From 6ee9d9132d5ae0b90ca7fd7d17cc93b061c44a6d Mon Sep 17 00:00:00 2001 From: Derek Lee Date: Thu, 20 Jul 2023 11:50:19 -0700 Subject: [PATCH 1/2] libvirt: sev-snp support Adds SEV-SNP support for libvirt. Signed-off-by: Derek Lee --- go.mod | 2 +- go.sum | 18 ++ install/overlays/libvirt/kustomization.yaml | 40 ++--- pkg/adaptor/cloud/libvirt/libvirt.go | 182 +++++++++++++++++--- pkg/adaptor/cloud/libvirt/libvirt_test.go | 4 +- pkg/adaptor/cloud/libvirt/manager.go | 22 ++- pkg/adaptor/cloud/libvirt/provider.go | 10 +- pkg/adaptor/cloud/libvirt/types.go | 51 ++++-- 8 files changed, 255 insertions(+), 74 deletions(-) diff --git a/go.mod b/go.mod index 3087af96c..2e32da565 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 k8s.io/cri-api v0.23.1 libvirt.org/go/libvirt v1.8002.0 - libvirt.org/go/libvirtxml v1.8002.0 + libvirt.org/go/libvirtxml v1.9004.0 ) require ( diff --git a/go.sum b/go.sum index 3fae5d516..401c47e57 100644 --- a/go.sum +++ b/go.sum @@ -65,6 +65,7 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5/go.mod h1:v4VVB6oBMz/c9fRY6vZrwr5xKRWOH5NPDjQZlPk0Gbs= contrib.go.opencensus.io/exporter/stackdriver v0.13.4/go.mod h1:aXENhDJ1Y4lIg4EUaVTwzvYETVNZk10Pu26tevFKLUc= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0= @@ -357,6 +358,7 @@ github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4S github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= +github.com/containerd/cgroups v1.0.5-0.20220625035431-cf7417bca682/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= @@ -392,6 +394,7 @@ github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= +github.com/containerd/cri-containerd v1.19.0/go.mod h1:wxbGdReWGCalzGOEpifoHeYCK4xAgnj4o/4bVB+9voU= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= @@ -600,6 +603,7 @@ github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU= +github.com/frankban/quicktest v1.13.1/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= @@ -628,6 +632,7 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gorp/gorp/v3 v3.0.2/go.mod h1:BJ3q1ejpV8cVALtcXvXaXyTOlMmJhWDxTmncaR6rwBY= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-ini/ini v1.28.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= @@ -858,6 +863,7 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= @@ -1109,6 +1115,7 @@ github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/insomniacslk/dhcp v0.0.0-20220119180841-3c283ff8b7dd/go.mod h1:h+MxyHxRg9NH3terB1nfRIUaQEcI0XOVkdR9LNBlp8E= +github.com/intel-go/cpuid v0.0.0-20210602155658-5747e5cec0d9/go.mod h1:RmeVYf9XrPRbRc3XIx0gLYA8qOFvNoPOfaEZduRlEp4= github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= @@ -1222,6 +1229,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= @@ -1322,6 +1330,8 @@ github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcK github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZmb2XgV7o= github.com/mdlayher/raw v0.0.0-20190606142536-fef19f00fc18/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg= github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg= +github.com/mdlayher/socket v0.2.0/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E= +github.com/mdlayher/vsock v1.1.0/go.mod h1:nsVhPsVuBBwAKh6i6PzdNoke6/TNYTjkxoRKAp/+pXs= github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg= github.com/mgechev/revive v1.1.2/go.mod h1:bnXsMr+ZTH09V5rssEI+jHAZ4z+ZdyhgO/zsy3EhK+0= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= @@ -1516,6 +1526,7 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= @@ -1588,6 +1599,7 @@ github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1-0.20210923151022-86f73c517451 h1:d1PiN4RxzIFXCJTvRkvSkKqwtRAl5ZV4lATKtQI0B7I= +github.com/rogpeppe/go-internal v1.8.1-0.20210923151022-86f73c517451/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rootless-containers/rootlesskit v1.0.1/go.mod h1:t2UAiYagxrJ+wmpFAUIZPcqsm4k2B7ve6g7lILKbloc= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= @@ -1829,6 +1841,7 @@ github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhu github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= github.com/zmap/zcrypto v0.0.0-20210811211718-6f9bc4aff20f/go.mod h1:y/9hjFEub4DtQxTHp/pqticBgdYeCwL97vojV3lsvHY= github.com/zmap/zlint/v3 v3.3.1-0.20211019173530-cb17369b4628/go.mod h1:O+4OXRfNLKqOyDl4eKZ1SBlYudKGUBGRFcv+m1KLr28= +gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965/go.mod h1:TBB3sR7/jg4RCThC/cgT4fB8mAbbMO307TycfgeR59w= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -1873,6 +1886,7 @@ go.opentelemetry.io/contrib/propagators v0.19.0/go.mod h1:4QOdZClXISU5S43xZxk5tY go.opentelemetry.io/otel v0.19.0/go.mod h1:j9bF567N9EfomkSidSfmMwIwIBuP37AMAIzVW85OxSg= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= +go.opentelemetry.io/otel/exporters/jaeger v1.0.0/go.mod h1:q10N1AolE1JjqKrFJK2tYw0iZpmX+HBaXBtuCzRnBGQ= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8= @@ -2565,6 +2579,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= @@ -2647,6 +2662,7 @@ k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= @@ -2671,6 +2687,8 @@ libvirt.org/go/libvirt v1.8002.0 h1:X8gz2Sa1ek4S5FznpDpeRz6JpNb7NdkfzTii5GMIwDY= libvirt.org/go/libvirt v1.8002.0/go.mod h1:1WiFE8EjZfq+FCVog+rvr1yatKbKZ9FaFMZgEqxEJqQ= libvirt.org/go/libvirtxml v1.8002.0 h1:ES5bU3/G/dykJ4WIO5NJ3cTvvr5xSCaqwjYeRxkTX40= libvirt.org/go/libvirtxml v1.8002.0/go.mod h1:7Oq2BLDstLr/XtoQD8Fr3mfDNrzlI3utYKySXF2xkng= +libvirt.org/go/libvirtxml v1.9004.0 h1:h+nhEZCABCnK4go0GLRN2WZhIhRrLAqsz84t553oiM4= +libvirt.org/go/libvirtxml v1.9004.0/go.mod h1:7Oq2BLDstLr/XtoQD8Fr3mfDNrzlI3utYKySXF2xkng= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= diff --git a/install/overlays/libvirt/kustomization.yaml b/install/overlays/libvirt/kustomization.yaml index 560697c1e..6bc230381 100644 --- a/install/overlays/libvirt/kustomization.yaml +++ b/install/overlays/libvirt/kustomization.yaml @@ -13,14 +13,18 @@ generatorOptions: disableNameSuffixHash: true configMapGenerator: -- name: peer-pods-cm - namespace: confidential-containers-system - literals: +- literals: - CLOUD_PROVIDER="libvirt" - - LIBVIRT_URI="qemu+ssh://root@192.168.122.1/system?no_verify=1" #set - - LIBVIRT_NET="default" # set - - LIBVIRT_POOL="default" # set + - LIBVIRT_URI="qemu:///system" + - LIBVIRT_NET="default" + - LIBVIRT_POOL="default" + - LIBVIRT_VOL_NAME="podvm-base.qcow2" + name: peer-pods-cm + namespace: confidential-containers-system #- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2 + #- LIBVIRT_LAUNCH_SECURITY="" # sev or s390pv + #- LIBVIRT_FIRMWARE="" # OVMF path (for SEV) + #- LIBVIRT_FIRMWARE_VARS="" #OVMF_VARS path (for SEV) #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 ##TLS_SETTINGS @@ -30,29 +34,13 @@ configMapGenerator: #- TLS_SKIP_VERIFY="" # for testing only ##TLS_SETTINGS + #- auth.json # set - path to auth.json pull credentials file secretGenerator: - name: auth-json-secret namespace: confidential-containers-system - files: - #- auth.json # set - path to auth.json pull credentials file -- name: ssh-key-secret +- files: + - id_rsa + name: ssh-key-secret namespace: confidential-containers-system - files: # key generation example: ssh-keygen -f ./id_rsa -N "" && sudo cat id_rsa.pub >> /root/.ssh/authorized_keys - #- id_rsa # set - path to private key - name: peer-pods-secret namespace: confidential-containers-system - literals: -##TLS_SETTINGS -#- name: certs-for-tls -# namespace: confidential-containers-system -# files: -# - # set - relative path to ca.crt, located either in the same folder as the kustomization.yaml file or within a subfolder -# - # set - relative path to client.crt, located either in the same folder as the kustomization.yaml file or within a subfolder -# - # set - relative path to client.key, located either in the same folder as the kustomization.yaml file or within a subfolder -##TLS_SETTINGS - -patchesStrategicMerge: - #- cri_runtime_endpoint.yaml # set (modify host's runtime cri socket path in the file, default is /run/containerd/containerd.sock) -##TLS_SETTINGS - #- tls_certs_volume_mount.yaml # set (for tls) -##TLS_SETTINGS diff --git a/pkg/adaptor/cloud/libvirt/libvirt.go b/pkg/adaptor/cloud/libvirt/libvirt.go index b63d41e3e..1302d1c8a 100644 --- a/pkg/adaptor/cloud/libvirt/libvirt.go +++ b/pkg/adaptor/cloud/libvirt/libvirt.go @@ -36,6 +36,30 @@ type domainConfig struct { cidataDisk string } +// https://www.amd.com/system/files/TechDocs/55766_SEV-KM_API_Specification.pdf +type sevGuestPolicy struct { + noDebug bool + noKeyShare bool + es bool + noSend bool + domain bool + sev bool +} + +// Struct bitmap to unsigned integer (needed for enabling sev) +func (s *sevGuestPolicy) getGuestPolicy() uint { + bitmap := []bool{s.noDebug, s.noKeyShare, s.es, s.noSend, s.domain, s.sev} + res := uint(0) + + for i := 0; i < len(bitmap); i++ { + if bitmap[i] { + res |= 1 << i + } + } + + return res +} + func createCloudInitISO(v *vmConfig, libvirtClient *libvirtClient) string { logger.Printf("Create cloudInit iso\n") cloudInitIso := libvirtClient.dataDir + "/" + v.name + "-cloudinit.iso" @@ -157,6 +181,20 @@ func getHostCapabilities(conn *libvirt.Connect) (*libvirtxml.Caps, error) { return caps, nil } +func getDomainCapabilities(conn *libvirt.Connect, emulatorbin string, arch string, machine string, virttype string, flags uint32) (*libvirtxml.DomainCaps, error) { + capsXML, err := conn.GetDomainCapabilities(emulatorbin, arch, machine, virttype, flags) + if err != nil { + return nil, fmt.Errorf("unable to get domain capabilities, cause: %w", err) + } + caps := &libvirtxml.DomainCaps{} + err = xml.Unmarshal([]byte(capsXML), caps) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal domain capabilities, cause: %w", err) + + } + return caps, nil +} + // lookupMachine finds the machine name from the set of available machines func lookupMachine(machines []libvirtxml.CapsGuestMachine, targetmachine string) string { for _, machine := range machines { @@ -193,7 +231,7 @@ func getCanonicalMachineName(caps *libvirtxml.Caps, arch string, virttype string return "", fmt.Errorf("cannot find machine type %s for %s/%s in %v", targetmachine, virttype, arch, caps) } -func createDomainXMLs390x(client *libvirtClient, cfg *domainConfig) (*libvirtxml.Domain, error) { +func createDomainXMLs390x(client *libvirtClient, cfg *domainConfig, vm *vmConfig) (*libvirtxml.Domain, error) { guest, err := getGuestForArchType(client.caps, archS390x, typeHardwareVirtualMachine) if err != nil { @@ -244,7 +282,7 @@ func createDomainXMLs390x(client *libvirtClient, cfg *domainConfig) (*libvirtxml }, } - return &libvirtxml.Domain{ + domain := &libvirtxml.Domain{ Type: "kvm", Name: cfg.name, Description: "This Virtual Machine is the peer-pod VM", @@ -311,19 +349,21 @@ func createDomainXMLs390x(client *libvirtClient, cfg *domainConfig) (*libvirtxml }, }, }, - }, nil + } + + return domain, nil } -func createDomainXMLx86_64(client *libvirtClient, cfg *domainConfig) (*libvirtxml.Domain, error) { +func createDomainXMLx86_64(client *libvirtClient, cfg *domainConfig, vm *vmConfig) (*libvirtxml.Domain, error) { var diskControllerAddr uint = 0 - return &libvirtxml.Domain{ + domain := &libvirtxml.Domain{ Type: "kvm", Name: cfg.name, Description: "This Virtual Machine is the peer-pod VM", - Memory: &libvirtxml.DomainMemory{Value: uint(cfg.mem), Unit: "GiB", DumpCore: "on"}, - VCPU: &libvirtxml.DomainVCPU{Value: uint(cfg.cpu)}, + Memory: &libvirtxml.DomainMemory{Value: cfg.mem, Unit: "GiB", DumpCore: "on"}, + VCPU: &libvirtxml.DomainVCPU{Value: cfg.cpu}, OS: &libvirtxml.DomainOS{ Type: &libvirtxml.DomainOSType{Arch: "x86_64", Type: typeHardwareVirtualMachine}, }, @@ -378,16 +418,110 @@ func createDomainXMLx86_64(client *libvirtClient, cfg *domainConfig) (*libvirtxm }, }, }, - }, nil + } + + switch l := vm.launchSecurityType; l { + case NoLaunchSecurity: + return domain, nil + case SEV: + return enableSEVSNP(client, cfg, vm, domain) + default: + return nil, fmt.Errorf("launch Security type is not supported for this domain: %s", l) + } + +} + +func enableSEVSNP(client *libvirtClient, cfg *domainConfig, vm *vmConfig, domain *libvirtxml.Domain) (*libvirtxml.Domain, error) { + + if vm.launchSecurityType != SEV { + return nil, fmt.Errorf("launch Seurity must be set as SEV to enable SEV-SNP") + } + + const sevMachine = "q35" + var diskControllerAddr uint = 0 + + var domCapflags uint32 = 0 + arch := "x86_64" + virttype := "qemu" + + // Determine whether machine supports SEV + guest, err := getGuestForArchType(client.caps, arch, "hvm") + if err != nil { + return nil, fmt.Errorf("unable to find guest machine to determine SEV capabilities") + } + emulator := guest.Arch.Emulator + domCaps, err := getDomainCapabilities(client.connection, emulator, arch, sevMachine, virttype, domCapflags) + if err != nil { + return nil, fmt.Errorf("unable to determine guest domain capabilities: %+v", err) + } + if domCaps.Features.SEV.Supported != "yes" { + return nil, fmt.Errorf("SEV is not supported for this domain") + } + + // Enable Launch Security + guestPolicyStruct := sevGuestPolicy{ + noDebug: false, + noKeyShare: true, + es: true, + noSend: false, + domain: false, + sev: false, + } + + guestPolicy := guestPolicyStruct.getGuestPolicy() + + domain.LaunchSecurity = &libvirtxml.DomainLaunchSecurity{ + SEV: &libvirtxml.DomainLaunchSecuritySEV{ + CBitPos: &domCaps.Features.SEV.CBitPos, + ReducedPhysBits: &domCaps.Features.SEV.ReducedPhysBits, + Policy: &guestPolicy, + }, + } + + domain.OS.Type.Machine = sevMachine + domain.OS.Loader = &libvirtxml.DomainLoader{ + Path: vm.firmware, + Readonly: "yes", + Secure: "yes", + Stateless: "yes", + Type: "pflash", + } + // Secure boot requires SMM feature enabled + domain.Features.SMM = &libvirtxml.DomainFeatureSMM{State: "on"} + + // Must allocate memory (8 GiB) + extra for qemu to use to calculate total memory limit + domain.MemoryTune = &libvirtxml.DomainMemoryTune{ + HardLimit: &libvirtxml.DomainMemoryTuneLimit{ + Value: 8912896, + Unit: "KiB", + }, + } + + // IDE controllers are unsupported for q35 machines, so must override cidatadisk + domain.Devices.Disks[1] = libvirtxml.DomainDisk{ + Device: "disk", + Driver: &libvirtxml.DomainDiskDriver{Name: "qemu", Type: "raw"}, + Source: &libvirtxml.DomainDiskSource{ + File: &libvirtxml.DomainDiskSourceFile{File: cfg.cidataDisk}, + }, + // Bus cannot be sata. Related? + // https://github.com/vagrant-libvirt/vagrant-libvirt/issues/1444 + Target: &libvirtxml.DomainDiskTarget{Dev: "sdb", Bus: "scsi"}, + Address: &libvirtxml.DomainAddress{ + Drive: &libvirtxml.DomainAddressDrive{ + Controller: &diskControllerAddr, Bus: &diskControllerAddr, Target: &diskControllerAddr, Unit: &diskControllerAddr}}, + } + + return domain, nil } // createDomainXML detects the machine type of the libvirt host and will return a libvirt XML for that machine type -func createDomainXML(client *libvirtClient, cfg *domainConfig) (*libvirtxml.Domain, error) { +func createDomainXML(client *libvirtClient, cfg *domainConfig, vm *vmConfig) (*libvirtxml.Domain, error) { switch client.nodeInfo.Model { case archS390x: - return createDomainXMLs390x(client, cfg) + return createDomainXMLs390x(client, cfg, vm) default: - return createDomainXMLx86_64(client, cfg) + return createDomainXMLx86_64(client, cfg, vm) } } @@ -399,7 +533,7 @@ func CreateDomain(ctx context.Context, libvirtClient *libvirtClient, v *vmConfig exists, err := checkDomainExistsByName(v.name, libvirtClient) if err != nil { - return nil, fmt.Errorf("Error in checking instance: %s", err) + return nil, fmt.Errorf("error in checking instance: %s", err) } if exists { logger.Printf("Instance already exists ") @@ -411,7 +545,7 @@ func CreateDomain(ctx context.Context, libvirtClient *libvirtClient, v *vmConfig rootVolName := v.name + "-root.qcow2" err = createVolume(rootVolName, v.rootDiskSize, libvirtClient.volName, libvirtClient) if err != nil { - return nil, fmt.Errorf("Error in creating volume: %s", err) + return nil, fmt.Errorf("error in creating volume: %s", err) } cloudInitIso := createCloudInitISO(v, libvirtClient) @@ -419,17 +553,17 @@ func CreateDomain(ctx context.Context, libvirtClient *libvirtClient, v *vmConfig isoVolName := v.name + "-cloudinit.iso" isoVolFile, err := uploadIso(cloudInitIso, isoVolName, libvirtClient) if err != nil { - return nil, fmt.Errorf("Error in uploading iso volume: %s", err) + return nil, fmt.Errorf("error in uploading iso volume: %s", err) } rootVol, err := getVolume(libvirtClient, rootVolName) if err != nil { - return nil, fmt.Errorf("Error retrieving volume: %s", err) + return nil, fmt.Errorf("error retrieving volume: %s", err) } rootVolFile, err := rootVol.GetPath() if err != nil { - return nil, fmt.Errorf("Error retrieving volume path: %s", err) + return nil, fmt.Errorf("error retrieving volume path: %s", err) } domainCfg := domainConfig{ @@ -441,33 +575,33 @@ func CreateDomain(ctx context.Context, libvirtClient *libvirtClient, v *vmConfig cidataDisk: isoVolFile, } - domCfg, err := createDomainXML(libvirtClient, &domainCfg) + domCfg, err := createDomainXML(libvirtClient, &domainCfg, v) if err != nil { - return nil, fmt.Errorf("error building the libvirt XML, cause: %w", err) + return nil, fmt.Errorf("error building the libvirt XML, cause: %+v", err) } logger.Printf("Create XML for '%s'", v.name) domXML, err := domCfg.Marshal() if err != nil { - return nil, fmt.Errorf("Failed to create domain xml: %s", err) + return nil, fmt.Errorf("failed to create domain xml: %+v", err) } logger.Printf("Creating VM '%s'", v.name) dom, err := libvirtClient.connection.DomainDefineXML(domXML) if err != nil { - return nil, fmt.Errorf("Failed to define domain: %s", err) + return nil, fmt.Errorf("failed to define domain: %+v", err) } // Start Domain. logger.Printf("Starting VM '%s'", v.name) err = dom.Create() if err != nil { - return nil, fmt.Errorf("Failed to start VM: %s", err) + return nil, fmt.Errorf("failed to start VM: %+v", err) } id, err := dom.GetID() if err != nil { - return nil, fmt.Errorf("Failed to get domain ID: %s", err) + return nil, fmt.Errorf("failed to get domain ID: %+v", err) } v.instanceId = strconv.FormatUint(uint64(id), 10) @@ -479,7 +613,7 @@ func CreateDomain(ctx context.Context, libvirtClient *libvirtClient, v *vmConfig domInterface, err := dom.ListAllInterfaceAddresses(libvirt.DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE) if err != nil { - return nil, fmt.Errorf("Failed to get domain interfaces: %s", err) + return nil, fmt.Errorf("failed to get domain interfaces: %+v", err) } logger.Printf("domain IP details %v", domInterface) @@ -489,7 +623,7 @@ func CreateDomain(ctx context.Context, libvirtClient *libvirtClient, v *vmConfig logger.Printf("VM IP %s", domInterface[0].Addrs[0].Addr) addr, err := netip.ParseAddr(domInterface[0].Addrs[0].Addr) if err != nil { - return nil, fmt.Errorf("Failed to parse address: %s", err) + return nil, fmt.Errorf("failed to parse address: %+v", err) } v.ips = append(v.ips, addr) logger.Printf("VM IP list %v", v.ips) diff --git a/pkg/adaptor/cloud/libvirt/libvirt_test.go b/pkg/adaptor/cloud/libvirt/libvirt_test.go index bf9321b01..f968895ee 100644 --- a/pkg/adaptor/cloud/libvirt/libvirt_test.go +++ b/pkg/adaptor/cloud/libvirt/libvirt_test.go @@ -98,7 +98,9 @@ func TestCreateDomainXMLs390x(t *testing.T) { cidataDisk: "/var/lib/libvirt/images/cidata.iso", } - domCfg, err := createDomainXML(client, &domainCfg) + vm := vmConfig{} + + domCfg, err := createDomainXML(client, &domainCfg, &vm) if err != nil { t.Error(err) } diff --git a/pkg/adaptor/cloud/libvirt/manager.go b/pkg/adaptor/cloud/libvirt/manager.go index 68c7238fd..cef6bfc2f 100644 --- a/pkg/adaptor/cloud/libvirt/manager.go +++ b/pkg/adaptor/cloud/libvirt/manager.go @@ -16,29 +16,35 @@ var libvirtcfg Config type Manager struct{} const ( - defaultURI = "qemu:///system" - defaultPoolName = "default" - defaultNetworkName = "default" - defaultDataDir = "/var/lib/libvirt/images" - defaultVolName = "podvm-base.qcow2" + defaultURI = "qemu:///system" + defaultPoolName = "default" + defaultNetworkName = "default" + defaultDataDir = "/var/lib/libvirt/images" + defaultVolName = "podvm-base.qcow2" + defaultLaunchSecurity = "" + defaultFirmware = "/usr/share/edk2/ovmf/OVMF_CODE.secboot.fd" ) -func (_ *Manager) ParseCmd(flags *flag.FlagSet) { +func (*Manager) ParseCmd(flags *flag.FlagSet) { flags.StringVar(&libvirtcfg.URI, "uri", defaultURI, "libvirt URI") flags.StringVar(&libvirtcfg.PoolName, "pool-name", defaultPoolName, "libvirt storage pool") flags.StringVar(&libvirtcfg.NetworkName, "network-name", defaultNetworkName, "libvirt network pool") flags.StringVar(&libvirtcfg.DataDir, "data-dir", defaultDataDir, "libvirt storage dir") + flags.StringVar(&libvirtcfg.LaunchSecurity, "launch-security", defaultLaunchSecurity, "Libvirt's LaunchSecurity element. SEV or s390-pv") + flags.StringVar(&libvirtcfg.Firmware, "firmware", defaultFirmware, "Path to OVMF") } -func (_ *Manager) LoadEnv() { +func (*Manager) LoadEnv() { cloud.DefaultToEnv(&libvirtcfg.URI, "LIBVIRT_URI", defaultURI) cloud.DefaultToEnv(&libvirtcfg.PoolName, "LIBVIRT_POOL", defaultPoolName) cloud.DefaultToEnv(&libvirtcfg.NetworkName, "LIBVIRT_NET", defaultNetworkName) cloud.DefaultToEnv(&libvirtcfg.VolName, "LIBVIRT_VOL_NAME", defaultVolName) + cloud.DefaultToEnv(&libvirtcfg.LaunchSecurity, "LIBVIRT_LAUNCH_SECURITY", defaultLaunchSecurity) + cloud.DefaultToEnv(&libvirtcfg.Firmware, "LIBVIRT_FIRMWARE", defaultFirmware) } -func (_ *Manager) NewProvider() (cloud.Provider, error) { +func (*Manager) NewProvider() (cloud.Provider, error) { return NewProvider(&libvirtcfg) } diff --git a/pkg/adaptor/cloud/libvirt/provider.go b/pkg/adaptor/cloud/libvirt/provider.go index 704b9ab09..7429f4758 100644 --- a/pkg/adaptor/cloud/libvirt/provider.go +++ b/pkg/adaptor/cloud/libvirt/provider.go @@ -56,7 +56,15 @@ func (p *libvirtProvider) CreateInstance(ctx context.Context, podName, sandboxID } // TODO: Specify the maximum instance name length in Libvirt - vm := &vmConfig{name: instanceName, userData: userData} + vm := &vmConfig{name: instanceName, userData: userData, firmware: p.serviceConfig.Firmware, firmwareVars: p.serviceConfig.FirmwareVars} + + switch p.serviceConfig.LaunchSecurity { + case "sev": + vm.launchSecurityType = SEV + case "s390-pv": + vm.launchSecurityType = S390PV + } + result, err := CreateDomain(ctx, p.libvirtClient, vm) if err != nil { logger.Printf("failed to create an instance : %v", err) diff --git a/pkg/adaptor/cloud/libvirt/types.go b/pkg/adaptor/cloud/libvirt/types.go index 79f53c10d..91bee9c4a 100644 --- a/pkg/adaptor/cloud/libvirt/types.go +++ b/pkg/adaptor/cloud/libvirt/types.go @@ -13,22 +13,26 @@ import ( ) type Config struct { - URI string - PoolName string - NetworkName string - DataDir string - VolName string + URI string + PoolName string + NetworkName string + DataDir string + VolName string + LaunchSecurity string + Firmware string } type vmConfig struct { - name string - cpu uint - mem uint - rootDiskSize uint64 - userData string - metaData string - ips []netip.Addr - instanceId string //keeping it consistent with sandbox.vsi + name string + cpu uint + mem uint + rootDiskSize uint64 + userData string + metaData string + ips []netip.Addr + instanceId string //keeping it consistent with sandbox.vsi + launchSecurityType LaunchSecurityType + firmware string } type createDomainOutput struct { @@ -56,3 +60,24 @@ type libvirtClient struct { // host capabilities caps *libvirtxml.Caps } + +type LaunchSecurityType int + +const ( + NoLaunchSecurity LaunchSecurityType = iota + SEV + S390PV +) + +func (l LaunchSecurityType) String() string { + switch l { + case NoLaunchSecurity: + return "" + case SEV: + return "SEV" + case S390PV: + return "S390PV" + default: + return "unknown" + } +} From f73d4e2d5761bb9a6163fdf28f7190a3722a27a0 Mon Sep 17 00:00:00 2001 From: Derek Lee Date: Thu, 13 Jul 2023 19:56:11 -0400 Subject: [PATCH 2/2] test/libvirt: allow libvirt conn URI to be specified The libvirt tests currently assumes that it is always running on a local instance and hard codes "qemu:///system" as the URI to create the Connection object with. This allows us to specify libvirt_conn_uri in the libvirt properties file to change it to a different instance. Also updates test/provisioner/e2e/README.md to add this option to the documentation and clarify the difference with libvirt_uri. Fixes: #1164 Signed-off-by: Derek Lee --- test/e2e/README.md | 3 ++- test/provisioner/provision_libvirt.go | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 7f456e147..58e14997b 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -58,8 +58,9 @@ Use the properties on the table below for Libvirt: |---|---|---| |libvirt_network|Libvirt Network|"default"| |libvirt_storage|Libvirt storage pool|"default"| -|libvirt_url|Libvirt connection URI|"qemu+ssh://root@192.168.122.1/system?no_verify=1"| |libvirt_vol_name|Volume name|"podvm-base.qcow2"| +|libvirt_uri|Libvirt pod URI|"qemu+ssh://root@192.168.122.1/system?no_verify=1"| +|libvirt_conn_uri|Libvirt host URI|"qemu:///system"| |libvirt_ssh_key_file|Path to SSH private key|| |pause_image|k8s pause image|| |vxlan_port| VXLAN port number|| diff --git a/test/provisioner/provision_libvirt.go b/test/provisioner/provision_libvirt.go index 2985052c5..f01f7b8d9 100644 --- a/test/provisioner/provision_libvirt.go +++ b/test/provisioner/provision_libvirt.go @@ -73,8 +73,11 @@ func NewLibvirtProvisioner(properties map[string]string) (CloudProvisioner, erro vol_name = properties["libvirt_vol_name"] } - // TODO: accept a different URI. - conn, err := libvirt.NewConnect("qemu:///system") + conn_uri := "qemu:///system" + if properties["libvirt_conn_uri"] != "" { + conn_uri = properties["libvirt_conn_uri"] + } + conn, err := libvirt.NewConnect(conn_uri) if err != nil { return nil, err }