Skip to content

Commit

Permalink
constellation-lib: move kubecmd package usage (#2673)
Browse files Browse the repository at this point in the history 
* Reduce external dependencies of kubecmd package
* Add kubecmd wrapper to constellation-lib
* Update CLI code to use constellation-lib
* Move kubecmd package to subpackage of constellation-lib
* Initialise helm and kubecmd clients when kubeConfig is set

---------

Signed-off-by: Daniel Weiße <[email protected]>
  • Loading branch information
@daniel-weisse
daniel-weisse authored Dec 5, 2023
1 parent c07c333 commit 3691def
Show file tree
Hide file tree
Showing 22 changed files with 745 additions and 708 deletions.
21 changes: 10 additions & 11 deletions CODEOWNERS
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,23 @@
.golangci.yml @katexochen
/3rdparty/gcp-guest-agent @malt3
/bazel @malt3
/bazel/ci @katexochen
/bazel/container @katexochen
/bazel @malt3
/bazel/sh @katexochen
/bootstrapper @3u13r
/internal/cloudcmd @daniel-weisse
/internal/cmd/upgrade* @derpsteb
/internal/featureset @malt3
/internal/helm @derpsteb
/internal/kubecmd @daniel-weisse
/internal/libvirt @daniel-weisse
/internal/terraform @elchead
/internal/upgrade @elchead
/internal/state @elchead
/cli/internal/cloudcmd @daniel-weisse
/cli/internal/cmd/upgrade* @derpsteb
/cli/internal/libvirt @daniel-weisse
/cli/internal/terraform @elchead
/csi @daniel-weisse
/debugd @malt3
/disk-mapper @daniel-weisse
/docs @thomasten
/e2e @3u13r
/hack/azure-snp-report-verify @derpsteb
/hack/bazel-deps-mirror @malt3
/hack/cli-k8s-compatibility @derpsteb
/hack/clidocgen @thomasten
/hack/cli-k8s-compatibility @derpsteb
/hack/fetch-broken-e2e @katexochen
/hack/gocoverage @katexochen
/hack/oci-pin @malt3
Expand All @@ -38,11 +33,14 @@
/internal/cloud @3u13r
/internal/compatibility @derpsteb
/internal/config @derpsteb
/internal/constellation/kubecmd @daniel-weisse
/internal/containerimage @malt3
/internal/crypto @thomasten
/internal/cryptsetup @daniel-weisse
/internal/featureset @malt3
/internal/file @daniel-weisse
/internal/grpc @thomasten
/internal/helm @derpsteb
/internal/imagefetcher @malt3
/internal/installer @3u13r
/internal/kms @daniel-weisse
Expand All @@ -54,6 +52,7 @@
/internal/retry @katexochen
/internal/semver @derpsteb
/internal/sigstore @elchead
/internal/state @elchead
/internal/staticupload @malt3
/internal/versions @3u13r
/joinservice @daniel-weisse
Expand Down
5 changes: 3 additions & 2 deletions cli/internal/cmd/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,14 +73,15 @@ go_library(
"//internal/config/migration",
"//internal/constants",
"//internal/constellation",
"//internal/constellation/kubecmd",
"//internal/crypto",
"//internal/featureset",
"//internal/file",
"//internal/grpc/dialer",
"//internal/grpc/retry",
"//internal/helm",
"//internal/imagefetcher",
"//internal/kms/uri",
"//internal/kubecmd",
# keep
"//internal/license",
"//internal/logger",
Expand Down Expand Up @@ -164,6 +165,7 @@ go_test(
"//internal/config",
"//internal/constants",
"//internal/constellation",
"//internal/constellation/kubecmd",
"//internal/crypto",
"//internal/crypto/testvector",
"//internal/file",
Expand All @@ -172,7 +174,6 @@ go_test(
"//internal/grpc/testdialer",
"//internal/helm",
"//internal/kms/uri",
"//internal/kubecmd",
"//internal/logger",
"//internal/semver",
"//internal/state",
Expand Down
131 changes: 90 additions & 41 deletions cli/internal/cmd/apply.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,23 +22,27 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/initproto"
"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
"github.com/edgelesssys/constellation/v2/internal/api/versionsapi"
"github.com/edgelesssys/constellation/v2/internal/atls"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/compatibility"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/constellation"
"github.com/edgelesssys/constellation/v2/internal/constellation/kubecmd"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/grpc/dialer"
"github.com/edgelesssys/constellation/v2/internal/helm"
"github.com/edgelesssys/constellation/v2/internal/imagefetcher"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
"github.com/edgelesssys/constellation/v2/internal/kubecmd"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/state"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/spf13/afero"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
)

Expand Down Expand Up @@ -225,13 +229,7 @@ func runApply(cmd *cobra.Command, _ []string) error {
newDialer := func(validator atls.Validator) *dialer.Dialer {
return dialer.New(nil, validator, &net.Dialer{})
}
newKubeUpgrader := func(w io.Writer, kubeConfigPath string, log debugLog) (kubernetesUpgrader, error) {
kubeConfig, err := fileHandler.Read(kubeConfigPath)
if err != nil {
return nil, fmt.Errorf("reading kubeconfig: %w", err)
}
return kubecmd.New(w, kubeConfig, fileHandler, log)
}

newHelmClient := func(kubeConfigPath string, log debugLog) (helmApplier, error) {
kubeConfig, err := fileHandler.Read(kubeConfigPath)
if err != nil {
Expand Down Expand Up @@ -264,9 +262,8 @@ func runApply(cmd *cobra.Command, _ []string) error {
spinner: spinner,
merger: &kubeconfigMerger{log: log},
newHelmClient: newHelmClient,
newDialer: newDialer,
newKubeUpgrader: newKubeUpgrader,
newInfraApplier: newInfraApplier,
imageFetcher: imagefetcher.New(),
applier: applier,
}

Expand All @@ -287,15 +284,15 @@ type applyCmd struct {

merger configMerger

applier applier
imageFetcher imageFetcher
applier applier

newHelmClient func(kubeConfigPath string, log debugLog) (helmApplier, error)
newDialer func(validator atls.Validator) *dialer.Dialer
newKubeUpgrader func(out io.Writer, kubeConfigPath string, log debugLog) (kubernetesUpgrader, error)
newInfraApplier func(context.Context) (cloudApplier, func(), error)
}

type applier interface {
SetKubeConfig(kubeConfig []byte) error
CheckLicense(ctx context.Context, csp cloudprovider.Provider, licenseID string) (int, error)
GenerateMasterSecret() (uri.MasterSecret, error)
GenerateMeasurementSalt() ([]byte, error)
Expand All @@ -309,6 +306,13 @@ type applier interface {
*initproto.InitSuccessResponse,
error,
)
ExtendClusterConfigCertSANs(ctx context.Context, clusterEndpoint, customEndpoint string, additionalAPIServerCertSANs []string) error
GetClusterAttestationConfig(ctx context.Context, variant variant.Variant) (config.AttestationCfg, error)
ApplyJoinConfig(ctx context.Context, newAttestConfig config.AttestationCfg, measurementSalt []byte) error
UpgradeNodeImage(ctx context.Context, imageVersion semver.Semver, imageReference string, force bool) error
UpgradeKubernetesVersion(ctx context.Context, kubernetesVersion versions.ValidK8sVersion, force bool) error
BackupCRDs(ctx context.Context, fileHandler file.Handler, upgradeDir string) ([]apiextensionsv1.CustomResourceDefinition, error)
BackupCRs(ctx context.Context, fileHandler file.Handler, crds []apiextensionsv1.CustomResourceDefinition, upgradeDir string) error
}

type warnLog interface {
Expand Down Expand Up @@ -415,42 +419,57 @@ func (a *applyCmd) apply(
}
}

if a.flags.skipPhases.contains(skipAttestationConfigPhase, skipCertSANsPhase, skipHelmPhase, skipK8sPhase, skipImagePhase) {
cmd.Print(bufferedOutput.String())
return nil
}

// From now on we can assume a valid Kubernetes admin config file exists
var kubeUpgrader kubernetesUpgrader
if !a.flags.skipPhases.contains(skipAttestationConfigPhase, skipCertSANsPhase, skipHelmPhase, skipK8sPhase, skipImagePhase) {
a.log.Debugf("Creating Kubernetes client using %s", a.flags.pathPrefixer.PrefixPrintablePath(constants.AdminConfFilename))
kubeUpgrader, err = a.newKubeUpgrader(cmd.OutOrStdout(), constants.AdminConfFilename, a.log)
if err != nil {
return err
}
kubeConfig, err := a.fileHandler.Read(constants.AdminConfFilename)
if err != nil {
return fmt.Errorf("reading kubeconfig: %w", err)
}
if err := a.applier.SetKubeConfig(kubeConfig); err != nil {
return err
}

// Apply Attestation Config
if !a.flags.skipPhases.contains(skipAttestationConfigPhase) {
a.log.Debugf("Applying new attestation config to cluster")
if err := a.applyJoinConfig(cmd, kubeUpgrader, conf.GetAttestationConfig(), stateFile.ClusterValues.MeasurementSalt); err != nil {
if err := a.applyJoinConfig(cmd, conf.GetAttestationConfig(), stateFile.ClusterValues.MeasurementSalt); err != nil {
return fmt.Errorf("applying attestation config: %w", err)
}
}

// Extend API Server Cert SANs
if !a.flags.skipPhases.contains(skipCertSANsPhase) {
sans := append([]string{stateFile.Infrastructure.ClusterEndpoint, conf.CustomEndpoint}, stateFile.Infrastructure.APIServerCertSANs...)
if err := kubeUpgrader.ExtendClusterConfigCertSANs(cmd.Context(), sans); err != nil {
if err := a.applier.ExtendClusterConfigCertSANs(
cmd.Context(),
stateFile.Infrastructure.ClusterEndpoint,
conf.CustomEndpoint,
stateFile.Infrastructure.APIServerCertSANs,
); err != nil {
return fmt.Errorf("extending cert SANs: %w", err)
}
}

// Apply Helm Charts
if !a.flags.skipPhases.contains(skipHelmPhase) {
if err := a.runHelmApply(cmd, conf, stateFile, kubeUpgrader, upgradeDir); err != nil {
if err := a.runHelmApply(cmd, conf, stateFile, upgradeDir); err != nil {
return err
}
}

// Upgrade NodeVersion object
if !(a.flags.skipPhases.contains(skipK8sPhase, skipImagePhase)) {
if err := a.runK8sUpgrade(cmd, conf, kubeUpgrader); err != nil {
// Upgrade node image
if !a.flags.skipPhases.contains(skipImagePhase) {
if err := a.runNodeImageUpgrade(cmd, conf); err != nil {
return err
}
}

// Upgrade Kubernetes version
if !a.flags.skipPhases.contains(skipK8sPhase) {
if err := a.runK8sVersionUpgrade(cmd, conf); err != nil {
return err
}
}
Expand Down Expand Up @@ -598,15 +617,14 @@ func (a *applyCmd) validateInputs(cmd *cobra.Command, configFetcher attestationc

// applyJoinConfig creates or updates the cluster's join config.
// If the config already exists, and is different from the new config, the user is asked to confirm the upgrade.
func (a *applyCmd) applyJoinConfig(
cmd *cobra.Command, kubeUpgrader kubernetesUpgrader, newConfig config.AttestationCfg, measurementSalt []byte,
func (a *applyCmd) applyJoinConfig(cmd *cobra.Command, newConfig config.AttestationCfg, measurementSalt []byte,
) error {
clusterAttestationConfig, err := kubeUpgrader.GetClusterAttestationConfig(cmd.Context(), newConfig.GetVariant())
clusterAttestationConfig, err := a.applier.GetClusterAttestationConfig(cmd.Context(), newConfig.GetVariant())
if err != nil {
a.log.Debugf("Getting cluster attestation config failed: %s", err)
if k8serrors.IsNotFound(err) {
a.log.Debugf("Creating new join config")
return kubeUpgrader.ApplyJoinConfig(cmd.Context(), newConfig, measurementSalt)
return a.applier.ApplyJoinConfig(cmd.Context(), newConfig, measurementSalt)
}
return fmt.Errorf("getting cluster attestation config: %w", err)
}
Expand Down Expand Up @@ -638,27 +656,37 @@ func (a *applyCmd) applyJoinConfig(
}
}

if err := kubeUpgrader.ApplyJoinConfig(cmd.Context(), newConfig, measurementSalt); err != nil {
if err := a.applier.ApplyJoinConfig(cmd.Context(), newConfig, measurementSalt); err != nil {
return fmt.Errorf("updating attestation config: %w", err)
}
cmd.Println("Successfully updated the cluster's attestation config")

return nil
}

// runK8sUpgrade upgrades image and Kubernetes version of the Constellation cluster.
func (a *applyCmd) runK8sUpgrade(cmd *cobra.Command, conf *config.Config, kubeUpgrader kubernetesUpgrader,
) error {
err := kubeUpgrader.UpgradeNodeVersion(
cmd.Context(), conf, a.flags.force,
a.flags.skipPhases.contains(skipImagePhase),
a.flags.skipPhases.contains(skipK8sPhase),
)
func (a *applyCmd) runNodeImageUpgrade(cmd *cobra.Command, conf *config.Config) error {
provider := conf.GetProvider()
attestationVariant := conf.GetAttestationConfig().GetVariant()
region := conf.GetRegion()
imageReference, err := a.imageFetcher.FetchReference(cmd.Context(), provider, attestationVariant, conf.Image, region)
if err != nil {
return fmt.Errorf("fetching image reference: %w", err)
}

imageVersionInfo, err := versionsapi.NewVersionFromShortPath(conf.Image, versionsapi.VersionKindImage)
if err != nil {
return fmt.Errorf("parsing version from image short path: %w", err)
}
imageVersion, err := semver.New(imageVersionInfo.Version())
if err != nil {
return fmt.Errorf("parsing image version: %w", err)
}

err = a.applier.UpgradeNodeImage(cmd.Context(), imageVersion, imageReference, a.flags.force)
var upgradeErr *compatibility.InvalidUpgradeError
switch {
case errors.Is(err, kubecmd.ErrInProgress):
cmd.PrintErrln("Skipping image and Kubernetes upgrades. Another upgrade is in progress.")
cmd.PrintErrln("Skipping image upgrade: Another upgrade is already in progress.")
case errors.As(err, &upgradeErr):
cmd.PrintErrln(err)
case err != nil:
Expand All @@ -668,6 +696,19 @@ func (a *applyCmd) runK8sUpgrade(cmd *cobra.Command, conf *config.Config, kubeUp
return nil
}

func (a *applyCmd) runK8sVersionUpgrade(cmd *cobra.Command, conf *config.Config) error {
err := a.applier.UpgradeKubernetesVersion(cmd.Context(), conf.KubernetesVersion, a.flags.force)
var upgradeErr *compatibility.InvalidUpgradeError
switch {
case errors.As(err, &upgradeErr):
cmd.PrintErrln(err)
case err != nil:
return fmt.Errorf("upgrading Kubernetes version: %w", err)
}

return nil
}

// checkCreateFilesClean ensures that the workspace is clean before creating a new cluster.
func (a *applyCmd) checkCreateFilesClean() error {
if err := a.checkInitFilesClean(); err != nil {
Expand Down Expand Up @@ -803,3 +844,11 @@ func (wl warnLogger) Infof(fmtStr string, args ...any) {
func (wl warnLogger) Warnf(fmtStr string, args ...any) {
wl.cmd.PrintErrf("Warning: %s\n", fmt.Sprintf(fmtStr, args...))
}

// imageFetcher gets an image reference from the versionsapi.
type imageFetcher interface {
FetchReference(ctx context.Context,
provider cloudprovider.Provider, attestationVariant variant.Variant,
image, region string,
) (string, error)
}
15 changes: 11 additions & 4 deletions cli/internal/cmd/apply_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,10 +193,11 @@ func TestBackupHelmCharts(t *testing.T) {

a := applyCmd{
fileHandler: file.NewHandler(afero.NewMemMapFs()),
applier: &stubConstellApplier{stubKubernetesUpgrader: tc.backupClient},
log: logger.NewTest(t),
}

err := a.backupHelmCharts(context.Background(), tc.backupClient, tc.helmApplier, tc.includesUpgrades, "")
err := a.backupHelmCharts(context.Background(), tc.helmApplier, tc.includesUpgrades, "")
if tc.wantErr {
assert.Error(err)
return
Expand Down Expand Up @@ -494,23 +495,29 @@ func newPhases(phases ...skipPhase) skipPhases {

type stubConstellApplier struct {
checkLicenseErr error
masterSecret uri.MasterSecret
measurementSalt []byte
generateMasterSecretErr error
generateMeasurementSaltErr error
initErr error
initResponse *initproto.InitSuccessResponse
*stubKubernetesUpgrader
}

func (s *stubConstellApplier) SetKubeConfig([]byte) error { return nil }

func (s *stubConstellApplier) CheckLicense(context.Context, cloudprovider.Provider, string) (int, error) {
return 0, s.checkLicenseErr
}

func (s *stubConstellApplier) GenerateMasterSecret() (uri.MasterSecret, error) {
return uri.MasterSecret{}, s.generateMasterSecretErr
return s.masterSecret, s.generateMasterSecretErr
}

func (s *stubConstellApplier) GenerateMeasurementSalt() ([]byte, error) {
return nil, s.generateMeasurementSaltErr
return s.measurementSalt, s.generateMeasurementSaltErr
}

func (s *stubConstellApplier) Init(context.Context, atls.Validator, *state.State, io.Writer, constellation.InitPayload) (*initproto.InitSuccessResponse, error) {
return nil, s.initErr
return s.initResponse, s.initErr
}
Loading

0 comments on commit 3691def

Please sign in to comment.