From 750443e379f552d6cf822666e732f1d6c8df4fca Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 29 Oct 2024 14:40:38 +0000 Subject: [PATCH 01/19] slog on deploy Signed-off-by: Austin Abro --- src/cmd/destroy.go | 2 +- src/internal/git/repository.go | 6 ++++ src/internal/packager/helm/chart.go | 26 +++++++++----- src/internal/packager/helm/destroy.go | 5 +-- src/internal/packager/helm/post-render.go | 9 +++++ src/internal/packager/helm/utils.go | 10 ++++-- src/internal/packager/helm/zarf.go | 4 +-- src/internal/packager/images/push.go | 3 ++ src/internal/packager/template/template.go | 7 ++-- src/pkg/cluster/data.go | 8 +++++ src/pkg/cluster/state.go | 8 +++-- src/pkg/logger/logger.go | 14 ++++++++ src/pkg/packager/deploy.go | 40 +++++++++++++++++++--- src/pkg/packager/prepare.go | 2 +- src/pkg/packager/sources/tarball.go | 5 +++ 15 files changed, 121 insertions(+), 28 deletions(-) diff --git a/src/cmd/destroy.go b/src/cmd/destroy.go index 6951efd96b..f3aeaa0f25 100644 --- a/src/cmd/destroy.go +++ b/src/cmd/destroy.go @@ -83,7 +83,7 @@ var destroyCmd = &cobra.Command{ } } else { // Perform chart uninstallation - helm.Destroy(removeComponents) + helm.Destroy(ctx, removeComponents) // If Zarf didn't deploy the cluster, only delete the ZarfNamespace if err := c.DeleteZarfNamespace(ctx); err != nil { diff --git a/src/internal/git/repository.go b/src/internal/git/repository.go index 9d2b2d4918..a0e2ac3ab6 100644 --- a/src/internal/git/repository.go +++ b/src/internal/git/repository.go @@ -18,6 +18,7 @@ import ( "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/zarf-dev/zarf/src/pkg/logger" "github.com/zarf-dev/zarf/src/pkg/message" "github.com/zarf-dev/zarf/src/pkg/transform" "github.com/zarf-dev/zarf/src/pkg/utils" @@ -146,6 +147,7 @@ func (r *Repository) Path() string { // Push pushes the repository to the remote git server. func (r *Repository) Push(ctx context.Context, address, username, password string) error { + l := logger.From(ctx) repo, err := git.PlainOpen(r.path) if err != nil { return fmt.Errorf("not a valid git repo or unable to open: %w", err) @@ -195,10 +197,13 @@ func (r *Repository) Push(ctx context.Context, address, username, password strin err = repo.FetchContext(ctx, fetchOptions) if errors.Is(err, transport.ErrRepositoryNotFound) { message.Debugf("Repo not yet available offline, skipping fetch...") + l.Debug("Repo not yet available offline, skipping fetch") } else if errors.Is(err, git.ErrForceNeeded) { message.Debugf("Repo fetch requires force, skipping fetch...") + l.Debug("Repo fetch requires force, skipping fetch") } else if errors.Is(err, git.NoErrAlreadyUpToDate) { message.Debugf("Repo already up-to-date, skipping fetch...") + l.Debug("Repo already up-to-date, skipping fetch") } else if err != nil { return fmt.Errorf("unable to fetch the git repo prior to push: %w", err) } @@ -217,6 +222,7 @@ func (r *Repository) Push(ctx context.Context, address, username, password strin }) if errors.Is(err, git.NoErrAlreadyUpToDate) { message.Debug("Repo already up-to-date") + l.Debug("repo already up-to-date") } else if errors.Is(err, plumbing.ErrObjectNotFound) { return fmt.Errorf("unable to push repo due to likely shallow clone: %s", err.Error()) } else if err != nil { diff --git a/src/internal/packager/helm/chart.go b/src/internal/packager/helm/chart.go index 954b1d6144..0443fa7423 100644 --- a/src/internal/packager/helm/chart.go +++ b/src/internal/packager/helm/chart.go @@ -28,21 +28,24 @@ import ( "github.com/zarf-dev/zarf/src/config" "github.com/zarf-dev/zarf/src/internal/healthchecks" + "github.com/zarf-dev/zarf/src/pkg/logger" "github.com/zarf-dev/zarf/src/pkg/message" "github.com/zarf-dev/zarf/src/types" ) // InstallOrUpgradeChart performs a helm install of the given chart. func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings, string, error) { - fromMessage := h.chart.URL - if fromMessage == "" { - fromMessage = "Zarf-generated helm chart" + l := logger.From(ctx) + source := h.chart.URL + if source == "" { + source = "Zarf-generated" } - spinner := message.NewProgressSpinner("Processing helm chart %s:%s from %s", + spinner := message.NewProgressSpinner("Processing helm chart %s:%s source: %s", h.chart.Name, h.chart.Version, - fromMessage) + source) defer spinner.Stop() + l.Info("processing helm chart", "name", h.chart.Name, "version", h.chart.Version, "source", source) // If no release name is specified, use the chart name. if h.chart.ReleaseName == "" { @@ -50,7 +53,7 @@ func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings, } // Setup K8s connection. - err := h.createActionConfig(h.chart.Namespace, spinner) + err := h.createActionConfig(ctx, h.chart.Namespace, spinner) if err != nil { return nil, "", fmt.Errorf("unable to initialize the K8s client: %w", err) } @@ -72,15 +75,18 @@ func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings, releases, histErr := histClient.Run(h.chart.ReleaseName) spinner.Updatef("Checking for existing helm deployment") + l.Debug("checking for existing helm deployment") if errors.Is(histErr, driver.ErrReleaseNotFound) { // No prior release, try to install it. spinner.Updatef("Attempting chart installation") + l.Info("preforming chart install") release, err = h.installChart(helmCtx, postRender) } else if histErr == nil && len(releases) > 0 { // Otherwise, there is a prior release so upgrade it. spinner.Updatef("Attempting chart upgrade") + l.Info("preforming chart upgrade") lastRelease := releases[len(releases)-1] @@ -117,6 +123,7 @@ func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings, // Attempt to rollback on a failed upgrade. spinner.Updatef("Performing chart rollback") + l.Info("preforming chart rollback") err = h.rollbackChart(h.chart.ReleaseName, previouslyDeployedVersion) if err != nil { return nil, "", fmt.Errorf("%w: unable to rollback: %w", installErr, err) @@ -136,6 +143,7 @@ func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings, if !h.chart.NoWait { // Ensure we don't go past the timeout by using a context initialized with the helm timeout spinner.Updatef("Running health checks") + l.Info("running health checks") if err := healthchecks.WaitForReadyRuntime(helmCtx, h.cluster.Watcher, runtimeObjs); err != nil { return nil, "", err } @@ -151,7 +159,7 @@ func (h *Helm) TemplateChart(ctx context.Context) (manifest string, chartValues spinner := message.NewProgressSpinner("Templating helm chart %s", h.chart.Name) defer spinner.Stop() - err = h.createActionConfig(h.chart.Namespace, spinner) + err = h.createActionConfig(ctx, h.chart.Namespace, spinner) // Setup K8s connection. if err != nil { @@ -215,7 +223,7 @@ func (h *Helm) TemplateChart(ctx context.Context) (manifest string, chartValues // RemoveChart removes a chart from the cluster. func (h *Helm) RemoveChart(namespace string, name string, spinner *message.Spinner) error { // Establish a new actionConfig for the namespace. - _ = h.createActionConfig(namespace, spinner) + _ = h.createActionConfig(context.TODO(), namespace, spinner) // Perform the uninstall. response, err := h.uninstallChart(name) message.Debug(response) @@ -228,7 +236,7 @@ func (h *Helm) UpdateReleaseValues(ctx context.Context, updatedValues map[string spinner := message.NewProgressSpinner("Updating values for helm release %s", h.chart.ReleaseName) defer spinner.Stop() - err := h.createActionConfig(h.chart.Namespace, spinner) + err := h.createActionConfig(ctx, h.chart.Namespace, spinner) if err != nil { return fmt.Errorf("unable to initialize the K8s client: %w", err) } diff --git a/src/internal/packager/helm/destroy.go b/src/internal/packager/helm/destroy.go index 44c519a00a..6383dffb01 100644 --- a/src/internal/packager/helm/destroy.go +++ b/src/internal/packager/helm/destroy.go @@ -5,6 +5,7 @@ package helm import ( + "context" "regexp" "github.com/zarf-dev/zarf/src/pkg/cluster" @@ -13,14 +14,14 @@ import ( ) // Destroy removes ZarfInitPackage charts from the cluster and optionally all Zarf-installed charts. -func Destroy(purgeAllZarfInstallations bool) { +func Destroy(ctx context.Context, purgeAllZarfInstallations bool) { spinner := message.NewProgressSpinner("Removing Zarf-installed charts") defer spinner.Stop() h := Helm{} // Initially load the actionConfig without a namespace - err := h.createActionConfig("", spinner) + err := h.createActionConfig(ctx, "", spinner) if err != nil { // Don't fatal since this is a removal action spinner.Errorf(err, "Unable to initialize the K8s client") diff --git a/src/internal/packager/helm/post-render.go b/src/internal/packager/helm/post-render.go index c316375acb..b80109ffd0 100644 --- a/src/internal/packager/helm/post-render.go +++ b/src/internal/packager/helm/post-render.go @@ -16,6 +16,7 @@ import ( "github.com/defenseunicorns/pkg/helpers/v2" "github.com/zarf-dev/zarf/src/config" "github.com/zarf-dev/zarf/src/pkg/cluster" + "github.com/zarf-dev/zarf/src/pkg/logger" "github.com/zarf-dev/zarf/src/pkg/message" "github.com/zarf-dev/zarf/src/pkg/utils" "github.com/zarf-dev/zarf/src/types" @@ -117,6 +118,7 @@ func (r *renderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { } func (r *renderer) adoptAndUpdateNamespaces(ctx context.Context) error { + l := logger.From(ctx) c := r.cluster namespaceList, err := r.cluster.Clientset.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) if err != nil { @@ -142,6 +144,7 @@ func (r *renderer) adoptAndUpdateNamespaces(ctx context.Context) error { // https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/#initial-namespaces if slices.Contains([]string{"default", "kube-node-lease", "kube-public", "kube-system"}, name) { message.Warnf("Refusing to adopt the initial namespace: %s", name) + l.Warn("refusing to adopt initial namespace", "name", name) } else { // This is an existing namespace to adopt _, err := c.Clientset.CoreV1().Namespaces().Update(ctx, namespace, metav1.UpdateOptions{}) @@ -181,6 +184,7 @@ func (r *renderer) adoptAndUpdateNamespaces(ctx context.Context) error { }() if err != nil { message.WarnErrf(err, "Problem creating registry secret for the %s namespace", name) + l.Warn("problem creating registry secret", "namespace", name, "error", err.Error()) } // Create or update the zarf git server secret @@ -201,6 +205,7 @@ func (r *renderer) adoptAndUpdateNamespaces(ctx context.Context) error { }() if err != nil { message.WarnErrf(err, "Problem creating git server secret for the %s namespace", name) + l.Warn("problem creating git server secret", "namespace", name, "error", err.Error()) } } } @@ -208,6 +213,7 @@ func (r *renderer) adoptAndUpdateNamespaces(ctx context.Context) error { } func (r *renderer) editHelmResources(ctx context.Context, resources []releaseutil.Manifest, finalManifestsOutput *bytes.Buffer) error { + l := logger.From(ctx) dc, err := dynamic.NewForConfig(r.cluster.RestConfig) if err != nil { return err @@ -231,8 +237,10 @@ func (r *renderer) editHelmResources(ctx context.Context, resources []releaseuti // parse the namespace resource so it can be applied out-of-band by zarf instead of helm to avoid helm ns shenanigans if err := runtime.DefaultUnstructuredConverter.FromUnstructured(rawData.UnstructuredContent(), namespace); err != nil { message.WarnErrf(err, "could not parse namespace %s", rawData.GetName()) + l.Warn("failed to parse namespace", "name", rawData.GetName(), "error", err) } else { message.Debugf("Matched helm namespace %s for zarf annotation", namespace.Name) + l.Debug("matched helm namespace for zarf annotation", "name", namespace.Name) namespace.Labels = cluster.AdoptZarfManagedLabels(namespace.Labels) // Add it to the stack r.namespaces[namespace.Name] = namespace @@ -253,6 +261,7 @@ func (r *renderer) editHelmResources(ctx context.Context, resources []releaseuti if key, keyExists := labels[cluster.ZarfConnectLabelName]; keyExists { // If there is a zarf-connect label message.Debugf("Match helm service %s for zarf connection %s", rawData.GetName(), key) + l.Debug("match helm service for zarf connection", "service", rawData.GetName(), "connection-key", key) // Add the connectString for processing later in the deployment r.connectStrings[key] = types.ConnectString{ diff --git a/src/internal/packager/helm/utils.go b/src/internal/packager/helm/utils.go index 8d3d5f3202..3a6e618f80 100644 --- a/src/internal/packager/helm/utils.go +++ b/src/internal/packager/helm/utils.go @@ -5,9 +5,12 @@ package helm import ( + "context" "fmt" + "log/slog" "github.com/defenseunicorns/pkg/helpers/v2" + "github.com/zarf-dev/zarf/src/pkg/logger" "github.com/zarf-dev/zarf/src/pkg/message" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" @@ -60,7 +63,7 @@ func (h *Helm) parseChartValues() (chartutil.Values, error) { return helpers.MergeMapRecursive(chartValues, h.valuesOverrides), nil } -func (h *Helm) createActionConfig(namespace string, spinner *message.Spinner) error { +func (h *Helm) createActionConfig(ctx context.Context, namespace string, spinner *message.Spinner) error { // Initialize helm SDK actionConfig := new(action.Configuration) // Set the settings for the helm SDK @@ -70,7 +73,10 @@ func (h *Helm) createActionConfig(namespace string, spinner *message.Spinner) er h.settings.SetNamespace(namespace) // Setup K8s connection - err := actionConfig.Init(h.settings.RESTClientGetter(), namespace, "", spinner.Updatef) + // TODO (austinabro321) this code cannot be merged as is + l := logger.From(ctx) + logLogger := slog.NewLogLogger(l.Handler(), slog.LevelInfo) + err := actionConfig.Init(h.settings.RESTClientGetter(), namespace, "", logLogger.Printf) // Set the actionConfig is the received Helm pointer h.actionConfig = actionConfig diff --git a/src/internal/packager/helm/zarf.go b/src/internal/packager/helm/zarf.go index b6945a6e8c..4783f6557e 100644 --- a/src/internal/packager/helm/zarf.go +++ b/src/internal/packager/helm/zarf.go @@ -81,7 +81,7 @@ func (h *Helm) UpdateZarfAgentValues(ctx context.Context) error { return err } - err = h.createActionConfig(cluster.ZarfNamespaceName, spinner) + err = h.createActionConfig(ctx, cluster.ZarfNamespaceName, spinner) if err != nil { return err } @@ -111,7 +111,7 @@ func (h *Helm) UpdateZarfAgentValues(ctx context.Context) error { Value: agentImage.Tag, }, }) - applicationTemplates, err := template.GetZarfTemplates("zarf-agent", h.state) + applicationTemplates, err := template.GetZarfTemplates(ctx, "zarf-agent", h.state) if err != nil { return fmt.Errorf("error setting up the templates: %w", err) } diff --git a/src/internal/packager/images/push.go b/src/internal/packager/images/push.go index e04c628023..f432d960db 100644 --- a/src/internal/packager/images/push.go +++ b/src/internal/packager/images/push.go @@ -14,6 +14,7 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/zarf-dev/zarf/src/pkg/cluster" + "github.com/zarf-dev/zarf/src/pkg/logger" "github.com/zarf-dev/zarf/src/pkg/message" "github.com/zarf-dev/zarf/src/pkg/transform" "github.com/zarf-dev/zarf/src/pkg/utils" @@ -21,6 +22,7 @@ import ( // Push pushes images to a registry. func Push(ctx context.Context, cfg PushConfig) error { + l := logger.From(ctx) logs.Warn.SetOutput(&message.DebugWriter{}) logs.Progress.SetOutput(&message.DebugWriter{}) @@ -67,6 +69,7 @@ func Push(ctx context.Context, cfg PushConfig) error { }() for refInfo, img := range toPush { message.Infof("Pushing %s", refInfo.Reference) + l.Info("pushing image", "name", refInfo.Reference) // If this is not a no checksum image push it for use with the Zarf agent if !cfg.NoChecksum { offlineNameCRC, err := transform.ImageTransformHost(registryURL, refInfo.Reference) diff --git a/src/internal/packager/template/template.go b/src/internal/packager/template/template.go index ec53877db1..233fa5d7bd 100644 --- a/src/internal/packager/template/template.go +++ b/src/internal/packager/template/template.go @@ -39,7 +39,7 @@ func GetZarfVariableConfig(ctx context.Context) *variables.VariableConfig { } // GetZarfTemplates returns the template keys and values to be used for templating. -func GetZarfTemplates(componentName string, state *types.ZarfState) (templateMap map[string]*variables.TextTemplate, err error) { +func GetZarfTemplates(ctx context.Context, componentName string, state *types.ZarfState) (templateMap map[string]*variables.TextTemplate, err error) { templateMap = make(map[string]*variables.TextTemplate) if state != nil { @@ -98,7 +98,7 @@ func GetZarfTemplates(componentName string, state *types.ZarfState) (templateMap } } - debugPrintTemplateMap(templateMap) + debugPrintTemplateMap(ctx, templateMap) return templateMap, nil } @@ -123,7 +123,7 @@ func generateHtpasswd(regInfo *types.RegistryInfo) (string, error) { return "", nil } -func debugPrintTemplateMap(templateMap map[string]*variables.TextTemplate) { +func debugPrintTemplateMap(ctx context.Context, templateMap map[string]*variables.TextTemplate) { debugText := "templateMap = { " for key, template := range templateMap { @@ -137,4 +137,5 @@ func debugPrintTemplateMap(templateMap map[string]*variables.TextTemplate) { debugText += " }" message.Debug(debugText) + logger.From(ctx).Debug(debugText) } diff --git a/src/pkg/cluster/data.go b/src/pkg/cluster/data.go index 68148003a7..81d8cdd460 100644 --- a/src/pkg/cluster/data.go +++ b/src/pkg/cluster/data.go @@ -25,6 +25,7 @@ import ( "github.com/zarf-dev/zarf/src/api/v1alpha1" "github.com/zarf-dev/zarf/src/config" "github.com/zarf-dev/zarf/src/pkg/layout" + "github.com/zarf-dev/zarf/src/pkg/logger" "github.com/zarf-dev/zarf/src/pkg/message" "github.com/zarf-dev/zarf/src/pkg/utils" "github.com/zarf-dev/zarf/src/pkg/utils/exec" @@ -33,6 +34,7 @@ import ( // HandleDataInjection waits for the target pod(s) to come up and inject the data into them // todo: this currently requires kubectl but we should have enough k8s work to make this native now. func (c *Cluster) HandleDataInjection(ctx context.Context, data v1alpha1.ZarfDataInjection, componentPath *layout.ComponentPaths, dataIdx int) error { + l := logger.From(ctx) injectionCompletionMarker := filepath.Join(componentPath.DataInjections, config.GetDataInjectionMarker()) if err := os.WriteFile(injectionCompletionMarker, []byte("🦄"), helpers.ReadWriteUser); err != nil { return fmt.Errorf("unable to create the data injection completion marker: %w", err) @@ -91,6 +93,7 @@ func (c *Cluster) HandleDataInjection(ctx context.Context, data v1alpha1.ZarfDat kubectlBinPath := "kubectl" if err != nil { message.Warnf("Unable to get the zarf executable path, falling back to host kubectl: %s", err) + l.Warn("unable to get the zarf executable path, falling back to host kubectl", "error", err) } else { kubectlBinPath = fmt.Sprintf("%s tools kubectl", zarfCommand) } @@ -171,6 +174,7 @@ type podFilter func(pod corev1.Pod) bool // If the timeout is reached, an empty list will be returned. // TODO: Test, refactor and/or remove. func waitForPodsAndContainers(ctx context.Context, clientset kubernetes.Interface, target podLookup, include podFilter) ([]corev1.Pod, error) { + l := logger.From(ctx) readyPods, err := retry.DoWithData(func() ([]corev1.Pod, error) { listOpts := metav1.ListOptions{ LabelSelector: target.Selector, @@ -180,6 +184,7 @@ func waitForPodsAndContainers(ctx context.Context, clientset kubernetes.Interfac return nil, err } message.Debugf("Found %d pods for target %#v", len(podList.Items), target) + l.Debug("found pods matching the target", "count", len(podList.Items), "target", target) // Sort the pods from newest to oldest sort.Slice(podList.Items, func(i, j int) bool { return podList.Items[i].CreationTimestamp.After(podList.Items[j].CreationTimestamp.Time) @@ -188,6 +193,7 @@ func waitForPodsAndContainers(ctx context.Context, clientset kubernetes.Interfac readyPods := []corev1.Pod{} for _, pod := range podList.Items { message.Debugf("Testing pod %q", pod.Name) + l.Debug("testing pod", "name", pod.Name) // If an include function is provided, only keep pods that return true if include != nil && !include(pod) { @@ -197,6 +203,7 @@ func waitForPodsAndContainers(ctx context.Context, clientset kubernetes.Interfac // Handle container targeting if target.Container != "" { message.Debugf("Testing pod %q for container %q", pod.Name, target.Container) + l.Debug("testing container", "name", target.Container, "source-pod", pod.Name) // Check the status of initContainers for a running match for _, initContainer := range pod.Status.InitContainerStatuses { @@ -219,6 +226,7 @@ func waitForPodsAndContainers(ctx context.Context, clientset kubernetes.Interfac } else { status := pod.Status.Phase message.Debugf("Testing pod %q phase, want (%q) got (%q)", pod.Name, corev1.PodRunning, status) + l.Debug(fmt.Sprintf("checking pod for %s status", corev1.PodRunning), "pod", pod.Name, "status", status) // Regular status checking without a container if status == corev1.PodRunning { readyPods = append(readyPods, pod) diff --git a/src/pkg/cluster/state.go b/src/pkg/cluster/state.go index f2279b0221..3744255d02 100644 --- a/src/pkg/cluster/state.go +++ b/src/pkg/cluster/state.go @@ -20,6 +20,7 @@ import ( "github.com/defenseunicorns/pkg/helpers/v2" "github.com/zarf-dev/zarf/src/config" "github.com/zarf-dev/zarf/src/config/lang" + "github.com/zarf-dev/zarf/src/pkg/logger" "github.com/zarf-dev/zarf/src/pkg/message" "github.com/zarf-dev/zarf/src/pkg/pki" "github.com/zarf-dev/zarf/src/types" @@ -205,7 +206,7 @@ func (c *Cluster) LoadZarfState(ctx context.Context) (*types.ZarfState, error) { if err != nil { return nil, fmt.Errorf("%w: %w", stateErr, err) } - c.debugPrintZarfState(state) + c.debugPrintZarfState(ctx, state) return state, nil } @@ -230,7 +231,7 @@ func (c *Cluster) sanitizeZarfState(state *types.ZarfState) *types.ZarfState { return state } -func (c *Cluster) debugPrintZarfState(state *types.ZarfState) { +func (c *Cluster) debugPrintZarfState(ctx context.Context, state *types.ZarfState) { if state == nil { return } @@ -242,11 +243,12 @@ func (c *Cluster) debugPrintZarfState(state *types.ZarfState) { return } message.Debugf("ZarfState - %s", string(b)) + logger.From(ctx).Debug("", "state", string(b)) } // SaveZarfState takes a given state and persists it to the Zarf/zarf-state secret. func (c *Cluster) SaveZarfState(ctx context.Context, state *types.ZarfState) error { - c.debugPrintZarfState(state) + c.debugPrintZarfState(ctx, state) data, err := json.Marshal(&state) if err != nil { diff --git a/src/pkg/logger/logger.go b/src/pkg/logger/logger.go index ccbddf7901..9bb225c994 100644 --- a/src/pkg/logger/logger.go +++ b/src/pkg/logger/logger.go @@ -211,3 +211,17 @@ func Default() *slog.Logger { func SetDefault(l *slog.Logger) { defaultLogger.Store(l) } + +func DebugLogFunc(logger *slog.Logger) func(format string, v ...interface{}) { + return func(format string, v ...interface{}) { + message := fmt.Sprintf(format, v...) + logger.Debug(message) + } +} + +func InfoLogFunc(logger *slog.Logger) func(format string, v ...interface{}) { + return func(format string, v ...interface{}) { + message := fmt.Sprintf(format, v...) + logger.Info(message) + } +} diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 26079df5d4..06dec3f6b3 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -34,6 +34,7 @@ import ( "github.com/zarf-dev/zarf/src/internal/packager/template" "github.com/zarf-dev/zarf/src/pkg/cluster" "github.com/zarf-dev/zarf/src/pkg/layout" + "github.com/zarf-dev/zarf/src/pkg/logger" "github.com/zarf-dev/zarf/src/pkg/message" "github.com/zarf-dev/zarf/src/pkg/packager/actions" "github.com/zarf-dev/zarf/src/pkg/packager/filters" @@ -56,6 +57,7 @@ func (p *Packager) resetRegistryHPA(ctx context.Context) { // Deploy attempts to deploy the given PackageConfig. func (p *Packager) Deploy(ctx context.Context) error { + l := logger.From(ctx) isInteractive := !config.CommonOptions.Confirm deployFilter := filters.Combine( @@ -89,12 +91,18 @@ func (p *Packager) Deploy(ctx context.Context) error { return err } warnings = append(warnings, validateWarnings...) + for _, warning := range validateWarnings { + l.Warn(warning) + } sbomViewFiles, sbomWarnings, err := p.layout.SBOMs.StageSBOMViewFiles() if err != nil { return err } warnings = append(warnings, sbomWarnings...) + for _, warning := range sbomWarnings { + l.Warn(warning) + } // Confirm the overall package deployment if !p.confirmAction(config.ZarfDeployStage, warnings, sbomViewFiles) { @@ -124,10 +132,12 @@ func (p *Packager) Deploy(ctx context.Context) error { } if len(deployedComponents) == 0 { message.Warn("No components were selected for deployment. Inspect the package to view the available components and select components interactively or by name with \"--components\"") + l.Warn("No components were selected for deployment. Inspect the package to view the available components and select components interactively or by name with \"--components\"") } // Notify all the things about the successful deployment message.Successf("Zarf deployment complete") + l.Info("Zarf deployed successfully") err = p.printTablesForDeployment(ctx, deployedComponents) if err != nil { @@ -139,6 +149,7 @@ func (p *Packager) Deploy(ctx context.Context) error { // deployComponents loops through a list of ZarfComponents and deploys them. func (p *Packager) deployComponents(ctx context.Context) ([]types.DeployedComponent, error) { + l := logger.From(ctx) deployedComponents := []types.DeployedComponent{} // Process all the components we are deploying @@ -165,6 +176,7 @@ func (p *Packager) deployComponents(ctx context.Context) ([]types.DeployedCompon installedCharts, err := p.cluster.GetInstalledChartsForComponent(ctx, p.cfg.Pkg.Metadata.Name, component) if err != nil { message.Debugf("Unable to fetch installed Helm charts for component '%s': %s", component.Name, err.Error()) + l.Debug("unable to fetch installed Helm charts", "component", component.Name, "error", err.Error()) } deployedComponent.InstalledCharts = installedCharts } @@ -186,6 +198,7 @@ func (p *Packager) deployComponents(ctx context.Context) ([]types.DeployedCompon onFailure := func() { if err := actions.Run(ctx, onDeploy.Defaults, onDeploy.OnFailure, p.variableConfig); err != nil { message.Debugf("unable to run component failure action: %s", err.Error()) + l.Debug("unable to run component failure action", "error", err.Error()) } } @@ -195,6 +208,7 @@ func (p *Packager) deployComponents(ctx context.Context) ([]types.DeployedCompon if p.isConnectedToCluster() { if _, err := p.cluster.RecordPackageDeployment(ctx, p.cfg.Pkg, deployedComponents); err != nil { message.Debugf("Unable to record package deployment for component %q: this will affect features like `zarf package remove`: %s", component.Name, err.Error()) + l.Debug("unable to record package deployment", "component", component.Name, "error", err.Error()) } } return nil, fmt.Errorf("unable to deploy component %q: %w", component.Name, deployErr) @@ -205,6 +219,7 @@ func (p *Packager) deployComponents(ctx context.Context) ([]types.DeployedCompon if p.isConnectedToCluster() { if _, err := p.cluster.RecordPackageDeployment(ctx, p.cfg.Pkg, deployedComponents); err != nil { message.Debugf("Unable to record package deployment for component %q: this will affect features like `zarf package remove`: %s", component.Name, err.Error()) + l.Debug("unable to record package deployment", "component", component.Name, "error", err.Error()) } } @@ -218,6 +233,7 @@ func (p *Packager) deployComponents(ctx context.Context) ([]types.DeployedCompon } func (p *Packager) deployInitComponent(ctx context.Context, component v1alpha1.ZarfComponent) ([]types.InstalledChart, error) { + l := logger.From(ctx) hasExternalRegistry := p.cfg.InitOpts.RegistryInfo.Address != "" isSeedRegistry := component.Name == "zarf-seed-registry" isRegistry := component.Name == "zarf-registry" @@ -239,6 +255,7 @@ func (p *Packager) deployInitComponent(ctx context.Context, component v1alpha1.Z if hasExternalRegistry && (isSeedRegistry || isInjector || isRegistry) { message.Notef("Not deploying the component (%s) since external registry information was provided during `zarf init`", component.Name) + l.Info("skipping init package component since external registry information was provided", "component", component.Name) return nil, nil } @@ -274,6 +291,7 @@ func (p *Packager) deployInitComponent(ctx context.Context, component v1alpha1.Z // Deploy a Zarf Component. func (p *Packager) deployComponent(ctx context.Context, component v1alpha1.ZarfComponent, noImgChecksum bool, noImgPush bool) ([]types.InstalledChart, error) { + l := logger.From(ctx) // Toggles for general deploy operations componentPath := p.layout.Components.Dirs[component.Name] @@ -301,13 +319,14 @@ func (p *Packager) deployComponent(ctx context.Context, component v1alpha1.ZarfC if hasImages && !p.hpaModified && p.state.RegistryInfo.IsInternal() { if err := p.cluster.DisableRegHPAScaleDown(ctx); err != nil { message.Debugf("unable to disable the registry HPA scale down: %s", err.Error()) + l.Debug("unable to disable the registry HPA scale down", "error", err.Error()) } else { p.hpaModified = true } } } - err := p.populateComponentAndStateTemplates(component.Name) + err := p.populateComponentAndStateTemplates(ctx, component.Name) if err != nil { return nil, err } @@ -317,7 +336,7 @@ func (p *Packager) deployComponent(ctx context.Context, component v1alpha1.ZarfC } if hasFiles { - if err := p.processComponentFiles(component, componentPath.Files); err != nil { + if err := p.processComponentFiles(ctx, component, componentPath.Files); err != nil { return nil, fmt.Errorf("unable to process the component files: %w", err) } } @@ -357,6 +376,7 @@ func (p *Packager) deployComponent(ctx context.Context, component v1alpha1.ZarfC healthCheckContext, cancel := context.WithTimeout(ctx, p.cfg.DeployOpts.Timeout) defer cancel() spinner := message.NewProgressSpinner("Running health checks") + l.Info("running health checks") defer spinner.Stop() if err = healthchecks.Run(healthCheckContext, p.cluster.Watcher, component.HealthChecks); err != nil { return nil, fmt.Errorf("health checks failed: %w", err) @@ -372,8 +392,11 @@ func (p *Packager) deployComponent(ctx context.Context, component v1alpha1.ZarfC } // Move files onto the host of the machine performing the deployment. -func (p *Packager) processComponentFiles(component v1alpha1.ZarfComponent, pkgLocation string) error { +func (p *Packager) processComponentFiles(ctx context.Context, component v1alpha1.ZarfComponent, pkgLocation string) error { + l := logger.From(ctx) + l.Handler() spinner := message.NewProgressSpinner("Copying %d files", len(component.Files)) + l.Info("Copying files", "count", len(component.Files)) defer spinner.Stop() for fileIdx, file := range component.Files { @@ -387,6 +410,7 @@ func (p *Packager) processComponentFiles(component v1alpha1.ZarfComponent, pkgLo // If a shasum is specified check it again on deployment as well if file.Shasum != "" { spinner.Updatef("Validating SHASUM for %s", file.Target) + l.Info("Validating SHASUM", "file", file.Target) if err := helpers.SHAsMatch(fileLocation, file.Shasum); err != nil { return err } @@ -455,9 +479,11 @@ func (p *Packager) processComponentFiles(component v1alpha1.ZarfComponent, pkgLo // setupState fetches the current ZarfState from the k8s cluster and sets the packager to use it func (p *Packager) setupState(ctx context.Context) error { + l := logger.From(ctx) // If we are touching K8s, make sure we can talk to it once per deployment spinner := message.NewProgressSpinner("Loading the Zarf State from the Kubernetes cluster") defer spinner.Stop() + l.Info("loading the Zarf State from the Kubernetes cluster") state, err := p.cluster.LoadZarfState(ctx) // We ignore the error if in YOLO mode because Zarf should not be initiated. @@ -475,6 +501,7 @@ func (p *Packager) setupState(ctx context.Context) error { // Try to create the zarf namespace spinner.Updatef("Creating the Zarf namespace") + l.Info("creating the Zarf namespace") zarfNamespace := cluster.NewZarfManagedNamespace(cluster.ZarfNamespaceName) err := func() error { _, err := p.cluster.Clientset.CoreV1().Namespaces().Create(ctx, zarfNamespace, metav1.CreateOptions{}) @@ -499,6 +526,9 @@ func (p *Packager) setupState(ctx context.Context) error { message.Warn("This package is in YOLO mode, but the cluster was already initialized with 'zarf init'. " + "This may cause issues if the package does not exclude any charts or manifests from the Zarf Agent using " + "the pod or namespace label `zarf.dev/agent: ignore'.") + l.Warn("This package is in YOLO mode, but the cluster was already initialized with 'zarf init'. " + + "This may cause issues if the package does not exclude any charts or manifests from the Zarf Agent using " + + "the pod or namespace label `zarf.dev/agent: ignore'.") } p.state = state @@ -507,8 +537,8 @@ func (p *Packager) setupState(ctx context.Context) error { return nil } -func (p *Packager) populateComponentAndStateTemplates(componentName string) error { - applicationTemplates, err := template.GetZarfTemplates(componentName, p.state) +func (p *Packager) populateComponentAndStateTemplates(ctx context.Context, componentName string) error { + applicationTemplates, err := template.GetZarfTemplates(ctx, componentName, p.state) if err != nil { return err } diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 1e43335898..89b5ebe625 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -140,7 +140,7 @@ func (p *Packager) findImages(ctx context.Context) (map[string][]string, error) if err != nil { return nil, err } - err = p.populateComponentAndStateTemplates(component.Name) + err = p.populateComponentAndStateTemplates(ctx, component.Name) if err != nil { return nil, err } diff --git a/src/pkg/packager/sources/tarball.go b/src/pkg/packager/sources/tarball.go index 5b556f78e1..e5d142b593 100644 --- a/src/pkg/packager/sources/tarball.go +++ b/src/pkg/packager/sources/tarball.go @@ -17,6 +17,7 @@ import ( "github.com/mholt/archiver/v3" "github.com/zarf-dev/zarf/src/api/v1alpha1" "github.com/zarf-dev/zarf/src/pkg/layout" + "github.com/zarf-dev/zarf/src/pkg/logger" "github.com/zarf-dev/zarf/src/pkg/message" "github.com/zarf-dev/zarf/src/pkg/packager/filters" "github.com/zarf-dev/zarf/src/pkg/zoci" @@ -37,6 +38,8 @@ type TarballSource struct { func (s *TarballSource) LoadPackage(ctx context.Context, dst *layout.PackagePaths, filter filters.ComponentFilterStrategy, unarchiveAll bool) (pkg v1alpha1.ZarfPackage, warnings []string, err error) { spinner := message.NewProgressSpinner("Loading package from %q", s.PackageSource) defer spinner.Stop() + l := logger.From(ctx) + l.Info("loading package", "source", s.PackageSource) if s.Shasum != "" { if err := helpers.SHAsMatch(s.PackageSource, s.Shasum); err != nil { @@ -100,12 +103,14 @@ func (s *TarballSource) LoadPackage(ctx context.Context, dst *layout.PackagePath if !dst.IsLegacyLayout() { spinner := message.NewProgressSpinner("Validating full package checksums") defer spinner.Stop() + l.Info("validating package checksums") if err := ValidatePackageIntegrity(dst, pkg.Metadata.AggregateChecksum, false); err != nil { return pkg, nil, err } spinner.Success() + l.Debug("done validating package checksums") if !s.SkipSignatureValidation { if err := ValidatePackageSignature(ctx, dst, s.PublicKeyPath); err != nil { From 0425bebc2a887e452bb9903e4c7d0fbd9538098e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 29 Oct 2024 14:40:47 +0000 Subject: [PATCH 02/19] delete unused functions Signed-off-by: Austin Abro --- src/pkg/logger/logger.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/pkg/logger/logger.go b/src/pkg/logger/logger.go index 9bb225c994..ccbddf7901 100644 --- a/src/pkg/logger/logger.go +++ b/src/pkg/logger/logger.go @@ -211,17 +211,3 @@ func Default() *slog.Logger { func SetDefault(l *slog.Logger) { defaultLogger.Store(l) } - -func DebugLogFunc(logger *slog.Logger) func(format string, v ...interface{}) { - return func(format string, v ...interface{}) { - message := fmt.Sprintf(format, v...) - logger.Debug(message) - } -} - -func InfoLogFunc(logger *slog.Logger) func(format string, v ...interface{}) { - return func(format string, v ...interface{}) { - message := fmt.Sprintf(format, v...) - logger.Info(message) - } -} From f600cff1943752b8f27366c9ae8a7a9dfabd634b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 29 Oct 2024 15:52:39 +0000 Subject: [PATCH 03/19] introduce use slog Signed-off-by: Austin Abro --- src/cmd/root.go | 1 + src/internal/packager/helm/utils.go | 14 ++++++++++---- src/types/runtime.go | 2 ++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/cmd/root.go b/src/cmd/root.go index 7899388317..eb7b73f54c 100644 --- a/src/cmd/root.go +++ b/src/cmd/root.go @@ -95,6 +95,7 @@ func preRun(cmd *cobra.Command, _ []string) error { var disableMessage bool if LogFormat != "" { disableMessage = true + config.CommonOptions.UseSlog = true } err = setupMessage(messageCfg{ level: LogLevelCLI, diff --git a/src/internal/packager/helm/utils.go b/src/internal/packager/helm/utils.go index 3a6e618f80..b677037a0b 100644 --- a/src/internal/packager/helm/utils.go +++ b/src/internal/packager/helm/utils.go @@ -10,6 +10,7 @@ import ( "log/slog" "github.com/defenseunicorns/pkg/helpers/v2" + "github.com/zarf-dev/zarf/src/config" "github.com/zarf-dev/zarf/src/pkg/logger" "github.com/zarf-dev/zarf/src/pkg/message" "helm.sh/helm/v3/pkg/action" @@ -73,10 +74,15 @@ func (h *Helm) createActionConfig(ctx context.Context, namespace string, spinner h.settings.SetNamespace(namespace) // Setup K8s connection - // TODO (austinabro321) this code cannot be merged as is - l := logger.From(ctx) - logLogger := slog.NewLogLogger(l.Handler(), slog.LevelInfo) - err := actionConfig.Init(h.settings.RESTClientGetter(), namespace, "", logLogger.Printf) + var helmLogger action.DebugLog + if config.CommonOptions.UseSlog { + l := logger.From(ctx) + helmLogger = slog.NewLogLogger(l.Handler(), slog.LevelDebug).Printf + } else { + helmLogger = spinner.Updatef + } + + err := actionConfig.Init(h.settings.RESTClientGetter(), namespace, "", helmLogger) // Set the actionConfig is the received Helm pointer h.actionConfig = actionConfig diff --git a/src/types/runtime.go b/src/types/runtime.go index aad1bd000c..649281a398 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -24,6 +24,8 @@ type ZarfCommonOptions struct { TempDirectory string // Number of concurrent layer operations to perform when interacting with a remote package OCIConcurrency int + // UseSlog tells Zarf whether or not to use the slogger over pterm when passing to external libraries + UseSlog bool } // ZarfPackageOptions tracks the user-defined preferences during common package operations. From 76c5a2ed32abfec5ddddf91fb040b5466ade62d1 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 29 Oct 2024 15:53:16 +0000 Subject: [PATCH 04/19] added todo Signed-off-by: Austin Abro --- src/types/runtime.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/types/runtime.go b/src/types/runtime.go index 649281a398..d2b9d94f1d 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -25,6 +25,7 @@ type ZarfCommonOptions struct { // Number of concurrent layer operations to perform when interacting with a remote package OCIConcurrency int // UseSlog tells Zarf whether or not to use the slogger over pterm when passing to external libraries + // TODO (@austinabro321) delete this when we officially switch over to slog UseSlog bool } From 9b3056a676ad377eca27829b1f245463b33957b7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 29 Oct 2024 15:54:55 +0000 Subject: [PATCH 05/19] move context Signed-off-by: Austin Abro --- src/internal/packager/helm/chart.go | 4 ++-- src/internal/packager/helm/destroy.go | 2 +- src/pkg/packager/remove.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/internal/packager/helm/chart.go b/src/internal/packager/helm/chart.go index 0443fa7423..e8b1309d9b 100644 --- a/src/internal/packager/helm/chart.go +++ b/src/internal/packager/helm/chart.go @@ -221,9 +221,9 @@ func (h *Helm) TemplateChart(ctx context.Context) (manifest string, chartValues } // RemoveChart removes a chart from the cluster. -func (h *Helm) RemoveChart(namespace string, name string, spinner *message.Spinner) error { +func (h *Helm) RemoveChart(ctx context.Context, namespace string, name string, spinner *message.Spinner) error { // Establish a new actionConfig for the namespace. - _ = h.createActionConfig(context.TODO(), namespace, spinner) + _ = h.createActionConfig(ctx, namespace, spinner) // Perform the uninstall. response, err := h.uninstallChart(name) message.Debug(response) diff --git a/src/internal/packager/helm/destroy.go b/src/internal/packager/helm/destroy.go index 6383dffb01..bf94ead9eb 100644 --- a/src/internal/packager/helm/destroy.go +++ b/src/internal/packager/helm/destroy.go @@ -54,7 +54,7 @@ func Destroy(ctx context.Context, purgeAllZarfInstallations bool) { // Filter on zarf releases if zarfPrefix.MatchString(release.Name) { spinner.Updatef("Uninstalling helm chart %s/%s", release.Namespace, release.Name) - if err = h.RemoveChart(release.Namespace, release.Name, spinner); err != nil { + if err = h.RemoveChart(ctx, release.Namespace, release.Name, spinner); err != nil { // Don't fatal since this is a removal action spinner.Errorf(err, "Unable to uninstall the chart") } diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index 322db5570f..b1149e343d 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -185,7 +185,7 @@ func (p *Packager) removeComponent(ctx context.Context, deployedPackage *types.D spinner.Updatef("Uninstalling chart '%s' from the '%s' component", chart.ChartName, deployedComponent.Name) helmCfg := helm.NewClusterOnly(p.cfg, p.variableConfig, p.state, p.cluster) - if err := helmCfg.RemoveChart(chart.Namespace, chart.ChartName, spinner); err != nil { + if err := helmCfg.RemoveChart(ctx, chart.Namespace, chart.ChartName, spinner); err != nil { if !errors.Is(err, driver.ErrReleaseNotFound) { onFailure() return deployedPackage, fmt.Errorf("unable to uninstall the helm chart %s in the namespace %s: %w", From bf29b07b8a5a9951d251ba0543714de2a7d65ad5 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 29 Oct 2024 17:07:03 +0000 Subject: [PATCH 06/19] add flag to enable logger Signed-off-by: Austin Abro --- src/cmd/root.go | 2 ++ src/pkg/logger/logger.go | 23 +++++++++++++++++++++++ src/pkg/logger/logger_test.go | 4 ++++ 3 files changed, 29 insertions(+) diff --git a/src/cmd/root.go b/src/cmd/root.go index 7899388317..63513c7104 100644 --- a/src/cmd/root.go +++ b/src/cmd/root.go @@ -95,6 +95,8 @@ func preRun(cmd *cobra.Command, _ []string) error { var disableMessage bool if LogFormat != "" { disableMessage = true + ctx := logger.WithLoggingEnabled(ctx, true) + cmd.SetContext(ctx) } err = setupMessage(messageCfg{ level: LogLevelCLI, diff --git a/src/pkg/logger/logger.go b/src/pkg/logger/logger.go index ccbddf7901..18ca259351 100644 --- a/src/pkg/logger/logger.go +++ b/src/pkg/logger/logger.go @@ -170,6 +170,29 @@ func WithContext(ctx context.Context, logger *slog.Logger) context.Context { return context.WithValue(ctx, defaultCtxKey, logger) } +type ctxKeyEnabled struct{} + +var defaultCtxKeyEnabled = ctxKeyEnabled{} + +// WithLoggingEnabled allows stores a value to determine whether or not slog logging is enabled +func WithLoggingEnabled(ctx context.Context, enabled bool) context.Context { + return context.WithValue(ctx, defaultCtxKeyEnabled, enabled) +} + +// Enabled returns true if slog logging is enabled +func Enabled(ctx context.Context) bool { + if ctx == nil { + return false + } + enabled := ctx.Value(defaultCtxKeyEnabled) + switch v := enabled.(type) { + case bool: + return v + default: + return false + } +} + // From takes a context and reads out a *slog.Logger. If From does not find a value it will return a discarding logger // similar to log-format "none". func From(ctx context.Context) *slog.Logger { diff --git a/src/pkg/logger/logger_test.go b/src/pkg/logger/logger_test.go index 409922c7b2..74695ca063 100644 --- a/src/pkg/logger/logger_test.go +++ b/src/pkg/logger/logger_test.go @@ -224,4 +224,8 @@ func TestContext(t *testing.T) { res := From(ctx) require.NotNil(t, res) }) + t.Run("can add a flag to the context to determine if enabled", func(t *testing.T) { + ctx := WithLoggingEnabled(context.Background(), true) + require.True(t, Enabled(ctx)) + }) } From cea30a34512a78aaf967d7adc75a166de6ea1908 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 29 Oct 2024 17:10:31 +0000 Subject: [PATCH 07/19] enable slogger Signed-off-by: Austin Abro --- src/internal/packager/images/pull.go | 1 + src/internal/packager/template/template.go | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 8d3b8201ae..7d5baf8464 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -144,6 +144,7 @@ func Pull(ctx context.Context, cfg PullConfig) (map[transform.Image]v1.Image, er // TODO(mkcp): Remove message on logger release message.Warnf("Falling back to local 'docker', failed to find the manifest on a remote: %s", err.Error()) + l.Warn("Falling back to local 'docker', failed to find the manifest on a remote", "error", err.Error()) // Attempt to connect to the local docker daemon. cli, err := client.NewClientWithOpts(client.FromEnv) diff --git a/src/internal/packager/template/template.go b/src/internal/packager/template/template.go index ec53877db1..afb8b5d409 100644 --- a/src/internal/packager/template/template.go +++ b/src/internal/packager/template/template.go @@ -8,6 +8,7 @@ import ( "context" "encoding/base64" "fmt" + "log/slog" "strings" "github.com/zarf-dev/zarf/src/api/v1alpha1" @@ -35,7 +36,13 @@ func GetZarfVariableConfig(ctx context.Context) *variables.VariableConfig { return interactive.PromptVariable(variable) } - return variables.New("zarf", prompt, logger.From(ctx)) + var l *slog.Logger + if logger.Enabled(ctx) { + l = logger.From(ctx) + } else { + l = slog.New(message.ZarfHandler{}) + } + return variables.New("zarf", prompt, l) } // GetZarfTemplates returns the template keys and values to be used for templating. From 3fc8432de385d86c9d6acbb1807ec9594eb35828 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 29 Oct 2024 17:11:50 +0000 Subject: [PATCH 08/19] todo Signed-off-by: Austin Abro --- src/pkg/logger/logger.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pkg/logger/logger.go b/src/pkg/logger/logger.go index 18ca259351..bd17269772 100644 --- a/src/pkg/logger/logger.go +++ b/src/pkg/logger/logger.go @@ -170,6 +170,7 @@ func WithContext(ctx context.Context, logger *slog.Logger) context.Context { return context.WithValue(ctx, defaultCtxKey, logger) } +// TODO (@austinabro321) once we switch over to the new logger completely the enabled key & logic should be deleted type ctxKeyEnabled struct{} var defaultCtxKeyEnabled = ctxKeyEnabled{} From 9ec0f9ce8de3cc2a9752b54d37257a41483922ea Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 29 Oct 2024 18:05:22 +0000 Subject: [PATCH 09/19] refactor enabled Signed-off-by: Austin Abro --- src/internal/packager/template/template.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/internal/packager/template/template.go b/src/internal/packager/template/template.go index afb8b5d409..a77d09de93 100644 --- a/src/internal/packager/template/template.go +++ b/src/internal/packager/template/template.go @@ -36,13 +36,10 @@ func GetZarfVariableConfig(ctx context.Context) *variables.VariableConfig { return interactive.PromptVariable(variable) } - var l *slog.Logger if logger.Enabled(ctx) { - l = logger.From(ctx) - } else { - l = slog.New(message.ZarfHandler{}) + return variables.New("zarf", prompt, logger.From(ctx)) } - return variables.New("zarf", prompt, l) + return variables.New("zarf", prompt, slog.New(message.ZarfHandler{})) } // GetZarfTemplates returns the template keys and values to be used for templating. From 47d87f0ef399a807df28bb31e72f38ab7627575c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 29 Oct 2024 18:26:33 +0000 Subject: [PATCH 10/19] more logs Signed-off-by: Austin Abro --- src/internal/packager/images/push.go | 3 --- src/pkg/packager/deploy.go | 10 +++++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/internal/packager/images/push.go b/src/internal/packager/images/push.go index f432d960db..1930754e45 100644 --- a/src/internal/packager/images/push.go +++ b/src/internal/packager/images/push.go @@ -10,7 +10,6 @@ import ( "github.com/avast/retry-go/v4" "github.com/google/go-containerregistry/pkg/crane" - "github.com/google/go-containerregistry/pkg/logs" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/zarf-dev/zarf/src/pkg/cluster" @@ -23,8 +22,6 @@ import ( // Push pushes images to a registry. func Push(ctx context.Context, cfg PushConfig) error { l := logger.From(ctx) - logs.Warn.SetOutput(&message.DebugWriter{}) - logs.Progress.SetOutput(&message.DebugWriter{}) toPush := map[transform.Image]v1.Image{} // Build an image list from the references diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 06dec3f6b3..2f7024bde9 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -396,11 +396,12 @@ func (p *Packager) processComponentFiles(ctx context.Context, component v1alpha1 l := logger.From(ctx) l.Handler() spinner := message.NewProgressSpinner("Copying %d files", len(component.Files)) - l.Info("Copying files", "count", len(component.Files)) + l.Info("copying files", "count", len(component.Files)) defer spinner.Stop() for fileIdx, file := range component.Files { spinner.Updatef("Loading %s", file.Target) + l.Info("loading file", "name", file.Target) fileLocation := filepath.Join(pkgLocation, strconv.Itoa(fileIdx), filepath.Base(file.Target)) if helpers.InvalidPath(fileLocation) { @@ -410,7 +411,7 @@ func (p *Packager) processComponentFiles(ctx context.Context, component v1alpha1 // If a shasum is specified check it again on deployment as well if file.Shasum != "" { spinner.Updatef("Validating SHASUM for %s", file.Target) - l.Info("Validating SHASUM", "file", file.Target) + l.Debug("Validating SHASUM", "file", file.Target) if err := helpers.SHAsMatch(fileLocation, file.Shasum); err != nil { return err } @@ -441,6 +442,7 @@ func (p *Packager) processComponentFiles(ctx context.Context, component v1alpha1 // If the file is a text file, template it if isText { spinner.Updatef("Templating %s", file.Target) + l.Debug("template file", "name", file.Target) if err := p.variableConfig.ReplaceTextTemplate(subFile); err != nil { return fmt.Errorf("unable to template file %s: %w", subFile, err) } @@ -449,6 +451,7 @@ func (p *Packager) processComponentFiles(ctx context.Context, component v1alpha1 // Copy the file to the destination spinner.Updatef("Saving %s", file.Target) + l.Debug("saving file", "name", file.Target) err = helpers.CreatePathAndCopy(fileLocation, file.Target) if err != nil { return fmt.Errorf("unable to copy file %s to %s: %w", fileLocation, file.Target, err) @@ -473,6 +476,7 @@ func (p *Packager) processComponentFiles(ctx context.Context, component v1alpha1 } spinner.Success() + l.Debug("done copying files") return nil } @@ -483,7 +487,7 @@ func (p *Packager) setupState(ctx context.Context) error { // If we are touching K8s, make sure we can talk to it once per deployment spinner := message.NewProgressSpinner("Loading the Zarf State from the Kubernetes cluster") defer spinner.Stop() - l.Info("loading the Zarf State from the Kubernetes cluster") + l.Debug("loading the Zarf State from the Kubernetes cluster") state, err := p.cluster.LoadZarfState(ctx) // We ignore the error if in YOLO mode because Zarf should not be initiated. From d04c7738a221624e9aa8a267b191ea62497fa25b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 29 Oct 2024 19:10:37 +0000 Subject: [PATCH 11/19] logs Signed-off-by: Austin Abro --- src/pkg/packager/deploy.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 2f7024bde9..744dd37483 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -137,7 +137,7 @@ func (p *Packager) Deploy(ctx context.Context) error { // Notify all the things about the successful deployment message.Successf("Zarf deployment complete") - l.Info("Zarf deployed successfully") + l.Debug("Zarf deployment complete") err = p.printTablesForDeployment(ctx, deployedComponents) if err != nil { @@ -769,6 +769,8 @@ func (p *Packager) installChartAndManifests(ctx context.Context, componentPaths return installedCharts, nil } +// TODO once packager is refactored to load the Zarf package and cluster objects in the cmd package +// printing should be moved to cmd func (p *Packager) printTablesForDeployment(ctx context.Context, componentsToDeploy []types.DeployedComponent) error { // If not init config, print the application connection table if !p.cfg.Pkg.IsInitConfig() { From 13105b9550c58936bd0899a084241ee58c8e8780 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 29 Oct 2024 19:46:12 +0000 Subject: [PATCH 12/19] adding context Signed-off-by: Austin Abro --- src/cmd/initialize.go | 7 ++++--- src/cmd/tools/zarf.go | 5 +++-- src/internal/packager2/layout/import.go | 4 ++-- src/internal/packager2/pull.go | 2 +- src/pkg/packager/common.go | 5 +++-- src/pkg/packager/composer/oci.go | 2 +- src/pkg/packager/creator/differential.go | 2 +- src/pkg/packager/creator/normal.go | 3 +-- src/pkg/packager/publish.go | 4 ++-- src/pkg/packager/sources/new.go | 4 ++-- src/pkg/packager/sources/new_test.go | 9 +++++---- src/pkg/packager/sources/tarball.go | 5 ++++- src/pkg/zoci/common.go | 13 ++++++++++--- src/test/e2e/14_oci_compose_test.go | 6 +++--- 14 files changed, 42 insertions(+), 29 deletions(-) diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index 70e73c1223..0543f97749 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -36,6 +36,7 @@ var initCmd = &cobra.Command{ Long: lang.CmdInitLong, Example: lang.CmdInitExample, RunE: func(cmd *cobra.Command, _ []string) error { + ctx := cmd.Context() if err := validateInitFlags(); err != nil { return fmt.Errorf("invalid command flags were provided: %w", err) } @@ -50,7 +51,7 @@ var initCmd = &cobra.Command{ return err } - src, err := sources.New(&pkgConfig.PkgOpts) + src, err := sources.New(ctx, &pkgConfig.PkgOpts) if err != nil { return err } @@ -65,7 +66,7 @@ var initCmd = &cobra.Command{ } defer pkgClient.ClearTempPaths() - err = pkgClient.Deploy(cmd.Context()) + err = pkgClient.Deploy(ctx) if err != nil { return err } @@ -142,7 +143,7 @@ func downloadInitPackage(ctx context.Context, cacheDirectory string) (string, er // If the user wants to download the init-package, download it if confirmDownload { - remote, err := zoci.NewRemote(url, oci.PlatformForArch(config.GetArch())) + remote, err := zoci.NewRemote(ctx, url, oci.PlatformForArch(config.GetArch())) if err != nil { return "", err } diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index 1bf56fd574..d33af49cdf 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -222,13 +222,14 @@ var downloadInitCmd = &cobra.Command{ Use: "download-init", Short: lang.CmdToolsDownloadInitShort, RunE: func(cmd *cobra.Command, _ []string) error { + ctx := cmd.Context() url := zoci.GetInitPackageURL(config.CLIVersion) - remote, err := zoci.NewRemote(url, oci.PlatformForArch(config.GetArch())) + remote, err := zoci.NewRemote(ctx, url, oci.PlatformForArch(config.GetArch())) if err != nil { return fmt.Errorf("unable to download the init package: %w", err) } source := &sources.OCISource{Remote: remote} - _, err = source.Collect(cmd.Context(), outputDirectory) + _, err = source.Collect(ctx, outputDirectory) if err != nil { return fmt.Errorf("unable to download the init package: %w", err) } diff --git a/src/internal/packager2/layout/import.go b/src/internal/packager2/layout/import.go index a1c917eee0..6c29aa31b5 100644 --- a/src/internal/packager2/layout/import.go +++ b/src/internal/packager2/layout/import.go @@ -54,7 +54,7 @@ func resolveImports(ctx context.Context, pkg v1alpha1.ZarfPackage, packagePath, return v1alpha1.ZarfPackage{}, err } } else if component.Import.URL != "" { - remote, err := zoci.NewRemote(component.Import.URL, zoci.PlatformForSkeleton()) + remote, err := zoci.NewRemote(ctx, component.Import.URL, zoci.PlatformForSkeleton()) if err != nil { return v1alpha1.ZarfPackage{}, err } @@ -166,7 +166,7 @@ func fetchOCISkeleton(ctx context.Context, component v1alpha1.ZarfComponent, pac } // Get the descriptor for the component. - remote, err := zoci.NewRemote(component.Import.URL, zoci.PlatformForSkeleton()) + remote, err := zoci.NewRemote(ctx, component.Import.URL, zoci.PlatformForSkeleton()) if err != nil { return "", err } diff --git a/src/internal/packager2/pull.go b/src/internal/packager2/pull.go index 538facc5b9..5f9dc2b5b7 100644 --- a/src/internal/packager2/pull.go +++ b/src/internal/packager2/pull.go @@ -99,7 +99,7 @@ func pullOCI(ctx context.Context, src, tarPath, shasum string, filter filters.Co src = fmt.Sprintf("%s@sha256:%s", src, shasum) } arch := config.GetArch() - remote, err := zoci.NewRemote(src, oci.PlatformForArch(arch)) + remote, err := zoci.NewRemote(ctx, src, oci.PlatformForArch(arch)) if err != nil { return false, err } diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index 442335cec0..f74a9d68c2 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -8,11 +8,12 @@ import ( "context" "errors" "fmt" - "github.com/zarf-dev/zarf/src/pkg/logger" "os" "slices" "strings" + "github.com/zarf-dev/zarf/src/pkg/logger" + "github.com/Masterminds/semver/v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -106,7 +107,7 @@ func New(cfg *types.PackagerConfig, mods ...Modifier) (*Packager, error) { // Fill the source if it wasn't provided - note source can be nil if the package is being created if pkgr.source == nil && pkgr.cfg.CreateOpts.BaseDir == "" { - pkgr.source, err = sources.New(&pkgr.cfg.PkgOpts) + pkgr.source, err = sources.New(pkgr.ctx, &pkgr.cfg.PkgOpts) if err != nil { return nil, err } diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 4d2589f717..4a49ce569f 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -28,7 +28,7 @@ func (ic *ImportChain) getRemote(ctx context.Context, url string) (*zoci.Remote, return ic.remote, nil } var err error - ic.remote, err = zoci.NewRemote(url, zoci.PlatformForSkeleton()) + ic.remote, err = zoci.NewRemote(ctx, url, zoci.PlatformForSkeleton()) if err != nil { return nil, err } diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index a373bcc4f2..d3aea389d9 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -25,7 +25,7 @@ func loadDifferentialData(ctx context.Context, diffPkgPath string) (diffData *ty diffLayout := layout.New(tmpdir) defer os.RemoveAll(diffLayout.Base) - src, err := sources.New(&types.ZarfPackageOptions{ + src, err := sources.New(ctx, &types.ZarfPackageOptions{ PackageSource: diffPkgPath, }) if err != nil { diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index d13d05e595..c97cab63c1 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -292,8 +292,7 @@ func (pc *PackageCreator) Output(ctx context.Context, dst *layout.PackagePaths, if err != nil { return err } - // TODO(mkcp): Port zoci.NewRemote to new logger - remote, err := zoci.NewRemote(ref, oci.PlatformForArch(config.GetArch())) + remote, err := zoci.NewRemote(ctx, ref, oci.PlatformForArch(config.GetArch())) if err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 95d4c66a36..34c54bd546 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -40,7 +40,7 @@ func (p *Packager) Publish(ctx context.Context) (err error) { arch := config.GetArch() - dstRemote, err := zoci.NewRemote(p.cfg.PublishOpts.PackageDestination, oci.PlatformForArch(arch)) + dstRemote, err := zoci.NewRemote(ctx, p.cfg.PublishOpts.PackageDestination, oci.PlatformForArch(arch)) if err != nil { return err } @@ -95,7 +95,7 @@ func (p *Packager) Publish(ctx context.Context) (err error) { } else { platform = oci.PlatformForArch(p.cfg.Pkg.Build.Architecture) } - remote, err := zoci.NewRemote(ref, platform) + remote, err := zoci.NewRemote(ctx, ref, platform) if err != nil { return err } diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index 65d0af6762..063901fac8 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -59,7 +59,7 @@ func Identify(pkgSrc string) string { } // New returns a new PackageSource based on the provided package options. -func New(pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { +func New(ctx context.Context, pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { var source PackageSource pkgSrc := pkgOpts.PackageSource @@ -70,7 +70,7 @@ func New(pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { pkgSrc = fmt.Sprintf("%s@sha256:%s", pkgSrc, pkgOpts.Shasum) } arch := config.GetArch() - remote, err := zoci.NewRemote(pkgSrc, oci.PlatformForArch(arch)) + remote, err := zoci.NewRemote(ctx, pkgSrc, oci.PlatformForArch(arch)) if err != nil { return nil, err } diff --git a/src/pkg/packager/sources/new_test.go b/src/pkg/packager/sources/new_test.go index 17d1481192..eb14403778 100644 --- a/src/pkg/packager/sources/new_test.go +++ b/src/pkg/packager/sources/new_test.go @@ -93,7 +93,7 @@ func TestNewPackageSource(t *testing.T) { t.Parallel() require.Equal(t, tt.expectedIdentify, Identify(tt.src)) - ps, err := New(&types.ZarfPackageOptions{PackageSource: tt.src}) + ps, err := New(context.Background(), &types.ZarfPackageOptions{PackageSource: tt.src}) require.NoError(t, err) require.IsType(t, tt.expectedType, ps) }) @@ -166,8 +166,9 @@ func TestPackageSource(t *testing.T) { PackageSource: tt.src, Shasum: tt.shasum, } + ctx := context.Background() - ps, err := New(opts) + ps, err := New(ctx, opts) require.NoError(t, err) packageDir := t.TempDir() pkgLayout := layout.New(packageDir) @@ -180,7 +181,7 @@ func TestPackageSource(t *testing.T) { require.Empty(t, warnings) require.Equal(t, expectedPkg, pkg) - ps, err = New(opts) + ps, err = New(ctx, opts) require.NoError(t, err) metadataDir := t.TempDir() metadataLayout := layout.New(metadataDir) @@ -189,7 +190,7 @@ func TestPackageSource(t *testing.T) { require.Empty(t, warnings) require.Equal(t, expectedPkg, metadata) - ps, err = New(opts) + ps, err = New(ctx, opts) require.NoError(t, err) collectDir := t.TempDir() fp, err := ps.Collect(context.Background(), collectDir) diff --git a/src/pkg/packager/sources/tarball.go b/src/pkg/packager/sources/tarball.go index e5d142b593..ac1d99c66c 100644 --- a/src/pkg/packager/sources/tarball.go +++ b/src/pkg/packager/sources/tarball.go @@ -12,6 +12,7 @@ import ( "io" "os" "path/filepath" + "time" "github.com/defenseunicorns/pkg/helpers/v2" "github.com/mholt/archiver/v3" @@ -36,9 +37,10 @@ type TarballSource struct { // LoadPackage loads a package from a tarball. func (s *TarballSource) LoadPackage(ctx context.Context, dst *layout.PackagePaths, filter filters.ComponentFilterStrategy, unarchiveAll bool) (pkg v1alpha1.ZarfPackage, warnings []string, err error) { + l := logger.From(ctx) spinner := message.NewProgressSpinner("Loading package from %q", s.PackageSource) defer spinner.Stop() - l := logger.From(ctx) + start := time.Now() l.Info("loading package", "source", s.PackageSource) if s.Shasum != "" { @@ -141,6 +143,7 @@ func (s *TarballSource) LoadPackage(ctx context.Context, dst *layout.PackagePath } spinner.Success() + l.Debug("done loading package", "source", s.PackageSource, "duration", time.Since(start)) return pkg, warnings, nil } diff --git a/src/pkg/zoci/common.go b/src/pkg/zoci/common.go index 29e9f34564..b83dd62dbe 100644 --- a/src/pkg/zoci/common.go +++ b/src/pkg/zoci/common.go @@ -5,11 +5,13 @@ package zoci import ( + "context" "log/slog" "github.com/defenseunicorns/pkg/oci" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/zarf-dev/zarf/src/config" + "github.com/zarf-dev/zarf/src/pkg/logger" "github.com/zarf-dev/zarf/src/pkg/message" ) @@ -29,12 +31,17 @@ type Remote struct { // NewRemote returns an oras remote repository client and context for the given url // with zarf opination embedded -func NewRemote(url string, platform ocispec.Platform, mods ...oci.Modifier) (*Remote, error) { - logger := slog.New(message.ZarfHandler{}) +func NewRemote(ctx context.Context, url string, platform ocispec.Platform, mods ...oci.Modifier) (*Remote, error) { + var l *slog.Logger + if logger.Enabled(ctx) { + l = logger.From(ctx) + } else { + l = slog.New(message.ZarfHandler{}) + } modifiers := append([]oci.Modifier{ oci.WithPlainHTTP(config.CommonOptions.PlainHTTP), oci.WithInsecureSkipVerify(config.CommonOptions.InsecureSkipTLSVerify), - oci.WithLogger(logger), + oci.WithLogger(l), oci.WithUserAgent("zarf/" + config.CLIVersion), }, mods...) remote, err := oci.NewOrasRemote(url, platform, modifiers...) diff --git a/src/test/e2e/14_oci_compose_test.go b/src/test/e2e/14_oci_compose_test.go index 6a983c7fc3..3f8c0a452b 100644 --- a/src/test/e2e/14_oci_compose_test.go +++ b/src/test/e2e/14_oci_compose_test.go @@ -189,18 +189,18 @@ func (suite *PublishCopySkeletonSuite) Test_3_Copy() { dstRegistry := testutil.SetupInMemoryRegistry(testutil.TestContext(t), t, 31890) dstRef := strings.Replace(ref, suite.Reference.Registry, dstRegistry, 1) + ctx := testutil.TestContext(t) - src, err := zoci.NewRemote(ref, oci.PlatformForArch(e2e.Arch), oci.WithPlainHTTP(true)) + src, err := zoci.NewRemote(ctx, ref, oci.PlatformForArch(e2e.Arch), oci.WithPlainHTTP(true)) suite.NoError(err) - dst, err := zoci.NewRemote(dstRef, oci.PlatformForArch(e2e.Arch), oci.WithPlainHTTP(true)) + dst, err := zoci.NewRemote(ctx, dstRef, oci.PlatformForArch(e2e.Arch), oci.WithPlainHTTP(true)) suite.NoError(err) reg, err := remote.NewRegistry(strings.Split(dstRef, "/")[0]) suite.NoError(err) reg.PlainHTTP = true attempt := 0 - ctx := testutil.TestContext(t) for attempt <= 5 { err = reg.Ping(ctx) if err == nil { From 122f5f176642c50308c79c467941fc878e371b39 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 29 Oct 2024 19:56:53 +0000 Subject: [PATCH 13/19] add logs for repo push Signed-off-by: Austin Abro --- src/pkg/packager/deploy.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 744dd37483..3a5928e3bd 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -582,7 +582,9 @@ func (p *Packager) pushImagesToRegistry(ctx context.Context, componentImages []s // Push all of the components git repos to the configured git server. func (p *Packager) pushReposToRepository(ctx context.Context, reposPath string, repos []string) error { + l := logger.From(ctx) for _, repoURL := range repos { + l.Info("pushing repository", "name", repoURL, "server", p.state.GitServer.Address) repository, err := git.Open(reposPath, repoURL) if err != nil { return err From 5f484039a80b9c2a4bb4e94dc0ff8b857a3c4237 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 30 Oct 2024 12:30:21 +0000 Subject: [PATCH 14/19] adding logs Signed-off-by: Austin Abro --- src/internal/packager/helm/chart.go | 2 ++ src/internal/packager/helm/utils.go | 4 +--- src/internal/packager/template/template.go | 2 ++ src/pkg/cluster/data.go | 1 + src/pkg/packager/deploy.go | 14 +++++++++----- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/internal/packager/helm/chart.go b/src/internal/packager/helm/chart.go index e8b1309d9b..5d65efd76d 100644 --- a/src/internal/packager/helm/chart.go +++ b/src/internal/packager/helm/chart.go @@ -36,6 +36,7 @@ import ( // InstallOrUpgradeChart performs a helm install of the given chart. func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings, string, error) { l := logger.From(ctx) + start := time.Now() source := h.chart.URL if source == "" { source = "Zarf-generated" @@ -149,6 +150,7 @@ func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings, } } spinner.Success() + l.Debug("done processing helm chart", "name", h.chart.Name, "duration", time.Since(start)) // return any collected connect strings for zarf connect. return postRender.connectStrings, h.chart.ReleaseName, nil diff --git a/src/internal/packager/helm/utils.go b/src/internal/packager/helm/utils.go index 0f96d2595e..d66ddd1bc4 100644 --- a/src/internal/packager/helm/utils.go +++ b/src/internal/packager/helm/utils.go @@ -73,12 +73,10 @@ func (h *Helm) createActionConfig(ctx context.Context, namespace string, spinner h.settings.SetNamespace(namespace) // Setup K8s connection - var helmLogger action.DebugLog + helmLogger := spinner.Updatef if logger.Enabled(ctx) { l := logger.From(ctx) helmLogger = slog.NewLogLogger(l.Handler(), slog.LevelDebug).Printf - } else { - helmLogger = spinner.Updatef } err := actionConfig.Init(h.settings.RESTClientGetter(), namespace, "", helmLogger) diff --git a/src/internal/packager/template/template.go b/src/internal/packager/template/template.go index 1a1daace8a..c39c6ead33 100644 --- a/src/internal/packager/template/template.go +++ b/src/internal/packager/template/template.go @@ -128,6 +128,8 @@ func generateHtpasswd(regInfo *types.RegistryInfo) (string, error) { } func debugPrintTemplateMap(ctx context.Context, templateMap map[string]*variables.TextTemplate) { + // TODO (@austinabro321) sanitize the template by making a copy and changing the actual keys + // then use json.MarshallIdent to create the json debugText := "templateMap = { " for key, template := range templateMap { diff --git a/src/pkg/cluster/data.go b/src/pkg/cluster/data.go index 81d8cdd460..f8b589dc24 100644 --- a/src/pkg/cluster/data.go +++ b/src/pkg/cluster/data.go @@ -63,6 +63,7 @@ func (c *Cluster) HandleDataInjection(ctx context.Context, data v1alpha1.ZarfDat } message.Debugf("Attempting to inject data into %s", data.Target) + l.Debug("preforming data injection", "target", data.Target) source := filepath.Join(componentPath.DataInjections, filepath.Base(data.Target.Path)) if helpers.InvalidPath(source) { diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 3a5928e3bd..6823545f11 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -58,6 +58,7 @@ func (p *Packager) resetRegistryHPA(ctx context.Context) { // Deploy attempts to deploy the given PackageConfig. func (p *Packager) Deploy(ctx context.Context) error { l := logger.From(ctx) + start := time.Now() isInteractive := !config.CommonOptions.Confirm deployFilter := filters.Combine( @@ -137,7 +138,7 @@ func (p *Packager) Deploy(ctx context.Context) error { // Notify all the things about the successful deployment message.Successf("Zarf deployment complete") - l.Debug("Zarf deployment complete") + l.Debug("Zarf deployment complete", "duration", time.Since(start)) err = p.printTablesForDeployment(ctx, deployedComponents) if err != nil { @@ -292,11 +293,12 @@ func (p *Packager) deployInitComponent(ctx context.Context, component v1alpha1.Z // Deploy a Zarf Component. func (p *Packager) deployComponent(ctx context.Context, component v1alpha1.ZarfComponent, noImgChecksum bool, noImgPush bool) ([]types.InstalledChart, error) { l := logger.From(ctx) + start := time.Now() // Toggles for general deploy operations componentPath := p.layout.Components.Dirs[component.Name] - // All components now require a name message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) + l.Info("deploying component", "name", component.Name) hasImages := len(component.Images) > 0 && !noImgPush hasCharts := len(component.Charts) > 0 @@ -388,6 +390,7 @@ func (p *Packager) deployComponent(ctx context.Context, component v1alpha1.ZarfC if err != nil { return nil, err } + l.Debug("done deploying component", "name", component.Name, "duration", time.Since(start)) return charts, nil } @@ -396,6 +399,7 @@ func (p *Packager) processComponentFiles(ctx context.Context, component v1alpha1 l := logger.From(ctx) l.Handler() spinner := message.NewProgressSpinner("Copying %d files", len(component.Files)) + start := time.Now() l.Info("copying files", "count", len(component.Files)) defer spinner.Stop() @@ -476,7 +480,7 @@ func (p *Packager) processComponentFiles(ctx context.Context, component v1alpha1 } spinner.Success() - l.Debug("done copying files") + l.Debug("done copying files", "duration", time.Since(start)) return nil } @@ -771,8 +775,8 @@ func (p *Packager) installChartAndManifests(ctx context.Context, componentPaths return installedCharts, nil } -// TODO once packager is refactored to load the Zarf package and cluster objects in the cmd package -// printing should be moved to cmd +// TODO once deploy is refactored to load the Zarf package and cluster objects in the cmd package +// table printing should be moved to cmd func (p *Packager) printTablesForDeployment(ctx context.Context, componentsToDeploy []types.DeployedComponent) error { // If not init config, print the application connection table if !p.cfg.Pkg.IsInitConfig() { From d36bb31fa223f100fa31b5b61bb7c9139a8abe45 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 30 Oct 2024 12:31:23 +0000 Subject: [PATCH 15/19] avoid merge conflicts Signed-off-by: Austin Abro --- src/pkg/cluster/state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/cluster/state.go b/src/pkg/cluster/state.go index 3744255d02..0ad4dac449 100644 --- a/src/pkg/cluster/state.go +++ b/src/pkg/cluster/state.go @@ -243,7 +243,7 @@ func (c *Cluster) debugPrintZarfState(ctx context.Context, state *types.ZarfStat return } message.Debugf("ZarfState - %s", string(b)) - logger.From(ctx).Debug("", "state", string(b)) + logger.From(ctx).Debug("cluster.debugPrintZarfState", "state", sanitized) } // SaveZarfState takes a given state and persists it to the Zarf/zarf-state secret. From 8543061d37008dbd6a5217d7e68b262b8b1fe513 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 30 Oct 2024 12:59:57 +0000 Subject: [PATCH 16/19] small style changes Signed-off-by: Austin Abro --- src/internal/packager/helm/utils.go | 1 - src/internal/packager/template/template.go | 2 +- src/pkg/zoci/common.go | 4 +--- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/internal/packager/helm/utils.go b/src/internal/packager/helm/utils.go index d66ddd1bc4..38b9e1f889 100644 --- a/src/internal/packager/helm/utils.go +++ b/src/internal/packager/helm/utils.go @@ -78,7 +78,6 @@ func (h *Helm) createActionConfig(ctx context.Context, namespace string, spinner l := logger.From(ctx) helmLogger = slog.NewLogLogger(l.Handler(), slog.LevelDebug).Printf } - err := actionConfig.Init(h.settings.RESTClientGetter(), namespace, "", helmLogger) // Set the actionConfig is the received Helm pointer diff --git a/src/internal/packager/template/template.go b/src/internal/packager/template/template.go index c39c6ead33..f069698425 100644 --- a/src/internal/packager/template/template.go +++ b/src/internal/packager/template/template.go @@ -129,7 +129,7 @@ func generateHtpasswd(regInfo *types.RegistryInfo) (string, error) { func debugPrintTemplateMap(ctx context.Context, templateMap map[string]*variables.TextTemplate) { // TODO (@austinabro321) sanitize the template by making a copy and changing the actual keys - // then use json.MarshallIdent to create the json + // then use json.MarshalIndent to create the json debugText := "templateMap = { " for key, template := range templateMap { diff --git a/src/pkg/zoci/common.go b/src/pkg/zoci/common.go index b83dd62dbe..ab8f92f3bf 100644 --- a/src/pkg/zoci/common.go +++ b/src/pkg/zoci/common.go @@ -32,11 +32,9 @@ type Remote struct { // NewRemote returns an oras remote repository client and context for the given url // with zarf opination embedded func NewRemote(ctx context.Context, url string, platform ocispec.Platform, mods ...oci.Modifier) (*Remote, error) { - var l *slog.Logger + l := slog.New(message.ZarfHandler{}) if logger.Enabled(ctx) { l = logger.From(ctx) - } else { - l = slog.New(message.ZarfHandler{}) } modifiers := append([]oci.Modifier{ oci.WithPlainHTTP(config.CommonOptions.PlainHTTP), From 31ed212c38c791d474d2518897924d42992fd254 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 30 Oct 2024 16:28:01 +0000 Subject: [PATCH 17/19] make it more clear that we use helm Signed-off-by: Austin Abro --- src/internal/packager/helm/chart.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/internal/packager/helm/chart.go b/src/internal/packager/helm/chart.go index 5714f590ae..619dc6b62a 100644 --- a/src/internal/packager/helm/chart.go +++ b/src/internal/packager/helm/chart.go @@ -82,13 +82,13 @@ func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings, if errors.Is(histErr, driver.ErrReleaseNotFound) { // No prior release, try to install it. spinner.Updatef("Attempting chart installation") - l.Info("preforming chart install") + l.Info("preforming Helm install") release, err = h.installChart(helmCtx, postRender) } else if histErr == nil && len(releases) > 0 { // Otherwise, there is a prior release so upgrade it. spinner.Updatef("Attempting chart upgrade") - l.Info("preforming chart upgrade") + l.Info("preforming Helm upgrade") lastRelease := releases[len(releases)-1] From ef2a807d725a7abb5cd65cbf332c0bb1c816a337 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 31 Oct 2024 13:36:42 +0000 Subject: [PATCH 18/19] capitalizes and things I missed Signed-off-by: Austin Abro --- src/internal/git/repository.go | 6 +++--- src/internal/packager/helm/chart.go | 8 ++++---- src/pkg/packager/deploy.go | 5 ++--- src/pkg/packager/sources/tarball.go | 4 ++-- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/internal/git/repository.go b/src/internal/git/repository.go index a0e2ac3ab6..be0215044b 100644 --- a/src/internal/git/repository.go +++ b/src/internal/git/repository.go @@ -197,13 +197,13 @@ func (r *Repository) Push(ctx context.Context, address, username, password strin err = repo.FetchContext(ctx, fetchOptions) if errors.Is(err, transport.ErrRepositoryNotFound) { message.Debugf("Repo not yet available offline, skipping fetch...") - l.Debug("Repo not yet available offline, skipping fetch") + l.Debug("repo not yet available offline, skipping fetch") } else if errors.Is(err, git.ErrForceNeeded) { message.Debugf("Repo fetch requires force, skipping fetch...") - l.Debug("Repo fetch requires force, skipping fetch") + l.Debug("repo fetch requires force, skipping fetch") } else if errors.Is(err, git.NoErrAlreadyUpToDate) { message.Debugf("Repo already up-to-date, skipping fetch...") - l.Debug("Repo already up-to-date, skipping fetch") + l.Debug("repo already up-to-date, skipping fetch") } else if err != nil { return fmt.Errorf("unable to fetch the git repo prior to push: %w", err) } diff --git a/src/internal/packager/helm/chart.go b/src/internal/packager/helm/chart.go index 619dc6b62a..b908541229 100644 --- a/src/internal/packager/helm/chart.go +++ b/src/internal/packager/helm/chart.go @@ -82,13 +82,13 @@ func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings, if errors.Is(histErr, driver.ErrReleaseNotFound) { // No prior release, try to install it. spinner.Updatef("Attempting chart installation") - l.Info("preforming Helm install") + l.Info("preforming Helm install", "chart", h.chart.Name) release, err = h.installChart(helmCtx, postRender) } else if histErr == nil && len(releases) > 0 { // Otherwise, there is a prior release so upgrade it. spinner.Updatef("Attempting chart upgrade") - l.Info("preforming Helm upgrade") + l.Info("preforming Helm upgrade", "chart", h.chart.Name) lastRelease := releases[len(releases)-1] @@ -125,7 +125,7 @@ func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings, // Attempt to rollback on a failed upgrade. spinner.Updatef("Performing chart rollback") - l.Info("preforming chart rollback") + l.Info("preforming Helm rollback", "chart", h.chart.Name) err = h.rollbackChart(h.chart.ReleaseName, previouslyDeployedVersion) if err != nil { return nil, "", fmt.Errorf("%w: unable to rollback: %w", installErr, err) @@ -145,7 +145,7 @@ func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings, if !h.chart.NoWait { // Ensure we don't go past the timeout by using a context initialized with the helm timeout spinner.Updatef("Running health checks") - l.Info("running health checks") + l.Info("running health checks", "chart", h.chart.Name) if err := healthchecks.WaitForReadyRuntime(helmCtx, h.cluster.Watcher, runtimeObjs); err != nil { return nil, "", err } diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 6823545f11..8fce3d100d 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -133,7 +133,7 @@ func (p *Packager) Deploy(ctx context.Context) error { } if len(deployedComponents) == 0 { message.Warn("No components were selected for deployment. Inspect the package to view the available components and select components interactively or by name with \"--components\"") - l.Warn("No components were selected for deployment. Inspect the package to view the available components and select components interactively or by name with \"--components\"") + l.Warn("no components were selected for deployment. Inspect the package to view the available components and select components interactively or by name with \"--components\"") } // Notify all the things about the successful deployment @@ -397,7 +397,6 @@ func (p *Packager) deployComponent(ctx context.Context, component v1alpha1.ZarfC // Move files onto the host of the machine performing the deployment. func (p *Packager) processComponentFiles(ctx context.Context, component v1alpha1.ZarfComponent, pkgLocation string) error { l := logger.From(ctx) - l.Handler() spinner := message.NewProgressSpinner("Copying %d files", len(component.Files)) start := time.Now() l.Info("copying files", "count", len(component.Files)) @@ -768,7 +767,7 @@ func (p *Packager) installChartAndManifests(ctx context.Context, componentPaths connectStrings, installedChartName, err := helmCfg.InstallOrUpgradeChart(ctx) if err != nil { return nil, err - } + } installedCharts = append(installedCharts, types.InstalledChart{Namespace: manifest.Namespace, ChartName: installedChartName, ConnectStrings: connectStrings}) } diff --git a/src/pkg/packager/sources/tarball.go b/src/pkg/packager/sources/tarball.go index ac1d99c66c..b7a00dc159 100644 --- a/src/pkg/packager/sources/tarball.go +++ b/src/pkg/packager/sources/tarball.go @@ -105,14 +105,14 @@ func (s *TarballSource) LoadPackage(ctx context.Context, dst *layout.PackagePath if !dst.IsLegacyLayout() { spinner := message.NewProgressSpinner("Validating full package checksums") defer spinner.Stop() - l.Info("validating package checksums") + l.Info("validating package checksums", "source", s.PackageSource) if err := ValidatePackageIntegrity(dst, pkg.Metadata.AggregateChecksum, false); err != nil { return pkg, nil, err } spinner.Success() - l.Debug("done validating package checksums") + l.Debug("done validating package checksums", "source", s.PackageSource) if !s.SkipSignatureValidation { if err := ValidatePackageSignature(ctx, dst, s.PublicKeyPath); err != nil { From ad895a6db469d03ba05be62b1e9ff2ce5fc514a4 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 31 Oct 2024 13:46:08 +0000 Subject: [PATCH 19/19] fix performing spelling Signed-off-by: Austin Abro --- src/internal/packager/helm/chart.go | 6 +++--- src/pkg/cluster/data.go | 2 +- src/pkg/packager/deploy.go | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/internal/packager/helm/chart.go b/src/internal/packager/helm/chart.go index b908541229..0bb12a03bb 100644 --- a/src/internal/packager/helm/chart.go +++ b/src/internal/packager/helm/chart.go @@ -82,13 +82,13 @@ func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings, if errors.Is(histErr, driver.ErrReleaseNotFound) { // No prior release, try to install it. spinner.Updatef("Attempting chart installation") - l.Info("preforming Helm install", "chart", h.chart.Name) + l.Info("performing Helm install", "chart", h.chart.Name) release, err = h.installChart(helmCtx, postRender) } else if histErr == nil && len(releases) > 0 { // Otherwise, there is a prior release so upgrade it. spinner.Updatef("Attempting chart upgrade") - l.Info("preforming Helm upgrade", "chart", h.chart.Name) + l.Info("performing Helm upgrade", "chart", h.chart.Name) lastRelease := releases[len(releases)-1] @@ -125,7 +125,7 @@ func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings, // Attempt to rollback on a failed upgrade. spinner.Updatef("Performing chart rollback") - l.Info("preforming Helm rollback", "chart", h.chart.Name) + l.Info("performing Helm rollback", "chart", h.chart.Name) err = h.rollbackChart(h.chart.ReleaseName, previouslyDeployedVersion) if err != nil { return nil, "", fmt.Errorf("%w: unable to rollback: %w", installErr, err) diff --git a/src/pkg/cluster/data.go b/src/pkg/cluster/data.go index f8b589dc24..a85f52c7ab 100644 --- a/src/pkg/cluster/data.go +++ b/src/pkg/cluster/data.go @@ -63,7 +63,7 @@ func (c *Cluster) HandleDataInjection(ctx context.Context, data v1alpha1.ZarfDat } message.Debugf("Attempting to inject data into %s", data.Target) - l.Debug("preforming data injection", "target", data.Target) + l.Debug("performing data injection", "target", data.Target) source := filepath.Join(componentPath.DataInjections, filepath.Base(data.Target.Path)) if helpers.InvalidPath(source) { diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 8fce3d100d..6ecf6cc229 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -506,7 +506,6 @@ func (p *Packager) setupState(ctx context.Context) error { // YOLO mode, so minimal state needed state.Distro = "YOLO" - // Try to create the zarf namespace spinner.Updatef("Creating the Zarf namespace") l.Info("creating the Zarf namespace") zarfNamespace := cluster.NewZarfManagedNamespace(cluster.ZarfNamespaceName)