Merge remote-tracking branch 'origin/master' into k8s-plugin

Signed-off-by: Shinnosuke Sawada-Dazai <[email protected]>

CONTRIBUTING.md

@@ -2,6 +2,7 @@
 
 If you're interested in contributing to PipeCD, this document will provide clear instructions on how to get involved.
 
+> Note: Don't bother reading policies and flows, just want to manipulate the code? Jump to the [Development](#development) section.
 
 The [Open Source Guides](https://opensource.guide/) website offers a collection of resources for individuals, communities, and companies who want to learn how to run and contribute to an open source project. Both contributors and newcomers to open source will find the following guides especially helpful:
 

cmd/pipecd/README.md

@@ -4,6 +4,7 @@
 ## Prerequisites
 
 - [Go 1.22 or later](https://go.dev/)
+- [NodeJS v20 or later](https://nodejs.org/en/)
 - [Docker](https://www.docker.com/)
 - [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) (If you want to run Control Plane locally)
 - [helm 3.8](https://helm.sh/docs/intro/install/) (If you want to run Control Plane locally)
@@ -22,7 +23,7 @@ For the full list of available commands, please see the Makefile at the root of
 
 ## How to run Control Plane locally
 
-1. Start running a Kubernetes cluster
+1. Start running a Kubernetes cluster (If you don't have any Kubernetes cluster to use)
 
     ``` console
     make kind-up
@@ -53,4 +54,4 @@ For the full list of available commands, please see the Makefile at the root of
     Point your web browser to [http://localhost:8080](http://localhost:8080) to login with the configured static admin account: project = `quickstart`, username = `hello-pipecd`, password = `hello-pipecd`.
 
 ## How to run Piped locally and add an application to your cluster
-See [How to run Piped agent locally](https://github.com/pipe-cd/pipecd/tree/master/cmd/piped#how-to-run-piped-agent-locally).
+See [How to run Piped agent locally](https://github.com/pipe-cd/pipecd/tree/master/cmd/piped#how-to-run-piped-agent-locally).

docs/content/en/docs-dev/feature-status/_index.md

@@ -67,9 +67,9 @@ Please note that the phases (Incubating, Alpha, Beta, and Stable) are applied to
 | Quick sync deployment | Beta |
 | Deployment with a defined pipeline (e.g. canary, analysis) | Beta |
 | [Automated rollback](../user-guide/managing-application/rolling-back-a-deployment/) | Beta |
-| [Automated configuration drift detection](../user-guide/managing-application/configuration-drift-detection/) | Incubating |
-| [Application live state](../user-guide/managing-application/application-live-state/) | Incubating |
-| [Plan preview](../user-guide/plan-preview) | Incubating |
+| [Automated configuration drift detection](../user-guide/managing-application/configuration-drift-detection/) | Alpha |
+| [Application live state](../user-guide/managing-application/application-live-state/) | Alpha |
+| [Plan preview](../user-guide/plan-preview) | Alpha |
 | [Manifest attachment](../user-guide/managing-application/manifest-attachment) | Alpha |
 
 ### Amazon ECS
@@ -79,14 +79,16 @@ Please note that the phases (Incubating, Alpha, Beta, and Stable) are applied to
 | Quick sync deployment | Alpha |
 | Deployment with a defined pipeline (e.g. canary, analysis) | Alpha |
 | [Automated rollback](../user-guide/managing-application/rolling-back-a-deployment/) | Beta |
-| [Automated configuration drift detection](../user-guide/managing-application/configuration-drift-detection/) | Incubating |
-| [Application live state](../user-guide/managing-application/application-live-state/) | Incubating |
+| [Automated configuration drift detection](../user-guide/managing-application/configuration-drift-detection/) | Alpha *1 |
+| [Application live state](../user-guide/managing-application/application-live-state/) | Alpha *1 |
 | Quick sync deployment for [ECS Service Discovery](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/service-discovery.html) | Alpha |
 | Deployment with a defined pipeline for [ECS Service Discovery](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/service-discovery.html) | Alpha |
 | Support [AWS App Mesh](https://aws.amazon.com/app-mesh/) | Incubating |
 | [Plan preview](../user-guide/plan-preview) | Alpha |
 | [Manifest attachment](../user-guide/managing-application/manifest-attachment) | Alpha |
 
+*1.  Not supported yet for standalone tasks.
+
 ## Piped Agent
 
 | Feature | Phase |

docs/content/en/docs-dev/installation/install-piped/required-permissions.md

@@ -26,7 +26,12 @@ You need IAM actions like the following example. You can restrict `Resource`.
                 "ecs:DeleteTaskSet",
                 "ecs:DeregisterTaskDefinition",
                 "ecs:DescribeServices",
+                "ecs:DescribeTaskDefinition",
                 "ecs:DescribeTaskSets",
+                "ecs:DescribeTasks",
+                "ecs:ListClusters",
+                "ecs:ListServices",
+                "ecs:ListTasks",
                 "ecs:RegisterTaskDefinition",
                 "ecs:RunTask",
                 "ecs:TagResource",
@@ -71,6 +76,7 @@ You need IAM actions like the following example. You can restrict `Resource`.
                 "lambda:CreateFunction",
                 "lambda:GetAlias",
                 "lambda:GetFunction",
+                "lambda:ListFunctions",
                 "lambda:PublishVersion",
                 "lambda:TagResource",
                 "lambda:UntagResource",

pkg/app/piped/controller/controller.go

@@ -138,6 +138,9 @@ type controller struct {
 	// WaitGroup for waiting the completions of all planners, schedulers.
 	wg sync.WaitGroup
 
+	// workingDirRemovalCh is used to single-threaded removal of working directory.
+	workingDirRemovalCh chan string
+
 	workspaceDir string
 	syncInternal time.Duration
 	gracePeriod  time.Duration
@@ -186,6 +189,8 @@ func NewController(
 		mostRecentlySuccessfulCommits:         make(map[string]string),
 		mostRecentlySuccessfulConfigFilenames: make(map[string]string),
 
+		workingDirRemovalCh: make(chan string),
+
 		syncInternal: 10 * time.Second,
 		gracePeriod:  gracePeriod,
 		logger:       lg,
@@ -219,6 +224,16 @@ func (c *controller) Run(ctx context.Context) error {
 		close(lpStoppedCh)
 	}()
 
+	// Start workspace cleaner.
+	// This will remove the workspace directory of the completed planner/scheduler.
+	go func() {
+		for ws := range c.workingDirRemovalCh {
+			if err := os.RemoveAll(ws); err != nil {
+				c.logger.Error("failed to remove working directory", zap.String("workDir", ws), zap.Error(err))
+			}
+		}
+	}()
+
 	ticker := time.NewTicker(c.syncInternal)
 	defer ticker.Stop()
 	c.logger.Info("start syncing planners and schedulers")
@@ -246,10 +261,19 @@ func (c *controller) shutdown(cancel func(), stoppedCh <-chan error) error {
 	cancel()
 	err := <-stoppedCh
 
+	// Stop the workspace cleaner.
+	// all calls of removeWorkingDir is completed because the c.wg.Wait() is done.
+	// so we can close the channel safely.
+	close(c.workingDirRemovalCh)
+
 	c.logger.Info("controller has been stopped")
 	return err
 }
 
+func (c *controller) removeWorkingDir(ws string) {
+	c.workingDirRemovalCh <- ws
+}
+
 // checkCommands lists all unhandled commands for running deployments
 // and forwards them to their planners and schedulers.
 func (c *controller) checkCommands() {
@@ -483,10 +507,8 @@ func (c *controller) startNewPlanner(ctx context.Context, d *model.Deployment) (
 	)
 
 	cleanup := func() {
-		logger.Info("cleaning up working directory for planner")
-		if err := os.RemoveAll(workingDir); err != nil {
-			logger.Warn("failed to clean working directory", zap.Error(err))
-		}
+		c.removeWorkingDir(workingDir)
+		logger.Info("cleaned working directory for planner")
 	}
 
 	// Start running planner.
@@ -627,15 +649,8 @@ func (c *controller) startNewScheduler(ctx context.Context, d *model.Deployment)
 	)
 
 	cleanup := func() {
-		logger.Info("cleaning up working directory for scheduler", zap.String("working-dir", workingDir))
-		err := os.RemoveAll(workingDir)
-		if err == nil {
-			return
-		}
-		logger.Warn("failed to clean working directory",
-			zap.String("working-dir", workingDir),
-			zap.Error(err),
-		)
+		c.removeWorkingDir(workingDir)
+		logger.Info("cleaned working directory for scheduler", zap.String("working-dir", workingDir))
 	}
 
 	// Start running scheduler.

pkg/app/piped/deploysource/deploysource.go

@@ -90,7 +90,7 @@ func (p *provider) Get(ctx context.Context, lw io.Writer) (*DeploySource, error)
 
 	if !p.done {
 		p.source, p.err = p.prepare(ctx, lw)
-		p.done = true
+		p.done = p.err == nil // If there is an error, we should re-prepare it next time.
 	}
 
 	if p.err != nil {
@@ -114,7 +114,7 @@ func (p *provider) GetReadOnly(ctx context.Context, lw io.Writer) (*DeploySource
 
 	if !p.done {
 		p.source, p.err = p.prepare(ctx, lw)
-		p.done = true
+		p.done = p.err == nil // If there is an error, we should re-prepare it next time.
 	}
 
 	if p.err != nil {

pkg/app/piped/executor/kubernetes/rollback.go

@@ -195,6 +195,13 @@ func (e *rollbackExecutor) ensureScriptRunRollback(ctx context.Context) model.St
 		return model.StageStatus_STAGE_SUCCESS
 	}
 
+	ds, err := e.TargetDSP.Get(ctx, e.LogPersister)
+	if err != nil {
+		e.LogPersister.Errorf("Failed to prepare target deploy source data (%v)", err)
+		return model.StageStatus_STAGE_FAILURE
+	}
+	e.appDir = ds.AppDir
+
 	envStr, ok := e.Stage.Metadata["env"]
 	env := make(map[string]string, 0)
 	if ok {

pkg/app/pipedv1/deploysource/deploysource.go

@@ -89,7 +89,7 @@ func (p *provider) Get(ctx context.Context, lw io.Writer) (*DeploySource, error)
 
 	if !p.done {
 		p.source, p.err = p.prepare(ctx, lw)
-		p.done = true
+		p.done = p.err == nil // If there is an error, we should re-prepare it next time.
 	}
 
 	if p.err != nil {