Merge branch 'flyteorg:master' into cicd-doc-update

charts/flyte-binary/README.md

@@ -116,6 +116,8 @@ Chart for basic single Flyte executable deployment
 | flyte-core-components.admin.disableClusterResourceManager | bool | `false` |  |
 | flyte-core-components.admin.disableScheduler | bool | `false` |  |
 | flyte-core-components.admin.disabled | bool | `false` |  |
+| flyte-core-components.admin.seedProjectsWithDetails[0].description | string | `"Default project setup."` |  |
+| flyte-core-components.admin.seedProjectsWithDetails[0].name | string | `"flytesnacks"` |  |
 | flyte-core-components.admin.seedProjects[0] | string | `"flytesnacks"` |  |
 | flyte-core-components.dataCatalog.disabled | bool | `false` |  |
 | flyte-core-components.propeller.disableWebhook | bool | `false` |  |

charts/flyte-binary/values.yaml

@@ -20,6 +20,14 @@ flyte-core-components:
     # seedProjects flyte projects to create by default
     seedProjects:
       - flytesnacks
+    # seedProjectsWithDetails flyte projects to create by default with description
+    # If there is an overlap between seedProjects and seedProjectsWithDetails,
+    # the description provided in seedProjectsWithDetails will take precedence.
+    # For seedProjects without a corresponding description in seedProjectsWithDetails,
+    # a default description will be auto-generated for the project.
+    seedProjectsWithDetails:
+      - name: flytesnacks
+        description: Default project setup.
   # propeller Configuration to disable propeller or any of its components
   propeller:
     # disabled Disables flytepropeller

cmd/single/config.go

@@ -1,6 +1,9 @@
 package single
 
-import "github.com/flyteorg/flyte/flytestdlib/config"
+import (
+	adminRepositoriesConfig "github.com/flyteorg/flyte/flyteadmin/pkg/repositories/config"
+	"github.com/flyteorg/flyte/flytestdlib/config"
+)
 
 //go:generate pflags Config --default-var=DefaultConfig
 
@@ -21,10 +24,11 @@ type Propeller struct {
 }
 
 type Admin struct {
-	Disabled                      bool     `json:"disabled" pflag:",Disables flyteadmin in the single binary mode"`
-	DisableScheduler              bool     `json:"disableScheduler" pflag:",Disables Native scheduler in the single binary mode"`
-	DisableClusterResourceManager bool     `json:"disableClusterResourceManager" pflag:",Disables Cluster resource manager"`
-	SeedProjects                  []string `json:"seedProjects" pflag:",flyte projects to create by default."`
+	Disabled                      bool                                  `json:"disabled" pflag:",Disables flyteadmin in the single binary mode"`
+	DisableScheduler              bool                                  `json:"disableScheduler" pflag:",Disables Native scheduler in the single binary mode"`
+	DisableClusterResourceManager bool                                  `json:"disableClusterResourceManager" pflag:",Disables Cluster resource manager"`
+	SeedProjects                  []string                              `json:"seedProjects" pflag:",flyte projects to create by default."`
+	SeedProjectsWithDetails       []adminRepositoriesConfig.SeedProject `json:"seedProjectsWithDetails" pflag:",,Detailed configuration for Flyte projects to be created by default."`
 }
 
 type DataCatalog struct {

cmd/single/start.go

@@ -22,6 +22,7 @@ import (
 	datacatalog "github.com/flyteorg/flyte/datacatalog/pkg/rpc/datacatalogservice"
 	"github.com/flyteorg/flyte/flyteadmin/pkg/clusterresource"
 	"github.com/flyteorg/flyte/flyteadmin/pkg/common"
+	adminRepositoriesConfig "github.com/flyteorg/flyte/flyteadmin/pkg/repositories/config"
 	"github.com/flyteorg/flyte/flyteadmin/pkg/runtime"
 	adminServer "github.com/flyteorg/flyte/flyteadmin/pkg/server"
 	"github.com/flyteorg/flyte/flyteadmin/plugins"
@@ -75,8 +76,9 @@ func startAdmin(ctx context.Context, cfg Admin) error {
 	if len(cfg.SeedProjects) != 0 {
 		projects = cfg.SeedProjects
 	}
-	logger.Infof(ctx, "Seeding default projects...", projects)
-	if err := adminServer.SeedProjects(ctx, projects); err != nil {
+	seedProjects := adminRepositoriesConfig.MergeSeedProjectsWithUniqueNames(projects, cfg.SeedProjectsWithDetails)
+	logger.Infof(ctx, "Seeding default projects... %v", seedProjects)
+	if err := adminServer.SeedProjects(ctx, seedProjects); err != nil {
 		return err
 	}
 

deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml

@@ -92,6 +92,9 @@ data:
         disabled: false
         seedProjects:
         - flytesnacks
+        seedProjectsWithDetails:
+        - description: Default project setup.
+          name: flytesnacks
       dataCatalog:
         disabled: false
       propeller:
@@ -360,7 +363,7 @@ spec:
         app.kubernetes.io/instance: flyte
         app.kubernetes.io/component: flyte-binary
       annotations:
-        checksum/configuration: faaefbd3b3b2ddfd4e718bd77c02c632c75e7111dad0a6e25dc415dc88add73f
+        checksum/configuration: 886440a42b3eeec802cfe60d37885f69e35ffd83e53e625b3c877da5e8c7eb38
         checksum/configuration-secret: d5d93f4e67780b21593dc3799f0f6682aab0765e708e4020939975d14d44f929
         checksum/cluster-resource-templates: 7dfa59f3d447e9c099b8f8ffad3af466fecbc9cf9f8c97295d9634254a55d4ae
     spec:

docs/deployment/configuration/monitoring.rst

@@ -5,7 +5,7 @@ Monitoring
 
 .. tags:: Infrastructure, Advanced
 
-.. tip:: The Flyte core team publishes and maintains Grafana dashboards built using Prometheus data sources, which can be found `here <https://grafana.com/grafana/dashboards?search=flyte>`__.
+.. tip:: The Flyte core team publishes and maintains Grafana dashboards built using Prometheus data sources. You can import them to your Grafana instance from the `Grafana marketplace <https://grafana.com/orgs/flyteorg/dashboards>`__.
 
 Metrics for Executions
 ======================
@@ -87,53 +87,81 @@ Flyte Backend is written in Golang and exposes stats using Prometheus. The stats
 
 Both ``flyteadmin`` and ``flytepropeller`` are instrumented to expose metrics. To visualize these metrics, Flyte provides three Grafana dashboards, each with a different focus:
 
-- **User-facing dashboards**: Dashboards that can be used to triage/investigate/observe performance and characteristics of workflows and tasks.
-  The user-facing dashboard is published under ID `13980 <https://grafana.com/grafana/dashboards/13980>`__ in the Grafana marketplace.
+- **User-facing dashboard**: it can be used to investigate performance and characteristics of workflow and task executions. It's published under ID `22146 <https://grafana.com/grafana/dashboards/22146-flyte-user-dashboard-via-prometheus/>`__ in the Grafana marketplace.
 
 - **System Dashboards**: Dashboards that are useful for the system maintainer to investigate the status and performance of their Flyte deployments. These are further divided into:
-        - `DataPlane/FlytePropeller <https://grafana.com/grafana/dashboards/13979>`__: execution engine status and performance.
-        - `ControlPlane/Flyteadmin <https://grafana.com/grafana/dashboards/13981>`__: API-level monitoring.
+        - Data plane (``flytepropeller``) - `21719 <https://grafana.com/grafana/dashboards/21719-flyte-propeller-dashboard-via-prometheus/>`__: execution engine status and performance.
+        - Control plane (``flyteadmin``) - `21720 <https://grafana.com/grafana/dashboards/21720-flyteadmin-dashboard-via-prometheus/>`__: API-level monitoring.
 
-The corresponding JSON files for each dashboard are also located at ``deployment/stats/prometheus``.
+The corresponding JSON files for each dashboard are also located in the ``flyte`` repository at `deployment/stats/prometheus <https://github.com/flyteorg/flyte/tree/master/deployment/stats/prometheus>`__.
 
 .. note::
 
     The dashboards are basic dashboards and do not include all the metrics exposed by Flyte.
     Feel free to use the scripts provided `here <https://github.com/flyteorg/flyte/tree/master/stats>`__ to improve and -hopefully- contribute the improved dashboards.
 
-How to use the dashboards
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-1. We recommend installing and configuring the Prometheus operator as described in `their docs <https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md>`__.
-This is especially true if you plan to use the Service Monitors provided by the `flyte-core <https://github.com/flyteorg/flyte/blob/master/charts/flyte-core/templates/propeller/service-monitor.yaml>`__ Helm chart.
-
-2. Enable the Prometheus instance to use Service Monitors in the namespace where Flyte is running, configuring the following keys in the ``prometheus`` resource:
-
-.. code-block:: yaml
-
-   spec:
-    serviceMonitorSelector: {}
-    serviceMonitorNamespaceSelector: {}
-
-.. note::
-
-   The above example configuration lets Prometheus use any ``ServiceMonitor`` in any namespace in the cluster. Adjust the configuration to reduce the scope if needed.
-
-3. Once you have installed and configured the Prometheus operator, enable the Service Monitors in the Helm chart by configuring the following keys in your ``values`` file:
-
-.. code-block:: yaml
-
-   flyteadmin:
-     serviceMonitor:
-       enabled: true
-   
-   flytepropeller:
-     serviceMonitor:
-       enabled: true
-
+Setup instructions
+~~~~~~~~~~~~~~~~~~
+
+The dashboards rely on a working Prometheus deployment with access to your Kubernetes cluster and Flyte pods.
+Additionally, the user dashboard uses metrics that come from ``kube-state-metrics``. Both of these requirements can be fulfilled by installing the `kube-prometheus-stack <https://github.com/kubernetes/kube-state-metrics>`__.
+
+Once the prerequisites are in place, follow the instructions in this section to configure metrics scraping for the corresponding Helm chart:
+
+.. tabs::
+
+   .. group-tab:: flyte-core
+
+      Save the following in a ``flyte-monitoring-overrides.yaml`` file and run a ``helm upgrade`` operation pointing to that ``--values`` file:
+
+      .. code-block:: yaml
+
+         flyteadmin:
+           serviceMonitor:
+           enabled: true
+           labels:
+             release: kube-prometheus-stack #This is particular to the kube-prometheus-stacl
+           selectorLabels:
+             - app.kubernetes.io/name: flyteadmin
+         flytepropeller:
+           serviceMonitor:
+             enabled: true
+             labels:
+               release: kube-prometheus-stack
+             selectorLabels:
+               - app.kubernetes.io/name: flytepropeller
+           service:
+             enabled: true
+
+      The above configuration enables the ``serviceMonitor`` that Prometheus can then use to automatically discover services and scrape metrics from them.
+
+   .. group-tab:: flyte-binary
+
+      Save the following in a ``flyte-monitoring-overrides.yaml`` file and run a ``helm upgrade`` operation pointing to that ``--values`` file:
+
+      .. code-block:: yaml
+
+         configuration:
+           inline:
+             propeller:
+               prof-port: 10254
+               metrics-prefix: "flyte:"
+             scheduler:
+               profilerPort: 10254
+               metricsScope: "flyte:"
+             flyteadmin:
+               profilerPort: 10254
+         service:
+           extraPorts:
+           - name: http-metrics
+             protocol: TCP
+             port: 10254
+
+      The above configuration enables the ``serviceMonitor`` that Prometheus can then use to automatically discover services and scrape metrics from them.
+
 .. note::
 
    By default, the ``ServiceMonitor`` is configured with a ``scrapeTimeout`` of 30s and ``interval`` of 60s. You can customize these values if needed.
 
-With the above configuration in place you should be able to import the dashboards in your Grafana instance.
+With the above configuration completed, you should be able to import the dashboards in your Grafana instance.
 

flyteadmin/cmd/entrypoints/migrate.go

@@ -6,6 +6,7 @@ import (
 	"github.com/spf13/cobra"
 	_ "gorm.io/driver/postgres" // Required to import database driver.
 
+	"github.com/flyteorg/flyte/flyteadmin/pkg/repositories/config"
 	"github.com/flyteorg/flyte/flyteadmin/pkg/server"
 )
 
@@ -40,7 +41,8 @@ var seedProjectsCmd = &cobra.Command{
 	Short: "Seed projects in the database.",
 	RunE: func(cmd *cobra.Command, args []string) error {
 		ctx := context.Background()
-		return server.SeedProjects(ctx, args)
+		seedProjects := config.UniqueProjectsFromNames(args)
+		return server.SeedProjects(ctx, seedProjects)
 	},
 }
 

flyteadmin/pkg/repositories/config/seed_data.go

@@ -10,16 +10,61 @@ import (
 	"github.com/flyteorg/flyte/flytestdlib/logger"
 )
 
+type SeedProject struct {
+	Name        string `json:"name" pflag:",Name of flyte project to create"`
+	Description string `json:"description" pflag:",Description of flyte project to create"`
+}
+
+func UniqueProjectsFromNames(names []string) []SeedProject {
+	return uniqueProjects(names, nil)
+}
+
+// MergeSeedProjectsWithUniqueNames merges seed projects from names and details while maintaining uniqueness
+func MergeSeedProjectsWithUniqueNames(seedProjects []string, seedProjectsWithDetails []SeedProject) []SeedProject {
+	return uniqueProjects(seedProjects, seedProjectsWithDetails)
+}
+
+func uniqueProjects(seedProjects []string, seedProjectsWithDetails []SeedProject) []SeedProject {
+	// Track unique project names
+	seen := make(map[string]struct{})
+
+	// Create the final result slice
+	var combinedProjects []SeedProject
+
+	// First, add all projects from SeedProjectsWithDetails to the map
+	for _, project := range seedProjectsWithDetails {
+		// Handle the duplication
+		if _, exists := seen[project.Name]; !exists {
+			seen[project.Name] = struct{}{}
+			combinedProjects = append(combinedProjects, project)
+		}
+	}
+
+	// Process SeedProjects
+	for _, projectName := range seedProjects {
+		// Check if project not exists in SeedProjectsWithDetails
+		if _, exists := seen[projectName]; !exists {
+			seen[projectName] = struct{}{}
+			combinedProjects = append(combinedProjects, SeedProject{
+				Name:        projectName,
+				Description: fmt.Sprintf("%s description", projectName),
+			})
+		}
+	}
+
+	return combinedProjects
+}
+
 // Returns a function to seed the database with default values.
-func SeedProjects(db *gorm.DB, projects []string) error {
+func SeedProjects(db *gorm.DB, projects []SeedProject) error {
 	tx := db.Begin()
 	for _, project := range projects {
 		projectModel := models.Project{
-			Identifier:  project,
-			Name:        project,
-			Description: fmt.Sprintf("%s description", project),
+			Identifier:  project.Name,
+			Name:        project.Name,
+			Description: project.Description,
 		}
-		if err := tx.Where(models.Project{Identifier: project}).Omit("id").FirstOrCreate(&projectModel).Error; err != nil {
+		if err := tx.Where(models.Project{Identifier: project.Name}).Omit("id").FirstOrCreate(&projectModel).Error; err != nil {
 			logger.Warningf(context.Background(), "failed to save project [%s]", project)
 			tx.Rollback()
 			return err