From ab59008830745bc408f42b3bfea557468da1eef8 Mon Sep 17 00:00:00 2001 From: ppalmes Date: Sun, 12 May 2024 22:49:48 +0100 Subject: [PATCH] argo install Signed-off-by: ppalmes --- ...01.install_cluster_influxdb_persistence.sh | 185 +++++++++++++++++ e2eai/02.install_metallb.sh | 57 +++++ e2eai/03.0.install-minio.sh | 42 ++++ e2eai/03.1.buggy-install-helm-minio.sh | 44 ++++ e2eai/04.0.install_argocd+minio.sh | 116 +++++++++++ e2eai/04.1.install_argo+minio.sh | 15 ++ e2eai/05.install_argo_workflows.sh | 118 +++++++++++ e2eai/05.install_argo_workflows_helm.sh | 195 ++++++++++++++++++ e2eai/twoblocks.jl | 128 ++++++++++++ 9 files changed, 900 insertions(+) create mode 100644 e2eai/01.install_cluster_influxdb_persistence.sh create mode 100644 e2eai/02.install_metallb.sh create mode 100644 e2eai/03.0.install-minio.sh create mode 100644 e2eai/03.1.buggy-install-helm-minio.sh create mode 100644 e2eai/04.0.install_argocd+minio.sh create mode 100644 e2eai/04.1.install_argo+minio.sh create mode 100644 e2eai/05.install_argo_workflows.sh create mode 100644 e2eai/05.install_argo_workflows_helm.sh create mode 100644 e2eai/twoblocks.jl diff --git a/e2eai/01.install_cluster_influxdb_persistence.sh b/e2eai/01.install_cluster_influxdb_persistence.sh new file mode 100644 index 0000000..37c87e3 --- /dev/null +++ b/e2eai/01.install_cluster_influxdb_persistence.sh @@ -0,0 +1,185 @@ +# 1. run kubespray to start a cluster: ./kubespray/install_k8s.sh or +1. install k0s at ./k0sctl/run.sh + +2. enable pvc/pv/storageclass using nfs-provisioner plugin and make it default storage class + +kubectl create namespace persistence + +helm -n persistence repo add \ + nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/ + +# for kube1 +helm -n persistence install nfs-subdir-external-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \ + --set nfs.server=10.242.7.29 --set nfs.path=/mnt/disk2/common + +# for mccojhub +helm -n persistence install nfs-subdir-external-provisioner \ + nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \ + --set nfs.server=10.38.225.120 --set nfs.path=/mnt/diskxvdc/nfsprovisioner + +# for mccoeu +helm -n persistence install nfs-subdir-external-provisioner \ + nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \ + --set nfs.server=10.242.7.227 --set nfs.path=/mnt/diskxvdc/nfsprovisioner + +kubectl -n persistence get pv,pvc,replicaset,deployment +kubectl -n persistence create -f ./nfs-provisioner/test-claim.yaml -f ./nfs-provisioner/test-pod.yaml +kubectl -n persistence get pv,pvc,replicaset,deployment +kubectl -n persistence create -f ./nfs-provisioner/test-another-pod.yaml +kubectl -n persistence delete -f ./nfs-provisioner/test-pod.yaml -f ./nfs-provisioner/test-another-pod.yaml -f ./nfs-provisioner/test-claim.yaml + +3. create as default storageclass +kubectl patch storageclass nfs-client -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' +kb -n persistence get storageclass + +4. install jupyterhub (optional) +# install jupyterhub +kubectl create namespace jhub + +helm repo add jupyterhub https://hub.jupyter.org/helm-chart/ +helm repo update +helm search repo jupyterhub/jupyterhub +helm pull jupyterhub/jupyterhub + +helm upgrade \ + --install jhubr jupyterhub/jupyterhub \ + --namespace jhub \ + --create-namespace \ + --version="3.2.1" \ + --values values.yaml + +# delete jupyterhub +# helm -n jhub delete jhubr + +5. install prometheus (must be done in the remote control-plane) + - cd kube-devsandbox + - change storageclass to nfs-client + +# ssh to mccoeu1 +ssh root@mccoeu1.sl.cloud9.ibm.com +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +kubectl create ns prometheus +#helm install prometheus-stack prometheus-community/kube-prometheus-stack --values ./manifests/monitoring/prom-config.yml -n prometheus +helm pull prometheus-community/kube-prometheus-stack +# edit ./kube-prometheus-stack/values.yaml +helm install prometheus-stack prometheus-community/kube-prometheus-stack --values ./kube-prometheus-stack/values.yaml -n prometheus + +# validate +kubectl --namespace prometheus get pods -l "release=prometheus-stack" +kubectl -n prometheus patch svc prometheus-stack-kube-prom-prometheus -p '{"spec": {"type": "NodePort"}}' +kubectl -n prometheus patch svc prometheus-stack-grafana -p '{"spec": {"type": "NodePort"}}' +kubectl -n prometheus patch svc prometheus-stack-kube-state-metrics -p '{"spec": {"type": "NodePort"}}' + +7. install influxdb +kb create ns influxdb2 +helm repo add influxdata https://helm.influxdata.com +helm search repo influxdata + +helm -n influxdb2 upgrade --install influxdb2 \ + --set persistence.enabled=true,persistence.size=200Gi\ + influxdata/influxdb2 + +# make it default +kubectl patch storageclass nfs-client -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' +kubectl get storageclass + +kb get svc -n influxdb2 +kubectl -n influxdb2 patch svc influxdb2 -p '{"spec": {"type": "NodePort"}}' + +echo $(kubectl get secret influxdb2-auth -o "jsonpath={.data['admin-password']}" --namespace influxdb2 | base64 --decode) + +kb -n influxdb2 get svc +kb -n influxdb2 get all + +prom-scraper: http://mccoeu1.sl.cloud9.ibm.com:31194/metrics + +#helm -n influxdb2 delete influxdb2 +#kb delete ns influxdb2 + +# telegraf to remote write to influxdb from prometheus +https://www.influxdata.com/blog/prometheus-remote-write-support-with-influxdb-2-0/ + +#export INFLUX_TOKEN=G0w5c7PiGgyOIV1-xEXaGZd5f2WDs2QaNnkWLu3x57_xQjUZd72xAVG7hKJnXpxwEPPLYNruhp1gyq8wR_lcTg== +#telegraf --config http://mccoeu1.sl.cloud9.ibm.com:30958/api/v2/telegrafs/0b2dbf9cbcd27000 + +8. Install the Latest Telegraf + +You can install the latest Telegraf by visiting the InfluxData Downloads page. If you already have Telegraf installed on your system, make sure it's up to date. You will need version 1.9.2 or higher. +https://portal.influxdata.com/downloads/ + +wget -q https://repos.influxdata.com/influxdata-archive_compat.key +echo '393e8779c89ac8d958f81f942f9ad7fb82a25e133faddaf92e15b16e6ac9ce4c influxdata-archive_compat.key' | sha256sum -c && cat influxdata-archive_compat.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg > /dev/null +echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg] https://repos.influxdata.com/debian stable main' | sudo tee /etc/apt/sources.list.d/influxdata.list + +sudo apt-get update && sudo apt-get install telegraf + +8.1 Configure your API Token + +Your API token is required for pushing data into InfluxDB. You can copy the following command to your terminal window to set an environment variable with your API token. + +# for mccoeu1 + +ssh root@mccoeu1.sl.cloud9.ibm.com +download the config from influxdb telegraf and copy to /etc/telegraf/telegraf.d + +export INFLUX_TOKEN=KuIqTk01Z93vJiWD8gUWJfodt3P5n9ihvE107pnt6-Nmm4gX4ww3w_vMdfdaGbznQCar1RkQhelcs6iBqhepLg== +telegraf --config /etc/telegraf/telegraf.conf --config-directory /etc/telegraf/telegraf.d + +8.2 Start Telegraf: for influxdb2 instruction + +# for mccoeu1 +# download the created telegraf config from influxdb and copy to telegraf.d/ +ssh mcceoeu1.sl.cloud9.ibm.com +vi /etc/telegraf/telegraf.d/prom-telegraf.conf + + [[inputs.prometheus]] + urls = ["http://mccoeu1.sl.cloud9.ibm.com:31143/api/cart/metrics", "http://mccoeu1.sl.cloud9.ibm.com:31143/api/payment/metrics","http://mccoeu1.sl.cloud9.ibm.com:32496/metrics"] + metric_version = 2 + + [[outputs.influxdb_v2]] + urls = ["http://mccoeu1.sl.cloud9.ibm.com:30958"] + token = "G0w5c7PiGgyOIV1-xEXaGZd5f2WDs2QaNnkWLu3x57_xQjUZd72xAVG7hKJnXpxwEPPLYNruhp1gyq8wR_lcTg==" + organization = "influxdata" + bucket = "prometheus" + +Finally, you can run the following command to start the Telegraf agent running on your machine. + +ssh mcceoeu1.sl.cloud9.ibm.com +nohup telegraf --config /etc/telegraf/telegraf.conf --config-directory /etc/telegraf/telegraf.d & + +#telegraf --config http://mccoeu1.sl.cloud9.ibm.com:30958/api/v2/telegrafs/0b2dbf9cbcd27000 + +# for mccosmall +# download the created telegraf config from influxdb and copy to telegraf.d/ +export INFLUX_TOKEN=3Qbra2TGkkW09GMwXn06QQox8ZdUoCexwvYqMeGvtqtUpBiVP6uoQ_7EkZcZ_nw9O507F8Y3dDahWZd9ujBf-A== + + [[inputs.prometheus]] + urls = ["http://mccosmall1.sl.cloud9.ibm.com:32196/api/cart/metrics", "http://mccosmall1.sl.cloud9.ibm.com:32196/api/payment/metrics","http://mccosmall1.sl.cloud9.ibm.com:32509/metrics"] + metric_version = 2 + +[[outputs.influxdb_v2]] +urls = ["http://mccosmall1.sl.cloud9.ibm.com:32347"] +token = "$INFLUX_TOKEN" +organization = "influxdata" +bucket = "prometheus" + +ssh mcceoeu1.sl.cloud9.ibm.com +nohup telegraf --config /etc/telegraf/telegraf.conf --config-directory /etc/telegraf/telegraf.d & + +# 8. update prometheus values.yaml: old influxdb instruction +# +# # ssh to mccoeu1 +# ssh mccoeu1.sl.clou9.ibm.com +# +# add these: this is old version of influxdb +# +# scrape_configs: +# - job_name: 'prometheus' +# static_configs: +# - targets: ['http://mccoeu1.sl.cloud9.ibm.com:31025'] +# +# remote_write: +# - url: "http://mccoeu1.sl.cloud9.ibm.com:8080/receive" +# +# helm upgrade --install prometheus-stack prometheus-community/kube-prometheus-stack --values ./manifests/monitoring/prom-config.yml -n observability diff --git a/e2eai/02.install_metallb.sh b/e2eai/02.install_metallb.sh new file mode 100644 index 0000000..e75dc4b --- /dev/null +++ b/e2eai/02.install_metallb.sh @@ -0,0 +1,57 @@ +# refs: https://metallb.universe.tf/ +# kubectl edit configmap -n kube-system kube-proxy + +# ClusterIP vs NodePort vs LoadBalancer +# - every pod has a private IP +# - a service is a static IP +# - ClusterIP is an internal service +# - NodePort port-forward from outside to ClusterIP service +# - a Loadbalancer performs dynamic port-forwarding from outside to ClusterIP service + +kubectl create ns metallb-system + +# install by manifest +kubectl apply -f metallb.yaml + +# install by helm +# helm repo add metallb https://metallb.github.io/metallb +# helm install metallb metallb/metallb + +# configure using layer 2 +# addresses based on the cluster nodes network range + +cat < kubectl -n argocd port-forward svc/argocd-server 8080:80 +#> argocd login localhost:8080 +# +# or use loadbalancer to login +kubectl -n argocd patch svc argocd-server -p '{"spec": {"type": "LoadBalancer"}}' +# +argo_ip=$(kubectl -n argocd get svc | grep argocd-server | grep LoadBalancer|tr -s " " ","|cut -d"," -f4) +initial_password=$(argocd admin initial-password -n argocd) +current_context=$(kubectl config current-context) +argocd login ${argo_ip} --username admin --password ${initial_password} --insecure +argocd cluster add ${current_context} --in-cluster --upsert +# +# and update the password +argocd account update-password + +#-- Create minio-operator +kubectl create namespace minio-operator +argocd app create minio-operator \ + --repo https://github.com/cniackz/minio-argocd.git \ + --path minio-operator \ + --dest-namespace minio-operator \ + --dest-server https://kubernetes.default.svc --insecure --upsert + +#-- Login to the svc by LoadBalancer or NodePort +# source: https://min.io/docs/minio/kubernetes/upstream/operations/install-deploy-manage/minio-operator-console.html +kubectl -n minio-operator patch svc console -p '{"spec": {"type": "LoadBalancer"}}' +jwtoken=$(kubectl get secret/console-sa-secret -n minio-operator -o json | jq -r '.data.token' | base64 -d) +# or jwtoken=$(kubectl -n minio-operator get secret console-sa-secret -o jsonpath='{.data.token}' | base64 --decode) +minio_operator=$(kubectl -n minio-operator get svc| grep console | grep LoadBalancer|tr -s " " ","|cut -d"," -f4) +open $(echo http://${minio_operator}:9090) +echo $jwtoken +# +# or login by proxy +#> kubectl minio proxy +#> open http://localhost:9090 + +#-- Create a tenant via argocd +kubectl create namespace minio-tenant +argocd app create minio-tenant \ + --repo https://github.com/cniackz/minio-argocd.git \ + --path minio-tenant \ + --dest-namespace minio-tenant \ + --dest-server https://kubernetes.default.svc \ + --insecure \ + --upsert +# +# Note: this is the orriginal helm install of tenant outside argocd +#> helm search repo minio +#> helm install --namespace minio-tenant \ +#> --create-namespace tenant minio-operator/tenant + +#-- Access tenant console +# +# using port-forward +#> kubectl --namespace minio-tenant port-forward svc/myminio-console 9443:9443 +# +# or using LoadBalancer to access +kubectl -n minio-tenant patch svc myminio-console -p '{"spec": {"type": "LoadBalancer"}}' +minio_tenant=$(kubectl -n minio-tenant get svc| grep myminio-console | grep LoadBalancer|tr -s " " ","|cut -d"," -f4) +# +# values.yaml info of password +# secrets: +# name: myminio-env-configuration +# # MinIO root user and password +# accessKey: minio +# secretKey: minio123 +# ## MinIO Tenant Definition +# info from https://github.com/cniackz/minio-argocd/blob/main/minio-tenant/values.yaml +# user: minio, password: minio123 +# +export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" +open $(echo https://${minio_tenant}:9443) + + +# Setting up Artifact Bucket Credentials +# +# connect argo with minio: https://medium.com/@michael.cook.talk/argo-workflows-minio-nginx-8911b988b5c8 +# create bucket in minio console +# create Access Key in minio console +# Access Key: FxcWDBqCHApF1HuF +echo -n FxcWDBqCHApF1HuF | base64 +# Secret Key: DshXCJ42dlSq6PgwTxCEwkRB09nywo3K +echo -n DshXCJ42dlSq6PgwTxCEwkRB09nywo3K | base64 +# write these base64 values into artifact-bucket-credential.yaml + +ACCESS_KEY=FxcWDBqCHApF1HuF +SECRET_KEY=DshXCJ42dlSq6PgwTxCEwkRB09nywo3K + +# Add access credential so that argo workflow can connect to COS +kubectl -n argo apply -f artifact-bucket-credential.yaml +kubectl create secret -n argo generic my-minio-cred --from-literal=access-key=$ACCESS_KEY --from-literal=secret-key=$SECRET_KEY + +# edit configmap to add access credentials +kubectl -n argo get configmap +kubectl -n argo edit configmap workflow-controller-configmap +data: + artifactRepository: | + archiveLogs: true + s3: + bucket: my-bucket + endpoint: argo-artifacts.default:9000 + insecure: true + accessKeySecret: + name: my-minio-cred + key: access-key + secretKeySecret: + name: my-minio-cred + key: secret-key diff --git a/e2eai/04.1.install_argo+minio.sh b/e2eai/04.1.install_argo+minio.sh new file mode 100644 index 0000000..85be204 --- /dev/null +++ b/e2eai/04.1.install_argo+minio.sh @@ -0,0 +1,15 @@ +helm repo add minio-helm https://charts.min.io/ +helm repo update + +#> helm install --namespace minio --create-namespace --set rootUser=rootuser,rootPassword=rootpass123 --generate-name minio-helm/minio + +helm install --namespace minio --create-namespace \ + --set resources.requests.memory=512Mi --set replicas=1 \ + --set persistence.enabled=false --set mode=standalone \ + --generate-name minio-helm/minio + +rootPassword=$(kubectl get secret --namespace minio myminio -o jsonpath="{.data.rootPassword}") +rootUser=$(kubectl get secret --namespace minio myminio -o jsonpath="{.data.rootUser}") + +kubectl -n argo edit configmap workflow-controller-configmap +kubectl -n argo edit configmap argo-worklows-argo-workflows-workflow-controller-configmap diff --git a/e2eai/05.install_argo_workflows.sh b/e2eai/05.install_argo_workflows.sh new file mode 100644 index 0000000..8c3e394 --- /dev/null +++ b/e2eai/05.install_argo_workflows.sh @@ -0,0 +1,118 @@ +# cd to github repo +cd argo-workflow/argo-workflows +git checkout v3.5.5 +cd manifests +# install minio, postgres, argo-server, httpbin +kubectl -n argo apply -f ./quick-start-minimal.yaml +kubectl -n argo delete -f argo-workflow/argo-workflows/manifests/quick-start-minimal.yaml + +# create default service account +kubectl -n argo create rolebinding default-admin --clusterrole=admin --serviceaccount=argo:default + +# for load balancer +kubectl -n argo patch svc argo-server -p '{"spec": {"type": "LoadBalancer"}}' +kubectl -n argo patch svc minio -p '{"spec": {"type": "LoadBalancer"}}' +kubectl -n argo patch svc httpbin -p '{"spec": {"type": "LoadBalancer"}}' +export MINIO=$(kubectl -n argo get svc |grep minio|tr -s " " " " |cut -d" " -f4) +export ARGO_SERVER=$(kubectl -n argo get svc |grep argo-server|tr -s " " " " |cut -d" " -f4) +open "https://${ARGO_SERVER}:2746" +open "http://${MINIO}:9001" + +# for nodeport +kubectl -n argo patch svc argo-server -p '{"spec": {"type": "NodePort"}}' +kubectl -n argo patch svc minio -p '{"spec": {"type": "NodePort"}}' +kubectl -n argo patch svc httpbin -p '{"spec": {"type": "NodePort"}}' +export NODE=mccoeu1.sl.cloud9.ibm.com +export MINIO=http://mccoeu1.sl.cloud9.ibm.com +export MINIO_PORT=$(kubectl -n argo get svc |grep minio|tr -s "[ /]" ":" |cut -d":" -f8) +export ARGO_PORT=$(kubectl -n argo get svc |grep argo-server|tr -s "[ /]" ":" |cut -d":" -f6) +export MINIO_PORT=30594 +open "http://${NODE}:${MINIO_PORT}" +open "https://${NODE}:${ARGO_PORT}" + + +#kubectl -n argo port-forward svc/argo-server 2746:2746 +#export ARGO_SECURE=true +#argo -n argo server --auth-mode=server + +# delete all workflows +argo -n argo delete --all + +# examples +argo -n argo submit ./yaml-examples/hello1.yaml +argo -n argo submit ./yaml-examples/hello1.yaml -p message="hello universe" +argo -n argo submit ./yaml-examples/hello2.yaml +argo -n argo submit ./yaml-examples/dag-parallel.yaml +argo -n argo submit ./yaml-examples/run-scripts.yaml +argo -n argo submit ./yaml-examples/output-parameters.yaml + +# secret +kubectl -n argo create secret generic my-secret --from-literal=mypassword=S00perS3cretPa55word +argo -n argo submit ./yaml-examples/secret-example.yaml +argo -n argo submit --serviceaccount argo-sa ./yaml-examples/argo-workflow-example-volumeclaim1.yaml + +# create pvc first called my-existing-volume +kubectl -n argo apply -f ./yaml-examples/pvc.yaml +argo -n argo submit --serviceaccount argo-sa ./yaml-examples/argo-workflow-example-volumeclaim2.yaml +#kubectl -n argo delete -f ./yaml-examples/pvc.yaml + +# template level +argo -n argo submit ./yaml-examples/argo-workflow-example-volumeclaim3.yaml +#argo -n argo logs template-level-volume-kzwm6 + +# sequence example +argo -n argo submit ./yaml-examples/withSequence-example.yaml +argo -n argo submit ./yaml-examples/withItems-example.yaml +argo -n argo submit ./yaml-examples/withItems-example-complex.yaml +argo -n argo submit ./yaml-examples/withParam-example-complex.yaml +argo -n argo submit ./yaml-examples/withParam-example-more-complex.yaml +argo -n argo submit ./yaml-examples/coinflip.yaml +argo -n argo submit ./yaml-examples/coinflip-recursion.yaml +argo -n argo submit ./yaml-examples/retry-strategy-example.yaml +argo -n argo submit ./yaml-examples/manage-k8s-resources-example.yaml +argo -n argo submit ./yaml-examples/daemon-containers-example.yaml +argo -n argo submit ./yaml-examples/sidecar-containers-example.yaml +argo -n argo submit ./yaml-examples/dind-containers-example.yaml + +argo -n argo submit ./yaml-examples/variable-reference-example.yaml +argo -n argo template create ./yaml-examples/loop-test-workflowtemplate.yaml +argo -n argo template create ./yaml-examples/enum-dropdown-wftemplates.yaml +argo -n argo submit --from=wftmpl/loop-test +argo template create https://raw.githubusercontent.com/argoproj/argo-workflows/main/examples/workflow-template/templates.yaml +argo submit https://raw.githubusercontent.com/argoproj/argo-workflows/main/examples/workflow-template/hello-world.yaml + +# workflow templates can be managed using kubectl apply -f and kubectl get wftmpl +kubectl -n argo apply -f https://raw.githubusercontent.com/argoproj-labs/argo-workflows-catalog/master/templates/hello-world/manifests.yaml +kubectl -n argo apply -f https://raw.githubusercontent.com/argoproj/argo-workflows/main/examples/cluster-workflow-template/clustertemplates.yaml + +#argo -n argo submit ./template-deadline-example.yaml +#argo -n argo submit ./workflow-deadline-example.yaml +#argo -n argo submit ./exit-handler-example.yaml +#argo -n argo submit ./suspend-example.yaml +#argo -n argo submit ./ci-example.yaml +#argo -n argo submit ./ci-influxdb-example.yaml + +# configure first artifact storage +# https://argo-workflows.readthedocs.io/en/release-3.5/configure-artifact-repository/ +argo -n argo submit ./yaml-examples/artifact-passing.yaml + +# customized images +argo -n argo submit ./yaml-examples/automl.yaml +argo -n argo submit ./yaml-examples/amlp.yaml +argo -n argo submit ./yaml-examples/tsml.yaml + +# using template +argo -n argo template create ./yaml-examples/combine-workflowtemplate.yaml +argo -n argo submit ./yaml-examples/combine-run-from-workflowtemplate.yaml + +# demo workflows +argo -n argo submit ./yaml-examples/hello.yaml +argo -n argo submit ./yaml-examples/sequential-preprocessing.yaml +argo -n argo submit ./yaml-examples/dag-rust-ml.yaml +argo -n argo submit ./yaml-examples/parallel.yaml +argo -n argo submit ./yaml-examples/parallel-combine.yaml + +# cluster template +kubectl -n argo apply -f ./yaml-examples/parallel-cluster-template.yaml +argo -n argo submit ./yaml-examples/parallel-cluster-template-workflow.yaml + diff --git a/e2eai/05.install_argo_workflows_helm.sh b/e2eai/05.install_argo_workflows_helm.sh new file mode 100644 index 0000000..f2d708f --- /dev/null +++ b/e2eai/05.install_argo_workflows_helm.sh @@ -0,0 +1,195 @@ +# old doc: https://pipekit.io/blog/using-helm-charts-to-deploy-argo-workflows +# https://github.com/argoproj/argo-helm +# https://argo-workflows.readthedocs.io/en/latest/installation/ +# https://argo-workflows.readthedocs.io/en/latest/walk-through/volumes/ + +helm repo add argo https://argoproj.github.io/argo-helm + +helm search repo argo +# argo/argo 1.0.0 v2.12.5 A Helm chart for Argo Workflows +# argo/argo-cd 6.6.0 v2.10.2 A Helm chart for Argo CD, a declarative, GitOps... +# argo/argo-ci 1.0.0 v1.0.0-alpha2 A Helm chart for Argo-CI +# argo/argo-events 2.4.3 v1.9.1 A Helm chart for Argo Events, the event-driven ... +# argo/argo-lite 0.1.0 Lighweight workflow engine for Kubernetes +# argo/argo-rollouts 2.34.3 v1.6.6 A Helm chart for Argo Rollouts +# argo/argo-workflows 0.40.14 v3.5.5 A Helm chart for Argo Workflows +# argo/argocd-applicationset 1.12.1 v0.4.1 A Helm chart for installing ArgoCD ApplicationSet +# argo/argocd-apps 1.6.2 A Helm chart for managing additional Argo CD Ap... +# argo/argocd-image-updater 0.9.5 v0.12.2 A Helm chart for Argo CD Image Updater, a tool ... +# argo/argocd-notifications 1.8.1 v1.2.1 A Helm chart for ArgoCD notifications, an add-o... + +helm -n argo install \ + --namespace argo \ + --create-namespace \ + argo-worklows argo/argo-workflows + + +kubectl --namespace argo get services -o wide | grep argo-worklows-argo-workflows-server + +# create default service account +kubectl -n argo create rolebinding default-admin --clusterrole=admin --serviceaccount=argo:default + +# patch svc as load-balancer +kubectl -n argo patch svc argo-worklows-argo-workflows-server -p '{"spec": {"type": "LoadBalancer"}}' + +# change auth mode of server +# https://argo-workflows.readthedocs.io/en/latest/argo-server-auth-mode/ +# https://argoproj.github.io/argo-workflows/argo-server-auth-mode/ +# https://github.com/argoproj/argo-workflows/issues/9868 +cat < argo-role.yaml +kubectl -n argo apply -f argo-role.yaml + +# create service account +kubectl -n argo create sa argo-sa +# kubectl -n argo create sa argo-workflow + +# create role-binding +cat > role-binding.yaml <"rb")) +const pt = SKPreprocessor("PowerTransformer",Dict(:name=>"pt")) +const norm = SKPreprocessor("Normalizer",Dict(:name=>"norm")) +const mx = SKPreprocessor("MinMaxScaler",Dict(:name=>"mx")) +const std = SKPreprocessor("StandardScaler",Dict(:name=>"std")) +# define extractors +const pca = SKPreprocessor("PCA",Dict(:name=>"pca")) +const fa = SKPreprocessor("FactorAnalysis",Dict(:name=>"fa")) +const ica = SKPreprocessor("FastICA",Dict(:name=>"ica")) +# define learners +const rf = SKLearner("RandomForestClassifier",Dict(:name => "rf")) +const ada = SKLearner("AdaBoostClassifier",Dict(:name => "ada")) +const gb = SKLearner("GradientBoostingClassifier",Dict(:name => "gb")) +const lsvc = SKLearner("LinearSVC",Dict(:name => "lsvc")) +const rbfsvc = SKLearner("SVC",Dict(:name => "rbfsvc")) +const dt = SKLearner("DecisionTreeClassifier",Dict(:name =>"dt")) +# preprocessing +const noop = Identity(Dict(:name =>"noop")) +const ohe = OneHotEncoder(Dict(:name=>"ohe")) +const catf = CatFeatureSelector(Dict(:name=>"catf")) +const numf = NumFeatureSelector(Dict(:name=>"numf")) + +const vscalers = [rb,pt,norm,mx,std,noop] +const vextractors = [pca,fa,ica,noop] +const vlearners = [rf,gb,lsvc,rbfsvc,ada,dt] +const learnerdict = Dict("rf"=>rf,"gb"=>gb,"lsvc"=>lsvc,"rbfsvc"=>rbfsvc,"ada"=>ada,"dt"=>dt) + + +function oneblock_pipeline_factory(scalers,extractors,learners) + results = @distributed (vcat) for lr in learners + @distributed (vcat) for xt in extractors + @distributed (vcat) for sc in scalers + # baseline preprocessing + prep = @pipeline ((catf |> ohe) + numf) + # one-block prp + expx = @pipeline prep |> (sc |> xt) |> lr + scn = sc.name[1:end - 4];xtn = xt.name[1:end - 4]; lrn = lr.name[1:end - 4] + pname = "($scn |> $xtn) |> $lrn" + DataFrame(Description=pname,Pipeline=expx) + end + end + end + return results +end + +function evaluate_pipeline(dfpipelines,X,Y;folds=3) + res=@distributed (vcat) for prow in eachrow(dfpipelines) + perf = crossvalidate(prow.Pipeline,X,Y,"balanced_accuracy_score";nfolds=folds) + DataFrame(;Description=prow.Description,mean=perf.mean,sd=perf.std,prow.Pipeline) + end + return res +end + +function twoblock_pipeline_factory(scalers,extractors,learners) + results = @distributed (vcat) for lr in learners + @distributed (vcat) for xt1 in extractors + @distributed (vcat) for xt2 in extractors + @distributed (vcat) for sc1 in scalers + @distributed (vcat) for sc2 in scalers + prep = @pipeline ((catf |> ohe) + numf) + expx = @pipeline prep |> ((sc1 |> xt1) + (sc2 |> xt2)) |> lr + scn1 = sc1.name[1:end - 4];xtn1 = xt1.name[1:end - 4]; + scn2 = sc2.name[1:end - 4];xtn2 = xt2.name[1:end - 4]; + lrn = lr.name[1:end - 4] + pname = "($scn1 |> $xtn1) + ($scn2 |> $xtn2) |> $lrn" + DataFrame(Description=pname,Pipeline=expx) + end + end + end + end + end + return results +end + +function model_selection_pipeline(learners) + results = @distributed (vcat) for lr in learners + prep = @pipeline ((catf |> ohe) + numf) + expx = @pipeline prep |> (rb |> pca) |> lr + pname = "(rb |> pca) |> $(lr.name[1:end-4])" + DataFrame(Description=pname,Pipeline=expx) + end + return results +end + +function lname(n::Learner) + n.name[1:end-4] +end + +function twoblockspipelinesearch(X::DataFrame,Y::Vector;scalers=vscalers,extractors=vextractors,learners=vlearners,nfolds=3) + dfpipes = model_selection_pipeline(vlearners) + # find the best model by evaluating the models + modelsperf = evaluate_pipeline(dfpipes,X,Y;folds=nfolds) + sort!(modelsperf,:mean, rev = true) + # get the string name of the top model + bestm = filter(x->occursin(x,modelsperf.Description[1]),lname.(vlearners))[1] + # get corresponding model object + bestmodel = learnerdict[bestm] + # use the best model to generate pipeline search + dfp = twoblock_pipeline_factory(vscalers,vextractors,[bestmodel]) + # evaluate the pipeline + bestp=evaluate_pipeline(dfp,X,Y;folds=nfolds) + sort!(bestp,:mean, rev = true) + show(bestp;allrows=false,truncate=1,allcols=false) + println() + optmodel = bestp[1,:] + return bestp +end + +end