diff --git a/demo/resource-usage.demo b/demo/resource-usage.demo new file mode 100644 index 000000000..5aa96215a --- /dev/null +++ b/demo/resource-usage.demo @@ -0,0 +1,28 @@ +# Let's getting started with kwokctl! +kwokctl create cluster --enable-metrics-server -c ./kustomize/metrics/resource/metrics-resource.yaml -c ./kustomize/metrics/usage/usage-from-annotation.yaml + +kwokctl scale node --replicas 2 +kwokctl scale pod --replicas 8 + +# Wait for a while to let the metrics server collect the metrics. +sleep 45 + +# Now we can check the metrics. +kubectl top node +kubectl top pod + +# Let's add some usage metrics to the pods. +kubectl patch pod pod-000000 --type=json -p='[{"op":"add","path":"/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwok.x-k8s.io/usage-memory":"10000Mi"}}]' + +# Wait for a while to let the metrics server collect the metrics. +sleep 15 + +# Now we can check the metrics again. +kubectl top node +kubectl top pod + +# Delete the cluster. +kwokctl delete cluster + +# That's all, enjoy it! +clear diff --git a/demo/resource-usage.svg b/demo/resource-usage.svg new file mode 100644 index 000000000..319baeec2 --- /dev/null +++ b/demo/resource-usage.svg @@ -0,0 +1 @@ +~/sigs.k8s.io/kwok$~/sigs.k8s.io/kwok$#~/sigs.k8s.io/kwok$#L~/sigs.k8s.io/kwok$#Le~/sigs.k8s.io/kwok$#Let~/sigs.k8s.io/kwok$#Let'~/sigs.k8s.io/kwok$#Let's~/sigs.k8s.io/kwok$#Let'sgetting~/sigs.k8s.io/kwok$#Let'sgettingstarted~/sigs.k8s.io/kwok$#Let'sgettingstartedwith~/sigs.k8s.io/kwok$#Let'sgettingstartedwithkwokctl!~/sigs.k8s.io/kwok$k~/sigs.k8s.io/kwok$kw~/sigs.k8s.io/kwok$kwo~/sigs.k8s.io/kwok$kwok~/sigs.k8s.io/kwok$kwokc~/sigs.k8s.io/kwok$kwokct~/sigs.k8s.io/kwok$kwokctl~/sigs.k8s.io/kwok$kwokctlcreate~/sigs.k8s.io/kwok$kwokctlcreatecluster~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c./kustomize/metrics/resource/metrics-resource.yamlics/resource/metrics-resource.yaml-cics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-from-annotation.yamlClusteriscreatingcluster=kwokClusteriscreatedelapsed=1scluster=kwokClusterisstartingcluster=kwokClusterisstartedelapsed=1.1scluster=kwokLoadresourcescounter=2elapsed=14mscluster=kwokYoucannowuseyourclusterwith:kubectlcluster-info--contextkwok-kwokThanksforusingkwok!~/sigs.k8s.io/kwok$kwokctls~/sigs.k8s.io/kwok$kwokctlsc~/sigs.k8s.io/kwok$kwokctlsca~/sigs.k8s.io/kwok$kwokctlscal~/sigs.k8s.io/kwok$kwokctlscale~/sigs.k8s.io/kwok$kwokctlscalenode~/sigs.k8s.io/kwok$kwokctlscalenode--replicas~/sigs.k8s.io/kwok$kwokctlscalenode--replicas2Noresourcefound,usedefaultresourceresource=nodecluster=kwokLoadresourcescounter=2elapsed=21msresource=nodesreplicas=2name=nodecluster=kwok~/sigs.k8s.io/kwok$kwokctlscalepod~/sigs.k8s.io/kwok$kwokctlscalepod--replicas~/sigs.k8s.io/kwok$kwokctlscalepod--replicas8Noresourcefound,usedefaultresourceresource=podcluster=kwokLoadresourcescounter=8elapsed=40msnamespace=defaultresource=podsreplicas=8name=podcluster=kwok~/sigs.k8s.io/kwok$#W~/sigs.k8s.io/kwok$#Wa~/sigs.k8s.io/kwok$#Wai~/sigs.k8s.io/kwok$#Wait~/sigs.k8s.io/kwok$#Waitf~/sigs.k8s.io/kwok$#Waitfo~/sigs.k8s.io/kwok$#Waitfor~/sigs.k8s.io/kwok$#Waitfora~/sigs.k8s.io/kwok$#Waitforaw~/sigs.k8s.io/kwok$#Waitforawh~/sigs.k8s.io/kwok$#Waitforawhi~/sigs.k8s.io/kwok$#Waitforawhil~/sigs.k8s.io/kwok$#Waitforawhile~/sigs.k8s.io/kwok$#Waitforawhilet~/sigs.k8s.io/kwok$#Waitforawhileto~/sigs.k8s.io/kwok$#Waitforawhiletol~/sigs.k8s.io/kwok$#Waitforawhiletole~/sigs.k8s.io/kwok$#Waitforawhiletolet~/sigs.k8s.io/kwok$#Waitforawhiletolett~/sigs.k8s.io/kwok$#Waitforawhiletoletth~/sigs.k8s.io/kwok$#Waitforawhiletoletthe~/sigs.k8s.io/kwok$#Waitforawhiletoletthem~/sigs.k8s.io/kwok$#Waitforawhiletolettheme~/sigs.k8s.io/kwok$#Waitforawhiletoletthemet~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetr~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetri~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetric~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetrics~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricss~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsse~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsser~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsserv~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsserve~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsserver~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsserverc~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsserverco~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsservercol~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsservercoll~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsservercolle~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsservercollec~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsservercollect~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsservercollectt~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsservercollectth~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsservercollectthe~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsservercollectthem~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsservercollecttheme~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsservercollectthemet~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsservercollectthemetr~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsservercollectthemetri~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsservercollectthemetric~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsservercollectthemetrics~/sigs.k8s.io/kwok$#Waitforawhiletoletthemetricsservercollectthemetrics.~/sigs.k8s.io/kwok$s~/sigs.k8s.io/kwok$sl~/sigs.k8s.io/kwok$sle~/sigs.k8s.io/kwok$slee~/sigs.k8s.io/kwok$sleep~/sigs.k8s.io/kwok$sleep45~/sigs.k8s.io/kwok$#N~/sigs.k8s.io/kwok$#No~/sigs.k8s.io/kwok$#Now~/sigs.k8s.io/kwok$#Noww~/sigs.k8s.io/kwok$#Nowwe~/sigs.k8s.io/kwok$#Nowwec~/sigs.k8s.io/kwok$#Nowweca~/sigs.k8s.io/kwok$#Nowwecan~/sigs.k8s.io/kwok$#Nowwecanc~/sigs.k8s.io/kwok$#Nowwecanch~/sigs.k8s.io/kwok$#Nowwecanche~/sigs.k8s.io/kwok$#Nowwecanchec~/sigs.k8s.io/kwok$#Nowwecancheck~/sigs.k8s.io/kwok$#Nowwecancheckt~/sigs.k8s.io/kwok$#Nowwecancheckth~/sigs.k8s.io/kwok$#Nowwecancheckthe~/sigs.k8s.io/kwok$#Nowwecancheckthem~/sigs.k8s.io/kwok$#Nowwecanchecktheme~/sigs.k8s.io/kwok$#Nowwecancheckthemet~/sigs.k8s.io/kwok$#Nowwecancheckthemetr~/sigs.k8s.io/kwok$#Nowwecancheckthemetri~/sigs.k8s.io/kwok$#Nowwecancheckthemetric~/sigs.k8s.io/kwok$#Nowwecancheckthemetrics~/sigs.k8s.io/kwok$#Nowwecancheckthemetrics.~/sigs.k8s.io/kwok$ku~/sigs.k8s.io/kwok$kub~/sigs.k8s.io/kwok$kube~/sigs.k8s.io/kwok$kubec~/sigs.k8s.io/kwok$kubect~/sigs.k8s.io/kwok$kubectl~/sigs.k8s.io/kwok$kubectlt~/sigs.k8s.io/kwok$kubectlto~/sigs.k8s.io/kwok$kubectltop~/sigs.k8s.io/kwok$kubectltopn~/sigs.k8s.io/kwok$kubectltopno~/sigs.k8s.io/kwok$kubectltopnod~/sigs.k8s.io/kwok$kubectltopnodeNAMECPU(cores)CPU%MEMORY(bytes)MEMORY%node-0000003m0%3Mi0%node-0000016m0%5Mi0%~/sigs.k8s.io/kwok$kubectltopp~/sigs.k8s.io/kwok$kubectltoppo~/sigs.k8s.io/kwok$kubectltoppodNAMECPU(cores)MEMORY(bytes)pod-0000001m1Mipod-0000012m1Mipod-0000021m1Mipod-0000031m1Mipod-0000042m1Mipod-0000052m1Mipod-0000062m1Mipod-0000072m1Mi~/sigs.k8s.io/kwok$#Let'sadd~/sigs.k8s.io/kwok$#Let'saddsome~/sigs.k8s.io/kwok$#Let'saddsomeusage~/sigs.k8s.io/kwok$#Let'saddsomeusagemetrics~/sigs.k8s.io/kwok$#Let'saddsomeusagemetricsto~/sigs.k8s.io/kwok$#Let'saddsomeusagemetricstothe~/sigs.k8s.io/kwok$#Let'saddsomeusagemetricstothepods.~/sigs.k8s.io/kwok$kubectlpatch~/sigs.k8s.io/kwok$kubectlpatchpod~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op":"add","path":"/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwok.x-k8s.io/usage-memory":"10000Mi"}}]'pod/pod-000000patched~/sigs.k8s.io/kwok$sleep15~/sigs.k8s.io/kwok$#Nowwecancheckthemetricsagain.node-00000010003m31%10002Mi3%node-0000015m0%5Mi0%pod-00000010002m10000Mipod-0000011m1Mipod-0000022m1Mipod-0000032m1Mipod-0000041m1Mipod-0000051m1Mipod-0000061m1Mipod-0000071m1Mi~/sigs.k8s.io/kwok$#Delete~/sigs.k8s.io/kwok$#Deletethe~/sigs.k8s.io/kwok$#Deletethecluster.~/sigs.k8s.io/kwok$kwokctldelete~/sigs.k8s.io/kwok$kwokctldeleteclusterClusterisstoppingcluster=kwokClusterisstoppedelapsed=0.4scluster=kwokClusterisdeletingcluster=kwokClusterisdeletedelapsed=0.5scluster=kwok~/sigs.k8s.io/kwok$#That's~/sigs.k8s.io/kwok$#That'sall,~/sigs.k8s.io/kwok$#That'sall,enjoy~/sigs.k8s.io/kwok$#That'sall,enjoyit!~/sigs.k8s.io/kwok$clear~/sigs.k8s.io/kwok$#Let'sg~/sigs.k8s.io/kwok$#Let'sge~/sigs.k8s.io/kwok$#Let'sget~/sigs.k8s.io/kwok$#Let'sgett~/sigs.k8s.io/kwok$#Let'sgetti~/sigs.k8s.io/kwok$#Let'sgettin~/sigs.k8s.io/kwok$#Let'sgettings~/sigs.k8s.io/kwok$#Let'sgettingst~/sigs.k8s.io/kwok$#Let'sgettingsta~/sigs.k8s.io/kwok$#Let'sgettingstar~/sigs.k8s.io/kwok$#Let'sgettingstart~/sigs.k8s.io/kwok$#Let'sgettingstarte~/sigs.k8s.io/kwok$#Let'sgettingstartedw~/sigs.k8s.io/kwok$#Let'sgettingstartedwi~/sigs.k8s.io/kwok$#Let'sgettingstartedwit~/sigs.k8s.io/kwok$#Let'sgettingstartedwithk~/sigs.k8s.io/kwok$#Let'sgettingstartedwithkw~/sigs.k8s.io/kwok$#Let'sgettingstartedwithkwo~/sigs.k8s.io/kwok$#Let'sgettingstartedwithkwok~/sigs.k8s.io/kwok$#Let'sgettingstartedwithkwokc~/sigs.k8s.io/kwok$#Let'sgettingstartedwithkwokct~/sigs.k8s.io/kwok$#Let'sgettingstartedwithkwokctl~/sigs.k8s.io/kwok$kwokctlc~/sigs.k8s.io/kwok$kwokctlcr~/sigs.k8s.io/kwok$kwokctlcre~/sigs.k8s.io/kwok$kwokctlcrea~/sigs.k8s.io/kwok$kwokctlcreat~/sigs.k8s.io/kwok$kwokctlcreatec~/sigs.k8s.io/kwok$kwokctlcreatecl~/sigs.k8s.io/kwok$kwokctlcreateclu~/sigs.k8s.io/kwok$kwokctlcreateclus~/sigs.k8s.io/kwok$kwokctlcreateclust~/sigs.k8s.io/kwok$kwokctlcreatecluste~/sigs.k8s.io/kwok$kwokctlcreatecluster-~/sigs.k8s.io/kwok$kwokctlcreatecluster--~/sigs.k8s.io/kwok$kwokctlcreatecluster--e~/sigs.k8s.io/kwok$kwokctlcreatecluster--en~/sigs.k8s.io/kwok$kwokctlcreatecluster--ena~/sigs.k8s.io/kwok$kwokctlcreatecluster--enab~/sigs.k8s.io/kwok$kwokctlcreatecluster--enabl~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-m~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-me~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-met~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metr~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metri~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metric~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-s~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-se~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-ser~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-serv~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-serve~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c.~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c./~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c./k~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c./ku~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c./kus~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c./kust~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c./kusto~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c./kustom~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c./kustomi~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c./kustomiz~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c./kustomize~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c./kustomize/~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c./kustomize/m~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c./kustomize/me~/sigs.k8s.io/kwok$kwokctlcreatecluster--enable-metrics-server-c./kustomize/metiicicsics/ics/rics/reics/resics/resoics/resouics/resourics/resourcics/resourceics/resource/ics/resource/mics/resource/meics/resource/metics/resource/metrics/resource/metriics/resource/metricics/resource/metricsics/resource/metrics-ics/resource/metrics-rics/resource/metrics-reics/resource/metrics-resics/resource/metrics-resoics/resource/metrics-resouics/resource/metrics-resourics/resource/metrics-resourcics/resource/metrics-resourceics/resource/metrics-resource.ics/resource/metrics-resource.yics/resource/metrics-resource.yaics/resource/metrics-resource.yamics/resource/metrics-resource.yaml-ics/resource/metrics-resource.yaml-c.ics/resource/metrics-resource.yaml-c./ics/resource/metrics-resource.yaml-c./kics/resource/metrics-resource.yaml-c./kuics/resource/metrics-resource.yaml-c./kusics/resource/metrics-resource.yaml-c./kustics/resource/metrics-resource.yaml-c./kustoics/resource/metrics-resource.yaml-c./kustomics/resource/metrics-resource.yaml-c./kustomiics/resource/metrics-resource.yaml-c./kustomizics/resource/metrics-resource.yaml-c./kustomizeics/resource/metrics-resource.yaml-c./kustomize/ics/resource/metrics-resource.yaml-c./kustomize/mics/resource/metrics-resource.yaml-c./kustomize/meics/resource/metrics-resource.yaml-c./kustomize/metics/resource/metrics-resource.yaml-c./kustomize/metrics/resource/metrics-resource.yaml-c./kustomize/metriics/resource/metrics-resource.yaml-c./kustomize/metricics/resource/metrics-resource.yaml-c./kustomize/metricsics/resource/metrics-resource.yaml-c./kustomize/metrics/ics/resource/metrics-resource.yaml-c./kustomize/metrics/uics/resource/metrics-resource.yaml-c./kustomize/metrics/usics/resource/metrics-resource.yaml-c./kustomize/metrics/usaics/resource/metrics-resource.yaml-c./kustomize/metrics/usagics/resource/metrics-resource.yaml-c./kustomize/metrics/usageics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/ics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/uics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usaics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usagics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usageics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-ics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-fics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-frics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-froics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-fromics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-from-ics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-from-aics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-from-anics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-from-annics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-from-annoics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-from-annotics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-from-annotaics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-from-annotatics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-from-annotatiics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-from-annotatioics/resource/metrics-resource.yaml-c./kustomize/metrics/usage/usage-from-annotationyyayam~/sigs.k8s.io/kwok$kwokctlscalen~/sigs.k8s.io/kwok$kwokctlscaleno~/sigs.k8s.io/kwok$kwokctlscalenod~/sigs.k8s.io/kwok$kwokctlscalenode-~/sigs.k8s.io/kwok$kwokctlscalenode--~/sigs.k8s.io/kwok$kwokctlscalenode--r~/sigs.k8s.io/kwok$kwokctlscalenode--re~/sigs.k8s.io/kwok$kwokctlscalenode--rep~/sigs.k8s.io/kwok$kwokctlscalenode--repl~/sigs.k8s.io/kwok$kwokctlscalenode--repli~/sigs.k8s.io/kwok$kwokctlscalenode--replic~/sigs.k8s.io/kwok$kwokctlscalenode--replica~/sigs.k8s.io/kwok$kwokctlscalep~/sigs.k8s.io/kwok$kwokctlscalepo~/sigs.k8s.io/kwok$kwokctlscalepod-~/sigs.k8s.io/kwok$kwokctlscalepod--~/sigs.k8s.io/kwok$kwokctlscalepod--r~/sigs.k8s.io/kwok$kwokctlscalepod--re~/sigs.k8s.io/kwok$kwokctlscalepod--rep~/sigs.k8s.io/kwok$kwokctlscalepod--repl~/sigs.k8s.io/kwok$kwokctlscalepod--repli~/sigs.k8s.io/kwok$kwokctlscalepod--replic~/sigs.k8s.io/kwok$kwokctlscalepod--replica~/sigs.k8s.io/kwok$sleep4node-0000016mpod-000002~/sigs.k8s.io/kwok$#Let'sa~/sigs.k8s.io/kwok$#Let'sad~/sigs.k8s.io/kwok$#Let'sadds~/sigs.k8s.io/kwok$#Let'saddso~/sigs.k8s.io/kwok$#Let'saddsom~/sigs.k8s.io/kwok$#Let'saddsomeu~/sigs.k8s.io/kwok$#Let'saddsomeus~/sigs.k8s.io/kwok$#Let'saddsomeusa~/sigs.k8s.io/kwok$#Let'saddsomeusag~/sigs.k8s.io/kwok$#Let'saddsomeusagem~/sigs.k8s.io/kwok$#Let'saddsomeusageme~/sigs.k8s.io/kwok$#Let'saddsomeusagemet~/sigs.k8s.io/kwok$#Let'saddsomeusagemetr~/sigs.k8s.io/kwok$#Let'saddsomeusagemetri~/sigs.k8s.io/kwok$#Let'saddsomeusagemetric~/sigs.k8s.io/kwok$#Let'saddsomeusagemetricst~/sigs.k8s.io/kwok$#Let'saddsomeusagemetricstot~/sigs.k8s.io/kwok$#Let'saddsomeusagemetricstoth~/sigs.k8s.io/kwok$#Let'saddsomeusagemetricstothep~/sigs.k8s.io/kwok$#Let'saddsomeusagemetricstothepo~/sigs.k8s.io/kwok$#Let'saddsomeusagemetricstothepod~/sigs.k8s.io/kwok$#Let'saddsomeusagemetricstothepods~/sigs.k8s.io/kwok$kubectlp~/sigs.k8s.io/kwok$kubectlpa~/sigs.k8s.io/kwok$kubectlpat~/sigs.k8s.io/kwok$kubectlpatc~/sigs.k8s.io/kwok$kubectlpatchp~/sigs.k8s.io/kwok$kubectlpatchpo~/sigs.k8s.io/kwok$kubectlpatchpodp~/sigs.k8s.io/kwok$kubectlpatchpodpo~/sigs.k8s.io/kwok$kubectlpatchpodpod~/sigs.k8s.io/kwok$kubectlpatchpodpod-~/sigs.k8s.io/kwok$kubectlpatchpodpod-0~/sigs.k8s.io/kwok$kubectlpatchpodpod-00~/sigs.k8s.io/kwok$kubectlpatchpodpod-000~/sigs.k8s.io/kwok$kubectlpatchpodpod-0000~/sigs.k8s.io/kwok$kubectlpatchpodpod-00000~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000-~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--t~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--ty~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--typ~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=j~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=js~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=jso~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p=~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"o~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op"~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op":~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op":"~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op":"a~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op":"ad~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op":"add~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op":"add"~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op":"add",~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op":"add","~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op":"add","p~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op":"add","pa~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op":"add","pat~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op":"add","path~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op":"add","path"~/sigs.k8s.io/kwok$kubectlpatchpodpod-000000--type=json-p='[{"op":"add","path"://m/me/met/meta/metad/metada/metadat/metadata/metadata//metadata/a/metadata/an/metadata/ann/metadata/anno/metadata/annot/metadata/annota/metadata/annotat/metadata/annotati/metadata/annotatio/metadata/annotation/metadata/annotations/metadata/annotations"/metadata/annotations",/metadata/annotations","/metadata/annotations","v/metadata/annotations","va/metadata/annotations","val/metadata/annotations","valu/metadata/annotations","value/metadata/annotations","value"/metadata/annotations","value":/metadata/annotations","value":{/metadata/annotations","value":{"/metadata/annotations","value":{"k/metadata/annotations","value":{"kw/metadata/annotations","value":{"kwo/metadata/annotations","value":{"kwok/metadata/annotations","value":{"kwok./metadata/annotations","value":{"kwok.x/metadata/annotations","value":{"kwok.x-/metadata/annotations","value":{"kwok.x-k/metadata/annotations","value":{"kwok.x-k8/metadata/annotations","value":{"kwok.x-k8s/metadata/annotations","value":{"kwok.x-k8s./metadata/annotations","value":{"kwok.x-k8s.i/metadata/annotations","value":{"kwok.x-k8s.io/metadata/annotations","value":{"kwok.x-k8s.io//metadata/annotations","value":{"kwok.x-k8s.io/u/metadata/annotations","value":{"kwok.x-k8s.io/us/metadata/annotations","value":{"kwok.x-k8s.io/usa/metadata/annotations","value":{"kwok.x-k8s.io/usag/metadata/annotations","value":{"kwok.x-k8s.io/usage/metadata/annotations","value":{"kwok.x-k8s.io/usage-/metadata/annotations","value":{"kwok.x-k8s.io/usage-c/metadata/annotations","value":{"kwok.x-k8s.io/usage-cp/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu"/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"1/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"100/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"1000/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m"/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m",/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","k/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kw/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwo/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwok/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwok./metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwok.x/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwok.x-/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwok.x-k/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwok.x-k8/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwok.x-k8s/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwok.x-k8s./metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwok.x-k8s.i/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwok.x-k8s.io/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwok.x-k8s.io//metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwok.x-k8s.io/u/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwok.x-k8s.io/us/metadata/annotations","value":{"kwok.x-k8s.io/usage-cpu":"10000m","kwok.x-k8s.io/usaee-e-me-mee-meme-memoe-memore-memorye-memory"e-memory":e-memory":"e-memory":"1e-memory":"10e-memory":"100e-memory":"1000e-memory":"10000e-memory":"10000Me-memory":"10000Mie-memory":"10000Mi"e-memory":"10000Mi"}e-memory":"10000Mi"}}e-memory":"10000Mi"}}]~/sigs.k8s.io/kwok$sleep1~/sigs.k8s.io/kwok$#Nowwecancheckthemetricsa~/sigs.k8s.io/kwok$#Nowwecancheckthemetricsag~/sigs.k8s.io/kwok$#Nowwecancheckthemetricsaga~/sigs.k8s.io/kwok$#Nowwecancheckthemetricsagai~/sigs.k8s.io/kwok$#Nowwecancheckthemetricsagain~/sigs.k8s.io/kwok$#D~/sigs.k8s.io/kwok$#De~/sigs.k8s.io/kwok$#Del~/sigs.k8s.io/kwok$#Dele~/sigs.k8s.io/kwok$#Delet~/sigs.k8s.io/kwok$#Deletet~/sigs.k8s.io/kwok$#Deleteth~/sigs.k8s.io/kwok$#Deletethec~/sigs.k8s.io/kwok$#Deletethecl~/sigs.k8s.io/kwok$#Deletetheclu~/sigs.k8s.io/kwok$#Deletetheclus~/sigs.k8s.io/kwok$#Deletetheclust~/sigs.k8s.io/kwok$#Deletethecluste~/sigs.k8s.io/kwok$#Deletethecluster~/sigs.k8s.io/kwok$kwokctld~/sigs.k8s.io/kwok$kwokctlde~/sigs.k8s.io/kwok$kwokctldel~/sigs.k8s.io/kwok$kwokctldele~/sigs.k8s.io/kwok$kwokctldelet~/sigs.k8s.io/kwok$kwokctldeletec~/sigs.k8s.io/kwok$kwokctldeletecl~/sigs.k8s.io/kwok$kwokctldeleteclu~/sigs.k8s.io/kwok$kwokctldeleteclus~/sigs.k8s.io/kwok$kwokctldeleteclust~/sigs.k8s.io/kwok$kwokctldeletecluste~/sigs.k8s.io/kwok$#T~/sigs.k8s.io/kwok$#Th~/sigs.k8s.io/kwok$#Tha~/sigs.k8s.io/kwok$#That~/sigs.k8s.io/kwok$#That'~/sigs.k8s.io/kwok$#That'sa~/sigs.k8s.io/kwok$#That'sal~/sigs.k8s.io/kwok$#That'sall~/sigs.k8s.io/kwok$#That'sall,e~/sigs.k8s.io/kwok$#That'sall,en~/sigs.k8s.io/kwok$#That'sall,enj~/sigs.k8s.io/kwok$#That'sall,enjo~/sigs.k8s.io/kwok$#That'sall,enjoyi~/sigs.k8s.io/kwok$#That'sall,enjoyit~/sigs.k8s.io/kwok$c~/sigs.k8s.io/kwok$cl~/sigs.k8s.io/kwok$cle~/sigs.k8s.io/kwok$clea \ No newline at end of file diff --git a/kustomize/metrics/resource/README.md b/kustomize/metrics/resource/README.md new file mode 100644 index 000000000..b0201d59c --- /dev/null +++ b/kustomize/metrics/resource/README.md @@ -0,0 +1,5 @@ +# Metrics Resource + +This Metrics simulates kubelet's `/metrics/resource` endpoint. +Please refer to [Metrics](https://kwok.sigs.k8s.io/docs/user/metrics-configuration) for more on how it works. + diff --git a/kustomize/metrics/usage/README.md b/kustomize/metrics/usage/README.md new file mode 100644 index 000000000..23d2d24d6 --- /dev/null +++ b/kustomize/metrics/usage/README.md @@ -0,0 +1,8 @@ +# Resource Usage + +This ResourceUsage simulates the resource usage of Pod(s) based on information collected from the respective annotations. + +Provided two annotations for Pod(s): + +- `kwok.x-k8s.io/usage-cpu` +- `kwok.x-k8s.io/usage-memory` diff --git a/site/config.yaml b/site/config.yaml index 8f105bc2f..873003345 100644 --- a/site/config.yaml +++ b/site/config.yaml @@ -215,6 +215,18 @@ menu: - identifier: attach pageRef: "/docs/user/attach-configuration" parent: configuration + - identifier: metrics + pageRef: "/docs/user/metrics-configuration" + parent: configuration + - identifier: resource-usage + pageRef: "/docs/user/resource-usage-configuration" + parent: configuration + - identifier: go-template + pageRef: "/docs/user/go-template" + parent: configuration + - identifier: cel-expressions + pageRef: "/docs/user/cel-expressions" + parent: configuration # Design Children - identifier: introduction diff --git a/site/content/en/docs/user/_index.md b/site/content/en/docs/user/_index.md index 46814c02e..0d38a7f70 100644 --- a/site/content/en/docs/user/_index.md +++ b/site/content/en/docs/user/_index.md @@ -37,6 +37,7 @@ If any special concerns, you can configure KWOK with options and stages. - [Exec] - [Logs] - [Attach] +- [ResourceUsage] I hope this helps you get started with KWOK! Good luck and have fun! @@ -53,3 +54,4 @@ I hope this helps you get started with KWOK! Good luck and have fun! [Exec]: {{< relref "/docs/user/exec-configuration" >}} [Logs]: {{< relref "/docs/user/logs-configuration" >}} [Attach]: {{< relref "/docs/user/attach-configuration" >}} +[ResourceUsage]: {{< relref "/docs/user/resource-usage-configuration" >}} diff --git a/site/content/en/docs/user/cel-expressions.md b/site/content/en/docs/user/cel-expressions.md new file mode 100644 index 000000000..f8133c498 --- /dev/null +++ b/site/content/en/docs/user/cel-expressions.md @@ -0,0 +1,77 @@ +--- +title: "CEL Expressions in `kwok`" +--- + +# Notes on CEL Expressions in `kwok` + +The page provides a concise note on writing CEL expressions in `kwok` CRs. + +Below is the list of all CRs in `kwok` that contains CEL based fields. +* [Metric] +* [ResourceUsage] +* [ClusterResourceUsage] + + +You must follow [the CEL language specification] when writing the expressions. +For predefined functions of CEL, please refer to [CEL predefined functions]. + +Besides the built-in functions, `kwok` also provides some customized extension functions. +An exhaustive list of all the extension functions with their usages is given below. + +* `Now()`: takes no parameters and returns the current timestamp. +* `Rand()`: takes no parameters and returns a random `float64` value. +* `SinceSecond()` returns the seconds elapsed since a given resource (`pod` or `node`) was created. + For example: `SinceSecond(pod)`, `node.SinceSecond(node)`. +* `UnixSecond()` returns the Unix time of a given time of type `time.Time`. + For example: , `UnixSecond(Now())`, `UnixSecond(node.metadata.creationTimestamp)`. +* `Quantity()` returns a float64 value of a given Quantity value. For example: `Quantity("100m")`, `Quantity("10Mi")`. +* `Usage()` returns the current instantaneous resource usage with the simulation data in [ResourceUsage (ClusterResourceUsage)]. + For example: `Usage(pod, "memory")`, `Usage(node, "memory")`, `Usage(pod, "memory", container.name)` return the + current working set of a resource (pod, node or container) in bytes. +* `CumulativeUsage()` returns the cumulative resource usage in seconds with the simulation data given in [ResourceUsage (ClusterResourceUsage)]. + For example: `CumulativeUsage(pod, "cpu")`, `CumulativeUsage(node, "cpu")`, `CumulativeUsage(pod, "cpu", container.name)` + return a cumulative cpu time consumed by a resource (pod, node or container) in core-seconds. + +Additionally, `kwok` provides three special CEL variables `node`, `pod`, and `container` that could be used +in the expressions. +The three variables are set to the corresponding node, pod, container resource object respectively and users can +reference any nested fields of the resource objects simply via the CEL field selection expression (`e.f` format). +For example, you could use expression `node.metadata.name` to obtain the node name. + +{{< hint "info" >}} + +The functions with at least one parameter can be called in a receiver call-style. +That is, a function call like `f(e1, e2)` can also be called in style `e1.f(e2)`. For example, you can use `pod.Usage("memory")` +as an alternative to `Usage(pod, "memory")`. + +{{< /hint >}} + + +It is worth noting that the use of some extension functions is restricted to specific CRs and contexts in the sense +that they are not generic but designed for special evaluating tasks. +The detailed limitations are described below. + +## Functions Limitation + +Function `Usage()` and `CumulativeUsage()` can only be used in the Metric resource. +For other functions listed above, users are also allowed to use them in ResourceUsage and ClusterResourceUsage +to build dynamic resource usage patterns. + +The reason behind is that when `kwok` evaluates functions `Usage()` or `CumulativeUsage()`, +it actually takes the simulation data given in ResourceUsage and ClusterResourceUsage to obtain metric values. +Therefore, please ensure that the associated ResourceUsage or ClusterResourceUsage with the needed resource types +(cpu or memory) are also provided when using function `Usage()` and `CumulativeUsage()`. + +## Variables Limitation + +When using the three special CEL variables `node`, `pod`, and `container` in Metric resource, you should follow the below rules. +* When `dimension` is `node`: only `node` variable can be used. +* When `dimension` is `pod`: only `node`, `pod` can be used. +* When `dimension` is `container`: `node`, `pod`, `container` all can be used. + + +[Metric]: {{< relref "/docs/generated/apis" >}}#kwok.x-k8s.io/v1alpha1.Metric +[ResourceUsage]: {{< relref "/docs/generated/apis" >}}#kwok.x-k8s.io/v1alpha1.ResourceUsage +[ClusterResourceUsage]: {{< relref "/docs/generated/apis" >}}#kwok.x-k8s.io/v1alpha1.ClusterResourceUsage +[the CEL language specification]: https://github.com/google/cel-spec/blob/master/doc/langdef.md +[CEL predefined functions]: https://github.com/google/cel-spec/blob/master/doc/langdef.md#list-of-standard-definitions diff --git a/site/content/en/docs/user/go-template.md b/site/content/en/docs/user/go-template.md new file mode 100644 index 000000000..51b6a27ba --- /dev/null +++ b/site/content/en/docs/user/go-template.md @@ -0,0 +1,26 @@ +--- +title: "Go Template in `kwok`" +--- + +# Notes on Go Template in `kwok` + + +The page provides a concise note on writing go templates in kwok CRs. + + +Currently, only `Stage` CR has go template based fields (`Spec.Next.StatusTemplate`). + + +You must follow [the go text template syntax] when writing the templates. +For predefined functions of go text template, please refer to [go text template functions]. +Besides the built-in functions, `kwok` also supports [sprig template functions]. + +It is worth noting that the "context" (which is denoted by the period character `.` ) to a template in `kwok` is set to the +referenced Kubernetes resource. +For example, you can use `.metadata.name` in a template to obtain the corresponding Kubernetes resource name. + + + +[the go text template syntax]: https://pkg.go.dev/text/template +[go text template functions]: https://pkg.go.dev/text/template#hdr-Functions +[sprig template functions]: https://masterminds.github.io/sprig/ diff --git a/site/content/en/docs/user/kwok-in-cluster.md b/site/content/en/docs/user/kwok-in-cluster.md index 5909ed287..7a2e38bfa 100644 --- a/site/content/en/docs/user/kwok-in-cluster.md +++ b/site/content/en/docs/user/kwok-in-cluster.md @@ -35,6 +35,21 @@ NOTE: This configures the pod/node emulation behavior, if not it will do nothing kubectl apply -f "https://github.com/${KWOK_REPO}/releases/download/${KWOK_LATEST_RELEASE}/stage-fast.yaml" ``` +## Set up default CRs of resource usage (optional) + +This allows to simulate the resource usage of nodes, pods and containers. + +``` bash +kubectl apply -f "https://github.com/${KWOK_REPO}/releases/download/${KWOK_LATEST_RELEASE}/metrics-usage.yaml" +``` + +The above configuration sets the CPU and memory usage of all the containers managed by `kwok` to `1m` and to `1Mi` respectively. +To override the defaults, you can add annotation `"kwok.x-k8s.io/usage-cpu"` (for cpu usage) and +`"kwok.x-k8s.io/usage-memory"` (for memory usage) with any quantity value you want to the fake pods. + +The resource usage simulation used above is annotation-based and the configuration is available at [here][resource usage from annotation]. +For the explanation of how it works and more complex resource usage simulation methods, please refer to [ResourceUsage configuration]. + ## Old way to deploy kwok Old way to deploy kwok is [here][kwok in cluster old]. @@ -45,3 +60,5 @@ Now, you can use `kwok` to [manage nodes and pods] in the Kubernetes cluster. [manage nodes and pods]: {{< relref "/docs/user/kwok-manage-nodes-and-pods" >}} [kwok in cluster old]: {{< relref "/docs/user/kwok-in-cluster-old" >}} +[resource usage from annotation]: https://github.com/kubernetes-sigs/kwok/tree/main/kustomize/metrics/usage +[ResourceUsage configuration]: {{< relref "/docs/user/resource-usage-configuration" >}} diff --git a/site/content/en/docs/user/metrics-configuration.md b/site/content/en/docs/user/metrics-configuration.md new file mode 100644 index 000000000..de70665f2 --- /dev/null +++ b/site/content/en/docs/user/metrics-configuration.md @@ -0,0 +1,108 @@ +--- +title: "Metrics" +--- + +# Metrics Configuration + +{{< hint "info" >}} + +This document walks you through how to configure the Metrics feature. + +{{< /hint >}} + +## What is a Metrics? + +The [Metrics] is a [`kwok` Configuration][configuration] that allows users to define and simulate metrics endpoints exposed by kubelet. + +The YAML below shows all the fields of a Metrics resource: + +``` yaml +kind: Metrics +apiVersion: kwok.x-k8s.io/v1alpha1 +metadata: + name: +spec: + path: + metrics: + - name: + help: + kind: + dimension: + labels: + - name: + value: + value: # for counter and gauge + buckets: # for histogram + - le: + value: + hidden: +``` + +There are total four metric-related endpoints in kubelet: `/metrics`, `/metrics/resource`, `/metrics/probe` and `/metrics/cadvisor`, +all of which are exposed with a Prometheus style. The Metrics resource is capable of simulating endpoints with such style. + +To simulate a metric endpoint, first, you need to specify the RESTful `path` of the endpoint, +which will be installed and exposed by the metric service of `kwok` at port `10247` after applied. +The `path` must start with `/metrics`, otherwise, `kwok` will not install it. + + +{{< hint "info" >}} +Starting from metrics-server 0.7.0, it is allowed to specify the path to scrape metrics for a node. +Specifically, metrics-server will check if a node has annotation `metrics.k8s.io/resource-metrics-path` +and use it as the target metric scrape path. +Combined with the Metric CR, the feature makes it possible to integrate `kwok` and metrics-server. +For a fake node, by adding that annotation and setting its value to the `path` +specified in a Metric resource, metrics-server will collect data from the endpoints exposed by `kwok` instead of +scrapping from kubelet. +{{< /hint >}} + +Besides, compared to kubelet, which only exposes the metric of the node it is located on, `kwok` needs to expose the +metrics of all the fake nodes it manages. Instead of creating a separate Metric CR for each fake node, it is possible +to bind all the metrics endpoints from different nodes into a single `path`. Metric CR allows for a built-in +`{nodeName}` path parameter to be included in the `path` field. For example: `/metrics/nodes/{nodeName}/metrics/resource`. +With `{nodeName}`, a single `path` is able to differentiate the metric data from different nodes. + + +The `metrics` field are used to customize the return body of the installed metrics endpoint. + +The descriptions of each sub-field are available at [Metric API][Metric]. +For readers' convenience, we also mirror the documents here with some additional notes. + +`metrics` is a list of specific configuration items, with each corresponding to a Prometheus style metric: +* `name` defines the metric name. +* `labels` defines the metric labels, with each item corresponding to a specific metric label. + - `name` is a const string that provides the label name. + - `value` is represented as a CEL expression that dynamically determines the label value. + For example: you can use `node.metadata.name` to reference the node name as the label value. +* `help` defines the help string of a metric. +* `kind` defines the type of the metric: `counter`, `gauge`, or `histogram`. +* `dimension` defines where the data comes from. It could be `node`, `pod`, or `container`. +* `value` is a CEL expression that defines the metric value if `kind` is `counter` or `gauge`. + Please refer to [CEL expressions in `kwok`] for more detailed instructions that might be helpful to simulate the metric value. +* `buckets` is exclusively for customizing the data of the metric of kind `histogram`. + - `le`, which defines the histogram bucket’s upper threshold, has the same meaning as the one of Prometheus histogram bucket. + That is, each bucket contains values less than or equal to `le`. + - `value` is a CEL expression that provides the value of the bucket. + - `hidden` indicates whether to show the bucket in the metric. + But the value of the bucket will be calculated and cumulated into the next bucket. + +Please refer to [Metrics for kubelet's "metrics/resource" endpoint][metrics resource endpoint] for a detailed example. + + +## Out-of-box Metric Config + +`kwok` currently provides the [Metrics config][metrics resource endpoint] that is capable of +simulating kubelet's `"metrics/resource"` endpoint. + +To integrate the simulated endpoint with metrics-server (required version >= 0.7.0), add the +`"metrics.k8s.io/resource-metrics-path": "/metrics/nodes//metrics/resource"` annotation to the fake +nodes managed by `kwok`. + + + + +[configuration]: {{< relref "/docs/user/configuration" >}} +[Metrics]: {{< relref "/docs/generated/apis" >}}#kwok.x-k8s.io/v1alpha1.Metrics +[CEL expressions in `kwok`]: {{< relref "/docs/user/cel-expressions" >}} +[metrics resource endpoint]: https://github.com/kubernetes-sigs/kwok/blob/main/kustomize/metrics/resource +[ResourceUsage (ClusterResourceUsage)]: {{< relref "/docs/user/resource-usage-configuration" >}} diff --git a/site/content/en/docs/user/resource-usage-configuration.md b/site/content/en/docs/user/resource-usage-configuration.md new file mode 100644 index 000000000..bffe52dcc --- /dev/null +++ b/site/content/en/docs/user/resource-usage-configuration.md @@ -0,0 +1,135 @@ +--- +title: ResourceUsage +--- + +# ResourceUsage Configuration + +{{< hint "info" >}} + +This document walks you through how to simulate the resource usage of pod(s). + +{{< /hint >}} + +## What is ResourceUsage? + +[ResourceUsage] is a [`kwok` Configuration][configuration] that allows users to define and simulate the resource usages of a single pod. + +The YAML below shows all the fields of a ResourceUsage resource: + +``` yaml +kind: ResourceUsage +apiVersion: kwok.x-k8s.io/v1alpha1 +metadata: + name: + namespace: +spec: + usages: + - containers: + - + usage: + cpu: + value: + expression: + memory: + value: + expression: +``` + +To associate a ResourceUsage with a certain pod to be simulated, users must ensure `metadata.name` and `metadata.namespace` +are inconsistent with the name and namespace of the target pod. + +The resource usages of a pod are specified via `usages` field. +The `usages` field are organized by groups, with each corresponding to a collection of containers that shares a same resource usage simulation setting. +Each group consists of a list of container names (`containers`) and the shared resource usage setting (`usage`). + +{{< hint "info" >}} +If `containers` is not given in a group, the `usage` in that group will be applied to all containers of the target pod. +{{< /hint >}} + +You can simply set a static [Quantity value] (`100Mi`, `1000m`, etc.) via `cpu.value` and `memory.value` to define the cpu and memory resource usage respectively. +Besides, users are also allowed to provide a [CEL expression] via `expressions` to describe the resource usage more flexibly. For example, +the following expression tries to extract the cpu resource usage from the pod's annotation if it has or use a default value. + +```yaml +expression: | + "kwok.x-k8s.io/usage-cpu" in pod.metadata.annotations + ? Quantity(pod.metadata.annotations["kwok.x-k8s.io/usage-cpu"]) + : Quantity("1m") +``` + +{{< hint "info" >}} +1. `value` has higher priority than `expressions` if both are set. +2. Quantity value must be explicitly wrapped by `Quantity` function in CEL expressions. +{{< /hint >}} + +With CEL expressions, it is even possible to simulate resource usages dynamically. For example, the following expression +yields memory usage that grows linearly with time. +```yaml +expression: (pod.SinceSecond() / 60.0) * Quantity("1Mi") +``` +Please refer to [CEL expressions in `kwok`] for an exhausted list that may be helpful to configure dynamic resource usage. + + +### ClusterResourceUsage + +In addition to simulating a single pod, users can also simulate the resource usage for multiple pods via [ClusterResourceUsage]. + +The YAML below shows all the fields of a ClusterResourceUsage resource: + +``` yaml +kind: ClusterResourceUsage +apiVersion: kwok.x-k8s.io/v1alpha1 +metadata: + name: +spec: + selector: + matchNamespaces: + - + matchNames: + - + usages: + - containers: + - + usage: + cpu: + value: + expression: + memory: + value: + expression: +``` +Compared to ResourceUsage, whose `metadata.name` and `metadata.namespace` are required to match the associated pod, +ClusterResourceUsage has an additional `selector` field for specifying the target pods to be simulated. +`matchNamespaces` and `matchNames` are both represented as list,which are designed to take pod collections by different levels: + +1. If `matchNamespaces` is empty, ClusterResourceUsage will be applied to all pods that are managed by `kwok` and whose names listed in `matchNames`. +2. If `matchNames` is empty, ClusterResourceUsage will be applied to all pods managed by `kwok` and under namespaces listed in `matchNamespaces`. +3. If `matchNames` and `matchNamespaces` are both unset, ClusterResourceUsage will be applied to all pods that `kwok` manages. + +The `usages` field of ClusterResourceUsage has the same semantic with the one in ResourceUsage. + +Please refer to [pod resource usage from annotation] for a concrete example. + +## Where to get the simulation data? + +The resource usages defined in ResourceUsage and ClusterResourceUsage resources can be fetched from the metric service of `kwok` at port `10247` with path `/metrics/nodes/{nodeName}/metrics/resource`, +where `{nodeName}` is the name of the fake node that the pod is scheduled to. +The returned metrics are similar to the response from kubelet's `/metrics/resource` endpoint. + +Please refer to [`kwok` Metric][Metric] about how to integrate `kwok` simulated metrics endpoints with metrics-server. + +## Dependencies + +ResourceUsage or ClusterResourceUsage only takes effect when the [Metric] feature is also enabled and +[the default Metric resource] that simulates kubelet's `/metrics/resource` endpoint is applied. + + +[configuration]: {{< relref "/docs/user/configuration" >}} +[ResourceUsage]: {{< relref "/docs/generated/apis" >}}#kwok.x-k8s.io/v1alpha1.ResourceUsage +[ClusterResourceUsage]: {{< relref "/docs/generated/apis" >}}#kwok.x-k8s.io/v1alpha1.ClusterResourceUsage +[Quantity value]: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-units-in-kubernetes +[CEL expression]: https://github.com/google/cel-spec/blob/master/doc/langdef.md +[Metric]: {{< relref "/docs/user/metrics-configuration" >}} +[the default Metric resource]: https://github.com/kubernetes-sigs/kwok/blob/main/kustomize/metrics/resource +[pod resource usage from annotation]: https://github.com/kubernetes-sigs/kwok/blob/main/kustomize/metrics/usage/usage-from-annotation.yaml +[CEL expressions in `kwok`]: {{< relref "/docs/user/cel-expressions" >}} diff --git a/site/content/en/docs/user/stages-configuration.md b/site/content/en/docs/user/stages-configuration.md index a469c6293..12bf7a113 100644 --- a/site/content/en/docs/user/stages-configuration.md +++ b/site/content/en/docs/user/stages-configuration.md @@ -65,8 +65,9 @@ users can specify the conditions that need to be met for the stage to be applied and the changes that will be made to the resource when the stage is applied. The `next` field allows users to define the new status of the resource using the `statusTemplate` field, and even `delete` the resource. -`statusTemplate` and `delete` are the two fundamental fields in `next` that respectively represent the two basic phases -of resource lifecycle simulation: status update and resource deletion. +`statusTemplate` and `delete` are the two fundamental fields in `next` that respectively represent the two basic phases of resource lifecycle simulation: status update and resource deletion. +`statusTemplate` provides a way to define resource status based on go template rendering. Please see +[go template in `kwok`] for more detailed instructions. `delete: true` has higher priority than a non-empty `statusTemplate`, which means `kwok` will delete the resource rather than update its status if both are set. @@ -257,3 +258,4 @@ This example shows how to configure the simplest and fastest stages of Pod resou [Stage API]: {{< relref "/docs/generated/apis" >}}#kwok.x-k8s.io/v1alpha1.Stage [Resource Lifecycle Simulation Controller]: {{< relref "/docs/design/architecture" >}} [How Delay is Calculated]: {{< relref "/docs/user/stages-configuration#how-delay-is-calculated" >}} +[go template in `kwok`]: {{< relref "/docs/user/go-template" >}} diff --git a/site/static/img/demo/resource-usage.svg b/site/static/img/demo/resource-usage.svg new file mode 120000 index 000000000..2492dab3c --- /dev/null +++ b/site/static/img/demo/resource-usage.svg @@ -0,0 +1 @@ +../../../../demo/resource-usage.svg \ No newline at end of file