From 52b8728e5c0ea1ba65c0730c12bce985b64f8bab Mon Sep 17 00:00:00 2001 From: chenk Date: Wed, 8 May 2024 13:25:53 +0300 Subject: [PATCH] feat: node-collector platfrom auto detection (#116) * feat: node-collector platfrom auto detection Signed-off-by: chenk --- CONTRIBUTING.md | 133 ++++++++++++ Makefile | 10 +- README.md | 102 ++++++++- go.mod | 1 + go.sum | 2 + pkg/collector/cluster.go | 41 +++- pkg/collector/collect.go | 108 ++++++---- pkg/collector/collect_test.go | 105 ++++++++++ pkg/collector/config/config.yaml | 193 +++++++++--------- .../k8s-cis-1.23.0.yaml} | 2 +- pkg/collector/info.go | 15 +- pkg/collector/testdata/fixture/mapping.yaml | 30 +++ pkg/command/k8s.go | 2 +- pkg/command/root.go | 8 +- 14 files changed, 599 insertions(+), 153 deletions(-) create mode 100644 CONTRIBUTING.md rename pkg/collector/config/{k8s/cis-123.yaml => specs/k8s-cis-1.23.0.yaml} (99%) create mode 100644 pkg/collector/testdata/fixture/mapping.yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1af484e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,133 @@ +# Contributing + +These guidelines will help you get started with the k8s-Node-Collector project. + +## Table of Contents + +- [Contributing](#contributing) + - [Table of Contents](#table-of-contents) + - [Contribution Workflow](#contribution-workflow) + - [Issues and Discussions](#issues-and-discussions) + - [Pull Requests](#pull-requests) + - [Conventional Commits](#conventional-commits) + - [Set up your Development Environment](#set-up-your-development-environment) + - [Build Binaries](#build-binaries) + - [running node-collector binary](#running-node-collector-binary) + - [Testing](#testing) + - [Run unit Tests](#run-unit-tests) + +## Contribution Workflow + +### Issues and Discussions + +- Feel free to open issues for any reason as long as you make it clear what this issue is about: bug/feature/proposal/comment. +- For questions and general discussions, please do not open an issue, and instead create a discussion in the "Discussions" tab. +- Please spend a minimal amount of time giving due diligence to existing issues or discussions. Your topic might be a duplicate. If it is, please add your comment to the existing one. +- Please give your issue or discussion a meaningful title that will be clear for future users. +- The issue should clearly explain the reason for opening, the proposal if you have any, and any relevant technical information. +- For technical questions, please explain in detail what you were trying to do, provide an error message if applicable, and your versions of k8s-node-collector and your environment. + +### Pull Requests + +- Every Pull Request should have an associated Issue unless it is a trivial fix. +- Your PR is more likely to be accepted if it focuses on just one change. +- Describe what the PR does. There's no convention enforced, but please try to be concise and descriptive. Treat the PR description as a commit message. Titles that start with "fix"/"add"/"improve"/"remove" are good examples. +- There's no need to add or tag reviewers, if your PR is left unattended for too long, you can add a comment to bring it up to attention, optionally "@" mention one of the maintainers that was involved with the issue. +- If a reviewer commented on your code or asked for changes, please remember to mark the discussion as resolved after you address it and re-request a review. +- When addressing comments, try to fix each suggestion in a separate commit. +- Tests are not required at this point as k8s-node-collector is evolving fast, but if you can include tests that will be appreciated. + +#### Conventional Commits + +It is not that strict, but we use the [Conventional commits](https://www.conventionalcommits.org) in this repository. +Each commit message doesn't have to follow conventions as long as it is clear and descriptive since it will be squashed and merged. + +## Set up your Development Environment + +- Install Go + + The project requires [Go 1.22.2][go-download] or later. We also assume that you're familiar with + Go's [GOPATH workspace][go-code] convention, and have the appropriate environment variables set. +- Get the source code: + +```sh +git clone git@github.com:aquasecurity/k8s-node-collector.git +cd k8s-node-collector +``` + +- Access to a Kubernetes cluster. We assume that you're using a [KIND][kind] cluster. To create a single-node KIND + cluster, run: + +```sh +kind create cluster +``` + +## Build Binaries + +| Binary | Image | Description | +|----------------------|------------------------------------------------|---------------------------------------------------------------| +| `node-collector` | `ghcr.io/aquasecurity/node-collector:dev` | k8s-node-collector | + +To build node-collector binary, run: + +```sh +make build +``` + +### running node-collector binary + +when running node-collector binary it will run cis-spec based on version mapping define in [config.yaml](./pkg/collector/config/config.yaml) +or you can define you on spec by using the flag `--spec k8s-cis` and `--version 1.23.0` + +```sh +node-collector --help + +A tool which provide a way to extract file info which is not accessible via pre-define commands + +Usage: + node-collector [flags] + node-collector [command] + +Available Commands: + completion Generate the autocompletion script for the specified shell + help Help about any command + k8s k8s-node-collector extract file system info from cluster Node + +Flags: + -h, --help help for node-collector + -n, --node string node name + -o, --output string Output format. One of table|json (default "json") + -s, --spec string spec name. example: k8s-cis + -v, --version string spec version. example 1.23.0 +``` + +This uses the `go build` command and builds binaries in the `./bin` directory. + +To build all k8s-node-collector binary into Docker images, run: + +copy `node-collector` binary to ./build/node-collector + +```sh + mv ./cmd/node-collector/node-collector ./build/node-collector/node-collector +``` + +build docker image + +```sh +make build:docker +``` + +## Testing + +We generally require tests to be added for all, but the most trivial of changes. However, unit tests alone don't +provide guarantees about the behaviour of k8s-node-collector. To verify that each Go module correctly interacts with its +collaborators, more coarse grained integration tests might be required. + +### Run unit Tests + +To run all tests with code coverage enabled, run: + +```sh +make test +``` + diff --git a/Makefile b/Makefile index c9504ec..48e5622 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ GOTEST=$(GOCMD) test all: - $(info "completed running make file for go-opa-validate") + $(info "completed running make file for k8s node collector") fmt: @go fmt ./... tidy: @@ -14,4 +14,10 @@ tidy: test: $(GOTEST) ./... -.PHONY: install-req fmt lint tidy test imports . \ No newline at end of file +build: + cd ./cmd/node-collector && go build -o node-collector main.go + +build-docker: + docker build -t ghcr.io/aquasecurity/node-collector:dev . + +.PHONY: install-req fmt lint tidy test imports . diff --git a/README.md b/README.md index 3228bdd..4f8c8b9 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,107 @@ + +# k8s-node-collector + [![GitHub Release][release-img]][release] [![Build Action][action-build-img]][action-build] [![Release snapshot Action][action-release-snapshot-img]][action-release-snapshot] [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/aquasecurity/k8s-node-collector/blob/main/LICENSE) -# k8s-node-collector +The k8s-Node-collector is an open-source collector that gathers Node information (file system and process data) from Kubernetes nodes and outputs it in a JSON format. -k8s-Node-collector is an open source collector who collect Node information (fs and process data) and output in a table/json format. +## Executing Collector specifications -## k8s-node-collector as job in k8s +The node-collector executes a collector specification,example [k8s-cis-1.23.0](./pkg/collector/config/specs/k8s-cis-1.23.0.yaml). +Each specification must include: -- simple k8s cluster run following job +- `name:` any other platfrom used, example (k8s-cis, aks-cis, gke-cis and etc) +- `version:` of the cis-benchmark it represent (example: 1.23.0) + +for executing a specific spec need to pass the `--spec k8s-cis` and `--version 1.23.0` flags + +If no collector spec has been specified. the node-collector will try to auto detect the matching spec by platrom type and version as define in [version_mapping data](./pkg/collector/config/config.yaml) +example: + +```yaml +k8s: + - op: "=" + cluster_version: "1.21" + spec: k8s-cis-1.21.0 + - op: ">" + cluster_version: "1.21" + spec: k8s-cis-1.23.0 +``` + +In the example provided, there are two rules; the first matching rule will obtain the appropriate specification. +Any native k8s cluser with version equal to 1.21 will obtain the `k8s-cis-1.21.0` collector specification it no match found +any native k8s cluser with version grather to 1.21 will obtain the `k8s-cis-1.23.0` + +## Adding new collector specifications + +In order to Add a new specifications, add a new yaml file to this path : `.pkg/collector/config/specs/` +with the following file naming convesion <`platform`-`cis`-`spec_version`> +example: `gke-cis-1.24.0` + +Each collector specification audit includes the following fields + +```yaml +--- +version: "1.23.0" +name: aks-cis +title: Node Specification for info collector +collectors: + - key: < name to hold the audit command output> + title: + nodeType: <node type - master | worker> + audit: <audit shell command> +``` + +### General spec data +`name` - name of the spec (example: `aks-cis`) +`version` - version of the spec (example: `1.23.0`) +`title` - short description of the overall spec + +### Specific audit data + +`key` - parameter name to hold the audit shell command output +`title` - title of the audit shell command +`nodeType` - define the node type on which shell command should be executed (master | worker) +`audit` - a shell command that collect information and return the result (errors must be supressed) + +## Config file + +The k8s-node-collector use a config file which help to obtain binaries and config files path based on different platfrom (rancher, natinv k8s and etc) +for example: + +```yaml +kubelet: + bins: + - kubelet + - hyperkube kubelet + confs: + - /etc/kubernetes/kubelet-config.yaml + - /var/lib/kubelet/config.yaml +``` + +The node collector will obtain the kubelet binary name and config file path based on the platfrom it runs on. +when writing an `audit` shell command the params from config files can be used to collect the appropriate data via config params +example, collect the kubelet config.yaml configuration file ownership: + +```sh +stat -c %U:%G $kubelet.confs ``` + +## Run s k8s job + +- simple k8s cluster run following job + +```sh kubectl apply -f job.yaml ``` - Check k8s pod status -``` +```sh kubectl get pods NAME READY STATUS RESTARTS AGE @@ -26,10 +110,12 @@ node-collector-ng2z7 0/1 Completed 0 6m1 - Check k8s pod audit output -``` +```sh kubectl logs node-collector-ng2z7 ``` +## k8s-node-collector output + - json output ```json @@ -314,9 +400,9 @@ kubectl logs node-collector-ng2z7 } ``` -* job cleanup +### job cleanup -``` +```sh kubectl delete -f job.yaml ``` diff --git a/go.mod b/go.mod index 56675ae..2dee036 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.22.0 toolchain go1.22.2 require ( + github.com/Masterminds/semver v1.5.0 github.com/olekukonko/tablewriter v0.0.5 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index ce9ba31..7f610a2 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= diff --git a/pkg/collector/cluster.go b/pkg/collector/cluster.go index a7fd080..7d07a55 100644 --- a/pkg/collector/cluster.go +++ b/pkg/collector/cluster.go @@ -17,6 +17,17 @@ import ( "k8s.io/client-go/tools/clientcmd" ) +const ( + native = "k8s" + gke = "gke" + aks = "aks" + eks = "eks" + rke2 = "rke2" + k3s = "k3s" + ocp = "ocp" + microk8s = "microk8s" +) + type Cluster struct { clientSet *kubernetes.Clientset cConfig clientcmd.ClientConfig @@ -64,11 +75,30 @@ func (cluster *Cluster) Platfrom() (Platform, error) { if len(v) != 0 { return Platform{Name: "ocp", Version: majorVersion(v)}, nil } - version, err := cluster.clientSet.ServerVersion() + nodeName := cluster.getNodeName() + semVersion, err := cluster.clientSet.ServerVersion() if err != nil { return Platform{}, err } - return getPlatformInfoFromVersion(version.GitVersion), nil + p := getPlatformInfoFromVersion(semVersion.GitVersion) + var name string + switch { + case strings.Contains(p.Version, k3s): + name = k3s + case strings.Contains(p.Version, rke2): + name = rke2 + case strings.Contains(p.Version, microk8s): + name = microk8s + case strings.Contains(nodeName, aks): + name = aks + case strings.Contains(nodeName, eks): + name = eks + case strings.Contains(nodeName, gke): + name = gke + default: + name = "k8s" + } + return Platform{Name: name, Version: p.Version}, nil } func getPlatformInfoFromVersion(s string) Platform { @@ -103,6 +133,13 @@ func (cluster *Cluster) getOpenShiftVersion(ctx context.Context) string { } return version } +func (cluster *Cluster) getNodeName() string { + nodes, err := cluster.clientSet.CoreV1().Nodes().List(context.Background(), v1.ListOptions{}) + if err != nil { + return "k8s" + } + return nodes.Items[0].Name +} func (cluster *Cluster) getDynamicClient(gvr schema.GroupVersionResource) dynamic.ResourceInterface { return cluster.dynamicClient.Resource(gvr).Namespace("") diff --git a/pkg/collector/collect.go b/pkg/collector/collect.go index 6ac14d5..83efb57 100644 --- a/pkg/collector/collect.go +++ b/pkg/collector/collect.go @@ -2,6 +2,7 @@ package collector import ( "context" + "encoding/base64" "encoding/json" "errors" "os" @@ -14,25 +15,18 @@ import ( "strconv" + "github.com/Masterminds/semver" "github.com/spf13/cobra" ) -type SpecVersion struct { - Name string - Version string -} - -var platfromSpec = map[string]SpecVersion{ - "k8s-1.23": { - Name: "k8s-cis", - Version: "1.23", - }, -} +const ( + // Version is the version of the output + defaultSpec = "k8s-cis-1.23.0" +) // CollectData run spec audit command and output it result data -func CollectData(cmd *cobra.Command, target string) error { +func CollectData(cmd *cobra.Command) error { log.SetFlags(log.LstdFlags | log.Lmicroseconds) - cluster, err := GetCluster() if err != nil { return err @@ -45,10 +39,6 @@ func CollectData(cmd *cobra.Command, target string) error { log.Println("Increase --timeout value") } }() - p, err := cluster.Platfrom() - if err != nil { - return err - } shellCmd := NewShellCmd() nodeType, err := shellCmd.FindNodeType() if err != nil { @@ -59,19 +49,17 @@ func CollectData(cmd *cobra.Command, target string) error { return err } cm := configParams(lp, shellCmd) - infoCollectorMap, err := LoadConfig(target, cm) + infoCollectorMap, err := LoadConfig(cm) if err != nil { return err } - specName := cmd.Flag("spec").Value.String() - specVersion := cmd.Flag("version").Value.String() - sv := SpecVersion{Name: specName, Version: specVersion} - if len(sv.Name) == 0 || len(sv.Version) == 0 { - sv = specByPlatfromVersion(p) + sv, err := specID(cmd, cluster, lp) + if err != nil { + return err } for _, infoCollector := range infoCollectorMap { nodeInfo := make(map[string]*Info) - if !(infoCollector.Version == sv.Version && infoCollector.Name == sv.Name) { + if fmt.Sprintf("%s-%s", infoCollector.Name, infoCollector.Version) != sv { continue } for _, ci := range infoCollector.Collectors { @@ -86,14 +74,17 @@ func CollectData(cmd *cobra.Command, target string) error { nodeInfo[ci.Key] = &Info{Values: values} } nodeName := cmd.Flag("node").Value.String() - nodeConfig, err := loadNodeConfig(ctx, *cluster, nodeName) - if err == nil { - mapping, err := LoadKubeletMapping() - if err != nil { - return err + kubeletConfig := cmd.Flag("kubelet-config").Value.String() + if nodeName != "" || kubeletConfig != "" { + nodeConfig, err := loadNodeConfig(ctx, *cluster, nodeName, kubeletConfig) + if err == nil { + mapping, err := LoadKubeletMapping() + if err != nil { + return err + } + configVal := getValuesFromkubeletConfig(nodeConfig, mapping) + mergeConfigValues(nodeInfo, configVal) } - configVal := getValuesFromkubeletConfig(nodeConfig, mapping) - mergeConfigValues(nodeInfo, configVal) } nodeData := Node{ APIVersion: Version, @@ -111,8 +102,37 @@ func CollectData(cmd *cobra.Command, target string) error { return nil } -func loadNodeConfig(ctx context.Context, cluster Cluster, nodeName string) (map[string]interface{}, error) { - data, err := cluster.clientSet.RESTClient().Get().AbsPath(fmt.Sprintf("/api/v1/nodes/%s/proxy/configz", nodeName)).DoRaw(ctx) +func specID(cmd *cobra.Command, cluster *Cluster, lp *Config) (string, error) { + specName := cmd.Flag("spec-name").Value.String() + specVersion := cmd.Flag("spec-version").Value.String() + clusterVersion := cmd.Flag("cluster-version").Value.String() + switch { + case specName != "" && specVersion != "": + return fmt.Sprintf("%s-%s", specName, specVersion), nil + case specName != "" && clusterVersion != "": + return specByPlatfromVersion( + Platform{ + Name: strings.TrimSuffix(specName, "-cis"), + Version: majorVersion(clusterVersion), + }, + lp.VersionMapping), nil + default: // auto detect spec by platform type (k8s, aks, eks and etc) and version + p, err := cluster.Platfrom() + if err != nil { + return "", err + } + return specByPlatfromVersion(p, lp.VersionMapping), nil + } +} + +func loadNodeConfig(ctx context.Context, cluster Cluster, nodeName string, kubeletConfig string) (map[string]interface{}, error) { + var data []byte + var err error + if kubeletConfig != "" { + data, err = base64.StdEncoding.DecodeString(kubeletConfig) + } else { + data, err = cluster.clientSet.RESTClient().Get().AbsPath(fmt.Sprintf("/api/v1/nodes/%s/proxy/configz", nodeName)).DoRaw(ctx) + } if err != nil { return nil, err } @@ -124,8 +144,26 @@ func loadNodeConfig(ctx context.Context, cluster Cluster, nodeName string) (map[ return nodeConfig, nil } -func specByPlatfromVersion(platfrom Platform) SpecVersion { - return platfromSpec[fmt.Sprintf("%s-%s", platfrom.Name, platfrom.Version)] +func specByPlatfromVersion(platfrom Platform, versionSpecMapper map[string][]SpecVersion) string { + speVersions, ok := versionSpecMapper[platfrom.Name] + if ok { + for _, cisVer := range speVersions { + c, err := semver.NewConstraint(fmt.Sprintf("%s %s", cisVer.Op, cisVer.Version)) + if err != nil { + // default to basic k8s spec + return defaultSpec + } + v, err := semver.NewVersion(platfrom.Version) + if err != nil { + // default to basic k8s spec + return defaultSpec + } + if ok, _ = c.Validate(v); ok { + return cisVer.CisSpec + } + } + } + return defaultSpec } func getValuesFromkubeletConfig(nodeConfig map[string]interface{}, configMapper map[string]string) map[string]*Info { diff --git a/pkg/collector/collect_test.go b/pkg/collector/collect_test.go index 7890716..c5209d6 100644 --- a/pkg/collector/collect_test.go +++ b/pkg/collector/collect_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/kyaml/yaml" ) func TestParseNodeConfig(t *testing.T) { @@ -67,3 +68,107 @@ func TestParseNodeConfig(t *testing.T) { }) } } + +func TestSpecByVersionName(t *testing.T) { + tests := []struct { + name string + versionMappingfile string + platfrom Platform + wantSpec string + }{ + { + name: "k8s cis spec", + versionMappingfile: "./testdata/fixture/mapping.yaml", + platfrom: Platform{Name: "k8s", Version: "1.21"}, + wantSpec: "k8s-cis-1.23.0", + }, + { + name: "aks cis spec", + versionMappingfile: "./testdata/fixture/mapping.yaml", + platfrom: Platform{Name: "aks", Version: "1.21"}, + wantSpec: "aks-cis-1.0.0", + }, + { + name: "eks cis spec", + versionMappingfile: "./testdata/fixture/mapping.yaml", + platfrom: Platform{Name: "eks", Version: "1.21"}, + wantSpec: "eks-cis-1.2.0", + }, + { + name: "gke cis spec", + versionMappingfile: "./testdata/fixture/mapping.yaml", + platfrom: Platform{Name: "gke", Version: "1.21"}, + wantSpec: "gke-cis-1.2.0", + }, + { + name: "rke2 cis spec", + versionMappingfile: "./testdata/fixture/mapping.yaml", + platfrom: Platform{Name: "rke2", Version: "1.21"}, + wantSpec: "rke2-cis-1.24.0", + }, + { + name: "ocp cis spec", + versionMappingfile: "./testdata/fixture/mapping.yaml", + platfrom: Platform{Name: "ocp", Version: "4.0"}, + wantSpec: "rh-cis-1.0", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f, err := os.ReadFile(tt.versionMappingfile) + assert.NoError(t, err) + var config Config + err = yaml.Unmarshal(f, &config) + assert.NoError(t, err) + gotSpec := specByPlatfromVersion(tt.platfrom, config.VersionMapping) + assert.Equal(t, gotSpec, tt.wantSpec) + }) + } +} + +func TestPlatfromVersion(t *testing.T) { + tests := []struct { + name string + version string + want string + }{ + { + name: "k8s version", + version: "v1.23.2", + want: "1.23", + }, + { + name: "eks version", + version: "v1.23.17-eks-8ccc7ba", + want: "1.23", + }, + { + name: "aks version", + version: "v1.23.17-eks-8ccc7ba", + want: "1.23", + }, + { + name: "gke version", + version: "v1.23.10-gke.2300", + want: "1.23", + }, + { + name: "rke2 version", + version: "v1.23.11+rke2r1", + want: "1.23", + }, + { + name: "ocp version", + version: "v1.23.15+c763d11", + want: "1.23", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getPlatformInfoFromVersion(tt.version) + assert.Equal(t, got.Version, tt.want) + }) + } +} diff --git a/pkg/collector/config/config.yaml b/pkg/collector/config/config.yaml index 5cefba3..7027c12 100644 --- a/pkg/collector/config/config.yaml +++ b/pkg/collector/config/config.yaml @@ -11,7 +11,6 @@ node: - /etc/kubernetes/manifests/talos-kube-apiserver.yaml - /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml defaultconf: /etc/kubernetes/manifests/kube-apiserver.yaml - controllermanager: confs: - /etc/kubernetes/manifests/kube-controller-manager.yaml @@ -27,7 +26,6 @@ node: - /var/lib/kube-controller-manager/kubeconfig - /system/secrets/kubernetes/kube-controller-manager/kubeconfig defaultkubeconfig: /etc/kubernetes/controller-manager.conf - scheduler: confs: - /etc/kubernetes/manifests/kube-scheduler.yaml @@ -62,85 +60,73 @@ node: - /var/lib/rancher/k3s/server/db/etcd/config defaultconf: /etc/kubernetes/manifests/etcd.yaml defaultdatadir: /var/lib/etcd/default.etcd - flanneld: defaultconf: /etc/sysconfig/flanneld - kubernetes: - defaultconf: "/etc/kubernetes/config" - + defaultconf: /etc/kubernetes/config kubelet: cafile: - - "/etc/kubernetes/pki/ca.crt" - - "/etc/kubernetes/certs/ca.crt" - - "/etc/kubernetes/cert/ca.pem" - - "/var/snap/microk8s/current/certs/ca.crt" - - "/var/lib/rancher/rke2/agent/server.crt" - - "/var/lib/rancher/rke2/agent/client-ca.crt" - - "/var/lib/rancher/k3s/agent/client-ca.crt" + - /etc/kubernetes/pki/ca.crt + - /etc/kubernetes/certs/ca.crt + - /etc/kubernetes/cert/ca.pem + - /var/snap/microk8s/current/certs/ca.crt + - /var/lib/rancher/rke2/agent/server.crt + - /var/lib/rancher/rke2/agent/client-ca.crt + - /var/lib/rancher/k3s/agent/client-ca.crt svc: - # These paths must also be included - # in the 'confs' property below - - "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" - - "/etc/systemd/system/kubelet.service" - - "/lib/systemd/system/kubelet.service" - - "/etc/systemd/system/snap.kubelet.daemon.service" - - "/etc/systemd/system/snap.microk8s.daemon-kubelet.service" - - "/etc/systemd/system/atomic-openshift-node.service" - - "/etc/systemd/system/origin-node.service" + - /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + - /etc/systemd/system/kubelet.service + - /lib/systemd/system/kubelet.service + - /etc/systemd/system/snap.kubelet.daemon.service + - /etc/systemd/system/snap.microk8s.daemon-kubelet.service + - /etc/systemd/system/atomic-openshift-node.service + - /etc/systemd/system/origin-node.service bins: - - "hyperkube kubelet" - - "kubelet" + - hyperkube kubelet + - kubelet kubeconfig: - - "/etc/kubernetes/kubelet.conf" - - "/etc/kubernetes/kubelet-kubeconfig.conf" - - "/var/lib/kubelet/kubeconfig" - - "/etc/kubernetes/kubelet-kubeconfig" - - "/etc/kubernetes/kubelet/kubeconfig" - - "/etc/kubernetes/ssl/kubecfg-kube-node.yaml" - - "/var/snap/microk8s/current/credentials/kubelet.config" - - "/etc/kubernetes/kubeconfig-kubelet" - - "/var/lib/rancher/rke2/agent/kubelet.kubeconfig" - - "/var/lib/rancher/k3s/server/cred/admin.kubeconfig" - - "/var/lib/rancher/k3s/agent/kubelet.kubeconfig" + - /etc/kubernetes/kubelet.conf + - /etc/kubernetes/kubelet-kubeconfig.conf + - /var/lib/kubelet/kubeconfig + - /etc/kubernetes/kubelet-kubeconfig + - /etc/kubernetes/kubelet/kubeconfig + - /etc/kubernetes/ssl/kubecfg-kube-node.yaml + - /var/snap/microk8s/current/credentials/kubelet.config + - /etc/kubernetes/kubeconfig-kubelet + - /var/lib/rancher/rke2/agent/kubelet.kubeconfig + - /var/lib/rancher/k3s/server/cred/admin.kubeconfig + - /var/lib/rancher/k3s/agent/kubelet.kubeconfig confs: - - "/etc/kubernetes/kubelet-config.yaml" - - "/var/lib/kubelet/config.yaml" - - "/var/lib/kubelet/config.yml" - - "/etc/kubernetes/kubelet/kubelet-config.json" - - "/etc/kubernetes/kubelet/config" - - "/home/kubernetes/kubelet-config.yaml" - - "/home/kubernetes/kubelet-config.yml" - - "/etc/default/kubeletconfig.json" - - "/etc/default/kubelet" - - "/var/lib/kubelet/kubeconfig" - - "/var/snap/kubelet/current/args" - - "/var/snap/microk8s/current/args/kubelet" - ## Due to the fact that the kubelet might be configured - ## without a kubelet-config file, we use a work-around - ## of pointing to the systemd service file (which can also - ## hold kubelet configuration). - ## Note: The following paths must match the one under 'svc' - - "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" - - "/etc/systemd/system/kubelet.service" - - "/lib/systemd/system/kubelet.service" - - "/etc/systemd/system/snap.kubelet.daemon.service" - - "/etc/systemd/system/snap.microk8s.daemon-kubelet.service" - - "/etc/kubernetes/kubelet.yaml" - - "/var/lib/rancher/rke2/agent/kubelet.kubeconfig" - - defaultconf: "/var/lib/kubelet/config.yaml" - defaultsvc: "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" - defaultkubeconfig: "/etc/kubernetes/kubelet.conf" - defaultcafile: "/etc/kubernetes/pki/ca.crt" - + - /etc/kubernetes/kubelet-config.yaml + - /var/lib/kubelet/config.yaml + - /var/lib/kubelet/config.yml + - /etc/kubernetes/kubelet/kubelet-config.json + - /etc/kubernetes/kubelet/config + - /home/kubernetes/kubelet-config.yaml + - /home/kubernetes/kubelet-config.yml + - /etc/default/kubeletconfig.json + - /etc/default/kubelet + - /var/lib/kubelet/kubeconfig + - /var/snap/kubelet/current/args + - /var/snap/microk8s/current/args/kubelet + - /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + - /etc/systemd/system/kubelet.service + - /lib/systemd/system/kubelet.service + - /etc/systemd/system/snap.kubelet.daemon.service + - /etc/systemd/system/snap.microk8s.daemon-kubelet.service + - /etc/kubernetes/kubelet.yaml + - /var/lib/rancher/rke2/agent/kubelet.kubeconfig + defaultconf: /var/lib/kubelet/config.yaml + defaultsvc: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + defaultkubeconfig: /etc/kubernetes/kubelet.conf + defaultcafile: /etc/kubernetes/pki/ca.crt proxy: bins: - - "kube-proxy" - - "hyperkube proxy" - - "hyperkube kube-proxy" - - "proxy" - - "openshift start network" + - kube-proxy + - hyperkube proxy + - hyperkube kube-proxy + - proxy + - openshift start network confs: - /etc/kubernetes/proxy - /etc/kubernetes/addons/kube-proxy-daemonset.yaml @@ -148,33 +134,46 @@ node: - /var/snap/kube-proxy/current/args - /var/snap/microk8s/current/args/kube-proxy kubeconfig: - - "/etc/kubernetes/kubelet-kubeconfig" - - "/etc/kubernetes/kubelet-kubeconfig.conf" - - "/etc/kubernetes/kubelet/config" - - "/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml" - - "/var/lib/kubelet/kubeconfig" - - "/var/snap/microk8s/current/credentials/proxy.config" - - "/var/lib/rancher/rke2/agent/kubeproxy.kubeconfig" - - "/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig" + - /etc/kubernetes/kubelet-kubeconfig + - /etc/kubernetes/kubelet-kubeconfig.conf + - /etc/kubernetes/kubelet/config + - /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + - /var/lib/kubelet/kubeconfig + - /var/snap/microk8s/current/credentials/proxy.config + - /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + - /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig svc: - - "/lib/systemd/system/kube-proxy.service" - - "/etc/systemd/system/snap.microk8s.daemon-proxy.service" + - /lib/systemd/system/kube-proxy.service + - /etc/systemd/system/snap.microk8s.daemon-proxy.service defaultconf: /etc/kubernetes/addons/kube-proxy-daemonset.yaml - defaultkubeconfig: "/etc/kubernetes/proxy.conf" - + defaultkubeconfig: /etc/kubernetes/proxy.conf +# op: "=", ">", "<", ">=", "<=" version_mapping: - "1.23": "cis-1.23" - "1.24": "cis-1.24" - "1.25": "cis-1.7" - "1.26": "cis-1.8" - "eks-1.2.0": "eks-1.2.0" - "gke-1.0": "gke-1.0" - "gke-1.2.0": "gke-1.2.0" - "ocp-4.0": "rh-1.0" - "aks-1.0": "aks-1.0" - "ack-1.0": "ack-1.0" - "cis-1.6-k3s": "cis-1.6-k3s" - "cis-1.24-microk8s": "cis-1.24-microk8s" - "tkgi-1.2.53": "tkgi-1.2.53" - "k3s-cis-1.24": "k3s-cis-1.24" - "rke2-cis-1.24": "rke2-cis-1.24" + k8s: + - op: ">=" + cluster_version: "1.21" + spec: k8s-cis-1.23.0 + gke: + - op: ">=" + cluster_version: "1.21" + spec: gke-cis-1.2.0 + eks: + - op: ">=" + cluster_version: "1.21" + spec: eks-cis-1.2.0 + aks: + - op: ">=" + cluster_version: "1.21" + spec: aks-cis-1.0 + rke2: + - op: ">=" + cluster_version: "1.21" + spec: rke2-cis-1.24 + microk8s: + - op: ">=" + cluster_version: "1.21" + spec: microk8s-cis-1.24 + ocp: + - op: ">=" + cluster_version: "4.0" + spec: rh-cis-1.0 diff --git a/pkg/collector/config/k8s/cis-123.yaml b/pkg/collector/config/specs/k8s-cis-1.23.0.yaml similarity index 99% rename from pkg/collector/config/k8s/cis-123.yaml rename to pkg/collector/config/specs/k8s-cis-1.23.0.yaml index 65e8f85..a51b98c 100644 --- a/pkg/collector/config/k8s/cis-123.yaml +++ b/pkg/collector/config/specs/k8s-cis-1.23.0.yaml @@ -1,5 +1,5 @@ --- -version: "1.23" +version: "1.23.0" name: k8s-cis title: Node Specification for info collector collectors: diff --git a/pkg/collector/info.go b/pkg/collector/info.go index 06d935a..9dfccfb 100644 --- a/pkg/collector/info.go +++ b/pkg/collector/info.go @@ -16,15 +16,15 @@ const ( Kind = "NodeInfo" ) -//go:embed config/k8s +//go:embed config/specs var config embed.FS //go:embed config var params embed.FS // LoadConfig load audit commands specification from config file -func LoadConfig(target string, configMap map[string]string) (map[string]*SpecInfo, error) { - fullPath := fmt.Sprintf("%s/%s", configFolder, target) +func LoadConfig(configMap map[string]string) (map[string]*SpecInfo, error) { + fullPath := fmt.Sprintf("%s/%s", configFolder, "specs") dirEntries, err := config.ReadDir(fullPath) if err != nil { return nil, err @@ -121,9 +121,16 @@ type Info struct { } type Config struct { - Node NodeParams `yaml:"node"` + Node NodeParams `yaml:"node"` + VersionMapping map[string][]SpecVersion `yaml:"version_mapping"` } +type SpecVersion struct { + Name string + Version string `yaml:"cluster_version"` + Op string `yaml:"op"` + CisSpec string `yaml:"spec"` +} type NodeParams struct { APIserver Params `yaml:"apiserver"` ControllerManager Params `yaml:"controllermanager"` diff --git a/pkg/collector/testdata/fixture/mapping.yaml b/pkg/collector/testdata/fixture/mapping.yaml new file mode 100644 index 0000000..b56e6ef --- /dev/null +++ b/pkg/collector/testdata/fixture/mapping.yaml @@ -0,0 +1,30 @@ +--- +version_mapping: + k8s: + - op: ">=" + cluster_version: "1.21" + spec: k8s-cis-1.23.0 + gke: + - op: ">=" + cluster_version: "1.21" + spec: gke-cis-1.2.0 + eks: + - op: ">=" + cluster_version: "1.21" + spec: eks-cis-1.2.0 + aks: + - op: ">=" + cluster_version: "1.21" + spec: aks-cis-1.0.0 + rke2: + - op: ">=" + cluster_version: "1.21" + spec: rke2-cis-1.24.0 + microk8s: + - op: ">=" + cluster_version: "1.21" + spec: microk8s-cis-1.24.0 + ocp: + - op: ">=" + cluster_version: "4.0" + spec: rh-cis-1.0 diff --git a/pkg/command/k8s.go b/pkg/command/k8s.go index 4c2f3dd..33bdc56 100644 --- a/pkg/command/k8s.go +++ b/pkg/command/k8s.go @@ -19,7 +19,7 @@ var k8sCmd = &cobra.Command{ Long: `A tool which provide a way to extract k8s info which is not accessible via apiserver from node cluster based on pre-define commands`, RunE: func() func(cmd *cobra.Command, args []string) error { return func(cmd *cobra.Command, args []string) error { - return collector.CollectData(cmd, subCommandK8s) + return collector.CollectData(cmd) } }(), } diff --git a/pkg/command/root.go b/pkg/command/root.go index a6996de..860c81f 100644 --- a/pkg/command/root.go +++ b/pkg/command/root.go @@ -9,9 +9,11 @@ import ( func init() { rootCmd.PersistentFlags().StringP("output", "o", "json", "Output format. One of table|json") - rootCmd.PersistentFlags().StringP("spec", "s", "k8s-cis", " spec name. default: k8s-cis") - rootCmd.PersistentFlags().StringP("version", "v", "1.23", "spec version. default: 1.23") - rootCmd.PersistentFlags().StringP("node", "n", "minikube", "node name. default: minikube") + rootCmd.PersistentFlags().StringP("spec-name", "s", "", "spec name. example: k8s-cis") + rootCmd.PersistentFlags().StringP("spec-version", "v", "", "spec version. example 1.23.0") + rootCmd.PersistentFlags().StringP("cluster-version", "c", "", "cluser version. example 1.23.0") + rootCmd.PersistentFlags().StringP("node", "n", "", "node name") + rootCmd.PersistentFlags().StringP("kubelet-config", "", "", "kubelet config via api /api/v1/nodes/<>/proxy/configz encoded in base64") } var rootCmd = &cobra.Command{