diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bbddf02d..fa637a58 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,16 +10,16 @@ env: TEST_VERBOSITY: 2 jobs: - license-check: - name: License check + license-checks: + name: License checks runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v2 - - name: License check - run: make license_check + - name: Run checks + run: make license-checks unit-tests: name: Unit tests (${{ matrix.python-version }}/${{ matrix.os }}) @@ -116,7 +116,7 @@ jobs: TEST_STACK_VERSION: ${{ matrix.stack-version }} run: | mkdir ~/elastic-stack-cache - docker compose pull -q + make stack-pull docker save -o ~/elastic-stack-cache/elasticsearch-${{ matrix.stack-version }}.tar \ docker.elastic.co/elasticsearch/elasticsearch:${{ matrix.stack-version }} docker save -o ~/elastic-stack-cache/kibana-${{ matrix.stack-version }}.tar \ @@ -125,7 +125,7 @@ jobs: - name: Start Elastic Stack ${{ matrix.stack-version }} env: TEST_STACK_VERSION: ${{ matrix.stack-version }} - run: make up + run: make stack-up - name: Run online tests env: @@ -133,18 +133,108 @@ jobs: TEST_DETECTION_RULES_URI: ${{ matrix.detection-rules-uri }} TEST_SIGNALS_QUERIES: ${{ matrix.signals_queries }} TEST_SIGNALS_RULES: ${{ matrix.signals_rules }} - run: make online_tests + run: make online-tests - name: Stop Elastic Stack ${{ matrix.stack-version }} - run: make down + run: make stack-down + + docker-sanity: + name: Docker image sanity + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Install dependencies + run: make prereq + + - name: Build image + run: make docker-build + + - name: Run sanity checks + run: make docker-sanity + + package-sanity: + name: Python package sanity + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Install dependencies + run: make prereq + + - name: Build package + run: make pkg-build + + - name: Install package + run: make pkg-install + + - name: Try package + run: make pkg-try + + kubernetes-sanity: + name: Kubernetes sanity + runs-on: ${{ matrix.os }} + needs: + - docker-sanity + + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + + steps: + - name: Checkout code + uses: actions/checkout@v1 + + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: ">=1.16" + + - name: Install Kind + run: | + go install sigs.k8s.io/kind@latest + kind version + + - name: Build image + run: make docker-build + + - name: Create cluster + run: | + make kind-up + kubectl get nodes -o wide + kubectl get pods -o wide + kubectl get services -o wide + + - name: Run sanity checks + run: make sanity-checks + + - name: Destroy Kind cluster + run: make kind-down publish: name: Publish runs-on: ubuntu-latest needs: - - license-check + - license-checks - unit-tests - online-tests + - kubernetes-sanity + - package-sanity if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') steps: @@ -160,13 +250,13 @@ jobs: run: make prereq - name: Build package - run: make pkg_build + run: make pkg-build - name: Install package - run: make pkg_install + run: make pkg-install - name: Try package - run: make pkg_try + run: make pkg-try - name: Publish to TestPyPI uses: pypa/gh-action-pypi-publish@v1.5.0 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..3bd31658 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +# syntax=docker/dockerfile:1 + +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM python:alpine +WORKDIR /app + +COPY requirements.txt requirements.txt +RUN pip3 install --user -r requirements.txt + +COPY etc etc +COPY geneve geneve + +EXPOSE 5000 + +ENV FLASK_APP=geneve/webapi.py +CMD [ "python3", "-m" , "flask", "run", "--host=0.0.0.0", "-p 5000" ] diff --git a/Makefile b/Makefile index 77537898..ac8653aa 100644 --- a/Makefile +++ b/Makefile @@ -16,16 +16,55 @@ lint: tests: tests/*.py $(PYTHON) -m pytest -raP tests/test_*.py -online_tests: tests/*.py +online-tests: tests/*.py $(PYTHON) -m pytest -raP tests/test_emitter_*.py -up: - docker compose up --wait --quiet-pull +sanity-checks: + for n in `seq 30`; do \ + curl -s --fail http://localhost:30000/api/v1/version && exit 0 || sleep 1; \ +done; exit 1 -down: - docker compose down +stack-pull: + cd tests && docker compose pull -q -license_check: +stack-up: + cd tests && docker compose up --wait --quiet-pull + +stack-down: + cd tests && docker compose down + +docker-build: + -docker image rm geneve + docker build -q -t geneve . + +docker-run: + docker run -p 127.0.0.1:30000:5000 --rm --name geneve geneve + +docker-sanity: GENEVE_VERSION=$(shell $(PYTHON) -c "import geneve; print(geneve.version)") +docker-sanity: + docker run -p 127.0.0.1:30000:5000 --rm --name geneve-test -d geneve + [ "`$(MAKE) -s sanity-checks`" = '{"version":"$(GENEVE_VERSION)"}' ] || \ + (docker container stop geneve-test; exit 1) + docker container stop geneve-test + +docker-push: GENEVE_VERSION=$(shell $(PYTHON) -c "import geneve; print(geneve.version)") +docker-push: + docker tag geneve:latest $(DOCKER_REGISTRY)/geneve:latest + docker tag geneve:latest $(DOCKER_REGISTRY)/geneve:$(GENEVE_VERSION) + docker push -q $(DOCKER_REGISTRY)/geneve:latest + docker push -q $(DOCKER_REGISTRY)/geneve:$(GENEVE_VERSION) + docker image rm $(DOCKER_REGISTRY)/geneve:latest $(DOCKER_REGISTRY)/geneve:$(GENEVE_VERSION) + +kind-up: + kind create cluster --config=etc/kind-config.yml + kind load docker-image geneve + kubectl apply -f etc/pods/geneve.yml + kubectl apply -f etc/services/geneve.yml + +kind-down: + kind delete cluster + +license-checks: bash scripts/license_check.sh run: @@ -33,17 +72,20 @@ run: $(PYTHON) -m geneve --help $(PYTHON) -m geneve -pkg_build: +flask: + FLASK_APP=geneve/webapi.py $(PYTHON) -m flask run + +pkg-build: $(PYTHON) -m build -pkg_install: +pkg-install: $(PYTHON) -m pip install --force-reinstall dist/geneve-*.whl -pkg_try: +pkg-try: geneve --version geneve --help geneve -package: pkg_build pkg_install pkg_try +package: pkg-build pkg-install pkg-try -.PHONY: lint tests online_tests run up down +.PHONY: lint tests online-tests run flask stack-up stack-down license-checks package docker docker-sanity diff --git a/etc/kind-config.yml b/etc/kind-config.yml new file mode 100644 index 00000000..e3af0f7d --- /dev/null +++ b/etc/kind-config.yml @@ -0,0 +1,9 @@ +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +nodes: + - role: control-plane + extraPortMappings: + - containerPort: 30000 + hostPort: 30000 + - role: worker + - role: worker diff --git a/etc/pods/geneve.yml b/etc/pods/geneve.yml new file mode 100644 index 00000000..1f6d6784 --- /dev/null +++ b/etc/pods/geneve.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: geneve + labels: + app: geneve +spec: + containers: + - name: geneve + image: geneve + imagePullPolicy: Never diff --git a/etc/services/geneve.yml b/etc/services/geneve.yml new file mode 100644 index 00000000..8e75d1ab --- /dev/null +++ b/etc/services/geneve.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: geneve +spec: + type: NodePort + selector: + app: geneve + ports: + - port: 5000 + nodePort: 30000 diff --git a/geneve/webapi.py b/geneve/webapi.py new file mode 100644 index 00000000..c701ed9c --- /dev/null +++ b/geneve/webapi.py @@ -0,0 +1,92 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys +import logging +from itertools import islice + +from . import version +from .events_emitter import SourceEvents +from .utils import root_dir, load_schema, load_rules + +from flask import Flask, request, jsonify +app = Flask("geneve") +app.config.from_prefixed_env("GENEVE") + +logging.basicConfig(level=logging.DEBUG) + +rule_tags = app.config.get("RULE_TAGS", "") +if rule_tags: + rule_tags = set(x.strip().lower() for x in rule_tags.split(",") if x.strip()) + if rule_tags: + app.logger.info("Rule tags: {}".format(", ".join(sorted(rule_tags)))) + +schema_uri = app.config.get("SCHEMA_URI", "./etc/ecs-8.1.0.tar.gz") +app.logger.debug(f"Loading {schema_uri}...") +schema = load_schema(schema_uri, "generated/ecs/ecs_flat.yml", root_dir) + +detection_rules_uri = app.config.get("DETECTION_RULES_URI", "./etc/detection-rules-8.1.0.tar.gz") +app.logger.debug(f"Loading {detection_rules_uri}...") +rules = load_rules(detection_rules_uri, "rules/**/*.toml", root_dir) + +source_events = SourceEvents(schema) +loaded_rules = [] +for rule in rules: + if not rule_tags or rule_tags.issubset(x.lower() for x in rule.tags): + try: + source_events.add_rule(rule) + loaded_rules.append(rule) + rule.path = str(rule.path) + except Exception as e: + app.logger.warning(f"{e}: {rule.path}") + continue + +if not source_events: + app.logger.error(f"Examined {len(rules)} rules, none was loaded.") + sys.exit(1) + +app.logger.info(f"Loaded {len(source_events)} rules") + + +@app.route("/api/v1/version", methods=["GET"]) +def get_version(): + ret = { + "version": version + } + return jsonify(ret) + + +@app.route("/api/v1/rules", methods=["GET"]) +def get_rules(): + return jsonify([vars(x) for x in loaded_rules]) + + +@app.route("/api/v1/query", methods=["GET"]) +def query(): + query = request.args.get("query") + count = request.args.get("count", default=1, type=int) + source_events = SourceEvents(schema) + source_events.add_query(query) + docs = [event.doc for events in islice(source_events, count) for event in events] + return jsonify(docs) + + +@app.route("/api/v1/emit", methods=["GET"]) +def emit(): + count = request.args.get("count", default=1, type=int) + docs = [event.doc for events in islice(source_events, count) for event in events] + return jsonify(docs) diff --git a/requirements.txt b/requirements.txt index c6d04789..e8ba2e88 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ click eql>=0.9.12 elasticsearch flake8 +flask nbformat pytest pytoml diff --git a/setup.cfg b/setup.cfg index 8dcd0e9c..fa849a04 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,6 +27,7 @@ license_file = LICENSE.txt packages = geneve geneve.kql + geneve.utils install_requires = click eql>=0.9.12 @@ -35,6 +36,10 @@ install_requires = requests python_requires = >=3.8.0 +[options.extras_require] +webapi = + flask + [options.entry_points] console_scripts = geneve = geneve.cli:main diff --git a/docker-compose.yml b/tests/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to tests/docker-compose.yml