-
Notifications
You must be signed in to change notification settings - Fork 28
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #53 from magicsword-io/feat/multi-sigma-versions
split into frontend and backend to support multiple sigma versions in parallel
Showing
20 changed files
with
957 additions
and
919 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,30 +1,17 @@ | ||
# Use the specified Python version | ||
FROM python:3.11.4-slim-buster | ||
|
||
# Configure Poetry | ||
ENV POETRY_VERSION=1.6.1 | ||
ENV POETRY_HOME=/opt/poetry | ||
ENV POETRY_VENV=/opt/poetry-venv | ||
ENV POETRY_CACHE_DIR=/opt/.cache | ||
# install dependencies | ||
RUN apt-get update | ||
RUN apt-get install -y git curl jq | ||
COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv | ||
|
||
# Install poetry separated from system interpreter | ||
RUN python3 -m venv $POETRY_VENV \ | ||
&& $POETRY_VENV/bin/pip install -U pip setuptools \ | ||
&& $POETRY_VENV/bin/pip install poetry==${POETRY_VERSION} | ||
|
||
# Add `poetry` to PATH | ||
ENV PATH="${PATH}:${POETRY_VENV}/bin" | ||
|
||
# Set the working directory | ||
WORKDIR /app | ||
|
||
# Install dependencies | ||
COPY poetry.lock pyproject.toml ./ | ||
RUN poetry install | ||
|
||
# Copy the flask app to the working directory | ||
# define work directory | ||
WORKDIR /app/ | ||
COPY . /app | ||
|
||
# Run the application | ||
# install backend | ||
RUN cd backend && ./setup-sigma-versions.sh | ||
|
||
# launch front- and backend | ||
EXPOSE 8000 | ||
CMD [ "poetry", "run", "python", "./run.py" ] | ||
ENTRYPOINT ["./entrypoint.sh"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
#!/bin/bash | ||
|
||
# Specify the directory to search in (or use the current directory) | ||
directory="./" | ||
|
||
# Iterate over all subdirectories | ||
for dir in "$directory"/*/; do | ||
if [ -d "$dir" ]; then | ||
version=$(basename $dir) | ||
echo "Launching sigconverter backend for sigma version: $version" | ||
./$version/.venv/bin/python ./backend.py & | ||
fi | ||
done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
[project] | ||
name = "sigconverter-backend" | ||
version = "1.0.0" | ||
description = "backend for the sigconverter projects" | ||
readme = "README.md" | ||
requires-python = ">=3.10" | ||
authors = [{ name = "Magic Sword", email = "info@magicsword.io" }] | ||
dependencies = [ | ||
"flask>=3.0.3", | ||
"setuptools>=75.1.0", | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
#!/bin/bash | ||
|
||
# fetch 10 latest versions of sigma-cli | ||
SIGMA_VERSIONS=$(curl -s https://pypi.org/pypi/sigma-cli/json | jq -r '.releases | keys | .[-10:] | .[]') | ||
|
||
# prepare virtualenv for each version | ||
for VERSION in $SIGMA_VERSIONS; do | ||
# prepare folder to contain a single version | ||
mkdir $VERSION | ||
cp pyproject.toml uv.lock $VERSION | ||
cd $VERSION | ||
uv venv && uv -q pip sync pyproject.toml | ||
|
||
# fetch all plugins from plugin directory json and install latest compatible plugins available | ||
uv -q add sigma-cli==$VERSION | ||
curl https://raw.githubusercontent.com/SigmaHQ/pySigma-plugin-directory/refs/heads/main/pySigma-plugins-v1.json | jq '.plugins[].package' | xargs -n 1 uv add -q | ||
|
||
# remove if installed because of https://github.com/redsand/pySigma-backend-hawk/issues/1 | ||
uv -q remove pySigma-backend-hawk | ||
|
||
# TODO: some problems with kusto backend, disable for now | ||
uv -q remove pySigma-backend-kusto | ||
|
||
# remove unused pyparsing imports in older version, see https://github.com/SigmaHQ/pySigma/pull/289#issuecomment-2410153076 | ||
find ./ -iwholename "*sigma/conversion/base.py" -exec sed -i "/from pyparsing import Set/d" {} + | ||
find ./ -iwholename "*sigma/exceptions.py" -exec sed -i "/from pyparsing import List/d" {} + | ||
cd .. | ||
done |
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
#!/bin/sh | ||
|
||
cd backend/ && ./launch-backends.sh && cd .. | ||
cd frontend && uv run frontend.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,68 @@ | ||
#!/usr/bin/env python | ||
# -*- coding: utf-8 -*- | ||
import os | ||
import re | ||
import requests | ||
from flask import Flask, render_template, request, jsonify | ||
|
||
app = Flask(__name__) | ||
sigma_versions = [ | ||
os.path.basename(it.path) for it in os.scandir("../backend/") if it.is_dir() | ||
] | ||
|
||
|
||
def version_key(version): | ||
return tuple(map(int, version.split("."))) | ||
|
||
|
||
def get_port_from_version(version): | ||
pattern = r"^\d+\.\d+\.\d+$" | ||
if re.match(pattern, version): | ||
return int(f'8{version.replace(".", "")}') | ||
else: | ||
return None | ||
|
||
|
||
@app.route("/") | ||
def home(): | ||
return render_template("index.html") | ||
|
||
|
||
@app.route("/api/v1/sigma-versions", methods=["GET"]) | ||
def get_versions(): | ||
return jsonify(sorted(sigma_versions, key=version_key, reverse=True)) | ||
|
||
|
||
@app.route("/api/v1/<version>/targets", methods=["GET"]) | ||
def get_targets(version): | ||
port = get_port_from_version(version) | ||
return requests.get( | ||
f"http://localhost:{port}/api/v1/targets", params=dict(request.args) | ||
).json() | ||
|
||
|
||
@app.route("/api/v1/<version>/formats", methods=["GET"]) | ||
def get_formats(version): | ||
port = get_port_from_version(version) | ||
return requests.get( | ||
f"http://localhost:{port}/api/v1/formats", params=dict(request.args) | ||
).json() | ||
|
||
|
||
@app.route("/api/v1/<version>/pipelines", methods=["GET"]) | ||
def get_pipelines(version): | ||
port = get_port_from_version(version) | ||
return requests.get( | ||
f"http://localhost:{port}/api/v1/pipelines", params=dict(request.args) | ||
).json() | ||
|
||
|
||
@app.route("/api/v1/<version>/convert", methods=["POST"]) | ||
def convert(version): | ||
port = get_port_from_version(version) | ||
payload = request.json | ||
return requests.post(f"http://localhost:{port}/api/v1/convert", json=payload).text | ||
|
||
|
||
if __name__ == "__main__": | ||
app.run(host="0.0.0.0", port=int(os.environ.get("PORT", 8000))) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
[project] | ||
name = "sigconverter-frontend" | ||
version = "1.0.0" | ||
description = "frontend for the sigconverter projects" | ||
readme = "README.md" | ||
requires-python = ">=3.10" | ||
dependencies = ["flask>=3.0.3", "requests>=2.32.3"] | ||
authors = [{ name = "Magic Sword", email = "info@magicsword.io" }] |
File renamed without changes.
File renamed without changes.
File renamed without changes
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
Binary file not shown.