Skip to content

Commit

Permalink
Merge pull request #53 from magicsword-io/feat/multi-sigma-versions
Browse files Browse the repository at this point in the history
split into frontend and backend to support multiple sigma versions in parallel
josehelps authored Nov 6, 2024
2 parents 20166da + 2619363 commit be9f4e0
Showing 20 changed files with 957 additions and 919 deletions.
32 changes: 17 additions & 15 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
@@ -4,6 +4,8 @@ on:
push:
branches:
- main
schedule:
- cron: "0 0 * * 0"

env:
SERVICE_NAME: sigconverter
@@ -14,23 +16,23 @@ jobs:
runs-on: ubuntu-latest

steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Checkout code
uses: actions/checkout@v3

- name: Setup Google Cloud SDK
uses: google-github-actions/setup-gcloud@v0.2.1
with:
project_id: ${{ secrets.PROJECT_ID }}
service_account_key: ${{ secrets.GCLOUD_AUTH }}
- name: Setup Google Cloud SDK
uses: google-github-actions/setup-gcloud@v0.2.1
with:
project_id: ${{ secrets.PROJECT_ID }}
service_account_key: ${{ secrets.GCLOUD_AUTH }}

- name: Configure Docker
run: gcloud auth configure-docker
- name: Configure Docker
run: gcloud auth configure-docker

- name: Build Docker image
run: docker build -t ${{ env.IMAGE_NAME }} .
- name: Build Docker image
run: docker build -t ${{ env.IMAGE_NAME }} .

- name: Push Docker image to Google Container Registry
run: gcloud builds submit --tag gcr.io/${{ secrets.PROJECT_ID }}/${{ env.IMAGE_NAME }}
- name: Push Docker image to Google Container Registry
run: gcloud builds submit --tag gcr.io/${{ secrets.PROJECT_ID }}/${{ env.IMAGE_NAME }}

- name: Deploy to Google Cloud Run
run: gcloud run deploy ${{ env.SERVICE_NAME }} --image gcr.io/${{ secrets.PROJECT_ID }}/${{ env.IMAGE_NAME }} --platform managed --region us-central1
- name: Deploy to Google Cloud Run
run: gcloud run deploy ${{ env.SERVICE_NAME }} --image gcr.io/${{ secrets.PROJECT_ID }}/${{ env.IMAGE_NAME }} --platform managed --region us-central1
29 changes: 17 additions & 12 deletions .github/workflows/packages-test.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Backend Packages Test
name: Frontend Packages Test

on: # yamllint disable-line rule:truthy
push:
@@ -12,18 +12,23 @@ on: # yamllint disable-line rule:truthy
- main

jobs:
test-poetry-package:
pip-package:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3.3.0
with:
submodules: true
- uses: actions/checkout@v3.3.0
with:
submodules: true

- name: Set up Python 3.11
uses: actions/setup-python@v4.5.0
with:
python-version: 3.11
- name: Set up Python 3.11
uses: actions/setup-python@v4.5.0
with:
python-version: 3.11

- name: Test poetry package installation
run: |
python -m pip install poetry && poetry install
- name: Install the latest version of uv
uses: astral-sh/setup-uv@v3
with:
version: "latest"

- name: Test uv package installation
run: uv venv && uv pip sync pyproject.toml
working-directory: frontend
35 changes: 11 additions & 24 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,30 +1,17 @@
# Use the specified Python version
FROM python:3.11.4-slim-buster

# Configure Poetry
ENV POETRY_VERSION=1.6.1
ENV POETRY_HOME=/opt/poetry
ENV POETRY_VENV=/opt/poetry-venv
ENV POETRY_CACHE_DIR=/opt/.cache
# install dependencies
RUN apt-get update
RUN apt-get install -y git curl jq
COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv

# Install poetry separated from system interpreter
RUN python3 -m venv $POETRY_VENV \
&& $POETRY_VENV/bin/pip install -U pip setuptools \
&& $POETRY_VENV/bin/pip install poetry==${POETRY_VERSION}

# Add `poetry` to PATH
ENV PATH="${PATH}:${POETRY_VENV}/bin"

# Set the working directory
WORKDIR /app

# Install dependencies
COPY poetry.lock pyproject.toml ./
RUN poetry install

# Copy the flask app to the working directory
# define work directory
WORKDIR /app/
COPY . /app

# Run the application
# install backend
RUN cd backend && ./setup-sigma-versions.sh

# launch front- and backend
EXPOSE 8000
CMD [ "poetry", "run", "python", "./run.py" ]
ENTRYPOINT ["./entrypoint.sh"]
83 changes: 50 additions & 33 deletions run.py → backend/backend.py
Original file line number Diff line number Diff line change
@@ -4,8 +4,8 @@
import os
import yaml
import base64
import json
from flask import Flask, jsonify, render_template, request, Response
import importlib.metadata as metadata
from flask import Flask, jsonify, request, Response

from sigma.conversion.base import Backend
from sigma.plugins import InstalledSigmaPlugins
@@ -20,43 +20,59 @@
backends = plugins.backends
pipeline_resolver = plugins.get_pipeline_resolver()
pipelines = list(pipeline_resolver.list_pipelines())
pipelines_names = [p[0] for p in pipelines]


@app.route("/")
def home():
formats = []
for backend in backends.keys():
for name, description in plugins.backends[backend].formats.items():
formats.append(
{"name": name, "description": description, "backend": backend}
@app.route("/api/v1/targets", methods=["GET"])
def get_targets():
response = []
for name, backend in backends.items():
response.append(
{"name": name, "description": backend.name}
)

for name, pipeline in pipelines:
if len(pipeline.allowed_backends) > 0:
pipeline.backends = ", ".join(pipeline.allowed_backends)
else:
pipeline.backends = "all"

return render_template(
"index.html", backends=backends, pipelines=pipelines, formats=formats
)


@app.route("/getpipelines", methods=["GET"])
return jsonify(response)

@app.route("/api/v1/formats", methods=["GET"])
def get_formats():
args = request.args
response = []
if len(args) == 0:
for backend in backends.keys():
for name, description in plugins.backends[backend].formats.items():
response.append(
{"name": name, "description": description, "target": backend}
)
elif "target" in args:
target = args.get("target")
for backend in backends.keys():
if backend == target:
for name, description in plugins.backends[backend].formats.items():
response.append(
{"name": name, "description": description}
)

return jsonify(response)

@app.route("/api/v1/pipelines", methods=["GET"])
def get_pipelines():
return jsonify(pipelines_names)


@app.route("/sigma", methods=["POST"])
args = request.args
response = []
if len(args) == 0:
for name, pipeline in pipelines:
response.append({"name": name, "targets": list(pipeline.allowed_backends)})
elif "target" in args:
target = args.get("target")
for name, pipeline in pipelines:
if (len(pipeline.allowed_backends) == 0) or (target in pipeline.allowed_backends):
response.append({"name": name, "targets": list(pipeline.allowed_backends)})
return jsonify(response)


@app.route("/api/v1/convert", methods=["POST"])
def convert():
# get params from request
rule = str(base64.b64decode(request.json["rule"]), "utf-8")
# check if input is valid yaml
try:
yaml.safe_load(rule)
except:
print("error")
return Response(
f"YamlError: Malformed yaml file", status=400, mimetype="text/html"
)
@@ -84,7 +100,7 @@ def convert():
try:
backend_class = backends[target]
except:
return Response(f"Unknown Backend", status=400, mimetype="text/html")
return Response(f"Unknown Target", status=400, mimetype="text/html")

try:
processing_pipeline = pipeline_resolver.resolve(pipeline)
@@ -109,6 +125,7 @@ def convert():

return result


if __name__ == "__main__":
app.run(host="0.0.0.0", port=int(os.environ.get("PORT", 8000)))
current_version = metadata.version("sigma-cli")
port = int(f'8{current_version.replace(".","")}')
app.run(host="0.0.0.0", port=int(os.environ.get("PORT", port)))
13 changes: 13 additions & 0 deletions backend/launch-backends.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#!/bin/bash

# Specify the directory to search in (or use the current directory)
directory="./"

# Iterate over all subdirectories
for dir in "$directory"/*/; do
if [ -d "$dir" ]; then
version=$(basename $dir)
echo "Launching sigconverter backend for sigma version: $version"
./$version/.venv/bin/python ./backend.py &
fi
done
11 changes: 11 additions & 0 deletions backend/pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
[project]
name = "sigconverter-backend"
version = "1.0.0"
description = "backend for the sigconverter projects"
readme = "README.md"
requires-python = ">=3.10"
authors = [{ name = "Magic Sword", email = "info@magicsword.io" }]
dependencies = [
"flask>=3.0.3",
"setuptools>=75.1.0",
]
28 changes: 28 additions & 0 deletions backend/setup-sigma-versions.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#!/bin/bash

# fetch 10 latest versions of sigma-cli
SIGMA_VERSIONS=$(curl -s https://pypi.org/pypi/sigma-cli/json | jq -r '.releases | keys | .[-10:] | .[]')

# prepare virtualenv for each version
for VERSION in $SIGMA_VERSIONS; do
# prepare folder to contain a single version
mkdir $VERSION
cp pyproject.toml uv.lock $VERSION
cd $VERSION
uv venv && uv -q pip sync pyproject.toml

# fetch all plugins from plugin directory json and install latest compatible plugins available
uv -q add sigma-cli==$VERSION
curl https://raw.githubusercontent.com/SigmaHQ/pySigma-plugin-directory/refs/heads/main/pySigma-plugins-v1.json | jq '.plugins[].package' | xargs -n 1 uv add -q

# remove if installed because of https://github.com/redsand/pySigma-backend-hawk/issues/1
uv -q remove pySigma-backend-hawk

# TODO: some problems with kusto backend, disable for now
uv -q remove pySigma-backend-kusto

# remove unused pyparsing imports in older version, see https://github.com/SigmaHQ/pySigma/pull/289#issuecomment-2410153076
find ./ -iwholename "*sigma/conversion/base.py" -exec sed -i "/from pyparsing import Set/d" {} +
find ./ -iwholename "*sigma/exceptions.py" -exec sed -i "/from pyparsing import List/d" {} +
cd ..
done
163 changes: 163 additions & 0 deletions backend/uv.lock

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions entrypoint.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
#!/bin/sh

cd backend/ && ./launch-backends.sh && cd ..
cd frontend && uv run frontend.py
68 changes: 68 additions & 0 deletions frontend/frontend.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import re
import requests
from flask import Flask, render_template, request, jsonify

app = Flask(__name__)
sigma_versions = [
os.path.basename(it.path) for it in os.scandir("../backend/") if it.is_dir()
]


def version_key(version):
return tuple(map(int, version.split(".")))


def get_port_from_version(version):
pattern = r"^\d+\.\d+\.\d+$"
if re.match(pattern, version):
return int(f'8{version.replace(".", "")}')
else:
return None


@app.route("/")
def home():
return render_template("index.html")


@app.route("/api/v1/sigma-versions", methods=["GET"])
def get_versions():
return jsonify(sorted(sigma_versions, key=version_key, reverse=True))


@app.route("/api/v1/<version>/targets", methods=["GET"])
def get_targets(version):
port = get_port_from_version(version)
return requests.get(
f"http://localhost:{port}/api/v1/targets", params=dict(request.args)
).json()


@app.route("/api/v1/<version>/formats", methods=["GET"])
def get_formats(version):
port = get_port_from_version(version)
return requests.get(
f"http://localhost:{port}/api/v1/formats", params=dict(request.args)
).json()


@app.route("/api/v1/<version>/pipelines", methods=["GET"])
def get_pipelines(version):
port = get_port_from_version(version)
return requests.get(
f"http://localhost:{port}/api/v1/pipelines", params=dict(request.args)
).json()


@app.route("/api/v1/<version>/convert", methods=["POST"])
def convert(version):
port = get_port_from_version(version)
payload = request.json
return requests.post(f"http://localhost:{port}/api/v1/convert", json=payload).text


if __name__ == "__main__":
app.run(host="0.0.0.0", port=int(os.environ.get("PORT", 8000)))
8 changes: 8 additions & 0 deletions frontend/pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
[project]
name = "sigconverter-frontend"
version = "1.0.0"
description = "frontend for the sigconverter projects"
readme = "README.md"
requires-python = ">=3.10"
dependencies = ["flask>=3.0.3", "requests>=2.32.3"]
authors = [{ name = "Magic Sword", email = "info@magicsword.io" }]
File renamed without changes.
File renamed without changes.
File renamed without changes
323 changes: 265 additions & 58 deletions static/js/index.js → frontend/static/js/index.js

Large diffs are not rendered by default.

54 changes: 37 additions & 17 deletions templates/index.html → frontend/templates/index.html
Original file line number Diff line number Diff line change
@@ -47,35 +47,35 @@
<div class="lg:col-span-1 self-center">
<div class="grid md:grid-cols-1 lg:grid-cols-3 gap-4 sm:p-4">
<div class="lg:px-1">
<label for="select-backend" class="text-sigma-blue">Backend:</label>
<select id="select-backend" class="select-sigma" autocomplete="off">
{%- for name, backend in backends.items() -%}
<option value="{{ name }}">{{ name }}</option>
{%- endfor -%}
<label for="select-backend" class="text-sigma-blue">Target:</label>
<select id="select-backend" class="select-sigma" placeholder="target selection" autocomplete="off">
</select>
</div>
<div class="lg:px-1">
<label for="select-format" class="text-sigma-blue">Format:</label>
<select id="select-format" class="select-sigma" autocomplete="off">
{%- for format in formats -%}
<option hidden backend="{{ format.backend }}" value="{{ format.name }}">{{ format.name }}</option>
{%- endfor -%}
<select id="select-format" class="select-sigma" placeholder="format selection" autocomplete="off">
</select>
</div>
<div class="lg:px-1">
<label for="select-pipeline" class="text-sigma-blue">Pipeline:</label>
<select id="select-pipeline" class="select-sigma" name="pipeline[]" placeholder="select pipelines..." multiple autocomplete="off">
{%- for name, pipeline in pipelines -%}
<option hidden value="{{ name }}" backend="{{ pipeline.backends }}">{{ name }}</option>
{%- endfor -%}
</select>
</div>
</div>
</div>
<div class="lg:col-span-1 lg:px-2">
<label for="cli-code" class="text-sigma-blue">CLI:</label>
<pre class="border border-sigma-blue"><code id="cli-code" class="language-bash text-sm">run this cli command for the same result</code></pre>

<div class="grid grid-cols-12 content-center">
<div class="col-span-11 lg:px-2">
<label for="cli-code" class="text-sigma-blue">CLI:</label>
<pre class="border border-sigma-blue"><code id="cli-code" class="language-bash text-sm">run this cli command for the same result</code></pre>
</div>
<!-- <div class="col-span-1"> -->
<span class="col-span-1 text-white self-center pl-3 pt-3 cursor-pointer">
<i id="settings-btn" class="fas fa-cog text-2xl"></i>
</span>
<!-- </div> -->
</div>

</div>
</div>
<div id="content-section" class="mx-10 mt-5">
@@ -100,8 +100,8 @@
- Internal Research
- https://tools.thehacker.recipes/mimikatz/modules
author: Florian Roth (rule), David ANDRE (additional keywords)
date: 2021/12/20
modified: 2022/04/27
date: 2021-12-20
modified: 2022-04-27
logsource:
category: process_creation
product: windows
@@ -184,6 +184,26 @@

</div>

<!-- Setting Modal -->
<div id="settings-modal" class="fixed inset-0 bg-gray-800 bg-opacity-50 flex items-center justify-center hidden">
<div class="bg-sigma-dark border-sigma-blue border rounded-lg shadow-lg w-1/3 p-6 relative">
<!-- Modal Close Button (Top Right Corner) -->
<button id="settings-close-btn" class="absolute top-2 right-2 text-sigma-blue hover:text-white text-2xl font-bold">&times;</button>

<!-- Modal Header -->
<div class="mb-4">
<h2 class="text-xl text-sigma-blue font-semibold">Settings</h2>
</div>

<!-- Modal Body with Select Form -->
<div class="mb-4">
<label for="select-sigma-version" class="block text-white font-medium mb-2">Select a sigma-cli version:</label>
<select id="select-sigma-version" class="select-sigma">
</select>
</div>
</div>
</div>

<!-- tom-select -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/tom-select/2.2.2/js/tom-select.complete.js" integrity="sha512-KfTOBVJv8qnV1b+2tsbTLepS7+RAgmVV0Odk6cj1eHxbR8WFX99gwIWOutwFAUlsve3FaGG3VxoPWWLRnehX1w==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<!-- PrismJS -->
265 changes: 265 additions & 0 deletions frontend/uv.lock

Large diffs are not rendered by default.

726 changes: 0 additions & 726 deletions poetry.lock

This file was deleted.

34 changes: 0 additions & 34 deletions pyproject.toml

This file was deleted.

Binary file removed static/.DS_Store
Binary file not shown.

0 comments on commit be9f4e0

Please sign in to comment.