From ad0a57d2b090e4427f6d0612a95472c8cf689a3c Mon Sep 17 00:00:00 2001 From: ngn Date: Fri, 5 Apr 2024 23:35:38 +0300 Subject: [PATCH] database/api rework --- .github/workflows/publish.yml | 6 +- .gitignore | 3 +- README.md | 105 ++++++++-------- api/.gitignore | 4 + api/Dockerfile | 19 +++ api/lib/__init__.py | 99 +++++++++++++++ api/main.py | 53 ++++++++ api/requirements.txt | 3 + assets/web.png | Bin 26265 -> 0 bytes database/.gitignore | 2 - database/Dockerfile | 13 -- database/go.mod | 18 --- database/go.sum | 27 ---- database/lib/db.go | 87 ------------- database/lib/routes.go | 76 ----------- database/main.go | 47 ------- database/static/index.css | 117 ----------------- database/static/index.html | 23 ---- database/static/index.js | 54 -------- scanner/.clang-format | 225 +++++++++++++++++++++++++++++++++ scanner/Dockerfile | 5 +- scanner/Makefile | 5 +- scanner/inc/api.h | 11 ++ scanner/inc/db.h | 12 -- scanner/inc/log.h | 24 ++-- scanner/inc/net.h | 136 ++++++++++---------- scanner/inc/op.h | 19 ++- scanner/inc/pool.h | 28 +++++ scanner/inc/util.h | 24 ++-- scanner/main.c | 153 +++++++++++----------- scanner/util/{db.c => api.c} | 84 ++++++------- scanner/util/log.c | 45 ++++--- scanner/util/net.c | 97 +++++++------- scanner/util/op.c | 230 +++++++++++++++++++--------------- scanner/util/pool.c | 118 +++++++++++++++++ scanner/util/util.c | 220 +++++++++++++++----------------- 36 files changed, 1136 insertions(+), 1056 deletions(-) create mode 100644 api/.gitignore create mode 100644 api/Dockerfile create mode 100644 api/lib/__init__.py create mode 100644 api/main.py create mode 100644 api/requirements.txt delete mode 100644 assets/web.png delete mode 100644 database/.gitignore delete mode 100644 database/Dockerfile delete mode 100644 database/go.mod delete mode 100644 database/go.sum delete mode 100644 database/lib/db.go delete mode 100644 database/lib/routes.go delete mode 100644 database/main.go delete mode 100644 database/static/index.css delete mode 100644 database/static/index.html delete mode 100644 database/static/index.js create mode 100644 scanner/.clang-format create mode 100644 scanner/inc/api.h delete mode 100644 scanner/inc/db.h create mode 100644 scanner/inc/pool.h rename scanner/util/{db.c => api.c} (50%) create mode 100644 scanner/util/pool.c diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9c2fcdf..c654767 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -28,6 +28,6 @@ jobs: docker build . --tag ghcr.io/ngn13/massacr/scanner:latest docker push ghcr.io/ngn13/massacr/scanner:latest - cd ../database - docker build . --tag ghcr.io/ngn13/massacr/database:latest - docker push ghcr.io/ngn13/massacr/database:latest + cd ../api + docker build . --tag ghcr.io/ngn13/massacr/api:latest + docker push ghcr.io/ngn13/massacr/api:latest diff --git a/.gitignore b/.gitignore index 21edbcf..f9709ca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -data/ -compose.yml docker-compose.yml +compose.yml diff --git a/README.md b/README.md index 19a845c..381ebaa 100644 --- a/README.md +++ b/README.md @@ -1,64 +1,53 @@ -

- massacr 🩸 mass IP/port scanner -
- a tool for scanning the entire internet -
-
- -

- ---- +# massacr 🩸 mass IP/port scanner toolkit +An extensible toolkit for scanning the internet for TCP ports using SYN packets. +Consists of different tools and servers that interact which each other: +``` +Scanner -> API -> Handler -> MongoDB -> Mongo-Express +``` +- [Scanner](scanner/): SYN port scanner written in C, sends requests to the API with curl +- [API](api/lib): Simple web API written with Flask, provides data to threaded handler +- [Handler](api/main.py): A simple extensible Python function to process provided data. When its done processing, +it saves the processed data to the MongoDB database. Default handler gathers extra information about HTTP(S) servers. +- [MongoDB](https://www.mongodb.com/what-is-mongodb): NoSQL database for storing all the data +- [Mongo-Express](https://github.com/mongo-express/mongo-express): Web-based MongoDB admin interface to interact with the data ## Deploy -### Docker -The project contains a scanner and a simple database with a web interface. Easiest way to deploy these two is to use -`docker-compose`. Here is an example configuration: +Since there are multiple components of massacr, easiest +way to deploy is to use `docker-compose`, here is an example configuration: ```yml version: "3" services: scanner: image: ghcr.io/ngn13/massacr/scanner - command: --url=http://database:3231 --pwd=securepassword --limit=100 + command: --url=http://api:5000 --limit=100 depends_on: - - database + - api - database: - image: ghcr.io/ngn13/massacr/database + api: + image: ghcr.io/ngn13/massacr/api restart: unless-stopped environment: - - PASSWORD=securepassword - ports: - - "127.0.0.1:3231:3231" + - API_MONGO=mongodb://mongo + depends_on: + - mongo + + mongo: + image: mongo volumes: - - ./data:/app/data -``` -after deploying the containers, you can access the web interface at `http://localhost:3231`. + - ./db:/data/db:rw -### From the source -Another way to deploy these two applications, is to build them from the source. -To build from source, first install all the dependencies and build tools: -```bash -build-esssential libnet libnet-dev curl curl-dev go -``` -Then clone the repository: -```bash -git clone https://github.com/ngn13/massacr.git -``` -Now change directory into the database and run the go build command: -```bash -cd massacr/database && go build . -``` -Now change directory into the scanner and run the make command: -```bash -cd ../scanner && make + interface: + image: mongo-express + depends_on: + - mongo + environment: + - ME_CONFIG_MONGODB_URL=mongodb://mongo + ports: + - "127.0.0.1:8081:8081" ``` +after deploying the containers, you can access the web interface at `http://localhost:8081`. ## Configuration -### Database -All the configuration options for the database are set using environment variables: -- `PASSWORD`: password, default is `default` -- `PORT`: port for the web server, default is `3231` - ### Scanner You can list all the options with `--help`: ``` @@ -68,17 +57,18 @@ You can list all the options with `--help`: --ports => Ports to scan for --limit => Packets per second limit --debug => Enable debug output ---url => Database HTTP(S) URL ---pwd => Database password +--url => API HTTP(S) URL +--password => API password ``` -- Options are set using the `--