diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile
index 094676f21..fb07dbcaa 100644
--- a/Dockerfiles/file-monitor.Dockerfile
+++ b/Dockerfiles/file-monitor.Dockerfile
@@ -42,6 +42,8 @@ ARG EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA=false
ARG CLAMD_SOCKET_FILE=/tmp/clamd.ctl
ARG EXTRACTED_FILE_ENABLE_YARA=false
ARG EXTRACTED_FILE_YARA_CUSTOM_ONLY=false
+ARG EXTRACTED_FILE_ENABLE_CAPA=false
+ARG EXTRACTED_FILE_CAPA_VERBOSE=false
ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH
ENV ZEEK_LOG_DIRECTORY $ZEEK_LOG_DIRECTORY
@@ -64,10 +66,14 @@ ENV EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA $EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA
ENV CLAMD_SOCKET_FILE $CLAMD_SOCKET_FILE
ENV EXTRACTED_FILE_ENABLE_YARA $EXTRACTED_FILE_ENABLE_YARA
ENV EXTRACTED_FILE_YARA_CUSTOM_ONLY $EXTRACTED_FILE_YARA_CUSTOM_ONLY
+ENV EXTRACTED_FILE_ENABLE_CAPA $EXTRACTED_FILE_ENABLE_CAPA
+ENV EXTRACTED_FILE_CAPA_VERBOSE $EXTRACTED_FILE_CAPA_VERBOSE
ENV YARA_VERSION "4.0.2"
ENV YARA_URL "https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz"
ENV YARA_RULES_URL "https://codeload.github.com/Neo23x0/signature-base/tar.gz/master"
ENV YARA_RULES_DIR "/yara-rules"
+ENV CAPA_URL "https://github.com/fireeye/capa"
+ENV CAPA_RULES_DIR "/opt/capa-rules"
ENV SRC_BASE_DIR "/usr/local/src"
RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list && \
@@ -80,6 +86,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
clamav-freshclam \
curl \
gcc \
+ git \
libclamunrar9 \
libjansson-dev \
libjansson4 \
@@ -89,11 +96,16 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
libssl1.1 \
libtool \
make \
- pkg-config && \
+ pkg-config \
+ unzip && \
apt-get -y -q install \
inotify-tools \
libzmq5 \
psmisc \
+ python \
+ python-dev \
+ python-pip \
+ python-backports-shutil-get-terminal-size \
python3 \
python3-bs4 \
python3-dev \
@@ -101,7 +113,8 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
python3-pyinotify \
python3-requests \
python3-zmq && \
- pip3 install clamd supervisor yara-python && \
+ pip3 install clamd supervisor yara-python python-magic psutil && \
+ pip2 install flare-capa && \
mkdir -p "${SRC_BASE_DIR}" && \
cd "${SRC_BASE_DIR}" && \
curl -sSL "${YARA_URL}" | tar xzf - -C "${SRC_BASE_DIR}" && \
@@ -114,19 +127,29 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
--enable-dotnet && \
make && \
make install && \
- cd /tmp && \
rm -rf "${SRC_BASE_DIR}"/yara* && \
- mkdir -p ./Neo23x0 && \
+ cd /tmp && \
+ mkdir -p ./Neo23x0 && \
curl -sSL "$YARA_RULES_URL" | tar xzvf - -C ./Neo23x0 --strip-components 1 && \
mkdir -p "${YARA_RULES_DIR}" && \
cp ./Neo23x0/yara/* ./Neo23x0/vendor/yara/* "${YARA_RULES_DIR}"/ && \
cp ./Neo23x0/LICENSE "${YARA_RULES_DIR}"/_LICENSE && \
rm -rf /tmp/Neo23x0 && \
+ cd /tmp && \
+ git clone --depth 1 --single-branch --branch "v$(/usr/local/bin/capa --version 2>&1 | awk '{print $2}')" "${CAPA_URL}" /tmp/capa && \
+ cd /tmp/capa && \
+ git submodule init rules && \
+ git submodule update --depth 1 rules && \
+ cd /tmp && \
+ rm -rf "${CAPA_RULES_DIR}" && \
+ mv /tmp/capa/rules "${CAPA_RULES_DIR}" && \
+ rm -rf "${CAPA_RULES_DIR}"/.git* /tmp/capa && \
apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --purge remove \
automake \
build-essential \
gcc \
gcc-8 \
+ git \
libc6-dev \
libgcc-8-dev \
libjansson-dev \
@@ -134,7 +157,9 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
libssl-dev \
libtool \
make \
- python3-dev && \
+ python-dev \
+ python3-dev \
+ unzip && \
apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages autoremove && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* && \
@@ -142,7 +167,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
curl -s -S -L -o /var/lib/clamav/daily.cvd http://database.clamav.net/daily.cvd && \
curl -s -S -L -o /var/lib/clamav/bytecode.cvd http://database.clamav.net/bytecode.cvd && \
groupadd --gid ${DEFAULT_GID} ${PGROUP} && \
- useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} && \
+ useradd -m --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} && \
usermod -a -G tty ${PUSER} && \
mkdir -p /var/log/clamav /var/lib/clamav && \
chown -R ${PUSER}:${PGROUP} /var/log/clamav /var/lib/clamav && \
@@ -161,6 +186,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/vtot_scan.py && \
ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/clam_scan.py && \
ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/yara_scan.py && \
+ ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/capa_scan.py && \
ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/malass_scan.py
ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
diff --git a/Dockerfiles/moloch.Dockerfile b/Dockerfiles/moloch.Dockerfile
index 2fda87d75..a730dc52a 100644
--- a/Dockerfiles/moloch.Dockerfile
+++ b/Dockerfiles/moloch.Dockerfile
@@ -4,7 +4,7 @@ FROM debian:buster-slim AS build
ENV DEBIAN_FRONTEND noninteractive
-ENV MOLOCH_VERSION "2.4.0"
+ENV MOLOCH_VERSION "2.4.1"
ENV MOLOCHDIR "/data/moloch"
ENV MOLOCH_URL "https://codeload.github.com/aol/moloch/tar.gz/v${MOLOCH_VERSION}"
ENV MOLOCH_LOCALELASTICSEARCH no
diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile
index 0c44193eb..43f50ddee 100644
--- a/Dockerfiles/zeek.Dockerfile
+++ b/Dockerfiles/zeek.Dockerfile
@@ -16,7 +16,7 @@ ENV SRC_BASE_DIR "/usr/local/src"
ENV ZEEK_DIR "/opt/zeek"
ENV ZEEK_PATCH_DIR "${SRC_BASE_DIR}/zeek-patches"
ENV ZEEK_SRC_DIR "${SRC_BASE_DIR}/zeek-${ZEEK_VERSION}"
-ENV ZEEK_VERSION "3.0.8"
+ENV ZEEK_VERSION "3.0.10"
# using clang now instead of gcc because Spicy depends on it
ENV LLVM_VERSION "10"
diff --git a/README.md b/README.md
index bf5c3adf5..a3eb368e5 100644
--- a/README.md
+++ b/README.md
@@ -157,22 +157,22 @@ You can then observe that the images have been retrieved by running `docker imag
```
$ docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
-malcolmnetsec/curator 2.3.0 xxxxxxxxxxxx 40 hours ago 256MB
-malcolmnetsec/elastalert 2.3.0 xxxxxxxxxxxx 40 hours ago 410MB
-malcolmnetsec/elasticsearch-oss 2.3.0 xxxxxxxxxxxx 40 hours ago 690MB
-malcolmnetsec/file-monitor 2.3.0 xxxxxxxxxxxx 39 hours ago 470MB
-malcolmnetsec/file-upload 2.3.0 xxxxxxxxxxxx 39 hours ago 199MB
-malcolmnetsec/filebeat-oss 2.3.0 xxxxxxxxxxxx 39 hours ago 555MB
-malcolmnetsec/freq 2.3.0 xxxxxxxxxxxx 39 hours ago 390MB
-malcolmnetsec/htadmin 2.3.0 xxxxxxxxxxxx 39 hours ago 180MB
-malcolmnetsec/kibana-oss 2.3.0 xxxxxxxxxxxx 40 hours ago 1.16GB
-malcolmnetsec/logstash-oss 2.3.0 xxxxxxxxxxxx 39 hours ago 1.41GB
-malcolmnetsec/moloch 2.3.0 xxxxxxxxxxxx 17 hours ago 683MB
-malcolmnetsec/name-map-ui 2.3.0 xxxxxxxxxxxx 39 hours ago 137MB
-malcolmnetsec/nginx-proxy 2.3.0 xxxxxxxxxxxx 39 hours ago 120MB
-malcolmnetsec/pcap-capture 2.3.0 xxxxxxxxxxxx 39 hours ago 111MB
-malcolmnetsec/pcap-monitor 2.3.0 xxxxxxxxxxxx 39 hours ago 157MB
-malcolmnetsec/zeek 2.3.0 xxxxxxxxxxxx 39 hours ago 887MB
+malcolmnetsec/curator 2.4.0 xxxxxxxxxxxx 40 hours ago 256MB
+malcolmnetsec/elastalert 2.4.0 xxxxxxxxxxxx 40 hours ago 410MB
+malcolmnetsec/elasticsearch-oss 2.4.0 xxxxxxxxxxxx 40 hours ago 690MB
+malcolmnetsec/file-monitor 2.4.0 xxxxxxxxxxxx 39 hours ago 470MB
+malcolmnetsec/file-upload 2.4.0 xxxxxxxxxxxx 39 hours ago 199MB
+malcolmnetsec/filebeat-oss 2.4.0 xxxxxxxxxxxx 39 hours ago 555MB
+malcolmnetsec/freq 2.4.0 xxxxxxxxxxxx 39 hours ago 390MB
+malcolmnetsec/htadmin 2.4.0 xxxxxxxxxxxx 39 hours ago 180MB
+malcolmnetsec/kibana-oss 2.4.0 xxxxxxxxxxxx 40 hours ago 1.16GB
+malcolmnetsec/logstash-oss 2.4.0 xxxxxxxxxxxx 39 hours ago 1.41GB
+malcolmnetsec/moloch 2.4.0 xxxxxxxxxxxx 17 hours ago 683MB
+malcolmnetsec/name-map-ui 2.4.0 xxxxxxxxxxxx 39 hours ago 137MB
+malcolmnetsec/nginx-proxy 2.4.0 xxxxxxxxxxxx 39 hours ago 120MB
+malcolmnetsec/pcap-capture 2.4.0 xxxxxxxxxxxx 39 hours ago 111MB
+malcolmnetsec/pcap-monitor 2.4.0 xxxxxxxxxxxx 39 hours ago 157MB
+malcolmnetsec/zeek 2.4.0 xxxxxxxxxxxx 39 hours ago 887MB
```
#### Import from pre-packaged tarballs
@@ -219,6 +219,7 @@ Malcolm leverages the following excellent open source tools, among others.
* [Kibana](https://www.elastic.co/products/kibana) - for creating additional ad-hoc visualizations and dashboards beyond that which is provided by Moloch Viewer
* [Zeek](https://www.zeek.org/index.html) - a network analysis framework and IDS
* [Yara](https://github.com/VirusTotal/yara) - a tool used to identify and classify malware samples
+* [Capa](https://github.com/fireeye/capa) - a tool for detecting capabilities in executable files
* [ClamAV](https://www.clamav.net/) - an antivirus engine for scanning files extracted by Zeek
* [CyberChef](https://github.com/gchq/CyberChef) - a "swiss-army knife" data conversion tool
* [jQuery File Upload](https://github.com/blueimp/jQuery-File-Upload) - for uploading PCAP files and Zeek logs for processing
@@ -238,9 +239,10 @@ Malcolm leverages the following excellent open source tools, among others.
* Corelight's [community ID](https://github.com/corelight/zeek-community-id) flow hashing plugin
* Corelight's [ripple20](https://github.com/corelight/ripple20) plugin
* Corelight's [SIGred](https://github.com/corelight/SIGred) plugin
+ * Corelight's [Zerologon](https://github.com/corelight/zerologon) plugin
* J-Gras' [Zeek::AF_Packet](https://github.com/J-Gras/zeek-af_packet-plugin) plugin
* Johanna Amann's [CVE-2020-0601](https://github.com/0xxon/cve-2020-0601) ECC certificate validation plugin and [CVE-2020-13777](https://github.com/0xxon/cve-2020-13777) GnuTLS unencrypted session ticket detection plugin
- * Lexi Brent's [EternalSafety](https://github.com/lexibrent/zeek-EternalSafety) plugin
+ * Lexi Brent's [EternalSafety](https://github.com/0xl3x1/zeek-EternalSafety) plugin
* MITRE Cyber Analytics Repository's [Bro/Zeek ATT&CK-Based Analytics (BZAR)](https://github.com/mitre-attack/car/tree/master/implementations) script
* Salesforce's [gQUIC](https://github.com/salesforce/GQUIC_Protocol_Analyzer) analyzer
* Salesforce's [HASSH](https://github.com/salesforce/hassh) SSH fingerprinting plugin
@@ -520,11 +522,15 @@ Various other environment variables inside of `docker-compose.yml` can be tweake
* `VTOT_API2_KEY` – used to specify a [VirusTotal Public API v.20](https://www.virustotal.com/en/documentation/public-api/) key, which, if specified, will be used to submit hashes of [Zeek-extracted files](#ZeekFileExtraction) to VirusTotal
-* `EXTRACTED_FILE_ENABLE_YARA` – if set to `true`, [Zeek-extracted files](#ZeekFileExtraction) will be scanned with Yara
+* `EXTRACTED_FILE_ENABLE_YARA` – if set to `true`, [Zeek-extracted files](#ZeekFileExtraction) will be scanned with [Yara](https://github.com/VirusTotal/yara)
* `EXTRACTED_FILE_YARA_CUSTOM_ONLY` – if set to `true`, Malcolm will bypass the default [Yara ruleset](https://github.com/Neo23x0/signature-base) and use only user-defined rules in `./yara/rules`
-* `EXTRACTED_FILE_ENABLE_CLAMAV` – if set to `true`, [Zeek-extracted files](#ZeekFileExtraction) will be scanned with ClamAV
+* `EXTRACTED_FILE_ENABLE_CAPA` – if set to `true`, [Zeek-extracted files](#ZeekFileExtraction) that are determined to be PE (portable executable) files will be scanned with [Capa](https://github.com/fireeye/capa)
+
+* `EXTRACTED_FILE_CAPA_VERBOSE` – if set to `true`, all Capa rule hits will be logged; otherwise (`false`) only [MITRE ATT&CK technique](https://attack.mitre.org/techniques) classifications will be logged
+
+* `EXTRACTED_FILE_ENABLE_CLAMAV` – if set to `true`, [Zeek-extracted files](#ZeekFileExtraction) will be scanned with [ClamAV](https://www.clamav.net/)
* `EXTRACTED_FILE_ENABLE_FRESHCLAM` – if set to `true`, ClamAV will periodically update virus databases
@@ -1272,8 +1278,9 @@ Extracted files can be examined through any of the following methods:
* submitting file hashes to [**VirusTotal**](https://www.virustotal.com/en/#search); to enable this method, specify the `VTOT_API2_KEY` [environment variable in `docker-compose.yml`](#DockerComposeYml)
* scanning files with [**ClamAV**](https://www.clamav.net/); to enable this method, set the `EXTRACTED_FILE_ENABLE_CLAMAV` [environment variable in `docker-compose.yml`](#DockerComposeYml) to `true`
* scanning files with [**Yara**](https://github.com/VirusTotal/yara); to enable this method, set the `EXTRACTED_FILE_ENABLE_YARA` [environment variable in `docker-compose.yml`](#DockerComposeYml) to `true`
+* scanning PE (portable executable) files with [**Capa**](https://github.com/fireeye/capa); to enable this method, set the `EXTRACTED_FILE_ENABLE_CAPA` [environment variable in `docker-compose.yml`](#DockerComposeYml) to `true`
-Files which are flagged as potentially malicious via either of these methods will be logged as Zeek `signatures.log` entries, and can be viewed in the **Signatures** dashboard in Kibana.
+Files which are flagged via any of these methods will be logged as Zeek `signatures.log` entries, and can be viewed in the **Signatures** dashboard in Kibana.
The `EXTRACTED_FILE_PRESERVATION` [environment variable in `docker-compose.yml`](#DockerComposeYml) determines the behavior for preservation of Zeek-extracted files:
@@ -1423,7 +1430,7 @@ Building the ISO may take 30 minutes or more depending on your system. As the bu
```
…
-Finished, created "/malcolm-build/malcolm-iso/malcolm-2.3.0.iso"
+Finished, created "/malcolm-build/malcolm-iso/malcolm-2.4.0.iso"
…
```
@@ -1438,7 +1445,7 @@ A system installed from the resulting ISO will load the Malcolm Docker images up
### Installation
-The ISO medium boots on systems that support EFI-mode booting. The installer is designed to require as little user input as possible. For this reason, there are NO user prompts and confirmations about partitioning and reformatting hard disks for use by the operating system. The installer assumes that all non-removable storage media (eg., SSD, HDD, NVMe, etc.) are available for use and ⛔🆘😭💀 ***will partition and format them without warning*** 💀😭🆘⛔.
+The installer is designed to require as little user input as possible. For this reason, there are NO user prompts and confirmations about partitioning and reformatting hard disks for use by the operating system. The installer assumes that all non-removable storage media (eg., SSD, HDD, NVMe, etc.) are available for use and ⛔🆘😭💀 ***will partition and format them without warning*** 💀😭🆘⛔.
The installer will ask for several pieces of information prior to installing the Malcolm base operating system:
@@ -1822,22 +1829,22 @@ Pulling zeek ... done
user@host:~/Malcolm$ docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
-malcolmnetsec/curator 2.3.0 xxxxxxxxxxxx 40 hours ago 256MB
-malcolmnetsec/elastalert 2.3.0 xxxxxxxxxxxx 40 hours ago 410MB
-malcolmnetsec/elasticsearch-oss 2.3.0 xxxxxxxxxxxx 40 hours ago 690MB
-malcolmnetsec/file-monitor 2.3.0 xxxxxxxxxxxx 39 hours ago 470MB
-malcolmnetsec/file-upload 2.3.0 xxxxxxxxxxxx 39 hours ago 199MB
-malcolmnetsec/filebeat-oss 2.3.0 xxxxxxxxxxxx 39 hours ago 555MB
-malcolmnetsec/freq 2.3.0 xxxxxxxxxxxx 39 hours ago 390MB
-malcolmnetsec/htadmin 2.3.0 xxxxxxxxxxxx 39 hours ago 180MB
-malcolmnetsec/kibana-oss 2.3.0 xxxxxxxxxxxx 40 hours ago 1.16GB
-malcolmnetsec/logstash-oss 2.3.0 xxxxxxxxxxxx 39 hours ago 1.41GB
-malcolmnetsec/moloch 2.3.0 xxxxxxxxxxxx 17 hours ago 683MB
-malcolmnetsec/name-map-ui 2.3.0 xxxxxxxxxxxx 39 hours ago 137MB
-malcolmnetsec/nginx-proxy 2.3.0 xxxxxxxxxxxx 39 hours ago 120MB
-malcolmnetsec/pcap-capture 2.3.0 xxxxxxxxxxxx 39 hours ago 111MB
-malcolmnetsec/pcap-monitor 2.3.0 xxxxxxxxxxxx 39 hours ago 157MB
-malcolmnetsec/zeek 2.3.0 xxxxxxxxxxxx 39 hours ago 887MB
+malcolmnetsec/curator 2.4.0 xxxxxxxxxxxx 40 hours ago 256MB
+malcolmnetsec/elastalert 2.4.0 xxxxxxxxxxxx 40 hours ago 410MB
+malcolmnetsec/elasticsearch-oss 2.4.0 xxxxxxxxxxxx 40 hours ago 690MB
+malcolmnetsec/file-monitor 2.4.0 xxxxxxxxxxxx 39 hours ago 470MB
+malcolmnetsec/file-upload 2.4.0 xxxxxxxxxxxx 39 hours ago 199MB
+malcolmnetsec/filebeat-oss 2.4.0 xxxxxxxxxxxx 39 hours ago 555MB
+malcolmnetsec/freq 2.4.0 xxxxxxxxxxxx 39 hours ago 390MB
+malcolmnetsec/htadmin 2.4.0 xxxxxxxxxxxx 39 hours ago 180MB
+malcolmnetsec/kibana-oss 2.4.0 xxxxxxxxxxxx 40 hours ago 1.16GB
+malcolmnetsec/logstash-oss 2.4.0 xxxxxxxxxxxx 39 hours ago 1.41GB
+malcolmnetsec/moloch 2.4.0 xxxxxxxxxxxx 17 hours ago 683MB
+malcolmnetsec/name-map-ui 2.4.0 xxxxxxxxxxxx 39 hours ago 137MB
+malcolmnetsec/nginx-proxy 2.4.0 xxxxxxxxxxxx 39 hours ago 120MB
+malcolmnetsec/pcap-capture 2.4.0 xxxxxxxxxxxx 39 hours ago 111MB
+malcolmnetsec/pcap-monitor 2.4.0 xxxxxxxxxxxx 39 hours ago 157MB
+malcolmnetsec/zeek 2.4.0 xxxxxxxxxxxx 39 hours ago 887MB
```
Finally, we can start Malcolm. When Malcolm starts it will stream informational and debug messages to the console. If you wish, you can safely close the console or use `Ctrl+C` to stop these messages; Malcolm will continue running in the background.
diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml
index a2f4bceaf..2a1343c70 100644
--- a/docker-compose-standalone.yml
+++ b/docker-compose-standalone.yml
@@ -49,6 +49,8 @@ x-zeek-variables: &zeek-variables
VTOT_REQUESTS_PER_MINUTE : 4
EXTRACTED_FILE_ENABLE_YARA : 'false'
EXTRACTED_FILE_YARA_CUSTOM_ONLY : 'false'
+ EXTRACTED_FILE_ENABLE_CAPA : 'false'
+ EXTRACTED_FILE_CAPA_VERBOSE : 'false'
EXTRACTED_FILE_ENABLE_CLAMAV : 'false'
EXTRACTED_FILE_ENABLE_FRESHCLAM : 'false'
EXTRACTED_FILE_PIPELINE_DEBUG : 'false'
@@ -124,7 +126,7 @@ x-pcap-capture-variables: &pcap-capture-variables
services:
elasticsearch:
- image: malcolmnetsec/elasticsearch-oss:2.3.0
+ image: malcolmnetsec/elasticsearch-oss:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -159,7 +161,7 @@ services:
retries: 3
start_period: 180s
kibana:
- image: malcolmnetsec/kibana-oss:2.3.0
+ image: malcolmnetsec/kibana-oss:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -185,7 +187,7 @@ services:
retries: 3
start_period: 210s
elastalert:
- image: malcolmnetsec/elastalert:2.3.0
+ image: malcolmnetsec/elastalert:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -213,7 +215,7 @@ services:
retries: 3
start_period: 210s
curator:
- image: malcolmnetsec/curator:2.3.0
+ image: malcolmnetsec/curator:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -232,7 +234,7 @@ services:
retries: 3
start_period: 30s
logstash:
- image: malcolmnetsec/logstash-oss:2.3.0
+ image: malcolmnetsec/logstash-oss:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -265,7 +267,7 @@ services:
retries: 3
start_period: 600s
filebeat:
- image: malcolmnetsec/filebeat-oss:2.3.0
+ image: malcolmnetsec/filebeat-oss:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -302,7 +304,7 @@ services:
retries: 3
start_period: 60s
moloch:
- image: malcolmnetsec/moloch:2.3.0
+ image: malcolmnetsec/moloch:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -313,7 +315,7 @@ services:
<< : *process-variables
<< : *common-upload-variables
<< : *moloch-variables
- MOLOCH_VERSION : '2.4.0'
+ MOLOCH_VERSION : '2.4.1'
VIRTUAL_HOST : 'moloch.malcolm.local'
ES_HOST : 'elasticsearch'
ES_PORT : 9200
@@ -341,7 +343,7 @@ services:
retries: 3
start_period: 210s
zeek:
- image: malcolmnetsec/zeek:2.3.0
+ image: malcolmnetsec/zeek:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -367,7 +369,7 @@ services:
retries: 3
start_period: 60s
file-monitor:
- image: malcolmnetsec/file-monitor:2.3.0
+ image: malcolmnetsec/file-monitor:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -388,7 +390,7 @@ services:
retries: 3
start_period: 60s
pcap-capture:
- image: malcolmnetsec/pcap-capture:2.3.0
+ image: malcolmnetsec/pcap-capture:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -414,7 +416,7 @@ services:
retries: 3
start_period: 60s
pcap-monitor:
- image: malcolmnetsec/pcap-monitor:2.3.0
+ image: malcolmnetsec/pcap-monitor:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -437,7 +439,7 @@ services:
retries: 3
start_period: 90s
upload:
- image: malcolmnetsec/file-upload:2.3.0
+ image: malcolmnetsec/file-upload:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -463,7 +465,7 @@ services:
retries: 3
start_period: 60s
htadmin:
- image: malcolmnetsec/htadmin:2.3.0
+ image: malcolmnetsec/htadmin:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -485,7 +487,7 @@ services:
retries: 3
start_period: 60s
freq:
- image: malcolmnetsec/freq:2.3.0
+ image: malcolmnetsec/freq:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -503,7 +505,7 @@ services:
retries: 3
start_period: 60s
name-map-ui:
- image: malcolmnetsec/name-map-ui:2.3.0
+ image: malcolmnetsec/name-map-ui:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -524,7 +526,7 @@ services:
retries: 3
start_period: 60s
nginx-proxy:
- image: malcolmnetsec/nginx-proxy:2.3.0
+ image: malcolmnetsec/nginx-proxy:2.4.0
restart: "no"
stdin_open: false
tty: true
diff --git a/docker-compose.yml b/docker-compose.yml
index 571cd9505..9f00df0b8 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -49,6 +49,8 @@ x-zeek-variables: &zeek-variables
VTOT_REQUESTS_PER_MINUTE : 4
EXTRACTED_FILE_ENABLE_YARA : 'false'
EXTRACTED_FILE_YARA_CUSTOM_ONLY : 'false'
+ EXTRACTED_FILE_ENABLE_CAPA : 'false'
+ EXTRACTED_FILE_CAPA_VERBOSE : 'false'
EXTRACTED_FILE_ENABLE_CLAMAV : 'false'
EXTRACTED_FILE_ENABLE_FRESHCLAM : 'false'
EXTRACTED_FILE_PIPELINE_DEBUG : 'false'
@@ -127,7 +129,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/elasticsearch.Dockerfile
- image: malcolmnetsec/elasticsearch-oss:2.3.0
+ image: malcolmnetsec/elasticsearch-oss:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -165,7 +167,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/kibana.Dockerfile
- image: malcolmnetsec/kibana-oss:2.3.0
+ image: malcolmnetsec/kibana-oss:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -194,7 +196,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/elastalert.Dockerfile
- image: malcolmnetsec/elastalert:2.3.0
+ image: malcolmnetsec/elastalert:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -225,7 +227,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/curator.Dockerfile
- image: malcolmnetsec/curator:2.3.0
+ image: malcolmnetsec/curator:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -249,7 +251,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/logstash.Dockerfile
- image: malcolmnetsec/logstash-oss:2.3.0
+ image: malcolmnetsec/logstash-oss:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -287,7 +289,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/filebeat.Dockerfile
- image: malcolmnetsec/filebeat-oss:2.3.0
+ image: malcolmnetsec/filebeat-oss:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -328,7 +330,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/moloch.Dockerfile
- image: malcolmnetsec/moloch:2.3.0
+ image: malcolmnetsec/moloch:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -339,7 +341,7 @@ services:
<< : *process-variables
<< : *common-upload-variables
<< : *moloch-variables
- MOLOCH_VERSION : '2.4.0'
+ MOLOCH_VERSION : '2.4.1'
VIRTUAL_HOST : 'moloch.malcolm.local'
ES_HOST : 'elasticsearch'
ES_PORT : 9200
@@ -373,7 +375,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/zeek.Dockerfile
- image: malcolmnetsec/zeek:2.3.0
+ image: malcolmnetsec/zeek:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -403,7 +405,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/file-monitor.Dockerfile
- image: malcolmnetsec/file-monitor:2.3.0
+ image: malcolmnetsec/file-monitor:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -427,7 +429,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/pcap-capture.Dockerfile
- image: malcolmnetsec/pcap-capture:2.3.0
+ image: malcolmnetsec/pcap-capture:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -456,7 +458,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/pcap-monitor.Dockerfile
- image: malcolmnetsec/pcap-monitor:2.3.0
+ image: malcolmnetsec/pcap-monitor:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -482,7 +484,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/file-upload.Dockerfile
- image: malcolmnetsec/file-upload:2.3.0
+ image: malcolmnetsec/file-upload:2.4.0
restart: "no"
stdin_open: false
tty: true
@@ -508,7 +510,7 @@ services:
retries: 3
start_period: 60s
htadmin:
- image: malcolmnetsec/htadmin:2.3.0
+ image: malcolmnetsec/htadmin:2.4.0
build:
context: .
dockerfile: Dockerfiles/htadmin.Dockerfile
@@ -533,7 +535,7 @@ services:
retries: 3
start_period: 60s
freq:
- image: malcolmnetsec/freq:2.3.0
+ image: malcolmnetsec/freq:2.4.0
build:
context: .
dockerfile: Dockerfiles/freq.Dockerfile
@@ -554,7 +556,7 @@ services:
retries: 3
start_period: 60s
name-map-ui:
- image: malcolmnetsec/name-map-ui:2.3.0
+ image: malcolmnetsec/name-map-ui:2.4.0
build:
context: .
dockerfile: Dockerfiles/name-map-ui.Dockerfile
@@ -581,7 +583,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/nginx.Dockerfile
- image: malcolmnetsec/nginx-proxy:2.3.0
+ image: malcolmnetsec/nginx-proxy:2.4.0
restart: "no"
stdin_open: false
tty: true
diff --git a/file-monitor/docker-entrypoint.sh b/file-monitor/docker-entrypoint.sh
index f4a7e1f8e..eacfed534 100755
--- a/file-monitor/docker-entrypoint.sh
+++ b/file-monitor/docker-entrypoint.sh
@@ -10,6 +10,10 @@ if [[ -z $EXTRACTED_FILE_ENABLE_YARA ]]; then
EXTRACTED_FILE_ENABLE_YARA=false
fi
+if [[ -z $EXTRACTED_FILE_ENABLE_CAPA ]]; then
+ EXTRACTED_FILE_ENABLE_CAPA=false
+fi
+
if [[ -z $EXTRACTED_FILE_ENABLE_MALASS ]]; then
[[ ${#MALASS_HOST} -gt 1 ]] && EXTRACTED_FILE_ENABLE_MALASS=true || EXTRACTED_FILE_ENABLE_MALASS=false
fi
@@ -20,6 +24,7 @@ fi
export EXTRACTED_FILE_ENABLE_CLAMAV
export EXTRACTED_FILE_ENABLE_YARA
+export EXTRACTED_FILE_ENABLE_CAPA
export EXTRACTED_FILE_ENABLE_MALASS
export EXTRACTED_FILE_ENABLE_VTOT
diff --git a/file-monitor/supervisord.conf b/file-monitor/supervisord.conf
index df82c688b..29150d906 100644
--- a/file-monitor/supervisord.conf
+++ b/file-monitor/supervisord.conf
@@ -36,7 +36,7 @@ stdout_logfile_maxbytes=0
redirect_stderr=true
[group:scanners]
-programs=virustotal,clamav,yara,malass
+programs=virustotal,clamav,yara,capa,malass
[program:virustotal]
command=/usr/local/bin/vtot_scan.py
@@ -89,6 +89,24 @@ stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
redirect_stderr=true
+[program:capa]
+command=/usr/local/bin/capa_scan.py
+ --verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG)s
+ --extra-verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA)s
+ --start-sleep %(ENV_EXTRACTED_FILE_SCANNER_START_SLEEP)s
+ --capa %(ENV_EXTRACTED_FILE_ENABLE_CAPA)s
+ --capa-rules "%(ENV_CAPA_RULES_DIR)s"
+ --capa-verbose %(ENV_EXTRACTED_FILE_CAPA_VERBOSE)s
+autostart=%(ENV_EXTRACTED_FILE_ENABLE_CAPA)s
+startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s
+startretries=0
+stopasgroup=true
+killasgroup=true
+directory=/data/zeek/extract_files
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
[program:malass]
command=/usr/local/bin/malass_scan.py
--verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG)s
diff --git a/logstash/maps/notice_reference.yaml b/logstash/maps/notice_reference.yaml
index a20a375f6..2e4f215e1 100644
--- a/logstash/maps/notice_reference.yaml
+++ b/logstash/maps/notice_reference.yaml
@@ -1,4 +1,4 @@
-"EternalSafety": "https://github.com/lexibrent/zeek-EternalSafety"
+"EternalSafety": "https://github.com/0xl3x1/zeek-EternalSafety"
"ATTACK": "https://github.com/mitre-attack/bzar"
"HTTPATTACKS": "https://github.com/precurse/zeek-httpattacks"
"Corelight": "https://github.com/corelight"
diff --git a/logstash/pipelines/enrichment/20_enriched_to_ecs.conf b/logstash/pipelines/enrichment/20_enriched_to_ecs.conf
index 7802dc757..8eb1b0d8a 100644
--- a/logstash/pipelines/enrichment/20_enriched_to_ecs.conf
+++ b/logstash/pipelines/enrichment/20_enriched_to_ecs.conf
@@ -1,6 +1,6 @@
filter {
- # Map enriched fields to ECS where possible (see https://github.com/idaholab/Malcolm/issues/79)
+ # Map enriched fields to ECS where possible (see https://github.com/idaholab/Malcolm/issues/16)
# For now I will add fields rather than rename them. This will preserve backwards compatibility
# but the records will be somewhat bigger. I'll have to address what (if anything) to do with upgrades.
diff --git a/logstash/pipelines/zeek/20_zeek_to_ecs.conf b/logstash/pipelines/zeek/20_zeek_to_ecs.conf
index 6d970136d..d6f6c9dc4 100644
--- a/logstash/pipelines/zeek/20_zeek_to_ecs.conf
+++ b/logstash/pipelines/zeek/20_zeek_to_ecs.conf
@@ -1,7 +1,6 @@
filter {
- # Map zeek fields to ECS where possible (see https://github.com/idaholab/Malcolm/issues/79)
-
+ # Map zeek fields to ECS where possible (see https://github.com/idaholab/Malcolm/issues/16)
# For now I will add fields rather than rename them. This will preserve backwards compatibility
# but the records will be somewhat bigger. I'll have to address what (if anything) to do with upgrades.
@@ -633,7 +632,7 @@ filter {
}
} else if ([zeek_notice][category] == "EternalSafety") {
- # populate threat information for EternalSafety from lexibrent/zeek-EternalSafety plugin
+ # populate threat information for EternalSafety from 0xl3x1/zeek-EternalSafety plugin
mutate { id => "mutate_add_field_ecs_threat_framework_eternal_safety"
add_field => { "[threat][framework]" => "EternalSafety" } }
if ([zeek_notice][sub_category]) { mutate { id => "mutate_add_field_ecs_threat_technique_name_eternal"
diff --git a/malcolm-iso/build.sh b/malcolm-iso/build.sh
index 144838a60..92db518e1 100755
--- a/malcolm-iso/build.sh
+++ b/malcolm-iso/build.sh
@@ -66,9 +66,9 @@ if [ -d "$WORKDIR" ]; then
# put the date in the grub.cfg entries and configure installation options
sed -i "s/\(Install Malcolm Base\)/\1 $(date +'%Y-%m-%d %H:%M:%S')/g" ./config/includes.binary/boot/grub/grub.cfg
- cp ./config/includes.binary/install/preseed.cfg ./config/includes.binary/install/preseed_crypto.cfg
+ cp ./config/includes.binary/install/preseed_multipar.cfg ./config/includes.binary/install/preseed_multipar_crypto.cfg
cp ./config/includes.binary/install/preseed_base.cfg ./config/includes.binary/install/preseed_minimal.cfg
- sed -i "s@\(partman-auto/method[[:space:]]*string[[:space:]]*\)lvm@\1crypto@g" ./config/includes.binary/install/preseed_crypto.cfg
+ sed -i "s@\(partman-auto/method[[:space:]]*string[[:space:]]*\)lvm@\1crypto@g" ./config/includes.binary/install/preseed_multipar_crypto.cfg
# make sure we install the newer kernel, firmwares, and kernel headers
echo "linux-image-$(uname -r)" > ./config/package-lists/kernel.list.chroot
@@ -160,6 +160,10 @@ if [ -d "$WORKDIR" ]; then
cp "$SCRIPT_PATH"/../docs/images/favicon/favicon16.png ./config/includes.chroot/usr/share/icons/hicolor/16x16/malcolm.png
chown -R root:root ./config/includes.chroot/usr/share/images ./config/includes.chroot/usr/share/icons
+ mkdir -p ./config/includes.installer
+ cp -v ./config/includes.binary/install/* ./config/includes.installer/
+ cp -v ./config/includes.chroot/usr/local/bin/preseed_partman_determine_disk.sh ./config/includes.installer/
+
lb config \
--image-name "$IMAGE_NAME" \
--debian-installer live \
diff --git a/malcolm-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/malcolm-iso/config/hooks/normal/0169-pip-installs.hook.chroot
index 5e39cd2e8..feb7b9204 100755
--- a/malcolm-iso/config/hooks/normal/0169-pip-installs.hook.chroot
+++ b/malcolm-iso/config/hooks/normal/0169-pip-installs.hook.chroot
@@ -10,5 +10,6 @@ pip3 install --no-compile --no-cache-dir --force-reinstall --upgrade \
debinterface \
docker-compose \
netifaces \
+ psutil \
pythondialog \
requests[security]
diff --git a/malcolm-iso/config/hooks/normal/0991-security-performance.hook.chroot b/malcolm-iso/config/hooks/normal/0991-security-performance.hook.chroot
index adbb13ed0..f2daccc27 100755
--- a/malcolm-iso/config/hooks/normal/0991-security-performance.hook.chroot
+++ b/malcolm-iso/config/hooks/normal/0991-security-performance.hook.chroot
@@ -1,16 +1,45 @@
#!/bin/bash
# configure firewall
+sed -i "s/LOGLEVEL=.*/LOGLEVEL=off/" /etc/ufw/ufw.conf
+sed -i 's/DEFAULT_FORWARD_POLICY=.*/DEFAULT_FORWARD_POLICY="ACCEPT"/' /etc/default/ufw
+sed -i "s/#net\/ipv4\/ip_forward=1/net\/ipv4\/ip_forward=1/" /etc/ufw/sysctl.conf
+read -r -d '' MASQUERADECFG <<- EOM
+# NAT table rules
+*nat
+:POSTROUTING ACCEPT [0:0]
+-A POSTROUTING ! -o docker0 -s 172.29.0.0/16 -j MASQUERADE
+COMMIT
+EOM
+awk '1' <(echo "$MASQUERADECFG") /etc/ufw/before.rules > /tmp/before.rules && \
+ cat /tmp/before.rules > /etc/ufw/before.rules && \
+ rm -f /tmp/before.rules
/usr/sbin/ufw --force enable
/usr/sbin/ufw default deny incoming
/usr/sbin/ufw default allow outgoing
-/usr/sbin/ufw allow ssh
-/usr/sbin/ufw allow ntp
-/usr/sbin/ufw allow 443/tcp
-/usr/sbin/ufw allow 5044
-/usr/sbin/ufw allow 5601/tcp
-/usr/sbin/ufw allow 8443/tcp
-/usr/sbin/ufw allow 9200/tcp
+UFW_ALLOW_RULES=(
+ https
+ ntp
+ ssh
+ 5044
+ 5601/tcp
+ 8443/tcp
+ 9200/tcp
+)
+for i in ${UFW_ALLOW_RULES[@]}; do
+ ufw allow "$i"
+done
+
+# docker (disallow overriding firewall)
+mkdir -p /etc/docker/
+cat << 'EOF' > /etc/docker/daemon.json
+{
+ "iptables": false,
+ "default-address-pools":[
+ {"base":"172.29.0.0/16","size":24}
+ ]
+}
+EOF
# performance parameters for networking, disk, etc.
cat << 'EOF' >> /etc/sysctl.conf
diff --git a/malcolm-iso/config/hooks/normal/0998-localepurge.hook.chroot b/malcolm-iso/config/hooks/normal/0998-localepurge.hook.chroot
deleted file mode 100755
index 48e8ea2e3..000000000
--- a/malcolm-iso/config/hooks/normal/0998-localepurge.hook.chroot
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2020 Battelle Energy Alliance, LLC. All rights reserved.
-
-# remove excess locales
-if [ -f /etc/localepurge-preseed.cfg ] ; then
- debconf-set-selections < /etc/localepurge-preseed.cfg
- apt-get -y install localepurge
- dpkg-reconfigure localepurge
- localepurge
-fi
diff --git a/malcolm-iso/config/includes.binary/boot/grub/grub.cfg b/malcolm-iso/config/includes.binary/boot/grub/grub.cfg
index be3c50c95..00b4f557a 100644
--- a/malcolm-iso/config/includes.binary/boot/grub/grub.cfg
+++ b/malcolm-iso/config/includes.binary/boot/grub/grub.cfg
@@ -1,3 +1,5 @@
+# Copyright (c) 2020 Battelle Energy Alliance, LLC. All rights reserved.
+
set default=0
set timeout=-1
@@ -20,27 +22,27 @@ insmod play
play 960 440 1 0 4 440 1
menuentry "Live system" {
- linux /live/vmlinuz boot=live components username=analyst nosplash persistence persistence-encryption=none,luks random.trust_cpu=on elevator=deadline cgroup_enable=memory swapaccount=1 cgroup.memory=nokmem
+ linux /live/vmlinuz boot=live components username=analyst nosplash random.trust_cpu=on elevator=deadline cgroup_enable=memory swapaccount=1 cgroup.memory=nokmem
initrd /live/initrd.img
}
-menuentry "Install Malcolm Base (quick install)" {
- linux /install/vmlinuz auto=true priority=high vga=normal locales=en_US.UTF-8 keyboard-layouts=us preseed/file=/cdrom/install/preseed.cfg
+menuentry "Install Malcolm (quick install)" {
+ linux /install/vmlinuz preseed/file=/cdrom/install/preseed_multipar.cfg auto=true priority=high vga=normal locales=en_US.UTF-8 keyboard-layouts=us
initrd /install/initrd.gz
}
-menuentry "Install Malcolm Base (encrypted quick install)" {
- linux /install/vmlinuz auto=true priority=high vga=normal locales=en_US.UTF-8 keyboard-layouts=us preseed/file=/cdrom/install/preseed_crypto.cfg
+menuentry "Install Malcolm (encrypted quick install)" {
+ linux /install/vmlinuz preseed/file=/cdrom/install/preseed_multipar_crypto.cfg auto=true priority=high vga=normal locales=en_US.UTF-8 keyboard-layouts=us
initrd /install/initrd.gz
}
-menuentry "Install Malcolm Base (advanced configuration)" {
- linux /install/vmlinuz auto=true priority=high vga=normal preseed/file=/cdrom/install/preseed_minimal.cfg
+menuentry "Install Malcolm (advanced configuration)" {
+ linux /install/vmlinuz preseed/file=/cdrom/install/preseed_minimal.cfg auto=true priority=high vga=normal
initrd /install/initrd.gz
}
-menuentry "Install Malcolm Base (virtual machine single partition quick install)" {
- linux /install/vmlinuz auto=true priority=high vga=normal locales=en_US.UTF-8 keyboard-layouts=us preseed/file=/cdrom/install/preseed_vmware.cfg
+menuentry "Install Malcolm (virtual machine single partition quick install)" {
+ linux /install/vmlinuz preseed/file=/cdrom/install/preseed_vmware.cfg auto=true priority=high vga=normal locales=en_US.UTF-8 keyboard-layouts=us
initrd /install/initrd.gz
}
diff --git a/malcolm-iso/config/includes.binary/install/preseed_base.cfg b/malcolm-iso/config/includes.binary/install/preseed_base.cfg
index 34b104866..255251da1 100644
--- a/malcolm-iso/config/includes.binary/install/preseed_base.cfg
+++ b/malcolm-iso/config/includes.binary/install/preseed_base.cfg
@@ -6,15 +6,17 @@ d-i time/zone string Universal
d-i clock-setup/ntp boolean false
d-i clock-setup/ntp-server string 0.debian.pool.ntp.org
-d-i localepurge/nopurge multiselect en, en_US, en_us.UTF-8, C.UTF-8
-d-i localepurge/use-dpkg-feature boolean false
-d-i localepurge/none_selected boolean false
-d-i localepurge/verbose boolean false
-d-i localepurge/dontbothernew boolean false
-d-i localepurge/quickndirtycalc boolean true
-d-i localepurge/mandelete boolean true
-d-i localepurge/showfreedspace boolean false
-d-i localepurge/remove_no note
+d-i popularity-contest/participate boolean false
+
+localepurge localepurge/dontbothernew boolean false
+localepurge localepurge/mandelete boolean true
+localepurge localepurge/none_selected boolean false
+localepurge localepurge/nopurge multiselect en, en_US, en_us.UTF-8, C.UTF-8
+localepurge localepurge/quickndirtycalc boolean true
+localepurge localepurge/remove_no note
+localepurge localepurge/showfreedspace boolean false
+localepurge localepurge/use-dpkg-feature boolean false
+localepurge localepurge/verbose boolean false
# d-i passwd/username string analyst
# d-i passwd/user-fullname string analyst
@@ -39,11 +41,10 @@ d-i preseed/late_command string \
in-target sed -r -i 's@(^.+\s+/(tmp|var/tmp)\s+ext4\s+.*defaults)@\1,nosuid,nodev,noexec@g' /etc/fstab; \
in-target sed -r -i 's@(^.+/media/cdrom[0-9]*.+)(noauto)(.*)@\1\2,nosuid,nodev,noexec\3@g' /etc/fstab; \
in-target sed -r -i 's@(^.+\s+/(home)\s+ext4\s+.*defaults)@\1,nosuid,nodev@g' /etc/fstab; \
- in-target bash -c "echo '\EFI\debian\grubx64.efi' > /boot/efi/startup.nsh"; \
+ in-target bash -c "( echo '\EFI\debian\grubx64.efi' > /boot/efi/startup.nsh ) || true"; \
in-target sed -i 's#^\(GRUB_CMDLINE_LINUX_DEFAULT="quiet\)"$#\1 random.trust_cpu=on elevator=deadline cgroup_enable=memory swapaccount=1 cgroup.memory=nokmem apparmor=1 security=apparmor ipv6.disable=1 audit=1"#' /etc/default/grub; \
in-target sed -i 's#^\(GRUB_CMDLINE_LINUX="\)"$#\1apparmor=1 security=apparmor audit=1"#' /etc/default/grub; \
in-target sed -i 's#^\(GRUB_DISTRIBUTOR=\).*$#\1"Hedgehog"#' /etc/default/grub; \
in-target cp /usr/share/images/desktop-base/Malcolm_background.png /boot/grub; \
in-target bash /usr/local/bin/preseed_late_user_config.sh; \
- in-target grub-mkconfig -o /boot/grub/grub.cfg; \
- in-target bash -c "(dpkg -s localepurge >/dev/null 2>&1) && (debconf-set-selections < /etc/localepurge-preseed.cfg) && dpkg-reconfigure localepurge && localepurge";
+ in-target grub-mkconfig -o /boot/grub/grub.cfg;
diff --git a/malcolm-iso/config/includes.binary/install/preseed.cfg b/malcolm-iso/config/includes.binary/install/preseed_multipar.cfg
similarity index 95%
rename from malcolm-iso/config/includes.binary/install/preseed.cfg
rename to malcolm-iso/config/includes.binary/install/preseed_multipar.cfg
index d1a4a66a7..5720a24c0 100644
--- a/malcolm-iso/config/includes.binary/install/preseed.cfg
+++ b/malcolm-iso/config/includes.binary/install/preseed_multipar.cfg
@@ -11,7 +11,7 @@ d-i preseed/include string preseed_base.cfg
# install root filesystem on smallest non-USB disk
d-i partman/early_command string \
- ROOT_DISK=$(parted_devices | egrep "^($(find /sys/block -mindepth 1 -maxdepth 1 -type l \( -name '[hs]d*' -o -name 'nvme*' \) -exec ls -l '{}' ';' | grep -v "usb" | sed 's@^.*\([hs]d[a-z]\+\|nvme[0-9]\+\).*$@/dev/\1@' | sed -e :a -e '$!N; s/\n/|/; ta'))" | sort -k2n | head -1 | cut -f1); \
+ ROOT_DISK=$(sh /preseed_partman_determine_disk.sh); \
pvremove -ff -y "$ROOT_DISK"*; \
debconf-set partman-auto/disk "$ROOT_DISK"; \
debconf-set grub-installer/bootdev "$ROOT_DISK"; \
@@ -88,7 +88,7 @@ d-i partman-auto/expert_recipe string \
filesystem{ ext4 } \
mountpoint{ / } \
. \
- 4000 8000 12000 ext4 \
+ 12000 16000 24000 ext4 \
$defaultignore{ } \
$lvmok{ } \
in_vg { main } lv_name{ var } \
diff --git a/malcolm-iso/config/includes.binary/install/preseed_vmware.cfg b/malcolm-iso/config/includes.binary/install/preseed_vmware.cfg
index ced2fc714..87cea5293 100644
--- a/malcolm-iso/config/includes.binary/install/preseed_vmware.cfg
+++ b/malcolm-iso/config/includes.binary/install/preseed_vmware.cfg
@@ -11,7 +11,7 @@ d-i preseed/include string preseed_base.cfg
# install root filesystem on smallest non-USB disk
d-i partman/early_command string \
- ROOT_DISK=$(parted_devices | egrep "^($(find /sys/block -mindepth 1 -maxdepth 1 -type l \( -name '[hs]d*' -o -name 'nvme*' \) -exec ls -l '{}' ';' | grep -v "usb" | sed 's@^.*\([hs]d[a-z]\+\|nvme[0-9]\+\).*$@/dev/\1@' | sed -e :a -e '$!N; s/\n/|/; ta'))" | sort -k2n | head -1 | cut -f1); \
+ ROOT_DISK=$(sh /preseed_partman_determine_disk.sh); \
pvremove -ff -y "$ROOT_DISK"*; \
debconf-set partman-auto/disk "$ROOT_DISK"; \
debconf-set grub-installer/bootdev "$ROOT_DISK"; \
diff --git a/malcolm-iso/config/includes.binary/isolinux/advanced.cfg b/malcolm-iso/config/includes.binary/isolinux/advanced.cfg
new file mode 100644
index 000000000..743039292
--- /dev/null
+++ b/malcolm-iso/config/includes.binary/isolinux/advanced.cfg
@@ -0,0 +1,29 @@
+label live
+menu label ^Live system
+kernel /live/vmlinuz
+append boot=live components username=analyst nosplash random.trust_cpu=on elevator=deadline cgroup_enable=memory swapaccount=1 cgroup.memory=nokmem initrd=/live/initrd.img --
+
+label install
+menu label ^Install Malcolm (quick install)
+kernel /install/vmlinuz
+append file=/preseed_multipar.cfg initrd=/install/initrd.gz auto=true priority=high locales=en_US.UTF-8 keyboard-layouts=us --
+
+label installenc
+menu label ^Install Malcolm (encrypted quick install)
+kernel /install/vmlinuz
+append file=/preseed_multipar_crypto.cfg initrd=/install/initrd.gz auto=true priority=high locales=en_US.UTF-8 keyboard-layouts=us --
+
+label installadv
+menu label ^Install Malcolm (advanced configuration)
+kernel /install/vmlinuz
+append file=/preseed_minimal.cfg initrd=/install/initrd.gz auto=true priority=high --
+
+label installvm
+menu label ^Install Malcolm (virtual machine single partition quick install)
+kernel /install/vmlinuz
+append file=/preseed_vmware.cfg initrd=/install/initrd.gz auto=true priority=high locales=en_US.UTF-8 keyboard-layouts=us --
+
+label rescue
+menu label ^Rescue system in text mode
+kernel /install/vmlinuz
+append rescue/enable=true initrd=/install/initrd.gz --
diff --git a/malcolm-iso/config/includes.binary/isolinux/install.cfg b/malcolm-iso/config/includes.binary/isolinux/install.cfg
new file mode 100644
index 000000000..e69de29bb
diff --git a/malcolm-iso/config/includes.chroot/etc/localepurge-preseed.cfg b/malcolm-iso/config/includes.chroot/etc/localepurge-preseed.cfg
deleted file mode 100644
index ac377e228..000000000
--- a/malcolm-iso/config/includes.chroot/etc/localepurge-preseed.cfg
+++ /dev/null
@@ -1,9 +0,0 @@
-localepurge localepurge/nopurge multiselect en, en_US, en_us.UTF-8, C.UTF-8
-localepurge localepurge/use-dpkg-feature boolean false
-localepurge localepurge/none_selected boolean false
-localepurge localepurge/verbose boolean false
-localepurge localepurge/dontbothernew boolean false
-localepurge localepurge/quickndirtycalc boolean true
-localepurge localepurge/mandelete boolean true
-localepurge localepurge/showfreedspace boolean false
-localepurge localepurge/remove_no note
\ No newline at end of file
diff --git a/malcolm-iso/config/package-lists/grub.list.binary b/malcolm-iso/config/package-lists/grub.list.binary
new file mode 100644
index 000000000..bed168d86
--- /dev/null
+++ b/malcolm-iso/config/package-lists/grub.list.binary
@@ -0,0 +1,3 @@
+grub-pc-bin
+grub-efi-amd64-bin
+grub-efi-amd64
diff --git a/malcolm-iso/config/package-lists/system.list.chroot b/malcolm-iso/config/package-lists/system.list.chroot
index d66ceecfc..f3c8dbd4e 100644
--- a/malcolm-iso/config/package-lists/system.list.chroot
+++ b/malcolm-iso/config/package-lists/system.list.chroot
@@ -52,9 +52,6 @@ gnupg1
gnupg2
gpart
gparted
-grub-efi-amd64
-grub-efi-amd64-bin
-grub-efi-ia32-bin
gvfs
gvfs-backends
gvfs-daemons
@@ -80,6 +77,7 @@ libssl-dev
libykpers-1-1
libyubikey0
lm-sensors
+localepurge
lshw
lsof
lvm2
diff --git a/malcolm-iso/vagrant/Vagrantfile b/malcolm-iso/vagrant/Vagrantfile
index e7625f2af..6e6045700 100644
--- a/malcolm-iso/vagrant/Vagrantfile
+++ b/malcolm-iso/vagrant/Vagrantfile
@@ -40,7 +40,7 @@ Vagrant.configure("2") do |config|
export KERNEL_VERSION=$(apt-cache search linux-image-5 | grep -Pv -- '(-(rt|cloud)-amd64|amd64-(dbg|unsigned))' | sort -r --sort=version | awk '{print $1}' | head -n 1 | sed 's/^linux-image-//' | sed 's/-amd64$//')
apt-get -t buster-backports install -y \
linux-image-$KERNEL_VERSION-amd64 linux-headers-$KERNEL_VERSION-amd64 linux-headers-$KERNEL_VERSION-common \
- dkms build-essential linux-kbuild-5.6 linux-compiler-gcc-8-x86 \
+ dkms build-essential linux-kbuild-5.7 linux-compiler-gcc-8-x86 \
firmware-linux firmware-linux-nonfree firmware-misc-nonfree firmware-amd-graphics
ls /dev/disk/by-id/ata-* | grep -v '\\-part' | head -n 1 | xargs -r -l grub-install
STEP1
diff --git a/scripts/beats/windows_vm_example/Malcolm_Windows_Forwarder_Download_and_Config.ps1 b/scripts/beats/windows_vm_example/Malcolm_Windows_Forwarder_Download_and_Config.ps1
index 64957a1de..480ef9f62 100644
--- a/scripts/beats/windows_vm_example/Malcolm_Windows_Forwarder_Download_and_Config.ps1
+++ b/scripts/beats/windows_vm_example/Malcolm_Windows_Forwarder_Download_and_Config.ps1
@@ -1,5 +1,5 @@
# configure a windows host to forward auditbeat and winlogbeat logs
-# to Malcolm (see https://github.com/idaholab/Malcolm/tree/development/scripts/beats)
+# to Malcolm (see https://github.com/idaholab/Malcolm/tree/master/scripts/beats)
$beatversion = "7.6.2"
@@ -37,7 +37,7 @@ function Download-Beat {
((Get-Content -path "C:\\Program Files\\$beat\\install-service-$beat.ps1" -Raw) -replace 'ProgramData','Program Files') | Set-Content -Path "C:\\Program Files\\$beat\\install-service-$beat.ps1"
((Get-Content -path "C:\\Program Files\\$beat\\install-service-$beat.ps1" -Raw) -replace ' -path',' --path') | Set-Content -Path "C:\\Program Files\\$beat\\install-service-$beat.ps1"
- Invoke-WebRequest -UseBasicParsing -OutFile "C:\\Program Files\\$beat\\$beat.yml" -Uri https://raw.githubusercontent.com/idaholab/Malcolm/development/scripts/beats/windows_vm_example/$beat.yml
+ Invoke-WebRequest -UseBasicParsing -OutFile "C:\\Program Files\\$beat\\$beat.yml" -Uri https://raw.githubusercontent.com/idaholab/Malcolm/master/scripts/beats/windows_vm_example/$beat.yml
(Get-Content "C:\\Program Files\\$beat\\$beat.yml") | Set-Content "C:\\Program Files\\$beat\\$beat.yml"
}
diff --git a/scripts/install.py b/scripts/install.py
index 87631e446..c8c1229e7 100755
--- a/scripts/install.py
+++ b/scripts/install.py
@@ -373,6 +373,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path, expose_logstash_default=Fa
filePreserveMode = None
vtotApiKey = '0'
yaraScan = False
+ capaScan = False
clamAvScan = False
clamAvUpdate = False
@@ -387,6 +388,8 @@ def tweak_malcolm_runtime(self, malcolm_install_path, expose_logstash_default=Fa
clamAvUpdate = InstallerYesOrNo('Download updated ClamAV virus signatures periodically?', default=True)
if InstallerYesOrNo('Scan extracted files with Yara?', default=False):
yaraScan = True
+ if InstallerYesOrNo('Scan extracted PE files with Capa?', default=False):
+ capaScan = True
if InstallerYesOrNo('Lookup extracted file hashes with VirusTotal?', default=False):
while (len(vtotApiKey) <= 1):
vtotApiKey = InstallerAskForString('Enter VirusTotal API key')
@@ -473,6 +476,9 @@ def tweak_malcolm_runtime(self, malcolm_install_path, expose_logstash_default=Fa
elif 'EXTRACTED_FILE_ENABLE_YARA' in line:
# file scanning via yara
line = re.sub(r'(EXTRACTED_FILE_ENABLE_YARA\s*:\s*)(\S+)', r'\g<1>{}'.format("'true'" if yaraScan else "'false'"), line)
+ elif 'EXTRACTED_FILE_ENABLE_CAPA' in line:
+ # PE file scanning via capa
+ line = re.sub(r'(EXTRACTED_FILE_ENABLE_CAPA\s*:\s*)(\S+)', r'\g<1>{}'.format("'true'" if capaScan else "'false'"), line)
elif 'EXTRACTED_FILE_ENABLE_CLAMAV' in line:
# file scanning via clamav
line = re.sub(r'(EXTRACTED_FILE_ENABLE_CLAMAV\s*:\s*)(\S+)', r'\g<1>{}'.format("'true'" if clamAvScan else "'false'"), line)
diff --git a/sensor-iso/README.md b/sensor-iso/README.md
index ea3408d65..0ba6d4696 100644
--- a/sensor-iso/README.md
+++ b/sensor-iso/README.md
@@ -60,7 +60,7 @@ The boot menu of the sensor installer image provides several options:
## Installer
-The ISO medium boots on systems that support EFI-mode booting. The sensor installer is designed to require as little user input as possible. For this reason, there are NO user prompts and confirmations about partitioning and reformatting hard disks for use by the sensor. The installer assumes that all non-removable storage media (eg., SSD, HDD, NVMe, etc.) are available for use and ⛔🆘😭💀 ***will partition and format them without warning*** 💀😭🆘⛔.
+The sensor installer is designed to require as little user input as possible. For this reason, there are NO user prompts and confirmations about partitioning and reformatting hard disks for use by the sensor. The installer assumes that all non-removable storage media (eg., SSD, HDD, NVMe, etc.) are available for use and ⛔🆘😭💀 ***will partition and format them without warning*** 💀😭🆘⛔.
The installer will ask for a few pieces of information prior to installing the sensor operating system:
@@ -209,6 +209,7 @@ You'll be prompted to specify which engine(s) to use to analyze extracted files.
* scanning files with [**ClamAV**](https://www.clamav.net/); to enable this method, select **ZEEK_FILE_SCAN_CLAMAV** when specifying scanners for Zeek-carved files
* submitting file hashes to [**VirusTotal**](https://www.virustotal.com/en/#search); to enable this method, select **ZEEK_FILE_SCAN_VTOT** when specifying scanners for Zeek-carved files, then manually edit `/opt/sensor/sensor_ctl/control_vars.conf` and specify your [VirusTotal API key](https://developers.virustotal.com/reference) in `VTOT_API2_KEY`
* scanning files with [**Yara**](https://github.com/VirusTotal/yara); to enable this method, select **ZEEK_FILE_SCAN_YARA** when specifying scanners for Zeek-carved files
+* scanning portable executable (PE) files with [**Capa**](https://github.com/fireeye/capa); to enable this method, select **ZEEK_FILE_SCAN_CAPA** when specifying scanners for Zeek-carved files
Files which are flagged as potentially malicious will be logged as Zeek `signatures.log` entries, and can be viewed in the **Signatures** dashboard in [Kibana](https://github.com/idaholab/malcolm#KibanaVisualizations) when forwarded to Malcolm.
@@ -379,6 +380,7 @@ tcpdump:tcpdump-enp8s0 STOPPED Not started
zeek:logger RUNNING pid 14434, uptime 8 days, 20:22:32
zeek:virustotal RUNNING pid 14435, uptime 8 days, 20:22:32
zeek:yara RUNNING pid 14435, uptime 8 days, 20:22:32
+zeek:capa RUNNING pid 14435, uptime 8 days, 20:22:32
zeek:clamav RUNNING pid 14435, uptime 8 days, 20:22:32
zeek:watcher RUNNING pid 14441, uptime 8 days, 20:22:32
zeek:zeekctl RUNNING pid 14433, uptime 8 days, 20:22:32
@@ -402,7 +404,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu
```
…
-Finished, created "/sensor-build/hedgehog-2.3.0.iso"
+Finished, created "/sensor-build/hedgehog-2.4.0.iso"
…
```
@@ -639,7 +641,7 @@ moloch_2.2.3-1_amd64.deb
netsniff-ng_0.6.6-1_amd64.deb 100% 330KB 52.1MB/s 00:00
packetbeat-tweaked-7.6.2-amd64.deb 100% 14MB 59.2MB/s 00:00
protologbeat 100% 56MB 38.1MB/s 00:01
-zeek_3.0.8-1_amd64.deb 100% 26MB 63.1MB/s 00:00
+zeek_3.0.10-1_amd64.deb 100% 26MB 63.1MB/s 00:00
```
12. Replace the old `/usr/local/bin/protologbeat` with the new one:
@@ -667,7 +669,7 @@ The following packages will be REMOVED:
After this operation, 160 MB disk space will be freed.
Do you want to continue? [Y/n] y
(Reading database ... 118490 files and directories currently installed.)
-Removing zeek (3.0.8-1) ...
+Removing zeek (3.0.10-1) ...
dpkg: warning: while removing zeek, directory '/opt/zeek/spool' not empty so not removed
dpkg: warning: while removing zeek, directory '/opt/zeek/share/zeek/site' not empty so not removed
dpkg: warning: while removing zeek, directory '/opt/zeek/lib' not empty so not removed
@@ -695,8 +697,8 @@ Preparing to unpack .../netsniff-ng_0.6.6-1_amd64.deb ...
Unpacking netsniff-ng (0.6.6-1) over (0.6.6-1) ...
Preparing to unpack .../packetbeat-tweaked-7.6.2-amd64.deb ...
Unpacking packetbeat (7.6.2) over (6.8.4) ...
-Preparing to unpack .../zeek_3.0.8-1_amd64.deb ...
-Unpacking zeek (3.0.8-1) over (3.0.0-1) ...
+Preparing to unpack .../zeek_3.0.10-1_amd64.deb ...
+Unpacking zeek (3.0.10-1) over (3.0.0-1) ...
Setting up auditbeat (7.6.2) ...
Installing new version of [...]
[...]
@@ -712,7 +714,7 @@ Setting up netsniff-ng (0.6.6-1) ...
Setting up packetbeat (7.6.2) ...
Installing new version of [...]
[...]
-Setting up zeek (3.0.8-1) ...
+Setting up zeek (3.0.10-1) ...
Processing triggers for systemd (232-25+deb9u12) ...
Processing triggers for man-db (2.7.6.1-2) ...
```
diff --git a/sensor-iso/build.sh b/sensor-iso/build.sh
index 812f05d8a..ff93ef1d1 100755
--- a/sensor-iso/build.sh
+++ b/sensor-iso/build.sh
@@ -54,10 +54,10 @@ if [ -d "$WORKDIR" ]; then
# put the date in the grub.cfg entries and configure installation options
sed -i "s/\(Install Hedgehog Linux\)/\1 $(date +'%Y-%m-%d %H:%M:%S')/g" ./config/includes.binary/boot/grub/grub.cfg
- cp ./config/includes.binary/install/preseed.cfg ./config/includes.binary/install/preseed_crypto.cfg
+ cp ./config/includes.binary/install/preseed_multipar.cfg ./config/includes.binary/install/preseed_multipar_crypto.cfg
cp ./config/includes.binary/install/preseed_base.cfg ./config/includes.binary/install/preseed_minimal.cfg
- sed -i "s@\(partman-auto/method[[:space:]]*string[[:space:]]*\)lvm@\1crypto@g" ./config/includes.binary/install/preseed_crypto.cfg
- sed -i "s@\(/etc/capture_storage_format\)@\1.crypt@g" ./config/includes.binary/install/preseed_crypto.cfg
+ sed -i "s@\(partman-auto/method[[:space:]]*string[[:space:]]*\)lvm@\1crypto@g" ./config/includes.binary/install/preseed_multipar_crypto.cfg
+ sed -i "s@\(/etc/capture_storage_format\)@\1.crypt@g" ./config/includes.binary/install/preseed_multipar_crypto.cfg
sed -i "s@\(/etc/capture_storage_format\)@\1.none@g" ./config/includes.binary/install/preseed_minimal.cfg
# create a hook for installing Python packages required by interface
@@ -163,6 +163,10 @@ if [ -d "$WORKDIR" ]; then
ln -r -s ./config/includes.chroot/usr/share/images/hedgehog/*wallpaper*.png ./config/includes.chroot/usr/share/images/desktop-base/
find "$SCRIPT_PATH/docs/logo/font" -type f -name "*.ttf" -exec cp "{}" ./config/includes.chroot/usr/share/fonts/truetype/ubuntu/ \;
+ mkdir -p ./config/includes.installer
+ cp -v ./config/includes.binary/install/* ./config/includes.installer/
+ cp -v ./config/includes.chroot/usr/local/bin/preseed_partman_determine_disk.sh ./config/includes.installer/
+
lb config \
--image-name "$IMAGE_NAME" \
--debian-installer live \
diff --git a/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot
index 36f87c0df..5b5d5fa96 100755
--- a/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot
+++ b/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot
@@ -11,15 +11,21 @@ export ASM="clang-10"
# python 3
pip3 install --no-compile --no-cache-dir --force-reinstall --upgrade \
beautifulsoup4 \
- colorama \
clamd \
+ colorama \
debinterface \
ipaddress \
netifaces \
+ psutil \
pyinotify \
+ python-magic \
pythondialog \
pyzmq \
requests \
scapy \
yara-python \
zkg
+
+# python 2
+pip2 install --system --no-compile --no-cache-dir --force-reinstall --upgrade \
+ flare-capa
diff --git a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot
index 008e6669c..1679512a8 100755
--- a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot
+++ b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot
@@ -7,7 +7,7 @@ NETSNIFF_URL="https://github.com/netsniff-ng/netsniff-ng/archive/v$NETSNIFF_VER.
SPICY_DIR="/opt/spicy"
ZEEK_DIR="/opt/zeek"
-ZEEK_VER="3.0.8"
+ZEEK_VER="3.0.10"
ZEEK_URL="https://old.zeek.org/downloads/zeek-$ZEEK_VER.tar.gz"
ZEEK_PATCH_URLS=(
# nothing here for now
@@ -30,6 +30,9 @@ YARA_URL="https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz"
YARA_RULES_URL="https://codeload.github.com/Neo23x0/signature-base/tar.gz/master"
YARA_RULES_DIR="/opt/yara-rules"
+CAPA_URL="https://github.com/fireeye/capa"
+CAPA_RULES_DIR="/opt/capa-rules"
+
mkdir -p /opt/hedgehog_install_artifacts/
# some environment variables needed for build using clang
@@ -154,6 +157,21 @@ tar czf yara-rules-hedgehog.tar.gz "$(basename "${YARA_RULES_DIR}")"
mv ./yara-rules-hedgehog.tar.gz /opt/hedgehog_install_artifacts/
###
+# capa (installed via pip) rules
+cd /tmp
+git clone --depth 1 --single-branch --branch "v$(/usr/local/bin/capa --version 2>&1 | awk '{print $2}')" "${CAPA_URL}" ./capa
+cd ./capa
+git submodule init rules
+git submodule update --depth 1 rules
+cd ../
+rm -rf "${CAPA_RULES_DIR}"
+mv ./capa/rules "${CAPA_RULES_DIR}"
+rm -rf "${CAPA_RULES_DIR}"/.git* ./capa
+
+cd "${CAPA_RULES_DIR}"/..
+tar czf capa-rules-hedgehog.tar.gz "$(basename "${CAPA_RULES_DIR}")"
+mv ./capa-rules-hedgehog.tar.gz /opt/hedgehog_install_artifacts/
+
# update clamav signatures
freshclam --stdout --quiet --no-warnings
###
diff --git a/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot b/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot
index 8b2878ac1..fea0c75a5 100755
--- a/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot
+++ b/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot
@@ -4,7 +4,7 @@
# remove development packages
apt-get -y --purge remove checkinstall bison google-perftools gdb git libc6-dbg ninja-build \
- $(dpkg --get-selections | grep -Pv "(^(libyaml-dev|dpkg|libgcc)|deinstall$)" | cut -f1 | grep -P -- '-dev(:\w+)?$') || true
+ $(dpkg --get-selections | grep -Pv "(^(libyaml-dev|dpkg|libgcc|libpcap|libclang)|deinstall$)" | cut -f1 | grep -P -- '-dev(:\w+)?$') || true
rm -rf /opt/cmake /var/spool/ccache
# remove unwanted packages
diff --git a/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot b/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot
index a323eb0ff..611f87734 100755
--- a/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot
+++ b/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot
@@ -2,12 +2,18 @@
# Copyright (c) 2020 Battelle Energy Alliance, LLC. All rights reserved.
-# enable firewall, disallow everything in except SSH
+# configure firewall
+sed -i "s/LOGLEVEL=.*/LOGLEVEL=off/" /etc/ufw/ufw.conf
/usr/sbin/ufw --force enable
/usr/sbin/ufw default deny incoming
/usr/sbin/ufw default allow outgoing
-/usr/sbin/ufw allow ssh
-/usr/sbin/ufw allow ntp
+UFW_ALLOW_RULES=(
+ ntp
+ ssh
+)
+for i in ${UFW_ALLOW_RULES[@]}; do
+ ufw allow "$i"
+done
# performance parameters for networking, disk, etc.
cat << 'EOF' >> /etc/sysctl.conf
diff --git a/sensor-iso/config/hooks/normal/0998-localepurge.hook.chroot b/sensor-iso/config/hooks/normal/0998-localepurge.hook.chroot
deleted file mode 100755
index 48e8ea2e3..000000000
--- a/sensor-iso/config/hooks/normal/0998-localepurge.hook.chroot
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2020 Battelle Energy Alliance, LLC. All rights reserved.
-
-# remove excess locales
-if [ -f /etc/localepurge-preseed.cfg ] ; then
- debconf-set-selections < /etc/localepurge-preseed.cfg
- apt-get -y install localepurge
- dpkg-reconfigure localepurge
- localepurge
-fi
diff --git a/sensor-iso/config/includes.binary/boot/grub/grub.cfg b/sensor-iso/config/includes.binary/boot/grub/grub.cfg
index d27d89a07..7dfefa205 100644
--- a/sensor-iso/config/includes.binary/boot/grub/grub.cfg
+++ b/sensor-iso/config/includes.binary/boot/grub/grub.cfg
@@ -32,22 +32,22 @@ menuentry "Live system (fully in RAM)" {
}
menuentry "Install Hedgehog Linux (quick install)" {
- linux /install/vmlinuz auto=true priority=critical vga=normal locales=en_US.UTF-8 keyboard-layouts=us preseed/file=/cdrom/install/preseed.cfg
+ linux /install/vmlinuz preseed/file=/cdrom/install/preseed_multipar.cfg auto=true priority=critical vga=normal locales=en_US.UTF-8 keyboard-layouts=us
initrd /install/initrd.gz
}
menuentry "Install Hedgehog Linux (encrypted quick install)" {
- linux /install/vmlinuz auto=true priority=critical vga=normal locales=en_US.UTF-8 keyboard-layouts=us preseed/file=/cdrom/install/preseed_crypto.cfg
+ linux /install/vmlinuz preseed/file=/cdrom/install/preseed_multipar_crypto.cfg auto=true priority=critical vga=normal locales=en_US.UTF-8 keyboard-layouts=us
initrd /install/initrd.gz
}
menuentry "Install Hedgehog Linux (advanced configuration)" {
- linux /install/vmlinuz auto=true priority=high vga=normal preseed/file=/cdrom/install/preseed_minimal.cfg
+ linux /install/vmlinuz preseed/file=/cdrom/install/preseed_minimal.cfg auto=true priority=high vga=normal
initrd /install/initrd.gz
}
menuentry "Install Hedgehog Linux (virtual machine single partition quick install)" {
- linux /install/vmlinuz auto=true priority=critical vga=normal locales=en_US.UTF-8 keyboard-layouts=us preseed/file=/cdrom/install/preseed_vmware.cfg
+ linux /install/vmlinuz preseed/file=/cdrom/install/preseed_vmware.cfg auto=true priority=critical vga=normal locales=en_US.UTF-8 keyboard-layouts=us
initrd /install/initrd.gz
}
diff --git a/sensor-iso/config/includes.binary/install/preseed_base.cfg b/sensor-iso/config/includes.binary/install/preseed_base.cfg
index bc5cbae96..de9e04772 100644
--- a/sensor-iso/config/includes.binary/install/preseed_base.cfg
+++ b/sensor-iso/config/includes.binary/install/preseed_base.cfg
@@ -17,6 +17,18 @@ d-i time/zone string Universal
d-i clock-setup/ntp boolean false
d-i clock-setup/ntp-server string 0.debian.pool.ntp.org
+d-i popularity-contest/participate boolean false
+
+localepurge localepurge/dontbothernew boolean false
+localepurge localepurge/mandelete boolean true
+localepurge localepurge/none_selected boolean false
+localepurge localepurge/nopurge multiselect en, en_US, en_us.UTF-8, C.UTF-8
+localepurge localepurge/quickndirtycalc boolean true
+localepurge localepurge/remove_no note
+localepurge localepurge/showfreedspace boolean false
+localepurge localepurge/use-dpkg-feature boolean false
+localepurge localepurge/verbose boolean false
+
d-i passwd/username string sensor
d-i passwd/user-fullname string sensor
d-i passwd/user-default-groups string audio cdrom video netdev plugdev vboxsf
@@ -42,11 +54,10 @@ d-i preseed/late_command string \
in-target sed -r -i 's@(^.+\s+/(tmp|var/tmp)\s+ext4\s+.*defaults)@\1,nosuid,nodev,noexec@g' /etc/fstab; \
in-target sed -r -i 's@(^.+/media/cdrom[0-9]*.+)(noauto)(.*)@\1\2,nosuid,nodev,noexec\3@g' /etc/fstab; \
in-target sed -r -i 's@(^.+\s+/(home)\s+ext4\s+.*defaults)@\1,nosuid,nodev@g' /etc/fstab; \
- in-target bash -c "echo '\EFI\debian\grubx64.efi' > /boot/efi/startup.nsh"; \
+ in-target bash -c "( echo '\EFI\debian\grubx64.efi' > /boot/efi/startup.nsh ) || true"; \
in-target sed -i 's#^\(GRUB_CMDLINE_LINUX_DEFAULT="quiet\)"$#\1 random.trust_cpu=on elevator=deadline cgroup_enable=memory swapaccount=1 cgroup.memory=nokmem apparmor=1 security=apparmor ipv6.disable=1 audit=1"#' /etc/default/grub; \
in-target sed -i 's#^\(GRUB_CMDLINE_LINUX="\)"$#\1apparmor=1 security=apparmor audit=1"#' /etc/default/grub; \
in-target sed -i 's#^\(GRUB_DISTRIBUTOR=\).*$#\1"Hedgehog"#' /etc/default/grub; \
in-target cp /usr/share/images/desktop-base/hedgehog-wallpaper-plain.png /boot/grub; \
in-target bash /usr/local/bin/preseed_late_user_config.sh; \
- in-target grub-mkconfig -o /boot/grub/grub.cfg; \
- in-target bash -c "(dpkg -s localepurge >/dev/null 2>&1) && (debconf-set-selections < /etc/localepurge-preseed.cfg) && dpkg-reconfigure localepurge && localepurge";
+ in-target grub-mkconfig -o /boot/grub/grub.cfg;
diff --git a/sensor-iso/config/includes.binary/install/preseed.cfg b/sensor-iso/config/includes.binary/install/preseed_multipar.cfg
similarity index 93%
rename from sensor-iso/config/includes.binary/install/preseed.cfg
rename to sensor-iso/config/includes.binary/install/preseed_multipar.cfg
index d43f02ad3..821cd8d0f 100644
--- a/sensor-iso/config/includes.binary/install/preseed.cfg
+++ b/sensor-iso/config/includes.binary/install/preseed_multipar.cfg
@@ -11,10 +11,10 @@ d-i preseed/include string preseed_base.cfg
# install root filesystem on smallest non-USB disk
d-i partman/early_command string \
- SENSOR_ROOT_DISK=$(parted_devices | egrep "^($(find /sys/block -mindepth 1 -maxdepth 1 -type l \( -name '[hs]d*' -o -name 'nvme*' \) -exec ls -l '{}' ';' | grep -v "usb" | sed 's@^.*\([hs]d[a-z]\+\|nvme[0-9]\+\).*$@/dev/\1@' | sed -e :a -e '$!N; s/\n/|/; ta'))" | sort -k2n | head -1 | cut -f1); \
- pvremove -ff -y "$SENSOR_ROOT_DISK"*; \
- debconf-set partman-auto/disk "$SENSOR_ROOT_DISK"; \
- debconf-set grub-installer/bootdev "$SENSOR_ROOT_DISK"; \
+ ROOT_DISK=$(sh /preseed_partman_determine_disk.sh); \
+ pvremove -ff -y "$ROOT_DISK"*; \
+ debconf-set partman-auto/disk "$ROOT_DISK"; \
+ debconf-set grub-installer/bootdev "$ROOT_DISK"; \
sed -i.bak 's/-f $id\/skip_erase/-d $id/g' /lib/partman/lib/crypto-base.sh;
d-i grub-installer/only_debian boolean true
@@ -22,6 +22,7 @@ d-i grub-installer/with_other_os boolean true
d-i partman-auto/method string lvm
d-i partman-auto-lvm/new_vg_name string main
+d-i partman-auto-lvm/guided_size string max
d-i partman-lvm/device_remove_lvm boolean true
d-i partman-lvm/confirm boolean true
diff --git a/sensor-iso/config/includes.binary/install/preseed_vmware.cfg b/sensor-iso/config/includes.binary/install/preseed_vmware.cfg
index 0a199b98c..87cea5293 100644
--- a/sensor-iso/config/includes.binary/install/preseed_vmware.cfg
+++ b/sensor-iso/config/includes.binary/install/preseed_vmware.cfg
@@ -11,10 +11,10 @@ d-i preseed/include string preseed_base.cfg
# install root filesystem on smallest non-USB disk
d-i partman/early_command string \
- SENSOR_ROOT_DISK=$(parted_devices | egrep "^($(find /sys/block -mindepth 1 -maxdepth 1 -type l \( -name '[hs]d*' -o -name 'nvme*' \) -exec ls -l '{}' ';' | grep -v "usb" | sed 's@^.*\([hs]d[a-z]\+\|nvme[0-9]\+\).*$@/dev/\1@' | sed -e :a -e '$!N; s/\n/|/; ta'))" | sort -k2n | head -1 | cut -f1); \
- pvremove -ff -y "$SENSOR_ROOT_DISK"*; \
- debconf-set partman-auto/disk "$SENSOR_ROOT_DISK"; \
- debconf-set grub-installer/bootdev "$SENSOR_ROOT_DISK"; \
+ ROOT_DISK=$(sh /preseed_partman_determine_disk.sh); \
+ pvremove -ff -y "$ROOT_DISK"*; \
+ debconf-set partman-auto/disk "$ROOT_DISK"; \
+ debconf-set grub-installer/bootdev "$ROOT_DISK"; \
sed -i.bak 's/-f $id\/skip_erase/-d $id/g' /lib/partman/lib/crypto-base.sh;
d-i grub-installer/only_debian boolean true
@@ -22,6 +22,7 @@ d-i grub-installer/with_other_os boolean true
d-i partman-auto/method string lvm
d-i partman-auto-lvm/new_vg_name string main
+d-i partman-auto-lvm/guided_size string max
d-i partman-lvm/device_remove_lvm boolean true
d-i partman-lvm/confirm boolean true
diff --git a/sensor-iso/config/includes.binary/isolinux/advanced.cfg b/sensor-iso/config/includes.binary/isolinux/advanced.cfg
new file mode 100644
index 000000000..1999e2ec0
--- /dev/null
+++ b/sensor-iso/config/includes.binary/isolinux/advanced.cfg
@@ -0,0 +1,34 @@
+label live
+menu label ^Live system
+kernel /live/vmlinuz
+append boot=live components username=sensor nosplash random.trust_cpu=on elevator=deadline cgroup_enable=memory swapaccount=1 cgroup.memory=nokmem initrd=/live/initrd.img --
+
+label liveram
+menu label ^Live system (fully in RAM)
+kernel /live/vmlinuz
+append boot=live toram components username=sensor nosplash random.trust_cpu=on elevator=deadline cgroup_enable=memory swapaccount=1 cgroup.memory=nokmem initrd=/live/initrd.img --
+
+label install
+menu label ^Install Hedgehog Linux (quick install)
+kernel /install/vmlinuz
+append file=/preseed_multipar.cfg initrd=/install/initrd.gz auto=true priority=critical locales=en_US.UTF-8 keyboard-layouts=us --
+
+label installenc
+menu label ^Install Hedgehog Linux (encrypted quick install)
+kernel /install/vmlinuz
+append file=/preseed_multipar_crypto.cfg initrd=/install/initrd.gz auto=true priority=critical locales=en_US.UTF-8 keyboard-layouts=us --
+
+label installadv
+menu label ^Install Hedgehog Linux (advanced configuration)
+kernel /install/vmlinuz
+append file=/preseed_minimal.cfg initrd=/install/initrd.gz auto=true priority=high --
+
+label installvm
+menu label ^Install Hedgehog Linux (virtual machine single partition quick install)
+kernel /install/vmlinuz
+append file=/preseed_vmware.cfg initrd=/install/initrd.gz auto=true priority=critical locales=en_US.UTF-8 keyboard-layouts=us --
+
+label rescue
+menu label ^Rescue system in text mode
+kernel /install/vmlinuz
+append rescue/enable=true initrd=/install/initrd.gz --
diff --git a/sensor-iso/config/includes.binary/isolinux/install.cfg b/sensor-iso/config/includes.binary/isolinux/install.cfg
new file mode 100644
index 000000000..e69de29bb
diff --git a/sensor-iso/config/includes.chroot/etc/localepurge-preseed.cfg b/sensor-iso/config/includes.chroot/etc/localepurge-preseed.cfg
deleted file mode 100644
index ac377e228..000000000
--- a/sensor-iso/config/includes.chroot/etc/localepurge-preseed.cfg
+++ /dev/null
@@ -1,9 +0,0 @@
-localepurge localepurge/nopurge multiselect en, en_US, en_us.UTF-8, C.UTF-8
-localepurge localepurge/use-dpkg-feature boolean false
-localepurge localepurge/none_selected boolean false
-localepurge localepurge/verbose boolean false
-localepurge localepurge/dontbothernew boolean false
-localepurge localepurge/quickndirtycalc boolean true
-localepurge localepurge/mandelete boolean true
-localepurge localepurge/showfreedspace boolean false
-localepurge localepurge/remove_no note
\ No newline at end of file
diff --git a/sensor-iso/config/includes.chroot/opt/zeek/bin/zeekdeploy.sh b/sensor-iso/config/includes.chroot/opt/zeek/bin/zeekdeploy.sh
index 1b7d9da2b..4b9d2c7c4 100755
--- a/sensor-iso/config/includes.chroot/opt/zeek/bin/zeekdeploy.sh
+++ b/sensor-iso/config/includes.chroot/opt/zeek/bin/zeekdeploy.sh
@@ -63,8 +63,10 @@ fi
ZEEK_LOG_PATH="$($REALPATH "$ZEEK_LOG_PATH")"
ARCHIVE_PATH="$ZEEK_LOG_PATH/logs"
WORK_PATH="$ZEEK_LOG_PATH/spool"
+TMP_PATH="$ZEEK_INSTALL_PATH/spool/tmp"
EXTRACT_FILES_PATH="$ZEEK_LOG_PATH/extract_files"
-mkdir -p "$ARCHIVE_PATH" "$WORK_PATH" "$EXTRACT_FILES_PATH"
+mkdir -p "$ARCHIVE_PATH" "$WORK_PATH" "$EXTRACT_FILES_PATH" "$TMP_PATH"
+export TMP="$TMP_PATH"
# if file extraction is enabled and file extraction script exists, set up the argument for zeek to use it
[[ -z $ZEEK_RULESET ]] && ZEEK_RULESET="local"
@@ -128,7 +130,7 @@ for IFACE in ${CAPTURE_INTERFACE//,/ }; do
type=worker
host=localhost
interface=$IFACE
-env_vars=ZEEK_EXTRACTOR_MODE=$ZEEK_EXTRACTOR_MODE,ZEEK_EXTRACTOR_PATH=$EXTRACT_FILES_PATH/
+env_vars=ZEEK_EXTRACTOR_MODE=$ZEEK_EXTRACTOR_MODE,ZEEK_EXTRACTOR_PATH=$EXTRACT_FILES_PATH/,TMP=$TMP_PATH
EOF
# if af_packet is available in the kernel, write it out as well
if [ $AF_PACKET_SUPPORT -gt 0 ] && [ $ZEEK_LB_PROCS -gt 0 ]; then
@@ -154,6 +156,7 @@ pushd "$ZEEK_LOG_PATH" >/dev/null 2>&1
function finish {
echo "Stopping via \"$ZEEK_CTL\"" >&2
"$ZEEK_CTL" stop
+ rm -f "$TMP_PATH"/*
}
trap finish EXIT
diff --git a/sensor-iso/config/package-lists/grub.list.binary b/sensor-iso/config/package-lists/grub.list.binary
new file mode 100644
index 000000000..bed168d86
--- /dev/null
+++ b/sensor-iso/config/package-lists/grub.list.binary
@@ -0,0 +1,3 @@
+grub-pc-bin
+grub-efi-amd64-bin
+grub-efi-amd64
diff --git a/sensor-iso/config/package-lists/system.list.chroot b/sensor-iso/config/package-lists/system.list.chroot
index d168371d2..5f200ba3c 100644
--- a/sensor-iso/config/package-lists/system.list.chroot
+++ b/sensor-iso/config/package-lists/system.list.chroot
@@ -61,9 +61,6 @@ gnupg2
google-perftools
gpart
gparted
-grub-efi-amd64
-grub-efi-amd64-bin
-grub-efi-ia32-bin
gvfs
gvfs-backends
gvfs-daemons
@@ -128,6 +125,7 @@ libzmq5
llvm-10
llvm-10-dev
lm-sensors
+localepurge
locales-all
lshw
lsof
@@ -165,7 +163,9 @@ psmisc
pv
pwgen
python
+python-backports-shutil-get-terminal-size
python-dev
+python-pip
python3
python3-dev
python3-pip
diff --git a/sensor-iso/docs/Notes.md b/sensor-iso/docs/Notes.md
index 2ca89114b..854ce142a 100644
--- a/sensor-iso/docs/Notes.md
+++ b/sensor-iso/docs/Notes.md
@@ -321,7 +321,7 @@ This may require opening a firewall port to the host running Moloch viewer to al
# Zeek
-At the time of writing, the [current stable release](https://github.com/zeek/zeek/blob/release/NEWS) of Zeek is [v3.0.8](https://github.com/zeek/zeek/releases/tag/v3.0.8). The notes in this section apply to that version, although some may apply to others as well.
+At the time of writing, the [current stable release](https://github.com/zeek/zeek/blob/release/NEWS) of Zeek is [v3.0.10](https://github.com/zeek/zeek/releases/tag/v3.0.10). The notes in this section apply to that version, although some may apply to others as well.
## Compiling Zeek from source
@@ -330,7 +330,7 @@ The following bash script was used to download, [build and install](https://docs
```bash
#!/bin/bash
-ZEEK_VER="3.0.8"
+ZEEK_VER="3.0.10"
ZEEK_URL="https://old.zeek.org/downloads/zeek-$ZEEK_VER.tar.gz"
ZEEK_PATCH_URLS=(
# nothing here for now
@@ -370,7 +370,7 @@ Hedgehog Linux utilizest he following third party Zeek packages:
* Corelight's [bro-xor-exe](https://github.com/corelight/bro-xor-exe-plugin) plugin
* Corelight's [community ID](https://github.com/corelight/bro-community-id) flow hashing plugin
* J-Gras' [Zeek::AF_Packet](https://github.com/J-Gras/zeek-af_packet-plugin) plugin
-* Lexi Brent's [EternalSafety](https://github.com/lexibrent/zeek-EternalSafety) plugin
+* Lexi Brent's [EternalSafety](https://github.com/0xl3x1/zeek-EternalSafety) plugin
* MITRE Cyber Analytics Repository's [Bro/Zeek ATT&CK-Based Analytics (BZAR)](https://github.com/mitre-attack/car/tree/master/implementations) script
* Salesforce's [gQUIC](https://github.com/salesforce/GQUIC_Protocol_Analyzer) analyzer
* Salesforce's [HASSH](https://github.com/salesforce/hassh) SSH fingerprinting plugin
@@ -456,7 +456,7 @@ ZKG_GITHUB_URLS=(
https://github.com/amzn/zeek-plugin-tds
https://github.com/corelight/bro-community-id
https://github.com/corelight/bro-xor-exe-plugin
- https://github.com/lexibrent/zeek-EternalSafety
+ https://github.com/0xl3x1/zeek-EternalSafety
https://github.com/salesforce/hassh
https://github.com/salesforce/ja3
)
diff --git a/sensor-iso/interface/requirements.txt b/sensor-iso/interface/requirements.txt
index 25c8115b0..79bfc26dc 100644
--- a/sensor-iso/interface/requirements.txt
+++ b/sensor-iso/interface/requirements.txt
@@ -8,7 +8,7 @@ idna==2.10
itsdangerous==1.1.0
Jinja2==2.11.2
MarkupSafe==1.1.1
-psutil==5.7.2
+psutil
python-dotenv==0.14.0
requests==2.24.0
six==1.15.0
diff --git a/sensor-iso/interface/sensor_ctl/control_vars.conf b/sensor-iso/interface/sensor_ctl/control_vars.conf
index f8aa3b278..9949b91e6 100644
--- a/sensor-iso/interface/sensor_ctl/control_vars.conf
+++ b/sensor-iso/interface/sensor_ctl/control_vars.conf
@@ -55,12 +55,15 @@ export MALASS_HOST=""
export MALASS_PORT=80
export EXTRACTED_FILE_YARA_CUSTOM_ONLY=false
export YARA_RULES_DIR=/opt/yara-rules
+export CAPA_VERBOSE=false
+export CAPA_RULES_DIR=/opt/capa-rules
export ZEEK_FILE_WATCH=false
export ZEEK_FILE_SCAN_CLAMAV=false
export ZEEK_FILE_SCAN_VTOT=false
export ZEEK_FILE_SCAN_MALASS=false
export ZEEK_FILE_SCAN_YARA=false
+export ZEEK_FILE_SCAN_CAPA=false
export AUTOSTART_AUDITBEAT=false
export AUTOSTART_CLAMAV_UPDATES=false
diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/zeek.conf b/sensor-iso/interface/sensor_ctl/supervisor.d/zeek.conf
index 61773a440..6770990c4 100644
--- a/sensor-iso/interface/sensor_ctl/supervisor.d/zeek.conf
+++ b/sensor-iso/interface/sensor_ctl/supervisor.d/zeek.conf
@@ -1,5 +1,5 @@
[group:zeek]
-programs=zeekctl,watcher,virustotal,clamav,yara,malass,logger
+programs=zeekctl,watcher,virustotal,clamav,yara,capa,malass,logger
[program:zeekctl]
command=/opt/zeek/bin/zeekdeploy.sh
@@ -64,6 +64,20 @@ autostart=%(ENV_ZEEK_FILE_SCAN_YARA)s
directory=%(ENV_ZEEK_LOG_PATH)s
user=sensor
+[program:capa]
+command=/usr/bin/python3.7 /usr/local/bin/zeek_carve_scanner.py
+ --start-sleep 20
+ --capa %(ENV_ZEEK_FILE_SCAN_CAPA)s
+ --capa-rules "%(ENV_CAPA_RULES_DIR)s"
+ --capa-verbose %(ENV_CAPA_VERBOSE)s
+startsecs=30
+startretries=3
+stopasgroup=true
+killasgroup=true
+autostart=%(ENV_ZEEK_FILE_SCAN_CAPA)s
+directory=%(ENV_ZEEK_LOG_PATH)s
+user=sensor
+
[program:malass]
command=/usr/bin/python3.7 /usr/local/bin/zeek_carve_scanner.py
--start-sleep 20
diff --git a/sensor-iso/moloch/Dockerfile b/sensor-iso/moloch/Dockerfile
index 8786de405..be3270286 100644
--- a/sensor-iso/moloch/Dockerfile
+++ b/sensor-iso/moloch/Dockerfile
@@ -6,7 +6,7 @@ LABEL maintainer="malcolm.netsec@gmail.com"
ENV DEBIAN_FRONTEND noninteractive
-ENV MOLOCH_VERSION "2.4.0"
+ENV MOLOCH_VERSION "2.4.1"
ENV MOLOCHDIR "/opt/moloch"
RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list && \
diff --git a/sensor-iso/vagrant/Vagrantfile b/sensor-iso/vagrant/Vagrantfile
index e56b11491..de53361f6 100644
--- a/sensor-iso/vagrant/Vagrantfile
+++ b/sensor-iso/vagrant/Vagrantfile
@@ -40,7 +40,7 @@ Vagrant.configure("2") do |config|
export KERNEL_VERSION=$(apt-cache search linux-image-5 | grep -Pv -- '(-(rt|cloud)-amd64|amd64-(dbg|unsigned))' | sort -r --sort=version | awk '{print $1}' | head -n 1 | sed 's/^linux-image-//' | sed 's/-amd64$//')
apt-get -t buster-backports install -y \
linux-image-$KERNEL_VERSION-amd64 linux-headers-$KERNEL_VERSION-amd64 linux-headers-$KERNEL_VERSION-common \
- dkms build-essential linux-kbuild-5.6 linux-compiler-gcc-8-x86 \
+ dkms build-essential linux-kbuild-5.7 linux-compiler-gcc-8-x86 \
firmware-linux firmware-linux-nonfree firmware-misc-nonfree firmware-amd-graphics
ls /dev/disk/by-id/ata-* | grep -v '\\-part' | head -n 1 | xargs -r -l grub-install
STEP1
diff --git a/shared/bin/preseed_partman_determine_disk.sh b/shared/bin/preseed_partman_determine_disk.sh
new file mode 100755
index 000000000..847787a88
--- /dev/null
+++ b/shared/bin/preseed_partman_determine_disk.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+parted_devices | egrep "^($(find /sys/block -mindepth 1 -maxdepth 1 -type l \( -name '[hs]d*' -o -name 'nvme*' \) -exec ls -l '{}' ';' | grep -v "usb" | sed 's@^.*\([hs]d[a-z]\+\|nvme[0-9]\+\).*$@/dev/\1@' | sed -e :a -e '$!N; s/\n/|/; ta'))" | sort -k2n | head -1 | cut -f1
diff --git a/shared/bin/zeek_carve_logger.py b/shared/bin/zeek_carve_logger.py
index b8bb23ddc..fce354f7c 100755
--- a/shared/bin/zeek_carve_logger.py
+++ b/shared/bin/zeek_carve_logger.py
@@ -153,6 +153,7 @@ def main():
scanners = set()
fileScanCounts = defaultdict(AtomicInt)
+ fileScanHits = defaultdict(AtomicInt)
# open and write out header for our super legit zeek signature.log file
with open(broSigLogSpec, 'w+', 1) if (broSigLogSpec is not None) else nullcontext() as broSigFile:
@@ -209,11 +210,12 @@ def main():
fileName = scanResult[FILE_SCAN_RESULT_FILE]
fileNameBase = os.path.basename(fileName)
- # we may quarantine the file if fileScanCount < len(scanners), but we won't delete it so the rest of the scanners can find it
+ # we won't delete or move/quarantine a file until fileScanCount < len(scanners)
fileScanCount = fileScanCounts[fileNameBase].increment()
if triggered:
# this file had a "hit" in one of the virus engines, log it!
+ fileScanHitCount = fileScanHits[fileNameBase].increment()
# format the line as it should appear in the signatures log file
fileSpecFields = extracted_filespec_to_fields(fileName)
@@ -233,43 +235,47 @@ def main():
else:
print(broLineStr, file=broSigFile, flush=True)
+ else:
+ fileScanHitCount = fileScanHits[fileNameBase].value()
+
# finally, what to do with the file itself
if os.path.isfile(fileName):
- if triggered and (args.preserveMode != PRESERVE_NONE):
+ # once all of the scanners have had their turn...
+ if (fileScanCount >= len(scanners)):
fileScanCounts.pop(fileNameBase, None)
+ fileScanHits.pop(fileNameBase, None)
- # move triggering file to quarantine
- if not same_file_or_dir(fileName, os.path.join(quarantineDir, fileNameBase)): # unless it's already there
-
- try:
- shutil.move(fileName, quarantineDir)
- if debug: eprint(f"{scriptName}:\t⏩\t{fileName} ({fileScanCount}/{len(scanners)})")
- except Exception as e:
- eprint(f"{scriptName}:\t❗\t🚫\t{fileName} move exception: {e}")
- # hm move failed, delete it i guess?
- os.remove(fileName)
+ if (fileScanHitCount > 0) and (args.preserveMode != PRESERVE_NONE):
- elif (fileScanCount >= len(scanners)):
- fileScanCounts.pop(fileNameBase, None)
-
- if not same_file_or_dir(quarantineDir, os.path.dirname(fileName)): # don't move or delete if it's already quarantined
+ # move triggering file to quarantine
+ if not same_file_or_dir(fileName, os.path.join(quarantineDir, fileNameBase)): # unless it's somehow already there
- if (args.preserveMode == PRESERVE_ALL):
- # move non-triggering file to preserved directory
try:
- shutil.move(fileName, preserveDir)
- if verboseDebug: eprint(f"{scriptName}:\t⏩\t{fileName} ({fileScanCount}/{len(scanners)})")
+ shutil.move(fileName, quarantineDir)
+ if debug: eprint(f"{scriptName}:\t⏩\t{fileName} ({fileScanCount}/{len(scanners)})")
except Exception as e:
eprint(f"{scriptName}:\t❗\t🚫\t{fileName} move exception: {e}")
# hm move failed, delete it i guess?
os.remove(fileName)
- else:
- # delete the file
- os.remove(fileName)
- fileScanCounts.pop(fileNameBase, None)
- if verboseDebug: eprint(f"{scriptName}:\t🚫\t{fileName} ({fileScanCount}/{len(scanners)})")
+ else:
+ if not same_file_or_dir(quarantineDir, os.path.dirname(fileName)): # don't move or delete if it's somehow already quarantined
+
+ if (args.preserveMode == PRESERVE_ALL):
+ # move non-triggering file to preserved directory
+ try:
+ shutil.move(fileName, preserveDir)
+ if verboseDebug: eprint(f"{scriptName}:\t⏩\t{fileName} ({fileScanCount}/{len(scanners)})")
+ except Exception as e:
+ eprint(f"{scriptName}:\t❗\t🚫\t{fileName} move exception: {e}")
+ # hm move failed, delete it i guess?
+ os.remove(fileName)
+
+ else:
+ # delete the file
+ os.remove(fileName)
+ if verboseDebug: eprint(f"{scriptName}:\t🚫\t{fileName} ({fileScanCount}/{len(scanners)})")
# graceful shutdown
if debug:
diff --git a/shared/bin/zeek_carve_scanner.py b/shared/bin/zeek_carve_scanner.py
index 903ed7d21..1fdb364c9 100755
--- a/shared/bin/zeek_carve_scanner.py
+++ b/shared/bin/zeek_carve_scanner.py
@@ -57,9 +57,16 @@ def debug_toggle_handler(signum, frame):
###################################################################################################
# look for a file to scan (probably in its original directory, but possibly already moved to quarantine)
-def locate_file(fileName):
+def locate_file(fileInfo):
global verboseDebug
+ if isinstance(fileInfo, dict) and (FILE_SCAN_RESULT_FILE in fileInfo):
+ fileName = fileInfo[FILE_SCAN_RESULT_FILE]
+ elif isinstance(fileInfo, str):
+ fileName = fileInfo
+ else:
+ fileName = None
+
if fileName is not None:
if os.path.isfile(fileName):
@@ -100,6 +107,7 @@ def scanFileWorker(checkConnInfo, carvedFileSub):
# scanned_files_socket.SNDTIMEO = 5000
if debug: eprint(f"{scriptName}[{scanWorkerId}]:\tconnected to sink at {SINK_PORT}")
+ fileInfo = None
fileName = None
retrySubmitFile = False # todo: maximum file retry count?
@@ -120,26 +128,28 @@ def scanFileWorker(checkConnInfo, carvedFileSub):
if shuttingDown:
break
- if retrySubmitFile and (fileName is not None) and (locate_file(fileName) is not None):
+ if retrySubmitFile and (fileInfo is not None) and (locate_file(fileInfo) is not None):
# we were unable to submit the file for processing, so try again
time.sleep(1)
- if debug: eprint(f"{scriptName}[{scanWorkerId}]:\t🔃\t{fileName}")
+ if debug: eprint(f"{scriptName}[{scanWorkerId}]:\t🔃\t{json.dumps(fileInfo)}")
else:
retrySubmitFile = False
- # read a filename from the subscription
- fileName = carvedFileSub.Pull(scanWorkerId=scanWorkerId)
+ # read watched file information from the subscription
+ fileInfo = carvedFileSub.Pull(scanWorkerId=scanWorkerId)
- fileName = locate_file(fileName)
+ fileName = locate_file(fileInfo)
if (fileName is not None) and os.path.isfile(fileName):
# file exists, submit for scanning
- if debug: eprint(f"{scriptName}[{scanWorkerId}]:\t🔎\t{fileName}")
+ if debug: eprint(f"{scriptName}[{scanWorkerId}]:\t🔎\t{json.dumps(fileInfo)}")
requestComplete = False
scanResult = None
+ fileSize = int(fileInfo[FILE_SCAN_RESULT_FILE_SIZE]) if isinstance(fileInfo[FILE_SCAN_RESULT_FILE_SIZE], int) or (isinstance(fileInfo[FILE_SCAN_RESULT_FILE_SIZE], str) and fileInfo[FILE_SCAN_RESULT_FILE_SIZE].isdecimal()) else None
scan = AnalyzerScan(provider=checkConnInfo, name=fileName,
- submissionResponse=checkConnInfo.submit(fileName=fileName, block=False))
-
+ size=fileSize,
+ fileType=fileInfo[FILE_SCAN_RESULT_FILE_TYPE],
+ submissionResponse=checkConnInfo.submit(fileName=fileName, fileSize=fileSize, fileType=fileInfo[FILE_SCAN_RESULT_FILE_TYPE], block=False))
if scan.submissionResponse is not None:
if debug: eprint(f"{scriptName}[{scanWorkerId}]:\t🔍\t{fileName}")
@@ -161,7 +171,7 @@ def scanFileWorker(checkConnInfo, carvedFileSub):
if response.success:
# successful scan, report the scan results
- scanResult = response.result
+ scanResult = response
elif isinstance(response.result, dict) and ("error" in response.result):
# scan errored out, report the error
@@ -233,6 +243,9 @@ def main():
parser.add_argument('--clamav-socket', dest='clamAvSocket', help="ClamAV socket filename", metavar='', type=str, required=False, default=None)
parser.add_argument('--yara', dest='enableYara', metavar='true|false', help="Enable Yara", type=str2bool, nargs='?', const=True, default=False, required=False)
parser.add_argument('--yara-custom-only', dest='yaraCustomOnly', metavar='true|false', help="Ignore default Yara rules", type=str2bool, nargs='?', const=True, default=False, required=False)
+ parser.add_argument('--capa', dest='enableCapa', metavar='true|false', help="Enable Capa", type=str2bool, nargs='?', const=True, default=False, required=False)
+ parser.add_argument('--capa-rules', dest='capaRulesDir', help="Capa Rules Directory", metavar='', type=str, required=False)
+ parser.add_argument('--capa-verbose', dest='capaVerbose', metavar='true|false', help="Log all capa rules, not just MITRE ATT&CK technique classifications", type=str2bool, nargs='?', const=True, default=False, required=False)
try:
parser.error = parser.exit
@@ -273,6 +286,8 @@ def main():
yaraDirs.append(YARA_RULES_DIR)
yaraDirs.append(YARA_CUSTOM_RULES_DIR)
checkConnInfo = YaraScan(debug=debug, verboseDebug=verboseDebug, rulesDirs=yaraDirs)
+ elif args.enableCapa:
+ checkConnInfo = CapaScan(debug=debug, verboseDebug=verboseDebug, rulesDir=args.capaRulesDir, verboseHits=args.capaVerbose)
else:
if not args.enableClamAv:
eprint('No scanner specified, defaulting to ClamAV')
diff --git a/shared/bin/zeek_carve_utils.py b/shared/bin/zeek_carve_utils.py
index cdc8d5d6f..7cd937ee8 100644
--- a/shared/bin/zeek_carve_utils.py
+++ b/shared/bin/zeek_carve_utils.py
@@ -22,6 +22,7 @@
from collections import defaultdict
from datetime import datetime
from multiprocessing import RawValue
+from subprocess import (PIPE, Popen)
from threading import get_ident
from threading import Lock
@@ -42,6 +43,8 @@
###################################################################################################
FILE_SCAN_RESULT_SCANNER = "scanner"
FILE_SCAN_RESULT_FILE = "file"
+FILE_SCAN_RESULT_FILE_SIZE = "size"
+FILE_SCAN_RESULT_FILE_TYPE = "type"
FILE_SCAN_RESULT_ENGINES = "engines"
FILE_SCAN_RESULT_HITS = "hits"
FILE_SCAN_RESULT_MESSAGE = "message"
@@ -87,6 +90,19 @@
YARA_ENGINE_ID = 'Yara'
YARA_MAX_REQS = 8 # maximum scanning threads concurrently
YARA_CHECK_INTERVAL = 0.1
+YARA_RUN_TIMEOUT_SEC = 300
+
+###################################################################################################
+# Capa
+CAPA_MAX_REQS = 4 # maximum scanning threads concurrently
+CAPA_SUBMIT_TIMEOUT_SEC = 60
+CAPA_ENGINE_ID = 'Capa'
+CAPA_CHECK_INTERVAL = 0.1
+CAPA_MIMES_TO_SCAN = ('application/bat', 'application/ecmascript', 'application/javascript', 'application/PowerShell', 'application/vnd.microsoft.portable-executable', 'application/x-bat', 'application/x-dosexec', 'application/x-executable', 'application/x-msdos-program', 'application/x-msdownload', 'application/x-pe-app-32bit-i386', 'application/x-sh', 'text/jscript', 'text/vbscript', 'text/x-python', 'text/x-shellscript')
+CAPA_VIV_SUFFIX = '.viv'
+CAPA_VIV_MIME = 'data'
+CAPA_ATTACK_KEY = 'att&ck'
+CAPA_RUN_TIMEOUT_SEC = 300
###################################################################################################
@@ -122,12 +138,16 @@ def signature_types_line(cls):
# AnalyzerScan
# .provider - a FileScanProvider subclass doing the scan/lookup
# .name - the filename to be scanned
+# .size - the size (in bytes) of the file
+# .fileType - the file's mime type
# .submissionResponse - a unique identifier to be returned by the provider with which to check status
class AnalyzerScan:
- __slots__ = ('provider', 'name', 'submissionResponse')
- def __init__(self, provider=None, name=None, submissionResponse=None):
+ __slots__ = ('provider', 'name', 'size', 'fileType', 'submissionResponse')
+ def __init__(self, provider=None, name=None, size=None, fileType=None, submissionResponse=None):
self.provider = provider
self.name = name
+ self.size = size
+ self.fileType = fileType
self.submissionResponse = submissionResponse
# AnalyzerResult
@@ -135,10 +155,11 @@ def __init__(self, provider=None, name=None, submissionResponse=None):
# .success - requesting the status was done successfully (whether or not it was finished)
# .result - the "result" of the scan/lookup, in whatever format is native to the provider
class AnalyzerResult:
- __slots__ = ('finished', 'success', 'result')
- def __init__(self, finished=False, success=False, result=None):
+ __slots__ = ('finished', 'success', 'verbose', 'result')
+ def __init__(self, finished=False, success=False, verbose=False, result=None):
self.finished = finished
self.success = success
+ self.verbose = verbose
self.result = result
# the filename parts used by our Zeek instance for extracted files:
@@ -178,6 +199,12 @@ def sha256sum(filename):
h.update(mv[:n])
return h.hexdigest()
+###################################################################################################
+# recursive dictionary key search
+def dictsearch(d, target):
+ val = filter(None, [[b] if a == target else dictsearch(b, target) if isinstance(b, dict) else None for a, b in d.items()])
+ return [i for b in val for i in b]
+
###################################################################################################
# filespec to various fields as per the extractor zeek script (/opt/zeek/share/zeek/site/extractor.zeek)
# source-fuid-uid-time.ext
@@ -214,6 +241,61 @@ def touch(filename):
open(filename, 'a').close()
os.utime(filename, None)
+###################################################################################################
+# run command with arguments and return its exit code, stdout, and stderr
+def check_output_input(*popenargs, **kwargs):
+
+ if 'stdout' in kwargs:
+ raise ValueError('stdout argument not allowed, it will be overridden')
+
+ if 'stderr' in kwargs:
+ raise ValueError('stderr argument not allowed, it will be overridden')
+
+ if 'input' in kwargs and kwargs['input']:
+ if 'stdin' in kwargs:
+ raise ValueError('stdin and input arguments may not both be used')
+ inputdata = kwargs['input']
+ kwargs['stdin'] = PIPE
+ else:
+ inputdata = None
+ kwargs.pop('input', None)
+
+ process = Popen(*popenargs, stdout=PIPE, stderr=PIPE, **kwargs)
+ try:
+ output, errput = process.communicate(input=inputdata)
+ except:
+ process.kill()
+ process.wait()
+ raise
+
+ retcode = process.poll()
+
+ return retcode, output, errput
+
+###################################################################################################
+# run command with arguments and return its exit code and output
+def run_process(command, stdout=True, stderr=True, stdin=None, cwd=None, env=None, debug=False):
+
+ retcode = -1
+ output = []
+
+ try:
+ # run the command
+ retcode, cmdout, cmderr = check_output_input(command, input=stdin.encode() if stdin else None, cwd=cwd, env=env)
+
+ # split the output on newlines to return a list
+ if stderr and (len(cmderr) > 0): output.extend(cmderr.decode(sys.getdefaultencoding()).split('\n'))
+ if stdout and (len(cmdout) > 0): output.extend(cmdout.decode(sys.getdefaultencoding()).split('\n'))
+
+ except (FileNotFoundError, OSError, IOError) as e:
+ if stderr:
+ output.append("Command {} not found or unable to execute".format(command))
+
+ if debug:
+ eprint("{}{} returned {}: {}".format(command, "({})".format(stdin[:80] + bool(stdin[80:]) * '...' if stdin else ""), retcode, output))
+
+ return retcode, output
+
###################################################################################################
class AtomicInt:
def __init__(self, value=0):
@@ -259,18 +341,20 @@ def __init__(self, debug=False, verboseDebug=False, host="localhost", port=VENTI
# ---------------------------------------------------------------------------------
def Pull(self, scanWorkerId=0):
+ fileinfo = defaultdict(str)
+
with self.lock:
- # accept a filename from newFilesSocket
+ # accept a fileinfo dict from newFilesSocket
try:
- filename = self.newFilesSocket.recv_string()
+ fileinfo.update(json.loads(self.newFilesSocket.recv_string()))
except zmq.Again as timeout:
- # no file received due to timeout, return "None" which means no file available
- filename = None
+ # no file received due to timeout, return empty dict. which means no file available
+ pass
if self.verboseDebug:
- eprint(f"{self.scriptName}[{scanWorkerId}]:\t{'📨' if (filename is not None) else '🕑'}\t{filename if (filename is not None) else '(recv)'}")
+ eprint(f"{self.scriptName}[{scanWorkerId}]:\t{'📨' if (FILE_SCAN_RESULT_FILE in fileinfo) else '🕑'}\t{fileinfo[FILE_SCAN_RESULT_FILE] if (FILE_SCAN_RESULT_FILE in fileinfo) else '(recv)'}")
- return filename
+ return fileinfo
###################################################################################################
class FileScanProvider(ABC):
@@ -294,7 +378,7 @@ def check_interval(cls):
pass
@abstractmethod
- def submit(self, fileName=None, block=False, timeout=0):
+ def submit(self, fileName=None, fileSize=None, fileType=None, block=False, timeout=0):
# returns something that can be passed into check_result for checking the scan status
pass
@@ -339,7 +423,7 @@ def check_interval():
# VirusTotalSearch does the request and gets the response immediately;
# the subsequent call to check_result (using submit's response as input)
# will always return "True" since the work has already been done
- def submit(self, fileName=None, block=False, timeout=0):
+ def submit(self, fileName=None, fileSize=None, fileType=None, block=False, timeout=0):
if timeout is None:
timeout = self.reqLimitSec+5
@@ -479,7 +563,7 @@ def check_interval():
# ---------------------------------------------------------------------------------
# upload a file to scan with Malass, respecting rate limiting. return submitted transaction ID
- def submit(self, fileName=None, block=False, timeout=MAL_SUBMIT_TIMEOUT_SEC):
+ def submit(self, fileName=None, fileSize=None, fileType=None, block=False, timeout=MAL_SUBMIT_TIMEOUT_SEC):
submittedTransactionId = None
allowed = False
@@ -652,7 +736,7 @@ def check_interval():
# ---------------------------------------------------------------------------------
# submit a file to scan with ClamAV, respecting rate limiting. return scan result
- def submit(self, fileName=None, block=False, timeout=CLAM_SUBMIT_TIMEOUT_SEC):
+ def submit(self, fileName=None, fileSize=None, fileType=None, block=False, timeout=CLAM_SUBMIT_TIMEOUT_SEC):
clamavResult = AnalyzerResult()
allowed = False
connected = False
@@ -741,10 +825,7 @@ def format(fileName, response):
else:
result[FILE_SCAN_RESULT_MESSAGE] = "Error or invalid response"
- if isinstance(resp, dict) and ('error' in resp):
- result[FILE_SCAN_RESULT_DESCRIPTION] = f"{resp['error']}"
- else:
- result[FILE_SCAN_RESULT_DESCRIPTION] = f"{resp}"
+ result[FILE_SCAN_RESULT_DESCRIPTION] = f"{resp}"
return result
@@ -792,7 +873,7 @@ def check_interval():
# ---------------------------------------------------------------------------------
# submit a file to scan with Yara, respecting rate limiting. return scan result
- def submit(self, fileName=None, block=False, timeout=YARA_SUBMIT_TIMEOUT_SEC):
+ def submit(self, fileName=None, fileSize=None, fileType=None, block=False, timeout=YARA_SUBMIT_TIMEOUT_SEC):
yaraResult = AnalyzerResult()
allowed = False
matches = []
@@ -813,13 +894,15 @@ def submit(self, fileName=None, block=False, timeout=YARA_SUBMIT_TIMEOUT_SEC):
if allowed:
try:
if self.verboseDebug: eprint(f'{get_ident()} Yara scanning: {fileName}')
- yaraResult.result = self.compiledRules.match(fileName)
+ yaraResult.result = self.compiledRules.match(fileName, timeout=YARA_RUN_TIMEOUT_SEC)
if self.verboseDebug: eprint(f'{get_ident()} Yara scan result: {yaraResult.result}')
yaraResult.success = (yaraResult.result is not None)
yaraResult.finished = True
except Exception as e:
if yaraResult.result is None:
- yaraResult.result = str(e)
+ yaraResult.result = {"error" : str(e)}
+ yaraResult.success = False
+ yaraResult.finished = True
if self.debug: eprint(f'{get_ident()} Yara scan error: {yaraResult.result}')
finally:
self.scanningFilesCount.decrement()
@@ -868,4 +951,147 @@ def format(fileName, response):
result[FILE_SCAN_RESULT_MESSAGE] = "Error or invalid response"
result[FILE_SCAN_RESULT_DESCRIPTION] = f"{resp}"
+ return result
+
+###################################################################################################
+# class for scanning a file with Capa
+class CapaScan(FileScanProvider):
+
+ # ---------------------------------------------------------------------------------
+ # constructor
+ def __init__(self, debug=False, verboseDebug=False, rulesDir=None, verboseHits=False):
+ self.scanningFilesCount = AtomicInt(value=0)
+ self.rulesDir = rulesDir
+ self.debug = debug
+ self.verboseDebug = verboseDebug
+ self.verboseHits = verboseHits
+
+ @staticmethod
+ def scanner_name():
+ return 'capa'
+
+ @staticmethod
+ def max_requests():
+ return CAPA_MAX_REQS
+
+ @staticmethod
+ def check_interval():
+ return CAPA_CHECK_INTERVAL
+
+ # ---------------------------------------------------------------------------------
+ # submit a file to scan with Capa, respecting rate limiting. return scan result
+ def submit(self, fileName=None, fileSize=None, fileType=None, block=False, timeout=CAPA_SUBMIT_TIMEOUT_SEC):
+ capaResult = AnalyzerResult(verbose=self.verboseHits)
+
+ if (fileType is not None) and (fileType in CAPA_MIMES_TO_SCAN):
+ allowed = False
+
+ # timeout only applies if block=True
+ timeoutTime = int(time.time()) + timeout
+
+ # while limit only repeats if block=True
+ while (not allowed) and (not capaResult.finished):
+
+ # first make sure we haven't exceeded rate limits
+ if (self.scanningFilesCount.increment() <= CAPA_MAX_REQS):
+ # we've got fewer than the allowed requests open, so we're good to go!
+ allowed = True
+ else:
+ self.scanningFilesCount.decrement()
+
+ if allowed:
+ try:
+ if self.verboseDebug: eprint(f'{get_ident()} Capa scanning: {fileName}')
+
+ if (self.rulesDir is not None):
+ cmd = ['timeout', '-k', '10', '-s', 'TERM', str(CAPA_RUN_TIMEOUT_SEC), 'capa', '--quiet', '-r', self.rulesDir, '--json', '--color', 'never', fileName]
+ else:
+ cmd = ['timeout', '-k', '10', '-s', 'TERM', str(CAPA_RUN_TIMEOUT_SEC), 'capa', '--quiet', '--json', '--color', 'never', fileName]
+ capaErr, capaOut = run_process(cmd, stderr=False, debug=self.debug)
+ if (capaErr == 0) and (len(capaOut) > 0) and (len(capaOut[0]) > 0):
+ # load the JSON output from capa into the .result
+ try:
+ capaResult.result = json.loads(capaOut[0])
+ except (ValueError, TypeError):
+ capaResult.result = {"error" : f"Invalid response: {'; '.join(capaOut)}"}
+
+ else:
+ # probably failed because it's not an executable, ignore it
+ capaResult.result = {"error" : str(capaErr)}
+
+ if self.verboseDebug: eprint(f'{get_ident()} Capa scan result: {capaResult.result}')
+ capaResult.success = (capaResult.result is not None)
+ capaResult.finished = True
+
+ except Exception as e:
+ if capaResult.result is None:
+ capaResult.result = str(e)
+ if self.debug: eprint(f'{get_ident()} Capa scan error: {capaResult.result}')
+
+ finally:
+ self.scanningFilesCount.decrement()
+ try:
+ if os.path.isfile(fileName + CAPA_VIV_SUFFIX):
+ os.remove(fileName + CAPA_VIV_SUFFIX)
+ except Exception as fe:
+ pass
+
+ elif block and (nowTime < timeoutTime):
+ # rate limited, wait for a bit and come around and try again
+ time.sleep(1)
+
+ else:
+ break
+
+ else:
+ # not an executable, don't need to scan it
+ capaResult.result = {}
+ capaResult.success = True
+ capaResult.finished = True
+
+ return capaResult
+
+ # ---------------------------------------------------------------------------------
+ # return the result of the previously scanned file
+ def check_result(self, capaResult):
+ return capaResult if isinstance(capaResult, AnalyzerResult) else AnalyzerResult(finished=True, success=False, result=None)
+
+ # ---------------------------------------------------------------------------------
+ # static method for formatting the response summaryDict (from check_result)
+ @staticmethod
+ def format(fileName, response):
+ result = {FILE_SCAN_RESULT_SCANNER : CapaScan.scanner_name(),
+ FILE_SCAN_RESULT_FILE : fileName,
+ FILE_SCAN_RESULT_ENGINES : 1,
+ FILE_SCAN_RESULT_HITS : 0,
+ FILE_SCAN_RESULT_MESSAGE : None,
+ FILE_SCAN_RESULT_DESCRIPTION : None}
+
+ if isinstance(response, AnalyzerResult):
+ resp = response.result
+ verboseHits = response.verbose
+ else:
+ resp = response
+ verboseHits = False
+
+ if isinstance(resp, dict):
+ hits = []
+ if 'rules' in resp and isinstance(resp['rules'], dict):
+ hits.extend([item.replace('[', '[ATT&CK ') for sublist in dictsearch(resp['rules'], CAPA_ATTACK_KEY) for item in sublist])
+ if verboseHits:
+ hits.extend(list(resp['rules'].keys()))
+
+ result[FILE_SCAN_RESULT_HITS] = len(hits)
+ if (len(hits) > 0):
+ hits = list(set(hits))
+ cnt = Counter(hits)
+ # short message is most common signature name (todo: they won't have duplicate names, so I guess this is just going to take the first...)
+ result[FILE_SCAN_RESULT_MESSAGE] = cnt.most_common(1)[0][0]
+ # long description is list of the signature names and the engines which generated them
+ result[FILE_SCAN_RESULT_DESCRIPTION] = ";".join([f"{x}<{CAPA_ENGINE_ID}>" for x in hits])
+
+ else:
+ result[FILE_SCAN_RESULT_MESSAGE] = "Error or invalid response"
+ result[FILE_SCAN_RESULT_DESCRIPTION] = f"{resp}"
+
return result
\ No newline at end of file
diff --git a/shared/bin/zeek_carve_watcher.py b/shared/bin/zeek_carve_watcher.py
index b8ce2e6cd..e291f0d22 100755
--- a/shared/bin/zeek_carve_watcher.py
+++ b/shared/bin/zeek_carve_watcher.py
@@ -12,6 +12,8 @@
import argparse
import copy
import glob
+import json
+import magic
import os
import pathlib
import pyinotify
@@ -76,15 +78,27 @@ def _method_name(self, event):
if debug: eprint(f"{scriptName}:\t👓\t{event.pathname}")
if (not event.dir) and os.path.isfile(event.pathname):
- if (args.minBytes <= os.path.getsize(event.pathname) <= args.maxBytes):
- # the entity is a right-sized file, and it exists, so send it to get scanned
-
- if debug: eprint(f"{scriptName}:\t📩\t{event.pathname}")
- try:
- self.ventilator_socket.send_string(event.pathname)
- if debug: eprint(f"{scriptName}:\t📫\t{event.pathname}")
- except zmq.Again as timeout:
- if verboseDebug: eprint(f"{scriptName}:\t🕑\t{event.pathname}")
+
+ fileSize = os.path.getsize(event.pathname)
+ if (args.minBytes <= fileSize <= args.maxBytes):
+
+ fileType = magic.from_file(event.pathname, mime=True)
+ if (pathlib.Path(event.pathname).suffix != CAPA_VIV_SUFFIX) and (fileType != CAPA_VIV_MIME):
+ # the entity is a right-sized file, is not a capa .viv cache file, and it exists, so send it to get scanned
+
+ fileInfo = json.dumps({ FILE_SCAN_RESULT_FILE : event.pathname,
+ FILE_SCAN_RESULT_FILE_SIZE : fileSize,
+ FILE_SCAN_RESULT_FILE_TYPE : fileType })
+ if debug: eprint(f"{scriptName}:\t📩\t{fileInfo}")
+ try:
+ self.ventilator_socket.send_string(fileInfo)
+ if debug: eprint(f"{scriptName}:\t📫\t{event.pathname}")
+ except zmq.Again as timeout:
+ if verboseDebug: eprint(f"{scriptName}:\t🕑\t{event.pathname}")
+
+ else:
+ # temporary capa .viv file, just ignore it as it will get cleaned up by the scanner when it's done
+ if debug: eprint(f"{scriptName}:\t🚧\t{event.pathname}")
else:
# too small/big to care about, delete it
diff --git a/shared/bin/zeek_install_plugins.sh b/shared/bin/zeek_install_plugins.sh
index fc032633c..976275b86 100755
--- a/shared/bin/zeek_install_plugins.sh
+++ b/shared/bin/zeek_install_plugins.sh
@@ -78,8 +78,9 @@ ZKG_GITHUB_URLS=(
https://github.com/corelight/ripple20
https://github.com/corelight/SIGRed
https://github.com/corelight/zeek-community-id
+ https://github.com/corelight/zerologon
https://github.com/cybera/zeek-sniffpass
- https://github.com/lexibrent/zeek-EternalSafety
+ https://github.com/0xl3x1/zeek-EternalSafety
https://github.com/mitre-attack/bzar
https://github.com/precurse/zeek-httpattacks
https://github.com/salesforce/hassh