Merge pull request #1383 from m-1-k-3/manual_pre_checking_not_supported

Refactoring, enable threading

README.md

@@ -30,7 +30,7 @@ Author(s): Michael Messner, Pascal Eckmann, Benedikt Kühne
 # EMBA
 ## The security analyzer for firmware of embedded devices
 
-*EMBA* is designed as the central firmware analysis and SBOM tool for penetration testers, product security teams, developers and responsible product managers. It supports the complete security analysis process starting with *firmware extraction*, doing *static analysis* and *dynamic analysis* via emulation, building the SBOM and finally generating a web report. *EMBA* automatically discovers possible weak spots and vulnerabilities in firmware. Examples are insecure binaries, old and outdated software components, potentially vulnerable scripts, or hard-coded passwords. *EMBA* is a command line tool with the possibility to generate an easy-to-use web report for further analysis.
+*EMBA* is designed as the central firmware analysis and SBOM tool for penetration testers, product security teams, developers and responsible product managers. It supports the complete security analysis process starting with *firmware extraction*, doing *static analysis* and *dynamic analysis* via emulation, building the SBOM and finally generating a web based vulnerability report. *EMBA* automatically discovers possible weak spots and vulnerabilities in firmware. Examples are insecure binaries, old and outdated software components, potentially vulnerable scripts, or hard-coded passwords. *EMBA* is a command line tool with the possibility to generate an easy-to-use web report for further analysis.
 
 *EMBA* assists the penetration testers, product security teams and developers in the identification of weak spots and vulnerabilities in the firmware image. *EMBA* provides as much information as possible about the firmware, that the tester can decide on focus areas and is responsible for verifying and interpreting the results.
 

emba

@@ -770,7 +770,7 @@ main() {
         if [[ -f "${LOG_DIR}"/f50_base_aggregator.txt ]]; then
           cat "${LOG_DIR}"/f50_base_aggregator.txt
         fi
-        D_RETURN=0
+        D_RETURN=$(docker inspect "${MAIN_CONTAINER_}" --format='{{.State.ExitCode}}')
       fi
     fi
     enable_strict_mode "${STRICT_MODE}" 0

helpers/helpers_emba_defaults.sh

@@ -90,7 +90,7 @@ set_defaults() {
   export MINIMUM_GPT_PRIO=1     # everything above this value gets checked
 
   export SHORT_PATH=0           # short paths in cli output
-  export THREADED=0             # 0 -> single thread
+  export THREADED=1             # 0 -> single thread
                                 # 1 -> multi threaded
   export YARA=0                 # default: disable yara tests
   export OVERWRITE_LOG=0        # automaticially overwrite log directory, if necessary

helpers/helpers_emba_dependency_check.sh

@@ -449,6 +449,9 @@ dependency_check()
     fi
   fi
 
+  if ! [[ -f "${CONFIG_DIR}"/gh_action ]]; then
+    check_dep_file "NVD CVE database" "${EXT_DIR}""/nvd-json-data-feeds/README.md"
+  fi
   # Python virtual environment in external directory
   check_dep_file "Python virtual environment" "${EXT_DIR}""/emba_venv/bin/activate"
 

helpers/helpers_emba_parameter_parser.sh

@@ -164,9 +164,9 @@ emba_parameter_parsing() {
       S)
         export STRICT_MODE=1
         ;;
-      t)
-        export THREADED=1
-        ;;
+#      t)
+#        export THREADED=1
+#        ;;
       T)
         check_int "${OPTARG}"
         export MAX_MOD_THREADS=""

helpers/helpers_emba_prepare.sh

@@ -419,12 +419,12 @@ prepare_file_arr() {
   print_output "[*] Unique files auto detection for ${ORANGE}${lFIRMWARE_PATH}${NC} (could take some time)\\n"
 
   export FILE_ARR=()
-  readarray -t FILE_ARR < <(find "${lFIRMWARE_PATH}" -xdev "${EXCL_FIND[@]}" -type f -print0|xargs -r -0 -P 16 -I % sh -c 'md5sum %' | sort -u -k1,1 | cut -d\  -f3- )
+  readarray -t FILE_ARR < <(find "${lFIRMWARE_PATH}" -xdev "${EXCL_FIND[@]}" -type f -print0|xargs -r -0 -P 16 -I % sh -c 'md5sum % || true' 2>/dev/null | sort -u -k1,1 | cut -d\  -f3- || true)
   # readarray -t FILE_ARR < <(find "${lFIRMWARE_PATH}" -xdev "${EXCL_FIND[@]}" -type f -exec md5sum {} \; 2>/dev/null | sort -u -k1,1 | cut -d\  -f3- )
   # RTOS handling:
   if [[ -f ${lFIRMWARE_PATH} && ${RTOS} -eq 1 ]]; then
     # readarray -t FILE_ARR_RTOS < <(find "${OUTPUT_DIR}" -xdev -type f -exec md5sum {} \; 2>/dev/null | sort -u -k1,1 | cut -d\  -f3- )
-    readarray -t FILE_ARR_RTOS < <(find "${OUTPUT_DIR}" -xdev -type f -print0|xargs -r -0 -P 16 -I % sh -c 'md5sum %' | sort -u -k1,1 | cut -d\  -f3- )
+    readarray -t FILE_ARR_RTOS < <(find "${OUTPUT_DIR}" -xdev -type f -print0|xargs -r -0 -P 16 -I % sh -c 'md5sum % || true' 2>/dev/null | sort -u -k1,1 | cut -d\  -f3- )
     FILE_ARR+=( "${FILE_ARR_RTOS[@]}" )
     FILE_ARR+=( "${lFIRMWARE_PATH}" )
   fi
@@ -454,7 +454,7 @@ prepare_binary_arr() {
 
   # In some firmwares we miss the exec permissions in the complete firmware. In such a case we try to find ELF files and unique it
   # readarray -t lBINARIES_TMP_ARR < <(find "${lFIRMWARE_PATH}" "${EXCL_FIND[@]}" -type f -exec file {} \; -exec grep "ELF\|PE32" | cut -d: -f1 || true)
-  readarray -t lBINARIES_TMP_ARR < <(find "${lFIRMWARE_PATH}" "${EXCL_FIND[@]}" -type f -print0|xargs -r -0 -P 16 -I % sh -c 'file % | grep "ELF\|PE32" | cut -d: -f1' || true)
+  readarray -t lBINARIES_TMP_ARR < <(find "${lFIRMWARE_PATH}" "${EXCL_FIND[@]}" -type f -print0|xargs -r -0 -P 16 -I % sh -c 'file % | grep "ELF\|PE32" | cut -d: -f1' 2>/dev/null || true)
   if [[ -v lBINARIES_TMP_ARR[@] ]]; then
     for lBINARY in "${lBINARIES_TMP_ARR[@]}"; do
       if [[ -f "${lBINARY}" ]]; then
@@ -555,7 +555,7 @@ detect_root_dir_helper() {
 
   if [[ "${SBOM_MINIMAL:-0}" -eq 0 ]]; then
     # mapfile -t lINTERPRETER_FULL_PATH_ARR < <(find "${lSEARCH_PATH}" -ignore_readdir_race -type f -exec file -b {} \; 2>/dev/null | grep "ELF" | grep "interpreter" | sed s/.*interpreter\ // | sed 's/,\ .*$//' | sort -u 2>/dev/null || true)
-    mapfile -t lINTERPRETER_FULL_PATH_ARR < <(find "${lSEARCH_PATH}" -ignore_readdir_race -type f -print0|xargs -r -0 -P 16 -I % sh -c 'file -b % | grep "ELF" | grep "interpreter" | sed "s/.*interpreter\ //" | sed "s/,\ .*$//"' | sort -u 2>/dev/null || true)
+    mapfile -t lINTERPRETER_FULL_PATH_ARR < <(find "${lSEARCH_PATH}" -ignore_readdir_race -type f -print0|xargs -r -0 -P 16 -I % sh -c 'file -b % | grep "ELF" | grep "interpreter" | sed "s/.*interpreter\ //" | sed "s/,\ .*$//"' 2>/dev/null | sort -u || true)
 
     if [[ "${#lINTERPRETER_FULL_PATH_ARR[@]}" -gt 0 ]]; then
       for lINTERPRETER_PATH in "${lINTERPRETER_FULL_PATH_ARR[@]}"; do

helpers/helpers_emba_print.sh

@@ -664,7 +664,8 @@ print_help()
   echo -e "${CYAN}""-m [MODULE_NO.]""${NC}""   Test only with set modules [e.g. -m p05 -m s10 ... or -m p to run all p modules]"
   echo -e "                                    (multiple usage possible, case insensitive)"
   echo -e "${CYAN}""-p [PROFILE]""${NC}""      EMBA starts with a pre-defined profile (stored in ./scan-profiles)"
-  echo -e "${CYAN}""-t""${NC}""                Activate multi threading (destroys regular console output)"
+  # Threading is now only available via profile parameter. In default mode EMBA is running in threading mode
+#  echo -e "${CYAN}""-t""${NC}""                Activate multi threading (destroys regular console output)"
   echo -e "${CYAN}""-P""${NC}""                Overwrite auto MAX_MODS (maximum modules in parallel) configuration"
   echo -e "${CYAN}""-T""${NC}""                Overwrite auto MAX_MOD_THREADS (maximum threads per module) configuration"
   echo -e "\\nDeveloper options"

helpers/helpers_emba_sbom_helpers.sh

@@ -243,7 +243,7 @@ build_sbom_json_component_arr() {
   lCOMPONENT_ARR+=( "description=${lAPP_DESC_NEW//\ /%SPACE%}" )
 
   if [[ ! -d "${SBOM_LOG_PATH}" ]]; then
-    mkdir "${SBOM_LOG_PATH}" || true
+    mkdir "${SBOM_LOG_PATH}" 2>/dev/null || true
   fi
 
   jo -n -- "${lCOMPONENT_ARR[@]}" > "${SBOM_LOG_PATH}/${lPACKAGING_SYSTEM}_${lAPP_NAME}_${SBOM_COMP_BOM_REF:-NA}.json"

helpers/helpers_emba_status_bar.sh

@@ -252,6 +252,7 @@ update_box_modules() {
   local MODULES_LOCAL=()
   local MODULES_EMBA=()
   local MODULE_FILE=""
+  local lMODULE_NAME=""
   local lLINES=""
 
   if [[ -f "${STATUS_TMP_PATH}" ]] ; then
@@ -266,18 +267,19 @@ update_box_modules() {
     for MODULE_FILE in "${MODULES[@]}" ; do
       if ( file "${MODULE_FILE}" | grep -q "shell script" ) && ! [[ "${MODULE_FILE}" =~ \ |\' ]]; then
         # if system emulation is not enabled, we do not count the L modules
-        if [[ "$(basename "${MODULE_FILE}")" =~ ^L[0-9]* ]] && [[ "${FULL_EMULATION}" -ne 1 ]]; then
+        lMODULE_NAME="$(basename "${MODULE_FILE}")"
+        if [[ "${lMODULE_NAME}" =~ ^L[0-9]* ]] && [[ "${FULL_EMULATION}" -ne 1 ]]; then
           continue
         fi
         # if diffing is not enabled, we do not count the diffing modules
-        if [[ "$(basename "${MODULE_FILE}")" =~ ^D[0-9]* ]] && [[ -z "${FIRMWARE_PATH1}" ]]; then
+        if [[ "${lMODULE_NAME}" =~ ^D[0-9]* ]] && [[ -z "${FIRMWARE_PATH1}" ]]; then
           continue
         fi
         # we do not count the quest modules
-        if [[ "$(basename "${MODULE_FILE}")" =~ ^Q[0-9]* ]]; then
+        if [[ "${lMODULE_NAME}" =~ ^Q[0-9]* ]]; then
           continue
         fi
-        if [[ "${MODULE_BLACKLIST[*]}" == *"$(basename -s .sh "${MODULE_FILE}")"* ]]; then
+        if [[ "${MODULE_BLACKLIST[*]}" == *"${lMODULE_NAME%\.sh}"* ]]; then
           continue
         fi
         (( COUNT_MODULES+=1 ))

modules/D10_firmware_diffing.sh

@@ -84,9 +84,9 @@ D10_firmware_diffing() {
     lNEG_LOG=1
     if [[ "${THREADED}" -eq 1 ]]; then
       analyse_fw_files "${lFW_FILE1}" &
-      local TMP_PID="$!"
-      store_kill_pids "${TMP_PID}"
-      lWAIT_PIDS_D10_ARR+=( "${TMP_PID}" )
+      local lTMP_PID="$!"
+      store_kill_pids "${lTMP_PID}"
+      lWAIT_PIDS_D10_ARR+=( "${lTMP_PID}" )
       max_pids_protection "${MAX_MOD_THREADS}" "${lWAIT_PIDS_D10_ARR[@]}"
     else
       # echo "Testing ${lFW_FILE1}"
@@ -97,9 +97,9 @@ D10_firmware_diffing() {
   for lFW_FILE2 in "${lFW_FILES2_ARR[@]}"; do
     if [[ "${THREADED}" -eq 1 ]]; then
       check_for_new_files "${lFW_FILE2}" &
-      local TMP_PID="$!"
-      store_kill_pids "${TMP_PID}"
-      lWAIT_PIDS_D10_ARR+=( "${TMP_PID}" )
+      local lTMP_PID="$!"
+      store_kill_pids "${lTMP_PID}"
+      lWAIT_PIDS_D10_ARR+=( "${lTMP_PID}" )
       max_pids_protection "${MAX_MOD_THREADS}" "${lWAIT_PIDS_D10_ARR[@]}"
     else
       check_for_new_files "${lFW_FILE2}"