diff --git a/.github/categories/02_privacy_apple_bot_ipset.txt b/.github/categories/02_privacy_applebot_ipset.txt similarity index 100% rename from .github/categories/02_privacy_apple_bot_ipset.txt rename to .github/categories/02_privacy_applebot_ipset.txt diff --git a/.github/categories/02_privacy_facebook_bot_ipset.txt b/.github/categories/02_privacy_facebook_bot_ipset.txt deleted file mode 100644 index da0b55710..000000000 --- a/.github/categories/02_privacy_facebook_bot_ipset.txt +++ /dev/null @@ -1 +0,0 @@ -Privacy \ No newline at end of file diff --git a/.github/descriptions/02_privacy_facebook_bot_ipset.txt b/.github/descriptions/02_privacy_facebook_bot_ipset.txt deleted file mode 100644 index 5dc1d656b..000000000 --- a/.github/descriptions/02_privacy_facebook_bot_ipset.txt +++ /dev/null @@ -1,3 +0,0 @@ -# This list contains ip addresses associated with Facebook bots / crawlers. -# -# These are useful if you do not wish for Facebook to crawl your data. \ No newline at end of file diff --git a/.github/descriptions/02_privacy_rssapi_ipset.txt b/.github/descriptions/02_privacy_rssapi_ipset.txt new file mode 100644 index 000000000..b79deece1 --- /dev/null +++ b/.github/descriptions/02_privacy_rssapi_ipset.txt @@ -0,0 +1,2 @@ +# Subscribe to RSS feeds and receive nearly real-time updates via webhooks. +# Consume any RSS, ATOM, and JSON feed. \ No newline at end of file diff --git a/.github/descriptions/02_privacy_telegram_ipset.txt b/.github/descriptions/02_privacy_telegram_ipset.txt new file mode 100644 index 000000000..fcde4d261 --- /dev/null +++ b/.github/descriptions/02_privacy_telegram_ipset.txt @@ -0,0 +1,2 @@ +# Telegram Messenger, commonly known as Telegram, is a cloud-based, cross-platform, social media +# and instant messaging service. \ No newline at end of file diff --git a/.github/descriptions/02_privacy_uptimerobot_ipset.txt b/.github/descriptions/02_privacy_uptimerobot_ipset.txt new file mode 100644 index 000000000..9b15ef61b --- /dev/null +++ b/.github/descriptions/02_privacy_uptimerobot_ipset.txt @@ -0,0 +1,7 @@ +# UptimeRobot is a service that monitors your website’s uptime and alerts you when it goes +# down. By integrating UptimeRobot with RunCloud, you can monitor all of your websites hosted +# on RunCloud servers, and access a number of features and benefits from both. +# +# It achieves this by sending requests to your website at regular intervals and checking the +# response status code, response time, and keyword presence. If the response is not satisfactory, +# UptimeRobot will notify you via email, SMS, webhook, or other method of your choice. \ No newline at end of file diff --git a/.github/descriptions/02_privacy_webpagetest_ipset.txt b/.github/descriptions/02_privacy_webpagetest_ipset.txt new file mode 100644 index 000000000..5f50ca175 --- /dev/null +++ b/.github/descriptions/02_privacy_webpagetest_ipset.txt @@ -0,0 +1,2 @@ +# WebPageTest is an open-source tool designed to help website owners and developers monitor and +# optimize their website’s performance. \ No newline at end of file diff --git a/.github/expires/02_privacy_facebook_bot_ipset.txt b/.github/expires/02_privacy_ahrefs_ipset.txt similarity index 100% rename from .github/expires/02_privacy_facebook_bot_ipset.txt rename to .github/expires/02_privacy_ahrefs_ipset.txt diff --git a/.github/expires/02_privacy_bunnycdn_ipset.txt b/.github/expires/02_privacy_bunnycdn_ipset.txt new file mode 100644 index 000000000..3ef8bf8ce --- /dev/null +++ b/.github/expires/02_privacy_bunnycdn_ipset.txt @@ -0,0 +1 @@ +6 hours \ No newline at end of file diff --git a/.github/expires/02_privacy_cloudflarecdn_ipset.txt b/.github/expires/02_privacy_cloudflarecdn_ipset.txt new file mode 100644 index 000000000..3ef8bf8ce --- /dev/null +++ b/.github/expires/02_privacy_cloudflarecdn_ipset.txt @@ -0,0 +1 @@ +6 hours \ No newline at end of file diff --git a/.github/expires/02_privacy_duckduckgo_ipset.txt b/.github/expires/02_privacy_duckduckgo_ipset.txt new file mode 100644 index 000000000..3ef8bf8ce --- /dev/null +++ b/.github/expires/02_privacy_duckduckgo_ipset.txt @@ -0,0 +1 @@ +6 hours \ No newline at end of file diff --git a/.github/expires/02_privacy_pingdom_ipset.txt b/.github/expires/02_privacy_pingdom_ipset.txt new file mode 100644 index 000000000..3ef8bf8ce --- /dev/null +++ b/.github/expires/02_privacy_pingdom_ipset.txt @@ -0,0 +1 @@ +6 hours \ No newline at end of file diff --git a/.github/expires/02_privacy_rssapi_ipset.txt b/.github/expires/02_privacy_rssapi_ipset.txt new file mode 100644 index 000000000..3ef8bf8ce --- /dev/null +++ b/.github/expires/02_privacy_rssapi_ipset.txt @@ -0,0 +1 @@ +6 hours \ No newline at end of file diff --git a/.github/expires/02_privacy_stripe_api_ipset.txt b/.github/expires/02_privacy_stripe_api_ipset.txt new file mode 100644 index 000000000..3ef8bf8ce --- /dev/null +++ b/.github/expires/02_privacy_stripe_api_ipset.txt @@ -0,0 +1 @@ +6 hours \ No newline at end of file diff --git a/.github/expires/02_privacy_uptimerobot_ipset.txt b/.github/expires/02_privacy_uptimerobot_ipset.txt new file mode 100644 index 000000000..3ef8bf8ce --- /dev/null +++ b/.github/expires/02_privacy_uptimerobot_ipset.txt @@ -0,0 +1 @@ +6 hours \ No newline at end of file diff --git a/.github/expires/02_privacy_webpagetest_ipset.txt b/.github/expires/02_privacy_webpagetest_ipset.txt new file mode 100644 index 000000000..3ef8bf8ce --- /dev/null +++ b/.github/expires/02_privacy_webpagetest_ipset.txt @@ -0,0 +1 @@ +6 hours \ No newline at end of file diff --git a/.github/scripts/bl-block.sh b/.github/scripts/bl-block.sh new file mode 100644 index 000000000..3836652e9 --- /dev/null +++ b/.github/scripts/bl-block.sh @@ -0,0 +1,280 @@ +#!/bin/bash + +# # +# @for https://github.com/Aetherinox/csf-firewall +# @workflow blocklist-generate.yml +# @type bash script +# @summary generate ipset by fetching locally specified file in /blocks/ repo folder +# copies local ipsets from .github/blocks/${ARG_BLOCKS_CAT}/*.ipset +# +# @terminal .github/scripts/bl-block.sh \ +# blocklists/02_privacy_general.ipset \ +# privacy +# +# @workflow # Privacy β€Ί General +# chmod +x ".github/scripts/bl-block.sh" +# run_general=".github/scripts/bl-block.sh 02_privacy_general.ipset privacy" +# eval "./$run_general" +# +# @command bl-block.sh +# bl-block.sh 02_privacy_general.ipset privacy +# +# πŸ“ .github +# πŸ“ blocks +# πŸ“ privacy +# πŸ“„ *.txt +# πŸ“ scripts +# πŸ“„ bl-block.sh +# πŸ“ workflows +# πŸ“„ blocklist-generate.yml +# +# # + +# # +# Arguments +# +# This bash script has the following arguments: +# +# ARG_SAVEFILE (str) file to save IP addresses into +# ARG_BLOCKS_CAT (str) which blocks folder to inject static IP addresses from +# # + +APP_FILE=$(basename "$0") +ARG_SAVEFILE=$1 +ARG_BLOCKS_CAT=$2 + +# # +# Validation checks +# # + +if [[ -z "${ARG_SAVEFILE}" ]]; then + echo -e " β­• No output file specified for saving by script ${APP_FILE}" + echo -e + exit 1 +fi + +if [[ -z "${ARG_BLOCKS_CAT}" ]]; then + echo -e " β­• Aborting -- no static file category specified. ex: privacy" + exit 1 +fi + +# # +# Define > General +# # + +SECONDS=0 # set seconds count for beginning of script +APP_DIR=${PWD} # returns the folder this script is being executed in +APP_REPO="Aetherinox/csf-firewall" # repository +APP_REPO_BRANCH="main" # repository branch +APP_OUT="" # each ip fetched from stdin will be stored in this var +APP_FILE_PERM="${ARG_SAVEFILE}" # perm file when building ipset list +COUNT_LINES=0 # number of lines in doc +COUNT_TOTAL_SUBNET=0 # number of IPs in all subnets combined +COUNT_TOTAL_IP=0 # number of single IPs (counts each line) +BLOCKS_COUNT_TOTAL_IP=0 # number of ips for one particular file +BLOCKS_COUNT_TOTAL_SUBNET=0 # number of subnets for one particular file +TEMPL_NOW=`date -u` # get current date in utc format +TEMPL_ID="${APP_FILE_PERM//[^[:alnum:]]/_}" # ipset id, /description/* and /category/* files must match this value +TEMPL_UUID=$(uuidgen -m -N "${TEMPL_ID}" -n @url) # uuid associated to each release +APP_AGENT="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" +TEMPL_DESC=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/descriptions/${TEMPL_ID}.txt") +TEMPL_CAT=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/categories/${TEMPL_ID}.txt") +TEMPL_EXP=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/expires/${TEMPL_ID}.txt") +TEMP_URL_SRC=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/url-source/${TEMPL_ID}.txt") +REGEX_URL='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]\.[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' +REGEX_ISNUM='^[0-9]+$' + +# # +# Default Values +# # + +if [[ "$TEMPL_DESC" == *"404: Not Found"* ]]; then + TEMPL_DESC="# No description provided" +fi + +if [[ "$TEMPL_CAT" == *"404: Not Found"* ]]; then + TEMPL_CAT="Uncategorized" +fi + +if [[ "$TEMPL_EXP" == *"404: Not Found"* ]]; then + TEMPL_EXP="6 hours" +fi + +if [[ "$TEMP_URL_SRC" == *"404: Not Found"* ]]; then + TEMP_URL_SRC="None" +fi + +# # +# Output > Header +# # + +echo -e +echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" +echo -e " Blocklist - ${APP_FILE_PERM} (${ARG_BLOCKS_CAT})" +echo -e " ID: ${TEMPL_ID}" +echo -e " UUID: ${TEMPL_UUID}" +echo -e " CATEGORY: ${TEMPL_CAT}" +echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" + +# # +# output +# # + +echo -e +echo -e " ⭐ Starting" + +# # +# Create or Clean file +# # + +if [ -f $APP_FILE_PERM ]; then + echo -e " πŸ“„ Clean ${APP_FILE_PERM}" + echo -e + > ${APP_FILE_PERM} # clean file +else + echo -e " πŸ“ Create ${APP_FILE_PERM}" + echo -e + mkdir -p $(dirname "${APP_FILE_PERM}") + touch ${APP_FILE_PERM} +fi + +# # +# Add Static Files +# # + +if [ -d .github/blocks/ ]; then + for APP_FILE_TEMP in .github/blocks/${ARG_BLOCKS_CAT}/*.ipset; do + echo -e " πŸ“’ Adding static file ${APP_FILE_TEMP}" + + # # + # calculate how many IPs are in a subnet + # if you want to calculate the USABLE IP addresses, subtract -2 from any subnet not ending with 31 or 32. + # + # for our purpose, we want to block them all in the event that the network has reconfigured their network / broadcast IPs, + # so we will count every IP in the block. + # # + + BLOCKS_COUNT_TOTAL_IP=0 + BLOCKS_COUNT_TOTAL_SUBNET=0 + + for line in $(cat ${APP_FILE_TEMP}); do + + # is ipv6 + if [ "$line" != "${line#*:[0-9a-fA-F]}" ]; then + if [[ $line =~ /[0-9]{1,3}$ ]]; then + COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet + BLOCKS_COUNT_TOTAL_SUBNET=`expr $BLOCKS_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet + else + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` # GLOBAL count ip + BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + 1` # LOCAL count ip + fi + + # is subnet + elif [[ $line =~ /[0-9]{1,2}$ ]]; then + ips=$(( 1 << (32 - ${line#*/}) )) + + if [[ $ips =~ $REGEX_ISNUM ]]; then + CIDR=$(echo $line | sed 's:.*/::') + + # uncomment if you want to count ONLY usable IP addresses + # subtract - 2 from any cidr not ending with 31 or 32 + # if [[ $CIDR != "31" ]] && [[ $CIDR != "32" ]]; then + # BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP - 2` + # COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP - 2` + # fi + + BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + $ips` # LOCAL count IPs in subnet + BLOCKS_COUNT_TOTAL_SUBNET=`expr $BLOCKS_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet + + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + $ips` # GLOBAL count IPs in subnet + COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet + fi + + # is normal IP + elif [[ $line =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + 1` + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` + fi + done + + # # + # Count lines and subnets + # # + + COUNT_LINES=$(wc -l < ${APP_FILE_TEMP}) # GLOBAL count ip lines + COUNT_LINES=$(printf "%'d" "$COUNT_LINES") # GLOBAL add commas to thousands + COUNT_TOTAL_IP=$(printf "%'d" "$COUNT_TOTAL_IP") # GLOBAL add commas to thousands + COUNT_TOTAL_SUBNET=$(printf "%'d" "$COUNT_TOTAL_SUBNET") # GLOBAL add commas to thousands + + BLOCKS_COUNT_TOTAL_IP=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_IP") # LOCAL add commas to thousands + BLOCKS_COUNT_TOTAL_SUBNET=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_SUBNET") # LOCAL add commas to thousands + + echo -e " πŸš› Move ${APP_FILE_TEMP} to ${APP_FILE_PERM}" + cat ${APP_FILE_TEMP} >> ${APP_FILE_PERM} # copy .tmp contents to real file + + echo -e " βž• Added ${BLOCKS_COUNT_TOTAL_IP} IPs and ${BLOCKS_COUNT_TOTAL_SUBNET} Subnets to ${APP_FILE_TEMP}" + echo -e + done +fi + +# # +# Sort +# - sort lines numerically and create .sort file +# - move re-sorted text from .sort over to real file +# - remove .sort temp file +# # + +APP_OUT=$(cat ${APP_FILE_PERM} | grep -v "^#" | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${APP_FILE_PERM}.sort) +sed -i 's/[[:blank:]]*$//' ${APP_FILE_PERM}.sort +> ${APP_FILE_PERM} +cat ${APP_FILE_PERM}.sort >> ${APP_FILE_PERM} +rm ${APP_FILE_PERM}.sort + +# # +# ed +# 0a top of file +# # + +ed -s ${APP_FILE_PERM} < +# # πŸ“ .github # πŸ“ scripts -# πŸ“„ bl-template.sh +# πŸ“„ bl-format.sh # πŸ“ workflows # πŸ“„ blocklist-generate.yml # -# activated from github workflow: -# - .github/workflows/blocklist-generate.yml -# -# @uage whois -h whois.radb.net -- '-i origin AS32934' | grep ^route | awk '{gsub("(route:|route6:)","");print}' | awk '{gsub(/ /,""); print}' | .github/scripts/bl-template.sh ip.txt # # # # @@ -28,6 +34,7 @@ # ARG_SAVEFILE (str) file to save IP addresses into # # +APP_FILE=$(basename "$0") ARG_SAVEFILE=$1 # # @@ -35,7 +42,7 @@ ARG_SAVEFILE=$1 # # if [[ -z "${ARG_SAVEFILE}" ]]; then - echo -e " β­• No output file specified for bl-template" + echo -e " β­• No output file specified for saving by script ${APP_FILE}" echo -e exit 1 fi @@ -44,44 +51,47 @@ fi # Define > General # # -REPO="Aetherinox/dev-kw" # repository -SECONDS=0 # set seconds count for beginning of script -NOW=`date -u` # get current date in utc format -OUTPUT="" # each ip fetched from stdin will be stored in this var -FILE_TEMP="${ARG_SAVEFILE}.tmp" # temp file when building ipset list -FOLDER_SAVE="blocklists" # folder where to save .ipset file -COUNT_LINES=0 # number of lines in doc -COUNT_TOTAL_SUBNET=0 # number of IPs in all subnets combined -COUNT_TOTAL_IP=0 # number of single IPs (counts each line) -BLOCKS_COUNT_TOTAL_IP=0 # number of ips for one particular file -BLOCKS_COUNT_TOTAL_SUBNET=0 # number of subnets for one particular file -ID="${ARG_SAVEFILE//[^[:alnum:]]/_}" # ipset id, /description/* and /category/* files must match this value -UUID=$(uuidgen -m -N "${ID}" -n @url) # uuid associated to each release -CURL_AGENT="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" -DESCRIPTION=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/descriptions/${ID}.txt") -CATEGORY=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/categories/${ID}.txt") -EXPIRES=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/expires/${ID}.txt") -URL_SOURCE=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/url-source/${ID}.txt") -regexURL='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]\.[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' +SECONDS=0 # set seconds count for beginning of script +APP_DIR=${PWD} # returns the folder this script is being executed in +APP_REPO="Aetherinox/csf-firewall" # repository +APP_REPO_BRANCH="main" # repository branch +APP_OUT="" # each ip fetched from stdin will be stored in this var +APP_FILE_TEMP="${ARG_SAVEFILE}.tmp" # temp file when building ipset list +APP_FILE_PERM="${ARG_SAVEFILE}" # perm file when building ipset list +COUNT_LINES=0 # number of lines in doc +COUNT_TOTAL_SUBNET=0 # number of IPs in all subnets combined +COUNT_TOTAL_IP=0 # number of single IPs (counts each line) +BLOCKS_COUNT_TOTAL_IP=0 # number of ips for one particular file +BLOCKS_COUNT_TOTAL_SUBNET=0 # number of subnets for one particular file +TEMPL_NOW=`date -u` # get current date in utc format +TEMPL_ID="${APP_FILE_PERM//[^[:alnum:]]/_}" # ipset id, /description/* and /category/* files must match this value +TEMPL_UUID=$(uuidgen -m -N "${TEMPL_ID}" -n @url) # uuid associated to each release +APP_AGENT="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" +TEMPL_DESC=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/descriptions/${TEMPL_ID}.txt") +TEMPL_CAT=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/categories/${TEMPL_ID}.txt") +TEMPL_EXP=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/expires/${TEMPL_ID}.txt") +TEMP_URL_SRC=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/url-source/${TEMPL_ID}.txt") +REGEX_URL='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]\.[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' +REGEX_ISNUM='^[0-9]+$' # # # Default Values # # -if [[ "$DESCRIPTION" == *"404: Not Found"* ]]; then - DESCRIPTION="# No description provided" +if [[ "$TEMPL_DESC" == *"404: Not Found"* ]]; then + TEMPL_DESC="# No description provided" fi -if [[ "$CATEGORY" == *"404: Not Found"* ]]; then - CATEGORY="Uncategorized" +if [[ "$TEMPL_CAT" == *"404: Not Found"* ]]; then + TEMPL_CAT="Uncategorized" fi -if [[ "$EXPIRES" == *"404: Not Found"* ]]; then - EXPIRES="6 hours" +if [[ "$TEMPL_EXP" == *"404: Not Found"* ]]; then + TEMPL_EXP="6 hours" fi -if [[ "$URL_SOURCE" == *"404: Not Found"* ]]; then - URL_SOURCE="None" +if [[ "$TEMP_URL_SRC" == *"404: Not Found"* ]]; then + TEMP_URL_SRC="None" fi # # @@ -138,9 +148,10 @@ process_v6() { echo -e echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" -echo -e " Blocklist - ${ARG_SAVEFILE}" -echo -e " ID: ${ID}" -echo -e " CATEGORY: ${CATEGORY}" +echo -e " Blocklist - ${APP_FILE_PERM}" +echo -e " ID: ${TEMPL_ID}" +echo -e " UUID: ${TEMPL_UUID}" +echo -e " CATEGORY: ${TEMPL_CAT}" echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" # # @@ -154,21 +165,22 @@ echo -e " ⭐ Starting" # Create or Clean file # # -if [ -f $ARG_SAVEFILE ]; then - echo -e " πŸ“„ Cleaning ${ARG_SAVEFILE}" +if [ -f $APP_FILE_PERM ]; then + echo -e " πŸ“„ Clean ${APP_FILE_PERM}" echo -e - > ${ARG_SAVEFILE} # clean file + > ${APP_FILE_PERM} # clean file else - echo -e " πŸ“„ Creating ${ARG_SAVEFILE}" + echo -e " πŸ“ Create ${APP_FILE_PERM}" echo -e - touch ${ARG_SAVEFILE} + mkdir -p $(dirname "${APP_FILE_PERM}") + touch ${APP_FILE_PERM} fi # # # Get IP list # # -echo -e " 🌎 Downloading IP blacklist to ${ARG_SAVEFILE}" +echo -e " 🌎 Downloading IP blacklist to ${APP_FILE_PERM}" # # # Read stdin @@ -180,7 +192,7 @@ echo -e " 🌎 Downloading IP blacklist to ${ARG_SAVEFILE}" while IFS= read -r ip || [[ -n "$ip" ]]; do line=${ip%$'\r'} if process_v4 "$line" || process_v6 "$line"; then - OUTPUT+="${line}"$'\n' + APP_OUT+="${line}"$'\n' fi done @@ -188,11 +200,11 @@ done # Get IP list # # -list_ips=$(echo "${OUTPUT}" | grep -v "^#" | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${FILE_TEMP}) -sed -i '/[#;]/{s/#.*//;s/;.*//;/^$/d}' ${FILE_TEMP} # remove # and ; comments -sed -i 's/\-.*//' ${FILE_TEMP} # remove hyphens for ip ranges -sed -i 's/[[:blank:]]*$//' ${FILE_TEMP} # remove space / tab from EOL -sed -i '/^\s*$/d' ${FILE_TEMP} # remove empty lines +list_ips=$(echo "${APP_OUT}" | grep -v "^#" | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${APP_FILE_TEMP}) +sed -i '/[#;]/{s/#.*//;s/;.*//;/^$/d}' ${APP_FILE_TEMP} # remove # and ; comments +sed -i 's/\-.*//' ${APP_FILE_TEMP} # remove hyphens for ip ranges +sed -i 's/[[:blank:]]*$//' ${APP_FILE_TEMP} # remove space / tab from EOL +sed -i '/^\s*$/d' ${APP_FILE_TEMP} # remove empty lines # # # calculate how many IPs are in a subnet @@ -202,7 +214,7 @@ sed -i '/^\s*$/d' ${FILE_TEMP} # remove # so we will count every IP in the block. # # -for line in $(cat ${FILE_TEMP}); do +for line in $(cat ${APP_FILE_TEMP}); do # is ipv6 if [ "$line" != "${line#*:[0-9a-fA-F]}" ]; then @@ -218,21 +230,21 @@ for line in $(cat ${FILE_TEMP}); do elif [[ $line =~ /[0-9]{1,2}$ ]]; then ips=$(( 1 << (32 - ${line#*/}) )) - regexIsNum='^[0-9]+$' - if [[ $ips =~ $regexIsNum ]]; then + if [[ $ips =~ $REGEX_ISNUM ]]; then CIDR=$(echo $line | sed 's:.*/::') + # uncomment if you want to count ONLY usable IP addresses # subtract - 2 from any cidr not ending with 31 or 32 # if [[ $CIDR != "31" ]] && [[ $CIDR != "32" ]]; then # BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP - 2` # COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP - 2` # fi - BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + $ips` # LOCAL count IPs in subnet - BLOCKS_COUNT_TOTAL_SUBNET=`expr $BLOCKS_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet + BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + $ips` # LOCAL count IPs in subnet + BLOCKS_COUNT_TOTAL_SUBNET=`expr $BLOCKS_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet - COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + $ips` # GLOBAL count IPs in subnet - COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + $ips` # GLOBAL count IPs in subnet + COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet fi # is normal IP @@ -246,43 +258,42 @@ done # Count lines and subnets # # -COUNT_LINES=$(wc -l < ${FILE_TEMP}) # GLOBAL count ip lines -COUNT_LINES=$(printf "%'d" "$COUNT_LINES") # GLOBAL add commas to thousands -COUNT_TOTAL_IP=$(printf "%'d" "$COUNT_TOTAL_IP") # GLOBAL add commas to thousands -COUNT_TOTAL_SUBNET=$(printf "%'d" "$COUNT_TOTAL_SUBNET") # GLOBAL add commas to thousands +COUNT_LINES=$(wc -l < ${APP_FILE_TEMP}) # GLOBAL count ip lines +COUNT_LINES=$(printf "%'d" "$COUNT_LINES") # GLOBAL add commas to thousands +COUNT_TOTAL_IP=$(printf "%'d" "$COUNT_TOTAL_IP") # GLOBAL add commas to thousands +COUNT_TOTAL_SUBNET=$(printf "%'d" "$COUNT_TOTAL_SUBNET") # GLOBAL add commas to thousands -BLOCKS_COUNT_TOTAL_IP=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_IP") # LOCAL add commas to thousands -BLOCKS_COUNT_TOTAL_SUBNET=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_SUBNET") # LOCAL add commas to thousands +BLOCKS_COUNT_TOTAL_IP=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_IP") # LOCAL add commas to thousands +BLOCKS_COUNT_TOTAL_SUBNET=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_SUBNET") # LOCAL add commas to thousands -echo -e " πŸš› Move ${FILE_TEMP} to ${ARG_SAVEFILE}" -cat ${FILE_TEMP} >> ${ARG_SAVEFILE} # copy .tmp contents to real file -rm ${FILE_TEMP} # delete temp file +echo -e " πŸš› Move ${APP_FILE_TEMP} to ${APP_FILE_PERM}" +cat ${APP_FILE_TEMP} >> ${APP_FILE_PERM} # copy .tmp contents to real file +rm ${APP_FILE_TEMP} # delete temp file -echo -e " βž• Added ${BLOCKS_COUNT_TOTAL_IP} IPs and ${BLOCKS_COUNT_TOTAL_SUBNET} Subnets to ${FILE_TEMP}" -echo -e +echo -e " βž• Added ${BLOCKS_COUNT_TOTAL_IP} IPs and ${BLOCKS_COUNT_TOTAL_SUBNET} Subnets to ${APP_FILE_TEMP}" # # # ed # 0a top of file # # -ed -s ${ARG_SAVEFILE} < [ , {...} ] -# bl-htm.sh 02_privacy_yandex.ipset -# # - - -# # -# Arguments -# -# This bash script has the following arguments: -# -# ARG_SAVEFILE (str) file to save IP addresses into -# { ... } (varg) list of URLs to API end-points -# # - -ARG_SAVEFILE=$1 - -# # -# Validation checks -# # - -if [[ -z "${ARG_SAVEFILE}" ]]; then - echo -e " β­• No output file specified for downloader script" - echo -e - exit 1 -fi - -if test "$#" -lt 2; then - echo -e " β­• Aborting -- did not provide URL arguments" - exit 1 -fi - -# # -# Define > General -# # - -CURL_AGENT="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" -FOLDER_SAVETO="blocklists" -SECONDS=0 -NOW=`date -u` -COUNT_LINES=0 # number of lines in doc -COUNT_TOTAL_SUBNET=0 # number of IPs in all subnets combined -COUNT_TOTAL_IP=0 # number of single IPs (counts each line) -ID="${ARG_SAVEFILE//[^[:alnum:]]/_}" -DESCRIPTION=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/Aetherinox/csf-firewall/main/.github/descriptions/${ID}.txt") -CATEGORY=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/Aetherinox/csf-firewall/main/.github/categories/${ID}.txt") -EXPIRES=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/Aetherinox/csf-firewall/main/.github/expires/${ID}.txt") -regexURL='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]\.[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' - -# # -# Default Values -# # - -DESCRIPTION=$([ "${DESCRIPTION}" == *"404: Not Found"* ] && echo "# No description provided" || echo "${DESCRIPTION}") -CATEGORY=$([ "${CATEGORY}" == *"404: Not Found"* ] && echo "Uncategorized" || echo "${CATEGORY}") -EXPIRES=$([ "${EXPIRES}" == *"404: Not Found"* ] && echo "6 hours" || echo "${EXPIRES}") - -# # -# Output > Header -# # - -echo -e -echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" -echo -e " Blocklist - ${ARG_SAVEFILE}" -echo -e " ID: ${ID}" -echo -e " CATEGORY: ${CATEGORY}" -echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" - -# # -# output -# # - -echo -e -echo -e " ⭐ Starting" - -# # -# Create or Clean file -# # - -if [ -f $ARG_SAVEFILE ]; then - echo -e " πŸ“„ Cleaning ${ARG_SAVEFILE}" - echo -e - > ${ARG_SAVEFILE} # clean file -else - echo -e " πŸ“„ Creating ${ARG_SAVEFILE}" - echo -e - touch ${ARG_SAVEFILE} -fi - -# # -# Func > Download List -# # - -download_list() -{ - - local fnUrl=$1 - local fnFile=$2 - local tempFile="${2}.tmp" - local DL_COUNT_TOTAL_IP=0 - local DL_COUNT_TOTAL_SUBNET=0 - - echo -e " 🌎 Downloading IP blacklist to ${tempFile}" - - jsonOutput=$(curl -Ss -A "${CURL_AGENT}" ${fnUrl} | html2text | grep -v "^#" | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}' | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${tempFile}) - sed -i 's/\-.*//' ${tempFile} # remove hyphens for ip ranges - sed -i '/[#;]/{s/#.*//;s/;.*//;/^$/d}' ${tempFile} # remove # and ; comments - sed -i 's/[[:blank:]]*$//' ${tempFile} # remove space / tab from EOL - sed -i '/^\s*$/d' ${tempFile} # remove empty lines - - # # - # calculate how many IPs are in a subnet - # if you want to calculate the USABLE IP addresses, subtract -2 from any subnet not ending with 31 or 32. - # - # for our purpose, we want to block them all in the event that the network has reconfigured their network / broadcast IPs, - # so we will count every IP in the block. - # # - - for line in $(cat ${tempFile}); do - # is ipv6 - if [ "$line" != "${line#*:[0-9a-fA-F]}" ]; then - COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` # GLOBAL count subnet - DL_COUNT_TOTAL_IP=`expr $DL_COUNT_TOTAL_IP + 1` # LOCAL count subnet - - # is subnet - elif [[ $line =~ /[0-9]{1,2}$ ]]; then - ips=$(( 1 << (32 - ${line#*/}) )) - - regexIsNum='^[0-9]+$' - if [[ $ips =~ $regexIsNum ]]; then - CIDR=$(echo $line | sed 's:.*/::') - - # subtract - 2 from any cidr not ending with 31 or 32 - # if [[ $CIDR != "31" ]] && [[ $CIDR != "32" ]]; then - # COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP - 2` - # DL_COUNT_TOTAL_IP=`expr $DL_COUNT_TOTAL_IP - 2` - # fi - - COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + $ips` # GLOBAL count IPs in subnet - COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet - - DL_COUNT_TOTAL_IP=`expr $DL_COUNT_TOTAL_IP + $ips` # LOCAL count IPs in subnet - DL_COUNT_TOTAL_SUBNET=`expr $DL_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet - fi - - # is normal IP - elif [[ $line =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` - DL_COUNT_TOTAL_IP=`expr $DL_COUNT_TOTAL_IP + 1` - fi - done - - # # - # Count lines and subnets - # # - - DL_COUNT_TOTAL_IP=$(printf "%'d" "$DL_COUNT_TOTAL_IP") # LOCAL add commas to thousands - DL_COUNT_TOTAL_SUBNET=$(printf "%'d" "$DL_COUNT_TOTAL_SUBNET") # LOCAL add commas to thousands - - # # - # Move temp file to final - # # - - echo -e " πŸš› Move ${tempFile} to ${fnFile}" - cat ${tempFile} >> ${fnFile} # copy .tmp contents to real file - rm ${tempFile} # delete temp file - - echo -e " βž• Added ${DL_COUNT_TOTAL_IP} IPs and ${DL_COUNT_TOTAL_SUBNET} subnets to ${fnFile}" -} - -# # -# Download lists -# # - -for arg in "${@:2}"; do - if [[ $arg =~ $regexURL ]]; then - download_list ${arg} ${ARG_SAVEFILE} - echo -e - fi -done - -# # -# Sort -# - sort lines numerically and create .sort file -# - move re-sorted text from .sort over to real file -# - remove .sort temp file -# # - -sorting=$(cat ${ARG_SAVEFILE} | grep -v "^#" | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${ARG_SAVEFILE}.sort) -> ${ARG_SAVEFILE} -cat ${ARG_SAVEFILE}.sort >> ${ARG_SAVEFILE} -rm ${ARG_SAVEFILE}.sort - -# # -# Format Counts -# # - -COUNT_LINES=$(wc -l < ${ARG_SAVEFILE}) # count ip lines -COUNT_LINES=$(printf "%'d" "$COUNT_LINES") # GLOBAL add commas to thousands - -# # -# Format count totals since we no longer need to add -# # - -COUNT_TOTAL_IP=$(printf "%'d" "$COUNT_TOTAL_IP") # GLOBAL add commas to thousands -COUNT_TOTAL_SUBNET=$(printf "%'d" "$COUNT_TOTAL_SUBNET") # GLOBAL add commas to thousands - -# # -# ed -# 0a top of file -# # - -ed -s ${ARG_SAVEFILE} < +# +# +# {...} +# +# πŸ“ .github +# πŸ“ scripts +# πŸ“„ bl-html.sh +# πŸ“ workflows +# πŸ“„ blocklist-generate.yml +# +# # + +# # +# Arguments +# +# This bash script has the following arguments: +# +# ARG_SAVEFILE (str) file to save IP addresses into +# { ... } (varg) list of URLs to API end-points +# # + +APP_FILE=$(basename "$0") +ARG_SAVEFILE=$1 + +# # +# Validation checks +# # + +if [[ -z "${ARG_SAVEFILE}" ]]; then + echo -e " β­• No output file specified for saving by script ${APP_FILE}" + echo -e " Usage: " + echo -e + exit 1 +fi + +if test "$#" -lt 2; then + echo -e " β­• Aborting -- did not provide URL arguments" + exit 1 +fi + +# # +# Define > General +# # + +SECONDS=0 # set seconds count for beginning of script +APP_DIR=${PWD} # returns the folder this script is being executed in +APP_REPO="Aetherinox/csf-firewall" # repository +APP_REPO_BRANCH="main" # repository branch +APP_OUT="" # each ip fetched from stdin will be stored in this var +APP_FILE_PERM="${ARG_SAVEFILE}" # perm file when building ipset list +COUNT_LINES=0 # number of lines in doc +COUNT_TOTAL_SUBNET=0 # number of IPs in all subnets combined +COUNT_TOTAL_IP=0 # number of single IPs (counts each line) +TEMPL_NOW=`date -u` # get current date in utc format +TEMPL_ID="${APP_FILE_PERM//[^[:alnum:]]/_}" # ipset id, /description/* and /category/* files must match this value +TEMPL_UUID=$(uuidgen -m -N "${TEMPL_ID}" -n @url) # uuid associated to each release +APP_AGENT="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" +TEMPL_DESC=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/descriptions/${TEMPL_ID}.txt") +TEMPL_CAT=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/categories/${TEMPL_ID}.txt") +TEMPL_EXP=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/expires/${TEMPL_ID}.txt") +TEMP_URL_SRC=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/url-source/${TEMPL_ID}.txt") +REGEX_URL='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]\.[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' +REGEX_ISNUM='^[0-9]+$' + +# # +# Default Values +# # + +if [[ "$TEMPL_DESC" == *"404: Not Found"* ]]; then + TEMPL_DESC="# No description provided" +fi + +if [[ "$TEMPL_CAT" == *"404: Not Found"* ]]; then + TEMPL_CAT="Uncategorized" +fi + +if [[ "$TEMPL_EXP" == *"404: Not Found"* ]]; then + TEMPL_EXP="6 hours" +fi + +if [[ "$TEMP_URL_SRC" == *"404: Not Found"* ]]; then + TEMP_URL_SRC="None" +fi + +# # +# Output > Header +# # + +echo -e +echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" +echo -e " Blocklist - ${APP_FILE_PERM}" +echo -e " ID: ${TEMPL_ID}" +echo -e " UUID: ${TEMPL_UUID}" +echo -e " CATEGORY: ${TEMPL_CAT}" +echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" + +# # +# output +# # + +echo -e +echo -e " ⭐ Starting" + +# # +# Create or Clean file +# # + +if [ -f $APP_FILE_PERM ]; then + echo -e " πŸ“„ Clean ${APP_FILE_PERM}" + echo -e + > ${APP_FILE_PERM} # clean file +else + echo -e " πŸ“ Create ${APP_FILE_PERM}" + echo -e + mkdir -p $(dirname "${APP_FILE_PERM}") + touch ${APP_FILE_PERM} +fi + +# # +# Func > Download List +# # + +download_list() +{ + + local fnUrl=$1 + local fnFile=$2 + local tempFile="${2}.tmp" + local DL_COUNT_TOTAL_IP=0 + local DL_COUNT_TOTAL_SUBNET=0 + + echo -e " 🌎 Downloading IP blacklist to ${tempFile}" + + APP_OUT=$(curl -sSL -A "${APP_AGENT}" ${fnUrl} | html2text | grep -v "^#" | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}' | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${tempFile}) + sed -i 's/\-.*//' ${tempFile} # remove hyphens for ip ranges + sed -i '/[#;]/{s/#.*//;s/;.*//;/^$/d}' ${tempFile} # remove # and ; comments + sed -i 's/[[:blank:]]*$//' ${tempFile} # remove space / tab from EOL + sed -i '/^\s*$/d' ${tempFile} # remove empty lines + + # # + # calculate how many IPs are in a subnet + # if you want to calculate the USABLE IP addresses, subtract -2 from any subnet not ending with 31 or 32. + # + # for our purpose, we want to block them all in the event that the network has reconfigured their network / broadcast IPs, + # so we will count every IP in the block. + # # + + for line in $(cat ${tempFile}); do + # is ipv6 + if [ "$line" != "${line#*:[0-9a-fA-F]}" ]; then + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` # GLOBAL count subnet + DL_COUNT_TOTAL_IP=`expr $DL_COUNT_TOTAL_IP + 1` # LOCAL count subnet + + # is subnet + elif [[ $line =~ /[0-9]{1,2}$ ]]; then + ips=$(( 1 << (32 - ${line#*/}) )) + + if [[ $ips =~ $REGEX_ISNUM ]]; then + CIDR=$(echo $line | sed 's:.*/::') + + # uncomment if you want to count ONLY usable IP addresses + # subtract - 2 from any cidr not ending with 31 or 32 + # if [[ $CIDR != "31" ]] && [[ $CIDR != "32" ]]; then + # COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP - 2` + # DL_COUNT_TOTAL_IP=`expr $DL_COUNT_TOTAL_IP - 2` + # fi + + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + $ips` # GLOBAL count IPs in subnet + COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet + + DL_COUNT_TOTAL_IP=`expr $DL_COUNT_TOTAL_IP + $ips` # LOCAL count IPs in subnet + DL_COUNT_TOTAL_SUBNET=`expr $DL_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet + fi + + # is normal IP + elif [[ $line =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` + DL_COUNT_TOTAL_IP=`expr $DL_COUNT_TOTAL_IP + 1` + fi + done + + # # + # Count lines and subnets + # # + + DL_COUNT_TOTAL_IP=$(printf "%'d" "$DL_COUNT_TOTAL_IP") # LOCAL add commas to thousands + DL_COUNT_TOTAL_SUBNET=$(printf "%'d" "$DL_COUNT_TOTAL_SUBNET") # LOCAL add commas to thousands + + # # + # Move temp file to final + # # + + echo -e " πŸš› Move ${tempFile} to ${fnFile}" + cat ${tempFile} >> ${fnFile} # copy .tmp contents to real file + rm ${tempFile} # delete temp file + + echo -e " βž• Added ${DL_COUNT_TOTAL_IP} IPs and ${DL_COUNT_TOTAL_SUBNET} subnets to ${fnFile}" +} + +# # +# Download lists +# # + +for arg in "${@:2}"; do + if [[ $arg =~ $REGEX_URL ]]; then + download_list ${arg} ${APP_FILE_PERM} + echo -e + fi +done + +# # +# Sort +# - sort lines numerically and create .sort file +# - move re-sorted text from .sort over to real file +# - remove .sort temp file +# # + +sorting=$(cat ${APP_FILE_PERM} | grep -v "^#" | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${APP_FILE_PERM}.sort) +> ${APP_FILE_PERM} +cat ${APP_FILE_PERM}.sort >> ${APP_FILE_PERM} +rm ${APP_FILE_PERM}.sort + +# # +# Format Counts +# # + +COUNT_LINES=$(wc -l < ${APP_FILE_PERM}) # count ip lines +COUNT_LINES=$(printf "%'d" "$COUNT_LINES") # GLOBAL add commas to thousands + +# # +# Format count totals since we no longer need to add +# # + +COUNT_TOTAL_IP=$(printf "%'d" "$COUNT_TOTAL_IP") # GLOBAL add commas to thousands +COUNT_TOTAL_SUBNET=$(printf "%'d" "$COUNT_TOTAL_SUBNET") # GLOBAL add commas to thousands + +# # +# ed +# 0a top of file +# # + +ed -s ${APP_FILE_PERM} < +# +# +# {...} +# +# bl-htmlip.sh blocklists/01_highrisk.ipset https://maxmind.com/en/high-risk-ip-sample-list '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}' +# +# πŸ“ .github +# πŸ“ scripts +# πŸ“„ bl-htmlip.sh +# πŸ“ workflows +# πŸ“„ blocklist-generate.yml +# +# # + +# # +# Arguments +# +# This bash script has the following arguments: +# +# ARG_SAVEFILE (str) file to save IP addresses into +# ARG_URL (str) url to grab text from +# # + +APP_FILE=$(basename "$0") +ARG_SAVEFILE=$1 +ARG_URL=$2 + +# # +# Validation checks +# # + +if [[ -z "${ARG_SAVEFILE}" ]]; then + echo -e " β­• No output file specified for saving by script ${APP_FILE}" + echo -e + exit 1 +fi + +if [[ -z "${ARG_URL}" ]]; then + echo -e " β­• Aborting -- no url specified" + exit 1 +fi + +# # +# Define > General +# # + +SECONDS=0 # set seconds count for beginning of script +APP_DIR=${PWD} # returns the folder this script is being executed in +APP_REPO="Aetherinox/csf-firewall" # repository +APP_REPO_BRANCH="main" # repository branch +APP_OUT="" # results of curl command +APP_FILE_TEMP="${ARG_SAVEFILE}.tmp" # temp file when building ipset list +APP_FILE_PERM="${ARG_SAVEFILE}" # perm file when building ipset list +COUNT_LINES=0 # number of lines in doc +COUNT_TOTAL_SUBNET=0 # number of IPs in all subnets combined +COUNT_TOTAL_IP=0 # number of single IPs (counts each line) +BLOCKS_COUNT_TOTAL_IP=0 # number of ips for one particular file +BLOCKS_COUNT_TOTAL_SUBNET=0 # number of subnets for one particular file +TEMPL_NOW=`date -u` # get current date in utc format +TEMPL_ID="${APP_FILE_PERM//[^[:alnum:]]/_}" # ipset id, /description/* and /category/* files must match this value +TEMPL_UUID=$(uuidgen -m -N "${TEMPL_ID}" -n @url) # uuid associated to each release +APP_AGENT="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" +TEMPL_DESC=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/descriptions/${TEMPL_ID}.txt") +TEMPL_CAT=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/categories/${TEMPL_ID}.txt") +TEMPL_EXP=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/expires/${TEMPL_ID}.txt") +TEMP_URL_SRC=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/url-source/${TEMPL_ID}.txt") +REGEX_URL='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]\.[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' +REGEX_ISNUM='^[0-9]+$' + +# # +# Default Values +# # + +if [[ "$TEMPL_DESC" == *"404: Not Found"* ]]; then + TEMPL_DESC="# No description provided" +fi + +if [[ "$TEMPL_CAT" == *"404: Not Found"* ]]; then + TEMPL_CAT="Uncategorized" +fi + +if [[ "$TEMPL_EXP" == *"404: Not Found"* ]]; then + TEMPL_EXP="6 hours" +fi + +if [[ "$TEMP_URL_SRC" == *"404: Not Found"* ]]; then + TEMP_URL_SRC="None" +fi + +# # +# Output > Header +# # + +echo -e +echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" +echo -e " Blocklist - ${APP_FILE_PERM}" +echo -e " ID: ${TEMPL_ID}" +echo -e " UUID: ${TEMPL_UUID}" +echo -e " CATEGORY: ${TEMPL_CAT}" +echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" + +# # +# output +# # + +echo -e +echo -e " ⭐ Starting" + +# # +# Create or Clean file +# # + +if [ -f $APP_FILE_PERM ]; then + echo -e " πŸ“„ Clean ${APP_FILE_PERM}" + echo -e + > ${APP_FILE_PERM} # clean file +else + echo -e " πŸ“ Create ${APP_FILE_PERM}" + echo -e + mkdir -p $(dirname "${APP_FILE_PERM}") + touch ${APP_FILE_PERM} +fi + +# # +# Get IP list +# # + +echo -e " 🌎 Downloading IP blacklist to ${APP_FILE_PERM}" + +# # +# Get IP list +# # + +APP_OUT=$(curl -sSL -A "${APP_AGENT}" ${ARG_URL} | html2text | grep -v "^#" | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}' | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${APP_FILE_TEMP}) +sed -i '/[#;]/{s/#.*//;s/;.*//;/^$/d}' ${APP_FILE_TEMP} # remove # and ; comments +sed -i 's/\-.*//' ${APP_FILE_TEMP} # remove hyphens for ip ranges +sed -i 's/[[:blank:]]*$//' ${APP_FILE_TEMP} # remove space / tab from EOL +sed -i '/^\s*$/d' ${APP_FILE_TEMP} # remove empty lines + +# # +# calculate how many IPs are in a subnet +# if you want to calculate the USABLE IP addresses, subtract -2 from any subnet not ending with 31 or 32. +# +# for our purpose, we want to block them all in the event that the network has reconfigured their network / broadcast IPs, +# so we will count every IP in the block. +# # + +for line in $(cat ${APP_FILE_TEMP}); do + + # is ipv6 + if [ "$line" != "${line#*:[0-9a-fA-F]}" ]; then + if [[ $line =~ /[0-9]{1,3}$ ]]; then + COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet + BLOCKS_COUNT_TOTAL_SUBNET=`expr $BLOCKS_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet + else + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` # GLOBAL count ip + BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + 1` # LOCAL count ip + fi; + + # is subnet + elif [[ $line =~ /[0-9]{1,2}$ ]]; then + ips=$(( 1 << (32 - ${line#*/}) )) + + if [[ $ips =~ $REGEX_ISNUM ]]; then + CIDR=$(echo $line | sed 's:.*/::') + + # uncomment if you want to count ONLY usable IP addresses + # subtract - 2 from any cidr not ending with 31 or 32 + # if [[ $CIDR != "31" ]] && [[ $CIDR != "32" ]]; then + # BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP - 2` + # COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP - 2` + # fi + + BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + $ips` # LOCAL count IPs in subnet + BLOCKS_COUNT_TOTAL_SUBNET=`expr $BLOCKS_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet + + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + $ips` # GLOBAL count IPs in subnet + COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet + fi + + # is normal IP + elif [[ $line =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + 1` + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` + fi +done + +# # +# Count lines and subnets +# # + +COUNT_LINES=$(wc -l < ${APP_FILE_TEMP}) # GLOBAL count ip lines +COUNT_LINES=$(printf "%'d" "$COUNT_LINES") # GLOBAL add commas to thousands +COUNT_TOTAL_IP=$(printf "%'d" "$COUNT_TOTAL_IP") # GLOBAL add commas to thousands +COUNT_TOTAL_SUBNET=$(printf "%'d" "$COUNT_TOTAL_SUBNET") # GLOBAL add commas to thousands + +BLOCKS_COUNT_TOTAL_IP=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_IP") # LOCAL add commas to thousands +BLOCKS_COUNT_TOTAL_SUBNET=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_SUBNET") # LOCAL add commas to thousands + +echo -e " πŸš› Move ${APP_FILE_TEMP} to ${APP_FILE_PERM}" +cat ${APP_FILE_TEMP} >> ${APP_FILE_PERM} # copy .tmp contents to real file +rm ${APP_FILE_TEMP} # delete temp file + +echo -e " βž• Added ${BLOCKS_COUNT_TOTAL_IP} IPs and ${BLOCKS_COUNT_TOTAL_SUBNET} Subnets to ${APP_FILE_TEMP}" + +# # +# ed +# 0a top of file +# # + +ed -s ${APP_FILE_PERM} < -# bl-static.sh 02_privacy_general.ipset privacy -# # - -# # -# Arguments -# -# This bash script has the following arguments: -# -# ARG_SAVEFILE (str) file to save IP addresses into -# ARG_BLOCKS_CAT (str) which blocks folder to inject static IP addresses from -# # - -ARG_SAVEFILE=$1 -ARG_URL=$2 -ARG_GREP=$3 - -# # -# Validation checks -# # - -if [[ -z "${ARG_SAVEFILE}" ]]; then - echo -e " β­• No output file specified for bl-htmltext" - echo -e - exit 1 -fi - -if [[ -z "${ARG_URL}" ]]; then - echo -e " β­• Aborting -- no url specified" - exit 1 -fi - -if [[ -z "${ARG_GREP}" ]]; then - echo -e " β­• Aborting -- no grep query specified" - exit 1 -fi - -# # -# Define > General -# # - -REPO="Aetherinox/dev-kw" # repository -SECONDS=0 # set seconds count for beginning of script -NOW=`date -u` # get current date in utc format -FILE_TEMP="${ARG_SAVEFILE}.tmp" # temp file when building ipset list -FOLDER_SAVE="blocklists" # folder where to save .ipset file -COUNT_LINES=0 # number of lines in doc -COUNT_TOTAL_SUBNET=0 # number of IPs in all subnets combined -COUNT_TOTAL_IP=0 # number of single IPs (counts each line) -ID="${ARG_SAVEFILE//[^[:alnum:]]/_}" # ipset id, /description/* and /category/* files must match this value -UUID=$(uuidgen -m -N "${ID}" -n @url) # uuid associated to each release -CURL_AGENT="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" -DESCRIPTION=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/descriptions/${ID}.txt") -CATEGORY=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/categories/${ID}.txt") -EXPIRES=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/expires/${ID}.txt") -URL_SOURCE=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/url-source/${ID}.txt") -regexURL='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]\.[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' - -# # -# Default Values -# # - -if [[ "$DESCRIPTION" == *"404: Not Found"* ]]; then - DESCRIPTION="# No description provided" -fi - -if [[ "$CATEGORY" == *"404: Not Found"* ]]; then - CATEGORY="Uncategorized" -fi - -if [[ "$EXPIRES" == *"404: Not Found"* ]]; then - EXPIRES="6 hours" -fi - -if [[ "$URL_SOURCE" == *"404: Not Found"* ]]; then - URL_SOURCE="None" -fi - -# # -# Output > Header -# # - -echo -e -echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" -echo -e " Blocklist - ${ARG_SAVEFILE}" -echo -e " ID: ${ID}" -echo -e " CATEGORY: ${CATEGORY}" -echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" - -# # -# output -# # - -echo -e -echo -e " ⭐ Starting" - -# # -# Create or Clean file -# # - -if [ -f $ARG_SAVEFILE ]; then - echo -e " πŸ“„ Cleaning ${ARG_SAVEFILE}" - echo -e - > ${ARG_SAVEFILE} # clean file -else - echo -e " πŸ“„ Creating ${ARG_SAVEFILE}" - echo -e - touch ${ARG_SAVEFILE} -fi - -# # -# Get IP list -# # - -echo -e " 🌎 Downloading IP blacklist to ${ARG_SAVEFILE}" - -# # -# Get IP list -# # - -jsonOutput=$(curl -Ss -A "${CURL_AGENT}" ${ARG_URL} | html2text | grep -v "^#" | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}' | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${FILE_TEMP}) -sed -i '/[#;]/{s/#.*//;s/;.*//;/^$/d}' ${FILE_TEMP} # remove # and ; comments -sed -i 's/\-.*//' ${FILE_TEMP} # remove hyphens for ip ranges -sed -i 's/[[:blank:]]*$//' ${FILE_TEMP} # remove space / tab from EOL -sed -i '/^\s*$/d' ${FILE_TEMP} # remove empty lines - -# # -# calculate how many IPs are in a subnet -# if you want to calculate the USABLE IP addresses, subtract -2 from any subnet not ending with 31 or 32. -# -# for our purpose, we want to block them all in the event that the network has reconfigured their network / broadcast IPs, -# so we will count every IP in the block. -# # - -BLOCKS_COUNT_TOTAL_IP=0 -BLOCKS_COUNT_TOTAL_SUBNET=0 - -for line in $(cat ${FILE_TEMP}); do - - # is ipv6 - if [ "$line" != "${line#*:[0-9a-fA-F]}" ]; then - if [[ $line =~ /[0-9]{1,3}$ ]]; then - COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet - BLOCKS_COUNT_TOTAL_SUBNET=`expr $BLOCKS_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet - else - COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` # GLOBAL count ip - BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + 1` # LOCAL count ip - fi; - - # is subnet - elif [[ $line =~ /[0-9]{1,2}$ ]]; then - ips=$(( 1 << (32 - ${line#*/}) )) - - regexIsNum='^[0-9]+$' - if [[ $ips =~ $regexIsNum ]]; then - CIDR=$(echo $line | sed 's:.*/::') - - # subtract - 2 from any cidr not ending with 31 or 32 - # if [[ $CIDR != "31" ]] && [[ $CIDR != "32" ]]; then - # BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP - 2` - # COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP - 2` - # fi - - BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + $ips` # LOCAL count IPs in subnet - BLOCKS_COUNT_TOTAL_SUBNET=`expr $BLOCKS_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet - - COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + $ips` # GLOBAL count IPs in subnet - COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet - fi - - # is normal IP - elif [[ $line =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + 1` - COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` - fi -done - -# # -# Count lines and subnets -# # - -COUNT_LINES=$(wc -l < ${FILE_TEMP}) # GLOBAL count ip lines -COUNT_LINES=$(printf "%'d" "$COUNT_LINES") # GLOBAL add commas to thousands -COUNT_TOTAL_IP=$(printf "%'d" "$COUNT_TOTAL_IP") # GLOBAL add commas to thousands -COUNT_TOTAL_SUBNET=$(printf "%'d" "$COUNT_TOTAL_SUBNET") # GLOBAL add commas to thousands - -BLOCKS_COUNT_TOTAL_IP=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_IP") # LOCAL add commas to thousands -BLOCKS_COUNT_TOTAL_SUBNET=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_SUBNET") # LOCAL add commas to thousands - -echo -e " πŸš› Move ${FILE_TEMP} to ${ARG_SAVEFILE}" -cat ${FILE_TEMP} >> ${ARG_SAVEFILE} # copy .tmp contents to real file -rm ${FILE_TEMP} # delete temp file - -echo -e " βž• Added ${BLOCKS_COUNT_TOTAL_IP} IPs and ${BLOCKS_COUNT_TOTAL_SUBNET} Subnets to ${FILE_TEMP}" -echo -e - -# # -# ed -# 0a top of file -# # - -ed -s ${ARG_SAVEFILE} < +# +# +# bl-json.sh 02_privacy_google.ipset https://api.domain.lan/googlebot.json '.prefixes | .[] |.ipv4Prefix//empty,.ipv6Prefix//empty' +# # πŸ“ .github # πŸ“ scripts # πŸ“„ bl-json.sh # πŸ“ workflows # πŸ“„ blocklist-generate.yml # -# activated from github workflow: -# - .github/workflows/blocklist-generate.yml -# -# within github workflow, run: -# chmod +x ".github/scripts/bl-json.sh" -# run_google=".github/scripts/bl-json.sh ${{ vars.API_02_GOOGLE_OUT }} ${{secrets.API_02_GOOGLE_URL}} '.prefixes | .[] |.ipv4Prefix//empty,.ipv6Prefix//empty'" -# eval "./$run_google" -# -# allows you to specify a .json file, and the query to use for data extraction. -# -# @uage bl-json.sh -# bl-json.sh 02_privacy_google.ipset https://api.domain.lan/googlebot.json '.prefixes | .[] |.ipv4Prefix//empty,.ipv6Prefix//empty' # # # # @@ -36,6 +41,7 @@ # ARG_JSON_QRY (str) jq rules which pull the needed ip addresses # # +APP_FILE=$(basename "$0") ARG_SAVEFILE=$1 ARG_JSON_URL=$2 ARG_JSON_QRY=$3 @@ -45,12 +51,12 @@ ARG_JSON_QRY=$3 # # if [[ -z "${ARG_SAVEFILE}" ]]; then - echo -e " β­• No output file specified for bl-json" + echo -e " β­• No output file specified for saving by script ${APP_FILE}" echo -e exit 1 fi -if [[ -z "${ARG_JSON_URL}" ]] || [[ ! $ARG_JSON_URL =~ $regexURL ]]; then +if [[ -z "${ARG_JSON_URL}" ]] || [[ ! $ARG_JSON_URL =~ $REGEX_URL ]]; then echo -e " β­• Invalid URL specified for ${ARG_SAVEFILE}" echo -e exit 1 @@ -66,40 +72,47 @@ fi # Define > General # # -REPO="Aetherinox/dev-kw" # repository -SECONDS=0 # set seconds count for beginning of script -NOW=`date -u` # get current date in utc format -FOLDER_SAVE="blocklists" # folder where to save .ipset file -COUNT_LINES=0 # number of lines in doc -COUNT_TOTAL_SUBNET=0 # number of IPs in all subnets combined -COUNT_TOTAL_IP=0 # number of single IPs (counts each line) -ID="${ARG_SAVEFILE//[^[:alnum:]]/_}" # ipset id, /description/* and /category/* files must match this value -UUID=$(uuidgen -m -N "${ID}" -n @url) # uuid associated to each release -CURL_AGENT="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" -DESCRIPTION=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/descriptions/${ID}.txt") -CATEGORY=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/categories/${ID}.txt") -EXPIRES=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/expires/${ID}.txt") -URL_SOURCE=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/url-source/${ID}.txt") -regexURL='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]\.[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' +SECONDS=0 # set seconds count for beginning of script +APP_DIR=${PWD} # returns the folder this script is being executed in +APP_REPO="Aetherinox/csf-firewall" # repository +APP_REPO_BRANCH="main" # repository branch +APP_OUT="" # each ip fetched from stdin will be stored in this var +APP_FILE_TEMP="${ARG_SAVEFILE}.tmp" # temp file when building ipset list +APP_FILE_PERM="${ARG_SAVEFILE}" # perm file when building ipset list +COUNT_LINES=0 # number of lines in doc +COUNT_TOTAL_SUBNET=0 # number of IPs in all subnets combined +COUNT_TOTAL_IP=0 # number of single IPs (counts each line) +BLOCKS_COUNT_TOTAL_IP=0 # number of ips for one particular file +BLOCKS_COUNT_TOTAL_SUBNET=0 # number of subnets for one particular file +TEMPL_NOW=`date -u` # get current date in utc format +TEMPL_ID="${APP_FILE_PERM//[^[:alnum:]]/_}" # ipset id, /description/* and /category/* files must match this value +TEMPL_UUID=$(uuidgen -m -N "${TEMPL_ID}" -n @url) # uuid associated to each release +APP_AGENT="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" +TEMPL_DESC=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/descriptions/${TEMPL_ID}.txt") +TEMPL_CAT=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/categories/${TEMPL_ID}.txt") +TEMPL_EXP=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/expires/${TEMPL_ID}.txt") +TEMP_URL_SRC=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/url-source/${TEMPL_ID}.txt") +REGEX_URL='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]\.[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' +REGEX_ISNUM='^[0-9]+$' # # # Default Values # # -if [[ "$DESCRIPTION" == *"404: Not Found"* ]]; then - DESCRIPTION="# No description provided" +if [[ "$TEMPL_DESC" == *"404: Not Found"* ]]; then + TEMPL_DESC="# No description provided" fi -if [[ "$CATEGORY" == *"404: Not Found"* ]]; then - CATEGORY="Uncategorized" +if [[ "$TEMPL_CAT" == *"404: Not Found"* ]]; then + TEMPL_CAT="Uncategorized" fi -if [[ "$EXPIRES" == *"404: Not Found"* ]]; then - EXPIRES="6 hours" +if [[ "$TEMPL_EXP" == *"404: Not Found"* ]]; then + TEMPL_EXP="6 hours" fi -if [[ "$URL_SOURCE" == *"404: Not Found"* ]]; then - URL_SOURCE="None" +if [[ "$TEMP_URL_SRC" == *"404: Not Found"* ]]; then + TEMP_URL_SRC="None" fi # # @@ -108,9 +121,10 @@ fi echo -e echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" -echo -e " Blocklist - ${ARG_SAVEFILE}" -echo -e " ID: ${ID}" -echo -e " CATEGORY: ${CATEGORY}" +echo -e " Blocklist - ${APP_FILE_PERM}" +echo -e " ID: ${TEMPL_ID}" +echo -e " UUID: ${TEMPL_UUID}" +echo -e " CATEGORY: ${TEMPL_CAT}" echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" # # @@ -124,32 +138,32 @@ echo -e " ⭐ Starting" # Create or Clean file # # -if [ -f $ARG_SAVEFILE ]; then - echo -e " πŸ“„ Cleaning ${ARG_SAVEFILE}" +if [ -f $APP_FILE_PERM ]; then + echo -e " πŸ“„ Clean ${APP_FILE_PERM}" echo -e - > ${ARG_SAVEFILE} # clean file + > ${APP_FILE_PERM} # clean file else - echo -e " πŸ“„ Creating ${ARG_SAVEFILE}" + echo -e " πŸ“ Create ${APP_FILE_PERM}" echo -e - touch ${ARG_SAVEFILE} + mkdir -p $(dirname "${APP_FILE_PERM}") + touch ${APP_FILE_PERM} fi # # # Get IP list # # -echo -e " 🌎 Downloading IP blacklist to ${ARG_SAVEFILE}" +echo -e " 🌎 Downloading IP blacklist to ${APP_FILE_PERM}" # # # Get IP list # # -tempFile="${ARG_SAVEFILE}.tmp" -jsonOutput=$(curl -Ss -A "${CURL_AGENT}" ${ARG_JSON_URL} | jq -r "${ARG_JSON_QRY}" | grep -v "^#" | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${tempFile}) -sed -i '/[#;]/{s/#.*//;s/;.*//;/^$/d}' ${tempFile} # remove # and ; comments -sed -i 's/\-.*//' ${tempFile} # remove hyphens for ip ranges -sed -i 's/[[:blank:]]*$//' ${tempFile} # remove space / tab from EOL -sed -i '/^\s*$/d' ${tempFile} # remove empty lines +jsonOutput=$(curl -sSL -A "${APP_AGENT}" ${ARG_JSON_URL} | jq -r "${ARG_JSON_QRY}" | grep -v "^#" | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${APP_FILE_TEMP}) +sed -i '/[#;]/{s/#.*//;s/;.*//;/^$/d}' ${APP_FILE_TEMP} # remove # and ; comments +sed -i 's/\-.*//' ${APP_FILE_TEMP} # remove hyphens for ip ranges +sed -i 's/[[:blank:]]*$//' ${APP_FILE_TEMP} # remove space / tab from EOL +sed -i '/^\s*$/d' ${APP_FILE_TEMP} # remove empty lines # # # calculate how many IPs are in a subnet @@ -159,10 +173,7 @@ sed -i '/^\s*$/d' ${tempFile} # remove # so we will count every IP in the block. # # -BLOCKS_COUNT_TOTAL_IP=0 -BLOCKS_COUNT_TOTAL_SUBNET=0 - -for line in $(cat ${tempFile}); do +for line in $(cat ${APP_FILE_TEMP}); do # is ipv6 if [ "$line" != "${line#*:[0-9a-fA-F]}" ]; then @@ -178,21 +189,21 @@ for line in $(cat ${tempFile}); do elif [[ $line =~ /[0-9]{1,2}$ ]]; then ips=$(( 1 << (32 - ${line#*/}) )) - regexIsNum='^[0-9]+$' - if [[ $ips =~ $regexIsNum ]]; then + if [[ $ips =~ $REGEX_ISNUM ]]; then CIDR=$(echo $line | sed 's:.*/::') + # uncomment if you want to count ONLY usable IP addresses # subtract - 2 from any cidr not ending with 31 or 32 # if [[ $CIDR != "31" ]] && [[ $CIDR != "32" ]]; then # BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP - 2` # COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP - 2` # fi - BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + $ips` # LOCAL count IPs in subnet - BLOCKS_COUNT_TOTAL_SUBNET=`expr $BLOCKS_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet + BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + $ips` # LOCAL count IPs in subnet + BLOCKS_COUNT_TOTAL_SUBNET=`expr $BLOCKS_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet - COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + $ips` # GLOBAL count IPs in subnet - COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + $ips` # GLOBAL count IPs in subnet + COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet fi # is normal IP @@ -206,43 +217,42 @@ done # Count lines and subnets # # -COUNT_LINES=$(wc -l < ${tempFile}) # GLOBAL count ip lines -COUNT_LINES=$(printf "%'d" "$COUNT_LINES") # GLOBAL add commas to thousands -COUNT_TOTAL_IP=$(printf "%'d" "$COUNT_TOTAL_IP") # GLOBAL add commas to thousands -COUNT_TOTAL_SUBNET=$(printf "%'d" "$COUNT_TOTAL_SUBNET") # GLOBAL add commas to thousands +COUNT_LINES=$(wc -l < ${APP_FILE_TEMP}) # GLOBAL count ip lines +COUNT_LINES=$(printf "%'d" "$COUNT_LINES") # GLOBAL add commas to thousands +COUNT_TOTAL_IP=$(printf "%'d" "$COUNT_TOTAL_IP") # GLOBAL add commas to thousands +COUNT_TOTAL_SUBNET=$(printf "%'d" "$COUNT_TOTAL_SUBNET") # GLOBAL add commas to thousands -BLOCKS_COUNT_TOTAL_IP=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_IP") # LOCAL add commas to thousands -BLOCKS_COUNT_TOTAL_SUBNET=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_SUBNET") # LOCAL add commas to thousands +BLOCKS_COUNT_TOTAL_IP=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_IP") # LOCAL add commas to thousands +BLOCKS_COUNT_TOTAL_SUBNET=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_SUBNET") # LOCAL add commas to thousands -echo -e " πŸš› Move ${tempFile} to ${ARG_SAVEFILE}" -cat ${tempFile} >> ${ARG_SAVEFILE} # copy .tmp contents to real file -rm ${tempFile} # delete temp file +echo -e " πŸš› Move ${APP_FILE_TEMP} to ${APP_FILE_PERM}" +cat ${APP_FILE_TEMP} >> ${APP_FILE_PERM} # copy .tmp contents to real file +rm ${APP_FILE_TEMP} # delete temp file -echo -e " βž• Added ${BLOCKS_COUNT_TOTAL_IP} IPs and ${BLOCKS_COUNT_TOTAL_SUBNET} Subnets to ${tempFile}" -echo -e +echo -e " βž• Added ${BLOCKS_COUNT_TOTAL_IP} IPs and ${BLOCKS_COUNT_TOTAL_SUBNET} Subnets to ${APP_FILE_TEMP}" # # # ed # 0a top of file # # -ed -s ${ARG_SAVEFILE} < +# +# +# {...} +# bl-master.sh 01_master.ipset URL_1 +# bl-master.sh 01_master.ipset URL_1 URL_2 URL_3 +# # πŸ“ .github -# πŸ“ blocks -# πŸ“ bruteforce -# πŸ“„ *.txt # πŸ“ scripts # πŸ“„ bl-master.sh # πŸ“ workflows # πŸ“„ blocklist-generate.yml # -# activated from github workflow: -# - .github/workflows/blocklist-generate.yml -# -# within github workflow, run: -# chmod +x ".github/scripts/bl-master.sh" -# run_master=".github/scripts/bl-master.sh ${{ vars.API_01_OUT }} false ${{ secrets.API_01_FILE_01 }} ${{ secrets.API_01_FILE_02 }} ${{ secrets.API_01_FILE_03 }}" -# eval "./$run_master" -# -# downloads a list of .txt / .ipset IP addresses in single file. -# generates a header to place at the top. -# -# @uage bl-master.sh [ , {...} ] -# bl-master.sh 01_master.ipset false API_URL_1 -# bl-master.sh 01_master.ipset true API_URL_1 API_URL_2 API_URL_3 # # # # @@ -39,29 +42,23 @@ # This bash script has the following arguments: # # ARG_SAVEFILE (str) file to save IP addresses into -# ARG_BOOL_DND (bool) add `#do not delete` to end of each line # { ... } (varg) list of URLs to API end-points # # +APP_FILE=$(basename "$0") ARG_SAVEFILE=$1 -ARG_BOOL_DND=$2 # # # Validation checks # # if [[ -z "${ARG_SAVEFILE}" ]]; then - echo -e " β­• No output file specified for downloader script" + echo -e " β­• No output file specified for saving by script ${APP_FILE}" echo -e exit 1 fi -if [[ -z "${ARG_BOOL_DND}" ]]; then - echo -e " β­• Aborting -- DND not specified" - exit 1 -fi - -if test "$#" -lt 3; then +if test "$#" -lt 2; then echo -e " β­• Aborting -- did not provide URL arguments" exit 1 fi @@ -70,40 +67,46 @@ fi # Define > General # # -REPO="Aetherinox/dev-kw" # repository -SECONDS=0 # set seconds count for beginning of script -NOW=`date -u` # get current date in utc format -FOLDER_SAVE="blocklists" # folder where to save .ipset file -COUNT_LINES=0 # number of lines in doc -COUNT_TOTAL_SUBNET=0 # number of IPs in all subnets combined -COUNT_TOTAL_IP=0 # number of single IPs (counts each line) -ID="${ARG_SAVEFILE//[^[:alnum:]]/_}" # ipset id, /description/* and /category/* files must match this value -UUID=$(uuidgen -m -N "${ID}" -n @url) # uuid associated to each release -CURL_AGENT="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" -DESCRIPTION=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/descriptions/${ID}.txt") -CATEGORY=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/categories/${ID}.txt") -EXPIRES=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/expires/${ID}.txt") -URL_SOURCE=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/url-source/${ID}.txt") -regexURL='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]\.[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' +SECONDS=0 # set seconds count for beginning of script +APP_DIR=${PWD} # returns the folder this script is being executed in +APP_REPO="Aetherinox/csf-firewall" # repository +APP_REPO_BRANCH="main" # repository branch +APP_OUT="" # each ip fetched from stdin will be stored in this var +APP_FILE_PERM="${ARG_SAVEFILE}" # perm file when building ipset list +COUNT_LINES=0 # number of lines in doc +COUNT_TOTAL_SUBNET=0 # number of IPs in all subnets combined +COUNT_TOTAL_IP=0 # number of single IPs (counts each line) +BLOCKS_COUNT_TOTAL_IP=0 # number of ips for one particular file +BLOCKS_COUNT_TOTAL_SUBNET=0 # number of subnets for one particular file +TEMPL_NOW=`date -u` # get current date in utc format +TEMPL_ID="${APP_FILE_PERM//[^[:alnum:]]/_}" # ipset id, /description/* and /category/* files must match this value +TEMPL_UUID=$(uuidgen -m -N "${TEMPL_ID}" -n @url) # uuid associated to each release +APP_AGENT="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" +TEMPL_DESC=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/descriptions/${TEMPL_ID}.txt") +TEMPL_CAT=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/categories/${TEMPL_ID}.txt") +TEMPL_EXP=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/expires/${TEMPL_ID}.txt") +TEMP_URL_SRC=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/url-source/${TEMPL_ID}.txt") +REGEX_URL='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]\.[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' +REGEX_ISNUM='^[0-9]+$' # # # Default Values # # -if [[ "$DESCRIPTION" == *"404: Not Found"* ]]; then - DESCRIPTION="# No description provided" +if [[ "$TEMPL_DESC" == *"404: Not Found"* ]]; then + TEMPL_DESC="# No description provided" fi -if [[ "$CATEGORY" == *"404: Not Found"* ]]; then - CATEGORY="Uncategorized" +if [[ "$TEMPL_CAT" == *"404: Not Found"* ]]; then + TEMPL_CAT="Uncategorized" fi -if [[ "$EXPIRES" == *"404: Not Found"* ]]; then - EXPIRES="6 hours" +if [[ "$TEMPL_EXP" == *"404: Not Found"* ]]; then + TEMPL_EXP="6 hours" fi -if [[ "$URL_SOURCE" == *"404: Not Found"* ]]; then - URL_SOURCE="None" +if [[ "$TEMP_URL_SRC" == *"404: Not Found"* ]]; then + TEMP_URL_SRC="None" fi # # @@ -112,9 +115,10 @@ fi echo -e echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" -echo -e " Blocklist - ${ARG_SAVEFILE}" -echo -e " ID: ${ID}" -echo -e " CATEGORY: ${CATEGORY}" +echo -e " Blocklist - ${APP_FILE_PERM}" +echo -e " ID: ${TEMPL_ID}" +echo -e " UUID: ${TEMPL_UUID}" +echo -e " CATEGORY: ${TEMPL_CAT}" echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" # # @@ -128,14 +132,15 @@ echo -e " ⭐ Starting" # Create or Clean file # # -if [ -f $ARG_SAVEFILE ]; then - echo -e " πŸ“„ Cleaning ${ARG_SAVEFILE}" +if [ -f $APP_FILE_PERM ]; then + echo -e " πŸ“„ Clean ${APP_FILE_PERM}" echo -e - > ${ARG_SAVEFILE} # clean file + > ${APP_FILE_PERM} # clean file else - echo -e " πŸ“„ Creating ${ARG_SAVEFILE}" + echo -e " πŸ“ Create ${APP_FILE_PERM}" echo -e - touch ${ARG_SAVEFILE} + mkdir -p $(dirname "${APP_FILE_PERM}") + touch ${APP_FILE_PERM} fi # # @@ -153,16 +158,11 @@ download_list() echo -e " 🌎 Downloading IP blacklist to ${tempFile}" - curl -sS -A "${CURL_AGENT}" ${fnUrl} -o ${tempFile} >/dev/null 2>&1 # download file - sed -i 's/\-.*//' ${tempFile} # remove hyphens for ip ranges - sed -i '/[#;]/{s/#.*//;s/;.*//;/^$/d}' ${tempFile} # remove # and ; comments - sed -i 's/[[:blank:]]*$//' ${tempFile} # remove space / tab from EOL - sed -i '/^\s*$/d' ${tempFile} # remove empty lines - - if [ "$ARG_BOOL_DND" = true ] ; then - echo -e " β­• Enabled \`# do not delete\`" - sed -i 's/$/\t\t\t\#\ do\ not\ delete/' ${tempFile} # add csf `# do not delete` to end of each line - fi + curl -sSL -A "${CURL_AGENT}" ${fnUrl} -o ${tempFile} >/dev/null 2>&1 # download file + sed -i 's/\-.*//' ${tempFile} # remove hyphens for ip ranges + sed -i '/[#;]/{s/#.*//;s/;.*//;/^$/d}' ${tempFile} # remove # and ; comments + sed -i 's/[[:blank:]]*$//' ${tempFile} # remove space / tab from EOL + sed -i '/^\s*$/d' ${tempFile} # remove empty lines # # # calculate how many IPs are in a subnet @@ -175,28 +175,28 @@ download_list() for line in $(cat ${tempFile}); do # is ipv6 if [ "$line" != "${line#*:[0-9a-fA-F]}" ]; then - COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` # GLOBAL count subnet - DL_COUNT_TOTAL_IP=`expr $DL_COUNT_TOTAL_IP + 1` # LOCAL count subnet + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` # GLOBAL count subnet + DL_COUNT_TOTAL_IP=`expr $DL_COUNT_TOTAL_IP + 1` # LOCAL count subnet # is subnet elif [[ $line =~ /[0-9]{1,2}$ ]]; then ips=$(( 1 << (32 - ${line#*/}) )) - regexIsNum='^[0-9]+$' - if [[ $ips =~ $regexIsNum ]]; then + if [[ $ips =~ $REGEX_ISNUM ]]; then CIDR=$(echo $line | sed 's:.*/::') + # uncomment if you want to count ONLY usable IP addresses # subtract - 2 from any cidr not ending with 31 or 32 # if [[ $CIDR != "31" ]] && [[ $CIDR != "32" ]]; then # COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP - 2` # DL_COUNT_TOTAL_IP=`expr $DL_COUNT_TOTAL_IP - 2` # fi - COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + $ips` # GLOBAL count IPs in subnet - COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + $ips` # GLOBAL count IPs in subnet + COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet - DL_COUNT_TOTAL_IP=`expr $DL_COUNT_TOTAL_IP + $ips` # LOCAL count IPs in subnet - DL_COUNT_TOTAL_SUBNET=`expr $DL_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet + DL_COUNT_TOTAL_IP=`expr $DL_COUNT_TOTAL_IP + $ips` # LOCAL count IPs in subnet + DL_COUNT_TOTAL_SUBNET=`expr $DL_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet fi # is normal IP @@ -210,16 +210,16 @@ download_list() # Count lines and subnets # # - DL_COUNT_TOTAL_IP=$(printf "%'d" "$DL_COUNT_TOTAL_IP") # LOCAL add commas to thousands - DL_COUNT_TOTAL_SUBNET=$(printf "%'d" "$DL_COUNT_TOTAL_SUBNET") # LOCAL add commas to thousands + DL_COUNT_TOTAL_IP=$(printf "%'d" "$DL_COUNT_TOTAL_IP") # LOCAL add commas to thousands + DL_COUNT_TOTAL_SUBNET=$(printf "%'d" "$DL_COUNT_TOTAL_SUBNET") # LOCAL add commas to thousands # # # Move temp file to final # # echo -e " πŸš› Move ${tempFile} to ${fnFile}" - cat ${tempFile} >> ${fnFile} # copy .tmp contents to real file - rm ${tempFile} # delete temp file + cat ${tempFile} >> ${fnFile} # copy .tmp contents to real file + rm ${tempFile} # delete temp file echo -e " βž• Added ${DL_COUNT_TOTAL_IP} IPs and ${DL_COUNT_TOTAL_SUBNET} subnets to ${fnFile}" } @@ -228,9 +228,9 @@ download_list() # Download lists # # -for arg in "${@:3}"; do - if [[ $arg =~ $regexURL ]]; then - download_list ${arg} ${ARG_SAVEFILE} +for arg in "${@:2}"; do + if [[ $arg =~ $REGEX_URL ]]; then + download_list ${arg} ${APP_FILE_PERM} echo -e fi done @@ -240,8 +240,8 @@ done # # if [ -d .github/blocks/ ]; then - for tempFile in .github/blocks/bruteforce/*.ipset; do - echo -e " πŸ“’ Adding static file ${tempFile}" + for APP_FILE_TEMP in .github/blocks/bruteforce/*.ipset; do + echo -e " πŸ“’ Adding static file ${APP_FILE_TEMP}" # # # calculate how many IPs are in a subnet @@ -254,7 +254,8 @@ if [ -d .github/blocks/ ]; then BLOCKS_COUNT_TOTAL_IP=0 BLOCKS_COUNT_TOTAL_SUBNET=0 - for line in $(cat ${tempFile}); do + for line in $(cat ${APP_FILE_TEMP}); do + # is ipv6 if [ "$line" != "${line#*:[0-9a-fA-F]}" ]; then COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet BLOCKS_COUNT_TOTAL_SUBNET=`expr $BLOCKS_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet @@ -263,10 +264,10 @@ if [ -d .github/blocks/ ]; then elif [[ $line =~ /[0-9]{1,2}$ ]]; then ips=$(( 1 << (32 - ${line#*/}) )) - regexIsNum='^[0-9]+$' - if [[ $ips =~ $regexIsNum ]]; then + if [[ $ips =~ $REGEX_ISNUM ]]; then CIDR=$(echo $line | sed 's:.*/::') + # uncomment if you want to count ONLY usable IP addresses # subtract - 2 from any cidr not ending with 31 or 32 # if [[ $CIDR != "31" ]] && [[ $CIDR != "32" ]]; then # BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP - 2` @@ -294,10 +295,10 @@ if [ -d .github/blocks/ ]; then BLOCKS_COUNT_TOTAL_IP=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_IP") # LOCAL add commas to thousands BLOCKS_COUNT_TOTAL_SUBNET=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_SUBNET") # LOCAL add commas to thousands - echo -e " πŸš› Move ${tempFile} to ${ARG_SAVEFILE}" - cat ${tempFile} >> ${ARG_SAVEFILE} # copy .tmp contents to real file + echo -e " πŸš› Move ${APP_FILE_TEMP} to ${APP_FILE_PERM}" + cat ${APP_FILE_TEMP} >> ${APP_FILE_PERM} # copy .tmp contents to real file - echo -e " βž• Added ${BLOCKS_COUNT_TOTAL_IP} IPs and ${BLOCKS_COUNT_TOTAL_SUBNET} Subnets to ${tempFile}" + echo -e " βž• Added ${BLOCKS_COUNT_TOTAL_IP} IPs and ${BLOCKS_COUNT_TOTAL_SUBNET} Subnets to ${APP_FILE_TEMP}" echo -e done fi @@ -309,16 +310,16 @@ fi # - remove .sort temp file # # -sorting=$(cat ${ARG_SAVEFILE} | grep -v "^#" | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${ARG_SAVEFILE}.sort) -> ${ARG_SAVEFILE} -cat ${ARG_SAVEFILE}.sort >> ${ARG_SAVEFILE} -rm ${ARG_SAVEFILE}.sort +sorting=$(cat ${APP_FILE_PERM} | grep -v "^#" | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${APP_FILE_PERM}.sort) +> ${APP_FILE_PERM} +cat ${APP_FILE_PERM}.sort >> ${APP_FILE_PERM} +rm ${APP_FILE_PERM}.sort # # # Format Counts # # -COUNT_LINES=$(wc -l < ${ARG_SAVEFILE}) # count ip lines +COUNT_LINES=$(wc -l < ${APP_FILE_PERM}) # count ip lines COUNT_LINES=$(printf "%'d" "$COUNT_LINES") # GLOBAL add commas to thousands # # @@ -333,23 +334,23 @@ COUNT_TOTAL_SUBNET=$(printf "%'d" "$COUNT_TOTAL_SUBNET") # GLOBAL # 0a top of file # # -ed -s ${ARG_SAVEFILE} < +# +# +# {...} +# bl-plain.sh 03_spam_spamhaus.ipset URL_1 +# bl-plain.sh 03_spam_spamhaus.ipset URL_1 URL_2 URL_3 +# # πŸ“ .github # πŸ“ scripts -# πŸ“„ bl-download.sh +# πŸ“„ bl-plain.sh # πŸ“ workflows # πŸ“„ blocklist-generate.yml # -# activated from github workflow: -# - .github/workflows/blocklist-generate.yml -# -# within github workflow, run: -# chmod +x ".github/scripts/bl-download.sh" -# run_master=".github/scripts/bl-download.sh ${{ vars.API_01_OUT }} false ${{ secrets.API_01_FILE_01 }} ${{ secrets.API_01_FILE_02 }} ${{ secrets.API_01_FILE_03 }}" -# eval "./$run_master" -# -# downloads a list of .txt / .ipset IP addresses in single file. -# generates a header to place at the top. -# -# @uage bl-download.sh [ , {...} ] -# bl-download.sh 01_master.ipset false API_URL_1 -# bl-download.sh 01_master.ipset true API_URL_1 API_URL_2 API_URL_3 # # # # @@ -36,25 +40,19 @@ # This bash script has the following arguments: # # ARG_SAVEFILE (str) file to save IP addresses into -# ARG_BOOL_DND (bool) add `#do not delete` to end of each line # { ... } (varg) list of URLs to download files from # # +APP_FILE=$(basename "$0") ARG_SAVEFILE=$1 -ARG_BOOL_DND=$2 if [[ -z "${ARG_SAVEFILE}" ]]; then - echo -e " β­• No output file specified for downloader script" + echo -e " β­• No output file specified for saving by script ${APP_FILE}" echo -e exit 1 fi -if [[ -z "${ARG_BOOL_DND}" ]]; then - echo -e " β­• Aborting -- DND not specified" - exit 1 -fi - -if test "$#" -lt 3; then +if test "$#" -lt 2; then echo -e " β­• Aborting -- did not provide URL arguments" exit 1 fi @@ -63,40 +61,44 @@ fi # Define > General # # -REPO="Aetherinox/dev-kw" # repository -SECONDS=0 # set seconds count for beginning of script -NOW=`date -u` # get current date in utc format -FOLDER_SAVE="blocklists" # folder where to save .ipset file -COUNT_LINES=0 # number of lines in doc -COUNT_TOTAL_SUBNET=0 # number of IPs in all subnets combined -COUNT_TOTAL_IP=0 # number of single IPs (counts each line) -ID="${ARG_SAVEFILE//[^[:alnum:]]/_}" # ipset id, /description/* and /category/* files must match this value -UUID=$(uuidgen -m -N "${ID}" -n @url) # uuid associated to each release -CURL_AGENT="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" -DESCRIPTION=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/descriptions/${ID}.txt") -CATEGORY=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/categories/${ID}.txt") -EXPIRES=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/expires/${ID}.txt") -URL_SOURCE=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/url-source/${ID}.txt") -regexURL='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]\.[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' +SECONDS=0 # set seconds count for beginning of script +APP_DIR=${PWD} # returns the folder this script is being executed in +APP_REPO="Aetherinox/csf-firewall" # repository +APP_REPO_BRANCH="main" # repository branch +APP_FILE_PERM="${ARG_SAVEFILE}" # perm file when building ipset list +APP_OUT="" # each ip fetched from stdin will be stored in this var +COUNT_LINES=0 # number of lines in doc +COUNT_TOTAL_SUBNET=0 # number of IPs in all subnets combined +COUNT_TOTAL_IP=0 # number of single IPs (counts each line) +TEMPL_NOW=`date -u` # get current date in utc format +TEMPL_ID="${APP_FILE_PERM//[^[:alnum:]]/_}" # ipset id, /description/* and /category/* files must match this value +TEMPL_UUID=$(uuidgen -m -N "${TEMPL_ID}" -n @url) # uuid associated to each release +APP_AGENT="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" +TEMPL_DESC=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/descriptions/${TEMPL_ID}.txt") +TEMPL_CAT=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/categories/${TEMPL_ID}.txt") +TEMPL_EXP=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/expires/${TEMPL_ID}.txt") +TEMP_URL_SRC=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/url-source/${TEMPL_ID}.txt") +REGEX_URL='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]\.[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' +REGEX_ISNUM='^[0-9]+$' # # # Default Values # # -if [[ "$DESCRIPTION" == *"404: Not Found"* ]]; then - DESCRIPTION="# No description provided" +if [[ "$TEMPL_DESC" == *"404: Not Found"* ]]; then + TEMPL_DESC="# No description provided" fi -if [[ "$CATEGORY" == *"404: Not Found"* ]]; then - CATEGORY="Uncategorized" +if [[ "$TEMPL_CAT" == *"404: Not Found"* ]]; then + TEMPL_CAT="Uncategorized" fi -if [[ "$EXPIRES" == *"404: Not Found"* ]]; then - EXPIRES="6 hours" +if [[ "$TEMPL_EXP" == *"404: Not Found"* ]]; then + TEMPL_EXP="6 hours" fi -if [[ "$URL_SOURCE" == *"404: Not Found"* ]]; then - URL_SOURCE="None" +if [[ "$TEMP_URL_SRC" == *"404: Not Found"* ]]; then + TEMP_URL_SRC="None" fi # # @@ -105,9 +107,10 @@ fi echo -e echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" -echo -e " Blocklist - ${ARG_SAVEFILE}" -echo -e " ID: ${ID}" -echo -e " CATEGORY: ${CATEGORY}" +echo -e " Blocklist - ${APP_FILE_PERM}" +echo -e " ID: ${TEMPL_ID}" +echo -e " UUID: ${TEMPL_UUID}" +echo -e " CATEGORY: ${TEMPL_CAT}" echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" # # @@ -121,14 +124,15 @@ echo -e " ⭐ Starting" # Create or Clean file # # -if [ -f $ARG_SAVEFILE ]; then - echo -e " πŸ“„ Cleaning ${ARG_SAVEFILE}" +if [ -f $APP_FILE_PERM ]; then + echo -e " πŸ“„ Clean ${APP_FILE_PERM}" echo -e - > ${ARG_SAVEFILE} # clean file + > ${APP_FILE_PERM} # clean file else - echo -e " πŸ“„ Creating ${ARG_SAVEFILE}" + echo -e " πŸ“ Create ${APP_FILE_PERM}" echo -e - touch ${ARG_SAVEFILE} + mkdir -p $(dirname "${APP_FILE_PERM}") + touch ${APP_FILE_PERM} fi # # @@ -146,17 +150,12 @@ download_list() echo -e " 🌎 Downloading IP blacklist to ${tempFile}" - curl -sS -A "${CURL_AGENT}" ${fnUrl} -o ${tempFile} >/dev/null 2>&1 # download file + curl -sSL -A "${CURL_AGENT}" ${fnUrl} -o ${tempFile} >/dev/null 2>&1 # download file sed -i 's/\-.*//' ${tempFile} # remove hyphens for ip ranges sed -i '/[#;]/{s/#.*//;s/;.*//;/^$/d}' ${tempFile} # remove # and ; comments sed -i 's/[[:blank:]]*$//' ${tempFile} # remove space / tab from EOL sed -i '/^\s*$/d' ${tempFile} # remove empty lines - if [ "$ARG_BOOL_DND" = true ] ; then - echo -e " β­• Enabled \`# do not delete\`" - sed -i 's/$/\t\t\t\#\ do\ not\ delete/' ${tempFile} # add csf `# do not delete` to end of each line - fi - # # # calculate how many IPs are in a subnet # if you want to calculate the USABLE IP addresses, subtract -2 from any subnet not ending with 31 or 32. @@ -168,17 +167,17 @@ download_list() for line in $(cat ${tempFile}); do # is ipv6 if [ "$line" != "${line#*:[0-9a-fA-F]}" ]; then - COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` # GLOBAL count subnet - DL_COUNT_TOTAL_IP=`expr $DL_COUNT_TOTAL_IP + 1` # LOCAL count subnet + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` # GLOBAL count subnet + DL_COUNT_TOTAL_IP=`expr $DL_COUNT_TOTAL_IP + 1` # LOCAL count subnet # is subnet elif [[ $line =~ /[0-9]{1,2}$ ]]; then ips=$(( 1 << (32 - ${line#*/}) )) - regexIsNum='^[0-9]+$' - if [[ $ips =~ $regexIsNum ]]; then + if [[ $ips =~ $REGEX_ISNUM ]]; then CIDR=$(echo $line | sed 's:.*/::') + # uncomment if you want to count ONLY usable IP addresses # subtract - 2 from any cidr not ending with 31 or 32 # if [[ $CIDR != "31" ]] && [[ $CIDR != "32" ]]; then # COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP - 2` @@ -224,9 +223,9 @@ download_list() # Download lists # # -for arg in "${@:3}"; do - if [[ $arg =~ $regexURL ]]; then - download_list ${arg} ${ARG_SAVEFILE} +for arg in "${@:2}"; do + if [[ $arg =~ $REGEX_URL ]]; then + download_list ${arg} ${APP_FILE_PERM} echo -e fi done @@ -238,16 +237,16 @@ done # - remove .sort temp file # # -sorting=$(cat ${ARG_SAVEFILE} | grep -v "^#" | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${ARG_SAVEFILE}.sort) -> ${ARG_SAVEFILE} -cat ${ARG_SAVEFILE}.sort >> ${ARG_SAVEFILE} -rm ${ARG_SAVEFILE}.sort +sorting=$(cat ${APP_FILE_PERM} | grep -v "^#" | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${APP_FILE_PERM}.sort) +> ${APP_FILE_PERM} +cat ${APP_FILE_PERM}.sort >> ${APP_FILE_PERM} +rm ${APP_FILE_PERM}.sort # # # Format Counts # # -COUNT_LINES=$(wc -l < ${ARG_SAVEFILE}) # count ip lines +COUNT_LINES=$(wc -l < ${APP_FILE_PERM}) # count ip lines COUNT_LINES=$(printf "%'d" "$COUNT_LINES") # GLOBAL add commas to thousands # # @@ -255,23 +254,23 @@ COUNT_LINES=$(printf "%'d" "$COUNT_LINES") # GL # 0a top of file # # -ed -s ${ARG_SAVEFILE} < +# +# πŸ“ .github +# πŸ“ scripts +# πŸ“„ bl-spf.sh +# πŸ“ workflows +# πŸ“„ blocklist-generate.yml +# +# # + +# # +# Arguments +# +# This bash script has the following arguments: +# +# ARG_SAVEFILE (str) file to save IP addresses into +# # + +APP_FILE=$(basename "$0") +ARG_SAVEFILE=$1 + +# # +# Validation checks +# # + +if [[ -z "${ARG_SAVEFILE}" ]]; then + echo -e " β­• No output file specified for saving by script ${APP_FILE}" + echo -e + exit 1 +fi + +# # +# Define > General +# # + +SECONDS=0 # set seconds count for beginning of script +APP_DIR=${PWD} # returns the folder this script is being executed in +APP_REPO="Aetherinox/csf-firewall" # repository +APP_REPO_BRANCH="main" # repository branch +APP_OUT="" # each ip fetched from stdin will be stored in this var +APP_FILE_TEMP="${ARG_SAVEFILE}.tmp" # temp file when building ipset list +APP_FILE_PERM="${ARG_SAVEFILE}" # perm file when building ipset list +COUNT_LINES=0 # number of lines in doc +COUNT_TOTAL_SUBNET=0 # number of IPs in all subnets combined +COUNT_TOTAL_IP=0 # number of single IPs (counts each line) +BLOCKS_COUNT_TOTAL_IP=0 # number of ips for one particular file +BLOCKS_COUNT_TOTAL_SUBNET=0 # number of subnets for one particular file +TEMPL_NOW=`date -u` # get current date in utc format +TEMPL_ID="${APP_FILE_PERM//[^[:alnum:]]/_}" # ipset id, /description/* and /category/* files must match this value +TEMPL_UUID=$(uuidgen -m -N "${TEMPL_ID}" -n @url) # uuid associated to each release +APP_AGENT="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" +TEMPL_DESC=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/descriptions/${TEMPL_ID}.txt") +TEMPL_CAT=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/categories/${TEMPL_ID}.txt") +TEMPL_EXP=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/expires/${TEMPL_ID}.txt") +TEMP_URL_SRC=$(curl -sSL -A "${APP_AGENT}" "https://raw.githubusercontent.com/${APP_REPO}/${APP_REPO_BRANCH}/.github/url-source/${TEMPL_ID}.txt") +REGEX_URL='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]\.[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' +REGEX_ISNUM='^[0-9]+$' + +# # +# Default Values +# # + +if [[ "$TEMPL_DESC" == *"404: Not Found"* ]]; then + TEMPL_DESC="# No description provided" +fi + +if [[ "$TEMPL_CAT" == *"404: Not Found"* ]]; then + TEMPL_CAT="Uncategorized" +fi + +if [[ "$TEMPL_EXP" == *"404: Not Found"* ]]; then + TEMPL_EXP="6 hours" +fi + +if [[ "$TEMP_URL_SRC" == *"404: Not Found"* ]]; then + TEMP_URL_SRC="None" +fi + +# # +# Validate ipv4 / CIDR +# +# @arg str ipv4 +# @usage if process_v4 "${entry#*:}"; then +# # + +process_v4() { + local ip_cidr="$1" + + if [[ $ip_cidr =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]{1,2})?$ ]]; then + IFS='/' read -r ip cidr <<< "$ip_cidr" + IFS='.' read -r a b c d <<< "$ip" + [[ $a -le 255 && $b -le 255 && $c -le 255 && $d -le 255 && (-z "$cidr" || ($cidr -ge 0 && $cidr -le 32)) ]] + + return $? + fi + + return 1 +} + +# # +# Validate ipv6 / CIDR +# +# @arg str ipv6 +# @usage if process_v6 "${entry#*:}"; then +# # + +process_v6() { + local ip_cidr="$1" + + # # + # Remove square brackets (if present) for URL format. + # # + + ip_cidr=${ip_cidr#[} + ip_cidr=${ip_cidr%]} + + if [[ $ip_cidr =~ ^([0-9a-fA-F:]+)(/[0-9]{1,3})?$ ]]; then + IFS='/' read -r ip cidr <<< "$ip_cidr" + + # # + # Use grep to check if ip:port is valid + # # + + if echo "$ip" | grep -qP '^(?:(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:(?:(:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$'; then + [[ -z "$cidr" || ($cidr -ge 0 && $cidr -le 128) ]] + return $? + fi + fi + + return 1 +} + +# # +# Parse > Extract SPF +# +# @usage parse_spf_record "${entry#*=}" ;; +# parse_spf_record "_spf.domain.com" ;; +# # + +parse_spf_record() { + declare -a dns_record + + local fqdn=${1} + local fqdn_i=${#fqdn} + + if [[ fqdn_i -lt 64 ]]; then + dns_record=( $( dig txt "${fqdn}" | grep -oE 'v=spf[0-9] [^"]+' ) ) + + for entry in ${dns_record[*]}; do + case ${entry} in + a ) + dig +short "${fqdn}" >/dev/null 2>&1;; + mx ) + parse_mx_record "${fqdn}" >/dev/null 2>&1 ;; + ip4:* ) + if process_v4 "${entry#*:}"; then + echo "${entry#*:}" + fi ;; + ip6:* ) + if process_v6 "${entry#*:}"; then + echo "${entry#*:}" + fi ;; + redirect=* ) + parse_spf_record "${entry#*=}" ;; + include:* ) + parse_spf_record "${entry#*:}" ;; + esac + done + fi +} + +# # +# Parse > MX Record +# +# @usage parse_mx_record "${entry#*=}" ;; +# parse_mx_record "_spf.domain.com" ;; +# # + +parse_mx_record() { + declare -a mx_records + + local fqdn=${1} + mx_records=($(dig +short mx "${fqdn}" | cut -d\ -f2 >/dev/null 2>&1)) + + for entry in ${mx_records[*]}; do + dig +short "${entry}" >/dev/null 2>&1 + done +} + +# # +# Sort Results +# +# @usage line=$(parse_spf_record "${ip}" | sort_results) +# # + +sort_results(){ + declare -a ipv4 ipv6 + + while read -r line ; do + if [[ ${line} =~ : ]] ; then + ipv6+=("${line}") + else + ipv4+=("${line}") + fi + done + + [[ -v ipv4[@] ]] && printf '%s\n' "${ipv4[@]}" | sort -g -t. -k1,1 -k 2,2 -k 3,3 -k 4,4 | uniq + [[ -v ipv6[@] ]] && printf '%s\n' "${ipv6[@]}" | sort -g -t: -k1,1 -k 2,2 -k 3,3 -k 4,4 -k 5,5 -k 6,6 -k 7,7 -k 8,8 | uniq +} + +# # +# Output > Header +# # + +echo -e +echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" +echo -e " Blocklist - ${APP_FILE_PERM}" +echo -e " ID: ${TEMPL_ID}" +echo -e " UUID: ${TEMPL_UUID}" +echo -e " CATEGORY: ${TEMPL_CAT}" +echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" + +# # +# output +# # + +echo -e +echo -e " ⭐ Starting" + +# # +# Create or Clean file +# # + +if [ -f $APP_FILE_PERM ]; then + echo -e " πŸ“„ Clean ${APP_FILE_PERM}" + echo -e + > ${APP_FILE_PERM} # clean file +else + echo -e " πŸ“ Create ${APP_FILE_PERM}" + echo -e + mkdir -p $(dirname "${APP_FILE_PERM}") + touch ${APP_FILE_PERM} +fi + +# # +# Get IP list +# # + +echo -e " 🌎 Downloading IP blacklist to ${APP_FILE_PERM}" + +# # +# Read stdin +# +# each line from stdin gets read in the while loop +# add newline at end which then be moved over to our temp file +# # + +while IFS= read -r ip || [[ -n "$ip" ]]; do + line=$(parse_spf_record "${ip}" | sort_results) + APP_OUT+="${line}"$'\n' +done + +# # +# Get IP list +# # + +list_ips=$(echo "${APP_OUT}" | grep -v "^#" | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${APP_FILE_TEMP}) +sed -i '/[#;]/{s/#.*//;s/;.*//;/^$/d}' ${APP_FILE_TEMP} # remove # and ; comments +sed -i 's/\-.*//' ${APP_FILE_TEMP} # remove hyphens for ip ranges +sed -i 's/[[:blank:]]*$//' ${APP_FILE_TEMP} # remove space / tab from EOL +sed -i '/^\s*$/d' ${APP_FILE_TEMP} # remove empty lines + +# # +# calculate how many IPs are in a subnet +# if you want to calculate the USABLE IP addresses, subtract -2 from any subnet not ending with 31 or 32. +# +# for our purpose, we want to block them all in the event that the network has reconfigured their network / broadcast IPs, +# so we will count every IP in the block. +# # + +for line in $(cat ${APP_FILE_TEMP}); do + + # is ipv6 + if [ "$line" != "${line#*:[0-9a-fA-F]}" ]; then + if [[ $line =~ /[0-9]{1,3}$ ]]; then + COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet + BLOCKS_COUNT_TOTAL_SUBNET=`expr $BLOCKS_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet + else + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` # GLOBAL count ip + BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + 1` # LOCAL count ip + fi; + + # is subnet + elif [[ $line =~ /[0-9]{1,2}$ ]]; then + ips=$(( 1 << (32 - ${line#*/}) )) + + if [[ $ips =~ $REGEX_ISNUM ]]; then + CIDR=$(echo $line | sed 's:.*/::') + + # uncomment if you want to count ONLY usable IP addresses + # subtract - 2 from any cidr not ending with 31 or 32 + # if [[ $CIDR != "31" ]] && [[ $CIDR != "32" ]]; then + # BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP - 2` + # COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP - 2` + # fi + + BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + $ips` # LOCAL count IPs in subnet + BLOCKS_COUNT_TOTAL_SUBNET=`expr $BLOCKS_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet + + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + $ips` # GLOBAL count IPs in subnet + COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet + fi + + # is normal IP + elif [[ $line =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + 1` + COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` + fi +done + +# # +# Count lines and subnets +# # + +COUNT_LINES=$(wc -l < ${APP_FILE_TEMP}) # GLOBAL count ip lines +COUNT_LINES=$(printf "%'d" "$COUNT_LINES") # GLOBAL add commas to thousands +COUNT_TOTAL_IP=$(printf "%'d" "$COUNT_TOTAL_IP") # GLOBAL add commas to thousands +COUNT_TOTAL_SUBNET=$(printf "%'d" "$COUNT_TOTAL_SUBNET") # GLOBAL add commas to thousands + +BLOCKS_COUNT_TOTAL_IP=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_IP") # LOCAL add commas to thousands +BLOCKS_COUNT_TOTAL_SUBNET=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_SUBNET") # LOCAL add commas to thousands + +echo -e " πŸš› Move ${APP_FILE_TEMP} to ${APP_FILE_PERM}" +cat ${APP_FILE_TEMP} >> ${APP_FILE_PERM} # copy .tmp contents to real file +rm ${APP_FILE_TEMP} # delete temp file + +echo -e " βž• Added ${BLOCKS_COUNT_TOTAL_IP} IPs and ${BLOCKS_COUNT_TOTAL_SUBNET} Subnets to ${APP_FILE_TEMP}" + +# # +# ed +# 0a top of file +# # + +ed -s ${APP_FILE_PERM} < -# bl-static.sh 02_privacy_general.ipset privacy -# # - -# # -# Arguments -# -# This bash script has the following arguments: -# -# ARG_SAVEFILE (str) file to save IP addresses into -# ARG_BLOCKS_CAT (str) which blocks folder to inject static IP addresses from -# # - -ARG_SAVEFILE=$1 -ARG_BLOCKS_CAT=$2 - -# # -# Validation checks -# # - -if [[ -z "${ARG_SAVEFILE}" ]]; then - echo -e " β­• No output file specified for bl-static" - echo -e - exit 1 -fi - -if [[ -z "${ARG_BLOCKS_CAT}" ]]; then - echo -e " β­• Aborting -- no static file category specified. ex: privacy" - exit 1 -fi - -# # -# Define > General -# # - -REPO="Aetherinox/dev-kw" # repository -SECONDS=0 # set seconds count for beginning of script -NOW=`date -u` # get current date in utc format -FOLDER_SAVE="blocklists" # folder where to save .ipset file -COUNT_LINES=0 # number of lines in doc -COUNT_TOTAL_SUBNET=0 # number of IPs in all subnets combined -COUNT_TOTAL_IP=0 # number of single IPs (counts each line) -ID="${ARG_SAVEFILE//[^[:alnum:]]/_}" # ipset id, /description/* and /category/* files must match this value -UUID=$(uuidgen -m -N "${ID}" -n @url) # uuid associated to each release -CURL_AGENT="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" -DESCRIPTION=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/descriptions/${ID}.txt") -CATEGORY=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/categories/${ID}.txt") -EXPIRES=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/expires/${ID}.txt") -URL_SOURCE=$(curl -sS -A "${CURL_AGENT}" "https://raw.githubusercontent.com/${REPO}/main/.github/url-source/${ID}.txt") -regexURL='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]\.[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' - -# # -# Default Values -# # - -if [[ "$DESCRIPTION" == *"404: Not Found"* ]]; then - DESCRIPTION="# No description provided" -fi - -if [[ "$CATEGORY" == *"404: Not Found"* ]]; then - CATEGORY="Uncategorized" -fi - -if [[ "$EXPIRES" == *"404: Not Found"* ]]; then - EXPIRES="6 hours" -fi - -if [[ "$URL_SOURCE" == *"404: Not Found"* ]]; then - URL_SOURCE="None" -fi - -# # -# Output > Header -# # - -echo -e -echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" -echo -e " Blocklist - ${ARG_SAVEFILE} (${ARG_BLOCKS_CAT})" -echo -e " ID: ${ID}" -echo -e " CATEGORY: ${CATEGORY}" -echo -e " ──────────────────────────────────────────────────────────────────────────────────────────────" - -# # -# output -# # - -echo -e -echo -e " ⭐ Starting" - -# # -# Create or Clean file -# # - -if [ -f $ARG_SAVEFILE ]; then - echo -e " πŸ“„ Cleaning ${ARG_SAVEFILE}" - echo -e - > ${ARG_SAVEFILE} # clean file -else - echo -e " πŸ“„ Creating ${ARG_SAVEFILE}" - echo -e - touch ${ARG_SAVEFILE} -fi - -# # -# Add Static Files -# # - -if [ -d .github/blocks/ ]; then - for tempFile in .github/blocks/${ARG_BLOCKS_CAT}/*.ipset; do - echo -e " πŸ“’ Adding static file ${tempFile}" - - # # - # calculate how many IPs are in a subnet - # if you want to calculate the USABLE IP addresses, subtract -2 from any subnet not ending with 31 or 32. - # - # for our purpose, we want to block them all in the event that the network has reconfigured their network / broadcast IPs, - # so we will count every IP in the block. - # # - - BLOCKS_COUNT_TOTAL_IP=0 - BLOCKS_COUNT_TOTAL_SUBNET=0 - - for line in $(cat ${tempFile}); do - - # is ipv6 - if [ "$line" != "${line#*:[0-9a-fA-F]}" ]; then - if [[ $line =~ /[0-9]{1,3}$ ]]; then - COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet - BLOCKS_COUNT_TOTAL_SUBNET=`expr $BLOCKS_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet - else - COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` # GLOBAL count ip - BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + 1` # LOCAL count ip - fi - - # is subnet - elif [[ $line =~ /[0-9]{1,2}$ ]]; then - ips=$(( 1 << (32 - ${line#*/}) )) - - regexIsNum='^[0-9]+$' - if [[ $ips =~ $regexIsNum ]]; then - CIDR=$(echo $line | sed 's:.*/::') - - # subtract - 2 from any cidr not ending with 31 or 32 - # if [[ $CIDR != "31" ]] && [[ $CIDR != "32" ]]; then - # BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP - 2` - # COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP - 2` - # fi - - BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + $ips` # LOCAL count IPs in subnet - BLOCKS_COUNT_TOTAL_SUBNET=`expr $BLOCKS_COUNT_TOTAL_SUBNET + 1` # LOCAL count subnet - - COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + $ips` # GLOBAL count IPs in subnet - COUNT_TOTAL_SUBNET=`expr $COUNT_TOTAL_SUBNET + 1` # GLOBAL count subnet - fi - - # is normal IP - elif [[ $line =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - BLOCKS_COUNT_TOTAL_IP=`expr $BLOCKS_COUNT_TOTAL_IP + 1` - COUNT_TOTAL_IP=`expr $COUNT_TOTAL_IP + 1` - fi - done - - # # - # Count lines and subnets - # # - - COUNT_LINES=$(wc -l < ${tempFile}) # GLOBAL count ip lines - COUNT_LINES=$(printf "%'d" "$COUNT_LINES") # GLOBAL add commas to thousands - COUNT_TOTAL_IP=$(printf "%'d" "$COUNT_TOTAL_IP") # GLOBAL add commas to thousands - COUNT_TOTAL_SUBNET=$(printf "%'d" "$COUNT_TOTAL_SUBNET") # GLOBAL add commas to thousands - - BLOCKS_COUNT_TOTAL_IP=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_IP") # LOCAL add commas to thousands - BLOCKS_COUNT_TOTAL_SUBNET=$(printf "%'d" "$BLOCKS_COUNT_TOTAL_SUBNET") # LOCAL add commas to thousands - - echo -e " πŸš› Move ${tempFile} to ${ARG_SAVEFILE}" - cat ${tempFile} >> ${ARG_SAVEFILE} # copy .tmp contents to real file - - echo -e " βž• Added ${BLOCKS_COUNT_TOTAL_IP} IPs and ${BLOCKS_COUNT_TOTAL_SUBNET} Subnets to ${tempFile}" - echo -e - done -fi - -# # -# Sort -# - sort lines numerically and create .sort file -# - move re-sorted text from .sort over to real file -# - remove .sort temp file -# # - -sorting=$(cat ${ARG_SAVEFILE} | grep -v "^#" | sort -n | awk '{if (++dup[$0] == 1) print $0;}' > ${ARG_SAVEFILE}.sort) -sed -i 's/[[:blank:]]*$//' ${ARG_SAVEFILE}.sort -> ${ARG_SAVEFILE} -cat ${ARG_SAVEFILE}.sort >> ${ARG_SAVEFILE} -rm ${ARG_SAVEFILE}.sort - -# # -# ed -# 0a top of file -# # - -ed -s ${ARG_SAVEFILE} < Cache Packages + # # + + - name: "🧱 Cache Packages" + uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: ipcalc ed html2text whois uuid-runtime + version: 1.0 # # # Job > Blocklist > Master @@ -121,11 +136,13 @@ jobs: # Set Permissions chmod +x ".github/scripts/bl-master.sh" - chmod +x ".github/scripts/bl-template.sh" - chmod +x ".github/scripts/bl-htmltext.sh" - chmod +x ".github/scripts/bl-static.sh" + chmod +x ".github/scripts/bl-format.sh" + chmod +x ".github/scripts/bl-htmlip.sh" + chmod +x ".github/scripts/bl-html.sh" + chmod +x ".github/scripts/bl-block.sh" chmod +x ".github/scripts/bl-json.sh" - chmod +x ".github/scripts/bl-download.sh" + chmod +x ".github/scripts/bl-plain.sh" + chmod +x ".github/scripts/bl-spf.sh" # # # Generate > Set Env Variables @@ -144,10 +161,10 @@ jobs: - name: "🧱 Generate β€Ί Master" id: task_blocklist_generate_master run: | - run_master=".github/scripts/bl-master.sh ${{ vars.API_01_OUT }} false ${{ secrets.API_01_FILE_01 }} ${{ secrets.API_01_FILE_02 }} ${{ secrets.API_01_FILE_03 }} ${{ secrets.API_01_FILE_04 }} ${{ secrets.API_01_FILE_05 }} ${{ secrets.API_01_FILE_06 }} ${{ secrets.API_01_FILE_07 }} ${{ secrets.API_01_FILE_08 }}" + run_master=".github/scripts/bl-master.sh ${{ vars.API_01_OUT }} ${{ secrets.API_01_FILE_01 }} ${{ secrets.API_01_FILE_02 }} ${{ secrets.API_01_FILE_03 }} ${{ secrets.API_01_FILE_04 }} ${{ secrets.API_01_FILE_05 }} ${{ secrets.API_01_FILE_06 }} ${{ secrets.API_01_FILE_07 }} ${{ secrets.API_01_FILE_08 }}" eval "./$run_master" - run_highrisk=".github/scripts/bl-htmltext.sh ${{ vars.API_01_HIGHRISK_OUT }} ${{ secrets.API_01_HIGHRISK_URL }} '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}'" + run_highrisk=".github/scripts/bl-htmlip.sh blocklists/${{ vars.API_01_HIGHRISK_OUT }} ${{ secrets.API_01_HIGHRISK_URL }} '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}'" eval "./$run_highrisk" # # @@ -158,65 +175,84 @@ jobs: id: task_blocklist_generate_privacy run: | - run_general=".github/scripts/bl-static.sh ${{ vars.API_02_GENERAL_OUT }} privacy" + # Privacy β€Ί General + run_general=".github/scripts/bl-block.sh blocklists/${{ vars.API_02_GENERAL_OUT }} privacy" eval "./$run_general" - run_google=".github/scripts/bl-json.sh ${{ vars.API_02_GOOGLE_OUT }} ${{secrets.API_02_GOOGLE_URL}} '.prefixes | .[] |.ipv4Prefix//empty,.ipv6Prefix//empty'" + # Privacy β€Ί Google + run_google=".github/scripts/bl-json.sh blocklists/02_privacy_google.ipset https://developers.google.com/search/apis/ipranges/googlebot.json '.prefixes | .[] |.ipv4Prefix//empty,.ipv6Prefix//empty'" eval "./$run_google" - run_cloudfront=".github/scripts/bl-json.sh ${{ vars.API_02_CLOUDFRONT_OUT }} ${{ secrets.API_02_CLOUDFRONT_URL }} 'map(.[]) | sort | .[]'" + # Privacy β€Ί Cloudfront + run_cloudfront=".github/scripts/bl-json.sh blocklists/02_privacy_cloudfront.ipset https://d7uri8nf7uskq.cloudfront.net/tools/list-cloudfront-ips 'map(.[]) | sort | .[]'" eval "./$run_cloudfront" - run_bing=".github/scripts/bl-json.sh ${{ vars.API_02_BING_OUT }} ${{ secrets.API_02_BING_URL }} '.prefixes | .[] |.ipv4Prefix//empty,.ipv6Prefix//empty'" + # Privacy β€Ί Bing + run_bing=".github/scripts/bl-json.sh blocklists/02_privacy_bing.ipset https://bing.com/toolbox/bingbot.json '.prefixes | .[] |.ipv4Prefix//empty,.ipv6Prefix//empty'" eval "./$run_bing" - run_fastly=".github/scripts/bl-json.sh ${{ vars.API_02_FASTLY_OUT }} ${{ secrets.API_02_FASTLY_URL }} 'map(.[]) | .[]'" + # Privacy β€Ί Fastly + run_fastly=".github/scripts/bl-json.sh blocklists/02_privacy_fastly.ipset https://api.fastly.com/public-ip-list 'map(.[]) | .[]'" eval "./$run_fastly" - run_amz_aws=".github/scripts/bl-json.sh ${{ vars.API_02_AMAZON_AWS_OUT }} ${{ secrets.API_02_AMAZON_URL }} '.prefixes[] | select(.service==\"AMAZON\") | .ip_prefix'" + # Privacy β€Ί Amazon AWS + run_amz_aws=".github/scripts/bl-json.sh blocklists/02_privacy_amazon_aws.ipset https://ip-ranges.amazonaws.com/ip-ranges.json '.prefixes[] | select(.service==\"AMAZON\") | .ip_prefix'" eval "./$run_amz_aws" - run_amz_ec2=".github/scripts/bl-json.sh ${{ vars.API_02_AMAZON_EC2_OUT }} ${{ secrets.API_02_AMAZON_URL }} '.prefixes[] | select(.service==\"EC2\") | .ip_prefix'" + # Privacy β€Ί Amazon EC2 + run_amz_ec2=".github/scripts/bl-json.sh blocklists/02_privacy_amazon_ec2.ipset https://ip-ranges.amazonaws.com/ip-ranges.json '.prefixes[] | select(.service==\"EC2\") | .ip_prefix'" eval "./$run_amz_ec2" - # Privacy > Ahrefs - curl -sS -A "${{ env.USERAGENT }}" https://api.ahrefs.com/v3/public/crawler-ips | jq -r '.ips[].ip_address | select( . != null )' | $GITHUB_WORKSPACE/.github/scripts/bl-template.sh 02_privacy_ahrefs.ipset + # Privacy β€Ί Facebook + whois -h whois.radb.net -- '-i origin AS32934' | grep ^route | awk '{gsub("(route:|route6:)","");print}' | awk '{gsub(/ /,""); print}' | $GITHUB_WORKSPACE/.github/scripts/bl-format.sh blocklists/02_privacy_facebook.ipset + + # Privacy β€Ί Ahrefs + curl -sSL -A "${{ env.USERAGENT }}" https://api.ahrefs.com/v3/public/crawler-ips | jq -r '.ips[].ip_address | select( . != null )' | $GITHUB_WORKSPACE/.github/scripts/bl-format.sh blocklists/02_privacy_ahrefs.ipset - # Privacy > DuckDuckGo - curl -sS -A "${{ env.USERAGENT }}" https://raw.githubusercontent.com/duckduckgo/duckduckgo-help-pages/master/_docs/results/duckduckbot.md | grep "^\- " | awk '{gsub("-",""); print}' | awk '{gsub(/ /,""); print}' | $GITHUB_WORKSPACE/.github/scripts/bl-template.sh 02_privacy_duckduckgo.ipset + # Privacy β€Ί DuckDuckGo + curl -sSL -A "${{ env.USERAGENT }}" https://raw.githubusercontent.com/duckduckgo/duckduckgo-help-pages/master/_docs/results/duckduckbot.md | grep "^\- " | awk '{gsub("-",""); print}' | awk '{gsub(/ /,""); print}' | $GITHUB_WORKSPACE/.github/scripts/bl-format.sh blocklists/02_privacy_duckduckgo.ipset - # Privacy > Telegram - curl -sS -A "${{ env.USERAGENT }}" https://core.telegram.org/resources/cidr.txt | $GITHUB_WORKSPACE/.github/scripts/bl-template.sh 02_privacy_telegram.ipset + # Privacy β€Ί Telegram + curl -sSL -A "${{ env.USERAGENT }}" https://core.telegram.org/resources/cidr.txt | $GITHUB_WORKSPACE/.github/scripts/bl-format.sh blocklists/02_privacy_telegram.ipset - # Privacy > Uptime Robot - curl -sS -A "${{ env.USERAGENT }}" https://uptimerobot.com/inc/files/ips/IPv4andIPv6.txt | $GITHUB_WORKSPACE/.github/scripts/bl-template.sh 02_privacy_uptimerobot.ipset + # Privacy β€Ί Uptime Robot + curl -sSL -A "${{ env.USERAGENT }}" https://uptimerobot.com/inc/files/ips/IPv4andIPv6.txt | $GITHUB_WORKSPACE/.github/scripts/bl-format.sh blocklists/02_privacy_uptimerobot.ipset - # Privacy > Pingdom - PINGDOM_IPv4=$(curl -sS -A "${{ env.USERAGENT }}" https://my.pingdom.com/probes/ipv4) - PINGDOM_IPv6=$(curl -sS -A "${{ env.USERAGENT }}" https://my.pingdom.com/probes/ipv6) + # Privacy β€Ί Pingdom + PINGDOM_IPv4=$(curl -sSL -A "${{ env.USERAGENT }}" https://my.pingdom.com/probes/ipv4) + PINGDOM_IPv6=$(curl -sSL -A "${{ env.USERAGENT }}" https://my.pingdom.com/probes/ipv6) PINGDOM_LIST="${PINGDOM_IPv4} ${PINGDOM_IPv6}" - echo "$PINGDOM_LIST" | .github/scripts/bl-template.sh 02_privacy_pingdom.ipset + echo "$PINGDOM_LIST" | $GITHUB_WORKSPACE/.github/scripts/bl-format.sh blocklists/02_privacy_pingdom.ipset - # Privacy > Stripe - curl -sS -A "${{ env.USERAGENT }}" https://stripe.com/files/ips/ips_webhooks.txt | $GITHUB_WORKSPACE/.github/scripts/bl-template.sh 02_privacy_stripe.ipset + # Privacy β€Ί Stripe β€Ί API + curl -sSL -A "${{ env.USERAGENT }}" https://stripe.com/files/ips/ips_api.txt | $GITHUB_WORKSPACE/.github/scripts/bl-format.sh blocklists/02_privacy_stripe_api.ipset - # Privacy > RSS API - curl -sS -A "${{ env.USERAGENT }}" https://rssapi.net/ips.txt | $GITHUB_WORKSPACE/.github/scripts/bl-template.sh 02_privacy_rssapi.ipset + # Privacy β€Ί Stripe β€Ί Webhooks + curl -sSL -A "${{ env.USERAGENT }}" https://stripe.com/files/ips/ips_webhooks.txt | $GITHUB_WORKSPACE/.github/scripts/bl-format.sh blocklists/02_privacy_stripe_webhooks.ipset - # Privacy > WebPageTest - curl -sS -A "${{ env.USERAGENT }}" https://www.webpagetest.org/addresses.php?f=json | jq -r '.data[].addresses[] | select( . != null )' | $GITHUB_WORKSPACE/.github/scripts/bl-template.sh 02_privacy_webpagetest.ipset + # Privacy β€Ί Stripe β€Ί Armada Gator + curl -sSL -A "${{ env.USERAGENT }}" https://stripe.com/files/ips/ips_armada_gator.txt | $GITHUB_WORKSPACE/.github/scripts/bl-format.sh blocklists/02_privacy_stripe_armada_gator.ipset + + # Privacy β€Ί RSS API + curl -sSL -A "${{ env.USERAGENT }}" https://rssapi.net/ips.txt | $GITHUB_WORKSPACE/.github/scripts/bl-format.sh blocklists/02_privacy_rssapi.ipset + + # Privacy β€Ί WebPageTest + curl -sSL -A "${{ env.USERAGENT }}" https://www.webpagetest.org/addresses.php?f=json | jq -r '.data[].addresses[] | select( . != null )' | $GITHUB_WORKSPACE/.github/scripts/bl-format.sh blocklists/02_privacy_webpagetest.ipset # Privacy > Bunny CDN - BUNNYCDN_IPv4=$(curl -sS -A "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" https://api.bunny.net/system/edgeserverlist/plain) - BUNNYCDN_IPv6=$(curl -sS -A "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" https://api.bunny.net/system/edgeserverlist/ipv6 | jq -r '.[] | select( . != null )') + BUNNYCDN_IPv4=$(curl -sSL -A "${{ env.USERAGENT }}" https://api.bunny.net/system/edgeserverlist/plain) + BUNNYCDN_IPv6=$(curl -sSL -A "${{ env.USERAGENT }}" https://api.bunny.net/system/edgeserverlist/ipv6 | jq -r '.[] | select( . != null )') BUNNYCDN_LIST="${BUNNYCDN_IPv4} ${BUNNYCDN_IPv6}" - echo "$BUNNYCDN_LIST" | .github/scripts/bl-template.sh 02_privacy_bunnycdn.ipset + echo "$BUNNYCDN_LIST" | $GITHUB_WORKSPACE/.github/scripts/bl-format.sh blocklists/02_privacy_bunnycdn.ipset - # Privacy > Cloudflare CDN - CLOUDFLARE_IPv4=$(curl -sS -A "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" https://www.cloudflare.com/ips-v4) - CLOUDFLARE_IPv6=$(curl -sS -A "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" https://www.cloudflare.com/ips-v6) + # Privacy β€Ί Cloudflare CDN + CLOUDFLARE_IPv4=$(curl -sSL -A "${{ env.USERAGENT }}" https://www.cloudflare.com/ips-v4) + CLOUDFLARE_IPv6=$(curl -sSL -A "${{ env.USERAGENT }}" https://www.cloudflare.com/ips-v6) CLOUDFLARE_LIST="${CLOUDFLARE_IPv4} ${CLOUDFLARE_IPv6}" - echo "$CLOUDFLARE_LIST" | .github/scripts/bl-template.sh 02_privacy_cloudflarecdn.ipset + echo "$CLOUDFLARE_LIST" | $GITHUB_WORKSPACE/.github/scripts/bl-format.sh blocklists/02_privacy_cloudflarecdn.ipset + + # Privacy β€Ί AppleBot + curl -sSL -A "${{ env.USERAGENT }}" https://search.developer.apple.com/applebot.json | jq -r '.prefixes | .[] |.ipv4Prefix//empty,.ipv6Prefix//empty' | $GITHUB_WORKSPACE/.github/scripts/bl-format.sh blocklists/02_privacy_applebot.ipset # # # Generate > Spam @@ -225,7 +261,7 @@ jobs: - name: "🧱 Generate β€Ί Spam" id: task_blocklist_generate_spam run: | - run_spamhaus=".github/scripts/bl-download.sh ${{ vars.API_03_SPAM_SPAMHAUS_OUT }} false ${{ secrets.API_03_SPAM_SPAMHAUS_URL }}" + run_spamhaus=".github/scripts/bl-plain.sh blocklists/${{ vars.API_03_SPAM_SPAMHAUS_OUT }} ${{ secrets.API_03_SPAM_SPAMHAUS_URL }}" eval "./$run_spamhaus" # # @@ -238,8 +274,8 @@ jobs: id: task_blocklist_spam_generate_forums if: github.event_name == 'schedule' && github.event.schedule == '0 2 * * *' run: | - chmod +x ".github/scripts/bl-download.sh" - run_forums=".github/scripts/bl-download.sh ${{ vars.API_03_SPAM_FORUMS_OUT }} false ${{ secrets.API_03_SPAM_FORUMS_URL }}" + chmod +x ".github/scripts/bl-plain.sh" + run_forums=".github/scripts/bl-plain.sh blocklists/${{ vars.API_03_SPAM_FORUMS_OUT }} ${{ secrets.API_03_SPAM_FORUMS_URL }}" eval "./$run_forums" # # @@ -251,7 +287,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: blocklist-latest - path: ./blocklists + path: ./ retention-days: 1 # # @@ -284,7 +320,7 @@ jobs: uses: actions/download-artifact@v4 with: name: blocklist-latest - path: ./blocklists + path: ./ # # # Commit > Precommit