From 6728162635540e41ad91c423933988d795e0df08 Mon Sep 17 00:00:00 2001 From: Aetherinox Date: Wed, 30 Oct 2024 04:22:16 -0700 Subject: [PATCH] chore: update geolite --- .github/scripts/bl-geolite2.sh | 50 ++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/.github/scripts/bl-geolite2.sh b/.github/scripts/bl-geolite2.sh index 6e76784a3..bac3ddab5 100644 --- a/.github/scripts/bl-geolite2.sh +++ b/.github/scripts/bl-geolite2.sh @@ -152,6 +152,7 @@ get_country_name() "gp") echo "Guadeloupe" ;; "gq") echo "Equatorial Guinea" ;; "gr") echo "Greece" ;; + "gs") echo "South Georgia and the South Sandwich Islands" ;; "gt") echo "Guatemala" ;; "gu") echo "Guam" ;; "gw") echo "Guinea-Bissau" ;; @@ -161,6 +162,7 @@ get_country_name() "hr") echo "Croatia" ;; "ht") echo "Haiti" ;; "hu") echo "Hungary" ;; + "hm") echo "Heard Island and McDonald Islands" ;; "id") echo "Indonesia" ;; "ie") echo "Ireland" ;; "il") echo "Israel" ;; @@ -280,6 +282,7 @@ get_country_name() "sz") echo "Eswatini" ;; "tc") echo "Turks Caicos Islands" ;; "td") echo "Chad" ;; + "tf") echo "French Southern Territories" ;; "tg") echo "Togo" ;; "th") echo "Thailand" ;; "tj") echo "Tajikistan" ;; @@ -402,7 +405,7 @@ af["ss"]="SS" # South Sudan # # declare -A an -an["tf"]="TR" # French Southern Territories +an["tf"]="TF" # French Southern Territories an["hm"]="HM" # Heard Island and McDonald Islands an["bv"]="BV" # Bouvet Island an["gs"]="GS" # South Georgia and the South Sandwich Islands @@ -554,7 +557,7 @@ oc["nz"]="NZ" # New Zealand oc["fj"]="FJ" # Fiji oc["pf"]="PF" # French Polynesia oc["pn"]="PN" # Pitcairn Islands -oc["ki"]="Ki" # Kiribati +oc["ki"]="KI" # Kiribati oc["tk"]="TK" # Tokelau oc["to"]="TO" # Tonga oc["wf"]="WF" # Wallis and Futuna @@ -692,6 +695,17 @@ function MAP_BUILD() OIFS=$IFS IFS=',' while read -ra LINE; do + + if [[ $APP_DEBUG == "true" ]]; then + echo -e "ID ........... ${LINE[0]}" + echo -e "Lang ......... ${LINE[1]}" + echo -e "Ctry Abbrev .. ${LINE[2]}" + echo -e "Ctry ......... ${LINE[3]}" + echo -e "City (ISO) ... ${LINE[4]}" + echo -e "City ......... ${LINE[5]}" + echo -e "\n" + fi + # echo "geonameid: ${LINE[0]}, country ISO code: ${LINE[4]}" COUNTRY_CODE="${LINE[4]}" @@ -755,9 +769,11 @@ function GENERATE_IPv4 # add ip to ipset file # # - if [[ $APP_DEBUG == "true" ]]; then - echo -e " 📄 Add ${SUBNET} to ${IPSET_FILE}" + if [ -z "${COUNTRY_CODE}" ] || [ "${COUNTRY_CODE}" == "" ] || [ "${COUNTRY_CODE}" == "_" ]; then + echo -e " 📄 Missing | Country Belonged To: ${LINE[1]} - 2: ${LINE[2]} | Subnet ${SUBNET} | Country ${COUNTRY_CODE} | File ${IPSET_FILE} | NAME ${SET_NAME}" + echo -e "+ Missing | ID ${ID} - ${LINE[2]} | Subnet ${SUBNET} | Country ${COUNTRY_CODE} | File ${IPSET_FILE} | NAME ${SET_NAME}" >> "${APP_THIS_FILE}-ipv4-missing.log" fi + echo "${SUBNET}" >> $IPSET_FILE done < <(sed -e 1d "${TEMPDIR}/${APP_GEO_IPV4_CSV}") @@ -895,20 +911,20 @@ function GENERATE_CONTINENTS() # key = antarctica # value = Antartica # - # COUNTRY_NAME = South America - # COUNTRY_ID = south_america + # CONTINENT_NAME = South America + # CONTINENT_ID = south_america # FILE_CONTINENT_TEMP = blocklists/country/geolite/ipv4/continent_europe.tmp # FILE_CONTINENT_PERM = blocklists/country/geolite/ipv4/continent_europe.ipset # # for key in "${!continents[@]}"; do - COUNTRY_NAME=${continents[$key]} - COUNTRY_ID=$( echo "$COUNTRY_NAME" | sed 's/ /_/g' | tr -d "[.,/\\-\=\+\{\[\]\}\!\@\#\$\%\^\*\'\\\(\)]" | tr '[:upper:]' '[:lower:]') - FILE_CONTINENT_TEMP="$APP_DIR_IPV4/continent_$COUNTRY_ID.$APP_TARGET_EXT_TMP" - FILE_CONTINENT_PERM="$APP_TARGET_DIR/continent_$COUNTRY_ID.$APP_TARGET_EXT_PROD" + CONTINENT_NAME=${continents[$key]} + CONTINENT_ID=$( echo "$CONTINENT_NAME" | sed 's/ /_/g' | tr -d "[.,/\\-\=\+\{\[\]\}\!\@\#\$\%\^\*\'\\\(\)]" | tr '[:upper:]' '[:lower:]') + FILE_CONTINENT_TEMP="$APP_DIR_IPV4/continent_$CONTINENT_ID.$APP_TARGET_EXT_TMP" + FILE_CONTINENT_PERM="$APP_TARGET_DIR/continent_$CONTINENT_ID.$APP_TARGET_EXT_PROD" echo -e - echo -e " 🌎 Generate Continent ${COUNTRY_NAME}" + echo -e " 🌎 Generate Continent ${CONTINENT_NAME} (${CONTINENT_ID})" # # # Return each country's ips to be included in continent file @@ -917,12 +933,20 @@ function GENERATE_CONTINENTS() # # for country in $(eval echo \${$key${i}[@]}); do + CONTINENT_COUNTRY_NAME=$(get_country_name "$country") + echo -e " 🌎 Continent ${CONTINENT_NAME} -> Adding country ${CONTINENT_COUNTRY_NAME}" + # blocklists/country/geolite/ipv4/JE.tmp FILE_TARGET="$APP_DIR_IPV4/$country.$APP_TARGET_EXT_TMP" # check if a specific country file exists, if so, open and grab all the IPs in the list. They need to be copied to $FILE_CONTINENT_TEMP if [ -f "$FILE_TARGET" ]; then + echo -e " 📒 Importing file ${FILE_TARGET} to ${FILE_CONTINENT_TEMP}" APP_OUTPUT=$(cat "$FILE_TARGET" | sort_results | awk '{if (++dup[$0] == 1) print $0;}' >> ${FILE_CONTINENT_TEMP}) + + echo -e "" + else + echo -e " ⭕ Could not find target file $FILE_TARGET" fi done @@ -961,8 +985,6 @@ function GENERATE_CONTINENTS() fi done - APP_OUTPUT=$(cat "$FILE_CONTINENT_TEMP" | sort_results | awk '{if (++dup[$0] == 1) print $0;}' >> ${FILE_CONTINENT_TEMP}) - # # # Continents > Format block count # # @@ -1029,7 +1051,7 @@ ed -s ${FILE_CONTINENT_PERM} <