Skip to content

🧱 Blocklist › Generate #66

🧱 Blocklist › Generate

🧱 Blocklist › Generate #66

# #
# @usage https://github.com/Aetherinox/csf-firewall
# @type github workflow
#
# used in combination with .github/scripts/bl-download.sh
#
# download AbuseIPDB ip list after list of ips are downloaded, merges them with a static list
# that is not updated as often which contains a list of long-term abusive ip addresses
#
# local test requires the same structure as the github workflow
# 📁 .github
# 📁 blocks
# 📁 bruteforce
# 📄 01.ipset
# 📁 privacy
# 📄 01.ipset
# 📁 scripts
# 📄 bl-download.sh
# 📄 bl-htmltext.sh
# 📄 bl-json.sh
# 📄 bl-master.sh
# 📄 bl-static.sh
# 📁 workflows
# 📄 blocklist-generate.yml
# #
name: "🧱 Blocklist › Generate"
run-name: "🧱 Blocklist › Generate"
# #
# triggers
# #
on:
workflow_dispatch:
schedule:
- cron: '0 */6 * * *'
# #
# environment variables
# #
env:
BOT_NAME_1: EuropaServ
BOT_NAME_DEPENDABOT: dependabot[bot]
# #
# jobs
# #
jobs:
# #
# Job > Setup
# #
blocklist-setup:
name: >-
📦 Setup
runs-on: apollo-x64
steps:
- name: "✅ Start"
id: task_setup_start
run: |
echo "Starting blocklist build script"
# #
# Job > Checkout
# #
- name: "☑️ Checkout"
id: task_setup_checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
# #
# Generate > Install Packages
# #
- name: "🧱 Install Packages"
id: task_setup_install
run: |
sudo apt-get install -y ipcalc
sudo apt-get install -y ed
sudo apt-get install -y html2text
# #
# Job > Blocklist > Master
# #
blocklist-generate:
name: >-
📋 Generate › Blocklist
runs-on: apollo-x64
needs: [ blocklist-setup ]
steps:
# #
# Generate > Checkout
# #
- name: "☑️ Checkout"
id: task_blocklist_generate_checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
# #
# Generate > Master
# #
- name: "🧱 Generate › Master"
id: task_blocklist_generate_master
run: |
chmod +x ".github/scripts/bl-master.sh"
run_master=".github/scripts/bl-master.sh ${{ vars.API_01_OUT }} false ${{ secrets.API_01_FILE_01 }} ${{ secrets.API_01_FILE_02 }} ${{ secrets.API_01_FILE_03 }} ${{ secrets.API_01_FILE_04 }} ${{ secrets.API_01_FILE_05 }} ${{ secrets.API_01_FILE_06 }} ${{ secrets.API_01_FILE_07 }} ${{ secrets.API_01_FILE_08 }}"
eval "./$run_master"
chmod +x ".github/scripts/bl-htmltext.sh"
run_highrisk=".github/scripts/bl-htmltext.sh ${{ vars.API_01_HIGHRISK_OUT }} ${{ secrets.API_01_HIGHRISK_URL }} '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}'"
eval "./$run_highrisk"
# #
# Generate > Privacy
# #
- name: "🧱 Generate › Privacy"
id: task_blocklist_generate_privacy
run: |
chmod +x ".github/scripts/bl-static.sh"
run_general=".github/scripts/bl-static.sh ${{ vars.API_02_GENERAL_OUT }} privacy"
eval "./$run_general"
chmod +x ".github/scripts/bl-json.sh"
run_google=".github/scripts/bl-json.sh ${{ vars.API_02_GOOGLE_OUT }} ${{secrets.API_02_GOOGLE_URL}} '.prefixes | .[] |.ipv4Prefix//empty,.ipv6Prefix//empty'"
eval "./$run_google"
chmod +x ".github/scripts/bl-json.sh"
run_cloudfront=".github/scripts/bl-json.sh ${{ vars.API_02_CLOUDFRONT_OUT }} ${{ secrets.API_02_CLOUDFRONT_URL }} 'map(.[]) | sort | .[]'"
eval "./$run_cloudfront"
chmod +x ".github/scripts/bl-json.sh"
run_bing=".github/scripts/bl-json.sh ${{ vars.API_02_BING_OUT }} ${{ secrets.API_02_BING_URL }} '.prefixes | .[] |.ipv4Prefix//empty,.ipv6Prefix//empty'"
eval "./$run_bing"
chmod +x ".github/scripts/bl-json.sh"
run_fastly=".github/scripts/bl-json.sh ${{ vars.API_02_FASTLY_OUT }} ${{ secrets.API_02_FASTLY_URL }} 'map(.[]) | .[]'"
eval "./$run_fastly"
chmod +x ".github/scripts/bl-json.sh"
run_amz_aws=".github/scripts/bl-json.sh ${{ vars.API_02_AMAZON_AWS_OUT }} ${{ secrets.API_02_AMAZON_URL }} '.prefixes[] | select(.service==\"AMAZON\") | .ip_prefix'"
eval "./$run_amz_aws"
chmod +x ".github/scripts/bl-json.sh"
run_amz_ec2=".github/scripts/bl-json.sh ${{ vars.API_02_AMAZON_EC2_OUT }} ${{ secrets.API_02_AMAZON_URL }} '.prefixes[] | select(.service==\"EC2\") | .ip_prefix'"
eval "./$run_amz_ec2"
chmod +x ".github/scripts/bl-htm.sh"
run_yandex=".github/scripts/bl-htm.sh ${{ vars.API_02_YANDEX_OUT }} ${{ secrets.API_02_YANDEX_URL }}"
eval "./$run_yandex"
chmod +x ".github/scripts/bl-htm.sh"
run_apple_bot=".github/scripts/bl-htm.sh ${{ vars.API_02_APPLE_BOT_OUT }} ${{ secrets.API_02_APPLE_BOT_URL }}"
eval "./$run_apple_bot"
chmod +x ".github/scripts/bl-htm.sh"
run_facebook_bot=".github/scripts/bl-htm.sh ${{ vars.API_02_FACEBOOK_BOT_OUT }} ${{ secrets.API_02_FACEBOOK_BOT_URL }}"
eval "./$run_facebook_bot"
# #
# Generate > Spam
# #
- name: "🧱 Generate › Spam"
id: task_blocklist_generate_spam
run: |
chmod +x ".github/scripts/bl-download.sh"
run_spamhaus=".github/scripts/bl-download.sh ${{ vars.API_03_SPAM_SPAMHAUS_OUT }} false ${{ secrets.API_03_SPAM_SPAMHAUS_URL }}"
eval "./$run_spamhaus"
# #
# Generate > Artifact > Upload
# #
- name: "🎁 Generate › Upload Artifact"
id: task_blocklist_generate_artifact_upload
uses: actions/upload-artifact@v4
with:
name: blocklist-latest
path: ./blocklists
retention-days: 1
# #
# Job > Commit
# #
blocklist-commit:
name: >-
📋 Commit
runs-on: apollo-x64
needs: [ blocklist-setup, blocklist-generate ]
steps:
# #
# Generate > Checkout
# #
- name: "☑️ Commit › Checkout"
id: task_blocklist_master_checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
# #
# Generate > Artifact > Download
# #
- name: "🎁 Commit › Download Artifact"
id: task_commit_artifact_download
uses: actions/download-artifact@v4
with:
name: blocklist-latest
path: ./blocklists
# #
# Commit > Precommit
# #
- name: "📦 Commit › Pre-commit"
id: task_commit_pre
run: |
now=$(date '+%m/%d/%Y %H:%M')
commit_label="Sync" >> $GITHUB_ENV
commit_message="\`️️🔒 $commit_label 🔒\` \`$now\`" >> $GITHUB_ENV
echo "COMMIT_MESSAGE=$(echo $commit_message)" >> $GITHUB_ENV
echo "NOW=$(echo $now)" >> $GITHUB_ENV
# #
# GPG Key
# #
- name: "📦 Commit › GPG Key"
id: task_commit_gpg
uses: crazy-max/ghaction-import-gpg@v6
with:
gpg_private_key: ${{ secrets.ADMINSERV_GPG_KEY_ASC }}
passphrase: ${{ secrets.ADMINSERV_GPG_PASSPHRASE }}
git_user_signingkey: true
git_commit_gpgsign: true
# #
# Commit > Commit
# #
- name: "📦 Commit › Execute"
id: task_commit_execute
uses: stefanzweifel/git-auto-commit-action@v5
with:
commit_message: ${{ env.COMMIT_MESSAGE }}
commit_author: "${{ steps.task_commit_gpg.outputs.name }} <${{ steps.task_commit_gpg.outputs.email }}>"
commit_user_name: ${{ steps.task_commit_gpg.outputs.name }}
commit_user_email: ${{ steps.task_commit_gpg.outputs.email }}