🧱 Blocklist › Generate #71
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # # | |
| # @usage https://github.com/Aetherinox/csf-firewall | |
| # @type github workflow | |
| # | |
| # used in combination with .github/scripts/bl-download.sh | |
| # | |
| # download AbuseIPDB ip list after list of ips are downloaded, merges them with a static list | |
| # that is not updated as often which contains a list of long-term abusive ip addresses | |
| # | |
| # local test requires the same structure as the github workflow | |
| # 📁 .github | |
| # 📁 blocks | |
| # 📁 bruteforce | |
| # 📄 01.ipset | |
| # 📁 privacy | |
| # 📄 01.ipset | |
| # 📁 scripts | |
| # 📄 bl-download.sh | |
| # 📄 bl-htmltext.sh | |
| # 📄 bl-json.sh | |
| # 📄 bl-master.sh | |
| # 📄 bl-static.sh | |
| # 📁 workflows | |
| # 📄 blocklist-generate.yml | |
| # # | |
| name: "🧱 Blocklist › Generate" | |
| run-name: "🧱 Blocklist › Generate" | |
| # # | |
| # triggers | |
| # # | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 0,6,12,18 * * *' | |
| - cron: '0 2 * * *' | |
| # # | |
| # environment variables | |
| # # | |
| env: | |
| BOT_NAME_1: EuropaServ | |
| BOT_NAME_DEPENDABOT: dependabot[bot] | |
| # # | |
| # jobs | |
| # # | |
| jobs: | |
| # # | |
| # Job > Setup | |
| # # | |
| blocklist-setup: | |
| name: >- | |
| 📦 Setup | |
| runs-on: apollo-x64 | |
| steps: | |
| - name: "✅ Start" | |
| id: task_setup_start | |
| run: | | |
| echo "Starting blocklist build script" | |
| # # | |
| # Job > Checkout | |
| # # | |
| - name: "☑️ Checkout" | |
| id: task_setup_checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # # | |
| # Generate > Install Packages | |
| # # | |
| - name: "🧱 Install Packages" | |
| id: task_setup_install | |
| run: | | |
| sudo apt-get install -y ipcalc | |
| sudo apt-get install -y ed | |
| sudo apt-get install -y html2text | |
| sudo apt-get install -y whois | |
| sudo apt-get install -y uuid-runtime | |
| # # | |
| # Job > Blocklist > Master | |
| # # | |
| blocklist-generate: | |
| name: >- | |
| 📋 Generate › Blocklist | |
| runs-on: apollo-x64 | |
| needs: [ blocklist-setup ] | |
| steps: | |
| # # | |
| # Generate > Checkout | |
| # # | |
| - name: "☑️ Checkout" | |
| id: task_blocklist_generate_checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # # | |
| # Generate > Set Template Permissions | |
| # # | |
| - name: "☑️ Set Permissions" | |
| id: task_blocklist_generate_perms | |
| run: | | |
| # Set Permissions | |
| chmod +x ".github/scripts/bl-master.sh" | |
| chmod +x ".github/scripts/bl-template.sh" | |
| chmod +x ".github/scripts/bl-htmltext.sh" | |
| chmod +x ".github/scripts/bl-static.sh" | |
| chmod +x ".github/scripts/bl-json.sh" | |
| chmod +x ".github/scripts/bl-download.sh" | |
| # # | |
| # Generate > Set Env Variables | |
| # # | |
| - name: "📦 Set Env Variables" | |
| id: task_commit_pre | |
| run: | | |
| useragent="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" | |
| echo "USERAGENT=$(echo $useragent)" >> $GITHUB_ENV | |
| # # | |
| # Generate > Master | |
| # # | |
| - name: "🧱 Generate › Master" | |
| id: task_blocklist_generate_master | |
| run: | | |
| run_master=".github/scripts/bl-master.sh ${{ vars.API_01_OUT }} false ${{ secrets.API_01_FILE_01 }} ${{ secrets.API_01_FILE_02 }} ${{ secrets.API_01_FILE_03 }} ${{ secrets.API_01_FILE_04 }} ${{ secrets.API_01_FILE_05 }} ${{ secrets.API_01_FILE_06 }} ${{ secrets.API_01_FILE_07 }} ${{ secrets.API_01_FILE_08 }}" | |
| eval "./$run_master" | |
| run_highrisk=".github/scripts/bl-htmltext.sh ${{ vars.API_01_HIGHRISK_OUT }} ${{ secrets.API_01_HIGHRISK_URL }} '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}'" | |
| eval "./$run_highrisk" | |
| # # | |
| # Generate > Privacy | |
| # # | |
| - name: "🧱 Generate › Privacy" | |
| id: task_blocklist_generate_privacy | |
| run: | | |
| run_general=".github/scripts/bl-static.sh ${{ vars.API_02_GENERAL_OUT }} privacy" | |
| eval "./$run_general" | |
| run_google=".github/scripts/bl-json.sh ${{ vars.API_02_GOOGLE_OUT }} ${{secrets.API_02_GOOGLE_URL}} '.prefixes | .[] |.ipv4Prefix//empty,.ipv6Prefix//empty'" | |
| eval "./$run_google" | |
| run_cloudfront=".github/scripts/bl-json.sh ${{ vars.API_02_CLOUDFRONT_OUT }} ${{ secrets.API_02_CLOUDFRONT_URL }} 'map(.[]) | sort | .[]'" | |
| eval "./$run_cloudfront" | |
| run_bing=".github/scripts/bl-json.sh ${{ vars.API_02_BING_OUT }} ${{ secrets.API_02_BING_URL }} '.prefixes | .[] |.ipv4Prefix//empty,.ipv6Prefix//empty'" | |
| eval "./$run_bing" | |
| run_fastly=".github/scripts/bl-json.sh ${{ vars.API_02_FASTLY_OUT }} ${{ secrets.API_02_FASTLY_URL }} 'map(.[]) | .[]'" | |
| eval "./$run_fastly" | |
| run_amz_aws=".github/scripts/bl-json.sh ${{ vars.API_02_AMAZON_AWS_OUT }} ${{ secrets.API_02_AMAZON_URL }} '.prefixes[] | select(.service==\"AMAZON\") | .ip_prefix'" | |
| eval "./$run_amz_aws" | |
| run_amz_ec2=".github/scripts/bl-json.sh ${{ vars.API_02_AMAZON_EC2_OUT }} ${{ secrets.API_02_AMAZON_URL }} '.prefixes[] | select(.service==\"EC2\") | .ip_prefix'" | |
| eval "./$run_amz_ec2" | |
| # Privacy > Ahrefs | |
| curl -sS -A "${{ env.USERAGENT }}" https://api.ahrefs.com/v3/public/crawler-ips | jq -r '.ips[].ip_address | select( . != null )' | $GITHUB_WORKSPACE/.github/scripts/bl-template.sh 02_privacy_ahrefs.ipset | |
| # Privacy > DuckDuckGo | |
| curl -sS -A "${{ env.USERAGENT }}" https://raw.githubusercontent.com/duckduckgo/duckduckgo-help-pages/master/_docs/results/duckduckbot.md | grep "^\- " | awk '{gsub("-",""); print}' | awk '{gsub(/ /,""); print}' | $GITHUB_WORKSPACE/.github/scripts/bl-template.sh 02_privacy_duckduckgo.ipset | |
| # Privacy > Telegram | |
| curl -sS -A "${{ env.USERAGENT }}" https://core.telegram.org/resources/cidr.txt | $GITHUB_WORKSPACE/.github/scripts/bl-template.sh 02_privacy_telegram.ipset | |
| # Privacy > Uptime Robot | |
| curl -sS -A "${{ env.USERAGENT }}" https://uptimerobot.com/inc/files/ips/IPv4andIPv6.txt | $GITHUB_WORKSPACE/.github/scripts/bl-template.sh 02_privacy_uptimerobot.ipset | |
| # Privacy > Pingdom | |
| PINGDOM_IPv4=$(curl -sS -A "${{ env.USERAGENT }}" https://my.pingdom.com/probes/ipv4) | |
| PINGDOM_IPv6=$(curl -sS -A "${{ env.USERAGENT }}" https://my.pingdom.com/probes/ipv6) | |
| PINGDOM_LIST="${PINGDOM_IPv4} ${PINGDOM_IPv6}" | |
| echo "$PINGDOM_LIST" | .github/scripts/bl-template.sh 02_privacy_pingdom.ipset | |
| # Privacy > Stripe | |
| curl -sS -A "${{ env.USERAGENT }}" https://stripe.com/files/ips/ips_webhooks.txt | $GITHUB_WORKSPACE/.github/scripts/bl-template.sh 02_privacy_stripe.ipset | |
| # Privacy > RSS API | |
| curl -sS -A "${{ env.USERAGENT }}" https://rssapi.net/ips.txt | $GITHUB_WORKSPACE/.github/scripts/bl-template.sh 02_privacy_rssapi.ipset | |
| # Privacy > WebPageTest | |
| curl -sS -A "${{ env.USERAGENT }}" https://www.webpagetest.org/addresses.php?f=json | jq -r '.data[].addresses[] | select( . != null )' | $GITHUB_WORKSPACE/.github/scripts/bl-template.sh 02_privacy_webpagetest.ipset | |
| # Privacy > Bunny CDN | |
| BUNNYCDN_IPv4=$(curl -sS -A "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" https://api.bunny.net/system/edgeserverlist/plain) | |
| BUNNYCDN_IPv6=$(curl -sS -A "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" https://api.bunny.net/system/edgeserverlist/ipv6 | jq -r '.[] | select( . != null )') | |
| BUNNYCDN_LIST="${BUNNYCDN_IPv4} ${BUNNYCDN_IPv6}" | |
| echo "$BUNNYCDN_LIST" | .github/scripts/bl-template.sh 02_privacy_bunnycdn.ipset | |
| # Privacy > Cloudflare CDN | |
| CLOUDFLARE_IPv4=$(curl -sS -A "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" https://www.cloudflare.com/ips-v4) | |
| CLOUDFLARE_IPv6=$(curl -sS -A "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" https://www.cloudflare.com/ips-v6) | |
| CLOUDFLARE_LIST="${CLOUDFLARE_IPv4} ${CLOUDFLARE_IPv6}" | |
| echo "$CLOUDFLARE_LIST" | .github/scripts/bl-template.sh 02_privacy_cloudflarecdn.ipset | |
| # # | |
| # Generate > Spam | |
| # # | |
| - name: "🧱 Generate › Spam" | |
| id: task_blocklist_generate_spam | |
| run: | | |
| run_spamhaus=".github/scripts/bl-download.sh ${{ vars.API_03_SPAM_SPAMHAUS_OUT }} false ${{ secrets.API_03_SPAM_SPAMHAUS_URL }}" | |
| eval "./$run_spamhaus" | |
| # # | |
| # Generate > Spam > Forums | |
| # | |
| # only updated once per day (at 1am UTC) | |
| # # | |
| - name: "🧱 Generate › Spam › Forums (1/day)" | |
| id: task_blocklist_spam_generate_forums | |
| if: github.event_name == 'schedule' && github.event.schedule == '0 2 * * *' | |
| run: | | |
| chmod +x ".github/scripts/bl-download.sh" | |
| run_forums=".github/scripts/bl-download.sh ${{ vars.API_03_SPAM_FORUMS_OUT }} false ${{ secrets.API_03_SPAM_FORUMS_URL }}" | |
| eval "./$run_forums" | |
| # # | |
| # Generate > Artifact > Upload | |
| # # | |
| - name: "🎁 Generate › Upload Artifact" | |
| id: task_blocklist_generate_artifact_upload | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: blocklist-latest | |
| path: ./blocklists | |
| retention-days: 1 | |
| # # | |
| # Job > Commit | |
| # # | |
| blocklist-commit: | |
| name: >- | |
| 📋 Commit | |
| runs-on: apollo-x64 | |
| needs: [ blocklist-setup, blocklist-generate ] | |
| steps: | |
| # # | |
| # Generate > Checkout | |
| # # | |
| - name: "☑️ Commit › Checkout" | |
| id: task_blocklist_master_checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # # | |
| # Generate > Artifact > Download | |
| # # | |
| - name: "🎁 Commit › Download Artifact" | |
| id: task_commit_artifact_download | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: blocklist-latest | |
| path: ./blocklists | |
| # # | |
| # Commit > Precommit | |
| # # | |
| - name: "📦 Commit › Pre-commit" | |
| id: task_commit_pre | |
| run: | | |
| now=$(date '+%m/%d/%Y %H:%M') | |
| commit_label="Sync" >> $GITHUB_ENV | |
| commit_message="\`️️🔒 $commit_label 🔒\` \`$now\`" >> $GITHUB_ENV | |
| echo "COMMIT_MESSAGE=$(echo $commit_message)" >> $GITHUB_ENV | |
| echo "NOW=$(echo $now)" >> $GITHUB_ENV | |
| # # | |
| # GPG Key | |
| # # | |
| - name: "📦 Commit › GPG Key" | |
| id: task_commit_gpg | |
| uses: crazy-max/ghaction-import-gpg@v6 | |
| with: | |
| gpg_private_key: ${{ secrets.ADMINSERV_GPG_KEY_ASC }} | |
| passphrase: ${{ secrets.ADMINSERV_GPG_PASSPHRASE }} | |
| git_user_signingkey: true | |
| git_commit_gpgsign: true | |
| # # | |
| # Commit > Commit | |
| # # | |
| - name: "📦 Commit › Execute" | |
| id: task_commit_execute | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| commit_message: ${{ env.COMMIT_MESSAGE }} | |
| commit_author: "${{ steps.task_commit_gpg.outputs.name }} <${{ steps.task_commit_gpg.outputs.email }}>" | |
| commit_user_name: ${{ steps.task_commit_gpg.outputs.name }} | |
| commit_user_email: ${{ steps.task_commit_gpg.outputs.email }} |