Add Microsoft Public IP Space & streamline scripts

examples/v2-alpine_cloudflare_rate-limit/Caddyfile

@@ -1,5 +1,5 @@
 #
-# This example adds a rate limit for all IPs in "abuseipdb/s100-7d.ipv4.Caddyfile" and "127.18.0.1"
+# This example adds a rate limit for all IPs in "abuseipdb/s100-7d.ipv4.caddyfile" and "127.18.0.1"
 # Check https://github.com/mholt/caddy-ratelimit for more details about the rate_limit module
 #
 :80 {
@@ -16,7 +16,8 @@
     @rateLimitedIPs {
         # Add your own IP here to check if the rate limit works as expected
         remote_ip 172.18.0.1
-        import abuseipdb/s100-7d.ipv4.Caddyfile
+        import abuseipdb/s100-7d.ipv4.caddyfile
+        import microsoft-public-ip-space/current.caddyfile
     }
 
     rate_limit @rateLimitedIPs {

images/v2-alpine_cloudflare_rate-limit/Dockerfile

@@ -5,16 +5,39 @@ RUN xcaddy build \
     --with github.com/caddy-dns/cloudflare \
     --with github.com/mholt/caddy-ratelimit
 
-# Add the abuseipdb download script
-COPY /images/v2-alpine_cloudflare_rate-limit/bin/download_abuseipdb.sh /usr/bin/download_abuseipdb.sh
-RUN chmod +x /usr/bin/download_abuseipdb.sh
-RUN /usr/bin/download_abuseipdb.sh "abuseipdb-s100-7d.ipv4" "/tmp/s100-7d.ipv4.Caddyfile"
-
 FROM caddy:2-alpine
 
+# Defines in which directory the blocklist should be stored. A subdirectory of /etc/caddy is recommended so the blocklist can be used in a Caddyfile (import abuseipdb/...)
+ENV ABUSE_IP_DB_LOCAL_BASE_DIRECTORY="/etc/caddy/abuseipdb"
+# The filename where the blocklist is stored inside of ABUSE_IP_DB_LOCAL_BASE_DIRECTORY
+ENV ABUSE_IP_DB_LOCAL_FILENAME="s100-7d.ipv4.caddyfile"
+# We use the 7d blocklist, because it's a good mix of "up to date" and "too shortlived"
+# Check https://github.com/borestad/blocklist-abuseipdb for all available options
+ENV ABUSE_IP_DB_REMOTE_FILENAME="abuseipdb-s100-7d.ipv4"
+# As the minimum expected IPs/rows we use 20000, on 2024-10-04 the blocklist had 47703 rows so this should be safe
+ENV ABUSE_IP_DB_MINIMUM_ENTRY_COUNT=20000
+
+# Defines in which directory the Microsoft Public IP space should be stored.
+# A subdirectory of /etc/caddy is recommended so the blocklist can be used in a Caddyfile (import microsoft-public-ip-space/...)
+ENV MICROSOFT_PUBLIC_IP_SPACE_LOCAL_BASE_DIRECTORY="/etc/caddy/microsoft-public-ip-space"
+# The filename where the Microsoft Public IP Space is stored inside of MICROSOFT_PUBLIC_IP_SPACE_LOCAL_BASE_DIRECTORY
+ENV MICROSOFT_PUBLIC_IP_SPACE_LOCAL_FILENAME="current.caddyfile"
+
 # Copy the caddy binary from the builder image
 COPY --from=builder /usr/bin/caddy /usr/bin/caddy
 
-# Add the default rate limited IPs files
-RUN mkdir "/etc/caddy/abuseipdb"
-COPY --from=builder /tmp/s100-7d.ipv4.Caddyfile /etc/caddy/abuseipdb/
+# Add AbuseIPDB scripts
+COPY /images/v2-alpine_cloudflare_rate-limit/bin/abuseipdb_cron.sh /usr/local/bin/
+COPY /images/v2-alpine_cloudflare_rate-limit/bin/abuseipdb_update.sh /usr/local/bin/
+# Ensure the AbuseIPDB base directory exists
+RUN mkdir "${ABUSE_IP_DB_LOCAL_BASE_DIRECTORY}"
+# Download & process the selected AbuseIPDB blocklist
+RUN /usr/local/bin/abuseipdb_update.sh
+
+# Add Microsoft Public IP Space scripts
+COPY /images/v2-alpine_cloudflare_rate-limit/bin/microsoft-public-ip-space_cron.sh /usr/local/bin/
+COPY /images/v2-alpine_cloudflare_rate-limit/bin/microsoft-public-ip-space_update.sh /usr/local/bin/
+# Ensure the Microsoft Public IP Space base directory exists
+RUN mkdir "${MICROSOFT_PUBLIC_IP_SPACE_LOCAL_BASE_DIRECTORY}"
+# Download & process the Microsoft Public IP space
+RUN /usr/local/bin/microsoft-public-ip-space_update.sh

images/v2-alpine_cloudflare_rate-limit/bin/abuseipdb_cron.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+#
+# Execute this script with a cron job on the host system to keep the AbuseIPDB list up to date without pulling the latest image
+#
+
+# Update the AbuseIPDB blocklist
+/usr/local/bin/abuseipdb_update.sh
+# Reload caddy to apply the new updated AbuseIPDB list
+caddy reload --config /etc/caddy/Caddyfile

images/v2-alpine_cloudflare_rate-limit/bin/abuseipdb_update.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+#
+# This script expects the following environment variables to exist:
+# ABUSE_IP_DB_LOCAL_BASE_DIRECTORY
+# ABUSE_IP_DB_LOCAL_FILENAME
+# ABUSE_IP_DB_REMOTE_FILENAME
+# ABUSE_IP_DB_MINIMUM_ENTRY_COUNT
+#
+# Check the Dockerfile for a detailed explanation of these environment variables
+#
+
+DOWNLOAD_URL="https://raw.githubusercontent.com/borestad/blocklist-abuseipdb/refs/heads/main/${ABUSE_IP_DB_REMOTE_FILENAME}"
+OUTPUT_FILE="${ABUSE_IP_DB_LOCAL_BASE_DIRECTORY}/${ABUSE_IP_DB_LOCAL_FILENAME}"
+
+# Download the AbuseIPDB blocklist
+TEMP_DOWNLOAD_FILE=$(mktemp)
+if ! wget "${DOWNLOAD_URL}" -O "${TEMP_DOWNLOAD_FILE}"
+then
+    echo "Failed to download the AbuseIPDB blocklist"
+    exit 1
+fi
+
+LINE_COUNT=$(wc -l < "${TEMP_DOWNLOAD_FILE}")
+if [ "${LINE_COUNT}" -lt "${ABUSE_IP_DB_MINIMUM_ENTRY_COUNT}" ]
+then
+  echo "Too few IPs in the list (${TEMP_DOWNLOAD_FILE}). Expected: ${ABUSE_IP_DB_MINIMUM_ENTRY_COUNT} Actual: ${LINE_COUNT}"
+  exit 1
+fi
+
+echo "Successfully downloaded the AbuseIPDB blocklist to ${TEMP_DOWNLOAD_FILE}"
+
+# Truncate the output file, otherwise running this script multiple times would append the result every time
+true > "${OUTPUT_FILE}"
+
+# Loop through each IP in the AbuseIPDB file, excluding comments, and add remote_ip before each IP so it can be imported in a Caddyfile
+for IP in $(grep -hv '^#' "${TEMP_DOWNLOAD_FILE}"); do
+    echo "remote_ip $IP" >> "${OUTPUT_FILE}"
+done

images/v2-alpine_cloudflare_rate-limit/bin/microsoft-public-ip-space_cron.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+#
+# Execute this script with a cron job on the host system to keep the Microsoft Public IP Space up to date without pulling the latest image
+#
+
+# Update the AbuseIPDB blocklist
+/usr/local/bin/microsoft-public-ip-space_update.sh
+# Reload caddy to apply the new updated AbuseIPDB list
+caddy reload --config /etc/caddy/Caddyfile

images/v2-alpine_cloudflare_rate-limit/bin/microsoft-public-ip-space_update.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+# Microsoft landing page which contains the URL to the current public IP CSV
+PAGE_URL="https://www.microsoft.com/en-us/download/details.aspx?id=53602"
+
+# Microsoft blocks requests from wget without a valid user agent, so we fake one
+USER_AGENT="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36"
+
+# Output file name
+OUTPUT_FILE="${MICROSOFT_PUBLIC_IP_SPACE_LOCAL_BASE_DIRECTORY}/${MICROSOFT_PUBLIC_IP_SPACE_LOCAL_FILENAME}"
+
+# Determine the current CSV URL
+LATEST_CSV_URL=$(wget --user-agent="$USER_AGENT" -q -O - "${PAGE_URL}" | grep -oE 'https://download\.microsoft\.com/download/[^\"]+\.csv' | head -n 1)
+
+if [ -z "${LATEST_CSV_URL}" ]; then
+    echo "Failed to determine the latest CSV URL"
+    exit 1
+fi
+
+LATEST_CSV_DATA=$(wget --user-agent="$USER_AGENT" -q -O - "${LATEST_CSV_URL}")
+
+LINE_COUNT=$(echo "${LATEST_CSV_DATA}" | wc -l)
+if [ "${LINE_COUNT}" -lt 100 ]
+then
+  echo "Too few IPs in the list. Expected: 100 Actual: ${LINE_COUNT}"
+  exit 1
+fi
+
+# Truncate the output file, otherwise running this script multiple times would append the result every time
+true > "${OUTPUT_FILE}"
+
+# Remove header row from the CSV
+LATEST_CSV_DATA=$(echo "${LATEST_CSV_DATA}" | tail -n +2)
+
+# Get first column (IP range)
+LATEST_CSV_DATA=$(echo "${LATEST_CSV_DATA}" | cut -d',' -f1)
+
+# Filter IPv6 addresses
+LATEST_CSV_DATA=$(echo "${LATEST_CSV_DATA}" | grep -v ':')
+
+echo "${LATEST_CSV_DATA}" | while read -r IP_RANGE
+do
+    echo "remote_ip ${IP_RANGE}" >> "${OUTPUT_FILE}"
+done