skip internal ip addresses for real_ip and remove from forwarded

.github/workflows/commit.yml

@@ -8,6 +8,8 @@ jobs:
   test:
     name: test
     runs-on: ubuntu-latest
+    env:
+      LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
     steps:
       - name: checkout
         uses: actions/checkout@v2

.gitignore

@@ -3,3 +3,4 @@ logstash/logstash-oss-*-linux-x86_64.tar.gz
 logstash/plugins.zip
 logstash/awscliv2.zip
 .DS_Store
+volume/

docker-compose.yml

@@ -20,6 +20,7 @@ services:
     environment:
       - DEBUG=${DEBUG-}
       - DOCKER_HOST=unix:///var/run/docker.sock
+      - LOCALSTACK_AUTH_TOKEN=${LOCALSTACK_AUTH_TOKEN}
     volumes:
       - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
       - "/var/run/docker.sock:/var/run/docker.sock"

logstash/logstash.conf

@@ -77,12 +77,31 @@ filter{
   # for message like: catalog-dev.data.gov - [2023-12-15T21:11:49.508628938Z] "GET /0000000 HTTP/1.1" 404 0 21445 "-" "Mozilla/5.0 ..." "127.0.x.x:xxxxx" "10.xx.2.10:xxxxx" x_forwarded_for:"108.xx.xxx.xxx, 64.252.66.xxx, 127.0.x.x" x_forwarded_proto:"https" ... ... b3:xxx
   grok {
       match => {
-        "log_data" => '%{HOSTNAME:hostname} - \[%{TIMESTAMP_ISO8601:timestamp}\] "%{WORD:http_method} %{GREEDYDATA:request} %{DATA:http_version}" %{NUMBER:status} %{NUMBER:bytes_sent} %{NUMBER:bytes_received} "%{DATA:http_referer}" "%{DATA:http_user_agent}" %{GREEDYDATA:skip} x_forwarded_for:"%{IP:real_ip}(?:, %{GREEDYDATA:forwarded_ips})*" x_forwarded_proto:%{GREEDYDATA:skip}'
+        "log_data" => '%{HOSTNAME:hostname} - \[%{TIMESTAMP_ISO8601:timestamp}\] "%{WORD:http_method} %{GREEDYDATA:request} %{DATA:http_version}" %{NUMBER:status} %{NUMBER:bytes_sent} %{NUMBER:bytes_received} "%{DATA:http_referer}" "%{DATA:http_user_agent}" %{GREEDYDATA:skip} x_forwarded_for:"%{DATA:x_forwarded_for}" x_forwarded_proto:%{GREEDYDATA:skip}'
         tag_on_failure => []
         overwrite => ["message"]
         break_on_match => false
       }
   }
+
+  ruby {
+    code => '
+      xff = event.get("x_forwarded_for")
+
+      if xff
+        ips = xff.split(",").map(&:strip).reject(&:empty?)
+        selected = ips[0]
+
+        if selected =~ /\A127(?:\.\d{1,3}){3}\z/ || selected == "::1"
+          selected = ips[1] || selected
+        end
+
+        event.set("real_ip", selected) if selected
+        event.set("forwarded_ips", ips[1..-1].join(", ")) if ips.length > 1
+      end
+    '
+  }
+
   # for message like: 2023-12-14 20:31:53,839 INFO  [ckan.config.middleware.flask_app]  404 /dataset/xxxx render time 0.023 seconds
   grok {
       match => {