diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index f80f988..d9aa0d0 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -8,6 +8,8 @@ jobs: test: name: test runs-on: ubuntu-latest + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} steps: - name: checkout uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index 13920ff..bd11885 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ logstash/logstash-oss-*-linux-x86_64.tar.gz logstash/plugins.zip logstash/awscliv2.zip .DS_Store +volume/ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index eb6efa4..aebc35d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,7 @@ services: environment: - DEBUG=${DEBUG-} - DOCKER_HOST=unix:///var/run/docker.sock + - LOCALSTACK_AUTH_TOKEN=${LOCALSTACK_AUTH_TOKEN} volumes: - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" - "/var/run/docker.sock:/var/run/docker.sock" diff --git a/logstash/logstash.conf b/logstash/logstash.conf index 9d9b597..c317771 100755 --- a/logstash/logstash.conf +++ b/logstash/logstash.conf @@ -77,12 +77,31 @@ filter{ # for message like: catalog-dev.data.gov - [2023-12-15T21:11:49.508628938Z] "GET /0000000 HTTP/1.1" 404 0 21445 "-" "Mozilla/5.0 ..." "127.0.x.x:xxxxx" "10.xx.2.10:xxxxx" x_forwarded_for:"108.xx.xxx.xxx, 64.252.66.xxx, 127.0.x.x" x_forwarded_proto:"https" ... ... b3:xxx grok { match => { - "log_data" => '%{HOSTNAME:hostname} - \[%{TIMESTAMP_ISO8601:timestamp}\] "%{WORD:http_method} %{GREEDYDATA:request} %{DATA:http_version}" %{NUMBER:status} %{NUMBER:bytes_sent} %{NUMBER:bytes_received} "%{DATA:http_referer}" "%{DATA:http_user_agent}" %{GREEDYDATA:skip} x_forwarded_for:"%{IP:real_ip}(?:, %{GREEDYDATA:forwarded_ips})*" x_forwarded_proto:%{GREEDYDATA:skip}' + "log_data" => '%{HOSTNAME:hostname} - \[%{TIMESTAMP_ISO8601:timestamp}\] "%{WORD:http_method} %{GREEDYDATA:request} %{DATA:http_version}" %{NUMBER:status} %{NUMBER:bytes_sent} %{NUMBER:bytes_received} "%{DATA:http_referer}" "%{DATA:http_user_agent}" %{GREEDYDATA:skip} x_forwarded_for:"%{DATA:x_forwarded_for}" x_forwarded_proto:%{GREEDYDATA:skip}' tag_on_failure => [] overwrite => ["message"] break_on_match => false } } + + ruby { + code => ' + xff = event.get("x_forwarded_for") + + if xff + ips = xff.split(",").map(&:strip).reject(&:empty?) + selected = ips[0] + + if selected =~ /\A127(?:\.\d{1,3}){3}\z/ || selected == "::1" + selected = ips[1] || selected + end + + event.set("real_ip", selected) if selected + event.set("forwarded_ips", ips[1..-1].join(", ")) if ips.length > 1 + end + ' + } + # for message like: 2023-12-14 20:31:53,839 INFO [ckan.config.middleware.flask_app] 404 /dataset/xxxx render time 0.023 seconds grok { match => {