From 348824af0905dcae8d465aaeb25ee6f6c9f9c15a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Fernandes?= Date: Thu, 28 Sep 2023 21:44:03 +0100 Subject: [PATCH] revision --- README.md | 5 ++-- .../rootfs/etc/nginx/conf.d/default.conf | 10 ++----- .../rootfs/etc/telegraf/telegraf.d/nginx.conf | 15 ++++++---- influxdb/by_nginx_client_ip.flux | 27 +++++++++++++++++ influxdb/by_nginx_host.flux | 26 ++++++++++++++++ influxdb/by_nginx_request.flux | 30 +++++++++++++++++++ influxdb/by_nginx_resp_code.flux | 23 ++++++++++++++ influxdb/by_nginx_verb.flux | 23 ++++++++++++++ influxdb/get_nginx_hosts.flux | 17 +++++++++++ influxdb/testess.flux | 19 ++++++++++++ 10 files changed, 179 insertions(+), 16 deletions(-) create mode 100644 influxdb/by_nginx_client_ip.flux create mode 100644 influxdb/by_nginx_host.flux create mode 100644 influxdb/by_nginx_request.flux create mode 100644 influxdb/by_nginx_resp_code.flux create mode 100644 influxdb/by_nginx_verb.flux create mode 100644 influxdb/get_nginx_hosts.flux create mode 100644 influxdb/testess.flux diff --git a/README.md b/README.md index 80ddc69..06569c7 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,8 @@ Notify Influxdb. # nginx - telegraf -(no tusing prometheus must same steps) -https://faun.pub/total-nginx-monitoring-with-application-performance-and-a-bit-more-using-8fc6d731051b +sources: +- https://faun.pub/total-nginx-monitoring-with-application-performance-and-a-bit-more-using-8fc6d731051b +- https://www.influxdata.com/blog/telegraf-correlate-log-metrics-data-performance-bottlenecks/ diff --git a/docker/nginx/rootfs/etc/nginx/conf.d/default.conf b/docker/nginx/rootfs/etc/nginx/conf.d/default.conf index d1cd5e6..9b492a6 100644 --- a/docker/nginx/rootfs/etc/nginx/conf.d/default.conf +++ b/docker/nginx/rootfs/etc/nginx/conf.d/default.conf @@ -1,9 +1,3 @@ -server_names_hash_bucket_size 64; -log_format telegraf-log '$remote_addr - $remote_user [$time_local]' - '"$request" $status $body_bytes_sent' - '"$http_referer" "$http_user_agent"' - '"$request_time" "$upstream_connect_time"' - '$http_host "xxx" "yyy"'; - #'"$geoip_city" "$geoip_city_country_code"'; -access_log /var/log/nginx/access-file.log telegraf-log; +log_format vhosts ' $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent $host "$http_referer" "$http_user_agent"'; +access_log /var/log/nginx/vhosts-access.log vhosts; diff --git a/docker/telegraf/rootfs/etc/telegraf/telegraf.d/nginx.conf b/docker/telegraf/rootfs/etc/telegraf/telegraf.d/nginx.conf index f04fa51..b8f8785 100644 --- a/docker/telegraf/rootfs/etc/telegraf/telegraf.d/nginx.conf +++ b/docker/telegraf/rootfs/etc/telegraf/telegraf.d/nginx.conf @@ -1,13 +1,16 @@ [[inputs.nginx]] urls = ["http://wansrv01:8080/stub_status"] response_timeout = "5s" + + [[inputs.tail]] - files = ["/var/log/nginx/access-file.log"] + files = ["/var/log/nginx/vhosts-access.log"] from_beginning = true - name_override = "nginx_access_log" - grok_patterns = ["%{CUSTOM_LOG_FORMAT}"] - grok_custom_patterns = ''' - CUSTOM_LOG_FORMAT %{CLIENT:client_ip} %{NOTSPACE:ident} %{NOTSPACE:auth} \[%{HTTPDATE:ts:ts-httpd}\]"(?:%{WORD:verb:tag} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version:float})?|%{DATA})" %{NUMBER:resp_code:tag} (?:%{NUMBER:resp_bytes:int}|-)%{QS:referrer} %{QS:agent}%{QS:request_time} %{QS:upstream_connect_time}%{NOTSPACE:http_host:tag} %{QS:geoip_city} %{QS:country_code} - ''' + name_override = "nginx_access-logv3" + grok_patterns = ["%{CUSTOM_LOG}"] + grok_custom_patterns = ''' + CUSTOM_LOG %{COMMON_LOG_FORMAT} %{NOTSPACE:nginx_host} "%{DATA:referrer}" "%{DATA:agent}" + ''' data_format = "grok" + diff --git a/influxdb/by_nginx_client_ip.flux b/influxdb/by_nginx_client_ip.flux new file mode 100644 index 0000000..52df15c --- /dev/null +++ b/influxdb/by_nginx_client_ip.flux @@ -0,0 +1,27 @@ + +//////////////////////////////////////////////////////////////////// +v = { + "timeRangeStart": 2023-06-16T14:44:00Z, + "timeRangeStop": 2023-06-18T14:44:00Z +} + +bucket = "telegraf" + +//////////////////////////////////////////////////////////////////// +hosts = ${httpHost:json} + +hostsLength = hosts |> length() +measurement = "nginx_access-logv3" +from(bucket: "${bucket}") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => r["_measurement"] == measurement and (r["_field"] == "client_ip" or r["_field"] == "nginx_host")) + |> pivot(rowKey:["_time"], + columnKey: ["_field"], + valueColumn: "_value") + |> filter(fn: (r) => if hostsLength>0 then contains(value: r["nginx_host"], set: hosts) else true) + |> duplicate(column: "client_ip", as: "_value") + |> group(columns: ["client_ip"]) + |>count() + |> group() + |> sort(columns: ["_value"], desc: true) + \ No newline at end of file diff --git a/influxdb/by_nginx_host.flux b/influxdb/by_nginx_host.flux new file mode 100644 index 0000000..76666a3 --- /dev/null +++ b/influxdb/by_nginx_host.flux @@ -0,0 +1,26 @@ + +//////////////////////////////////////////////////////////////////// +v = { + "timeRangeStart": 2023-06-15T14:44:00Z, + "timeRangeStop": 2023-06-18T14:44:00Z +} +httpHosts = ["git.limbosolutions.com"] +bucket = "telegraf" + +//////////////////////////////////////////////////////////////////// + +measurement = "nginx_access-logv3" +from(bucket: "${bucket}") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => r["_measurement"] == measurement and r["_field"] == "nginx_host") + |> duplicate(column: "_value", as: "nginx_host") + |> group(columns: ["nginx_host"]) + |> count() + |> group() + |> sort(columns: ["_value"], desc: true) + + + + + + diff --git a/influxdb/by_nginx_request.flux b/influxdb/by_nginx_request.flux new file mode 100644 index 0000000..b1415eb --- /dev/null +++ b/influxdb/by_nginx_request.flux @@ -0,0 +1,30 @@ + +//////////////////////////////////////////////////////////////////// +v = { + "timeRangeStart": 2023-06-16T14:44:00Z, + "timeRangeStop": 2023-06-18T14:44:00Z +} + +bucket = "telegraf" + +//////////////////////////////////////////////////////////////////// +hosts = ${httpHost:json} +hostsLength = hosts |> length() +measurement = "nginx_access-logv3" +from(bucket: "${bucket}") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => r["_measurement"] == measurement and (r["_field"] == "request" or r["_field"] == "nginx_host")) + |> pivot(rowKey:["_time"], + columnKey: ["_field"], + valueColumn: "_value") + |> filter(fn: (r) => if hostsLength>0 then contains(value: r["nginx_host"], set: hosts) else true) + |> group() + |> duplicate(column: "request", as: "_value") + |> group(columns: ["request"]) + |>count() + |> group() + |> sort(columns: ["_value"], desc: true) + + + + diff --git a/influxdb/by_nginx_resp_code.flux b/influxdb/by_nginx_resp_code.flux new file mode 100644 index 0000000..3948a53 --- /dev/null +++ b/influxdb/by_nginx_resp_code.flux @@ -0,0 +1,23 @@ + +//////////////////////////////////////////////////////////////////// +v = { + "timeRangeStart": 2023-06-16T14:44:00Z, + "timeRangeStop": 2023-06-18T14:44:00Z +} + +bucket = "telegraf" + +//////////////////////////////////////////////////////////////////// +hosts = ${httpHost:json} + +hostsLength = hosts |> length() +measurement = "nginx_access-logv3" +from(bucket: "${bucket}") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => r["_measurement"] == measurement and r["_field"] == "nginx_host") + |> filter(fn: (r) => if hostsLength>0 then contains(value: r["_value"], set: hosts) else true) + |> group(columns: ["resp_code"]) + |> count() + |> group() + |> sort(columns: ["_value"], desc: true) + \ No newline at end of file diff --git a/influxdb/by_nginx_verb.flux b/influxdb/by_nginx_verb.flux new file mode 100644 index 0000000..dd8170a --- /dev/null +++ b/influxdb/by_nginx_verb.flux @@ -0,0 +1,23 @@ + +//////////////////////////////////////////////////////////////////// +v = { + "timeRangeStart": 2023-06-16T14:44:00Z, + "timeRangeStop": 2023-06-18T14:44:00Z +} + +bucket = "telegraf" + +//////////////////////////////////////////////////////////////////// +hosts = ${httpHost:json} + +hostsLength = hosts |> length() +measurement = "nginx_access-logv3" +from(bucket: "${bucket}") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => r["_measurement"] == measurement and r["_field"] == "nginx_host") + |> filter(fn: (r) => if hostsLength>0 then contains(value: r["_value"], set: hosts) else true) + |> group(columns: ["verb"]) + |> count() + |> group() + |> sort(columns: ["_value"], desc: true) + \ No newline at end of file diff --git a/influxdb/get_nginx_hosts.flux b/influxdb/get_nginx_hosts.flux new file mode 100644 index 0000000..d40ce7f --- /dev/null +++ b/influxdb/get_nginx_hosts.flux @@ -0,0 +1,17 @@ +//////////////////////////////////////////////////////////////////// +v = { + "timeRangeStart": 2023-06-15T14:44:00Z, + "timeRangeStop": 2023-06-18T14:44:00Z +} +httpHosts = ["git.limbosolutions.com"] +bucket = "telegraf" + +//////////////////////////////////////////////////////////////////// + +measurement = "nginx_access-logv3" +from(bucket: "${bucket}") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => r["_measurement"] == measurement and r["_field"] == "nginx_host") + |> keep(columns: ["_value"]) + |> distinct() + diff --git a/influxdb/testess.flux b/influxdb/testess.flux new file mode 100644 index 0000000..e165333 --- /dev/null +++ b/influxdb/testess.flux @@ -0,0 +1,19 @@ + +//////////////////////////////////////////////////////////////////// +v = { + "timeRangeStart": 2023-06-15T14:44:00Z, + "timeRangeStop": 2023-06-18T14:44:00Z +} +httpHosts = ["git.limbosolutions.com"] +bucket = "telegraf" + +//////////////////////////////////////////////////////////////////// + +measurement = "nginx_access-logv3" +from(bucket: "${bucket}") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => r["_measurement"] == measurement and r["_field"] == "nginx_host" and r._value=="frontdoor-f7fje4hfd0hpc3h9.z01.azurefd.net") + + + + 66.240.205.34 - - [18/Jun/2023:10:01:01 +0000] "H\x00\x00\x00tj\xA8\x9E#D\x98+\xCA\xF0\xA7\xBBl\xC5\x19\xD7\x8D\xB6\x18\xEDJ\x1En\xC1\xF9xu[l\xF0E\x1D-j\xEC\xD4xL\xC9r\xC9\x15\x10u\xE0%\x86Rtg\x05fv\x86]%\xCC\x80\x0C\xE8\xCF\xAE\x00\xB5\xC0f\xC8\x8DD\xC5\x09\xF4" 400 157 cloud.limbosolutions.com "-" "-" \ No newline at end of file