feat: telegraf logs and metrics

README.md

@@ -28,3 +28,9 @@ Notify Influxdb.
 ### Duplicati - data job Options
 
 
+# nginx - telegraf
+
+(no tusing prometheus must same steps)
+https://faun.pub/total-nginx-monitoring-with-application-performance-and-a-bit-more-using-8fc6d731051b
+
+

docker/docker-compose.yaml

@@ -6,6 +6,7 @@ services:
       - nginx-conf.d:/etc/nginx/conf.d
       - letsencrypt-conf:/etc/letsencrypt
       - certbot-www:/var/www/certbot
+      - nginx-log:/var/log/nginx
     ports:
       - 443:443
       - 80:80
@@ -15,11 +16,11 @@ services:
   duplicati:
     image: duplicati-mysqlclient:latest
     environment:
-      - INFLUXDB_SERVER=${INFLUXDB_SERVER}
+      - INFLUXDB_SERVER=${DUPLICATI_INFLUXDB_SERVER}
-      - INFLUXDB_BUCKET=${INFLUXDB_BUCKET}
+      - INFLUXDB_BUCKET=${DUPLICATI_INFLUXDB_BUCKET}
-      - INFLUXDB_ORG=${INFLUXDB_ORG}
+      - INFLUXDB_ORG=${DUPLICATI_INFLUXDB_ORG}
-      - INFLUXDB_TOKEN=${INFLUXDB_TOKEN}
+      - INFLUXDB_TOKEN=${DUPLICATI_INFLUXDB_TOKEN}
-      - INFLUXDB_CLIENT_HOSTNAME=${INFLUXDB_CLIENT_HOSTNAME}
+      - INFLUXDB_CLIENT_HOSTNAME=${DUPLICATI_CLIENT_HOSTNAME}
     volumes:
       - duplicati_tmpdata:/data
       - nginx-conf.d:/bck/nginx-conf.d
@@ -32,33 +33,39 @@ services:
   code-server:
     image: lscr.io/linuxserver/code-server:latest
     environment:
-      #- PUID=1000
+      - PUID=0
-      #- PGID=1000
+      - PGID=0
       - TZ=Europe/London
-      - PASSWORD=${ADMIN_PASSWORD} #optional
+      - PASSWORD=${CODE_WEB_PASSWORD}
-      #- HASHED_PASSWORD= #optional
+      #- SUDO_PASSWORD=${CODE_WEB_PASSWORD}
-      #- SUDO_PASSWORD=password #optional
-      #- SUDO_PASSWORD_HASH= #optional
       #- PROXY_DOMAIN=code-server.my.domain #optional
-      - DEFAULT_WORKSPACE=/config/workspace #optional
+      #- DEFAULT_WORKSPACE=/config/workspace #optional
     volumes:
       - code-server_config:/config
       - nginx-conf.d:/nginx/conf.d
       - letsencrypt-conf:/letsencrypt/conf
+      - telegraf-etc:/telegraf-etc
+      - nginx-log:/nginx/log
     ports:
       - 8444:8443
     restart: unless-stopped
-  prometheus-exporter:
+  telegraf:
-    image: nginx/nginx-prometheus-exporter
+    image: telegraf
-    command:
-      - -nginx.scrape-uri
-      - http://wan_reverse_proxy-nginx-1:8080/stub_status
-    ports:
-      - 9113:9113
     restart: unless-stopped
+    volumes:
+      - telegraf-etc:/etc/telegraf
+      - nginx-log:/var/log/nginx
+    environment:
+    - INFLUXDB_URL=${TELEGRAF_INFLUXDB_URL}
+    - INFLUXDB_BUCKET=${TELEGRAF_INFLUXDB_BUCKET}
+    - INFLUXDB_ORG=${TELEGRAF_INFLUXDB_ORG}
+    - INFLUXDB_TOKEN=${TELEGRAF_INFLUXDB_TOKEN}
+    - INFLUXDB_CLIENT_HOSTNAME=${TELEGRAF_CLIENT_HOSTNAME}   
 volumes:
   nginx-conf.d:
   letsencrypt-conf:
   certbot-www:
   duplicati_tmpdata:
   code-server_config:
+  telegraf-etc:
+  nginx-log:

docker/nginx/rootfs/etc/nginx/conf.d/default.conf

@@ -0,0 +1,9 @@
+server_names_hash_bucket_size  64;
+log_format telegraf-log '$remote_addr - $remote_user [$time_local]'
+                  '"$request" $status $body_bytes_sent'
+                  '"$http_referer" "$http_user_agent"'
+                  '"$request_time" "$upstream_connect_time"'
+                  '$http_host "xxx" "yyy"';
+                  #'"$geoip_city" "$geoip_city_country_code"';
+access_log  /var/log/nginx/access-file.log  telegraf-log;
+

docker/telegraf/rootfs/etc/telegraf/telegraf.d/influxdb.conf

@@ -0,0 +1,19 @@
+[[outputs.influxdb_v2]] 
+
+
+#   ## The URLs of the InfluxDB cluster nodes.
+#   ##
+#   ## Multiple URLs can be specified for a single cluster, only ONE of the
+#   ## urls will be written to each interval.
+#   ##   ex: urls = ["https://us-west-2-1.aws.cloud2.influxdata.com"]
+urls = ["${INFLUXDB_URL}"]
+#
+#   ## Token for authentication.
+token = "${INFLUXDB_TOKEN}"
+#
+#   ## Organization is the name of the organization you wish to write to.
+organization = "${INFLUXDB_ORG}"
+#
+#   ## Destination bucket to write into.
+bucket = "${INFLUXDB_BUCKET}"
+#

docker/telegraf/rootfs/etc/telegraf/telegraf.d/nginx.conf

@@ -0,0 +1,13 @@
+[[inputs.nginx]]
+   	 urls = ["http://wansrv01:8080/stub_status"]
+   	 response_timeout = "5s"
+[[inputs.tail]]
+	files = ["/var/log/nginx/access-file.log"]
+	from_beginning = true
+    name_override = "nginx_access_log"
+	grok_patterns = ["%{CUSTOM_LOG_FORMAT}"]
+  	grok_custom_patterns = '''
+        CUSTOM_LOG_FORMAT %{CLIENT:client_ip} %{NOTSPACE:ident} %{NOTSPACE:auth} \[%{HTTPDATE:ts:ts-httpd}\]"(?:%{WORD:verb:tag} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version:float})?|%{DATA})" %{NUMBER:resp_code:tag} (?:%{NUMBER:resp_bytes:int}|-)%{QS:referrer} %{QS:agent}%{QS:request_time} %{QS:upstream_connect_time}%{NOTSPACE:http_host:tag} %{QS:geoip_city} %{QS:country_code}
+      '''
+  	data_format = "grok"
+