feat: cleaning up some metrics & adding labels

config/alloy/metrics.alloy.nix

@@ -27,16 +27,32 @@
 
     prometheus.relabel "filter_metrics" {
       rule {
+        source_labels = ["__name__"]
+        regex         = ".*_build_info"
         action        = "drop"
-        source_labels = [ "env" ]
-        regex = "dev"
       }
       rule {
-        action = "replace"
-        regex = "127\\.0\\.0\\.1"
-        target_label = "instance"
+        source_labels = ["__name__"]
+        regex         = "go_.*"
+        action        = "drop"
+      }
+      rule {
+        source_labels = [ "env" ]
+        regex         = "dev"
+        action        = "drop"
+      }
+      rule {
+        target_label  = "host"
+        replacement   = "${tools.build_hostname container}"
+      }
+      rule {
+        target_label  = "host_ip"
         replacement   = "${tools.build_ip container}"
       }
+      rule {
+        target_label  = "service"
+        replacement   = "alloy"
+      }
       forward_to = [prometheus.remote_write.metrics_service.receiver]
     }
 

config/metrics-prometheus.nix

@@ -21,7 +21,14 @@ in
       {
         job_name = "prometheus";
         static_configs = [
-          { targets = [ "localhost:9090" ]; }
+          {
+            targets = [ "localhost:9090" ];
+            labels = {
+              host = tools.build_hostname "metrics";
+              host_ip = tools.build_ip "metrics";
+              service = "prometheus";
+            };
+          }
         ];
       }
     ]
@@ -33,8 +40,13 @@ in
         in
         {
           job_name = container;
-          static_configs = map (port: {
+          static_configs = lib.mapAttrsToList (service: port: {
             targets = [ "${container_ip}:${toString port}" ];
+            labels = {
+              host = tools.build_hostname container;
+              host_ip = tools.build_ip container;
+              service = service;
+            };
           }) def.logging.prometheusPorts;
         }
       ) config.my-lxc

config/proxy-traefik.nix

@@ -16,6 +16,37 @@ let
   ];
 in
 {
+  environment.etc."alloy/logs-traefik.alloy".text =
+    (import ./alloy/default-journal-logger.alloy.nix {
+      inherit tools;
+      container = "proxy";
+      service = "traefik";
+      additional_stages = ''
+        stage.regex {
+          expression = "^(?P<client_ip>\\S+) (?P<ident>\\S+) (?P<auth_id>\\S+) \\[(?P<timestamp>[^\\]]+)\\] \"(?P<method>\\S+) (?P<path>\\S+) HTTP/(?P<http_version>\\S+)\" (?P<status>\\d+) (?P<bytes_sent>\\d+) \"(?P<referrer>[^\"]*)\" \"(?P<user_agent>[^\"]*)\" (?P<bytes_received>\\d+) \"(?P<router>[^\"]*)\" \"(?P<upstream>[^\"]*)\" (?P<duration>\\d+)ms$"
+        }
+
+        stage.timestamp {
+          source = "timestamp"
+          format = "02/Jan/2006:15:04:05 -0700"
+        }
+
+        stage.labels {
+        values = {
+          client_ip = "",
+          ident = "",
+          auth_id = "",
+          method = "",
+          status = "",
+          referrer = "",
+          router = "",
+          upstream = "",
+          }
+        }
+
+      '';
+    }).out;
+
   services = {
     traefik = {
       enable = true;

config/yarrr-arr.nix

@@ -5,7 +5,7 @@
   ...
 }:
 {
-  environment.etc."yarrr.env".source = config.age.secrets.yarrr-env;
+  environment.etc."yarrr.env".source = config.age.secrets.yarrr-env.path;
 
   services = {
     bazarr = {

containers/db.nix

@@ -24,7 +24,9 @@
     logging = {
       enable = true;
       metricsEnable = true;
-      prometheusPorts = [ 9187 ];
+      prometheusPorts = {
+        postgres = 9187;
+      };
     };
     private = true;
     auth = true;

containers/dns.nix

@@ -24,9 +24,9 @@
     logging = {
       enable = true;
       metricsEnable = true;
-      prometheusPorts = [
-        9167
-      ];
+      prometheusPorts = {
+        unbound = 9167;
+      };
     };
     private = true;
     auth = true;

containers/yarrr.nix

@@ -30,9 +30,9 @@ in
     logging = {
       enable = true;
       metricsEnable = true;
-      prometheusPorts = [
-        9708
-      ];
+      prometheusPorts = {
+        yarrr = 9708;
+      };
     };
     otherDomains = [
       {

modules/containers.nix

@@ -142,9 +142,9 @@ in
                 enable = mkEnableOption "Whether to enable default logs collection";
                 metricsEnable = mkEnableOption "Whether to enable default metrics collection";
                 prometheusPorts = mkOption {
-                  type = listOf int;
-                  description = "Ports of Prometheus Exporters";
-                  default = [ ];
+                  type = attrsOf int;
+                  description = "Service => ports of Prometheus Exporters";
+                  default = { };
                 };
                 alloyConfig = mkOption {
                   type = attrsOf path;