fix: fix default journal logger integration

config/_globals.nix.example

@@ -18,6 +18,7 @@
     country_code = "eu";
     currency = "EUR";
     dns_provider = "cloudflare";
+    retention = "15d";
 
     other_hosts = [
       {

config/alloy/metrics.alloy.nix

@@ -27,15 +27,31 @@
 
     prometheus.relabel "filter_metrics" {
       rule {
-        action = "drop"
-        source_labels = [ "env" ]
-        regex = "dev"
+        source_labels = ["__name__"]
+        regex         = ".*_build_info"
+        action        = "drop"
       }
       rule {
-        action = "replace"
-        regex = "127\\.0\\.0\\.1"
-        target_label = "instance"
-        replacement = "${tools.build_ip container}"
+        source_labels = ["__name__"]
+        regex         = "go_.*"
+        action        = "drop"
+      }
+      rule {
+        source_labels = [ "env" ]
+        regex         = "dev"
+        action        = "drop"
+      }
+      rule {
+        target_label  = "host"
+        replacement   = "${tools.build_hostname container}"
+      }
+      rule {
+        target_label  = "host_ip"
+        replacement   = "${tools.build_ip container}"
+      }
+      rule {
+        target_label  = "service"
+        replacement   = "alloy"
       }
       forward_to = [prometheus.remote_write.metrics_service.receiver]
     }

config/alloy/proxy-traefik.alloy.nix

@@ -1,54 +0,0 @@
-{ config, tools, ... }:
-let
-  hostname = tools.build_hostname "proxy";
-  ip = tools.build_ip "proxy";
-in
-{
-  out = ''
-    loki.relabel "trf_journal" {
-      forward_to = []
-      rule {
-        source_labels = ["__journal__priority_keyword"]
-        target_label = "level"
-      }
-      rule {
-        source_labels = ["__journal__SYSLOG_IDENTIFIER"]
-        target_label = "app"
-      }
-    }
-
-    loki.source.journal "trf_journal_scrape" {
-      forward_to = [loki.process.trf_router.receiver]
-      matches = "_SYSTEMD_UNIT=traefik.service"
-      relabel_rules = loki.relabel.trf_journal.rules
-      labels = {
-        service = "traefik",
-        host = "${hostname}",
-        host_ip = "${ip}",
-      }
-    }
-    loki.process "trf_router" {
-      stage.regex {
-        expression = "^(?P<datetime>\\S+) (?P<level>\\w{3}) (?P<message>.*)$"
-      }
-      stage.timestamp {
-        source = "datetime"
-        format = "2006-01-02 15:04:05-07:00"
-      }
-      stage.replace {
-        source = "level"
-        expression = "INF"
-        replace = "INFO"
-      }
-      stage.labels {
-        values = {
-          level = "level",
-        }
-      }
-      stage.output {
-        source = "message"
-      }
-      forward_to = [loki.write.grafana_loki.receiver]
-    }
-  '';
-}

config/dns-adguardhome.nix

@@ -26,7 +26,8 @@ in
 
         stage.timestamp {
           source = "timestamp"
-          format = "2006-01-02 15:04:05"
+          format = "2006/01/02 15:04:05.999999"
+          location = "${config.globals.default_tz}"
         }
 
         stage.labels {

config/dns-unbound.nix

@@ -7,7 +7,7 @@ let
   mask_cidr = tools.mask_cidr; # build_ip_cidr 0 config.globals.cidr;
 in
 {
-  environment.etc."alloy/logs-adguardhome.alloy".text =
+  environment.etc."alloy/logs-unbound.alloy".text =
     (import ./alloy/default-journal-logger.alloy.nix {
       inherit tools;
       container = "dns";

config/metrics-prometheus.nix

@@ -13,6 +13,7 @@ in
     extraFlags = [
       "--web.enable-otlp-receiver"
       "--web.enable-remote-write-receiver"
+      "--storage.tsdb.retention.time=${config.globals.retention}"
     ];
     globalConfig = {
       scrape_interval = "30s";
@@ -21,7 +22,14 @@ in
       {
         job_name = "prometheus";
         static_configs = [
-          { targets = [ "localhost:9090" ]; }
+          {
+            targets = [ "localhost:9090" ];
+            labels = {
+              host = tools.build_hostname "metrics";
+              host_ip = tools.build_ip "metrics";
+              service = "prometheus";
+            };
+          }
         ];
       }
     ]
@@ -33,8 +41,13 @@ in
         in
         {
           job_name = container;
-          static_configs = map (port: {
+          static_configs = lib.mapAttrsToList (service: port: {
             targets = [ "${container_ip}:${toString port}" ];
+            labels = {
+              host = tools.build_hostname container;
+              host_ip = tools.build_ip container;
+              service = service;
+            };
           }) def.logging.prometheusPorts;
         }
       ) config.my-lxc

config/monitoring-loki.nix

@@ -29,9 +29,13 @@
           kvstore.store = "inmemory";
         };
         replication_factor = 1;
-        path_prefix = "/tmp/loki";
+        path_prefix = "/var/lib/loki";
+      };
+      storage_config.filesystem.directory = "/var/lib/loki/chunks";
+      table_manager = {
+        retention_deletes_enabled = true;
+        retention_period = config.globals.retention;
       };
-      storage_config.filesystem.directory = "/tmp/loki/chunks";
     };
   };
 }

config/proxy-traefik.nix

@@ -16,6 +16,15 @@ let
   ];
 in
 {
+  # environment.etc."alloy/logs-traefik.alloy".text =
+  #   (import ./alloy/default-journal-logger.alloy.nix {
+  #     inherit tools;
+  #     container = "proxy";
+  #     service = "traefik";
+  #
+  #     '';
+  #   }).out;
+
   services = {
     traefik = {
       enable = true;

config/yarrr-arr.nix

@@ -5,7 +5,7 @@
   ...
 }:
 {
-  environment.etc."yarrr.env".source = config.age.secrets.yarrr-env;
+  environment.etc."yarrr.env".source = config.age.secrets.yarrr-env.path;
 
   services = {
     bazarr = {

containers/db.nix

@@ -24,7 +24,9 @@
     logging = {
       enable = true;
       metricsEnable = true;
-      prometheusPorts = [ 9187 ];
+      prometheusPorts = {
+        postgres = 9187;
+      };
     };
     private = true;
     auth = true;

containers/dns.nix

@@ -24,9 +24,9 @@
     logging = {
       enable = true;
       metricsEnable = true;
-      prometheusPorts = [
-        9167
-      ];
+      prometheusPorts = {
+        unbound = 9167;
+      };
     };
     private = true;
     auth = true;

containers/proxy.nix

@@ -22,10 +22,30 @@
     logging = {
       enable = true;
       metricsEnable = true;
-      alloyConfig = {
-        # probably move to default-journal...
-        "logs-traefik" = ../config/alloy/proxy-traefik.alloy.nix;
-      };
+      prometheusPorts.traefik = 8082;
+      journalLoggers.traefik = ''
+        stage.regex {
+          expression = "^(?P<client_ip>\\S+) (?P<ident>\\S+) (?P<auth_id>\\S+) \\[(?P<timestamp>[^\\]]+)\\] \"(?P<method>\\S+) (?P<path>\\S+) HTTP/(?P<http_version>\\S+)\" (?P<status>\\d+) (?P<bytes_sent>\\d+) \"(?P<referrer>[^\"]*)\" \"(?P<user_agent>[^\"]*)\" (?P<bytes_received>\\d+) \"(?P<router>[^\"]*)\" \"(?P<upstream>[^\"]*)\" (?P<duration>\\d+)ms$"
+        }
+
+        stage.timestamp {
+          source = "timestamp"
+          format = "02/Jan/2006:15:04:05 -0700"
+        }
+
+        stage.labels {
+        values = {
+          client_ip = "",
+          ident = "",
+          auth_id = "",
+          method = "",
+          status = "",
+          referrer = "",
+          router = "",
+          upstream = "",
+          }
+        }
+      '';
     };
     private = true;
     auth = true;

containers/yarrr.nix

@@ -30,9 +30,9 @@ in
     logging = {
       enable = true;
       metricsEnable = true;
-      prometheusPorts = [
-        9708
-      ];
+      prometheusPorts = {
+        yarrr = 9708;
+      };
     };
     otherDomains = [
       {

modules/containers-nixos-logging.nix

@@ -34,15 +34,14 @@ in
     }) def.logging.alloyConfig)
     ++ (lib.mapAttrsToList (service: additional_stages: {
       "alloy/${container}-${service}.alloy".text =
-        import ../config/alloy/default-journal-logger.alloy.nix
-          {
-            inherit
-              tools
-              container
-              service
-              additional_stages
-              ;
-          };
+        (import ../config/alloy/default-journal-logger.alloy.nix {
+          inherit
+            tools
+            container
+            service
+            additional_stages
+            ;
+        }).out;
     }) def.logging.journalLoggers)
   );
 

modules/containers.nix

@@ -142,9 +142,9 @@ in
                 enable = mkEnableOption "Whether to enable default logs collection";
                 metricsEnable = mkEnableOption "Whether to enable default metrics collection";
                 prometheusPorts = mkOption {
-                  type = listOf int;
-                  description = "Ports of Prometheus Exporters";
-                  default = [ ];
+                  type = attrsOf int;
+                  description = "Service => ports of Prometheus Exporters";
+                  default = { };
                 };
                 alloyConfig = mkOption {
                   type = attrsOf path;
@@ -240,6 +240,7 @@ in
           default_tz = mkOption { type = str; };
           country_code = mkOption { type = str; };
           currency = mkOption { type = str; };
+          retention = mkOption { type = str; };
           services = mkOption {
             type = submodule {
               log_sink = mkOption { type = str; }; # ip:port