feat: a bit of cleanup & keeping 15 days of metrics/logs

config/_globals.nix.example

@@ -18,6 +18,7 @@
     country_code = "eu";
     currency = "EUR";
     dns_provider = "cloudflare";
+    retention = "15d";
 
     other_hosts = [
       {

config/alloy/proxy-traefik.alloy.nix

@@ -1,54 +0,0 @@
-{ config, tools, ... }:
-let
-  hostname = tools.build_hostname "proxy";
-  ip = tools.build_ip "proxy";
-in
-{
-  out = ''
-    loki.relabel "trf_journal" {
-      forward_to = []
-      rule {
-        source_labels = ["__journal__priority_keyword"]
-        target_label = "level"
-      }
-      rule {
-        source_labels = ["__journal__SYSLOG_IDENTIFIER"]
-        target_label = "app"
-      }
-    }
-
-    loki.source.journal "trf_journal_scrape" {
-      forward_to = [loki.process.trf_router.receiver]
-      matches = "_SYSTEMD_UNIT=traefik.service"
-      relabel_rules = loki.relabel.trf_journal.rules
-      labels = {
-        service = "traefik",
-        host = "${hostname}",
-        host_ip = "${ip}",
-      }
-    }
-    loki.process "trf_router" {
-      stage.regex {
-        expression = "^(?P<datetime>\\S+) (?P<level>\\w{3}) (?P<message>.*)$"
-      }
-      stage.timestamp {
-        source = "datetime"
-        format = "2006-01-02 15:04:05-07:00"
-      }
-      stage.replace {
-        source = "level"
-        expression = "INF"
-        replace = "INFO"
-      }
-      stage.labels {
-        values = {
-          level = "level",
-        }
-      }
-      stage.output {
-        source = "message"
-      }
-      forward_to = [loki.write.grafana_loki.receiver]
-    }
-  '';
-}

config/metrics-prometheus.nix

@@ -13,6 +13,7 @@ in
     extraFlags = [
       "--web.enable-otlp-receiver"
       "--web.enable-remote-write-receiver"
+      "--storage.tsdb.retention.time=${config.globals.retention}"
     ];
     globalConfig = {
       scrape_interval = "30s";

config/monitoring-loki.nix

@@ -29,9 +29,13 @@
           kvstore.store = "inmemory";
         };
         replication_factor = 1;
-        path_prefix = "/tmp/loki";
+        path_prefix = "/var/lib/loki";
+      };
+      storage_config.filesystem.directory = "/var/lib/loki/chunks";
+      table_manager = {
+        retention_deletes_enabled = true;
+        retention_period = config.globals.retention;
       };
-      storage_config.filesystem.directory = "/tmp/loki/chunks";
     };
   };
 }

config/proxy-traefik.nix

@@ -16,36 +16,14 @@ let
   ];
 in
 {
-  environment.etc."alloy/logs-traefik.alloy".text =
+  # environment.etc."alloy/logs-traefik.alloy".text =
-    (import ./alloy/default-journal-logger.alloy.nix {
+  #   (import ./alloy/default-journal-logger.alloy.nix {
-      inherit tools;
+  #     inherit tools;
-      container = "proxy";
+  #     container = "proxy";
-      service = "traefik";
+  #     service = "traefik";
-      additional_stages = ''
+  #
-        stage.regex {
+  #     '';
-          expression = "^(?P<client_ip>\\S+) (?P<ident>\\S+) (?P<auth_id>\\S+) \\[(?P<timestamp>[^\\]]+)\\] \"(?P<method>\\S+) (?P<path>\\S+) HTTP/(?P<http_version>\\S+)\" (?P<status>\\d+) (?P<bytes_sent>\\d+) \"(?P<referrer>[^\"]*)\" \"(?P<user_agent>[^\"]*)\" (?P<bytes_received>\\d+) \"(?P<router>[^\"]*)\" \"(?P<upstream>[^\"]*)\" (?P<duration>\\d+)ms$"
+  #   }).out;
-        }
-
-        stage.timestamp {
-          source = "timestamp"
-          format = "02/Jan/2006:15:04:05 -0700"
-        }
-
-        stage.labels {
-        values = {
-          client_ip = "",
-          ident = "",
-          auth_id = "",
-          method = "",
-          status = "",
-          referrer = "",
-          router = "",
-          upstream = "",
-          }
-        }
-
-      '';
-    }).out;
 
   services = {
     traefik = {

containers/proxy.nix

@@ -22,10 +22,30 @@
     logging = {
       enable = true;
       metricsEnable = true;
-      alloyConfig = {
+      prometheusPorts.traefik = 8082;
-        # probably move to default-journal...
+      journalLoggers.traefik = ''
-        "logs-traefik" = ../config/alloy/proxy-traefik.alloy.nix;
+        stage.regex {
-      };
+          expression = "^(?P<client_ip>\\S+) (?P<ident>\\S+) (?P<auth_id>\\S+) \\[(?P<timestamp>[^\\]]+)\\] \"(?P<method>\\S+) (?P<path>\\S+) HTTP/(?P<http_version>\\S+)\" (?P<status>\\d+) (?P<bytes_sent>\\d+) \"(?P<referrer>[^\"]*)\" \"(?P<user_agent>[^\"]*)\" (?P<bytes_received>\\d+) \"(?P<router>[^\"]*)\" \"(?P<upstream>[^\"]*)\" (?P<duration>\\d+)ms$"
+        }
+
+        stage.timestamp {
+          source = "timestamp"
+          format = "02/Jan/2006:15:04:05 -0700"
+        }
+
+        stage.labels {
+        values = {
+          client_ip = "",
+          ident = "",
+          auth_id = "",
+          method = "",
+          status = "",
+          referrer = "",
+          router = "",
+          upstream = "",
+          }
+        }
+      '';
     };
     private = true;
     auth = true;

modules/containers.nix

@@ -240,6 +240,7 @@ in
           default_tz = mkOption { type = str; };
           country_code = mkOption { type = str; };
           currency = mkOption { type = str; };
+          retention = mkOption { type = str; };
           services = mkOption {
             type = submodule {
               log_sink = mkOption { type = str; }; # ip:port