From f64ee1322f5e8a8e1d6008768db3fe4736460ac1 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Sun, 17 May 2026 17:55:00 +0200
Subject: [PATCH 01/54] overleaf: init

---
 constants.nix                                 |   5 +
 hosts/rx4/services/default.nix                |   1 +
 hosts/rx4/services/overleaf-oci.nix           |  18 ++
 hosts/sid/services/nginx.nix                  |   5 +
 modules/nixos/default.nix                     |   1 +
 modules/nixos/overleaf-oci/default.nix        | 257 ++++++++++++++++++
 .../overleaf-oci/mongodb-init-replica-set.js  |   3 +
 7 files changed, 290 insertions(+)
 create mode 100644 hosts/rx4/services/overleaf-oci.nix
 create mode 100644 modules/nixos/overleaf-oci/default.nix
 create mode 100644 modules/nixos/overleaf-oci/mongodb-init-replica-set.js

diff --git a/constants.nix b/constants.nix
index 3dc7ded..5dfa26e 100644
--- a/constants.nix
+++ b/constants.nix
@@ -37,6 +37,11 @@ rec {
       fqdn = "ai." + domain;
       port = 8083;
     };
+    overleaf-oci = rec {
+      subdomain = "of";
+      fqdn = subdomain + "." + domain;
+      port = 8081;
+    };
     rss-bridge = rec {
       subdomain = "rss-bridge";
       fqdn = subdomain + "." + domain;
diff --git a/hosts/rx4/services/default.nix b/hosts/rx4/services/default.nix
index a61584e..df12843 100644
--- a/hosts/rx4/services/default.nix
+++ b/hosts/rx4/services/default.nix
@@ -16,6 +16,7 @@
     ./miniflux.nix
     ./netdata.nix
     ./nginx.nix
+    ./overleaf-oci.nix
     ./open-webui-oci.nix
     ./print-server.nix
     ./rsshub-oci.nix
diff --git a/hosts/rx4/services/overleaf-oci.nix b/hosts/rx4/services/overleaf-oci.nix
new file mode 100644
index 0000000..575012f
--- /dev/null
+++ b/hosts/rx4/services/overleaf-oci.nix
@@ -0,0 +1,18 @@
+{ outputs, constants, ... }:
+
+let
+  inherit (constants.services.overleaf-oci) port subdomain;
+in
+{
+  imports = [ outputs.nixosModules.overleaf-oci ];
+
+  services.overleaf-oci = {
+    enable = true;
+    inherit port;
+    reverseProxy = {
+      enable = true;
+      inherit subdomain;
+      forceSSL = false;
+    };
+  };
+}
diff --git a/hosts/sid/services/nginx.nix b/hosts/sid/services/nginx.nix
index d1e6227..1c2f432 100644
--- a/hosts/sid/services/nginx.nix
+++ b/hosts/sid/services/nginx.nix
@@ -77,6 +77,11 @@ in
         error_log /var/log/nginx/open-webui-error.log debug;
       '';
     };
+    virtualHosts."${constants.services.overleaf-oci.fqdn}" = mkVirtualHost {
+      inherit ssl;
+      address = constants.hosts.rx4.ip;
+      port = constants.services.overleaf-oci.port;
+    };
     virtualHosts."${constants.services.rsshub-oci.fqdn}" = mkVirtualHost {
       inherit ssl;
       address = constants.hosts.rx4.ip;
diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix
index 540f4ee..1028fdf 100644
--- a/modules/nixos/default.nix
+++ b/modules/nixos/default.nix
@@ -6,6 +6,7 @@
   forgejo-runner = import ./forgejo-runner;
   gnome = import ./gnome;
   monero = import ./monero;
+  overleaf-oci = import ./overleaf-oci;
   pki = import ./pki;
   rsshub-oci = import ./rsshub-oci;
   tailscale = import ./tailscale;
diff --git a/modules/nixos/overleaf-oci/default.nix b/modules/nixos/overleaf-oci/default.nix
new file mode 100644
index 0000000..6be248b
--- /dev/null
+++ b/modules/nixos/overleaf-oci/default.nix
@@ -0,0 +1,257 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.overleaf-oci;
+  domain = config.networking.domain;
+  subdomain = cfg.reverseProxy.subdomain;
+  fqdn = if (cfg.reverseProxy.enable && subdomain != "") then "${subdomain}.${domain}" else domain;
+
+  mongodbInitReplicaSet = ./mongodb-init-replica-set.js;
+
+  images = {
+    # https://hub.docker.com/r/sharelatex/sharelatex
+    sharelatex = pkgs.dockerTools.pullImage {
+      imageName = "sharelatex/sharelatex";
+      imageDigest = "sha256:fa5e5a049721b0973aa6ddabd15cb04da2d492f38c685fda0805dab97d4f3c8a";
+      hash = "sha256-2Om6b+xPcBFObMKAlZ+qsvPw1xd8B/8ODyI5dsWoaQA=";
+      finalImageName = "sharelatex/sharelatex";
+      finalImageTag = "6.1.2";
+    };
+    # https://hub.docker.com/_/mongo
+    mongo = pkgs.dockerTools.pullImage {
+      imageName = "mongo";
+      imageDigest = "sha256:ce32a4b67580c982e2020b07d4ffafdbd9ce474088434294d89c6bd9257f7788";
+      hash = "sha256-NtMSeH5hYD69uF7gZWPppxoI9esyPTfxaw50yh0ChpY=";
+      finalImageName = "mongo";
+      finalImageTag = "8.3.1";
+    };
+    # https://hub.docker.com/_/redis
+    redis = pkgs.dockerTools.pullImage {
+      imageName = "redis";
+      imageDigest = "sha256:0c341492924cad6f5483f9133e43bd6c51ecdecbcadfac5b51657393b6a7936c";
+      hash = "sha256-nU5+UdvA+29sbr7dAD2jpKYACxYb3BfStlOE/25ET+w=";
+      finalImageName = "redis";
+      finalImageTag = "8.6.3";
+    };
+  };
+
+  defaultEnv = {
+    EMAIL_CONFIRMATION_DISABLED = "true";
+    ENABLED_LINKED_FILE_TYPES = "project_file,project_output_file";
+    ENABLE_CONVERSIONS = "true";
+    OVERLEAF_APP_NAME = "Overleaf Community Edition";
+    OVERLEAF_MONGO_URL = "mongodb://mongo/sharelatex";
+    OVERLEAF_REDIS_HOST = "redis";
+    REDIS_HOST = "redis";
+  };
+
+  inherit (lib)
+    mkEnableOption
+    mkIf
+    mkOption
+    mkOverride
+    optional
+    types
+    ;
+  inherit (lib.utils)
+    mkReverseProxyOption
+    mkVirtualHost
+    ;
+in
+{
+  options.services.overleaf-oci = {
+    enable = mkEnableOption "Overleaf service stack with Podman";
+    port = mkOption {
+      type = types.port;
+      default = 80;
+      description = "The port Overleaf will listen on.";
+    };
+    environment = mkOption {
+      type = types.attrsOf types.str;
+      default = { };
+      description = "Extra environment variables for Overleaf.";
+    };
+    environmentFile = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      description = "Environment file for secrets.";
+    };
+    reverseProxy = mkReverseProxyOption "Overleaf" "overleaf";
+  };
+
+  config = mkIf cfg.enable {
+    virtualisation.podman = {
+      enable = true;
+      autoPrune.enable = true;
+      dockerCompat = true;
+    };
+
+    services.nginx.virtualHosts = mkIf cfg.reverseProxy.enable {
+      "${fqdn}" = mkVirtualHost {
+        inherit (cfg) port;
+        ssl = cfg.reverseProxy.forceSSL;
+      };
+    };
+
+    networking.firewall.interfaces =
+      let
+        matchAll = if !config.networking.nftables.enable then "podman+" else "podman*";
+      in
+      {
+        "${matchAll}".allowedUDPPorts = [ 53 ];
+      };
+
+    virtualisation.oci-containers.backend = "podman";
+
+    virtualisation.oci-containers.containers = {
+      overleaf-mongo = {
+        image = with images.mongo; imageName + ":" + imageTag;
+        imageFile = images.mongo;
+        environment = {
+          MONGO_INITDB_DATABASE = "sharelatex";
+        };
+        volumes = [
+          "overleaf_mongo-data:/data/db:rw"
+          "${mongodbInitReplicaSet}:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js:rw"
+        ];
+        cmd = [
+          "--replSet"
+          "overleaf"
+        ];
+        log-driver = "journald";
+        extraOptions = [
+          "--network-alias=mongo"
+          "--network=overleaf_default"
+          "--add-host=mongo:127.0.0.1"
+          "--health-cmd=echo 'db.stats().ok' | mongosh localhost:27017/test --quiet"
+          "--health-interval=10s"
+          "--health-retries=5"
+          "--health-timeout=10s"
+        ];
+      };
+
+      overleaf-redis = {
+        image = with images.redis; imageName + ":" + imageTag;
+        imageFile = images.redis;
+        volumes = [ "overleaf_redis-data:/data:rw" ];
+        log-driver = "journald";
+        extraOptions = [
+          "--network-alias=redis"
+          "--network=overleaf_default"
+        ];
+      };
+
+      overleaf-sharelatex = {
+        image = with images.sharelatex; imageName + ":" + imageTag;
+        imageFile = images.sharelatex;
+        log-driver = "journald";
+        environment = defaultEnv // cfg.environment;
+        environmentFiles = optional (cfg.environmentFile != null) cfg.environmentFile;
+        volumes = [ "overleaf_data:/var/lib/overleaf:rw" ];
+        ports = [ "${toString cfg.port}:80/tcp" ];
+        extraOptions = [
+          "--network-alias=sharelatex"
+          "--network=overleaf_default"
+        ];
+      };
+    };
+
+    systemd.services =
+      let
+        commonServiceCfg = {
+          serviceConfig.Restart = mkOverride 90 "always";
+          partOf = [ "podman-compose-overleaf-root.target" ];
+          wantedBy = [ "podman-compose-overleaf-root.target" ];
+        };
+      in
+      {
+        podman-overleaf-mongo = commonServiceCfg // {
+          after = [
+            "podman-network-overleaf_default.service"
+            "podman-volume-overleaf_mongo-data.service"
+          ];
+          requires = [
+            "podman-network-overleaf_default.service"
+            "podman-volume-overleaf_mongo-data.service"
+          ];
+        };
+
+        podman-overleaf-redis = commonServiceCfg // {
+          after = [
+            "podman-network-overleaf_default.service"
+            "podman-volume-overleaf_redis-data.service"
+          ];
+          requires = [
+            "podman-network-overleaf_default.service"
+            "podman-volume-overleaf_redis-data.service"
+          ];
+        };
+
+        podman-overleaf-sharelatex = commonServiceCfg // {
+          after = [
+            "podman-network-overleaf_default.service"
+            "podman-overleaf-mongo.service"
+            "podman-overleaf-redis.service"
+          ];
+          requires = [
+            "podman-network-overleaf_default.service"
+            "podman-overleaf-mongo.service"
+            "podman-overleaf-redis.service"
+          ];
+          serviceConfig.ExecStartPre = [
+            "${pkgs.bash}/bin/bash -c 'until ${pkgs.podman}/bin/podman healthcheck run overleaf-mongo; do sleep 2; done'"
+          ];
+        };
+
+        podman-network-overleaf_default = {
+          path = [ pkgs.podman ];
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+            ExecStop = "podman network rm -f overleaf_default";
+          };
+          script = ''
+            podman network inspect overleaf_default || podman network create overleaf_default
+          '';
+          partOf = [ "podman-compose-overleaf-root.target" ];
+          wantedBy = [ "podman-compose-overleaf-root.target" ];
+        };
+
+        podman-volume-overleaf_redis-data = {
+          path = [ pkgs.podman ];
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+          };
+          script = ''
+            podman volume inspect overleaf_redis-data || podman volume create overleaf_redis-data
+          '';
+          partOf = [ "podman-compose-overleaf-root.target" ];
+          wantedBy = [ "podman-compose-overleaf-root.target" ];
+        };
+
+        podman-volume-overleaf_mongo-data = {
+          path = [ pkgs.podman ];
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+          };
+          script = ''
+            podman volume inspect overleaf_mongo-data || podman volume create overleaf_mongo-data
+          '';
+          partOf = [ "podman-compose-overleaf-root.target" ];
+          wantedBy = [ "podman-compose-overleaf-root.target" ];
+        };
+      };
+
+    systemd.targets."podman-compose-overleaf-root" = {
+      unitConfig.Description = "Root target for Overleaf services.";
+      wantedBy = [ "multi-user.target" ];
+    };
+  };
+}
diff --git a/modules/nixos/overleaf-oci/mongodb-init-replica-set.js b/modules/nixos/overleaf-oci/mongodb-init-replica-set.js
new file mode 100644
index 0000000..30af660
--- /dev/null
+++ b/modules/nixos/overleaf-oci/mongodb-init-replica-set.js
@@ -0,0 +1,3 @@
+/* eslint-disable no-undef */
+
+rs.initiate({ _id: 'overleaf', members: [{ _id: 0, host: 'mongo:27017' }] })

From 1bb2b7c194aa5ec35bce360849b5725a9edd3eaf Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Sun, 17 May 2026 18:54:10 +0200
Subject: [PATCH 02/54] replace webdav with samba

---
 constants.nix                  |  4 --
 hosts/rx4/secrets/secrets.yaml |  7 +--
 hosts/rx4/services/default.nix |  2 +-
 hosts/rx4/services/samba.nix   | 27 +++++++++++
 hosts/rx4/services/webdav.nix  | 86 ----------------------------------
 hosts/sid/services/coredns.nix |  1 -
 hosts/sid/services/nginx.nix   |  8 ----
 hosts/sid/services/step-ca.nix |  1 -
 8 files changed, 30 insertions(+), 106 deletions(-)
 create mode 100644 hosts/rx4/services/samba.nix
 delete mode 100644 hosts/rx4/services/webdav.nix

diff --git a/constants.nix b/constants.nix
index 3dc7ded..d49999c 100644
--- a/constants.nix
+++ b/constants.nix
@@ -49,9 +49,5 @@ rec {
       fqdn = "pw." + intranet;
       port = 8222;
     };
-    webdav = {
-      fqdn = "dav." + intranet;
-      port = 8080;
-    };
   };
 }
diff --git a/hosts/rx4/secrets/secrets.yaml b/hosts/rx4/secrets/secrets.yaml
index 0aa47f7..a591f81 100644
--- a/hosts/rx4/secrets/secrets.yaml
+++ b/hosts/rx4/secrets/secrets.yaml
@@ -13,9 +13,6 @@ syncthing:
     gui-pw: ENC[AES256_GCM,data:mN4rxYr5DZgvbpIkwSFIuPvviJE=,iv:Kyl3mZFOejVwEwBCKteJQpgbCosREp9C4T4JYhWz6KQ=,tag:6myk9lr/44CH/hyUPgRH0Q==,type:str]
 forgejo-runner:
     token: ENC[AES256_GCM,data:DZgi6ocpV0MplgQ6Et85vHxmkMfC4qYbLLdyRuj/4z8tJauz1w6DUQ==,iv:+SZYsv6sDn2Nc1WxhTn0dJGN9nXYZw16/HVtXJGXpHc=,tag:8Oa5mC7cUy85+lXHbRcCcg==,type:str]
-webdav:
-    user: ENC[AES256_GCM,data:vCLx,iv:Nra/FprNfd02HpvqOb5uYK+IGRFHhNwnFXWrX71c0C0=,tag:TjbKKOKBTq31o/5MxmqIsA==,type:str]
-    pass: ENC[AES256_GCM,data:jfIoob6R6OhqKa2EujRzTQbvIlA=,iv:HvB088H2Z2uLCveT4YfNEdkK5VU0lBFD5FrZhx79fg0=,tag:1RnrfeUEURx0C575GTxi9A==,type:str]
 vaultwarden:
     admin-token: ENC[AES256_GCM,data:HhD0xNZ/Ep7pCOX1j6p/M/ZZ3gs=,iv:7QT71KlYz+HQYBhiRavpiXS9sNS2PoJiM/WkxM3Hk/g=,tag:SYTRWpyA2+WMSMiRM8mvew==,type:str]
     smtp-password: ENC[AES256_GCM,data:eQo7op5+74EID6689hL0/J1pq2s=,iv:JqrEqxabWGydRuJJ/27e1q+4YnQhTQ1bKRSsOvjQ+bE=,tag:weqnrhqK+LGEfAacBcuPUA==,type:str]
@@ -39,7 +36,7 @@ sops:
             NE5yK3ZaOG5PdXNSUnlIUmFSSmRFancKk57hCmo79HvI3hzzgQvgOK7oK5/dcQR8
             f3R4OGF5+212VXEHR/hAEbKzV7CY4y6HhFyrGZ9bUKm1RrxtnVqUyA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-05-02T17:10:11Z"
-    mac: ENC[AES256_GCM,data:uf5TqZaevyUUjW6pM6K8c4CZFFdwTXFGIaHmYr5Q4XFR1uW3kBsVLeQKxq26duLuQ4UiZkUpW27a/PW797Z+iIpBdqbnoQ35q7RnOW+GpnAv8TaRW1PpqQ+JR3/R0LMXsi3cMt7ioG2ad1bIHztiNz+SmePiv3Yt9WxQ7PIqBdY=,iv:dAzuyKSo0OW+j02AH0chCdLBm7Wv6PZgqZrEWhEVnxQ=,tag:k6EKWHHY4fwTd03d4TVcNg==,type:str]
+    lastmodified: "2026-05-17T16:35:00Z"
+    mac: ENC[AES256_GCM,data:U2WT4ENx8I9sr3byj7fQjv3H+mQTlhTI1HL9tufryKcUGjvb35ChwkIBcvEiYLa8udOR631sWwN4dCqZ4qwtCQ3MNjR8s1P6HqhzXeAPwyxfMLPZG1mbKXvYpamkxAOq8RxVHnVsPbrvFsxc57J11SI5IUfWT5T5GPQyJ+U8gMs=,iv:/xDaNV0fgKf9z+sql4BwwyIO/LQhRm3TrMhgaYZsPuE=,tag:Y0bfT1ZuiJ05F/+EwyzbSg==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.12.1
diff --git a/hosts/rx4/services/default.nix b/hosts/rx4/services/default.nix
index a61584e..6cb2dc6 100644
--- a/hosts/rx4/services/default.nix
+++ b/hosts/rx4/services/default.nix
@@ -19,10 +19,10 @@
     ./open-webui-oci.nix
     ./print-server.nix
     ./rsshub-oci.nix
+    ./samba.nix
     ./vaultwarden.nix
 
     # ./alditalk-extender.nix # FIXME
-    # ./webdav.nix # FIXME
   ];
 
   # bootstrap
diff --git a/hosts/rx4/services/samba.nix b/hosts/rx4/services/samba.nix
new file mode 100644
index 0000000..2696005
--- /dev/null
+++ b/hosts/rx4/services/samba.nix
@@ -0,0 +1,27 @@
+{ config, ... }:
+
+{
+  services.samba = {
+    enable = true;
+    openFirewall = false;
+    nmbd.enable = false;
+    winbindd.enable = false;
+    settings = {
+      global = {
+        workgroup = "WORKGROUP";
+        "server string" = config.networking.hostName;
+        security = "user";
+        "map to guest" = "Bad User";
+        "guest account" = "nobody";
+      };
+      share = {
+        path = "/home/sid";
+        browseable = "yes";
+        "read only" = "yes";
+        "guest ok" = "yes";
+        "force user" = "sid";
+        "directory mask" = "0750";
+      };
+    };
+  };
+}
diff --git a/hosts/rx4/services/webdav.nix b/hosts/rx4/services/webdav.nix
deleted file mode 100644
index 46d01a9..0000000
--- a/hosts/rx4/services/webdav.nix
+++ /dev/null
@@ -1,86 +0,0 @@
-{ constants, config, ... }:
-
-# FIXME: floccus throws error: NetworkError when attempting to fetch resource.
-
-let
-  cfg = config.services.webdav;
-
-  inherit (constants.services.webdav) fqdn port;
-in
-{
-  services.webdav = {
-    enable = true;
-    environmentFile = config.sops.templates."webdav/env-file".path;
-
-    settings = {
-      inherit port;
-      address = "127.0.0.1";
-      prefix = "/";
-      directory = "/srv/webdav";
-      users = [
-        {
-          username = "{env}WEBDAV_USER";
-          password = "{env}WEBDAV_PASS";
-          permissions = "CRUD";
-        }
-      ];
-    };
-  };
-
-  systemd.tmpfiles.rules = [
-    "d ${cfg.settings.directory} 0750 ${cfg.user} ${cfg.group} -"
-  ];
-
-  networking.firewall.allowedTCPPorts = [ port ];
-
-  services.nginx = {
-    enable = true;
-    virtualHosts."${fqdn}" = {
-      listen = [
-        {
-          addr = "0.0.0.0";
-          inherit port;
-        }
-      ];
-      locations."/" = {
-        proxyPass = "http://127.0.0.1:${toString port}";
-        extraConfig = ''
-          add_header 'Access-Control-Allow-Origin' '*' always;
-          add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PROPFIND, OPTIONS' always;
-          add_header 'Access-Control-Allow-Headers' 'Authorization,DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Depth' always;
-
-          if ($request_method = 'OPTIONS') {
-              add_header 'Access-Control-Allow-Origin' '*';
-              add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PROPFIND, OPTIONS';
-              add_header 'Access-Control-Allow-Headers' 'Authorization,DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Depth';
-              return 204;
-          }
-        '';
-      };
-    };
-  };
-
-  sops =
-    let
-      owner = cfg.user;
-      group = cfg.group;
-      mode = "0400";
-    in
-    {
-      secrets = {
-        "webdav/user" = {
-          inherit owner group mode;
-        };
-        "webdav/pass" = {
-          inherit owner group mode;
-        };
-      };
-      templates."webdav/env-file" = {
-        inherit owner group mode;
-        content = ''
-          WEBDAV_USER=${config.sops.placeholder."webdav/user"}
-          WEBDAV_PASS=${config.sops.placeholder."webdav/pass"}
-        '';
-      };
-    };
-}
diff --git a/hosts/sid/services/coredns.nix b/hosts/sid/services/coredns.nix
index c7af795..13c3096 100644
--- a/hosts/sid/services/coredns.nix
+++ b/hosts/sid/services/coredns.nix
@@ -20,7 +20,6 @@
 
           ${hosts.sid.ip} ${services.netdata.fqdn}
           ${hosts.sid.ip} ${services.vaultwarden.fqdn}
-          ${hosts.sid.ip} ${services.webdav.fqdn}
 
           fallthrough
         }
diff --git a/hosts/sid/services/nginx.nix b/hosts/sid/services/nginx.nix
index d1e6227..81eace3 100644
--- a/hosts/sid/services/nginx.nix
+++ b/hosts/sid/services/nginx.nix
@@ -91,14 +91,6 @@ in
         };
       };
     };
-    virtualHosts."${constants.services.webdav.fqdn}" = {
-      useACMEHost = "sid-internal";
-      forceSSL = ssl;
-      locations."/" = {
-        proxyPass = "http://${constants.hosts.rx4.ip}:${toString constants.services.webdav.port}";
-        proxyWebsockets = true;
-      };
-    };
     # FIXME
     # virtualHosts."print.sid.ovh" = {
     #   enableACME = true;
diff --git a/hosts/sid/services/step-ca.nix b/hosts/sid/services/step-ca.nix
index d3abb11..21d04d4 100644
--- a/hosts/sid/services/step-ca.nix
+++ b/hosts/sid/services/step-ca.nix
@@ -82,7 +82,6 @@ in
       extraDomainNames = [
         constants.services.netdata.fqdn
         # constants.services.vaultwarden.fqdn
-        constants.services.webdav.fqdn
       ];
       server = "https://${constants.ca-fqdn}:${toString cfg.port}/acme/acme/directory";
       group = "nginx";

From 41ce9b892b6e81dfaaaa0a717b3890df50d02cb4 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Sun, 17 May 2026 22:44:00 +0200
Subject: [PATCH 03/54] rm coredns and step-ca. use hs extra dns records.
 resolve vaultwarden

---
 constants.nix                       |  2 +-
 hosts/rx4/secrets/secrets.yaml      |  5 +--
 hosts/rx4/services/vaultwarden.nix  | 47 ++++++++++++++++++++++++++++-
 hosts/sid/services/coredns.nix      |  5 ---
 hosts/sid/services/default.nix      |  5 +--
 hosts/sid/services/headscale.nix    | 13 ++++++++
 hosts/sid/services/mailserver.nix   |  8 +++--
 hosts/sid/services/nginx.nix        | 18 -----------
 hosts/sid/services/step-ca.nix      |  4 +--
 modules/nixos/tailscale/default.nix |  2 +-
 10 files changed, 73 insertions(+), 36 deletions(-)

diff --git a/constants.nix b/constants.nix
index d49999c..7a2a16e 100644
--- a/constants.nix
+++ b/constants.nix
@@ -46,7 +46,7 @@ rec {
       port = 1200;
     };
     vaultwarden = {
-      fqdn = "pw." + intranet;
+      fqdn = "pw." + domain;
       port = 8222;
     };
   };
diff --git a/hosts/rx4/secrets/secrets.yaml b/hosts/rx4/secrets/secrets.yaml
index a591f81..eaa951b 100644
--- a/hosts/rx4/secrets/secrets.yaml
+++ b/hosts/rx4/secrets/secrets.yaml
@@ -16,6 +16,7 @@ forgejo-runner:
 vaultwarden:
     admin-token: ENC[AES256_GCM,data:HhD0xNZ/Ep7pCOX1j6p/M/ZZ3gs=,iv:7QT71KlYz+HQYBhiRavpiXS9sNS2PoJiM/WkxM3Hk/g=,tag:SYTRWpyA2+WMSMiRM8mvew==,type:str]
     smtp-password: ENC[AES256_GCM,data:eQo7op5+74EID6689hL0/J1pq2s=,iv:JqrEqxabWGydRuJJ/27e1q+4YnQhTQ1bKRSsOvjQ+bE=,tag:weqnrhqK+LGEfAacBcuPUA==,type:str]
+hetzner-api-key: ENC[AES256_GCM,data:casjNOXzuQDWgnSFftbBMygA8kGpGiZDqup08faWO9kfjvgOyWOXeqPd2VA1ND8yfM2LvoLYvPs6gUWtni2ldQ==,iv:p2W24uhJgBvpi3g4+cHw0/XbbTM5oYCPHreMBUR4CNs=,tag:lpwjZGoJe/91+CHX/hAkKA==,type:str]
 sops:
     age:
         - recipient: age19yeqvv28fgrtk6jsh3xyaf0lch86kna6rcz4dwe962yyyyevu30sx474xy
@@ -36,7 +37,7 @@ sops:
             NE5yK3ZaOG5PdXNSUnlIUmFSSmRFancKk57hCmo79HvI3hzzgQvgOK7oK5/dcQR8
             f3R4OGF5+212VXEHR/hAEbKzV7CY4y6HhFyrGZ9bUKm1RrxtnVqUyA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-05-17T16:35:00Z"
-    mac: ENC[AES256_GCM,data:U2WT4ENx8I9sr3byj7fQjv3H+mQTlhTI1HL9tufryKcUGjvb35ChwkIBcvEiYLa8udOR631sWwN4dCqZ4qwtCQ3MNjR8s1P6HqhzXeAPwyxfMLPZG1mbKXvYpamkxAOq8RxVHnVsPbrvFsxc57J11SI5IUfWT5T5GPQyJ+U8gMs=,iv:/xDaNV0fgKf9z+sql4BwwyIO/LQhRm3TrMhgaYZsPuE=,tag:Y0bfT1ZuiJ05F/+EwyzbSg==,type:str]
+    lastmodified: "2026-05-17T20:34:39Z"
+    mac: ENC[AES256_GCM,data:lSSotIfDcS6oJpSDSe2hLx1M9L8a+bjkPstcPv1h2ohSiOu8WGAwTy4lsKD1n9rnhTzFmMqi2Xgh4K0n3WiqWFBeNcA6UeM7+a6PcDtUeCC3JKsP/XZvCoPq5uBwUWcovRSm4UElaL5MteZkV3e+qZWeUpZCTWWWEjYBYnHPLpQ=,iv:t4Up4DuTuQyQQNa7lmZK6kt5O0/aShXSF2XBj9Y6/z8=,tag:oNmP8e7jEZ3ttPkwXkWSZw==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.12.1
diff --git a/hosts/rx4/services/vaultwarden.nix b/hosts/rx4/services/vaultwarden.nix
index 6f00505..eed5712 100644
--- a/hosts/rx4/services/vaultwarden.nix
+++ b/hosts/rx4/services/vaultwarden.nix
@@ -6,6 +6,7 @@
 
 let
   inherit (constants) domain;
+  inherit (constants.hosts.rx4) ip;
   inherit (constants.services.vaultwarden) fqdn port;
 in
 {
@@ -21,6 +22,7 @@ in
     environmentFile = [ config.sops.templates."vaultwarden/env-file".path ];
 
     config = {
+      ENABLE_WEBSOCKET = true;
       SIGNUPS_ALLOWED = false;
 
       SMTP_FROM = "vaultwarden@${domain}";
@@ -30,12 +32,50 @@ in
       SMTP_SECURITY = "starttls";
       SMTP_USERNAME = "vaultwarden@${domain}";
 
-      ROCKET_ADDRESS = "0.0.0.0";
+      ROCKET_ADDRESS = "127.0.0.1";
       ROCKET_PORT = port;
       ROCKET_LOG = "critical";
     };
   };
 
+  services.nginx.virtualHosts."${fqdn}" = {
+    useACMEHost = "pw-custom";
+    forceSSL = true;
+    listen = [
+      {
+        addr = "${ip}:443";
+        ssl = true;
+      }
+    ];
+    locations = {
+      "/" = {
+        proxyPass = "http://127.0.0.1:${toString port}";
+      };
+      "= /notifications/alerts" = {
+        proxyPass = "http://127.0.0.1:${toString port}";
+        proxyWebsockets = true;
+      };
+      "= /notifications/hub" = {
+        proxyPass = "http://127.0.0.1:${toString port}";
+        proxyWebsockets = true;
+      };
+    };
+  };
+
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "admin@${domain}";
+    certs."pw-custom" = {
+      domain = fqdn;
+      dnsProvider = "hetzner";
+      dnsResolver = "1.1.1.1:53";
+      credentialFiles = {
+        HETZNER_API_TOKEN_FILE = config.sops.secrets.hetzner-api-key.path;
+      };
+      group = "nginx";
+    };
+  };
+
   sops =
     let
       owner = config.users.users.vaultwarden.name;
@@ -50,6 +90,11 @@ in
         "vaultwarden/smtp-password" = {
           inherit owner group mode;
         };
+        hetzner-api-key = {
+          inherit mode;
+          owner = "acme";
+          group = "acme";
+        };
       };
       templates = {
         "vaultwarden/env-file" = {
diff --git a/hosts/sid/services/coredns.nix b/hosts/sid/services/coredns.nix
index 13c3096..b7e2f37 100644
--- a/hosts/sid/services/coredns.nix
+++ b/hosts/sid/services/coredns.nix
@@ -14,12 +14,7 @@
         hosts {
           ${hosts.sid.ip} ${ca-fqdn} 
 
-          ${hosts.rx4.ip} rx4.tail
-          ${hosts.sid.ip} sid.tail
-          ${hosts.vde.ip} vde.tail
-
           ${hosts.sid.ip} ${services.netdata.fqdn}
-          ${hosts.sid.ip} ${services.vaultwarden.fqdn}
 
           fallthrough
         }
diff --git a/hosts/sid/services/default.nix b/hosts/sid/services/default.nix
index c1079d8..9baf6fb 100644
--- a/hosts/sid/services/default.nix
+++ b/hosts/sid/services/default.nix
@@ -10,7 +10,6 @@
 
     outputs.nixosModules.tailscale
 
-    ./coredns.nix
     ./headscale.nix
     ./mailserver.nix
     ./matrix-synapse.nix
@@ -18,6 +17,8 @@
     ./nginx.nix
     ./radicale.nix
     ./rss-bridge.nix
-    ./step-ca.nix
+
+    # ./coredns.nix
+    # ./step-ca.nix
   ];
 }
diff --git a/hosts/sid/services/headscale.nix b/hosts/sid/services/headscale.nix
index 6c7148f..b9492db 100644
--- a/hosts/sid/services/headscale.nix
+++ b/hosts/sid/services/headscale.nix
@@ -1,5 +1,6 @@
 {
   inputs,
+  constants,
   ...
 }:
 
@@ -24,5 +25,17 @@
       enable = true;
       subdomain = "hs";
     };
+    settings.dns.extra_records = [
+      {
+        name = constants.services.vaultwarden.fqdn;
+        type = "A";
+        value = constants.hosts.rx4.ip;
+      }
+      {
+        name = constants.services.netdata.fqdn;
+        type = "A";
+        value = constants.hosts.sid.ip;
+      }
+    ];
   };
 }
diff --git a/hosts/sid/services/mailserver.nix b/hosts/sid/services/mailserver.nix
index f3af274..024665c 100644
--- a/hosts/sid/services/mailserver.nix
+++ b/hosts/sid/services/mailserver.nix
@@ -1,4 +1,4 @@
-{ inputs, config, ... }:
+{ inputs, ... }:
 
 {
   imports = [ inputs.synix.nixosModules.mailserver ];
@@ -6,10 +6,12 @@
   mailserver = {
     enable = true;
     stateVersion = 3;
-    localDnsResolver = !config.services.coredns.enable;
     accounts = {
       sid = {
-        aliases = [ "postmaster" ];
+        aliases = [
+          "admin"
+          "postmaster"
+        ];
       };
       vaultwarden = { };
     };
diff --git a/hosts/sid/services/nginx.nix b/hosts/sid/services/nginx.nix
index 81eace3..0844781 100644
--- a/hosts/sid/services/nginx.nix
+++ b/hosts/sid/services/nginx.nix
@@ -56,15 +56,6 @@ in
       address = constants.hosts.rx4.ip;
       port = constants.services.miniflux.port;
     };
-    virtualHosts."${constants.services.netdata.fqdn}" = {
-      useACMEHost = "sid-internal";
-      forceSSL = ssl;
-      locations."/" = {
-        # proxyPass = "http://${constants.hosts.sid.ip}:${toString constants.services.netdata.port}";
-        proxyPass = "http://127.0.0.1:${toString constants.services.netdata.port}";
-        proxyWebsockets = true;
-      };
-    };
     virtualHosts."${constants.services.open-webui-oci.fqdn}" = mkVirtualHost {
       inherit ssl;
       address = constants.hosts.rx4.ip;
@@ -82,15 +73,6 @@ in
       address = constants.hosts.rx4.ip;
       port = constants.services.rsshub-oci.port;
     };
-    virtualHosts."${constants.services.vaultwarden.fqdn}" = {
-      useACMEHost = "sid-internal";
-      forceSSL = ssl;
-      locations = {
-        "/" = {
-          proxyPass = "http://${constants.hosts.rx4.ip}:${toString constants.services.vaultwarden.port}";
-        };
-      };
-    };
     # FIXME
     # virtualHosts."print.sid.ovh" = {
     #   enableACME = true;
diff --git a/hosts/sid/services/step-ca.nix b/hosts/sid/services/step-ca.nix
index 21d04d4..e2570c9 100644
--- a/hosts/sid/services/step-ca.nix
+++ b/hosts/sid/services/step-ca.nix
@@ -78,10 +78,8 @@ in
   security.acme = {
     certs."sid-internal" = {
       # domain = constants.intranet;
-      domain = constants.services.vaultwarden.fqdn;
+      domain = constants.services.netdata.fqdn;
       extraDomainNames = [
-        constants.services.netdata.fqdn
-        # constants.services.vaultwarden.fqdn
       ];
       server = "https://${constants.ca-fqdn}:${toString cfg.port}/acme/acme/directory";
       group = "nginx";
diff --git a/modules/nixos/tailscale/default.nix b/modules/nixos/tailscale/default.nix
index 884847a..f42bb85 100644
--- a/modules/nixos/tailscale/default.nix
+++ b/modules/nixos/tailscale/default.nix
@@ -11,7 +11,7 @@
         loginServer = "https://hs.sid.ovh";
         authKeyFile = config.sops.secrets."tailscale/personal-key".path;
         enableSSH = true;
-        acceptDNS = false; # use coredns
+        acceptDNS = true;
       };
     };
   };

From 5c8d94d03d01de896b984b653dd8ec07ab69cf85 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Sun, 17 May 2026 22:45:37 +0200
Subject: [PATCH 04/54] change netdata fqdn

---
 constants.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/constants.nix b/constants.nix
index 7a2a16e..91b8456 100644
--- a/constants.nix
+++ b/constants.nix
@@ -30,7 +30,7 @@ rec {
       port = 8085;
     };
     netdata = {
-      fqdn = "netdata." + intranet;
+      fqdn = "mon." + domain;
       port = 19999;
     };
     open-webui-oci = {

From 27492ea730c7f9131150a29eb5df5b7abec698a4 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Sun, 17 May 2026 23:01:56 +0200
Subject: [PATCH 05/54] rm step-ca and coredns

---
 constants.nix                      |   2 +-
 hosts/rx4/services/nginx.nix       |  19 ++++++
 hosts/rx4/services/vaultwarden.nix |  21 ++----
 hosts/sid/services/coredns.nix     |  31 ---------
 hosts/sid/services/default.nix     |   3 -
 hosts/sid/services/headscale.nix   |   5 --
 hosts/sid/services/step-ca.nix     | 105 -----------------------------
 7 files changed, 24 insertions(+), 162 deletions(-)
 delete mode 100644 hosts/sid/services/coredns.nix
 delete mode 100644 hosts/sid/services/step-ca.nix

diff --git a/constants.nix b/constants.nix
index 91b8456..e8de9ad 100644
--- a/constants.nix
+++ b/constants.nix
@@ -30,7 +30,7 @@ rec {
       port = 8085;
     };
     netdata = {
-      fqdn = "mon." + domain;
+      # fqdn = "mon." + domain;
       port = 19999;
     };
     open-webui-oci = {
diff --git a/hosts/rx4/services/nginx.nix b/hosts/rx4/services/nginx.nix
index c4c24af..cae8e31 100644
--- a/hosts/rx4/services/nginx.nix
+++ b/hosts/rx4/services/nginx.nix
@@ -7,6 +7,8 @@
 
 let
   cfg = config.services.nginx;
+
+  inherit (constants) domain;
 in
 {
   imports = [
@@ -34,4 +36,21 @@ in
       };
     };
   };
+
+  security.acme = {
+    acceptTerms = true;
+    defaults = {
+      email = "admin@${domain}";
+      dnsProvider = "hetzner";
+      credentialFiles = {
+        HETZNER_API_TOKEN_FILE = config.sops.secrets.hetzner-api-key.path;
+      };
+    };
+  };
+
+  sops.secrets.hetzner-api-key = {
+    mode = "0400";
+    owner = "acme";
+    group = "acme";
+  };
 }
diff --git a/hosts/rx4/services/vaultwarden.nix b/hosts/rx4/services/vaultwarden.nix
index eed5712..cffaeae 100644
--- a/hosts/rx4/services/vaultwarden.nix
+++ b/hosts/rx4/services/vaultwarden.nix
@@ -62,18 +62,10 @@ in
     };
   };
 
-  security.acme = {
-    acceptTerms = true;
-    defaults.email = "admin@${domain}";
-    certs."pw-custom" = {
-      domain = fqdn;
-      dnsProvider = "hetzner";
-      dnsResolver = "1.1.1.1:53";
-      credentialFiles = {
-        HETZNER_API_TOKEN_FILE = config.sops.secrets.hetzner-api-key.path;
-      };
-      group = "nginx";
-    };
+  security.acme.certs."pw-custom" = {
+    domain = fqdn;
+    postRun = "systemctl restart vaultwarden.service";
+    group = "nginx";
   };
 
   sops =
@@ -90,11 +82,6 @@ in
         "vaultwarden/smtp-password" = {
           inherit owner group mode;
         };
-        hetzner-api-key = {
-          inherit mode;
-          owner = "acme";
-          group = "acme";
-        };
       };
       templates = {
         "vaultwarden/env-file" = {
diff --git a/hosts/sid/services/coredns.nix b/hosts/sid/services/coredns.nix
deleted file mode 100644
index b7e2f37..0000000
--- a/hosts/sid/services/coredns.nix
+++ /dev/null
@@ -1,31 +0,0 @@
-{ constants, ... }:
-
-{
-  services.resolved.enable = false;
-  networking.resolvconf.enable = false;
-
-  networking.nameservers = [ "127.0.0.1" ];
-
-  services.coredns = {
-    enable = true;
-    config = with constants; ''
-      .:53 {
-        bind 0.0.0.0
-        hosts {
-          ${hosts.sid.ip} ${ca-fqdn} 
-
-          ${hosts.sid.ip} ${services.netdata.fqdn}
-
-          fallthrough
-        }
-        forward . 1.1.1.1 8.8.8.8
-        cache 30
-        log
-        errors
-      }
-    '';
-  };
-
-  networking.firewall.allowedUDPPorts = [ 53 ];
-  networking.firewall.allowedTCPPorts = [ 53 ];
-}
diff --git a/hosts/sid/services/default.nix b/hosts/sid/services/default.nix
index 9baf6fb..7ca9678 100644
--- a/hosts/sid/services/default.nix
+++ b/hosts/sid/services/default.nix
@@ -17,8 +17,5 @@
     ./nginx.nix
     ./radicale.nix
     ./rss-bridge.nix
-
-    # ./coredns.nix
-    # ./step-ca.nix
   ];
 }
diff --git a/hosts/sid/services/headscale.nix b/hosts/sid/services/headscale.nix
index b9492db..0d4a03f 100644
--- a/hosts/sid/services/headscale.nix
+++ b/hosts/sid/services/headscale.nix
@@ -31,11 +31,6 @@
         type = "A";
         value = constants.hosts.rx4.ip;
       }
-      {
-        name = constants.services.netdata.fqdn;
-        type = "A";
-        value = constants.hosts.sid.ip;
-      }
     ];
   };
 }
diff --git a/hosts/sid/services/step-ca.nix b/hosts/sid/services/step-ca.nix
deleted file mode 100644
index e2570c9..0000000
--- a/hosts/sid/services/step-ca.nix
+++ /dev/null
@@ -1,105 +0,0 @@
-{
-  constants,
-  config,
-  pkgs,
-  ...
-}:
-
-let
-  cfg = config.services.step-ca;
-in
-{
-  services.step-ca = {
-    enable = true;
-    address = "0.0.0.0";
-    port = 8443;
-    openFirewall = true;
-    intermediatePasswordFile = config.sops.secrets."step-ca/password".path;
-    # nix-shell -p step-cli --run "step ca init"
-    settings = {
-      # FIXME: nix-store paths do not work
-      # root = ../../../certs/root_ca.crt;
-      # crt = ../../../certs/intermediate_ca.crt;
-      # FIXME: not reproducible
-      root = "/var/lib/step-ca/certs/root_ca.crt";
-      crt = "/var/lib/step-ca/certs/intermediate_ca.crt";
-      key = config.sops.secrets."step-ca/intermediate-key".path;
-      dnsNames = [
-        constants.ca-fqdn
-        constants.hosts.sid.ip
-      ];
-      logger = {
-        format = "text";
-      };
-      db = {
-        type = "badgerv2";
-        dataSource = "/var/lib/step-ca/db";
-      };
-      authority = {
-        provisioners = [
-          {
-            type = "ACME";
-            name = "acme";
-          }
-          {
-            type = "JWK";
-            name = "sid@sid.ovh";
-            key = {
-              use = "sig";
-              kty = "EC";
-              kid = "w3fV4U-frlyTnBMg4yNYrLsn8_mY98H8HthoscpoVrg";
-              crv = "P-256";
-              alg = "ES256";
-              x = "KZCDecn4sb87T3UO6JsIzJVtr4Aa0UcYzYDNBUM6F7M";
-              y = "CbGHn9tXQbV0Ur2VuXITLnWgfxCRmKEoUdMUmrP9Qkw";
-            };
-            encryptedKey = "eyJhbGciOiJQQkVTMi1IUzI1NitBMTI4S1ciLCJjdHkiOiJqd2sranNvbiIsImVuYyI6IkEyNTZHQ00iLCJwMmMiOjYwMDAwMCwicDJzIjoiZS1MUDhDYlE4dzVuMF9nUGhXOWtGdyJ9.rgsqo58rJFWaociSqiPg3E1alAeqoHWubJi4n2uoUFYp3YTWaYZzqA.6P6oimHsKGdCWruo.fNaDr50IXCtCe7W7VIXuS3rlfin_R0nogNpIJ9C6szYg8k10UylircUs14Zl1EHQ9lFeJovb1y1uljzBajMGkOAGlMvashrphVkXiSxHWKDhzbrItJx3qChLtSLJJtXiXPbJQKCAeBjztqPuTw6dI4Z6IR9---kiTvzF6I9KE8afGFlMSubGjr9FnqgiOb2JiZuTfcBGDx78puxdWzUrEEVlliHdv2agbKhY0b13x-obaTIWwlqLFbasv7kPneJ8Ggp7IHHr5uDcUrqVKkTfBrD0lelXm6SwJTHGMkty6inlwSflT9mxvkNq7OGV9triPQc8AGVv0c7t7dHoX_E.tSjJqttCS6zLI_-7zPdXNQ";
-          }
-        ];
-      };
-      tls = {
-        cipherSuites = [
-          "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256"
-          "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"
-        ];
-        renegotiation = false;
-      };
-    };
-  };
-
-  environment.systemPackages = [
-    pkgs.step-cli
-  ];
-
-  systemd.tmpfiles.rules = [
-    "d /var/lib/acme/acme-challenge 0755 acme nginx"
-  ];
-
-  security.acme = {
-    certs."sid-internal" = {
-      # domain = constants.intranet;
-      domain = constants.services.netdata.fqdn;
-      extraDomainNames = [
-      ];
-      server = "https://${constants.ca-fqdn}:${toString cfg.port}/acme/acme/directory";
-      group = "nginx";
-    };
-  };
-
-  sops =
-    let
-      owner = "step-ca";
-      group = "step-ca";
-      mode = "0400";
-    in
-    {
-      secrets = {
-        "step-ca/password" = {
-          inherit owner group mode;
-        };
-        "step-ca/intermediate-key" = {
-          inherit owner group mode;
-        };
-      };
-    };
-}

From c31744739f6d54ae226c0799237b436722553cba Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Sun, 17 May 2026 23:25:21 +0200
Subject: [PATCH 06/54] fix vw smtp config

---
 constants.nix                      | 4 ++++
 hosts/rx4/services/vaultwarden.nix | 6 +++---
 hosts/sid/services/mailserver.nix  | 6 +++++-
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/constants.nix b/constants.nix
index e8de9ad..805fbf9 100644
--- a/constants.nix
+++ b/constants.nix
@@ -25,6 +25,10 @@ rec {
       subdomain = "f";
       fqdn = subdomain + "." + domain;
     };
+    mailserver = rec {
+      subdomain = "mail";
+      fqdn = subdomain + "." + domain;
+    };
     miniflux = {
       fqdn = "rss." + domain;
       port = 8085;
diff --git a/hosts/rx4/services/vaultwarden.nix b/hosts/rx4/services/vaultwarden.nix
index cffaeae..d9f8092 100644
--- a/hosts/rx4/services/vaultwarden.nix
+++ b/hosts/rx4/services/vaultwarden.nix
@@ -27,9 +27,9 @@ in
 
       SMTP_FROM = "vaultwarden@${domain}";
       SMTP_FROM_NAME = "${domain} Vaultwarden server";
-      SMTP_HOST = constants.hosts.sid.ip;
-      SMTP_PORT = 587;
-      SMTP_SECURITY = "starttls";
+      SMTP_HOST = constants.services.mailserver.fqdn;
+      SMTP_PORT = 465;
+      SMTP_SECURITY = "force_tls";
       SMTP_USERNAME = "vaultwarden@${domain}";
 
       ROCKET_ADDRESS = "127.0.0.1";
diff --git a/hosts/sid/services/mailserver.nix b/hosts/sid/services/mailserver.nix
index 024665c..c70433e 100644
--- a/hosts/sid/services/mailserver.nix
+++ b/hosts/sid/services/mailserver.nix
@@ -1,10 +1,14 @@
-{ inputs, ... }:
+{ inputs, constants, ... }:
 
+let
+  inherit (constants.services.mailserver) subdomain;
+in
 {
   imports = [ inputs.synix.nixosModules.mailserver ];
 
   mailserver = {
     enable = true;
+    inherit subdomain;
     stateVersion = 3;
     accounts = {
       sid = {

From 9360ae4543602b62aae8d99a4da72add46c630ee Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Mon, 18 May 2026 18:42:52 +0200
Subject: [PATCH 07/54] add nixos mcp server

---
 flake.lock                            |  8 +++----
 hosts/rx4/services/open-webui-oci.nix | 31 ++++++++++++++++++++++++++-
 2 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/flake.lock b/flake.lock
index da7918e..0f12baa 100644
--- a/flake.lock
+++ b/flake.lock
@@ -5199,11 +5199,11 @@
         "stylix": "stylix_6"
       },
       "locked": {
-        "lastModified": 1778016348,
-        "narHash": "sha256-C8PtC95r1KJync8qDEroIont1VT8tiwsjonYjwGLhbY=",
+        "lastModified": 1779122512,
+        "narHash": "sha256-fMhsa8ms/0mR6wmuc+Eoe3Lj31pFO1hcfiZvvCOFC2I=",
         "ref": "release-25.11",
-        "rev": "8ad8b1f633f6c3875032a0ead0e87255dff4ab3c",
-        "revCount": 57,
+        "rev": "896743958561cb191f4e4d738d823008126cf175",
+        "revCount": 83,
         "type": "git",
         "url": "https://git.sid.ovh/sid/synix.git"
       },
diff --git a/hosts/rx4/services/open-webui-oci.nix b/hosts/rx4/services/open-webui-oci.nix
index 5c43197..506b5dc 100644
--- a/hosts/rx4/services/open-webui-oci.nix
+++ b/hosts/rx4/services/open-webui-oci.nix
@@ -2,12 +2,19 @@
   inputs,
   constants,
   config,
+  lib,
   pkgs,
   ...
 }:
 
+let
+  inherit (lib) getExe;
+in
 {
-  imports = [ inputs.synix.nixosModules.open-webui-oci ];
+  imports = [
+    inputs.synix.nixosModules.open-webui-oci
+    inputs.synix.nixosModules.mcpo
+  ];
 
   services.open-webui-oci = {
     enable = true;
@@ -21,6 +28,28 @@
     };
   };
 
+  services.mcpo = {
+    enable = true;
+    package = pkgs.synix.mcpo;
+    port = 8765;
+    settings = {
+      mcpServers = {
+        # fetcher-mcp = {
+        #   command = getExe pkgs.synix.fetcher-mcp;
+        #   url = "http://127.0.0.1:8001/fetcher-mcp";
+        # };
+        nixos = {
+          command = getExe pkgs.nix;
+          args = [
+            "run"
+            "github:utensils/mcp-nixos"
+            "--"
+          ];
+        };
+      };
+    };
+  };
+
   # sops = {
   #   secrets."open-webui-oci/stt-api-key" = { };
   #   secrets."open-webui-oci/tts-api-key" = { };

From 2b621988e485a2c3905de50778f2322eee0e6197 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Mon, 18 May 2026 18:45:44 +0200
Subject: [PATCH 08/54] update synix input

---
 flake.lock | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/flake.lock b/flake.lock
index 0f12baa..b4c5943 100644
--- a/flake.lock
+++ b/flake.lock
@@ -5199,11 +5199,11 @@
         "stylix": "stylix_6"
       },
       "locked": {
-        "lastModified": 1779122512,
-        "narHash": "sha256-fMhsa8ms/0mR6wmuc+Eoe3Lj31pFO1hcfiZvvCOFC2I=",
+        "lastModified": 1779122723,
+        "narHash": "sha256-hEL93URxv2e1vhKxA++m0rszGgZUmWKKPFxa4t1LIi4=",
         "ref": "release-25.11",
-        "rev": "896743958561cb191f4e4d738d823008126cf175",
-        "revCount": 83,
+        "rev": "7d73c46b94fa6a7562b7b3ef300f39aa71734d26",
+        "revCount": 85,
         "type": "git",
         "url": "https://git.sid.ovh/sid/synix.git"
       },

From 1410a5914061f4cd08eb2851fb36999c4be90e0b Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Mon, 18 May 2026 19:52:38 +0200
Subject: [PATCH 09/54] open-webui: enable fetcher-mcp

---
 hosts/rx4/services/open-webui-oci.nix | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/hosts/rx4/services/open-webui-oci.nix b/hosts/rx4/services/open-webui-oci.nix
index 506b5dc..f22297d 100644
--- a/hosts/rx4/services/open-webui-oci.nix
+++ b/hosts/rx4/services/open-webui-oci.nix
@@ -34,10 +34,9 @@ in
     port = 8765;
     settings = {
       mcpServers = {
-        # fetcher-mcp = {
-        #   command = getExe pkgs.synix.fetcher-mcp;
-        #   url = "http://127.0.0.1:8001/fetcher-mcp";
-        # };
+        fetcher-mcp = {
+          command = getExe pkgs.synix.fetcher-mcp;
+        };
         nixos = {
           command = getExe pkgs.nix;
           args = [

From 949f403a20bf00f07306b0d8cef2fefe8bf43b1e Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Mon, 18 May 2026 20:40:38 +0200
Subject: [PATCH 10/54] rm custom root ca. add remote journald

---
 modules/nixos/common/default.nix  |  3 +--
 modules/nixos/common/journald.nix |  6 ++++++
 modules/nixos/default.nix         |  1 -
 modules/nixos/pki/default.nix     |  3 ---
 modules/nixos/pki/root_ca.crt     | 12 ------------
 5 files changed, 7 insertions(+), 18 deletions(-)
 create mode 100644 modules/nixos/common/journald.nix
 delete mode 100644 modules/nixos/pki/default.nix
 delete mode 100644 modules/nixos/pki/root_ca.crt

diff --git a/modules/nixos/common/default.nix b/modules/nixos/common/default.nix
index 0415b9f..10028cd 100644
--- a/modules/nixos/common/default.nix
+++ b/modules/nixos/common/default.nix
@@ -2,11 +2,10 @@
 
 {
   imports = [
+    ./journald.nix
     ./nix.nix
     ./overlays.nix
 
-    ../pki
-
     inputs.synix.nixosModules.device.server
   ];
 
diff --git a/modules/nixos/common/journald.nix b/modules/nixos/common/journald.nix
new file mode 100644
index 0000000..d31ada4
--- /dev/null
+++ b/modules/nixos/common/journald.nix
@@ -0,0 +1,6 @@
+{
+  services.journald.upload = {
+    enable = true;
+    settings.Upload.URL = "http://100.64.0.5:19532";
+  };
+}
diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix
index 540f4ee..f831ea8 100644
--- a/modules/nixos/default.nix
+++ b/modules/nixos/default.nix
@@ -6,7 +6,6 @@
   forgejo-runner = import ./forgejo-runner;
   gnome = import ./gnome;
   monero = import ./monero;
-  pki = import ./pki;
   rsshub-oci = import ./rsshub-oci;
   tailscale = import ./tailscale;
   xfce = import ./xfce;
diff --git a/modules/nixos/pki/default.nix b/modules/nixos/pki/default.nix
deleted file mode 100644
index 9b849f6..0000000
--- a/modules/nixos/pki/default.nix
+++ /dev/null
@@ -1,3 +0,0 @@
-{
-  security.pki.certificateFiles = [ ./root_ca.crt ];
-}
diff --git a/modules/nixos/pki/root_ca.crt b/modules/nixos/pki/root_ca.crt
deleted file mode 100644
index 44abf61..0000000
--- a/modules/nixos/pki/root_ca.crt
+++ /dev/null
@@ -1,12 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIBrzCCAVWgAwIBAgIQDV0M0pLkCXvARpa+ipSx8jAKBggqhkjOPQQDAjA2MRUw
-EwYDVQQKEwxzaWQtaW50ZXJuYWwxHTAbBgNVBAMTFHNpZC1pbnRlcm5hbCBSb290
-IENBMB4XDTI2MDQxODIwMzkwMloXDTM2MDQxNTIwMzkwMlowNjEVMBMGA1UEChMM
-c2lkLWludGVybmFsMR0wGwYDVQQDExRzaWQtaW50ZXJuYWwgUm9vdCBDQTBZMBMG
-ByqGSM49AgEGCCqGSM49AwEHA0IABCH2VmIwKEjdma4UymD7RWuGcaT2algrL5nm
-TE0NzP8giezdU9bEP487AvUPPibSYDWxdp4ycbl6qNVTiy29xkmjRTBDMA4GA1Ud
-DwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEBMB0GA1UdDgQWBBRaiBACRDZk
-HZMU9y8YsUF4WPB+5TAKBggqhkjOPQQDAgNIADBFAiAh+b49V2VTnT6nRCRM0Qwq
-ruzayrrnmF7pIxi9PVFwBQIhANQsL3ok4gCTRAnT0mUXSyWexzSESZ1lkpLYiyoj
-RgLi
------END CERTIFICATE-----

From cc27c3bde41737bb216971004f0d77da0287f020 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Mon, 18 May 2026 21:50:20 +0200
Subject: [PATCH 11/54] add AGENTS.md

---
 AGENTS.md | 390 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 390 insertions(+)
 create mode 100644 AGENTS.md

diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 0000000..0ba9650
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,390 @@
+# AGENTS.md
+
+This file teaches AI agents how to navigate and work with this NixOS
+configuration repository (`sid.ovh`).
+
+---
+
+## Repository Overview
+
+This is a NixOS flake managing three hosts:
+
+| Host  | IP (Tailscale) | Role                                              |
+|-------|----------------|---------------------------------------------------|
+| `sid` | `100.64.0.6`   | VPS — reverse proxy, mail, matrix, headscale      |
+| `rx4` | `100.64.0.10`  | Home server — open-webui, forgejo, vaultwarden, … |
+| `vde` | `100.64.0.1`   | Desktop / workstation (not in use at the moment)  |
+
+Deployment is done via `deploy-rs` through a Forgejo CI pipeline
+(`.forgejo/workflows/deploy-configs.yml`).
+
+---
+
+## The `synix` Flake Input
+
+### What it is
+
+`synix` is the owner's **personal NixOS library flake**, hosted at:
+
+```
+https://git.sid.ovh/sid/synix.git
+```
+
+It provides:
+- **NixOS modules** (`inputs.synix.nixosModules.*`) — opinionated,
+  reusable service configurations used heavily across all three hosts.
+- **Home Manager modules** (`inputs.synix.homeModules.*`) — desktop /
+  user-space configurations (Hyprland, nixvim, waybar, …).
+- **Packages** (`inputs.synix.packages.<system>.*`, also accessible via
+  the `synix-packages` overlay as `pkgs.synix.*`) — custom packages not
+  in nixpkgs.
+- **Overlays** (`inputs.synix.overlays.*`) — nixpkgs modifications.
+- **A utility library** (`inputs.synix.lib` / `lib.utils`) — helper
+  functions such as `mkVirtualHost` and `mkReverseProxyOption`.
+- **Templates** — starter flake templates for servers, desktops, VMs, etc.
+- **Apps** — convenience scripts (`deploy`, `rebuild`, `install`, …).
+
+### How it is consumed in `sid.ovh`
+
+In `flake.nix`:
+
+```nix
+inputs = {
+  synix.url = "git+https://git.sid.ovh/sid/synix.git?ref=release-25.11";
+  synix.inputs.nixpkgs.follows = "nixpkgs";
+  # development overrides (commented out normally):
+  # synix.url = "git+https://git.sid.ovh/sid/synix.git?ref=develop";
+  # synix.url = "git+file:///home/sid/src/synix";  # local checkout
+  ...
+};
+```
+
+The active branch is **`release-25.11`**, aligned with `nixpkgs`
+`nixos-25.11`. A `develop` branch exists for unstable work. A local
+checkout can be used for testing by switching to the `git+file://` URL.
+
+---
+
+## `synix` Module Reference
+
+All NixOS modules are under `modules/nixos/` in the synix repo and
+exposed as `inputs.synix.nixosModules.<name>`.
+
+### Infrastructure / base modules
+
+| Module name                        | Path in synix                     | Purpose |
+|------------------------------------|-----------------------------------|---------|
+| `common`                           | `modules/nixos/common/`           | Base system config (boot, networking, nix settings, zsh, sudo, htop, locale) |
+| `device.server`                    | `modules/nixos/device/server.nix` | Server profile (no GUI, sensible server defaults) |
+| `device.desktop`                   | `modules/nixos/device/desktop.nix`| Desktop profile |
+| `device.laptop`                    | `modules/nixos/device/laptop.nix` | Laptop profile |
+| `device.vm`                        | `modules/nixos/device/vm.nix`     | VM profile |
+| `sops`                             | `modules/nixos/sops/`             | sops-nix wiring (age keys, secret paths) |
+| `openssh`                          | `modules/nixos/openssh/`          | Hardened OpenSSH configuration |
+| `normalUsers`                      | `modules/nixos/normalUsers/`      | Declarative normal-user creation with SSH keys |
+| `tailscale`                        | `modules/nixos/tailscale/`        | Tailscale VPN with multiple tailnet support |
+| `nginx`                            | `modules/nixos/nginx/`            | Opinionated nginx base (ACME, SSL helpers) |
+
+### Service modules used in `sid.ovh`
+
+| Module name          | Path in synix                        | Used by host | Notes |
+|----------------------|--------------------------------------|--------------|-------|
+| `headscale`          | `modules/nixos/headscale/`           | sid          | Self-hosted Tailscale control plane |
+| `headplane`          | `modules/nixos/headplane/`           | sid          | Headscale web UI |
+| `mailserver`         | `modules/nixos/mailserver/`          | sid          | Wraps `nixos-mailserver` |
+| `matrix-synapse`     | `modules/nixos/matrix-synapse/`      | sid          | Matrix homeserver + bridges (WhatsApp, Signal) + LiveKit |
+| `maubot`             | `modules/nixos/maubot/`              | sid          | Matrix bot framework |
+| `baibot`             | `modules/nixos/baibot/`              | sid          | AI Matrix bot |
+| `coturn`             | `modules/nixos/coturn/`              | sid          | TURN/STUN server (Matrix calls) |
+| `radicale`           | `modules/nixos/radicale/`            | sid          | CalDAV/CardDAV server |
+| `rss-bridge`         | `modules/nixos/rss-bridge/`          | sid          | RSS-Bridge PHP app |
+| `miniflux`           | `modules/nixos/miniflux/`            | rx4          | RSS reader |
+| `jirafeau`           | `modules/nixos/jirafeau/`            | rx4          | File sharing |
+| `open-webui-oci`     | `modules/nixos/open-webui-oci/`      | rx4          | Open WebUI via Podman OCI container |
+| `mcpo`               | `modules/nixos/mcpo/`                | rx4          | MCP-to-OpenAPI proxy (AI tool server) |
+| `print-server`       | `modules/nixos/print-server/`        | rx4          | CUPS print server |
+| `virtualisation`     | `modules/nixos/virtualisation/`      | vde          | VFIO / KVM / QEMU / kvmfr setup |
+
+### Other available modules (not currently active in `sid.ovh`)
+
+`audio`, `bluetooth`, `amd`, `nvidia`, `jellyfin`, `i2pd`,
+`ftp-webserver`, `webPage`, `windows-oci`, `librechat-oci`,
+`nostr-relay`, `ollama`, `cifsMount`, `hyprland` (NixOS-level),
+`normalUsers`.
+
+---
+
+## `synix` Packages (`pkgs.synix.*`)
+
+Custom packages provided by synix and available via the
+`synix-packages` overlay (applied in
+`modules/nixos/common/overlays.nix`):
+
+| Package name                  | What it is |
+|-------------------------------|------------|
+| `pkgs.synix.mcpo`             | MCP-to-OpenAPI proxy server |
+| `pkgs.synix.fetcher-mcp`      | MCP tool: web page fetcher |
+| `pkgs.synix.baibot`           | AI Matrix bot |
+| `pkgs.synix.jirafeau`         | Jirafeau file-sharing webapp |
+| `pkgs.synix.jirafeau-cli`     | CLI client for Jirafeau |
+| `pkgs.synix.marker-pdf`       | PDF-to-markdown converter |
+| `pkgs.synix.mcpo`             | MCP proxy |
+| `pkgs.synix.systemctl-tui`    | TUI for systemctl |
+| `pkgs.synix.quicknote`        | Quick note shell script |
+| `pkgs.synix.synix-docs`       | Built MkDocs documentation site |
+| `pkgs.synix.bulk-rename`      | Batch file renaming script |
+| `pkgs.synix.pyman`            | Python man-page helper |
+
+The full list lives in `pkgs/default.nix` in the synix repo. All
+packages are also accessible as `inputs.synix.packages.<system>.<name>`.
+
+---
+
+## `synix` Utility Library (`lib.utils`)
+
+Exposed as `inputs.synix.lib` and merged into the flake's `lib` in
+`flake.nix`:
+
+```nix
+lib = nixpkgs.lib.extend (final: prev: inputs.synix.lib or { });
+```
+
+Key helpers (defined in `lib/utils.nix`):
+
+| Function                  | What it does |
+|---------------------------|--------------|
+| `lib.utils.mkVirtualHost` | Builds a standard nginx `virtualHosts` entry with optional SSL, upstream address, port, and extra config |
+| `lib.utils.mkReverseProxyOption` | Generates a standard NixOS option set for enabling a reverse-proxy sub-config on a module |
+
+Usage example from `hosts/sid/services/nginx.nix`:
+
+```nix
+virtualHosts."${constants.services.forgejo.fqdn}" = mkVirtualHost {
+  inherit ssl;
+  address = constants.hosts.rx4.ip;
+  port    = constants.services.forgejo.port;
+};
+```
+
+---
+
+## How `synix` Modules Are Imported
+
+In each host's `default.nix`, synix modules are imported directly from
+the `inputs` special arg:
+
+```nix
+# hosts/rx4/default.nix
+imports = [
+  inputs.synix.nixosModules.common          # base system config
+  inputs.synix.nixosModules.device.server   # server profile
+  inputs.synix.nixosModules.openssh         # hardened SSH
+  # …service-specific modules in hosts/rx4/services/
+];
+```
+
+Service files then add more synix modules as needed:
+
+```nix
+# hosts/rx4/services/open-webui-oci.nix
+imports = [
+  inputs.synix.nixosModules.open-webui-oci
+  inputs.synix.nixosModules.mcpo
+];
+```
+
+---
+
+## Overlays Applied
+
+`modules/nixos/common/overlays.nix` applies five overlays to every host:
+
+| Overlay name            | Source           | Accessible as        |
+|-------------------------|------------------|----------------------|
+| `synix-packages`        | `synix`          | `pkgs.synix.*`       |
+| `local-packages`        | `pkgs/` in repo  | `pkgs.local.*`       |
+| `modifications`         | `overlays/`      | replaces pkgs in place |
+| `old-stable-packages`   | `nixpkgs-25.05`  | `pkgs.old-stable.*`  |
+| `unstable-packages`     | `nixos-unstable` | `pkgs.unstable.*`    |
+
+`synix` also supplies its own `modifications` overlay
+(`inputs.synix.overlays.modifications`), merged into the local
+`modifications` overlay in `overlays/default.nix`.
+
+---
+
+## Centralized Logging Architecture
+
+All hosts ship their systemd journal to a **central receiver running on
+`sid`** over HTTP using `systemd-journal-remote` / `systemd-journal-upload`.
+
+### Receiver (`sid`)
+
+```nix
+services.journald.remote = {
+  enable   = true;
+  listen   = "http";
+  port     = 19532;
+  settings.Remote.SplitMode = "host";  # one journal dir per sending host
+};
+
+users.users.sid.extraGroups = [
+  "systemd-journal"
+  "systemd-journal-remote"
+];
+```
+
+Key points:
+- Listens on **`http://0.0.0.0:19532`** (plain HTTP, Tailscale-only network).
+- `SplitMode = "host"` stores each remote host's entries under
+  `/var/log/journal/remote/`, separated by hostname.
+
+### Senders (all hosts via `modules/nixos/common/journald.nix`)
+
+```nix
+services.journald.upload = {
+  enable = true;
+  settings.Upload.URL = "http://100.64.0.5:19532";
+};
+```
+
+> **Note:** The upload URL uses `100.64.0.5` — the Tailscale transport
+> IP for `sid` (distinct from the advertised IP `100.64.0.6` in
+> `constants.nix`; verify with `ip addr show tailscale0` on `sid` if
+> queries fail).
+
+This module is applied to **every host** via
+`outputs.nixosModules.common` → `modules/nixos/common/default.nix`.
+
+---
+
+## Querying Remote Journals
+
+All queries run **on `sid`**, reading from `/var/log/journal/remote/`.
+
+### General pattern
+
+```bash
+journalctl -D /var/log/journal/remote/ \
+  _HOSTNAME=<hostname> \
+  [SYSTEMD_UNIT=<unit>] \
+  [other filters…] \
+  --no-pager [-n <lines>] [-f] [-S <since>] [-U <until>]
+```
+
+| Flag / Field                              | Purpose |
+|-------------------------------------------|---------|
+| `-D /var/log/journal/remote/`             | Read the remote journal directory |
+| `_HOSTNAME=rx4`                           | Filter to entries from host `rx4` |
+| `SYSTEMD_UNIT=podman-open-webui.service`  | Filter to a specific systemd unit |
+| `--no-pager`                              | Don't page output; good for scripts |
+| `-n 20`                                   | Show last 20 lines |
+| `-f`                                      | Follow (tail) mode |
+| `-S` / `-U`                               | Since / Until time bounds |
+
+### Canonical example — last 20 lines of open-webui on rx4
+
+```bash
+journalctl \
+  -D /var/log/journal/remote/ \
+  _HOSTNAME=rx4 \
+  SYSTEMD_UNIT=podman-open-webui.service \
+  --no-pager -n 20
+```
+
+This works because:
+1. `rx4` uploads its journal to `sid:19532`.
+2. The receiver stores it under `/var/log/journal/remote/` with
+   `SplitMode=host`, preserving the `_HOSTNAME` field.
+3. `_HOSTNAME=rx4` narrows to that host's entries.
+4. `SYSTEMD_UNIT=podman-open-webui.service` targets the OCI container
+   unit for Open WebUI (defined in
+   `hosts/rx4/services/open-webui-oci.nix`).
+
+### Other useful queries
+
+```bash
+# All logs from rx4 in the last hour
+journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 --no-pager -S "1 hour ago"
+
+# Forgejo on rx4
+journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 \
+  SYSTEMD_UNIT=forgejo.service --no-pager -n 50
+
+# Vaultwarden on rx4
+journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 \
+  SYSTEMD_UNIT=vaultwarden.service --no-pager -n 50
+
+# Nginx on rx4
+journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 \
+  SYSTEMD_UNIT=nginx.service --no-pager -n 50
+
+# Matrix-synapse on sid
+journalctl -D /var/log/journal/remote/ _HOSTNAME=sid \
+  SYSTEMD_UNIT=matrix-synapse.service --no-pager -n 50
+
+# Follow logs live
+journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 \
+  SYSTEMD_UNIT=podman-open-webui.service -f
+```
+
+### OCI container unit names
+
+Containers managed by `virtualisation.oci-containers` get units named
+`podman-<container-attr>.service`:
+
+| Container attr         | Unit name                              | Host |
+|------------------------|----------------------------------------|------|
+| `open-webui`           | `podman-open-webui.service`            | rx4  |
+| `rsshub-rsshub`        | `podman-rsshub-rsshub.service`         | rx4  |
+| `rsshub-redis`         | `podman-rsshub-redis.service`          | rx4  |
+| `rsshub-browserless`   | `podman-rsshub-browserless.service`    | rx4  |
+| `rsshub-real-browser`  | `podman-rsshub-real-browser.service`   | rx4  |
+
+---
+
+## Permissions
+
+User `sid` must be in both `systemd-journal` and
+`systemd-journal-remote` to read `/var/log/journal/remote/`:
+
+```nix
+users.users.sid.extraGroups = [
+  "systemd-journal"
+  "systemd-journal-remote"
+];
+```
+
+Verify with `id sid` on `sid`.
+
+---
+
+## Key File Locations
+
+| Path                                    | Purpose |
+|-----------------------------------------|---------|
+| `flake.nix`                             | Flake inputs, host configs, deploy nodes |
+| `constants.nix`                         | Canonical IPs, FQDNs, service ports |
+| `modules/nixos/common/journald.nix`     | Enables `journald.upload` on all hosts |
+| `modules/nixos/common/default.nix`      | Imports all common modules |
+| `modules/nixos/common/overlays.nix`     | Applies all five overlays |
+| `hosts/<name>/default.nix`             | Per-host top-level import list |
+| `hosts/<name>/services/`               | Per-host service configurations |
+| `hosts/<name>/secrets/secrets.yaml`    | sops-encrypted secrets for that host |
+| `overlays/default.nix`                 | Local overlay definitions |
+| `pkgs/`                                 | Local packages (`pkgs.local.*`) |
+| `users/sid/`                            | User account + SSH public key |
+
+---
+
+## Troubleshooting
+
+| Symptom | Check |
+|---------|-------|
+| No journal entries for a host | `systemctl status systemd-journal-upload` on the source host |
+| Permission denied reading remote journal | `id sid` on `sid` — needs `systemd-journal` + `systemd-journal-remote` |
+| Wrong unit name | `systemctl list-units --type=service` on the source host |
+| Receiver unreachable | `ss -tlnp \| grep 19532` on `sid` |
+| Upload URL IP mismatch | Upload uses `100.64.0.5`; verify `ip addr show tailscale0` on `sid` |
+| synix module not found | Check the branch in `flake.nix`; run `nix flake update synix` to refresh |
+| Want to test a synix change locally | Switch input URL to `git+file:///home/sid/src/synix` |

From aa5190781234170cb2bf42ee1cbbc0fcf82de1b1 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 00:26:20 +0200
Subject: [PATCH 12/54] add librechat-oci

---
 constants.nix                        |  4 +++
 hosts/rx4/services/default.nix       |  1 +
 hosts/rx4/services/librechat-oci.nix | 44 ++++++++++++++++++++++++++++
 3 files changed, 49 insertions(+)
 create mode 100644 hosts/rx4/services/librechat-oci.nix

diff --git a/constants.nix b/constants.nix
index 805fbf9..7b3578e 100644
--- a/constants.nix
+++ b/constants.nix
@@ -25,6 +25,10 @@ rec {
       subdomain = "f";
       fqdn = subdomain + "." + domain;
     };
+    librechat-oci = {
+      fqdn = "lc." + domain;
+      port = 3080;
+    };
     mailserver = rec {
       subdomain = "mail";
       fqdn = subdomain + "." + domain;
diff --git a/hosts/rx4/services/default.nix b/hosts/rx4/services/default.nix
index 6cb2dc6..8db7081 100644
--- a/hosts/rx4/services/default.nix
+++ b/hosts/rx4/services/default.nix
@@ -13,6 +13,7 @@
 
     ./forgejo.nix
     ./jirafeau.nix
+    ./librechat-oci.nix
     ./miniflux.nix
     ./netdata.nix
     ./nginx.nix
diff --git a/hosts/rx4/services/librechat-oci.nix b/hosts/rx4/services/librechat-oci.nix
new file mode 100644
index 0000000..07ab8d3
--- /dev/null
+++ b/hosts/rx4/services/librechat-oci.nix
@@ -0,0 +1,44 @@
+{
+  inputs,
+  constants,
+  config,
+  ...
+}:
+
+let
+  inherit (constants) domain;
+  inherit (constants.hosts.rx4) ip;
+  inherit (constants.services.librechat-oci) fqdn port;
+in
+{
+  imports = [
+    inputs.synix.nixosModules.librechat-oci
+  ];
+
+  services.librechat-oci = {
+    enable = true;
+    inherit port;
+    externalUrl = "https://${fqdn}";
+  };
+
+  services.nginx.virtualHosts."${fqdn}" = {
+    useACMEHost = fqdn;
+    forceSSL = true;
+    listen = [
+      {
+        addr = "${ip}:443";
+        ssl = true;
+      }
+    ];
+    locations."/" = {
+      proxyPass = "http://127.0.0.1:${toString port}";
+      proxyWebsockets = true;
+    };
+  };
+
+  security.acme.certs."${fqdn}" = {
+    domain = fqdn;
+    postRun = "systemctl restart podman-librechat.service";
+    group = "nginx";
+  };
+}

From 6b4474e1cac4b590ba8212beb2d21cc98035d862 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 00:26:45 +0200
Subject: [PATCH 13/54] streamline AGENTS.md

---
 AGENTS.md | 115 +++++++++++++++++++-----------------------------------
 1 file changed, 40 insertions(+), 75 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md
index 0ba9650..8defb09 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -1,7 +1,6 @@
 # AGENTS.md
 
-This file teaches AI agents how to navigate and work with this NixOS
-configuration repository (`sid.ovh`).
+This file teaches AI agents how to navigate and work with this NixOS configuration repository (`sid.ovh`).
 
 ---
 
@@ -9,14 +8,13 @@ configuration repository (`sid.ovh`).
 
 This is a NixOS flake managing three hosts:
 
-| Host  | IP (Tailscale) | Role                                              |
-|-------|----------------|---------------------------------------------------|
-| `sid` | `100.64.0.6`   | VPS — reverse proxy, mail, matrix, headscale      |
-| `rx4` | `100.64.0.10`  | Home server — open-webui, forgejo, vaultwarden, … |
-| `vde` | `100.64.0.1`   | Desktop / workstation (not in use at the moment)  |
+| Host  | IP (Tailscale) | Role                                                 |
+|-------|----------------|------------------------------------------------------|
+| `sid` | `100.64.0.6`   | VPS - reverse proxy, mail, matrix, headscale         |
+| `rx4` | `100.64.0.10`  | Home server - open-webui, forgejo, vaultwarden, etc. |
+| `vde` | `100.64.0.1`   | Desktop / workstation (not in use at the moment)     |
 
-Deployment is done via `deploy-rs` through a Forgejo CI pipeline
-(`.forgejo/workflows/deploy-configs.yml`).
+Deployment is done via `deploy-rs` through a Forgejo CI pipeline (`.forgejo/workflows/deploy-configs.yml`).
 
 ---
 
@@ -24,25 +22,20 @@ Deployment is done via `deploy-rs` through a Forgejo CI pipeline
 
 ### What it is
 
-`synix` is the owner's **personal NixOS library flake**, hosted at:
+`synix` is a **NixOS library flake**, hosted at:
 
 ```
 https://git.sid.ovh/sid/synix.git
 ```
 
 It provides:
-- **NixOS modules** (`inputs.synix.nixosModules.*`) — opinionated,
-  reusable service configurations used heavily across all three hosts.
-- **Home Manager modules** (`inputs.synix.homeModules.*`) — desktop /
-  user-space configurations (Hyprland, nixvim, waybar, …).
-- **Packages** (`inputs.synix.packages.<system>.*`, also accessible via
-  the `synix-packages` overlay as `pkgs.synix.*`) — custom packages not
-  in nixpkgs.
-- **Overlays** (`inputs.synix.overlays.*`) — nixpkgs modifications.
-- **A utility library** (`inputs.synix.lib` / `lib.utils`) — helper
-  functions such as `mkVirtualHost` and `mkReverseProxyOption`.
-- **Templates** — starter flake templates for servers, desktops, VMs, etc.
-- **Apps** — convenience scripts (`deploy`, `rebuild`, `install`, …).
+- **NixOS modules** (`inputs.synix.nixosModules.*`): opinionated, reusable service configurations used heavily across all three hosts.
+- **Home Manager modules** (`inputs.synix.homeModules.*`): desktop / user-space configurations (Hyprland, nixvim, waybar, etc.).
+- **Packages** (`inputs.synix.packages.<system>.*`, also accessible via the `synix-packages` overlay as `pkgs.synix.*`): custom packages not in nixpkgs.
+- **Overlays** (`inputs.synix.overlays.*`): nixpkgs modifications.
+- **A utility library** (`inputs.synix.lib` / `lib.utils`): helper functions such as `mkVirtualHost` and `mkReverseProxyOption`.
+- **Templates**: starter flake templates for servers, desktops, VMs, etc.
+- **Apps**: convenience scripts (`deploy`, `rebuild`, `install`, etc.).
 
 ### How it is consumed in `sid.ovh`
 
@@ -59,16 +52,13 @@ inputs = {
 };
 ```
 
-The active branch is **`release-25.11`**, aligned with `nixpkgs`
-`nixos-25.11`. A `develop` branch exists for unstable work. A local
-checkout can be used for testing by switching to the `git+file://` URL.
+The active branch is **`release-25.11`**, aligned with `nixpkgs` at `nixos-25.11`. A `develop` branch exists for unstable work. A local checkout can be used for testing by switching to the `git+file://` URL.
 
 ---
 
 ## `synix` Module Reference
 
-All NixOS modules are under `modules/nixos/` in the synix repo and
-exposed as `inputs.synix.nixosModules.<name>`.
+All NixOS modules are under `modules/nixos/` in the synix repo and exposed as `inputs.synix.nixosModules.<name>`.
 
 ### Infrastructure / base modules
 
@@ -107,18 +97,13 @@ exposed as `inputs.synix.nixosModules.<name>`.
 
 ### Other available modules (not currently active in `sid.ovh`)
 
-`audio`, `bluetooth`, `amd`, `nvidia`, `jellyfin`, `i2pd`,
-`ftp-webserver`, `webPage`, `windows-oci`, `librechat-oci`,
-`nostr-relay`, `ollama`, `cifsMount`, `hyprland` (NixOS-level),
-`normalUsers`.
+`audio`, `bluetooth`, `amd`, `nvidia`, `jellyfin`, `i2pd`, `ftp-webserver`, `webPage`, `windows-oci`, `librechat-oci`, `nostr-relay`, `ollama`, `cifsMount`, `hyprland` (NixOS-level), `normalUsers`.
 
 ---
 
 ## `synix` Packages (`pkgs.synix.*`)
 
-Custom packages provided by synix and available via the
-`synix-packages` overlay (applied in
-`modules/nixos/common/overlays.nix`):
+Custom packages provided by synix and available via the `synix-packages` overlay (applied in `modules/nixos/common/overlays.nix`):
 
 | Package name                  | What it is |
 |-------------------------------|------------|
@@ -135,15 +120,13 @@ Custom packages provided by synix and available via the
 | `pkgs.synix.bulk-rename`      | Batch file renaming script |
 | `pkgs.synix.pyman`            | Python man-page helper |
 
-The full list lives in `pkgs/default.nix` in the synix repo. All
-packages are also accessible as `inputs.synix.packages.<system>.<name>`.
+The full list lives in `pkgs/default.nix` in the synix repo. All packages are also accessible as `inputs.synix.packages.<system>.<name>`.
 
 ---
 
 ## `synix` Utility Library (`lib.utils`)
 
-Exposed as `inputs.synix.lib` and merged into the flake's `lib` in
-`flake.nix`:
+Exposed as `inputs.synix.lib` and merged into the flake's `lib` in `flake.nix`:
 
 ```nix
 lib = nixpkgs.lib.extend (final: prev: inputs.synix.lib or { });
@@ -170,8 +153,7 @@ virtualHosts."${constants.services.forgejo.fqdn}" = mkVirtualHost {
 
 ## How `synix` Modules Are Imported
 
-In each host's `default.nix`, synix modules are imported directly from
-the `inputs` special arg:
+In each host's `default.nix`, synix modules are imported directly from the `inputs` special arg:
 
 ```nix
 # hosts/rx4/default.nix
@@ -179,7 +161,7 @@ imports = [
   inputs.synix.nixosModules.common          # base system config
   inputs.synix.nixosModules.device.server   # server profile
   inputs.synix.nixosModules.openssh         # hardened SSH
-  # …service-specific modules in hosts/rx4/services/
+  # service-specific modules in hosts/rx4/services/
 ];
 ```
 
@@ -207,18 +189,15 @@ imports = [
 | `old-stable-packages`   | `nixpkgs-25.05`  | `pkgs.old-stable.*`  |
 | `unstable-packages`     | `nixos-unstable` | `pkgs.unstable.*`    |
 
-`synix` also supplies its own `modifications` overlay
-(`inputs.synix.overlays.modifications`), merged into the local
-`modifications` overlay in `overlays/default.nix`.
+`synix` also supplies its own `modifications` overlay (`inputs.synix.overlays.modifications`), merged into the local `modifications` overlay in `overlays/default.nix`.
 
 ---
 
 ## Centralized Logging Architecture
 
-All hosts ship their systemd journal to a **central receiver running on
-`sid`** over HTTP using `systemd-journal-remote` / `systemd-journal-upload`.
+All hosts ship their systemd journal to a **central receiver running on `sid`** over HTTP using `systemd-journal-remote` / `systemd-journal-upload`.
 
-### Receiver (`sid`)
+### Receiver (local host `pc` which is not part of this flake)
 
 ```nix
 services.journald.remote = {
@@ -248,19 +227,15 @@ services.journald.upload = {
 };
 ```
 
-> **Note:** The upload URL uses `100.64.0.5` — the Tailscale transport
-> IP for `sid` (distinct from the advertised IP `100.64.0.6` in
-> `constants.nix`; verify with `ip addr show tailscale0` on `sid` if
-> queries fail).
+> **Note:** The upload URL uses `100.64.0.5` - the Tailscale transport IP for host `pc` (which is not part of this flake). Verify with `ip addr show tailscale0` and `hostname` if you are on host `pc` with IP `100.64.0.5`.
 
-This module is applied to **every host** via
-`outputs.nixosModules.common` → `modules/nixos/common/default.nix`.
+This module is applied to **every host** via `outputs.nixosModules.common` -> `modules/nixos/common/default.nix`.
 
 ---
 
 ## Querying Remote Journals
 
-All queries run **on `sid`**, reading from `/var/log/journal/remote/`.
+All queries run **on `pc`**, reading from `/var/log/journal/remote/`.
 
 ### General pattern
 
@@ -293,13 +268,10 @@ journalctl \
 ```
 
 This works because:
-1. `rx4` uploads its journal to `sid:19532`.
-2. The receiver stores it under `/var/log/journal/remote/` with
-   `SplitMode=host`, preserving the `_HOSTNAME` field.
+1. `rx4` uploads its journal to `pc:19532`.
+2. The receiver stores it under `/var/log/journal/remote/` with `SplitMode=host`, preserving the `_HOSTNAME` field.
 3. `_HOSTNAME=rx4` narrows to that host's entries.
-4. `SYSTEMD_UNIT=podman-open-webui.service` targets the OCI container
-   unit for Open WebUI (defined in
-   `hosts/rx4/services/open-webui-oci.nix`).
+4. `SYSTEMD_UNIT=podman-open-webui.service` targets the OCI container unit for Open WebUI (defined in `hosts/rx4/services/open-webui-oci.nix`).
 
 ### Other useful queries
 
@@ -308,30 +280,24 @@ This works because:
 journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 --no-pager -S "1 hour ago"
 
 # Forgejo on rx4
-journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 \
-  SYSTEMD_UNIT=forgejo.service --no-pager -n 50
+journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 SYSTEMD_UNIT=forgejo.service --no-pager -n 50
 
 # Vaultwarden on rx4
-journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 \
-  SYSTEMD_UNIT=vaultwarden.service --no-pager -n 50
+journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 SYSTEMD_UNIT=vaultwarden.service --no-pager -n 50
 
 # Nginx on rx4
-journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 \
-  SYSTEMD_UNIT=nginx.service --no-pager -n 50
+journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 SYSTEMD_UNIT=nginx.service --no-pager -n 50
 
 # Matrix-synapse on sid
-journalctl -D /var/log/journal/remote/ _HOSTNAME=sid \
-  SYSTEMD_UNIT=matrix-synapse.service --no-pager -n 50
+journalctl -D /var/log/journal/remote/ _HOSTNAME=sid SYSTEMD_UNIT=matrix-synapse.service --no-pager -n 50
 
 # Follow logs live
-journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 \
-  SYSTEMD_UNIT=podman-open-webui.service -f
+journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 SYSTEMD_UNIT=podman-open-webui.service -f
 ```
 
 ### OCI container unit names
 
-Containers managed by `virtualisation.oci-containers` get units named
-`podman-<container-attr>.service`:
+Containers managed by `virtualisation.oci-containers` get units named `podman-<container-attr>.service`:
 
 | Container attr         | Unit name                              | Host |
 |------------------------|----------------------------------------|------|
@@ -345,8 +311,7 @@ Containers managed by `virtualisation.oci-containers` get units named
 
 ## Permissions
 
-User `sid` must be in both `systemd-journal` and
-`systemd-journal-remote` to read `/var/log/journal/remote/`:
+User `sid` must be in both `systemd-journal` and `systemd-journal-remote` to read `/var/log/journal/remote/`:
 
 ```nix
 users.users.sid.extraGroups = [
@@ -355,7 +320,7 @@ users.users.sid.extraGroups = [
 ];
 ```
 
-Verify with `id sid` on `sid`.
+Verify with `id sid` on `pc`.
 
 ---
 

From d6f7ce6d46177e62624de036612fa028b4384daf Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 00:43:49 +0200
Subject: [PATCH 14/54] disable journald remote for now

---
 modules/nixos/common/default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/nixos/common/default.nix b/modules/nixos/common/default.nix
index 10028cd..075c2dd 100644
--- a/modules/nixos/common/default.nix
+++ b/modules/nixos/common/default.nix
@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ./journald.nix
+    # ./journald.nix # FIXME: start systemd-journal-upload.service after tailscaled
     ./nix.nix
     ./overlays.nix
 

From de573124cef800af2d08593cf264fc08a8c443ae Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 12:56:26 +0200
Subject: [PATCH 15/54] add mcp log server via journald

---
 hosts/rx4/services/default.nix                |  1 +
 hosts/sid/services/default.nix                |  1 +
 modules/nixos/common/default.nix              |  1 -
 modules/nixos/default.nix                     |  2 +
 modules/nixos/journald-remote/default.nix     | 57 ++++++++++++++++++
 modules/nixos/journald-remote/pyproject.toml  |  9 +++
 modules/nixos/journald-remote/server.py       | 60 +++++++++++++++++++
 .../default.nix}                              |  2 +-
 8 files changed, 131 insertions(+), 2 deletions(-)
 create mode 100644 modules/nixos/journald-remote/default.nix
 create mode 100644 modules/nixos/journald-remote/pyproject.toml
 create mode 100644 modules/nixos/journald-remote/server.py
 rename modules/nixos/{common/journald.nix => journald-upload/default.nix} (52%)

diff --git a/hosts/rx4/services/default.nix b/hosts/rx4/services/default.nix
index 8db7081..3814e36 100644
--- a/hosts/rx4/services/default.nix
+++ b/hosts/rx4/services/default.nix
@@ -10,6 +10,7 @@
     inputs.clients.nixosModules.syncthing
 
     outputs.nixosModules.tailscale
+    outputs.nixosModules.journald-upload
 
     ./forgejo.nix
     ./jirafeau.nix
diff --git a/hosts/sid/services/default.nix b/hosts/sid/services/default.nix
index 7ca9678..521af0b 100644
--- a/hosts/sid/services/default.nix
+++ b/hosts/sid/services/default.nix
@@ -9,6 +9,7 @@
     inputs.synix.nixosModules.openssh
 
     outputs.nixosModules.tailscale
+    outputs.nixosModules.journald-remote
 
     ./headscale.nix
     ./mailserver.nix
diff --git a/modules/nixos/common/default.nix b/modules/nixos/common/default.nix
index 075c2dd..eba84dd 100644
--- a/modules/nixos/common/default.nix
+++ b/modules/nixos/common/default.nix
@@ -2,7 +2,6 @@
 
 {
   imports = [
-    # ./journald.nix # FIXME: start systemd-journal-upload.service after tailscaled
     ./nix.nix
     ./overlays.nix
 
diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix
index f831ea8..2e35096 100644
--- a/modules/nixos/default.nix
+++ b/modules/nixos/default.nix
@@ -5,6 +5,8 @@
   forgejo = import ./forgejo;
   forgejo-runner = import ./forgejo-runner;
   gnome = import ./gnome;
+  journald-remote = import ./journald-remote;
+  journald-upload = import ./journald-upload;
   monero = import ./monero;
   rsshub-oci = import ./rsshub-oci;
   tailscale = import ./tailscale;
diff --git a/modules/nixos/journald-remote/default.nix b/modules/nixos/journald-remote/default.nix
new file mode 100644
index 0000000..17012a6
--- /dev/null
+++ b/modules/nixos/journald-remote/default.nix
@@ -0,0 +1,57 @@
+{
+  pkgs,
+  lib,
+  ...
+}:
+
+let
+  python = pkgs.python3Packages;
+
+  mcp-log-server = python.buildPythonApplication {
+    pname = "mcp-log-server";
+    version = "1.0.0";
+
+    src = ./.;
+
+    pyproject = true;
+    build-system = [ python.setuptools ];
+
+    propagatedBuildInputs = with python; [
+      fastmcp
+    ];
+
+    meta.mainProgram = "mcp-log-server";
+  };
+in
+{
+  services.journald.remote = {
+    enable = true;
+    listen = "http";
+    port = 19532;
+    settings.Remote.SplitMode = "host";
+  };
+
+  users.users.sid.extraGroups = [
+    "systemd-journal"
+    "systemd-journal-remote"
+  ];
+
+  systemd.services.mcp-log-server = {
+    description = "AI Log Access MCP Server";
+    after = [
+      "network.target"
+      "multi-user.target"
+      "systemd-journald.service"
+    ];
+    wantedBy = [ "multi-user.target" ];
+
+    script = lib.getExe mcp-log-server;
+
+    serviceConfig = {
+      User = "root";
+      Group = "root";
+      Environment = "PYTHONUNBUFFERED=1";
+      Restart = "on-failure";
+    };
+  };
+}
diff --git a/modules/nixos/journald-remote/pyproject.toml b/modules/nixos/journald-remote/pyproject.toml
new file mode 100644
index 0000000..809656b
--- /dev/null
+++ b/modules/nixos/journald-remote/pyproject.toml
@@ -0,0 +1,9 @@
+[build-system]
+requires = ["setuptools"]
+
+[project]
+name = "mcp-log-server"
+version = "1.0.0"
+
+[project.scripts]
+mcp-log-server = "server:main"
diff --git a/modules/nixos/journald-remote/server.py b/modules/nixos/journald-remote/server.py
new file mode 100644
index 0000000..39c8f28
--- /dev/null
+++ b/modules/nixos/journald-remote/server.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+import json
+import subprocess
+from fastmcp import FastMCP
+
+mcp = FastMCP('ClusterLogQuery')
+
+@mcp.tool()
+async def search_cluster_logs(keyword: str, unit: str = '*', limit: int = 500):
+    """
+    Search centralized systemd logs collected on this server.
+
+    Args:
+        keyword: The string to search for (e.g. 'oom-kill', 'failed').
+        unit: The systemd unit to filter (e.g. 'sshd'). If '*', matches all.
+        limit: The number of log lines to return (Max 1000).
+    """
+    
+    limit = min(max(limit, 10), 1000)
+
+    cmd = [
+        'journalctl',
+        '-n', str(limit),
+        '-o', 'json',
+        '--no-pager',
+        '--directory', '/var/log/journal/remote'
+    ]
+
+    if unit != '*':
+        cmd.append(f'UNIT={unit}')
+    if keyword:
+        cmd.append(keyword)
+
+    try:
+        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, _ = proc.communicate()
+
+        logs = []
+        for line in stdout.decode('utf-8').strip().split('\n'):
+            if not line: continue
+            try:
+                log = json.loads(line)
+                logs.append({
+                    'timestamp': log.get('TIME_ISO8601'),
+                    'hostname': log.get('_HOSTNAME'),
+                    'unit': log.get('_COMM'),
+                    'message': log.get('MESSAGE')
+                })
+            except: pass
+
+        return json.dumps(logs)
+
+    except Exception as e:
+        return f'Error querying logs: {str(e)}'
+
+def main():
+    mcp.run(transport='sse', host='0.0.0.0', port=9500)
+
+if __name__ == '__main__':
+    main()
diff --git a/modules/nixos/common/journald.nix b/modules/nixos/journald-upload/default.nix
similarity index 52%
rename from modules/nixos/common/journald.nix
rename to modules/nixos/journald-upload/default.nix
index d31ada4..9d8c0c5 100644
--- a/modules/nixos/common/journald.nix
+++ b/modules/nixos/journald-upload/default.nix
@@ -1,6 +1,6 @@
 {
   services.journald.upload = {
     enable = true;
-    settings.Upload.URL = "http://100.64.0.5:19532";
+    settings.Upload.URL = "http://100.64.0.6:19532";
   };
 }

From f6aca90009570d7aa1d88fbf1cfc50edcc993d2b Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 13:00:26 +0200
Subject: [PATCH 16/54] disable librechat

---
 hosts/rx4/services/default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hosts/rx4/services/default.nix b/hosts/rx4/services/default.nix
index 3814e36..2dd7f7c 100644
--- a/hosts/rx4/services/default.nix
+++ b/hosts/rx4/services/default.nix
@@ -14,7 +14,7 @@
 
     ./forgejo.nix
     ./jirafeau.nix
-    ./librechat-oci.nix
+    # ./librechat-oci.nix
     ./miniflux.nix
     ./netdata.nix
     ./nginx.nix

From 72265ffed718fb722aeb3c5bc55b676833422139 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 13:06:13 +0200
Subject: [PATCH 17/54] start systemd-journal-upload after tailscaled

---
 modules/nixos/journald-upload/default.nix | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/modules/nixos/journald-upload/default.nix b/modules/nixos/journald-upload/default.nix
index 9d8c0c5..02d6b43 100644
--- a/modules/nixos/journald-upload/default.nix
+++ b/modules/nixos/journald-upload/default.nix
@@ -3,4 +3,15 @@
     enable = true;
     settings.Upload.URL = "http://100.64.0.6:19532";
   };
+
+  systemd.services.systemd-journal-upload = {
+    after = [
+      "network-online.target"
+      "tailscaled.service"
+    ];
+    wants = [
+      "network-online.target"
+      "tailscaled.service"
+    ];
+  };
 }

From a5323520e21f3a07168a155566e99d1574d5d256 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 13:10:26 +0200
Subject: [PATCH 18/54] rm journald groups from user sid

---
 modules/nixos/journald-remote/default.nix | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/modules/nixos/journald-remote/default.nix b/modules/nixos/journald-remote/default.nix
index 17012a6..797eef3 100644
--- a/modules/nixos/journald-remote/default.nix
+++ b/modules/nixos/journald-remote/default.nix
@@ -31,11 +31,6 @@ in
     settings.Remote.SplitMode = "host";
   };
 
-  users.users.sid.extraGroups = [
-    "systemd-journal"
-    "systemd-journal-remote"
-  ];
-
   systemd.services.mcp-log-server = {
     description = "AI Log Access MCP Server";
     after = [

From 0c68ed16f7ac7db4cf02c54c0d7d463c7ee20162 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 14:03:35 +0200
Subject: [PATCH 19/54] journald-upload needs to wait for tailscaled

---
 modules/nixos/journald-upload/default.nix | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/modules/nixos/journald-upload/default.nix b/modules/nixos/journald-upload/default.nix
index 02d6b43..49c8907 100644
--- a/modules/nixos/journald-upload/default.nix
+++ b/modules/nixos/journald-upload/default.nix
@@ -6,12 +6,10 @@
 
   systemd.services.systemd-journal-upload = {
     after = [
-      "network-online.target"
       "tailscaled.service"
-    ];
-    wants = [
       "network-online.target"
-      "tailscaled.service"
     ];
+    wants = [ "network-online.target" ];
+    timeoutStartSec = 10; # Tailscale can be slow to start
   };
 }

From 00af06d380c2b78c9da20d0e751b974755429850 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 14:11:00 +0200
Subject: [PATCH 20/54] fix journald upload service config

---
 modules/nixos/journald-upload/default.nix | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/nixos/journald-upload/default.nix b/modules/nixos/journald-upload/default.nix
index 49c8907..dede56b 100644
--- a/modules/nixos/journald-upload/default.nix
+++ b/modules/nixos/journald-upload/default.nix
@@ -10,6 +10,8 @@
       "network-online.target"
     ];
     wants = [ "network-online.target" ];
-    timeoutStartSec = 10; # Tailscale can be slow to start
+    serviceConfig = {
+      TimeoutStartSec = 20; # Tailscale is slow
+    };
   };
 }

From 7e4f2a1a075508354cc6e56732e0811edbf41838 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 14:26:44 +0200
Subject: [PATCH 21/54] remove AGENTS.md

---
 AGENTS.md | 355 ------------------------------------------------------
 1 file changed, 355 deletions(-)
 delete mode 100644 AGENTS.md

diff --git a/AGENTS.md b/AGENTS.md
deleted file mode 100644
index 8defb09..0000000
--- a/AGENTS.md
+++ /dev/null
@@ -1,355 +0,0 @@
-# AGENTS.md
-
-This file teaches AI agents how to navigate and work with this NixOS configuration repository (`sid.ovh`).
-
----
-
-## Repository Overview
-
-This is a NixOS flake managing three hosts:
-
-| Host  | IP (Tailscale) | Role                                                 |
-|-------|----------------|------------------------------------------------------|
-| `sid` | `100.64.0.6`   | VPS - reverse proxy, mail, matrix, headscale         |
-| `rx4` | `100.64.0.10`  | Home server - open-webui, forgejo, vaultwarden, etc. |
-| `vde` | `100.64.0.1`   | Desktop / workstation (not in use at the moment)     |
-
-Deployment is done via `deploy-rs` through a Forgejo CI pipeline (`.forgejo/workflows/deploy-configs.yml`).
-
----
-
-## The `synix` Flake Input
-
-### What it is
-
-`synix` is a **NixOS library flake**, hosted at:
-
-```
-https://git.sid.ovh/sid/synix.git
-```
-
-It provides:
-- **NixOS modules** (`inputs.synix.nixosModules.*`): opinionated, reusable service configurations used heavily across all three hosts.
-- **Home Manager modules** (`inputs.synix.homeModules.*`): desktop / user-space configurations (Hyprland, nixvim, waybar, etc.).
-- **Packages** (`inputs.synix.packages.<system>.*`, also accessible via the `synix-packages` overlay as `pkgs.synix.*`): custom packages not in nixpkgs.
-- **Overlays** (`inputs.synix.overlays.*`): nixpkgs modifications.
-- **A utility library** (`inputs.synix.lib` / `lib.utils`): helper functions such as `mkVirtualHost` and `mkReverseProxyOption`.
-- **Templates**: starter flake templates for servers, desktops, VMs, etc.
-- **Apps**: convenience scripts (`deploy`, `rebuild`, `install`, etc.).
-
-### How it is consumed in `sid.ovh`
-
-In `flake.nix`:
-
-```nix
-inputs = {
-  synix.url = "git+https://git.sid.ovh/sid/synix.git?ref=release-25.11";
-  synix.inputs.nixpkgs.follows = "nixpkgs";
-  # development overrides (commented out normally):
-  # synix.url = "git+https://git.sid.ovh/sid/synix.git?ref=develop";
-  # synix.url = "git+file:///home/sid/src/synix";  # local checkout
-  ...
-};
-```
-
-The active branch is **`release-25.11`**, aligned with `nixpkgs` at `nixos-25.11`. A `develop` branch exists for unstable work. A local checkout can be used for testing by switching to the `git+file://` URL.
-
----
-
-## `synix` Module Reference
-
-All NixOS modules are under `modules/nixos/` in the synix repo and exposed as `inputs.synix.nixosModules.<name>`.
-
-### Infrastructure / base modules
-
-| Module name                        | Path in synix                     | Purpose |
-|------------------------------------|-----------------------------------|---------|
-| `common`                           | `modules/nixos/common/`           | Base system config (boot, networking, nix settings, zsh, sudo, htop, locale) |
-| `device.server`                    | `modules/nixos/device/server.nix` | Server profile (no GUI, sensible server defaults) |
-| `device.desktop`                   | `modules/nixos/device/desktop.nix`| Desktop profile |
-| `device.laptop`                    | `modules/nixos/device/laptop.nix` | Laptop profile |
-| `device.vm`                        | `modules/nixos/device/vm.nix`     | VM profile |
-| `sops`                             | `modules/nixos/sops/`             | sops-nix wiring (age keys, secret paths) |
-| `openssh`                          | `modules/nixos/openssh/`          | Hardened OpenSSH configuration |
-| `normalUsers`                      | `modules/nixos/normalUsers/`      | Declarative normal-user creation with SSH keys |
-| `tailscale`                        | `modules/nixos/tailscale/`        | Tailscale VPN with multiple tailnet support |
-| `nginx`                            | `modules/nixos/nginx/`            | Opinionated nginx base (ACME, SSL helpers) |
-
-### Service modules used in `sid.ovh`
-
-| Module name          | Path in synix                        | Used by host | Notes |
-|----------------------|--------------------------------------|--------------|-------|
-| `headscale`          | `modules/nixos/headscale/`           | sid          | Self-hosted Tailscale control plane |
-| `headplane`          | `modules/nixos/headplane/`           | sid          | Headscale web UI |
-| `mailserver`         | `modules/nixos/mailserver/`          | sid          | Wraps `nixos-mailserver` |
-| `matrix-synapse`     | `modules/nixos/matrix-synapse/`      | sid          | Matrix homeserver + bridges (WhatsApp, Signal) + LiveKit |
-| `maubot`             | `modules/nixos/maubot/`              | sid          | Matrix bot framework |
-| `baibot`             | `modules/nixos/baibot/`              | sid          | AI Matrix bot |
-| `coturn`             | `modules/nixos/coturn/`              | sid          | TURN/STUN server (Matrix calls) |
-| `radicale`           | `modules/nixos/radicale/`            | sid          | CalDAV/CardDAV server |
-| `rss-bridge`         | `modules/nixos/rss-bridge/`          | sid          | RSS-Bridge PHP app |
-| `miniflux`           | `modules/nixos/miniflux/`            | rx4          | RSS reader |
-| `jirafeau`           | `modules/nixos/jirafeau/`            | rx4          | File sharing |
-| `open-webui-oci`     | `modules/nixos/open-webui-oci/`      | rx4          | Open WebUI via Podman OCI container |
-| `mcpo`               | `modules/nixos/mcpo/`                | rx4          | MCP-to-OpenAPI proxy (AI tool server) |
-| `print-server`       | `modules/nixos/print-server/`        | rx4          | CUPS print server |
-| `virtualisation`     | `modules/nixos/virtualisation/`      | vde          | VFIO / KVM / QEMU / kvmfr setup |
-
-### Other available modules (not currently active in `sid.ovh`)
-
-`audio`, `bluetooth`, `amd`, `nvidia`, `jellyfin`, `i2pd`, `ftp-webserver`, `webPage`, `windows-oci`, `librechat-oci`, `nostr-relay`, `ollama`, `cifsMount`, `hyprland` (NixOS-level), `normalUsers`.
-
----
-
-## `synix` Packages (`pkgs.synix.*`)
-
-Custom packages provided by synix and available via the `synix-packages` overlay (applied in `modules/nixos/common/overlays.nix`):
-
-| Package name                  | What it is |
-|-------------------------------|------------|
-| `pkgs.synix.mcpo`             | MCP-to-OpenAPI proxy server |
-| `pkgs.synix.fetcher-mcp`      | MCP tool: web page fetcher |
-| `pkgs.synix.baibot`           | AI Matrix bot |
-| `pkgs.synix.jirafeau`         | Jirafeau file-sharing webapp |
-| `pkgs.synix.jirafeau-cli`     | CLI client for Jirafeau |
-| `pkgs.synix.marker-pdf`       | PDF-to-markdown converter |
-| `pkgs.synix.mcpo`             | MCP proxy |
-| `pkgs.synix.systemctl-tui`    | TUI for systemctl |
-| `pkgs.synix.quicknote`        | Quick note shell script |
-| `pkgs.synix.synix-docs`       | Built MkDocs documentation site |
-| `pkgs.synix.bulk-rename`      | Batch file renaming script |
-| `pkgs.synix.pyman`            | Python man-page helper |
-
-The full list lives in `pkgs/default.nix` in the synix repo. All packages are also accessible as `inputs.synix.packages.<system>.<name>`.
-
----
-
-## `synix` Utility Library (`lib.utils`)
-
-Exposed as `inputs.synix.lib` and merged into the flake's `lib` in `flake.nix`:
-
-```nix
-lib = nixpkgs.lib.extend (final: prev: inputs.synix.lib or { });
-```
-
-Key helpers (defined in `lib/utils.nix`):
-
-| Function                  | What it does |
-|---------------------------|--------------|
-| `lib.utils.mkVirtualHost` | Builds a standard nginx `virtualHosts` entry with optional SSL, upstream address, port, and extra config |
-| `lib.utils.mkReverseProxyOption` | Generates a standard NixOS option set for enabling a reverse-proxy sub-config on a module |
-
-Usage example from `hosts/sid/services/nginx.nix`:
-
-```nix
-virtualHosts."${constants.services.forgejo.fqdn}" = mkVirtualHost {
-  inherit ssl;
-  address = constants.hosts.rx4.ip;
-  port    = constants.services.forgejo.port;
-};
-```
-
----
-
-## How `synix` Modules Are Imported
-
-In each host's `default.nix`, synix modules are imported directly from the `inputs` special arg:
-
-```nix
-# hosts/rx4/default.nix
-imports = [
-  inputs.synix.nixosModules.common          # base system config
-  inputs.synix.nixosModules.device.server   # server profile
-  inputs.synix.nixosModules.openssh         # hardened SSH
-  # service-specific modules in hosts/rx4/services/
-];
-```
-
-Service files then add more synix modules as needed:
-
-```nix
-# hosts/rx4/services/open-webui-oci.nix
-imports = [
-  inputs.synix.nixosModules.open-webui-oci
-  inputs.synix.nixosModules.mcpo
-];
-```
-
----
-
-## Overlays Applied
-
-`modules/nixos/common/overlays.nix` applies five overlays to every host:
-
-| Overlay name            | Source           | Accessible as        |
-|-------------------------|------------------|----------------------|
-| `synix-packages`        | `synix`          | `pkgs.synix.*`       |
-| `local-packages`        | `pkgs/` in repo  | `pkgs.local.*`       |
-| `modifications`         | `overlays/`      | replaces pkgs in place |
-| `old-stable-packages`   | `nixpkgs-25.05`  | `pkgs.old-stable.*`  |
-| `unstable-packages`     | `nixos-unstable` | `pkgs.unstable.*`    |
-
-`synix` also supplies its own `modifications` overlay (`inputs.synix.overlays.modifications`), merged into the local `modifications` overlay in `overlays/default.nix`.
-
----
-
-## Centralized Logging Architecture
-
-All hosts ship their systemd journal to a **central receiver running on `sid`** over HTTP using `systemd-journal-remote` / `systemd-journal-upload`.
-
-### Receiver (local host `pc` which is not part of this flake)
-
-```nix
-services.journald.remote = {
-  enable   = true;
-  listen   = "http";
-  port     = 19532;
-  settings.Remote.SplitMode = "host";  # one journal dir per sending host
-};
-
-users.users.sid.extraGroups = [
-  "systemd-journal"
-  "systemd-journal-remote"
-];
-```
-
-Key points:
-- Listens on **`http://0.0.0.0:19532`** (plain HTTP, Tailscale-only network).
-- `SplitMode = "host"` stores each remote host's entries under
-  `/var/log/journal/remote/`, separated by hostname.
-
-### Senders (all hosts via `modules/nixos/common/journald.nix`)
-
-```nix
-services.journald.upload = {
-  enable = true;
-  settings.Upload.URL = "http://100.64.0.5:19532";
-};
-```
-
-> **Note:** The upload URL uses `100.64.0.5` - the Tailscale transport IP for host `pc` (which is not part of this flake). Verify with `ip addr show tailscale0` and `hostname` if you are on host `pc` with IP `100.64.0.5`.
-
-This module is applied to **every host** via `outputs.nixosModules.common` -> `modules/nixos/common/default.nix`.
-
----
-
-## Querying Remote Journals
-
-All queries run **on `pc`**, reading from `/var/log/journal/remote/`.
-
-### General pattern
-
-```bash
-journalctl -D /var/log/journal/remote/ \
-  _HOSTNAME=<hostname> \
-  [SYSTEMD_UNIT=<unit>] \
-  [other filters…] \
-  --no-pager [-n <lines>] [-f] [-S <since>] [-U <until>]
-```
-
-| Flag / Field                              | Purpose |
-|-------------------------------------------|---------|
-| `-D /var/log/journal/remote/`             | Read the remote journal directory |
-| `_HOSTNAME=rx4`                           | Filter to entries from host `rx4` |
-| `SYSTEMD_UNIT=podman-open-webui.service`  | Filter to a specific systemd unit |
-| `--no-pager`                              | Don't page output; good for scripts |
-| `-n 20`                                   | Show last 20 lines |
-| `-f`                                      | Follow (tail) mode |
-| `-S` / `-U`                               | Since / Until time bounds |
-
-### Canonical example — last 20 lines of open-webui on rx4
-
-```bash
-journalctl \
-  -D /var/log/journal/remote/ \
-  _HOSTNAME=rx4 \
-  SYSTEMD_UNIT=podman-open-webui.service \
-  --no-pager -n 20
-```
-
-This works because:
-1. `rx4` uploads its journal to `pc:19532`.
-2. The receiver stores it under `/var/log/journal/remote/` with `SplitMode=host`, preserving the `_HOSTNAME` field.
-3. `_HOSTNAME=rx4` narrows to that host's entries.
-4. `SYSTEMD_UNIT=podman-open-webui.service` targets the OCI container unit for Open WebUI (defined in `hosts/rx4/services/open-webui-oci.nix`).
-
-### Other useful queries
-
-```bash
-# All logs from rx4 in the last hour
-journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 --no-pager -S "1 hour ago"
-
-# Forgejo on rx4
-journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 SYSTEMD_UNIT=forgejo.service --no-pager -n 50
-
-# Vaultwarden on rx4
-journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 SYSTEMD_UNIT=vaultwarden.service --no-pager -n 50
-
-# Nginx on rx4
-journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 SYSTEMD_UNIT=nginx.service --no-pager -n 50
-
-# Matrix-synapse on sid
-journalctl -D /var/log/journal/remote/ _HOSTNAME=sid SYSTEMD_UNIT=matrix-synapse.service --no-pager -n 50
-
-# Follow logs live
-journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 SYSTEMD_UNIT=podman-open-webui.service -f
-```
-
-### OCI container unit names
-
-Containers managed by `virtualisation.oci-containers` get units named `podman-<container-attr>.service`:
-
-| Container attr         | Unit name                              | Host |
-|------------------------|----------------------------------------|------|
-| `open-webui`           | `podman-open-webui.service`            | rx4  |
-| `rsshub-rsshub`        | `podman-rsshub-rsshub.service`         | rx4  |
-| `rsshub-redis`         | `podman-rsshub-redis.service`          | rx4  |
-| `rsshub-browserless`   | `podman-rsshub-browserless.service`    | rx4  |
-| `rsshub-real-browser`  | `podman-rsshub-real-browser.service`   | rx4  |
-
----
-
-## Permissions
-
-User `sid` must be in both `systemd-journal` and `systemd-journal-remote` to read `/var/log/journal/remote/`:
-
-```nix
-users.users.sid.extraGroups = [
-  "systemd-journal"
-  "systemd-journal-remote"
-];
-```
-
-Verify with `id sid` on `pc`.
-
----
-
-## Key File Locations
-
-| Path                                    | Purpose |
-|-----------------------------------------|---------|
-| `flake.nix`                             | Flake inputs, host configs, deploy nodes |
-| `constants.nix`                         | Canonical IPs, FQDNs, service ports |
-| `modules/nixos/common/journald.nix`     | Enables `journald.upload` on all hosts |
-| `modules/nixos/common/default.nix`      | Imports all common modules |
-| `modules/nixos/common/overlays.nix`     | Applies all five overlays |
-| `hosts/<name>/default.nix`             | Per-host top-level import list |
-| `hosts/<name>/services/`               | Per-host service configurations |
-| `hosts/<name>/secrets/secrets.yaml`    | sops-encrypted secrets for that host |
-| `overlays/default.nix`                 | Local overlay definitions |
-| `pkgs/`                                 | Local packages (`pkgs.local.*`) |
-| `users/sid/`                            | User account + SSH public key |
-
----
-
-## Troubleshooting
-
-| Symptom | Check |
-|---------|-------|
-| No journal entries for a host | `systemctl status systemd-journal-upload` on the source host |
-| Permission denied reading remote journal | `id sid` on `sid` — needs `systemd-journal` + `systemd-journal-remote` |
-| Wrong unit name | `systemctl list-units --type=service` on the source host |
-| Receiver unreachable | `ss -tlnp \| grep 19532` on `sid` |
-| Upload URL IP mismatch | Upload uses `100.64.0.5`; verify `ip addr show tailscale0` on `sid` |
-| synix module not found | Check the branch in `flake.nix`; run `nix flake update synix` to refresh |
-| Want to test a synix change locally | Switch input URL to `git+file:///home/sid/src/synix` |

From 35df4f1551c863ce0c2a17154f630337f3164eb6 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 18:48:43 +0200
Subject: [PATCH 22/54] rm journald-remote for now

---
 hosts/rx4/services/default.nix | 2 +-
 hosts/sid/services/default.nix | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/hosts/rx4/services/default.nix b/hosts/rx4/services/default.nix
index 2dd7f7c..dab1f79 100644
--- a/hosts/rx4/services/default.nix
+++ b/hosts/rx4/services/default.nix
@@ -10,7 +10,7 @@
     inputs.clients.nixosModules.syncthing
 
     outputs.nixosModules.tailscale
-    outputs.nixosModules.journald-upload
+    # outputs.nixosModules.journald-upload # FIXME
 
     ./forgejo.nix
     ./jirafeau.nix
diff --git a/hosts/sid/services/default.nix b/hosts/sid/services/default.nix
index 521af0b..b40b165 100644
--- a/hosts/sid/services/default.nix
+++ b/hosts/sid/services/default.nix
@@ -9,7 +9,7 @@
     inputs.synix.nixosModules.openssh
 
     outputs.nixosModules.tailscale
-    outputs.nixosModules.journald-remote
+    # outputs.nixosModules.journald-remote
 
     ./headscale.nix
     ./mailserver.nix

From 966a4c473a9963136e0eb4fbd50747dc4a531528 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 18:50:14 +0200
Subject: [PATCH 23/54] re-enable librechat

---
 hosts/rx4/services/default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hosts/rx4/services/default.nix b/hosts/rx4/services/default.nix
index dab1f79..1014d6f 100644
--- a/hosts/rx4/services/default.nix
+++ b/hosts/rx4/services/default.nix
@@ -14,7 +14,7 @@
 
     ./forgejo.nix
     ./jirafeau.nix
-    # ./librechat-oci.nix
+    ./librechat-oci.nix
     ./miniflux.nix
     ./netdata.nix
     ./nginx.nix

From f97186cefc13aa6ac5a736d7d95c5cd58d903dcd Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 19:32:47 +0200
Subject: [PATCH 24/54] update synix input

---
 flake.lock | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/flake.lock b/flake.lock
index b4c5943..b5376a3 100644
--- a/flake.lock
+++ b/flake.lock
@@ -5199,11 +5199,11 @@
         "stylix": "stylix_6"
       },
       "locked": {
-        "lastModified": 1779122723,
-        "narHash": "sha256-hEL93URxv2e1vhKxA++m0rszGgZUmWKKPFxa4t1LIi4=",
+        "lastModified": 1779211946,
+        "narHash": "sha256-PZsNGNnwgiFh8sm8POIEeDf/JVB0+lhGEApp/wS4XCk=",
         "ref": "release-25.11",
-        "rev": "7d73c46b94fa6a7562b7b3ef300f39aa71734d26",
-        "revCount": 85,
+        "rev": "fe6b0d6c47b29b95697c310997f19bf70a29e8a6",
+        "revCount": 89,
         "type": "git",
         "url": "https://git.sid.ovh/sid/synix.git"
       },

From d038353260d1e384ef63b8efa0f3f638563b4ca7 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 19:43:48 +0200
Subject: [PATCH 25/54] librechat: add jwt tokens

---
 hosts/rx4/secrets/secrets.yaml       |  7 +++++--
 hosts/rx4/services/librechat-oci.nix | 11 ++++++++++-
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/hosts/rx4/secrets/secrets.yaml b/hosts/rx4/secrets/secrets.yaml
index eaa951b..0589c8a 100644
--- a/hosts/rx4/secrets/secrets.yaml
+++ b/hosts/rx4/secrets/secrets.yaml
@@ -17,6 +17,9 @@ vaultwarden:
     admin-token: ENC[AES256_GCM,data:HhD0xNZ/Ep7pCOX1j6p/M/ZZ3gs=,iv:7QT71KlYz+HQYBhiRavpiXS9sNS2PoJiM/WkxM3Hk/g=,tag:SYTRWpyA2+WMSMiRM8mvew==,type:str]
     smtp-password: ENC[AES256_GCM,data:eQo7op5+74EID6689hL0/J1pq2s=,iv:JqrEqxabWGydRuJJ/27e1q+4YnQhTQ1bKRSsOvjQ+bE=,tag:weqnrhqK+LGEfAacBcuPUA==,type:str]
 hetzner-api-key: ENC[AES256_GCM,data:casjNOXzuQDWgnSFftbBMygA8kGpGiZDqup08faWO9kfjvgOyWOXeqPd2VA1ND8yfM2LvoLYvPs6gUWtni2ldQ==,iv:p2W24uhJgBvpi3g4+cHw0/XbbTM5oYCPHreMBUR4CNs=,tag:lpwjZGoJe/91+CHX/hAkKA==,type:str]
+librechat:
+    jwt-token: ENC[AES256_GCM,data:/NZfZsvg4mDCgB3prDbyPEXIOuN/WSWP3dmSYlvTn7TRSO6oKtnSz20zC0FLvwDAn5QvBYvBKF+LnYjXJeUNkw==,iv:vgESrSyy6IoCMNHG0eL05c9k7Z+tdNb88u5sz+4cYCI=,tag:/WPi7v3hrgKPgwdV0ZE2Bg==,type:str]
+    jwt-refresh-token: ENC[AES256_GCM,data:w/gHj+dXgGk4BcT1ueIdVujjgYWzUGgY8TG/ci8WUDkU12aPcqi6Kuqe55Did0s2AH1Am+1cToy/Q8QiOnt7QQ==,iv:5LJ8ht5yZlql+TayLwU3CNhAd9DUjGw8sRamwbwm7JA=,tag:GJ9zaU7p36oZsOnXeifyyw==,type:str]
 sops:
     age:
         - recipient: age19yeqvv28fgrtk6jsh3xyaf0lch86kna6rcz4dwe962yyyyevu30sx474xy
@@ -37,7 +40,7 @@ sops:
             NE5yK3ZaOG5PdXNSUnlIUmFSSmRFancKk57hCmo79HvI3hzzgQvgOK7oK5/dcQR8
             f3R4OGF5+212VXEHR/hAEbKzV7CY4y6HhFyrGZ9bUKm1RrxtnVqUyA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-05-17T20:34:39Z"
-    mac: ENC[AES256_GCM,data:lSSotIfDcS6oJpSDSe2hLx1M9L8a+bjkPstcPv1h2ohSiOu8WGAwTy4lsKD1n9rnhTzFmMqi2Xgh4K0n3WiqWFBeNcA6UeM7+a6PcDtUeCC3JKsP/XZvCoPq5uBwUWcovRSm4UElaL5MteZkV3e+qZWeUpZCTWWWEjYBYnHPLpQ=,iv:t4Up4DuTuQyQQNa7lmZK6kt5O0/aShXSF2XBj9Y6/z8=,tag:oNmP8e7jEZ3ttPkwXkWSZw==,type:str]
+    lastmodified: "2026-05-19T17:41:34Z"
+    mac: ENC[AES256_GCM,data:UPpz15iUrysYMovpNFLGyAnw1TZ8mmGUo4HDCPlyGI8ADo0v8RfhGjBL/0H0EIA4UX6D+EfRpp4wNacvTdgapQmKHd4H2Q4uDxRUJAHaAkBQVljiuTAEf+8aF/99/U5nEoYrUba15zV8WOONDD7CnzMm+fOosjJuZwKd+akt0KQ=,iv:+nzB0ffdB4PGsnaQ5x9WzWrhfcVQqv1WENUEJOAYbyE=,tag:VvEgvSyBUZixRK3MgCpFvQ==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.12.1
diff --git a/hosts/rx4/services/librechat-oci.nix b/hosts/rx4/services/librechat-oci.nix
index 07ab8d3..80db7a1 100644
--- a/hosts/rx4/services/librechat-oci.nix
+++ b/hosts/rx4/services/librechat-oci.nix
@@ -6,7 +6,6 @@
 }:
 
 let
-  inherit (constants) domain;
   inherit (constants.hosts.rx4) ip;
   inherit (constants.services.librechat-oci) fqdn port;
 in
@@ -19,6 +18,7 @@ in
     enable = true;
     inherit port;
     externalUrl = "https://${fqdn}";
+    environmentFile = config.sops.templates.librechat-env-file.path;
   };
 
   services.nginx.virtualHosts."${fqdn}" = {
@@ -41,4 +41,13 @@ in
     postRun = "systemctl restart podman-librechat.service";
     group = "nginx";
   };
+
+  sops = {
+    secrets."librechat/jwt-token" = { }; # openssl rand -hex 32
+    secrets."librechat/jwt-refresh-token" = { }; # openssl rand -hex 32
+    templates.librechat-env-file.content = ''
+      JET_TOKEN=${config.sops.placeholder."librechat/jwt-token"}
+      JET_REFRESH_TOKEN=${config.sops.placeholder."librechat/jwt-refresh-token"}
+    '';
+  };
 }

From afb49c74073a597f9a98022068292826fed2ccb4 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 20:59:18 +0200
Subject: [PATCH 26/54] new librechat api

---
 flake.lock                           |  8 +++----
 hosts/rx4/secrets/secrets.yaml       |  7 ++++--
 hosts/rx4/services/librechat-oci.nix | 34 +++++++++++++++++++++++-----
 3 files changed, 37 insertions(+), 12 deletions(-)

diff --git a/flake.lock b/flake.lock
index b5376a3..8036d42 100644
--- a/flake.lock
+++ b/flake.lock
@@ -5199,11 +5199,11 @@
         "stylix": "stylix_6"
       },
       "locked": {
-        "lastModified": 1779211946,
-        "narHash": "sha256-PZsNGNnwgiFh8sm8POIEeDf/JVB0+lhGEApp/wS4XCk=",
+        "lastModified": 1779216861,
+        "narHash": "sha256-fmgvPXXlrVJhIRGNjuYbrr5sHNFxFdlU3kdkmmzPirY=",
         "ref": "release-25.11",
-        "rev": "fe6b0d6c47b29b95697c310997f19bf70a29e8a6",
-        "revCount": 89,
+        "rev": "a7daa3b9f33cf266218a04c41f4c45af1c7e5207",
+        "revCount": 92,
         "type": "git",
         "url": "https://git.sid.ovh/sid/synix.git"
       },
diff --git a/hosts/rx4/secrets/secrets.yaml b/hosts/rx4/secrets/secrets.yaml
index 0589c8a..2c01e49 100644
--- a/hosts/rx4/secrets/secrets.yaml
+++ b/hosts/rx4/secrets/secrets.yaml
@@ -20,6 +20,9 @@ hetzner-api-key: ENC[AES256_GCM,data:casjNOXzuQDWgnSFftbBMygA8kGpGiZDqup08faWO9k
 librechat:
     jwt-token: ENC[AES256_GCM,data:/NZfZsvg4mDCgB3prDbyPEXIOuN/WSWP3dmSYlvTn7TRSO6oKtnSz20zC0FLvwDAn5QvBYvBKF+LnYjXJeUNkw==,iv:vgESrSyy6IoCMNHG0eL05c9k7Z+tdNb88u5sz+4cYCI=,tag:/WPi7v3hrgKPgwdV0ZE2Bg==,type:str]
     jwt-refresh-token: ENC[AES256_GCM,data:w/gHj+dXgGk4BcT1ueIdVujjgYWzUGgY8TG/ci8WUDkU12aPcqi6Kuqe55Did0s2AH1Am+1cToy/Q8QiOnt7QQ==,iv:5LJ8ht5yZlql+TayLwU3CNhAd9DUjGw8sRamwbwm7JA=,tag:GJ9zaU7p36oZsOnXeifyyw==,type:str]
+    creds-key: ENC[AES256_GCM,data:EljwEqFByJaOjd8lRFGwo/FyXHUtl5an0xS1EjRe+kmpo5z4P33EUKbMeeIl69rEcziMHZQLiadzSEcS2cb2uA==,iv:sidBN6VTBeFhMUtN67HZuyofiXCeGFG4tuMRckLZv84=,tag:n7vI8LuPgER3J6r6Q6Jkjg==,type:str]
+    creds-iv: ENC[AES256_GCM,data:oc0sPm5RM/7AbH3vdDLJ2m0q6C7eAAME0GPbiojHZUspP8Cto5QX5WKnUjUVLLcvgK+t6pnu7BEmAuD3PLr11A==,iv:Z6XJmlqv0ULFiwqHyRO5v7lb/iyv4g9aSTV4xw9VTXU=,tag:7kptbQwc6lBZ70aXw7wOVA==,type:str]
+    meili-master-key: ENC[AES256_GCM,data:eugFl40a6Ks3ba8hcn83WS76AwA0TXkhu3K4gSrbNHtXRliLQCWhGTEvoaQSeb7whmpszh4zh8cKSxByBdhJiQ==,iv:rrWlcVyBlrE5dnBBFWjheIo6SgQTbkzqskGQvQczR+U=,tag:fjKOSVoPxomA3qUw+baV4w==,type:str]
 sops:
     age:
         - recipient: age19yeqvv28fgrtk6jsh3xyaf0lch86kna6rcz4dwe962yyyyevu30sx474xy
@@ -40,7 +43,7 @@ sops:
             NE5yK3ZaOG5PdXNSUnlIUmFSSmRFancKk57hCmo79HvI3hzzgQvgOK7oK5/dcQR8
             f3R4OGF5+212VXEHR/hAEbKzV7CY4y6HhFyrGZ9bUKm1RrxtnVqUyA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-05-19T17:41:34Z"
-    mac: ENC[AES256_GCM,data:UPpz15iUrysYMovpNFLGyAnw1TZ8mmGUo4HDCPlyGI8ADo0v8RfhGjBL/0H0EIA4UX6D+EfRpp4wNacvTdgapQmKHd4H2Q4uDxRUJAHaAkBQVljiuTAEf+8aF/99/U5nEoYrUba15zV8WOONDD7CnzMm+fOosjJuZwKd+akt0KQ=,iv:+nzB0ffdB4PGsnaQ5x9WzWrhfcVQqv1WENUEJOAYbyE=,tag:VvEgvSyBUZixRK3MgCpFvQ==,type:str]
+    lastmodified: "2026-05-19T18:58:47Z"
+    mac: ENC[AES256_GCM,data:UMx1EdkZs4qcnSzGn5V4rbhRTaR8elLbN1nzvnhLOXjjhtLawuVNasJpBQonQi7vDw3X/XlZLgJSCVeCGT0YRjYDpxd00xle72BydH33uUXstJMCg/9atiXCAQhabPdhwhU/srWFh6rxasZFskTI/S+H3/uKIICOapne0TQz7E4=,iv:nxkpQX8ftZEx8gI39wyfJZtvjB4AVQNzmS40fZ/O05g=,tag:ksgNMNv0fzBDp2XhXXjhNQ==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.12.1
diff --git a/hosts/rx4/services/librechat-oci.nix b/hosts/rx4/services/librechat-oci.nix
index 80db7a1..6213f50 100644
--- a/hosts/rx4/services/librechat-oci.nix
+++ b/hosts/rx4/services/librechat-oci.nix
@@ -18,7 +18,16 @@ in
     enable = true;
     inherit port;
     externalUrl = "https://${fqdn}";
-    environmentFile = config.sops.templates.librechat-env-file.path;
+
+    # environment = {
+    #   ALLOW_REGISTRATION = "true";
+    # };
+
+    environmentFiles = {
+      librechat = config.sops.templates.librechat-env.path;
+      meilisearch = config.sops.templates.meili-env.path;
+      ragApi = null;
+    };
   };
 
   services.nginx.virtualHosts."${fqdn}" = {
@@ -43,11 +52,24 @@ in
   };
 
   sops = {
-    secrets."librechat/jwt-token" = { }; # openssl rand -hex 32
-    secrets."librechat/jwt-refresh-token" = { }; # openssl rand -hex 32
-    templates.librechat-env-file.content = ''
-      JET_TOKEN=${config.sops.placeholder."librechat/jwt-token"}
-      JET_REFRESH_TOKEN=${config.sops.placeholder."librechat/jwt-refresh-token"}
+    # generate with:
+    # openssl rand -hex 32
+    secrets."librechat/jwt-secret" = { };
+    secrets."librechat/jwt-refresh-secret" = { };
+    secrets."librechat/creds-key" = { };
+    secrets."librechat/creds-iv" = { };
+    secrets."librechat/meili-master-key" = { };
+
+    templates.librechat-env.content = ''
+      JWT_SECRET=${config.sops.placeholder."librechat/jwt-secret"}
+      JWT_REFRESH_SECRET=${config.sops.placeholder."librechat/jwt-refresh-secret"}
+      CREDS_KEY=${config.sops.placeholder."librechat/creds-key"}
+      CREDS_IV=${config.sops.placeholder."librechat/creds-iv"}
+      MEILI_MASTER_KEY=${config.sops.placeholder."librechat/meili-master-key"}
+    '';
+
+    templates.meili-env.content = ''
+      MEILI_MASTER_KEY=${config.sops.placeholder."librechat/meili-master-key"}
     '';
   };
 }

From e4a429ebe63baf54f689764d001c48eaa0ff4d0f Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 21:03:12 +0200
Subject: [PATCH 27/54] revert to old api

---
 hosts/rx4/services/librechat-oci.nix | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/hosts/rx4/services/librechat-oci.nix b/hosts/rx4/services/librechat-oci.nix
index 6213f50..e4b915b 100644
--- a/hosts/rx4/services/librechat-oci.nix
+++ b/hosts/rx4/services/librechat-oci.nix
@@ -18,16 +18,11 @@ in
     enable = true;
     inherit port;
     externalUrl = "https://${fqdn}";
+    environmentFile = config.sops.templates.librechat-env-file.path;
 
     # environment = {
     #   ALLOW_REGISTRATION = "true";
     # };
-
-    environmentFiles = {
-      librechat = config.sops.templates.librechat-env.path;
-      meilisearch = config.sops.templates.meili-env.path;
-      ragApi = null;
-    };
   };
 
   services.nginx.virtualHosts."${fqdn}" = {
@@ -60,16 +55,12 @@ in
     secrets."librechat/creds-iv" = { };
     secrets."librechat/meili-master-key" = { };
 
-    templates.librechat-env.content = ''
+    templates.librechat-env-file.content = ''
       JWT_SECRET=${config.sops.placeholder."librechat/jwt-secret"}
       JWT_REFRESH_SECRET=${config.sops.placeholder."librechat/jwt-refresh-secret"}
       CREDS_KEY=${config.sops.placeholder."librechat/creds-key"}
       CREDS_IV=${config.sops.placeholder."librechat/creds-iv"}
       MEILI_MASTER_KEY=${config.sops.placeholder."librechat/meili-master-key"}
     '';
-
-    templates.meili-env.content = ''
-      MEILI_MASTER_KEY=${config.sops.placeholder."librechat/meili-master-key"}
-    '';
   };
 }

From 47f63fba59eaf0dbde870acc30261dabc99d87f7 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 21:06:31 +0200
Subject: [PATCH 28/54] librechat: fix secrets

---
 hosts/rx4/secrets/secrets.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/hosts/rx4/secrets/secrets.yaml b/hosts/rx4/secrets/secrets.yaml
index 2c01e49..e78419f 100644
--- a/hosts/rx4/secrets/secrets.yaml
+++ b/hosts/rx4/secrets/secrets.yaml
@@ -18,8 +18,8 @@ vaultwarden:
     smtp-password: ENC[AES256_GCM,data:eQo7op5+74EID6689hL0/J1pq2s=,iv:JqrEqxabWGydRuJJ/27e1q+4YnQhTQ1bKRSsOvjQ+bE=,tag:weqnrhqK+LGEfAacBcuPUA==,type:str]
 hetzner-api-key: ENC[AES256_GCM,data:casjNOXzuQDWgnSFftbBMygA8kGpGiZDqup08faWO9kfjvgOyWOXeqPd2VA1ND8yfM2LvoLYvPs6gUWtni2ldQ==,iv:p2W24uhJgBvpi3g4+cHw0/XbbTM5oYCPHreMBUR4CNs=,tag:lpwjZGoJe/91+CHX/hAkKA==,type:str]
 librechat:
-    jwt-token: ENC[AES256_GCM,data:/NZfZsvg4mDCgB3prDbyPEXIOuN/WSWP3dmSYlvTn7TRSO6oKtnSz20zC0FLvwDAn5QvBYvBKF+LnYjXJeUNkw==,iv:vgESrSyy6IoCMNHG0eL05c9k7Z+tdNb88u5sz+4cYCI=,tag:/WPi7v3hrgKPgwdV0ZE2Bg==,type:str]
-    jwt-refresh-token: ENC[AES256_GCM,data:w/gHj+dXgGk4BcT1ueIdVujjgYWzUGgY8TG/ci8WUDkU12aPcqi6Kuqe55Did0s2AH1Am+1cToy/Q8QiOnt7QQ==,iv:5LJ8ht5yZlql+TayLwU3CNhAd9DUjGw8sRamwbwm7JA=,tag:GJ9zaU7p36oZsOnXeifyyw==,type:str]
+    jwt-secret: ENC[AES256_GCM,data:/OJr23Sw975byjyHN6yqWxuk5FeRfLdQYYOPYJeDHTjzq9X78c3VHqdvnN2a9ZUEtzRi1sx6YLIjNkxBkGbvuQ==,iv:2D0iBj2U3iy3JPtKZBWP5nCfmXMA2/pBhBKUD2f5DoM=,tag:0ZYNxBhUdCBOne0otcG2iQ==,type:str]
+    jwt-refresh-secret: ENC[AES256_GCM,data:qIaunHUMTUFyp88whrxe65eM3Mfi3EX0ieWOUCmYYojSKQQRudh8d4Cb1zMqPbXJLG3zqTVCaZl9xwQn5K4Z/g==,iv:k5+oSCd0TzdOmIUe8BQBesofjvjuRiPXdLT6H9yQf18=,tag:4wcJjX7MvJNx19PCxgqyhw==,type:str]
     creds-key: ENC[AES256_GCM,data:EljwEqFByJaOjd8lRFGwo/FyXHUtl5an0xS1EjRe+kmpo5z4P33EUKbMeeIl69rEcziMHZQLiadzSEcS2cb2uA==,iv:sidBN6VTBeFhMUtN67HZuyofiXCeGFG4tuMRckLZv84=,tag:n7vI8LuPgER3J6r6Q6Jkjg==,type:str]
     creds-iv: ENC[AES256_GCM,data:oc0sPm5RM/7AbH3vdDLJ2m0q6C7eAAME0GPbiojHZUspP8Cto5QX5WKnUjUVLLcvgK+t6pnu7BEmAuD3PLr11A==,iv:Z6XJmlqv0ULFiwqHyRO5v7lb/iyv4g9aSTV4xw9VTXU=,tag:7kptbQwc6lBZ70aXw7wOVA==,type:str]
     meili-master-key: ENC[AES256_GCM,data:eugFl40a6Ks3ba8hcn83WS76AwA0TXkhu3K4gSrbNHtXRliLQCWhGTEvoaQSeb7whmpszh4zh8cKSxByBdhJiQ==,iv:rrWlcVyBlrE5dnBBFWjheIo6SgQTbkzqskGQvQczR+U=,tag:fjKOSVoPxomA3qUw+baV4w==,type:str]
@@ -43,7 +43,7 @@ sops:
             NE5yK3ZaOG5PdXNSUnlIUmFSSmRFancKk57hCmo79HvI3hzzgQvgOK7oK5/dcQR8
             f3R4OGF5+212VXEHR/hAEbKzV7CY4y6HhFyrGZ9bUKm1RrxtnVqUyA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-05-19T18:58:47Z"
-    mac: ENC[AES256_GCM,data:UMx1EdkZs4qcnSzGn5V4rbhRTaR8elLbN1nzvnhLOXjjhtLawuVNasJpBQonQi7vDw3X/XlZLgJSCVeCGT0YRjYDpxd00xle72BydH33uUXstJMCg/9atiXCAQhabPdhwhU/srWFh6rxasZFskTI/S+H3/uKIICOapne0TQz7E4=,iv:nxkpQX8ftZEx8gI39wyfJZtvjB4AVQNzmS40fZ/O05g=,tag:ksgNMNv0fzBDp2XhXXjhNQ==,type:str]
+    lastmodified: "2026-05-19T19:05:23Z"
+    mac: ENC[AES256_GCM,data:X2ELeFFUNEonCDZqJ5a9JCJ8U1EysxcIfbZM751NMK9PvJM8wbRC7MUg5cM/r25Gmua+voch9piTfmL77bJaCq8p6p9EwcBNxc1Weo5sWsHQ5J78MOAoO0wuDSBibOdI7CYEmFC8tRSoEdRQWRBoIOVCyP40fk5fEHhGYOAg9hE=,iv:8PGZRzEq2ezWvdKPi47cECvZD2wJpTDysgLZY3LYOcs=,tag:7TmaiYJ5MEj1+80i2jUgGw==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.12.1

From 04e1748ef9359ffc341cda79d732afd984ef5006 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 21:25:58 +0200
Subject: [PATCH 29/54] update synix input

---
 flake.lock | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/flake.lock b/flake.lock
index 8036d42..8a711b2 100644
--- a/flake.lock
+++ b/flake.lock
@@ -5199,11 +5199,11 @@
         "stylix": "stylix_6"
       },
       "locked": {
-        "lastModified": 1779216861,
-        "narHash": "sha256-fmgvPXXlrVJhIRGNjuYbrr5sHNFxFdlU3kdkmmzPirY=",
+        "lastModified": 1779218747,
+        "narHash": "sha256-KCQ0m93bGGfH4/cyfIshTqllSPIqb/dJLVgRf3iUS68=",
         "ref": "release-25.11",
-        "rev": "a7daa3b9f33cf266218a04c41f4c45af1c7e5207",
-        "revCount": 92,
+        "rev": "b6546f3ab00ffa8ab39631a76644c718f4fafe6d",
+        "revCount": 94,
         "type": "git",
         "url": "https://git.sid.ovh/sid/synix.git"
       },

From e693ba0340662d69b431086a3ff86980a454bdff Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 21:54:01 +0200
Subject: [PATCH 30/54] librechat: add openrouter api key

---
 hosts/rx4/secrets/secrets.yaml       | 5 +++--
 hosts/rx4/services/librechat-oci.nix | 2 ++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/hosts/rx4/secrets/secrets.yaml b/hosts/rx4/secrets/secrets.yaml
index e78419f..1c8c79b 100644
--- a/hosts/rx4/secrets/secrets.yaml
+++ b/hosts/rx4/secrets/secrets.yaml
@@ -23,6 +23,7 @@ librechat:
     creds-key: ENC[AES256_GCM,data:EljwEqFByJaOjd8lRFGwo/FyXHUtl5an0xS1EjRe+kmpo5z4P33EUKbMeeIl69rEcziMHZQLiadzSEcS2cb2uA==,iv:sidBN6VTBeFhMUtN67HZuyofiXCeGFG4tuMRckLZv84=,tag:n7vI8LuPgER3J6r6Q6Jkjg==,type:str]
     creds-iv: ENC[AES256_GCM,data:oc0sPm5RM/7AbH3vdDLJ2m0q6C7eAAME0GPbiojHZUspP8Cto5QX5WKnUjUVLLcvgK+t6pnu7BEmAuD3PLr11A==,iv:Z6XJmlqv0ULFiwqHyRO5v7lb/iyv4g9aSTV4xw9VTXU=,tag:7kptbQwc6lBZ70aXw7wOVA==,type:str]
     meili-master-key: ENC[AES256_GCM,data:eugFl40a6Ks3ba8hcn83WS76AwA0TXkhu3K4gSrbNHtXRliLQCWhGTEvoaQSeb7whmpszh4zh8cKSxByBdhJiQ==,iv:rrWlcVyBlrE5dnBBFWjheIo6SgQTbkzqskGQvQczR+U=,tag:fjKOSVoPxomA3qUw+baV4w==,type:str]
+    openrouter-key: ENC[AES256_GCM,data:63GqQMzasD/+mrVZ4fl6XT2Dv3esfInc4dsgu+kLNX8HARBuid5CtUpSeZe6Yb9GmlDXNKDOjhT8m3jh9x1Bl9X7eNS8KBkruw==,iv:Tp03eyT30Df0PBT10r2YXuVoxParzzjO9gsr3uF4KOA=,tag:YoygwjLFeV8fzKzdlwktPQ==,type:str]
 sops:
     age:
         - recipient: age19yeqvv28fgrtk6jsh3xyaf0lch86kna6rcz4dwe962yyyyevu30sx474xy
@@ -43,7 +44,7 @@ sops:
             NE5yK3ZaOG5PdXNSUnlIUmFSSmRFancKk57hCmo79HvI3hzzgQvgOK7oK5/dcQR8
             f3R4OGF5+212VXEHR/hAEbKzV7CY4y6HhFyrGZ9bUKm1RrxtnVqUyA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-05-19T19:05:23Z"
-    mac: ENC[AES256_GCM,data:X2ELeFFUNEonCDZqJ5a9JCJ8U1EysxcIfbZM751NMK9PvJM8wbRC7MUg5cM/r25Gmua+voch9piTfmL77bJaCq8p6p9EwcBNxc1Weo5sWsHQ5J78MOAoO0wuDSBibOdI7CYEmFC8tRSoEdRQWRBoIOVCyP40fk5fEHhGYOAg9hE=,iv:8PGZRzEq2ezWvdKPi47cECvZD2wJpTDysgLZY3LYOcs=,tag:7TmaiYJ5MEj1+80i2jUgGw==,type:str]
+    lastmodified: "2026-05-19T19:53:49Z"
+    mac: ENC[AES256_GCM,data:x38s31Fzmthgyez+eZ9U2DBzZUXlsENWOUikYS3q95pdFryU5DiB/GQI8Q/2TSzCdqJf+WqqF40jJ7+XUS31KYqAmaFe0w30yidDvzWcU1E0ax+XmuR9OijlLuxFFg9ogGhC6JTV4ZBcoQkr/cgT44qdSolRqEckZICdPNI2USQ=,iv:nFj3dceHf2Ziuk511XACYiwgalVLpL6RCIsRe4AQKws=,tag:cQNABlAfHkPqD/n6ahTOEg==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.12.1
diff --git a/hosts/rx4/services/librechat-oci.nix b/hosts/rx4/services/librechat-oci.nix
index e4b915b..5dd91bf 100644
--- a/hosts/rx4/services/librechat-oci.nix
+++ b/hosts/rx4/services/librechat-oci.nix
@@ -54,6 +54,7 @@ in
     secrets."librechat/creds-key" = { };
     secrets."librechat/creds-iv" = { };
     secrets."librechat/meili-master-key" = { };
+    secrets."librechat/openrouter-key" = { };
 
     templates.librechat-env-file.content = ''
       JWT_SECRET=${config.sops.placeholder."librechat/jwt-secret"}
@@ -61,6 +62,7 @@ in
       CREDS_KEY=${config.sops.placeholder."librechat/creds-key"}
       CREDS_IV=${config.sops.placeholder."librechat/creds-iv"}
       MEILI_MASTER_KEY=${config.sops.placeholder."librechat/meili-master-key"}
+      OPENROUTER_KEY==${config.sops.placeholder}."librechat/openrouter-key"}
     '';
   };
 }

From 3dca169832c215908eaf60cd1b370d816865aea4 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 21:54:40 +0200
Subject: [PATCH 31/54] librechat: fix secrets typo

---
 hosts/rx4/services/librechat-oci.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hosts/rx4/services/librechat-oci.nix b/hosts/rx4/services/librechat-oci.nix
index 5dd91bf..65fa112 100644
--- a/hosts/rx4/services/librechat-oci.nix
+++ b/hosts/rx4/services/librechat-oci.nix
@@ -62,7 +62,7 @@ in
       CREDS_KEY=${config.sops.placeholder."librechat/creds-key"}
       CREDS_IV=${config.sops.placeholder."librechat/creds-iv"}
       MEILI_MASTER_KEY=${config.sops.placeholder."librechat/meili-master-key"}
-      OPENROUTER_KEY==${config.sops.placeholder}."librechat/openrouter-key"}
+      OPENROUTER_KEY=${config.sops.placeholder}."librechat/openrouter-key"}
     '';
   };
 }

From e988008e9791a966662be871df35761534d7ac5c Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 21:55:24 +0200
Subject: [PATCH 32/54] librechat: fix secrets typo

---
 hosts/rx4/services/librechat-oci.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hosts/rx4/services/librechat-oci.nix b/hosts/rx4/services/librechat-oci.nix
index 65fa112..b8b8d73 100644
--- a/hosts/rx4/services/librechat-oci.nix
+++ b/hosts/rx4/services/librechat-oci.nix
@@ -62,7 +62,7 @@ in
       CREDS_KEY=${config.sops.placeholder."librechat/creds-key"}
       CREDS_IV=${config.sops.placeholder."librechat/creds-iv"}
       MEILI_MASTER_KEY=${config.sops.placeholder."librechat/meili-master-key"}
-      OPENROUTER_KEY=${config.sops.placeholder}."librechat/openrouter-key"}
+      OPENROUTER_KEY=${config.sops.placeholder."librechat/openrouter-key"}
     '';
   };
 }

From d051ec20c3935b025c55f278ee91cf643928fdbc Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 22:30:25 +0200
Subject: [PATCH 33/54] librechat: add config file

---
 flake.lock                           |  8 ++++----
 hosts/rx4/services/librechat-oci.nix |  1 +
 hosts/rx4/services/librechat.yaml    | 27 +++++++++++++++++++++++++++
 3 files changed, 32 insertions(+), 4 deletions(-)
 create mode 100644 hosts/rx4/services/librechat.yaml

diff --git a/flake.lock b/flake.lock
index 8a711b2..b0a6409 100644
--- a/flake.lock
+++ b/flake.lock
@@ -5199,11 +5199,11 @@
         "stylix": "stylix_6"
       },
       "locked": {
-        "lastModified": 1779218747,
-        "narHash": "sha256-KCQ0m93bGGfH4/cyfIshTqllSPIqb/dJLVgRf3iUS68=",
+        "lastModified": 1779222589,
+        "narHash": "sha256-pFlaPXus8e+mY9C7/xQhBwux6tPk5P30K2uaN2Qluh0=",
         "ref": "release-25.11",
-        "rev": "b6546f3ab00ffa8ab39631a76644c718f4fafe6d",
-        "revCount": 94,
+        "rev": "1ab817090ff5989578caefd8786e9450b37e3da5",
+        "revCount": 96,
         "type": "git",
         "url": "https://git.sid.ovh/sid/synix.git"
       },
diff --git a/hosts/rx4/services/librechat-oci.nix b/hosts/rx4/services/librechat-oci.nix
index b8b8d73..02ddf20 100644
--- a/hosts/rx4/services/librechat-oci.nix
+++ b/hosts/rx4/services/librechat-oci.nix
@@ -17,6 +17,7 @@ in
   services.librechat-oci = {
     enable = true;
     inherit port;
+    configFile = ./librechat.yaml;
     externalUrl = "https://${fqdn}";
     environmentFile = config.sops.templates.librechat-env-file.path;
 
diff --git a/hosts/rx4/services/librechat.yaml b/hosts/rx4/services/librechat.yaml
new file mode 100644
index 0000000..2459252
--- /dev/null
+++ b/hosts/rx4/services/librechat.yaml
@@ -0,0 +1,27 @@
+version: 1.3.11
+cache: true
+
+interface:
+  modelSelect: true
+  parameters: true
+  presets: true
+  prompts:
+    use: true
+    create: true
+  bookmarks: true
+  multiConvo: true
+
+endpoints:
+  custom:
+    - name: 'OpenRouter'
+      apiKey: '${OPENROUTER_KEY}'
+      baseURL: 'https://openrouter.ai/api/v1'
+      headers:
+        x-librechat-body-parentmessageid: '{{LIBRECHAT_BODY_PARENTMESSAGEID}}'
+      models:
+        default: ['deepseek/deepseek-v4-flash']
+        fetch: true
+      titleConvo: true
+      titleModel: 'deepseek/deepseek-v4-flash'
+      dropParams: ['stop']
+      modelDisplayLabel: 'OpenRouter'

From 974f11c88170ec58b0dabdbd45d5e3d850dd6808 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Tue, 19 May 2026 22:48:22 +0200
Subject: [PATCH 34/54] librechat: add searxng

---
 hosts/rx4/services/librechat-oci.nix |  7 ++--
 hosts/rx4/services/librechat.yaml    | 48 +++++++++++++++++++++++-----
 2 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/hosts/rx4/services/librechat-oci.nix b/hosts/rx4/services/librechat-oci.nix
index 02ddf20..41c9928 100644
--- a/hosts/rx4/services/librechat-oci.nix
+++ b/hosts/rx4/services/librechat-oci.nix
@@ -21,9 +21,10 @@ in
     externalUrl = "https://${fqdn}";
     environmentFile = config.sops.templates.librechat-env-file.path;
 
-    # environment = {
-    #   ALLOW_REGISTRATION = "true";
-    # };
+    environment = {
+      # ALLOW_REGISTRATION = "true";
+      SEARXNG_INSTANCE_URL = "https://searxng.website/";
+    };
   };
 
   services.nginx.virtualHosts."${fqdn}" = {
diff --git a/hosts/rx4/services/librechat.yaml b/hosts/rx4/services/librechat.yaml
index 2459252..d5f6d95 100644
--- a/hosts/rx4/services/librechat.yaml
+++ b/hosts/rx4/services/librechat.yaml
@@ -2,6 +2,7 @@ version: 1.3.11
 cache: true
 
 interface:
+  customWelcome: "Such compose. Much yaml. Wow"
   modelSelect: true
   parameters: true
   presets: true
@@ -11,17 +12,48 @@ interface:
   bookmarks: true
   multiConvo: true
 
+  agents: true
+  fileSearch: true
+  webSearch: true
+
+webSearch:
+  searchProvider: "searxng"
+  searxngInstanceUrl: "${SEARXNG_INSTANCE_URL}"
+  rerankerType: "none" # TODO: add cohere rerank
+
 endpoints:
+  agents:
+    disableBuilder: false
+    recursionLimit: 50
+    maxRecursionLimit: 100
+
+    maxCitations: 30
+    maxCitationsPerFile: 7
+    minRelevanceScore: 0.45
+
+    capabilities:
+      - "deferred_tools"
+      - "execute_code"
+      - "file_search"
+      - "web_search"
+      - "artifacts"
+      - "subagents"
+      - "actions"
+      - "context"
+      - "skills"
+      - "tools"
+      - "chain"
+
   custom:
-    - name: 'OpenRouter'
-      apiKey: '${OPENROUTER_KEY}'
-      baseURL: 'https://openrouter.ai/api/v1'
+    - name: "OpenRouter"
+      apiKey: "${OPENROUTER_KEY}"
+      baseURL: "https://openrouter.ai/api/v1"
       headers:
-        x-librechat-body-parentmessageid: '{{LIBRECHAT_BODY_PARENTMESSAGEID}}'
+        x-librechat-body-parentmessageid: "{{LIBRECHAT_BODY_PARENTMESSAGEID}}"
       models:
-        default: ['deepseek/deepseek-v4-flash']
+        default: ["deepseek/deepseek-v4-flash"]
         fetch: true
       titleConvo: true
-      titleModel: 'deepseek/deepseek-v4-flash'
-      dropParams: ['stop']
-      modelDisplayLabel: 'OpenRouter'
+      titleModel: "deepseek/deepseek-v4-flash"
+      dropParams: ["stop"]
+      modelDisplayLabel: "OpenRouter"

From ded81c4cdde01fb0541f6a1bc6f4fc8992f1c6d2 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Fri, 22 May 2026 18:52:41 +0200
Subject: [PATCH 35/54] librechat: TODO add cohere rerank

---
 hosts/rx4/services/librechat.yaml | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/hosts/rx4/services/librechat.yaml b/hosts/rx4/services/librechat.yaml
index d5f6d95..3176e9d 100644
--- a/hosts/rx4/services/librechat.yaml
+++ b/hosts/rx4/services/librechat.yaml
@@ -11,36 +11,31 @@ interface:
     create: true
   bookmarks: true
   multiConvo: true
-
   agents: true
   fileSearch: true
   webSearch: true
 
+# TODO: add cohere rerank
 webSearch:
   searchProvider: "searxng"
   searxngInstanceUrl: "${SEARXNG_INSTANCE_URL}"
-  rerankerType: "none" # TODO: add cohere rerank
 
 endpoints:
   agents:
     disableBuilder: false
     recursionLimit: 50
     maxRecursionLimit: 100
-
     maxCitations: 30
     maxCitationsPerFile: 7
     minRelevanceScore: 0.45
-
     capabilities:
       - "deferred_tools"
       - "execute_code"
       - "file_search"
       - "web_search"
       - "artifacts"
-      - "subagents"
       - "actions"
       - "context"
-      - "skills"
       - "tools"
       - "chain"
 
@@ -51,9 +46,8 @@ endpoints:
       headers:
         x-librechat-body-parentmessageid: "{{LIBRECHAT_BODY_PARENTMESSAGEID}}"
       models:
-        default: ["deepseek/deepseek-v4-flash"]
+        default: ["meta-llama/llama-3-70b-instruct"]
         fetch: true
       titleConvo: true
-      titleModel: "deepseek/deepseek-v4-flash"
+      titleModel: "meta-llama/llama-3-70b-instruct"
       dropParams: ["stop"]
-      modelDisplayLabel: "OpenRouter"

From 4da48d5f99791b2388153b491a4a9f97d746badf Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Fri, 22 May 2026 19:10:46 +0200
Subject: [PATCH 36/54] replace journald witrh loki+promtail

---
 hosts/rx4/services/default.nix               |  2 +-
 hosts/sid/services/default.nix               |  3 +-
 modules/nixos/default.nix                    |  4 +-
 modules/nixos/journald-remote/default.nix    | 52 ----------------
 modules/nixos/journald-remote/pyproject.toml |  9 ---
 modules/nixos/journald-remote/server.py      | 60 -------------------
 modules/nixos/journald-upload/default.nix    | 17 ------
 modules/nixos/loki/default.nix               | 62 ++++++++++++++++++++
 modules/nixos/promtail/default.nix           | 43 ++++++++++++++
 9 files changed, 110 insertions(+), 142 deletions(-)
 delete mode 100644 modules/nixos/journald-remote/default.nix
 delete mode 100644 modules/nixos/journald-remote/pyproject.toml
 delete mode 100644 modules/nixos/journald-remote/server.py
 delete mode 100644 modules/nixos/journald-upload/default.nix
 create mode 100644 modules/nixos/loki/default.nix
 create mode 100644 modules/nixos/promtail/default.nix

diff --git a/hosts/rx4/services/default.nix b/hosts/rx4/services/default.nix
index 1014d6f..5201854 100644
--- a/hosts/rx4/services/default.nix
+++ b/hosts/rx4/services/default.nix
@@ -10,7 +10,7 @@
     inputs.clients.nixosModules.syncthing
 
     outputs.nixosModules.tailscale
-    # outputs.nixosModules.journald-upload # FIXME
+    outputs.nixosModules.promtail
 
     ./forgejo.nix
     ./jirafeau.nix
diff --git a/hosts/sid/services/default.nix b/hosts/sid/services/default.nix
index b40b165..1be8328 100644
--- a/hosts/sid/services/default.nix
+++ b/hosts/sid/services/default.nix
@@ -9,7 +9,8 @@
     inputs.synix.nixosModules.openssh
 
     outputs.nixosModules.tailscale
-    # outputs.nixosModules.journald-remote
+    outputs.nixosModules.loki
+    outputs.nixosModules.promtail
 
     ./headscale.nix
     ./mailserver.nix
diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix
index 2e35096..d3511ee 100644
--- a/modules/nixos/default.nix
+++ b/modules/nixos/default.nix
@@ -5,9 +5,9 @@
   forgejo = import ./forgejo;
   forgejo-runner = import ./forgejo-runner;
   gnome = import ./gnome;
-  journald-remote = import ./journald-remote;
-  journald-upload = import ./journald-upload;
+  loki = import ./loki;
   monero = import ./monero;
+  promtail = import ./promtail;
   rsshub-oci = import ./rsshub-oci;
   tailscale = import ./tailscale;
   xfce = import ./xfce;
diff --git a/modules/nixos/journald-remote/default.nix b/modules/nixos/journald-remote/default.nix
deleted file mode 100644
index 797eef3..0000000
--- a/modules/nixos/journald-remote/default.nix
+++ /dev/null
@@ -1,52 +0,0 @@
-{
-  pkgs,
-  lib,
-  ...
-}:
-
-let
-  python = pkgs.python3Packages;
-
-  mcp-log-server = python.buildPythonApplication {
-    pname = "mcp-log-server";
-    version = "1.0.0";
-
-    src = ./.;
-
-    pyproject = true;
-    build-system = [ python.setuptools ];
-
-    propagatedBuildInputs = with python; [
-      fastmcp
-    ];
-
-    meta.mainProgram = "mcp-log-server";
-  };
-in
-{
-  services.journald.remote = {
-    enable = true;
-    listen = "http";
-    port = 19532;
-    settings.Remote.SplitMode = "host";
-  };
-
-  systemd.services.mcp-log-server = {
-    description = "AI Log Access MCP Server";
-    after = [
-      "network.target"
-      "multi-user.target"
-      "systemd-journald.service"
-    ];
-    wantedBy = [ "multi-user.target" ];
-
-    script = lib.getExe mcp-log-server;
-
-    serviceConfig = {
-      User = "root";
-      Group = "root";
-      Environment = "PYTHONUNBUFFERED=1";
-      Restart = "on-failure";
-    };
-  };
-}
diff --git a/modules/nixos/journald-remote/pyproject.toml b/modules/nixos/journald-remote/pyproject.toml
deleted file mode 100644
index 809656b..0000000
--- a/modules/nixos/journald-remote/pyproject.toml
+++ /dev/null
@@ -1,9 +0,0 @@
-[build-system]
-requires = ["setuptools"]
-
-[project]
-name = "mcp-log-server"
-version = "1.0.0"
-
-[project.scripts]
-mcp-log-server = "server:main"
diff --git a/modules/nixos/journald-remote/server.py b/modules/nixos/journald-remote/server.py
deleted file mode 100644
index 39c8f28..0000000
--- a/modules/nixos/journald-remote/server.py
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/env python3
-import json
-import subprocess
-from fastmcp import FastMCP
-
-mcp = FastMCP('ClusterLogQuery')
-
-@mcp.tool()
-async def search_cluster_logs(keyword: str, unit: str = '*', limit: int = 500):
-    """
-    Search centralized systemd logs collected on this server.
-
-    Args:
-        keyword: The string to search for (e.g. 'oom-kill', 'failed').
-        unit: The systemd unit to filter (e.g. 'sshd'). If '*', matches all.
-        limit: The number of log lines to return (Max 1000).
-    """
-    
-    limit = min(max(limit, 10), 1000)
-
-    cmd = [
-        'journalctl',
-        '-n', str(limit),
-        '-o', 'json',
-        '--no-pager',
-        '--directory', '/var/log/journal/remote'
-    ]
-
-    if unit != '*':
-        cmd.append(f'UNIT={unit}')
-    if keyword:
-        cmd.append(keyword)
-
-    try:
-        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout, _ = proc.communicate()
-
-        logs = []
-        for line in stdout.decode('utf-8').strip().split('\n'):
-            if not line: continue
-            try:
-                log = json.loads(line)
-                logs.append({
-                    'timestamp': log.get('TIME_ISO8601'),
-                    'hostname': log.get('_HOSTNAME'),
-                    'unit': log.get('_COMM'),
-                    'message': log.get('MESSAGE')
-                })
-            except: pass
-
-        return json.dumps(logs)
-
-    except Exception as e:
-        return f'Error querying logs: {str(e)}'
-
-def main():
-    mcp.run(transport='sse', host='0.0.0.0', port=9500)
-
-if __name__ == '__main__':
-    main()
diff --git a/modules/nixos/journald-upload/default.nix b/modules/nixos/journald-upload/default.nix
deleted file mode 100644
index dede56b..0000000
--- a/modules/nixos/journald-upload/default.nix
+++ /dev/null
@@ -1,17 +0,0 @@
-{
-  services.journald.upload = {
-    enable = true;
-    settings.Upload.URL = "http://100.64.0.6:19532";
-  };
-
-  systemd.services.systemd-journal-upload = {
-    after = [
-      "tailscaled.service"
-      "network-online.target"
-    ];
-    wants = [ "network-online.target" ];
-    serviceConfig = {
-      TimeoutStartSec = 20; # Tailscale is slow
-    };
-  };
-}
diff --git a/modules/nixos/loki/default.nix b/modules/nixos/loki/default.nix
new file mode 100644
index 0000000..a360c0d
--- /dev/null
+++ b/modules/nixos/loki/default.nix
@@ -0,0 +1,62 @@
+{
+  services.loki = {
+    enable = true;
+    configuration = {
+      auth_enabled = false;
+      server.http_listen_port = 3100;
+
+      common = {
+        ring = {
+          instance_addr = "127.0.0.1";
+          kvstore.store = "inmemory";
+        };
+        replication_factor = 1;
+        path_prefix = "/var/lib/loki";
+      };
+
+      schema_config = {
+        configs = [
+          {
+            from = "2020-10-24";
+            store = "tsdb";
+            object_store = "filesystem";
+            schema = "v13";
+            index = {
+              prefix = "index_";
+              period = "24h";
+            };
+          }
+        ];
+      };
+
+      storage_config = {
+        filesystem = {
+          directory = "/var/lib/loki/chunks";
+        };
+      };
+    };
+  };
+
+  services.grafana = {
+    enable = true;
+    settings = {
+      server = {
+        http_addr = "0.0.0.0";
+        http_port = 3003;
+      };
+    };
+
+    provision = {
+      enable = true;
+      datasources.settings.datasources = [
+        {
+          name = "Loki";
+          type = "loki";
+          access = "proxy";
+          url = "http://127.0.0.1:3100";
+          isDefault = true;
+        }
+      ];
+    };
+  };
+}
diff --git a/modules/nixos/promtail/default.nix b/modules/nixos/promtail/default.nix
new file mode 100644
index 0000000..cdb99de
--- /dev/null
+++ b/modules/nixos/promtail/default.nix
@@ -0,0 +1,43 @@
+{ config, constants, ... }:
+
+{
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 9080;
+        grpc_listen_port = 0;
+      };
+
+      clients = [
+        {
+          url = "http://${constants.hosts.sid.ip}:3100/loki/api/v1/push";
+        }
+      ];
+
+      scrape_configs = [
+        {
+          job_name = "journal";
+          journal = {
+            max_age = "12h";
+            path = "/var/log/journal";
+
+            labels = {
+              job = "systemd-journal";
+              host = config.networking.hostName;
+            };
+          };
+
+          relabel_configs = [
+            {
+              source_labels = [ "__journal__systemd_unit" ];
+              target_label = "unit";
+            }
+          ];
+        }
+      ];
+    };
+  };
+
+  users.users.promtail.extraGroups = [ "systemd-journal" ];
+}

From e0f0581d27dc4ccf34a333392bab6d3d39ef4dae Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Fri, 22 May 2026 19:16:45 +0200
Subject: [PATCH 37/54] grafana: add anonymous auth

---
 modules/nixos/loki/default.nix | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/modules/nixos/loki/default.nix b/modules/nixos/loki/default.nix
index a360c0d..d5261e6 100644
--- a/modules/nixos/loki/default.nix
+++ b/modules/nixos/loki/default.nix
@@ -44,6 +44,11 @@
         http_addr = "0.0.0.0";
         http_port = 3003;
       };
+      "auth.anonymous" = {
+        enabled = true;
+        org_name = "Main Org.";
+        org_role = "Admin";
+      };
     };
 
     provision = {

From e6bfb11be0313379ea09272b4d7fcda6897922ce Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Fri, 22 May 2026 19:20:36 +0200
Subject: [PATCH 38/54] grafana: add logs dashboard

---
 modules/nixos/loki/dashboards/logs.json | 75 +++++++++++++++++++++++++
 modules/nixos/loki/default.nix          |  7 +++
 2 files changed, 82 insertions(+)
 create mode 100644 modules/nixos/loki/dashboards/logs.json

diff --git a/modules/nixos/loki/dashboards/logs.json b/modules/nixos/loki/dashboards/logs.json
new file mode 100644
index 0000000..b929e9b
--- /dev/null
+++ b/modules/nixos/loki/dashboards/logs.json
@@ -0,0 +1,75 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Graphics --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 1,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "Loki"
+      },
+      "gridPos": {
+        "h": 24,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "Loki"
+          },
+          "expr": "{job=\"systemd-journal\"}",
+          "refId": "A"
+        }
+      ],
+      "title": "System Logs",
+      "type": "logs"
+    }
+  ],
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "System Logs",
+  "uid": "system-logs",
+  "version": 1
+}
diff --git a/modules/nixos/loki/default.nix b/modules/nixos/loki/default.nix
index d5261e6..f6f6045 100644
--- a/modules/nixos/loki/default.nix
+++ b/modules/nixos/loki/default.nix
@@ -60,6 +60,13 @@
           access = "proxy";
           url = "http://127.0.0.1:3100";
           isDefault = true;
+          uid = "Loki";
+        }
+      ];
+      dashboards.settings.providers = [
+        {
+          name = "default";
+          options.path = ./dashboards;
         }
       ];
     };

From 476939b9b55ed2982e7f7401d26b904ef158391f Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Fri, 22 May 2026 20:00:00 +0200
Subject: [PATCH 39/54] loki: add best practices

---
 modules/nixos/loki/default.nix | 42 ++++++++++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/modules/nixos/loki/default.nix b/modules/nixos/loki/default.nix
index f6f6045..8baaa0a 100644
--- a/modules/nixos/loki/default.nix
+++ b/modules/nixos/loki/default.nix
@@ -3,7 +3,12 @@
     enable = true;
     configuration = {
       auth_enabled = false;
-      server.http_listen_port = 3100;
+
+      server = {
+        http_listen_address = "0.0.0.0";
+        http_listen_port = 3100;
+        grpc_listen_port = 9096;
+      };
 
       common = {
         ring = {
@@ -14,10 +19,32 @@
         path_prefix = "/var/lib/loki";
       };
 
+      ingester = {
+        wal = {
+          enabled = true;
+          dir = "/var/lib/loki/wal";
+        };
+        chunk_encoding = "snappy";
+        chunk_idle_period = "30m";
+        max_chunk_age = "2h";
+        chunk_target_size = 1572864;
+        chunk_block_size = 262144;
+      };
+
+      limits_config = {
+        reject_old_samples = true;
+        reject_old_samples_max_age = "168h";
+        ingestion_rate_mb = 10;
+        ingestion_burst_size_mb = 20;
+        per_stream_rate_limit = "3MB";
+        per_stream_rate_limit_burst = "15MB";
+        max_line_size = "256KB";
+      };
+
       schema_config = {
         configs = [
           {
-            from = "2020-10-24";
+            from = "2026-01-01";
             store = "tsdb";
             object_store = "filesystem";
             schema = "v13";
@@ -34,6 +61,15 @@
           directory = "/var/lib/loki/chunks";
         };
       };
+
+      compactor = {
+        working_directory = "/var/lib/loki/compactor";
+        compaction_interval = "10m";
+        retention_enabled = true;
+        retention_delete_delay = "2h";
+        retention_delete_worker_count = 150;
+        delete_request_store = "filesystem";
+      };
     };
   };
 
@@ -71,4 +107,6 @@
       ];
     };
   };
+
+  networking.firewall.allowedTCPPorts = [ 3100 ];
 }

From f2ebf6626245944016dfd086f8e885950b882e09 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Fri, 22 May 2026 20:09:24 +0200
Subject: [PATCH 40/54] loki: add apiVersion

---
 modules/nixos/loki/dashboards/logs.json |  2 +-
 modules/nixos/loki/default.nix          | 23 +++++++++++++----------
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/modules/nixos/loki/dashboards/logs.json b/modules/nixos/loki/dashboards/logs.json
index b929e9b..2009e52 100644
--- a/modules/nixos/loki/dashboards/logs.json
+++ b/modules/nixos/loki/dashboards/logs.json
@@ -24,7 +24,7 @@
     {
       "datasource": {
         "type": "loki",
-        "uid": "Loki"
+        "uid": "loki"
       },
       "gridPos": {
         "h": 24,
diff --git a/modules/nixos/loki/default.nix b/modules/nixos/loki/default.nix
index 8baaa0a..e5fdd14 100644
--- a/modules/nixos/loki/default.nix
+++ b/modules/nixos/loki/default.nix
@@ -89,16 +89,19 @@
 
     provision = {
       enable = true;
-      datasources.settings.datasources = [
-        {
-          name = "Loki";
-          type = "loki";
-          access = "proxy";
-          url = "http://127.0.0.1:3100";
-          isDefault = true;
-          uid = "Loki";
-        }
-      ];
+      datasources.settings = {
+        apiVersion = 1;
+        datasources = [
+          {
+            name = "Loki";
+            type = "loki";
+            access = "proxy";
+            url = "http://127.0.0.1:3100";
+            isDefault = true;
+            uid = "loki";
+          }
+        ];
+      };
       dashboards.settings.providers = [
         {
           name = "default";

From 3b335c6ba42fd4e2a11bdede9eb4c2a58f9c882a Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Fri, 22 May 2026 20:17:55 +0200
Subject: [PATCH 41/54] netdata: add systemd-journal

---
 hosts/rx4/services/netdata.nix | 4 ++++
 hosts/sid/services/netdata.nix | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/hosts/rx4/services/netdata.nix b/hosts/rx4/services/netdata.nix
index 30d720d..92361c3 100644
--- a/hosts/rx4/services/netdata.nix
+++ b/hosts/rx4/services/netdata.nix
@@ -17,6 +17,10 @@
     };
   };
 
+  services.journald.storage = "persistent";
+
+  users.users.netdata.extraGroups = [ "systemd-journal" ];
+
   sops =
     let
       owner = config.services.netdata.user;
diff --git a/hosts/sid/services/netdata.nix b/hosts/sid/services/netdata.nix
index 005f9e1..724b0f3 100644
--- a/hosts/sid/services/netdata.nix
+++ b/hosts/sid/services/netdata.nix
@@ -36,6 +36,10 @@ in
     NETDATA_USER_CONFIG_DIR = "/etc/netdata/conf.d";
   };
 
+  services.journald.storage = "persistent";
+
+  users.users.netdata.extraGroups = [ "systemd-journal" ];
+
   sops =
     let
       owner = config.services.netdata.user;

From 5ee15884b44526dd05705ec2b899c114b40ab03d Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Fri, 22 May 2026 20:41:42 +0200
Subject: [PATCH 42/54] finally free again

---
 hosts/rx4/services/default.nix      |  1 -
 hosts/rx4/services/netdata.nix      |  2 ++
 hosts/rx4/services/print-server.nix | 12 ------------
 hosts/sid/services/netdata.nix      |  4 +---
 modules/nixos/common/default.nix    |  2 --
 5 files changed, 3 insertions(+), 18 deletions(-)
 delete mode 100644 hosts/rx4/services/print-server.nix

diff --git a/hosts/rx4/services/default.nix b/hosts/rx4/services/default.nix
index 5201854..ae599a8 100644
--- a/hosts/rx4/services/default.nix
+++ b/hosts/rx4/services/default.nix
@@ -19,7 +19,6 @@
     ./netdata.nix
     ./nginx.nix
     ./open-webui-oci.nix
-    ./print-server.nix
     ./rsshub-oci.nix
     ./samba.nix
     ./vaultwarden.nix
diff --git a/hosts/rx4/services/netdata.nix b/hosts/rx4/services/netdata.nix
index 92361c3..956f612 100644
--- a/hosts/rx4/services/netdata.nix
+++ b/hosts/rx4/services/netdata.nix
@@ -1,12 +1,14 @@
 {
   config,
   constants,
+  pkgs,
   ...
 }:
 
 {
   services.netdata = {
     enable = true;
+    package = pkgs.netdata.override { withCloudUi = false; };
     config.global = {
       "debug log" = "syslog";
       "access log" = "syslog";
diff --git a/hosts/rx4/services/print-server.nix b/hosts/rx4/services/print-server.nix
deleted file mode 100644
index 64037fd..0000000
--- a/hosts/rx4/services/print-server.nix
+++ /dev/null
@@ -1,12 +0,0 @@
-{
-  inputs,
-  ...
-}:
-
-{
-  imports = [
-    inputs.synix.nixosModules.print-server
-  ];
-
-  services.print-server.enable = true;
-}
diff --git a/hosts/sid/services/netdata.nix b/hosts/sid/services/netdata.nix
index 724b0f3..5cec75d 100644
--- a/hosts/sid/services/netdata.nix
+++ b/hosts/sid/services/netdata.nix
@@ -10,9 +10,7 @@ in
 {
   services.netdata = {
     enable = true;
-    package = pkgs.netdata.override {
-      withCloudUi = true;
-    };
+    package = pkgs.netdata.override { withCloudUi = false; };
     config.global = {
       "debug log" = "syslog";
       "access log" = "syslog";
diff --git a/modules/nixos/common/default.nix b/modules/nixos/common/default.nix
index eba84dd..71660e5 100644
--- a/modules/nixos/common/default.nix
+++ b/modules/nixos/common/default.nix
@@ -7,6 +7,4 @@
 
     inputs.synix.nixosModules.device.server
   ];
-
-  nixpkgs.config.allowUnfree = true;
 }

From ac4e31bb7d595362257896e80e4176cfcb600827 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Fri, 22 May 2026 21:05:07 +0200
Subject: [PATCH 43/54] netdata: add old ui

---
 constants.nix                  |  2 +-
 hosts/sid/services/netdata.nix | 36 ++++++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/constants.nix b/constants.nix
index 7b3578e..aeb5aea 100644
--- a/constants.nix
+++ b/constants.nix
@@ -38,7 +38,7 @@ rec {
       port = 8085;
     };
     netdata = {
-      # fqdn = "mon." + domain;
+      fqdn = "mon." + domain;
       port = 19999;
     };
     open-webui-oci = {
diff --git a/hosts/sid/services/netdata.nix b/hosts/sid/services/netdata.nix
index 5cec75d..f4b504f 100644
--- a/hosts/sid/services/netdata.nix
+++ b/hosts/sid/services/netdata.nix
@@ -1,11 +1,32 @@
 {
   config,
+  constants,
+  lib,
   pkgs,
   ...
 }:
 
 let
   email = "sid@${config.networking.domain}";
+
+  netdata-dashboard = pkgs.stdenvNoCC.mkDerivation {
+    pname = "netdata-dashboard";
+    version = "2.31.0";
+
+    src = pkgs.fetchurl {
+      url = "https://github.com/netdata/dashboard/releases/download/v2.31.0/dashboard.tar.gz";
+      hash = "sha256-n7M7Y8LIb4tbgQ8wQIr5bMKxLT5fPDID5LnX47ayH/o=";
+    };
+
+    dontUnpack = true;
+
+    installPhase = ''
+      mkdir -p $out
+      tar -xzf $src --strip-components=1 -C $out
+    '';
+
+    meta.license = lib.licenses.gpl3Only;
+  };
 in
 {
   services.netdata = {
@@ -34,6 +55,21 @@ in
     NETDATA_USER_CONFIG_DIR = "/etc/netdata/conf.d";
   };
 
+  services.nginx.virtualHosts."${constants.services.netdata.fqdn}" = {
+    enableACME = true;
+    forceSSL = true;
+
+    locations."/" = {
+      root = netdata-dashboard;
+      tryFiles = "$uri $uri/ /index.html";
+    };
+
+    locations."~ ^/(api|v[0-9]+|netdata.conf|registry|stream)(/|$)" = {
+      proxyPass = "http://127.0.0.1:${toString constants.services.netdata.port}";
+      recommendedProxySettings = true;
+    };
+  };
+
   services.journald.storage = "persistent";
 
   users.users.netdata.extraGroups = [ "systemd-journal" ];

From 69d12812dfff9faddcab574f64fc75eb3553fc21 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Fri, 22 May 2026 21:14:48 +0200
Subject: [PATCH 44/54] netdata: also proxy version.txt

---
 hosts/sid/services/netdata.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hosts/sid/services/netdata.nix b/hosts/sid/services/netdata.nix
index f4b504f..b69a02b 100644
--- a/hosts/sid/services/netdata.nix
+++ b/hosts/sid/services/netdata.nix
@@ -64,7 +64,7 @@ in
       tryFiles = "$uri $uri/ /index.html";
     };
 
-    locations."~ ^/(api|v[0-9]+|netdata.conf|registry|stream)(/|$)" = {
+    locations."~ ^/(api|v[0-9]+|netdata.conf|registry|stream|version.txt)(/|$)" = {
       proxyPass = "http://127.0.0.1:${toString constants.services.netdata.port}";
       recommendedProxySettings = true;
     };

From eac7803895890b42fb8235bf85f1927de38e3810 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Fri, 22 May 2026 21:17:28 +0200
Subject: [PATCH 45/54] netdata: ui should only be reachable inside tailnet

---
 hosts/sid/services/netdata.nix | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/hosts/sid/services/netdata.nix b/hosts/sid/services/netdata.nix
index b69a02b..046f2eb 100644
--- a/hosts/sid/services/netdata.nix
+++ b/hosts/sid/services/netdata.nix
@@ -55,20 +55,21 @@ in
     NETDATA_USER_CONFIG_DIR = "/etc/netdata/conf.d";
   };
 
-  services.nginx.virtualHosts."${constants.services.netdata.fqdn}" = {
-    enableACME = true;
-    forceSSL = true;
-
-    locations."/" = {
-      root = netdata-dashboard;
-      tryFiles = "$uri $uri/ /index.html";
-    };
-
-    locations."~ ^/(api|v[0-9]+|netdata.conf|registry|stream|version.txt)(/|$)" = {
-      proxyPass = "http://127.0.0.1:${toString constants.services.netdata.port}";
-      recommendedProxySettings = true;
-    };
-  };
+  # TODO: move into Tailnet
+  # services.nginx.virtualHosts."${constants.services.netdata.fqdn}" = {
+  #   enableACME = true;
+  #   forceSSL = true;
+  #
+  #   locations."/" = {
+  #     root = netdata-dashboard;
+  #     tryFiles = "$uri $uri/ /index.html";
+  #   };
+  #
+  #   locations."~ ^/(api|v[0-9]+|netdata.conf|registry|stream|version.txt)(/|$)" = {
+  #     proxyPass = "http://127.0.0.1:${toString constants.services.netdata.port}";
+  #     recommendedProxySettings = true;
+  #   };
+  # };
 
   services.journald.storage = "persistent";
 

From 84d04fa1ad97716f2433f1acd23aee5628e3ef92 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Fri, 22 May 2026 21:23:54 +0200
Subject: [PATCH 46/54] netdata: make ui available in tailnet

---
 hosts/sid/secrets/secrets.yaml |  5 ++--
 hosts/sid/services/netdata.nix | 51 ++++++++++++++++++++++++----------
 2 files changed, 39 insertions(+), 17 deletions(-)

diff --git a/hosts/sid/secrets/secrets.yaml b/hosts/sid/secrets/secrets.yaml
index 8fb0e4a..f94edbf 100644
--- a/hosts/sid/secrets/secrets.yaml
+++ b/hosts/sid/secrets/secrets.yaml
@@ -35,6 +35,7 @@ radicale:
 step-ca:
     password: ENC[AES256_GCM,data:8/6NA3WpII0LmDOp5ISnHKeaXn5LM4gpiI47JTso23c=,iv:fi2eMGG1lOwdK5+98Hp7vZ101GKRip5Xgq9k+vnC9yI=,tag:oENvvsEbKSHFfLoXcJlPkg==,type:str]
     intermediate-key: ENC[AES256_GCM,data:yGZLSd7ydx9wNFpWWPcyUBwZQZbyziGleCWSxurFniBCauw2h4hcPc4c4I/7cjl1vRUv41WfzWu1PtXnZ3lNHOC6tTbiikHFBgGiHk2Lhddx+NESUWmgNiejJR/UDW4T25W9OHxwLCV9pmHf4fjyT/REymGIB7kbcRryWqcWtoZWYaL7JooJornm5mMU1Be+MCfxusTGQA4gQsT5/bu20iEGPwgY3fEgZLQWzKFI2kD2lYlMC8CRxoZO32uTizzooW1+zKng1qSZ7aobFJsbSKRYpYDv9Vvfwltcczb+xo+yZL3pfoEiqAxPzeG/48lRVNf1nftM5esBRGIIPr9BV9+7fbe5DFbSRDtAWspEnp9R5ENj1rbNint/fjCcStg3OfFMdv6N8cQyIpQyHCiBLiG4z+xyFcn0iW4=,iv:BhUoeaoetI5vJk9wOHhBI2ebHWCPeXz8U2ta/xEeUxM=,tag:7xg5ilOSJP1rFlSmmZVZUg==,type:str]
+hetzner-api-key: ENC[AES256_GCM,data:NhgWjitvgJrcBEDSkZH0S0VmaW37NupkiEUcQDZe/6oYyrE/VgEwrGSag/s2Fgv6uHmSsdbv1vqdc0iDO8GJ8w==,iv:ChEicL0jtjQrgn8CCUnrzErRr3YVdDhMbvcIlI3t7H8=,tag:cjjbEEYqEyNa5qDZCytjxw==,type:str]
 sops:
     age:
         - recipient: age19yeqvv28fgrtk6jsh3xyaf0lch86kna6rcz4dwe962yyyyevu30sx474xy
@@ -55,7 +56,7 @@ sops:
             RzhnczA0S1pxcXZncGpWVHNYQW96L28K+ytH3PPyg4+wibpAQhp02RiSfZ83EDRB
             UJ8UV1d+51D0e2A1sI95r2AzDj4jfwUnI+LYDPC/qEpsu5LFLGVyeg==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-05-02T17:10:22Z"
-    mac: ENC[AES256_GCM,data:f4KQ26/zvg2nLLeW5qVeI8uH2GmPpJUKohNu68nEiIjP5AT53zjBaGoLOTGl9+oVRomSOGZtLGkJGaExB6tLMon5HN6xkQbugqvq08UkZ7FnR1Sa8/OtTr/+eexPNzF8VSdZE2TZCboUSQODV8+0Cy5T918g5kedxnT62SyY4As=,iv:P4TnpJvHwnZPl7kRNjv9d1WLZP9J0sg6R3KbdDMJqyc=,tag:ylYOcg6825jT29lWUaFRYA==,type:str]
+    lastmodified: "2026-05-22T19:19:21Z"
+    mac: ENC[AES256_GCM,data:hOtmWizEaIcybM14UEDsXw4GNQZob5SoFn49bWeccxA3dkGlYl67kVkDJGg0cQIO1qr/vGcZ8h/OmnOxU3geP0DaflG0h1/40lDQ3+E6BTb6HP2JmhgEmlRBRBdv87cRDHnDytBzcWARTvff3SsP2J2pLpLBTDiihlaZaiQYtgU=,iv:TvFpvcTydXO3fbh5x9ZXIOtMChlE7WXl2Xx2a9ujh00=,tag:XHvsZh6r9fzbbYFWWQyI5g==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.12.1
diff --git a/hosts/sid/services/netdata.nix b/hosts/sid/services/netdata.nix
index 046f2eb..fbfa89d 100644
--- a/hosts/sid/services/netdata.nix
+++ b/hosts/sid/services/netdata.nix
@@ -55,21 +55,36 @@ in
     NETDATA_USER_CONFIG_DIR = "/etc/netdata/conf.d";
   };
 
-  # TODO: move into Tailnet
-  # services.nginx.virtualHosts."${constants.services.netdata.fqdn}" = {
-  #   enableACME = true;
-  #   forceSSL = true;
-  #
-  #   locations."/" = {
-  #     root = netdata-dashboard;
-  #     tryFiles = "$uri $uri/ /index.html";
-  #   };
-  #
-  #   locations."~ ^/(api|v[0-9]+|netdata.conf|registry|stream|version.txt)(/|$)" = {
-  #     proxyPass = "http://127.0.0.1:${toString constants.services.netdata.port}";
-  #     recommendedProxySettings = true;
-  #   };
-  # };
+  services.nginx.virtualHosts."${constants.services.netdata.fqdn}" = {
+    useACMEHost = constants.services.netdata.fqdn;
+    forceSSL = true;
+    listen = [
+      {
+        addr = "${constants.hosts.sid.ip}:443";
+        ssl = true;
+      }
+    ];
+
+    locations."/" = {
+      root = netdata-dashboard;
+      tryFiles = "$uri $uri/ /index.html";
+    };
+
+    locations."~ ^/(api|v[0-9]+|netdata.conf|registry|stream|version.txt)(/|$)" = {
+      proxyPass = "http://127.0.0.1:${toString constants.services.netdata.port}";
+      recommendedProxySettings = true;
+    };
+  };
+
+  security.acme = {
+    acceptTerms = true;
+    certs."${constants.services.netdata.fqdn}" = {
+      domain = constants.services.netdata.fqdn;
+      dnsProvider = "hetzner";
+      credentialFiles.HETZNER_API_TOKEN_FILE = config.sops.secrets.hetzner-api-key.path;
+      group = "nginx";
+    };
+  };
 
   services.journald.storage = "persistent";
 
@@ -83,6 +98,12 @@ in
       restartUnits = [ "netdata.service" ];
     in
     {
+      secrets.hetzner-api-key = {
+        inherit mode;
+        owner = "acme";
+        group = "acme";
+      };
+
       secrets."netdata/stream/rx4/uuid" = {
         inherit
           owner

From 68412567057655cd017c927c812c358965f3c468 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Fri, 22 May 2026 21:27:58 +0200
Subject: [PATCH 47/54] netdata: override acme host defaults

---
 hosts/sid/services/netdata.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hosts/sid/services/netdata.nix b/hosts/sid/services/netdata.nix
index fbfa89d..5b02855 100644
--- a/hosts/sid/services/netdata.nix
+++ b/hosts/sid/services/netdata.nix
@@ -80,6 +80,7 @@ in
     acceptTerms = true;
     certs."${constants.services.netdata.fqdn}" = {
       domain = constants.services.netdata.fqdn;
+      webroot = lib.mkForce null;
       dnsProvider = "hetzner";
       credentialFiles.HETZNER_API_TOKEN_FILE = config.sops.secrets.hetzner-api-key.path;
       group = "nginx";

From 1b7f7cbcad5c70cee526e62ab0a26b874b4a5fb5 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Fri, 22 May 2026 21:30:48 +0200
Subject: [PATCH 48/54] disable loki and promtail

---
 hosts/rx4/services/default.nix | 2 +-
 hosts/sid/services/default.nix | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/hosts/rx4/services/default.nix b/hosts/rx4/services/default.nix
index ae599a8..8694146 100644
--- a/hosts/rx4/services/default.nix
+++ b/hosts/rx4/services/default.nix
@@ -10,7 +10,7 @@
     inputs.clients.nixosModules.syncthing
 
     outputs.nixosModules.tailscale
-    outputs.nixosModules.promtail
+    # outputs.nixosModules.promtail
 
     ./forgejo.nix
     ./jirafeau.nix
diff --git a/hosts/sid/services/default.nix b/hosts/sid/services/default.nix
index 1be8328..ee63f99 100644
--- a/hosts/sid/services/default.nix
+++ b/hosts/sid/services/default.nix
@@ -9,8 +9,8 @@
     inputs.synix.nixosModules.openssh
 
     outputs.nixosModules.tailscale
-    outputs.nixosModules.loki
-    outputs.nixosModules.promtail
+    # outputs.nixosModules.loki
+    # outputs.nixosModules.promtail
 
     ./headscale.nix
     ./mailserver.nix

From b6194de7bdde23ed8bb12eb10d4bd4775cc675be Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Sat, 23 May 2026 12:18:21 +0200
Subject: [PATCH 49/54] netdata: proxy child nodes

---
 hosts/sid/services/netdata.nix | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/hosts/sid/services/netdata.nix b/hosts/sid/services/netdata.nix
index 5b02855..6733c2a 100644
--- a/hosts/sid/services/netdata.nix
+++ b/hosts/sid/services/netdata.nix
@@ -74,6 +74,11 @@ in
       proxyPass = "http://127.0.0.1:${toString constants.services.netdata.port}";
       recommendedProxySettings = true;
     };
+
+    locations."~ ^/host/" = {
+      proxyPass = "http://127.0.0.1:${toString constants.services.netdata.port}";
+      recommendedProxySettings = true;
+    };
   };
 
   security.acme = {
@@ -126,6 +131,7 @@ in
           [${config.sops.placeholder."netdata/stream/rx4/uuid"}]
           enabled = yes
           default history = 3600
+          allow from = *
         '';
       };
     };

From 24fe9cf6d0c291d1ce047f2f7c4283ea69cbf1f8 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Sat, 23 May 2026 12:25:37 +0200
Subject: [PATCH 50/54] netdata: rm host path

---
 hosts/sid/services/netdata.nix | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/hosts/sid/services/netdata.nix b/hosts/sid/services/netdata.nix
index 6733c2a..56496aa 100644
--- a/hosts/sid/services/netdata.nix
+++ b/hosts/sid/services/netdata.nix
@@ -74,11 +74,6 @@ in
       proxyPass = "http://127.0.0.1:${toString constants.services.netdata.port}";
       recommendedProxySettings = true;
     };
-
-    locations."~ ^/host/" = {
-      proxyPass = "http://127.0.0.1:${toString constants.services.netdata.port}";
-      recommendedProxySettings = true;
-    };
   };
 
   security.acme = {

From 612ea64ae9c199a8013a12a6e69451bd1324a595 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Sat, 23 May 2026 12:32:46 +0200
Subject: [PATCH 51/54] netdata: trying host path 302

---
 hosts/sid/services/netdata.nix | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/hosts/sid/services/netdata.nix b/hosts/sid/services/netdata.nix
index 56496aa..c0b9594 100644
--- a/hosts/sid/services/netdata.nix
+++ b/hosts/sid/services/netdata.nix
@@ -74,6 +74,10 @@ in
       proxyPass = "http://127.0.0.1:${toString constants.services.netdata.port}";
       recommendedProxySettings = true;
     };
+
+    locations."~ ^/host/([^/]+)/?$" = {
+      return = "302 /#/node/$1";
+    };
   };
 
   security.acme = {

From 3bc424eeeefd347ae9325718d8e7ae0ef3f90c11 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Sat, 23 May 2026 12:36:24 +0200
Subject: [PATCH 52/54] netdata: host path fix

---
 hosts/sid/services/netdata.nix | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/hosts/sid/services/netdata.nix b/hosts/sid/services/netdata.nix
index c0b9594..6733c2a 100644
--- a/hosts/sid/services/netdata.nix
+++ b/hosts/sid/services/netdata.nix
@@ -75,8 +75,9 @@ in
       recommendedProxySettings = true;
     };
 
-    locations."~ ^/host/([^/]+)/?$" = {
-      return = "302 /#/node/$1";
+    locations."~ ^/host/" = {
+      proxyPass = "http://127.0.0.1:${toString constants.services.netdata.port}";
+      recommendedProxySettings = true;
     };
   };
 

From 51fed84b2a0572515ac458626e8cbcd2070a2656 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Mon, 25 May 2026 12:38:30 +0200
Subject: [PATCH 53/54] librechat: replace openrouter with requesty

---
 hosts/rx4/services/librechat-oci.nix | 5 +++--
 hosts/rx4/services/librechat.yaml    | 6 +++---
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/hosts/rx4/services/librechat-oci.nix b/hosts/rx4/services/librechat-oci.nix
index 41c9928..25149dc 100644
--- a/hosts/rx4/services/librechat-oci.nix
+++ b/hosts/rx4/services/librechat-oci.nix
@@ -56,7 +56,8 @@ in
     secrets."librechat/creds-key" = { };
     secrets."librechat/creds-iv" = { };
     secrets."librechat/meili-master-key" = { };
-    secrets."librechat/openrouter-key" = { };
+
+    secrets."librechat/requesty-key" = { };
 
     templates.librechat-env-file.content = ''
       JWT_SECRET=${config.sops.placeholder."librechat/jwt-secret"}
@@ -64,7 +65,7 @@ in
       CREDS_KEY=${config.sops.placeholder."librechat/creds-key"}
       CREDS_IV=${config.sops.placeholder."librechat/creds-iv"}
       MEILI_MASTER_KEY=${config.sops.placeholder."librechat/meili-master-key"}
-      OPENROUTER_KEY=${config.sops.placeholder."librechat/openrouter-key"}
+      REQUESTY_KEY=${config.sops.placeholder."librechat/requesty-key"}
     '';
   };
 }
diff --git a/hosts/rx4/services/librechat.yaml b/hosts/rx4/services/librechat.yaml
index 3176e9d..327012a 100644
--- a/hosts/rx4/services/librechat.yaml
+++ b/hosts/rx4/services/librechat.yaml
@@ -40,9 +40,9 @@ endpoints:
       - "chain"
 
   custom:
-    - name: "OpenRouter"
-      apiKey: "${OPENROUTER_KEY}"
-      baseURL: "https://openrouter.ai/api/v1"
+    - name: "Requesty"
+      apiKey: "${REQUESTY_KEY}"
+      baseURL: "https://router.requesty.ai/v1"
       headers:
         x-librechat-body-parentmessageid: "{{LIBRECHAT_BODY_PARENTMESSAGEID}}"
       models:

From 19d60288b3f2035c4a9a5e54d0180c01c18baca9 Mon Sep 17 00:00:00 2001
From: sid <sid@sid.ovh>
Date: Mon, 25 May 2026 20:15:44 +0200
Subject: [PATCH 54/54] add requesty api key

---
 hosts/rx4/secrets/secrets.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/hosts/rx4/secrets/secrets.yaml b/hosts/rx4/secrets/secrets.yaml
index 1c8c79b..2b66560 100644
--- a/hosts/rx4/secrets/secrets.yaml
+++ b/hosts/rx4/secrets/secrets.yaml
@@ -23,7 +23,7 @@ librechat:
     creds-key: ENC[AES256_GCM,data:EljwEqFByJaOjd8lRFGwo/FyXHUtl5an0xS1EjRe+kmpo5z4P33EUKbMeeIl69rEcziMHZQLiadzSEcS2cb2uA==,iv:sidBN6VTBeFhMUtN67HZuyofiXCeGFG4tuMRckLZv84=,tag:n7vI8LuPgER3J6r6Q6Jkjg==,type:str]
     creds-iv: ENC[AES256_GCM,data:oc0sPm5RM/7AbH3vdDLJ2m0q6C7eAAME0GPbiojHZUspP8Cto5QX5WKnUjUVLLcvgK+t6pnu7BEmAuD3PLr11A==,iv:Z6XJmlqv0ULFiwqHyRO5v7lb/iyv4g9aSTV4xw9VTXU=,tag:7kptbQwc6lBZ70aXw7wOVA==,type:str]
     meili-master-key: ENC[AES256_GCM,data:eugFl40a6Ks3ba8hcn83WS76AwA0TXkhu3K4gSrbNHtXRliLQCWhGTEvoaQSeb7whmpszh4zh8cKSxByBdhJiQ==,iv:rrWlcVyBlrE5dnBBFWjheIo6SgQTbkzqskGQvQczR+U=,tag:fjKOSVoPxomA3qUw+baV4w==,type:str]
-    openrouter-key: ENC[AES256_GCM,data:63GqQMzasD/+mrVZ4fl6XT2Dv3esfInc4dsgu+kLNX8HARBuid5CtUpSeZe6Yb9GmlDXNKDOjhT8m3jh9x1Bl9X7eNS8KBkruw==,iv:Tp03eyT30Df0PBT10r2YXuVoxParzzjO9gsr3uF4KOA=,tag:YoygwjLFeV8fzKzdlwktPQ==,type:str]
+    requesty-key: ENC[AES256_GCM,data:vxr+m3c9qu6ChFvuAbBCFrneDP9xDIPJBRmB9diw5uSQD9XDl0IK954OzmMMXaSl2AeHgY8WiugvgvQjUwywjKG6TxxMEYMFsQkMpSnV7xHYv2MAE/TIC74CsHRL823MesUQ7agoIasFtjr+CnZ5RRUNHZOG,iv:pa64BT1yay5vYwn/XBdK7meYzOBk4M+MmgLzaHR5Hfw=,tag:L+lUPTE0ZI303jtVefIrvQ==,type:str]
 sops:
     age:
         - recipient: age19yeqvv28fgrtk6jsh3xyaf0lch86kna6rcz4dwe962yyyyevu30sx474xy
@@ -44,7 +44,7 @@ sops:
             NE5yK3ZaOG5PdXNSUnlIUmFSSmRFancKk57hCmo79HvI3hzzgQvgOK7oK5/dcQR8
             f3R4OGF5+212VXEHR/hAEbKzV7CY4y6HhFyrGZ9bUKm1RrxtnVqUyA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-05-19T19:53:49Z"
-    mac: ENC[AES256_GCM,data:x38s31Fzmthgyez+eZ9U2DBzZUXlsENWOUikYS3q95pdFryU5DiB/GQI8Q/2TSzCdqJf+WqqF40jJ7+XUS31KYqAmaFe0w30yidDvzWcU1E0ax+XmuR9OijlLuxFFg9ogGhC6JTV4ZBcoQkr/cgT44qdSolRqEckZICdPNI2USQ=,iv:nFj3dceHf2Ziuk511XACYiwgalVLpL6RCIsRe4AQKws=,tag:cQNABlAfHkPqD/n6ahTOEg==,type:str]
+    lastmodified: "2026-05-25T18:14:59Z"
+    mac: ENC[AES256_GCM,data:eh/jcKrqyCTh+2n4phHQ2LKF71DaCDwrrfXms6HaD0ER4xVOkYERTe7IN4cX//qjY/91wSzAzwLg3yphWK4k920tiYTBog9LcWUz6l6X5lpmKHQp+vdoQH41WrA1ZgOcXzSfmZoblcD1qoJNCaHGt5N8hjXRcUc3lEqcPrdoC7A=,iv:8kBd9Daai3wJgzxONX4eIkeZLMzJO2DX439sBv/pER4=,tag:l8Q3gzMHoSTCdOqwzaKgCA==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.12.1