Merge pull request 'replace journald witrh loki+promtail' (#86) from develop into master

Reviewed-on: #86

hosts/rx4/services/default.nix

@@ -10,7 +10,7 @@
     inputs.clients.nixosModules.syncthing
 
     outputs.nixosModules.tailscale
-    # outputs.nixosModules.journald-upload # FIXME
+    outputs.nixosModules.promtail
 
     ./forgejo.nix
     ./jirafeau.nix

hosts/sid/services/default.nix

@@ -9,7 +9,8 @@
     inputs.synix.nixosModules.openssh
 
     outputs.nixosModules.tailscale
-    # outputs.nixosModules.journald-remote
+    outputs.nixosModules.loki
+    outputs.nixosModules.promtail
 
     ./headscale.nix
     ./mailserver.nix

modules/nixos/default.nix

@@ -5,9 +5,9 @@
   forgejo = import ./forgejo;
   forgejo-runner = import ./forgejo-runner;
   gnome = import ./gnome;
-  journald-remote = import ./journald-remote;
+  loki = import ./loki;
-  journald-upload = import ./journald-upload;
   monero = import ./monero;
+  promtail = import ./promtail;
   rsshub-oci = import ./rsshub-oci;
   tailscale = import ./tailscale;
   xfce = import ./xfce;

modules/nixos/journald-remote/default.nix

@@ -1,52 +0,0 @@
-{
-  pkgs,
-  lib,
-  ...
-}:
-
-let
-  python = pkgs.python3Packages;
-
-  mcp-log-server = python.buildPythonApplication {
-    pname = "mcp-log-server";
-    version = "1.0.0";
-
-    src = ./.;
-
-    pyproject = true;
-    build-system = [ python.setuptools ];
-
-    propagatedBuildInputs = with python; [
-      fastmcp
-    ];
-
-    meta.mainProgram = "mcp-log-server";
-  };
-in
-{
-  services.journald.remote = {
-    enable = true;
-    listen = "http";
-    port = 19532;
-    settings.Remote.SplitMode = "host";
-  };
-
-  systemd.services.mcp-log-server = {
-    description = "AI Log Access MCP Server";
-    after = [
-      "network.target"
-      "multi-user.target"
-      "systemd-journald.service"
-    ];
-    wantedBy = [ "multi-user.target" ];
-
-    script = lib.getExe mcp-log-server;
-
-    serviceConfig = {
-      User = "root";
-      Group = "root";
-      Environment = "PYTHONUNBUFFERED=1";
-      Restart = "on-failure";
-    };
-  };
-}

modules/nixos/journald-remote/pyproject.toml

@@ -1,9 +0,0 @@
-[build-system]
-requires = ["setuptools"]
-
-[project]
-name = "mcp-log-server"
-version = "1.0.0"
-
-[project.scripts]
-mcp-log-server = "server:main"

modules/nixos/journald-remote/server.py

@@ -1,60 +0,0 @@
-#!/usr/bin/env python3
-import json
-import subprocess
-from fastmcp import FastMCP
-
-mcp = FastMCP('ClusterLogQuery')
-
-@mcp.tool()
-async def search_cluster_logs(keyword: str, unit: str = '*', limit: int = 500):
-    """
-    Search centralized systemd logs collected on this server.
-
-    Args:
-        keyword: The string to search for (e.g. 'oom-kill', 'failed').
-        unit: The systemd unit to filter (e.g. 'sshd'). If '*', matches all.
-        limit: The number of log lines to return (Max 1000).
-    """
-    
-    limit = min(max(limit, 10), 1000)
-
-    cmd = [
-        'journalctl',
-        '-n', str(limit),
-        '-o', 'json',
-        '--no-pager',
-        '--directory', '/var/log/journal/remote'
-    ]
-
-    if unit != '*':
-        cmd.append(f'UNIT={unit}')
-    if keyword:
-        cmd.append(keyword)
-
-    try:
-        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout, _ = proc.communicate()
-
-        logs = []
-        for line in stdout.decode('utf-8').strip().split('\n'):
-            if not line: continue
-            try:
-                log = json.loads(line)
-                logs.append({
-                    'timestamp': log.get('TIME_ISO8601'),
-                    'hostname': log.get('_HOSTNAME'),
-                    'unit': log.get('_COMM'),
-                    'message': log.get('MESSAGE')
-                })
-            except: pass
-
-        return json.dumps(logs)
-
-    except Exception as e:
-        return f'Error querying logs: {str(e)}'
-
-def main():
-    mcp.run(transport='sse', host='0.0.0.0', port=9500)
-
-if __name__ == '__main__':
-    main()

modules/nixos/journald-upload/default.nix

@@ -1,17 +0,0 @@
-{
-  services.journald.upload = {
-    enable = true;
-    settings.Upload.URL = "http://100.64.0.6:19532";
-  };
-
-  systemd.services.systemd-journal-upload = {
-    after = [
-      "tailscaled.service"
-      "network-online.target"
-    ];
-    wants = [ "network-online.target" ];
-    serviceConfig = {
-      TimeoutStartSec = 20; # Tailscale is slow
-    };
-  };
-}

modules/nixos/loki/default.nix

@@ -0,0 +1,62 @@
+{
+  services.loki = {
+    enable = true;
+    configuration = {
+      auth_enabled = false;
+      server.http_listen_port = 3100;
+
+      common = {
+        ring = {
+          instance_addr = "127.0.0.1";
+          kvstore.store = "inmemory";
+        };
+        replication_factor = 1;
+        path_prefix = "/var/lib/loki";
+      };
+
+      schema_config = {
+        configs = [
+          {
+            from = "2020-10-24";
+            store = "tsdb";
+            object_store = "filesystem";
+            schema = "v13";
+            index = {
+              prefix = "index_";
+              period = "24h";
+            };
+          }
+        ];
+      };
+
+      storage_config = {
+        filesystem = {
+          directory = "/var/lib/loki/chunks";
+        };
+      };
+    };
+  };
+
+  services.grafana = {
+    enable = true;
+    settings = {
+      server = {
+        http_addr = "0.0.0.0";
+        http_port = 3003;
+      };
+    };
+
+    provision = {
+      enable = true;
+      datasources.settings.datasources = [
+        {
+          name = "Loki";
+          type = "loki";
+          access = "proxy";
+          url = "http://127.0.0.1:3100";
+          isDefault = true;
+        }
+      ];
+    };
+  };
+}

modules/nixos/promtail/default.nix

@@ -0,0 +1,43 @@
+{ config, constants, ... }:
+
+{
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 9080;
+        grpc_listen_port = 0;
+      };
+
+      clients = [
+        {
+          url = "http://${constants.hosts.sid.ip}:3100/loki/api/v1/push";
+        }
+      ];
+
+      scrape_configs = [
+        {
+          job_name = "journal";
+          journal = {
+            max_age = "12h";
+            path = "/var/log/journal";
+
+            labels = {
+              job = "systemd-journal";
+              host = config.networking.hostName;
+            };
+          };
+
+          relabel_configs = [
+            {
+              source_labels = [ "__journal__systemd_unit" ];
+              target_label = "unit";
+            }
+          ];
+        }
+      ];
+    };
+  };
+
+  users.users.promtail.extraGroups = [ "systemd-journal" ];
+}