From 4da48d5f99791b2388153b491a4a9f97d746badf Mon Sep 17 00:00:00 2001 From: sid Date: Fri, 22 May 2026 19:10:46 +0200 Subject: [PATCH] replace journald witrh loki+promtail --- hosts/rx4/services/default.nix | 2 +- hosts/sid/services/default.nix | 3 +- modules/nixos/default.nix | 4 +- modules/nixos/journald-remote/default.nix | 52 ---------------- modules/nixos/journald-remote/pyproject.toml | 9 --- modules/nixos/journald-remote/server.py | 60 ------------------- modules/nixos/journald-upload/default.nix | 17 ------ modules/nixos/loki/default.nix | 62 ++++++++++++++++++++ modules/nixos/promtail/default.nix | 43 ++++++++++++++ 9 files changed, 110 insertions(+), 142 deletions(-) delete mode 100644 modules/nixos/journald-remote/default.nix delete mode 100644 modules/nixos/journald-remote/pyproject.toml delete mode 100644 modules/nixos/journald-remote/server.py delete mode 100644 modules/nixos/journald-upload/default.nix create mode 100644 modules/nixos/loki/default.nix create mode 100644 modules/nixos/promtail/default.nix diff --git a/hosts/rx4/services/default.nix b/hosts/rx4/services/default.nix index 1014d6f..5201854 100644 --- a/hosts/rx4/services/default.nix +++ b/hosts/rx4/services/default.nix @@ -10,7 +10,7 @@ inputs.clients.nixosModules.syncthing outputs.nixosModules.tailscale - # outputs.nixosModules.journald-upload # FIXME + outputs.nixosModules.promtail ./forgejo.nix ./jirafeau.nix diff --git a/hosts/sid/services/default.nix b/hosts/sid/services/default.nix index b40b165..1be8328 100644 --- a/hosts/sid/services/default.nix +++ b/hosts/sid/services/default.nix @@ -9,7 +9,8 @@ inputs.synix.nixosModules.openssh outputs.nixosModules.tailscale - # outputs.nixosModules.journald-remote + outputs.nixosModules.loki + outputs.nixosModules.promtail ./headscale.nix ./mailserver.nix diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix index 2e35096..d3511ee 100644 --- a/modules/nixos/default.nix +++ b/modules/nixos/default.nix @@ -5,9 +5,9 @@ forgejo = import ./forgejo; forgejo-runner = import ./forgejo-runner; gnome = import ./gnome; - journald-remote = import ./journald-remote; - journald-upload = import ./journald-upload; + loki = import ./loki; monero = import ./monero; + promtail = import ./promtail; rsshub-oci = import ./rsshub-oci; tailscale = import ./tailscale; xfce = import ./xfce; diff --git a/modules/nixos/journald-remote/default.nix b/modules/nixos/journald-remote/default.nix deleted file mode 100644 index 797eef3..0000000 --- a/modules/nixos/journald-remote/default.nix +++ /dev/null @@ -1,52 +0,0 @@ -{ - pkgs, - lib, - ... -}: - -let - python = pkgs.python3Packages; - - mcp-log-server = python.buildPythonApplication { - pname = "mcp-log-server"; - version = "1.0.0"; - - src = ./.; - - pyproject = true; - build-system = [ python.setuptools ]; - - propagatedBuildInputs = with python; [ - fastmcp - ]; - - meta.mainProgram = "mcp-log-server"; - }; -in -{ - services.journald.remote = { - enable = true; - listen = "http"; - port = 19532; - settings.Remote.SplitMode = "host"; - }; - - systemd.services.mcp-log-server = { - description = "AI Log Access MCP Server"; - after = [ - "network.target" - "multi-user.target" - "systemd-journald.service" - ]; - wantedBy = [ "multi-user.target" ]; - - script = lib.getExe mcp-log-server; - - serviceConfig = { - User = "root"; - Group = "root"; - Environment = "PYTHONUNBUFFERED=1"; - Restart = "on-failure"; - }; - }; -} diff --git a/modules/nixos/journald-remote/pyproject.toml b/modules/nixos/journald-remote/pyproject.toml deleted file mode 100644 index 809656b..0000000 --- a/modules/nixos/journald-remote/pyproject.toml +++ /dev/null @@ -1,9 +0,0 @@ -[build-system] -requires = ["setuptools"] - -[project] -name = "mcp-log-server" -version = "1.0.0" - -[project.scripts] -mcp-log-server = "server:main" diff --git a/modules/nixos/journald-remote/server.py b/modules/nixos/journald-remote/server.py deleted file mode 100644 index 39c8f28..0000000 --- a/modules/nixos/journald-remote/server.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -import json -import subprocess -from fastmcp import FastMCP - -mcp = FastMCP('ClusterLogQuery') - -@mcp.tool() -async def search_cluster_logs(keyword: str, unit: str = '*', limit: int = 500): - """ - Search centralized systemd logs collected on this server. - - Args: - keyword: The string to search for (e.g. 'oom-kill', 'failed'). - unit: The systemd unit to filter (e.g. 'sshd'). If '*', matches all. - limit: The number of log lines to return (Max 1000). - """ - - limit = min(max(limit, 10), 1000) - - cmd = [ - 'journalctl', - '-n', str(limit), - '-o', 'json', - '--no-pager', - '--directory', '/var/log/journal/remote' - ] - - if unit != '*': - cmd.append(f'UNIT={unit}') - if keyword: - cmd.append(keyword) - - try: - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, _ = proc.communicate() - - logs = [] - for line in stdout.decode('utf-8').strip().split('\n'): - if not line: continue - try: - log = json.loads(line) - logs.append({ - 'timestamp': log.get('TIME_ISO8601'), - 'hostname': log.get('_HOSTNAME'), - 'unit': log.get('_COMM'), - 'message': log.get('MESSAGE') - }) - except: pass - - return json.dumps(logs) - - except Exception as e: - return f'Error querying logs: {str(e)}' - -def main(): - mcp.run(transport='sse', host='0.0.0.0', port=9500) - -if __name__ == '__main__': - main() diff --git a/modules/nixos/journald-upload/default.nix b/modules/nixos/journald-upload/default.nix deleted file mode 100644 index dede56b..0000000 --- a/modules/nixos/journald-upload/default.nix +++ /dev/null @@ -1,17 +0,0 @@ -{ - services.journald.upload = { - enable = true; - settings.Upload.URL = "http://100.64.0.6:19532"; - }; - - systemd.services.systemd-journal-upload = { - after = [ - "tailscaled.service" - "network-online.target" - ]; - wants = [ "network-online.target" ]; - serviceConfig = { - TimeoutStartSec = 20; # Tailscale is slow - }; - }; -} diff --git a/modules/nixos/loki/default.nix b/modules/nixos/loki/default.nix new file mode 100644 index 0000000..a360c0d --- /dev/null +++ b/modules/nixos/loki/default.nix @@ -0,0 +1,62 @@ +{ + services.loki = { + enable = true; + configuration = { + auth_enabled = false; + server.http_listen_port = 3100; + + common = { + ring = { + instance_addr = "127.0.0.1"; + kvstore.store = "inmemory"; + }; + replication_factor = 1; + path_prefix = "/var/lib/loki"; + }; + + schema_config = { + configs = [ + { + from = "2020-10-24"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + }; + + storage_config = { + filesystem = { + directory = "/var/lib/loki/chunks"; + }; + }; + }; + }; + + services.grafana = { + enable = true; + settings = { + server = { + http_addr = "0.0.0.0"; + http_port = 3003; + }; + }; + + provision = { + enable = true; + datasources.settings.datasources = [ + { + name = "Loki"; + type = "loki"; + access = "proxy"; + url = "http://127.0.0.1:3100"; + isDefault = true; + } + ]; + }; + }; +} diff --git a/modules/nixos/promtail/default.nix b/modules/nixos/promtail/default.nix new file mode 100644 index 0000000..cdb99de --- /dev/null +++ b/modules/nixos/promtail/default.nix @@ -0,0 +1,43 @@ +{ config, constants, ... }: + +{ + services.promtail = { + enable = true; + configuration = { + server = { + http_listen_port = 9080; + grpc_listen_port = 0; + }; + + clients = [ + { + url = "http://${constants.hosts.sid.ip}:3100/loki/api/v1/push"; + } + ]; + + scrape_configs = [ + { + job_name = "journal"; + journal = { + max_age = "12h"; + path = "/var/log/journal"; + + labels = { + job = "systemd-journal"; + host = config.networking.hostName; + }; + }; + + relabel_configs = [ + { + source_labels = [ "__journal__systemd_unit" ]; + target_label = "unit"; + } + ]; + } + ]; + }; + }; + + users.users.promtail.extraGroups = [ "systemd-journal" ]; +} -- 2.51.2