diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..8defb09 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,355 @@ +# AGENTS.md + +This file teaches AI agents how to navigate and work with this NixOS configuration repository (`sid.ovh`). + +--- + +## Repository Overview + +This is a NixOS flake managing three hosts: + +| Host | IP (Tailscale) | Role | +|-------|----------------|------------------------------------------------------| +| `sid` | `100.64.0.6` | VPS - reverse proxy, mail, matrix, headscale | +| `rx4` | `100.64.0.10` | Home server - open-webui, forgejo, vaultwarden, etc. | +| `vde` | `100.64.0.1` | Desktop / workstation (not in use at the moment) | + +Deployment is done via `deploy-rs` through a Forgejo CI pipeline (`.forgejo/workflows/deploy-configs.yml`). + +--- + +## The `synix` Flake Input + +### What it is + +`synix` is a **NixOS library flake**, hosted at: + +``` +https://git.sid.ovh/sid/synix.git +``` + +It provides: +- **NixOS modules** (`inputs.synix.nixosModules.*`): opinionated, reusable service configurations used heavily across all three hosts. +- **Home Manager modules** (`inputs.synix.homeModules.*`): desktop / user-space configurations (Hyprland, nixvim, waybar, etc.). +- **Packages** (`inputs.synix.packages..*`, also accessible via the `synix-packages` overlay as `pkgs.synix.*`): custom packages not in nixpkgs. +- **Overlays** (`inputs.synix.overlays.*`): nixpkgs modifications. +- **A utility library** (`inputs.synix.lib` / `lib.utils`): helper functions such as `mkVirtualHost` and `mkReverseProxyOption`. +- **Templates**: starter flake templates for servers, desktops, VMs, etc. +- **Apps**: convenience scripts (`deploy`, `rebuild`, `install`, etc.). + +### How it is consumed in `sid.ovh` + +In `flake.nix`: + +```nix +inputs = { + synix.url = "git+https://git.sid.ovh/sid/synix.git?ref=release-25.11"; + synix.inputs.nixpkgs.follows = "nixpkgs"; + # development overrides (commented out normally): + # synix.url = "git+https://git.sid.ovh/sid/synix.git?ref=develop"; + # synix.url = "git+file:///home/sid/src/synix"; # local checkout + ... +}; +``` + +The active branch is **`release-25.11`**, aligned with `nixpkgs` at `nixos-25.11`. A `develop` branch exists for unstable work. A local checkout can be used for testing by switching to the `git+file://` URL. + +--- + +## `synix` Module Reference + +All NixOS modules are under `modules/nixos/` in the synix repo and exposed as `inputs.synix.nixosModules.`. + +### Infrastructure / base modules + +| Module name | Path in synix | Purpose | +|------------------------------------|-----------------------------------|---------| +| `common` | `modules/nixos/common/` | Base system config (boot, networking, nix settings, zsh, sudo, htop, locale) | +| `device.server` | `modules/nixos/device/server.nix` | Server profile (no GUI, sensible server defaults) | +| `device.desktop` | `modules/nixos/device/desktop.nix`| Desktop profile | +| `device.laptop` | `modules/nixos/device/laptop.nix` | Laptop profile | +| `device.vm` | `modules/nixos/device/vm.nix` | VM profile | +| `sops` | `modules/nixos/sops/` | sops-nix wiring (age keys, secret paths) | +| `openssh` | `modules/nixos/openssh/` | Hardened OpenSSH configuration | +| `normalUsers` | `modules/nixos/normalUsers/` | Declarative normal-user creation with SSH keys | +| `tailscale` | `modules/nixos/tailscale/` | Tailscale VPN with multiple tailnet support | +| `nginx` | `modules/nixos/nginx/` | Opinionated nginx base (ACME, SSL helpers) | + +### Service modules used in `sid.ovh` + +| Module name | Path in synix | Used by host | Notes | +|----------------------|--------------------------------------|--------------|-------| +| `headscale` | `modules/nixos/headscale/` | sid | Self-hosted Tailscale control plane | +| `headplane` | `modules/nixos/headplane/` | sid | Headscale web UI | +| `mailserver` | `modules/nixos/mailserver/` | sid | Wraps `nixos-mailserver` | +| `matrix-synapse` | `modules/nixos/matrix-synapse/` | sid | Matrix homeserver + bridges (WhatsApp, Signal) + LiveKit | +| `maubot` | `modules/nixos/maubot/` | sid | Matrix bot framework | +| `baibot` | `modules/nixos/baibot/` | sid | AI Matrix bot | +| `coturn` | `modules/nixos/coturn/` | sid | TURN/STUN server (Matrix calls) | +| `radicale` | `modules/nixos/radicale/` | sid | CalDAV/CardDAV server | +| `rss-bridge` | `modules/nixos/rss-bridge/` | sid | RSS-Bridge PHP app | +| `miniflux` | `modules/nixos/miniflux/` | rx4 | RSS reader | +| `jirafeau` | `modules/nixos/jirafeau/` | rx4 | File sharing | +| `open-webui-oci` | `modules/nixos/open-webui-oci/` | rx4 | Open WebUI via Podman OCI container | +| `mcpo` | `modules/nixos/mcpo/` | rx4 | MCP-to-OpenAPI proxy (AI tool server) | +| `print-server` | `modules/nixos/print-server/` | rx4 | CUPS print server | +| `virtualisation` | `modules/nixos/virtualisation/` | vde | VFIO / KVM / QEMU / kvmfr setup | + +### Other available modules (not currently active in `sid.ovh`) + +`audio`, `bluetooth`, `amd`, `nvidia`, `jellyfin`, `i2pd`, `ftp-webserver`, `webPage`, `windows-oci`, `librechat-oci`, `nostr-relay`, `ollama`, `cifsMount`, `hyprland` (NixOS-level), `normalUsers`. + +--- + +## `synix` Packages (`pkgs.synix.*`) + +Custom packages provided by synix and available via the `synix-packages` overlay (applied in `modules/nixos/common/overlays.nix`): + +| Package name | What it is | +|-------------------------------|------------| +| `pkgs.synix.mcpo` | MCP-to-OpenAPI proxy server | +| `pkgs.synix.fetcher-mcp` | MCP tool: web page fetcher | +| `pkgs.synix.baibot` | AI Matrix bot | +| `pkgs.synix.jirafeau` | Jirafeau file-sharing webapp | +| `pkgs.synix.jirafeau-cli` | CLI client for Jirafeau | +| `pkgs.synix.marker-pdf` | PDF-to-markdown converter | +| `pkgs.synix.mcpo` | MCP proxy | +| `pkgs.synix.systemctl-tui` | TUI for systemctl | +| `pkgs.synix.quicknote` | Quick note shell script | +| `pkgs.synix.synix-docs` | Built MkDocs documentation site | +| `pkgs.synix.bulk-rename` | Batch file renaming script | +| `pkgs.synix.pyman` | Python man-page helper | + +The full list lives in `pkgs/default.nix` in the synix repo. All packages are also accessible as `inputs.synix.packages..`. + +--- + +## `synix` Utility Library (`lib.utils`) + +Exposed as `inputs.synix.lib` and merged into the flake's `lib` in `flake.nix`: + +```nix +lib = nixpkgs.lib.extend (final: prev: inputs.synix.lib or { }); +``` + +Key helpers (defined in `lib/utils.nix`): + +| Function | What it does | +|---------------------------|--------------| +| `lib.utils.mkVirtualHost` | Builds a standard nginx `virtualHosts` entry with optional SSL, upstream address, port, and extra config | +| `lib.utils.mkReverseProxyOption` | Generates a standard NixOS option set for enabling a reverse-proxy sub-config on a module | + +Usage example from `hosts/sid/services/nginx.nix`: + +```nix +virtualHosts."${constants.services.forgejo.fqdn}" = mkVirtualHost { + inherit ssl; + address = constants.hosts.rx4.ip; + port = constants.services.forgejo.port; +}; +``` + +--- + +## How `synix` Modules Are Imported + +In each host's `default.nix`, synix modules are imported directly from the `inputs` special arg: + +```nix +# hosts/rx4/default.nix +imports = [ + inputs.synix.nixosModules.common # base system config + inputs.synix.nixosModules.device.server # server profile + inputs.synix.nixosModules.openssh # hardened SSH + # service-specific modules in hosts/rx4/services/ +]; +``` + +Service files then add more synix modules as needed: + +```nix +# hosts/rx4/services/open-webui-oci.nix +imports = [ + inputs.synix.nixosModules.open-webui-oci + inputs.synix.nixosModules.mcpo +]; +``` + +--- + +## Overlays Applied + +`modules/nixos/common/overlays.nix` applies five overlays to every host: + +| Overlay name | Source | Accessible as | +|-------------------------|------------------|----------------------| +| `synix-packages` | `synix` | `pkgs.synix.*` | +| `local-packages` | `pkgs/` in repo | `pkgs.local.*` | +| `modifications` | `overlays/` | replaces pkgs in place | +| `old-stable-packages` | `nixpkgs-25.05` | `pkgs.old-stable.*` | +| `unstable-packages` | `nixos-unstable` | `pkgs.unstable.*` | + +`synix` also supplies its own `modifications` overlay (`inputs.synix.overlays.modifications`), merged into the local `modifications` overlay in `overlays/default.nix`. + +--- + +## Centralized Logging Architecture + +All hosts ship their systemd journal to a **central receiver running on `sid`** over HTTP using `systemd-journal-remote` / `systemd-journal-upload`. + +### Receiver (local host `pc` which is not part of this flake) + +```nix +services.journald.remote = { + enable = true; + listen = "http"; + port = 19532; + settings.Remote.SplitMode = "host"; # one journal dir per sending host +}; + +users.users.sid.extraGroups = [ + "systemd-journal" + "systemd-journal-remote" +]; +``` + +Key points: +- Listens on **`http://0.0.0.0:19532`** (plain HTTP, Tailscale-only network). +- `SplitMode = "host"` stores each remote host's entries under + `/var/log/journal/remote/`, separated by hostname. + +### Senders (all hosts via `modules/nixos/common/journald.nix`) + +```nix +services.journald.upload = { + enable = true; + settings.Upload.URL = "http://100.64.0.5:19532"; +}; +``` + +> **Note:** The upload URL uses `100.64.0.5` - the Tailscale transport IP for host `pc` (which is not part of this flake). Verify with `ip addr show tailscale0` and `hostname` if you are on host `pc` with IP `100.64.0.5`. + +This module is applied to **every host** via `outputs.nixosModules.common` -> `modules/nixos/common/default.nix`. + +--- + +## Querying Remote Journals + +All queries run **on `pc`**, reading from `/var/log/journal/remote/`. + +### General pattern + +```bash +journalctl -D /var/log/journal/remote/ \ + _HOSTNAME= \ + [SYSTEMD_UNIT=] \ + [other filters…] \ + --no-pager [-n ] [-f] [-S ] [-U ] +``` + +| Flag / Field | Purpose | +|-------------------------------------------|---------| +| `-D /var/log/journal/remote/` | Read the remote journal directory | +| `_HOSTNAME=rx4` | Filter to entries from host `rx4` | +| `SYSTEMD_UNIT=podman-open-webui.service` | Filter to a specific systemd unit | +| `--no-pager` | Don't page output; good for scripts | +| `-n 20` | Show last 20 lines | +| `-f` | Follow (tail) mode | +| `-S` / `-U` | Since / Until time bounds | + +### Canonical example — last 20 lines of open-webui on rx4 + +```bash +journalctl \ + -D /var/log/journal/remote/ \ + _HOSTNAME=rx4 \ + SYSTEMD_UNIT=podman-open-webui.service \ + --no-pager -n 20 +``` + +This works because: +1. `rx4` uploads its journal to `pc:19532`. +2. The receiver stores it under `/var/log/journal/remote/` with `SplitMode=host`, preserving the `_HOSTNAME` field. +3. `_HOSTNAME=rx4` narrows to that host's entries. +4. `SYSTEMD_UNIT=podman-open-webui.service` targets the OCI container unit for Open WebUI (defined in `hosts/rx4/services/open-webui-oci.nix`). + +### Other useful queries + +```bash +# All logs from rx4 in the last hour +journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 --no-pager -S "1 hour ago" + +# Forgejo on rx4 +journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 SYSTEMD_UNIT=forgejo.service --no-pager -n 50 + +# Vaultwarden on rx4 +journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 SYSTEMD_UNIT=vaultwarden.service --no-pager -n 50 + +# Nginx on rx4 +journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 SYSTEMD_UNIT=nginx.service --no-pager -n 50 + +# Matrix-synapse on sid +journalctl -D /var/log/journal/remote/ _HOSTNAME=sid SYSTEMD_UNIT=matrix-synapse.service --no-pager -n 50 + +# Follow logs live +journalctl -D /var/log/journal/remote/ _HOSTNAME=rx4 SYSTEMD_UNIT=podman-open-webui.service -f +``` + +### OCI container unit names + +Containers managed by `virtualisation.oci-containers` get units named `podman-.service`: + +| Container attr | Unit name | Host | +|------------------------|----------------------------------------|------| +| `open-webui` | `podman-open-webui.service` | rx4 | +| `rsshub-rsshub` | `podman-rsshub-rsshub.service` | rx4 | +| `rsshub-redis` | `podman-rsshub-redis.service` | rx4 | +| `rsshub-browserless` | `podman-rsshub-browserless.service` | rx4 | +| `rsshub-real-browser` | `podman-rsshub-real-browser.service` | rx4 | + +--- + +## Permissions + +User `sid` must be in both `systemd-journal` and `systemd-journal-remote` to read `/var/log/journal/remote/`: + +```nix +users.users.sid.extraGroups = [ + "systemd-journal" + "systemd-journal-remote" +]; +``` + +Verify with `id sid` on `pc`. + +--- + +## Key File Locations + +| Path | Purpose | +|-----------------------------------------|---------| +| `flake.nix` | Flake inputs, host configs, deploy nodes | +| `constants.nix` | Canonical IPs, FQDNs, service ports | +| `modules/nixos/common/journald.nix` | Enables `journald.upload` on all hosts | +| `modules/nixos/common/default.nix` | Imports all common modules | +| `modules/nixos/common/overlays.nix` | Applies all five overlays | +| `hosts//default.nix` | Per-host top-level import list | +| `hosts//services/` | Per-host service configurations | +| `hosts//secrets/secrets.yaml` | sops-encrypted secrets for that host | +| `overlays/default.nix` | Local overlay definitions | +| `pkgs/` | Local packages (`pkgs.local.*`) | +| `users/sid/` | User account + SSH public key | + +--- + +## Troubleshooting + +| Symptom | Check | +|---------|-------| +| No journal entries for a host | `systemctl status systemd-journal-upload` on the source host | +| Permission denied reading remote journal | `id sid` on `sid` — needs `systemd-journal` + `systemd-journal-remote` | +| Wrong unit name | `systemctl list-units --type=service` on the source host | +| Receiver unreachable | `ss -tlnp \| grep 19532` on `sid` | +| Upload URL IP mismatch | Upload uses `100.64.0.5`; verify `ip addr show tailscale0` on `sid` | +| synix module not found | Check the branch in `flake.nix`; run `nix flake update synix` to refresh | +| Want to test a synix change locally | Switch input URL to `git+file:///home/sid/src/synix` | diff --git a/constants.nix b/constants.nix index 805fbf9..7b3578e 100644 --- a/constants.nix +++ b/constants.nix @@ -25,6 +25,10 @@ rec { subdomain = "f"; fqdn = subdomain + "." + domain; }; + librechat-oci = { + fqdn = "lc." + domain; + port = 3080; + }; mailserver = rec { subdomain = "mail"; fqdn = subdomain + "." + domain; diff --git a/hosts/rx4/services/default.nix b/hosts/rx4/services/default.nix index 6cb2dc6..8db7081 100644 --- a/hosts/rx4/services/default.nix +++ b/hosts/rx4/services/default.nix @@ -13,6 +13,7 @@ ./forgejo.nix ./jirafeau.nix + ./librechat-oci.nix ./miniflux.nix ./netdata.nix ./nginx.nix diff --git a/hosts/rx4/services/librechat-oci.nix b/hosts/rx4/services/librechat-oci.nix new file mode 100644 index 0000000..07ab8d3 --- /dev/null +++ b/hosts/rx4/services/librechat-oci.nix @@ -0,0 +1,44 @@ +{ + inputs, + constants, + config, + ... +}: + +let + inherit (constants) domain; + inherit (constants.hosts.rx4) ip; + inherit (constants.services.librechat-oci) fqdn port; +in +{ + imports = [ + inputs.synix.nixosModules.librechat-oci + ]; + + services.librechat-oci = { + enable = true; + inherit port; + externalUrl = "https://${fqdn}"; + }; + + services.nginx.virtualHosts."${fqdn}" = { + useACMEHost = fqdn; + forceSSL = true; + listen = [ + { + addr = "${ip}:443"; + ssl = true; + } + ]; + locations."/" = { + proxyPass = "http://127.0.0.1:${toString port}"; + proxyWebsockets = true; + }; + }; + + security.acme.certs."${fqdn}" = { + domain = fqdn; + postRun = "systemctl restart podman-librechat.service"; + group = "nginx"; + }; +}