Merge branch 'main' into worktree-agent-bench-m6

This commit is contained in:
sir_melonchen
2026-10-15 19:40:34 +02:00
26 changed files with 1520 additions and 56 deletions
+29 -2
View File
@@ -99,6 +99,8 @@ jobs:
run: make lint-shell
- name: SPDX headers
run: make lint-spdx
- name: No hard-coded product names (ADR 0018)
run: make lint-brand
release-dry-run:
name: Release build (dry run)
@@ -126,10 +128,35 @@ jobs:
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: "24"
- name: Build release artefacts
run: make release
- name: Install nfpm
run: scripts/ci/install-tools.sh "$HOME/.local/bin" nfpm
- name: Build release artefacts and packages
run: make release PACKAGES=1
- name: Verify checksums
run: cd dist && sha256sum -c SHA256SUMS
- name: Inspect packages
run: |
ls -l dist/*.deb dist/*.rpm
# 3 binaries x 2 architectures x (deb, rpm)
test "$(ls dist/*.deb dist/*.rpm | wc -l)" -eq 12
for deb in dist/*_amd64.deb; do
dpkg-deb --info "$deb" >/dev/null
dpkg-deb --contents "$deb" >contents.txt
grep -q ' ./usr/bin/' contents.txt
grep -q ' ./usr/lib/systemd/system/.*\.service$' contents.txt
grep -q ' ./usr/share/doc/.*/LICENSE$' contents.txt
done
- name: Release signature (throwaway key)
run: |
command -v openssl >/dev/null || (apt-get update -qq && apt-get install -y -qq openssl)
scripts/sign-release.sh selftest
umask 077
key="$(mktemp)"
openssl genpkey -algorithm ed25519 -out "$key"
RELEASE_SIGNING_KEY="$(cat "$key")" scripts/sign-release.sh sign dist/SHA256SUMS
scripts/sign-release.sh public-key "$key" >"$key.pub"
scripts/sign-release.sh verify dist/SHA256SUMS dist/SHA256SUMS.sig "$key.pub"
rm -f "$key" "$key.pub" dist/SHA256SUMS.sig
- name: Smoke test of the released server binary
env:
DDI_DATABASE_URL: postgres://ddi:ddi-ci-only@postgres:5432/ddi?sslmode=disable
+31 -3
View File
@@ -23,12 +23,40 @@ jobs:
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: "24"
- name: Ensure jq
run: command -v jq >/dev/null || (apt-get update -qq && apt-get install -y -qq jq)
- name: Ensure jq and openssl
run: |
command -v jq >/dev/null && command -v openssl >/dev/null ||
(apt-get update -qq && apt-get install -y -qq jq openssl)
- name: Install nfpm
run: scripts/ci/install-tools.sh "$HOME/.local/bin" nfpm
- name: Build
run: make release VERSION="${{ github.ref_name }}"
run: make release PACKAGES=1 VERSION="${{ github.ref_name }}"
- name: Verify checksums
run: cd dist && sha256sum -c SHA256SUMS
# Ed25519 signature over SHA256SUMS (ADR 0024, docs/RELEASE.md) if the
# secret RELEASE_SIGNING_KEY (PEM private key) is set. The key only
# exists in a private temporary file during signing.
- name: Sign checksums
env:
RELEASE_SIGNING_KEY: ${{ secrets.RELEASE_SIGNING_KEY }}
run: |
if [[ -z "${RELEASE_SIGNING_KEY:-}" ]]; then
echo "::warning::RELEASE_SIGNING_KEY is not set: the release is not signed"
exit 0
fi
scripts/sign-release.sh sign dist/SHA256SUMS
# The installer must accept the signature: verify with its public
# key if one is configured there.
pub="$(mktemp)"
sed -n -e 's/^[[:space:]]*//' -e 's/.*\(-----BEGIN PUBLIC KEY-----\)/\1/' \
-e 's/\(-----END PUBLIC KEY-----\).*/\1/' \
-e '/-----BEGIN PUBLIC KEY-----/,/-----END PUBLIC KEY-----/p' scripts/install.sh >"$pub" 2>/dev/null || true
if [[ -s "$pub" ]]; then
scripts/sign-release.sh verify dist/SHA256SUMS dist/SHA256SUMS.sig "$pub"
else
echo "::warning::no public key in scripts/install.sh: the installer will not check the signature"
fi
rm -f "$pub"
- name: Upload to release
env:
GITEA_API_URL: ${{ github.server_url }}/api/v1
+24 -2
View File
@@ -25,6 +25,10 @@ BUILD_DATE ?= $(shell git log -1 --format=%cI 2>/dev/null)
PLATFORMS ?= linux/amd64 linux/arm64
BIN_DIR ?= bin
DIST_DIR ?= dist
# nfpm for make packages / PACKAGES=1; scripts/ci/install-tools.sh installs
# the pinned version (NFPM_VERSION there).
NFPM ?= nfpm
export NFPM
COMPOSE_DEV := docker compose -f deploy/compose/compose.dev.yaml
@@ -103,7 +107,7 @@ fmt:
## lint: run all linters
.PHONY: lint
lint: lint-go lint-proto lint-web lint-shell lint-spdx
lint: lint-go lint-proto lint-web lint-shell lint-spdx lint-brand
.PHONY: lint-go
lint-go:
@@ -127,6 +131,11 @@ lint-shell:
lint-spdx:
scripts/check-spdx.sh
## lint-brand: no hard-coded product names in units, packages, installer (ADR 0018)
.PHONY: lint-brand
lint-brand:
scripts/check-brand.sh
## test: run Go and frontend unit tests
.PHONY: test
test: test-go test-web
@@ -167,10 +176,23 @@ ci: lint check-generated test vuln build
# --------------------------------------------------------------------------- release
## release: cross-compile tarballs and SHA256SUMS into dist/
## release: cross-compile tarballs and SHA256SUMS into dist/ (PACKAGES=1: also .deb/.rpm)
.PHONY: release
release: web
scripts/build-release.sh -o $(DIST_DIR) -p "$(PLATFORMS)"
ifeq ($(PACKAGES),1)
scripts/build-packages.sh -d $(DIST_DIR) -p "$(PLATFORMS)"
endif
## packages: .deb and .rpm from the tarballs in dist/ (after make release; needs nfpm)
.PHONY: packages
packages:
scripts/build-packages.sh -d $(DIST_DIR) -p "$(PLATFORMS)"
## sign-release: sign dist/SHA256SUMS with the Ed25519 key in RELEASE_SIGNING_KEY_FILE
.PHONY: sign-release
sign-release:
scripts/sign-release.sh sign $(DIST_DIR)/SHA256SUMS "$(RELEASE_SIGNING_KEY_FILE)"
# --------------------------------------------------------------------------- development
+39
View File
@@ -0,0 +1,39 @@
# SPDX-License-Identifier: Apache-2.0
#
# Template of the nfpm configuration of the .deb and .rpm packages (ADR 0024),
# shared by all binaries. scripts/build-packages.sh replaces the placeholders
# (at-sign NAME at-sign) per binary and architecture (ADR 0018: no product
# names in this file). The
# packages contain only the binary, its systemd unit and the license text;
# configuration, users and services are set up by the installer.
name: "@PKG_NAME@"
arch: "@PKG_ARCH@"
platform: linux
version: "@PKG_VERSION@"
version_schema: semver
release: "1"
section: net
priority: optional
maintainer: "@PKG_MAINTAINER@"
vendor: "@PKG_VENDOR@"
homepage: "@PKG_HOMEPAGE@"
license: "@PKG_LICENSE@"
description: "@PKG_DESCRIPTION@"
contents:
- src: "@PKG_SRC_DIR@/binary"
dst: "/usr/bin/@PKG_NAME@"
file_info:
mode: 0755
- src: "@PKG_SRC_DIR@/unit.service"
dst: "/usr/lib/systemd/system/@PKG_NAME@.service"
file_info:
mode: 0644
- src: "@PKG_SRC_DIR@/LICENSE"
dst: "/usr/share/doc/@PKG_NAME@/LICENSE"
file_info:
mode: 0644
scripts:
postinstall: "@PKG_SCRIPTS_DIR@/postinstall.sh"
postremove: "@PKG_SCRIPTS_DIR@/postremove.sh"
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
# SPDX-License-Identifier: Apache-2.0
#
# Package hook (deb postinst, rpm %post): make systemd see the new or
# updated unit. Enabling and starting services is left to the installer.
if [ -d /run/systemd/system ]; then
systemctl daemon-reload >/dev/null 2>&1 || true
fi
exit 0
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
# SPDX-License-Identifier: Apache-2.0
#
# Package hook (deb postrm, rpm %postun): make systemd forget the removed
# unit. Stopping and disabling services is left to the installer.
if [ -d /run/systemd/system ]; then
systemctl daemon-reload >/dev/null 2>&1 || true
fi
exit 0
+4 -1
View File
@@ -201,14 +201,17 @@ Server und Agents lesen eine YAML-Datei (Server: Pfad aus `DDI_CONFIG_FILE`, Bei
| `DDI_LOG_LEVEL`, `DDI_LOG_FORMAT` | Logging |
| `DDI_SESSION_LIFETIME`, `DDI_SESSION_IDLE_TIMEOUT` | Sitzungen |
| `DDI_LICENSE_GRACE_PERIOD`, `KEYGEN_URL`, `KEYGEN_PRODUCT_ID` | Lizenz (siehe [LICENSING.md](LICENSING.md)) |
| `DDI_RETENTION_AUDIT_DAYS`, `DDI_RETENTION_REVISIONS_KEEP` | Aufbewahrung (`retention.audit_days`, `retention.revisions_keep`; 0 = alles behalten) |
`KEYGEN_ACCOUNT_ID` und `KEYGEN_PUBLIC_KEY` gelten nur für Builds ohne eingebettete Werte (Entwicklung). Dauerangaben verstehen Go-Notation und Tage (`14d`).
**Aufbewahrung** ([0024](adr/0024-auslieferung-installer-m6.md)): Ist `retention.audit_days` oder `retention.revisions_keep` gesetzt, räumt der Server fünf Minuten nach dem Start und danach täglich auf. Gelöscht werden Audit-Einträge, die älter als die angegebene Zahl Tage sind, und Revisionen jenseits der neuesten `revisions_keep` (samt Dateien). Nie gelöscht werden die neueste Revision, Revisionen, die ein Agent anwendet, als letzte funktionierende kennt oder als Ziel hat, und Revisionen mit offenen Deployments. Ein Aufräumlauf, der etwas löscht, schreibt den Audit-Eintrag `retention.cleanup` mit den Anzahlen.
## 8. Build, CI und Auslieferung
- `Makefile` als Einstieg: `build`, `test`, `lint`, `fmt`, `generate`, `web`, `dev-up`/`dev-down`, `release`.
- CI: Gitea Actions unter `.gitea/workflows/` ([0003](adr/0003-git-host-gitea-actions.md)). Geprüft werden Go (Format, `vet`, `golangci-lint`, Tests, `govulncheck`), Protobuf (`buf lint`, generierter Code aktuell), Web (ESLint, Typecheck, Build, Tests, `npm audit`), Shell (`shellcheck`), SPDX-Header und die Entwicklungsumgebung (Compose-Smoke-Test).
- Release bei Tag `v*`: Binaries für `linux/amd64` und `linux/arm64`, Tarballs, `SHA256SUMS`, Upload an das Gitea-Release.
- Release bei Tag `v*`: Binaries für `linux/amd64` und `linux/arm64`, Tarballs, `.deb`/`.rpm` (nfpm), `SHA256SUMS` und – mit Secret `RELEASE_SIGNING_KEY` – die Ed25519-Signatur `SHA256SUMS.sig`, Upload an das Gitea-Release ([RELEASE.md](RELEASE.md)).
- Installer `scripts/install.sh` (M6) lädt die Artefakte und prüft die Prüfsummen.
## 9. Entwicklungsumgebung
+77
View File
@@ -0,0 +1,77 @@
# Release
Wie ein Release gebaut, signiert und veröffentlicht wird ([ADR 0003](adr/0003-git-host-gitea-actions.md), [ADR 0024](adr/0024-auslieferung-installer-m6.md)).
## Ablauf
1. Meilenstein fertig, CI auf `main` grün.
2. Tag setzen und pushen: `git tag -a v0.7.0 -m "M6 Auslieferung" && git push origin v0.7.0`.
3. Der Workflow `.gitea/workflows/release.yml` baut mit `make release PACKAGES=1 VERSION=<tag>`:
- Tarballs `<binary>_<tag>_linux_<arch>.tar.gz` für `amd64` und `arm64`,
- Pakete `<binary>_<version>-1_<arch>.deb` und `<binary>-<version>-1.<arch>.rpm` (nfpm, siehe unten),
- `SHA256SUMS` über Tarballs und Pakete,
- `SHA256SUMS.sig`, falls das Secret `RELEASE_SIGNING_KEY` gesetzt ist.
4. `scripts/release-upload.sh` legt das Gitea-Release an (Tags mit `-` werden als Vorabversion markiert) und lädt alle Dateien aus `dist/` hoch; vorhandene Dateien gleichen Namens werden ersetzt, der Workflow kann also erneut laufen.
Lokal: `make release` (nur Tarballs), `make release PACKAGES=1` bzw. nach `make release` ein `make packages` (braucht `nfpm`), `make sign-release RELEASE_SIGNING_KEY_FILE=<pfad>`.
## Pakete (`.deb`, `.rpm`)
`scripts/build-packages.sh` nimmt die Binaries aus den Tarballs in `dist/` (dieselben Dateien, keine zweite Übersetzung) und baut je Binary und Architektur ein `.deb` und ein `.rpm` mit [nfpm](https://nfpm.goreleaser.com). nfpm wird in der CI von `scripts/ci/install-tools.sh` in einer festen Version mit Prüfsumme installiert (`NFPM_VERSION`); lokal genügt ein `nfpm` im `PATH` oder `make packages NFPM=/pfad/zu/nfpm`.
| Paket | Inhalt |
| --- | --- |
| Name = Binary-Name (`BIN_SERVER`, `BIN_AGENT_DNS`, `BIN_AGENT_DHCP` aus dem Makefile) | `/usr/bin/<binary>` (0755) |
| | `/usr/lib/systemd/system/<binary>.service` (aus `deploy/systemd/*.service.in`, gerendert mit `/usr/bin`) |
| | `/usr/share/doc/<binary>/LICENSE` (Server proprietär, Agents Apache-2.0) |
- Paketversion: Tag ohne `v` (`v0.7.0` → `0.7.0`, Release `1`); Builds ohne Versions-Tag erhalten `0.0.0-dev.<beschreibung>`.
- Die Pakete enthalten **keine** Konfiguration, legen keine Benutzer an und starten nichts. Die Paket-Skripte rufen nur `systemctl daemon-reload` auf. Eingerichtet wird mit `scripts/install.sh`.
- Konfiguration: Vorlage `deploy/nfpm/package.yaml.in` mit Platzhaltern (ADR 0018). `PKG_MAINTAINER`, `PKG_VENDOR` und `PKG_HOMEPAGE` lassen sich per Umgebung überschreiben.
## Signatur
`SHA256SUMS.sig` ist eine Ed25519-Signatur (64 Byte, binär) über `SHA256SUMS`, erzeugt mit `openssl pkeyutl -sign -rawin`. Weil `SHA256SUMS` alle Artefakte abdeckt, genügt eine Signatur. Der Installer prüft sie mit dem öffentlichen Schlüssel aus seinem Variablenblock.
### Schlüsselpaar erzeugen (einmalig, offline)
```sh
umask 077
openssl genpkey -algorithm ed25519 -out release-signing.pem # privat
openssl pkey -in release-signing.pem -pubout -out release-signing.pub.pem
```
Der private Schlüssel gehört in einen Passwortmanager bzw. auf einen Offline-Datenträger, nicht ins Repository. Wer ihn hat, kann Releases signieren, die jeder Installer annimmt.
### Gitea-Secret setzen
Repository → Einstellungen → Actions → Secrets → **Secret hinzufügen**: Name `RELEASE_SIGNING_KEY`, Wert = kompletter Inhalt von `release-signing.pem` (einschließlich der Zeilen `-----BEGIN PRIVATE KEY-----`/`-----END PRIVATE KEY-----`).
Der Workflow schreibt den Schlüssel nur für die Dauer des Signierens in eine Datei mit Modus 0600 (`umask 077`) und löscht sie danach; er wird nie ausgegeben. Ohne Secret entsteht kein `SHA256SUMS.sig`, der Workflow warnt.
### Öffentlichen Schlüssel in den Installer eintragen
In `scripts/install.sh` im Variablenblock (zwischen `# --- names (ADR 0018) ---` und `# --- end names ---`) den Inhalt von `release-signing.pub.pem` als `RELEASE_PUBLIC_KEY_PEM` eintragen:
```sh
RELEASE_PUBLIC_KEY_PEM="-----BEGIN PUBLIC KEY-----
MCowBQYDK2VwAyEA…
-----END PUBLIC KEY-----"
```
Ist der Schlüssel gesetzt, bricht der Installer bei fehlender oder ungültiger Signatur ab; ohne Schlüssel prüft er nur die Prüfsummen und weist darauf hin. Der Release-Workflow prüft die frische Signatur mit dem Schlüssel aus `scripts/install.sh`, damit ein falsch eingetragener Schlüssel schon beim Release auffällt.
### Prüfen von Hand
```sh
scripts/sign-release.sh verify SHA256SUMS SHA256SUMS.sig release-signing.pub.pem
# entspricht
openssl pkeyutl -verify -pubin -inkey release-signing.pub.pem -rawin -in SHA256SUMS -sigfile SHA256SUMS.sig
sha256sum -c SHA256SUMS
```
Die CI (Job „Release build (dry run)“) erzeugt bei jedem Lauf einen Wegwerf-Schlüssel und prüft Signieren, Verifizieren sowie die Ablehnung einer veränderten Datei und eines fremden Schlüssels (`scripts/sign-release.sh selftest`).
### Schlüsselwechsel
Neues Schlüsselpaar erzeugen, Secret ersetzen, öffentlichen Schlüssel in `scripts/install.sh` tauschen, neues Release taggen. Bereits verteilte Installer-Skripte kennen nur den alten Schlüssel und lehnen Releases mit dem neuen ab; sie müssen durch das Skript des neuen Releases ersetzt werden.
@@ -56,3 +56,24 @@ Brief §10 verlangt `scripts/install.sh` für die Rollen `server`, `agent-dns` u
- Gemischte Distributionen in einem Cluster funktionieren, weil `zone_dir` und die Kea-Pfade überall gleich sind.
- Die Kea-Installation hängt standardmäßig vom ISC-Repository ab; ohne Internetzugang bleibt `--kea-source distro` oder ein eigener Spiegel.
- Agents haben die Rechte ihres Dienstes; ein kompromittierter Agent kann diesen Dienst umkonfigurieren, aber nicht mehr.
## Präzisierungen (Umsetzung)
### `license set-key`
- Liest den Schlüssel von stdin (höchstens 4096 Byte Eingabe, nach dem Trimmen nicht leer, eine Zeile, höchstens 512 Zeichen) und ruft denselben Service wie `PUT /api/v1/license` auf; der Audit-Eintrag `license.set_key` trägt den Akteur `system` (wie `bootstrap-admin`). Ausgegeben werden Status, Grund, Lizenzname, Ablauf, Grenzen und Features, nie der Schlüssel.
- Exit-Codes: `0` aktiviert, `1` sonstiger Fehler (Konfiguration, Datenbank), `2` Eingabe leer/zu groß/mehrzeilig, `3` Schlüssel abgelehnt (`license_invalid`, `license_in_use`), `4` Lizenzserver nicht erreichbar oder Antwort nicht vertrauenswürdig. Wie bei `PUT /license` wird bei `3` und `4` nichts gespeichert; der Installer behandelt `4` als Warnung.
### Aufbewahrung
- Die Löschung braucht keine Migration: Der Unveränderlichkeits-Trigger der Revisionen greift nur bei `UPDATE`; Dateien werden per `ON DELETE CASCADE` mitgelöscht.
- Der Aufräumlauf läuft fünf Minuten nach dem Start und dann alle 24 Stunden, in einer Transaktion mit der Revisionssperre (wie Erzeugen und Ausrollen). Er läuft auch im Lizenz-Lesemodus.
- Geschützt sind die neuesten `revisions_keep` Revisionen (mindestens die neueste) sowie jede Revision, die in `agents.applied_revision`, `last_good_revision` oder `target_revision` steht oder ein Deployment mit Ergebnis `pending` hat.
- **Deployments bleiben erhalten**, auch wenn ihre Revision gelöscht ist: Sie speichern nur die Nummer und sind Teil der Historie. Der Diff einer Revision gegen eine gelöschte Vorgängerin liefert `not_found`; ein Rollback auf eine gelöschte Revision ebenso.
- Audit-Einträge werden nach `at` gelöscht. Hat der Lauf etwas gelöscht, schreibt er `retention.cleanup` (Objekttyp `system`) mit den Anzahlen und dem Bereich der gelöschten Revisionsnummern. Die Anzeige „offene Änderungen“ bleibt korrekt, weil sie ohne den Audit-Eintrag der Revision auf deren Zeitpunkt zurückfällt.
### Pakete, Signatur, Namensprüfung
- **Pakete:** `scripts/build-packages.sh` (`make packages`, `make release PACKAGES=1`) verpackt die Binaries aus den Tarballs (keine zweite Übersetzung) mit nfpm 2.48.0, installiert mit Prüfsumme über `scripts/ci/install-tools.sh` (kein `go run`: nfpm gehört nicht in den Modulgraphen). Paketname = Binary-Name; Inhalt `/usr/bin/<binary>`, `/usr/lib/systemd/system/<binary>.service` (gerendert mit `/usr/bin`), `/usr/share/doc/<binary>/LICENSE`; Paket-Skripte nur `systemctl daemon-reload`. Keine Abhängigkeiten (PostgreSQL, BIND und Kea richtet der Installer ein, Kea je nach `--kea-source` mit anderen Paketnamen). Danach umfasst `SHA256SUMS` Tarballs und Pakete. Details: [RELEASE.md](../RELEASE.md).
- **Signatur:** `scripts/sign-release.sh` (`sign`, `verify`, `public-key`, `selftest`); die Signatur ist binär (64 Byte). Der Release-Workflow prüft die neue Signatur mit dem öffentlichen Schlüssel aus `scripts/install.sh`, falls dort einer steht. Die CI prüft Signieren und Verifizieren bei jedem Lauf mit einem Wegwerf-Schlüssel.
- **Namensprüfung:** `scripts/check-brand.sh` (`make lint-brand`, Job „Shell“) sucht `ddi-suite`, `ddi-server`, `ddi-agent-dns`, `ddi-agent-dhcp` (ohne Groß-/Kleinschreibung) in `deploy/systemd/*.in`, `deploy/nfpm/*` und `scripts/install.sh`. Erlaubt sind die Namen nur im Block zwischen den Zeilen `# --- names (ADR 0018) ---` und `# --- end names ---`, die im Installer genau einmal vorkommen müssen. Die nfpm-Konfiguration ist deshalb eine Vorlage (`deploy/nfpm/package.yaml.in`) mit Platzhaltern wie die Units.
+158
View File
@@ -0,0 +1,158 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: Apache-2.0
#
# Builds .deb and .rpm packages with nfpm from the release tarballs in DIST
# (made by scripts/build-release.sh) and rewrites DIST/SHA256SUMS to cover
# tarballs and packages (ADR 0024).
#
# Usage: scripts/build-packages.sh [-d DIST] [-p "linux/amd64 linux/arm64"]
#
# Each package holds the binary from its tarball as /usr/bin/<binary>, the
# systemd unit rendered for /usr/bin and the license text. Names and version
# come from the environment (Makefile, ADR 0018): VERSION, PRODUCT,
# DISPLAY_NAME, BIN_SERVER, BIN_AGENT_DNS, BIN_AGENT_DHCP. NFPM names the
# nfpm binary (default: nfpm from PATH, installed by scripts/ci/install-tools.sh).
# Optional: PKG_MAINTAINER, PKG_VENDOR, PKG_HOMEPAGE.
set -euo pipefail
PRODUCT="${PRODUCT:-ddi-suite}"
DISPLAY_NAME="${DISPLAY_NAME:-DDI-Suite}"
BIN_SERVER="${BIN_SERVER:-ddi-server}"
BIN_AGENT_DNS="${BIN_AGENT_DNS:-ddi-agent-dns}"
BIN_AGENT_DHCP="${BIN_AGENT_DHCP:-ddi-agent-dhcp}"
NFPM="${NFPM:-nfpm}"
dist="dist"
platforms="linux/amd64 linux/arm64"
usage() {
sed -n '4,16p' "$0" | sed 's/^# \{0,1\}//'
}
while getopts ":d:p:h" opt; do
case "$opt" in
d) dist="$OPTARG" ;;
p) platforms="$OPTARG" ;;
h)
usage
exit 0
;;
*)
usage >&2
exit 2
;;
esac
done
root="$(git rev-parse --show-toplevel)"
cd "$root"
if ! command -v "$NFPM" >/dev/null; then
echo "nfpm not found (set NFPM or run scripts/ci/install-tools.sh)" >&2
exit 1
fi
VERSION="${VERSION:-$(git describe --tags --always --dirty 2>/dev/null || echo dev)}"
BUILD_DATE="${BUILD_DATE:-$(git log -1 --format=%cI 2>/dev/null || date -u +%Y-%m-%dT%H:%M:%SZ)}"
# Package versions must be semantic versions: v1.2.3 -> 1.2.3; anything else
# (a commit hash, "dev") becomes a 0.0.0 pre-release that sorts below every
# real release.
if [[ "$VERSION" =~ ^v?([0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.+-]*)?)$ ]]; then
pkg_version="${BASH_REMATCH[1]}"
else
pkg_version="0.0.0-dev.$(tr -c '0-9A-Za-z.\n' '.' <<<"$VERSION")"
fi
# Reproducible timestamps inside the packages.
SOURCE_DATE_EPOCH="$(date -u -d "$BUILD_DATE" +%s 2>/dev/null || echo 0)"
export SOURCE_DATE_EPOCH
declare -A descriptions=(
["$BIN_SERVER"]="${DISPLAY_NAME} management server (IPAM, DNS and DHCP)"
["$BIN_AGENT_DNS"]="${DISPLAY_NAME} agent for BIND 9"
["$BIN_AGENT_DHCP"]="${DISPLAY_NAME} agent for Kea DHCPv4"
)
declare -A licenses=(
["$BIN_SERVER"]="LICENSES/LicenseRef-DDI-Proprietary.txt"
["$BIN_AGENT_DNS"]="LICENSES/Apache-2.0.txt"
["$BIN_AGENT_DHCP"]="LICENSES/Apache-2.0.txt"
)
declare -A license_names=(
["$BIN_SERVER"]="Proprietary"
["$BIN_AGENT_DNS"]="Apache-2.0"
["$BIN_AGENT_DHCP"]="Apache-2.0"
)
workdir="$(mktemp -d)"
trap 'rm -rf "$workdir"' EXIT
# Git Bash on Windows (development): nfpm.exe needs Windows paths.
if command -v cygpath >/dev/null; then
workdir="$(cygpath -m "$workdir")"
fi
export PRODUCT DISPLAY_NAME BIN_SERVER BIN_AGENT_DNS BIN_AGENT_DHCP
scripts/render-units.sh /usr/bin "$workdir/units"
PKG_MAINTAINER="${PKG_MAINTAINER:-${DISPLAY_NAME} <maintainer@example.invalid>}"
PKG_VENDOR="${PKG_VENDOR:-$DISPLAY_NAME}"
PKG_HOMEPAGE="${PKG_HOMEPAGE:-https://git.beugsiud.de/beug.k/DDI}"
# render_config NAME ARCH SRC_DIR LICENSE DESCRIPTION > nfpm.yaml
render_config() {
local -a subst=(
PKG_NAME "$1" PKG_ARCH "$2" PKG_SRC_DIR "$3" PKG_LICENSE "$4" PKG_DESCRIPTION "$5"
PKG_VERSION "$pkg_version" PKG_MAINTAINER "$PKG_MAINTAINER" PKG_VENDOR "$PKG_VENDOR"
PKG_HOMEPAGE "$PKG_HOMEPAGE" PKG_SCRIPTS_DIR "$root/deploy/nfpm"
)
local -a expr=()
local i value
for ((i = 0; i < ${#subst[@]}; i += 2)); do
value="${subst[i + 1]}"
# The values end up in sed replacements and double-quoted YAML.
if [[ "$value" == *[\|\&\\\"]* ]]; then
echo "unsafe value for ${subst[i]}: ${value}" >&2
exit 2
fi
expr+=(-e "s|@${subst[i]}@|${value}|g")
done
sed "${expr[@]}" deploy/nfpm/package.yaml.in
}
for platform in $platforms; do
goos="${platform%/*}"
goarch="${platform#*/}"
for bin in "$BIN_SERVER" "$BIN_AGENT_DNS" "$BIN_AGENT_DHCP"; do
name="${bin}_${VERSION}_${goos}_${goarch}"
tarball="${dist}/${name}.tar.gz"
if [[ ! -f "$tarball" ]]; then
echo "missing ${tarball}: run make release first" >&2
exit 1
fi
src="${workdir}/${name}"
mkdir -p "$src"
tar -xzf "$tarball" -C "$workdir" "${name}/${bin}"
mv "${workdir}/${name}/${bin}" "${src}/binary"
cp "$workdir/units/${bin}.service" "${src}/unit.service"
cp "${licenses[$bin]}" "${src}/LICENSE"
render_config "$bin" "$goarch" "$src" "${license_names[$bin]}" "${descriptions[$bin]}" >"${src}/nfpm.yaml"
if grep -q '@[A-Z_]*@' "${src}/nfpm.yaml"; then
echo "unreplaced placeholder in the nfpm configuration of ${name}" >&2
exit 1
fi
for packager in deb rpm; do
echo "packaging ${bin} ${pkg_version} ${goarch} (${packager})"
"$NFPM" package --config "${src}/nfpm.yaml" --packager "$packager" --target "$dist/" >/dev/null
done
done
done
(
cd "$dist"
shopt -s nullglob
sha256sum -- *.tar.gz *.deb *.rpm >SHA256SUMS
)
echo "artefacts in ${dist}:"
cat "${dist}/SHA256SUMS"
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: Apache-2.0
#
# Fails if systemd unit templates, nfpm package files or the installer
# contain hard-coded product or binary names (ADR 0018). Names belong in the
# variables of the Makefile and the scripts; templates use placeholders.
#
# Usage: scripts/check-brand.sh [FILE...]
# Without FILE the default set is checked: deploy/systemd/*.in,
# deploy/nfpm/*, scripts/install.sh. Missing files are skipped.
#
# The installer may define the names once, in a block between the lines
# # --- names (ADR 0018) ---
# # --- end names ---
# Each marker must appear exactly once, in this order; everything outside the
# block is checked. The same applies to any checked file containing a marker.
set -euo pipefail
readonly BEGIN_MARKER="# --- names (ADR 0018) ---"
readonly END_MARKER="# --- end names ---"
# Literal names that must not appear (case-insensitive).
readonly PATTERN='ddi-suite|ddi-server|ddi-agent-dns|ddi-agent-dhcp'
cd "$(git rev-parse --show-toplevel)"
files=("$@")
if ((${#files[@]} == 0)); then
shopt -s nullglob
files=(deploy/systemd/*.in deploy/nfpm/* scripts/install.sh)
shopt -u nullglob
fi
failed=0
checked=0
for f in "${files[@]}"; do
[[ -f "$f" ]] || continue
checked=$((checked + 1))
begins="$(grep -cxF -- "$BEGIN_MARKER" "$f" || true)"
ends="$(grep -cxF -- "$END_MARKER" "$f" || true)"
if [[ "$f" == scripts/install.sh ]] || ((begins + ends > 0)); then
if ((begins != 1 || ends != 1)); then
echo "$f: expected exactly one '${BEGIN_MARKER}' and one '${END_MARKER}' line" >&2
failed=1
continue
fi
fi
# "line:text" for every offending line outside the names block.
hits="$(awk -v b="$BEGIN_MARKER" -v e="$END_MARKER" -v pat="$PATTERN" '
$0 == b { inside = 1; next }
$0 == e { if (!inside) print NR ":end marker before begin marker"; inside = 0; next }
!inside && tolower($0) ~ pat { print NR ":" $0 }
' "$f")"
if [[ -n "$hits" ]]; then
while IFS= read -r line; do
echo "${f}:${line}" >&2
done <<<"$hits"
failed=1
fi
done
if ((failed)); then
echo "hard-coded product or binary names found (ADR 0018): use the variables or placeholders" >&2
exit 1
fi
echo "brand check passed (${checked} files)"
+51 -21
View File
@@ -2,9 +2,10 @@
# SPDX-License-Identifier: Apache-2.0
#
# Installs pinned, checksum-verified development tools on Linux (amd64/arm64)
# for CI: golangci-lint, buf, shellcheck.
# for CI: golangci-lint, buf, shellcheck, nfpm.
#
# Usage: scripts/ci/install-tools.sh [BINDIR] (default: $HOME/.local/bin)
# Usage: scripts/ci/install-tools.sh [BINDIR] [TOOL...]
# BINDIR defaults to $HOME/.local/bin; without TOOL all tools are installed.
# In GitHub/Gitea Actions the directory is appended to $GITHUB_PATH.
set -euo pipefail
@@ -12,25 +13,35 @@ set -euo pipefail
readonly GOLANGCI_LINT_VERSION="2.14.0"
readonly BUF_VERSION="1.73.0"
readonly SHELLCHECK_VERSION="0.11.0"
readonly NFPM_VERSION="2.48.0"
bindir="${1:-$HOME/.local/bin}"
shift || true
tools=("$@")
if ((${#tools[@]} == 0)); then
tools=(golangci-lint buf shellcheck nfpm)
fi
case "$(uname -m)" in
x86_64 | amd64)
golangci_arch="amd64"
buf_arch="x86_64"
shellcheck_arch="x86_64"
nfpm_arch="x86_64"
golangci_sha="ab90aeb7b066f92a33415b638a50fe5344bbb75a0d32ad30cc248d88f81032ab"
buf_sha="8f2986298ad08f0cc1bf999b9797b7c383adf32d7edf0f73d6f1e1a701baeac1"
shellcheck_sha="b7af85e41cc99489dcc21d66c6d5f3685138f06d34651e6d34b42ec6d54fe6f6"
nfpm_sha="6d7f9a28b2f397934403950fcbba83f598b08d02edbe79a0815b3b4897341750"
;;
aarch64 | arm64)
golangci_arch="arm64"
buf_arch="aarch64"
shellcheck_arch="aarch64"
nfpm_arch="arm64"
golangci_sha="ee7ec5f3453d15ddf106fae5a4d6c71737712348a979d1fe9cd52ec7ea299bae"
buf_sha="902b75267db7f4391e99b7fa0756050e5354234cc0437ef50eee9c788950c7a3"
shellcheck_sha="68a8133197a50beb8803f8d42f9908d1af1c5540d4bb05fdfca8c1fa47decefc"
nfpm_sha="a9a75213763783b7669719e505474c748c5ad7dc66591a195619deb50f85c2f4"
;;
*)
echo "unsupported architecture: $(uname -m)" >&2
@@ -48,26 +59,45 @@ fetch() {
echo "$3 $2" | sha256sum -c --quiet -
}
name="golangci-lint-${GOLANGCI_LINT_VERSION}-linux-${golangci_arch}"
fetch "https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCI_LINT_VERSION}/${name}.tar.gz" \
"$workdir/golangci.tar.gz" "$golangci_sha"
tar -xzf "$workdir/golangci.tar.gz" -C "$workdir"
install -m 0755 "$workdir/${name}/golangci-lint" "$bindir/golangci-lint"
fetch "https://github.com/bufbuild/buf/releases/download/v${BUF_VERSION}/buf-Linux-${buf_arch}" \
"$workdir/buf" "$buf_sha"
install -m 0755 "$workdir/buf" "$bindir/buf"
name="shellcheck-v${SHELLCHECK_VERSION}"
fetch "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/${name}.linux.${shellcheck_arch}.tar.gz" \
"$workdir/shellcheck.tar.gz" "$shellcheck_sha"
tar -xzf "$workdir/shellcheck.tar.gz" -C "$workdir"
install -m 0755 "$workdir/${name}/shellcheck" "$bindir/shellcheck"
for tool in "${tools[@]}"; do
case "$tool" in
golangci-lint)
name="golangci-lint-${GOLANGCI_LINT_VERSION}-linux-${golangci_arch}"
fetch "https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCI_LINT_VERSION}/${name}.tar.gz" \
"$workdir/golangci.tar.gz" "$golangci_sha"
tar -xzf "$workdir/golangci.tar.gz" -C "$workdir"
install -m 0755 "$workdir/${name}/golangci-lint" "$bindir/golangci-lint"
"$bindir/golangci-lint" version
;;
buf)
fetch "https://github.com/bufbuild/buf/releases/download/v${BUF_VERSION}/buf-Linux-${buf_arch}" \
"$workdir/buf" "$buf_sha"
install -m 0755 "$workdir/buf" "$bindir/buf"
"$bindir/buf" --version
;;
shellcheck)
name="shellcheck-v${SHELLCHECK_VERSION}"
fetch "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/${name}.linux.${shellcheck_arch}.tar.gz" \
"$workdir/shellcheck.tar.gz" "$shellcheck_sha"
tar -xzf "$workdir/shellcheck.tar.gz" -C "$workdir"
install -m 0755 "$workdir/${name}/shellcheck" "$bindir/shellcheck"
"$bindir/shellcheck" --version | head -n 2
;;
nfpm)
fetch "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_${nfpm_arch}.tar.gz" \
"$workdir/nfpm.tar.gz" "$nfpm_sha"
tar -xzf "$workdir/nfpm.tar.gz" -C "$workdir" nfpm
install -m 0755 "$workdir/nfpm" "$bindir/nfpm"
"$bindir/nfpm" --version >"$workdir/nfpm.version"
grep GitVersion "$workdir/nfpm.version"
;;
*)
echo "unknown tool: $tool" >&2
exit 2
;;
esac
done
if [[ -n "${GITHUB_PATH:-}" ]]; then
echo "$bindir" >>"$GITHUB_PATH"
fi
"$bindir/golangci-lint" version
"$bindir/buf" --version
"$bindir/shellcheck" --version | head -n 2
+131
View File
@@ -0,0 +1,131 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: Apache-2.0
#
# Signs and verifies release checksums with Ed25519 (ADR 0024, docs/RELEASE.md).
# The signature is the raw 64-byte output of
# `openssl pkeyutl -sign -rawin` over SHA256SUMS, stored as SHA256SUMS.sig.
#
# Usage:
# scripts/sign-release.sh sign SUMS [KEY_FILE]
# writes SUMS.sig; without KEY_FILE the PEM private key is taken from
# the environment variable RELEASE_SIGNING_KEY (it is written to a
# private temporary file and removed again, never printed)
# scripts/sign-release.sh verify SUMS SIG PUBLIC_KEY_FILE
# exits 0 only if SIG is a valid signature of SUMS
# scripts/sign-release.sh public-key KEY_FILE
# prints the PEM public key that belongs to a private key
# scripts/sign-release.sh selftest
# signs and verifies with a throwaway key (CI); a tampered file and a
# foreign key must fail
#
# Requires OpenSSL 3.
set -euo pipefail
usage() {
sed -n '4,21p' "$0" | sed 's/^# \{0,1\}//'
}
die() {
echo "$*" >&2
exit 1
}
command -v openssl >/dev/null || die "openssl is required"
# sign SUMS KEY_FILE
sign_with_file() {
local sums="$1" key="$2"
[[ -f "$sums" ]] || die "no such file: $sums"
[[ -f "$key" ]] || die "no such key file: $key"
openssl pkeyutl -sign -rawin -inkey "$key" -in "$sums" -out "${sums}.sig"
[[ "$(wc -c <"${sums}.sig")" -eq 64 ]] || die "unexpected signature size (is the key an Ed25519 key?)"
echo "signed ${sums} -> ${sums}.sig"
}
cmd_sign() {
(($# >= 1 && $# <= 2)) || {
usage >&2
exit 2
}
if (($# == 2)); then
sign_with_file "$1" "$2"
return
fi
[[ -n "${RELEASE_SIGNING_KEY:-}" ]] || die "RELEASE_SIGNING_KEY is not set and no key file was given"
local dir
dir="$(umask 077 && mktemp -d)"
# shellcheck disable=SC2064 # expand now: $dir is local
trap "rm -rf '$dir'" EXIT
(umask 077 && printf '%s\n' "$RELEASE_SIGNING_KEY" >"$dir/key.pem")
sign_with_file "$1" "$dir/key.pem"
rm -rf "$dir"
trap - EXIT
}
cmd_verify() {
(($# == 3)) || {
usage >&2
exit 2
}
local sums="$1" sig="$2" pub="$3"
for f in "$sums" "$sig" "$pub"; do
[[ -f "$f" ]] || die "no such file: $f"
done
if openssl pkeyutl -verify -pubin -inkey "$pub" -rawin -in "$sums" -sigfile "$sig" >/dev/null 2>&1; then
echo "signature of ${sums} is valid"
else
die "signature of ${sums} is NOT valid"
fi
}
cmd_public_key() {
(($# == 1)) || {
usage >&2
exit 2
}
openssl pkey -in "$1" -pubout
}
cmd_selftest() {
local dir
dir="$(umask 077 && mktemp -d)"
# shellcheck disable=SC2064 # expand now: $dir is local
trap "rm -rf '$dir'" EXIT
openssl genpkey -algorithm ed25519 -out "$dir/key.pem" 2>/dev/null
openssl pkey -in "$dir/key.pem" -pubout -out "$dir/pub.pem"
openssl genpkey -algorithm ed25519 -out "$dir/other.pem" 2>/dev/null
openssl pkey -in "$dir/other.pem" -pubout -out "$dir/other-pub.pem"
printf '%s example_v0.0.0_linux_amd64.tar.gz\n' "$(printf 'x' | sha256sum | cut -d' ' -f1)" >"$dir/SHA256SUMS"
# Signing from the environment, as the release workflow does.
RELEASE_SIGNING_KEY="$(cat "$dir/key.pem")" bash "$0" sign "$dir/SHA256SUMS" >/dev/null
cmd_verify "$dir/SHA256SUMS" "$dir/SHA256SUMS.sig" "$dir/pub.pem" >/dev/null ||
die "selftest: valid signature rejected"
if bash "$0" verify "$dir/SHA256SUMS" "$dir/SHA256SUMS.sig" "$dir/other-pub.pem" >/dev/null 2>&1; then
die "selftest: signature accepted with a foreign key"
fi
printf 'tampered\n' >>"$dir/SHA256SUMS"
if bash "$0" verify "$dir/SHA256SUMS" "$dir/SHA256SUMS.sig" "$dir/pub.pem" >/dev/null 2>&1; then
die "selftest: tampered file accepted"
fi
echo "selftest passed: sign, verify, foreign key and tampered file rejected"
}
(($# >= 1)) || {
usage >&2
exit 2
}
cmd="$1"
shift
case "$cmd" in
sign) cmd_sign "$@" ;;
verify) cmd_verify "$@" ;;
public-key) cmd_public_key "$@" ;;
selftest) cmd_selftest "$@" ;;
-h | --help | help) usage ;;
*)
usage >&2
exit 2
;;
esac
+171
View File
@@ -0,0 +1,171 @@
// SPDX-License-Identifier: LicenseRef-DDI-Proprietary
package main
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"strconv"
"strings"
"time"
"git.beugsiud.de/beug.k/DDI/internal/brand"
"git.beugsiud.de/beug.k/DDI/server/internal/app"
"git.beugsiud.de/beug.k/DDI/server/internal/config"
"git.beugsiud.de/beug.k/DDI/server/internal/service"
)
// Exit codes of `license set-key` besides 0 (stored) and 1 (other errors,
// e.g. configuration or database). The installer uses them to decide
// whether to warn or to fail.
const (
exitUsage = 2 // bad arguments, empty or oversized input
exitKeyRejected = 3 // the license service rejected the key
exitLicenseSvc = 4 // the license service is unreachable or untrusted
)
// maxKeyInput bounds what is read from stdin; maxKeyLength matches the
// limit of the license manager.
const (
maxKeyInput = 4096
maxKeyLength = 512
)
// exitError carries a specific exit code.
type exitError struct {
code int
err error
}
func (e *exitError) Error() string { return e.err.Error() }
func (e *exitError) Unwrap() error { return e.err }
func licenseUsage(w io.Writer) {
_, _ = fmt.Fprintf(w, `Usage: %[1]s license set-key < keyfile
set-key read a license key from stdin, validate and activate it online
(like PUT /api/v1/license) and print the resulting license state
Exit codes: 0 activated, 1 other error, 2 empty or oversized input,
3 key rejected, 4 license server unreachable (nothing was changed).
`, brand.ServerBinary)
}
func cmdLicense(ctx context.Context, args []string, lookup config.LookupFunc, stdin io.Reader, stdout, logOut io.Writer) error {
if len(args) == 0 || args[0] == "help" || args[0] == "-h" || args[0] == "--help" {
if len(args) == 0 {
licenseUsage(logOut)
return &exitError{code: exitUsage, err: errors.New("license: missing subcommand")}
}
licenseUsage(stdout)
return nil
}
switch args[0] {
case "set-key":
if len(args) > 1 {
return &exitError{code: exitUsage, err: errors.New("license set-key takes no arguments; pass the key on stdin")}
}
return cmdLicenseSetKey(ctx, lookup, stdin, stdout, logOut)
default:
licenseUsage(logOut)
return &exitError{code: exitUsage, err: fmt.Errorf("license: unknown subcommand %q", args[0])}
}
}
// readKey reads the license key from r: trimmed, not empty, bounded.
func readKey(r io.Reader) (string, error) {
data, err := io.ReadAll(io.LimitReader(r, maxKeyInput+1))
if err != nil {
return "", fmt.Errorf("read license key from stdin: %w", err)
}
if len(data) > maxKeyInput {
return "", &exitError{code: exitUsage, err: fmt.Errorf("input on stdin is larger than %d bytes; expected a single license key", maxKeyInput)}
}
key := string(bytes.TrimSpace(data))
switch {
case key == "":
return "", &exitError{code: exitUsage, err: errors.New("no license key on stdin")}
case len(key) > maxKeyLength:
return "", &exitError{code: exitUsage, err: fmt.Errorf("license key is longer than %d characters", maxKeyLength)}
case strings.ContainsAny(key, "\r\n"):
return "", &exitError{code: exitUsage, err: errors.New("expected a single license key, got several lines")}
}
return key, nil
}
func cmdLicenseSetKey(ctx context.Context, lookup config.LookupFunc, stdin io.Reader, stdout, logOut io.Writer) error {
if f, ok := stdin.(*os.File); ok {
if fi, err := f.Stat(); err == nil && fi.Mode()&os.ModeCharDevice != 0 {
_, _ = fmt.Fprintln(logOut, "reading the license key from stdin (finish with Ctrl-D)")
}
}
key, err := readKey(stdin)
if err != nil {
return err
}
cfg, log, err := setup(lookup, logOut)
if err != nil {
return err
}
core, err := app.Open(ctx, cfg, log, true)
if err != nil {
return err
}
defer core.Close()
st, err := core.Service.SetLicenseKey(ctx, service.SystemActor, key)
if err != nil {
return licenseCLIError(err)
}
return printLicense(stdout, st)
}
// licenseCLIError maps service errors to messages and exit codes.
func licenseCLIError(err error) error {
var se *service.Error
if !errors.As(err, &se) {
return err
}
msg := se.Message
if pc := se.Params["provider_code"]; pc != "" {
msg += " (" + pc + ")"
}
switch se.Kind {
case service.KindUnavailable:
return &exitError{code: exitLicenseSvc, err: fmt.Errorf("license key not stored: %s [%s]", msg, se.Code)}
case service.KindInvalid, service.KindConflict:
return &exitError{code: exitKeyRejected, err: fmt.Errorf("license key rejected: %s [%s]", msg, se.Code)}
}
return err
}
func printLicense(w io.Writer, st service.LicenseStatus) error {
limit := func(v *int) string {
if v == nil {
return "unlimited"
}
return strconv.Itoa(*v)
}
day := func(t *time.Time) string {
if t == nil {
return "never"
}
return t.UTC().Format(time.RFC3339)
}
features := strings.Join(st.Features, ", ")
if features == "" {
features = "-"
}
name := st.Name
if st.LicenseID != "" {
name = strings.TrimSpace(name + " (" + st.LicenseID + ")")
}
_, err := fmt.Fprintf(w, "status: %s\nreason: %s\nlicense: %s\nexpires: %s\n"+
"max agents: %s\nmax addresses: %s (in use: %d)\nfeatures: %s\n",
st.State, st.Reason, name, day(st.ExpiresAt), limit(st.Limits.MaxAgents), limit(st.Limits.MaxAddresses),
st.Usage.Addresses, features)
return err
}
+190
View File
@@ -0,0 +1,190 @@
// SPDX-License-Identifier: LicenseRef-DDI-Proprietary
package main
import (
"bytes"
"context"
"crypto/ed25519"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5"
)
const testKeygenAccount = "9de54794-8e94-4250-83f6-87e3d5686a0b"
// fakeKeygen answers the requests of license activation with signed
// responses: "GOOD-KEY" is valid (no machine activation needed), every
// other key does not exist.
func fakeKeygen(t *testing.T) (url string, pub ed25519.PublicKey) {
t.Helper()
pub, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
path := strings.TrimPrefix(r.URL.Path, "/v1/accounts/"+testKeygenAccount)
body, _ := io.ReadAll(r.Body)
var resp any
switch {
case r.Method == http.MethodPost && path == "/licenses/actions/validate-key":
var req struct {
Meta struct {
Key string `json:"key"`
Nonce int64 `json:"nonce"`
} `json:"meta"`
}
_ = json.Unmarshal(body, &req)
meta := map[string]any{"valid": false, "code": "NOT_FOUND", "detail": "does not exist", "nonce": req.Meta.Nonce}
var data any
if req.Meta.Key == "GOOD-KEY" {
meta["valid"], meta["code"], meta["detail"] = true, "VALID", "valid"
data = map[string]any{"id": "lic-1", "type": "licenses", "attributes": map[string]any{
"name": "CLI Test", "expiry": "2030-01-01T00:00:00Z", "status": "ACTIVE",
"metadata": map[string]any{"maxAgents": 5},
}}
}
resp = map[string]any{"meta": meta, "data": data}
case r.Method == http.MethodGet && path == "/licenses/lic-1/entitlements":
resp = map[string]any{"data": []any{map[string]any{"attributes": map[string]any{"code": "HA"}}}}
default:
w.WriteHeader(http.StatusNotFound)
return
}
out, _ := json.Marshal(resp)
sum := sha256.Sum256(out)
digest := "sha-256=" + base64.StdEncoding.EncodeToString(sum[:])
date := time.Now().UTC().Format(http.TimeFormat)
signing := "(request-target): " + strings.ToLower(r.Method) + " " + r.URL.RequestURI() + "\nhost: " + r.Host +
"\ndate: " + date + "\ndigest: " + digest
sig := ed25519.Sign(priv, []byte(signing))
w.Header().Set("Date", date)
w.Header().Set("Digest", digest)
w.Header().Set("Keygen-Signature", `keyid="`+testKeygenAccount+`", algorithm="ed25519", signature="`+
base64.StdEncoding.EncodeToString(sig)+`", headers="(request-target) host date digest"`)
w.Header().Set("Content-Type", "application/vnd.api+json")
_, _ = w.Write(out)
}))
t.Cleanup(srv.Close)
return srv.URL, pub
}
func TestReadKey(t *testing.T) {
if k, err := readKey(strings.NewReader(" ABCD-1234 \n")); err != nil || k != "ABCD-1234" {
t.Errorf("trimmed key: %q %v", k, err)
}
for name, in := range map[string]string{
"empty": "",
"blank": " \n\t\n",
"too long": strings.Repeat("A", maxKeyLength+1),
"oversized": strings.Repeat("A", maxKeyInput+1),
"two lines": "KEY-1\nKEY-2\n",
"blank line": "KEY-1\r\nKEY-2",
} {
_, err := readKey(strings.NewReader(in))
ee, ok := err.(*exitError) //nolint:errorlint // readKey returns it unwrapped
if !ok || ee.code != exitUsage {
t.Errorf("%s: %v", name, err)
}
}
}
func TestLicenseCommandUsage(t *testing.T) {
var out, errOut bytes.Buffer
if code := run(context.Background(), []string{"license"}, lookupFrom(nil), nil, &out, &errOut); code != exitUsage {
t.Errorf("without subcommand: %d", code)
}
if code := run(context.Background(), []string{"license", "frob"}, lookupFrom(nil), nil, &out, &errOut); code != exitUsage {
t.Errorf("unknown subcommand: %d", code)
}
if code := run(context.Background(), []string{"license", "set-key", "KEY"}, lookupFrom(nil), nil, &out, &errOut); code != exitUsage ||
!strings.Contains(errOut.String(), "stdin") {
t.Errorf("key as argument: %d %q", code, errOut.String())
}
out.Reset()
if code := run(context.Background(), []string{"license", "help"}, lookupFrom(nil), nil, &out, &errOut); code != 0 ||
!strings.Contains(out.String(), "set-key") {
t.Errorf("help: %d %q", code, out.String())
}
errOut.Reset()
if code := run(context.Background(), []string{"license", "set-key"}, lookupFrom(nil), strings.NewReader("\n"), &out, &errOut); code != exitUsage ||
!strings.Contains(errOut.String(), "no license key") {
t.Errorf("empty stdin: %d %q", code, errOut.String())
}
}
func TestLicenseSetKey(t *testing.T) {
dbURL := testDatabase(t)
dir := t.TempDir()
keyFile := filepath.Join(dir, "master.key")
if code := run(context.Background(), []string{"gen-master-key", "-out", keyFile}, lookupFrom(nil), nil, io.Discard, io.Discard); code != 0 {
t.Fatal("gen-master-key failed")
}
keygenURL, pub := fakeKeygen(t)
env := map[string]string{
"DDI_DATABASE_URL": dbURL, "DDI_MASTER_KEY_FILE": keyFile, "DDI_STATE_DIR": dir, "DDI_LOG_LEVEL": "error",
"KEYGEN_URL": keygenURL, "KEYGEN_ACCOUNT_ID": testKeygenAccount, "KEYGEN_PUBLIC_KEY": hex.EncodeToString(pub),
}
setKey := func(key string, env map[string]string) (int, string, string) {
var out, errOut bytes.Buffer
code := run(context.Background(), []string{"license", "set-key"}, lookupFrom(env), strings.NewReader(key), &out, &errOut)
return code, out.String(), errOut.String()
}
code, out, errOut := setKey("UNKNOWN-KEY\n", env)
if code != exitKeyRejected || !strings.Contains(errOut, "license_invalid") || !strings.Contains(errOut, "NOT_FOUND") {
t.Errorf("unknown key: %d %q", code, errOut)
}
if strings.Contains(out+errOut, "UNKNOWN-KEY") {
t.Error("the key must not be printed")
}
unreachable := map[string]string{}
for k, v := range env {
unreachable[k] = v
}
unreachable["KEYGEN_URL"] = "http://127.0.0.1:1"
if code, _, errOut := setKey("GOOD-KEY", unreachable); code != exitLicenseSvc || !strings.Contains(errOut, "license_server_unreachable") {
t.Errorf("unreachable license server: %d %q", code, errOut)
}
code, out, errOut = setKey(" GOOD-KEY\n", env)
if code != 0 {
t.Fatalf("set-key: %d %s", code, errOut)
}
for _, want := range []string{"status: valid", "CLI Test (lic-1)", "expires: 2030-01-01T00:00:00Z",
"max agents: 5", "max addresses: unlimited", "features: HA"} {
if !strings.Contains(out, want) {
t.Errorf("output lacks %q:\n%s", want, out)
}
}
if strings.Contains(out+errOut, "GOOD-KEY") {
t.Error("the key must not be printed")
}
conn, err := pgx.Connect(context.Background(), dbURL)
if err != nil {
t.Fatal(err)
}
defer conn.Close(context.Background()) //nolint:errcheck // test
var actor string
var n int
if err := conn.QueryRow(context.Background(),
`SELECT min(actor_name), count(*) FROM audit_log WHERE action = 'license.set_key'`).Scan(&actor, &n); err != nil {
t.Fatal(err)
}
if n != 1 || actor != "system" {
t.Errorf("audit: %d entries by %q", n, actor)
}
}
+10 -4
View File
@@ -26,7 +26,7 @@ import (
func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
code := run(ctx, os.Args[1:], os.LookupEnv, os.Stdout, os.Stderr)
code := run(ctx, os.Args[1:], os.LookupEnv, os.Stdin, os.Stdout, os.Stderr)
stop()
os.Exit(code)
}
@@ -39,15 +39,16 @@ Commands:
migrate apply database migrations and exit
bootstrap-admin create the first administrator and print its one-time password
gen-master-key write a new master key file (refuses to overwrite)
license set-key read a license key from stdin and activate it (%[1]s license help)
version print version information
help show this help
Configuration: YAML file from $DDI_CONFIG_FILE (if set), overridden by
Configuration: YAML file from $%[2]s (if set), overridden by
environment variables; see docs/ARCHITECTURE.md.
`, brand.ServerBinary)
`, brand.ServerBinary, config.EnvName("CONFIG_FILE"))
}
func run(ctx context.Context, args []string, lookup config.LookupFunc, stdout, stderr io.Writer) int {
func run(ctx context.Context, args []string, lookup config.LookupFunc, stdin io.Reader, stdout, stderr io.Writer) int {
cmd := "serve"
if len(args) > 0 {
cmd, args = args[0], args[1:]
@@ -62,6 +63,8 @@ func run(ctx context.Context, args []string, lookup config.LookupFunc, stdout, s
err = cmdBootstrapAdmin(ctx, args, lookup, stdout, stderr)
case "gen-master-key":
err = cmdGenMasterKey(args, lookup, stdout)
case "license":
err = cmdLicense(ctx, args, lookup, stdin, stdout, stderr)
case "version", "--version":
err = buildinfo.Print(stdout, brand.ServerBinary)
case "help", "-h", "--help":
@@ -73,6 +76,9 @@ func run(ctx context.Context, args []string, lookup config.LookupFunc, stdout, s
}
if err != nil {
_, _ = fmt.Fprintf(stderr, "%s: %v\n", brand.ServerBinary, err)
if ee, ok := errors.AsType[*exitError](err); ok {
return ee.code
}
return 1
}
return 0
+11 -11
View File
@@ -28,28 +28,28 @@ func lookupFrom(m map[string]string) func(string) (string, bool) {
func TestRunVersionAndHelp(t *testing.T) {
var out, errOut bytes.Buffer
if code := run(context.Background(), []string{"version"}, lookupFrom(nil), &out, &errOut); code != 0 {
if code := run(context.Background(), []string{"version"}, lookupFrom(nil), nil, &out, &errOut); code != 0 {
t.Fatalf("version exit code %d", code)
}
if !strings.HasPrefix(out.String(), "ddi-server ") {
t.Errorf("version output %q", out.String())
}
out.Reset()
if code := run(context.Background(), []string{"help"}, lookupFrom(nil), &out, &errOut); code != 0 || !strings.Contains(out.String(), "bootstrap-admin") {
if code := run(context.Background(), []string{"help"}, lookupFrom(nil), nil, &out, &errOut); code != 0 || !strings.Contains(out.String(), "bootstrap-admin") {
t.Errorf("help: code %d, output %q", code, out.String())
}
}
func TestRunUnknownCommand(t *testing.T) {
var out, errOut bytes.Buffer
if code := run(context.Background(), []string{"frobnicate"}, lookupFrom(nil), &out, &errOut); code != 2 {
if code := run(context.Background(), []string{"frobnicate"}, lookupFrom(nil), nil, &out, &errOut); code != 2 {
t.Errorf("exit code %d, want 2", code)
}
}
func TestRunInvalidConfig(t *testing.T) {
var errOut bytes.Buffer
code := run(context.Background(), []string{"serve"}, lookupFrom(map[string]string{"DDI_HTTP_LISTEN": "bad"}), io.Discard, &errOut)
code := run(context.Background(), []string{"serve"}, lookupFrom(map[string]string{"DDI_HTTP_LISTEN": "bad"}), nil, io.Discard, &errOut)
if code != 1 || !strings.Contains(errOut.String(), "invalid configuration") {
t.Errorf("code %d, stderr %q", code, errOut.String())
}
@@ -58,7 +58,7 @@ func TestRunInvalidConfig(t *testing.T) {
func TestServeFailsWithoutMasterKey(t *testing.T) {
var errOut bytes.Buffer
env := map[string]string{"DDI_MASTER_KEY_FILE": filepath.Join(t.TempDir(), "missing.key"), "DDI_DATABASE_URL": "postgres://x"}
if code := run(context.Background(), []string{"serve"}, lookupFrom(env), io.Discard, &errOut); code != 1 || !strings.Contains(errOut.String(), "master key") {
if code := run(context.Background(), []string{"serve"}, lookupFrom(env), nil, io.Discard, &errOut); code != 1 || !strings.Contains(errOut.String(), "master key") {
t.Errorf("code %d, stderr %q", code, errOut.String())
}
}
@@ -66,14 +66,14 @@ func TestServeFailsWithoutMasterKey(t *testing.T) {
func TestGenMasterKey(t *testing.T) {
path := filepath.Join(t.TempDir(), "sub", "master.key")
var out, errOut bytes.Buffer
if code := run(context.Background(), []string{"gen-master-key", "-out", path}, lookupFrom(nil), &out, &errOut); code != 0 {
if code := run(context.Background(), []string{"gen-master-key", "-out", path}, lookupFrom(nil), nil, &out, &errOut); code != 0 {
t.Fatalf("code %d: %s", code, errOut.String())
}
data, err := os.ReadFile(path)
if err != nil || !strings.Contains(string(data), "k1 ") {
t.Fatalf("key file: %q %v", data, err)
}
if code := run(context.Background(), []string{"gen-master-key", "-out", path}, lookupFrom(nil), &out, &errOut); code != 1 {
if code := run(context.Background(), []string{"gen-master-key", "-out", path}, lookupFrom(nil), nil, &out, &errOut); code != 1 {
t.Error("existing key file must not be overwritten")
}
}
@@ -141,7 +141,7 @@ func TestServeEndToEnd(t *testing.T) {
dbURL := testDatabase(t)
dir := t.TempDir()
keyFile := filepath.Join(dir, "master.key")
if code := run(context.Background(), []string{"gen-master-key", "-out", keyFile}, lookupFrom(nil), io.Discard, io.Discard); code != 0 {
if code := run(context.Background(), []string{"gen-master-key", "-out", keyFile}, lookupFrom(nil), nil, io.Discard, io.Discard); code != 0 {
t.Fatal("gen-master-key failed")
}
addr := freeAddr(t)
@@ -152,20 +152,20 @@ func TestServeEndToEnd(t *testing.T) {
}
var out bytes.Buffer
if code := run(context.Background(), []string{"bootstrap-admin", "-username", "root"}, lookupFrom(env), &out, io.Discard); code != 0 {
if code := run(context.Background(), []string{"bootstrap-admin", "-username", "root"}, lookupFrom(env), nil, &out, io.Discard); code != 0 {
t.Fatalf("bootstrap-admin failed: %s", out.String())
}
if !strings.Contains(out.String(), "username: root") || !strings.Contains(out.String(), "password: ") {
t.Errorf("bootstrap output %q", out.String())
}
if code := run(context.Background(), []string{"bootstrap-admin"}, lookupFrom(env), io.Discard, io.Discard); code != 1 {
if code := run(context.Background(), []string{"bootstrap-admin"}, lookupFrom(env), nil, io.Discard, io.Discard); code != 1 {
t.Error("second bootstrap must fail")
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan int, 1)
var logs bytes.Buffer
go func() { done <- run(ctx, []string{"serve"}, lookupFrom(env), io.Discard, &syncWriter{w: &logs}) }()
go func() { done <- run(ctx, []string{"serve"}, lookupFrom(env), nil, io.Discard, &syncWriter{w: &logs}) }()
base := fmt.Sprintf("http://%s", addr)
get := func(path string) (int, string) {
+38
View File
@@ -182,6 +182,7 @@ func Serve(ctx context.Context, core *Core) error {
defer stopJobs()
go core.License.Run(jobs)
go cleanup(jobs, core)
go retention(jobs, core, retentionStartDelay, retentionInterval)
log.Info("starting server", slog.String("version", info.Version), slog.String("commit", info.Commit),
slog.String("listen", cfg.HTTP.Listen), slog.Bool("tls", cfg.TLSEnabled()),
@@ -236,6 +237,43 @@ func Serve(ctx context.Context, core *Core) error {
return nil
}
// The retention cleanup runs shortly after the start (not during it, so
// that a restart loop does not hammer the database) and then daily.
const (
retentionStartDelay = 5 * time.Minute
retentionInterval = 24 * time.Hour
)
// retention applies the configured retention policy (ADR 0024).
func retention(ctx context.Context, core *Core, delay, interval time.Duration) {
policy := service.RetentionPolicy{AuditDays: core.Config.Retention.AuditDays,
RevisionsKeep: core.Config.Retention.RevisionsKeep}
if !policy.Enabled() {
return
}
log := core.Log.With(slog.String("component", "retention"))
t := time.NewTimer(delay)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
}
res, err := core.Service.ApplyRetention(ctx, policy)
switch {
case err != nil:
log.Warn("retention cleanup failed", slog.Any("error", err))
case res.AuditDeleted > 0 || len(res.RevisionsDeleted) > 0:
log.Info("retention cleanup", slog.Int64("audit_entries_deleted", res.AuditDeleted),
slog.Int("revisions_deleted", len(res.RevisionsDeleted)))
default:
log.Debug("retention cleanup: nothing to delete")
}
t.Reset(interval)
}
}
// cleanup removes expired sessions and old enrollment tokens periodically.
func cleanup(ctx context.Context, core *Core) {
t := time.NewTicker(15 * time.Minute)
+64
View File
@@ -0,0 +1,64 @@
// SPDX-License-Identifier: LicenseRef-DDI-Proprietary
package app
import (
"context"
"io"
"log/slog"
"testing"
"time"
"git.beugsiud.de/beug.k/DDI/server/internal/config"
"git.beugsiud.de/beug.k/DDI/server/internal/service"
"git.beugsiud.de/beug.k/DDI/server/internal/store"
"git.beugsiud.de/beug.k/DDI/server/internal/store/storetest"
)
func TestRetentionJob(t *testing.T) {
st := storetest.New(t)
ctx := context.Background()
if err := st.Q().InsertAudit(ctx, &store.AuditEntry{ActorName: "admin", Action: "old.one", ObjectType: "test"}); err != nil {
t.Fatal(err)
}
if _, err := st.Pool().Exec(ctx, `UPDATE audit_log SET at = now() - interval '10 days'`); err != nil {
t.Fatal(err)
}
log := slog.New(slog.NewTextHandler(io.Discard, nil))
cfg := config.Defaults()
core := &Core{Config: cfg, Store: st, Log: log, Service: service.New(service.Options{Store: st, Logger: log})}
// Disabled policy: returns at once.
done := make(chan struct{})
go func() { retention(ctx, core, 0, time.Hour); close(done) }()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("disabled retention job did not return")
}
core.Config.Retention.AuditDays = 7
jobCtx, cancel := context.WithCancel(ctx)
done = make(chan struct{})
go func() { retention(jobCtx, core, 10*time.Millisecond, time.Hour); close(done) }()
deadline := time.Now().Add(10 * time.Second)
for {
entries, err := st.Q().ListAudit(ctx, store.AuditFilter{Action: "old.one"})
if err != nil {
t.Fatal(err)
}
if len(entries) == 0 {
break
}
if time.Now().After(deadline) {
t.Fatal("old audit entry was not deleted")
}
time.Sleep(20 * time.Millisecond)
}
cancel()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("retention job did not stop")
}
}
+41 -9
View File
@@ -71,15 +71,28 @@ func (d Duration) Std() time.Duration { return time.Duration(d) }
// Config is the server configuration.
type Config struct {
HTTP HTTP `yaml:"http"`
AgentAPI AgentAPI `yaml:"agent_api"`
Database Database `yaml:"database"`
MasterKeyFile string `yaml:"master_key_file"`
StateDir string `yaml:"state_dir"`
Log Log `yaml:"log"`
Session Session `yaml:"session"`
License License `yaml:"license"`
ShutdownTimeout Duration `yaml:"shutdown_timeout"`
HTTP HTTP `yaml:"http"`
AgentAPI AgentAPI `yaml:"agent_api"`
Database Database `yaml:"database"`
MasterKeyFile string `yaml:"master_key_file"`
StateDir string `yaml:"state_dir"`
Log Log `yaml:"log"`
Session Session `yaml:"session"`
License License `yaml:"license"`
Retention Retention `yaml:"retention"`
ShutdownTimeout Duration `yaml:"shutdown_timeout"`
}
// Retention limits how long history is kept (ADR 0024). A daily cleanup
// applies it; 0 keeps everything.
type Retention struct {
// AuditDays deletes audit entries older than this many days.
AuditDays int `yaml:"audit_days"`
// RevisionsKeep keeps the newest this many configuration revisions.
// The latest revision, revisions an agent runs, falls back to or is
// told to apply, and revisions with pending deployments are always
// kept in addition.
RevisionsKeep int `yaml:"revisions_keep"`
}
// HTTP configures the web UI/API listener.
@@ -289,6 +302,17 @@ func str(dst func(*Config) *string) func(*Config, string) error {
return func(c *Config, v string) error { *dst(c) = v; return nil }
}
func integer(dst func(*Config) *int) func(*Config, string) error {
return func(c *Config, v string) error {
n, err := strconv.Atoi(strings.TrimSpace(v))
if err != nil {
return fmt.Errorf("invalid number %q", v)
}
*dst(c) = n
return nil
}
}
func dur(dst func(*Config) *Duration) func(*Config, string) error {
return func(c *Config, v string) error {
d, err := ParseDuration(v)
@@ -326,6 +350,8 @@ var envBindings = []envBinding{
{name: "SESSION_LIFETIME", set: dur(func(c *Config) *Duration { return &c.Session.Lifetime })},
{name: "SESSION_IDLE_TIMEOUT", set: dur(func(c *Config) *Duration { return &c.Session.IdleTimeout })},
{name: "LICENSE_GRACE_PERIOD", set: dur(func(c *Config) *Duration { return &c.License.GracePeriod })},
{name: "RETENTION_AUDIT_DAYS", set: integer(func(c *Config) *int { return &c.Retention.AuditDays })},
{name: "RETENTION_REVISIONS_KEEP", set: integer(func(c *Config) *int { return &c.Retention.RevisionsKeep })},
// Names given by the project brief, accepted without prefix.
{name: "KEYGEN_URL", raw: true, set: str(func(c *Config) *string { return &c.License.KeygenURL })},
{name: "KEYGEN_ACCOUNT_ID", raw: true, set: str(func(c *Config) *string { return &c.License.AccountID })},
@@ -389,6 +415,12 @@ func (c Config) Validate() error {
if c.Database.MaxConns < 1 {
errs = append(errs, errors.New("database.max_conns must be at least 1"))
}
if c.Retention.AuditDays < 0 || c.Retention.AuditDays > 36500 {
errs = append(errs, errors.New("retention.audit_days must be between 0 (keep all) and 36500"))
}
if c.Retention.RevisionsKeep < 0 {
errs = append(errs, errors.New("retention.revisions_keep must not be negative (0 keeps all)"))
}
errs = append(errs, c.AgentAPI.validate()...)
return errors.Join(errs...)
}
+16 -3
View File
@@ -43,6 +43,9 @@ func TestDefaults(t *testing.T) {
if !strings.HasSuffix(filepath.ToSlash(cfg.MasterKeyFile), "/master.key") {
t.Errorf("master key default %q", cfg.MasterKeyFile)
}
if cfg.Retention != (Retention{}) {
t.Errorf("retention must default to keeping everything: %+v", cfg.Retention)
}
}
func TestFileAndEnvPrecedence(t *testing.T) {
@@ -57,11 +60,15 @@ license:
grace_period: 7d
session:
idle_timeout: 30m
retention:
audit_days: 365
revisions_keep: 50
`)
cfg, err := Load(path, env(map[string]string{
"DDI_DATABASE_URL": "postgres://env",
"KEYGEN_PRODUCT_ID": "prod-1",
"DDI_LOG_LEVEL": "debug",
"DDI_DATABASE_URL": "postgres://env",
"KEYGEN_PRODUCT_ID": "prod-1",
"DDI_LOG_LEVEL": "debug",
"DDI_RETENTION_REVISIONS_KEEP": "20",
}))
if err != nil {
t.Fatal(err)
@@ -81,6 +88,9 @@ session:
if cfg.Session.IdleTimeout.Std() != 30*time.Minute || cfg.Log.Level != "debug" {
t.Errorf("session/log: %+v %+v", cfg.Session, cfg.Log)
}
if cfg.Retention.AuditDays != 365 || cfg.Retention.RevisionsKeep != 20 {
t.Errorf("retention: %+v", cfg.Retention)
}
}
func TestDatabaseURLFile(t *testing.T) {
@@ -116,6 +126,9 @@ func TestValidation(t *testing.T) {
"http keygen": {map[string]string{"KEYGEN_URL": "http://license.example.test"}, "keygen_url"},
"relative public": {map[string]string{"DDI_PUBLIC_URL": "/ddi"}, "public_url"},
"bad grace in days": {map[string]string{"DDI_LICENSE_GRACE_PERIOD": "xd"}, "LICENSE_GRACE_PERIOD"},
"audit days text": {map[string]string{"DDI_RETENTION_AUDIT_DAYS": "1y"}, "RETENTION_AUDIT_DAYS"},
"negative audit": {map[string]string{"DDI_RETENTION_AUDIT_DAYS": "-1"}, "retention.audit_days"},
"negative keep": {map[string]string{"DDI_RETENTION_REVISIONS_KEEP": "-3"}, "retention.revisions_keep"},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
@@ -146,6 +146,16 @@ func TestBINDZoneTransfer(t *testing.T) {
script := `set -e
mkdir -p /var/cache/bind/p /var/cache/bind/s ` + secZoneDir + `/secondary
named -g -c /etc/bind/primary.conf >/tmp/primary.log 2>&1 &
# Start the secondary only when the primary serves its zones: a refresh
# that fails first is retried only after the zone's retry interval.
i=0
until grep -q 'running' /tmp/primary.log; do
if [ $i -ge 30 ]; then
echo "--- primary did not start ---"; cat /tmp/primary.log; exit 1
fi
sleep 1
i=$((i + 1))
done
named -g -c /etc/bind/secondary.conf >/tmp/secondary.log 2>&1 &
i=0
while [ $i -lt 60 ]; do
+74
View File
@@ -0,0 +1,74 @@
// SPDX-License-Identifier: LicenseRef-DDI-Proprietary
package service
import (
"context"
"time"
"git.beugsiud.de/beug.k/DDI/server/internal/store"
)
// RetentionPolicy says how much history to keep (ADR 0024); zero values
// keep everything.
type RetentionPolicy struct {
AuditDays int
RevisionsKeep int
}
// RetentionResult reports what a cleanup removed.
type RetentionResult struct {
AuditDeleted int64 `json:"audit_deleted"`
RevisionsDeleted []int64 `json:"revisions_deleted"`
}
// Enabled reports whether the policy removes anything at all.
func (p RetentionPolicy) Enabled() bool { return p.AuditDays > 0 || p.RevisionsKeep > 0 }
// ApplyRetention deletes audit entries older than AuditDays and revisions
// beyond the newest RevisionsKeep (see store.PruneRevisions for the ones
// that are always kept). It runs in one transaction and, if anything was
// deleted, records that in the audit log. It works in license read-only
// mode as well (housekeeping).
func (s *Service) ApplyRetention(ctx context.Context, p RetentionPolicy) (RetentionResult, error) {
res := RetentionResult{RevisionsDeleted: []int64{}}
if !p.Enabled() {
return res, nil
}
err := s.inTx(ctx, func(tx *store.Tx) error {
if p.AuditDays > 0 {
n, err := tx.DeleteAuditBefore(ctx, s.now().Add(-time.Duration(p.AuditDays)*24*time.Hour))
if err != nil {
return err
}
res.AuditDeleted = n
}
if p.RevisionsKeep > 0 {
// Serialize with revision creation and pushes.
if err := tx.LockRevisions(ctx); err != nil {
return err
}
deleted, err := tx.PruneRevisions(ctx, p.RevisionsKeep)
if err != nil {
return err
}
if deleted != nil {
res.RevisionsDeleted = deleted
}
}
if res.AuditDeleted == 0 && len(res.RevisionsDeleted) == 0 {
return nil
}
after := map[string]any{"audit_days": p.AuditDays, "revisions_keep": p.RevisionsKeep,
"audit_deleted": res.AuditDeleted, "revisions_deleted": len(res.RevisionsDeleted)}
if n := len(res.RevisionsDeleted); n > 0 {
// The list can be long on the first run; the range is enough.
after["revisions_oldest"], after["revisions_newest"] = res.RevisionsDeleted[0], res.RevisionsDeleted[n-1]
}
return s.audit(ctx, tx, SystemActor, "retention.cleanup", "system", "", "retention", nil, after)
})
if err != nil {
return RetentionResult{RevisionsDeleted: []int64{}}, err
}
return res, nil
}
+65
View File
@@ -0,0 +1,65 @@
// SPDX-License-Identifier: LicenseRef-DDI-Proprietary
package service
import (
"context"
"encoding/json"
"slices"
"testing"
"time"
"git.beugsiud.de/beug.k/DDI/server/internal/store"
"git.beugsiud.de/beug.k/DDI/server/internal/store/storetest"
)
func TestApplyRetention(t *testing.T) {
st := storetest.New(t)
ctx := context.Background()
svc := New(Options{Store: st, Keyring: testKeyring(t)})
now := time.Now().UTC().Truncate(time.Microsecond)
for n := int64(1); n <= 4; n++ {
if err := st.Q().InsertRevision(ctx, &store.Revision{Number: n, CreatedAt: now, CreatedByName: "admin"}, nil); err != nil {
t.Fatal(err)
}
}
for _, action := range []string{"old.one", "new.one"} {
if err := st.Q().InsertAudit(ctx, &store.AuditEntry{ActorName: "admin", Action: action, ObjectType: "test"}); err != nil {
t.Fatal(err)
}
}
if _, err := st.Pool().Exec(ctx, `UPDATE audit_log SET at = now() - interval '100 days' WHERE action = 'old.one'`); err != nil {
t.Fatal(err)
}
if res, err := svc.ApplyRetention(ctx, RetentionPolicy{}); err != nil || res.AuditDeleted != 0 || len(res.RevisionsDeleted) != 0 {
t.Fatalf("disabled policy: %+v %v", res, err)
}
policy := RetentionPolicy{AuditDays: 90, RevisionsKeep: 2}
res, err := svc.ApplyRetention(ctx, policy)
if err != nil {
t.Fatal(err)
}
if res.AuditDeleted != 1 || !slices.Equal(res.RevisionsDeleted, []int64{1, 2}) {
t.Errorf("result %+v", res)
}
entries, err := st.Q().ListAudit(ctx, store.AuditFilter{Action: "retention.cleanup"})
if err != nil || len(entries) != 1 {
t.Fatalf("cleanup audit entries %d %v", len(entries), err)
}
var after map[string]any
if err := json.Unmarshal(entries[0].After, &after); err != nil || after["audit_deleted"] != 1.0 ||
after["revisions_deleted"] != 2.0 || after["revisions_oldest"] != 1.0 || after["revisions_newest"] != 2.0 ||
entries[0].ActorName != "system" {
t.Errorf("audit entry %+v %s", entries[0], entries[0].After)
}
// Nothing left to delete: no further audit entry.
if res, err := svc.ApplyRetention(ctx, policy); err != nil || res.AuditDeleted != 0 || len(res.RevisionsDeleted) != 0 {
t.Errorf("second run %+v %v", res, err)
}
if entries, _ := st.Q().ListAudit(ctx, store.AuditFilter{Action: "retention.cleanup"}); len(entries) != 1 {
t.Errorf("second run wrote %d cleanup entries", len(entries))
}
}
+63
View File
@@ -0,0 +1,63 @@
// SPDX-License-Identifier: LicenseRef-DDI-Proprietary
package store
import (
"context"
"slices"
"time"
)
// DeleteAuditBefore deletes audit entries written before t0 (retention,
// ADR 0024) and returns how many were removed.
func (t *Tx) DeleteAuditBefore(ctx context.Context, t0 time.Time) (int64, error) {
tag, err := t.q.Exec(ctx, `DELETE FROM audit_log WHERE at < $1`, t0)
if err != nil {
return 0, mapErr(err)
}
return tag.RowsAffected(), nil
}
// PruneRevisions deletes configuration revisions (with their files) that
// are older than the newest keep revisions (retention, ADR 0024) and
// returns the deleted numbers in ascending order. keep must be at least 1,
// so the latest revision always stays. Never deleted are revisions that an
// agent runs (applied_revision), falls back to (last_good_revision) or is
// told to apply (target_revision), and revisions with pending deployments.
// Deployment rows keep their revision number.
//
// The caller should hold LockRevisions so that no revision is created or
// pushed concurrently.
func (t *Tx) PruneRevisions(ctx context.Context, keep int) ([]int64, error) {
if keep < 1 {
keep = 1
}
rows, err := t.q.Query(ctx, `
DELETE FROM config_revisions r
WHERE r.number < (
SELECT min(k.number) FROM (
SELECT number FROM config_revisions ORDER BY number DESC LIMIT $1) k)
AND NOT EXISTS (
SELECT 1 FROM agents a
WHERE r.number IN (a.applied_revision, a.last_good_revision, a.target_revision))
AND NOT EXISTS (
SELECT 1 FROM deployments d WHERE d.revision = r.number AND d.outcome = 'pending')
RETURNING r.number`, keep)
if err != nil {
return nil, mapErr(err)
}
defer rows.Close()
var out []int64
for rows.Next() {
var n int64
if err := rows.Scan(&n); err != nil {
return nil, mapErr(err)
}
out = append(out, n)
}
if err := rows.Err(); err != nil {
return nil, mapErr(err)
}
slices.Sort(out) // RETURNING has no defined order
return out, nil
}
+118
View File
@@ -0,0 +1,118 @@
// SPDX-License-Identifier: LicenseRef-DDI-Proprietary
package store_test
import (
"context"
"slices"
"testing"
"time"
"git.beugsiud.de/beug.k/DDI/server/internal/store"
"git.beugsiud.de/beug.k/DDI/server/internal/store/storetest"
)
func TestDeleteAuditBefore(t *testing.T) {
st := storetest.New(t)
ctx := context.Background()
for _, action := range []string{"old.one", "old.two", "new.one"} {
if err := st.Q().InsertAudit(ctx, &store.AuditEntry{ActorName: "admin", Action: action, ObjectType: "test"}); err != nil {
t.Fatal(err)
}
}
if _, err := st.Pool().Exec(ctx, `UPDATE audit_log SET at = now() - interval '40 days' WHERE action LIKE 'old.%'`); err != nil {
t.Fatal(err)
}
n, err := st.Q().DeleteAuditBefore(ctx, time.Now().Add(-30*24*time.Hour))
if err != nil || n != 2 {
t.Fatalf("deleted %d, %v", n, err)
}
left, err := st.Q().ListAudit(ctx, store.AuditFilter{})
if err != nil || len(left) != 1 || left[0].Action != "new.one" {
t.Errorf("left %+v %v", left, err)
}
if n, err := st.Q().DeleteAuditBefore(ctx, time.Now().Add(-30*24*time.Hour)); err != nil || n != 0 {
t.Errorf("second run deleted %d, %v", n, err)
}
}
func TestPruneRevisions(t *testing.T) {
st := storetest.New(t)
ctx := context.Background()
now := time.Now().UTC().Truncate(time.Microsecond)
for n := int64(1); n <= 10; n++ {
files := []store.RevisionFile{{Role: "dns", Path: "ddi.conf", Mode: 0o644, Content: []byte("rev")}}
if err := st.Q().InsertRevision(ctx, &store.Revision{Number: n, CreatedAt: now, CreatedByName: "admin"}, files); err != nil {
t.Fatal(err)
}
}
newAgent(t, st, agentA, "dns", "ns1", "0a")
newAgent(t, st, agentB, "dns", "ns2", "0b")
// Agent A runs 3 and falls back to 2, agent B is told to apply 4, a
// push of 5 to agent A is pending, revision 1 has a finished report.
if _, err := st.Q().UpdateAgentStatus(ctx, agentA, store.AgentStatus{AppliedRevision: ptr(int64(3)),
LastGoodRevision: ptr(int64(2)), SeenAt: now}); err != nil {
t.Fatal(err)
}
if err := st.Q().SetAgentTarget(ctx, agentB, 4); err != nil {
t.Fatal(err)
}
requester := "admin"
if err := st.Q().InsertPushDeployment(ctx, &store.Deployment{AgentID: agentA, Revision: 5, StartedAt: now,
RequestedByName: &requester}); err != nil {
t.Fatal(err)
}
finished := now.Add(time.Second)
if _, err := st.Q().InsertDeployment(ctx, &store.Deployment{AgentID: agentB, Revision: 1, Trigger: "pull",
Outcome: store.OutcomeSucceeded, StartedAt: now, FinishedAt: &finished}); err != nil {
t.Fatal(err)
}
var deleted []int64
err := st.InTx(ctx, func(tx *store.Tx) error {
if err := tx.LockRevisions(ctx); err != nil {
return err
}
var err error
deleted, err = tx.PruneRevisions(ctx, 3)
return err
})
if err != nil {
t.Fatal(err)
}
if !slices.Equal(deleted, []int64{1, 6, 7}) {
t.Errorf("deleted %v, want [1 6 7]", deleted)
}
revs, err := st.Q().ListRevisions(ctx, 100, 0)
if err != nil {
t.Fatal(err)
}
var kept []int64
for _, r := range revs {
kept = append(kept, r.Number)
}
if !slices.Equal(kept, []int64{10, 9, 8, 5, 4, 3, 2}) {
t.Errorf("kept %v", kept)
}
if files, err := st.Q().RevisionFiles(ctx, 1, ""); err != nil || len(files) != 0 {
t.Errorf("files of a deleted revision: %d %v", len(files), err)
}
if files, err := st.Q().RevisionFiles(ctx, 8, ""); err != nil || len(files) != 1 {
t.Errorf("files of a kept revision: %d %v", len(files), err)
}
var reports int
if err := st.Pool().QueryRow(ctx, `SELECT count(*) FROM deployments WHERE revision = 1`).Scan(&reports); err != nil || reports != 1 {
t.Errorf("deployment rows of deleted revisions stay: %d %v", reports, err)
}
if again, err := st.Q().PruneRevisions(ctx, 3); err != nil || len(again) != 0 {
t.Errorf("second run: %v %v", again, err)
}
// keep < 1 still keeps the latest revision.
if last, err := st.Q().PruneRevisions(ctx, 0); err != nil || !slices.Equal(last, []int64{8, 9}) {
t.Errorf("keep 0: %v %v", last, err)
}
if latest, err := st.Q().LatestRevision(ctx); err != nil || latest.Number != 10 {
t.Errorf("latest %+v %v", latest, err)
}
}