Merge branch 'main' into worktree-agent-bench-m6
This commit is contained in:
+29
-2
@@ -99,6 +99,8 @@ jobs:
|
||||
run: make lint-shell
|
||||
- name: SPDX headers
|
||||
run: make lint-spdx
|
||||
- name: No hard-coded product names (ADR 0018)
|
||||
run: make lint-brand
|
||||
|
||||
release-dry-run:
|
||||
name: Release build (dry run)
|
||||
@@ -126,10 +128,35 @@ jobs:
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: "24"
|
||||
- name: Build release artefacts
|
||||
run: make release
|
||||
- name: Install nfpm
|
||||
run: scripts/ci/install-tools.sh "$HOME/.local/bin" nfpm
|
||||
- name: Build release artefacts and packages
|
||||
run: make release PACKAGES=1
|
||||
- name: Verify checksums
|
||||
run: cd dist && sha256sum -c SHA256SUMS
|
||||
- name: Inspect packages
|
||||
run: |
|
||||
ls -l dist/*.deb dist/*.rpm
|
||||
# 3 binaries x 2 architectures x (deb, rpm)
|
||||
test "$(ls dist/*.deb dist/*.rpm | wc -l)" -eq 12
|
||||
for deb in dist/*_amd64.deb; do
|
||||
dpkg-deb --info "$deb" >/dev/null
|
||||
dpkg-deb --contents "$deb" >contents.txt
|
||||
grep -q ' ./usr/bin/' contents.txt
|
||||
grep -q ' ./usr/lib/systemd/system/.*\.service$' contents.txt
|
||||
grep -q ' ./usr/share/doc/.*/LICENSE$' contents.txt
|
||||
done
|
||||
- name: Release signature (throwaway key)
|
||||
run: |
|
||||
command -v openssl >/dev/null || (apt-get update -qq && apt-get install -y -qq openssl)
|
||||
scripts/sign-release.sh selftest
|
||||
umask 077
|
||||
key="$(mktemp)"
|
||||
openssl genpkey -algorithm ed25519 -out "$key"
|
||||
RELEASE_SIGNING_KEY="$(cat "$key")" scripts/sign-release.sh sign dist/SHA256SUMS
|
||||
scripts/sign-release.sh public-key "$key" >"$key.pub"
|
||||
scripts/sign-release.sh verify dist/SHA256SUMS dist/SHA256SUMS.sig "$key.pub"
|
||||
rm -f "$key" "$key.pub" dist/SHA256SUMS.sig
|
||||
- name: Smoke test of the released server binary
|
||||
env:
|
||||
DDI_DATABASE_URL: postgres://ddi:ddi-ci-only@postgres:5432/ddi?sslmode=disable
|
||||
|
||||
@@ -23,12 +23,40 @@ jobs:
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: "24"
|
||||
- name: Ensure jq
|
||||
run: command -v jq >/dev/null || (apt-get update -qq && apt-get install -y -qq jq)
|
||||
- name: Ensure jq and openssl
|
||||
run: |
|
||||
command -v jq >/dev/null && command -v openssl >/dev/null ||
|
||||
(apt-get update -qq && apt-get install -y -qq jq openssl)
|
||||
- name: Install nfpm
|
||||
run: scripts/ci/install-tools.sh "$HOME/.local/bin" nfpm
|
||||
- name: Build
|
||||
run: make release VERSION="${{ github.ref_name }}"
|
||||
run: make release PACKAGES=1 VERSION="${{ github.ref_name }}"
|
||||
- name: Verify checksums
|
||||
run: cd dist && sha256sum -c SHA256SUMS
|
||||
# Ed25519 signature over SHA256SUMS (ADR 0024, docs/RELEASE.md) if the
|
||||
# secret RELEASE_SIGNING_KEY (PEM private key) is set. The key only
|
||||
# exists in a private temporary file during signing.
|
||||
- name: Sign checksums
|
||||
env:
|
||||
RELEASE_SIGNING_KEY: ${{ secrets.RELEASE_SIGNING_KEY }}
|
||||
run: |
|
||||
if [[ -z "${RELEASE_SIGNING_KEY:-}" ]]; then
|
||||
echo "::warning::RELEASE_SIGNING_KEY is not set: the release is not signed"
|
||||
exit 0
|
||||
fi
|
||||
scripts/sign-release.sh sign dist/SHA256SUMS
|
||||
# The installer must accept the signature: verify with its public
|
||||
# key if one is configured there.
|
||||
pub="$(mktemp)"
|
||||
sed -n -e 's/^[[:space:]]*//' -e 's/.*\(-----BEGIN PUBLIC KEY-----\)/\1/' \
|
||||
-e 's/\(-----END PUBLIC KEY-----\).*/\1/' \
|
||||
-e '/-----BEGIN PUBLIC KEY-----/,/-----END PUBLIC KEY-----/p' scripts/install.sh >"$pub" 2>/dev/null || true
|
||||
if [[ -s "$pub" ]]; then
|
||||
scripts/sign-release.sh verify dist/SHA256SUMS dist/SHA256SUMS.sig "$pub"
|
||||
else
|
||||
echo "::warning::no public key in scripts/install.sh: the installer will not check the signature"
|
||||
fi
|
||||
rm -f "$pub"
|
||||
- name: Upload to release
|
||||
env:
|
||||
GITEA_API_URL: ${{ github.server_url }}/api/v1
|
||||
|
||||
@@ -25,6 +25,10 @@ BUILD_DATE ?= $(shell git log -1 --format=%cI 2>/dev/null)
|
||||
PLATFORMS ?= linux/amd64 linux/arm64
|
||||
BIN_DIR ?= bin
|
||||
DIST_DIR ?= dist
|
||||
# nfpm for make packages / PACKAGES=1; scripts/ci/install-tools.sh installs
|
||||
# the pinned version (NFPM_VERSION there).
|
||||
NFPM ?= nfpm
|
||||
export NFPM
|
||||
|
||||
COMPOSE_DEV := docker compose -f deploy/compose/compose.dev.yaml
|
||||
|
||||
@@ -103,7 +107,7 @@ fmt:
|
||||
|
||||
## lint: run all linters
|
||||
.PHONY: lint
|
||||
lint: lint-go lint-proto lint-web lint-shell lint-spdx
|
||||
lint: lint-go lint-proto lint-web lint-shell lint-spdx lint-brand
|
||||
|
||||
.PHONY: lint-go
|
||||
lint-go:
|
||||
@@ -127,6 +131,11 @@ lint-shell:
|
||||
lint-spdx:
|
||||
scripts/check-spdx.sh
|
||||
|
||||
## lint-brand: no hard-coded product names in units, packages, installer (ADR 0018)
|
||||
.PHONY: lint-brand
|
||||
lint-brand:
|
||||
scripts/check-brand.sh
|
||||
|
||||
## test: run Go and frontend unit tests
|
||||
.PHONY: test
|
||||
test: test-go test-web
|
||||
@@ -167,10 +176,23 @@ ci: lint check-generated test vuln build
|
||||
|
||||
# --------------------------------------------------------------------------- release
|
||||
|
||||
## release: cross-compile tarballs and SHA256SUMS into dist/
|
||||
## release: cross-compile tarballs and SHA256SUMS into dist/ (PACKAGES=1: also .deb/.rpm)
|
||||
.PHONY: release
|
||||
release: web
|
||||
scripts/build-release.sh -o $(DIST_DIR) -p "$(PLATFORMS)"
|
||||
ifeq ($(PACKAGES),1)
|
||||
scripts/build-packages.sh -d $(DIST_DIR) -p "$(PLATFORMS)"
|
||||
endif
|
||||
|
||||
## packages: .deb and .rpm from the tarballs in dist/ (after make release; needs nfpm)
|
||||
.PHONY: packages
|
||||
packages:
|
||||
scripts/build-packages.sh -d $(DIST_DIR) -p "$(PLATFORMS)"
|
||||
|
||||
## sign-release: sign dist/SHA256SUMS with the Ed25519 key in RELEASE_SIGNING_KEY_FILE
|
||||
.PHONY: sign-release
|
||||
sign-release:
|
||||
scripts/sign-release.sh sign $(DIST_DIR)/SHA256SUMS "$(RELEASE_SIGNING_KEY_FILE)"
|
||||
|
||||
# --------------------------------------------------------------------------- development
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Template of the nfpm configuration of the .deb and .rpm packages (ADR 0024),
|
||||
# shared by all binaries. scripts/build-packages.sh replaces the placeholders
|
||||
# (at-sign NAME at-sign) per binary and architecture (ADR 0018: no product
|
||||
# names in this file). The
|
||||
# packages contain only the binary, its systemd unit and the license text;
|
||||
# configuration, users and services are set up by the installer.
|
||||
name: "@PKG_NAME@"
|
||||
arch: "@PKG_ARCH@"
|
||||
platform: linux
|
||||
version: "@PKG_VERSION@"
|
||||
version_schema: semver
|
||||
release: "1"
|
||||
section: net
|
||||
priority: optional
|
||||
maintainer: "@PKG_MAINTAINER@"
|
||||
vendor: "@PKG_VENDOR@"
|
||||
homepage: "@PKG_HOMEPAGE@"
|
||||
license: "@PKG_LICENSE@"
|
||||
description: "@PKG_DESCRIPTION@"
|
||||
|
||||
contents:
|
||||
- src: "@PKG_SRC_DIR@/binary"
|
||||
dst: "/usr/bin/@PKG_NAME@"
|
||||
file_info:
|
||||
mode: 0755
|
||||
- src: "@PKG_SRC_DIR@/unit.service"
|
||||
dst: "/usr/lib/systemd/system/@PKG_NAME@.service"
|
||||
file_info:
|
||||
mode: 0644
|
||||
- src: "@PKG_SRC_DIR@/LICENSE"
|
||||
dst: "/usr/share/doc/@PKG_NAME@/LICENSE"
|
||||
file_info:
|
||||
mode: 0644
|
||||
|
||||
scripts:
|
||||
postinstall: "@PKG_SCRIPTS_DIR@/postinstall.sh"
|
||||
postremove: "@PKG_SCRIPTS_DIR@/postremove.sh"
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Package hook (deb postinst, rpm %post): make systemd see the new or
|
||||
# updated unit. Enabling and starting services is left to the installer.
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload >/dev/null 2>&1 || true
|
||||
fi
|
||||
exit 0
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Package hook (deb postrm, rpm %postun): make systemd forget the removed
|
||||
# unit. Stopping and disabling services is left to the installer.
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload >/dev/null 2>&1 || true
|
||||
fi
|
||||
exit 0
|
||||
@@ -201,14 +201,17 @@ Server und Agents lesen eine YAML-Datei (Server: Pfad aus `DDI_CONFIG_FILE`, Bei
|
||||
| `DDI_LOG_LEVEL`, `DDI_LOG_FORMAT` | Logging |
|
||||
| `DDI_SESSION_LIFETIME`, `DDI_SESSION_IDLE_TIMEOUT` | Sitzungen |
|
||||
| `DDI_LICENSE_GRACE_PERIOD`, `KEYGEN_URL`, `KEYGEN_PRODUCT_ID` | Lizenz (siehe [LICENSING.md](LICENSING.md)) |
|
||||
| `DDI_RETENTION_AUDIT_DAYS`, `DDI_RETENTION_REVISIONS_KEEP` | Aufbewahrung (`retention.audit_days`, `retention.revisions_keep`; 0 = alles behalten) |
|
||||
|
||||
`KEYGEN_ACCOUNT_ID` und `KEYGEN_PUBLIC_KEY` gelten nur für Builds ohne eingebettete Werte (Entwicklung). Dauerangaben verstehen Go-Notation und Tage (`14d`).
|
||||
|
||||
**Aufbewahrung** ([0024](adr/0024-auslieferung-installer-m6.md)): Ist `retention.audit_days` oder `retention.revisions_keep` gesetzt, räumt der Server fünf Minuten nach dem Start und danach täglich auf. Gelöscht werden Audit-Einträge, die älter als die angegebene Zahl Tage sind, und Revisionen jenseits der neuesten `revisions_keep` (samt Dateien). Nie gelöscht werden die neueste Revision, Revisionen, die ein Agent anwendet, als letzte funktionierende kennt oder als Ziel hat, und Revisionen mit offenen Deployments. Ein Aufräumlauf, der etwas löscht, schreibt den Audit-Eintrag `retention.cleanup` mit den Anzahlen.
|
||||
|
||||
## 8. Build, CI und Auslieferung
|
||||
|
||||
- `Makefile` als Einstieg: `build`, `test`, `lint`, `fmt`, `generate`, `web`, `dev-up`/`dev-down`, `release`.
|
||||
- CI: Gitea Actions unter `.gitea/workflows/` ([0003](adr/0003-git-host-gitea-actions.md)). Geprüft werden Go (Format, `vet`, `golangci-lint`, Tests, `govulncheck`), Protobuf (`buf lint`, generierter Code aktuell), Web (ESLint, Typecheck, Build, Tests, `npm audit`), Shell (`shellcheck`), SPDX-Header und die Entwicklungsumgebung (Compose-Smoke-Test).
|
||||
- Release bei Tag `v*`: Binaries für `linux/amd64` und `linux/arm64`, Tarballs, `SHA256SUMS`, Upload an das Gitea-Release.
|
||||
- Release bei Tag `v*`: Binaries für `linux/amd64` und `linux/arm64`, Tarballs, `.deb`/`.rpm` (nfpm), `SHA256SUMS` und – mit Secret `RELEASE_SIGNING_KEY` – die Ed25519-Signatur `SHA256SUMS.sig`, Upload an das Gitea-Release ([RELEASE.md](RELEASE.md)).
|
||||
- Installer `scripts/install.sh` (M6) lädt die Artefakte und prüft die Prüfsummen.
|
||||
|
||||
## 9. Entwicklungsumgebung
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
# Release
|
||||
|
||||
Wie ein Release gebaut, signiert und veröffentlicht wird ([ADR 0003](adr/0003-git-host-gitea-actions.md), [ADR 0024](adr/0024-auslieferung-installer-m6.md)).
|
||||
|
||||
## Ablauf
|
||||
|
||||
1. Meilenstein fertig, CI auf `main` grün.
|
||||
2. Tag setzen und pushen: `git tag -a v0.7.0 -m "M6 Auslieferung" && git push origin v0.7.0`.
|
||||
3. Der Workflow `.gitea/workflows/release.yml` baut mit `make release PACKAGES=1 VERSION=<tag>`:
|
||||
- Tarballs `<binary>_<tag>_linux_<arch>.tar.gz` für `amd64` und `arm64`,
|
||||
- Pakete `<binary>_<version>-1_<arch>.deb` und `<binary>-<version>-1.<arch>.rpm` (nfpm, siehe unten),
|
||||
- `SHA256SUMS` über Tarballs und Pakete,
|
||||
- `SHA256SUMS.sig`, falls das Secret `RELEASE_SIGNING_KEY` gesetzt ist.
|
||||
4. `scripts/release-upload.sh` legt das Gitea-Release an (Tags mit `-` werden als Vorabversion markiert) und lädt alle Dateien aus `dist/` hoch; vorhandene Dateien gleichen Namens werden ersetzt, der Workflow kann also erneut laufen.
|
||||
|
||||
Lokal: `make release` (nur Tarballs), `make release PACKAGES=1` bzw. nach `make release` ein `make packages` (braucht `nfpm`), `make sign-release RELEASE_SIGNING_KEY_FILE=<pfad>`.
|
||||
|
||||
## Pakete (`.deb`, `.rpm`)
|
||||
|
||||
`scripts/build-packages.sh` nimmt die Binaries aus den Tarballs in `dist/` (dieselben Dateien, keine zweite Übersetzung) und baut je Binary und Architektur ein `.deb` und ein `.rpm` mit [nfpm](https://nfpm.goreleaser.com). nfpm wird in der CI von `scripts/ci/install-tools.sh` in einer festen Version mit Prüfsumme installiert (`NFPM_VERSION`); lokal genügt ein `nfpm` im `PATH` oder `make packages NFPM=/pfad/zu/nfpm`.
|
||||
|
||||
| Paket | Inhalt |
|
||||
| --- | --- |
|
||||
| Name = Binary-Name (`BIN_SERVER`, `BIN_AGENT_DNS`, `BIN_AGENT_DHCP` aus dem Makefile) | `/usr/bin/<binary>` (0755) |
|
||||
| | `/usr/lib/systemd/system/<binary>.service` (aus `deploy/systemd/*.service.in`, gerendert mit `/usr/bin`) |
|
||||
| | `/usr/share/doc/<binary>/LICENSE` (Server proprietär, Agents Apache-2.0) |
|
||||
|
||||
- Paketversion: Tag ohne `v` (`v0.7.0` → `0.7.0`, Release `1`); Builds ohne Versions-Tag erhalten `0.0.0-dev.<beschreibung>`.
|
||||
- Die Pakete enthalten **keine** Konfiguration, legen keine Benutzer an und starten nichts. Die Paket-Skripte rufen nur `systemctl daemon-reload` auf. Eingerichtet wird mit `scripts/install.sh`.
|
||||
- Konfiguration: Vorlage `deploy/nfpm/package.yaml.in` mit Platzhaltern (ADR 0018). `PKG_MAINTAINER`, `PKG_VENDOR` und `PKG_HOMEPAGE` lassen sich per Umgebung überschreiben.
|
||||
|
||||
## Signatur
|
||||
|
||||
`SHA256SUMS.sig` ist eine Ed25519-Signatur (64 Byte, binär) über `SHA256SUMS`, erzeugt mit `openssl pkeyutl -sign -rawin`. Weil `SHA256SUMS` alle Artefakte abdeckt, genügt eine Signatur. Der Installer prüft sie mit dem öffentlichen Schlüssel aus seinem Variablenblock.
|
||||
|
||||
### Schlüsselpaar erzeugen (einmalig, offline)
|
||||
|
||||
```sh
|
||||
umask 077
|
||||
openssl genpkey -algorithm ed25519 -out release-signing.pem # privat
|
||||
openssl pkey -in release-signing.pem -pubout -out release-signing.pub.pem
|
||||
```
|
||||
|
||||
Der private Schlüssel gehört in einen Passwortmanager bzw. auf einen Offline-Datenträger, nicht ins Repository. Wer ihn hat, kann Releases signieren, die jeder Installer annimmt.
|
||||
|
||||
### Gitea-Secret setzen
|
||||
|
||||
Repository → Einstellungen → Actions → Secrets → **Secret hinzufügen**: Name `RELEASE_SIGNING_KEY`, Wert = kompletter Inhalt von `release-signing.pem` (einschließlich der Zeilen `-----BEGIN PRIVATE KEY-----`/`-----END PRIVATE KEY-----`).
|
||||
|
||||
Der Workflow schreibt den Schlüssel nur für die Dauer des Signierens in eine Datei mit Modus 0600 (`umask 077`) und löscht sie danach; er wird nie ausgegeben. Ohne Secret entsteht kein `SHA256SUMS.sig`, der Workflow warnt.
|
||||
|
||||
### Öffentlichen Schlüssel in den Installer eintragen
|
||||
|
||||
In `scripts/install.sh` im Variablenblock (zwischen `# --- names (ADR 0018) ---` und `# --- end names ---`) den Inhalt von `release-signing.pub.pem` als `RELEASE_PUBLIC_KEY_PEM` eintragen:
|
||||
|
||||
```sh
|
||||
RELEASE_PUBLIC_KEY_PEM="-----BEGIN PUBLIC KEY-----
|
||||
MCowBQYDK2VwAyEA…
|
||||
-----END PUBLIC KEY-----"
|
||||
```
|
||||
|
||||
Ist der Schlüssel gesetzt, bricht der Installer bei fehlender oder ungültiger Signatur ab; ohne Schlüssel prüft er nur die Prüfsummen und weist darauf hin. Der Release-Workflow prüft die frische Signatur mit dem Schlüssel aus `scripts/install.sh`, damit ein falsch eingetragener Schlüssel schon beim Release auffällt.
|
||||
|
||||
### Prüfen von Hand
|
||||
|
||||
```sh
|
||||
scripts/sign-release.sh verify SHA256SUMS SHA256SUMS.sig release-signing.pub.pem
|
||||
# entspricht
|
||||
openssl pkeyutl -verify -pubin -inkey release-signing.pub.pem -rawin -in SHA256SUMS -sigfile SHA256SUMS.sig
|
||||
sha256sum -c SHA256SUMS
|
||||
```
|
||||
|
||||
Die CI (Job „Release build (dry run)“) erzeugt bei jedem Lauf einen Wegwerf-Schlüssel und prüft Signieren, Verifizieren sowie die Ablehnung einer veränderten Datei und eines fremden Schlüssels (`scripts/sign-release.sh selftest`).
|
||||
|
||||
### Schlüsselwechsel
|
||||
|
||||
Neues Schlüsselpaar erzeugen, Secret ersetzen, öffentlichen Schlüssel in `scripts/install.sh` tauschen, neues Release taggen. Bereits verteilte Installer-Skripte kennen nur den alten Schlüssel und lehnen Releases mit dem neuen ab; sie müssen durch das Skript des neuen Releases ersetzt werden.
|
||||
@@ -56,3 +56,24 @@ Brief §10 verlangt `scripts/install.sh` für die Rollen `server`, `agent-dns` u
|
||||
- Gemischte Distributionen in einem Cluster funktionieren, weil `zone_dir` und die Kea-Pfade überall gleich sind.
|
||||
- Die Kea-Installation hängt standardmäßig vom ISC-Repository ab; ohne Internetzugang bleibt `--kea-source distro` oder ein eigener Spiegel.
|
||||
- Agents haben die Rechte ihres Dienstes; ein kompromittierter Agent kann diesen Dienst umkonfigurieren, aber nicht mehr.
|
||||
|
||||
## Präzisierungen (Umsetzung)
|
||||
|
||||
### `license set-key`
|
||||
|
||||
- Liest den Schlüssel von stdin (höchstens 4096 Byte Eingabe, nach dem Trimmen nicht leer, eine Zeile, höchstens 512 Zeichen) und ruft denselben Service wie `PUT /api/v1/license` auf; der Audit-Eintrag `license.set_key` trägt den Akteur `system` (wie `bootstrap-admin`). Ausgegeben werden Status, Grund, Lizenzname, Ablauf, Grenzen und Features, nie der Schlüssel.
|
||||
- Exit-Codes: `0` aktiviert, `1` sonstiger Fehler (Konfiguration, Datenbank), `2` Eingabe leer/zu groß/mehrzeilig, `3` Schlüssel abgelehnt (`license_invalid`, `license_in_use`), `4` Lizenzserver nicht erreichbar oder Antwort nicht vertrauenswürdig. Wie bei `PUT /license` wird bei `3` und `4` nichts gespeichert; der Installer behandelt `4` als Warnung.
|
||||
|
||||
### Aufbewahrung
|
||||
|
||||
- Die Löschung braucht keine Migration: Der Unveränderlichkeits-Trigger der Revisionen greift nur bei `UPDATE`; Dateien werden per `ON DELETE CASCADE` mitgelöscht.
|
||||
- Der Aufräumlauf läuft fünf Minuten nach dem Start und dann alle 24 Stunden, in einer Transaktion mit der Revisionssperre (wie Erzeugen und Ausrollen). Er läuft auch im Lizenz-Lesemodus.
|
||||
- Geschützt sind die neuesten `revisions_keep` Revisionen (mindestens die neueste) sowie jede Revision, die in `agents.applied_revision`, `last_good_revision` oder `target_revision` steht oder ein Deployment mit Ergebnis `pending` hat.
|
||||
- **Deployments bleiben erhalten**, auch wenn ihre Revision gelöscht ist: Sie speichern nur die Nummer und sind Teil der Historie. Der Diff einer Revision gegen eine gelöschte Vorgängerin liefert `not_found`; ein Rollback auf eine gelöschte Revision ebenso.
|
||||
- Audit-Einträge werden nach `at` gelöscht. Hat der Lauf etwas gelöscht, schreibt er `retention.cleanup` (Objekttyp `system`) mit den Anzahlen und dem Bereich der gelöschten Revisionsnummern. Die Anzeige „offene Änderungen“ bleibt korrekt, weil sie ohne den Audit-Eintrag der Revision auf deren Zeitpunkt zurückfällt.
|
||||
|
||||
### Pakete, Signatur, Namensprüfung
|
||||
|
||||
- **Pakete:** `scripts/build-packages.sh` (`make packages`, `make release PACKAGES=1`) verpackt die Binaries aus den Tarballs (keine zweite Übersetzung) mit nfpm 2.48.0, installiert mit Prüfsumme über `scripts/ci/install-tools.sh` (kein `go run`: nfpm gehört nicht in den Modulgraphen). Paketname = Binary-Name; Inhalt `/usr/bin/<binary>`, `/usr/lib/systemd/system/<binary>.service` (gerendert mit `/usr/bin`), `/usr/share/doc/<binary>/LICENSE`; Paket-Skripte nur `systemctl daemon-reload`. Keine Abhängigkeiten (PostgreSQL, BIND und Kea richtet der Installer ein, Kea je nach `--kea-source` mit anderen Paketnamen). Danach umfasst `SHA256SUMS` Tarballs und Pakete. Details: [RELEASE.md](../RELEASE.md).
|
||||
- **Signatur:** `scripts/sign-release.sh` (`sign`, `verify`, `public-key`, `selftest`); die Signatur ist binär (64 Byte). Der Release-Workflow prüft die neue Signatur mit dem öffentlichen Schlüssel aus `scripts/install.sh`, falls dort einer steht. Die CI prüft Signieren und Verifizieren bei jedem Lauf mit einem Wegwerf-Schlüssel.
|
||||
- **Namensprüfung:** `scripts/check-brand.sh` (`make lint-brand`, Job „Shell“) sucht `ddi-suite`, `ddi-server`, `ddi-agent-dns`, `ddi-agent-dhcp` (ohne Groß-/Kleinschreibung) in `deploy/systemd/*.in`, `deploy/nfpm/*` und `scripts/install.sh`. Erlaubt sind die Namen nur im Block zwischen den Zeilen `# --- names (ADR 0018) ---` und `# --- end names ---`, die im Installer genau einmal vorkommen müssen. Die nfpm-Konfiguration ist deshalb eine Vorlage (`deploy/nfpm/package.yaml.in`) mit Platzhaltern wie die Units.
|
||||
|
||||
Executable
+158
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Builds .deb and .rpm packages with nfpm from the release tarballs in DIST
|
||||
# (made by scripts/build-release.sh) and rewrites DIST/SHA256SUMS to cover
|
||||
# tarballs and packages (ADR 0024).
|
||||
#
|
||||
# Usage: scripts/build-packages.sh [-d DIST] [-p "linux/amd64 linux/arm64"]
|
||||
#
|
||||
# Each package holds the binary from its tarball as /usr/bin/<binary>, the
|
||||
# systemd unit rendered for /usr/bin and the license text. Names and version
|
||||
# come from the environment (Makefile, ADR 0018): VERSION, PRODUCT,
|
||||
# DISPLAY_NAME, BIN_SERVER, BIN_AGENT_DNS, BIN_AGENT_DHCP. NFPM names the
|
||||
# nfpm binary (default: nfpm from PATH, installed by scripts/ci/install-tools.sh).
|
||||
# Optional: PKG_MAINTAINER, PKG_VENDOR, PKG_HOMEPAGE.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PRODUCT="${PRODUCT:-ddi-suite}"
|
||||
DISPLAY_NAME="${DISPLAY_NAME:-DDI-Suite}"
|
||||
BIN_SERVER="${BIN_SERVER:-ddi-server}"
|
||||
BIN_AGENT_DNS="${BIN_AGENT_DNS:-ddi-agent-dns}"
|
||||
BIN_AGENT_DHCP="${BIN_AGENT_DHCP:-ddi-agent-dhcp}"
|
||||
NFPM="${NFPM:-nfpm}"
|
||||
|
||||
dist="dist"
|
||||
platforms="linux/amd64 linux/arm64"
|
||||
|
||||
usage() {
|
||||
sed -n '4,16p' "$0" | sed 's/^# \{0,1\}//'
|
||||
}
|
||||
|
||||
while getopts ":d:p:h" opt; do
|
||||
case "$opt" in
|
||||
d) dist="$OPTARG" ;;
|
||||
p) platforms="$OPTARG" ;;
|
||||
h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
root="$(git rev-parse --show-toplevel)"
|
||||
cd "$root"
|
||||
|
||||
if ! command -v "$NFPM" >/dev/null; then
|
||||
echo "nfpm not found (set NFPM or run scripts/ci/install-tools.sh)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
VERSION="${VERSION:-$(git describe --tags --always --dirty 2>/dev/null || echo dev)}"
|
||||
BUILD_DATE="${BUILD_DATE:-$(git log -1 --format=%cI 2>/dev/null || date -u +%Y-%m-%dT%H:%M:%SZ)}"
|
||||
|
||||
# Package versions must be semantic versions: v1.2.3 -> 1.2.3; anything else
|
||||
# (a commit hash, "dev") becomes a 0.0.0 pre-release that sorts below every
|
||||
# real release.
|
||||
if [[ "$VERSION" =~ ^v?([0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.+-]*)?)$ ]]; then
|
||||
pkg_version="${BASH_REMATCH[1]}"
|
||||
else
|
||||
pkg_version="0.0.0-dev.$(tr -c '0-9A-Za-z.\n' '.' <<<"$VERSION")"
|
||||
fi
|
||||
|
||||
# Reproducible timestamps inside the packages.
|
||||
SOURCE_DATE_EPOCH="$(date -u -d "$BUILD_DATE" +%s 2>/dev/null || echo 0)"
|
||||
export SOURCE_DATE_EPOCH
|
||||
|
||||
declare -A descriptions=(
|
||||
["$BIN_SERVER"]="${DISPLAY_NAME} management server (IPAM, DNS and DHCP)"
|
||||
["$BIN_AGENT_DNS"]="${DISPLAY_NAME} agent for BIND 9"
|
||||
["$BIN_AGENT_DHCP"]="${DISPLAY_NAME} agent for Kea DHCPv4"
|
||||
)
|
||||
declare -A licenses=(
|
||||
["$BIN_SERVER"]="LICENSES/LicenseRef-DDI-Proprietary.txt"
|
||||
["$BIN_AGENT_DNS"]="LICENSES/Apache-2.0.txt"
|
||||
["$BIN_AGENT_DHCP"]="LICENSES/Apache-2.0.txt"
|
||||
)
|
||||
declare -A license_names=(
|
||||
["$BIN_SERVER"]="Proprietary"
|
||||
["$BIN_AGENT_DNS"]="Apache-2.0"
|
||||
["$BIN_AGENT_DHCP"]="Apache-2.0"
|
||||
)
|
||||
|
||||
workdir="$(mktemp -d)"
|
||||
trap 'rm -rf "$workdir"' EXIT
|
||||
# Git Bash on Windows (development): nfpm.exe needs Windows paths.
|
||||
if command -v cygpath >/dev/null; then
|
||||
workdir="$(cygpath -m "$workdir")"
|
||||
fi
|
||||
|
||||
export PRODUCT DISPLAY_NAME BIN_SERVER BIN_AGENT_DNS BIN_AGENT_DHCP
|
||||
scripts/render-units.sh /usr/bin "$workdir/units"
|
||||
|
||||
PKG_MAINTAINER="${PKG_MAINTAINER:-${DISPLAY_NAME} <maintainer@example.invalid>}"
|
||||
PKG_VENDOR="${PKG_VENDOR:-$DISPLAY_NAME}"
|
||||
PKG_HOMEPAGE="${PKG_HOMEPAGE:-https://git.beugsiud.de/beug.k/DDI}"
|
||||
|
||||
# render_config NAME ARCH SRC_DIR LICENSE DESCRIPTION > nfpm.yaml
|
||||
render_config() {
|
||||
local -a subst=(
|
||||
PKG_NAME "$1" PKG_ARCH "$2" PKG_SRC_DIR "$3" PKG_LICENSE "$4" PKG_DESCRIPTION "$5"
|
||||
PKG_VERSION "$pkg_version" PKG_MAINTAINER "$PKG_MAINTAINER" PKG_VENDOR "$PKG_VENDOR"
|
||||
PKG_HOMEPAGE "$PKG_HOMEPAGE" PKG_SCRIPTS_DIR "$root/deploy/nfpm"
|
||||
)
|
||||
local -a expr=()
|
||||
local i value
|
||||
for ((i = 0; i < ${#subst[@]}; i += 2)); do
|
||||
value="${subst[i + 1]}"
|
||||
# The values end up in sed replacements and double-quoted YAML.
|
||||
if [[ "$value" == *[\|\&\\\"]* ]]; then
|
||||
echo "unsafe value for ${subst[i]}: ${value}" >&2
|
||||
exit 2
|
||||
fi
|
||||
expr+=(-e "s|@${subst[i]}@|${value}|g")
|
||||
done
|
||||
sed "${expr[@]}" deploy/nfpm/package.yaml.in
|
||||
}
|
||||
|
||||
for platform in $platforms; do
|
||||
goos="${platform%/*}"
|
||||
goarch="${platform#*/}"
|
||||
for bin in "$BIN_SERVER" "$BIN_AGENT_DNS" "$BIN_AGENT_DHCP"; do
|
||||
name="${bin}_${VERSION}_${goos}_${goarch}"
|
||||
tarball="${dist}/${name}.tar.gz"
|
||||
if [[ ! -f "$tarball" ]]; then
|
||||
echo "missing ${tarball}: run make release first" >&2
|
||||
exit 1
|
||||
fi
|
||||
src="${workdir}/${name}"
|
||||
mkdir -p "$src"
|
||||
tar -xzf "$tarball" -C "$workdir" "${name}/${bin}"
|
||||
mv "${workdir}/${name}/${bin}" "${src}/binary"
|
||||
cp "$workdir/units/${bin}.service" "${src}/unit.service"
|
||||
cp "${licenses[$bin]}" "${src}/LICENSE"
|
||||
|
||||
render_config "$bin" "$goarch" "$src" "${license_names[$bin]}" "${descriptions[$bin]}" >"${src}/nfpm.yaml"
|
||||
if grep -q '@[A-Z_]*@' "${src}/nfpm.yaml"; then
|
||||
echo "unreplaced placeholder in the nfpm configuration of ${name}" >&2
|
||||
exit 1
|
||||
fi
|
||||
for packager in deb rpm; do
|
||||
echo "packaging ${bin} ${pkg_version} ${goarch} (${packager})"
|
||||
"$NFPM" package --config "${src}/nfpm.yaml" --packager "$packager" --target "$dist/" >/dev/null
|
||||
done
|
||||
done
|
||||
done
|
||||
|
||||
(
|
||||
cd "$dist"
|
||||
shopt -s nullglob
|
||||
sha256sum -- *.tar.gz *.deb *.rpm >SHA256SUMS
|
||||
)
|
||||
echo "artefacts in ${dist}:"
|
||||
cat "${dist}/SHA256SUMS"
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Fails if systemd unit templates, nfpm package files or the installer
|
||||
# contain hard-coded product or binary names (ADR 0018). Names belong in the
|
||||
# variables of the Makefile and the scripts; templates use placeholders.
|
||||
#
|
||||
# Usage: scripts/check-brand.sh [FILE...]
|
||||
# Without FILE the default set is checked: deploy/systemd/*.in,
|
||||
# deploy/nfpm/*, scripts/install.sh. Missing files are skipped.
|
||||
#
|
||||
# The installer may define the names once, in a block between the lines
|
||||
# # --- names (ADR 0018) ---
|
||||
# # --- end names ---
|
||||
# Each marker must appear exactly once, in this order; everything outside the
|
||||
# block is checked. The same applies to any checked file containing a marker.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
readonly BEGIN_MARKER="# --- names (ADR 0018) ---"
|
||||
readonly END_MARKER="# --- end names ---"
|
||||
# Literal names that must not appear (case-insensitive).
|
||||
readonly PATTERN='ddi-suite|ddi-server|ddi-agent-dns|ddi-agent-dhcp'
|
||||
|
||||
cd "$(git rev-parse --show-toplevel)"
|
||||
|
||||
files=("$@")
|
||||
if ((${#files[@]} == 0)); then
|
||||
shopt -s nullglob
|
||||
files=(deploy/systemd/*.in deploy/nfpm/* scripts/install.sh)
|
||||
shopt -u nullglob
|
||||
fi
|
||||
|
||||
failed=0
|
||||
checked=0
|
||||
for f in "${files[@]}"; do
|
||||
[[ -f "$f" ]] || continue
|
||||
checked=$((checked + 1))
|
||||
begins="$(grep -cxF -- "$BEGIN_MARKER" "$f" || true)"
|
||||
ends="$(grep -cxF -- "$END_MARKER" "$f" || true)"
|
||||
if [[ "$f" == scripts/install.sh ]] || ((begins + ends > 0)); then
|
||||
if ((begins != 1 || ends != 1)); then
|
||||
echo "$f: expected exactly one '${BEGIN_MARKER}' and one '${END_MARKER}' line" >&2
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
# "line:text" for every offending line outside the names block.
|
||||
hits="$(awk -v b="$BEGIN_MARKER" -v e="$END_MARKER" -v pat="$PATTERN" '
|
||||
$0 == b { inside = 1; next }
|
||||
$0 == e { if (!inside) print NR ":end marker before begin marker"; inside = 0; next }
|
||||
!inside && tolower($0) ~ pat { print NR ":" $0 }
|
||||
' "$f")"
|
||||
if [[ -n "$hits" ]]; then
|
||||
while IFS= read -r line; do
|
||||
echo "${f}:${line}" >&2
|
||||
done <<<"$hits"
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
|
||||
if ((failed)); then
|
||||
echo "hard-coded product or binary names found (ADR 0018): use the variables or placeholders" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "brand check passed (${checked} files)"
|
||||
+51
-21
@@ -2,9 +2,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Installs pinned, checksum-verified development tools on Linux (amd64/arm64)
|
||||
# for CI: golangci-lint, buf, shellcheck.
|
||||
# for CI: golangci-lint, buf, shellcheck, nfpm.
|
||||
#
|
||||
# Usage: scripts/ci/install-tools.sh [BINDIR] (default: $HOME/.local/bin)
|
||||
# Usage: scripts/ci/install-tools.sh [BINDIR] [TOOL...]
|
||||
# BINDIR defaults to $HOME/.local/bin; without TOOL all tools are installed.
|
||||
# In GitHub/Gitea Actions the directory is appended to $GITHUB_PATH.
|
||||
|
||||
set -euo pipefail
|
||||
@@ -12,25 +13,35 @@ set -euo pipefail
|
||||
readonly GOLANGCI_LINT_VERSION="2.14.0"
|
||||
readonly BUF_VERSION="1.73.0"
|
||||
readonly SHELLCHECK_VERSION="0.11.0"
|
||||
readonly NFPM_VERSION="2.48.0"
|
||||
|
||||
bindir="${1:-$HOME/.local/bin}"
|
||||
shift || true
|
||||
tools=("$@")
|
||||
if ((${#tools[@]} == 0)); then
|
||||
tools=(golangci-lint buf shellcheck nfpm)
|
||||
fi
|
||||
|
||||
case "$(uname -m)" in
|
||||
x86_64 | amd64)
|
||||
golangci_arch="amd64"
|
||||
buf_arch="x86_64"
|
||||
shellcheck_arch="x86_64"
|
||||
nfpm_arch="x86_64"
|
||||
golangci_sha="ab90aeb7b066f92a33415b638a50fe5344bbb75a0d32ad30cc248d88f81032ab"
|
||||
buf_sha="8f2986298ad08f0cc1bf999b9797b7c383adf32d7edf0f73d6f1e1a701baeac1"
|
||||
shellcheck_sha="b7af85e41cc99489dcc21d66c6d5f3685138f06d34651e6d34b42ec6d54fe6f6"
|
||||
nfpm_sha="6d7f9a28b2f397934403950fcbba83f598b08d02edbe79a0815b3b4897341750"
|
||||
;;
|
||||
aarch64 | arm64)
|
||||
golangci_arch="arm64"
|
||||
buf_arch="aarch64"
|
||||
shellcheck_arch="aarch64"
|
||||
nfpm_arch="arm64"
|
||||
golangci_sha="ee7ec5f3453d15ddf106fae5a4d6c71737712348a979d1fe9cd52ec7ea299bae"
|
||||
buf_sha="902b75267db7f4391e99b7fa0756050e5354234cc0437ef50eee9c788950c7a3"
|
||||
shellcheck_sha="68a8133197a50beb8803f8d42f9908d1af1c5540d4bb05fdfca8c1fa47decefc"
|
||||
nfpm_sha="a9a75213763783b7669719e505474c748c5ad7dc66591a195619deb50f85c2f4"
|
||||
;;
|
||||
*)
|
||||
echo "unsupported architecture: $(uname -m)" >&2
|
||||
@@ -48,26 +59,45 @@ fetch() {
|
||||
echo "$3 $2" | sha256sum -c --quiet -
|
||||
}
|
||||
|
||||
name="golangci-lint-${GOLANGCI_LINT_VERSION}-linux-${golangci_arch}"
|
||||
fetch "https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCI_LINT_VERSION}/${name}.tar.gz" \
|
||||
"$workdir/golangci.tar.gz" "$golangci_sha"
|
||||
tar -xzf "$workdir/golangci.tar.gz" -C "$workdir"
|
||||
install -m 0755 "$workdir/${name}/golangci-lint" "$bindir/golangci-lint"
|
||||
|
||||
fetch "https://github.com/bufbuild/buf/releases/download/v${BUF_VERSION}/buf-Linux-${buf_arch}" \
|
||||
"$workdir/buf" "$buf_sha"
|
||||
install -m 0755 "$workdir/buf" "$bindir/buf"
|
||||
|
||||
name="shellcheck-v${SHELLCHECK_VERSION}"
|
||||
fetch "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/${name}.linux.${shellcheck_arch}.tar.gz" \
|
||||
"$workdir/shellcheck.tar.gz" "$shellcheck_sha"
|
||||
tar -xzf "$workdir/shellcheck.tar.gz" -C "$workdir"
|
||||
install -m 0755 "$workdir/${name}/shellcheck" "$bindir/shellcheck"
|
||||
for tool in "${tools[@]}"; do
|
||||
case "$tool" in
|
||||
golangci-lint)
|
||||
name="golangci-lint-${GOLANGCI_LINT_VERSION}-linux-${golangci_arch}"
|
||||
fetch "https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCI_LINT_VERSION}/${name}.tar.gz" \
|
||||
"$workdir/golangci.tar.gz" "$golangci_sha"
|
||||
tar -xzf "$workdir/golangci.tar.gz" -C "$workdir"
|
||||
install -m 0755 "$workdir/${name}/golangci-lint" "$bindir/golangci-lint"
|
||||
"$bindir/golangci-lint" version
|
||||
;;
|
||||
buf)
|
||||
fetch "https://github.com/bufbuild/buf/releases/download/v${BUF_VERSION}/buf-Linux-${buf_arch}" \
|
||||
"$workdir/buf" "$buf_sha"
|
||||
install -m 0755 "$workdir/buf" "$bindir/buf"
|
||||
"$bindir/buf" --version
|
||||
;;
|
||||
shellcheck)
|
||||
name="shellcheck-v${SHELLCHECK_VERSION}"
|
||||
fetch "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/${name}.linux.${shellcheck_arch}.tar.gz" \
|
||||
"$workdir/shellcheck.tar.gz" "$shellcheck_sha"
|
||||
tar -xzf "$workdir/shellcheck.tar.gz" -C "$workdir"
|
||||
install -m 0755 "$workdir/${name}/shellcheck" "$bindir/shellcheck"
|
||||
"$bindir/shellcheck" --version | head -n 2
|
||||
;;
|
||||
nfpm)
|
||||
fetch "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_${nfpm_arch}.tar.gz" \
|
||||
"$workdir/nfpm.tar.gz" "$nfpm_sha"
|
||||
tar -xzf "$workdir/nfpm.tar.gz" -C "$workdir" nfpm
|
||||
install -m 0755 "$workdir/nfpm" "$bindir/nfpm"
|
||||
"$bindir/nfpm" --version >"$workdir/nfpm.version"
|
||||
grep GitVersion "$workdir/nfpm.version"
|
||||
;;
|
||||
*)
|
||||
echo "unknown tool: $tool" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -n "${GITHUB_PATH:-}" ]]; then
|
||||
echo "$bindir" >>"$GITHUB_PATH"
|
||||
fi
|
||||
|
||||
"$bindir/golangci-lint" version
|
||||
"$bindir/buf" --version
|
||||
"$bindir/shellcheck" --version | head -n 2
|
||||
|
||||
Executable
+131
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Signs and verifies release checksums with Ed25519 (ADR 0024, docs/RELEASE.md).
|
||||
# The signature is the raw 64-byte output of
|
||||
# `openssl pkeyutl -sign -rawin` over SHA256SUMS, stored as SHA256SUMS.sig.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/sign-release.sh sign SUMS [KEY_FILE]
|
||||
# writes SUMS.sig; without KEY_FILE the PEM private key is taken from
|
||||
# the environment variable RELEASE_SIGNING_KEY (it is written to a
|
||||
# private temporary file and removed again, never printed)
|
||||
# scripts/sign-release.sh verify SUMS SIG PUBLIC_KEY_FILE
|
||||
# exits 0 only if SIG is a valid signature of SUMS
|
||||
# scripts/sign-release.sh public-key KEY_FILE
|
||||
# prints the PEM public key that belongs to a private key
|
||||
# scripts/sign-release.sh selftest
|
||||
# signs and verifies with a throwaway key (CI); a tampered file and a
|
||||
# foreign key must fail
|
||||
#
|
||||
# Requires OpenSSL 3.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
sed -n '4,21p' "$0" | sed 's/^# \{0,1\}//'
|
||||
}
|
||||
|
||||
die() {
|
||||
echo "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
command -v openssl >/dev/null || die "openssl is required"
|
||||
|
||||
# sign SUMS KEY_FILE
|
||||
sign_with_file() {
|
||||
local sums="$1" key="$2"
|
||||
[[ -f "$sums" ]] || die "no such file: $sums"
|
||||
[[ -f "$key" ]] || die "no such key file: $key"
|
||||
openssl pkeyutl -sign -rawin -inkey "$key" -in "$sums" -out "${sums}.sig"
|
||||
[[ "$(wc -c <"${sums}.sig")" -eq 64 ]] || die "unexpected signature size (is the key an Ed25519 key?)"
|
||||
echo "signed ${sums} -> ${sums}.sig"
|
||||
}
|
||||
|
||||
cmd_sign() {
|
||||
(($# >= 1 && $# <= 2)) || {
|
||||
usage >&2
|
||||
exit 2
|
||||
}
|
||||
if (($# == 2)); then
|
||||
sign_with_file "$1" "$2"
|
||||
return
|
||||
fi
|
||||
[[ -n "${RELEASE_SIGNING_KEY:-}" ]] || die "RELEASE_SIGNING_KEY is not set and no key file was given"
|
||||
local dir
|
||||
dir="$(umask 077 && mktemp -d)"
|
||||
# shellcheck disable=SC2064 # expand now: $dir is local
|
||||
trap "rm -rf '$dir'" EXIT
|
||||
(umask 077 && printf '%s\n' "$RELEASE_SIGNING_KEY" >"$dir/key.pem")
|
||||
sign_with_file "$1" "$dir/key.pem"
|
||||
rm -rf "$dir"
|
||||
trap - EXIT
|
||||
}
|
||||
|
||||
cmd_verify() {
|
||||
(($# == 3)) || {
|
||||
usage >&2
|
||||
exit 2
|
||||
}
|
||||
local sums="$1" sig="$2" pub="$3"
|
||||
for f in "$sums" "$sig" "$pub"; do
|
||||
[[ -f "$f" ]] || die "no such file: $f"
|
||||
done
|
||||
if openssl pkeyutl -verify -pubin -inkey "$pub" -rawin -in "$sums" -sigfile "$sig" >/dev/null 2>&1; then
|
||||
echo "signature of ${sums} is valid"
|
||||
else
|
||||
die "signature of ${sums} is NOT valid"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_public_key() {
|
||||
(($# == 1)) || {
|
||||
usage >&2
|
||||
exit 2
|
||||
}
|
||||
openssl pkey -in "$1" -pubout
|
||||
}
|
||||
|
||||
cmd_selftest() {
|
||||
local dir
|
||||
dir="$(umask 077 && mktemp -d)"
|
||||
# shellcheck disable=SC2064 # expand now: $dir is local
|
||||
trap "rm -rf '$dir'" EXIT
|
||||
openssl genpkey -algorithm ed25519 -out "$dir/key.pem" 2>/dev/null
|
||||
openssl pkey -in "$dir/key.pem" -pubout -out "$dir/pub.pem"
|
||||
openssl genpkey -algorithm ed25519 -out "$dir/other.pem" 2>/dev/null
|
||||
openssl pkey -in "$dir/other.pem" -pubout -out "$dir/other-pub.pem"
|
||||
printf '%s example_v0.0.0_linux_amd64.tar.gz\n' "$(printf 'x' | sha256sum | cut -d' ' -f1)" >"$dir/SHA256SUMS"
|
||||
|
||||
# Signing from the environment, as the release workflow does.
|
||||
RELEASE_SIGNING_KEY="$(cat "$dir/key.pem")" bash "$0" sign "$dir/SHA256SUMS" >/dev/null
|
||||
cmd_verify "$dir/SHA256SUMS" "$dir/SHA256SUMS.sig" "$dir/pub.pem" >/dev/null ||
|
||||
die "selftest: valid signature rejected"
|
||||
if bash "$0" verify "$dir/SHA256SUMS" "$dir/SHA256SUMS.sig" "$dir/other-pub.pem" >/dev/null 2>&1; then
|
||||
die "selftest: signature accepted with a foreign key"
|
||||
fi
|
||||
printf 'tampered\n' >>"$dir/SHA256SUMS"
|
||||
if bash "$0" verify "$dir/SHA256SUMS" "$dir/SHA256SUMS.sig" "$dir/pub.pem" >/dev/null 2>&1; then
|
||||
die "selftest: tampered file accepted"
|
||||
fi
|
||||
echo "selftest passed: sign, verify, foreign key and tampered file rejected"
|
||||
}
|
||||
|
||||
(($# >= 1)) || {
|
||||
usage >&2
|
||||
exit 2
|
||||
}
|
||||
cmd="$1"
|
||||
shift
|
||||
case "$cmd" in
|
||||
sign) cmd_sign "$@" ;;
|
||||
verify) cmd_verify "$@" ;;
|
||||
public-key) cmd_public_key "$@" ;;
|
||||
selftest) cmd_selftest "$@" ;;
|
||||
-h | --help | help) usage ;;
|
||||
*)
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,171 @@
|
||||
// SPDX-License-Identifier: LicenseRef-DDI-Proprietary
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.beugsiud.de/beug.k/DDI/internal/brand"
|
||||
"git.beugsiud.de/beug.k/DDI/server/internal/app"
|
||||
"git.beugsiud.de/beug.k/DDI/server/internal/config"
|
||||
"git.beugsiud.de/beug.k/DDI/server/internal/service"
|
||||
)
|
||||
|
||||
// Exit codes of `license set-key` besides 0 (stored) and 1 (other errors,
|
||||
// e.g. configuration or database). The installer uses them to decide
|
||||
// whether to warn or to fail.
|
||||
const (
|
||||
exitUsage = 2 // bad arguments, empty or oversized input
|
||||
exitKeyRejected = 3 // the license service rejected the key
|
||||
exitLicenseSvc = 4 // the license service is unreachable or untrusted
|
||||
)
|
||||
|
||||
// maxKeyInput bounds what is read from stdin; maxKeyLength matches the
|
||||
// limit of the license manager.
|
||||
const (
|
||||
maxKeyInput = 4096
|
||||
maxKeyLength = 512
|
||||
)
|
||||
|
||||
// exitError carries a specific exit code.
|
||||
type exitError struct {
|
||||
code int
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *exitError) Error() string { return e.err.Error() }
|
||||
func (e *exitError) Unwrap() error { return e.err }
|
||||
|
||||
func licenseUsage(w io.Writer) {
|
||||
_, _ = fmt.Fprintf(w, `Usage: %[1]s license set-key < keyfile
|
||||
|
||||
set-key read a license key from stdin, validate and activate it online
|
||||
(like PUT /api/v1/license) and print the resulting license state
|
||||
|
||||
Exit codes: 0 activated, 1 other error, 2 empty or oversized input,
|
||||
3 key rejected, 4 license server unreachable (nothing was changed).
|
||||
`, brand.ServerBinary)
|
||||
}
|
||||
|
||||
func cmdLicense(ctx context.Context, args []string, lookup config.LookupFunc, stdin io.Reader, stdout, logOut io.Writer) error {
|
||||
if len(args) == 0 || args[0] == "help" || args[0] == "-h" || args[0] == "--help" {
|
||||
if len(args) == 0 {
|
||||
licenseUsage(logOut)
|
||||
return &exitError{code: exitUsage, err: errors.New("license: missing subcommand")}
|
||||
}
|
||||
licenseUsage(stdout)
|
||||
return nil
|
||||
}
|
||||
switch args[0] {
|
||||
case "set-key":
|
||||
if len(args) > 1 {
|
||||
return &exitError{code: exitUsage, err: errors.New("license set-key takes no arguments; pass the key on stdin")}
|
||||
}
|
||||
return cmdLicenseSetKey(ctx, lookup, stdin, stdout, logOut)
|
||||
default:
|
||||
licenseUsage(logOut)
|
||||
return &exitError{code: exitUsage, err: fmt.Errorf("license: unknown subcommand %q", args[0])}
|
||||
}
|
||||
}
|
||||
|
||||
// readKey reads the license key from r: trimmed, not empty, bounded.
|
||||
func readKey(r io.Reader) (string, error) {
|
||||
data, err := io.ReadAll(io.LimitReader(r, maxKeyInput+1))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read license key from stdin: %w", err)
|
||||
}
|
||||
if len(data) > maxKeyInput {
|
||||
return "", &exitError{code: exitUsage, err: fmt.Errorf("input on stdin is larger than %d bytes; expected a single license key", maxKeyInput)}
|
||||
}
|
||||
key := string(bytes.TrimSpace(data))
|
||||
switch {
|
||||
case key == "":
|
||||
return "", &exitError{code: exitUsage, err: errors.New("no license key on stdin")}
|
||||
case len(key) > maxKeyLength:
|
||||
return "", &exitError{code: exitUsage, err: fmt.Errorf("license key is longer than %d characters", maxKeyLength)}
|
||||
case strings.ContainsAny(key, "\r\n"):
|
||||
return "", &exitError{code: exitUsage, err: errors.New("expected a single license key, got several lines")}
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func cmdLicenseSetKey(ctx context.Context, lookup config.LookupFunc, stdin io.Reader, stdout, logOut io.Writer) error {
|
||||
if f, ok := stdin.(*os.File); ok {
|
||||
if fi, err := f.Stat(); err == nil && fi.Mode()&os.ModeCharDevice != 0 {
|
||||
_, _ = fmt.Fprintln(logOut, "reading the license key from stdin (finish with Ctrl-D)")
|
||||
}
|
||||
}
|
||||
key, err := readKey(stdin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg, log, err := setup(lookup, logOut)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
core, err := app.Open(ctx, cfg, log, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer core.Close()
|
||||
st, err := core.Service.SetLicenseKey(ctx, service.SystemActor, key)
|
||||
if err != nil {
|
||||
return licenseCLIError(err)
|
||||
}
|
||||
return printLicense(stdout, st)
|
||||
}
|
||||
|
||||
// licenseCLIError maps service errors to messages and exit codes.
|
||||
func licenseCLIError(err error) error {
|
||||
var se *service.Error
|
||||
if !errors.As(err, &se) {
|
||||
return err
|
||||
}
|
||||
msg := se.Message
|
||||
if pc := se.Params["provider_code"]; pc != "" {
|
||||
msg += " (" + pc + ")"
|
||||
}
|
||||
switch se.Kind {
|
||||
case service.KindUnavailable:
|
||||
return &exitError{code: exitLicenseSvc, err: fmt.Errorf("license key not stored: %s [%s]", msg, se.Code)}
|
||||
case service.KindInvalid, service.KindConflict:
|
||||
return &exitError{code: exitKeyRejected, err: fmt.Errorf("license key rejected: %s [%s]", msg, se.Code)}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func printLicense(w io.Writer, st service.LicenseStatus) error {
|
||||
limit := func(v *int) string {
|
||||
if v == nil {
|
||||
return "unlimited"
|
||||
}
|
||||
return strconv.Itoa(*v)
|
||||
}
|
||||
day := func(t *time.Time) string {
|
||||
if t == nil {
|
||||
return "never"
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
features := strings.Join(st.Features, ", ")
|
||||
if features == "" {
|
||||
features = "-"
|
||||
}
|
||||
name := st.Name
|
||||
if st.LicenseID != "" {
|
||||
name = strings.TrimSpace(name + " (" + st.LicenseID + ")")
|
||||
}
|
||||
_, err := fmt.Fprintf(w, "status: %s\nreason: %s\nlicense: %s\nexpires: %s\n"+
|
||||
"max agents: %s\nmax addresses: %s (in use: %d)\nfeatures: %s\n",
|
||||
st.State, st.Reason, name, day(st.ExpiresAt), limit(st.Limits.MaxAgents), limit(st.Limits.MaxAddresses),
|
||||
st.Usage.Addresses, features)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
// SPDX-License-Identifier: LicenseRef-DDI-Proprietary
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
const testKeygenAccount = "9de54794-8e94-4250-83f6-87e3d5686a0b"
|
||||
|
||||
// fakeKeygen answers the requests of license activation with signed
|
||||
// responses: "GOOD-KEY" is valid (no machine activation needed), every
|
||||
// other key does not exist.
|
||||
func fakeKeygen(t *testing.T) (url string, pub ed25519.PublicKey) {
|
||||
t.Helper()
|
||||
pub, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
path := strings.TrimPrefix(r.URL.Path, "/v1/accounts/"+testKeygenAccount)
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
var resp any
|
||||
switch {
|
||||
case r.Method == http.MethodPost && path == "/licenses/actions/validate-key":
|
||||
var req struct {
|
||||
Meta struct {
|
||||
Key string `json:"key"`
|
||||
Nonce int64 `json:"nonce"`
|
||||
} `json:"meta"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &req)
|
||||
meta := map[string]any{"valid": false, "code": "NOT_FOUND", "detail": "does not exist", "nonce": req.Meta.Nonce}
|
||||
var data any
|
||||
if req.Meta.Key == "GOOD-KEY" {
|
||||
meta["valid"], meta["code"], meta["detail"] = true, "VALID", "valid"
|
||||
data = map[string]any{"id": "lic-1", "type": "licenses", "attributes": map[string]any{
|
||||
"name": "CLI Test", "expiry": "2030-01-01T00:00:00Z", "status": "ACTIVE",
|
||||
"metadata": map[string]any{"maxAgents": 5},
|
||||
}}
|
||||
}
|
||||
resp = map[string]any{"meta": meta, "data": data}
|
||||
case r.Method == http.MethodGet && path == "/licenses/lic-1/entitlements":
|
||||
resp = map[string]any{"data": []any{map[string]any{"attributes": map[string]any{"code": "HA"}}}}
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
out, _ := json.Marshal(resp)
|
||||
sum := sha256.Sum256(out)
|
||||
digest := "sha-256=" + base64.StdEncoding.EncodeToString(sum[:])
|
||||
date := time.Now().UTC().Format(http.TimeFormat)
|
||||
signing := "(request-target): " + strings.ToLower(r.Method) + " " + r.URL.RequestURI() + "\nhost: " + r.Host +
|
||||
"\ndate: " + date + "\ndigest: " + digest
|
||||
sig := ed25519.Sign(priv, []byte(signing))
|
||||
w.Header().Set("Date", date)
|
||||
w.Header().Set("Digest", digest)
|
||||
w.Header().Set("Keygen-Signature", `keyid="`+testKeygenAccount+`", algorithm="ed25519", signature="`+
|
||||
base64.StdEncoding.EncodeToString(sig)+`", headers="(request-target) host date digest"`)
|
||||
w.Header().Set("Content-Type", "application/vnd.api+json")
|
||||
_, _ = w.Write(out)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv.URL, pub
|
||||
}
|
||||
|
||||
func TestReadKey(t *testing.T) {
|
||||
if k, err := readKey(strings.NewReader(" ABCD-1234 \n")); err != nil || k != "ABCD-1234" {
|
||||
t.Errorf("trimmed key: %q %v", k, err)
|
||||
}
|
||||
for name, in := range map[string]string{
|
||||
"empty": "",
|
||||
"blank": " \n\t\n",
|
||||
"too long": strings.Repeat("A", maxKeyLength+1),
|
||||
"oversized": strings.Repeat("A", maxKeyInput+1),
|
||||
"two lines": "KEY-1\nKEY-2\n",
|
||||
"blank line": "KEY-1\r\nKEY-2",
|
||||
} {
|
||||
_, err := readKey(strings.NewReader(in))
|
||||
ee, ok := err.(*exitError) //nolint:errorlint // readKey returns it unwrapped
|
||||
if !ok || ee.code != exitUsage {
|
||||
t.Errorf("%s: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLicenseCommandUsage(t *testing.T) {
|
||||
var out, errOut bytes.Buffer
|
||||
if code := run(context.Background(), []string{"license"}, lookupFrom(nil), nil, &out, &errOut); code != exitUsage {
|
||||
t.Errorf("without subcommand: %d", code)
|
||||
}
|
||||
if code := run(context.Background(), []string{"license", "frob"}, lookupFrom(nil), nil, &out, &errOut); code != exitUsage {
|
||||
t.Errorf("unknown subcommand: %d", code)
|
||||
}
|
||||
if code := run(context.Background(), []string{"license", "set-key", "KEY"}, lookupFrom(nil), nil, &out, &errOut); code != exitUsage ||
|
||||
!strings.Contains(errOut.String(), "stdin") {
|
||||
t.Errorf("key as argument: %d %q", code, errOut.String())
|
||||
}
|
||||
out.Reset()
|
||||
if code := run(context.Background(), []string{"license", "help"}, lookupFrom(nil), nil, &out, &errOut); code != 0 ||
|
||||
!strings.Contains(out.String(), "set-key") {
|
||||
t.Errorf("help: %d %q", code, out.String())
|
||||
}
|
||||
errOut.Reset()
|
||||
if code := run(context.Background(), []string{"license", "set-key"}, lookupFrom(nil), strings.NewReader("\n"), &out, &errOut); code != exitUsage ||
|
||||
!strings.Contains(errOut.String(), "no license key") {
|
||||
t.Errorf("empty stdin: %d %q", code, errOut.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLicenseSetKey(t *testing.T) {
|
||||
dbURL := testDatabase(t)
|
||||
dir := t.TempDir()
|
||||
keyFile := filepath.Join(dir, "master.key")
|
||||
if code := run(context.Background(), []string{"gen-master-key", "-out", keyFile}, lookupFrom(nil), nil, io.Discard, io.Discard); code != 0 {
|
||||
t.Fatal("gen-master-key failed")
|
||||
}
|
||||
keygenURL, pub := fakeKeygen(t)
|
||||
env := map[string]string{
|
||||
"DDI_DATABASE_URL": dbURL, "DDI_MASTER_KEY_FILE": keyFile, "DDI_STATE_DIR": dir, "DDI_LOG_LEVEL": "error",
|
||||
"KEYGEN_URL": keygenURL, "KEYGEN_ACCOUNT_ID": testKeygenAccount, "KEYGEN_PUBLIC_KEY": hex.EncodeToString(pub),
|
||||
}
|
||||
setKey := func(key string, env map[string]string) (int, string, string) {
|
||||
var out, errOut bytes.Buffer
|
||||
code := run(context.Background(), []string{"license", "set-key"}, lookupFrom(env), strings.NewReader(key), &out, &errOut)
|
||||
return code, out.String(), errOut.String()
|
||||
}
|
||||
|
||||
code, out, errOut := setKey("UNKNOWN-KEY\n", env)
|
||||
if code != exitKeyRejected || !strings.Contains(errOut, "license_invalid") || !strings.Contains(errOut, "NOT_FOUND") {
|
||||
t.Errorf("unknown key: %d %q", code, errOut)
|
||||
}
|
||||
if strings.Contains(out+errOut, "UNKNOWN-KEY") {
|
||||
t.Error("the key must not be printed")
|
||||
}
|
||||
|
||||
unreachable := map[string]string{}
|
||||
for k, v := range env {
|
||||
unreachable[k] = v
|
||||
}
|
||||
unreachable["KEYGEN_URL"] = "http://127.0.0.1:1"
|
||||
if code, _, errOut := setKey("GOOD-KEY", unreachable); code != exitLicenseSvc || !strings.Contains(errOut, "license_server_unreachable") {
|
||||
t.Errorf("unreachable license server: %d %q", code, errOut)
|
||||
}
|
||||
|
||||
code, out, errOut = setKey(" GOOD-KEY\n", env)
|
||||
if code != 0 {
|
||||
t.Fatalf("set-key: %d %s", code, errOut)
|
||||
}
|
||||
for _, want := range []string{"status: valid", "CLI Test (lic-1)", "expires: 2030-01-01T00:00:00Z",
|
||||
"max agents: 5", "max addresses: unlimited", "features: HA"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("output lacks %q:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
if strings.Contains(out+errOut, "GOOD-KEY") {
|
||||
t.Error("the key must not be printed")
|
||||
}
|
||||
|
||||
conn, err := pgx.Connect(context.Background(), dbURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close(context.Background()) //nolint:errcheck // test
|
||||
var actor string
|
||||
var n int
|
||||
if err := conn.QueryRow(context.Background(),
|
||||
`SELECT min(actor_name), count(*) FROM audit_log WHERE action = 'license.set_key'`).Scan(&actor, &n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 || actor != "system" {
|
||||
t.Errorf("audit: %d entries by %q", n, actor)
|
||||
}
|
||||
}
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
code := run(ctx, os.Args[1:], os.LookupEnv, os.Stdout, os.Stderr)
|
||||
code := run(ctx, os.Args[1:], os.LookupEnv, os.Stdin, os.Stdout, os.Stderr)
|
||||
stop()
|
||||
os.Exit(code)
|
||||
}
|
||||
@@ -39,15 +39,16 @@ Commands:
|
||||
migrate apply database migrations and exit
|
||||
bootstrap-admin create the first administrator and print its one-time password
|
||||
gen-master-key write a new master key file (refuses to overwrite)
|
||||
license set-key read a license key from stdin and activate it (%[1]s license help)
|
||||
version print version information
|
||||
help show this help
|
||||
|
||||
Configuration: YAML file from $DDI_CONFIG_FILE (if set), overridden by
|
||||
Configuration: YAML file from $%[2]s (if set), overridden by
|
||||
environment variables; see docs/ARCHITECTURE.md.
|
||||
`, brand.ServerBinary)
|
||||
`, brand.ServerBinary, config.EnvName("CONFIG_FILE"))
|
||||
}
|
||||
|
||||
func run(ctx context.Context, args []string, lookup config.LookupFunc, stdout, stderr io.Writer) int {
|
||||
func run(ctx context.Context, args []string, lookup config.LookupFunc, stdin io.Reader, stdout, stderr io.Writer) int {
|
||||
cmd := "serve"
|
||||
if len(args) > 0 {
|
||||
cmd, args = args[0], args[1:]
|
||||
@@ -62,6 +63,8 @@ func run(ctx context.Context, args []string, lookup config.LookupFunc, stdout, s
|
||||
err = cmdBootstrapAdmin(ctx, args, lookup, stdout, stderr)
|
||||
case "gen-master-key":
|
||||
err = cmdGenMasterKey(args, lookup, stdout)
|
||||
case "license":
|
||||
err = cmdLicense(ctx, args, lookup, stdin, stdout, stderr)
|
||||
case "version", "--version":
|
||||
err = buildinfo.Print(stdout, brand.ServerBinary)
|
||||
case "help", "-h", "--help":
|
||||
@@ -73,6 +76,9 @@ func run(ctx context.Context, args []string, lookup config.LookupFunc, stdout, s
|
||||
}
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintf(stderr, "%s: %v\n", brand.ServerBinary, err)
|
||||
if ee, ok := errors.AsType[*exitError](err); ok {
|
||||
return ee.code
|
||||
}
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
|
||||
@@ -28,28 +28,28 @@ func lookupFrom(m map[string]string) func(string) (string, bool) {
|
||||
|
||||
func TestRunVersionAndHelp(t *testing.T) {
|
||||
var out, errOut bytes.Buffer
|
||||
if code := run(context.Background(), []string{"version"}, lookupFrom(nil), &out, &errOut); code != 0 {
|
||||
if code := run(context.Background(), []string{"version"}, lookupFrom(nil), nil, &out, &errOut); code != 0 {
|
||||
t.Fatalf("version exit code %d", code)
|
||||
}
|
||||
if !strings.HasPrefix(out.String(), "ddi-server ") {
|
||||
t.Errorf("version output %q", out.String())
|
||||
}
|
||||
out.Reset()
|
||||
if code := run(context.Background(), []string{"help"}, lookupFrom(nil), &out, &errOut); code != 0 || !strings.Contains(out.String(), "bootstrap-admin") {
|
||||
if code := run(context.Background(), []string{"help"}, lookupFrom(nil), nil, &out, &errOut); code != 0 || !strings.Contains(out.String(), "bootstrap-admin") {
|
||||
t.Errorf("help: code %d, output %q", code, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUnknownCommand(t *testing.T) {
|
||||
var out, errOut bytes.Buffer
|
||||
if code := run(context.Background(), []string{"frobnicate"}, lookupFrom(nil), &out, &errOut); code != 2 {
|
||||
if code := run(context.Background(), []string{"frobnicate"}, lookupFrom(nil), nil, &out, &errOut); code != 2 {
|
||||
t.Errorf("exit code %d, want 2", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunInvalidConfig(t *testing.T) {
|
||||
var errOut bytes.Buffer
|
||||
code := run(context.Background(), []string{"serve"}, lookupFrom(map[string]string{"DDI_HTTP_LISTEN": "bad"}), io.Discard, &errOut)
|
||||
code := run(context.Background(), []string{"serve"}, lookupFrom(map[string]string{"DDI_HTTP_LISTEN": "bad"}), nil, io.Discard, &errOut)
|
||||
if code != 1 || !strings.Contains(errOut.String(), "invalid configuration") {
|
||||
t.Errorf("code %d, stderr %q", code, errOut.String())
|
||||
}
|
||||
@@ -58,7 +58,7 @@ func TestRunInvalidConfig(t *testing.T) {
|
||||
func TestServeFailsWithoutMasterKey(t *testing.T) {
|
||||
var errOut bytes.Buffer
|
||||
env := map[string]string{"DDI_MASTER_KEY_FILE": filepath.Join(t.TempDir(), "missing.key"), "DDI_DATABASE_URL": "postgres://x"}
|
||||
if code := run(context.Background(), []string{"serve"}, lookupFrom(env), io.Discard, &errOut); code != 1 || !strings.Contains(errOut.String(), "master key") {
|
||||
if code := run(context.Background(), []string{"serve"}, lookupFrom(env), nil, io.Discard, &errOut); code != 1 || !strings.Contains(errOut.String(), "master key") {
|
||||
t.Errorf("code %d, stderr %q", code, errOut.String())
|
||||
}
|
||||
}
|
||||
@@ -66,14 +66,14 @@ func TestServeFailsWithoutMasterKey(t *testing.T) {
|
||||
func TestGenMasterKey(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "sub", "master.key")
|
||||
var out, errOut bytes.Buffer
|
||||
if code := run(context.Background(), []string{"gen-master-key", "-out", path}, lookupFrom(nil), &out, &errOut); code != 0 {
|
||||
if code := run(context.Background(), []string{"gen-master-key", "-out", path}, lookupFrom(nil), nil, &out, &errOut); code != 0 {
|
||||
t.Fatalf("code %d: %s", code, errOut.String())
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil || !strings.Contains(string(data), "k1 ") {
|
||||
t.Fatalf("key file: %q %v", data, err)
|
||||
}
|
||||
if code := run(context.Background(), []string{"gen-master-key", "-out", path}, lookupFrom(nil), &out, &errOut); code != 1 {
|
||||
if code := run(context.Background(), []string{"gen-master-key", "-out", path}, lookupFrom(nil), nil, &out, &errOut); code != 1 {
|
||||
t.Error("existing key file must not be overwritten")
|
||||
}
|
||||
}
|
||||
@@ -141,7 +141,7 @@ func TestServeEndToEnd(t *testing.T) {
|
||||
dbURL := testDatabase(t)
|
||||
dir := t.TempDir()
|
||||
keyFile := filepath.Join(dir, "master.key")
|
||||
if code := run(context.Background(), []string{"gen-master-key", "-out", keyFile}, lookupFrom(nil), io.Discard, io.Discard); code != 0 {
|
||||
if code := run(context.Background(), []string{"gen-master-key", "-out", keyFile}, lookupFrom(nil), nil, io.Discard, io.Discard); code != 0 {
|
||||
t.Fatal("gen-master-key failed")
|
||||
}
|
||||
addr := freeAddr(t)
|
||||
@@ -152,20 +152,20 @@ func TestServeEndToEnd(t *testing.T) {
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
if code := run(context.Background(), []string{"bootstrap-admin", "-username", "root"}, lookupFrom(env), &out, io.Discard); code != 0 {
|
||||
if code := run(context.Background(), []string{"bootstrap-admin", "-username", "root"}, lookupFrom(env), nil, &out, io.Discard); code != 0 {
|
||||
t.Fatalf("bootstrap-admin failed: %s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "username: root") || !strings.Contains(out.String(), "password: ") {
|
||||
t.Errorf("bootstrap output %q", out.String())
|
||||
}
|
||||
if code := run(context.Background(), []string{"bootstrap-admin"}, lookupFrom(env), io.Discard, io.Discard); code != 1 {
|
||||
if code := run(context.Background(), []string{"bootstrap-admin"}, lookupFrom(env), nil, io.Discard, io.Discard); code != 1 {
|
||||
t.Error("second bootstrap must fail")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan int, 1)
|
||||
var logs bytes.Buffer
|
||||
go func() { done <- run(ctx, []string{"serve"}, lookupFrom(env), io.Discard, &syncWriter{w: &logs}) }()
|
||||
go func() { done <- run(ctx, []string{"serve"}, lookupFrom(env), nil, io.Discard, &syncWriter{w: &logs}) }()
|
||||
|
||||
base := fmt.Sprintf("http://%s", addr)
|
||||
get := func(path string) (int, string) {
|
||||
|
||||
@@ -182,6 +182,7 @@ func Serve(ctx context.Context, core *Core) error {
|
||||
defer stopJobs()
|
||||
go core.License.Run(jobs)
|
||||
go cleanup(jobs, core)
|
||||
go retention(jobs, core, retentionStartDelay, retentionInterval)
|
||||
|
||||
log.Info("starting server", slog.String("version", info.Version), slog.String("commit", info.Commit),
|
||||
slog.String("listen", cfg.HTTP.Listen), slog.Bool("tls", cfg.TLSEnabled()),
|
||||
@@ -236,6 +237,43 @@ func Serve(ctx context.Context, core *Core) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// The retention cleanup runs shortly after the start (not during it, so
|
||||
// that a restart loop does not hammer the database) and then daily.
|
||||
const (
|
||||
retentionStartDelay = 5 * time.Minute
|
||||
retentionInterval = 24 * time.Hour
|
||||
)
|
||||
|
||||
// retention applies the configured retention policy (ADR 0024).
|
||||
func retention(ctx context.Context, core *Core, delay, interval time.Duration) {
|
||||
policy := service.RetentionPolicy{AuditDays: core.Config.Retention.AuditDays,
|
||||
RevisionsKeep: core.Config.Retention.RevisionsKeep}
|
||||
if !policy.Enabled() {
|
||||
return
|
||||
}
|
||||
log := core.Log.With(slog.String("component", "retention"))
|
||||
t := time.NewTimer(delay)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
res, err := core.Service.ApplyRetention(ctx, policy)
|
||||
switch {
|
||||
case err != nil:
|
||||
log.Warn("retention cleanup failed", slog.Any("error", err))
|
||||
case res.AuditDeleted > 0 || len(res.RevisionsDeleted) > 0:
|
||||
log.Info("retention cleanup", slog.Int64("audit_entries_deleted", res.AuditDeleted),
|
||||
slog.Int("revisions_deleted", len(res.RevisionsDeleted)))
|
||||
default:
|
||||
log.Debug("retention cleanup: nothing to delete")
|
||||
}
|
||||
t.Reset(interval)
|
||||
}
|
||||
}
|
||||
|
||||
// cleanup removes expired sessions and old enrollment tokens periodically.
|
||||
func cleanup(ctx context.Context, core *Core) {
|
||||
t := time.NewTicker(15 * time.Minute)
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
// SPDX-License-Identifier: LicenseRef-DDI-Proprietary
|
||||
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.beugsiud.de/beug.k/DDI/server/internal/config"
|
||||
"git.beugsiud.de/beug.k/DDI/server/internal/service"
|
||||
"git.beugsiud.de/beug.k/DDI/server/internal/store"
|
||||
"git.beugsiud.de/beug.k/DDI/server/internal/store/storetest"
|
||||
)
|
||||
|
||||
func TestRetentionJob(t *testing.T) {
|
||||
st := storetest.New(t)
|
||||
ctx := context.Background()
|
||||
if err := st.Q().InsertAudit(ctx, &store.AuditEntry{ActorName: "admin", Action: "old.one", ObjectType: "test"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.Pool().Exec(ctx, `UPDATE audit_log SET at = now() - interval '10 days'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
log := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
cfg := config.Defaults()
|
||||
core := &Core{Config: cfg, Store: st, Log: log, Service: service.New(service.Options{Store: st, Logger: log})}
|
||||
|
||||
// Disabled policy: returns at once.
|
||||
done := make(chan struct{})
|
||||
go func() { retention(ctx, core, 0, time.Hour); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("disabled retention job did not return")
|
||||
}
|
||||
|
||||
core.Config.Retention.AuditDays = 7
|
||||
jobCtx, cancel := context.WithCancel(ctx)
|
||||
done = make(chan struct{})
|
||||
go func() { retention(jobCtx, core, 10*time.Millisecond, time.Hour); close(done) }()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
entries, err := st.Q().ListAudit(ctx, store.AuditFilter{Action: "old.one"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("old audit entry was not deleted")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("retention job did not stop")
|
||||
}
|
||||
}
|
||||
@@ -71,15 +71,28 @@ func (d Duration) Std() time.Duration { return time.Duration(d) }
|
||||
|
||||
// Config is the server configuration.
|
||||
type Config struct {
|
||||
HTTP HTTP `yaml:"http"`
|
||||
AgentAPI AgentAPI `yaml:"agent_api"`
|
||||
Database Database `yaml:"database"`
|
||||
MasterKeyFile string `yaml:"master_key_file"`
|
||||
StateDir string `yaml:"state_dir"`
|
||||
Log Log `yaml:"log"`
|
||||
Session Session `yaml:"session"`
|
||||
License License `yaml:"license"`
|
||||
ShutdownTimeout Duration `yaml:"shutdown_timeout"`
|
||||
HTTP HTTP `yaml:"http"`
|
||||
AgentAPI AgentAPI `yaml:"agent_api"`
|
||||
Database Database `yaml:"database"`
|
||||
MasterKeyFile string `yaml:"master_key_file"`
|
||||
StateDir string `yaml:"state_dir"`
|
||||
Log Log `yaml:"log"`
|
||||
Session Session `yaml:"session"`
|
||||
License License `yaml:"license"`
|
||||
Retention Retention `yaml:"retention"`
|
||||
ShutdownTimeout Duration `yaml:"shutdown_timeout"`
|
||||
}
|
||||
|
||||
// Retention limits how long history is kept (ADR 0024). A daily cleanup
|
||||
// applies it; 0 keeps everything.
|
||||
type Retention struct {
|
||||
// AuditDays deletes audit entries older than this many days.
|
||||
AuditDays int `yaml:"audit_days"`
|
||||
// RevisionsKeep keeps the newest this many configuration revisions.
|
||||
// The latest revision, revisions an agent runs, falls back to or is
|
||||
// told to apply, and revisions with pending deployments are always
|
||||
// kept in addition.
|
||||
RevisionsKeep int `yaml:"revisions_keep"`
|
||||
}
|
||||
|
||||
// HTTP configures the web UI/API listener.
|
||||
@@ -289,6 +302,17 @@ func str(dst func(*Config) *string) func(*Config, string) error {
|
||||
return func(c *Config, v string) error { *dst(c) = v; return nil }
|
||||
}
|
||||
|
||||
func integer(dst func(*Config) *int) func(*Config, string) error {
|
||||
return func(c *Config, v string) error {
|
||||
n, err := strconv.Atoi(strings.TrimSpace(v))
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid number %q", v)
|
||||
}
|
||||
*dst(c) = n
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func dur(dst func(*Config) *Duration) func(*Config, string) error {
|
||||
return func(c *Config, v string) error {
|
||||
d, err := ParseDuration(v)
|
||||
@@ -326,6 +350,8 @@ var envBindings = []envBinding{
|
||||
{name: "SESSION_LIFETIME", set: dur(func(c *Config) *Duration { return &c.Session.Lifetime })},
|
||||
{name: "SESSION_IDLE_TIMEOUT", set: dur(func(c *Config) *Duration { return &c.Session.IdleTimeout })},
|
||||
{name: "LICENSE_GRACE_PERIOD", set: dur(func(c *Config) *Duration { return &c.License.GracePeriod })},
|
||||
{name: "RETENTION_AUDIT_DAYS", set: integer(func(c *Config) *int { return &c.Retention.AuditDays })},
|
||||
{name: "RETENTION_REVISIONS_KEEP", set: integer(func(c *Config) *int { return &c.Retention.RevisionsKeep })},
|
||||
// Names given by the project brief, accepted without prefix.
|
||||
{name: "KEYGEN_URL", raw: true, set: str(func(c *Config) *string { return &c.License.KeygenURL })},
|
||||
{name: "KEYGEN_ACCOUNT_ID", raw: true, set: str(func(c *Config) *string { return &c.License.AccountID })},
|
||||
@@ -389,6 +415,12 @@ func (c Config) Validate() error {
|
||||
if c.Database.MaxConns < 1 {
|
||||
errs = append(errs, errors.New("database.max_conns must be at least 1"))
|
||||
}
|
||||
if c.Retention.AuditDays < 0 || c.Retention.AuditDays > 36500 {
|
||||
errs = append(errs, errors.New("retention.audit_days must be between 0 (keep all) and 36500"))
|
||||
}
|
||||
if c.Retention.RevisionsKeep < 0 {
|
||||
errs = append(errs, errors.New("retention.revisions_keep must not be negative (0 keeps all)"))
|
||||
}
|
||||
errs = append(errs, c.AgentAPI.validate()...)
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
@@ -43,6 +43,9 @@ func TestDefaults(t *testing.T) {
|
||||
if !strings.HasSuffix(filepath.ToSlash(cfg.MasterKeyFile), "/master.key") {
|
||||
t.Errorf("master key default %q", cfg.MasterKeyFile)
|
||||
}
|
||||
if cfg.Retention != (Retention{}) {
|
||||
t.Errorf("retention must default to keeping everything: %+v", cfg.Retention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileAndEnvPrecedence(t *testing.T) {
|
||||
@@ -57,11 +60,15 @@ license:
|
||||
grace_period: 7d
|
||||
session:
|
||||
idle_timeout: 30m
|
||||
retention:
|
||||
audit_days: 365
|
||||
revisions_keep: 50
|
||||
`)
|
||||
cfg, err := Load(path, env(map[string]string{
|
||||
"DDI_DATABASE_URL": "postgres://env",
|
||||
"KEYGEN_PRODUCT_ID": "prod-1",
|
||||
"DDI_LOG_LEVEL": "debug",
|
||||
"DDI_DATABASE_URL": "postgres://env",
|
||||
"KEYGEN_PRODUCT_ID": "prod-1",
|
||||
"DDI_LOG_LEVEL": "debug",
|
||||
"DDI_RETENTION_REVISIONS_KEEP": "20",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -81,6 +88,9 @@ session:
|
||||
if cfg.Session.IdleTimeout.Std() != 30*time.Minute || cfg.Log.Level != "debug" {
|
||||
t.Errorf("session/log: %+v %+v", cfg.Session, cfg.Log)
|
||||
}
|
||||
if cfg.Retention.AuditDays != 365 || cfg.Retention.RevisionsKeep != 20 {
|
||||
t.Errorf("retention: %+v", cfg.Retention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatabaseURLFile(t *testing.T) {
|
||||
@@ -116,6 +126,9 @@ func TestValidation(t *testing.T) {
|
||||
"http keygen": {map[string]string{"KEYGEN_URL": "http://license.example.test"}, "keygen_url"},
|
||||
"relative public": {map[string]string{"DDI_PUBLIC_URL": "/ddi"}, "public_url"},
|
||||
"bad grace in days": {map[string]string{"DDI_LICENSE_GRACE_PERIOD": "xd"}, "LICENSE_GRACE_PERIOD"},
|
||||
"audit days text": {map[string]string{"DDI_RETENTION_AUDIT_DAYS": "1y"}, "RETENTION_AUDIT_DAYS"},
|
||||
"negative audit": {map[string]string{"DDI_RETENTION_AUDIT_DAYS": "-1"}, "retention.audit_days"},
|
||||
"negative keep": {map[string]string{"DDI_RETENTION_REVISIONS_KEEP": "-3"}, "retention.revisions_keep"},
|
||||
}
|
||||
for name, tt := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
|
||||
@@ -146,6 +146,16 @@ func TestBINDZoneTransfer(t *testing.T) {
|
||||
script := `set -e
|
||||
mkdir -p /var/cache/bind/p /var/cache/bind/s ` + secZoneDir + `/secondary
|
||||
named -g -c /etc/bind/primary.conf >/tmp/primary.log 2>&1 &
|
||||
# Start the secondary only when the primary serves its zones: a refresh
|
||||
# that fails first is retried only after the zone's retry interval.
|
||||
i=0
|
||||
until grep -q 'running' /tmp/primary.log; do
|
||||
if [ $i -ge 30 ]; then
|
||||
echo "--- primary did not start ---"; cat /tmp/primary.log; exit 1
|
||||
fi
|
||||
sleep 1
|
||||
i=$((i + 1))
|
||||
done
|
||||
named -g -c /etc/bind/secondary.conf >/tmp/secondary.log 2>&1 &
|
||||
i=0
|
||||
while [ $i -lt 60 ]; do
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
// SPDX-License-Identifier: LicenseRef-DDI-Proprietary
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.beugsiud.de/beug.k/DDI/server/internal/store"
|
||||
)
|
||||
|
||||
// RetentionPolicy says how much history to keep (ADR 0024); zero values
|
||||
// keep everything.
|
||||
type RetentionPolicy struct {
|
||||
AuditDays int
|
||||
RevisionsKeep int
|
||||
}
|
||||
|
||||
// RetentionResult reports what a cleanup removed.
|
||||
type RetentionResult struct {
|
||||
AuditDeleted int64 `json:"audit_deleted"`
|
||||
RevisionsDeleted []int64 `json:"revisions_deleted"`
|
||||
}
|
||||
|
||||
// Enabled reports whether the policy removes anything at all.
|
||||
func (p RetentionPolicy) Enabled() bool { return p.AuditDays > 0 || p.RevisionsKeep > 0 }
|
||||
|
||||
// ApplyRetention deletes audit entries older than AuditDays and revisions
|
||||
// beyond the newest RevisionsKeep (see store.PruneRevisions for the ones
|
||||
// that are always kept). It runs in one transaction and, if anything was
|
||||
// deleted, records that in the audit log. It works in license read-only
|
||||
// mode as well (housekeeping).
|
||||
func (s *Service) ApplyRetention(ctx context.Context, p RetentionPolicy) (RetentionResult, error) {
|
||||
res := RetentionResult{RevisionsDeleted: []int64{}}
|
||||
if !p.Enabled() {
|
||||
return res, nil
|
||||
}
|
||||
err := s.inTx(ctx, func(tx *store.Tx) error {
|
||||
if p.AuditDays > 0 {
|
||||
n, err := tx.DeleteAuditBefore(ctx, s.now().Add(-time.Duration(p.AuditDays)*24*time.Hour))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res.AuditDeleted = n
|
||||
}
|
||||
if p.RevisionsKeep > 0 {
|
||||
// Serialize with revision creation and pushes.
|
||||
if err := tx.LockRevisions(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
deleted, err := tx.PruneRevisions(ctx, p.RevisionsKeep)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if deleted != nil {
|
||||
res.RevisionsDeleted = deleted
|
||||
}
|
||||
}
|
||||
if res.AuditDeleted == 0 && len(res.RevisionsDeleted) == 0 {
|
||||
return nil
|
||||
}
|
||||
after := map[string]any{"audit_days": p.AuditDays, "revisions_keep": p.RevisionsKeep,
|
||||
"audit_deleted": res.AuditDeleted, "revisions_deleted": len(res.RevisionsDeleted)}
|
||||
if n := len(res.RevisionsDeleted); n > 0 {
|
||||
// The list can be long on the first run; the range is enough.
|
||||
after["revisions_oldest"], after["revisions_newest"] = res.RevisionsDeleted[0], res.RevisionsDeleted[n-1]
|
||||
}
|
||||
return s.audit(ctx, tx, SystemActor, "retention.cleanup", "system", "", "retention", nil, after)
|
||||
})
|
||||
if err != nil {
|
||||
return RetentionResult{RevisionsDeleted: []int64{}}, err
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
// SPDX-License-Identifier: LicenseRef-DDI-Proprietary
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.beugsiud.de/beug.k/DDI/server/internal/store"
|
||||
"git.beugsiud.de/beug.k/DDI/server/internal/store/storetest"
|
||||
)
|
||||
|
||||
func TestApplyRetention(t *testing.T) {
|
||||
st := storetest.New(t)
|
||||
ctx := context.Background()
|
||||
svc := New(Options{Store: st, Keyring: testKeyring(t)})
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
for n := int64(1); n <= 4; n++ {
|
||||
if err := st.Q().InsertRevision(ctx, &store.Revision{Number: n, CreatedAt: now, CreatedByName: "admin"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, action := range []string{"old.one", "new.one"} {
|
||||
if err := st.Q().InsertAudit(ctx, &store.AuditEntry{ActorName: "admin", Action: action, ObjectType: "test"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := st.Pool().Exec(ctx, `UPDATE audit_log SET at = now() - interval '100 days' WHERE action = 'old.one'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if res, err := svc.ApplyRetention(ctx, RetentionPolicy{}); err != nil || res.AuditDeleted != 0 || len(res.RevisionsDeleted) != 0 {
|
||||
t.Fatalf("disabled policy: %+v %v", res, err)
|
||||
}
|
||||
|
||||
policy := RetentionPolicy{AuditDays: 90, RevisionsKeep: 2}
|
||||
res, err := svc.ApplyRetention(ctx, policy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.AuditDeleted != 1 || !slices.Equal(res.RevisionsDeleted, []int64{1, 2}) {
|
||||
t.Errorf("result %+v", res)
|
||||
}
|
||||
entries, err := st.Q().ListAudit(ctx, store.AuditFilter{Action: "retention.cleanup"})
|
||||
if err != nil || len(entries) != 1 {
|
||||
t.Fatalf("cleanup audit entries %d %v", len(entries), err)
|
||||
}
|
||||
var after map[string]any
|
||||
if err := json.Unmarshal(entries[0].After, &after); err != nil || after["audit_deleted"] != 1.0 ||
|
||||
after["revisions_deleted"] != 2.0 || after["revisions_oldest"] != 1.0 || after["revisions_newest"] != 2.0 ||
|
||||
entries[0].ActorName != "system" {
|
||||
t.Errorf("audit entry %+v %s", entries[0], entries[0].After)
|
||||
}
|
||||
|
||||
// Nothing left to delete: no further audit entry.
|
||||
if res, err := svc.ApplyRetention(ctx, policy); err != nil || res.AuditDeleted != 0 || len(res.RevisionsDeleted) != 0 {
|
||||
t.Errorf("second run %+v %v", res, err)
|
||||
}
|
||||
if entries, _ := st.Q().ListAudit(ctx, store.AuditFilter{Action: "retention.cleanup"}); len(entries) != 1 {
|
||||
t.Errorf("second run wrote %d cleanup entries", len(entries))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// SPDX-License-Identifier: LicenseRef-DDI-Proprietary
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DeleteAuditBefore deletes audit entries written before t0 (retention,
|
||||
// ADR 0024) and returns how many were removed.
|
||||
func (t *Tx) DeleteAuditBefore(ctx context.Context, t0 time.Time) (int64, error) {
|
||||
tag, err := t.q.Exec(ctx, `DELETE FROM audit_log WHERE at < $1`, t0)
|
||||
if err != nil {
|
||||
return 0, mapErr(err)
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
// PruneRevisions deletes configuration revisions (with their files) that
|
||||
// are older than the newest keep revisions (retention, ADR 0024) and
|
||||
// returns the deleted numbers in ascending order. keep must be at least 1,
|
||||
// so the latest revision always stays. Never deleted are revisions that an
|
||||
// agent runs (applied_revision), falls back to (last_good_revision) or is
|
||||
// told to apply (target_revision), and revisions with pending deployments.
|
||||
// Deployment rows keep their revision number.
|
||||
//
|
||||
// The caller should hold LockRevisions so that no revision is created or
|
||||
// pushed concurrently.
|
||||
func (t *Tx) PruneRevisions(ctx context.Context, keep int) ([]int64, error) {
|
||||
if keep < 1 {
|
||||
keep = 1
|
||||
}
|
||||
rows, err := t.q.Query(ctx, `
|
||||
DELETE FROM config_revisions r
|
||||
WHERE r.number < (
|
||||
SELECT min(k.number) FROM (
|
||||
SELECT number FROM config_revisions ORDER BY number DESC LIMIT $1) k)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM agents a
|
||||
WHERE r.number IN (a.applied_revision, a.last_good_revision, a.target_revision))
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM deployments d WHERE d.revision = r.number AND d.outcome = 'pending')
|
||||
RETURNING r.number`, keep)
|
||||
if err != nil {
|
||||
return nil, mapErr(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []int64
|
||||
for rows.Next() {
|
||||
var n int64
|
||||
if err := rows.Scan(&n); err != nil {
|
||||
return nil, mapErr(err)
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, mapErr(err)
|
||||
}
|
||||
slices.Sort(out) // RETURNING has no defined order
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
// SPDX-License-Identifier: LicenseRef-DDI-Proprietary
|
||||
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.beugsiud.de/beug.k/DDI/server/internal/store"
|
||||
"git.beugsiud.de/beug.k/DDI/server/internal/store/storetest"
|
||||
)
|
||||
|
||||
func TestDeleteAuditBefore(t *testing.T) {
|
||||
st := storetest.New(t)
|
||||
ctx := context.Background()
|
||||
for _, action := range []string{"old.one", "old.two", "new.one"} {
|
||||
if err := st.Q().InsertAudit(ctx, &store.AuditEntry{ActorName: "admin", Action: action, ObjectType: "test"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := st.Pool().Exec(ctx, `UPDATE audit_log SET at = now() - interval '40 days' WHERE action LIKE 'old.%'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, err := st.Q().DeleteAuditBefore(ctx, time.Now().Add(-30*24*time.Hour))
|
||||
if err != nil || n != 2 {
|
||||
t.Fatalf("deleted %d, %v", n, err)
|
||||
}
|
||||
left, err := st.Q().ListAudit(ctx, store.AuditFilter{})
|
||||
if err != nil || len(left) != 1 || left[0].Action != "new.one" {
|
||||
t.Errorf("left %+v %v", left, err)
|
||||
}
|
||||
if n, err := st.Q().DeleteAuditBefore(ctx, time.Now().Add(-30*24*time.Hour)); err != nil || n != 0 {
|
||||
t.Errorf("second run deleted %d, %v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneRevisions(t *testing.T) {
|
||||
st := storetest.New(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
for n := int64(1); n <= 10; n++ {
|
||||
files := []store.RevisionFile{{Role: "dns", Path: "ddi.conf", Mode: 0o644, Content: []byte("rev")}}
|
||||
if err := st.Q().InsertRevision(ctx, &store.Revision{Number: n, CreatedAt: now, CreatedByName: "admin"}, files); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
newAgent(t, st, agentA, "dns", "ns1", "0a")
|
||||
newAgent(t, st, agentB, "dns", "ns2", "0b")
|
||||
// Agent A runs 3 and falls back to 2, agent B is told to apply 4, a
|
||||
// push of 5 to agent A is pending, revision 1 has a finished report.
|
||||
if _, err := st.Q().UpdateAgentStatus(ctx, agentA, store.AgentStatus{AppliedRevision: ptr(int64(3)),
|
||||
LastGoodRevision: ptr(int64(2)), SeenAt: now}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.Q().SetAgentTarget(ctx, agentB, 4); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requester := "admin"
|
||||
if err := st.Q().InsertPushDeployment(ctx, &store.Deployment{AgentID: agentA, Revision: 5, StartedAt: now,
|
||||
RequestedByName: &requester}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
finished := now.Add(time.Second)
|
||||
if _, err := st.Q().InsertDeployment(ctx, &store.Deployment{AgentID: agentB, Revision: 1, Trigger: "pull",
|
||||
Outcome: store.OutcomeSucceeded, StartedAt: now, FinishedAt: &finished}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var deleted []int64
|
||||
err := st.InTx(ctx, func(tx *store.Tx) error {
|
||||
if err := tx.LockRevisions(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
var err error
|
||||
deleted, err = tx.PruneRevisions(ctx, 3)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(deleted, []int64{1, 6, 7}) {
|
||||
t.Errorf("deleted %v, want [1 6 7]", deleted)
|
||||
}
|
||||
revs, err := st.Q().ListRevisions(ctx, 100, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var kept []int64
|
||||
for _, r := range revs {
|
||||
kept = append(kept, r.Number)
|
||||
}
|
||||
if !slices.Equal(kept, []int64{10, 9, 8, 5, 4, 3, 2}) {
|
||||
t.Errorf("kept %v", kept)
|
||||
}
|
||||
if files, err := st.Q().RevisionFiles(ctx, 1, ""); err != nil || len(files) != 0 {
|
||||
t.Errorf("files of a deleted revision: %d %v", len(files), err)
|
||||
}
|
||||
if files, err := st.Q().RevisionFiles(ctx, 8, ""); err != nil || len(files) != 1 {
|
||||
t.Errorf("files of a kept revision: %d %v", len(files), err)
|
||||
}
|
||||
var reports int
|
||||
if err := st.Pool().QueryRow(ctx, `SELECT count(*) FROM deployments WHERE revision = 1`).Scan(&reports); err != nil || reports != 1 {
|
||||
t.Errorf("deployment rows of deleted revisions stay: %d %v", reports, err)
|
||||
}
|
||||
|
||||
if again, err := st.Q().PruneRevisions(ctx, 3); err != nil || len(again) != 0 {
|
||||
t.Errorf("second run: %v %v", again, err)
|
||||
}
|
||||
// keep < 1 still keeps the latest revision.
|
||||
if last, err := st.Q().PruneRevisions(ctx, 0); err != nil || !slices.Equal(last, []int64{8, 9}) {
|
||||
t.Errorf("keep 0: %v %v", last, err)
|
||||
}
|
||||
if latest, err := st.Q().LatestRevision(ctx); err != nil || latest.Number != 10 {
|
||||
t.Errorf("latest %+v %v", latest, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user