diff --git a/agent/cmd/ddi-agent-dhcp/main.go b/agent/cmd/ddi-agent-dhcp/main.go index 3230491..018a0b0 100644 --- a/agent/cmd/ddi-agent-dhcp/main.go +++ b/agent/cmd/ddi-agent-dhcp/main.go @@ -4,6 +4,7 @@ package main import ( + "errors" "os" "git.beugsiud.de/beug.k/DDI/agent/core" @@ -11,5 +12,10 @@ import ( ) func main() { - os.Exit(core.RunCLI(brand.AgentDHCPBinary, core.KindDHCP, os.Args[1:], os.Stdout, os.Stderr)) + os.Exit(core.Main(brand.AgentDHCPBinary, core.KindDHCP, newRole, os.Args[1:], os.Stdout, os.Stderr)) +} + +// newRole is a placeholder until the DHCP role package is wired in. +func newRole(func(v any) error) (core.Role, error) { + return nil, errors.New("DHCP role not available in this build") } diff --git a/agent/cmd/ddi-agent-dns/main.go b/agent/cmd/ddi-agent-dns/main.go index a32150c..2f2c027 100644 --- a/agent/cmd/ddi-agent-dns/main.go +++ b/agent/cmd/ddi-agent-dns/main.go @@ -4,6 +4,7 @@ package main import ( + "errors" "os" "git.beugsiud.de/beug.k/DDI/agent/core" @@ -11,5 +12,10 @@ import ( ) func main() { - os.Exit(core.RunCLI(brand.AgentDNSBinary, core.KindDNS, os.Args[1:], os.Stdout, os.Stderr)) + os.Exit(core.Main(brand.AgentDNSBinary, core.KindDNS, newRole, os.Args[1:], os.Stdout, os.Stderr)) +} + +// newRole is a placeholder until the DNS role package is wired in. +func newRole(func(v any) error) (core.Role, error) { + return nil, errors.New("DNS role not available in this build") } diff --git a/agent/core/agent.go b/agent/core/agent.go new file mode 100644 index 0000000..58a5ed0 --- /dev/null +++ b/agent/core/agent.go @@ -0,0 +1,612 @@ +// SPDX-License-Identifier: Apache-2.0 + +package core + +import ( + "context" + "crypto/ed25519" + "errors" + "fmt" + "log/slog" + "reflect" + "slices" + "strings" + "sync" + "time" + "unicode/utf8" + + "git.beugsiud.de/beug.k/DDI/internal/bundle" +) + +// Timing defaults of the agent that are not configurable. +const ( + healthRetryInterval = time.Second + reportRetryInterval = 30 * time.Second + reportTimeout = 30 * time.Second + statusTimeout = 10 * time.Second + serviceVersionMaxAge = 10 * time.Minute + // maxMessageLen bounds the message of a deployment report. + maxMessageLen = 4096 +) + +// Agent drives a role: it applies the revisions announced by a Source and +// reports outcomes, health and leases. +type Agent struct { + kind Kind + cfg *Config + role Role + store *Store + agentID string + bundleKey ed25519.PublicKey + source Source + log *slog.Logger + + // Intervals, shortened by tests. + healthRetry time.Duration + reportRetry time.Duration + + // roleMu serializes calls into the role: only one apply at a time, and + // health checks for heartbeats are skipped while an apply runs. + roleMu sync.Mutex + + mu sync.Mutex + st *State + current *Revision // applied revision, nil if none or not loadable + health Health + healthAt time.Time + version string + versionAt time.Time + + qmu sync.Mutex + queue []applyRequest + wake chan struct{} + + flushCh chan struct{} + leaseCh chan struct{} +} + +// applyRequest asks for an apply of a revision. +type applyRequest struct { + revision uint64 + trigger Trigger + deploymentID string +} + +// NewAgent prepares an agent. The identity provides the agent ID and the +// bundle signing key. +func NewAgent(kind Kind, cfg *Config, role Role, store *Store, id *Identity, src Source, log *slog.Logger) (*Agent, error) { + if role.Kind() != kind { + return nil, fmt.Errorf("role %q does not match agent role %q", role.Kind(), kind) + } + st, err := store.LoadState() + if err != nil { + return nil, err + } + a := &Agent{ + kind: kind, cfg: cfg, role: role, store: store, agentID: id.AgentID, + bundleKey: id.BundleKey, source: src, log: log, st: st, + healthRetry: healthRetryInterval, reportRetry: reportRetryInterval, + wake: make(chan struct{}, 1), flushCh: make(chan struct{}, 1), leaseCh: make(chan struct{}, 1), + } + if st.Applied != 0 { + rev, err := store.LoadRevision(st.Applied) + if err != nil { + log.Warn("applied revision is not available locally", "revision", st.Applied, "error", err) + } else { + a.current = &rev + } + } + return a, nil +} + +// Run works until ctx is canceled. The managed service keeps running with +// its current configuration afterwards. +func (a *Agent) Run(ctx context.Context) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + a.prune() + var wg sync.WaitGroup + wg.Go(func() { a.applyLoop(ctx) }) + wg.Go(func() { a.reportLoop(ctx) }) + if ls, ok := a.role.(LeaseSource); ok { + wg.Go(func() { a.leaseLoop(ctx, ls) }) + } + err := a.source.Run(ctx, a) + stopped := ctx.Err() != nil + cancel() + wg.Wait() + if stopped { + return nil + } + return err +} + +// state returns a copy of the state (without the report queue). +func (a *Agent) state() State { + a.mu.Lock() + defer a.mu.Unlock() + st := *a.st + st.PendingReports = nil + return st +} + +// Connected implements SourceHandler. +func (a *Agent) Connected() { + notify(a.flushCh) + notify(a.leaseCh) +} + +// Desired implements SourceHandler. +func (a *Agent) Desired(revision uint64, autoApply bool) { + if revision == 0 { + return + } + if !autoApply { + a.log.Debug("desired revision not applied in manual mode", "revision", revision) + return + } + a.enqueue(applyRequest{revision: revision, trigger: TriggerPull}) +} + +// Push implements SourceHandler. +func (a *Agent) Push(revision uint64, deploymentID string) { + a.enqueue(applyRequest{revision: revision, trigger: TriggerPush, deploymentID: deploymentID}) +} + +// Status implements SourceHandler. It runs a health check unless an apply +// is in progress, in which case the last result is returned. +func (a *Agent) Status(ctx context.Context) AgentStatus { + if a.roleMu.TryLock() { + a.refreshStatus(ctx) + a.roleMu.Unlock() + } + a.mu.Lock() + defer a.mu.Unlock() + return AgentStatus{ + Applied: a.st.Applied, LastGood: a.st.LastGood, + Health: a.health, CheckedAt: a.healthAt, ServiceVersion: a.version, + } +} + +// refreshStatus runs a health check and, if stale, reads the service +// version. The caller holds roleMu. +func (a *Agent) refreshStatus(ctx context.Context) { + ctx, cancel := context.WithTimeout(ctx, statusTimeout) + defer cancel() + a.mu.Lock() + cur := a.current + needVersion := a.version == "" || time.Since(a.versionAt) > serviceVersionMaxAge + a.mu.Unlock() + h := a.role.Health(ctx, cur) + var version string + if needVersion { + v, err := a.role.ServiceVersion(ctx) + if err != nil { + a.log.Warn("cannot determine the service version", "error", err) + } + version = v + } + a.mu.Lock() + a.health, a.healthAt = h, time.Now() + if needVersion && version != "" { + a.version, a.versionAt = version, time.Now() + } + a.mu.Unlock() +} + +// enqueue adds a request. A pull is dropped if an equal or newer revision +// is already queued; a push replaces queued pulls up to its revision. +func (a *Agent) enqueue(r applyRequest) { + a.qmu.Lock() + if r.trigger == TriggerPull { + for _, q := range a.queue { + if q.revision >= r.revision { + a.qmu.Unlock() + return + } + } + } else { + a.queue = slices.DeleteFunc(a.queue, func(q applyRequest) bool { + return q.trigger == TriggerPull && q.revision <= r.revision + }) + } + a.queue = append(a.queue, r) + a.qmu.Unlock() + notify(a.wake) +} + +func (a *Agent) dequeue() (applyRequest, bool) { + a.qmu.Lock() + defer a.qmu.Unlock() + if len(a.queue) == 0 { + return applyRequest{}, false + } + r := a.queue[0] + a.queue = a.queue[1:] + return r, true +} + +// applyLoop processes requests one at a time. +func (a *Agent) applyLoop(ctx context.Context) { + for { + r, ok := a.dequeue() + if !ok { + select { + case <-ctx.Done(): + return + case <-a.wake: + continue + } + } + if ctx.Err() != nil { + return + } + a.handle(ctx, r) + } +} + +// handle decides whether a request leads to an apply. +func (a *Agent) handle(ctx context.Context, r applyRequest) { + st := a.state() + log := a.log.With("revision", r.revision, "trigger", r.trigger, "deployment_id", r.deploymentID) + switch { + case r.trigger == TriggerPull && r.revision <= st.Applied: + log.Debug("revision already applied or older", "applied", st.Applied) + return + case r.trigger == TriggerPull && r.revision <= st.LastUnsuccessful: + log.Debug("revision failed before; waiting for a newer revision or an explicit push", + "last_unsuccessful", st.LastUnsuccessful) + return + case r.revision == 0 || r.revision < st.Applied: + now := time.Now() + log.Warn("refusing to apply a revision older than the applied one", "applied", st.Applied) + a.finish(log, DeploymentReport{ + DeploymentID: r.deploymentID, Revision: r.revision, Trigger: r.trigger, Outcome: OutcomeRejected, + Message: fmt.Sprintf("revision %d is older than the applied revision %d", r.revision, st.Applied), + ActiveRevision: st.Applied, StartedAt: now, FinishedAt: now, + }, nil) + return + } + a.apply(ctx, r, st, log) +} + +// apply runs the pipeline of docs/adr/0021: fetch, verify, store, Validate, +// Activate, Health, rollback. Shutdown aborts it until Activate starts; from +// then on it is completed (bounded by the configured timeouts) so that the +// service is never left between two revisions. +func (a *Agent) apply(ctx context.Context, r applyRequest, st State, log *slog.Logger) { + started := time.Now() + report := DeploymentReport{ + DeploymentID: r.deploymentID, Revision: r.revision, Trigger: r.trigger, + ActiveRevision: st.Applied, StartedAt: started, + } + log.Info("applying revision", "applied", st.Applied, "last_good", st.LastGood) + + reject := func(msg string, err error) { + report.Outcome = OutcomeRejected + report.Message = fmt.Sprintf("%s: %v", msg, err) + log.Error("revision rejected", "error", report.Message) + a.finish(log, report, func(s *State) { s.LastUnsuccessful = max(s.LastUnsuccessful, r.revision) }) + } + + fctx, cancel := context.WithTimeout(ctx, a.cfg.ApplyTimeout) + b, err := a.source.FetchBundle(fctx, r.revision) + cancel() + if err != nil { + if ctx.Err() != nil { + return + } + log.Warn("cannot fetch bundle", "error", err) + if r.trigger == TriggerPush { + // Transport problems do not block later pulls of this revision. + report.Outcome = OutcomeRejected + report.Message = "fetch bundle: " + err.Error() + a.finish(log, report, nil) + } + return + } + minRev := st.Applied + 1 + if r.trigger == TriggerPush { + minRev = st.Applied // an explicit push may re-apply the current revision + } + m, contents, err := a.verify(b, r.revision, minRev) + if err != nil { + reject("bundle verification failed", err) + return + } + rev, err := a.store.StoreRevision(m, b.Manifest, b.Signature, contents) + if err != nil { + reject("store revision", err) + return + } + if ctx.Err() != nil { + log.Info("shutdown before validation; revision not applied") + return + } + + a.roleMu.Lock() + defer a.roleMu.Unlock() + rctx := context.WithoutCancel(ctx) + + vctx, cancel := context.WithTimeout(rctx, a.cfg.ApplyTimeout) + err = a.role.Validate(vctx, rev) + cancel() + if err != nil { + reject("validation failed", err) + return + } + if ctx.Err() != nil { + log.Info("shutdown before activation; revision not applied") + return + } + + a.mu.Lock() + prev := a.current + a.mu.Unlock() + actx, cancel := context.WithTimeout(rctx, a.cfg.ApplyTimeout) + err = a.role.Activate(actx, rev, prev) + cancel() + activated := err == nil + var reason string + if activated { + h := a.waitHealthy(rctx, &rev) + if h.State == HealthOK { + report.Outcome = OutcomeSucceeded + report.ActiveRevision = rev.Number + log.Info("revision applied") + a.finish(log, report, func(s *State) { + s.Applied, s.LastGood = rev.Number, rev.Number + a.current = &rev + a.health, a.healthAt = h, time.Now() + }) + return + } + reason = "health check failed: " + describeHealth(h) + } else { + reason = "activation failed: " + err.Error() + } + log.Error("revision unhealthy, rolling back", "error", reason, "last_good", st.LastGood) + a.rollback(rctx, log, report, rev, activated, reason, st) +} + +// rollback restores the last good revision after a failed activation. +func (a *Agent) rollback(ctx context.Context, log *slog.Logger, report DeploymentReport, rev Revision, activated bool, reason string, st State) { + // What is installed if the rollback cannot restore anything: the new + // revision if it was activated, otherwise (the role undid its own + // changes) the previous one. + installed := st.Applied + if activated { + installed = rev.Number + } + fail := func(msg string) { + report.Outcome = OutcomeFailed + report.Message = reason + "; " + msg + report.ActiveRevision = installed + log.Error("deployment failed", "error", report.Message) + a.finish(log, report, func(s *State) { + s.LastUnsuccessful = max(s.LastUnsuccessful, rev.Number) + if installed != s.Applied { + s.Applied = installed + a.current = &rev + } + }) + } + if st.LastGood == 0 || st.LastGood == rev.Number { + fail("no last good revision to roll back to") + return + } + good, err := a.store.LoadRevision(st.LastGood) + if err != nil { + fail(fmt.Sprintf("rollback to revision %d impossible: %v", st.LastGood, err)) + return + } + actx, cancel := context.WithTimeout(ctx, a.cfg.ApplyTimeout) + err = a.role.Activate(actx, good, &rev) + cancel() + if err != nil { + fail(fmt.Sprintf("rollback to revision %d failed: %v", good.Number, err)) + return + } + installed = good.Number + h := a.waitHealthy(ctx, &good) + if h.State != HealthOK { + report.Outcome = OutcomeFailed + report.Message = fmt.Sprintf("%s; revision %d restored but unhealthy: %s", reason, good.Number, describeHealth(h)) + } else { + report.Outcome = OutcomeRolledBack + report.Message = fmt.Sprintf("%s; rolled back to revision %d", reason, good.Number) + } + report.ActiveRevision = good.Number + log.Warn("deployment rolled back", "outcome", report.Outcome, "active_revision", good.Number) + a.finish(log, report, func(s *State) { + s.LastUnsuccessful = max(s.LastUnsuccessful, rev.Number) + s.Applied = good.Number + a.current = &good + a.health, a.healthAt = h, time.Now() + }) +} + +// verify checks a fetched bundle against the bundle signing key. +func (a *Agent) verify(b *Bundle, revision, minRev uint64) (*bundle.Manifest, map[string][]byte, error) { + contents := make(map[string][]byte, len(b.Artifacts)) + for _, art := range b.Artifacts { + if _, dup := contents[art.Path]; dup { + return nil, nil, fmt.Errorf("duplicate artifact %s", art.Path) + } + contents[art.Path] = art.Content + } + m, err := bundle.Verify(a.bundleKey, b.Manifest, b.Signature, contents, bundle.Expect{ + AgentID: a.agentID, Role: string(a.kind), MinRevision: minRev, + }) + if err != nil { + return nil, nil, err + } + if m.Revision != revision { + return nil, nil, fmt.Errorf("bundle holds revision %d instead of %d", m.Revision, revision) + } + for _, f := range m.Files { + if reservedArtifact(f.Path) { + return nil, nil, fmt.Errorf("artifact path %s is reserved", f.Path) + } + } + return m, contents, nil +} + +// waitHealthy calls Health every healthRetry until it reports OK or the +// health timeout has passed, and returns the last result. +func (a *Agent) waitHealthy(ctx context.Context, rev *Revision) Health { + ctx, cancel := context.WithTimeout(ctx, a.cfg.HealthTimeout) + defer cancel() + t := time.NewTimer(0) + defer t.Stop() + for { + h := a.role.Health(ctx, rev) + if h.State == HealthOK { + return h + } + t.Reset(a.healthRetry) + select { + case <-ctx.Done(): + return h + case <-t.C: + } + } +} + +// finish records the outcome: it updates the state (update runs under the +// state lock), queues the report, persists both and triggers the report +// delivery. +func (a *Agent) finish(log *slog.Logger, r DeploymentReport, update func(*State)) { + r.Message = truncate(r.Message, maxMessageLen) + if r.FinishedAt.IsZero() { + r.FinishedAt = time.Now() + } + a.mu.Lock() + if update != nil { + update(a.st) + } + last := r + a.st.LastResult = &last + a.st.PendingReports = append(a.st.PendingReports, r) + a.st.UpdatedAt = time.Now().UTC() + err := a.store.SaveState(a.st) + a.mu.Unlock() + if err != nil { + log.Error("cannot save state", "error", err) + } + a.prune() + notify(a.flushCh) +} + +// prune removes old revisions, keeping the applied and last good one. +func (a *Agent) prune() { + st := a.state() + if err := a.store.Prune(KeepRevisions, st.Applied, st.LastGood); err != nil { + a.log.Warn("cannot prune old revisions", "error", err) + } +} + +// reportLoop delivers queued deployment reports. +func (a *Agent) reportLoop(ctx context.Context) { + t := time.NewTicker(a.reportRetry) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-a.flushCh: + case <-t.C: + } + a.flushReports(ctx) + } +} + +// flushReports sends queued reports in order until one fails. +func (a *Agent) flushReports(ctx context.Context) { + for ctx.Err() == nil { + a.mu.Lock() + if len(a.st.PendingReports) == 0 { + a.mu.Unlock() + return + } + r := a.st.PendingReports[0] + a.mu.Unlock() + + rctx, cancel := context.WithTimeout(ctx, reportTimeout) + err := a.source.ReportDeployment(rctx, r) + cancel() + if err != nil { + a.log.Debug("cannot deliver deployment report; will retry", "revision", r.Revision, + "deployment_id", r.DeploymentID, "error", err) + return + } + a.log.Info("deployment reported", "revision", r.Revision, "deployment_id", r.DeploymentID, "outcome", r.Outcome) + a.mu.Lock() + if i := slices.IndexFunc(a.st.PendingReports, func(p DeploymentReport) bool { return reflect.DeepEqual(p, r) }); i >= 0 { + a.st.PendingReports = slices.Delete(a.st.PendingReports, i, i+1) + } + err = a.store.SaveState(a.st) + a.mu.Unlock() + if err != nil { + a.log.Error("cannot save state", "error", err) + } + } +} + +// describeHealth renders a non-OK health result for reports. +func describeHealth(h Health) string { + var b strings.Builder + b.WriteString(h.State.String()) + if h.Message != "" { + b.WriteString(": " + h.Message) + } + for _, c := range h.Checks { + if c.State == HealthOK { + continue + } + fmt.Fprintf(&b, "; %s: %s", c.Name, c.State) + if c.Message != "" { + b.WriteString(" (" + c.Message + ")") + } + } + return b.String() +} + +// String returns the lowercase name of the state. +func (s HealthState) String() string { + switch s { + case HealthOK: + return "ok" + case HealthDegraded: + return "degraded" + case HealthFailed: + return "failed" + default: + return "unknown" + } +} + +// truncate shortens s to at most n bytes without splitting a UTF-8 +// sequence. +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + const marker = " [truncated]" + cut := n - len(marker) + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + return s[:cut] + marker +} + +// notify performs a non-blocking send on a 1-buffered channel. +func notify(ch chan struct{}) { + select { + case ch <- struct{}{}: + default: + } +} + +// errRoleNotAvailable is returned by the placeholder role factory. +var errRoleNotAvailable = errors.New("role not available in this build") diff --git a/agent/core/agent_test.go b/agent/core/agent_test.go new file mode 100644 index 0000000..ab3810d --- /dev/null +++ b/agent/core/agent_test.go @@ -0,0 +1,548 @@ +// SPDX-License-Identifier: Apache-2.0 + +package core + +import ( + "context" + "errors" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" +) + +func TestApplySucceeded(t *testing.T) { + f := newFixture(t, KindDNS) + f.addBundle(t, 1) + f.pull(1) + + r := f.lastResult(t) + if r.Outcome != OutcomeSucceeded || r.Revision != 1 || r.ActiveRevision != 1 || r.Trigger != TriggerPull { + t.Fatalf("result = %+v", r) + } + if r.StartedAt.IsZero() || r.FinishedAt.Before(r.StartedAt) { + t.Errorf("timestamps = %v .. %v", r.StartedAt, r.FinishedAt) + } + st := f.agent.state() + if st.Applied != 1 || st.LastGood != 1 || st.LastUnsuccessful != 0 { + t.Errorf("state = %+v", st) + } + if got, want := f.role.callLog(), []string{"validate 1", "activate 1 prev none"}; !slices.Equal(got, want) { + t.Errorf("calls = %q, want %q", got, want) + } + // Stored revision: manifest, signature and artifacts. + dir := filepath.Join(f.store.Dir(), "revisions", "1") + for _, name := range []string{"manifest.json", "manifest.sig", "ddi-suite.conf", filepath.Join("zones", "db.example.com")} { + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + t.Errorf("stored revision lacks %s: %v", name, err) + } + } + // The report is persisted until it is delivered. + saved, err := f.store.LoadState() + if err != nil { + t.Fatal(err) + } + if len(saved.PendingReports) != 1 || saved.Applied != 1 { + t.Fatalf("saved state = %+v", saved) + } + f.agent.flushReports(context.Background()) + if rep := f.src.reported(); len(rep) != 1 || rep[0].Outcome != OutcomeSucceeded { + t.Fatalf("reports = %+v", rep) + } + if saved, _ := f.store.LoadState(); len(saved.PendingReports) != 0 { + t.Errorf("pending reports after delivery = %d", len(saved.PendingReports)) + } + + // Next revision: Activate gets the previous one. + f.addBundle(t, 2) + f.role.resetCalls() + f.pull(2) + if got, want := f.role.callLog(), []string{"validate 2", "activate 2 prev 1"}; !slices.Equal(got, want) { + t.Errorf("calls = %q, want %q", got, want) + } + if st := f.agent.state(); st.Applied != 2 || st.LastGood != 2 { + t.Errorf("state = %+v", st) + } +} + +func TestApplyRejectedOnValidate(t *testing.T) { + f := newFixture(t, KindDNS) + f.addBundle(t, 1) + f.addBundle(t, 2) + f.pull(1) + f.role.validateErr[2] = errors.New("zone example.com: bad SOA") + f.role.resetCalls() + f.pull(2) + + r := f.lastResult(t) + if r.Outcome != OutcomeRejected || r.ActiveRevision != 1 || !strings.Contains(r.Message, "bad SOA") { + t.Fatalf("result = %+v", r) + } + if got := f.role.callLog(); !slices.Equal(got, []string{"validate 2"}) { + t.Errorf("calls = %q", got) + } + st := f.agent.state() + if st.Applied != 1 || st.LastGood != 1 || st.LastUnsuccessful != 2 { + t.Errorf("state = %+v", st) + } + + // No automatic retry of the same revision. + f.role.resetCalls() + f.src.fetches = 0 + f.pull(2) + if len(f.role.callLog()) != 0 || f.src.fetches != 0 { + t.Errorf("rejected revision was retried: calls %q, fetches %d", f.role.callLog(), f.src.fetches) + } + + // An explicit push retries it. + delete(f.role.validateErr, 2) + f.push(2, "dep-7") + r = f.lastResult(t) + if r.Outcome != OutcomeSucceeded || r.DeploymentID != "dep-7" || r.Trigger != TriggerPush { + t.Fatalf("push result = %+v", r) + } + + // A newer revision is pulled again. + f.addBundle(t, 3) + f.pull(3) + if st := f.agent.state(); st.Applied != 3 { + t.Errorf("state = %+v", st) + } +} + +func TestRollbackOnActivateError(t *testing.T) { + f := newFixture(t, KindDNS) + f.addBundle(t, 1) + f.addBundle(t, 2) + f.pull(1) + f.role.activateErr[2] = errors.New("rndc reconfig failed") + f.role.resetCalls() + f.pull(2) + + r := f.lastResult(t) + if r.Outcome != OutcomeRolledBack || r.ActiveRevision != 1 || !strings.Contains(r.Message, "rndc reconfig failed") { + t.Fatalf("result = %+v", r) + } + want := []string{"validate 2", "activate 2 prev 1", "activate 1 prev 2"} + if got := f.role.callLog(); !slices.Equal(got, want) { + t.Errorf("calls = %q, want %q", got, want) + } + st := f.agent.state() + if st.Applied != 1 || st.LastGood != 1 || st.LastUnsuccessful != 2 { + t.Errorf("state = %+v", st) + } + // Not retried by a later poll. + f.role.resetCalls() + f.pull(2) + if len(f.role.callLog()) != 0 { + t.Errorf("rolled back revision retried: %q", f.role.callLog()) + } +} + +func TestRollbackOnHealthFailure(t *testing.T) { + f := newFixture(t, KindDNS) + f.addBundle(t, 1) + f.addBundle(t, 2) + f.pull(1) + f.role.unhealthy[2] = true + f.pull(2) + + r := f.lastResult(t) + if r.Outcome != OutcomeRolledBack || r.ActiveRevision != 1 { + t.Fatalf("result = %+v", r) + } + if !strings.Contains(r.Message, "serial mismatch") || !strings.Contains(r.Message, "rolled back to revision 1") { + t.Errorf("message = %q", r.Message) + } + if n := f.role.healthCalls[2]; n < 2 { + t.Errorf("health of revision 2 checked %d times, want retries", n) + } + if f.role.active != 1 { + t.Errorf("active = %d", f.role.active) + } + if st := f.agent.state(); st.Applied != 1 || st.LastGood != 1 { + t.Errorf("state = %+v", st) + } +} + +func TestFailedWithoutLastGood(t *testing.T) { + f := newFixture(t, KindDNS) + f.addBundle(t, 1) + f.role.unhealthy[1] = true + f.pull(1) + + r := f.lastResult(t) + if r.Outcome != OutcomeFailed || r.ActiveRevision != 1 || !strings.Contains(r.Message, "no last good revision") { + t.Fatalf("result = %+v", r) + } + st := f.agent.state() + if st.Applied != 1 || st.LastGood != 0 || st.LastUnsuccessful != 1 { + t.Errorf("state = %+v", st) + } + + // Activation error without last good: the previous state stays. + g := newFixture(t, KindDNS) + g.addBundle(t, 1) + g.role.activateErr[1] = errors.New("boom") + g.pull(1) + if r := g.lastResult(t); r.Outcome != OutcomeFailed || r.ActiveRevision != 0 { + t.Fatalf("result = %+v", r) + } + if st := g.agent.state(); st.Applied != 0 { + t.Errorf("state = %+v", st) + } +} + +func TestFailedWhenRollbackFails(t *testing.T) { + f := newFixture(t, KindDNS) + f.addBundle(t, 1) + f.addBundle(t, 2) + f.pull(1) + f.role.unhealthy[2] = true + f.role.activateErr[1] = errors.New("restore failed") + f.pull(2) + + r := f.lastResult(t) + if r.Outcome != OutcomeFailed || r.ActiveRevision != 2 || !strings.Contains(r.Message, "rollback to revision 1 failed: restore failed") { + t.Fatalf("result = %+v", r) + } + if st := f.agent.state(); st.Applied != 2 || st.LastGood != 1 { + t.Errorf("state = %+v", st) + } +} + +func TestFailedWhenRollbackUnhealthy(t *testing.T) { + f := newFixture(t, KindDNS) + f.addBundle(t, 1) + f.addBundle(t, 2) + f.pull(1) + f.role.unhealthy[1] = true + f.role.unhealthy[2] = true + f.pull(2) + + r := f.lastResult(t) + if r.Outcome != OutcomeFailed || r.ActiveRevision != 1 || !strings.Contains(r.Message, "restored but unhealthy") { + t.Fatalf("result = %+v", r) + } + if st := f.agent.state(); st.Applied != 1 || st.LastGood != 1 || st.LastUnsuccessful != 2 { + t.Errorf("state = %+v", st) + } +} + +func TestReplayProtection(t *testing.T) { + f := newFixture(t, KindDNS) + for rev := uint64(1); rev <= 3; rev++ { + f.addBundle(t, rev) + } + f.pull(3) + if st := f.agent.state(); st.Applied != 3 { + t.Fatalf("state = %+v", st) + } + + // Older revisions: a pull is ignored, a push is rejected. + f.role.resetCalls() + f.pull(2) + if len(f.role.callLog()) != 0 || len(f.agent.state().PendingReports) != 0 { + t.Errorf("older pull was processed") + } + f.push(2, "dep-old") + r := f.lastResult(t) + if r.Outcome != OutcomeRejected || r.Revision != 2 || !strings.Contains(r.Message, "older") || r.ActiveRevision != 3 { + t.Fatalf("result = %+v", r) + } + if len(f.role.callLog()) != 0 { + t.Errorf("role called for an older revision: %q", f.role.callLog()) + } + + // The server answers revision 4 with the (validly signed) bundle of 2. + f.src.bundles[4] = f.src.bundles[2] + f.pull(4) + if r := f.lastResult(t); r.Outcome != OutcomeRejected || !strings.Contains(r.Message, "older than 4") { + t.Fatalf("result = %+v", r) + } + + // An explicit push of the applied revision re-applies it. + f.push(3, "dep-again") + if r := f.lastResult(t); r.Outcome != OutcomeSucceeded || r.Revision != 3 { + t.Fatalf("result = %+v", r) + } + if st := f.agent.state(); st.Applied != 3 { + t.Errorf("state = %+v", st) + } +} + +func TestRejectedBundles(t *testing.T) { + cases := map[string]func(f *fixture) *Bundle{ + "bad signature": func(f *fixture) *Bundle { + b := signBundle(t, f.signer, testAgentID, KindDNS, 1) + b.Signature[0] ^= 1 + return b + }, + "other agent": func(f *fixture) *Bundle { + return signBundle(t, f.signer, "11111111-2222-4333-8444-555555555555", KindDNS, 1) + }, + "other role": func(f *fixture) *Bundle { return signBundle(t, f.signer, testAgentID, KindDHCP, 1) }, + "other key": func(*fixture) *Bundle { + _, other := newSigningKey(t) + return signBundle(t, other, testAgentID, KindDNS, 1) + }, + "tampered content": func(f *fixture) *Bundle { + b := signBundle(t, f.signer, testAgentID, KindDNS, 1) + b.Artifacts[0].Content = []byte("evil") + return b + }, + "duplicate artifact": func(f *fixture) *Bundle { + b := signBundle(t, f.signer, testAgentID, KindDNS, 1) + b.Artifacts = append(b.Artifacts, b.Artifacts[0]) + return b + }, + } + for name, mk := range cases { + t.Run(name, func(t *testing.T) { + f := newFixture(t, KindDNS) + f.src.bundles[1] = mk(f) + f.pull(1) + r := f.lastResult(t) + if r.Outcome != OutcomeRejected || !strings.Contains(r.Message, "verification failed") { + t.Fatalf("result = %+v", r) + } + if len(f.role.callLog()) != 0 { + t.Errorf("role called: %q", f.role.callLog()) + } + if revs, _ := f.store.ListRevisions(); len(revs) != 0 { + t.Errorf("rejected bundle stored: %v", revs) + } + }) + } +} + +func TestFetchError(t *testing.T) { + f := newFixture(t, KindDNS) + f.src.fetchErr = errors.New("unavailable") + f.pull(1) + if st := f.agent.state(); st.LastResult != nil || st.LastUnsuccessful != 0 { + t.Fatalf("pull fetch error recorded: %+v", st) + } + f.push(1, "dep-1") + r := f.lastResult(t) + if r.Outcome != OutcomeRejected || !strings.Contains(r.Message, "fetch bundle: unavailable") { + t.Fatalf("result = %+v", r) + } + // A transport problem does not block a later pull. + f.src.fetchErr = nil + f.addBundle(t, 1) + f.pull(1) + if st := f.agent.state(); st.Applied != 1 { + t.Errorf("state = %+v", st) + } +} + +func TestShutdownBeforeValidation(t *testing.T) { + f := newFixture(t, KindDNS) + f.addBundle(t, 1) + ctx, cancel := context.WithCancel(context.Background()) + f.src.onFetch = cancel + f.agent.handle(ctx, applyRequest{revision: 1, trigger: TriggerPull}) + if len(f.role.callLog()) != 0 { + t.Errorf("role called after shutdown: %q", f.role.callLog()) + } + if st := f.agent.state(); st.LastResult != nil || st.Applied != 0 { + t.Errorf("state = %+v", st) + } +} + +func TestReportQueueWhileServerDown(t *testing.T) { + f := newFixture(t, KindDNS) + f.src.setReportErr(errors.New("server down")) + f.addBundle(t, 1) + f.addBundle(t, 2) + f.pull(1) + f.role.validateErr[2] = errors.New("bad") + f.pull(2) + f.agent.flushReports(context.Background()) + if n := len(f.src.reported()); n != 0 { + t.Fatalf("%d reports delivered while down", n) + } + + // After a restart the queue is still there and is sent in order. + g := newFixtureWithStore(t, KindDNS, f.store, f.signer) + if st, _ := g.store.LoadState(); len(st.PendingReports) != 2 { + t.Fatalf("pending reports after restart = %d", len(st.PendingReports)) + } + g.agent.flushReports(context.Background()) + rep := g.src.reported() + if len(rep) != 2 || rep[0].Revision != 1 || rep[1].Revision != 2 || rep[1].Outcome != OutcomeRejected { + t.Fatalf("reports = %+v", rep) + } + if st, _ := g.store.LoadState(); len(st.PendingReports) != 0 { + t.Errorf("pending after delivery = %d", len(st.PendingReports)) + } +} + +func TestPendingReportsBounded(t *testing.T) { + f := newFixture(t, KindDNS) + f.src.setReportErr(errors.New("down")) + for i := 0; i < maxPendingReports+5; i++ { + f.push(1, "dep") // fetch fails: rejected report each time + } + st, err := f.store.LoadState() + if err != nil { + t.Fatal(err) + } + if len(st.PendingReports) != maxPendingReports { + t.Errorf("pending = %d, want %d", len(st.PendingReports), maxPendingReports) + } +} + +func TestPruneKeepsAppliedAndLastGood(t *testing.T) { + f := newFixture(t, KindDNS) + f.addBundle(t, 1) + f.pull(1) + for rev := uint64(2); rev <= 14; rev++ { + f.addBundle(t, rev) + f.role.validateErr[rev] = errors.New("bad") + f.push(rev, "") + } + revs, err := f.store.ListRevisions() + if err != nil { + t.Fatal(err) + } + if !slices.Contains(revs, 1) || len(revs) != KeepRevisions+1 || revs[len(revs)-1] != 14 { + t.Errorf("revisions = %v", revs) + } +} + +func TestEnqueue(t *testing.T) { + f := newFixture(t, KindDNS) + a := f.agent + a.enqueue(applyRequest{revision: 5, trigger: TriggerPull}) + a.enqueue(applyRequest{revision: 4, trigger: TriggerPull}) // dropped: 5 queued + a.enqueue(applyRequest{revision: 5, trigger: TriggerPull}) // dropped: duplicate + a.enqueue(applyRequest{revision: 6, trigger: TriggerPush, deploymentID: "d"}) + a.enqueue(applyRequest{revision: 7, trigger: TriggerPull}) + var got []applyRequest + for { + r, ok := a.dequeue() + if !ok { + break + } + got = append(got, r) + } + want := []applyRequest{{revision: 6, trigger: TriggerPush, deploymentID: "d"}, {revision: 7, trigger: TriggerPull}} + if !slices.Equal(got, want) { + t.Errorf("queue = %+v, want %+v", got, want) + } +} + +func TestDesiredAndPush(t *testing.T) { + f := newFixture(t, KindDNS) + f.agent.Desired(0, true) + f.agent.Desired(3, false) // manual mode + if _, ok := f.agent.dequeue(); ok { + t.Fatal("request queued for revision 0 or manual mode") + } + f.agent.Desired(3, true) + f.agent.Push(4, "dep") + r1, _ := f.agent.dequeue() + r2, _ := f.agent.dequeue() + if r1.revision != 3 || r1.trigger != TriggerPull || r2.revision != 4 || r2.deploymentID != "dep" { + t.Errorf("requests = %+v %+v", r1, r2) + } +} + +func TestRunAppliesAndReports(t *testing.T) { + f := newFixture(t, KindDHCP) + f.addBundle(t, 1) + f.role.setLeases([]Lease{{IP: mustAddr("10.0.0.5"), MAC: "aa:bb:cc:dd:ee:ff", State: LeaseActive}}) + f.src.run = func(ctx context.Context, h SourceHandler) error { + h.Connected() + h.Desired(1, true) + <-ctx.Done() + return ctx.Err() + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- f.agent.Run(ctx) }() + waitFor(t, "deployment report", func() bool { return len(f.src.reported()) == 1 }) + waitFor(t, "lease snapshot", func() bool { return len(f.src.leaseBatches()) > 0 }) + if b := f.src.leaseBatches()[0]; !b.FullSync || len(b.Leases) != 1 { + t.Errorf("first lease batch = %+v", b) + } + cancel() + select { + case err := <-done: + if err != nil { + t.Errorf("Run = %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Run did not return") + } +} + +func TestRunSourceError(t *testing.T) { + f := newFixture(t, KindDNS) + f.src.run = func(context.Context, SourceHandler) error { return errors.New("broken source") } + if err := f.agent.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "broken source") { + t.Errorf("Run = %v", err) + } +} + +func TestStatus(t *testing.T) { + f := newFixture(t, KindDNS) + f.addBundle(t, 1) + f.pull(1) + st := f.agent.Status(context.Background()) + if st.Applied != 1 || st.LastGood != 1 || st.Health.State != HealthOK || st.ServiceVersion != "Fake 1.0" || st.CheckedAt.IsZero() { + t.Errorf("status = %+v", st) + } + // While the role is busy the cached result is returned. + f.role.unhealthy[1] = true + f.agent.roleMu.Lock() + cached := f.agent.Status(context.Background()) + f.agent.roleMu.Unlock() + if cached.Health.State != HealthOK { + t.Errorf("cached status = %+v", cached) + } + if fresh := f.agent.Status(context.Background()); fresh.Health.State != HealthFailed { + t.Errorf("fresh status = %+v", fresh) + } +} + +func TestNewAgentLoadsAppliedRevision(t *testing.T) { + f := newFixture(t, KindDNS) + f.addBundle(t, 1) + f.pull(1) + g := newFixtureWithStore(t, KindDNS, f.store, f.signer) + if g.agent.current == nil || g.agent.current.Number != 1 || len(g.agent.current.Files) != 2 { + t.Fatalf("current = %+v", g.agent.current) + } + // Role mismatch. + id := &Identity{AgentID: testAgentID} + if _, err := NewAgent(KindDHCP, testConfig(f.store.Dir()), newFakeRole(KindDNS), f.store, id, f.src, testLogger()); err == nil { + t.Error("role mismatch accepted") + } +} + +func TestDescribeHealthAndTruncate(t *testing.T) { + h := Health{State: HealthDegraded, Message: "slow", Checks: []HealthCheck{ + {Name: "a", State: HealthOK}, {Name: "b", State: HealthFailed, Message: "down"}, {Name: "c", State: HealthUnknown}, + }} + if got, want := describeHealth(h), "degraded: slow; b: failed (down); c: unknown"; got != want { + t.Errorf("describeHealth = %q, want %q", got, want) + } + if got := truncate("short", 10); got != "short" { + t.Errorf("truncate = %q", got) + } + long := strings.Repeat("ä", 100) // 200 bytes + got := truncate(long, 50) + if len(got) > 50 || !strings.HasSuffix(got, "[truncated]") || !strings.HasPrefix(got, "ää") { + t.Errorf("truncate = %q (%d bytes)", got, len(got)) + } + for _, r := range strings.TrimSuffix(got, " [truncated]") { + if r != 'ä' { + t.Fatalf("truncate split a rune: %q", got) + } + } +} diff --git a/agent/core/cli.go b/agent/core/cli.go index 19bf7c6..d545793 100644 --- a/agent/core/cli.go +++ b/agent/core/cli.go @@ -1,24 +1,37 @@ // SPDX-License-Identifier: Apache-2.0 // Package core is the role-independent part of the agents: command line, -// enrollment, configuration source, verification and apply pipeline -// (see docs/ARCHITECTURE.md, section 4). M0 provides the command line frame; -// enrollment and the apply pipeline follow in M3. +// configuration, local state, enrollment, the connection to the management +// server and the apply pipeline (see docs/ARCHITECTURE.md, section 4, and +// docs/adr/0021-agent-strecke-m3.md). The roles (BIND, Kea) implement Role. package core import ( + "context" + "errors" + "flag" "fmt" "io" + "log/slog" + "os" + "os/signal" + "strings" + "syscall" + "time" + "git.beugsiud.de/beug.k/DDI/internal/agentpki" "git.beugsiud.de/beug.k/DDI/internal/buildinfo" + "git.beugsiud.de/beug.k/DDI/internal/logging" ) -// RunCLI dispatches the agent command line and returns the process exit code. -func RunCLI(binary string, role Kind, args []string, stdout, stderr io.Writer) int { +// Main dispatches the agent command line and returns the process exit code. +// factory creates the role for the run command. +func Main(binary string, kind Kind, factory RoleFactory, args []string, stdout, stderr io.Writer) int { if len(args) == 0 { - usage(stderr, binary, role) + usage(stderr, binary, kind) return 2 } + c := &cli{binary: binary, kind: kind, factory: factory, stdout: stdout, stderr: stderr} switch args[0] { case "version", "--version": if err := buildinfo.Print(stdout, binary); err != nil { @@ -26,22 +39,269 @@ func RunCLI(binary string, role Kind, args []string, stdout, stderr io.Writer) i } return 0 case "help", "-h", "--help": - usage(stdout, binary, role) + usage(stdout, binary, kind) return 0 + case "enroll": + return c.enroll(args[1:]) + case "run": + return c.run(args[1:]) + case "status": + return c.status(args[1:]) default: _, _ = fmt.Fprintf(stderr, "%s: unknown command %q\n\n", binary, args[0]) - usage(stderr, binary, role) + usage(stderr, binary, kind) return 2 } } -func usage(w io.Writer, binary string, role Kind) { - _, _ = fmt.Fprintf(w, `Usage: %s +func usage(w io.Writer, binary string, kind Kind) { + _, _ = fmt.Fprintf(w, `Usage: %[1]s [flags] -Agent role: %s +Agent role: %[2]s Commands: + enroll register with the management server + --server --token --ca-hash sha256: + [--server-name ] [--force] + run run the agent (until SIGINT/SIGTERM) + status show identity, revisions and the last result version print version information help show this help -`, binary, role) + +Common flags: + --config configuration file (default %[3]s) +`, binary, kind, DefaultConfigPath(kind)) +} + +type cli struct { + binary string + kind Kind + factory RoleFactory + stdout io.Writer + stderr io.Writer +} + +func (c *cli) flags(name string) (*flag.FlagSet, *string) { + fs := flag.NewFlagSet(c.binary+" "+name, flag.ContinueOnError) + fs.SetOutput(c.stderr) + cfg := fs.String("config", DefaultConfigPath(c.kind), "configuration file") + return fs, cfg +} + +func (c *cli) fail(err error) int { + _, _ = fmt.Fprintf(c.stderr, "%s: %v\n", c.binary, err) + return 1 +} + +// parse parses flags and rejects positional arguments. +func parse(fs *flag.FlagSet, args []string) int { + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() > 0 { + _, _ = fmt.Fprintf(fs.Output(), "unexpected argument %q\n", fs.Arg(0)) + return 2 + } + return -1 +} + +func (c *cli) enroll(args []string) int { + fs, cfgPath := c.flags("enroll") + var opts EnrollOptions + fs.StringVar(&opts.Server, "server", "", "agent API address of the management server (host:port)") + fs.StringVar(&opts.Token, "token", "", "one-time enrollment token") + fs.StringVar(&opts.CAHash, "ca-hash", "", "fingerprint of the agent CA (sha256:)") + fs.StringVar(&opts.ServerName, "server-name", "", "TLS server name (default: host of --server)") + fs.BoolVar(&opts.Force, "force", false, "replace an existing identity") + if code := parse(fs, args); code >= 0 { + return code + } + if opts.Server == "" || opts.Token == "" || opts.CAHash == "" { + _, _ = fmt.Fprintln(c.stderr, "enroll needs --server, --token and --ca-hash") + return 2 + } + cfg, err := LoadConfig(*cfgPath, c.kind) + if err != nil { + return c.fail(err) + } + store, unlock, err := openLocked(cfg) + if err != nil { + return c.fail(err) + } + defer unlock() + opts.Kind = c.kind + opts.Hostname, _ = os.Hostname() + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + id, err := Enroll(ctx, store, opts) + if err != nil { + return c.fail(err) + } + _, _ = fmt.Fprintf(c.stdout, "Enrolled as agent %s (%s) with %s.\nIdentity stored in %s.\n", + id.AgentID, c.kind, id.Server, store.Dir()) + return 0 +} + +func openLocked(cfg *Config) (*Store, func(), error) { + store, err := OpenStore(cfg.StateDir) + if err != nil { + return nil, nil, err + } + unlock, err := store.Lock() + if err != nil { + return nil, nil, err + } + return store, unlock, nil +} + +func (c *cli) run(args []string) int { + fs, cfgPath := c.flags("run") + if code := parse(fs, args); code >= 0 { + return code + } + cfg, err := LoadConfig(*cfgPath, c.kind) + if err != nil { + return c.fail(err) + } + log, err := logging.New(c.stderr, cfg.Log) + if err != nil { + return c.fail(err) + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := runAgent(ctx, c.kind, c.factory, cfg, log); err != nil { + log.Error("agent stopped", "error", err) + return 1 + } + return 0 +} + +// runAgent runs the agent until ctx is canceled. +func runAgent(ctx context.Context, kind Kind, factory RoleFactory, cfg *Config, log *slog.Logger) error { + if factory == nil { + return errRoleNotAvailable + } + store, unlock, err := openLocked(cfg) + if err != nil { + return err + } + defer unlock() + id, err := store.LoadIdentity() + if err != nil { + return err + } + if id.Role != kind { + return fmt.Errorf("identity in %s belongs to a %s agent", store.Dir(), id.Role) + } + role, err := factory(cfg.DecodeRole) + if err != nil { + return fmt.Errorf("%s role: %w", kind, err) + } + log = log.With("agent_id", id.AgentID, "role", string(kind)) + hostname, _ := os.Hostname() + src := NewServerSource(store, id, hostname, log) + agent, err := NewAgent(kind, cfg, role, store, id, src, log) + if err != nil { + return err + } + log.Info("agent started", "version", buildinfo.Get().Version, "server", id.Server, "state_dir", store.Dir()) + err = agent.Run(ctx) + log.Info("agent stopped; the managed service keeps its configuration") + return err +} + +func (c *cli) status(args []string) int { + fs, cfgPath := c.flags("status") + if code := parse(fs, args); code >= 0 { + return code + } + cfg, err := LoadConfig(*cfgPath, c.kind) + if err != nil { + return c.fail(err) + } + store, err := OpenStore(cfg.StateDir) + if err != nil { + return c.fail(err) + } + if err := writeStatus(c.stdout, store); err != nil { + return c.fail(err) + } + return 0 +} + +// writeStatus prints identity, revisions and the last result. +func writeStatus(w io.Writer, store *Store) error { + var b strings.Builder + line := func(k, format string, a ...any) { fmt.Fprintf(&b, "%-20s %s\n", k+":", fmt.Sprintf(format, a...)) } + line("State directory", "%s", store.Dir()) + id, err := store.LoadIdentity() + switch { + case errors.Is(err, ErrNotEnrolled): + line("Identity", "not enrolled") + case err != nil: + return err + default: + leaf := id.Certificate.Leaf + line("Agent ID", "%s", id.AgentID) + line("Role", "%s", id.Role) + line("Server", "%s (TLS name %s)", id.Server, id.ServerName) + line("Enrolled", "%s", id.EnrolledAt.Format(time.RFC3339)) + line("Certificate", "valid until %s, renewal from %s", leaf.NotAfter.UTC().Format(time.RFC3339), + renewalDue(leaf).UTC().Format(time.RFC3339)) + line("CA fingerprint", "%s", agentpki.Fingerprint(id.CA)) + } + st, err := store.LoadState() + if err != nil { + return err + } + revs, err := store.ListRevisions() + if err != nil { + return err + } + line("Applied revision", "%s", revOrNone(st.Applied)) + line("Last good revision", "%s", revOrNone(st.LastGood)) + if st.LastUnsuccessful != 0 { + line("Last unsuccessful", "%d (retried only by a newer revision or an explicit push)", st.LastUnsuccessful) + } + nums := make([]string, len(revs)) + for i, n := range revs { + nums[i] = fmt.Sprint(n) + } + line("Stored revisions", "%s", orNone(strings.Join(nums, ", "))) + if r := st.LastResult; r != nil { + dep := "" + if r.DeploymentID != "" { + dep = ", deployment " + r.DeploymentID + } + line("Last result", "revision %d %s at %s (%s%s)", r.Revision, r.Outcome, + r.FinishedAt.UTC().Format(time.RFC3339), r.Trigger, dep) + if r.Message != "" { + line("Message", "%s", r.Message) + } + } else { + line("Last result", "none") + } + line("Pending reports", "%d", len(st.PendingReports)) + if st.LeaseSequence != 0 { + line("Lease sequence", "%d", st.LeaseSequence) + } + _, err = io.WriteString(w, b.String()) + return err +} + +func revOrNone(n uint64) string { + if n == 0 { + return "none" + } + return fmt.Sprint(n) +} + +func orNone(s string) string { + if s == "" { + return "none" + } + return s } diff --git a/agent/core/cli_test.go b/agent/core/cli_test.go index ff9a76c..7a3e897 100644 --- a/agent/core/cli_test.go +++ b/agent/core/cli_test.go @@ -4,11 +4,16 @@ package core import ( "bytes" + "context" + "errors" + "os" + "path/filepath" "strings" "testing" + "time" ) -func TestRunCLI(t *testing.T) { +func TestMainBasics(t *testing.T) { tests := []struct { name string args []string @@ -20,14 +25,19 @@ func TestRunCLI(t *testing.T) { {name: "version", args: []string{"version"}, wantCode: 0, wantStdout: "ddi-agent-test "}, {name: "--version", args: []string{"--version"}, wantCode: 0, wantStdout: "ddi-agent-test "}, {name: "help", args: []string{"help"}, wantCode: 0, wantStdout: "Agent role: dns"}, + {name: "help lists enroll", args: []string{"--help"}, wantCode: 0, wantStdout: "--ca-hash sha256:"}, {name: "unknown", args: []string{"explode"}, wantCode: 2, wantStderr: `unknown command "explode"`}, + {name: "enroll without flags", args: []string{"enroll"}, wantCode: 2, wantStderr: "enroll needs --server"}, + {name: "bad flag", args: []string{"run", "--bogus"}, wantCode: 2, wantStderr: "flag provided but not defined"}, + {name: "extra argument", args: []string{"status", "extra"}, wantCode: 2, wantStderr: `unexpected argument "extra"`}, + {name: "flag help", args: []string{"status", "-h"}, wantCode: 0, wantStderr: "-config"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var out, errOut bytes.Buffer - code := RunCLI("ddi-agent-test", KindDNS, tt.args, &out, &errOut) + code := Main("ddi-agent-test", KindDNS, nil, tt.args, &out, &errOut) if code != tt.wantCode { - t.Errorf("code = %d, want %d", code, tt.wantCode) + t.Errorf("code = %d, want %d (stderr %q)", code, tt.wantCode, errOut.String()) } if tt.wantStdout != "" && !strings.Contains(out.String(), tt.wantStdout) { t.Errorf("stdout %q does not contain %q", out.String(), tt.wantStdout) @@ -38,3 +48,146 @@ func TestRunCLI(t *testing.T) { }) } } + +// writeConfig writes a configuration with its own state directory. +func writeConfig(t *testing.T, extra string) (cfgPath, stateDir string) { + t.Helper() + dir := t.TempDir() + stateDir = filepath.Join(dir, "state") + cfgPath = filepath.Join(dir, "agent.yaml") + data := "state_dir: " + filepath.ToSlash(stateDir) + "\nlog:\n format: text\n" + extra + if err := os.WriteFile(cfgPath, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + return cfgPath, stateDir +} + +func TestMainEnrollStatusRun(t *testing.T) { + srv := newFakeServer(t, KindDNS) + addr := srv.start(t) + cfgPath, stateDir := writeConfig(t, "dns:\n zone_dir: /var/lib/bind/x\n") + + var out, errOut bytes.Buffer + if code := Main("ddi-agent-dns", KindDNS, nil, []string{"status", "--config", cfgPath}, &out, &errOut); code != 0 { + t.Fatalf("status before enroll = %d: %s", code, errOut.String()) + } + if !strings.Contains(out.String(), "not enrolled") { + t.Errorf("status = %q", out.String()) + } + + // run without identity fails. + out.Reset() + errOut.Reset() + factory := func(decode func(any) error) (Role, error) { return newFakeRole(KindDNS), nil } + if code := Main("ddi-agent-dns", KindDNS, factory, []string{"run", "--config", cfgPath}, &out, &errOut); code != 1 { + t.Fatalf("run before enroll = %d", code) + } + if !strings.Contains(errOut.String(), "not enrolled") { + t.Errorf("stderr = %q", errOut.String()) + } + + args := []string{"enroll", "--config", cfgPath, "--server", addr, "--token", srv.token, + "--ca-hash", srv.fingerprint(), "--server-name", "localhost"} + out.Reset() + errOut.Reset() + if code := Main("ddi-agent-dns", KindDNS, nil, args, &out, &errOut); code != 0 { + t.Fatalf("enroll = %d: %s", code, errOut.String()) + } + if !strings.Contains(out.String(), srv.agentID) { + t.Errorf("enroll output = %q", out.String()) + } + out.Reset() + errOut.Reset() + if code := Main("ddi-agent-dns", KindDNS, nil, args, &out, &errOut); code != 1 || !strings.Contains(errOut.String(), "--force") { + t.Errorf("second enroll = %d: %s", code, errOut.String()) + } + + // Run the agent until revision 1 is reported. + srv.with(func(f *fakeServer) { f.desired = 1 }) + cfg, err := LoadConfig(cfgPath, KindDNS) + if err != nil { + t.Fatal(err) + } + var gotSection struct { + ZoneDir string `yaml:"zone_dir"` + } + role := newFakeRole(KindDNS) + factory = func(decode func(any) error) (Role, error) { + if err := decode(&gotSection); err != nil { + return nil, err + } + return role, nil + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- runAgent(ctx, KindDNS, factory, cfg, testLogger()) }() + waitFor(t, "deployment report", func() bool { return len(serverReports(srv)) == 1 }) + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("runAgent = %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("runAgent did not return") + } + if gotSection.ZoneDir != "/var/lib/bind/x" { + t.Errorf("role section = %+v", gotSection) + } + + out.Reset() + errOut.Reset() + if code := Main("ddi-agent-dns", KindDNS, nil, []string{"status", "--config", cfgPath}, &out, &errOut); code != 0 { + t.Fatalf("status = %d: %s", code, errOut.String()) + } + for _, want := range []string{srv.agentID, "Applied revision:", "revision 1 succeeded", "Stored revisions: 1", stateDir} { + if !strings.Contains(out.String(), want) { + t.Errorf("status lacks %q:\n%s", want, out.String()) + } + } +} + +func TestRunAgentErrors(t *testing.T) { + cfgPath, _ := writeConfig(t, "") + cfg, err := LoadConfig(cfgPath, KindDNS) + if err != nil { + t.Fatal(err) + } + if err := runAgent(context.Background(), KindDNS, nil, cfg, testLogger()); !errors.Is(err, errRoleNotAvailable) { + t.Errorf("nil factory: %v", err) + } + // Enrolled as a DHCP agent, run as DNS agent. + store, err := OpenStore(cfg.StateDir) + if err != nil { + t.Fatal(err) + } + pki := newTestPKI(t) + id, key := testIdentity(t, pki) + id.Role = KindDHCP + if err := store.SaveIdentity(id, key, false); err != nil { + t.Fatal(err) + } + factory := func(func(any) error) (Role, error) { return newFakeRole(KindDNS), nil } + if err := runAgent(context.Background(), KindDNS, factory, cfg, testLogger()); err == nil || !strings.Contains(err.Error(), "dhcp agent") { + t.Errorf("role mismatch: %v", err) + } + if err := runAgent(context.Background(), KindDHCP, func(func(any) error) (Role, error) { + return nil, errors.New("kea missing") + }, cfg, testLogger()); err == nil || !strings.Contains(err.Error(), "kea missing") { + t.Errorf("factory error: %v", err) + } +} + +func TestMainConfigErrors(t *testing.T) { + cfgPath, _ := writeConfig(t, "bogus: 1\n") + for _, args := range [][]string{ + {"run", "--config", cfgPath}, + {"status", "--config", cfgPath}, + {"enroll", "--config", cfgPath, "--server", "x:1", "--token", "a.b", "--ca-hash", "sha256:00"}, + } { + var out, errOut bytes.Buffer + if code := Main("ddi-agent-dns", KindDNS, nil, args, &out, &errOut); code != 1 || !strings.Contains(errOut.String(), "bogus") { + t.Errorf("%v: code %d, stderr %q", args[0], code, errOut.String()) + } + } +} diff --git a/agent/core/config.go b/agent/core/config.go new file mode 100644 index 0000000..8f17cea --- /dev/null +++ b/agent/core/config.go @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: Apache-2.0 + +package core + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "time" + + "go.yaml.in/yaml/v3" + + "git.beugsiud.de/beug.k/DDI/internal/brand" + "git.beugsiud.de/beug.k/DDI/internal/logging" +) + +// Configuration defaults (docs/adr/0021-agent-strecke-m3.md). +const ( + DefaultApplyTimeout = 5 * time.Minute + DefaultHealthTimeout = 30 * time.Second + DefaultLeaseInterval = 30 * time.Second + DefaultLeaseFullSync = time.Hour +) + +// Config is the agent configuration file (/agent-.yaml). +// Server address, server name and the identity are not configured here; the +// enroll command stores them in the state directory. +type Config struct { + // StateDir holds identity, revisions and state.json (ADR 0011). + StateDir string `yaml:"state_dir"` + // Log configures the structured logger (written to stderr). + Log logging.Options `yaml:"log"` + // ApplyTimeout bounds each role call (Validate, Activate) of an apply. + ApplyTimeout time.Duration `yaml:"apply_timeout"` + // HealthTimeout is how long the health check of a freshly activated + // revision may take until it reports OK. + HealthTimeout time.Duration `yaml:"health_timeout"` + // LeaseInterval is the lease polling interval (DHCP). + LeaseInterval time.Duration `yaml:"lease_interval"` + // LeaseFullSync is the interval of full lease snapshots. + LeaseFullSync time.Duration `yaml:"lease_full_sync"` + + // role holds the undecoded section of the agent's role. + role yaml.Node +} + +// fileConfig mirrors Config for strict decoding, including the role +// sections that are handed to the role factory undecoded. +type fileConfig struct { + StateDir string `yaml:"state_dir"` + Log logging.Options `yaml:"log"` + ApplyTimeout time.Duration `yaml:"apply_timeout"` + HealthTimeout time.Duration `yaml:"health_timeout"` + LeaseInterval time.Duration `yaml:"lease_interval"` + LeaseFullSync time.Duration `yaml:"lease_full_sync"` + DNS yaml.Node `yaml:"dns"` + DHCP yaml.Node `yaml:"dhcp"` +} + +// DefaultConfigPath returns /agent-.yaml. +func DefaultConfigPath(kind Kind) string { + return filepath.Join(brand.ConfigDir, "agent-"+string(kind)+".yaml") +} + +// DefaultStateDir returns /agent-. +func DefaultStateDir(kind Kind) string { + return filepath.Join(brand.StateDir, "agent-"+string(kind)) +} + +// LoadConfig reads the configuration file of the role. A missing file yields +// the defaults. +func LoadConfig(path string, kind Kind) (*Config, error) { + data, err := os.ReadFile(path) //nolint:gosec // path is given by the operator + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("read configuration: %w", err) + } + cfg, err := ParseConfig(data, kind) + if err != nil { + return nil, fmt.Errorf("configuration %s: %w", path, err) + } + return cfg, nil +} + +// ParseConfig decodes a configuration (strictly: unknown keys are errors), +// applies defaults and validates it. +func ParseConfig(data []byte, kind Kind) (*Config, error) { + var fc fileConfig + if len(bytes.TrimSpace(data)) > 0 { + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(true) + if err := dec.Decode(&fc); err != nil && !errors.Is(err, io.EOF) { + return nil, err + } + } + cfg := &Config{ + StateDir: fc.StateDir, Log: fc.Log, ApplyTimeout: fc.ApplyTimeout, + HealthTimeout: fc.HealthTimeout, LeaseInterval: fc.LeaseInterval, LeaseFullSync: fc.LeaseFullSync, + } + switch kind { + case KindDNS: + cfg.role = fc.DNS + if !isEmptyNode(&fc.DHCP) { + return nil, errors.New("section dhcp is not used by the DNS agent") + } + case KindDHCP: + cfg.role = fc.DHCP + if !isEmptyNode(&fc.DNS) { + return nil, errors.New("section dns is not used by the DHCP agent") + } + default: + return nil, fmt.Errorf("unknown role %q", kind) + } + if cfg.StateDir == "" { + cfg.StateDir = DefaultStateDir(kind) + } + setDefault(&cfg.ApplyTimeout, DefaultApplyTimeout) + setDefault(&cfg.HealthTimeout, DefaultHealthTimeout) + setDefault(&cfg.LeaseInterval, DefaultLeaseInterval) + setDefault(&cfg.LeaseFullSync, DefaultLeaseFullSync) + if err := cfg.validate(); err != nil { + return nil, err + } + return cfg, nil +} + +func setDefault(d *time.Duration, def time.Duration) { + if *d == 0 { + *d = def + } +} + +func (c *Config) validate() error { + for _, d := range []struct { + name string + v time.Duration + }{ + {"apply_timeout", c.ApplyTimeout}, {"health_timeout", c.HealthTimeout}, + {"lease_interval", c.LeaseInterval}, {"lease_full_sync", c.LeaseFullSync}, + } { + if d.v < time.Second { + return fmt.Errorf("%s must be at least 1s", d.name) + } + } + if c.LeaseFullSync < c.LeaseInterval { + return errors.New("lease_full_sync must not be shorter than lease_interval") + } + if _, err := logging.New(io.Discard, c.Log); err != nil { + return err + } + return nil +} + +// isEmptyNode reports whether a section is absent or null. +func isEmptyNode(n *yaml.Node) bool { + return n.Kind == 0 || (n.Kind == yaml.ScalarNode && n.Tag == "!!null") +} + +// DecodeRole decodes the role's section into v (strictly). An absent section +// leaves v unchanged so that roles keep their defaults. +func (c *Config) DecodeRole(v any) error { + if isEmptyNode(&c.role) { + return nil + } + data, err := yaml.Marshal(&c.role) + if err != nil { + return fmt.Errorf("encode role section: %w", err) + } + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(true) + if err := dec.Decode(v); err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("role section: %w", err) + } + return nil +} diff --git a/agent/core/config_test.go b/agent/core/config_test.go new file mode 100644 index 0000000..426798b --- /dev/null +++ b/agent/core/config_test.go @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: Apache-2.0 + +package core + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestParseConfigDefaults(t *testing.T) { + for _, data := range []string{"", " \n", "# only a comment\n"} { + cfg, err := ParseConfig([]byte(data), KindDNS) + if err != nil { + t.Fatalf("%q: %v", data, err) + } + if cfg.StateDir != DefaultStateDir(KindDNS) || cfg.ApplyTimeout != DefaultApplyTimeout || + cfg.HealthTimeout != 30*time.Second || cfg.LeaseInterval != 30*time.Second || cfg.LeaseFullSync != time.Hour { + t.Errorf("defaults = %+v", cfg) + } + var role struct{ X int } + role.X = 7 + if err := cfg.DecodeRole(&role); err != nil || role.X != 7 { + t.Errorf("absent role section: %v, %+v", err, role) + } + } + if !strings.HasSuffix(filepath.ToSlash(DefaultConfigPath(KindDHCP)), "/agent-dhcp.yaml") { + t.Errorf("DefaultConfigPath = %s", DefaultConfigPath(KindDHCP)) + } +} + +func TestParseConfigFull(t *testing.T) { + data := ` +state_dir: /tmp/agent +log: + level: debug + format: text +apply_timeout: 2m +health_timeout: 45s +lease_interval: 10s +lease_full_sync: 30m +dhcp: + config_file: /etc/kea/kea-dhcp4.conf + control_socket: /run/kea/ctrl.sock +` + cfg, err := ParseConfig([]byte(data), KindDHCP) + if err != nil { + t.Fatal(err) + } + if cfg.StateDir != "/tmp/agent" || cfg.Log.Level != "debug" || cfg.Log.Format != "text" || + cfg.ApplyTimeout != 2*time.Minute || cfg.HealthTimeout != 45*time.Second || + cfg.LeaseInterval != 10*time.Second || cfg.LeaseFullSync != 30*time.Minute { + t.Errorf("config = %+v", cfg) + } + var role struct { + ConfigFile string `yaml:"config_file"` + ControlSocket string `yaml:"control_socket"` + } + if err := cfg.DecodeRole(&role); err != nil { + t.Fatal(err) + } + if role.ConfigFile != "/etc/kea/kea-dhcp4.conf" || role.ControlSocket != "/run/kea/ctrl.sock" { + t.Errorf("role = %+v", role) + } + // Strict decoding of the role section. + var narrow struct { + ConfigFile string `yaml:"config_file"` + } + if err := cfg.DecodeRole(&narrow); err == nil { + t.Error("unknown role key accepted") + } +} + +func TestParseConfigErrors(t *testing.T) { + cases := map[string]string{ + "unknown key": "flavor: blue\n", + "other role": "dhcp:\n config_file: x\n", + "short timeout": "health_timeout: 10ms\n", + "full sync": "lease_interval: 2m\nlease_full_sync: 1m\n", + "bad level": "log:\n level: loud\n", + "bad duration": "apply_timeout: soon\n", + "not a mapping": "- a\n- b\n", + "unknown log key": "log:\n flavor: x\n", + "negative timeout": "apply_timeout: -5s\n", + } + for name, data := range cases { + if _, err := ParseConfig([]byte(data), KindDNS); err == nil { + t.Errorf("%s: accepted", name) + } + } + if _, err := ParseConfig(nil, Kind("ntp")); err == nil { + t.Error("unknown role accepted") + } + if _, err := ParseConfig([]byte("dns:\n zone_dir: x\n"), KindDHCP); err == nil { + t.Error("dns section accepted by the DHCP agent") + } + // A null section is fine. + if _, err := ParseConfig([]byte("dhcp:\n"), KindDNS); err != nil { + t.Errorf("null section: %v", err) + } +} + +func TestLoadConfig(t *testing.T) { + dir := t.TempDir() + cfg, err := LoadConfig(filepath.Join(dir, "missing.yaml"), KindDNS) + if err != nil || cfg.StateDir != DefaultStateDir(KindDNS) { + t.Fatalf("missing file: %+v, %v", cfg, err) + } + path := filepath.Join(dir, "agent-dns.yaml") + if err := os.WriteFile(path, []byte("state_dir: "+filepath.ToSlash(dir)+"\n"), 0o600); err != nil { + t.Fatal(err) + } + cfg, err = LoadConfig(path, KindDNS) + if err != nil || cfg.StateDir != filepath.ToSlash(dir) { + t.Fatalf("file: %+v, %v", cfg, err) + } + if err := os.WriteFile(path, []byte("bogus: 1\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadConfig(path, KindDNS); err == nil || !strings.Contains(err.Error(), path) { + t.Errorf("error does not name the file: %v", err) + } + if _, err := LoadConfig(dir, KindDNS); err == nil { + t.Error("directory accepted as configuration file") + } +} diff --git a/agent/core/enroll.go b/agent/core/enroll.go new file mode 100644 index 0000000..af14e84 --- /dev/null +++ b/agent/core/enroll.go @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: Apache-2.0 + +package core + +import ( + "context" + "crypto/ed25519" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net" + "strings" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + "git.beugsiud.de/beug.k/DDI/internal/agentpki" + "git.beugsiud.de/beug.k/DDI/internal/buildinfo" + agentv1 "git.beugsiud.de/beug.k/DDI/proto/gen/ddi/agent/v1" +) + +// EnrollOptions are the parameters of the enroll command. +type EnrollOptions struct { + Kind Kind + // Server is the agent API address (host:port; an https:// prefix is + // tolerated). + Server string + // Token is the one-time enrollment token ".". + Token string + // CAHash is the fingerprint "sha256:" of the agent CA. + CAHash string + // ServerName is the TLS server name (default: host part of Server). + ServerName string + // Force replaces an existing identity. + Force bool + Hostname string +} + +// normalizeServer validates host:port and returns the address and its host. +func normalizeServer(s string) (addr, host string, err error) { + s = strings.TrimSpace(s) + s = strings.TrimPrefix(s, "https://") + s = strings.TrimSuffix(s, "/") + host, port, err := net.SplitHostPort(s) + if err != nil || host == "" || port == "" { + return "", "", fmt.Errorf("server address %q must have the form host:port", s) + } + return s, host, nil +} + +// Enroll exchanges the token for a client certificate and stores the +// identity. The server is authenticated by the CA fingerprint: the TLS chain +// it presents must contain a CA with that fingerprint which issued the +// server certificate for ServerName. +func Enroll(ctx context.Context, store *Store, opts EnrollOptions) (*Identity, error) { + addr, host, err := normalizeServer(opts.Server) + if err != nil { + return nil, err + } + if opts.Token == "" || !strings.Contains(opts.Token, ".") { + return nil, errors.New("enrollment token must have the form .") + } + fingerprint, err := agentpki.ParseFingerprint(opts.CAHash) + if err != nil { + return nil, err + } + serverName := opts.ServerName + if serverName == "" { + serverName = host + } + if store.Enrolled() && !opts.Force { + return nil, errAlreadyEnrolled + } + role, err := roleToProto(opts.Kind) + if err != nil { + return nil, err + } + + var ( + caMu sync.Mutex + pinnedCA *x509.Certificate + ) + tlsCfg := &tls.Config{ + MinVersion: tls.VersionTLS13, + ServerName: serverName, + // The server is verified against the CA fingerprint below instead + // of a system trust store. + InsecureSkipVerify: true, //nolint:gosec // verified in VerifyPeerCertificate + // No session resumption, so every handshake runs the check below. + SessionTicketsDisabled: true, + VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + ca, err := agentpki.VerifyChainWithFingerprint(rawCerts, fingerprint, serverName) + if err != nil { + return err + } + caMu.Lock() + pinnedCA = ca + caMu.Unlock() + return nil + }, + } + conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg))) + if err != nil { + return nil, fmt.Errorf("connect to %s: %w", addr, err) + } + defer func() { _ = conn.Close() }() + + key, csr, err := newKeyAndCSR(opts.Hostname) + if err != nil { + return nil, err + } + rctx, cancel := context.WithTimeout(ctx, rpcTimeout) + defer cancel() + resp, err := agentv1.NewAgentServiceClient(conn).Enroll(rctx, &agentv1.EnrollRequest{ + Token: opts.Token, CsrDer: csr, Role: role, Hostname: opts.Hostname, + AgentVersion: buildinfo.Get().Version, + }) + if err != nil { + return nil, fmt.Errorf("enroll: %w", err) + } + caMu.Lock() + ca := pinnedCA + caMu.Unlock() + if ca == nil { + return nil, errors.New("enroll: server CA was not verified") + } + if chain := resp.GetCaCertificatesDer(); len(chain) > 0 { + root, err := x509.ParseCertificate(chain[len(chain)-1]) + if err != nil || !root.Equal(ca) { + return nil, errors.New("enroll: CA in the response differs from the verified server CA") + } + } + if !agentpki.ValidAgentID(resp.GetAgentId()) { + return nil, fmt.Errorf("enroll: server assigned an invalid agent ID %q", resp.GetAgentId()) + } + cert, err := checkIssued(resp.GetCertificateDer(), ca, resp.GetAgentId(), key) + if err != nil { + return nil, fmt.Errorf("enroll: %w", err) + } + if len(resp.GetBundleSigningPublicKey()) != ed25519.PublicKeySize { + return nil, errors.New("enroll: invalid bundle signing key") + } + id := &Identity{ + AgentID: resp.GetAgentId(), Role: opts.Kind, Server: addr, ServerName: serverName, + EnrolledAt: time.Now().UTC(), CA: ca, Certificate: keyPair(key, cert), + BundleKey: ed25519.PublicKey(resp.GetBundleSigningPublicKey()), + } + if err := store.SaveIdentity(id, key, opts.Force); err != nil { + return nil, err + } + if opts.Force { + // Reports of the previous identity must not be sent with the new one. + st, err := store.LoadState() + if err == nil && len(st.PendingReports) > 0 { + st.PendingReports = nil + err = store.SaveState(st) + } + if err != nil { + return nil, err + } + } + return id, nil +} + +func roleToProto(k Kind) (agentv1.Role, error) { + switch k { + case KindDNS: + return agentv1.Role_ROLE_DNS, nil + case KindDHCP: + return agentv1.Role_ROLE_DHCP, nil + default: + return agentv1.Role_ROLE_UNSPECIFIED, fmt.Errorf("unknown role %q", k) + } +} diff --git a/agent/core/helpers_test.go b/agent/core/helpers_test.go new file mode 100644 index 0000000..a8dfe31 --- /dev/null +++ b/agent/core/helpers_test.go @@ -0,0 +1,697 @@ +// SPDX-License-Identifier: Apache-2.0 + +package core + +import ( + "context" + "crypto" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "fmt" + "io" + "log/slog" + "math/big" + "net" + "net/url" + "os" + "path/filepath" + "slices" + "sync" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" + + "git.beugsiud.de/beug.k/DDI/internal/agentpki" + "git.beugsiud.de/beug.k/DDI/internal/bundle" + agentv1 "git.beugsiud.de/beug.k/DDI/proto/gen/ddi/agent/v1" +) + +const testAgentID = "0b6f3c1e-5d2a-4c8e-9f10-123456789abc" + +func testLogger() *slog.Logger { + if testing.Verbose() { + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})) + } + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +// waitFor polls cond until it holds or the timeout expires. +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timeout waiting for %s", what) + } + time.Sleep(10 * time.Millisecond) + } +} + +// ---- PKI ---- + +type testPKI struct { + ca *x509.Certificate + caKey *ecdsa.PrivateKey + serverCert tls.Certificate +} + +func newTestPKI(t *testing.T) *testPKI { + t.Helper() + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test agent CA"}, + NotBefore: time.Now().Add(-24 * time.Hour), NotAfter: time.Now().Add(24 * time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &caKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + ca, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + p := &testPKI{ca: ca, caKey: caKey} + srvKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + srvDER, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{ + SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "localhost"}, + DNSNames: []string{"localhost"}, IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }, ca, &srvKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + p.serverCert = tls.Certificate{Certificate: [][]byte{srvDER, ca.Raw}, PrivateKey: srvKey} + return p +} + +func (p *testPKI) issueClient(pub crypto.PublicKey, agentID string, notBefore, notAfter time.Time) (*x509.Certificate, error) { + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 127)) + if err != nil { + return nil, err + } + der, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{ + SerialNumber: serial, Subject: pkix.Name{CommonName: agentID}, + URIs: []*url.URL{agentpki.AgentURI(agentID)}, + NotBefore: notBefore, NotAfter: notAfter, + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }, p.ca, pub, p.caKey) + if err != nil { + return nil, err + } + return x509.ParseCertificate(der) +} + +// ---- bundles ---- + +func newSigningKey(t *testing.T) (ed25519.PublicKey, ed25519.PrivateKey) { + t.Helper() + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + return pub, priv +} + +// testArtifacts returns the artifacts of a test revision. +func testArtifacts(rev uint64) []bundle.Artifact { + return []bundle.Artifact{ + {Path: "ddi-suite.conf", Mode: 0o640, Content: fmt.Appendf(nil, "// revision %d\n", rev)}, + {Path: "zones/db.example.com", Mode: 0o644, Content: fmt.Appendf(nil, "example.com. SOA ns 1 %d\n", rev)}, + } +} + +func signBundle(t testing.TB, key ed25519.PrivateKey, agentID string, role Kind, rev uint64) *Bundle { + t.Helper() + arts := testArtifacts(rev) + manifest, sig, err := bundle.Sign(key, agentID, string(role), rev, time.Unix(1_700_000_000, 0), arts) + if err != nil { + t.Fatal(err) + } + b := &Bundle{Manifest: manifest, Signature: sig} + for _, a := range arts { + b.Artifacts = append(b.Artifacts, BundleArtifact{Path: a.Path, Content: a.Content}) + } + return b +} + +// ---- fake role ---- + +type fakeRole struct { + kind Kind + + mu sync.Mutex + validateErr map[uint64]error + activateErr map[uint64]error + unhealthy map[uint64]bool + active uint64 + calls []string + healthCalls map[uint64]int + leases []Lease + leasesErr error + versionErr error +} + +func newFakeRole(kind Kind) *fakeRole { + return &fakeRole{ + kind: kind, validateErr: map[uint64]error{}, activateErr: map[uint64]error{}, + unhealthy: map[uint64]bool{}, healthCalls: map[uint64]int{}, + } +} + +func (r *fakeRole) Kind() Kind { return r.kind } + +func (r *fakeRole) ServiceVersion(context.Context) (string, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.versionErr != nil { + return "", r.versionErr + } + return "Fake 1.0", nil +} + +// checkFiles verifies that the stored revision holds its artifacts. +func checkFiles(rev Revision) error { + for _, f := range rev.Files { + data, err := os.ReadFile(filepath.Join(rev.Dir, filepath.FromSlash(f.Path))) + if err != nil { + return err + } + if int64(len(data)) != f.Size { + return fmt.Errorf("%s: size %d, want %d", f.Path, len(data), f.Size) + } + } + return nil +} + +func (r *fakeRole) Validate(_ context.Context, rev Revision) error { + r.mu.Lock() + defer r.mu.Unlock() + r.calls = append(r.calls, fmt.Sprintf("validate %d", rev.Number)) + if err := checkFiles(rev); err != nil { + return err + } + return r.validateErr[rev.Number] +} + +func (r *fakeRole) Activate(_ context.Context, rev Revision, prev *Revision) error { + r.mu.Lock() + defer r.mu.Unlock() + p := "none" + if prev != nil { + p = fmt.Sprint(prev.Number) + } + r.calls = append(r.calls, fmt.Sprintf("activate %d prev %s", rev.Number, p)) + if err := r.activateErr[rev.Number]; err != nil { + return err + } + r.active = rev.Number + return nil +} + +func (r *fakeRole) Health(_ context.Context, expected *Revision) Health { + r.mu.Lock() + defer r.mu.Unlock() + var n uint64 + if expected != nil { + n = expected.Number + } + r.healthCalls[n]++ + if r.unhealthy[r.active] { + return Health{State: HealthFailed, Message: "service down", + Checks: []HealthCheck{{Name: "soa", State: HealthFailed, Message: "serial mismatch"}, {Name: "status", State: HealthOK}}} + } + return Health{State: HealthOK} +} + +func (r *fakeRole) Leases(context.Context) ([]Lease, error) { + r.mu.Lock() + defer r.mu.Unlock() + return slices.Clone(r.leases), r.leasesErr +} + +func (r *fakeRole) setLeases(l []Lease) { + r.mu.Lock() + defer r.mu.Unlock() + r.leases = l +} + +func (r *fakeRole) callLog() []string { + r.mu.Lock() + defer r.mu.Unlock() + return slices.Clone(r.calls) +} + +func (r *fakeRole) activeRevision() uint64 { + r.mu.Lock() + defer r.mu.Unlock() + return r.active +} + +func (r *fakeRole) resetCalls() { + r.mu.Lock() + defer r.mu.Unlock() + r.calls = nil +} + +// fakeLeaseRole is a role with a LeaseSource; fakeRole itself has the +// method too, so wrap it to hide Leases for the DNS case. +type dnsOnlyRole struct{ *fakeRole } + +func (d dnsOnlyRole) Leases() {} // shadows LeaseSource.Leases with another signature + +// ---- fake source ---- + +type fakeSource struct { + mu sync.Mutex + bundles map[uint64]*Bundle + fetchErr error + reportErr error + leaseErr error + reports []DeploymentReport + batches []LeaseBatch + fetches int + onFetch func() + run func(ctx context.Context, h SourceHandler) error +} + +func newFakeSource() *fakeSource { return &fakeSource{bundles: map[uint64]*Bundle{}} } + +func (s *fakeSource) Run(ctx context.Context, h SourceHandler) error { + if s.run != nil { + return s.run(ctx, h) + } + <-ctx.Done() + return ctx.Err() +} + +func (s *fakeSource) FetchBundle(_ context.Context, rev uint64) (*Bundle, error) { + s.mu.Lock() + s.fetches++ + b, err, hook := s.bundles[rev], s.fetchErr, s.onFetch + s.mu.Unlock() + if hook != nil { + hook() + } + if err != nil { + return nil, err + } + if b == nil { + return nil, fmt.Errorf("revision %d not found", rev) + } + return b, nil +} + +func (s *fakeSource) ReportDeployment(_ context.Context, r DeploymentReport) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.reportErr != nil { + return s.reportErr + } + s.reports = append(s.reports, r) + return nil +} + +func (s *fakeSource) ReportLeases(_ context.Context, b LeaseBatch) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.leaseErr != nil { + return s.leaseErr + } + b.Leases = slices.Clone(b.Leases) + s.batches = append(s.batches, b) + return nil +} + +func (s *fakeSource) setReportErr(err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.reportErr = err +} + +func (s *fakeSource) reported() []DeploymentReport { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.reports) +} + +func (s *fakeSource) leaseBatches() []LeaseBatch { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.batches) +} + +// ---- agent fixture ---- + +type fixture struct { + agent *Agent + role *fakeRole + src *fakeSource + store *Store + signer ed25519.PrivateKey +} + +func testConfig(dir string) *Config { + return &Config{ + StateDir: dir, ApplyTimeout: 5 * time.Second, HealthTimeout: 300 * time.Millisecond, + LeaseInterval: time.Second, LeaseFullSync: time.Hour, + } +} + +func newFixture(t *testing.T, kind Kind) *fixture { + t.Helper() + store, err := OpenStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return newFixtureWithStore(t, kind, store, nil) +} + +func newFixtureWithStore(t *testing.T, kind Kind, store *Store, signer ed25519.PrivateKey) *fixture { + t.Helper() + if signer == nil { + _, signer = newSigningKey(t) + } + role := newFakeRole(kind) + src := newFakeSource() + id := &Identity{AgentID: testAgentID, Role: kind, BundleKey: signer.Public().(ed25519.PublicKey)} + a, err := NewAgent(kind, testConfig(store.Dir()), role, store, id, src, testLogger()) + if err != nil { + t.Fatal(err) + } + a.healthRetry = 20 * time.Millisecond + a.reportRetry = 50 * time.Millisecond + return &fixture{agent: a, role: role, src: src, store: store, signer: signer} +} + +// addBundle makes revision rev available from the fake source. +func (f *fixture) addBundle(t *testing.T, rev uint64) { + t.Helper() + f.src.mu.Lock() + defer f.src.mu.Unlock() + f.src.bundles[rev] = signBundle(t, f.signer, testAgentID, f.agent.kind, rev) +} + +func (f *fixture) pull(rev uint64) { + f.agent.handle(context.Background(), applyRequest{revision: rev, trigger: TriggerPull}) +} + +func (f *fixture) push(rev uint64, dep string) { + f.agent.handle(context.Background(), applyRequest{revision: rev, trigger: TriggerPush, deploymentID: dep}) +} + +func (f *fixture) lastResult(t *testing.T) DeploymentReport { + t.Helper() + st := f.agent.state() + if st.LastResult == nil { + t.Fatal("no result recorded") + } + return *st.LastResult +} + +// ---- fake gRPC server ---- + +type fakeServer struct { + agentv1.UnimplementedAgentServiceServer + pki *testPKI + signKey ed25519.PrivateKey + role Kind + + mu sync.Mutex + token string + agentID string + serial *big.Int + enrollValidity [2]time.Duration // NotBefore offset, NotAfter offset for Enroll + desired uint64 + mode agentv1.OperatingMode + pullInterval time.Duration + heartbeat time.Duration + streamDisabled bool + hellos []*agentv1.Hello + heartbeats int + acks []*agentv1.CommandAck + reports []*agentv1.ReportDeploymentRequest + leaseReports []*agentv1.ReportLeasesRequest + renewals int + polls int + commands chan *agentv1.Command + enrollments int +} + +func newFakeServer(t *testing.T, role Kind) *fakeServer { + t.Helper() + _, key := newSigningKey(t) + return &fakeServer{ + pki: newTestPKI(t), signKey: key, role: role, token: "0123456789abcdef.secret", + agentID: "11111111-2222-4333-8444-555555555555", + enrollValidity: [2]time.Duration{-time.Minute, 14 * 24 * time.Hour}, + mode: agentv1.OperatingMode_OPERATING_MODE_AUTO_PULL, + pullInterval: 50 * time.Millisecond, heartbeat: 50 * time.Millisecond, + commands: make(chan *agentv1.Command, 10), + } +} + +// start serves the fake on 127.0.0.1 with mTLS and returns its address. +func (f *fakeServer) start(t *testing.T) string { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + pool.AddCert(f.pki.ca) + srv := grpc.NewServer( + grpc.Creds(credentials.NewTLS(&tls.Config{ + MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{f.pki.serverCert}, + ClientAuth: tls.VerifyClientCertIfGiven, ClientCAs: pool, + })), + grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{MinTime: 10 * time.Second, PermitWithoutStream: true}), + ) + agentv1.RegisterAgentServiceServer(srv, f) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + return lis.Addr().String() +} + +func (f *fakeServer) fingerprint() string { return agentpki.Fingerprint(f.pki.ca) } + +func (f *fakeServer) authenticate(ctx context.Context) error { + p, ok := peer.FromContext(ctx) + if !ok { + return status.Error(codes.Unauthenticated, "no peer") + } + ti, ok := p.AuthInfo.(credentials.TLSInfo) + if !ok || len(ti.State.PeerCertificates) == 0 { + return status.Error(codes.Unauthenticated, "client certificate required") + } + cert := ti.State.PeerCertificates[0] + id, err := agentpki.AgentID(cert) + if err != nil { + return status.Error(codes.Unauthenticated, err.Error()) + } + f.mu.Lock() + defer f.mu.Unlock() + if id != f.agentID || f.serial == nil || cert.SerialNumber.Cmp(f.serial) != 0 { + return status.Error(codes.Unauthenticated, "unknown certificate") + } + return nil +} + +func (f *fakeServer) issue(csrDER []byte, notBefore, notAfter time.Time) (*x509.Certificate, error) { + csr, err := x509.ParseCertificateRequest(csrDER) + if err != nil { + return nil, err + } + if err := csr.CheckSignature(); err != nil { + return nil, err + } + f.mu.Lock() + id := f.agentID + f.mu.Unlock() + cert, err := f.pki.issueClient(csr.PublicKey, id, notBefore, notAfter) + if err != nil { + return nil, err + } + f.mu.Lock() + f.serial = cert.SerialNumber + f.mu.Unlock() + return cert, nil +} + +func (f *fakeServer) Enroll(_ context.Context, req *agentv1.EnrollRequest) (*agentv1.EnrollResponse, error) { + f.mu.Lock() + token, validity := f.token, f.enrollValidity + f.enrollments++ + f.mu.Unlock() + if req.GetToken() != token { + return nil, status.Error(codes.PermissionDenied, "invalid enrollment token") + } + want, _ := roleToProto(f.role) + if req.GetRole() != want { + return nil, status.Error(codes.InvalidArgument, "wrong role") + } + now := time.Now() + cert, err := f.issue(req.GetCsrDer(), now.Add(validity[0]), now.Add(validity[1])) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + f.mu.Lock() + defer f.mu.Unlock() + return &agentv1.EnrollResponse{ + AgentId: f.agentID, CertificateDer: cert.Raw, CaCertificatesDer: [][]byte{f.pki.ca.Raw}, + BundleSigningPublicKey: f.signKey.Public().(ed25519.PublicKey), + }, nil +} + +func (f *fakeServer) RenewCertificate(ctx context.Context, req *agentv1.RenewCertificateRequest) (*agentv1.RenewCertificateResponse, error) { + if err := f.authenticate(ctx); err != nil { + return nil, err + } + now := time.Now() + cert, err := f.issue(req.GetCsrDer(), now.Add(-time.Minute), now.Add(14*24*time.Hour)) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + f.mu.Lock() + f.renewals++ + f.mu.Unlock() + return &agentv1.RenewCertificateResponse{CertificateDer: cert.Raw, CaCertificatesDer: [][]byte{f.pki.ca.Raw}}, nil +} + +func (f *fakeServer) Connect(stream agentv1.AgentService_ConnectServer) error { + f.mu.Lock() + disabled := f.streamDisabled + f.mu.Unlock() + if disabled { + return status.Error(codes.Unimplemented, "streams disabled") + } + if err := f.authenticate(stream.Context()); err != nil { + return err + } + first, err := stream.Recv() + if err != nil { + return err + } + if first.GetHello() == nil { + return status.Error(codes.InvalidArgument, "Hello expected") + } + f.mu.Lock() + f.hellos = append(f.hellos, first.GetHello()) + w := &agentv1.Welcome{ + Mode: f.mode, PullInterval: durationpb.New(f.pullInterval), + HeartbeatInterval: durationpb.New(f.heartbeat), DesiredRevision: f.desired, + } + f.mu.Unlock() + if err := stream.Send(&agentv1.ConnectResponse{Message: &agentv1.ConnectResponse_Welcome{Welcome: w}}); err != nil { + return err + } + recvErr := make(chan error, 1) + go func() { + for { + msg, err := stream.Recv() + if err != nil { + recvErr <- err + return + } + f.mu.Lock() + switch { + case msg.GetHeartbeat() != nil: + f.heartbeats++ + case msg.GetCommandAck() != nil: + f.acks = append(f.acks, msg.GetCommandAck()) + } + f.mu.Unlock() + } + }() + for { + select { + case err := <-recvErr: + if errors.Is(err, io.EOF) { + return nil + } + return err + case <-stream.Context().Done(): + return nil + case c := <-f.commands: + if err := stream.Send(&agentv1.ConnectResponse{Message: &agentv1.ConnectResponse_Command{Command: c}}); err != nil { + return err + } + } + } +} + +func (f *fakeServer) GetDesiredState(ctx context.Context, _ *agentv1.GetDesiredStateRequest) (*agentv1.GetDesiredStateResponse, error) { + if err := f.authenticate(ctx); err != nil { + return nil, err + } + f.mu.Lock() + defer f.mu.Unlock() + f.polls++ + return &agentv1.GetDesiredStateResponse{DesiredRevision: f.desired, Mode: f.mode, PullInterval: durationpb.New(f.pullInterval)}, nil +} + +func (f *fakeServer) FetchBundle(ctx context.Context, req *agentv1.FetchBundleRequest) (*agentv1.FetchBundleResponse, error) { + if err := f.authenticate(ctx); err != nil { + return nil, err + } + f.mu.Lock() + id := f.agentID + f.mu.Unlock() + arts := testArtifacts(req.GetRevision()) + manifest, sig, err := bundle.Sign(f.signKey, id, string(f.role), req.GetRevision(), time.Unix(1_700_000_000, 0), arts) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + resp := &agentv1.FetchBundleResponse{Manifest: manifest, Signature: sig} + for _, a := range arts { + resp.Artifacts = append(resp.Artifacts, &agentv1.Artifact{Path: a.Path, Content: a.Content}) + } + return resp, nil +} + +func (f *fakeServer) ReportDeployment(ctx context.Context, req *agentv1.ReportDeploymentRequest) (*agentv1.ReportDeploymentResponse, error) { + if err := f.authenticate(ctx); err != nil { + return nil, err + } + f.mu.Lock() + defer f.mu.Unlock() + f.reports = append(f.reports, req) + return &agentv1.ReportDeploymentResponse{}, nil +} + +func (f *fakeServer) ReportLeases(ctx context.Context, req *agentv1.ReportLeasesRequest) (*agentv1.ReportLeasesResponse, error) { + if err := f.authenticate(ctx); err != nil { + return nil, err + } + f.mu.Lock() + defer f.mu.Unlock() + f.leaseReports = append(f.leaseReports, req) + return &agentv1.ReportLeasesResponse{AcknowledgedSequence: req.GetSequence()}, nil +} + +// snapshot returns a copy of the recorded data under the lock. +func (f *fakeServer) with(fn func(f *fakeServer)) { + f.mu.Lock() + defer f.mu.Unlock() + fn(f) +} diff --git a/agent/core/identity.go b/agent/core/identity.go new file mode 100644 index 0000000..ab8a3ff --- /dev/null +++ b/agent/core/identity.go @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: Apache-2.0 + +package core + +import ( + "crypto" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "time" + + "git.beugsiud.de/beug.k/DDI/internal/agentpki" +) + +// Files in identity/. +const ( + keyFile = "key.pem" + certFile = "cert.pem" + caFile = "ca.pem" + bundleKeyFile = "bundle-signing.pub" + agentFile = "agent.json" + newSuffix = ".new" +) + +// ErrNotEnrolled is returned by LoadIdentity if the agent has no identity. +var ErrNotEnrolled = errors.New("agent is not enrolled (run the enroll command first)") + +// errAlreadyEnrolled is returned by SaveIdentity without force. +var errAlreadyEnrolled = errors.New("agent is already enrolled (use --force to enroll again)") + +// Identity is the enrolled identity of the agent. +type Identity struct { + AgentID string + Role Kind + Server string // host:port of the agent API + ServerName string // TLS server name + EnrolledAt time.Time + CA *x509.Certificate + // Certificate holds the client certificate (Leaf set) and its key. + Certificate tls.Certificate + BundleKey ed25519.PublicKey +} + +// agentInfo is identity/agent.json. +type agentInfo struct { + AgentID string `json:"agent_id"` + Role Kind `json:"role"` + Server string `json:"server"` + ServerName string `json:"server_name"` + EnrolledAt time.Time `json:"enrolled_at"` + CAFingerprint string `json:"ca_fingerprint"` +} + +func (s *Store) identityPath(name string) string { + return filepath.Join(s.dir, identityDir, name) +} + +// Enrolled reports whether an identity exists. +func (s *Store) Enrolled() bool { + _, err := os.Stat(s.identityPath(agentFile)) + return err == nil +} + +// LoadIdentity reads identity/. It completes an interrupted certificate +// renewal (see SaveCertificate). +func (s *Store) LoadIdentity() (*Identity, error) { + data, err := os.ReadFile(s.identityPath(agentFile)) + if errors.Is(err, fs.ErrNotExist) { + return nil, ErrNotEnrolled + } + if err != nil { + return nil, fmt.Errorf("read identity: %w", err) + } + var info agentInfo + if err := json.Unmarshal(data, &info); err != nil { + return nil, fmt.Errorf("decode %s: %w", agentFile, err) + } + if !agentpki.ValidAgentID(info.AgentID) { + return nil, fmt.Errorf("%s holds an invalid agent ID", agentFile) + } + id := &Identity{ + AgentID: info.AgentID, Role: info.Role, Server: info.Server, + ServerName: info.ServerName, EnrolledAt: info.EnrolledAt, + } + if id.CA, err = readCertPEM(s.identityPath(caFile)); err != nil { + return nil, err + } + if id.BundleKey, err = readBundleKey(s.identityPath(bundleKeyFile)); err != nil { + return nil, err + } + if id.Certificate, err = s.loadKeyPair(); err != nil { + return nil, err + } + return id, nil +} + +// loadKeyPair loads key.pem and cert.pem. If they do not match and a +// cert.pem.new matching the key exists, a renewal was interrupted after the +// key was replaced; the new certificate is moved into place. +func (s *Store) loadKeyPair() (tls.Certificate, error) { + keyPEM, err := os.ReadFile(s.identityPath(keyFile)) + if err != nil { + return tls.Certificate{}, fmt.Errorf("read identity: %w", err) + } + certPEM, err := os.ReadFile(s.identityPath(certFile)) + if err != nil { + return tls.Certificate{}, fmt.Errorf("read identity: %w", err) + } + pair, err := tls.X509KeyPair(certPEM, keyPEM) + if err == nil { + s.removeStaleNew() + return pair, nil + } + newCert, nerr := os.ReadFile(s.identityPath(certFile + newSuffix)) + if nerr != nil { + return tls.Certificate{}, fmt.Errorf("load client certificate: %w", err) + } + pair, nerr = tls.X509KeyPair(newCert, keyPEM) + if nerr != nil { + return tls.Certificate{}, fmt.Errorf("load client certificate: %w", err) + } + if err := os.Rename(s.identityPath(certFile+newSuffix), s.identityPath(certFile)); err != nil { + return tls.Certificate{}, fmt.Errorf("complete certificate renewal: %w", err) + } + syncDir(filepath.Join(s.dir, identityDir)) + return pair, nil +} + +func (s *Store) removeStaleNew() { + _ = os.Remove(s.identityPath(keyFile + newSuffix)) + _ = os.Remove(s.identityPath(certFile + newSuffix)) +} + +// SaveIdentity writes a new identity directory. Without force it refuses to +// replace an existing identity. +func (s *Store) SaveIdentity(id *Identity, key crypto.Signer, force bool) error { + if s.Enrolled() && !force { + return errAlreadyEnrolled + } + keyPEM, err := encodeKey(key) + if err != nil { + return err + } + pubDER, err := x509.MarshalPKIXPublicKey(id.BundleKey) + if err != nil { + return fmt.Errorf("encode bundle signing key: %w", err) + } + info, err := json.MarshalIndent(agentInfo{ + AgentID: id.AgentID, Role: id.Role, Server: id.Server, ServerName: id.ServerName, + EnrolledAt: id.EnrolledAt.UTC(), CAFingerprint: agentpki.Fingerprint(id.CA), + }, "", " ") + if err != nil { + return fmt.Errorf("encode identity: %w", err) + } + tmp, err := os.MkdirTemp(s.dir, tmpPrefix+identityDir+"-") + if err != nil { + return fmt.Errorf("create identity directory: %w", err) + } + defer func() { _ = os.RemoveAll(tmp) }() + if err := os.Chmod(tmp, 0o700); err != nil { //nolint:gosec // directories need the search bit + return fmt.Errorf("create identity directory: %w", err) + } + files := []struct { + name string + data []byte + perm fs.FileMode + }{ + {keyFile, keyPEM, 0o600}, + {certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: id.Certificate.Leaf.Raw}), 0o644}, + {caFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: id.CA.Raw}), 0o644}, + {bundleKeyFile, pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: pubDER}), 0o644}, + {agentFile, append(info, '\n'), 0o644}, + } + for _, f := range files { + if err := writeFileSync(filepath.Join(tmp, f.name), f.data, f.perm); err != nil { + return err + } + } + syncDir(tmp) + if err := replaceDir(tmp, filepath.Join(s.dir, identityDir)); err != nil { + return err + } + syncDir(s.dir) + return nil +} + +// SaveCertificate stores a renewed key and certificate. Both are written as +// *.new first; key.pem is replaced before cert.pem, and LoadIdentity +// completes the second rename if the process died in between. +func (s *Store) SaveCertificate(key crypto.Signer, cert *x509.Certificate) error { + keyPEM, err := encodeKey(key) + if err != nil { + return err + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}) + keyNew, certNew := s.identityPath(keyFile+newSuffix), s.identityPath(certFile+newSuffix) + if err := writeFileAtomic(keyNew, keyPEM, 0o600); err != nil { + return err + } + if err := writeFileAtomic(certNew, certPEM, 0o644); err != nil { + return err + } + if err := os.Rename(keyNew, s.identityPath(keyFile)); err != nil { + return fmt.Errorf("replace key: %w", err) + } + if err := os.Rename(certNew, s.identityPath(certFile)); err != nil { + return fmt.Errorf("replace certificate: %w", err) + } + syncDir(filepath.Join(s.dir, identityDir)) + return nil +} + +func readCertPEM(path string) (*x509.Certificate, error) { + data, err := os.ReadFile(path) //nolint:gosec // path built from the state directory + if err != nil { + return nil, fmt.Errorf("read identity: %w", err) + } + block, _ := pem.Decode(data) + if block == nil || block.Type != "CERTIFICATE" { + return nil, fmt.Errorf("%s holds no certificate", filepath.Base(path)) + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, fmt.Errorf("parse %s: %w", filepath.Base(path), err) + } + return cert, nil +} + +func readBundleKey(path string) (ed25519.PublicKey, error) { + data, err := os.ReadFile(path) //nolint:gosec // path built from the state directory + if err != nil { + return nil, fmt.Errorf("read identity: %w", err) + } + block, _ := pem.Decode(data) + if block == nil || block.Type != "PUBLIC KEY" { + return nil, fmt.Errorf("%s holds no public key", filepath.Base(path)) + } + pub, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("parse %s: %w", filepath.Base(path), err) + } + key, ok := pub.(ed25519.PublicKey) + if !ok { + return nil, fmt.Errorf("%s is not an Ed25519 key", filepath.Base(path)) + } + return key, nil +} + +func encodeKey(key crypto.Signer) ([]byte, error) { + der, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, fmt.Errorf("encode private key: %w", err) + } + return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), nil +} + +// newKeyAndCSR generates an ECDSA P-256 key and a CSR for it. The server +// sets subject and SANs itself; commonName is informational. +func newKeyAndCSR(commonName string) (*ecdsa.PrivateKey, []byte, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, nil, fmt.Errorf("generate key: %w", err) + } + csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{ + Subject: pkix.Name{CommonName: commonName}, + }, key) + if err != nil { + return nil, nil, fmt.Errorf("create certificate request: %w", err) + } + return key, csr, nil +} + +// checkIssued verifies a certificate issued by the server: it must chain to +// ca for client authentication, carry agentID and certify key. +func checkIssued(der []byte, ca *x509.Certificate, agentID string, key *ecdsa.PrivateKey) (*x509.Certificate, error) { + cert, err := x509.ParseCertificate(der) + if err != nil { + return nil, fmt.Errorf("parse issued certificate: %w", err) + } + roots := x509.NewCertPool() + roots.AddCert(ca) + // Tolerate a server clock that is slightly ahead of ours. + at := time.Now() + if at.Before(cert.NotBefore) { + at = cert.NotBefore + } + if _, err := cert.Verify(x509.VerifyOptions{ + Roots: roots, CurrentTime: at, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }); err != nil { + return nil, fmt.Errorf("verify issued certificate: %w", err) + } + id, err := agentpki.AgentID(cert) + if err != nil { + return nil, err + } + if id != agentID { + return nil, fmt.Errorf("issued certificate is for agent %s, not %s", id, agentID) + } + pub, ok := cert.PublicKey.(*ecdsa.PublicKey) + if !ok || !pub.Equal(&key.PublicKey) { + return nil, errors.New("issued certificate does not certify the generated key") + } + return cert, nil +} + +// keyPair combines a key and its certificate. +func keyPair(key *ecdsa.PrivateKey, cert *x509.Certificate) tls.Certificate { + return tls.Certificate{Certificate: [][]byte{cert.Raw}, PrivateKey: key, Leaf: cert} +} diff --git a/agent/core/leases.go b/agent/core/leases.go new file mode 100644 index 0000000..ac3e5db --- /dev/null +++ b/agent/core/leases.go @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: Apache-2.0 + +package core + +import ( + "context" + "fmt" + "net/netip" + "slices" + "time" +) + +// maxLeasesPerBatch bounds one ReportLeases request (about 100 bytes per +// lease, far below the default gRPC message limit of 4 MiB). +const maxLeasesPerBatch = 5000 + +// leasesTimeout bounds one LeaseSource.Leases call. +const leasesTimeout = 30 * time.Second + +// leaseTracker remembers what the source was last told. +type leaseTracker struct { + // known holds the leases as last reported, by address. + known map[netip.Addr]Lease + // needFull forces the next report to be a full snapshot (at start, + // after a new connection and after a failed report). + needFull bool + lastFull time.Time +} + +// leaseLoop polls the role for leases and reports changes. A full snapshot +// is sent at start, after each new connection, after a failed report and +// every LeaseFullSync. +func (a *Agent) leaseLoop(ctx context.Context, ls LeaseSource) { + t := time.NewTicker(a.cfg.LeaseInterval) + defer t.Stop() + tr := &leaseTracker{needFull: true} + for { + a.pollLeases(ctx, ls, tr) + select { + case <-ctx.Done(): + return + case <-t.C: + case <-a.leaseCh: + tr.needFull = true + } + } +} + +// pollLeases runs one polling round. +func (a *Agent) pollLeases(ctx context.Context, ls LeaseSource, tr *leaseTracker) { + if !a.roleMu.TryLock() { + return // an apply is running; try again next round + } + lctx, cancel := context.WithTimeout(ctx, leasesTimeout) + cur, err := ls.Leases(lctx) + cancel() + a.roleMu.Unlock() + if err != nil { + if ctx.Err() == nil { + a.log.Warn("cannot read leases", "error", err) + } + return + } + full := tr.needFull || time.Since(tr.lastFull) >= a.cfg.LeaseFullSync + changes, next := diffLeases(tr.known, cur) + toSend := changes + if full { + toSend = sortedLeases(next) + } else if len(changes) == 0 { + return + } + if err := a.sendLeases(ctx, toSend, full); err != nil { + if ctx.Err() == nil { + a.log.Warn("cannot report leases; next report is a full snapshot", "error", err) + } + tr.needFull = true + return + } + tr.known = next + tr.needFull = false + if full { + tr.lastFull = time.Now() + } +} + +// sendLeases sends leases in batches of maxLeasesPerBatch. For a full +// snapshot only the first batch carries FullSync; the server replaces its +// view with it and applies the following batches as changes, so a snapshot +// split into several requests ends in the same state. If any batch fails, +// the caller sends a new full snapshot later. +func (a *Agent) sendLeases(ctx context.Context, leases []Lease, full bool) error { + first := true + for len(leases) > 0 || first { + n := min(len(leases), maxLeasesPerBatch) + seq, err := a.nextLeaseSequence() + if err != nil { + return err + } + b := LeaseBatch{Sequence: seq, FullSync: full && first, Leases: leases[:n]} + if err := a.source.ReportLeases(ctx, b); err != nil { + return err + } + a.log.Debug("leases reported", "sequence", seq, "full_sync", b.FullSync, "count", n) + leases = leases[n:] + first = false + } + return nil +} + +// nextLeaseSequence increments and persists the lease sequence before it is +// used, so that a restart never reuses a number. +func (a *Agent) nextLeaseSequence() (uint64, error) { + a.mu.Lock() + defer a.mu.Unlock() + a.st.LeaseSequence++ + a.st.UpdatedAt = time.Now().UTC() + if err := a.store.SaveState(a.st); err != nil { + return 0, fmt.Errorf("persist lease sequence: %w", err) + } + return a.st.LeaseSequence, nil +} + +// diffLeases compares the current leases with the known ones. It returns the +// changes (new or modified leases, and known leases that disappeared, marked +// expired) and the new known set. +func diffLeases(known map[netip.Addr]Lease, cur []Lease) ([]Lease, map[netip.Addr]Lease) { + next := make(map[netip.Addr]Lease, len(cur)) + for _, l := range cur { + if l.IP.IsValid() { + next[l.IP] = l + } + } + var changes []Lease + for ip, l := range next { + if old, ok := known[ip]; !ok || !leaseEqual(old, l) { + changes = append(changes, l) + } + } + for ip, l := range known { + if _, ok := next[ip]; !ok { + l.State = LeaseExpired + changes = append(changes, l) + } + } + slices.SortFunc(changes, func(x, y Lease) int { return x.IP.Compare(y.IP) }) + return changes, next +} + +func sortedLeases(m map[netip.Addr]Lease) []Lease { + out := make([]Lease, 0, len(m)) + for _, l := range m { + out = append(out, l) + } + slices.SortFunc(out, func(x, y Lease) int { return x.IP.Compare(y.IP) }) + return out +} + +func leaseEqual(x, y Lease) bool { + return x.IP == y.IP && x.MAC == y.MAC && x.Hostname == y.Hostname && x.State == y.State && + x.Expires.Equal(y.Expires) && x.Updated.Equal(y.Updated) && x.SubnetID == y.SubnetID && + x.ClientID == y.ClientID +} diff --git a/agent/core/leases_test.go b/agent/core/leases_test.go new file mode 100644 index 0000000..c26ff17 --- /dev/null +++ b/agent/core/leases_test.go @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: Apache-2.0 + +package core + +import ( + "context" + "errors" + "fmt" + "net/netip" + "testing" + "time" +) + +func mustAddr(s string) netip.Addr { return netip.MustParseAddr(s) } + +func lease(ip, mac string) Lease { + return Lease{ + IP: mustAddr(ip), MAC: mac, Hostname: "host-" + ip, State: LeaseActive, + Expires: time.Unix(1_800_000_000, 0), Updated: time.Unix(1_700_000_000, 0), SubnetID: 1, + } +} + +func TestDiffLeases(t *testing.T) { + a, b, c := lease("10.0.0.1", "aa:00:00:00:00:01"), lease("10.0.0.2", "aa:00:00:00:00:02"), lease("10.0.0.3", "aa:00:00:00:00:03") + changes, known := diffLeases(nil, []Lease{b, a, {}}) + if len(changes) != 2 || changes[0].IP != a.IP || changes[1].IP != b.IP || len(known) != 2 { + t.Fatalf("initial changes = %+v", changes) + } + if changes, _ := diffLeases(known, []Lease{a, b}); len(changes) != 0 { + t.Errorf("unchanged leases reported: %+v", changes) + } + b2 := b + b2.Expires = b.Expires.Add(time.Hour) + changes, next := diffLeases(known, []Lease{b2, c}) + if len(changes) != 3 { + t.Fatalf("changes = %+v", changes) + } + if changes[0].IP != a.IP || changes[0].State != LeaseExpired { + t.Errorf("disappeared lease = %+v", changes[0]) + } + if !changes[1].Expires.Equal(b2.Expires) || changes[2].IP != c.IP { + t.Errorf("changes = %+v", changes) + } + if len(next) != 2 { + t.Errorf("next = %v", next) + } + // Equal instants in different locations are not a change. + b3 := b2 + b3.Expires = b2.Expires.In(time.FixedZone("x", 3600)) + if changes, _ := diffLeases(next, []Lease{b3, c}); len(changes) != 0 { + t.Errorf("time zone change reported: %+v", changes) + } +} + +func TestPollLeases(t *testing.T) { + f := newFixture(t, KindDHCP) + ctx := context.Background() + tr := &leaseTracker{needFull: true} + a, b := lease("10.0.0.1", "aa:00:00:00:00:01"), lease("10.0.0.2", "aa:00:00:00:00:02") + f.role.setLeases([]Lease{a, b}) + + f.agent.pollLeases(ctx, f.role, tr) + batches := f.src.leaseBatches() + if len(batches) != 1 || !batches[0].FullSync || batches[0].Sequence != 1 || len(batches[0].Leases) != 2 { + t.Fatalf("batches = %+v", batches) + } + + // No change: nothing is sent. + f.agent.pollLeases(ctx, f.role, tr) + if n := len(f.src.leaseBatches()); n != 1 { + t.Fatalf("%d batches after an unchanged poll", n) + } + + // b released, a gone, c new: one delta. + b.State = LeaseReleased + c := lease("10.0.0.3", "aa:00:00:00:00:03") + f.role.setLeases([]Lease{b, c}) + f.agent.pollLeases(ctx, f.role, tr) + batches = f.src.leaseBatches() + if len(batches) != 2 { + t.Fatalf("batches = %+v", batches) + } + d := batches[1] + if d.FullSync || d.Sequence != 2 || len(d.Leases) != 3 || d.Leases[0].State != LeaseExpired || d.Leases[1].State != LeaseReleased { + t.Fatalf("delta = %+v", d) + } + + // A failed report forces a full snapshot next time. + f.src.mu.Lock() + f.src.leaseErr = errors.New("down") + f.src.mu.Unlock() + c.Hostname = "renamed" + f.role.setLeases([]Lease{b, c}) + f.agent.pollLeases(ctx, f.role, tr) + if !tr.needFull { + t.Fatal("failure did not request a full snapshot") + } + f.src.mu.Lock() + f.src.leaseErr = nil + f.src.mu.Unlock() + f.agent.pollLeases(ctx, f.role, tr) + batches = f.src.leaseBatches() + last := batches[len(batches)-1] + if !last.FullSync || len(last.Leases) != 2 || last.Sequence != 4 { + t.Fatalf("snapshot after failure = %+v", last) + } + + // The sequence survives a restart. + st, err := f.store.LoadState() + if err != nil { + t.Fatal(err) + } + if st.LeaseSequence != 4 { + t.Errorf("persisted sequence = %d", st.LeaseSequence) + } + + // The full sync interval triggers a snapshot. + tr.lastFull = time.Now().Add(-2 * time.Hour) + f.agent.pollLeases(ctx, f.role, tr) + batches = f.src.leaseBatches() + if last := batches[len(batches)-1]; !last.FullSync || last.Sequence != 5 { + t.Errorf("periodic snapshot = %+v", last) + } + + // Errors of the role and a busy role skip the round. + f.role.mu.Lock() + f.role.leasesErr = errors.New("socket gone") + f.role.mu.Unlock() + n := len(f.src.leaseBatches()) + tr.needFull = true + f.agent.pollLeases(ctx, f.role, tr) + f.role.mu.Lock() + f.role.leasesErr = nil + f.role.mu.Unlock() + f.agent.roleMu.Lock() + f.agent.pollLeases(ctx, f.role, tr) + f.agent.roleMu.Unlock() + if len(f.src.leaseBatches()) != n { + t.Error("leases reported despite an error or a running apply") + } +} + +func TestPollLeasesEmptySnapshot(t *testing.T) { + f := newFixture(t, KindDHCP) + tr := &leaseTracker{needFull: true} + f.agent.pollLeases(context.Background(), f.role, tr) + b := f.src.leaseBatches() + if len(b) != 1 || !b[0].FullSync || len(b[0].Leases) != 0 { + t.Fatalf("batches = %+v", b) + } +} + +func TestLeaseBatching(t *testing.T) { + f := newFixture(t, KindDHCP) + var leases []Lease + for i := 0; i < 2*maxLeasesPerBatch+1; i++ { + ip := netip.AddrFrom4([4]byte{10, byte(i >> 16), byte(i >> 8), byte(i)}) + leases = append(leases, Lease{IP: ip, MAC: fmt.Sprintf("aa:00:00:%02x:%02x:%02x", byte(i>>16), byte(i>>8), byte(i)), State: LeaseActive}) + } + f.role.setLeases(leases) + f.agent.pollLeases(context.Background(), f.role, &leaseTracker{needFull: true}) + b := f.src.leaseBatches() + if len(b) != 3 { + t.Fatalf("%d batches", len(b)) + } + if !b[0].FullSync || b[1].FullSync || b[2].FullSync { + t.Errorf("full_sync flags = %v %v %v", b[0].FullSync, b[1].FullSync, b[2].FullSync) + } + if len(b[0].Leases) != maxLeasesPerBatch || len(b[2].Leases) != 1 || b[0].Sequence != 1 || b[2].Sequence != 3 { + t.Errorf("batches: %d/%d/%d leases, sequences %d..%d", len(b[0].Leases), len(b[1].Leases), len(b[2].Leases), b[0].Sequence, b[2].Sequence) + } +} + +func TestLeaseLoopNotStartedForDNS(t *testing.T) { + f := newFixture(t, KindDNS) + role := dnsOnlyRole{f.role} + a, err := NewAgent(KindDNS, f.agent.cfg, role, f.store, &Identity{AgentID: testAgentID, BundleKey: f.agent.bundleKey}, f.src, testLogger()) + if err != nil { + t.Fatal(err) + } + if _, ok := a.role.(LeaseSource); ok { + t.Fatal("DNS test role implements LeaseSource") + } + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + if err := a.Run(ctx); err != nil { + t.Fatal(err) + } + if n := len(f.src.leaseBatches()); n != 0 { + t.Errorf("%d lease batches from a DNS agent", n) + } +} diff --git a/agent/core/lock_flock.go b/agent/core/lock_flock.go new file mode 100644 index 0000000..03bdfb8 --- /dev/null +++ b/agent/core/lock_flock.go @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: Apache-2.0 + +//go:build linux || darwin || freebsd || netbsd || openbsd || dragonfly + +package core + +import ( + "errors" + "fmt" + "os" + "syscall" +) + +// lockPath takes an exclusive, non-blocking flock on path. The lock is +// released by the returned function or when the process exits. +func lockPath(path string) (func(), error) { + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600) //nolint:gosec // path built from the state directory + if err != nil { + return nil, fmt.Errorf("open lock file: %w", err) + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { //nolint:gosec // file descriptors fit into int + _ = f.Close() + if errors.Is(err, syscall.EWOULDBLOCK) { + return nil, fmt.Errorf("state directory is locked by another agent process (%s)", path) + } + return nil, fmt.Errorf("lock state directory: %w", err) + } + return func() { + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) //nolint:gosec // file descriptors fit into int + _ = f.Close() + }, nil +} diff --git a/agent/core/lock_other.go b/agent/core/lock_other.go new file mode 100644 index 0000000..7f734f4 --- /dev/null +++ b/agent/core/lock_other.go @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 + +//go:build !(linux || darwin || freebsd || netbsd || openbsd || dragonfly) + +package core + +// lockPath is a no-op on platforms without flock. The agents only run on +// Linux; this keeps the package buildable elsewhere (e.g. for development). +func lockPath(string) (func(), error) { + return func() {}, nil +} diff --git a/agent/core/server_source.go b/agent/core/server_source.go new file mode 100644 index 0000000..95c600a --- /dev/null +++ b/agent/core/server_source.go @@ -0,0 +1,599 @@ +// SPDX-License-Identifier: Apache-2.0 + +package core + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "log/slog" + "math/rand/v2" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/keepalive" + "google.golang.org/protobuf/types/known/timestamppb" + + "git.beugsiud.de/beug.k/DDI/internal/agentpki" + "git.beugsiud.de/beug.k/DDI/internal/buildinfo" + "git.beugsiud.de/beug.k/DDI/internal/bundle" + agentv1 "git.beugsiud.de/beug.k/DDI/proto/gen/ddi/agent/v1" +) + +// Defaults of the server connection (docs/adr/0021-agent-strecke-m3.md). +const ( + defaultPullInterval = 60 * time.Second + defaultHeartbeatInterval = 30 * time.Second + keepaliveInterval = 30 * time.Second + keepaliveTimeout = 10 * time.Second + backoffMin = time.Second + backoffMax = 60 * time.Second + welcomeTimeout = 30 * time.Second + rpcTimeout = 30 * time.Second + // renewRetryInterval spaces failed renewal attempts (also with an + // expired certificate, which the server accepts for 30 days). + renewRetryInterval = 15 * time.Minute + // oldConnGrace keeps the connection of a replaced certificate open for + // calls in flight. + oldConnGrace = 30 * time.Second + // maxBundleMessage allows bundles up to the bundle size limit. + maxBundleMessage = bundle.MaxTotalSize + 16<<20 +) + +// errReconnect ends a session after the certificate was renewed. +var errReconnect = errors.New("reconnecting with the renewed certificate") + +// ServerSource is the Source backed by the management server: an mTLS gRPC +// connection with the Connect control stream and GetDesiredState polling as +// fallback (docs/adr/0007-server-agent-grpc-mtls.md). It also renews the +// client certificate. +type ServerSource struct { + store *Store + agentID string + ca *x509.Certificate + server string + serverName string + hostname string + log *slog.Logger + + // Timing, shortened by tests. + backoffMin, backoffMax time.Duration + welcomeTimeout time.Duration + renewRetry time.Duration + // dialOptions are appended to the defaults (tests). + dialOptions []grpc.DialOption + + mu sync.Mutex + cert tls.Certificate + conn *grpc.ClientConn + pullInterval time.Duration + heartbeatInterval time.Duration + lastRenewTry time.Time + + renewMu sync.Mutex +} + +// NewServerSource creates the source for an enrolled identity. +func NewServerSource(store *Store, id *Identity, hostname string, log *slog.Logger) *ServerSource { + return &ServerSource{ + store: store, agentID: id.AgentID, ca: id.CA, server: id.Server, serverName: id.ServerName, + hostname: hostname, log: log, cert: id.Certificate, + backoffMin: backoffMin, backoffMax: backoffMax, welcomeTimeout: welcomeTimeout, + renewRetry: renewRetryInterval, pullInterval: defaultPullInterval, + heartbeatInterval: defaultHeartbeatInterval, + } +} + +// client returns the gRPC client, creating the connection on first use. +// Connections are established lazily by gRPC. +func (s *ServerSource) client() (agentv1.AgentServiceClient, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.conn == nil { + opts := append([]grpc.DialOption{ + grpc.WithTransportCredentials(credentials.NewTLS(agentpki.ClientTLSConfig(s.ca, s.cert, s.serverName))), + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: keepaliveInterval, Timeout: keepaliveTimeout, PermitWithoutStream: true, + }), + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxBundleMessage)), + }, s.dialOptions...) + conn, err := grpc.NewClient(s.server, opts...) + if err != nil { + return nil, fmt.Errorf("connect to %s: %w", s.server, err) + } + s.conn = conn + } + return agentv1.NewAgentServiceClient(s.conn), nil +} + +// replaceConn drops the current connection (after a certificate renewal); +// the next call dials with the new certificate. +func (s *ServerSource) replaceConn() { + s.mu.Lock() + old := s.conn + s.conn = nil + s.mu.Unlock() + if old != nil { + time.AfterFunc(oldConnGrace, func() { _ = old.Close() }) + } +} + +func (s *ServerSource) closeConn() { + s.mu.Lock() + defer s.mu.Unlock() + if s.conn != nil { + _ = s.conn.Close() + s.conn = nil + } +} + +func (s *ServerSource) intervals() (pull, heartbeat time.Duration) { + s.mu.Lock() + defer s.mu.Unlock() + return s.pullInterval, s.heartbeatInterval +} + +func (s *ServerSource) setIntervals(pull, heartbeat time.Duration) { + s.mu.Lock() + defer s.mu.Unlock() + if pull > 0 { + s.pullInterval = pull + } + if heartbeat > 0 { + s.heartbeatInterval = heartbeat + } +} + +// Run implements Source: it keeps a Connect stream open, reconnecting with +// exponential backoff and jitter, and polls GetDesiredState every pull +// interval while no stream can be established. +func (s *ServerSource) Run(ctx context.Context, h SourceHandler) error { + defer s.closeConn() + backoff := s.backoffMin + var lastPoll time.Time + reachable := false + for ctx.Err() == nil { + s.maybeRenew(ctx) + established, err := s.session(ctx, h) + if ctx.Err() != nil { + break + } + wait := true + switch { + case errors.Is(err, errReconnect): + wait = false + backoff = s.backoffMin + case established: + s.log.Warn("control stream to the server closed", "error", err) + backoff = s.backoffMin + reachable = false + default: + s.log.Warn("cannot open control stream", "server", s.server, "error", err) + pull, _ := s.intervals() + if time.Since(lastPoll) >= pull { + lastPoll = time.Now() + if err := s.poll(ctx, h); err != nil { + s.log.Warn("cannot poll the desired state", "error", err) + reachable = false + } else if !reachable { + reachable = true + h.Connected() + } + } + } + if wait { + if !sleepCtx(ctx, jitter(backoff)) { + break + } + backoff = min(backoff*2, s.backoffMax) + } + } + return ctx.Err() +} + +// session runs one Connect stream. established reports whether the server +// answered the Hello with a Welcome. +func (s *ServerSource) session(ctx context.Context, h SourceHandler) (established bool, err error) { + cl, err := s.client() + if err != nil { + return false, err + } + sctx, cancel := context.WithCancel(ctx) + defer cancel() + stream, err := cl.Connect(sctx) + if err != nil { + return false, err + } + defer func() { _ = stream.CloseSend() }() + + st := h.Status(sctx) + if err := stream.Send(&agentv1.ConnectRequest{Message: &agentv1.ConnectRequest_Hello{Hello: &agentv1.Hello{ + AgentVersion: buildinfo.Get().Version, AppliedRevision: st.Applied, LastGoodRevision: st.LastGood, + Health: healthToProto(st.Health, st.CheckedAt), ServiceVersion: st.ServiceVersion, Hostname: s.hostname, + }}}); err != nil { + return false, err + } + + msgs := make(chan *agentv1.ConnectResponse) + recvErr := make(chan error, 1) + go func() { + for { + msg, err := stream.Recv() + if err != nil { + recvErr <- err + return + } + select { + case msgs <- msg: + case <-sctx.Done(): + return + } + } + }() + + timer := time.NewTimer(s.welcomeTimeout) + defer timer.Stop() + select { + case <-ctx.Done(): + return false, ctx.Err() + case err := <-recvErr: + return false, err + case <-timer.C: + return false, errors.New("server did not answer the Hello in time") + case msg := <-msgs: + w := msg.GetWelcome() + if w == nil { + return false, errors.New("server did not answer the Hello with a Welcome") + } + s.log.Info("connected to the server", "server", s.server, "mode", w.GetMode().String(), + "desired_revision", w.GetDesiredRevision()) + s.welcome(w, h, true) + } + + _, hbInterval := s.intervals() + heartbeat := time.NewTicker(hbInterval) + defer heartbeat.Stop() + renew := time.NewTimer(s.untilRenewal()) + defer renew.Stop() + for { + select { + case <-ctx.Done(): + return true, ctx.Err() + case err := <-recvErr: + return true, err + case msg := <-msgs: + switch m := msg.GetMessage().(type) { + case *agentv1.ConnectResponse_Welcome: + s.welcome(m.Welcome, h, false) + _, hb := s.intervals() + heartbeat.Reset(hb) + case *agentv1.ConnectResponse_Command: + if err := s.command(ctx, stream, m.Command, h); err != nil { + return true, err + } + } + case <-heartbeat.C: + st := h.Status(sctx) + if err := stream.Send(&agentv1.ConnectRequest{Message: &agentv1.ConnectRequest_Heartbeat{Heartbeat: &agentv1.Heartbeat{ + AppliedRevision: st.Applied, LastGoodRevision: st.LastGood, Health: healthToProto(st.Health, st.CheckedAt), + }}}); err != nil { + return true, err + } + case <-renew.C: + if s.maybeRenew(ctx) { + return true, errReconnect + } + renew.Reset(s.untilRenewal()) + } + } +} + +// welcome applies the settings of a Welcome and announces its desired +// revision. +func (s *ServerSource) welcome(w *agentv1.Welcome, h SourceHandler, first bool) { + s.setIntervals(w.GetPullInterval().AsDuration(), w.GetHeartbeatInterval().AsDuration()) + if first { + h.Connected() + } + h.Desired(w.GetDesiredRevision(), w.GetMode() != agentv1.OperatingMode_OPERATING_MODE_MANUAL) +} + +// command acknowledges and executes a server command. +func (s *ServerSource) command(ctx context.Context, stream agentv1.AgentService_ConnectClient, c *agentv1.Command, h SourceHandler) error { + ack := func(accepted bool, msg string) error { + return stream.Send(&agentv1.ConnectRequest{Message: &agentv1.ConnectRequest_CommandAck{CommandAck: &agentv1.CommandAck{ + CommandId: c.GetCommandId(), Accepted: accepted, Message: msg, + }}}) + } + log := s.log.With("command_id", c.GetCommandId()) + switch a := c.GetAction().(type) { + case *agentv1.Command_ApplyRevision: + rev, dep := a.ApplyRevision.GetRevision(), a.ApplyRevision.GetDeploymentId() + if rev == 0 { + log.Warn("apply command without revision") + return ack(false, "revision is missing") + } + log.Info("apply command received", "revision", rev, "deployment_id", dep) + if err := ack(true, ""); err != nil { + return err + } + h.Push(rev, dep) + return nil + case *agentv1.Command_RenewCertificate: + log.Info("certificate renewal requested by the server") + if err := ack(true, ""); err != nil { + return err + } + if err := s.renew(ctx); err != nil { + log.Error("certificate renewal failed", "error", err) + return nil + } + return errReconnect + default: + log.Warn("unknown command") + return ack(false, "unknown command") + } +} + +// poll asks for the desired state once. +func (s *ServerSource) poll(ctx context.Context, h SourceHandler) error { + cl, err := s.client() + if err != nil { + return err + } + st := h.Status(ctx) + rctx, cancel := context.WithTimeout(ctx, rpcTimeout) + defer cancel() + resp, err := cl.GetDesiredState(rctx, &agentv1.GetDesiredStateRequest{AppliedRevision: st.Applied}) + if err != nil { + return err + } + s.setIntervals(resp.GetPullInterval().AsDuration(), 0) + s.log.Debug("desired state polled", "desired_revision", resp.GetDesiredRevision(), "mode", resp.GetMode().String()) + h.Desired(resp.GetDesiredRevision(), resp.GetMode() != agentv1.OperatingMode_OPERATING_MODE_MANUAL) + return nil +} + +// leaf returns the current client certificate. +func (s *ServerSource) leaf() *x509.Certificate { + s.mu.Lock() + defer s.mu.Unlock() + return s.cert.Leaf +} + +// renewalDue returns when the certificate should be renewed: when half of +// its lifetime has passed. +func renewalDue(c *x509.Certificate) time.Time { + return c.NotBefore.Add(c.NotAfter.Sub(c.NotBefore) / 2) +} + +// untilRenewal returns the time until the next renewal attempt. +func (s *ServerSource) untilRenewal() time.Duration { + due := renewalDue(s.leaf()) + s.mu.Lock() + last := s.lastRenewTry + s.mu.Unlock() + if !last.IsZero() && time.Now().After(due) { + due = last.Add(s.renewRetry) + } + return max(time.Until(due), 0) +} + +// maybeRenew renews the certificate if half of its lifetime has passed +// (or it expired), spacing failed attempts by renewRetry. +func (s *ServerSource) maybeRenew(ctx context.Context) bool { + leaf := s.leaf() + now := time.Now() + if now.Before(renewalDue(leaf)) { + return false + } + s.mu.Lock() + if !s.lastRenewTry.IsZero() && now.Sub(s.lastRenewTry) < s.renewRetry { + s.mu.Unlock() + return false + } + s.lastRenewTry = now + s.mu.Unlock() + if err := s.renew(ctx); err != nil { + if now.After(leaf.NotAfter) { + s.log.Error("client certificate expired and renewal failed; the agent must be enrolled again if this persists", + "not_after", leaf.NotAfter, "error", err) + } else { + s.log.Warn("certificate renewal failed", "not_after", leaf.NotAfter, "error", err) + } + return false + } + return true +} + +// renew obtains a certificate for a new key and stores it. +func (s *ServerSource) renew(ctx context.Context) error { + s.renewMu.Lock() + defer s.renewMu.Unlock() + key, csr, err := newKeyAndCSR(s.agentID) + if err != nil { + return err + } + cl, err := s.client() + if err != nil { + return err + } + rctx, cancel := context.WithTimeout(ctx, rpcTimeout) + defer cancel() + resp, err := cl.RenewCertificate(rctx, &agentv1.RenewCertificateRequest{CsrDer: csr}) + if err != nil { + return err + } + if chain := resp.GetCaCertificatesDer(); len(chain) > 0 { + if root, err := x509.ParseCertificate(chain[len(chain)-1]); err != nil || !root.Equal(s.ca) { + s.log.Warn("server sent a different CA certificate; keeping the enrolled CA") + } + } + cert, err := checkIssued(resp.GetCertificateDer(), s.ca, s.agentID, key) + if err != nil { + return err + } + if err := s.store.SaveCertificate(key, cert); err != nil { + return err + } + s.mu.Lock() + s.cert = keyPair(key, cert) + s.lastRenewTry = time.Time{} + s.mu.Unlock() + s.replaceConn() + s.log.Info("client certificate renewed", "not_after", cert.NotAfter) + return nil +} + +// FetchBundle implements Source. +func (s *ServerSource) FetchBundle(ctx context.Context, revision uint64) (*Bundle, error) { + cl, err := s.client() + if err != nil { + return nil, err + } + resp, err := cl.FetchBundle(ctx, &agentv1.FetchBundleRequest{Revision: revision}) + if err != nil { + return nil, err + } + b := &Bundle{Manifest: resp.GetManifest(), Signature: resp.GetSignature()} + for _, a := range resp.GetArtifacts() { + b.Artifacts = append(b.Artifacts, BundleArtifact{Path: a.GetPath(), Content: a.GetContent()}) + } + return b, nil +} + +// ReportDeployment implements Source. +func (s *ServerSource) ReportDeployment(ctx context.Context, r DeploymentReport) error { + cl, err := s.client() + if err != nil { + return err + } + _, err = cl.ReportDeployment(ctx, &agentv1.ReportDeploymentRequest{ + DeploymentId: r.DeploymentID, Revision: r.Revision, Trigger: triggerToProto(r.Trigger), + Outcome: outcomeToProto(r.Outcome), Message: r.Message, ActiveRevision: r.ActiveRevision, + StartedAt: timestamppb.New(r.StartedAt), FinishedAt: timestamppb.New(r.FinishedAt), + }) + return err +} + +// ReportLeases implements Source. +func (s *ServerSource) ReportLeases(ctx context.Context, b LeaseBatch) error { + cl, err := s.client() + if err != nil { + return err + } + req := &agentv1.ReportLeasesRequest{Sequence: b.Sequence, FullSync: b.FullSync, Leases: make([]*agentv1.Lease, len(b.Leases))} + for i, l := range b.Leases { + req.Leases[i] = leaseToProto(l) + } + rctx, cancel := context.WithTimeout(ctx, rpcTimeout) + defer cancel() + resp, err := cl.ReportLeases(rctx, req) + if err != nil { + return err + } + if ack := resp.GetAcknowledgedSequence(); ack != b.Sequence { + s.log.Debug("lease report acknowledged with another sequence", "sequence", b.Sequence, "acknowledged", ack) + } + return nil +} + +func healthToProto(h Health, at time.Time) *agentv1.HealthStatus { + out := &agentv1.HealthStatus{State: healthStateToProto(h.State), Message: h.Message} + if !at.IsZero() { + out.CheckedAt = timestamppb.New(at) + } + for _, c := range h.Checks { + out.Checks = append(out.Checks, &agentv1.HealthCheck{Name: c.Name, State: healthStateToProto(c.State), Message: c.Message}) + } + return out +} + +func healthStateToProto(s HealthState) agentv1.HealthState { + switch s { + case HealthOK: + return agentv1.HealthState_HEALTH_STATE_OK + case HealthDegraded: + return agentv1.HealthState_HEALTH_STATE_DEGRADED + case HealthFailed: + return agentv1.HealthState_HEALTH_STATE_FAILED + default: + return agentv1.HealthState_HEALTH_STATE_UNSPECIFIED + } +} + +func triggerToProto(t Trigger) agentv1.DeploymentTrigger { + switch t { + case TriggerPull: + return agentv1.DeploymentTrigger_DEPLOYMENT_TRIGGER_PULL + case TriggerPush: + return agentv1.DeploymentTrigger_DEPLOYMENT_TRIGGER_PUSH + default: + return agentv1.DeploymentTrigger_DEPLOYMENT_TRIGGER_UNSPECIFIED + } +} + +func outcomeToProto(o Outcome) agentv1.DeploymentOutcome { + switch o { + case OutcomeSucceeded: + return agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_SUCCEEDED + case OutcomeRejected: + return agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_REJECTED + case OutcomeRolledBack: + return agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_ROLLED_BACK + case OutcomeFailed: + return agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_FAILED + default: + return agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_UNSPECIFIED + } +} + +func leaseToProto(l Lease) *agentv1.Lease { + out := &agentv1.Lease{ + Ip: l.IP.String(), Mac: l.MAC, Hostname: l.Hostname, State: leaseStateToProto(l.State), + SubnetId: l.SubnetID, ClientId: l.ClientID, + } + if !l.Expires.IsZero() { + out.ExpiresAt = timestamppb.New(l.Expires) + } + if !l.Updated.IsZero() { + out.UpdatedAt = timestamppb.New(l.Updated) + } + return out +} + +func leaseStateToProto(s LeaseState) agentv1.LeaseState { + switch s { + case LeaseActive: + return agentv1.LeaseState_LEASE_STATE_ACTIVE + case LeaseExpired: + return agentv1.LeaseState_LEASE_STATE_EXPIRED + case LeaseReleased: + return agentv1.LeaseState_LEASE_STATE_RELEASED + case LeaseDeclined: + return agentv1.LeaseState_LEASE_STATE_DECLINED + default: + return agentv1.LeaseState_LEASE_STATE_UNSPECIFIED + } +} + +// jitter returns d randomized by ±20 %. +func jitter(d time.Duration) time.Duration { + return time.Duration(float64(d) * (0.8 + 0.4*rand.Float64())) //nolint:gosec // jitter needs no cryptographic randomness +} + +// sleepCtx waits for d; it returns false if ctx ended first. +func sleepCtx(ctx context.Context, d time.Duration) bool { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return false + case <-t.C: + return true + } +} diff --git a/agent/core/server_source_test.go b/agent/core/server_source_test.go new file mode 100644 index 0000000..48819d0 --- /dev/null +++ b/agent/core/server_source_test.go @@ -0,0 +1,412 @@ +// SPDX-License-Identifier: Apache-2.0 + +package core + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "git.beugsiud.de/beug.k/DDI/internal/agentpki" + agentv1 "git.beugsiud.de/beug.k/DDI/proto/gen/ddi/agent/v1" +) + +// enrollWith enrolls a new store against the fake server. +func enrollWith(t *testing.T, srv *fakeServer, addr string) (*Store, *Identity) { + t.Helper() + store := openTestStore(t) + id, err := Enroll(context.Background(), store, EnrollOptions{ + Kind: srv.role, Server: addr, Token: srv.token, CAHash: srv.fingerprint(), + ServerName: "localhost", Hostname: "test-host", + }) + if err != nil { + t.Fatalf("Enroll: %v", err) + } + return store, id +} + +func TestEnroll(t *testing.T) { + srv := newFakeServer(t, KindDNS) + addr := srv.start(t) + store, id := enrollWith(t, srv, addr) + if id.AgentID != srv.agentID || id.Server != addr || id.ServerName != "localhost" || !id.CA.Equal(srv.pki.ca) { + t.Fatalf("identity = %+v", id) + } + loaded, err := store.LoadIdentity() + if err != nil { + t.Fatal(err) + } + if loaded.AgentID != srv.agentID || !loaded.BundleKey.Equal(srv.signKey.Public()) || loaded.Role != KindDNS { + t.Errorf("stored identity = %+v", loaded) + } + if got, err := agentpki.AgentID(loaded.Certificate.Leaf); err != nil || got != srv.agentID { + t.Errorf("certificate agent ID = %q, %v", got, err) + } + + // Enrolling again needs --force. + opts := EnrollOptions{Kind: KindDNS, Server: addr, Token: srv.token, CAHash: srv.fingerprint(), ServerName: "localhost"} + if _, err := Enroll(context.Background(), store, opts); !errors.Is(err, errAlreadyEnrolled) { + t.Fatalf("second Enroll = %v", err) + } + srv.with(func(f *fakeServer) { f.agentID = "22222222-2222-4333-8444-555555555555" }) + st := &State{Applied: 3, PendingReports: []DeploymentReport{{Revision: 3}}} + if err := store.SaveState(st); err != nil { + t.Fatal(err) + } + opts.Force = true + id2, err := Enroll(context.Background(), store, opts) + if err != nil || id2.AgentID != "22222222-2222-4333-8444-555555555555" { + t.Fatalf("forced Enroll = %+v, %v", id2, err) + } + if st, _ := store.LoadState(); st.Applied != 3 || len(st.PendingReports) != 0 { + t.Errorf("state after forced enroll = %+v", st) + } +} + +func TestEnrollDefaultsServerNameToHost(t *testing.T) { + srv := newFakeServer(t, KindDHCP) + addr := srv.start(t) // 127.0.0.1:port; the server certificate has the IP SAN + store := openTestStore(t) + id, err := Enroll(context.Background(), store, EnrollOptions{ + Kind: KindDHCP, Server: "https://" + addr + "/", Token: srv.token, CAHash: strings.ToUpper(srv.fingerprint()[7:]), + }) + if err == nil { + t.Fatalf("fingerprint without prefix accepted: %+v", id) + } + id, err = Enroll(context.Background(), store, EnrollOptions{ + Kind: KindDHCP, Server: "https://" + addr + "/", Token: srv.token, CAHash: "sha256:" + strings.ToUpper(srv.fingerprint()[7:]), + }) + if err != nil { + t.Fatal(err) + } + if id.ServerName != "127.0.0.1" || id.Server != addr { + t.Errorf("identity = %+v", id) + } +} + +func TestEnrollFailures(t *testing.T) { + srv := newFakeServer(t, KindDNS) + addr := srv.start(t) + other := newTestPKI(t) + cases := map[string]EnrollOptions{ + "wrong fingerprint": {Server: addr, Token: srv.token, CAHash: agentpki.Fingerprint(other.ca), ServerName: "localhost"}, + "wrong server name": {Server: addr, Token: srv.token, CAHash: srv.fingerprint(), ServerName: "evil.example.com"}, + "wrong token": {Server: addr, Token: "0123456789abcdef.wrong", CAHash: srv.fingerprint(), ServerName: "localhost"}, + "bad token": {Server: addr, Token: "nodot", CAHash: srv.fingerprint()}, + "bad address": {Server: "localhost", Token: srv.token, CAHash: srv.fingerprint()}, + "bad fingerprint": {Server: addr, Token: srv.token, CAHash: "sha256:abc"}, + } + for name, opts := range cases { + t.Run(name, func(t *testing.T) { + opts.Kind = KindDNS + store := openTestStore(t) + if _, err := Enroll(context.Background(), store, opts); err == nil { + t.Fatal("accepted") + } + if store.Enrolled() { + t.Error("identity stored after a failed enrollment") + } + }) + } + // Role mismatch: the server refuses. + store := openTestStore(t) + if _, err := Enroll(context.Background(), store, EnrollOptions{ + Kind: KindDHCP, Server: addr, Token: srv.token, CAHash: srv.fingerprint(), ServerName: "localhost", + }); err == nil { + t.Error("wrong role accepted") + } + if _, err := Enroll(context.Background(), store, EnrollOptions{ + Kind: Kind("ntp"), Server: addr, Token: srv.token, CAHash: srv.fingerprint(), + }); err == nil { + t.Error("unknown role accepted") + } + srv.with(func(f *fakeServer) { + if f.enrollments == 0 { + t.Error("server never reached") + } + }) +} + +// runAgentAgainst runs an agent with a ServerSource until the test ends. +func runAgentAgainst(t *testing.T, store *Store, id *Identity, kind Kind) (*fakeRole, *ServerSource) { + t.Helper() + role := newFakeRole(kind) + src := NewServerSource(store, id, "test-host", testLogger()) + src.backoffMin, src.backoffMax = 10*time.Millisecond, 100*time.Millisecond + a, err := NewAgent(kind, testConfig(store.Dir()), role, store, id, src, testLogger()) + if err != nil { + t.Fatal(err) + } + a.healthRetry = 20 * time.Millisecond + a.reportRetry = 50 * time.Millisecond + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- a.Run(ctx) }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + if err != nil { + t.Errorf("Run = %v", err) + } + case <-time.After(10 * time.Second): + t.Error("Run did not return after cancel") + } + }) + return role, src +} + +func serverReports(srv *fakeServer) []*agentv1.ReportDeploymentRequest { + var out []*agentv1.ReportDeploymentRequest + srv.with(func(f *fakeServer) { out = append(out, f.reports...) }) + return out +} + +func TestServerSourceStreamAndPush(t *testing.T) { + srv := newFakeServer(t, KindDNS) + addr := srv.start(t) + store, id := enrollWith(t, srv, addr) + srv.with(func(f *fakeServer) { f.desired = 1 }) + role, _ := runAgentAgainst(t, store, id, KindDNS) + + waitFor(t, "report of revision 1", func() bool { return len(serverReports(srv)) == 1 }) + r := serverReports(srv)[0] + if r.GetRevision() != 1 || r.GetOutcome() != agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_SUCCEEDED || + r.GetTrigger() != agentv1.DeploymentTrigger_DEPLOYMENT_TRIGGER_PULL || r.GetActiveRevision() != 1 || + r.GetStartedAt() == nil || r.GetFinishedAt() == nil { + t.Fatalf("report = %v", r) + } + srv.with(func(f *fakeServer) { + if len(f.hellos) == 0 || f.hellos[0].GetHostname() != "test-host" || f.hellos[0].GetAgentVersion() == "" { + t.Errorf("hellos = %v", f.hellos) + } + }) + waitFor(t, "heartbeats", func() bool { + n := 0 + srv.with(func(f *fakeServer) { n = f.heartbeats }) + return n >= 2 + }) + + // Push over the stream. + srv.commands <- &agentv1.Command{CommandId: "c1", Action: &agentv1.Command_ApplyRevision{ + ApplyRevision: &agentv1.ApplyRevisionCommand{Revision: 2, DeploymentId: "dep-2"}, + }} + waitFor(t, "report of revision 2", func() bool { return len(serverReports(srv)) == 2 }) + r = serverReports(srv)[1] + if r.GetRevision() != 2 || r.GetDeploymentId() != "dep-2" || r.GetTrigger() != agentv1.DeploymentTrigger_DEPLOYMENT_TRIGGER_PUSH || + r.GetOutcome() != agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_SUCCEEDED { + t.Fatalf("report = %v", r) + } + srv.with(func(f *fakeServer) { + if len(f.acks) != 1 || f.acks[0].GetCommandId() != "c1" || !f.acks[0].GetAccepted() { + t.Errorf("acks = %v", f.acks) + } + }) + if n := role.activeRevision(); n != 2 { + t.Errorf("active = %d", n) + } + + // Invalid and unknown commands are refused. + srv.commands <- &agentv1.Command{CommandId: "c2", Action: &agentv1.Command_ApplyRevision{ApplyRevision: &agentv1.ApplyRevisionCommand{}}} + srv.commands <- &agentv1.Command{CommandId: "c3"} + waitFor(t, "refusals", func() bool { + n := 0 + srv.with(func(f *fakeServer) { n = len(f.acks) }) + return n == 3 + }) + srv.with(func(f *fakeServer) { + if f.acks[1].GetAccepted() || f.acks[2].GetAccepted() { + t.Errorf("acks = %v", f.acks) + } + }) +} + +func TestServerSourceManualMode(t *testing.T) { + srv := newFakeServer(t, KindDNS) + addr := srv.start(t) + store, id := enrollWith(t, srv, addr) + srv.with(func(f *fakeServer) { + f.desired = 1 + f.mode = agentv1.OperatingMode_OPERATING_MODE_MANUAL + }) + runAgentAgainst(t, store, id, KindDNS) + waitFor(t, "hello", func() bool { + n := 0 + srv.with(func(f *fakeServer) { n = f.heartbeats }) + return n >= 3 + }) + if n := len(serverReports(srv)); n != 0 { + t.Errorf("manual mode applied the desired revision (%d reports)", n) + } +} + +func TestServerSourcePollingFallback(t *testing.T) { + srv := newFakeServer(t, KindDNS) + addr := srv.start(t) + store, id := enrollWith(t, srv, addr) + srv.with(func(f *fakeServer) { + f.desired = 1 + f.streamDisabled = true + }) + runAgentAgainst(t, store, id, KindDNS) + waitFor(t, "report via polling", func() bool { return len(serverReports(srv)) == 1 }) + srv.with(func(f *fakeServer) { + if f.polls == 0 { + t.Error("GetDesiredState never called") + } + }) + // A newer revision is picked up by a later poll. + srv.with(func(f *fakeServer) { f.desired = 2 }) + waitFor(t, "report of revision 2", func() bool { return len(serverReports(srv)) == 2 }) + if st, _ := store.LoadState(); st.Applied != 2 { + t.Errorf("state = %+v", st) + } +} + +func TestServerSourceRenewsCertificate(t *testing.T) { + srv := newFakeServer(t, KindDNS) + // More than half of the certificate's lifetime has passed. + srv.enrollValidity = [2]time.Duration{-3 * time.Hour, time.Hour} + addr := srv.start(t) + store, id := enrollWith(t, srv, addr) + oldCert, _ := os.ReadFile(filepath.Join(store.Dir(), "identity", "cert.pem")) + runAgentAgainst(t, store, id, KindDNS) + + waitFor(t, "renewal and hello", func() bool { + ok := false + srv.with(func(f *fakeServer) { ok = f.renewals == 1 && len(f.hellos) >= 1 }) + return ok + }) + newCert, _ := os.ReadFile(filepath.Join(store.Dir(), "identity", "cert.pem")) + if string(newCert) == string(oldCert) { + t.Error("cert.pem unchanged after renewal") + } + loaded, err := store.LoadIdentity() + if err != nil { + t.Fatal(err) + } + if time.Until(loaded.Certificate.Leaf.NotAfter) < 24*time.Hour { + t.Errorf("renewed certificate expires %v", loaded.Certificate.Leaf.NotAfter) + } +} + +func TestServerSourceRenewCommand(t *testing.T) { + srv := newFakeServer(t, KindDNS) + addr := srv.start(t) + store, id := enrollWith(t, srv, addr) + runAgentAgainst(t, store, id, KindDNS) + waitFor(t, "first hello", func() bool { + n := 0 + srv.with(func(f *fakeServer) { n = len(f.hellos) }) + return n == 1 + }) + srv.commands <- &agentv1.Command{CommandId: "r1", Action: &agentv1.Command_RenewCertificate{RenewCertificate: &agentv1.RenewCertificateCommand{}}} + // The old certificate is no longer accepted, so the second Hello proves + // that the agent reconnected with the new one. + waitFor(t, "reconnect with the renewed certificate", func() bool { + ok := false + srv.with(func(f *fakeServer) { ok = f.renewals == 1 && len(f.hellos) >= 2 }) + return ok + }) +} + +func TestServerSourceReportsLeases(t *testing.T) { + srv := newFakeServer(t, KindDHCP) + addr := srv.start(t) + store, id := enrollWith(t, srv, addr) + role, _ := runAgentAgainst(t, store, id, KindDHCP) + _ = role + waitFor(t, "full lease sync", func() bool { + ok := false + srv.with(func(f *fakeServer) { + for _, r := range f.leaseReports { + if r.GetFullSync() { + ok = true + } + } + }) + return ok + }) +} + +func TestLeaseToProto(t *testing.T) { + l := lease("10.0.0.9", "aa:bb:cc:dd:ee:ff") + l.ClientID = "01:aa" + p := leaseToProto(l) + if p.GetIp() != "10.0.0.9" || p.GetMac() != l.MAC || p.GetState() != agentv1.LeaseState_LEASE_STATE_ACTIVE || + !p.GetExpiresAt().AsTime().Equal(l.Expires) || p.GetSubnetId() != 1 || p.GetClientId() != "01:aa" { + t.Errorf("lease = %v", p) + } + if p := leaseToProto(Lease{IP: mustAddr("10.0.0.1")}); p.ExpiresAt != nil || p.UpdatedAt != nil { + t.Errorf("zero times encoded: %v", p) + } + for s, want := range map[LeaseState]agentv1.LeaseState{ + LeaseUnknown: agentv1.LeaseState_LEASE_STATE_UNSPECIFIED, LeaseExpired: agentv1.LeaseState_LEASE_STATE_EXPIRED, + LeaseReleased: agentv1.LeaseState_LEASE_STATE_RELEASED, LeaseDeclined: agentv1.LeaseState_LEASE_STATE_DECLINED, + } { + if got := leaseStateToProto(s); got != want { + t.Errorf("%v -> %v, want %v", s, got, want) + } + } + for o, want := range map[Outcome]agentv1.DeploymentOutcome{ + OutcomeRejected: agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_REJECTED, OutcomeRolledBack: agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_ROLLED_BACK, + OutcomeFailed: agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_FAILED, Outcome("x"): agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_UNSPECIFIED, + } { + if got := outcomeToProto(o); got != want { + t.Errorf("%v -> %v, want %v", o, got, want) + } + } + if triggerToProto(Trigger("x")) != agentv1.DeploymentTrigger_DEPLOYMENT_TRIGGER_UNSPECIFIED { + t.Error("unknown trigger mapped") + } + h := healthToProto(Health{State: HealthDegraded, Checks: []HealthCheck{{Name: "a", State: HealthFailed}}}, time.Time{}) + if h.GetState() != agentv1.HealthState_HEALTH_STATE_DEGRADED || h.CheckedAt != nil || h.GetChecks()[0].GetState() != agentv1.HealthState_HEALTH_STATE_FAILED { + t.Errorf("health = %v", h) + } +} + +func TestServerSourceUnreachable(t *testing.T) { + srv := newFakeServer(t, KindDNS) + addr := srv.start(t) + store, id := enrollWith(t, srv, addr) + id.Server = "127.0.0.1:1" // nothing listens there + src := NewServerSource(store, id, "h", testLogger()) + defer src.closeConn() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if err := src.ReportDeployment(ctx, DeploymentReport{Revision: 1}); err == nil { + t.Error("report to an unreachable server succeeded") + } + if _, err := src.FetchBundle(ctx, 1); err == nil { + t.Error("fetch from an unreachable server succeeded") + } + if code := status.Code(src.ReportLeases(ctx, LeaseBatch{Sequence: 1})); code == codes.OK { + t.Error("lease report to an unreachable server succeeded") + } +} + +func TestJitterAndRenewalDue(t *testing.T) { + for i := 0; i < 100; i++ { + if d := jitter(time.Second); d < 800*time.Millisecond || d > 1200*time.Millisecond { + t.Fatalf("jitter = %v", d) + } + } + pki := newTestPKI(t) + id, _ := testIdentity(t, pki) + leaf := id.Certificate.Leaf + if due := renewalDue(leaf); !due.Equal(leaf.NotBefore.Add(leaf.NotAfter.Sub(leaf.NotBefore) / 2)) { + t.Errorf("renewalDue = %v", due) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if sleepCtx(ctx, time.Hour) { + t.Error("sleepCtx ignored the context") + } +} diff --git a/agent/core/source.go b/agent/core/source.go new file mode 100644 index 0000000..b089497 --- /dev/null +++ b/agent/core/source.go @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 + +package core + +import ( + "context" + "time" +) + +// Source is where the agent learns its desired revision, fetches bundles and +// reports results (docs/adr/0010-agent-binaries-pro-rolle.md). M3 provides +// ServerSource (gRPC); a FileSource for standalone operation (M7) implements +// the same interface without changes to the apply pipeline. +type Source interface { + // Run delivers events to h until ctx is canceled. It returns ctx.Err() + // on shutdown or an error if the source cannot work at all. + Run(ctx context.Context, h SourceHandler) error + // FetchBundle returns the signed bundle of a revision. The caller + // verifies it. + FetchBundle(ctx context.Context, revision uint64) (*Bundle, error) + // ReportDeployment delivers an apply outcome. + ReportDeployment(ctx context.Context, r DeploymentReport) error + // ReportLeases delivers a batch of lease observations. + ReportLeases(ctx context.Context, b LeaseBatch) error +} + +// SourceHandler receives the events of a Source. Its methods must not +// block for long. +type SourceHandler interface { + // Connected is called whenever the source (re)established contact; + // the agent then sends queued reports and a full lease snapshot. + Connected() + // Desired announces the revision the agent should run. autoApply is + // false in manual mode, where only explicit pushes are applied. + Desired(revision uint64, autoApply bool) + // Push asks for an explicit apply of a revision. Unlike Desired, it + // also retries a revision that failed before. + Push(revision uint64, deploymentID string) + // Status returns what the agent reports in Hello and heartbeats. + Status(ctx context.Context) AgentStatus +} + +// AgentStatus is the agent's view of itself. +type AgentStatus struct { + Applied uint64 + LastGood uint64 + Health Health + CheckedAt time.Time + ServiceVersion string +} + +// Bundle is a signed bundle as delivered by a Source. +type Bundle struct { + Manifest []byte + Signature []byte + Artifacts []BundleArtifact +} + +// BundleArtifact is one artifact of a Bundle. +type BundleArtifact struct { + Path string + Content []byte +} + +// LeaseBatch is one lease report. Sequence is monotonic per agent. A batch +// with FullSync replaces the server's view of all leases of this agent; the +// following batches (higher sequence) are changes on top of it. +type LeaseBatch struct { + Sequence uint64 + FullSync bool + Leases []Lease +} diff --git a/agent/core/store.go b/agent/core/store.go new file mode 100644 index 0000000..9126a69 --- /dev/null +++ b/agent/core/store.go @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: Apache-2.0 + +package core + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "git.beugsiud.de/beug.k/DDI/internal/bundle" +) + +// Layout of the state directory (docs/adr/0011-agent-zustand-in-dateien.md). +const ( + identityDir = "identity" + revisionsDir = "revisions" + stateFile = "state.json" + lockFile = "agent.lock" + manifestFile = "manifest.json" + signatureFile = "manifest.sig" + tmpPrefix = ".tmp-" + oldPrefix = ".old-" +) + +// KeepRevisions is the number of stored revisions kept by Prune (the applied +// and the last good revision are kept in addition). +const KeepRevisions = 10 + +// maxPendingReports bounds the queue of unsent deployment reports. +const maxPendingReports = 100 + +// Store manages the agent's state directory. Writes are atomic: files are +// written to a temporary file, synced and renamed; directories are built +// under a temporary name and renamed into place. +type Store struct { + dir string +} + +// OpenStore creates the state directory if necessary. +func OpenStore(dir string) (*Store, error) { + if dir == "" { + return nil, errors.New("state directory is not set") + } + if err := os.MkdirAll(filepath.Join(dir, revisionsDir), 0o750); err != nil { + return nil, fmt.Errorf("create state directory: %w", err) + } + return &Store{dir: dir}, nil +} + +// Dir returns the state directory. +func (s *Store) Dir() string { return s.dir } + +// Lock takes the exclusive lock of the state directory. The returned +// function releases it. +func (s *Store) Lock() (func(), error) { + return lockPath(filepath.Join(s.dir, lockFile)) +} + +// Trigger says why a revision was applied. +type Trigger string + +// Triggers. +const ( + TriggerPull Trigger = "pull" + TriggerPush Trigger = "push" +) + +// Outcome is the result of applying a revision. +type Outcome string + +// Outcomes (see DeploymentOutcome in the protocol). +const ( + OutcomeSucceeded Outcome = "succeeded" + OutcomeRejected Outcome = "rejected" + OutcomeRolledBack Outcome = "rolled_back" + OutcomeFailed Outcome = "failed" +) + +// DeploymentReport is the outcome of one apply, queued in state.json until +// the source accepted it. +type DeploymentReport struct { + DeploymentID string `json:"deployment_id,omitempty"` + Revision uint64 `json:"revision"` + Trigger Trigger `json:"trigger"` + Outcome Outcome `json:"outcome"` + Message string `json:"message,omitempty"` + ActiveRevision uint64 `json:"active_revision"` + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at"` +} + +// State is persisted in state.json. +type State struct { + // Applied is the revision whose configuration is installed (0 = none). + Applied uint64 `json:"applied_revision"` + // LastGood is the last revision that passed its health check. + LastGood uint64 `json:"last_good_revision"` + // LastUnsuccessful is the highest revision that was rejected, rolled + // back or failed. Pulls never retry it; only a newer revision or an + // explicit push does. + LastUnsuccessful uint64 `json:"last_unsuccessful_revision,omitempty"` + // LastResult is the most recent apply outcome. + LastResult *DeploymentReport `json:"last_result,omitempty"` + // LeaseSequence is the sequence number of the last lease report. + LeaseSequence uint64 `json:"lease_sequence,omitempty"` + // PendingReports are deployment reports not yet accepted by the server. + PendingReports []DeploymentReport `json:"pending_reports,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +// LoadState reads state.json; a missing file yields the zero state. +func (s *Store) LoadState() (*State, error) { + var st State + data, err := os.ReadFile(filepath.Join(s.dir, stateFile)) + if errors.Is(err, fs.ErrNotExist) { + return &st, nil + } + if err != nil { + return nil, fmt.Errorf("read state: %w", err) + } + if err := json.Unmarshal(data, &st); err != nil { + return nil, fmt.Errorf("decode %s: %w", stateFile, err) + } + return &st, nil +} + +// SaveState writes state.json atomically. +func (s *Store) SaveState(st *State) error { + if n := len(st.PendingReports); n > maxPendingReports { + st.PendingReports = slices.Clone(st.PendingReports[n-maxPendingReports:]) + } + data, err := json.MarshalIndent(st, "", " ") + if err != nil { + return fmt.Errorf("encode state: %w", err) + } + return writeFileAtomic(filepath.Join(s.dir, stateFile), append(data, '\n'), 0o600) +} + +// revisionPath returns the directory of revision n. +func (s *Store) revisionPath(n uint64) string { + return filepath.Join(s.dir, revisionsDir, strconv.FormatUint(n, 10)) +} + +// reservedArtifact reports whether an artifact path would collide with the +// files the store keeps next to the artifacts. +func reservedArtifact(p string) bool { + return p == manifestFile || p == signatureFile +} + +// StoreRevision writes a verified bundle to revisions// (manifest, +// signature and the artifacts at their manifest paths) and returns it. An +// existing directory for the same number is replaced. +func (s *Store) StoreRevision(m *bundle.Manifest, manifest, signature []byte, contents map[string][]byte) (Revision, error) { + parent := filepath.Join(s.dir, revisionsDir) + tmp, err := os.MkdirTemp(parent, tmpPrefix) + if err != nil { + return Revision{}, fmt.Errorf("create revision directory: %w", err) + } + defer func() { _ = os.RemoveAll(tmp) }() + if err := writeFileSync(filepath.Join(tmp, manifestFile), manifest, 0o640); err != nil { + return Revision{}, err + } + if err := writeFileSync(filepath.Join(tmp, signatureFile), signature, 0o640); err != nil { + return Revision{}, err + } + dirs := map[string]bool{tmp: true} + for _, f := range m.Files { + if err := bundle.CheckPath(f.Path); err != nil { + return Revision{}, err + } + if reservedArtifact(f.Path) { + return Revision{}, fmt.Errorf("artifact path %s is reserved", f.Path) + } + target := filepath.Join(tmp, filepath.FromSlash(f.Path)) + dir := filepath.Dir(target) + if !dirs[dir] { + if err := os.MkdirAll(dir, 0o750); err != nil { + return Revision{}, fmt.Errorf("create directory for %s: %w", f.Path, err) + } + dirs[dir] = true + } + if err := writeFileSync(target, contents[f.Path], fs.FileMode(f.Mode&0o777)); err != nil { + return Revision{}, err + } + } + for d := range dirs { + syncDir(d) + } + final := s.revisionPath(m.Revision) + if err := replaceDir(tmp, final); err != nil { + return Revision{}, err + } + syncDir(parent) + return Revision{Number: m.Revision, Dir: final, Files: slices.Clone(m.Files)}, nil +} + +// LoadRevision reads a stored revision. Stored revisions were verified +// before they were written (ADR 0009), so only the manifest is parsed. +func (s *Store) LoadRevision(n uint64) (Revision, error) { + dir := s.revisionPath(n) + data, err := os.ReadFile(filepath.Join(dir, manifestFile)) //nolint:gosec // path built from the state directory + if err != nil { + return Revision{}, fmt.Errorf("load revision %d: %w", n, err) + } + var m bundle.Manifest + if err := json.Unmarshal(data, &m); err != nil { + return Revision{}, fmt.Errorf("load revision %d: %w", n, err) + } + if m.Revision != n { + return Revision{}, fmt.Errorf("load revision %d: manifest is for revision %d", n, m.Revision) + } + return Revision{Number: n, Dir: dir, Files: m.Files}, nil +} + +// ListRevisions returns the stored revision numbers in ascending order. +func (s *Store) ListRevisions() ([]uint64, error) { + entries, err := os.ReadDir(filepath.Join(s.dir, revisionsDir)) + if err != nil { + return nil, fmt.Errorf("list revisions: %w", err) + } + var out []uint64 + for _, e := range entries { + if !e.IsDir() { + continue + } + n, err := strconv.ParseUint(e.Name(), 10, 64) + if err != nil || n == 0 || strconv.FormatUint(n, 10) != e.Name() { + continue + } + out = append(out, n) + } + slices.Sort(out) + return out, nil +} + +// Prune keeps the newest keep revisions plus the protected ones, removes the +// others and cleans up leftovers of interrupted writes. +func (s *Store) Prune(keep int, protect ...uint64) error { + parent := filepath.Join(s.dir, revisionsDir) + entries, err := os.ReadDir(parent) + if err != nil { + return fmt.Errorf("prune revisions: %w", err) + } + for _, e := range entries { + if strings.HasPrefix(e.Name(), tmpPrefix) || strings.HasPrefix(e.Name(), oldPrefix) { + _ = os.RemoveAll(filepath.Join(parent, e.Name())) + } + } + revs, err := s.ListRevisions() + if err != nil { + return err + } + var errs []error + for i, n := range revs { + if len(revs)-i <= keep || slices.Contains(protect, n) { + continue + } + if err := os.RemoveAll(s.revisionPath(n)); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// writeFileSync creates path with data and syncs it. +func writeFileSync(path string, data []byte, perm fs.FileMode) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) //nolint:gosec // path built from the state directory + if err != nil { + return fmt.Errorf("write %s: %w", filepath.Base(path), err) + } + if err := finishFile(f, data, perm); err != nil { + return fmt.Errorf("write %s: %w", filepath.Base(path), err) + } + return nil +} + +func finishFile(f *os.File, data []byte, perm fs.FileMode) error { + _, err := f.Write(data) + if err == nil { + err = f.Chmod(perm) // not subject to the umask + if errors.Is(err, errors.ErrUnsupported) { + err = nil + } + } + if err == nil { + err = f.Sync() + } + if cerr := f.Close(); err == nil { + err = cerr + } + return err +} + +// writeFileAtomic replaces path with data: temporary file, fsync, rename, +// fsync of the directory. +func writeFileAtomic(target string, data []byte, perm fs.FileMode) error { + dir := filepath.Dir(target) + f, err := os.CreateTemp(dir, tmpPrefix+filepath.Base(target)+"-") + if err != nil { + return fmt.Errorf("write %s: %w", filepath.Base(target), err) + } + tmp := f.Name() + if err := finishFile(f, data, perm); err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("write %s: %w", filepath.Base(target), err) + } + if err := os.Rename(tmp, target); err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("write %s: %w", filepath.Base(target), err) + } + syncDir(dir) + return nil +} + +// replaceDir moves the fully written directory tmp to final. An existing +// final directory is moved aside first and removed afterwards. +func replaceDir(tmp, final string) error { + parent := filepath.Dir(final) + var old string + if _, err := os.Stat(final); err == nil { + old = filepath.Join(parent, oldPrefix+filepath.Base(final)+"-"+randomSuffix()) + if err := os.Rename(final, old); err != nil { + return fmt.Errorf("replace %s: %w", filepath.Base(final), err) + } + } + if err := os.Rename(tmp, final); err != nil { + if old != "" { + _ = os.Rename(old, final) + } + return fmt.Errorf("replace %s: %w", filepath.Base(final), err) + } + if old != "" { + _ = os.RemoveAll(old) + } + return nil +} + +func randomSuffix() string { + var b [6]byte + _, _ = rand.Read(b[:]) + return hex.EncodeToString(b[:]) +} + +// syncDir flushes directory metadata (renames). Errors are ignored because +// some platforms cannot sync directories; the data itself was synced. +func syncDir(dir string) { + d, err := os.Open(dir) //nolint:gosec // path built from the state directory + if err != nil { + return + } + _ = d.Sync() + _ = d.Close() +} diff --git a/agent/core/store_test.go b/agent/core/store_test.go new file mode 100644 index 0000000..ce02129 --- /dev/null +++ b/agent/core/store_test.go @@ -0,0 +1,382 @@ +// SPDX-License-Identifier: Apache-2.0 + +package core + +import ( + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "io/fs" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + "time" + + "git.beugsiud.de/beug.k/DDI/internal/bundle" +) + +func openTestStore(t *testing.T) *Store { + t.Helper() + s, err := OpenStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return s +} + +// storeTestRevision verifies and stores a signed test bundle. +func storeTestRevision(t *testing.T, s *Store, key ed25519.PrivateKey, rev uint64) Revision { + t.Helper() + b := signBundle(t, key, testAgentID, KindDNS, rev) + contents := map[string][]byte{} + for _, a := range b.Artifacts { + contents[a.Path] = a.Content + } + m, err := bundle.Verify(key.Public().(ed25519.PublicKey), b.Manifest, b.Signature, contents, + bundle.Expect{AgentID: testAgentID, Role: string(KindDNS)}) + if err != nil { + t.Fatal(err) + } + r, err := s.StoreRevision(m, b.Manifest, b.Signature, contents) + if err != nil { + t.Fatal(err) + } + return r +} + +func TestOpenStoreEmpty(t *testing.T) { + if _, err := OpenStore(""); err == nil { + t.Error("empty state directory accepted") + } +} + +func TestStoreRevision(t *testing.T) { + s := openTestStore(t) + _, key := newSigningKey(t) + r := storeTestRevision(t, s, key, 7) + if r.Number != 7 || r.Dir != filepath.Join(s.Dir(), "revisions", "7") || len(r.Files) != 2 { + t.Fatalf("revision = %+v", r) + } + data, err := os.ReadFile(filepath.Join(r.Dir, "zones", "db.example.com")) + if err != nil || !strings.Contains(string(data), "SOA ns 1 7") { + t.Fatalf("artifact = %q, %v", data, err) + } + if runtime.GOOS != "windows" { + fi, err := os.Stat(filepath.Join(r.Dir, "ddi-suite.conf")) + if err != nil || fi.Mode().Perm() != 0o640 { + t.Errorf("mode = %v, %v", fi.Mode(), err) + } + } + loaded, err := s.LoadRevision(7) + if err != nil || loaded.Number != 7 || loaded.Dir != r.Dir || len(loaded.Files) != 2 { + t.Fatalf("LoadRevision = %+v, %v", loaded, err) + } + // Storing the same number again replaces the directory. + if err := os.WriteFile(filepath.Join(r.Dir, "stale"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + storeTestRevision(t, s, key, 7) + if _, err := os.Stat(filepath.Join(r.Dir, "stale")); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("old directory content survived: %v", err) + } + if _, err := s.LoadRevision(8); err == nil { + t.Error("missing revision loaded") + } + entries, _ := os.ReadDir(filepath.Join(s.Dir(), "revisions")) + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".") { + t.Errorf("leftover %s", e.Name()) + } + } +} + +func TestStoreRevisionReservedPath(t *testing.T) { + s := openTestStore(t) + m := &bundle.Manifest{Revision: 1, Files: []bundle.File{{Path: "manifest.json", Mode: 0o644}}} + if _, err := s.StoreRevision(m, []byte("{}"), []byte("sig"), map[string][]byte{"manifest.json": nil}); err == nil { + t.Error("reserved artifact path accepted") + } + if revs, _ := s.ListRevisions(); len(revs) != 0 { + t.Errorf("revisions = %v", revs) + } +} + +func TestLoadRevisionMismatch(t *testing.T) { + s := openTestStore(t) + _, key := newSigningKey(t) + r := storeTestRevision(t, s, key, 3) + if err := os.Rename(r.Dir, filepath.Join(s.Dir(), "revisions", "4")); err != nil { + t.Fatal(err) + } + if _, err := s.LoadRevision(4); err == nil { + t.Error("manifest of another revision accepted") + } +} + +func TestPrune(t *testing.T) { + s := openTestStore(t) + _, key := newSigningKey(t) + for rev := uint64(1); rev <= 15; rev++ { + storeTestRevision(t, s, key, rev) + } + for _, name := range []string{".tmp-123", ".old-5-abc", "notanumber", "007"} { + if err := os.Mkdir(filepath.Join(s.Dir(), "revisions", name), 0o750); err != nil { + t.Fatal(err) + } + } + if err := s.Prune(KeepRevisions, 2, 3); err != nil { + t.Fatal(err) + } + revs, err := s.ListRevisions() + if err != nil { + t.Fatal(err) + } + want := []uint64{2, 3, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + if !slices.Equal(revs, want) { + t.Errorf("revisions = %v, want %v", revs, want) + } + for _, name := range []string{".tmp-123", ".old-5-abc"} { + if _, err := os.Stat(filepath.Join(s.Dir(), "revisions", name)); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("%s not removed", name) + } + } + if _, err := os.Stat(filepath.Join(s.Dir(), "revisions", "notanumber")); err != nil { + t.Error("foreign directory removed") + } +} + +func TestStateRoundTrip(t *testing.T) { + s := openTestStore(t) + st, err := s.LoadState() + if err != nil || st.Applied != 0 { + t.Fatalf("empty state = %+v, %v", st, err) + } + now := time.Now().UTC().Truncate(time.Second) + st = &State{ + Applied: 5, LastGood: 4, LastUnsuccessful: 5, LeaseSequence: 9, + LastResult: &DeploymentReport{Revision: 5, Outcome: OutcomeFailed, Trigger: TriggerPush, StartedAt: now, FinishedAt: now}, + PendingReports: []DeploymentReport{{Revision: 5, Outcome: OutcomeFailed, DeploymentID: "d"}}, + } + if err := s.SaveState(st); err != nil { + t.Fatal(err) + } + got, err := s.LoadState() + if err != nil { + t.Fatal(err) + } + if got.Applied != 5 || got.LastGood != 4 || got.LeaseSequence != 9 || got.LastResult.Outcome != OutcomeFailed || + len(got.PendingReports) != 1 || got.PendingReports[0].DeploymentID != "d" { + t.Errorf("state = %+v", got) + } + // Human-readable JSON with the documented keys. + raw, _ := os.ReadFile(filepath.Join(s.Dir(), "state.json")) + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, k := range []string{"applied_revision", "last_good_revision", "pending_reports", "lease_sequence"} { + if _, ok := m[k]; !ok { + t.Errorf("state.json lacks %s", k) + } + } + entries, _ := os.ReadDir(s.Dir()) + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".tmp-") { + t.Errorf("leftover temporary file %s", e.Name()) + } + } + if err := os.WriteFile(filepath.Join(s.Dir(), "state.json"), []byte("{broken"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := s.LoadState(); err == nil { + t.Error("broken state.json accepted") + } +} + +func TestLock(t *testing.T) { + s := openTestStore(t) + unlock, err := s.Lock() + if err != nil { + t.Fatal(err) + } + if runtime.GOOS == "linux" { + if _, err := s.Lock(); err == nil || !strings.Contains(err.Error(), "locked") { + t.Errorf("second lock = %v", err) + } + } + unlock() + unlock2, err := s.Lock() + if err != nil { + t.Fatalf("lock after unlock: %v", err) + } + unlock2() +} + +// testIdentity creates an identity signed by pki. +func testIdentity(t *testing.T, pki *testPKI) (*Identity, *ecdsa.PrivateKey) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + cert, err := pki.issueClient(&key.PublicKey, testAgentID, time.Now().Add(-time.Hour), time.Now().Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + pub, _ := newSigningKey(t) + return &Identity{ + AgentID: testAgentID, Role: KindDNS, Server: "ddi.example.com:8444", ServerName: "ddi.example.com", + EnrolledAt: time.Now().UTC().Truncate(time.Second), CA: pki.ca, Certificate: keyPair(key, cert), BundleKey: pub, + }, key +} + +func TestIdentityRoundTrip(t *testing.T) { + s := openTestStore(t) + if _, err := s.LoadIdentity(); !errors.Is(err, ErrNotEnrolled) { + t.Fatalf("LoadIdentity = %v", err) + } + pki := newTestPKI(t) + id, key := testIdentity(t, pki) + if err := s.SaveIdentity(id, key, false); err != nil { + t.Fatal(err) + } + if !s.Enrolled() { + t.Error("not enrolled after SaveIdentity") + } + got, err := s.LoadIdentity() + if err != nil { + t.Fatal(err) + } + if got.AgentID != id.AgentID || got.Server != id.Server || got.ServerName != id.ServerName || got.Role != KindDNS || + !got.CA.Equal(pki.ca) || !got.BundleKey.Equal(id.BundleKey) || !got.Certificate.Leaf.Equal(id.Certificate.Leaf) || + !got.EnrolledAt.Equal(id.EnrolledAt) { + t.Errorf("identity = %+v", got) + } + if runtime.GOOS != "windows" { + fi, err := os.Stat(filepath.Join(s.Dir(), "identity", "key.pem")) + if err != nil || fi.Mode().Perm() != 0o600 { + t.Errorf("key.pem mode = %v, %v", fi.Mode(), err) + } + di, err := os.Stat(filepath.Join(s.Dir(), "identity")) + if err != nil || di.Mode().Perm() != 0o700 { + t.Errorf("identity dir mode = %v, %v", di.Mode(), err) + } + } + // Already enrolled. + if err := s.SaveIdentity(id, key, false); !errors.Is(err, errAlreadyEnrolled) { + t.Errorf("second SaveIdentity = %v", err) + } + id2, key2 := testIdentity(t, pki) + id2.Server = "other.example.com:8444" + if err := s.SaveIdentity(id2, key2, true); err != nil { + t.Fatal(err) + } + if got, err := s.LoadIdentity(); err != nil || got.Server != "other.example.com:8444" { + t.Errorf("forced identity = %+v, %v", got, err) + } + entries, _ := os.ReadDir(s.Dir()) + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".") { + t.Errorf("leftover %s", e.Name()) + } + } +} + +func TestSaveCertificateAndRecovery(t *testing.T) { + s := openTestStore(t) + pki := newTestPKI(t) + id, key := testIdentity(t, pki) + if err := s.SaveIdentity(id, key, false); err != nil { + t.Fatal(err) + } + newKey, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + newCert, err := pki.issueClient(&newKey.PublicKey, testAgentID, time.Now().Add(-time.Minute), time.Now().Add(2*time.Hour)) + if err != nil { + t.Fatal(err) + } + if err := s.SaveCertificate(newKey, newCert); err != nil { + t.Fatal(err) + } + got, err := s.LoadIdentity() + if err != nil || !got.Certificate.Leaf.Equal(newCert) { + t.Fatalf("renewed identity = %v", err) + } + + // Simulate a crash between the two renames: key.pem is new, cert.pem + // old, cert.pem.new holds the matching certificate. + key3, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + cert3, err := pki.issueClient(&key3.PublicKey, testAgentID, time.Now().Add(-time.Minute), time.Now().Add(3*time.Hour)) + if err != nil { + t.Fatal(err) + } + keyPEM, _ := encodeKey(key3) + dir := filepath.Join(s.Dir(), "identity") + if err := os.WriteFile(filepath.Join(dir, "key.pem"), keyPEM, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "cert.pem.new"), pemCert(cert3), 0o600); err != nil { + t.Fatal(err) + } + got, err = s.LoadIdentity() + if err != nil || !got.Certificate.Leaf.Equal(cert3) { + t.Fatalf("recovered identity: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "cert.pem.new")); !errors.Is(err, fs.ErrNotExist) { + t.Error("cert.pem.new left behind") + } + + // A mismatch without a recoverable certificate is an error. + key4, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + keyPEM, _ = encodeKey(key4) + if err := os.WriteFile(filepath.Join(dir, "key.pem"), keyPEM, 0o600); err != nil { + t.Fatal(err) + } + if _, err := s.LoadIdentity(); err == nil { + t.Error("mismatched key pair accepted") + } +} + +func pemCert(c *x509.Certificate) []byte { + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: c.Raw}) +} + +func TestCheckIssued(t *testing.T) { + pki := newTestPKI(t) + key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + cert, err := pki.issueClient(&key.PublicKey, testAgentID, time.Now().Add(-time.Minute), time.Now().Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + if _, err := checkIssued(cert.Raw, pki.ca, testAgentID, key); err != nil { + t.Fatalf("checkIssued = %v", err) + } + if _, err := checkIssued(cert.Raw, pki.ca, "11111111-2222-4333-8444-555555555555", key); err == nil { + t.Error("other agent ID accepted") + } + other, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if _, err := checkIssued(cert.Raw, pki.ca, testAgentID, other); err == nil { + t.Error("certificate for another key accepted") + } + if _, err := checkIssued(cert.Raw, newTestPKI(t).ca, testAgentID, key); err == nil { + t.Error("certificate of another CA accepted") + } + if _, err := checkIssued([]byte("junk"), pki.ca, testAgentID, key); err == nil { + t.Error("junk accepted") + } + // A server clock slightly ahead is tolerated. + future, err := pki.issueClient(&key.PublicKey, testAgentID, time.Now().Add(2*time.Minute), time.Now().Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + if _, err := checkIssued(future.Raw, pki.ca, testAgentID, key); err != nil { + t.Errorf("future NotBefore: %v", err) + } +}