beug.k/DDI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 12 Wiki Activity

Merge branch 'worktree-agent-a2c569cc905081672'

Browse Source
This commit is contained in:
beug.k@beugsiud.de
2026-10-13 19:10:13 +02:00
parent 6c1f7249b0 3065175673
commit c6162e00d2
20 changed files with 5326 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/cmd/ddi-agent-dhcp/main.go
+7 -1
View File
@@ -4,6 +4,7 @@
package main
import (
"errors"
"os"
"git.beugsiud.de/beug.k/DDI/agent/core"
@@ -11,5 +12,10 @@ import (
)
func main() {
os.Exit(core.RunCLI(brand.AgentDHCPBinary, core.KindDHCP, os.Args[1:], os.Stdout, os.Stderr))
os.Exit(core.Main(brand.AgentDHCPBinary, core.KindDHCP, newRole, os.Args[1:], os.Stdout, os.Stderr))
}
// newRole is a placeholder until the DHCP role package is wired in.
func newRole(func(v any) error) (core.Role, error) {
return nil, errors.New("DHCP role not available in this build")
}
agent/cmd/ddi-agent-dns/main.go
+7 -1
View File
@@ -4,6 +4,7 @@
package main
import (
"errors"
"os"
"git.beugsiud.de/beug.k/DDI/agent/core"
@@ -11,5 +12,10 @@ import (
)
func main() {
os.Exit(core.RunCLI(brand.AgentDNSBinary, core.KindDNS, os.Args[1:], os.Stdout, os.Stderr))
os.Exit(core.Main(brand.AgentDNSBinary, core.KindDNS, newRole, os.Args[1:], os.Stdout, os.Stderr))
}
// newRole is a placeholder until the DNS role package is wired in.
func newRole(func(v any) error) (core.Role, error) {
return nil, errors.New("DNS role not available in this build")
}
agent/core/agent.go
+612
View File
@@ -0,0 +1,612 @@
// SPDX-License-Identifier: Apache-2.0
package core
import (
"context"
"crypto/ed25519"
"errors"
"fmt"
"log/slog"
"reflect"
"slices"
"strings"
"sync"
"time"
"unicode/utf8"
"git.beugsiud.de/beug.k/DDI/internal/bundle"
)
// Timing defaults of the agent that are not configurable.
const (
healthRetryInterval = time.Second
reportRetryInterval = 30 * time.Second
reportTimeout = 30 * time.Second
statusTimeout = 10 * time.Second
serviceVersionMaxAge = 10 * time.Minute
// maxMessageLen bounds the message of a deployment report.
maxMessageLen = 4096
)
// Agent drives a role: it applies the revisions announced by a Source and
// reports outcomes, health and leases.
type Agent struct {
kind Kind
cfg *Config
role Role
store *Store
agentID string
bundleKey ed25519.PublicKey
source Source
log *slog.Logger
// Intervals, shortened by tests.
healthRetry time.Duration
reportRetry time.Duration
// roleMu serializes calls into the role: only one apply at a time, and
// health checks for heartbeats are skipped while an apply runs.
roleMu sync.Mutex
mu sync.Mutex
st *State
current *Revision // applied revision, nil if none or not loadable
health Health
healthAt time.Time
version string
versionAt time.Time
qmu sync.Mutex
queue []applyRequest
wake chan struct{}
flushCh chan struct{}
leaseCh chan struct{}
}
// applyRequest asks for an apply of a revision.
type applyRequest struct {
revision uint64
trigger Trigger
deploymentID string
}
// NewAgent prepares an agent. The identity provides the agent ID and the
// bundle signing key.
func NewAgent(kind Kind, cfg *Config, role Role, store *Store, id *Identity, src Source, log *slog.Logger) (*Agent, error) {
if role.Kind() != kind {
return nil, fmt.Errorf("role %q does not match agent role %q", role.Kind(), kind)
}
st, err := store.LoadState()
if err != nil {
return nil, err
}
a := &Agent{
kind: kind, cfg: cfg, role: role, store: store, agentID: id.AgentID,
bundleKey: id.BundleKey, source: src, log: log, st: st,
healthRetry: healthRetryInterval, reportRetry: reportRetryInterval,
wake: make(chan struct{}, 1), flushCh: make(chan struct{}, 1), leaseCh: make(chan struct{}, 1),
}
if st.Applied != 0 {
rev, err := store.LoadRevision(st.Applied)
if err != nil {
log.Warn("applied revision is not available locally", "revision", st.Applied, "error", err)
} else {
a.current = &rev
}
}
return a, nil
}
// Run works until ctx is canceled. The managed service keeps running with
// its current configuration afterwards.
func (a *Agent) Run(ctx context.Context) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
a.prune()
var wg sync.WaitGroup
wg.Go(func() { a.applyLoop(ctx) })
wg.Go(func() { a.reportLoop(ctx) })
if ls, ok := a.role.(LeaseSource); ok {
wg.Go(func() { a.leaseLoop(ctx, ls) })
}
err := a.source.Run(ctx, a)
stopped := ctx.Err() != nil
cancel()
wg.Wait()
if stopped {
return nil
}
return err
}
// state returns a copy of the state (without the report queue).
func (a *Agent) state() State {
a.mu.Lock()
defer a.mu.Unlock()
st := *a.st
st.PendingReports = nil
return st
}
// Connected implements SourceHandler.
func (a *Agent) Connected() {
notify(a.flushCh)
notify(a.leaseCh)
}
// Desired implements SourceHandler.
func (a *Agent) Desired(revision uint64, autoApply bool) {
if revision == 0 {
return
}
if !autoApply {
a.log.Debug("desired revision not applied in manual mode", "revision", revision)
return
}
a.enqueue(applyRequest{revision: revision, trigger: TriggerPull})
}
// Push implements SourceHandler.
func (a *Agent) Push(revision uint64, deploymentID string) {
a.enqueue(applyRequest{revision: revision, trigger: TriggerPush, deploymentID: deploymentID})
}
// Status implements SourceHandler. It runs a health check unless an apply
// is in progress, in which case the last result is returned.
func (a *Agent) Status(ctx context.Context) AgentStatus {
if a.roleMu.TryLock() {
a.refreshStatus(ctx)
a.roleMu.Unlock()
}
a.mu.Lock()
defer a.mu.Unlock()
return AgentStatus{
Applied: a.st.Applied, LastGood: a.st.LastGood,
Health: a.health, CheckedAt: a.healthAt, ServiceVersion: a.version,
}
}
// refreshStatus runs a health check and, if stale, reads the service
// version. The caller holds roleMu.
func (a *Agent) refreshStatus(ctx context.Context) {
ctx, cancel := context.WithTimeout(ctx, statusTimeout)
defer cancel()
a.mu.Lock()
cur := a.current
needVersion := a.version == "" || time.Since(a.versionAt) > serviceVersionMaxAge
a.mu.Unlock()
h := a.role.Health(ctx, cur)
var version string
if needVersion {
v, err := a.role.ServiceVersion(ctx)
if err != nil {
a.log.Warn("cannot determine the service version", "error", err)
}
version = v
}
a.mu.Lock()
a.health, a.healthAt = h, time.Now()
if needVersion && version != "" {
a.version, a.versionAt = version, time.Now()
}
a.mu.Unlock()
}
// enqueue adds a request. A pull is dropped if an equal or newer revision
// is already queued; a push replaces queued pulls up to its revision.
func (a *Agent) enqueue(r applyRequest) {
a.qmu.Lock()
if r.trigger == TriggerPull {
for _, q := range a.queue {
if q.revision >= r.revision {
a.qmu.Unlock()
return
}
}
} else {
a.queue = slices.DeleteFunc(a.queue, func(q applyRequest) bool {
return q.trigger == TriggerPull && q.revision <= r.revision
})
}
a.queue = append(a.queue, r)
a.qmu.Unlock()
notify(a.wake)
}
func (a *Agent) dequeue() (applyRequest, bool) {
a.qmu.Lock()
defer a.qmu.Unlock()
if len(a.queue) == 0 {
return applyRequest{}, false
}
r := a.queue[0]
a.queue = a.queue[1:]
return r, true
}
// applyLoop processes requests one at a time.
func (a *Agent) applyLoop(ctx context.Context) {
for {
r, ok := a.dequeue()
if !ok {
select {
case <-ctx.Done():
return
case <-a.wake:
continue
}
}
if ctx.Err() != nil {
return
}
a.handle(ctx, r)
}
}
// handle decides whether a request leads to an apply.
func (a *Agent) handle(ctx context.Context, r applyRequest) {
st := a.state()
log := a.log.With("revision", r.revision, "trigger", r.trigger, "deployment_id", r.deploymentID)
switch {
case r.trigger == TriggerPull && r.revision <= st.Applied:
log.Debug("revision already applied or older", "applied", st.Applied)
return
case r.trigger == TriggerPull && r.revision <= st.LastUnsuccessful:
log.Debug("revision failed before; waiting for a newer revision or an explicit push",
"last_unsuccessful", st.LastUnsuccessful)
return
case r.revision == 0 || r.revision < st.Applied:
now := time.Now()
log.Warn("refusing to apply a revision older than the applied one", "applied", st.Applied)
a.finish(log, DeploymentReport{
DeploymentID: r.deploymentID, Revision: r.revision, Trigger: r.trigger, Outcome: OutcomeRejected,
Message: fmt.Sprintf("revision %d is older than the applied revision %d", r.revision, st.Applied),
ActiveRevision: st.Applied, StartedAt: now, FinishedAt: now,
}, nil)
return
}
a.apply(ctx, r, st, log)
}
// apply runs the pipeline of docs/adr/0021: fetch, verify, store, Validate,
// Activate, Health, rollback. Shutdown aborts it until Activate starts; from
// then on it is completed (bounded by the configured timeouts) so that the
// service is never left between two revisions.
func (a *Agent) apply(ctx context.Context, r applyRequest, st State, log *slog.Logger) {
started := time.Now()
report := DeploymentReport{
DeploymentID: r.deploymentID, Revision: r.revision, Trigger: r.trigger,
ActiveRevision: st.Applied, StartedAt: started,
}
log.Info("applying revision", "applied", st.Applied, "last_good", st.LastGood)
reject := func(msg string, err error) {
report.Outcome = OutcomeRejected
report.Message = fmt.Sprintf("%s: %v", msg, err)
log.Error("revision rejected", "error", report.Message)
a.finish(log, report, func(s *State) { s.LastUnsuccessful = max(s.LastUnsuccessful, r.revision) })
}
fctx, cancel := context.WithTimeout(ctx, a.cfg.ApplyTimeout)
b, err := a.source.FetchBundle(fctx, r.revision)
cancel()
if err != nil {
if ctx.Err() != nil {
return
}
log.Warn("cannot fetch bundle", "error", err)
if r.trigger == TriggerPush {
// Transport problems do not block later pulls of this revision.
report.Outcome = OutcomeRejected
report.Message = "fetch bundle: " + err.Error()
a.finish(log, report, nil)
}
return
}
minRev := st.Applied + 1
if r.trigger == TriggerPush {
minRev = st.Applied // an explicit push may re-apply the current revision
}
m, contents, err := a.verify(b, r.revision, minRev)
if err != nil {
reject("bundle verification failed", err)
return
}
rev, err := a.store.StoreRevision(m, b.Manifest, b.Signature, contents)
if err != nil {
reject("store revision", err)
return
}
if ctx.Err() != nil {
log.Info("shutdown before validation; revision not applied")
return
}
a.roleMu.Lock()
defer a.roleMu.Unlock()
rctx := context.WithoutCancel(ctx)
vctx, cancel := context.WithTimeout(rctx, a.cfg.ApplyTimeout)
err = a.role.Validate(vctx, rev)
cancel()
if err != nil {
reject("validation failed", err)
return
}
if ctx.Err() != nil {
log.Info("shutdown before activation; revision not applied")
return
}
a.mu.Lock()
prev := a.current
a.mu.Unlock()
actx, cancel := context.WithTimeout(rctx, a.cfg.ApplyTimeout)
err = a.role.Activate(actx, rev, prev)
cancel()
activated := err == nil
var reason string
if activated {
h := a.waitHealthy(rctx, &rev)
if h.State == HealthOK {
report.Outcome = OutcomeSucceeded
report.ActiveRevision = rev.Number
log.Info("revision applied")
a.finish(log, report, func(s *State) {
s.Applied, s.LastGood = rev.Number, rev.Number
a.current = &rev
a.health, a.healthAt = h, time.Now()
})
return
}
reason = "health check failed: " + describeHealth(h)
} else {
reason = "activation failed: " + err.Error()
}
log.Error("revision unhealthy, rolling back", "error", reason, "last_good", st.LastGood)
a.rollback(rctx, log, report, rev, activated, reason, st)
}
// rollback restores the last good revision after a failed activation.
func (a *Agent) rollback(ctx context.Context, log *slog.Logger, report DeploymentReport, rev Revision, activated bool, reason string, st State) {
// What is installed if the rollback cannot restore anything: the new
// revision if it was activated, otherwise (the role undid its own
// changes) the previous one.
installed := st.Applied
if activated {
installed = rev.Number
}
fail := func(msg string) {
report.Outcome = OutcomeFailed
report.Message = reason + "; " + msg
report.ActiveRevision = installed
log.Error("deployment failed", "error", report.Message)
a.finish(log, report, func(s *State) {
s.LastUnsuccessful = max(s.LastUnsuccessful, rev.Number)
if installed != s.Applied {
s.Applied = installed
a.current = &rev
}
})
}
if st.LastGood == 0 || st.LastGood == rev.Number {
fail("no last good revision to roll back to")
return
}
good, err := a.store.LoadRevision(st.LastGood)
if err != nil {
fail(fmt.Sprintf("rollback to revision %d impossible: %v", st.LastGood, err))
return
}
actx, cancel := context.WithTimeout(ctx, a.cfg.ApplyTimeout)
err = a.role.Activate(actx, good, &rev)
cancel()
if err != nil {
fail(fmt.Sprintf("rollback to revision %d failed: %v", good.Number, err))
return
}
installed = good.Number
h := a.waitHealthy(ctx, &good)
if h.State != HealthOK {
report.Outcome = OutcomeFailed
report.Message = fmt.Sprintf("%s; revision %d restored but unhealthy: %s", reason, good.Number, describeHealth(h))
} else {
report.Outcome = OutcomeRolledBack
report.Message = fmt.Sprintf("%s; rolled back to revision %d", reason, good.Number)
}
report.ActiveRevision = good.Number
log.Warn("deployment rolled back", "outcome", report.Outcome, "active_revision", good.Number)
a.finish(log, report, func(s *State) {
s.LastUnsuccessful = max(s.LastUnsuccessful, rev.Number)
s.Applied = good.Number
a.current = &good
a.health, a.healthAt = h, time.Now()
})
}
// verify checks a fetched bundle against the bundle signing key.
func (a *Agent) verify(b *Bundle, revision, minRev uint64) (*bundle.Manifest, map[string][]byte, error) {
contents := make(map[string][]byte, len(b.Artifacts))
for _, art := range b.Artifacts {
if _, dup := contents[art.Path]; dup {
return nil, nil, fmt.Errorf("duplicate artifact %s", art.Path)
}
contents[art.Path] = art.Content
}
m, err := bundle.Verify(a.bundleKey, b.Manifest, b.Signature, contents, bundle.Expect{
AgentID: a.agentID, Role: string(a.kind), MinRevision: minRev,
})
if err != nil {
return nil, nil, err
}
if m.Revision != revision {
return nil, nil, fmt.Errorf("bundle holds revision %d instead of %d", m.Revision, revision)
}
for _, f := range m.Files {
if reservedArtifact(f.Path) {
return nil, nil, fmt.Errorf("artifact path %s is reserved", f.Path)
}
}
return m, contents, nil
}
// waitHealthy calls Health every healthRetry until it reports OK or the
// health timeout has passed, and returns the last result.
func (a *Agent) waitHealthy(ctx context.Context, rev *Revision) Health {
ctx, cancel := context.WithTimeout(ctx, a.cfg.HealthTimeout)
defer cancel()
t := time.NewTimer(0)
defer t.Stop()
for {
h := a.role.Health(ctx, rev)
if h.State == HealthOK {
return h
}
t.Reset(a.healthRetry)
select {
case <-ctx.Done():
return h
case <-t.C:
}
}
}
// finish records the outcome: it updates the state (update runs under the
// state lock), queues the report, persists both and triggers the report
// delivery.
func (a *Agent) finish(log *slog.Logger, r DeploymentReport, update func(*State)) {
r.Message = truncate(r.Message, maxMessageLen)
if r.FinishedAt.IsZero() {
r.FinishedAt = time.Now()
}
a.mu.Lock()
if update != nil {
update(a.st)
}
last := r
a.st.LastResult = &last
a.st.PendingReports = append(a.st.PendingReports, r)
a.st.UpdatedAt = time.Now().UTC()
err := a.store.SaveState(a.st)
a.mu.Unlock()
if err != nil {
log.Error("cannot save state", "error", err)
}
a.prune()
notify(a.flushCh)
}
// prune removes old revisions, keeping the applied and last good one.
func (a *Agent) prune() {
st := a.state()
if err := a.store.Prune(KeepRevisions, st.Applied, st.LastGood); err != nil {
a.log.Warn("cannot prune old revisions", "error", err)
}
}
// reportLoop delivers queued deployment reports.
func (a *Agent) reportLoop(ctx context.Context) {
t := time.NewTicker(a.reportRetry)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-a.flushCh:
case <-t.C:
}
a.flushReports(ctx)
}
}
// flushReports sends queued reports in order until one fails.
func (a *Agent) flushReports(ctx context.Context) {
for ctx.Err() == nil {
a.mu.Lock()
if len(a.st.PendingReports) == 0 {
a.mu.Unlock()
return
}
r := a.st.PendingReports[0]
a.mu.Unlock()
rctx, cancel := context.WithTimeout(ctx, reportTimeout)
err := a.source.ReportDeployment(rctx, r)
cancel()
if err != nil {
a.log.Debug("cannot deliver deployment report; will retry", "revision", r.Revision,
"deployment_id", r.DeploymentID, "error", err)
return
}
a.log.Info("deployment reported", "revision", r.Revision, "deployment_id", r.DeploymentID, "outcome", r.Outcome)
a.mu.Lock()
if i := slices.IndexFunc(a.st.PendingReports, func(p DeploymentReport) bool { return reflect.DeepEqual(p, r) }); i >= 0 {
a.st.PendingReports = slices.Delete(a.st.PendingReports, i, i+1)
}
err = a.store.SaveState(a.st)
a.mu.Unlock()
if err != nil {
a.log.Error("cannot save state", "error", err)
}
}
}
// describeHealth renders a non-OK health result for reports.
func describeHealth(h Health) string {
var b strings.Builder
b.WriteString(h.State.String())
if h.Message != "" {
b.WriteString(": " + h.Message)
}
for _, c := range h.Checks {
if c.State == HealthOK {
continue
}
fmt.Fprintf(&b, "; %s: %s", c.Name, c.State)
if c.Message != "" {
b.WriteString(" (" + c.Message + ")")
}
}
return b.String()
}
// String returns the lowercase name of the state.
func (s HealthState) String() string {
switch s {
case HealthOK:
return "ok"
case HealthDegraded:
return "degraded"
case HealthFailed:
return "failed"
default:
return "unknown"
}
}
// truncate shortens s to at most n bytes without splitting a UTF-8
// sequence.
func truncate(s string, n int) string {
if len(s) <= n {
return s
}
const marker = " [truncated]"
cut := n - len(marker)
for cut > 0 && !utf8.RuneStart(s[cut]) {
cut--
}
return s[:cut] + marker
}
// notify performs a non-blocking send on a 1-buffered channel.
func notify(ch chan struct{}) {
select {
case ch <- struct{}{}:
default:
}
}
// errRoleNotAvailable is returned by the placeholder role factory.
var errRoleNotAvailable = errors.New("role not available in this build")
agent/core/agent_test.go
+548
View File
@@ -0,0 +1,548 @@
// SPDX-License-Identifier: Apache-2.0
package core
import (
"context"
"errors"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
)
func TestApplySucceeded(t *testing.T) {
f := newFixture(t, KindDNS)
f.addBundle(t, 1)
f.pull(1)
r := f.lastResult(t)
if r.Outcome != OutcomeSucceeded || r.Revision != 1 || r.ActiveRevision != 1 || r.Trigger != TriggerPull {
t.Fatalf("result = %+v", r)
}
if r.StartedAt.IsZero() || r.FinishedAt.Before(r.StartedAt) {
t.Errorf("timestamps = %v .. %v", r.StartedAt, r.FinishedAt)
}
st := f.agent.state()
if st.Applied != 1 || st.LastGood != 1 || st.LastUnsuccessful != 0 {
t.Errorf("state = %+v", st)
}
if got, want := f.role.callLog(), []string{"validate 1", "activate 1 prev none"}; !slices.Equal(got, want) {
t.Errorf("calls = %q, want %q", got, want)
}
// Stored revision: manifest, signature and artifacts.
dir := filepath.Join(f.store.Dir(), "revisions", "1")
for _, name := range []string{"manifest.json", "manifest.sig", "ddi-suite.conf", filepath.Join("zones", "db.example.com")} {
if _, err := os.Stat(filepath.Join(dir, name)); err != nil {
t.Errorf("stored revision lacks %s: %v", name, err)
}
}
// The report is persisted until it is delivered.
saved, err := f.store.LoadState()
if err != nil {
t.Fatal(err)
}
if len(saved.PendingReports) != 1 || saved.Applied != 1 {
t.Fatalf("saved state = %+v", saved)
}
f.agent.flushReports(context.Background())
if rep := f.src.reported(); len(rep) != 1 || rep[0].Outcome != OutcomeSucceeded {
t.Fatalf("reports = %+v", rep)
}
if saved, _ := f.store.LoadState(); len(saved.PendingReports) != 0 {
t.Errorf("pending reports after delivery = %d", len(saved.PendingReports))
}
// Next revision: Activate gets the previous one.
f.addBundle(t, 2)
f.role.resetCalls()
f.pull(2)
if got, want := f.role.callLog(), []string{"validate 2", "activate 2 prev 1"}; !slices.Equal(got, want) {
t.Errorf("calls = %q, want %q", got, want)
}
if st := f.agent.state(); st.Applied != 2 || st.LastGood != 2 {
t.Errorf("state = %+v", st)
}
}
func TestApplyRejectedOnValidate(t *testing.T) {
f := newFixture(t, KindDNS)
f.addBundle(t, 1)
f.addBundle(t, 2)
f.pull(1)
f.role.validateErr[2] = errors.New("zone example.com: bad SOA")
f.role.resetCalls()
f.pull(2)
r := f.lastResult(t)
if r.Outcome != OutcomeRejected || r.ActiveRevision != 1 || !strings.Contains(r.Message, "bad SOA") {
t.Fatalf("result = %+v", r)
}
if got := f.role.callLog(); !slices.Equal(got, []string{"validate 2"}) {
t.Errorf("calls = %q", got)
}
st := f.agent.state()
if st.Applied != 1 || st.LastGood != 1 || st.LastUnsuccessful != 2 {
t.Errorf("state = %+v", st)
}
// No automatic retry of the same revision.
f.role.resetCalls()
f.src.fetches = 0
f.pull(2)
if len(f.role.callLog()) != 0 || f.src.fetches != 0 {
t.Errorf("rejected revision was retried: calls %q, fetches %d", f.role.callLog(), f.src.fetches)
}
// An explicit push retries it.
delete(f.role.validateErr, 2)
f.push(2, "dep-7")
r = f.lastResult(t)
if r.Outcome != OutcomeSucceeded || r.DeploymentID != "dep-7" || r.Trigger != TriggerPush {
t.Fatalf("push result = %+v", r)
}
// A newer revision is pulled again.
f.addBundle(t, 3)
f.pull(3)
if st := f.agent.state(); st.Applied != 3 {
t.Errorf("state = %+v", st)
}
}
func TestRollbackOnActivateError(t *testing.T) {
f := newFixture(t, KindDNS)
f.addBundle(t, 1)
f.addBundle(t, 2)
f.pull(1)
f.role.activateErr[2] = errors.New("rndc reconfig failed")
f.role.resetCalls()
f.pull(2)
r := f.lastResult(t)
if r.Outcome != OutcomeRolledBack || r.ActiveRevision != 1 || !strings.Contains(r.Message, "rndc reconfig failed") {
t.Fatalf("result = %+v", r)
}
want := []string{"validate 2", "activate 2 prev 1", "activate 1 prev 2"}
if got := f.role.callLog(); !slices.Equal(got, want) {
t.Errorf("calls = %q, want %q", got, want)
}
st := f.agent.state()
if st.Applied != 1 || st.LastGood != 1 || st.LastUnsuccessful != 2 {
t.Errorf("state = %+v", st)
}
// Not retried by a later poll.
f.role.resetCalls()
f.pull(2)
if len(f.role.callLog()) != 0 {
t.Errorf("rolled back revision retried: %q", f.role.callLog())
}
}
func TestRollbackOnHealthFailure(t *testing.T) {
f := newFixture(t, KindDNS)
f.addBundle(t, 1)
f.addBundle(t, 2)
f.pull(1)
f.role.unhealthy[2] = true
f.pull(2)
r := f.lastResult(t)
if r.Outcome != OutcomeRolledBack || r.ActiveRevision != 1 {
t.Fatalf("result = %+v", r)
}
if !strings.Contains(r.Message, "serial mismatch") || !strings.Contains(r.Message, "rolled back to revision 1") {
t.Errorf("message = %q", r.Message)
}
if n := f.role.healthCalls[2]; n < 2 {
t.Errorf("health of revision 2 checked %d times, want retries", n)
}
if f.role.active != 1 {
t.Errorf("active = %d", f.role.active)
}
if st := f.agent.state(); st.Applied != 1 || st.LastGood != 1 {
t.Errorf("state = %+v", st)
}
}
func TestFailedWithoutLastGood(t *testing.T) {
f := newFixture(t, KindDNS)
f.addBundle(t, 1)
f.role.unhealthy[1] = true
f.pull(1)
r := f.lastResult(t)
if r.Outcome != OutcomeFailed || r.ActiveRevision != 1 || !strings.Contains(r.Message, "no last good revision") {
t.Fatalf("result = %+v", r)
}
st := f.agent.state()
if st.Applied != 1 || st.LastGood != 0 || st.LastUnsuccessful != 1 {
t.Errorf("state = %+v", st)
}
// Activation error without last good: the previous state stays.
g := newFixture(t, KindDNS)
g.addBundle(t, 1)
g.role.activateErr[1] = errors.New("boom")
g.pull(1)
if r := g.lastResult(t); r.Outcome != OutcomeFailed || r.ActiveRevision != 0 {
t.Fatalf("result = %+v", r)
}
if st := g.agent.state(); st.Applied != 0 {
t.Errorf("state = %+v", st)
}
}
func TestFailedWhenRollbackFails(t *testing.T) {
f := newFixture(t, KindDNS)
f.addBundle(t, 1)
f.addBundle(t, 2)
f.pull(1)
f.role.unhealthy[2] = true
f.role.activateErr[1] = errors.New("restore failed")
f.pull(2)
r := f.lastResult(t)
if r.Outcome != OutcomeFailed || r.ActiveRevision != 2 || !strings.Contains(r.Message, "rollback to revision 1 failed: restore failed") {
t.Fatalf("result = %+v", r)
}
if st := f.agent.state(); st.Applied != 2 || st.LastGood != 1 {
t.Errorf("state = %+v", st)
}
}
func TestFailedWhenRollbackUnhealthy(t *testing.T) {
f := newFixture(t, KindDNS)
f.addBundle(t, 1)
f.addBundle(t, 2)
f.pull(1)
f.role.unhealthy[1] = true
f.role.unhealthy[2] = true
f.pull(2)
r := f.lastResult(t)
if r.Outcome != OutcomeFailed || r.ActiveRevision != 1 || !strings.Contains(r.Message, "restored but unhealthy") {
t.Fatalf("result = %+v", r)
}
if st := f.agent.state(); st.Applied != 1 || st.LastGood != 1 || st.LastUnsuccessful != 2 {
t.Errorf("state = %+v", st)
}
}
func TestReplayProtection(t *testing.T) {
f := newFixture(t, KindDNS)
for rev := uint64(1); rev <= 3; rev++ {
f.addBundle(t, rev)
}
f.pull(3)
if st := f.agent.state(); st.Applied != 3 {
t.Fatalf("state = %+v", st)
}
// Older revisions: a pull is ignored, a push is rejected.
f.role.resetCalls()
f.pull(2)
if len(f.role.callLog()) != 0 || len(f.agent.state().PendingReports) != 0 {
t.Errorf("older pull was processed")
}
f.push(2, "dep-old")
r := f.lastResult(t)
if r.Outcome != OutcomeRejected || r.Revision != 2 || !strings.Contains(r.Message, "older") || r.ActiveRevision != 3 {
t.Fatalf("result = %+v", r)
}
if len(f.role.callLog()) != 0 {
t.Errorf("role called for an older revision: %q", f.role.callLog())
}
// The server answers revision 4 with the (validly signed) bundle of 2.
f.src.bundles[4] = f.src.bundles[2]
f.pull(4)
if r := f.lastResult(t); r.Outcome != OutcomeRejected || !strings.Contains(r.Message, "older than 4") {
t.Fatalf("result = %+v", r)
}
// An explicit push of the applied revision re-applies it.
f.push(3, "dep-again")
if r := f.lastResult(t); r.Outcome != OutcomeSucceeded || r.Revision != 3 {
t.Fatalf("result = %+v", r)
}
if st := f.agent.state(); st.Applied != 3 {
t.Errorf("state = %+v", st)
}
}
func TestRejectedBundles(t *testing.T) {
cases := map[string]func(f *fixture) *Bundle{
"bad signature": func(f *fixture) *Bundle {
b := signBundle(t, f.signer, testAgentID, KindDNS, 1)
b.Signature[0] ^= 1
return b
},
"other agent": func(f *fixture) *Bundle {
return signBundle(t, f.signer, "11111111-2222-4333-8444-555555555555", KindDNS, 1)
},
"other role": func(f *fixture) *Bundle { return signBundle(t, f.signer, testAgentID, KindDHCP, 1) },
"other key": func(*fixture) *Bundle {
_, other := newSigningKey(t)
return signBundle(t, other, testAgentID, KindDNS, 1)
},
"tampered content": func(f *fixture) *Bundle {
b := signBundle(t, f.signer, testAgentID, KindDNS, 1)
b.Artifacts[0].Content = []byte("evil")
return b
},
"duplicate artifact": func(f *fixture) *Bundle {
b := signBundle(t, f.signer, testAgentID, KindDNS, 1)
b.Artifacts = append(b.Artifacts, b.Artifacts[0])
return b
},
}
for name, mk := range cases {
t.Run(name, func(t *testing.T) {
f := newFixture(t, KindDNS)
f.src.bundles[1] = mk(f)
f.pull(1)
r := f.lastResult(t)
if r.Outcome != OutcomeRejected || !strings.Contains(r.Message, "verification failed") {
t.Fatalf("result = %+v", r)
}
if len(f.role.callLog()) != 0 {
t.Errorf("role called: %q", f.role.callLog())
}
if revs, _ := f.store.ListRevisions(); len(revs) != 0 {
t.Errorf("rejected bundle stored: %v", revs)
}
})
}
}
func TestFetchError(t *testing.T) {
f := newFixture(t, KindDNS)
f.src.fetchErr = errors.New("unavailable")
f.pull(1)
if st := f.agent.state(); st.LastResult != nil || st.LastUnsuccessful != 0 {
t.Fatalf("pull fetch error recorded: %+v", st)
}
f.push(1, "dep-1")
r := f.lastResult(t)
if r.Outcome != OutcomeRejected || !strings.Contains(r.Message, "fetch bundle: unavailable") {
t.Fatalf("result = %+v", r)
}
// A transport problem does not block a later pull.
f.src.fetchErr = nil
f.addBundle(t, 1)
f.pull(1)
if st := f.agent.state(); st.Applied != 1 {
t.Errorf("state = %+v", st)
}
}
func TestShutdownBeforeValidation(t *testing.T) {
f := newFixture(t, KindDNS)
f.addBundle(t, 1)
ctx, cancel := context.WithCancel(context.Background())
f.src.onFetch = cancel
f.agent.handle(ctx, applyRequest{revision: 1, trigger: TriggerPull})
if len(f.role.callLog()) != 0 {
t.Errorf("role called after shutdown: %q", f.role.callLog())
}
if st := f.agent.state(); st.LastResult != nil || st.Applied != 0 {
t.Errorf("state = %+v", st)
}
}
func TestReportQueueWhileServerDown(t *testing.T) {
f := newFixture(t, KindDNS)
f.src.setReportErr(errors.New("server down"))
f.addBundle(t, 1)
f.addBundle(t, 2)
f.pull(1)
f.role.validateErr[2] = errors.New("bad")
f.pull(2)
f.agent.flushReports(context.Background())
if n := len(f.src.reported()); n != 0 {
t.Fatalf("%d reports delivered while down", n)
}
// After a restart the queue is still there and is sent in order.
g := newFixtureWithStore(t, KindDNS, f.store, f.signer)
if st, _ := g.store.LoadState(); len(st.PendingReports) != 2 {
t.Fatalf("pending reports after restart = %d", len(st.PendingReports))
}
g.agent.flushReports(context.Background())
rep := g.src.reported()
if len(rep) != 2 || rep[0].Revision != 1 || rep[1].Revision != 2 || rep[1].Outcome != OutcomeRejected {
t.Fatalf("reports = %+v", rep)
}
if st, _ := g.store.LoadState(); len(st.PendingReports) != 0 {
t.Errorf("pending after delivery = %d", len(st.PendingReports))
}
}
func TestPendingReportsBounded(t *testing.T) {
f := newFixture(t, KindDNS)
f.src.setReportErr(errors.New("down"))
for i := 0; i < maxPendingReports+5; i++ {
f.push(1, "dep") // fetch fails: rejected report each time
}
st, err := f.store.LoadState()
if err != nil {
t.Fatal(err)
}
if len(st.PendingReports) != maxPendingReports {
t.Errorf("pending = %d, want %d", len(st.PendingReports), maxPendingReports)
}
}
func TestPruneKeepsAppliedAndLastGood(t *testing.T) {
f := newFixture(t, KindDNS)
f.addBundle(t, 1)
f.pull(1)
for rev := uint64(2); rev <= 14; rev++ {
f.addBundle(t, rev)
f.role.validateErr[rev] = errors.New("bad")
f.push(rev, "")
}
revs, err := f.store.ListRevisions()
if err != nil {
t.Fatal(err)
}
if !slices.Contains(revs, 1) || len(revs) != KeepRevisions+1 || revs[len(revs)-1] != 14 {
t.Errorf("revisions = %v", revs)
}
}
func TestEnqueue(t *testing.T) {
f := newFixture(t, KindDNS)
a := f.agent
a.enqueue(applyRequest{revision: 5, trigger: TriggerPull})
a.enqueue(applyRequest{revision: 4, trigger: TriggerPull}) // dropped: 5 queued
a.enqueue(applyRequest{revision: 5, trigger: TriggerPull}) // dropped: duplicate
a.enqueue(applyRequest{revision: 6, trigger: TriggerPush, deploymentID: "d"})
a.enqueue(applyRequest{revision: 7, trigger: TriggerPull})
var got []applyRequest
for {
r, ok := a.dequeue()
if !ok {
break
}
got = append(got, r)
}
want := []applyRequest{{revision: 6, trigger: TriggerPush, deploymentID: "d"}, {revision: 7, trigger: TriggerPull}}
if !slices.Equal(got, want) {
t.Errorf("queue = %+v, want %+v", got, want)
}
}
func TestDesiredAndPush(t *testing.T) {
f := newFixture(t, KindDNS)
f.agent.Desired(0, true)
f.agent.Desired(3, false) // manual mode
if _, ok := f.agent.dequeue(); ok {
t.Fatal("request queued for revision 0 or manual mode")
}
f.agent.Desired(3, true)
f.agent.Push(4, "dep")
r1, _ := f.agent.dequeue()
r2, _ := f.agent.dequeue()
if r1.revision != 3 || r1.trigger != TriggerPull || r2.revision != 4 || r2.deploymentID != "dep" {
t.Errorf("requests = %+v %+v", r1, r2)
}
}
func TestRunAppliesAndReports(t *testing.T) {
f := newFixture(t, KindDHCP)
f.addBundle(t, 1)
f.role.setLeases([]Lease{{IP: mustAddr("10.0.0.5"), MAC: "aa:bb:cc:dd:ee:ff", State: LeaseActive}})
f.src.run = func(ctx context.Context, h SourceHandler) error {
h.Connected()
h.Desired(1, true)
<-ctx.Done()
return ctx.Err()
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- f.agent.Run(ctx) }()
waitFor(t, "deployment report", func() bool { return len(f.src.reported()) == 1 })
waitFor(t, "lease snapshot", func() bool { return len(f.src.leaseBatches()) > 0 })
if b := f.src.leaseBatches()[0]; !b.FullSync || len(b.Leases) != 1 {
t.Errorf("first lease batch = %+v", b)
}
cancel()
select {
case err := <-done:
if err != nil {
t.Errorf("Run = %v", err)
}
case <-time.After(10 * time.Second):
t.Fatal("Run did not return")
}
}
func TestRunSourceError(t *testing.T) {
f := newFixture(t, KindDNS)
f.src.run = func(context.Context, SourceHandler) error { return errors.New("broken source") }
if err := f.agent.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "broken source") {
t.Errorf("Run = %v", err)
}
}
func TestStatus(t *testing.T) {
f := newFixture(t, KindDNS)
f.addBundle(t, 1)
f.pull(1)
st := f.agent.Status(context.Background())
if st.Applied != 1 || st.LastGood != 1 || st.Health.State != HealthOK || st.ServiceVersion != "Fake 1.0" || st.CheckedAt.IsZero() {
t.Errorf("status = %+v", st)
}
// While the role is busy the cached result is returned.
f.role.unhealthy[1] = true
f.agent.roleMu.Lock()
cached := f.agent.Status(context.Background())
f.agent.roleMu.Unlock()
if cached.Health.State != HealthOK {
t.Errorf("cached status = %+v", cached)
}
if fresh := f.agent.Status(context.Background()); fresh.Health.State != HealthFailed {
t.Errorf("fresh status = %+v", fresh)
}
}
func TestNewAgentLoadsAppliedRevision(t *testing.T) {
f := newFixture(t, KindDNS)
f.addBundle(t, 1)
f.pull(1)
g := newFixtureWithStore(t, KindDNS, f.store, f.signer)
if g.agent.current == nil || g.agent.current.Number != 1 || len(g.agent.current.Files) != 2 {
t.Fatalf("current = %+v", g.agent.current)
}
// Role mismatch.
id := &Identity{AgentID: testAgentID}
if _, err := NewAgent(KindDHCP, testConfig(f.store.Dir()), newFakeRole(KindDNS), f.store, id, f.src, testLogger()); err == nil {
t.Error("role mismatch accepted")
}
}
func TestDescribeHealthAndTruncate(t *testing.T) {
h := Health{State: HealthDegraded, Message: "slow", Checks: []HealthCheck{
{Name: "a", State: HealthOK}, {Name: "b", State: HealthFailed, Message: "down"}, {Name: "c", State: HealthUnknown},
}}
if got, want := describeHealth(h), "degraded: slow; b: failed (down); c: unknown"; got != want {
t.Errorf("describeHealth = %q, want %q", got, want)
}
if got := truncate("short", 10); got != "short" {
t.Errorf("truncate = %q", got)
}
long := strings.Repeat("ä", 100) // 200 bytes
got := truncate(long, 50)
if len(got) > 50 || !strings.HasSuffix(got, "[truncated]") || !strings.HasPrefix(got, "ää") {
t.Errorf("truncate = %q (%d bytes)", got, len(got))
}
for _, r := range strings.TrimSuffix(got, " [truncated]") {
if r != 'ä' {
t.Fatalf("truncate split a rune: %q", got)
}
}
}
agent/core/cli.go
+272 -12
View File
@@ -1,24 +1,37 @@
// SPDX-License-Identifier: Apache-2.0
// Package core is the role-independent part of the agents: command line,
// enrollment, configuration source, verification and apply pipeline
// (see docs/ARCHITECTURE.md, section 4). M0 provides the command line frame;
// enrollment and the apply pipeline follow in M3.
// configuration, local state, enrollment, the connection to the management
// server and the apply pipeline (see docs/ARCHITECTURE.md, section 4, and
// docs/adr/0021-agent-strecke-m3.md). The roles (BIND, Kea) implement Role.
package core
import (
"context"
"errors"
"flag"
"fmt"
"io"
"log/slog"
"os"
"os/signal"
"strings"
"syscall"
"time"
"git.beugsiud.de/beug.k/DDI/internal/agentpki"
"git.beugsiud.de/beug.k/DDI/internal/buildinfo"
"git.beugsiud.de/beug.k/DDI/internal/logging"
)
// RunCLI dispatches the agent command line and returns the process exit code.
func RunCLI(binary string, role Kind, args []string, stdout, stderr io.Writer) int {
// Main dispatches the agent command line and returns the process exit code.
// factory creates the role for the run command.
func Main(binary string, kind Kind, factory RoleFactory, args []string, stdout, stderr io.Writer) int {
if len(args) == 0 {
usage(stderr, binary, role)
usage(stderr, binary, kind)
return 2
}
c := &cli{binary: binary, kind: kind, factory: factory, stdout: stdout, stderr: stderr}
switch args[0] {
case "version", "--version":
if err := buildinfo.Print(stdout, binary); err != nil {
@@ -26,22 +39,269 @@ func RunCLI(binary string, role Kind, args []string, stdout, stderr io.Writer) i
}
return 0
case "help", "-h", "--help":
usage(stdout, binary, role)
usage(stdout, binary, kind)
return 0
case "enroll":
return c.enroll(args[1:])
case "run":
return c.run(args[1:])
case "status":
return c.status(args[1:])
default:
_, _ = fmt.Fprintf(stderr, "%s: unknown command %q\n\n", binary, args[0])
usage(stderr, binary, role)
usage(stderr, binary, kind)
return 2
}
}
func usage(w io.Writer, binary string, role Kind) {
_, _ = fmt.Fprintf(w, `Usage: %s <command>
func usage(w io.Writer, binary string, kind Kind) {
_, _ = fmt.Fprintf(w, `Usage: %[1]s <command> [flags]
Agent role: %s
Agent role: %[2]s
Commands:
enroll register with the management server
--server <host:port> --token <token> --ca-hash sha256:<hex>
[--server-name <name>] [--force]
run run the agent (until SIGINT/SIGTERM)
status show identity, revisions and the last result
version print version information
help show this help
`, binary, role)
Common flags:
--config <file> configuration file (default %[3]s)
`, binary, kind, DefaultConfigPath(kind))
}
type cli struct {
binary string
kind Kind
factory RoleFactory
stdout io.Writer
stderr io.Writer
}
func (c *cli) flags(name string) (*flag.FlagSet, *string) {
fs := flag.NewFlagSet(c.binary+" "+name, flag.ContinueOnError)
fs.SetOutput(c.stderr)
cfg := fs.String("config", DefaultConfigPath(c.kind), "configuration file")
return fs, cfg
}
func (c *cli) fail(err error) int {
_, _ = fmt.Fprintf(c.stderr, "%s: %v\n", c.binary, err)
return 1
}
// parse parses flags and rejects positional arguments.
func parse(fs *flag.FlagSet, args []string) int {
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if fs.NArg() > 0 {
_, _ = fmt.Fprintf(fs.Output(), "unexpected argument %q\n", fs.Arg(0))
return 2
}
return -1
}
func (c *cli) enroll(args []string) int {
fs, cfgPath := c.flags("enroll")
var opts EnrollOptions
fs.StringVar(&opts.Server, "server", "", "agent API address of the management server (host:port)")
fs.StringVar(&opts.Token, "token", "", "one-time enrollment token")
fs.StringVar(&opts.CAHash, "ca-hash", "", "fingerprint of the agent CA (sha256:<hex>)")
fs.StringVar(&opts.ServerName, "server-name", "", "TLS server name (default: host of --server)")
fs.BoolVar(&opts.Force, "force", false, "replace an existing identity")
if code := parse(fs, args); code >= 0 {
return code
}
if opts.Server == "" || opts.Token == "" || opts.CAHash == "" {
_, _ = fmt.Fprintln(c.stderr, "enroll needs --server, --token and --ca-hash")
return 2
}
cfg, err := LoadConfig(*cfgPath, c.kind)
if err != nil {
return c.fail(err)
}
store, unlock, err := openLocked(cfg)
if err != nil {
return c.fail(err)
}
defer unlock()
opts.Kind = c.kind
opts.Hostname, _ = os.Hostname()
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
id, err := Enroll(ctx, store, opts)
if err != nil {
return c.fail(err)
}
_, _ = fmt.Fprintf(c.stdout, "Enrolled as agent %s (%s) with %s.\nIdentity stored in %s.\n",
id.AgentID, c.kind, id.Server, store.Dir())
return 0
}
func openLocked(cfg *Config) (*Store, func(), error) {
store, err := OpenStore(cfg.StateDir)
if err != nil {
return nil, nil, err
}
unlock, err := store.Lock()
if err != nil {
return nil, nil, err
}
return store, unlock, nil
}
func (c *cli) run(args []string) int {
fs, cfgPath := c.flags("run")
if code := parse(fs, args); code >= 0 {
return code
}
cfg, err := LoadConfig(*cfgPath, c.kind)
if err != nil {
return c.fail(err)
}
log, err := logging.New(c.stderr, cfg.Log)
if err != nil {
return c.fail(err)
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := runAgent(ctx, c.kind, c.factory, cfg, log); err != nil {
log.Error("agent stopped", "error", err)
return 1
}
return 0
}
// runAgent runs the agent until ctx is canceled.
func runAgent(ctx context.Context, kind Kind, factory RoleFactory, cfg *Config, log *slog.Logger) error {
if factory == nil {
return errRoleNotAvailable
}
store, unlock, err := openLocked(cfg)
if err != nil {
return err
}
defer unlock()
id, err := store.LoadIdentity()
if err != nil {
return err
}
if id.Role != kind {
return fmt.Errorf("identity in %s belongs to a %s agent", store.Dir(), id.Role)
}
role, err := factory(cfg.DecodeRole)
if err != nil {
return fmt.Errorf("%s role: %w", kind, err)
}
log = log.With("agent_id", id.AgentID, "role", string(kind))
hostname, _ := os.Hostname()
src := NewServerSource(store, id, hostname, log)
agent, err := NewAgent(kind, cfg, role, store, id, src, log)
if err != nil {
return err
}
log.Info("agent started", "version", buildinfo.Get().Version, "server", id.Server, "state_dir", store.Dir())
err = agent.Run(ctx)
log.Info("agent stopped; the managed service keeps its configuration")
return err
}
func (c *cli) status(args []string) int {
fs, cfgPath := c.flags("status")
if code := parse(fs, args); code >= 0 {
return code
}
cfg, err := LoadConfig(*cfgPath, c.kind)
if err != nil {
return c.fail(err)
}
store, err := OpenStore(cfg.StateDir)
if err != nil {
return c.fail(err)
}
if err := writeStatus(c.stdout, store); err != nil {
return c.fail(err)
}
return 0
}
// writeStatus prints identity, revisions and the last result.
func writeStatus(w io.Writer, store *Store) error {
var b strings.Builder
line := func(k, format string, a ...any) { fmt.Fprintf(&b, "%-20s %s\n", k+":", fmt.Sprintf(format, a...)) }
line("State directory", "%s", store.Dir())
id, err := store.LoadIdentity()
switch {
case errors.Is(err, ErrNotEnrolled):
line("Identity", "not enrolled")
case err != nil:
return err
default:
leaf := id.Certificate.Leaf
line("Agent ID", "%s", id.AgentID)
line("Role", "%s", id.Role)
line("Server", "%s (TLS name %s)", id.Server, id.ServerName)
line("Enrolled", "%s", id.EnrolledAt.Format(time.RFC3339))
line("Certificate", "valid until %s, renewal from %s", leaf.NotAfter.UTC().Format(time.RFC3339),
renewalDue(leaf).UTC().Format(time.RFC3339))
line("CA fingerprint", "%s", agentpki.Fingerprint(id.CA))
}
st, err := store.LoadState()
if err != nil {
return err
}
revs, err := store.ListRevisions()
if err != nil {
return err
}
line("Applied revision", "%s", revOrNone(st.Applied))
line("Last good revision", "%s", revOrNone(st.LastGood))
if st.LastUnsuccessful != 0 {
line("Last unsuccessful", "%d (retried only by a newer revision or an explicit push)", st.LastUnsuccessful)
}
nums := make([]string, len(revs))
for i, n := range revs {
nums[i] = fmt.Sprint(n)
}
line("Stored revisions", "%s", orNone(strings.Join(nums, ", ")))
if r := st.LastResult; r != nil {
dep := ""
if r.DeploymentID != "" {
dep = ", deployment " + r.DeploymentID
}
line("Last result", "revision %d %s at %s (%s%s)", r.Revision, r.Outcome,
r.FinishedAt.UTC().Format(time.RFC3339), r.Trigger, dep)
if r.Message != "" {
line("Message", "%s", r.Message)
}
} else {
line("Last result", "none")
}
line("Pending reports", "%d", len(st.PendingReports))
if st.LeaseSequence != 0 {
line("Lease sequence", "%d", st.LeaseSequence)
}
_, err = io.WriteString(w, b.String())
return err
}
func revOrNone(n uint64) string {
if n == 0 {
return "none"
}
return fmt.Sprint(n)
}
func orNone(s string) string {
if s == "" {
return "none"
}
return s
}
agent/core/cli_test.go
+156 -3
View File
@@ -4,11 +4,16 @@ package core
import (
"bytes"
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestRunCLI(t *testing.T) {
func TestMainBasics(t *testing.T) {
tests := []struct {
name string
args []string
@@ -20,14 +25,19 @@ func TestRunCLI(t *testing.T) {
{name: "version", args: []string{"version"}, wantCode: 0, wantStdout: "ddi-agent-test "},
{name: "--version", args: []string{"--version"}, wantCode: 0, wantStdout: "ddi-agent-test "},
{name: "help", args: []string{"help"}, wantCode: 0, wantStdout: "Agent role: dns"},
{name: "help lists enroll", args: []string{"--help"}, wantCode: 0, wantStdout: "--ca-hash sha256:<hex>"},
{name: "unknown", args: []string{"explode"}, wantCode: 2, wantStderr: `unknown command "explode"`},
{name: "enroll without flags", args: []string{"enroll"}, wantCode: 2, wantStderr: "enroll needs --server"},
{name: "bad flag", args: []string{"run", "--bogus"}, wantCode: 2, wantStderr: "flag provided but not defined"},
{name: "extra argument", args: []string{"status", "extra"}, wantCode: 2, wantStderr: `unexpected argument "extra"`},
{name: "flag help", args: []string{"status", "-h"}, wantCode: 0, wantStderr: "-config"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var out, errOut bytes.Buffer
code := RunCLI("ddi-agent-test", KindDNS, tt.args, &out, &errOut)
code := Main("ddi-agent-test", KindDNS, nil, tt.args, &out, &errOut)
if code != tt.wantCode {
t.Errorf("code = %d, want %d", code, tt.wantCode)
t.Errorf("code = %d, want %d (stderr %q)", code, tt.wantCode, errOut.String())
}
if tt.wantStdout != "" && !strings.Contains(out.String(), tt.wantStdout) {
t.Errorf("stdout %q does not contain %q", out.String(), tt.wantStdout)
@@ -38,3 +48,146 @@ func TestRunCLI(t *testing.T) {
})
}
}
// writeConfig writes a configuration with its own state directory.
func writeConfig(t *testing.T, extra string) (cfgPath, stateDir string) {
t.Helper()
dir := t.TempDir()
stateDir = filepath.Join(dir, "state")
cfgPath = filepath.Join(dir, "agent.yaml")
data := "state_dir: " + filepath.ToSlash(stateDir) + "\nlog:\n format: text\n" + extra
if err := os.WriteFile(cfgPath, []byte(data), 0o600); err != nil {
t.Fatal(err)
}
return cfgPath, stateDir
}
func TestMainEnrollStatusRun(t *testing.T) {
srv := newFakeServer(t, KindDNS)
addr := srv.start(t)
cfgPath, stateDir := writeConfig(t, "dns:\n zone_dir: /var/lib/bind/x\n")
var out, errOut bytes.Buffer
if code := Main("ddi-agent-dns", KindDNS, nil, []string{"status", "--config", cfgPath}, &out, &errOut); code != 0 {
t.Fatalf("status before enroll = %d: %s", code, errOut.String())
}
if !strings.Contains(out.String(), "not enrolled") {
t.Errorf("status = %q", out.String())
}
// run without identity fails.
out.Reset()
errOut.Reset()
factory := func(decode func(any) error) (Role, error) { return newFakeRole(KindDNS), nil }
if code := Main("ddi-agent-dns", KindDNS, factory, []string{"run", "--config", cfgPath}, &out, &errOut); code != 1 {
t.Fatalf("run before enroll = %d", code)
}
if !strings.Contains(errOut.String(), "not enrolled") {
t.Errorf("stderr = %q", errOut.String())
}
args := []string{"enroll", "--config", cfgPath, "--server", addr, "--token", srv.token,
"--ca-hash", srv.fingerprint(), "--server-name", "localhost"}
out.Reset()
errOut.Reset()
if code := Main("ddi-agent-dns", KindDNS, nil, args, &out, &errOut); code != 0 {
t.Fatalf("enroll = %d: %s", code, errOut.String())
}
if !strings.Contains(out.String(), srv.agentID) {
t.Errorf("enroll output = %q", out.String())
}
out.Reset()
errOut.Reset()
if code := Main("ddi-agent-dns", KindDNS, nil, args, &out, &errOut); code != 1 || !strings.Contains(errOut.String(), "--force") {
t.Errorf("second enroll = %d: %s", code, errOut.String())
}
// Run the agent until revision 1 is reported.
srv.with(func(f *fakeServer) { f.desired = 1 })
cfg, err := LoadConfig(cfgPath, KindDNS)
if err != nil {
t.Fatal(err)
}
var gotSection struct {
ZoneDir string `yaml:"zone_dir"`
}
role := newFakeRole(KindDNS)
factory = func(decode func(any) error) (Role, error) {
if err := decode(&gotSection); err != nil {
return nil, err
}
return role, nil
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- runAgent(ctx, KindDNS, factory, cfg, testLogger()) }()
waitFor(t, "deployment report", func() bool { return len(serverReports(srv)) == 1 })
cancel()
select {
case err := <-done:
if err != nil {
t.Fatalf("runAgent = %v", err)
}
case <-time.After(10 * time.Second):
t.Fatal("runAgent did not return")
}
if gotSection.ZoneDir != "/var/lib/bind/x" {
t.Errorf("role section = %+v", gotSection)
}
out.Reset()
errOut.Reset()
if code := Main("ddi-agent-dns", KindDNS, nil, []string{"status", "--config", cfgPath}, &out, &errOut); code != 0 {
t.Fatalf("status = %d: %s", code, errOut.String())
}
for _, want := range []string{srv.agentID, "Applied revision:", "revision 1 succeeded", "Stored revisions: 1", stateDir} {
if !strings.Contains(out.String(), want) {
t.Errorf("status lacks %q:\n%s", want, out.String())
}
}
}
func TestRunAgentErrors(t *testing.T) {
cfgPath, _ := writeConfig(t, "")
cfg, err := LoadConfig(cfgPath, KindDNS)
if err != nil {
t.Fatal(err)
}
if err := runAgent(context.Background(), KindDNS, nil, cfg, testLogger()); !errors.Is(err, errRoleNotAvailable) {
t.Errorf("nil factory: %v", err)
}
// Enrolled as a DHCP agent, run as DNS agent.
store, err := OpenStore(cfg.StateDir)
if err != nil {
t.Fatal(err)
}
pki := newTestPKI(t)
id, key := testIdentity(t, pki)
id.Role = KindDHCP
if err := store.SaveIdentity(id, key, false); err != nil {
t.Fatal(err)
}
factory := func(func(any) error) (Role, error) { return newFakeRole(KindDNS), nil }
if err := runAgent(context.Background(), KindDNS, factory, cfg, testLogger()); err == nil || !strings.Contains(err.Error(), "dhcp agent") {
t.Errorf("role mismatch: %v", err)
}
if err := runAgent(context.Background(), KindDHCP, func(func(any) error) (Role, error) {
return nil, errors.New("kea missing")
}, cfg, testLogger()); err == nil || !strings.Contains(err.Error(), "kea missing") {
t.Errorf("factory error: %v", err)
}
}
func TestMainConfigErrors(t *testing.T) {
cfgPath, _ := writeConfig(t, "bogus: 1\n")
for _, args := range [][]string{
{"run", "--config", cfgPath},
{"status", "--config", cfgPath},
{"enroll", "--config", cfgPath, "--server", "x:1", "--token", "a.b", "--ca-hash", "sha256:00"},
} {
var out, errOut bytes.Buffer
if code := Main("ddi-agent-dns", KindDNS, nil, args, &out, &errOut); code != 1 || !strings.Contains(errOut.String(), "bogus") {
t.Errorf("%v: code %d, stderr %q", args[0], code, errOut.String())
}
}
}
agent/core/config.go
+178
View File
@@ -0,0 +1,178 @@
// SPDX-License-Identifier: Apache-2.0
package core
import (
"bytes"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"time"
"go.yaml.in/yaml/v3"
"git.beugsiud.de/beug.k/DDI/internal/brand"
"git.beugsiud.de/beug.k/DDI/internal/logging"
)
// Configuration defaults (docs/adr/0021-agent-strecke-m3.md).
const (
DefaultApplyTimeout = 5 * time.Minute
DefaultHealthTimeout = 30 * time.Second
DefaultLeaseInterval = 30 * time.Second
DefaultLeaseFullSync = time.Hour
)
// Config is the agent configuration file (<ConfigDir>/agent-<role>.yaml).
// Server address, server name and the identity are not configured here; the
// enroll command stores them in the state directory.
type Config struct {
// StateDir holds identity, revisions and state.json (ADR 0011).
StateDir string `yaml:"state_dir"`
// Log configures the structured logger (written to stderr).
Log logging.Options `yaml:"log"`
// ApplyTimeout bounds each role call (Validate, Activate) of an apply.
ApplyTimeout time.Duration `yaml:"apply_timeout"`
// HealthTimeout is how long the health check of a freshly activated
// revision may take until it reports OK.
HealthTimeout time.Duration `yaml:"health_timeout"`
// LeaseInterval is the lease polling interval (DHCP).
LeaseInterval time.Duration `yaml:"lease_interval"`
// LeaseFullSync is the interval of full lease snapshots.
LeaseFullSync time.Duration `yaml:"lease_full_sync"`
// role holds the undecoded section of the agent's role.
role yaml.Node
}
// fileConfig mirrors Config for strict decoding, including the role
// sections that are handed to the role factory undecoded.
type fileConfig struct {
StateDir string `yaml:"state_dir"`
Log logging.Options `yaml:"log"`
ApplyTimeout time.Duration `yaml:"apply_timeout"`
HealthTimeout time.Duration `yaml:"health_timeout"`
LeaseInterval time.Duration `yaml:"lease_interval"`
LeaseFullSync time.Duration `yaml:"lease_full_sync"`
DNS yaml.Node `yaml:"dns"`
DHCP yaml.Node `yaml:"dhcp"`
}
// DefaultConfigPath returns <brand.ConfigDir>/agent-<role>.yaml.
func DefaultConfigPath(kind Kind) string {
return filepath.Join(brand.ConfigDir, "agent-"+string(kind)+".yaml")
}
// DefaultStateDir returns <brand.StateDir>/agent-<role>.
func DefaultStateDir(kind Kind) string {
return filepath.Join(brand.StateDir, "agent-"+string(kind))
}
// LoadConfig reads the configuration file of the role. A missing file yields
// the defaults.
func LoadConfig(path string, kind Kind) (*Config, error) {
data, err := os.ReadFile(path) //nolint:gosec // path is given by the operator
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return nil, fmt.Errorf("read configuration: %w", err)
}
cfg, err := ParseConfig(data, kind)
if err != nil {
return nil, fmt.Errorf("configuration %s: %w", path, err)
}
return cfg, nil
}
// ParseConfig decodes a configuration (strictly: unknown keys are errors),
// applies defaults and validates it.
func ParseConfig(data []byte, kind Kind) (*Config, error) {
var fc fileConfig
if len(bytes.TrimSpace(data)) > 0 {
dec := yaml.NewDecoder(bytes.NewReader(data))
dec.KnownFields(true)
if err := dec.Decode(&fc); err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
}
cfg := &Config{
StateDir: fc.StateDir, Log: fc.Log, ApplyTimeout: fc.ApplyTimeout,
HealthTimeout: fc.HealthTimeout, LeaseInterval: fc.LeaseInterval, LeaseFullSync: fc.LeaseFullSync,
}
switch kind {
case KindDNS:
cfg.role = fc.DNS
if !isEmptyNode(&fc.DHCP) {
return nil, errors.New("section dhcp is not used by the DNS agent")
}
case KindDHCP:
cfg.role = fc.DHCP
if !isEmptyNode(&fc.DNS) {
return nil, errors.New("section dns is not used by the DHCP agent")
}
default:
return nil, fmt.Errorf("unknown role %q", kind)
}
if cfg.StateDir == "" {
cfg.StateDir = DefaultStateDir(kind)
}
setDefault(&cfg.ApplyTimeout, DefaultApplyTimeout)
setDefault(&cfg.HealthTimeout, DefaultHealthTimeout)
setDefault(&cfg.LeaseInterval, DefaultLeaseInterval)
setDefault(&cfg.LeaseFullSync, DefaultLeaseFullSync)
if err := cfg.validate(); err != nil {
return nil, err
}
return cfg, nil
}
func setDefault(d *time.Duration, def time.Duration) {
if *d == 0 {
*d = def
}
}
func (c *Config) validate() error {
for _, d := range []struct {
name string
v time.Duration
}{
{"apply_timeout", c.ApplyTimeout}, {"health_timeout", c.HealthTimeout},
{"lease_interval", c.LeaseInterval}, {"lease_full_sync", c.LeaseFullSync},
} {
if d.v < time.Second {
return fmt.Errorf("%s must be at least 1s", d.name)
}
}
if c.LeaseFullSync < c.LeaseInterval {
return errors.New("lease_full_sync must not be shorter than lease_interval")
}
if _, err := logging.New(io.Discard, c.Log); err != nil {
return err
}
return nil
}
// isEmptyNode reports whether a section is absent or null.
func isEmptyNode(n *yaml.Node) bool {
return n.Kind == 0 || (n.Kind == yaml.ScalarNode && n.Tag == "!!null")
}
// DecodeRole decodes the role's section into v (strictly). An absent section
// leaves v unchanged so that roles keep their defaults.
func (c *Config) DecodeRole(v any) error {
if isEmptyNode(&c.role) {
return nil
}
data, err := yaml.Marshal(&c.role)
if err != nil {
return fmt.Errorf("encode role section: %w", err)
}
dec := yaml.NewDecoder(bytes.NewReader(data))
dec.KnownFields(true)
if err := dec.Decode(v); err != nil && !errors.Is(err, io.EOF) {
return fmt.Errorf("role section: %w", err)
}
return nil
}
agent/core/config_test.go
+128
View File
@@ -0,0 +1,128 @@
// SPDX-License-Identifier: Apache-2.0
package core
import (
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestParseConfigDefaults(t *testing.T) {
for _, data := range []string{"", " \n", "# only a comment\n"} {
cfg, err := ParseConfig([]byte(data), KindDNS)
if err != nil {
t.Fatalf("%q: %v", data, err)
}
if cfg.StateDir != DefaultStateDir(KindDNS) || cfg.ApplyTimeout != DefaultApplyTimeout ||
cfg.HealthTimeout != 30*time.Second || cfg.LeaseInterval != 30*time.Second || cfg.LeaseFullSync != time.Hour {
t.Errorf("defaults = %+v", cfg)
}
var role struct{ X int }
role.X = 7
if err := cfg.DecodeRole(&role); err != nil || role.X != 7 {
t.Errorf("absent role section: %v, %+v", err, role)
}
}
if !strings.HasSuffix(filepath.ToSlash(DefaultConfigPath(KindDHCP)), "/agent-dhcp.yaml") {
t.Errorf("DefaultConfigPath = %s", DefaultConfigPath(KindDHCP))
}
}
func TestParseConfigFull(t *testing.T) {
data := `
state_dir: /tmp/agent
log:
level: debug
format: text
apply_timeout: 2m
health_timeout: 45s
lease_interval: 10s
lease_full_sync: 30m
dhcp:
config_file: /etc/kea/kea-dhcp4.conf
control_socket: /run/kea/ctrl.sock
`
cfg, err := ParseConfig([]byte(data), KindDHCP)
if err != nil {
t.Fatal(err)
}
if cfg.StateDir != "/tmp/agent" || cfg.Log.Level != "debug" || cfg.Log.Format != "text" ||
cfg.ApplyTimeout != 2*time.Minute || cfg.HealthTimeout != 45*time.Second ||
cfg.LeaseInterval != 10*time.Second || cfg.LeaseFullSync != 30*time.Minute {
t.Errorf("config = %+v", cfg)
}
var role struct {
ConfigFile string `yaml:"config_file"`
ControlSocket string `yaml:"control_socket"`
}
if err := cfg.DecodeRole(&role); err != nil {
t.Fatal(err)
}
if role.ConfigFile != "/etc/kea/kea-dhcp4.conf" || role.ControlSocket != "/run/kea/ctrl.sock" {
t.Errorf("role = %+v", role)
}
// Strict decoding of the role section.
var narrow struct {
ConfigFile string `yaml:"config_file"`
}
if err := cfg.DecodeRole(&narrow); err == nil {
t.Error("unknown role key accepted")
}
}
func TestParseConfigErrors(t *testing.T) {
cases := map[string]string{
"unknown key": "flavor: blue\n",
"other role": "dhcp:\n config_file: x\n",
"short timeout": "health_timeout: 10ms\n",
"full sync": "lease_interval: 2m\nlease_full_sync: 1m\n",
"bad level": "log:\n level: loud\n",
"bad duration": "apply_timeout: soon\n",
"not a mapping": "- a\n- b\n",
"unknown log key": "log:\n flavor: x\n",
"negative timeout": "apply_timeout: -5s\n",
}
for name, data := range cases {
if _, err := ParseConfig([]byte(data), KindDNS); err == nil {
t.Errorf("%s: accepted", name)
}
}
if _, err := ParseConfig(nil, Kind("ntp")); err == nil {
t.Error("unknown role accepted")
}
if _, err := ParseConfig([]byte("dns:\n zone_dir: x\n"), KindDHCP); err == nil {
t.Error("dns section accepted by the DHCP agent")
}
// A null section is fine.
if _, err := ParseConfig([]byte("dhcp:\n"), KindDNS); err != nil {
t.Errorf("null section: %v", err)
}
}
func TestLoadConfig(t *testing.T) {
dir := t.TempDir()
cfg, err := LoadConfig(filepath.Join(dir, "missing.yaml"), KindDNS)
if err != nil || cfg.StateDir != DefaultStateDir(KindDNS) {
t.Fatalf("missing file: %+v, %v", cfg, err)
}
path := filepath.Join(dir, "agent-dns.yaml")
if err := os.WriteFile(path, []byte("state_dir: "+filepath.ToSlash(dir)+"\n"), 0o600); err != nil {
t.Fatal(err)
}
cfg, err = LoadConfig(path, KindDNS)
if err != nil || cfg.StateDir != filepath.ToSlash(dir) {
t.Fatalf("file: %+v, %v", cfg, err)
}
if err := os.WriteFile(path, []byte("bogus: 1\n"), 0o600); err != nil {
t.Fatal(err)
}
if _, err := LoadConfig(path, KindDNS); err == nil || !strings.Contains(err.Error(), path) {
t.Errorf("error does not name the file: %v", err)
}
if _, err := LoadConfig(dir, KindDNS); err == nil {
t.Error("directory accepted as configuration file")
}
}
agent/core/enroll.go
+177
View File
@@ -0,0 +1,177 @@
// SPDX-License-Identifier: Apache-2.0
package core
import (
"context"
"crypto/ed25519"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"net"
"strings"
"sync"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"git.beugsiud.de/beug.k/DDI/internal/agentpki"
"git.beugsiud.de/beug.k/DDI/internal/buildinfo"
agentv1 "git.beugsiud.de/beug.k/DDI/proto/gen/ddi/agent/v1"
)
// EnrollOptions are the parameters of the enroll command.
type EnrollOptions struct {
Kind Kind
// Server is the agent API address (host:port; an https:// prefix is
// tolerated).
Server string
// Token is the one-time enrollment token "<id>.<secret>".
Token string
// CAHash is the fingerprint "sha256:<hex>" of the agent CA.
CAHash string
// ServerName is the TLS server name (default: host part of Server).
ServerName string
// Force replaces an existing identity.
Force bool
Hostname string
}
// normalizeServer validates host:port and returns the address and its host.
func normalizeServer(s string) (addr, host string, err error) {
s = strings.TrimSpace(s)
s = strings.TrimPrefix(s, "https://")
s = strings.TrimSuffix(s, "/")
host, port, err := net.SplitHostPort(s)
if err != nil || host == "" || port == "" {
return "", "", fmt.Errorf("server address %q must have the form host:port", s)
}
return s, host, nil
}
// Enroll exchanges the token for a client certificate and stores the
// identity. The server is authenticated by the CA fingerprint: the TLS chain
// it presents must contain a CA with that fingerprint which issued the
// server certificate for ServerName.
func Enroll(ctx context.Context, store *Store, opts EnrollOptions) (*Identity, error) {
addr, host, err := normalizeServer(opts.Server)
if err != nil {
return nil, err
}
if opts.Token == "" || !strings.Contains(opts.Token, ".") {
return nil, errors.New("enrollment token must have the form <id>.<secret>")
}
fingerprint, err := agentpki.ParseFingerprint(opts.CAHash)
if err != nil {
return nil, err
}
serverName := opts.ServerName
if serverName == "" {
serverName = host
}
if store.Enrolled() && !opts.Force {
return nil, errAlreadyEnrolled
}
role, err := roleToProto(opts.Kind)
if err != nil {
return nil, err
}
var (
caMu sync.Mutex
pinnedCA *x509.Certificate
)
tlsCfg := &tls.Config{
MinVersion: tls.VersionTLS13,
ServerName: serverName,
// The server is verified against the CA fingerprint below instead
// of a system trust store.
InsecureSkipVerify: true, //nolint:gosec // verified in VerifyPeerCertificate
// No session resumption, so every handshake runs the check below.
SessionTicketsDisabled: true,
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
ca, err := agentpki.VerifyChainWithFingerprint(rawCerts, fingerprint, serverName)
if err != nil {
return err
}
caMu.Lock()
pinnedCA = ca
caMu.Unlock()
return nil
},
}
conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg)))
if err != nil {
return nil, fmt.Errorf("connect to %s: %w", addr, err)
}
defer func() { _ = conn.Close() }()
key, csr, err := newKeyAndCSR(opts.Hostname)
if err != nil {
return nil, err
}
rctx, cancel := context.WithTimeout(ctx, rpcTimeout)
defer cancel()
resp, err := agentv1.NewAgentServiceClient(conn).Enroll(rctx, &agentv1.EnrollRequest{
Token: opts.Token, CsrDer: csr, Role: role, Hostname: opts.Hostname,
AgentVersion: buildinfo.Get().Version,
})
if err != nil {
return nil, fmt.Errorf("enroll: %w", err)
}
caMu.Lock()
ca := pinnedCA
caMu.Unlock()
if ca == nil {
return nil, errors.New("enroll: server CA was not verified")
}
if chain := resp.GetCaCertificatesDer(); len(chain) > 0 {
root, err := x509.ParseCertificate(chain[len(chain)-1])
if err != nil || !root.Equal(ca) {
return nil, errors.New("enroll: CA in the response differs from the verified server CA")
}
}
if !agentpki.ValidAgentID(resp.GetAgentId()) {
return nil, fmt.Errorf("enroll: server assigned an invalid agent ID %q", resp.GetAgentId())
}
cert, err := checkIssued(resp.GetCertificateDer(), ca, resp.GetAgentId(), key)
if err != nil {
return nil, fmt.Errorf("enroll: %w", err)
}
if len(resp.GetBundleSigningPublicKey()) != ed25519.PublicKeySize {
return nil, errors.New("enroll: invalid bundle signing key")
}
id := &Identity{
AgentID: resp.GetAgentId(), Role: opts.Kind, Server: addr, ServerName: serverName,
EnrolledAt: time.Now().UTC(), CA: ca, Certificate: keyPair(key, cert),
BundleKey: ed25519.PublicKey(resp.GetBundleSigningPublicKey()),
}
if err := store.SaveIdentity(id, key, opts.Force); err != nil {
return nil, err
}
if opts.Force {
// Reports of the previous identity must not be sent with the new one.
st, err := store.LoadState()
if err == nil && len(st.PendingReports) > 0 {
st.PendingReports = nil
err = store.SaveState(st)
}
if err != nil {
return nil, err
}
}
return id, nil
}
func roleToProto(k Kind) (agentv1.Role, error) {
switch k {
case KindDNS:
return agentv1.Role_ROLE_DNS, nil
case KindDHCP:
return agentv1.Role_ROLE_DHCP, nil
default:
return agentv1.Role_ROLE_UNSPECIFIED, fmt.Errorf("unknown role %q", k)
}
}
agent/core/helpers_test.go
+697
View File
@@ -0,0 +1,697 @@
// SPDX-License-Identifier: Apache-2.0
package core
import (
"context"
"crypto"
"crypto/ecdsa"
"crypto/ed25519"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"errors"
"fmt"
"io"
"log/slog"
"math/big"
"net"
"net/url"
"os"
"path/filepath"
"slices"
"sync"
"testing"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/keepalive"
"google.golang.org/grpc/peer"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/durationpb"
"git.beugsiud.de/beug.k/DDI/internal/agentpki"
"git.beugsiud.de/beug.k/DDI/internal/bundle"
agentv1 "git.beugsiud.de/beug.k/DDI/proto/gen/ddi/agent/v1"
)
const testAgentID = "0b6f3c1e-5d2a-4c8e-9f10-123456789abc"
func testLogger() *slog.Logger {
if testing.Verbose() {
return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}))
}
return slog.New(slog.NewTextHandler(io.Discard, nil))
}
// waitFor polls cond until it holds or the timeout expires.
func waitFor(t *testing.T, what string, cond func() bool) {
t.Helper()
deadline := time.Now().Add(15 * time.Second)
for !cond() {
if time.Now().After(deadline) {
t.Fatalf("timeout waiting for %s", what)
}
time.Sleep(10 * time.Millisecond)
}
}
// ---- PKI ----
type testPKI struct {
ca *x509.Certificate
caKey *ecdsa.PrivateKey
serverCert tls.Certificate
}
func newTestPKI(t *testing.T) *testPKI {
t.Helper()
caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
tmpl := &x509.Certificate{
SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test agent CA"},
NotBefore: time.Now().Add(-24 * time.Hour), NotAfter: time.Now().Add(24 * time.Hour),
IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &caKey.PublicKey, caKey)
if err != nil {
t.Fatal(err)
}
ca, err := x509.ParseCertificate(der)
if err != nil {
t.Fatal(err)
}
p := &testPKI{ca: ca, caKey: caKey}
srvKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
srvDER, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{
SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "localhost"},
DNSNames: []string{"localhost"}, IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)},
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
}, ca, &srvKey.PublicKey, caKey)
if err != nil {
t.Fatal(err)
}
p.serverCert = tls.Certificate{Certificate: [][]byte{srvDER, ca.Raw}, PrivateKey: srvKey}
return p
}
func (p *testPKI) issueClient(pub crypto.PublicKey, agentID string, notBefore, notAfter time.Time) (*x509.Certificate, error) {
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 127))
if err != nil {
return nil, err
}
der, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{
SerialNumber: serial, Subject: pkix.Name{CommonName: agentID},
URIs: []*url.URL{agentpki.AgentURI(agentID)},
NotBefore: notBefore, NotAfter: notAfter,
KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}, p.ca, pub, p.caKey)
if err != nil {
return nil, err
}
return x509.ParseCertificate(der)
}
// ---- bundles ----
func newSigningKey(t *testing.T) (ed25519.PublicKey, ed25519.PrivateKey) {
t.Helper()
pub, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
return pub, priv
}
// testArtifacts returns the artifacts of a test revision.
func testArtifacts(rev uint64) []bundle.Artifact {
return []bundle.Artifact{
{Path: "ddi-suite.conf", Mode: 0o640, Content: fmt.Appendf(nil, "// revision %d\n", rev)},
{Path: "zones/db.example.com", Mode: 0o644, Content: fmt.Appendf(nil, "example.com. SOA ns 1 %d\n", rev)},
}
}
func signBundle(t testing.TB, key ed25519.PrivateKey, agentID string, role Kind, rev uint64) *Bundle {
t.Helper()
arts := testArtifacts(rev)
manifest, sig, err := bundle.Sign(key, agentID, string(role), rev, time.Unix(1_700_000_000, 0), arts)
if err != nil {
t.Fatal(err)
}
b := &Bundle{Manifest: manifest, Signature: sig}
for _, a := range arts {
b.Artifacts = append(b.Artifacts, BundleArtifact{Path: a.Path, Content: a.Content})
}
return b
}
// ---- fake role ----
type fakeRole struct {
kind Kind
mu sync.Mutex
validateErr map[uint64]error
activateErr map[uint64]error
unhealthy map[uint64]bool
active uint64
calls []string
healthCalls map[uint64]int
leases []Lease
leasesErr error
versionErr error
}
func newFakeRole(kind Kind) *fakeRole {
return &fakeRole{
kind: kind, validateErr: map[uint64]error{}, activateErr: map[uint64]error{},
unhealthy: map[uint64]bool{}, healthCalls: map[uint64]int{},
}
}
func (r *fakeRole) Kind() Kind { return r.kind }
func (r *fakeRole) ServiceVersion(context.Context) (string, error) {
r.mu.Lock()
defer r.mu.Unlock()
if r.versionErr != nil {
return "", r.versionErr
}
return "Fake 1.0", nil
}
// checkFiles verifies that the stored revision holds its artifacts.
func checkFiles(rev Revision) error {
for _, f := range rev.Files {
data, err := os.ReadFile(filepath.Join(rev.Dir, filepath.FromSlash(f.Path)))
if err != nil {
return err
}
if int64(len(data)) != f.Size {
return fmt.Errorf("%s: size %d, want %d", f.Path, len(data), f.Size)
}
}
return nil
}
func (r *fakeRole) Validate(_ context.Context, rev Revision) error {
r.mu.Lock()
defer r.mu.Unlock()
r.calls = append(r.calls, fmt.Sprintf("validate %d", rev.Number))
if err := checkFiles(rev); err != nil {
return err
}
return r.validateErr[rev.Number]
}
func (r *fakeRole) Activate(_ context.Context, rev Revision, prev *Revision) error {
r.mu.Lock()
defer r.mu.Unlock()
p := "none"
if prev != nil {
p = fmt.Sprint(prev.Number)
}
r.calls = append(r.calls, fmt.Sprintf("activate %d prev %s", rev.Number, p))
if err := r.activateErr[rev.Number]; err != nil {
return err
}
r.active = rev.Number
return nil
}
func (r *fakeRole) Health(_ context.Context, expected *Revision) Health {
r.mu.Lock()
defer r.mu.Unlock()
var n uint64
if expected != nil {
n = expected.Number
}
r.healthCalls[n]++
if r.unhealthy[r.active] {
return Health{State: HealthFailed, Message: "service down",
Checks: []HealthCheck{{Name: "soa", State: HealthFailed, Message: "serial mismatch"}, {Name: "status", State: HealthOK}}}
}
return Health{State: HealthOK}
}
func (r *fakeRole) Leases(context.Context) ([]Lease, error) {
r.mu.Lock()
defer r.mu.Unlock()
return slices.Clone(r.leases), r.leasesErr
}
func (r *fakeRole) setLeases(l []Lease) {
r.mu.Lock()
defer r.mu.Unlock()
r.leases = l
}
func (r *fakeRole) callLog() []string {
r.mu.Lock()
defer r.mu.Unlock()
return slices.Clone(r.calls)
}
func (r *fakeRole) activeRevision() uint64 {
r.mu.Lock()
defer r.mu.Unlock()
return r.active
}
func (r *fakeRole) resetCalls() {
r.mu.Lock()
defer r.mu.Unlock()
r.calls = nil
}
// fakeLeaseRole is a role with a LeaseSource; fakeRole itself has the
// method too, so wrap it to hide Leases for the DNS case.
type dnsOnlyRole struct{ *fakeRole }
func (d dnsOnlyRole) Leases() {} // shadows LeaseSource.Leases with another signature
// ---- fake source ----
type fakeSource struct {
mu sync.Mutex
bundles map[uint64]*Bundle
fetchErr error
reportErr error
leaseErr error
reports []DeploymentReport
batches []LeaseBatch
fetches int
onFetch func()
run func(ctx context.Context, h SourceHandler) error
}
func newFakeSource() *fakeSource { return &fakeSource{bundles: map[uint64]*Bundle{}} }
func (s *fakeSource) Run(ctx context.Context, h SourceHandler) error {
if s.run != nil {
return s.run(ctx, h)
}
<-ctx.Done()
return ctx.Err()
}
func (s *fakeSource) FetchBundle(_ context.Context, rev uint64) (*Bundle, error) {
s.mu.Lock()
s.fetches++
b, err, hook := s.bundles[rev], s.fetchErr, s.onFetch
s.mu.Unlock()
if hook != nil {
hook()
}
if err != nil {
return nil, err
}
if b == nil {
return nil, fmt.Errorf("revision %d not found", rev)
}
return b, nil
}
func (s *fakeSource) ReportDeployment(_ context.Context, r DeploymentReport) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.reportErr != nil {
return s.reportErr
}
s.reports = append(s.reports, r)
return nil
}
func (s *fakeSource) ReportLeases(_ context.Context, b LeaseBatch) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.leaseErr != nil {
return s.leaseErr
}
b.Leases = slices.Clone(b.Leases)
s.batches = append(s.batches, b)
return nil
}
func (s *fakeSource) setReportErr(err error) {
s.mu.Lock()
defer s.mu.Unlock()
s.reportErr = err
}
func (s *fakeSource) reported() []DeploymentReport {
s.mu.Lock()
defer s.mu.Unlock()
return slices.Clone(s.reports)
}
func (s *fakeSource) leaseBatches() []LeaseBatch {
s.mu.Lock()
defer s.mu.Unlock()
return slices.Clone(s.batches)
}
// ---- agent fixture ----
type fixture struct {
agent *Agent
role *fakeRole
src *fakeSource
store *Store
signer ed25519.PrivateKey
}
func testConfig(dir string) *Config {
return &Config{
StateDir: dir, ApplyTimeout: 5 * time.Second, HealthTimeout: 300 * time.Millisecond,
LeaseInterval: time.Second, LeaseFullSync: time.Hour,
}
}
func newFixture(t *testing.T, kind Kind) *fixture {
t.Helper()
store, err := OpenStore(t.TempDir())
if err != nil {
t.Fatal(err)
}
return newFixtureWithStore(t, kind, store, nil)
}
func newFixtureWithStore(t *testing.T, kind Kind, store *Store, signer ed25519.PrivateKey) *fixture {
t.Helper()
if signer == nil {
_, signer = newSigningKey(t)
}
role := newFakeRole(kind)
src := newFakeSource()
id := &Identity{AgentID: testAgentID, Role: kind, BundleKey: signer.Public().(ed25519.PublicKey)}
a, err := NewAgent(kind, testConfig(store.Dir()), role, store, id, src, testLogger())
if err != nil {
t.Fatal(err)
}
a.healthRetry = 20 * time.Millisecond
a.reportRetry = 50 * time.Millisecond
return &fixture{agent: a, role: role, src: src, store: store, signer: signer}
}
// addBundle makes revision rev available from the fake source.
func (f *fixture) addBundle(t *testing.T, rev uint64) {
t.Helper()
f.src.mu.Lock()
defer f.src.mu.Unlock()
f.src.bundles[rev] = signBundle(t, f.signer, testAgentID, f.agent.kind, rev)
}
func (f *fixture) pull(rev uint64) {
f.agent.handle(context.Background(), applyRequest{revision: rev, trigger: TriggerPull})
}
func (f *fixture) push(rev uint64, dep string) {
f.agent.handle(context.Background(), applyRequest{revision: rev, trigger: TriggerPush, deploymentID: dep})
}
func (f *fixture) lastResult(t *testing.T) DeploymentReport {
t.Helper()
st := f.agent.state()
if st.LastResult == nil {
t.Fatal("no result recorded")
}
return *st.LastResult
}
// ---- fake gRPC server ----
type fakeServer struct {
agentv1.UnimplementedAgentServiceServer
pki *testPKI
signKey ed25519.PrivateKey
role Kind
mu sync.Mutex
token string
agentID string
serial *big.Int
enrollValidity [2]time.Duration // NotBefore offset, NotAfter offset for Enroll
desired uint64
mode agentv1.OperatingMode
pullInterval time.Duration
heartbeat time.Duration
streamDisabled bool
hellos []*agentv1.Hello
heartbeats int
acks []*agentv1.CommandAck
reports []*agentv1.ReportDeploymentRequest
leaseReports []*agentv1.ReportLeasesRequest
renewals int
polls int
commands chan *agentv1.Command
enrollments int
}
func newFakeServer(t *testing.T, role Kind) *fakeServer {
t.Helper()
_, key := newSigningKey(t)
return &fakeServer{
pki: newTestPKI(t), signKey: key, role: role, token: "0123456789abcdef.secret",
agentID: "11111111-2222-4333-8444-555555555555",
enrollValidity: [2]time.Duration{-time.Minute, 14 * 24 * time.Hour},
mode: agentv1.OperatingMode_OPERATING_MODE_AUTO_PULL,
pullInterval: 50 * time.Millisecond, heartbeat: 50 * time.Millisecond,
commands: make(chan *agentv1.Command, 10),
}
}
// start serves the fake on 127.0.0.1 with mTLS and returns its address.
func (f *fakeServer) start(t *testing.T) string {
t.Helper()
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
pool := x509.NewCertPool()
pool.AddCert(f.pki.ca)
srv := grpc.NewServer(
grpc.Creds(credentials.NewTLS(&tls.Config{
MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{f.pki.serverCert},
ClientAuth: tls.VerifyClientCertIfGiven, ClientCAs: pool,
})),
grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{MinTime: 10 * time.Second, PermitWithoutStream: true}),
)
agentv1.RegisterAgentServiceServer(srv, f)
go func() { _ = srv.Serve(lis) }()
t.Cleanup(srv.Stop)
return lis.Addr().String()
}
func (f *fakeServer) fingerprint() string { return agentpki.Fingerprint(f.pki.ca) }
func (f *fakeServer) authenticate(ctx context.Context) error {
p, ok := peer.FromContext(ctx)
if !ok {
return status.Error(codes.Unauthenticated, "no peer")
}
ti, ok := p.AuthInfo.(credentials.TLSInfo)
if !ok || len(ti.State.PeerCertificates) == 0 {
return status.Error(codes.Unauthenticated, "client certificate required")
}
cert := ti.State.PeerCertificates[0]
id, err := agentpki.AgentID(cert)
if err != nil {
return status.Error(codes.Unauthenticated, err.Error())
}
f.mu.Lock()
defer f.mu.Unlock()
if id != f.agentID || f.serial == nil || cert.SerialNumber.Cmp(f.serial) != 0 {
return status.Error(codes.Unauthenticated, "unknown certificate")
}
return nil
}
func (f *fakeServer) issue(csrDER []byte, notBefore, notAfter time.Time) (*x509.Certificate, error) {
csr, err := x509.ParseCertificateRequest(csrDER)
if err != nil {
return nil, err
}
if err := csr.CheckSignature(); err != nil {
return nil, err
}
f.mu.Lock()
id := f.agentID
f.mu.Unlock()
cert, err := f.pki.issueClient(csr.PublicKey, id, notBefore, notAfter)
if err != nil {
return nil, err
}
f.mu.Lock()
f.serial = cert.SerialNumber
f.mu.Unlock()
return cert, nil
}
func (f *fakeServer) Enroll(_ context.Context, req *agentv1.EnrollRequest) (*agentv1.EnrollResponse, error) {
f.mu.Lock()
token, validity := f.token, f.enrollValidity
f.enrollments++
f.mu.Unlock()
if req.GetToken() != token {
return nil, status.Error(codes.PermissionDenied, "invalid enrollment token")
}
want, _ := roleToProto(f.role)
if req.GetRole() != want {
return nil, status.Error(codes.InvalidArgument, "wrong role")
}
now := time.Now()
cert, err := f.issue(req.GetCsrDer(), now.Add(validity[0]), now.Add(validity[1]))
if err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
}
f.mu.Lock()
defer f.mu.Unlock()
return &agentv1.EnrollResponse{
AgentId: f.agentID, CertificateDer: cert.Raw, CaCertificatesDer: [][]byte{f.pki.ca.Raw},
BundleSigningPublicKey: f.signKey.Public().(ed25519.PublicKey),
}, nil
}
func (f *fakeServer) RenewCertificate(ctx context.Context, req *agentv1.RenewCertificateRequest) (*agentv1.RenewCertificateResponse, error) {
if err := f.authenticate(ctx); err != nil {
return nil, err
}
now := time.Now()
cert, err := f.issue(req.GetCsrDer(), now.Add(-time.Minute), now.Add(14*24*time.Hour))
if err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
}
f.mu.Lock()
f.renewals++
f.mu.Unlock()
return &agentv1.RenewCertificateResponse{CertificateDer: cert.Raw, CaCertificatesDer: [][]byte{f.pki.ca.Raw}}, nil
}
func (f *fakeServer) Connect(stream agentv1.AgentService_ConnectServer) error {
f.mu.Lock()
disabled := f.streamDisabled
f.mu.Unlock()
if disabled {
return status.Error(codes.Unimplemented, "streams disabled")
}
if err := f.authenticate(stream.Context()); err != nil {
return err
}
first, err := stream.Recv()
if err != nil {
return err
}
if first.GetHello() == nil {
return status.Error(codes.InvalidArgument, "Hello expected")
}
f.mu.Lock()
f.hellos = append(f.hellos, first.GetHello())
w := &agentv1.Welcome{
Mode: f.mode, PullInterval: durationpb.New(f.pullInterval),
HeartbeatInterval: durationpb.New(f.heartbeat), DesiredRevision: f.desired,
}
f.mu.Unlock()
if err := stream.Send(&agentv1.ConnectResponse{Message: &agentv1.ConnectResponse_Welcome{Welcome: w}}); err != nil {
return err
}
recvErr := make(chan error, 1)
go func() {
for {
msg, err := stream.Recv()
if err != nil {
recvErr <- err
return
}
f.mu.Lock()
switch {
case msg.GetHeartbeat() != nil:
f.heartbeats++
case msg.GetCommandAck() != nil:
f.acks = append(f.acks, msg.GetCommandAck())
}
f.mu.Unlock()
}
}()
for {
select {
case err := <-recvErr:
if errors.Is(err, io.EOF) {
return nil
}
return err
case <-stream.Context().Done():
return nil
case c := <-f.commands:
if err := stream.Send(&agentv1.ConnectResponse{Message: &agentv1.ConnectResponse_Command{Command: c}}); err != nil {
return err
}
}
}
}
func (f *fakeServer) GetDesiredState(ctx context.Context, _ *agentv1.GetDesiredStateRequest) (*agentv1.GetDesiredStateResponse, error) {
if err := f.authenticate(ctx); err != nil {
return nil, err
}
f.mu.Lock()
defer f.mu.Unlock()
f.polls++
return &agentv1.GetDesiredStateResponse{DesiredRevision: f.desired, Mode: f.mode, PullInterval: durationpb.New(f.pullInterval)}, nil
}
func (f *fakeServer) FetchBundle(ctx context.Context, req *agentv1.FetchBundleRequest) (*agentv1.FetchBundleResponse, error) {
if err := f.authenticate(ctx); err != nil {
return nil, err
}
f.mu.Lock()
id := f.agentID
f.mu.Unlock()
arts := testArtifacts(req.GetRevision())
manifest, sig, err := bundle.Sign(f.signKey, id, string(f.role), req.GetRevision(), time.Unix(1_700_000_000, 0), arts)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
resp := &agentv1.FetchBundleResponse{Manifest: manifest, Signature: sig}
for _, a := range arts {
resp.Artifacts = append(resp.Artifacts, &agentv1.Artifact{Path: a.Path, Content: a.Content})
}
return resp, nil
}
func (f *fakeServer) ReportDeployment(ctx context.Context, req *agentv1.ReportDeploymentRequest) (*agentv1.ReportDeploymentResponse, error) {
if err := f.authenticate(ctx); err != nil {
return nil, err
}
f.mu.Lock()
defer f.mu.Unlock()
f.reports = append(f.reports, req)
return &agentv1.ReportDeploymentResponse{}, nil
}
func (f *fakeServer) ReportLeases(ctx context.Context, req *agentv1.ReportLeasesRequest) (*agentv1.ReportLeasesResponse, error) {
if err := f.authenticate(ctx); err != nil {
return nil, err
}
f.mu.Lock()
defer f.mu.Unlock()
f.leaseReports = append(f.leaseReports, req)
return &agentv1.ReportLeasesResponse{AcknowledgedSequence: req.GetSequence()}, nil
}
// snapshot returns a copy of the recorded data under the lock.
func (f *fakeServer) with(fn func(f *fakeServer)) {
f.mu.Lock()
defer f.mu.Unlock()
fn(f)
}
agent/core/identity.go
+320
View File
@@ -0,0 +1,320 @@
// SPDX-License-Identifier: Apache-2.0
package core
import (
"crypto"
"crypto/ecdsa"
"crypto/ed25519"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"time"
"git.beugsiud.de/beug.k/DDI/internal/agentpki"
)
// Files in identity/.
const (
keyFile = "key.pem"
certFile = "cert.pem"
caFile = "ca.pem"
bundleKeyFile = "bundle-signing.pub"
agentFile = "agent.json"
newSuffix = ".new"
)
// ErrNotEnrolled is returned by LoadIdentity if the agent has no identity.
var ErrNotEnrolled = errors.New("agent is not enrolled (run the enroll command first)")
// errAlreadyEnrolled is returned by SaveIdentity without force.
var errAlreadyEnrolled = errors.New("agent is already enrolled (use --force to enroll again)")
// Identity is the enrolled identity of the agent.
type Identity struct {
AgentID string
Role Kind
Server string // host:port of the agent API
ServerName string // TLS server name
EnrolledAt time.Time
CA *x509.Certificate
// Certificate holds the client certificate (Leaf set) and its key.
Certificate tls.Certificate
BundleKey ed25519.PublicKey
}
// agentInfo is identity/agent.json.
type agentInfo struct {
AgentID string `json:"agent_id"`
Role Kind `json:"role"`
Server string `json:"server"`
ServerName string `json:"server_name"`
EnrolledAt time.Time `json:"enrolled_at"`
CAFingerprint string `json:"ca_fingerprint"`
}
func (s *Store) identityPath(name string) string {
return filepath.Join(s.dir, identityDir, name)
}
// Enrolled reports whether an identity exists.
func (s *Store) Enrolled() bool {
_, err := os.Stat(s.identityPath(agentFile))
return err == nil
}
// LoadIdentity reads identity/. It completes an interrupted certificate
// renewal (see SaveCertificate).
func (s *Store) LoadIdentity() (*Identity, error) {
data, err := os.ReadFile(s.identityPath(agentFile))
if errors.Is(err, fs.ErrNotExist) {
return nil, ErrNotEnrolled
}
if err != nil {
return nil, fmt.Errorf("read identity: %w", err)
}
var info agentInfo
if err := json.Unmarshal(data, &info); err != nil {
return nil, fmt.Errorf("decode %s: %w", agentFile, err)
}
if !agentpki.ValidAgentID(info.AgentID) {
return nil, fmt.Errorf("%s holds an invalid agent ID", agentFile)
}
id := &Identity{
AgentID: info.AgentID, Role: info.Role, Server: info.Server,
ServerName: info.ServerName, EnrolledAt: info.EnrolledAt,
}
if id.CA, err = readCertPEM(s.identityPath(caFile)); err != nil {
return nil, err
}
if id.BundleKey, err = readBundleKey(s.identityPath(bundleKeyFile)); err != nil {
return nil, err
}
if id.Certificate, err = s.loadKeyPair(); err != nil {
return nil, err
}
return id, nil
}
// loadKeyPair loads key.pem and cert.pem. If they do not match and a
// cert.pem.new matching the key exists, a renewal was interrupted after the
// key was replaced; the new certificate is moved into place.
func (s *Store) loadKeyPair() (tls.Certificate, error) {
keyPEM, err := os.ReadFile(s.identityPath(keyFile))
if err != nil {
return tls.Certificate{}, fmt.Errorf("read identity: %w", err)
}
certPEM, err := os.ReadFile(s.identityPath(certFile))
if err != nil {
return tls.Certificate{}, fmt.Errorf("read identity: %w", err)
}
pair, err := tls.X509KeyPair(certPEM, keyPEM)
if err == nil {
s.removeStaleNew()
return pair, nil
}
newCert, nerr := os.ReadFile(s.identityPath(certFile + newSuffix))
if nerr != nil {
return tls.Certificate{}, fmt.Errorf("load client certificate: %w", err)
}
pair, nerr = tls.X509KeyPair(newCert, keyPEM)
if nerr != nil {
return tls.Certificate{}, fmt.Errorf("load client certificate: %w", err)
}
if err := os.Rename(s.identityPath(certFile+newSuffix), s.identityPath(certFile)); err != nil {
return tls.Certificate{}, fmt.Errorf("complete certificate renewal: %w", err)
}
syncDir(filepath.Join(s.dir, identityDir))
return pair, nil
}
func (s *Store) removeStaleNew() {
_ = os.Remove(s.identityPath(keyFile + newSuffix))
_ = os.Remove(s.identityPath(certFile + newSuffix))
}
// SaveIdentity writes a new identity directory. Without force it refuses to
// replace an existing identity.
func (s *Store) SaveIdentity(id *Identity, key crypto.Signer, force bool) error {
if s.Enrolled() && !force {
return errAlreadyEnrolled
}
keyPEM, err := encodeKey(key)
if err != nil {
return err
}
pubDER, err := x509.MarshalPKIXPublicKey(id.BundleKey)
if err != nil {
return fmt.Errorf("encode bundle signing key: %w", err)
}
info, err := json.MarshalIndent(agentInfo{
AgentID: id.AgentID, Role: id.Role, Server: id.Server, ServerName: id.ServerName,
EnrolledAt: id.EnrolledAt.UTC(), CAFingerprint: agentpki.Fingerprint(id.CA),
}, "", " ")
if err != nil {
return fmt.Errorf("encode identity: %w", err)
}
tmp, err := os.MkdirTemp(s.dir, tmpPrefix+identityDir+"-")
if err != nil {
return fmt.Errorf("create identity directory: %w", err)
}
defer func() { _ = os.RemoveAll(tmp) }()
if err := os.Chmod(tmp, 0o700); err != nil { //nolint:gosec // directories need the search bit
return fmt.Errorf("create identity directory: %w", err)
}
files := []struct {
name string
data []byte
perm fs.FileMode
}{
{keyFile, keyPEM, 0o600},
{certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: id.Certificate.Leaf.Raw}), 0o644},
{caFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: id.CA.Raw}), 0o644},
{bundleKeyFile, pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: pubDER}), 0o644},
{agentFile, append(info, '\n'), 0o644},
}
for _, f := range files {
if err := writeFileSync(filepath.Join(tmp, f.name), f.data, f.perm); err != nil {
return err
}
}
syncDir(tmp)
if err := replaceDir(tmp, filepath.Join(s.dir, identityDir)); err != nil {
return err
}
syncDir(s.dir)
return nil
}
// SaveCertificate stores a renewed key and certificate. Both are written as
// *.new first; key.pem is replaced before cert.pem, and LoadIdentity
// completes the second rename if the process died in between.
func (s *Store) SaveCertificate(key crypto.Signer, cert *x509.Certificate) error {
keyPEM, err := encodeKey(key)
if err != nil {
return err
}
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})
keyNew, certNew := s.identityPath(keyFile+newSuffix), s.identityPath(certFile+newSuffix)
if err := writeFileAtomic(keyNew, keyPEM, 0o600); err != nil {
return err
}
if err := writeFileAtomic(certNew, certPEM, 0o644); err != nil {
return err
}
if err := os.Rename(keyNew, s.identityPath(keyFile)); err != nil {
return fmt.Errorf("replace key: %w", err)
}
if err := os.Rename(certNew, s.identityPath(certFile)); err != nil {
return fmt.Errorf("replace certificate: %w", err)
}
syncDir(filepath.Join(s.dir, identityDir))
return nil
}
func readCertPEM(path string) (*x509.Certificate, error) {
data, err := os.ReadFile(path) //nolint:gosec // path built from the state directory
if err != nil {
return nil, fmt.Errorf("read identity: %w", err)
}
block, _ := pem.Decode(data)
if block == nil || block.Type != "CERTIFICATE" {
return nil, fmt.Errorf("%s holds no certificate", filepath.Base(path))
}
cert, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return nil, fmt.Errorf("parse %s: %w", filepath.Base(path), err)
}
return cert, nil
}
func readBundleKey(path string) (ed25519.PublicKey, error) {
data, err := os.ReadFile(path) //nolint:gosec // path built from the state directory
if err != nil {
return nil, fmt.Errorf("read identity: %w", err)
}
block, _ := pem.Decode(data)
if block == nil || block.Type != "PUBLIC KEY" {
return nil, fmt.Errorf("%s holds no public key", filepath.Base(path))
}
pub, err := x509.ParsePKIXPublicKey(block.Bytes)
if err != nil {
return nil, fmt.Errorf("parse %s: %w", filepath.Base(path), err)
}
key, ok := pub.(ed25519.PublicKey)
if !ok {
return nil, fmt.Errorf("%s is not an Ed25519 key", filepath.Base(path))
}
return key, nil
}
func encodeKey(key crypto.Signer) ([]byte, error) {
der, err := x509.MarshalPKCS8PrivateKey(key)
if err != nil {
return nil, fmt.Errorf("encode private key: %w", err)
}
return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), nil
}
// newKeyAndCSR generates an ECDSA P-256 key and a CSR for it. The server
// sets subject and SANs itself; commonName is informational.
func newKeyAndCSR(commonName string) (*ecdsa.PrivateKey, []byte, error) {
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return nil, nil, fmt.Errorf("generate key: %w", err)
}
csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{
Subject: pkix.Name{CommonName: commonName},
}, key)
if err != nil {
return nil, nil, fmt.Errorf("create certificate request: %w", err)
}
return key, csr, nil
}
// checkIssued verifies a certificate issued by the server: it must chain to
// ca for client authentication, carry agentID and certify key.
func checkIssued(der []byte, ca *x509.Certificate, agentID string, key *ecdsa.PrivateKey) (*x509.Certificate, error) {
cert, err := x509.ParseCertificate(der)
if err != nil {
return nil, fmt.Errorf("parse issued certificate: %w", err)
}
roots := x509.NewCertPool()
roots.AddCert(ca)
// Tolerate a server clock that is slightly ahead of ours.
at := time.Now()
if at.Before(cert.NotBefore) {
at = cert.NotBefore
}
if _, err := cert.Verify(x509.VerifyOptions{
Roots: roots, CurrentTime: at, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}); err != nil {
return nil, fmt.Errorf("verify issued certificate: %w", err)
}
id, err := agentpki.AgentID(cert)
if err != nil {
return nil, err
}
if id != agentID {
return nil, fmt.Errorf("issued certificate is for agent %s, not %s", id, agentID)
}
pub, ok := cert.PublicKey.(*ecdsa.PublicKey)
if !ok || !pub.Equal(&key.PublicKey) {
return nil, errors.New("issued certificate does not certify the generated key")
}
return cert, nil
}
// keyPair combines a key and its certificate.
func keyPair(key *ecdsa.PrivateKey, cert *x509.Certificate) tls.Certificate {
return tls.Certificate{Certificate: [][]byte{cert.Raw}, PrivateKey: key, Leaf: cert}
}
agent/core/leases.go
+162
View File
@@ -0,0 +1,162 @@
// SPDX-License-Identifier: Apache-2.0
package core
import (
"context"
"fmt"
"net/netip"
"slices"
"time"
)
// maxLeasesPerBatch bounds one ReportLeases request (about 100 bytes per
// lease, far below the default gRPC message limit of 4 MiB).
const maxLeasesPerBatch = 5000
// leasesTimeout bounds one LeaseSource.Leases call.
const leasesTimeout = 30 * time.Second
// leaseTracker remembers what the source was last told.
type leaseTracker struct {
// known holds the leases as last reported, by address.
known map[netip.Addr]Lease
// needFull forces the next report to be a full snapshot (at start,
// after a new connection and after a failed report).
needFull bool
lastFull time.Time
}
// leaseLoop polls the role for leases and reports changes. A full snapshot
// is sent at start, after each new connection, after a failed report and
// every LeaseFullSync.
func (a *Agent) leaseLoop(ctx context.Context, ls LeaseSource) {
t := time.NewTicker(a.cfg.LeaseInterval)
defer t.Stop()
tr := &leaseTracker{needFull: true}
for {
a.pollLeases(ctx, ls, tr)
select {
case <-ctx.Done():
return
case <-t.C:
case <-a.leaseCh:
tr.needFull = true
}
}
}
// pollLeases runs one polling round.
func (a *Agent) pollLeases(ctx context.Context, ls LeaseSource, tr *leaseTracker) {
if !a.roleMu.TryLock() {
return // an apply is running; try again next round
}
lctx, cancel := context.WithTimeout(ctx, leasesTimeout)
cur, err := ls.Leases(lctx)
cancel()
a.roleMu.Unlock()
if err != nil {
if ctx.Err() == nil {
a.log.Warn("cannot read leases", "error", err)
}
return
}
full := tr.needFull || time.Since(tr.lastFull) >= a.cfg.LeaseFullSync
changes, next := diffLeases(tr.known, cur)
toSend := changes
if full {
toSend = sortedLeases(next)
} else if len(changes) == 0 {
return
}
if err := a.sendLeases(ctx, toSend, full); err != nil {
if ctx.Err() == nil {
a.log.Warn("cannot report leases; next report is a full snapshot", "error", err)
}
tr.needFull = true
return
}
tr.known = next
tr.needFull = false
if full {
tr.lastFull = time.Now()
}
}
// sendLeases sends leases in batches of maxLeasesPerBatch. For a full
// snapshot only the first batch carries FullSync; the server replaces its
// view with it and applies the following batches as changes, so a snapshot
// split into several requests ends in the same state. If any batch fails,
// the caller sends a new full snapshot later.
func (a *Agent) sendLeases(ctx context.Context, leases []Lease, full bool) error {
first := true
for len(leases) > 0 || first {
n := min(len(leases), maxLeasesPerBatch)
seq, err := a.nextLeaseSequence()
if err != nil {
return err
}
b := LeaseBatch{Sequence: seq, FullSync: full && first, Leases: leases[:n]}
if err := a.source.ReportLeases(ctx, b); err != nil {
return err
}
a.log.Debug("leases reported", "sequence", seq, "full_sync", b.FullSync, "count", n)
leases = leases[n:]
first = false
}
return nil
}
// nextLeaseSequence increments and persists the lease sequence before it is
// used, so that a restart never reuses a number.
func (a *Agent) nextLeaseSequence() (uint64, error) {
a.mu.Lock()
defer a.mu.Unlock()
a.st.LeaseSequence++
a.st.UpdatedAt = time.Now().UTC()
if err := a.store.SaveState(a.st); err != nil {
return 0, fmt.Errorf("persist lease sequence: %w", err)
}
return a.st.LeaseSequence, nil
}
// diffLeases compares the current leases with the known ones. It returns the
// changes (new or modified leases, and known leases that disappeared, marked
// expired) and the new known set.
func diffLeases(known map[netip.Addr]Lease, cur []Lease) ([]Lease, map[netip.Addr]Lease) {
next := make(map[netip.Addr]Lease, len(cur))
for _, l := range cur {
if l.IP.IsValid() {
next[l.IP] = l
}
}
var changes []Lease
for ip, l := range next {
if old, ok := known[ip]; !ok || !leaseEqual(old, l) {
changes = append(changes, l)
}
}
for ip, l := range known {
if _, ok := next[ip]; !ok {
l.State = LeaseExpired
changes = append(changes, l)
}
}
slices.SortFunc(changes, func(x, y Lease) int { return x.IP.Compare(y.IP) })
return changes, next
}
func sortedLeases(m map[netip.Addr]Lease) []Lease {
out := make([]Lease, 0, len(m))
for _, l := range m {
out = append(out, l)
}
slices.SortFunc(out, func(x, y Lease) int { return x.IP.Compare(y.IP) })
return out
}
func leaseEqual(x, y Lease) bool {
return x.IP == y.IP && x.MAC == y.MAC && x.Hostname == y.Hostname && x.State == y.State &&
x.Expires.Equal(y.Expires) && x.Updated.Equal(y.Updated) && x.SubnetID == y.SubnetID &&
x.ClientID == y.ClientID
}
agent/core/leases_test.go
+192
View File
@@ -0,0 +1,192 @@
// SPDX-License-Identifier: Apache-2.0
package core
import (
"context"
"errors"
"fmt"
"net/netip"
"testing"
"time"
)
func mustAddr(s string) netip.Addr { return netip.MustParseAddr(s) }
func lease(ip, mac string) Lease {
return Lease{
IP: mustAddr(ip), MAC: mac, Hostname: "host-" + ip, State: LeaseActive,
Expires: time.Unix(1_800_000_000, 0), Updated: time.Unix(1_700_000_000, 0), SubnetID: 1,
}
}
func TestDiffLeases(t *testing.T) {
a, b, c := lease("10.0.0.1", "aa:00:00:00:00:01"), lease("10.0.0.2", "aa:00:00:00:00:02"), lease("10.0.0.3", "aa:00:00:00:00:03")
changes, known := diffLeases(nil, []Lease{b, a, {}})
if len(changes) != 2 || changes[0].IP != a.IP || changes[1].IP != b.IP || len(known) != 2 {
t.Fatalf("initial changes = %+v", changes)
}
if changes, _ := diffLeases(known, []Lease{a, b}); len(changes) != 0 {
t.Errorf("unchanged leases reported: %+v", changes)
}
b2 := b
b2.Expires = b.Expires.Add(time.Hour)
changes, next := diffLeases(known, []Lease{b2, c})
if len(changes) != 3 {
t.Fatalf("changes = %+v", changes)
}
if changes[0].IP != a.IP || changes[0].State != LeaseExpired {
t.Errorf("disappeared lease = %+v", changes[0])
}
if !changes[1].Expires.Equal(b2.Expires) || changes[2].IP != c.IP {
t.Errorf("changes = %+v", changes)
}
if len(next) != 2 {
t.Errorf("next = %v", next)
}
// Equal instants in different locations are not a change.
b3 := b2
b3.Expires = b2.Expires.In(time.FixedZone("x", 3600))
if changes, _ := diffLeases(next, []Lease{b3, c}); len(changes) != 0 {
t.Errorf("time zone change reported: %+v", changes)
}
}
func TestPollLeases(t *testing.T) {
f := newFixture(t, KindDHCP)
ctx := context.Background()
tr := &leaseTracker{needFull: true}
a, b := lease("10.0.0.1", "aa:00:00:00:00:01"), lease("10.0.0.2", "aa:00:00:00:00:02")
f.role.setLeases([]Lease{a, b})
f.agent.pollLeases(ctx, f.role, tr)
batches := f.src.leaseBatches()
if len(batches) != 1 || !batches[0].FullSync || batches[0].Sequence != 1 || len(batches[0].Leases) != 2 {
t.Fatalf("batches = %+v", batches)
}
// No change: nothing is sent.
f.agent.pollLeases(ctx, f.role, tr)
if n := len(f.src.leaseBatches()); n != 1 {
t.Fatalf("%d batches after an unchanged poll", n)
}
// b released, a gone, c new: one delta.
b.State = LeaseReleased
c := lease("10.0.0.3", "aa:00:00:00:00:03")
f.role.setLeases([]Lease{b, c})
f.agent.pollLeases(ctx, f.role, tr)
batches = f.src.leaseBatches()
if len(batches) != 2 {
t.Fatalf("batches = %+v", batches)
}
d := batches[1]
if d.FullSync || d.Sequence != 2 || len(d.Leases) != 3 || d.Leases[0].State != LeaseExpired || d.Leases[1].State != LeaseReleased {
t.Fatalf("delta = %+v", d)
}
// A failed report forces a full snapshot next time.
f.src.mu.Lock()
f.src.leaseErr = errors.New("down")
f.src.mu.Unlock()
c.Hostname = "renamed"
f.role.setLeases([]Lease{b, c})
f.agent.pollLeases(ctx, f.role, tr)
if !tr.needFull {
t.Fatal("failure did not request a full snapshot")
}
f.src.mu.Lock()
f.src.leaseErr = nil
f.src.mu.Unlock()
f.agent.pollLeases(ctx, f.role, tr)
batches = f.src.leaseBatches()
last := batches[len(batches)-1]
if !last.FullSync || len(last.Leases) != 2 || last.Sequence != 4 {
t.Fatalf("snapshot after failure = %+v", last)
}
// The sequence survives a restart.
st, err := f.store.LoadState()
if err != nil {
t.Fatal(err)
}
if st.LeaseSequence != 4 {
t.Errorf("persisted sequence = %d", st.LeaseSequence)
}
// The full sync interval triggers a snapshot.
tr.lastFull = time.Now().Add(-2 * time.Hour)
f.agent.pollLeases(ctx, f.role, tr)
batches = f.src.leaseBatches()
if last := batches[len(batches)-1]; !last.FullSync || last.Sequence != 5 {
t.Errorf("periodic snapshot = %+v", last)
}
// Errors of the role and a busy role skip the round.
f.role.mu.Lock()
f.role.leasesErr = errors.New("socket gone")
f.role.mu.Unlock()
n := len(f.src.leaseBatches())
tr.needFull = true
f.agent.pollLeases(ctx, f.role, tr)
f.role.mu.Lock()
f.role.leasesErr = nil
f.role.mu.Unlock()
f.agent.roleMu.Lock()
f.agent.pollLeases(ctx, f.role, tr)
f.agent.roleMu.Unlock()
if len(f.src.leaseBatches()) != n {
t.Error("leases reported despite an error or a running apply")
}
}
func TestPollLeasesEmptySnapshot(t *testing.T) {
f := newFixture(t, KindDHCP)
tr := &leaseTracker{needFull: true}
f.agent.pollLeases(context.Background(), f.role, tr)
b := f.src.leaseBatches()
if len(b) != 1 || !b[0].FullSync || len(b[0].Leases) != 0 {
t.Fatalf("batches = %+v", b)
}
}
func TestLeaseBatching(t *testing.T) {
f := newFixture(t, KindDHCP)
var leases []Lease
for i := 0; i < 2*maxLeasesPerBatch+1; i++ {
ip := netip.AddrFrom4([4]byte{10, byte(i >> 16), byte(i >> 8), byte(i)})
leases = append(leases, Lease{IP: ip, MAC: fmt.Sprintf("aa:00:00:%02x:%02x:%02x", byte(i>>16), byte(i>>8), byte(i)), State: LeaseActive})
}
f.role.setLeases(leases)
f.agent.pollLeases(context.Background(), f.role, &leaseTracker{needFull: true})
b := f.src.leaseBatches()
if len(b) != 3 {
t.Fatalf("%d batches", len(b))
}
if !b[0].FullSync || b[1].FullSync || b[2].FullSync {
t.Errorf("full_sync flags = %v %v %v", b[0].FullSync, b[1].FullSync, b[2].FullSync)
}
if len(b[0].Leases) != maxLeasesPerBatch || len(b[2].Leases) != 1 || b[0].Sequence != 1 || b[2].Sequence != 3 {
t.Errorf("batches: %d/%d/%d leases, sequences %d..%d", len(b[0].Leases), len(b[1].Leases), len(b[2].Leases), b[0].Sequence, b[2].Sequence)
}
}
func TestLeaseLoopNotStartedForDNS(t *testing.T) {
f := newFixture(t, KindDNS)
role := dnsOnlyRole{f.role}
a, err := NewAgent(KindDNS, f.agent.cfg, role, f.store, &Identity{AgentID: testAgentID, BundleKey: f.agent.bundleKey}, f.src, testLogger())
if err != nil {
t.Fatal(err)
}
if _, ok := a.role.(LeaseSource); ok {
t.Fatal("DNS test role implements LeaseSource")
}
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
defer cancel()
if err := a.Run(ctx); err != nil {
t.Fatal(err)
}
if n := len(f.src.leaseBatches()); n != 0 {
t.Errorf("%d lease batches from a DNS agent", n)
}
}
agent/core/lock_flock.go
+32
View File
@@ -0,0 +1,32 @@
// SPDX-License-Identifier: Apache-2.0
//go:build linux || darwin || freebsd || netbsd || openbsd || dragonfly
package core
import (
"errors"
"fmt"
"os"
"syscall"
)
// lockPath takes an exclusive, non-blocking flock on path. The lock is
// released by the returned function or when the process exits.
func lockPath(path string) (func(), error) {
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600) //nolint:gosec // path built from the state directory
if err != nil {
return nil, fmt.Errorf("open lock file: %w", err)
}
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { //nolint:gosec // file descriptors fit into int
_ = f.Close()
if errors.Is(err, syscall.EWOULDBLOCK) {
return nil, fmt.Errorf("state directory is locked by another agent process (%s)", path)
}
return nil, fmt.Errorf("lock state directory: %w", err)
}
return func() {
_ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) //nolint:gosec // file descriptors fit into int
_ = f.Close()
}, nil
}
agent/core/lock_other.go
+11
View File
@@ -0,0 +1,11 @@
// SPDX-License-Identifier: Apache-2.0
//go:build !(linux || darwin || freebsd || netbsd || openbsd || dragonfly)
package core
// lockPath is a no-op on platforms without flock. The agents only run on
// Linux; this keeps the package buildable elsewhere (e.g. for development).
func lockPath(string) (func(), error) {
return func() {}, nil
}
agent/core/server_source.go
+599
View File
@@ -0,0 +1,599 @@
// SPDX-License-Identifier: Apache-2.0
package core
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"log/slog"
"math/rand/v2"
"sync"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/keepalive"
"google.golang.org/protobuf/types/known/timestamppb"
"git.beugsiud.de/beug.k/DDI/internal/agentpki"
"git.beugsiud.de/beug.k/DDI/internal/buildinfo"
"git.beugsiud.de/beug.k/DDI/internal/bundle"
agentv1 "git.beugsiud.de/beug.k/DDI/proto/gen/ddi/agent/v1"
)
// Defaults of the server connection (docs/adr/0021-agent-strecke-m3.md).
const (
defaultPullInterval = 60 * time.Second
defaultHeartbeatInterval = 30 * time.Second
keepaliveInterval = 30 * time.Second
keepaliveTimeout = 10 * time.Second
backoffMin = time.Second
backoffMax = 60 * time.Second
welcomeTimeout = 30 * time.Second
rpcTimeout = 30 * time.Second
// renewRetryInterval spaces failed renewal attempts (also with an
// expired certificate, which the server accepts for 30 days).
renewRetryInterval = 15 * time.Minute
// oldConnGrace keeps the connection of a replaced certificate open for
// calls in flight.
oldConnGrace = 30 * time.Second
// maxBundleMessage allows bundles up to the bundle size limit.
maxBundleMessage = bundle.MaxTotalSize + 16<<20
)
// errReconnect ends a session after the certificate was renewed.
var errReconnect = errors.New("reconnecting with the renewed certificate")
// ServerSource is the Source backed by the management server: an mTLS gRPC
// connection with the Connect control stream and GetDesiredState polling as
// fallback (docs/adr/0007-server-agent-grpc-mtls.md). It also renews the
// client certificate.
type ServerSource struct {
store *Store
agentID string
ca *x509.Certificate
server string
serverName string
hostname string
log *slog.Logger
// Timing, shortened by tests.
backoffMin, backoffMax time.Duration
welcomeTimeout time.Duration
renewRetry time.Duration
// dialOptions are appended to the defaults (tests).
dialOptions []grpc.DialOption
mu sync.Mutex
cert tls.Certificate
conn *grpc.ClientConn
pullInterval time.Duration
heartbeatInterval time.Duration
lastRenewTry time.Time
renewMu sync.Mutex
}
// NewServerSource creates the source for an enrolled identity.
func NewServerSource(store *Store, id *Identity, hostname string, log *slog.Logger) *ServerSource {
return &ServerSource{
store: store, agentID: id.AgentID, ca: id.CA, server: id.Server, serverName: id.ServerName,
hostname: hostname, log: log, cert: id.Certificate,
backoffMin: backoffMin, backoffMax: backoffMax, welcomeTimeout: welcomeTimeout,
renewRetry: renewRetryInterval, pullInterval: defaultPullInterval,
heartbeatInterval: defaultHeartbeatInterval,
}
}
// client returns the gRPC client, creating the connection on first use.
// Connections are established lazily by gRPC.
func (s *ServerSource) client() (agentv1.AgentServiceClient, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.conn == nil {
opts := append([]grpc.DialOption{
grpc.WithTransportCredentials(credentials.NewTLS(agentpki.ClientTLSConfig(s.ca, s.cert, s.serverName))),
grpc.WithKeepaliveParams(keepalive.ClientParameters{
Time: keepaliveInterval, Timeout: keepaliveTimeout, PermitWithoutStream: true,
}),
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxBundleMessage)),
}, s.dialOptions...)
conn, err := grpc.NewClient(s.server, opts...)
if err != nil {
return nil, fmt.Errorf("connect to %s: %w", s.server, err)
}
s.conn = conn
}
return agentv1.NewAgentServiceClient(s.conn), nil
}
// replaceConn drops the current connection (after a certificate renewal);
// the next call dials with the new certificate.
func (s *ServerSource) replaceConn() {
s.mu.Lock()
old := s.conn
s.conn = nil
s.mu.Unlock()
if old != nil {
time.AfterFunc(oldConnGrace, func() { _ = old.Close() })
}
}
func (s *ServerSource) closeConn() {
s.mu.Lock()
defer s.mu.Unlock()
if s.conn != nil {
_ = s.conn.Close()
s.conn = nil
}
}
func (s *ServerSource) intervals() (pull, heartbeat time.Duration) {
s.mu.Lock()
defer s.mu.Unlock()
return s.pullInterval, s.heartbeatInterval
}
func (s *ServerSource) setIntervals(pull, heartbeat time.Duration) {
s.mu.Lock()
defer s.mu.Unlock()
if pull > 0 {
s.pullInterval = pull
}
if heartbeat > 0 {
s.heartbeatInterval = heartbeat
}
}
// Run implements Source: it keeps a Connect stream open, reconnecting with
// exponential backoff and jitter, and polls GetDesiredState every pull
// interval while no stream can be established.
func (s *ServerSource) Run(ctx context.Context, h SourceHandler) error {
defer s.closeConn()
backoff := s.backoffMin
var lastPoll time.Time
reachable := false
for ctx.Err() == nil {
s.maybeRenew(ctx)
established, err := s.session(ctx, h)
if ctx.Err() != nil {
break
}
wait := true
switch {
case errors.Is(err, errReconnect):
wait = false
backoff = s.backoffMin
case established:
s.log.Warn("control stream to the server closed", "error", err)
backoff = s.backoffMin
reachable = false
default:
s.log.Warn("cannot open control stream", "server", s.server, "error", err)
pull, _ := s.intervals()
if time.Since(lastPoll) >= pull {
lastPoll = time.Now()
if err := s.poll(ctx, h); err != nil {
s.log.Warn("cannot poll the desired state", "error", err)
reachable = false
} else if !reachable {
reachable = true
h.Connected()
}
}
}
if wait {
if !sleepCtx(ctx, jitter(backoff)) {
break
}
backoff = min(backoff*2, s.backoffMax)
}
}
return ctx.Err()
}
// session runs one Connect stream. established reports whether the server
// answered the Hello with a Welcome.
func (s *ServerSource) session(ctx context.Context, h SourceHandler) (established bool, err error) {
cl, err := s.client()
if err != nil {
return false, err
}
sctx, cancel := context.WithCancel(ctx)
defer cancel()
stream, err := cl.Connect(sctx)
if err != nil {
return false, err
}
defer func() { _ = stream.CloseSend() }()
st := h.Status(sctx)
if err := stream.Send(&agentv1.ConnectRequest{Message: &agentv1.ConnectRequest_Hello{Hello: &agentv1.Hello{
AgentVersion: buildinfo.Get().Version, AppliedRevision: st.Applied, LastGoodRevision: st.LastGood,
Health: healthToProto(st.Health, st.CheckedAt), ServiceVersion: st.ServiceVersion, Hostname: s.hostname,
}}}); err != nil {
return false, err
}
msgs := make(chan *agentv1.ConnectResponse)
recvErr := make(chan error, 1)
go func() {
for {
msg, err := stream.Recv()
if err != nil {
recvErr <- err
return
}
select {
case msgs <- msg:
case <-sctx.Done():
return
}
}
}()
timer := time.NewTimer(s.welcomeTimeout)
defer timer.Stop()
select {
case <-ctx.Done():
return false, ctx.Err()
case err := <-recvErr:
return false, err
case <-timer.C:
return false, errors.New("server did not answer the Hello in time")
case msg := <-msgs:
w := msg.GetWelcome()
if w == nil {
return false, errors.New("server did not answer the Hello with a Welcome")
}
s.log.Info("connected to the server", "server", s.server, "mode", w.GetMode().String(),
"desired_revision", w.GetDesiredRevision())
s.welcome(w, h, true)
}
_, hbInterval := s.intervals()
heartbeat := time.NewTicker(hbInterval)
defer heartbeat.Stop()
renew := time.NewTimer(s.untilRenewal())
defer renew.Stop()
for {
select {
case <-ctx.Done():
return true, ctx.Err()
case err := <-recvErr:
return true, err
case msg := <-msgs:
switch m := msg.GetMessage().(type) {
case *agentv1.ConnectResponse_Welcome:
s.welcome(m.Welcome, h, false)
_, hb := s.intervals()
heartbeat.Reset(hb)
case *agentv1.ConnectResponse_Command:
if err := s.command(ctx, stream, m.Command, h); err != nil {
return true, err
}
}
case <-heartbeat.C:
st := h.Status(sctx)
if err := stream.Send(&agentv1.ConnectRequest{Message: &agentv1.ConnectRequest_Heartbeat{Heartbeat: &agentv1.Heartbeat{
AppliedRevision: st.Applied, LastGoodRevision: st.LastGood, Health: healthToProto(st.Health, st.CheckedAt),
}}}); err != nil {
return true, err
}
case <-renew.C:
if s.maybeRenew(ctx) {
return true, errReconnect
}
renew.Reset(s.untilRenewal())
}
}
}
// welcome applies the settings of a Welcome and announces its desired
// revision.
func (s *ServerSource) welcome(w *agentv1.Welcome, h SourceHandler, first bool) {
s.setIntervals(w.GetPullInterval().AsDuration(), w.GetHeartbeatInterval().AsDuration())
if first {
h.Connected()
}
h.Desired(w.GetDesiredRevision(), w.GetMode() != agentv1.OperatingMode_OPERATING_MODE_MANUAL)
}
// command acknowledges and executes a server command.
func (s *ServerSource) command(ctx context.Context, stream agentv1.AgentService_ConnectClient, c *agentv1.Command, h SourceHandler) error {
ack := func(accepted bool, msg string) error {
return stream.Send(&agentv1.ConnectRequest{Message: &agentv1.ConnectRequest_CommandAck{CommandAck: &agentv1.CommandAck{
CommandId: c.GetCommandId(), Accepted: accepted, Message: msg,
}}})
}
log := s.log.With("command_id", c.GetCommandId())
switch a := c.GetAction().(type) {
case *agentv1.Command_ApplyRevision:
rev, dep := a.ApplyRevision.GetRevision(), a.ApplyRevision.GetDeploymentId()
if rev == 0 {
log.Warn("apply command without revision")
return ack(false, "revision is missing")
}
log.Info("apply command received", "revision", rev, "deployment_id", dep)
if err := ack(true, ""); err != nil {
return err
}
h.Push(rev, dep)
return nil
case *agentv1.Command_RenewCertificate:
log.Info("certificate renewal requested by the server")
if err := ack(true, ""); err != nil {
return err
}
if err := s.renew(ctx); err != nil {
log.Error("certificate renewal failed", "error", err)
return nil
}
return errReconnect
default:
log.Warn("unknown command")
return ack(false, "unknown command")
}
}
// poll asks for the desired state once.
func (s *ServerSource) poll(ctx context.Context, h SourceHandler) error {
cl, err := s.client()
if err != nil {
return err
}
st := h.Status(ctx)
rctx, cancel := context.WithTimeout(ctx, rpcTimeout)
defer cancel()
resp, err := cl.GetDesiredState(rctx, &agentv1.GetDesiredStateRequest{AppliedRevision: st.Applied})
if err != nil {
return err
}
s.setIntervals(resp.GetPullInterval().AsDuration(), 0)
s.log.Debug("desired state polled", "desired_revision", resp.GetDesiredRevision(), "mode", resp.GetMode().String())
h.Desired(resp.GetDesiredRevision(), resp.GetMode() != agentv1.OperatingMode_OPERATING_MODE_MANUAL)
return nil
}
// leaf returns the current client certificate.
func (s *ServerSource) leaf() *x509.Certificate {
s.mu.Lock()
defer s.mu.Unlock()
return s.cert.Leaf
}
// renewalDue returns when the certificate should be renewed: when half of
// its lifetime has passed.
func renewalDue(c *x509.Certificate) time.Time {
return c.NotBefore.Add(c.NotAfter.Sub(c.NotBefore) / 2)
}
// untilRenewal returns the time until the next renewal attempt.
func (s *ServerSource) untilRenewal() time.Duration {
due := renewalDue(s.leaf())
s.mu.Lock()
last := s.lastRenewTry
s.mu.Unlock()
if !last.IsZero() && time.Now().After(due) {
due = last.Add(s.renewRetry)
}
return max(time.Until(due), 0)
}
// maybeRenew renews the certificate if half of its lifetime has passed
// (or it expired), spacing failed attempts by renewRetry.
func (s *ServerSource) maybeRenew(ctx context.Context) bool {
leaf := s.leaf()
now := time.Now()
if now.Before(renewalDue(leaf)) {
return false
}
s.mu.Lock()
if !s.lastRenewTry.IsZero() && now.Sub(s.lastRenewTry) < s.renewRetry {
s.mu.Unlock()
return false
}
s.lastRenewTry = now
s.mu.Unlock()
if err := s.renew(ctx); err != nil {
if now.After(leaf.NotAfter) {
s.log.Error("client certificate expired and renewal failed; the agent must be enrolled again if this persists",
"not_after", leaf.NotAfter, "error", err)
} else {
s.log.Warn("certificate renewal failed", "not_after", leaf.NotAfter, "error", err)
}
return false
}
return true
}
// renew obtains a certificate for a new key and stores it.
func (s *ServerSource) renew(ctx context.Context) error {
s.renewMu.Lock()
defer s.renewMu.Unlock()
key, csr, err := newKeyAndCSR(s.agentID)
if err != nil {
return err
}
cl, err := s.client()
if err != nil {
return err
}
rctx, cancel := context.WithTimeout(ctx, rpcTimeout)
defer cancel()
resp, err := cl.RenewCertificate(rctx, &agentv1.RenewCertificateRequest{CsrDer: csr})
if err != nil {
return err
}
if chain := resp.GetCaCertificatesDer(); len(chain) > 0 {
if root, err := x509.ParseCertificate(chain[len(chain)-1]); err != nil || !root.Equal(s.ca) {
s.log.Warn("server sent a different CA certificate; keeping the enrolled CA")
}
}
cert, err := checkIssued(resp.GetCertificateDer(), s.ca, s.agentID, key)
if err != nil {
return err
}
if err := s.store.SaveCertificate(key, cert); err != nil {
return err
}
s.mu.Lock()
s.cert = keyPair(key, cert)
s.lastRenewTry = time.Time{}
s.mu.Unlock()
s.replaceConn()
s.log.Info("client certificate renewed", "not_after", cert.NotAfter)
return nil
}
// FetchBundle implements Source.
func (s *ServerSource) FetchBundle(ctx context.Context, revision uint64) (*Bundle, error) {
cl, err := s.client()
if err != nil {
return nil, err
}
resp, err := cl.FetchBundle(ctx, &agentv1.FetchBundleRequest{Revision: revision})
if err != nil {
return nil, err
}
b := &Bundle{Manifest: resp.GetManifest(), Signature: resp.GetSignature()}
for _, a := range resp.GetArtifacts() {
b.Artifacts = append(b.Artifacts, BundleArtifact{Path: a.GetPath(), Content: a.GetContent()})
}
return b, nil
}
// ReportDeployment implements Source.
func (s *ServerSource) ReportDeployment(ctx context.Context, r DeploymentReport) error {
cl, err := s.client()
if err != nil {
return err
}
_, err = cl.ReportDeployment(ctx, &agentv1.ReportDeploymentRequest{
DeploymentId: r.DeploymentID, Revision: r.Revision, Trigger: triggerToProto(r.Trigger),
Outcome: outcomeToProto(r.Outcome), Message: r.Message, ActiveRevision: r.ActiveRevision,
StartedAt: timestamppb.New(r.StartedAt), FinishedAt: timestamppb.New(r.FinishedAt),
})
return err
}
// ReportLeases implements Source.
func (s *ServerSource) ReportLeases(ctx context.Context, b LeaseBatch) error {
cl, err := s.client()
if err != nil {
return err
}
req := &agentv1.ReportLeasesRequest{Sequence: b.Sequence, FullSync: b.FullSync, Leases: make([]*agentv1.Lease, len(b.Leases))}
for i, l := range b.Leases {
req.Leases[i] = leaseToProto(l)
}
rctx, cancel := context.WithTimeout(ctx, rpcTimeout)
defer cancel()
resp, err := cl.ReportLeases(rctx, req)
if err != nil {
return err
}
if ack := resp.GetAcknowledgedSequence(); ack != b.Sequence {
s.log.Debug("lease report acknowledged with another sequence", "sequence", b.Sequence, "acknowledged", ack)
}
return nil
}
func healthToProto(h Health, at time.Time) *agentv1.HealthStatus {
out := &agentv1.HealthStatus{State: healthStateToProto(h.State), Message: h.Message}
if !at.IsZero() {
out.CheckedAt = timestamppb.New(at)
}
for _, c := range h.Checks {
out.Checks = append(out.Checks, &agentv1.HealthCheck{Name: c.Name, State: healthStateToProto(c.State), Message: c.Message})
}
return out
}
func healthStateToProto(s HealthState) agentv1.HealthState {
switch s {
case HealthOK:
return agentv1.HealthState_HEALTH_STATE_OK
case HealthDegraded:
return agentv1.HealthState_HEALTH_STATE_DEGRADED
case HealthFailed:
return agentv1.HealthState_HEALTH_STATE_FAILED
default:
return agentv1.HealthState_HEALTH_STATE_UNSPECIFIED
}
}
func triggerToProto(t Trigger) agentv1.DeploymentTrigger {
switch t {
case TriggerPull:
return agentv1.DeploymentTrigger_DEPLOYMENT_TRIGGER_PULL
case TriggerPush:
return agentv1.DeploymentTrigger_DEPLOYMENT_TRIGGER_PUSH
default:
return agentv1.DeploymentTrigger_DEPLOYMENT_TRIGGER_UNSPECIFIED
}
}
func outcomeToProto(o Outcome) agentv1.DeploymentOutcome {
switch o {
case OutcomeSucceeded:
return agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_SUCCEEDED
case OutcomeRejected:
return agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_REJECTED
case OutcomeRolledBack:
return agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_ROLLED_BACK
case OutcomeFailed:
return agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_FAILED
default:
return agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_UNSPECIFIED
}
}
func leaseToProto(l Lease) *agentv1.Lease {
out := &agentv1.Lease{
Ip: l.IP.String(), Mac: l.MAC, Hostname: l.Hostname, State: leaseStateToProto(l.State),
SubnetId: l.SubnetID, ClientId: l.ClientID,
}
if !l.Expires.IsZero() {
out.ExpiresAt = timestamppb.New(l.Expires)
}
if !l.Updated.IsZero() {
out.UpdatedAt = timestamppb.New(l.Updated)
}
return out
}
func leaseStateToProto(s LeaseState) agentv1.LeaseState {
switch s {
case LeaseActive:
return agentv1.LeaseState_LEASE_STATE_ACTIVE
case LeaseExpired:
return agentv1.LeaseState_LEASE_STATE_EXPIRED
case LeaseReleased:
return agentv1.LeaseState_LEASE_STATE_RELEASED
case LeaseDeclined:
return agentv1.LeaseState_LEASE_STATE_DECLINED
default:
return agentv1.LeaseState_LEASE_STATE_UNSPECIFIED
}
}
// jitter returns d randomized by ±20 %.
func jitter(d time.Duration) time.Duration {
return time.Duration(float64(d) * (0.8 + 0.4*rand.Float64())) //nolint:gosec // jitter needs no cryptographic randomness
}
// sleepCtx waits for d; it returns false if ctx ended first.
func sleepCtx(ctx context.Context, d time.Duration) bool {
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
return false
case <-t.C:
return true
}
}
agent/core/server_source_test.go
+412
View File
@@ -0,0 +1,412 @@
// SPDX-License-Identifier: Apache-2.0
package core
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"git.beugsiud.de/beug.k/DDI/internal/agentpki"
agentv1 "git.beugsiud.de/beug.k/DDI/proto/gen/ddi/agent/v1"
)
// enrollWith enrolls a new store against the fake server.
func enrollWith(t *testing.T, srv *fakeServer, addr string) (*Store, *Identity) {
t.Helper()
store := openTestStore(t)
id, err := Enroll(context.Background(), store, EnrollOptions{
Kind: srv.role, Server: addr, Token: srv.token, CAHash: srv.fingerprint(),
ServerName: "localhost", Hostname: "test-host",
})
if err != nil {
t.Fatalf("Enroll: %v", err)
}
return store, id
}
func TestEnroll(t *testing.T) {
srv := newFakeServer(t, KindDNS)
addr := srv.start(t)
store, id := enrollWith(t, srv, addr)
if id.AgentID != srv.agentID || id.Server != addr || id.ServerName != "localhost" || !id.CA.Equal(srv.pki.ca) {
t.Fatalf("identity = %+v", id)
}
loaded, err := store.LoadIdentity()
if err != nil {
t.Fatal(err)
}
if loaded.AgentID != srv.agentID || !loaded.BundleKey.Equal(srv.signKey.Public()) || loaded.Role != KindDNS {
t.Errorf("stored identity = %+v", loaded)
}
if got, err := agentpki.AgentID(loaded.Certificate.Leaf); err != nil || got != srv.agentID {
t.Errorf("certificate agent ID = %q, %v", got, err)
}
// Enrolling again needs --force.
opts := EnrollOptions{Kind: KindDNS, Server: addr, Token: srv.token, CAHash: srv.fingerprint(), ServerName: "localhost"}
if _, err := Enroll(context.Background(), store, opts); !errors.Is(err, errAlreadyEnrolled) {
t.Fatalf("second Enroll = %v", err)
}
srv.with(func(f *fakeServer) { f.agentID = "22222222-2222-4333-8444-555555555555" })
st := &State{Applied: 3, PendingReports: []DeploymentReport{{Revision: 3}}}
if err := store.SaveState(st); err != nil {
t.Fatal(err)
}
opts.Force = true
id2, err := Enroll(context.Background(), store, opts)
if err != nil || id2.AgentID != "22222222-2222-4333-8444-555555555555" {
t.Fatalf("forced Enroll = %+v, %v", id2, err)
}
if st, _ := store.LoadState(); st.Applied != 3 || len(st.PendingReports) != 0 {
t.Errorf("state after forced enroll = %+v", st)
}
}
func TestEnrollDefaultsServerNameToHost(t *testing.T) {
srv := newFakeServer(t, KindDHCP)
addr := srv.start(t) // 127.0.0.1:port; the server certificate has the IP SAN
store := openTestStore(t)
id, err := Enroll(context.Background(), store, EnrollOptions{
Kind: KindDHCP, Server: "https://" + addr + "/", Token: srv.token, CAHash: strings.ToUpper(srv.fingerprint()[7:]),
})
if err == nil {
t.Fatalf("fingerprint without prefix accepted: %+v", id)
}
id, err = Enroll(context.Background(), store, EnrollOptions{
Kind: KindDHCP, Server: "https://" + addr + "/", Token: srv.token, CAHash: "sha256:" + strings.ToUpper(srv.fingerprint()[7:]),
})
if err != nil {
t.Fatal(err)
}
if id.ServerName != "127.0.0.1" || id.Server != addr {
t.Errorf("identity = %+v", id)
}
}
func TestEnrollFailures(t *testing.T) {
srv := newFakeServer(t, KindDNS)
addr := srv.start(t)
other := newTestPKI(t)
cases := map[string]EnrollOptions{
"wrong fingerprint": {Server: addr, Token: srv.token, CAHash: agentpki.Fingerprint(other.ca), ServerName: "localhost"},
"wrong server name": {Server: addr, Token: srv.token, CAHash: srv.fingerprint(), ServerName: "evil.example.com"},
"wrong token": {Server: addr, Token: "0123456789abcdef.wrong", CAHash: srv.fingerprint(), ServerName: "localhost"},
"bad token": {Server: addr, Token: "nodot", CAHash: srv.fingerprint()},
"bad address": {Server: "localhost", Token: srv.token, CAHash: srv.fingerprint()},
"bad fingerprint": {Server: addr, Token: srv.token, CAHash: "sha256:abc"},
}
for name, opts := range cases {
t.Run(name, func(t *testing.T) {
opts.Kind = KindDNS
store := openTestStore(t)
if _, err := Enroll(context.Background(), store, opts); err == nil {
t.Fatal("accepted")
}
if store.Enrolled() {
t.Error("identity stored after a failed enrollment")
}
})
}
// Role mismatch: the server refuses.
store := openTestStore(t)
if _, err := Enroll(context.Background(), store, EnrollOptions{
Kind: KindDHCP, Server: addr, Token: srv.token, CAHash: srv.fingerprint(), ServerName: "localhost",
}); err == nil {
t.Error("wrong role accepted")
}
if _, err := Enroll(context.Background(), store, EnrollOptions{
Kind: Kind("ntp"), Server: addr, Token: srv.token, CAHash: srv.fingerprint(),
}); err == nil {
t.Error("unknown role accepted")
}
srv.with(func(f *fakeServer) {
if f.enrollments == 0 {
t.Error("server never reached")
}
})
}
// runAgentAgainst runs an agent with a ServerSource until the test ends.
func runAgentAgainst(t *testing.T, store *Store, id *Identity, kind Kind) (*fakeRole, *ServerSource) {
t.Helper()
role := newFakeRole(kind)
src := NewServerSource(store, id, "test-host", testLogger())
src.backoffMin, src.backoffMax = 10*time.Millisecond, 100*time.Millisecond
a, err := NewAgent(kind, testConfig(store.Dir()), role, store, id, src, testLogger())
if err != nil {
t.Fatal(err)
}
a.healthRetry = 20 * time.Millisecond
a.reportRetry = 50 * time.Millisecond
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- a.Run(ctx) }()
t.Cleanup(func() {
cancel()
select {
case err := <-done:
if err != nil {
t.Errorf("Run = %v", err)
}
case <-time.After(10 * time.Second):
t.Error("Run did not return after cancel")
}
})
return role, src
}
func serverReports(srv *fakeServer) []*agentv1.ReportDeploymentRequest {
var out []*agentv1.ReportDeploymentRequest
srv.with(func(f *fakeServer) { out = append(out, f.reports...) })
return out
}
func TestServerSourceStreamAndPush(t *testing.T) {
srv := newFakeServer(t, KindDNS)
addr := srv.start(t)
store, id := enrollWith(t, srv, addr)
srv.with(func(f *fakeServer) { f.desired = 1 })
role, _ := runAgentAgainst(t, store, id, KindDNS)
waitFor(t, "report of revision 1", func() bool { return len(serverReports(srv)) == 1 })
r := serverReports(srv)[0]
if r.GetRevision() != 1 || r.GetOutcome() != agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_SUCCEEDED ||
r.GetTrigger() != agentv1.DeploymentTrigger_DEPLOYMENT_TRIGGER_PULL || r.GetActiveRevision() != 1 ||
r.GetStartedAt() == nil || r.GetFinishedAt() == nil {
t.Fatalf("report = %v", r)
}
srv.with(func(f *fakeServer) {
if len(f.hellos) == 0 || f.hellos[0].GetHostname() != "test-host" || f.hellos[0].GetAgentVersion() == "" {
t.Errorf("hellos = %v", f.hellos)
}
})
waitFor(t, "heartbeats", func() bool {
n := 0
srv.with(func(f *fakeServer) { n = f.heartbeats })
return n >= 2
})
// Push over the stream.
srv.commands <- &agentv1.Command{CommandId: "c1", Action: &agentv1.Command_ApplyRevision{
ApplyRevision: &agentv1.ApplyRevisionCommand{Revision: 2, DeploymentId: "dep-2"},
}}
waitFor(t, "report of revision 2", func() bool { return len(serverReports(srv)) == 2 })
r = serverReports(srv)[1]
if r.GetRevision() != 2 || r.GetDeploymentId() != "dep-2" || r.GetTrigger() != agentv1.DeploymentTrigger_DEPLOYMENT_TRIGGER_PUSH ||
r.GetOutcome() != agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_SUCCEEDED {
t.Fatalf("report = %v", r)
}
srv.with(func(f *fakeServer) {
if len(f.acks) != 1 || f.acks[0].GetCommandId() != "c1" || !f.acks[0].GetAccepted() {
t.Errorf("acks = %v", f.acks)
}
})
if n := role.activeRevision(); n != 2 {
t.Errorf("active = %d", n)
}
// Invalid and unknown commands are refused.
srv.commands <- &agentv1.Command{CommandId: "c2", Action: &agentv1.Command_ApplyRevision{ApplyRevision: &agentv1.ApplyRevisionCommand{}}}
srv.commands <- &agentv1.Command{CommandId: "c3"}
waitFor(t, "refusals", func() bool {
n := 0
srv.with(func(f *fakeServer) { n = len(f.acks) })
return n == 3
})
srv.with(func(f *fakeServer) {
if f.acks[1].GetAccepted() || f.acks[2].GetAccepted() {
t.Errorf("acks = %v", f.acks)
}
})
}
func TestServerSourceManualMode(t *testing.T) {
srv := newFakeServer(t, KindDNS)
addr := srv.start(t)
store, id := enrollWith(t, srv, addr)
srv.with(func(f *fakeServer) {
f.desired = 1
f.mode = agentv1.OperatingMode_OPERATING_MODE_MANUAL
})
runAgentAgainst(t, store, id, KindDNS)
waitFor(t, "hello", func() bool {
n := 0
srv.with(func(f *fakeServer) { n = f.heartbeats })
return n >= 3
})
if n := len(serverReports(srv)); n != 0 {
t.Errorf("manual mode applied the desired revision (%d reports)", n)
}
}
func TestServerSourcePollingFallback(t *testing.T) {
srv := newFakeServer(t, KindDNS)
addr := srv.start(t)
store, id := enrollWith(t, srv, addr)
srv.with(func(f *fakeServer) {
f.desired = 1
f.streamDisabled = true
})
runAgentAgainst(t, store, id, KindDNS)
waitFor(t, "report via polling", func() bool { return len(serverReports(srv)) == 1 })
srv.with(func(f *fakeServer) {
if f.polls == 0 {
t.Error("GetDesiredState never called")
}
})
// A newer revision is picked up by a later poll.
srv.with(func(f *fakeServer) { f.desired = 2 })
waitFor(t, "report of revision 2", func() bool { return len(serverReports(srv)) == 2 })
if st, _ := store.LoadState(); st.Applied != 2 {
t.Errorf("state = %+v", st)
}
}
func TestServerSourceRenewsCertificate(t *testing.T) {
srv := newFakeServer(t, KindDNS)
// More than half of the certificate's lifetime has passed.
srv.enrollValidity = [2]time.Duration{-3 * time.Hour, time.Hour}
addr := srv.start(t)
store, id := enrollWith(t, srv, addr)
oldCert, _ := os.ReadFile(filepath.Join(store.Dir(), "identity", "cert.pem"))
runAgentAgainst(t, store, id, KindDNS)
waitFor(t, "renewal and hello", func() bool {
ok := false
srv.with(func(f *fakeServer) { ok = f.renewals == 1 && len(f.hellos) >= 1 })
return ok
})
newCert, _ := os.ReadFile(filepath.Join(store.Dir(), "identity", "cert.pem"))
if string(newCert) == string(oldCert) {
t.Error("cert.pem unchanged after renewal")
}
loaded, err := store.LoadIdentity()
if err != nil {
t.Fatal(err)
}
if time.Until(loaded.Certificate.Leaf.NotAfter) < 24*time.Hour {
t.Errorf("renewed certificate expires %v", loaded.Certificate.Leaf.NotAfter)
}
}
func TestServerSourceRenewCommand(t *testing.T) {
srv := newFakeServer(t, KindDNS)
addr := srv.start(t)
store, id := enrollWith(t, srv, addr)
runAgentAgainst(t, store, id, KindDNS)
waitFor(t, "first hello", func() bool {
n := 0
srv.with(func(f *fakeServer) { n = len(f.hellos) })
return n == 1
})
srv.commands <- &agentv1.Command{CommandId: "r1", Action: &agentv1.Command_RenewCertificate{RenewCertificate: &agentv1.RenewCertificateCommand{}}}
// The old certificate is no longer accepted, so the second Hello proves
// that the agent reconnected with the new one.
waitFor(t, "reconnect with the renewed certificate", func() bool {
ok := false
srv.with(func(f *fakeServer) { ok = f.renewals == 1 && len(f.hellos) >= 2 })
return ok
})
}
func TestServerSourceReportsLeases(t *testing.T) {
srv := newFakeServer(t, KindDHCP)
addr := srv.start(t)
store, id := enrollWith(t, srv, addr)
role, _ := runAgentAgainst(t, store, id, KindDHCP)
_ = role
waitFor(t, "full lease sync", func() bool {
ok := false
srv.with(func(f *fakeServer) {
for _, r := range f.leaseReports {
if r.GetFullSync() {
ok = true
}
}
})
return ok
})
}
func TestLeaseToProto(t *testing.T) {
l := lease("10.0.0.9", "aa:bb:cc:dd:ee:ff")
l.ClientID = "01:aa"
p := leaseToProto(l)
if p.GetIp() != "10.0.0.9" || p.GetMac() != l.MAC || p.GetState() != agentv1.LeaseState_LEASE_STATE_ACTIVE ||
!p.GetExpiresAt().AsTime().Equal(l.Expires) || p.GetSubnetId() != 1 || p.GetClientId() != "01:aa" {
t.Errorf("lease = %v", p)
}
if p := leaseToProto(Lease{IP: mustAddr("10.0.0.1")}); p.ExpiresAt != nil || p.UpdatedAt != nil {
t.Errorf("zero times encoded: %v", p)
}
for s, want := range map[LeaseState]agentv1.LeaseState{
LeaseUnknown: agentv1.LeaseState_LEASE_STATE_UNSPECIFIED, LeaseExpired: agentv1.LeaseState_LEASE_STATE_EXPIRED,
LeaseReleased: agentv1.LeaseState_LEASE_STATE_RELEASED, LeaseDeclined: agentv1.LeaseState_LEASE_STATE_DECLINED,
} {
if got := leaseStateToProto(s); got != want {
t.Errorf("%v -> %v, want %v", s, got, want)
}
}
for o, want := range map[Outcome]agentv1.DeploymentOutcome{
OutcomeRejected: agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_REJECTED, OutcomeRolledBack: agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_ROLLED_BACK,
OutcomeFailed: agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_FAILED, Outcome("x"): agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_UNSPECIFIED,
} {
if got := outcomeToProto(o); got != want {
t.Errorf("%v -> %v, want %v", o, got, want)
}
}
if triggerToProto(Trigger("x")) != agentv1.DeploymentTrigger_DEPLOYMENT_TRIGGER_UNSPECIFIED {
t.Error("unknown trigger mapped")
}
h := healthToProto(Health{State: HealthDegraded, Checks: []HealthCheck{{Name: "a", State: HealthFailed}}}, time.Time{})
if h.GetState() != agentv1.HealthState_HEALTH_STATE_DEGRADED || h.CheckedAt != nil || h.GetChecks()[0].GetState() != agentv1.HealthState_HEALTH_STATE_FAILED {
t.Errorf("health = %v", h)
}
}
func TestServerSourceUnreachable(t *testing.T) {
srv := newFakeServer(t, KindDNS)
addr := srv.start(t)
store, id := enrollWith(t, srv, addr)
id.Server = "127.0.0.1:1" // nothing listens there
src := NewServerSource(store, id, "h", testLogger())
defer src.closeConn()
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := src.ReportDeployment(ctx, DeploymentReport{Revision: 1}); err == nil {
t.Error("report to an unreachable server succeeded")
}
if _, err := src.FetchBundle(ctx, 1); err == nil {
t.Error("fetch from an unreachable server succeeded")
}
if code := status.Code(src.ReportLeases(ctx, LeaseBatch{Sequence: 1})); code == codes.OK {
t.Error("lease report to an unreachable server succeeded")
}
}
func TestJitterAndRenewalDue(t *testing.T) {
for i := 0; i < 100; i++ {
if d := jitter(time.Second); d < 800*time.Millisecond || d > 1200*time.Millisecond {
t.Fatalf("jitter = %v", d)
}
}
pki := newTestPKI(t)
id, _ := testIdentity(t, pki)
leaf := id.Certificate.Leaf
if due := renewalDue(leaf); !due.Equal(leaf.NotBefore.Add(leaf.NotAfter.Sub(leaf.NotBefore) / 2)) {
t.Errorf("renewalDue = %v", due)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if sleepCtx(ctx, time.Hour) {
t.Error("sleepCtx ignored the context")
}
}
agent/core/source.go
+72
View File
@@ -0,0 +1,72 @@
// SPDX-License-Identifier: Apache-2.0
package core
import (
"context"
"time"
)
// Source is where the agent learns its desired revision, fetches bundles and
// reports results (docs/adr/0010-agent-binaries-pro-rolle.md). M3 provides
// ServerSource (gRPC); a FileSource for standalone operation (M7) implements
// the same interface without changes to the apply pipeline.
type Source interface {
// Run delivers events to h until ctx is canceled. It returns ctx.Err()
// on shutdown or an error if the source cannot work at all.
Run(ctx context.Context, h SourceHandler) error
// FetchBundle returns the signed bundle of a revision. The caller
// verifies it.
FetchBundle(ctx context.Context, revision uint64) (*Bundle, error)
// ReportDeployment delivers an apply outcome.
ReportDeployment(ctx context.Context, r DeploymentReport) error
// ReportLeases delivers a batch of lease observations.
ReportLeases(ctx context.Context, b LeaseBatch) error
}
// SourceHandler receives the events of a Source. Its methods must not
// block for long.
type SourceHandler interface {
// Connected is called whenever the source (re)established contact;
// the agent then sends queued reports and a full lease snapshot.
Connected()
// Desired announces the revision the agent should run. autoApply is
// false in manual mode, where only explicit pushes are applied.
Desired(revision uint64, autoApply bool)
// Push asks for an explicit apply of a revision. Unlike Desired, it
// also retries a revision that failed before.
Push(revision uint64, deploymentID string)
// Status returns what the agent reports in Hello and heartbeats.
Status(ctx context.Context) AgentStatus
}
// AgentStatus is the agent's view of itself.
type AgentStatus struct {
Applied uint64
LastGood uint64
Health Health
CheckedAt time.Time
ServiceVersion string
}
// Bundle is a signed bundle as delivered by a Source.
type Bundle struct {
Manifest []byte
Signature []byte
Artifacts []BundleArtifact
}
// BundleArtifact is one artifact of a Bundle.
type BundleArtifact struct {
Path string
Content []byte
}
// LeaseBatch is one lease report. Sequence is monotonic per agent. A batch
// with FullSync replaces the server's view of all leases of this agent; the
// following batches (higher sequence) are changes on top of it.
type LeaseBatch struct {
Sequence uint64
FullSync bool
Leases []Lease
}
agent/core/store.go
+362
View File
@@ -0,0 +1,362 @@
// SPDX-License-Identifier: Apache-2.0
package core
import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"time"
"git.beugsiud.de/beug.k/DDI/internal/bundle"
)
// Layout of the state directory (docs/adr/0011-agent-zustand-in-dateien.md).
const (
identityDir = "identity"
revisionsDir = "revisions"
stateFile = "state.json"
lockFile = "agent.lock"
manifestFile = "manifest.json"
signatureFile = "manifest.sig"
tmpPrefix = ".tmp-"
oldPrefix = ".old-"
)
// KeepRevisions is the number of stored revisions kept by Prune (the applied
// and the last good revision are kept in addition).
const KeepRevisions = 10
// maxPendingReports bounds the queue of unsent deployment reports.
const maxPendingReports = 100
// Store manages the agent's state directory. Writes are atomic: files are
// written to a temporary file, synced and renamed; directories are built
// under a temporary name and renamed into place.
type Store struct {
dir string
}
// OpenStore creates the state directory if necessary.
func OpenStore(dir string) (*Store, error) {
if dir == "" {
return nil, errors.New("state directory is not set")
}
if err := os.MkdirAll(filepath.Join(dir, revisionsDir), 0o750); err != nil {
return nil, fmt.Errorf("create state directory: %w", err)
}
return &Store{dir: dir}, nil
}
// Dir returns the state directory.
func (s *Store) Dir() string { return s.dir }
// Lock takes the exclusive lock of the state directory. The returned
// function releases it.
func (s *Store) Lock() (func(), error) {
return lockPath(filepath.Join(s.dir, lockFile))
}
// Trigger says why a revision was applied.
type Trigger string
// Triggers.
const (
TriggerPull Trigger = "pull"
TriggerPush Trigger = "push"
)
// Outcome is the result of applying a revision.
type Outcome string
// Outcomes (see DeploymentOutcome in the protocol).
const (
OutcomeSucceeded Outcome = "succeeded"
OutcomeRejected Outcome = "rejected"
OutcomeRolledBack Outcome = "rolled_back"
OutcomeFailed Outcome = "failed"
)
// DeploymentReport is the outcome of one apply, queued in state.json until
// the source accepted it.
type DeploymentReport struct {
DeploymentID string `json:"deployment_id,omitempty"`
Revision uint64 `json:"revision"`
Trigger Trigger `json:"trigger"`
Outcome Outcome `json:"outcome"`
Message string `json:"message,omitempty"`
ActiveRevision uint64 `json:"active_revision"`
StartedAt time.Time `json:"started_at"`
FinishedAt time.Time `json:"finished_at"`
}
// State is persisted in state.json.
type State struct {
// Applied is the revision whose configuration is installed (0 = none).
Applied uint64 `json:"applied_revision"`
// LastGood is the last revision that passed its health check.
LastGood uint64 `json:"last_good_revision"`
// LastUnsuccessful is the highest revision that was rejected, rolled
// back or failed. Pulls never retry it; only a newer revision or an
// explicit push does.
LastUnsuccessful uint64 `json:"last_unsuccessful_revision,omitempty"`
// LastResult is the most recent apply outcome.
LastResult *DeploymentReport `json:"last_result,omitempty"`
// LeaseSequence is the sequence number of the last lease report.
LeaseSequence uint64 `json:"lease_sequence,omitempty"`
// PendingReports are deployment reports not yet accepted by the server.
PendingReports []DeploymentReport `json:"pending_reports,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
}
// LoadState reads state.json; a missing file yields the zero state.
func (s *Store) LoadState() (*State, error) {
var st State
data, err := os.ReadFile(filepath.Join(s.dir, stateFile))
if errors.Is(err, fs.ErrNotExist) {
return &st, nil
}
if err != nil {
return nil, fmt.Errorf("read state: %w", err)
}
if err := json.Unmarshal(data, &st); err != nil {
return nil, fmt.Errorf("decode %s: %w", stateFile, err)
}
return &st, nil
}
// SaveState writes state.json atomically.
func (s *Store) SaveState(st *State) error {
if n := len(st.PendingReports); n > maxPendingReports {
st.PendingReports = slices.Clone(st.PendingReports[n-maxPendingReports:])
}
data, err := json.MarshalIndent(st, "", " ")
if err != nil {
return fmt.Errorf("encode state: %w", err)
}
return writeFileAtomic(filepath.Join(s.dir, stateFile), append(data, '\n'), 0o600)
}
// revisionPath returns the directory of revision n.
func (s *Store) revisionPath(n uint64) string {
return filepath.Join(s.dir, revisionsDir, strconv.FormatUint(n, 10))
}
// reservedArtifact reports whether an artifact path would collide with the
// files the store keeps next to the artifacts.
func reservedArtifact(p string) bool {
return p == manifestFile || p == signatureFile
}
// StoreRevision writes a verified bundle to revisions/<n>/ (manifest,
// signature and the artifacts at their manifest paths) and returns it. An
// existing directory for the same number is replaced.
func (s *Store) StoreRevision(m *bundle.Manifest, manifest, signature []byte, contents map[string][]byte) (Revision, error) {
parent := filepath.Join(s.dir, revisionsDir)
tmp, err := os.MkdirTemp(parent, tmpPrefix)
if err != nil {
return Revision{}, fmt.Errorf("create revision directory: %w", err)
}
defer func() { _ = os.RemoveAll(tmp) }()
if err := writeFileSync(filepath.Join(tmp, manifestFile), manifest, 0o640); err != nil {
return Revision{}, err
}
if err := writeFileSync(filepath.Join(tmp, signatureFile), signature, 0o640); err != nil {
return Revision{}, err
}
dirs := map[string]bool{tmp: true}
for _, f := range m.Files {
if err := bundle.CheckPath(f.Path); err != nil {
return Revision{}, err
}
if reservedArtifact(f.Path) {
return Revision{}, fmt.Errorf("artifact path %s is reserved", f.Path)
}
target := filepath.Join(tmp, filepath.FromSlash(f.Path))
dir := filepath.Dir(target)
if !dirs[dir] {
if err := os.MkdirAll(dir, 0o750); err != nil {
return Revision{}, fmt.Errorf("create directory for %s: %w", f.Path, err)
}
dirs[dir] = true
}
if err := writeFileSync(target, contents[f.Path], fs.FileMode(f.Mode&0o777)); err != nil {
return Revision{}, err
}
}
for d := range dirs {
syncDir(d)
}
final := s.revisionPath(m.Revision)
if err := replaceDir(tmp, final); err != nil {
return Revision{}, err
}
syncDir(parent)
return Revision{Number: m.Revision, Dir: final, Files: slices.Clone(m.Files)}, nil
}
// LoadRevision reads a stored revision. Stored revisions were verified
// before they were written (ADR 0009), so only the manifest is parsed.
func (s *Store) LoadRevision(n uint64) (Revision, error) {
dir := s.revisionPath(n)
data, err := os.ReadFile(filepath.Join(dir, manifestFile)) //nolint:gosec // path built from the state directory
if err != nil {
return Revision{}, fmt.Errorf("load revision %d: %w", n, err)
}
var m bundle.Manifest
if err := json.Unmarshal(data, &m); err != nil {
return Revision{}, fmt.Errorf("load revision %d: %w", n, err)
}
if m.Revision != n {
return Revision{}, fmt.Errorf("load revision %d: manifest is for revision %d", n, m.Revision)
}
return Revision{Number: n, Dir: dir, Files: m.Files}, nil
}
// ListRevisions returns the stored revision numbers in ascending order.
func (s *Store) ListRevisions() ([]uint64, error) {
entries, err := os.ReadDir(filepath.Join(s.dir, revisionsDir))
if err != nil {
return nil, fmt.Errorf("list revisions: %w", err)
}
var out []uint64
for _, e := range entries {
if !e.IsDir() {
continue
}
n, err := strconv.ParseUint(e.Name(), 10, 64)
if err != nil || n == 0 || strconv.FormatUint(n, 10) != e.Name() {
continue
}
out = append(out, n)
}
slices.Sort(out)
return out, nil
}
// Prune keeps the newest keep revisions plus the protected ones, removes the
// others and cleans up leftovers of interrupted writes.
func (s *Store) Prune(keep int, protect ...uint64) error {
parent := filepath.Join(s.dir, revisionsDir)
entries, err := os.ReadDir(parent)
if err != nil {
return fmt.Errorf("prune revisions: %w", err)
}
for _, e := range entries {
if strings.HasPrefix(e.Name(), tmpPrefix) || strings.HasPrefix(e.Name(), oldPrefix) {
_ = os.RemoveAll(filepath.Join(parent, e.Name()))
}
}
revs, err := s.ListRevisions()
if err != nil {
return err
}
var errs []error
for i, n := range revs {
if len(revs)-i <= keep || slices.Contains(protect, n) {
continue
}
if err := os.RemoveAll(s.revisionPath(n)); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
// writeFileSync creates path with data and syncs it.
func writeFileSync(path string, data []byte, perm fs.FileMode) error {
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) //nolint:gosec // path built from the state directory
if err != nil {
return fmt.Errorf("write %s: %w", filepath.Base(path), err)
}
if err := finishFile(f, data, perm); err != nil {
return fmt.Errorf("write %s: %w", filepath.Base(path), err)
}
return nil
}
func finishFile(f *os.File, data []byte, perm fs.FileMode) error {
_, err := f.Write(data)
if err == nil {
err = f.Chmod(perm) // not subject to the umask
if errors.Is(err, errors.ErrUnsupported) {
err = nil
}
}
if err == nil {
err = f.Sync()
}
if cerr := f.Close(); err == nil {
err = cerr
}
return err
}
// writeFileAtomic replaces path with data: temporary file, fsync, rename,
// fsync of the directory.
func writeFileAtomic(target string, data []byte, perm fs.FileMode) error {
dir := filepath.Dir(target)
f, err := os.CreateTemp(dir, tmpPrefix+filepath.Base(target)+"-")
if err != nil {
return fmt.Errorf("write %s: %w", filepath.Base(target), err)
}
tmp := f.Name()
if err := finishFile(f, data, perm); err != nil {
_ = os.Remove(tmp)
return fmt.Errorf("write %s: %w", filepath.Base(target), err)
}
if err := os.Rename(tmp, target); err != nil {
_ = os.Remove(tmp)
return fmt.Errorf("write %s: %w", filepath.Base(target), err)
}
syncDir(dir)
return nil
}
// replaceDir moves the fully written directory tmp to final. An existing
// final directory is moved aside first and removed afterwards.
func replaceDir(tmp, final string) error {
parent := filepath.Dir(final)
var old string
if _, err := os.Stat(final); err == nil {
old = filepath.Join(parent, oldPrefix+filepath.Base(final)+"-"+randomSuffix())
if err := os.Rename(final, old); err != nil {
return fmt.Errorf("replace %s: %w", filepath.Base(final), err)
}
}
if err := os.Rename(tmp, final); err != nil {
if old != "" {
_ = os.Rename(old, final)
}
return fmt.Errorf("replace %s: %w", filepath.Base(final), err)
}
if old != "" {
_ = os.RemoveAll(old)
}
return nil
}
func randomSuffix() string {
var b [6]byte
_, _ = rand.Read(b[:])
return hex.EncodeToString(b[:])
}
// syncDir flushes directory metadata (renames). Errors are ignored because
// some platforms cannot sync directories; the data itself was synced.
func syncDir(dir string) {
d, err := os.Open(dir) //nolint:gosec // path built from the state directory
if err != nil {
return
}
_ = d.Sync()
_ = d.Close()
}
agent/core/store_test.go
+382
View File
@@ -0,0 +1,382 @@
// SPDX-License-Identifier: Apache-2.0
package core
import (
"crypto/ecdsa"
"crypto/ed25519"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"encoding/json"
"encoding/pem"
"errors"
"io/fs"
"os"
"path/filepath"
"runtime"
"slices"
"strings"
"testing"
"time"
"git.beugsiud.de/beug.k/DDI/internal/bundle"
)
func openTestStore(t *testing.T) *Store {
t.Helper()
s, err := OpenStore(t.TempDir())
if err != nil {
t.Fatal(err)
}
return s
}
// storeTestRevision verifies and stores a signed test bundle.
func storeTestRevision(t *testing.T, s *Store, key ed25519.PrivateKey, rev uint64) Revision {
t.Helper()
b := signBundle(t, key, testAgentID, KindDNS, rev)
contents := map[string][]byte{}
for _, a := range b.Artifacts {
contents[a.Path] = a.Content
}
m, err := bundle.Verify(key.Public().(ed25519.PublicKey), b.Manifest, b.Signature, contents,
bundle.Expect{AgentID: testAgentID, Role: string(KindDNS)})
if err != nil {
t.Fatal(err)
}
r, err := s.StoreRevision(m, b.Manifest, b.Signature, contents)
if err != nil {
t.Fatal(err)
}
return r
}
func TestOpenStoreEmpty(t *testing.T) {
if _, err := OpenStore(""); err == nil {
t.Error("empty state directory accepted")
}
}
func TestStoreRevision(t *testing.T) {
s := openTestStore(t)
_, key := newSigningKey(t)
r := storeTestRevision(t, s, key, 7)
if r.Number != 7 || r.Dir != filepath.Join(s.Dir(), "revisions", "7") || len(r.Files) != 2 {
t.Fatalf("revision = %+v", r)
}
data, err := os.ReadFile(filepath.Join(r.Dir, "zones", "db.example.com"))
if err != nil || !strings.Contains(string(data), "SOA ns 1 7") {
t.Fatalf("artifact = %q, %v", data, err)
}
if runtime.GOOS != "windows" {
fi, err := os.Stat(filepath.Join(r.Dir, "ddi-suite.conf"))
if err != nil || fi.Mode().Perm() != 0o640 {
t.Errorf("mode = %v, %v", fi.Mode(), err)
}
}
loaded, err := s.LoadRevision(7)
if err != nil || loaded.Number != 7 || loaded.Dir != r.Dir || len(loaded.Files) != 2 {
t.Fatalf("LoadRevision = %+v, %v", loaded, err)
}
// Storing the same number again replaces the directory.
if err := os.WriteFile(filepath.Join(r.Dir, "stale"), []byte("x"), 0o600); err != nil {
t.Fatal(err)
}
storeTestRevision(t, s, key, 7)
if _, err := os.Stat(filepath.Join(r.Dir, "stale")); !errors.Is(err, fs.ErrNotExist) {
t.Errorf("old directory content survived: %v", err)
}
if _, err := s.LoadRevision(8); err == nil {
t.Error("missing revision loaded")
}
entries, _ := os.ReadDir(filepath.Join(s.Dir(), "revisions"))
for _, e := range entries {
if strings.HasPrefix(e.Name(), ".") {
t.Errorf("leftover %s", e.Name())
}
}
}
func TestStoreRevisionReservedPath(t *testing.T) {
s := openTestStore(t)
m := &bundle.Manifest{Revision: 1, Files: []bundle.File{{Path: "manifest.json", Mode: 0o644}}}
if _, err := s.StoreRevision(m, []byte("{}"), []byte("sig"), map[string][]byte{"manifest.json": nil}); err == nil {
t.Error("reserved artifact path accepted")
}
if revs, _ := s.ListRevisions(); len(revs) != 0 {
t.Errorf("revisions = %v", revs)
}
}
func TestLoadRevisionMismatch(t *testing.T) {
s := openTestStore(t)
_, key := newSigningKey(t)
r := storeTestRevision(t, s, key, 3)
if err := os.Rename(r.Dir, filepath.Join(s.Dir(), "revisions", "4")); err != nil {
t.Fatal(err)
}
if _, err := s.LoadRevision(4); err == nil {
t.Error("manifest of another revision accepted")
}
}
func TestPrune(t *testing.T) {
s := openTestStore(t)
_, key := newSigningKey(t)
for rev := uint64(1); rev <= 15; rev++ {
storeTestRevision(t, s, key, rev)
}
for _, name := range []string{".tmp-123", ".old-5-abc", "notanumber", "007"} {
if err := os.Mkdir(filepath.Join(s.Dir(), "revisions", name), 0o750); err != nil {
t.Fatal(err)
}
}
if err := s.Prune(KeepRevisions, 2, 3); err != nil {
t.Fatal(err)
}
revs, err := s.ListRevisions()
if err != nil {
t.Fatal(err)
}
want := []uint64{2, 3, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
if !slices.Equal(revs, want) {
t.Errorf("revisions = %v, want %v", revs, want)
}
for _, name := range []string{".tmp-123", ".old-5-abc"} {
if _, err := os.Stat(filepath.Join(s.Dir(), "revisions", name)); !errors.Is(err, fs.ErrNotExist) {
t.Errorf("%s not removed", name)
}
}
if _, err := os.Stat(filepath.Join(s.Dir(), "revisions", "notanumber")); err != nil {
t.Error("foreign directory removed")
}
}
func TestStateRoundTrip(t *testing.T) {
s := openTestStore(t)
st, err := s.LoadState()
if err != nil || st.Applied != 0 {
t.Fatalf("empty state = %+v, %v", st, err)
}
now := time.Now().UTC().Truncate(time.Second)
st = &State{
Applied: 5, LastGood: 4, LastUnsuccessful: 5, LeaseSequence: 9,
LastResult: &DeploymentReport{Revision: 5, Outcome: OutcomeFailed, Trigger: TriggerPush, StartedAt: now, FinishedAt: now},
PendingReports: []DeploymentReport{{Revision: 5, Outcome: OutcomeFailed, DeploymentID: "d"}},
}
if err := s.SaveState(st); err != nil {
t.Fatal(err)
}
got, err := s.LoadState()
if err != nil {
t.Fatal(err)
}
if got.Applied != 5 || got.LastGood != 4 || got.LeaseSequence != 9 || got.LastResult.Outcome != OutcomeFailed ||
len(got.PendingReports) != 1 || got.PendingReports[0].DeploymentID != "d" {
t.Errorf("state = %+v", got)
}
// Human-readable JSON with the documented keys.
raw, _ := os.ReadFile(filepath.Join(s.Dir(), "state.json"))
var m map[string]any
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
for _, k := range []string{"applied_revision", "last_good_revision", "pending_reports", "lease_sequence"} {
if _, ok := m[k]; !ok {
t.Errorf("state.json lacks %s", k)
}
}
entries, _ := os.ReadDir(s.Dir())
for _, e := range entries {
if strings.HasPrefix(e.Name(), ".tmp-") {
t.Errorf("leftover temporary file %s", e.Name())
}
}
if err := os.WriteFile(filepath.Join(s.Dir(), "state.json"), []byte("{broken"), 0o600); err != nil {
t.Fatal(err)
}
if _, err := s.LoadState(); err == nil {
t.Error("broken state.json accepted")
}
}
func TestLock(t *testing.T) {
s := openTestStore(t)
unlock, err := s.Lock()
if err != nil {
t.Fatal(err)
}
if runtime.GOOS == "linux" {
if _, err := s.Lock(); err == nil || !strings.Contains(err.Error(), "locked") {
t.Errorf("second lock = %v", err)
}
}
unlock()
unlock2, err := s.Lock()
if err != nil {
t.Fatalf("lock after unlock: %v", err)
}
unlock2()
}
// testIdentity creates an identity signed by pki.
func testIdentity(t *testing.T, pki *testPKI) (*Identity, *ecdsa.PrivateKey) {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
cert, err := pki.issueClient(&key.PublicKey, testAgentID, time.Now().Add(-time.Hour), time.Now().Add(time.Hour))
if err != nil {
t.Fatal(err)
}
pub, _ := newSigningKey(t)
return &Identity{
AgentID: testAgentID, Role: KindDNS, Server: "ddi.example.com:8444", ServerName: "ddi.example.com",
EnrolledAt: time.Now().UTC().Truncate(time.Second), CA: pki.ca, Certificate: keyPair(key, cert), BundleKey: pub,
}, key
}
func TestIdentityRoundTrip(t *testing.T) {
s := openTestStore(t)
if _, err := s.LoadIdentity(); !errors.Is(err, ErrNotEnrolled) {
t.Fatalf("LoadIdentity = %v", err)
}
pki := newTestPKI(t)
id, key := testIdentity(t, pki)
if err := s.SaveIdentity(id, key, false); err != nil {
t.Fatal(err)
}
if !s.Enrolled() {
t.Error("not enrolled after SaveIdentity")
}
got, err := s.LoadIdentity()
if err != nil {
t.Fatal(err)
}
if got.AgentID != id.AgentID || got.Server != id.Server || got.ServerName != id.ServerName || got.Role != KindDNS ||
!got.CA.Equal(pki.ca) || !got.BundleKey.Equal(id.BundleKey) || !got.Certificate.Leaf.Equal(id.Certificate.Leaf) ||
!got.EnrolledAt.Equal(id.EnrolledAt) {
t.Errorf("identity = %+v", got)
}
if runtime.GOOS != "windows" {
fi, err := os.Stat(filepath.Join(s.Dir(), "identity", "key.pem"))
if err != nil || fi.Mode().Perm() != 0o600 {
t.Errorf("key.pem mode = %v, %v", fi.Mode(), err)
}
di, err := os.Stat(filepath.Join(s.Dir(), "identity"))
if err != nil || di.Mode().Perm() != 0o700 {
t.Errorf("identity dir mode = %v, %v", di.Mode(), err)
}
}
// Already enrolled.
if err := s.SaveIdentity(id, key, false); !errors.Is(err, errAlreadyEnrolled) {
t.Errorf("second SaveIdentity = %v", err)
}
id2, key2 := testIdentity(t, pki)
id2.Server = "other.example.com:8444"
if err := s.SaveIdentity(id2, key2, true); err != nil {
t.Fatal(err)
}
if got, err := s.LoadIdentity(); err != nil || got.Server != "other.example.com:8444" {
t.Errorf("forced identity = %+v, %v", got, err)
}
entries, _ := os.ReadDir(s.Dir())
for _, e := range entries {
if strings.HasPrefix(e.Name(), ".") {
t.Errorf("leftover %s", e.Name())
}
}
}
func TestSaveCertificateAndRecovery(t *testing.T) {
s := openTestStore(t)
pki := newTestPKI(t)
id, key := testIdentity(t, pki)
if err := s.SaveIdentity(id, key, false); err != nil {
t.Fatal(err)
}
newKey, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
newCert, err := pki.issueClient(&newKey.PublicKey, testAgentID, time.Now().Add(-time.Minute), time.Now().Add(2*time.Hour))
if err != nil {
t.Fatal(err)
}
if err := s.SaveCertificate(newKey, newCert); err != nil {
t.Fatal(err)
}
got, err := s.LoadIdentity()
if err != nil || !got.Certificate.Leaf.Equal(newCert) {
t.Fatalf("renewed identity = %v", err)
}
// Simulate a crash between the two renames: key.pem is new, cert.pem
// old, cert.pem.new holds the matching certificate.
key3, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
cert3, err := pki.issueClient(&key3.PublicKey, testAgentID, time.Now().Add(-time.Minute), time.Now().Add(3*time.Hour))
if err != nil {
t.Fatal(err)
}
keyPEM, _ := encodeKey(key3)
dir := filepath.Join(s.Dir(), "identity")
if err := os.WriteFile(filepath.Join(dir, "key.pem"), keyPEM, 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "cert.pem.new"), pemCert(cert3), 0o600); err != nil {
t.Fatal(err)
}
got, err = s.LoadIdentity()
if err != nil || !got.Certificate.Leaf.Equal(cert3) {
t.Fatalf("recovered identity: %v", err)
}
if _, err := os.Stat(filepath.Join(dir, "cert.pem.new")); !errors.Is(err, fs.ErrNotExist) {
t.Error("cert.pem.new left behind")
}
// A mismatch without a recoverable certificate is an error.
key4, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
keyPEM, _ = encodeKey(key4)
if err := os.WriteFile(filepath.Join(dir, "key.pem"), keyPEM, 0o600); err != nil {
t.Fatal(err)
}
if _, err := s.LoadIdentity(); err == nil {
t.Error("mismatched key pair accepted")
}
}
func pemCert(c *x509.Certificate) []byte {
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: c.Raw})
}
func TestCheckIssued(t *testing.T) {
pki := newTestPKI(t)
key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
cert, err := pki.issueClient(&key.PublicKey, testAgentID, time.Now().Add(-time.Minute), time.Now().Add(time.Hour))
if err != nil {
t.Fatal(err)
}
if _, err := checkIssued(cert.Raw, pki.ca, testAgentID, key); err != nil {
t.Fatalf("checkIssued = %v", err)
}
if _, err := checkIssued(cert.Raw, pki.ca, "11111111-2222-4333-8444-555555555555", key); err == nil {
t.Error("other agent ID accepted")
}
other, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if _, err := checkIssued(cert.Raw, pki.ca, testAgentID, other); err == nil {
t.Error("certificate for another key accepted")
}
if _, err := checkIssued(cert.Raw, newTestPKI(t).ca, testAgentID, key); err == nil {
t.Error("certificate of another CA accepted")
}
if _, err := checkIssued([]byte("junk"), pki.ca, testAgentID, key); err == nil {
t.Error("junk accepted")
}
// A server clock slightly ahead is tolerated.
future, err := pki.issueClient(&key.PublicKey, testAgentID, time.Now().Add(2*time.Minute), time.Now().Add(time.Hour))
if err != nil {
t.Fatal(err)
}
if _, err := checkIssued(future.Raw, pki.ca, testAgentID, key); err != nil {
t.Errorf("future NotBefore: %v", err)
}
}