DDI/agent/core/agent_test.go

// SPDX-License-Identifier: Apache-2.0

package core

import (
	"context"
	"errors"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"testing"
	"time"
)

func TestApplySucceeded(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.addBundle(t, 1)
	f.pull(1)

	r := f.lastResult(t)
	if r.Outcome != OutcomeSucceeded || r.Revision != 1 || r.ActiveRevision != 1 || r.Trigger != TriggerPull {
		t.Fatalf("result = %+v", r)
	}
	if r.StartedAt.IsZero() || r.FinishedAt.Before(r.StartedAt) {
		t.Errorf("timestamps = %v .. %v", r.StartedAt, r.FinishedAt)
	}
	st := f.agent.state()
	if st.Applied != 1 || st.LastGood != 1 || st.LastUnsuccessful != 0 {
		t.Errorf("state = %+v", st)
	}
	if got, want := f.role.callLog(), []string{"validate 1", "activate 1 prev none"}; !slices.Equal(got, want) {
		t.Errorf("calls = %q, want %q", got, want)
	}
	// Stored revision: manifest, signature and artifacts.
	dir := filepath.Join(f.store.Dir(), "revisions", "1")
	for _, name := range []string{"manifest.json", "manifest.sig", "ddi-suite.conf", filepath.Join("zones", "db.example.com")} {
		if _, err := os.Stat(filepath.Join(dir, name)); err != nil {
			t.Errorf("stored revision lacks %s: %v", name, err)
		}
	}
	// The report is persisted until it is delivered.
	saved, err := f.store.LoadState()
	if err != nil {
		t.Fatal(err)
	}
	if len(saved.PendingReports) != 1 || saved.Applied != 1 {
		t.Fatalf("saved state = %+v", saved)
	}
	f.agent.flushReports(context.Background())
	if rep := f.src.reported(); len(rep) != 1 || rep[0].Outcome != OutcomeSucceeded {
		t.Fatalf("reports = %+v", rep)
	}
	if saved, _ := f.store.LoadState(); len(saved.PendingReports) != 0 {
		t.Errorf("pending reports after delivery = %d", len(saved.PendingReports))
	}

	// Next revision: Activate gets the previous one.
	f.addBundle(t, 2)
	f.role.resetCalls()
	f.pull(2)
	if got, want := f.role.callLog(), []string{"validate 2", "activate 2 prev 1"}; !slices.Equal(got, want) {
		t.Errorf("calls = %q, want %q", got, want)
	}
	if st := f.agent.state(); st.Applied != 2 || st.LastGood != 2 {
		t.Errorf("state = %+v", st)
	}
}

func TestApplyRejectedOnValidate(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.addBundle(t, 1)
	f.addBundle(t, 2)
	f.pull(1)
	f.role.validateErr[2] = errors.New("zone example.com: bad SOA")
	f.role.resetCalls()
	f.pull(2)

	r := f.lastResult(t)
	if r.Outcome != OutcomeRejected || r.ActiveRevision != 1 || !strings.Contains(r.Message, "bad SOA") {
		t.Fatalf("result = %+v", r)
	}
	if got := f.role.callLog(); !slices.Equal(got, []string{"validate 2"}) {
		t.Errorf("calls = %q", got)
	}
	st := f.agent.state()
	if st.Applied != 1 || st.LastGood != 1 || st.LastUnsuccessful != 2 {
		t.Errorf("state = %+v", st)
	}

	// No automatic retry of the same revision.
	f.role.resetCalls()
	f.src.fetches = 0
	f.pull(2)
	if len(f.role.callLog()) != 0 || f.src.fetches != 0 {
		t.Errorf("rejected revision was retried: calls %q, fetches %d", f.role.callLog(), f.src.fetches)
	}

	// An explicit push retries it.
	delete(f.role.validateErr, 2)
	f.push(2, "dep-7")
	r = f.lastResult(t)
	if r.Outcome != OutcomeSucceeded || r.DeploymentID != "dep-7" || r.Trigger != TriggerPush {
		t.Fatalf("push result = %+v", r)
	}

	// A newer revision is pulled again.
	f.addBundle(t, 3)
	f.pull(3)
	if st := f.agent.state(); st.Applied != 3 {
		t.Errorf("state = %+v", st)
	}
}

func TestRollbackOnActivateError(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.addBundle(t, 1)
	f.addBundle(t, 2)
	f.pull(1)
	f.role.activateErr[2] = errors.New("rndc reconfig failed")
	f.role.resetCalls()
	f.pull(2)

	r := f.lastResult(t)
	if r.Outcome != OutcomeRolledBack || r.ActiveRevision != 1 || !strings.Contains(r.Message, "rndc reconfig failed") {
		t.Fatalf("result = %+v", r)
	}
	want := []string{"validate 2", "activate 2 prev 1", "activate 1 prev 2"}
	if got := f.role.callLog(); !slices.Equal(got, want) {
		t.Errorf("calls = %q, want %q", got, want)
	}
	st := f.agent.state()
	if st.Applied != 1 || st.LastGood != 1 || st.LastUnsuccessful != 2 {
		t.Errorf("state = %+v", st)
	}
	// Not retried by a later poll.
	f.role.resetCalls()
	f.pull(2)
	if len(f.role.callLog()) != 0 {
		t.Errorf("rolled back revision retried: %q", f.role.callLog())
	}
}

func TestRollbackOnHealthFailure(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.addBundle(t, 1)
	f.addBundle(t, 2)
	f.pull(1)
	f.role.unhealthy[2] = true
	f.pull(2)

	r := f.lastResult(t)
	if r.Outcome != OutcomeRolledBack || r.ActiveRevision != 1 {
		t.Fatalf("result = %+v", r)
	}
	if !strings.Contains(r.Message, "serial mismatch") || !strings.Contains(r.Message, "rolled back to revision 1") {
		t.Errorf("message = %q", r.Message)
	}
	if n := f.role.healthCalls[2]; n < 2 {
		t.Errorf("health of revision 2 checked %d times, want retries", n)
	}
	if f.role.active != 1 {
		t.Errorf("active = %d", f.role.active)
	}
	if st := f.agent.state(); st.Applied != 1 || st.LastGood != 1 {
		t.Errorf("state = %+v", st)
	}
}

func TestFailedWithoutLastGood(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.addBundle(t, 1)
	f.role.unhealthy[1] = true
	f.pull(1)

	r := f.lastResult(t)
	if r.Outcome != OutcomeFailed || r.ActiveRevision != 1 || !strings.Contains(r.Message, "no last good revision") {
		t.Fatalf("result = %+v", r)
	}
	st := f.agent.state()
	if st.Applied != 1 || st.LastGood != 0 || st.LastUnsuccessful != 1 {
		t.Errorf("state = %+v", st)
	}

	// Activation error without last good: the previous state stays.
	g := newFixture(t, KindDNS)
	g.addBundle(t, 1)
	g.role.activateErr[1] = errors.New("boom")
	g.pull(1)
	if r := g.lastResult(t); r.Outcome != OutcomeFailed || r.ActiveRevision != 0 {
		t.Fatalf("result = %+v", r)
	}
	if st := g.agent.state(); st.Applied != 0 {
		t.Errorf("state = %+v", st)
	}
}

func TestFailedWhenRollbackFails(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.addBundle(t, 1)
	f.addBundle(t, 2)
	f.pull(1)
	f.role.unhealthy[2] = true
	f.role.activateErr[1] = errors.New("restore failed")
	f.pull(2)

	r := f.lastResult(t)
	if r.Outcome != OutcomeFailed || r.ActiveRevision != 2 || !strings.Contains(r.Message, "rollback to revision 1 failed: restore failed") {
		t.Fatalf("result = %+v", r)
	}
	if st := f.agent.state(); st.Applied != 2 || st.LastGood != 1 {
		t.Errorf("state = %+v", st)
	}
}

func TestFailedWhenRollbackUnhealthy(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.addBundle(t, 1)
	f.addBundle(t, 2)
	f.pull(1)
	f.role.unhealthy[1] = true
	f.role.unhealthy[2] = true
	f.pull(2)

	r := f.lastResult(t)
	if r.Outcome != OutcomeFailed || r.ActiveRevision != 1 || !strings.Contains(r.Message, "restored but unhealthy") {
		t.Fatalf("result = %+v", r)
	}
	if st := f.agent.state(); st.Applied != 1 || st.LastGood != 1 || st.LastUnsuccessful != 2 {
		t.Errorf("state = %+v", st)
	}
}

func TestReplayProtection(t *testing.T) {
	f := newFixture(t, KindDNS)
	for rev := uint64(1); rev <= 3; rev++ {
		f.addBundle(t, rev)
	}
	f.pull(3)
	if st := f.agent.state(); st.Applied != 3 {
		t.Fatalf("state = %+v", st)
	}

	// Older revisions: a pull is ignored, a push is rejected.
	f.role.resetCalls()
	f.pull(2)
	if len(f.role.callLog()) != 0 || len(f.agent.state().PendingReports) != 0 {
		t.Errorf("older pull was processed")
	}
	f.push(2, "dep-old")
	r := f.lastResult(t)
	if r.Outcome != OutcomeRejected || r.Revision != 2 || !strings.Contains(r.Message, "older") || r.ActiveRevision != 3 {
		t.Fatalf("result = %+v", r)
	}
	if len(f.role.callLog()) != 0 {
		t.Errorf("role called for an older revision: %q", f.role.callLog())
	}

	// The server answers revision 4 with the (validly signed) bundle of 2.
	f.src.bundles[4] = f.src.bundles[2]
	f.pull(4)
	if r := f.lastResult(t); r.Outcome != OutcomeRejected || !strings.Contains(r.Message, "older than 4") {
		t.Fatalf("result = %+v", r)
	}

	// An explicit push of the applied revision re-applies it.
	f.push(3, "dep-again")
	if r := f.lastResult(t); r.Outcome != OutcomeSucceeded || r.Revision != 3 {
		t.Fatalf("result = %+v", r)
	}
	if st := f.agent.state(); st.Applied != 3 {
		t.Errorf("state = %+v", st)
	}
}

func TestRejectedBundles(t *testing.T) {
	cases := map[string]func(f *fixture) *Bundle{
		"bad signature": func(f *fixture) *Bundle {
			b := signBundle(t, f.signer, testAgentID, KindDNS, 1)
			b.Signature[0] ^= 1
			return b
		},
		"other agent": func(f *fixture) *Bundle {
			return signBundle(t, f.signer, "11111111-2222-4333-8444-555555555555", KindDNS, 1)
		},
		"other role": func(f *fixture) *Bundle { return signBundle(t, f.signer, testAgentID, KindDHCP, 1) },
		"other key": func(*fixture) *Bundle {
			_, other := newSigningKey(t)
			return signBundle(t, other, testAgentID, KindDNS, 1)
		},
		"tampered content": func(f *fixture) *Bundle {
			b := signBundle(t, f.signer, testAgentID, KindDNS, 1)
			b.Artifacts[0].Content = []byte("evil")
			return b
		},
		"duplicate artifact": func(f *fixture) *Bundle {
			b := signBundle(t, f.signer, testAgentID, KindDNS, 1)
			b.Artifacts = append(b.Artifacts, b.Artifacts[0])
			return b
		},
	}
	for name, mk := range cases {
		t.Run(name, func(t *testing.T) {
			f := newFixture(t, KindDNS)
			f.src.bundles[1] = mk(f)
			f.pull(1)
			r := f.lastResult(t)
			if r.Outcome != OutcomeRejected || !strings.Contains(r.Message, "verification failed") {
				t.Fatalf("result = %+v", r)
			}
			if len(f.role.callLog()) != 0 {
				t.Errorf("role called: %q", f.role.callLog())
			}
			if revs, _ := f.store.ListRevisions(); len(revs) != 0 {
				t.Errorf("rejected bundle stored: %v", revs)
			}
		})
	}
}

func TestFetchError(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.src.fetchErr = errors.New("unavailable")
	f.pull(1)
	if st := f.agent.state(); st.LastResult != nil || st.LastUnsuccessful != 0 {
		t.Fatalf("pull fetch error recorded: %+v", st)
	}
	f.push(1, "dep-1")
	r := f.lastResult(t)
	if r.Outcome != OutcomeRejected || !strings.Contains(r.Message, "fetch bundle: unavailable") {
		t.Fatalf("result = %+v", r)
	}
	// A transport problem does not block a later pull.
	f.src.fetchErr = nil
	f.addBundle(t, 1)
	f.pull(1)
	if st := f.agent.state(); st.Applied != 1 {
		t.Errorf("state = %+v", st)
	}
}

func TestShutdownBeforeValidation(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.addBundle(t, 1)
	ctx, cancel := context.WithCancel(context.Background())
	f.src.onFetch = cancel
	f.agent.handle(ctx, applyRequest{revision: 1, trigger: TriggerPull})
	if len(f.role.callLog()) != 0 {
		t.Errorf("role called after shutdown: %q", f.role.callLog())
	}
	if st := f.agent.state(); st.LastResult != nil || st.Applied != 0 {
		t.Errorf("state = %+v", st)
	}
}

func TestReportQueueWhileServerDown(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.src.setReportErr(errors.New("server down"))
	f.addBundle(t, 1)
	f.addBundle(t, 2)
	f.pull(1)
	f.role.validateErr[2] = errors.New("bad")
	f.pull(2)
	f.agent.flushReports(context.Background())
	if n := len(f.src.reported()); n != 0 {
		t.Fatalf("%d reports delivered while down", n)
	}

	// After a restart the queue is still there and is sent in order.
	g := newFixtureWithStore(t, KindDNS, f.store, f.signer)
	if st, _ := g.store.LoadState(); len(st.PendingReports) != 2 {
		t.Fatalf("pending reports after restart = %d", len(st.PendingReports))
	}
	g.agent.flushReports(context.Background())
	rep := g.src.reported()
	if len(rep) != 2 || rep[0].Revision != 1 || rep[1].Revision != 2 || rep[1].Outcome != OutcomeRejected {
		t.Fatalf("reports = %+v", rep)
	}
	if st, _ := g.store.LoadState(); len(st.PendingReports) != 0 {
		t.Errorf("pending after delivery = %d", len(st.PendingReports))
	}
}

func TestPendingReportsBounded(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.src.setReportErr(errors.New("down"))
	for i := 0; i < maxPendingReports+5; i++ {
		f.push(1, "dep") // fetch fails: rejected report each time
	}
	st, err := f.store.LoadState()
	if err != nil {
		t.Fatal(err)
	}
	if len(st.PendingReports) != maxPendingReports {
		t.Errorf("pending = %d, want %d", len(st.PendingReports), maxPendingReports)
	}
}

func TestPruneKeepsAppliedAndLastGood(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.addBundle(t, 1)
	f.pull(1)
	for rev := uint64(2); rev <= 14; rev++ {
		f.addBundle(t, rev)
		f.role.validateErr[rev] = errors.New("bad")
		f.push(rev, "")
	}
	revs, err := f.store.ListRevisions()
	if err != nil {
		t.Fatal(err)
	}
	if !slices.Contains(revs, 1) || len(revs) != KeepRevisions+1 || revs[len(revs)-1] != 14 {
		t.Errorf("revisions = %v", revs)
	}
}

func TestEnqueue(t *testing.T) {
	f := newFixture(t, KindDNS)
	a := f.agent
	a.enqueue(applyRequest{revision: 5, trigger: TriggerPull})
	a.enqueue(applyRequest{revision: 4, trigger: TriggerPull}) // dropped: 5 queued
	a.enqueue(applyRequest{revision: 5, trigger: TriggerPull}) // dropped: duplicate
	a.enqueue(applyRequest{revision: 6, trigger: TriggerPush, deploymentID: "d"})
	a.enqueue(applyRequest{revision: 7, trigger: TriggerPull})
	var got []applyRequest
	for {
		r, ok := a.dequeue()
		if !ok {
			break
		}
		got = append(got, r)
	}
	want := []applyRequest{{revision: 6, trigger: TriggerPush, deploymentID: "d"}, {revision: 7, trigger: TriggerPull}}
	if !slices.Equal(got, want) {
		t.Errorf("queue = %+v, want %+v", got, want)
	}
}

func TestDesiredAndPush(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.agent.Desired(0, true)
	f.agent.Desired(3, false) // manual mode
	if _, ok := f.agent.dequeue(); ok {
		t.Fatal("request queued for revision 0 or manual mode")
	}
	f.agent.Desired(3, true)
	f.agent.Push(4, "dep")
	r1, _ := f.agent.dequeue()
	r2, _ := f.agent.dequeue()
	if r1.revision != 3 || r1.trigger != TriggerPull || r2.revision != 4 || r2.deploymentID != "dep" {
		t.Errorf("requests = %+v %+v", r1, r2)
	}
}

func TestRunAppliesAndReports(t *testing.T) {
	f := newFixture(t, KindDHCP)
	f.addBundle(t, 1)
	f.role.setLeases([]Lease{{IP: mustAddr("10.0.0.5"), MAC: "aa:bb:cc:dd:ee:ff", State: LeaseActive}})
	f.src.run = func(ctx context.Context, h SourceHandler) error {
		h.Connected()
		h.Desired(1, true)
		<-ctx.Done()
		return ctx.Err()
	}
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() { done <- f.agent.Run(ctx) }()
	waitFor(t, "deployment report", func() bool { return len(f.src.reported()) == 1 })
	waitFor(t, "lease snapshot", func() bool { return len(f.src.leaseBatches()) > 0 })
	if b := f.src.leaseBatches()[0]; !b.FullSync || len(b.Leases) != 1 {
		t.Errorf("first lease batch = %+v", b)
	}
	cancel()
	select {
	case err := <-done:
		if err != nil {
			t.Errorf("Run = %v", err)
		}
	case <-time.After(10 * time.Second):
		t.Fatal("Run did not return")
	}
}

func TestRunSourceError(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.src.run = func(context.Context, SourceHandler) error { return errors.New("broken source") }
	if err := f.agent.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "broken source") {
		t.Errorf("Run = %v", err)
	}
}

func TestStatus(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.addBundle(t, 1)
	f.pull(1)
	st := f.agent.Status(context.Background())
	if st.Applied != 1 || st.LastGood != 1 || st.Health.State != HealthOK || st.ServiceVersion != "Fake 1.0" || st.CheckedAt.IsZero() {
		t.Errorf("status = %+v", st)
	}
	// While the role is busy the cached result is returned.
	f.role.unhealthy[1] = true
	f.agent.roleMu.Lock()
	cached := f.agent.Status(context.Background())
	f.agent.roleMu.Unlock()
	if cached.Health.State != HealthOK {
		t.Errorf("cached status = %+v", cached)
	}
	if fresh := f.agent.Status(context.Background()); fresh.Health.State != HealthFailed {
		t.Errorf("fresh status = %+v", fresh)
	}
}

func TestNewAgentLoadsAppliedRevision(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.addBundle(t, 1)
	f.pull(1)
	g := newFixtureWithStore(t, KindDNS, f.store, f.signer)
	if g.agent.current == nil || g.agent.current.Number != 1 || len(g.agent.current.Files) != 2 {
		t.Fatalf("current = %+v", g.agent.current)
	}
	// Role mismatch.
	id := &Identity{AgentID: testAgentID}
	if _, err := NewAgent(KindDHCP, testConfig(f.store.Dir()), newFakeRole(KindDNS), f.store, id, f.src, testLogger()); err == nil {
		t.Error("role mismatch accepted")
	}
}

func TestDescribeHealthAndTruncate(t *testing.T) {
	h := Health{State: HealthDegraded, Message: "slow", Checks: []HealthCheck{
		{Name: "a", State: HealthOK}, {Name: "b", State: HealthFailed, Message: "down"}, {Name: "c", State: HealthUnknown},
	}}
	if got, want := describeHealth(h), "degraded: slow; b: failed (down); c: unknown"; got != want {
		t.Errorf("describeHealth = %q, want %q", got, want)
	}
	if got := truncate("short", 10); got != "short" {
		t.Errorf("truncate = %q", got)
	}
	long := strings.Repeat("ä", 100) // 200 bytes
	got := truncate(long, 50)
	if len(got) > 50 || !strings.HasSuffix(got, "[truncated]") || !strings.HasPrefix(got, "ää") {
		t.Errorf("truncate = %q (%d bytes)", got, len(got))
	}
	for _, r := range strings.TrimSuffix(got, " [truncated]") {
		if r != 'ä' {
			t.Fatalf("truncate split a rune: %q", got)
		}
	}
}