DDI/agent/core/push_test.go

// SPDX-License-Identifier: Apache-2.0

package core

import (
	"context"
	"errors"
	"fmt"
	"slices"
	"strings"
	"testing"
	"time"

	"google.golang.org/grpc/codes"

	agentv1 "git.beugsiud.de/beug.k/DDI/proto/gen/ddi/agent/v1"
)

// Tests for push deployments of docs/adr/0022-ausrollen-pull-push-m4.md.

// countCalls counts the role calls that start with prefix.
func (r *fakeRole) countCalls(prefix string) int {
	n := 0
	for _, c := range r.callLog() {
		if strings.HasPrefix(c, prefix) {
			n++
		}
	}
	return n
}

// activations counts the activations of a revision.
func (r *fakeRole) activations(rev uint64) int {
	return r.countCalls(fmt.Sprintf("activate %d ", rev))
}

// validations counts the validations of a revision.
func (r *fakeRole) validations(rev uint64) int {
	return r.countCalls(fmt.Sprintf("validate %d", rev))
}

func (f *fixture) pending() []DeploymentReport {
	f.agent.mu.Lock()
	defer f.agent.mu.Unlock()
	return slices.Clone(f.agent.st.PendingReports)
}

func TestPushAppliedOncePerDeployment(t *testing.T) {
	ctx := context.Background()
	f := newFixture(t, KindDNS)
	f.addBundle(t, 1)
	f.push(1, "dep-1")
	if n := f.role.activations(1); n != 1 {
		t.Fatalf("activations of revision 1 = %d", n)
	}

	// The server resends the open push (reconnect, polling) while the
	// report is still queued: not applied again, the queued report is
	// delivered once.
	f.push(1, "dep-1")
	if n := f.role.activations(1); n != 1 {
		t.Errorf("resent push applied again (%d activations)", n)
	}
	if p := f.pending(); len(p) != 1 || p[0].DeploymentID != "dep-1" || p[0].Outcome != OutcomeSucceeded {
		t.Fatalf("pending reports = %+v", p)
	}
	f.agent.flushReports(ctx)
	if r := f.src.reported(); len(r) != 1 {
		t.Fatalf("reports = %+v", r)
	}

	// Resent after the report was delivered (the server did not record
	// it): the remembered report is delivered again.
	f.push(1, "dep-1")
	if n := f.role.activations(1); n != 1 {
		t.Errorf("resent push applied again (%d activations)", n)
	}
	f.agent.flushReports(ctx)
	r := f.src.reported()
	if len(r) != 2 || r[1].DeploymentID != "dep-1" || r[1].Outcome != OutcomeSucceeded ||
		!r[1].StartedAt.Equal(r[0].StartedAt) || r[1].Trigger != TriggerPush {
		t.Fatalf("reports = %+v", r)
	}

	// A new deployment of the same revision is applied again.
	f.push(1, "dep-2")
	if n := f.role.activations(1); n != 2 {
		t.Errorf("activations of revision 1 = %d, want 2", n)
	}

	// Pushes without a deployment ID are never deduplicated (M3 behavior).
	f.push(1, "")
	f.push(1, "")
	if n := f.role.activations(1); n != 4 {
		t.Errorf("activations of revision 1 = %d, want 4", n)
	}
}

func TestRejectedPushNotRetriedForSameDeployment(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.addBundle(t, 1)
	f.pull(1)
	f.addBundle(t, 2)
	f.role.mu.Lock()
	f.role.validateErr[2] = errors.New("bad zone")
	f.role.mu.Unlock()
	f.push(2, "dep-3")
	if r := f.lastResult(t); r.Outcome != OutcomeRejected || r.DeploymentID != "dep-3" {
		t.Fatalf("result = %+v", r)
	}
	f.role.mu.Lock()
	delete(f.role.validateErr, 2)
	f.role.mu.Unlock()

	f.push(2, "dep-3")
	if n := f.role.validations(2); n != 1 {
		t.Errorf("validations of revision 2 = %d, want 1", n)
	}
	if st := f.agent.state(); st.Applied != 1 {
		t.Errorf("applied = %d", st.Applied)
	}
	// A new push of the same revision retries it.
	f.push(2, "dep-4")
	if r := f.lastResult(t); r.Outcome != OutcomeSucceeded || r.DeploymentID != "dep-4" {
		t.Errorf("result = %+v", r)
	}
}

func TestHandledDeploymentsPersistedAndBounded(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.addBundle(t, 1)
	for i := range maxHandledDeployments + 5 {
		f.push(1, fmt.Sprintf("dep-%d", i))
	}
	st, err := f.store.LoadState()
	if err != nil {
		t.Fatal(err)
	}
	h := st.HandledDeployments
	if len(h) != maxHandledDeployments || h[0].DeploymentID != "dep-5" ||
		h[len(h)-1].DeploymentID != fmt.Sprintf("dep-%d", maxHandledDeployments+4) {
		t.Fatalf("handled deployments = %+v", h)
	}

	// After a restart, resent pushes are still recognized; forgotten ones
	// are applied again.
	g := newFixtureWithStore(t, KindDNS, f.store, f.signer)
	g.addBundle(t, 1)
	g.push(1, fmt.Sprintf("dep-%d", maxHandledDeployments+4))
	if n := g.role.activations(1); n != 0 {
		t.Errorf("remembered deployment applied again after restart (%d activations)", n)
	}
	g.push(1, "dep-0")
	if n := g.role.activations(1); n != 1 {
		t.Errorf("forgotten deployment: %d activations, want 1", n)
	}
}

func TestRememberDeployment(t *testing.T) {
	var st State
	st.rememberDeployment(DeploymentReport{Revision: 1}) // pulls are not remembered
	st.rememberDeployment(DeploymentReport{DeploymentID: "a", Revision: 1, Message: strings.Repeat("x", 5000)})
	st.rememberDeployment(DeploymentReport{DeploymentID: "b", Revision: 2})
	st.rememberDeployment(DeploymentReport{DeploymentID: "a", Revision: 3})
	if len(st.HandledDeployments) != 2 || st.HandledDeployments[0].DeploymentID != "b" ||
		st.HandledDeployments[1].Revision != 3 {
		t.Fatalf("handled = %+v", st.HandledDeployments)
	}
	st.rememberDeployment(DeploymentReport{DeploymentID: "c", Message: strings.Repeat("y", 5000)})
	if r, ok := st.handledDeployment("c"); !ok || len(r.Message) > handledMessageLen {
		t.Errorf("handled c = %v, message length %d", ok, len(r.Message))
	}
	if _, ok := st.handledDeployment(""); ok {
		t.Error("empty deployment ID found")
	}
	if _, ok := st.handledDeployment("x"); ok {
		t.Error("unknown deployment ID found")
	}
}

func TestEnqueueDeduplicatesPushes(t *testing.T) {
	f := newFixture(t, KindDNS)
	a := f.agent
	a.enqueue(applyRequest{revision: 3, trigger: TriggerPush, deploymentID: "d"})
	a.enqueue(applyRequest{revision: 3, trigger: TriggerPush, deploymentID: "d"})
	a.enqueue(applyRequest{revision: 3, trigger: TriggerPush, deploymentID: "e"})
	a.enqueue(applyRequest{revision: 3, trigger: TriggerPush})
	a.enqueue(applyRequest{revision: 3, trigger: TriggerPush})
	var got []string
	for {
		r, ok := a.dequeue()
		if !ok {
			break
		}
		got = append(got, r.deploymentID)
	}
	if want := []string{"d", "e", "", ""}; !slices.Equal(got, want) {
		t.Errorf("queued deployments = %q, want %q", got, want)
	}
}

func TestRefusedReportIsDropped(t *testing.T) {
	f := newFixture(t, KindDNS)
	f.addBundle(t, 1)
	f.addBundle(t, 2)
	f.pull(1)
	f.push(2, "dep-gone")
	f.src.setReportErr(fmt.Errorf("%w: deployment closed", ErrReportRefused))
	f.agent.flushReports(context.Background())
	if p := f.pending(); len(p) != 0 {
		t.Errorf("refused reports still queued: %+v", p)
	}
}

func TestServerSourceReportRefused(t *testing.T) {
	srv := newFakeServer(t, KindDNS)
	addr := srv.start(t)
	store, id := enrollWith(t, srv, addr)
	src := NewServerSource(store, id, "h", testLogger())
	defer src.closeConn()
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	rep := DeploymentReport{DeploymentID: "d", Revision: 1, Trigger: TriggerPush, Outcome: OutcomeSucceeded,
		ActiveRevision: 1, StartedAt: time.Now(), FinishedAt: time.Now()}
	if err := src.ReportDeployment(ctx, rep); err != nil {
		t.Fatalf("report = %v", err)
	}
	srv.with(func(f *fakeServer) { f.reportCode = codes.FailedPrecondition })
	if err := src.ReportDeployment(ctx, rep); !errors.Is(err, ErrReportRefused) {
		t.Errorf("refused report = %v", err)
	}
	srv.with(func(f *fakeServer) { f.reportCode = codes.Unavailable })
	if err := src.ReportDeployment(ctx, rep); err == nil || errors.Is(err, ErrReportRefused) {
		t.Errorf("unavailable server = %v", err)
	}
}

func TestServerSourcePollingPush(t *testing.T) {
	srv := newFakeServer(t, KindDNS)
	addr := srv.start(t)
	store, id := enrollWith(t, srv, addr)
	srv.with(func(f *fakeServer) {
		f.desired = 1
		f.mode = agentv1.OperatingMode_OPERATING_MODE_MANUAL
		f.streamDisabled = true
	})
	role, _ := runAgentAgainst(t, store, id, KindDNS)
	polls := func() int {
		n := 0
		srv.with(func(f *fakeServer) { n = f.polls })
		return n
	}
	waitFor(t, "polls", func() bool { return polls() >= 3 })
	if n := len(serverReports(srv)); n != 0 {
		t.Fatalf("manual mode applied the desired revision (%d reports)", n)
	}

	// An open push is announced by polling and applied in manual mode.
	srv.with(func(f *fakeServer) {
		f.desired = 2
		f.pushDeployment = "dep-poll"
	})
	waitFor(t, "report of the push", func() bool { return len(serverReports(srv)) >= 1 })
	r := serverReports(srv)[0]
	if r.GetRevision() != 2 || r.GetDeploymentId() != "dep-poll" ||
		r.GetTrigger() != agentv1.DeploymentTrigger_DEPLOYMENT_TRIGGER_PUSH ||
		r.GetOutcome() != agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_SUCCEEDED {
		t.Fatalf("report = %v", r)
	}
	after := polls()
	waitFor(t, "more polls", func() bool { return polls() >= after+3 })
	if n := role.activations(2); n != 1 {
		t.Errorf("activations of revision 2 = %d, want 1", n)
	}
	// A poll racing with the report may cause the same report to be sent
	// again, never a second apply.
	for _, rr := range serverReports(srv) {
		if rr.GetDeploymentId() != "dep-poll" || !rr.GetStartedAt().AsTime().Equal(r.GetStartedAt().AsTime()) {
			t.Errorf("unexpected report %v", rr)
		}
	}
}

func TestServerSourceModeChangeAndResentPush(t *testing.T) {
	srv := newFakeServer(t, KindDNS)
	addr := srv.start(t)
	store, id := enrollWith(t, srv, addr)
	srv.with(func(f *fakeServer) {
		f.desired = 1
		f.mode = agentv1.OperatingMode_OPERATING_MODE_MANUAL
	})
	role, _ := runAgentAgainst(t, store, id, KindDNS)
	counts := func() (hellos, heartbeats, acks int) {
		srv.with(func(f *fakeServer) { hellos, heartbeats, acks = len(f.hellos), f.heartbeats, len(f.acks) })
		return
	}
	waitFor(t, "heartbeats in manual mode", func() bool { _, hb, _ := counts(); return hb >= 3 })
	if n := len(serverReports(srv)); n != 0 {
		t.Fatalf("manual mode applied the desired revision (%d reports)", n)
	}

	// Switching to auto_pull: the server closes the stream and the new
	// Welcome carries the mode.
	srv.with(func(f *fakeServer) { f.mode = agentv1.OperatingMode_OPERATING_MODE_AUTO_PULL })
	srv.kick <- struct{}{}
	waitFor(t, "report of revision 1", func() bool { return len(serverReports(srv)) == 1 })
	if r := serverReports(srv)[0]; r.GetRevision() != 1 || r.GetTrigger() != agentv1.DeploymentTrigger_DEPLOYMENT_TRIGGER_PULL {
		t.Fatalf("report = %v", r)
	}
	if h, _, _ := counts(); h < 2 {
		t.Errorf("hellos = %d, want a reconnect", h)
	}

	// Back to manual with a newer revision: it is not applied.
	srv.with(func(f *fakeServer) {
		f.mode = agentv1.OperatingMode_OPERATING_MODE_MANUAL
		f.desired = 2
	})
	srv.kick <- struct{}{}
	waitFor(t, "third hello", func() bool { h, _, _ := counts(); return h >= 3 })
	_, hb, _ := counts()
	waitFor(t, "heartbeats after the mode change", func() bool { _, n, _ := counts(); return n >= hb+3 })
	if n := role.activations(2); n != 0 || len(serverReports(srv)) != 1 {
		t.Fatalf("manual mode applied revision 2 (%d activations)", n)
	}

	// The explicit push applies it.
	push := func(cmdID string) *agentv1.Command {
		return &agentv1.Command{CommandId: cmdID, Action: &agentv1.Command_ApplyRevision{
			ApplyRevision: &agentv1.ApplyRevisionCommand{Revision: 2, DeploymentId: "dep-2"},
		}}
	}
	srv.commands <- push("c1")
	waitFor(t, "report of the push", func() bool { return len(serverReports(srv)) == 2 })
	first := serverReports(srv)[1]
	if first.GetRevision() != 2 || first.GetDeploymentId() != "dep-2" ||
		first.GetTrigger() != agentv1.DeploymentTrigger_DEPLOYMENT_TRIGGER_PUSH ||
		first.GetOutcome() != agentv1.DeploymentOutcome_DEPLOYMENT_OUTCOME_SUCCEEDED {
		t.Fatalf("report = %v", first)
	}

	// The server resends the push (as after a reconnect) after the agent
	// has dropped the delivered report from its queue: acknowledged, not
	// applied again, the remembered report is delivered again.
	waitFor(t, "empty report queue", func() bool {
		st, err := store.LoadState()
		return err == nil && len(st.PendingReports) == 0
	})
	srv.commands <- push("c2")
	waitFor(t, "second ack and report", func() bool {
		_, _, acks := counts()
		return acks == 2 && len(serverReports(srv)) == 3
	})
	if n := role.activations(2); n != 1 {
		t.Errorf("activations of revision 2 = %d, want 1", n)
	}
	again := serverReports(srv)[2]
	if again.GetDeploymentId() != "dep-2" || !again.GetStartedAt().AsTime().Equal(first.GetStartedAt().AsTime()) {
		t.Errorf("repeated report = %v", again)
	}
	srv.with(func(f *fakeServer) {
		if !f.acks[1].GetAccepted() || f.acks[1].GetCommandId() != "c2" {
			t.Errorf("acks = %v", f.acks)
		}
	})
}