feat(helm): add example Agents with the kagent-tools skills

Makefile

@@ -9,6 +9,9 @@ HELM_REPO ?= oci://ghcr.io/kagent-dev
 HELM_DIST_FOLDER ?= dist
 HELM_SOURCE_DIR ?= helm
 RUNTIME_IMAGE_REPOSITORY ?= ghcr.io/kagent-dev/kagent
+# The kagent-tools image whose skills the chart's example Agents mount. Its tag
+# is the kagent-tools chart version that helm/kagent depends on.
+TOOLS_IMAGE_REPOSITORY ?= ghcr.io/kagent-dev/kagent/tools
 # Release packaging requires an explicit metadata directory; local builds have a default.
 RUNTIME_BUILD_METADATA_DIR = $(or $(RUNTIME_IMAGE_METADATA_DIR),$(CURDIR)/build/runtime-images)
 RUNTIME_IMAGE_METADATA_FILE = $(RUNTIME_BUILD_METADATA_DIR)/$(patsubst build-%,%,$@).json
@@ -619,9 +622,15 @@ helm-release: ## Package release charts from RUNTIME_IMAGE_METADATA_DIR without
 	@test -n "$(RUNTIME_IMAGE_METADATA_DIR)" || { echo "RUNTIME_IMAGE_METADATA_DIR is required for release chart packaging" >&2; exit 1; }
 	bash scripts/package-release-charts.sh "$(VERSION)" "$(RUNTIME_IMAGE_METADATA_DIR)" "$(HELM_SOURCE_DIR)" "$(HELM_DIST_FOLDER)" "$(RUNTIME_IMAGE_REPOSITORY)"
 
+.PHONY: tools-image-metadata
+tools-image-metadata: ## Record the digest of the kagent-tools image the chart pins in RUNTIME_IMAGE_METADATA_DIR
+	@test -n "$(RUNTIME_IMAGE_METADATA_DIR)" || { echo "RUNTIME_IMAGE_METADATA_DIR is required for release chart packaging" >&2; exit 1; }
+	bash scripts/tools-image-metadata.sh "$(HELM_SOURCE_DIR)/kagent/Chart-template.yaml" "$(TOOLS_IMAGE_REPOSITORY)" "$(RUNTIME_IMAGE_METADATA_DIR)/kagent-tools.json"
+
 .PHONY: helm-publish
 helm-publish: ## Package and push all Helm charts to the OCI registry
-helm-publish: helm-release
+helm-publish: tools-image-metadata
+	$(MAKE) --no-print-directory helm-release
 	helm push "$(HELM_DIST_FOLDER)/kagent-crds-$(VERSION).tgz" $(HELM_REPO)/kagent/helm
 	helm push "$(HELM_DIST_FOLDER)/kagent-$(VERSION).tgz" $(HELM_REPO)/kagent/helm
 

docs/architecture/configuration-and-compilation.md

@@ -95,6 +95,16 @@ Local installations can override individual defaults with
 `config.harnessImages`; `global.imageRegistry` rewrites both packaged defaults
 and explicit overrides.
 
+The chart's optional example Agents mount skills from the kagent-tools image.
+kagent-tools releases from its own repository, so no build in this repository
+records the image digest. Before packaging, `make helm-publish` runs `make
+tools-image-metadata`. That target looks up the digest of the kagent-tools
+version in the chart's dependencies and writes it to
+`RUNTIME_IMAGE_METADATA_DIR/kagent-tools.json`. If that file exists, `make
+helm-release` validates it and packages it as `files/kagent-tools-image.json`.
+If the file is missing, the example Agents render only with an explicit
+`exampleAgents.skillsImage`.
+
 The outer compiler sets the selected image on the compiled revision before hashing
 it. Adapters receive the requested Harness spec; provenance records that spec
 without injected defaults. A missing or invalid

docs/env.md

@@ -180,6 +180,7 @@ This reference covers user-configurable settings for the controller, CLI, standa
 | `KAGENT_E2E_RUN_UPGRADE_TESTS` | String | `false` | Run upgrade integration tests when exactly true. |
 | `KAGENT_E2E_SANDBOX_NAMESPACE` | String | `kagent` | Namespace for sandbox E2E resources. |
 | `KAGENT_E2E_SANDBOX_WORKER_POOL` | String | `kagent-default` | Worker pool used by sandbox E2E resources. |
+| `KAGENT_E2E_TOOLS_SKILLS_IMAGE` | String | `(none)` | Digest-pinned kagent-tools image with the opt/kagent-tools-skills Agent Plugin for the example Agents E2E test. When unset, that test skips. |
 | `KAGENT_E2E_UI_LOOP_EXTENSION_PORT` | String | `(none)` | Example-extension browser-test port. Defaults to KAGENT_E2E_UI_LOOP_PORT plus 50. |
 | `KAGENT_E2E_UI_LOOP_LIVE` | Boolean | `false` | Run UI browser tests against a real cluster when exactly true. |
 | `KAGENT_E2E_UI_LOOP_LIVE_PORT` | String | `8301` | Live-cluster UI browser-test port. |

go/core/pkg/env/testing.go

@@ -56,6 +56,7 @@ var (
 	_                         = RegisterStringVar("KAGENT_E2E_BYO_IMAGE", "", "Digest-pinned BYO image substituted into E2E lifecycle manifests.", ComponentTesting)
 	_                         = RegisterStringVar("KAGENT_E2E_CLAUDE_IMAGE", "", "Digest-pinned Claude harness image substituted into E2E lifecycle manifests.", ComponentTesting)
 	_                         = RegisterStringVar("KAGENT_E2E_CODEX_IMAGE", "", "Digest-pinned Codex harness image substituted into E2E lifecycle manifests.", ComponentTesting)
+	E2EToolsSkillsImage       = RegisterStringVar("KAGENT_E2E_TOOLS_SKILLS_IMAGE", "", "Digest-pinned kagent-tools image with the opt/kagent-tools-skills Agent Plugin for the example Agents E2E test. When unset, that test skips.", ComponentTesting)
 	E2ESandboxNamespace       = RegisterStringVar("KAGENT_E2E_SANDBOX_NAMESPACE", "kagent", "Namespace for sandbox E2E resources.", ComponentTesting)
 	E2ESandboxWorkerPool      = RegisterStringVar("KAGENT_E2E_SANDBOX_WORKER_POOL", "kagent-default", "Worker pool used by sandbox E2E resources.", ComponentTesting)
 	E2EOTLPListenAddress      = RegisterStringVar("KAGENT_E2E_OTLP_LISTEN_ADDRESS", "", "Listen address for the E2E suite's OTLP trace receiver. Unset disables the shared receiver.", ComponentTesting)

go/core/test/e2e/README.md

@@ -7,6 +7,15 @@ the three `config.harnessImages` defaults matching the explicit Harness
 fixtures. The test checks runtime readiness, effective-image status, and that
 the controller leaves the user spec unchanged.
 
+`TestExampleAgentToolsSkills` runs each of the chart's three example Agents
+on every harness with a mock model. It reads each Agent's name, description,
+and skills from `helm/kagent/templates/example-agents.yaml`. It needs
+`KAGENT_E2E_TOOLS_SKILLS_IMAGE`: a digest-pinned kagent-tools image in the local
+test registry that contains the `opt/kagent-tools-skills` Agent Plugin. Without
+that variable, the test skips. For each Agent, the test checks that the model
+sees only the Agent's own skills and that one of them loads. The test leaves
+out the tool server binding because the test install does not run kagent-tools.
+
 The standalone sandbox suite needs a Substrate WorkerPool with available capacity.
 Set `config.sandboxes.guestImage` to the
 guest image built for the test. The controller passes the pinned image reference

go/core/test/e2e/example_agent_skills_test.go

@@ -0,0 +1,163 @@
+package e2e_test
+
+import (
+	"encoding/json"
+	"os"
+	"regexp"
+	"slices"
+	"strings"
+	"testing"
+
+	a2atype "github.com/a2aproject/a2a-go/v2/a2a"
+	"github.com/kagent-dev/mockllm"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/kagent-dev/kagent/go/api/v1alpha3"
+	kagentenv "github.com/kagent-dev/kagent/go/core/pkg/env"
+)
+
+const exampleAgentsTemplate = "../../../../helm/kagent/templates/example-agents.yaml"
+
+// exampleAgentLoads names, for each example Agent in the chart, a question the
+// mock model answers by loading one of the Agent's skills, and a tool that only
+// that skill's body names.
+var exampleAgentLoads = map[string]struct{ question, skill, tool string }{
+	"cluster-ops": {"Which Helm releases are installed?", "helm-release-management", "helm_list_releases"},
+	"networking":  {"Why do requests fail inside the Istio mesh?", "istio-traffic-debugging", "istio_proxy_status"},
+	"security":    {"Is Kubescape healthy in this cluster?", "kubescape-security-posture", "kubescape_check_health"},
+}
+
+type exampleAgent struct {
+	name, description string
+	skills            []string
+}
+
+var (
+	exampleAgentNamePattern        = regexp.MustCompile(`(?m)^  name: \{\{ printf "%s-([a-z-]+)" \$prefix`)
+	exampleAgentDescriptionPattern = regexp.MustCompile(`(?m)^      description: "([^"]+)"$`)
+	exampleAgentSkillsPattern      = regexp.MustCompile(`(?m)^          skills:\n((?:            - \S+(?:\n|\z))+)`)
+)
+
+// readExampleAgents reads each example Agent's name, description and skills
+// from the chart template, so the test follows the chart.
+func readExampleAgents(t *testing.T) []exampleAgent {
+	t.Helper()
+	data, err := os.ReadFile(exampleAgentsTemplate)
+	require.NoError(t, err)
+	var agents []exampleAgent
+	for doc := range strings.SplitSeq(string(data), "\n---\n") {
+		if !strings.Contains(doc, "\nkind: Agent\n") {
+			continue
+		}
+		name := exampleAgentNamePattern.FindStringSubmatch(doc)
+		description := exampleAgentDescriptionPattern.FindStringSubmatch(doc)
+		skills := exampleAgentSkillsPattern.FindStringSubmatch(doc)
+		require.True(t, name != nil && description != nil && skills != nil, "cannot read an example Agent in %s:\n%s", exampleAgentsTemplate, doc)
+		agent := exampleAgent{name: name[1], description: description[1]}
+		for line := range strings.SplitSeq(strings.TrimSpace(skills[1]), "\n") {
+			agent.skills = append(agent.skills, strings.TrimPrefix(strings.TrimSpace(line), "- "))
+		}
+		agents = append(agents, agent)
+	}
+	names := make([]string, 0, len(agents))
+	for _, agent := range agents {
+		names = append(names, agent.name)
+	}
+	want := make([]string, 0, len(exampleAgentLoads))
+	for name := range exampleAgentLoads {
+		want = append(want, name)
+	}
+	require.ElementsMatch(t, want, names, "every example Agent in %s needs an entry in exampleAgentLoads", exampleAgentsTemplate)
+	return agents
+}
+
+// TestExampleAgentToolsSkills builds each of the chart's example Agents on
+// every harness with a mock model. Each test Agent gets the kagent-tools image
+// as an Agent Plugin, the chart Agent's skills, and the built-in prompts. The
+// test leaves out the chart's tool server binding because the E2E install does
+// not run kagent-tools.
+func TestExampleAgentToolsSkills(t *testing.T) {
+	t.Parallel()
+	target := interactionTarget(t)
+	image := kagentenv.E2EToolsSkillsImage.Get()
+	if image == "" {
+		t.Skip("KAGENT_E2E_TOOLS_SKILLS_IMAGE is not set. Set it to a kagent-tools image with the opt/kagent-tools-skills folder.")
+	}
+	agents := readExampleAgents(t)
+	for _, agent := range agents {
+		load := exampleAgentLoads[agent.name]
+		require.Contains(t, agent.skills, load.skill, "%s must select the skill it loads", agent.name)
+		var others []string
+		for _, other := range agents {
+			if other.name != agent.name {
+				others = append(others, other.skills...)
+			}
+		}
+		t.Run(agent.name, func(t *testing.T) {
+			t.Parallel()
+			forEachHarness(t, func(t *testing.T, harness testHarness) {
+				t.Parallel()
+				model := startExampleAgentSkillsMock(t, harness.skillsRoot(), load.question, load.skill, load.tool)
+				kube := interactionKubeClient(t)
+				config := harness.createModel(t, kube, reachableModelURL(t, model.URL), nil)
+				template := &v1alpha3.AgentTemplate{
+					ObjectMeta: metav1.ObjectMeta{GenerateName: "example-" + agent.name + "-", Namespace: "kagent", Labels: harness.labels()},
+					Spec: v1alpha3.AgentTemplateSpec{
+						ModelConfig: &v1alpha3.ObjectReference{Name: config.Name},
+						Description: agent.description,
+						PromptTemplate: &v1alpha3.AgentTemplatePromptTemplateSpec{
+							DataSources: []v1alpha3.AgentTemplatePromptSource{{Name: "kagent-builtin-prompts", Alias: "builtin"}},
+						},
+						SystemPrompt: "Answer questions about the Kubernetes cluster where kagent runs. When a\n" +
+							"question matches one of your skills, load that skill and follow it. Each\n" +
+							"skill names the kagent-tools MCP tools it uses.\n\n" +
+							"{{include \"builtin/skills-usage\"}}\n\n" +
+							"{{include \"builtin/kubernetes-context\"}}\n\n" +
+							"{{include \"builtin/tool-usage-best-practices\"}}\n\n" +
+							"{{include \"builtin/safety-guardrails\"}}\n",
+						Plugins: []v1alpha3.PluginBundle{{
+							Source: v1alpha3.ArtifactSource{OCI: image, Path: "opt/kagent-tools-skills"},
+							Skills: agent.skills,
+						}},
+					},
+				}
+				// Ready requires the runtime to mount the image and install the skills.
+				createAndWaitInteractionTemplate(t, harness, kube, template)
+				fixture := newInteractionFixtureForTemplate(t, harness, target, template.Name)
+				_, _, task := fixture.send(t, load.question)
+				require.Equal(t, a2atype.TaskStateCompleted, task.Status.State, taskText(task))
+				require.Contains(t, taskText(task), "EXAMPLE_SKILL_LOADED")
+
+				// Claude also sends requests without a body. Count only model turns.
+				requests := slices.DeleteFunc(model.Requests("", ""), func(r recordedModelRequest) bool { return len(r.Body) == 0 })
+				require.GreaterOrEqual(t, len(requests), 2, "the model must be called again after loading the skill")
+				first := string(requests[0].Body)
+				for _, skill := range agent.skills {
+					require.Contains(t, first, skill, "skill must be advertised to the model")
+				}
+				for _, skill := range others {
+					require.NotContains(t, first, skill, "another example Agent's skill must not be advertised")
+				}
+				require.Contains(t, first, "## Safety Guidelines", "built-in prompts must be included")
+				// Discovery exposes metadata only. The skill body names its MCP tools.
+				require.NotContains(t, first, load.tool)
+				loaded := false
+				for _, request := range requests[1:] {
+					loaded = loaded || strings.Contains(string(request.Body), load.tool)
+				}
+				require.True(t, loaded, "the model did not receive the %s skill body", load.skill)
+			})
+		})
+	}
+}
+
+func startExampleAgentSkillsMock(t *testing.T, root, question, skill, tool string) *modelRecorder {
+	t.Helper()
+	fixture, err := interactionMocks.ReadFile("mocks/invoke_example_agent_skills.json")
+	require.NoError(t, err)
+	replacer := strings.NewReplacer("ROOT", root, "QUESTION", question, "SKILL_NAME", skill, "SKILL_TOOL", tool)
+	var config mockllm.Config
+	require.NoError(t, json.Unmarshal([]byte(replacer.Replace(string(fixture))), &config))
+	return startModelRecorder(t, mockLLMProxy(t, startMockLLMConfig(t, config)), nil)
+}

go/core/test/e2e/interaction_test.go

@@ -50,7 +50,7 @@ import (
 )
 
 //go:embed mocks/invoke_agent.json mocks/invoke_golang_hitl_ask_user.json mocks/invoke_mcp_agent.json mocks/invoke_shared_agent.json mocks/invoke_structured_output.json mocks/invoke_native_ask_user.json
-//go:embed mocks/invoke_mounted_skills.json mocks/invoke_egress_skill.json
+//go:embed mocks/invoke_mounted_skills.json mocks/invoke_egress_skill.json mocks/invoke_example_agent_skills.json
 var interactionMocks embed.FS
 
 const structuredOutputSchema = `{"type":"object","properties":{"answer":{"type":"integer"},"explanation":{"type":"string"}},"required":["answer","explanation"],"additionalProperties":false}`