Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/helm-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ on:
- "manifests/**"
- "scripts/publish-helm-repo.sh"
- "scripts/test-helm-charts.sh"
- "scripts/test_routing_rbac.py"
- "tests/helm/**"
- ".github/workflows/helm-release.yml"
push:
Expand All @@ -16,6 +17,7 @@ on:
- "manifests/**"
- "scripts/publish-helm-repo.sh"
- "scripts/test-helm-charts.sh"
- "scripts/test_routing_rbac.py"
- "tests/helm/**"
- ".github/workflows/helm-release.yml"
workflow_dispatch:
Expand Down Expand Up @@ -64,6 +66,9 @@ jobs:
with:
version: ${{ env.HELM_VERSION }}

- name: Check routing collector RBAC
run: python3 scripts/test_routing_rbac.py

- name: Lint and render
run: scripts/test-helm-charts.sh "$CHART"

Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/terraform-validate.yml
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,9 @@ jobs:
- name: compare Allow/Deny action sets
run: python3 scripts/check_iam_policy_drift.py

- name: Check routing collector RBAC
run: python3 scripts/test_routing_rbac.py

shellcheck:
runs-on: ubuntu-latest
steps:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -542,6 +542,7 @@
"iam:ListGroupPolicies",
"iam:ListGroups",
"iam:ListGroupsForUser",
"iam:ListInstanceProfiles",
"iam:ListMFADevices",
"iam:ListOpenIDConnectProviders",
"iam:ListPolicies",
Expand Down
14 changes: 13 additions & 1 deletion aws-integration-setup/terraform-v6/modules/k8s-resources/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,19 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" {
}

rule {
non_resource_urls = ["/version"]
api_groups = ["gateway.networking.k8s.io"]
resources = ["gatewayclasses", "gateways", "httproutes", "referencegrants"]
verbs = ["list"]
}

rule {
api_groups = ["networking.istio.io"]
resources = ["virtualservices", "serviceentries", "destinationrules"]
verbs = ["list"]
}

rule {
non_resource_urls = ["/version", "/api", "/apis", "/apis/gateway.networking.k8s.io/v1", "/apis/gateway.networking.k8s.io/v1beta1", "/apis/gateway.networking.k8s.io/v1alpha2", "/apis/gateway.networking.k8s.io/v1alpha3", "/apis/networking.istio.io/v1", "/apis/networking.istio.io/v1beta1", "/apis/networking.istio.io/v1alpha3"]
verbs = ["get"]
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,14 @@ locals {
api_groups = ["admissionregistration.k8s.io"]
resources = ["validatingwebhookconfigurations", "mutatingwebhookconfigurations", "validatingadmissionpolicies", "validatingadmissionpolicybindings"]
},
{
api_groups = ["gateway.networking.k8s.io"]
resources = ["gatewayclasses", "gateways", "httproutes", "referencegrants"]
},
{
api_groups = ["networking.istio.io"]
resources = ["virtualservices", "serviceentries", "destinationrules"]
},
]
}

Expand All @@ -56,7 +64,7 @@ resource "kubernetes_cluster_role_v1" "nullify_readonly" {
}

rule {
non_resource_urls = ["/version"]
non_resource_urls = ["/version", "/api", "/apis", "/apis/gateway.networking.k8s.io/v1", "/apis/gateway.networking.k8s.io/v1beta1", "/apis/gateway.networking.k8s.io/v1alpha2", "/apis/gateway.networking.k8s.io/v1alpha3", "/apis/networking.istio.io/v1", "/apis/networking.istio.io/v1beta1", "/apis/networking.istio.io/v1alpha3"]
verbs = ["get"]
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ run "collector_only_is_the_default" {
}

assert {
condition = kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].image == "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0"
condition = kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].image == "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.48.0"
error_message = "The collector image must default to the pinned Nullify ECR Public tag"
}

Expand Down Expand Up @@ -135,18 +135,15 @@ run "managed_scan_rbac_only" {
}

assert {
condition = length(kubernetes_cluster_role_v1.nullify_readonly[0].rule) == 8
error_message = "The managed-scan ClusterRole must have seven resource rules and one /version rule"
condition = length(kubernetes_cluster_role_v1.nullify_readonly[0].rule) == 10
error_message = "The managed-scan ClusterRole must have nine resource rules and one discovery rule"
}

assert {
condition = sum([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 7) : length(rule.resources)]) == 26
error_message = "The managed-scan ClusterRole must list exactly the 26 config kinds; secrets are omitted because list returns values"
condition = sum([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 9) : length(rule.resources)]) == 33
error_message = "The managed-scan ClusterRole must list exactly the 33 config and routing kinds; secrets are omitted because list returns values"
}

# Pins the set, not just the size: swapping one kind for another keeps every
# count assertion green. Anything else that binds this group -- a chart, a
# manifest -- has to grant exactly these kinds.
assert {
condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].api_groups) == tolist([""]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].resources) == toset(["nodes", "namespaces", "pods", "services", "persistentvolumeclaims", "persistentvolumes", "configmaps", "resourcequotas", "limitranges", "serviceaccounts"])
error_message = "Rule 0 must grant exactly the core API group kinds the scanner lists"
Expand Down Expand Up @@ -183,13 +180,32 @@ run "managed_scan_rbac_only" {
}

assert {
condition = alltrue([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 7) : tolist(rule.verbs) == tolist(["list"])])
condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[7].api_groups) == tolist(["gateway.networking.k8s.io"]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[7].resources) == toset(["gatewayclasses", "gateways", "httproutes", "referencegrants"])
error_message = "Rule 7 must grant exactly the four Gateway API routing kinds"
}

assert {
condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[8].api_groups) == tolist(["networking.istio.io"]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[8].resources) == toset(["virtualservices", "serviceentries", "destinationrules"])
error_message = "Rule 8 must grant exactly the three Istio routing kinds"
}

assert {
condition = alltrue([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 9) : tolist(rule.verbs) == tolist(["list"])])
error_message = "Resource rules must grant list only"
}

assert {
condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[7].non_resource_urls) == tolist(["/version"]) && tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[7].verbs) == tolist(["get"])
error_message = "The only non-resource grant must be get /version"
condition = toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[9].non_resource_urls) == toset([
"/version", "/api", "/apis",
"/apis/gateway.networking.k8s.io/v1",
"/apis/gateway.networking.k8s.io/v1beta1",
"/apis/gateway.networking.k8s.io/v1alpha2",
"/apis/gateway.networking.k8s.io/v1alpha3",
"/apis/networking.istio.io/v1",
"/apis/networking.istio.io/v1beta1",
"/apis/networking.istio.io/v1alpha3",
]) && tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[9].verbs) == tolist(["get"])
error_message = "The only non-resource grant must be get on the exact collector discovery URLs"
}

assert {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ variable "kubernetes_namespace" {
variable "collector_image" {
type = string
description = "Container image for the Kubernetes collector. Defaults to a pinned tag of Nullify's ECR Public image"
default = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0"
default = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.48.0"
}

variable "cronjob_schedule" {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ data "aws_iam_policy_document" "readonly_policy_part2" {
"iam:ListGroupPolicies",
"iam:ListGroups",
"iam:ListGroupsForUser",
"iam:ListInstanceProfiles",
"iam:ListMFADevices",
"iam:ListOpenIDConnectProviders",
"iam:ListPolicies",
Expand Down
14 changes: 13 additions & 1 deletion aws-integration-setup/terraform/modules/k8s-resources/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,19 @@ resource "kubernetes_cluster_role" "nullify_readonly_role" {
}

rule {
non_resource_urls = ["/version"]
api_groups = ["gateway.networking.k8s.io"]
resources = ["gatewayclasses", "gateways", "httproutes", "referencegrants"]
verbs = ["list"]
}

rule {
api_groups = ["networking.istio.io"]
resources = ["virtualservices", "serviceentries", "destinationrules"]
verbs = ["list"]
}

rule {
non_resource_urls = ["/version", "/api", "/apis", "/apis/gateway.networking.k8s.io/v1", "/apis/gateway.networking.k8s.io/v1beta1", "/apis/gateway.networking.k8s.io/v1alpha2", "/apis/gateway.networking.k8s.io/v1alpha3", "/apis/networking.istio.io/v1", "/apis/networking.istio.io/v1beta1", "/apis/networking.istio.io/v1alpha3"]
verbs = ["get"]
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,14 @@ locals {
api_groups = ["admissionregistration.k8s.io"]
resources = ["validatingwebhookconfigurations", "mutatingwebhookconfigurations", "validatingadmissionpolicies", "validatingadmissionpolicybindings"]
},
{
api_groups = ["gateway.networking.k8s.io"]
resources = ["gatewayclasses", "gateways", "httproutes", "referencegrants"]
},
{
api_groups = ["networking.istio.io"]
resources = ["virtualservices", "serviceentries", "destinationrules"]
},
]
}

Expand All @@ -56,7 +64,7 @@ resource "kubernetes_cluster_role_v1" "nullify_readonly" {
}

rule {
non_resource_urls = ["/version"]
non_resource_urls = ["/version", "/api", "/apis", "/apis/gateway.networking.k8s.io/v1", "/apis/gateway.networking.k8s.io/v1beta1", "/apis/gateway.networking.k8s.io/v1alpha2", "/apis/gateway.networking.k8s.io/v1alpha3", "/apis/networking.istio.io/v1", "/apis/networking.istio.io/v1beta1", "/apis/networking.istio.io/v1alpha3"]
verbs = ["get"]
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ run "collector_only_is_the_default" {
}

assert {
condition = kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].image == "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0"
condition = kubernetes_cron_job_v1.k8s_collector[0].spec[0].job_template[0].spec[0].template[0].spec[0].container[0].image == "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.48.0"
error_message = "The collector image must default to the pinned Nullify ECR Public tag"
}

Expand Down Expand Up @@ -135,18 +135,15 @@ run "managed_scan_rbac_only" {
}

assert {
condition = length(kubernetes_cluster_role_v1.nullify_readonly[0].rule) == 8
error_message = "The managed-scan ClusterRole must have seven resource rules and one /version rule"
condition = length(kubernetes_cluster_role_v1.nullify_readonly[0].rule) == 10
error_message = "The managed-scan ClusterRole must have nine resource rules and one discovery rule"
}

assert {
condition = sum([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 7) : length(rule.resources)]) == 26
error_message = "The managed-scan ClusterRole must list exactly the 26 config kinds; secrets are omitted because list returns values"
condition = sum([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 9) : length(rule.resources)]) == 33
error_message = "The managed-scan ClusterRole must list exactly the 33 config and routing kinds; secrets are omitted because list returns values"
}

# Pins the set, not just the size: swapping one kind for another keeps every
# count assertion green. Anything else that binds this group -- a chart, a
# manifest -- has to grant exactly these kinds.
assert {
condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].api_groups) == tolist([""]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[0].resources) == toset(["nodes", "namespaces", "pods", "services", "persistentvolumeclaims", "persistentvolumes", "configmaps", "resourcequotas", "limitranges", "serviceaccounts"])
error_message = "Rule 0 must grant exactly the core API group kinds the scanner lists"
Expand Down Expand Up @@ -183,13 +180,32 @@ run "managed_scan_rbac_only" {
}

assert {
condition = alltrue([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 7) : tolist(rule.verbs) == tolist(["list"])])
condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[7].api_groups) == tolist(["gateway.networking.k8s.io"]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[7].resources) == toset(["gatewayclasses", "gateways", "httproutes", "referencegrants"])
error_message = "Rule 7 must grant exactly the four Gateway API routing kinds"
}

assert {
condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[8].api_groups) == tolist(["networking.istio.io"]) && toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[8].resources) == toset(["virtualservices", "serviceentries", "destinationrules"])
error_message = "Rule 8 must grant exactly the three Istio routing kinds"
}

assert {
condition = alltrue([for rule in slice(kubernetes_cluster_role_v1.nullify_readonly[0].rule, 0, 9) : tolist(rule.verbs) == tolist(["list"])])
error_message = "Resource rules must grant list only"
}

assert {
condition = tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[7].non_resource_urls) == tolist(["/version"]) && tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[7].verbs) == tolist(["get"])
error_message = "The only non-resource grant must be get /version"
condition = toset(kubernetes_cluster_role_v1.nullify_readonly[0].rule[9].non_resource_urls) == toset([
"/version", "/api", "/apis",
"/apis/gateway.networking.k8s.io/v1",
"/apis/gateway.networking.k8s.io/v1beta1",
"/apis/gateway.networking.k8s.io/v1alpha2",
"/apis/gateway.networking.k8s.io/v1alpha3",
"/apis/networking.istio.io/v1",
"/apis/networking.istio.io/v1beta1",
"/apis/networking.istio.io/v1alpha3",
]) && tolist(kubernetes_cluster_role_v1.nullify_readonly[0].rule[9].verbs) == tolist(["get"])
error_message = "The only non-resource grant must be get on the exact collector discovery URLs"
}

assert {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ variable "kubernetes_namespace" {
variable "collector_image" {
type = string
description = "Container image for the Kubernetes collector. Defaults to a pinned tag of Nullify's ECR Public image"
default = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.46.0"
default = "public.ecr.aws/w4o2j2x4/integrations:k8s-collector-3.48.0"
}

variable "cronjob_schedule" {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ data "aws_iam_policy_document" "readonly_policy_part2" {
"iam:ListGroupPolicies",
"iam:ListGroups",
"iam:ListGroupsForUser",
"iam:ListInstanceProfiles",
"iam:ListMFADevices",
"iam:ListOpenIDConnectProviders",
"iam:ListPolicies",
Expand Down
2 changes: 1 addition & 1 deletion helm-charts/nullify-k8s-collector/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ apiVersion: v2
name: nullify-k8s-collector
description: A Helm chart for Nullify's Kubernetes information collector for cloud security scanning
type: application
version: 0.4.1
version: 0.5.0
kubeVersion: ">=1.21.0-0"
maintainers:
- name: Nullify
Expand Down
25 changes: 23 additions & 2 deletions helm-charts/nullify-k8s-collector/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ This Helm chart deploys a Kubernetes collector for the Nullify platform to gathe
- **GKE:** the cluster registered with Nullify (see [GKE](#gke-with-workload-identity-federation))
- Egress from the collector pod over HTTPS to `public.ecr.aws`, and to STS and S3 in the Nullify bucket's region

> **Image tag:** the default `collector.image.tag` is `k8s-collector-3.46.0`,
> **Image tag:** the default `collector.image.tag` is `k8s-collector-3.48.0`,
> the same pin as the Terraform `k8s-resources` module.

## Supported platforms
Expand Down Expand Up @@ -59,7 +59,7 @@ The chart refuses to render when a required value is missing or still a placehol
| `serviceAccount.name` | Name of the service account | `nullify-k8s-collector-sa` |
| `serviceAccount.namespace` | Namespace of every namespaced object; `""` means the release namespace. Must already exist. | `nullify` |
| `collector.image.repository` | Image repository | `public.ecr.aws/w4o2j2x4/integrations` |
| `collector.image.tag` | Image tag | `k8s-collector-3.46.0` |
| `collector.image.tag` | Image tag | `k8s-collector-3.48.0` |
| `collector.image.pullPolicy` | Pull policy | `IfNotPresent` |
| `collector.schedule` | CronJob schedule | `0 0 * * *` (daily at midnight) |
| `collector.s3.bucket` | S3 bucket for storing data (from Nullify configure page) | `""` (required) |
Expand Down Expand Up @@ -248,3 +248,24 @@ To enable debug logging for troubleshooting:
```

Debug mode enables the `ENABLE_DEBUG_LOG` environment variable, which provides more detailed logging information to help diagnose collection issues.

## Routing inventory

Collector 3.48.0 lists GatewayClass, Gateway, HTTPRoute and ReferenceGrant
from `gateway.networking.k8s.io`, and VirtualService, ServiceEntry and
DestinationRule from `networking.istio.io`. Only installed, supported API
versions are listed. Discovery inspects experimental Gateway API version
metadata to distinguish an absent kind from an unsupported version; it never
lists experimental routing objects. Routing failures stay isolated from core
cluster collection. The scan payload records complete (including empty),
absent, denied, failed and unsupported outcomes for each resource kind.

The chart grants `list` on these seven resources and `get` on API discovery
endpoints. It does not grant Secret access or write/watch permissions.
Header/query literals (including negative Istio header matches), infrastructure
annotation/label values, direct-response bodies and status messages are redacted; certificate and
credential references are retained. Collected declarations do not prove
that a route is programmed or receiving traffic.

Upgrading an existing collector requires both the new image and the updated
ClusterRole. Publish collector 3.48.0 before installing this chart version.
8 changes: 7 additions & 1 deletion helm-charts/nullify-k8s-collector/templates/clusterrole.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,12 @@ rules:
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["list"]
- nonResourceURLs: ["/version"]
- apiGroups: ["gateway.networking.k8s.io"]
resources: ["gatewayclasses", "gateways", "httproutes", "referencegrants"]
verbs: ["list"]
- apiGroups: ["networking.istio.io"]
resources: ["virtualservices", "serviceentries", "destinationrules"]
verbs: ["list"]
- nonResourceURLs: ["/version", "/api", "/apis", "/apis/gateway.networking.k8s.io/v1", "/apis/gateway.networking.k8s.io/v1beta1", "/apis/gateway.networking.k8s.io/v1alpha2", "/apis/gateway.networking.k8s.io/v1alpha3", "/apis/networking.istio.io/v1", "/apis/networking.istio.io/v1beta1", "/apis/networking.istio.io/v1alpha3"]
verbs: ["get"]
{{- end }}
2 changes: 1 addition & 1 deletion helm-charts/nullify-k8s-collector/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ collector:
# on k8s-collector-latest: a moved tag is what every Helm install runs.
image:
repository: public.ecr.aws/w4o2j2x4/integrations
tag: k8s-collector-3.46.0
tag: k8s-collector-3.48.0
pullPolicy: IfNotPresent

# Resource limits
Expand Down
Loading
Loading