From f9b018205e982a995f88104df467d209b72f1e79 Mon Sep 17 00:00:00 2001 From: Yuriy Bezsonov Date: Mon, 9 Nov 2026 21:32:48 +0100 Subject: [PATCH 1/3] fix(cleanup): Delete PVC-backed EBS volumes, define log_success java-performance.sh and java-on-amazon-eks.sh stop every StatefulSet and Deployment that mounts a PersistentVolumeClaim, delete the claims and wait until their PersistentVolumes are gone. Without this, the gp3 volumes of Prometheus, Grafana and Pyroscope (28 GiB) stayed billed as `available` after the stack was deleted. Stopping Pyroscope also ends its writes into the workshop bucket, which failed the bucket's deletion. The plan shows the claim count. All three cleanup scripts define log_success: the final report_stack_retained called it and exited 127 after a successful cleanup. --- infra/scripts/cleanup/java-on-amazon-eks.sh | 63 ++++++++++++++++++++- infra/scripts/cleanup/java-on-aws.sh | 1 + infra/scripts/cleanup/java-performance.sh | 63 ++++++++++++++++++++- 3 files changed, 125 insertions(+), 2 deletions(-) diff --git a/infra/scripts/cleanup/java-on-amazon-eks.sh b/infra/scripts/cleanup/java-on-amazon-eks.sh index fd860183..2397bd22 100755 --- a/infra/scripts/cleanup/java-on-amazon-eks.sh +++ b/infra/scripts/cleanup/java-on-amazon-eks.sh @@ -42,6 +42,7 @@ declare -a SECURITY_GROUPS=() declare -a STACK_RESOURCE_IDS=() log_info() { printf '[INFO] %s\n' "$*"; } +log_success() { printf '[OK] %s\n' "$*"; } log_warn() { printf '[WARN] %s\n' "$*" >&2; } log_error() { printf '[ERROR] %s\n' "$*" >&2; } record_error() { log_error "$*"; ERRORS=$((ERRORS + 1)); } @@ -399,9 +400,10 @@ discover_resources() { } print_plan() { - local ingress_count service_count association_count + local ingress_count service_count association_count pvc_count ingress_count=$(kubectl_cli get ingress --all-namespaces -o json | jq '.items | length') service_count=$(kubectl_cli get service --all-namespaces -o json | jq '[.items[] | select(.spec.type == "LoadBalancer")] | length') + pvc_count=$(kubectl_cli get pvc --all-namespaces -o json | jq '.items | length') association_count=$(aws_cli eks list-pod-identity-associations --cluster-name "$EKS_CLUSTER_NAME" \ --query 'length(associations)' --output text) @@ -409,6 +411,7 @@ print_plan() { printf ' Kubernetes Ingresses: %s\n' "$ingress_count" printf ' Kubernetes LoadBalancer Services: %s\n' "$service_count" printf ' EKS Pod Identity associations: %s\n' "$association_count" + printf ' Kubernetes volume claims: %s\n' "$pvc_count" printf ' Load balancers in VPC: %d\n' "${#RUNTIME_LOAD_BALANCERS[@]}" printf ' Target groups in VPC: %d\n' "${#RUNTIME_TARGET_GROUPS[@]}" printf ' Tagged runtime ECR repositories: %d\n' "${#RUNTIME_ECR_REPOSITORIES[@]}" @@ -434,6 +437,63 @@ kubernetes_dependencies_gone() { [[ "$ingress_count" -eq 0 && "$service_count" -eq 0 ]] } +pods_with_claims_gone() { + local count + count=$(kubectl_cli get pods --all-namespaces -o json 2>/dev/null \ + | jq '[.items[] | select([.spec.volumes[]? | select(.persistentVolumeClaim)] | length > 0)] | length') || return 1 + [[ "$count" -eq 0 ]] +} + +volumes_gone() { + local name + for name in "$@"; do + kubectl_cli get pv "$name" >/dev/null 2>&1 && return 1 + done + return 0 +} + +# PersistentVolumeClaims are backed by EBS volumes that outlive the cluster: once it is +# deleted, no CSI driver is left to delete them, and they stay billed as `available`. +# Stop every workload that mounts a claim (this also stops Pyroscope writing blocks into +# the stack's bucket, which would fail the bucket's deletion), delete the claims, and wait +# until their PersistentVolumes, and with them the EBS volumes, are gone. +cleanup_persistent_volumes() { + local pvc_json workload_json namespace kind name + local -a volume_names=() + pvc_json=$(kubectl_cli get pvc --all-namespaces -o json) || { + record_error "Could not enumerate PersistentVolumeClaims" + return + } + [[ "$(jq '.items | length' <<<"$pvc_json")" -gt 0 ]] || return 0 + mapfile -t volume_names < <(jq -r '.items[].spec.volumeName // empty' <<<"$pvc_json") + + workload_json=$(kubectl_cli get statefulsets,deployments --all-namespaces -o json) || { + record_error "Could not enumerate StatefulSets and Deployments" + return + } + log_info "Stopping every workload that mounts a PersistentVolumeClaim" + while IFS=$'\t' read -r namespace kind name; do + [[ -n "$namespace" && -n "$name" ]] || continue + kubectl_cli scale "$kind/$name" --namespace "$namespace" --replicas=0 >/dev/null \ + || record_error "Failed to scale $kind $namespace/$name to 0" + done < <(jq -r '.items[] + | select(((.spec.volumeClaimTemplates // []) | length > 0) + or ([.spec.template.spec.volumes[]? | select(.persistentVolumeClaim)] | length > 0)) + | [.metadata.namespace, (.kind | ascii_downcase), .metadata.name] | @tsv' <<<"$workload_json") + wait_until "pods that mount PersistentVolumeClaims to stop" pods_with_claims_gone || true + + log_info "Deleting every PersistentVolumeClaim from validated EKS cluster $EKS_CLUSTER_NAME" + while IFS=$'\t' read -r namespace name; do + [[ -n "$namespace" && -n "$name" ]] || continue + kubectl_cli delete pvc "$name" --namespace "$namespace" --ignore-not-found \ + --wait=true --timeout="${WAIT_SECONDS}s" >/dev/null || record_error "Failed to delete PersistentVolumeClaim $namespace/$name" + done < <(jq -r '.items[] | [.metadata.namespace,.metadata.name] | @tsv' <<<"$pvc_json") + if (( ${#volume_names[@]} > 0 )); then + wait_until "PersistentVolumes and their EBS volumes to be deleted" volumes_gone "${volume_names[@]}" \ + && log_info "PersistentVolumes deleted: ${volume_names[*]}" + fi +} + cleanup_eks_dependencies() { local namespace name association association_json association_tags ingress_json service_json association_output ingress_json=$(kubectl_cli get ingress --all-namespaces -o json) || { @@ -663,6 +723,7 @@ main() { fi confirm_cleanup + cleanup_persistent_volumes cleanup_eks_dependencies cleanup_vpc_load_balancers cleanup_runtime_ecr diff --git a/infra/scripts/cleanup/java-on-aws.sh b/infra/scripts/cleanup/java-on-aws.sh index 69479506..fa2c0dad 100755 --- a/infra/scripts/cleanup/java-on-aws.sh +++ b/infra/scripts/cleanup/java-on-aws.sh @@ -37,6 +37,7 @@ declare -a SECURITY_GROUPS=() declare -a STACK_RESOURCE_IDS=() log_info() { printf '[INFO] %s\n' "$*"; } +log_success() { printf '[OK] %s\n' "$*"; } log_warn() { printf '[WARN] %s\n' "$*" >&2; } log_error() { printf '[ERROR] %s\n' "$*" >&2; } record_error() { log_error "$*"; ERRORS=$((ERRORS + 1)); } diff --git a/infra/scripts/cleanup/java-performance.sh b/infra/scripts/cleanup/java-performance.sh index 84d8db48..9c9b4cf1 100755 --- a/infra/scripts/cleanup/java-performance.sh +++ b/infra/scripts/cleanup/java-performance.sh @@ -42,6 +42,7 @@ declare -a SECURITY_GROUPS=() declare -a STACK_RESOURCE_IDS=() log_info() { printf '[INFO] %s\n' "$*"; } +log_success() { printf '[OK] %s\n' "$*"; } log_warn() { printf '[WARN] %s\n' "$*" >&2; } log_error() { printf '[ERROR] %s\n' "$*" >&2; } record_error() { log_error "$*"; ERRORS=$((ERRORS + 1)); } @@ -399,9 +400,10 @@ discover_resources() { } print_plan() { - local ingress_count service_count association_count + local ingress_count service_count association_count pvc_count ingress_count=$(kubectl_cli get ingress --all-namespaces -o json | jq '.items | length') service_count=$(kubectl_cli get service --all-namespaces -o json | jq '[.items[] | select(.spec.type == "LoadBalancer")] | length') + pvc_count=$(kubectl_cli get pvc --all-namespaces -o json | jq '.items | length') association_count=$(aws_cli eks list-pod-identity-associations --cluster-name "$EKS_CLUSTER_NAME" \ --query 'length(associations)' --output text) @@ -409,6 +411,7 @@ print_plan() { printf ' Kubernetes Ingresses: %s\n' "$ingress_count" printf ' Kubernetes LoadBalancer Services: %s\n' "$service_count" printf ' EKS Pod Identity associations: %s\n' "$association_count" + printf ' Kubernetes volume claims: %s\n' "$pvc_count" printf ' Load balancers in VPC: %d\n' "${#RUNTIME_LOAD_BALANCERS[@]}" printf ' Target groups in VPC: %d\n' "${#RUNTIME_TARGET_GROUPS[@]}" printf ' Tagged runtime ECR repositories: %d\n' "${#RUNTIME_ECR_REPOSITORIES[@]}" @@ -434,6 +437,63 @@ kubernetes_dependencies_gone() { [[ "$ingress_count" -eq 0 && "$service_count" -eq 0 ]] } +pods_with_claims_gone() { + local count + count=$(kubectl_cli get pods --all-namespaces -o json 2>/dev/null \ + | jq '[.items[] | select([.spec.volumes[]? | select(.persistentVolumeClaim)] | length > 0)] | length') || return 1 + [[ "$count" -eq 0 ]] +} + +volumes_gone() { + local name + for name in "$@"; do + kubectl_cli get pv "$name" >/dev/null 2>&1 && return 1 + done + return 0 +} + +# PersistentVolumeClaims are backed by EBS volumes that outlive the cluster: once it is +# deleted, no CSI driver is left to delete them, and they stay billed as `available`. +# Stop every workload that mounts a claim (this also stops Pyroscope writing blocks into +# the stack's bucket, which would fail the bucket's deletion), delete the claims, and wait +# until their PersistentVolumes, and with them the EBS volumes, are gone. +cleanup_persistent_volumes() { + local pvc_json workload_json namespace kind name + local -a volume_names=() + pvc_json=$(kubectl_cli get pvc --all-namespaces -o json) || { + record_error "Could not enumerate PersistentVolumeClaims" + return + } + [[ "$(jq '.items | length' <<<"$pvc_json")" -gt 0 ]] || return 0 + mapfile -t volume_names < <(jq -r '.items[].spec.volumeName // empty' <<<"$pvc_json") + + workload_json=$(kubectl_cli get statefulsets,deployments --all-namespaces -o json) || { + record_error "Could not enumerate StatefulSets and Deployments" + return + } + log_info "Stopping every workload that mounts a PersistentVolumeClaim" + while IFS=$'\t' read -r namespace kind name; do + [[ -n "$namespace" && -n "$name" ]] || continue + kubectl_cli scale "$kind/$name" --namespace "$namespace" --replicas=0 >/dev/null \ + || record_error "Failed to scale $kind $namespace/$name to 0" + done < <(jq -r '.items[] + | select(((.spec.volumeClaimTemplates // []) | length > 0) + or ([.spec.template.spec.volumes[]? | select(.persistentVolumeClaim)] | length > 0)) + | [.metadata.namespace, (.kind | ascii_downcase), .metadata.name] | @tsv' <<<"$workload_json") + wait_until "pods that mount PersistentVolumeClaims to stop" pods_with_claims_gone || true + + log_info "Deleting every PersistentVolumeClaim from validated EKS cluster $EKS_CLUSTER_NAME" + while IFS=$'\t' read -r namespace name; do + [[ -n "$namespace" && -n "$name" ]] || continue + kubectl_cli delete pvc "$name" --namespace "$namespace" --ignore-not-found \ + --wait=true --timeout="${WAIT_SECONDS}s" >/dev/null || record_error "Failed to delete PersistentVolumeClaim $namespace/$name" + done < <(jq -r '.items[] | [.metadata.namespace,.metadata.name] | @tsv' <<<"$pvc_json") + if (( ${#volume_names[@]} > 0 )); then + wait_until "PersistentVolumes and their EBS volumes to be deleted" volumes_gone "${volume_names[@]}" \ + && log_info "PersistentVolumes deleted: ${volume_names[*]}" + fi +} + cleanup_eks_dependencies() { local namespace name association association_json association_tags ingress_json service_json association_output ingress_json=$(kubectl_cli get ingress --all-namespaces -o json) || { @@ -663,6 +723,7 @@ main() { fi confirm_cleanup + cleanup_persistent_volumes cleanup_eks_dependencies cleanup_vpc_load_balancers cleanup_runtime_ecr From 50e1ece8a44cad317e2c1a971a1d839666e1584f Mon Sep 17 00:00:00 2001 From: Yuriy Bezsonov Date: Tue, 10 Nov 2026 07:17:50 +0100 Subject: [PATCH 2/3] fix(java-performance): Restart a stalled Kyverno reports controller in the enforce block The reports controller of Kyverno v1.19.1 can stop producing reports without an error (kyverno#17848; the counter fix kyverno#17085 is only in the 1.20 release candidates). The enforce block of the content now restarts it after 90 s without a PolicyReport. The comment at the chart pin says to bump it and drop the restart once a stable release has the fix. ws-test regenerated from the content (the own-account setup blocks are test=false and only shift line numbers). --- .../deploy/java-performance/perf-profiler.sh | 3 +++ infra/scripts/ws-test/java-performance.sh | 14 ++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/infra/scripts/deploy/java-performance/perf-profiler.sh b/infra/scripts/deploy/java-performance/perf-profiler.sh index dafb42a7..4ab1525c 100755 --- a/infra/scripts/deploy/java-performance/perf-profiler.sh +++ b/infra/scripts/deploy/java-performance/perf-profiler.sh @@ -29,6 +29,9 @@ CLUSTER_NAME="${PREFIX:-workshop}-eks" # stamped into the sideca APP_UID="${APP_UID:-1000}" # UID the profiled app runs as PYROSCOPE_URL="${PYROSCOPE_URL:-http://pyroscope.monitoring:4040}" KYVERNO_CHART_VERSION="3.9.1" # Kyverno v1.19.1; MutatingPolicy v1beta1 needs >= this +# The reports controller can stall without an error (kyverno#17848; #17085 is only in 1.20 rcs), so the +# content's enforce block restarts it after 90 s without a PolicyReport. Once a stable release has the +# fix, bump this pin and remove the restart from ../java-performance/content/enforce/index.en.md. build() { log_info "Building and pushing perf-profiler image..." diff --git a/infra/scripts/ws-test/java-performance.sh b/infra/scripts/ws-test/java-performance.sh index a58010b7..b0b12a53 100755 --- a/infra/scripts/ws-test/java-performance.sh +++ b/infra/scripts/ws-test/java-performance.sh @@ -12,16 +12,16 @@ ws_source_environment '/etc/profile.d/workshop.sh' ws_begin_page 'Set up the environment' 10 'setup/index.en.md' -ws_run_block 1 'Check the environment' 'In the Shell terminal:' 25 26 'bash' '' <<'WS_TEST_BLOCK_10_1' +ws_run_block 1 'Check the environment' 'In the Shell terminal:' 65 66 'bash' '' <<'WS_TEST_BLOCK_10_1' curl -sf localhost:8090/actuator/health && echo kubectl -n unicorn-store-spring get pods WS_TEST_BLOCK_10_1 -ws_skip_block 2 'Check the environment' 'Check the environment' 31 32 'text' '' 'copy action disabled' +ws_skip_block 2 'Check the environment' 'Check the environment' 71 72 'text' '' 'copy action disabled' -ws_skip_block 3 'Start Claude Code' 'In the Claude terminal:' 47 47 'bash' '' 'interactive terminal UI' +ws_skip_block 3 'Start Claude Code' 'In the Claude terminal:' 87 87 'bash' '' 'interactive terminal UI' -ws_run_block 4 'Start Claude Code' 'With -p, Claude Code prints the answer and exits. Later calls with -c continue the same session:' 65 65 'bash' 'non-interactive' <<'WS_TEST_BLOCK_10_4' +ws_run_block 4 'Start Claude Code' 'With -p, Claude Code prints the answer and exits. Later calls with -c continue the same session:' 105 105 'bash' 'non-interactive' <<'WS_TEST_BLOCK_10_4' cd ~/environment && claude -p "List the MCP tools you can see from perf-sensor and eks-mcp, names only." WS_TEST_BLOCK_10_4 @@ -272,10 +272,12 @@ WS_TEST_BLOCK_500_1 ws_skip_block 2 'Review the policy' '- Item 10 assumption: the grace-period rule assumes Spring Boot'"'"'s 30 s default drain. The app sets no spring.lifecycle.timeout-per-shutdown-phase, so that holds today.' 93 105 'yaml' '' 'copy action disabled' -ws_run_block 3 'Apply and test' 'In the Shell terminal:' 117 129 'bash' '' <<'WS_TEST_BLOCK_500_3' +ws_run_block 3 'Apply and test' 'In the Shell terminal:' 117 131 'bash' '' <<'WS_TEST_BLOCK_500_3' kubectl apply -f ~/environment/unicorn-store-spring/k8s/policy.yaml # the background scan reports the running Deployment +timeout 90 sh -c 'until kubectl -n unicorn-store-spring get policyreport -o name | grep -q .; do sleep 5; done' \ + || kubectl -n kyverno rollout restart deploy/kyverno-reports-controller until kubectl -n unicorn-store-spring get policyreport -o name | grep -q .; do sleep 5; done kubectl -n unicorn-store-spring get policyreport @@ -288,7 +290,7 @@ git -C ~/environment/unicorn-store-spring add -A \ && git -C ~/environment/unicorn-store-spring commit -q -m "kyverno policy" WS_TEST_BLOCK_500_3 -ws_skip_block 4 'Apply and test' 'Apply and test' 134 138 'text' '' 'copy action disabled' +ws_skip_block 4 'Apply and test' 'Apply and test' 136 140 'text' '' 'copy action disabled' ws_end_page From 39c14d0137ec53b3d75335a791ee0ec783704435 Mon Sep 17 00:00:00 2001 From: Yuriy Bezsonov Date: Tue, 10 Nov 2026 07:22:36 +0100 Subject: [PATCH 3/3] fix(cdk): Log groups belong to the stack and are deleted with it Every Lambda function, the CodeBuild project and the IDE bootstrap log to a log group the stack owns (Lambda.stackLogGroup: generated name, 7 days, RemovalPolicy.DESTROY). Before, Lambda, CodeBuild and the CloudWatch agent created their own groups (/aws/lambda/..., /aws/codebuild/..., workshop-ide-bootstrap-), and nine of them outlived the stack. CloudFormation deletes each group after the resource that writes to it, so the logs of the custom resources' Delete calls go too. A generated name keeps a group left by an earlier deployment from blocking the create. Only the java-performance stack is regenerated (GIT_BRANCH=main); the other workshops pick this up at their next generation. --- .../com/constructs/CfnPreDeleteCleanup.java | 5 +- .../java/sample/com/constructs/CodeBuild.java | 5 + .../main/java/sample/com/constructs/Ide.java | 6 +- .../java/sample/com/constructs/Lambda.java | 17 ++ .../java/sample/com/constructs/Unicorn.java | 1 + infra/cdk/src/main/resources/userdata.sh | 4 +- infra/cfn/java-performance-stack.yaml | 181 +++++++++++++++++- 7 files changed, 205 insertions(+), 14 deletions(-) diff --git a/infra/cdk/src/main/java/sample/com/constructs/CfnPreDeleteCleanup.java b/infra/cdk/src/main/java/sample/com/constructs/CfnPreDeleteCleanup.java index 065b91ff..60c40389 100644 --- a/infra/cdk/src/main/java/sample/com/constructs/CfnPreDeleteCleanup.java +++ b/infra/cdk/src/main/java/sample/com/constructs/CfnPreDeleteCleanup.java @@ -21,8 +21,8 @@ /** * CfnPreDeleteCleanup construct for cleaning up resources before stack deletion. * - GuardDuty VPC endpoints that block VPC deletion - * - CloudWatch log groups with workshop- or unicornstore- prefix * - S3 bucket contents for workshop- buckets + * Its own log group belongs to the stack (Lambda.stackLogGroup), like every function's. */ public class CfnPreDeleteCleanup extends Construct { @@ -120,7 +120,8 @@ public CfnPreDeleteCleanup(final Construct scope, final String id, final CfnPreD .code(Code.fromInline(loadFile("/lambda/cfn-pre-delete-cleanup.py"))) .role(lambdaRole) .timeout(Duration.minutes(10)) - .description("Cleanup VPC endpoints, CloudWatch logs, and S3 buckets before stack deletion") + .logGroup(Lambda.stackLogGroup(this, "FunctionLogGroup")) + .description("Cleanup VPC endpoints and S3 buckets before stack deletion") .build(); // Create Custom Resource that triggers cleanup on stack delete diff --git a/infra/cdk/src/main/java/sample/com/constructs/CodeBuild.java b/infra/cdk/src/main/java/sample/com/constructs/CodeBuild.java index 5a990c75..a9369625 100644 --- a/infra/cdk/src/main/java/sample/com/constructs/CodeBuild.java +++ b/infra/cdk/src/main/java/sample/com/constructs/CodeBuild.java @@ -103,6 +103,11 @@ public CodeBuild(final Construct scope, final String id, final CodeBuildProps pr .buildSpec(BuildSpec.fromObjectToYaml(new Yaml().load(props.getBuildSpec()))) .environmentVariables(codeBuildEnvVars) .timeout(props.getTimeout()) + .logging(LoggingOptions.builder() + .cloudWatch(CloudWatchLoggingOptions.builder() + .logGroup(Lambda.stackLogGroup(this, "ProjectLogGroup")) + .build()) + .build()) .build(); lambdaRole.addToPolicy(PolicyStatement.Builder.create() diff --git a/infra/cdk/src/main/java/sample/com/constructs/Ide.java b/infra/cdk/src/main/java/sample/com/constructs/Ide.java index 22172895..c63288a1 100644 --- a/infra/cdk/src/main/java/sample/com/constructs/Ide.java +++ b/infra/cdk/src/main/java/sample/com/constructs/Ide.java @@ -424,7 +424,9 @@ public Ide(final Construct scope, final String id, final IdeProps props) { String gitBranch = props.getGitBranch(); String templateType = props.getTemplateType(); - // Load UserData from external template file and substitute variables + // Load UserData from external template file and substitute variables. The bootstrap log + // group belongs to the stack, so it is deleted with it (after the instance). + var bootstrapLogGroup = Lambda.stackLogGroup(this, "BootstrapLogGroup"); String userDataTemplate = loadFile("/userdata.sh"); String userDataContent = userDataTemplate .replace("${GIT_BRANCH}", gitBranch) @@ -436,6 +438,7 @@ public Ide(final Construct scope, final String id, final IdeProps props) { .replace("${ARCH}", props.getIdeArch().getUnameValue()) .replace("${IDE_TYPE}", props.getIdeType().getScriptName()) .replace("${WAIT_CONDITION_HANDLE_URL}", waitHandle.getRef()) + .replace("${LOG_GROUP_NAME}", bootstrapLogGroup.getLogGroupName()) .replace("${PREFIX}", prefix); userData.addCommands(userDataContent); @@ -578,6 +581,7 @@ private String getIdePassword(String instanceName) { .runtime(Runtime.PYTHON_3_13) .timeout(Duration.minutes(3)) .functionName(prefix + "-ide-password") + .logGroup(Lambda.stackLogGroup(this, "PasswordFunctionLogGroup")) .build(); ideSecretsManagerPassword.grantRead(passwordFunction); diff --git a/infra/cdk/src/main/java/sample/com/constructs/Lambda.java b/infra/cdk/src/main/java/sample/com/constructs/Lambda.java index f8d767a5..e450f402 100644 --- a/infra/cdk/src/main/java/sample/com/constructs/Lambda.java +++ b/infra/cdk/src/main/java/sample/com/constructs/Lambda.java @@ -1,10 +1,13 @@ package sample.com.constructs; import software.amazon.awscdk.Duration; +import software.amazon.awscdk.RemovalPolicy; import software.amazon.awscdk.services.iam.Role; import software.amazon.awscdk.services.lambda.Code; import software.amazon.awscdk.services.lambda.Function; import software.amazon.awscdk.services.lambda.Runtime; +import software.amazon.awscdk.services.logs.LogGroup; +import software.amazon.awscdk.services.logs.RetentionDays; import software.constructs.Construct; import java.io.IOException; @@ -58,6 +61,20 @@ public Lambda(final Construct scope, final String id, final LambdaProps props, f .timeout(props.getTimeout()) .functionName(props.getFunctionName()) .role(props.getRole()) + .logGroup(stackLogGroup(this, "LogGroup")) + .build(); + } + + /** + * A log group the stack owns: CloudFormation deletes it after the function that writes + * to it, so the logs of a custom resource's Delete call go too. Without it, Lambda + * creates /aws/lambda/ itself and the group outlives the stack. The name is + * generated, so a group left by an earlier deployment can't block the create. + */ + public static LogGroup stackLogGroup(final Construct scope, final String id) { + return LogGroup.Builder.create(scope, id) + .retention(RetentionDays.ONE_WEEK) + .removalPolicy(RemovalPolicy.DESTROY) .build(); } diff --git a/infra/cdk/src/main/java/sample/com/constructs/Unicorn.java b/infra/cdk/src/main/java/sample/com/constructs/Unicorn.java index a479d6a0..652f137e 100644 --- a/infra/cdk/src/main/java/sample/com/constructs/Unicorn.java +++ b/infra/cdk/src/main/java/sample/com/constructs/Unicorn.java @@ -82,6 +82,7 @@ private void createDatabaseSetup(UnicornProps props) { .timeout(Duration.minutes(3)) .vpc(props.getVpc()) .securityGroups(List.of(database.getDatabaseSecurityGroup())) + .logGroup(Lambda.stackLogGroup(this, "UnicornStoreDatabaseSetupLogGroup")) .build(); // Grant permissions to setup function diff --git a/infra/cdk/src/main/resources/userdata.sh b/infra/cdk/src/main/resources/userdata.sh index 200d63ed..dee3b9b5 100644 --- a/infra/cdk/src/main/resources/userdata.sh +++ b/infra/cdk/src/main/resources/userdata.sh @@ -18,8 +18,8 @@ export IDE_TYPE="${IDE_TYPE}" export WAIT_CONDITION_HANDLE_URL="${WAIT_CONDITION_HANDLE_URL}" export PREFIX="${PREFIX}" -# Setup logging - use PREFIX for log group name -LOG_GROUP_NAME="${PREFIX}-ide-bootstrap-$(date +%Y%m%d-%H%M%S)" +# Setup logging into the stack's bootstrap log group (Ide.java), deleted with the stack +LOG_GROUP_NAME="${LOG_GROUP_NAME}" echo "Bootstrap logs will be written to CloudWatch log group: $LOG_GROUP_NAME" # Install CloudWatch agent for logging diff --git a/infra/cfn/java-performance-stack.yaml b/infra/cfn/java-performance-stack.yaml index 88b001ed..380a0082 100644 --- a/infra/cfn/java-performance-stack.yaml +++ b/infra/cfn/java-performance-stack.yaml @@ -345,9 +345,12 @@ Resources: raise print(f"Timeout waiting for endpoint deletion after {max_wait}s") - Description: Cleanup VPC endpoints, CloudWatch logs, and S3 buckets before stack deletion + Description: Cleanup VPC endpoints and S3 buckets before stack deletion FunctionName: workshop-cfn-pre-delete-cleanup Handler: index.lambda_handler + LoggingConfig: + LogGroup: + Ref: CfnPreDeleteCleanupFunctionLogGroupE77CC8F5 Role: Fn::GetAtt: - CfnPreDeleteCleanupRole2BF62B3D @@ -363,6 +366,20 @@ Resources: Value: cloudformation Timeout: 600 Type: AWS::Lambda::Function + CfnPreDeleteCleanupFunctionLogGroupE77CC8F5: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete CfnPreDeleteCleanupRole2BF62B3D: Properties: AssumeRolePolicyDocument: @@ -490,7 +507,7 @@ Resources: - CodeBuildReportLambdaFunctionA3C396F7 - CodeBuildStartLambdaFunction8349284F Properties: - ContentHash: "1792832530668" + ContentHash: "1794291664247" ProjectName: Ref: CodeBuildProjectA0FF5539 ServiceToken: @@ -653,16 +670,21 @@ Resources: Environment: ComputeType: BUILD_GENERAL1_MEDIUM EnvironmentVariables: - - Name: TEMPLATE_TYPE - Type: PLAINTEXT - Value: java-performance - Name: GIT_BRANCH Type: PLAINTEXT Value: main + - Name: TEMPLATE_TYPE + Type: PLAINTEXT + Value: java-performance Image: aws/codebuild/amazonlinux2-x86_64-standard:5.0 ImagePullCredentialsType: CODEBUILD PrivilegedMode: false Type: LINUX_CONTAINER + LogsConfig: + CloudWatchLogs: + GroupName: + Ref: CodeBuildProjectLogGroup8E24F361 + Status: ENABLED Name: workshop-setup ServiceRole: Fn::GetAtt: @@ -697,6 +719,20 @@ Resources: Value: cloudformation TimeoutInMinutes: 30 Type: AWS::CodeBuild::Project + CodeBuildProjectLogGroup8E24F361: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete CodeBuildReportLambdaFunctionA3C396F7: DependsOn: - CodeBuildLambdaRoleDefaultPolicyFB35F0AF @@ -815,6 +851,9 @@ Resources: Ref: CodeBuildPendingBuilds19869454 FunctionName: workshop-setup-report Handler: index.lambda_handler + LoggingConfig: + LogGroup: + Ref: CodeBuildReportLambdaLogGroup7EC4A659 Role: Fn::GetAtt: - CodeBuildLambdaRole655C06B4 @@ -830,6 +869,20 @@ Resources: Value: cloudformation Timeout: 120 Type: AWS::Lambda::Function + CodeBuildReportLambdaLogGroup7EC4A659: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete CodeBuildRoleDefaultPolicy196BAF24: Properties: PolicyDocument: @@ -841,6 +894,14 @@ Resources: - elasticloadbalancing.amazonaws.com Effect: Allow Resource: arn:aws:iam::*:role/aws-service-role/* + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: + Fn::GetAtt: + - CodeBuildProjectLogGroup8E24F361 + - Arn - Action: - logs:CreateLogGroup - logs:CreateLogStream @@ -993,6 +1054,9 @@ Resources: Ref: CodeBuildPendingBuilds19869454 FunctionName: workshop-setup-start Handler: index.lambda_handler + LoggingConfig: + LogGroup: + Ref: CodeBuildStartLambdaLogGroup8C94F258 Role: Fn::GetAtt: - CodeBuildLambdaRole655C06B4 @@ -1008,6 +1072,20 @@ Resources: Value: cloudformation Timeout: 120 Type: AWS::Lambda::Function + CodeBuildStartLambdaLogGroup8C94F258: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete DatabaseCluster5B53A178: DeletionPolicy: Delete Properties: @@ -1527,6 +1605,20 @@ Resources: - Key: WorkshopOwner Value: cloudformation Type: AWS::EKS::Addon + IdeBootstrapLogGroupE41C8C91: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete IdeDistribution042A6660: DeletionPolicy: Delete DependsOn: @@ -1679,8 +1771,11 @@ Resources: " export PREFIX="workshop" - # Setup logging - use PREFIX for log group name - LOG_GROUP_NAME="workshop-ide-bootstrap-$(date +%Y%m%d-%H%M%S)" + # Setup logging into the stack's bootstrap log group (Ide.java), deleted with the stack + LOG_GROUP_NAME=" + - Ref: IdeBootstrapLogGroupE41C8C91 + - |- + " echo "Bootstrap logs will be written to CloudWatch log group: $LOG_GROUP_NAME" # Install CloudWatch agent for logging @@ -1955,6 +2050,9 @@ Resources: cfnresponse.send(event, context, status, responseData, physical_id) FunctionName: workshop-ide-launcher Handler: index.lambda_handler + LoggingConfig: + LogGroup: + Ref: IdeInstanceLauncherLogGroup5E716EFF Role: Fn::GetAtt: - IdeLambdaRole82392143 @@ -1970,6 +2068,20 @@ Resources: Value: cloudformation Timeout: 300 Type: AWS::Lambda::Function + IdeInstanceLauncherLogGroup5E716EFF: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete IdeInstanceProfile61B92038: Properties: InstanceProfileName: workshop-ide-instance-profile @@ -2144,6 +2256,9 @@ Resources: cfnresponse.send(event, context, status, responseData, physical_id) FunctionName: workshop-ide-password Handler: index.lambda_handler + LoggingConfig: + LogGroup: + Ref: IdePasswordFunctionLogGroup75ECD53B Role: Fn::GetAtt: - IdePasswordFunctionServiceRole971421B6 @@ -2159,6 +2274,20 @@ Resources: Value: cloudformation Timeout: 180 Type: AWS::Lambda::Function + IdePasswordFunctionLogGroup75ECD53B: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete IdePasswordFunctionServiceRole971421B6: Properties: AssumeRolePolicyDocument: @@ -2547,6 +2676,9 @@ Resources: cfnresponse.send(event, context, status, responseData, 'CustomResourcePhysicalID') FunctionName: workshop-ide-prefixlist Handler: index.lambda_handler + LoggingConfig: + LogGroup: + Ref: IdePrefixListLookupLogGroup8650FFBA Role: Fn::GetAtt: - IdeLambdaRole82392143 @@ -2562,6 +2694,20 @@ Resources: Value: cloudformation Timeout: 180 Type: AWS::Lambda::Function + IdePrefixListLookupLogGroup8650FFBA: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete IdePrefixListResourceD2EAC007: DeletionPolicy: Delete Properties: @@ -2923,6 +3069,9 @@ Resources: cfnresponse.send(event, context, status, responseData, 'CustomResourcePhysicalID') FunctionName: unicornstore-database-setup Handler: index.lambda_handler + LoggingConfig: + LogGroup: + Ref: UnicornUnicornStoreDatabaseSetupLogGroup79C2BE46 Role: Fn::GetAtt: - UnicornUnicornStoreDatabaseSetupFunctionServiceRole61942171 @@ -3019,6 +3168,20 @@ Resources: Roles: - Ref: UnicornUnicornStoreDatabaseSetupFunctionServiceRole61942171 Type: AWS::IAM::Policy + UnicornUnicornStoreDatabaseSetupLogGroup79C2BE46: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete UnicornUnicornStoreDatabaseSetupResourceCDB8BDDF: DeletionPolicy: Delete DependsOn: @@ -3498,7 +3661,7 @@ Resources: - Ref: AWS::AccountId - "-" - Ref: AWS::Region - - "-20261024110210" + - "-20261110072104" PublicAccessBlockConfiguration: BlockPublicAcls: true BlockPublicPolicy: true @@ -3582,7 +3745,7 @@ Resources: - Ref: AWS::AccountId - "-" - Ref: AWS::Region - - "-20261024110210" + - "-20261110072104" LoggingConfiguration: DestinationBucketName: Ref: WorkshopBucketAccessLogs476BAB88