diff --git a/infra/cdk/src/main/java/sample/com/constructs/CfnPreDeleteCleanup.java b/infra/cdk/src/main/java/sample/com/constructs/CfnPreDeleteCleanup.java index 065b91ff..60c40389 100644 --- a/infra/cdk/src/main/java/sample/com/constructs/CfnPreDeleteCleanup.java +++ b/infra/cdk/src/main/java/sample/com/constructs/CfnPreDeleteCleanup.java @@ -21,8 +21,8 @@ /** * CfnPreDeleteCleanup construct for cleaning up resources before stack deletion. * - GuardDuty VPC endpoints that block VPC deletion - * - CloudWatch log groups with workshop- or unicornstore- prefix * - S3 bucket contents for workshop- buckets + * Its own log group belongs to the stack (Lambda.stackLogGroup), like every function's. */ public class CfnPreDeleteCleanup extends Construct { @@ -120,7 +120,8 @@ public CfnPreDeleteCleanup(final Construct scope, final String id, final CfnPreD .code(Code.fromInline(loadFile("/lambda/cfn-pre-delete-cleanup.py"))) .role(lambdaRole) .timeout(Duration.minutes(10)) - .description("Cleanup VPC endpoints, CloudWatch logs, and S3 buckets before stack deletion") + .logGroup(Lambda.stackLogGroup(this, "FunctionLogGroup")) + .description("Cleanup VPC endpoints and S3 buckets before stack deletion") .build(); // Create Custom Resource that triggers cleanup on stack delete diff --git a/infra/cdk/src/main/java/sample/com/constructs/CodeBuild.java b/infra/cdk/src/main/java/sample/com/constructs/CodeBuild.java index 5a990c75..a9369625 100644 --- a/infra/cdk/src/main/java/sample/com/constructs/CodeBuild.java +++ b/infra/cdk/src/main/java/sample/com/constructs/CodeBuild.java @@ -103,6 +103,11 @@ public CodeBuild(final Construct scope, final String id, final CodeBuildProps pr .buildSpec(BuildSpec.fromObjectToYaml(new Yaml().load(props.getBuildSpec()))) .environmentVariables(codeBuildEnvVars) .timeout(props.getTimeout()) + .logging(LoggingOptions.builder() + .cloudWatch(CloudWatchLoggingOptions.builder() + .logGroup(Lambda.stackLogGroup(this, "ProjectLogGroup")) + .build()) + .build()) .build(); lambdaRole.addToPolicy(PolicyStatement.Builder.create() diff --git a/infra/cdk/src/main/java/sample/com/constructs/Ide.java b/infra/cdk/src/main/java/sample/com/constructs/Ide.java index 22172895..c63288a1 100644 --- a/infra/cdk/src/main/java/sample/com/constructs/Ide.java +++ b/infra/cdk/src/main/java/sample/com/constructs/Ide.java @@ -424,7 +424,9 @@ public Ide(final Construct scope, final String id, final IdeProps props) { String gitBranch = props.getGitBranch(); String templateType = props.getTemplateType(); - // Load UserData from external template file and substitute variables + // Load UserData from external template file and substitute variables. The bootstrap log + // group belongs to the stack, so it is deleted with it (after the instance). + var bootstrapLogGroup = Lambda.stackLogGroup(this, "BootstrapLogGroup"); String userDataTemplate = loadFile("/userdata.sh"); String userDataContent = userDataTemplate .replace("${GIT_BRANCH}", gitBranch) @@ -436,6 +438,7 @@ public Ide(final Construct scope, final String id, final IdeProps props) { .replace("${ARCH}", props.getIdeArch().getUnameValue()) .replace("${IDE_TYPE}", props.getIdeType().getScriptName()) .replace("${WAIT_CONDITION_HANDLE_URL}", waitHandle.getRef()) + .replace("${LOG_GROUP_NAME}", bootstrapLogGroup.getLogGroupName()) .replace("${PREFIX}", prefix); userData.addCommands(userDataContent); @@ -578,6 +581,7 @@ private String getIdePassword(String instanceName) { .runtime(Runtime.PYTHON_3_13) .timeout(Duration.minutes(3)) .functionName(prefix + "-ide-password") + .logGroup(Lambda.stackLogGroup(this, "PasswordFunctionLogGroup")) .build(); ideSecretsManagerPassword.grantRead(passwordFunction); diff --git a/infra/cdk/src/main/java/sample/com/constructs/Lambda.java b/infra/cdk/src/main/java/sample/com/constructs/Lambda.java index f8d767a5..e450f402 100644 --- a/infra/cdk/src/main/java/sample/com/constructs/Lambda.java +++ b/infra/cdk/src/main/java/sample/com/constructs/Lambda.java @@ -1,10 +1,13 @@ package sample.com.constructs; import software.amazon.awscdk.Duration; +import software.amazon.awscdk.RemovalPolicy; import software.amazon.awscdk.services.iam.Role; import software.amazon.awscdk.services.lambda.Code; import software.amazon.awscdk.services.lambda.Function; import software.amazon.awscdk.services.lambda.Runtime; +import software.amazon.awscdk.services.logs.LogGroup; +import software.amazon.awscdk.services.logs.RetentionDays; import software.constructs.Construct; import java.io.IOException; @@ -58,6 +61,20 @@ public Lambda(final Construct scope, final String id, final LambdaProps props, f .timeout(props.getTimeout()) .functionName(props.getFunctionName()) .role(props.getRole()) + .logGroup(stackLogGroup(this, "LogGroup")) + .build(); + } + + /** + * A log group the stack owns: CloudFormation deletes it after the function that writes + * to it, so the logs of a custom resource's Delete call go too. Without it, Lambda + * creates /aws/lambda/ itself and the group outlives the stack. The name is + * generated, so a group left by an earlier deployment can't block the create. + */ + public static LogGroup stackLogGroup(final Construct scope, final String id) { + return LogGroup.Builder.create(scope, id) + .retention(RetentionDays.ONE_WEEK) + .removalPolicy(RemovalPolicy.DESTROY) .build(); } diff --git a/infra/cdk/src/main/java/sample/com/constructs/Unicorn.java b/infra/cdk/src/main/java/sample/com/constructs/Unicorn.java index a479d6a0..652f137e 100644 --- a/infra/cdk/src/main/java/sample/com/constructs/Unicorn.java +++ b/infra/cdk/src/main/java/sample/com/constructs/Unicorn.java @@ -82,6 +82,7 @@ private void createDatabaseSetup(UnicornProps props) { .timeout(Duration.minutes(3)) .vpc(props.getVpc()) .securityGroups(List.of(database.getDatabaseSecurityGroup())) + .logGroup(Lambda.stackLogGroup(this, "UnicornStoreDatabaseSetupLogGroup")) .build(); // Grant permissions to setup function diff --git a/infra/cdk/src/main/resources/userdata.sh b/infra/cdk/src/main/resources/userdata.sh index 200d63ed..dee3b9b5 100644 --- a/infra/cdk/src/main/resources/userdata.sh +++ b/infra/cdk/src/main/resources/userdata.sh @@ -18,8 +18,8 @@ export IDE_TYPE="${IDE_TYPE}" export WAIT_CONDITION_HANDLE_URL="${WAIT_CONDITION_HANDLE_URL}" export PREFIX="${PREFIX}" -# Setup logging - use PREFIX for log group name -LOG_GROUP_NAME="${PREFIX}-ide-bootstrap-$(date +%Y%m%d-%H%M%S)" +# Setup logging into the stack's bootstrap log group (Ide.java), deleted with the stack +LOG_GROUP_NAME="${LOG_GROUP_NAME}" echo "Bootstrap logs will be written to CloudWatch log group: $LOG_GROUP_NAME" # Install CloudWatch agent for logging diff --git a/infra/cfn/java-performance-stack.yaml b/infra/cfn/java-performance-stack.yaml index 88b001ed..380a0082 100644 --- a/infra/cfn/java-performance-stack.yaml +++ b/infra/cfn/java-performance-stack.yaml @@ -345,9 +345,12 @@ Resources: raise print(f"Timeout waiting for endpoint deletion after {max_wait}s") - Description: Cleanup VPC endpoints, CloudWatch logs, and S3 buckets before stack deletion + Description: Cleanup VPC endpoints and S3 buckets before stack deletion FunctionName: workshop-cfn-pre-delete-cleanup Handler: index.lambda_handler + LoggingConfig: + LogGroup: + Ref: CfnPreDeleteCleanupFunctionLogGroupE77CC8F5 Role: Fn::GetAtt: - CfnPreDeleteCleanupRole2BF62B3D @@ -363,6 +366,20 @@ Resources: Value: cloudformation Timeout: 600 Type: AWS::Lambda::Function + CfnPreDeleteCleanupFunctionLogGroupE77CC8F5: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete CfnPreDeleteCleanupRole2BF62B3D: Properties: AssumeRolePolicyDocument: @@ -490,7 +507,7 @@ Resources: - CodeBuildReportLambdaFunctionA3C396F7 - CodeBuildStartLambdaFunction8349284F Properties: - ContentHash: "1792832530668" + ContentHash: "1794291664247" ProjectName: Ref: CodeBuildProjectA0FF5539 ServiceToken: @@ -653,16 +670,21 @@ Resources: Environment: ComputeType: BUILD_GENERAL1_MEDIUM EnvironmentVariables: - - Name: TEMPLATE_TYPE - Type: PLAINTEXT - Value: java-performance - Name: GIT_BRANCH Type: PLAINTEXT Value: main + - Name: TEMPLATE_TYPE + Type: PLAINTEXT + Value: java-performance Image: aws/codebuild/amazonlinux2-x86_64-standard:5.0 ImagePullCredentialsType: CODEBUILD PrivilegedMode: false Type: LINUX_CONTAINER + LogsConfig: + CloudWatchLogs: + GroupName: + Ref: CodeBuildProjectLogGroup8E24F361 + Status: ENABLED Name: workshop-setup ServiceRole: Fn::GetAtt: @@ -697,6 +719,20 @@ Resources: Value: cloudformation TimeoutInMinutes: 30 Type: AWS::CodeBuild::Project + CodeBuildProjectLogGroup8E24F361: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete CodeBuildReportLambdaFunctionA3C396F7: DependsOn: - CodeBuildLambdaRoleDefaultPolicyFB35F0AF @@ -815,6 +851,9 @@ Resources: Ref: CodeBuildPendingBuilds19869454 FunctionName: workshop-setup-report Handler: index.lambda_handler + LoggingConfig: + LogGroup: + Ref: CodeBuildReportLambdaLogGroup7EC4A659 Role: Fn::GetAtt: - CodeBuildLambdaRole655C06B4 @@ -830,6 +869,20 @@ Resources: Value: cloudformation Timeout: 120 Type: AWS::Lambda::Function + CodeBuildReportLambdaLogGroup7EC4A659: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete CodeBuildRoleDefaultPolicy196BAF24: Properties: PolicyDocument: @@ -841,6 +894,14 @@ Resources: - elasticloadbalancing.amazonaws.com Effect: Allow Resource: arn:aws:iam::*:role/aws-service-role/* + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: + Fn::GetAtt: + - CodeBuildProjectLogGroup8E24F361 + - Arn - Action: - logs:CreateLogGroup - logs:CreateLogStream @@ -993,6 +1054,9 @@ Resources: Ref: CodeBuildPendingBuilds19869454 FunctionName: workshop-setup-start Handler: index.lambda_handler + LoggingConfig: + LogGroup: + Ref: CodeBuildStartLambdaLogGroup8C94F258 Role: Fn::GetAtt: - CodeBuildLambdaRole655C06B4 @@ -1008,6 +1072,20 @@ Resources: Value: cloudformation Timeout: 120 Type: AWS::Lambda::Function + CodeBuildStartLambdaLogGroup8C94F258: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete DatabaseCluster5B53A178: DeletionPolicy: Delete Properties: @@ -1527,6 +1605,20 @@ Resources: - Key: WorkshopOwner Value: cloudformation Type: AWS::EKS::Addon + IdeBootstrapLogGroupE41C8C91: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete IdeDistribution042A6660: DeletionPolicy: Delete DependsOn: @@ -1679,8 +1771,11 @@ Resources: " export PREFIX="workshop" - # Setup logging - use PREFIX for log group name - LOG_GROUP_NAME="workshop-ide-bootstrap-$(date +%Y%m%d-%H%M%S)" + # Setup logging into the stack's bootstrap log group (Ide.java), deleted with the stack + LOG_GROUP_NAME=" + - Ref: IdeBootstrapLogGroupE41C8C91 + - |- + " echo "Bootstrap logs will be written to CloudWatch log group: $LOG_GROUP_NAME" # Install CloudWatch agent for logging @@ -1955,6 +2050,9 @@ Resources: cfnresponse.send(event, context, status, responseData, physical_id) FunctionName: workshop-ide-launcher Handler: index.lambda_handler + LoggingConfig: + LogGroup: + Ref: IdeInstanceLauncherLogGroup5E716EFF Role: Fn::GetAtt: - IdeLambdaRole82392143 @@ -1970,6 +2068,20 @@ Resources: Value: cloudformation Timeout: 300 Type: AWS::Lambda::Function + IdeInstanceLauncherLogGroup5E716EFF: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete IdeInstanceProfile61B92038: Properties: InstanceProfileName: workshop-ide-instance-profile @@ -2144,6 +2256,9 @@ Resources: cfnresponse.send(event, context, status, responseData, physical_id) FunctionName: workshop-ide-password Handler: index.lambda_handler + LoggingConfig: + LogGroup: + Ref: IdePasswordFunctionLogGroup75ECD53B Role: Fn::GetAtt: - IdePasswordFunctionServiceRole971421B6 @@ -2159,6 +2274,20 @@ Resources: Value: cloudformation Timeout: 180 Type: AWS::Lambda::Function + IdePasswordFunctionLogGroup75ECD53B: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete IdePasswordFunctionServiceRole971421B6: Properties: AssumeRolePolicyDocument: @@ -2547,6 +2676,9 @@ Resources: cfnresponse.send(event, context, status, responseData, 'CustomResourcePhysicalID') FunctionName: workshop-ide-prefixlist Handler: index.lambda_handler + LoggingConfig: + LogGroup: + Ref: IdePrefixListLookupLogGroup8650FFBA Role: Fn::GetAtt: - IdeLambdaRole82392143 @@ -2562,6 +2694,20 @@ Resources: Value: cloudformation Timeout: 180 Type: AWS::Lambda::Function + IdePrefixListLookupLogGroup8650FFBA: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete IdePrefixListResourceD2EAC007: DeletionPolicy: Delete Properties: @@ -2923,6 +3069,9 @@ Resources: cfnresponse.send(event, context, status, responseData, 'CustomResourcePhysicalID') FunctionName: unicornstore-database-setup Handler: index.lambda_handler + LoggingConfig: + LogGroup: + Ref: UnicornUnicornStoreDatabaseSetupLogGroup79C2BE46 Role: Fn::GetAtt: - UnicornUnicornStoreDatabaseSetupFunctionServiceRole61942171 @@ -3019,6 +3168,20 @@ Resources: Roles: - Ref: UnicornUnicornStoreDatabaseSetupFunctionServiceRole61942171 Type: AWS::IAM::Policy + UnicornUnicornStoreDatabaseSetupLogGroup79C2BE46: + DeletionPolicy: Delete + Properties: + RetentionInDays: 7 + Tags: + - Key: WorkshopDeploymentId + Value: + Ref: AWS::StackId + - Key: WorkshopId + Value: java-performance + - Key: WorkshopOwner + Value: cloudformation + Type: AWS::Logs::LogGroup + UpdateReplacePolicy: Delete UnicornUnicornStoreDatabaseSetupResourceCDB8BDDF: DeletionPolicy: Delete DependsOn: @@ -3498,7 +3661,7 @@ Resources: - Ref: AWS::AccountId - "-" - Ref: AWS::Region - - "-20261024110210" + - "-20261110072104" PublicAccessBlockConfiguration: BlockPublicAcls: true BlockPublicPolicy: true @@ -3582,7 +3745,7 @@ Resources: - Ref: AWS::AccountId - "-" - Ref: AWS::Region - - "-20261024110210" + - "-20261110072104" LoggingConfiguration: DestinationBucketName: Ref: WorkshopBucketAccessLogs476BAB88 diff --git a/infra/scripts/cleanup/java-on-amazon-eks.sh b/infra/scripts/cleanup/java-on-amazon-eks.sh index fd860183..2397bd22 100755 --- a/infra/scripts/cleanup/java-on-amazon-eks.sh +++ b/infra/scripts/cleanup/java-on-amazon-eks.sh @@ -42,6 +42,7 @@ declare -a SECURITY_GROUPS=() declare -a STACK_RESOURCE_IDS=() log_info() { printf '[INFO] %s\n' "$*"; } +log_success() { printf '[OK] %s\n' "$*"; } log_warn() { printf '[WARN] %s\n' "$*" >&2; } log_error() { printf '[ERROR] %s\n' "$*" >&2; } record_error() { log_error "$*"; ERRORS=$((ERRORS + 1)); } @@ -399,9 +400,10 @@ discover_resources() { } print_plan() { - local ingress_count service_count association_count + local ingress_count service_count association_count pvc_count ingress_count=$(kubectl_cli get ingress --all-namespaces -o json | jq '.items | length') service_count=$(kubectl_cli get service --all-namespaces -o json | jq '[.items[] | select(.spec.type == "LoadBalancer")] | length') + pvc_count=$(kubectl_cli get pvc --all-namespaces -o json | jq '.items | length') association_count=$(aws_cli eks list-pod-identity-associations --cluster-name "$EKS_CLUSTER_NAME" \ --query 'length(associations)' --output text) @@ -409,6 +411,7 @@ print_plan() { printf ' Kubernetes Ingresses: %s\n' "$ingress_count" printf ' Kubernetes LoadBalancer Services: %s\n' "$service_count" printf ' EKS Pod Identity associations: %s\n' "$association_count" + printf ' Kubernetes volume claims: %s\n' "$pvc_count" printf ' Load balancers in VPC: %d\n' "${#RUNTIME_LOAD_BALANCERS[@]}" printf ' Target groups in VPC: %d\n' "${#RUNTIME_TARGET_GROUPS[@]}" printf ' Tagged runtime ECR repositories: %d\n' "${#RUNTIME_ECR_REPOSITORIES[@]}" @@ -434,6 +437,63 @@ kubernetes_dependencies_gone() { [[ "$ingress_count" -eq 0 && "$service_count" -eq 0 ]] } +pods_with_claims_gone() { + local count + count=$(kubectl_cli get pods --all-namespaces -o json 2>/dev/null \ + | jq '[.items[] | select([.spec.volumes[]? | select(.persistentVolumeClaim)] | length > 0)] | length') || return 1 + [[ "$count" -eq 0 ]] +} + +volumes_gone() { + local name + for name in "$@"; do + kubectl_cli get pv "$name" >/dev/null 2>&1 && return 1 + done + return 0 +} + +# PersistentVolumeClaims are backed by EBS volumes that outlive the cluster: once it is +# deleted, no CSI driver is left to delete them, and they stay billed as `available`. +# Stop every workload that mounts a claim (this also stops Pyroscope writing blocks into +# the stack's bucket, which would fail the bucket's deletion), delete the claims, and wait +# until their PersistentVolumes, and with them the EBS volumes, are gone. +cleanup_persistent_volumes() { + local pvc_json workload_json namespace kind name + local -a volume_names=() + pvc_json=$(kubectl_cli get pvc --all-namespaces -o json) || { + record_error "Could not enumerate PersistentVolumeClaims" + return + } + [[ "$(jq '.items | length' <<<"$pvc_json")" -gt 0 ]] || return 0 + mapfile -t volume_names < <(jq -r '.items[].spec.volumeName // empty' <<<"$pvc_json") + + workload_json=$(kubectl_cli get statefulsets,deployments --all-namespaces -o json) || { + record_error "Could not enumerate StatefulSets and Deployments" + return + } + log_info "Stopping every workload that mounts a PersistentVolumeClaim" + while IFS=$'\t' read -r namespace kind name; do + [[ -n "$namespace" && -n "$name" ]] || continue + kubectl_cli scale "$kind/$name" --namespace "$namespace" --replicas=0 >/dev/null \ + || record_error "Failed to scale $kind $namespace/$name to 0" + done < <(jq -r '.items[] + | select(((.spec.volumeClaimTemplates // []) | length > 0) + or ([.spec.template.spec.volumes[]? | select(.persistentVolumeClaim)] | length > 0)) + | [.metadata.namespace, (.kind | ascii_downcase), .metadata.name] | @tsv' <<<"$workload_json") + wait_until "pods that mount PersistentVolumeClaims to stop" pods_with_claims_gone || true + + log_info "Deleting every PersistentVolumeClaim from validated EKS cluster $EKS_CLUSTER_NAME" + while IFS=$'\t' read -r namespace name; do + [[ -n "$namespace" && -n "$name" ]] || continue + kubectl_cli delete pvc "$name" --namespace "$namespace" --ignore-not-found \ + --wait=true --timeout="${WAIT_SECONDS}s" >/dev/null || record_error "Failed to delete PersistentVolumeClaim $namespace/$name" + done < <(jq -r '.items[] | [.metadata.namespace,.metadata.name] | @tsv' <<<"$pvc_json") + if (( ${#volume_names[@]} > 0 )); then + wait_until "PersistentVolumes and their EBS volumes to be deleted" volumes_gone "${volume_names[@]}" \ + && log_info "PersistentVolumes deleted: ${volume_names[*]}" + fi +} + cleanup_eks_dependencies() { local namespace name association association_json association_tags ingress_json service_json association_output ingress_json=$(kubectl_cli get ingress --all-namespaces -o json) || { @@ -663,6 +723,7 @@ main() { fi confirm_cleanup + cleanup_persistent_volumes cleanup_eks_dependencies cleanup_vpc_load_balancers cleanup_runtime_ecr diff --git a/infra/scripts/cleanup/java-on-aws.sh b/infra/scripts/cleanup/java-on-aws.sh index 69479506..fa2c0dad 100755 --- a/infra/scripts/cleanup/java-on-aws.sh +++ b/infra/scripts/cleanup/java-on-aws.sh @@ -37,6 +37,7 @@ declare -a SECURITY_GROUPS=() declare -a STACK_RESOURCE_IDS=() log_info() { printf '[INFO] %s\n' "$*"; } +log_success() { printf '[OK] %s\n' "$*"; } log_warn() { printf '[WARN] %s\n' "$*" >&2; } log_error() { printf '[ERROR] %s\n' "$*" >&2; } record_error() { log_error "$*"; ERRORS=$((ERRORS + 1)); } diff --git a/infra/scripts/cleanup/java-performance.sh b/infra/scripts/cleanup/java-performance.sh index 84d8db48..9c9b4cf1 100755 --- a/infra/scripts/cleanup/java-performance.sh +++ b/infra/scripts/cleanup/java-performance.sh @@ -42,6 +42,7 @@ declare -a SECURITY_GROUPS=() declare -a STACK_RESOURCE_IDS=() log_info() { printf '[INFO] %s\n' "$*"; } +log_success() { printf '[OK] %s\n' "$*"; } log_warn() { printf '[WARN] %s\n' "$*" >&2; } log_error() { printf '[ERROR] %s\n' "$*" >&2; } record_error() { log_error "$*"; ERRORS=$((ERRORS + 1)); } @@ -399,9 +400,10 @@ discover_resources() { } print_plan() { - local ingress_count service_count association_count + local ingress_count service_count association_count pvc_count ingress_count=$(kubectl_cli get ingress --all-namespaces -o json | jq '.items | length') service_count=$(kubectl_cli get service --all-namespaces -o json | jq '[.items[] | select(.spec.type == "LoadBalancer")] | length') + pvc_count=$(kubectl_cli get pvc --all-namespaces -o json | jq '.items | length') association_count=$(aws_cli eks list-pod-identity-associations --cluster-name "$EKS_CLUSTER_NAME" \ --query 'length(associations)' --output text) @@ -409,6 +411,7 @@ print_plan() { printf ' Kubernetes Ingresses: %s\n' "$ingress_count" printf ' Kubernetes LoadBalancer Services: %s\n' "$service_count" printf ' EKS Pod Identity associations: %s\n' "$association_count" + printf ' Kubernetes volume claims: %s\n' "$pvc_count" printf ' Load balancers in VPC: %d\n' "${#RUNTIME_LOAD_BALANCERS[@]}" printf ' Target groups in VPC: %d\n' "${#RUNTIME_TARGET_GROUPS[@]}" printf ' Tagged runtime ECR repositories: %d\n' "${#RUNTIME_ECR_REPOSITORIES[@]}" @@ -434,6 +437,63 @@ kubernetes_dependencies_gone() { [[ "$ingress_count" -eq 0 && "$service_count" -eq 0 ]] } +pods_with_claims_gone() { + local count + count=$(kubectl_cli get pods --all-namespaces -o json 2>/dev/null \ + | jq '[.items[] | select([.spec.volumes[]? | select(.persistentVolumeClaim)] | length > 0)] | length') || return 1 + [[ "$count" -eq 0 ]] +} + +volumes_gone() { + local name + for name in "$@"; do + kubectl_cli get pv "$name" >/dev/null 2>&1 && return 1 + done + return 0 +} + +# PersistentVolumeClaims are backed by EBS volumes that outlive the cluster: once it is +# deleted, no CSI driver is left to delete them, and they stay billed as `available`. +# Stop every workload that mounts a claim (this also stops Pyroscope writing blocks into +# the stack's bucket, which would fail the bucket's deletion), delete the claims, and wait +# until their PersistentVolumes, and with them the EBS volumes, are gone. +cleanup_persistent_volumes() { + local pvc_json workload_json namespace kind name + local -a volume_names=() + pvc_json=$(kubectl_cli get pvc --all-namespaces -o json) || { + record_error "Could not enumerate PersistentVolumeClaims" + return + } + [[ "$(jq '.items | length' <<<"$pvc_json")" -gt 0 ]] || return 0 + mapfile -t volume_names < <(jq -r '.items[].spec.volumeName // empty' <<<"$pvc_json") + + workload_json=$(kubectl_cli get statefulsets,deployments --all-namespaces -o json) || { + record_error "Could not enumerate StatefulSets and Deployments" + return + } + log_info "Stopping every workload that mounts a PersistentVolumeClaim" + while IFS=$'\t' read -r namespace kind name; do + [[ -n "$namespace" && -n "$name" ]] || continue + kubectl_cli scale "$kind/$name" --namespace "$namespace" --replicas=0 >/dev/null \ + || record_error "Failed to scale $kind $namespace/$name to 0" + done < <(jq -r '.items[] + | select(((.spec.volumeClaimTemplates // []) | length > 0) + or ([.spec.template.spec.volumes[]? | select(.persistentVolumeClaim)] | length > 0)) + | [.metadata.namespace, (.kind | ascii_downcase), .metadata.name] | @tsv' <<<"$workload_json") + wait_until "pods that mount PersistentVolumeClaims to stop" pods_with_claims_gone || true + + log_info "Deleting every PersistentVolumeClaim from validated EKS cluster $EKS_CLUSTER_NAME" + while IFS=$'\t' read -r namespace name; do + [[ -n "$namespace" && -n "$name" ]] || continue + kubectl_cli delete pvc "$name" --namespace "$namespace" --ignore-not-found \ + --wait=true --timeout="${WAIT_SECONDS}s" >/dev/null || record_error "Failed to delete PersistentVolumeClaim $namespace/$name" + done < <(jq -r '.items[] | [.metadata.namespace,.metadata.name] | @tsv' <<<"$pvc_json") + if (( ${#volume_names[@]} > 0 )); then + wait_until "PersistentVolumes and their EBS volumes to be deleted" volumes_gone "${volume_names[@]}" \ + && log_info "PersistentVolumes deleted: ${volume_names[*]}" + fi +} + cleanup_eks_dependencies() { local namespace name association association_json association_tags ingress_json service_json association_output ingress_json=$(kubectl_cli get ingress --all-namespaces -o json) || { @@ -663,6 +723,7 @@ main() { fi confirm_cleanup + cleanup_persistent_volumes cleanup_eks_dependencies cleanup_vpc_load_balancers cleanup_runtime_ecr diff --git a/infra/scripts/deploy/java-performance/perf-profiler.sh b/infra/scripts/deploy/java-performance/perf-profiler.sh index dafb42a7..4ab1525c 100755 --- a/infra/scripts/deploy/java-performance/perf-profiler.sh +++ b/infra/scripts/deploy/java-performance/perf-profiler.sh @@ -29,6 +29,9 @@ CLUSTER_NAME="${PREFIX:-workshop}-eks" # stamped into the sideca APP_UID="${APP_UID:-1000}" # UID the profiled app runs as PYROSCOPE_URL="${PYROSCOPE_URL:-http://pyroscope.monitoring:4040}" KYVERNO_CHART_VERSION="3.9.1" # Kyverno v1.19.1; MutatingPolicy v1beta1 needs >= this +# The reports controller can stall without an error (kyverno#17848; #17085 is only in 1.20 rcs), so the +# content's enforce block restarts it after 90 s without a PolicyReport. Once a stable release has the +# fix, bump this pin and remove the restart from ../java-performance/content/enforce/index.en.md. build() { log_info "Building and pushing perf-profiler image..." diff --git a/infra/scripts/ws-test/java-performance.sh b/infra/scripts/ws-test/java-performance.sh index a58010b7..b0b12a53 100755 --- a/infra/scripts/ws-test/java-performance.sh +++ b/infra/scripts/ws-test/java-performance.sh @@ -12,16 +12,16 @@ ws_source_environment '/etc/profile.d/workshop.sh' ws_begin_page 'Set up the environment' 10 'setup/index.en.md' -ws_run_block 1 'Check the environment' 'In the Shell terminal:' 25 26 'bash' '' <<'WS_TEST_BLOCK_10_1' +ws_run_block 1 'Check the environment' 'In the Shell terminal:' 65 66 'bash' '' <<'WS_TEST_BLOCK_10_1' curl -sf localhost:8090/actuator/health && echo kubectl -n unicorn-store-spring get pods WS_TEST_BLOCK_10_1 -ws_skip_block 2 'Check the environment' 'Check the environment' 31 32 'text' '' 'copy action disabled' +ws_skip_block 2 'Check the environment' 'Check the environment' 71 72 'text' '' 'copy action disabled' -ws_skip_block 3 'Start Claude Code' 'In the Claude terminal:' 47 47 'bash' '' 'interactive terminal UI' +ws_skip_block 3 'Start Claude Code' 'In the Claude terminal:' 87 87 'bash' '' 'interactive terminal UI' -ws_run_block 4 'Start Claude Code' 'With -p, Claude Code prints the answer and exits. Later calls with -c continue the same session:' 65 65 'bash' 'non-interactive' <<'WS_TEST_BLOCK_10_4' +ws_run_block 4 'Start Claude Code' 'With -p, Claude Code prints the answer and exits. Later calls with -c continue the same session:' 105 105 'bash' 'non-interactive' <<'WS_TEST_BLOCK_10_4' cd ~/environment && claude -p "List the MCP tools you can see from perf-sensor and eks-mcp, names only." WS_TEST_BLOCK_10_4 @@ -272,10 +272,12 @@ WS_TEST_BLOCK_500_1 ws_skip_block 2 'Review the policy' '- Item 10 assumption: the grace-period rule assumes Spring Boot'"'"'s 30 s default drain. The app sets no spring.lifecycle.timeout-per-shutdown-phase, so that holds today.' 93 105 'yaml' '' 'copy action disabled' -ws_run_block 3 'Apply and test' 'In the Shell terminal:' 117 129 'bash' '' <<'WS_TEST_BLOCK_500_3' +ws_run_block 3 'Apply and test' 'In the Shell terminal:' 117 131 'bash' '' <<'WS_TEST_BLOCK_500_3' kubectl apply -f ~/environment/unicorn-store-spring/k8s/policy.yaml # the background scan reports the running Deployment +timeout 90 sh -c 'until kubectl -n unicorn-store-spring get policyreport -o name | grep -q .; do sleep 5; done' \ + || kubectl -n kyverno rollout restart deploy/kyverno-reports-controller until kubectl -n unicorn-store-spring get policyreport -o name | grep -q .; do sleep 5; done kubectl -n unicorn-store-spring get policyreport @@ -288,7 +290,7 @@ git -C ~/environment/unicorn-store-spring add -A \ && git -C ~/environment/unicorn-store-spring commit -q -m "kyverno policy" WS_TEST_BLOCK_500_3 -ws_skip_block 4 'Apply and test' 'Apply and test' 134 138 'text' '' 'copy action disabled' +ws_skip_block 4 'Apply and test' 'Apply and test' 136 140 'text' '' 'copy action disabled' ws_end_page