From 1b0f82d0bd58534e91873404fb4d98dcd294f43c Mon Sep 17 00:00:00 2001 From: Nithin Chandran R Date: Tue, 15 Sep 2026 11:24:41 +0000 Subject: [PATCH] feat(apigw-sfn-express-athena-cdk): Add synchronous Athena query API with zero Lambda Amazon API Gateway invokes an AWS Step Functions Express Workflow synchronously (StartSyncExecution) that starts an Amazon Athena query, polls it to completion, and returns the result rows in one HTTP call. Amazon Athena queries an AWS Glue table over Amazon S3. No AWS Lambda functions in the request path. Deployed and tested end-to-end in us-east-1. --- apigw-sfn-express-athena-cdk/.gitignore | 9 + apigw-sfn-express-athena-cdk/README.md | 115 ++++++ apigw-sfn-express-athena-cdk/cdk/.gitignore | 9 + apigw-sfn-express-athena-cdk/cdk/bin/app.ts | 22 ++ apigw-sfn-express-athena-cdk/cdk/cdk.json | 27 ++ .../cdk/lib/apigw-sfn-express-athena-stack.ts | 374 ++++++++++++++++++ apigw-sfn-express-athena-cdk/cdk/package.json | 24 ++ .../cdk/tsconfig.json | 23 ++ .../example-pattern.json | 115 ++++++ 9 files changed, 718 insertions(+) create mode 100644 apigw-sfn-express-athena-cdk/.gitignore create mode 100644 apigw-sfn-express-athena-cdk/README.md create mode 100644 apigw-sfn-express-athena-cdk/cdk/.gitignore create mode 100644 apigw-sfn-express-athena-cdk/cdk/bin/app.ts create mode 100644 apigw-sfn-express-athena-cdk/cdk/cdk.json create mode 100644 apigw-sfn-express-athena-cdk/cdk/lib/apigw-sfn-express-athena-stack.ts create mode 100644 apigw-sfn-express-athena-cdk/cdk/package.json create mode 100644 apigw-sfn-express-athena-cdk/cdk/tsconfig.json create mode 100644 apigw-sfn-express-athena-cdk/example-pattern.json diff --git a/apigw-sfn-express-athena-cdk/.gitignore b/apigw-sfn-express-athena-cdk/.gitignore new file mode 100644 index 0000000000..af14d7bf27 --- /dev/null +++ b/apigw-sfn-express-athena-cdk/.gitignore @@ -0,0 +1,9 @@ +node_modules +*.js +!jest.config.js +*.d.ts +cdk.out +cdk.context.json +.cdk.staging +*.log +package-lock.json diff --git a/apigw-sfn-express-athena-cdk/README.md b/apigw-sfn-express-athena-cdk/README.md new file mode 100644 index 0000000000..20fdd247b5 --- /dev/null +++ b/apigw-sfn-express-athena-cdk/README.md @@ -0,0 +1,115 @@ +# Amazon API Gateway to AWS Step Functions Express to Amazon Athena (synchronous, zero AWS Lambda) + +Run an Amazon Athena query and return its rows in a single synchronous Amazon API Gateway request through an AWS Step Functions Express Workflow, using zero AWS Lambda functions. + +This pattern is implemented with the AWS Cloud Development Kit (AWS CDK) in TypeScript. + +Learn more about this pattern at Serverless Land Patterns: https://serverlessland.com/patterns/apigw-sfn-express-athena-cdk + +Important: this application uses various AWS services and there are costs associated with these services after the Free Tier usage - please see the [AWS Pricing page](https://aws.amazon.com/pricing/) for details. You are responsible for any AWS costs incurred. No warranty is implied in this example. + +## Architecture + +A client sends an HTTP `POST` to an Amazon API Gateway REST API with a JSON body containing an Amazon Athena SQL `queryString`. Amazon API Gateway invokes an AWS Step Functions Express Workflow with `StartSyncExecution`, so the HTTP call blocks until the workflow completes and returns its output in the same response. + +The Express Workflow uses only native AWS Step Functions service integrations (no AWS Lambda). Because Express Workflows do not support the `.sync` (RUN_JOB) integration pattern, the workflow starts the query and then polls it to completion: + +1. `AthenaStartQueryExecution` with the `REQUEST_RESPONSE` integration pattern starts the query against an AWS Glue database and table and returns immediately with a query execution ID. +2. A `Wait` state pauses, then `GetQueryExecution` reads the query status and a `Choice` state loops back to `Wait` until the query reaches `SUCCEEDED` (or fails). +3. `AthenaGetQueryResults` reads the completed query's result rows. + +A final `Pass` state shapes the response down to the Amazon Athena result rows, which flow back through AWS Step Functions and Amazon API Gateway to the caller. + +Amazon Athena reads a sample CSV dataset stored in Amazon S3 (`data/` prefix) via an AWS Glue external table whose schema is declared in AWS CDK, and writes query results to a separate `athena-results/` prefix in the same Amazon S3 bucket. An Amazon Athena workgroup pins the result location and enforces server-side encryption. + +``` +Client + | POST { "queryString": "SELECT ..." } + v +Amazon API Gateway (REST) + | states:StartSyncExecution + v +AWS Step Functions Express Workflow + | AthenaStartQueryExecution (REQUEST_RESPONSE) + | --> Wait --> GetQueryExecution --> Choice(status) --loop until SUCCEEDED--> + | --> AthenaGetQueryResults --> FormatResponse + v +Amazon Athena --(AWS Glue table)--> Amazon S3 (data/ read, athena-results/ write) + | + v +Rows returned synchronously to the client +``` + +Because every Amazon Athena call is a native AWS Step Functions service integration, there are no AWS Lambda functions anywhere in this pattern. + +## Requirements + +* [Create an AWS account](https://portal.aws.amazon.com/gp/aws/developer/registration/index.html) if you do not already have one and log in. The IAM user that you use must have sufficient permissions to make necessary AWS service calls and manage AWS resources. +* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) installed and configured +* [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) installed +* [Node.js 18+](https://nodejs.org/en/download/) installed +* [AWS Cloud Development Kit (AWS CDK) v2](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html) installed +* An AWS account bootstrapped for the AWS CDK (`cdk bootstrap`) + +## Deployment Instructions + +1. Create a new directory, navigate to that directory in a terminal and clone the GitHub repository: + ``` + git clone https://github.com/aws-samples/serverless-patterns + ``` +2. Change directory to the pattern directory: + ``` + cd apigw-sfn-express-athena-cdk/cdk + ``` +3. Install dependencies: + ``` + npm install + ``` +4. Deploy the stack to your default AWS account and region: + ``` + npx cdk deploy + ``` +5. Note the `ApiEndpoint` value from the stack outputs. You will use it for testing. + +## Testing + +The headline of this pattern is that a single synchronous HTTP call returns Amazon Athena result rows, with no AWS Lambda in the path. + +After deployment, send a `POST` to the `ApiEndpoint` output with an Amazon Athena SQL query in the body. Replace `` with your stack output: + +``` +curl -X POST '' \ + -H 'Content-Type: application/json' \ + -d '{"queryString": "SELECT customer, SUM(amount) AS total FROM orders GROUP BY customer ORDER BY total DESC"}' +``` + +The response returns synchronously in the same call and contains the Amazon Athena result rows, for example: + +```json +{ + "rows": [ + { "Data": [ { "VarCharValue": "customer" }, { "VarCharValue": "total" } ] }, + { "Data": [ { "VarCharValue": "Initech" }, { "VarCharValue": "499.75" } ] }, + { "Data": [ { "VarCharValue": "Umbrella" }, { "VarCharValue": "359.88" } ] } + ] +} +``` + +The first row is the column header. This proves the full Amazon API Gateway -> AWS Step Functions Express -> Amazon Athena query-and-return round trip completed in one request/response. + +You can query any column of the sample `orders` table (`order_id`, `customer`, `product`, `quantity`, `amount`, `order_date`). + +## Cleanup + +1. Delete the stack: + ``` + cd apigw-sfn-express-athena-cdk/cdk + npx cdk destroy + ``` + + Warning: `cdk destroy` deletes the Amazon S3 bucket and ALL of its contents, including the sample dataset and every Amazon Athena query result written during testing. The bucket is configured with `RemovalPolicy.DESTROY` and auto-delete so the pattern cleans up fully. Do not store data you want to keep in this bucket. + +---- +Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +SPDX-License-Identifier: MIT-0 diff --git a/apigw-sfn-express-athena-cdk/cdk/.gitignore b/apigw-sfn-express-athena-cdk/cdk/.gitignore new file mode 100644 index 0000000000..af14d7bf27 --- /dev/null +++ b/apigw-sfn-express-athena-cdk/cdk/.gitignore @@ -0,0 +1,9 @@ +node_modules +*.js +!jest.config.js +*.d.ts +cdk.out +cdk.context.json +.cdk.staging +*.log +package-lock.json diff --git a/apigw-sfn-express-athena-cdk/cdk/bin/app.ts b/apigw-sfn-express-athena-cdk/cdk/bin/app.ts new file mode 100644 index 0000000000..52a03677e9 --- /dev/null +++ b/apigw-sfn-express-athena-cdk/cdk/bin/app.ts @@ -0,0 +1,22 @@ +#!/usr/bin/env node +// Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT-0 + +import 'source-map-support/register'; +import * as cdk from 'aws-cdk-lib'; +import { ApigwSfnExpressAthenaStack } from '../lib/apigw-sfn-express-athena-stack'; + +const app = new cdk.App(); + +new ApigwSfnExpressAthenaStack(app, 'ApigwSfnExpressAthenaStack', { + // Uses the account/region from the ambient CDK environment + // (CDK_DEFAULT_ACCOUNT / CDK_DEFAULT_REGION). No hardcoded values. + env: { + account: process.env.CDK_DEFAULT_ACCOUNT, + region: process.env.CDK_DEFAULT_REGION, + }, + description: + 'Amazon API Gateway -> AWS Step Functions Express -> Amazon Athena synchronous query, zero AWS Lambda (uksb-sfn-athena)', +}); + +app.synth(); diff --git a/apigw-sfn-express-athena-cdk/cdk/cdk.json b/apigw-sfn-express-athena-cdk/cdk/cdk.json new file mode 100644 index 0000000000..2b005ac678 --- /dev/null +++ b/apigw-sfn-express-athena-cdk/cdk/cdk.json @@ -0,0 +1,27 @@ +{ + "app": "npx ts-node --prefer-ts-exts bin/app.ts", + "watch": { + "include": ["**"], + "exclude": [ + "README.md", + "cdk*.json", + "**/*.d.ts", + "**/*.js", + "tsconfig.json", + "package*.json", + "node_modules", + "cdk.out" + ] + }, + "context": { + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/core:target-partitions": ["aws", "aws-cn"], + "@aws-cdk/aws-iam:minimizePolicies": true, + "@aws-cdk/core:validateSnapshotRemovalPolicy": true, + "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, + "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, + "@aws-cdk/core:stackRelativeExports": true, + "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true + } +} diff --git a/apigw-sfn-express-athena-cdk/cdk/lib/apigw-sfn-express-athena-stack.ts b/apigw-sfn-express-athena-cdk/cdk/lib/apigw-sfn-express-athena-stack.ts new file mode 100644 index 0000000000..a455407acc --- /dev/null +++ b/apigw-sfn-express-athena-cdk/cdk/lib/apigw-sfn-express-athena-stack.ts @@ -0,0 +1,374 @@ +// Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT-0 + +import { + Stack, + StackProps, + RemovalPolicy, + Duration, + Aws, + CfnOutput, +} from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import * as cr from 'aws-cdk-lib/custom-resources'; +import * as glue from 'aws-cdk-lib/aws-glue'; +import * as athena from 'aws-cdk-lib/aws-athena'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as logs from 'aws-cdk-lib/aws-logs'; +import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; +import * as tasks from 'aws-cdk-lib/aws-stepfunctions-tasks'; +import * as apigw from 'aws-cdk-lib/aws-apigateway'; + +/** + * Amazon API Gateway (REST) -> AWS Step Functions Express Workflow (synchronous + * StartSyncExecution) -> Amazon Athena (StartQueryExecution + GetQueryResults via + * native Step Functions service integrations) -> results returned to the caller in + * a single request/response. ZERO AWS Lambda functions. + */ +export class ApigwSfnExpressAthenaStack extends Stack { + constructor(scope: Construct, id: string, props?: StackProps) { + super(scope, id, props); + + const dataPrefix = 'data/'; + const resultsPrefix = 'athena-results/'; + + // --------------------------------------------------------------------- + // Amazon S3 bucket: sample dataset (data/) + Athena query results + // (athena-results/). Encrypted with S3-managed keys, TLS enforced, + // public access fully blocked. + // --------------------------------------------------------------------- + const dataBucket = new s3.Bucket(this, 'DataBucket', { + encryption: s3.BucketEncryption.S3_MANAGED, + enforceSSL: true, + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + // WARNING: DESTROY + autoDeleteObjects deletes the bucket and all data + // (sample dataset AND query results) on `cdk destroy`. See README. + removalPolicy: RemovalPolicy.DESTROY, + autoDeleteObjects: true, + }); + + // Seed a tiny sample CSV dataset into the data/ prefix so the pattern is + // queryable immediately after deploy with no manual data-loading step. + // Uses an AwsCustomResource calling s3:PutObject with the CSV inline — + // no asset bundling or bundled-CLI download (more robust than + // BucketDeployment across build hosts). + const seedData = new cr.AwsCustomResource(this, 'SampleDataDeployment', { + onCreate: { + service: 'S3', + action: 'putObject', + parameters: { + Bucket: dataBucket.bucketName, + Key: `${dataPrefix}orders.csv`, + Body: SAMPLE_CSV, + ContentType: 'text/csv', + }, + physicalResourceId: cr.PhysicalResourceId.of(`${dataPrefix}orders.csv`), + }, + onUpdate: { + service: 'S3', + action: 'putObject', + parameters: { + Bucket: dataBucket.bucketName, + Key: `${dataPrefix}orders.csv`, + Body: SAMPLE_CSV, + ContentType: 'text/csv', + }, + physicalResourceId: cr.PhysicalResourceId.of(`${dataPrefix}orders.csv`), + }, + policy: cr.AwsCustomResourcePolicy.fromStatements([ + new iam.PolicyStatement({ + actions: ['s3:PutObject'], + resources: [`${dataBucket.bucketArn}/${dataPrefix}*`], + }), + ]), + }); + seedData.node.addDependency(dataBucket); + + // --------------------------------------------------------------------- + // AWS Glue database + external table over the CSV data in Amazon S3. + // The table schema is declared in CDK so Athena can query it directly. + // --------------------------------------------------------------------- + const glueDb = new glue.CfnDatabase(this, 'GlueDatabase', { + catalogId: Aws.ACCOUNT_ID, + databaseInput: { + name: `orders_db_${Aws.ACCOUNT_ID}`, + description: 'Sample database for the apigw-sfn-express-athena pattern', + }, + }); + + const glueTable = new glue.CfnTable(this, 'GlueTable', { + catalogId: Aws.ACCOUNT_ID, + databaseName: `orders_db_${Aws.ACCOUNT_ID}`, + tableInput: { + name: 'orders', + description: 'Sample orders table backed by CSV in Amazon S3', + tableType: 'EXTERNAL_TABLE', + parameters: { + classification: 'csv', + 'skip.header.line.count': '1', + areColumnsQuoted: 'false', + }, + storageDescriptor: { + columns: [ + { name: 'order_id', type: 'string' }, + { name: 'customer', type: 'string' }, + { name: 'product', type: 'string' }, + { name: 'quantity', type: 'int' }, + { name: 'amount', type: 'double' }, + { name: 'order_date', type: 'string' }, + ], + location: `s3://${dataBucket.bucketName}/${dataPrefix}`, + inputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + outputFormat: + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + serdeInfo: { + serializationLibrary: + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', + parameters: { + 'field.delim': ',', + 'serialization.format': ',', + }, + }, + }, + }, + }); + glueTable.addDependency(glueDb); + + // --------------------------------------------------------------------- + // Amazon Athena workgroup with the results location pinned to the + // athena-results/ prefix and results encrypted with SSE-S3. + // --------------------------------------------------------------------- + const workgroupName = `apigw-sfn-express-athena-${Aws.ACCOUNT_ID}`; + const workgroup = new athena.CfnWorkGroup(this, 'AthenaWorkgroup', { + name: workgroupName, + recursiveDeleteOption: true, + workGroupConfiguration: { + enforceWorkGroupConfiguration: true, + publishCloudWatchMetricsEnabled: true, + resultConfiguration: { + outputLocation: `s3://${dataBucket.bucketName}/${resultsPrefix}`, + encryptionConfiguration: { encryptionOption: 'SSE_S3' }, + }, + }, + }); + + // ARN helpers scoped to THIS account/region (no hardcoded account IDs). + const workgroupArn = `arn:${Aws.PARTITION}:athena:${Aws.REGION}:${Aws.ACCOUNT_ID}:workgroup/${workgroupName}`; + const catalogArn = `arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:catalog`; + const databaseArn = `arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:database/orders_db_${Aws.ACCOUNT_ID}`; + const tableArn = `arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:table/orders_db_${Aws.ACCOUNT_ID}/orders`; + + // --------------------------------------------------------------------- + // Step Functions EXPRESS state machine. + // StartQueryExecution (REQUEST_RESPONSE) -> Wait -> GetQueryExecution + // -> Choice(status) -> GetQueryResults -> FormatResponse + // Express workflows do NOT support the '.sync' (RUN_JOB) Athena + // integration, so we start the query, then poll its status in a + // Wait/Choice loop until it reaches SUCCEEDED before fetching results. + // All native Step Functions service integrations -> no AWS Lambda. + // --------------------------------------------------------------------- + const startQuery = new tasks.AthenaStartQueryExecution(this, 'StartAthenaQuery', { + // REQUEST_RESPONSE returns immediately with a QueryExecutionId. Express + // state machines only support this integration pattern for Athena. + integrationPattern: sfn.IntegrationPattern.REQUEST_RESPONSE, + queryString: sfn.JsonPath.stringAt('$.queryString'), + workGroup: workgroupName, + queryExecutionContext: { + databaseName: `orders_db_${Aws.ACCOUNT_ID}`, + }, + resultPath: '$.queryExecution', + }); + + // Retry transient Athena throttling/service errors, then Catch any + // remaining failure and surface a clean error payload to the caller. + startQuery.addRetry({ + errors: ['Athena.AthenaException', 'States.TaskFailed'], + interval: Duration.seconds(2), + maxAttempts: 3, + backoffRate: 2, + }); + + // Poll the query status: Wait -> GetQueryExecution -> Choice. + const waitForQuery = new sfn.Wait(this, 'WaitForQuery', { + time: sfn.WaitTime.duration(Duration.seconds(2)), + }); + + const getQueryExecution = new tasks.CallAwsService(this, 'GetQueryExecution', { + service: 'athena', + action: 'getQueryExecution', + parameters: { + QueryExecutionId: sfn.JsonPath.stringAt( + '$.queryExecution.QueryExecutionId', + ), + }, + iamResources: [workgroupArn], + resultPath: '$.queryStatus', + }); + getQueryExecution.addRetry({ + errors: ['Athena.AthenaException', 'States.TaskFailed'], + interval: Duration.seconds(2), + maxAttempts: 3, + backoffRate: 2, + }); + + const getResults = new tasks.AthenaGetQueryResults(this, 'GetAthenaResults', { + queryExecutionId: sfn.JsonPath.stringAt( + '$.queryExecution.QueryExecutionId', + ), + resultPath: '$.queryResults', + }); + getResults.addRetry({ + errors: ['Athena.AthenaException', 'States.TaskFailed'], + interval: Duration.seconds(2), + maxAttempts: 3, + backoffRate: 2, + }); + + const queryFailed = new sfn.Pass(this, 'QueryFailed', { + parameters: { + 'error.$': '$.error', + message: 'Athena query failed', + }, + }); + + // Shape the response: return only the Athena result rows to the caller. + const formatResponse = new sfn.Pass(this, 'FormatResponse', { + parameters: { + 'rows.$': '$.queryResults.ResultSet.Rows', + }, + }); + + // Branch on the Athena query state. + const queryStatePath = '$.queryStatus.QueryExecution.Status.State'; + const checkQueryState = new sfn.Choice(this, 'CheckQueryState') + .when( + sfn.Condition.stringEquals(queryStatePath, 'SUCCEEDED'), + getResults, + ) + .when( + sfn.Condition.or( + sfn.Condition.stringEquals(queryStatePath, 'FAILED'), + sfn.Condition.stringEquals(queryStatePath, 'CANCELLED'), + ), + queryFailed, + ) + .otherwise(waitForQuery); + + startQuery.addCatch(queryFailed, { errors: ['States.ALL'], resultPath: '$.error' }); + getQueryExecution.addCatch(queryFailed, { errors: ['States.ALL'], resultPath: '$.error' }); + getResults.addCatch(queryFailed, { errors: ['States.ALL'], resultPath: '$.error' }); + + // Amazon API Gateway's StepFunctionsRestApi integration wraps the request + // body under a "body" key (input shape: {body, querystring, path}). Lift + // the caller's queryString to the top level so the query task can read it. + const extractInput = new sfn.Pass(this, 'ExtractInput', { + parameters: { + 'queryString.$': '$.body.queryString', + }, + }); + + const definition = extractInput + .next(startQuery) + .next(waitForQuery) + .next(getQueryExecution) + .next(checkQueryState); + getResults.next(formatResponse); + + const logGroup = new logs.LogGroup(this, 'StateMachineLogGroup', { + retention: logs.RetentionDays.ONE_WEEK, + removalPolicy: RemovalPolicy.DESTROY, + }); + + const stateMachine = new sfn.StateMachine(this, 'AthenaExpressStateMachine', { + definitionBody: sfn.DefinitionBody.fromChainable(definition), + stateMachineType: sfn.StateMachineType.EXPRESS, + timeout: Duration.seconds(60), + tracingEnabled: true, + logs: { + destination: logGroup, + level: sfn.LogLevel.ALL, + includeExecutionData: true, + }, + }); + + // --------------------------------------------------------------------- + // Least-privilege IAM for the state-machine role. + // --------------------------------------------------------------------- + // Athena query-execution actions, scoped to this workgroup ARN. + stateMachine.addToRolePolicy( + new iam.PolicyStatement({ + actions: [ + 'athena:StartQueryExecution', + 'athena:GetQueryExecution', + 'athena:GetQueryResults', + 'athena:StopQueryExecution', + ], + resources: [workgroupArn], + }), + ); + + // AWS Glue metadata reads Athena performs on the caller's behalf, scoped + // to the specific catalog / database / table this pattern owns. + stateMachine.addToRolePolicy( + new iam.PolicyStatement({ + actions: [ + 'glue:GetDatabase', + 'glue:GetTable', + 'glue:GetTables', + 'glue:GetPartition', + 'glue:GetPartitions', + ], + resources: [catalogArn, databaseArn, tableArn], + }), + ); + + // Amazon S3: read the sample data, write query results, and list the + // bucket (Athena requires ListBucket to resolve the results location). + dataBucket.grantRead(stateMachine, `${dataPrefix}*`); + dataBucket.grantReadWrite(stateMachine, `${resultsPrefix}*`); + stateMachine.addToRolePolicy( + new iam.PolicyStatement({ + actions: ['s3:GetBucketLocation', 's3:ListBucket'], + resources: [dataBucket.bucketArn], + }), + ); + + // --------------------------------------------------------------------- + // Amazon API Gateway (REST) -> StartSyncExecution on the Express state + // machine. StepFunctionsRestApi wires the AWS service integration and + // grants the API role sync-execution permission on THIS state machine. + // --------------------------------------------------------------------- + const api = new apigw.StepFunctionsRestApi(this, 'AthenaQueryApi', { + stateMachine, + // Express + StartSyncExecution: request/response in one HTTP call. + useDefaultMethodResponses: true, + deployOptions: { + stageName: 'prod', + tracingEnabled: true, + }, + }); + + new CfnOutput(this, 'ApiEndpoint', { + value: api.url, + description: 'POST here with {"queryString":"..."} to run a synchronous Athena query', + }); + new CfnOutput(this, 'DataBucketName', { value: dataBucket.bucketName }); + new CfnOutput(this, 'AthenaWorkgroupName', { value: workgroup.name! }); + new CfnOutput(this, 'GlueDatabaseName', { + value: `orders_db_${Aws.ACCOUNT_ID}`, + }); + } +} + +// A tiny sample dataset (6 columns, a handful of rows) seeded into S3 so the +// pattern is queryable immediately after deploy. +const SAMPLE_CSV = [ + 'order_id,customer,product,quantity,amount,order_date', + 'O-1001,Acme Corp,Widget,10,199.90,2026-01-05', + 'O-1002,Globex,Gadget,3,89.97,2026-01-06', + 'O-1003,Initech,Widget,25,499.75,2026-01-07', + 'O-1004,Acme Corp,Sprocket,7,140.00,2026-01-08', + 'O-1005,Umbrella,Gadget,12,359.88,2026-01-09', + 'O-1006,Globex,Widget,5,99.95,2026-01-10', +].join('\n'); diff --git a/apigw-sfn-express-athena-cdk/cdk/package.json b/apigw-sfn-express-athena-cdk/cdk/package.json new file mode 100644 index 0000000000..fa97a4d7d7 --- /dev/null +++ b/apigw-sfn-express-athena-cdk/cdk/package.json @@ -0,0 +1,24 @@ +{ + "name": "apigw-sfn-express-athena-cdk", + "version": "1.0.0", + "description": "Amazon API Gateway to AWS Step Functions Express to Amazon Athena synchronous query (zero AWS Lambda)", + "bin": { + "app": "bin/app.js" + }, + "scripts": { + "build": "tsc", + "watch": "tsc -w", + "cdk": "cdk" + }, + "devDependencies": { + "@types/node": "20.11.30", + "aws-cdk": "2.147.0", + "ts-node": "^10.9.2", + "typescript": "~5.4.5" + }, + "dependencies": { + "aws-cdk-lib": "2.147.0", + "constructs": "^10.3.0", + "source-map-support": "^0.5.21" + } +} diff --git a/apigw-sfn-express-athena-cdk/cdk/tsconfig.json b/apigw-sfn-express-athena-cdk/cdk/tsconfig.json new file mode 100644 index 0000000000..507608a99f --- /dev/null +++ b/apigw-sfn-express-athena-cdk/cdk/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["es2020"], + "declaration": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "noImplicitThis": true, + "alwaysStrict": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": false, + "inlineSourceMap": true, + "inlineSources": true, + "experimentalDecorators": true, + "strictPropertyInitialization": false, + "typeRoots": ["./node_modules/@types"] + }, + "exclude": ["node_modules", "cdk.out"] +} diff --git a/apigw-sfn-express-athena-cdk/example-pattern.json b/apigw-sfn-express-athena-cdk/example-pattern.json new file mode 100644 index 0000000000..e31437c83f --- /dev/null +++ b/apigw-sfn-express-athena-cdk/example-pattern.json @@ -0,0 +1,115 @@ +{ + "title": "Amazon API Gateway to AWS Step Functions Express to Amazon Athena", + "description": "Run an Amazon Athena query and return its rows in one synchronous Amazon API Gateway call through an AWS Step Functions Express Workflow, with zero AWS Lambda functions.", + "language": "TypeScript", + "level": "300", + "framework": "AWS CDK", + "services": { + "from": { + "serviceName": "Amazon API Gateway", + "serviceURL": "/api-gateway/" + }, + "to": { + "serviceName": "AWS Step Functions", + "serviceURL": "/step-functions/" + } + }, + "introBox": { + "headline": "How it works", + "text": [ + "This pattern deploys an Amazon API Gateway REST API that invokes an AWS Step Functions Express Workflow synchronously with StartSyncExecution.", + "The Express Workflow uses native AWS Step Functions service integrations to call Amazon Athena. Because Express Workflows do not support the .sync (RUN_JOB) pattern, AthenaStartQueryExecution runs with the REQUEST_RESPONSE pattern, a Wait/Choice loop polls GetQueryExecution until the query succeeds, then AthenaGetQueryResults reads the rows. A final Pass state shapes the response.", + "Amazon Athena queries an AWS Glue external table over a sample CSV dataset in Amazon S3 and writes results to a separate Amazon S3 prefix pinned by an Amazon Athena workgroup.", + "The result rows are returned to the caller in a single synchronous request/response. There are no AWS Lambda functions anywhere in the pattern." + ] + }, + "gitHub": { + "template": { + "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/apigw-sfn-express-athena-cdk", + "templateURL": "serverless-patterns/apigw-sfn-express-athena-cdk", + "projectFolder": "apigw-sfn-express-athena-cdk", + "readmeURL": "https://github.com/aws-samples/serverless-patterns/tree/main/apigw-sfn-express-athena-cdk/README.md" + } + }, + "resources": { + "bullets": [ + { + "text": "Call Amazon Athena with AWS Step Functions", + "link": "https://docs.aws.amazon.com/step-functions/latest/dg/connect-athena.html" + }, + { + "text": "Synchronous Express Workflows", + "link": "https://docs.aws.amazon.com/step-functions/latest/dg/express-synchronous.html" + } + ] + }, + "deploy": { + "text": [ + "cd apigw-sfn-express-athena-cdk/cdk", + "npm install", + "npx cdk deploy" + ] + }, + "testing": { + "text": [ + "See the GitHub repo for detailed testing instructions." + ] + }, + "cleanup": { + "text": [ + "Delete the stack: cd apigw-sfn-express-athena-cdk/cdk then npx cdk destroy.", + "Warning: this deletes the Amazon S3 bucket and all sample data and Amazon Athena query results." + ] + }, + "authors": [ + { + "name": "Nithin Chandran R", + "image": "https://avatars.githubusercontent.com/NithinChandranR-AWS", + "bio": "Cloud Support / Serverless enthusiast building serverless patterns.", + "linkedin": "", + "twitter": "" + } + ], + "patternArch": { + "icon1": { + "x": 15, + "y": 50, + "service": "apigw", + "label": "Amazon API Gateway (REST)" + }, + "icon2": { + "x": 45, + "y": 50, + "service": "sfn", + "label": "AWS Step Functions Express" + }, + "icon3": { + "x": 70, + "y": 50, + "service": "athena", + "label": "Amazon Athena" + }, + "icon4": { + "x": 90, + "y": 50, + "service": "s3", + "label": "Amazon S3" + }, + "line1": { + "from": "icon1", + "to": "icon2", + "label": "StartSyncExecution" + }, + "line2": { + "from": "icon2", + "to": "icon3", + "label": "Start query + poll" + }, + "line3": { + "from": "icon3", + "to": "icon4", + "label": "Read data / write results" + } + }, + "patternType": "Serverless" +}