add timestamp to clickhouse

Author: carderne

internal-packages/clickhouse/schema/037_add_queue_timestamp_to_task_runs_v2.sql

@@ -0,0 +1,7 @@
+-- +goose Up
+ALTER TABLE trigger_dev.task_runs_v2
+  ADD COLUMN IF NOT EXISTS queue_timestamp Nullable(DateTime64(3)) AFTER created_at;
+
+-- +goose Down
+ALTER TABLE trigger_dev.task_runs_v2
+  DROP COLUMN IF EXISTS queue_timestamp;

internal-packages/clickhouse/src/taskRuns.test.ts

@@ -29,13 +29,15 @@ describe("Task Runs V2", () => {
     });
 
     const now = Date.now();
+    const queueTimestamp = now + 30_000;
     const taskRunData: TaskRunInsertArray = [
       "env_1234", // environment_id
       "org_1234", // organization_id
       "project_1234", // project_id
       "run_1234", // run_id
       now, // updated_at
       now, // created_at
+      queueTimestamp, // queue_timestamp
       "PENDING", // status
       "DEVELOPMENT", // environment_type
       "friendly_1234", // friendly_id
@@ -105,6 +107,7 @@ describe("Task Runs V2", () => {
       schema: z.object({
         environment_id: z.string(),
         run_id: z.string(),
+        queue_timestamp: z.coerce.date().nullable(),
         concurrency_key: z.string(),
         bulk_action_group_ids: z.array(z.string()),
       }),
@@ -121,6 +124,7 @@ describe("Task Runs V2", () => {
         expect.objectContaining({
           environment_id: "env_1234",
           run_id: "run_1234",
+          queue_timestamp: new Date(queueTimestamp),
           concurrency_key: "concurrency_key_1234",
           bulk_action_group_ids: ["bulk_action_group_id_1234", "bulk_action_group_id_1235"],
         }),
@@ -183,6 +187,7 @@ describe("Task Runs V2", () => {
         "run_mixed", // run_id
         now, // updated_at
         now, // created_at
+        null, // queue_timestamp
         "COMPLETED_SUCCESSFULLY", // status
         "DEVELOPMENT", // environment_type
         "friendly_mixed", // friendly_id
@@ -282,6 +287,7 @@ describe("Task Runs V2", () => {
       "cma45oli70002qrdy47w0j4n7", // run_id
       createdAt, // updated_at
       createdAt, // created_at
+      null, // queue_timestamp
       "PENDING", // status
       "PRODUCTION", // environment_type
       "run_cma45oli70002qrdy47w0j4n7", // friendly_id
@@ -339,6 +345,7 @@ describe("Task Runs V2", () => {
       "cma45oli70002qrdy47w0j4n7", // run_id
       createdAt, // updated_at
       createdAt, // created_at
+      null, // queue_timestamp
       "COMPLETED_SUCCESSFULLY", // status
       "PRODUCTION", // environment_type
       "run_cma45oli70002qrdy47w0j4n7", // friendly_id
@@ -443,6 +450,7 @@ describe("Task Runs V2", () => {
         "cma45oli70002qrdy47w0j4n7", // run_id
         createdAt, // updated_at
         createdAt, // created_at
+        null, // queue_timestamp
         "PENDING", // status
         "PRODUCTION", // environment_type
         "run_cma45oli70002qrdy47w0j4n7", // friendly_id
@@ -555,6 +563,7 @@ describe("Task Runs V2", () => {
         "root_run_1", // run_id
         baseCreatedAt, // updated_at
         baseCreatedAt, // created_at
+        null, // queue_timestamp
         "EXECUTING", // status
         "DEVELOPMENT", // environment_type
         "run_root_1", // friendly_id
@@ -612,6 +621,7 @@ describe("Task Runs V2", () => {
         "child_a",
         baseCreatedAt + 1_000,
         baseCreatedAt + 1_000,
+        null, // queue_timestamp
         "PENDING",
         "DEVELOPMENT",
         "run_child_a",
@@ -673,6 +683,7 @@ describe("Task Runs V2", () => {
         "child_b",
         baseCreatedAt + 2_000,
         baseCreatedAt + 2_000,
+        null, // queue_timestamp
         "EXECUTING",
         "DEVELOPMENT",
         "run_child_b",
@@ -730,6 +741,7 @@ describe("Task Runs V2", () => {
         "child_deleted",
         baseCreatedAt + 3_000,
         baseCreatedAt + 3_000,
+        null, // queue_timestamp
         "PENDING",
         "DEVELOPMENT",
         "run_child_deleted",
@@ -907,6 +919,7 @@ describe("Task Runs V2", () => {
         "cma45oli70002qrdy47w0j4n7",
         createdAt,
         createdAt,
+        null, // queue_timestamp
         "PENDING",
         "PRODUCTION",
         "run_cma45oli70002qrdy47w0j4n7",
@@ -1010,6 +1023,7 @@ describe("Task Runs V2", () => {
         "cma45oli70002qrdy47w0j4n7",
         createdAt,
         createdAt,
+        null, // queue_timestamp
         "PENDING",
         "PRODUCTION",
         "run_cma45oli70002qrdy47w0j4n7",
@@ -1113,6 +1127,7 @@ describe("Task Runs V2", () => {
         "cma45oli70002qrdy47w0j4n7",
         createdAt,
         createdAt,
+        null, // queue_timestamp
         "PENDING",
         "PRODUCTION",
         "run_cma45oli70002qrdy47w0j4n7",

internal-packages/clickhouse/src/taskRuns.ts

@@ -9,6 +9,7 @@ export const TaskRunV2 = z.object({
   run_id: z.string(),
   updated_at: z.number().int(),
   created_at: z.number().int(),
+  queue_timestamp: z.number().int().nullish(),
   status: z.string(),
   environment_type: z.string(),
   friendly_id: z.string(),
@@ -69,6 +70,7 @@ export const TASK_RUN_COLUMNS = [
   "run_id",
   "updated_at",
   "created_at",
+  "queue_timestamp",
   "status",
   "environment_type",
   "friendly_id",
@@ -138,6 +140,7 @@ export type TaskRunFieldTypes = {
   run_id: string;
   updated_at: number;
   created_at: number;
+  queue_timestamp: number | null;
   status: string;
   environment_type: string;
   friendly_id: string;
@@ -306,6 +309,7 @@ export type TaskRunInsertArray = [
   run_id: string,
   updated_at: number,
   created_at: number,
+  queue_timestamp: number | null,
   status: string,
   environment_type: string,
   friendly_id: string,