fix(app): keep replay chunks from different page loads apart

.changeset/replay-chunks-across-page-loads.md

@@ -0,0 +1,16 @@
+---
+'@hyperdx/app': patch
+---
+
+fix: keep session replay events from different page loads apart
+
+Browser SDKs up to `@clickhouse/clickstack-browser` 1.2.0 number replay events
+from 1 again on every page load. When a page closed before all chunks of a large
+event were sent, its leftover chunk could be joined with a later page's event of
+the same number. That page's snapshot then failed to parse and did not render.
+Chunks are now grouped by event number and full-precision timestamp, and replay
+rows are ordered by the full-precision timestamp, so rows of two page loads in
+the same second no longer interleave.
+
+The app's own browser telemetry now uses `@clickhouse/clickstack-browser` 1.2.1,
+whose replay event numbers no longer repeat across page loads.

packages/app/package.json

@@ -27,7 +27,7 @@
   },
   "dependencies": {
     "@clickhouse/click-ui": "^0.12.0",
-    "@clickhouse/clickstack-browser": "^1.2.0",
+    "@clickhouse/clickstack-browser": "^1.2.1",
     "@clickhouse/clickstack-node": "^1.0.2",
     "@codemirror/autocomplete": "^6.0.0",
     "@codemirror/lang-json": "^6.0.1",

packages/app/src/__tests__/sessions.test.ts

@@ -14,6 +14,7 @@ import {
   buildRRWebStreamChartConfig,
   buildSessionsQueryFilters,
   fetchServiceScopeFilters,
+  getRRWebTimestampExpression,
 } from '@/sessions';
 
 jest.mock('@hyperdx/common-utils/dist/core/renderChartConfig', () => ({
@@ -39,14 +40,55 @@ describe('buildRRWebStreamChartConfig', () => {
     limit: 1000000,
     offset: 0,
     getSessionSourceFieldExpression,
+    preciseTimestampExpression: 'Timestamp',
   });
 
   it('orders by timestamp with numeric offset and chunk tiebreaks', () => {
     // Chunks of one rrweb event share a single timestamp, so without the
     // tiebreaks ClickHouse can return them in arbitrary order and the replay
     // reassembly gets scrambled (ClickHouse/ClickStack#2569). The casts matter
     // too: the attributes are strings, and lexicographically '10' < '2'.
+    // `Timestamp` comes before the offset because older SDKs restart the
+    // offset on every page load, and TimestampTime only has seconds.
     expect(config.orderBy).toBe(
+      "TimestampTime ASC, Timestamp ASC, toUInt64OrZero(LogAttributes['rr-web.offset']) ASC, toUInt64OrZero(LogAttributes['rr-web.chunk']) ASC",
+    );
+  });
+
+  it('does not repeat the timestamp when the source already uses Timestamp', () => {
+    const jsonConfig = buildRRWebStreamChartConfig({
+      source: { ...config, timestampValueExpression: 'Timestamp' },
+      serviceName: 'my-service',
+      sessionId: 'session-1',
+      startDate: new Date(1000),
+      endDate: new Date(2000),
+      limit: 1000000,
+      offset: 0,
+      getSessionSourceFieldExpression,
+      preciseTimestampExpression: 'Timestamp',
+    });
+    expect(jsonConfig.orderBy).toBe(
+      "Timestamp ASC, toUInt64OrZero(LogAttributes['rr-web.offset']) ASC, toUInt64OrZero(LogAttributes['rr-web.chunk']) ASC",
+    );
+  });
+
+  it('falls back to the source timestamp for tables without Timestamp', () => {
+    const fallbackConfig = buildRRWebStreamChartConfig({
+      source: config,
+      serviceName: 'my-service',
+      sessionId: 'session-1',
+      startDate: new Date(1000),
+      endDate: new Date(2000),
+      limit: 1000000,
+      offset: 0,
+      getSessionSourceFieldExpression,
+      preciseTimestampExpression: 'TimestampTime',
+    });
+    expect(fallbackConfig.select).toContainEqual({
+      valueExpression: 'TimestampTime',
+      alias: 'ts',
+    });
+    expect(fallbackConfig.orderBy).toBe(
       "TimestampTime ASC, toUInt64OrZero(LogAttributes['rr-web.offset']) ASC, toUInt64OrZero(LogAttributes['rr-web.chunk']) ASC",
     );
   });
@@ -60,6 +102,7 @@ describe('buildRRWebStreamChartConfig', () => {
           alias: 'tcks',
         },
         { valueExpression: "LogAttributes['rr-web.event']", alias: 'ev' },
+        { valueExpression: 'Timestamp', alias: 'ts' },
       ]),
     );
   });
@@ -83,13 +126,82 @@ describe('buildRRWebStreamChartConfig', () => {
       limit: 1000000,
       offset: 0,
       getSessionSourceFieldExpression,
+      preciseTimestampExpression: 'Timestamp',
     });
     expect(otelConfig.where).toBe(
       'ServiceName:"my-service" AND LogAttributes.session.id:"session-1"',
     );
   });
 });
 
+describe('getRRWebTimestampExpression', () => {
+  const source = {
+    from: { databaseName: 'default', tableName: 'hyperdx_sessions' },
+    timestampValueExpression: 'TimestampTime',
+    connection: 'conn',
+  };
+
+  function metadataWithColumns(columns: string[] | Error) {
+    const metadata = new Metadata(
+      new ClickhouseClient({ host: '' }),
+      new MetadataCache(),
+    );
+    const getColumns = jest.spyOn(metadata, 'getColumns');
+    if (columns instanceof Error) {
+      getColumns.mockRejectedValue(columns);
+    } else {
+      getColumns.mockResolvedValue(
+        columns.map(name => ({
+          name,
+          type: name === 'Timestamp' ? 'DateTime64(9)' : 'String',
+          codec_expression: '',
+          comment: '',
+          default_expression: '',
+          default_type: '',
+          ttl_expression: '',
+        })),
+      );
+    }
+    return { metadata, getColumns };
+  }
+
+  it('uses Timestamp when the session table has it', async () => {
+    const { metadata, getColumns } = metadataWithColumns([
+      'Timestamp',
+      'TimestampTime',
+      'Body',
+    ]);
+    expect(await getRRWebTimestampExpression(source, metadata)).toBe(
+      'Timestamp',
+    );
+    expect(getColumns).toHaveBeenCalledWith({
+      databaseName: 'default',
+      tableName: 'hyperdx_sessions',
+      connectionId: 'conn',
+    });
+  });
+
+  it("falls back to the source's first timestamp without Timestamp", async () => {
+    const { metadata } = metadataWithColumns(['TimestampTime', 'Body']);
+    expect(
+      await getRRWebTimestampExpression(
+        { ...source, timestampValueExpression: 'TimestampTime, EventTime' },
+        metadata,
+      ),
+    ).toBe('TimestampTime');
+  });
+
+  it('falls back instead of failing when the columns lookup fails', async () => {
+    const { metadata } = metadataWithColumns(new Error('DESCRIBE failed'));
+    const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+    expect(await getRRWebTimestampExpression(source, metadata)).toBe(
+      'TimestampTime',
+    );
+    expect(warn).toHaveBeenCalled();
+    warn.mockRestore();
+  });
+});
+
 describe('fetchServiceScopeFilters', () => {
   const traceSource: TTraceSource = {
     id: 'traces',

packages/app/src/sessions.ts

@@ -485,6 +485,45 @@ function isAbortError(e: unknown): boolean {
   );
 }
 
+/**
+ * Session sources map no precise timestamp, and the player already assumes
+ * the OTel log columns (Body, LogAttributes, ServiceName). Their `Timestamp`
+ * is DateTime64(9); the default `TimestampTime` has only seconds.
+ */
+const SESSION_PRECISE_TIMESTAMP_COLUMN = 'Timestamp';
+
+/**
+ * `Timestamp` when the session table has it, else the source's own
+ * timestamp. With the fallback, chunks are grouped at that precision, which
+ * still tells apart page loads in different seconds. The columns are
+ * usually cached already: the player's field expressions read them.
+ */
+export async function getRRWebTimestampExpression(
+  source: {
+    from: TSessionSource['from'];
+    timestampValueExpression: string;
+    connection: string;
+  },
+  metadata: Metadata,
+): Promise<string> {
+  const fallback = getFirstTimestampValueExpression(
+    source.timestampValueExpression,
+  );
+  try {
+    const column = await metadata.getColumn({
+      databaseName: source.from.databaseName,
+      tableName: source.from.tableName,
+      column: SESSION_PRECISE_TIMESTAMP_COLUMN,
+      connectionId: source.connection,
+    });
+    return column != null ? SESSION_PRECISE_TIMESTAMP_COLUMN : fallback;
+  } catch (error) {
+    // Don't fail the replay over a lookup that only sharpens chunk grouping
+    console.warn('Failed to read session table columns', error);
+    return fallback;
+  }
+}
+
 // OPTIMIZATION STRATEGY
 //
 // 1. Write a clickhouse query to divide a session into different chunks, where each chunk has a start time. Maybe each chunk contains 100 events.
@@ -497,11 +536,18 @@ function isAbortError(e: unknown): boolean {
  * Large rrweb events are split by the recorder into multiple log records
  * ("chunks") that all share the same event timestamp, so ordering by the
  * timestamp alone lets ClickHouse return chunks of one event in arbitrary
- * order, scrambling reassembly. `rr-web.offset` is a monotonically increasing
- * counter stamped on every record, and `rr-web.chunk` is the 1-based chunk
- * index within an event — ordering by both (numerically — the attributes are
- * stored as strings) makes the stream order deterministic.
+ * order, scrambling reassembly. `rr-web.offset` is a counter stamped on every
+ * record, and `rr-web.chunk` is the 1-based chunk index within an event —
+ * ordering by both (numerically — the attributes are stored as strings) makes
+ * the stream order deterministic.
  * https://github.com/ClickHouse/ClickStack/issues/2569
+ *
+ * SDKs up to clickstack-browser 1.2.0 restart `rr-web.offset` on every page
+ * load, so the full-precision timestamp (see getRRWebTimestampExpression)
+ * sorts ahead of it: at second precision, rows of two page loads in the same
+ * second interleave. The source's own timestamp stays first to match the
+ * table's sort key; in the default schema it is `toDateTime(Timestamp)`, so
+ * `Timestamp` only breaks its ties.
  */
 export function buildRRWebStreamChartConfig({
   source,
@@ -513,6 +559,7 @@ export function buildRRWebStreamChartConfig({
   limit,
   offset,
   getSessionSourceFieldExpression,
+  preciseTimestampExpression,
 }: {
   source: {
     from: TSessionSource['from'];
@@ -527,6 +574,7 @@ export function buildRRWebStreamChartConfig({
   limit: number;
   offset: number;
   getSessionSourceFieldExpression: FieldExpressionGenerator;
+  preciseTimestampExpression: string;
 }) {
   const eventAttributeExpression = (key: string) =>
     getSessionSourceFieldExpression(
@@ -557,6 +605,11 @@ export function buildRRWebStreamChartConfig({
         valueExpression: `${eventAttributeExpression('rr-web.event')}`,
         alias: 'ev',
       },
+      // Tells apart events of different page loads that share an `ev`
+      {
+        valueExpression: preciseTimestampExpression,
+        alias: 'ts',
+      },
     ],
     dateRange: [startDate, endDate] as [Date, Date],
     from: source.from,
@@ -566,7 +619,13 @@ export function buildRRWebStreamChartConfig({
     implicitColumnExpression:
       SESSION_TABLE_EXPRESSIONS.implicitColumnExpression,
     connection: source.connection,
-    orderBy: `${source.timestampValueExpression} ASC, toUInt64OrZero(${eventAttributeExpression('rr-web.offset')}) ASC, toUInt64OrZero(${eventAttributeExpression('rr-web.chunk')}) ASC`,
+    orderBy: [
+      ...new Set([source.timestampValueExpression, preciseTimestampExpression]),
+      `toUInt64OrZero(${eventAttributeExpression('rr-web.offset')})`,
+      `toUInt64OrZero(${eventAttributeExpression('rr-web.chunk')})`,
+    ]
+      .map(expression => `${expression} ASC`)
+      .join(', '),
     limit: {
       limit,
       offset,
@@ -649,6 +708,10 @@ export function useRRWebEventStream(
 
       const MAX_LIMIT = 1e6;
 
+      const preciseTimestampExpression = await getRRWebTimestampExpression(
+        source,
+        metadata,
+      );
       const query = await renderChartConfig(
         buildRRWebStreamChartConfig({
           source,
@@ -660,6 +723,7 @@ export function useRRWebEventStream(
           limit: Math.min(MAX_LIMIT, parseInt(queryLimit)),
           offset: parseInt(offset),
           getSessionSourceFieldExpression,
+          preciseTimestampExpression,
         }),
         metadata,
         source.querySettings,