Merge branch 'judicial-snake' into judicial-snake

Author: max-ostapenko

Makefile

@@ -11,4 +11,9 @@ tf_plan:
 
 tf_apply:
 	terraform -chdir=infra/tf init && terraform -chdir=infra/tf apply -auto-approve
-	cd infra/bigquery-export/ && npm install && npm run buildpack
+
+bigquery_export_deploy:
+	cd infra/bigquery-export && npm install && npm run buildpack
+
+bigquery_export_spark_deploy:
+	cd infra/bigquery_export_spark && gcloud builds submit --region=global --tag us-docker.pkg.dev/httparchive/bigquery-spark-procedures/firestore_export:latest

README.md

@@ -47,7 +47,7 @@ Consumers:
 
 ### Triggering workflows
 
-In order to unify the workflow triggering mechanism, we use [a Cloud Run function](./src/README.md) that can be invoked in a number of ways (e.g. listen to PubSub messages), do intermediate checks and trigger the particular Dataform workflow execution configuration.
+In order to unify the workflow triggering mechanism, we use [a Cloud Run function](./infra/README.md) that can be invoked in a number of ways (e.g. listen to PubSub messages), do intermediate checks and trigger the particular Dataform workflow execution configuration.
 
 ## Contributing
 
@@ -59,5 +59,38 @@ In order to unify the workflow triggering mechanism, we use [a Cloud Run functio
 
 #### Workspace hints
 
-1. In `workflow_settings.yaml` set `env_name: dev` to process sampled data.
-2. In `includes/constants.js` set `today` or other variables to a custome value.
+1. In `workflow_settings.yaml` set `environment: dev` to process sampled data.
+2. For development and testing, you can modify variables in `includes/constants.js`, but note that these are programmatically generated.
+
+## Repository Structure
+
+- `definitions/` - Contains the core Dataform SQL definitions and declarations
+  - `output/` - Contains the main pipeline transformation logic
+  - `declarations/` - Contains referenced tables/views declarations and other resources definitions
+- `includes/` - Contains shared JavaScript utilities and constants
+- `infra/` - Infrastructure code and deployment configurations
+  - `dataform-trigger/` - Cloud Run function for workflow automation
+  - `tf/` - Terraform configurations
+  - `bigquery-export/` - BigQuery export configurations
+- `docs/` - Additional documentation
+
+## Development Setup
+
+1. Install dependencies:
+
+    ```bash
+    npm install
+    ```
+
+2. Available Scripts:
+
+    - `npm run format` - Format code using Standard.js, fix Markdown issues, and format Terraform files
+    - `npm run lint` - Run linting checks on JavaScript, Markdown files, and compile Dataform configs
+
+## Code Quality
+
+This repository uses:
+
+- Standard.js for JavaScript code style
+- Markdownlint for Markdown file formatting
+- Dataform's built-in compiler for SQL validation

definitions/declarations/httparchive.js

@@ -42,7 +42,7 @@ ORDER BY cnt_pages DESC
 )
 
 operate('create_reservation_assignment')
-  .tags(['crawl_complete'])
+  // .tags(['crawl_complete'])
   .queries(ctx => `
 CREATE ASSIGNMENT
 \`httparchive.region-us.retrospective-reprocessing.pipeline\`
@@ -52,8 +52,8 @@ OPTIONS (
 `)
 
 operate('drop_reservation_assignment')
-  .dependencies(['requests_10k'])
-  .tags(['crawl_complete'])
+  // .dependencies(['requests_10k'])
+  // .tags(['crawl_complete'])
   .queries(ctx => `
 DROP ASSIGNMENT IF EXISTS
 \`httparchive.region-us.retrospective-reprocessing.pipeline\`

definitions/output/reports/reports_dynamic.js

@@ -23,9 +23,21 @@ if (iterations.length === 1) {
       }).preOps(ctx => `
 --DELETE FROM ${ctx.self()}
 --WHERE date = '${params.date}';
-  `).query(ctx => `
-/* {"dataform_trigger": "report_complete", "date": "${params.date}", "name": "${metric.id}", "type": "${sql.type}"} */` +
-sql.query(ctx, params))
+      `).query(
+        ctx => sql.query(ctx, params)
+      ).postOps(ctx => `
+SELECT
+  reports.run_export_job(
+    JSON '''{
+      "destination": "cloud_storage",
+      "config": {
+        "bucket": "httparchive",
+        "name": "reports/${constants.environment}/${metric.id}_${sql.type}_${params.date}.json"
+      },
+      "query": "SELECT FORMAT_DATE('%Y_%m_%d', date) AS date, * EXCEPT(date) FROM ${ctx.self()}"
+    }'''
+  );
+      `)
     })
   })
 } else {
@@ -38,10 +50,23 @@ sql.query(ctx, params))
 DELETE FROM reports.${metric.id}_${sql.type}
 WHERE date = '${params.date}';
 
-/* {"dataform_trigger": "report_complete", "date": "${params.date}", "name": "${metric.id}", "type": "${sql.type}"} */
-INSERT INTO reports.${metric.id}_${sql.type}` +
-        sql.query(ctx, params))
+INSERT INTO reports.${metric.id}_${sql.type}` + sql.query(ctx, params)
+        ).postOps(ctx => `
+SELECT
+  reports.run_export_job(
+    JSON '''{
+      "dataform_trigger": "report_complete",
+      "date": "${params.date}",
+      "name": "${metric.id}",
+      "type": "${sql.type}",
+      "environment": "${constants.environment}"
+    }'''
+  );
+        `)
       })
     })
   })
 }
+
+// --"query": "SELECT * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${params.date}'"
+// `${this.storagePath}${date.replaceAll('-', '_')}/${metric}.json`

definitions/output/reports/tech_report_adoption.js

@@ -13,7 +13,6 @@ publish('tech_report_adoption', {
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "tech_report_complete", "date": "${pastMonth}", "name": "adoption", "type": "report"} */
 SELECT
   date,
   geo,
@@ -32,4 +31,15 @@ GROUP BY
   rank,
   technology,
   version
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "dataform_trigger": "tech_report_complete",
+        "date": "${pastMonth}",
+        "name": "adoption",
+        "type": "report",
+        "environment": "${constants.environment}"
+      }'''
+    );
 `)

definitions/output/reports/tech_report_audits.js

@@ -67,7 +67,6 @@ return Object.keys(auditMap).map(function(key) {
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "tech_report_complete", "date": "${pastMonth}", "name": "audits", "type": "report"} */
 SELECT
   date,
   geo,
@@ -86,4 +85,15 @@ GROUP BY
   rank,
   technology,
   version
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "dataform_trigger": "tech_report_complete",
+        "date": "${pastMonth}",
+        "name": "audits",
+        "type": "report",
+        "environment": "${constants.environment}"
+      }'''
+    );
 `)

definitions/output/reports/tech_report_categories.js

@@ -5,7 +5,6 @@ publish('tech_report_categories', {
   type: 'table',
   tags: ['tech_report']
 }).query(ctx => `
-/* {"dataform_trigger": "tech_report_complete", "name": "categories", "type": "dict"} */
 WITH pages AS (
   SELECT DISTINCT
     client,
@@ -91,4 +90,14 @@ FROM (
   FROM pages
   GROUP BY client
 )
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "dataform_trigger": "tech_report_complete",
+        "name": "categories",
+        "type": "dict",
+        "environment": "${constants.environment}"
+      }'''
+    );
 `)

definitions/output/reports/tech_report_core_web_vitals.js

@@ -68,7 +68,6 @@ return Object.values(vitals)
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "tech_report_complete", "date": "${pastMonth}", "name": "core_web_vitals", "type": "report"} */
 SELECT
   date,
   geo,
@@ -100,4 +99,15 @@ GROUP BY
   rank,
   technology,
   version
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "dataform_trigger": "tech_report_complete",
+        "date": "${pastMonth}",
+        "name": "core_web_vitals",
+        "type": "report",
+        "environment": "${constants.environment}"
+      }'''
+    );
 `)

definitions/output/reports/tech_report_lighthouse.js

@@ -54,7 +54,6 @@ return Object.values(lighthouse)
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "tech_report_complete", "date": "${pastMonth}", "name": "lighthouse", "type": "report"} */
 SELECT
   date,
   geo,
@@ -76,4 +75,15 @@ GROUP BY
   rank,
   technology,
   version
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "dataform_trigger": "tech_report_complete",
+        "date": "${pastMonth}",
+        "name": "lighthouse",
+        "type": "report",
+        "environment": "${constants.environment}"
+      }'''
+    );
 `)

definitions/output/reports/tech_report_page_weight.js

@@ -48,7 +48,6 @@ return Object.values(pageWeight)
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "tech_report_complete", "date": "${pastMonth}", "name": "page_weight", "type": "report"} */
 SELECT
   date,
   geo,
@@ -69,4 +68,15 @@ GROUP BY
   rank,
   technology,
   version
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "dataform_trigger": "tech_report_complete",
+        "date": "${pastMonth}",
+        "name": "page_weight",
+        "type": "report",
+        "environment": "${constants.environment}"
+      }'''
+    );
 `)