remote functions (#67)

* remote function

* connection

* masthead update

* formatting

* extend description

* reservation off

* tf update

* dataform export routine

* bq connections

* spark procedure role

* docker update

* lint

* lint

* lint

* lint

* more spark roles

* mh submodule

* submodule

* lint

* lint

* use connections from dataform

* sync with latest version

* fix package versions

* remove submodule

* update packages

* test

* test

* rewrite triggers

* adjust bq export

* mh roles update

* cleanup

* packages update

* test

* packages update

* arguments renamed

* current nodejs

* Update definitions/output/reports/tech_report_technologies.js

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* remove reservation

* switch order

* deactivate spark procedure

* deactivate standard exports

* updated function calling

* standard reports export draft

* update description

* dependabot update

* fix month

* fix query generation

* update

* fix env var

* fix export config

* formatting

* fix

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: max-ostapenko

.github/dependabot.yml

@@ -10,6 +10,18 @@ updates:
     schedule:
       interval: "weekly"
   - package-ecosystem: "npm"
-    directory: "/src"
+    directory: "/infra/bigquery-export"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "npm"
+    directory: "infra/dataform-export"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "npm"
+    directory: "infra/dataform-trigger"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "terraform"
+    directory: "infra/tf/"
     schedule:
       interval: "weekly"

Makefile

@@ -11,4 +11,9 @@ tf_plan:
 
 tf_apply:
 	terraform -chdir=infra/tf init && terraform -chdir=infra/tf apply -auto-approve
-	cd infra/bigquery-export/ && npm install && npm run buildpack
+
+bigquery_export_deploy:
+	cd infra/bigquery-export && npm install && npm run buildpack
+
+#bigquery_export_spark_deploy:
+#	cd infra/bigquery_export_spark && gcloud builds submit --region=global --tag us-docker.pkg.dev/httparchive/bigquery-spark-procedures/firestore_export:latest

README.md

@@ -47,7 +47,7 @@ Consumers:
 
 ### Triggering workflows
 
-In order to unify the workflow triggering mechanism, we use [a Cloud Run function](./src/README.md) that can be invoked in a number of ways (e.g. listen to PubSub messages), do intermediate checks and trigger the particular Dataform workflow execution configuration.
+In order to unify the workflow triggering mechanism, we use [a Cloud Run function](./infra/README.md) that can be invoked in a number of ways (e.g. listen to PubSub messages), do intermediate checks and trigger the particular Dataform workflow execution configuration.
 
 ## Contributing
 
@@ -59,5 +59,38 @@ In order to unify the workflow triggering mechanism, we use [a Cloud Run functio
 
 #### Workspace hints
 
-1. In `workflow_settings.yaml` set `env_name: dev` to process sampled data.
-2. In `includes/constants.js` set `today` or other variables to a custome value.
+1. In `workflow_settings.yaml` set `environment: dev` to process sampled data.
+2. For development and testing, you can modify variables in `includes/constants.js`, but note that these are programmatically generated.
+
+## Repository Structure
+
+- `definitions/` - Contains the core Dataform SQL definitions and declarations
+  - `output/` - Contains the main pipeline transformation logic
+  - `declarations/` - Contains referenced tables/views declarations and other resources definitions
+- `includes/` - Contains shared JavaScript utilities and constants
+- `infra/` - Infrastructure code and deployment configurations
+  - `dataform-trigger/` - Cloud Run function for workflow automation
+  - `tf/` - Terraform configurations
+  - `bigquery-export/` - BigQuery export configurations
+- `docs/` - Additional documentation
+
+## Development Setup
+
+1. Install dependencies:
+
+    ```bash
+    npm install
+    ```
+
+2. Available Scripts:
+
+    - `npm run format` - Format code using Standard.js, fix Markdown issues, and format Terraform files
+    - `npm run lint` - Run linting checks on JavaScript, Markdown files, and compile Dataform configs
+
+## Code Quality
+
+This repository uses:
+
+- Standard.js for JavaScript code style
+- Markdownlint for Markdown file formatting
+- Dataform's built-in compiler for SQL validation

definitions/output/reports/cwv_tech_adoption.js

@@ -13,7 +13,6 @@ publish('cwv_tech_adoption', {
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "report_cwv_tech_complete", "date": "${pastMonth}", "name": "adoption", "type": "report"} */
 SELECT
   date,
   app AS technology,
@@ -30,4 +29,18 @@ GROUP BY
   app,
   rank,
   geo
-`)
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "destination": "firestore",
+        "config": {
+          "database": "tech-report-apis-${constants.environment}",
+          "collection": "adoption",
+          "type": "report",
+          "date": "${pastMonth}"
+        },
+        "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
+      }'''
+    );
+  `)

definitions/output/reports/cwv_tech_categories.js

@@ -5,7 +5,6 @@ publish('cwv_tech_categories', {
   type: 'table',
   tags: ['crux_ready']
 }).query(ctx => `
-/* {"dataform_trigger": "report_cwv_tech_complete", "name": "categories", "type": "dict"} */
 WITH pages AS (
   SELECT DISTINCT
     client,
@@ -50,7 +49,7 @@ technology_stats AS (
   SELECT
     technology,
     category_obj AS categories,
-    SUM(origins.dektop + origins.mobile) AS total_origins
+    SUM(origins.desktop + origins.mobile) AS total_origins
   FROM ${ctx.ref('reports', 'cwv_tech_technologies')}
   GROUP BY
     technology,
@@ -91,4 +90,17 @@ SELECT
   ) AS origins,
   NULL AS technologies
 FROM total_pages
-`)
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "destination": "firestore",
+        "config": {
+          "database": "tech-report-apis-${constants.environment}",
+          "collection": "categories",
+          "type": "dict"
+        },
+        "query": "SELECT * FROM ${ctx.self()}"
+      }'''
+    );
+  `)

definitions/output/reports/cwv_tech_core_web_vitals.js

@@ -68,7 +68,6 @@ return Object.values(vitals)
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "report_cwv_tech_complete", "date": "${pastMonth}", "name": "core_web_vitals", "type": "report"} */
 SELECT
   date,
   app AS technology,
@@ -98,4 +97,18 @@ GROUP BY
   app,
   rank,
   geo
-`)
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "destination": "firestore",
+        "config": {
+          "database": "tech-report-apis-${constants.environment}",
+          "collection": "core_web_vitals",
+          "type": "report",
+          "date": "${pastMonth}"
+        },
+        "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
+      }'''
+    );
+  `)

definitions/output/reports/cwv_tech_lighthouse.js

@@ -54,7 +54,6 @@ return Object.values(lighthouse)
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "report_cwv_tech_complete", "date": "${pastMonth}", "name": "lighthouse", "type": "report"} */
 SELECT
   date,
   app AS technology,
@@ -75,4 +74,18 @@ GROUP BY
   app,
   rank,
   geo
-`)
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "destination": "firestore",
+        "config": {
+          "database": "tech-report-apis-${constants.environment}",
+          "collection": "lighthouse",
+          "type": "report",
+          "date": "${pastMonth}"
+        },
+        "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
+      }'''
+    );
+  `)

definitions/output/reports/cwv_tech_page_weight.js

@@ -46,7 +46,6 @@ return Object.values(pageWeight)
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "report_cwv_tech_complete", "date": "${pastMonth}", "name": "page_weight", "type": "report"} */
 SELECT
   date,
   app AS technology,
@@ -65,4 +64,18 @@ GROUP BY
   app,
   rank,
   geo
-`)
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "destination": "firestore",
+        "config": {
+          "database": "tech-report-apis-${constants.environment}",
+          "collection": "page_weight",
+          "type": "report",
+          "date": "${pastMonth}"
+        },
+        "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
+      }'''
+    );
+  `)

definitions/output/reports/cwv_tech_technologies.js

@@ -5,7 +5,6 @@ publish('cwv_tech_technologies', {
   type: 'table',
   tags: ['crux_ready']
 }).query(ctx => `
-/* {"dataform_trigger": "report_cwv_tech_complete", "name": "technologies", "type": "dict"} */
 WITH pages AS (
   SELECT DISTINCT
     client,
@@ -86,4 +85,17 @@ SELECT
     MAX(IF(client = 'mobile', origins, 0)) AS mobile
   ) AS origins
 FROM total_pages
-`)
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "destination": "firestore",
+        "config": {
+          "database": "tech-report-apis-${constants.environment}",
+          "collection": "technologies",
+          "type": "dict"
+        },
+        "query": "SELECT * FROM ${ctx.self()}"
+      }'''
+    );
+  `)

definitions/output/reports/reports_dynamic.js

@@ -1,11 +1,48 @@
 const configs = new reports.HTTPArchiveReports()
 const metrics = configs.listMetrics()
 
+const bucket = 'httparchive'
+const storagePath = '/reports/'
+
+function generateExportQuery (metric, sql, params, ctx) {
+  let query = ''
+  if (sql.type === 'histogram') {
+    query = `
+SELECT
+  * EXCEPT(date)
+FROM ${ctx.self()}
+WHERE date = '${params.date}'
+`
+  } else if (sql.type === 'timeseries') {
+    query = `
+SELECT
+  FORMAT_DATE('%Y_%m_%d', date) AS date,
+  * EXCEPT(date)
+FROM ${ctx.self()}
+`
+  } else {
+    throw new Error('Unknown SQL type')
+  }
+
+  const queryOutput = query.replace(/[\r\n]+/g, ' ')
+  return queryOutput
+}
+
+function generateExportPath (metric, sql, params) {
+  if (sql.type === 'histogram') {
+    return `${storagePath}${params.date.replaceAll('-', '_')}/${metric.id}.json`
+  } else if (sql.type === 'timeseries') {
+    return `${storagePath}${metric.id}.json`
+  } else {
+    throw new Error('Unknown SQL type')
+  }
+}
+
 const iterations = []
 for (
-  let month = constants.currentMonth; month >= constants.currentMonth; month = constants.fnPastMonth(month)) {
+  let date = constants.currentMonth; date >= constants.currentMonth; date = constants.fnPastMonth(date)) {
   iterations.push({
-    date: month,
+    date,
     devRankFilter: constants.devRankFilter
   })
 }
@@ -18,29 +55,52 @@ if (iterations.length === 1) {
         type: 'incremental',
         protected: true,
         bigquery: sql.type === 'histogram' ? { partitionBy: 'date', clusterBy: ['client'] } : {},
-        schema: 'reports',
-        tags: ['crawl_complete', 'http_reports']
+        schema: 'reports'
+        // tags: ['crawl_complete', 'http_reports']
       }).preOps(ctx => `
 --DELETE FROM ${ctx.self()}
 --WHERE date = '${params.date}';
-  `).query(ctx => `
-/* {"dataform_trigger": "report_complete", "date": "${params.date}", "name": "${metric.id}", "type": "${sql.type}"} */` +
-sql.query(ctx, params))
+      `).query(
+        ctx => sql.query(ctx, params)
+      ).postOps(ctx => `
+SELECT
+  reports.run_export_job(
+    JSON '''{
+      "destination": "cloud_storage",
+      "config": {
+        "bucket": "${bucket}",
+        "name": "${generateExportPath(metric, sql, params)}"
+      },
+      "query": "${generateExportQuery(metric, sql, params, ctx)}"
+    }'''
+  );
+      `)
     })
   })
 } else {
   iterations.forEach((params, i) => {
     metrics.forEach(metric => {
       metric.SQL.forEach(sql => {
         operate(metric.id + '_' + sql.type + '_' + params.date, {
-          tags: ['crawl_complete']
+          // tags: ['crawl_complete']
         }).queries(ctx => `
 DELETE FROM reports.${metric.id}_${sql.type}
 WHERE date = '${params.date}';
 
-/* {"dataform_trigger": "report_complete", "date": "${params.date}", "name": "${metric.id}", "type": "${sql.type}"} */
-INSERT INTO reports.${metric.id}_${sql.type}` +
-        sql.query(ctx, params))
+INSERT INTO reports.${metric.id}_${sql.type}` + sql.query(ctx, params)
+        ).postOps(ctx => `
+  SELECT
+  reports.run_export_job(
+    JSON '''{
+      "destination": "cloud_storage",
+      "config": {
+        "bucket": "${bucket}",
+        "name": "${generateExportPath(metric, sql, params)}"
+      },
+      "query": "${generateExportQuery(metric, sql, params, ctx)}"
+    }'''
+  );
+        `)
       })
     })
   })