Merge branch 'main' into judicial-snake

Author: max-ostapenko

definitions/declarations/httparchive.js

@@ -4,34 +4,7 @@
     schema: 'crawl_staging',
     name: table
   })
-)
-
-// See https://github.com/HTTPArchive/dataform/issues/43
-assert('corrupted_technology_values')
-  .tags(['crawl_complete'])
-  .query(ctx => `
-SELECT
-  date,
-  client,
-  tech,
-  COUNT(DISTINCT page) AS cnt_pages,
-  ARRAY_AGG(DISTINCT page LIMIT 3) AS sample_pages
-FROM ${ctx.ref('crawl_staging', 'pages')} AS pages
-LEFT JOIN pages.technologies AS tech
-LEFT JOIN tech.categories AS category
-WHERE
-  date = '${constants.currentMonth}' AND
-  (
-    tech.technology NOT IN (SELECT DISTINCT name FROM wappalyzer.technologies)
-    OR category NOT IN (SELECT DISTINCT name FROM wappalyzer.categories)
-    OR ARRAY_LENGTH(tech.categories) = 0
-  )
-GROUP BY
-  date,
-  client,
-  tech
-ORDER BY cnt_pages DESC
-`);
+);
 
 // Wappalyzer tables source: https://github.com/HTTPArchive/wappalyzer/blob/main/.github/workflows/upload.yml
 ['technologies', 'categories'].forEach(table =>

definitions/output/crawl/pages.js

@@ -1,3 +1,30 @@
+// See https://github.com/HTTPArchive/dataform/issues/43
+assert('corrupted_technology_values')
+  .tags(['crawl_complete'])
+  .query(ctx => `
+SELECT
+  date,
+  client,
+  tech,
+  COUNT(DISTINCT page) AS cnt_pages,
+  ARRAY_AGG(DISTINCT page LIMIT 3) AS sample_pages
+FROM ${ctx.ref('crawl_staging', 'pages')} AS pages
+LEFT JOIN pages.technologies AS tech
+LEFT JOIN tech.categories AS category
+WHERE
+  date = '${constants.currentMonth}' AND
+  (
+    tech.technology NOT IN (SELECT DISTINCT name FROM wappalyzer.technologies)
+    OR category NOT IN (SELECT DISTINCT name FROM wappalyzer.categories)
+    OR ARRAY_LENGTH(tech.categories) = 0
+  )
+GROUP BY
+  date,
+  client,
+  tech
+ORDER BY cnt_pages DESC
+  `)
+
 publish('pages', {
   type: 'incremental',
   protected: true,
@@ -47,7 +74,8 @@ publish('pages', {
     technologies: 'Technologies detected at runtime (see https://www.wappalyzer.com/)',
     metadata: 'Additional metadata about the test'
   },
-  tags: ['crawl_complete']
+  tags: ['crawl_complete'],
+  dependOnDependencyAssertions: true
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}
 WHERE date = '${constants.currentMonth}' AND

definitions/output/crawl/requests.js

@@ -38,23 +38,27 @@ publish('requests', {
   },
   tags: ['crawl_complete']
 }).preOps(ctx => `
-FOR client_value IN (SELECT * FROM UNNEST(['desktop', 'mobile']) AS client) DO
-  FOR is_root_page_value IN (SELECT * FROM UNNEST([TRUE, FALSE]) AS is_root_page) DO
+FOR client_var IN (SELECT * FROM UNNEST(['desktop', 'mobile']) AS value) DO
+  FOR is_root_page_var IN (SELECT * FROM UNNEST([TRUE, FALSE]) AS value) DO
+    FOR rank_lt_50M_var IN (SELECT * FROM UNNEST([TRUE, FALSE]) AS value) DO
 
-    -- Delete old entries
-    DELETE FROM ${ctx.self()}
-    WHERE date = '${constants.currentMonth}'
-      AND client = client_value.client
-      AND is_root_page = is_root_page_value.is_root_page;
+      -- Delete old entries
+      DELETE FROM ${ctx.self()}
+      WHERE date = '${constants.currentMonth}' AND
+        client = client_var.value AND
+        is_root_page = is_root_page_var.value AND
+        (rank < 50000000) = rank_lt_50M_var.value;
 
-    -- Insert new entries
-    INSERT INTO ${ctx.self()}
-    SELECT *
-    FROM ${ctx.ref('crawl_staging', 'requests')}
-    WHERE date = '${constants.currentMonth}' AND
-      client = client_value.client AND
-      is_root_page = is_root_page_value.is_root_page ${constants.devRankFilter};
+      -- Insert new entries
+      INSERT INTO ${ctx.self()}
+      SELECT *
+      FROM ${ctx.ref('crawl_staging', 'requests')}
+      WHERE date = '${constants.currentMonth}' AND
+        client = client_var.value AND
+        is_root_page = is_root_page_var.value AND
+        (rank < 50000000) = rank_lt_50M_var.value ${constants.devRankFilter};
 
+    END FOR;
   END FOR;
 END FOR;
 `).query(ctx => `

definitions/output/reports/cwv_tech_categories.js

@@ -50,7 +50,7 @@ technology_stats AS (
   SELECT
     technology,
     category_obj AS categories,
-    SUM(origins) AS total_origins
+    SUM(origins.dektop + origins.mobile) AS total_origins
   FROM ${ctx.ref('reports', 'cwv_tech_technologies')}
   GROUP BY
     technology,

package-lock.json

@@ -2,7 +2,7 @@
   "name": "crawl-data",
   "author": "@max-ostapenko",
   "dependencies": {
-    "@dataform/core": "3.0.15"
+    "@dataform/core": "^3.0.19"
   },
   "scripts": {
     "format": "npx standard --fix; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint . --fix; terraform -chdir=infra/tf fmt -recursive",