lint

Author: max-ostapenko

definitions/declarations/httparchive.js

@@ -4,35 +4,7 @@
     schema: 'crawl_staging',
     name: table
   })
-)
-
-// See https://github.com/HTTPArchive/dataform/issues/43
-assert('corrupted_technology_values')
-  .tags(['crawl_complete'])
-  .query(ctx => `
-SELECT
-  date,
-  client,
-  tech,
-  COUNT(DISTINCT page) AS cnt_pages,
-  ARRAY_AGG(DISTINCT page LIMIT 3) AS sample_pages
-FROM ${ctx.ref('crawl_staging', 'pages')} AS pages
-LEFT JOIN pages.technologies AS tech
-LEFT JOIN tech.categories AS category
-WHERE
-  date = '${constants.currentMonth}' AND
-  (
-    tech.technology NOT IN (SELECT DISTINCT name FROM wappalyzer.technologies)
-    OR category NOT IN (SELECT DISTINCT name FROM wappalyzer.categories)
-    OR ARRAY_LENGTH(tech.categories) = 0
-  )
-GROUP BY
-  date,
-  client,
-  tech
-HAVING cnt_pages > 20
-ORDER BY cnt_pages DESC
-`);
+);
 
 // Wappalyzer tables source: https://github.com/HTTPArchive/wappalyzer/blob/main/.github/workflows/upload.yml
 ['technologies', 'categories'].forEach(table =>

definitions/output/crawl/pages.js

@@ -1,3 +1,30 @@
+// See https://github.com/HTTPArchive/dataform/issues/43
+assert('corrupted_technology_values')
+  .tags(['crawl_complete'])
+  .query(ctx => `
+SELECT
+  date,
+  client,
+  tech,
+  COUNT(DISTINCT page) AS cnt_pages,
+  ARRAY_AGG(DISTINCT page LIMIT 3) AS sample_pages
+FROM ${ctx.ref('crawl_staging', 'pages')} AS pages
+LEFT JOIN pages.technologies AS tech
+LEFT JOIN tech.categories AS category
+WHERE
+  date = '${constants.currentMonth}' AND
+  (
+    tech.technology NOT IN (SELECT DISTINCT name FROM wappalyzer.technologies)
+    OR category NOT IN (SELECT DISTINCT name FROM wappalyzer.categories)
+    OR ARRAY_LENGTH(tech.categories) = 0
+  )
+GROUP BY
+  date,
+  client,
+  tech
+ORDER BY cnt_pages DESC
+  `)
+
 publish('pages', {
   type: 'incremental',
   protected: true,