From 959ad5e960318e04bc4718cdfc71653fda0dac66 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Tue, 21 Jan 2025 09:18:12 +0100
Subject: [PATCH] clean no valid technologies (#53)

---
 definitions/output/crawl/pages.js | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/definitions/output/crawl/pages.js b/definitions/output/crawl/pages.js
index b39624f..098521e 100644
--- a/definitions/output/crawl/pages.js
+++ b/definitions/output/crawl/pages.js
@@ -73,7 +73,7 @@ WHERE date = '${constants.currentMonth}' AND
   ${constants.devRankFilter}
 `).postOps(ctx => `
 CREATE TEMP TABLE technologies_cleaned AS (
-  WITH wappalyzer AS (
+  WITH technologies AS (
     SELECT DISTINCT
       name AS technology,
       categories
@@ -101,13 +101,13 @@ CREATE TEMP TABLE technologies_cleaned AS (
     LEFT JOIN pages.categories AS category
     WHERE
       -- Technology is corrupted
-      technology NOT IN (SELECT DISTINCT technology FROM wappalyzer) OR
+      technology NOT IN (SELECT DISTINCT technology FROM technologies) OR
       -- Technology's category is corrupted
       CONCAT(technology, category) NOT IN (
         SELECT DISTINCT
           CONCAT(technology, category)
-        FROM wappalyzer
-        LEFT JOIN wappalyzer.categories AS category
+        FROM technologies
+        INNER JOIN technologies.categories AS category
       )
   ),
 
@@ -118,14 +118,14 @@ CREATE TEMP TABLE technologies_cleaned AS (
       page,
       ARRAY_AGG(STRUCT(
         pages.technology,
-        wappalyzer.categories,
+        technologies.categories,
         pages.info
       )) AS technologies
     FROM pages
     INNER JOIN impacted_pages
     USING (client, page)
-    INNER JOIN wappalyzer
-    ON pages.technology = wappalyzer.technology
+    INNER JOIN technologies
+    USING (technology)
     GROUP BY
       client,
       page
@@ -134,8 +134,10 @@ CREATE TEMP TABLE technologies_cleaned AS (
   SELECT
     client,
     page,
-    technologies
-  FROM reconstructed_technologies
+    reconstructed_technologies.technologies
+  FROM impacted_pages
+  LEFT JOIN reconstructed_technologies
+  USING(client,page)
 );
 
 -- Update the crawl.pages table with the cleaned and restored technologies