Privacy 2024 queries

sql/2024/privacy/ads_accounts_distribution.sql

@@ -0,0 +1,64 @@
+WITH publishers AS (
+  SELECT
+    page,
+    JSON_QUERY(custom_metrics, '$.ads.ads.account_types') AS ads_account_types,
+    JSON_QUERY(custom_metrics, '$.ads.app_ads.account_types') AS app_ads_account_types
+  FROM `httparchive.all.pages`
+  WHERE date = '2024-06-01' AND
+    is_root_page = TRUE AND
+    (CAST(JSON_VALUE(custom_metrics, '$.ads.ads.account_count') AS INT64) > 0 OR
+      CAST(JSON_VALUE(custom_metrics, '$.ads.app_ads.account_count') AS INT64) > 0)
+), ads_accounts AS (
+  SELECT
+    page,
+    CEIL(CAST(JSON_VALUE(ads_account_types, '$.direct.account_count') AS INT64) / 100) * 100 AS direct_account_count_bucket,
+    CEIL(CAST(JSON_VALUE(ads_account_types, '$.reseller.account_count') AS INT64) / 100) * 100 AS reseller_account_count_bucket,
+    COUNT(DISTINCT page) OVER () AS total_pages
+  FROM publishers
+), app_ads_accounts AS (
+  SELECT
+    page,
+    CEIL(CAST(JSON_VALUE(app_ads_account_types, '$.direct.account_count') AS INT64) / 100) * 100 AS direct_account_count_bucket,
+    CEIL(CAST(JSON_VALUE(app_ads_account_types, '$.reseller.account_count') AS INT64) / 100) * 100 AS reseller_account_count_bucket,
+    COUNT(DISTINCT page) OVER () AS total_pages
+  FROM publishers
+)
+
+SELECT
+  'ads' AS source,
+  'direct' AS account_type,
+  direct_account_count_bucket AS account_count_bucket,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(DISTINCT page) AS number_of_pages
+FROM ads_accounts
+GROUP BY source, direct_account_count_bucket
+UNION ALL
+SELECT
+  'ads' AS source,
+  'reseller' AS account_type,
+  reseller_account_count_bucket AS account_count_bucket,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(DISTINCT page) AS number_of_pages
+FROM ads_accounts
+GROUP BY source, reseller_account_count_bucket
+UNION ALL
+SELECT
+  'app_ads' AS source,
+  'direct' AS account_type,
+  direct_account_count_bucket AS account_count_bucket,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(DISTINCT page) AS number_of_pages
+FROM app_ads_accounts
+GROUP BY source, direct_account_count_bucket
+UNION ALL
+SELECT
+  'app_ads' AS source,
+  'reseller' AS account_type,
+  reseller_account_count_bucket AS account_count_bucket,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(DISTINCT page) AS number_of_pages
+FROM app_ads_accounts
+GROUP BY source, reseller_account_count_bucket
+
+ORDER BY account_count_bucket ASC
+LIMIT 1000

sql/2024/privacy/ads_and_sellers_graph.sql

@@ -0,0 +1,114 @@
+WITH RECURSIVE pages AS (
+  SELECT
+    CASE page -- Publisher websites may redirect to an SSP domain, and need to use redirected domain instead of page domain. CASE needs to be replaced with a more robust solution from HTTPArchive/custom-metrics#136.
+      WHEN 'https://www.chunkbase.com/' THEN 'cafemedia.com'
+      ELSE NET.REG_DOMAIN(page)
+    END AS page_domain,
+    JSON_QUERY(ANY_VALUE(custom_metrics), '$.ads') AS ads_metrics
+  FROM `httparchive.all.pages`
+  WHERE date = '2024-06-01' AND
+    is_root_page = TRUE
+  GROUP BY page_domain
+), ads AS (
+  SELECT
+    page_domain,
+    JSON_QUERY(ads_metrics, '$.ads.account_types') AS ad_accounts
+  FROM pages
+  WHERE
+    CAST(JSON_VALUE(ads_metrics, '$.ads.account_count') AS INT64) > 0
+), sellers AS (
+  SELECT
+    page_domain,
+    JSON_QUERY(ads_metrics, '$.sellers.seller_types') AS ad_sellers
+  FROM pages
+  WHERE
+    CAST(JSON_VALUE(ads_metrics, '$.sellers.seller_count') AS INT64) > 0
+), relationships_web AS (
+  SELECT
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS demand,
+    'Web' AS supply,
+    'direct' AS relationship,
+    page_domain AS publisher
+  FROM ads, UNNEST(JSON_VALUE_ARRAY(ad_accounts, '$.direct.domains')) AS domain
+  UNION ALL
+  SELECT
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS demand,
+    'Web' AS supply,
+    'indirect' AS relationship,
+    page_domain AS publisher
+  FROM ads, UNNEST(JSON_VALUE_ARRAY(ad_accounts, '$.reseller.domains')) AS domain
+  UNION ALL
+  SELECT
+    page_domain AS demand,
+    'Web' AS supply,
+    'direct' AS relationship,
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS publisher
+  FROM sellers, UNNEST(JSON_VALUE_ARRAY(ad_sellers, '$.publisher.domains')) AS domain
+  UNION ALL
+  SELECT
+    page_domain AS demand,
+    'Web' AS supply,
+    'direct' AS relationship,
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS publisher
+  FROM sellers, UNNEST(JSON_VALUE_ARRAY(ad_sellers, '$.both.domains')) AS domain
+), relationships_adtech AS (
+  SELECT
+    page_domain AS demand,
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS supply,
+    'indirect' AS relationship
+  FROM sellers, UNNEST(JSON_VALUE_ARRAY(ad_sellers, '$.intermediary.domains')) AS domain
+  UNION ALL
+  SELECT
+    page_domain AS demand,
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS supply,
+    'indirect' AS relationship
+  FROM sellers, UNNEST(JSON_VALUE_ARRAY(ad_sellers, '$.both.domains')) AS domain
+), nodes AS (
+  (
+    SELECT
+      demand,
+      supply,
+      CONCAT(demand, '-', supply) AS path,
+      relationship,
+      HLL_COUNT.INIT(publisher) AS supply_sketch
+    FROM relationships_web
+    GROUP BY demand, supply, relationship
+  )
+  UNION ALL
+  (
+    SELECT
+      relationships_grouped.demand AS demand,
+      relationships_grouped.supply AS supply,
+      CONCAT(relationships_grouped.demand, '-', nodes.path) AS path,
+      relationships_grouped.relationship AS relationship,
+      nodes.supply_sketch AS supply_sketch
+    FROM (
+      SELECT
+        demand,
+        supply,
+        relationship
+      FROM relationships_adtech
+      GROUP BY
+        demand,
+        supply,
+        relationship
+    ) AS relationships_grouped
+    INNER JOIN nodes
+    ON relationships_grouped.supply = nodes.demand AND
+      nodes.supply_sketch IS NOT NULL AND
+      nodes.relationship = 'indirect' AND
+      relationships_grouped.demand IS NOT NULL AND
+      STRPOS(nodes.path, relationships_grouped.demand) = 0
+  )
+)
+
+SELECT
+  supply,
+  demand,
+  HLL_COUNT.MERGE(supply_sketch) AS publishers_count,
+  relationship,
+  path
+FROM nodes
+GROUP BY demand, supply, relationship, path
+ORDER BY publishers_count DESC
+LIMIT 5000

sql/2024/privacy/ads_lines_distribution.sql

@@ -0,0 +1,45 @@
+WITH RECURSIVE pages AS (
+  SELECT
+    CASE page -- publisher websites may redirect to an SSP domain, and need to use redirected domain instead of page domain
+      WHEN 'https://www.chunkbase.com/' THEN 'cafemedia.com'
+      ELSE NET.REG_DOMAIN(page)
+    END AS page,
+    CAST(JSON_VALUE(custom_metrics, '$.ads.ads.line_count') AS INT64) AS ads_line_count,
+    CAST(JSON_VALUE(custom_metrics, '$.ads.app_ads.line_count') AS INT64) AS app_ads_line_count
+  FROM `httparchive.all.pages`
+  WHERE date = '2024-06-01' AND
+    is_root_page = TRUE
+), ads AS (
+  SELECT
+    page,
+    CEIL(ads_line_count / 100) * 100 AS line_count_bucket,
+    COUNT(DISTINCT page) OVER () AS total_pages
+  FROM pages
+  WHERE ads_line_count > 0
+), app_ads AS (
+  SELECT
+    page,
+    CEIL(app_ads_line_count / 100) * 100 AS line_count_bucket,
+    COUNT(DISTINCT page) OVER () AS total_pages
+  FROM pages
+  WHERE app_ads_line_count > 0
+)
+
+SELECT
+  'ads.txt' AS type,
+  line_count_bucket,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(DISTINCT page) AS number_of_pages
+FROM ads
+GROUP BY line_count_bucket
+HAVING line_count_bucket <= 10000
+UNION ALL
+SELECT
+  'app-ads.txt' AS type,
+  line_count_bucket,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(DISTINCT page) AS number_of_pages
+FROM app_ads
+GROUP BY line_count_bucket
+HAVING line_count_bucket <= 10000
+ORDER BY type, line_count_bucket ASC

sql/2024/privacy/ccpa_most_common_phrases.sql

@@ -0,0 +1,31 @@
+WITH pages_with_phrase AS (
+  SELECT
+    client,
+    rank_grouping,
+    page,
+    COUNT(DISTINCT page) OVER (PARTITION BY client, rank_grouping) AS total_pages_with_phrase_in_rank_group,
+    JSON_QUERY_ARRAY(custom_metrics, '$.privacy.ccpa_link.CCPALinkPhrases') AS ccpa_link_phrases
+  FROM `httparchive.all.pages`, --TABLESAMPLE SYSTEM (0.01 PERCENT)
+    UNNEST([1000, 10000, 100000, 1000000, 10000000, 100000000]) AS rank_grouping
+  WHERE date = '2024-06-01' AND
+    is_root_page = true AND
+    rank <= rank_grouping AND
+    array_length(JSON_QUERY_ARRAY(custom_metrics, '$.privacy.ccpa_link.CCPALinkPhrases')) > 0
+)
+
+SELECT
+  client,
+  rank_grouping,
+  link_phrase,
+  COUNT(DISTINCT page) AS num_pages,
+  COUNT(DISTINCT page) / any_value(total_pages_with_phrase_in_rank_group) AS pct_pages
+FROM pages_with_phrase,
+  UNNEST(ccpa_link_phrases) AS link_phrase
+GROUP BY
+  link_phrase,
+  rank_grouping,
+  client
+ORDER BY
+  rank_grouping,
+  client,
+  num_pages DESC

sql/2024/privacy/ccpa_prevalence.sql

@@ -0,0 +1,27 @@
+WITH pages AS (
+  SELECT
+    client,
+    rank_grouping,
+    page,
+    JSON_VALUE(custom_metrics, '$.privacy.ccpa_link.hasCCPALink') AS has_ccpa_link
+  FROM `httparchive.all.pages`, --    TABLESAMPLE SYSTEM (0.0025 PERCENT)
+    UNNEST([1000, 10000, 100000, 1000000, 10000000, 100000000]) AS rank_grouping
+  WHERE date = '2024-06-01' AND
+    is_root_page = true AND
+    rank <= rank_grouping
+)
+
+SELECT
+  client,
+  rank_grouping,
+  has_ccpa_link,
+  COUNT(DISTINCT page) AS num_pages
+FROM pages
+GROUP BY
+  has_ccpa_link,
+  rank_grouping,
+  client
+ORDER BY
+  rank_grouping,
+  client,
+  has_ccpa_link

sql/2024/privacy/common_ads_variables.sql

@@ -0,0 +1,29 @@
+WITH RECURSIVE pages AS (
+  SELECT
+    page,
+    JSON_QUERY(custom_metrics, '$.ads.ads') AS ads_metrics
+  FROM `httparchive.all.pages`
+  WHERE
+    date = '2024-06-01' AND
+    is_root_page = TRUE AND
+    CAST(JSON_VALUE(custom_metrics, '$.ads.ads.account_count') AS INT64) > 0
+), ads AS (
+  SELECT
+    page,
+    variable,
+    COUNT(DISTINCT page) OVER() AS total_publishers
+  FROM pages,
+    UNNEST(JSON_VALUE_ARRAY(ads_metrics, '$.variables')) AS variable
+  WHERE
+    CAST(JSON_VALUE(ads_metrics, '$.account_types.reseller.account_count') AS INT64) > 0 OR
+    CAST(JSON_VALUE(ads_metrics, '$.account_types.direct.account_count') AS INT64) > 0
+)
+
+SELECT
+  variable,
+  COUNT(DISTINCT page) / ANY_VALUE(total_publishers) AS pct_publishers,
+  COUNT(DISTINCT page) AS number_of_publishers
+FROM ads
+GROUP BY variable
+ORDER BY pct_publishers DESC
+LIMIT 100

sql/2024/privacy/cookies_top_first_party_names.sql

@@ -0,0 +1,35 @@
+-- Most common cookie names, by number of domains on which they appear. Goal is to identify common trackers that use first-party cookies across sites.
+
+WITH pages AS (
+  SELECT
+    client,
+    root_page,
+    custom_metrics,
+    COUNT(DISTINCT net.host(root_page)) OVER(PARTITION BY client) AS total_domains
+  FROM `httparchive.all.pages`
+  WHERE date = '2024-06-01'
+), cookies AS (
+  SELECT
+    client,
+    cookie,
+    NET.HOST(JSON_VALUE(cookie, '$.domain')) AS cookie_host,
+    NET.HOST(root_page) AS firstparty_host,
+    total_domains
+  FROM pages,
+    UNNEST(JSON_QUERY_ARRAY(custom_metrics, '$.cookies')) AS cookie
+)
+
+SELECT
+  client,
+  COUNT(DISTINCT firstparty_host) AS domain_count,
+  COUNT(DISTINCT firstparty_host) / any_value(total_domains) AS pct_domains,
+  JSON_VALUE(cookie, '$.name') AS cookie_name
+FROM cookies
+WHERE firstparty_host LIKE '%' || cookie_host
+GROUP BY
+  client,
+  cookie_name
+ORDER BY
+  domain_count DESC,
+  client DESC
+LIMIT 500

sql/2024/privacy/cookies_top_third_party_domains.sql

@@ -0,0 +1,35 @@
+WITH pages AS (
+  SELECT
+    page,
+    client,
+    root_page,
+    custom_metrics,
+    COUNT(DISTINCT page) OVER (PARTITION BY client) AS total_pages
+  FROM `httparchive.all.pages`
+  WHERE date = '2024-06-01'
+), cookies AS (
+  SELECT
+    client,
+    page,
+    cookie,
+    NET.HOST(JSON_VALUE(cookie, '$.domain')) AS cookie_host,
+    NET.HOST(root_page) AS firstparty_host,
+    total_pages
+  FROM pages,
+    UNNEST(JSON_QUERY_ARRAY(custom_metrics, '$.cookies')) AS cookie
+)
+
+SELECT
+  client,
+  cookie_host,
+  COUNT(DISTINCT page) AS page_count,
+  COUNT(DISTINCT page) / any_value(total_pages) AS pct_pages
+FROM cookies
+WHERE firstparty_host NOT LIKE '%' || cookie_host
+GROUP BY
+  client,
+  cookie_host
+ORDER BY
+  page_count DESC,
+  client
+LIMIT 500