Privacy 2024 Chapter

sql/2024/privacy/ads_accounts_distribution.sql

@@ -0,0 +1,64 @@
+WITH publishers AS (
+  SELECT
+    page,
+    JSON_QUERY(custom_metrics, '$.ads.ads.account_types') AS ads_account_types,
+    JSON_QUERY(custom_metrics, '$.ads.app_ads.account_types') AS app_ads_account_types
+  FROM `httparchive.all.pages`
+  WHERE date = '2024-06-01' AND
+    is_root_page = TRUE AND
+    (CAST(JSON_VALUE(custom_metrics, '$.ads.ads.account_count') AS INT64) > 0 OR
+      CAST(JSON_VALUE(custom_metrics, '$.ads.app_ads.account_count') AS INT64) > 0)
+), ads_accounts AS (
+  SELECT
+    page,
+    CEIL(CAST(JSON_VALUE(ads_account_types, '$.direct.account_count') AS INT64) / 100) * 100 AS direct_account_count_bucket,
+    CEIL(CAST(JSON_VALUE(ads_account_types, '$.reseller.account_count') AS INT64) / 100) * 100 AS reseller_account_count_bucket,
+    COUNT(DISTINCT page) OVER () AS total_pages
+  FROM publishers
+), app_ads_accounts AS (
+  SELECT
+    page,
+    CEIL(CAST(JSON_VALUE(app_ads_account_types, '$.direct.account_count') AS INT64) / 100) * 100 AS direct_account_count_bucket,
+    CEIL(CAST(JSON_VALUE(app_ads_account_types, '$.reseller.account_count') AS INT64) / 100) * 100 AS reseller_account_count_bucket,
+    COUNT(DISTINCT page) OVER () AS total_pages
+  FROM publishers
+)
+
+SELECT
+  'ads' AS source,
+  'direct' AS account_type,
+  direct_account_count_bucket AS account_count_bucket,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(DISTINCT page) AS number_of_pages
+FROM ads_accounts
+GROUP BY source, direct_account_count_bucket
+UNION ALL
+SELECT
+  'ads' AS source,
+  'reseller' AS account_type,
+  reseller_account_count_bucket AS account_count_bucket,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(DISTINCT page) AS number_of_pages
+FROM ads_accounts
+GROUP BY source, reseller_account_count_bucket
+UNION ALL
+SELECT
+  'app_ads' AS source,
+  'direct' AS account_type,
+  direct_account_count_bucket AS account_count_bucket,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(DISTINCT page) AS number_of_pages
+FROM app_ads_accounts
+GROUP BY source, direct_account_count_bucket
+UNION ALL
+SELECT
+  'app_ads' AS source,
+  'reseller' AS account_type,
+  reseller_account_count_bucket AS account_count_bucket,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(DISTINCT page) AS number_of_pages
+FROM app_ads_accounts
+GROUP BY source, reseller_account_count_bucket
+
+ORDER BY account_count_bucket ASC
+LIMIT 1000

sql/2024/privacy/ads_and_sellers_graph.sql

@@ -0,0 +1,114 @@
+WITH RECURSIVE pages AS (
+  SELECT
+    CASE page -- publisher websites may redirect to an SSP domain, and need to use redirected domain instead of page domain
+      WHEN 'https://www.chunkbase.com/' THEN 'cafemedia.com'
+      ELSE NET.REG_DOMAIN(page)
+    END AS page,
+    custom_metrics
+  FROM `httparchive.all.pages`
+  WHERE date = '2024-06-01' AND
+    client = 'mobile' AND
+    is_root_page = TRUE
+), ads AS (
+  SELECT
+    page,
+    JSON_QUERY(custom_metrics, '$.ads.ads.account_types') AS ad_accounts
+  FROM pages
+  WHERE
+    CAST(JSON_VALUE(custom_metrics, '$.ads.ads.account_count') AS INT64) > 0
+), sellers AS (
+  SELECT
+    page,
+    JSON_QUERY(custom_metrics, '$.ads.sellers.seller_types') AS ad_sellers
+  FROM pages
+  WHERE
+    CAST(JSON_VALUE(custom_metrics, '$.ads.sellers.seller_count') AS INT64) > 0
+), relationships_web AS (
+  SELECT
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS demand,
+    'Web' AS supply,
+    'direct' AS relationship,
+    page AS publisher
+  FROM ads, UNNEST(JSON_VALUE_ARRAY(ad_accounts, '$.direct.domains')) AS domain
+  UNION ALL
+  SELECT
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS demand,
+    'Web' AS supply,
+    'indirect' AS relationship,
+    page AS publisher
+  FROM ads, UNNEST(JSON_VALUE_ARRAY(ad_accounts, '$.reseller.domains')) AS domain
+  UNION ALL
+  SELECT
+    page AS demand,
+    'Web' AS supply,
+    'direct' AS relationship,
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS publisher
+  FROM sellers, UNNEST(JSON_VALUE_ARRAY(ad_sellers, '$.publisher.domains')) AS domain
+  UNION ALL
+  SELECT
+    page AS demand,
+    'Web' AS supply,
+    'direct' AS relationship,
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS publisher
+  FROM sellers, UNNEST(JSON_VALUE_ARRAY(ad_sellers, '$.both.domains')) AS domain
+), relationships_adtech AS (
+  SELECT
+    page AS demand,
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS supply,
+    'indirect' AS relationship
+  FROM sellers, UNNEST(JSON_VALUE_ARRAY(ad_sellers, '$.intermediary.domains')) AS domain
+  UNION ALL
+  SELECT
+    page AS demand,
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS supply,
+    'indirect' AS relationship
+  FROM sellers, UNNEST(JSON_VALUE_ARRAY(ad_sellers, '$.both.domains')) AS domain
+), nodes AS (
+  (
+    SELECT
+      demand,
+      supply,
+      CONCAT(demand, '-', supply) AS path,
+      relationship,
+      HLL_COUNT.INIT(publisher) AS supply_sketch
+    FROM relationships_web
+    GROUP BY demand, supply, relationship
+  )
+  UNION ALL
+  (
+    SELECT
+      relationships_grouped.demand AS demand,
+      relationships_grouped.supply AS supply,
+      CONCAT(relationships_grouped.demand, '-', nodes.path) AS path,
+      relationships_grouped.relationship AS relationship,
+      nodes.supply_sketch AS supply_sketch
+    FROM (
+      SELECT
+        demand,
+        supply,
+        relationship
+      FROM relationships_adtech
+      GROUP BY
+        demand,
+        supply,
+        relationship
+    ) AS relationships_grouped
+    INNER JOIN nodes
+    ON relationships_grouped.supply = nodes.demand AND
+      nodes.supply_sketch IS NOT NULL AND
+      nodes.relationship = 'indirect' AND
+      relationships_grouped.demand IS NOT NULL AND
+      STRPOS(nodes.path, relationships_grouped.demand) = 0
+  )
+)
+
+SELECT
+  supply,
+  demand,
+  HLL_COUNT.MERGE(supply_sketch) AS publishers_count,
+  relationship,
+  path
+FROM nodes
+GROUP BY demand, supply, relationship, path
+ORDER BY publishers_count DESC
+LIMIT 5000

sql/2024/privacy/ads_lines_distribution.sql

@@ -0,0 +1,45 @@
+WITH RECURSIVE pages AS (
+  SELECT
+    CASE page -- publisher websites may redirect to an SSP domain, and need to use redirected domain instead of page domain
+      WHEN 'https://www.chunkbase.com/' THEN 'cafemedia.com'
+      ELSE NET.REG_DOMAIN(page)
+    END AS page,
+    CAST(JSON_VALUE(custom_metrics, '$.ads.ads.line_count') AS INT64) AS ads_line_count,
+    CAST(JSON_VALUE(custom_metrics, '$.ads.app_ads.line_count') AS INT64) AS app_ads_line_count
+  FROM `httparchive.all.pages`
+  WHERE date = '2024-06-01' AND
+    is_root_page = TRUE
+), ads AS (
+  SELECT
+    page,
+    CEIL(ads_line_count / 100) * 100 AS line_count_bucket,
+    COUNT(DISTINCT page) OVER () AS total_pages
+  FROM pages
+  WHERE ads_line_count > 0
+), app_ads AS (
+  SELECT
+    page,
+    CEIL(app_ads_line_count / 100) * 100 AS line_count_bucket,
+    COUNT(DISTINCT page) OVER () AS total_pages
+  FROM pages
+  WHERE app_ads_line_count > 0
+)
+
+SELECT
+  'ads.txt' AS type,
+  line_count_bucket,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(DISTINCT page) AS number_of_pages
+FROM ads
+GROUP BY line_count_bucket
+HAVING line_count_bucket <= 10000
+UNION ALL
+SELECT
+  'app-ads.txt' AS type,
+  line_count_bucket,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(DISTINCT page) AS number_of_pages
+FROM app_ads
+GROUP BY line_count_bucket
+HAVING line_count_bucket <= 10000
+ORDER BY type, line_count_bucket ASC

sql/2024/privacy/ccpa_most_common_phrases.sql

@@ -0,0 +1,5 @@
+WITH pages_with_phrase AS (
+  SELECT client, rank_grouping, page, count(DISTINCT page) OVER (PARTITION BY client, rank_grouping) AS total_pages_with_phrase_in_rank_group, JSON_QUERY_ARRAY(custom_metrics, '$.privacy.ccpa_link.CCPALinkPhrases') AS ccpa_link_phrases FROM `httparchive.all.pages`, --TABLESAMPLE SYSTEM (0.01 PERCENT)
+    UNNEST([1000, 10000, 100000, 1000000, 10000000, 100000000]) AS rank_grouping WHERE date = '2024-06-01' AND is_root_page = true AND rank <= rank_grouping AND array_length(JSON_QUERY_ARRAY(custom_metrics, '$.privacy.ccpa_link.CCPALinkPhrases')) > 0
+)
+SELECT client, rank_grouping, link_phrase, count(DISTINCT page) AS num_pages, count(DISTINCT page) / any_value(total_pages_with_phrase_in_rank_group) AS pct_pages FROM pages_with_phrase, unnest(ccpa_link_phrases) link_phrase GROUP BY link_phrase, rank_grouping, client ORDER BY rank_grouping, client, num_pages DESC

sql/2024/privacy/ccpa_prevalence.sql

@@ -0,0 +1,6 @@
+WITH pages AS (
+  SELECT client, rank_grouping, page, JSON_VALUE(custom_metrics, '$.privacy.ccpa_link.hasCCPALink') AS has_ccpa_link FROM `httparchive.all.pages`,
+    --    TABLESAMPLE SYSTEM (0.0025 PERCENT)
+    UNNEST([1000, 10000, 100000, 1000000, 10000000, 100000000]) AS rank_grouping WHERE date = '2024-06-01' AND is_root_page = true AND rank <= rank_grouping
+)
+SELECT client, rank_grouping, has_ccpa_link, count(DISTINCT page) AS num_pages FROM pages GROUP BY has_ccpa_link, rank_grouping, client ORDER BY rank_grouping, client, has_ccpa_link

sql/2024/privacy/common_ads_variables.sql

@@ -0,0 +1,29 @@
+WITH RECURSIVE pages AS (
+  SELECT
+    page,
+    JSON_QUERY(custom_metrics, '$.ads.ads') AS ads_metrics
+  FROM `httparchive.all.pages`
+  WHERE
+    date = '2024-06-01' AND
+    is_root_page = TRUE AND
+    CAST(JSON_VALUE(custom_metrics, '$.ads.ads.account_count') AS INT64) > 0
+), ads AS (
+  SELECT
+    page,
+    variable,
+    COUNT(DISTINCT page) OVER() AS total_publishers
+  FROM pages,
+    UNNEST(JSON_VALUE_ARRAY(ads_metrics, '$.variables')) AS variable
+  WHERE
+    CAST(JSON_VALUE(ads_metrics, '$.account_types.reseller.account_count') AS INT64) > 0 OR
+    CAST(JSON_VALUE(ads_metrics, '$.account_types.direct.account_count') AS INT64) > 0
+)
+
+SELECT
+  variable,
+  COUNT(DISTINCT page) / ANY_VALUE(total_publishers) AS pct_publishers,
+  COUNT(DISTINCT page) AS number_of_publishers
+FROM ads
+GROUP BY variable
+ORDER BY pct_publishers DESC
+LIMIT 100

sql/2024/privacy/cookies_top_first_party_names.sql

@@ -0,0 +1,10 @@
+-- Most common cookie names, by number of domains on which they appear. Goal is to identify common trackers that use first-party cookies across sites.
+
+WITH pages AS (
+  SELECT client, root_page, custom_metrics, count(DISTINCT net.host(root_page)) OVER(PARTITION BY client) AS total_domains FROM `httparchive.all.pages` -- TABLESAMPLE SYSTEM (0.00001 PERCENT)
+  WHERE date = '2024-06-01'
+),
+cookies AS (
+  SELECT client, cookie, net.host(JSON_VALUE(cookie, '$.domain')) AS cookie_host, net.host(root_page) AS firstparty_host, total_domains FROM pages, UNNEST(JSON_QUERY_ARRAY(custom_metrics, '$.cookies')) cookie
+)
+SELECT client, count(DISTINCT firstparty_host) AS domain_count, count(DISTINCT firstparty_host) / any_value(total_domains) AS pct_domains, JSON_VALUE(cookie, '$.name') AS cookie_name FROM cookies WHERE firstparty_host LIKE '%' || cookie_host GROUP BY client, cookie_name ORDER BY domain_count DESC, client DESC LIMIT 500

sql/2024/privacy/cookies_top_third_party_domains.sql

@@ -0,0 +1,8 @@
+WITH pages AS (
+  SELECT page, client, root_page, custom_metrics, count(DISTINCT page) OVER (PARTITION BY client) AS total_pages FROM `httparchive.all.pages`  -- TABLESAMPLE SYSTEM (0.1 PERCENT)
+  WHERE date = '2024-06-01'
+),
+cookies AS (
+  SELECT client, page, cookie, net.host(JSON_VALUE(cookie, '$.domain')) AS cookie_host, net.host(root_page) AS firstparty_host, total_pages FROM pages, UNNEST(JSON_QUERY_ARRAY(custom_metrics, '$.cookies')) cookie
+)
+SELECT client, cookie_host, count(DISTINCT page) AS page_count, count(DISTINCT page) / any_value(total_pages) AS pct_pages FROM cookies WHERE firstparty_host NOT LIKE '%' || cookie_host GROUP BY client, cookie_host ORDER BY page_count DESC, client LIMIT 500

sql/2024/privacy/cookies_top_third_party_names.sql

@@ -0,0 +1,10 @@
+-- Most common cookie names, by number of domains on which they appear. Goal is to identify common trackers that set cookies using many domains.
+
+WITH pages AS (
+  SELECT client, root_page, custom_metrics, count(DISTINCT net.host(root_page)) OVER(PARTITION BY client) AS total_domains FROM `httparchive.all.pages` -- TABLESAMPLE SYSTEM (0.00001 PERCENT)
+  WHERE date = '2024-06-01'
+),
+cookies AS (
+  SELECT client, cookie, net.host(JSON_VALUE(cookie, '$.domain')) AS cookie_host, net.host(root_page) AS firstparty_host, total_domains FROM pages, UNNEST(JSON_QUERY_ARRAY(custom_metrics, '$.cookies')) cookie
+)
+SELECT client, count(DISTINCT firstparty_host) AS domain_count, count(DISTINCT firstparty_host) / any_value(total_domains) AS pct_domains, JSON_VALUE(cookie, '$.name') AS cookie_name FROM cookies WHERE firstparty_host NOT LIKE '%' || cookie_host GROUP BY client, cookie_name ORDER BY domain_count DESC, client DESC LIMIT 500

sql/2024/privacy/easylist-tracker-detection.sql

@@ -0,0 +1,38 @@
+CREATE TEMP FUNCTION
+CheckDomainInURL(url STRING, domain STRING)
+RETURNS INT64
+LANGUAGE js AS """
+  return url.includes(domain) ? 1 : 0;
+""";
+
+-- We need to use the `easylist_adservers.csv` to populate the table to get the list of domains to block
+-- https://github.com/easylist/easylist/blob/master/easylist/easylist_adservers.txt
+WITH easylist_data AS (
+  SELECT string_field_0
+  FROM `httparchive.almanac.easylist_adservers`
+),
+requests_data AS (
+  SELECT url
+  FROM `httparchive.all.requests`
+  WHERE
+    date = '2024-06-01' AND
+    is_root_page = TRUE
+),
+block_status AS (
+  SELECT
+    r.url,
+    MAX(
+      CASE
+        WHEN CheckDomainInURL(r.url, e.string_field_0) = 1 THEN 1
+        ELSE 0
+      END
+    ) AS should_block
+  FROM requests_data r
+  LEFT JOIN easylist_data e
+  ON CheckDomainInURL(r.url, e.string_field_0) = 1
+  GROUP BY r.url
+)
+SELECT
+  COUNT(0) AS blocked_url_count
+FROM block_status
+WHERE should_block = 1;

sql/2024/privacy/fingerprinting_most_common_apis.sql

@@ -0,0 +1,19 @@
+CREATE TEMP FUNCTION getFingerprintingTypes(input STRING)
+RETURNS ARRAY<STRING>
+LANGUAGE js AS """
+if (input) {
+  try {
+    return Object.keys(JSON.parse(input))
+  } catch (e) {
+    return []
+  }
+} else {
+  return []
+}
+""";
+
+WITH pages AS (
+  SELECT client, page, fingerprinting_type, count(DISTINCT page) OVER (PARTITION BY client) AS total_pages FROM `httparchive.all.pages`, --TABLESAMPLE SYSTEM (0.01 PERCENT)
+    unnest(getFingerprintingTypes(JSON_EXTRACT(custom_metrics, '$.privacy.fingerprinting.counts'))) AS fingerprinting_type WHERE date = '2024-06-01'
+)
+SELECT client, fingerprinting_type, count(DISTINCT page) AS page_count, count(DISTINCT page) / any_value(total_pages) AS pct_pages FROM pages GROUP BY client, fingerprinting_type ORDER BY page_count DESC

sql/2024/privacy/fingerprinting_most_common_scripts.sql

@@ -0,0 +1,6 @@
+WITH pages AS (
+  SELECT page, client, custom_metrics, count(DISTINCT page) OVER (PARTITION BY client) AS total_pages FROM `httparchive.all.pages` --TABLESAMPLE SYSTEM (0.001 PERCENT)
+  WHERE date = '2024-06-01'
+)
+SELECT client, script, count(DISTINCT page) AS page_count, count(DISTINCT page) / any_value(total_pages) AS pct_pages FROM pages,
+  unnest(JSON_QUERY_ARRAY(custom_metrics, '$.privacy.fingerprinting.likelyFingerprintingScripts')) AS script GROUP BY client, script ORDER BY page_count DESC LIMIT 100;

sql/2024/privacy/fingerprinting_script_count.sql

@@ -0,0 +1,5 @@
+WITH pages AS (
+  SELECT page, client, array_length(JSON_QUERY_ARRAY(custom_metrics, '$.privacy.fingerprinting.likelyFingerprintingScripts')) AS script_count, count(DISTINCT page) OVER (PARTITION BY client) AS total_pages FROM `httparchive.all.pages` --TABLESAMPLE SYSTEM (0.01 PERCENT)
+  WHERE date = '2024-06-01'
+)
+SELECT script_count, client, count(DISTINCT page) AS page_count, count(DISTINCT page) / any_value(total_pages) AS pct_pages FROM pages GROUP BY script_count, client ORDER BY script_count ASC;