From b231da894f958bb3ddaac80bc49688257a844f07 Mon Sep 17 00:00:00 2001 From: Michael Solati Date: Wed, 21 Aug 2024 14:48:03 -0700 Subject: [PATCH] PWA Queries for 2024 --- sql/2024/pwa/assetlink_usage.sql | 61 +++++++++++ sql/2024/pwa/fugu.sql | 39 +++++++ sql/2024/pwa/install_events.sql | 69 ++++++++++++ sql/2024/pwa/lighthouse_pwa_audits.sql | 73 +++++++++++++ sql/2024/pwa/lighthouse_pwa_score.sql | 55 ++++++++++ .../pwa/manifests_and_service_workers.sql | 21 ++++ sql/2024/pwa/manifests_description.sql | 28 +++++ sql/2024/pwa/manifests_file_handlers.sql | 28 +++++ sql/2024/pwa/manifests_iarc.sql | 28 +++++ sql/2024/pwa/manifests_lang.sql | 28 +++++ sql/2024/pwa/manifests_not_json_parsable.sql | 52 +++++++++ sql/2024/pwa/manifests_note_taking.sql | 28 +++++ .../pwa/manifests_preferring_native_apps.sql | 46 ++++++++ sql/2024/pwa/manifests_protocol_handlers.sql | 28 +++++ sql/2024/pwa/manifests_screenshots.sql | 28 +++++ .../manifests_screenshots_platform_narrow.sql | 30 ++++++ .../manifests_screenshots_platform_wide.sql | 30 ++++++ sql/2024/pwa/manifests_share_target.sql | 28 +++++ sql/2024/pwa/manifests_shortcuts.sql | 28 +++++ .../pwa/manifests_window_controls_overlay.sql | 30 ++++++ sql/2024/pwa/popular_pwa_libraries.sql | 100 ++++++++++++++++++ sql/2024/pwa/popular_pwa_libraries_helper.sql | 48 +++++++++ sql/2024/pwa/pwa_blink_usage.sql | 21 ++++ .../pwa/pwa_notification_acceptance_rates.sql | 47 ++++++++ ...otification_acceptance_rates_over_time.sql | 27 +++++ sql/2024/pwa/sw_adoption_over_time.sql | 15 +++ .../pwa/sw_adoption_over_time_ranking.sql | 57 ++++++++++ sql/2024/pwa/sw_events.sql | 46 ++++++++ sql/2024/pwa/sw_methods.sql | 44 ++++++++ sql/2024/pwa/sw_objects.sql | 47 ++++++++ sql/2024/pwa/sw_objects_name_only.sql | 47 ++++++++ sql/2024/pwa/sw_registration_properties.sql | 48 +++++++++ .../sw_registration_properties_name_only.sql | 48 +++++++++ sql/2024/pwa/top_manifest_categories.sql | 86 +++++++++++++++ sql/2024/pwa/top_manifest_display_values.sql | 56 ++++++++++ sql/2024/pwa/top_manifest_icon_sizes.sql | 88 +++++++++++++++ sql/2024/pwa/top_manifest_orientations.sql | 52 +++++++++ sql/2024/pwa/top_manifest_properties.sql | 92 ++++++++++++++++ sql/2024/pwa/workbox_methods.sql | 61 +++++++++++ sql/2024/pwa/workbox_packages.sql | 60 +++++++++++ sql/2024/pwa/workbox_usage.sql | 18 ++++ sql/2024/pwa/workbox_versions.sql | 60 +++++++++++ 42 files changed, 1926 insertions(+) create mode 100644 sql/2024/pwa/assetlink_usage.sql create mode 100644 sql/2024/pwa/fugu.sql create mode 100644 sql/2024/pwa/install_events.sql create mode 100644 sql/2024/pwa/lighthouse_pwa_audits.sql create mode 100644 sql/2024/pwa/lighthouse_pwa_score.sql create mode 100644 sql/2024/pwa/manifests_and_service_workers.sql create mode 100644 sql/2024/pwa/manifests_description.sql create mode 100644 sql/2024/pwa/manifests_file_handlers.sql create mode 100644 sql/2024/pwa/manifests_iarc.sql create mode 100644 sql/2024/pwa/manifests_lang.sql create mode 100644 sql/2024/pwa/manifests_not_json_parsable.sql create mode 100644 sql/2024/pwa/manifests_note_taking.sql create mode 100644 sql/2024/pwa/manifests_preferring_native_apps.sql create mode 100644 sql/2024/pwa/manifests_protocol_handlers.sql create mode 100644 sql/2024/pwa/manifests_screenshots.sql create mode 100644 sql/2024/pwa/manifests_screenshots_platform_narrow.sql create mode 100644 sql/2024/pwa/manifests_screenshots_platform_wide.sql create mode 100644 sql/2024/pwa/manifests_share_target.sql create mode 100644 sql/2024/pwa/manifests_shortcuts.sql create mode 100644 sql/2024/pwa/manifests_window_controls_overlay.sql create mode 100644 sql/2024/pwa/popular_pwa_libraries.sql create mode 100644 sql/2024/pwa/popular_pwa_libraries_helper.sql create mode 100644 sql/2024/pwa/pwa_blink_usage.sql create mode 100644 sql/2024/pwa/pwa_notification_acceptance_rates.sql create mode 100644 sql/2024/pwa/pwa_notification_acceptance_rates_over_time.sql create mode 100644 sql/2024/pwa/sw_adoption_over_time.sql create mode 100644 sql/2024/pwa/sw_adoption_over_time_ranking.sql create mode 100644 sql/2024/pwa/sw_events.sql create mode 100644 sql/2024/pwa/sw_methods.sql create mode 100644 sql/2024/pwa/sw_objects.sql create mode 100644 sql/2024/pwa/sw_objects_name_only.sql create mode 100644 sql/2024/pwa/sw_registration_properties.sql create mode 100644 sql/2024/pwa/sw_registration_properties_name_only.sql create mode 100644 sql/2024/pwa/top_manifest_categories.sql create mode 100644 sql/2024/pwa/top_manifest_display_values.sql create mode 100644 sql/2024/pwa/top_manifest_icon_sizes.sql create mode 100644 sql/2024/pwa/top_manifest_orientations.sql create mode 100644 sql/2024/pwa/top_manifest_properties.sql create mode 100644 sql/2024/pwa/workbox_methods.sql create mode 100644 sql/2024/pwa/workbox_packages.sql create mode 100644 sql/2024/pwa/workbox_usage.sql create mode 100644 sql/2024/pwa/workbox_versions.sql diff --git a/sql/2024/pwa/assetlink_usage.sql b/sql/2024/pwa/assetlink_usage.sql new file mode 100644 index 00000000000..109bdcaab19 --- /dev/null +++ b/sql/2024/pwa/assetlink_usage.sql @@ -0,0 +1,61 @@ +#standardSQL +# assetlink usage + +SELECT + 'PWA sites' AS type, + _TABLE_SUFFIX AS client, + COUNT(0) AS freq, + total, + COUNT(0) / total AS pct +FROM + `httparchive.pages.2024_06_01_*` +JOIN + ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' + GROUP BY + _TABLE_SUFFIX + ) +USING (_TABLE_SUFFIX) +WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' AND + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT_SCALAR(JSON_VALUE(payload, '$._well-known'), "$['/.well-known/assetlinks.json'].found") = 'true' +GROUP BY + client, + total +UNION ALL +SELECT + 'All sites' AS type, + _TABLE_SUFFIX AS client, + COUNT(0) AS freq, + total, + COUNT(0) / total AS pct +FROM + `httparchive.pages.2024_06_01_*` +JOIN + ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.pages.2024_06_01_*` + GROUP BY + _TABLE_SUFFIX + ) +USING (_TABLE_SUFFIX) +WHERE + JSON_EXTRACT_SCALAR(JSON_VALUE(payload, '$._well-known'), "$['/.well-known/assetlinks.json'].found") = 'true' +GROUP BY + client, + total +ORDER BY + type DESC, + freq / total DESC, + client diff --git a/sql/2024/pwa/fugu.sql b/sql/2024/pwa/fugu.sql new file mode 100644 index 00000000000..21d0ec93396 --- /dev/null +++ b/sql/2024/pwa/fugu.sql @@ -0,0 +1,39 @@ +#standardSQL +CREATE TEMP FUNCTION getFuguAPIs(data STRING) +RETURNS ARRAY +LANGUAGE js AS ''' +const $ = JSON.parse(data); +return Object.keys($); +'''; + +SELECT + _TABLE_SUFFIX AS client, + fuguAPI, + COUNT(DISTINCT url) AS pages, + total, + COUNT(DISTINCT url) / total AS pct, + ARRAY_TO_STRING(ARRAY_AGG(DISTINCT url LIMIT 50), ' ') AS sample_urls +FROM + `httparchive.pages.2024_06_01_*` +JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.pages.2024_06_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (_TABLE_SUFFIX), + UNNEST(getFuguAPIs(JSON_QUERY(payload, '$."_fugu-apis"'))) AS fuguAPI +WHERE + JSON_QUERY(payload, '$."_fugu-apis"') != '[]' +GROUP BY + fuguAPI, + client, + total +HAVING + COUNT(DISTINCT url) >= 10 +ORDER BY + pct DESC, + client; diff --git a/sql/2024/pwa/install_events.sql b/sql/2024/pwa/install_events.sql new file mode 100644 index 00000000000..e3369343ded --- /dev/null +++ b/sql/2024/pwa/install_events.sql @@ -0,0 +1,69 @@ +#standardSQL +# SW install events + +CREATE TEMPORARY FUNCTION getInstallEvents(payload STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + var payloadJSON = JSON.parse(payload); + + /* YouTube iFrames account for a lot of these, so we exclude them */ + /* Cannot use filter as it is a complex object and not a straight array */ + function filterYouTube(info) { + + var objectKeys = Object.keys(info); + objectKeys = objectKeys.trim().split(','); + for(var i = 0; i < objectKeys.length; i++) { + if(objectKeys[i].toLowerCase().includes('youtube')) { + delete info[objectKeys[i]]; + } + } + return info; + } + + var windowEventListenersInfo = Object.values(filterYouTube(payloadJSON.windowEventListenersInfo)).flat(); + var windowPropertiesInfo = Object.values(filterYouTube(payloadJSON.windowPropertiesInfo)).flat() + + return [...new Set([...windowEventListenersInfo ,...windowPropertiesInfo])]; +} catch (e) { + return []; +} +'''; + +SELECT + _TABLE_SUFFIX AS client, + install_event, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.pages.2024_06_01_*`, + UNNEST(getInstallEvents(JSON_EXTRACT(payload, '$._pwa'))) AS install_event +JOIN + ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.pages.2024_06_01_*` + WHERE + -- This condition filters out tests that might have broken when running the 'pwa' metric + -- as even pages without any pwa capabilities will have a _pwa object with empty fields + JSON_EXTRACT(payload, '$._pwa') != '[]' + GROUP BY + _TABLE_SUFFIX + ) +USING (_TABLE_SUFFIX) +WHERE + ( + JSON_EXTRACT(payload, '$._pwa.windowEventListenersInfo') != '[]' OR + JSON_EXTRACT(payload, '$._pwa.windowPropertiesInfo') != '[]' + ) AND + install_event != '' AND + install_event != '[]' +GROUP BY + client, + total, + install_event +ORDER BY + freq / total DESC, + client diff --git a/sql/2024/pwa/lighthouse_pwa_audits.sql b/sql/2024/pwa/lighthouse_pwa_audits.sql new file mode 100644 index 00000000000..ecf1a3f9bd7 --- /dev/null +++ b/sql/2024/pwa/lighthouse_pwa_audits.sql @@ -0,0 +1,73 @@ +#standardSQL +# Get summary of all lighthouse PWA audits, for both PWA pages and all pages +# Note scores, weightings, groups and descriptions may be off in mixed months when new versions of Lighthouse roles out + +CREATE TEMPORARY FUNCTION getAudits(auditRefs STRING, audits STRING) +RETURNS ARRAY> LANGUAGE js AS ''' +var auditrefs = JSON.parse(auditRefs); +var audits = JSON.parse(audits); +var results = []; +for (auditref of auditrefs) { + results.push({ + id: auditref.id, + weight: auditref.weight, + audit_group: auditref.group, + description: audits[auditref.id].description, + score: audits[auditref.id].score + }); +} +return results; +'''; + +SELECT + _TABLE_SUFFIX AS client, + 'PWA Sites' AS type, + audits.id AS id, + COUNTIF(audits.score > 0) AS num_pages, + COUNT(0) AS total, + COUNTIF(audits.score IS NOT NULL) AS total_applicable, + SAFE_DIVIDE(COUNTIF(audits.score > 0), COUNTIF(audits.score IS NOT NULL)) AS pct, + APPROX_QUANTILES(audits.weight, 100)[OFFSET(50)] AS median_weight, + MAX(audits.audit_group) AS audit_group, + MAX(audits.description) AS description +FROM + `httparchive.lighthouse.2024_06_01_*`, + UNNEST(getAudits(JSON_EXTRACT(report, '$.categories.pwa.auditRefs'), JSON_EXTRACT(report, '$.audits'))) AS audits +JOIN + ( + SELECT + _TABLE_SUFFIX, + url + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' AND + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' + ) +USING (_TABLE_SUFFIX, url) +GROUP BY + client, + audits.id +UNION ALL +SELECT + _TABLE_SUFFIX AS client, + 'ALL Sites' AS type, + audits.id AS id, + COUNTIF(audits.score > 0) AS num_pages, + COUNT(0) AS total, + COUNTIF(audits.score IS NOT NULL) AS total_applicable, + SAFE_DIVIDE(COUNTIF(audits.score > 0), COUNTIF(audits.score IS NOT NULL)) AS pct, + APPROX_QUANTILES(audits.weight, 100)[OFFSET(50)] AS median_weight, + MAX(audits.audit_group) AS audit_group, + MAX(audits.description) AS description +FROM + `httparchive.lighthouse.2024_06_01_*`, + UNNEST(getAudits(JSON_EXTRACT(report, '$.categories.pwa.auditRefs'), JSON_EXTRACT(report, '$.audits'))) AS audits +GROUP BY + client, + audits.id +ORDER BY + client, + type DESC, + median_weight DESC, + id diff --git a/sql/2024/pwa/lighthouse_pwa_score.sql b/sql/2024/pwa/lighthouse_pwa_score.sql new file mode 100644 index 00000000000..6a64fc3b21d --- /dev/null +++ b/sql/2024/pwa/lighthouse_pwa_score.sql @@ -0,0 +1,55 @@ +#standardSQL +# Percentiles of lighthouse pwa score +SELECT + client, + '2024_06_01' AS date, + 'PWA Sites' AS type, + percentile, + APPROX_QUANTILES(score, 1000)[OFFSET(percentile * 10)] * 100 AS score +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url, + CAST(JSON_EXTRACT(report, '$.categories.pwa.score') AS NUMERIC) AS score + FROM + `httparchive.lighthouse.2024_06_01_*`) +JOIN + ( + SELECT + _TABLE_SUFFIX AS client, + url + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' AND + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' + ) +USING (client, url), + UNNEST([10, 25, 50, 75, 90]) AS percentile +GROUP BY + client, + date, + percentile +UNION ALL +SELECT + client, + '2024_06_01' AS date, + 'All Sites' AS type, + percentile, + APPROX_QUANTILES(score, 1000)[OFFSET(percentile * 10)] * 100 AS score +FROM ( + SELECT + _TABLE_SUFFIX AS client, + CAST(JSON_EXTRACT(report, '$.categories.pwa.score') AS NUMERIC) AS score + FROM + `httparchive.lighthouse.2024_06_01_*`), + UNNEST([10, 25, 50, 75, 90]) AS percentile +GROUP BY + client, + date, + percentile +ORDER BY + client, + date, + type, + percentile diff --git a/sql/2024/pwa/manifests_and_service_workers.sql b/sql/2024/pwa/manifests_and_service_workers.sql new file mode 100644 index 00000000000..c67c2274e73 --- /dev/null +++ b/sql/2024/pwa/manifests_and_service_workers.sql @@ -0,0 +1,21 @@ +#standardSQL +# Counting Manifests and Service Workers +SELECT + client, + SAFE_DIVIDE(SUM(ServiceWorker), SUM(COUNT(0)) OVER (PARTITION BY client)) AS ServiceWorkers, + SAFE_DIVIDE(SUM(manifests), SUM(COUNT(0)) OVER (PARTITION BY client)) AS Manifests, + SAFE_DIVIDE(COUNTIF(ServiceWorker > 0 OR manifests > 0), SUM(COUNT(0)) OVER (PARTITION BY client)) AS Either, + SAFE_DIVIDE(COUNTIF(ServiceWorker > 0 AND manifests > 0), SUM(COUNT(0)) OVER (PARTITION BY client)) AS Both, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total +FROM ( + SELECT + _TABLE_SUFFIX AS client, + IF(JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true', 1, 0) AS ServiceWorker, + IF(JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}', 1, 0) AS manifests + FROM + `httparchive.pages.2024_06_01_*` + ) +GROUP BY + client +ORDER BY + client diff --git a/sql/2024/pwa/manifests_description.sql b/sql/2024/pwa/manifests_description.sql new file mode 100644 index 00000000000..548d2dd5109 --- /dev/null +++ b/sql/2024/pwa/manifests_description.sql @@ -0,0 +1,28 @@ +#standardSQL +# % manifests with description for service worker pages and all pages + +CREATE TEMP FUNCTION hasDescription(manifest STRING) +RETURNS BOOLEAN LANGUAGE js AS ''' +try { + var $ = Object.values(JSON.parse(manifest))[0]; + return $.hasOwnProperty('description') && $.description != ''; +} catch (e) { + return null; +} +'''; + +SELECT + 'PWA Pages' AS type, + _TABLE_SUFFIX AS client, + hasDescription(JSON_EXTRACT(payload, '$._pwa.manifests')) AS hasDescription, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + hasDescription diff --git a/sql/2024/pwa/manifests_file_handlers.sql b/sql/2024/pwa/manifests_file_handlers.sql new file mode 100644 index 00000000000..059e25f84b6 --- /dev/null +++ b/sql/2024/pwa/manifests_file_handlers.sql @@ -0,0 +1,28 @@ +#standardSQL +# % manifests with file handlers for service worker pages and all pages + +CREATE TEMP FUNCTION hasFileHandlers(manifest STRING) +RETURNS BOOLEAN LANGUAGE js AS ''' +try { + var $ = Object.values(JSON.parse(manifest))[0]; + return $.hasOwnProperty('file_handlers') && $.file_handlers.length > 0; +} catch (e) { + return null; +} +'''; + +SELECT + 'PWA Pages' AS type, + _TABLE_SUFFIX AS client, + hasFileHandlers(JSON_EXTRACT(payload, '$._pwa.manifests')) AS hasFileHandlers, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + hasFileHandlers diff --git a/sql/2024/pwa/manifests_iarc.sql b/sql/2024/pwa/manifests_iarc.sql new file mode 100644 index 00000000000..8738089ea95 --- /dev/null +++ b/sql/2024/pwa/manifests_iarc.sql @@ -0,0 +1,28 @@ +#standardSQL +# % manifests with iarc rating for service worker pages and all pages + +CREATE TEMP FUNCTION hasIARC(manifest STRING) +RETURNS BOOLEAN LANGUAGE js AS ''' +try { + var $ = Object.values(JSON.parse(manifest))[0]; + return $.hasOwnProperty('iarc_rating_id') && $.iarc_rating_id != ''; +} catch (e) { + return null; +} +'''; + +SELECT + 'PWA Pages' AS type, + _TABLE_SUFFIX AS client, + hasIARC(JSON_EXTRACT(payload, '$._pwa.manifests')) AS hasIARC, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + hasIARC diff --git a/sql/2024/pwa/manifests_lang.sql b/sql/2024/pwa/manifests_lang.sql new file mode 100644 index 00000000000..9a8298700b4 --- /dev/null +++ b/sql/2024/pwa/manifests_lang.sql @@ -0,0 +1,28 @@ +#standardSQL +# % manifests with language for service worker pages and all pages + +CREATE TEMP FUNCTION hasLang(manifest STRING) +RETURNS BOOLEAN LANGUAGE js AS ''' +try { + var $ = Object.values(JSON.parse(manifest))[0]; + return $.hasOwnProperty('lang') && $.lang != ''; +} catch (e) { + return null; +} +'''; + +SELECT + 'PWA Pages' AS type, + _TABLE_SUFFIX AS client, + hasLang(JSON_EXTRACT(payload, '$._pwa.manifests')) AS hasLang, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + hasLang diff --git a/sql/2024/pwa/manifests_not_json_parsable.sql b/sql/2024/pwa/manifests_not_json_parsable.sql new file mode 100644 index 00000000000..61b97d6a102 --- /dev/null +++ b/sql/2024/pwa/manifests_not_json_parsable.sql @@ -0,0 +1,52 @@ +#standardSQL +# Manifests that are not JSON parsable for service worker pages and all pages + +CREATE TEMP FUNCTION canParseManifest(manifest STRING) +RETURNS BOOLEAN LANGUAGE js AS ''' +try { + var manifestJSON = Object.values(JSON.parse(manifest))[0]; + if (typeof manifestJSON === 'string' && manifestJSON.trim() != '') { + return false; + } + if (typeof manifestJSON === 'string' && manifestJSON.trim() === '') { + return null; + } + return true; +} catch { + return false; +} +'''; + +SELECT + 'PWA Pages' AS type, + _TABLE_SUFFIX AS client, + canParseManifest(JSON_EXTRACT(payload, '$._pwa.manifests')) AS can_parse, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + can_parse +UNION ALL +SELECT + 'All Pages' AS type, + _TABLE_SUFFIX AS client, + canParseManifest(JSON_EXTRACT(payload, '$._pwa.manifests')) AS can_parse, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +GROUP BY + client, + can_parse +ORDER BY + type DESC, + freq / total DESC, + can_parse, + client diff --git a/sql/2024/pwa/manifests_note_taking.sql b/sql/2024/pwa/manifests_note_taking.sql new file mode 100644 index 00000000000..802403fbd4e --- /dev/null +++ b/sql/2024/pwa/manifests_note_taking.sql @@ -0,0 +1,28 @@ +#standardSQL +# % manifests with note taking for service worker pages and all pages + +CREATE TEMP FUNCTION hasNoteTaking(manifest STRING) +RETURNS BOOLEAN LANGUAGE js AS ''' +try { + var $ = Object.values(JSON.parse(manifest))[0]; + return $.hasOwnProperty('note_taking') && $.note_taking != {}; +} catch (e) { + return null; +} +'''; + +SELECT + 'PWA Pages' AS type, + _TABLE_SUFFIX AS client, + hasNoteTaking(JSON_EXTRACT(payload, '$._pwa.manifests')) AS hasNoteTaking, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + hasNoteTaking diff --git a/sql/2024/pwa/manifests_preferring_native_apps.sql b/sql/2024/pwa/manifests_preferring_native_apps.sql new file mode 100644 index 00000000000..5ad9bdc7dfc --- /dev/null +++ b/sql/2024/pwa/manifests_preferring_native_apps.sql @@ -0,0 +1,46 @@ +#standardSQL +# % manifests preferring native apps for service worker pages and all pages + +CREATE TEMP FUNCTION prefersNative(manifest STRING) +RETURNS BOOLEAN LANGUAGE js AS ''' +try { + var $ = Object.values(JSON.parse(manifest))[0]; + return $.prefer_related_applications == true && $.related_applications.length > 0; +} catch (e) { + return null; +} +'''; + +SELECT + 'PWA Pages' AS type, + _TABLE_SUFFIX AS client, + prefersNative(JSON_EXTRACT(payload, '$._pwa.manifests')) AS prefersNative, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + prefersNative +UNION ALL +SELECT + 'All Pages' AS type, + _TABLE_SUFFIX AS client, + prefersNative(JSON_EXTRACT(payload, '$._pwa.manifests')) AS prefersNative, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +GROUP BY + client, + prefersNative +ORDER BY + type DESC, + freq / total DESC, + prefersNative, + client diff --git a/sql/2024/pwa/manifests_protocol_handlers.sql b/sql/2024/pwa/manifests_protocol_handlers.sql new file mode 100644 index 00000000000..2bf2621387c --- /dev/null +++ b/sql/2024/pwa/manifests_protocol_handlers.sql @@ -0,0 +1,28 @@ +#standardSQL +# % manifests with protocol handlers for service worker pages and all pages + +CREATE TEMP FUNCTION hasProtocolHandlers(manifest STRING) +RETURNS BOOLEAN LANGUAGE js AS ''' +try { + var $ = Object.values(JSON.parse(manifest))[0]; + return $.hasOwnProperty('protocol_handlers') && $.protocol_handlers.length > 0; +} catch (e) { + return null; +} +'''; + +SELECT + 'PWA Pages' AS type, + _TABLE_SUFFIX AS client, + hasProtocolHandlers(JSON_EXTRACT(payload, '$._pwa.manifests')) AS hasProtocolHandlers, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + hasProtocolHandlers diff --git a/sql/2024/pwa/manifests_screenshots.sql b/sql/2024/pwa/manifests_screenshots.sql new file mode 100644 index 00000000000..fc68c9a4976 --- /dev/null +++ b/sql/2024/pwa/manifests_screenshots.sql @@ -0,0 +1,28 @@ +#standardSQL +# % manifests with screenshots for service worker pages and all pages + +CREATE TEMP FUNCTION hasScreenshots(manifest STRING) +RETURNS BOOLEAN LANGUAGE js AS ''' +try { + var $ = Object.values(JSON.parse(manifest))[0]; + return $.hasOwnProperty('screenshots') && $.screenshots.length > 0; +} catch (e) { + return null; +} +'''; + +SELECT + 'PWA Pages' AS type, + _TABLE_SUFFIX AS client, + hasScreenshots(JSON_EXTRACT(payload, '$._pwa.manifests')) AS hasScreenshots, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + hasScreenshots diff --git a/sql/2024/pwa/manifests_screenshots_platform_narrow.sql b/sql/2024/pwa/manifests_screenshots_platform_narrow.sql new file mode 100644 index 00000000000..41f5e46fda9 --- /dev/null +++ b/sql/2024/pwa/manifests_screenshots_platform_narrow.sql @@ -0,0 +1,30 @@ +#standardSQL +# % manifests with narrow platform screenshots for service worker pages and all pages + +CREATE TEMP FUNCTION hasScreenshotsPlatformNarrow(manifest STRING) +RETURNS BOOLEAN LANGUAGE js AS ''' +try { + var manifest = Object.values(JSON.parse(manifest))[0]; + var screenshots = manifest.screenshots; + var check = screenshots.some(obj => obj.platform == 'narrow'); + return check; +} catch (e) { + return null; +} +'''; + +SELECT + 'PWA Pages' AS type, + _TABLE_SUFFIX AS client, + hasScreenshotsPlatformNarrow(JSON_EXTRACT(payload, '$._pwa.manifests')) AS platformNarrow, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + platformNarrow diff --git a/sql/2024/pwa/manifests_screenshots_platform_wide.sql b/sql/2024/pwa/manifests_screenshots_platform_wide.sql new file mode 100644 index 00000000000..2fee358ed22 --- /dev/null +++ b/sql/2024/pwa/manifests_screenshots_platform_wide.sql @@ -0,0 +1,30 @@ +#standardSQL +# % manifests with wide platform screenshots for service worker pages and all pages + +CREATE TEMP FUNCTION hasScreenshotsPlatformWide(manifest STRING) +RETURNS BOOLEAN LANGUAGE js AS ''' +try { + var manifest = Object.values(JSON.parse(manifest))[0]; + var screenshots = manifest.screenshots; + var check = screenshots.some(obj => obj.platform == 'wide'); + return check; +} catch (e) { + return null; +} +'''; + +SELECT + 'PWA Pages' AS type, + _TABLE_SUFFIX AS client, + hasScreenshotsPlatformWide(JSON_EXTRACT(payload, '$._pwa.manifests')) AS platformWide, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + platformWide diff --git a/sql/2024/pwa/manifests_share_target.sql b/sql/2024/pwa/manifests_share_target.sql new file mode 100644 index 00000000000..926c38d1071 --- /dev/null +++ b/sql/2024/pwa/manifests_share_target.sql @@ -0,0 +1,28 @@ +#standardSQL +# % manifests with web share target for service worker pages and all pages + +CREATE TEMP FUNCTION hasShareTarget(manifest STRING) +RETURNS BOOLEAN LANGUAGE js AS ''' +try { + var $ = Object.values(JSON.parse(manifest))[0]; + return $.hasOwnProperty('share_target') && $.share_target != {}; +} catch (e) { + return null; +} +'''; + +SELECT + 'PWA Pages' AS type, + _TABLE_SUFFIX AS client, + hasShareTarget(JSON_EXTRACT(payload, '$._pwa.manifests')) AS hasShareTarget, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + hasShareTarget diff --git a/sql/2024/pwa/manifests_shortcuts.sql b/sql/2024/pwa/manifests_shortcuts.sql new file mode 100644 index 00000000000..c321a91315d --- /dev/null +++ b/sql/2024/pwa/manifests_shortcuts.sql @@ -0,0 +1,28 @@ +#standardSQL +# % manifests with shortcuts for service worker pages and all pages + +CREATE TEMP FUNCTION hasShortcuts(manifest STRING) +RETURNS BOOLEAN LANGUAGE js AS ''' +try { + var $ = Object.values(JSON.parse(manifest))[0]; + return $.hasOwnProperty('shortcuts') && $.shortcuts.length > 0; +} catch (e) { + return null; +} +'''; + +SELECT + 'PWA Pages' AS type, + _TABLE_SUFFIX AS client, + hasShortcuts(JSON_EXTRACT(payload, '$._pwa.manifests')) AS hasShortcuts, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + hasShortcuts diff --git a/sql/2024/pwa/manifests_window_controls_overlay.sql b/sql/2024/pwa/manifests_window_controls_overlay.sql new file mode 100644 index 00000000000..1b2032b5db7 --- /dev/null +++ b/sql/2024/pwa/manifests_window_controls_overlay.sql @@ -0,0 +1,30 @@ +#standardSQL +# % manifests with window controls overlay display override for service worker pages and all pages + +CREATE TEMP FUNCTION hasWindowControlsOverlay(manifest STRING) +RETURNS BOOLEAN LANGUAGE js AS ''' +try { + var manifest = Object.values(JSON.parse(manifest))[0]; + var display_override = manifest.display_override; + var check = display_override.includes("window-controls-overlay"); + return check; +} catch (e) { + return null; +} +'''; + +SELECT + 'PWA Pages' AS type, + _TABLE_SUFFIX AS client, + hasWindowControlsOverlay(JSON_EXTRACT(payload, '$._pwa.manifests')) AS hasWindowControlsOverlay, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + hasWindowControlsOverlay diff --git a/sql/2024/pwa/popular_pwa_libraries.sql b/sql/2024/pwa/popular_pwa_libraries.sql new file mode 100644 index 00000000000..5a6332d4118 --- /dev/null +++ b/sql/2024/pwa/popular_pwa_libraries.sql @@ -0,0 +1,100 @@ +#standardSQL +# Popular PWA script +CREATE TEMPORARY FUNCTION getSWLibraries(importScriptsInfo STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + /* 'importScriptsInfo' returns an array of script that might import other script + The final array of script comes from the combination of both */ + var ObjKeys = Object.keys(JSON.parse(importScriptsInfo)); + var ObjValues = Object.values(JSON.parse(importScriptsInfo)); + var script = ObjKeys.concat(ObjValues); + /* Replacing spaces and commas */ + for (var i = 0; i < script.length; i++) { + script[i] = script[i].toString().trim().replace(/'/g, ""); + } + + /* Creating a Set to eliminate duplicates and transforming back to an array to respect the function signature */ + return Array.from(new Set(script)); +} catch (e) { + return []; +} +'''; + +SELECT + client, + COUNTIF(workbox > 0) / total AS workbox, + COUNTIF(sw_toolbox > 0) / total AS sw_toolbox, + COUNTIF(firebase > 0) / total AS firebase, + COUNTIF(OneSignalSDK > 0) / total AS OneSignalSDK, + COUNTIF(najva > 0) / total AS najva, + COUNTIF(upush > 0) / total AS upush, + COUNTIF(cache_polyfill > 0) / total AS cache_polyfill, + COUNTIF(analytics_helper > 0) / total AS analytics_helper, + COUNTIF(recaptcha > 0) / total AS recaptcha, + COUNTIF(pwabuilder > 0) / total AS pwabuilder, + COUNTIF(pushprofit > 0) / total AS pushprofit, + COUNTIF(sendpulse > 0) / total AS sendpulse, + COUNTIF(quora > 0) / total AS quora, + COUNTIF(none_of_the_above > 0) / total AS none_of_the_above, + COUNTIF(importscripts > 0) / total AS uses_importscript, + total +FROM + ( + SELECT + _TABLE_SUFFIX AS client, + url, + COUNT(0) AS importscripts, + COUNTIF(LOWER(script) LIKE '%workbox%') AS workbox, + COUNTIF(LOWER(script) LIKE '%sw-toolbox%') AS sw_toolbox, + COUNTIF(LOWER(script) LIKE '%firebase%') AS firebase, + COUNTIF(LOWER(script) LIKE '%onesignalsdk%') AS OneSignalSDK, + COUNTIF(LOWER(script) LIKE '%najva%') AS najva, + COUNTIF(LOWER(script) LIKE '%upush%') AS upush, + COUNTIF(LOWER(script) LIKE '%cache-polyfill%') AS cache_polyfill, + COUNTIF(LOWER(script) LIKE '%analytics-helper%') AS analytics_helper, + COUNTIF(LOWER(script) LIKE '%recaptcha%') AS recaptcha, + COUNTIF(LOWER(script) LIKE '%pwabuilder%') AS pwabuilder, + COUNTIF(LOWER(script) LIKE '%pushprofit%') AS pushprofit, + COUNTIF(LOWER(script) LIKE '%sendpulse%') AS sendpulse, + COUNTIF(LOWER(script) LIKE '%quore%') AS quora, + COUNTIF(LOWER(script) NOT LIKE '%workbox%' AND + LOWER(script) NOT LIKE '%sw-toolbox%' AND + LOWER(script) NOT LIKE '%firebase%' AND + LOWER(script) NOT LIKE '%onesignalsdk%' AND + LOWER(script) NOT LIKE '%najva%' AND + LOWER(script) NOT LIKE '%upush%' AND + LOWER(script) NOT LIKE '%cache-polyfill.js%' AND + LOWER(script) NOT LIKE '%analytics-helper.js%' AND + LOWER(script) NOT LIKE '%recaptcha%' AND + LOWER(script) NOT LIKE '%pwabuilder%' AND + LOWER(script) NOT LIKE '%pushprofit%' AND + LOWER(script) NOT LIKE '%sendpulse%' AND + LOWER(script) NOT LIKE '%quora%') AS none_of_the_above + FROM + `httparchive.pages.2024_06_01_*`, + UNNEST(getSWLibraries(JSON_EXTRACT(payload, '$._pwa.importScriptsInfo'))) AS script + WHERE + JSON_EXTRACT(payload, '$._pwa.importScriptsInfo') != '[]' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' + GROUP BY + _TABLE_SUFFIX, + url + ) +JOIN + ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(0) AS total + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' + GROUP BY + client + ) +USING (client) +GROUP BY + client, + total +ORDER BY + client diff --git a/sql/2024/pwa/popular_pwa_libraries_helper.sql b/sql/2024/pwa/popular_pwa_libraries_helper.sql new file mode 100644 index 00000000000..00163ccee27 --- /dev/null +++ b/sql/2024/pwa/popular_pwa_libraries_helper.sql @@ -0,0 +1,48 @@ +#standardSQL +# Use this sql to find popular library imports for popular_pwa_libraries.sql +# And also other importscripts used in service workers +CREATE TEMPORARY FUNCTION getSWLibraries(importScriptsInfo STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + /* 'importScriptsInfo' returns an array of libraries that might import other libraries + The final array of libraries comes from the combination of both */ + var ObjKeys = Object.keys(JSON.parse(importScriptsInfo)); + var ObjValues = Object.values(JSON.parse(importScriptsInfo)); + var libraries = ObjKeys.concat(ObjValues); + /* Replacing spaces and commas */ + for (var i = 0; i < libraries.length; i++) { + libraries[i] = libraries[i].toString().trim().replace(/'/g, ""); + } + + /* Creating a Set to eliminate duplicates and transforming back to an array to respect the function signature */ + return Array.from(new Set(libraries)); +} catch (e) { + return []; +} +'''; + +SELECT + _TABLE_SUFFIX AS client, + script, + COUNT(DISTINCT url) AS freq +FROM + `httparchive.pages.2024_06_01_*`, + UNNEST(getSWLibraries(JSON_EXTRACT(payload, '$._pwa.importScriptsInfo'))) AS script +WHERE + JSON_EXTRACT(payload, '$._pwa.importScriptsInfo') != '[]' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' AND + LOWER(script) NOT LIKE '%workbox%' AND + LOWER(script) NOT LIKE '%sw-toolbox%' AND + LOWER(script) NOT LIKE '%firebase%' AND + LOWER(script) NOT LIKE '%onesignalsdk%' AND + LOWER(script) NOT LIKE '%najva%' AND + LOWER(script) NOT LIKE '%upush%' AND + LOWER(script) NOT LIKE '%cache-polyfill.js%' AND + LOWER(script) NOT LIKE '%analytics-helper.js%' AND + LOWER(script) NOT LIKE '%recaptcha%' AND + LOWER(script) NOT LIKE '%pwabuilder%' +GROUP BY + _TABLE_SUFFIX, + script +ORDER BY + freq DESC diff --git a/sql/2024/pwa/pwa_blink_usage.sql b/sql/2024/pwa/pwa_blink_usage.sql new file mode 100644 index 00000000000..90a98e01137 --- /dev/null +++ b/sql/2024/pwa/pwa_blink_usage.sql @@ -0,0 +1,21 @@ +#standardSQL +# PWA features tracked in blink_features.usage +SELECT DISTINCT + client, + feature, + num_urls, + total_urls, + num_urls / total_urls AS pct +FROM + `httparchive.blink_features.usage` +WHERE + yyyymmdd = '20240601' AND + ( + feature LIKE '%ServiceWorker%' OR + feature LIKE '%BackgroundSync%' OR + feature LIKE '%GetInstalledRelatedApps%' + ) +ORDER BY + num_urls DESC, + client, + feature diff --git a/sql/2024/pwa/pwa_notification_acceptance_rates.sql b/sql/2024/pwa/pwa_notification_acceptance_rates.sql new file mode 100644 index 00000000000..2795812fc02 --- /dev/null +++ b/sql/2024/pwa/pwa_notification_acceptance_rates.sql @@ -0,0 +1,47 @@ +#standardSQL +# 13_16b: Web Push Notification CRUX stats (10th / 25th / 50th / 75th / 90th / 100th percentile) for PWA origins +# Based on https://github.com/HTTPArchive/almanac.httparchive.org/blob/main/sql/2020/ecommerce/webpushstats_ecommsites.sql + +SELECT + date, + client, + percentile, + APPROX_QUANTILES(notification_permission_accept, 1000 RESPECT NULLS)[OFFSET(percentile * 10)] AS notification_permission_accept, + APPROX_QUANTILES(notification_permission_deny, 1000 RESPECT NULLS)[OFFSET(percentile * 10)] AS notification_permission_deny, + APPROX_QUANTILES(notification_permission_ignore, 1000 RESPECT NULLS)[OFFSET(percentile * 10)] AS notification_permission_ignore, + APPROX_QUANTILES(notification_permission_dismiss, 1000 RESPECT NULLS)[OFFSET(percentile * 10)] AS notification_permission_dismiss +FROM + `chrome-ux-report.materialized.metrics_summary` +JOIN + ( + SELECT + _TABLE_SUFFIX AS client, + RTRIM(url, '/') AS origin, + COUNT(0) AS total + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' + GROUP BY + _TABLE_SUFFIX, + url + ) +USING + (origin), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +WHERE + date IN ('2024-06-01') AND + ( + notification_permission_accept IS NOT NULL OR + notification_permission_deny IS NOT NULL OR + notification_permission_ignore IS NOT NULL OR + notification_permission_dismiss IS NOT NULL + ) +GROUP BY + date, + client, + percentile +ORDER BY + date, + client, + percentile diff --git a/sql/2024/pwa/pwa_notification_acceptance_rates_over_time.sql b/sql/2024/pwa/pwa_notification_acceptance_rates_over_time.sql new file mode 100644 index 00000000000..142e2f46552 --- /dev/null +++ b/sql/2024/pwa/pwa_notification_acceptance_rates_over_time.sql @@ -0,0 +1,27 @@ +#standardSQL +# Total Noitification Acceptence Rates across all origins +# Note: not weighted by domain popularity nor number of notifications shown + +SELECT + date, + IF(device = 'phone', 'mobile', device) AS client, + SUM(notification_permission_accept) / SUM(notification_permission_accept + notification_permission_deny + notification_permission_ignore + notification_permission_dismiss) AS accept, + SUM(notification_permission_deny) / SUM(notification_permission_accept + notification_permission_deny + notification_permission_ignore + notification_permission_dismiss) AS deny, + SUM(notification_permission_ignore) / SUM(notification_permission_accept + notification_permission_deny + notification_permission_ignore + notification_permission_dismiss) AS _ignore, + SUM(notification_permission_dismiss) / SUM(notification_permission_accept + notification_permission_deny + notification_permission_ignore + notification_permission_dismiss) AS dismiss +FROM + `chrome-ux-report.materialized.device_summary` +WHERE + ( + notification_permission_accept IS NOT NULL OR + notification_permission_deny IS NOT NULL OR + notification_permission_ignore IS NOT NULL OR + notification_permission_dismiss IS NOT NULL + ) AND + device IN ('desktop', 'phone') +GROUP BY + date, + device +ORDER BY + date DESC, + device diff --git a/sql/2024/pwa/sw_adoption_over_time.sql b/sql/2024/pwa/sw_adoption_over_time.sql new file mode 100644 index 00000000000..fced747b0ad --- /dev/null +++ b/sql/2024/pwa/sw_adoption_over_time.sql @@ -0,0 +1,15 @@ +#standardSQL +# SW adoption over time +SELECT DISTINCT + yyyymmdd AS date, + client, + num_urls AS freq, + total_urls AS total, + pct_urls AS pct +FROM + `httparchive.blink_features.usage` +WHERE + feature = 'ServiceWorkerControlledPage' +ORDER BY + date DESC, + client diff --git a/sql/2024/pwa/sw_adoption_over_time_ranking.sql b/sql/2024/pwa/sw_adoption_over_time_ranking.sql new file mode 100644 index 00000000000..eb2765e58c8 --- /dev/null +++ b/sql/2024/pwa/sw_adoption_over_time_ranking.sql @@ -0,0 +1,57 @@ +#standardSQL +# SW adoption over time, including ranking +SELECT + yyyymmdd AS date, + client, + rank_grouping, + CASE + WHEN rank_grouping = 10000000 THEN 'all' + ELSE FORMAT("%'d", rank_grouping) + END AS ranking, + COUNT(0) AS freq, + total, + COUNT(0) / total AS pct +FROM + ( + SELECT DISTINCT + FORMAT_DATE('%Y%m%d', yyyymmdd) AS yyyymmdd, + client, + url, + rank + FROM + `httparchive.blink_features.features` + WHERE + feature = 'ServiceWorkerControlledPage' AND + yyyymmdd >= '2024-05-01' + ), + UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping +JOIN ( + SELECT + REPLACE(SUBSTR(_TABLE_SUFFIX, 0, 10), '_', '') AS yyyymmdd, + SUBSTR(_TABLE_SUFFIX, 12) AS client, + rank_grouping, + COUNT(0) AS total + FROM + `httparchive.summary_pages.*`, + UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping + WHERE + _TABLE_SUFFIX > '2024_05_01' AND + rank <= rank_grouping + GROUP BY + yyyymmdd, + rank_grouping, + client +) +USING + (yyyymmdd, client, rank_grouping) +WHERE + rank <= rank_grouping +GROUP BY + yyyymmdd, + client, + total, + rank_grouping +ORDER BY + date DESC, + rank_grouping, + client diff --git a/sql/2024/pwa/sw_events.sql b/sql/2024/pwa/sw_events.sql new file mode 100644 index 00000000000..64749a181c1 --- /dev/null +++ b/sql/2024/pwa/sw_events.sql @@ -0,0 +1,46 @@ +#standardSQL +# SW events + +CREATE TEMPORARY FUNCTION getSWEvents(payload STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + var payloadJSON = JSON.parse(payload); + var swEventListenersInfo = (Object.values(payloadJSON.swEventListenersInfo)).flat(); + var swPropertiesInfo = (Object.values(payloadJSON.swPropertiesInfo)).flat(); + return [...new Set([...swEventListenersInfo ,...swPropertiesInfo])]; +} catch (e) { + return []; +} +'''; + +SELECT + _TABLE_SUFFIX AS client, + event, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.pages.2024_06_01_*`, + UNNEST(getSWEvents(JSON_EXTRACT(payload, '$._pwa'))) AS event +JOIN + ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' + GROUP BY + _TABLE_SUFFIX + ) +USING (_TABLE_SUFFIX) +WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + total, + event +ORDER BY + freq / total DESC, + client diff --git a/sql/2024/pwa/sw_methods.sql b/sql/2024/pwa/sw_methods.sql new file mode 100644 index 00000000000..85937409e3d --- /dev/null +++ b/sql/2024/pwa/sw_methods.sql @@ -0,0 +1,44 @@ +#standardSQL +# SW methods +CREATE TEMPORARY FUNCTION getSWMethods(swMethodsInfo STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + var swMethods = JSON.parse(swMethodsInfo); + return Array.from(new Set(Object.values(swMethods).flat())); +} catch (e) { + return []; +} +'''; + +SELECT + _TABLE_SUFFIX AS client, + sw_method, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.pages.2024_06_01_*`, + UNNEST(getSWMethods(JSON_EXTRACT(payload, '$._pwa.swMethodsInfo'))) AS sw_method +JOIN + ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' + GROUP BY + _TABLE_SUFFIX + ) +USING (_TABLE_SUFFIX) +WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' AND + JSON_EXTRACT(payload, '$._pwa.swMethodsInfo') != '[]' +GROUP BY + client, + total, + sw_method +ORDER BY + freq / total DESC, + client diff --git a/sql/2024/pwa/sw_objects.sql b/sql/2024/pwa/sw_objects.sql new file mode 100644 index 00000000000..1e8bb710ebc --- /dev/null +++ b/sql/2024/pwa/sw_objects.sql @@ -0,0 +1,47 @@ +#standardSQL +# SW objects +CREATE TEMPORARY FUNCTION getSWObjects(swObjectsInfo STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + var swObjects = Object.values(JSON.parse(swObjectsInfo)); + if (typeof swObjects != 'string') { + swObjects = swObjects.toString(); + } + swObjects = swObjects.trim().split(','); + return Array.from(new Set(swObjects)); +} catch (e) { + return []; +} +'''; +SELECT + _TABLE_SUFFIX AS client, + sw_object, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.pages.2024_06_01_*`, + UNNEST(getSWObjects(JSON_EXTRACT(payload, '$._pwa.swObjectsInfo'))) AS sw_object +JOIN + ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' + GROUP BY + _TABLE_SUFFIX + ) +USING (_TABLE_SUFFIX) +WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' AND + JSON_EXTRACT(payload, '$._pwa.swObjectsInfo') != '[]' +GROUP BY + client, + total, + sw_object +ORDER BY + freq / total DESC, + client diff --git a/sql/2024/pwa/sw_objects_name_only.sql b/sql/2024/pwa/sw_objects_name_only.sql new file mode 100644 index 00000000000..ac4acdef045 --- /dev/null +++ b/sql/2024/pwa/sw_objects_name_only.sql @@ -0,0 +1,47 @@ +#standardSQL +# SW objects name only +CREATE TEMPORARY FUNCTION getSWObjects(swObjectsInfo STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + var swObjects = Object.values(JSON.parse(swObjectsInfo)); + if (typeof swObjects != 'string') { + swObjects = swObjects.toString(); + } + swObjects = swObjects.trim().split(',').map(x => x.split('.')[0]); + return Array.from(new Set(swObjects)); +} catch (e) { + return []; +} +'''; +SELECT + _TABLE_SUFFIX AS client, + sw_object, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.pages.2024_06_01_*`, + UNNEST(getSWObjects(JSON_EXTRACT(payload, '$._pwa.swObjectsInfo'))) AS sw_object +JOIN + ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' + GROUP BY + _TABLE_SUFFIX + ) +USING (_TABLE_SUFFIX) +WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' AND + JSON_EXTRACT(payload, '$._pwa.swObjectsInfo') != '[]' +GROUP BY + client, + total, + sw_object +ORDER BY + freq / total DESC, + client diff --git a/sql/2024/pwa/sw_registration_properties.sql b/sql/2024/pwa/sw_registration_properties.sql new file mode 100644 index 00000000000..ef070f63ab7 --- /dev/null +++ b/sql/2024/pwa/sw_registration_properties.sql @@ -0,0 +1,48 @@ +#standardSQL +# SW registration properties +CREATE TEMPORARY FUNCTION getSWRegistrationProperties(swRegistrationPropertiesInfo STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + var swRegistrationProperties = Object.values(JSON.parse(swRegistrationPropertiesInfo)); + if (typeof swRegistrationProperties != 'string') { + swRegistrationProperties = swRegistrationProperties.toString(); + } + swRegistrationProperties = swRegistrationProperties.trim().split(','); + return Array.from(new Set(swRegistrationProperties)); +} catch (e) { + return []; +} +'''; + +SELECT + _TABLE_SUFFIX AS client, + sw_registration_properties, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.pages.2024_06_01_*`, + UNNEST(getSWRegistrationProperties(JSON_EXTRACT(payload, '$._pwa.swRegistrationPropertiesInfo'))) AS sw_registration_properties +JOIN + ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' + GROUP BY + _TABLE_SUFFIX + ) +USING (_TABLE_SUFFIX) +WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' AND + JSON_EXTRACT(payload, '$._pwa.swRegistrationPropertiesInfo') != '[]' +GROUP BY + client, + total, + sw_registration_properties +ORDER BY + freq / total DESC, + client diff --git a/sql/2024/pwa/sw_registration_properties_name_only.sql b/sql/2024/pwa/sw_registration_properties_name_only.sql new file mode 100644 index 00000000000..df814f04327 --- /dev/null +++ b/sql/2024/pwa/sw_registration_properties_name_only.sql @@ -0,0 +1,48 @@ +#standardSQL +# SW registration properties name only +CREATE TEMPORARY FUNCTION getSWRegistrationProperties(swRegistrationPropertiesInfo STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + var swRegistrationProperties = Object.values(JSON.parse(swRegistrationPropertiesInfo)); + if (typeof swRegistrationProperties != 'string') { + swRegistrationProperties = swRegistrationProperties.toString(); + } + swRegistrationProperties = swRegistrationProperties.trim().split(',').map(x => x.split('.')[0]); + return Array.from(new Set(swRegistrationProperties)); +} catch (e) { + return []; +} +'''; + +SELECT + _TABLE_SUFFIX AS client, + sw_registration_properties, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.pages.2024_06_01_*`, + UNNEST(getSWRegistrationProperties(JSON_EXTRACT(payload, '$._pwa.swRegistrationPropertiesInfo'))) AS sw_registration_properties +JOIN + ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' + GROUP BY + _TABLE_SUFFIX + ) +USING (_TABLE_SUFFIX) +WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' AND + JSON_EXTRACT(payload, '$._pwa.swRegistrationPropertiesInfo') != '[]' +GROUP BY + client, + total, + sw_registration_properties +ORDER BY + freq / total DESC, + client diff --git a/sql/2024/pwa/top_manifest_categories.sql b/sql/2024/pwa/top_manifest_categories.sql new file mode 100644 index 00000000000..bd99231dfaf --- /dev/null +++ b/sql/2024/pwa/top_manifest_categories.sql @@ -0,0 +1,86 @@ +#standardSQL +# Top manifest categories +CREATE TEMPORARY FUNCTION getCategories(manifest STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + var $ = Object.values(JSON.parse(manifest))[0]; + var categories = $.categories; + if (typeof categories == 'string') { + return [categories]; + } + return categories; +} catch (e) { + return null; +} +'''; + +WITH totals AS ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total, + COUNTIF(JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true') AS pwa_total + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' + GROUP BY + _TABLE_SUFFIX +), + +manifests_categories AS ( + SELECT + 'All Sites' AS type, + _TABLE_SUFFIX AS client, + category, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct, + COUNTIF(JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true') AS pwa_freq, + pwa_total, + COUNTIF(JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true') / pwa_total AS pwa_pct + FROM + `httparchive.pages.2024_06_01_*`, + UNNEST(getCategories(JSON_EXTRACT(payload, '$._pwa.manifests'))) AS category + JOIN + totals + USING (_TABLE_SUFFIX) + WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' + GROUP BY + client, + category, + total, + pwa_total + HAVING + category IS NOT NULL + ORDER BY + type DESC, + freq / total DESC, + category, + client +) + +SELECT + 'PWA Sites' AS type, + client, + category, + pwa_freq AS freq, + pwa_total AS total, + pwa_pct AS pct +FROM + manifests_categories +UNION ALL +SELECT + 'All Sites' AS type, + client, + category, + freq, + total, + pct +FROM + manifests_categories +ORDER BY + type DESC, + pct DESC, + category, + client diff --git a/sql/2024/pwa/top_manifest_display_values.sql b/sql/2024/pwa/top_manifest_display_values.sql new file mode 100644 index 00000000000..5a129946c72 --- /dev/null +++ b/sql/2024/pwa/top_manifest_display_values.sql @@ -0,0 +1,56 @@ +#standardSQL +# Top most used display values in manifest files + +CREATE TEMP FUNCTION getDisplay(manifest STRING) RETURNS STRING LANGUAGE js AS ''' +try { + var $ = Object.values(JSON.parse(manifest))[0]; + if (!('display' in $)) { + return '(not set)'; + } + return $.display; +} catch { + return '(not set)' +} +'''; + +SELECT + 'PWA Sites' AS type, + _TABLE_SUFFIX AS client, + getDisplay(JSON_EXTRACT(payload, '$._pwa.manifests')) AS display, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + display +QUALIFY + display IS NOT NULL AND + freq > 100 +UNION ALL +SELECT + 'All Sites' AS type, + _TABLE_SUFFIX AS client, + getDisplay(JSON_EXTRACT(payload, '$._pwa.manifests')) AS display, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' +GROUP BY + client, + display +QUALIFY + display IS NOT NULL AND + freq > 100 +ORDER BY + type DESC, + freq / total DESC, + display, + client diff --git a/sql/2024/pwa/top_manifest_icon_sizes.sql b/sql/2024/pwa/top_manifest_icon_sizes.sql new file mode 100644 index 00000000000..e7403af265e --- /dev/null +++ b/sql/2024/pwa/top_manifest_icon_sizes.sql @@ -0,0 +1,88 @@ +#standardSQL +# Top manifest icon sizes +CREATE TEMPORARY FUNCTION getIconSizes(manifest STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + var $ = Object.values(JSON.parse(manifest))[0]; + return $.icons.map(icon => icon.sizes); +} catch (e) { + return null; +} +'''; + +WITH totals AS ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total, + COUNTIF(JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true') AS pwa_total + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' + GROUP BY + _TABLE_SUFFIX +), + +manifests_icon_sizes AS ( + SELECT + 'All Sites' AS type, + _TABLE_SUFFIX AS client, + size, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct, + COUNTIF(JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true') AS pwa_freq, + pwa_total, + COUNTIF(JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true') / pwa_total AS pwa_pct + FROM + `httparchive.pages.2024_06_01_*`, + UNNEST(getIconSizes(JSON_EXTRACT(payload, '$._pwa.manifests'))) AS size + JOIN + totals + USING (_TABLE_SUFFIX) + WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' + GROUP BY + client, + size, + total, + pwa_total + HAVING + size IS NOT NULL + ORDER BY + type DESC, + freq / total DESC, + size, + client +) + +SELECT + 'PWA Sites' AS type, + client, + size, + pwa_freq AS freq, + pwa_total AS total, + pwa_pct AS pct +FROM + manifests_icon_sizes +WHERE + size IS NOT NULL AND + freq > 100 +UNION ALL +SELECT + 'All Sites' AS type, + client, + size, + freq, + total, + pct +FROM + manifests_icon_sizes +WHERE + size IS NOT NULL AND + freq > 100 +ORDER BY + type DESC, + pct DESC, + size, + client diff --git a/sql/2024/pwa/top_manifest_orientations.sql b/sql/2024/pwa/top_manifest_orientations.sql new file mode 100644 index 00000000000..9500090a5bb --- /dev/null +++ b/sql/2024/pwa/top_manifest_orientations.sql @@ -0,0 +1,52 @@ +#standardSQL +# Top manifest orientations + +CREATE TEMP FUNCTION getOrientation(manifest STRING) RETURNS STRING LANGUAGE js AS ''' +try { + var $ = Object.values(JSON.parse(manifest))[0]; + if (!('orientation' in $)) { + return '(not set)'; + } + return $.orientation; +} catch { + return '(not set)' +} +'''; + +SELECT + 'PWA Sites' AS type, + _TABLE_SUFFIX AS client, + getOrientation(JSON_EXTRACT(payload, '$._pwa.manifests')) AS orientation, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + type, + client, + orientation +UNION ALL +SELECT + 'All Sites' AS type, + _TABLE_SUFFIX AS client, + getOrientation(JSON_EXTRACT(payload, '$._pwa.manifests')) AS orientation, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' +GROUP BY + type, + client, + orientation +ORDER BY + type DESC, + freq / total DESC, + orientation, + client diff --git a/sql/2024/pwa/top_manifest_properties.sql b/sql/2024/pwa/top_manifest_properties.sql new file mode 100644 index 00000000000..7c323b0275f --- /dev/null +++ b/sql/2024/pwa/top_manifest_properties.sql @@ -0,0 +1,92 @@ +#standardSQL +# Top manifest properties + +CREATE TEMP FUNCTION getManifestProps(manifest STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + var manifestJSON = Object.values(JSON.parse(manifest))[0]; + if (typeof manifestJSON === 'string') { + return null; + } + return Object.keys(manifestJSON); +} catch { + return null +} +'''; + +WITH totals AS ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total, + COUNTIF(JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true') AS pwa_total + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' + GROUP BY + _TABLE_SUFFIX +), + +manifests_properties AS ( + SELECT + 'All Sites' AS type, + _TABLE_SUFFIX AS client, + property, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct, + COUNTIF(JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true') AS pwa_freq, + pwa_total, + COUNTIF(JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true') / pwa_total AS pwa_pct + FROM + `httparchive.pages.2024_06_01_*`, + UNNEST(getManifestProps(JSON_EXTRACT(payload, '$._pwa.manifests'))) AS property + JOIN + totals + USING (_TABLE_SUFFIX) + WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' + GROUP BY + client, + property, + total, + pwa_total + HAVING + property IS NOT NULL + ORDER BY + type DESC, + freq / total DESC, + property, + client +) + +SELECT + 'PWA Sites' AS type, + client, + property, + pwa_freq AS freq, + pwa_total AS total, + pwa_pct AS pct +FROM + manifests_properties +WHERE + property IS NOT NULL AND + freq > 100 +UNION ALL +SELECT + 'All Sites' AS type, + client, + property, + freq, + total, + pct +FROM + manifests_properties +WHERE + property IS NOT NULL AND + freq > 100 +ORDER BY + type DESC, + pct DESC, + property, + client diff --git a/sql/2024/pwa/workbox_methods.sql b/sql/2024/pwa/workbox_methods.sql new file mode 100644 index 00000000000..d3756caffb3 --- /dev/null +++ b/sql/2024/pwa/workbox_methods.sql @@ -0,0 +1,61 @@ +#standardSQL +#Workbox methods +CREATE TEMPORARY FUNCTION getWorkboxMethods(workboxInfo STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + var workboxPackageMethods = Object.values(JSON.parse(workboxInfo)); + if (typeof workboxPackageMethods == 'string') { + workboxPackageMethods = [workboxPackageMethods]; + } + var workboxMethods = []; + /* Replacing spaces and commas */ + for (var i = 0; i < workboxPackageMethods.length; i++) { + var workboxItems = workboxPackageMethods[i].toString().trim().split(','); + for(var j = 0; j < workboxItems.length; j++) { + if(workboxItems[j].indexOf(':') == -1) { + if (workboxItems[j].trim().startsWith('workbox.')) { + workboxMethods.push(workboxItems[j].trim().substring(8)); + } + } + } + } + return Array.from(new Set(workboxMethods)); +} catch (e) { + return []; +} +'''; + +SELECT + _TABLE_SUFFIX AS client, + workbox_method, + REGEXP_EXTRACT(workbox_method, r'^([^.]+)') AS module_only, + REGEXP_EXTRACT(workbox_method, r'^[^.]+\.([^.]+)') AS method_only, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.pages.2024_06_01_*`, + UNNEST(getWorkboxMethods(JSON_EXTRACT(payload, '$._pwa.workboxInfo'))) AS workbox_method +JOIN + ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' + GROUP BY + _TABLE_SUFFIX + ) +USING (_TABLE_SUFFIX) +WHERE + JSON_EXTRACT(payload, '$._pwa.workboxInfo') != '[]' AND JSON_EXTRACT(payload, '$._pwa.workboxInfo') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + workbox_method, + total +ORDER BY + pct DESC, + client diff --git a/sql/2024/pwa/workbox_packages.sql b/sql/2024/pwa/workbox_packages.sql new file mode 100644 index 00000000000..27fa47ea3c7 --- /dev/null +++ b/sql/2024/pwa/workbox_packages.sql @@ -0,0 +1,60 @@ +#standardSQL +#Workbox package and methods +CREATE TEMPORARY FUNCTION getWorkboxPackages(workboxInfo STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + var workboxPackageMethods = Object.values(JSON.parse(workboxInfo)); + if (typeof workboxPackageMethods == 'string') { + workboxPackageMethods = [workboxPackageMethods]; + } + var workboxPackages = []; + /* Replacing spaces and commas */ + for (var i = 0; i < workboxPackageMethods.length; i++) { + var workboxItems = workboxPackageMethods[i].toString().trim().split(','); + for(var j = 0; j < workboxItems.length; j++) { + var workboxItem = workboxItems[j]; + var firstColonIndex = workboxItem.indexOf(':'); + if(firstColonIndex > -1) { + var workboxPackage = workboxItem.trim().substring(firstColonIndex + 1, workboxItem.indexOf(':', firstColonIndex + 1)); + workboxPackages.push(workboxPackage); + } + } + } + return Array.from(new Set(workboxPackages)); +} catch (e) { + return []; +} +'''; + +SELECT + _TABLE_SUFFIX AS client, + workbox_package, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.pages.2024_06_01_*`, + UNNEST(getWorkboxPackages(JSON_EXTRACT(payload, '$._pwa.workboxInfo'))) AS workbox_package +JOIN + ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' + GROUP BY + _TABLE_SUFFIX + ) +USING (_TABLE_SUFFIX) +WHERE + JSON_EXTRACT(payload, '$._pwa.workboxInfo') != '[]' AND JSON_EXTRACT(payload, '$._pwa.workboxInfo') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client, + workbox_package, + total +ORDER BY + pct DESC, + client diff --git a/sql/2024/pwa/workbox_usage.sql b/sql/2024/pwa/workbox_usage.sql new file mode 100644 index 00000000000..1fd1d82ea19 --- /dev/null +++ b/sql/2024/pwa/workbox_usage.sql @@ -0,0 +1,18 @@ +#standardSQL +# Workbox usage + +SELECT + _TABLE_SUFFIX AS client, + COUNTIF(JSON_EXTRACT(payload, '$._pwa.workboxInfo') != '[]' AND JSON_EXTRACT(payload, '$._pwa.workboxInfo') != '{}') AS freq, + SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNTIF(JSON_EXTRACT(payload, '$._pwa.workboxInfo') != '[]' AND JSON_EXTRACT(payload, '$._pwa.workboxInfo') != '{}') / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct +FROM + `httparchive.pages.2024_06_01_*` +WHERE + JSON_EXTRACT(payload, '$._pwa.manifests') != '[]' AND JSON_EXTRACT(payload, '$._pwa.manifests') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + client +ORDER BY + freq / total DESC, + client diff --git a/sql/2024/pwa/workbox_versions.sql b/sql/2024/pwa/workbox_versions.sql new file mode 100644 index 00000000000..595404f42f3 --- /dev/null +++ b/sql/2024/pwa/workbox_versions.sql @@ -0,0 +1,60 @@ +#standardSQL +#Workbox versions +CREATE TEMPORARY FUNCTION getWorkboxVersions(workboxInfo STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + var workboxPackageMethods = Object.values(JSON.parse(workboxInfo)); + if (typeof workboxPackageMethods == 'string') { + workboxPackageMethods = [workboxPackageMethods]; + } + var workboxVersions = []; + /* Replacing spaces and commas */ + for (var i = 0; i < workboxPackageMethods.length; i++) { + var workboxItems = workboxPackageMethods[i].toString().trim().split(','); + for(var j = 0; j < workboxItems.length; j++) { + var workboxItem = workboxItems[j]; + var firstColonIndex = workboxItem.indexOf(':'); + if(firstColonIndex > -1) { + var workboxVersion = workboxItem.trim().substring(workboxItem.indexOf(':', firstColonIndex + 1) + 1); + workboxVersions.push(workboxVersion); + } + } + } + return Array.from(new Set(workboxVersions)); +} catch (e) { + return []; +} +'''; + +SELECT + _TABLE_SUFFIX AS client, + workbox_version, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.pages.2024_06_01_*`, + UNNEST(getWorkboxVersions(JSON_EXTRACT(payload, '$._pwa.workboxInfo'))) AS workbox_version +JOIN + ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.pages.2024_06_01_*` + WHERE + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' + GROUP BY + _TABLE_SUFFIX + ) +USING (_TABLE_SUFFIX) +WHERE + JSON_EXTRACT(payload, '$._pwa.workboxInfo') != '[]' AND JSON_EXTRACT(payload, '$._pwa.workboxInfo') != '{}' AND + JSON_EXTRACT(payload, '$._pwa.serviceWorkerHeuristic') = 'true' +GROUP BY + _TABLE_SUFFIX, + workbox_version, + total +ORDER BY + pct DESC, + client