From 09c9330da8f80e94327cfb245cb198ce83d9128a Mon Sep 17 00:00:00 2001 From: Nurullah Date: Wed, 14 Aug 2024 19:05:20 +0200 Subject: [PATCH 01/77] Queries of CMS 2024 (replicated from 2022's version) --- sql/2024/cms/cms_adoption.sql | 153 ++ sql/2024/cms/cms_adoption_by_geo.sql | 47 + sql/2024/cms/cms_adoption_by_rank.sql | 52 + sql/2024/cms/cms_adoption_by_region.sql | 1244 +++++++++++++++++ sql/2024/cms/cms_adoption_by_subregion.sql | 1244 +++++++++++++++++ sql/2024/cms/core_web_vitals.sql | 58 + sql/2024/cms/core_web_vitals_by_geo.sql | 67 + sql/2024/cms/core_web_vitals_yoy.sql | 164 +++ sql/2024/cms/image_format_popularity.sql | 66 + .../lighthouse_category_scores_per_cms.sql | 34 + sql/2024/cms/page_weight_distribution.sql | 36 + sql/2024/cms/resource_weights.sql | 40 + sql/2024/cms/top_cms.sql | 108 ++ sql/2024/cms/top_cms_by_geo.sql | 68 + sql/2024/cms/top_cms_by_rank.sql | 52 + sql/2024/cms/wordpress_page_builders.sql | 35 + sql/2024/cms/wordpress_resources.sql | 38 + 17 files changed, 3506 insertions(+) create mode 100644 sql/2024/cms/cms_adoption.sql create mode 100644 sql/2024/cms/cms_adoption_by_geo.sql create mode 100644 sql/2024/cms/cms_adoption_by_rank.sql create mode 100644 sql/2024/cms/cms_adoption_by_region.sql create mode 100644 sql/2024/cms/cms_adoption_by_subregion.sql create mode 100644 sql/2024/cms/core_web_vitals.sql create mode 100644 sql/2024/cms/core_web_vitals_by_geo.sql create mode 100644 sql/2024/cms/core_web_vitals_yoy.sql create mode 100644 sql/2024/cms/image_format_popularity.sql create mode 100644 sql/2024/cms/lighthouse_category_scores_per_cms.sql create mode 100644 sql/2024/cms/page_weight_distribution.sql create mode 100644 sql/2024/cms/resource_weights.sql create mode 100644 sql/2024/cms/top_cms.sql create mode 100644 sql/2024/cms/top_cms_by_geo.sql create mode 100644 sql/2024/cms/top_cms_by_rank.sql create mode 100644 sql/2024/cms/wordpress_page_builders.sql create mode 100644 sql/2024/cms/wordpress_resources.sql diff --git a/sql/2024/cms/cms_adoption.sql b/sql/2024/cms/cms_adoption.sql new file mode 100644 index 00000000000..d219eb7352c --- /dev/null +++ b/sql/2024/cms/cms_adoption.sql @@ -0,0 +1,153 @@ +#standardSQL +# CMS adoption over time + +SELECT + _TABLE_SUFFIX AS client, + 2024 AS year, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.technologies.2024_06_01_*` +JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2024_06_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (_TABLE_SUFFIX) +WHERE + category = 'CMS' +GROUP BY + client, + total +UNION ALL + +SELECT + _TABLE_SUFFIX AS client, + 2023 AS year, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.technologies.2023_06_01_*` +JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2023_06_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (_TABLE_SUFFIX) +WHERE + category = 'CMS' +GROUP BY + client, + total +UNION ALL + +SELECT + _TABLE_SUFFIX AS client, + 2022 AS year, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.technologies.2022_06_01_*` +JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2022_06_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (_TABLE_SUFFIX) +WHERE + category = 'CMS' +GROUP BY + client, + total +UNION ALL + + +SELECT + _TABLE_SUFFIX AS client, + 2021 AS year, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.technologies.2021_07_01_*` +JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2021_07_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (_TABLE_SUFFIX) +WHERE + category = 'CMS' +GROUP BY + client, + total +UNION ALL +SELECT + _TABLE_SUFFIX AS client, + 2020 AS year, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.technologies.2020_08_01_*` +JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2020_08_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (_TABLE_SUFFIX) +WHERE + category = 'CMS' +GROUP BY + client, + total +UNION ALL +SELECT + _TABLE_SUFFIX AS client, + 2019 AS year, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.technologies.2019_07_01_*` +JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2019_07_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (_TABLE_SUFFIX) +WHERE + category = 'CMS' +GROUP BY + client, + total +ORDER BY + year DESC, + pct DESC diff --git a/sql/2024/cms/cms_adoption_by_geo.sql b/sql/2024/cms/cms_adoption_by_geo.sql new file mode 100644 index 00000000000..56c8b589f21 --- /dev/null +++ b/sql/2024/cms/cms_adoption_by_geo.sql @@ -0,0 +1,47 @@ +#standardSQL +# All CMS popularity per geo +WITH geo_summary AS ( + SELECT + `chrome-ux-report`.experimental.GET_COUNTRY(country_code) AS geo, + IF(device = 'desktop', 'desktop', 'mobile') AS client, + origin, + COUNT(DISTINCT origin) OVER (PARTITION BY country_code, IF(device = 'desktop', 'desktop', 'mobile')) AS total + FROM + `chrome-ux-report.materialized.country_summary` + WHERE + yyyymm = 202406 +) + +SELECT + * +FROM ( + SELECT + client, + geo, + COUNT(0) AS pages, + ANY_VALUE(total) AS total, + COUNT(0) / ANY_VALUE(total) AS pct + FROM ( + SELECT DISTINCT + geo, + client, + total, + CONCAT(origin, '/') AS url + FROM + geo_summary + ) JOIN ( + SELECT DISTINCT + _TABLE_SUFFIX AS client, + url + FROM + `httparchive.technologies.2024_06_01_*` + WHERE + category = 'CMS' + ) USING (client, url) + GROUP BY + client, + geo) +WHERE + pages > 1000 +ORDER BY + pages DESC diff --git a/sql/2024/cms/cms_adoption_by_rank.sql b/sql/2024/cms/cms_adoption_by_rank.sql new file mode 100644 index 00000000000..e416cf1b724 --- /dev/null +++ b/sql/2024/cms/cms_adoption_by_rank.sql @@ -0,0 +1,52 @@ +#standardSQL +# CMS adoption per rank +SELECT + client, + cms, + rank, + COUNT(DISTINCT url) AS pages, + ANY_VALUE(total) AS total, + COUNT(DISTINCT url) / ANY_VALUE(total) AS pct +FROM ( + SELECT DISTINCT + _TABLE_SUFFIX AS client, + app AS cms, + url + FROM + `httparchive.technologies.2024_06_01_*` + WHERE + category = 'CMS') +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + url, + rank + FROM + `httparchive.summary_pages.2024_06_01_*`) +USING + (client, url) +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + rank_magnitude AS rank, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2024_06_01_*`, + UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude + WHERE + rank <= rank_magnitude + GROUP BY + _TABLE_SUFFIX, + rank_magnitude) +USING + (client, rank), + UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude +WHERE + rank <= rank_magnitude +GROUP BY + client, + cms, + rank +ORDER BY + rank, + pages DESC diff --git a/sql/2024/cms/cms_adoption_by_region.sql b/sql/2024/cms/cms_adoption_by_region.sql new file mode 100644 index 00000000000..4baa0442d7d --- /dev/null +++ b/sql/2024/cms/cms_adoption_by_region.sql @@ -0,0 +1,1244 @@ +#standardSQL +# All CMS popularity per geo +CREATE TEMP FUNCTION GET_GEO(country_code STRING, geo STRING) RETURNS STRING LANGUAGE js AS ''' +var countries = { + "af": { + "name": "Afghanistan", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "ax": { + "name": "Åland Islands", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "al": { + "name": "Albania", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "dz": { + "name": "Algeria", + "region": "Africa", + "sub-region": "Northern Africa" + }, + "as": { + "name": "American Samoa", + "region": "Oceania", + "sub-region": "Polynesia" + }, + "ad": { + "name": "Andorra", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "ao": { + "name": "Angola", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "ai": { + "name": "Anguilla", + "region": "Americas", + "sub-region": "Caribbean" + }, + "ag": { + "name": "Antigua and Barbuda", + "region": "Americas", + "sub-region": "Caribbean" + }, + "ar": { + "name": "Argentina", + "region": "Americas", + "sub-region": "South America" + }, + "am": { + "name": "Armenia", + "region": "Asia", + "sub-region": "Western Asia" + }, + "aw": { + "name": "Aruba", + "region": "Americas", + "sub-region": "Caribbean" + }, + "au": { + "name": "Australia", + "region": "Oceania", + "sub-region": "Australia and New Zealand" + }, + "at": { + "name": "Austria", + "region": "Europe", + "sub-region": "Western Europe" + }, + "az": { + "name": "Azerbaijan", + "region": "Asia", + "sub-region": "Western Asia" + }, + "bs": { + "name": "Bahamas", + "region": "Americas", + "sub-region": "Caribbean" + }, + "bh": { + "name": "Bahrain", + "region": "Asia", + "sub-region": "Western Asia" + }, + "bd": { + "name": "Bangladesh", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "bb": { + "name": "Barbados", + "region": "Americas", + "sub-region": "Caribbean" + }, + "by": { + "name": "Belarus", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "be": { + "name": "Belgium", + "region": "Europe", + "sub-region": "Western Europe" + }, + "bz": { + "name": "Belize", + "region": "Americas", + "sub-region": "Central America" + }, + "bj": { + "name": "Benin", + "region": "Africa", + "sub-region": "Western Africa" + }, + "bm": { + "name": "Bermuda", + "region": "Americas", + "sub-region": "Northern America" + }, + "bt": { + "name": "Bhutan", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "bo": { + "name": "Bolivia (Plurinational State of)", + "region": "Americas", + "sub-region": "South America" + }, + "bq": { + "name": "Bonaire, Sint Eustatius and Saba", + "region": "Americas", + "sub-region": "Caribbean" + }, + "ba": { + "name": "Bosnia and Herzegovina", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "bw": { + "name": "Botswana", + "region": "Africa", + "sub-region": "Southern Africa" + }, + "br": { + "name": "Brazil", + "region": "Americas", + "sub-region": "South America" + }, + "io": { + "name": "British Indian Ocean Territory", + "region": null, + "sub-region": null + }, + "bn": { + "name": "Brunei Darussalam", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "bg": { + "name": "Kosovo", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "bf": { + "name": "Burkina Faso", + "region": "Africa", + "sub-region": "Western Africa" + }, + "bi": { + "name": "Burundi", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "kh": { + "name": "Cambodia", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "cm": { + "name": "Cameroon", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "ca": { + "name": "Canada", + "region": "Americas", + "sub-region": "Northern America" + }, + "cv": { + "name": "Cabo Verde", + "region": "Africa", + "sub-region": "Western Africa" + }, + "ky": { + "name": "Cayman Islands", + "region": "Americas", + "sub-region": "Caribbean" + }, + "cf": { + "name": "Central African Republic", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "td": { + "name": "Chad", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "cl": { + "name": "Chile", + "region": "Americas", + "sub-region": "South America" + }, + "cn": { + "name": "China", + "region": "Asia", + "sub-region": "Eastern Asia" + }, + "cx": { + "name": "Christmas Island", + "region": null, + "sub-region": null + }, + "co": { + "name": "Colombia", + "region": "Americas", + "sub-region": "South America" + }, + "km": { + "name": "Comoros", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "cg": { + "name": "Congo", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "cd": { + "name": "Congo (Democratic Republic of the)", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "ck": { + "name": "Cook Islands", + "region": "Oceania", + "sub-region": "Polynesia" + }, + "cr": { + "name": "Costa Rica", + "region": "Americas", + "sub-region": "Central America" + }, + "ci": { + "name": "Côte d'Ivoire", + "region": "Africa", + "sub-region": "Western Africa" + }, + "hr": { + "name": "Croatia", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "cu": { + "name": "Cuba", + "region": "Americas", + "sub-region": "Caribbean" + }, + "cw": { + "name": "Curaçao", + "region": "Americas", + "sub-region": "Caribbean" + }, + "cy": { + "name": "Cyprus", + "region": "Asia", + "sub-region": "Western Asia" + }, + "cz": { + "name": "Czech Republic", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "dk": { + "name": "Denmark", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "dj": { + "name": "Djibouti", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "dm": { + "name": "Dominica", + "region": "Americas", + "sub-region": "Caribbean" + }, + "do": { + "name": "Dominican Republic", + "region": "Americas", + "sub-region": "Caribbean" + }, + "ec": { + "name": "Ecuador", + "region": "Americas", + "sub-region": "South America" + }, + "eg": { + "name": "Egypt", + "region": "Africa", + "sub-region": "Northern Africa" + }, + "sv": { + "name": "El Salvador", + "region": "Americas", + "sub-region": "Central America" + }, + "gq": { + "name": "Equatorial Guinea", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "er": { + "name": "Eritrea", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "ee": { + "name": "Estonia", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "et": { + "name": "Ethiopia", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "fk": { + "name": "Falkland Islands (Malvinas)", + "region": "Americas", + "sub-region": "South America" + }, + "fo": { + "name": "Faroe Islands", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "fj": { + "name": "Fiji", + "region": "Oceania", + "sub-region": "Melanesia" + }, + "fi": { + "name": "Finland", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "fr": { + "name": "France", + "region": "Europe", + "sub-region": "Western Europe" + }, + "gf": { + "name": "French Guiana", + "region": "Americas", + "sub-region": "South America" + }, + "pf": { + "name": "French Polynesia", + "region": "Oceania", + "sub-region": "Polynesia" + }, + "ga": { + "name": "Gabon", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "gm": { + "name": "Gambia", + "region": "Africa", + "sub-region": "Western Africa" + }, + "ge": { + "name": "Georgia", + "region": "Asia", + "sub-region": "Western Asia" + }, + "de": { + "name": "Germany", + "region": "Europe", + "sub-region": "Western Europe" + }, + "gh": { + "name": "Ghana", + "region": "Africa", + "sub-region": "Western Africa" + }, + "gi": { + "name": "Gibraltar", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "gr": { + "name": "Greece", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "gl": { + "name": "Greenland", + "region": "Americas", + "sub-region": "Northern America" + }, + "gd": { + "name": "Grenada", + "region": "Americas", + "sub-region": "Caribbean" + }, + "gp": { + "name": "Guadeloupe", + "region": "Americas", + "sub-region": "Caribbean" + }, + "gu": { + "name": "Guam", + "region": "Oceania", + "sub-region": "Micronesia" + }, + "gt": { + "name": "Guatemala", + "region": "Americas", + "sub-region": "Central America" + }, + "gg": { + "name": "Guernsey", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "gn": { + "name": "Guinea", + "region": "Africa", + "sub-region": "Western Africa" + }, + "gw": { + "name": "Guinea-Bissau", + "region": "Africa", + "sub-region": "Western Africa" + }, + "gy": { + "name": "Guyana", + "region": "Americas", + "sub-region": "South America" + }, + "ht": { + "name": "Haiti", + "region": "Americas", + "sub-region": "Caribbean" + }, + "hn": { + "name": "Honduras", + "region": "Americas", + "sub-region": "Central America" + }, + "hk": { + "name": "Hong Kong", + "region": "Asia", + "sub-region": "Eastern Asia" + }, + "hu": { + "name": "Hungary", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "is": { + "name": "Iceland", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "in": { + "name": "India", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "id": { + "name": "Indonesia", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "ir": { + "name": "Iran (Islamic Republic of)", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "iq": { + "name": "Iraq", + "region": "Asia", + "sub-region": "Western Asia" + }, + "ie": { + "name": "Ireland", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "im": { + "name": "Isle of Man", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "il": { + "name": "Israel", + "region": "Asia", + "sub-region": "Western Asia" + }, + "it": { + "name": "Italy", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "jm": { + "name": "Jamaica", + "region": "Americas", + "sub-region": "Caribbean" + }, + "jp": { + "name": "Japan", + "region": "Asia", + "sub-region": "Eastern Asia" + }, + "je": { + "name": "Jersey", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "jo": { + "name": "Jordan", + "region": "Asia", + "sub-region": "Western Asia" + }, + "kz": { + "name": "Kazakhstan", + "region": "Asia", + "sub-region": "Central Asia" + }, + "ke": { + "name": "Kenya", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "ki": { + "name": "Kiribati", + "region": "Oceania", + "sub-region": "Micronesia" + }, + "kp": { + "name": "Korea (Democratic People's Republic of)", + "region": "Asia", + "sub-region": "Eastern Asia" + }, + "kr": { + "name": "Korea (Republic of)", + "region": "Asia", + "sub-region": "Eastern Asia" + }, + "kw": { + "name": "Kuwait", + "region": "Asia", + "sub-region": "Western Asia" + }, + "kg": { + "name": "Kyrgyzstan", + "region": "Asia", + "sub-region": "Central Asia" + }, + "la": { + "name": "Lao People's Democratic Republic", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "lv": { + "name": "Latvia", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "lb": { + "name": "Lebanon", + "region": "Asia", + "sub-region": "Western Asia" + }, + "ls": { + "name": "Lesotho", + "region": "Africa", + "sub-region": "Southern Africa" + }, + "lr": { + "name": "Liberia", + "region": "Africa", + "sub-region": "Western Africa" + }, + "ly": { + "name": "Libya", + "region": "Africa", + "sub-region": "Northern Africa" + }, + "li": { + "name": "Liechtenstein", + "region": "Europe", + "sub-region": "Western Europe" + }, + "lt": { + "name": "Lithuania", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "lu": { + "name": "Luxembourg", + "region": "Europe", + "sub-region": "Western Europe" + }, + "mo": { + "name": "Macao", + "region": "Asia", + "sub-region": "Eastern Asia" + }, + "mk": { + "name": "Macedonia (the former Yugoslav Republic of)", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "mg": { + "name": "Madagascar", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "mw": { + "name": "Malawi", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "my": { + "name": "Malaysia", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "mv": { + "name": "Maldives", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "ml": { + "name": "Mali", + "region": "Africa", + "sub-region": "Western Africa" + }, + "mt": { + "name": "Malta", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "mh": { + "name": "Marshall Islands", + "region": "Oceania", + "sub-region": "Micronesia" + }, + "mq": { + "name": "Martinique", + "region": "Americas", + "sub-region": "Caribbean" + }, + "mr": { + "name": "Mauritania", + "region": "Africa", + "sub-region": "Western Africa" + }, + "mu": { + "name": "Mauritius", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "yt": { + "name": "Mayotte", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "mx": { + "name": "Mexico", + "region": "Americas", + "sub-region": "Central America" + }, + "fm": { + "name": "Micronesia (Federated States of)", + "region": "Oceania", + "sub-region": "Micronesia" + }, + "md": { + "name": "Moldova (Republic of)", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "mc": { + "name": "Monaco", + "region": "Europe", + "sub-region": "Western Europe" + }, + "mn": { + "name": "Mongolia", + "region": "Asia", + "sub-region": "Eastern Asia" + }, + "me": { + "name": "Montenegro", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "ms": { + "name": "Montserrat", + "region": "Americas", + "sub-region": "Caribbean" + }, + "ma": { + "name": "Morocco", + "region": "Africa", + "sub-region": "Northern Africa" + }, + "mz": { + "name": "Mozambique", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "mm": { + "name": "Myanmar", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "na": { + "name": "Namibia", + "region": "Africa", + "sub-region": "Southern Africa" + }, + "nr": { + "name": "Nauru", + "region": "Oceania", + "sub-region": "Micronesia" + }, + "np": { + "name": "Nepal", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "nl": { + "name": "Netherlands", + "region": "Europe", + "sub-region": "Western Europe" + }, + "nc": { + "name": "New Caledonia", + "region": "Oceania", + "sub-region": "Melanesia" + }, + "nz": { + "name": "New Zealand", + "region": "Oceania", + "sub-region": "Australia and New Zealand" + }, + "ni": { + "name": "Nicaragua", + "region": "Americas", + "sub-region": "Central America" + }, + "ne": { + "name": "Niger", + "region": "Africa", + "sub-region": "Western Africa" + }, + "ng": { + "name": "Nigeria", + "region": "Africa", + "sub-region": "Western Africa" + }, + "nf": { + "name": "Norfolk Island", + "region": "Oceania", + "sub-region": "Australia and New Zealand" + }, + "mp": { + "name": "Northern Mariana Islands", + "region": "Oceania", + "sub-region": "Micronesia" + }, + "no": { + "name": "Norway", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "om": { + "name": "Oman", + "region": "Asia", + "sub-region": "Western Asia" + }, + "pk": { + "name": "Pakistan", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "pw": { + "name": "Palau", + "region": "Oceania", + "sub-region": "Micronesia" + }, + "ps": { + "name": "Palestine, State of", + "region": "Asia", + "sub-region": "Western Asia" + }, + "pa": { + "name": "Panama", + "region": "Americas", + "sub-region": "Central America" + }, + "pg": { + "name": "Papua New Guinea", + "region": "Oceania", + "sub-region": "Melanesia" + }, + "py": { + "name": "Paraguay", + "region": "Americas", + "sub-region": "South America" + }, + "pe": { + "name": "Peru", + "region": "Americas", + "sub-region": "South America" + }, + "ph": { + "name": "Philippines", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "pl": { + "name": "Poland", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "pt": { + "name": "Portugal", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "pr": { + "name": "Puerto Rico", + "region": "Americas", + "sub-region": "Caribbean" + }, + "qa": { + "name": "Qatar", + "region": "Asia", + "sub-region": "Western Asia" + }, + "re": { + "name": "Réunion", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "ro": { + "name": "Romania", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "ru": { + "name": "Russian Federation", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "rw": { + "name": "Rwanda", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "bl": { + "name": "Saint Barthélemy", + "region": "Americas", + "sub-region": "Caribbean" + }, + "sh": { + "name": "Saint Helena, Ascension and Tristan da Cunha", + "region": "Africa", + "sub-region": "Western Africa" + }, + "kn": { + "name": "Saint Kitts and Nevis", + "region": "Americas", + "sub-region": "Caribbean" + }, + "lc": { + "name": "Saint Lucia", + "region": "Americas", + "sub-region": "Caribbean" + }, + "mf": { + "name": "Saint Martin (French part)", + "region": "Americas", + "sub-region": "Caribbean" + }, + "pm": { + "name": "Saint Pierre and Miquelon", + "region": "Americas", + "sub-region": "Northern America" + }, + "vc": { + "name": "Saint Vincent and the Grenadines", + "region": "Americas", + "sub-region": "Caribbean" + }, + "ws": { + "name": "Samoa", + "region": "Oceania", + "sub-region": "Polynesia" + }, + "sm": { + "name": "San Marino", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "st": { + "name": "Sao Tome and Principe", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "sa": { + "name": "Saudi Arabia", + "region": "Asia", + "sub-region": "Western Asia" + }, + "sn": { + "name": "Senegal", + "region": "Africa", + "sub-region": "Western Africa" + }, + "rs": { + "name": "Serbia", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "sc": { + "name": "Seychelles", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "sl": { + "name": "Sierra Leone", + "region": "Africa", + "sub-region": "Western Africa" + }, + "sg": { + "name": "Singapore", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "sx": { + "name": "Sint Maarten (Dutch part)", + "region": "Americas", + "sub-region": "Caribbean" + }, + "sk": { + "name": "Slovakia", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "si": { + "name": "Slovenia", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "sb": { + "name": "Solomon Islands", + "region": "Oceania", + "sub-region": "Melanesia" + }, + "so": { + "name": "Somalia", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "za": { + "name": "South Africa", + "region": "Africa", + "sub-region": "Southern Africa" + }, + "ss": { + "name": "South Sudan", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "es": { + "name": "Spain", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "lk": { + "name": "Sri Lanka", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "sd": { + "name": "Sudan", + "region": "Africa", + "sub-region": "Northern Africa" + }, + "sr": { + "name": "Suriname", + "region": "Americas", + "sub-region": "South America" + }, + "sj": { + "name": "Svalbard and Jan Mayen", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "sz": { + "name": "Swaziland", + "region": "Africa", + "sub-region": "Southern Africa" + }, + "se": { + "name": "Sweden", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "ch": { + "name": "Switzerland", + "region": "Europe", + "sub-region": "Western Europe" + }, + "sy": { + "name": "Syrian Arab Republic", + "region": "Asia", + "sub-region": "Western Asia" + }, + "tw": { + "name": "Taiwan, Province of China", + "region": "Asia", + "sub-region": "Eastern Asia" + }, + "tj": { + "name": "Tajikistan", + "region": "Asia", + "sub-region": "Central Asia" + }, + "tz": { + "name": "Tanzania, United Republic of", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "th": { + "name": "Thailand", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "tl": { + "name": "Timor-Leste", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "tg": { + "name": "Togo", + "region": "Africa", + "sub-region": "Western Africa" + }, + "to": { + "name": "Tonga", + "region": "Oceania", + "sub-region": "Polynesia" + }, + "tt": { + "name": "Trinidad and Tobago", + "region": "Americas", + "sub-region": "Caribbean" + }, + "tn": { + "name": "Tunisia", + "region": "Africa", + "sub-region": "Northern Africa" + }, + "tr": { + "name": "Turkey", + "region": "Asia", + "sub-region": "Western Asia" + }, + "tm": { + "name": "Turkmenistan", + "region": "Asia", + "sub-region": "Central Asia" + }, + "tc": { + "name": "Turks and Caicos Islands", + "region": "Americas", + "sub-region": "Caribbean" + }, + "tv": { + "name": "Tuvalu", + "region": "Oceania", + "sub-region": "Polynesia" + }, + "ug": { + "name": "Uganda", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "ua": { + "name": "Ukraine", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "ae": { + "name": "United Arab Emirates", + "region": "Asia", + "sub-region": "Western Asia" + }, + "gb": { + "name": "United Kingdom of Great Britain and Northern Ireland", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "us": { + "name": "United States of America", + "region": "Americas", + "sub-region": "Northern America" + }, + "uy": { + "name": "Uruguay", + "region": "Americas", + "sub-region": "South America" + }, + "uz": { + "name": "Uzbekistan", + "region": "Asia", + "sub-region": "Central Asia" + }, + "vu": { + "name": "Vanuatu", + "region": "Oceania", + "sub-region": "Melanesia" + }, + "ve": { + "name": "Venezuela (Bolivarian Republic of)", + "region": "Americas", + "sub-region": "South America" + }, + "vn": { + "name": "Viet Nam", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "vg": { + "name": "Virgin Islands (British)", + "region": "Americas", + "sub-region": "Caribbean" + }, + "vi": { + "name": "Virgin Islands (U.S.)", + "region": "Americas", + "sub-region": "Caribbean" + }, + "eh": { + "name": "Western Sahara", + "region": "Africa", + "sub-region": "Northern Africa" + }, + "ye": { + "name": "Yemen", + "region": "Asia", + "sub-region": "Western Asia" + }, + "zm": { + "name": "Zambia", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "zw": { + "name": "Zimbabwe", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "xk": { + "name": "Kosovo", + "region": "Europe", + "sub-region": "Eastern Europe" + } +}; + +return countries[country_code][geo]; +'''; + +WITH geo_summary AS ( + SELECT + GET_GEO(country_code, 'region') AS region, + IF(device = 'desktop', 'desktop', 'mobile') AS client, + origin, + COUNT(DISTINCT origin) OVER (PARTITION BY GET_GEO(country_code, 'region'), IF(device = 'desktop', 'desktop', 'mobile')) AS total + FROM + `chrome-ux-report.materialized.country_summary` + WHERE + yyyymm = 202406 +) + +SELECT + * +FROM ( + SELECT + client, + region, + COUNT(0) AS pages, + ANY_VALUE(total) AS total, + COUNT(0) / ANY_VALUE(total) AS pct + FROM ( + SELECT DISTINCT + region, + client, + total, + CONCAT(origin, '/') AS url + FROM + geo_summary + ) JOIN ( + SELECT DISTINCT + _TABLE_SUFFIX AS client, + url + FROM + `httparchive.technologies.2024_06_01_*` + WHERE + category = 'CMS' + ) USING (client, url) + GROUP BY + client, + region) +WHERE + pages > 1000 +ORDER BY + pages DESC diff --git a/sql/2024/cms/cms_adoption_by_subregion.sql b/sql/2024/cms/cms_adoption_by_subregion.sql new file mode 100644 index 00000000000..ec9fdf141e4 --- /dev/null +++ b/sql/2024/cms/cms_adoption_by_subregion.sql @@ -0,0 +1,1244 @@ +#standardSQL +# All CMS popularity per geo +CREATE TEMP FUNCTION GET_GEO(country_code STRING, geo STRING) RETURNS STRING LANGUAGE js AS ''' +var countries = { + "af": { + "name": "Afghanistan", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "ax": { + "name": "Åland Islands", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "al": { + "name": "Albania", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "dz": { + "name": "Algeria", + "region": "Africa", + "sub-region": "Northern Africa" + }, + "as": { + "name": "American Samoa", + "region": "Oceania", + "sub-region": "Polynesia" + }, + "ad": { + "name": "Andorra", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "ao": { + "name": "Angola", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "ai": { + "name": "Anguilla", + "region": "Americas", + "sub-region": "Caribbean" + }, + "ag": { + "name": "Antigua and Barbuda", + "region": "Americas", + "sub-region": "Caribbean" + }, + "ar": { + "name": "Argentina", + "region": "Americas", + "sub-region": "South America" + }, + "am": { + "name": "Armenia", + "region": "Asia", + "sub-region": "Western Asia" + }, + "aw": { + "name": "Aruba", + "region": "Americas", + "sub-region": "Caribbean" + }, + "au": { + "name": "Australia", + "region": "Oceania", + "sub-region": "Australia and New Zealand" + }, + "at": { + "name": "Austria", + "region": "Europe", + "sub-region": "Western Europe" + }, + "az": { + "name": "Azerbaijan", + "region": "Asia", + "sub-region": "Western Asia" + }, + "bs": { + "name": "Bahamas", + "region": "Americas", + "sub-region": "Caribbean" + }, + "bh": { + "name": "Bahrain", + "region": "Asia", + "sub-region": "Western Asia" + }, + "bd": { + "name": "Bangladesh", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "bb": { + "name": "Barbados", + "region": "Americas", + "sub-region": "Caribbean" + }, + "by": { + "name": "Belarus", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "be": { + "name": "Belgium", + "region": "Europe", + "sub-region": "Western Europe" + }, + "bz": { + "name": "Belize", + "region": "Americas", + "sub-region": "Central America" + }, + "bj": { + "name": "Benin", + "region": "Africa", + "sub-region": "Western Africa" + }, + "bm": { + "name": "Bermuda", + "region": "Americas", + "sub-region": "Northern America" + }, + "bt": { + "name": "Bhutan", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "bo": { + "name": "Bolivia (Plurinational State of)", + "region": "Americas", + "sub-region": "South America" + }, + "bq": { + "name": "Bonaire, Sint Eustatius and Saba", + "region": "Americas", + "sub-region": "Caribbean" + }, + "ba": { + "name": "Bosnia and Herzegovina", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "bw": { + "name": "Botswana", + "region": "Africa", + "sub-region": "Southern Africa" + }, + "br": { + "name": "Brazil", + "region": "Americas", + "sub-region": "South America" + }, + "io": { + "name": "British Indian Ocean Territory", + "region": null, + "sub-region": null + }, + "bn": { + "name": "Brunei Darussalam", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "bg": { + "name": "Kosovo", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "bf": { + "name": "Burkina Faso", + "region": "Africa", + "sub-region": "Western Africa" + }, + "bi": { + "name": "Burundi", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "kh": { + "name": "Cambodia", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "cm": { + "name": "Cameroon", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "ca": { + "name": "Canada", + "region": "Americas", + "sub-region": "Northern America" + }, + "cv": { + "name": "Cabo Verde", + "region": "Africa", + "sub-region": "Western Africa" + }, + "ky": { + "name": "Cayman Islands", + "region": "Americas", + "sub-region": "Caribbean" + }, + "cf": { + "name": "Central African Republic", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "td": { + "name": "Chad", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "cl": { + "name": "Chile", + "region": "Americas", + "sub-region": "South America" + }, + "cn": { + "name": "China", + "region": "Asia", + "sub-region": "Eastern Asia" + }, + "cx": { + "name": "Christmas Island", + "region": null, + "sub-region": null + }, + "co": { + "name": "Colombia", + "region": "Americas", + "sub-region": "South America" + }, + "km": { + "name": "Comoros", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "cg": { + "name": "Congo", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "cd": { + "name": "Congo (Democratic Republic of the)", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "ck": { + "name": "Cook Islands", + "region": "Oceania", + "sub-region": "Polynesia" + }, + "cr": { + "name": "Costa Rica", + "region": "Americas", + "sub-region": "Central America" + }, + "ci": { + "name": "Côte d'Ivoire", + "region": "Africa", + "sub-region": "Western Africa" + }, + "hr": { + "name": "Croatia", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "cu": { + "name": "Cuba", + "region": "Americas", + "sub-region": "Caribbean" + }, + "cw": { + "name": "Curaçao", + "region": "Americas", + "sub-region": "Caribbean" + }, + "cy": { + "name": "Cyprus", + "region": "Asia", + "sub-region": "Western Asia" + }, + "cz": { + "name": "Czech Republic", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "dk": { + "name": "Denmark", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "dj": { + "name": "Djibouti", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "dm": { + "name": "Dominica", + "region": "Americas", + "sub-region": "Caribbean" + }, + "do": { + "name": "Dominican Republic", + "region": "Americas", + "sub-region": "Caribbean" + }, + "ec": { + "name": "Ecuador", + "region": "Americas", + "sub-region": "South America" + }, + "eg": { + "name": "Egypt", + "region": "Africa", + "sub-region": "Northern Africa" + }, + "sv": { + "name": "El Salvador", + "region": "Americas", + "sub-region": "Central America" + }, + "gq": { + "name": "Equatorial Guinea", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "er": { + "name": "Eritrea", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "ee": { + "name": "Estonia", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "et": { + "name": "Ethiopia", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "fk": { + "name": "Falkland Islands (Malvinas)", + "region": "Americas", + "sub-region": "South America" + }, + "fo": { + "name": "Faroe Islands", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "fj": { + "name": "Fiji", + "region": "Oceania", + "sub-region": "Melanesia" + }, + "fi": { + "name": "Finland", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "fr": { + "name": "France", + "region": "Europe", + "sub-region": "Western Europe" + }, + "gf": { + "name": "French Guiana", + "region": "Americas", + "sub-region": "South America" + }, + "pf": { + "name": "French Polynesia", + "region": "Oceania", + "sub-region": "Polynesia" + }, + "ga": { + "name": "Gabon", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "gm": { + "name": "Gambia", + "region": "Africa", + "sub-region": "Western Africa" + }, + "ge": { + "name": "Georgia", + "region": "Asia", + "sub-region": "Western Asia" + }, + "de": { + "name": "Germany", + "region": "Europe", + "sub-region": "Western Europe" + }, + "gh": { + "name": "Ghana", + "region": "Africa", + "sub-region": "Western Africa" + }, + "gi": { + "name": "Gibraltar", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "gr": { + "name": "Greece", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "gl": { + "name": "Greenland", + "region": "Americas", + "sub-region": "Northern America" + }, + "gd": { + "name": "Grenada", + "region": "Americas", + "sub-region": "Caribbean" + }, + "gp": { + "name": "Guadeloupe", + "region": "Americas", + "sub-region": "Caribbean" + }, + "gu": { + "name": "Guam", + "region": "Oceania", + "sub-region": "Micronesia" + }, + "gt": { + "name": "Guatemala", + "region": "Americas", + "sub-region": "Central America" + }, + "gg": { + "name": "Guernsey", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "gn": { + "name": "Guinea", + "region": "Africa", + "sub-region": "Western Africa" + }, + "gw": { + "name": "Guinea-Bissau", + "region": "Africa", + "sub-region": "Western Africa" + }, + "gy": { + "name": "Guyana", + "region": "Americas", + "sub-region": "South America" + }, + "ht": { + "name": "Haiti", + "region": "Americas", + "sub-region": "Caribbean" + }, + "hn": { + "name": "Honduras", + "region": "Americas", + "sub-region": "Central America" + }, + "hk": { + "name": "Hong Kong", + "region": "Asia", + "sub-region": "Eastern Asia" + }, + "hu": { + "name": "Hungary", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "is": { + "name": "Iceland", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "in": { + "name": "India", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "id": { + "name": "Indonesia", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "ir": { + "name": "Iran (Islamic Republic of)", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "iq": { + "name": "Iraq", + "region": "Asia", + "sub-region": "Western Asia" + }, + "ie": { + "name": "Ireland", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "im": { + "name": "Isle of Man", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "il": { + "name": "Israel", + "region": "Asia", + "sub-region": "Western Asia" + }, + "it": { + "name": "Italy", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "jm": { + "name": "Jamaica", + "region": "Americas", + "sub-region": "Caribbean" + }, + "jp": { + "name": "Japan", + "region": "Asia", + "sub-region": "Eastern Asia" + }, + "je": { + "name": "Jersey", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "jo": { + "name": "Jordan", + "region": "Asia", + "sub-region": "Western Asia" + }, + "kz": { + "name": "Kazakhstan", + "region": "Asia", + "sub-region": "Central Asia" + }, + "ke": { + "name": "Kenya", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "ki": { + "name": "Kiribati", + "region": "Oceania", + "sub-region": "Micronesia" + }, + "kp": { + "name": "Korea (Democratic People's Republic of)", + "region": "Asia", + "sub-region": "Eastern Asia" + }, + "kr": { + "name": "Korea (Republic of)", + "region": "Asia", + "sub-region": "Eastern Asia" + }, + "kw": { + "name": "Kuwait", + "region": "Asia", + "sub-region": "Western Asia" + }, + "kg": { + "name": "Kyrgyzstan", + "region": "Asia", + "sub-region": "Central Asia" + }, + "la": { + "name": "Lao People's Democratic Republic", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "lv": { + "name": "Latvia", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "lb": { + "name": "Lebanon", + "region": "Asia", + "sub-region": "Western Asia" + }, + "ls": { + "name": "Lesotho", + "region": "Africa", + "sub-region": "Southern Africa" + }, + "lr": { + "name": "Liberia", + "region": "Africa", + "sub-region": "Western Africa" + }, + "ly": { + "name": "Libya", + "region": "Africa", + "sub-region": "Northern Africa" + }, + "li": { + "name": "Liechtenstein", + "region": "Europe", + "sub-region": "Western Europe" + }, + "lt": { + "name": "Lithuania", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "lu": { + "name": "Luxembourg", + "region": "Europe", + "sub-region": "Western Europe" + }, + "mo": { + "name": "Macao", + "region": "Asia", + "sub-region": "Eastern Asia" + }, + "mk": { + "name": "Macedonia (the former Yugoslav Republic of)", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "mg": { + "name": "Madagascar", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "mw": { + "name": "Malawi", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "my": { + "name": "Malaysia", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "mv": { + "name": "Maldives", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "ml": { + "name": "Mali", + "region": "Africa", + "sub-region": "Western Africa" + }, + "mt": { + "name": "Malta", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "mh": { + "name": "Marshall Islands", + "region": "Oceania", + "sub-region": "Micronesia" + }, + "mq": { + "name": "Martinique", + "region": "Americas", + "sub-region": "Caribbean" + }, + "mr": { + "name": "Mauritania", + "region": "Africa", + "sub-region": "Western Africa" + }, + "mu": { + "name": "Mauritius", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "yt": { + "name": "Mayotte", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "mx": { + "name": "Mexico", + "region": "Americas", + "sub-region": "Central America" + }, + "fm": { + "name": "Micronesia (Federated States of)", + "region": "Oceania", + "sub-region": "Micronesia" + }, + "md": { + "name": "Moldova (Republic of)", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "mc": { + "name": "Monaco", + "region": "Europe", + "sub-region": "Western Europe" + }, + "mn": { + "name": "Mongolia", + "region": "Asia", + "sub-region": "Eastern Asia" + }, + "me": { + "name": "Montenegro", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "ms": { + "name": "Montserrat", + "region": "Americas", + "sub-region": "Caribbean" + }, + "ma": { + "name": "Morocco", + "region": "Africa", + "sub-region": "Northern Africa" + }, + "mz": { + "name": "Mozambique", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "mm": { + "name": "Myanmar", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "na": { + "name": "Namibia", + "region": "Africa", + "sub-region": "Southern Africa" + }, + "nr": { + "name": "Nauru", + "region": "Oceania", + "sub-region": "Micronesia" + }, + "np": { + "name": "Nepal", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "nl": { + "name": "Netherlands", + "region": "Europe", + "sub-region": "Western Europe" + }, + "nc": { + "name": "New Caledonia", + "region": "Oceania", + "sub-region": "Melanesia" + }, + "nz": { + "name": "New Zealand", + "region": "Oceania", + "sub-region": "Australia and New Zealand" + }, + "ni": { + "name": "Nicaragua", + "region": "Americas", + "sub-region": "Central America" + }, + "ne": { + "name": "Niger", + "region": "Africa", + "sub-region": "Western Africa" + }, + "ng": { + "name": "Nigeria", + "region": "Africa", + "sub-region": "Western Africa" + }, + "nf": { + "name": "Norfolk Island", + "region": "Oceania", + "sub-region": "Australia and New Zealand" + }, + "mp": { + "name": "Northern Mariana Islands", + "region": "Oceania", + "sub-region": "Micronesia" + }, + "no": { + "name": "Norway", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "om": { + "name": "Oman", + "region": "Asia", + "sub-region": "Western Asia" + }, + "pk": { + "name": "Pakistan", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "pw": { + "name": "Palau", + "region": "Oceania", + "sub-region": "Micronesia" + }, + "ps": { + "name": "Palestine, State of", + "region": "Asia", + "sub-region": "Western Asia" + }, + "pa": { + "name": "Panama", + "region": "Americas", + "sub-region": "Central America" + }, + "pg": { + "name": "Papua New Guinea", + "region": "Oceania", + "sub-region": "Melanesia" + }, + "py": { + "name": "Paraguay", + "region": "Americas", + "sub-region": "South America" + }, + "pe": { + "name": "Peru", + "region": "Americas", + "sub-region": "South America" + }, + "ph": { + "name": "Philippines", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "pl": { + "name": "Poland", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "pt": { + "name": "Portugal", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "pr": { + "name": "Puerto Rico", + "region": "Americas", + "sub-region": "Caribbean" + }, + "qa": { + "name": "Qatar", + "region": "Asia", + "sub-region": "Western Asia" + }, + "re": { + "name": "Réunion", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "ro": { + "name": "Romania", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "ru": { + "name": "Russian Federation", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "rw": { + "name": "Rwanda", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "bl": { + "name": "Saint Barthélemy", + "region": "Americas", + "sub-region": "Caribbean" + }, + "sh": { + "name": "Saint Helena, Ascension and Tristan da Cunha", + "region": "Africa", + "sub-region": "Western Africa" + }, + "kn": { + "name": "Saint Kitts and Nevis", + "region": "Americas", + "sub-region": "Caribbean" + }, + "lc": { + "name": "Saint Lucia", + "region": "Americas", + "sub-region": "Caribbean" + }, + "mf": { + "name": "Saint Martin (French part)", + "region": "Americas", + "sub-region": "Caribbean" + }, + "pm": { + "name": "Saint Pierre and Miquelon", + "region": "Americas", + "sub-region": "Northern America" + }, + "vc": { + "name": "Saint Vincent and the Grenadines", + "region": "Americas", + "sub-region": "Caribbean" + }, + "ws": { + "name": "Samoa", + "region": "Oceania", + "sub-region": "Polynesia" + }, + "sm": { + "name": "San Marino", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "st": { + "name": "Sao Tome and Principe", + "region": "Africa", + "sub-region": "Middle Africa" + }, + "sa": { + "name": "Saudi Arabia", + "region": "Asia", + "sub-region": "Western Asia" + }, + "sn": { + "name": "Senegal", + "region": "Africa", + "sub-region": "Western Africa" + }, + "rs": { + "name": "Serbia", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "sc": { + "name": "Seychelles", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "sl": { + "name": "Sierra Leone", + "region": "Africa", + "sub-region": "Western Africa" + }, + "sg": { + "name": "Singapore", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "sx": { + "name": "Sint Maarten (Dutch part)", + "region": "Americas", + "sub-region": "Caribbean" + }, + "sk": { + "name": "Slovakia", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "si": { + "name": "Slovenia", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "sb": { + "name": "Solomon Islands", + "region": "Oceania", + "sub-region": "Melanesia" + }, + "so": { + "name": "Somalia", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "za": { + "name": "South Africa", + "region": "Africa", + "sub-region": "Southern Africa" + }, + "ss": { + "name": "South Sudan", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "es": { + "name": "Spain", + "region": "Europe", + "sub-region": "Southern Europe" + }, + "lk": { + "name": "Sri Lanka", + "region": "Asia", + "sub-region": "Southern Asia" + }, + "sd": { + "name": "Sudan", + "region": "Africa", + "sub-region": "Northern Africa" + }, + "sr": { + "name": "Suriname", + "region": "Americas", + "sub-region": "South America" + }, + "sj": { + "name": "Svalbard and Jan Mayen", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "sz": { + "name": "Swaziland", + "region": "Africa", + "sub-region": "Southern Africa" + }, + "se": { + "name": "Sweden", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "ch": { + "name": "Switzerland", + "region": "Europe", + "sub-region": "Western Europe" + }, + "sy": { + "name": "Syrian Arab Republic", + "region": "Asia", + "sub-region": "Western Asia" + }, + "tw": { + "name": "Taiwan, Province of China", + "region": "Asia", + "sub-region": "Eastern Asia" + }, + "tj": { + "name": "Tajikistan", + "region": "Asia", + "sub-region": "Central Asia" + }, + "tz": { + "name": "Tanzania, United Republic of", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "th": { + "name": "Thailand", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "tl": { + "name": "Timor-Leste", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "tg": { + "name": "Togo", + "region": "Africa", + "sub-region": "Western Africa" + }, + "to": { + "name": "Tonga", + "region": "Oceania", + "sub-region": "Polynesia" + }, + "tt": { + "name": "Trinidad and Tobago", + "region": "Americas", + "sub-region": "Caribbean" + }, + "tn": { + "name": "Tunisia", + "region": "Africa", + "sub-region": "Northern Africa" + }, + "tr": { + "name": "Turkey", + "region": "Asia", + "sub-region": "Western Asia" + }, + "tm": { + "name": "Turkmenistan", + "region": "Asia", + "sub-region": "Central Asia" + }, + "tc": { + "name": "Turks and Caicos Islands", + "region": "Americas", + "sub-region": "Caribbean" + }, + "tv": { + "name": "Tuvalu", + "region": "Oceania", + "sub-region": "Polynesia" + }, + "ug": { + "name": "Uganda", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "ua": { + "name": "Ukraine", + "region": "Europe", + "sub-region": "Eastern Europe" + }, + "ae": { + "name": "United Arab Emirates", + "region": "Asia", + "sub-region": "Western Asia" + }, + "gb": { + "name": "United Kingdom of Great Britain and Northern Ireland", + "region": "Europe", + "sub-region": "Northern Europe" + }, + "us": { + "name": "United States of America", + "region": "Americas", + "sub-region": "Northern America" + }, + "uy": { + "name": "Uruguay", + "region": "Americas", + "sub-region": "South America" + }, + "uz": { + "name": "Uzbekistan", + "region": "Asia", + "sub-region": "Central Asia" + }, + "vu": { + "name": "Vanuatu", + "region": "Oceania", + "sub-region": "Melanesia" + }, + "ve": { + "name": "Venezuela (Bolivarian Republic of)", + "region": "Americas", + "sub-region": "South America" + }, + "vn": { + "name": "Viet Nam", + "region": "Asia", + "sub-region": "South-Eastern Asia" + }, + "vg": { + "name": "Virgin Islands (British)", + "region": "Americas", + "sub-region": "Caribbean" + }, + "vi": { + "name": "Virgin Islands (U.S.)", + "region": "Americas", + "sub-region": "Caribbean" + }, + "eh": { + "name": "Western Sahara", + "region": "Africa", + "sub-region": "Northern Africa" + }, + "ye": { + "name": "Yemen", + "region": "Asia", + "sub-region": "Western Asia" + }, + "zm": { + "name": "Zambia", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "zw": { + "name": "Zimbabwe", + "region": "Africa", + "sub-region": "Eastern Africa" + }, + "xk": { + "name": "Kosovo", + "region": "Europe", + "sub-region": "Eastern Europe" + } +}; + +return countries[country_code][geo]; +'''; + +WITH geo_summary AS ( + SELECT + GET_GEO(country_code, 'sub-region') AS sub_region, + IF(device = 'desktop', 'desktop', 'mobile') AS client, + origin, + COUNT(DISTINCT origin) OVER (PARTITION BY GET_GEO(country_code, 'sub-region'), IF(device = 'desktop', 'desktop', 'mobile')) AS total + FROM + `chrome-ux-report.materialized.country_summary` + WHERE + yyyymm = 202406 +) + +SELECT + * +FROM ( + SELECT + client, + sub_region, + COUNT(0) AS pages, + ANY_VALUE(total) AS total, + COUNT(0) / ANY_VALUE(total) AS pct + FROM ( + SELECT DISTINCT + sub_region, + client, + total, + CONCAT(origin, '/') AS url + FROM + geo_summary + ) JOIN ( + SELECT DISTINCT + _TABLE_SUFFIX AS client, + url + FROM + `httparchive.technologies.2024_06_01_*` + WHERE + category = 'CMS' + ) USING (client, url) + GROUP BY + client, + sub_region) +WHERE + pages > 1000 +ORDER BY + pages DESC diff --git a/sql/2024/cms/core_web_vitals.sql b/sql/2024/cms/core_web_vitals.sql new file mode 100644 index 00000000000..713b6840674 --- /dev/null +++ b/sql/2024/cms/core_web_vitals.sql @@ -0,0 +1,58 @@ +# cms passing core web vitals +CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + good / (good + needs_improvement + poor) >= 0.75 +); + +CREATE TEMP FUNCTION IS_NON_ZERO (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + good + needs_improvement + poor > 0 +); + +SELECT + client, + cms, + COUNT(DISTINCT origin) AS origins, + # Origins with good LCP divided by origins with any LCP. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_good_lcp, + + # Origins with good FID divided by origins with any FID. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_good_fid, + + # Origins with good CLS divided by origins with any CLS. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_good_cls, + + # Origins with good LCP, FID (optional), and CLS divided by origins with any LCP and CLS. + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND + IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_good_cwv +FROM + `chrome-ux-report.materialized.device_summary` +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + url, + app AS cms + FROM + `httparchive.technologies.2024_06_01_*` + WHERE + category = 'CMS') +ON + CONCAT(origin, '/') = url AND + IF(device = 'desktop', 'desktop', 'mobile') = client +WHERE + date = '2024-06-01' +GROUP BY + client, + cms +ORDER BY + origins DESC diff --git a/sql/2024/cms/core_web_vitals_by_geo.sql b/sql/2024/cms/core_web_vitals_by_geo.sql new file mode 100644 index 00000000000..d2aa29de0cc --- /dev/null +++ b/sql/2024/cms/core_web_vitals_by_geo.sql @@ -0,0 +1,67 @@ +# cms passing core web vitals +CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + good / (good + needs_improvement + poor) >= 0.75 +); + +CREATE TEMP FUNCTION IS_NON_ZERO (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + good + needs_improvement + poor > 0 +); + +SELECT + client, + `chrome-ux-report`.experimental.GET_COUNTRY(country_code) AS geo, + cms, + COUNT(DISTINCT origin) AS origins, + # Origins with good LCP divided by origins with any LCP. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_good_lcp, + + # Origins with good FID divided by origins with any FID. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_good_fid, + + # Origins with good CLS divided by origins with any CLS. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_good_cls, + + # Origins with good LCP, FID (optional), and CLS divided by origins with any LCP and CLS. FID is optional! + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND + IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_good_cwv +FROM ( + SELECT + *, + CONCAT(origin, '/') AS url, + IF(device = 'desktop', 'desktop', 'mobile') AS client + FROM + `chrome-ux-report.materialized.country_summary` + WHERE + yyyymm = 202406 AND + device IN ('desktop', 'phone')) +JOIN ( + SELECT DISTINCT + _TABLE_SUFFIX AS client, + url, + app AS cms + FROM + `httparchive.technologies.2024_06_01_*` + WHERE + category = 'CMS') +USING + (client, url) +GROUP BY + client, + geo, + cms +HAVING + origins > 1000 +ORDER BY + origins DESC diff --git a/sql/2024/cms/core_web_vitals_yoy.sql b/sql/2024/cms/core_web_vitals_yoy.sql new file mode 100644 index 00000000000..aa49e7d54a0 --- /dev/null +++ b/sql/2024/cms/core_web_vitals_yoy.sql @@ -0,0 +1,164 @@ +# cms passing core web vitals +CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + good / (good + needs_improvement + poor) >= 0.75 +); + +CREATE TEMP FUNCTION IS_NON_ZERO (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + good + needs_improvement + poor > 0 +); + +SELECT + 2024 AS year, + client, + cms, + COUNT(DISTINCT origin) AS origins, + # Origins with good LCP divided by origins with any LCP. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_good_lcp, + + # Origins with good FID divided by origins with any FID. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_good_fid, + + # Origins with good CLS divided by origins with any CLS. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_good_cls, + + # Origins with good LCP, FID (optional), and CLS divided by origins with any LCP and CLS. + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND + IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_good_cwv +FROM + `chrome-ux-report.materialized.device_summary` +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + url, + app AS cms + FROM + `httparchive.technologies.2024_06_01_*` + WHERE + category = 'CMS') +ON + CONCAT(origin, '/') = url AND + IF(device = 'desktop', 'desktop', 'mobile') = client +WHERE + date = '2024-06-01' +GROUP BY + client, + cms +UNION ALL + + + +SELECT + 2023 AS year, + client, + cms, + COUNT(DISTINCT origin) AS origins, + # Origins with good LCP divided by origins with any LCP. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_good_lcp, + + # Origins with good FID divided by origins with any FID. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_good_fid, + + # Origins with good CLS divided by origins with any CLS. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_good_cls, + + # Origins with good LCP, FID (optional), and CLS divided by origins with any LCP and CLS. + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND + IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_good_cwv +FROM + `chrome-ux-report.materialized.device_summary` +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + url, + app AS cms + FROM + `httparchive.technologies.2023_06_01_*` + WHERE + category = 'CMS') +ON + CONCAT(origin, '/') = url AND + IF(device = 'desktop', 'desktop', 'mobile') = client +WHERE + date = '2023-06-01' +GROUP BY + client, + cms + + +union all + +SELECT + 2022 AS year, + client, + cms, + COUNT(DISTINCT origin) AS origins, + # Origins with good LCP divided by origins with any LCP. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_good_lcp, + + # Origins with good FID divided by origins with any FID. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_good_fid, + + # Origins with good CLS divided by origins with any CLS. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_good_cls, + + # Origins with good LCP, FID (optional), and CLS divided by origins with any LCP and CLS. + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND + IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_good_cwv +FROM + `chrome-ux-report.materialized.device_summary` +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + url, + app AS cms + FROM + `httparchive.technologies.2022_06_01_*` + WHERE + category = 'CMS') +ON + CONCAT(origin, '/') = url AND + IF(device = 'desktop', 'desktop', 'mobile') = client +WHERE + date = '2022-06-01' +GROUP BY + client, + cms + + ORDER BY + origins DESC diff --git a/sql/2024/cms/image_format_popularity.sql b/sql/2024/cms/image_format_popularity.sql new file mode 100644 index 00000000000..ba07be36c17 --- /dev/null +++ b/sql/2024/cms/image_format_popularity.sql @@ -0,0 +1,66 @@ +#standardSQL +# Image format popularity by CMS +SELECT + client, + cms, + ANY_VALUE(pages) AS pages, + format, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY client, cms) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client, cms) AS pct +FROM ( + SELECT DISTINCT + _TABLE_SUFFIX AS client, + url, + app AS cms + FROM + `httparchive.technologies.2024_06_01_*` + WHERE + category = 'CMS') +JOIN ( + SELECT + client, + page AS url, + CASE + WHEN json_value(summary, "$.mimeType") = 'image/avif' THEN 'avif' + WHEN json_value(summary, "$.mimeType") = 'image/webp' THEN 'webp' + WHEN json_value(summary, "$.mimeType") = 'image/jpeg' THEN 'jpg' + WHEN json_value(summary, "$.mimeType") = 'image/png' THEN 'png' + WHEN json_value(summary, "$.mimeType") = 'image/gif' THEN 'gif' + WHEN json_value(summary, "$.mimeType") = 'image/svg+xml' THEN 'svg' + WHEN json_value(summary, "$.mimeType") = 'image/x-icon' THEN 'ico' + WHEN json_value(summary, "$.mimeType") = 'image/vnd.microsoft.icon' THEN 'ico' + WHEN json_value(summary, "$.mimeType") = 'image/jpg' THEN 'jpg' + WHEN json_value(summary, "$.mimeType") = 'image/bmp' THEN 'bmp' + WHEN json_value(summary, "$.mimeType") = 'binary/octet-stream' THEN 'binary/octet-stream' + ELSE 'other/unknown' -- To handle any unexpected formats + END AS format + FROM + httparchive.all.requests + WHERE + date = '2024-06-01' AND + type = 'image') +USING + (client, url) +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + app AS cms, + COUNT(DISTINCT url) AS pages + FROM + `httparchive.technologies.2024_06_01_*` + WHERE + category = 'CMS' + GROUP BY + client, + cms) +USING + (client, cms) +WHERE + pages > 1000 +GROUP BY + client, + cms, + format +ORDER BY + freq DESC diff --git a/sql/2024/cms/lighthouse_category_scores_per_cms.sql b/sql/2024/cms/lighthouse_category_scores_per_cms.sql new file mode 100644 index 00000000000..2e0af7a87e2 --- /dev/null +++ b/sql/2024/cms/lighthouse_category_scores_per_cms.sql @@ -0,0 +1,34 @@ +#standardSQL +# Lighthouse category scores per CMS +SELECT + client, + cms, + COUNT(DISTINCT url) AS freq, + APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.performance.score') AS NUMERIC), 1000)[OFFSET(500)] * 100 AS median_performance, + APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.accessibility.score') AS NUMERIC), 1000)[OFFSET(500)] * 100 AS median_accessibility, + APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.seo.score') AS NUMERIC), 1000)[OFFSET(500)] * 100 AS median_seo, + APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$."best-practices".score') AS NUMERIC), 1000)[OFFSET(500)] * 100 AS median_best_practices +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url, + JSON_EXTRACT(report, '$.categories') AS categories + FROM + `httparchive.lighthouse.2024_06_01_*`) +JOIN ( + SELECT DISTINCT + _TABLE_SUFFIX AS client, + app AS cms, + url + FROM + `httparchive.technologies.2024_06_01_*` + WHERE + category = 'CMS') +USING + (url, client) +GROUP BY + client, + cms +ORDER BY + freq DESC, + client diff --git a/sql/2024/cms/page_weight_distribution.sql b/sql/2024/cms/page_weight_distribution.sql new file mode 100644 index 00000000000..a407f961db4 --- /dev/null +++ b/sql/2024/cms/page_weight_distribution.sql @@ -0,0 +1,36 @@ +#standardSQL +# Total page weight distribution by CMS +SELECT + percentile, + client, + cms, + COUNT(0) AS pages, + APPROX_QUANTILES(total_kb, 1000)[OFFSET(percentile * 10)] AS total_kb +FROM ( + SELECT DISTINCT + _TABLE_SUFFIX AS client, + url, + app AS cms + FROM + `httparchive.technologies.2024_06_01_*` + WHERE + category = 'CMS') +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + url, + bytesTotal / 1024 AS total_kb + FROM + `httparchive.summary_pages.2024_06_01_*`) +USING + (client, url), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client, + cms +HAVING + pages > 1000 +ORDER BY + percentile, + pages DESC diff --git a/sql/2024/cms/resource_weights.sql b/sql/2024/cms/resource_weights.sql new file mode 100644 index 00000000000..b1dc6842488 --- /dev/null +++ b/sql/2024/cms/resource_weights.sql @@ -0,0 +1,40 @@ +#standardSQL +# Median resource weights by CMS +SELECT + client, + cms, + COUNT(0) AS pages, + APPROX_QUANTILES(total_kb, 1000)[OFFSET(500)] AS median_total_kb, + APPROX_QUANTILES(html_kb, 1000)[OFFSET(500)] AS median_html_kb, + APPROX_QUANTILES(js_kb, 1000)[OFFSET(500)] AS median_js_kb, + APPROX_QUANTILES(css_kb, 1000)[OFFSET(500)] AS median_css_kb, + APPROX_QUANTILES(img_kb, 1000)[OFFSET(500)] AS median_img_kb, + APPROX_QUANTILES(font_kb, 1000)[OFFSET(500)] AS median_font_kb +FROM ( + SELECT DISTINCT + _TABLE_SUFFIX AS client, + url, + app AS cms + FROM + `httparchive.technologies.2024_06_01_*` + WHERE + category = 'CMS') +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + url, + bytesTotal / 1024 AS total_kb, + bytesHtml / 1024 AS html_kb, + bytesJS / 1024 AS js_kb, + bytesCSS / 1024 AS css_kb, + bytesImg / 1024 AS img_kb, + bytesFont / 1024 AS font_kb + FROM + `httparchive.summary_pages.2024_06_01_*`) +USING + (client, url) +GROUP BY + client, + cms +ORDER BY + pages DESC diff --git a/sql/2024/cms/top_cms.sql b/sql/2024/cms/top_cms.sql new file mode 100644 index 00000000000..e009a7597d6 --- /dev/null +++ b/sql/2024/cms/top_cms.sql @@ -0,0 +1,108 @@ +#standardSQL +# Top CMS platforms, compared to 2020 +SELECT + _TABLE_SUFFIX AS client, + 2024 AS year, + app AS cms, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.technologies.2024_06_01_*` +JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2024_06_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (_TABLE_SUFFIX) +WHERE + category = 'CMS' +GROUP BY + client, + total, + cms +UNION ALL +SELECT + _TABLE_SUFFIX AS client, + 2023 AS year, + app AS cms, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.technologies.2023_06_01_*` +JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2023_06_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (_TABLE_SUFFIX) +WHERE + category = 'CMS' +GROUP BY + client, + total, + cms +UNION ALL +SELECT + _TABLE_SUFFIX AS client, + 2022 AS year, + app AS cms, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.technologies.2022_08_01_*` +JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2022_08_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (_TABLE_SUFFIX) +WHERE + category = 'CMS' +GROUP BY + client, + total, + cms +UNION ALL +SELECT + _TABLE_SUFFIX AS client, + 2021 AS year, + app AS cms, + COUNT(DISTINCT url) AS freq, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.technologies.2021_07_01_*` +JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2021_07_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (_TABLE_SUFFIX) +WHERE + category = 'CMS' +GROUP BY + client, + total, + cms +ORDER BY + year DESC, + pct DESC diff --git a/sql/2024/cms/top_cms_by_geo.sql b/sql/2024/cms/top_cms_by_geo.sql new file mode 100644 index 00000000000..c2a3eaa463f --- /dev/null +++ b/sql/2024/cms/top_cms_by_geo.sql @@ -0,0 +1,68 @@ +#standardSQL +# CMS popularity per geo +WITH geo_summary AS ( + SELECT + `chrome-ux-report`.experimental.GET_COUNTRY(country_code) AS geo, + IF(device = 'desktop', 'desktop', 'mobile') AS client, + origin, + COUNT(DISTINCT origin) OVER (PARTITION BY country_code, IF(device = 'desktop', 'desktop', 'mobile')) AS total + FROM + `chrome-ux-report.materialized.country_summary` + WHERE + # We're intentionally using April 2022 CrUX data here. + # That's because there's a two month lag between CrUX and HA datasets. + # Since we're only JOINing with the CrUX dataset to see which URLs + # belong to different countries (as opposed to CWV field data) + # it's not necessary to look at the 202207 dataset. + yyyymm = 202406 + UNION ALL + SELECT + 'ALL' AS geo, + IF(device = 'desktop', 'desktop', 'mobile') AS client, + origin, + COUNT(DISTINCT origin) OVER (PARTITION BY IF(device = 'desktop', 'desktop', 'mobile')) AS total + FROM + `chrome-ux-report.materialized.device_summary` + WHERE + yyyymm = 202406 +) + +SELECT + * +FROM ( + SELECT + client, + geo, + cms, + COUNT(0) AS pages, + ANY_VALUE(total) AS total, + COUNT(DISTINCT url) / ANY_VALUE(total) AS pct + FROM ( + SELECT DISTINCT + geo, + client, + CONCAT(origin, '/') AS url, + total + FROM + geo_summary + ) JOIN ( + SELECT DISTINCT + _TABLE_SUFFIX AS client, + category, + app AS cms, + url + FROM + `httparchive.technologies.2024_06_01_*` + WHERE + app IS NOT NULL AND + category = 'CMS' AND + app != '' + ) USING (client, url) + GROUP BY + client, + geo, + cms) +WHERE + pages > 1000 +ORDER BY + pages DESC diff --git a/sql/2024/cms/top_cms_by_rank.sql b/sql/2024/cms/top_cms_by_rank.sql new file mode 100644 index 00000000000..c01ddb4931f --- /dev/null +++ b/sql/2024/cms/top_cms_by_rank.sql @@ -0,0 +1,52 @@ +#standardSQL +# CMS adoption per rank +SELECT + client, + cms, + rank, + COUNT(DISTINCT url) AS pages, + ANY_VALUE(total) AS total, + COUNT(DISTINCT url) / ANY_VALUE(total) AS pct +FROM ( + SELECT DISTINCT + _TABLE_SUFFIX AS client, + app AS cms, + url + FROM + `httparchive.technologies.2024_06_01_*` + WHERE + category = 'CMS') +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + url, + rank_magnitude AS rank + FROM + `httparchive.summary_pages.2024_06_01_*`, + UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude + WHERE + rank <= rank_magnitude) +USING + (client, url) +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + rank_magnitude AS rank, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2024_06_01_*`, + UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude + WHERE + rank <= rank_magnitude + GROUP BY + _TABLE_SUFFIX, + rank_magnitude) +USING + (client, rank) +GROUP BY + client, + cms, + rank +ORDER BY + rank, + pages DESC diff --git a/sql/2024/cms/wordpress_page_builders.sql b/sql/2024/cms/wordpress_page_builders.sql new file mode 100644 index 00000000000..fe9fe08eb08 --- /dev/null +++ b/sql/2024/cms/wordpress_page_builders.sql @@ -0,0 +1,35 @@ +#standardSQL +# Top WordPress page builder combinations +SELECT + client, + page_builders, + COUNT(0) AS pages, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct +FROM ( + SELECT DISTINCT + _TABLE_SUFFIX AS client, + url + FROM + `httparchive.technologies.2024_06_01_*` + WHERE + app = 'WordPress') +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + url, + ARRAY_TO_STRING(ARRAY_AGG(app ORDER BY app), ', ') AS page_builders + FROM + `httparchive.technologies.2024_06_01_*` + WHERE + category = 'Page builders' + GROUP BY + client, + url) +USING + (client, url) +GROUP BY + client, + page_builders +ORDER BY + pct DESC diff --git a/sql/2024/cms/wordpress_resources.sql b/sql/2024/cms/wordpress_resources.sql new file mode 100644 index 00000000000..696572f20b7 --- /dev/null +++ b/sql/2024/cms/wordpress_resources.sql @@ -0,0 +1,38 @@ +#standardSQL +# Distribution of WordPress resource types by path +SELECT + percentile, + client, + path, + APPROX_QUANTILES(freq, 1000)[OFFSET(percentile * 10)] AS freq +FROM ( + SELECT + _TABLE_SUFFIX AS client, + page, + REGEXP_EXTRACT(url, r'/(themes|plugins|wp-includes)/') AS path, + COUNT(0) AS freq + FROM + (SELECT _TABLE_SUFFIX, url AS page FROM `httparchive.technologies.2024_06_01_*` WHERE app = 'WordPress') + JOIN + (SELECT _TABLE_SUFFIX, pageid, url AS page FROM `httparchive.summary_pages.2024_06_01_*`) + USING + (_TABLE_SUFFIX, page) + JOIN + (SELECT _TABLE_SUFFIX, pageid, url FROM `httparchive.summary_requests.2024_06_01_*`) + USING + (_TABLE_SUFFIX, pageid) + GROUP BY + client, + page, + path + HAVING + path IS NOT NULL), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client, + path +ORDER BY + percentile, + client, + path From 8fea7b162aeaa824e02fee1fc461e1e67d0a807c Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Sat, 24 Aug 2024 09:33:15 +0200 Subject: [PATCH 02/77] Update cms_adoption_by_geo.sql --- sql/2024/cms/cms_adoption_by_geo.sql | 30 ++++++++++++++++------------ 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_geo.sql b/sql/2024/cms/cms_adoption_by_geo.sql index 56c8b589f21..cb013f444c9 100644 --- a/sql/2024/cms/cms_adoption_by_geo.sql +++ b/sql/2024/cms/cms_adoption_by_geo.sql @@ -8,10 +8,9 @@ WITH geo_summary AS ( COUNT(DISTINCT origin) OVER (PARTITION BY country_code, IF(device = 'desktop', 'desktop', 'mobile')) AS total FROM `chrome-ux-report.materialized.country_summary` - WHERE + WHERE yyyymm = 202406 ) - SELECT * FROM ( @@ -22,22 +21,27 @@ FROM ( ANY_VALUE(total) AS total, COUNT(0) / ANY_VALUE(total) AS pct FROM ( - SELECT DISTINCT - geo, + SELECT + DISTINCT geo, client, total, - CONCAT(origin, '/') AS url + CONCAT(origin, '/') AS page FROM - geo_summary - ) JOIN ( - SELECT DISTINCT - _TABLE_SUFFIX AS client, - url + geo_summary ) + JOIN ( + SELECT + client, + page FROM - `httparchive.technologies.2024_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - category = 'CMS' - ) USING (client, url) + date = '2024-06-01' + AND cats= 'CMS' ) + USING + (client, + page) GROUP BY client, geo) From 5a47ac2f8cfabe331ebd565af8b28ea1dd7e1062 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Sat, 24 Aug 2024 09:47:35 +0200 Subject: [PATCH 03/77] Update cms_adoption_by_rank.sql --- sql/2024/cms/cms_adoption_by_rank.sql | 47 +++++++++++++++------------ 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_rank.sql b/sql/2024/cms/cms_adoption_by_rank.sql index e416cf1b724..1743891d68a 100644 --- a/sql/2024/cms/cms_adoption_by_rank.sql +++ b/sql/2024/cms/cms_adoption_by_rank.sql @@ -1,51 +1,58 @@ -#standardSQL -# CMS adoption per rank +#standardSQL # CMS adoption per rank SELECT client, - cms, + technology, rank, - COUNT(DISTINCT url) AS pages, + COUNT(DISTINCT page) AS pages, ANY_VALUE(total) AS total, - COUNT(DISTINCT url) / ANY_VALUE(total) AS pct + COUNT(DISTINCT page) / ANY_VALUE(total) AS pct FROM ( - SELECT DISTINCT - _TABLE_SUFFIX AS client, - app AS cms, - url + SELECT + DISTINCT client, + technology, + page FROM - `httparchive.technologies.2024_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - category = 'CMS') + date = '2024-06-01' + AND cats= 'CMS' ) JOIN ( SELECT - _TABLE_SUFFIX AS client, - url, + client, + page, rank FROM - `httparchive.summary_pages.2024_06_01_*`) + `httparchive.all.pages` + WHERE + date = '2024-06-01' ) USING - (client, url) + (client, + page) JOIN ( SELECT - _TABLE_SUFFIX AS client, + client, rank_magnitude AS rank, COUNT(0) AS total FROM - `httparchive.summary_pages.2024_06_01_*`, + `httparchive.all.pages`, UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude WHERE rank <= rank_magnitude + AND date = '2024-06-01' GROUP BY - _TABLE_SUFFIX, + client, rank_magnitude) USING - (client, rank), + (client, + rank), UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude WHERE rank <= rank_magnitude GROUP BY client, - cms, + technology, rank ORDER BY rank, From 4621030a9d624d149d81b3584b9c1462592e569e Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Sat, 24 Aug 2024 09:55:50 +0200 Subject: [PATCH 04/77] Update cms_adoption_by_region.sql --- sql/2024/cms/cms_adoption_by_region.sql | 1962 +++++++++-------------- 1 file changed, 745 insertions(+), 1217 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_region.sql b/sql/2024/cms/cms_adoption_by_region.sql index 4baa0442d7d..6fccf04584f 100644 --- a/sql/2024/cms/cms_adoption_by_region.sql +++ b/sql/2024/cms/cms_adoption_by_region.sql @@ -1,1214 +1,737 @@ -#standardSQL -# All CMS popularity per geo -CREATE TEMP FUNCTION GET_GEO(country_code STRING, geo STRING) RETURNS STRING LANGUAGE js AS ''' -var countries = { - "af": { - "name": "Afghanistan", - "region": "Asia", - "sub-region": "Southern Asia" - }, - "ax": { - "name": "Åland Islands", - "region": "Europe", - "sub-region": "Northern Europe" - }, - "al": { - "name": "Albania", - "region": "Europe", - "sub-region": "Southern Europe" - }, - "dz": { - "name": "Algeria", - "region": "Africa", - "sub-region": "Northern Africa" - }, - "as": { - "name": "American Samoa", - "region": "Oceania", - "sub-region": "Polynesia" - }, - "ad": { - "name": "Andorra", - "region": "Europe", - "sub-region": "Southern Europe" - }, - "ao": { - "name": "Angola", - "region": "Africa", - "sub-region": "Middle Africa" - }, - "ai": { - "name": "Anguilla", - "region": "Americas", - "sub-region": "Caribbean" - }, - "ag": { - "name": "Antigua and Barbuda", - "region": "Americas", - "sub-region": "Caribbean" - }, - "ar": { - "name": "Argentina", - "region": "Americas", - "sub-region": "South America" - }, - "am": { - "name": "Armenia", - "region": "Asia", - "sub-region": "Western Asia" - }, - "aw": { - "name": "Aruba", - "region": "Americas", - "sub-region": "Caribbean" - }, - "au": { - "name": "Australia", - "region": "Oceania", - "sub-region": "Australia and New Zealand" - }, - "at": { - "name": "Austria", - "region": "Europe", - "sub-region": "Western Europe" - }, - "az": { - "name": "Azerbaijan", - "region": "Asia", - "sub-region": "Western Asia" - }, - "bs": { - "name": "Bahamas", - "region": "Americas", - "sub-region": "Caribbean" - }, - "bh": { - "name": "Bahrain", - "region": "Asia", - "sub-region": "Western Asia" - }, - "bd": { - "name": "Bangladesh", - "region": "Asia", - "sub-region": "Southern Asia" - }, - "bb": { - "name": "Barbados", - "region": "Americas", - "sub-region": "Caribbean" - }, - "by": { - "name": "Belarus", - "region": "Europe", - "sub-region": "Eastern Europe" - }, - "be": { - "name": "Belgium", - "region": "Europe", - "sub-region": "Western Europe" - }, - "bz": { - "name": "Belize", - "region": "Americas", - "sub-region": "Central America" - }, - "bj": { - "name": "Benin", - "region": "Africa", - "sub-region": "Western Africa" - }, - "bm": { - "name": "Bermuda", - "region": "Americas", - "sub-region": "Northern America" - }, - "bt": { - "name": "Bhutan", - "region": "Asia", - "sub-region": "Southern Asia" - }, - "bo": { - "name": "Bolivia (Plurinational State of)", - "region": "Americas", - "sub-region": "South America" - }, - "bq": { - "name": "Bonaire, Sint Eustatius and Saba", - "region": "Americas", - "sub-region": "Caribbean" - }, - "ba": { - "name": "Bosnia and Herzegovina", - "region": "Europe", - "sub-region": "Southern Europe" - }, - "bw": { - "name": "Botswana", - "region": "Africa", - "sub-region": "Southern Africa" - }, - "br": { - "name": "Brazil", - "region": "Americas", - "sub-region": "South America" - }, - "io": { - "name": "British Indian Ocean Territory", - "region": null, - "sub-region": null - }, - "bn": { - "name": "Brunei Darussalam", - "region": "Asia", - "sub-region": "South-Eastern Asia" - }, - "bg": { - "name": "Kosovo", - "region": "Europe", - "sub-region": "Eastern Europe" - }, - "bf": { - "name": "Burkina Faso", - "region": "Africa", - "sub-region": "Western Africa" - }, - "bi": { - "name": "Burundi", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "kh": { - "name": "Cambodia", - "region": "Asia", - "sub-region": "South-Eastern Asia" - }, - "cm": { - "name": "Cameroon", - "region": "Africa", - "sub-region": "Middle Africa" - }, - "ca": { - "name": "Canada", - "region": "Americas", - "sub-region": "Northern America" - }, - "cv": { - "name": "Cabo Verde", - "region": "Africa", - "sub-region": "Western Africa" - }, - "ky": { - "name": "Cayman Islands", - "region": "Americas", - "sub-region": "Caribbean" - }, - "cf": { - "name": "Central African Republic", - "region": "Africa", - "sub-region": "Middle Africa" - }, - "td": { - "name": "Chad", - "region": "Africa", - "sub-region": "Middle Africa" - }, - "cl": { - "name": "Chile", - "region": "Americas", - "sub-region": "South America" - }, - "cn": { - "name": "China", - "region": "Asia", - "sub-region": "Eastern Asia" - }, - "cx": { - "name": "Christmas Island", - "region": null, - "sub-region": null - }, - "co": { - "name": "Colombia", - "region": "Americas", - "sub-region": "South America" - }, - "km": { - "name": "Comoros", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "cg": { - "name": "Congo", - "region": "Africa", - "sub-region": "Middle Africa" - }, - "cd": { - "name": "Congo (Democratic Republic of the)", - "region": "Africa", - "sub-region": "Middle Africa" - }, - "ck": { - "name": "Cook Islands", - "region": "Oceania", - "sub-region": "Polynesia" - }, - "cr": { - "name": "Costa Rica", - "region": "Americas", - "sub-region": "Central America" - }, - "ci": { - "name": "Côte d'Ivoire", - "region": "Africa", - "sub-region": "Western Africa" - }, - "hr": { - "name": "Croatia", - "region": "Europe", - "sub-region": "Southern Europe" - }, - "cu": { - "name": "Cuba", - "region": "Americas", - "sub-region": "Caribbean" - }, - "cw": { - "name": "Curaçao", - "region": "Americas", - "sub-region": "Caribbean" - }, - "cy": { - "name": "Cyprus", - "region": "Asia", - "sub-region": "Western Asia" - }, - "cz": { - "name": "Czech Republic", - "region": "Europe", - "sub-region": "Eastern Europe" - }, - "dk": { - "name": "Denmark", - "region": "Europe", - "sub-region": "Northern Europe" - }, - "dj": { - "name": "Djibouti", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "dm": { - "name": "Dominica", - "region": "Americas", - "sub-region": "Caribbean" - }, - "do": { - "name": "Dominican Republic", - "region": "Americas", - "sub-region": "Caribbean" - }, - "ec": { - "name": "Ecuador", - "region": "Americas", - "sub-region": "South America" - }, - "eg": { - "name": "Egypt", - "region": "Africa", - "sub-region": "Northern Africa" - }, - "sv": { - "name": "El Salvador", - "region": "Americas", - "sub-region": "Central America" - }, - "gq": { - "name": "Equatorial Guinea", - "region": "Africa", - "sub-region": "Middle Africa" - }, - "er": { - "name": "Eritrea", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "ee": { - "name": "Estonia", - "region": "Europe", - "sub-region": "Northern Europe" - }, - "et": { - "name": "Ethiopia", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "fk": { - "name": "Falkland Islands (Malvinas)", - "region": "Americas", - "sub-region": "South America" - }, - "fo": { - "name": "Faroe Islands", - "region": "Europe", - "sub-region": "Northern Europe" - }, - "fj": { - "name": "Fiji", - "region": "Oceania", - "sub-region": "Melanesia" - }, - "fi": { - "name": "Finland", - "region": "Europe", - "sub-region": "Northern Europe" - }, - "fr": { - "name": "France", - "region": "Europe", - "sub-region": "Western Europe" - }, - "gf": { - "name": "French Guiana", - "region": "Americas", - "sub-region": "South America" - }, - "pf": { - "name": "French Polynesia", - "region": "Oceania", - "sub-region": "Polynesia" - }, - "ga": { - "name": "Gabon", - "region": "Africa", - "sub-region": "Middle Africa" - }, - "gm": { - "name": "Gambia", - "region": "Africa", - "sub-region": "Western Africa" - }, - "ge": { - "name": "Georgia", - "region": "Asia", - "sub-region": "Western Asia" - }, - "de": { - "name": "Germany", - "region": "Europe", - "sub-region": "Western Europe" - }, - "gh": { - "name": "Ghana", - "region": "Africa", - "sub-region": "Western Africa" - }, - "gi": { - "name": "Gibraltar", - "region": "Europe", - "sub-region": "Southern Europe" - }, - "gr": { - "name": "Greece", - "region": "Europe", - "sub-region": "Southern Europe" - }, - "gl": { - "name": "Greenland", - "region": "Americas", - "sub-region": "Northern America" - }, - "gd": { - "name": "Grenada", - "region": "Americas", - "sub-region": "Caribbean" - }, - "gp": { - "name": "Guadeloupe", - "region": "Americas", - "sub-region": "Caribbean" - }, - "gu": { - "name": "Guam", - "region": "Oceania", - "sub-region": "Micronesia" - }, - "gt": { - "name": "Guatemala", - "region": "Americas", - "sub-region": "Central America" - }, - "gg": { - "name": "Guernsey", - "region": "Europe", - "sub-region": "Northern Europe" - }, - "gn": { - "name": "Guinea", - "region": "Africa", - "sub-region": "Western Africa" - }, - "gw": { - "name": "Guinea-Bissau", - "region": "Africa", - "sub-region": "Western Africa" - }, - "gy": { - "name": "Guyana", - "region": "Americas", - "sub-region": "South America" - }, - "ht": { - "name": "Haiti", - "region": "Americas", - "sub-region": "Caribbean" - }, - "hn": { - "name": "Honduras", - "region": "Americas", - "sub-region": "Central America" - }, - "hk": { - "name": "Hong Kong", - "region": "Asia", - "sub-region": "Eastern Asia" - }, - "hu": { - "name": "Hungary", - "region": "Europe", - "sub-region": "Eastern Europe" - }, - "is": { - "name": "Iceland", - "region": "Europe", - "sub-region": "Northern Europe" - }, - "in": { - "name": "India", - "region": "Asia", - "sub-region": "Southern Asia" - }, - "id": { - "name": "Indonesia", - "region": "Asia", - "sub-region": "South-Eastern Asia" - }, - "ir": { - "name": "Iran (Islamic Republic of)", - "region": "Asia", - "sub-region": "Southern Asia" - }, - "iq": { - "name": "Iraq", - "region": "Asia", - "sub-region": "Western Asia" - }, - "ie": { - "name": "Ireland", - "region": "Europe", - "sub-region": "Northern Europe" - }, - "im": { - "name": "Isle of Man", - "region": "Europe", - "sub-region": "Northern Europe" - }, - "il": { - "name": "Israel", - "region": "Asia", - "sub-region": "Western Asia" - }, - "it": { - "name": "Italy", - "region": "Europe", - "sub-region": "Southern Europe" - }, - "jm": { - "name": "Jamaica", - "region": "Americas", - "sub-region": "Caribbean" - }, - "jp": { - "name": "Japan", - "region": "Asia", - "sub-region": "Eastern Asia" - }, - "je": { - "name": "Jersey", - "region": "Europe", - "sub-region": "Northern Europe" - }, - "jo": { - "name": "Jordan", - "region": "Asia", - "sub-region": "Western Asia" - }, - "kz": { - "name": "Kazakhstan", - "region": "Asia", - "sub-region": "Central Asia" - }, - "ke": { - "name": "Kenya", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "ki": { - "name": "Kiribati", - "region": "Oceania", - "sub-region": "Micronesia" - }, - "kp": { - "name": "Korea (Democratic People's Republic of)", - "region": "Asia", - "sub-region": "Eastern Asia" - }, - "kr": { - "name": "Korea (Republic of)", - "region": "Asia", - "sub-region": "Eastern Asia" - }, - "kw": { - "name": "Kuwait", - "region": "Asia", - "sub-region": "Western Asia" - }, - "kg": { - "name": "Kyrgyzstan", - "region": "Asia", - "sub-region": "Central Asia" - }, - "la": { - "name": "Lao People's Democratic Republic", - "region": "Asia", - "sub-region": "South-Eastern Asia" - }, - "lv": { - "name": "Latvia", - "region": "Europe", - "sub-region": "Northern Europe" - }, - "lb": { - "name": "Lebanon", - "region": "Asia", - "sub-region": "Western Asia" - }, - "ls": { - "name": "Lesotho", - "region": "Africa", - "sub-region": "Southern Africa" - }, - "lr": { - "name": "Liberia", - "region": "Africa", - "sub-region": "Western Africa" - }, - "ly": { - "name": "Libya", - "region": "Africa", - "sub-region": "Northern Africa" - }, - "li": { - "name": "Liechtenstein", - "region": "Europe", - "sub-region": "Western Europe" - }, - "lt": { - "name": "Lithuania", - "region": "Europe", - "sub-region": "Northern Europe" - }, - "lu": { - "name": "Luxembourg", - "region": "Europe", - "sub-region": "Western Europe" - }, - "mo": { - "name": "Macao", - "region": "Asia", - "sub-region": "Eastern Asia" - }, - "mk": { - "name": "Macedonia (the former Yugoslav Republic of)", - "region": "Europe", - "sub-region": "Southern Europe" - }, - "mg": { - "name": "Madagascar", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "mw": { - "name": "Malawi", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "my": { - "name": "Malaysia", - "region": "Asia", - "sub-region": "South-Eastern Asia" - }, - "mv": { - "name": "Maldives", - "region": "Asia", - "sub-region": "Southern Asia" - }, - "ml": { - "name": "Mali", - "region": "Africa", - "sub-region": "Western Africa" - }, - "mt": { - "name": "Malta", - "region": "Europe", - "sub-region": "Southern Europe" - }, - "mh": { - "name": "Marshall Islands", - "region": "Oceania", - "sub-region": "Micronesia" - }, - "mq": { - "name": "Martinique", - "region": "Americas", - "sub-region": "Caribbean" - }, - "mr": { - "name": "Mauritania", - "region": "Africa", - "sub-region": "Western Africa" - }, - "mu": { - "name": "Mauritius", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "yt": { - "name": "Mayotte", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "mx": { - "name": "Mexico", - "region": "Americas", - "sub-region": "Central America" - }, - "fm": { - "name": "Micronesia (Federated States of)", - "region": "Oceania", - "sub-region": "Micronesia" - }, - "md": { - "name": "Moldova (Republic of)", - "region": "Europe", - "sub-region": "Eastern Europe" - }, - "mc": { - "name": "Monaco", - "region": "Europe", - "sub-region": "Western Europe" - }, - "mn": { - "name": "Mongolia", - "region": "Asia", - "sub-region": "Eastern Asia" - }, - "me": { - "name": "Montenegro", - "region": "Europe", - "sub-region": "Southern Europe" - }, - "ms": { - "name": "Montserrat", - "region": "Americas", - "sub-region": "Caribbean" - }, - "ma": { - "name": "Morocco", - "region": "Africa", - "sub-region": "Northern Africa" - }, - "mz": { - "name": "Mozambique", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "mm": { - "name": "Myanmar", - "region": "Asia", - "sub-region": "South-Eastern Asia" - }, - "na": { - "name": "Namibia", - "region": "Africa", - "sub-region": "Southern Africa" - }, - "nr": { - "name": "Nauru", - "region": "Oceania", - "sub-region": "Micronesia" - }, - "np": { - "name": "Nepal", - "region": "Asia", - "sub-region": "Southern Asia" - }, - "nl": { - "name": "Netherlands", - "region": "Europe", - "sub-region": "Western Europe" - }, - "nc": { - "name": "New Caledonia", - "region": "Oceania", - "sub-region": "Melanesia" - }, - "nz": { - "name": "New Zealand", - "region": "Oceania", - "sub-region": "Australia and New Zealand" - }, - "ni": { - "name": "Nicaragua", - "region": "Americas", - "sub-region": "Central America" - }, - "ne": { - "name": "Niger", - "region": "Africa", - "sub-region": "Western Africa" - }, - "ng": { - "name": "Nigeria", - "region": "Africa", - "sub-region": "Western Africa" - }, - "nf": { - "name": "Norfolk Island", - "region": "Oceania", - "sub-region": "Australia and New Zealand" - }, - "mp": { - "name": "Northern Mariana Islands", - "region": "Oceania", - "sub-region": "Micronesia" - }, - "no": { - "name": "Norway", - "region": "Europe", - "sub-region": "Northern Europe" - }, - "om": { - "name": "Oman", - "region": "Asia", - "sub-region": "Western Asia" - }, - "pk": { - "name": "Pakistan", - "region": "Asia", - "sub-region": "Southern Asia" - }, - "pw": { - "name": "Palau", - "region": "Oceania", - "sub-region": "Micronesia" - }, - "ps": { - "name": "Palestine, State of", - "region": "Asia", - "sub-region": "Western Asia" - }, - "pa": { - "name": "Panama", - "region": "Americas", - "sub-region": "Central America" - }, - "pg": { - "name": "Papua New Guinea", - "region": "Oceania", - "sub-region": "Melanesia" - }, - "py": { - "name": "Paraguay", - "region": "Americas", - "sub-region": "South America" - }, - "pe": { - "name": "Peru", - "region": "Americas", - "sub-region": "South America" - }, - "ph": { - "name": "Philippines", - "region": "Asia", - "sub-region": "South-Eastern Asia" - }, - "pl": { - "name": "Poland", - "region": "Europe", - "sub-region": "Eastern Europe" - }, - "pt": { - "name": "Portugal", - "region": "Europe", - "sub-region": "Southern Europe" - }, - "pr": { - "name": "Puerto Rico", - "region": "Americas", - "sub-region": "Caribbean" - }, - "qa": { - "name": "Qatar", - "region": "Asia", - "sub-region": "Western Asia" - }, - "re": { - "name": "Réunion", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "ro": { - "name": "Romania", - "region": "Europe", - "sub-region": "Eastern Europe" - }, - "ru": { - "name": "Russian Federation", - "region": "Europe", - "sub-region": "Eastern Europe" - }, - "rw": { - "name": "Rwanda", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "bl": { - "name": "Saint Barthélemy", - "region": "Americas", - "sub-region": "Caribbean" - }, - "sh": { - "name": "Saint Helena, Ascension and Tristan da Cunha", - "region": "Africa", - "sub-region": "Western Africa" - }, - "kn": { - "name": "Saint Kitts and Nevis", - "region": "Americas", - "sub-region": "Caribbean" - }, - "lc": { - "name": "Saint Lucia", - "region": "Americas", - "sub-region": "Caribbean" - }, - "mf": { - "name": "Saint Martin (French part)", - "region": "Americas", - "sub-region": "Caribbean" - }, - "pm": { - "name": "Saint Pierre and Miquelon", - "region": "Americas", - "sub-region": "Northern America" - }, - "vc": { - "name": "Saint Vincent and the Grenadines", - "region": "Americas", - "sub-region": "Caribbean" - }, - "ws": { - "name": "Samoa", - "region": "Oceania", - "sub-region": "Polynesia" - }, - "sm": { - "name": "San Marino", - "region": "Europe", - "sub-region": "Southern Europe" - }, - "st": { - "name": "Sao Tome and Principe", - "region": "Africa", - "sub-region": "Middle Africa" - }, - "sa": { - "name": "Saudi Arabia", - "region": "Asia", - "sub-region": "Western Asia" - }, - "sn": { - "name": "Senegal", - "region": "Africa", - "sub-region": "Western Africa" - }, - "rs": { - "name": "Serbia", - "region": "Europe", - "sub-region": "Southern Europe" - }, - "sc": { - "name": "Seychelles", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "sl": { - "name": "Sierra Leone", - "region": "Africa", - "sub-region": "Western Africa" - }, - "sg": { - "name": "Singapore", - "region": "Asia", - "sub-region": "South-Eastern Asia" - }, - "sx": { - "name": "Sint Maarten (Dutch part)", - "region": "Americas", - "sub-region": "Caribbean" - }, - "sk": { - "name": "Slovakia", - "region": "Europe", - "sub-region": "Eastern Europe" - }, - "si": { - "name": "Slovenia", - "region": "Europe", - "sub-region": "Southern Europe" - }, - "sb": { - "name": "Solomon Islands", - "region": "Oceania", - "sub-region": "Melanesia" - }, - "so": { - "name": "Somalia", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "za": { - "name": "South Africa", - "region": "Africa", - "sub-region": "Southern Africa" - }, - "ss": { - "name": "South Sudan", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "es": { - "name": "Spain", - "region": "Europe", - "sub-region": "Southern Europe" - }, - "lk": { - "name": "Sri Lanka", - "region": "Asia", - "sub-region": "Southern Asia" - }, - "sd": { - "name": "Sudan", - "region": "Africa", - "sub-region": "Northern Africa" - }, - "sr": { - "name": "Suriname", - "region": "Americas", - "sub-region": "South America" - }, - "sj": { - "name": "Svalbard and Jan Mayen", - "region": "Europe", - "sub-region": "Northern Europe" - }, - "sz": { - "name": "Swaziland", - "region": "Africa", - "sub-region": "Southern Africa" - }, - "se": { - "name": "Sweden", - "region": "Europe", - "sub-region": "Northern Europe" - }, - "ch": { - "name": "Switzerland", - "region": "Europe", - "sub-region": "Western Europe" - }, - "sy": { - "name": "Syrian Arab Republic", - "region": "Asia", - "sub-region": "Western Asia" - }, - "tw": { - "name": "Taiwan, Province of China", - "region": "Asia", - "sub-region": "Eastern Asia" - }, - "tj": { - "name": "Tajikistan", - "region": "Asia", - "sub-region": "Central Asia" - }, - "tz": { - "name": "Tanzania, United Republic of", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "th": { - "name": "Thailand", - "region": "Asia", - "sub-region": "South-Eastern Asia" - }, - "tl": { - "name": "Timor-Leste", - "region": "Asia", - "sub-region": "South-Eastern Asia" - }, - "tg": { - "name": "Togo", - "region": "Africa", - "sub-region": "Western Africa" - }, - "to": { - "name": "Tonga", - "region": "Oceania", - "sub-region": "Polynesia" - }, - "tt": { - "name": "Trinidad and Tobago", - "region": "Americas", - "sub-region": "Caribbean" - }, - "tn": { - "name": "Tunisia", - "region": "Africa", - "sub-region": "Northern Africa" - }, - "tr": { - "name": "Turkey", - "region": "Asia", - "sub-region": "Western Asia" - }, - "tm": { - "name": "Turkmenistan", - "region": "Asia", - "sub-region": "Central Asia" - }, - "tc": { - "name": "Turks and Caicos Islands", - "region": "Americas", - "sub-region": "Caribbean" - }, - "tv": { - "name": "Tuvalu", - "region": "Oceania", - "sub-region": "Polynesia" - }, - "ug": { - "name": "Uganda", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "ua": { - "name": "Ukraine", - "region": "Europe", - "sub-region": "Eastern Europe" - }, - "ae": { - "name": "United Arab Emirates", - "region": "Asia", - "sub-region": "Western Asia" - }, - "gb": { - "name": "United Kingdom of Great Britain and Northern Ireland", - "region": "Europe", - "sub-region": "Northern Europe" - }, - "us": { - "name": "United States of America", - "region": "Americas", - "sub-region": "Northern America" - }, - "uy": { - "name": "Uruguay", - "region": "Americas", - "sub-region": "South America" - }, - "uz": { - "name": "Uzbekistan", - "region": "Asia", - "sub-region": "Central Asia" - }, - "vu": { - "name": "Vanuatu", - "region": "Oceania", - "sub-region": "Melanesia" - }, - "ve": { - "name": "Venezuela (Bolivarian Republic of)", - "region": "Americas", - "sub-region": "South America" - }, - "vn": { - "name": "Viet Nam", - "region": "Asia", - "sub-region": "South-Eastern Asia" - }, - "vg": { - "name": "Virgin Islands (British)", - "region": "Americas", - "sub-region": "Caribbean" - }, - "vi": { - "name": "Virgin Islands (U.S.)", - "region": "Americas", - "sub-region": "Caribbean" - }, - "eh": { - "name": "Western Sahara", - "region": "Africa", - "sub-region": "Northern Africa" - }, - "ye": { - "name": "Yemen", - "region": "Asia", - "sub-region": "Western Asia" - }, - "zm": { - "name": "Zambia", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "zw": { - "name": "Zimbabwe", - "region": "Africa", - "sub-region": "Eastern Africa" - }, - "xk": { - "name": "Kosovo", - "region": "Europe", - "sub-region": "Eastern Europe" - } -}; - -return countries[country_code][geo]; -'''; - -WITH geo_summary AS ( +#standardSQL # ALL CMS popularity per geo +CREATE TEMP FUNCTION + GET_GEO(country_code STRING, + geo STRING) + RETURNS STRING + LANGUAGE js AS ''' var countries = { "af": { "name": "Afghanistan", + "region": "Asia", + "sub-region": "Southern Asia" }, + "ax": { "name": "Åland Islands", + "region": "Europe", + "sub-region": "Northern Europe" }, + "al": { "name": "Albania", + "region": "Europe", + "sub-region": "Southern Europe" }, + "dz": { "name": "Algeria", + "region": "Africa", + "sub-region": "Northern Africa" }, + "as": { "name": "American Samoa", + "region": "Oceania", + "sub-region": "Polynesia" }, + "ad": { "name": "Andorra", + "region": "Europe", + "sub-region": "Southern Europe" }, + "ao": { "name": "Angola", + "region": "Africa", + "sub-region": "Middle Africa" }, + "ai": { "name": "Anguilla", + "region": "Americas", + "sub-region": "Caribbean" }, + "ag": { "name": "Antigua and Barbuda", + "region": "Americas", + "sub-region": "Caribbean" }, + "ar": { "name": "Argentina", + "region": "Americas", + "sub-region": "South America" }, + "am": { "name": "Armenia", + "region": "Asia", + "sub-region": "Western Asia" }, + "aw": { "name": "Aruba", + "region": "Americas", + "sub-region": "Caribbean" }, + "au": { "name": "Australia", + "region": "Oceania", + "sub-region": "Australia and New Zealand" }, + "at": { "name": "Austria", + "region": "Europe", + "sub-region": "Western Europe" }, + "az": { "name": "Azerbaijan", + "region": "Asia", + "sub-region": "Western Asia" }, + "bs": { "name": "Bahamas", + "region": "Americas", + "sub-region": "Caribbean" }, + "bh": { "name": "Bahrain", + "region": "Asia", + "sub-region": "Western Asia" }, + "bd": { "name": "Bangladesh", + "region": "Asia", + "sub-region": "Southern Asia" }, + "bb": { "name": "Barbados", + "region": "Americas", + "sub-region": "Caribbean" }, + "by": { "name": "Belarus", + "region": "Europe", + "sub-region": "Eastern Europe" }, + "be": { "name": "Belgium", + "region": "Europe", + "sub-region": "Western Europe" }, + "bz": { "name": "Belize", + "region": "Americas", + "sub-region": "Central America" }, + "bj": { "name": "Benin", + "region": "Africa", + "sub-region": "Western Africa" }, + "bm": { "name": "Bermuda", + "region": "Americas", + "sub-region": "Northern America" }, + "bt": { "name": "Bhutan", + "region": "Asia", + "sub-region": "Southern Asia" }, + "bo": { "name": "Bolivia (Plurinational State of)", + "region": "Americas", + "sub-region": "South America" }, + "bq": { "name": "Bonaire, Sint Eustatius and Saba", + "region": "Americas", + "sub-region": "Caribbean" }, + "ba": { "name": "Bosnia and Herzegovina", + "region": "Europe", + "sub-region": "Southern Europe" }, + "bw": { "name": "Botswana", + "region": "Africa", + "sub-region": "Southern Africa" }, + "br": { "name": "Brazil", + "region": "Americas", + "sub-region": "South America" }, + "io": { "name": "British Indian Ocean Territory", + "region": null, + "sub-region": null }, + "bn": { "name": "Brunei Darussalam", + "region": "Asia", + "sub-region": "South-Eastern Asia" }, + "bg": { "name": "Kosovo", + "region": "Europe", + "sub-region": "Eastern Europe" }, + "bf": { "name": "Burkina Faso", + "region": "Africa", + "sub-region": "Western Africa" }, + "bi": { "name": "Burundi", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "kh": { "name": "Cambodia", + "region": "Asia", + "sub-region": "South-Eastern Asia" }, + "cm": { "name": "Cameroon", + "region": "Africa", + "sub-region": "Middle Africa" }, + "ca": { "name": "Canada", + "region": "Americas", + "sub-region": "Northern America" }, + "cv": { "name": "Cabo Verde", + "region": "Africa", + "sub-region": "Western Africa" }, + "ky": { "name": "Cayman Islands", + "region": "Americas", + "sub-region": "Caribbean" }, + "cf": { "name": "Central African Republic", + "region": "Africa", + "sub-region": "Middle Africa" }, + "td": { "name": "Chad", + "region": "Africa", + "sub-region": "Middle Africa" }, + "cl": { "name": "Chile", + "region": "Americas", + "sub-region": "South America" }, + "cn": { "name": "China", + "region": "Asia", + "sub-region": "Eastern Asia" }, + "cx": { "name": "Christmas Island", + "region": null, + "sub-region": null }, + "co": { "name": "Colombia", + "region": "Americas", + "sub-region": "South America" }, + "km": { "name": "Comoros", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "cg": { "name": "Congo", + "region": "Africa", + "sub-region": "Middle Africa" }, + "cd": { "name": "Congo (Democratic Republic of the)", + "region": "Africa", + "sub-region": "Middle Africa" }, + "ck": { "name": "Cook Islands", + "region": "Oceania", + "sub-region": "Polynesia" }, + "cr": { "name": "Costa Rica", + "region": "Americas", + "sub-region": "Central America" }, + "ci": { "name": "Côte d'Ivoire", + "region": "Africa", + "sub-region": "Western Africa" }, + "hr": { "name": "Croatia", + "region": "Europe", + "sub-region": "Southern Europe" }, + "cu": { "name": "Cuba", + "region": "Americas", + "sub-region": "Caribbean" }, + "cw": { "name": "Curaçao", + "region": "Americas", + "sub-region": "Caribbean" }, + "cy": { "name": "Cyprus", + "region": "Asia", + "sub-region": "Western Asia" }, + "cz": { "name": "Czech Republic", + "region": "Europe", + "sub-region": "Eastern Europe" }, + "dk": { "name": "Denmark", + "region": "Europe", + "sub-region": "Northern Europe" }, + "dj": { "name": "Djibouti", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "dm": { "name": "Dominica", + "region": "Americas", + "sub-region": "Caribbean" }, + "do": { "name": "Dominican Republic", + "region": "Americas", + "sub-region": "Caribbean" }, + "ec": { "name": "Ecuador", + "region": "Americas", + "sub-region": "South America" }, + "eg": { "name": "Egypt", + "region": "Africa", + "sub-region": "Northern Africa" }, + "sv": { "name": "El Salvador", + "region": "Americas", + "sub-region": "Central America" }, + "gq": { "name": "Equatorial Guinea", + "region": "Africa", + "sub-region": "Middle Africa" }, + "er": { "name": "Eritrea", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "ee": { "name": "Estonia", + "region": "Europe", + "sub-region": "Northern Europe" }, + "et": { "name": "Ethiopia", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "fk": { "name": "Falkland Islands (Malvinas)", + "region": "Americas", + "sub-region": "South America" }, + "fo": { "name": "Faroe Islands", + "region": "Europe", + "sub-region": "Northern Europe" }, + "fj": { "name": "Fiji", + "region": "Oceania", + "sub-region": "Melanesia" }, + "fi": { "name": "Finland", + "region": "Europe", + "sub-region": "Northern Europe" }, + "fr": { "name": "France", + "region": "Europe", + "sub-region": "Western Europe" }, + "gf": { "name": "French Guiana", + "region": "Americas", + "sub-region": "South America" }, + "pf": { "name": "French Polynesia", + "region": "Oceania", + "sub-region": "Polynesia" }, + "ga": { "name": "Gabon", + "region": "Africa", + "sub-region": "Middle Africa" }, + "gm": { "name": "Gambia", + "region": "Africa", + "sub-region": "Western Africa" }, + "ge": { "name": "Georgia", + "region": "Asia", + "sub-region": "Western Asia" }, + "de": { "name": "Germany", + "region": "Europe", + "sub-region": "Western Europe" }, + "gh": { "name": "Ghana", + "region": "Africa", + "sub-region": "Western Africa" }, + "gi": { "name": "Gibraltar", + "region": "Europe", + "sub-region": "Southern Europe" }, + "gr": { "name": "Greece", + "region": "Europe", + "sub-region": "Southern Europe" }, + "gl": { "name": "Greenland", + "region": "Americas", + "sub-region": "Northern America" }, + "gd": { "name": "Grenada", + "region": "Americas", + "sub-region": "Caribbean" }, + "gp": { "name": "Guadeloupe", + "region": "Americas", + "sub-region": "Caribbean" }, + "gu": { "name": "Guam", + "region": "Oceania", + "sub-region": "Micronesia" }, + "gt": { "name": "Guatemala", + "region": "Americas", + "sub-region": "Central America" }, + "gg": { "name": "Guernsey", + "region": "Europe", + "sub-region": "Northern Europe" }, + "gn": { "name": "Guinea", + "region": "Africa", + "sub-region": "Western Africa" }, + "gw": { "name": "Guinea-Bissau", + "region": "Africa", + "sub-region": "Western Africa" }, + "gy": { "name": "Guyana", + "region": "Americas", + "sub-region": "South America" }, + "ht": { "name": "Haiti", + "region": "Americas", + "sub-region": "Caribbean" }, + "hn": { "name": "Honduras", + "region": "Americas", + "sub-region": "Central America" }, + "hk": { "name": "Hong Kong", + "region": "Asia", + "sub-region": "Eastern Asia" }, + "hu": { "name": "Hungary", + "region": "Europe", + "sub-region": "Eastern Europe" }, + "is": { "name": "Iceland", + "region": "Europe", + "sub-region": "Northern Europe" }, + "in": { "name": "India", + "region": "Asia", + "sub-region": "Southern Asia" }, + "id": { "name": "Indonesia", + "region": "Asia", + "sub-region": "South-Eastern Asia" }, + "ir": { "name": "Iran (Islamic Republic of)", + "region": "Asia", + "sub-region": "Southern Asia" }, + "iq": { "name": "Iraq", + "region": "Asia", + "sub-region": "Western Asia" }, + "ie": { "name": "Ireland", + "region": "Europe", + "sub-region": "Northern Europe" }, + "im": { "name": "Isle of Man", + "region": "Europe", + "sub-region": "Northern Europe" }, + "il": { "name": "Israel", + "region": "Asia", + "sub-region": "Western Asia" }, + "it": { "name": "Italy", + "region": "Europe", + "sub-region": "Southern Europe" }, + "jm": { "name": "Jamaica", + "region": "Americas", + "sub-region": "Caribbean" }, + "jp": { "name": "Japan", + "region": "Asia", + "sub-region": "Eastern Asia" }, + "je": { "name": "Jersey", + "region": "Europe", + "sub-region": "Northern Europe" }, + "jo": { "name": "Jordan", + "region": "Asia", + "sub-region": "Western Asia" }, + "kz": { "name": "Kazakhstan", + "region": "Asia", + "sub-region": "Central Asia" }, + "ke": { "name": "Kenya", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "ki": { "name": "Kiribati", + "region": "Oceania", + "sub-region": "Micronesia" }, + "kp": { "name": "Korea (Democratic People's Republic of)", + "region": "Asia", + "sub-region": "Eastern Asia" }, + "kr": { "name": "Korea (Republic of)", + "region": "Asia", + "sub-region": "Eastern Asia" }, + "kw": { "name": "Kuwait", + "region": "Asia", + "sub-region": "Western Asia" }, + "kg": { "name": "Kyrgyzstan", + "region": "Asia", + "sub-region": "Central Asia" }, + "la": { "name": "Lao People's Democratic Republic", + "region": "Asia", + "sub-region": "South-Eastern Asia" }, + "lv": { "name": "Latvia", + "region": "Europe", + "sub-region": "Northern Europe" }, + "lb": { "name": "Lebanon", + "region": "Asia", + "sub-region": "Western Asia" }, + "ls": { "name": "Lesotho", + "region": "Africa", + "sub-region": "Southern Africa" }, + "lr": { "name": "Liberia", + "region": "Africa", + "sub-region": "Western Africa" }, + "ly": { "name": "Libya", + "region": "Africa", + "sub-region": "Northern Africa" }, + "li": { "name": "Liechtenstein", + "region": "Europe", + "sub-region": "Western Europe" }, + "lt": { "name": "Lithuania", + "region": "Europe", + "sub-region": "Northern Europe" }, + "lu": { "name": "Luxembourg", + "region": "Europe", + "sub-region": "Western Europe" }, + "mo": { "name": "Macao", + "region": "Asia", + "sub-region": "Eastern Asia" }, + "mk": { "name": "Macedonia (the former Yugoslav Republic of)", + "region": "Europe", + "sub-region": "Southern Europe" }, + "mg": { "name": "Madagascar", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "mw": { "name": "Malawi", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "my": { "name": "Malaysia", + "region": "Asia", + "sub-region": "South-Eastern Asia" }, + "mv": { "name": "Maldives", + "region": "Asia", + "sub-region": "Southern Asia" }, + "ml": { "name": "Mali", + "region": "Africa", + "sub-region": "Western Africa" }, + "mt": { "name": "Malta", + "region": "Europe", + "sub-region": "Southern Europe" }, + "mh": { "name": "Marshall Islands", + "region": "Oceania", + "sub-region": "Micronesia" }, + "mq": { "name": "Martinique", + "region": "Americas", + "sub-region": "Caribbean" }, + "mr": { "name": "Mauritania", + "region": "Africa", + "sub-region": "Western Africa" }, + "mu": { "name": "Mauritius", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "yt": { "name": "Mayotte", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "mx": { "name": "Mexico", + "region": "Americas", + "sub-region": "Central America" }, + "fm": { "name": "Micronesia (Federated States of)", + "region": "Oceania", + "sub-region": "Micronesia" }, + "md": { "name": "Moldova (Republic of)", + "region": "Europe", + "sub-region": "Eastern Europe" }, + "mc": { "name": "Monaco", + "region": "Europe", + "sub-region": "Western Europe" }, + "mn": { "name": "Mongolia", + "region": "Asia", + "sub-region": "Eastern Asia" }, + "me": { "name": "Montenegro", + "region": "Europe", + "sub-region": "Southern Europe" }, + "ms": { "name": "Montserrat", + "region": "Americas", + "sub-region": "Caribbean" }, + "ma": { "name": "Morocco", + "region": "Africa", + "sub-region": "Northern Africa" }, + "mz": { "name": "Mozambique", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "mm": { "name": "Myanmar", + "region": "Asia", + "sub-region": "South-Eastern Asia" }, + "na": { "name": "Namibia", + "region": "Africa", + "sub-region": "Southern Africa" }, + "nr": { "name": "Nauru", + "region": "Oceania", + "sub-region": "Micronesia" }, + "np": { "name": "Nepal", + "region": "Asia", + "sub-region": "Southern Asia" }, + "nl": { "name": "Netherlands", + "region": "Europe", + "sub-region": "Western Europe" }, + "nc": { "name": "New Caledonia", + "region": "Oceania", + "sub-region": "Melanesia" }, + "nz": { "name": "New Zealand", + "region": "Oceania", + "sub-region": "Australia and New Zealand" }, + "ni": { "name": "Nicaragua", + "region": "Americas", + "sub-region": "Central America" }, + "ne": { "name": "Niger", + "region": "Africa", + "sub-region": "Western Africa" }, + "ng": { "name": "Nigeria", + "region": "Africa", + "sub-region": "Western Africa" }, + "nf": { "name": "Norfolk Island", + "region": "Oceania", + "sub-region": "Australia and New Zealand" }, + "mp": { "name": "Northern Mariana Islands", + "region": "Oceania", + "sub-region": "Micronesia" }, + "no": { "name": "Norway", + "region": "Europe", + "sub-region": "Northern Europe" }, + "om": { "name": "Oman", + "region": "Asia", + "sub-region": "Western Asia" }, + "pk": { "name": "Pakistan", + "region": "Asia", + "sub-region": "Southern Asia" }, + "pw": { "name": "Palau", + "region": "Oceania", + "sub-region": "Micronesia" }, + "ps": { "name": "Palestine, State of", + "region": "Asia", + "sub-region": "Western Asia" }, + "pa": { "name": "Panama", + "region": "Americas", + "sub-region": "Central America" }, + "pg": { "name": "Papua New Guinea", + "region": "Oceania", + "sub-region": "Melanesia" }, + "py": { "name": "Paraguay", + "region": "Americas", + "sub-region": "South America" }, + "pe": { "name": "Peru", + "region": "Americas", + "sub-region": "South America" }, + "ph": { "name": "Philippines", + "region": "Asia", + "sub-region": "South-Eastern Asia" }, + "pl": { "name": "Poland", + "region": "Europe", + "sub-region": "Eastern Europe" }, + "pt": { "name": "Portugal", + "region": "Europe", + "sub-region": "Southern Europe" }, + "pr": { "name": "Puerto Rico", + "region": "Americas", + "sub-region": "Caribbean" }, + "qa": { "name": "Qatar", + "region": "Asia", + "sub-region": "Western Asia" }, + "re": { "name": "Réunion", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "ro": { "name": "Romania", + "region": "Europe", + "sub-region": "Eastern Europe" }, + "ru": { "name": "Russian Federation", + "region": "Europe", + "sub-region": "Eastern Europe" }, + "rw": { "name": "Rwanda", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "bl": { "name": "Saint Barthélemy", + "region": "Americas", + "sub-region": "Caribbean" }, + "sh": { "name": "Saint Helena, Ascension and Tristan da Cunha", + "region": "Africa", + "sub-region": "Western Africa" }, + "kn": { "name": "Saint Kitts and Nevis", + "region": "Americas", + "sub-region": "Caribbean" }, + "lc": { "name": "Saint Lucia", + "region": "Americas", + "sub-region": "Caribbean" }, + "mf": { "name": "Saint Martin (French part)", + "region": "Americas", + "sub-region": "Caribbean" }, + "pm": { "name": "Saint Pierre and Miquelon", + "region": "Americas", + "sub-region": "Northern America" }, + "vc": { "name": "Saint Vincent and the Grenadines", + "region": "Americas", + "sub-region": "Caribbean" }, + "ws": { "name": "Samoa", + "region": "Oceania", + "sub-region": "Polynesia" }, + "sm": { "name": "San Marino", + "region": "Europe", + "sub-region": "Southern Europe" }, + "st": { "name": "Sao Tome and Principe", + "region": "Africa", + "sub-region": "Middle Africa" }, + "sa": { "name": "Saudi Arabia", + "region": "Asia", + "sub-region": "Western Asia" }, + "sn": { "name": "Senegal", + "region": "Africa", + "sub-region": "Western Africa" }, + "rs": { "name": "Serbia", + "region": "Europe", + "sub-region": "Southern Europe" }, + "sc": { "name": "Seychelles", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "sl": { "name": "Sierra Leone", + "region": "Africa", + "sub-region": "Western Africa" }, + "sg": { "name": "Singapore", + "region": "Asia", + "sub-region": "South-Eastern Asia" }, + "sx": { "name": "Sint Maarten (Dutch part)", + "region": "Americas", + "sub-region": "Caribbean" }, + "sk": { "name": "Slovakia", + "region": "Europe", + "sub-region": "Eastern Europe" }, + "si": { "name": "Slovenia", + "region": "Europe", + "sub-region": "Southern Europe" }, + "sb": { "name": "Solomon Islands", + "region": "Oceania", + "sub-region": "Melanesia" }, + "so": { "name": "Somalia", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "za": { "name": "South Africa", + "region": "Africa", + "sub-region": "Southern Africa" }, + "ss": { "name": "South Sudan", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "es": { "name": "Spain", + "region": "Europe", + "sub-region": "Southern Europe" }, + "lk": { "name": "Sri Lanka", + "region": "Asia", + "sub-region": "Southern Asia" }, + "sd": { "name": "Sudan", + "region": "Africa", + "sub-region": "Northern Africa" }, + "sr": { "name": "Suriname", + "region": "Americas", + "sub-region": "South America" }, + "sj": { "name": "Svalbard and Jan Mayen", + "region": "Europe", + "sub-region": "Northern Europe" }, + "sz": { "name": "Swaziland", + "region": "Africa", + "sub-region": "Southern Africa" }, + "se": { "name": "Sweden", + "region": "Europe", + "sub-region": "Northern Europe" }, + "ch": { "name": "Switzerland", + "region": "Europe", + "sub-region": "Western Europe" }, + "sy": { "name": "Syrian Arab Republic", + "region": "Asia", + "sub-region": "Western Asia" }, + "tw": { "name": "Taiwan, Province of China", + "region": "Asia", + "sub-region": "Eastern Asia" }, + "tj": { "name": "Tajikistan", + "region": "Asia", + "sub-region": "Central Asia" }, + "tz": { "name": "Tanzania, United Republic of", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "th": { "name": "Thailand", + "region": "Asia", + "sub-region": "South-Eastern Asia" }, + "tl": { "name": "Timor-Leste", + "region": "Asia", + "sub-region": "South-Eastern Asia" }, + "tg": { "name": "Togo", + "region": "Africa", + "sub-region": "Western Africa" }, + "to": { "name": "Tonga", + "region": "Oceania", + "sub-region": "Polynesia" }, + "tt": { "name": "Trinidad and Tobago", + "region": "Americas", + "sub-region": "Caribbean" }, + "tn": { "name": "Tunisia", + "region": "Africa", + "sub-region": "Northern Africa" }, + "tr": { "name": "Turkey", + "region": "Asia", + "sub-region": "Western Asia" }, + "tm": { "name": "Turkmenistan", + "region": "Asia", + "sub-region": "Central Asia" }, + "tc": { "name": "Turks and Caicos Islands", + "region": "Americas", + "sub-region": "Caribbean" }, + "tv": { "name": "Tuvalu", + "region": "Oceania", + "sub-region": "Polynesia" }, + "ug": { "name": "Uganda", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "ua": { "name": "Ukraine", + "region": "Europe", + "sub-region": "Eastern Europe" }, + "ae": { "name": "United Arab Emirates", + "region": "Asia", + "sub-region": "Western Asia" }, + "gb": { "name": "United Kingdom of Great Britain and Northern Ireland", + "region": "Europe", + "sub-region": "Northern Europe" }, + "us": { "name": "United States of America", + "region": "Americas", + "sub-region": "Northern America" }, + "uy": { "name": "Uruguay", + "region": "Americas", + "sub-region": "South America" }, + "uz": { "name": "Uzbekistan", + "region": "Asia", + "sub-region": "Central Asia" }, + "vu": { "name": "Vanuatu", + "region": "Oceania", + "sub-region": "Melanesia" }, + "ve": { "name": "Venezuela (Bolivarian Republic of)", + "region": "Americas", + "sub-region": "South America" }, + "vn": { "name": "Viet Nam", + "region": "Asia", + "sub-region": "South-Eastern Asia" }, + "vg": { "name": "Virgin Islands (British)", + "region": "Americas", + "sub-region": "Caribbean" }, + "vi": { "name": "Virgin Islands (U.S.)", + "region": "Americas", + "sub-region": "Caribbean" }, + "eh": { "name": "Western Sahara", + "region": "Africa", + "sub-region": "Northern Africa" }, + "ye": { "name": "Yemen", + "region": "Asia", + "sub-region": "Western Asia" }, + "zm": { "name": "Zambia", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "zw": { "name": "Zimbabwe", + "region": "Africa", + "sub-region": "Eastern Africa" }, + "xk": { "name": "Kosovo", + "region": "Europe", + "sub-region": "Eastern Europe" } }; return countries[country_code][geo]; '''; +WITH + geo_summary AS ( SELECT - GET_GEO(country_code, 'region') AS region, - IF(device = 'desktop', 'desktop', 'mobile') AS client, + GET_GEO(country_code, + 'region') AS region, + IF + (device = 'desktop', 'desktop', 'mobile') AS client, origin, - COUNT(DISTINCT origin) OVER (PARTITION BY GET_GEO(country_code, 'region'), IF(device = 'desktop', 'desktop', 'mobile')) AS total + COUNT(DISTINCT origin) OVER (PARTITION BY GET_GEO(country_code, 'region'), + IF + (device = 'desktop', 'desktop', 'mobile')) AS total FROM `chrome-ux-report.materialized.country_summary` WHERE - yyyymm = 202406 -) - + yyyymm = 202406 ) SELECT * FROM ( @@ -1219,22 +742,27 @@ FROM ( ANY_VALUE(total) AS total, COUNT(0) / ANY_VALUE(total) AS pct FROM ( - SELECT DISTINCT - region, + SELECT + DISTINCT region, client, total, - CONCAT(origin, '/') AS url + CONCAT(origin, '/') AS page FROM - geo_summary - ) JOIN ( - SELECT DISTINCT - _TABLE_SUFFIX AS client, - url + geo_summary ) + JOIN ( + SELECT + client, + page FROM - `httparchive.technologies.2024_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - category = 'CMS' - ) USING (client, url) + date = '2024-06-01' + AND cats= 'CMS' ) + USING + (client, + page) GROUP BY client, region) From dd03b17b09fe965d34ebad1d9705cc2174ff3a73 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Sat, 24 Aug 2024 10:01:44 +0200 Subject: [PATCH 05/77] Update cms_adoption_by_subregion.sql --- sql/2024/cms/cms_adoption_by_subregion.sql | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_subregion.sql b/sql/2024/cms/cms_adoption_by_subregion.sql index ec9fdf141e4..3dada5eaa65 100644 --- a/sql/2024/cms/cms_adoption_by_subregion.sql +++ b/sql/2024/cms/cms_adoption_by_subregion.sql @@ -1223,18 +1223,23 @@ FROM ( sub_region, client, total, - CONCAT(origin, '/') AS url + CONCAT(origin, '/') AS page FROM geo_summary ) JOIN ( - SELECT DISTINCT - _TABLE_SUFFIX AS client, - url + SELECT + client, + page FROM - `httparchive.technologies.2024_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - category = 'CMS' - ) USING (client, url) + date = '2024-06-01' + AND cats= 'CMS' ) + USING + (client, + page) GROUP BY client, sub_region) From ecce24c30ab0b002797233fbc20f549e2f5f95cb Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Sat, 24 Aug 2024 11:02:40 +0200 Subject: [PATCH 06/77] Update cms_adoption.sql --- sql/2024/cms/cms_adoption.sql | 151 ++++++++++------------------------ 1 file changed, 44 insertions(+), 107 deletions(-) diff --git a/sql/2024/cms/cms_adoption.sql b/sql/2024/cms/cms_adoption.sql index d219eb7352c..f93d0d60b56 100644 --- a/sql/2024/cms/cms_adoption.sql +++ b/sql/2024/cms/cms_adoption.sql @@ -1,150 +1,87 @@ -#standardSQL -# CMS adoption over time - +#standardSQL # CMS adoption OVER time SELECT - _TABLE_SUFFIX AS client, - 2024 AS year, - COUNT(DISTINCT url) AS freq, - total, - COUNT(DISTINCT url) / total AS pct -FROM - `httparchive.technologies.2024_06_01_*` -JOIN ( - SELECT - _TABLE_SUFFIX, - COUNT(0) AS total - FROM - `httparchive.summary_pages.2024_06_01_*` - GROUP BY - _TABLE_SUFFIX) -USING - (_TABLE_SUFFIX) -WHERE - category = 'CMS' -GROUP BY client, - total -UNION ALL - -SELECT - _TABLE_SUFFIX AS client, - 2023 AS year, - COUNT(DISTINCT url) AS freq, + 2024 AS year, + COUNT(DISTINCT page) AS freq, total, - COUNT(DISTINCT url) / total AS pct + COUNT(DISTINCT page) / total AS pct FROM - `httparchive.technologies.2023_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats JOIN ( SELECT - _TABLE_SUFFIX, + client, COUNT(0) AS total FROM - `httparchive.summary_pages.2023_06_01_*` + `httparchive.all.pages` + WHERE + date = '2024-06-01' GROUP BY - _TABLE_SUFFIX) + client) USING - (_TABLE_SUFFIX) + (client) WHERE - category = 'CMS' + cats = 'CMS' + AND date = '2024-06-01' GROUP BY client, - total + total UNION ALL - SELECT - _TABLE_SUFFIX AS client, - 2022 AS year, - COUNT(DISTINCT url) AS freq, - total, - COUNT(DISTINCT url) / total AS pct -FROM - `httparchive.technologies.2022_06_01_*` -JOIN ( - SELECT - _TABLE_SUFFIX, - COUNT(0) AS total - FROM - `httparchive.summary_pages.2022_06_01_*` - GROUP BY - _TABLE_SUFFIX) -USING - (_TABLE_SUFFIX) -WHERE - category = 'CMS' -GROUP BY client, - total -UNION ALL - - -SELECT - _TABLE_SUFFIX AS client, - 2021 AS year, - COUNT(DISTINCT url) AS freq, + 2023 AS year, + COUNT(DISTINCT page) AS freq, total, - COUNT(DISTINCT url) / total AS pct + COUNT(DISTINCT page) / total AS pct FROM - `httparchive.technologies.2021_07_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats JOIN ( SELECT - _TABLE_SUFFIX, + client, COUNT(0) AS total FROM - `httparchive.summary_pages.2021_07_01_*` + `httparchive.all.pages` + WHERE + date = '2023-06-01' GROUP BY - _TABLE_SUFFIX) + client) USING - (_TABLE_SUFFIX) + (client) WHERE - category = 'CMS' + cats = 'CMS' + AND date = '2023-06-01' GROUP BY client, total UNION ALL SELECT - _TABLE_SUFFIX AS client, - 2020 AS year, - COUNT(DISTINCT url) AS freq, - total, - COUNT(DISTINCT url) / total AS pct -FROM - `httparchive.technologies.2020_08_01_*` -JOIN ( - SELECT - _TABLE_SUFFIX, - COUNT(0) AS total - FROM - `httparchive.summary_pages.2020_08_01_*` - GROUP BY - _TABLE_SUFFIX) -USING - (_TABLE_SUFFIX) -WHERE - category = 'CMS' -GROUP BY client, - total -UNION ALL -SELECT - _TABLE_SUFFIX AS client, - 2019 AS year, - COUNT(DISTINCT url) AS freq, + 2022 AS year, + COUNT(DISTINCT page) AS freq, total, - COUNT(DISTINCT url) / total AS pct + COUNT(DISTINCT page) / total AS pct FROM - `httparchive.technologies.2019_07_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats JOIN ( SELECT - _TABLE_SUFFIX, + client, COUNT(0) AS total FROM - `httparchive.summary_pages.2019_07_01_*` + `httparchive.all.pages` + WHERE + date = '2022-06-01' GROUP BY - _TABLE_SUFFIX) + client) USING - (_TABLE_SUFFIX) + (client) WHERE - category = 'CMS' + cats = 'CMS' + AND date = '2022-06-01' GROUP BY client, total From 0e763c3e36dc40f202e0c3899743034e6ce66b41 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Sat, 24 Aug 2024 12:13:50 +0200 Subject: [PATCH 07/77] Update core_web_vitals_by_geo.sql --- sql/2024/cms/core_web_vitals_by_geo.sql | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sql/2024/cms/core_web_vitals_by_geo.sql b/sql/2024/cms/core_web_vitals_by_geo.sql index d2aa29de0cc..9f63d3c97c5 100644 --- a/sql/2024/cms/core_web_vitals_by_geo.sql +++ b/sql/2024/cms/core_web_vitals_by_geo.sql @@ -48,13 +48,16 @@ FROM ( device IN ('desktop', 'phone')) JOIN ( SELECT DISTINCT - _TABLE_SUFFIX AS client, - url, - app AS cms + client, + page as url, + technologies AS cms FROM - `httparchive.technologies.2024_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - category = 'CMS') + cats = 'CMS' and + date = "2024-06-01") USING (client, url) GROUP BY From e75f4754ba9fb8cfe6e3fadcdf754c348b5f1136 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Sat, 24 Aug 2024 12:26:25 +0200 Subject: [PATCH 08/77] Update core_web_vitals_by_geo.sql --- sql/2024/cms/core_web_vitals_by_geo.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/2024/cms/core_web_vitals_by_geo.sql b/sql/2024/cms/core_web_vitals_by_geo.sql index 9f63d3c97c5..f2d8129ddc7 100644 --- a/sql/2024/cms/core_web_vitals_by_geo.sql +++ b/sql/2024/cms/core_web_vitals_by_geo.sql @@ -1,4 +1,5 @@ # cms passing core web vitals +# core_web_vitals_by_geo CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( good / (good + needs_improvement + poor) >= 0.75 ); @@ -50,7 +51,7 @@ JOIN ( SELECT DISTINCT client, page as url, - technologies AS cms + technologies.technology AS cms FROM `httparchive.all.pages`, UNNEST (technologies) AS technologies, From 264e0c40148a4fa60608139fed020a9f64f874f3 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Sat, 24 Aug 2024 12:35:19 +0200 Subject: [PATCH 09/77] Update core_web_vitals_yoy.sql --- sql/2024/cms/core_web_vitals_yoy.sql | 51 +++++++++++++++++----------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/sql/2024/cms/core_web_vitals_yoy.sql b/sql/2024/cms/core_web_vitals_yoy.sql index aa49e7d54a0..0c354d4ca0e 100644 --- a/sql/2024/cms/core_web_vitals_yoy.sql +++ b/sql/2024/cms/core_web_vitals_yoy.sql @@ -1,4 +1,5 @@ # cms passing core web vitals +# core_web_vitals_yoy.sql CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( good / (good + needs_improvement + poor) >= 0.75 ); @@ -17,21 +18,22 @@ SELECT COUNT(DISTINCT IF(IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_good_lcp, - # Origins with good FID divided by origins with any FID. + # Origins with good INP divided by origins with any INP. SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_good_fid, + COUNT(DISTINCT IF(IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_good_fid, + # Origins with good CLS divided by origins with any CLS. SAFE_DIVIDE( COUNT(DISTINCT IF(IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_good_cls, - # Origins with good LCP, FID (optional), and CLS divided by origins with any LCP and CLS. + # Origins with good LCP, INP (optional), and CLS divided by origins with any LCP and CLS. SAFE_DIVIDE( COUNT(DISTINCT IF( IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND - IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND + IS_GOOD(fast_inp, avg_inp, slow_inp) IS NOT FALSE AND IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), COUNT(DISTINCT IF( IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND @@ -40,13 +42,16 @@ FROM `chrome-ux-report.materialized.device_summary` JOIN ( SELECT - _TABLE_SUFFIX AS client, - url, - app AS cms + client, + page url, + technologies.technology AS cms FROM - `httparchive.technologies.2024_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - category = 'CMS') + cats = 'CMS' + and date ="2024-06-01") ON CONCAT(origin, '/') = url AND IF(device = 'desktop', 'desktop', 'mobile') = client @@ -92,13 +97,16 @@ FROM `chrome-ux-report.materialized.device_summary` JOIN ( SELECT - _TABLE_SUFFIX AS client, - url, - app AS cms + client, + page as url, + technologies.technology AS cms FROM - `httparchive.technologies.2023_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - category = 'CMS') + cats = 'CMS' + and date ="2024-06-01") ON CONCAT(origin, '/') = url AND IF(device = 'desktop', 'desktop', 'mobile') = client @@ -144,13 +152,16 @@ FROM `chrome-ux-report.materialized.device_summary` JOIN ( SELECT - _TABLE_SUFFIX AS client, - url, - app AS cms + client, + page as url, + technologies.technology AS cms FROM - `httparchive.technologies.2022_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - category = 'CMS') + cats = 'CMS' + and date ="2024-06-01") ON CONCAT(origin, '/') = url AND IF(device = 'desktop', 'desktop', 'mobile') = client From db7ef12308b745e46d4f885e50e5adeab9283c3f Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Sat, 24 Aug 2024 12:37:21 +0200 Subject: [PATCH 10/77] Update core_web_vitals_yoy.sql --- sql/2024/cms/core_web_vitals_yoy.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2024/cms/core_web_vitals_yoy.sql b/sql/2024/cms/core_web_vitals_yoy.sql index 0c354d4ca0e..1b30b286d2f 100644 --- a/sql/2024/cms/core_web_vitals_yoy.sql +++ b/sql/2024/cms/core_web_vitals_yoy.sql @@ -106,7 +106,7 @@ JOIN ( UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' - and date ="2024-06-01") + and date ="2023-06-01") ON CONCAT(origin, '/') = url AND IF(device = 'desktop', 'desktop', 'mobile') = client @@ -161,7 +161,7 @@ JOIN ( UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' - and date ="2024-06-01") + and date ="2022-06-01") ON CONCAT(origin, '/') = url AND IF(device = 'desktop', 'desktop', 'mobile') = client From 3dab3d8ba4e514bb27d67774ccf6b48523e04244 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Sat, 24 Aug 2024 12:45:06 +0200 Subject: [PATCH 11/77] Update core_web_vitals_yoy.sql --- sql/2024/cms/core_web_vitals_yoy.sql | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sql/2024/cms/core_web_vitals_yoy.sql b/sql/2024/cms/core_web_vitals_yoy.sql index 1b30b286d2f..e01388ddce8 100644 --- a/sql/2024/cms/core_web_vitals_yoy.sql +++ b/sql/2024/cms/core_web_vitals_yoy.sql @@ -21,7 +21,7 @@ SELECT # Origins with good INP divided by origins with any INP. SAFE_DIVIDE( COUNT(DISTINCT IF(IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_good_fid, + COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_good_inp, # Origins with good CLS divided by origins with any CLS. @@ -74,21 +74,21 @@ SELECT COUNT(DISTINCT IF(IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_good_lcp, - # Origins with good FID divided by origins with any FID. + # Origins with good INP divided by origins with any INP. SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_good_fid, + COUNT(DISTINCT IF(IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_good_inp, # Origins with good CLS divided by origins with any CLS. SAFE_DIVIDE( COUNT(DISTINCT IF(IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_good_cls, - # Origins with good LCP, FID (optional), and CLS divided by origins with any LCP and CLS. + # Origins with good LCP, INP (optional), and CLS divided by origins with any LCP and CLS. SAFE_DIVIDE( COUNT(DISTINCT IF( IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND - IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND + IS_GOOD(fast_inp, avg_inp, slow_inp) IS NOT FALSE AND IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), COUNT(DISTINCT IF( IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND @@ -129,21 +129,21 @@ SELECT COUNT(DISTINCT IF(IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_good_lcp, - # Origins with good FID divided by origins with any FID. + # Origins with good INP divided by origins with any INP. SAFE_DIVIDE( - COUNT(DISTINCT IF(IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), - COUNT(DISTINCT IF(IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_good_fid, + COUNT(DISTINCT IF(IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_good_inp, # Origins with good CLS divided by origins with any CLS. SAFE_DIVIDE( COUNT(DISTINCT IF(IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_good_cls, - # Origins with good LCP, FID (optional), and CLS divided by origins with any LCP and CLS. + # Origins with good LCP, INP (optional), and CLS divided by origins with any LCP and CLS. SAFE_DIVIDE( COUNT(DISTINCT IF( IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND - IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND + IS_GOOD(fast_inp, avg_inp, slow_inp) IS NOT FALSE AND IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), COUNT(DISTINCT IF( IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND From b2273850a01e7180284233dafe6aefbcec7527d9 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Sat, 24 Aug 2024 13:53:42 +0200 Subject: [PATCH 12/77] Update image_format_popularity.sql --- sql/2024/cms/image_format_popularity.sql | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sql/2024/cms/image_format_popularity.sql b/sql/2024/cms/image_format_popularity.sql index ba07be36c17..be03d208a60 100644 --- a/sql/2024/cms/image_format_popularity.sql +++ b/sql/2024/cms/image_format_popularity.sql @@ -44,13 +44,16 @@ USING (client, url) JOIN ( SELECT - _TABLE_SUFFIX AS client, - app AS cms, - COUNT(DISTINCT url) AS pages + client, + technologies.technology AS cms, + COUNT(DISTINCT page) AS pages FROM - `httparchive.technologies.2024_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - category = 'CMS' + cats = 'CMS' and + date = "2024-06-01" GROUP BY client, cms) From 62ac99ac1e30b5ca7bdf8dc8b9734152d8cd3b1d Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Sat, 24 Aug 2024 13:59:23 +0200 Subject: [PATCH 13/77] Update image_format_popularity.sql --- sql/2024/cms/image_format_popularity.sql | 65 +++++++++++++----------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/sql/2024/cms/image_format_popularity.sql b/sql/2024/cms/image_format_popularity.sql index be03d208a60..8e6552e4f4e 100644 --- a/sql/2024/cms/image_format_popularity.sql +++ b/sql/2024/cms/image_format_popularity.sql @@ -1,5 +1,4 @@ -#standardSQL -# Image format popularity by CMS +#standardSQL # Image format popularity BY CMS SELECT client, cms, @@ -9,56 +8,62 @@ SELECT SUM(COUNT(0)) OVER (PARTITION BY client, cms) AS total, COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client, cms) AS pct FROM ( - SELECT DISTINCT - _TABLE_SUFFIX AS client, - url, - app AS cms + SELECT + DISTINCT client, + page AS url, + technologies.technology AS cms FROM - `httparchive.technologies.2024_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - category = 'CMS') + cats = 'CMS' + AND date = "2024-06-01" ) JOIN ( - SELECT + SELECT client, page AS url, - CASE - WHEN json_value(summary, "$.mimeType") = 'image/avif' THEN 'avif' - WHEN json_value(summary, "$.mimeType") = 'image/webp' THEN 'webp' - WHEN json_value(summary, "$.mimeType") = 'image/jpeg' THEN 'jpg' - WHEN json_value(summary, "$.mimeType") = 'image/png' THEN 'png' - WHEN json_value(summary, "$.mimeType") = 'image/gif' THEN 'gif' - WHEN json_value(summary, "$.mimeType") = 'image/svg+xml' THEN 'svg' - WHEN json_value(summary, "$.mimeType") = 'image/x-icon' THEN 'ico' - WHEN json_value(summary, "$.mimeType") = 'image/vnd.microsoft.icon' THEN 'ico' - WHEN json_value(summary, "$.mimeType") = 'image/jpg' THEN 'jpg' - WHEN json_value(summary, "$.mimeType") = 'image/bmp' THEN 'bmp' - WHEN json_value(summary, "$.mimeType") = 'binary/octet-stream' THEN 'binary/octet-stream' - ELSE 'other/unknown' -- To handle any unexpected formats - END AS format + CASE + WHEN JSON_VALUE(summary, "$.mimeType") = 'image/avif' THEN 'avif' + WHEN JSON_VALUE(summary, "$.mimeType") = 'image/webp' THEN 'webp' + WHEN JSON_VALUE(summary, "$.mimeType") = 'image/jpeg' THEN 'jpg' + WHEN JSON_VALUE(summary, "$.mimeType") = 'image/png' THEN 'png' + WHEN JSON_VALUE(summary, "$.mimeType") = 'image/gif' THEN 'gif' + WHEN JSON_VALUE(summary, "$.mimeType") = 'image/svg+xml' THEN 'svg' + WHEN JSON_VALUE(summary, "$.mimeType") = 'image/x-icon' THEN 'ico' + WHEN JSON_VALUE(summary, "$.mimeType") = 'image/vnd.microsoft.icon' THEN 'ico' + WHEN JSON_VALUE(summary, "$.mimeType") = 'image/jpg' THEN 'jpg' + WHEN JSON_VALUE(summary, "$.mimeType") = 'image/bmp' THEN 'bmp' + WHEN JSON_VALUE(summary, "$.mimeType") = 'binary/octet-stream' THEN 'binary/octet-stream' + ELSE 'other/unknown' -- TO handle ANY unexpected formats + END + AS format FROM httparchive.all.requests WHERE - date = '2024-06-01' AND - type = 'image') + date = '2024-06-01' + AND type = 'image') USING - (client, url) + (client, + url) JOIN ( SELECT client, technologies.technology AS cms, COUNT(DISTINCT page) AS pages FROM - `httparchive.all.pages`, + `httparchive.all.pages`, UNNEST (technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE - cats = 'CMS' and - date = "2024-06-01" + cats = 'CMS' + AND date = "2024-06-01" GROUP BY client, cms) USING - (client, cms) + (client, + cms) WHERE pages > 1000 GROUP BY From 712a594aa81715bd6dfb1b1f89ac7a8504f6319e Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Sat, 24 Aug 2024 14:07:19 +0200 Subject: [PATCH 14/77] Update lighthouse_category_scores_per_cms.sql --- .../lighthouse_category_scores_per_cms.sql | 47 ++++++++++++------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/sql/2024/cms/lighthouse_category_scores_per_cms.sql b/sql/2024/cms/lighthouse_category_scores_per_cms.sql index 2e0af7a87e2..e6d5395a464 100644 --- a/sql/2024/cms/lighthouse_category_scores_per_cms.sql +++ b/sql/2024/cms/lighthouse_category_scores_per_cms.sql @@ -1,31 +1,46 @@ -#standardSQL +#standardSQL # Lighthouse category scores per CMS +# lighthouse_category_scores_per_cms.sql SELECT client, cms, COUNT(DISTINCT url) AS freq, - APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.performance.score') AS NUMERIC), 1000)[OFFSET(500)] * 100 AS median_performance, - APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.accessibility.score') AS NUMERIC), 1000)[OFFSET(500)] * 100 AS median_accessibility, - APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.seo.score') AS NUMERIC), 1000)[OFFSET(500)] * 100 AS median_seo, - APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$."best-practices".score') AS NUMERIC), 1000)[OFFSET(500)] * 100 AS median_best_practices + APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.performance.score') AS NUMERIC), 1000)[ +OFFSET + (500)] * 100 AS median_performance, + APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.accessibility.score') AS NUMERIC), 1000)[ +OFFSET + (500)] * 100 AS median_accessibility, + APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.seo.score') AS NUMERIC), 1000)[ +OFFSET + (500)] * 100 AS median_seo, + APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$."best-practices".score') AS NUMERIC), 1000)[ +OFFSET + (500)] * 100 AS median_best_practices FROM ( SELECT - _TABLE_SUFFIX AS client, - url, - JSON_EXTRACT(report, '$.categories') AS categories + client, + page AS url, + JSON_EXTRACT(lighthouse, '$.categories') AS categories FROM - `httparchive.lighthouse.2024_06_01_*`) + `httparchive.all.pages` + WHERE + date="2024-06-01") JOIN ( - SELECT DISTINCT - _TABLE_SUFFIX AS client, - app AS cms, - url + SELECT + DISTINCT client, + technologies.technology AS cms, + page AS url FROM - `httparchive.technologies.2024_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - category = 'CMS') + cats = 'CMS' + AND date ="2024-06-01") USING - (url, client) + (url, + client) GROUP BY client, cms From 4c096b5919b9bf8b4dd8ba651f514d160a1be16f Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Sat, 24 Aug 2024 14:14:32 +0200 Subject: [PATCH 15/77] Update page_weight_distribution.sql --- sql/2024/cms/page_weight_distribution.sql | 37 ++++++++++++++--------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/sql/2024/cms/page_weight_distribution.sql b/sql/2024/cms/page_weight_distribution.sql index a407f961db4..b7e6c233e2c 100644 --- a/sql/2024/cms/page_weight_distribution.sql +++ b/sql/2024/cms/page_weight_distribution.sql @@ -1,29 +1,38 @@ -#standardSQL -# Total page weight distribution by CMS +#standardSQL +# Total page weight distribution BY CMS +# page_weight_distribution.sql SELECT percentile, client, cms, COUNT(0) AS pages, - APPROX_QUANTILES(total_kb, 1000)[OFFSET(percentile * 10)] AS total_kb + APPROX_QUANTILES(total_kb, 1000)[ +OFFSET + (percentile * 10)] AS total_kb FROM ( - SELECT DISTINCT - _TABLE_SUFFIX AS client, - url, - app AS cms + SELECT + DISTINCT client, + page AS url, + technologies.technology AS cms FROM - `httparchive.technologies.2024_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - category = 'CMS') + cats = 'CMS' + AND date="2024-06-01") JOIN ( SELECT - _TABLE_SUFFIX AS client, - url, - bytesTotal / 1024 AS total_kb + client, + page AS url, + cast(json_value(summary, "$.bytesTotal") as int64) / 1024 AS total_kb FROM - `httparchive.summary_pages.2024_06_01_*`) + `httparchive.all.pages` + WHERE + date="2024-06-01") USING - (client, url), + (client, + url), UNNEST([10, 25, 50, 75, 90, 100]) AS percentile GROUP BY percentile, From 9e73147edcd5f2f1236c3867e424d6561a00fdf0 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:13:16 +0200 Subject: [PATCH 16/77] Update page_weight_distribution.sql Fixing linter issues. --- sql/2024/cms/page_weight_distribution.sql | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/sql/2024/cms/page_weight_distribution.sql b/sql/2024/cms/page_weight_distribution.sql index b7e6c233e2c..f4604243691 100644 --- a/sql/2024/cms/page_weight_distribution.sql +++ b/sql/2024/cms/page_weight_distribution.sql @@ -6,12 +6,10 @@ SELECT client, cms, COUNT(0) AS pages, - APPROX_QUANTILES(total_kb, 1000)[ -OFFSET - (percentile * 10)] AS total_kb + APPROX_QUANTILES(total_kb, 1000)[OFFSET(percentile * 10)] AS total_kb FROM ( - SELECT - DISTINCT client, + SELECT DISTINCT + client, page AS url, technologies.technology AS cms FROM @@ -20,16 +18,19 @@ FROM ( UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' - AND date="2024-06-01") + AND + date= '2024-06-01' + ) JOIN ( SELECT client, page AS url, - cast(json_value(summary, "$.bytesTotal") as int64) / 1024 AS total_kb + cast(json_value(summary, '$.bytesTotal') as int64) / 1024 AS total_kb FROM `httparchive.all.pages` WHERE - date="2024-06-01") + date = '2024-06-01' +) USING (client, url), From abd613c4b700f1f62489088761bac250e99eed03 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:28:12 +0200 Subject: [PATCH 17/77] Update page_weight_distribution.sql Fixing linting. --- sql/2024/cms/page_weight_distribution.sql | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sql/2024/cms/page_weight_distribution.sql b/sql/2024/cms/page_weight_distribution.sql index f4604243691..b8160ae97ec 100644 --- a/sql/2024/cms/page_weight_distribution.sql +++ b/sql/2024/cms/page_weight_distribution.sql @@ -14,18 +14,17 @@ FROM ( technologies.technology AS cms FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE - cats = 'CMS' - AND - date= '2024-06-01' + cats = 'CMS' AND + date = '2024-06-01' ) JOIN ( SELECT client, page AS url, - cast(json_value(summary, '$.bytesTotal') as int64) / 1024 AS total_kb + cast(json_value(summary, '$.bytesTotal') AS INT64) / 1024 AS total_kb FROM `httparchive.all.pages` WHERE From eded9a6875c19bb77e26fef4d744821bc87bb15e Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:32:04 +0200 Subject: [PATCH 18/77] Update cms_adoption_by_geo.sql Fixing the linter errors. --- sql/2024/cms/cms_adoption_by_geo.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_geo.sql b/sql/2024/cms/cms_adoption_by_geo.sql index cb013f444c9..3496b1cea2b 100644 --- a/sql/2024/cms/cms_adoption_by_geo.sql +++ b/sql/2024/cms/cms_adoption_by_geo.sql @@ -8,7 +8,7 @@ WITH geo_summary AS ( COUNT(DISTINCT origin) OVER (PARTITION BY country_code, IF(device = 'desktop', 'desktop', 'mobile')) AS total FROM `chrome-ux-report.materialized.country_summary` - WHERE + WHERE yyyymm = 202406 ) SELECT @@ -30,7 +30,7 @@ FROM ( geo_summary ) JOIN ( SELECT - client, + client, page FROM `httparchive.all.pages`, @@ -38,7 +38,8 @@ FROM ( UNNEST(technologies.categories) AS cats WHERE date = '2024-06-01' - AND cats= 'CMS' ) + AND cats= 'CMS' + ) USING (client, page) From daaef3068b776771a082d5339843d6d8074334a2 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:33:30 +0200 Subject: [PATCH 19/77] Update cms_adoption_by_subregion.sql Linter errors --- sql/2024/cms/cms_adoption_by_subregion.sql | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_subregion.sql b/sql/2024/cms/cms_adoption_by_subregion.sql index 3dada5eaa65..c9be3dfa0d7 100644 --- a/sql/2024/cms/cms_adoption_by_subregion.sql +++ b/sql/2024/cms/cms_adoption_by_subregion.sql @@ -1227,16 +1227,17 @@ FROM ( FROM geo_summary ) JOIN ( - SELECT - client, - page + SELECT + client, + page FROM `httparchive.all.pages`, UNNEST (technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE date = '2024-06-01' - AND cats= 'CMS' ) + AND cats= 'CMS' + ) USING (client, page) From 30d58fc3323c6fd327e617df4363a8a758fbdffc Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:36:48 +0200 Subject: [PATCH 20/77] Update core_web_vitals_yoy.sql Fixing linter errors. --- sql/2024/cms/core_web_vitals_yoy.sql | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/sql/2024/cms/core_web_vitals_yoy.sql b/sql/2024/cms/core_web_vitals_yoy.sql index e01388ddce8..82b62a169a7 100644 --- a/sql/2024/cms/core_web_vitals_yoy.sql +++ b/sql/2024/cms/core_web_vitals_yoy.sql @@ -22,7 +22,6 @@ SELECT SAFE_DIVIDE( COUNT(DISTINCT IF(IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_good_inp, - # Origins with good CLS divided by origins with any CLS. SAFE_DIVIDE( @@ -50,8 +49,9 @@ JOIN ( UNNEST (technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE - cats = 'CMS' - and date ="2024-06-01") + cats = 'CMS' AND + date = '2024-06-01' +) ON CONCAT(origin, '/') = url AND IF(device = 'desktop', 'desktop', 'mobile') = client @@ -60,9 +60,8 @@ WHERE GROUP BY client, cms -UNION ALL - +UNION ALL SELECT 2023 AS year, @@ -98,15 +97,16 @@ FROM JOIN ( SELECT client, - page as url, + page AS url, technologies.technology AS cms FROM `httparchive.all.pages`, UNNEST (technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE - cats = 'CMS' - and date ="2023-06-01") + cats = 'CMS' AND + date = '2023-06-01' +) ON CONCAT(origin, '/') = url AND IF(device = 'desktop', 'desktop', 'mobile') = client @@ -116,8 +116,7 @@ GROUP BY client, cms - -union all +UNION ALL SELECT 2022 AS year, @@ -160,8 +159,9 @@ JOIN ( UNNEST (technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE - cats = 'CMS' - and date ="2022-06-01") + cats = 'CMS' AND + date = '2022-06-01' +) ON CONCAT(origin, '/') = url AND IF(device = 'desktop', 'desktop', 'mobile') = client From 71d9f3315d1774cfd7118c667984e3747914e368 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:38:28 +0200 Subject: [PATCH 21/77] Update lighthouse_category_scores_per_cms.sql Fixing linter errors. --- sql/2024/cms/lighthouse_category_scores_per_cms.sql | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sql/2024/cms/lighthouse_category_scores_per_cms.sql b/sql/2024/cms/lighthouse_category_scores_per_cms.sql index e6d5395a464..c51e63d6242 100644 --- a/sql/2024/cms/lighthouse_category_scores_per_cms.sql +++ b/sql/2024/cms/lighthouse_category_scores_per_cms.sql @@ -1,4 +1,4 @@ -#standardSQL +#standardSQL # Lighthouse category scores per CMS # lighthouse_category_scores_per_cms.sql SELECT @@ -25,7 +25,8 @@ FROM ( FROM `httparchive.all.pages` WHERE - date="2024-06-01") + date = '2024-06-01' +) JOIN ( SELECT DISTINCT client, @@ -36,8 +37,9 @@ JOIN ( UNNEST (technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE - cats = 'CMS' - AND date ="2024-06-01") + cats = 'CMS' AND + date = '2024-06-01' +) USING (url, client) From a689f97827d0bcb97ee9a13e1ba2c5cf135b49b3 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:39:17 +0200 Subject: [PATCH 22/77] Update page_weight_distribution.sql removing white space. --- sql/2024/cms/page_weight_distribution.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/cms/page_weight_distribution.sql b/sql/2024/cms/page_weight_distribution.sql index b8160ae97ec..70e50762b99 100644 --- a/sql/2024/cms/page_weight_distribution.sql +++ b/sql/2024/cms/page_weight_distribution.sql @@ -1,4 +1,4 @@ -#standardSQL +#standardSQL # Total page weight distribution BY CMS # page_weight_distribution.sql SELECT From ac5a7a06fc37a38a4cb5b2be7726082a3c1ca975 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:41:46 +0200 Subject: [PATCH 23/77] Update cms_adoption.sql Fixing linting issues. --- sql/2024/cms/cms_adoption.sql | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sql/2024/cms/cms_adoption.sql b/sql/2024/cms/cms_adoption.sql index f93d0d60b56..a09f5f5220a 100644 --- a/sql/2024/cms/cms_adoption.sql +++ b/sql/2024/cms/cms_adoption.sql @@ -7,7 +7,7 @@ SELECT COUNT(DISTINCT page) / total AS pct FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats JOIN ( SELECT @@ -22,8 +22,8 @@ JOIN ( USING (client) WHERE - cats = 'CMS' - AND date = '2024-06-01' + cats = 'CMS' AND + date = '2024-06-01' GROUP BY client, total @@ -36,7 +36,7 @@ SELECT COUNT(DISTINCT page) / total AS pct FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats JOIN ( SELECT @@ -51,8 +51,8 @@ JOIN ( USING (client) WHERE - cats = 'CMS' - AND date = '2023-06-01' + cats = 'CMS' AND + date = '2023-06-01' GROUP BY client, total @@ -65,7 +65,7 @@ SELECT COUNT(DISTINCT page) / total AS pct FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats JOIN ( SELECT @@ -80,8 +80,8 @@ JOIN ( USING (client) WHERE - cats = 'CMS' - AND date = '2022-06-01' + cats = 'CMS' AND + date = '2022-06-01' GROUP BY client, total From e8f57c1f94fba3e512bca7d6f2bd52abcdb38907 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:45:05 +0200 Subject: [PATCH 24/77] Update cms_adoption_by_geo.sql Fixing linting issues. --- sql/2024/cms/cms_adoption_by_geo.sql | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_geo.sql b/sql/2024/cms/cms_adoption_by_geo.sql index 3496b1cea2b..d80dac2d1a5 100644 --- a/sql/2024/cms/cms_adoption_by_geo.sql +++ b/sql/2024/cms/cms_adoption_by_geo.sql @@ -21,24 +21,25 @@ FROM ( ANY_VALUE(total) AS total, COUNT(0) / ANY_VALUE(total) AS pct FROM ( - SELECT - DISTINCT geo, + SELECT DISTINCT + geo, client, total, CONCAT(origin, '/') AS page FROM - geo_summary ) + geo_summary + ) JOIN ( SELECT client, page FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE - date = '2024-06-01' - AND cats= 'CMS' + date = '2024-06-01' AND + cats= 'CMS' ) USING (client, From 762dbcb9c87a9919b242148fe0ff0ac470c80cc4 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:47:22 +0200 Subject: [PATCH 25/77] Update cms_adoption_by_rank.sql Fixing linting issues. --- sql/2024/cms/cms_adoption_by_rank.sql | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_rank.sql b/sql/2024/cms/cms_adoption_by_rank.sql index 1743891d68a..acff2c53964 100644 --- a/sql/2024/cms/cms_adoption_by_rank.sql +++ b/sql/2024/cms/cms_adoption_by_rank.sql @@ -7,17 +7,18 @@ SELECT ANY_VALUE(total) AS total, COUNT(DISTINCT page) / ANY_VALUE(total) AS pct FROM ( - SELECT - DISTINCT client, + SELECT DISTINCT + client, technology, page FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE - date = '2024-06-01' - AND cats= 'CMS' ) + date = '2024-06-01' AND + cats= 'CMS' +) JOIN ( SELECT client, @@ -26,7 +27,8 @@ JOIN ( FROM `httparchive.all.pages` WHERE - date = '2024-06-01' ) + date = '2024-06-01' +) USING (client, page) @@ -39,8 +41,8 @@ JOIN ( `httparchive.all.pages`, UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude WHERE - rank <= rank_magnitude - AND date = '2024-06-01' + rank <= rank_magnitude AND + date = '2024-06-01' GROUP BY client, rank_magnitude) From f2e1a388900355afdefb38425b330595ead43bde Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:51:46 +0200 Subject: [PATCH 26/77] Update cms_adoption_by_region.sql Conforming with linter. --- sql/2024/cms/cms_adoption_by_region.sql | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_region.sql b/sql/2024/cms/cms_adoption_by_region.sql index 6fccf04584f..6e5012a29d2 100644 --- a/sql/2024/cms/cms_adoption_by_region.sql +++ b/sql/2024/cms/cms_adoption_by_region.sql @@ -1,9 +1,8 @@ #standardSQL # ALL CMS popularity per geo CREATE TEMP FUNCTION - GET_GEO(country_code STRING, - geo STRING) - RETURNS STRING - LANGUAGE js AS ''' var countries = { "af": { "name": "Afghanistan", +GET_GEO(country_code STRING, geo STRING) +RETURNS STRING +LANGUAGE js AS ''' var countries = { "af": { "name": "Afghanistan", "region": "Asia", "sub-region": "Southern Asia" }, "ax": { "name": "Åland Islands", @@ -719,19 +718,20 @@ CREATE TEMP FUNCTION "sub-region": "Eastern Europe" } }; return countries[country_code][geo]; '''; WITH geo_summary AS ( - SELECT - GET_GEO(country_code, +SELECT + GET_GEO(country_code, 'region') AS region, IF (device = 'desktop', 'desktop', 'mobile') AS client, origin, COUNT(DISTINCT origin) OVER (PARTITION BY GET_GEO(country_code, 'region'), - IF + IF (device = 'desktop', 'desktop', 'mobile')) AS total FROM `chrome-ux-report.materialized.country_summary` WHERE - yyyymm = 202406 ) + yyyymm = 202406 + ) SELECT * FROM ( @@ -748,7 +748,8 @@ FROM ( total, CONCAT(origin, '/') AS page FROM - geo_summary ) + geo_summary + ) JOIN ( SELECT client, @@ -758,8 +759,9 @@ FROM ( UNNEST (technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE - date = '2024-06-01' - AND cats= 'CMS' ) + date = '2024-06-01' AND + cats= 'CMS' + ) USING (client, page) From 5c281efef7b82a223173b46978d460be9f10b41d Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:53:57 +0200 Subject: [PATCH 27/77] Update cms_adoption_by_subregion.sql Fixing linter errors --- sql/2024/cms/cms_adoption_by_subregion.sql | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_subregion.sql b/sql/2024/cms/cms_adoption_by_subregion.sql index c9be3dfa0d7..a7139bac7eb 100644 --- a/sql/2024/cms/cms_adoption_by_subregion.sql +++ b/sql/2024/cms/cms_adoption_by_subregion.sql @@ -1226,24 +1226,26 @@ FROM ( CONCAT(origin, '/') AS page FROM geo_summary - ) JOIN ( - SELECT - client, - page + ) + JOIN ( + SELECT + client, + page FROM `httparchive.all.pages`, UNNEST (technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE - date = '2024-06-01' - AND cats= 'CMS' + date = '2024-06-01' AND + cats = 'CMS' ) USING (client, page) GROUP BY client, - sub_region) + sub_region +) WHERE pages > 1000 ORDER BY From 2018a311d7b6e4998904ce2514caa1d50fdd1763 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:55:41 +0200 Subject: [PATCH 28/77] Update core_web_vitals_by_geo.sql Linting errors. --- sql/2024/cms/core_web_vitals_by_geo.sql | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sql/2024/cms/core_web_vitals_by_geo.sql b/sql/2024/cms/core_web_vitals_by_geo.sql index f2d8129ddc7..006b0bddadf 100644 --- a/sql/2024/cms/core_web_vitals_by_geo.sql +++ b/sql/2024/cms/core_web_vitals_by_geo.sql @@ -50,15 +50,16 @@ FROM ( JOIN ( SELECT DISTINCT client, - page as url, + page AS url, technologies.technology AS cms FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE - cats = 'CMS' and - date = "2024-06-01") + cats = 'CMS' AND + date = '2024-06-01' +) USING (client, url) GROUP BY From 54ab614225452f068119711a80add2b509f44f15 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:59:54 +0200 Subject: [PATCH 29/77] Update core_web_vitals_yoy.sql Fixing linting --- sql/2024/cms/core_web_vitals_yoy.sql | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/sql/2024/cms/core_web_vitals_yoy.sql b/sql/2024/cms/core_web_vitals_yoy.sql index 82b62a169a7..2328461e792 100644 --- a/sql/2024/cms/core_web_vitals_yoy.sql +++ b/sql/2024/cms/core_web_vitals_yoy.sql @@ -22,7 +22,6 @@ SELECT SAFE_DIVIDE( COUNT(DISTINCT IF(IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_good_inp, - # Origins with good CLS divided by origins with any CLS. SAFE_DIVIDE( COUNT(DISTINCT IF(IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), @@ -46,7 +45,7 @@ JOIN ( technologies.technology AS cms FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' AND @@ -101,7 +100,7 @@ JOIN ( technologies.technology AS cms FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' AND @@ -152,10 +151,10 @@ FROM JOIN ( SELECT client, - page as url, + page AS url, technologies.technology AS cms FROM - `httparchive.all.pages`, + `httparchive.all.pages`, UNNEST (technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE @@ -170,6 +169,5 @@ WHERE GROUP BY client, cms - - ORDER BY +ORDER BY origins DESC From ba856fb14325b31c3c46cb0f74193f57261facd2 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:02:05 +0200 Subject: [PATCH 30/77] Update image_format_popularity.sql Fixing linting --- sql/2024/cms/image_format_popularity.sql | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sql/2024/cms/image_format_popularity.sql b/sql/2024/cms/image_format_popularity.sql index 8e6552e4f4e..3593a559512 100644 --- a/sql/2024/cms/image_format_popularity.sql +++ b/sql/2024/cms/image_format_popularity.sql @@ -8,8 +8,7 @@ SELECT SUM(COUNT(0)) OVER (PARTITION BY client, cms) AS total, COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client, cms) AS pct FROM ( - SELECT - DISTINCT client, + SELECT DISTINCT client, page AS url, technologies.technology AS cms FROM @@ -18,7 +17,8 @@ FROM ( UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' - AND date = "2024-06-01" ) + AND date = '2024-06-01' +) JOIN ( SELECT client, @@ -42,7 +42,8 @@ JOIN ( httparchive.all.requests WHERE date = '2024-06-01' - AND type = 'image') + AND type = 'image' +) USING (client, url) @@ -57,7 +58,7 @@ JOIN ( UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' - AND date = "2024-06-01" + AND date = '2024-06-01' GROUP BY client, cms) From c0e7e2562513f42288250437f784de8a74ca0cba Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:03:54 +0200 Subject: [PATCH 31/77] Update lighthouse_category_scores_per_cms.sql Updating for linting. --- sql/2024/cms/lighthouse_category_scores_per_cms.sql | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/sql/2024/cms/lighthouse_category_scores_per_cms.sql b/sql/2024/cms/lighthouse_category_scores_per_cms.sql index c51e63d6242..0744e9d5632 100644 --- a/sql/2024/cms/lighthouse_category_scores_per_cms.sql +++ b/sql/2024/cms/lighthouse_category_scores_per_cms.sql @@ -6,17 +6,13 @@ SELECT cms, COUNT(DISTINCT url) AS freq, APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.performance.score') AS NUMERIC), 1000)[ -OFFSET - (500)] * 100 AS median_performance, + OFFSET(500)] * 100 AS median_performance, APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.accessibility.score') AS NUMERIC), 1000)[ -OFFSET - (500)] * 100 AS median_accessibility, + OFFSET(500)] * 100 AS median_accessibility, APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.seo.score') AS NUMERIC), 1000)[ -OFFSET - (500)] * 100 AS median_seo, + OFFSET(500)] * 100 AS median_seo, APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$."best-practices".score') AS NUMERIC), 1000)[ -OFFSET - (500)] * 100 AS median_best_practices + OFFSET(500)] * 100 AS median_best_practices FROM ( SELECT client, From bd734bd3288523efb0907cf29c60203fd6e71652 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:07:49 +0200 Subject: [PATCH 32/77] Update lighthouse_category_scores_per_cms.sql Linting... --- .../cms/lighthouse_category_scores_per_cms.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sql/2024/cms/lighthouse_category_scores_per_cms.sql b/sql/2024/cms/lighthouse_category_scores_per_cms.sql index 0744e9d5632..b3172be2cde 100644 --- a/sql/2024/cms/lighthouse_category_scores_per_cms.sql +++ b/sql/2024/cms/lighthouse_category_scores_per_cms.sql @@ -6,13 +6,13 @@ SELECT cms, COUNT(DISTINCT url) AS freq, APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.performance.score') AS NUMERIC), 1000)[ - OFFSET(500)] * 100 AS median_performance, + OFFSET(500)] * 100 AS median_performance, APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.accessibility.score') AS NUMERIC), 1000)[ - OFFSET(500)] * 100 AS median_accessibility, + OFFSET(500)] * 100 AS median_accessibility, APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.seo.score') AS NUMERIC), 1000)[ - OFFSET(500)] * 100 AS median_seo, + OFFSET(500)] * 100 AS median_seo, APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$."best-practices".score') AS NUMERIC), 1000)[ - OFFSET(500)] * 100 AS median_best_practices + OFFSET(500)] * 100 AS median_best_practices FROM ( SELECT client, @@ -24,13 +24,13 @@ FROM ( date = '2024-06-01' ) JOIN ( - SELECT - DISTINCT client, + SELECT DISTINCT + client, technologies.technology AS cms, page AS url FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' AND From a6dd16c074c5e5c293960eba04e8631b3906e6b5 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:11:23 +0200 Subject: [PATCH 33/77] Update image_format_popularity.sql Linting issues. --- sql/2024/cms/image_format_popularity.sql | 38 ++++++++++++------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/sql/2024/cms/image_format_popularity.sql b/sql/2024/cms/image_format_popularity.sql index 3593a559512..44128169b99 100644 --- a/sql/2024/cms/image_format_popularity.sql +++ b/sql/2024/cms/image_format_popularity.sql @@ -13,7 +13,7 @@ FROM ( technologies.technology AS cms FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' @@ -24,25 +24,25 @@ JOIN ( client, page AS url, CASE - WHEN JSON_VALUE(summary, "$.mimeType") = 'image/avif' THEN 'avif' - WHEN JSON_VALUE(summary, "$.mimeType") = 'image/webp' THEN 'webp' - WHEN JSON_VALUE(summary, "$.mimeType") = 'image/jpeg' THEN 'jpg' - WHEN JSON_VALUE(summary, "$.mimeType") = 'image/png' THEN 'png' - WHEN JSON_VALUE(summary, "$.mimeType") = 'image/gif' THEN 'gif' - WHEN JSON_VALUE(summary, "$.mimeType") = 'image/svg+xml' THEN 'svg' - WHEN JSON_VALUE(summary, "$.mimeType") = 'image/x-icon' THEN 'ico' - WHEN JSON_VALUE(summary, "$.mimeType") = 'image/vnd.microsoft.icon' THEN 'ico' - WHEN JSON_VALUE(summary, "$.mimeType") = 'image/jpg' THEN 'jpg' - WHEN JSON_VALUE(summary, "$.mimeType") = 'image/bmp' THEN 'bmp' - WHEN JSON_VALUE(summary, "$.mimeType") = 'binary/octet-stream' THEN 'binary/octet-stream' + WHEN JSON_VALUE(summary, '$.mimeType') = 'image/avif' THEN 'avif' + WHEN JSON_VALUE(summary, '$.mimeType') = 'image/webp' THEN 'webp' + WHEN JSON_VALUE(summary, '$.mimeType') = 'image/jpeg' THEN 'jpg' + WHEN JSON_VALUE(summary, '$.mimeType') = 'image/png' THEN 'png' + WHEN JSON_VALUE(summary, '$.mimeType') = 'image/gif' THEN 'gif' + WHEN JSON_VALUE(summary, '$.mimeType') = 'image/svg+xml' THEN 'svg' + WHEN JSON_VALUE(summary, '$.mimeType') = 'image/x-icon' THEN 'ico' + WHEN JSON_VALUE(summary, '$.mimeType') = 'image/vnd.microsoft.icon' THEN 'ico' + WHEN JSON_VALUE(summary, '$.mimeType') = 'image/jpg' THEN 'jpg' + WHEN JSON_VALUE(summary, '$.mimeType') = 'image/bmp' THEN 'bmp' + WHEN JSON_VALUE(summary, '$.mimeType') = 'binary/octet-stream' THEN 'binary/octet-stream' ELSE 'other/unknown' -- TO handle ANY unexpected formats - END + END AS format FROM - httparchive.all.requests + `httparchive.all.requests` WHERE - date = '2024-06-01' - AND type = 'image' + date = '2024-06-01' AND + type = 'image' ) USING (client, @@ -54,11 +54,11 @@ JOIN ( COUNT(DISTINCT page) AS pages FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE - cats = 'CMS' - AND date = '2024-06-01' + cats = 'CMS' AND + date = '2024-06-01' GROUP BY client, cms) From f5123a529dc7c585ba24968751352e12e524dfd7 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:14:38 +0200 Subject: [PATCH 34/77] Update core_web_vitals_yoy.sql Linting --- sql/2024/cms/core_web_vitals_yoy.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/cms/core_web_vitals_yoy.sql b/sql/2024/cms/core_web_vitals_yoy.sql index 2328461e792..7bb1723e936 100644 --- a/sql/2024/cms/core_web_vitals_yoy.sql +++ b/sql/2024/cms/core_web_vitals_yoy.sql @@ -155,7 +155,7 @@ JOIN ( technologies.technology AS cms FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' AND From 7a38765c0fed82d3a0e62c293950136af755f392 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:17:39 +0200 Subject: [PATCH 35/77] Update cms_adoption_by_subregion.sql Linting --- sql/2024/cms/cms_adoption_by_subregion.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_subregion.sql b/sql/2024/cms/cms_adoption_by_subregion.sql index a7139bac7eb..cc30b6f5014 100644 --- a/sql/2024/cms/cms_adoption_by_subregion.sql +++ b/sql/2024/cms/cms_adoption_by_subregion.sql @@ -1226,14 +1226,14 @@ FROM ( CONCAT(origin, '/') AS page FROM geo_summary - ) + ) JOIN ( SELECT client, page FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE date = '2024-06-01' AND From bdb9870577791fffbdc30ba14f006cdb30accb9c Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:20:08 +0200 Subject: [PATCH 36/77] Update cms_adoption_by_region.sql Linting --- sql/2024/cms/cms_adoption_by_region.sql | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_region.sql b/sql/2024/cms/cms_adoption_by_region.sql index 6e5012a29d2..60b54feff91 100644 --- a/sql/2024/cms/cms_adoption_by_region.sql +++ b/sql/2024/cms/cms_adoption_by_region.sql @@ -717,16 +717,14 @@ LANGUAGE js AS ''' var countries = { "af": { "name": "Afghanistan", "region": "Europe", "sub-region": "Eastern Europe" } }; return countries[country_code][geo]; '''; WITH - geo_summary AS ( -SELECT +geo_summary AS ( + SELECT GET_GEO(country_code, 'region') AS region, - IF - (device = 'desktop', 'desktop', 'mobile') AS client, + IF(device = 'desktop', 'desktop', 'mobile') AS client, origin, COUNT(DISTINCT origin) OVER (PARTITION BY GET_GEO(country_code, 'region'), - IF - (device = 'desktop', 'desktop', 'mobile')) AS total + IF(device = 'desktop', 'desktop', 'mobile')) AS total FROM `chrome-ux-report.materialized.country_summary` WHERE From 850a6bfc1e849f88b0eb9db170c4e9dd5f5e21bb Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:20:48 +0200 Subject: [PATCH 37/77] Update cms_adoption_by_geo.sql Linting --- sql/2024/cms/cms_adoption_by_geo.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_geo.sql b/sql/2024/cms/cms_adoption_by_geo.sql index d80dac2d1a5..5db4bc16652 100644 --- a/sql/2024/cms/cms_adoption_by_geo.sql +++ b/sql/2024/cms/cms_adoption_by_geo.sql @@ -21,7 +21,7 @@ FROM ( ANY_VALUE(total) AS total, COUNT(0) / ANY_VALUE(total) AS pct FROM ( - SELECT DISTINCT + SELECT DISTINCT geo, client, total, @@ -39,7 +39,7 @@ FROM ( UNNEST(technologies.categories) AS cats WHERE date = '2024-06-01' AND - cats= 'CMS' + cats = 'CMS' ) USING (client, From 6da6fe8d4bcb4da26e844497cebee22d41920742 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:22:14 +0200 Subject: [PATCH 38/77] Update cms_adoption_by_subregion.sql Fixing whitespace issue --- sql/2024/cms/cms_adoption_by_subregion.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/cms/cms_adoption_by_subregion.sql b/sql/2024/cms/cms_adoption_by_subregion.sql index cc30b6f5014..e42ee8639cc 100644 --- a/sql/2024/cms/cms_adoption_by_subregion.sql +++ b/sql/2024/cms/cms_adoption_by_subregion.sql @@ -1226,7 +1226,7 @@ FROM ( CONCAT(origin, '/') AS page FROM geo_summary - ) +) JOIN ( SELECT client, From eae26b629bbed29fe86c133fe502246c06773521 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:22:58 +0200 Subject: [PATCH 39/77] Update cms_adoption_by_rank.sql Whitespace --- sql/2024/cms/cms_adoption_by_rank.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_rank.sql b/sql/2024/cms/cms_adoption_by_rank.sql index acff2c53964..84e9c6849af 100644 --- a/sql/2024/cms/cms_adoption_by_rank.sql +++ b/sql/2024/cms/cms_adoption_by_rank.sql @@ -7,7 +7,7 @@ SELECT ANY_VALUE(total) AS total, COUNT(DISTINCT page) / ANY_VALUE(total) AS pct FROM ( - SELECT DISTINCT + SELECT DISTINCT client, technology, page @@ -16,8 +16,8 @@ FROM ( UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE - date = '2024-06-01' AND - cats= 'CMS' + date = '2024-06-01' AND + cats= 'CMS' ) JOIN ( SELECT @@ -41,7 +41,7 @@ JOIN ( `httparchive.all.pages`, UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude WHERE - rank <= rank_magnitude AND + rank <= rank_magnitude AND date = '2024-06-01' GROUP BY client, From 7c3711fd60c2c93ca6f869ccf4d9957a60858bbf Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:24:27 +0200 Subject: [PATCH 40/77] Update cms_adoption_by_geo.sql Spacing --- sql/2024/cms/cms_adoption_by_geo.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/cms/cms_adoption_by_geo.sql b/sql/2024/cms/cms_adoption_by_geo.sql index 5db4bc16652..dfe354e5520 100644 --- a/sql/2024/cms/cms_adoption_by_geo.sql +++ b/sql/2024/cms/cms_adoption_by_geo.sql @@ -28,7 +28,7 @@ FROM ( CONCAT(origin, '/') AS page FROM geo_summary - ) +) JOIN ( SELECT client, From a269f8f82da47b92470059d5bd6ad47fd802e198 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:28:10 +0200 Subject: [PATCH 41/77] Update lighthouse_category_scores_per_cms.sql spacing --- sql/2024/cms/lighthouse_category_scores_per_cms.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/cms/lighthouse_category_scores_per_cms.sql b/sql/2024/cms/lighthouse_category_scores_per_cms.sql index b3172be2cde..47e74a00880 100644 --- a/sql/2024/cms/lighthouse_category_scores_per_cms.sql +++ b/sql/2024/cms/lighthouse_category_scores_per_cms.sql @@ -24,7 +24,7 @@ FROM ( date = '2024-06-01' ) JOIN ( - SELECT DISTINCT + SELECT DISTINCT client, technologies.technology AS cms, page AS url From e7b511769ad207a03975a4de4dd06c95cdf749b9 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:28:58 +0200 Subject: [PATCH 42/77] Update image_format_popularity.sql Linting --- sql/2024/cms/image_format_popularity.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2024/cms/image_format_popularity.sql b/sql/2024/cms/image_format_popularity.sql index 44128169b99..b25f0c7e29f 100644 --- a/sql/2024/cms/image_format_popularity.sql +++ b/sql/2024/cms/image_format_popularity.sql @@ -16,8 +16,8 @@ FROM ( UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE - cats = 'CMS' - AND date = '2024-06-01' + cats = 'CMS' AND + date = '2024-06-01' ) JOIN ( SELECT From fc5c01e20658e945df255d86d38e0ee0c0e244fe Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:31:31 +0200 Subject: [PATCH 43/77] Update cms_adoption_by_subregion.sql Linting.. --- sql/2024/cms/cms_adoption_by_subregion.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/cms/cms_adoption_by_subregion.sql b/sql/2024/cms/cms_adoption_by_subregion.sql index e42ee8639cc..cc30b6f5014 100644 --- a/sql/2024/cms/cms_adoption_by_subregion.sql +++ b/sql/2024/cms/cms_adoption_by_subregion.sql @@ -1226,7 +1226,7 @@ FROM ( CONCAT(origin, '/') AS page FROM geo_summary -) + ) JOIN ( SELECT client, From bb7b07a0ef7f7651783fa0d3b503bca704ab5edc Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:34:24 +0200 Subject: [PATCH 44/77] Update cms_adoption_by_region.sql Linting --- sql/2024/cms/cms_adoption_by_region.sql | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_region.sql b/sql/2024/cms/cms_adoption_by_region.sql index 60b54feff91..0c4afceef4b 100644 --- a/sql/2024/cms/cms_adoption_by_region.sql +++ b/sql/2024/cms/cms_adoption_by_region.sql @@ -719,17 +719,17 @@ LANGUAGE js AS ''' var countries = { "af": { "name": "Afghanistan", WITH geo_summary AS ( SELECT - GET_GEO(country_code, + GET_GEO(country_code, 'region') AS region, - IF(device = 'desktop', 'desktop', 'mobile') AS client, + IF(device = 'desktop', 'desktop', 'mobile') AS client, origin, COUNT(DISTINCT origin) OVER (PARTITION BY GET_GEO(country_code, 'region'), - IF(device = 'desktop', 'desktop', 'mobile')) AS total + IF(device = 'desktop', 'desktop', 'mobile')) AS total FROM `chrome-ux-report.materialized.country_summary` WHERE yyyymm = 202406 - ) +) SELECT * FROM ( @@ -740,8 +740,7 @@ FROM ( ANY_VALUE(total) AS total, COUNT(0) / ANY_VALUE(total) AS pct FROM ( - SELECT - DISTINCT region, + SELECT DISTINCT region, client, total, CONCAT(origin, '/') AS page @@ -754,11 +753,11 @@ FROM ( page FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE date = '2024-06-01' AND - cats= 'CMS' + cats = 'CMS' ) USING (client, From 3af474cd06bff5ac07da9c518b46de95190b8d1b Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:34:52 +0200 Subject: [PATCH 45/77] Update cms_adoption_by_rank.sql spaces --- sql/2024/cms/cms_adoption_by_rank.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/cms/cms_adoption_by_rank.sql b/sql/2024/cms/cms_adoption_by_rank.sql index 84e9c6849af..91918498f2e 100644 --- a/sql/2024/cms/cms_adoption_by_rank.sql +++ b/sql/2024/cms/cms_adoption_by_rank.sql @@ -17,7 +17,7 @@ FROM ( UNNEST(technologies.categories) AS cats WHERE date = '2024-06-01' AND - cats= 'CMS' + cats = 'CMS' ) JOIN ( SELECT From eb33b2162f5e88571b502f5deeee95fcd095b90d Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:35:44 +0200 Subject: [PATCH 46/77] Update cms_adoption_by_geo.sql Spaces --- sql/2024/cms/cms_adoption_by_geo.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/cms/cms_adoption_by_geo.sql b/sql/2024/cms/cms_adoption_by_geo.sql index dfe354e5520..5db4bc16652 100644 --- a/sql/2024/cms/cms_adoption_by_geo.sql +++ b/sql/2024/cms/cms_adoption_by_geo.sql @@ -28,7 +28,7 @@ FROM ( CONCAT(origin, '/') AS page FROM geo_summary -) + ) JOIN ( SELECT client, From 61c15fa742a866bde45ae6307dca4f123c119bc1 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:41:46 +0200 Subject: [PATCH 47/77] Update cms_adoption_by_region.sql Indenting spaces --- sql/2024/cms/cms_adoption_by_region.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/cms/cms_adoption_by_region.sql b/sql/2024/cms/cms_adoption_by_region.sql index 0c4afceef4b..b9b38e5e9f9 100644 --- a/sql/2024/cms/cms_adoption_by_region.sql +++ b/sql/2024/cms/cms_adoption_by_region.sql @@ -724,7 +724,7 @@ geo_summary AS ( IF(device = 'desktop', 'desktop', 'mobile') AS client, origin, COUNT(DISTINCT origin) OVER (PARTITION BY GET_GEO(country_code, 'region'), - IF(device = 'desktop', 'desktop', 'mobile')) AS total + IF(device = 'desktop', 'desktop', 'mobile')) AS total FROM `chrome-ux-report.materialized.country_summary` WHERE From 52d93154d64978fb9397123773608bcba9fe3647 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 21:55:52 +0200 Subject: [PATCH 48/77] Update core_web_vitals_yoy.sql Fixing Alias --- sql/2024/cms/core_web_vitals_yoy.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/cms/core_web_vitals_yoy.sql b/sql/2024/cms/core_web_vitals_yoy.sql index 7bb1723e936..3dd9fd4932f 100644 --- a/sql/2024/cms/core_web_vitals_yoy.sql +++ b/sql/2024/cms/core_web_vitals_yoy.sql @@ -41,7 +41,7 @@ FROM JOIN ( SELECT client, - page url, + page AS url, technologies.technology AS cms FROM `httparchive.all.pages`, From 6d2a461924c2aa95a4baeadd665347217281481b Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:29:20 +0200 Subject: [PATCH 49/77] Update resource_weights.sql --- sql/2024/cms/resource_weights.sql | 35 ++++++++++++++++++------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/sql/2024/cms/resource_weights.sql b/sql/2024/cms/resource_weights.sql index b1dc6842488..cf1fdd25a0e 100644 --- a/sql/2024/cms/resource_weights.sql +++ b/sql/2024/cms/resource_weights.sql @@ -1,5 +1,7 @@ #standardSQL # Median resource weights by CMS +#resource_weights.sql + SELECT client, cms, @@ -12,25 +14,30 @@ SELECT APPROX_QUANTILES(font_kb, 1000)[OFFSET(500)] AS median_font_kb FROM ( SELECT DISTINCT - _TABLE_SUFFIX AS client, - url, - app AS cms + client, + page as url, + technologies.technology AS cms FROM - `httparchive.technologies.2024_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - category = 'CMS') + cats = 'CMS' + AND date ='2024-06-01') JOIN ( SELECT - _TABLE_SUFFIX AS client, - url, - bytesTotal / 1024 AS total_kb, - bytesHtml / 1024 AS html_kb, - bytesJS / 1024 AS js_kb, - bytesCSS / 1024 AS css_kb, - bytesImg / 1024 AS img_kb, - bytesFont / 1024 AS font_kb + client, + page as url, + cast(json_value(summary, "$.bytesTotal") as int64) / 1024 AS total_kb, + cast(json_value(summary, "$.bytesHtml") as int64) / 1024 AS html_kb, + cast(json_value(summary, "$.bytesJS") as int64) / 1024 AS js_kb, + cast(json_value(summary, "$.bytesCSS") as int64) / 1024 AS css_kb, + cast(json_value(summary, "$.bytesImg") as int64) / 1024 AS img_kb, + cast(json_value(summary, "$.bytesFont") as int64) / 1024 AS font_kb FROM - `httparchive.summary_pages.2024_06_01_*`) + `httparchive.all.pages` + WHERE + date = '2024-06-01') USING (client, url) GROUP BY From 66d581542b0ebdb1a761e5e6df525e07226ba63a Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:41:02 +0200 Subject: [PATCH 50/77] Update top_cms.sql --- sql/2024/cms/top_cms.sql | 107 +++++++++++++++++++++++---------------- 1 file changed, 64 insertions(+), 43 deletions(-) diff --git a/sql/2024/cms/top_cms.sql b/sql/2024/cms/top_cms.sql index e009a7597d6..5737d33287b 100644 --- a/sql/2024/cms/top_cms.sql +++ b/sql/2024/cms/top_cms.sql @@ -1,104 +1,125 @@ #standardSQL # Top CMS platforms, compared to 2020 +# top_cms.sql SELECT - _TABLE_SUFFIX AS client, + client, 2024 AS year, - app AS cms, - COUNT(DISTINCT url) AS freq, + technologies.technology AS cms, + COUNT(DISTINCT page) AS freq, total, - COUNT(DISTINCT url) / total AS pct + COUNT(DISTINCT page) / total AS pct FROM - `httparchive.technologies.2024_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats JOIN ( SELECT - _TABLE_SUFFIX, + client, COUNT(0) AS total FROM - `httparchive.summary_pages.2024_06_01_*` + `httparchive.all.pages` + WHERE + date = '2024-06-01' GROUP BY - _TABLE_SUFFIX) + client) USING - (_TABLE_SUFFIX) + (client) WHERE - category = 'CMS' + cats = 'CMS' AND + date = '2024-06-01' GROUP BY client, total, - cms + cms UNION ALL SELECT - _TABLE_SUFFIX AS client, + client, 2023 AS year, - app AS cms, - COUNT(DISTINCT url) AS freq, + technologies.technology AS cms, + COUNT(DISTINCT page) AS freq, total, - COUNT(DISTINCT url) / total AS pct + COUNT(DISTINCT page) / total AS pct FROM - `httparchive.technologies.2023_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats JOIN ( SELECT - _TABLE_SUFFIX, + client, COUNT(0) AS total FROM - `httparchive.summary_pages.2023_06_01_*` + `httparchive.all.pages` + WHERE + date = '2023-06-01' GROUP BY - _TABLE_SUFFIX) + client) USING - (_TABLE_SUFFIX) + (client) WHERE - category = 'CMS' + cats = 'CMS' AND + date = '2023-06-01' GROUP BY client, total, - cms + cms UNION ALL SELECT - _TABLE_SUFFIX AS client, + client, 2022 AS year, - app AS cms, - COUNT(DISTINCT url) AS freq, + technologies.technology AS cms, + COUNT(DISTINCT page) AS freq, total, - COUNT(DISTINCT url) / total AS pct + COUNT(DISTINCT page) / total AS pct FROM - `httparchive.technologies.2022_08_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats JOIN ( SELECT - _TABLE_SUFFIX, + client, COUNT(0) AS total FROM - `httparchive.summary_pages.2022_08_01_*` + `httparchive.all.pages` + WHERE + date = '2022-08-01' GROUP BY - _TABLE_SUFFIX) + client) USING - (_TABLE_SUFFIX) + (client) WHERE - category = 'CMS' + cats = 'CMS' AND + date = '2022-08-01' GROUP BY client, total, - cms + cms UNION ALL SELECT - _TABLE_SUFFIX AS client, + client, 2021 AS year, - app AS cms, - COUNT(DISTINCT url) AS freq, + technologies.technology AS cms, + COUNT(DISTINCT page) AS freq, total, - COUNT(DISTINCT url) / total AS pct + COUNT(DISTINCT page) / total AS pct FROM - `httparchive.technologies.2021_07_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats JOIN ( SELECT - _TABLE_SUFFIX, + client, COUNT(0) AS total FROM - `httparchive.summary_pages.2021_07_01_*` + `httparchive.all.pages` + WHERE + date = '2021-07-01' GROUP BY - _TABLE_SUFFIX) + client) USING - (_TABLE_SUFFIX) + (client) WHERE - category = 'CMS' + cats = 'CMS' AND + date = '2021-07-01' GROUP BY client, total, From cb1ad494e5f5a7c1376751e97c1f4cf92a6355ee Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:44:08 +0200 Subject: [PATCH 51/77] Update top_cms_by_geo.sql --- sql/2024/cms/top_cms_by_geo.sql | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/sql/2024/cms/top_cms_by_geo.sql b/sql/2024/cms/top_cms_by_geo.sql index c2a3eaa463f..45b26d05092 100644 --- a/sql/2024/cms/top_cms_by_geo.sql +++ b/sql/2024/cms/top_cms_by_geo.sql @@ -9,11 +9,6 @@ WITH geo_summary AS ( FROM `chrome-ux-report.materialized.country_summary` WHERE - # We're intentionally using April 2022 CrUX data here. - # That's because there's a two month lag between CrUX and HA datasets. - # Since we're only JOINing with the CrUX dataset to see which URLs - # belong to different countries (as opposed to CWV field data) - # it's not necessary to look at the 202207 dataset. yyyymm = 202406 UNION ALL SELECT @@ -47,16 +42,19 @@ FROM ( geo_summary ) JOIN ( SELECT DISTINCT - _TABLE_SUFFIX AS client, - category, - app AS cms, - url + client, + cats, + technologies.technology AS cms, + page as url FROM - `httparchive.technologies.2024_06_01_*` + `httparchive.all.pages`, + UNNEST (technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - app IS NOT NULL AND - category = 'CMS' AND - app != '' + technologies.technology IS NOT NULL AND + cats = 'CMS' AND + technologies.technology != '' AND + date = '2024-06-01' ) USING (client, url) GROUP BY client, From 9de9b225c2d39c63f60526edd0f0310f0d865f52 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:44:28 +0200 Subject: [PATCH 52/77] Update top_cms_by_geo.sql --- sql/2024/cms/top_cms_by_geo.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/2024/cms/top_cms_by_geo.sql b/sql/2024/cms/top_cms_by_geo.sql index 45b26d05092..3fe27788348 100644 --- a/sql/2024/cms/top_cms_by_geo.sql +++ b/sql/2024/cms/top_cms_by_geo.sql @@ -1,5 +1,6 @@ #standardSQL # CMS popularity per geo +# top_cms_by_geo.sql WITH geo_summary AS ( SELECT `chrome-ux-report`.experimental.GET_COUNTRY(country_code) AS geo, From 65838202c00796b4fac9ed50bb6d901548fcc2be Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Tue, 27 Aug 2024 13:01:43 +0200 Subject: [PATCH 53/77] Update resource_weights.sql --- sql/2024/cms/resource_weights.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/cms/resource_weights.sql b/sql/2024/cms/resource_weights.sql index cf1fdd25a0e..efc1d86d683 100644 --- a/sql/2024/cms/resource_weights.sql +++ b/sql/2024/cms/resource_weights.sql @@ -31,7 +31,7 @@ JOIN ( cast(json_value(summary, "$.bytesTotal") as int64) / 1024 AS total_kb, cast(json_value(summary, "$.bytesHtml") as int64) / 1024 AS html_kb, cast(json_value(summary, "$.bytesJS") as int64) / 1024 AS js_kb, - cast(json_value(summary, "$.bytesCSS") as int64) / 1024 AS css_kb, + cast(json_value(summary, "$.bytesCss") as int64) / 1024 AS css_kb, cast(json_value(summary, "$.bytesImg") as int64) / 1024 AS img_kb, cast(json_value(summary, "$.bytesFont") as int64) / 1024 AS font_kb FROM From 25a3cd175996033108da24e39c33c73eea8733dc Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Tue, 27 Aug 2024 13:17:22 +0200 Subject: [PATCH 54/77] Update top_cms_by_rank.sql --- sql/2024/cms/top_cms_by_rank.sql | 33 +++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/sql/2024/cms/top_cms_by_rank.sql b/sql/2024/cms/top_cms_by_rank.sql index c01ddb4931f..ba4ef0e8535 100644 --- a/sql/2024/cms/top_cms_by_rank.sql +++ b/sql/2024/cms/top_cms_by_rank.sql @@ -1,5 +1,7 @@ #standardSQL # CMS adoption per rank +# top_cms_by_rank.sql + SELECT client, cms, @@ -9,37 +11,42 @@ SELECT COUNT(DISTINCT url) / ANY_VALUE(total) AS pct FROM ( SELECT DISTINCT - _TABLE_SUFFIX AS client, - app AS cms, - url + client, + page AS url, + technologies.technology AS cms FROM - `httparchive.technologies.2024_06_01_*` + `httparchive.all.pages`, + UNNEST(technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - category = 'CMS') + cats = 'CMS' AND + date = '2024-06-01') JOIN ( SELECT - _TABLE_SUFFIX AS client, - url, + client, + page as url, rank_magnitude AS rank FROM - `httparchive.summary_pages.2024_06_01_*`, + `httparchive.all.pages`, UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude WHERE - rank <= rank_magnitude) + rank <= rank_magnitude AND + date = '2024-06-01') USING (client, url) JOIN ( SELECT - _TABLE_SUFFIX AS client, + client, rank_magnitude AS rank, COUNT(0) AS total FROM - `httparchive.summary_pages.2024_06_01_*`, + `httparchive.all.pages`, UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude WHERE - rank <= rank_magnitude + rank <= rank_magnitude AND + date = '2024-06-01' GROUP BY - _TABLE_SUFFIX, + client, rank_magnitude) USING (client, rank) From 35e71ff9b966b4095d506360adfe20b799a4a5d7 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Tue, 27 Aug 2024 13:24:14 +0200 Subject: [PATCH 55/77] Update wordpress_page_builders.sql --- sql/2024/cms/wordpress_page_builders.sql | 26 ++++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/sql/2024/cms/wordpress_page_builders.sql b/sql/2024/cms/wordpress_page_builders.sql index fe9fe08eb08..14fced8ee8e 100644 --- a/sql/2024/cms/wordpress_page_builders.sql +++ b/sql/2024/cms/wordpress_page_builders.sql @@ -1,5 +1,7 @@ #standardSQL # Top WordPress page builder combinations +# wordpress_page_builders.sql + SELECT client, page_builders, @@ -8,21 +10,27 @@ SELECT COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct FROM ( SELECT DISTINCT - _TABLE_SUFFIX AS client, - url + client, + page as url FROM - `httparchive.technologies.2024_06_01_*` + `httparchive.all.pages`, + UNNEST(technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - app = 'WordPress') + technologies.technology = 'WordPress' AND + date = '2024-06-01') JOIN ( SELECT - _TABLE_SUFFIX AS client, - url, - ARRAY_TO_STRING(ARRAY_AGG(app ORDER BY app), ', ') AS page_builders + client, + page as url, + ARRAY_TO_STRING(ARRAY_AGG(technologies.technology ORDER BY technologies.technology), ', ') AS page_builders FROM - `httparchive.technologies.2024_06_01_*` + `httparchive.all.pages`, + UNNEST(technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE - category = 'Page builders' + cats = 'Page builders' AND + date = '2024-06-01' GROUP BY client, url) From 18be4dd75b501f0a1fc0b360b5c4c40f7f6f8271 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Tue, 27 Aug 2024 13:39:20 +0200 Subject: [PATCH 56/77] Update wordpress_resources.sql --- sql/2024/cms/wordpress_resources.sql | 53 +++++++++++++++++++++------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/sql/2024/cms/wordpress_resources.sql b/sql/2024/cms/wordpress_resources.sql index 696572f20b7..d337825571d 100644 --- a/sql/2024/cms/wordpress_resources.sql +++ b/sql/2024/cms/wordpress_resources.sql @@ -1,26 +1,55 @@ -#standardSQL -# Distribution of WordPress resource types by path +#standardSQL +# Distribution OF WordPress resource types BY path +# wordpress_resources.sql + SELECT percentile, client, path, - APPROX_QUANTILES(freq, 1000)[OFFSET(percentile * 10)] AS freq + APPROX_QUANTILES(freq, 1000)[OFFSET (percentile * 10)] AS freq FROM ( SELECT - _TABLE_SUFFIX AS client, + client, page, REGEXP_EXTRACT(url, r'/(themes|plugins|wp-includes)/') AS path, COUNT(0) AS freq - FROM - (SELECT _TABLE_SUFFIX, url AS page FROM `httparchive.technologies.2024_06_01_*` WHERE app = 'WordPress') - JOIN - (SELECT _TABLE_SUFFIX, pageid, url AS page FROM `httparchive.summary_pages.2024_06_01_*`) + FROM ( + SELECT + client, + page + FROM + `httparchive.all.pages`, + UNNEST(technologies) AS technologies, + UNNEST(technologies.categories) AS cats + WHERE + technologies.technology = 'WordPress' AND + date = '2024-06-01') + JOIN ( + SELECT + client, + cast(json_value(summary, "$.pageid") as int64) as pageid, + page + FROM + `httparchive.all.pages`, + UNNEST(technologies) AS technologies, + UNNEST(technologies.categories) AS cats + WHERE + date = '2024-06-01') USING - (_TABLE_SUFFIX, page) - JOIN - (SELECT _TABLE_SUFFIX, pageid, url FROM `httparchive.summary_requests.2024_06_01_*`) + (client, + page) + JOIN ( + SELECT + client, + cast(json_value(summary, "$.pageid") as int64) as pageid, + page as url + FROM + `httparchive.all.pages` + WHERE + date = '2024-06-01') USING - (_TABLE_SUFFIX, pageid) + (client, + pageid) GROUP BY client, page, From edd8c0b43ee4123ae6e6a54bf26030652d623d40 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:18:58 +0200 Subject: [PATCH 57/77] Update cms_adoption.sql --- sql/2024/cms/cms_adoption.sql | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/sql/2024/cms/cms_adoption.sql b/sql/2024/cms/cms_adoption.sql index a09f5f5220a..c066acbcb4e 100644 --- a/sql/2024/cms/cms_adoption.sql +++ b/sql/2024/cms/cms_adoption.sql @@ -1,4 +1,7 @@ -#standardSQL # CMS adoption OVER time +#standardSQL +# CMS adoption OVER time +# cms_adoption.sql + SELECT client, 2024 AS year, @@ -16,14 +19,16 @@ JOIN ( FROM `httparchive.all.pages` WHERE - date = '2024-06-01' + date = '2024-06-01' AND + is_root_page GROUP BY client) USING (client) WHERE cats = 'CMS' AND - date = '2024-06-01' + date = '2024-06-01' AND + is_root_page GROUP BY client, total @@ -45,14 +50,16 @@ JOIN ( FROM `httparchive.all.pages` WHERE - date = '2023-06-01' + date = '2023-06-01' AND + is_root_page GROUP BY client) USING (client) WHERE cats = 'CMS' AND - date = '2023-06-01' + date = '2023-06-01' AND + is_root_page GROUP BY client, total @@ -74,14 +81,16 @@ JOIN ( FROM `httparchive.all.pages` WHERE - date = '2022-06-01' + date = '2022-06-01' AND + is_root_page GROUP BY client) USING (client) WHERE cats = 'CMS' AND - date = '2022-06-01' + date = '2022-06-01' AND + is_root_page GROUP BY client, total From 4af9d7fa059acc312b38db99a35c78dcb45d9250 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:19:21 +0200 Subject: [PATCH 58/77] Update cms_adoption_by_geo.sql --- sql/2024/cms/cms_adoption_by_geo.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/2024/cms/cms_adoption_by_geo.sql b/sql/2024/cms/cms_adoption_by_geo.sql index 5db4bc16652..241e03d9b1a 100644 --- a/sql/2024/cms/cms_adoption_by_geo.sql +++ b/sql/2024/cms/cms_adoption_by_geo.sql @@ -1,5 +1,7 @@ #standardSQL # All CMS popularity per geo +# cms_adoption_by_geo.sql + WITH geo_summary AS ( SELECT `chrome-ux-report`.experimental.GET_COUNTRY(country_code) AS geo, @@ -39,7 +41,8 @@ FROM ( UNNEST(technologies.categories) AS cats WHERE date = '2024-06-01' AND - cats = 'CMS' + cats = 'CMS' AND + is_root_page ) USING (client, From 6deec3efdcee0eb2e5186425aba680b3a399abfe Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:19:37 +0200 Subject: [PATCH 59/77] Update cms_adoption_by_rank.sql --- sql/2024/cms/cms_adoption_by_rank.sql | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_rank.sql b/sql/2024/cms/cms_adoption_by_rank.sql index 91918498f2e..6e3607e777a 100644 --- a/sql/2024/cms/cms_adoption_by_rank.sql +++ b/sql/2024/cms/cms_adoption_by_rank.sql @@ -1,4 +1,7 @@ -#standardSQL # CMS adoption per rank +#standardSQL +# CMS adoption per rank +# cms_adoption_by_rank.sql + SELECT client, technology, @@ -17,7 +20,8 @@ FROM ( UNNEST(technologies.categories) AS cats WHERE date = '2024-06-01' AND - cats = 'CMS' + cats = 'CMS' AND + is_root_page ) JOIN ( SELECT @@ -27,7 +31,8 @@ JOIN ( FROM `httparchive.all.pages` WHERE - date = '2024-06-01' + date = '2024-06-01' AND + is_root_page ) USING (client, @@ -42,7 +47,8 @@ JOIN ( UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude WHERE rank <= rank_magnitude AND - date = '2024-06-01' + date = '2024-06-01' AND + is_root_page GROUP BY client, rank_magnitude) From cb4773e131b9aa3d7fe65d944a527a676fe60ad4 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:19:53 +0200 Subject: [PATCH 60/77] Update cms_adoption_by_region.sql --- sql/2024/cms/cms_adoption_by_region.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/2024/cms/cms_adoption_by_region.sql b/sql/2024/cms/cms_adoption_by_region.sql index b9b38e5e9f9..3887bec7a2c 100644 --- a/sql/2024/cms/cms_adoption_by_region.sql +++ b/sql/2024/cms/cms_adoption_by_region.sql @@ -757,7 +757,8 @@ FROM ( UNNEST(technologies.categories) AS cats WHERE date = '2024-06-01' AND - cats = 'CMS' + cats = 'CMS' AND + is_root_page ) USING (client, From 2b265422fd44725ed23cc56671b9bcdab716826b Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:20:09 +0200 Subject: [PATCH 61/77] Update cms_adoption_by_subregion.sql --- sql/2024/cms/cms_adoption_by_subregion.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/2024/cms/cms_adoption_by_subregion.sql b/sql/2024/cms/cms_adoption_by_subregion.sql index cc30b6f5014..d98cd22b8aa 100644 --- a/sql/2024/cms/cms_adoption_by_subregion.sql +++ b/sql/2024/cms/cms_adoption_by_subregion.sql @@ -1237,7 +1237,8 @@ FROM ( UNNEST(technologies.categories) AS cats WHERE date = '2024-06-01' AND - cats = 'CMS' + cats = 'CMS' AND + is_root_page ) USING (client, From 0f5ccbcd1f735e051fa6bcfa83c0533f9cb6ecd9 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:21:00 +0200 Subject: [PATCH 62/77] Update core_web_vitals_by_geo.sql --- sql/2024/cms/core_web_vitals_by_geo.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/2024/cms/core_web_vitals_by_geo.sql b/sql/2024/cms/core_web_vitals_by_geo.sql index 006b0bddadf..95e0475cc3c 100644 --- a/sql/2024/cms/core_web_vitals_by_geo.sql +++ b/sql/2024/cms/core_web_vitals_by_geo.sql @@ -1,5 +1,6 @@ # cms passing core web vitals # core_web_vitals_by_geo +# core_web_vitals_by_geo.sql CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( good / (good + needs_improvement + poor) >= 0.75 ); @@ -58,7 +59,8 @@ JOIN ( UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' AND - date = '2024-06-01' + date = '2024-06-01' AND + is_root_page ) USING (client, url) From 2df989e253f554a13a819464a8d5bd8fb1ec352f Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:21:15 +0200 Subject: [PATCH 63/77] Update core_web_vitals_yoy.sql --- sql/2024/cms/core_web_vitals_yoy.sql | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sql/2024/cms/core_web_vitals_yoy.sql b/sql/2024/cms/core_web_vitals_yoy.sql index 3dd9fd4932f..78fd23e49ed 100644 --- a/sql/2024/cms/core_web_vitals_yoy.sql +++ b/sql/2024/cms/core_web_vitals_yoy.sql @@ -49,13 +49,14 @@ JOIN ( UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' AND - date = '2024-06-01' + date = '2024-06-01'AND + is_root_page ) ON CONCAT(origin, '/') = url AND IF(device = 'desktop', 'desktop', 'mobile') = client WHERE - date = '2024-06-01' + date = '2024-06-01' GROUP BY client, cms @@ -104,7 +105,8 @@ JOIN ( UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' AND - date = '2023-06-01' + date = '2023-06-01' AND + is_root_page ) ON CONCAT(origin, '/') = url AND @@ -159,13 +161,14 @@ JOIN ( UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' AND - date = '2022-06-01' + date = '2022-06-01' AND + is_root_page ) ON CONCAT(origin, '/') = url AND IF(device = 'desktop', 'desktop', 'mobile') = client WHERE - date = '2022-06-01' + date = '2022-06-01' GROUP BY client, cms From cdc06a2cd1beb2693406951f0e85cb2d20f505da Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:22:24 +0200 Subject: [PATCH 64/77] Update core_web_vitals.sql --- sql/2024/cms/core_web_vitals.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/2024/cms/core_web_vitals.sql b/sql/2024/cms/core_web_vitals.sql index 713b6840674..30d053c74f7 100644 --- a/sql/2024/cms/core_web_vitals.sql +++ b/sql/2024/cms/core_web_vitals.sql @@ -1,4 +1,5 @@ # cms passing core web vitals +# core_web_vitals.sql CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( good / (good + needs_improvement + poor) >= 0.75 ); @@ -50,7 +51,7 @@ ON CONCAT(origin, '/') = url AND IF(device = 'desktop', 'desktop', 'mobile') = client WHERE - date = '2024-06-01' + date = '2024-06-01' GROUP BY client, cms From 455f33a1e5f14180b860c6fe0bbd45e06f60be2d Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:22:46 +0200 Subject: [PATCH 65/77] Update image_format_popularity.sql --- sql/2024/cms/image_format_popularity.sql | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/sql/2024/cms/image_format_popularity.sql b/sql/2024/cms/image_format_popularity.sql index b25f0c7e29f..b5e20eb5b38 100644 --- a/sql/2024/cms/image_format_popularity.sql +++ b/sql/2024/cms/image_format_popularity.sql @@ -1,4 +1,7 @@ -#standardSQL # Image format popularity BY CMS +#standardSQL +# Image format popularity BY CMS +# image_format_popularity.sql + SELECT client, cms, @@ -17,7 +20,8 @@ FROM ( UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' AND - date = '2024-06-01' + date = '2024-06-01' AND + is_root_page ) JOIN ( SELECT @@ -42,7 +46,8 @@ JOIN ( `httparchive.all.requests` WHERE date = '2024-06-01' AND - type = 'image' + type = 'image' AND + is_root_page ) USING (client, @@ -58,7 +63,8 @@ JOIN ( UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' AND - date = '2024-06-01' + date = '2024-06-01' AND + is_root_page GROUP BY client, cms) From 20b89095072abe58383bf7ac49c6d9ff77f1f0a0 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:23:17 +0200 Subject: [PATCH 66/77] Update lighthouse_category_scores_per_cms.sql --- sql/2024/cms/lighthouse_category_scores_per_cms.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/2024/cms/lighthouse_category_scores_per_cms.sql b/sql/2024/cms/lighthouse_category_scores_per_cms.sql index 47e74a00880..77a74ad6727 100644 --- a/sql/2024/cms/lighthouse_category_scores_per_cms.sql +++ b/sql/2024/cms/lighthouse_category_scores_per_cms.sql @@ -21,7 +21,8 @@ FROM ( FROM `httparchive.all.pages` WHERE - date = '2024-06-01' + date = '2024-06-01' AND + is_root_page ) JOIN ( SELECT DISTINCT @@ -34,7 +35,8 @@ JOIN ( UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' AND - date = '2024-06-01' + date = '2024-06-01' AND + is_root_page ) USING (url, From a8c9b0f53451365605d96bd49fc73055bef1e7cc Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:25:09 +0200 Subject: [PATCH 67/77] Update top_cms.sql --- sql/2024/cms/top_cms.sql | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/sql/2024/cms/top_cms.sql b/sql/2024/cms/top_cms.sql index 5737d33287b..28e57599bb4 100644 --- a/sql/2024/cms/top_cms.sql +++ b/sql/2024/cms/top_cms.sql @@ -19,14 +19,16 @@ JOIN ( FROM `httparchive.all.pages` WHERE - date = '2024-06-01' + date = '2024-06-01' AND + is_root_page GROUP BY client) USING (client) WHERE cats = 'CMS' AND - date = '2024-06-01' + date = '2024-06-01' AND + is_root_page GROUP BY client, total, @@ -50,14 +52,16 @@ JOIN ( FROM `httparchive.all.pages` WHERE - date = '2023-06-01' + date = '2023-06-01' AND + is_root_page GROUP BY client) USING (client) WHERE cats = 'CMS' AND - date = '2023-06-01' + date = '2023-06-01' AND + is_root_page GROUP BY client, total, @@ -81,14 +85,16 @@ JOIN ( FROM `httparchive.all.pages` WHERE - date = '2022-08-01' + date = '2022-08-01' AND + is_root_page GROUP BY client) USING (client) WHERE cats = 'CMS' AND - date = '2022-08-01' + date = '2022-08-01' AND + is_root_page GROUP BY client, total, @@ -112,14 +118,16 @@ JOIN ( FROM `httparchive.all.pages` WHERE - date = '2021-07-01' + date = '2021-07-01' AND + is_root_page GROUP BY client) USING (client) WHERE cats = 'CMS' AND - date = '2021-07-01' + date = '2021-07-01' AND + is_root_page GROUP BY client, total, From ba7de650fa39e3423e1d7e91ff912f20ad0945d4 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:25:33 +0200 Subject: [PATCH 68/77] Update top_cms_by_geo.sql --- sql/2024/cms/top_cms_by_geo.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/2024/cms/top_cms_by_geo.sql b/sql/2024/cms/top_cms_by_geo.sql index 3fe27788348..50e9de9074a 100644 --- a/sql/2024/cms/top_cms_by_geo.sql +++ b/sql/2024/cms/top_cms_by_geo.sql @@ -55,7 +55,8 @@ FROM ( technologies.technology IS NOT NULL AND cats = 'CMS' AND technologies.technology != '' AND - date = '2024-06-01' + date = '2024-06-01' AND + is_root_page ) USING (client, url) GROUP BY client, From 6a4659b295e51a5e097afdb83e787d55787a67a9 Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:25:52 +0200 Subject: [PATCH 69/77] Update top_cms_by_rank.sql --- sql/2024/cms/top_cms_by_rank.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sql/2024/cms/top_cms_by_rank.sql b/sql/2024/cms/top_cms_by_rank.sql index ba4ef0e8535..456aa703fc7 100644 --- a/sql/2024/cms/top_cms_by_rank.sql +++ b/sql/2024/cms/top_cms_by_rank.sql @@ -20,7 +20,8 @@ FROM ( UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' AND - date = '2024-06-01') + date = '2024-06-01' AND + is_root_page) JOIN ( SELECT client, @@ -31,7 +32,8 @@ JOIN ( UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude WHERE rank <= rank_magnitude AND - date = '2024-06-01') + date = '2024-06-01' AND + is_root_page) USING (client, url) JOIN ( @@ -44,7 +46,8 @@ JOIN ( UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude WHERE rank <= rank_magnitude AND - date = '2024-06-01' + date = '2024-06-01' AND + is_root_page GROUP BY client, rank_magnitude) From 87a8e8ccc02b1fc3479d3f5be7e54fcd2c4d690b Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:26:40 +0200 Subject: [PATCH 70/77] Update wordpress_page_builders.sql --- sql/2024/cms/wordpress_page_builders.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/2024/cms/wordpress_page_builders.sql b/sql/2024/cms/wordpress_page_builders.sql index 14fced8ee8e..11246fe96cc 100644 --- a/sql/2024/cms/wordpress_page_builders.sql +++ b/sql/2024/cms/wordpress_page_builders.sql @@ -18,7 +18,8 @@ FROM ( UNNEST(technologies.categories) AS cats WHERE technologies.technology = 'WordPress' AND - date = '2024-06-01') + date = '2024-06-01' AND + is_root_page) JOIN ( SELECT client, @@ -30,7 +31,8 @@ JOIN ( UNNEST(technologies.categories) AS cats WHERE cats = 'Page builders' AND - date = '2024-06-01' + date = '2024-06-01' AND + is_root_page GROUP BY client, url) From 47105971b337a77d574fb6759576ff01a30b147b Mon Sep 17 00:00:00 2001 From: Nurullah Demir <13475745+nrllh@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:27:59 +0200 Subject: [PATCH 71/77] Update wordpress_page_builders.sql --- sql/2024/cms/wordpress_page_builders.sql | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sql/2024/cms/wordpress_page_builders.sql b/sql/2024/cms/wordpress_page_builders.sql index 11246fe96cc..14fced8ee8e 100644 --- a/sql/2024/cms/wordpress_page_builders.sql +++ b/sql/2024/cms/wordpress_page_builders.sql @@ -18,8 +18,7 @@ FROM ( UNNEST(technologies.categories) AS cats WHERE technologies.technology = 'WordPress' AND - date = '2024-06-01' AND - is_root_page) + date = '2024-06-01') JOIN ( SELECT client, @@ -31,8 +30,7 @@ JOIN ( UNNEST(technologies.categories) AS cats WHERE cats = 'Page builders' AND - date = '2024-06-01' AND - is_root_page + date = '2024-06-01' GROUP BY client, url) From a73fe0e39c0d4d2b119678661d38d76a023f4720 Mon Sep 17 00:00:00 2001 From: Kevin Farrugia Date: Tue, 15 Oct 2024 11:40:16 +0200 Subject: [PATCH 72/77] Fixed linting issues --- sql/2024/cms/cms_adoption.sql | 6 +++--- sql/2024/cms/core_web_vitals.sql | 2 +- sql/2024/cms/core_web_vitals_yoy.sql | 6 +++--- sql/2024/cms/resource_weights.sql | 24 +++++++++++----------- sql/2024/cms/top_cms.sql | 22 ++++++++++---------- sql/2024/cms/top_cms_by_geo.sql | 6 +++--- sql/2024/cms/top_cms_by_rank.sql | 2 +- sql/2024/cms/wordpress_page_builders.sql | 8 ++++---- sql/2024/cms/wordpress_resources.sql | 26 ++++++++++++------------ 9 files changed, 51 insertions(+), 51 deletions(-) diff --git a/sql/2024/cms/cms_adoption.sql b/sql/2024/cms/cms_adoption.sql index c066acbcb4e..41628dd8285 100644 --- a/sql/2024/cms/cms_adoption.sql +++ b/sql/2024/cms/cms_adoption.sql @@ -19,7 +19,7 @@ JOIN ( FROM `httparchive.all.pages` WHERE - date = '2024-06-01' AND + date = '2024-06-01' AND is_root_page GROUP BY client) @@ -58,7 +58,7 @@ USING (client) WHERE cats = 'CMS' AND - date = '2023-06-01' AND + date = '2023-06-01' AND is_root_page GROUP BY client, @@ -82,7 +82,7 @@ JOIN ( `httparchive.all.pages` WHERE date = '2022-06-01' AND - is_root_page + is_root_page GROUP BY client) USING diff --git a/sql/2024/cms/core_web_vitals.sql b/sql/2024/cms/core_web_vitals.sql index 30d053c74f7..91423ce1dde 100644 --- a/sql/2024/cms/core_web_vitals.sql +++ b/sql/2024/cms/core_web_vitals.sql @@ -51,7 +51,7 @@ ON CONCAT(origin, '/') = url AND IF(device = 'desktop', 'desktop', 'mobile') = client WHERE - date = '2024-06-01' + date = '2024-06-01' GROUP BY client, cms diff --git a/sql/2024/cms/core_web_vitals_yoy.sql b/sql/2024/cms/core_web_vitals_yoy.sql index 78fd23e49ed..b29be382e61 100644 --- a/sql/2024/cms/core_web_vitals_yoy.sql +++ b/sql/2024/cms/core_web_vitals_yoy.sql @@ -49,14 +49,14 @@ JOIN ( UNNEST(technologies.categories) AS cats WHERE cats = 'CMS' AND - date = '2024-06-01'AND + date = '2024-06-01' AND is_root_page ) ON CONCAT(origin, '/') = url AND IF(device = 'desktop', 'desktop', 'mobile') = client WHERE - date = '2024-06-01' + date = '2024-06-01' GROUP BY client, cms @@ -168,7 +168,7 @@ ON CONCAT(origin, '/') = url AND IF(device = 'desktop', 'desktop', 'mobile') = client WHERE - date = '2022-06-01' + date = '2022-06-01' GROUP BY client, cms diff --git a/sql/2024/cms/resource_weights.sql b/sql/2024/cms/resource_weights.sql index efc1d86d683..c86c3e853ed 100644 --- a/sql/2024/cms/resource_weights.sql +++ b/sql/2024/cms/resource_weights.sql @@ -15,28 +15,28 @@ SELECT FROM ( SELECT DISTINCT client, - page as url, + page AS url, technologies.technology AS cms FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats WHERE - cats = 'CMS' - AND date ='2024-06-01') + cats = 'CMS' AND + date = '2024-06-01') JOIN ( SELECT client, - page as url, - cast(json_value(summary, "$.bytesTotal") as int64) / 1024 AS total_kb, - cast(json_value(summary, "$.bytesHtml") as int64) / 1024 AS html_kb, - cast(json_value(summary, "$.bytesJS") as int64) / 1024 AS js_kb, - cast(json_value(summary, "$.bytesCss") as int64) / 1024 AS css_kb, - cast(json_value(summary, "$.bytesImg") as int64) / 1024 AS img_kb, - cast(json_value(summary, "$.bytesFont") as int64) / 1024 AS font_kb + page AS url, + cast(json_value(summary, '$.bytesTotal') AS INT64) / 1024 AS total_kb, + cast(json_value(summary, '$.bytesHtml') AS INT64) / 1024 AS html_kb, + cast(json_value(summary, '$.bytesJS') AS INT64) / 1024 AS js_kb, + cast(json_value(summary, '$.bytesCss') AS INT64) / 1024 AS css_kb, + cast(json_value(summary, '$.bytesImg') AS INT64) / 1024 AS img_kb, + cast(json_value(summary, '$.bytesFont') AS INT64) / 1024 AS font_kb FROM `httparchive.all.pages` - WHERE + WHERE date = '2024-06-01') USING (client, url) diff --git a/sql/2024/cms/top_cms.sql b/sql/2024/cms/top_cms.sql index 28e57599bb4..0e1708fe0d6 100644 --- a/sql/2024/cms/top_cms.sql +++ b/sql/2024/cms/top_cms.sql @@ -10,7 +10,7 @@ SELECT COUNT(DISTINCT page) / total AS pct FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats JOIN ( SELECT @@ -18,7 +18,7 @@ JOIN ( COUNT(0) AS total FROM `httparchive.all.pages` - WHERE + WHERE date = '2024-06-01' AND is_root_page GROUP BY @@ -32,7 +32,7 @@ WHERE GROUP BY client, total, - cms + cms UNION ALL SELECT client, @@ -43,7 +43,7 @@ SELECT COUNT(DISTINCT page) / total AS pct FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats JOIN ( SELECT @@ -51,7 +51,7 @@ JOIN ( COUNT(0) AS total FROM `httparchive.all.pages` - WHERE + WHERE date = '2023-06-01' AND is_root_page GROUP BY @@ -65,7 +65,7 @@ WHERE GROUP BY client, total, - cms + cms UNION ALL SELECT client, @@ -76,7 +76,7 @@ SELECT COUNT(DISTINCT page) / total AS pct FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats JOIN ( SELECT @@ -84,7 +84,7 @@ JOIN ( COUNT(0) AS total FROM `httparchive.all.pages` - WHERE + WHERE date = '2022-08-01' AND is_root_page GROUP BY @@ -98,7 +98,7 @@ WHERE GROUP BY client, total, - cms + cms UNION ALL SELECT client, @@ -109,7 +109,7 @@ SELECT COUNT(DISTINCT page) / total AS pct FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, + UNNEST(technologies) AS technologies, UNNEST(technologies.categories) AS cats JOIN ( SELECT @@ -117,7 +117,7 @@ JOIN ( COUNT(0) AS total FROM `httparchive.all.pages` - WHERE + WHERE date = '2021-07-01' AND is_root_page GROUP BY diff --git a/sql/2024/cms/top_cms_by_geo.sql b/sql/2024/cms/top_cms_by_geo.sql index 50e9de9074a..473a556e11e 100644 --- a/sql/2024/cms/top_cms_by_geo.sql +++ b/sql/2024/cms/top_cms_by_geo.sql @@ -46,11 +46,11 @@ FROM ( client, cats, technologies.technology AS cms, - page as url + page AS url FROM `httparchive.all.pages`, - UNNEST (technologies) AS technologies, - UNNEST(technologies.categories) AS cats + UNNEST(technologies) AS technologies, + UNNEST(technologies.categories) AS cats WHERE technologies.technology IS NOT NULL AND cats = 'CMS' AND diff --git a/sql/2024/cms/top_cms_by_rank.sql b/sql/2024/cms/top_cms_by_rank.sql index 456aa703fc7..da6da0e0e2d 100644 --- a/sql/2024/cms/top_cms_by_rank.sql +++ b/sql/2024/cms/top_cms_by_rank.sql @@ -25,7 +25,7 @@ FROM ( JOIN ( SELECT client, - page as url, + page AS url, rank_magnitude AS rank FROM `httparchive.all.pages`, diff --git a/sql/2024/cms/wordpress_page_builders.sql b/sql/2024/cms/wordpress_page_builders.sql index 14fced8ee8e..b853ad58053 100644 --- a/sql/2024/cms/wordpress_page_builders.sql +++ b/sql/2024/cms/wordpress_page_builders.sql @@ -11,23 +11,23 @@ SELECT FROM ( SELECT DISTINCT client, - page as url + page AS url FROM `httparchive.all.pages`, UNNEST(technologies) AS technologies, - UNNEST(technologies.categories) AS cats + UNNEST(technologies.categories) AS cats WHERE technologies.technology = 'WordPress' AND date = '2024-06-01') JOIN ( SELECT client, - page as url, + page AS url, ARRAY_TO_STRING(ARRAY_AGG(technologies.technology ORDER BY technologies.technology), ', ') AS page_builders FROM `httparchive.all.pages`, UNNEST(technologies) AS technologies, - UNNEST(technologies.categories) AS cats + UNNEST(technologies.categories) AS cats WHERE cats = 'Page builders' AND date = '2024-06-01' diff --git a/sql/2024/cms/wordpress_resources.sql b/sql/2024/cms/wordpress_resources.sql index d337825571d..9d609be1060 100644 --- a/sql/2024/cms/wordpress_resources.sql +++ b/sql/2024/cms/wordpress_resources.sql @@ -6,7 +6,7 @@ SELECT percentile, client, path, - APPROX_QUANTILES(freq, 1000)[OFFSET (percentile * 10)] AS freq + APPROX_QUANTILES(freq, 1000)[OFFSET(percentile * 10)] AS freq FROM ( SELECT client, @@ -18,22 +18,22 @@ FROM ( client, page FROM - `httparchive.all.pages`, - UNNEST(technologies) AS technologies, - UNNEST(technologies.categories) AS cats - WHERE - technologies.technology = 'WordPress' AND - date = '2024-06-01') + `httparchive.all.pages`, + UNNEST(technologies) AS technologies, + UNNEST(technologies.categories) AS cats + WHERE + technologies.technology = 'WordPress' AND + date = '2024-06-01') JOIN ( SELECT client, - cast(json_value(summary, "$.pageid") as int64) as pageid, + cast(json_value(summary, '$.pageid') AS INT64) AS pageid, page FROM `httparchive.all.pages`, UNNEST(technologies) AS technologies, - UNNEST(technologies.categories) AS cats - WHERE + UNNEST(technologies.categories) AS cats + WHERE date = '2024-06-01') USING (client, @@ -41,11 +41,11 @@ FROM ( JOIN ( SELECT client, - cast(json_value(summary, "$.pageid") as int64) as pageid, - page as url + cast(json_value(summary, '$.pageid') AS INT64) AS pageid, + page AS url FROM `httparchive.all.pages` - WHERE + WHERE date = '2024-06-01') USING (client, From d23ad7ea1f4347bedf397cae68f07c0c3ce23d31 Mon Sep 17 00:00:00 2001 From: Kevin Farrugia Date: Tue, 15 Oct 2024 11:51:30 +0200 Subject: [PATCH 73/77] Fixed linting issues --- sql/2024/cms/cms_adoption.sql | 2 +- sql/2024/cms/cms_adoption_by_rank.sql | 2 +- sql/2024/cms/image_format_popularity.sql | 2 +- sql/2024/cms/wordpress_resources.sql | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/2024/cms/cms_adoption.sql b/sql/2024/cms/cms_adoption.sql index 41628dd8285..37eb2b76600 100644 --- a/sql/2024/cms/cms_adoption.sql +++ b/sql/2024/cms/cms_adoption.sql @@ -1,4 +1,4 @@ -#standardSQL +#standardSQL # CMS adoption OVER time # cms_adoption.sql diff --git a/sql/2024/cms/cms_adoption_by_rank.sql b/sql/2024/cms/cms_adoption_by_rank.sql index 6e3607e777a..e69217eb9bf 100644 --- a/sql/2024/cms/cms_adoption_by_rank.sql +++ b/sql/2024/cms/cms_adoption_by_rank.sql @@ -1,4 +1,4 @@ -#standardSQL +#standardSQL # CMS adoption per rank # cms_adoption_by_rank.sql diff --git a/sql/2024/cms/image_format_popularity.sql b/sql/2024/cms/image_format_popularity.sql index b5e20eb5b38..09641aa08b7 100644 --- a/sql/2024/cms/image_format_popularity.sql +++ b/sql/2024/cms/image_format_popularity.sql @@ -1,4 +1,4 @@ -#standardSQL +#standardSQL # Image format popularity BY CMS # image_format_popularity.sql diff --git a/sql/2024/cms/wordpress_resources.sql b/sql/2024/cms/wordpress_resources.sql index 9d609be1060..4377c489e05 100644 --- a/sql/2024/cms/wordpress_resources.sql +++ b/sql/2024/cms/wordpress_resources.sql @@ -1,5 +1,5 @@ -#standardSQL -# Distribution OF WordPress resource types BY path +#standardSQL +# Distribution OF WordPress resource types BY path # wordpress_resources.sql SELECT From c260b084b9632e77c2a62ab9a8bdf41ced26be61 Mon Sep 17 00:00:00 2001 From: Kevin Farrugia Date: Tue, 15 Oct 2024 14:59:58 +0200 Subject: [PATCH 74/77] Added query for lighthouse scores per CMS YoY --- ...lighthouse_category_scores_per_cms_yoy.sql | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 sql/2024/cms/lighthouse_category_scores_per_cms_yoy.sql diff --git a/sql/2024/cms/lighthouse_category_scores_per_cms_yoy.sql b/sql/2024/cms/lighthouse_category_scores_per_cms_yoy.sql new file mode 100644 index 00000000000..c4d8b8d6626 --- /dev/null +++ b/sql/2024/cms/lighthouse_category_scores_per_cms_yoy.sql @@ -0,0 +1,56 @@ +#standardSQL +# Lighthouse category scores per CMS YoY +# lighthouse_category_scores_per_cms_yoy.sql +SELECT + CASE + WHEN date = '2024-06-01' THEN '2024' + ELSE '2023' + END AS year, + client, + cms, + COUNT(DISTINCT url) AS freq, + APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.performance.score') AS NUMERIC), 1000)[ + OFFSET(500)] * 100 AS median_performance, + APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.accessibility.score') AS NUMERIC), 1000)[ + OFFSET(500)] * 100 AS median_accessibility, + APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$.seo.score') AS NUMERIC), 1000)[ + OFFSET(500)] * 100 AS median_seo, + APPROX_QUANTILES(CAST(JSON_VALUE(categories, '$."best-practices".score') AS NUMERIC), 1000)[ + OFFSET(500)] * 100 AS median_best_practices +FROM ( + SELECT + client, + date, + page AS url, + JSON_EXTRACT(lighthouse, '$.categories') AS categories + FROM + `httparchive.all.pages` + WHERE + (date = '2024-06-01' OR date = '2023-06-01') AND + is_root_page +) +JOIN ( + SELECT DISTINCT + client, + date, + technologies.technology AS cms, + page AS url + FROM + `httparchive.all.pages`, + UNNEST(technologies) AS technologies, + UNNEST(technologies.categories) AS cats + WHERE + cats = 'CMS' AND + (date = '2024-06-01' OR date = '2023-06-01') AND + is_root_page +) +USING + (date, url, client) +GROUP BY + date, + client, + cms +ORDER BY + date, + client, + freq DESC From 7499370a1142d46c41f763c1ba86c61b6ebcf153 Mon Sep 17 00:00:00 2001 From: Kevin Farrugia Date: Thu, 31 Oct 2024 15:02:35 +0100 Subject: [PATCH 75/77] Update sql/2024/cms/cms_adoption_by_rank.sql Co-authored-by: Barry Pollard --- sql/2024/cms/cms_adoption_by_rank.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2024/cms/cms_adoption_by_rank.sql b/sql/2024/cms/cms_adoption_by_rank.sql index e69217eb9bf..293dfe197f6 100644 --- a/sql/2024/cms/cms_adoption_by_rank.sql +++ b/sql/2024/cms/cms_adoption_by_rank.sql @@ -44,7 +44,7 @@ JOIN ( COUNT(0) AS total FROM `httparchive.all.pages`, - UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude + UNNEST([1e3, 1e4, 1e5, 1e6, 1e7, 1e8]) AS rank_magnitude WHERE rank <= rank_magnitude AND date = '2024-06-01' AND @@ -55,7 +55,7 @@ JOIN ( USING (client, rank), - UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude + UNNEST([1e3, 1e4, 1e5, 1e6, 1e7, 1e8]) AS rank_magnitude WHERE rank <= rank_magnitude GROUP BY From 4258ae4afe06e727fd081e3b3f2ad536e6012400 Mon Sep 17 00:00:00 2001 From: Kevin Farrugia Date: Thu, 31 Oct 2024 15:02:43 +0100 Subject: [PATCH 76/77] Update sql/2024/cms/top_cms_by_rank.sql Co-authored-by: Barry Pollard --- sql/2024/cms/top_cms_by_rank.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2024/cms/top_cms_by_rank.sql b/sql/2024/cms/top_cms_by_rank.sql index da6da0e0e2d..c0a84010cd8 100644 --- a/sql/2024/cms/top_cms_by_rank.sql +++ b/sql/2024/cms/top_cms_by_rank.sql @@ -29,7 +29,7 @@ JOIN ( rank_magnitude AS rank FROM `httparchive.all.pages`, - UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude + UNNEST([1e3, 1e4, 1e5, 1e6, 1e7, 1e8]) AS rank_magnitude WHERE rank <= rank_magnitude AND date = '2024-06-01' AND @@ -43,7 +43,7 @@ JOIN ( COUNT(0) AS total FROM `httparchive.all.pages`, - UNNEST([1e3, 1e4, 1e5, 1e6, 1e7]) AS rank_magnitude + UNNEST([1e3, 1e4, 1e5, 1e6, 1e7, 1e8]) AS rank_magnitude WHERE rank <= rank_magnitude AND date = '2024-06-01' AND From bf5a2230004ff27de3d95241c33ad83fa06b2676 Mon Sep 17 00:00:00 2001 From: Kevin Farrugia Date: Fri, 1 Nov 2024 10:11:55 +0100 Subject: [PATCH 77/77] Removed redundant query cms_adoption_by_rank.sql --- sql/2024/cms/cms_adoption_by_geo.sql | 1 + sql/2024/cms/cms_adoption_by_rank.sql | 67 --------------------------- 2 files changed, 1 insertion(+), 67 deletions(-) delete mode 100644 sql/2024/cms/cms_adoption_by_rank.sql diff --git a/sql/2024/cms/cms_adoption_by_geo.sql b/sql/2024/cms/cms_adoption_by_geo.sql index 241e03d9b1a..c9371b732aa 100644 --- a/sql/2024/cms/cms_adoption_by_geo.sql +++ b/sql/2024/cms/cms_adoption_by_geo.sql @@ -13,6 +13,7 @@ WITH geo_summary AS ( WHERE yyyymm = 202406 ) + SELECT * FROM ( diff --git a/sql/2024/cms/cms_adoption_by_rank.sql b/sql/2024/cms/cms_adoption_by_rank.sql deleted file mode 100644 index 293dfe197f6..00000000000 --- a/sql/2024/cms/cms_adoption_by_rank.sql +++ /dev/null @@ -1,67 +0,0 @@ -#standardSQL -# CMS adoption per rank -# cms_adoption_by_rank.sql - -SELECT - client, - technology, - rank, - COUNT(DISTINCT page) AS pages, - ANY_VALUE(total) AS total, - COUNT(DISTINCT page) / ANY_VALUE(total) AS pct -FROM ( - SELECT DISTINCT - client, - technology, - page - FROM - `httparchive.all.pages`, - UNNEST(technologies) AS technologies, - UNNEST(technologies.categories) AS cats - WHERE - date = '2024-06-01' AND - cats = 'CMS' AND - is_root_page -) -JOIN ( - SELECT - client, - page, - rank - FROM - `httparchive.all.pages` - WHERE - date = '2024-06-01' AND - is_root_page -) -USING - (client, - page) -JOIN ( - SELECT - client, - rank_magnitude AS rank, - COUNT(0) AS total - FROM - `httparchive.all.pages`, - UNNEST([1e3, 1e4, 1e5, 1e6, 1e7, 1e8]) AS rank_magnitude - WHERE - rank <= rank_magnitude AND - date = '2024-06-01' AND - is_root_page - GROUP BY - client, - rank_magnitude) -USING - (client, - rank), - UNNEST([1e3, 1e4, 1e5, 1e6, 1e7, 1e8]) AS rank_magnitude -WHERE - rank <= rank_magnitude -GROUP BY - client, - technology, - rank -ORDER BY - rank, - pages DESC