From 80ad02e10e2ebefe7b1707cb504bd9d3cc598ea8 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Oct 2024 11:23:36 +0200 Subject: [PATCH 01/31] Ensure Python 3.13 and PostgreSQL 17 compatibility (#17752) This adds Python 3.13.0 to the trial test matrix Also updates `cffi` and `zope.interface` in the locked dependencies to make sure we have versions compatible with Python 3.13. For some reasons, they are not being picked up by dependabot. --- .ci/scripts/calculate_jobs.py | 6 +- changelog.d/17752.misc | 1 + poetry.lock | 212 ++++++++++++++++++---------------- 3 files changed, 115 insertions(+), 104 deletions(-) create mode 100644 changelog.d/17752.misc diff --git a/.ci/scripts/calculate_jobs.py b/.ci/scripts/calculate_jobs.py index 92bd192fd1f..15f2d94a810 100755 --- a/.ci/scripts/calculate_jobs.py +++ b/.ci/scripts/calculate_jobs.py @@ -53,7 +53,7 @@ def set_output(key: str, value: str): "database": "sqlite", "extras": "all", } - for version in ("3.9", "3.10", "3.11", "3.12") + for version in ("3.9", "3.10", "3.11", "3.12", "3.13") ) trial_postgres_tests = [ @@ -68,9 +68,9 @@ def set_output(key: str, value: str): if not IS_PR: trial_postgres_tests.append( { - "python-version": "3.12", + "python-version": "3.13", "database": "postgres", - "postgres-version": "16", + "postgres-version": "17", "extras": "all", } ) diff --git a/changelog.d/17752.misc b/changelog.d/17752.misc new file mode 100644 index 00000000000..f9f01a42420 --- /dev/null +++ b/changelog.d/17752.misc @@ -0,0 +1 @@ +Add Python 3.13 and Postgres 17 to the test matrix. diff --git a/poetry.lock b/poetry.lock index 42d7f03c5c9..1102e89d12f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -147,75 +147,78 @@ files = [ [[package]] name = "cffi" -version = "1.15.1" +version = "1.17.1" description = "Foreign Function Interface for Python calling C code." optional = false -python-versions = "*" +python-versions = ">=3.8" files = [ - {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, - {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, - {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, - {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, - {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, - {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, - {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, - {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, - {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, - {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, - {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, - {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, - {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, - {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, - {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, - {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, - {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, - {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, - {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, + {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, + {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be"}, + {file = "cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c"}, + {file = "cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"}, + {file = "cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655"}, + {file = "cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0"}, + {file = "cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4"}, + {file = "cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93"}, + {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3"}, + {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8"}, + {file = "cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65"}, + {file = "cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903"}, + {file = "cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e"}, + {file = "cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd"}, + {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed"}, + {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9"}, + {file = "cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d"}, + {file = "cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a"}, + {file = "cffi-1.17.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1"}, + {file = "cffi-1.17.1-cp38-cp38-win32.whl", hash = "sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8"}, + {file = "cffi-1.17.1-cp38-cp38-win_amd64.whl", hash = "sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1"}, + {file = "cffi-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16"}, + {file = "cffi-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e"}, + {file = "cffi-1.17.1-cp39-cp39-win32.whl", hash = "sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7"}, + {file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"}, + {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, ] [package.dependencies] @@ -3028,50 +3031,57 @@ test = ["zope.testrunner"] [[package]] name = "zope-interface" -version = "6.0" +version = "7.1.0" description = "Interfaces for Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "zope.interface-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f299c020c6679cb389814a3b81200fe55d428012c5e76da7e722491f5d205990"}, - {file = "zope.interface-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ee4b43f35f5dc15e1fec55ccb53c130adb1d11e8ad8263d68b1284b66a04190d"}, - {file = "zope.interface-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a158846d0fca0a908c1afb281ddba88744d403f2550dc34405c3691769cdd85"}, - {file = "zope.interface-6.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f72f23bab1848edb7472309e9898603141644faec9fd57a823ea6b4d1c4c8995"}, - {file = "zope.interface-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48f4d38cf4b462e75fac78b6f11ad47b06b1c568eb59896db5b6ec1094eb467f"}, - {file = "zope.interface-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:87b690bbee9876163210fd3f500ee59f5803e4a6607d1b1238833b8885ebd410"}, - {file = "zope.interface-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f2363e5fd81afb650085c6686f2ee3706975c54f331b426800b53531191fdf28"}, - {file = "zope.interface-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:af169ba897692e9cd984a81cb0f02e46dacdc07d6cf9fd5c91e81f8efaf93d52"}, - {file = "zope.interface-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa90bac61c9dc3e1a563e5babb3fd2c0c1c80567e815442ddbe561eadc803b30"}, - {file = "zope.interface-6.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:89086c9d3490a0f265a3c4b794037a84541ff5ffa28bb9c24cc9f66566968464"}, - {file = "zope.interface-6.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:809fe3bf1a91393abc7e92d607976bbb8586512913a79f2bf7d7ec15bd8ea518"}, - {file = "zope.interface-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:0ec9653825f837fbddc4e4b603d90269b501486c11800d7c761eee7ce46d1bbb"}, - {file = "zope.interface-6.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:790c1d9d8f9c92819c31ea660cd43c3d5451df1df61e2e814a6f99cebb292788"}, - {file = "zope.interface-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b39b8711578dcfd45fc0140993403b8a81e879ec25d53189f3faa1f006087dca"}, - {file = "zope.interface-6.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eba51599370c87088d8882ab74f637de0c4f04a6d08a312dce49368ba9ed5c2a"}, - {file = "zope.interface-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ee934f023f875ec2cfd2b05a937bd817efcc6c4c3f55c5778cbf78e58362ddc"}, - {file = "zope.interface-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:042f2381118b093714081fd82c98e3b189b68db38ee7d35b63c327c470ef8373"}, - {file = "zope.interface-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:dfbbbf0809a3606046a41f8561c3eada9db811be94138f42d9135a5c47e75f6f"}, - {file = "zope.interface-6.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:424d23b97fa1542d7be882eae0c0fc3d6827784105264a8169a26ce16db260d8"}, - {file = "zope.interface-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e538f2d4a6ffb6edfb303ce70ae7e88629ac6e5581870e66c306d9ad7b564a58"}, - {file = "zope.interface-6.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12175ca6b4db7621aedd7c30aa7cfa0a2d65ea3a0105393e05482d7a2d367446"}, - {file = "zope.interface-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c3d7dfd897a588ec27e391edbe3dd320a03684457470415870254e714126b1f"}, - {file = "zope.interface-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:b3f543ae9d3408549a9900720f18c0194ac0fe810cecda2a584fd4dca2eb3bb8"}, - {file = "zope.interface-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d0583b75f2e70ec93f100931660328965bb9ff65ae54695fb3fa0a1255daa6f2"}, - {file = "zope.interface-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:23ac41d52fd15dd8be77e3257bc51bbb82469cf7f5e9a30b75e903e21439d16c"}, - {file = "zope.interface-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99856d6c98a326abbcc2363827e16bd6044f70f2ef42f453c0bd5440c4ce24e5"}, - {file = "zope.interface-6.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1592f68ae11e557b9ff2bc96ac8fc30b187e77c45a3c9cd876e3368c53dc5ba8"}, - {file = "zope.interface-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4407b1435572e3e1610797c9203ad2753666c62883b921318c5403fb7139dec2"}, - {file = "zope.interface-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:5171eb073474a5038321409a630904fd61f12dd1856dd7e9d19cd6fe092cbbc5"}, - {file = "zope.interface-6.0.tar.gz", hash = "sha256:aab584725afd10c710b8f1e6e208dbee2d0ad009f57d674cb9d1b3964037275d"}, + {file = "zope.interface-7.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2bd9e9f366a5df08ebbdc159f8224904c1c5ce63893984abb76954e6fbe4381a"}, + {file = "zope.interface-7.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:661d5df403cd3c5b8699ac480fa7f58047a3253b029db690efa0c3cf209993ef"}, + {file = "zope.interface-7.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91b6c30689cfd87c8f264acb2fc16ad6b3c72caba2aec1bf189314cf1a84ca33"}, + {file = "zope.interface-7.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b6a4924f5bad9fe21d99f66a07da60d75696a136162427951ec3cb223a5570d"}, + {file = "zope.interface-7.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80a3c00b35f6170be5454b45abe2719ea65919a2f09e8a6e7b1362312a872cd3"}, + {file = "zope.interface-7.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:b936d61dbe29572fd2cfe13e30b925e5383bed1aba867692670f5a2a2eb7b4e9"}, + {file = "zope.interface-7.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ac20581fc6cd7c754f6dff0ae06fedb060fa0e9ea6309d8be8b2701d9ea51c4"}, + {file = "zope.interface-7.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:848b6fa92d7c8143646e64124ed46818a0049a24ecc517958c520081fd147685"}, + {file = "zope.interface-7.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec1ef1fdb6f014d5886b97e52b16d0f852364f447d2ab0f0c6027765777b6667"}, + {file = "zope.interface-7.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bcff5c09d0215f42ba64b49205a278e44413d9bf9fa688fd9e42bfe472b5f4f"}, + {file = "zope.interface-7.1.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07add15de0cc7e69917f7d286b64d54125c950aeb43efed7a5ea7172f000fbc1"}, + {file = "zope.interface-7.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:9940d5bc441f887c5f375ec62bcf7e7e495a2d5b1da97de1184a88fb567f06af"}, + {file = "zope.interface-7.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f245d039f72e6f802902375755846f5de1ee1e14c3e8736c078565599bcab621"}, + {file = "zope.interface-7.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6159e767d224d8f18deff634a1d3722e68d27488c357f62ebeb5f3e2f5288b1f"}, + {file = "zope.interface-7.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e956b1fd7f3448dd5e00f273072e73e50dfafcb35e4227e6d5af208075593c9"}, + {file = "zope.interface-7.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff115ef91c0eeac69cd92daeba36a9d8e14daee445b504eeea2b1c0b55821984"}, + {file = "zope.interface-7.1.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bec001798ab62c3fc5447162bf48496ae9fba02edc295a9e10a0b0c639a6452e"}, + {file = "zope.interface-7.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:124149e2d42067b9c6597f4dafdc7a0983d0163868f897b7bb5dc850b14f9a87"}, + {file = "zope.interface-7.1.0-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:9733a9a0f94ef53d7aa64661811b20875b5bc6039034c6e42fb9732170130573"}, + {file = "zope.interface-7.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5fcf379b875c610b5a41bc8a891841533f98de0520287d7f85e25386cd10d3e9"}, + {file = "zope.interface-7.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0a45b5af9f72c805ee668d1479480ca85169312211bed6ed18c343e39307d5f"}, + {file = "zope.interface-7.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4af4a12b459a273b0b34679a5c3dc5e34c1847c3dd14a628aa0668e19e638ea2"}, + {file = "zope.interface-7.1.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a735f82d2e3ed47ca01a20dfc4c779b966b16352650a8036ab3955aad151ed8a"}, + {file = "zope.interface-7.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:5501e772aff595e3c54266bc1bfc5858e8f38974ce413a8f1044aae0f32a83a3"}, + {file = "zope.interface-7.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ec59fe53db7d32abb96c6d4efeed84aab4a7c38c62d7a901a9b20c09dd936e7a"}, + {file = "zope.interface-7.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e53c291debef523b09e1fe3dffe5f35dde164f1c603d77f770b88a1da34b7ed6"}, + {file = "zope.interface-7.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:711eebc77f2092c6a8b304bad0b81a6ce3cf5490b25574e7309fbc07d881e3af"}, + {file = "zope.interface-7.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a00ead2e24c76436e1b457a5132d87f83858330f6c923640b7ef82d668525d1"}, + {file = "zope.interface-7.1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e28ea0bc4b084fc93a483877653a033062435317082cdc6388dec3438309faf"}, + {file = "zope.interface-7.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:27cfb5205d68b12682b6e55ab8424662d96e8ead19550aad0796b08dd2c9a45e"}, + {file = "zope.interface-7.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9e3e48f3dea21c147e1b10c132016cb79af1159facca9736d231694ef5a740a8"}, + {file = "zope.interface-7.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a99240b1d02dc469f6afbe7da1bf617645e60290c272968f4e53feec18d7dce8"}, + {file = "zope.interface-7.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc8a318162123eddbdf22fcc7b751288ce52e4ad096d3766ff1799244352449d"}, + {file = "zope.interface-7.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7b25db127db3e6b597c5f74af60309c4ad65acd826f89609662f0dc33a54728"}, + {file = "zope.interface-7.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a29ac607e970b5576547f0e3589ec156e04de17af42839eedcf478450687317"}, + {file = "zope.interface-7.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:a14c9decf0eb61e0892631271d500c1e306c7b6901c998c7035e194d9150fdd1"}, + {file = "zope_interface-7.1.0.tar.gz", hash = "sha256:3f005869a1a05e368965adb2075f97f8ee9a26c61898a9e52a9764d93774f237"}, ] [package.dependencies] setuptools = "*" [package.extras] -docs = ["Sphinx", "repoze.sphinx.autointerface"] -test = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] -testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] +docs = ["Sphinx", "furo", "repoze.sphinx.autointerface"] +test = ["coverage[toml]", "zope.event", "zope.testing"] +testing = ["coverage[toml]", "zope.event", "zope.testing"] [[package]] name = "zope-schema" From a5e16a4ab5a7db41b6dc978da1d833c39874813f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 22 Oct 2024 05:06:46 -0500 Subject: [PATCH 02/31] Sliding Sync: Reset `forgotten` status when membership changes (like rejoining a room) (#17835) Reset `sliding_sync_membership_snapshots` -> `forgotten` status when membership changes (like rejoining a room). Fix https://github.com/element-hq/synapse/issues/17781 ### What was the problem before? Previously, if someone used `/forget` on one of their rooms, it would update `sliding_sync_membership_snapshots` as expected but when someone rejoined the room (or had any membership change), the upsert didn't overwrite and reset the `forgotten` status so it remained `forgotten` and invisible down the Sliding Sync endpoint. --- changelog.d/17835.bugfix | 1 + synapse/storage/databases/main/events.py | 11 +- .../databases/main/events_bg_updates.py | 118 ++++++++++++++++++ synapse/storage/databases/main/roommember.py | 1 + synapse/storage/schema/__init__.py | 2 + ..._membership_snapshots_forgotten_column.sql | 21 ++++ synapse/types/storage/__init__.py | 3 + .../client/sliding_sync/test_sliding_sync.py | 115 ++++++++++++++++- tests/rest/client/test_rooms.py | 62 +++++++++ tests/storage/test_sliding_sync_tables.py | 103 +++++++++++++++ 10 files changed, 433 insertions(+), 4 deletions(-) create mode 100644 changelog.d/17835.bugfix create mode 100644 synapse/storage/schema/main/delta/88/02_fix_sliding_sync_membership_snapshots_forgotten_column.sql diff --git a/changelog.d/17835.bugfix b/changelog.d/17835.bugfix new file mode 100644 index 00000000000..b970598c354 --- /dev/null +++ b/changelog.d/17835.bugfix @@ -0,0 +1 @@ +Fix a bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync that would cause rooms to stay forgotten and hidden even after rejoining. diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index c0b7d8107d7..e5eae9cee99 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -1863,10 +1863,10 @@ def _update_current_state_txn( txn.execute_batch( f""" INSERT INTO sliding_sync_membership_snapshots - (room_id, user_id, sender, membership_event_id, membership, event_stream_ordering, event_instance_name + (room_id, user_id, sender, membership_event_id, membership, forgotten, event_stream_ordering, event_instance_name {("," + ", ".join(sliding_sync_snapshot_keys)) if sliding_sync_snapshot_keys else ""}) VALUES ( - ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, (SELECT stream_ordering FROM events WHERE event_id = ?), (SELECT COALESCE(instance_name, 'master') FROM events WHERE event_id = ?) {("," + ", ".join("?" for _ in sliding_sync_snapshot_values)) if sliding_sync_snapshot_values else ""} @@ -1876,6 +1876,7 @@ def _update_current_state_txn( sender = EXCLUDED.sender, membership_event_id = EXCLUDED.membership_event_id, membership = EXCLUDED.membership, + forgotten = EXCLUDED.forgotten, event_stream_ordering = EXCLUDED.event_stream_ordering {("," + ", ".join(f"{key} = EXCLUDED.{key}" for key in sliding_sync_snapshot_keys)) if sliding_sync_snapshot_keys else ""} """, @@ -1886,6 +1887,9 @@ def _update_current_state_txn( membership_info.sender, membership_info.membership_event_id, membership_info.membership, + # Since this is a new membership, it isn't forgotten anymore (which + # matches how Synapse currently thinks about the forgotten status) + 0, # XXX: We do not use `membership_info.membership_event_stream_ordering` here # because it is an unreliable value. See XXX note above. membership_info.membership_event_id, @@ -2901,6 +2905,9 @@ def _store_room_members_txn( "sender": event.sender, "membership_event_id": event.event_id, "membership": event.membership, + # Since this is a new membership, it isn't forgotten anymore (which + # matches how Synapse currently thinks about the forgotten status) + "forgotten": 0, "event_stream_ordering": event.internal_metadata.stream_ordering, "event_instance_name": event.internal_metadata.instance_name, } diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index a8723f94bc8..4b0bdd79c67 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -304,6 +304,12 @@ def __init__( _BackgroundUpdates.SLIDING_SYNC_MEMBERSHIP_SNAPSHOTS_BG_UPDATE, self._sliding_sync_membership_snapshots_bg_update, ) + # Add a background update to fix data integrity issue in the + # `sliding_sync_membership_snapshots` -> `forgotten` column + self.db_pool.updates.register_background_update_handler( + _BackgroundUpdates.SLIDING_SYNC_MEMBERSHIP_SNAPSHOTS_FIX_FORGOTTEN_COLUMN_BG_UPDATE, + self._sliding_sync_membership_snapshots_fix_forgotten_column_bg_update, + ) # We want this to run on the main database at startup before we start processing # events. @@ -2429,6 +2435,118 @@ def _fill_table_txn(txn: LoggingTransaction) -> None: return len(memberships_to_update_rows) + async def _sliding_sync_membership_snapshots_fix_forgotten_column_bg_update( + self, progress: JsonDict, batch_size: int + ) -> int: + """ + Background update to update the `sliding_sync_membership_snapshots` -> + `forgotten` column to be in sync with the `room_memberships` table. + + Because of previously flawed code (now fixed); any room that someone has + forgotten and subsequently re-joined or had any new membership on, we need to go + and update the column to match the `room_memberships` table as it has fallen out + of sync. + """ + last_event_stream_ordering = progress.get( + "last_event_stream_ordering", -(1 << 31) + ) + + def _txn( + txn: LoggingTransaction, + ) -> int: + """ + Returns: + The number of rows updated. + """ + + # To simplify things, we can just recheck any row in + # `sliding_sync_membership_snapshots` with `forgotten=1` + txn.execute( + """ + SELECT + s.room_id, + s.user_id, + s.membership_event_id, + s.event_stream_ordering, + m.forgotten + FROM sliding_sync_membership_snapshots AS s + INNER JOIN room_memberships AS m ON (s.membership_event_id = m.event_id) + WHERE s.event_stream_ordering > ? + AND s.forgotten = 1 + ORDER BY s.event_stream_ordering ASC + LIMIT ? + """, + (last_event_stream_ordering, batch_size), + ) + + memberships_to_update_rows = cast( + List[Tuple[str, str, str, int, int]], + txn.fetchall(), + ) + if not memberships_to_update_rows: + return 0 + + # Assemble the values to update + # + # (room_id, user_id) + key_values: List[Tuple[str, str]] = [] + # (forgotten,) + value_values: List[Tuple[int]] = [] + for ( + room_id, + user_id, + _membership_event_id, + _event_stream_ordering, + forgotten, + ) in memberships_to_update_rows: + key_values.append( + ( + room_id, + user_id, + ) + ) + value_values.append((forgotten,)) + + # Update all of the rows in one go + self.db_pool.simple_update_many_txn( + txn, + table="sliding_sync_membership_snapshots", + key_names=("room_id", "user_id"), + key_values=key_values, + value_names=("forgotten",), + value_values=value_values, + ) + + # Update the progress + ( + _room_id, + _user_id, + _membership_event_id, + event_stream_ordering, + _forgotten, + ) = memberships_to_update_rows[-1] + self.db_pool.updates._background_update_progress_txn( + txn, + _BackgroundUpdates.SLIDING_SYNC_MEMBERSHIP_SNAPSHOTS_FIX_FORGOTTEN_COLUMN_BG_UPDATE, + { + "last_event_stream_ordering": event_stream_ordering, + }, + ) + + return len(memberships_to_update_rows) + + num_rows = await self.db_pool.runInteraction( + "_sliding_sync_membership_snapshots_fix_forgotten_column_bg_update", + _txn, + ) + + if not num_rows: + await self.db_pool.updates._end_background_update( + _BackgroundUpdates.SLIDING_SYNC_MEMBERSHIP_SNAPSHOTS_FIX_FORGOTTEN_COLUMN_BG_UPDATE + ) + + return num_rows + def _resolve_stale_data_in_sliding_sync_tables( txn: LoggingTransaction, diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index c77e009d030..4249cf77e55 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -1375,6 +1375,7 @@ def f(txn: LoggingTransaction) -> None: keyvalues={"user_id": user_id, "room_id": room_id}, updatevalues={"forgotten": 1}, ) + # Handle updating the `sliding_sync_membership_snapshots` table self.db_pool.simple_update_txn( txn, table="sliding_sync_membership_snapshots", diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index f171f4568a0..934e1cccedb 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -153,6 +153,8 @@ Changes in SCHEMA_VERSION = 88 - MSC4140: Add `delayed_events` table that keeps track of events that are to be posted in response to a resettable timeout or an on-demand action. + - Add background update to fix data integrity issue in the + `sliding_sync_membership_snapshots` -> `forgotten` column """ diff --git a/synapse/storage/schema/main/delta/88/02_fix_sliding_sync_membership_snapshots_forgotten_column.sql b/synapse/storage/schema/main/delta/88/02_fix_sliding_sync_membership_snapshots_forgotten_column.sql new file mode 100644 index 00000000000..4de46af2fc0 --- /dev/null +++ b/synapse/storage/schema/main/delta/88/02_fix_sliding_sync_membership_snapshots_forgotten_column.sql @@ -0,0 +1,21 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2024 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Add a background update to update the `sliding_sync_membership_snapshots` -> +-- `forgotten` column to be in sync with the `room_memberships` table. +-- +-- For any room that someone has forgotten and subsequently re-joined or had any new +-- membership on, we need to go and update the column to match the `room_memberships` +-- table as it has fallen out of sync. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (8802, 'sliding_sync_membership_snapshots_fix_forgotten_column_bg_update', '{}'); diff --git a/synapse/types/storage/__init__.py b/synapse/types/storage/__init__.py index fae5449bcc3..b5fa20a41a5 100644 --- a/synapse/types/storage/__init__.py +++ b/synapse/types/storage/__init__.py @@ -45,3 +45,6 @@ class _BackgroundUpdates: SLIDING_SYNC_MEMBERSHIP_SNAPSHOTS_BG_UPDATE = ( "sliding_sync_membership_snapshots_bg_update" ) + SLIDING_SYNC_MEMBERSHIP_SNAPSHOTS_FIX_FORGOTTEN_COLUMN_BG_UPDATE = ( + "sliding_sync_membership_snapshots_fix_forgotten_column_bg_update" + ) diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index ea3ca57957d..578cb384cd7 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -240,6 +240,7 @@ def _create_remote_invite_room_for_user( self, invitee_user_id: str, unsigned_invite_room_state: Optional[List[StrippedStateEvent]], + invite_room_id: Optional[str] = None, ) -> str: """ Create a fake invite for a remote room and persist it. @@ -252,19 +253,23 @@ def _create_remote_invite_room_for_user( invitee_user_id: The person being invited unsigned_invite_room_state: List of stripped state events to assist the receiver in identifying the room. + invite_room_id: Optional remote room ID to be invited to. When unset, we + will generate one. Returns: The room ID of the remote invite room """ store = self.hs.get_datastores().main - invite_room_id = f"!test_room{self._remote_invite_count}:remote_server" + if invite_room_id is None: + invite_room_id = f"!test_room{self._remote_invite_count}:remote_server" invite_event_dict = { "room_id": invite_room_id, "sender": "@inviter:remote_server", "state_key": invitee_user_id, - "depth": 1, + # Just keep advancing the depth + "depth": self._remote_invite_count, "origin_server_ts": 1, "type": EventTypes.Member, "content": {"membership": Membership.INVITE}, @@ -679,6 +684,112 @@ def test_forgotten_up_to_date(self) -> None: exact=True, ) + def test_rejoin_forgotten_room(self) -> None: + """ + Make sure we can see a forgotten room again if we rejoin (or any new membership + like an invite) (no longer forgotten) + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + room_id = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + # User1 joins the room + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Make the Sliding Sync request + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 99]], + "required_state": [], + "timeline_limit": 0, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + # We should see the room (like normal) + self.assertIncludes( + set(response_body["lists"]["foo-list"]["ops"][0]["room_ids"]), + {room_id}, + exact=True, + ) + + # Leave and forget the room + self.helper.leave(room_id, user1_id, tok=user1_tok) + # User1 forgets the room + channel = self.make_request( + "POST", + f"/_matrix/client/r0/rooms/{room_id}/forget", + content={}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Re-join the room + self.helper.join(room_id, user1_id, tok=user1_tok) + + # We should see the room again after re-joining + response_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) + self.assertIncludes( + set(response_body["lists"]["foo-list"]["ops"][0]["room_ids"]), + {room_id}, + exact=True, + ) + + def test_invited_to_forgotten_remote_room(self) -> None: + """ + Make sure we can see a forgotten room again if we are invited again + (remote/federated out-of-band memberships) + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + # Create a remote room invite (out-of-band membership) + room_id = self._create_remote_invite_room_for_user(user1_id, None) + + # Make the Sliding Sync request + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 99]], + "required_state": [], + "timeline_limit": 0, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + # We should see the room (like normal) + self.assertIncludes( + set(response_body["lists"]["foo-list"]["ops"][0]["room_ids"]), + {room_id}, + exact=True, + ) + + # Leave and forget the room + self.helper.leave(room_id, user1_id, tok=user1_tok) + # User1 forgets the room + channel = self.make_request( + "POST", + f"/_matrix/client/r0/rooms/{room_id}/forget", + content={}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Get invited to the room again + # self.helper.join(room_id, user1_id, tok=user1_tok) + self._create_remote_invite_room_for_user(user1_id, None, invite_room_id=room_id) + + # We should see the room again after re-joining + response_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) + self.assertIncludes( + set(response_body["lists"]["foo-list"]["ops"][0]["room_ids"]), + {room_id}, + exact=True, + ) + def test_ignored_user_invites_initial_sync(self) -> None: """ Make sure we ignore invites if they are from one of the `m.ignored_user_list` on diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 2ecd37ca1a0..07600418ed8 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -2894,6 +2894,68 @@ def _check_for_reason(self, reason: str) -> None: self.assertEqual(event_content.get("reason"), reason, channel.result) +class RoomForgottenTestCase(unittest.HomeserverTestCase): + """ + Test forget/forgotten rooms + """ + + servlets = [ + synapse.rest.admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + + def test_room_not_forgotten_after_unban(self) -> None: + """ + Test what happens when someone is banned from a room, they forget the room, and + some time later are unbanned. + + Currently, when they are unbanned, the room isn't forgotten anymore which may or + may not be expected. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + room_id = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # User1 is banned and forgets the room + self.helper.ban(room_id, src=user2_id, targ=user1_id, tok=user2_tok) + # User1 forgets the room + self.get_success(self.store.forget(user1_id, room_id)) + + # The room should show up as forgotten + forgotten_room_ids = self.get_success( + self.store.get_forgotten_rooms_for_user(user1_id) + ) + self.assertIncludes(forgotten_room_ids, {room_id}, exact=True) + + # Unban user1 + self.helper.change_membership( + room=room_id, + src=user2_id, + targ=user1_id, + membership=Membership.LEAVE, + tok=user2_tok, + ) + + # Room is no longer forgotten because it's a new membership + # + # XXX: Is this how we actually want it to behave? It seems like ideally, the + # room forgotten status should only be reset when the user decides to join again + # (or is invited/knocks). This way the room remains forgotten for any ban/leave + # transitions. + forgotten_room_ids = self.get_success( + self.store.get_forgotten_rooms_for_user(user1_id) + ) + self.assertIncludes(forgotten_room_ids, set(), exact=True) + + class LabelsTestCase(unittest.HomeserverTestCase): servlets = [ synapse.rest.admin.register_servlets_for_client_rest_resource, diff --git a/tests/storage/test_sliding_sync_tables.py b/tests/storage/test_sliding_sync_tables.py index 35917505a42..53212f7c452 100644 --- a/tests/storage/test_sliding_sync_tables.py +++ b/tests/storage/test_sliding_sync_tables.py @@ -5014,3 +5014,106 @@ def test_membership_snapshots_background_update_catch_up_no_membership( }, exact=True, ) + + +class SlidingSyncMembershipSnapshotsTableFixForgottenColumnBackgroundUpdatesTestCase( + SlidingSyncTablesTestCaseBase +): + """ + Test the background updates that fixes `sliding_sync_membership_snapshots` -> + `forgotten` column. + """ + + def test_membership_snapshots_fix_forgotten_column_background_update(self) -> None: + """ + Test that the background update, updates the `sliding_sync_membership_snapshots` + -> `forgotten` column to be in sync with the `room_memberships` table. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + room_id = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + # User1 joins the room + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Leave and forget the room + self.helper.leave(room_id, user1_id, tok=user1_tok) + # User1 forgets the room + channel = self.make_request( + "POST", + f"/_matrix/client/r0/rooms/{room_id}/forget", + content={}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Re-join the room + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Reset `sliding_sync_membership_snapshots` table as if the `forgotten` column + # got out of sync from the `room_memberships` table from the previous flawed + # code. + self.get_success( + self.store.db_pool.simple_update_one( + table="sliding_sync_membership_snapshots", + keyvalues={"room_id": room_id, "user_id": user1_id}, + updatevalues={"forgotten": 1}, + desc="sliding_sync_membership_snapshots.test_membership_snapshots_fix_forgotten_column_background_update", + ) + ) + + # Insert and run the background update. + self.get_success( + self.store.db_pool.simple_insert( + "background_updates", + { + "update_name": _BackgroundUpdates.SLIDING_SYNC_MEMBERSHIP_SNAPSHOTS_FIX_FORGOTTEN_COLUMN_BG_UPDATE, + "progress_json": "{}", + }, + ) + ) + self.store.db_pool.updates._all_done = False + self.wait_for_background_updates() + + # Make sure the table is populated + + sliding_sync_membership_snapshots_results = ( + self._get_sliding_sync_membership_snapshots() + ) + self.assertIncludes( + set(sliding_sync_membership_snapshots_results.keys()), + { + (room_id, user1_id), + (room_id, user2_id), + }, + exact=True, + ) + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id) + ) + # Holds the info according to the current state when the user joined. + # + # We only care about checking on user1 as that's what we reset and expect to be + # correct now + self.assertEqual( + sliding_sync_membership_snapshots_results.get((room_id, user1_id)), + _SlidingSyncMembershipSnapshotResult( + room_id=room_id, + user_id=user1_id, + sender=user1_id, + membership_event_id=state_map[(EventTypes.Member, user1_id)].event_id, + membership=Membership.JOIN, + event_stream_ordering=state_map[ + (EventTypes.Member, user1_id) + ].internal_metadata.stream_ordering, + has_known_state=True, + room_type=None, + room_name=None, + is_encrypted=False, + tombstone_successor_room_id=None, + # We should see the room as no longer forgotten + forgotten=False, + ), + ) From 4b94a056bd27a05f278cca6a96511b3ea28d57f8 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 22 Oct 2024 11:56:08 +0100 Subject: [PATCH 03/31] 1.118.0rc1 --- CHANGES.md | 57 +++++++++++++++++++++++++++++++++++++++ changelog.d/17627.doc | 1 - changelog.d/17708.feature | 1 - changelog.d/17718.misc | 1 - changelog.d/17736.bugfix | 1 - changelog.d/17752.misc | 1 - changelog.d/17783.feature | 1 - changelog.d/17785.bugfix | 1 - changelog.d/17802.doc | 1 - changelog.d/17803.misc | 1 - changelog.d/17805.bugfix | 1 - changelog.d/17824.misc | 1 - changelog.d/17825.doc | 1 - changelog.d/17826.misc | 1 - changelog.d/17835.bugfix | 1 - changelog.d/17842.misc | 1 - debian/changelog | 6 +++++ pyproject.toml | 2 +- 18 files changed, 64 insertions(+), 16 deletions(-) delete mode 100644 changelog.d/17627.doc delete mode 100644 changelog.d/17708.feature delete mode 100644 changelog.d/17718.misc delete mode 100644 changelog.d/17736.bugfix delete mode 100644 changelog.d/17752.misc delete mode 100644 changelog.d/17783.feature delete mode 100644 changelog.d/17785.bugfix delete mode 100644 changelog.d/17802.doc delete mode 100644 changelog.d/17803.misc delete mode 100644 changelog.d/17805.bugfix delete mode 100644 changelog.d/17824.misc delete mode 100644 changelog.d/17825.doc delete mode 100644 changelog.d/17826.misc delete mode 100644 changelog.d/17835.bugfix delete mode 100644 changelog.d/17842.misc diff --git a/CHANGES.md b/CHANGES.md index ba45fe01564..1e3b5370fc0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,60 @@ +# Synapse 1.118.0rc1 (2024-10-22) + +### Notice of dropping Python 3.8 support in next release + +Python 3.8 is now [end-of-life](https://devguide.python.org/versions/). As per our [Deprecation Policy for Platform Dependencies](https://element-hq.github.io/synapse/latest/deprecation_policy.html#policy), Synpase will be dropping support for Python 3.8 in the next release; Synapse 1.119.0. + +1.118.x will be the final Synapse release to support Python 3.8. Python upgrade to a more recent Python version if necessary. + +### Python 3.13 and PostgreSQL 17 support + +On the other end of the spectrum, Synapse 1.118.0 is the first release to support [Python 3.13](https://www.python.org/downloads/release/python-3130/)! + +[PostgreSQL 17](https://www.postgresql.org/about/news/postgresql-17-released-2936/) is also supported as of this release. + +### Features + +- Added the `display_name_claim` option to the JWT configuration. This option allows specifying the claim key that contains the user's display name in the JWT payload. ([\#17708](https://github.com/element-hq/synapse/issues/17708)) +- Implement [MSC4210](https://github.com/matrix-org/matrix-spec-proposals/pull/4210): Remove legacy mentions. Contributed by @tulir @ Beeper. ([\#17783](https://github.com/element-hq/synapse/issues/17783)) + +### Bugfixes + +- Fix saving of PNG thumbnails, when the original image is in the CMYK color space. ([\#17736](https://github.com/element-hq/synapse/issues/17736)) +- Fix bug with sliding sync where the server would not return state that was added to the `required_state` config. ([\#17785](https://github.com/element-hq/synapse/issues/17785), [\#17805](https://github.com/element-hq/synapse/issues/17805)) +- Fix a bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync that would cause rooms to stay forgotten and hidden even after rejoining. ([\#17835](https://github.com/element-hq/synapse/issues/17835)) + +### Improved Documentation + +- Clarify when the `user_may_invite` and `user_may_send_3pid_invite` module callbacks are called. ([\#17627](https://github.com/element-hq/synapse/issues/17627)) +- Correct documentation to refer to the `--config-path` argument instead of `--config-file`. ([\#17802](https://github.com/element-hq/synapse/issues/17802)) +- Fix typo in `target_cache_memory_usage` docs. ([\#17825](https://github.com/element-hq/synapse/issues/17825)) + +### Internal Changes + +- Slight optimization when fetching state/events for Sliding Sync. ([\#17718](https://github.com/element-hq/synapse/issues/17718)) +- Add Python 3.13 and Postgres 17 to the test matrix. ([\#17752](https://github.com/element-hq/synapse/issues/17752)) +- Test github token before running release script steps. ([\#17803](https://github.com/element-hq/synapse/issues/17803)) +- Build debian packages for new Ubuntu versions, and stop building for no longer supported versions. ([\#17824](https://github.com/element-hq/synapse/issues/17824)) +- Enable the `.org.matrix.msc4028.encrypted_event` push rule by default in accordance with [MSC4028](https://github.com/matrix-org/matrix-spec-proposals/pull/4028). Note that the corresponding experimental feature must still be switched on for this push rule to have any effect. ([\#17826](https://github.com/element-hq/synapse/issues/17826)) +- Fix some typing issues uncovered by upgrading mypy to 1.11.x. ([\#17842](https://github.com/element-hq/synapse/issues/17842)) + + + +### Updates to locked dependencies + +* Bump mypy from 1.10.1 to 1.11.2. ([\#17842](https://github.com/element-hq/synapse/issues/17842)) +* Bump mypy-zope from 1.0.5 to 1.0.7. ([\#17827](https://github.com/element-hq/synapse/issues/17827)) +* Bump phonenumbers from 8.13.46 to 8.13.47. ([\#17797](https://github.com/element-hq/synapse/issues/17797)) +* Bump psycopg2 from 2.9.9 to 2.9.10. ([\#17843](https://github.com/element-hq/synapse/issues/17843)) +* Bump ruff from 0.6.8 to 0.6.9. ([\#17794](https://github.com/element-hq/synapse/issues/17794)) +* Bump sentry-sdk from 2.14.0 to 2.15.0. ([\#17795](https://github.com/element-hq/synapse/issues/17795)) +* Bump sentry-sdk from 2.15.0 to 2.16.0. ([\#17829](https://github.com/element-hq/synapse/issues/17829)) +* Bump sentry-sdk from 2.16.0 to 2.17.0. ([\#17844](https://github.com/element-hq/synapse/issues/17844)) +* Bump sigstore/cosign-installer from 3.6.0 to 3.7.0. ([\#17798](https://github.com/element-hq/synapse/issues/17798)) +* Bump tomli from 2.0.1 to 2.0.2. ([\#17796](https://github.com/element-hq/synapse/issues/17796)) +* Bump types-requests from 2.32.0.20240914 to 2.32.0.20241016. ([\#17841](https://github.com/element-hq/synapse/issues/17841)) +* Bump types-setuptools from 75.1.0.20240917 to 75.1.0.20241014. ([\#17828](https://github.com/element-hq/synapse/issues/17828)) + # Synapse 1.117.0 (2024-10-15) No significant changes since 1.117.0rc1. diff --git a/changelog.d/17627.doc b/changelog.d/17627.doc deleted file mode 100644 index 487a0aea0da..00000000000 --- a/changelog.d/17627.doc +++ /dev/null @@ -1 +0,0 @@ -Clarify when the `user_may_invite` and `user_may_send_3pid_invite` module callbacks are called. \ No newline at end of file diff --git a/changelog.d/17708.feature b/changelog.d/17708.feature deleted file mode 100644 index 90ec810f50d..00000000000 --- a/changelog.d/17708.feature +++ /dev/null @@ -1 +0,0 @@ -Added the `display_name_claim` option to the JWT configuration. This option allows specifying the claim key that contains the user's display name in the JWT payload. \ No newline at end of file diff --git a/changelog.d/17718.misc b/changelog.d/17718.misc deleted file mode 100644 index ea73a03f537..00000000000 --- a/changelog.d/17718.misc +++ /dev/null @@ -1 +0,0 @@ -Slight optimization when fetching state/events for Sliding Sync. diff --git a/changelog.d/17736.bugfix b/changelog.d/17736.bugfix deleted file mode 100644 index 0d3fd06962c..00000000000 --- a/changelog.d/17736.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix saving of PNG thumbnails, when the original image is in the CMYK color space. diff --git a/changelog.d/17752.misc b/changelog.d/17752.misc deleted file mode 100644 index f9f01a42420..00000000000 --- a/changelog.d/17752.misc +++ /dev/null @@ -1 +0,0 @@ -Add Python 3.13 and Postgres 17 to the test matrix. diff --git a/changelog.d/17783.feature b/changelog.d/17783.feature deleted file mode 100644 index ce8c2164185..00000000000 --- a/changelog.d/17783.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [MSC4210](https://github.com/matrix-org/matrix-spec-proposals/pull/4210): Remove legacy mentions. Contributed by @tulir @ Beeper. diff --git a/changelog.d/17785.bugfix b/changelog.d/17785.bugfix deleted file mode 100644 index df2898f54e3..00000000000 --- a/changelog.d/17785.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug with sliding sync where the server would not return state that was added to the `required_state` config. diff --git a/changelog.d/17802.doc b/changelog.d/17802.doc deleted file mode 100644 index 72e653d3c4a..00000000000 --- a/changelog.d/17802.doc +++ /dev/null @@ -1 +0,0 @@ -Correct documentation to refer to the `--config-path` argument instead of `--config-file`. diff --git a/changelog.d/17803.misc b/changelog.d/17803.misc deleted file mode 100644 index a267df8b83c..00000000000 --- a/changelog.d/17803.misc +++ /dev/null @@ -1 +0,0 @@ -Test github token before running release script steps. diff --git a/changelog.d/17805.bugfix b/changelog.d/17805.bugfix deleted file mode 100644 index df2898f54e3..00000000000 --- a/changelog.d/17805.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug with sliding sync where the server would not return state that was added to the `required_state` config. diff --git a/changelog.d/17824.misc b/changelog.d/17824.misc deleted file mode 100644 index 22574f00ec3..00000000000 --- a/changelog.d/17824.misc +++ /dev/null @@ -1 +0,0 @@ -Build debian packages for new Ubuntu versions, and stop building for no longer supported versions. diff --git a/changelog.d/17825.doc b/changelog.d/17825.doc deleted file mode 100644 index ee436674175..00000000000 --- a/changelog.d/17825.doc +++ /dev/null @@ -1 +0,0 @@ -Fix typo in `target_cache_memory_usage` docs. \ No newline at end of file diff --git a/changelog.d/17826.misc b/changelog.d/17826.misc deleted file mode 100644 index 9148c96a0d4..00000000000 --- a/changelog.d/17826.misc +++ /dev/null @@ -1 +0,0 @@ -Enable the `.org.matrix.msc4028.encrypted_event` push rule by default in accordance with [MSC4028](https://github.com/matrix-org/matrix-spec-proposals/pull/4028). Note that the corresponding experimental feature must still be switched on for this push rule to have any effect. \ No newline at end of file diff --git a/changelog.d/17835.bugfix b/changelog.d/17835.bugfix deleted file mode 100644 index b970598c354..00000000000 --- a/changelog.d/17835.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync that would cause rooms to stay forgotten and hidden even after rejoining. diff --git a/changelog.d/17842.misc b/changelog.d/17842.misc deleted file mode 100644 index 78af706c31b..00000000000 --- a/changelog.d/17842.misc +++ /dev/null @@ -1 +0,0 @@ -Fix some typing issues uncovered by upgrading mypy to 1.11.x. diff --git a/debian/changelog b/debian/changelog index 1995fbf6f63..9fb360c3302 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.118.0~rc1) stable; urgency=medium + + * New Synapse release 1.118.0rc1. + + -- Synapse Packaging team Tue, 22 Oct 2024 11:48:14 +0100 + matrix-synapse-py3 (1.117.0) stable; urgency=medium * New Synapse release 1.117.0. diff --git a/pyproject.toml b/pyproject.toml index 658771a89af..6cdd6850b11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.117.0" +version = "1.118.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 46c885f5b5ce85f47a8d8a259f66520b6f87d7ae Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 22 Oct 2024 12:00:40 +0100 Subject: [PATCH 04/31] fix spelling in changelog --- CHANGES.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1e3b5370fc0..38338f1fdbe 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,16 +1,14 @@ # Synapse 1.118.0rc1 (2024-10-22) -### Notice of dropping Python 3.8 support in next release +### Python 3.8 support will be dropped in the next release -Python 3.8 is now [end-of-life](https://devguide.python.org/versions/). As per our [Deprecation Policy for Platform Dependencies](https://element-hq.github.io/synapse/latest/deprecation_policy.html#policy), Synpase will be dropping support for Python 3.8 in the next release; Synapse 1.119.0. +Python 3.8 is now [end-of-life](https://devguide.python.org/versions/). As per our [Deprecation Policy for Platform Dependencies](https://element-hq.github.io/synapse/latest/deprecation_policy.html#policy), Synapse will be dropping support for Python 3.8 in the next release; Synapse 1.119.0. -1.118.x will be the final Synapse release to support Python 3.8. Python upgrade to a more recent Python version if necessary. +Synapse 1.118.x will be the final release to support Python 3.8. If you are running Synapse with Python 3.8, please upgrade before the 1.119.0 release, due in less than one month. ### Python 3.13 and PostgreSQL 17 support -On the other end of the spectrum, Synapse 1.118.0 is the first release to support [Python 3.13](https://www.python.org/downloads/release/python-3130/)! - -[PostgreSQL 17](https://www.postgresql.org/about/news/postgresql-17-released-2936/) is also supported as of this release. +On the other end of the spectrum, Synapse 1.118.0 is the first release to support [Python 3.13](https://www.python.org/downloads/release/python-3130/)! [PostgreSQL 17](https://www.postgresql.org/about/news/postgresql-17-released-2936/) is also supported as of this release. ### Features From 57efc8c03eb27048c721f389f0b56e7bea145be3 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 23 Oct 2024 18:26:01 +0100 Subject: [PATCH 05/31] Add media tests for a CMYK JPEG image (#17786) --- changelog.d/17786.misc | 1 + tests/media/test_media_storage.py | 64 ++++++++++++++++++++++++++++++- tests/rest/client/test_media.py | 4 +- tests/test_utils/__init__.py | 19 +++++++++ 4 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 changelog.d/17786.misc diff --git a/changelog.d/17786.misc b/changelog.d/17786.misc new file mode 100644 index 00000000000..59eb3f4dbdb --- /dev/null +++ b/changelog.d/17786.misc @@ -0,0 +1 @@ +Add a test for downloading and thumbnailing a CMYK JPEG. \ No newline at end of file diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index e50ff5fa780..034d9ece0b5 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -60,7 +60,7 @@ from tests import unittest from tests.server import FakeChannel -from tests.test_utils import SMALL_PNG +from tests.test_utils import SMALL_CMYK_JPEG, SMALL_PNG from tests.unittest import override_config from tests.utils import default_config @@ -187,6 +187,68 @@ class TestImage: # different versions of Pillow. ) +small_cmyk_jpeg = TestImage( + SMALL_CMYK_JPEG, + b"image/jpeg", + b".jpeg", + # These values were sourced simply by seeing at what the tests produced at + # the time of writing. If this changes, the tests will fail. + unhexlify( + b"ffd8ffe000104a46494600010100000100010000ffdb00430006" + b"040506050406060506070706080a100a0a09090a140e0f0c1017" + b"141818171416161a1d251f1a1b231c1616202c20232627292a29" + b"191f2d302d283025282928ffdb0043010707070a080a130a0a13" + b"281a161a28282828282828282828282828282828282828282828" + b"2828282828282828282828282828282828282828282828282828" + b"2828ffc00011080020002003012200021101031101ffc4001f00" + b"0001050101010101010000000000000000010203040506070809" + b"0a0bffc400b5100002010303020403050504040000017d010203" + b"00041105122131410613516107227114328191a1082342b1c115" + b"52d1f02433627282090a161718191a25262728292a3435363738" + b"393a434445464748494a535455565758595a636465666768696a" + b"737475767778797a838485868788898a92939495969798999aa2" + b"a3a4a5a6a7a8a9aab2b3b4b5b6b7b8b9bac2c3c4c5c6c7c8c9ca" + b"d2d3d4d5d6d7d8d9dae1e2e3e4e5e6e7e8e9eaf1f2f3f4f5f6f7" + b"f8f9faffc4001f01000301010101010101010100000000000001" + b"02030405060708090a0bffc400b5110002010204040304070504" + b"0400010277000102031104052131061241510761711322328108" + b"144291a1b1c109233352f0156272d10a162434e125f11718191a" + b"262728292a35363738393a434445464748494a53545556575859" + b"5a636465666768696a737475767778797a82838485868788898a" + b"92939495969798999aa2a3a4a5a6a7a8a9aab2b3b4b5b6b7b8b9" + b"bac2c3c4c5c6c7c8c9cad2d3d4d5d6d7d8d9dae2e3e4e5e6e7e8" + b"e9eaf2f3f4f5f6f7f8f9faffda000c03010002110311003f00fa" + b"a68a28a0028a28a0028a28a0028a28a00fffd9" + ), + unhexlify( + b"ffd8ffe000104a46494600010100000100010000ffdb00430006" + b"040506050406060506070706080a100a0a09090a140e0f0c1017" + b"141818171416161a1d251f1a1b231c1616202c20232627292a29" + b"191f2d302d283025282928ffdb0043010707070a080a130a0a13" + b"281a161a28282828282828282828282828282828282828282828" + b"2828282828282828282828282828282828282828282828282828" + b"2828ffc00011080001000103012200021101031101ffc4001f00" + b"0001050101010101010000000000000000010203040506070809" + b"0a0bffc400b5100002010303020403050504040000017d010203" + b"00041105122131410613516107227114328191a1082342b1c115" + b"52d1f02433627282090a161718191a25262728292a3435363738" + b"393a434445464748494a535455565758595a636465666768696a" + b"737475767778797a838485868788898a92939495969798999aa2" + b"a3a4a5a6a7a8a9aab2b3b4b5b6b7b8b9bac2c3c4c5c6c7c8c9ca" + b"d2d3d4d5d6d7d8d9dae1e2e3e4e5e6e7e8e9eaf1f2f3f4f5f6f7" + b"f8f9faffc4001f01000301010101010101010100000000000001" + b"02030405060708090a0bffc400b5110002010204040304070504" + b"0400010277000102031104052131061241510761711322328108" + b"144291a1b1c109233352f0156272d10a162434e125f11718191a" + b"262728292a35363738393a434445464748494a53545556575859" + b"5a636465666768696a737475767778797a82838485868788898a" + b"92939495969798999aa2a3a4a5a6a7a8a9aab2b3b4b5b6b7b8b9" + b"bac2c3c4c5c6c7c8c9cad2d3d4d5d6d7d8d9dae2e3e4e5e6e7e8" + b"e9eaf2f3f4f5f6f7f8f9faffda000c03010002110311003f00fa" + b"a68a28a00fffd9" + ), +) + small_lossless_webp = TestImage( unhexlify( b"524946461a000000574542505650384c0d0000002f0000001007" b"1011118888fe0700" diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index 42014e257e4..4060525efe3 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -66,6 +66,7 @@ SVG, TestImage, empty_file, + small_cmyk_jpeg, small_lossless_webp, small_png, small_png_with_transparency, @@ -1916,6 +1917,7 @@ def test_file_download(self) -> None: test_images = [ small_png, small_png_with_transparency, + small_cmyk_jpeg, small_lossless_webp, empty_file, SVG, @@ -2400,7 +2402,7 @@ def _test_thumbnail( if expected_body is not None: self.assertEqual( - channel.result["body"], expected_body, channel.result["body"] + channel.result["body"], expected_body, channel.result["body"].hex() ) else: # ensure that the result is at least some valid image diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py index 4d7adf72049..e3611852b79 100644 --- a/tests/test_utils/__init__.py +++ b/tests/test_utils/__init__.py @@ -23,6 +23,7 @@ Utilities for running the unit tests """ +import base64 import json import sys import warnings @@ -138,3 +139,21 @@ def json(cls, *, code: int = 200, payload: JsonSerializable) -> "FakeResponse": b"0000001f15c4890000000a49444154789c63000100000500010d" b"0a2db40000000049454e44ae426082" ) + +# A small CMYK-encoded JPEG image used in some tests. +# +# Generated with: +# img = PIL.Image.new('CMYK', (1, 1), (0, 0, 0, 0)) +# img.save('minimal_cmyk.jpg', 'JPEG') +# +# Resolution: 1x1, MIME type: image/jpeg, Extension: jpeg, Size: 4 KiB +SMALL_CMYK_JPEG = base64.b64decode(""" +/9j/7gAOQWRvYmUAZAAAAAAA/9sAQwAIBgYHBgUIBwcHCQkICgwUDQwLCww +ZEhMPFB0aHx4dGhwcICQuJyAiLCMcHCg3KSwwMTQ0NB8nOT04MjwuMzQy/8 +AAFAgAAQABBEMRAE0RAFkRAEsRAP/EAB8AAAEFAQEBAQEBAAAAAAAAAAABA +gMEBQYHCAkKC//EALUQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNR +YQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkN +ERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlp +eYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5 +ebn6Onq8fLz9PX29/j5+v/aAA4EQwBNAFkASwAAPwD3+vf69/r3+v/Z +""") From 1006c12eb20b28dc1c372517f56a9fa06574e7ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Oct 2024 10:35:37 +0100 Subject: [PATCH 06/31] Bump anyhow from 1.0.89 to 1.0.90 (#17858) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dfc8dc6047f..06d5c48be52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.89" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95" [[package]] name = "arc-swap" From 12d65a677845e8376160a2e836db50a19bd63a82 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Oct 2024 10:40:58 +0100 Subject: [PATCH 07/31] Bump cryptography from 43.0.1 to 43.0.3 (#17853) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 58 ++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/poetry.lock b/poetry.lock index 1102e89d12f..09c4661f37e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -360,38 +360,38 @@ files = [ [[package]] name = "cryptography" -version = "43.0.1" +version = "43.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-43.0.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d"}, - {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062"}, - {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962"}, - {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277"}, - {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a"}, - {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042"}, - {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494"}, - {file = "cryptography-43.0.1-cp37-abi3-win32.whl", hash = "sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2"}, - {file = "cryptography-43.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d"}, - {file = "cryptography-43.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d"}, - {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806"}, - {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85"}, - {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c"}, - {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1"}, - {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa"}, - {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4"}, - {file = "cryptography-43.0.1-cp39-abi3-win32.whl", hash = "sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47"}, - {file = "cryptography-43.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb"}, - {file = "cryptography-43.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034"}, - {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d"}, - {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289"}, - {file = "cryptography-43.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84"}, - {file = "cryptography-43.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365"}, - {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96"}, - {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172"}, - {file = "cryptography-43.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2"}, - {file = "cryptography-43.0.1.tar.gz", hash = "sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d"}, + {file = "cryptography-43.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e1ce50266f4f70bf41a2c6dc4358afadae90e2a1e5342d3c08883df1675374f"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:443c4a81bb10daed9a8f334365fe52542771f25aedaf889fd323a853ce7377d6"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:74f57f24754fe349223792466a709f8e0c093205ff0dca557af51072ff47ab18"}, + {file = "cryptography-43.0.3-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9762ea51a8fc2a88b70cf2995e5675b38d93bf36bd67d91721c309df184f49bd"}, + {file = "cryptography-43.0.3-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:81ef806b1fef6b06dcebad789f988d3b37ccaee225695cf3e07648eee0fc6b73"}, + {file = "cryptography-43.0.3-cp37-abi3-win32.whl", hash = "sha256:cbeb489927bd7af4aa98d4b261af9a5bc025bd87f0e3547e11584be9e9427be2"}, + {file = "cryptography-43.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:f46304d6f0c6ab8e52770addfa2fc41e6629495548862279641972b6215451cd"}, + {file = "cryptography-43.0.3-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:8ac43ae87929a5982f5948ceda07001ee5e83227fd69cf55b109144938d96984"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:846da004a5804145a5f441b8530b4bf35afbf7da70f82409f151695b127213d5"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f996e7268af62598f2fc1204afa98a3b5712313a55c4c9d434aef49cadc91d4"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f7b178f11ed3664fd0e995a47ed2b5ff0a12d893e41dd0494f406d1cf555cab7"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:c2e6fc39c4ab499049df3bdf567f768a723a5e8464816e8f009f121a5a9f4405"}, + {file = "cryptography-43.0.3-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e1be4655c7ef6e1bbe6b5d0403526601323420bcf414598955968c9ef3eb7d16"}, + {file = "cryptography-43.0.3-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:df6b6c6d742395dd77a23ea3728ab62f98379eff8fb61be2744d4679ab678f73"}, + {file = "cryptography-43.0.3-cp39-abi3-win32.whl", hash = "sha256:d56e96520b1020449bbace2b78b603442e7e378a9b3bd68de65c782db1507995"}, + {file = "cryptography-43.0.3-cp39-abi3-win_amd64.whl", hash = "sha256:0c580952eef9bf68c4747774cde7ec1d85a6e61de97281f2dba83c7d2c806362"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d03b5621a135bffecad2c73e9f4deb1a0f977b9a8ffe6f8e002bf6c9d07b918c"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a2a431ee15799d6db9fe80c82b055bae5a752bef645bba795e8e52687c69efe3"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:281c945d0e28c92ca5e5930664c1cefd85efe80e5c0d2bc58dd63383fda29f83"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f18c716be16bc1fea8e95def49edf46b82fccaa88587a45f8dc0ff6ab5d8e0a7"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:4a02ded6cd4f0a5562a8887df8b3bd14e822a90f97ac5e544c162899bc467664"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:53a583b6637ab4c4e3591a15bc9db855b8d9dee9a669b550f311480acab6eb08"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1ec0bcf7e17c0c5669d881b1cd38c4972fade441b27bda1051665faaa89bdcaa"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff"}, + {file = "cryptography-43.0.3.tar.gz", hash = "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805"}, ] [package.dependencies] @@ -404,7 +404,7 @@ nox = ["nox"] pep8test = ["check-sdist", "click", "mypy", "ruff"] sdist = ["build"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi", "cryptography-vectors (==43.0.1)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test = ["certifi", "cryptography-vectors (==43.0.3)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] [[package]] From 4e5410fdae9a5850e8cd771a87cfd185aa2c9dc4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Oct 2024 10:42:38 +0100 Subject: [PATCH 08/31] Bump types-psycopg2 from 2.9.21.20240819 to 2.9.21.20241019 (#17855) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 09c4661f37e..43063ac70fc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2783,13 +2783,13 @@ files = [ [[package]] name = "types-psycopg2" -version = "2.9.21.20240819" +version = "2.9.21.20241019" description = "Typing stubs for psycopg2" optional = false python-versions = ">=3.8" files = [ - {file = "types-psycopg2-2.9.21.20240819.tar.gz", hash = "sha256:4ed6b47464d6374fa64e5e3b234cea0f710e72123a4596d67ab50b7415a84666"}, - {file = "types_psycopg2-2.9.21.20240819-py3-none-any.whl", hash = "sha256:c9192311c27d7ad561eef705f1b2df1074f2cdcf445a98a6a2fcaaaad43278cf"}, + {file = "types-psycopg2-2.9.21.20241019.tar.gz", hash = "sha256:bca89b988d2ebd19bcd08b177d22a877ea8b841decb10ed130afcf39404612fa"}, + {file = "types_psycopg2-2.9.21.20241019-py3-none-any.whl", hash = "sha256:44d091e67732d16a941baae48cd7b53bf91911bc36888652447cf1ef0c1fb3f6"}, ] [[package]] From 5d0514f29b897ef98f8d2c0d60660b348d41ec79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Oct 2024 10:43:40 +0100 Subject: [PATCH 09/31] Bump serde_json from 1.0.128 to 1.0.132 (#17857) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 06d5c48be52..4b6775cecac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -505,9 +505,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", From 69e9b753734ebd28d5bed5f7828a8d34e86ec205 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Oct 2024 10:44:12 +0100 Subject: [PATCH 10/31] Bump types-setuptools from 75.1.0.20241014 to 75.2.0.20241019 (#17856) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 43063ac70fc..9ca72814a44 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2834,13 +2834,13 @@ urllib3 = ">=2" [[package]] name = "types-setuptools" -version = "75.1.0.20241014" +version = "75.2.0.20241019" description = "Typing stubs for setuptools" optional = false python-versions = ">=3.8" files = [ - {file = "types-setuptools-75.1.0.20241014.tar.gz", hash = "sha256:29b0560a8d4b4a91174be085847002c69abfcb048e20b33fc663005aedf56804"}, - {file = "types_setuptools-75.1.0.20241014-py3-none-any.whl", hash = "sha256:caab58366741fb99673d0138b6e2d760717f154cfb981b74fea5e8de40f0b703"}, + {file = "types-setuptools-75.2.0.20241019.tar.gz", hash = "sha256:86ea31b5f6df2c6b8f2dc8ae3f72b213607f62549b6fa2ed5866e5299f968694"}, + {file = "types_setuptools-75.2.0.20241019-py3-none-any.whl", hash = "sha256:2e48ff3acd4919471e80d5e3f049cce5c177e108d5d36d2d4cee3fa4d4104258"}, ] [[package]] From 6c51f8649d5b17eeb218da61583f0ca132fdd728 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 29 Oct 2024 10:09:25 +0000 Subject: [PATCH 11/31] Include the destination in the error of 'Destination mismatch' (#17830) To help debug problems such as https://github.com/element-hq/synapse/issues/17822 --- changelog.d/17830.misc | 1 + synapse/federation/transport/server/_base.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/17830.misc diff --git a/changelog.d/17830.misc b/changelog.d/17830.misc new file mode 100644 index 00000000000..b2cd91c0ad0 --- /dev/null +++ b/changelog.d/17830.misc @@ -0,0 +1 @@ +Include the destination in the error of 'Destination mismatch' on federation requests. diff --git a/synapse/federation/transport/server/_base.py b/synapse/federation/transport/server/_base.py index 9094201da00..cba309635b7 100644 --- a/synapse/federation/transport/server/_base.py +++ b/synapse/federation/transport/server/_base.py @@ -113,7 +113,7 @@ async def authenticate_request( ): raise AuthenticationError( HTTPStatus.UNAUTHORIZED, - "Destination mismatch in auth header", + f"Destination mismatch in auth header, received: {destination!r}", Codes.UNAUTHORIZED, ) if ( From b3b1db4057fea62244804ab83a138a8bb9a93404 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:30:10 +0100 Subject: [PATCH 12/31] 1.118.0 --- CHANGES.md | 7 +++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 38338f1fdbe..bf3e8d692df 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.118.0 (2024-10-29) + +No significant changes since 1.118.0rc1. + + + + # Synapse 1.118.0rc1 (2024-10-22) ### Python 3.8 support will be dropped in the next release diff --git a/debian/changelog b/debian/changelog index 9fb360c3302..384887888fb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.118.0) stable; urgency=medium + + * New Synapse release 1.118.0. + + -- Synapse Packaging team Tue, 29 Oct 2024 15:29:53 +0100 + matrix-synapse-py3 (1.118.0~rc1) stable; urgency=medium * New Synapse release 1.118.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 6cdd6850b11..5af9a4145bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.118.0rc1" +version = "1.118.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 4be3bd41fddd3d6bf3b772be9cf23b16899e089d Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Tue, 29 Oct 2024 17:05:22 +0100 Subject: [PATCH 13/31] Move announcements up --- CHANGES.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index bf3e8d692df..bcc83402104 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,11 +2,6 @@ No significant changes since 1.118.0rc1. - - - -# Synapse 1.118.0rc1 (2024-10-22) - ### Python 3.8 support will be dropped in the next release Python 3.8 is now [end-of-life](https://devguide.python.org/versions/). As per our [Deprecation Policy for Platform Dependencies](https://element-hq.github.io/synapse/latest/deprecation_policy.html#policy), Synapse will be dropping support for Python 3.8 in the next release; Synapse 1.119.0. @@ -17,6 +12,9 @@ Synapse 1.118.x will be the final release to support Python 3.8. If you are runn On the other end of the spectrum, Synapse 1.118.0 is the first release to support [Python 3.13](https://www.python.org/downloads/release/python-3130/)! [PostgreSQL 17](https://www.postgresql.org/about/news/postgresql-17-released-2936/) is also supported as of this release. + +# Synapse 1.118.0rc1 (2024-10-22) + ### Features - Added the `display_name_claim` option to the JWT configuration. This option allows specifying the claim key that contains the user's display name in the JWT payload. ([\#17708](https://github.com/element-hq/synapse/issues/17708)) From d427403c6776ca23a6946b67ce3fb2b3cdcb33be Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 29 Oct 2024 17:06:15 +0000 Subject: [PATCH 14/31] Fix check for outdated Rust library (#17861) This failed when install with poetry, so let's properly try and detect what's going on. --- changelog.d/17861.bugfix | 1 + synapse/util/rust.py | 87 ++++++++++++++++++++++++++++++---------- 2 files changed, 66 insertions(+), 22 deletions(-) create mode 100644 changelog.d/17861.bugfix diff --git a/changelog.d/17861.bugfix b/changelog.d/17861.bugfix new file mode 100644 index 00000000000..abee7a30f70 --- /dev/null +++ b/changelog.d/17861.bugfix @@ -0,0 +1 @@ +Fix detection when the built Rust library was outdated when using source installations. diff --git a/synapse/util/rust.py b/synapse/util/rust.py index 0e35d6d1881..37f43459f1c 100644 --- a/synapse/util/rust.py +++ b/synapse/util/rust.py @@ -19,9 +19,12 @@ # # +import json import os -import sys +import urllib.parse from hashlib import blake2b +from importlib.metadata import Distribution, PackageNotFoundError +from typing import Optional import synapse from synapse.synapse_rust import get_rust_file_digest @@ -32,22 +35,17 @@ def check_rust_lib_up_to_date() -> None: be rebuilt. """ - if not _dist_is_editable(): - return - - synapse_dir = os.path.dirname(synapse.__file__) - synapse_root = os.path.abspath(os.path.join(synapse_dir, "..")) - - # Double check we've not gone into site-packages... - if os.path.basename(synapse_root) == "site-packages": - return - - # ... and it looks like the root of a python project. - if not os.path.exists("pyproject.toml"): - return + # Get the location of the editable install. + synapse_root = get_synapse_source_directory() + if synapse_root is None: + return None # Get the hash of all Rust source files - hash = _hash_rust_files_in_directory(os.path.join(synapse_root, "rust", "src")) + rust_path = os.path.join(synapse_root, "rust", "src") + if not os.path.exists(rust_path): + return None + + hash = _hash_rust_files_in_directory(rust_path) if hash != get_rust_file_digest(): raise Exception("Rust module outdated. Please rebuild using `poetry install`") @@ -82,10 +80,55 @@ def _hash_rust_files_in_directory(directory: str) -> str: return hasher.hexdigest() -def _dist_is_editable() -> bool: - """Is distribution an editable install?""" - for path_item in sys.path: - egg_link = os.path.join(path_item, "matrix-synapse.egg-link") - if os.path.isfile(egg_link): - return True - return False +def get_synapse_source_directory() -> Optional[str]: + """Try and find the source directory of synapse for editable installs (like + those used in development). + + Returns None if not an editable install (or otherwise can't find the source + directory). + """ + + # Try and find the installed matrix-synapse package. + try: + package = Distribution.from_name("matrix-synapse") + except PackageNotFoundError: + # The package is not found, so it's not installed and so must be being + # pulled out from a local directory (usually the current one). + synapse_dir = os.path.dirname(synapse.__file__) + synapse_root = os.path.abspath(os.path.join(synapse_dir, "..")) + + # Double check we've not gone into site-packages... + if os.path.basename(synapse_root) == "site-packages": + return None + + # ... and it looks like the root of a python project. + if not os.path.exists("pyproject.toml"): + return None + + return synapse_root + + # Read the `direct_url.json` metadata for the package. This won't exist for + # packages installed via a repository/etc. + # c.f. https://packaging.python.org/en/latest/specifications/direct-url/ + direct_url_json = package.read_text("direct_url.json") + if direct_url_json is None: + return None + + # c.f. https://packaging.python.org/en/latest/specifications/direct-url/ for + # the format + direct_url_dict: dict = json.loads(direct_url_json) + + # `url` must exist as a key, and point to where we fetched the repo from. + project_url = urllib.parse.urlparse(direct_url_dict["url"]) + + # If its not a local file then we must have built the rust libs either a) + # after we downloaded the package, or b) we built the download wheel. + if project_url.scheme != "file": + return None + + # And finally if its not an editable install then the files can't have + # changed since we installed the package. + if not direct_url_dict.get("dir_info", {}).get("editable", False): + return None + + return project_url.path From 58deef5eba68c24987c6b2d95d397ee023a1c6ec Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 29 Oct 2024 11:50:13 -0700 Subject: [PATCH 15/31] Add admin handler to list of handlers used for background tasks (#17847) Fixes #17823 While we're at it, makes a change where the redactions are sent as the admin if the user is not a member of the server (otherwise these fail with a "User must be our own" message). --- changelog.d/17847.bugfix | 2 + docs/admin_api/user_admin_api.md | 3 + synapse/handlers/admin.py | 6 +- synapse/server.py | 1 + tests/rest/admin/test_user.py | 98 +++++++++++++++++++++++++++++++- 5 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 changelog.d/17847.bugfix diff --git a/changelog.d/17847.bugfix b/changelog.d/17847.bugfix new file mode 100644 index 00000000000..0ba39df94df --- /dev/null +++ b/changelog.d/17847.bugfix @@ -0,0 +1,2 @@ +Fix a bug in the admin redact endpoint where the background task would not run if a worker was specified in +the config option `run_background_tasks_on`. \ No newline at end of file diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index cb38e26005a..96a2994b7b4 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -1365,6 +1365,9 @@ _Added in Synapse 1.72.0._ ## Redact all the events of a user +This endpoint allows an admin to redact the events of a given user. There are no restrictions on redactions for a +local user. By default, we puppet the user who sent the message to redact it themselves. Redactions for non-local users are issued using the admin user, and will fail in rooms where the admin user is not admin/does not have the specified power level to issue redactions. + The API is ``` POST /_synapse/admin/v1/user/$user_id/redact diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 851fe57a177..d1989e9d2c6 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -73,6 +73,8 @@ def __init__(self, hs: "HomeServer"): self._redact_all_events, REDACT_ALL_EVENTS_ACTION_NAME ) + self.hs = hs + async def get_redact_task(self, redact_id: str) -> Optional[ScheduledTask]: """Get the current status of an active redaction process @@ -423,8 +425,10 @@ async def _redact_all_events( user_id = task.params.get("user_id") assert user_id is not None + # puppet the user if they're ours, otherwise use admin to redact requester = create_requester( - user_id, authenticated_entity=admin.user.to_string() + user_id if self.hs.is_mine_id(user_id) else admin.user.to_string(), + authenticated_entity=admin.user.to_string(), ) reason = task.params.get("reason") diff --git a/synapse/server.py b/synapse/server.py index 318c6abf3d8..c7b49188139 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -249,6 +249,7 @@ class HomeServer(metaclass=abc.ABCMeta): """ REQUIRED_ON_BACKGROUND_TASK_STARTUP = [ + "admin", "account_validity", "auth", "deactivate_account", diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index 6982c7291a6..f9ae50f40a8 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -23,6 +23,7 @@ import hmac import json import os +import time import urllib.parse from binascii import unhexlify from http import HTTPStatus @@ -56,6 +57,7 @@ from synapse.util import Clock from tests import unittest +from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.test_utils import SMALL_PNG from tests.unittest import override_config @@ -5127,7 +5129,6 @@ def test_redact_messages_all_rooms(self) -> None: """ Test that request to redact events in all rooms user is member of is successful """ - # join rooms, send some messages originals = [] for rm in [self.rm1, self.rm2, self.rm3]: @@ -5404,3 +5405,98 @@ def test_admin_redact_works_if_user_kicked_or_banned(self) -> None: matches.append((event_id, event)) # we redacted 6 messages self.assertEqual(len(matches), 6) + + +class UserRedactionBackgroundTaskTestCase(BaseMultiWorkerStreamTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + admin.register_servlets, + room.register_servlets, + sync.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.admin = self.register_user("thomas", "pass", True) + self.admin_tok = self.login("thomas", "pass") + + self.bad_user = self.register_user("teresa", "pass") + self.bad_user_tok = self.login("teresa", "pass") + + # create rooms - room versions 11+ store the `redacts` key in content while + # earlier ones don't so we use a mix of room versions + self.rm1 = self.helper.create_room_as( + self.admin, tok=self.admin_tok, room_version="7" + ) + self.rm2 = self.helper.create_room_as(self.admin, tok=self.admin_tok) + self.rm3 = self.helper.create_room_as( + self.admin, tok=self.admin_tok, room_version="11" + ) + + @override_config({"run_background_tasks_on": "worker1"}) + def test_redact_messages_all_rooms(self) -> None: + """ + Test that redact task successfully runs when `run_background_tasks_on` is specified + """ + self.make_worker_hs( + "synapse.app.generic_worker", + extra_config={ + "worker_name": "worker1", + "run_background_tasks_on": "worker1", + "redis": {"enabled": True}, + }, + ) + + # join rooms, send some messages + original_event_ids = set() + for rm in [self.rm1, self.rm2, self.rm3]: + join = self.helper.join(rm, self.bad_user, tok=self.bad_user_tok) + original_event_ids.add(join["event_id"]) + for i in range(15): + event = {"body": f"hello{i}", "msgtype": "m.text"} + res = self.helper.send_event( + rm, "m.room.message", event, tok=self.bad_user_tok, expect_code=200 + ) + original_event_ids.add(res["event_id"]) + + # redact all events in all rooms + channel = self.make_request( + "POST", + f"/_synapse/admin/v1/user/{self.bad_user}/redact", + content={"rooms": []}, + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + id = channel.json_body.get("redact_id") + + timeout_s = 10 + start_time = time.time() + redact_result = "" + while redact_result != "complete": + if start_time + timeout_s < time.time(): + self.fail("Timed out waiting for redactions.") + + channel2 = self.make_request( + "GET", + f"/_synapse/admin/v1/user/redact_status/{id}", + access_token=self.admin_tok, + ) + redact_result = channel2.json_body["status"] + if redact_result == "failed": + self.fail("Redaction task failed.") + + redaction_ids = set() + for rm in [self.rm1, self.rm2, self.rm3]: + filter = json.dumps({"types": [EventTypes.Redaction]}) + channel = self.make_request( + "GET", + f"rooms/{rm}/messages?filter={filter}&limit=50", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + + for event in channel.json_body["chunk"]: + if event["type"] == "m.room.redaction": + redaction_ids.add(event["redacts"]) + + self.assertIncludes(redaction_ids, original_event_ids, exact=True) From 83513b75f7b7d63d6ce2e1565568d8051ba658f8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 30 Oct 2024 10:51:04 +0000 Subject: [PATCH 16/31] Speed up sliding sync by computing extensions in parallel (#17884) The main change here is to add a helper function `gather_optional_coroutines`, which works in a similar way as `yieldable_gather_results` but takes a set of coroutines rather than a function --- changelog.d/17884.misc | 1 + synapse/handlers/sliding_sync/extensions.py | 39 +++++-- synapse/logging/context.py | 40 +++++++ synapse/util/async_helpers.py | 110 +++++++++++++++++++- tests/util/test_async_helpers.py | 108 ++++++++++++++++++- 5 files changed, 285 insertions(+), 13 deletions(-) create mode 100644 changelog.d/17884.misc diff --git a/changelog.d/17884.misc b/changelog.d/17884.misc new file mode 100644 index 00000000000..9dfa13f853c --- /dev/null +++ b/changelog.d/17884.misc @@ -0,0 +1 @@ +Minor speed-up of sliding sync by computing extensions results in parallel. diff --git a/synapse/handlers/sliding_sync/extensions.py b/synapse/handlers/sliding_sync/extensions.py index 0c77b525139..077887ec321 100644 --- a/synapse/handlers/sliding_sync/extensions.py +++ b/synapse/handlers/sliding_sync/extensions.py @@ -49,7 +49,10 @@ SlidingSyncConfig, SlidingSyncResult, ) -from synapse.util.async_helpers import concurrently_execute +from synapse.util.async_helpers import ( + concurrently_execute, + gather_optional_coroutines, +) if TYPE_CHECKING: from synapse.server import HomeServer @@ -97,26 +100,26 @@ async def get_extensions_response( if sync_config.extensions is None: return SlidingSyncResult.Extensions() - to_device_response = None + to_device_coro = None if sync_config.extensions.to_device is not None: - to_device_response = await self.get_to_device_extension_response( + to_device_coro = self.get_to_device_extension_response( sync_config=sync_config, to_device_request=sync_config.extensions.to_device, to_token=to_token, ) - e2ee_response = None + e2ee_coro = None if sync_config.extensions.e2ee is not None: - e2ee_response = await self.get_e2ee_extension_response( + e2ee_coro = self.get_e2ee_extension_response( sync_config=sync_config, e2ee_request=sync_config.extensions.e2ee, to_token=to_token, from_token=from_token, ) - account_data_response = None + account_data_coro = None if sync_config.extensions.account_data is not None: - account_data_response = await self.get_account_data_extension_response( + account_data_coro = self.get_account_data_extension_response( sync_config=sync_config, previous_connection_state=previous_connection_state, new_connection_state=new_connection_state, @@ -127,9 +130,9 @@ async def get_extensions_response( from_token=from_token, ) - receipts_response = None + receipts_coro = None if sync_config.extensions.receipts is not None: - receipts_response = await self.get_receipts_extension_response( + receipts_coro = self.get_receipts_extension_response( sync_config=sync_config, previous_connection_state=previous_connection_state, new_connection_state=new_connection_state, @@ -141,9 +144,9 @@ async def get_extensions_response( from_token=from_token, ) - typing_response = None + typing_coro = None if sync_config.extensions.typing is not None: - typing_response = await self.get_typing_extension_response( + typing_coro = self.get_typing_extension_response( sync_config=sync_config, actual_lists=actual_lists, actual_room_ids=actual_room_ids, @@ -153,6 +156,20 @@ async def get_extensions_response( from_token=from_token, ) + ( + to_device_response, + e2ee_response, + account_data_response, + receipts_response, + typing_response, + ) = await gather_optional_coroutines( + to_device_coro, + e2ee_coro, + account_data_coro, + receipts_coro, + typing_coro, + ) + return SlidingSyncResult.Extensions( to_device=to_device_response, e2ee=e2ee_response, diff --git a/synapse/logging/context.py b/synapse/logging/context.py index ae2b3d11c07..8a2dfeba13c 100644 --- a/synapse/logging/context.py +++ b/synapse/logging/context.py @@ -37,6 +37,7 @@ from types import TracebackType from typing import ( TYPE_CHECKING, + Any, Awaitable, Callable, Optional, @@ -850,6 +851,45 @@ def run_in_background( return d +def run_coroutine_in_background( + coroutine: typing.Coroutine[Any, Any, R], +) -> "defer.Deferred[R]": + """Run the coroutine, ensuring that the current context is restored after + return from the function, and that the sentinel context is set once the + deferred returned by the function completes. + + Useful for wrapping coroutines that you don't yield or await on (for + instance because you want to pass it to deferred.gatherResults()). + + This is a special case of `run_in_background` where we can accept a + coroutine directly rather than a function. We can do this because coroutines + do not run until called, and so calling an async function without awaiting + cannot change the log contexts. + """ + + current = current_context() + d = defer.ensureDeferred(coroutine) + + # The function may have reset the context before returning, so + # we need to restore it now. + ctx = set_current_context(current) + + # The original context will be restored when the deferred + # completes, but there is nothing waiting for it, so it will + # get leaked into the reactor or some other function which + # wasn't expecting it. We therefore need to reset the context + # here. + # + # (If this feels asymmetric, consider it this way: we are + # effectively forking a new thread of execution. We are + # probably currently within a ``with LoggingContext()`` block, + # which is supposed to have a single entry and exit point. But + # by spawning off another deferred, we are effectively + # adding a new exit point.) + d.addBoth(_set_context_cb, ctx) + return d + + T = TypeVar("T") diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 8618bb0651c..e1eb8a48632 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -51,7 +51,7 @@ ) import attr -from typing_extensions import Concatenate, Literal, ParamSpec +from typing_extensions import Concatenate, Literal, ParamSpec, Unpack from twisted.internet import defer from twisted.internet.defer import CancelledError @@ -61,6 +61,7 @@ from synapse.logging.context import ( PreserveLoggingContext, make_deferred_yieldable, + run_coroutine_in_background, run_in_background, ) from synapse.util import Clock @@ -344,6 +345,7 @@ async def yieldable_gather_results_delaying_cancellation( T2 = TypeVar("T2") T3 = TypeVar("T3") T4 = TypeVar("T4") +T5 = TypeVar("T5") @overload @@ -402,6 +404,112 @@ def gather_results( # type: ignore[misc] return deferred.addCallback(tuple) +@overload +async def gather_optional_coroutines( + *coroutines: Unpack[Tuple[Optional[Coroutine[Any, Any, T1]]]], +) -> Tuple[Optional[T1]]: ... + + +@overload +async def gather_optional_coroutines( + *coroutines: Unpack[ + Tuple[ + Optional[Coroutine[Any, Any, T1]], + Optional[Coroutine[Any, Any, T2]], + ] + ], +) -> Tuple[Optional[T1], Optional[T2]]: ... + + +@overload +async def gather_optional_coroutines( + *coroutines: Unpack[ + Tuple[ + Optional[Coroutine[Any, Any, T1]], + Optional[Coroutine[Any, Any, T2]], + Optional[Coroutine[Any, Any, T3]], + ] + ], +) -> Tuple[Optional[T1], Optional[T2], Optional[T3]]: ... + + +@overload +async def gather_optional_coroutines( + *coroutines: Unpack[ + Tuple[ + Optional[Coroutine[Any, Any, T1]], + Optional[Coroutine[Any, Any, T2]], + Optional[Coroutine[Any, Any, T3]], + Optional[Coroutine[Any, Any, T4]], + ] + ], +) -> Tuple[Optional[T1], Optional[T2], Optional[T3], Optional[T4]]: ... + + +@overload +async def gather_optional_coroutines( + *coroutines: Unpack[ + Tuple[ + Optional[Coroutine[Any, Any, T1]], + Optional[Coroutine[Any, Any, T2]], + Optional[Coroutine[Any, Any, T3]], + Optional[Coroutine[Any, Any, T4]], + Optional[Coroutine[Any, Any, T5]], + ] + ], +) -> Tuple[Optional[T1], Optional[T2], Optional[T3], Optional[T4], Optional[T5]]: ... + + +async def gather_optional_coroutines( + *coroutines: Unpack[Tuple[Optional[Coroutine[Any, Any, T1]], ...]], +) -> Tuple[Optional[T1], ...]: + """Helper function that allows waiting on multiple coroutines at once. + + The return value is a tuple of the return values of the coroutines in order. + + If a `None` is passed instead of a coroutine, it will be ignored and a None + is returned in the tuple. + + Note: For typechecking we need to have an explicit overload for each + distinct number of coroutines passed in. If you see type problems, it's + likely because you're using many arguments and you need to add a new + overload above. + """ + + try: + results = await make_deferred_yieldable( + defer.gatherResults( + [ + run_coroutine_in_background(coroutine) + for coroutine in coroutines + if coroutine is not None + ], + consumeErrors=True, + ) + ) + + results_iter = iter(results) + return tuple( + next(results_iter) if coroutine is not None else None + for coroutine in coroutines + ) + except defer.FirstError as dfe: + # unwrap the error from defer.gatherResults. + + # The raised exception's traceback only includes func() etc if + # the 'await' happens before the exception is thrown - ie if the failure + # happens *asynchronously* - otherwise Twisted throws away the traceback as it + # could be large. + # + # We could maybe reconstruct a fake traceback from Failure.frames. Or maybe + # we could throw Twisted into the fires of Mordor. + + # suppress exception chaining, because the FirstError doesn't tell us anything + # very interesting. + assert isinstance(dfe.subFailure.value, BaseException) + raise dfe.subFailure.value from None + + @attr.s(slots=True, auto_attribs=True) class _LinearizerEntry: # The number of things executing. diff --git a/tests/util/test_async_helpers.py b/tests/util/test_async_helpers.py index d82822d00dc..350a2b7c8cd 100644 --- a/tests/util/test_async_helpers.py +++ b/tests/util/test_async_helpers.py @@ -18,7 +18,7 @@ # # import traceback -from typing import Generator, List, NoReturn, Optional +from typing import Any, Coroutine, Generator, List, NoReturn, Optional, Tuple, TypeVar from parameterized import parameterized_class @@ -39,6 +39,7 @@ ObservableDeferred, concurrently_execute, delay_cancellation, + gather_optional_coroutines, stop_cancellation, timeout_deferred, ) @@ -46,6 +47,8 @@ from tests.server import get_clock from tests.unittest import TestCase +T = TypeVar("T") + class ObservableDeferredTest(TestCase): def test_succeed(self) -> None: @@ -588,3 +591,106 @@ def test_multiple_sleepers_wake(self) -> None: sleeper.wake("name") self.assertTrue(d1.called) self.assertTrue(d2.called) + + +class GatherCoroutineTests(TestCase): + """Tests for `gather_optional_coroutines`""" + + def make_coroutine(self) -> Tuple[Coroutine[Any, Any, T], "defer.Deferred[T]"]: + """Returns a coroutine and a deferred that it is waiting on to resolve""" + + d: "defer.Deferred[T]" = defer.Deferred() + + async def inner() -> T: + with PreserveLoggingContext(): + return await d + + return inner(), d + + def test_single(self) -> None: + "Test passing in a single coroutine works" + + with LoggingContext("test_ctx") as text_ctx: + deferred: "defer.Deferred[None]" + coroutine, deferred = self.make_coroutine() + + gather_deferred = defer.ensureDeferred( + gather_optional_coroutines(coroutine) + ) + + # We shouldn't have a result yet, and should be in the sentinel + # context. + self.assertNoResult(gather_deferred) + self.assertEqual(current_context(), SENTINEL_CONTEXT) + + # Resolving the deferred will resolve the coroutine + deferred.callback(None) + + # All coroutines have resolved, and so we should have the results + result = self.successResultOf(gather_deferred) + self.assertEqual(result, (None,)) + + # We should be back in the normal context. + self.assertEqual(current_context(), text_ctx) + + def test_multiple_resolve(self) -> None: + "Test passing in multiple coroutine that all resolve works" + + with LoggingContext("test_ctx") as test_ctx: + deferred1: "defer.Deferred[int]" + coroutine1, deferred1 = self.make_coroutine() + deferred2: "defer.Deferred[str]" + coroutine2, deferred2 = self.make_coroutine() + + gather_deferred = defer.ensureDeferred( + gather_optional_coroutines(coroutine1, coroutine2) + ) + + # We shouldn't have a result yet, and should be in the sentinel + # context. + self.assertNoResult(gather_deferred) + self.assertEqual(current_context(), SENTINEL_CONTEXT) + + # Even if we resolve one of the coroutines, we shouldn't have a result + # yet + deferred2.callback("test") + self.assertNoResult(gather_deferred) + self.assertEqual(current_context(), SENTINEL_CONTEXT) + + deferred1.callback(1) + + # All coroutines have resolved, and so we should have the results + result = self.successResultOf(gather_deferred) + self.assertEqual(result, (1, "test")) + + # We should be back in the normal context. + self.assertEqual(current_context(), test_ctx) + + def test_multiple_fail(self) -> None: + "Test passing in multiple coroutine where one fails does the right thing" + + with LoggingContext("test_ctx") as test_ctx: + deferred1: "defer.Deferred[int]" + coroutine1, deferred1 = self.make_coroutine() + deferred2: "defer.Deferred[str]" + coroutine2, deferred2 = self.make_coroutine() + + gather_deferred = defer.ensureDeferred( + gather_optional_coroutines(coroutine1, coroutine2) + ) + + # We shouldn't have a result yet, and should be in the sentinel + # context. + self.assertNoResult(gather_deferred) + self.assertEqual(current_context(), SENTINEL_CONTEXT) + + # Throw an exception in one of the coroutines + exc = Exception("test") + deferred2.errback(exc) + + # Expect the gather deferred to immediately fail + result_exc = self.failureResultOf(gather_deferred) + self.assertEqual(result_exc.value, exc) + + # We should be back in the normal context. + self.assertEqual(current_context(), test_ctx) From 9cd3545bcab2f75e933e3a19e7266dfd97987a31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:54:38 +0000 Subject: [PATCH 17/31] Bump regex from 1.11.0 to 1.11.1 (#17874) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b6775cecac..9d4a85b3d5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -444,9 +444,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", From 3e750ab0d8738c1126f79c0595a2a44d0d435c74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:54:48 +0000 Subject: [PATCH 18/31] Bump serde from 1.0.210 to 1.0.213 (#17875) --- Cargo.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d4a85b3d5f..c45e7e7e9fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -302,9 +302,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.82" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -485,18 +485,18 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.213" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "3ea7893ff5e2466df8d720bb615088341b295f849602c6956047f8f80f0e9bc1" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.213" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "7e85ad2009c50b58e87caa8cd6dac16bdf511bbfb7af6c33df902396aa480fa5" dependencies = [ "proc-macro2", "quote", @@ -551,9 +551,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "2.0.61" +version = "2.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c993ed8ccba56ae856363b1845da7266a7cb78e1d146c8a32d54b45a8b831fc9" +checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" dependencies = [ "proc-macro2", "quote", From bf03361c862f85abdc5fde3594299f7b4e228700 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:54:59 +0000 Subject: [PATCH 19/31] Bump anyhow from 1.0.90 to 1.0.91 (#17876) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c45e7e7e9fc..499cf0c9603 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.90" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95" +checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" [[package]] name = "arc-swap" From 330f170c0e2dad8da3cd397c40bad97f0d856a17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:55:17 +0000 Subject: [PATCH 20/31] Bump bytes from 1.7.2 to 1.8.0 (#17877) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 499cf0c9603..e2f22ba8124 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,9 +67,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytes" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] name = "cfg-if" From 6d65c3944bed99f56e4d636ab30784135ecc708a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:56:12 +0000 Subject: [PATCH 21/31] Bump python-multipart from 0.0.12 to 0.0.16 (#17879) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9ca72814a44..322eccb0c79 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1974,13 +1974,13 @@ six = ">=1.5" [[package]] name = "python-multipart" -version = "0.0.12" +version = "0.0.16" description = "A streaming multipart parser for Python" optional = false python-versions = ">=3.8" files = [ - {file = "python_multipart-0.0.12-py3-none-any.whl", hash = "sha256:43dcf96cf65888a9cd3423544dd0d75ac10f7aa0c3c28a175bbcd00c9ce1aebf"}, - {file = "python_multipart-0.0.12.tar.gz", hash = "sha256:045e1f98d719c1ce085ed7f7e1ef9d8ccc8c02ba02b5566d5f7521410ced58cb"}, + {file = "python_multipart-0.0.16-py3-none-any.whl", hash = "sha256:c2759b7b976ef3937214dfb592446b59dfaa5f04682a076f78b117c94776d87a"}, + {file = "python_multipart-0.0.16.tar.gz", hash = "sha256:8dee37b88dab9b59922ca173c35acb627cc12ec74019f5cd4578369c6df36554"}, ] [[package]] From 418fbba8dec740d23ec929391d1c0f7e7721d46c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:56:20 +0000 Subject: [PATCH 22/31] Bump phonenumbers from 8.13.47 to 8.13.48 (#17880) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 322eccb0c79..67814676252 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1451,13 +1451,13 @@ dev = ["jinja2"] [[package]] name = "phonenumbers" -version = "8.13.47" +version = "8.13.48" description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." optional = false python-versions = "*" files = [ - {file = "phonenumbers-8.13.47-py2.py3-none-any.whl", hash = "sha256:5d3c0142ef7055ca5551884352e3b6b93bfe002a0bc95b8eaba39b0e2184541b"}, - {file = "phonenumbers-8.13.47.tar.gz", hash = "sha256:53c5e7c6d431cafe4efdd44956078404ae9bc8b0eacc47be3105d3ccc88aaffa"}, + {file = "phonenumbers-8.13.48-py2.py3-none-any.whl", hash = "sha256:5c51939acefa390eb74119750afb10a85d3c628dc83fd62c52d6f532fcf5d205"}, + {file = "phonenumbers-8.13.48.tar.gz", hash = "sha256:62d8df9b0f3c3c41571c6b396f044ddd999d61631534001b8be7fdf7ba1b18f3"}, ] [[package]] From 5c781b578d5c2bfdf26138ce4345e1cc68525ec6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:57:36 +0000 Subject: [PATCH 23/31] Bump ruff from 0.6.9 to 0.7.1 (#17868) --- poetry.lock | 40 ++++++++++++++++++++-------------------- pyproject.toml | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/poetry.lock b/poetry.lock index 67814676252..f311e787f98 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2277,29 +2277,29 @@ files = [ [[package]] name = "ruff" -version = "0.6.9" +version = "0.7.1" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.6.9-py3-none-linux_armv6l.whl", hash = "sha256:064df58d84ccc0ac0fcd63bc3090b251d90e2a372558c0f057c3f75ed73e1ccd"}, - {file = "ruff-0.6.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:140d4b5c9f5fc7a7b074908a78ab8d384dd7f6510402267bc76c37195c02a7ec"}, - {file = "ruff-0.6.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:53fd8ca5e82bdee8da7f506d7b03a261f24cd43d090ea9db9a1dc59d9313914c"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:645d7d8761f915e48a00d4ecc3686969761df69fb561dd914a773c1a8266e14e"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eae02b700763e3847595b9d2891488989cac00214da7f845f4bcf2989007d577"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d5ccc9e58112441de8ad4b29dcb7a86dc25c5f770e3c06a9d57e0e5eba48829"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:417b81aa1c9b60b2f8edc463c58363075412866ae4e2b9ab0f690dc1e87ac1b5"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c866b631f5fbce896a74a6e4383407ba7507b815ccc52bcedabb6810fdb3ef7"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7b118afbb3202f5911486ad52da86d1d52305b59e7ef2031cea3425142b97d6f"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a67267654edc23c97335586774790cde402fb6bbdb3c2314f1fc087dee320bfa"}, - {file = "ruff-0.6.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3ef0cc774b00fec123f635ce5c547dac263f6ee9fb9cc83437c5904183b55ceb"}, - {file = "ruff-0.6.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:12edd2af0c60fa61ff31cefb90aef4288ac4d372b4962c2864aeea3a1a2460c0"}, - {file = "ruff-0.6.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:55bb01caeaf3a60b2b2bba07308a02fca6ab56233302406ed5245180a05c5625"}, - {file = "ruff-0.6.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:925d26471fa24b0ce5a6cdfab1bb526fb4159952385f386bdcc643813d472039"}, - {file = "ruff-0.6.9-py3-none-win32.whl", hash = "sha256:eb61ec9bdb2506cffd492e05ac40e5bc6284873aceb605503d8494180d6fc84d"}, - {file = "ruff-0.6.9-py3-none-win_amd64.whl", hash = "sha256:785d31851c1ae91f45b3d8fe23b8ae4b5170089021fbb42402d811135f0b7117"}, - {file = "ruff-0.6.9-py3-none-win_arm64.whl", hash = "sha256:a9641e31476d601f83cd602608739a0840e348bda93fec9f1ee816f8b6798b93"}, - {file = "ruff-0.6.9.tar.gz", hash = "sha256:b076ef717a8e5bc819514ee1d602bbdca5b4420ae13a9cf61a0c0a4f53a2baa2"}, + {file = "ruff-0.7.1-py3-none-linux_armv6l.whl", hash = "sha256:cb1bc5ed9403daa7da05475d615739cc0212e861b7306f314379d958592aaa89"}, + {file = "ruff-0.7.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:27c1c52a8d199a257ff1e5582d078eab7145129aa02721815ca8fa4f9612dc35"}, + {file = "ruff-0.7.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:588a34e1ef2ea55b4ddfec26bbe76bc866e92523d8c6cdec5e8aceefeff02d99"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94fc32f9cdf72dc75c451e5f072758b118ab8100727168a3df58502b43a599ca"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:985818742b833bffa543a84d1cc11b5e6871de1b4e0ac3060a59a2bae3969250"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32f1e8a192e261366c702c5fb2ece9f68d26625f198a25c408861c16dc2dea9c"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:699085bf05819588551b11751eff33e9ca58b1b86a6843e1b082a7de40da1565"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:344cc2b0814047dc8c3a8ff2cd1f3d808bb23c6658db830d25147339d9bf9ea7"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4316bbf69d5a859cc937890c7ac7a6551252b6a01b1d2c97e8fc96e45a7c8b4a"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79d3af9dca4c56043e738a4d6dd1e9444b6d6c10598ac52d146e331eb155a8ad"}, + {file = "ruff-0.7.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c5c121b46abde94a505175524e51891f829414e093cd8326d6e741ecfc0a9112"}, + {file = "ruff-0.7.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8422104078324ea250886954e48f1373a8fe7de59283d747c3a7eca050b4e378"}, + {file = "ruff-0.7.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:56aad830af8a9db644e80098fe4984a948e2b6fc2e73891538f43bbe478461b8"}, + {file = "ruff-0.7.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:658304f02f68d3a83c998ad8bf91f9b4f53e93e5412b8f2388359d55869727fd"}, + {file = "ruff-0.7.1-py3-none-win32.whl", hash = "sha256:b517a2011333eb7ce2d402652ecaa0ac1a30c114fbbd55c6b8ee466a7f600ee9"}, + {file = "ruff-0.7.1-py3-none-win_amd64.whl", hash = "sha256:f38c41fcde1728736b4eb2b18850f6d1e3eedd9678c914dede554a70d5241307"}, + {file = "ruff-0.7.1-py3-none-win_arm64.whl", hash = "sha256:19aa200ec824c0f36d0c9114c8ec0087082021732979a359d6f3c390a6ff2a37"}, + {file = "ruff-0.7.1.tar.gz", hash = "sha256:9d8a41d4aa2dad1575adb98a82870cf5db5f76b2938cf2206c22c940034a36f4"}, ] [[package]] @@ -3122,4 +3122,4 @@ user-search = ["pyicu"] [metadata] lock-version = "2.0" python-versions = "^3.8.0" -content-hash = "c8a22f901970b2f851151e731532757fd3acf7ba02930952636d2e6c5c9c0c90" +content-hash = "aa1f6d97809596c23a6d160c0c5804971dad0ba49e34b137bbfb79df038fe6f0" diff --git a/pyproject.toml b/pyproject.toml index 5af9a4145bf..93e9f38ca90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -320,7 +320,7 @@ all = [ # failing on new releases. Keeping lower bounds loose here means that dependabot # can bump versions without having to update the content-hash in the lockfile. # This helps prevents merge conflicts when running a batch of dependabot updates. -ruff = "0.6.9" +ruff = "0.7.1" # Type checking only works with the pydantic.v1 compat module from pydantic v2 pydantic = "^2" From 3ae80b0de4abeff2bdbbb9d8735057e6b301a670 Mon Sep 17 00:00:00 2001 From: Lama Date: Wed, 30 Oct 2024 19:55:23 +0300 Subject: [PATCH 24/31] Check if user is in room before being able to tag it (#17839) Fix #17819 --- changelog.d/17839.bugfix | 1 + synapse/handlers/room_member.py | 20 +++++++ synapse/rest/client/tags.py | 7 +++ tests/rest/client/test_tags.py | 95 +++++++++++++++++++++++++++++++++ 4 files changed, 123 insertions(+) create mode 100644 changelog.d/17839.bugfix create mode 100644 tests/rest/client/test_tags.py diff --git a/changelog.d/17839.bugfix b/changelog.d/17839.bugfix new file mode 100644 index 00000000000..57667a6df5d --- /dev/null +++ b/changelog.d/17839.bugfix @@ -0,0 +1 @@ +Check if user has membership in a room before tagging it. Contributed by Lama Alosaimi. \ No newline at end of file diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 75c60e3c34d..70cbbc352be 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -1190,6 +1190,26 @@ async def update_membership_locked( origin_server_ts=origin_server_ts, ) + async def check_for_any_membership_in_room( + self, *, user_id: str, room_id: str + ) -> None: + """ + Check if the user has any membership in the room and raise error if not. + + Args: + user_id: The user to check. + room_id: The room to check. + + Raises: + AuthError if the user doesn't have any membership in the room. + """ + result = await self.store.get_local_current_membership_for_user_in_room( + user_id=user_id, room_id=room_id + ) + + if result is None or result == (None, None): + raise AuthError(403, f"User {user_id} has no membership in room {room_id}") + async def _should_perform_remote_join( self, user_id: str, diff --git a/synapse/rest/client/tags.py b/synapse/rest/client/tags.py index 554bcb95dd8..b6648f3499f 100644 --- a/synapse/rest/client/tags.py +++ b/synapse/rest/client/tags.py @@ -78,6 +78,7 @@ def __init__(self, hs: "HomeServer"): super().__init__() self.auth = hs.get_auth() self.handler = hs.get_account_data_handler() + self.room_member_handler = hs.get_room_member_handler() async def on_PUT( self, request: SynapseRequest, user_id: str, room_id: str, tag: str @@ -85,6 +86,12 @@ async def on_PUT( requester = await self.auth.get_user_by_req(request) if user_id != requester.user.to_string(): raise AuthError(403, "Cannot add tags for other users.") + # Check if the user has any membership in the room and raise error if not. + # Although it's not harmful for users to tag random rooms, it's just superfluous + # data we don't need to track or allow. + await self.room_member_handler.check_for_any_membership_in_room( + user_id=user_id, room_id=room_id + ) body = parse_json_object_from_request(request) diff --git a/tests/rest/client/test_tags.py b/tests/rest/client/test_tags.py new file mode 100644 index 00000000000..5d596409e18 --- /dev/null +++ b/tests/rest/client/test_tags.py @@ -0,0 +1,95 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2024 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +"""Tests REST events for /tags paths.""" + +from http import HTTPStatus + +import synapse.rest.admin +from synapse.rest.client import login, room, tags + +from tests import unittest + + +class RoomTaggingTestCase(unittest.HomeserverTestCase): + """Tests /user/$user_id/rooms/$room_id/tags/$tag REST API.""" + + servlets = [ + room.register_servlets, + tags.register_servlets, + login.register_servlets, + synapse.rest.admin.register_servlets_for_client_rest_resource, + ] + + def test_put_tag_checks_room_membership(self) -> None: + """ + Test that a user can add a tag to a room if they have membership to the room. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + tag = "test_tag" + + # Make the request + channel = self.make_request( + "PUT", + f"/user/{user1_id}/rooms/{room_id}/tags/{tag}", + content={"order": 0.5}, + access_token=user1_tok, + ) + # Check that the request was successful + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + + def test_put_tag_fails_if_not_in_room(self) -> None: + """ + Test that a user cannot add a tag to a room if they don't have membership to the + room. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + # Create the room with user2 (user1 has no membership in the room) + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + tag = "test_tag" + + # Make the request + channel = self.make_request( + "PUT", + f"/user/{user1_id}/rooms/{room_id}/tags/{tag}", + content={"order": 0.5}, + access_token=user1_tok, + ) + # Check that the request failed with the correct error + self.assertEqual(channel.code, HTTPStatus.FORBIDDEN, channel.result) + + def test_put_tag_fails_if_room_does_not_exist(self) -> None: + """ + Test that a user cannot add a tag to a room if the room doesn't exist (therefore + no membership in the room.) + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = "!nonexistent:test" + tag = "test_tag" + + # Make the request + channel = self.make_request( + "PUT", + f"/user/{user1_id}/rooms/{room_id}/tags/{tag}", + content={"order": 0.5}, + access_token=user1_tok, + ) + # Check that the request failed with the correct error + self.assertEqual(channel.code, HTTPStatus.FORBIDDEN, channel.result) From 7987d5e638b3c60494f4695f67bc0000a804f68d Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 30 Oct 2024 19:34:11 -0500 Subject: [PATCH 25/31] Remove `Generator` in `_quarantine_media_txn()` (#17813) --- changelog.d/17813.bugfix | 1 + synapse/storage/databases/main/room.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/17813.bugfix diff --git a/changelog.d/17813.bugfix b/changelog.d/17813.bugfix new file mode 100644 index 00000000000..5dd276709b0 --- /dev/null +++ b/changelog.d/17813.bugfix @@ -0,0 +1 @@ +Avoid lost data on some database query retries. diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index e0b7b7e1949..33569a4391e 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -1175,7 +1175,7 @@ def _quarantine_media_txn( SET quarantined_by = ? WHERE media_origin = ? AND media_id = ? """, - ((quarantined_by, origin, media_id) for origin, media_id in remote_mxcs), + [(quarantined_by, origin, media_id) for origin, media_id in remote_mxcs], ) total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 From af59a99933a54abf34296f12531d56d9342e2592 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 30 Oct 2024 20:14:36 -0500 Subject: [PATCH 26/31] Remove `Generator` from 4 places in `PersistEventStore` (#17818) Context: https://github.com/matrix-org/synapse/issues/15439 (https://github.com/element-hq/synapse/issues/15439) Also see discussion in https://github.com/element-hq/synapse/pull/17813 --- changelog.d/17818.bugfix | 1 + synapse/storage/databases/main/events.py | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) create mode 100644 changelog.d/17818.bugfix diff --git a/changelog.d/17818.bugfix b/changelog.d/17818.bugfix new file mode 100644 index 00000000000..5dd276709b0 --- /dev/null +++ b/changelog.d/17818.bugfix @@ -0,0 +1 @@ +Avoid lost data on some database query retries. diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index e5eae9cee99..dd6ac909e9f 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -1686,7 +1686,7 @@ def _update_current_state_txn( """ txn.execute_batch( sql, - ( + [ ( stream_id, self._instance_name, @@ -1699,17 +1699,17 @@ def _update_current_state_txn( state_key, ) for etype, state_key in itertools.chain(to_delete, to_insert) - ), + ], ) # Now we actually update the current_state_events table txn.execute_batch( "DELETE FROM current_state_events" " WHERE room_id = ? AND type = ? AND state_key = ?", - ( + [ (room_id, etype, state_key) for etype, state_key in itertools.chain(to_delete, to_insert) - ), + ], ) # We include the membership in the current state table, hence we do @@ -1799,11 +1799,11 @@ def _update_current_state_txn( txn.execute_batch( "DELETE FROM local_current_membership" " WHERE room_id = ? AND user_id = ?", - ( + [ (room_id, state_key) for etype, state_key in itertools.chain(to_delete, to_insert) if etype == EventTypes.Member and self.is_mine_id(state_key) - ), + ], ) if to_insert: @@ -3208,7 +3208,7 @@ def _set_push_actions_for_event_and_users_txn( if notifiable_events: txn.execute_batch( sql, - ( + [ ( event.room_id, event.internal_metadata.stream_ordering, @@ -3216,18 +3216,18 @@ def _set_push_actions_for_event_and_users_txn( event.event_id, ) for event in notifiable_events - ), + ], ) # Now we delete the staging area for *all* events that were being # persisted. txn.execute_batch( "DELETE FROM event_push_actions_staging WHERE event_id = ?", - ( + [ (event.event_id,) for event, _ in all_events_and_contexts if event.internal_metadata.is_notifiable() - ), + ], ) def _remove_push_actions_for_event_id_txn( From 2e5fe3f187490c8d593313ce70336f61bb010f25 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 30 Oct 2024 20:15:57 -0500 Subject: [PATCH 27/31] Remove `Generator` in `store_search_entries_txn` (#17817) Context: https://github.com/matrix-org/synapse/issues/15439 (https://github.com/element-hq/synapse/issues/15439) Also see discussion in https://github.com/element-hq/synapse/pull/17813 --- changelog.d/17817.bugfix | 1 + synapse/storage/databases/main/search.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/17817.bugfix diff --git a/changelog.d/17817.bugfix b/changelog.d/17817.bugfix new file mode 100644 index 00000000000..5dd276709b0 --- /dev/null +++ b/changelog.d/17817.bugfix @@ -0,0 +1 @@ +Avoid lost data on some database query retries. diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index b436275f3fd..1d5c5e72ffc 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -94,7 +94,7 @@ def store_search_entries_txn( VALUES (?,?,?,to_tsvector('english', ?),?,?) """ - args1 = ( + args1 = [ ( entry.event_id, entry.room_id, @@ -104,7 +104,7 @@ def store_search_entries_txn( entry.origin_server_ts, ) for entry in entries - ) + ] txn.execute_batch(sql, args1) From 0c429fae1ded5b7348abf1d7695fec908018c3c1 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 30 Oct 2024 20:16:24 -0500 Subject: [PATCH 28/31] Remove `Generator` in `update_cached_last_access_time` (#17816) Context: https://github.com/matrix-org/synapse/issues/15439 (https://github.com/element-hq/synapse/issues/15439) Also see discussion in https://github.com/element-hq/synapse/pull/17813 --- changelog.d/17816.bugfix | 1 + synapse/storage/databases/main/media_repository.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelog.d/17816.bugfix diff --git a/changelog.d/17816.bugfix b/changelog.d/17816.bugfix new file mode 100644 index 00000000000..5dd276709b0 --- /dev/null +++ b/changelog.d/17816.bugfix @@ -0,0 +1 @@ +Avoid lost data on some database query retries. diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index 7617fd3ad49..7a96e254321 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -729,10 +729,10 @@ def update_cache_txn(txn: LoggingTransaction) -> None: txn.execute_batch( sql, - ( + [ (time_ms, media_origin, media_id) for media_origin, media_id in remote_media - ), + ], ) sql = ( @@ -740,7 +740,7 @@ def update_cache_txn(txn: LoggingTransaction) -> None: " WHERE media_id = ?" ) - txn.execute_batch(sql, ((time_ms, media_id) for media_id in local_media)) + txn.execute_batch(sql, [(time_ms, media_id) for media_id in local_media]) await self.db_pool.runInteraction( "update_cached_last_access_time", update_cache_txn From 034d472688a83c5a41bda8a27c485111ecf95138 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 30 Oct 2024 20:16:49 -0500 Subject: [PATCH 29/31] Remove `Generator` in `_purge_unreferenced_state_groups` twice (#17815) Context: https://github.com/matrix-org/synapse/issues/15439 (https://github.com/element-hq/synapse/issues/15439) Also see discussion in https://github.com/element-hq/synapse/pull/17813 --- changelog.d/17815.bugfix | 1 + synapse/storage/databases/state/store.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/17815.bugfix diff --git a/changelog.d/17815.bugfix b/changelog.d/17815.bugfix new file mode 100644 index 00000000000..5dd276709b0 --- /dev/null +++ b/changelog.d/17815.bugfix @@ -0,0 +1 @@ +Avoid lost data on some database query retries. diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py index aea71b8fcc5..875dba33496 100644 --- a/synapse/storage/databases/state/store.py +++ b/synapse/storage/databases/state/store.py @@ -804,11 +804,11 @@ def _purge_unreferenced_state_groups( logger.info("[purge] removing redundant state groups") txn.execute_batch( "DELETE FROM state_groups_state WHERE state_group = ?", - ((sg,) for sg in state_groups_to_delete), + [(sg,) for sg in state_groups_to_delete], ) txn.execute_batch( "DELETE FROM state_groups WHERE id = ?", - ((sg,) for sg in state_groups_to_delete), + [(sg,) for sg in state_groups_to_delete], ) @trace From 47fe6df0134441211588c0bf8fc8f1f18c489c38 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 30 Oct 2024 21:21:22 -0500 Subject: [PATCH 30/31] Remove `Generator` in `_prune_old_outbound_device_pokes` (#17814) Context: https://github.com/matrix-org/synapse/issues/15439 (https://github.com/element-hq/synapse/issues/15439) Also see discussion in https://github.com/element-hq/synapse/pull/17813 --- changelog.d/17814.bugfix | 1 + synapse/storage/databases/main/devices.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/17814.bugfix diff --git a/changelog.d/17814.bugfix b/changelog.d/17814.bugfix new file mode 100644 index 00000000000..5dd276709b0 --- /dev/null +++ b/changelog.d/17814.bugfix @@ -0,0 +1 @@ +Avoid lost data on some database query retries. diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index a83df4075a3..80889432531 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -1422,7 +1422,7 @@ def _prune_txn(txn: LoggingTransaction) -> None: DELETE FROM device_lists_outbound_last_success WHERE destination = ? AND user_id = ? """ - txn.execute_batch(sql, ((row[0], row[1]) for row in rows)) + txn.execute_batch(sql, [(row[0], row[1]) for row in rows]) logger.info("Pruned %d device list outbound pokes", count) From c705beebf7e95f25624ec38280192d96bc207626 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 31 Oct 2024 03:55:30 -0600 Subject: [PATCH 31/31] Support & use stable endpoints for MSC4151 (#17374) https://github.com/matrix-org/matrix-spec-proposals/pull/4151 has finished FCP. See https://github.com/element-hq/synapse/issues/17373 for unstable endpoint removal --------- Co-authored-by: Andrew Morgan --- changelog.d/17374.feature | 1 + synapse/rest/client/reporting.py | 35 ++++++++++++++++++----------- tests/rest/client/test_reporting.py | 31 ++----------------------- 3 files changed, 25 insertions(+), 42 deletions(-) create mode 100644 changelog.d/17374.feature diff --git a/changelog.d/17374.feature b/changelog.d/17374.feature new file mode 100644 index 00000000000..3321f189472 --- /dev/null +++ b/changelog.d/17374.feature @@ -0,0 +1 @@ +Support [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151)'s stable report room API. \ No newline at end of file diff --git a/synapse/rest/client/reporting.py b/synapse/rest/client/reporting.py index 97bd5d8c02c..949f0770351 100644 --- a/synapse/rest/client/reporting.py +++ b/synapse/rest/client/reporting.py @@ -20,11 +20,13 @@ # import logging +import re from http import HTTPStatus from typing import TYPE_CHECKING, Tuple from synapse._pydantic_compat import StrictStr from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError +from synapse.api.urls import CLIENT_API_PREFIX from synapse.http.server import HttpServer from synapse.http.servlet import ( RestServlet, @@ -105,18 +107,17 @@ async def on_POST( class ReportRoomRestServlet(RestServlet): """This endpoint lets clients report a room for abuse. - Whilst MSC4151 is not yet merged, this unstable endpoint is enabled on matrix.org - for content moderation purposes, and therefore backwards compatibility should be - carefully considered when changing anything on this endpoint. - - More details on the MSC: https://github.com/matrix-org/matrix-spec-proposals/pull/4151 + Introduced by MSC4151: https://github.com/matrix-org/matrix-spec-proposals/pull/4151 """ - PATTERNS = client_patterns( - "/org.matrix.msc4151/rooms/(?P[^/]*)/report$", - releases=[], - v1=False, - unstable=True, + # Cast the Iterable to a list so that we can `append` below. + PATTERNS = list( + client_patterns( + "/rooms/(?P[^/]*)/report$", + releases=("v3",), + unstable=False, + v1=False, + ) ) def __init__(self, hs: "HomeServer"): @@ -126,6 +127,16 @@ def __init__(self, hs: "HomeServer"): self.clock = hs.get_clock() self.store = hs.get_datastores().main + # TODO: Remove the unstable variant after 2-3 releases + # https://github.com/element-hq/synapse/issues/17373 + if hs.config.experimental.msc4151_enabled: + self.PATTERNS.append( + re.compile( + f"^{CLIENT_API_PREFIX}/unstable/org.matrix.msc4151" + "/rooms/(?P[^/]*)/report$" + ) + ) + class PostBody(RequestBodyModel): reason: StrictStr @@ -153,6 +164,4 @@ async def on_POST( def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: ReportEventRestServlet(hs).register(http_server) - - if hs.config.experimental.msc4151_enabled: - ReportRoomRestServlet(hs).register(http_server) + ReportRoomRestServlet(hs).register(http_server) diff --git a/tests/rest/client/test_reporting.py b/tests/rest/client/test_reporting.py index 009deb9cb05..723553979f7 100644 --- a/tests/rest/client/test_reporting.py +++ b/tests/rest/client/test_reporting.py @@ -156,58 +156,31 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.room_id = self.helper.create_room_as( self.other_user, tok=self.other_user_tok, is_public=True ) - self.report_path = ( - f"/_matrix/client/unstable/org.matrix.msc4151/rooms/{self.room_id}/report" - ) + self.report_path = f"/_matrix/client/v3/rooms/{self.room_id}/report" - @unittest.override_config( - { - "experimental_features": {"msc4151_enabled": True}, - } - ) def test_reason_str(self) -> None: data = {"reason": "this makes me sad"} self._assert_status(200, data) - @unittest.override_config( - { - "experimental_features": {"msc4151_enabled": True}, - } - ) def test_no_reason(self) -> None: data = {"not_reason": "for typechecking"} self._assert_status(400, data) - @unittest.override_config( - { - "experimental_features": {"msc4151_enabled": True}, - } - ) def test_reason_nonstring(self) -> None: data = {"reason": 42} self._assert_status(400, data) - @unittest.override_config( - { - "experimental_features": {"msc4151_enabled": True}, - } - ) def test_reason_null(self) -> None: data = {"reason": None} self._assert_status(400, data) - @unittest.override_config( - { - "experimental_features": {"msc4151_enabled": True}, - } - ) def test_cannot_report_nonexistent_room(self) -> None: """ Tests that we don't accept event reports for rooms which do not exist. """ channel = self.make_request( "POST", - "/_matrix/client/unstable/org.matrix.msc4151/rooms/!bloop:example.org/report", + "/_matrix/client/v3/rooms/!bloop:example.org/report", {"reason": "i am very sad"}, access_token=self.other_user_tok, shorthand=False,