From 0b2979f486eaf4cfeb1f6f1426854d287c0ab635 Mon Sep 17 00:00:00 2001 From: Russ Allbery Date: Thu, 2 Jun 2022 10:37:07 -0700 Subject: [PATCH 1/5] Log better exceptions on GCS failures Use logger.exception instead of logger.error when failing to retrieve files from GCS. --- src/crawlspace/handlers/external.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crawlspace/handlers/external.py b/src/crawlspace/handlers/external.py index bb25a1f..6db9776 100644 --- a/src/crawlspace/handlers/external.py +++ b/src/crawlspace/handlers/external.py @@ -49,7 +49,7 @@ def get_file( except GCSFileNotFoundError: raise HTTPException(status_code=404, detail="File not found") except Exception as e: - logger.error(f"Failed to retrieve {path}", error=str(e)) + logger.exception(f"Failed to retrieve {path}", error=str(e)) raise HTTPException( status_code=500, detail="Failed to retrieve file from GCS" ) @@ -91,7 +91,7 @@ def head_file( except GCSFileNotFoundError: raise HTTPException(status_code=404, detail="File not found") except Exception as e: - logger.error(f"Failed to retrieve {path}", error=str(e)) + logger.exception(f"Failed to retrieve {path}", error=str(e)) raise HTTPException( status_code=500, detail="Failed to retrieve file from GCS" ) From 1c9c65421f6604040b1d47a2a15bc0f22ac926ed Mon Sep 17 00:00:00 2001 From: Russ Allbery Date: Thu, 2 Jun 2022 10:37:41 -0700 Subject: [PATCH 2/5] Use production Safir logging by default We want JSON in the Kubernetes logs, not colorized, formatted log messages. --- src/crawlspace/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crawlspace/config.py b/src/crawlspace/config.py index f364428..09800a7 100644 --- a/src/crawlspace/config.py +++ b/src/crawlspace/config.py @@ -46,7 +46,7 @@ class Configuration: Set with the ``SAFIR_NAME`` environment variable. """ - profile: str = os.getenv("SAFIR_PROFILE", "development") + profile: str = os.getenv("SAFIR_PROFILE", "production") """Application run profile: "development" or "production". Set with the ``SAFIR_PROFILE`` environment variable. From 8eaaccc7544d82805a3c1b0ad2e734253d462725 Mon Sep 17 00:00:00 2001 From: Russ Allbery Date: Thu, 2 Jun 2022 10:49:24 -0700 Subject: [PATCH 3/5] Reload the storage blob before using it The Google libraries appear not to populate the blob metadata by default, but we need it for the HTTP headers. --- src/crawlspace/services/file.py | 1 + tests/support/gcs.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/crawlspace/services/file.py b/src/crawlspace/services/file.py index 09886cc..b7ce183 100644 --- a/src/crawlspace/services/file.py +++ b/src/crawlspace/services/file.py @@ -94,4 +94,5 @@ def get_file(self, path: str) -> CrawlspaceFile: blob = bucket.blob(path) if not blob.exists(): raise GCSFileNotFoundError(path) + blob.reload() return CrawlspaceFile.from_blob(path, blob) diff --git a/tests/support/gcs.py b/tests/support/gcs.py index 9d9e5ba..99ec1be 100644 --- a/tests/support/gcs.py +++ b/tests/support/gcs.py @@ -37,6 +37,9 @@ def open(self, mode: str) -> BufferedReader: assert mode == "rb" return self._path.open("rb") + def reload(self) -> None: + pass + class MockBucket(Mock): def __init__(self) -> None: From 583181ed8506984237b0da5c0b40a3de922a7867 Mon Sep 17 00:00:00 2001 From: Russ Allbery Date: Thu, 2 Jun 2022 11:05:56 -0700 Subject: [PATCH 4/5] Fix type of Content-Length header Needs to explicitly convert the blob size to a string. --- src/crawlspace/services/file.py | 2 +- tests/support/gcs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crawlspace/services/file.py b/src/crawlspace/services/file.py index b7ce183..a1edfc5 100644 --- a/src/crawlspace/services/file.py +++ b/src/crawlspace/services/file.py @@ -39,7 +39,7 @@ def from_blob(cls, path: str, blob: storage.Blob) -> CrawlspaceFile: """ headers = { "Cache-Control": f"private, max-age={config.cache_max_age}", - "Content-Length": blob.size, + "Content-Length": str(blob.size), "Last-Modified": format_datetime(blob.updated, usegmt=True), "Etag": blob.etag, } diff --git a/tests/support/gcs.py b/tests/support/gcs.py index 99ec1be..b20533c 100644 --- a/tests/support/gcs.py +++ b/tests/support/gcs.py @@ -25,7 +25,7 @@ def __init__(self, name: str) -> None: self._path = Path(__file__).parent.parent / "files" / name self._exists = self._path.exists() if self._exists: - self.size = str(self._path.stat().st_size) + self.size = self._path.stat().st_size mtime = self._path.stat().st_mtime self.updated = datetime.fromtimestamp(mtime, tz=timezone.utc) self.etag = str(self._path.stat().st_ino) From 8ee1f2caa00660c1176246fd9784d3e6444cb8bb Mon Sep 17 00:00:00 2001 From: Russ Allbery Date: Thu, 2 Jun 2022 12:17:43 -0700 Subject: [PATCH 5/5] Update dependencies --- requirements/dev.txt | 90 +++++++++++++++++++++---------------------- requirements/main.txt | 6 +-- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 4136dd4..bcbf7b7 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -29,48 +29,48 @@ cfgv==3.3.1 \ --hash=sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426 \ --hash=sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736 # via pre-commit -coverage[toml]==6.4 \ - --hash=sha256:00c8544510f3c98476bbd58201ac2b150ffbcce46a8c3e4fb89ebf01998f806a \ - --hash=sha256:016d7f5cf1c8c84f533a3c1f8f36126fbe00b2ec0ccca47cc5731c3723d327c6 \ - --hash=sha256:03014a74023abaf5a591eeeaf1ac66a73d54eba178ff4cb1fa0c0a44aae70383 \ - --hash=sha256:033ebec282793bd9eb988d0271c211e58442c31077976c19c442e24d827d356f \ - --hash=sha256:21e6686a95025927775ac501e74f5940cdf6fe052292f3a3f7349b0abae6d00f \ - --hash=sha256:26f8f92699756cb7af2b30720de0c5bb8d028e923a95b6d0c891088025a1ac8f \ - --hash=sha256:2e76bd16f0e31bc2b07e0fb1379551fcd40daf8cdf7e24f31a29e442878a827c \ - --hash=sha256:341e9c2008c481c5c72d0e0dbf64980a4b2238631a7f9780b0fe2e95755fb018 \ - --hash=sha256:3cfd07c5889ddb96a401449109a8b97a165be9d67077df6802f59708bfb07720 \ - --hash=sha256:4002f9e8c1f286e986fe96ec58742b93484195defc01d5cc7809b8f7acb5ece3 \ - --hash=sha256:50ed480b798febce113709846b11f5d5ed1e529c88d8ae92f707806c50297abf \ - --hash=sha256:543e172ce4c0de533fa892034cce260467b213c0ea8e39da2f65f9a477425211 \ - --hash=sha256:5a78cf2c43b13aa6b56003707c5203f28585944c277c1f3f109c7b041b16bd39 \ - --hash=sha256:5cd698341626f3c77784858427bad0cdd54a713115b423d22ac83a28303d1d95 \ - --hash=sha256:60c2147921da7f4d2d04f570e1838db32b95c5509d248f3fe6417e91437eaf41 \ - --hash=sha256:62d382f7d77eeeaff14b30516b17bcbe80f645f5cf02bb755baac376591c653c \ - --hash=sha256:69432946f154c6add0e9ede03cc43b96e2ef2733110a77444823c053b1ff5166 \ - --hash=sha256:727dafd7f67a6e1cad808dc884bd9c5a2f6ef1f8f6d2f22b37b96cb0080d4f49 \ - --hash=sha256:742fb8b43835078dd7496c3c25a1ec8d15351df49fb0037bffb4754291ef30ce \ - --hash=sha256:750e13834b597eeb8ae6e72aa58d1d831b96beec5ad1d04479ae3772373a8088 \ - --hash=sha256:7b546cf2b1974ddc2cb222a109b37c6ed1778b9be7e6b0c0bc0cf0438d9e45a6 \ - --hash=sha256:83bd142cdec5e4a5c4ca1d4ff6fa807d28460f9db919f9f6a31babaaa8b88426 \ - --hash=sha256:8d2e80dd3438e93b19e1223a9850fa65425e77f2607a364b6fd134fcd52dc9df \ - --hash=sha256:9229d074e097f21dfe0643d9d0140ee7433814b3f0fc3706b4abffd1e3038632 \ - --hash=sha256:968ed5407f9460bd5a591cefd1388cc00a8f5099de9e76234655ae48cfdbe2c3 \ - --hash=sha256:9c82f2cd69c71698152e943f4a5a6b83a3ab1db73b88f6e769fabc86074c3b08 \ - --hash=sha256:a00441f5ea4504f5abbc047589d09e0dc33eb447dc45a1a527c8b74bfdd32c65 \ - --hash=sha256:a022394996419142b33a0cf7274cb444c01d2bb123727c4bb0b9acabcb515dea \ - --hash=sha256:af5b9ee0fc146e907aa0f5fb858c3b3da9199d78b7bb2c9973d95550bd40f701 \ - --hash=sha256:b5578efe4038be02d76c344007b13119b2b20acd009a88dde8adec2de4f630b5 \ - --hash=sha256:b84ab65444dcc68d761e95d4d70f3cfd347ceca5a029f2ffec37d4f124f61311 \ - --hash=sha256:c53ad261dfc8695062fc8811ac7c162bd6096a05a19f26097f411bdf5747aee7 \ - --hash=sha256:cc173f1ce9ffb16b299f51c9ce53f66a62f4d975abe5640e976904066f3c835d \ - --hash=sha256:d548edacbf16a8276af13063a2b0669d58bbcfca7c55a255f84aac2870786a61 \ - --hash=sha256:d55fae115ef9f67934e9f1103c9ba826b4c690e4c5bcf94482b8b2398311bf9c \ - --hash=sha256:d8099ea680201c2221f8468c372198ceba9338a5fec0e940111962b03b3f716a \ - --hash=sha256:e35217031e4b534b09f9b9a5841b9344a30a6357627761d4218818b865d45055 \ - --hash=sha256:e4f52c272fdc82e7c65ff3f17a7179bc5f710ebc8ce8a5cadac81215e8326740 \ - --hash=sha256:e637ae0b7b481905358624ef2e81d7fb0b1af55f5ff99f9ba05442a444b11e45 \ - --hash=sha256:eef5292b60b6de753d6e7f2d128d5841c7915fb1e3321c3a1fe6acfe76c38052 \ - --hash=sha256:fb45fe08e1abc64eb836d187b20a59172053999823f7f6ef4f18a819c44ba16f +coverage[toml]==6.4.1 \ + --hash=sha256:01c5615d13f3dd3aa8543afc069e5319cfa0c7d712f6e04b920431e5c564a749 \ + --hash=sha256:106c16dfe494de3193ec55cac9640dd039b66e196e4641fa8ac396181578b982 \ + --hash=sha256:129cd05ba6f0d08a766d942a9ed4b29283aff7b2cccf5b7ce279d50796860bb3 \ + --hash=sha256:145f296d00441ca703a659e8f3eb48ae39fb083baba2d7ce4482fb2723e050d9 \ + --hash=sha256:1480ff858b4113db2718848d7b2d1b75bc79895a9c22e76a221b9d8d62496428 \ + --hash=sha256:269eaa2c20a13a5bf17558d4dc91a8d078c4fa1872f25303dddcbba3a813085e \ + --hash=sha256:26dff09fb0d82693ba9e6231248641d60ba606150d02ed45110f9ec26404ed1c \ + --hash=sha256:2bd9a6fc18aab8d2e18f89b7ff91c0f34ff4d5e0ba0b33e989b3cd4194c81fd9 \ + --hash=sha256:309ce4a522ed5fca432af4ebe0f32b21d6d7ccbb0f5fcc99290e71feba67c264 \ + --hash=sha256:3384f2a3652cef289e38100f2d037956194a837221edd520a7ee5b42d00cc605 \ + --hash=sha256:342d4aefd1c3e7f620a13f4fe563154d808b69cccef415415aece4c786665397 \ + --hash=sha256:39ee53946bf009788108b4dd2894bf1349b4e0ca18c2016ffa7d26ce46b8f10d \ + --hash=sha256:4321f075095a096e70aff1d002030ee612b65a205a0a0f5b815280d5dc58100c \ + --hash=sha256:4803e7ccf93230accb928f3a68f00ffa80a88213af98ed338a57ad021ef06815 \ + --hash=sha256:4ce1b258493cbf8aec43e9b50d89982346b98e9ffdfaae8ae5793bc112fb0068 \ + --hash=sha256:664a47ce62fe4bef9e2d2c430306e1428ecea207ffd68649e3b942fa8ea83b0b \ + --hash=sha256:75ab269400706fab15981fd4bd5080c56bd5cc07c3bccb86aab5e1d5a88dc8f4 \ + --hash=sha256:83c4e737f60c6936460c5be330d296dd5b48b3963f48634c53b3f7deb0f34ec4 \ + --hash=sha256:84631e81dd053e8a0d4967cedab6db94345f1c36107c71698f746cb2636c63e3 \ + --hash=sha256:84e65ef149028516c6d64461b95a8dbcfce95cfd5b9eb634320596173332ea84 \ + --hash=sha256:865d69ae811a392f4d06bde506d531f6a28a00af36f5c8649684a9e5e4a85c83 \ + --hash=sha256:87f4f3df85aa39da00fd3ec4b5abeb7407e82b68c7c5ad181308b0e2526da5d4 \ + --hash=sha256:8c08da0bd238f2970230c2a0d28ff0e99961598cb2e810245d7fc5afcf1254e8 \ + --hash=sha256:961e2fb0680b4f5ad63234e0bf55dfb90d302740ae9c7ed0120677a94a1590cb \ + --hash=sha256:9b3e07152b4563722be523e8cd0b209e0d1a373022cfbde395ebb6575bf6790d \ + --hash=sha256:a7f3049243783df2e6cc6deafc49ea123522b59f464831476d3d1448e30d72df \ + --hash=sha256:bf5601c33213d3cb19d17a796f8a14a9eaa5e87629a53979a5981e3e3ae166f6 \ + --hash=sha256:cec3a0f75c8f1031825e19cd86ee787e87cf03e4fd2865c79c057092e69e3a3b \ + --hash=sha256:d42c549a8f41dc103a8004b9f0c433e2086add8a719da00e246e17cbe4056f72 \ + --hash=sha256:d67d44996140af8b84284e5e7d398e589574b376fb4de8ccd28d82ad8e3bea13 \ + --hash=sha256:d9c80df769f5ec05ad21ea34be7458d1dc51ff1fb4b2219e77fe24edf462d6df \ + --hash=sha256:e57816f8ffe46b1df8f12e1b348f06d164fd5219beba7d9433ba79608ef011cc \ + --hash=sha256:ee2ddcac99b2d2aec413e36d7a429ae9ebcadf912946b13ffa88e7d4c9b712d6 \ + --hash=sha256:f02cbbf8119db68455b9d763f2f8737bb7db7e43720afa07d8eb1604e5c5ae28 \ + --hash=sha256:f1d5aa2703e1dab4ae6cf416eb0095304f49d004c39e9db1d86f57924f43006b \ + --hash=sha256:f5b66caa62922531059bc5ac04f836860412f7f88d38a476eda0a6f11d4724f4 \ + --hash=sha256:f69718750eaae75efe506406c490d6fc5a6161d047206cc63ce25527e8a3adad \ + --hash=sha256:fb73e0011b8793c053bfa85e53129ba5f0250fdc0392c1591fd35d915ec75c46 \ + --hash=sha256:fd180ed867e289964404051a958f7cccabdeed423f91a899829264bb7974d3d3 \ + --hash=sha256:fdb6f7bd51c2d1714cea40718f6149ad9be6a2ee7d93b19e9f00934c0f2a74d9 \ + --hash=sha256:ffa9297c3a453fba4717d06df579af42ab9a28022444cae7fa605af4df612d54 # via # -r requirements/dev.in # pytest-cov @@ -78,9 +78,9 @@ distlib==0.3.4 \ --hash=sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b \ --hash=sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579 # via virtualenv -filelock==3.7.0 \ - --hash=sha256:b795f1b42a61bbf8ec7113c341dad679d772567b936fbd1bf43c9a238e673e20 \ - --hash=sha256:c7b5fdb219b398a5b28c8e4c1893ef5f98ece6a38c6ab2c22e26ec161556fed6 +filelock==3.7.1 \ + --hash=sha256:37def7b658813cda163b56fc564cdc75e86d338246458c4c28ae84cabefa2404 \ + --hash=sha256:3a0fd85166ad9dbab54c9aec96737b744106dc5f15c0b09a6744a445299fcf04 # via virtualenv h11==0.12.0 \ --hash=sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6 \ diff --git a/requirements/main.txt b/requirements/main.txt index 63f2cb8..68cce6c 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -302,9 +302,9 @@ rsa==4.8 \ --hash=sha256:5c6bd9dc7a543b7fe4304a631f8a8a3b674e2bbfc49c2ae96200cdbe55df6b17 \ --hash=sha256:95c5d300c4e879ee69708c428ba566c59478fd653cc3a22243eeb8ed846950bb # via google-auth -safir==3.0.3 \ - --hash=sha256:2bf4c069b3bb3fa819e69ba2d7bf2e4a406db7341bfa31f3d9057da10b2f6518 \ - --hash=sha256:2de146ff512b3e74e353d8dc44a497d48bc7336dd712a06a07b718f22e26329c +safir==3.1.0 \ + --hash=sha256:d32eafce4180fc245bd6e804b3e537159521dfe6d319994143012c2214a5ab10 \ + --hash=sha256:e46e49f54360763379897b889940e734545e601657cdec9a3a4c005d3dc9587b # via -r requirements/main.in six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \