Merge branch 'main' into auto-deploy

# Conflicts:
#	.circleci/config.yml

.circleci/config.yml

@@ -34,9 +34,6 @@ jobs:
           command: |
             poetry config virtualenvs.in-project true
             poetry install
-      - run:
-          name: Save commit hash in a file
-          command: echo $CIRCLE_SHA1 > .git-commit
       - save_cache:
           key: << pipeline.parameters.cache-prefix >>-{{ arch }}-{{ checksum "poetry.lock" }}
           paths:
@@ -90,9 +87,16 @@ jobs:
       - run:
           name: Build version number
           command: |
-            export VERSION=$(poetry version -s)$CIRCLE_BUILD_NUM
-            echo "Version number: $VERSION"
+            # Set the version
+            if [[ $CIRCLE_TAG ]]; then
+                export VERSION=$CIRCLE_TAG
+            else
+                export VERSION=$(poetry version -s)$CIRCLE_BUILD_NUM+${CIRCLE_SHA1:0:7}
+            fi
+            # Write it in a file
             echo "export VERSION=\"$VERSION\"" >> .env_vars
+            # Display some debug info
+            echo "Building a wheel release with version $VERSION, build number: $CIRCLE_BUILD_NUM, commit hash: ${CIRCLE_SHA1:0:7}, tag: $CIRCLE_TAG."
       - run:
           name: Build a distributable package
           command: |
@@ -127,7 +131,7 @@ jobs:
             echo "Create a GitLab CI pipeline for deploying Hydra version $VERSION on dev with tag light..."
             curl --request POST \
               --header "PRIVATE-TOKEN: ${GITLAB_PRIVATE_TOKEN}" \
-              --data '{ "ref": "main", "variables": [{"key": "SITE", "value": "dev.data.gouv.fr" }, {"key": "APP", "value": "hydra"}, {"key": "TAGS", "value": "light" }] }' \
+              --data '{ "ref": "main", "variables": [{"key": "SITE", "value": "dev.data.gouv.fr" }, {"key": "APP", "value": "hydra"}, {"key": "TAGS", "value": "light" }, {"key": "VERSION", "value": "$VERSION" }] }' \
               https://gitlab.com/api/v4/projects/${GITLAB_PROJECT_ID}/pipeline
 
   publish:

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 ## Current (in progress)
 
+- Save git commit hash in CI and use it for health check [#182](https://github.com/datagouv/hydra/pull/182) and [#185](https://github.com/datagouv/hydra/pull/185)
+- Add comment column/field to ressources exceptions [#191](https://github.com/datagouv/hydra/pull/191)
+
+## 2.0.2 (2024-10-07)
+
+- Fix typos in README in curl commands examples [#189](https://github.com/datagouv/hydra/pull/189)
+- Bump csv-detective to 0.7.3 [#192](https://github.com/datagouv/hydra/pull/192)
+
+## 2.0.1 (2024-10-04)
+
 - Refactor function to get no_backoff domains and add PostgreSQL indexes to improve DB queries perfs [#171](https://github.com/datagouv/hydra/pull/171)
 - Clean changelog and remove useless section in pyproject.toml [#175](https://github.com/datagouv/hydra/pull/175)
 - Refactor purge_checks CLI to use a date limit instead of a number [#174](https://github.com/datagouv/hydra/pull/174)

README.md

@@ -266,30 +266,42 @@ $ curl -s "http://localhost:8000/api/checks/aggregate?group_by=domain&created_at
 #### Adding a resource exception
 
 ```bash
-$ curl  -X POST http://localhost:8000/api/resources-exceptions
-        -H "Authorization: Bearer <myAPIkey>"
-        -d "{'resource_id': 'f868cca6-8da1-4369-a78d-47463f19a9a3', 'table_indexes': {'SIRET': "index", "immatriculation": "index"}}"
+$ curl   -X POST http://localhost:8000/api/resources-exceptions \
+         -H 'Authorization: Bearer <myAPIkey>' \
+         -d '{
+            "resource_id": "123e4567-e89b-12d3-a456-426614174000",
+            "table_indexes": {
+                  "siren": "index"
+            },
+            "comment": "This is a comment for the resource exception."
+         }'
 ```
 
-...or, if you don't want to add table indexes:
+...or, if you don't want to add table indexes and a comment:
 ```bash
-$ curl  -X POST http://localhost:8000/api/resources-exceptions
-        -H "Authorization: Bearer <myAPIkey>"
-        -d "{'resource_id': 'f868cca6-8da1-4369-a78d-47463f19a9a3'}"
+$ curl  -X POST localhost:8000/api/resources-exceptions \
+        -H 'Authorization: Bearer <myAPIkey>" \
+        -d '{"resource_id": "f868cca6-8da1-4369-a78d-47463f19a9a3"}'
 ```
 
 #### Updating a resource exception
 
 ```bash
-$ curl  -X PUT http://localhost:8000/api/resources-exceptions/f868cca6-8da1-4369-a78d-47463f19a9a3
-        -H "Authorization: Bearer <myAPIkey>"
-         -d "{'table_indexes': {'SIRET': "index", "immatriculation": "index"}}"
+$ curl   -X PUT http://localhost:8000/api/resources-exceptions/f868cca6-8da1-4369-a78d-47463f19a9a3 \
+         -H "Authorization: Bearer <myAPIkey>" \
+         -d '{
+            "table_indexes": {
+                  "siren": "index",
+                  "code_postal": "index"
+            },
+            "comment": "Updated comment for the resource exception."
+         }'
 ```
 
 #### Deleting a resource exception
 
 ```bash
-$ curl  -X DELETE http://localhost:8000/api/resources-exceptions/f868cca6-8da1-4369-a78d-47463f19a9a3
+$ curl  -X DELETE http://localhost:8000/api/resources-exceptions/f868cca6-8da1-4369-a78d-47463f19a9a3 \
         -H "Authorization: Bearer <myAPIkey>"
 ```
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "udata-hydra"
-version = "2.0.1.dev"
+version = "2.0.3.dev"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = ["Opendata Team <[email protected]>"]
 license = "MIT"
@@ -12,7 +12,7 @@ aiocontextvars = "^0.2.2"
 aiohttp = "^3.10.3"
 asyncpg = "^0.29.0"
 coloredlogs = "^15.0.1"
-csv-detective = "0.7.2"
+csv-detective = "0.7.3"
 dateparser = "^1.1.7"
 humanfriendly = "^10.0"
 marshmallow = "^3.14.1"

tests/conftest.py

@@ -153,6 +153,7 @@ async def setup_catalog_with_resource_exception(setup_catalog):
     await ResourceException.insert(
         resource_id=RESOURCE_EXCEPTION_ID,
         table_indexes=RESOURCE_EXCEPTION_TABLE_INDEXES,
+        comment="This is a test comment.",
     )
 
 

tests/test_api/test_api_resources_exceptions.py

@@ -59,12 +59,14 @@ async def test_create_resource_exception(
         json={
             "resource_id": RESOURCE_ID,
             "table_indexes": RESOURCE_EXCEPTION_TABLE_INDEXES,
+            "comment": "This is a test comment.",
         },
     )
     assert resp.status == 201
     data: dict = await resp.json()
     assert data["resource_id"] == RESOURCE_ID
     assert json.loads(data["table_indexes"]) == RESOURCE_EXCEPTION_TABLE_INDEXES
+    assert data["comment"] == "This is a test comment."
 
     # Test posting the same resource exception
     resp = await client.post(
@@ -121,12 +123,16 @@ async def test_update_resource_exception(
     resp = await client.put(
         path=f"/api/resources-exceptions/{RESOURCE_EXCEPTION_ID}",
         headers=api_headers,
-        json={"table_indexes": RESOURCE_EXCEPTION_TABLE_INDEXES},
+        json={
+            "table_indexes": RESOURCE_EXCEPTION_TABLE_INDEXES,
+            "comment": "Updated test comment.",
+        },
     )
     assert resp.status == 200
     data: dict = await resp.json()
     assert data["resource_id"] == RESOURCE_EXCEPTION_ID
     assert json.loads(data["table_indexes"]) == RESOURCE_EXCEPTION_TABLE_INDEXES
+    assert data["comment"] == "Updated test comment."
 
 
 async def test_delete_resource_exception(

udata_hydra/__init__.py

@@ -31,13 +31,6 @@ def configure(self) -> None:
         self.configuration = configuration
         self.check()
 
-        # Read commit hash from .git-commit file
-        try:
-            with open(Path.cwd() / ".git-commit", "r") as f:
-                self.configuration["GIT_COMMIT"] = f.read().strip()
-        except FileNotFoundError:
-            log.warning("No .git-commit file")
-
         # add project metadata to config
         self.configuration["APP_NAME"] = "udata-hydra"
         self.configuration["APP_VERSION"] = importlib.metadata.version("udata-hydra")

udata_hydra/db/resource_exception.py

@@ -32,7 +32,12 @@ async def get_by_resource_id(cls, resource_id: str) -> Record | None:
             return await connection.fetchrow(q, resource_id)
 
     @classmethod
-    async def insert(cls, resource_id: str, table_indexes: dict[str, str] | None = {}) -> Record:
+    async def insert(
+        cls,
+        resource_id: str,
+        table_indexes: dict[str, str] | None = None,
+        comment: str | None = None,
+    ) -> Record:
         """
         Insert a new resource_exception
         table_indexes is a JSON object of column names and index types
@@ -45,30 +50,30 @@ async def insert(cls, resource_id: str, table_indexes: dict[str, str] | None = {
         if not resource:
             raise ValueError("Resource not found")
 
-        if table_indexes:
+        if table_indexes is None:
+            table_indexes = {}
+        else:
             valid, error = ResourceExceptionSchema.are_table_indexes_valid(table_indexes)
             if not valid:
                 raise ValueError(error)
 
-            async with pool.acquire() as connection:
-                q = f"""
-                    INSERT INTO resources_exceptions (resource_id, table_indexes)
-                    VALUES ('{resource_id}', '{json.dumps(table_indexes)}')
-                    RETURNING *;
-                """
-                return await connection.fetchrow(q)
-
-        else:
-            async with pool.acquire() as connection:
-                q = f"""
-                    INSERT INTO resources_exceptions (resource_id)
-                    VALUES ('{resource_id}')
-                    RETURNING *;
-                """
-                return await connection.fetchrow(q)
+        async with pool.acquire() as connection:
+            q = """
+                INSERT INTO resources_exceptions (resource_id, table_indexes, comment)
+                VALUES ($1, $2, $3)
+                RETURNING *;
+            """
+            return await connection.fetchrow(
+                q, resource_id, json.dumps(table_indexes) if table_indexes else None, comment
+            )
 
     @classmethod
-    async def update(cls, resource_id: str, table_indexes: dict[str, str] | None = {}) -> Record:
+    async def update(
+        cls,
+        resource_id: str,
+        table_indexes: dict[str, str] | None = None,
+        comment: str | None = None,
+    ) -> Record:
         """
         Update a resource_exception
         table_indexes is a JSON object of column names and index types
@@ -81,29 +86,23 @@ async def update(cls, resource_id: str, table_indexes: dict[str, str] | None = {
         if not resource:
             raise ValueError("Resource not found")
 
-        if table_indexes:
+        if table_indexes is None:
+            table_indexes = {}
+        else:
             valid, error = ResourceExceptionSchema.are_table_indexes_valid(table_indexes)
             if not valid:
                 raise ValueError(error)
 
-            async with pool.acquire() as connection:
-                q = f"""
-                    UPDATE resources_exceptions
-                    SET table_indexes = '{json.dumps(table_indexes)}'
-                    WHERE resource_id = '{resource_id}'
-                    RETURNING *;
-                """
-                return await connection.fetchrow(q)
-
-        else:
-            async with pool.acquire() as connection:
-                q = f"""
-                    UPDATE resources_exceptions
-                    SET table_indexes = NULL
-                    WHERE resource_id = '{resource_id}'
-                    RETURNING *;
-                """
-                return await connection.fetchrow(q)
+        async with pool.acquire() as connection:
+            q = """
+                UPDATE resources_exceptions
+                SET table_indexes = $2, comment = $3
+                WHERE resource_id = $1
+                RETURNING *;
+            """
+            return await connection.fetchrow(
+                q, resource_id, json.dumps(table_indexes) if table_indexes else None, comment
+            )
 
     @classmethod
     async def delete(cls, resource_id: str) -> None:

udata_hydra/migrations/main/20241004_add_comment_column_to_resources_exceptions.sql

@@ -0,0 +1,4 @@
+-- add comment column to resources_exceptions table
+
+ALTER TABLE resources_exceptions
+ADD COLUMN comment VARCHAR(255);

udata_hydra/routes/resources_exceptions.py

@@ -30,20 +30,22 @@ async def create_resource_exception(request: web.Request) -> web.Response:
     try:
         payload = await request.json()
         resource_id: str = payload["resource_id"]
-        table_indexes: dict[str, str] | None = payload.get("table_indexes", None)
+        table_indexes: dict[str, str] | None = payload.get("table_indexes")
         # format should be like this:
         #    {
         #       "column_name": "index_type",
         #       "column_name": "index_type"
         #       ...
         #    },
+        comment: str | None = payload.get("comment")
     except Exception as err:
         raise web.HTTPBadRequest(text=json.dumps({"error": str(err)}))
 
     try:
         resource_exception: Record = await ResourceException.insert(
             resource_id=resource_id,
             table_indexes=table_indexes,
+            comment=comment,
         )
     except ValueError as err:
         raise web.HTTPBadRequest(text=f"Resource exception could not be created: {str(err)}")
@@ -68,7 +70,8 @@ async def update_resource_exception(request: web.Request) -> web.Response:
 
     try:
         payload = await request.json()
-        table_indexes: dict[str, str] | None = payload.get("table_indexes", None)
+        table_indexes: dict[str, str] | None = payload.get("table_indexes")
+        comment: str | None = payload.get("comment")
         if table_indexes:
             valid, error = ResourceExceptionSchema.are_table_indexes_valid(table_indexes)
             if not valid:
@@ -79,6 +82,7 @@ async def update_resource_exception(request: web.Request) -> web.Response:
     resource_exception: Record = await ResourceException.update(
         resource_id=resource_id,
         table_indexes=table_indexes,
+        comment=comment,
     )
 
     return web.json_response(ResourceExceptionSchema().dump(dict(resource_exception)))

udata_hydra/routes/status.py

@@ -125,7 +125,6 @@ async def get_health(request: web.Request) -> web.Response:
     return web.json_response(
         {
             "version": config.APP_VERSION,
-            "commit": config.GIT_COMMIT or "unknown",
             "environment": config.ENVIRONMENT or "unknown",
         }
     )

udata_hydra/schemas/resource_exception.py

@@ -7,6 +7,7 @@ class ResourceExceptionSchema(Schema):
     id = fields.Str(required=True)
     resource_id = fields.Str(required=True)
     table_indexes = fields.Str(allow_none=True)
+    comment = fields.Str(allow_none=True)
 
     @staticmethod
     def are_table_indexes_valid(table_indexes: dict[str, str]) -> tuple[bool, str | None]: