From 5c2b600b8bbc2f0b9781451543afab2eb4a5bad7 Mon Sep 17 00:00:00 2001 From: devketanpro Date: Wed, 27 Nov 2024 17:23:34 +0530 Subject: [PATCH 1/2] udpate section cv mapping to item[subject] [BOR-28] --- server/borsen/io/feed_parsers/ritzau.py | 25 +++++++++++++----- server/tests/io/feed_parsers/ritzau_test.py | 15 ++++++++--- server/tests/io/fixtures/vocabularies.json | 28 ++++++++++----------- 3 files changed, 44 insertions(+), 24 deletions(-) diff --git a/server/borsen/io/feed_parsers/ritzau.py b/server/borsen/io/feed_parsers/ritzau.py index ac2ad01..5a33048 100644 --- a/server/borsen/io/feed_parsers/ritzau.py +++ b/server/borsen/io/feed_parsers/ritzau.py @@ -24,30 +24,43 @@ class RitzauFeedParser(BaseRitzauFeedParser): def __init__(self): super().__init__() self.default_mapping.update( - {"anpa_category": {"xpath": "sectionID/text()", "filter": self.section_category_filter}} + { + "subject": { + "xpath": "sectionID/text()", + "filter": self.section_category_filter, + } + } ) def do_mapping(self, item, item_xml, setting_param_name=None, namespaces=None): item = super().do_mapping(item, item_xml, setting_param_name, namespaces) # Add Default Category: - sections_cv_items = self.get_cv_items("sections") - default_section = [section for section in sections_cv_items if section["qcode"] == "generelt"] + sections_cv_items = self.get_cv_items("categories") + default_section = [ + section + for section in sections_cv_items + if str(section["qcode"]) == "generelt" + ] if default_section: - item["anpa_category"].append(default_section[0]) + item.setdefault("anpa_category", []).append(default_section[0]) return item def section_category_filter(self, category): voc_categories = self.get_cv_items("sections") if voc_categories: - categories_cv = {str(i["ritzau_section_id"]): i for i in voc_categories if "ritzau_section_id" in i} + categories_cv = { + str(i["ritzau_section_id"]): i + for i in voc_categories + if "ritzau_section_id" in i + } else: categories_cv = {} populated_categories = [] - match = categories_cv.get(category) + match = categories_cv.get(str(category)) if match: populated_categories.append(match) return populated_categories diff --git a/server/tests/io/feed_parsers/ritzau_test.py b/server/tests/io/feed_parsers/ritzau_test.py index a0ae645..06f84fa 100644 --- a/server/tests/io/feed_parsers/ritzau_test.py +++ b/server/tests/io/feed_parsers/ritzau_test.py @@ -28,7 +28,9 @@ class BaseRitzauTestCase(TestCase): def setUp(self): super().setUp() voc_file = os.path.join( - os.path.abspath(os.path.dirname(os.path.dirname(borsen.__file__))), "tests/io/fixtures", "vocabularies.json" + os.path.abspath(os.path.dirname(os.path.dirname(borsen.__file__))), + "tests/io/fixtures", + "vocabularies.json", ) AppPopulateCommand().run(voc_file) for key in dir(settings): @@ -57,7 +59,7 @@ def test_can_parse(self): def test_content(self): item = self.item self.assertEqual( - item["anpa_category"], + item["subject"], [ { "name": "Sandbox", @@ -66,14 +68,19 @@ def test_content(self): "translations": {"name": {"da": "Sandbox"}}, "ritzau_section_id": "4", "scheme": "sections", - }, + } + ], + ) + self.assertEqual( + item["anpa_category"], + [ { "name": "Generelt", "qcode": "generelt", "subject": None, "translations": {"name": {"da": "Generelt"}}, "ritzau_section_id": "", - "scheme": "sections", + "scheme": "categories", }, ], ) diff --git a/server/tests/io/fixtures/vocabularies.json b/server/tests/io/fixtures/vocabularies.json index aebf9e3..e8bfb7f 100644 --- a/server/tests/io/fixtures/vocabularies.json +++ b/server/tests/io/fixtures/vocabularies.json @@ -11,7 +11,7 @@ "is_active": true, "name": "foo", "qcode": "f1", - "ritzau_section_id": 1, + "ritzau_section_id": "1", "subject": "", "translations": { "name": { @@ -24,7 +24,7 @@ "is_active": true, "name": "bar", "qcode": "f2", - "ritzau_section_id": 2, + "ritzau_section_id": "2", "subject": "", "translations": { "name": { @@ -32,6 +32,18 @@ "es": "article-es" } } + }, + { + "is_active":true, + "name":"Generelt", + "qcode":"generelt", + "subject":null, + "translations":{ + "name":{ + "da":"Generelt" + } + }, + "ritzau_section_id":"" } ], "schema": { @@ -110,18 +122,6 @@ } }, "ritzau_section_id": "" - }, - { - "is_active":true, - "name":"Generelt", - "qcode":"generelt", - "subject":null, - "translations":{ - "name":{ - "da":"Generelt" - } - }, - "ritzau_section_id":"" } ], "schema":{ From 66562f8ac07c59ea00ccf967d92c80272c0dfa03 Mon Sep 17 00:00:00 2001 From: devketanpro Date: Wed, 27 Nov 2024 17:39:52 +0530 Subject: [PATCH 2/2] refactore code --- server/borsen/io/feed_parsers/ritzau.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/server/borsen/io/feed_parsers/ritzau.py b/server/borsen/io/feed_parsers/ritzau.py index 5a33048..7fa1a38 100644 --- a/server/borsen/io/feed_parsers/ritzau.py +++ b/server/borsen/io/feed_parsers/ritzau.py @@ -36,12 +36,8 @@ def do_mapping(self, item, item_xml, setting_param_name=None, namespaces=None): item = super().do_mapping(item, item_xml, setting_param_name, namespaces) # Add Default Category: - sections_cv_items = self.get_cv_items("categories") - default_section = [ - section - for section in sections_cv_items - if str(section["qcode"]) == "generelt" - ] + categories_cv_items = self.get_cv_items("categories") + default_section = [section for section in categories_cv_items if str(section["qcode"]) == "generelt"] if default_section: item.setdefault("anpa_category", []).append(default_section[0]) @@ -49,18 +45,14 @@ def do_mapping(self, item, item_xml, setting_param_name=None, namespaces=None): return item def section_category_filter(self, category): - voc_categories = self.get_cv_items("sections") - if voc_categories: - categories_cv = { - str(i["ritzau_section_id"]): i - for i in voc_categories - if "ritzau_section_id" in i - } + cv_sections_items = self.get_cv_items("sections") + if cv_sections_items: + section_cv = {str(i["ritzau_section_id"]): i for i in cv_sections_items if "ritzau_section_id" in i} else: - categories_cv = {} + section_cv = {} populated_categories = [] - match = categories_cv.get(str(category)) + match = section_cv.get(str(category)) if match: populated_categories.append(match) return populated_categories