From c50260b29486693fed081175a4ad1940033d1af5 Mon Sep 17 00:00:00 2001 From: leonidastri Date: Tue, 21 Jan 2025 19:38:32 +0100 Subject: [PATCH 1/6] feat: change csv parse functionality --- groups_import.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/groups_import.py b/groups_import.py index a95cf846a..23178474e 100644 --- a/groups_import.py +++ b/groups_import.py @@ -3,6 +3,8 @@ __copyright__ = 'Copyright (c) 2018-2024, Utrecht University' __license__ = 'GPLv3, see LICENSE' +import csv + from typing import Dict, List, Set, Tuple from iteration_utilities import duplicates, unique_everseen @@ -170,15 +172,15 @@ def parse_data(ctx: 'rule.Context', csv_header_and_data: str) -> Tuple: extracted_data = [] csv_lines = csv_header_and_data.splitlines() - header = csv_lines[0] - import_lines = csv_lines[1:] + csv_reader = csv.reader(csv_lines) + header = next(csv_reader) + import_lines = list(csv_reader) # List of dicts each containing label / list of values pairs. lines = [] - header_cols = header.split(',') + header_cols = header for import_line in import_lines: - data = import_line.split(',') - if len(data) != len(header_cols): + if len(import_line) != len(header_cols): return [], 'Amount of header columns differs from data columns.' # A kind of MultiDict # each key is a header column @@ -195,8 +197,8 @@ def parse_data(ctx: 'rule.Context', csv_header_and_data: str) -> Tuple: if header_cols[x] not in line_dict: line_dict[header_cols[x]] = [] - if len(data[x]): - line_dict[header_cols[x]].append(data[x]) + if len(import_line[x]): + line_dict[header_cols[x]].append(import_line[x]) lines.append(line_dict) From 7bfd382ce986d733fa964449568556a5482ded9b Mon Sep 17 00:00:00 2001 From: leonidastri Date: Wed, 22 Jan 2025 10:48:55 +0100 Subject: [PATCH 2/6] fix: lint --- groups_import.py | 1 - 1 file changed, 1 deletion(-) diff --git a/groups_import.py b/groups_import.py index 23178474e..cdb111bef 100644 --- a/groups_import.py +++ b/groups_import.py @@ -4,7 +4,6 @@ __license__ = 'GPLv3, see LICENSE' import csv - from typing import Dict, List, Set, Tuple from iteration_utilities import duplicates, unique_everseen From 0fec1bb2ffd112b8221a7cce7826a3b0f22fe001 Mon Sep 17 00:00:00 2001 From: leonidastri Date: Wed, 22 Jan 2025 10:51:32 +0100 Subject: [PATCH 3/6] refactor: delete unecessary variable --- groups_import.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/groups_import.py b/groups_import.py index cdb111bef..d50cc215c 100644 --- a/groups_import.py +++ b/groups_import.py @@ -172,12 +172,11 @@ def parse_data(ctx: 'rule.Context', csv_header_and_data: str) -> Tuple: csv_lines = csv_header_and_data.splitlines() csv_reader = csv.reader(csv_lines) - header = next(csv_reader) + header_cols = next(csv_reader) import_lines = list(csv_reader) # List of dicts each containing label / list of values pairs. lines = [] - header_cols = header for import_line in import_lines: if len(import_line) != len(header_cols): return [], 'Amount of header columns differs from data columns.' From 180ef2aa1ff38975ef3054fc12636ea26a8bfbb3 Mon Sep 17 00:00:00 2001 From: leonidastri Date: Thu, 23 Jan 2025 16:43:17 +0100 Subject: [PATCH 4/6] feat: csv parse commas test --- unit-tests/files/with-commas.csv | 5 +++++ unit-tests/files/without-commas.csv | 5 +++++ unit-tests/test_group_import.py | 12 ++++++++++++ 3 files changed, 22 insertions(+) create mode 100644 unit-tests/files/with-commas.csv create mode 100644 unit-tests/files/without-commas.csv diff --git a/unit-tests/files/with-commas.csv b/unit-tests/files/with-commas.csv new file mode 100644 index 000000000..44b4370db --- /dev/null +++ b/unit-tests/files/with-commas.csv @@ -0,0 +1,5 @@ +category,subcategory,groupname,manager,member,viewer +test-automation,csv-test,"csv,test,group1",groupmanager@yoda.test,researcher@yoda.test,viewer@yoda.test +test-automation,csv-test,"csv,test,group2",groupmanager@yoda.test,researcher@yoda.test,viewer@yoda.test +test-automation,csv-test,"csv,test,group3",groupmanager@yoda.test,researcher@yoda.test,viewer@yoda.test +test-automation,csv-test,"csv,test,group4",groupmanager@yoda.test,researcher@yoda.test,viewer@yoda.test diff --git a/unit-tests/files/without-commas.csv b/unit-tests/files/without-commas.csv new file mode 100644 index 000000000..f11e6f7ee --- /dev/null +++ b/unit-tests/files/without-commas.csv @@ -0,0 +1,5 @@ +category,subcategory,groupname,manager,member,viewer +test-automation,csv-test,csv-test-group1,groupmanager@yoda.test,researcher@yoda.test,viewer@yoda.test +test-automation,csv-test,csv-test-group2,groupmanager@yoda.test,researcher@yoda.test,viewer@yoda.test +test-automation,csv-test,csv-test-group3,groupmanager@yoda.test,researcher@yoda.test,viewer@yoda.test +test-automation,csv-test,csv-test-group4,groupmanager@yoda.test,researcher@yoda.test,viewer@yoda.test diff --git a/unit-tests/test_group_import.py b/unit-tests/test_group_import.py index ddfcf2fa7..b1fc0b2ef 100644 --- a/unit-tests/test_group_import.py +++ b/unit-tests/test_group_import.py @@ -221,3 +221,15 @@ def test_parse_csv_file_duplicates(self): no_duplicate_data, no_duplicate_err = self.parse_csv_file("files/without-duplicates2.csv") self.assertNotEqual(no_duplicate_data, []) self.assertEqual(no_duplicate_err, '') + + + def test_parse_csv_file_commas(self): + # CSV file with commas + commas_data, commas_err = self.parse_csv_file("files/with-commas.csv") + self.assertEqual(commas_data, []) + self.assertIn("Data error", commas_err) + + # CSV file without duplicates + no_commas_data, no_commas_err = self.parse_csv_file("files/without-commas.csv") + self.assertNotEqual(no_commas_data, []) + self.assertEqual(no_commas_err, '') From f55dcd44e25b809c86e709bfcba5f48999a5fa05 Mon Sep 17 00:00:00 2001 From: leonidastri Date: Thu, 23 Jan 2025 16:47:38 +0100 Subject: [PATCH 5/6] fix: lint --- unit-tests/test_group_import.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/unit-tests/test_group_import.py b/unit-tests/test_group_import.py index b1fc0b2ef..98bf7f3a1 100644 --- a/unit-tests/test_group_import.py +++ b/unit-tests/test_group_import.py @@ -222,14 +222,13 @@ def test_parse_csv_file_duplicates(self): self.assertNotEqual(no_duplicate_data, []) self.assertEqual(no_duplicate_err, '') - def test_parse_csv_file_commas(self): # CSV file with commas commas_data, commas_err = self.parse_csv_file("files/with-commas.csv") self.assertEqual(commas_data, []) self.assertIn("Data error", commas_err) - # CSV file without duplicates + # CSV file without duplicates no_commas_data, no_commas_err = self.parse_csv_file("files/without-commas.csv") self.assertNotEqual(no_commas_data, []) - self.assertEqual(no_commas_err, '') + self.assertEqual(no_commas_err, '') From 453d9811fdf0b28f0054c8eb00fa24ac7465fea6 Mon Sep 17 00:00:00 2001 From: Lazlo Westerhof Date: Wed, 29 Jan 2025 11:24:20 +0100 Subject: [PATCH 6/6] Update copyright --- groups_import.py | 2 +- unit-tests/test_group_import.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/groups_import.py b/groups_import.py index d50cc215c..ce6a138e2 100644 --- a/groups_import.py +++ b/groups_import.py @@ -1,6 +1,6 @@ """Functions related to importing group data.""" -__copyright__ = 'Copyright (c) 2018-2024, Utrecht University' +__copyright__ = 'Copyright (c) 2018-2025, Utrecht University' __license__ = 'GPLv3, see LICENSE' import csv diff --git a/unit-tests/test_group_import.py b/unit-tests/test_group_import.py index 98bf7f3a1..d2880f88c 100644 --- a/unit-tests/test_group_import.py +++ b/unit-tests/test_group_import.py @@ -1,7 +1,6 @@ -"""Unit tests for the groups functionality -""" +"""Unit tests for the groups functionality.""" -__copyright__ = 'Copyright (c) 2019-2024, Utrecht University' +__copyright__ = 'Copyright (c) 2019-2025, Utrecht University' __license__ = 'GPLv3, see LICENSE' import io