Parse decoupled csv files to sql

.gitignore

@@ -1,3 +1,7 @@
+# Python environment
+env_scrapper/**
+csv_to_sql/sql_parser/**
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -114,3 +118,5 @@ node_modules/
 # Intellij
 .idea
 
+
+

csv_to_sql/README.md

@@ -0,0 +1,2 @@
+# Csv parser
+

csv_to_sql/configparser.ini

@@ -0,0 +1,9 @@
+[course]
+csv = courses
+faculties_col = faculties
+
+[faculty]
+csv = faculties
+
+[course_faculty]
+

csv_to_sql/requirements.txt

@@ -0,0 +1,10 @@
+astunparse==1.6.3
+Jinja2==3.0.3
+MarkupSafe==2.0.1
+numpy==1.21.4
+pandas==1.3.4
+pdoc==8.0.1
+Pygments==2.10.0
+python-dateutil==2.8.2
+pytz==2021.3
+six==1.16.0

csv_to_sql/src/__main__.py

@@ -0,0 +1,26 @@
+
+from .faculty import Faculty
+from .course import Course 
+from .course_faculty import Course_Faculty 
+
+import configparser as cp 
+import os 
+
+# Rename the files by the order it should be executed in the database. 
+def rename_file(order: int, filename: str):
+    path = "./data/sql"
+    new_name = f"{path}/{order}_{filename}.sql"
+    old_name = f"{path}/{filename}.sql"
+    os.rename(old_name, new_name)
+
+# Order that the files should be added to the database. 
+order = ["faculty", "course", "course_faculty"]
+
+config = cp.ConfigParser()
+config.read("./configparser.ini")
+faculty = Faculty(config).parse()
+course = Course(config).parse()
+course_faculty = Course_Faculty(config).parse() 
+
+for i, filename in enumerate(order): 
+    rename_file(i+1, filename)

csv_to_sql/src/course.py

@@ -0,0 +1,30 @@
+from .parser import Parser 
+import configparser as cp
+
+class Course(Parser):
+    def __init__(self, config: cp.ConfigParser):
+        self.config = config
+        super().__init__("course", config['course']['csv'])
+
+    def parse(self):   
+        cols_list = next(self.f_reader)   
+
+        # Get faculties index. 
+        faculties_col_name = self.config['course']['faculties_col']
+        faculties_index = cols_list.index(faculties_col_name) 
+
+        # Drop faculties col. 
+        del cols_list[faculties_index]
+        cols = self.get_cols(cols_list)  
+
+        # Generate inserts 
+        for course_id, row in enumerate(self.f_reader): 
+            del row[faculties_index]    # Remove faculties position.
+            values = self.get_values(course_id, row)
+            insert = self.sql_get_insert(cols, values) 
+            self.f_sql.write(insert)
+
+
+
+
+

csv_to_sql/src/course_faculty.py

@@ -0,0 +1,33 @@
+import configparser as cp
+from .parser import Parser 
+import pandas as pd 
+from ast import literal_eval 
+
+class Course_Faculty(Parser):
+    def __init__(self, config: cp.ConfigParser): 
+        self.config = config
+        super().__init__("course_faculty", None) 
+        self.df_faculty = pd.read_csv(self.get_input_filepath(config['faculty']['csv']), index_col=False) 
+        self.df_course = pd.read_csv(self.get_input_filepath(config['course']['csv']), index_col=False)  
+
+    def get_faculty_id(self, faculty_acronym: str): 
+        return self.df_faculty[self.df_faculty['acronym'] == faculty_acronym].index[0]
+
+
+    def parse(self):  
+        cols = self.get_cols(["course_id", "faculty_id"], with_id=False)   
+
+        # Get faculties index. 
+        faculties_col_name = self.config['course']['faculties_col'] 
+
+        # For each course get's the faculties ids that it's associated. 
+        for course_id, faculties in enumerate(self.df_course[faculties_col_name]):     
+            # Acronym to id 
+            faculties_acronyms = literal_eval(faculties)
+            faculties_ids = list(map(self.get_faculty_id, faculties_acronyms))     
+            # For each id create on instance in the table
+            for faculty_id in faculties_ids:   
+                values = self.get_values(None, [course_id, faculty_id], with_id=False)
+                insert = self.sql_get_insert(cols, values)
+                self.f_sql.write(insert)
+

csv_to_sql/src/faculty.py

@@ -0,0 +1,15 @@
+from .parser import Parser 
+import configparser as cp
+
+class Faculty(Parser):
+    def __init__(self, config: cp.ConfigParser): 
+        self.config = config 
+        super().__init__("faculty", config['faculty']['csv'])
+
+    def parse(self):
+        cols_list = next(self.f_reader)
+        cols = self.get_cols(cols_list)
+        for faculty_id, row in enumerate(self.f_reader):   
+            values = self.get_values(faculty_id, row)
+            insert = self.sql_get_insert(cols, values)
+            self.f_sql.write(insert)

csv_to_sql/src/parser.py

@@ -0,0 +1,50 @@
+from abc import abstractclassmethod
+import os 
+import csv 
+
+class Parser: 
+    def __init__(self, table_name: str, csv_name: str = None):
+        self.current_path = os.path.dirname(os.path.abspath(__file__))    
+        self.table_name = table_name
+        self.csv_name = csv_name 
+
+        # Creating sql.
+        self.f_sql = open(self.get_output_filepath(), "w", encoding="utf-8")    
+
+        # Reading csv. 
+        if csv_name is not None: 
+            f = open(self.get_input_filepath() , "r")  
+            self.f_reader = csv.reader(f) 
+
+
+    def add_brackets_vals(self, x: str):
+        return f"'{x}'"  
+
+    def add_brackets_cols(self, x: str):  
+        return f"`{x}`" 
+
+    def sql_get_insert(self, cols: list, values: list): 
+        return f"INSERT INTO {self.table_name} ({cols}) VALUES ({values}); \n"
+
+    def get_input_filepath(self, csv_name=None): 
+        if csv_name is None:
+            return f"{self.current_path}/../data/raw/{self.csv_name}.csv" 
+        return f"{self.current_path}/../data/raw/{csv_name}.csv" 
+
+    def get_output_filepath(self):
+        return f"{self.current_path}/../data/sql/{self.table_name}.sql"
+
+    def get_cols(self, cols_list, with_id=True):  
+        if with_id:
+            return ','.join(list(map(self.add_brackets_cols, ['id'] + cols_list)))     
+        return ','.join(list(map(self.add_brackets_cols, cols_list)))     
+
+    def get_values(self, id_, row, with_id=True): 
+        if with_id:
+            return ','.join([str(id_)] + list(map(self.add_brackets_vals, row)))     
+        return ','.join(list(map(self.add_brackets_vals, row)))     
+
+
+    @abstractclassmethod 
+    def parser(self):
+        pass 

scrape_to_csv/.gitignore

@@ -0,0 +1 @@
+output