Merge pull request #257 from pagopa/PAGOPA-1239-sviluppo-adeguamento-…

…script-inclusione-attributes

Pagopa 1239 sviluppo adeguamento script inclusione attributes

scripts/utils/pythonExportIban/README.md

@@ -2,7 +2,7 @@
 
 To populate Iban tables it is necessary to run the command:
 
-`./populate_tables.sh <hostIpAddress> <hostPort> <DatabaseName> <DatabaseUsername> <DatabasePassword> <pathToOracleDriver>`
+`./populate_tables.sh <pathToPdaCsvFile>`
 
 The requirements for the correct compilation of the script are explained in `requirements.md` 
 

scripts/utils/pythonExportIban/exporterIbanMasterToCsv.py

@@ -1,33 +1,26 @@
 import csv
-from datetime import datetime
-import jaydebeapi
-import sys
+import os
 
-host = "jdbc:oracle:thin:@" + sys.argv[1] + ":" + sys.argv[2] + "/" + sys.argv[3]
-username = sys.argv[4]
-password = sys.argv[5]
-driver = sys.argv[6]
-connection  = jaydebeapi.connect(
-    "oracle.jdbc.driver.OracleDriver",
-    host,
-    [username, password],
-    driver)
+import oracledb
+
+connection = oracledb.connect(
+    dsn=os.environ['SPRING_DATASOURCE_HOST'],
+    port=os.environ['SPRING_DATASOURCE_PORT'],
+    user=os.environ['SPRING_DATASOURCE_USERNAME'],
+    password=os.environ['SPRING_DATASOURCE_PASSWORD']
+)
 cursor = connection.cursor()
 
 with open('./IbanCsv/IbanView.csv', 'r', newline='') as source, open('./IbanCsv/Iban_Master_output.csv', 'w', newline='') as result:
     csvreader = csv.reader(source, delimiter=',')
     csvwriter = csv.writer(result, delimiter=',')
-
-    csvwriter.writerow(['denominazioneEnte','codiceFiscale','iban','stato','dataAttivazioneIban','dataPubblicazioneIban','descrizione','etichetta'])
-    next(csvreader)
     # Process data rows
     for row in csvreader:
         cursor.execute(f"Select id_dominio from NODO4_CFG.pa where obj_id={row[0]}")
         result_set = cursor.fetchall()
         if(result_set == ''):
-            print(f"bad luck {row}")
-        rowToWrite = [row[4], result_set[0][0], row[1], "ENABLED", row[2], row[3], row[4],]
-        print(rowToWrite)
+            print(f"Problem with {row[1]}")
+        rowToWrite = [row[4], result_set[0][0], row[1], "ENABLED", row[2], row[3]]
         csvwriter.writerow(rowToWrite)
 
 cursor.close()

scripts/utils/pythonExportIban/exporterIbanToCsv.py

@@ -9,14 +9,11 @@
     csv_reader = csv.reader(csv_file, delimiter=',')
     line_count = 0
     for row in csv_reader:
-        print(row[2])
         if line_count == 0:
-            print(f'Column names are {", ".join(row)}')
             line_count += 1
         else:
             if(thisdictDate.get(row[2]) != None):
-                print(row[2])
-                if(datetime.strptime(thisdictDate.get(row[2]), "%Y-%m-%d %H:%M:%S.%f") > datetime.strptime((row[4]), "%Y-%m-%d %H:%M:%S.%f")):
+                if(datetime.strptime(thisdictDate.get(row[2]), "%Y-%m-%d %H:%M:%S") > datetime.strptime((row[4]), "%Y-%m-%d %H:%M:%S")):
                     thisdictDate.update({row[2]: row[4]})
                     thisdictPa.update({row[2]: row[1]})
             else:
@@ -27,14 +24,12 @@
     csvreader = csv.reader(source, delimiter=',')
     csvwriter = csv.writer(result, delimiter=',')
 
-    csvwriter.writerow(['iban', 'fiscal_code', 'description', 'publication_date'])
-    next(csvreader)
     # Process data rows
     for row in csvreader:
         new_date = ''
         if(row[4] != ''):
-            d = datetime.strptime(thisdictDate.get(row[2]), "%Y-%m-%d %H:%M:%S.%f")
-            new_date = d.strftime("%Y-%m-%d %H:%M:%S.%f")
+            d = datetime.strptime(thisdictDate.get(row[2]), "%Y-%m-%d %H:%M:%S")
+            new_date = d.strftime("%Y-%m-%d %H:%M:%S")
         if(thisdictPa.get(row[2]) != None):
             rowToWrite = [row[2], thisdictPa.get(row[2]), row[0], new_date]
             thisdictPa.pop(row[2], None)

scripts/utils/pythonExportIban/exporterViewToCsv.py

@@ -1,25 +1,19 @@
-import jaydebeapi
-import csv
 import numpy as np
-import sys
+import os
 
-host = "jdbc:oracle:thin:@" + sys.argv[1] + ":" + sys.argv[2] + "/" + sys.argv[3]
-username = sys.argv[4]
-password = sys.argv[5]
-driver = sys.argv[6]
-connection  = jaydebeapi.connect(
-    "oracle.jdbc.driver.OracleDriver",
-    host,
-    [username, password],
-    driver)
+import oracledb
+
+connection = oracledb.connect(
+    dsn=os.environ['SPRING_DATASOURCE_HOST'],
+    port=os.environ['SPRING_DATASOURCE_PORT'],
+    user=os.environ['SPRING_DATASOURCE_USERNAME'],
+    password=os.environ['SPRING_DATASOURCE_PASSWORD']
+)
 cursor = connection.cursor()
 
 cursor.execute("Select * from NODO4_CFG.IBAN_VALIDI_PER_PA")
 result_set = cursor.fetchall()
 a = np.asarray(result_set)
-row_to_be_added = np.array(['FK_PA', 'IBAN_ACCREDITO', 'DATA_INIZIO_VALIDITA', 'DATA_PUBBLICAZIONE', 'RAGIONE_SOCIALE', 'ID_MERCHANT', 'ID_BANCA_SELLER', 'CHIAVE_AVVIO', 'CHIAVE_ESITO', 'OBJ_ID', 'MASTER_OBJ'])
-a = np.r_[[row_to_be_added], a]
-print(a)
 np.savetxt('./IbanCsv/IbanView.csv', a, delimiter=',', fmt='%s')
 
 cursor.close()

scripts/utils/pythonExportIban/exportingIbanAttributesMasterToDB.py

@@ -0,0 +1,33 @@
+import os
+import csv
+import oracledb
+import sys
+
+connection = oracledb.connect(
+    dsn=os.environ['SPRING_DATASOURCE_HOST'],
+    port=os.environ['SPRING_DATASOURCE_PORT'],
+    user=os.environ['SPRING_DATASOURCE_USERNAME'],
+    password=os.environ['SPRING_DATASOURCE_PASSWORD']
+)
+cursor = connection.cursor()
+
+cursor.execute(f"Select obj_id from NODO4_CFG.iban_attributes where ATTRIBUTE_NAME = '0201138TS'")
+iban_attribute = cursor.fetchall()
+
+data = []
+with open(sys.argv[1]) as csv_file:
+    csv_reader = csv.reader(csv_file, delimiter=';')
+    for row in csv_reader:
+        cursor.execute("""
+        Select obj_id from NODO4_CFG.iban_master where fk_pa=(Select obj_id from NODO4_CFG.pa where id_Dominio=:id_dominio) and fk_iban=(Select obj_id from NODO4_CFG.iban where iban=:iban)
+        """, id_dominio=row[1], iban=row[3])
+        iban_master_element = cursor.fetchall()
+        if(len(iban_master_element) == 1 and row[8] == '0201138TS'):
+            data.append([iban_master_element[0][0], iban_attribute[0][0]])
+
+cursor.executemany("INSERT INTO NODO4_CFG.iban_attributes_master (FK_IBAN_MASTER, FK_IBAN_ATTRIBUTE) VALUES (:1, :2)", data)
+connection.commit()
+
+cursor.close()
+connection.close()
+

scripts/utils/pythonExportIban/exportingIbanAttributesToDB.py

@@ -0,0 +1,19 @@
+import os
+import csv
+import oracledb
+import sys
+
+connection = oracledb.connect(
+    dsn=os.environ['SPRING_DATASOURCE_HOST'],
+    port=os.environ['SPRING_DATASOURCE_PORT'],
+    user=os.environ['SPRING_DATASOURCE_USERNAME'],
+    password=os.environ['SPRING_DATASOURCE_PASSWORD']
+)
+cursor = connection.cursor()
+
+cursor.execute(f"INSERT INTO NODO4_CFG.iban_attributes (ATTRIBUTE_NAME, ATTRIBUTE_DESCRIPTION) VALUES ('0201138TS', 'Canone Unico Patrimoniale - CORPORATE (0201138TS)')")
+connection.commit()
+
+cursor.close()
+connection.close()
+

scripts/utils/pythonExportIban/exportingIbanMastertoDB.py

@@ -1,23 +1,21 @@
-import jaydebeapi
 import csv
-import sys
+import os
 
-host = "jdbc:oracle:thin:@" + sys.argv[1] + ":" + sys.argv[2] + "/" + sys.argv[3]
-username = sys.argv[4]
-password = sys.argv[5]
-driver = sys.argv[6]
-connection  = jaydebeapi.connect(
-    "oracle.jdbc.driver.OracleDriver",
-    host,
-    [username, password],
-    driver)
+import oracledb
+
+connection = oracledb.connect(
+    dsn=os.environ['SPRING_DATASOURCE_HOST'],
+    port=os.environ['SPRING_DATASOURCE_PORT'],
+    user=os.environ['SPRING_DATASOURCE_USERNAME'],
+    password=os.environ['SPRING_DATASOURCE_PASSWORD']
+)
 cursor = connection.cursor()
+data = []
 with open('./IbanCsv/Iban_Master_output.csv') as csv_file:
-    csv_reader = csv.reader(csv_file, delimiter=',')
-    next(csv_reader)
-    for row in csv_reader:
-        print(row)
-        cursor.execute(f"INSERT INTO NODO4_CFG.iban_master (inserted_date, validity_date, state, fk_pa, fk_iban, description) VALUES (to_timestamp('{row[4]}','YYYY-MM-DD HH24:MI:SS.FF6'), to_timestamp('{row[5]}','YYYY-MM-DD HH24:MI:SS.FF6'), '{row[3]}', (Select obj_id from NODO4_CFG.pa where id_dominio = '{row[1]}'), (Select obj_id from NODO4_CFG.iban where iban = '{row[2]}'), '{row[6]}')")
-
+    data = list(csv.reader(csv_file))
+cursor.executemany("""
+        INSERT INTO NODO4_CFG.iban_master (description, fk_pa, fk_iban, state, validity_date, inserted_date)
+        VALUES (:1, (Select obj_id from NODO4_CFG.pa where id_dominio = :2), (Select obj_id from NODO4_CFG.iban where iban = :3), :4,to_timestamp(:5,'YYYY-MM-DD HH24:MI:SS'), to_timestamp(:6,'YYYY-MM-DD HH24:MI:SS'))""", data)
+connection.commit()
 cursor.close()
 connection.close()

scripts/utils/pythonExportIban/exportingIbanToDB.py

@@ -1,28 +1,21 @@
-import jaydebeapi
 import csv
-import sys
+import os
 
-host = "jdbc:oracle:thin:@" + sys.argv[1] + ":" + sys.argv[2] + "/" + sys.argv[3]
-username = sys.argv[4]
-password = sys.argv[5]
-driver = sys.argv[6]
-connection  = jaydebeapi.connect(
-    "oracle.jdbc.driver.OracleDriver",
-    host,
-    [username, password],
-    driver)
-cursor = connection.cursor()
+import oracledb
 
+connection = oracledb.connect(
+    dsn=os.environ['SPRING_DATASOURCE_HOST'],
+    port=os.environ['SPRING_DATASOURCE_PORT'],
+    user=os.environ['SPRING_DATASOURCE_USERNAME'],
+    password=os.environ['SPRING_DATASOURCE_PASSWORD']
+)
+cursor = connection.cursor()
+data = []
 with open('./IbanCsv/Iban_output.csv') as csv_file:
-    csv_reader = csv.reader(csv_file, delimiter=',')
-    next(csv_reader)
-    for row in csv_reader:
-        print(row)
-        cursor.execute("Select count(*) from NODO4_CFG.iban")
-        print(cursor.fetchall())
-        cursor.execute(f"INSERT INTO NODO4_CFG.IBAN (IBAN, FISCAL_CODE, DESCRIPTION, DUE_DATE) VALUES ('{row[0]}', '{row[1]}', '{row[2]}', to_timestamp('{row[3]}','YYYY-MM-DD HH24:MI:SS.FF6'))")
-cursor.execute("Select count(*) from NODO4_CFG.iban")
-result_set = cursor.fetchall()
-
+    data = list(csv.reader(csv_file))
+cursor.executemany("""
+        INSERT INTO NODO4_CFG.IBAN (IBAN, FISCAL_CODE, DESCRIPTION, DUE_DATE)
+        VALUES (:1, :2, :3, to_timestamp(:4,'YYYY-MM-DD HH24:MI:SS.FF6'))""", data)
+connection.commit()
 cursor.close()
 connection.close()

scripts/utils/pythonExportIban/populate_tables.sh

@@ -1,39 +1,38 @@
 #!/bin/bash
 
-DB_HOST_IPADDRESS=$1
-DB_HOST_PORT=$2
-DB_HOST_NAME=$3
-DB_USERNAME=$4
-DB_PASSWORD=$5
-DB_DRIVER_PATH=$6
+PDA_CSV_FILE=$1
 
 echo "Create file where View will be saved"
 
 mkdir ./IbanCsv
 
 touch ./IbanCsv/IbanView.csv
 
-python3 exporterViewToCsv.py $DB_HOST_IPADDRESS $DB_HOST_PORT $DB_HOST_NAME $DB_USERNAME $DB_PASSWORD $DB_DRIVER_PATH
+python3 exporterViewToCsv.py
 
 echo "Original data from existing view exported"
 
-sed -i '' "s/\'//g" ./IbanCsv/IbanView.csv
+sed -i "s/[']//g" ./IbanCsv/IbanView.csv
 
 echo "All ' eliminated"
 
-python3 exporterIbanMasterToCsv.py DB_HOST_IPADDRESS DB_HOST_PORT DB_HOST_NAME DB_USERNAME DB_PASSWORD DB_DRIVER_PATH
+python3 exporterIbanMasterToCsv.py
 
 python3 exporterIbanToCsv.py
 
 echo "Pre process done"
 
-python3 exportingIbanToDB.py DB_HOST_IPADDRESS DB_HOST_PORT DB_HOST_NAME DB_USERNAME DB_PASSWORD DB_DRIVER_PATH
+python3 exportingIbanToDB.py
 
 echo "Populated Iban table"
 
-python3 exportingIbanMastertoDB.py DB_HOST_IPADDRESS DB_HOST_PORT DB_HOST_NAME DB_USERNAME DB_PASSWORD DB_DRIVER_PATH
+python3 exportingIbanMastertoDB.py
 
-echo "Populated Iban Master table, goodbye"
+echo "Populated Iban Master table"
+
+python3 exportingIbanAttributesToDB.py $PDA_CSV_FILE
+
+echo "Populated Iban Attributes and Iban Attributes Master table"
 
 
 

scripts/utils/pythonExportIban/requirements.md

@@ -2,7 +2,5 @@
 
 To correctly compile the scripts, it is necesssary to install the following dependecies:
 
-- JVM installed
 - pip3
-- jaydebeapi
 - numpy

scripts/utils/pythonExportIban/rollbackFiles/rollback.py

@@ -0,0 +1,30 @@
+import os
+
+import oracledb
+
+connection = oracledb.connect(
+    dsn=os.environ['SPRING_DATASOURCE_HOST'],
+    port=os.environ['SPRING_DATASOURCE_PORT'],
+    user=os.environ['SPRING_DATASOURCE_USERNAME'],
+    password=os.environ['SPRING_DATASOURCE_PASSWORD']
+)
+cursor = connection.cursor()
+print("Removing constraints")
+cursor.execute("ALTER TABLE IBAN_ATTRIBUTES_MASTER DROP CONSTRAINT FK_IBAN_ATTRIBUTES_MASTER_TO_IBAN_ATTRIBUTES")
+cursor.execute("ALTER TABLE IBAN_ATTRIBUTES_MASTER DROP CONSTRAINT FK_IBAN_ATTRIBUTES_MASTER_TO_IBAN_MASTER")
+cursor.execute("ALTER TABLE IBAN_MASTER DROP CONSTRAINT FK_IBAN_MASTER_TO_IBAN")
+print("Rollback IBAN_ATTRIBUTES_MASTER")
+cursor.execute("TRUNCATE TABLE NODO4_CFG.IBAN_ATTRIBUTES_MASTER")
+print("Rollback IBAN_ATTRIBUTES")
+cursor.execute("TRUNCATE TABLE NODO4_CFG.IBAN_ATTRIBUTES")
+print("Rollback IBAN_MASTER")
+cursor.execute("TRUNCATE TABLE NODO4_CFG.IBAN_MASTER")
+print("Rollback IBAN")
+cursor.execute("TRUNCATE TABLE NODO4_CFG.IBAN")
+print("Reinstating constraints")
+cursor.execute("ALTER TABLE \"NODO4_CFG\".\"IBAN_ATTRIBUTES_MASTER\" ADD CONSTRAINT \"FK_IBAN_ATTRIBUTES_MASTER_TO_IBAN_ATTRIBUTES\" FOREIGN KEY (\"FK_IBAN_ATTRIBUTE\") REFERENCES \"NODO4_CFG\".\"IBAN_ATTRIBUTES\" (\"OBJ_ID\") ENABLE")
+cursor.execute("ALTER TABLE \"NODO4_CFG\".\"IBAN_ATTRIBUTES_MASTER\" ADD CONSTRAINT \"FK_IBAN_ATTRIBUTES_MASTER_TO_IBAN_MASTER\" FOREIGN KEY (\"FK_IBAN_MASTER\") REFERENCES \"NODO4_CFG\".\"IBAN_MASTER\" (\"OBJ_ID\") ENABLE")
+cursor.execute("ALTER TABLE \"NODO4_CFG\".\"IBAN_MASTER\" ADD CONSTRAINT \"FK_IBAN_MASTER_TO_IBAN\" FOREIGN KEY (\"FK_IBAN\") REFERENCES \"NODO4_CFG\".\"IBAN\" (\"OBJ_ID\") ENABLE")
+connection.commit()
+cursor.close()
+connection.close()

scripts/utils/pythonExportIban/rollbackFiles/rollback_iban.sh

@@ -1,15 +1,10 @@
 #!/bin/bash
 
-DB_HOST_IPADDRESS=$1
-DB_HOST_PORT=$2
-DB_HOST_NAME=$3
-DB_USERNAME=$4
-DB_PASSWORD=$5
-DB_DRIVER_PATH=$6
+echo "Rollback all"
 
-python3 deleteAll.py DB_HOST_IPADDRESS DB_HOST_PORT DB_HOST_NAME DB_USERNAME DB_PASSWORD DB_DRIVER_PATH
+python3 rollback.py
 
-echo "Deleted all record in tables IBAN and IBAN_MASTER"
+echo "Deleted all record in tables IBAN, IBAN_MASTER, IBAN_ATTRIBUTES and IBAN_ATTRIBUTES_MASTER"