output.py

# -*- coding: utf-8 -*-

# Form implementation generated from reading ui file 'TestUI.ui'
#
# Created by: PyQt5 UI code generator 5.15.4
#
# WARNING: Any manual changes made to this file will be lost when pyuic5 is
# run again.  Do not edit this file unless you know what you are doing.
import shutil
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtWidgets import QMessageBox
from openpyxl import load_workbook
from plumreader import PlumReader
# import tika
# tika.initVM()
# from tika import parser
import multiprocessing
import time
import PDF_Convert
import os
import pdfplumber
import TataPolcity
import icici_reader
import hdfc_reader
from plumber import *
from bajaj_parser import *
from openpyxl.styles import PatternFill
import WordFile
from pathlib import Path
import traceback

redFill = PatternFill(start_color='EAE51B',
                      end_color='EAE51B',
                      fill_type='solid')


def createCleanupDirectory():
    requiredDirectory = os.environ['LOCALAPPDATA'] + "//PDF_Reader//TEMP_DATA"
    if not os.path.exists(requiredDirectory):
        os.makedirs(requiredDirectory)

    return requiredDirectory


class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        MainWindow.setObjectName("MainWindow")
        MainWindow.resize(572, 443)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Ignored, QtWidgets.QSizePolicy.Ignored)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(MainWindow.sizePolicy().hasHeightForWidth())
        MainWindow.setSizePolicy(sizePolicy)
        MainWindow.setMinimumSize(QtCore.QSize(572, 443))
        MainWindow.setMaximumSize(QtCore.QSize(572, 443))
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap("res/icons8_microsoft_excel.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        MainWindow.setWindowIcon(icon)
        MainWindow.setAnimated(True)
        MainWindow.setUnifiedTitleAndToolBarOnMac(True)
        self.centralwidget = QtWidgets.QWidget(MainWindow)
        self.centralwidget.setObjectName("centralwidget")
        self.policySelector = QtWidgets.QPushButton(self.centralwidget)
        self.policySelector.setGeometry(QtCore.QRect(62, 60, 101, 31))
        self.policySelector.setContextMenuPolicy(QtCore.Qt.PreventContextMenu)
        self.policySelector.setToolTip("")
        self.policySelector.setIconSize(QtCore.QSize(26, 20))
        self.policySelector.setAutoDefault(False)
        self.policySelector.setObjectName("policySelector")
        self.policySelector.clicked.connect(self.press_it)
        self.plainTextEdit = QtWidgets.QPlainTextEdit(self.centralwidget)
        self.plainTextEdit.setGeometry(QtCore.QRect(170, 60, 351, 31))
        self.plainTextEdit.setToolTip("")
        self.plainTextEdit.setReadOnly(True)
        self.plainTextEdit.setObjectName("plainTextEdit")
        self.plainTextEdit_2 = QtWidgets.QPlainTextEdit(self.centralwidget)
        self.plainTextEdit_2.setGeometry(QtCore.QRect(170, 220, 351, 31))
        self.plainTextEdit_2.setToolTip("")
        self.plainTextEdit_2.setReadOnly(True)
        self.plainTextEdit_2.setObjectName("plainTextEdit_2")
        self.plainTextEdit_3 = QtWidgets.QPlainTextEdit(self.centralwidget)
        self.plainTextEdit_3.setGeometry(QtCore.QRect(170, 110, 351, 31))
        self.plainTextEdit_3.setToolTip("")
        self.plainTextEdit_3.setReadOnly(False)
        self.plainTextEdit_3.setObjectName("plainTextEdit_3")
        self.plainTextEdit_4 = QtWidgets.QPlainTextEdit(self.centralwidget)
        self.plainTextEdit_4.setGeometry(QtCore.QRect(170, 160, 351, 31))
        self.plainTextEdit_4.setToolTip("")
        self.plainTextEdit_4.setReadOnly(False)
        self.plainTextEdit_4.setObjectName("plainTextEdit_4")
        self.excelFilePath = QtWidgets.QPushButton(self.centralwidget)
        self.excelFilePath.setGeometry(QtCore.QRect(60, 220, 101, 31))
        self.excelFilePath.setContextMenuPolicy(QtCore.Qt.PreventContextMenu)
        self.excelFilePath.setToolTip("")
        self.excelFilePath.setIconSize(QtCore.QSize(26, 20))
        self.excelFilePath.setAutoDefault(False)
        self.excelFilePath.setObjectName("excelFilePath")
        self.excelFilePath.clicked.connect(self.excel_file_selector)
        self.label = QtWidgets.QLabel(self.centralwidget)
        self.label.setGeometry(QtCore.QRect(64, 116, 91, 20))
        self.label.setAlignment(QtCore.Qt.AlignCenter)
        self.label.setObjectName("label")
        self.label_2 = QtWidgets.QLabel(self.centralwidget)
        self.label_2.setGeometry(QtCore.QRect(60, 160, 91, 20))
        self.label_2.setAlignment(QtCore.Qt.AlignCenter)
        self.label_2.setObjectName("label_2")
        self.progressBar = QtWidgets.QProgressBar(self.centralwidget)
        self.progressBar.setGeometry(QtCore.QRect(20, 280, 251, 23))
        self.progressBar.setToolTip("")
        self.progressBar.setProperty("value", 0)
        self.progressBar.setObjectName("progressBar")
        self.runProcess = QtWidgets.QPushButton(self.centralwidget)
        self.runProcess.setGeometry(QtCore.QRect(430, 280, 93, 28))
        self.runProcess.setToolTip("")
        self.runProcess.setObjectName("runProcess")
        self.runProcess.clicked.connect(self.start_pdf_processing)
        self.plainTextEdit_5 = QtWidgets.QPlainTextEdit(self.centralwidget)
        self.plainTextEdit_5.setGeometry(QtCore.QRect(20, 330, 301, 51))
        self.plainTextEdit_5.setToolTip("")
        self.plainTextEdit_5.setReadOnly(True)
        self.plainTextEdit_5.setObjectName("plainTextEdit_5")
        self.label_3 = QtWidgets.QLabel(self.centralwidget)
        self.label_3.setGeometry(QtCore.QRect(20, 310, 91, 20))
        self.label_3.setAlignment(QtCore.Qt.AlignLeading | QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter)
        self.label_3.setObjectName("label_3")
        self.resetButton = QtWidgets.QPushButton(self.centralwidget)
        self.resetButton.setGeometry(QtCore.QRect(510, 0, 51, 31))
        self.resetButton.setText("")
        icon1 = QtGui.QIcon()
        icon1.addPixmap(QtGui.QPixmap("res/icons8_reset.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.resetButton.setIcon(icon1)
        self.resetButton.setFlat(True)
        self.resetButton.setObjectName("resetButton")
        MainWindow.setCentralWidget(self.centralwidget)
        self.statusbar = QtWidgets.QStatusBar(MainWindow)
        self.statusbar.setObjectName("statusbar")
        MainWindow.setStatusBar(self.statusbar)
        self.menubar = QtWidgets.QMenuBar(MainWindow)
        self.menubar.setGeometry(QtCore.QRect(0, 0, 572, 26))
        self.menubar.setObjectName("menubar")
        MainWindow.setMenuBar(self.menubar)

        self.retranslateUi(MainWindow)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

    def launchDialog(self):
        print(" I m in the callback")
        file_filter = 'Data File (*.xlsx *.csv *.dat);; Excel File (*.xlsx *.xls)'
        response = QtWidgets.QFileDialog.getOpenFileNames()
        print(response)

    def excel_file_selector(self):
        filename, _filter = QtWidgets.QFileDialog.getOpenFileName(caption='Select a target data file',
                                                                  filter='Data File (*.xlsx *.xls)')
        self.plainTextEdit_2.setPlainText(filename)

    @staticmethod
    def rename_file(fl, xl_row):
        pth = Path(fl)
        name_without_extension = pth.stem
        ext = pth.suffix
        if xl_row is not None:
            new_file_name = f"Done_{xl_row}_{name_without_extension}"
        else:
            new_file_name = f"NOT_Done_{name_without_extension}"
        pth.rename(Path(pth.parent, new_file_name + ext))
        return os.path.join(pth.parent, new_file_name + ext)

    def press_it(self):
        filename, _filter = QtWidgets.QFileDialog.getOpenFileNames(caption='Select policy files...',
                                                                   filter='Policy File (*.pdf )')
        # print(dir(filename))
        print(filename)
        # fnames = ";".join(filename)
        # print(fnames)
        if len(filename) > 0:
            fnames = ";".join(filename)
            #     print(fnames)
            self.plainTextEdit.setPlainText(fnames)

    def retranslateUi(self, MainWindow):
        _translate = QtCore.QCoreApplication.translate
        MainWindow.setWindowTitle(_translate("MainWindow", "POLICY DATA EXTRACTOR"))
        self.policySelector.setText(_translate("MainWindow", "Policy Files"))
        self.excelFilePath.setText(_translate("MainWindow", "Excel File"))
        self.label.setText(_translate("MainWindow", "Agent Name"))
        self.label_2.setText(_translate("MainWindow", "Paid By"))
        self.runProcess.setText(_translate("MainWindow", "Start"))
        self.label_3.setText(_translate("MainWindow", "Log"))

    def show_warning_box(self, display_text):
        msgBox = QMessageBox()
        msgBox.setIcon(QMessageBox.Warning)
        msgBox.setText(display_text)
        msgBox.setWindowTitle("Warning")
        msgBox.setStandardButtons(QMessageBox.Ok)
        # msgBox.buttonClicked.connect(self.msgButtonClick)

        returnValue = msgBox.exec()

    def fill_excel_sheet(self, data_dictionary, filep):
        excel_path = self.plainTextEdit_2.toPlainText()
        wb = load_workbook(excel_path)
        ws = wb[wb.sheetnames[0]]
        mx_row = ws.max_row
        mx_row += 1
        for key, value in data_dictionary.items():
            if key == 'Y':
                newname = self.rename_file(filep, mx_row)
                value = '=HYPERLINK("{}", "{}")'.format(newname, data_dictionary['D'])
            if value is not None:
                if isinstance(value, str):
                    value = value.upper()
                ws[key + str(mx_row)].value = value
            else:
                ws[key + str(mx_row)].fill = redFill
        wb.save(excel_path)
        return mx_row

    def process_files(self):
        file_paths = self.plainTextEdit.toPlainText().split(';')
        total_files = len(file_paths)
        cnt = 0
        Bajaj_New_Formats = []
        UnreadFiles = []
        show_pop = False
        for each_file in file_paths:
            cnt += 1
            excel_row = None
            print(each_file)
            try:
                self.plainTextEdit_5.setPlainText('Reading file : {0}'.format(each_file))
                # print(rk.get_policy_type())
                prd = PlumReader(each_file)
                parsed = prd.raw_text
                # print(parsed["content"])
                cs = Classify(parsed)

                policy_type = cs.get_policy()
                # print(cs.get_policy_type())
                print("Policy Type is: {}".format(policy_type))
                data_dict = {
                    'A': None,
                    'B': None,
                    'C': None,
                    'D': None,
                    'E': None,
                    'F': None,
                    'G': None,
                    'H': None,
                    'I': None,
                    'J': None,
                    'K': None,
                    'L': None,
                    'M': None,
                    'N': None,
                    'O': None,
                    'P': None,
                    'Q': None,
                    'R': None,
                    'S': None,
                    'T': None,
                    'U': None,
                    'V': None,
                    'W': None,
                    'X': None,
                    'Y': None
                }
                if policy_type == 'digit':
                    pdf = pdfplumber.open(each_file)
                    rk = Digit_RickShaw(pdf,each_file)
                    pdf.close()
                    data_dict['D'], data_dict['A'] = rk.get_policy_num()
                    data_dict['B'], data_dict['C'] = rk.get_customer_name()
                    data_dict['E'] = rk.get_policy_expiry()
                    data_dict['F'] = rk.get_mobile_num()
                    data_dict['I'], data_dict['G'], data_dict['H'] = rk.get_insurance_type()
                    data_dict['J'] = rk.get_make()
                    data_dict['K'] = rk.get_model()
                    data_dict['L'] = float(rk.get_final_premium())
                    data_dict['N'] = float(rk.get_od_premium())
                    data_dict['M'] = float(rk.get_tp_premium())
                    data_dict['O'] = float(rk.get_net_premium())
                    data_dict['R'] = self.plainTextEdit_3.toPlainText()
                    data_dict['S'] = self.plainTextEdit_4.toPlainText()
                    data_dict['A'] = convert_date_format(data_dict['A'].strip())
                    data_dict['E'] = convert_date_format(data_dict['E'].strip())
                    data_dict['P'] = data_dict[rk.get_payout()]
                    data_dict['Y'] = None
                    data_dict['Q'] = rk.get_code()
                    self.fill_excel_sheet(data_dict,each_file)
                    # print(rk.get_customer_name())
                    # print(rk.get_policy_num())
                    # # print(rk.get_invoice_date())
                    # print(rk.get_policy_expiry())
                    # print(rk.get_mobile_num())
                    # print(rk.get_insurance_type())
                    # print(rk.get_make())
                    # print(rk.get_model())
                    # print(rk.get_final_premium())
                    # print(rk.get_net_premium())
                    # print(rk.get_od_premium())
                    pass

                elif policy_type == 'bajaj':
                    if 'facebook' in parsed:
                        Bajaj_New_Formats.append(each_file)
                    else:

                        pdf = pdfplumber.open(each_file)

                        rk = BajajPolicy(pdf, each_file, self.plainTextEdit_5)
                        pdf.close()
                        data_dict = rk.get_all_policy_details()
                        data_dict['R'] = self.plainTextEdit_3.toPlainText()
                        data_dict['S'] = self.plainTextEdit_4.toPlainText()
                        data_dict['P'] = data_dict['O']
                        data_dict['A'] = convert_date_format(data_dict['A'].strip())
                        data_dict['E'] = convert_date_format(data_dict['E'].strip())
                        data_dict['Y'] = None
                        excel_row = self.fill_excel_sheet(data_dict,each_file)

                elif policy_type == 'tata':
                    cs = TataPolcity.TataPolicy(each_file)
                    data_dict = cs.get_policy_info()
                    data_dict['A'] = data_dict['A'].strip()
                    data_dict['E'] = data_dict['E'].strip()
                    data_dict['R'] = self.plainTextEdit_3.toPlainText()
                    data_dict['S'] = self.plainTextEdit_4.toPlainText()
                    data_dict['Y'] = '=HYPERLINK("{}", "{}")'.format(each_file, data_dict['D'])

                    excel_row = self.fill_excel_sheet(data_dict,each_file)
                elif policy_type =="icici":
                    reader = icici_reader.ICICI(each_file)
                    data_dict = reader.get_policy_info()
                    print(data_dict)
                    data_dict['A'] = data_dict['A'].strip()
                    data_dict['E'] = data_dict['E'].strip()
                    data_dict['R'] = self.plainTextEdit_3.toPlainText()
                    data_dict['S'] = self.plainTextEdit_4.toPlainText()
                    data_dict['Y'] = '=HYPERLINK("{}", "{}")'.format(each_file, data_dict['D'])
                    # print(data_dict)
                    excel_row = self.fill_excel_sheet(data_dict, each_file)
                elif policy_type =="HDFC":
                    reader = hdfc_reader.HDFC(each_file)
                    data_dict = reader.get_policy_info()
                    # print(data_dict)
                    data_dict['A'] = data_dict['A'].strip()
                    data_dict['E'] = data_dict['E'].strip()
                    data_dict['R'] = self.plainTextEdit_3.toPlainText()
                    data_dict['S'] = self.plainTextEdit_4.toPlainText()
                    data_dict['Y'] = '=HYPERLINK("{}", "{}")'.format(each_file, data_dict['D'])
                    # print(data_dict)
                    excel_row = self.fill_excel_sheet(data_dict, each_file)

                else:
                    print("Script for ", each_file, " is not implimented yet")
                self.progressBar.setValue(int(cnt * 100.0 / total_files))
            except Exception as e:
                print(e)
                traceback.print_exc()
                dest_fpath = createCleanupDirectory()
                shutil.copy(each_file, dest_fpath)
                show_pop = True
            # self.rename_file(each_file,excel_row)
        # Converting to Word Files
        converter = PDF_Convert.pdfconverter(Bajaj_New_Formats)
        self.progressBar.setValue(5)
        self.plainTextEdit_5.setPlainText('Now processing new formats of Bajaj')
        Bajaj_New_Word_Files = converter.convert_pdfs_doc()
        for cnt, each_file in enumerate(Bajaj_New_Word_Files):
            excel_row=None
            try:
                self.plainTextEdit_5.setPlainText('Reading file : {0}'.format(each_file))
                pdf = WordFile.BajajDataExtractor(each_file)
                data_dict = pdf.get_all_details()
                data_dict['R'] = self.plainTextEdit_3.toPlainText()
                data_dict['S'] = self.plainTextEdit_4.toPlainText()
                data_dict['P'] = data_dict['O']
                data_dict['A'] = convert_date_format(data_dict['A'].strip())
                data_dict['E'] = convert_date_format(data_dict['E'].strip())
                data_dict['Y'] = None
                # rk = BajajPolicy(pdf, each_file, self.plainTextEdit_5)
                # data_dict = rk.get_all_policy_details()
                excel_row = self.fill_excel_sheet(data_dict,each_file[:-5] + ".pdf")
            except:
                dest_fpath = createCleanupDirectory()
                shutil.copy(each_file[:-5] + ".pdf", dest_fpath)
                show_pop = True
            os.remove(each_file)
            # self.rename_file(each_file, excel_row)
            self.progressBar.setValue(int((cnt + 1) * 100.0 / len(Bajaj_New_Word_Files)))
        if show_pop:
            self.show_warning_box("A few PDF were not processed successfully. Please check the temp directory")

    def start_pdf_processing(self):
        start_time = time.time()
        excel_path = self.plainTextEdit_2.toPlainText()
        try:
            wb = load_workbook(excel_path)
            wb.save(excel_path)
        except:
            self.show_warning_box("The excel file seems to be opened in a software. Please close it and proceed.")
        self.process_files()

        self.progressBar.setValue(0)
        end_time = time.time()
        self.plainTextEdit_5.setPlainText(
            "Total time taken to parse the files : {0} Seconds".format(round(end_time - start_time, 2)))


if __name__ == "__main__":
    multiprocessing.freeze_support()
    import sys

    app = QtWidgets.QApplication(sys.argv)
    MainWindow = QtWidgets.QMainWindow()
    ui = Ui_MainWindow()
    ui.setupUi(MainWindow)
    MainWindow.show()
    # QtWidgets.QFileDialog.getOpenFileNames('test', 'D:/', filter="docx")
    sys.exit(app.exec_())