Skip to content

Commit

Permalink
separated execution parameters, paths protections, dialogs stack, crc…
Browse files Browse the repository at this point in the history 
… hidden
  • Loading branch information
piotrj committed Nov 29, 2023
1 parent a73ba16 commit afa4767
Show file tree
Hide file tree
Showing 4 changed files with 356 additions and 252 deletions.
85 changes: 52 additions & 33 deletions src/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,7 @@ def prepare_customdata_pool_rec(self,scan_like_data,parent_path):
matched = False

rule_nr=-1
for expressions,use_smin,smin_int,use_smax,smax_int,executable,shell,timeout,crc in cde_list:
for expressions,use_smin,smin_int,use_smax,smax_int,executable,parameters,shell,timeout,crc in cde_list:
if self.abort_action:
break
if matched:
Expand Down Expand Up @@ -397,6 +397,13 @@ def prepare_customdata_pool_rec(self,scan_like_data,parent_path):

def extract_customdata_update(self):
self.info_line_current = self.exe.info
self_header = self.header

self_header.files_cde_errors_quant = self.exe.files_cde_errors_quant

self_header.files_cde_quant = self.exe.files_cde_quant
self_header.files_cde_size = self.exe.files_cde_size
self_header.files_cde_size_extracted = self.exe.files_cde_size_extracted

def extract_customdata(self):
self_header = self.header
Expand Down Expand Up @@ -424,21 +431,21 @@ def extract_customdata(self):
for (scan_like_list,subpath,rule_nr,size) in self.customdata_pool.values():
if self.abort_action:
break
expressions,use_smin,smin_int,use_smax,smax_int,executable,shell,timeout,do_crc = cde_list[rule_nr]
expressions,use_smin,smin_int,use_smax,smax_int,executable,parameters,shell,timeout,do_crc = cde_list[rule_nr]

full_file_path = normpath(abspath(sep.join([scan_path,subpath]))).replace('/',sep)

#cde_run_list = list(executable) + [full_file_path]

io_list.append( [ list(executable),full_file_path,timeout,shell,do_crc,size,scan_like_list,rule_nr ] )
io_list.append( [ executable,parameters,full_file_path,timeout,shell,do_crc,size,scan_like_list,rule_nr ] )

#############################################################
self.exe=Executor(io_list,self.extract_customdata_update)

self.exe.run()
#############################################################
for io_list_elem in io_list:
executable,full_file_path,timeout,shell,do_crc,size,scan_like_list,rule_nr,result_tuple = io_list_elem
executable,parameters,full_file_path,timeout,shell,do_crc,size,scan_like_list,rule_nr,result_tuple = io_list_elem
if do_crc:
returncode,output,crc_val = result_tuple
else:
Expand Down Expand Up @@ -468,13 +475,13 @@ def extract_customdata(self):
if self.abort_action:
break

expressions,use_smin,smin_int,use_smax,smax_int,executable,shell,timeout,crc = cde_list[rule_nr]
expressions,use_smin,smin_int,use_smax,smax_int,executable,parameters,shell,timeout,crc = cde_list[rule_nr]

full_file_path = normpath(abspath(sep.join([scan_path,subpath]))).replace('/',sep)

size = scan_like_list[0]

cde_run_list = list(executable) + [full_file_path]
cde_run_list = list(executable) + list(parameters) + [full_file_path]

if crc:
self.info_line_current = f'{subpath} CRC calculation ({bytes_to_str(size)})'
Expand Down Expand Up @@ -806,8 +813,13 @@ def find_items_sort(self,what,reverse):
print('unknown sorting',what,mod)

def prepare_info(self):
bytes_to_str_mod = lambda x : bytes_to_str(x) if type(x) == int else x

info_list = []

self.txtinfo = 'init'
self.txtinfo_basic = 'init-basic'

try:
self.FILE_SIZE = stat(self.file_path).st_size
except Exception as e:
Expand All @@ -821,30 +833,30 @@ def prepare_info(self):
self_header = self.header

local_time = strftime('%Y/%m/%d %H:%M:%S',localtime(self.header.creation_time))
info_list.append(f'name: {self_header.label}')
info_list.append(f'scan path: {self_header.scan_path}')
info_list.append(f'size: {bytes_to_str(self_header.sum_size)}')
info_list.append(f'host: {self_header.creation_host}')
info_list.append(f'OS: {self_header.creation_os}')
info_list.append(f'creation time: {local_time}')
info_list.append(f'file: {self.FILE_NAME}')
info_list.append(f'')
info_list.append(f'file size: {bytes_to_str(self.FILE_SIZE)}')
info_list.append(f'record label : {self_header.label}')
info_list.append(f'')
info_list.append(f'internal sizes:')
info_list.append(f'scanned path : {self_header.scan_path}')
info_list.append(f'scanned space : {bytes_to_str(self_header.sum_size)}')
info_list.append(f'scanned files : {fnumber(self_header.quant_files)}')
info_list.append(f'scanned folders : {fnumber(self_header.quant_folders)}')
info_list.append(f'')
info_list.append( ' compressed decompressed')
info_list.append(f'creation host : {self_header.creation_host} ({self_header.creation_os})')
info_list.append(f'creation time : {local_time}')

self.txtinfo_basic = '\n'.join(info_list)

bytes_to_str_mod = lambda x : bytes_to_str(x) if type(x) == int else x
info_list.append(f'')
info_list.append(f'database file : {self.FILE_NAME} ({bytes_to_str(self.FILE_SIZE)})')
info_list.append(f'')
info_list.append(f'internal sizes : compressed decompressed')
info_list.append(f'')

info_list.append(f'header :{bytes_to_str_mod(zip_file_info["header"]).rjust(14)}{bytes_to_str_mod(self.zipinfo["header"]).rjust(14)}')
info_list.append(f'filestructure :{bytes_to_str_mod(zip_file_info["filestructure"]).rjust(14)}{bytes_to_str_mod(self.zipinfo["filestructure"]).rjust(14)}')
info_list.append(f'file names :{bytes_to_str_mod(zip_file_info["filenames"]).rjust(14)}{bytes_to_str_mod(self.zipinfo["filenames"]).rjust(14)}')
info_list.append(f'custom data :{bytes_to_str_mod(zip_file_info["customdata"]).rjust(14)}{bytes_to_str_mod(self.zipinfo["customdata"]).rjust(14)}')
info_list.append(f'header :{bytes_to_str_mod(zip_file_info["header"]).rjust(14)}{bytes_to_str_mod(self.zipinfo["header"]).rjust(14)}')
info_list.append(f'filestructure :{bytes_to_str_mod(zip_file_info["filestructure"]).rjust(14)}{bytes_to_str_mod(self.zipinfo["filestructure"]).rjust(14)}')
info_list.append(f'file names :{bytes_to_str_mod(zip_file_info["filenames"]).rjust(14)}{bytes_to_str_mod(self.zipinfo["filenames"]).rjust(14)}')

info_list.append(f'\nquant_files:{fnumber(self_header.quant_files)}')
info_list.append(f'quant_folders:{fnumber(self_header.quant_folders)}')
info_list.append(f'sum_size:{bytes_to_str(self_header.sum_size)}')
if zip_file_info["customdata"]:
info_list.append(f'custom data :{bytes_to_str_mod(zip_file_info["customdata"]).rjust(14)}{bytes_to_str_mod(self.zipinfo["customdata"]).rjust(14)}')

try:
if self_header.cde_list:
Expand Down Expand Up @@ -884,10 +896,13 @@ def save(self,file_path=None,compression_level=16):
self.zipinfo['filenames'] = len(filenames_ser)
zip_file.writestr('filenames',cctx.compress(filenames_ser))

self.info_line = f'saving {filename} (Custom Data)'
customdata_ser = dumps(self.customdata)
self.zipinfo['customdata'] = len(customdata_ser)
zip_file.writestr('customdata',cctx.compress(customdata_ser))
if self.customdata:
self.info_line = f'saving {filename} (Custom Data)'
customdata_ser = dumps(self.customdata)
self.zipinfo['customdata'] = len(customdata_ser)
zip_file.writestr('customdata',cctx.compress(customdata_ser))
else:
self.zipinfo['customdata'] = 0

self.prepare_info()

Expand Down Expand Up @@ -953,10 +968,14 @@ def decompress_customdata(self):
dctx = ZstdDecompressor()
with ZipFile(self.file_path, "r") as zip_file:

customdata_ser_comp = zip_file.read('customdata')
customdata_ser = dctx.decompress(customdata_ser_comp)
self.customdata = loads( customdata_ser )
self.zipinfo['customdata'] = len(customdata_ser)
try:
customdata_ser_comp = zip_file.read('customdata')
customdata_ser = dctx.decompress(customdata_ser_comp)
self.customdata = loads( customdata_ser )
self.zipinfo['customdata'] = len(customdata_ser)
except:
self.customdata = []
self.zipinfo['customdata'] = 0

self.decompressed_customdata = True

Expand Down
4 changes: 3 additions & 1 deletion src/dialogs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ def unlock(self):

def focusin(self):
global locked_by_child
#print('\nlocked_by_child\n',[print(k,v) for k,v in locked_by_child.items()])
if child_widget := locked_by_child[self.widget]:
child_widget.focus_set()

Expand Down Expand Up @@ -188,6 +189,8 @@ def hide(self,force_hide=False):
except Exception as e:
pass

locked_by_child[self.parent]=None

if self.post_close:
self.post_close()

Expand All @@ -197,7 +200,6 @@ def hide(self,force_hide=False):
else:
self.parent.focus_set()

locked_by_child[self.parent]=None

self.wait_var.set(True)
self.parent.config(cursor="")
Expand Down
69 changes: 58 additions & 11 deletions src/executor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,32 @@
from hashlib import sha1
from os import sep

from os import name as os_name
from sys import getsizeof

windows = bool(os_name=='nt')

def get_command_list(executable,parameters,full_file_path,shell=False):
if windows:
full_file_path = full_file_path.replace('/',sep)

if shell:
executable=f'"{executable}"'
full_file_path = f'"{full_file_path}"'

if parameters:
parameters = parameters.replace('"$"','$')
if '$' not in parameters:
return None
else:
parameters='$'

parameters_list = [p_elem.replace('$',full_file_path) for p_elem in parameters.split() if p_elem]

single_command_list = [executable] + parameters_list

return single_command_list

class Executor :
def __init__(self,io_list,callback):
self.io_list=io_list
Expand All @@ -43,6 +69,13 @@ def __init__(self,io_list,callback):
self.timeout=None
self.info = ''

self.files_cde_errors_quant = 0

self.files_cde_quant = 0
self.files_cde_size = 0
self.files_cde_size_extracted = 0


def calc_crc(self,fullpath,size):
CRC_BUFFER_SIZE=4*1024*1024
buf = bytearray(CRC_BUFFER_SIZE)
Expand Down Expand Up @@ -92,9 +125,10 @@ def abort_now(self):
def run(self):
for single_command_combo in self.io_list:
self_results_list_append = single_command_combo.append
executable,full_file_path,timeout,shell,do_crc,size = single_command_combo[0:6]
executable,parameters,full_file_path,timeout,shell,do_crc,size = single_command_combo[0:7]
do_crc = False #TODO hidden option

single_command_list = executable + [full_file_path.replace('/',sep)]
single_command_list = get_command_list(executable,parameters,full_file_path,shell)

if self.keep_running:
try:
Expand Down Expand Up @@ -142,18 +176,31 @@ def run(self):
returncode=100 if killed else 101

if killed:
returncode = 102
output = output + '\nKilled.'
if do_crc:
crc=self.calc_crc(full_file_path,size)
self_results_list_append(tuple([returncode,output,crc]))
else:
self_results_list_append(tuple([returncode,output]))
#if do_crc:
# crc=self.calc_crc(full_file_path,size)
# self_results_list_append(tuple([returncode,output,crc]))
#else:
# self_results_list_append(tuple([returncode,output]))

else:
if do_crc:
self_results_list_append((300,'CDE aborted',0))
else:
self_results_list_append((300,'CDE aborted'))
returncode = 300
output = 'CDE aborted'
#if do_crc:
# self_results_list_append((300,'CDE aborted',0))
#else:
# self_results_list_append((300,'CDE aborted'))

self_results_list_append(tuple([returncode,output]))

if returncode:
self.files_cde_errors_quant+=1

self.files_cde_quant += 1
self.files_cde_size = size
self.files_cde_size_extracted += getsizeof(output)


def kill(self,pid):
proc = Process(pid)
Expand Down
Loading

0 comments on commit afa4767

Please sign in to comment.