Skip to content

Commit

Permalink
garbage collector activated manually in critical sections, search opt…
Browse files Browse the repository at this point in the history 
…imizations
  • Loading branch information
piotrj committed Jan 2, 2024
1 parent dddc4c4 commit 3b079e4
Show file tree
Hide file tree
Showing 3 changed files with 111 additions and 46 deletions.
62 changes: 36 additions & 26 deletions src/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
windows = bool(os_name=='nt')

if windows:
from subprocess import CREATE_NO_WINDOW,HIGH_PRIORITY_CLASS
from subprocess import CREATE_NO_WINDOW
else:
from os import getpgid, killpg

Expand Down Expand Up @@ -217,20 +217,16 @@ def kill_subprocess(subproc):
except Exception as ke:
print(f'kill_subprocess: {ke}')

def compress_with_stats(data,compression):
def compress_with_header_update(header,data,compression,datalabel,zip_file):
t0 = perf_counter()
data_ser = dumps(data)
data_ser_compr = ZstdCompressor(level=compression,threads=-1).compress(data_ser)
t1 = perf_counter()
tdiff=t1-t0

return data_ser_compr,tdiff,asizeof(data_ser_compr),asizeof(data_ser),asizeof(data)

def compress_with_header_update(header,data,compression,datalabel,zip_file):
data_compr,tdiff,size_1,size_2,size_3 = compress_with_stats(data,compression)
header.zipinfo[datalabel]=(size_1,size_2,size_3)
header.compression_time[datalabel] = tdiff
zip_file.writestr(datalabel,data_compr)
header.zipinfo[datalabel]=(asizeof(data_ser_compr),asizeof(data_ser),asizeof(data))
header.compression_time[datalabel] = tdiff
zip_file.writestr(datalabel,data_ser_compr)

CREATION_CODE = 0
EXPORT_CODE = 1
Expand Down Expand Up @@ -848,7 +844,8 @@ def find_items(self,
size_min,size_max,
timestamp_min,timestamp_max,
name_search_kind,name_func_to_call,
cd_search_kind,cd_func_to_call):
cd_search_kind,cd_func_to_call,
print_info_fn):

self.find_results = []

Expand All @@ -872,7 +869,7 @@ def find_items(self,
search_list_pop = search_list.pop
search_list_append = search_list.append

cd_search_kind_is_regezp_glob_or_fuzzy = bool(cd_search_kind in ('regexp','glob','fuzzy'))
cd_search_kind_is_regexp_glob_or_fuzzy = bool(cd_search_kind in ('regexp','glob','fuzzy'))
cd_search_kind_is_dont_or_without = bool(cd_search_kind in ('dont','without'))

when_folder_may_apply = bool(cd_search_kind_is_dont_or_without and not use_size and not use_timestamp)
Expand All @@ -893,11 +890,6 @@ def find_items(self,
#if check_abort():
# break

search_progress_update_quant+=1
if search_progress_update_quant>1024:
results_queue_put([search_progress])
search_progress_update_quant=0

search_progress +=1

name_nr,code,size,mtime = data_entry[0:4]
Expand Down Expand Up @@ -961,7 +953,7 @@ def find_items(self,
if not name_func_to_call(name):
continue
except Exception as e:
self.log.error('find_items(1):%s',str(e) )
print_info_fn(f'find_items(1):{e}' )
continue

#oczywistosc
Expand All @@ -971,18 +963,27 @@ def find_items(self,
if cd_search_kind_is_any:
if not has_cd or not cd_ok:
continue
elif cd_search_kind_is_regezp_glob_or_fuzzy:
elif cd_search_kind_is_regexp_glob_or_fuzzy:
if has_cd and cd_ok:
cd_data = self_customdata[cd_nr][2]
else:
continue

if cd_func_to_call:
try:
if not cd_func_to_call(cd_data):
continue
if True: #entire file
if not cd_func_to_call(cd_data):
continue
else:
found=False
for line in cd_data.splitlines():
if cd_func_to_call(line):
found=True
break
if not found:
continue
except Exception as e:
self.log.error('find_items(2):%s',str(e) )
print_info_fn(f'find_items(2):{e}' )
continue

else:
Expand All @@ -1000,6 +1001,12 @@ def find_items(self,
results_queue_put([search_progress,size,mtime,*next_level])
search_progress_update_quant=0

if search_progress_update_quant>1024:
results_queue_put([search_progress])
search_progress_update_quant=0
else:
search_progress_update_quant+=1

results_queue_put([search_progress])
results_queue_put(True)

Expand Down Expand Up @@ -1345,6 +1352,7 @@ def repack_record(self,record,new_label,new_compression,keep_cd,update_callback)
messages = []

new_file_path = sep.join([self.db_dir,f'rep.{int(time())}.dat'])

try:
src_file = record.file_path

Expand Down Expand Up @@ -1470,7 +1478,7 @@ def find_items_in_records_check(self,
filename_fuzzy_threshold,cd_fuzzy_threshold):

sel_range = [range_par] if range_par else self.records
self.files_search_quant = sum([record.header.quant_files for record in sel_range])
self.files_search_quant = sum([record.header.quant_files+record.header.quant_folders for record in sel_range])

if self.files_search_quant==0:
return 1
Expand Down Expand Up @@ -1530,14 +1538,16 @@ def find_items_in_records(self,
records_to_process.sort(reverse = True,key = lambda x : x.header.quant_files)

record_commnad_list={}
is_frozen = bool(getattr(sys, 'frozen', False))

for record_nr,record in enumerate(records_to_process):
if windows:
if getattr(sys, 'frozen', False):
if is_frozen:
curr_command_list = record_commnad_list[record_nr] = ['record.exe', 'load',record.file_path]
else:
curr_command_list = record_commnad_list[record_nr] = ['python','src\\record.py', 'load',record.file_path]
else:
if getattr(sys, 'frozen', False):
if is_frozen:
curr_command_list = record_commnad_list[record_nr] = ['./record', 'load',record.file_path]
else:
curr_command_list = record_commnad_list[record_nr] = ['python3','./src/record.py', 'load',record.file_path]
Expand Down Expand Up @@ -1628,8 +1638,8 @@ def threaded_run(record_nr,commands_list,results_list,progress_list,info_list,pr
else:
info_list[record_nr].append(line.strip())
except Exception as e:
print('threaded_run work error:',e,line)
info_list[record_nr].append(f'threaded_run work error: {e} line: {line}')
print(f'threaded_run work error:{e} line:{line}')
info_list[record_nr].append(f'threaded_run work error:{e} line:{line}')
else:
if subprocess_poll() is not None:
break
Expand Down
Loading

0 comments on commit 3b079e4

Please sign in to comment.