From 1c4c3062594995d9fd0cb8a1042a8c7f493bdeb6 Mon Sep 17 00:00:00 2001 From: chiangtw Date: Tue, 7 Mar 2023 13:27:19 +0800 Subject: [PATCH] speed up the file checking process --- pyega3/libs/data_file.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyega3/libs/data_file.py b/pyega3/libs/data_file.py index af9048f..5b179a3 100644 --- a/pyega3/libs/data_file.py +++ b/pyega3/libs/data_file.py @@ -138,6 +138,8 @@ def download_file(self, output_file, num_connections=1, max_slice_size=DEFAULT_S min(chunk_len, file_size - chunk_start_pos), options, pbar) for chunk_start_pos in range(0, file_size, chunk_len)] + all_slices = set([(param[1], param[2]) for param in params]) + for file in os.listdir(temporary_directory): match = re.match(r"(.*)-from-(\d*)-len-(\d*).*", file) file_id = match.group(1) @@ -147,7 +149,7 @@ def download_file(self, output_file, num_connections=1, max_slice_size=DEFAULT_S if file_id != self.id: continue - if (int(file_from), int(file_length)) in [(param[1], param[2]) for param in params]: + if (int(file_from), int(file_length)) in all_slices: continue logging.warning(f'Deleting the leftover {file} temporary file because the MAX_SLICE_SIZE parameter ('