Skip to content

Commit

Permalink
Rclone logs usinf /shell cat rlog.txt
Browse files Browse the repository at this point in the history 
- Some respect for /users
- Other minor fixes

Signed-off-by: anasty17 <[email protected]>
  • Loading branch information
anasty17 committed Apr 23, 2023
1 parent 08766f5 commit 65c6c38
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 29 deletions.
56 changes: 29 additions & 27 deletions bot/helper/mirror_utils/rclone_utils/transfer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ async def __create_rc_sa(self, remote, remote_opts):
await f.write(text)
return sa_conf_file

async def __start_download(self, cmd):
async def __start_download(self, cmd, remote_type):
self.__proc = await create_subprocess_exec(*cmd, stdout=PIPE, stderr=PIPE)
_, return_code = await gather(self.__progress(), self.__proc.wait())

Expand All @@ -111,13 +111,13 @@ async def __start_download(self, cmd):
error = (await self.__proc.stderr.read()).decode().strip()
LOGGER.error(error)

if 'RATE_LIMIT_EXCEEDED' in error and config_dict['USE_SERVICE_ACCOUNTS']:
if remote_type == 'drive' and 'RATE_LIMIT_EXCEEDED' in error and config_dict['USE_SERVICE_ACCOUNTS']:
if self.__sa_number != 0 and self.__sa_count < self.__sa_number:
remote = self.__switchServiceAccount()
cmd[6] = f"{remote}:{cmd[6].split(':', 1)[1]}"
if self.__is_cancelled:
return
return self.__start_download(cmd)
return self.__start_download(cmd, remote_type)
else:
LOGGER.info(
f"Reached maximum number of service accounts switching, which is {self.__sa_count}")
Expand Down Expand Up @@ -147,7 +147,7 @@ async def download(self, remote, rc_path, config_path, path):
elif remote_type != 'drive':
cmd.extend(('--retries-sleep', '3s'))

await self.__start_download(cmd)
await self.__start_download(cmd, remote_type)

async def __get_gdrive_link(self, config_path, remote, rc_path, mime_type):
if mime_type == 'Folder':
Expand Down Expand Up @@ -177,7 +177,7 @@ async def __get_gdrive_link(self, config_path, remote, rc_path, mime_type):
link = ''
return link, destination

async def __start_upload(self, cmd):
async def __start_upload(self, cmd, remote_type):
self.__proc = await create_subprocess_exec(*cmd, stdout=PIPE, stderr=PIPE)
_, return_code = await gather(self.__progress(), self.__proc.wait())

Expand All @@ -189,11 +189,11 @@ async def __start_upload(self, cmd):
elif return_code != 0:
error = (await self.__proc.stderr.read()).decode().strip()
LOGGER.error(error)
if 'RATE_LIMIT_EXCEEDED' in error and config_dict['USE_SERVICE_ACCOUNTS']:
if remote_type == 'drive' and 'RATE_LIMIT_EXCEEDED' in error and config_dict['USE_SERVICE_ACCOUNTS']:
if self.__sa_number != 0 and self.__sa_count < self.__sa_number:
remote = self.__switchServiceAccount()
cmd[7] = f"{remote}:{cmd[7].split(':', 1)[1]}"
return False if self.__is_cancelled else self.__start_upload(cmd)
return False if self.__is_cancelled else self.__start_upload(cmd, remote_type)
else:
LOGGER.info(
f"Reached maximum number of service accounts switching, which is {self.__sa_count}")
Expand Down Expand Up @@ -246,7 +246,7 @@ async def upload(self, path, size):
elif remote_type != 'drive':
cmd.extend(('--retries-sleep', '3s'))

result = await self.__start_upload(cmd)
result = await self.__start_upload(cmd, remote_type)
if not result:
return

Expand Down Expand Up @@ -274,17 +274,23 @@ async def upload(self, path, size):
LOGGER.info(f'Upload Done. Path: {destination}')
await self.__listener.onUploadComplete(link, size, files, folders, mime_type, self.name, destination)

async def clone(self, config_path, remote, source, destination, rcflags, mime_type):
remote_opts = await self.__get_remote_options(config_path, remote)
remote_type = remote_opts['type']
async def clone(self, config_path, src_remote, src_path, destination, rcflags, mime_type):
dst_remote, dst_path = destination.split(':', 1)

cmd = await self.__getUpdatedCommand(config_path, f'{remote}:{source}', destination, rcflags, 'copy')
if remote_type == 'drive' and not rcflags:
cmd.extend(('--drive-chunk-size', '64M', '--tpslimit', '3', '--transfers',
'3', '--drive-upload-cutoff', '32M', '--drive-acknowledge-abuse'))
elif remote_type != 'drive':
cmd.extend(('--retries-sleep', '3s'))
cmd.append('--server-side-across-configs')
src_remote_opts, dst_remote_opt = await gather(self.__get_remote_options(config_path, src_remote),
self.__get_remote_options(config_path, dst_remote))

src_remote_type, dst_remote_type = src_remote_opts['type'], dst_remote_opt['type']

cmd = await self.__getUpdatedCommand(config_path, f'{src_remote}:{src_path}', destination, rcflags, 'copy')
if not rcflags:
if src_remote_type == 'drive' and dst_remote_type != 'drive':
cmd.append('--drive-acknowledge-abuse')
elif dst_remote_type == 'drive' and src_remote_type != 'drive':
cmd.extend(('--drive-chunk-size', '64M',
'--drive-upload-cutoff', '32M'))
elif src_remote_type == 'drive':
cmd.extend(('--tpslimit', '3', '--transfers', '3'))

self.__proc = await create_subprocess_exec(*cmd, stdout=PIPE, stderr=PIPE)
_, return_code = await gather(self.__progress(), self.__proc.wait())
Expand All @@ -300,17 +306,12 @@ async def clone(self, config_path, remote, source, destination, rcflags, mime_ty
await self.__listener.onUploadError(error[:4000])
return None, None
else:
dst_remote, dst_path = destination.split(':', 1)
remote_opts = await self.__get_remote_options(config_path, dst_remote)
remote_type = remote_opts['type']
if remote_type == 'drive':
dst_remote, dst_path = destination.split(':', 1)
if dst_remote_opt == 'drive':
link, destination = await self.__get_gdrive_link(config_path, dst_remote, dst_path, mime_type)
return (None, None) if self.__is_cancelled else (link, destination)
else:
if mime_type != 'Folder':
destination += f'/{self.name}' if destination.split(':', 1)[
1] else self.name
destination += f'/{self.name}' if dst_path else self.name

cmd = ['rclone', 'link', '--config', config_path, destination]
res, err, code = await cmd_exec(cmd)
Expand All @@ -323,14 +324,15 @@ async def clone(self, config_path, remote, source, destination, rcflags, mime_ty
elif code != -9:
LOGGER.error(
f'while getting link. Path: {destination} | Stderr: {err}')
await self.__listener.onUploadError(error[:4000])
await self.__listener.onUploadError(err[:4000])
return None, None

@staticmethod
async def __getUpdatedCommand(config_path, source, destination, rcflags, method):
ext = '*.{' + ','.join(GLOBAL_EXTENSION_FILTER) + '}'
cmd = ['rclone', method, '--fast-list', '--config', config_path, '-P', source, destination,
'--exclude', ext, '--ignore-case', '--low-level-retries', '1', '-M']
'--exclude', ext, '--ignore-case', '--low-level-retries', '1', '-M', '--log-file',
'rlog.txt', '--log-level', 'DEBUG']
if rcflags:
rcflags = rcflags.split('|')
for flag in rcflags:
Expand Down
11 changes: 9 additions & 2 deletions bot/modules/users_settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,8 @@ async def edit_user_settings(client, query):
await query.answer()
buttons = ButtonMaker()
if user_dict.get('lprefix', False) or config_dict['LEECH_FILENAME_PREFIX']:
buttons.ibutton("Remove Leech Prefix", f"userset {user_id} rlprefix")
buttons.ibutton("Remove Leech Prefix",
f"userset {user_id} rlprefix")
buttons.ibutton("Back", f"userset {user_id} back")
buttons.ibutton("Close", f"userset {user_id} close")
await editMessage(message, 'Send Leech Filename Prefix. You can add HTML tags. Timeout: 60 sec', buttons.build_menu(1))
Expand All @@ -352,7 +353,13 @@ async def edit_user_settings(client, query):


async def send_users_settings(client, message):
if msg := ''.join(f'<code>{u}</code>: {escape(str(d))}\n\n' for u, d in user_data.items()):
if user_data:
msg = ''
for u, d in user_data.items():
kmsg = f'\n<b>{u}:</b>\n'
if vmsg := ''.join(f'{k}: <code>{v}</code>\n' for k, v in d.items() if v):
msg += kmsg + vmsg

msg_ecd = msg.encode()
if len(msg_ecd) > 4000:
with BytesIO(msg_ecd) as ofile:
Expand Down

0 comments on commit 65c6c38

Please sign in to comment.