From 85ef488be5b2e0feb78d303e1bdf8c08debc79a2 Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Thu, 26 May 2022 20:57:27 +0800 Subject: [PATCH] feat: remove upload to aliyundrive Signed-off-by: Tianling Shen --- .github/workflows/update-mirror.yml | 18 -- README.md | 3 +- aliyundrive-uploader/AliyunDrive.py | 345 --------------------- aliyundrive-uploader/Client.py | 235 -------------- aliyundrive-uploader/common.py | 252 --------------- aliyundrive-uploader/gen-alidriveconf.sh | 23 -- aliyundrive-uploader/main.py | 105 ------- aliyundrive-uploader/requirements.txt | 7 - aliyundrive-uploader/sqlite.py | 378 ----------------------- 9 files changed, 1 insertion(+), 1365 deletions(-) delete mode 100644 aliyundrive-uploader/AliyunDrive.py delete mode 100644 aliyundrive-uploader/Client.py delete mode 100644 aliyundrive-uploader/common.py delete mode 100755 aliyundrive-uploader/gen-alidriveconf.sh delete mode 100644 aliyundrive-uploader/main.py delete mode 100644 aliyundrive-uploader/requirements.txt delete mode 100644 aliyundrive-uploader/sqlite.py diff --git a/.github/workflows/update-mirror.yml b/.github/workflows/update-mirror.yml index 536c8a7..72706e5 100644 --- a/.github/workflows/update-mirror.yml +++ b/.github/workflows/update-mirror.yml @@ -104,24 +104,6 @@ jobs: [ -d "uploader-keys" ] || mkdir -p "uploader-keys" cp -f "$UPLOADER_CONF" "uploader-keys/$UPLOADER_TYPE.json" - - name: Upload Files to AliyunDrive - env: - UPLOADER_TYPE: "alidrive" - ALI_DRIVE_ID: ${{ secrets.ALI_DRIVE_ID }} - ALI_REFRESH_TOKEN: ${{ secrets.ALI_REFRESH_TOKEN }} - run: | - ./uploader-cache/detect-upload-files.sh - [ "$(ls -A "package-sources"/* | wc -l)" -gt "0" ] || exit 0 - pushd aliyundrive-uploader - [ -f "../uploader-keys/$UPLOADER_TYPE.json" ] && cp -f "../uploader-keys/$UPLOADER_TYPE.json" "config.json" || ./gen-alidriveconf.sh - [ -f "../uploader-keys/$UPLOADER_TYPE.db" ] && cp -f "../uploader-keys/$UPLOADER_TYPE.db" "db.db" - python3 main.py || echo "Never mind." - popd - ./uploader-cache/gen-downloaded-hash.sh - [ -d "uploader-keys" ] || mkdir -p "uploader-keys" - cp -f "aliyundrive-uploader/config.json" "uploader-keys/$UPLOADER_TYPE.json" - cp -f "aliyundrive-uploader/db.db" "uploader-keys/$UPLOADER_TYPE.db" - cleanup-workflow-runs: runs-on: ubuntu-18.04 needs: update-mirror diff --git a/README.md b/README.md index 4bbc323..bcef21c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ ## Reference [SuLingGG/OpenWrt-Mini/update-immortalwrt-1806-dl-mirror.yml](https://github.com/SuLingGG/OpenWrt-Mini/blob/381524b297bc346c3afbf779eef8601ffe6c469f/.github/workflows/update-immortalwrt-1806-dl-mirror.yml).
-[MoeClub/OneList/OneDriveUploader](https://github.com/MoeClub/OneList/tree/206b44d9a129d383211806f02a600a96893e2445/OneDriveUploader).
-[Hidove/aliyundrive-uploader](https://github.com/Hidove/aliyundrive-uploader/tree/d16b104bc924b390c82789338dd9f60e7ed994ba). +[MoeClub/OneList/OneDriveUploader](https://github.com/MoeClub/OneList/tree/206b44d9a129d383211806f02a600a96893e2445/OneDriveUploader). diff --git a/aliyundrive-uploader/AliyunDrive.py b/aliyundrive-uploader/AliyunDrive.py deleted file mode 100644 index 72d7560..0000000 --- a/aliyundrive-uploader/AliyunDrive.py +++ /dev/null @@ -1,345 +0,0 @@ -# -*- coding: utf-8 -*- -# +------------------------------------------------------------------- -# | 阿里云盘上传类 -# +------------------------------------------------------------------- -# | Author: Pluto -# +------------------------------------------------------------------- - -import json -import math -import os -import requests -from tqdm import tqdm - -from common import DATA, LOCK_TOKEN_REFRESH -import common - -requests.packages.urllib3.disable_warnings() - -""" -status: 0：未上传，1：上传成功，2：正在上传，-1：上传失败 -""" - - -class AliyunDrive: - def __init__(self, drive_id, root_path, chunk_size=10485760): - self.status = 0 - self.create_time = 0 - self.start_time = common.get_timestamp() - self.finish_time = 0 - self.spend_time = 0 - self.drive_id = drive_id - self.root_path = root_path - self.chunk_size = chunk_size - self.filepath = None - self.realpath = None - self.filename = None - self.hash = None - self.part_info_list = [] - self.part_upload_url_list = [] - self.upload_id = 0 - self.file_id = 0 - self.part_number = 0 - self.filesize = 0 - self.headers = { - 'authorization': DATA['access_token'], - 'content-type': 'application/json;charset=UTF-8' - } - self.id = None - - def load_task(self, task): - tmp = [ - "id", - "filepath", - "realpath", - "filesize", - "hash", - "status", - "create_time", - "finish_time", - "spend_time", - "drive_id", - "file_id", - "upload_id", - "part_number", - "chunk_size", - ] - if task['drive_id'] == '' or int(task['drive_id']) == 0: - task['drive_id'] = self.__getattribute__('drive_id') - for v in tmp: - self.__setattr__(v, task[v]) - - def load_file(self, filepath, realpath): - self.filepath = filepath - self.realpath = realpath - self.filename = os.path.basename(self.realpath) - self.print('【{filename}】正在校检文件中，耗时与文件大小有关'.format(filename=self.filename), 'info') - self.hash = common.get_hash(self.realpath) - self.filesize = os.path.getsize(self.realpath) - - self.part_info_list = [] - for i in range(0, math.ceil(self.filesize / self.chunk_size)): - self.part_info_list.append({ - 'part_number': i + 1 - }) - - message = '''================================================= - 文件名：{filename} - hash：{hash} - 文件大小：{filesize} - 文件路径：{filepath} -================================================= -'''.format(filename=self.filename, hash=self.hash, filesize=self.filesize, filepath=self.realpath) - self.print(message, 'info') - - def token_refresh(self): - LOCK_TOKEN_REFRESH.acquire() - try: - data = {"refresh_token": DATA['config']['REFRESH_TOKEN']} - post = requests.post( - 'https://websv.aliyundrive.com/token/refresh', - data=json.dumps(data), - headers={ - 'content-type': 'application/json;charset=UTF-8' - }, - verify=False, - timeout=3 - ) - try: - post_json = post.json() - # 刷新配置中的token - common.set_config('REFRESH_TOKEN', post_json['refresh_token']) - - except Exception as e: - self.print('refresh_token已经失效', 'warn') - raise e - - DATA['access_token'] = post_json['access_token'] - self.headers = { - 'authorization': DATA['access_token'], - 'content-type': 'application/json;charset=UTF-8' - } - DATA['config']['REFRESH_TOKEN'] = post_json['refresh_token'] - finally: - LOCK_TOKEN_REFRESH.release() - return True - - def create(self, parent_file_id): - create_data = { - "drive_id": self.drive_id, - "part_info_list": self.part_info_list, - "parent_file_id": parent_file_id, - "name": self.filename, - "type": "file", - "check_name_mode": "auto_rename", - "size": self.filesize, - "content_hash": self.hash, - "content_hash_name": 'sha1' - } - # 覆盖已有文件 - if DATA['config']['OVERWRITE']: - create_data['check_name_mode'] = 'refuse' - request_post = requests.post( - 'https://api.aliyundrive.com/v2/file/create', - # 'https://api.aliyundrive.com/adrive/v2/file/createWithFolders', - data=json.dumps(create_data), - headers=self.headers, - verify=False - ) - requests_post_json = request_post.json() - if not self.check_auth(requests_post_json): - return self.create(parent_file_id) - # 覆盖已有文件 - if DATA['config']['OVERWRITE'] and requests_post_json.get('exist'): - if self.recycle(requests_post_json.get('file_id')): - self.print('【%s】原有文件回收成功' % self.filename, 'info') - self.print('【%s】重新上传新文件中' % self.filename, 'info') - return self.create(parent_file_id) - - self.part_upload_url_list = requests_post_json.get('part_info_list', []) - self.file_id = requests_post_json.get('file_id') - self.upload_id = requests_post_json.get('upload_id') - common.save_task(self.id, { - 'drive_id': self.drive_id, - 'file_id': self.file_id, - 'upload_id': self.upload_id, - }) - return requests_post_json - - def get_upload_url(self): - self.print('【%s】上传地址已失效正在获取新的上传地址' % self.filename, 'info') - requests_data = { - "drive_id": self.drive_id, - "file_id": self.file_id, - "part_info_list": self.part_info_list, - "upload_id": self.upload_id, - } - requests_post = requests.post( - 'https://api.aliyundrive.com/v2/file/get_upload_url', - data=json.dumps(requests_data), - headers=self.headers, - verify=False - ) - requests_post_json = requests_post.json() - if not self.check_auth(requests_post_json): - return self.get_upload_url() - self.print('【%s】上传地址刷新成功' % self.filename, 'info') - return requests_post_json.get('part_info_list') - - def upload(self): - with open(self.realpath, "rb") as f: - task_log_id = common.log('正在上传【%s】0%%' % self.filename, self.id, 'info') - with tqdm.wrapattr(f, "read", desc='正在上传【%s】' % self.filename, miniters=1, - initial=self.part_number * self.chunk_size, - total=self.filesize, - ascii=True - ) as fs: - - while self.part_number < len(self.part_upload_url_list): - upload_url = self.part_upload_url_list[self.part_number]['upload_url'] - total_size = min(self.chunk_size, self.filesize) - fs.seek(self.part_number * total_size) - res = requests.put( - url=upload_url, - data=common.read_in_chunks(fs, 16 * 1024, total_size), - verify=False, - timeout=None - ) - if 400 <= res.status_code < 600: - common_get_xml_value = common.get_xml_tag_value(res.text, 'Message') - if common_get_xml_value == 'Request has expired.': - self.part_upload_url_list = self.get_upload_url() - continue - common_get_xml_value = common.get_xml_tag_value(res.text, 'Code') - if common_get_xml_value == 'PartNotSequential': - self.part_number -= 1 - continue - elif common_get_xml_value == 'PartAlreadyExist': - pass - else: - self.print(res.text, 'error') - # res.raise_for_status() - return False - self.part_number += 1 - common.update_task_log(task_log_id, - '正在上传【%s】%.2f%%' % ( - self.filename, ((self.part_number * total_size) / self.filesize) * 100)) - udata = { - "part_number": self.part_number, - } - common.save_task(self.id, udata) - - return True - - def complete(self): - complete_data = { - "drive_id": self.drive_id, - "file_id": self.file_id, - "upload_id": self.upload_id - } - complete_post = requests.post( - 'https://api.aliyundrive.com/v2/file/complete', json.dumps(complete_data), - headers=self.headers, - verify=False - ) - - requests_post_json = complete_post.json() - if not self.check_auth(requests_post_json): - return self.complete() - self.finish_time = common.get_timestamp() - self.spend_time = self.finish_time - self.start_time - - if 'file_id' in requests_post_json: - self.print('【{filename}】上传成功！消耗{s}秒'.format(filename=self.filename, s=self.spend_time), 'success') - return True - else: - self.print('【{filename}】上传失败！消耗{s}秒'.format(filename=self.filename, s=self.spend_time), 'warn') - return False - - def create_folder(self, folder_name, parent_folder_id): - create_data = { - "drive_id": self.drive_id, - "parent_file_id": parent_folder_id, - "name": folder_name, - "check_name_mode": "refuse", - "type": "folder" - } - requests_post = requests.post( - 'https://api.aliyundrive.com/adrive/v2/file/createWithFolders', - data=json.dumps(create_data), - headers=self.headers, - verify=False - ) - - requests_post_json = requests_post.json() - if not self.check_auth(requests_post_json): - return self.create_folder(folder_name, parent_folder_id) - return requests_post_json.get('file_id') - - def get_parent_folder_id(self, filepath): - self.print('检索目录中', 'info') - filepath_split = (self.root_path + filepath.lstrip(os.sep)).split(os.sep) - del filepath_split[len(filepath_split) - 1] - path_name = os.sep.join(filepath_split) - if path_name not in DATA['folder_id_dict']: - parent_folder_id = 'root' - parent_folder_name = os.sep - if len(filepath_split) > 0: - for folder in filepath_split: - if folder in ['', 'root']: - continue - parent_folder_id = self.create_folder(folder, parent_folder_id) - parent_folder_name = parent_folder_name.rstrip(os.sep) + os.sep + folder - DATA['folder_id_dict'][parent_folder_name] = parent_folder_id - else: - parent_folder_id = DATA['folder_id_dict'][path_name] - self.print('已存在目录，无需创建', 'info') - - self.print('目录id获取成功{parent_folder_id}'.format(parent_folder_id=parent_folder_id), 'info') - return parent_folder_id - - def recycle(self, file_id): - # https://api.aliyundrive.com/v2/batch - requests_data = { - "requests": [ - { - "body": { - "drive_id": self.drive_id, - "file_id": file_id - }, - "headers": { - "Content-Type": "application/json" - }, - "id": file_id, - "method": "POST", - "url": "/recyclebin/trash" - } - ], - "resource": "file" - } - requests_post = requests.post( - 'https://api.aliyundrive.com/v2/batch', - data=json.dumps(requests_data), - headers=self.headers, - verify=False - ) - requests_post_json = requests_post.json() - if not self.check_auth(requests_post_json): - return self.recycle(file_id) - return True - - def check_auth(self, response_json): - if response_json.get('code') == 'AccessTokenInvalid': - self.print('AccessToken已失效，尝试刷新AccessToken中', 'info') - if self.token_refresh(): - self.print('AccessToken刷新成功，准备返回', 'info') - return False - self.print('无法刷新AccessToken，准备退出', 'error') - if 'code' in response_json.keys(): - self.print(response_json, 'error') - common.suicide() - return True - - def print(self, message, print_type='info'): - func = 'print_' + print_type - return getattr(common, func)(message, self.id) diff --git a/aliyundrive-uploader/Client.py b/aliyundrive-uploader/Client.py deleted file mode 100644 index 956e31e..0000000 --- a/aliyundrive-uploader/Client.py +++ /dev/null @@ -1,235 +0,0 @@ -# -*- coding: utf-8 -*- -# +------------------------------------------------------------------- -# | Client.py -# +------------------------------------------------------------------- -# | Author: Pluto -# +------------------------------------------------------------------- - -# 配置信息 - -import json -import os -import sqlite3 -import sys - -from AliyunDrive import AliyunDrive -from common import get_running_path, get_config, DATA, get_config_file_path, qualify_path, \ - get_all_file_relative, LOCK, get_db_file_path, save_task, get_timestamp, date, suicide -import common - - -class Client(): - tasks = [] - database_file = get_db_file_path() - - def __init__(self): - pass - - def __upload(self, drive): - try: - drive.upload() - except Exception as e: - status = False - for index in range(int(DATA['config']['RETRY'])): - self.print('【%s】正在尝试第%d次重试！' % (drive.filename, index + 1), 'warn', drive.id) - if drive.upload(): - status = True - break - if not status: - drive.status = -1 - return drive - # 提交 - if drive.complete(): - drive.status = 1 - else: - drive.status = -1 - return drive - - def init_config(self): - config = { - "REFRESH_TOKEN": "refresh_token", - "DRIVE_ID": "drive_id", - "ROOT_PATH": "root", - "FILE_PATH": get_running_path(), - "MULTITHREADING": False, - "MAX_WORKERS": 5, - "CHUNK_SIZE": 104857600, - "RESUME": False, - "OVERWRITE": False, - "RETRY": 0, - "RESIDENT": False, - "VERSIONS": "v2021.0729.1800" - } - if not os.path.exists(get_config_file_path()): - self.print('请配置好config.json后重试', 'error') - with open(get_config_file_path(), 'w') as f: - f.write(json.dumps(config)) - suicide() - try: - config.update(get_config()) - DATA['config'] = config - - except Exception as e: - self.print('请配置好config.json后重试', 'error') - raise e - - def init_command_line_parameter(self): - unset_keys = [] - - for k in range(len(sys.argv)): - if sys.argv[k] == '--resident' or sys.argv[k] == '-r': - DATA['config']['RESIDENT'] = True - unset_keys.append(k) - - for v in unset_keys: - del sys.argv[v] - - # 命令分配 - if len(sys.argv) == 3: - # 网盘保存路径 - DATA['config']['ROOT_PATH'] = sys.argv[2] - # 读取文件路径 - abspath = os.path.abspath(sys.argv[1]) - - elif len(sys.argv) == 2: - # 读取文件路径 - abspath = os.path.abspath(sys.argv[1]) - else: - # 读取配置文件里的 - abspath = DATA['config']['FILE_PATH'] - - DATA['config']['FILE_PATH'] = os.path.dirname(abspath) - DATA['config']['ROOT_PATH'] = qualify_path(DATA['config']['ROOT_PATH']) - if not DATA['config']['RESIDENT']: - if os.path.exists(abspath): - if os.path.isdir(abspath): - # 目录上传 - self.tasks = get_all_file_relative(abspath) - self.tasks = list(map(lambda x: os.path.basename(abspath) + os.sep + x, self.tasks)) - else: - # 单文件上传 - self.tasks = [os.path.basename(abspath)] - else: - - self.print('该文件夹不存在：%s，请重试' % abspath, 'error') - # 获取目录的父目录以上传该目录并且格式化目录 - - DATA['config']['FILE_PATH'] = qualify_path(DATA['config']['FILE_PATH']) - - def init_database(self): - conn = sqlite3.connect(self.database_file) - cursor = conn.cursor() - cursor.execute('''create table IF NOT EXISTS task -( - id INTEGER - primary key autoincrement, - filepath TEXT default '' not null, - realpath TEXT default '' not null, - filesize INTEGER, - hash TEXT default '' not null, - status INTEGER default 0 not null, - drive_id TEXT default '' not null, - file_id TEXT default '' not null, - upload_id TEXT default '' not null, - part_number INTEGER default 0 not null, - chunk_size INTEGER default 104857600 not null, - create_time INTEGER default 0 not null, - finish_time INTEGER default 0 not null, - spend_time INTEGER default 0 not null -);''') - cursor.execute('''create table IF NOT EXISTS task_log -( - id INTEGER not null - constraint task_log_pk - primary key autoincrement, - task_id INTEGER, - log_level TEXT default 'info' not null, - content TEXT default '' not null, - create_time INTEGER default 0 not null -);''') - - def upload_file(self, task): - save_task(task['id'], { - 'status': 2 - }) - drive = AliyunDrive(DATA['config']['DRIVE_ID'], DATA['config']['ROOT_PATH'], DATA['config']['CHUNK_SIZE']) - # 加载任务队列 - drive.load_task(task) - # 刷新token - # drive.token_refresh() - if not os.path.exists(task['realpath']): - drive.status = -1 - return drive - drive.load_file(task['filepath'], task['realpath']) - # 创建目录 - LOCK.acquire() - try: - parent_folder_id = drive.get_parent_folder_id(drive.filepath) - finally: - LOCK.release() - # 断点续传 - if DATA['config']['RESUME'] and DATA['config']['DRIVE_ID'] == task['drive_id']: - if 0 not in [ - drive.drive_id, - drive.part_number, - drive.chunk_size, - ] and not drive.file_id and not drive.upload_id: - # 获取上传地址 - drive.part_upload_url_list = drive.get_upload_url() - # 上传 - return self.__upload(drive) - - # 创建上传 - create_post_json = drive.create(parent_folder_id) - if 'rapid_upload' in create_post_json and create_post_json['rapid_upload']: - drive.finish_time = get_timestamp() - drive.spend_time = drive.finish_time - drive.start_time - - self.print('【{filename}】秒传成功！消耗{s}秒'.format(filename=drive.filename, s=drive.spend_time), 'success', - drive.id) - drive.status = 1 - return drive - # 上传 - return self.__upload(drive) - - def save_task(self, task): - task_id = task.id - tmp = [ - "filepath", - "realpath", - "filesize", - "hash", - "status", - "create_time", - "finish_time", - "spend_time", - "drive_id", - "file_id", - "upload_id", - "part_number", - "chunk_size", - ] - - data = {} - for v in tmp: - data[v] = task.__getattribute__(v) - if data[v] is None: - data[v] = '' - - return save_task(task_id, data) - - def print_config_info(self): - s = '' - for k in DATA['config'].keys(): - s += "\n\t\t%s：%s" % (k, DATA['config'][k]) - - content = '''================================================= - 阿里云盘上传工具启动成功 - 当前时间：%s%s -================================================= -''' % (date(get_timestamp()), s) - self.print(content, 'info') - - def print(self, message, print_type, id=0): - func = 'print_' + print_type - return getattr(common, func)(message, id) diff --git a/aliyundrive-uploader/common.py b/aliyundrive-uploader/common.py deleted file mode 100644 index da170a6..0000000 --- a/aliyundrive-uploader/common.py +++ /dev/null @@ -1,252 +0,0 @@ -# -*- coding: utf-8 -*- -# +------------------------------------------------------------------- -# | 公共函数类 -# +------------------------------------------------------------------- -# | Author: Pluto -# +------------------------------------------------------------------- - -import hashlib -import json -import os -import sys -import threading -import time -from xml.dom.minidom import parseString - -from sqlite import sqlite - -LOCK = threading.Lock() -LOCK_TOKEN_REFRESH = threading.Lock() -DATA = { - 'access_token': '', - 'time_period': 600, - 'config': {}, - 'folder_id_dict': {}, - 'task_template': { - "filepath": None, - "filesize": 0, - "hash": '', - "status": 0, - "create_time": time.time(), - "finish_time": 0, - "spend_time": 0, - "drive_id": '0', - "file_id": '0', - "upload_id": '0', - "part_number": 0, - "chunk_size": 104857600, - } -} - - -def suicide(): - os._exit(1) - - -def ctrl_c(signalnum, frame): - suicide() - - -# 处理路径 -def qualify_path(path): - if not path: - return '' - return path.replace('/', os.sep).replace('\\\\', os.sep).rstrip(os.sep) + os.sep - - -# 获取运行目录 -def get_running_path(path=''): - if getattr(sys, 'frozen', False): - return os.path.dirname(sys.executable) + path - elif __file__: - return os.path.dirname(__file__) + path - - -def get_config_file_path(): - return get_running_path('/config.json') - - -def get_db_file_path(): - return get_running_path('/db.db') - - -# 读取配置项 -# @param key 取指定配置项，若不传则取所有配置[可选] -def get_config(key=None): - # 判断是否从文件读取配置 - if not os.path.exists(get_config_file_path()): return None - - with open(get_config_file_path(), 'rb') as f: - f_body = f.read().decode('utf-8') - if not f_body: return None - config = json.loads(f_body) - for value in [ - 'MULTITHREADING', - 'RESUME', - 'OVERWRITE', - 'RESIDENT', - ]: - if value in config: - DATA['config'][value] = bool(config[value]) - config['ROOT_PATH'] = qualify_path(config.get('ROOT_PATH')).rstrip(os.sep) - # 取指定配置项 - if key: - if key in config: return config[key] - return None - return config - - -def set_config(key, value): - config = get_config() - # 是否需要初始化配置项 - if not config: config = {} - # 是否需要设置配置值 - if key: - config[key] = value - with open(get_config_file_path(), 'w') as f: - f.write(json.dumps(config)) - f.flush() - return True - - -def get_db(): - return sqlite().dbfile(get_db_file_path()) - - -def get_hash(filepath, block_size=2 * 1024 * 1024): - with open(filepath, 'rb') as f: - sha1 = hashlib.sha1() - while True: - data = f.read(block_size) - if not data: - break - sha1.update(data) - return sha1.hexdigest() - - -def get_all_file(path): - result = [] - get_dir = os.listdir(path) - for i in get_dir: - sub_dir = os.path.join(path, i) - if os.path.isdir(sub_dir): - result.extend(get_all_file(sub_dir)) - else: - result.append(sub_dir) - return result - - -def get_all_file_relative(path): - result = [] - if not os.path.exists(path): - return result - get_dir = os.listdir(path) - for i in get_dir: - sub_dir = os.path.join(path, i) - if os.path.isdir(sub_dir): - all_file = get_all_file_relative(sub_dir) - all_file = map(lambda x: i + os.sep + x, all_file) - result.extend(all_file) - else: - result.append(i) - return result - - -def print_info(message, id=None): - message = message.__str__() - # i = random.randint(34, 37) - i = 36 - log(message, id, 'info') - print('\033[7;30;{i}m{message}\033[0m'.format(message=message, i=i)) - - -def print_warn(message, id=None): - message = message.__str__() - log(message, id, 'warn') - print('\033[7;30;33m{message}\033[0m'.format(message=message)) - - -def print_error(message, id=None): - message = message.__str__() - log(message, id, 'error') - print('\033[7;30;31m{message}\033[0m'.format(message=message)) - - -def print_success(message, id=None): - message = message.__str__() - log(message, id, 'success') - print('\033[7;30;32m{message}\033[0m'.format(message=message)) - - -def date(timestamp=None): - if not timestamp: - timestamp = get_timestamp() - return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp)) - - -def get_timestamp(): - return int(time.time()) - - -def log(message, id=None, log_level='info'): - task_log_id = None - if not id is None: - db = get_db() - idata = { - 'task_id': id, - 'content': message, - 'log_level': log_level, - 'create_time': get_timestamp(), - } - task_log_id = db.table('task_log').insert(idata) - file = get_running_path('/log/' + time.strftime("%Y-%m-%d", time.localtime()) + '.log') - if not os.path.exists(os.path.dirname(file)): - os.mkdir(os.path.dirname(file)) - with open(file, 'a') as f: - f.write('【{date}】{message}\n'.format(date=date(time.time()), message=message)) - return task_log_id - - -def update_task_log(task_log_id, message): - db = get_db() - return db.table('task_log').where('id=?', task_log_id).update({ - 'content': message - }) - - -def get_xml_tag_value(xml_string, tag_name): - DOMTree = parseString(xml_string) - DOMTree = DOMTree.documentElement - tag = DOMTree.getElementsByTagName(tag_name) - if len(tag) > 0: - for node in tag[0].childNodes: - if node.nodeType == node.TEXT_NODE: - return node.data - return False - - -def load_task(): - db = get_db() - return db.table('task').where('finish_time=?', 0).order('create_time asc').limit('25').select() - - -def save_task(task_id, udata): - db = get_db() - return db.table('task').where('id=?', (task_id,)).update(udata) - - -def create_task(data): - db = get_db() - db.table('task').insert(data) - - -def read_in_chunks(file_object, chunk_size=16 * 1024, total_size=10 * 1024 * 1024): - load_size = 0 - while True: - if load_size >= total_size: - break - data = file_object.read(chunk_size) - if not data: - break - load_size += 16 * 1024 - yield data diff --git a/aliyundrive-uploader/gen-alidriveconf.sh b/aliyundrive-uploader/gen-alidriveconf.sh deleted file mode 100755 index 32ad8e3..0000000 --- a/aliyundrive-uploader/gen-alidriveconf.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: GPL-3.0-only -# -# Copyright (C) 2021 ImmortalWrt.org - -cat > "$PWD/config.json" < -# +------------------------------------------------------------------- - -import os -import signal -import time -from concurrent.futures import ThreadPoolExecutor - -from AliyunDrive import AliyunDrive -from Client import Client -from common import DATA, print_error, get_db, get_timestamp, print_info, load_task, create_task, suicide, ctrl_c - -if __name__ != '__main__': - suicide() - - -signal.signal(signal.SIGINT, ctrl_c) -signal.signal(signal.SIGTERM, ctrl_c) - -client = Client() -# 数据库初始化 -client.init_database() -# 配置信息初始化 -client.init_config() -# 命令行参数初始化，读取文件列表 -client.init_command_line_parameter() -# 输出配置信息 -client.print_config_info() -db = get_db() -# 是否常驻运行 -if not DATA['config']['RESIDENT']: - for v in client.tasks: - tmp = DATA['task_template'].copy() - tmp.update({ - "filepath": v, - "realpath": DATA['config']['FILE_PATH'] + v, - "create_time": get_timestamp(), - }) - find = db.table('task').where('filepath=? and realpath=?', (tmp['filepath'], tmp['realpath'],)).find() - if find: - print_info('【%s】已存在任务队列中，跳过' % tmp['filepath']) - else: - create_task(tmp) - - -def thread(task): - drive = client.upload_file(task) - drive.finish_time = get_timestamp() - drive.spend_time = drive.finish_time - drive.start_time - if drive.status != 1: - print_error(os.path.basename(drive.filepath) + ' 上传失败') - client.save_task(drive) - - -def distribute_thread(tasks): - if not DATA['config']['MULTITHREADING'] or int(DATA['config']['MAX_WORKERS']) <= 0: - for task in tasks: - thread(task) - else: - with ThreadPoolExecutor(max_workers=int(DATA['config']['MAX_WORKERS'])) as executor: - for task in tasks: - # 提交线程 - executor.submit(thread, task) - - -# 定时任务 -def crontab(): - def crontab_tasks(): - # 定时刷新token - (AliyunDrive(DATA['config']['DRIVE_ID'], DATA['config']['ROOT_PATH'], - DATA['config']['CHUNK_SIZE'])).token_refresh() - - time_period = DATA['time_period'] - crontab_tasks() - while True: - if time_period <= 0: - try: - crontab_tasks() - except Exception as e: - print_error(e.__str__()) - finally: - time_period = DATA['time_period'] - else: - time_period -= 1 - time.sleep(1) - - -(ThreadPoolExecutor()).submit(crontab) - -is_RESIDENT = DATA['config']['RESIDENT'] -while True: - client.tasks = load_task() - if len(client.tasks) <= 0: - if not is_RESIDENT: - suicide() - else: - print_info('当前无任务，等待新的任务队列中', 0) - time.sleep(5) - continue - distribute_thread(client.tasks) diff --git a/aliyundrive-uploader/requirements.txt b/aliyundrive-uploader/requirements.txt deleted file mode 100644 index 9fe130a..0000000 --- a/aliyundrive-uploader/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -certifi==2020.12.5 -chardet==4.0.0 -idna==2.10 -requests==2.25.1 -tqdm==4.59.0 -urllib3==1.26.5 -wincertstore==0.2 \ No newline at end of file diff --git a/aliyundrive-uploader/sqlite.py b/aliyundrive-uploader/sqlite.py deleted file mode 100644 index a54d5b1..0000000 --- a/aliyundrive-uploader/sqlite.py +++ /dev/null @@ -1,378 +0,0 @@ -import sys - -import sqlite3 -import os, re, time - - -def get_running_path(path=''): - if getattr(sys, 'frozen', False): - return os.path.dirname(sys.executable) + path - elif __file__: - return os.path.dirname(__file__) + path - - -class sqlite(): - # ------------------------------ - # 数据库操作类 For sqlite3 - # ------------------------------ - DB_FILE = None # 数据库文件 - DB_CONN = None # 数据库连接对象 - DB_TABLE = "" # 被操作的表名称 - OPT_WHERE = "" # where条件 - OPT_LIMIT = "" # limit条件 - OPT_ORDER = "" # order条件 - OPT_GROUPBY = "" # group by条件 - OPT_FIELD = "*" # field条件 - OPT_PARAM = () # where值 - __LOCK = get_running_path() + 'sqlite_lock.pl' - - def __init__(self): - self.DB_FILE = 'data/default.db' - - def GetConn(self): - # 取数据库对象 - try: - if self.DB_CONN == None: - self.DB_CONN = sqlite3.connect(self.DB_FILE) - if sys.version_info[0] == 3: - self.DB_CONN.text_factory = lambda x: str(x, - encoding="utf-8", errors='ignore') - else: - self.DB_CONN.text_factory = lambda x: unicode(x, - "utf-8", "ignore") - except Exception as ex: - print(ex) - return "error: " + str(ex) - - def dbfile(self, path): - if not os.path.isfile(path): - raise RuntimeError("数据库文件不存在。") - self.DB_FILE = path - return self - - def table(self, table): - # 设置表名 - self.DB_TABLE = table - return self - - def where(self, where, param): - # WHERE条件 - if where: - self.OPT_WHERE = " WHERE " + where - self.OPT_PARAM = self.__to_tuple(param) - return self - - def __to_tuple(self, param): - # 将参数转换为tuple - if type(param) != tuple: - if type(param) == list: - param = tuple(param) - else: - param = (param,) - return param - - def order(self, order): - # ORDER条件 - if len(order): - self.OPT_ORDER = " ORDER BY " + order - return self - - def groupby(self, group): - if len(group): - self.OPT_GROUPBY = " GROUP BY " + group - return self - - def limit(self, limit): - # LIMIT条件 - if len(limit): - self.OPT_LIMIT = " LIMIT " + limit - return self - - def field(self, field): - # FIELD条件 - if len(field): - self.OPT_FIELD = field - return self - - def log(self, msg): - log_file = "/www/server/panel/logs/error.log" - if sys.version_info[0] == 3: - with open(log_file, "a", encoding="utf-8") as fp: - fp.write("\n" + msg) - else: - with open(log_file, "a") as fp: - fp.write("\n" + msg) - - def select(self): - # 查询数据集 - self.GetConn() - try: - self.__get_columns() - sql = "SELECT " + self.OPT_FIELD + " FROM " + self.DB_TABLE + self.OPT_WHERE + self.OPT_GROUPBY + self.OPT_ORDER + self.OPT_LIMIT - result = self.DB_CONN.execute(sql, self.OPT_PARAM) - data = result.fetchall() - # 构造字典系列 - if self.OPT_FIELD != "*": - fields = self.__format_field(self.OPT_FIELD.split(',')) - tmp = [] - for row in data: - i = 0 - tmp1 = {} - for key in fields: - tmp1[key] = row[i] - i += 1 - tmp.append(tmp1) - del (tmp1) - data = tmp - del (tmp) - else: - # 将元组转换成列表 - tmp = list(map(list, data)) - data = tmp - del (tmp) - self.__close() - return data - except Exception as ex: - return "error: " + str(ex) - - def get(self): - self.__get_columns() - return self.select() - - def __format_field(self, field): - fields = [] - for key in field: - s_as = re.search('\s+as\s+', key, flags=re.IGNORECASE) - if s_as: - as_tip = s_as.group() - key = key.split(as_tip)[1] - fields.append(key) - return fields - - def __get_columns(self): - if self.OPT_FIELD == '*': - tmp_cols = self.query('PRAGMA table_info(' + self.DB_TABLE + ')', ()) - cols = [] - for col in tmp_cols: - if len(col) > 2: cols.append(col[1]) - if len(cols) > 0: self.OPT_FIELD = ','.join(cols) - - def getField(self, keyName): - # 取回指定字段 - try: - result = self.field(keyName).select() - if len(result) != 0: - return result[0][keyName] - return result - except: - return None - - def setField(self, keyName, keyValue): - # 更新指定字段 - return self.save(keyName, (keyValue,)) - - def find(self): - # 取一行数据 - try: - result = self.limit("1").select() - if len(result) == 1: - return result[0] - return result - except: - return None - - def count(self): - # 取行数 - key = "COUNT(*)" - data = self.field(key).select() - try: - return int(data[0][key]) - except: - return 0 - - def add(self, keys, param): - # 插入数据 - self.write_lock() - self.GetConn() - self.DB_CONN.text_factory = str - try: - values = "" - for key in keys.split(','): - values += "?," - values = values[0:len(values) - 1] - sql = "INSERT INTO " + self.DB_TABLE + "(" + keys + ") " + "VALUES(" + values + ")" - result = self.DB_CONN.execute(sql, self.__to_tuple(param)) - id = result.lastrowid - self.__close() - self.DB_CONN.commit() - self.rm_lock() - return id - except Exception as ex: - return "error: " + str(ex) - - # 插入数据 - def insert(self, pdata): - if not pdata: return False - keys, param = self.__format_pdata(pdata) - return self.add(keys, param) - - # 更新数据 - def update(self, pdata): - if not pdata: return False - keys, param = self.__format_pdata(pdata) - return self.save(keys, param) - - # 构造数据 - def __format_pdata(self, pdata): - keys = pdata.keys() - keys_str = ','.join(keys) - param = [] - for k in keys: param.append(pdata[k]) - return keys_str, tuple(param) - - def addAll(self, keys, param): - # 插入数据 - self.write_lock() - self.GetConn() - self.DB_CONN.text_factory = str - try: - values = "" - for key in keys.split(','): - values += "?," - values = values[0:len(values) - 1] - sql = "INSERT INTO " + self.DB_TABLE + "(" + keys + ") " + "VALUES(" + values + ")" - result = self.DB_CONN.execute(sql, self.__to_tuple(param)) - self.rm_lock() - return True - except Exception as ex: - return "error: " + str(ex) - - def commit(self): - self.__close() - self.DB_CONN.commit() - - def save(self, keys, param): - # 更新数据 - self.write_lock() - self.GetConn() - self.DB_CONN.text_factory = str - try: - opt = "" - for key in keys.split(','): - opt += key + "=?," - opt = opt[0:len(opt) - 1] - sql = "UPDATE " + self.DB_TABLE + " SET " + opt + self.OPT_WHERE - # 处理拼接WHERE与UPDATE参数 - tmp = list(self.__to_tuple(param)) - for arg in self.OPT_PARAM: - tmp.append(arg) - self.OPT_PARAM = tuple(tmp) - result = self.DB_CONN.execute(sql, self.OPT_PARAM) - self.__close() - self.DB_CONN.commit() - self.rm_lock() - return result.rowcount - except Exception as ex: - return "error: " + str(ex) - - def delete(self, id=None): - # 删除数据 - self.write_lock() - self.GetConn() - try: - if id: - self.OPT_WHERE = " WHERE id=?" - self.OPT_PARAM = (id,) - sql = "DELETE FROM " + self.DB_TABLE + self.OPT_WHERE - result = self.DB_CONN.execute(sql, self.OPT_PARAM) - self.__close() - self.DB_CONN.commit() - self.rm_lock() - return result.rowcount - except Exception as ex: - return "error: " + str(ex) - - def execute(self, sql, param=(), auto_commit=True): - # 执行SQL语句返回受影响行 - self.write_lock() - self.GetConn() - try: - result = self.DB_CONN.execute(sql, self.__to_tuple(param)) - if auto_commit: - self.DB_CONN.commit() - self.rm_lock() - return result.rowcount - except Exception as ex: - return "error: " + str(ex) - - # 是否有锁 - def is_lock(self): - n = 0 - while os.path.exists(self.__LOCK): - n += 1 - if n > 100: - self.rm_lock() - break - time.sleep(0.01) - - # 写锁 - def write_lock(self): - self.is_lock() - with(open(self.__LOCK, 'w+'))as f: - f.write("True") - - # 解锁 - def rm_lock(self): - if os.path.exists(self.__LOCK): - os.remove(self.__LOCK) - - def query(self, sql, param=()): - # 执行SQL语句返回数据集 - self.GetConn() - try: - result = self.DB_CONN.execute(sql, self.__to_tuple(param)) - # self.log("result:" + str(result)) - # 将元组转换成列表 - data = list(map(list, result)) - return data - except Exception as ex: - return "error: " + str(ex) - - def create(self, name): - # 创建数据表 - self.write_lock() - self.GetConn() - with(open('data/' + name + '.sql', 'rb')) as f: - script = f.read().decode('utf-8') - result = self.DB_CONN.executescript(script) - self.DB_CONN.commit() - self.rm_lock() - return result.rowcount - - def fofile(self, filename): - # 执行脚本 - self.write_lock() - self.GetConn() - with(open(filename, 'rb')) as f: - script = f.read().decode('utf-8') - result = self.DB_CONN.executescript(script) - self.DB_CONN.commit() - self.rm_lock() - return result.rowcount - - def __close(self): - # 清理条件属性 - self.OPT_WHERE = "" - self.OPT_FIELD = "*" - self.OPT_ORDER = "" - self.OPT_LIMIT = "" - self.OPT_GROUPBY = "" - self.OPT_PARAM = () - - def close(self): - # 释放资源 - try: - self.DB_CONN.close() - self.DB_CONN = None - except: - pass