diff --git a/kubespider/api/values.py b/kubespider/api/values.py index f91302eb..fa96f970 100644 --- a/kubespider/api/values.py +++ b/kubespider/api/values.py @@ -4,6 +4,7 @@ from enum import Enum from api import types +from utils.helper import get_unique_hash FILE_TYPE_TO_PATH = { types.FILE_TYPE_COMMON: "Common", @@ -88,7 +89,7 @@ def __init__(self, url: str, path: str, self.path = path self.link_type = link_type self.file_type = file_type - self.uid = uid if uid is not None else url + self.uid = uid if uid else get_unique_hash(url) class Task(Extra): diff --git a/kubespider/core/period_server.py b/kubespider/core/period_server.py index f3fac252..c5f8102f 100644 --- a/kubespider/core/period_server.py +++ b/kubespider/core/period_server.py @@ -56,7 +56,7 @@ def run_single_provider(self, provider: SourceProvider) -> TypeError: err = None for source in links: - if helper.get_unique_hash(source.url) in state: + if source.uid in state: continue if source.link_type is None: source.link_type = link_type @@ -75,7 +75,7 @@ def run_single_provider(self, provider: SourceProvider) -> TypeError: ) break # add resource to state - state.append(helper.get_unique_hash(source.url)) + state.append(source.uid) notification_server.kubespider_notification_server.send_message( title=f"[{provider_name}] start download", url=source.url, path=source.path, diff --git a/kubespider/source_provider/alist_source_provider/provider.py b/kubespider/source_provider/alist_source_provider/provider.py index ce183e35..c4c4341e 100644 --- a/kubespider/source_provider/alist_source_provider/provider.py +++ b/kubespider/source_provider/alist_source_provider/provider.py @@ -6,7 +6,7 @@ from api import types from api.values import Event, Resource from utils.config_reader import AbsConfigReader -from utils.helper import get_request_controller, retry +from utils.helper import get_request_controller, retry, get_unique_hash class AlistSourceProvider(provider.SourceProvider): @@ -90,7 +90,7 @@ def fs_list(self, page, per_page, path) -> tuple: content = resp.get("data").get("content") alist_provider = resp.get("data").get("provider") total = resp.get("data").get("total") - if code: + if code == 200: return content, alist_provider, total raise ValueError(f"response error: {resp}") @@ -116,13 +116,17 @@ def get_all_files(self, path) -> list[Resource]: files += self.get_all_files(new_path) else: item["file_type"] = types.FILE_TYPE_COMMON - uri = f'{os.path.join("/d", os.path.join(item.get("path"), item.get("name")))}' - item["link"] = urljoin(self.host, quote(uri) + f'?modified={item.get("modified")}') + uri = os.path.join("/d", os.path.join(item.get("path", "").strip('/'), item.get("name", ""))) + md5 = (item.get("hash_info") or {}).get("md5", get_unique_hash(uri)) + sign = item.get("sign", "") + modified = item.get("modified", "") + item["link"] = urljoin(self.host, quote(uri) + f'?modified={modified}&sign={sign}') files.append(Resource( url=item.pop("link"), path=item.pop("path"), link_type=self.get_link_type(), file_type=item.pop("file_type"), + uid=md5, **item )) return files