From bdf26a8fecd1f9c0f49a353e64bcce26565ab9bc Mon Sep 17 00:00:00 2001 From: morvanzhou Date: Thu, 30 May 2024 14:00:16 +0800 Subject: [PATCH] feat(app): reorder config and change temp dir name --- .dockerignore | 1 + .gitignore | 1 + src/retk/config.py | 48 ++++++++++++++++++++++++----------- tests/test_api.py | 4 +-- tests/test_core_local.py | 20 +++++++-------- tests/test_llm.py | 23 +++++++++-------- tests/test_model_indexing.py | 2 +- tests/test_plugins.py | 10 ++++---- tests/test_run.py | 2 +- tests/test_version_manager.py | 4 +-- tests/utils.py | 2 +- 11 files changed, 70 insertions(+), 47 deletions(-) diff --git a/.dockerignore b/.dockerignore index 4d03c90..4b4c19f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -23,3 +23,4 @@ *.sw? /temp +/trails diff --git a/.gitignore b/.gitignore index 14477a9..0d6e634 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ /temp +/trails diff --git a/src/retk/config.py b/src/retk/config.py index f54553d..7927784 100644 --- a/src/retk/config.py +++ b/src/retk/config.py @@ -13,32 +13,48 @@ class Settings(BaseSettings): + # system settings ONE_USER: bool = Field(default=False, env='ONE_USER') RETHINK_SERVER_DEBUG: bool = Field(default=False, env='RETHINK_SERVER_DEBUG') VERIFY_REFERER: bool = Field(default=False, env='VERIFY_REFERER') PLUGINS: bool = Field(default=False, env='PLUGINS') RETHINK_LOCAL_STORAGE_PATH: Optional[DirectoryPath] = Field(env='RETHINK_LOCAL_STORAGE_PATH', default=None) + CAPTCHA_SALT: str = Field(env='CAPTCHA_SALT', default="") + MD_BACKUP_INTERVAL: int = Field(env="MD_BACKUP_INTERVAL", default=60 * 5) # 5 minutes + + # database settings: MongoDB DB_NAME: str = Field(env='DB_NAME', default="") DB_USER: str = Field(env='DB_USER', default="") DB_PASSWORD: str = Field(env='DB_PASSWORD', default="") DB_HOST: str = Field(env='DB_HOST', default="") DB_PORT: int = Field(env='DB_PORT', default=-1) DB_SALT: str = Field(env='BD_SALT', default="") - HUNYUAN_SECRET_ID: str = Field(env='HUNYUAN_SECRET_ID', default="") - HUNYUAN_SECRET_KEY: str = Field(env='HUNYUAN_SECRET_KEY', default="") + + # database settings: ElasticSearch ES_USER: str = Field(env='ES_USER', default="") ES_PASSWORD: str = Field(env='ES_PASSWORD', default="") ES_HOSTS: str = Field(env='ES_HOSTS', default="") ES_INDEX_ALIAS: str = Field(env='ES_INDEX_ALIAS', default="") - CAPTCHA_SALT: str = Field(env='CAPTCHA_SALT', default="") + + # database settings: COS + COS_SECRET_ID: str = Field(env='COS_SECRET_ID', default="") + COS_SECRET_KEY: str = Field(env="COS_SECRET_KEY", default="") + COS_REGION: Optional[str] = Field(env="COS_REGION", default=None) + COS_BUCKET_NAME: str = Field(env="COS_BUCKET_NAME", default="") + COS_DOMAIN: Optional[str] = Field(env="COS_DOMAIN", default=None) + + # hunyuan + HUNYUAN_SECRET_ID: str = Field(env='HUNYUAN_SECRET_ID', default="") + HUNYUAN_SECRET_KEY: str = Field(env='HUNYUAN_SECRET_KEY', default="") + + # aliyun + ALIYUN_DASHSCOPE_API_KEY: str = Field(env='ALIYUN_DASHSCOPE_API_KEY', default="") + + # Email client settings RETHINK_EMAIL: str = Field(env='RETHINK_EMAIL', default="") RETHINK_EMAIL_PASSWORD: str = Field(env='RETHINK_EMAIL_PASSWORD', default="") - JWT_KEY: bytes = Field(env='JWT_KEY', default=b"") - JWT_KEY_PUB: bytes = Field(env='JWT_KEY_PUB', default=b"") - JWT_REFRESH_EXPIRED_DAYS: int = Field(default=1, env='JWT_REFRESH_EXPIRED_DAYS') - JWT_ACCESS_EXPIRED_MINS: int = Field(default=5, env='JWT_ACCESS_EXPIRED_MINS') - REFRESH_TOKEN_EXPIRE_DELTA: datetime.timedelta = Field(default=datetime.timedelta(days=1)) - ACCESS_TOKEN_EXPIRE_DELTA: datetime.timedelta = Field(default=datetime.timedelta(minutes=15)) + + # OAuth settings OAUTH_REDIRECT_URL: str = Field(env='OAUTH_REDIRECT_URL', default="") OAUTH_CLIENT_ID_GITHUB: str = Field(env='OAUTH_CLIENT_ID_GITHUB', default="") OAUTH_CLIENT_SEC_GITHUB: str = Field(env='OAUTH_CLIENT_SEC_GITHUB', default="") @@ -49,12 +65,14 @@ class Settings(BaseSettings): OAUTH_CLIENT_SEC_FACEBOOK: str = Field(env='OAUTH_CLIENT_SEC_QQ', default="") OAUTH_CLIENT_ID_GOOGLE: str = Field(env='OAUTH_CLIENT_ID_GOOGLE', default="") OAUTH_CLIENT_SEC_GOOGLE: str = Field(env='OAUTH_CLIENT_SEC_GOOGLE', default="") - COS_SECRET_ID: str = Field(env='COS_SECRET_ID', default="") - COS_SECRET_KEY: str = Field(env="COS_SECRET_KEY", default="") - COS_REGION: Optional[str] = Field(env="COS_REGION", default=None) - COS_BUCKET_NAME: str = Field(env="COS_BUCKET_NAME", default="") - COS_DOMAIN: Optional[str] = Field(env="COS_DOMAIN", default=None) - MD_BACKUP_INTERVAL: int = Field(env="MD_BACKUP_INTERVAL", default=60 * 5) # 5 minutes + + # auth secret + JWT_KEY: bytes = Field(env='JWT_KEY', default=b"") + JWT_KEY_PUB: bytes = Field(env='JWT_KEY_PUB', default=b"") + JWT_REFRESH_EXPIRED_DAYS: int = Field(default=1, env='JWT_REFRESH_EXPIRED_DAYS') + JWT_ACCESS_EXPIRED_MINS: int = Field(default=5, env='JWT_ACCESS_EXPIRED_MINS') + REFRESH_TOKEN_EXPIRE_DELTA: datetime.timedelta = Field(default=datetime.timedelta(days=1)) + ACCESS_TOKEN_EXPIRE_DELTA: datetime.timedelta = Field(default=datetime.timedelta(minutes=15)) model_config = SettingsConfigDict( env_file=".env", diff --git a/tests/test_api.py b/tests/test_api.py index fee671c..4c48a5d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -145,8 +145,8 @@ async def asyncTearDown(self) -> None: scheduler.stop() await client.drop() self.client.close() - shutil.rmtree(Path(__file__).parent / "tmp" / const.settings.DOT_DATA / "files", ignore_errors=True) - shutil.rmtree(Path(__file__).parent / "tmp" / const.settings.DOT_DATA / "md", ignore_errors=True) + shutil.rmtree(Path(__file__).parent / "temp" / const.settings.DOT_DATA / "files", ignore_errors=True) + shutil.rmtree(Path(__file__).parent / "temp" / const.settings.DOT_DATA / "md", ignore_errors=True) @classmethod def tearDownClass(cls) -> None: diff --git a/tests/test_core_local.py b/tests/test_core_local.py index 6102ab3..3e8c222 100644 --- a/tests/test_core_local.py +++ b/tests/test_core_local.py @@ -31,14 +31,14 @@ class LocalModelsTest(unittest.IsolatedAsyncioTestCase): def setUpClass(cls) -> None: utils.set_env(".env.test.local") # create a fake image - tmp_path = Path(__file__).parent / "tmp" / "fake.png" + tmp_path = Path(__file__).parent / "temp" / "fake.png" tmp_path.parent.mkdir(parents=True, exist_ok=True) Image.new("RGB", (100, 100)).save(tmp_path) @classmethod def tearDownClass(cls) -> None: utils.drop_env(".env.test.local") - shutil.rmtree(Path(__file__).parent / "tmp", ignore_errors=True) + shutil.rmtree(Path(__file__).parent / "temp", ignore_errors=True) async def asyncSetUp(self) -> None: await client.init() @@ -51,8 +51,8 @@ async def asyncSetUp(self) -> None: async def asyncTearDown(self) -> None: await client.drop() - shutil.rmtree(Path(__file__).parent / "tmp" / const.settings.DOT_DATA / "files", ignore_errors=True) - shutil.rmtree(Path(__file__).parent / "tmp" / const.settings.DOT_DATA / "md", ignore_errors=True) + shutil.rmtree(Path(__file__).parent / "temp" / const.settings.DOT_DATA / "files", ignore_errors=True) + shutil.rmtree(Path(__file__).parent / "temp" / const.settings.DOT_DATA / "md", ignore_errors=True) async def test_user(self): u, code = await core.user.get_by_email(email=const.DEFAULT_USER["email"]) @@ -423,7 +423,7 @@ async def test_upload_image_vditor(self): u, code = await core.user.get(self.au.u.id) self.assertEqual(const.CodeEnum.OK, code) used_space = u["usedSpace"] - p = Path(__file__).parent / "tmp" / "fake.png" + p = Path(__file__).parent / "temp" / "fake.png" image = Image.open(p) buf = BytesIO() @@ -436,7 +436,7 @@ async def test_upload_image_vditor(self): res = await core.files.vditor_upload(au=self.au, files=[img_file]) self.assertIn("fake.png", res["succMap"]) self.assertTrue(".png" in res["succMap"]["fake.png"]) - local_file = Path(__file__).parent / "tmp" / const.settings.DOT_DATA / res["succMap"]["fake.png"][1:] + local_file = Path(__file__).parent / "temp" / const.settings.DOT_DATA / res["succMap"]["fake.png"][1:] self.assertTrue(local_file.exists()) local_file.unlink() image.close() @@ -450,7 +450,7 @@ async def test_upload_image_vditor(self): "retk.core.files.upload.httpx.AsyncClient.get", ) async def test_fetch_image_vditor(self, mock_get): - f = open(Path(__file__).parent / "tmp" / "fake.png", "rb") + f = open(Path(__file__).parent / "temp" / "fake.png", "rb") mock_get.return_value = httpx.Response( 200, content=f.read(), @@ -465,7 +465,7 @@ async def test_fetch_image_vditor(self, mock_get): self.assertEqual(const.CodeEnum.OK, code) self.assertTrue(new_url.endswith(".png")) self.assertTrue(new_url.startswith("/")) - local_file = Path(__file__).parent / "tmp" / const.settings.DOT_DATA / new_url[1:] + local_file = Path(__file__).parent / "temp" / const.settings.DOT_DATA / new_url[1:] self.assertTrue(local_file.exists()) local_file.unlink() @@ -498,14 +498,14 @@ async def test_node_version(self): au=self.au, md="[title](/qqq)\nbody", type_=const.NodeTypeEnum.MARKDOWN.value ) self.assertEqual(const.CodeEnum.OK, code) - md_path = Path(__file__).parent / "tmp" / const.settings.DOT_DATA / "md" / (node["id"] + ".md") + md_path = Path(__file__).parent / "temp" / const.settings.DOT_DATA / "md" / (node["id"] + ".md") self.assertTrue(md_path.exists()) time.sleep(1) _, _, code = await core.node.update_md(au=self.au, nid=node["id"], md="title2\nbody2") self.assertEqual(const.CodeEnum.OK, code) - hist_dir = Path(__file__).parent / "tmp" / const.settings.DOT_DATA / "md" / "hist" / node["id"] + hist_dir = Path(__file__).parent / "temp" / const.settings.DOT_DATA / "md" / "hist" / node["id"] self.assertEqual(1, len(list(hist_dir.glob("*.md")))) time.sleep(1) diff --git a/tests/test_llm.py b/tests/test_llm.py index f44c213..5114123 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -1,5 +1,4 @@ import json -import os import unittest from unittest.mock import patch, AsyncMock @@ -13,8 +12,8 @@ class HunyuanTest(unittest.IsolatedAsyncioTestCase): @classmethod def setUpClass(cls): - os.environ["HUNYUAN_SECRET_ID"] = "testid" - os.environ["HUNYUAN_SECRET_KEY"] = "testkey" + cls.sid = "testid" + cls.skey = "testkey" utils.set_env(".env.test.local") @classmethod @@ -31,23 +30,26 @@ def test_authorization(self): "EnableEnhancement": False, } payload = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") - m = llm.hunyuan.HunyuanLite() + m = llm.hunyuan.HunyuanLite( + secret_id=self.sid, secret_key=self.skey + ) auth = m.get_auth( action="ChatCompletions", payload=payload, timestamp=1716913478, content_type="application/json" ) - sid = os.environ["HUNYUAN_SECRET_ID"] self.assertEqual( - f"TC3-HMAC-SHA256 Credential={sid}/2024-05-28/hunyuan/tc3_request," + f"TC3-HMAC-SHA256 Credential={self.sid}/2024-05-28/hunyuan/tc3_request," f" SignedHeaders=content-type;host;x-tc-action," f" Signature=f628e271c4acdf72a4618fe59e3a31591f2ddedbd44e5befe6e02c05949b01b3", auth) async def test_hunyuan_auth_failed(self): - m = llm.hunyuan.HunyuanLite() - self.assertEqual("hunyuan-lite", m.model_name) + m = llm.hunyuan.HunyuanLite( + secret_id=self.sid, secret_key=self.skey + ) + self.assertEqual("hunyuan-lite", m.name) text, code = await m.complete([{"Role": "user", "Content": "你是谁"}]) self.assertEqual(const.CodeEnum.LLM_SERVICE_ERROR, code, msg=text) self.assertEqual("SecretId不存在，请输入正确的密钥。", text) @@ -69,8 +71,9 @@ async def test_hunyuan_complete(self, mock_post): } } ) - m = llm.hunyuan.HunyuanLite() - self.assertEqual("hunyuan-lite", m.model_name) + m = llm.hunyuan.HunyuanLite( + secret_id=self.sid, secret_key=self.skey + ) text, code = await m.complete([{"Role": "user", "Content": "你是谁"}]) self.assertEqual(const.CodeEnum.OK, code, msg=text) self.assertEqual("我是一个AI助手。", text) diff --git a/tests/test_model_indexing.py b/tests/test_model_indexing.py index 3d37788..4560480 100644 --- a/tests/test_model_indexing.py +++ b/tests/test_model_indexing.py @@ -27,7 +27,7 @@ async def case(keys, out_index): ) self.assertEqual(out_index, index) - mongo_path = os.path.join("tmp", "mongo") + mongo_path = os.path.join("temp", "mongo") os.makedirs(mongo_path, exist_ok=True) mongo = MongitaClientDisk(mongo_path) db = mongo["test_db"] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index c85d53b..05d55b7 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -17,7 +17,7 @@ class TestPlugin(retk.Plugin): version = "0.1.0" description = "A demo test plugin." author = "morvanzhou" - icon = "tmp/back.svg" + icon = "temp/back.svg" template = "

{h}

\n

{p}

" def __init__(self): @@ -53,7 +53,7 @@ class DemoCount(unittest.IsolatedAsyncioTestCase): @classmethod def setUpClass(cls) -> None: utils.set_env(".env.test.local") - tmp_dir = Path(__file__).parent / "tmp" + tmp_dir = Path(__file__).parent / "temp" tmp_dir.mkdir(exist_ok=True) with open(tmp_dir / "back.svg", "w") as f: f.write(svg) @@ -61,7 +61,7 @@ def setUpClass(cls) -> None: @classmethod def tearDownClass(cls) -> None: utils.drop_env(".env.test.local") - shutil.rmtree(Path(__file__).parent / "tmp", ignore_errors=True) + shutil.rmtree(Path(__file__).parent / "temp", ignore_errors=True) async def asyncSetUp(self) -> None: self.p = TestPlugin() @@ -78,8 +78,8 @@ async def asyncSetUp(self) -> None: async def asyncTearDown(self) -> None: retk.remove_plugin(self.p) await client.drop() - shutil.rmtree(Path(__file__).parent / "tmp" / const.settings.DOT_DATA / "files", ignore_errors=True) - shutil.rmtree(Path(__file__).parent / "tmp" / const.settings.DOT_DATA / "md", ignore_errors=True) + shutil.rmtree(Path(__file__).parent / "temp" / const.settings.DOT_DATA / "files", ignore_errors=True) + shutil.rmtree(Path(__file__).parent / "temp" / const.settings.DOT_DATA / "md", ignore_errors=True) def test_creation(self): self.assertEqual(self.p.id, "testPlugin") diff --git a/tests/test_run.py b/tests/test_run.py index 99d98dc..1d06b85 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -16,7 +16,7 @@ class TestRun(unittest.TestCase): @classmethod def setUpClass(cls) -> None: config.get_settings.cache_clear() - cls.path = p = Path(__file__).parent / "tmp" + cls.path = p = Path(__file__).parent / "temp" p.mkdir(exist_ok=True) @classmethod diff --git a/tests/test_version_manager.py b/tests/test_version_manager.py index 3a62c8d..d6a3021 100644 --- a/tests/test_version_manager.py +++ b/tests/test_version_manager.py @@ -8,7 +8,7 @@ class RecoverTest(unittest.TestCase): def setUp(self) -> None: - self.tmp_dir = Path(__file__).parent / "tmp" + self.tmp_dir = Path(__file__).parent / "temp" self.tmp_dir.mkdir(parents=True, exist_ok=True) def tearDown(self) -> None: @@ -41,7 +41,7 @@ def test_dump_load(self): class MigrateTest(unittest.TestCase): def setUp(self) -> None: - self.tmp_dir = Path(__file__).parent / "tmp" + self.tmp_dir = Path(__file__).parent / "temp" self.tmp_dir.mkdir(parents=True, exist_ok=True) def tearDown(self) -> None: diff --git a/tests/utils.py b/tests/utils.py index 557be3f..223129b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -12,7 +12,7 @@ def set_env(file=".env.test.local"): os.environ["VUE_APP_MODE"] = "local" os.environ["VUE_APP_API_URL"] = "http//127.0.0.1:8000" if file.endswith(".local"): - tmp = Path(__file__).parent / "tmp" + tmp = Path(__file__).parent / "temp" if tmp.exists(): shutil.rmtree(tmp, ignore_errors=True) tmp.mkdir(exist_ok=True)