From 920bf223809ed759568748aea936c5087149a368 Mon Sep 17 00:00:00 2001 From: jingfelix Date: Thu, 28 Dec 2023 22:47:50 +0800 Subject: [PATCH 1/7] feat: support GitHub OAuth and App install Signed-off-by: jingfelix --- .gitignore | 2 + deploy/Dockerfile | 2 +- pdm.lock | 102 +++++++++++++++++++++++++++++++++++++- requirements.txt | 4 ++ server/routes/__init__.py | 1 + server/routes/github.py | 49 ++++++++++++++++++ server/utils/github.py | 92 ++++++++++++++++++++++++++++++++++ 7 files changed, 250 insertions(+), 2 deletions(-) create mode 100644 server/routes/github.py create mode 100644 server/utils/github.py diff --git a/.gitignore b/.gitignore index 3a8816c9..e81f18c5 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,5 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +*.pem diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 4a48af88..e5e3c07a 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-bullseye +FROM python:3.10-bookworm RUN sed -i "s@http://deb.debian.org@http://mirrors.aliyun.com@g" /etc/apt/sources.list RUN sed -i "s@http://security.debian.org@http://mirrors.aliyun.com@g" /etc/apt/sources.list diff --git a/pdm.lock b/pdm.lock index 8ae18451..56ee2df0 100644 --- a/pdm.lock +++ b/pdm.lock @@ -94,6 +94,50 @@ files = [ {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, ] +[[package]] +name = "cffi" +version = "1.16.0" +requires_python = ">=3.8" +summary = "Foreign Function Interface for Python calling C code." +dependencies = [ + "pycparser", +] +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + [[package]] name = "click" version = "8.1.7" @@ -117,6 +161,40 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "cryptography" +version = "41.0.7" +requires_python = ">=3.7" +summary = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +dependencies = [ + "cffi>=1.12", +] +files = [ + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf"}, + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406"}, + {file = "cryptography-41.0.7-cp37-abi3-win32.whl", hash = "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d"}, + {file = "cryptography-41.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309"}, + {file = "cryptography-41.0.7.tar.gz", hash = "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc"}, +] + [[package]] name = "exceptiongroup" version = "1.2.0" @@ -280,6 +358,18 @@ files = [ {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, ] +[[package]] +name = "jwt" +version = "1.3.1" +requires_python = ">= 3.6" +summary = "JSON Web Token library for Python 3." +dependencies = [ + "cryptography!=3.4.0,>=3.1", +] +files = [ + {file = "jwt-1.3.1-py3-none-any.whl", hash = "sha256:61c9170f92e736b530655e75374681d4fcca9cfa8763ab42be57353b2b203494"}, +] + [[package]] name = "markupsafe" version = "2.1.3" @@ -319,6 +409,16 @@ files = [ {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, ] +[[package]] +name = "pycparser" +version = "2.21" +requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +summary = "C parser in Python" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + [[package]] name = "pycryptodome" version = "3.19.1" @@ -407,7 +507,7 @@ version = "2.0.23" requires_python = ">=3.7" summary = "Database Abstraction Library" dependencies = [ - "greenlet!=0.4.17; platform_machine == \"aarch64\" or (platform_machine == \"ppc64le\" or (platform_machine == \"x86_64\" or (platform_machine == \"amd64\" or (platform_machine == \"AMD64\" or (platform_machine == \"win32\" or platform_machine == \"WIN32\")))))", + "greenlet!=0.4.17; platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\"", "typing-extensions>=4.2.0", ] files = [ diff --git a/requirements.txt b/requirements.txt index dfc5f3c3..f16db853 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,8 +8,10 @@ ca-lark-oauth==0.0.5 ca-lark-sdk==0.0.7 ca-lark-webhook==0.0.3 certifi==2023.11.17 +cffi==1.16.0 click==8.1.7 colorama==0.4.6; platform_system == "Windows" +cryptography==41.0.7 exceptiongroup==1.2.0; python_version < "3.11" Flask==3.0.0 flask-cors==4.0.0 @@ -21,7 +23,9 @@ httpx==0.26.0 idna==3.6 itsdangerous==2.1.2 Jinja2==3.1.2 +jwt==1.3.1 MarkupSafe==2.1.3 +pycparser==2.21 pycryptodome==3.19.1 pymysql==1.1.0 python-dateutil==2.8.2 diff --git a/server/routes/__init__.py b/server/routes/__init__.py index 460b5644..25a22184 100644 --- a/server/routes/__init__.py +++ b/server/routes/__init__.py @@ -1 +1,2 @@ +from .github import * from .lark import * diff --git a/server/routes/github.py b/server/routes/github.py new file mode 100644 index 00000000..f3b5af6b --- /dev/null +++ b/server/routes/github.py @@ -0,0 +1,49 @@ +import os + +from app import app +from flask import Blueprint, abort, redirect, request +from utils.github import get_installation_token, get_jwt, register_by_code + +bp = Blueprint("github", __name__, url_prefix="/api/github") + + +@bp.route("/install", methods=["GET"]) +def github_install(): + installation_id = request.args.get("installation_id", None) + + if installation_id is None: + return redirect( + f"https://github.com/apps/{os.environ.get('GITHUB_APP_NAME')}/installations/new" + ) + + print(f"installation_id: {installation_id}") + + jwt = get_jwt( + os.environ.get("GITHUB_APP_PRIVATE_KEY_PATH"), + os.environ.get("GITHUB_APP_ID"), + ) + + installation_token = get_installation_token(jwt, installation_id) + if installation_token is None: + print("Failed to get installation token.") + + # TODO: 统一解决各类 http 请求失败的情况 + abort(500) + print(f"installation_token: {installation_token}") + + # 如果有 code 参数，则为该用户注册 + code = request.args.get("code", None) + if code is not None: + print(f"code: {code}") + + user_token = register_by_code(code) + if user_token is None: + print("Failed to register by code.") + abort(500) + + print(f"user_token: {user_token}") + + return "Success!" + + +app.register_blueprint(bp) diff --git a/server/utils/github.py b/server/utils/github.py new file mode 100644 index 00000000..e9eff48f --- /dev/null +++ b/server/utils/github.py @@ -0,0 +1,92 @@ +import os +import time + +import httpx +from jwt import JWT, jwk_from_pem + + +def get_jwt(pem_path: str, app_id: str) -> str: + """Generate a JSON Web Token (JWT) for authentication. + + Args: + pem_path (str): path to the private key file. + app_id (str): GitHub App's identifier. + + Returns: + str: JWT. + """ + + # Open PEM + with open(pem_path, "rb") as pem_file: + signing_key = jwk_from_pem(pem_file.read()) + + payload = { + # Issued at time + "iat": int(time.time()), + # JWT expiration time (10 minutes maximum) + "exp": int(time.time()) + 600, + # GitHub App's identifier + "iss": app_id, + } + + # Create JWT + jwt_instance = JWT() + encoded_jwt = jwt_instance.encode(payload, signing_key, alg="RS256") + + return encoded_jwt + + +def get_installation_token(jwt: str, installation_id: str) -> str | None: + """Get installation access token + + Args: + jwt (str): The JSON Web Token used for authentication. + installation_id (str): The ID of the installation. + + Returns: + str: The installation access token. + """ + + with httpx.Client() as client: + response = client.post( + f"https://api.github.com/app/installations/{installation_id}/access_tokens", + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {jwt}", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + if response.status_code == 200: + return None + + installation_token = response.json().get("token", None) + return installation_token + + return None + + +def register_by_code(code: str) -> str | None: + """Register by code + + Args: + code (str): The code returned by GitHub OAuth. + + Returns: + str: The user access token. + """ + + with httpx.Client() as client: + response = client.post( + "https://github.com/login/oauth/access_token", + params={ + "client_id": os.environ.get("CLIENT_ID"), + "client_secret": os.environ.get("CLIENT_SECRET"), + "code": code, + }, + ) + if response.status_code == 200: + return None + + return response.text + + return None From f3f68bb9d5c77964e377474e086b3cc1ee1473b1 Mon Sep 17 00:00:00 2001 From: jingfelix Date: Fri, 29 Dec 2023 13:57:54 +0800 Subject: [PATCH 2/7] feat: support GitHub register Signed-off-by: jingfelix --- pdm.lock | 14 ++++++++++++-- pyproject.toml | 2 ++ requirements.txt | 1 + server/routes/github.py | 29 +++++++++++++++++++++++++++++ server/utils/github.py | 11 +++++++---- 5 files changed, 51 insertions(+), 6 deletions(-) diff --git a/pdm.lock b/pdm.lock index 56ee2df0..2e649f45 100644 --- a/pdm.lock +++ b/pdm.lock @@ -4,8 +4,8 @@ [metadata] groups = ["default"] strategy = ["cross_platform"] -lock_version = "4.4" -content_hash = "sha256:377060918126987ee7854a1767311674ff9e2710607aaacf0de88245cf78b1cb" +lock_version = "4.4.1" +content_hash = "sha256:c1a5d1e22693f74c3174d55040fa95f5517b735391c8b4c2eced81269267fad9" [[package]] name = "anyio" @@ -549,6 +549,16 @@ files = [ {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, ] +[[package]] +name = "urllib3" +version = "2.1.0" +requires_python = ">=3.8" +summary = "HTTP library with thread-safe connection pooling, file post, and more." +files = [ + {file = "urllib3-2.1.0-py3-none-any.whl", hash = "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3"}, + {file = "urllib3-2.1.0.tar.gz", hash = "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54"}, +] + [[package]] name = "werkzeug" version = "3.0.1" diff --git a/pyproject.toml b/pyproject.toml index 17b4eef6..341843a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,8 @@ dependencies = [ "pymysql>=1.1.0", "click>=8.1.7", "bson>=0.5.10", + "jwt>=1.3.1", + "urllib3>=2.1.0", ] requires-python = ">=3.10" readme = "README.md" diff --git a/requirements.txt b/requirements.txt index f16db853..acb280ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,4 +34,5 @@ six==1.16.0 sniffio==1.3.0 sqlalchemy==2.0.23 typing-extensions==4.9.0 +urllib3==2.1.0 Werkzeug==3.0.1 diff --git a/server/routes/github.py b/server/routes/github.py index f3b5af6b..07c9d1b5 100644 --- a/server/routes/github.py +++ b/server/routes/github.py @@ -9,6 +9,13 @@ @bp.route("/install", methods=["GET"]) def github_install(): + """Install GitHub App. + + If not `installation_id`, redirect to install page. + If `installation_id`, get installation token. + + If `code`, register by code. + """ installation_id = request.args.get("installation_id", None) if installation_id is None: @@ -46,4 +53,26 @@ def github_install(): return "Success!" +@bp.route("/register", methods=["GET"]) +def github_register(): + """GitHub OAuth register. + + If not `code`, redirect to GitHub OAuth page. + If `code`, register by code. + """ + code = request.args.get("code", None) + if code is None: + return redirect( + f"https://github.com/login/oauth/authorize?client_id={os.environ.get('GITHUB_CLIENT_ID')}" + ) + + print(f"code: {code}") + user_token = register_by_code(code) + if user_token is None: + return "Failed to register by code." + + print(f"user_token: {user_token}") + return user_token + + app.register_blueprint(bp) diff --git a/server/utils/github.py b/server/utils/github.py index e9eff48f..044c6b15 100644 --- a/server/utils/github.py +++ b/server/utils/github.py @@ -1,5 +1,6 @@ import os import time +from urllib.parse import parse_qs import httpx from jwt import JWT, jwk_from_pem @@ -79,14 +80,16 @@ def register_by_code(code: str) -> str | None: response = client.post( "https://github.com/login/oauth/access_token", params={ - "client_id": os.environ.get("CLIENT_ID"), - "client_secret": os.environ.get("CLIENT_SECRET"), + "client_id": os.environ.get("GITHUB_CLIENT_ID"), + "client_secret": os.environ.get("GITHUB_CLIENT_SECRET"), "code": code, }, ) - if response.status_code == 200: + if response.status_code != 200: return None - return response.text + access_token = parse_qs(response.text).get("access_token", None) + if access_token is not None: + return access_token[0] return None From fc1d9af7da5264ea25458c7960b1756975d7b72b Mon Sep 17 00:00:00 2001 From: jingfelix Date: Fri, 29 Dec 2023 14:14:36 +0800 Subject: [PATCH 3/7] fix: use logging instead of print directly Signed-off-by: jingfelix --- server/routes/github.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/server/routes/github.py b/server/routes/github.py index 07c9d1b5..156fd306 100644 --- a/server/routes/github.py +++ b/server/routes/github.py @@ -1,3 +1,4 @@ +import logging import os from app import app @@ -23,7 +24,7 @@ def github_install(): f"https://github.com/apps/{os.environ.get('GITHUB_APP_NAME')}/installations/new" ) - print(f"installation_id: {installation_id}") + logging.debug(f"installation_id: {installation_id}") jwt = get_jwt( os.environ.get("GITHUB_APP_PRIVATE_KEY_PATH"), @@ -32,23 +33,23 @@ def github_install(): installation_token = get_installation_token(jwt, installation_id) if installation_token is None: - print("Failed to get installation token.") + logging.debug("Failed to get installation token.") # TODO: 统一解决各类 http 请求失败的情况 abort(500) - print(f"installation_token: {installation_token}") + logging.debug(f"installation_token: {installation_token}") # 如果有 code 参数，则为该用户注册 code = request.args.get("code", None) if code is not None: - print(f"code: {code}") + logging.debug(f"code: {code}") user_token = register_by_code(code) if user_token is None: - print("Failed to register by code.") + logging.debug("Failed to register by code.") abort(500) - print(f"user_token: {user_token}") + logging.debug(f"user_token: {user_token}") return "Success!" @@ -66,12 +67,12 @@ def github_register(): f"https://github.com/login/oauth/authorize?client_id={os.environ.get('GITHUB_CLIENT_ID')}" ) - print(f"code: {code}") + logging.debug(f"code: {code}") user_token = register_by_code(code) if user_token is None: return "Failed to register by code." - print(f"user_token: {user_token}") + logging.debug(f"user_token: {user_token}") return user_token From 50658cd6cbf61449b4d18ce61a309e1c4c09e8de Mon Sep 17 00:00:00 2001 From: Lloyd Zhou Date: Fri, 29 Dec 2023 21:17:52 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E5=A2=9E=E5=8A=A0command+=E5=8A=A8?= =?UTF-8?q?=E6=80=81=E7=94=9F=E6=88=90bot=20(#6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: init server code and deploy script * feat: init server code and deploy script. * add package by pdm * feat: update * feat: add session * feat: add model * feat: add model * feat: update config * feat: update config * feat: hotfix * feat: hotfix * feat: hotfix * feat: add mysql schema * feat: add create command * add command * add command * add missing file * update ca-lark-sdk * hotfix * hotfix * hotfix * update version * update * add celery task * add celery task * remove rabbitmq * update * add auth * add auth --- deploy/Dockerfile | 3 +- deploy/docker-compose.yml | 32 +++---- pdm.lock | 176 ++++++++++++++++++++++++++++++++++++-- pyproject.toml | 5 +- requirements.txt | 17 +++- server/celery_app.py | 25 ++++++ server/command/lark.py | 72 ++++++++++++++++ server/model/lark.py | 11 +++ server/model/schema.py | 19 +++- server/routes/__init__.py | 1 + server/routes/lark.py | 31 ++++--- server/routes/team.py | 20 +++++ server/tasks/__init__.py | 11 +++ server/tasks/lark.py | 9 ++ server/utils/auth.py | 13 +++ 15 files changed, 397 insertions(+), 48 deletions(-) create mode 100644 server/celery_app.py create mode 100644 server/command/lark.py create mode 100644 server/model/lark.py create mode 100644 server/routes/team.py create mode 100644 server/tasks/__init__.py create mode 100644 server/tasks/lark.py create mode 100644 server/utils/auth.py diff --git a/deploy/Dockerfile b/deploy/Dockerfile index e5e3c07a..887a8090 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,7 +1,6 @@ FROM python:3.10-bookworm -RUN sed -i "s@http://deb.debian.org@http://mirrors.aliyun.com@g" /etc/apt/sources.list -RUN sed -i "s@http://security.debian.org@http://mirrors.aliyun.com@g" /etc/apt/sources.list +RUN sed -i "s@http://deb.debian.org@http://mirrors.aliyun.com@g" /etc/apt/sources.list.d/debian.sources RUN ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime ADD ./requirements.txt /tmp/requirements.txt diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 31b0e3fa..73389028 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -1,10 +1,17 @@ version: '2' services: - gitmaya: - restart: always + worker: image: gitmaya volumes: - .env:/server/.env + command: celery -A tasks.celery worker -l DEBUG -c 2 + + beat: + extends: worker + command: celery -A tasks.celery beat -l DEBUG + + gitmaya: + extends: worker ports: - "8888" environment: @@ -15,22 +22,9 @@ services: image: redis:alpine ports: - "6379" - - rabbitmq: - restart: always - image: rabbitmq:3.7-management-alpine - environment: - RABBITMQ_ERLANG_COOKIE: "SWQOKODSQALRPCLNMEQG" - RABBITMQ_DEFAULT_USER: "rabbitmq" - RABBITMQ_DEFAULT_PASS: "rabbitmq" - RABBITMQ_DEFAULT_VHOST: "/" - VIRTUAL_HOST: rabbitmq.gitmaya.com - VIRTUAL_PORT: 15672 - ports: - - "15672" - - "5672" volumes: - - ./data/rabbitmq:/data/mnesia + - ./data/redis:/data + command: redis-server --save 20 1 --loglevel warning mysql: restart: always @@ -39,8 +33,8 @@ services: - ./data/mysql/data:/var/lib/mysql - ./data/mysql/conf.d:/etc/mysql/conf.d environment: - MYSQL_ROOT_PASSWORD: 'gitmaya' - MYSQL_DATABASE: 'gitmaya2023' + MYSQL_ROOT_PASSWORD: 'gitmaya2023' + MYSQL_DATABASE: 'gitmaya' TZ: 'Asia/Shanghai' ports: - "3306" diff --git a/pdm.lock b/pdm.lock index 2e649f45..65979d9a 100644 --- a/pdm.lock +++ b/pdm.lock @@ -4,8 +4,21 @@ [metadata] groups = ["default"] strategy = ["cross_platform"] -lock_version = "4.4.1" -content_hash = "sha256:c1a5d1e22693f74c3174d55040fa95f5517b735391c8b4c2eced81269267fad9" +lock_version = "4.4" +content_hash = "sha256:6aad74a90f0ad95144d877dcf48c872ce82df2a6ddbe42d04e67585c0734c5f8" + +[[package]] +name = "amqp" +version = "5.2.0" +requires_python = ">=3.6" +summary = "Low-level AMQP client for Python (fork of amqplib)." +dependencies = [ + "vine<6.0.0,>=5.0.0", +] +files = [ + {file = "amqp-5.2.0-py3-none-any.whl", hash = "sha256:827cb12fb0baa892aad844fd95258143bce4027fdac4fccddbc43330fd281637"}, + {file = "amqp-5.2.0.tar.gz", hash = "sha256:a1ecff425ad063ad42a486c902807d1482311481c8ad95a72694b2975e75f7fd"}, +] [[package]] name = "anyio" @@ -23,6 +36,26 @@ files = [ {file = "anyio-4.2.0.tar.gz", hash = "sha256:e1875bb4b4e2de1669f4bc7869b6d3f54231cdced71605e6e64c9be77e3be50f"}, ] +[[package]] +name = "async-timeout" +version = "4.0.3" +requires_python = ">=3.7" +summary = "Timeout context manager for asyncio programs" +files = [ + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, +] + +[[package]] +name = "billiard" +version = "4.2.0" +requires_python = ">=3.7" +summary = "Python multiprocessing fork with improvements and bugfixes" +files = [ + {file = "billiard-4.2.0-py3-none-any.whl", hash = "sha256:07aa978b308f334ff8282bd4a746e681b3513db5c9a514cbdd810cbbdc19714d"}, + {file = "billiard-4.2.0.tar.gz", hash = "sha256:9a3c3184cb275aa17a732f93f65b20c525d3d9f253722d26a82194803ade5a2c"}, +] + [[package]] name = "blinker" version = "1.7.0" @@ -60,7 +93,7 @@ files = [ [[package]] name = "ca-lark-sdk" -version = "0.0.7" +version = "0.0.8" requires_python = ">=3.8" summary = "lark(feishu) client" dependencies = [ @@ -68,12 +101,12 @@ dependencies = [ "pycryptodome", ] files = [ - {file = "ca-lark-sdk-0.0.7.tar.gz", hash = "sha256:07374e6cfeb97d96aed1358becc09529a66e292d50a94bb3b904306c7d2b6c93"}, + {file = "ca-lark-sdk-0.0.8.tar.gz", hash = "sha256:ee04d245a66d0174c28318e6dcd4d074a8c11eed96e224105d05dedb2db8b557"}, ] [[package]] name = "ca-lark-webhook" -version = "0.0.3" +version = "0.0.4" requires_python = ">=3.8" summary = "lark(feishu) client" dependencies = [ @@ -81,7 +114,28 @@ dependencies = [ "flask", ] files = [ - {file = "ca-lark-webhook-0.0.3.tar.gz", hash = "sha256:79d6da1ab96b207b542b8a20eabb0d8dc6563f87b84948d468183337f543cf22"}, + {file = "ca-lark-webhook-0.0.4.tar.gz", hash = "sha256:1ce7d6f92e61a12e434dada2fb12a2d6a18420576caa72db82afbbfaaa7ca3ce"}, +] + +[[package]] +name = "celery" +version = "5.3.6" +requires_python = ">=3.8" +summary = "Distributed Task Queue." +dependencies = [ + "billiard<5.0,>=4.2.0", + "click-didyoumean>=0.3.0", + "click-plugins>=1.1.1", + "click-repl>=0.2.0", + "click<9.0,>=8.1.2", + "kombu<6.0,>=5.3.4", + "python-dateutil>=2.8.2", + "tzdata>=2022.7", + "vine<6.0,>=5.1.0", +] +files = [ + {file = "celery-5.3.6-py3-none-any.whl", hash = "sha256:9da4ea0118d232ce97dff5ed4974587fb1c0ff5c10042eb15278487cdd27d1af"}, + {file = "celery-5.3.6.tar.gz", hash = "sha256:870cc71d737c0200c397290d730344cc991d13a057534353d124c9380267aab9"}, ] [[package]] @@ -151,6 +205,45 @@ files = [ {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] +[[package]] +name = "click-didyoumean" +version = "0.3.0" +requires_python = ">=3.6.2,<4.0.0" +summary = "Enables git-like *did-you-mean* feature in click" +dependencies = [ + "click>=7", +] +files = [ + {file = "click-didyoumean-0.3.0.tar.gz", hash = "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035"}, + {file = "click_didyoumean-0.3.0-py3-none-any.whl", hash = "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667"}, +] + +[[package]] +name = "click-plugins" +version = "1.1.1" +summary = "An extension module for click to enable registering CLI commands via setuptools entry-points." +dependencies = [ + "click>=4.0", +] +files = [ + {file = "click-plugins-1.1.1.tar.gz", hash = "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b"}, + {file = "click_plugins-1.1.1-py2.py3-none-any.whl", hash = "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8"}, +] + +[[package]] +name = "click-repl" +version = "0.3.0" +requires_python = ">=3.6" +summary = "REPL plugin for Click" +dependencies = [ + "click>=7.0", + "prompt-toolkit>=3.0.36", +] +files = [ + {file = "click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9"}, + {file = "click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812"}, +] + [[package]] name = "colorama" version = "0.4.6" @@ -370,6 +463,20 @@ files = [ {file = "jwt-1.3.1-py3-none-any.whl", hash = "sha256:61c9170f92e736b530655e75374681d4fcca9cfa8763ab42be57353b2b203494"}, ] +[[package]] +name = "kombu" +version = "5.3.4" +requires_python = ">=3.8" +summary = "Messaging library for Python." +dependencies = [ + "amqp<6.0.0,>=5.1.1", + "vine", +] +files = [ + {file = "kombu-5.3.4-py3-none-any.whl", hash = "sha256:63bb093fc9bb80cfb3a0972336a5cec1fa7ac5f9ef7e8237c6bf8dda9469313e"}, + {file = "kombu-5.3.4.tar.gz", hash = "sha256:0bb2e278644d11dea6272c17974a3dbb9688a949f3bb60aeb5b791329c44fadc"}, +] + [[package]] name = "markupsafe" version = "2.1.3" @@ -409,6 +516,19 @@ files = [ {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, ] +[[package]] +name = "prompt-toolkit" +version = "3.0.43" +requires_python = ">=3.7.0" +summary = "Library for building powerful interactive command lines in Python" +dependencies = [ + "wcwidth", +] +files = [ + {file = "prompt_toolkit-3.0.43-py3-none-any.whl", hash = "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"}, + {file = "prompt_toolkit-3.0.43.tar.gz", hash = "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d"}, +] + [[package]] name = "pycparser" version = "2.21" @@ -481,6 +601,19 @@ files = [ {file = "python_dotenv-1.0.0-py3-none-any.whl", hash = "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"}, ] +[[package]] +name = "redis" +version = "5.0.1" +requires_python = ">=3.7" +summary = "Python client for Redis database and key-value store" +dependencies = [ + "async-timeout>=4.0.2; python_full_version <= \"3.11.2\"", +] +files = [ + {file = "redis-5.0.1-py3-none-any.whl", hash = "sha256:ed4802971884ae19d640775ba3b03aa2e7bd5e8fb8dfaed2decce4d0fc48391f"}, + {file = "redis-5.0.1.tar.gz", hash = "sha256:0dab495cd5753069d3bc650a0dde8a8f9edde16fc5691b689a566eda58100d0f"}, +] + [[package]] name = "six" version = "1.16.0" @@ -507,7 +640,7 @@ version = "2.0.23" requires_python = ">=3.7" summary = "Database Abstraction Library" dependencies = [ - "greenlet!=0.4.17; platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\"", + "greenlet!=0.4.17; platform_machine == \"aarch64\" or (platform_machine == \"ppc64le\" or (platform_machine == \"x86_64\" or (platform_machine == \"amd64\" or (platform_machine == \"AMD64\" or (platform_machine == \"win32\" or platform_machine == \"WIN32\")))))", "typing-extensions>=4.2.0", ] files = [ @@ -549,6 +682,16 @@ files = [ {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, ] +[[package]] +name = "tzdata" +version = "2023.3" +requires_python = ">=2" +summary = "Provider of IANA time zone data" +files = [ + {file = "tzdata-2023.3-py2.py3-none-any.whl", hash = "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"}, + {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, +] + [[package]] name = "urllib3" version = "2.1.0" @@ -559,6 +702,25 @@ files = [ {file = "urllib3-2.1.0.tar.gz", hash = "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54"}, ] +[[package]] +name = "vine" +version = "5.1.0" +requires_python = ">=3.6" +summary = "Python promises." +files = [ + {file = "vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc"}, + {file = "vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0"}, +] + +[[package]] +name = "wcwidth" +version = "0.2.12" +summary = "Measures the displayed width of unicode strings in a terminal" +files = [ + {file = "wcwidth-0.2.12-py2.py3-none-any.whl", hash = "sha256:f26ec43d96c8cbfed76a5075dac87680124fa84e0855195a6184da9c187f133c"}, + {file = "wcwidth-0.2.12.tar.gz", hash = "sha256:f01c104efdf57971bcb756f054dd58ddec5204dd15fa31d6503ea57947d97c02"}, +] + [[package]] name = "werkzeug" version = "3.0.1" diff --git a/pyproject.toml b/pyproject.toml index 341843a4..52ec8545 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ authors = [ dependencies = [ "python-dotenv>=1.0.0", "ca-lark-oauth==0.0.5", - "ca-lark-webhook>=0.0.3", + "ca-lark-webhook==0.0.4", "flask-sqlalchemy>=3.1.1", "flask-cors>=4.0.0", "pymysql>=1.1.0", @@ -16,6 +16,9 @@ dependencies = [ "bson>=0.5.10", "jwt>=1.3.1", "urllib3>=2.1.0", + "ca-lark-sdk>=0.0.8", + "celery>=5.3.6", + "redis>=5.0.1", ] requires-python = ">=3.10" readme = "README.md" diff --git a/requirements.txt b/requirements.txt index acb280ed..97cff15e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,22 @@ # This file is @generated by PDM. # Please do not edit it manually. +amqp==5.2.0 anyio==4.2.0 +async-timeout==4.0.3; python_full_version <= "3.11.2" +billiard==4.2.0 blinker==1.7.0 bson==0.5.10 ca-lark-oauth==0.0.5 -ca-lark-sdk==0.0.7 -ca-lark-webhook==0.0.3 +ca-lark-sdk==0.0.8 +ca-lark-webhook==0.0.4 +celery==5.3.6 certifi==2023.11.17 cffi==1.16.0 click==8.1.7 +click-didyoumean==0.3.0 +click-plugins==1.1.1 +click-repl==0.3.0 colorama==0.4.6; platform_system == "Windows" cryptography==41.0.7 exceptiongroup==1.2.0; python_version < "3.11" @@ -24,15 +31,21 @@ idna==3.6 itsdangerous==2.1.2 Jinja2==3.1.2 jwt==1.3.1 +kombu==5.3.4 MarkupSafe==2.1.3 +prompt-toolkit==3.0.43 pycparser==2.21 pycryptodome==3.19.1 pymysql==1.1.0 python-dateutil==2.8.2 python-dotenv==1.0.0 +redis==5.0.1 six==1.16.0 sniffio==1.3.0 sqlalchemy==2.0.23 typing-extensions==4.9.0 +tzdata==2023.3 urllib3==2.1.0 +vine==5.1.0 +wcwidth==0.2.12 Werkzeug==3.0.1 diff --git a/server/celery_app.py b/server/celery_app.py new file mode 100644 index 00000000..5b6f046f --- /dev/null +++ b/server/celery_app.py @@ -0,0 +1,25 @@ +import env +from app import app +from celery import Celery + +app.config.setdefault("CELERY_BROKER_URL", "redis://redis:6379/0") +app.config.setdefault("CELERY_RESULT_BACKEND", "redis://redis:6379/0") +celery = Celery( + app.import_name, + broker=app.config["CELERY_BROKER_URL"], + backend=app.config["CELERY_RESULT_BACKEND"], +) + +celery.conf.update(app.config.get("CELERY_CONFIG", {})) +TaskBase = celery.Task + + +class ContextTask(TaskBase): + abstract = True + + def __call__(self, *args, **kwargs): + with app.app_context(): + return TaskBase.__call__(self, *args, **kwargs) + + +celery.Task = ContextTask diff --git a/server/command/lark.py b/server/command/lark.py new file mode 100644 index 00000000..c99754f7 --- /dev/null +++ b/server/command/lark.py @@ -0,0 +1,72 @@ +import logging + +import click +from app import app, db +from model.schema import IMApplication, ObjID + + +# create command function +@app.cli.command(name="larkapp") +@click.option("-a", "--app-id", "app_id", required=True, prompt="Feishu(Lark) APP ID") +@click.option( + "-s", "--app-secret", "app_secret", required=True, prompt="Feishu(Lark) APP SECRET" +) +@click.option( + "-e", "--encrypt-key", "encrypt_key", default="", prompt="Feishu(Lark) ENCRYPT KEY" +) +@click.option( + "-v", + "--verification-token", + "verification_token", + default="", + prompt="Feishu(Lark) VERIFICATION TOKEN", +) +@click.option("-h", "--host", "host", default="https://testapi.gitmaya.com") +def create_lark_app(app_id, app_secret, encrypt_key, verification_token, host): + # click.echo(f'create_lark_app {app_id} {app_secret} {encrypt_key} {verification_token} {host}') + permissions = "\n\t".join( + [ + "contact:contact:readonly_as_app", + "im:chat", + "im:message", + "im:resource", + ] + ) + events = "\n\t".join( + [ + "im.message.receive_v1", + ] + ) + click.echo(f"need permissions: \n{permissions}\n") + click.echo(f"need events: \n{events}\n") + click.echo(f"webhook: \n{host}/api/feishu/oauth") + + application = ( + db.session.query(IMApplication).filter(IMApplication.app_id == app_id).first() + ) + if not application: + application = IMApplication( + id=ObjID.new_id(), + app_id=app_id, + app_secret=app_secret, + extra=dict(encrypt_key=encrypt_key, verification_token=verification_token), + ) + db.session.add(application) + db.session.commit() + else: + db.session.query(IMApplication).filter( + IMApplication.id == application.id, + ).update( + dict( + app_id=app_id, + app_secret=app_secret, + extra=dict( + encrypt_key=encrypt_key, verification_token=verification_token + ), + ) + ) + db.session.commit() + click.echo(f"webhook: \n{host}/api/feishu/hook/{app_id}") + + click.confirm("success to save feishu app?", abort=True) + click.echo(f"you can publish you app.") diff --git a/server/model/lark.py b/server/model/lark.py new file mode 100644 index 00000000..86e1b092 --- /dev/null +++ b/server/model/lark.py @@ -0,0 +1,11 @@ +from .schema import IMApplication, db + + +def get_bot_by_app_id(app_id): + return ( + db.session.query(IMApplication) + .filter( + IMApplication.app_id == app_id, + ) + .first() + ) diff --git a/server/model/schema.py b/server/model/schema.py index cf572d93..70ec87b1 100644 --- a/server/model/schema.py +++ b/server/model/schema.py @@ -22,7 +22,7 @@ def processor(value): def result_processor(self, dialect, coltype): def processor(value): - if not isinstance(value, bytes): + if value and not isinstance(value, bytes): value = bytes(value) return str(bson.ObjectId(value)) if bson.ObjectId.is_valid(value) else value @@ -77,9 +77,20 @@ class Base(db.Model): __abstract__ = True id = db.Column(ObjID(12), primary_key=True) status = db.Column(db.Integer, nullable=True, default=0, server_default=text("0")) - created = db.Column(db.TIMESTAMP, nullable=False, default=datetime.utcnow) + created = db.Column( + db.TIMESTAMP, + nullable=False, + default=datetime.utcnow, + server_default=text("CURRENT_TIMESTAMP"), + comment="创建时间", + ) modified = db.Column( - db.TIMESTAMP, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow + db.TIMESTAMP, + nullable=False, + default=datetime.utcnow, + onupdate=datetime.utcnow, + server_default=text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"), + comment="修改时间", ) @@ -101,7 +112,7 @@ class Account(User): class BindUser(Base): - __tablename__ = "bint_user" + __tablename__ = "bind_user" user_id = db.Column(ObjID(12), ForeignKey("user.id"), nullable=True, comment="用户ID") # 这里如果是飞书租户，可能会有不同的name等，但是在github这边不管是哪一个org，都是一样的 # 这里如何统一？ diff --git a/server/routes/__init__.py b/server/routes/__init__.py index 25a22184..a7105778 100644 --- a/server/routes/__init__.py +++ b/server/routes/__init__.py @@ -1,2 +1,3 @@ from .github import * from .lark import * +from .team import * diff --git a/server/routes/lark.py b/server/routes/lark.py index 2a699e57..c1664e69 100644 --- a/server/routes/lark.py +++ b/server/routes/lark.py @@ -1,46 +1,51 @@ +import logging import os from app import app, session from connectai.lark.oauth import Server as OauthServerBase from connectai.lark.sdk import Bot, MarketBot from connectai.lark.webhook import LarkServer as LarkServerBase +from model.lark import get_bot_by_app_id -bot = Bot( - app_id=os.environ.get("APP_ID"), - app_secret=os.environ.get("APP_SECRET"), - encrypt_key=os.environ.get("ENCRYPT_KEY"), - verification_token=os.environ.get("VERIFICATION_TOKEN"), -) + +def get_bot(app_id): + with app.app_context(): + application = get_bot_by_app_id(app_id) + if application: + return Bot( + app_id=application.app_id, + app_secret=application.app_secret, + encrypt_key=application.extra.get("encrypt_key"), + verification_token=application.extra.get("verification_token"), + ) class LarkServer(LarkServerBase): def get_bot(self, app_id): - # TODO search in database and create Bot() - return bot + return get_bot(app_id) class OauthServer(OauthServerBase): def get_bot(self, app_id): - # TODO search in database and create Bot() - return bot + return get_bot(app_id) hook = LarkServer(prefix="/api/feishu/hook") oauth = OauthServer(prefix="/api/feishu/oauth") -@hook.on_bot_message(message_type="text", bot=bot) +@hook.on_bot_message(message_type="text") def on_text_message(bot, message_id, content, *args, **kwargs): text = content["text"] print("reply_text", message_id, text) bot.reply_text(message_id, "reply: " + text) -@oauth.on_bot_event(event_type="oauth:user_info", bot=bot) +@oauth.on_bot_event(event_type="oauth:user_info") def on_oauth_user_info(bot, event_id, user_info, *args, **kwargs): # oauth user_info print("oauth", user_info) - # TODO + # TODO save bind user session["user_id"] = user_info["union_id"] session.permanent = True return user_info diff --git a/server/routes/team.py b/server/routes/team.py new file mode 100644 index 00000000..f32ed6da --- /dev/null +++ b/server/routes/team.py @@ -0,0 +1,20 @@ +import logging + +from app import app +from flask import Blueprint, abort, jsonify, redirect, request +from utils.auth import authenticated + +bp = Blueprint("team", __name__, url_prefix="/api/team") + + +@bp.route("/", methods=["GET"]) +@authenticated +def get_team_list(): + """ + get team list + TODO + """ + return jsonify({"code": 0, "msg": "success", "data": [], "total": 0}) + + +app.register_blueprint(bp) diff --git a/server/tasks/__init__.py b/server/tasks/__init__.py new file mode 100644 index 00000000..5b2395dd --- /dev/null +++ b/server/tasks/__init__.py @@ -0,0 +1,11 @@ +from celery_app import celery + +from .lark import test_task + +celery.conf.beat_schedule = { + # "test_crontab_task": { + # "task": "tasks.crontab_task", + # "schedule": timedelta(hours=24), # 定时24hours执行一次 + # "args": (False) # 函数传参的值 + # }, +} diff --git a/server/tasks/lark.py b/server/tasks/lark.py new file mode 100644 index 00000000..606ca3e4 --- /dev/null +++ b/server/tasks/lark.py @@ -0,0 +1,9 @@ +import logging + +from celery_app import celery + + +@celery.task() +def test_task(): + logging.error("test_task") + return 1 diff --git a/server/utils/auth.py b/server/utils/auth.py new file mode 100644 index 00000000..e008d6e5 --- /dev/null +++ b/server/utils/auth.py @@ -0,0 +1,13 @@ +from functools import wraps + +from flask import abort, session + + +def authenticated(func): + @wraps(func) + def wrapper(*args, **kwargs): + if "user_id" not in session: + return abort(401) + return func(*args, **kwargs) + + return wrapper From 3cb3a2f7ae76dcf84a3da44e98521047c6253cba Mon Sep 17 00:00:00 2001 From: jingfelix Date: Fri, 29 Dec 2023 15:59:30 +0800 Subject: [PATCH 5/7] feat: enable GitHub Webhook Signature Signed-off-by: jingfelix --- server/routes/github.py | 20 ++++++++++++++++- server/utils/github.py | 50 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/server/routes/github.py b/server/routes/github.py index 156fd306..23674348 100644 --- a/server/routes/github.py +++ b/server/routes/github.py @@ -3,7 +3,12 @@ from app import app from flask import Blueprint, abort, redirect, request -from utils.github import get_installation_token, get_jwt, register_by_code +from utils.github import ( + get_installation_token, + get_jwt, + register_by_code, + verify_github_signature, +) bp = Blueprint("github", __name__, url_prefix="/api/github") @@ -76,4 +81,17 @@ def github_register(): return user_token +@bp.route("/hook", methods=["POST"]) +@verify_github_signature(os.environ.get("GITHUB_WEBHOOK_SECRET")) +def github_hook(): + """Receive GitHub webhook.""" + + x_github_event = request.headers.get("x-github-event") + logging.info(x_github_event) + + logging.debug(request.json) + + return "Receive Success!" + + app.register_blueprint(bp) diff --git a/server/utils/github.py b/server/utils/github.py index 044c6b15..f816abb1 100644 --- a/server/utils/github.py +++ b/server/utils/github.py @@ -1,8 +1,13 @@ +import hashlib +import hmac +import logging import os import time +from functools import wraps from urllib.parse import parse_qs import httpx +from flask import abort, request from jwt import JWT, jwk_from_pem @@ -57,7 +62,8 @@ def get_installation_token(jwt: str, installation_id: str) -> str | None: "X-GitHub-Api-Version": "2022-11-28", }, ) - if response.status_code == 200: + if response.status_code != 200: + logging.debug(f"Failed to get installation token. {response.text}") return None installation_token = response.json().get("token", None) @@ -93,3 +99,45 @@ def register_by_code(code: str) -> str | None: return access_token[0] return None + + +def verify_github_signature( + secret: str = os.environ.get("GITHUB_WEBHOOK_SECRET", "secret") +): + """Decorator to verify the signature of a GitHub webhook request. + + Args: + secret (str): The secret key used to sign the webhook request. + + Returns: + function: The decorated function. + """ + + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + signature = request.headers.get("x-hub-signature-256") + if not signature: + abort(400, "No signature provided.") + + # Verify the signature + body = request.get_data() + + hash_object = hmac.new( + secret.encode("utf-8"), + msg=body, + digestmod=hashlib.sha256, + ) + expected_signature = "sha256=" + hash_object.hexdigest() + + logging.debug(f"{expected_signature} {signature}") + + if not hmac.compare_digest(expected_signature, signature): + logging.debug("Invalid signature.") + abort(403, "Invalid signature.") + + return func(*args, **kwargs) + + return wrapper + + return decorator From eb77b4b05ab4076f6be8d0ce970fa8763aa20b78 Mon Sep 17 00:00:00 2001 From: jingfelix Date: Fri, 29 Dec 2023 22:45:53 +0800 Subject: [PATCH 6/7] feat: support GitHub login and authenticated Signed-off-by: jingfelix --- server/routes/github.py | 35 ++++++++++-------------- server/utils/github/__init__.py | 0 server/utils/{ => github}/github.py | 40 ++++++++++++++++++++++++---- server/utils/user.py | 41 +++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 26 deletions(-) create mode 100644 server/utils/github/__init__.py rename server/utils/{ => github}/github.py (80%) create mode 100644 server/utils/user.py diff --git a/server/routes/github.py b/server/routes/github.py index 23674348..e981e0e3 100644 --- a/server/routes/github.py +++ b/server/routes/github.py @@ -2,18 +2,21 @@ import os from app import app -from flask import Blueprint, abort, redirect, request -from utils.github import ( +from flask import Blueprint, abort, redirect, request, session +from utils.auth import authenticated + +from server.utils.github.github import ( get_installation_token, get_jwt, - register_by_code, verify_github_signature, ) +from server.utils.user import register bp = Blueprint("github", __name__, url_prefix="/api/github") @bp.route("/install", methods=["GET"]) +@authenticated def github_install(): """Install GitHub App. @@ -44,18 +47,6 @@ def github_install(): abort(500) logging.debug(f"installation_token: {installation_token}") - # 如果有 code 参数，则为该用户注册 - code = request.args.get("code", None) - if code is not None: - logging.debug(f"code: {code}") - - user_token = register_by_code(code) - if user_token is None: - logging.debug("Failed to register by code.") - abort(500) - - logging.debug(f"user_token: {user_token}") - return "Success!" @@ -72,13 +63,14 @@ def github_register(): f"https://github.com/login/oauth/authorize?client_id={os.environ.get('GITHUB_CLIENT_ID')}" ) - logging.debug(f"code: {code}") - user_token = register_by_code(code) - if user_token is None: + user_id = register(code) + if user_id is None: return "Failed to register by code." - logging.debug(f"user_token: {user_token}") - return user_token + session["user_id"] = user_id + + # TODO: 统一的返回格式 + return "Success!" @bp.route("/hook", methods=["POST"]) @@ -86,7 +78,8 @@ def github_register(): def github_hook(): """Receive GitHub webhook.""" - x_github_event = request.headers.get("x-github-event") + x_github_event = request.headers.get("x-github-event", None) + logging.info(x_github_event) logging.debug(request.json) diff --git a/server/utils/github/__init__.py b/server/utils/github/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/utils/github.py b/server/utils/github/github.py similarity index 80% rename from server/utils/github.py rename to server/utils/github/github.py index f816abb1..4b3e811f 100644 --- a/server/utils/github.py +++ b/server/utils/github/github.py @@ -72,7 +72,7 @@ def get_installation_token(jwt: str, installation_id: str) -> str | None: return None -def register_by_code(code: str) -> str | None: +def oauth_by_code(code: str) -> dict | None: """Register by code Args: @@ -94,11 +94,13 @@ def register_by_code(code: str) -> str | None: if response.status_code != 200: return None - access_token = parse_qs(response.text).get("access_token", None) - if access_token is not None: - return access_token[0] + try: + oauth_info = parse_qs(response.text) + except Exception as e: + logging.debug(e) + return None - return None + return oauth_info def verify_github_signature( @@ -141,3 +143,31 @@ def wrapper(*args, **kwargs): return wrapper return decorator + + +def get_user_info(access_token: str): + """Get user info by access token. + + Args: + access_token (str): The user access token. + + Returns: + dict: User info. + """ + + with httpx.Client() as client: + response = client.get( + "https://api.github.com/user", + headers={ + "Accept": "application/vnd.github.v3+json", + "Authorization": f"token {access_token}", + }, + ) + if response.status_code != 200: + logging.debug(f"Failed to get user info. {response.text}") + return None + + user_info = response.json() + return user_info + + return None diff --git a/server/utils/user.py b/server/utils/user.py new file mode 100644 index 00000000..1c1c5c85 --- /dev/null +++ b/server/utils/user.py @@ -0,0 +1,41 @@ +from app import app, db +from model.schema import BindUser, User + +from server.utils.github.github import oauth_by_code + + +def register(code: str) -> str | None: + """GitHub OAuth register. + + If not `code`, redirect to GitHub OAuth page. + If `code`, register by code. + """ + + oauth_info = oauth_by_code(code) + + user_info = oauth_info.get("user", None) + + # 使用 oauth_info 中的 access_token 获取用户信息 + + new_user = User( + email=user_info.get("email", None), + name=user_info.get("login", None), # TODO: 确认一下 login 和 name 哪个是唯一的 + avatar=user_info.get("avatar_url", None), + extra=user_info, + ) + + db.session.add(new_user) + + new_bind_user = BindUser( + user_id=new_user.id, + platform="github", + email=user_info.get("email", None), + avatar=user_info.get("avatar_url", None), + extra=oauth_info, + ) + + db.session.add(new_bind_user) + + db.session.commit() + + return new_user.id From 30932cacf6f2d797ce0bbdac7de57b1be47d3a27 Mon Sep 17 00:00:00 2001 From: jingfelix Date: Tue, 2 Jan 2024 17:00:50 +0800 Subject: [PATCH 7/7] feat: GitHub Register (basic) Signed-off-by: jingfelix --- server/model/schema.py | 3 +- server/routes/github.py | 37 ++++---------------- server/utils/github/{github.py => common.py} | 0 server/utils/user.py | 28 ++++++++++----- 4 files changed, 29 insertions(+), 39 deletions(-) rename server/utils/github/{github.py => common.py} (100%) diff --git a/server/model/schema.py b/server/model/schema.py index 70ec87b1..ae3cd27f 100644 --- a/server/model/schema.py +++ b/server/model/schema.py @@ -96,12 +96,13 @@ class Base(db.Model): class User(Base): __tablename__ = "user" + github_id = db.Column(db.String(128), nullable=True, comment="GitHub ID, 作为唯一标识") email = db.Column(db.String(128), nullable=True, comment="邮箱,这里考虑一下如何做唯一的用户") telephone = db.Column(db.String(128), nullable=True, comment="手机号") name = db.Column(db.String(128), nullable=True, comment="用户名") avatar = db.Column(db.String(128), nullable=True, comment="头像") extra = db.Column( - JSONStr(1024), nullable=True, server_default=text("'{}'"), comment="用户其他字段" + JSONStr(2048), nullable=True, server_default=text("'{}'"), comment="用户其他字段" ) diff --git a/server/routes/github.py b/server/routes/github.py index e981e0e3..446d85ff 100644 --- a/server/routes/github.py +++ b/server/routes/github.py @@ -3,14 +3,10 @@ from app import app from flask import Blueprint, abort, redirect, request, session +from model.schema import Team from utils.auth import authenticated - -from server.utils.github.github import ( - get_installation_token, - get_jwt, - verify_github_signature, -) -from server.utils.user import register +from utils.github.common import get_installation_token, get_jwt, verify_github_signature +from utils.user import register bp = Blueprint("github", __name__, url_prefix="/api/github") @@ -20,35 +16,14 @@ def github_install(): """Install GitHub App. - If not `installation_id`, redirect to install page. - If `installation_id`, get installation token. - - If `code`, register by code. + Redirect to GitHub App installation page. """ installation_id = request.args.get("installation_id", None) - if installation_id is None: return redirect( f"https://github.com/apps/{os.environ.get('GITHUB_APP_NAME')}/installations/new" ) - logging.debug(f"installation_id: {installation_id}") - - jwt = get_jwt( - os.environ.get("GITHUB_APP_PRIVATE_KEY_PATH"), - os.environ.get("GITHUB_APP_ID"), - ) - - installation_token = get_installation_token(jwt, installation_id) - if installation_token is None: - logging.debug("Failed to get installation token.") - - # TODO: 统一解决各类 http 请求失败的情况 - abort(500) - logging.debug(f"installation_token: {installation_token}") - - return "Success!" - @bp.route("/register", methods=["GET"]) def github_register(): @@ -58,18 +33,20 @@ def github_register(): If `code`, register by code. """ code = request.args.get("code", None) + if code is None: return redirect( f"https://github.com/login/oauth/authorize?client_id={os.environ.get('GITHUB_CLIENT_ID')}" ) + # 通过 code 注册；如果 user 已经存在，则一样会返回 user_id user_id = register(code) if user_id is None: return "Failed to register by code." + # 保存用户注册状态 session["user_id"] = user_id - # TODO: 统一的返回格式 return "Success!" diff --git a/server/utils/github/github.py b/server/utils/github/common.py similarity index 100% rename from server/utils/github/github.py rename to server/utils/github/common.py diff --git a/server/utils/user.py b/server/utils/user.py index 1c1c5c85..36ff98fa 100644 --- a/server/utils/user.py +++ b/server/utils/user.py @@ -1,32 +1,44 @@ from app import app, db -from model.schema import BindUser, User - -from server.utils.github.github import oauth_by_code +from model.schema import BindUser, ObjID, User +from utils.github.common import get_user_info, oauth_by_code def register(code: str) -> str | None: """GitHub OAuth register. - If not `code`, redirect to GitHub OAuth page. If `code`, register by code. """ - oauth_info = oauth_by_code(code) + oauth_info = oauth_by_code(code) # 获取 access token - user_info = oauth_info.get("user", None) + access_token = oauth_info.get("access_token", None)[0] # 这里要考虑取哪个，为什么会有多个？ # 使用 oauth_info 中的 access_token 获取用户信息 + user_info = get_user_info(access_token) + + # 查询 github_id 是否已经存在，若存在，则返回 user_id + github_id = user_info.get("id", None) + if github_id is not None: + user = User.query.filter_by(github_id=github_id).first() + if user is not None: + return user.id new_user = User( - email=user_info.get("email", None), - name=user_info.get("login", None), # TODO: 确认一下 login 和 name 哪个是唯一的 + id=ObjID.new_id(), + github_id=github_id, + email=user_info.get( + "email", None + ), # 这里的邮箱其实是公开邮箱，可能会获取不到 TODO: 换成使用用户邮箱 API 来获取 + name=user_info.get("login", None), avatar=user_info.get("avatar_url", None), extra=user_info, ) db.session.add(new_user) + db.session.commit() new_bind_user = BindUser( + id=ObjID.new_id(), user_id=new_user.id, platform="github", email=user_info.get("email", None),