diff --git a/Pipfile b/Pipfile index 120f94c..717758c 100644 --- a/Pipfile +++ b/Pipfile @@ -17,6 +17,7 @@ ipaddress = "*" [dev-packages] ipython = "*" +isort = "*" [requires] python_version = "3.10" diff --git a/Pipfile.lock b/Pipfile.lock index eb89931..5e7d168 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6167afb3d520ca03f00b201f05f4b46c1b068c8f014f71f2c9eb418a306b28ed" + "sha256": "03b1210b10ff76be1bc7f09f04f2ecc2e95925a5d9347938d87263ab398d94e5" }, "pipfile-spec": 6, "requires": { @@ -360,7 +360,7 @@ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.2" }, "redis": { @@ -384,7 +384,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "smmap": { @@ -535,6 +535,14 @@ "index": "pypi", "version": "==8.5.0" }, + "isort": { + "hashes": [ + "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7", + "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951" + ], + "index": "pypi", + "version": "==5.10.1" + }, "jedi": { "hashes": [ "sha256:637c9635fcf47945ceb91cd7f320234a7be540ded6f3e99a50cb6febdfd1ba8d", @@ -609,7 +617,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "stack-data": { diff --git a/cache_json/admin.py b/cache_json/admin.py deleted file mode 100644 index 8c38f3f..0000000 --- a/cache_json/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/cache_json/models.py b/cache_json/models.py deleted file mode 100644 index d468d96..0000000 --- a/cache_json/models.py +++ /dev/null @@ -1,22 +0,0 @@ -from django.db import models - -# Create your models here. -class Word(models.Model): - word = models.CharField(max_length=255, unique=True) - is_checked = models.BooleanField(default=False) - class Meta: - ordering = ['word'] - - def get_translates(self) -> list[str]: - return [translate.translate for translate in self.translate_set.all().order_by('score')] - - def __str__(self) -> str: - return self.word - -class Translate(models.Model): - word = models.ForeignKey(Word, on_delete=models.CASCADE) - translate = models.CharField(max_length=255) - score = models.IntegerField(default=1) - - def __str__(self) -> str: - return f'{self.word}-{self.translate}' diff --git a/cache_json/views.py b/cache_json/views.py deleted file mode 100644 index 91ea44a..0000000 --- a/cache_json/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/github_manager/admin.py b/github_manager/admin.py index 6fcb822..980bf19 100644 --- a/github_manager/admin.py +++ b/github_manager/admin.py @@ -1,4 +1,7 @@ from django.contrib import admin -from .models import PullRequest + +from .models import Moderator, PullRequest + # Register your models here. -admin.site.register(PullRequest) \ No newline at end of file +admin.site.register(PullRequest) +admin.site.register(Moderator) diff --git a/github_manager/messages.json b/github_manager/messages.json new file mode 100644 index 0000000..6020ee7 --- /dev/null +++ b/github_manager/messages.json @@ -0,0 +1,8 @@ +{ + "pull_request": { + "title": "Change '{word}' Translation to '{translation}'", + "head": "'{word}' has reached the minimum translations limit\nThe New default Translation is: '{translation}'\n\nThese are the 5 most commoun translations:\n", + "other_translation": "{index}. '{translation}'", + "footer": "To change the default translation, write: /set-default NumberOfTranslation\nTo delete Specific translation, write: /delete NumberOfTranslation" + } +} \ No newline at end of file diff --git a/github_manager/migrations/0001_initial.py b/github_manager/migrations/0001_initial.py index 8737487..6201896 100644 --- a/github_manager/migrations/0001_initial.py +++ b/github_manager/migrations/0001_initial.py @@ -1,5 +1,6 @@ -# Generated by Django 4.1.1 on 2022-09-17 18:03 +# Generated by Django 4.1.1 on 2022-09-30 08:32 +import django.db.models.deletion from django.db import migrations, models @@ -7,9 +8,26 @@ class Migration(migrations.Migration): initial = True - dependencies = [] + dependencies = [ + ("translation", "0001_initial"), + ] operations = [ + migrations.CreateModel( + name="Moderator", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("username", models.CharField(max_length=255)), + ], + ), migrations.CreateModel( name="PullRequest", fields=[ @@ -22,7 +40,14 @@ class Migration(migrations.Migration): verbose_name="ID", ), ), - ("prid", models.IntegerField()), + ("number", models.IntegerField()), + ( + "word", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="translation.word", + ), + ), ], ), ] diff --git a/github_manager/models.py b/github_manager/models.py index 6c46082..bfd9992 100644 --- a/github_manager/models.py +++ b/github_manager/models.py @@ -1,5 +1,15 @@ from django.db import models + # Create your models here. class PullRequest(models.Model): - prid = models.IntegerField() + number = models.IntegerField() + word = models.ForeignKey("translation.Word", on_delete=models.CASCADE) + + +class Moderator(models.Model): + username = models.CharField(max_length=255) + + @classmethod + def get_all_moderators(cls) -> list[str]: + return [mod.username for mod in cls.objects.all()] diff --git a/github_manager/tasks.py b/github_manager/tasks.py index 3dc40f4..4a4f2f6 100644 --- a/github_manager/tasks.py +++ b/github_manager/tasks.py @@ -1,72 +1,233 @@ -from translate.models import Word as TWord, Translate as TTranslate -from .models import PullRequest -from github import Github +import itertools import json import os +import re from pathlib import Path + from django.conf import settings -from cache_json.models import Word, Translate +from django.db.models import Count +from github import Github +from translation.models import SourceTranslation, UserTranslation, Word + +from .models import Moderator, PullRequest account = Github(os.environ.get("GITHUB_ACCOUNT_TOKEN")) -dictionary_repo = account.get_repo(os.environ.get('JSON_REPO')) -local_json_file: Path = (settings.BASE_DIR / 'local_dictionary.json').resolve() +account_id = account.get_user().id +dictionary_repo = account.get_repo(os.environ.get("JSON_REPO")) +local_json_file: Path = (settings.BASE_DIR / "local_dictionary.json").resolve() if not local_json_file.exists(): - raise Exception("Please run 'python3 project/prepare-project.py' before run Torjoman for the first time") + raise Exception( + "Please run 'python3 project/prepare-project.py' before run Torjoman for the first time" + ) + def check_for_update_json(): - file = dictionary_repo.get_contents(os.environ.get('JSON_FILE')) - repo_file_json= json.loads(file.decoded_content) - old_file_json = json.loads(local_json_file.read_text()) - if repo_file_json == old_file_json: - return - else: - print('Files Are not the same') - with open(str(local_json_file), 'w') as f: - json.dump(repo_file_json, f, ensure_ascii=False, indent=2) - update_source(repo_file_json) - -def update_source(data: list[dict[str, str | list]]): - source_words = [] # store words to check if some words have been deleted - data_words = [] - - for word in data: - data_words.append(word['word'].strip()) - if word['is_checked']: - print('Skip word {} because it is checked'.format(word['name'])) - w = TWord.objects.get(word=word['word']) - w.delete() - continue - w: Word = Word.objects.get_or_create(word=word['word'])[0] - w.save() - for translate in word['translates']: - try: - Translate.objects.get(word=w, translate=translate) - except Translate.DoesNotExist: - print(f'new translation for word {w.word}') - t: Translate = Translate.objects.get_or_create(word=w, translate=translate)[0] - t.save() - - tw: TWord = TWord.objects.get_or_create(word=w.word)[0] - [t.delete() for t in tw.translate_set.all()] - [t.delete() for t in tw.prs.all()] - tt: TTranslate = TTranslate(word=tw, translate=translate) - tt.save() - source_words = [w.word for w in Word.objects.all()] - diff = list(set(data_words) - set(source_words)) - for word in diff: - w = Word.objects.get(word=word) - w.delete() - w = TWord.objects.get(word=word) - w.delete() - -def push(_payload: dict): - print('New Push Event') - check_for_update_json() - print('Finished Push Event') - - -if local_json_file.read_text() == '{}': - check_for_update_json() - print('The first data extraction process has been completed') \ No newline at end of file + file = dictionary_repo.get_contents(os.environ.get("JSON_FILE")) + server = json.loads(file.decoded_content) + local = json.loads(local_json_file.read_text()) + if server == local: + print("There is no change") + return + else: + print("Files Are not the same") + update_source(local, server) + with open(str(local_json_file), "w") as f: + json.dump(server, f, ensure_ascii=False, indent=2) + + +def generate_dict_from_db(word: Word) -> list[dict]: + return [ + { + "word": w.word, + "translations": w.get_users_translations + if w.word == word.word + else w.get_source_translations, + "is_checked": w.is_checked, + } + for w in Word.objects.all() + ] + + +def update_source(local: list[dict], server: list[dict]): + rlocal = list(itertools.filterfalse(lambda x: x in local, server)) + rserver = list(itertools.filterfalse(lambda x: x in server, local)) + deleted_words: list[str] = [ + x["word"] for x in rserver if x and x["word"] not in [i["word"] for i in rlocal] + ] + for word in deleted_words: + w: Word = Word.objects.get(word=word) + w.delete() + for word in rlocal: + w: Word = Word.objects.get_or_create(word=word["word"])[0] + if word["translations"]: + for translation in word["translations"]: + st: SourceTranslation = SourceTranslation.objects.get_or_create( + word=w, translation=translation + )[0] + st.save() + # Reset users translations + for translation in w.usertranslation_set.all(): + translation.delete() + deleted_translations = [ + SourceTranslation.objects.get(word=w, translation=t) + for t in w.get_source_translations + if t not in word["translations"] + ] + for translation in deleted_translations: + translation.delete() + + +def handle_push(_payload: dict): + print("New Push Event") + check_for_update_json() + print("Finished Push Event") + + +def handle_pull_request(payload: dict): + # Verify whether the request is a pull request and if its owner is our account + if issue := payload.get("issue"): + if not issue.get("pull_request"): + pass + elif pr := payload.get("pull_request"): + if pr["user"]["id"] != account_id: + return + match payload["action"]: + case "closed": + PullRequest.objects.get(number=payload["number"]).delete() + word: str = re.search(r"'(?P.+)'\s", payload["issue"]["title"]).group( + "word" + ) + [ + ref + for ref in dictionary_repo.get_git_refs() + if ref.ref == f"refs/heads/Torjoman-{word.word}" + ][0].delete() + + +def handle_pr_comments(payload): + number = payload["issue"]["number"] + if not payload["comment"]["user"]["login"] in Moderator.get_all_moderators(): + return + r = re.search( + r"^/(?P.+) (?P\d+)$", payload["comment"]["body"] + ) + word: str = re.search(r"^'(?P.+)'\s", payload["issue"]["body"]).group("word") + translations: dict[str, str] = { + i: t + for i, t in re.findall( + r"(?P\d+)\. '(?P.+?)'", payload["issue"]["body"] + ) + } + print(word, translations) + match r.group("command"): + case "set-default": + w: Word = Word.objects.get(word=word) + tr: UserTranslation = UserTranslation.objects.get( + word=w, translation=translations[r.group("translation")] + ) + tr.score = w.usertranslation_set.first().score + 1 + tr.save() + update_pull_request(number, w) + case _: + pass + + +def update_json_file(word: Word, translations: list[str]) -> str: + """update_json_file Update word translations and return json as string. + + Args: + word (Word): The word to update. + translations (list[str]): List of translations. Order is important. + + Returns: + str: updated json file content as string + """ + + +def update_pull_request(number, word): + with open(settings.BASE_DIR / "github_manager" / "messages.json") as f: + messages = json.load(f)["pull_request"] + pull = dictionary_repo.get_pull(number) + f = dictionary_repo.get_contents( + os.environ.get("JSON_FILE"), ref=f"Torjoman-{word.word}" + ) + dictionary_repo.update_file( + os.environ.get("JSON_FILE"), + f"Update {word.word} Translation To {word.usertranslation_set.first()}", + json.dumps(generate_dict_from_db(word), indent=4, ensure_ascii=False), + f.sha, + branch=f"Torjoman-{word.word}", + ) + body = messages["head"].format( + word=word.word, translation=word.get_users_translations[0] + ) + for index, translation in enumerate(word.get_users_translations[:5]): + body += ( + messages["other_translation"].format( + index=index + 1, translation=translation + ) + + "\n" + ) + body += "\n" + messages["footer"] + pull.edit( + title=messages["title"].format( + word=word.word, translation=word.get_users_translations[0] + ), + body=body, + ) + + +def make_pull_request(): + """make_pull_request Get Words that have at least 5 translations and make a pull request""" + words: list[Word] = Word.objects.annotate( + num_usertranslation=Count("usertranslation") + ).filter(num_usertranslation__gt=3, pullrequest=None) + if not words: + return + with open(settings.BASE_DIR / "github_manager" / "messages.json") as f: + messages = json.load(f)["pull_request"] + source_branch = dictionary_repo.get_branch(dictionary_repo.default_branch) + for word in words: + try: + [ + ref + for ref in dictionary_repo.get_git_refs() + if ref.ref == f"refs/heads/Torjoman-{word.word}" + ][0].delete() + except: + pass + dictionary_repo.create_git_ref( + f"refs/heads/Torjoman-{word.word}", source_branch.commit.sha + ) + f = dictionary_repo.get_contents( + os.environ.get("JSON_FILE"), ref=f"Torjoman-{word.word}" + ) + dictionary_repo.update_file( + os.environ.get("JSON_FILE"), + f"Update {word.word} Translation To {word.usertranslation_set.first()}", + json.dumps(generate_dict_from_db(word), indent=4, ensure_ascii=False), + f.sha, + branch=f"Torjoman-{word.word}", + ) + body = messages["head"].format( + word=word.word, translation=word.get_users_translations[0] + ) + for index, translation in enumerate(word.get_users_translations[:5]): + body += ( + messages["other_translation"].format( + index=index + 1, translation=translation + ) + + "\n" + ) + body += "\n" + messages["footer"] + pull = dictionary_repo.create_pull( + title=messages["title"].format( + word=word.word, translation=word.get_users_translations[0] + ), + body=body, + head=f"Torjoman-{word.word}", + base="main", + ) + PullRequest.objects.create(number=pull.number, word=word) diff --git a/github_manager/webhook.py b/github_manager/webhook.py index 33685f3..2ae1172 100644 --- a/github_manager/webhook.py +++ b/github_manager/webhook.py @@ -1,40 +1,56 @@ import hmac -from hashlib import sha1 import json -from ninja import Router +import threading +from hashlib import sha1 + from django.conf import settings -from django.http import HttpResponse, HttpResponseForbidden, HttpResponseServerError +from django.http import ( + HttpResponse, HttpResponseForbidden, HttpResponseServerError, +) from django.utils.encoding import force_bytes -from .tasks import push -import threading +from ninja import Router +from .tasks import handle_pr_comments, handle_pull_request, handle_push router = Router() -@router.post('/') -def manage_webhooks(request): - # Verify the request signature - header_signature = request.META.get('HTTP_X_HUB_SIGNATURE') - if header_signature is None: - return HttpResponseForbidden('Permission denied.') - sha_name, signature = header_signature.split('=') - if sha_name != 'sha1': - return HttpResponseServerError('Operation not supported.', status=501) - - mac = hmac.new(force_bytes(settings.GITHUB_WEBHOOK_KEY), msg=force_bytes(request.body), digestmod=sha1) - if not hmac.compare_digest(force_bytes(mac.hexdigest()), force_bytes(signature)): - return HttpResponseForbidden('Permission denied.') +@router.post("/") +def manage_webhooks(request): + # Verify the request signature + header_signature = request.META.get("HTTP_X_HUB_SIGNATURE") + if header_signature is None: + return HttpResponseForbidden("Permission denied.") - # If request reached this point we are in a good shape - # Process the GitHub events - event = request.META.get('HTTP_X_GITHUB_EVENT', 'ping') + sha_name, signature = header_signature.split("=") + if sha_name != "sha1": + return HttpResponseServerError("Operation not supported.", status=501) - if event == 'ping': - return HttpResponse('pong') - elif event == 'push': - t = threading.Thread(target=push, args=(json.loads(request.body),)) - t.start() - return HttpResponse('success') - # In case we receive an event that's not ping or push - return HttpResponse(status=204) \ No newline at end of file + mac = hmac.new( + force_bytes(settings.GITHUB_WEBHOOK_KEY), + msg=force_bytes(request.body), + digestmod=sha1, + ) + if not hmac.compare_digest(force_bytes(mac.hexdigest()), force_bytes(signature)): + return HttpResponseForbidden("Permission denied.") + event = request.META.get("HTTP_X_GITHUB_EVENT", "ping") + match event: + case "ping": + return HttpResponse("pong") + case "push": + t = threading.Thread(target=handle_push, args=(json.loads(request.body),)) + t.start() + return HttpResponse("success") + case "pull_request": + t = threading.Thread( + target=handle_pull_request, args=(json.loads(request.body),) + ) + t.start() + return HttpResponse("success") + case "issue_comment": + t = threading.Thread( + target=handle_pr_comments, args=(json.loads(request.body),) + ) + t.start() + return HttpResponse("success") + return HttpResponse(status=204) diff --git a/json-example.json b/json-example.json index 763730f..d16ea58 100644 --- a/json-example.json +++ b/json-example.json @@ -2,14 +2,14 @@ { "word": "Hello", "is_checked": false, - "translates": [ + "translation": [ "Hola" ] }, { "word": "Ping", "is_checked": true, - "translates": [ + "translation": [ "Pong" ] } diff --git a/project/api.py b/project/api.py index 4afd3cc..d7a1be5 100644 --- a/project/api.py +++ b/project/api.py @@ -1,14 +1,14 @@ from django.conf import settings from ninja import NinjaAPI -from translate.api import router as translate_router -from translators.api import router as translators_router -from github_manager.webhook import router as github_manager_router +from github_manager.webhook import router as github_manager_router +from translation.api import router as translate_router +from translators.api import router as translators_router api = NinjaAPI( - title='Torjoman Core API', - version='1.0', - description='This is the Torjoman Core API', + title="Torjoman Core API", + version="1.0", + description="This is the Torjoman Core API", ) api.add_router("/translate", translate_router) diff --git a/project/prepare-project.py b/project/prepare-project.py index bfaa0e1..0a0801b 100644 --- a/project/prepare-project.py +++ b/project/prepare-project.py @@ -1,7 +1,8 @@ from pathlib import Path + BASE_DIR = Path(__file__).resolve().parent.parent -local_json_file: Path = (BASE_DIR / 'local_dictionary.json').resolve() +local_json_file: Path = (BASE_DIR / "local_dictionary.json").resolve() local_json_file.touch() -local_json_file.write_text('{}') +local_json_file.write_text("{}") diff --git a/project/settings.py b/project/settings.py index 3236412..5d2f678 100644 --- a/project/settings.py +++ b/project/settings.py @@ -11,6 +11,7 @@ """ from pathlib import Path + import environ env = environ.Env() @@ -26,13 +27,13 @@ # See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = env('SECRET_KEY') +SECRET_KEY = env("SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True ALLOWED_HOSTS = [] -ALLOWED_HOSTS.append(env('HOST')) +ALLOWED_HOSTS.append(env("HOST")) # Application definition @@ -43,13 +44,11 @@ "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", - 'ninja', - 'django_q', - - 'cache_json.apps.CacheJsonConfig', - 'translate.apps.TranslateConfig', - 'translators.apps.TranslatorsConfig', - 'github_manager.apps.GithubManagerConfig', + "django_q", + "ninja", + "translation.apps.TranslationConfig", + "translators.apps.TranslatorsConfig", + "github_manager.apps.GithubManagerConfig", ] MIDDLEWARE = [ @@ -88,14 +87,15 @@ DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': env("DATABASE_NAME"), - 'USER': env("DATABASE_USER"), - 'PASSWORD': env("DATABASE_PASSWORD"), - 'HOST': env("DATABASE_HOST"), - 'PORT': env("DATABASE_PORT"), - }} + "default": { + "ENGINE": "django.db.backends.postgresql_psycopg2", + "NAME": env("DATABASE_NAME"), + "USER": env("DATABASE_USER"), + "PASSWORD": env("DATABASE_PASSWORD"), + "HOST": env("DATABASE_HOST"), + "PORT": env("DATABASE_PORT"), + } +} # Password validation @@ -133,7 +133,7 @@ # https://docs.djangoproject.com/en/4.1/howto/static-files/ STATIC_URL = "static/" -STATIC_ROOT = BASE_DIR / 'static/' +STATIC_ROOT = BASE_DIR / "static/" # Default primary key field type # https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field @@ -143,14 +143,10 @@ Q_CLUSTER = { "name": "torjoman", - 'retry': 750, - 'timeout': 600, - 'redis': { - 'host': '127.0.0.1', - 'port': 6379, - 'db': 0 - } + "retry": 750, + "timeout": 600, + "redis": {"host": "127.0.0.1", "port": 6379, "db": 0}, } -GITHUB_WEBHOOK_KEY = env('GITHUB_WEBHOOK_KEY') \ No newline at end of file +GITHUB_WEBHOOK_KEY = env("GITHUB_WEBHOOK_KEY") diff --git a/project/urls.py b/project/urls.py index fbd5a60..9c3a2b3 100644 --- a/project/urls.py +++ b/project/urls.py @@ -1,8 +1,9 @@ from django.contrib import admin from django.urls import path + from .api import api urlpatterns = [ - path('admin/', admin.site.urls), - path('api/', api.urls), + path("admin/", admin.site.urls), + path("api/", api.urls), ] diff --git a/translate/__init__.py b/translate/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/translate/admin.py b/translate/admin.py deleted file mode 100644 index 346242a..0000000 --- a/translate/admin.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.contrib import admin -from .models import Word, Translate -# Register your models here. -class TranslateAdminInline(admin.TabularInline): - model = Translate - -class WordAdmin(admin.ModelAdmin): - inlines = (TranslateAdminInline, ) -admin.site.register(Word, WordAdmin) diff --git a/translate/api.py b/translate/api.py deleted file mode 100644 index ff78521..0000000 --- a/translate/api.py +++ /dev/null @@ -1,59 +0,0 @@ -from difflib import SequenceMatcher -from typing import List -from django.shortcuts import get_object_or_404 -from ninja import Router -from . import schemas -from django.conf import settings -from .models import Translate, Word -from cache_json.models import Word as SourceWord -from translators.models import Translator -from ninja.errors import AuthenticationError -#This is for Arabic Only -import pyarabic.araby as araby - - -def do_before_check(word: str) -> str: - return araby.strip_harakat(araby.strip_tatweel(word)) - -router = Router() - -# @router.get("/", response={200: schemas.Word, 404: schemas.Error}) -# def get_nontranslate_words(request): -# words = SourceWord.objects.filter(translate=None).order_by('word') -# if words.count() > 0: -# return 200, words.first() # Get a random nontranslate word - -# words = SourceWord.objects.filter(is_checked=False).order_by('word') -# if words.count() > 0: -# return 200, words.first() # Get a random word not checked word -# else: -# return 404, {'messgae': 'there_are_no_nontranslate_words'} - -@router.post("/", response=schemas.Word) -def recive_word(request, payload: schemas.WordIn): - w: Word = get_object_or_404(Word, word=payload.word) - try: - translator: Translator = Translator.objects.get(uuid=payload.uuid) - w.translators.add(translator) - w.save() - except Translator.DoesNotExist: - raise AuthenticationError - user_translate = do_before_check(payload.translate.strip()) - if user_translate == '': return w - is_exists = (False, None) - for t in w.translate_set.all(): - similarity = SequenceMatcher(None, t.translate, user_translate) - print(similarity.ratio()) - print(similarity.quick_ratio()) - if similarity.ratio() > 0.8: - is_exists = (True, t) - break - if not is_exists[0]: - t = Translate(word=w, translate=user_translate) - t.save() - else: - t: Translate = is_exists[1] - t.score += 1 - t.save() - - return w \ No newline at end of file diff --git a/translate/apps.py b/translate/apps.py deleted file mode 100644 index 6c1c6c8..0000000 --- a/translate/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class TranslateConfig(AppConfig): - default_auto_field = "django.db.models.BigAutoField" - name = "translate" diff --git a/translate/migrations/0001_initial.py b/translate/migrations/0001_initial.py deleted file mode 100644 index 53ea578..0000000 --- a/translate/migrations/0001_initial.py +++ /dev/null @@ -1,59 +0,0 @@ -# Generated by Django 4.1.1 on 2022-09-17 18:03 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ("translators", "0001_initial"), - ("github_manager", "0001_initial"), - ] - - operations = [ - migrations.CreateModel( - name="Word", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("word", models.CharField(max_length=255, unique=True)), - ( - "prs", - models.ManyToManyField(blank=True, to="github_manager.pullrequest"), - ), - ("translators", models.ManyToManyField(to="translators.translator")), - ], - ), - migrations.CreateModel( - name="Translate", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("translate", models.CharField(max_length=255)), - ("score", models.IntegerField(default=1)), - ( - "word", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to="translate.word" - ), - ), - ], - ), - ] diff --git a/translate/migrations/__init__.py b/translate/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/translate/models.py b/translate/models.py deleted file mode 100644 index ba62d37..0000000 --- a/translate/models.py +++ /dev/null @@ -1,20 +0,0 @@ -from django.db import models - -class Word(models.Model): - word = models.CharField(max_length=255, unique=True) - translators = models.ManyToManyField("translators.Translator") - prs = models.ManyToManyField('github_manager.PullRequest', blank=True) - - def get_translates(self) -> list["Translate"]: - return [translate.translate for translate in self.translate_set.all().order_by('score')] - - def __str__(self) -> str: - return self.word - -class Translate(models.Model): - word = models.ForeignKey(Word, on_delete=models.CASCADE) - translate = models.CharField(max_length=255) - score = models.IntegerField(default=1) - - def __str__(self) -> str: - return f'{self.word}-{self.translate}' diff --git a/translate/schemas.py b/translate/schemas.py deleted file mode 100644 index 911f465..0000000 --- a/translate/schemas.py +++ /dev/null @@ -1,17 +0,0 @@ -from typing import List -from ninja import ModelSchema, Schema, Field -from .models import (Word as WordModel) - -class Error(Schema): - message: str - -class WordIn(Schema): - uuid: str - word: str - translate: str - -class Word(ModelSchema): - translates: List[str] = Field(..., alias='get_translates') - class Config: - model = WordModel - model_fields = ['word'] \ No newline at end of file diff --git a/translate/tests.py b/translate/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/translate/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/translate/views.py b/translate/views.py deleted file mode 100644 index 91ea44a..0000000 --- a/translate/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/cache_json/__init__.py b/translation/__init__.py similarity index 100% rename from cache_json/__init__.py rename to translation/__init__.py diff --git a/translation/admin.py b/translation/admin.py new file mode 100644 index 0000000..8944e8c --- /dev/null +++ b/translation/admin.py @@ -0,0 +1,20 @@ +from django.contrib import admin + +from .models import SourceTranslation, UserTranslation, Word + +# Register your models here. + + +class UserTranslationAdminInline(admin.TabularInline): + model = UserTranslation + + +class SourceTranslationAdminInline(admin.TabularInline): + model = SourceTranslation + + +class WordAdmin(admin.ModelAdmin): + inlines = (SourceTranslationAdminInline, UserTranslationAdminInline) + + +admin.site.register(Word, WordAdmin) diff --git a/translation/api.py b/translation/api.py new file mode 100644 index 0000000..438dad7 --- /dev/null +++ b/translation/api.py @@ -0,0 +1,30 @@ +from django.shortcuts import get_object_or_404 +from ninja import Router +from ninja.errors import AuthenticationError + +from translators.models import Translator + +from . import schemas +from .models import Word +from .utils import modify_translation + +router = Router() + + +@router.post( + "/", + response={ + 200: schemas.Word, + }, +) +def receiving_translation(request, payload: schemas.WordIn): + w: Word = get_object_or_404(Word, word=payload.word) + try: + translator: Translator = Translator.objects.get(uuid=payload.uuid) + w.translators.add(translator) + w.save() + except Translator.DoesNotExist: + raise AuthenticationError + tranlation = modify_translation(payload.translation) + w.add_translation(tranlation) + return {"success": True} diff --git a/cache_json/apps.py b/translation/apps.py similarity index 60% rename from cache_json/apps.py rename to translation/apps.py index d30c37f..e950da2 100644 --- a/cache_json/apps.py +++ b/translation/apps.py @@ -1,6 +1,6 @@ from django.apps import AppConfig -class CacheJsonConfig(AppConfig): +class TranslationConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" - name = "cache_json" + name = "translation" diff --git a/cache_json/migrations/0001_initial.py b/translation/migrations/0001_initial.py similarity index 51% rename from cache_json/migrations/0001_initial.py rename to translation/migrations/0001_initial.py index 0d9bc4c..4d5cc6e 100644 --- a/cache_json/migrations/0001_initial.py +++ b/translation/migrations/0001_initial.py @@ -1,7 +1,7 @@ -# Generated by Django 4.1.1 on 2022-09-17 18:03 +# Generated by Django 4.1.1 on 2022-09-30 08:32 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): @@ -31,7 +31,7 @@ class Migration(migrations.Migration): }, ), migrations.CreateModel( - name="Translate", + name="UserTranslation", fields=[ ( "id", @@ -42,15 +42,46 @@ class Migration(migrations.Migration): verbose_name="ID", ), ), - ("translate", models.CharField(max_length=255)), - ("score", models.IntegerField(default=1)), + ("translation", models.CharField(max_length=255)), + ("score", models.PositiveIntegerField(default=0)), ( "word", models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, - to="cache_json.word", + to="translation.word", ), ), ], + options={ + "ordering": ("-score", "translation"), + "abstract": False, + }, + ), + migrations.CreateModel( + name="SourceTranslation", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("translation", models.CharField(max_length=255)), + ("is_default", models.BooleanField(default=False)), + ( + "word", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="translation.word", + ), + ), + ], + options={ + "ordering": ("-is_default", "translation"), + "abstract": False, + }, ), ] diff --git a/cache_json/migrations/__init__.py b/translation/migrations/__init__.py similarity index 100% rename from cache_json/migrations/__init__.py rename to translation/migrations/__init__.py diff --git a/translation/models.py b/translation/models.py new file mode 100644 index 0000000..e7b5511 --- /dev/null +++ b/translation/models.py @@ -0,0 +1,83 @@ +from django.db import models + +from .utils import get_close_match + + +class Word(models.Model): + word = models.CharField(max_length=255, unique=True) + is_checked = models.BooleanField(default=False) + + class Meta: + ordering = ["word"] + + @property + def get_source_translations(self) -> list[str]: + return self.sourcetranslation_set.objects.values_list("translation") + + @property + def get_users_translations(self) -> list[str]: + return self.usertranslation_set.objects.values_list("translation") + + def add_translation(self, translation: str) -> "UserTranslation": + """add_translation Add new `UserTranslation` to word and increase score if similar translation was found. + + Args: + translation (str): the translation to compare and add/increase score. + + Returns: + UserTranslation: The new/similar translation object. + None: if the translation was empty. + """ + if (translation := translation.strip()) == "": + return + match = get_close_match(translation, self.get_users_translations()) + if match: + t: UserTranslation = UserTranslation.objects.get( + word=self, translation=match + ) + t.score += 1 + t.save() + else: + t = UserTranslation(word=self, translation=translation) + t.save() + return t + + def __str__(self) -> str: + return self.word + + +class TranslationBase(models.Model): + word = models.ForeignKey(Word, on_delete=models.CASCADE) + translation = models.CharField(max_length=255) + + class Meta: + abstract = True + + def __str__(self) -> str: + return f"{self.word}-{self.translation}" + + +class SourceTranslation(TranslationBase): + is_default = models.BooleanField(default=False) + + def save(self, *args, **kwargs) -> None: + if self.is_default: + try: + last_default_translation = SourceTranslation.objects.get( + word=self.word, is_default=True + ) + last_default_translation.is_default = False + last_default_translation.save() + except SourceTranslation.DoesNotExist: + pass + return super().save(*args, **kwargs) + + class Meta(TranslationBase.Meta): + ordering = ("-is_default", "translation") + + +class UserTranslation(TranslationBase): + score = models.PositiveIntegerField(default=0) + + class Meta(TranslationBase.Meta): + ordering = ("-score", "translation") diff --git a/translation/schemas.py b/translation/schemas.py new file mode 100644 index 0000000..4874b8e --- /dev/null +++ b/translation/schemas.py @@ -0,0 +1,22 @@ +from uuid import UUID + +from ninja import Field, ModelSchema, Schema + +from .models import SourceTranslation, UserTranslation, Word + + +class WordIn(ModelSchema): + by: UUID + translation: str + + class Config: + model = Word + model_fields = ["word"] + + +class Word(ModelSchema): + translations: list[str] = Field(..., alias="get_source_translations") + + class Config: + model = Word + model_fields = ["word"] diff --git a/cache_json/tests.py b/translation/tests.py similarity index 100% rename from cache_json/tests.py rename to translation/tests.py diff --git a/translation/utils.py b/translation/utils.py new file mode 100644 index 0000000..80b8119 --- /dev/null +++ b/translation/utils.py @@ -0,0 +1,52 @@ +from difflib import SequenceMatcher +from heapq import nlargest as _nlargest + +import pyarabic.araby as araby # Language specific module + + +def get_close_match(word, possibilities, cutoff=0.8) -> (str | tuple[()]): + """Use SequenceMatcher to return a list of the indexes of the best + "good enough" matches. word is a sequence for which close matches + are desired (typically a string). + possibilities is a list of sequences against which to match word + (typically a list of strings). + Optional arg cutoff (default 0.8) is a float in [0, 1]. Possibilities + that don't score at least that similar to word are ignored. + """ + + if not 0.0 <= cutoff <= 1.0: + raise ValueError("cutoff must be in [0.0, 1.0]: %r" % (cutoff,)) + result = [] + s = SequenceMatcher() + s.set_seq2(word) + for idx, x in enumerate(possibilities): + s.set_seq1(x) + if ( + s.real_quick_ratio() >= cutoff + and s.quick_ratio() >= cutoff + and s.ratio() >= cutoff + ): + result.append((s.ratio(), idx)) + + # Move the best scorers to head of list + result = _nlargest(1, result) + + # Strip scores for the best n matches + if result: + return possibilities[result[0][1]] + else: + return () + + +def modify_translation(translation: str) -> str: + """modify_translation Changes the translation before comparing it with saved translations + + Args: + translation (str): the translation to modify + + Returns: + str: the modified translation + """ + return araby.strip_harakat( + araby.strip_tatweel(translation) + ) # remove the harakat and tatweel from the translation diff --git a/translators/admin.py b/translators/admin.py index 0885f1e..50cd49a 100644 --- a/translators/admin.py +++ b/translators/admin.py @@ -1,5 +1,7 @@ from django.contrib import admin -from .models import Translator, Platform + +from .models import Platform, Translator + # Register your models here. admin.site.register(Translator) -admin.site.register(Platform) \ No newline at end of file +admin.site.register(Platform) diff --git a/translators/api.py b/translators/api.py index 17aba1b..739751c 100644 --- a/translators/api.py +++ b/translators/api.py @@ -1,34 +1,33 @@ +from django.http import Http404 +from django.shortcuts import get_object_or_404 from ninja import Router + from . import schemas -from .models import Translator, Platform +from .models import Platform, Translator router = Router() - - -@router.post("/register", response={200: schemas.Translator, 400: schemas.Error, 404: schemas.Error}) + + +@router.post("/register", response={200: schemas.Translator}) def register(request, payload: schemas.TranslatorRegister): - platforms: dict[str, Platform] = {f'{platform.name}': platform for platform in Platform.objects.all()} - platforms_names = list(platforms.keys()) - if payload.platform not in platforms_names: - return 400, schemas.Error(message='platform_not_exists') - translator = Translator(name=payload.name, number_of_words=payload.number_of_words, send_time=payload.send_time) - translator.save() - platform = platforms[payload.platform] - platform.translators.add(translator) - platform.save() - return 200, translator + platforms = Platform.get_all_platforms() + if payload.platform not in platforms: + return (422,) + translator = payload.to_model() + translator.save() + platform = platforms[payload.platform] + platform.translators.add(translator) + platform.save() + return 200, translator -@router.post("/login", response={200: schemas.Translator, 400: schemas.Error, 404: schemas.Error}) +@router.post("/login", response={200: schemas.Translator}) def login(request, payload: schemas.TranslatorLogin): - try: translator = Translator.objects.get(uuid=payload.uuid) - except: return 404, schemas.Error(message='user_not_exists') - platforms: dict[str, Platform] = {f'{platform.name}': platform for platform in Platform.objects.all()} - platforms_names = list(platforms.keys()) - if payload.platform not in platforms_names: - return 400, schemas.Error(message='platform_not_exists') - platform = platforms[payload.platform] - platform.translators.add(translator) - platform.save() - return 200, translator - + translator = get_object_or_404(Translator, uuid=payload.uuid) + platforms = Platform.get_all_platforms() + if payload.platform not in platforms.keys(): + return (422,) + platform = platforms[payload.platform] + platform.translators.add(translator) + platform.save() + return 200, translator diff --git a/translators/migrations/0001_initial.py b/translators/migrations/0001_initial.py index 10d2372..622cc0f 100644 --- a/translators/migrations/0001_initial.py +++ b/translators/migrations/0001_initial.py @@ -1,14 +1,17 @@ -# Generated by Django 4.1.1 on 2022-09-17 18:03 +# Generated by Django 4.1.1 on 2022-09-30 08:32 -from django.db import migrations, models import uuid +from django.db import migrations, models + class Migration(migrations.Migration): initial = True - dependencies = [] + dependencies = [ + ("translation", "0001_initial"), + ] operations = [ migrations.CreateModel( @@ -33,6 +36,10 @@ class Migration(migrations.Migration): models.IntegerField(verbose_name="Number of words to send"), ), ("send_time", models.TimeField()), + ( + "translated_words", + models.ManyToManyField(blank=True, to="translation.word"), + ), ], ), migrations.CreateModel( diff --git a/translators/models.py b/translators/models.py index 39f029e..7edf148 100644 --- a/translators/models.py +++ b/translators/models.py @@ -1,21 +1,39 @@ import uuid + from django.db import models + # Create your models here. class Translator(models.Model): - uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) - name = models.CharField(max_length=255) - number_of_words = models.IntegerField('Number of words to send') - send_time = models.TimeField() - - def __str__(self) -> str: - return self.name - + uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) + name = models.CharField(max_length=255) + number_of_words = models.IntegerField("Number of words to send") + send_time = models.TimeField() + translated_words = models.ManyToManyField("translation.Word", blank=True) + + def get_platforms(self): + return self.platform_set.objects.values_list("name") + + def __str__(self) -> str: + return self.name + + class Platform(models.Model): - name = models.CharField(max_length=250) - base_url = models.URLField() - translators = models.ManyToManyField(Translator, blank=True) - is_active = models.BooleanField(default=True) - - def __str__(self) -> str: - return self.name \ No newline at end of file + name = models.CharField(max_length=250) + base_url = models.URLField() + translators = models.ManyToManyField(Translator, blank=True) + is_active = models.BooleanField( + default=True + ) # if True, torjoman will send untranslated words to this platform + + @classmethod + def get_all_platforms(cls) -> dict[str, "Platform"]: + """get_all_platforms get a dict of all platforms. + + Returns: + dict[str, Platform]: keys are platforms names, values are platforms instances + """ + return {platform.name: platform for platform in Platform.objects.all()} + + def __str__(self) -> str: + return self.name diff --git a/translators/schemas.py b/translators/schemas.py index 1313cd4..942760a 100644 --- a/translators/schemas.py +++ b/translators/schemas.py @@ -1,25 +1,35 @@ from typing import List from uuid import UUID -from ninja import ModelSchema, Schema, Field -from .models import (Translator as TranslatorModel) -class Error(Schema): - message: str +from ninja import Field, ModelSchema, Schema + +from .models import Translator as TranslatorModel + class TranslatorRegister(ModelSchema): - platform: str - class Config: - model = TranslatorModel - model_exclude = ['id'] - + platform: str + + class Config: + model = TranslatorModel + model_exclude = ["id"] + + def to_model(self) -> TranslatorModel: + return TranslatorModel.objects.create( + name=self.name, + number_of_words=self.number_of_words, + send_time=self.send_time, + ) + -class TranslatorLogin(Schema): - uuid: UUID - platform: str +class TranslatorLogin(ModelSchema): + platform: str + class Config: + model = TranslatorModel + model_fields = ["uuid"] class Translator(ModelSchema): - class Config: - model = TranslatorModel - model_exclude = ['id'] \ No newline at end of file + class Config: + model = TranslatorModel + model_exclude = ["id"] diff --git a/translators/tasks.py b/translators/tasks.py index 24f6469..81d5b0e 100644 --- a/translators/tasks.py +++ b/translators/tasks.py @@ -1,28 +1,31 @@ -from .models import Translator -from translate.models import Word from datetime import datetime + import requests +from translation.models import Word + +from .models import Translator + + def send_words(): - now = datetime.now() - # users = Translator.objects.filter(send_time=f"{now.hour}:{now.minute}") - users = Translator.objects.all() - print(f'Sending for {users.count()} users') - for user in users: - words: list[Word] = Word.objects.exclude(translators__in=[user]).order_by('word')[:user.number_of_words] - if words.count() < 1: continue - for platform in user.platform_set.filter(is_active=True): - data = { - 'uuid': str(user.uuid), - 'words': [ - { - 'word': word.word, - 'translates': word.get_translates() - } - for word in words - ] - } - res = requests.post( - f'{platform.base_url}/get_words', - json=data - ) + now = datetime.now() + # users = Translator.objects.filter(send_time=f"{now.hour}:{now.minute}") + users = Translator.objects.all() + print(f"Sending for {users.count()} users") + for user in users: + words: list[Word] = ( + Word.objects.filter(pullrequest=None) + .exclude(translators__in=[user]) + .order_by("word")[: user.number_of_words] + ) + if words.count() < 1: + continue + for platform in user.platform_set.filter(is_active=True): + data = { + "uuid": str(user.uuid), + "words": [ + {"word": word.word, "translates": word.get_translates()} + for word in words + ], + } + res = requests.post(f"{platform.base_url}/get_words", json=data) diff --git a/translators/views.py b/translators/views.py deleted file mode 100644 index 91ea44a..0000000 --- a/translators/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here.