diff --git a/.gitignore b/.gitignore index 0074830..eda1cdd 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,10 @@ mysql/sql/01_data.sql # django django/**/migrations/** django/university/models.py +django/statistics.sql + +# celery +django/celerybeat-schedule # stats related graphs django/university/stats_graphs/* diff --git a/django/entrypoint.sh b/django/entrypoint.sh index 8c8c702..fb723a6 100644 --- a/django/entrypoint.sh +++ b/django/entrypoint.sh @@ -25,7 +25,9 @@ python manage.py inspectdb > university/models.py python manage.py makemigrations python manage.py migrate university --fake - +# Initialize redis worker for celery and celery's beat scheduler in the background +celery -A tasks worker --loglevel=INFO & +celery -A tasks beat & # Initializes the API. exec $cmd diff --git a/django/manage.py b/django/manage.py index 3d7162a..6d9b541 100755 --- a/django/manage.py +++ b/django/manage.py @@ -3,7 +3,6 @@ import os import sys - def main(): """Run administrative tasks.""" os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tts_be.settings') @@ -17,6 +16,5 @@ def main(): ) from exc execute_from_command_line(sys.argv) - if __name__ == '__main__': main() diff --git a/django/requirements.txt b/django/requirements.txt index 39de5f0..3eab85a 100644 --- a/django/requirements.txt +++ b/django/requirements.txt @@ -6,3 +6,5 @@ djangorestframework==3.11.0 pytz==2021.3 sqlparse==0.4.2 mysqlclient==1.4.6 +celery==5.2.7 +redis==3.5.3 diff --git a/django/tasks.py b/django/tasks.py new file mode 100644 index 0000000..0520c18 --- /dev/null +++ b/django/tasks.py @@ -0,0 +1,23 @@ +from celery import Celery +from celery.schedules import crontab +import os + +app = Celery('tasks', broker="redis://tts-be-redis_service-1:6379") + +# Gets called after celery sets up. Creates a worker that runs the dump_statistics function at midnight and noon everyday +@app.on_after_configure.connect +def setup_periodic_tasks(sender, **kwargs): + sender.add_periodic_task( + crontab(minute='0', hour='0, 12'), + dump_statistics.s(), + name='dump statistics' + ) + +@app.task +def dump_statistics(): + command = "mysqldump -P {} -h db -u {} -p{} {} statistics > statistics.sql".format( + os.environ["MYSQL_PORT"], + os.environ["MYSQL_USER"], + os.environ["MYSQL_PASSWORD"], + os.environ["MYSQL_DATABASE"]) + os.system(command) diff --git a/django/university/stats.py b/django/university/stats.py deleted file mode 100644 index 3b64d7f..0000000 --- a/django/university/stats.py +++ /dev/null @@ -1,89 +0,0 @@ -import json -import os -from time import sleep - -""" - This singleton class is used to store the statistics of the requests made to the server. - It is used to store the number of requests made to the server for each course. -""" -class statistics: - CACHE_DIR = "./university/cache/" - REQ_CACHE_FILE = "statistics_cache.json" - REQ_CACH_PATH = CACHE_DIR + REQ_CACHE_FILE - - __instance = None - - def __init__(self, courses, year): - if statistics.__instance == None: - self.year = year - self.requests_stats = dict() # key: id value: number of requests - if self.load_cache(): - print("Loaded cache") - else: - print("Cache not found, initializing statistics to 0") - for course in courses: - self.requests_stats[course["id"]] = 0 - - statistics.__instance = self - - - - @staticmethod - def get_instance(): - return statistics.__instance - - def get_year(self): - return self.year - - def get_request_stats(self): - return self.requests_stats - - - def get_specific_stat(self, id): - return self.requests_stats[id] - - - def increment_requests_stats(self, id): - self.requests_stats[id] += 1 - - - def import_request_stats(self, filepath): - with open(filepath, 'r') as f: - self.requests_stats = json.load(f) - - - def cache_stats(self, filepath: str): - if not os.path.isdir(self.CACHE_DIR): - os.makedirs(self.CACHE_DIR) - with open(filepath, 'w') as f: - json.dump(self.requests_stats, f) - - def load_cache(self) -> bool: - if os.path.exists(self.REQ_CACH_PATH): - with open(self.REQ_CACH_PATH, 'r') as f: - cached_json_stats = json.load(f) - - for id in cached_json_stats: - self.requests_stats[int(id)] = cached_json_stats[id] - - return True - - return False - - - - - def export_request_stats(self, courses): - requests_stats_to_export = {} - for course in courses: - if course["id"] in self.requests_stats: - requests_stats_to_export[course["name"]] = self.requests_stats[course["id"]] - - return json.dumps(requests_stats_to_export, ensure_ascii=False) - - -def cache_statistics(): - stats = statistics.get_instance() - if stats != None: - stats.cache_stats(stats.REQ_CACH_PATH) - diff --git a/django/university/views.py b/django/university/views.py index db3eba3..1a327b1 100644 --- a/django/university/views.py +++ b/django/university/views.py @@ -6,21 +6,17 @@ from university.models import Professor from university.models import ScheduleProfessor from university.models import CourseMetadata +from university.models import Statistics from django.http import JsonResponse from django.core import serializers from rest_framework.decorators import api_view from django.db.models import Max -from university.stats import statistics, cache_statistics +from django.db import transaction import json import os +from django.utils import timezone # Create your views here. -""" - Initialization of statistics. -""" - -DEFAULT_YEAR = 2023 -statistics(Course.objects.filter(year=DEFAULT_YEAR).values(), DEFAULT_YEAR) def get_field(value): return value.field @@ -56,10 +52,18 @@ def course_units(request, course_id, year, semester): course_units.__dict__.update(course_units.course_unit.__dict__) del course_units.__dict__["_state"] json_data.append(course_units.__dict__) - - stats = statistics.get_instance() - if stats != None: - stats.increment_requests_stats(id=course_id) + + course = Course.objects.get(id = course_id) + + with transaction.atomic(): + statistics, created = Statistics.objects.select_for_update().get_or_create( + course_unit_id = course_id, + acronym = course.acronym, + defaults = {"visited_times": 0, "last_updated": timezone.now()}, + ) + statistics.visited_times += 1 + statistics.last_updated = timezone.now() + statistics.save() return JsonResponse(json_data, safe=False) @@ -119,13 +123,10 @@ def data(request): name = request.GET.get('name') password = request.GET.get('password') if name == os.environ['STATISTICS_NAME'] and password == os.environ['STATISTICS_PASS']: - stats = statistics.get_instance() - if stats != None: - json_data = stats.export_request_stats(Course.objects.filter(year=stats.get_year()).values()) - cache_statistics() - return HttpResponse(json.dumps(json_data), content_type='application/json') + json_data = serializers.serialize("json", Statistics.objects.all()) + return HttpResponse(json_data, content_type='application/json') else: - return HttpResponse(status=401) + return HttpResponse(status=401) """ Returns all the professors of a class of the schedule id diff --git a/docker-compose.yaml b/docker-compose.yaml index 9315c2b..dc8740a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -40,7 +40,8 @@ services: volumes: - ./fetcher/:/fetcher/ - - - - + redis_service: + image: redis:6.2-bullseye + restart: always + ports: + - '6379:6379' diff --git a/mysql/sql/00_schema_mysql.sql b/mysql/sql/00_schema_mysql.sql index 4ccb259..76c5bc8 100644 --- a/mysql/sql/00_schema_mysql.sql +++ b/mysql/sql/00_schema_mysql.sql @@ -118,6 +118,17 @@ CREATE TABLE `professor` ( ) ENGINE=InnoDB CHARSET = utf8 COLLATE = utf8_general_ci; +-- -------------------------------------------------------- +-- Table structure for table `statistics` +-- + +CREATE TABLE `statistics` ( + `course_unit_id` int(11) NOT NULL, + `acronym` varchar(10) NOT NULL, + `visited_times` int(11) NOT NULL, + `last_updated` datetime NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + -- Add primary keys alter TABLE faculty ADD PRIMARY KEY (`acronym`); @@ -135,6 +146,8 @@ alter TABLE course_metadata ADD FOREIGN KEY (`course_id`) REFERENCES `course`(`i alter TABLE schedule ADD PRIMARY KEY (`id`); alter TABLE schedule ADD FOREIGN KEY (`course_unit_id`) REFERENCES `course_unit`(`id`) ON DELETE CASCADE ON UPDATE CASCADE; +alter TABLE statistics ADD PRIMARY KEY (`course_unit_id`); + alter TABLE professor ADD PRIMARY KEY (`sigarra_id`); alter TABLE schedule_professor ADD PRIMARY KEY (`schedule_id`, `professor_sigarra_id`); @@ -163,8 +176,12 @@ CREATE UNIQUE INDEX `faculty_acronym` ON `faculty`(`acronym`); -- CREATE INDEX `schedule_course_unit_id` ON `schedule`(`course_unit_id`); +-- +-- Indexes for table `schedule` +-- +CREATE INDEX `statistics` ON `statistics`(`course_unit_id`); + -- -- Indexes for table `course_metadata` -- CREATE INDEX `course_metadata_index` ON `course_metadata`(`course_id`, `course_unit_id`, `course_unit_year`); -