Merge pull request #73 from NIAEFEUP/develop

Update main branch for release

.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+  pull_request:
+    branches: 
+      - main
+      - develop
+
+jobs:
+  audit:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [14.x]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Set up Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+
+    - name: Install dependencies
+      run: npm install
+
+    - name: Audit
+      run: npm run audit
+
+  lint:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [14.x]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Set up Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+
+    - name: Install dependencies
+      run: npm install
+
+    - name: Lint
+      run: npm run lint
+

.gitignore

@@ -1,104 +1,25 @@
 # Logs
 logs
 *.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-lerna-debug.log*
 
-# Diagnostic reports (https://nodejs.org/api/report.html)
-report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
-
-# Runtime data
-pids
-*.pid
-*.seed
-*.pid.lock
-
-# Directory for instrumented libs generated by jscoverage/JSCover
-lib-cov
-
-# Coverage directory used by tools like istanbul
-coverage
-*.lcov
-
-# nyc test coverage
-.nyc_output
-
-# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
-.grunt
-
-# Bower dependency directory (https://bower.io/)
-bower_components
-
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (https://nodejs.org/api/addons.html)
-build/Release
-
-# Dependency directories
-node_modules/
-jspm_packages/
-
-# TypeScript v1 declaration files
-typings/
-
-# TypeScript cache
-*.tsbuildinfo
-
-# Optional npm cache directory
-.npm
-
-# Optional eslint cache
-.eslintcache
-
-# Microbundle cache
-.rpt2_cache/
-.rts2_cache_cjs/
-.rts2_cache_es/
-.rts2_cache_umd/
-
-# Optional REPL history
-.node_repl_history
-
-# Output of 'npm pack'
-*.tgz
-
-# Yarn Integrity file
-.yarn-integrity
+# sigarra internal data cannot be committed
+mysql/sql/01_dump_mysql.sql 
 
 # dotenv environment variables file
 .env
-.env.test
-
-# parcel-bundler cache (https://parceljs.org/)
-.cache
-
-# Next.js build output
-.next
-
-# Nuxt.js build / generate output
-.nuxt
-dist
-
-# Gatsby files
-.cache/
-# Comment in the public line in if your project uses Gatsby and *not* Next.js
-# https://nextjs.org/blog/next-9-1#public-directory-support
-# public
-
-# vuepress build output
-.vuepress/dist
 
-# Serverless directories
-.serverless/
+# mysql data
+mysql/data/*
+mysql/sql/01_data.sql
+**__pycache__
 
-# FuseBox cache
-.fusebox/
+# django 
+django/**/migrations/**
+django/university/models.py
+django/statistics.sql
 
-# DynamoDB Local files
-.dynamodb/
+# celery
+django/celerybeat-schedule
 
-# TernJS port file
-.tern-port
+# stats related graphs
+django/university/stats_graphs/*

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.analysis.typeCheckingMode": "basic"
+}

Makefile

@@ -0,0 +1,31 @@
+.PHONY: all clean 
+
+MYSQL_DATA = ./mysql/sql
+
+all: clean_database
+	@echo [EXECUTING] ./scripts/$(EXEC)
+	@bash ./scripts/exec.sh
+
+download: clean_fetcher clean_database
+	@echo [CREATING] fetcher/data folder 
+	@-mkdir ./fetcher/data
+	@echo [DOWNLOADING] data from the source... 
+	@docker-compose run fetcher python ./update_data/download.py
+	@echo [REMOVING] data from mysql... 
+	@-rm $(MYSQL_DATA)/01_data.sql
+	@echo [MOVING] data from fetcher to sql... 
+	@mv ./fetcher/data/* ./mysql/sql
+
+upload: 
+	@echo [UPLOADING] data...
+	@docker-compose run fetcher python ./update_data/upload.py
+
+clean: clean_fetcher clean_database
+
+clean_fetcher: 
+	@echo [CLEANING] Removing folder fetcher/data...
+	@-rm -r ./fetcher/data
+
+clean_database:
+	@echo [CLEANING] Removing folder mysql/data...
+	@-rm -r ./mysql/data/

README.md

@@ -1 +1,45 @@
-# tts-be
+# TTS - backend
+The backend for timetable selector. 
+## Installation 
+### Prerequisites
+- `docker`
+- `docker-compose` 
+
+### Installing docker 
+to install docker, take a look on the [official website](https://www.docker.com/) and follow the [`Get docker`](https://docs.docker.com/get-docker/) section to install it. If you're using windows, make sure to have the [`wsl`](https://docs.microsoft.com/en-us/windows/wsl/install) installed.   
+
+In case you're using linux, after installing docker check the [`Manage Docker as a non-root user`](https://docs.docker.com/engine/install/linux-postinstall/), so you can use docker without the `sudo` command.  
+
+## Data
+
+The data is available the NIAEFEUP drive (Only for NIAEFEUP members):
+
+https://drive.google.com/drive/folders/1hyiwPwwPWhbAPeJm03c0MAo1HTF6s_zK?usp=sharing
+
+- The ```00_schema_mysql.sql``` corresponds to the schema for the most recent data.
+
+- Copy the ```01_data.sql``` and ```00_schema_mysql.sql``` of year and semester you desire to the ```mysql/sql``` folder.
+
+
+
+## Usage 
+### Development environment 
+You can start developing by building the local server with docker:
+
+```yaml
+docker-compose build . 
+```
+
+In case you have __already build the server before and want to build it again__, be sure to delete the folder in `mysql/data`. You can do this by running `sudo rm -r mysql/data/`. To make your life easier, you can simply run the `build_dev.sh` script: `sudo ./build_dev.sh`.   
+> The sudo permission is nevessary to delete the `mysql/data` folder. 
+
+```yaml
+docker-compose up 
+```
+
+As well as the build, the running command can also be executed with the `run_dev.sh` script by executing: `./run_dev.sh`. 
+
+
+> __WARNING__: it's likely that the first execution of `docker-compose up` after building won't work, since django doesn't wait for the database being populated to executed. Thus, if that's your ccase, execute it again. 
+
+

django/.dockerignore

@@ -0,0 +1 @@
+Dockerfile

django/.env.dev

@@ -0,0 +1,8 @@
+DEBUG=0
+SECRET_KEY=foo
+
+MYSQL_DATABASE=tts
+MYSQL_PASSWORD=root
+MYSQL_USER=root
+MYSQL_HOST=db
+MYSQL_PORT=3306

django/Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.8-slim-buster
+
+WORKDIR /usr/src/django/ 
+
+# Get's the output from the django in realtime. 
+ENV PYTHONUNBUFFERED 1
+ENV STATISTICS_NAME tts_be
+ENV STATISTICS_PASS batata_frita_123
+
+# Copy requirements 
+COPY ./requirements.txt ./requirements.txt 
+
+# Dependencies for mysqlclient 
+RUN apt-get update
+RUN apt-get -y install build-essential default-libmysqlclient-dev
+
+# Install mysql command to wait for the database initialization 
+RUN apt -y install default-mysql-client
+
+# Install the requirements
+RUN pip install -r requirements.txt
+
+EXPOSE 8000
+
+COPY ./entrypoint.sh ./entrypoint.sh  
+ENTRYPOINT ["sh", "/usr/src/django/entrypoint.sh"]

django/entrypoint.sh

@@ -0,0 +1,33 @@
+#!bin/sh 
+
+# WARNING: The script will not work if formated with CRLF. 
+
+# Configure the shell behaviour. 
+set -e
+if [[ ${DEBUG} == 1 ]]
+then set -x
+fi 
+
+# Get parameters.
+database_host="$1"    # The database host and should be provided the container name. 
+shift
+cmd="$@"
+
+# Waits for mysql initialization. 
+until mysql -h "$database_host" -u ${MYSQL_USER} -p${MYSQL_PASSWORD} ${MYSQL_DATABASE} -e 'select 1'; do
+  >&2 echo "MySQL is unavailable - sleeping"
+  sleep 4
+done
+>&2 echo "Mysql is up - executing command" 
+
+# Migrate the Django.
+python manage.py inspectdb > university/models.py
+python manage.py makemigrations
+python manage.py migrate university --fake
+
+# Initialize redis worker for celery and celery's beat scheduler in the background
+celery -A tasks worker --loglevel=INFO &
+celery -A tasks beat &
+
+# Initializes the API. 
+exec $cmd

django/manage.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tts_be.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+if __name__ == '__main__':
+    main()