diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cbb9c27..b4433da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,10 @@ jobs: SUPABASE_URL: ${{ vars[format('APP_{0}_SUPABASE_URL', needs.set_vars.outputs.APP_ENV)] }} SUPABASE_KEY: ${{ secrets[format('APP_{0}_SUPABASE_KEY', needs.set_vars.outputs.APP_ENV)] }} SECRET_KEY: ${{ secrets[format('APP_{0}_SECRET_KEY', needs.set_vars.outputs.APP_ENV)] }} + POSTGRES_DB_HOST: ${{ secrets[format('APP_{0}_POSTGRES_DB_HOST', needs.set_vars.outputs.APP_ENV)] }} + POSTGRES_DB_NAME: ${{ secrets[format('APP_{0}_POSTGRES_DB_NAME', needs.set_vars.outputs.APP_ENV)] }} + POSTGRES_DB_USER: ${{ secrets[format('APP_{0}_POSTGRES_DB_USER', needs.set_vars.outputs.APP_ENV)] }} + POSTGRES_DB_PASS: ${{ secrets[format('APP_{0}_POSTGRES_DB_PASS', needs.set_vars.outputs.APP_ENV)] }} steps: - name: Checkout code uses: actions/checkout@v2 @@ -87,6 +91,10 @@ jobs: echo "SUPABASE_URL=${SUPABASE_URL}" >> .env echo "SUPABASE_KEY=${SUPABASE_KEY}" >> .env echo "SECRET_KEY=${SECRET_KEY}" >> .env + echo "POSTGRES_DB_HOST=${POSTGRES_DB_HOST}" >> .env + echo "POSTGRES_DB_NAME=${POSTGRES_DB_NAME}" >> .env + echo "POSTGRES_DB_USER=${POSTGRES_DB_USER}" >> .env + echo "POSTGRES_DB_PASS=${POSTGRES_DB_PASS}" >> .env mv .env ${{ env.DOT_ENV_FILE_NAME }} - name: Copy env file to DEV Server diff --git a/app.py b/app.py index 07b82bf..027e1ea 100644 --- a/app.py +++ b/app.py @@ -1,17 +1,25 @@ from flask import Flask, jsonify,request,url_for -from db import SupabaseInterface from collections import defaultdict from flasgger import Swagger import re,os,traceback +from query import PostgresORM from utils import * from flask_cors import CORS,cross_origin from v2_app import v2 +from flask_sqlalchemy import SQLAlchemy +from models import db + app = Flask(__name__) CORS(app,supports_credentials=True) +app.config['SQLALCHEMY_DATABASE_URI'] = PostgresORM.get_postgres_uri() +app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + +db.init_app(app) + Swagger(app) GITHUB_TOKEN =os.getenv('GITHUB_TOKEN') @@ -45,61 +53,6 @@ def greeting(): - -@app.route('/get-data', methods=['GET']) -@cross_origin(supports_credentials=True) -@require_secret_key -def get_data(): - """ - Fetch data from Supabase. - --- - responses: - 200: - description: Data fetched successfully - schema: - type: array - items: - type: object - 500: - description: Error fetching data - schema: - type: object - properties: - error: - type: string - """ - try: - response = SupabaseInterface().get_instance().client.table('dmp_pr_updates').select('*').execute() - data = response.data - return jsonify(data) - except Exception as e: - return jsonify({'error': str(e)}), 200 - - - -@app.route('/v1/issues', methods=['GET']) -@require_secret_key -def v1get_issues(): - try: - response = SupabaseInterface().get_instance().client.table('dmp_issue_updates').select('*').execute() - data = response.data - - #group data based on issues - grouped_data = defaultdict(list) - for record in data: - issue_url = record['issue_url'] - grouped_data[issue_url].append({ - 'id': record['id'], - 'name': record['body_text'] - }) - - result = [{'issue_url': issue_url, 'issues': issues} for issue_url, issues in grouped_data.items()] - grouped_data = group_by_owner(result) - return jsonify(grouped_data) - - except Exception as e: - error_traceback = traceback.format_exc() - return jsonify({'error': str(e), 'traceback': error_traceback}), 200 @app.route('/issues', methods=['GET']) @@ -127,21 +80,18 @@ def get_issues(): type: string """ try: - # Fetch all issues with their details - response = SupabaseInterface().get_instance().client.table('dmp_orgs').select('*, dmp_issues(*)').execute() - res = [] - - for org in response.data: - obj = {} - issues = org['dmp_issues'] - obj['org_id'] = org['id'] - obj['org_name'] = org['name'] - renamed_issues = [{"id": issue["id"], "name": issue["title"]} for issue in issues] - obj['issues'] = renamed_issues - - res.append(obj) - - return jsonify({"issues": res}) + # Fetch all issues with their details + data = PostgresORM.get_issue_query() + response = [] + + for result in data: + response.append({ + 'org_id': result.org_id, + 'org_name': result.org_name, + 'issues': result.issues + }) + + return jsonify({"issues": response}) except Exception as e: error_traceback = traceback.format_exc() @@ -190,16 +140,15 @@ def get_issues_by_owner(owner): description: Error message """ try: - # Construct the GitHub URL based on the owner parameter - org_link = f"https://github.com/{owner}" - + # Fetch organization details from dmp_orgs table - response = SupabaseInterface().get_instance().client.table('dmp_orgs').select('name', 'description').eq('name', owner).execute() - - if not response.data: + response = PostgresORM.get_issue_owner(owner) + if not response: return jsonify({'error': "Organization not found"}), 404 - - return jsonify(response.data) + + orgs_dict = [org.to_dict() for org in response] + + return jsonify(orgs_dict) except Exception as e: error_traceback = traceback.format_exc() diff --git a/db.py b/db.py index 6abec14..e69de29 100644 --- a/db.py +++ b/db.py @@ -1,61 +0,0 @@ -import os, sys -from typing import Any -from supabase import create_client, Client -from supabase.lib.client_options import ClientOptions -from abc import ABC, abstractmethod - -client_options = ClientOptions(postgrest_client_timeout=None) - - - -class SupabaseInterface(): - - _instance = None - - def __init__(self): - if not SupabaseInterface._instance: - - # Load environment variables - from dotenv import load_dotenv - load_dotenv() - - SUPABASE_URL = os.getenv('SUPABASE_URL') - SUPABASE_KEY = os.getenv('SUPABASE_KEY') - self.client: Client = create_client(SUPABASE_URL, SUPABASE_KEY) - SupabaseInterface._instance = self - else: - SupabaseInterface._instance = self._instance - - - - @staticmethod - def get_instance(): - # Static method to retrieve the singleton instance - if not SupabaseInterface._instance: - # If no instance exists, create a new one - SupabaseInterface._instance = SupabaseInterface() - return SupabaseInterface._instance - - - def readAll(self, table): - data = self.client.table(f"{table}").select("*").execute() - return data.data - - def add_data(self, data,table_name): - data = self.client.table(table_name).insert(data).execute() - return data.data - - def add_data_filter(self, data, table_name): - # Construct the filter based on the provided column names and values - filter_data = {column: data[column] for column in ['dmp_id','issue_number','owner']} - - # Check if the data already exists in the table based on the filter - existing_data = self.client.table(table_name).select("*").eq('dmp_id',data['dmp_id']).execute() - - # If the data already exists, return without creating a new record - if existing_data.data: - return "Data already exists" - - # If the data doesn't exist, insert it into the table - new_data = self.client.table(table_name).insert(data).execute() - return new_data.data \ No newline at end of file diff --git a/models.py b/models.py new file mode 100644 index 0000000..b1c6040 --- /dev/null +++ b/models.py @@ -0,0 +1,141 @@ +from flask import Flask +from flask_sqlalchemy import SQLAlchemy +from datetime import datetime + +db = SQLAlchemy() + +class DmpOrg(db.Model): + __tablename__ = 'dmp_orgs' + + id = db.Column(db.Integer, primary_key=True, autoincrement=True) + created_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) + name = db.Column(db.String, nullable=False) + description = db.Column(db.Text, nullable=True) + link = db.Column(db.String, nullable=False) + repo_owner = db.Column(db.String, nullable=False) + + # Relationship to DmpIssueUpdate + issues = db.relationship('DmpIssueUpdate', backref='organization', lazy=True) + + # Updated relationship name to avoid conflict + dmp_issues = db.relationship('DmpIssue', backref='organization', lazy=True) + + def __repr__(self): + return f"" + + def to_dict(self): + return { + 'id': self.id, + 'created_at': self.created_at.isoformat(), + 'name': self.name, + 'description': self.description, + 'link': self.link, + 'repo_owner': self.repo_owner + } + +class DmpIssue(db.Model): + __tablename__ = 'dmp_issues' + + id = db.Column(db.Integer, primary_key=True, autoincrement=True) + issue_url = db.Column(db.String, nullable=False) + issue_number = db.Column(db.Integer, nullable=False) + mentor_username = db.Column(db.String, nullable=True) + contributor_username = db.Column(db.String, nullable=True) + title = db.Column(db.String, nullable=False) + org_id = db.Column(db.Integer, db.ForeignKey('dmp_orgs.id'), nullable=False) + description = db.Column(db.Text, nullable=True) + repo = db.Column(db.String, nullable=True) + + + # Relationship to Prupdates + pr_updates = db.relationship('Prupdates', backref='pr_details', lazy=True) + + def __repr__(self): + return f"" + + def to_dict(self): + return { + 'id': self.id, + 'issue_url': self.issue_url, + 'issue_number': self.issue_number, + 'mentor_username': self.mentor_username, + 'contributor_username': self.contributor_username, + 'title': self.title, + 'org_id': self.org_id, + 'description': self.description, + 'repo': self.repo + } + +class DmpIssueUpdate(db.Model): + __tablename__ = 'dmp_issue_updates' + + created_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) + body_text = db.Column(db.Text, nullable=False) + comment_link = db.Column(db.String, nullable=False) + comment_id = db.Column(db.BigInteger, primary_key=True, nullable=False) + comment_api = db.Column(db.String, nullable=False) + comment_updated_at = db.Column(db.DateTime, nullable=False) + dmp_id = db.Column(db.Integer, db.ForeignKey('dmp_orgs.id'), nullable=False) + created_by = db.Column(db.String, nullable=False) + + def __repr__(self): + return f"" + + def to_dict(self): + return { + 'created_at': self.created_at.isoformat(), + 'body_text': self.body_text, + 'comment_link': self.comment_link, + 'comment_id': self.comment_id, + 'comment_api': self.comment_api, + 'comment_updated_at': self.comment_updated_at.isoformat(), + 'dmp_id': self.dmp_id, + 'created_by': self.created_by + } + +class Prupdates(db.Model): + __tablename__ = 'dmp_pr_updates' + + created_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow) + pr_id = db.Column(db.Integer, nullable=False,primary_key=True) + status = db.Column(db.String, nullable=False) + title = db.Column(db.String, nullable=False) + pr_updated_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow) + merged_at = db.Column(db.DateTime) + closed_at = db.Column(db.DateTime) + dmp_id = db.Column(db.Integer, db.ForeignKey('dmp_issues.id'), nullable=False) # ForeignKey relationship + link = db.Column(db.String, nullable=False) + + def __repr__(self): + return f'' + + def to_dict(self): + return { + 'created_at': self.created_at.isoformat(), + 'pr_id': self.pr_id, + 'status': self.status, + 'title': self.title, + 'pr_updated_at': self.pr_updated_at.isoformat(), + 'merged_at': self.merged_at.isoformat() if self.merged_at else None, + 'closed_at': self.closed_at.isoformat() if self.closed_at else None, + 'dmp_id': self.dmp_id, + 'link': self.link + } + +class DmpWeekUpdate(db.Model): + __tablename__ = 'dmp_week_updates' + + id = db.Column(db.Integer, primary_key=True, autoincrement=True) + issue_url = db.Column(db.String, nullable=False) + week = db.Column(db.Integer, nullable=False) + total_task = db.Column(db.Integer, nullable=False) + completed_task = db.Column(db.Integer, nullable=False) + progress = db.Column(db.Integer, nullable=False) + task_data = db.Column(db.Text, nullable=False) + dmp_id = db.Column(db.Integer, nullable=False) + + def __repr__(self): + return f"" + +# if __name__ == '__main__': +# db.create_all() diff --git a/query.py b/query.py new file mode 100644 index 0000000..bcce7b1 --- /dev/null +++ b/query.py @@ -0,0 +1,67 @@ +from models import * +from sqlalchemy import func +import os +from dotenv import load_dotenv + + +load_dotenv() + + +class PostgresORM: + + def get_postgres_uri(): + DB_HOST = os.getenv('POSTGRES_DB_HOST') + DB_NAME = os.getenv('POSTGRES_DB_NAME') + DB_USER = os.getenv('POSTGRES_DB_USER') + DB_PASS = os.getenv('POSTGRES_DB_PASS') + + return f'postgresql://{DB_USER}:{DB_PASS}@{DB_HOST}/{DB_NAME}' + + def get_issue_query(): + results = ( + db.session.query( + DmpOrg.id.label('org_id'), + DmpOrg.name.label('org_name'), + func.json_agg( + func.json_build_object( + 'id', DmpIssue.id, + 'name', DmpIssue.title + ) + ).label('issues') + ) + .outerjoin(DmpIssue, DmpOrg.id == DmpIssue.org_id) + .group_by(DmpOrg.id) + .order_by(DmpOrg.id) + .all() + ) + + return results + + def get_issue_owner(name): + response = DmpOrg.query.filter_by(name=name).all() + return response + + def get_actual_owner_query(owner): + results = DmpOrg.query.filter(DmpOrg.name.like(f'%{owner}%')).all() + results = [val.to_dict() for val in results] + return results + + + def get_dmp_issues(issue_id): + results = DmpIssue.query.filter_by(id=issue_id).all() + results = [val.to_dict() for val in results] + return results + + + def get_dmp_issue_updates(dmp_issue_id): + results = DmpIssueUpdate.query.filter_by(dmp_id=dmp_issue_id).all() + results = [val.to_dict() for val in results] + return results + + + def get_pr_data(dmp_issue_id): + pr_updates = Prupdates.query.filter_by(dmp_id=dmp_issue_id).all() + pr_updates_dict = [pr_update.to_dict() for pr_update in pr_updates] + return pr_updates_dict + + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 34241d0..70a2cee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,7 @@ gunicorn==22.0.0 flasgger==0.9.7.1 markdown==3.4.1 requests==2.32.2 -flask-cors==4.0.1 \ No newline at end of file +flask-cors==4.0.1 +Flask-SQLAlchemy==3.1.1 +postgrest==0.16.4 +psycopg2-binary==2.9.9 \ No newline at end of file diff --git a/tests.py b/tests.py index f9cf2e6..3d66baf 100644 --- a/tests.py +++ b/tests.py @@ -1,18 +1,25 @@ import unittest from v2_utils import remove_unmatched_tags from app import app -import json,random +import json, random,os +from dotenv import load_dotenv +load_dotenv() class CustomTestResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) print(f"{test._testMethodName} - passed") - class CustomTestRunner(unittest.TextTestRunner): resultclass = CustomTestResult + def run(self, test): + result = super().run(test) + if result.wasSuccessful(): + print("All Testcases Passed") + return result + class TestRemoveUnmatchedTags(unittest.TestCase): """ @@ -20,37 +27,37 @@ class TestRemoveUnmatchedTags(unittest.TestCase): """ def test_remove_unmatched_tags_basic(self): input_text = "
Test content

" - expected_output = "
Test content
" + expected_output = "
    Test content
" self.assertEqual(remove_unmatched_tags(input_text), expected_output) def test_remove_unmatched_tags_unmatched_opening(self): input_text = "
Test content" - expected_output = "
Test content
" + expected_output = "
    Test content
" self.assertEqual(remove_unmatched_tags(input_text), expected_output) def test_remove_unmatched_tags_unmatched_closing(self): input_text = "

Test content

" - expected_output = "

Test content

" + expected_output = "

    Test content

" self.assertEqual(remove_unmatched_tags(input_text), expected_output) def test_remove_unmatched_tags_nested_tags(self): input_text = "

Test content

" - expected_output = "

Test content

" + expected_output = "

    Test content

" self.assertEqual(remove_unmatched_tags(input_text), expected_output) def test_remove_unmatched_tags_unmatched_nested_opening(self): input_text = "

Test content

" - expected_output = "

Test content

" + expected_output = "

    Test content

" self.assertEqual(remove_unmatched_tags(input_text), expected_output) def test_remove_unmatched_tags_unmatched_nested_closing(self): input_text = "
Test content

" - expected_output = "
Test content
" + expected_output = "
    Test content
" self.assertEqual(remove_unmatched_tags(input_text), expected_output) def test_remove_unmatched_tags_multiple_unmatched_tags(self): input_text = "
Test

Content

Here" - expected_output = "
Test

Content

Here" + expected_output = "
    Test

    Content

    Here
" self.assertEqual(remove_unmatched_tags(input_text), expected_output) def test_remove_unmatched_tags_text_with_no_tags(self): @@ -61,7 +68,7 @@ def test_remove_unmatched_tags_text_with_no_tags(self): def test_remove_unmatched_tags_empty_string(self): input_text = "" expected_output = "" - self.assertEqual(len(remove_unmatched_tags(input_text)),len(expected_output)) + self.assertEqual(len(remove_unmatched_tags(input_text)), len(expected_output)) class TestIssuesEndpoints(unittest.TestCase): @@ -70,13 +77,16 @@ def setUp(self): self.app = app.test_client() self.app.testing = True self.issues_data = None # To store issues data for use in subsequent tests + self.headers = { + 'x-secret-key':os.getenv('SECRET_KEY') + } # Fetch issues data during setup self._fetch_issues_data() def _fetch_issues_data(self): # Validate the /issues endpoint and store the issues data - response = self.app.get('/issues') + response = self.app.get('/issues',headers=self.headers) self.assertEqual(response.status_code, 200) data = json.loads(response.data) @@ -94,14 +104,14 @@ def test_get_issues_detail_success(self): # Use first data from /issues response to form the endpoint URL - index = random.randrange(1,len(self.issues_data)-1) + index = random.randrange(1, len(self.issues_data) - 1) sample_issue = self.issues_data[index]['issues'][0] issue_id = sample_issue['id'] orgname = self.issues_data[index]['org_name'] endpoint = f'/v2/issues/{orgname}/{issue_id}' - response = self.app.get(endpoint) + response = self.app.get(endpoint,headers=self.headers) self.assertEqual(response.status_code, 200) def test_get_repo_detail_success(self): @@ -110,13 +120,12 @@ def test_get_repo_detail_success(self): self.skipTest("Skipping detail test as /issues endpoint did not return data") # Use first data from /issues response to form the endpoint URL - index = random.randrange(1,len(self.issues_data)-1) + index = random.randrange(1, len(self.issues_data) - 1) orgname = self.issues_data[index]['org_name'] endpoint = f'/issues/{orgname}' - response = self.app.get(endpoint) + response = self.app.get(endpoint,headers=self.headers) self.assertEqual(response.status_code, 200) - if __name__ == '__main__': unittest.main(testRunner=CustomTestRunner()) diff --git a/utils.py b/utils.py index 082102c..9b31c18 100644 --- a/utils.py +++ b/utils.py @@ -250,7 +250,7 @@ def determine_week(input_date_str, start_date_str='2024-06-11'): try: # Convert the start date string to a datetime object start_date = datetime.strptime(start_date_str, '%Y-%m-%d') - input_date = parser.parse(input_date_str).replace(tzinfo=None) + input_date = parser.parse(input_date_str).replace(tzinfo=None) if type(input_date_str) == str else input_date_str.replace(tzinfo=None) # Calculate the difference in days difference_in_days = (input_date - start_date).days diff --git a/v2_app.py b/v2_app.py index 6d0d3c0..b4c2af8 100644 --- a/v2_app.py +++ b/v2_app.py @@ -2,9 +2,10 @@ from flask import Blueprint, jsonify, request import markdown from utils import require_secret_key -from db import SupabaseInterface from utils import determine_week from v2_utils import calculate_overall_progress, define_link_data, week_data_formatter +from query import PostgresORM + v2 = Blueprint('v2', __name__) @@ -12,32 +13,30 @@ @v2.route('/issues//', methods=['GET']) @require_secret_key def get_issues_by_owner_id_v2(owner, issue): + try: - SUPABASE_DB = SupabaseInterface().get_instance() # Fetch issue updates based on owner and issue number url = f"https://github.com/{owner}" - # import pdb;pdb.set_trace() - actual_owner = SUPABASE_DB.client.table('dmp_orgs').select('id','name','repo_owner').like('name',owner).execute().data + actual_owner = PostgresORM.get_actual_owner_query(owner) repo_owner =actual_owner[0]['repo_owner'] if actual_owner else "" #create url with repo owner url = f"https://github.com/{repo_owner}" if repo_owner else None - - dmp_issue_id = SUPABASE_DB.client.table('dmp_issues').select('*').eq('id', issue).execute() - if not dmp_issue_id.data: + dmp_issue_id = PostgresORM.get_dmp_issues(issue) + if not dmp_issue_id: print(f"url....{url}....{issue}") return jsonify({'error': "No data found in dmp_issue"}), 500 - dmp_issue_id = dmp_issue_id.data[0] - response = SUPABASE_DB.client.table('dmp_issue_updates').select('*').eq('dmp_id', dmp_issue_id['id']).execute() + dmp_issue_id = dmp_issue_id[0] - if not response.data: + response = PostgresORM.get_dmp_issue_updates(dmp_issue_id['id']) + if not response: print(f"dmp_issue_id....{response}....{dmp_issue_id}") return jsonify({'error': "No data found in dmp_issue_updates"}), 500 - data = response.data + data = response final_data = [] w_learn_url,w_goal_url,avg,cont_details,plain_text_body,plain_text_wurl = None,None,None,None,None,None @@ -84,10 +83,11 @@ def get_issues_by_owner_id_v2(owner, issue): "weekly_learnings":week_data_formatter(plain_text_wurl,"Learnings") } - pr_Data = SUPABASE_DB.client.table('dmp_pr_updates').select('*').eq('dmp_id', dmp_issue_id['id']).execute() + + pr_Data = PostgresORM.get_pr_data(dmp_issue_id['id']) transformed = {"pr_details": []} - if pr_Data.data: - for pr in pr_Data.data: + if pr_Data: + for pr in pr_Data: pr_status = pr.get("status", "") if pr_status == "closed" and pr.get("merged_at"): pr_status = "merged"