diff --git a/api/app.py b/api/app.py new file mode 100644 index 0000000..bcbc87e --- /dev/null +++ b/api/app.py @@ -0,0 +1,22 @@ +from flask import Flask +from webscraper.database import * +from webscraper.nyserda_scraper import * + +app = Flask(__name__) + +# flask run --port 5328 --debug + + +@app.route("/api/hello", methods=["GET"]) +def hello_world(): + return "Hello, World!" + + +@app.route("/api/nyserda_large", methods=["GET"]) +async def run_nyserda_large(): + nyserda_large_to_database() + return {"message": "Hello NYSERDA Large"} + + +if __name__ == "__main__": + app.run(port=5328, debug=True) diff --git a/api/webscraper/__init__.py b/api/webscraper/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/webscraper/__pycache__/database_constants.cpython-312.pyc b/api/webscraper/__pycache__/database_constants.cpython-312.pyc index abca84b..e31f734 100644 Binary files a/api/webscraper/__pycache__/database_constants.cpython-312.pyc and b/api/webscraper/__pycache__/database_constants.cpython-312.pyc differ diff --git a/api/webscraper/__pycache__/nyiso_scraper.cpython-312.pyc b/api/webscraper/__pycache__/nyiso_scraper.cpython-312.pyc index ccd35bc..64ccf93 100644 Binary files a/api/webscraper/__pycache__/nyiso_scraper.cpython-312.pyc and b/api/webscraper/__pycache__/nyiso_scraper.cpython-312.pyc differ diff --git a/api/webscraper/__pycache__/nyserda_scraper.cpython-312.pyc b/api/webscraper/__pycache__/nyserda_scraper.cpython-312.pyc index dd47d72..ec01f5f 100644 Binary files a/api/webscraper/__pycache__/nyserda_scraper.cpython-312.pyc and b/api/webscraper/__pycache__/nyserda_scraper.cpython-312.pyc differ diff --git a/api/webscraper/database.py b/api/webscraper/database.py index b322ada..209b557 100644 --- a/api/webscraper/database.py +++ b/api/webscraper/database.py @@ -1,27 +1,30 @@ import os +from dotenv import load_dotenv from datetime import datetime from dateutil import tz from supabase import create_client, Client from geocodio import GeocodioClient -from nyserda_scraper import query_nyserda_large, query_nyserda_solar_repeat -from nyiso_scraper import ( +from .nyserda_scraper import query_nyserda_large, query_nyserda_solar_repeat +from .nyiso_scraper import ( filter_nyiso_iq_sheet, filter_nyiso_cluster_sheet, filter_nyiso_in_service_sheet, filter_nyiso_withdrawn_sheets, ) -from ores_scraper import query_ores_noi, query_ores_under_review, query_ores_permitted -from utils.scraper_utils import ( +from .ores_scraper import query_ores_noi, query_ores_under_review, query_ores_permitted +from .utils.scraper_utils import ( geocode_lat_long, create_update_object, update_kdm, update_last_updated, ) -from database_constants import ( +from .database_constants import ( initial_kdm, ) +load_dotenv(os.path.join(os.path.dirname(__file__), "../../.env.local")) + url: str = os.environ.get("NEXT_PUBLIC_SUPABASE_URL") key: str = os.environ.get("NEXT_PUBLIC_SUPABASE_ANON_KEY") supabase: Client = create_client(url, key) @@ -48,7 +51,7 @@ def nyserda_large_to_database() -> None: In the case that the project is cancelled, we delete the project from the Supabase database. """ database = [] - database.extend(query_nyserda_large()) + database.extend(query_nyserda_large()[:10]) for project in database: if project.get("proposed_cod", None) is not None: ymd = datetime.strptime(project.get("proposed_cod"), "%Y").strftime( @@ -752,7 +755,7 @@ def ores_permitted_to_database() -> None: """ For testing """ -# nyserda_large_to_database() +nyserda_large_to_database() # nyserda_solar_to_database() # nyiso_to_database() # ores_noi_to_database() diff --git a/api/webscraper/nyiso_scraper.py b/api/webscraper/nyiso_scraper.py index 28192be..d6991ec 100644 --- a/api/webscraper/nyiso_scraper.py +++ b/api/webscraper/nyiso_scraper.py @@ -2,8 +2,8 @@ import pandas as pd from io import BytesIO import json -from utils.scraper_utils import clean_df_data -from database_constants import ( +from .utils.scraper_utils import clean_df_data +from .database_constants import ( renewable_energy_abbreviations, ) diff --git a/api/webscraper/nyserda_scraper.py b/api/webscraper/nyserda_scraper.py index 0ba7c3e..ae23966 100644 --- a/api/webscraper/nyserda_scraper.py +++ b/api/webscraper/nyserda_scraper.py @@ -1,7 +1,7 @@ import requests import json -from utils.scraper_utils import check_status, geocode_lat_long, standardize_label -from database_constants import renewable_energy_map, initial_kdm +from .utils.scraper_utils import check_status, geocode_lat_long, standardize_label +from .database_constants import renewable_energy_map, initial_kdm """ This scrapes data from the NYSERDA Large-scale Renewable Projects database. diff --git a/api/webscraper/ores_scraper.py b/api/webscraper/ores_scraper.py index 9daf72c..8c12bba 100644 --- a/api/webscraper/ores_scraper.py +++ b/api/webscraper/ores_scraper.py @@ -2,8 +2,8 @@ from bs4 import BeautifulSoup import pandas as pd from io import StringIO -from utils.scraper_utils import geocode_lat_long -from database_constants import initial_kdm +from .utils.scraper_utils import geocode_lat_long +from .database_constants import initial_kdm # url = "https://dps.ny.gov/ores-permit-applications" # page = requests.get(url) diff --git a/api/webscraper/server.py b/api/webscraper/server.py new file mode 100644 index 0000000..40ebe71 --- /dev/null +++ b/api/webscraper/server.py @@ -0,0 +1,50 @@ +from fastapi import FastAPI, Response +from database import * + +app = FastAPI() + + +@app.get("/favicon.ico") +def favicon(): + return Response(content=open("favicon.ico", "rb").read(), media_type="image/x-icon") + + +@app.get("/") +async def root(): + return {"message": "Hello World"} + + +@app.get("/nyserda_large") +async def run_nyserda_large(): + nyserda_large_to_database() + return {"message": "Hello NYSERDA Large"} + + +@app.get("/nyserda_solar") +async def run_nyserda_solar(): + nyserda_solar_to_database() + return {"message": "Hello NYSERDA Solar"} + + +@app.get("/nyiso") +async def run_nyiso(): + nyiso_to_database() + return {"message": "Hello NYISO"} + + +@app.get("/ores_noi") +async def run_ores_noi(): + ores_noi_to_database() + return {"message": "Hello ORES NOI"} + + +@app.get("/ores_under_review") +async def run_ores_under_review(): + ores_under_review_to_database() + return {"message": "Hello ORES Under Review"} + + +@app.get("/ores_permitted") +async def run_ores_permitted(): + ores_permitted_to_database() + return {"message": "Hello ORES Permitted"} diff --git a/next.config.mjs b/next.config.mjs index bfc08c1..abe7a9e 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -12,6 +12,14 @@ const nextConfig = { }, ], }, + async rewrites() { + return [ + { + source: '/api/:path*', + destination: 'http://127.0.0.1:5328/api/:path*', // Proxy to Backend + }, + ]; + }, }; export default nextConfig;