Merge pull request #1 from sudeepaviraj/dev

Dev

.env.example

@@ -0,0 +1,5 @@
+DATABASE_HOST="YOUR DATABASE HOST"
+DATABASE_PORT="YOUR DATABASE PORT"
+DATABASE_USER="YOUR DATABASE USER"
+DATABASE_PASSWORD="YOUR DATABASE PASSWORD"
+DATABASE_NAME="srilankav2"

DataScrape.py

@@ -1,10 +1,15 @@
+import time
+
 import requests
 from bs4 import BeautifulSoup
 import json
+import pandas as pd
+from Database import SelectQuery,Query
+from tkinter.messagebox import showinfo
 
 
-def GetProvinces() -> list:
 
+def GetProvinces() -> list:
     ProvinceList = []
 
     cookies = {
@@ -41,7 +46,6 @@ def GetProvinces() -> list:
 
 
 def GetProvinceDistricts(province: str) -> list:
-
     DistrictList = []
 
     cookies = {
@@ -88,9 +92,7 @@ def GetProvinceDistricts(province: str) -> list:
             pass
     return DistrictList
 
-
 def GetDistrictDevisions(district):
-
     DevisonList = []
 
     cookies = {
@@ -130,9 +132,11 @@ def GetDistrictDevisions(district):
         if (devision.getText() != "Select a Divisional Secretariat"):
             try:
                 DevisonList.append({"name": devision.getText()[
-                                   3:], "value": devision["value"]})
+                                            3:], "value": devision["value"]})
             except:
                 pass
         else:
             pass
     return DevisonList
+
+

DataStore.py

@@ -5,11 +5,19 @@
 
 
 for province in provinces:
-    Query(f"INSERT INTO provinces (id,name) VALUES({province['value']},'{province['name']}')")
+    if(province['name'][0]==" "):
+        Query(f"INSERT INTO provinces (id,name) VALUES({province['value']},'{province['name'][1:]}')")
+    else:
+        Query(f"INSERT INTO provinces (id,name) VALUES({province['value']},'{province['name']}')")
     districts = GetProvinceDistricts(province)
     for district in districts:
-        Query(f"INSERT INTO districts (id,name,province_id) VALUES({district['value']},'{district['name']}',{province['value']})")
+        if(district['name'][0]==" "):
+            Query(f"INSERT INTO districts (id,name,province_id) VALUES({district['value']},'{district['name'][1:]}',{province['value']})")
+        else:
+            Query(f"INSERT INTO districts (id,name,province_id) VALUES({district['value']},'{district['name']}',{province['value']})")
         divisions = GetDistrictDevisions(district)
         for division in divisions:
-            Query(f"INSERT INTO divisions (id,name,district_id) VALUES({division['value']},'{division['name']}',{district['value']})")
-
+            if(division['name'][0]==" "):
+                Query(f"INSERT INTO divisions (id,name,district_id) VALUES({division['value']},'{division['name'][1:]}',{district['value']})")
+            else:
+                Query(f"INSERT INTO divisions (id,name,district_id) VALUES({division['value']},'{division['name']}',{district['value']})")

Database.py

@@ -1,14 +1,17 @@
 import json
-
 import mysql.connector
+from dotenv.main import load_dotenv
+import os
+load_dotenv()
+
 
 def Query(sql):
     connection = mysql.connector.connect(
-        host="YOUR DATABASE HOST",
-        port="YOUR DATABASE PORT",
-        user="YOUR DATABASE HOST",
-        password="YOUR DATABASE PASSWORD",
-        database="YOUR DATABASE"
+        host=os.environ["DATABASE_HOST"],
+        port=os.environ["DATABASE_PORT"],
+        user=os.environ["DATABASE_USER"],
+        password=os.environ["DATABASE_PASSWORD"],
+        database=os.environ["DATABASE_NAME"]
     )
     cursor = connection.cursor()
 
@@ -23,24 +26,21 @@ def Query(sql):
 
 def SelectQuery(sql) -> list:
     connection = mysql.connector.connect(
-        host="YOUR DATABASE HOST",
-        port="YOUR DATABASE PORT",
-        user="YOUR DATABASE HOST",
-        password="YOUR DATABASE PASSWORD",
-        database="YOUR DATABASE"
+        host=os.environ["DATABASE_HOST"],
+        port=os.environ["DATABASE_PORT"],
+        user=os.environ["DATABASE_USER"],
+        password=os.environ["DATABASE_PASSWORD"],
+        database=os.environ["DATABASE_NAME"]
     )
     dataset = []
     try:
         curser = connection.cursor()
         curser.execute(sql)
-        responses = curser.fetchall()
         try:
-            for response in responses:
-                dataset.append({"id": response[0], "name": response[1].decode("utf-8")})
+            connection.close()
+            return curser.fetchall()
         except Exception as e:
-            for response in responses:
-                dataset.append({"id": response[0], "name": response[1]})
-        connection.close()
-        return dataset
+            connection.close()
+            return "fetch failed"
     except Exception as e:
-        raise SyntaxError(e)
+        raise SyntaxError(e)

app.py

@@ -7,39 +7,41 @@
 
 @app.route("/")
 def Home():
-    return '''<h1 id="sri-lanka-administrative-data-api">Sri Lanka Administrative Data API</h1>
+    return '''
+    <h1 id="sri-lanka-administrative-data-api">Sri Lanka Administrative Data API</h1>
 <p>👋 Welcome to the Sri Lanka Administrative Data API repository! This API provides access to provinces, districts, and division data in Sri Lanka.</p>
 <h2 id="installation">Installation</h2>
 <ol>
 <li>Clone the repository:</li>
 </ol>
-<pre><code class="lang-bash">git <span class="hljs-keyword">clone</span> <span class="hljs-title">https</span>://github.com/sudeepaviraj/gn-extract.git
+<pre><code class="lang-bash">git <span class="hljs-keyword">clone</span> <span class="hljs-title">https</span>://github.com/sudeepaviraj/administrative-api-srilanka.git
 </code></pre>
 <ol>
 <li>Navigate to the project directory:</li>
 </ol>
-<pre><code class="lang-bash"><span class="hljs-built_in">cd</span> gn-extract
+<pre><code class="lang-bash"><span class="hljs-built_in">cd</span> administrative-api-srilanka
 </code></pre>
 <ol>
 <li>Install the required dependencies:</li>
 </ol>
 <pre><code class="lang-bash">pip <span class="hljs-keyword">install</span> -r requirements.txt
 </code></pre>
 <h2 id="config">Config</h2>
-<p>You Need to config this before use</p>
+<p>You Need to configure this application before use</p>
+<ol>
+<li>Create a database named <code>srilankav2</code> in your sql server </li>
+</ol>
 <ol>
 <li>Import <code>srilanka.sql</code> to your own sql server</li>
 </ol>
 <ol>
-<li>Update connection values in <code>database.py</code></li>
+<li>Create a <code>.env</code> file and add database connection values </li>
 </ol>
-<pre><code>connection = mysql<span class="hljs-selector-class">.connector</span><span class="hljs-selector-class">.connect</span>(
-        host=<span class="hljs-string">"YOUR_DATABSE_HOST"</span>,
-        port=<span class="hljs-number">3306</span>,
-        user=<span class="hljs-string">"YOUR DATABASE USER"</span>,
-        password=<span class="hljs-string">"YOUR DATABASE PASSWORD"</span>,
-        database=<span class="hljs-string">"YOUR DATABASE"</span>
-    )
+<pre><code><span class="hljs-attr">DATABASE_HOST</span>=<span class="hljs-string">"YOUR DATABASE HOST"</span>
+<span class="hljs-attr">DATABASE_PORT</span>=<span class="hljs-string">"YOUR DATABASE PORT"</span>
+<span class="hljs-attr">DATABASE_USER</span>=<span class="hljs-string">"YOUR DATABASE USER"</span>
+<span class="hljs-attr">DATABASE_PASSWORD</span>=<span class="hljs-string">"YOUR DATABASE PASSWORD"</span>
+<span class="hljs-attr">DATABASE_NAME</span>=<span class="hljs-string">"srilankav2"</span>
 </code></pre><h2 id="usage">Usage</h2>
 <ol>
 <li>Run the application:</li>
@@ -54,7 +56,8 @@ def Home():
 <li><strong>Get all districts</strong>: <code>/districts</code></li>
 <li><strong>Get all divisions</strong>: <code>/divisions</code></li>
 <li><strong>Get all districts in a province</strong>: <code>/getdistricts?province={province_id}</code></li>
-<li><p><strong>Get all divisions in a district</strong>: <code>/getdivisons?district={district_id}</code></p>
+<li><strong>Get all divisions in a district</strong>: <code>/getdivisons?district={district_id}</code></li>
+<li><p><strong>Get all villages in a division</strong>: <code>/villages?division={division_id}</code></p>
 </li>
 <li><p>Make HTTP GET requests to the desired endpoints using tools like <code>curl</code> or <code>Postman</code> to retrieve the administrative data.</p>
 </li>
@@ -63,33 +66,58 @@ def Home():
 <pre><code class="lang-bash">curl <span class="hljs-string">http:</span><span class="hljs-comment">//localhost:5000/provinces</span>
 </code></pre>
 <p>Example response:</p>
-<pre><code class="lang-json">[
-    {
-        <span class="hljs-attr">"id"</span>: <span class="hljs-number">1</span>,
-        <span class="hljs-attr">"name"</span>: <span class="hljs-string">"Western"</span>
-    },
-    {
-        <span class="hljs-attr">"id"</span>: <span class="hljs-number">2</span>,
-        <span class="hljs-attr">"name"</span>: <span class="hljs-string">"Central"</span>
-    }
+<pre><code class="lang-json">[   
+  [
+    <span class="hljs-number">63</span>,
+    <span class="hljs-string">"Western"</span>,
+    <span class="hljs-string">"බස්නාහිර"</span>,
+    <span class="hljs-string">"மேற்கு"</span>
+  ],
+  [
+    <span class="hljs-number">64</span>,
+    <span class="hljs-string">"Central"</span>,
+    <span class="hljs-string">"මධ්‍යම"</span>,
+    <span class="hljs-string">"மத்திய"</span>
+  ]
 ]
 </code></pre>
+<h2 id="response-structure">Response Structure</h2>
+<p><code>provinces,districts and divisions</code></p>
+<pre><code class="lang-json"> [
+   <span class="hljs-string">"PROVINCE_ID"</span>,
+   <span class="hljs-string">"NAME_EN"</span>,
+   <span class="hljs-string">"NAME_SI"</span>,
+   <span class="hljs-string">"NAME_TA"</span>
+ ]
+</code></pre>
+<pre><code class="lang-villages```">```json
+ [
+   [
+    <span class="hljs-string">"LIFE_CODE"</span>,
+    <span class="hljs-string">"GN_CODE"</span>,
+    <span class="hljs-string">"MPA_CODE"</span>,
+    <span class="hljs-string">"NAME_EN"</span>,
+    <span class="hljs-string">"NAME_SI"</span>,
+    <span class="hljs-string">"NAME_TA"</span>
+  ]
+ ]
+</code></pre>
 <h2 id="demo-api-endpoint">Demo API Endpoint</h2>
 <pre><code class="lang-http://db.famed.cloud/```">
-<span class="hljs-meta">## *Special Note*</span>
+<span class="hljs-comment">## *Special Note*</span>
 
-**You can run sql queries form your own mysql server directly. <span class="hljs-keyword">To</span> <span class="hljs-keyword">do</span> that please use database file ```srilanka.sql``` <span class="hljs-keyword">to</span> your own mysql server**
+**You can <span class="hljs-built_in">run</span> sql queries form your own <span class="hljs-built_in">application</span> directly. To do <span class="hljs-keyword">that</span> please use database <span class="hljs-built_in">file</span> ```srilanka.sql``` <span class="hljs-keyword">to</span> your own mysql server**
 
-<span class="hljs-meta">## Contributing</span>
+<span class="hljs-comment">## Contributing</span>
 
-We welcome contributions <span class="hljs-keyword">to</span> improve this project! <span class="hljs-keyword">If</span> you<span class="hljs-comment">'d like to contribute, please follow these steps:</span>
+We welcome contributions <span class="hljs-keyword">to</span> improve this project! If you'd like <span class="hljs-keyword">to</span> contribute, please follow these steps:
 
-<span class="hljs-number">1.</span> Fork the repository.
+<span class="hljs-number">1.</span> Fork <span class="hljs-keyword">the</span> repository.
 
-<span class="hljs-number">2.</span> Create a <span class="hljs-keyword">new</span> branch <span class="hljs-keyword">for</span> your feature:
+<span class="hljs-number">2.</span> Create a new branch <span class="hljs-keyword">for</span> your feature:
 
 ```bash
-git checkout -b feature/your-feature-name
+git checkout -b feature/your-feature-<span class="hljs-built_in">name</span>
 </code></pre>
 <ol>
 <li>Make your changes and commit them:</li>
@@ -106,17 +134,18 @@ def Home():
 </ol>
 <h2 id="license">License</h2>
 <p>This project is licensed under the <a href="LICENSE">MIT License</a>.</p>
+
 '''
 
 @app.route("/provinces",methods=["GET"])
 def ProvincesList():
-    provinces = SelectQuery("SELECT * FROM provinces")
+    provinces = SelectQuery("SELECT id,name_en,name_si,name_ta FROM provinces")
     print(json.dumps({"provinces":provinces}))
     return json.dumps(provinces)
 
 @app.route("/districts",methods=["GET"])
 def DistrictsList():
-    districts= SelectQuery("SELECT id,name FROM districts")
+    districts= SelectQuery("SELECT id,name_en,name_si,name_ta FROM districts")
     return json.dumps(districts)
 
 @app.route("/getdistricts",methods=["GET"])
@@ -125,13 +154,13 @@ def DistrictsListSELECT():
     if id is None:
         return "Please Add Province Id",422
     else:
-        districts = SelectQuery(f"SELECT id,name FROM districts WHERE province_id = {id}")
+        districts = SelectQuery(f"SELECT id,name_en,name_si,name_ta FROM districts WHERE province_id = {id}")
         print(districts)
         return districts
 
 @app.route("/divisions",methods=["GET"])
 def DivisionsList():
-    divisions= SelectQuery("SELECT id,name FROM divisions")
+    divisions= SelectQuery("SELECT id,name_en,name_si,name_ta FROM divisions")
     return json.dumps(divisions)
 
 @app.route("/getdivisions",methods=["GET"])
@@ -140,9 +169,20 @@ def DivisionsListSELECT():
     if id is None:
         return "Please Add District Id", 422
     else:
-        divisions = SelectQuery(f"SELECT id,name FROM divisions WHERE district_id = {id}")
+        divisions = SelectQuery(f"SELECT id,name_en,name_si,name_ta FROM divisions WHERE district_id = {id}")
         return json.dumps(divisions)
 
+@app.route("/villages",methods=["GET"])
+def VillagesListSELECT():
+    id = request.args.get("division")
+    if id is None:
+        return "Please Add Division Id", 422
+    else:
+        divisions = SelectQuery(f"SELECT life_code, gn_code, mpa_code, name_en, name_si,name_ta FROM villages WHERE division_id = {id}")
+        return json.dumps(divisions)
+
+
+
 
 if __name__ == "__main__":
     app.run(debug=True)