Get overrides from GSheet to CSV

.github/workflows/update.yaml

@@ -54,6 +54,8 @@ jobs:
       run: pip install -U -r requirements.txt
 
     - name: "Run update.py"
+      env:
+        GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
       run: |
         python update.py 
 

gurs/addresses-overrides-geocoded.csv

@@ -0,0 +1,22 @@
+postOver,addressOver,lat,lon,street,streetAlt,housenumber,housenumberAppendix,city,cityAlt,municipalityPart,municipality,zipCode,zipName,statisticalRegion
+1000 Ljubljana,Rakovniška ulica 4,46.03660,14.52361,Rakovniška ulica,,4,,Ljubljana,,Četrtna skupnost Rudnik,Ljubljana,1000,Ljubljana,Osrednjeslovenska
+1000 Ljubljana,Čufarjeva ulica 11,46.05516,14.51021,Čufarjeva ulica,,11,,Ljubljana,,Četrtna skupnost Center,Ljubljana,1000,Ljubljana,Osrednjeslovenska
+1218 Komenda,Glavarjeva cesta 61a,46.20453,14.53935,Glavarjeva cesta,,61,A,Komenda,,,Komenda,1218,Komenda,Osrednjeslovenska
+1231 Ljubljana-Črnuče,Primožičeva ulica 2,46.10339,14.53177,Primožičeva ulica,,2,,Ljubljana,,Četrtna skupnost Črnuče,Ljubljana,1231,Ljubljana - Črnuče,Osrednjeslovenska
+1292 Ig pri Ljubljani,Banija 4,45.95897,14.52720,Banija,,4,,Ig,,,Ig,1292,Ig,Osrednjeslovenska
+1311 Turjak,Laporje 30,45.87088,14.61679,Laporje,,30,,Laporje,,,Velike Lašče,1311,Turjak,Osrednjeslovenska
+1312 Videm - Dobrepolje,Videm 37 a,45.84819,14.69546,Videm,,37,A,Videm,,Videm,Dobrepolje,1312,Videm - Dobrepolje,Osrednjeslovenska
+1354 Horjul,Slovenska cesta 17,46.02461,14.29536,Slovenska cesta,,17,,Horjul,,,Horjul,1354,Horjul,Osrednjeslovenska
+1360 Vrhnika,Stara cesta 4a,45.96590,14.29733,Stara cesta,,4,A,Vrhnika,,Vrhnika Center,Vrhnika,1360,Vrhnika,Osrednjeslovenska
+2000 Maribor,Kalohova ulica 18,46.55507,15.61747,Kalohova ulica,,18,,Maribor,,Studenci,Maribor,2000,Maribor,Podravska
+2370 Dravograd,Trg 4. julija 9,46.58921,15.02233,Trg 4. julija,,9,,Dravograd,,Dravograd,Dravograd,2370,Dravograd,Koroška
+3000 Celje,Gregorčičeva ulica 5,46.23224,15.26227,Gregorčičeva ulica,,5,,Celje,,Dolgo polje,Celje,3000,Celje,Savinjska
+3272 Rimske Toplice,Aškerčeva cesta 4,46.12412,15.19462,Aškerčeva cesta,,4,,Rimske Toplice,,Rimske Toplice,Laško,3272,Rimske Toplice,Savinjska
+3333 Ljubno ob Savinji,Foršt 28,46.34498,14.83276,Foršt,,28,,Ljubno ob Savinji,,,Ljubno,3333,Ljubno ob Savinji,Savinjska
+4000 Kranj,Ljubljanska cesta 24A,46.22457,14.35459,Ljubljanska cesta,,24,A,Kranj,,Stražišče,Kranj,4000,Kranj,Gorenjska
+4000 Kranj,Škofjeloška cesta 6,46.22819,14.35325,Škofjeloška cesta,,6,,Kranj,,Stražišče,Kranj,4000,Kranj,Gorenjska
+4220 Škofja Loka,Stara cesta 10,46.16618,14.31281,Stara cesta,,10,,Škofja Loka,,Kamnitnik,Škofja Loka,4220,Škofja Loka,Gorenjska
+4224 Gorenja vas,Trata 7,46.10233,14.14218,Trata,,7,,Gorenja vas,,Gorenja vas,Gorenja vas-Poljane,4224,Gorenja vas,Gorenjska
+4226 Žiri,Trg svobode 9,46.04754,14.11159,Trg svobode,,9,,Žiri,,,Žiri,4226,Žiri,Gorenjska
+4228 Železniki,Racovnik 29,46.22219,14.15786,Racovnik,,29,,Železniki,,Železniki,Železniki,4228,Železniki,Gorenjska
+5292 Šempeter pri Gorici,Cesta prekomorskih brigad 25,45.93053,13.63792,Cesta Prekomorskih brigad,,25,,Šempeter pri Gorici,,,Šempeter-Vrtojba,5290,Šempeter pri Gorici,Goriška

gurs/addresses-overrides.csv

@@ -0,0 +1,22 @@
+postOver,addressOver
+1000 Ljubljana,Rakovniška ulica 4
+1000 Ljubljana,Čufarjeva ulica 11
+1218 Komenda,Glavarjeva cesta 61a
+1231 Ljubljana-Črnuče,Primožičeva ulica 2
+1292 Ig pri Ljubljani,Banija 4
+1311 Turjak,Laporje 30
+1312 Videm - Dobrepolje,Videm 37 a
+1354 Horjul,Slovenska cesta 17
+1360 Vrhnika,Stara cesta 4a
+2000 Maribor,Kalohova ulica 18
+2370 Dravograd,Trg 4. julija 9
+3000 Celje,Gregorčičeva ulica 5
+3272 Rimske Toplice,Aškerčeva cesta 4
+3333 Ljubno ob Savinji,Foršt 28
+4000 Kranj,Ljubljanska cesta 24A
+4000 Kranj,Škofjeloška cesta 6
+4220 Škofja Loka,Stara cesta 10
+4224 Gorenja vas,Trata 7
+4226 Žiri,Trg svobode 9
+4228 Železniki,Racovnik 29
+5292 Šempeter pri Gorici,Cesta prekomorskih brigad 25

requirements.txt

@@ -3,3 +3,5 @@ pandas==1.3.*
 openpyxl==3.0.*
 requests==2.26.*
 bs4==0.0.1
+sheet2csv==1.0.6
+

update.py

@@ -8,6 +8,11 @@
 import glob
 import pandas as pd
 import subprocess
+import sheet2csv
+
+GOOGLE_API_KEY = os.environ["GOOGLE_API_KEY"]
+SHEET_OVERRIDES = "1gsIkUsvO-2_atHTsU9UcH2q69Js9PuvskTbtuY3eEWQ"
+RANGE_OVERRIDES = "Overrides!A1:AA"
 
 type_map = {
     'SPLOŠNA DEJAVNOST - SPLOŠNA AMBULANTA': 'gp',
@@ -40,7 +45,7 @@ def convert_to_csv(zzzsid_map):
         df['city'] = df['city'].str.strip()
         df['unit'] = df['unit'].str.strip()
         df['zzzsid'] = df['name'].map(zzzsid_map)
-        df = df.reindex(['doctor', 'type', 'accepts', 'availability', 'load', 'name', 'address', 'city', 'unit', 'zzzsid'], axis='columns')
+        df = df.reindex(['doctor', 'type', 'zzzsid', 'accepts', 'availability', 'load', 'name', 'address', 'city', 'unit'], axis='columns')
         doctors.append(df)
 
     doctors = pd.concat(doctors, ignore_index=True)
@@ -59,10 +64,26 @@ def convert_to_csv(zzzsid_map):
 
     # reindex:
     doctors.set_index(['doctor','type','id_inst'], inplace=True)
-
-    print(doctors)
     doctors.to_csv('csv/doctors.csv')
 
+
+def append_overrides():
+    filename = "csv/overrides.csv"
+    print(f"Get overrides from GSheet to {filename}")
+    try:
+        sheet2csv.sheet2csv(id=SHEET_OVERRIDES, range=RANGE_OVERRIDES, api_key=GOOGLE_API_KEY, filename=filename)
+    except Exception as e:
+        print("Failed to import {}".format(filename))
+        raise e
+
+    doctors = pd.read_csv('csv/doctors.csv', index_col=['doctor','type','id_inst'])
+    overrides = pd.read_csv('csv/overrides.csv', index_col=['doctor','type','id_inst'])
+
+    doctors = doctors.join(overrides)
+
+    doctors.to_csv('csv/doctors-overrides.csv')
+
+
 def geocode_addresses():
     xlsxAddresses = pd.read_csv('csv/dict-institutions.csv', usecols=['city','address']).rename(columns={'city':'cityZZZS','address':'addressZZZS'})
     apiAddresses = pd.read_csv('zzzs/institutions-all.csv', usecols=['posta','naslov']).rename(columns={'posta':'cityZZZS','naslov':'addressZZZS'})
@@ -72,7 +93,7 @@ def geocode_addresses():
     addresses['addressZZZS'] = addresses['addressZZZS'].str.upper()
     addresses.sort_values(by=['cityZZZS','addressZZZS'], inplace=True)
     addresses.drop_duplicates(inplace=True)
-    addresses.set_index(['cityZZZS','addressZZZS'], inplace=True) 
+    addresses.set_index(['cityZZZS','addressZZZS'], inplace=True)
 
     addresses.to_csv('gurs/addresses-zzzs.csv')
 
@@ -81,6 +102,17 @@ def geocode_addresses():
     except FileNotFoundError:
         print("geocodecsv not found, skipping.")
 
+    addresses = pd.read_csv('csv/doctors-overrides.csv', usecols=['post', 'address']).rename(columns={'post':'postOver','address':'addressOver'}).dropna()
+    addresses.sort_values(by=['postOver', 'addressOver'], inplace=True)
+    addresses.drop_duplicates(inplace=True)
+    addresses.set_index(['postOver', 'addressOver'], inplace=True)
+    addresses.to_csv('gurs/addresses-overrides.csv')
+
+    try:
+        subprocess.run(["geocodecsv", "-in", "gurs/addresses-overrides.csv", "-out", "gurs/addresses-overrides-geocoded.csv", "-zipCol", "1", "-addressCol", "2", "-appendAll"])
+    except FileNotFoundError:
+        print("geocodecsv not found, skipping.")
+
 
 def add_gurs_geodata():
     institutions = pd.read_csv('csv/dict-institutions.csv', index_col=['id_inst'])
@@ -91,9 +123,18 @@ def add_gurs_geodata():
 
     institutions = institutions.merge(dfgeo[['address','post','city','municipalityPart','municipality','lat','lon']], how = 'left', left_on = ['city','address'], right_index=True, suffixes=['_zzzs', ''])
     institutions.drop(['address_zzzs','city_zzzs'], axis='columns', inplace=True)
-
     institutions.to_csv('csv/dict-institutions.csv')
 
+    doctors = pd.read_csv('csv/doctors-overrides.csv', index_col=['doctor', 'type', 'id_inst'])
+    dfgeo=pd.read_csv('gurs/addresses-overrides-geocoded.csv', index_col=['postOver','addressOver'], dtype=str)
+    dfgeo.fillna('', inplace=True)
+    dfgeo['address'] = dfgeo.apply(lambda x: f'{x.street} {x.housenumber}{x.housenumberAppendix}', axis = 1)
+    dfgeo['post'] = dfgeo.apply(lambda x: f'{x.zipCode} {x.zipName}', axis = 1)
+
+    doctors = doctors.merge(dfgeo[['address','post','city','municipalityPart','municipality','lat','lon']], how = 'left', left_on = ['post','address'], right_index=True, suffixes=['Over', ''])
+    doctors.drop(['addressOver','postOver'], axis='columns', inplace=True)
+    doctors.to_csv('csv/doctors-overrides.csv')
+
 
 def get_zzzs_api_data_all():
     # https://api.zzzs.si/covid-sledilnik/0 ... 1600 by pages (of 100 records)
@@ -122,6 +163,7 @@ def get_zzzs_api_data_all():
     df.sort_values(by=[*df], inplace=True) # sort by all columns
     df.to_csv('zzzs/institutions-all.csv')
 
+
 def get_zzzs_api_data_by_category():
     # keys for ZZZS API calls, add as needed, see https://www.zzzs.si/zzzs-api/izvajalci-zdravstvenih-storitev/po-dejavnosti/
     zzzsApiKeys=[
@@ -240,9 +282,10 @@ def download_zzzs_xlsx_files():
 
 if __name__ == "__main__":
     download_zzzs_xlsx_files()
-    zzzsid_map = get_zzzs_api_data_by_category()
     get_zzzs_api_data_all()
+    zzzsid_map = get_zzzs_api_data_by_category()
     convert_to_csv(zzzsid_map)
+    append_overrides()
     geocode_addresses()
     add_gurs_geodata()
     add_zzzs_api_data()