Skip to content

Commit

Permalink
complet new resampling methode
Browse files Browse the repository at this point in the history
  • Loading branch information
@smercier
smercier committed Mar 14, 2019
1 parent 58870c3 commit 4572b5e
Show file tree
Hide file tree
Showing 10 changed files with 411 additions and 151 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ __pycache__/
*.shx
*.prj
*.dbf
*.log

# C extensions
*.so
Expand Down
Empty file added .sql
View file
Empty file.
77 changes: 60 additions & 17 deletions pgrastertime/commandline.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"""

import argparse, os, sys, time, glob

from datetime import datetime
from pgrastertime import init_config
from pgrastertime.readers import RasterReader
from pgrastertime.processes import LoadRaster
Expand Down Expand Up @@ -66,14 +66,19 @@ def parse_arguments():
)
parser.add_argument(
'--verbose', '-v',
action='count', default=0,
action='store_true',
help="Verbose"
)
parser.add_argument(
'--force', '-f', '--force-overwrite',
action='store_true',
help="Force overwrite of the historical data"
)
parser.add_argument(
'--dry-run', '-x',
action='store_true',
help="Force overwrite of the historical data"
)
parser.add_argument(
'--reset-data',
action='store_true',
Expand All @@ -92,6 +97,9 @@ def parse_arguments():
def main():
args = parse_arguments()

if not args.verbose:
args.verbose = False

# if not set use local.ini
if not (args.config_file):
inif = "local.ini"
Expand Down Expand Up @@ -158,30 +166,64 @@ def main():
error_list = []
ns=nb=er=0
if os.path.isdir(args.reader):

# we will inform each loop how many file left ...
xmlCounter = len(glob.glob1(args.reader,"*.xml"))


# We will log how much time it will take
start_time = time.time()
date_started = str(datetime.now())

# Print result in logfile
log_date = date_started.replace(" ","_").replace(":","-").split(".", 1)[0]
loggingfile = ("pgrastertime_%s.log" % log_date)

for file in os.listdir(args.reader):
if (os.path.splitext(file)[-1].lower() == '.xml'):
print ("extension:" + os.path.splitext(file)[-1].lower())
nb += 1
print ("\n Process file %d of %d ..." % (nb,xmlCounter))
if (XMLRastersObject(os.path.join(args.reader, file),
args.tablename,
args.force,
args.sqlfiles,
args.verbose).importRasters() != "SUCCESS"):
step = "\n Process file %d of %d ..." % (nb,xmlCounter)
print(step)
logfile = open(loggingfile, "a")
logfile.write(step)
logfile.close
if (XML2RastersResampling(os.path.join(args.reader, file),
args.tablename,
args.force,
args.sqlfiles,
args.verbose,
args.dry_run).importRasters() != "SUCCESS"):
er += 1
error_list.append(os.path.join(args.reader, file))
else:
ns += 1

# Print result of process
print ("\n Convert %d files of %d" % (ns,nb))



logfile = open(loggingfile, "a")
logfile.write("\n\n==== pgRastertime log file...\n")
logfile.write("Param : Target table -> %s \n" % args.tablename)
logfile.write("Param : Source directory -> %s \n" % args.reader)
logfile.write("Param : Force -> %s \n" % str(args.force))
logfile.write("Param : Post process -> %s \n" % args.sqlfiles)
logfile.write("====\n")
logfile.write("Import Started: %s \n" % date_started.split(".", 1)[0])
logfile.write("Import Ended: %s \n" % str(datetime.now()).split(".", 1)[0])
logfile.write("Number of XML file to process : %d \n" % xmlCounter)
logfile.write("Number of invalide XML file or fail porcess : %d \n" % er)
logfile.write("Execution took %s seconds to process \n" % str((time.time() - start_time)).split(".", 1)[0])

if len(error_list):
print ("Invalid or corrupt files list:")
print ("\n".join(error_list))

logfile.write("\n==== Invalid or corrupt files list:")
logfile.write("\n".join(error_list))

# pintf and close
logfile.close

with open(loggingfile, "r") as logfile:
for line in logfile:
print(line)
logfile.close

elif os.path.isfile(args.reader):
# user specify a file instead of a folder to process
Expand All @@ -196,6 +238,7 @@ def main():
args.tablename,
args.force,
args.sqlfiles,
args.verbose).importRasters()
args.verbose,
args.dry_run).importRasters()


77 changes: 42 additions & 35 deletions pgrastertime/processes/raster2pgsql.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,16 @@

class Raster2pgsql:

def __init__(self, rasterfile, tablename, filename, tile_id, date, resolution, verbose=False):
def __init__(self, rasterfile, tablename, filename, tile_id, date, resolution, source_tiff_file , verbose, dry_run):
self.rasterfile = rasterfile
self.tablename = tablename
self.filename = filename
self.tile_id = tile_id
self.date = date
self.resolution = resolution
self.source_tiff_file = source_tiff_file
self.verbose = verbose
self.dry_run = dry_run

def getConParam(self):
conDic = {}
Expand All @@ -30,49 +32,54 @@ def run(self):

## *NOTE:* sqlalchemy didn't able to manage result from some large raster file.
## Using system cmd trhough psql workaround work fine.
cmd = "raster2pgsql -Y -a -f raster -t 100x100 -n filename %s %s > %s.sql" % (
self.filename, self.tablename, self.filename)
cmd_ras2pg = "raster2pgsql -Y -a -f raster -t 100x100 -n filename %s %s > %s.sql" % (
self.rasterfile, self.tablename, self.rasterfile)

if self.verbose:
print(cmd)

if subprocess.call(cmd, shell=True) == 0:

# 2. Load database con info
user_param = self.getParamsDict()


# if raster2pgsql run w/ success, we import SQL file in database
cmd = "PGPASSWORD=%s psql -q -p %s -h %s -U %s -d %s -f %s.sql" % (
# if raster2pgsql run w/ success, we import SQL file in database
user_param = self.getConParam()
cmd_psql = "PGPASSWORD=%s psql -q -p %s -h %s -U %s -d %s -f %s.sql" % (
user_param['pg_pw'],
user_param['pg_port'],
user_param['pg_host'],
user_param['pg_user'],
user_param['pg_dbname'],
filename)

if self.verbose:
print(cmd)

if subprocess.call(cmd, shell=True) == 0:

## if raster file upload w/ success, we update some metadata in DFO model
sql = "UPDATE " + self.tablename + " set filename = '" + \
self.filename+"', tile_id="+str(self.tile_id)+",resolution = " + \
self.rasterfile)

## if raster file upload w/ success, we update some metadata in DFO model
sql = "UPDATE " + self.tablename + " set filename = '" + \
self.source_tiff_file.split("/")[-1]+"', tile_id="+str(self.tile_id)+",resolution = " + \
str(self.resolution)+", sys_period=tstzrange('"+str(self.date) + \
"', NULL) where filename = '"+self.filename.split("/")[-1] + \
"', NULL) where filename = '"+ self.rasterfile.split("/")[-1] + \
"' and resolution is null"
if self.verbose:
print(cmd)
try:
DBSession().execute(sql)
DBSession().commit()
except DatabaseError as error:
print('Fail to run SQL : %s ' % (error.args[0]))
return False

if self.verbose:
print(cmd_ras2pg)
print(cmd_psql)
print(sql)

if not self.dry_run:
# 1. create the sql file that load the raster data
if subprocess.call(cmd_ras2pg, shell=True) == 0:

# 2. insert the raster data in database
user_param = self.getConParam()
if subprocess.call(cmd_psql, shell=True) == 0:

# Ok now update the table with specific information not manage by raster2pgsql
try:
DBSession().execute(sql)
DBSession().commit()

# we need to delete temp file
print("delete file " + self.rasterfile + ".sql")
os.remove(self.rasterfile + ".sql")

except DatabaseError as error:
print('Fail to run SQL : %s ' % (error.args[0]))
return False

else:
return False
else:
return False

# everything is OK
return True
Loading

0 comments on commit 4572b5e

Please sign in to comment.