updated v0.4.0

- Fixed issue with placing reprojection request. - Downloader can now download partial as well as completely successful orders. - Added retry method for rate limit during downloading - General improvements
tyson-swetnam · Aug 19, 2019 · 8f16077 · 8f16077
1 parent b7fbd2b
commit 8f16077
Show file tree

Hide file tree

Showing 12 changed files with 88 additions and 48 deletions.
diff --git a/README.md b/README.md
@@ -289,6 +289,12 @@ A simple setup would be
 
 ## Changelog
 
+### v0.4.0
+- Fixed issue with placing reprojection request.
+- Downloader can now download partial as well as completely successful orders.
+- Added retry method for rate limit during downloading
+- General improvements
+
 ### v0.3.9
 - Removed deprecated bundles from bundles list.
 - Improved parameter description

diff --git a/dist/porder-0.3.9-py2.py3-none-any.whl b/dist/porder-0.3.9-py2.py3-none-any.whl
diff --git a/dist/porder-0.3.9.tar.gz b/dist/porder-0.3.9.tar.gz
diff --git a/dist/porder-0.4.0-py2.py3-none-any.whl b/dist/porder-0.4.0-py2.py3-none-any.whl
diff --git a/dist/porder-0.4.0.tar.gz b/dist/porder-0.4.0.tar.gz
diff --git a/porder/__init__.py b/porder/__init__.py
@@ -2,4 +2,4 @@
 
 __author__ = 'Samapriya Roy'
 __email__ = '[email protected]'
-__version__ = '0.3.9'
+__version__ = '0.4.0'
diff --git a/porder/async_down.py b/porder/async_down.py
@@ -84,7 +84,8 @@ def funct(url,final,ext):
         for z in bar(range(60)):
             time.sleep(1)
         response=SESSION.get(url).json()
-    if response['state']=='success':
+    if response['state']=='success' or response['state']=='partial':
+        print('Order completed with status: '+str(response['state']))
         for items in response['_links']['results']:
             url=(items['location'])
             url_to_check = url if url.startswith('https') else "http://%s" % url

diff --git a/porder/async_downloader.py b/porder/async_downloader.py
@@ -56,7 +56,34 @@ def check_for_redirects(url):
         if r.status_code == 429:  # Too many requests
             raise Exception("rate limit error")
 
-#Get the redirects and download
+
+@retry(
+    wait_exponential_multiplier=1000,
+    wait_exponential_max=10000)
+def downonly(redirect_url,local_path,ext,items):
+    result=SESSION.get(redirect_url)
+    if not os.path.exists(local_path) and result.status_code==200:
+        if ext is not None:
+            if local_path.endswith(ext):
+                print("Downloading: " + str(local_path))
+                obj = SmartDL(redirect_url, local_path)
+                obj.start()
+                path = obj.get_dest()
+        elif ext is None:
+                print("Downloading: " + str(local_path))
+                obj = SmartDL(redirect_url, local_path)
+                obj.start()
+                path = obj.get_dest()
+    elif result.status_code==429:
+        raise Exception("rate limit error")
+    else:
+        if int(result.status_code)!=200:
+            print("Encountered error with code: " + str(result.status_code)+' for '+str(os.path.split(items['name'])[-1]))
+        elif int(result.status_code)==200:
+            print("File already exists SKIPPING: "+str(os.path.split(items['name'])[-1]))
+
+
+# Get the redirects and download
 def asyncdownload(url,local,ext):
     response=SESSION.get(url).json()
     print("Polling ...")
@@ -65,32 +92,22 @@ def asyncdownload(url,local,ext):
         for z in bar(range(60)):
             time.sleep(1)
         response=SESSION.get(url).json()
-    if response['state']=='success':
+    if response['state']=='success' or response['state']=='partial':
+        print('Order completed with status: '+str(response['state']))
         for items in response['_links']['results']:
             url=(items['location'])
             url_to_check = url if url.startswith('https') else "http://%s" % url
             redirect_url = check_for_redirects(url_to_check)
             if redirect_url.startswith('https'):
                 #print('Processing redirect link for '+str(os.path.split(items['name'])[-1]))
                 local_path=os.path.join(local,str(os.path.split(items['name'])[-1]))
-                result=SESSION.get(redirect_url)
-                if not os.path.exists(local_path) and result.status_code==200:
-                    if ext is not None:
-                        if local_path.endswith(ext):
-                            print("Downloading: " + str(local_path))
-                            obj = SmartDL(redirect_url, local_path)
-                            obj.start()
-                            path = obj.get_dest()
-                    elif ext is None:
-                            print("Downloading: " + str(local_path))
-                            obj = SmartDL(redirect_url, local_path)
-                            obj.start()
-                            path = obj.get_dest()
-                else:
-                    if int(result.status_code)!=200:
-                        print("Encountered error with code: " + str(result.status_code)+' for '+str(os.path.split(items['name'])[-1]))
-                    elif int(result.status_code)==200:
-                        print("File already exists SKIPPING: "+str(os.path.split(items['name'])[-1]))
+                try:
+                    downonly(redirect_url,local_path,ext,items)
+                except Exception as e:
+                    print(e)
+                except (KeyboardInterrupt, SystemExit) as e:
+                    print('\n'+'Program escaped by User')
+                    sys.exit()
     else:
         print('Order Failed with state: '+str(response['state']))
 ##download(url="https://api.planet.com/compute/ops/orders/v2/0ee9e923-59fc-4c31-8632-9882cb342708",

diff --git a/porder/downloader.py b/porder/downloader.py
@@ -58,6 +58,35 @@ def check_for_redirects(url):
             raise Exception("rate limit error")
 
 #Get the redirects and download
+@retry(
+    wait_exponential_multiplier=1000,
+    wait_exponential_max=10000)
+def downonly(redirect_url,local_path,ext,items):
+    result=SESSION.get(redirect_url)
+    if not os.path.exists(local_path) and result.status_code==200:
+        if ext is not None:
+            if local_path.endswith(ext):
+                print("Downloading: " + str(local_path))
+                f = open(local_path, 'wb')
+                for chunk in result.iter_content(chunk_size=512 * 1024):
+                    if chunk:
+                        f.write(chunk)
+                f.close()
+        elif ext is None:
+            print("Downloading: " + str(local_path))
+            f = open(local_path, 'wb')
+            for chunk in result.iter_content(chunk_size=512 * 1024):
+                if chunk:
+                    f.write(chunk)
+            f.close()
+    elif result.status_code==429:
+        raise Exception("rate limit error")
+    else:
+        if int(result.status_code)!=200:
+            print("Encountered error with code: " + str(result.status_code)+' for '+str(os.path.split(items['name'])[-1]))
+        elif int(result.status_code)==200:
+            print("File already exists SKIPPING: "+str(os.path.split(items['name'])[-1]))
+
 def download(url,local,ext):
     response=SESSION.get(url).json()
     print("Polling ...")
@@ -66,36 +95,22 @@ def download(url,local,ext):
         for z in bar(range(60)):
             time.sleep(1)
         response=SESSION.get(url).json()
-    if response['state']=='success':
+    if response['state']=='success' or response['state']=='partial':
+        print('Order completed with status: '+str(response['state']))
         for items in response['_links']['results']:
             url=(items['location'])
             url_to_check = url if url.startswith('https') else "http://%s" % url
             redirect_url = check_for_redirects(url_to_check)
             if redirect_url.startswith('https'):
                 #print('Processing redirect link for '+str(os.path.split(items['name'])[-1]))
                 local_path=os.path.join(local,str(os.path.split(items['name'])[-1]))
-                result=SESSION.get(redirect_url)
-                if not os.path.exists(local_path) and result.status_code==200:
-                    if ext is not None:
-                        if local_path.endswith(ext):
-                            print("Downloading: " + str(local_path))
-                            f = open(local_path, 'wb')
-                            for chunk in result.iter_content(chunk_size=512 * 1024):
-                                if chunk:
-                                    f.write(chunk)
-                            f.close()
-                    elif ext is None:
-                        print("Downloading: " + str(local_path))
-                        f = open(local_path, 'wb')
-                        for chunk in result.iter_content(chunk_size=512 * 1024):
-                            if chunk:
-                                f.write(chunk)
-                        f.close()
-                else:
-                    if int(result.status_code)!=200:
-                        print("Encountered error with code: " + str(result.status_code)+' for '+str(os.path.split(items['name'])[-1]))
-                    elif int(result.status_code)==200:
-                        print("File already exists SKIPPING: "+str(os.path.split(items['name'])[-1]))
+                try:
+                    downonly(redirect_url,local_path,ext,items)
+                except Exception as e:
+                    print(e)
+                except (KeyboardInterrupt, SystemExit) as e:
+                    print('\n'+'Program escaped by User')
+                    sys.exit()
     else:
         print('Order Failed with state: '+str(response['state']))
 # download(url="https://api.planet.com/compute/ops/orders/v2/0ee9e923-59fc-4c31-8632-9882cb342708",

diff --git a/porder/multiproc_pydl.py b/porder/multiproc_pydl.py
@@ -125,7 +125,8 @@ def funct(url,final,ext):
         for z in bar(range(60)):
             time.sleep(1)
         response=SESSION.get(url).json()
-    if response['state']=='success':
+    if response['state']=='success' or response['state']=='partial':
+        print('Order completed with status: '+str(response['state']))
         for items in response['_links']['results']:
             url=(items['location'])
             url_to_check = url if url.startswith('https') else "http://%s" % url

diff --git a/porder/order_now.py b/porder/order_now.py
@@ -119,7 +119,7 @@ def order(**kwargs):
                     dbundle.update(dgcs)
                 elif items=='composite':
                     dbundle['tools'].append(dcomposite)
-                elif items=='reproject':
+                elif items=='projection':
                     dbundle['tools'].append(dreproject)
                 elif items=='ndvi':
                     dndvi={"pixel_type": "32R","ndvi": "(b4 - b3) / (b4+b3)"}

diff --git a/setup.py b/setup.py
@@ -20,7 +20,7 @@ def readme():
         return f.read()
 setuptools.setup(
     name='porder',
-    version='0.3.9',
+    version='0.4.0',
     packages=['porder'],
     url='https://github.com/samapriya/porder',
     package_data={'': ['bundles.json']},