Replace --directoy option with --destination to allow a filename for …

…the target file (#92)

mozdownload/scraper.py

@@ -87,7 +87,7 @@ def __init__(self):
 class Scraper(object):
     """Generic class to download an application from the Mozilla server"""
 
-    def __init__(self, directory, version, platform=None,
+    def __init__(self, destination, version, platform=None,
                  application='firefox', locale=None, extension=None,
                  username=None, password=None,
                  retry_attempts=0, retry_delay=10.,
@@ -99,7 +99,7 @@ def __init__(self, directory, version, platform=None,
         self._target = None
         self._binary = None
 
-        self.directory = directory
+        self.destination = destination
         if not locale:
             if application in MULTI_LOCALE_APPLICATIONS:
                 self.locale = 'multi'
@@ -242,8 +242,13 @@ def target(self):
         """Return the target file name of the build"""
 
         if self._target is None:
-            self._target = os.path.join(self.directory,
-                                        self.build_filename(self.binary))
+
+            # if destination contains filename
+            if os.path.splitext(self.destination)[1]:
+                self._target = self.destination
+            else:
+                self._target = os.path.join(self.destination,
+                                            self.build_filename(self.binary))
         return self._target
 
     def get_build_info(self):
@@ -279,15 +284,16 @@ def total_seconds(td):
 
         attempt = 0
 
-        if not os.path.isdir(self.directory):
-            os.makedirs(self.directory)
-
         # Don't re-download the file
         if os.path.isfile(os.path.abspath(self.target)):
             self.logger.info("File has already been downloaded: %s" %
                              (self.target))
             return
 
+        directory = os.path.dirname(self.target)
+        if not os.path.isdir(directory):
+            os.makedirs(directory)
+
         self.logger.info('Downloading from: %s' %
                          (urllib.unquote(self.final_url)))
         self.logger.info('Saving as: %s' % self.target)
@@ -558,8 +564,15 @@ def __init__(self, url, *args, **kwargs):
     @property
     def target(self):
         target = urlparse(self.final_url)
-        filename = target.path.rpartition('/')[-1] or target.hostname
-        return os.path.join(self.directory, filename)
+
+        # if destination is path to file
+        if os.path.splitext(self.destination)[1]:
+            target_file = self.destination
+        else:
+            source_filename = (target.path.rpartition('/')[-1] or
+                               target.hostname)
+            target_file = os.path.join(self.destination, source_filename)
+        return target_file
 
     @property
     def final_url(self):
@@ -1049,12 +1062,12 @@ def cli():
                       metavar='APPLICATION',
                       help='The name of the application to download, '
                            'default: "%default"')
-    parser.add_option('--directory', '-d',
-                      dest='directory',
+    parser.add_option('--destination', '-d',
+                      dest='destination',
                       default=os.getcwd(),
-                      metavar='DIRECTORY',
-                      help='Target directory for the download, default: '
-                           'current working directory')
+                      metavar='DESTINATION',
+                      help='Directory or file name to download the '
+                           'file to, default: current working directory')
     parser.add_option('--build-number',
                       dest='build_number',
                       type="int",
@@ -1202,7 +1215,7 @@ def cli():
                         'locale': options.locale,
                         'platform': options.platform,
                         'version': options.version,
-                        'directory': options.directory,
+                        'destination': options.destination,
                         'extension': options.extension,
                         'username': options.username,
                         'password': options.password,

tests/base_scraper/test_base_scraper.py

@@ -21,7 +21,7 @@ def test_platform_regex(self):
         """Test for correct platform_regex output"""
 
         for key in mozdownload.PLATFORM_FRAGMENTS:
-            scraper = mozdownload.Scraper(directory=self.temp_dir,
+            scraper = mozdownload.Scraper(destination=self.temp_dir,
                                           version=None,
                                           platform=key)
             self.assertEqual(scraper.platform_regex,
@@ -32,9 +32,9 @@ def test_download(self):
 
         filename = 'download_test.txt'
         # standard download
-        test_url = urljoin(self.wdir, 'download_test.txt')
+        test_url = urljoin(self.wdir, filename)
         scraper = mozdownload.DirectScraper(url=test_url,
-                                            directory=self.temp_dir,
+                                            destination=self.temp_dir,
                                             version=None,
                                             log_level='ERROR')
         scraper.download()
@@ -50,7 +50,7 @@ def test_download(self):
         # RequestException
         test_url1 = urljoin(self.wdir, 'does_not_exist.html')
         scraper1 = mozdownload.DirectScraper(url=test_url1,
-                                             directory=self.temp_dir,
+                                             destination=self.temp_dir,
                                              version=None,
                                              log_level='ERROR')
         self.assertRaises(requests.exceptions.RequestException,
@@ -59,7 +59,7 @@ def test_download(self):
         # Covering retry_attempts
         test_url2 = urljoin(self.wdir, 'does_not_exist.html')
         scraper2 = mozdownload.DirectScraper(url=test_url2,
-                                             directory=self.temp_dir,
+                                             destination=self.temp_dir,
                                              version=None,
                                              retry_attempts=3,
                                              retry_delay=1.0,
@@ -68,7 +68,7 @@ def test_download(self):
                           scraper2.download)
 
     def test_notimplementedexceptions(self):
-        scraper = mozdownload.Scraper(directory=self.temp_dir,
+        scraper = mozdownload.Scraper(destination=self.temp_dir,
                                       version=None, log_level='ERROR')
         for attr in ['binary', 'binary_regex', 'path_regex']:
             self.assertRaises(mozdownload.NotImplementedError, getattr,
@@ -83,14 +83,14 @@ def test_authentication(self):
         basic_auth_url = 'http://mozqa.com/data/mozqa.com/http_auth/basic/'
 
         # test with invalid authentication
-        scraper = mozdownload.DirectScraper(directory=self.temp_dir,
+        scraper = mozdownload.DirectScraper(destination=self.temp_dir,
                                             url=basic_auth_url,
                                             version=None,
                                             log_level='ERROR')
         self.assertRaises(requests.exceptions.HTTPError, scraper.download)
 
         # testing with valid authentication
-        scraper = mozdownload.DirectScraper(directory=self.temp_dir,
+        scraper = mozdownload.DirectScraper(destination=self.temp_dir,
                                             url=basic_auth_url,
                                             version=None,
                                             log_level='ERROR',
@@ -105,13 +105,51 @@ def test_optional_authentication(self):
         optional_auth_url = 'https://ci.mozilla.org/'
 
         # requires optional authentication with no data specified
-        scraper = mozdownload.DirectScraper(directory=self.temp_dir,
+        scraper = mozdownload.DirectScraper(destination=self.temp_dir,
                                             url=optional_auth_url,
                                             version=None,
                                             log_level='ERROR')
         scraper.download()
         self.assertTrue(os.path.isfile(os.path.join(self.temp_dir,
                                                     'ci.mozilla.org')))
 
+    def test_destination(self):
+        """Test for various destination scenarios"""
+
+        filename = 'download_test.txt'
+        test_url = urljoin(self.wdir, filename)
+
+        # destination is directory
+        scraper = mozdownload.DirectScraper(url=test_url,
+                                            destination=self.temp_dir,
+                                            version=None,
+                                            log_level='ERROR')
+        self.assertEqual(scraper.target, os.path.join(self.temp_dir, filename))
+
+        # destination is file
+        destination = os.path.join(self.temp_dir, filename)
+        scraper = mozdownload.DirectScraper(url=test_url,
+                                            destination=destination,
+                                            version=None,
+                                            log_level='ERROR')
+        self.assertEqual(scraper.target, destination)
+
+        # destination directory does not exist
+        destination = os.path.join(self.temp_dir, 'temp_folder', filename)
+        scraper = mozdownload.DirectScraper(url=test_url,
+                                            destination=destination,
+                                            version=None,
+                                            log_level='ERROR')
+        self.assertEqual(scraper.destination, destination)
+
+        # ensure that multiple non existing directories are created
+        destination = os.path.join(self.temp_dir, 'tmp1', 'tmp2', filename)
+        scraper = mozdownload.DirectScraper(url=test_url,
+                                            destination=destination,
+                                            version=None,
+                                            log_level='ERROR')
+        self.assertEqual(scraper.destination, destination)
+
+
 if __name__ == '__main__':
     unittest.main()

tests/daily_scraper/test_daily_indices.py

@@ -50,7 +50,7 @@ def test_build_indices(self):
         """Testing for correct build_index in DailyScraper"""
 
         for entry in test_params:
-            scraper = DailyScraper(directory=self.temp_dir, base_url=self.wdir,
+            scraper = DailyScraper(destination=self.temp_dir, base_url=self.wdir,
                                    version=None, log_level='ERROR',
                                    **entry['args'])
             self.assertEqual(scraper.build_index, entry['build_index'])

tests/daily_scraper/test_daily_scraper.py

@@ -315,7 +315,7 @@ def test_scraper(self):
         """Testing various download scenarios for DailyScraper"""
 
         for entry in tests:
-            scraper = DailyScraper(directory=self.temp_dir, base_url=self.wdir,
+            scraper = DailyScraper(destination=self.temp_dir, base_url=self.wdir,
                                    version=None, log_level='ERROR',
                                    **entry['args'])
 

tests/daily_scraper/test_invalid_branch.py

@@ -32,7 +32,7 @@ def test_invalid_branch(self):
 
         for entry in tests:
             self.assertRaises(NotFoundError, DailyScraper,
-                              directory=self.temp_dir,
+                              destination=self.temp_dir,
                               version=None,
                               base_url=self.wdir,
                               log_level='ERROR',

tests/daily_scraper/test_invalid_date.py

@@ -47,7 +47,7 @@ def test_scraper(self):
 
         for entry in tests:
             self.assertRaises(ValueError, DailyScraper,
-                              directory=self.temp_dir,
+                              destination=self.temp_dir,
                               version=None,
                               base_url=self.wdir,
                               log_level='ERROR',

tests/direct_scraper/test_direct_scraper.py

@@ -25,7 +25,7 @@ def tearDown(self):
     def test_url_download(self):
         test_url = 'https://mozqa.com/index.html'
         scraper = DirectScraper(url=test_url,
-                                directory=self.temp_dir,
+                                destination=self.temp_dir,
                                 version=None,
                                 log_level='ERROR')
         self.assertEqual(scraper.url, test_url)

tests/release_candidate_scraper/test_release_candidate_scraper.py

@@ -135,7 +135,7 @@ def test_scraper(self):
         """Testing various download scenarios for ReleaseCandidateScraper"""
 
         for entry in tests:
-            scraper = ReleaseCandidateScraper(directory=self.temp_dir,
+            scraper = ReleaseCandidateScraper(destination=self.temp_dir,
                                               base_url=self.wdir,
                                               log_level='ERROR',
                                               **entry['args'])

tests/release_candidate_scraper/test_release_candidate_scraper_indices.py

@@ -49,7 +49,7 @@ def test_build_indices(self):
         """Testing indices in choosing builds for ReleaseCandidateScraper"""
 
         for entry in test_params:
-            scraper = ReleaseCandidateScraper(directory=self.temp_dir,
+            scraper = ReleaseCandidateScraper(destination=self.temp_dir,
                                               base_url=self.wdir,
                                               log_level='ERROR',
                                               **entry['args'])

tests/release_scraper/test_release_scraper.py

@@ -134,12 +134,13 @@ def test_scraper(self):
         """Testing various download scenarios for ReleaseScraper"""
 
         for entry in tests:
-            scraper = ReleaseScraper(directory=self.temp_dir, base_url=self.wdir,
+            scraper = ReleaseScraper(destination=self.temp_dir, base_url=self.wdir,
                                      log_level='ERROR', **entry['args'])
             expected_target = os.path.join(self.temp_dir, entry['target'])
             self.assertEqual(scraper.target, expected_target)
             self.assertEqual(urllib.unquote(scraper.final_url),
                              urljoin(self.wdir, entry['target_url']))
 
+
 if __name__ == '__main__':
     unittest.main()

tests/tinderbox_scraper/test_invalid_date.py

@@ -56,7 +56,7 @@ def test_scraper(self):
 
         for entry in tests:
             self.assertRaises(ValueError, TinderboxScraper,
-                              directory=self.temp_dir,
+                              destination=self.temp_dir,
                               version=None,
                               base_url=self.wdir,
                               log_level='ERROR',

tests/tinderbox_scraper/test_tinderbox_scraper.py

@@ -248,13 +248,14 @@ def test_scraper(self):
         """Testing various download scenarios for TinderboxScraper"""
 
         for entry in tests:
-            scraper = TinderboxScraper(directory=self.temp_dir, version=None,
+            scraper = TinderboxScraper(destination=self.temp_dir, version=None,
                                        base_url=self.wdir, log_level='ERROR',
                                        **entry['args'])
             expected_target = os.path.join(self.temp_dir, entry['target'])
             self.assertEqual(scraper.target, expected_target)
             self.assertEqual(urllib.unquote(scraper.final_url),
                              urljoin(self.wdir, entry['target_url']))
 
+
 if __name__ == '__main__':
     unittest.main()

tests/try_scraper/test_invalid_changeset.py

@@ -29,7 +29,7 @@ def test_scraper(self):
 
         for entry in tests:
             self.assertRaises(NotFoundError, TryScraper,
-                              directory=self.temp_dir,
+                              destination=self.temp_dir,
                               version=None,
                               base_url=self.wdir,
                               log_level='ERROR',

tests/try_scraper/test_try_scraper.py

@@ -62,7 +62,7 @@ def test_scraper(self):
         """Testing various download scenarios for TryScraper"""
 
         for entry in tests:
-            scraper = TryScraper(directory=self.temp_dir, version=None,
+            scraper = TryScraper(destination=self.temp_dir, version=None,
                                  base_url=self.wdir, log_level='ERROR',
                                  **entry['args'])
             expected_target = os.path.join(self.temp_dir, entry['target'])