Skip to content

Commit

Permalink
test fix
Browse files Browse the repository at this point in the history
  • Loading branch information
jake authored and jake committed Mar 27, 2024
1 parent a00a601 commit 6a8f6b5
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 15 deletions.
4 changes: 3 additions & 1 deletion tests/test_api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,10 +266,12 @@ def test_upload_validate_identifier():

def test_download(tmpdir):
tmpdir.chdir()
last_mod_header = {"Last-Modified": "Tue, 14 Nov 2023 20:25:48 GMT"}
with IaRequestsMock() as rsps:
rsps.add(responses.GET,
f'{PROTOCOL}//archive.org/download/nasa/nasa_meta.xml',
body='test content')
body='test content',
adding_headers=last_mod_header)
rsps.add_metadata_mock('nasa')
download('nasa', 'nasa_meta.xml')
p = os.path.join(str(tmpdir), 'nasa')
Expand Down
54 changes: 40 additions & 14 deletions tests/test_item.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
DOWNLOAD_URL_RE = re.compile(f'{PROTOCOL}//archive.org/download/.*')
S3_URL_RE = re.compile(r'.*s3.us.archive.org/.*')

EXPECTED_LAST_MOD_HEADER = {"Last-Modified": "Tue, 14 Nov 2023 20:25:48 GMT"}
EXPECTED_S3_HEADERS = {
'content-length': '7557',
'x-archive-queue-derive': '1',
Expand Down Expand Up @@ -145,11 +146,15 @@ def test_get_files_no_matches(nasa_item):
def test_download(tmpdir, nasa_item):
tmpdir.chdir()
with IaRequestsMock() as rsps:
rsps.add(responses.GET, DOWNLOAD_URL_RE, body='test content')
rsps.add(responses.GET, DOWNLOAD_URL_RE,
body='test content',
adding_headers=EXPECTED_LAST_MOD_HEADER)
nasa_item.download(files='nasa_meta.xml')
assert len(tmpdir.listdir()) == 1
with IaRequestsMock() as rsps:
rsps.add(responses.GET, DOWNLOAD_URL_RE, body='new test content')
rsps.add(responses.GET, DOWNLOAD_URL_RE,
body='new test content',
adding_headers=EXPECTED_LAST_MOD_HEADER)
nasa_item.download(files='nasa_meta.xml')
with open('nasa/nasa_meta.xml') as fh:
assert fh.read() == 'new test content'
Expand All @@ -158,7 +163,9 @@ def test_download(tmpdir, nasa_item):
def test_download_io_error(tmpdir, nasa_item):
tmpdir.chdir()
with IaRequestsMock() as rsps:
rsps.add(responses.GET, DOWNLOAD_URL_RE, body='test content')
rsps.add(responses.GET, DOWNLOAD_URL_RE,
body='test content',
adding_headers=EXPECTED_LAST_MOD_HEADER)
nasa_item.download(files='nasa_meta.xml')
rsps.reset()
with pytest.raises(ConnectionError):
Expand All @@ -167,7 +174,9 @@ def test_download_io_error(tmpdir, nasa_item):

def test_download_ignore_errors(tmpdir, nasa_item):
with IaRequestsMock() as rsps:
rsps.add(responses.GET, DOWNLOAD_URL_RE, body='test content')
rsps.add(responses.GET, DOWNLOAD_URL_RE,
body='test content',
adding_headers=EXPECTED_LAST_MOD_HEADER)
nasa_item.download(files='nasa_meta.xml')
nasa_item.download(files='nasa_meta.xml', ignore_errors=True)

Expand All @@ -177,11 +186,13 @@ def test_download_ignore_existing(tmpdir, nasa_item):
with IaRequestsMock(
assert_all_requests_are_fired=False) as rsps:
rsps.add(responses.GET, DOWNLOAD_URL_RE,
body='test content')
body='test content',
adding_headers=EXPECTED_LAST_MOD_HEADER)
nasa_item.download(files='nasa_meta.xml', ignore_existing=True)

rsps.add(responses.GET, DOWNLOAD_URL_RE,
body='new test content')
body='new test content',
adding_headers=EXPECTED_LAST_MOD_HEADER)
nasa_item.download(files='nasa_meta.xml', ignore_existing=True)
with open('nasa/nasa_meta.xml') as fh:
assert fh.read() == 'test content'
Expand All @@ -190,11 +201,15 @@ def test_download_ignore_existing(tmpdir, nasa_item):
def test_download_clobber(tmpdir, nasa_item):
tmpdir.chdir()
with IaRequestsMock() as rsps:
rsps.add(responses.GET, DOWNLOAD_URL_RE, body='test content')
rsps.add(responses.GET, DOWNLOAD_URL_RE,
body='test content',
adding_headers=EXPECTED_LAST_MOD_HEADER)
nasa_item.download(files='nasa_meta.xml')

rsps.reset()
rsps.add(responses.GET, DOWNLOAD_URL_RE, body='new test content')
rsps.add(responses.GET, DOWNLOAD_URL_RE,
body='new test content',
adding_headers=EXPECTED_LAST_MOD_HEADER)
nasa_item.download(files='nasa_meta.xml')
assert load_file('nasa/nasa_meta.xml') == 'new test content'

Expand All @@ -205,8 +220,12 @@ def test_download_checksum(tmpdir, caplog):
# test overwrite based on checksum.
with IaRequestsMock() as rsps:
rsps.add_metadata_mock('nasa')
rsps.add(responses.GET, DOWNLOAD_URL_RE, body='test content')
rsps.add(responses.GET, DOWNLOAD_URL_RE, body='overwrite based on md5')
rsps.add(responses.GET, DOWNLOAD_URL_RE,
body='test content',
adding_headers=EXPECTED_LAST_MOD_HEADER)
rsps.add(responses.GET, DOWNLOAD_URL_RE,
body='overwrite based on md5',
adding_headers=EXPECTED_LAST_MOD_HEADER)

nasa_item = get_item('nasa')
nasa_item.download(files='nasa_meta.xml')
Expand All @@ -218,7 +237,8 @@ def test_download_checksum(tmpdir, caplog):
with caplog.at_level(logging.DEBUG):
rsps.reset()
rsps.add(responses.GET, DOWNLOAD_URL_RE,
body=load_test_data_file('nasa_meta.xml'))
body=load_test_data_file('nasa_meta.xml'),
adding_headers=EXPECTED_LAST_MOD_HEADER)
nasa_item.download(files='nasa_meta.xml', checksum=True, verbose=True)
nasa_item.download(files='nasa_meta.xml', checksum=True, verbose=True)

Expand All @@ -229,7 +249,9 @@ def test_download_checksum(tmpdir, caplog):
def test_download_destdir(tmpdir, nasa_item):
tmpdir.chdir()
with IaRequestsMock() as rsps:
rsps.add(responses.GET, DOWNLOAD_URL_RE, body='new destdir')
rsps.add(responses.GET, DOWNLOAD_URL_RE,
body='new destdir',
adding_headers=EXPECTED_LAST_MOD_HEADER)
dest = os.path.join(str(tmpdir), 'new destdir')
nasa_item.download(files='nasa_meta.xml', destdir=dest)
assert 'nasa' in os.listdir(dest)
Expand All @@ -241,7 +263,9 @@ def test_download_no_directory(tmpdir, nasa_item):
url_re = re.compile(f'{PROTOCOL}//archive.org/download/.*')
tmpdir.chdir()
with IaRequestsMock() as rsps:
rsps.add(responses.GET, url_re, body='no dest dir')
rsps.add(responses.GET, url_re,
body='no dest dir',
adding_headers=EXPECTED_LAST_MOD_HEADER)
nasa_item.download(files='nasa_meta.xml', no_directory=True)
with open(os.path.join(str(tmpdir), 'nasa_meta.xml')) as fh:
assert fh.read() == 'no dest dir'
Expand Down Expand Up @@ -278,9 +302,11 @@ def test_download_dry_run_on_the_fly_formats(tmpdir, capsys, nasa_item):
def test_download_verbose(tmpdir, capsys, nasa_item):
tmpdir.chdir()
with IaRequestsMock(assert_all_requests_are_fired=False) as rsps:
headers = {'content-length': '11'}
headers.update(EXPECTED_LAST_MOD_HEADER)
rsps.add(responses.GET, DOWNLOAD_URL_RE,
body='no dest dir',
adding_headers={'content-length': '11'})
adding_headers=headers)
nasa_item.download(files='nasa_meta.xml', verbose=True)
out, err = capsys.readouterr()
assert 'downloading nasa_meta.xml' in err
Expand Down

0 comments on commit 6a8f6b5

Please sign in to comment.