From 0d58299dd8f9565938e62c4cfd744c94332672c4 Mon Sep 17 00:00:00 2001 From: studentbrad Date: Thu, 16 Apr 2020 17:58:43 -0400 Subject: [PATCH 1/7] relocacted tests folder + housekeeping --- jobfunnel/tests/__init__.py | 0 setup.py | 2 +- {jobfunnel/tests => tests}/conftest.py | 8 +++++--- .../tests => tests}/json/cities_america.json | 0 .../tests => tests}/json/cities_canada.json | 0 {jobfunnel/tests => tests}/test_countries.py | 8 ++++---- {jobfunnel/tests => tests}/test_filters.py | 20 +++++++++++-------- {jobfunnel/tests => tests}/test_parse.py | 9 +++++---- {jobfunnel/tests => tests}/test_tools.py | 11 +++++----- {jobfunnel/tests => tests}/test_validate.py | 16 +++++++-------- 10 files changed, 40 insertions(+), 34 deletions(-) delete mode 100644 jobfunnel/tests/__init__.py rename {jobfunnel/tests => tests}/conftest.py (96%) rename {jobfunnel/tests => tests}/json/cities_america.json (100%) rename {jobfunnel/tests => tests}/json/cities_canada.json (100%) rename {jobfunnel/tests => tests}/test_countries.py (95%) rename {jobfunnel/tests => tests}/test_filters.py (84%) rename {jobfunnel/tests => tests}/test_parse.py (97%) rename {jobfunnel/tests => tests}/test_tools.py (96%) rename {jobfunnel/tests => tests}/test_validate.py (91%) diff --git a/jobfunnel/tests/__init__.py b/jobfunnel/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/setup.py b/setup.py index 3014aa20..db3724d1 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,6 @@ license = 'MIT License', python_requires = '>=3.6.0', install_requires = requires, - packages = find_packages(exclude=('demo',)), + packages = find_packages(exclude=('demo', 'tests')), include_package_data = True, entry_points = {'console_scripts': ['funnel = jobfunnel.__main__:main']}) diff --git a/jobfunnel/tests/conftest.py b/tests/conftest.py similarity index 96% rename from jobfunnel/tests/conftest.py rename to tests/conftest.py index 86da83a7..9cb2cdb9 100644 --- a/jobfunnel/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,11 @@ import pytest -from unittest.mock import patch -from ..config.parser import parse_config -from ..tools.tools import config_factory import sys +from unittest.mock import patch + +from jobfunnel.config.parser import parse_config +from jobfunnel.tools.tools import config_factory + @pytest.fixture() def configure_options(): diff --git a/jobfunnel/tests/json/cities_america.json b/tests/json/cities_america.json similarity index 100% rename from jobfunnel/tests/json/cities_america.json rename to tests/json/cities_america.json diff --git a/jobfunnel/tests/json/cities_canada.json b/tests/json/cities_canada.json similarity index 100% rename from jobfunnel/tests/json/cities_canada.json rename to tests/json/cities_canada.json diff --git a/jobfunnel/tests/test_countries.py b/tests/test_countries.py similarity index 95% rename from jobfunnel/tests/test_countries.py rename to tests/test_countries.py index e6628832..6f904e2f 100644 --- a/jobfunnel/tests/test_countries.py +++ b/tests/test_countries.py @@ -10,11 +10,11 @@ from typing import Union from unittest.mock import patch -from ..config.parser import parse_config +from jobfunnel.config.parser import parse_config +from jobfunnel.indeed import Indeed +from jobfunnel.monster import Monster +from jobfunnel.glassdoor import GlassDoor -from ..indeed import Indeed -from ..monster import Monster -from ..glassdoor import GlassDoor PROVIDERS = {'indeed': Indeed, 'monster': Monster, 'glassdoor': GlassDoor} diff --git a/jobfunnel/tests/test_filters.py b/tests/test_filters.py similarity index 84% rename from jobfunnel/tests/test_filters.py rename to tests/test_filters.py index cd859cbb..cb64d7cd 100644 --- a/jobfunnel/tests/test_filters.py +++ b/tests/test_filters.py @@ -1,8 +1,12 @@ -from ..tools.filters import tfidf_filter, id_filter, date_filter +import pytest + from collections import OrderedDict from datetime import datetime, timedelta from unittest.mock import patch +from jobfunnel.tools.filters import tfidf_filter, id_filter, date_filter + + attr_list = [[['blurb'], 'Looking for a passionate team player that is willing to learn new technologies. Our company X is still growing at an exponential rate. In order to be a perfect fit' ' you must tell us your favorite movie at the interview; favorite food; and a fun fact about yourself. The ideal fit will also know Python and SDLC.'], [['blurb'], 'Looking for a passionate developer that is willing to learn new technologies. Our company X is still growing at an exponential rate. In order to be a perfect fit' @@ -14,13 +18,13 @@ def test_date_filter(per_id_job_listings): new_job_listings = per_id_job_listings([attr_list[0], attr_list[1]]) - # Assign two different dates to the job_postings + # assign two different dates to the job_postings job_date = datetime.now() - timedelta(days=10) new_job_listings['0']['date'] = job_date.strftime('%Y-%m-%d') job_date = datetime.now() - timedelta(days=3) new_job_listings['1']['date'] = job_date.strftime('%Y-%m-%d') date_filter(new_job_listings, 5) - # Assert that that jobs older than 5 days have been removed + # assert that that jobs older than 5 days have been removed assert list(new_job_listings) == ['1'] @@ -30,16 +34,16 @@ def test_id_filter(per_id_job_listings): previous_job_listings = per_id_job_listings([attr_list[1], attr_list[3]]) id_filter(new_job_listings, previous_job_listings, new_job_listings['0']['provider']) - # Assert that the new job listings have been removed since they already exist + # assert that the new job listings have been removed since they already exist assert len(new_job_listings) == 0 - # Assert that the correct job ids are in the new filtered new_job_listings + # assert that the correct job ids are in the new filtered new_job_listings assert list(previous_job_listings) == ['0', '1'] def test_tfidf_filter_no_previous_scrape(per_id_job_listings): new_job_listings = per_id_job_listings(attr_list[0:4]) tfidf_filter(new_job_listings) - # Assert that the correct job ids are in the new filtered new_job_listings + # assert that the correct job ids are in the new filtered new_job_listings assert list(new_job_listings) == ['1', '3'] @@ -49,7 +53,7 @@ def test_tfidf_filter_with_previous_scrape(per_id_job_listings): previous_job_listings = per_id_job_listings( [attr_list[1], attr_list[3]], first_job_id=2) tfidf_filter(new_job_listings, previous_job_listings) - # Assert that the new job listings have been removed since they already exist + # assert that the new job listings have been removed since they already exist assert len(new_job_listings) == 0 - # Assert that the correct job ids are in the new filtered new_job_listings + # assert that the correct job ids are in the new filtered new_job_listings assert list(previous_job_listings) == ['2', '3'] diff --git a/jobfunnel/tests/test_parse.py b/tests/test_parse.py similarity index 97% rename from jobfunnel/tests/test_parse.py rename to tests/test_parse.py index a2bfd264..90e283cf 100644 --- a/jobfunnel/tests/test_parse.py +++ b/tests/test_parse.py @@ -3,10 +3,11 @@ import os import yaml -from unittest.mock import patch from pathlib import Path +from unittest.mock import patch + +from jobfunnel.config.parser import parse_config, parse_cli, cli_to_yaml, update_yaml, check_config_types, log_levels -from ..config.parser import parse_config, parse_cli, cli_to_yaml, update_yaml, check_config_types, log_levels config_dict = { 'output_path': 'fish', @@ -25,7 +26,7 @@ } cli_options = [ - ['', '-s', 'settins.yaml'], + ['', '-s', 'settings.yaml'], ['', '-o', '.'], ['', '-kw', 'java', 'python'], ['', '-p', 'ON'], @@ -70,7 +71,7 @@ def test_parse_cli_to_yaml_pass(option): cli_to_yaml(cli) -# Create config fixture to avoid code duplication +# create config fixture to avoid code duplication @pytest.fixture() def config_dependency(): diff --git a/jobfunnel/tests/test_tools.py b/tests/test_tools.py similarity index 96% rename from jobfunnel/tests/test_tools.py rename to tests/test_tools.py index 21f8f0e6..2889e0ec 100644 --- a/jobfunnel/tests/test_tools.py +++ b/tests/test_tools.py @@ -1,9 +1,10 @@ import pytest + +from dateutil.relativedelta import relativedelta from datetime import datetime, timedelta -from ..tools.tools import split_url, proxy_dict_to_url, config_factory, post_date_from_relative_post_age, filter_non_printables +from jobfunnel.tools.tools import split_url, proxy_dict_to_url, config_factory, post_date_from_relative_post_age, filter_non_printables -from dateutil.relativedelta import relativedelta URLS = [ { @@ -67,8 +68,7 @@ [['some_option'], 'option_value'] ] -# Test clean/dirty characters that may be on title and blurb fields - +# test clean/dirty characters that may be on title and blurb fields def test_filter_non_printables_clean_title(job_listings): job_list = job_listings(attr_list[0:1]) @@ -93,8 +93,7 @@ def test_filter_non_printables_diryt_blurb(job_listings): filter_non_printables(job_list[0]) assert job_list[0]['blurb'] == 'Develop and design software' -# Test job_listing dates with all possible formats - +# test job_listing dates with all possible formats def test_post_date_from_relative_post_age_just_posted_pass(job_listings): job_list = job_listings(attr_list[4:5]) diff --git a/jobfunnel/tests/test_validate.py b/tests/test_validate.py similarity index 91% rename from jobfunnel/tests/test_validate.py rename to tests/test_validate.py index bbcbcdf2..644c0f5e 100644 --- a/jobfunnel/tests/test_validate.py +++ b/tests/test_validate.py @@ -3,9 +3,9 @@ from unittest.mock import patch -from ..config.parser import parse_config -from ..config.validate import validate_config, validate_delay, validate_region -from ..tools.tools import config_factory +from jobfunnel.config.parser import parse_config +from jobfunnel.config.validate import validate_config, validate_delay, validate_region +from jobfunnel.tools.tools import config_factory # define config dictionaries that are not valid @@ -25,8 +25,8 @@ [['log_path'], 'data/jobfunnel_.log'], [['filter_list_path'], 'data/filter_list_.json'] ] -# Test all paths with invalid values +# test all paths with invalid values def test_filter_list_path_fail(configure_options): path_configs = config_factory( @@ -121,7 +121,7 @@ def test_delay_min_delay_fail(configure_options): assert str(e.value) == '(min)_delay' -# Test validate_delay with a min_delay greater than delay +# test validate_delay with a min_delay greater than delay def test_delay_min_delay_greater_than_delay_fail(configure_options): delay_configs = config_factory(configure_options(['']), attr_list[5: 6])[0] @@ -130,7 +130,7 @@ def test_delay_min_delay_greater_than_delay_fail(configure_options): assert str(e.value) == '(min)_delay' -# Test validate_delay with a delay less than 10(the minimum) +# test validate_delay with a delay less than 10(the minimum) def test_delay_less_than_10_fail(configure_options): delay_configs = config_factory(configure_options(['']), attr_list[7: 8])[0] @@ -139,7 +139,7 @@ def test_delay_less_than_10_fail(configure_options): assert str(e.value) == '(min)_delay' -# Test validate_delay with the original configuration +# test validate_delay with the original configuration def test_delay_pass(configure_options): validate_delay(configure_options([''])['delay_config']) @@ -155,7 +155,7 @@ def test_delay_max_listing_days_fail(configure_options): assert str(e.value) == 'max_listing_days' -# Test the integration of all parts with the config as a whole +# test the integration of all parts with the config as a whole @pytest.mark.parametrize('attribute', attr_list) def test_config_fail(configure_options, attribute): From f8de91837faaf51db556eac4cb5e6b46f99ae422 Mon Sep 17 00:00:00 2001 From: studentbrad Date: Thu, 16 Apr 2020 20:26:29 -0400 Subject: [PATCH 2/7] fixed relative path --- tests/test_parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_parse.py b/tests/test_parse.py index 90e283cf..86dd276e 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -87,7 +87,7 @@ def setup(default_path='config/settings.yaml', patch_path=None): """ # find the jobfunnel root dir jobfunnel_path = os.path.normpath( - os.path.join(os.path.dirname(__file__), '..')) + os.path.join(os.path.dirname(__file__), '../jobfunnel')) # load the default settings default_yaml_path = os.path.join(jobfunnel_path, default_path) From ad57f293415775fed30c3e34c1a374fa0b769387 Mon Sep 17 00:00:00 2001 From: studentbrad Date: Thu, 16 Apr 2020 20:26:57 -0400 Subject: [PATCH 3/7] fixed cov report path in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index dbda04c2..ea3a32c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,6 @@ install: before_script: 'flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics' script: - 'python -m jobfunnel -s demo/settings.yaml -o demo/' - - 'pytest --cov --cov-report=xml jobfunnel' + - 'pytest --cov --cov-report=xml' after_success: - 'bash <(curl -s https://codecov.io/bash)' From d23b0a8101556544d161adf98335bb323f6762d3 Mon Sep 17 00:00:00 2001 From: studentbrad Date: Thu, 16 Apr 2020 21:34:37 -0400 Subject: [PATCH 4/7] unittest failed at midnight --- tests/test_tools.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_tools.py b/tests/test_tools.py index 2889e0ec..c83971fc 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -110,13 +110,17 @@ def test_post_date_from_relative_post_age_today_pass(job_listings): def test_post_date_from_relative_post_age_1_hour_ago_pass(job_listings): job_list = job_listings(attr_list[6:7]) post_date_from_relative_post_age(job_list) - assert datetime.now().strftime('%Y-%m-%d') == job_list[0]['date'] + now = datetime.now() + assert now.strftime('%Y-%m-%d') == job_list[0]['date'] or \ + (now - timedelta(days=int(1))).strftime('%Y-%m-%d') == job_list[0]['date'] def test_post_date_from_relative_post_age_2_hours_ago_pass(job_listings): job_list = job_listings(attr_list[7:8]) post_date_from_relative_post_age(job_list) - assert datetime.now().strftime('%Y-%m-%d') == job_list[0]['date'] + now = datetime.now() + assert now.strftime('%Y-%m-%d') == job_list[0]['date'] or \ + (now - timedelta(days=int(2))).strftime('%Y-%m-%d') == job_list[0]['date'] def test_post_date_from_relative_ago_post_age_yesterday_ago_pass(job_listings): From 94798ba3557cd710363fef6f26133a9e6db695ec Mon Sep 17 00:00:00 2001 From: studentbrad Date: Thu, 16 Apr 2020 21:51:55 -0400 Subject: [PATCH 5/7] changed date pushback from two to one --- tests/test_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_tools.py b/tests/test_tools.py index c83971fc..30337220 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -120,7 +120,7 @@ def test_post_date_from_relative_post_age_2_hours_ago_pass(job_listings): post_date_from_relative_post_age(job_list) now = datetime.now() assert now.strftime('%Y-%m-%d') == job_list[0]['date'] or \ - (now - timedelta(days=int(2))).strftime('%Y-%m-%d') == job_list[0]['date'] + (now - timedelta(days=int(1))).strftime('%Y-%m-%d') == job_list[0]['date'] def test_post_date_from_relative_ago_post_age_yesterday_ago_pass(job_listings): From 9f64971f85208cd7b5c4ea4bb07c7377906e1ac0 Mon Sep 17 00:00:00 2001 From: studentbrad Date: Thu, 16 Apr 2020 22:22:16 -0400 Subject: [PATCH 6/7] updated version number --- jobfunnel/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobfunnel/__init__.py b/jobfunnel/__init__.py index 503eeb92..cfb007cb 100644 --- a/jobfunnel/__init__.py +++ b/jobfunnel/__init__.py @@ -1 +1 @@ -__version__ = '2.1.4' +__version__ = '2.1.5' From 7dc5b132107f2ce7d0061e93f8791f263645301b Mon Sep 17 00:00:00 2001 From: studentbrad Date: Fri, 17 Apr 2020 13:15:34 -0400 Subject: [PATCH 7/7] omit tests from coverage report --- .travis.yml | 2 +- tests/__init__.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 tests/__init__.py diff --git a/.travis.yml b/.travis.yml index ea3a32c3..334dbaed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,6 @@ install: before_script: 'flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics' script: - 'python -m jobfunnel -s demo/settings.yaml -o demo/' - - 'pytest --cov --cov-report=xml' + - 'pytest --cov=jobfunnel --cov-report=xml' after_success: - 'bash <(curl -s https://codecov.io/bash)' diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b