diff --git a/requirements/dev.txt b/requirements/dev.txt index d493eb2d..6ba63753 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -5,12 +5,12 @@ # pip-compile --allow-unsafe --config=pyproject.toml requirements/dev.in # alembic==1.11.1 - # via -r prod.txt + # via -r requirements/prod.txt asttokens==2.2.1 # via stack-data attrs==20.3.0 # via - # -r prod.txt + # -r requirements/prod.txt # jsonschema backcall==0.2.0 # via ipython @@ -18,76 +18,76 @@ build==0.8.0 # via pip-tools cachetools==4.2.2 # via - # -r prod.txt + # -r requirements/prod.txt # google-auth certifi==2022.12.7 # via - # -r prod.txt + # -r requirements/prod.txt # requests # sentry-sdk cffi==1.15.1 # via - # -r prod.txt + # -r requirements/prod.txt # cryptography charset-normalizer==2.0.1 # via - # -r prod.txt + # -r requirements/prod.txt # requests checkmatelib==1.0.15 - # via -r prod.txt + # via -r requirements/prod.txt click==8.1.3 # via pip-tools cryptography==41.0.2 # via - # -r prod.txt + # -r requirements/prod.txt # python-jose decorator==5.0.7 # via ipython ecdsa==0.18.0 # via - # -r prod.txt + # -r requirements/prod.txt # python-jose executing==1.2.0 # via stack-data factory-boy==3.2.1 - # via -r dev.in + # via -r requirements/dev.in faker==18.9.0 # via factory-boy google-auth==2.16.0 # via - # -r prod.txt + # -r requirements/prod.txt # google-auth-oauthlib google-auth-oauthlib==1.0.0 - # via -r prod.txt + # via -r requirements/prod.txt greenlet==2.0.2 # via - # -r prod.txt + # -r requirements/prod.txt # sqlalchemy gunicorn==20.1.0 - # via -r prod.txt + # via -r requirements/prod.txt h-assets==1.0.5 - # via -r prod.txt + # via -r requirements/prod.txt h-pyramid-sentry==1.2.4 - # via -r prod.txt + # via -r requirements/prod.txt h-vialib==1.2.1 - # via -r prod.txt + # via -r requirements/prod.txt hupper==1.10.2 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid idna==2.10 # via - # -r prod.txt + # -r requirements/prod.txt # requests importlib-metadata==6.0.0 # via - # -r prod.txt + # -r requirements/prod.txt # alembic # h-vialib # pip-sync-faster importlib-resources==5.12.0 # via - # -r prod.txt + # -r requirements/prod.txt # alembic # checkmatelib ipython==8.10.0 @@ -98,41 +98,41 @@ jedi==0.18.0 # via ipython jinja2==2.11.3 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid-jinja2 jsonschema==3.2.0 # via - # -r prod.txt + # -r requirements/prod.txt # checkmatelib mako==1.2.4 # via - # -r prod.txt + # -r requirements/prod.txt # alembic markupsafe==1.1.1 # via - # -r prod.txt + # -r requirements/prod.txt # jinja2 # mako # pyramid-jinja2 marshmallow==3.19.0 # via - # -r prod.txt + # -r requirements/prod.txt # webargs matplotlib-inline==0.1.3 # via ipython netaddr==0.8.0 # via - # -r prod.txt + # -r requirements/prod.txt # checkmatelib newrelic==8.8.1 - # via -r prod.txt + # via -r requirements/prod.txt oauthlib==3.2.2 # via - # -r prod.txt + # -r requirements/prod.txt # requests-oauthlib packaging==21.3 # via - # -r prod.txt + # -r requirements/prod.txt # build # marshmallow # webargs @@ -140,7 +140,7 @@ parso==0.8.2 # via jedi pastedeploy==2.1.1 # via - # -r prod.txt + # -r requirements/prod.txt # plaster-pastedeploy pep517==0.13.0 # via build @@ -149,53 +149,53 @@ pexpect==4.8.0 pickleshare==0.7.5 # via ipython pip-sync-faster==0.0.3 - # via -r dev.in + # via -r requirements/dev.in pip-tools==6.14.0 # via - # -r dev.in + # -r requirements/dev.in # pip-sync-faster plaster==1.0 # via - # -r prod.txt + # -r requirements/prod.txt # plaster-pastedeploy # pyramid plaster-pastedeploy==0.7 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid prompt-toolkit==3.0.36 # via ipython psycopg2==2.9.6 - # via -r prod.txt + # via -r requirements/prod.txt ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 # via stack-data pyasn1==0.4.8 # via - # -r prod.txt + # -r requirements/prod.txt # pyasn1-modules # python-jose # rsa pyasn1-modules==0.2.8 # via - # -r prod.txt + # -r requirements/prod.txt # google-auth pycparser==2.21 # via - # -r prod.txt + # -r requirements/prod.txt # cffi pygments==2.15.0 # via ipython pyjwt==2.7.0 - # via -r prod.txt + # via -r requirements/prod.txt pyparsing==3.0.7 # via - # -r prod.txt + # -r requirements/prod.txt # packaging pyramid==2.0 # via - # -r prod.txt + # -r requirements/prod.txt # h-assets # h-pyramid-sentry # pyramid-exclog @@ -204,59 +204,59 @@ pyramid==2.0 # pyramid-sanity # pyramid-services pyramid-exclog==1.1 - # via -r prod.txt + # via -r requirements/prod.txt pyramid-ipython==0.2 - # via -r dev.in + # via -r requirements/dev.in pyramid-jinja2==2.10 - # via -r prod.txt + # via -r requirements/prod.txt pyramid-sanity==1.0.3 - # via -r prod.txt + # via -r requirements/prod.txt pyramid-services==2.2 - # via -r prod.txt + # via -r requirements/prod.txt pyrsistent==0.17.3 # via - # -r prod.txt + # -r requirements/prod.txt # jsonschema python-dateutil==2.8.2 # via faker python-jose[cryptography]==3.3.0 # via - # -r prod.txt + # -r requirements/prod.txt # h-vialib requests==2.31.0 # via - # -r prod.txt + # -r requirements/prod.txt # checkmatelib # requests-oauthlib # youtube-transcript-api requests-oauthlib==1.3.0 # via - # -r prod.txt + # -r requirements/prod.txt # google-auth-oauthlib rsa==4.7.2 # via - # -r prod.txt + # -r requirements/prod.txt # google-auth # python-jose sentry-sdk==1.14.0 # via - # -r prod.txt + # -r requirements/prod.txt # h-pyramid-sentry six==1.15.0 # via - # -r prod.txt + # -r requirements/prod.txt # ecdsa # google-auth # jsonschema # python-dateutil sqlalchemy==2.0.19 # via - # -r prod.txt + # -r requirements/prod.txt # alembic stack-data==0.6.2 # via ipython supervisor==4.2.5 - # via -r dev.in + # via -r requirements/dev.in tomli==2.0.1 # via # build @@ -268,54 +268,54 @@ traitlets==5.0.5 # matplotlib-inline translationstring==1.4 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid typing-extensions==4.7.1 # via - # -r prod.txt + # -r requirements/prod.txt # alembic # sqlalchemy urllib3==1.26.15 # via - # -r prod.txt + # -r requirements/prod.txt # requests # sentry-sdk venusian==3.0.0 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid wcwidth==0.2.5 # via prompt-toolkit webargs==8.2.0 - # via -r prod.txt + # via -r requirements/prod.txt webob==1.8.6 # via - # -r prod.txt + # -r requirements/prod.txt # h-vialib # pyramid wheel==0.38.1 # via pip-tools whitenoise==6.5.0 - # via -r prod.txt + # via -r requirements/prod.txt wired==0.3 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid-services youtube-transcript-api==0.6.1 - # via -r prod.txt + # via -r requirements/prod.txt zipp==3.4.1 # via - # -r prod.txt + # -r requirements/prod.txt # importlib-metadata # importlib-resources zope-deprecation==4.4.0 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid # pyramid-jinja2 zope-interface==5.2.0 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid # pyramid-services # wired @@ -325,7 +325,7 @@ pip==23.1.2 # via pip-tools setuptools==67.7.2 # via - # -r prod.txt + # -r requirements/prod.txt # gunicorn # jsonschema # pastedeploy diff --git a/requirements/functests.txt b/requirements/functests.txt index 72d8e00f..5c98108f 100644 --- a/requirements/functests.txt +++ b/requirements/functests.txt @@ -5,10 +5,10 @@ # pip-compile --allow-unsafe --config=pyproject.toml requirements/functests.in # alembic==1.11.1 - # via -r prod.txt + # via -r requirements/prod.txt attrs==20.3.0 # via - # -r prod.txt + # -r requirements/prod.txt # jsonschema beautifulsoup4==4.9.3 # via webtest @@ -16,80 +16,80 @@ build==0.8.0 # via pip-tools cachetools==4.2.2 # via - # -r prod.txt + # -r requirements/prod.txt # google-auth certifi==2022.12.7 # via - # -r prod.txt + # -r requirements/prod.txt # requests # sentry-sdk cffi==1.15.1 # via - # -r prod.txt + # -r requirements/prod.txt # cryptography charset-normalizer==2.0.1 # via - # -r prod.txt + # -r requirements/prod.txt # requests checkmatelib==1.0.15 - # via -r prod.txt + # via -r requirements/prod.txt click==8.1.3 # via pip-tools cryptography==41.0.2 # via - # -r prod.txt + # -r requirements/prod.txt # python-jose ecdsa==0.18.0 # via - # -r prod.txt + # -r requirements/prod.txt # python-jose exceptiongroup==1.0.0rc9 # via pytest factory-boy==3.2.1 # via - # -r functests.in + # -r requirements/functests.in # pytest-factoryboy faker==18.9.0 # via factory-boy google-auth==2.16.0 # via - # -r prod.txt + # -r requirements/prod.txt # google-auth-oauthlib google-auth-oauthlib==1.0.0 - # via -r prod.txt + # via -r requirements/prod.txt greenlet==2.0.2 # via - # -r prod.txt + # -r requirements/prod.txt # sqlalchemy gunicorn==20.1.0 - # via -r prod.txt + # via -r requirements/prod.txt h-assets==1.0.5 - # via -r prod.txt + # via -r requirements/prod.txt h-matchers==1.2.15 - # via -r functests.in + # via -r requirements/functests.in h-pyramid-sentry==1.2.4 - # via -r prod.txt + # via -r requirements/prod.txt h-vialib==1.2.1 - # via -r prod.txt + # via -r requirements/prod.txt httpretty==1.1.4 - # via -r functests.in + # via -r requirements/functests.in hupper==1.10.2 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid idna==2.10 # via - # -r prod.txt + # -r requirements/prod.txt # requests importlib-metadata==6.0.0 # via - # -r prod.txt + # -r requirements/prod.txt # alembic # h-vialib # pip-sync-faster importlib-resources==5.12.0 # via - # -r prod.txt + # -r requirements/prod.txt # alembic # checkmatelib inflection==0.5.1 @@ -98,91 +98,91 @@ iniconfig==1.1.1 # via pytest jinja2==2.11.3 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid-jinja2 jsonschema==3.2.0 # via - # -r prod.txt + # -r requirements/prod.txt # checkmatelib mako==1.2.4 # via - # -r prod.txt + # -r requirements/prod.txt # alembic markupsafe==1.1.1 # via - # -r prod.txt + # -r requirements/prod.txt # jinja2 # mako # pyramid-jinja2 marshmallow==3.19.0 # via - # -r prod.txt + # -r requirements/prod.txt # webargs netaddr==0.8.0 # via - # -r prod.txt + # -r requirements/prod.txt # checkmatelib newrelic==8.8.1 - # via -r prod.txt + # via -r requirements/prod.txt oauthlib==3.2.2 # via - # -r prod.txt + # -r requirements/prod.txt # requests-oauthlib packaging==21.3 # via - # -r prod.txt + # -r requirements/prod.txt # build # marshmallow # pytest # webargs pastedeploy==2.1.1 # via - # -r prod.txt + # -r requirements/prod.txt # plaster-pastedeploy pep517==0.13.0 # via build pip-sync-faster==0.0.3 - # via -r functests.in + # via -r requirements/functests.in pip-tools==6.14.0 # via - # -r functests.in + # -r requirements/functests.in # pip-sync-faster plaster==1.0 # via - # -r prod.txt + # -r requirements/prod.txt # plaster-pastedeploy # pyramid plaster-pastedeploy==0.7 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid pluggy==0.13.1 # via pytest psycopg2==2.9.6 - # via -r prod.txt + # via -r requirements/prod.txt pyasn1==0.4.8 # via - # -r prod.txt + # -r requirements/prod.txt # pyasn1-modules # python-jose # rsa pyasn1-modules==0.2.8 # via - # -r prod.txt + # -r requirements/prod.txt # google-auth pycparser==2.21 # via - # -r prod.txt + # -r requirements/prod.txt # cffi pyjwt==2.7.0 - # via -r prod.txt + # via -r requirements/prod.txt pyparsing==3.0.7 # via - # -r prod.txt + # -r requirements/prod.txt # packaging pyramid==2.0 # via - # -r prod.txt + # -r requirements/prod.txt # h-assets # h-pyramid-sentry # pyramid-exclog @@ -190,51 +190,51 @@ pyramid==2.0 # pyramid-sanity # pyramid-services pyramid-exclog==1.1 - # via -r prod.txt + # via -r requirements/prod.txt pyramid-jinja2==2.10 - # via -r prod.txt + # via -r requirements/prod.txt pyramid-sanity==1.0.3 - # via -r prod.txt + # via -r requirements/prod.txt pyramid-services==2.2 - # via -r prod.txt + # via -r requirements/prod.txt pyrsistent==0.17.3 # via - # -r prod.txt + # -r requirements/prod.txt # jsonschema pytest==7.4.0 # via - # -r functests.in + # -r requirements/functests.in # pytest-factoryboy pytest-factoryboy==2.5.1 - # via -r functests.in + # via -r requirements/functests.in python-dateutil==2.8.2 # via faker python-jose[cryptography]==3.3.0 # via - # -r prod.txt + # -r requirements/prod.txt # h-vialib requests==2.31.0 # via - # -r prod.txt + # -r requirements/prod.txt # checkmatelib # requests-oauthlib # youtube-transcript-api requests-oauthlib==1.3.0 # via - # -r prod.txt + # -r requirements/prod.txt # google-auth-oauthlib rsa==4.7.2 # via - # -r prod.txt + # -r requirements/prod.txt # google-auth # python-jose sentry-sdk==1.14.0 # via - # -r prod.txt + # -r requirements/prod.txt # h-pyramid-sentry six==1.15.0 # via - # -r prod.txt + # -r requirements/prod.txt # ecdsa # google-auth # jsonschema @@ -243,7 +243,7 @@ soupsieve==2.1 # via beautifulsoup4 sqlalchemy==2.0.19 # via - # -r prod.txt + # -r requirements/prod.txt # alembic tomli==2.0.0 # via @@ -253,58 +253,58 @@ tomli==2.0.0 # pytest translationstring==1.4 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid typing-extensions==4.7.1 # via - # -r prod.txt + # -r requirements/prod.txt # alembic # pytest-factoryboy # sqlalchemy urllib3==1.26.15 # via - # -r prod.txt + # -r requirements/prod.txt # requests # sentry-sdk venusian==3.0.0 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid waitress==2.1.2 # via webtest webargs==8.2.0 - # via -r prod.txt + # via -r requirements/prod.txt webob==1.8.6 # via - # -r prod.txt + # -r requirements/prod.txt # h-vialib # pyramid # webtest webtest==3.0.0 - # via -r functests.in + # via -r requirements/functests.in wheel==0.38.1 # via pip-tools whitenoise==6.5.0 - # via -r prod.txt + # via -r requirements/prod.txt wired==0.3 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid-services youtube-transcript-api==0.6.1 - # via -r prod.txt + # via -r requirements/prod.txt zipp==3.4.1 # via - # -r prod.txt + # -r requirements/prod.txt # importlib-metadata # importlib-resources zope-deprecation==4.4.0 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid # pyramid-jinja2 zope-interface==5.2.0 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid # pyramid-services # wired @@ -314,7 +314,7 @@ pip==23.1.2 # via pip-tools setuptools==67.7.2 # via - # -r prod.txt + # -r requirements/prod.txt # gunicorn # jsonschema # pastedeploy diff --git a/requirements/lint.txt b/requirements/lint.txt index da1e7c8b..98366942 100644 --- a/requirements/lint.txt +++ b/requirements/lint.txt @@ -6,290 +6,290 @@ # alembic==1.11.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt astroid==2.15.5 # via pylint attrs==20.3.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # jsonschema beautifulsoup4==4.9.3 # via - # -r functests.txt + # -r requirements/functests.txt # webtest build==0.8.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pip-tools cachetools==4.2.2 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # google-auth certifi==2022.12.7 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # requests # sentry-sdk cffi==1.15.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # cryptography charset-normalizer==2.0.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # requests checkmatelib==1.0.15 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt click==8.1.3 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pip-tools coverage[toml]==7.2.7 - # via -r tests.txt + # via -r requirements/tests.txt cryptography==41.0.2 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # python-jose dill==0.3.4 # via pylint ecdsa==0.18.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # python-jose exceptiongroup==1.0.0rc9 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pytest factory-boy==3.2.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pytest-factoryboy faker==18.9.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # factory-boy freezegun==1.2.2 - # via -r tests.txt + # via -r requirements/tests.txt google-auth==2.16.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # google-auth-oauthlib google-auth-oauthlib==1.0.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt greenlet==2.0.2 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # sqlalchemy gunicorn==20.1.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt h-assets==1.0.5 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt h-matchers==1.2.15 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt h-pyramid-sentry==1.2.4 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt h-vialib==1.2.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt httpretty==1.1.4 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt hupper==1.10.2 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pyramid idna==2.10 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # requests importlib-metadata==6.0.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # alembic # h-vialib # pip-sync-faster importlib-resources==5.12.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # alembic # checkmatelib inflection==0.5.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pytest-factoryboy iniconfig==1.1.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pytest isort==5.12.0 # via pylint jinja2==2.11.3 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pyramid-jinja2 jsonschema==3.2.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # checkmatelib lazy-object-proxy==1.6.0 # via astroid mako==1.2.4 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # alembic markupsafe==1.1.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # jinja2 # mako # pyramid-jinja2 marshmallow==3.19.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # webargs mccabe==0.6.1 # via pylint netaddr==0.8.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # checkmatelib newrelic==8.8.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt oauthlib==3.2.2 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # requests-oauthlib packaging==21.3 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # build # marshmallow # pytest # webargs pastedeploy==2.1.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # plaster-pastedeploy pep517==0.13.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # build pip-sync-faster==0.0.3 # via - # -r functests.txt - # -r lint.in - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/lint.in + # -r requirements/tests.txt pip-tools==6.14.0 # via - # -r functests.txt - # -r lint.in - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/lint.in + # -r requirements/tests.txt # pip-sync-faster plaster==1.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # plaster-pastedeploy # pyramid plaster-pastedeploy==0.7 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pyramid platformdirs==2.2.0 # via pylint pluggy==0.13.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pytest psycopg2==2.9.6 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt pyasn1==0.4.8 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pyasn1-modules # python-jose # rsa pyasn1-modules==0.2.8 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # google-auth pycodestyle==2.10.0 - # via -r lint.in + # via -r requirements/lint.in pycparser==2.21 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # cffi pydocstyle==6.3.0 - # via -r lint.in + # via -r requirements/lint.in pyjwt==2.7.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt pylint==2.17.4 - # via -r lint.in + # via -r requirements/lint.in pyparsing==3.0.7 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # packaging pyramid==2.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # h-assets # h-pyramid-sentry # pyramid-exclog @@ -298,72 +298,72 @@ pyramid==2.0 # pyramid-services pyramid-exclog==1.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt pyramid-jinja2==2.10 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt pyramid-sanity==1.0.3 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt pyramid-services==2.2 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt pyrsistent==0.17.3 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # jsonschema pytest==7.4.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pytest-factoryboy pytest-factoryboy==2.5.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt python-dateutil==2.8.2 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # faker # freezegun python-jose[cryptography]==3.3.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # h-vialib requests==2.31.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # checkmatelib # requests-oauthlib # youtube-transcript-api requests-oauthlib==1.3.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # google-auth-oauthlib rsa==4.7.2 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # google-auth # python-jose sentry-sdk==1.14.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # h-pyramid-sentry six==1.15.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # ecdsa # google-auth # jsonschema @@ -372,19 +372,19 @@ snowballstemmer==2.2.0 # via pydocstyle soupsieve==2.1 # via - # -r functests.txt + # -r requirements/functests.txt # beautifulsoup4 sqlalchemy==2.0.19 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # alembic toml==0.10.2 - # via -r lint.in + # via -r requirements/lint.in tomli==2.0.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # build # coverage # pep517 @@ -395,13 +395,13 @@ tomlkit==0.11.0 # via pylint translationstring==1.4 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pyramid typing-extensions==4.7.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # alembic # astroid # pylint @@ -409,68 +409,68 @@ typing-extensions==4.7.1 # sqlalchemy urllib3==1.26.15 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # requests # sentry-sdk venusian==3.0.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pyramid waitress==2.1.2 # via - # -r functests.txt + # -r requirements/functests.txt # webtest webargs==8.2.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt webob==1.8.6 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # h-vialib # pyramid # webtest webtest==3.0.0 - # via -r functests.txt + # via -r requirements/functests.txt wheel==0.38.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pip-tools whitenoise==6.5.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt wired==0.3 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pyramid-services wrapt==1.12.1 # via astroid youtube-transcript-api==0.6.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt zipp==3.4.1 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # importlib-metadata # importlib-resources zope-deprecation==4.4.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pyramid # pyramid-jinja2 zope-interface==5.2.0 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pyramid # pyramid-services # wired @@ -478,13 +478,13 @@ zope-interface==5.2.0 # The following packages are considered to be unsafe in a requirements file: pip==23.1.2 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # pip-tools setuptools==67.7.2 # via - # -r functests.txt - # -r tests.txt + # -r requirements/functests.txt + # -r requirements/tests.txt # gunicorn # jsonschema # pastedeploy diff --git a/requirements/prod.txt b/requirements/prod.txt index e2571c32..4c468679 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -5,7 +5,7 @@ # pip-compile --allow-unsafe --config=pyproject.toml requirements/prod.in # alembic==1.11.1 - # via -r prod.in + # via -r requirements/prod.in attrs==20.3.0 # via jsonschema cachetools==4.2.2 @@ -19,7 +19,7 @@ cffi==1.15.1 charset-normalizer==2.0.1 # via requests checkmatelib==1.0.15 - # via -r prod.in + # via -r requirements/prod.in cryptography==41.0.2 # via python-jose ecdsa==0.18.0 @@ -27,17 +27,17 @@ ecdsa==0.18.0 google-auth==2.16.0 # via google-auth-oauthlib google-auth-oauthlib==1.0.0 - # via -r prod.in + # via -r requirements/prod.in greenlet==2.0.2 # via sqlalchemy gunicorn==20.1.0 - # via -r prod.in + # via -r requirements/prod.in h-assets==1.0.5 - # via -r prod.in + # via -r requirements/prod.in h-pyramid-sentry==1.2.4 - # via -r prod.in + # via -r requirements/prod.in h-vialib==1.2.1 - # via -r prod.in + # via -r requirements/prod.in hupper==1.10.2 # via pyramid idna==2.10 @@ -48,7 +48,7 @@ importlib-metadata==6.0.0 # h-vialib importlib-resources==5.12.0 # via - # -r prod.in + # -r requirements/prod.in # alembic # checkmatelib jinja2==2.11.3 @@ -64,12 +64,12 @@ markupsafe==1.1.1 # pyramid-jinja2 marshmallow==3.19.0 # via - # -r prod.in + # -r requirements/prod.in # webargs netaddr==0.8.0 # via checkmatelib newrelic==8.8.1 - # via -r prod.in + # via -r requirements/prod.in oauthlib==3.2.2 # via requests-oauthlib packaging==21.3 @@ -85,7 +85,7 @@ plaster==1.0 plaster-pastedeploy==0.7 # via pyramid psycopg2==2.9.6 - # via -r prod.in + # via -r requirements/prod.in pyasn1==0.4.8 # via # pyasn1-modules @@ -96,12 +96,12 @@ pyasn1-modules==0.2.8 pycparser==2.21 # via cffi pyjwt==2.7.0 - # via -r prod.in + # via -r requirements/prod.in pyparsing==3.0.7 # via packaging pyramid==2.0 # via - # -r prod.in + # -r requirements/prod.in # h-assets # h-pyramid-sentry # pyramid-exclog @@ -109,20 +109,20 @@ pyramid==2.0 # pyramid-sanity # pyramid-services pyramid-exclog==1.1 - # via -r prod.in + # via -r requirements/prod.in pyramid-jinja2==2.10 - # via -r prod.in + # via -r requirements/prod.in pyramid-sanity==1.0.3 - # via -r prod.in + # via -r requirements/prod.in pyramid-services==2.2 - # via -r prod.in + # via -r requirements/prod.in pyrsistent==0.17.3 # via jsonschema python-jose[cryptography]==3.3.0 # via h-vialib requests==2.31.0 # via - # -r prod.in + # -r requirements/prod.in # checkmatelib # requests-oauthlib # youtube-transcript-api @@ -141,7 +141,7 @@ six==1.15.0 # jsonschema sqlalchemy==2.0.19 # via - # -r prod.in + # -r requirements/prod.in # alembic translationstring==1.4 # via pyramid @@ -156,17 +156,17 @@ urllib3==1.26.15 venusian==3.0.0 # via pyramid webargs==8.2.0 - # via -r prod.in + # via -r requirements/prod.in webob==1.8.6 # via # h-vialib # pyramid whitenoise==6.5.0 - # via -r prod.in + # via -r requirements/prod.in wired==0.3 # via pyramid-services youtube-transcript-api==0.6.1 - # via -r prod.in + # via -r requirements/prod.in zipp==3.4.1 # via # importlib-metadata diff --git a/requirements/tests.txt b/requirements/tests.txt index b0f4593d..67a9ddbb 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -5,93 +5,93 @@ # pip-compile --allow-unsafe --config=pyproject.toml requirements/tests.in # alembic==1.11.1 - # via -r prod.txt + # via -r requirements/prod.txt attrs==20.3.0 # via - # -r prod.txt + # -r requirements/prod.txt # jsonschema build==0.8.0 # via pip-tools cachetools==4.2.2 # via - # -r prod.txt + # -r requirements/prod.txt # google-auth certifi==2022.12.7 # via - # -r prod.txt + # -r requirements/prod.txt # requests # sentry-sdk cffi==1.15.1 # via - # -r prod.txt + # -r requirements/prod.txt # cryptography charset-normalizer==2.0.1 # via - # -r prod.txt + # -r requirements/prod.txt # requests checkmatelib==1.0.15 - # via -r prod.txt + # via -r requirements/prod.txt click==8.1.3 # via pip-tools coverage[toml]==7.2.7 - # via -r tests.in + # via -r requirements/tests.in cryptography==41.0.2 # via - # -r prod.txt + # -r requirements/prod.txt # python-jose ecdsa==0.18.0 # via - # -r prod.txt + # -r requirements/prod.txt # python-jose exceptiongroup==1.0.0rc9 # via pytest factory-boy==3.2.1 # via - # -r tests.in + # -r requirements/tests.in # pytest-factoryboy faker==18.9.0 # via factory-boy freezegun==1.2.2 - # via -r tests.in + # via -r requirements/tests.in google-auth==2.16.0 # via - # -r prod.txt + # -r requirements/prod.txt # google-auth-oauthlib google-auth-oauthlib==1.0.0 - # via -r prod.txt + # via -r requirements/prod.txt greenlet==2.0.2 # via - # -r prod.txt + # -r requirements/prod.txt # sqlalchemy gunicorn==20.1.0 - # via -r prod.txt + # via -r requirements/prod.txt h-assets==1.0.5 - # via -r prod.txt + # via -r requirements/prod.txt h-matchers==1.2.15 - # via -r tests.in + # via -r requirements/tests.in h-pyramid-sentry==1.2.4 - # via -r prod.txt + # via -r requirements/prod.txt h-vialib==1.2.1 - # via -r prod.txt + # via -r requirements/prod.txt httpretty==1.1.4 - # via -r tests.in + # via -r requirements/tests.in hupper==1.10.2 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid idna==2.10 # via - # -r prod.txt + # -r requirements/prod.txt # requests importlib-metadata==6.0.0 # via - # -r prod.txt + # -r requirements/prod.txt # alembic # h-vialib # pip-sync-faster importlib-resources==5.12.0 # via - # -r prod.txt + # -r requirements/prod.txt # alembic # checkmatelib inflection==0.5.1 @@ -100,91 +100,91 @@ iniconfig==1.1.1 # via pytest jinja2==2.11.3 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid-jinja2 jsonschema==3.2.0 # via - # -r prod.txt + # -r requirements/prod.txt # checkmatelib mako==1.2.4 # via - # -r prod.txt + # -r requirements/prod.txt # alembic markupsafe==1.1.1 # via - # -r prod.txt + # -r requirements/prod.txt # jinja2 # mako # pyramid-jinja2 marshmallow==3.19.0 # via - # -r prod.txt + # -r requirements/prod.txt # webargs netaddr==0.8.0 # via - # -r prod.txt + # -r requirements/prod.txt # checkmatelib newrelic==8.8.1 - # via -r prod.txt + # via -r requirements/prod.txt oauthlib==3.2.2 # via - # -r prod.txt + # -r requirements/prod.txt # requests-oauthlib packaging==21.3 # via - # -r prod.txt + # -r requirements/prod.txt # build # marshmallow # pytest # webargs pastedeploy==2.1.1 # via - # -r prod.txt + # -r requirements/prod.txt # plaster-pastedeploy pep517==0.13.0 # via build pip-sync-faster==0.0.3 - # via -r tests.in + # via -r requirements/tests.in pip-tools==6.14.0 # via - # -r tests.in + # -r requirements/tests.in # pip-sync-faster plaster==1.0 # via - # -r prod.txt + # -r requirements/prod.txt # plaster-pastedeploy # pyramid plaster-pastedeploy==0.7 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid pluggy==0.13.1 # via pytest psycopg2==2.9.6 - # via -r prod.txt + # via -r requirements/prod.txt pyasn1==0.4.8 # via - # -r prod.txt + # -r requirements/prod.txt # pyasn1-modules # python-jose # rsa pyasn1-modules==0.2.8 # via - # -r prod.txt + # -r requirements/prod.txt # google-auth pycparser==2.21 # via - # -r prod.txt + # -r requirements/prod.txt # cffi pyjwt==2.7.0 - # via -r prod.txt + # via -r requirements/prod.txt pyparsing==3.0.7 # via - # -r prod.txt + # -r requirements/prod.txt # packaging pyramid==2.0 # via - # -r prod.txt + # -r requirements/prod.txt # h-assets # h-pyramid-sentry # pyramid-exclog @@ -192,60 +192,60 @@ pyramid==2.0 # pyramid-sanity # pyramid-services pyramid-exclog==1.1 - # via -r prod.txt + # via -r requirements/prod.txt pyramid-jinja2==2.10 - # via -r prod.txt + # via -r requirements/prod.txt pyramid-sanity==1.0.3 - # via -r prod.txt + # via -r requirements/prod.txt pyramid-services==2.2 - # via -r prod.txt + # via -r requirements/prod.txt pyrsistent==0.17.3 # via - # -r prod.txt + # -r requirements/prod.txt # jsonschema pytest==7.4.0 # via - # -r tests.in + # -r requirements/tests.in # pytest-factoryboy pytest-factoryboy==2.5.1 - # via -r tests.in + # via -r requirements/tests.in python-dateutil==2.8.2 # via # faker # freezegun python-jose[cryptography]==3.3.0 # via - # -r prod.txt + # -r requirements/prod.txt # h-vialib requests==2.31.0 # via - # -r prod.txt + # -r requirements/prod.txt # checkmatelib # requests-oauthlib # youtube-transcript-api requests-oauthlib==1.3.0 # via - # -r prod.txt + # -r requirements/prod.txt # google-auth-oauthlib rsa==4.7.2 # via - # -r prod.txt + # -r requirements/prod.txt # google-auth # python-jose sentry-sdk==1.14.0 # via - # -r prod.txt + # -r requirements/prod.txt # h-pyramid-sentry six==1.15.0 # via - # -r prod.txt + # -r requirements/prod.txt # ecdsa # google-auth # jsonschema # python-dateutil sqlalchemy==2.0.19 # via - # -r prod.txt + # -r requirements/prod.txt # alembic tomli==2.0.0 # via @@ -256,53 +256,53 @@ tomli==2.0.0 # pytest translationstring==1.4 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid typing-extensions==4.7.1 # via - # -r prod.txt + # -r requirements/prod.txt # alembic # pytest-factoryboy # sqlalchemy urllib3==1.26.15 # via - # -r prod.txt + # -r requirements/prod.txt # requests # sentry-sdk venusian==3.0.0 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid webargs==8.2.0 - # via -r prod.txt + # via -r requirements/prod.txt webob==1.8.6 # via - # -r prod.txt + # -r requirements/prod.txt # h-vialib # pyramid wheel==0.38.1 # via pip-tools whitenoise==6.5.0 - # via -r prod.txt + # via -r requirements/prod.txt wired==0.3 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid-services youtube-transcript-api==0.6.1 - # via -r prod.txt + # via -r requirements/prod.txt zipp==3.4.1 # via - # -r prod.txt + # -r requirements/prod.txt # importlib-metadata # importlib-resources zope-deprecation==4.4.0 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid # pyramid-jinja2 zope-interface==5.2.0 # via - # -r prod.txt + # -r requirements/prod.txt # pyramid # pyramid-services # wired @@ -312,7 +312,7 @@ pip==23.1.2 # via pip-tools setuptools==67.7.2 # via - # -r prod.txt + # -r requirements/prod.txt # gunicorn # jsonschema # pastedeploy diff --git a/tests/unit/via/services/youtube_api/__init__.py b/tests/unit/via/services/youtube_api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/via/services/youtube_api/_nested_data_test.py b/tests/unit/via/services/youtube_api/_nested_data_test.py new file mode 100644 index 00000000..e3bbb08a --- /dev/null +++ b/tests/unit/via/services/youtube_api/_nested_data_test.py @@ -0,0 +1,20 @@ +import pytest +from pytest import param + +from via.services.youtube_api._nested_data import safe_get + + +class TestSafeGet: + @pytest.mark.parametrize( + "data,path,expected", + ( + param({"a": {"b": 1}}, ["a", "b"], 1, id="nested_dict_key"), + param({"a": 1}, ["b"], ..., id="missing_dict_key"), + param({"a": None}, ["a"], None, id="null_not_default"), + param({"a": None}, ["a", "b"], ..., id="dict_key_into_none"), + param({"a": [{"b": 1}]}, ["a", 0, "b"], 1, id="array_key"), + param({"a": [{"b": 1}]}, ["a", 1, "b"], ..., id="missing_array_key"), + ), + ) + def test_it(self, data, path, expected): + assert safe_get(data, path, default=...) == expected diff --git a/tests/unit/via/services/youtube_api/client_test.py b/tests/unit/via/services/youtube_api/client_test.py new file mode 100644 index 00000000..9783240b --- /dev/null +++ b/tests/unit/via/services/youtube_api/client_test.py @@ -0,0 +1,77 @@ +from unittest.mock import sentinel + +import pytest + +from via.services.youtube_api import ( + CaptionTrack, + Transcript, + TranscriptText, + YouTubeAPIClient, +) + + +class TestYouTubeAPIClient: + def test_get_video_info(self, client, http_session, Video): + video = client.get_video_info("VIDEO_ID") + + http_session.post.assert_called_once_with( + "https://youtubei.googleapis.com/youtubei/v1/player", + json={ + "context": { + "client": { + "hl": "en", + "clientName": "WEB", + # Suspicious value right here... + "clientVersion": "2.20210721.00.00", + } + }, + "videoId": "VIDEO_ID", + }, + ) + response = http_session.post.return_value + response.json.assert_called_once_with() + + Video.from_v1_json.assert_called_once_with(data=response.json.return_value) + assert video == Video.from_v1_json.return_value + + def test_get_transcript(self, client, http_session): + caption_track = CaptionTrack("en", base_url=sentinel.url) + response = http_session.get.return_value + response.text = """ + + Hey there guys, + + Lichen' subscribe + + <font color="#A0AAB4">Buy my merch!</font> + + + """ + + transcript = client.get_transcript(caption_track) + + http_session.get.assert_called_once_with(url=caption_track.url) + assert transcript == Transcript( + track=caption_track, + text=[ + TranscriptText(text="Hey there guys,", start=0.21, duration=1.387), + TranscriptText(text="Lichen' subscribe", start=1.597, duration=0.0), + TranscriptText(text="Buy my merch!", start=4.327, duration=2.063), + ], + ) + + def test_get_transcript_with_no_url(self, client): + with pytest.raises(ValueError): + client.get_transcript(CaptionTrack("en", base_url=None)) + + @pytest.fixture + def client(self): + return YouTubeAPIClient() + + @pytest.fixture + def Video(self, patch): + return patch("via.services.youtube_api.client.Video") + + @pytest.fixture(autouse=True) + def http_session(self, patch): + return patch("via.services.youtube_api.client.HTTPService").return_value diff --git a/tests/unit/via/services/youtube_api/models_test.py b/tests/unit/via/services/youtube_api/models_test.py new file mode 100644 index 00000000..e257d034 --- /dev/null +++ b/tests/unit/via/services/youtube_api/models_test.py @@ -0,0 +1,200 @@ +from unittest.mock import sentinel + +import pytest +from h_matchers import Any +from pytest import param + +from via.services.youtube_api import Captions, CaptionTrack, Video + + +class TestCaptionTrack: + @pytest.mark.parametrize("kind", (None, True)) + def test_from_v1_json(self, kind): + data = { + "name": {"simpleText": "English (British) - Name"}, + "languageCode": "en-GB", + "baseUrl": sentinel.url, + } + if kind: + data["kind"] = kind + + caption_track = CaptionTrack.from_v1_json(data) + + assert caption_track == Any.instance_of(CaptionTrack).with_attrs( + { + "name": "Name", + "language_code": "en-gb", + "label": "English (British) - Name", + "kind": kind, + "base_url": sentinel.url, + } + ) + + @pytest.mark.parametrize( + "caption_track,id_string", + ( + (CaptionTrack(language_code="en"), "en"), + (CaptionTrack(language_code="en", kind="asr"), "en.a"), + (CaptionTrack(language_code="en", name="Hello"), "en..SGVsbG8="), + ( + CaptionTrack(language_code="en", translated_language_code="fr"), + "en...fr", + ), + # This combination isn't actually possible, but let's try everything at + # once + ( + CaptionTrack( + language_code="en-gb", + kind="asr", + name="Name", + translated_language_code="fr", + ), + "en-gb.a.TmFtZQ==.fr", + ), + ), + ) + def test_id(self, caption_track, id_string): + assert caption_track.id == id_string + + def test_is_auto_generated(self): + caption_track = CaptionTrack("en", kind="asr") + assert caption_track.is_auto_generated + + caption_track.kind = None + assert not caption_track.is_auto_generated + + @pytest.mark.parametrize( + "caption_track,url", + ( + ( + CaptionTrack("en", base_url="http://example.com?a=1"), + "http://example.com?a=1", + ), + ( + CaptionTrack( + "en", + base_url="http://example.com?a=1", + translated_language_code="fr", + ), + "http://example.com?a=1&tlang=fr", + ), + (CaptionTrack("en", base_url=None), None), + (CaptionTrack("en", base_url=None, translated_language_code="fr"), None), + ), + ) + def test_url(self, caption_track, url): + assert caption_track.url == url + + +class TestCaptions: + def test_from_v1_json(self, CaptionTrack): + captions = Captions.from_v1_json({"captionTracks": [{"track": "fake_dict"}]}) + + CaptionTrack.from_v1_json.assert_called_once_with({"track": "fake_dict"}) + assert captions == Any.instance_of(Captions).with_attrs( + {"tracks": [CaptionTrack.from_v1_json.return_value]} + ) + + def test_from_v1_json_minimal(self, CaptionTrack): + captions = Captions.from_v1_json({}) + + assert not captions.tracks + CaptionTrack.assert_not_called() + + @pytest.mark.parametrize( + "preferences,expected_label", + ( + param( + [CaptionTrack("en")], + "plain_en", + id="direct_match", + ), + param( + [CaptionTrack("de"), CaptionTrack("en-gb")], + "plain_en_gb", + id="miss_then_hit", + ), + param( + [ + CaptionTrack("de"), + CaptionTrack(Any.string.matching("^en-.*"), name="Name"), + ], + "named_en_gb", + id="wild_cards", + ), + param( + [CaptionTrack("fr", kind=None), CaptionTrack("en", kind="asr")], + "en_auto", + id="fallback_to_auto", + ), + param( + [CaptionTrack(Any(), name="Name")], + "named_en_gb", + id="same_level_sorting", + ), + param([CaptionTrack("fr")], None, id="miss"), + ), + ) + def test_find_matching_track(self, preferences, expected_label): + captions = Captions( + tracks=[ + CaptionTrack("en", label="plain_en"), + CaptionTrack("en-gb", label="plain_en_gb"), + CaptionTrack("en-us", name="Name", label="named_en_us"), + CaptionTrack("en-gb", name="Name", label="named_en_gb"), + CaptionTrack("en", kind="asr", label="en_auto"), + ] + ) + + caption_track = captions.find_matching_track(preferences) + + assert ( + caption_track.label == expected_label + if expected_label + else not caption_track + ) + + def test_find_matching_track_with_translation(self): + captions = Captions(tracks=[CaptionTrack("fr", label="plain_fr")]) + + caption_track = captions.find_matching_track( + [ + CaptionTrack( + language_code=Any(), + name=Any(), + kind=Any(), + translated_language_code="de", + ) + ] + ) + + assert caption_track.label == "plain_fr" + assert caption_track.translated_language_code == "de" + + @pytest.fixture + def CaptionTrack(self, patch): + return patch("via.services.youtube_api.models.CaptionTrack") + + +class TestVideo: + def test_from_v1_json(self, Captions): + video = Video.from_v1_json( + data={ + "captions": {"playerCaptionsTracklistRenderer": sentinel.captions}, + }, + ) + + Captions.from_v1_json.assert_called_once_with(sentinel.captions) + + assert video == Any.instance_of(Video).with_attrs( + {"caption": Captions.from_v1_json.return_value} + ) + + def test_from_v1_json_minimal(self, Captions): + Video.from_v1_json({}) + + Captions.from_v1_json.assert_called_once_with({}) + + @pytest.fixture + def Captions(self, patch): + return patch("via.services.youtube_api.models.Captions") diff --git a/via/services/youtube_api/__init__.py b/via/services/youtube_api/__init__.py new file mode 100644 index 00000000..014b3542 --- /dev/null +++ b/via/services/youtube_api/__init__.py @@ -0,0 +1,8 @@ +from via.services.youtube_api.client import YouTubeAPIClient +from via.services.youtube_api.models import ( + Captions, + CaptionTrack, + Transcript, + TranscriptText, + Video, +) diff --git a/via/services/youtube_api/_nested_data.py b/via/services/youtube_api/_nested_data.py new file mode 100644 index 00000000..85319b07 --- /dev/null +++ b/via/services/youtube_api/_nested_data.py @@ -0,0 +1,13 @@ +from typing import Iterable + + +def safe_get(data, path: Iterable, default=None): + """Get deeply nested items without exploding.""" + + for key in path: + try: + data = data[key] + except (KeyError, IndexError, TypeError): + return default + + return data diff --git a/via/services/youtube_api/client.py b/via/services/youtube_api/client.py new file mode 100644 index 00000000..84268b0f --- /dev/null +++ b/via/services/youtube_api/client.py @@ -0,0 +1,79 @@ +from xml.etree import ElementTree + +import requests + +from via.services import HTTPService +from via.services.youtube_api.models import ( + CaptionTrack, + Transcript, + TranscriptText, + Video, +) + + +class YouTubeAPIError(Exception): + """Something has gone wrong interacting with YouTube.""" + + +class YouTubeAPIClient: + """A client for interacting with YouTube and manipulating related URLs.""" + + def __init__(self): + session = requests.Session() + # Ensure any translations that Google provides are in English + session.headers["Accept-Language"] = "en-US" + self._http = HTTPService(session=session) + + def get_video_info(self, video_id: str) -> Video: + """Get information for a given YouTube video.""" + + response = self._http.post( + "https://youtubei.googleapis.com/youtubei/v1/player", + json={ + "context": { + "client": { + "hl": "en", + "clientName": "WEB", + # Suspicious value right here... + "clientVersion": "2.20210721.00.00", + } + }, + "videoId": video_id, + }, + ) + + return Video.from_v1_json(data=response.json()) + + def get_transcript(self, caption_track: CaptionTrack) -> Transcript: + """Get the transcript associated with a caption track. + + You can set the track `translated_language_code` to ensure we translate + the value before returning it. + """ + + if not caption_track.url: + raise ValueError("Cannot get a transcript without a URL") + + response = self._http.get(url=caption_track.url) + xml_elements = ElementTree.fromstring(response.text) + + return Transcript( + track=caption_track, + text=[ + TranscriptText( + text=self._strip_html(xml_element.text), + start=float(xml_element.attrib["start"]), + duration=float(xml_element.attrib.get("dur", "0.0")), + ) + for xml_element in xml_elements + if xml_element.text is not None + ], + ) + + @staticmethod + def _strip_html(xml_string): + """Remove all non-text content from an XML fragment or string.""" + + return "".join( + ElementTree.fromstring(f"{xml_string}").itertext() + ).strip() diff --git a/via/services/youtube_api/models.py b/via/services/youtube_api/models.py new file mode 100644 index 00000000..5c24ddb3 --- /dev/null +++ b/via/services/youtube_api/models.py @@ -0,0 +1,188 @@ +import base64 +from copy import deepcopy +from dataclasses import dataclass, field +from operator import attrgetter +from typing import List, Optional + +from via.services.youtube_api._nested_data import safe_get + + +@dataclass +class CaptionTrack: + """A source of transcription data, in a particular language.""" + + language_code: str + """Original language of the track.""" + + name: Optional[str] = None + """Human set name for the track.""" + + kind: Optional[str] = None + """Is this track automatically generated by audio to text AI?""" + + translated_language_code: Optional[str] = None + """Language to machine translate this into. Set this manually.""" + + label: Optional[str] = None + """Human readable name (determined by language + name).""" + + base_url: Optional[str] = None + """URL to download the original language text (as XML).""" + + @classmethod + def from_v1_json(cls, data: dict): + """Create an instance from a `captionTrack` section of JSON.""" + + label = data["name"]["simpleText"] + + return CaptionTrack( + name=label.split(" - ", 1)[-1] if " - " in label else None, + language_code=data["languageCode"].lower(), + label=label, + kind=data.get("kind", None), + base_url=data["baseUrl"], + ) + + @property + def id(self) -> str: # pylint: disable=invalid-name + if self.name: + # Ensure our ids don't contain wild characters + name = base64.b64encode(self.name.encode("utf-8")).decode("utf-8") + else: + name = None + + return ".".join( + part or "" + for part in [ + self.language_code, + "a" if self.is_auto_generated else None, + name, + self.translated_language_code, + ] + ).rstrip(".") + + @property + def is_auto_generated(self) -> bool: + """Get whether this caption track auto generated.""" + + return self.kind == "asr" + + @property + def url(self) -> Optional[str]: + """Get the URL to download a transcript of this caption track.""" + if not self.base_url: + return None + + url = self.base_url + + if self.translated_language_code: + url += f"&tlang={self.translated_language_code}" + + return url + + +@dataclass +class Captions: + """All information about captions.""" + + tracks: List[CaptionTrack] = field(default_factory=list) + """Available tracks to pick from.""" + + @classmethod + def from_v1_json(cls, data: dict): + """Create an instance from JSON. + + This is populated from the `captions.playerCaptionsTracklistRenderer` + section. + """ + + return Captions( + tracks=[ + CaptionTrack.from_v1_json(track) + for track in data.get("captionTracks", []) + ] + ) + + def find_matching_track( + self, preferences: List[CaptionTrack] + ) -> Optional[CaptionTrack]: + """ + Get a caption track which matching the preferences in order. + + This method takes the provided list of caption track objects and + searches the available tracks for those with matching details: + + * language_code + * name + * is_auto_generated / kind + * translation_language_code + + For a match to happen, we must match the first three items, and be + translatable to the last if present. + + Earlier items are higher priority. + + :param preferences: List of partially filled out caption track objects + which represent the caption track we would like. + """ + + def get_key(track: CaptionTrack): + return track.language_code, track.kind, track.name + + search_keys = [get_key(preference) for preference in preferences] + best_index, best_caption_track = None, None + + # Sort the tracks to keep the algorithm more stable! This only insulates + # us from sorting changes, not metadata changes. + for caption_track in sorted(self.tracks, key=attrgetter("id")): + try: + index = search_keys.index(get_key(caption_track)) + except ValueError: + continue + + # Items with lower indexes are first choices for the user + if best_index is None or best_index > index: + best_index, best_caption_track = index, deepcopy(caption_track) + + if best_index is None: + return None + + if target_language := preferences[best_index].translated_language_code: + # Convert the track to a translated language if required, we've + # checked above this is ok. + best_caption_track.translated_language_code = target_language + + return best_caption_track + + +@dataclass +class Video: + """Data for a video in YouTube.""" + + caption: Optional[Captions] = None + """Caption related information (tracks and languages).""" + + @classmethod + def from_v1_json(cls, data): + return Video( + caption=Captions.from_v1_json( + safe_get(data, ["captions", "playerCaptionsTracklistRenderer"], {}) + ) + ) + + +@dataclass +class TranscriptText: + """An individual row of transcript text.""" + + text: str + start: float + duration: float + + +@dataclass +class Transcript: + """A full transcript from a caption track.""" + + track: CaptionTrack + text: List[TranscriptText]