From b0d52f27bbd7bb7afab283888a06eea211ebcb18 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Mon, 16 Dec 2024 14:50:17 +0100 Subject: [PATCH 01/32] Draft of integration of browserforge fingerprint generation. TODO: Tests, JS page function for injecting fingerprint json. --- poetry.lock | 32 +++++++++++++++---- pyproject.toml | 5 +-- src/crawlee/browsers/_browser_pool.py | 17 +++++++++- .../_playwright_browser_controller.py | 20 ++++++++++++ .../browsers/_playwright_browser_plugin.py | 10 ++++++ .../_injected_page_function.py | 2 ++ 6 files changed, 76 insertions(+), 10 deletions(-) create mode 100644 src/crawlee/fingerprint_suite/_injected_page_function.py diff --git a/poetry.lock b/poetry.lock index ae49822ed3..fcf542ba1d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "annotated-types" @@ -402,6 +402,24 @@ files = [ [package.dependencies] cffi = ">=1.0.0" +[[package]] +name = "browserforge" +version = "1.2.1" +description = "Intelligent browser header & fingerprint generator" +optional = true +python-versions = "<4.0,>=3.8" +files = [ + {file = "browserforge-1.2.1-py3-none-any.whl", hash = "sha256:b2813b4de80b9c48c88700c93e3dfa6a64694d04f3263545e28bb03dd95df27e"}, + {file = "browserforge-1.2.1.tar.gz", hash = "sha256:7036d73fb066a4361a015b619079474c42d8b0ff415e1d874b62366de48d0b61"}, +] + +[package.dependencies] +click = "*" +typing_extensions = {version = "*", markers = "python_version < \"3.10\""} + +[package.extras] +all = ["orjson"] + [[package]] name = "build" version = "1.2.2.post1" @@ -429,13 +447,13 @@ virtualenv = ["virtualenv (>=20.0.35)"] [[package]] name = "certifi" -version = "2024.8.30" +version = "2024.12.14" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, - {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, + {file = "certifi-2024.12.14-py3-none-any.whl", hash = "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56"}, + {file = "certifi-2024.12.14.tar.gz", hash = "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"}, ] [[package]] @@ -3661,13 +3679,13 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", type = ["pytest-mypy"] [extras] -all = ["beautifulsoup4", "curl-cffi", "html5lib", "lxml", "parsel", "playwright"] +all = ["beautifulsoup4", "browserforge", "curl-cffi", "html5lib", "lxml", "parsel", "playwright"] beautifulsoup = ["beautifulsoup4", "html5lib", "lxml"] curl-impersonate = ["curl-cffi"] parsel = ["parsel"] -playwright = ["playwright"] +playwright = ["browserforge", "playwright"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "ba84ac0e96fc33b777b9c097e3e905b6493fdf1288cb38d56ff775984b9817e3" +content-hash = "7044fb232dd9a4e5cc8bdb75ecbeae3337e164bbef1d5551f9300f242b118e4a" diff --git a/pyproject.toml b/pyproject.toml index 44ffef1327..a2f118024e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ keywords = [ python = "^3.9" apify = { version = ">=2.0.0", optional = true } beautifulsoup4 = { version = ">=4.12.0", optional = true } +browserforge = { version = "*", optional = true } colorama = ">=0.4.0" cookiecutter = ">=2.6.0" curl-cffi = { version = ">=0.7.2", optional = true } @@ -94,10 +95,10 @@ types-psutil = "~5.9.5.20240205" types-python-dateutil = "~2.9.0.20240316" [tool.poetry.extras] -all = ["beautifulsoup4", "curl-cffi", "lxml", "html5lib", "parsel", "playwright"] +all = ["beautifulsoup4", "curl-cffi", "lxml", "html5lib", "parsel", "playwright", "browserforge"] beautifulsoup = ["beautifulsoup4", "lxml", "html5lib"] curl-impersonate = ["curl-cffi"] -playwright = ["playwright"] +playwright = ["playwright", "browserforge"] parsel = ["parsel"] [tool.poetry.scripts] diff --git a/src/crawlee/browsers/_browser_pool.py b/src/crawlee/browsers/_browser_pool.py index 07908286f5..29b2868ec3 100644 --- a/src/crawlee/browsers/_browser_pool.py +++ b/src/crawlee/browsers/_browser_pool.py @@ -51,6 +51,8 @@ def __init__( browser_inactive_threshold: timedelta = timedelta(seconds=10), identify_inactive_browsers_interval: timedelta = timedelta(seconds=20), close_inactive_browsers_interval: timedelta = timedelta(seconds=30), + use_fingerprints: bool = True, + fingerprint_generator_options: dict[str, Any] | None = None, ) -> None: """A default constructor. @@ -66,6 +68,8 @@ def __init__( close_inactive_browsers_interval: The interval at which the pool checks for inactive browsers and closes them. The browser is considered as inactive if it has no active pages and has been idle for the specified period. + use_fingerprints: Will inject fingerprints + fingerprint_generator_options: Override generated fingerprints with these specific values. """ self._plugins = plugins or [PlaywrightBrowserPlugin()] self._operation_timeout = operation_timeout @@ -95,6 +99,9 @@ def __init__( # Flag to indicate the context state. self._active = False + self._use_fingerprints = use_fingerprints + self._fingerprint_generator_options = fingerprint_generator_options + @classmethod def with_default_plugin( cls, @@ -103,6 +110,8 @@ def with_default_plugin( browser_options: Mapping[str, Any] | None = None, page_options: Mapping[str, Any] | None = None, headless: bool | None = None, + use_fingerprints: bool = True, + fingerprint_generator_options: dict[str, Any] | None = None, **kwargs: Any, ) -> BrowserPool: """Create a new instance with a single `PlaywrightBrowserPlugin` configured with the provided options. @@ -116,6 +125,8 @@ def with_default_plugin( Playwright's `browser_context.new_page` method. For more details, refer to the Playwright documentation: https://playwright.dev/python/docs/api/class-browsercontext#browser-context-new-page. headless: Whether to run the browser in headless mode. + use_fingerprints: Will inject fingerprints + fingerprint_generator_options: Override generated fingerprints with these specific values. kwargs: Additional arguments for default constructor. """ plugin_options: dict = defaultdict(dict) @@ -128,7 +139,11 @@ def with_default_plugin( if browser_type: plugin_options['browser_type'] = browser_type - plugin = PlaywrightBrowserPlugin(**plugin_options) + plugin = PlaywrightBrowserPlugin( + **plugin_options, + use_fingerprints=use_fingerprints, + fingerprint_generator_options=fingerprint_generator_options, + ) return cls(plugins=[plugin], **kwargs) @property diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index f0a91deb51..c6728493ea 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -5,6 +5,7 @@ from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, cast +from browserforge.fingerprints import FingerprintGenerator from playwright.async_api import BrowserContext, Page, ProxySettings from typing_extensions import override @@ -12,6 +13,7 @@ from crawlee.browsers._base_browser_controller import BaseBrowserController from crawlee.browsers._types import BrowserType from crawlee.fingerprint_suite import HeaderGenerator +from crawlee.fingerprint_suite._injected_page_function import javascript_stuff if TYPE_CHECKING: from collections.abc import Mapping @@ -38,6 +40,9 @@ def __init__( *, max_open_pages_per_browser: int = 20, header_generator: HeaderGenerator | None = _DEFAULT_HEADER_GENERATOR, + fingerprint_generator: FingerprintGenerator | None = None, + use_fingerprints: bool = True, + fingerprint_generator_options: dict[str, Any] | None = None, ) -> None: """A default constructor. @@ -47,15 +52,23 @@ def __init__( header_generator: An optional `HeaderGenerator` instance used to generate and manage HTTP headers for requests made by the browser. By default, a predefined header generator is used. Set to `None` to disable automatic header modifications. + fingerprint_generator: An optional `FingerprintGenerator` instance used to generate fingerprints, + use_fingerprints: Will inject fingerprints + fingerprint_generator_options: Override generated fingerprints with these specific values. """ self._browser = browser self._max_open_pages_per_browser = max_open_pages_per_browser self._header_generator = header_generator + fingerprint_generator_options = fingerprint_generator_options or {} + self._fingerprint_generator = fingerprint_generator or FingerprintGenerator(**fingerprint_generator_options) + self._browser_context: BrowserContext | None = None self._pages = list[Page]() self._last_page_opened_at = datetime.now(timezone.utc) + self._use_fingerprints = use_fingerprints + @property @override def pages(self) -> list[Page]: @@ -113,8 +126,15 @@ async def new_page( self._pages.append(page) self._last_page_opened_at = datetime.now(timezone.utc) + # Inject fingerprint + if self._use_fingerprints: + await self._inject_fingerprint_to_page(page) + return page + async def _inject_fingerprint_to_page(self, page: Page) -> None: + await page.evaluate(javascript_stuff, self._fingerprint_generator.generate().dumps()) + @override async def close(self, *, force: bool = False) -> None: if force: diff --git a/src/crawlee/browsers/_playwright_browser_plugin.py b/src/crawlee/browsers/_playwright_browser_plugin.py index cd5cda314a..16c36b0bff 100644 --- a/src/crawlee/browsers/_playwright_browser_plugin.py +++ b/src/crawlee/browsers/_playwright_browser_plugin.py @@ -38,6 +38,8 @@ def __init__( browser_options: Mapping[str, Any] | None = None, page_options: Mapping[str, Any] | None = None, max_open_pages_per_browser: int = 20, + use_fingerprints: bool = True, + fingerprint_generator_options: dict[str, Any] | None = None, ) -> None: """A default constructor. @@ -51,6 +53,9 @@ def __init__( https://playwright.dev/python/docs/api/class-browsercontext#browser-context-new-page. max_open_pages_per_browser: The maximum number of pages that can be opened in a single browser instance. Once reached, a new browser instance will be launched to handle the excess. + use_fingerprints: Will inject fingerprints + fingerprint_generator_options: Override generated fingerprints with these specific values. + """ self._browser_type = browser_type self._browser_options = browser_options or {} @@ -63,6 +68,9 @@ def __init__( # Flag to indicate the context state. self._active = False + self._use_fingerprints = use_fingerprints + self._fingerprint_generator_options = fingerprint_generator_options + @property @override def active(self) -> bool: @@ -128,4 +136,6 @@ async def new_browser(self) -> PlaywrightBrowserController: return PlaywrightBrowserController( browser, max_open_pages_per_browser=self._max_open_pages_per_browser, + use_fingerprints=self._use_fingerprints, + fingerprint_generator_options=self._fingerprint_generator_options, ) diff --git a/src/crawlee/fingerprint_suite/_injected_page_function.py b/src/crawlee/fingerprint_suite/_injected_page_function.py new file mode 100644 index 0000000000..65aa0e71e8 --- /dev/null +++ b/src/crawlee/fingerprint_suite/_injected_page_function.py @@ -0,0 +1,2 @@ +# This is function that should set the fingerprint on page object. +javascript_stuff = """(a) =>{ return a;}""" From be15847f87f6a0deb005056e0f76ee8ecc4ce7b8 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 17 Dec 2024 13:35:16 +0100 Subject: [PATCH 02/32] Works with page.evaluate. Todo: Make it work with page.add_init_script --- .../_playwright_browser_controller.py | 3 +- .../_injected_page_function.py | 850 +++++++++++++++++- .../test_playwright_crawler.py | 2 +- 3 files changed, 852 insertions(+), 3 deletions(-) diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index c6728493ea..1e90c31417 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -133,7 +133,8 @@ async def new_page( return page async def _inject_fingerprint_to_page(self, page: Page) -> None: - await page.evaluate(javascript_stuff, self._fingerprint_generator.generate().dumps()) + finger_print = self._fingerprint_generator.generate().dumps() + ret = await page.evaluate(javascript_stuff, finger_print) @override async def close(self, *, force: bool = False) -> None: diff --git a/src/crawlee/fingerprint_suite/_injected_page_function.py b/src/crawlee/fingerprint_suite/_injected_page_function.py index 65aa0e71e8..d43cdb7bf9 100644 --- a/src/crawlee/fingerprint_suite/_injected_page_function.py +++ b/src/crawlee/fingerprint_suite/_injected_page_function.py @@ -1,2 +1,850 @@ # This is function that should set the fingerprint on page object. -javascript_stuff = """(a) =>{ return a;}""" +javascript_stuff = r""" +(fp) =>{ + const isHeadlessChromium = /headless/i.test(navigator.userAgent) && navigator.plugins.length === 0; + const isChrome = navigator.userAgent.includes("Chrome"); + const isFirefox = navigator.userAgent.includes("Firefox"); + const isSafari = navigator.userAgent.includes("Safari") && !navigator.userAgent.includes("Chrome"); + + let slim = null; + function getSlim() { + if(slim === null) { + slim = window.slim || false; + if(typeof window.slim !== 'undefined') { + delete window.slim; + } + } + + return slim; + } + + // This file contains utils that are build and included on the window object with some randomized prefix. + + // some protections can mess with these to prevent the overrides - our script is first so we can reference the old values. + const cache = { + Reflect: { + get: Reflect.get.bind(Reflect), + apply: Reflect.apply.bind(Reflect), + }, + // Used in `makeNativeString` + nativeToStringStr: `${Function.toString}`, // => `function toString() { [native code] }` + }; + + /** + * @param masterObject Object to override. + * @param propertyName Property to override. + * @param proxyHandler Proxy handled with the new value. + */ + function overridePropertyWithProxy(masterObject, propertyName, proxyHandler) { + const originalObject = masterObject[propertyName]; + const proxy = new Proxy(masterObject[propertyName], stripProxyFromErrors(proxyHandler)); + + redefineProperty(masterObject, propertyName, { value: proxy }); + redirectToString(proxy, originalObject); + } + + const prototypeProxyHandler = { + setPrototypeOf: (target, newProto) => { + try { + throw new TypeError('Cyclic __proto__ value'); + } catch (e) { + const oldStack = e.stack; + const oldProto = Object.getPrototypeOf(target); + Object.setPrototypeOf(target, newProto); + try { + // shouldn't throw if prototype is okay, will throw if there is a prototype cycle (maximum call stack size exceeded). + target['nonexistentpropertytest']; + return true; + } + catch (err) { + Object.setPrototypeOf(target, oldProto); + if (oldStack.includes('Reflect.setPrototypeOf')) return false; + const newError = new TypeError('Cyclic __proto__ value'); + const stack = oldStack.split('\n'); + newError.stack = [stack[0], ...stack.slice(2)].join('\n'); + throw newError; + } + } + }, + } + +function useStrictModeExceptions(prop) { + if (['caller', 'callee', 'arguments'].includes(prop)) { + throw TypeError(`'caller', 'callee', and 'arguments' properties may not be accessed on strict mode functions or the arguments objects for calls to them`); + } + } + + /** + * @param masterObject Object to override. + * @param propertyName Property to override. + * @param proxyHandler ES6 Proxy handler object with a get handle only. + */ + function overrideGetterWithProxy(masterObject, propertyName, proxyHandler) { + const fn = Object.getOwnPropertyDescriptor(masterObject, propertyName).get; + const fnStr = fn.toString; // special getter function string + const proxyObj = new Proxy(fn, { + ...stripProxyFromErrors(proxyHandler), + ...prototypeProxyHandler, + }); + + redefineProperty(masterObject, propertyName, { get: proxyObj }); + redirectToString(proxyObj, fnStr); + } + + /** + * @param instance Instance to override. + * @param overrideObj New instance values. + */ + // eslint-disable-next-line no-unused-vars + function overrideInstancePrototype(instance, overrideObj) { + try { + Object.keys(overrideObj).forEach((key) => { + if (!(overrideObj[key] === null)) { + try { + overrideGetterWithProxy( + Object.getPrototypeOf(instance), + key, + makeHandler().getterValue(overrideObj[key]), + ); + } catch (e) { + return false; + // console.error(`Could not override property: ${key} on ${instance}. Reason: ${e.message} `); // some fingerprinting services can be listening + } + } + }); + } catch (e) { + console.error(e); + } + } + + /** + * Updates the .toString method in Function.prototype to return a native string representation of the function. + * @param {*} proxyObj + * @param {*} originalObj + */ + function redirectToString(proxyObj, originalObj) { + if(getSlim()) return; + + const handler = { + setPrototypeOf: (target, newProto) => { + try { + throw new TypeError('Cyclic __proto__ value'); + } catch (e) { + if (e.stack.includes('Reflect.setPrototypeOf')) return false; + // const stack = e.stack.split('\n'); + // e.stack = [stack[0], ...stack.slice(2)].join('\n'); + throw e; + } + }, + apply(target, ctx) { + // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + ""` + if (ctx === Function.prototype.toString) { + return makeNativeString('toString'); + } + + // `toString` targeted at our proxied Object detected + if (ctx === proxyObj) { + // Return the toString representation of our original object if possible + return makeNativeString(proxyObj.name); + } + + // Check if the toString prototype of the context is the same as the global prototype, + // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case + const hasSameProto = Object.getPrototypeOf( + Function.prototype.toString, + ).isPrototypeOf(ctx.toString); // eslint-disable-line no-prototype-builtins + + if (!hasSameProto) { + // Pass the call on to the local Function.prototype.toString instead + return ctx.toString(); + } + + if (Object.getPrototypeOf(ctx) === proxyObj){ + try { + return target.call(ctx); + } catch (err) { + err.stack = err.stack.replace( + 'at Object.toString (', + 'at Function.toString (', + ); + throw err; + }} + return target.call(ctx); + }, + get: function(target, prop, receiver) { + if (prop === 'toString') { + return new Proxy(target.toString, { + apply: function(tget, thisArg, argumentsList) { + try { + return tget.bind(thisArg)(...argumentsList); + } catch (err) { + if(Object.getPrototypeOf(thisArg) === tget){ + err.stack = err.stack.replace( + 'at Object.toString (', + 'at Function.toString (', + ); + } + + throw err; + } + } + }); + } + useStrictModeExceptions(prop); + return Reflect.get(...arguments); + } + }; + + const toStringProxy = new Proxy( + Function.prototype.toString, + stripProxyFromErrors(handler), + ); + redefineProperty(Function.prototype, 'toString', { + value: toStringProxy, + }); + } + + function makeNativeString(name = '') { + return cache.nativeToStringStr.replace('toString', name || ''); + } + + function redefineProperty(masterObject, propertyName, descriptorOverrides = {}) { + return Object.defineProperty(masterObject, propertyName, { + // Copy over the existing descriptors (writable, enumerable, configurable, etc) + ...(Object.getOwnPropertyDescriptor(masterObject, propertyName) || {}), + // Add our overrides (e.g. value, get()) + ...descriptorOverrides, + }); + } + + /** + * For all the traps in the passed proxy handler, we wrap them in a try/catch and modify the error stack if they throw. + * @param {*} handler A proxy handler object + * @returns A new proxy handler object with error stack modifications + */ + function stripProxyFromErrors(handler) { + const newHandler = {}; + // We wrap each trap in the handler in a try/catch and modify the error stack if they throw + const traps = Object.getOwnPropertyNames(handler); + traps.forEach((trap) => { + newHandler[trap] = function () { + try { + // Forward the call to the defined proxy handler + return handler[trap].apply(this, arguments || []); //eslint-disable-line + } catch (err) { + // Stack traces differ per browser, we only support chromium based ones currently + if (!err || !err.stack || !err.stack.includes(`at `)) { + throw err; + } + + // When something throws within one of our traps the Proxy will show up in error stacks + // An earlier implementation of this code would simply strip lines with a blacklist, + // but it makes sense to be more surgical here and only remove lines related to our Proxy. + // We try to use a known "anchor" line for that and strip it with everything above it. + // If the anchor line cannot be found for some reason we fall back to our blacklist approach. + + const stripWithBlacklist = (stack, stripFirstLine = true) => { + const blacklist = [ + `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply + `at Object.${trap} `, // e.g. Object.get or Object.apply + `at Object.newHandler. [as ${trap}] `, // caused by this very wrapper :-) + `at newHandler. [as ${trap}] `, // also caused by this wrapper :p + ]; + return ( + err.stack + .split('\n') + // Always remove the first (file) line in the stack (guaranteed to be our proxy) + .filter((line, index) => !(index === 1 && stripFirstLine)) + // Check if the line starts with one of our blacklisted strings + .filter((line) => !blacklist.some((bl) => line.trim().startsWith(bl))) + .join('\n') + ); + }; + + const stripWithAnchor = (stack, anchor) => { + const stackArr = stack.split('\n'); + anchor = anchor || `at Object.newHandler. [as ${trap}] `; // Known first Proxy line in chromium + const anchorIndex = stackArr.findIndex((line) => line.trim().startsWith(anchor)); + if (anchorIndex === -1) { + return false; // 404, anchor not found + } + // Strip everything from the top until we reach the anchor line + // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`) + stackArr.splice(1, anchorIndex); + return stackArr.join('\n'); + }; + + + const oldStackLines = err.stack.split('\n'); + Error.captureStackTrace(err); + const newStackLines = err.stack.split('\n'); + + err.stack = [newStackLines[0],oldStackLines[1],...newStackLines.slice(1)].join('\n'); + + if ((err.stack || '').includes('toString (')) { + err.stack = stripWithBlacklist(err.stack, false); + throw err; + } + + // Try using the anchor method, fallback to blacklist if necessary + err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack); + + throw err; // Re-throw our now sanitized error + } + }; + }); + return newHandler; + } + + // eslint-disable-next-line no-unused-vars + function overrideWebGl(webGl) { + // try to override WebGl + try { + const getParameterProxyHandler = { + apply: function (target, ctx, args) { + const param = (args || [])[0]; + const result = cache.Reflect.apply(target, ctx, args); + // UNMASKED_VENDOR_WEBGL + if (param === 37445) { + return webGl.vendor; + } + // UNMASKED_RENDERER_WEBGL + if (param === 37446) { + return webGl.renderer; + } + return result; + }, + get: function (target, prop, receiver) { + useStrictModeExceptions(prop); + return Reflect.get(...arguments); + }, + } + const addProxy = (obj, propName) => { + overridePropertyWithProxy(obj, propName, getParameterProxyHandler); + } + // For whatever weird reason loops don't play nice with Object.defineProperty, here's the next best thing: + addProxy(WebGLRenderingContext.prototype, 'getParameter'); + addProxy(WebGL2RenderingContext.prototype, 'getParameter'); + } catch (err) { + console.warn(err); + } + } + + // eslint-disable-next-line no-unused-vars + const overrideCodecs = (audioCodecs, videoCodecs) => { + try { + const codecs = { + ...Object.fromEntries(Object.entries(audioCodecs).map(([key, value]) => [`audio/${key}`, value])), + ...Object.fromEntries(Object.entries(videoCodecs).map(([key, value]) => [`video/${key}`, value])), + }; + + const findCodec = (codecString) => { + const [mime, codecSpec] = codecString.split(';'); + if (mime === 'video/mp4') { + if (codecSpec && codecSpec.includes('avc1.42E01E')) { // codec is missing from Chromium + return {name: mime, state: 'probably'}; + } + } + + const codec = Object.entries(codecs).find(([key]) => key === codecString.split(';')[0]); + if(codec) { + return {name: codec[0], state: codec[1]}; + } + + return undefined; + }; + + const canPlayType = { + // eslint-disable-next-line + apply: function (target, ctx, args) { + if (!args || !args.length) { + return target.apply(ctx, args); + } + const [codecString] = args; + const codec = findCodec(codecString); + + if (codec) { + return codec.state; + } + + // If the codec is not in our collected data use + return target.apply(ctx, args); + }, + }; + + overridePropertyWithProxy( + HTMLMediaElement.prototype, + 'canPlayType', + canPlayType, + ); + } catch (e) { + console.warn(e); + } + }; + + // eslint-disable-next-line no-unused-vars + function overrideBattery(batteryInfo) { + try { + const getBattery = { + ...prototypeProxyHandler, + // eslint-disable-next-line + apply: async function () { + return batteryInfo; + }, + }; + + if(navigator.getBattery) { // Firefox does not have this method - to be fixed + overridePropertyWithProxy( + Object.getPrototypeOf(navigator), + 'getBattery', + getBattery, + ); + } + } catch (e) { + console.warn(e); + } + } + + function overrideIntlAPI(language){ + try { + const innerHandler = { + construct(target, [locales, options]) { + return new target(locales ?? language, options); + }, + apply(target, _, [locales, options]) { + return target(locales ?? language, options); + } + }; + + overridePropertyWithProxy(window, 'Intl', { + get(target, key){ + if(typeof key !== 'string' || key[0].toLowerCase() === key[0]) return target[key]; + return new Proxy( + target[key], + innerHandler + ); + } + }); + } catch (e) { + console.warn(e); + } + } + + function makeHandler() { + return { + // Used by simple `navigator` getter evasions + getterValue: (value) => ({ + apply(target, ctx, args) { + // Let's fetch the value first, to trigger and escalate potential errors + // Illegal invocations like `navigator.__proto__.vendor` will throw here + const ret = cache.Reflect.apply(...arguments); // eslint-disable-line + if (args && args.length === 0) { + return value; + } + return ret; + }, + get: function (target, prop, receiver) { + useStrictModeExceptions(prop); + return Reflect.get(...arguments); + }, + }), + }; + } + + function overrideScreenByReassigning(target, newProperties) { + for (const [prop, value] of Object.entries(newProperties)) { + if (value > 0) { + // The 0 values are introduced by collecting in the hidden iframe. + // They are document sizes anyway so no need to test them or inject them. + target[prop] = value; + } + } + } + + // eslint-disable-next-line no-unused-vars + function overrideWindowDimensionsProps(props) { + try { + overrideScreenByReassigning(window, props); + } catch (e) { + console.warn(e); + } + } + + // eslint-disable-next-line no-unused-vars + function overrideDocumentDimensionsProps(props) { + try { + // FIX THIS = non-zero values here block the injecting process? + // overrideScreenByReassigning(window.document.body, props); + } catch (e) { + console.warn(e); + } + } + + function replace(target, key, value) { + if (target?.[key]) { + target[key] = value; + } + } + + // Replaces all the WebRTC related methods with a recursive ES6 Proxy + // This way, we don't have to model a mock WebRTC API and we still don't get any exceptions. + function blockWebRTC() { + const handler = { + get: () => { + return new Proxy(() => {}, handler); + }, + apply: () => { + return new Proxy(() => {}, handler); + }, + construct: () => { + return new Proxy(() => {}, handler); + }, + }; + + const ConstrProxy = new Proxy(Object, handler); + const proxy = new Proxy(() => {}, handler); + + replace(navigator.mediaDevices, 'getUserMedia', proxy); + replace(navigator, 'webkitGetUserMedia', proxy); + replace(navigator, 'mozGetUserMedia', proxy); + replace(navigator, 'getUserMedia`', proxy); + replace(window, 'webkitRTCPeerConnection', proxy); + + replace(window, 'RTCPeerConnection', ConstrProxy); + replace(window, 'MediaStreamTrack', ConstrProxy); + } + + // eslint-disable-next-line no-unused-vars + function overrideUserAgentData(userAgentData) { + try { + const { brands, mobile, platform, ...highEntropyValues } = userAgentData; + // Override basic properties + const getHighEntropyValues = { + // eslint-disable-next-line + apply: async function (target, ctx, args) { + // Just to throw original validation error + // Remove traces of our Proxy + const stripErrorStack = (stack) => stack + .split('\n') + .filter((line) => !line.includes('at Object.apply')) + .filter((line) => !line.includes('at Object.get')) + .join('\n'); + + try { + if (!args || !args.length) { + return target.apply(ctx, args); + } + const [hints] = args; + await target.apply(ctx, args); + + const data = { brands, mobile, platform }; + hints.forEach((hint) => { + data[hint] = highEntropyValues[hint]; + }); + return data; + } catch (err) { + err.stack = stripErrorStack(err.stack); + throw err; + } + }, + }; + + if(window.navigator.userAgentData){ // Firefox does not contain this property - to be fixed + overridePropertyWithProxy( + Object.getPrototypeOf(window.navigator.userAgentData), + 'getHighEntropyValues', + getHighEntropyValues, + ); + + overrideInstancePrototype(window.navigator.userAgentData, { brands, mobile, platform }); + } + } catch (e) { + console.warn(e); + } + }; + + function fixWindowChrome(){ + if(isChrome && !window.chrome){ + Object.defineProperty(window, 'chrome', { + writable: true, + enumerable: true, + configurable: false, + value: {} // incomplete, todo! + }) + } + } + + // heavily inspired by https://github.com/berstend/puppeteer-extra/, check it out! + function fixPermissions(){ + const isSecure = document.location.protocol.startsWith('https') + + if (isSecure) { + overrideGetterWithProxy(Notification, 'permission', { + apply() { + return 'default' + } + }); + } + + if (!isSecure) { + const handler = { + apply(target, ctx, args) { + const param = (args || [])[0] + + const isNotifications = + param && param.name && param.name === 'notifications' + if (!isNotifications) { + return utils.cache.Reflect.apply(...arguments) + } + + return Promise.resolve( + Object.setPrototypeOf( + { + state: 'denied', + onchange: null + }, + PermissionStatus.prototype + ) + ) + } + }; + + overridePropertyWithProxy(Permissions.prototype, 'query', handler) + } + } + + function fixIframeContentWindow(){ + try { + // Adds a contentWindow proxy to the provided iframe element + const addContentWindowProxy = iframe => { + const contentWindowProxy = { + get(target, key) { + if (key === 'self') { + return this + } + if (key === 'frameElement') { + return iframe + } + + if (key === '0') { + return undefined + } + return Reflect.get(target, key) + } + } + + if (!iframe.contentWindow) { + const proxy = new Proxy(window, contentWindowProxy) + Object.defineProperty(iframe, 'contentWindow', { + get() { + return proxy + }, + set(newValue) { + return newValue // contentWindow is immutable + }, + enumerable: true, + configurable: false + }) + } + } + + // Handles iframe element creation, augments `srcdoc` property so we can intercept further + const handleIframeCreation = (target, thisArg, args) => { + const iframe = target.apply(thisArg, args) + + // We need to keep the originals around + const _iframe = iframe + const _srcdoc = _iframe.srcdoc + + // Add hook for the srcdoc property + // We need to be very surgical here to not break other iframes by accident + Object.defineProperty(iframe, 'srcdoc', { + configurable: true, // Important, so we can reset this later + get: function() { + return _srcdoc + }, + set: function(newValue) { + addContentWindowProxy(this) + // Reset property, the hook is only needed once + Object.defineProperty(iframe, 'srcdoc', { + configurable: false, + writable: false, + value: _srcdoc + }) + _iframe.srcdoc = newValue + } + }) + return iframe + } + + // Adds a hook to intercept iframe creation events + const addIframeCreationSniffer = () => { + /* global document */ + const createElementHandler = { + // Make toString() native + get(target, key) { + return Reflect.get(target, key) + }, + apply: function(target, thisArg, args) { + if (`${args[0]}`.toLowerCase() === 'iframe') { + // Everything as usual + return handleIframeCreation(target, thisArg, args) + } + return target.apply(thisArg, args) + } + } + + // All this just due to iframes with srcdoc bug + overridePropertyWithProxy( + document, + 'createElement', + createElementHandler + ) + } + + // Let's go + addIframeCreationSniffer() + } catch (err) { + // warning message supressed (see https://github.com/apify/fingerprint-suite/issues/61). + // console.warn(err) + } + } + + function fixPluginArray() { + if(window.navigator.plugins.length !== 0){ + return; + } + + Object.defineProperty(navigator, 'plugins', { + get: () => { + const ChromiumPDFPlugin = Object.create(Plugin.prototype, { + description: { value: 'Portable Document Format', enumerable: false }, + filename: { value: 'internal-pdf-viewer', enumerable: false }, + name: { value: 'Chromium PDF Plugin', enumerable: false }, + }); + + return Object.create(PluginArray.prototype, { + length: { value: 1 }, + 0: { value: ChromiumPDFPlugin }, + }); + }, + }); + } + + function runHeadlessFixes(){ + try { + if( isHeadlessChromium ){ + fixWindowChrome(); + fixPermissions(); + fixIframeContentWindow(); + fixPluginArray(); + } + } catch (e) { + console.error(e); + } + } + + function overrideStatic(){ + try { + window.SharedArrayBuffer = undefined; + } catch (e) { + console.error(e); + } + } + + function inject(fp) { + const { + battery, + navigator: { + + extraProperties, + userAgentData, + webdriver, + ...navigatorProps + }, + screen: allScreenProps, + videoCard, + historyLength, + audioCodecs, + videoCodecs, + mockWebRTC, + slim, + // @ts-expect-error internal browser code + } = fp; + + const { + // window screen props + outerHeight, + outerWidth, + devicePixelRatio, + innerWidth, + innerHeight, + screenX, + pageXOffset, + pageYOffset, + + // Document screen props + clientWidth, + clientHeight, + // Ignore hdr for now. + + hasHDR, + // window.screen props + ...newScreen + } = allScreenProps; + + const windowScreenProps = { + innerHeight, + outerHeight, + outerWidth, + innerWidth, + screenX, + pageXOffset, + pageYOffset, + devicePixelRatio, + }; + const documentScreenProps = { + clientHeight, + clientWidth, + }; + + runHeadlessFixes(); + + if (mockWebRTC) blockWebRTC(); + + if (slim) { + // @ts-expect-error internal browser code + // eslint-disable-next-line dot-notation + window['slim'] = true; + } + + overrideIntlAPI(navigatorProps.language); + overrideStatic(); + + if (userAgentData) { + overrideUserAgentData(userAgentData); + } + + if (window.navigator.webdriver) { + navigatorProps.webdriver = false; + } + overrideInstancePrototype(window.navigator, navigatorProps); + + overrideInstancePrototype(window.screen, newScreen); + overrideWindowDimensionsProps(windowScreenProps); + overrideDocumentDimensionsProps(documentScreenProps); + + overrideInstancePrototype(window.history, { length: historyLength }); + + overrideWebGl(videoCard); + overrideCodecs(audioCodecs, videoCodecs); + + overrideBattery(battery); + + } + + +inject(JSON.parse(fp)) +return window.screen;} +""" + diff --git a/tests/unit/playwright_crawler/test_playwright_crawler.py b/tests/unit/playwright_crawler/test_playwright_crawler.py index 33cd69551d..84d49bc081 100644 --- a/tests/unit/playwright_crawler/test_playwright_crawler.py +++ b/tests/unit/playwright_crawler/test_playwright_crawler.py @@ -112,7 +112,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: async def test_firefox_headless_headers(httpbin: URL) -> None: - crawler = PlaywrightCrawler(headless=True, browser_type='firefox') + crawler = PlaywrightCrawler(headless=False, browser_type='firefox') headers = dict[str, str]() @crawler.router.default_handler From a9415ec62a810248ffa97d9c9f850fd13e7e6e9b Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 17 Dec 2024 16:21:12 +0100 Subject: [PATCH 03/32] Use add_init_script Added dev test TODO: Add proper tests Sync header generation with fingerprint generation (use gen from browserforge) --- .../_playwright_browser_controller.py | 6 ++-- .../_injected_page_function.py | 13 ++++---- .../playwright_crawler/_playwright_crawler.py | 7 ++++- .../test_playwright_crawler.py | 30 +++++++++++++++++++ 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index 1e90c31417..64a97ef32a 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -13,7 +13,7 @@ from crawlee.browsers._base_browser_controller import BaseBrowserController from crawlee.browsers._types import BrowserType from crawlee.fingerprint_suite import HeaderGenerator -from crawlee.fingerprint_suite._injected_page_function import javascript_stuff +from crawlee.fingerprint_suite._injected_page_function import create_init_script_with_fingerprint if TYPE_CHECKING: from collections.abc import Mapping @@ -60,7 +60,7 @@ def __init__( self._max_open_pages_per_browser = max_open_pages_per_browser self._header_generator = header_generator - fingerprint_generator_options = fingerprint_generator_options or {} + fingerprint_generator_options = fingerprint_generator_options or {'slim':True} self._fingerprint_generator = fingerprint_generator or FingerprintGenerator(**fingerprint_generator_options) self._browser_context: BrowserContext | None = None @@ -134,7 +134,7 @@ async def new_page( async def _inject_fingerprint_to_page(self, page: Page) -> None: finger_print = self._fingerprint_generator.generate().dumps() - ret = await page.evaluate(javascript_stuff, finger_print) + await page.add_init_script(create_init_script_with_fingerprint(finger_print)) @override async def close(self, *, force: bool = False) -> None: diff --git a/src/crawlee/fingerprint_suite/_injected_page_function.py b/src/crawlee/fingerprint_suite/_injected_page_function.py index d43cdb7bf9..df501af8f7 100644 --- a/src/crawlee/fingerprint_suite/_injected_page_function.py +++ b/src/crawlee/fingerprint_suite/_injected_page_function.py @@ -1,6 +1,7 @@ # This is function that should set the fingerprint on page object. -javascript_stuff = r""" -(fp) =>{ + +_unclosed_body = r""" +(() =>{ const isHeadlessChromium = /headless/i.test(navigator.userAgent) && navigator.plugins.length === 0; const isChrome = navigator.userAgent.includes("Chrome"); const isFirefox = navigator.userAgent.includes("Firefox"); @@ -68,7 +69,7 @@ }, } -function useStrictModeExceptions(prop) { + function useStrictModeExceptions(prop) { if (['caller', 'callee', 'arguments'].includes(prop)) { throw TypeError(`'caller', 'callee', and 'arguments' properties may not be accessed on strict mode functions or the arguments objects for calls to them`); } @@ -842,9 +843,7 @@ overrideBattery(battery); } - - -inject(JSON.parse(fp)) -return window.screen;} """ +def create_init_script_with_fingerprint(fingerprint: str): + return _unclosed_body + f'inject({fingerprint});' + '})()' diff --git a/src/crawlee/playwright_crawler/_playwright_crawler.py b/src/crawlee/playwright_crawler/_playwright_crawler.py index a609674581..86f8ddd2b4 100644 --- a/src/crawlee/playwright_crawler/_playwright_crawler.py +++ b/src/crawlee/playwright_crawler/_playwright_crawler.py @@ -74,6 +74,8 @@ def __init__( browser_options: Mapping[str, Any] | None = None, page_options: Mapping[str, Any] | None = None, headless: bool | None = None, + use_fingerprints: bool = True, + fingerprint_generator_options: dict[str, Any] | None = None, **kwargs: Unpack[BasicCrawlerOptions[PlaywrightCrawlingContext]], ) -> None: """A default constructor. @@ -92,6 +94,8 @@ def __init__( This option should not be used if `browser_pool` is provided. headless: Whether to run the browser in headless mode. This option should not be used if `browser_pool` is provided. + use_fingerprints: Will inject fingerprints + fingerprint_generator_options: Override generated fingerprints with these specific values. kwargs: Additional keyword arguments to pass to the underlying `BasicCrawler`. """ if browser_pool: @@ -109,6 +113,8 @@ def __init__( browser_type=browser_type, browser_options=browser_options, page_options=page_options, + use_fingerprints=use_fingerprints, + fingerprint_generator_options=fingerprint_generator_options, ) self._browser_pool = browser_pool @@ -168,7 +174,6 @@ async def _navigate( async with context.page: if context.request.headers: await context.page.set_extra_http_headers(context.request.headers.model_dump()) - # Navigate to the URL and get response. response = await context.page.goto(context.request.url) if response is None: diff --git a/tests/unit/playwright_crawler/test_playwright_crawler.py b/tests/unit/playwright_crawler/test_playwright_crawler.py index 84d49bc081..e2bfe212e3 100644 --- a/tests/unit/playwright_crawler/test_playwright_crawler.py +++ b/tests/unit/playwright_crawler/test_playwright_crawler.py @@ -168,3 +168,33 @@ async def request_handler(_context: PlaywrightCrawlingContext) -> None: await crawler.run(['https://example.com', str(httpbin)]) assert mock_hook.call_count == 2 + + +async def test_custom_fingerprint(httpbin: URL) -> None: + crawler = PlaywrightCrawler(headless=False, use_fingerprints=True,fingerprint_generator_options={ + "browser":"edge", + "os":"android", + "device":"mobile"}) + response_headers = dict[str, str]() + fingerprints = [] + + @crawler.router.default_handler + async def request_handler(context: PlaywrightCrawlingContext) -> None: + response = await context.response.text() + context_response_headers = dict(json.loads(response)).get('headers', {}) + + for key, val in context_response_headers.items(): + response_headers[key] = val + fingerprints.append(await context.page.evaluate("()=>window.navigator.userAgent")) + + + await crawler.run([Request.from_url(str(httpbin / 'get'))]) + + for fingerprint in fingerprints: + assert "EdgA" in fingerprint + assert "Android" in fingerprint + assert "Mobile" in fingerprint + + + + From 36727a1233da174eb48405e7277d790ae7ab5347 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 18 Dec 2024 11:42:52 +0100 Subject: [PATCH 04/32] WIP Added more tests. --- .../_playwright_browser_controller.py | 44 +++++++++------ .../_injected_page_function.py | 8 ++- .../test_playwright_crawler.py | 56 ++++++++++++++----- 3 files changed, 76 insertions(+), 32 deletions(-) diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index 64a97ef32a..1a8b274433 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -5,10 +5,11 @@ from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, cast -from browserforge.fingerprints import FingerprintGenerator +from browserforge.fingerprints import Fingerprint, FingerprintGenerator from playwright.async_api import BrowserContext, Page, ProxySettings from typing_extensions import override +from crawlee import HttpHeaders from crawlee._utils.docs import docs_group from crawlee.browsers._base_browser_controller import BaseBrowserController from crawlee.browsers._types import BrowserType @@ -40,7 +41,6 @@ def __init__( *, max_open_pages_per_browser: int = 20, header_generator: HeaderGenerator | None = _DEFAULT_HEADER_GENERATOR, - fingerprint_generator: FingerprintGenerator | None = None, use_fingerprints: bool = True, fingerprint_generator_options: dict[str, Any] | None = None, ) -> None: @@ -52,7 +52,6 @@ def __init__( header_generator: An optional `HeaderGenerator` instance used to generate and manage HTTP headers for requests made by the browser. By default, a predefined header generator is used. Set to `None` to disable automatic header modifications. - fingerprint_generator: An optional `FingerprintGenerator` instance used to generate fingerprints, use_fingerprints: Will inject fingerprints fingerprint_generator_options: Override generated fingerprints with these specific values. """ @@ -60,14 +59,14 @@ def __init__( self._max_open_pages_per_browser = max_open_pages_per_browser self._header_generator = header_generator - fingerprint_generator_options = fingerprint_generator_options or {'slim':True} - self._fingerprint_generator = fingerprint_generator or FingerprintGenerator(**fingerprint_generator_options) + self._fingerprint_generator = FingerprintGenerator(**(fingerprint_generator_options or {})) self._browser_context: BrowserContext | None = None self._pages = list[Page]() self._last_page_opened_at = datetime.now(timezone.utc) self._use_fingerprints = use_fingerprints + self._finger_print: Fingerprint | None = None @property @override @@ -110,8 +109,8 @@ async def new_page( page_options: Mapping[str, Any] | None = None, proxy_info: ProxyInfo | None = None, ) -> Page: - if not self._browser_context: - self._browser_context = await self._create_browser_context(proxy_info) + await self._set_fingerprint() + await self._set_browser_context(fingerprint=self._finger_print) if not self.has_free_capacity: raise ValueError('Cannot open more pages in this browser.') @@ -132,9 +131,25 @@ async def new_page( return page + async def _set_browser_context(self, proxy_info: ProxyInfo | None = None, fingerprint: Fingerprint | None = None) -> None: + if not self._browser_context: + if fingerprint: + headers = fingerprint.headers + elif self._header_generator: + common_headers = self._header_generator.get_common_headers() + sec_ch_ua_headers = self._header_generator.get_sec_ch_ua_headers(browser_type=self.browser_type) + user_agent_header = self._header_generator.get_user_agent_header(browser_type=self.browser_type) + headers = dict(common_headers | sec_ch_ua_headers | user_agent_header) + else: + headers = None + self._browser_context = await self._create_browser_context(proxy_info, headers) + + async def _set_fingerprint(self): + if self._use_fingerprints and not self._finger_print: + self._finger_print = self._fingerprint_generator.generate() + async def _inject_fingerprint_to_page(self, page: Page) -> None: - finger_print = self._fingerprint_generator.generate().dumps() - await page.add_init_script(create_init_script_with_fingerprint(finger_print)) + await page.add_init_script(create_init_script_with_fingerprint(self._finger_print.dumps())) @override async def close(self, *, force: bool = False) -> None: @@ -151,14 +166,11 @@ def _on_page_close(self, page: Page) -> None: """Handle actions after a page is closed.""" self._pages.remove(page) - async def _create_browser_context(self, proxy_info: ProxyInfo | None = None) -> BrowserContext: + async def _create_browser_context(self, proxy_info: ProxyInfo | None = None, headers: HttpHeaders | None = None) -> BrowserContext: """Create a new browser context with the specified proxy settings.""" - if self._header_generator: - common_headers = self._header_generator.get_common_headers() - sec_ch_ua_headers = self._header_generator.get_sec_ch_ua_headers(browser_type=self.browser_type) - user_agent_header = self._header_generator.get_user_agent_header(browser_type=self.browser_type) - extra_http_headers = dict(common_headers | sec_ch_ua_headers | user_agent_header) - user_agent = user_agent_header.get('User-Agent') + if headers: + extra_http_headers = headers + user_agent = headers.get('User-Agent') else: extra_http_headers = None user_agent = None diff --git a/src/crawlee/fingerprint_suite/_injected_page_function.py b/src/crawlee/fingerprint_suite/_injected_page_function.py index df501af8f7..27665d1b0a 100644 --- a/src/crawlee/fingerprint_suite/_injected_page_function.py +++ b/src/crawlee/fingerprint_suite/_injected_page_function.py @@ -1,7 +1,7 @@ # This is function that should set the fingerprint on page object. -_unclosed_body = r""" -(() =>{ +_js_function = r""" +((fp) =>{ const isHeadlessChromium = /headless/i.test(navigator.userAgent) && navigator.plugins.length === 0; const isChrome = navigator.userAgent.includes("Chrome"); const isFirefox = navigator.userAgent.includes("Firefox"); @@ -843,7 +843,9 @@ overrideBattery(battery); } +return inject(fp); +}) """ def create_init_script_with_fingerprint(fingerprint: str): - return _unclosed_body + f'inject({fingerprint});' + '})()' + return _js_function + f'({fingerprint})' diff --git a/tests/unit/playwright_crawler/test_playwright_crawler.py b/tests/unit/playwright_crawler/test_playwright_crawler.py index e2bfe212e3..c36d92f96b 100644 --- a/tests/unit/playwright_crawler/test_playwright_crawler.py +++ b/tests/unit/playwright_crawler/test_playwright_crawler.py @@ -8,6 +8,8 @@ from typing import TYPE_CHECKING from unittest import mock +from browserforge.fingerprints import Screen + from crawlee import Glob, Request from crawlee.fingerprint_suite._consts import ( PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA, @@ -84,7 +86,7 @@ async def request_handler(_context: PlaywrightCrawlingContext) -> None: async def test_chromium_headless_headers(httpbin: URL) -> None: - crawler = PlaywrightCrawler(headless=True, browser_type='chromium') + crawler = PlaywrightCrawler(headless=True, browser_type='chromium', use_fingerprints=False) headers = dict[str, str]() @crawler.router.default_handler @@ -112,7 +114,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: async def test_firefox_headless_headers(httpbin: URL) -> None: - crawler = PlaywrightCrawler(headless=False, browser_type='firefox') + crawler = PlaywrightCrawler(headless=False, browser_type='firefox', use_fingerprints=False) headers = dict[str, str]() @crawler.router.default_handler @@ -170,13 +172,19 @@ async def request_handler(_context: PlaywrightCrawlingContext) -> None: assert mock_hook.call_count == 2 -async def test_custom_fingerprint(httpbin: URL) -> None: - crawler = PlaywrightCrawler(headless=False, use_fingerprints=True,fingerprint_generator_options={ - "browser":"edge", - "os":"android", - "device":"mobile"}) +async def test_custom_fingerprint_uses_generator_options(httpbin: URL) -> None: + MIN_WIDTH = 300 + MAX_WIDTH = 600 + MIN_HEIGHT = 500 + MAX_HEIGHT = 1200 + crawler = PlaywrightCrawler(headless=True, use_fingerprints=True,fingerprint_generator_options={ + 'browser':'edge', + 'os':'android', + 'screen': Screen(min_width=MIN_WIDTH, max_width=MAX_WIDTH, min_height=MIN_HEIGHT, max_height=MAX_HEIGHT) + }) + response_headers = dict[str, str]() - fingerprints = [] + fingerprints = dict[str, str]() @crawler.router.default_handler async def request_handler(context: PlaywrightCrawlingContext) -> None: @@ -185,16 +193,38 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: for key, val in context_response_headers.items(): response_headers[key] = val - fingerprints.append(await context.page.evaluate("()=>window.navigator.userAgent")) + + for relevant_key in ('window.navigator.userAgent', 'window.navigator.userAgentData', 'window.screen.height', 'window.screen.width'): + fingerprints[relevant_key] = await context.page.evaluate(f'()=>{relevant_key}') + await crawler.run([Request.from_url(str(httpbin / 'get'))]) - for fingerprint in fingerprints: - assert "EdgA" in fingerprint - assert "Android" in fingerprint - assert "Mobile" in fingerprint + assert 'EdgA' in fingerprints['window.navigator.userAgent'] + assert fingerprints['window.navigator.userAgentData']['platform'] == 'Android' + assert MIN_WIDTH <= fingerprints['window.screen.width'] <= MAX_WIDTH + assert MIN_HEIGHT <= fingerprints['window.screen.height'] <= MAX_HEIGHT + + +async def test_custom_fingerprint_matches_header_user_agent(httpbin: URL) -> None: + """Test that generated fingerprint and header have matching user agent.""" + + crawler = PlaywrightCrawler(headless=True, use_fingerprints=True) + response_headers = dict[str, str]() + fingerprints = dict[str, str]() + + @crawler.router.default_handler + async def request_handler(context: PlaywrightCrawlingContext) -> None: + response = await context.response.text() + context_response_headers = dict(json.loads(response)).get('headers', {}) + + response_headers['User-Agent'] = context_response_headers['User-Agent'] + fingerprints['window.navigator.userAgent'] = await context.page.evaluate('()=>window.navigator.userAgent') + + await crawler.run([Request.from_url(str(httpbin / 'get'))]) + assert response_headers['User-Agent'] == fingerprints['window.navigator.userAgent'] From 42eff809d668a1f367afac40b75c4f361425ccf2 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 18 Dec 2024 13:29:37 +0100 Subject: [PATCH 05/32] Fix format, type check and tests. --- .../_playwright_browser_controller.py | 64 ++++++++++++------- .../_injected_page_function.py | 8 ++- .../playwright_crawler/_playwright_crawler.py | 1 + .../test_playwright_crawler.py | 42 ++++++------ 4 files changed, 71 insertions(+), 44 deletions(-) diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index 1a8b274433..bdc60a06b1 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -9,7 +9,6 @@ from playwright.async_api import BrowserContext, Page, ProxySettings from typing_extensions import override -from crawlee import HttpHeaders from crawlee._utils.docs import docs_group from crawlee.browsers._base_browser_controller import BaseBrowserController from crawlee.browsers._types import BrowserType @@ -66,7 +65,7 @@ def __init__( self._last_page_opened_at = datetime.now(timezone.utc) self._use_fingerprints = use_fingerprints - self._finger_print: Fingerprint | None = None + self._fingerprint: Fingerprint | None = None @property @override @@ -109,14 +108,15 @@ async def new_page( page_options: Mapping[str, Any] | None = None, proxy_info: ProxyInfo | None = None, ) -> Page: - await self._set_fingerprint() - await self._set_browser_context(fingerprint=self._finger_print) + if not self._browser_context: + await self._set_fingerprint() + await self._set_browser_context(fingerprint=self._fingerprint) if not self.has_free_capacity: raise ValueError('Cannot open more pages in this browser.') page_options = dict(page_options) if page_options else {} - page = await self._browser_context.new_page(**page_options) + page = await self._get_browser_context().new_page(**page_options) # Handle page close event page.on(event='close', f=self._on_page_close) @@ -131,25 +131,43 @@ async def new_page( return page - async def _set_browser_context(self, proxy_info: ProxyInfo | None = None, fingerprint: Fingerprint | None = None) -> None: + async def _set_browser_context( + self, proxy_info: ProxyInfo | None = None, fingerprint: Fingerprint | None = None + ) -> None: + """Set browser context. + + Set headers based on fingerprint if available to ensure consistency between headers and fingerprint. + Fallback to header generator if no fingerprint is available. + """ + if fingerprint: + headers = fingerprint.headers + elif self._header_generator: + common_headers = self._header_generator.get_common_headers() + sec_ch_ua_headers = self._header_generator.get_sec_ch_ua_headers(browser_type=self.browser_type) + user_agent_header = self._header_generator.get_user_agent_header(browser_type=self.browser_type) + headers = dict(common_headers | sec_ch_ua_headers | user_agent_header) + else: + headers = None + self._browser_context = await self._create_browser_context(proxy_info, headers) + + def _get_browser_context(self) -> BrowserContext: if not self._browser_context: - if fingerprint: - headers = fingerprint.headers - elif self._header_generator: - common_headers = self._header_generator.get_common_headers() - sec_ch_ua_headers = self._header_generator.get_sec_ch_ua_headers(browser_type=self.browser_type) - user_agent_header = self._header_generator.get_user_agent_header(browser_type=self.browser_type) - headers = dict(common_headers | sec_ch_ua_headers | user_agent_header) - else: - headers = None - self._browser_context = await self._create_browser_context(proxy_info, headers) - - async def _set_fingerprint(self): - if self._use_fingerprints and not self._finger_print: - self._finger_print = self._fingerprint_generator.generate() + raise RuntimeError('Browser context was not set yet.') + return self._browser_context + + async def _set_fingerprint(self) -> None: + if self._use_fingerprints and not self._fingerprint: + self._fingerprint = self._fingerprint_generator.generate() + + def _get_fingerprint(self) -> Fingerprint: + if not self._use_fingerprints: + raise RuntimeError('Fingerprint was is not allowed. use_fingerprints = False.') + if not self._fingerprint: + raise RuntimeError('Fingerprint was not set yet.') + return self._fingerprint async def _inject_fingerprint_to_page(self, page: Page) -> None: - await page.add_init_script(create_init_script_with_fingerprint(self._finger_print.dumps())) + await page.add_init_script(create_init_script_with_fingerprint(self._get_fingerprint().dumps())) @override async def close(self, *, force: bool = False) -> None: @@ -166,7 +184,9 @@ def _on_page_close(self, page: Page) -> None: """Handle actions after a page is closed.""" self._pages.remove(page) - async def _create_browser_context(self, proxy_info: ProxyInfo | None = None, headers: HttpHeaders | None = None) -> BrowserContext: + async def _create_browser_context( + self, proxy_info: ProxyInfo | None = None, headers: dict[str, str] | None = None + ) -> BrowserContext: """Create a new browser context with the specified proxy settings.""" if headers: extra_http_headers = headers diff --git a/src/crawlee/fingerprint_suite/_injected_page_function.py b/src/crawlee/fingerprint_suite/_injected_page_function.py index 27665d1b0a..a9fa5b3b1e 100644 --- a/src/crawlee/fingerprint_suite/_injected_page_function.py +++ b/src/crawlee/fingerprint_suite/_injected_page_function.py @@ -1,5 +1,5 @@ -# This is function that should set the fingerprint on page object. - +# flake8: noqa +# No formating check on injected JS code. _js_function = r""" ((fp) =>{ const isHeadlessChromium = /headless/i.test(navigator.userAgent) && navigator.plugins.length === 0; @@ -847,5 +847,7 @@ }) """ -def create_init_script_with_fingerprint(fingerprint: str): + +def create_init_script_with_fingerprint(fingerprint: str) -> str: + """Create string that contains Javascript function call that will set fingerprints in page.""" return _js_function + f'({fingerprint})' diff --git a/src/crawlee/playwright_crawler/_playwright_crawler.py b/src/crawlee/playwright_crawler/_playwright_crawler.py index 86f8ddd2b4..bf947bbf8d 100644 --- a/src/crawlee/playwright_crawler/_playwright_crawler.py +++ b/src/crawlee/playwright_crawler/_playwright_crawler.py @@ -74,6 +74,7 @@ def __init__( browser_options: Mapping[str, Any] | None = None, page_options: Mapping[str, Any] | None = None, headless: bool | None = None, + *, use_fingerprints: bool = True, fingerprint_generator_options: dict[str, Any] | None = None, **kwargs: Unpack[BasicCrawlerOptions[PlaywrightCrawlingContext]], diff --git a/tests/unit/playwright_crawler/test_playwright_crawler.py b/tests/unit/playwright_crawler/test_playwright_crawler.py index c36d92f96b..20864226c6 100644 --- a/tests/unit/playwright_crawler/test_playwright_crawler.py +++ b/tests/unit/playwright_crawler/test_playwright_crawler.py @@ -5,7 +5,7 @@ from __future__ import annotations import json -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from unittest import mock from browserforge.fingerprints import Screen @@ -173,18 +173,22 @@ async def request_handler(_context: PlaywrightCrawlingContext) -> None: async def test_custom_fingerprint_uses_generator_options(httpbin: URL) -> None: - MIN_WIDTH = 300 - MAX_WIDTH = 600 - MIN_HEIGHT = 500 - MAX_HEIGHT = 1200 - crawler = PlaywrightCrawler(headless=True, use_fingerprints=True,fingerprint_generator_options={ - 'browser':'edge', - 'os':'android', - 'screen': Screen(min_width=MIN_WIDTH, max_width=MAX_WIDTH, min_height=MIN_HEIGHT, max_height=MAX_HEIGHT) - }) + min_width = 300 + max_width = 600 + min_height = 500 + max_height = 1200 + crawler = PlaywrightCrawler( + headless=True, + use_fingerprints=True, + fingerprint_generator_options={ + 'browser': 'edge', + 'os': 'android', + 'screen': Screen(min_width=min_width, max_width=max_width, min_height=min_height, max_height=max_height), + }, + ) response_headers = dict[str, str]() - fingerprints = dict[str, str]() + fingerprints = dict[str, Any]() @crawler.router.default_handler async def request_handler(context: PlaywrightCrawlingContext) -> None: @@ -194,17 +198,20 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: for key, val in context_response_headers.items(): response_headers[key] = val - for relevant_key in ('window.navigator.userAgent', 'window.navigator.userAgentData', 'window.screen.height', 'window.screen.width'): + for relevant_key in ( + 'window.navigator.userAgent', + 'window.navigator.userAgentData', + 'window.screen.height', + 'window.screen.width', + ): fingerprints[relevant_key] = await context.page.evaluate(f'()=>{relevant_key}') - - await crawler.run([Request.from_url(str(httpbin / 'get'))]) assert 'EdgA' in fingerprints['window.navigator.userAgent'] assert fingerprints['window.navigator.userAgentData']['platform'] == 'Android' - assert MIN_WIDTH <= fingerprints['window.screen.width'] <= MAX_WIDTH - assert MIN_HEIGHT <= fingerprints['window.screen.height'] <= MAX_HEIGHT + assert min_width <= int(fingerprints['window.screen.width']) <= max_width + assert min_height <= int(fingerprints['window.screen.height']) <= max_height async def test_custom_fingerprint_matches_header_user_agent(httpbin: URL) -> None: @@ -225,6 +232,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: await crawler.run([Request.from_url(str(httpbin / 'get'))]) assert response_headers['User-Agent'] == fingerprints['window.navigator.userAgent'] - - - From 998cbb6f9a6fcd019e7e254eef8ab816184a58ce Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 18 Dec 2024 14:56:40 +0100 Subject: [PATCH 06/32] Fix rootcause for flakiness in fingerprint generation Use browser_pool_options from crawler to pass fingerprint related stuff to be similar to JS --- src/crawlee/browsers/_browser_pool.py | 10 +++++----- .../_playwright_browser_controller.py | 18 ++++++++++++++--- .../browsers/_playwright_browser_plugin.py | 4 ++-- .../playwright_crawler/_playwright_crawler.py | 14 +++++-------- .../test_playwright_crawler.py | 20 +++++++++++-------- 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/src/crawlee/browsers/_browser_pool.py b/src/crawlee/browsers/_browser_pool.py index 29b2868ec3..6c52eb0419 100644 --- a/src/crawlee/browsers/_browser_pool.py +++ b/src/crawlee/browsers/_browser_pool.py @@ -68,8 +68,8 @@ def __init__( close_inactive_browsers_interval: The interval at which the pool checks for inactive browsers and closes them. The browser is considered as inactive if it has no active pages and has been idle for the specified period. - use_fingerprints: Will inject fingerprints - fingerprint_generator_options: Override generated fingerprints with these specific values. + use_fingerprints: Inject generated fingerprints to page. + fingerprint_generator_options: Override generated fingerprints with these specific values, if possible. """ self._plugins = plugins or [PlaywrightBrowserPlugin()] self._operation_timeout = operation_timeout @@ -110,7 +110,7 @@ def with_default_plugin( browser_options: Mapping[str, Any] | None = None, page_options: Mapping[str, Any] | None = None, headless: bool | None = None, - use_fingerprints: bool = True, + use_fingerprints: bool = False, fingerprint_generator_options: dict[str, Any] | None = None, **kwargs: Any, ) -> BrowserPool: @@ -125,8 +125,8 @@ def with_default_plugin( Playwright's `browser_context.new_page` method. For more details, refer to the Playwright documentation: https://playwright.dev/python/docs/api/class-browsercontext#browser-context-new-page. headless: Whether to run the browser in headless mode. - use_fingerprints: Will inject fingerprints - fingerprint_generator_options: Override generated fingerprints with these specific values. + use_fingerprints: Inject generated fingerprints to page. + fingerprint_generator_options: Override generated fingerprints with these specific values, if possible. kwargs: Additional arguments for default constructor. """ plugin_options: dict = defaultdict(dict) diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index bdc60a06b1..5d2f83828d 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -51,8 +51,8 @@ def __init__( header_generator: An optional `HeaderGenerator` instance used to generate and manage HTTP headers for requests made by the browser. By default, a predefined header generator is used. Set to `None` to disable automatic header modifications. - use_fingerprints: Will inject fingerprints - fingerprint_generator_options: Override generated fingerprints with these specific values. + use_fingerprints: Inject generated fingerprints to page. + fingerprint_generator_options: Override generated fingerprints with these specific values, if possible. """ self._browser = browser self._max_open_pages_per_browser = max_open_pages_per_browser @@ -157,7 +157,19 @@ def _get_browser_context(self) -> BrowserContext: async def _set_fingerprint(self) -> None: if self._use_fingerprints and not self._fingerprint: - self._fingerprint = self._fingerprint_generator.generate() + while fingerprint := self._fingerprint_generator.generate(): + if self._is_good_fingerprint(fingerprint): + break + self._fingerprint = fingerprint + + @staticmethod + def _is_good_fingerprint(fingerprint: Fingerprint) -> bool: + """Check if fingerprint is ok to use. + + By trial and error it was found out that some generated fingerprints are not working well. + All fingerprints that were not working well had 'Te': 'trailers' in headers. + """ + return fingerprint.headers.get('Te', '') != 'trailers' def _get_fingerprint(self) -> Fingerprint: if not self._use_fingerprints: diff --git a/src/crawlee/browsers/_playwright_browser_plugin.py b/src/crawlee/browsers/_playwright_browser_plugin.py index 16c36b0bff..34c3e82cbf 100644 --- a/src/crawlee/browsers/_playwright_browser_plugin.py +++ b/src/crawlee/browsers/_playwright_browser_plugin.py @@ -53,8 +53,8 @@ def __init__( https://playwright.dev/python/docs/api/class-browsercontext#browser-context-new-page. max_open_pages_per_browser: The maximum number of pages that can be opened in a single browser instance. Once reached, a new browser instance will be launched to handle the excess. - use_fingerprints: Will inject fingerprints - fingerprint_generator_options: Override generated fingerprints with these specific values. + use_fingerprints: Inject generated fingerprints to page. + fingerprint_generator_options: Override generated fingerprints with these specific values, if possible. """ self._browser_type = browser_type diff --git a/src/crawlee/playwright_crawler/_playwright_crawler.py b/src/crawlee/playwright_crawler/_playwright_crawler.py index bf947bbf8d..35072eb15a 100644 --- a/src/crawlee/playwright_crawler/_playwright_crawler.py +++ b/src/crawlee/playwright_crawler/_playwright_crawler.py @@ -69,20 +69,20 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: def __init__( self, + *, browser_pool: BrowserPool | None = None, + browser_pool_options: Mapping[str, Any] | None = None, browser_type: BrowserType | None = None, browser_options: Mapping[str, Any] | None = None, page_options: Mapping[str, Any] | None = None, headless: bool | None = None, - *, - use_fingerprints: bool = True, - fingerprint_generator_options: dict[str, Any] | None = None, **kwargs: Unpack[BasicCrawlerOptions[PlaywrightCrawlingContext]], ) -> None: """A default constructor. Args: browser_pool: A `BrowserPool` instance to be used for launching the browsers and getting pages. + browser_pool_options: Arguments passed to `BrowserPool`. browser_type: The type of browser to launch ('chromium', 'firefox', or 'webkit'). This option should not be used if `browser_pool` is provided. browser_options: Keyword arguments to pass to the browser launch method. These options are provided @@ -95,8 +95,6 @@ def __init__( This option should not be used if `browser_pool` is provided. headless: Whether to run the browser in headless mode. This option should not be used if `browser_pool` is provided. - use_fingerprints: Will inject fingerprints - fingerprint_generator_options: Override generated fingerprints with these specific values. kwargs: Additional keyword arguments to pass to the underlying `BasicCrawler`. """ if browser_pool: @@ -114,8 +112,7 @@ def __init__( browser_type=browser_type, browser_options=browser_options, page_options=page_options, - use_fingerprints=use_fingerprints, - fingerprint_generator_options=fingerprint_generator_options, + **(browser_pool_options or {}), ) self._browser_pool = browser_pool @@ -175,8 +172,7 @@ async def _navigate( async with context.page: if context.request.headers: await context.page.set_extra_http_headers(context.request.headers.model_dump()) - response = await context.page.goto(context.request.url) - + response = await context.page.goto(context.request.url, wait_until='domcontentloaded') if response is None: raise SessionError(f'Failed to load the URL: {context.request.url}') diff --git a/tests/unit/playwright_crawler/test_playwright_crawler.py b/tests/unit/playwright_crawler/test_playwright_crawler.py index 20864226c6..623ef010ef 100644 --- a/tests/unit/playwright_crawler/test_playwright_crawler.py +++ b/tests/unit/playwright_crawler/test_playwright_crawler.py @@ -86,7 +86,7 @@ async def request_handler(_context: PlaywrightCrawlingContext) -> None: async def test_chromium_headless_headers(httpbin: URL) -> None: - crawler = PlaywrightCrawler(headless=True, browser_type='chromium', use_fingerprints=False) + crawler = PlaywrightCrawler(headless=True, browser_type='chromium') headers = dict[str, str]() @crawler.router.default_handler @@ -114,7 +114,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: async def test_firefox_headless_headers(httpbin: URL) -> None: - crawler = PlaywrightCrawler(headless=False, browser_type='firefox', use_fingerprints=False) + crawler = PlaywrightCrawler(headless=False, browser_type='firefox') headers = dict[str, str]() @crawler.router.default_handler @@ -179,11 +179,15 @@ async def test_custom_fingerprint_uses_generator_options(httpbin: URL) -> None: max_height = 1200 crawler = PlaywrightCrawler( headless=True, - use_fingerprints=True, - fingerprint_generator_options={ - 'browser': 'edge', - 'os': 'android', - 'screen': Screen(min_width=min_width, max_width=max_width, min_height=min_height, max_height=max_height), + browser_pool_options={ + 'use_fingerprints': True, + 'fingerprint_generator_options': { + 'browser': 'edge', + 'os': 'android', + 'screen': Screen( + min_width=min_width, max_width=max_width, min_height=min_height, max_height=max_height + ), + }, }, ) @@ -217,7 +221,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: async def test_custom_fingerprint_matches_header_user_agent(httpbin: URL) -> None: """Test that generated fingerprint and header have matching user agent.""" - crawler = PlaywrightCrawler(headless=True, use_fingerprints=True) + crawler = PlaywrightCrawler(headless=True, browser_pool_options={'use_fingerprints': True}) response_headers = dict[str, str]() fingerprints = dict[str, str]() From e1025c8e8322ef58f60f8c6026e18403b8561933 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 19 Dec 2024 10:34:20 +0100 Subject: [PATCH 07/32] Use browserforge.injector code for fingerprints --- .../_playwright_browser_controller.py | 107 +-- .../_injected_page_function.py | 853 ------------------ .../playwright_crawler/_playwright_crawler.py | 2 +- 3 files changed, 36 insertions(+), 926 deletions(-) delete mode 100644 src/crawlee/fingerprint_suite/_injected_page_function.py diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index 5d2f83828d..d66c6c8602 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, cast -from browserforge.fingerprints import Fingerprint, FingerprintGenerator +from browserforge.injectors.playwright import AsyncNewContext from playwright.async_api import BrowserContext, Page, ProxySettings from typing_extensions import override @@ -13,7 +13,6 @@ from crawlee.browsers._base_browser_controller import BaseBrowserController from crawlee.browsers._types import BrowserType from crawlee.fingerprint_suite import HeaderGenerator -from crawlee.fingerprint_suite._injected_page_function import create_init_script_with_fingerprint if TYPE_CHECKING: from collections.abc import Mapping @@ -58,14 +57,12 @@ def __init__( self._max_open_pages_per_browser = max_open_pages_per_browser self._header_generator = header_generator - self._fingerprint_generator = FingerprintGenerator(**(fingerprint_generator_options or {})) - self._browser_context: BrowserContext | None = None self._pages = list[Page]() self._last_page_opened_at = datetime.now(timezone.utc) self._use_fingerprints = use_fingerprints - self._fingerprint: Fingerprint | None = None + self._fingerprint_generator_options = fingerprint_generator_options @property @override @@ -109,8 +106,9 @@ async def new_page( proxy_info: ProxyInfo | None = None, ) -> Page: if not self._browser_context: - await self._set_fingerprint() - await self._set_browser_context(fingerprint=self._fingerprint) + await self._set_browser_context( + fingerprint_options=self._fingerprint_generator_options, proxy_info=proxy_info + ) if not self.has_free_capacity: raise ValueError('Cannot open more pages in this browser.') @@ -125,62 +123,54 @@ async def new_page( self._pages.append(page) self._last_page_opened_at = datetime.now(timezone.utc) - # Inject fingerprint - if self._use_fingerprints: - await self._inject_fingerprint_to_page(page) - return page async def _set_browser_context( - self, proxy_info: ProxyInfo | None = None, fingerprint: Fingerprint | None = None + self, proxy_info: ProxyInfo | None = None, fingerprint_options: dict | None = None ) -> None: """Set browser context. - Set headers based on fingerprint if available to ensure consistency between headers and fingerprint. - Fallback to header generator if no fingerprint is available. + Create context using `browserforge` if `_use_fingerprints` is True. + Create context without fingerprints with headers based header generator if available. """ - if fingerprint: - headers = fingerprint.headers - elif self._header_generator: + proxy = ( + ProxySettings( + server=f'{proxy_info.scheme}://{proxy_info.hostname}:{proxy_info.port}', + username=proxy_info.username, + password=proxy_info.password, + ) + if proxy_info + else None + ) + + if self._use_fingerprints: + self._browser_context = await AsyncNewContext( + browser=self._browser, fingerprint_options=(fingerprint_options or {}), proxy=proxy + ) + return + + if self._header_generator: common_headers = self._header_generator.get_common_headers() sec_ch_ua_headers = self._header_generator.get_sec_ch_ua_headers(browser_type=self.browser_type) user_agent_header = self._header_generator.get_user_agent_header(browser_type=self.browser_type) headers = dict(common_headers | sec_ch_ua_headers | user_agent_header) + extra_http_headers = headers + user_agent = headers.get('User-Agent') else: - headers = None - self._browser_context = await self._create_browser_context(proxy_info, headers) + extra_http_headers = None + user_agent = None + + self._browser_context = await self._browser.new_context( + user_agent=user_agent, + extra_http_headers=extra_http_headers, + proxy=proxy, + ) def _get_browser_context(self) -> BrowserContext: if not self._browser_context: raise RuntimeError('Browser context was not set yet.') return self._browser_context - async def _set_fingerprint(self) -> None: - if self._use_fingerprints and not self._fingerprint: - while fingerprint := self._fingerprint_generator.generate(): - if self._is_good_fingerprint(fingerprint): - break - self._fingerprint = fingerprint - - @staticmethod - def _is_good_fingerprint(fingerprint: Fingerprint) -> bool: - """Check if fingerprint is ok to use. - - By trial and error it was found out that some generated fingerprints are not working well. - All fingerprints that were not working well had 'Te': 'trailers' in headers. - """ - return fingerprint.headers.get('Te', '') != 'trailers' - - def _get_fingerprint(self) -> Fingerprint: - if not self._use_fingerprints: - raise RuntimeError('Fingerprint was is not allowed. use_fingerprints = False.') - if not self._fingerprint: - raise RuntimeError('Fingerprint was not set yet.') - return self._fingerprint - - async def _inject_fingerprint_to_page(self, page: Page) -> None: - await page.add_init_script(create_init_script_with_fingerprint(self._get_fingerprint().dumps())) - @override async def close(self, *, force: bool = False) -> None: if force: @@ -195,30 +185,3 @@ async def close(self, *, force: bool = False) -> None: def _on_page_close(self, page: Page) -> None: """Handle actions after a page is closed.""" self._pages.remove(page) - - async def _create_browser_context( - self, proxy_info: ProxyInfo | None = None, headers: dict[str, str] | None = None - ) -> BrowserContext: - """Create a new browser context with the specified proxy settings.""" - if headers: - extra_http_headers = headers - user_agent = headers.get('User-Agent') - else: - extra_http_headers = None - user_agent = None - - proxy = ( - ProxySettings( - server=f'{proxy_info.scheme}://{proxy_info.hostname}:{proxy_info.port}', - username=proxy_info.username, - password=proxy_info.password, - ) - if proxy_info - else None - ) - - return await self._browser.new_context( - user_agent=user_agent, - extra_http_headers=extra_http_headers, - proxy=proxy, - ) diff --git a/src/crawlee/fingerprint_suite/_injected_page_function.py b/src/crawlee/fingerprint_suite/_injected_page_function.py deleted file mode 100644 index a9fa5b3b1e..0000000000 --- a/src/crawlee/fingerprint_suite/_injected_page_function.py +++ /dev/null @@ -1,853 +0,0 @@ -# flake8: noqa -# No formating check on injected JS code. -_js_function = r""" -((fp) =>{ - const isHeadlessChromium = /headless/i.test(navigator.userAgent) && navigator.plugins.length === 0; - const isChrome = navigator.userAgent.includes("Chrome"); - const isFirefox = navigator.userAgent.includes("Firefox"); - const isSafari = navigator.userAgent.includes("Safari") && !navigator.userAgent.includes("Chrome"); - - let slim = null; - function getSlim() { - if(slim === null) { - slim = window.slim || false; - if(typeof window.slim !== 'undefined') { - delete window.slim; - } - } - - return slim; - } - - // This file contains utils that are build and included on the window object with some randomized prefix. - - // some protections can mess with these to prevent the overrides - our script is first so we can reference the old values. - const cache = { - Reflect: { - get: Reflect.get.bind(Reflect), - apply: Reflect.apply.bind(Reflect), - }, - // Used in `makeNativeString` - nativeToStringStr: `${Function.toString}`, // => `function toString() { [native code] }` - }; - - /** - * @param masterObject Object to override. - * @param propertyName Property to override. - * @param proxyHandler Proxy handled with the new value. - */ - function overridePropertyWithProxy(masterObject, propertyName, proxyHandler) { - const originalObject = masterObject[propertyName]; - const proxy = new Proxy(masterObject[propertyName], stripProxyFromErrors(proxyHandler)); - - redefineProperty(masterObject, propertyName, { value: proxy }); - redirectToString(proxy, originalObject); - } - - const prototypeProxyHandler = { - setPrototypeOf: (target, newProto) => { - try { - throw new TypeError('Cyclic __proto__ value'); - } catch (e) { - const oldStack = e.stack; - const oldProto = Object.getPrototypeOf(target); - Object.setPrototypeOf(target, newProto); - try { - // shouldn't throw if prototype is okay, will throw if there is a prototype cycle (maximum call stack size exceeded). - target['nonexistentpropertytest']; - return true; - } - catch (err) { - Object.setPrototypeOf(target, oldProto); - if (oldStack.includes('Reflect.setPrototypeOf')) return false; - const newError = new TypeError('Cyclic __proto__ value'); - const stack = oldStack.split('\n'); - newError.stack = [stack[0], ...stack.slice(2)].join('\n'); - throw newError; - } - } - }, - } - - function useStrictModeExceptions(prop) { - if (['caller', 'callee', 'arguments'].includes(prop)) { - throw TypeError(`'caller', 'callee', and 'arguments' properties may not be accessed on strict mode functions or the arguments objects for calls to them`); - } - } - - /** - * @param masterObject Object to override. - * @param propertyName Property to override. - * @param proxyHandler ES6 Proxy handler object with a get handle only. - */ - function overrideGetterWithProxy(masterObject, propertyName, proxyHandler) { - const fn = Object.getOwnPropertyDescriptor(masterObject, propertyName).get; - const fnStr = fn.toString; // special getter function string - const proxyObj = new Proxy(fn, { - ...stripProxyFromErrors(proxyHandler), - ...prototypeProxyHandler, - }); - - redefineProperty(masterObject, propertyName, { get: proxyObj }); - redirectToString(proxyObj, fnStr); - } - - /** - * @param instance Instance to override. - * @param overrideObj New instance values. - */ - // eslint-disable-next-line no-unused-vars - function overrideInstancePrototype(instance, overrideObj) { - try { - Object.keys(overrideObj).forEach((key) => { - if (!(overrideObj[key] === null)) { - try { - overrideGetterWithProxy( - Object.getPrototypeOf(instance), - key, - makeHandler().getterValue(overrideObj[key]), - ); - } catch (e) { - return false; - // console.error(`Could not override property: ${key} on ${instance}. Reason: ${e.message} `); // some fingerprinting services can be listening - } - } - }); - } catch (e) { - console.error(e); - } - } - - /** - * Updates the .toString method in Function.prototype to return a native string representation of the function. - * @param {*} proxyObj - * @param {*} originalObj - */ - function redirectToString(proxyObj, originalObj) { - if(getSlim()) return; - - const handler = { - setPrototypeOf: (target, newProto) => { - try { - throw new TypeError('Cyclic __proto__ value'); - } catch (e) { - if (e.stack.includes('Reflect.setPrototypeOf')) return false; - // const stack = e.stack.split('\n'); - // e.stack = [stack[0], ...stack.slice(2)].join('\n'); - throw e; - } - }, - apply(target, ctx) { - // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + ""` - if (ctx === Function.prototype.toString) { - return makeNativeString('toString'); - } - - // `toString` targeted at our proxied Object detected - if (ctx === proxyObj) { - // Return the toString representation of our original object if possible - return makeNativeString(proxyObj.name); - } - - // Check if the toString prototype of the context is the same as the global prototype, - // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case - const hasSameProto = Object.getPrototypeOf( - Function.prototype.toString, - ).isPrototypeOf(ctx.toString); // eslint-disable-line no-prototype-builtins - - if (!hasSameProto) { - // Pass the call on to the local Function.prototype.toString instead - return ctx.toString(); - } - - if (Object.getPrototypeOf(ctx) === proxyObj){ - try { - return target.call(ctx); - } catch (err) { - err.stack = err.stack.replace( - 'at Object.toString (', - 'at Function.toString (', - ); - throw err; - }} - return target.call(ctx); - }, - get: function(target, prop, receiver) { - if (prop === 'toString') { - return new Proxy(target.toString, { - apply: function(tget, thisArg, argumentsList) { - try { - return tget.bind(thisArg)(...argumentsList); - } catch (err) { - if(Object.getPrototypeOf(thisArg) === tget){ - err.stack = err.stack.replace( - 'at Object.toString (', - 'at Function.toString (', - ); - } - - throw err; - } - } - }); - } - useStrictModeExceptions(prop); - return Reflect.get(...arguments); - } - }; - - const toStringProxy = new Proxy( - Function.prototype.toString, - stripProxyFromErrors(handler), - ); - redefineProperty(Function.prototype, 'toString', { - value: toStringProxy, - }); - } - - function makeNativeString(name = '') { - return cache.nativeToStringStr.replace('toString', name || ''); - } - - function redefineProperty(masterObject, propertyName, descriptorOverrides = {}) { - return Object.defineProperty(masterObject, propertyName, { - // Copy over the existing descriptors (writable, enumerable, configurable, etc) - ...(Object.getOwnPropertyDescriptor(masterObject, propertyName) || {}), - // Add our overrides (e.g. value, get()) - ...descriptorOverrides, - }); - } - - /** - * For all the traps in the passed proxy handler, we wrap them in a try/catch and modify the error stack if they throw. - * @param {*} handler A proxy handler object - * @returns A new proxy handler object with error stack modifications - */ - function stripProxyFromErrors(handler) { - const newHandler = {}; - // We wrap each trap in the handler in a try/catch and modify the error stack if they throw - const traps = Object.getOwnPropertyNames(handler); - traps.forEach((trap) => { - newHandler[trap] = function () { - try { - // Forward the call to the defined proxy handler - return handler[trap].apply(this, arguments || []); //eslint-disable-line - } catch (err) { - // Stack traces differ per browser, we only support chromium based ones currently - if (!err || !err.stack || !err.stack.includes(`at `)) { - throw err; - } - - // When something throws within one of our traps the Proxy will show up in error stacks - // An earlier implementation of this code would simply strip lines with a blacklist, - // but it makes sense to be more surgical here and only remove lines related to our Proxy. - // We try to use a known "anchor" line for that and strip it with everything above it. - // If the anchor line cannot be found for some reason we fall back to our blacklist approach. - - const stripWithBlacklist = (stack, stripFirstLine = true) => { - const blacklist = [ - `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply - `at Object.${trap} `, // e.g. Object.get or Object.apply - `at Object.newHandler. [as ${trap}] `, // caused by this very wrapper :-) - `at newHandler. [as ${trap}] `, // also caused by this wrapper :p - ]; - return ( - err.stack - .split('\n') - // Always remove the first (file) line in the stack (guaranteed to be our proxy) - .filter((line, index) => !(index === 1 && stripFirstLine)) - // Check if the line starts with one of our blacklisted strings - .filter((line) => !blacklist.some((bl) => line.trim().startsWith(bl))) - .join('\n') - ); - }; - - const stripWithAnchor = (stack, anchor) => { - const stackArr = stack.split('\n'); - anchor = anchor || `at Object.newHandler. [as ${trap}] `; // Known first Proxy line in chromium - const anchorIndex = stackArr.findIndex((line) => line.trim().startsWith(anchor)); - if (anchorIndex === -1) { - return false; // 404, anchor not found - } - // Strip everything from the top until we reach the anchor line - // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`) - stackArr.splice(1, anchorIndex); - return stackArr.join('\n'); - }; - - - const oldStackLines = err.stack.split('\n'); - Error.captureStackTrace(err); - const newStackLines = err.stack.split('\n'); - - err.stack = [newStackLines[0],oldStackLines[1],...newStackLines.slice(1)].join('\n'); - - if ((err.stack || '').includes('toString (')) { - err.stack = stripWithBlacklist(err.stack, false); - throw err; - } - - // Try using the anchor method, fallback to blacklist if necessary - err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack); - - throw err; // Re-throw our now sanitized error - } - }; - }); - return newHandler; - } - - // eslint-disable-next-line no-unused-vars - function overrideWebGl(webGl) { - // try to override WebGl - try { - const getParameterProxyHandler = { - apply: function (target, ctx, args) { - const param = (args || [])[0]; - const result = cache.Reflect.apply(target, ctx, args); - // UNMASKED_VENDOR_WEBGL - if (param === 37445) { - return webGl.vendor; - } - // UNMASKED_RENDERER_WEBGL - if (param === 37446) { - return webGl.renderer; - } - return result; - }, - get: function (target, prop, receiver) { - useStrictModeExceptions(prop); - return Reflect.get(...arguments); - }, - } - const addProxy = (obj, propName) => { - overridePropertyWithProxy(obj, propName, getParameterProxyHandler); - } - // For whatever weird reason loops don't play nice with Object.defineProperty, here's the next best thing: - addProxy(WebGLRenderingContext.prototype, 'getParameter'); - addProxy(WebGL2RenderingContext.prototype, 'getParameter'); - } catch (err) { - console.warn(err); - } - } - - // eslint-disable-next-line no-unused-vars - const overrideCodecs = (audioCodecs, videoCodecs) => { - try { - const codecs = { - ...Object.fromEntries(Object.entries(audioCodecs).map(([key, value]) => [`audio/${key}`, value])), - ...Object.fromEntries(Object.entries(videoCodecs).map(([key, value]) => [`video/${key}`, value])), - }; - - const findCodec = (codecString) => { - const [mime, codecSpec] = codecString.split(';'); - if (mime === 'video/mp4') { - if (codecSpec && codecSpec.includes('avc1.42E01E')) { // codec is missing from Chromium - return {name: mime, state: 'probably'}; - } - } - - const codec = Object.entries(codecs).find(([key]) => key === codecString.split(';')[0]); - if(codec) { - return {name: codec[0], state: codec[1]}; - } - - return undefined; - }; - - const canPlayType = { - // eslint-disable-next-line - apply: function (target, ctx, args) { - if (!args || !args.length) { - return target.apply(ctx, args); - } - const [codecString] = args; - const codec = findCodec(codecString); - - if (codec) { - return codec.state; - } - - // If the codec is not in our collected data use - return target.apply(ctx, args); - }, - }; - - overridePropertyWithProxy( - HTMLMediaElement.prototype, - 'canPlayType', - canPlayType, - ); - } catch (e) { - console.warn(e); - } - }; - - // eslint-disable-next-line no-unused-vars - function overrideBattery(batteryInfo) { - try { - const getBattery = { - ...prototypeProxyHandler, - // eslint-disable-next-line - apply: async function () { - return batteryInfo; - }, - }; - - if(navigator.getBattery) { // Firefox does not have this method - to be fixed - overridePropertyWithProxy( - Object.getPrototypeOf(navigator), - 'getBattery', - getBattery, - ); - } - } catch (e) { - console.warn(e); - } - } - - function overrideIntlAPI(language){ - try { - const innerHandler = { - construct(target, [locales, options]) { - return new target(locales ?? language, options); - }, - apply(target, _, [locales, options]) { - return target(locales ?? language, options); - } - }; - - overridePropertyWithProxy(window, 'Intl', { - get(target, key){ - if(typeof key !== 'string' || key[0].toLowerCase() === key[0]) return target[key]; - return new Proxy( - target[key], - innerHandler - ); - } - }); - } catch (e) { - console.warn(e); - } - } - - function makeHandler() { - return { - // Used by simple `navigator` getter evasions - getterValue: (value) => ({ - apply(target, ctx, args) { - // Let's fetch the value first, to trigger and escalate potential errors - // Illegal invocations like `navigator.__proto__.vendor` will throw here - const ret = cache.Reflect.apply(...arguments); // eslint-disable-line - if (args && args.length === 0) { - return value; - } - return ret; - }, - get: function (target, prop, receiver) { - useStrictModeExceptions(prop); - return Reflect.get(...arguments); - }, - }), - }; - } - - function overrideScreenByReassigning(target, newProperties) { - for (const [prop, value] of Object.entries(newProperties)) { - if (value > 0) { - // The 0 values are introduced by collecting in the hidden iframe. - // They are document sizes anyway so no need to test them or inject them. - target[prop] = value; - } - } - } - - // eslint-disable-next-line no-unused-vars - function overrideWindowDimensionsProps(props) { - try { - overrideScreenByReassigning(window, props); - } catch (e) { - console.warn(e); - } - } - - // eslint-disable-next-line no-unused-vars - function overrideDocumentDimensionsProps(props) { - try { - // FIX THIS = non-zero values here block the injecting process? - // overrideScreenByReassigning(window.document.body, props); - } catch (e) { - console.warn(e); - } - } - - function replace(target, key, value) { - if (target?.[key]) { - target[key] = value; - } - } - - // Replaces all the WebRTC related methods with a recursive ES6 Proxy - // This way, we don't have to model a mock WebRTC API and we still don't get any exceptions. - function blockWebRTC() { - const handler = { - get: () => { - return new Proxy(() => {}, handler); - }, - apply: () => { - return new Proxy(() => {}, handler); - }, - construct: () => { - return new Proxy(() => {}, handler); - }, - }; - - const ConstrProxy = new Proxy(Object, handler); - const proxy = new Proxy(() => {}, handler); - - replace(navigator.mediaDevices, 'getUserMedia', proxy); - replace(navigator, 'webkitGetUserMedia', proxy); - replace(navigator, 'mozGetUserMedia', proxy); - replace(navigator, 'getUserMedia`', proxy); - replace(window, 'webkitRTCPeerConnection', proxy); - - replace(window, 'RTCPeerConnection', ConstrProxy); - replace(window, 'MediaStreamTrack', ConstrProxy); - } - - // eslint-disable-next-line no-unused-vars - function overrideUserAgentData(userAgentData) { - try { - const { brands, mobile, platform, ...highEntropyValues } = userAgentData; - // Override basic properties - const getHighEntropyValues = { - // eslint-disable-next-line - apply: async function (target, ctx, args) { - // Just to throw original validation error - // Remove traces of our Proxy - const stripErrorStack = (stack) => stack - .split('\n') - .filter((line) => !line.includes('at Object.apply')) - .filter((line) => !line.includes('at Object.get')) - .join('\n'); - - try { - if (!args || !args.length) { - return target.apply(ctx, args); - } - const [hints] = args; - await target.apply(ctx, args); - - const data = { brands, mobile, platform }; - hints.forEach((hint) => { - data[hint] = highEntropyValues[hint]; - }); - return data; - } catch (err) { - err.stack = stripErrorStack(err.stack); - throw err; - } - }, - }; - - if(window.navigator.userAgentData){ // Firefox does not contain this property - to be fixed - overridePropertyWithProxy( - Object.getPrototypeOf(window.navigator.userAgentData), - 'getHighEntropyValues', - getHighEntropyValues, - ); - - overrideInstancePrototype(window.navigator.userAgentData, { brands, mobile, platform }); - } - } catch (e) { - console.warn(e); - } - }; - - function fixWindowChrome(){ - if(isChrome && !window.chrome){ - Object.defineProperty(window, 'chrome', { - writable: true, - enumerable: true, - configurable: false, - value: {} // incomplete, todo! - }) - } - } - - // heavily inspired by https://github.com/berstend/puppeteer-extra/, check it out! - function fixPermissions(){ - const isSecure = document.location.protocol.startsWith('https') - - if (isSecure) { - overrideGetterWithProxy(Notification, 'permission', { - apply() { - return 'default' - } - }); - } - - if (!isSecure) { - const handler = { - apply(target, ctx, args) { - const param = (args || [])[0] - - const isNotifications = - param && param.name && param.name === 'notifications' - if (!isNotifications) { - return utils.cache.Reflect.apply(...arguments) - } - - return Promise.resolve( - Object.setPrototypeOf( - { - state: 'denied', - onchange: null - }, - PermissionStatus.prototype - ) - ) - } - }; - - overridePropertyWithProxy(Permissions.prototype, 'query', handler) - } - } - - function fixIframeContentWindow(){ - try { - // Adds a contentWindow proxy to the provided iframe element - const addContentWindowProxy = iframe => { - const contentWindowProxy = { - get(target, key) { - if (key === 'self') { - return this - } - if (key === 'frameElement') { - return iframe - } - - if (key === '0') { - return undefined - } - return Reflect.get(target, key) - } - } - - if (!iframe.contentWindow) { - const proxy = new Proxy(window, contentWindowProxy) - Object.defineProperty(iframe, 'contentWindow', { - get() { - return proxy - }, - set(newValue) { - return newValue // contentWindow is immutable - }, - enumerable: true, - configurable: false - }) - } - } - - // Handles iframe element creation, augments `srcdoc` property so we can intercept further - const handleIframeCreation = (target, thisArg, args) => { - const iframe = target.apply(thisArg, args) - - // We need to keep the originals around - const _iframe = iframe - const _srcdoc = _iframe.srcdoc - - // Add hook for the srcdoc property - // We need to be very surgical here to not break other iframes by accident - Object.defineProperty(iframe, 'srcdoc', { - configurable: true, // Important, so we can reset this later - get: function() { - return _srcdoc - }, - set: function(newValue) { - addContentWindowProxy(this) - // Reset property, the hook is only needed once - Object.defineProperty(iframe, 'srcdoc', { - configurable: false, - writable: false, - value: _srcdoc - }) - _iframe.srcdoc = newValue - } - }) - return iframe - } - - // Adds a hook to intercept iframe creation events - const addIframeCreationSniffer = () => { - /* global document */ - const createElementHandler = { - // Make toString() native - get(target, key) { - return Reflect.get(target, key) - }, - apply: function(target, thisArg, args) { - if (`${args[0]}`.toLowerCase() === 'iframe') { - // Everything as usual - return handleIframeCreation(target, thisArg, args) - } - return target.apply(thisArg, args) - } - } - - // All this just due to iframes with srcdoc bug - overridePropertyWithProxy( - document, - 'createElement', - createElementHandler - ) - } - - // Let's go - addIframeCreationSniffer() - } catch (err) { - // warning message supressed (see https://github.com/apify/fingerprint-suite/issues/61). - // console.warn(err) - } - } - - function fixPluginArray() { - if(window.navigator.plugins.length !== 0){ - return; - } - - Object.defineProperty(navigator, 'plugins', { - get: () => { - const ChromiumPDFPlugin = Object.create(Plugin.prototype, { - description: { value: 'Portable Document Format', enumerable: false }, - filename: { value: 'internal-pdf-viewer', enumerable: false }, - name: { value: 'Chromium PDF Plugin', enumerable: false }, - }); - - return Object.create(PluginArray.prototype, { - length: { value: 1 }, - 0: { value: ChromiumPDFPlugin }, - }); - }, - }); - } - - function runHeadlessFixes(){ - try { - if( isHeadlessChromium ){ - fixWindowChrome(); - fixPermissions(); - fixIframeContentWindow(); - fixPluginArray(); - } - } catch (e) { - console.error(e); - } - } - - function overrideStatic(){ - try { - window.SharedArrayBuffer = undefined; - } catch (e) { - console.error(e); - } - } - - function inject(fp) { - const { - battery, - navigator: { - - extraProperties, - userAgentData, - webdriver, - ...navigatorProps - }, - screen: allScreenProps, - videoCard, - historyLength, - audioCodecs, - videoCodecs, - mockWebRTC, - slim, - // @ts-expect-error internal browser code - } = fp; - - const { - // window screen props - outerHeight, - outerWidth, - devicePixelRatio, - innerWidth, - innerHeight, - screenX, - pageXOffset, - pageYOffset, - - // Document screen props - clientWidth, - clientHeight, - // Ignore hdr for now. - - hasHDR, - // window.screen props - ...newScreen - } = allScreenProps; - - const windowScreenProps = { - innerHeight, - outerHeight, - outerWidth, - innerWidth, - screenX, - pageXOffset, - pageYOffset, - devicePixelRatio, - }; - const documentScreenProps = { - clientHeight, - clientWidth, - }; - - runHeadlessFixes(); - - if (mockWebRTC) blockWebRTC(); - - if (slim) { - // @ts-expect-error internal browser code - // eslint-disable-next-line dot-notation - window['slim'] = true; - } - - overrideIntlAPI(navigatorProps.language); - overrideStatic(); - - if (userAgentData) { - overrideUserAgentData(userAgentData); - } - - if (window.navigator.webdriver) { - navigatorProps.webdriver = false; - } - overrideInstancePrototype(window.navigator, navigatorProps); - - overrideInstancePrototype(window.screen, newScreen); - overrideWindowDimensionsProps(windowScreenProps); - overrideDocumentDimensionsProps(documentScreenProps); - - overrideInstancePrototype(window.history, { length: historyLength }); - - overrideWebGl(videoCard); - overrideCodecs(audioCodecs, videoCodecs); - - overrideBattery(battery); - - } -return inject(fp); -}) -""" - - -def create_init_script_with_fingerprint(fingerprint: str) -> str: - """Create string that contains Javascript function call that will set fingerprints in page.""" - return _js_function + f'({fingerprint})' diff --git a/src/crawlee/playwright_crawler/_playwright_crawler.py b/src/crawlee/playwright_crawler/_playwright_crawler.py index 35072eb15a..c5ef06b354 100644 --- a/src/crawlee/playwright_crawler/_playwright_crawler.py +++ b/src/crawlee/playwright_crawler/_playwright_crawler.py @@ -172,7 +172,7 @@ async def _navigate( async with context.page: if context.request.headers: await context.page.set_extra_http_headers(context.request.headers.model_dump()) - response = await context.page.goto(context.request.url, wait_until='domcontentloaded') + response = await context.page.goto(context.request.url) if response is None: raise SessionError(f'Failed to load the URL: {context.request.url}') From 85ba877f67eeb3a2f1f19ff0cf4713c052fec99a Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 19 Dec 2024 11:02:51 +0100 Subject: [PATCH 08/32] Regenerate poetry lock after merge --- poetry.lock | 222 ++++++++++++++++++++++++++-------------------------- 1 file changed, 111 insertions(+), 111 deletions(-) diff --git a/poetry.lock b/poetry.lock index fcf542ba1d..6f62863e28 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2382,18 +2382,18 @@ files = [ [[package]] name = "pydantic" -version = "2.10.3" +version = "2.10.4" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.10.3-py3-none-any.whl", hash = "sha256:be04d85bbc7b65651c5f8e6b9976ed9c6f41782a55524cef079a34a0bb82144d"}, - {file = "pydantic-2.10.3.tar.gz", hash = "sha256:cb5ac360ce894ceacd69c403187900a02c4b20b693a9dd1d643e1effab9eadf9"}, + {file = "pydantic-2.10.4-py3-none-any.whl", hash = "sha256:597e135ea68be3a37552fb524bc7d0d66dcf93d395acd93a00682f1efcb8ee3d"}, + {file = "pydantic-2.10.4.tar.gz", hash = "sha256:82f12e9723da6de4fe2ba888b5971157b3be7ad914267dea8f05f82b28254f06"}, ] [package.dependencies] annotated-types = ">=0.6.0" -pydantic-core = "2.27.1" +pydantic-core = "2.27.2" typing-extensions = ">=4.12.2" [package.extras] @@ -2402,111 +2402,111 @@ timezone = ["tzdata"] [[package]] name = "pydantic-core" -version = "2.27.1" +version = "2.27.2" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_core-2.27.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:71a5e35c75c021aaf400ac048dacc855f000bdfed91614b4a726f7432f1f3d6a"}, - {file = "pydantic_core-2.27.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f82d068a2d6ecfc6e054726080af69a6764a10015467d7d7b9f66d6ed5afa23b"}, - {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:121ceb0e822f79163dd4699e4c54f5ad38b157084d97b34de8b232bcaad70278"}, - {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4603137322c18eaf2e06a4495f426aa8d8388940f3c457e7548145011bb68e05"}, - {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a33cd6ad9017bbeaa9ed78a2e0752c5e250eafb9534f308e7a5f7849b0b1bfb4"}, - {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15cc53a3179ba0fcefe1e3ae50beb2784dede4003ad2dfd24f81bba4b23a454f"}, - {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45d9c5eb9273aa50999ad6adc6be5e0ecea7e09dbd0d31bd0c65a55a2592ca08"}, - {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8bf7b66ce12a2ac52d16f776b31d16d91033150266eb796967a7e4621707e4f6"}, - {file = "pydantic_core-2.27.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:655d7dd86f26cb15ce8a431036f66ce0318648f8853d709b4167786ec2fa4807"}, - {file = "pydantic_core-2.27.1-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:5556470f1a2157031e676f776c2bc20acd34c1990ca5f7e56f1ebf938b9ab57c"}, - {file = "pydantic_core-2.27.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f69ed81ab24d5a3bd93861c8c4436f54afdf8e8cc421562b0c7504cf3be58206"}, - {file = "pydantic_core-2.27.1-cp310-none-win32.whl", hash = "sha256:f5a823165e6d04ccea61a9f0576f345f8ce40ed533013580e087bd4d7442b52c"}, - {file = "pydantic_core-2.27.1-cp310-none-win_amd64.whl", hash = "sha256:57866a76e0b3823e0b56692d1a0bf722bffb324839bb5b7226a7dbd6c9a40b17"}, - {file = "pydantic_core-2.27.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ac3b20653bdbe160febbea8aa6c079d3df19310d50ac314911ed8cc4eb7f8cb8"}, - {file = "pydantic_core-2.27.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a5a8e19d7c707c4cadb8c18f5f60c843052ae83c20fa7d44f41594c644a1d330"}, - {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f7059ca8d64fea7f238994c97d91f75965216bcbe5f695bb44f354893f11d52"}, - {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bed0f8a0eeea9fb72937ba118f9db0cb7e90773462af7962d382445f3005e5a4"}, - {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a3cb37038123447cf0f3ea4c74751f6a9d7afef0eb71aa07bf5f652b5e6a132c"}, - {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:84286494f6c5d05243456e04223d5a9417d7f443c3b76065e75001beb26f88de"}, - {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acc07b2cfc5b835444b44a9956846b578d27beeacd4b52e45489e93276241025"}, - {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4fefee876e07a6e9aad7a8c8c9f85b0cdbe7df52b8a9552307b09050f7512c7e"}, - {file = "pydantic_core-2.27.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:258c57abf1188926c774a4c94dd29237e77eda19462e5bb901d88adcab6af919"}, - {file = "pydantic_core-2.27.1-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:35c14ac45fcfdf7167ca76cc80b2001205a8d5d16d80524e13508371fb8cdd9c"}, - {file = "pydantic_core-2.27.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d1b26e1dff225c31897696cab7d4f0a315d4c0d9e8666dbffdb28216f3b17fdc"}, - {file = "pydantic_core-2.27.1-cp311-none-win32.whl", hash = "sha256:2cdf7d86886bc6982354862204ae3b2f7f96f21a3eb0ba5ca0ac42c7b38598b9"}, - {file = "pydantic_core-2.27.1-cp311-none-win_amd64.whl", hash = "sha256:3af385b0cee8df3746c3f406f38bcbfdc9041b5c2d5ce3e5fc6637256e60bbc5"}, - {file = "pydantic_core-2.27.1-cp311-none-win_arm64.whl", hash = "sha256:81f2ec23ddc1b476ff96563f2e8d723830b06dceae348ce02914a37cb4e74b89"}, - {file = "pydantic_core-2.27.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9cbd94fc661d2bab2bc702cddd2d3370bbdcc4cd0f8f57488a81bcce90c7a54f"}, - {file = "pydantic_core-2.27.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5f8c4718cd44ec1580e180cb739713ecda2bdee1341084c1467802a417fe0f02"}, - {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15aae984e46de8d376df515f00450d1522077254ef6b7ce189b38ecee7c9677c"}, - {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ba5e3963344ff25fc8c40da90f44b0afca8cfd89d12964feb79ac1411a260ac"}, - {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:992cea5f4f3b29d6b4f7f1726ed8ee46c8331c6b4eed6db5b40134c6fe1768bb"}, - {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0325336f348dbee6550d129b1627cb8f5351a9dc91aad141ffb96d4937bd9529"}, - {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7597c07fbd11515f654d6ece3d0e4e5093edc30a436c63142d9a4b8e22f19c35"}, - {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3bbd5d8cc692616d5ef6fbbbd50dbec142c7e6ad9beb66b78a96e9c16729b089"}, - {file = "pydantic_core-2.27.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:dc61505e73298a84a2f317255fcc72b710b72980f3a1f670447a21efc88f8381"}, - {file = "pydantic_core-2.27.1-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:e1f735dc43da318cad19b4173dd1ffce1d84aafd6c9b782b3abc04a0d5a6f5bb"}, - {file = "pydantic_core-2.27.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f4e5658dbffe8843a0f12366a4c2d1c316dbe09bb4dfbdc9d2d9cd6031de8aae"}, - {file = "pydantic_core-2.27.1-cp312-none-win32.whl", hash = "sha256:672ebbe820bb37988c4d136eca2652ee114992d5d41c7e4858cdd90ea94ffe5c"}, - {file = "pydantic_core-2.27.1-cp312-none-win_amd64.whl", hash = "sha256:66ff044fd0bb1768688aecbe28b6190f6e799349221fb0de0e6f4048eca14c16"}, - {file = "pydantic_core-2.27.1-cp312-none-win_arm64.whl", hash = "sha256:9a3b0793b1bbfd4146304e23d90045f2a9b5fd5823aa682665fbdaf2a6c28f3e"}, - {file = "pydantic_core-2.27.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f216dbce0e60e4d03e0c4353c7023b202d95cbaeff12e5fd2e82ea0a66905073"}, - {file = "pydantic_core-2.27.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a2e02889071850bbfd36b56fd6bc98945e23670773bc7a76657e90e6b6603c08"}, - {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42b0e23f119b2b456d07ca91b307ae167cc3f6c846a7b169fca5326e32fdc6cf"}, - {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:764be71193f87d460a03f1f7385a82e226639732214b402f9aa61f0d025f0737"}, - {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c00666a3bd2f84920a4e94434f5974d7bbc57e461318d6bb34ce9cdbbc1f6b2"}, - {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ccaa88b24eebc0f849ce0a4d09e8a408ec5a94afff395eb69baf868f5183107"}, - {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c65af9088ac534313e1963443d0ec360bb2b9cba6c2909478d22c2e363d98a51"}, - {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:206b5cf6f0c513baffaeae7bd817717140770c74528f3e4c3e1cec7871ddd61a"}, - {file = "pydantic_core-2.27.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:062f60e512fc7fff8b8a9d680ff0ddaaef0193dba9fa83e679c0c5f5fbd018bc"}, - {file = "pydantic_core-2.27.1-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:a0697803ed7d4af5e4c1adf1670af078f8fcab7a86350e969f454daf598c4960"}, - {file = "pydantic_core-2.27.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:58ca98a950171f3151c603aeea9303ef6c235f692fe555e883591103da709b23"}, - {file = "pydantic_core-2.27.1-cp313-none-win32.whl", hash = "sha256:8065914ff79f7eab1599bd80406681f0ad08f8e47c880f17b416c9f8f7a26d05"}, - {file = "pydantic_core-2.27.1-cp313-none-win_amd64.whl", hash = "sha256:ba630d5e3db74c79300d9a5bdaaf6200172b107f263c98a0539eeecb857b2337"}, - {file = "pydantic_core-2.27.1-cp313-none-win_arm64.whl", hash = "sha256:45cf8588c066860b623cd11c4ba687f8d7175d5f7ef65f7129df8a394c502de5"}, - {file = "pydantic_core-2.27.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:5897bec80a09b4084aee23f9b73a9477a46c3304ad1d2d07acca19723fb1de62"}, - {file = "pydantic_core-2.27.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d0165ab2914379bd56908c02294ed8405c252250668ebcb438a55494c69f44ab"}, - {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b9af86e1d8e4cfc82c2022bfaa6f459381a50b94a29e95dcdda8442d6d83864"}, - {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f6c8a66741c5f5447e047ab0ba7a1c61d1e95580d64bce852e3df1f895c4067"}, - {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a42d6a8156ff78981f8aa56eb6394114e0dedb217cf8b729f438f643608cbcd"}, - {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64c65f40b4cd8b0e049a8edde07e38b476da7e3aaebe63287c899d2cff253fa5"}, - {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdcf339322a3fae5cbd504edcefddd5a50d9ee00d968696846f089b4432cf78"}, - {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bf99c8404f008750c846cb4ac4667b798a9f7de673ff719d705d9b2d6de49c5f"}, - {file = "pydantic_core-2.27.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8f1edcea27918d748c7e5e4d917297b2a0ab80cad10f86631e488b7cddf76a36"}, - {file = "pydantic_core-2.27.1-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:159cac0a3d096f79ab6a44d77a961917219707e2a130739c64d4dd46281f5c2a"}, - {file = "pydantic_core-2.27.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:029d9757eb621cc6e1848fa0b0310310de7301057f623985698ed7ebb014391b"}, - {file = "pydantic_core-2.27.1-cp38-none-win32.whl", hash = "sha256:a28af0695a45f7060e6f9b7092558a928a28553366519f64083c63a44f70e618"}, - {file = "pydantic_core-2.27.1-cp38-none-win_amd64.whl", hash = "sha256:2d4567c850905d5eaaed2f7a404e61012a51caf288292e016360aa2b96ff38d4"}, - {file = "pydantic_core-2.27.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:e9386266798d64eeb19dd3677051f5705bf873e98e15897ddb7d76f477131967"}, - {file = "pydantic_core-2.27.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4228b5b646caa73f119b1ae756216b59cc6e2267201c27d3912b592c5e323b60"}, - {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b3dfe500de26c52abe0477dde16192ac39c98f05bf2d80e76102d394bd13854"}, - {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:aee66be87825cdf72ac64cb03ad4c15ffef4143dbf5c113f64a5ff4f81477bf9"}, - {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b748c44bb9f53031c8cbc99a8a061bc181c1000c60a30f55393b6e9c45cc5bd"}, - {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ca038c7f6a0afd0b2448941b6ef9d5e1949e999f9e5517692eb6da58e9d44be"}, - {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e0bd57539da59a3e4671b90a502da9a28c72322a4f17866ba3ac63a82c4498e"}, - {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ac6c2c45c847bbf8f91930d88716a0fb924b51e0c6dad329b793d670ec5db792"}, - {file = "pydantic_core-2.27.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b94d4ba43739bbe8b0ce4262bcc3b7b9f31459ad120fb595627eaeb7f9b9ca01"}, - {file = "pydantic_core-2.27.1-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:00e6424f4b26fe82d44577b4c842d7df97c20be6439e8e685d0d715feceb9fb9"}, - {file = "pydantic_core-2.27.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:38de0a70160dd97540335b7ad3a74571b24f1dc3ed33f815f0880682e6880131"}, - {file = "pydantic_core-2.27.1-cp39-none-win32.whl", hash = "sha256:7ccebf51efc61634f6c2344da73e366c75e735960b5654b63d7e6f69a5885fa3"}, - {file = "pydantic_core-2.27.1-cp39-none-win_amd64.whl", hash = "sha256:a57847b090d7892f123726202b7daa20df6694cbd583b67a592e856bff603d6c"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3fa80ac2bd5856580e242dbc202db873c60a01b20309c8319b5c5986fbe53ce6"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d950caa237bb1954f1b8c9227b5065ba6875ac9771bb8ec790d956a699b78676"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e4216e64d203e39c62df627aa882f02a2438d18a5f21d7f721621f7a5d3611d"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02a3d637bd387c41d46b002f0e49c52642281edacd2740e5a42f7017feea3f2c"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:161c27ccce13b6b0c8689418da3885d3220ed2eae2ea5e9b2f7f3d48f1d52c27"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:19910754e4cc9c63bc1c7f6d73aa1cfee82f42007e407c0f413695c2f7ed777f"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:e173486019cc283dc9778315fa29a363579372fe67045e971e89b6365cc035ed"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:af52d26579b308921b73b956153066481f064875140ccd1dfd4e77db89dbb12f"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:981fb88516bd1ae8b0cbbd2034678a39dedc98752f264ac9bc5839d3923fa04c"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5fde892e6c697ce3e30c61b239330fc5d569a71fefd4eb6512fc6caec9dd9e2f"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:816f5aa087094099fff7edabb5e01cc370eb21aa1a1d44fe2d2aefdfb5599b31"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c10c309e18e443ddb108f0ef64e8729363adbfd92d6d57beec680f6261556f3"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98476c98b02c8e9b2eec76ac4156fd006628b1b2d0ef27e548ffa978393fd154"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c3027001c28434e7ca5a6e1e527487051136aa81803ac812be51802150d880dd"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:7699b1df36a48169cdebda7ab5a2bac265204003f153b4bd17276153d997670a"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:1c39b07d90be6b48968ddc8c19e7585052088fd7ec8d568bb31ff64c70ae3c97"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:46ccfe3032b3915586e469d4972973f893c0a2bb65669194a5bdea9bacc088c2"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:62ba45e21cf6571d7f716d903b5b7b6d2617e2d5d67c0923dc47b9d41369f840"}, - {file = "pydantic_core-2.27.1.tar.gz", hash = "sha256:62a763352879b84aa31058fc931884055fd75089cccbd9d58bb6afd01141b235"}, + {file = "pydantic_core-2.27.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2d367ca20b2f14095a8f4fa1210f5a7b78b8a20009ecced6b12818f455b1e9fa"}, + {file = "pydantic_core-2.27.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:491a2b73db93fab69731eaee494f320faa4e093dbed776be1a829c2eb222c34c"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7969e133a6f183be60e9f6f56bfae753585680f3b7307a8e555a948d443cc05a"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3de9961f2a346257caf0aa508a4da705467f53778e9ef6fe744c038119737ef5"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e2bb4d3e5873c37bb3dd58714d4cd0b0e6238cebc4177ac8fe878f8b3aa8e74c"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:280d219beebb0752699480fe8f1dc61ab6615c2046d76b7ab7ee38858de0a4e7"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47956ae78b6422cbd46f772f1746799cbb862de838fd8d1fbd34a82e05b0983a"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:14d4a5c49d2f009d62a2a7140d3064f686d17a5d1a268bc641954ba181880236"}, + {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:337b443af21d488716f8d0b6164de833e788aa6bd7e3a39c005febc1284f4962"}, + {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:03d0f86ea3184a12f41a2d23f7ccb79cdb5a18e06993f8a45baa8dfec746f0e9"}, + {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7041c36f5680c6e0f08d922aed302e98b3745d97fe1589db0a3eebf6624523af"}, + {file = "pydantic_core-2.27.2-cp310-cp310-win32.whl", hash = "sha256:50a68f3e3819077be2c98110c1f9dcb3817e93f267ba80a2c05bb4f8799e2ff4"}, + {file = "pydantic_core-2.27.2-cp310-cp310-win_amd64.whl", hash = "sha256:e0fd26b16394ead34a424eecf8a31a1f5137094cabe84a1bcb10fa6ba39d3d31"}, + {file = "pydantic_core-2.27.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:8e10c99ef58cfdf2a66fc15d66b16c4a04f62bca39db589ae8cba08bc55331bc"}, + {file = "pydantic_core-2.27.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:26f32e0adf166a84d0cb63be85c562ca8a6fa8de28e5f0d92250c6b7e9e2aff7"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c19d1ea0673cd13cc2f872f6c9ab42acc4e4f492a7ca9d3795ce2b112dd7e15"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e68c4446fe0810e959cdff46ab0a41ce2f2c86d227d96dc3847af0ba7def306"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9640b0059ff4f14d1f37321b94061c6db164fbe49b334b31643e0528d100d99"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40d02e7d45c9f8af700f3452f329ead92da4c5f4317ca9b896de7ce7199ea459"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c1fd185014191700554795c99b347d64f2bb637966c4cfc16998a0ca700d048"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d81d2068e1c1228a565af076598f9e7451712700b673de8f502f0334f281387d"}, + {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1a4207639fb02ec2dbb76227d7c751a20b1a6b4bc52850568e52260cae64ca3b"}, + {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:3de3ce3c9ddc8bbd88f6e0e304dea0e66d843ec9de1b0042b0911c1663ffd474"}, + {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:30c5f68ded0c36466acede341551106821043e9afaad516adfb6e8fa80a4e6a6"}, + {file = "pydantic_core-2.27.2-cp311-cp311-win32.whl", hash = "sha256:c70c26d2c99f78b125a3459f8afe1aed4d9687c24fd677c6a4436bc042e50d6c"}, + {file = "pydantic_core-2.27.2-cp311-cp311-win_amd64.whl", hash = "sha256:08e125dbdc505fa69ca7d9c499639ab6407cfa909214d500897d02afb816e7cc"}, + {file = "pydantic_core-2.27.2-cp311-cp311-win_arm64.whl", hash = "sha256:26f0d68d4b235a2bae0c3fc585c585b4ecc51382db0e3ba402a22cbc440915e4"}, + {file = "pydantic_core-2.27.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9e0c8cfefa0ef83b4da9588448b6d8d2a2bf1a53c3f1ae5fca39eb3061e2f0b0"}, + {file = "pydantic_core-2.27.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:83097677b8e3bd7eaa6775720ec8e0405f1575015a463285a92bfdfe254529ef"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:172fce187655fece0c90d90a678424b013f8fbb0ca8b036ac266749c09438cb7"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:519f29f5213271eeeeb3093f662ba2fd512b91c5f188f3bb7b27bc5973816934"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05e3a55d124407fffba0dd6b0c0cd056d10e983ceb4e5dbd10dda135c31071d6"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c3ed807c7b91de05e63930188f19e921d1fe90de6b4f5cd43ee7fcc3525cb8c"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fb4aadc0b9a0c063206846d603b92030eb6f03069151a625667f982887153e2"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28ccb213807e037460326424ceb8b5245acb88f32f3d2777427476e1b32c48c4"}, + {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:de3cd1899e2c279b140adde9357c4495ed9d47131b4a4eaff9052f23398076b3"}, + {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:220f892729375e2d736b97d0e51466252ad84c51857d4d15f5e9692f9ef12be4"}, + {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a0fcd29cd6b4e74fe8ddd2c90330fd8edf2e30cb52acda47f06dd615ae72da57"}, + {file = "pydantic_core-2.27.2-cp312-cp312-win32.whl", hash = "sha256:1e2cb691ed9834cd6a8be61228471d0a503731abfb42f82458ff27be7b2186fc"}, + {file = "pydantic_core-2.27.2-cp312-cp312-win_amd64.whl", hash = "sha256:cc3f1a99a4f4f9dd1de4fe0312c114e740b5ddead65bb4102884b384c15d8bc9"}, + {file = "pydantic_core-2.27.2-cp312-cp312-win_arm64.whl", hash = "sha256:3911ac9284cd8a1792d3cb26a2da18f3ca26c6908cc434a18f730dc0db7bfa3b"}, + {file = "pydantic_core-2.27.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7d14bd329640e63852364c306f4d23eb744e0f8193148d4044dd3dacdaacbd8b"}, + {file = "pydantic_core-2.27.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:82f91663004eb8ed30ff478d77c4d1179b3563df6cdb15c0817cd1cdaf34d154"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71b24c7d61131bb83df10cc7e687433609963a944ccf45190cfc21e0887b08c9"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fa8e459d4954f608fa26116118bb67f56b93b209c39b008277ace29937453dc9"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce8918cbebc8da707ba805b7fd0b382816858728ae7fe19a942080c24e5b7cd1"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eda3f5c2a021bbc5d976107bb302e0131351c2ba54343f8a496dc8783d3d3a6a"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd8086fa684c4775c27f03f062cbb9eaa6e17f064307e86b21b9e0abc9c0f02e"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8d9b3388db186ba0c099a6d20f0604a44eabdeef1777ddd94786cdae158729e4"}, + {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7a66efda2387de898c8f38c0cf7f14fca0b51a8ef0b24bfea5849f1b3c95af27"}, + {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:18a101c168e4e092ab40dbc2503bdc0f62010e95d292b27827871dc85450d7ee"}, + {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ba5dd002f88b78a4215ed2f8ddbdf85e8513382820ba15ad5ad8955ce0ca19a1"}, + {file = "pydantic_core-2.27.2-cp313-cp313-win32.whl", hash = "sha256:1ebaf1d0481914d004a573394f4be3a7616334be70261007e47c2a6fe7e50130"}, + {file = "pydantic_core-2.27.2-cp313-cp313-win_amd64.whl", hash = "sha256:953101387ecf2f5652883208769a79e48db18c6df442568a0b5ccd8c2723abee"}, + {file = "pydantic_core-2.27.2-cp313-cp313-win_arm64.whl", hash = "sha256:ac4dbfd1691affb8f48c2c13241a2e3b60ff23247cbcf981759c768b6633cf8b"}, + {file = "pydantic_core-2.27.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d3e8d504bdd3f10835468f29008d72fc8359d95c9c415ce6e767203db6127506"}, + {file = "pydantic_core-2.27.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:521eb9b7f036c9b6187f0b47318ab0d7ca14bd87f776240b90b21c1f4f149320"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85210c4d99a0114f5a9481b44560d7d1e35e32cc5634c656bc48e590b669b145"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d716e2e30c6f140d7560ef1538953a5cd1a87264c737643d481f2779fc247fe1"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f66d89ba397d92f840f8654756196d93804278457b5fbede59598a1f9f90b228"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:669e193c1c576a58f132e3158f9dfa9662969edb1a250c54d8fa52590045f046"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdbe7629b996647b99c01b37f11170a57ae675375b14b8c13b8518b8320ced5"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d262606bf386a5ba0b0af3b97f37c83d7011439e3dc1a9298f21efb292e42f1a"}, + {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cabb9bcb7e0d97f74df8646f34fc76fbf793b7f6dc2438517d7a9e50eee4f14d"}, + {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:d2d63f1215638d28221f664596b1ccb3944f6e25dd18cd3b86b0a4c408d5ebb9"}, + {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bca101c00bff0adb45a833f8451b9105d9df18accb8743b08107d7ada14bd7da"}, + {file = "pydantic_core-2.27.2-cp38-cp38-win32.whl", hash = "sha256:f6f8e111843bbb0dee4cb6594cdc73e79b3329b526037ec242a3e49012495b3b"}, + {file = "pydantic_core-2.27.2-cp38-cp38-win_amd64.whl", hash = "sha256:fd1aea04935a508f62e0d0ef1f5ae968774a32afc306fb8545e06f5ff5cdf3ad"}, + {file = "pydantic_core-2.27.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:c10eb4f1659290b523af58fa7cffb452a61ad6ae5613404519aee4bfbf1df993"}, + {file = "pydantic_core-2.27.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ef592d4bad47296fb11f96cd7dc898b92e795032b4894dfb4076cfccd43a9308"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c61709a844acc6bf0b7dce7daae75195a10aac96a596ea1b776996414791ede4"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c5f762659e47fdb7b16956c71598292f60a03aa92f8b6351504359dbdba6cf"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4c9775e339e42e79ec99c441d9730fccf07414af63eac2f0e48e08fd38a64d76"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57762139821c31847cfb2df63c12f725788bd9f04bc2fb392790959b8f70f118"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d1e85068e818c73e048fe28cfc769040bb1f475524f4745a5dc621f75ac7630"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:097830ed52fd9e427942ff3b9bc17fab52913b2f50f2880dc4a5611446606a54"}, + {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:044a50963a614ecfae59bb1eaf7ea7efc4bc62f49ed594e18fa1e5d953c40e9f"}, + {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:4e0b4220ba5b40d727c7f879eac379b822eee5d8fff418e9d3381ee45b3b0362"}, + {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e4f4bb20d75e9325cc9696c6802657b58bc1dbbe3022f32cc2b2b632c3fbb96"}, + {file = "pydantic_core-2.27.2-cp39-cp39-win32.whl", hash = "sha256:cca63613e90d001b9f2f9a9ceb276c308bfa2a43fafb75c8031c4f66039e8c6e"}, + {file = "pydantic_core-2.27.2-cp39-cp39-win_amd64.whl", hash = "sha256:77d1bca19b0f7021b3a982e6f903dcd5b2b06076def36a652e3907f596e29f67"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2bf14caea37e91198329b828eae1618c068dfb8ef17bb33287a7ad4b61ac314e"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b0cb791f5b45307caae8810c2023a184c74605ec3bcbb67d13846c28ff731ff8"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:688d3fd9fcb71f41c4c015c023d12a79d1c4c0732ec9eb35d96e3388a120dcf3"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d591580c34f4d731592f0e9fe40f9cc1b430d297eecc70b962e93c5c668f15f"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:82f986faf4e644ffc189a7f1aafc86e46ef70372bb153e7001e8afccc6e54133"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:bec317a27290e2537f922639cafd54990551725fc844249e64c523301d0822fc"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:0296abcb83a797db256b773f45773da397da75a08f5fcaef41f2044adec05f50"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:0d75070718e369e452075a6017fbf187f788e17ed67a3abd47fa934d001863d9"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7e17b560be3c98a8e3aa66ce828bdebb9e9ac6ad5466fba92eb74c4c95cb1151"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c33939a82924da9ed65dab5a65d427205a73181d8098e79b6b426bdf8ad4e656"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:00bad2484fa6bda1e216e7345a798bd37c68fb2d97558edd584942aa41b7d278"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c817e2b40aba42bac6f457498dacabc568c3b7a986fc9ba7c8d9d260b71485fb"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:251136cdad0cb722e93732cb45ca5299fb56e1344a833640bf93b2803f8d1bfd"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d2088237af596f0a524d3afc39ab3b036e8adb054ee57cbb1dcf8e09da5b29cc"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d4041c0b966a84b4ae7a09832eb691a35aec90910cd2dbe7a208de59be77965b"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:8083d4e875ebe0b864ffef72a4304827015cff328a1be6e22cc850753bfb122b"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f141ee28a0ad2123b6611b6ceff018039df17f32ada8b534e6aa039545a3efb2"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7d0c8399fcc1848491f00e0314bd59fb34a9c008761bcb422a057670c3f65e35"}, + {file = "pydantic_core-2.27.2.tar.gz", hash = "sha256:eb026e5a4c1fee05726072337ff51d1efb6f59090b7da90d30ea58625b1ffb39"}, ] [package.dependencies] @@ -2625,20 +2625,20 @@ dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments [[package]] name = "pytest-asyncio" -version = "0.24.0" +version = "0.25.0" description = "Pytest support for asyncio" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "pytest_asyncio-0.24.0-py3-none-any.whl", hash = "sha256:a811296ed596b69bf0b6f3dc40f83bcaf341b155a269052d82efa2b25ac7037b"}, - {file = "pytest_asyncio-0.24.0.tar.gz", hash = "sha256:d081d828e576d85f875399194281e92bf8a68d60d72d1a2faf2feddb6c46b276"}, + {file = "pytest_asyncio-0.25.0-py3-none-any.whl", hash = "sha256:db5432d18eac6b7e28b46dcd9b69921b55c3b1086e85febfe04e70b18d9e81b3"}, + {file = "pytest_asyncio-0.25.0.tar.gz", hash = "sha256:8c0610303c9e0442a5db8604505fc0f545456ba1528824842b37b4a626cbf609"}, ] [package.dependencies] pytest = ">=8.2,<9" [package.extras] -docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1)"] testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] [[package]] @@ -3688,4 +3688,4 @@ playwright = ["browserforge", "playwright"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "7044fb232dd9a4e5cc8bdb75ecbeae3337e164bbef1d5551f9300f242b118e4a" +content-hash = "beb441c3ccaa042804b4ba0e5a4cc226c80b52413f78d422207a8cf2e2941afd" From 6e35c1df76c460de2e28b67ce93db56b0f2b3d20 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 19 Dec 2024 11:11:13 +0100 Subject: [PATCH 09/32] Remove unintentional change to headless test --- tests/unit/playwright_crawler/test_playwright_crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/playwright_crawler/test_playwright_crawler.py b/tests/unit/playwright_crawler/test_playwright_crawler.py index 623ef010ef..d1f7d094d3 100644 --- a/tests/unit/playwright_crawler/test_playwright_crawler.py +++ b/tests/unit/playwright_crawler/test_playwright_crawler.py @@ -114,7 +114,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: async def test_firefox_headless_headers(httpbin: URL) -> None: - crawler = PlaywrightCrawler(headless=False, browser_type='firefox') + crawler = PlaywrightCrawler(headless=True, browser_type='firefox') headers = dict[str, str]() @crawler.router.default_handler From ddfabeaf9f17829c241e9bea69bd9dd89a0a5661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Fri, 3 Jan 2025 15:05:26 +0100 Subject: [PATCH 10/32] chore: revert React version bump --- website/package.json | 12 +- website/yarn.lock | 520 ++++++++++++++++++++++++------------------- 2 files changed, 291 insertions(+), 241 deletions(-) diff --git a/website/package.json b/website/package.json index 096f330c7c..2550baec02 100644 --- a/website/package.json +++ b/website/package.json @@ -20,12 +20,12 @@ "@apify/tsconfig": "^0.1.0", "@docusaurus/module-type-aliases": "3.6.3", "@docusaurus/types": "3.6.3", - "@types/react": "^19.0.0", - "@typescript-eslint/eslint-plugin": "8.19.0", - "@typescript-eslint/parser": "8.19.0", + "@types/react": "^18.0.28", + "@typescript-eslint/eslint-plugin": "8.15.0", + "@typescript-eslint/parser": "8.15.0", "eslint": "8.57.0", "eslint-plugin-react": "7.37.2", - "eslint-plugin-react-hooks": "5.1.0", + "eslint-plugin-react-hooks": "5.0.0", "fs-extra": "^11.1.0", "patch-package": "^8.0.0", "path-browserify": "^1.0.1", @@ -51,8 +51,8 @@ "process": "^0.11.10", "prop-types": "^15.8.1", "raw-loader": "^4.0.2", - "react": "^19.0.0", - "react-dom": "^19.0.0", + "react": "^18.2.0", + "react-dom": "^18.2.0", "react-lite-youtube-embed": "^2.3.52", "stream-browserify": "^3.0.0", "unist-util-visit": "^5.0.0" diff --git a/website/yarn.lock b/website/yarn.lock index 7294ff0082..de3ddd214b 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -358,10 +358,10 @@ __metadata: languageName: node linkType: hard -"@apify/consts@npm:^2.35.0": - version: 2.35.0 - resolution: "@apify/consts@npm:2.35.0" - checksum: 10c0/b4e8a0655bb33e183d2ac4cffb47c758e432d10ea067ff9f8c2bc4ed0efe5e2b5131c98d899f60563b1d0afe2e9d0d50d066ec0124a966a234f4be37d37ec35d +"@apify/consts@npm:^2.36.0": + version: 2.36.0 + resolution: "@apify/consts@npm:2.36.0" + checksum: 10c0/49004de665590ca14e0115b38333c7141f43e45e696379535291f3bfecc448a088785810b4e9d037625ecaffe2ecebbcfb7adb3a3bfd2707533042bbb895d3ee languageName: node linkType: hard @@ -424,13 +424,13 @@ __metadata: languageName: node linkType: hard -"@apify/log@npm:^2.5.11": - version: 2.5.11 - resolution: "@apify/log@npm:2.5.11" +"@apify/log@npm:^2.5.12": + version: 2.5.12 + resolution: "@apify/log@npm:2.5.12" dependencies: - "@apify/consts": "npm:^2.35.0" + "@apify/consts": "npm:^2.36.0" ansi-colors: "npm:^4.1.1" - checksum: 10c0/af27b710bb6ad9248bcfff4657b7a0cfca7773ff2be2c410b7e7e0f88670e5ed6915755fb2c8d6412657b463268bbcaa266d9fc0eae27a2b3a0af11b7c7734b5 + checksum: 10c0/61a328a4c84062abe0de1ced1e36ca170c8dedd58c60d53f02e98690edc018f6b82edd50869bb73c524db6ed895f634367404928d838f98a60574da8bf53d73d languageName: node linkType: hard @@ -442,12 +442,12 @@ __metadata: linkType: hard "@apify/utilities@npm:^2.8.0": - version: 2.11.1 - resolution: "@apify/utilities@npm:2.11.1" + version: 2.12.1 + resolution: "@apify/utilities@npm:2.12.1" dependencies: - "@apify/consts": "npm:^2.35.0" - "@apify/log": "npm:^2.5.11" - checksum: 10c0/319d1b222a64fa704370763adc37b2151e60b997dd8f8b345d0c5f4c397cd5e7893d8e89eeb6ff974875ff9fda760c4b2626a95efabf65b644894e1435a5c5f4 + "@apify/consts": "npm:^2.36.0" + "@apify/log": "npm:^2.5.12" + checksum: 10c0/e76657df9f19ee1ad3c14a3d6a93d5662fc1aedb186e6a288c2d54ccc842161dfe8743a971af89a86d92c8f7379f90df9c89275305d3d06215c7beb17c22173c languageName: node linkType: hard @@ -2851,14 +2851,14 @@ __metadata: linkType: hard "@giscus/react@npm:^3.0.0": - version: 3.0.0 - resolution: "@giscus/react@npm:3.0.0" + version: 3.1.0 + resolution: "@giscus/react@npm:3.1.0" dependencies: - giscus: "npm:^1.5.0" + giscus: "npm:^1.6.0" peerDependencies: - react: ^16 || ^17 || ^18 - react-dom: ^16 || ^17 || ^18 - checksum: 10c0/134de49eb80d124a511fa33b38cbc00da24aa338caa37f32f5b89fe130365e6bc4c5583b0de3916b80ecd01cc2db27f904b3f8d633bb317fe8caad83baa125d0 + react: ^16 || ^17 || ^18 || ^19 + react-dom: ^16 || ^17 || ^18 || ^19 + checksum: 10c0/1347b3a729917a7c134dbf38ff4e15189d37447db3453dfbcb0a76b58f6044a32040d197a0744a093682bcefea14e27b8167dfe30f55d79f3f415054092104c9 languageName: node linkType: hard @@ -3172,55 +3172,73 @@ __metadata: languageName: node linkType: hard -"@shikijs/core@npm:1.24.4": - version: 1.24.4 - resolution: "@shikijs/core@npm:1.24.4" +"@shikijs/core@npm:1.26.1": + version: 1.26.1 + resolution: "@shikijs/core@npm:1.26.1" dependencies: - "@shikijs/engine-javascript": "npm:1.24.4" - "@shikijs/engine-oniguruma": "npm:1.24.4" - "@shikijs/types": "npm:1.24.4" - "@shikijs/vscode-textmate": "npm:^9.3.1" + "@shikijs/engine-javascript": "npm:1.26.1" + "@shikijs/engine-oniguruma": "npm:1.26.1" + "@shikijs/types": "npm:1.26.1" + "@shikijs/vscode-textmate": "npm:^10.0.1" "@types/hast": "npm:^3.0.4" hast-util-to-html: "npm:^9.0.4" - checksum: 10c0/4ebdb3022d6d7c0598f42c90f26dc039a758dbda168d0244a1265be805124c2c1846ee502e141605ae75bd2f154a0b99db803603ccc272a90ea96abaefa7cd2b + checksum: 10c0/c1b9748db7b6209b19eaabb68592e3c5ed6e4d1350ce4ffa6ab3c48d212421a33080b548f9e36457c8cc10386649dfa77a37650004e21c52f4ab93647a85ee5a languageName: node linkType: hard -"@shikijs/engine-javascript@npm:1.24.4": - version: 1.24.4 - resolution: "@shikijs/engine-javascript@npm:1.24.4" +"@shikijs/engine-javascript@npm:1.26.1": + version: 1.26.1 + resolution: "@shikijs/engine-javascript@npm:1.26.1" dependencies: - "@shikijs/types": "npm:1.24.4" - "@shikijs/vscode-textmate": "npm:^9.3.1" - oniguruma-to-es: "npm:0.8.1" - checksum: 10c0/dcd244a552f1d8e589140b908496eeeb2a960dfd761dfb3f5ceee00b9560867657267eebaab236fc601bdbd04783e88cd62c7a7c7cdff65e64b521f303df664f + "@shikijs/types": "npm:1.26.1" + "@shikijs/vscode-textmate": "npm:^10.0.1" + oniguruma-to-es: "npm:0.10.0" + checksum: 10c0/aa21dcc54d19e4db1b0ad39e83b8faca80899ea85792eb5f7b4907398bbf504fbc88fca6e3326ff76373f196cd60c9e409ad31440c183045efd8da0069a3d2ab languageName: node linkType: hard -"@shikijs/engine-oniguruma@npm:1.24.4": - version: 1.24.4 - resolution: "@shikijs/engine-oniguruma@npm:1.24.4" +"@shikijs/engine-oniguruma@npm:1.26.1": + version: 1.26.1 + resolution: "@shikijs/engine-oniguruma@npm:1.26.1" dependencies: - "@shikijs/types": "npm:1.24.4" - "@shikijs/vscode-textmate": "npm:^9.3.1" - checksum: 10c0/613180014ca639af9b281c5351d7d5c642c82e53dea7f4f6fd58abaf96c86d6cb06954d6e84f9b463be9d0a215381ee06d0ed14e057d5b770ee72871072334f0 + "@shikijs/types": "npm:1.26.1" + "@shikijs/vscode-textmate": "npm:^10.0.1" + checksum: 10c0/ea5b222459346ad77a0504d27b1e5b47953062a2954d7cdd0632851b0b163fe9bc62c78b505056c5fb0152b12bbb5d076829ffcb3b2440f545287d1283da6a6a languageName: node linkType: hard -"@shikijs/types@npm:1.24.4": - version: 1.24.4 - resolution: "@shikijs/types@npm:1.24.4" +"@shikijs/langs@npm:1.26.1": + version: 1.26.1 + resolution: "@shikijs/langs@npm:1.26.1" dependencies: - "@shikijs/vscode-textmate": "npm:^9.3.1" + "@shikijs/types": "npm:1.26.1" + checksum: 10c0/3299f06f206cf74421f55fdb18b41e1a0d9a22a25c51ce30300158b311491cc24b72876c38034dcab191ede80fd4cce1aa1947cea16259036644b03cbc81f2f6 + languageName: node + linkType: hard + +"@shikijs/themes@npm:1.26.1": + version: 1.26.1 + resolution: "@shikijs/themes@npm:1.26.1" + dependencies: + "@shikijs/types": "npm:1.26.1" + checksum: 10c0/b57ef7f52aec869b142ccffa4fa36e519c2af167e6163db79e6cd6dcf2e54fdf98380bbc73d33c34601f459ab5a73af32133aaad99908f78f8b4fc97373fa2f5 + languageName: node + linkType: hard + +"@shikijs/types@npm:1.26.1": + version: 1.26.1 + resolution: "@shikijs/types@npm:1.26.1" + dependencies: + "@shikijs/vscode-textmate": "npm:^10.0.1" "@types/hast": "npm:^3.0.4" - checksum: 10c0/43128e287c445ebdeb0666054d09a78a524e728dbb8dee4565eb27f6e93ad75db9a7e94d37437bcad3874df92e10da57961b18224825e6775904545aeb011a97 + checksum: 10c0/7f47a071c3ae844936a00b09ae1c973e5e2c9e501f7027a162bdfafc04c5a25ce8c5d593cdd5d83113254b3cda016e4d9fc9498e7808e96167e94e35f44d4f7b languageName: node linkType: hard -"@shikijs/vscode-textmate@npm:^9.3.1": - version: 9.3.1 - resolution: "@shikijs/vscode-textmate@npm:9.3.1" - checksum: 10c0/8db3aa96696d83d30a56670516b128191340830382f1d1edc3108c2f0a418e7cc405dd9f253bf8b0d00fe4426795669b2c4dac3a035ebfe965eda241c33bfe9d +"@shikijs/vscode-textmate@npm:^10.0.1": + version: 10.0.1 + resolution: "@shikijs/vscode-textmate@npm:10.0.1" + checksum: 10c0/acdbcf1b00d2503620ab50c2a23c7876444850ae0610c8e8b85a29587a333be40c9b98406ff17b9f87cbc64674dac6a2ada680374bde3e51a890e16cf1407490 languageName: node linkType: hard @@ -3544,14 +3562,14 @@ __metadata: linkType: hard "@types/express-serve-static-core@npm:*, @types/express-serve-static-core@npm:^5.0.0": - version: 5.0.2 - resolution: "@types/express-serve-static-core@npm:5.0.2" + version: 5.0.3 + resolution: "@types/express-serve-static-core@npm:5.0.3" dependencies: "@types/node": "npm:*" "@types/qs": "npm:*" "@types/range-parser": "npm:*" "@types/send": "npm:*" - checksum: 10c0/9f6ee50bd81f0aa6cc9df6ad2c2d221a3a63249da944db58ec8bb8681e77a5b3b3fdb1931bda73beb13cfaf9125731f835fe5256afb6a6da35b0eb08ccbdbfdf + checksum: 10c0/47cacb12d393f4272f46e5c95d7b06f9c32c528ba1cca31b7ee883f6f6ab7e8f40b92fac2333dea6c1f8130e098793f775441f048067514794662944e900189c languageName: node linkType: hard @@ -3742,11 +3760,11 @@ __metadata: linkType: hard "@types/node@npm:*, @types/node@npm:>=20": - version: 22.10.2 - resolution: "@types/node@npm:22.10.2" + version: 22.10.5 + resolution: "@types/node@npm:22.10.5" dependencies: undici-types: "npm:~6.20.0" - checksum: 10c0/2c7b71a040f1ef5320938eca8ebc946e6905caa9bbf3d5665d9b3774a8d15ea9fab1582b849a6d28c7fc80756a62c5666bc66b69f42f4d5dafd1ccb193cdb4ac + checksum: 10c0/6a0e7d1fe6a86ef6ee19c3c6af4c15542e61aea2f4cee655b6252efb356795f1f228bc8299921e82924e80ff8eca29b74d9dd0dd5cc1a90983f892f740b480df languageName: node linkType: hard @@ -3824,7 +3842,7 @@ __metadata: languageName: node linkType: hard -"@types/react@npm:*, @types/react@npm:^19.0.0": +"@types/react@npm:*": version: 19.0.2 resolution: "@types/react@npm:19.0.2" dependencies: @@ -3833,7 +3851,7 @@ __metadata: languageName: node linkType: hard -"@types/react@npm:^18.3.11": +"@types/react@npm:^18.0.28, @types/react@npm:^18.3.11": version: 18.3.18 resolution: "@types/react@npm:18.3.18" dependencies: @@ -3944,15 +3962,15 @@ __metadata: languageName: node linkType: hard -"@typescript-eslint/eslint-plugin@npm:8.19.0": - version: 8.19.0 - resolution: "@typescript-eslint/eslint-plugin@npm:8.19.0" +"@typescript-eslint/eslint-plugin@npm:8.15.0": + version: 8.15.0 + resolution: "@typescript-eslint/eslint-plugin@npm:8.15.0" dependencies: "@eslint-community/regexpp": "npm:^4.10.0" - "@typescript-eslint/scope-manager": "npm:8.19.0" - "@typescript-eslint/type-utils": "npm:8.19.0" - "@typescript-eslint/utils": "npm:8.19.0" - "@typescript-eslint/visitor-keys": "npm:8.19.0" + "@typescript-eslint/scope-manager": "npm:8.15.0" + "@typescript-eslint/type-utils": "npm:8.15.0" + "@typescript-eslint/utils": "npm:8.15.0" + "@typescript-eslint/visitor-keys": "npm:8.15.0" graphemer: "npm:^1.4.0" ignore: "npm:^5.3.1" natural-compare: "npm:^1.4.0" @@ -3960,99 +3978,108 @@ __metadata: peerDependencies: "@typescript-eslint/parser": ^8.0.0 || ^8.0.0-alpha.0 eslint: ^8.57.0 || ^9.0.0 - typescript: ">=4.8.4 <5.8.0" - checksum: 10c0/ceaa5063b68684b5608950b5e69f0caf1eadfc356cba82625240d6aae55f769faff599c38d35252dcb77a40d92e6fbf6d6264bc0c577d5c549da25061c3bd796 + peerDependenciesMeta: + typescript: + optional: true + checksum: 10c0/90ef10cc7d37a81abec4f4a3ffdfc3a0da8e99d949e03c75437e96e8ab2e896e34b85ab64718690180a7712581031b8611c5d8e7666d6ed4d60b9ace834d58e3 languageName: node linkType: hard -"@typescript-eslint/parser@npm:8.19.0": - version: 8.19.0 - resolution: "@typescript-eslint/parser@npm:8.19.0" +"@typescript-eslint/parser@npm:8.15.0": + version: 8.15.0 + resolution: "@typescript-eslint/parser@npm:8.15.0" dependencies: - "@typescript-eslint/scope-manager": "npm:8.19.0" - "@typescript-eslint/types": "npm:8.19.0" - "@typescript-eslint/typescript-estree": "npm:8.19.0" - "@typescript-eslint/visitor-keys": "npm:8.19.0" + "@typescript-eslint/scope-manager": "npm:8.15.0" + "@typescript-eslint/types": "npm:8.15.0" + "@typescript-eslint/typescript-estree": "npm:8.15.0" + "@typescript-eslint/visitor-keys": "npm:8.15.0" debug: "npm:^4.3.4" peerDependencies: eslint: ^8.57.0 || ^9.0.0 - typescript: ">=4.8.4 <5.8.0" - checksum: 10c0/064b0997963060490fc3f92c90cebc7c694f47a7657f7882ce9eb314786e0cf3e917bfccfad614d23038439d84e69a978bdc7054515b23201001dd427e524e64 + peerDependenciesMeta: + typescript: + optional: true + checksum: 10c0/19c25aea0dc51faa758701a5319a89950fd30494d9d645db8ced84fb60714c5e7d4b51fc4ee8ccb07ddefec88c51ee307ee7e49addd6330ee8f3e7ee9ba329fc languageName: node linkType: hard -"@typescript-eslint/scope-manager@npm:8.19.0": - version: 8.19.0 - resolution: "@typescript-eslint/scope-manager@npm:8.19.0" +"@typescript-eslint/scope-manager@npm:8.15.0": + version: 8.15.0 + resolution: "@typescript-eslint/scope-manager@npm:8.15.0" dependencies: - "@typescript-eslint/types": "npm:8.19.0" - "@typescript-eslint/visitor-keys": "npm:8.19.0" - checksum: 10c0/5052863d00db7ae939de27e91dc6c92df3c37a877e1ff44015ae9aa754d419b44d97d98b25fbb30a80dc58cf92606dad599e27f32b86d20c13b77ac12b4f2abc + "@typescript-eslint/types": "npm:8.15.0" + "@typescript-eslint/visitor-keys": "npm:8.15.0" + checksum: 10c0/c27dfdcea4100cc2d6fa967f857067cbc93155b55e648f9f10887a1b9372bb76cf864f7c804f3fa48d7868d9461cdef10bcea3dab7637d5337e8aa8042dc08b9 languageName: node linkType: hard -"@typescript-eslint/type-utils@npm:8.19.0": - version: 8.19.0 - resolution: "@typescript-eslint/type-utils@npm:8.19.0" +"@typescript-eslint/type-utils@npm:8.15.0": + version: 8.15.0 + resolution: "@typescript-eslint/type-utils@npm:8.15.0" dependencies: - "@typescript-eslint/typescript-estree": "npm:8.19.0" - "@typescript-eslint/utils": "npm:8.19.0" + "@typescript-eslint/typescript-estree": "npm:8.15.0" + "@typescript-eslint/utils": "npm:8.15.0" debug: "npm:^4.3.4" ts-api-utils: "npm:^1.3.0" peerDependencies: eslint: ^8.57.0 || ^9.0.0 - typescript: ">=4.8.4 <5.8.0" - checksum: 10c0/5a460b4d26fd68ded3567390cbac310500e94e9c69583fda3fb9930877663719e6831699bb6d85de6b940bcb7951a51ab1ef67c5fea8b76a13ea3a3783bbae28 + peerDependenciesMeta: + typescript: + optional: true + checksum: 10c0/20f09c79c83b38a962cf7eff10d47a2c01bcc0bab7bf6d762594221cd89023ef8c7aec26751c47b524f53f5c8d38bba55a282529b3df82d5f5ab4350496316f9 languageName: node linkType: hard -"@typescript-eslint/types@npm:8.19.0": - version: 8.19.0 - resolution: "@typescript-eslint/types@npm:8.19.0" - checksum: 10c0/0062e7dce5f374e293c97f1f50fe450187f6b0eaf4971c818e18ef2f6baf4e9aa4e8605fba8d8fc464a504ee1130527b71ecb39d31687c31825942b9f569d902 +"@typescript-eslint/types@npm:8.15.0": + version: 8.15.0 + resolution: "@typescript-eslint/types@npm:8.15.0" + checksum: 10c0/84abc6fd954aff13822a76ac49efdcb90a55c0025c20eee5d8cebcfb68faff33b79bbc711ea524e0209cecd90c5ee3a5f92babc7083c081d3a383a0710264a41 languageName: node linkType: hard -"@typescript-eslint/typescript-estree@npm:8.19.0": - version: 8.19.0 - resolution: "@typescript-eslint/typescript-estree@npm:8.19.0" +"@typescript-eslint/typescript-estree@npm:8.15.0": + version: 8.15.0 + resolution: "@typescript-eslint/typescript-estree@npm:8.15.0" dependencies: - "@typescript-eslint/types": "npm:8.19.0" - "@typescript-eslint/visitor-keys": "npm:8.19.0" + "@typescript-eslint/types": "npm:8.15.0" + "@typescript-eslint/visitor-keys": "npm:8.15.0" debug: "npm:^4.3.4" fast-glob: "npm:^3.3.2" is-glob: "npm:^4.0.3" minimatch: "npm:^9.0.4" semver: "npm:^7.6.0" ts-api-utils: "npm:^1.3.0" - peerDependencies: - typescript: ">=4.8.4 <5.8.0" - checksum: 10c0/ff47004588e8ff585af740b3e0bda07dc52310dbfeb2317eb4a723935740cf0c1953fc9ba57f14cf192bcfe373c46be833ba29d3303df8b501181bb852c7b822 + peerDependenciesMeta: + typescript: + optional: true + checksum: 10c0/3af5c129532db3575349571bbf64d32aeccc4f4df924ac447f5d8f6af8b387148df51965eb2c9b99991951d3dadef4f2509d7ce69bf34a2885d013c040762412 languageName: node linkType: hard -"@typescript-eslint/utils@npm:8.19.0": - version: 8.19.0 - resolution: "@typescript-eslint/utils@npm:8.19.0" +"@typescript-eslint/utils@npm:8.15.0": + version: 8.15.0 + resolution: "@typescript-eslint/utils@npm:8.15.0" dependencies: "@eslint-community/eslint-utils": "npm:^4.4.0" - "@typescript-eslint/scope-manager": "npm:8.19.0" - "@typescript-eslint/types": "npm:8.19.0" - "@typescript-eslint/typescript-estree": "npm:8.19.0" + "@typescript-eslint/scope-manager": "npm:8.15.0" + "@typescript-eslint/types": "npm:8.15.0" + "@typescript-eslint/typescript-estree": "npm:8.15.0" peerDependencies: eslint: ^8.57.0 || ^9.0.0 - typescript: ">=4.8.4 <5.8.0" - checksum: 10c0/7731f7fb66d54491769ca68fd04728138ceb6b785778ad491f8e9b2147802fa0ff480e520f6ea5fb73c8484d13a2ed3e35d44635f5bf4cfbdb04c313154097a9 + peerDependenciesMeta: + typescript: + optional: true + checksum: 10c0/65743f51845a1f6fd2d21f66ca56182ba33e966716bdca73d30b7a67c294e47889c322de7d7b90ab0818296cd33c628e5eeeb03cec7ef2f76c47de7a453eeda2 languageName: node linkType: hard -"@typescript-eslint/visitor-keys@npm:8.19.0": - version: 8.19.0 - resolution: "@typescript-eslint/visitor-keys@npm:8.19.0" +"@typescript-eslint/visitor-keys@npm:8.15.0": + version: 8.15.0 + resolution: "@typescript-eslint/visitor-keys@npm:8.15.0" dependencies: - "@typescript-eslint/types": "npm:8.19.0" + "@typescript-eslint/types": "npm:8.15.0" eslint-visitor-keys: "npm:^4.2.0" - checksum: 10c0/a293def05018bb2259506e23cd8f14349f4386d0e51231893fbddf96ef73c219d5f9fe17b82e3c104f5c23956dbd9b87af3cff5e84b887af243139a3b4bbbe0d + checksum: 10c0/02a954c3752c4328482a884eb1da06ca8fb72ae78ef28f1d854b18f3779406ed47263af22321cf3f65a637ec7584e5f483e34a263b5c8cec60ec85aebc263574 languageName: node linkType: hard @@ -5710,9 +5737,9 @@ __metadata: "@docusaurus/types": "npm:3.6.3" "@giscus/react": "npm:^3.0.0" "@mdx-js/react": "npm:^3.0.1" - "@types/react": "npm:^19.0.0" - "@typescript-eslint/eslint-plugin": "npm:8.19.0" - "@typescript-eslint/parser": "npm:8.19.0" + "@types/react": "npm:^18.0.28" + "@typescript-eslint/eslint-plugin": "npm:8.15.0" + "@typescript-eslint/parser": "npm:8.15.0" axios: "npm:^1.5.0" buffer: "npm:^6.0.3" clsx: "npm:^2.0.0" @@ -5720,7 +5747,7 @@ __metadata: docusaurus-gtm-plugin: "npm:^0.0.2" eslint: "npm:8.57.0" eslint-plugin-react: "npm:7.37.2" - eslint-plugin-react-hooks: "npm:5.1.0" + eslint-plugin-react-hooks: "npm:5.0.0" fs-extra: "npm:^11.1.0" patch-package: "npm:^8.0.0" path-browserify: "npm:^1.0.1" @@ -5729,8 +5756,8 @@ __metadata: process: "npm:^0.11.10" prop-types: "npm:^15.8.1" raw-loader: "npm:^4.0.2" - react: "npm:^19.0.0" - react-dom: "npm:^19.0.0" + react: "npm:^18.2.0" + react-dom: "npm:^18.2.0" react-lite-youtube-embed: "npm:^2.3.52" rimraf: "npm:^6.0.0" stream-browserify: "npm:^3.0.0" @@ -6655,9 +6682,9 @@ __metadata: languageName: node linkType: hard -"es-abstract@npm:^1.17.5, es-abstract@npm:^1.23.2, es-abstract@npm:^1.23.3, es-abstract@npm:^1.23.5, es-abstract@npm:^1.23.6": - version: 1.23.8 - resolution: "es-abstract@npm:1.23.8" +"es-abstract@npm:^1.17.5, es-abstract@npm:^1.23.2, es-abstract@npm:^1.23.3, es-abstract@npm:^1.23.5, es-abstract@npm:^1.23.6, es-abstract@npm:^1.23.9": + version: 1.23.9 + resolution: "es-abstract@npm:1.23.9" dependencies: array-buffer-byte-length: "npm:^1.0.2" arraybuffer.prototype.slice: "npm:^1.0.4" @@ -6670,10 +6697,11 @@ __metadata: es-define-property: "npm:^1.0.1" es-errors: "npm:^1.3.0" es-object-atoms: "npm:^1.0.0" - es-set-tostringtag: "npm:^2.0.3" + es-set-tostringtag: "npm:^2.1.0" es-to-primitive: "npm:^1.3.0" function.prototype.name: "npm:^1.1.8" - get-intrinsic: "npm:^1.2.6" + get-intrinsic: "npm:^1.2.7" + get-proto: "npm:^1.0.0" get-symbol-description: "npm:^1.1.0" globalthis: "npm:^1.0.4" gopd: "npm:^1.2.0" @@ -6694,11 +6722,12 @@ __metadata: object-inspect: "npm:^1.13.3" object-keys: "npm:^1.1.1" object.assign: "npm:^4.1.7" - own-keys: "npm:^1.0.0" + own-keys: "npm:^1.0.1" regexp.prototype.flags: "npm:^1.5.3" safe-array-concat: "npm:^1.1.3" safe-push-apply: "npm:^1.0.0" safe-regex-test: "npm:^1.1.0" + set-proto: "npm:^1.0.0" string.prototype.trim: "npm:^1.2.10" string.prototype.trimend: "npm:^1.0.9" string.prototype.trimstart: "npm:^1.0.8" @@ -6708,7 +6737,7 @@ __metadata: typed-array-length: "npm:^1.0.7" unbox-primitive: "npm:^1.1.0" which-typed-array: "npm:^1.1.18" - checksum: 10c0/5e3afb94ff8ad70801625e3d262a0384cc75e42574b6c2e89b33d255c03e15e1af72ca9fd459511b717ec25b79812520481c3b4d1f9bea6038bae1421225907b + checksum: 10c0/1de229c9e08fe13c17fe5abaec8221545dfcd57e51f64909599a6ae896df84b8fd2f7d16c60cb00d7bf495b9298ca3581aded19939d4b7276854a4b066f8422b languageName: node linkType: hard @@ -6766,14 +6795,15 @@ __metadata: languageName: node linkType: hard -"es-set-tostringtag@npm:^2.0.3": - version: 2.0.3 - resolution: "es-set-tostringtag@npm:2.0.3" +"es-set-tostringtag@npm:^2.0.3, es-set-tostringtag@npm:^2.1.0": + version: 2.1.0 + resolution: "es-set-tostringtag@npm:2.1.0" dependencies: - get-intrinsic: "npm:^1.2.4" + es-errors: "npm:^1.3.0" + get-intrinsic: "npm:^1.2.6" has-tostringtag: "npm:^1.0.2" - hasown: "npm:^2.0.1" - checksum: 10c0/f22aff1585eb33569c326323f0b0d175844a1f11618b86e193b386f8be0ea9474cfbe46df39c45d959f7aa8f6c06985dc51dd6bce5401645ec5a74c4ceaa836a + hasown: "npm:^2.0.2" + checksum: 10c0/ef2ca9ce49afe3931cb32e35da4dcb6d86ab02592cfc2ce3e49ced199d9d0bb5085fc7e73e06312213765f5efa47cc1df553a6a5154584b21448e9fb8355b1af languageName: node linkType: hard @@ -7013,12 +7043,12 @@ __metadata: languageName: node linkType: hard -"eslint-plugin-react-hooks@npm:5.1.0": - version: 5.1.0 - resolution: "eslint-plugin-react-hooks@npm:5.1.0" +"eslint-plugin-react-hooks@npm:5.0.0": + version: 5.0.0 + resolution: "eslint-plugin-react-hooks@npm:5.0.0" peerDependencies: eslint: ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0 - checksum: 10c0/37ef76e1d916d46ab8e93a596078efcf2162e2c653614437e0c54e31d02a5dadabec22802fab717effe257aeb4bdc20c2a710666a89ab1cf07e01e614dde75d8 + checksum: 10c0/bcb74b421f32e4203a7100405b57aab85526be4461e5a1da01bc537969a30012d2ee209a2c2a6cac543833a27188ce1e6ad71e4628d0bb4a2e5365cad86c5002 languageName: node linkType: hard @@ -7853,21 +7883,21 @@ __metadata: languageName: node linkType: hard -"get-intrinsic@npm:^1.2.4, get-intrinsic@npm:^1.2.5, get-intrinsic@npm:^1.2.6": - version: 1.2.6 - resolution: "get-intrinsic@npm:1.2.6" +"get-intrinsic@npm:^1.2.4, get-intrinsic@npm:^1.2.5, get-intrinsic@npm:^1.2.6, get-intrinsic@npm:^1.2.7": + version: 1.2.7 + resolution: "get-intrinsic@npm:1.2.7" dependencies: call-bind-apply-helpers: "npm:^1.0.1" - dunder-proto: "npm:^1.0.0" es-define-property: "npm:^1.0.1" es-errors: "npm:^1.3.0" es-object-atoms: "npm:^1.0.0" function-bind: "npm:^1.1.2" + get-proto: "npm:^1.0.0" gopd: "npm:^1.2.0" has-symbols: "npm:^1.1.0" hasown: "npm:^2.0.2" - math-intrinsics: "npm:^1.0.0" - checksum: 10c0/0f1ea6d807d97d074e8a31ac698213a12757fcfa9a8f4778263d2e4702c40fe83198aadd3dba2e99aabc2e4cf8a38345545dbb0518297d3df8b00b56a156c32a + math-intrinsics: "npm:^1.1.0" + checksum: 10c0/b475dec9f8bff6f7422f51ff4b7b8d0b68e6776ee83a753c1d627e3008c3442090992788038b37eff72e93e43dceed8c1acbdf2d6751672687ec22127933080d languageName: node linkType: hard @@ -7878,6 +7908,16 @@ __metadata: languageName: node linkType: hard +"get-proto@npm:^1.0.0, get-proto@npm:^1.0.1": + version: 1.0.1 + resolution: "get-proto@npm:1.0.1" + dependencies: + dunder-proto: "npm:^1.0.1" + es-object-atoms: "npm:^1.0.0" + checksum: 10c0/9224acb44603c5526955e83510b9da41baf6ae73f7398875fba50edc5e944223a89c4a72b070fcd78beb5f7bdda58ecb6294adc28f7acfc0da05f76a2399643c + languageName: node + linkType: hard + "get-stream@npm:^6.0.0, get-stream@npm:^6.0.1": version: 6.0.1 resolution: "get-stream@npm:6.0.1" @@ -7905,12 +7945,12 @@ __metadata: languageName: node linkType: hard -"giscus@npm:^1.5.0": - version: 1.5.0 - resolution: "giscus@npm:1.5.0" +"giscus@npm:^1.6.0": + version: 1.6.0 + resolution: "giscus@npm:1.6.0" dependencies: - lit: "npm:^3.1.2" - checksum: 10c0/2e94c0260128c402de16550d1ec1b5797dc2efbebed994371920fed1a7018f4b20377f147205c6e479e3e99d975a499db746ae813ca11e1a7d58009e6f059842 + lit: "npm:^3.2.1" + checksum: 10c0/2dc96e45591b38bbf8db7c9a0efbb25f598e57522cd90ca0cad23d573f88f5adbe4411c0c4cf61231f6dca57531d6fe4fbbcd12be78681cba3aa218eb4584e11 languageName: node linkType: hard @@ -8188,7 +8228,7 @@ __metadata: languageName: node linkType: hard -"has-tostringtag@npm:^1.0.0, has-tostringtag@npm:^1.0.2": +"has-tostringtag@npm:^1.0.2": version: 1.0.2 resolution: "has-tostringtag@npm:1.0.2" dependencies: @@ -8235,7 +8275,7 @@ __metadata: languageName: node linkType: hard -"hasown@npm:^2.0.0, hasown@npm:^2.0.1, hasown@npm:^2.0.2": +"hasown@npm:^2.0.0, hasown@npm:^2.0.2": version: 2.0.2 resolution: "hasown@npm:2.0.2" dependencies: @@ -8291,8 +8331,8 @@ __metadata: linkType: hard "hast-util-to-estree@npm:^3.0.0": - version: 3.1.0 - resolution: "hast-util-to-estree@npm:3.1.0" + version: 3.1.1 + resolution: "hast-util-to-estree@npm:3.1.1" dependencies: "@types/estree": "npm:^1.0.0" "@types/estree-jsx": "npm:^1.0.0" @@ -8307,10 +8347,10 @@ __metadata: mdast-util-mdxjs-esm: "npm:^2.0.0" property-information: "npm:^6.0.0" space-separated-tokens: "npm:^2.0.0" - style-to-object: "npm:^0.4.0" + style-to-object: "npm:^1.0.0" unist-util-position: "npm:^5.0.0" zwitch: "npm:^2.0.0" - checksum: 10c0/9003a8bac26a4580d5fc9f2a271d17330dd653266425e9f5539feecd2f7538868d6630a18f70698b8b804bf14c306418a3f4ab3119bb4692aca78b0c08b1291e + checksum: 10c0/90b4faa892171597daec5b5c691bba0f9bcacd10573ea0254898cf877ab3898e7d95de73ee99cfcec371d9824183c0631dfbbeb1085c4c22f7b90b4904324749 languageName: node linkType: hard @@ -8809,13 +8849,6 @@ __metadata: languageName: node linkType: hard -"inline-style-parser@npm:0.1.1": - version: 0.1.1 - resolution: "inline-style-parser@npm:0.1.1" - checksum: 10c0/08832a533f51a1e17619f2eabf2f5ec5e956d6dcba1896351285c65df022c9420de61d73256e1dca8015a52abf96cc84ddc3b73b898b22de6589d3962b5e501b - languageName: node - linkType: hard - "inline-style-parser@npm:0.2.4": version: 0.2.4 resolution: "inline-style-parser@npm:0.2.4" @@ -8910,11 +8943,14 @@ __metadata: linkType: hard "is-async-function@npm:^2.0.0": - version: 2.0.0 - resolution: "is-async-function@npm:2.0.0" + version: 2.1.0 + resolution: "is-async-function@npm:2.1.0" dependencies: - has-tostringtag: "npm:^1.0.0" - checksum: 10c0/787bc931576aad525d751fc5ce211960fe91e49ac84a5c22d6ae0bc9541945fbc3f686dc590c3175722ce4f6d7b798a93f6f8ff4847fdb2199aea6f4baf5d668 + call-bound: "npm:^1.0.3" + get-proto: "npm:^1.0.1" + has-tostringtag: "npm:^1.0.2" + safe-regex-test: "npm:^1.1.0" + checksum: 10c0/5209b858c6d18d88a9fb56dea202a050d53d4b722448cc439fdca859b36e23edf27ee8c18958ba49330f1a71b8846576273f4581e1c0bb9d403738129d852fdb languageName: node linkType: hard @@ -9050,11 +9086,14 @@ __metadata: linkType: hard "is-generator-function@npm:^1.0.10": - version: 1.0.10 - resolution: "is-generator-function@npm:1.0.10" + version: 1.1.0 + resolution: "is-generator-function@npm:1.1.0" dependencies: - has-tostringtag: "npm:^1.0.0" - checksum: 10c0/df03514df01a6098945b5a0cfa1abff715807c8e72f57c49a0686ad54b3b74d394e2d8714e6f709a71eb00c9630d48e73ca1796c1ccc84ac95092c1fecc0d98b + call-bound: "npm:^1.0.3" + get-proto: "npm:^1.0.0" + has-tostringtag: "npm:^1.0.2" + safe-regex-test: "npm:^1.1.0" + checksum: 10c0/fdfa96c8087bf36fc4cd514b474ba2ff404219a4dd4cfa6cf5426404a1eed259bdcdb98f082a71029a48d01f27733e3436ecc6690129a7ec09cb0434bee03a2a languageName: node linkType: hard @@ -9337,16 +9376,16 @@ __metadata: linkType: hard "iterator.prototype@npm:^1.1.4": - version: 1.1.4 - resolution: "iterator.prototype@npm:1.1.4" + version: 1.1.5 + resolution: "iterator.prototype@npm:1.1.5" dependencies: define-data-property: "npm:^1.1.4" es-object-atoms: "npm:^1.0.0" get-intrinsic: "npm:^1.2.6" + get-proto: "npm:^1.0.0" has-symbols: "npm:^1.1.0" - reflect.getprototypeof: "npm:^1.0.8" set-function-name: "npm:^2.0.2" - checksum: 10c0/e63fcb5c1094192f43795b836fae9149a7dc2d445425958045e8e193df428407f909efca21bfdf0d885668ae8204681984afac7dd75478118e62f3cd3959c538 + checksum: 10c0/f7a262808e1b41049ab55f1e9c29af7ec1025a000d243b83edf34ce2416eedd56079b117fa59376bb4a724110690f13aa8427f2ee29a09eec63a7e72367626d0 languageName: node linkType: hard @@ -9713,7 +9752,7 @@ __metadata: languageName: node linkType: hard -"lit@npm:^3.1.2": +"lit@npm:^3.2.1": version: 3.2.1 resolution: "lit@npm:3.2.1" dependencies: @@ -9819,7 +9858,7 @@ __metadata: languageName: node linkType: hard -"loose-envify@npm:^1.0.0, loose-envify@npm:^1.2.0, loose-envify@npm:^1.3.1, loose-envify@npm:^1.4.0": +"loose-envify@npm:^1.0.0, loose-envify@npm:^1.1.0, loose-envify@npm:^1.2.0, loose-envify@npm:^1.3.1, loose-envify@npm:^1.4.0": version: 1.4.0 resolution: "loose-envify@npm:1.4.0" dependencies: @@ -9954,7 +9993,7 @@ __metadata: languageName: node linkType: hard -"math-intrinsics@npm:^1.0.0, math-intrinsics@npm:^1.1.0": +"math-intrinsics@npm:^1.1.0": version: 1.1.0 resolution: "math-intrinsics@npm:1.1.0" checksum: 10c0/7579ff94e899e2f76ab64491d76cf606274c874d8f2af4a442c016bd85688927fcfca157ba6bf74b08e9439dc010b248ce05b96cc7c126a354c3bae7fcb48b7f @@ -9989,14 +10028,14 @@ __metadata: linkType: hard "mdast-util-find-and-replace@npm:^3.0.0, mdast-util-find-and-replace@npm:^3.0.1": - version: 3.0.1 - resolution: "mdast-util-find-and-replace@npm:3.0.1" + version: 3.0.2 + resolution: "mdast-util-find-and-replace@npm:3.0.2" dependencies: "@types/mdast": "npm:^4.0.0" escape-string-regexp: "npm:^5.0.0" unist-util-is: "npm:^6.0.0" unist-util-visit-parents: "npm:^6.0.0" - checksum: 10c0/1faca98c4ee10a919f23b8cc6d818e5bb6953216a71dfd35f51066ed5d51ef86e5063b43dcfdc6061cd946e016a9f0d44a1dccadd58452cf4ed14e39377f00cb + checksum: 10c0/c8417a35605d567772ff5c1aa08363ff3010b0d60c8ea68c53cba09bf25492e3dd261560425c1756535f3b7107f62e7ff3857cdd8fb1e62d1b2cc2ea6e074ca2 languageName: node linkType: hard @@ -11351,14 +11390,14 @@ __metadata: languageName: node linkType: hard -"oniguruma-to-es@npm:0.8.1": - version: 0.8.1 - resolution: "oniguruma-to-es@npm:0.8.1" +"oniguruma-to-es@npm:0.10.0": + version: 0.10.0 + resolution: "oniguruma-to-es@npm:0.10.0" dependencies: emoji-regex-xs: "npm:^1.0.0" - regex: "npm:^5.0.2" - regex-recursion: "npm:^5.0.0" - checksum: 10c0/994373a52a424c6ecb81960ee46efdd03c5fbf3d0dd692ec7f164cc29e3a57cebd3ea069b4c1499f1b26cc36b5ef28b42012cedccd3c300f52c4a7acb5c15cbf + regex: "npm:^5.1.1" + regex-recursion: "npm:^5.1.1" + checksum: 10c0/d54c3975515d655186e51973bbbfa41c3191eedaca2fdb909c10d1ce463c4bbbfd075705f58552a724615e3e0d301de05123cb5cad8eac1255c36e75b8562fd5 languageName: node linkType: hard @@ -11413,7 +11452,7 @@ __metadata: languageName: node linkType: hard -"own-keys@npm:^1.0.0": +"own-keys@npm:^1.0.1": version: 1.0.1 resolution: "own-keys@npm:1.0.1" dependencies: @@ -12947,14 +12986,15 @@ __metadata: languageName: node linkType: hard -"react-dom@npm:^19.0.0": - version: 19.0.0 - resolution: "react-dom@npm:19.0.0" +"react-dom@npm:^18.2.0": + version: 18.3.1 + resolution: "react-dom@npm:18.3.1" dependencies: - scheduler: "npm:^0.25.0" + loose-envify: "npm:^1.1.0" + scheduler: "npm:^0.23.2" peerDependencies: - react: ^19.0.0 - checksum: 10c0/a36ce7ab507b237ae2759c984cdaad4af4096d8199fb65b3815c16825e5cfeb7293da790a3fc2184b52bfba7ba3ff31c058c01947aff6fd1a3701632aabaa6a9 + react: ^18.3.1 + checksum: 10c0/a752496c1941f958f2e8ac56239172296fcddce1365ce45222d04a1947e0cc5547df3e8447f855a81d6d39f008d7c32eab43db3712077f09e3f67c4874973e85 languageName: node linkType: hard @@ -13098,10 +13138,12 @@ __metadata: languageName: node linkType: hard -"react@npm:^19.0.0": - version: 19.0.0 - resolution: "react@npm:19.0.0" - checksum: 10c0/9cad8f103e8e3a16d15cb18a0d8115d8bd9f9e1ce3420310aea381eb42aa0a4f812cf047bb5441349257a05fba8a291515691e3cb51267279b2d2c3253f38471 +"react@npm:^18.2.0": + version: 18.3.1 + resolution: "react@npm:18.3.1" + dependencies: + loose-envify: "npm:^1.1.0" + checksum: 10c0/283e8c5efcf37802c9d1ce767f302dd569dd97a70d9bb8c7be79a789b9902451e0d16334b05d73299b20f048cbc3c7d288bbbde10b701fa194e2089c237dbea3 languageName: node linkType: hard @@ -13213,19 +13255,19 @@ __metadata: languageName: node linkType: hard -"reflect.getprototypeof@npm:^1.0.6, reflect.getprototypeof@npm:^1.0.8, reflect.getprototypeof@npm:^1.0.9": - version: 1.0.9 - resolution: "reflect.getprototypeof@npm:1.0.9" +"reflect.getprototypeof@npm:^1.0.6, reflect.getprototypeof@npm:^1.0.9": + version: 1.0.10 + resolution: "reflect.getprototypeof@npm:1.0.10" dependencies: call-bind: "npm:^1.0.8" define-properties: "npm:^1.2.1" - dunder-proto: "npm:^1.0.1" - es-abstract: "npm:^1.23.6" + es-abstract: "npm:^1.23.9" es-errors: "npm:^1.3.0" - get-intrinsic: "npm:^1.2.6" - gopd: "npm:^1.2.0" + es-object-atoms: "npm:^1.0.0" + get-intrinsic: "npm:^1.2.7" + get-proto: "npm:^1.0.1" which-builtin-type: "npm:^1.2.1" - checksum: 10c0/db42118a8699fa8b5856e6aa06eac32498a7bbc3c22832729049501733d060662bf16f204c546db87df8bb78b36491ecd6b3b0478c0a27be6c8302cc0770a42e + checksum: 10c0/7facec28c8008876f8ab98e80b7b9cb4b1e9224353fd4756dda5f2a4ab0d30fa0a5074777c6df24e1e0af463a2697513b0a11e548d99cf52f21f7bc6ba48d3ac languageName: node linkType: hard @@ -13261,7 +13303,7 @@ __metadata: languageName: node linkType: hard -"regex-recursion@npm:^5.0.0": +"regex-recursion@npm:^5.1.1": version: 5.1.1 resolution: "regex-recursion@npm:5.1.1" dependencies: @@ -13278,7 +13320,7 @@ __metadata: languageName: node linkType: hard -"regex@npm:^5.0.2, regex@npm:^5.1.1": +"regex@npm:^5.1.1": version: 5.1.1 resolution: "regex@npm:5.1.1" dependencies: @@ -13288,14 +13330,16 @@ __metadata: linkType: hard "regexp.prototype.flags@npm:^1.5.3": - version: 1.5.3 - resolution: "regexp.prototype.flags@npm:1.5.3" + version: 1.5.4 + resolution: "regexp.prototype.flags@npm:1.5.4" dependencies: - call-bind: "npm:^1.0.7" + call-bind: "npm:^1.0.8" define-properties: "npm:^1.2.1" es-errors: "npm:^1.3.0" + get-proto: "npm:^1.0.1" + gopd: "npm:^1.2.0" set-function-name: "npm:^2.0.2" - checksum: 10c0/e1a7c7dc42cc91abf73e47a269c4b3a8f225321b7f617baa25821f6a123a91d23a73b5152f21872c566e699207e1135d075d2251cd3e84cc96d82a910adf6020 + checksum: 10c0/83b88e6115b4af1c537f8dabf5c3744032cb875d63bc05c288b1b8c0ef37cbe55353f95d8ca817e8843806e3e150b118bc624e4279b24b4776b4198232735a77 languageName: node linkType: hard @@ -13773,10 +13817,12 @@ __metadata: languageName: node linkType: hard -"scheduler@npm:^0.25.0": - version: 0.25.0 - resolution: "scheduler@npm:0.25.0" - checksum: 10c0/a4bb1da406b613ce72c1299db43759526058fdcc413999c3c3e0db8956df7633acf395cb20eb2303b6a65d658d66b6585d344460abaee8080b4aa931f10eaafe +"scheduler@npm:^0.23.2": + version: 0.23.2 + resolution: "scheduler@npm:0.23.2" + dependencies: + loose-envify: "npm:^1.1.0" + checksum: 10c0/26383305e249651d4c58e6705d5f8425f153211aef95f15161c151f7b8de885f24751b377e4a0b3dd42cce09aad3f87a61dab7636859c0d89b7daf1a1e2a5c78 languageName: node linkType: hard @@ -13966,6 +14012,17 @@ __metadata: languageName: node linkType: hard +"set-proto@npm:^1.0.0": + version: 1.0.0 + resolution: "set-proto@npm:1.0.0" + dependencies: + dunder-proto: "npm:^1.0.1" + es-errors: "npm:^1.3.0" + es-object-atoms: "npm:^1.0.0" + checksum: 10c0/ca5c3ccbba479d07c30460e367e66337cec825560b11e8ba9c5ebe13a2a0d6021ae34eddf94ff3dfe17a3104dc1f191519cb6c48378b503e5c3f36393938776a + languageName: node + linkType: hard + "setprototypeof@npm:1.1.0": version: 1.1.0 resolution: "setprototypeof@npm:1.1.0" @@ -14045,16 +14102,18 @@ __metadata: linkType: hard "shiki@npm:^1.16.2": - version: 1.24.4 - resolution: "shiki@npm:1.24.4" - dependencies: - "@shikijs/core": "npm:1.24.4" - "@shikijs/engine-javascript": "npm:1.24.4" - "@shikijs/engine-oniguruma": "npm:1.24.4" - "@shikijs/types": "npm:1.24.4" - "@shikijs/vscode-textmate": "npm:^9.3.1" + version: 1.26.1 + resolution: "shiki@npm:1.26.1" + dependencies: + "@shikijs/core": "npm:1.26.1" + "@shikijs/engine-javascript": "npm:1.26.1" + "@shikijs/engine-oniguruma": "npm:1.26.1" + "@shikijs/langs": "npm:1.26.1" + "@shikijs/themes": "npm:1.26.1" + "@shikijs/types": "npm:1.26.1" + "@shikijs/vscode-textmate": "npm:^10.0.1" "@types/hast": "npm:^3.0.4" - checksum: 10c0/c3758f67ea997f7b9ce4fe32cf50178a5f26fa49fdb2c3550115764d0a55441f63bd89e659e6ba133bdaeac57bfad485412c1b19c19f3ae523c8ffd4334b0c38 + checksum: 10c0/1a9ca4d8e1adfc8c6342ffd4760781667b7c3e7647231b12c82b8087bcf782b0c5189fb950d740f89dc38f403f3707807d10831f3ec9fcd28569297b42e292e5 languageName: node linkType: hard @@ -14575,15 +14634,6 @@ __metadata: languageName: node linkType: hard -"style-to-object@npm:^0.4.0": - version: 0.4.4 - resolution: "style-to-object@npm:0.4.4" - dependencies: - inline-style-parser: "npm:0.1.1" - checksum: 10c0/3a733080da66952881175b17d65f92985cf94c1ca358a92cf21b114b1260d49b94a404ed79476047fb95698d64c7e366ca7443f0225939e2fb34c38bbc9c7639 - languageName: node - linkType: hard - "style-to-object@npm:^1.0.0": version: 1.0.8 resolution: "style-to-object@npm:1.0.8" @@ -15731,11 +15781,11 @@ __metadata: linkType: hard "yaml@npm:^2.2.2, yaml@npm:^2.5.1": - version: 2.6.1 - resolution: "yaml@npm:2.6.1" + version: 2.7.0 + resolution: "yaml@npm:2.7.0" bin: yaml: bin.mjs - checksum: 10c0/aebf07f61c72b38c74d2b60c3a3ccf89ee4da45bcd94b2bfb7899ba07a5257625a7c9f717c65a6fc511563d48001e01deb1d9e55f0133f3e2edf86039c8c1be7 + checksum: 10c0/886a7d2abbd70704b79f1d2d05fe9fb0aa63aefb86e1cb9991837dced65193d300f5554747a872b4b10ae9a12bc5d5327e4d04205f70336e863e35e89d8f4ea9 languageName: node linkType: hard From 1b8e6a3dbeac70daa510f7fe385aba103b122f0f Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Fri, 10 Jan 2025 14:49:30 +0100 Subject: [PATCH 11/32] Add ScreenFingerprint and NavigatorFingerprint --- .../fingerprint_suite/_camoufox_adapter.py | 1 + src/crawlee/fingerprint_suite/_types.py | 147 ++++++++++++++++++ .../test_camoufox_integration.py | 6 + 3 files changed, 154 insertions(+) create mode 100644 src/crawlee/fingerprint_suite/_camoufox_adapter.py create mode 100644 src/crawlee/fingerprint_suite/_types.py create mode 100644 tests/unit/fingerprint_suite/test_camoufox_integration.py diff --git a/src/crawlee/fingerprint_suite/_camoufox_adapter.py b/src/crawlee/fingerprint_suite/_camoufox_adapter.py new file mode 100644 index 0000000000..b86160980b --- /dev/null +++ b/src/crawlee/fingerprint_suite/_camoufox_adapter.py @@ -0,0 +1 @@ +"""Input and ouput adapter for camoufox fingerprint handling""" diff --git a/src/crawlee/fingerprint_suite/_types.py b/src/crawlee/fingerprint_suite/_types.py new file mode 100644 index 0000000000..501f88fd60 --- /dev/null +++ b/src/crawlee/fingerprint_suite/_types.py @@ -0,0 +1,147 @@ +from typing import Annotated + +from pydantic import BaseModel, Field + +class ScreenFingerprint(BaseModel): + """ + Collection of various attributes from following sources: + + https://developer.mozilla.org/en-US/docs/Web/API/Screen + https://developer.mozilla.org/en-US/docs/Web/API/ScreenDetailed + https://developer.mozilla.org/en-US/docs/Web/API/Window + https://developer.mozilla.org/en-US/docs/Web/API/Element/clientWidth + """ + + avail_height: Annotated[float, Field(alias="availHeight")] + """https://developer.mozilla.org/en-US/docs/Web/API/Screen/availHeight""" + + avail_width: Annotated[float, Field(alias="availWidth")] + """https://developer.mozilla.org/en-US/docs/Web/API/Screen/availWidth""" + + avail_top: Annotated[float, Field(alias="availTop")] + """https://developer.mozilla.org/en-US/docs/Web/API/ScreenDetailed/availTop""" + + avail_left: Annotated[float, Field(alias="availLeft")] + """https://developer.mozilla.org/en-US/docs/Web/API/ScreenDetailed/left""" + + color_depth: Annotated[float, Field(alias="colorDepth")] + """https://developer.mozilla.org/en-US/docs/Web/API/Screen/colorDepth""" + + height: float + """https://developer.mozilla.org/en-US/docs/Web/API/Screen/height""" + + pixel_depth: Annotated[float, Field(alias="pixelDepth")] + """https://developer.mozilla.org/en-US/docs/Web/API/Screen/pixelDepth""" + + width: float + """https://developer.mozilla.org/en-US/docs/Web/API/Screen/width""" + + device_pixel_ratio: Annotated[float, Field(alias="devicePixelRatio")] + """https://developer.mozilla.org/en-US/docs/Web/API/Window/devicePixelRatio""" + + page_x_offset: Annotated[float, Field(alias="pageXOffset")] + """https://developer.mozilla.org/en-US/docs/Web/API/Window/scrollX""" + + page_y_offset: Annotated[float, Field(alias="pageYOffset")] + """https://developer.mozilla.org/en-US/docs/Web/API/Window/scrollY""" + + inner_height: Annotated[float, Field(alias="innerHeight")] + """https://developer.mozilla.org/en-US/docs/Web/API/Window/innerHeight""" + + outer_height: Annotated[float, Field(alias="outerHeight")] + """https://developer.mozilla.org/en-US/docs/Web/API/Window/outerHeight""" + + outer_width: Annotated[float, Field(alias="outerWidth")] + """https://developer.mozilla.org/en-US/docs/Web/API/Window/outerWidth""" + + inner_width: Annotated[float, Field(alias="innerWidth")] + """https://developer.mozilla.org/en-US/docs/Web/API/Window/innerWidth""" + + screen_x: Annotated[float, Field(alias="screenX")] # Why screenY not present in JS? + """https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent/screenX""" + + client_width: Annotated[float, Field(alias="clientWidth")] + """https://developer.mozilla.org/en-US/docs/Web/API/Element/clientWidth""" + + client_height: Annotated[float, Field(alias="clientHeight")] + """https://developer.mozilla.org/en-US/docs/Web/API/Element/clientHeight""" + + has_hdr: Annotated[bool, Field(alias="hasHDR")] # What is this? A placeholder? + + +class NavigatorFingerprint(BaseModel): + """ + Collection of various attributes from following sources: + + https://developer.mozilla.org/en-US/docs/Web/API/Navigator + """ + + user_agent: Annotated[str, Field(alias="userAgent")] + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/userAgent""" + + user_agent_data: Annotated[str, Field(alias="userAgentData")] + """https://developer.mozilla.org/en-US/docs/Web/API/WorkerNavigator/userAgentData""" + + do_not_track: Annotated[str, Field(alias="doNotTrack")] + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/doNotTrack""" + + app_code_name: Annotated[str, Field(alias="appCodeName")] + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/appCodeName""" + + app_name: Annotated[str, Field(alias="appName")] + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/appName""" + + app_version: Annotated[str, Field(alias="appVersion")] + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/appVersion""" + + oscpu: str + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/oscpu""" + + webdriver: str + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/webdriver""" + + language: str + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/language""" + + languages: list[str] + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/languages""" + + platform: str + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/platform""" + + device_memory: Annotated[float|None, Field(alias="deviceMemory")] = None # Firefox does not have deviceMemory available + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/deviceMemory""" + + hardware_concurrency: Annotated[float, Field(alias="hardwareConcurrency")] + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/hardwareConcurrency""" + + product: str + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/product""" + + productSub: str + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/productSub""" + + vendor: str + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/vendor""" + + vendorSub: str + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/vendorSub""" + + max_touch_points: Annotated[float|None, Field(alias="maxTouchPoints")] = None + """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/maxTouchPoints""" + + extraProperties: dict[str, str] + + +class Fingerprint(BaseModel): + """Represents specific browser fingerprint collection. + + Collection of browser settings, attributes and capabilities is commonly referred to as a browser fingerprint. + Such fingerprint can be used to collect various information or track or group users, or thus also detect web + crawlers by inspecting suspiciously looking browser fingerprints. + This object contains attributes that are sub set of such specific fingerprint. + See `https://docs.apify.com/academy/anti-scraping/mitigation/generating-fingerprints` + TODO: Update guide with Python example.""" + + screen: ScreenFingerprint + navigator: NavigatorFingerprint diff --git a/tests/unit/fingerprint_suite/test_camoufox_integration.py b/tests/unit/fingerprint_suite/test_camoufox_integration.py new file mode 100644 index 0000000000..14a831fcdf --- /dev/null +++ b/tests/unit/fingerprint_suite/test_camoufox_integration.py @@ -0,0 +1,6 @@ + +from crawlee.fingerprint_suite._types import ScreenFingerprint + + +def test_camoufox_fingerprint_generator_conversion(): + pass From 9828a36dc9bc4181f719432912e2138f31097c4a Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Mon, 13 Jan 2025 10:32:31 +0100 Subject: [PATCH 12/32] Add Fingerprint and their options types Draft adapter and basic test. --- .../_browserforge_adapter.py | 19 ++++ .../fingerprint_suite/_camoufox_adapter.py | 1 - .../_fingerprint_generator.py | 11 ++ src/crawlee/fingerprint_suite/_types.py | 101 ++++++++++++++++-- tests/unit/fingerprint_suite/test_adapters.py | 14 +++ .../test_camoufox_integration.py | 6 -- 6 files changed, 136 insertions(+), 16 deletions(-) create mode 100644 src/crawlee/fingerprint_suite/_browserforge_adapter.py delete mode 100644 src/crawlee/fingerprint_suite/_camoufox_adapter.py create mode 100644 src/crawlee/fingerprint_suite/_fingerprint_generator.py create mode 100644 tests/unit/fingerprint_suite/test_adapters.py delete mode 100644 tests/unit/fingerprint_suite/test_camoufox_integration.py diff --git a/src/crawlee/fingerprint_suite/_browserforge_adapter.py b/src/crawlee/fingerprint_suite/_browserforge_adapter.py new file mode 100644 index 0000000000..a42a915854 --- /dev/null +++ b/src/crawlee/fingerprint_suite/_browserforge_adapter.py @@ -0,0 +1,19 @@ +"""Input and ouput adapter for camoufox fingerprint handling""" +from browserforge.fingerprints import Fingerprint as bf_Fingerprint, FingerprintGenerator as bf_FingerprintGenerator +from typing_extensions import override + +from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator +from crawlee.fingerprint_suite._types import Fingerprint, FingerprintGeneratorOptions + +class FingerprintGenerator(AbstractFingerprintGenerator): + + @staticmethod + def get_fingerprint(bf_fingerprint: bf_Fingerprint) -> Fingerprint: + return Fingerprint.model_validate(bf_fingerprint, from_attributes=True) + + @override + @staticmethod + def generate(options: FingerprintGeneratorOptions | None = None) -> Fingerprint: + bf_fingerprint = bf_FingerprintGenerator().generate(**(options or {})) + return Fingerprint.model_validate(bf_fingerprint, from_attributes=True) + diff --git a/src/crawlee/fingerprint_suite/_camoufox_adapter.py b/src/crawlee/fingerprint_suite/_camoufox_adapter.py deleted file mode 100644 index b86160980b..0000000000 --- a/src/crawlee/fingerprint_suite/_camoufox_adapter.py +++ /dev/null @@ -1 +0,0 @@ -"""Input and ouput adapter for camoufox fingerprint handling""" diff --git a/src/crawlee/fingerprint_suite/_fingerprint_generator.py b/src/crawlee/fingerprint_suite/_fingerprint_generator.py new file mode 100644 index 0000000000..896445b995 --- /dev/null +++ b/src/crawlee/fingerprint_suite/_fingerprint_generator.py @@ -0,0 +1,11 @@ +from abc import ABC + +from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions, Fingerprint + + +class AbstractFingerprintGenerator(ABC): + + @staticmethod + def generate(options: FingerprintGeneratorOptions | None = None) -> Fingerprint: + ... + diff --git a/src/crawlee/fingerprint_suite/_types.py b/src/crawlee/fingerprint_suite/_types.py index 501f88fd60..368086d4b7 100644 --- a/src/crawlee/fingerprint_suite/_types.py +++ b/src/crawlee/fingerprint_suite/_types.py @@ -1,7 +1,14 @@ -from typing import Annotated +from typing import Annotated, Literal from pydantic import BaseModel, Field +from crawlee.browsers._types import BrowserType + + +SupportedOperatingSystems= Literal["windows", "macos", "linux", "android", "ios"] +SupportedDevices = Literal["desktop", "mobile"] +SupportedHttpVersion = Literal["1", "2"] + class ScreenFingerprint(BaseModel): """ Collection of various attributes from following sources: @@ -9,7 +16,7 @@ class ScreenFingerprint(BaseModel): https://developer.mozilla.org/en-US/docs/Web/API/Screen https://developer.mozilla.org/en-US/docs/Web/API/ScreenDetailed https://developer.mozilla.org/en-US/docs/Web/API/Window - https://developer.mozilla.org/en-US/docs/Web/API/Element/clientWidth + https://developer.mozilla.org/en-US/docs/Web/API/Element """ avail_height: Annotated[float, Field(alias="availHeight")] @@ -79,8 +86,22 @@ class NavigatorFingerprint(BaseModel): user_agent: Annotated[str, Field(alias="userAgent")] """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/userAgent""" - user_agent_data: Annotated[str, Field(alias="userAgentData")] - """https://developer.mozilla.org/en-US/docs/Web/API/WorkerNavigator/userAgentData""" + + + user_agent_data: Annotated[dict[str, str | list[dict[str, str]] | bool], Field(alias="userAgentData")] + """https://developer.mozilla.org/en-US/docs/Web/API/WorkerNavigator/userAgentData + + In JS this is just userAgentData: Record, but it can contain more stuff like + + mobile = False + 'fullVersionList' = [ + {'brand': 'Google Chrome', 'version': '131.0.6778.86'}, + {'brand': 'Chromium', 'version': '131.0.6778.86'}, + {'brand': 'Not_A Brand', 'version': '24.0.0.0'} + ] + so more generic type should be allowed + + """ do_not_track: Annotated[str, Field(alias="doNotTrack")] """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/doNotTrack""" @@ -118,21 +139,25 @@ class NavigatorFingerprint(BaseModel): product: str """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/product""" - productSub: str + product_sub: Annotated[str, Field(alias="productSub")] """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/productSub""" vendor: str """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/vendor""" - vendorSub: str + vendor_sub: Annotated[str, Field(alias="vendorSub")] """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/vendorSub""" max_touch_points: Annotated[float|None, Field(alias="maxTouchPoints")] = None """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/maxTouchPoints""" - extraProperties: dict[str, str] + extra_properties: Annotated[dict[str, str], Field(alias="extraProperties")] = [] +class VideoCard(BaseModel): + rendered: str + vendor: str + class Fingerprint(BaseModel): """Represents specific browser fingerprint collection. @@ -140,8 +165,66 @@ class Fingerprint(BaseModel): Such fingerprint can be used to collect various information or track or group users, or thus also detect web crawlers by inspecting suspiciously looking browser fingerprints. This object contains attributes that are sub set of such specific fingerprint. - See `https://docs.apify.com/academy/anti-scraping/mitigation/generating-fingerprints` - TODO: Update guide with Python example.""" + See `https://docs.apify.com/academy/anti-scraping/mitigation/generating-fingerprints` TODO: Update guide with Python example. + """ screen: ScreenFingerprint navigator: NavigatorFingerprint + video_codecs: Annotated[dict[str, str], Field(alias="videoCodecs")] = None + audio_codecs: Annotated[dict[str, str], Field(alias="audioCodecs")] = None + plugins_data: Annotated[dict[str, str], Field(alias="pluginsData")] = None + battery: dict[str, str] | None = None + video_card: VideoCard + multimedia_devices: Annotated[list[str], Field(alias="multimediaDevices")] + fonts: list[str] = [] + mock_web_rtc: Annotated[bool, Field(alias="mockWebRTC")] + slim: Annotated[bool|None, Field(alias="slim")]=None + + +class ScreenOptions(BaseModel): + """Defines the screen dimensions of the generated fingerprint.""" + min_width: Annotated[float|None, Field(alias="minWidth")] = None + max_width: Annotated[float | None, Field(alias="maxWidth")] = None + min_height: Annotated[float | None, Field(alias="minHeight")] = None + max_height: Annotated[float | None, Field(alias="maxHeight")] = None + +class Browser: + name: BrowserType + """Name of the browser.""" + min_version: Annotated[float|None, Field(alias="minVersion")] = None + """Minimum version of browser used.""" + max_version: Annotated[float | None, Field(alias="maxVersion")] = None + """Maximum version of browser used.""" + http_version: Annotated[SupportedHttpVersion | None, Field(alias="httpVersion")] = None + """HTTP version to be used for header generation (the headers differ depending on the version).""" + + + +class HeaderGeneratorOptions(BaseModel): + browsers: BrowserType + """List of BrowserSpecifications to generate the headers for.""" + + operating_systems: Annotated[list[SupportedOperatingSystems], Field(alias="operatingSystems")] + """List of operating systems to generate the headers for.""" + + devices: list[SupportedDevices] + """List of devices to generate the headers for.""" + + locales: list[str] + """List of at most 10 languages to include in the [Accept-Language] + (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language) request header + in the language format accepted by that header, for example `en`, `en-US` or `de`.""" + + http_version: Annotated[SupportedHttpVersion, Field(alias="httpVersion")] + """HTTP version to be used for header generation (the headers differ depending on the version).""" + + strict: bool + """If true, the generator will throw an error if it cannot generate headers based on the input.""" + +class FingerprintGeneratorOptions(BaseModel): + header_options: HeaderGeneratorOptions + screen: ScreenOptions + mock_web_rtc: Annotated[bool, Field(alias="mockWebRTC")] = None + """Whether to mock WebRTC when injecting the fingerprint.""" + slim: Annotated[bool | None, Field(alias="slim")] = None + """Disables performance-heavy evasions when injecting the fingerprint.""" diff --git a/tests/unit/fingerprint_suite/test_adapters.py b/tests/unit/fingerprint_suite/test_adapters.py new file mode 100644 index 0000000000..b5357be9e2 --- /dev/null +++ b/tests/unit/fingerprint_suite/test_adapters.py @@ -0,0 +1,14 @@ +import pytest + +from crawlee.fingerprint_suite._browserforge_adapter import FingerprintGenerator as bf_FingerprintGenerator +from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator + + +@pytest.mark.skip(reason="Injector not implemented yet so we use browserforge injector.") +@pytest.mark.parametrize("fingerprint_generator",[ + pytest.param(bf_FingerprintGenerator, id="Browserforge"), +]) +def test_fingerprint_generator_has_default(fingerprint_generator: AbstractFingerprintGenerator): + """Test that header generator can work without any options.""" + assert fingerprint_generator.generate() + diff --git a/tests/unit/fingerprint_suite/test_camoufox_integration.py b/tests/unit/fingerprint_suite/test_camoufox_integration.py deleted file mode 100644 index 14a831fcdf..0000000000 --- a/tests/unit/fingerprint_suite/test_camoufox_integration.py +++ /dev/null @@ -1,6 +0,0 @@ - -from crawlee.fingerprint_suite._types import ScreenFingerprint - - -def test_camoufox_fingerprint_generator_conversion(): - pass From f733c078cf7a0a8fca0443ca26bddc439bb908cb Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Mon, 13 Jan 2025 16:47:18 +0100 Subject: [PATCH 13/32] Add adapter tests --- .../_browserforge_adapter.py | 34 +++++++--- src/crawlee/fingerprint_suite/_types.py | 36 ++++++++--- tests/unit/fingerprint_suite/test_adapters.py | 63 ++++++++++++++++++- 3 files changed, 114 insertions(+), 19 deletions(-) diff --git a/src/crawlee/fingerprint_suite/_browserforge_adapter.py b/src/crawlee/fingerprint_suite/_browserforge_adapter.py index a42a915854..9141612024 100644 --- a/src/crawlee/fingerprint_suite/_browserforge_adapter.py +++ b/src/crawlee/fingerprint_suite/_browserforge_adapter.py @@ -1,19 +1,33 @@ -"""Input and ouput adapter for camoufox fingerprint handling""" -from browserforge.fingerprints import Fingerprint as bf_Fingerprint, FingerprintGenerator as bf_FingerprintGenerator +from browserforge.fingerprints import Fingerprint as bf_Fingerprint, FingerprintGenerator as bf_FingerprintGenerator, \ + Screen from typing_extensions import override from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator -from crawlee.fingerprint_suite._types import Fingerprint, FingerprintGeneratorOptions +from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions class FingerprintGenerator(AbstractFingerprintGenerator): - @staticmethod - def get_fingerprint(bf_fingerprint: bf_Fingerprint) -> Fingerprint: - return Fingerprint.model_validate(bf_fingerprint, from_attributes=True) - @override @staticmethod - def generate(options: FingerprintGeneratorOptions | None = None) -> Fingerprint: - bf_fingerprint = bf_FingerprintGenerator().generate(**(options or {})) - return Fingerprint.model_validate(bf_fingerprint, from_attributes=True) + def generate(options: FingerprintGeneratorOptions | None = None, strict: bool = False) -> bf_Fingerprint: + options = options or FingerprintGeneratorOptions() + bf_options = FingerprintGenerator._prepare_options(options) + bf_fingerprint = bf_FingerprintGenerator().generate( + screen = Screen(**(bf_options["screen"] or {})), + mock_webrtc = bf_options["mock_web_rtc"], + slim=bf_options["slim"], + **bf_options["header_options"]) + return bf_fingerprint + + @staticmethod + def _prepare_options(options: FingerprintGeneratorOptions) -> dict[any,any]: + bf_options = options.model_dump() + if bf_options["header_options"] is None: + bf_options["header_options"] = dict() + else: + bf_options["header_options"]["browser"] = bf_options["header_options"].pop("browsers", None) + bf_options["header_options"]["os"] = bf_options["header_options"].pop("operating_systems", None) + bf_options["header_options"]["device"] = bf_options["header_options"].pop("devices", None) + bf_options["header_options"]["locale"] = bf_options["header_options"].pop("locales", None) + return bf_options diff --git a/src/crawlee/fingerprint_suite/_types.py b/src/crawlee/fingerprint_suite/_types.py index 368086d4b7..ceaac47435 100644 --- a/src/crawlee/fingerprint_suite/_types.py +++ b/src/crawlee/fingerprint_suite/_types.py @@ -188,6 +188,10 @@ class ScreenOptions(BaseModel): min_height: Annotated[float | None, Field(alias="minHeight")] = None max_height: Annotated[float | None, Field(alias="maxHeight")] = None + class Config: + extra = "forbid" + populate_by_name = True + class Browser: name: BrowserType """Name of the browser.""" @@ -198,33 +202,49 @@ class Browser: http_version: Annotated[SupportedHttpVersion | None, Field(alias="httpVersion")] = None """HTTP version to be used for header generation (the headers differ depending on the version).""" + class Config: + extra = "forbid" + populate_by_name = True class HeaderGeneratorOptions(BaseModel): - browsers: BrowserType + browsers: list[BrowserType] | None = None """List of BrowserSpecifications to generate the headers for.""" - operating_systems: Annotated[list[SupportedOperatingSystems], Field(alias="operatingSystems")] + operating_systems: Annotated[list[SupportedOperatingSystems]| None, Field(alias="operatingSystems")] = None """List of operating systems to generate the headers for.""" - devices: list[SupportedDevices] + devices: list[SupportedDevices]| None = None """List of devices to generate the headers for.""" - locales: list[str] + locales: list[str]| None = None """List of at most 10 languages to include in the [Accept-Language] (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language) request header in the language format accepted by that header, for example `en`, `en-US` or `de`.""" - http_version: Annotated[SupportedHttpVersion, Field(alias="httpVersion")] + http_version: Annotated[SupportedHttpVersion| None, Field(alias="httpVersion")]= None """HTTP version to be used for header generation (the headers differ depending on the version).""" - strict: bool + strict: bool| None = None """If true, the generator will throw an error if it cannot generate headers based on the input.""" + class Config: + extra = "forbid" + populate_by_name = True + class FingerprintGeneratorOptions(BaseModel): - header_options: HeaderGeneratorOptions - screen: ScreenOptions + """All generator options are optional. If any value si s not specified, then a default value will be used. + + Default values are implementation detail of used fingerprint generator. + Specific default values should not be relied upon. Use explicit values if it matters for your use case. + """ + + header_options: HeaderGeneratorOptions | None = None + screen: ScreenOptions | None = None mock_web_rtc: Annotated[bool, Field(alias="mockWebRTC")] = None """Whether to mock WebRTC when injecting the fingerprint.""" slim: Annotated[bool | None, Field(alias="slim")] = None """Disables performance-heavy evasions when injecting the fingerprint.""" + class Config: + extra = "forbid" + populate_by_name = True diff --git a/tests/unit/fingerprint_suite/test_adapters.py b/tests/unit/fingerprint_suite/test_adapters.py index b5357be9e2..4a4d71dee6 100644 --- a/tests/unit/fingerprint_suite/test_adapters.py +++ b/tests/unit/fingerprint_suite/test_adapters.py @@ -2,9 +2,9 @@ from crawlee.fingerprint_suite._browserforge_adapter import FingerprintGenerator as bf_FingerprintGenerator from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator +from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions, ScreenOptions, HeaderGeneratorOptions -@pytest.mark.skip(reason="Injector not implemented yet so we use browserforge injector.") @pytest.mark.parametrize("fingerprint_generator",[ pytest.param(bf_FingerprintGenerator, id="Browserforge"), ]) @@ -12,3 +12,64 @@ def test_fingerprint_generator_has_default(fingerprint_generator: AbstractFinger """Test that header generator can work without any options.""" assert fingerprint_generator.generate() + + +@pytest.mark.parametrize("fingerprint_generator",[ + pytest.param(bf_FingerprintGenerator, id="Browserforge"), +]) +def test_fingerprint_generator_some_options(fingerprint_generator: AbstractFingerprintGenerator): + """Test that header generator can work with only some options.""" + options = FingerprintGeneratorOptions(screen=ScreenOptions(min_width = 500), mockWebRTC=True) + + fingerprint = fingerprint_generator.generate(options=options) + + assert fingerprint.mockWebRTC == True + assert fingerprint.screen.availWidth >= 500 + + +@pytest.mark.parametrize("fingerprint_generator",[ + pytest.param(bf_FingerprintGenerator, id="Browserforge"), +]) +def test_fingerprint_generator_all_options(fingerprint_generator: AbstractFingerprintGenerator): + """Test that header generator can work with all the options. Some most basic checks of fingerprint. + + Fingerprint generation option might have no effect if there is no fingerprint sample present in collected data. + """ + min_width = 600 + max_width = 1800 + min_height = 400 + max_height = 1200 + + options = FingerprintGeneratorOptions( + screen=ScreenOptions( + min_width = min_width, + max_width=max_width, + min_height=min_height, + max_height=max_height, + ), + mockWebRTC=True, + slim=False, + header_options = HeaderGeneratorOptions( + strict = True, + browsers = ["firefox"], + operating_systems = ["windows"], + devices = ["mobile"], + locales = ["en"], # This does not seem to generate any other values than `en-US` regardless of the input + http_version = "2", # Http1 does not work in browserforge + + ) + ) + + fingerprint = fingerprint_generator.generate(options=options) + + assert fingerprint.screen.availWidth >= min_width + assert fingerprint.screen.availWidth <= max_width + assert fingerprint.screen.availHeight >= min_height + assert fingerprint.screen.availHeight <= max_height + + assert fingerprint.mockWebRTC == True + assert fingerprint.slim == False + assert "Firefox" in fingerprint.navigator.userAgent + assert "Win" in fingerprint.navigator.oscpu + assert "en-US" in fingerprint.navigator.languages + From 97011d902923530f6a4d29ac97c44a624807bab0 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Mon, 13 Jan 2025 17:33:49 +0100 Subject: [PATCH 14/32] Integrate into pw_crawler TODO: Solve circular imports --- .../_playwright_browser_controller.py | 19 +++++---- .../browsers/_playwright_browser_plugin.py | 2 +- .../_browserforge_adapter.py | 40 ++++++++++--------- tests/unit/fingerprint_suite/test_adapters.py | 23 ++++------- 4 files changed, 41 insertions(+), 43 deletions(-) diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index 0e67679be6..e2d183bf2e 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -13,6 +13,7 @@ from crawlee.browsers._base_browser_controller import BaseBrowserController from crawlee.browsers._types import BrowserType from crawlee.fingerprint_suite import HeaderGenerator +from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator if TYPE_CHECKING: from collections.abc import Mapping @@ -43,8 +44,7 @@ def __init__( *, max_open_pages_per_browser: int = 20, header_generator: HeaderGenerator | None = _DEFAULT_HEADER_GENERATOR, - use_fingerprints: bool = True, - fingerprint_generator_options: dict[str, Any] | None = None, + fingerprint_generator: AbstractFingerprintGenerator | None = None, ) -> None: """A default constructor. @@ -54,18 +54,21 @@ def __init__( header_generator: An optional `HeaderGenerator` instance used to generate and manage HTTP headers for requests made by the browser. By default, a predefined header generator is used. Set to `None` to disable automatic header modifications. - use_fingerprints: Inject generated fingerprints to page. - fingerprint_generator_options: Override generated fingerprints with these specific values, if possible. + fingerprint_generator: An optional instance of implementation of `AbstractFingerprintGenerator` that is used + to generate browser fingerprints together with headers. """ + if header_generator and fingerprint_generator: + raise ValueError("Do not use `header_generator` and `fingerprint_generator` arguments at the same time. " + "Choose only one. `fingerprint_generator` generates headers as well.") self._browser = browser self._max_open_pages_per_browser = max_open_pages_per_browser self._header_generator = header_generator + self._fingerprint_generator = fingerprint_generator self._browser_context: BrowserContext | None = None self._pages = list[Page]() self._last_page_opened_at = datetime.now(timezone.utc) - self._use_fingerprints = use_fingerprints self._fingerprint_generator_options = fingerprint_generator_options @property @@ -152,8 +155,8 @@ async def _set_browser_context( ) -> None: """Set browser context. - Create context using `browserforge` if `_use_fingerprints` is True. - Create context without fingerprints with headers based header generator if available. + Create context using `browserforge` if `self._fingerprint_generator` exists. + Create context without fingerprints with headers based on header generator if available. """ browser_new_context_options = dict(browser_new_context_options) if browser_new_context_options else {} @@ -167,7 +170,7 @@ async def _set_browser_context( password=proxy_info.password, ) - if self._use_fingerprints: + if self._fingerprint_generator: self._browser_context = await AsyncNewContext( browser=self._browser, fingerprint_options=(fingerprint_options or {}), **browser_new_context_options ) diff --git a/src/crawlee/browsers/_playwright_browser_plugin.py b/src/crawlee/browsers/_playwright_browser_plugin.py index 69792ac7dc..ebafa29316 100644 --- a/src/crawlee/browsers/_playwright_browser_plugin.py +++ b/src/crawlee/browsers/_playwright_browser_plugin.py @@ -148,6 +148,6 @@ async def new_browser(self) -> PlaywrightBrowserController: return PlaywrightBrowserController( browser, max_open_pages_per_browser=self._max_open_pages_per_browser, - use_fingerprints=self._use_fingerprints, + fingerprint_generator=self.fingerprint_generator, fingerprint_generator_options=self._fingerprint_generator_options, ) diff --git a/src/crawlee/fingerprint_suite/_browserforge_adapter.py b/src/crawlee/fingerprint_suite/_browserforge_adapter.py index 9141612024..d0a8014176 100644 --- a/src/crawlee/fingerprint_suite/_browserforge_adapter.py +++ b/src/crawlee/fingerprint_suite/_browserforge_adapter.py @@ -1,3 +1,5 @@ +from copy import deepcopy + from browserforge.fingerprints import Fingerprint as bf_Fingerprint, FingerprintGenerator as bf_FingerprintGenerator, \ Screen from typing_extensions import override @@ -7,27 +9,29 @@ class FingerprintGenerator(AbstractFingerprintGenerator): - @override - @staticmethod - def generate(options: FingerprintGeneratorOptions | None = None, strict: bool = False) -> bf_Fingerprint: - options = options or FingerprintGeneratorOptions() - bf_options = FingerprintGenerator._prepare_options(options) + def __init__(self, fingerprint_generator_options: FingerprintGeneratorOptions | None = None, strict: bool = False): + self._fingerprint_generator_options = FingerprintGenerator._prepare_options( + fingerprint_generator_options or FingerprintGeneratorOptions()) + self._strict = strict - bf_fingerprint = bf_FingerprintGenerator().generate( - screen = Screen(**(bf_options["screen"] or {})), - mock_webrtc = bf_options["mock_web_rtc"], - slim=bf_options["slim"], - **bf_options["header_options"]) + @override + def generate(self) -> bf_Fingerprint: + bf_fingerprint = bf_FingerprintGenerator().generate(**self._fingerprint_generator_options) return bf_fingerprint @staticmethod def _prepare_options(options: FingerprintGeneratorOptions) -> dict[any,any]: - bf_options = options.model_dump() - if bf_options["header_options"] is None: - bf_options["header_options"] = dict() + raw_options = options.model_dump() + bf_options = {} + if raw_options["header_options"] is None: + header_options = dict() else: - bf_options["header_options"]["browser"] = bf_options["header_options"].pop("browsers", None) - bf_options["header_options"]["os"] = bf_options["header_options"].pop("operating_systems", None) - bf_options["header_options"]["device"] = bf_options["header_options"].pop("devices", None) - bf_options["header_options"]["locale"] = bf_options["header_options"].pop("locales", None) - return bf_options + header_options = deepcopy(raw_options["header_options"]) + header_options["browser"] = header_options.pop("browsers", None) + header_options["os"] = header_options.pop("operating_systems", None) + header_options["device"] = header_options.pop("devices", None) + header_options["locale"] = header_options.pop("locales", None) + + bf_options["mock_webrtc"] = raw_options["mock_web_rtc"] + bf_options["screen"] = Screen(**(raw_options.get("screen") or {})) + return {**bf_options , **header_options} diff --git a/tests/unit/fingerprint_suite/test_adapters.py b/tests/unit/fingerprint_suite/test_adapters.py index 4a4d71dee6..fba65298d6 100644 --- a/tests/unit/fingerprint_suite/test_adapters.py +++ b/tests/unit/fingerprint_suite/test_adapters.py @@ -1,36 +1,27 @@ import pytest -from crawlee.fingerprint_suite._browserforge_adapter import FingerprintGenerator as bf_FingerprintGenerator +from crawlee.fingerprint_suite._browserforge_adapter import FingerprintGenerator from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions, ScreenOptions, HeaderGeneratorOptions -@pytest.mark.parametrize("fingerprint_generator",[ - pytest.param(bf_FingerprintGenerator, id="Browserforge"), -]) -def test_fingerprint_generator_has_default(fingerprint_generator: AbstractFingerprintGenerator): +def test_fingerprint_generator_has_default(): """Test that header generator can work without any options.""" - assert fingerprint_generator.generate() + assert FingerprintGenerator().generate() -@pytest.mark.parametrize("fingerprint_generator",[ - pytest.param(bf_FingerprintGenerator, id="Browserforge"), -]) -def test_fingerprint_generator_some_options(fingerprint_generator: AbstractFingerprintGenerator): +def test_fingerprint_generator_some_options(): """Test that header generator can work with only some options.""" options = FingerprintGeneratorOptions(screen=ScreenOptions(min_width = 500), mockWebRTC=True) - fingerprint = fingerprint_generator.generate(options=options) + fingerprint = FingerprintGenerator(fingerprint_generator_options=options).generate() assert fingerprint.mockWebRTC == True assert fingerprint.screen.availWidth >= 500 -@pytest.mark.parametrize("fingerprint_generator",[ - pytest.param(bf_FingerprintGenerator, id="Browserforge"), -]) -def test_fingerprint_generator_all_options(fingerprint_generator: AbstractFingerprintGenerator): +def test_fingerprint_generator_all_options(): """Test that header generator can work with all the options. Some most basic checks of fingerprint. Fingerprint generation option might have no effect if there is no fingerprint sample present in collected data. @@ -60,7 +51,7 @@ def test_fingerprint_generator_all_options(fingerprint_generator: AbstractFinger ) ) - fingerprint = fingerprint_generator.generate(options=options) + fingerprint = FingerprintGenerator(fingerprint_generator_options=options).generate() assert fingerprint.screen.availWidth >= min_width assert fingerprint.screen.availWidth <= max_width From debe900a7a57dcd548889db8985bcad51dfbe4c3 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 14 Jan 2025 10:15:15 +0100 Subject: [PATCH 15/32] Further integration into our code. --- src/crawlee/browsers/_browser_pool.py | 18 ++++------- .../_playwright_browser_controller.py | 7 ++--- .../browsers/_playwright_browser_plugin.py | 14 ++++----- src/crawlee/fingerprint_suite/__init__.py | 2 ++ .../_browserforge_adapter.py | 16 +++++----- .../_fingerprint_generator.py | 30 ++++++++++++++++--- .../fingerprint_suite/_header_generator.py | 2 +- src/crawlee/fingerprint_suite/_types.py | 12 ++++---- .../_playwright/test_playwright_crawler.py | 25 ++++++++-------- 9 files changed, 69 insertions(+), 57 deletions(-) diff --git a/src/crawlee/browsers/_browser_pool.py b/src/crawlee/browsers/_browser_pool.py index 2f32659f88..4b33010fce 100644 --- a/src/crawlee/browsers/_browser_pool.py +++ b/src/crawlee/browsers/_browser_pool.py @@ -16,7 +16,8 @@ from crawlee._utils.recurring_task import RecurringTask from crawlee.browsers._base_browser_controller import BaseBrowserController from crawlee.browsers._playwright_browser_plugin import PlaywrightBrowserPlugin -from crawlee.browsers._types import BrowserType, CrawleePage +from crawlee.browsers._types import CrawleePage, BrowserType +from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -51,8 +52,6 @@ def __init__( browser_inactive_threshold: timedelta = timedelta(seconds=10), identify_inactive_browsers_interval: timedelta = timedelta(seconds=20), close_inactive_browsers_interval: timedelta = timedelta(seconds=30), - use_fingerprints: bool = True, - fingerprint_generator_options: dict[str, Any] | None = None, ) -> None: """A default constructor. @@ -68,8 +67,6 @@ def __init__( close_inactive_browsers_interval: The interval at which the pool checks for inactive browsers and closes them. The browser is considered as inactive if it has no active pages and has been idle for the specified period. - use_fingerprints: Inject generated fingerprints to page. - fingerprint_generator_options: Override generated fingerprints with these specific values, if possible. """ self._plugins = plugins or [PlaywrightBrowserPlugin()] self._operation_timeout = operation_timeout @@ -99,8 +96,6 @@ def __init__( # Flag to indicate the context state. self._active = False - self._use_fingerprints = use_fingerprints - self._fingerprint_generator_options = fingerprint_generator_options @classmethod def with_default_plugin( @@ -110,8 +105,7 @@ def with_default_plugin( browser_launch_options: Mapping[str, Any] | None = None, browser_new_context_options: Mapping[str, Any] | None = None, headless: bool | None = None, - use_fingerprints: bool = False, - fingerprint_generator_options: dict[str, Any] | None = None, + fingerprint_generator: AbstractFingerprintGenerator | None = None, **kwargs: Any, ) -> BrowserPool: """Create a new instance with a single `PlaywrightBrowserPlugin` configured with the provided options. @@ -125,8 +119,7 @@ def with_default_plugin( are provided directly to Playwright's `browser.new_context` method. For more details, refer to the Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context. headless: Whether to run the browser in headless mode. - use_fingerprints: Inject generated fingerprints to page. - fingerprint_generator_options: Override generated fingerprints with these specific values, if possible. + fingerprint_generator: kwargs: Additional arguments for default constructor. """ plugin_options: dict = defaultdict(dict) @@ -141,8 +134,7 @@ def with_default_plugin( plugin = PlaywrightBrowserPlugin( **plugin_options, - use_fingerprints=use_fingerprints, - fingerprint_generator_options=fingerprint_generator_options, + fingerprint_generator=fingerprint_generator, ) return cls(plugins=[plugin], **kwargs) diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index e2d183bf2e..fc669d97e9 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -57,7 +57,7 @@ def __init__( fingerprint_generator: An optional instance of implementation of `AbstractFingerprintGenerator` that is used to generate browser fingerprints together with headers. """ - if header_generator and fingerprint_generator: + if fingerprint_generator and header_generator is not self._DEFAULT_HEADER_GENERATOR: raise ValueError("Do not use `header_generator` and `fingerprint_generator` arguments at the same time. " "Choose only one. `fingerprint_generator` generates headers as well.") self._browser = browser @@ -69,7 +69,6 @@ def __init__( self._pages = list[Page]() self._last_page_opened_at = datetime.now(timezone.utc) - self._fingerprint_generator_options = fingerprint_generator_options @property @override @@ -129,7 +128,6 @@ async def new_page( if not self._browser_context: await self._set_browser_context( browser_new_context_options=browser_new_context_options, - fingerprint_options=self._fingerprint_generator_options, proxy_info=proxy_info, ) @@ -151,7 +149,6 @@ async def _set_browser_context( self, browser_new_context_options: Mapping[str, Any] | None = None, proxy_info: ProxyInfo | None = None, - fingerprint_options: dict | None = None, ) -> None: """Set browser context. @@ -172,7 +169,7 @@ async def _set_browser_context( if self._fingerprint_generator: self._browser_context = await AsyncNewContext( - browser=self._browser, fingerprint_options=(fingerprint_options or {}), **browser_new_context_options + browser=self._browser, fingerprint=self._fingerprint_generator.generate(), **browser_new_context_options ) return diff --git a/src/crawlee/browsers/_playwright_browser_plugin.py b/src/crawlee/browsers/_playwright_browser_plugin.py index ebafa29316..db1a717658 100644 --- a/src/crawlee/browsers/_playwright_browser_plugin.py +++ b/src/crawlee/browsers/_playwright_browser_plugin.py @@ -12,6 +12,7 @@ from crawlee._utils.docs import docs_group from crawlee.browsers._base_browser_plugin import BaseBrowserPlugin from crawlee.browsers._playwright_browser_controller import PlaywrightBrowserController +from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator if TYPE_CHECKING: from collections.abc import Mapping @@ -38,8 +39,7 @@ def __init__( browser_launch_options: Mapping[str, Any] | None = None, browser_new_context_options: Mapping[str, Any] | None = None, max_open_pages_per_browser: int = 20, - use_fingerprints: bool = True, - fingerprint_generator_options: dict[str, Any] | None = None, + fingerprint_generator: AbstractFingerprintGenerator | None = None, ) -> None: """A default constructor. @@ -53,9 +53,7 @@ def __init__( Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context. max_open_pages_per_browser: The maximum number of pages that can be opened in a single browser instance. Once reached, a new browser instance will be launched to handle the excess. - use_fingerprints: Inject generated fingerprints to page. - fingerprint_generator_options: Override generated fingerprints with these specific values, if possible. - + fingerprint_generator: Use fingerprint generator to generate headers with consistent browser fingerprints. """ self._browser_type = browser_type self._browser_launch_options = browser_launch_options or {} @@ -68,8 +66,7 @@ def __init__( # Flag to indicate the context state. self._active = False - self._use_fingerprints = use_fingerprints - self._fingerprint_generator_options = fingerprint_generator_options + self._fingerprint_generator = fingerprint_generator @property @override @@ -148,6 +145,5 @@ async def new_browser(self) -> PlaywrightBrowserController: return PlaywrightBrowserController( browser, max_open_pages_per_browser=self._max_open_pages_per_browser, - fingerprint_generator=self.fingerprint_generator, - fingerprint_generator_options=self._fingerprint_generator_options, + fingerprint_generator=self._fingerprint_generator, ) diff --git a/src/crawlee/fingerprint_suite/__init__.py b/src/crawlee/fingerprint_suite/__init__.py index e07b43caae..10d87a8f41 100644 --- a/src/crawlee/fingerprint_suite/__init__.py +++ b/src/crawlee/fingerprint_suite/__init__.py @@ -1 +1,3 @@ +from ._fingerprint_generator import AbstractFingerprintGenerator from ._header_generator import HeaderGenerator +from ._browserforge_adapter import FingerprintGenerator as DefaultFingerprintGenerator diff --git a/src/crawlee/fingerprint_suite/_browserforge_adapter.py b/src/crawlee/fingerprint_suite/_browserforge_adapter.py index d0a8014176..4f2f91f3c4 100644 --- a/src/crawlee/fingerprint_suite/_browserforge_adapter.py +++ b/src/crawlee/fingerprint_suite/_browserforge_adapter.py @@ -21,17 +21,19 @@ def generate(self) -> bf_Fingerprint: @staticmethod def _prepare_options(options: FingerprintGeneratorOptions) -> dict[any,any]: + """Adapt options for `browserforge.fingerprints.FingerprintGenerator`.""" raw_options = options.model_dump() bf_options = {} if raw_options["header_options"] is None: - header_options = dict() + bf_header_options = dict() else: - header_options = deepcopy(raw_options["header_options"]) - header_options["browser"] = header_options.pop("browsers", None) - header_options["os"] = header_options.pop("operating_systems", None) - header_options["device"] = header_options.pop("devices", None) - header_options["locale"] = header_options.pop("locales", None) + bf_header_options = deepcopy(raw_options["header_options"]) + bf_header_options["browser"] = bf_header_options.pop("browsers", None) + bf_header_options["os"] = bf_header_options.pop("operating_systems", None) + bf_header_options["device"] = bf_header_options.pop("devices", None) + bf_header_options["locale"] = bf_header_options.pop("locales", None) bf_options["mock_webrtc"] = raw_options["mock_web_rtc"] bf_options["screen"] = Screen(**(raw_options.get("screen") or {})) - return {**bf_options , **header_options} + bf_options["slim"] = raw_options["slim"] + return {**bf_options , **bf_header_options} diff --git a/src/crawlee/fingerprint_suite/_fingerprint_generator.py b/src/crawlee/fingerprint_suite/_fingerprint_generator.py index 896445b995..2c5815bdd3 100644 --- a/src/crawlee/fingerprint_suite/_fingerprint_generator.py +++ b/src/crawlee/fingerprint_suite/_fingerprint_generator.py @@ -1,11 +1,33 @@ -from abc import ABC +from abc import ABC, abstractmethod -from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions, Fingerprint +from browserforge.fingerprints import Fingerprint + +from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions class AbstractFingerprintGenerator(ABC): - @staticmethod - def generate(options: FingerprintGeneratorOptions | None = None) -> Fingerprint: + @abstractmethod + def __init__(self, fingerprint_generator_options: FingerprintGeneratorOptions, strict: bool = False): + """A default constructor. + + Args: + fingerprint_generator_options: Options used for generating fingerprints. + strict: If set to True, it will raise error if it is not possible to generate fingerprints based on the + fingerprint_generator_options. Default behavior is relaxation of fingerprint_generator_options until it + is possible to generate a fingerprint. + """ + ... + + @abstractmethod + def generate(self) -> Fingerprint: + """Method that is capable of generating fingerprints. + + If generator needs some settings or arguments, then it is expected to be done in `init`. + + This is experimental feature. + Return type is temporarily set to `Fingerprint` from `browserforge`. This is subject to change and most likely + it will change to custom `Fingerprint` class defined in this repo later. + """ ... diff --git a/src/crawlee/fingerprint_suite/_header_generator.py b/src/crawlee/fingerprint_suite/_header_generator.py index 50d753d3d9..d3e8ccacb4 100644 --- a/src/crawlee/fingerprint_suite/_header_generator.py +++ b/src/crawlee/fingerprint_suite/_header_generator.py @@ -18,7 +18,7 @@ ) if TYPE_CHECKING: - from crawlee.browsers._types import BrowserType + from crawlee.fingerprint_suite._types import BrowserType @docs_group('Classes') diff --git a/src/crawlee/fingerprint_suite/_types.py b/src/crawlee/fingerprint_suite/_types.py index ceaac47435..f247d55644 100644 --- a/src/crawlee/fingerprint_suite/_types.py +++ b/src/crawlee/fingerprint_suite/_types.py @@ -1,13 +1,13 @@ +from __future__ import annotations + from typing import Annotated, Literal from pydantic import BaseModel, Field -from crawlee.browsers._types import BrowserType - - SupportedOperatingSystems= Literal["windows", "macos", "linux", "android", "ios"] SupportedDevices = Literal["desktop", "mobile"] SupportedHttpVersion = Literal["1", "2"] +SupportedBrowserType = Literal['chromium', 'firefox', 'webkit', 'edge'] class ScreenFingerprint(BaseModel): """ @@ -193,7 +193,7 @@ class Config: populate_by_name = True class Browser: - name: BrowserType + name: SupportedBrowserType """Name of the browser.""" min_version: Annotated[float|None, Field(alias="minVersion")] = None """Minimum version of browser used.""" @@ -208,7 +208,7 @@ class Config: class HeaderGeneratorOptions(BaseModel): - browsers: list[BrowserType] | None = None + browsers: list[SupportedBrowserType] | None = None """List of BrowserSpecifications to generate the headers for.""" operating_systems: Annotated[list[SupportedOperatingSystems]| None, Field(alias="operatingSystems")] = None @@ -248,3 +248,5 @@ class FingerprintGeneratorOptions(BaseModel): class Config: extra = "forbid" populate_by_name = True + + diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py index 9fbed967dd..2d01bbf54c 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any from unittest import mock -from browserforge.fingerprints import Screen +from crawlee.fingerprint_suite import DefaultFingerprintGenerator from crawlee import Glob, Request from crawlee._types import EnqueueStrategy @@ -20,6 +20,7 @@ PW_CHROMIUM_HEADLESS_DEFAULT_USER_AGENT, PW_FIREFOX_HEADLESS_DEFAULT_USER_AGENT, ) +from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions, HeaderGeneratorOptions, ScreenOptions if TYPE_CHECKING: from yarl import URL @@ -197,18 +198,15 @@ async def test_custom_fingerprint_uses_generator_options(httpbin: URL) -> None: max_width = 600 min_height = 500 max_height = 1200 + + fingerprint_options = FingerprintGeneratorOptions( + header_options=HeaderGeneratorOptions(browsers=["firefox"], operating_systems=["android"]), + screen=ScreenOptions(min_width=min_width, max_width=max_width, min_height=min_height, max_height=max_height) + ) + crawler = PlaywrightCrawler( headless=True, - browser_pool_options={ - 'use_fingerprints': True, - 'fingerprint_generator_options': { - 'browser': 'edge', - 'os': 'android', - 'screen': Screen( - min_width=min_width, max_width=max_width, min_height=min_height, max_height=max_height - ), - }, - }, + browser_pool_options={'fingerprint_generator': DefaultFingerprintGenerator(fingerprint_options)} ) response_headers = dict[str, str]() @@ -232,7 +230,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: await crawler.run([Request.from_url(str(httpbin / 'get'))]) - assert 'EdgA' in fingerprints['window.navigator.userAgent'] + assert 'Firefox' in fingerprints['window.navigator.userAgent'] assert fingerprints['window.navigator.userAgentData']['platform'] == 'Android' assert min_width <= int(fingerprints['window.screen.width']) <= max_width assert min_height <= int(fingerprints['window.screen.height']) <= max_height @@ -241,7 +239,8 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: async def test_custom_fingerprint_matches_header_user_agent(httpbin: URL) -> None: """Test that generated fingerprint and header have matching user agent.""" - crawler = PlaywrightCrawler(headless=True, browser_pool_options={'use_fingerprints': True}) + crawler = PlaywrightCrawler(headless=True, + browser_pool_options={'fingerprint_generator': DefaultFingerprintGenerator()}) response_headers = dict[str, str]() fingerprints = dict[str, str]() From 3d8340c15ffbada57ffd8dfa7e4392938d92f193 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 14 Jan 2025 10:44:19 +0100 Subject: [PATCH 16/32] Finalize draft. Remove types unused in this change. Format, lint, type check. --- src/crawlee/browsers/_browser_pool.py | 8 +- .../_playwright_browser_controller.py | 11 +- .../browsers/_playwright_browser_plugin.py | 5 +- src/crawlee/fingerprint_suite/__init__.py | 2 +- .../_browserforge_adapter.py | 44 ++-- .../_fingerprint_generator.py | 13 +- .../fingerprint_suite/_header_generator.py | 6 +- src/crawlee/fingerprint_suite/_types.py | 235 ++---------------- .../_playwright/test_playwright_crawler.py | 15 +- tests/unit/fingerprint_suite/test_adapters.py | 52 ++-- 10 files changed, 101 insertions(+), 290 deletions(-) diff --git a/src/crawlee/browsers/_browser_pool.py b/src/crawlee/browsers/_browser_pool.py index 4b33010fce..74b35bb887 100644 --- a/src/crawlee/browsers/_browser_pool.py +++ b/src/crawlee/browsers/_browser_pool.py @@ -16,14 +16,14 @@ from crawlee._utils.recurring_task import RecurringTask from crawlee.browsers._base_browser_controller import BaseBrowserController from crawlee.browsers._playwright_browser_plugin import PlaywrightBrowserPlugin -from crawlee.browsers._types import CrawleePage, BrowserType -from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator +from crawlee.browsers._types import BrowserType, CrawleePage if TYPE_CHECKING: from collections.abc import Mapping, Sequence from types import TracebackType from crawlee.browsers._base_browser_plugin import BaseBrowserPlugin + from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator from crawlee.proxy_configuration import ProxyInfo logger = getLogger(__name__) @@ -96,7 +96,6 @@ def __init__( # Flag to indicate the context state. self._active = False - @classmethod def with_default_plugin( cls, @@ -119,7 +118,8 @@ def with_default_plugin( are provided directly to Playwright's `browser.new_context` method. For more details, refer to the Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context. headless: Whether to run the browser in headless mode. - fingerprint_generator: + fingerprint_generator: An optional instance of implementation of `AbstractFingerprintGenerator` that is used + to generate browser fingerprints together with consistent headers. kwargs: Additional arguments for default constructor. """ plugin_options: dict = defaultdict(dict) diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index fc669d97e9..1e65a78784 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -13,13 +13,13 @@ from crawlee.browsers._base_browser_controller import BaseBrowserController from crawlee.browsers._types import BrowserType from crawlee.fingerprint_suite import HeaderGenerator -from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator if TYPE_CHECKING: from collections.abc import Mapping from playwright.async_api import Browser + from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator from crawlee.proxy_configuration import ProxyInfo from logging import getLogger @@ -55,11 +55,13 @@ def __init__( requests made by the browser. By default, a predefined header generator is used. Set to `None` to disable automatic header modifications. fingerprint_generator: An optional instance of implementation of `AbstractFingerprintGenerator` that is used - to generate browser fingerprints together with headers. + to generate browser fingerprints together with consistent headers. """ if fingerprint_generator and header_generator is not self._DEFAULT_HEADER_GENERATOR: - raise ValueError("Do not use `header_generator` and `fingerprint_generator` arguments at the same time. " - "Choose only one. `fingerprint_generator` generates headers as well.") + raise ValueError( + 'Do not use `header_generator` and `fingerprint_generator` arguments at the same time. ' + 'Choose only one. `fingerprint_generator` generates headers as well.' + ) self._browser = browser self._max_open_pages_per_browser = max_open_pages_per_browser self._header_generator = header_generator @@ -69,7 +71,6 @@ def __init__( self._pages = list[Page]() self._last_page_opened_at = datetime.now(timezone.utc) - @property @override def pages(self) -> list[Page]: diff --git a/src/crawlee/browsers/_playwright_browser_plugin.py b/src/crawlee/browsers/_playwright_browser_plugin.py index db1a717658..91fc4782b6 100644 --- a/src/crawlee/browsers/_playwright_browser_plugin.py +++ b/src/crawlee/browsers/_playwright_browser_plugin.py @@ -12,13 +12,13 @@ from crawlee._utils.docs import docs_group from crawlee.browsers._base_browser_plugin import BaseBrowserPlugin from crawlee.browsers._playwright_browser_controller import PlaywrightBrowserController -from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator if TYPE_CHECKING: from collections.abc import Mapping from types import TracebackType from crawlee.browsers._types import BrowserType + from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator logger = getLogger(__name__) @@ -53,7 +53,8 @@ def __init__( Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context. max_open_pages_per_browser: The maximum number of pages that can be opened in a single browser instance. Once reached, a new browser instance will be launched to handle the excess. - fingerprint_generator: Use fingerprint generator to generate headers with consistent browser fingerprints. + fingerprint_generator: An optional instance of implementation of `AbstractFingerprintGenerator` that is used + to generate browser fingerprints together with consistent headers. """ self._browser_type = browser_type self._browser_launch_options = browser_launch_options or {} diff --git a/src/crawlee/fingerprint_suite/__init__.py b/src/crawlee/fingerprint_suite/__init__.py index 10d87a8f41..f2acb2b6aa 100644 --- a/src/crawlee/fingerprint_suite/__init__.py +++ b/src/crawlee/fingerprint_suite/__init__.py @@ -1,3 +1,3 @@ +from ._browserforge_adapter import FingerprintGenerator as DefaultFingerprintGenerator from ._fingerprint_generator import AbstractFingerprintGenerator from ._header_generator import HeaderGenerator -from ._browserforge_adapter import FingerprintGenerator as DefaultFingerprintGenerator diff --git a/src/crawlee/fingerprint_suite/_browserforge_adapter.py b/src/crawlee/fingerprint_suite/_browserforge_adapter.py index 4f2f91f3c4..8ec460d7d8 100644 --- a/src/crawlee/fingerprint_suite/_browserforge_adapter.py +++ b/src/crawlee/fingerprint_suite/_browserforge_adapter.py @@ -1,39 +1,41 @@ +from __future__ import annotations + from copy import deepcopy +from typing import Any -from browserforge.fingerprints import Fingerprint as bf_Fingerprint, FingerprintGenerator as bf_FingerprintGenerator, \ - Screen +from browserforge.fingerprints import Fingerprint as bf_Fingerprint +from browserforge.fingerprints import FingerprintGenerator as bf_FingerprintGenerator +from browserforge.fingerprints import Screen from typing_extensions import override from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions -class FingerprintGenerator(AbstractFingerprintGenerator): - def __init__(self, fingerprint_generator_options: FingerprintGeneratorOptions | None = None, strict: bool = False): - self._fingerprint_generator_options = FingerprintGenerator._prepare_options( - fingerprint_generator_options or FingerprintGeneratorOptions()) +class FingerprintGenerator(AbstractFingerprintGenerator): + def __init__(self, options: FingerprintGeneratorOptions | None = None, *, strict: bool = False) -> None: + self._options = FingerprintGenerator._prepare_options(options or FingerprintGeneratorOptions()) self._strict = strict @override def generate(self) -> bf_Fingerprint: - bf_fingerprint = bf_FingerprintGenerator().generate(**self._fingerprint_generator_options) - return bf_fingerprint + return bf_FingerprintGenerator().generate(**self._options) @staticmethod - def _prepare_options(options: FingerprintGeneratorOptions) -> dict[any,any]: + def _prepare_options(options: FingerprintGeneratorOptions) -> dict[Any, Any]: """Adapt options for `browserforge.fingerprints.FingerprintGenerator`.""" raw_options = options.model_dump() bf_options = {} - if raw_options["header_options"] is None: - bf_header_options = dict() + if raw_options['header_options'] is None: + bf_header_options = {} else: - bf_header_options = deepcopy(raw_options["header_options"]) - bf_header_options["browser"] = bf_header_options.pop("browsers", None) - bf_header_options["os"] = bf_header_options.pop("operating_systems", None) - bf_header_options["device"] = bf_header_options.pop("devices", None) - bf_header_options["locale"] = bf_header_options.pop("locales", None) - - bf_options["mock_webrtc"] = raw_options["mock_web_rtc"] - bf_options["screen"] = Screen(**(raw_options.get("screen") or {})) - bf_options["slim"] = raw_options["slim"] - return {**bf_options , **bf_header_options} + bf_header_options = deepcopy(raw_options['header_options']) + bf_header_options['browser'] = bf_header_options.pop('browsers', None) + bf_header_options['os'] = bf_header_options.pop('operating_systems', None) + bf_header_options['device'] = bf_header_options.pop('devices', None) + bf_header_options['locale'] = bf_header_options.pop('locales', None) + + bf_options['mock_webrtc'] = raw_options['mock_web_rtc'] + bf_options['screen'] = Screen(**(raw_options.get('screen') or {})) + bf_options['slim'] = raw_options['slim'] + return {**bf_options, **bf_header_options} diff --git a/src/crawlee/fingerprint_suite/_fingerprint_generator.py b/src/crawlee/fingerprint_suite/_fingerprint_generator.py index 2c5815bdd3..5cdac40c88 100644 --- a/src/crawlee/fingerprint_suite/_fingerprint_generator.py +++ b/src/crawlee/fingerprint_suite/_fingerprint_generator.py @@ -6,16 +6,14 @@ class AbstractFingerprintGenerator(ABC): - @abstractmethod - def __init__(self, fingerprint_generator_options: FingerprintGeneratorOptions, strict: bool = False): + def __init__(self, options: FingerprintGeneratorOptions, *, strict: bool = False) -> None: """A default constructor. Args: - fingerprint_generator_options: Options used for generating fingerprints. - strict: If set to True, it will raise error if it is not possible to generate fingerprints based on the - fingerprint_generator_options. Default behavior is relaxation of fingerprint_generator_options until it - is possible to generate a fingerprint. + options: Options used for generating fingerprints. + strict: If set to `True`, it will raise error if it is not possible to generate fingerprints based on the + `options`. Default behavior is relaxation of `options` until it is possible to generate a fingerprint. """ ... @@ -23,11 +21,8 @@ def __init__(self, fingerprint_generator_options: FingerprintGeneratorOptions, s def generate(self) -> Fingerprint: """Method that is capable of generating fingerprints. - If generator needs some settings or arguments, then it is expected to be done in `init`. - This is experimental feature. Return type is temporarily set to `Fingerprint` from `browserforge`. This is subject to change and most likely it will change to custom `Fingerprint` class defined in this repo later. """ ... - diff --git a/src/crawlee/fingerprint_suite/_header_generator.py b/src/crawlee/fingerprint_suite/_header_generator.py index d3e8ccacb4..5e821336c8 100644 --- a/src/crawlee/fingerprint_suite/_header_generator.py +++ b/src/crawlee/fingerprint_suite/_header_generator.py @@ -18,7 +18,7 @@ ) if TYPE_CHECKING: - from crawlee.fingerprint_suite._types import BrowserType + from crawlee.fingerprint_suite._types import SupportedBrowserType @docs_group('Classes') @@ -45,7 +45,7 @@ def get_random_user_agent_header(self) -> HttpHeaders: def get_user_agent_header( self, *, - browser_type: BrowserType = 'chromium', + browser_type: SupportedBrowserType = 'chromium', ) -> HttpHeaders: """Get the User-Agent header based on the browser type.""" headers = dict[str, str]() @@ -67,7 +67,7 @@ def get_user_agent_header( def get_sec_ch_ua_headers( self, *, - browser_type: BrowserType = 'chromium', + browser_type: SupportedBrowserType = 'chromium', ) -> HttpHeaders: """Get the Sec-Ch-Ua headers based on the browser type.""" headers = dict[str, str]() diff --git a/src/crawlee/fingerprint_suite/_types.py b/src/crawlee/fingerprint_suite/_types.py index f247d55644..8617f76601 100644 --- a/src/crawlee/fingerprint_suite/_types.py +++ b/src/crawlee/fingerprint_suite/_types.py @@ -4,249 +4,68 @@ from pydantic import BaseModel, Field -SupportedOperatingSystems= Literal["windows", "macos", "linux", "android", "ios"] -SupportedDevices = Literal["desktop", "mobile"] -SupportedHttpVersion = Literal["1", "2"] +SupportedOperatingSystems = Literal['windows', 'macos', 'linux', 'android', 'ios'] +SupportedDevices = Literal['desktop', 'mobile'] +SupportedHttpVersion = Literal['1', '2'] SupportedBrowserType = Literal['chromium', 'firefox', 'webkit', 'edge'] -class ScreenFingerprint(BaseModel): - """ - Collection of various attributes from following sources: - - https://developer.mozilla.org/en-US/docs/Web/API/Screen - https://developer.mozilla.org/en-US/docs/Web/API/ScreenDetailed - https://developer.mozilla.org/en-US/docs/Web/API/Window - https://developer.mozilla.org/en-US/docs/Web/API/Element - """ - - avail_height: Annotated[float, Field(alias="availHeight")] - """https://developer.mozilla.org/en-US/docs/Web/API/Screen/availHeight""" - - avail_width: Annotated[float, Field(alias="availWidth")] - """https://developer.mozilla.org/en-US/docs/Web/API/Screen/availWidth""" - - avail_top: Annotated[float, Field(alias="availTop")] - """https://developer.mozilla.org/en-US/docs/Web/API/ScreenDetailed/availTop""" - - avail_left: Annotated[float, Field(alias="availLeft")] - """https://developer.mozilla.org/en-US/docs/Web/API/ScreenDetailed/left""" - - color_depth: Annotated[float, Field(alias="colorDepth")] - """https://developer.mozilla.org/en-US/docs/Web/API/Screen/colorDepth""" - - height: float - """https://developer.mozilla.org/en-US/docs/Web/API/Screen/height""" - - pixel_depth: Annotated[float, Field(alias="pixelDepth")] - """https://developer.mozilla.org/en-US/docs/Web/API/Screen/pixelDepth""" - - width: float - """https://developer.mozilla.org/en-US/docs/Web/API/Screen/width""" - - device_pixel_ratio: Annotated[float, Field(alias="devicePixelRatio")] - """https://developer.mozilla.org/en-US/docs/Web/API/Window/devicePixelRatio""" - - page_x_offset: Annotated[float, Field(alias="pageXOffset")] - """https://developer.mozilla.org/en-US/docs/Web/API/Window/scrollX""" - - page_y_offset: Annotated[float, Field(alias="pageYOffset")] - """https://developer.mozilla.org/en-US/docs/Web/API/Window/scrollY""" - - inner_height: Annotated[float, Field(alias="innerHeight")] - """https://developer.mozilla.org/en-US/docs/Web/API/Window/innerHeight""" - - outer_height: Annotated[float, Field(alias="outerHeight")] - """https://developer.mozilla.org/en-US/docs/Web/API/Window/outerHeight""" - - outer_width: Annotated[float, Field(alias="outerWidth")] - """https://developer.mozilla.org/en-US/docs/Web/API/Window/outerWidth""" - - inner_width: Annotated[float, Field(alias="innerWidth")] - """https://developer.mozilla.org/en-US/docs/Web/API/Window/innerWidth""" - - screen_x: Annotated[float, Field(alias="screenX")] # Why screenY not present in JS? - """https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent/screenX""" - - client_width: Annotated[float, Field(alias="clientWidth")] - """https://developer.mozilla.org/en-US/docs/Web/API/Element/clientWidth""" - - client_height: Annotated[float, Field(alias="clientHeight")] - """https://developer.mozilla.org/en-US/docs/Web/API/Element/clientHeight""" - - has_hdr: Annotated[bool, Field(alias="hasHDR")] # What is this? A placeholder? - - -class NavigatorFingerprint(BaseModel): - """ - Collection of various attributes from following sources: - - https://developer.mozilla.org/en-US/docs/Web/API/Navigator - """ - - user_agent: Annotated[str, Field(alias="userAgent")] - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/userAgent""" - - - - user_agent_data: Annotated[dict[str, str | list[dict[str, str]] | bool], Field(alias="userAgentData")] - """https://developer.mozilla.org/en-US/docs/Web/API/WorkerNavigator/userAgentData - - In JS this is just userAgentData: Record, but it can contain more stuff like - - mobile = False - 'fullVersionList' = [ - {'brand': 'Google Chrome', 'version': '131.0.6778.86'}, - {'brand': 'Chromium', 'version': '131.0.6778.86'}, - {'brand': 'Not_A Brand', 'version': '24.0.0.0'} - ] - so more generic type should be allowed - - """ - - do_not_track: Annotated[str, Field(alias="doNotTrack")] - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/doNotTrack""" - - app_code_name: Annotated[str, Field(alias="appCodeName")] - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/appCodeName""" - - app_name: Annotated[str, Field(alias="appName")] - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/appName""" - - app_version: Annotated[str, Field(alias="appVersion")] - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/appVersion""" - - oscpu: str - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/oscpu""" - - webdriver: str - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/webdriver""" - - language: str - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/language""" - - languages: list[str] - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/languages""" - - platform: str - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/platform""" - - device_memory: Annotated[float|None, Field(alias="deviceMemory")] = None # Firefox does not have deviceMemory available - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/deviceMemory""" - - hardware_concurrency: Annotated[float, Field(alias="hardwareConcurrency")] - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/hardwareConcurrency""" - - product: str - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/product""" - - product_sub: Annotated[str, Field(alias="productSub")] - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/productSub""" - - vendor: str - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/vendor""" - - vendor_sub: Annotated[str, Field(alias="vendorSub")] - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/vendorSub""" - - max_touch_points: Annotated[float|None, Field(alias="maxTouchPoints")] = None - """https://developer.mozilla.org/en-US/docs/Web/API/Navigator/maxTouchPoints""" - - extra_properties: Annotated[dict[str, str], Field(alias="extraProperties")] = [] - - -class VideoCard(BaseModel): - rendered: str - vendor: str - -class Fingerprint(BaseModel): - """Represents specific browser fingerprint collection. - - Collection of browser settings, attributes and capabilities is commonly referred to as a browser fingerprint. - Such fingerprint can be used to collect various information or track or group users, or thus also detect web - crawlers by inspecting suspiciously looking browser fingerprints. - This object contains attributes that are sub set of such specific fingerprint. - See `https://docs.apify.com/academy/anti-scraping/mitigation/generating-fingerprints` TODO: Update guide with Python example. - """ - - screen: ScreenFingerprint - navigator: NavigatorFingerprint - video_codecs: Annotated[dict[str, str], Field(alias="videoCodecs")] = None - audio_codecs: Annotated[dict[str, str], Field(alias="audioCodecs")] = None - plugins_data: Annotated[dict[str, str], Field(alias="pluginsData")] = None - battery: dict[str, str] | None = None - video_card: VideoCard - multimedia_devices: Annotated[list[str], Field(alias="multimediaDevices")] - fonts: list[str] = [] - mock_web_rtc: Annotated[bool, Field(alias="mockWebRTC")] - slim: Annotated[bool|None, Field(alias="slim")]=None - class ScreenOptions(BaseModel): - """Defines the screen dimensions of the generated fingerprint.""" - min_width: Annotated[float|None, Field(alias="minWidth")] = None - max_width: Annotated[float | None, Field(alias="maxWidth")] = None - min_height: Annotated[float | None, Field(alias="minHeight")] = None - max_height: Annotated[float | None, Field(alias="maxHeight")] = None + """Defines the screen constrains for the fingerprint generator .""" - class Config: - extra = "forbid" - populate_by_name = True - -class Browser: - name: SupportedBrowserType - """Name of the browser.""" - min_version: Annotated[float|None, Field(alias="minVersion")] = None - """Minimum version of browser used.""" - max_version: Annotated[float | None, Field(alias="maxVersion")] = None - """Maximum version of browser used.""" - http_version: Annotated[SupportedHttpVersion | None, Field(alias="httpVersion")] = None - """HTTP version to be used for header generation (the headers differ depending on the version).""" + min_width: Annotated[float | None, Field(alias='minWidth')] = None + max_width: Annotated[float | None, Field(alias='maxWidth')] = None + min_height: Annotated[float | None, Field(alias='minHeight')] = None + max_height: Annotated[float | None, Field(alias='maxHeight')] = None class Config: - extra = "forbid" + extra = 'forbid' populate_by_name = True class HeaderGeneratorOptions(BaseModel): + """Collection of header related attributes that can be used by the fingerprint generator.""" + browsers: list[SupportedBrowserType] | None = None """List of BrowserSpecifications to generate the headers for.""" - operating_systems: Annotated[list[SupportedOperatingSystems]| None, Field(alias="operatingSystems")] = None + operating_systems: Annotated[list[SupportedOperatingSystems] | None, Field(alias='operatingSystems')] = None """List of operating systems to generate the headers for.""" - devices: list[SupportedDevices]| None = None + devices: list[SupportedDevices] | None = None """List of devices to generate the headers for.""" - locales: list[str]| None = None + locales: list[str] | None = None """List of at most 10 languages to include in the [Accept-Language] (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language) request header in the language format accepted by that header, for example `en`, `en-US` or `de`.""" - http_version: Annotated[SupportedHttpVersion| None, Field(alias="httpVersion")]= None + http_version: Annotated[SupportedHttpVersion | None, Field(alias='httpVersion')] = None """HTTP version to be used for header generation (the headers differ depending on the version).""" - strict: bool| None = None + strict: bool | None = None """If true, the generator will throw an error if it cannot generate headers based on the input.""" class Config: - extra = "forbid" + extra = 'forbid' populate_by_name = True + class FingerprintGeneratorOptions(BaseModel): - """All generator options are optional. If any value si s not specified, then a default value will be used. + """Collection of fingerprint related attributes that can be used by the fingerprint generator. - Default values are implementation detail of used fingerprint generator. - Specific default values should not be relied upon. Use explicit values if it matters for your use case. - """ + All generator options are optional. If any value is not specified, then `None` is set in the options. + Default values for options set to `None` are implementation detail of used fingerprint generator. + Specific default values should not be relied upon. Use explicit values if it matters for your use case. + """ header_options: HeaderGeneratorOptions | None = None screen: ScreenOptions | None = None - mock_web_rtc: Annotated[bool, Field(alias="mockWebRTC")] = None + mock_web_rtc: Annotated[bool | None, Field(alias='mockWebRTC')] = None """Whether to mock WebRTC when injecting the fingerprint.""" - slim: Annotated[bool | None, Field(alias="slim")] = None + slim: Annotated[bool | None, Field(alias='slim')] = None """Disables performance-heavy evasions when injecting the fingerprint.""" + class Config: - extra = "forbid" + extra = 'forbid' populate_by_name = True - - diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py index 2d01bbf54c..69009ee308 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py @@ -8,11 +8,10 @@ from typing import TYPE_CHECKING, Any from unittest import mock -from crawlee.fingerprint_suite import DefaultFingerprintGenerator - from crawlee import Glob, Request from crawlee._types import EnqueueStrategy from crawlee.crawlers import PlaywrightCrawler +from crawlee.fingerprint_suite import DefaultFingerprintGenerator from crawlee.fingerprint_suite._consts import ( PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA, PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_MOBILE, @@ -200,13 +199,12 @@ async def test_custom_fingerprint_uses_generator_options(httpbin: URL) -> None: max_height = 1200 fingerprint_options = FingerprintGeneratorOptions( - header_options=HeaderGeneratorOptions(browsers=["firefox"], operating_systems=["android"]), - screen=ScreenOptions(min_width=min_width, max_width=max_width, min_height=min_height, max_height=max_height) + header_options=HeaderGeneratorOptions(browsers=['firefox'], operating_systems=['android']), + screen=ScreenOptions(min_width=min_width, max_width=max_width, min_height=min_height, max_height=max_height), ) crawler = PlaywrightCrawler( - headless=True, - browser_pool_options={'fingerprint_generator': DefaultFingerprintGenerator(fingerprint_options)} + headless=True, browser_pool_options={'fingerprint_generator': DefaultFingerprintGenerator(fingerprint_options)} ) response_headers = dict[str, str]() @@ -239,8 +237,9 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: async def test_custom_fingerprint_matches_header_user_agent(httpbin: URL) -> None: """Test that generated fingerprint and header have matching user agent.""" - crawler = PlaywrightCrawler(headless=True, - browser_pool_options={'fingerprint_generator': DefaultFingerprintGenerator()}) + crawler = PlaywrightCrawler( + headless=True, browser_pool_options={'fingerprint_generator': DefaultFingerprintGenerator()} + ) response_headers = dict[str, str]() fingerprints = dict[str, str]() diff --git a/tests/unit/fingerprint_suite/test_adapters.py b/tests/unit/fingerprint_suite/test_adapters.py index fba65298d6..793a223cdb 100644 --- a/tests/unit/fingerprint_suite/test_adapters.py +++ b/tests/unit/fingerprint_suite/test_adapters.py @@ -1,27 +1,23 @@ -import pytest - from crawlee.fingerprint_suite._browserforge_adapter import FingerprintGenerator -from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator -from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions, ScreenOptions, HeaderGeneratorOptions +from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions, HeaderGeneratorOptions, ScreenOptions -def test_fingerprint_generator_has_default(): +def test_fingerprint_generator_has_default() -> None: """Test that header generator can work without any options.""" assert FingerprintGenerator().generate() - -def test_fingerprint_generator_some_options(): +def test_fingerprint_generator_some_options() -> None: """Test that header generator can work with only some options.""" - options = FingerprintGeneratorOptions(screen=ScreenOptions(min_width = 500), mockWebRTC=True) + options = FingerprintGeneratorOptions(screen=ScreenOptions(min_width=500), mock_web_rtc=True) - fingerprint = FingerprintGenerator(fingerprint_generator_options=options).generate() + fingerprint = FingerprintGenerator(options=options).generate() - assert fingerprint.mockWebRTC == True + assert fingerprint.mockWebRTC is True assert fingerprint.screen.availWidth >= 500 -def test_fingerprint_generator_all_options(): +def test_fingerprint_generator_all_options() -> None: """Test that header generator can work with all the options. Some most basic checks of fingerprint. Fingerprint generation option might have no effect if there is no fingerprint sample present in collected data. @@ -33,34 +29,32 @@ def test_fingerprint_generator_all_options(): options = FingerprintGeneratorOptions( screen=ScreenOptions( - min_width = min_width, + min_width=min_width, max_width=max_width, min_height=min_height, max_height=max_height, ), - mockWebRTC=True, + mock_web_rtc=True, slim=False, - header_options = HeaderGeneratorOptions( - strict = True, - browsers = ["firefox"], - operating_systems = ["windows"], - devices = ["mobile"], - locales = ["en"], # This does not seem to generate any other values than `en-US` regardless of the input - http_version = "2", # Http1 does not work in browserforge - - ) + header_options=HeaderGeneratorOptions( + strict=True, + browsers=['firefox'], + operating_systems=['windows'], + devices=['mobile'], + locales=['en'], # This does not seem to generate any other values than `en-US` regardless of the input + http_version='2', # Http1 does not work in browserforge + ), ) - fingerprint = FingerprintGenerator(fingerprint_generator_options=options).generate() + fingerprint = FingerprintGenerator(options=options).generate() assert fingerprint.screen.availWidth >= min_width assert fingerprint.screen.availWidth <= max_width assert fingerprint.screen.availHeight >= min_height assert fingerprint.screen.availHeight <= max_height - assert fingerprint.mockWebRTC == True - assert fingerprint.slim == False - assert "Firefox" in fingerprint.navigator.userAgent - assert "Win" in fingerprint.navigator.oscpu - assert "en-US" in fingerprint.navigator.languages - + assert fingerprint.mockWebRTC is True + assert fingerprint.slim is False + assert 'Firefox' in fingerprint.navigator.userAgent + assert 'Win' in fingerprint.navigator.oscpu + assert 'en-US' in fingerprint.navigator.languages From 3d9b170e4335b980bac31bff0318a48afb918d36 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 14 Jan 2025 11:07:54 +0100 Subject: [PATCH 17/32] Set fiongerprint generator as top level argument to pw crawler --- .../_playwright_browser_controller.py | 24 ++++++++----------- .../_playwright/_playwright_crawler.py | 21 +++++++++++----- .../_playwright/test_playwright_crawler.py | 8 ++----- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index 1e65a78784..b1136b2f21 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -127,7 +127,7 @@ async def new_page( ValueError: If the browser has reached the maximum number of open pages. """ if not self._browser_context: - await self._set_browser_context( + self._browser_context = await self._create_browser_context( browser_new_context_options=browser_new_context_options, proxy_info=proxy_info, ) @@ -135,7 +135,7 @@ async def new_page( if not self.has_free_capacity: raise ValueError('Cannot open more pages in this browser.') - page = await self._get_browser_context().new_page() + page = await self._browser_context.new_page() # Handle page close event page.on(event='close', f=self._on_page_close) @@ -146,15 +146,17 @@ async def new_page( return page - async def _set_browser_context( + async def _create_browser_context( self, browser_new_context_options: Mapping[str, Any] | None = None, proxy_info: ProxyInfo | None = None, - ) -> None: + ) -> BrowserContext: """Set browser context. - Create context using `browserforge` if `self._fingerprint_generator` exists. - Create context without fingerprints with headers based on header generator if available. + Create context with fingerprints and headers using with `self._fingerprint_generator` if available. + Create context without fingerprints, but with headers based on `self._header_generator` if available. + Create context without headers and without fingerprints if neither `self._header_generator` nor + `self._fingerprint_generator` is available. """ browser_new_context_options = dict(browser_new_context_options) if browser_new_context_options else {} @@ -169,10 +171,9 @@ async def _set_browser_context( ) if self._fingerprint_generator: - self._browser_context = await AsyncNewContext( + return await AsyncNewContext( browser=self._browser, fingerprint=self._fingerprint_generator.generate(), **browser_new_context_options ) - return if self._header_generator: common_headers = self._header_generator.get_common_headers() @@ -187,12 +188,7 @@ async def _set_browser_context( 'extra_http_headers', extra_http_headers ) - self._browser_context = await self._browser.new_context(**browser_new_context_options) - - def _get_browser_context(self) -> BrowserContext: - if not self._browser_context: - raise RuntimeError('Browser context was not set yet.') - return self._browser_context + return await self._browser.new_context(**browser_new_context_options) @override async def close(self, *, force: bool = False) -> None: diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py index d3c0aa4099..57a9c964bb 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py +++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py @@ -25,6 +25,7 @@ from crawlee._types import BasicCrawlingContext, EnqueueLinksKwargs from crawlee.browsers._types import BrowserType + from crawlee.fingerprint_suite import AbstractFingerprintGenerator @docs_group('Classes') @@ -72,10 +73,10 @@ def __init__( self, *, browser_pool: BrowserPool | None = None, - browser_pool_options: Mapping[str, Any] | None = None, browser_type: BrowserType | None = None, browser_launch_options: Mapping[str, Any] | None = None, browser_new_context_options: Mapping[str, Any] | None = None, + fingerprint_generator: AbstractFingerprintGenerator | None = None, headless: bool | None = None, **kwargs: Unpack[BasicCrawlerOptions[PlaywrightCrawlingContext]], ) -> None: @@ -83,7 +84,6 @@ def __init__( Args: browser_pool: A `BrowserPool` instance to be used for launching the browsers and getting pages. - browser_pool_options: Arguments passed to `BrowserPool`. browser_type: The type of browser to launch ('chromium', 'firefox', or 'webkit'). This option should not be used if `browser_pool` is provided. browser_launch_options: Keyword arguments to pass to the browser launch method. These options are provided @@ -94,6 +94,8 @@ def __init__( are provided directly to Playwright's `browser.new_context` method. For more details, refer to the [Playwright documentation](https://playwright.dev/python/docs/api/class-browser#browser-new-context). This option should not be used if `browser_pool` is provided. + fingerprint_generator: An optional instance of implementation of `AbstractFingerprintGenerator` that is used + to generate browser fingerprints together with consistent headers. headless: Whether to run the browser in headless mode. This option should not be used if `browser_pool` is provided. kwargs: Additional keyword arguments to pass to the underlying `BasicCrawler`. @@ -102,11 +104,18 @@ def __init__( # Raise an exception if browser_pool is provided together with other browser-related arguments. if any( param is not None - for param in (headless, browser_type, browser_launch_options, browser_new_context_options) + for param in ( + headless, + browser_type, + browser_launch_options, + browser_new_context_options, + fingerprint_generator, + ) ): raise ValueError( - 'You cannot provide `headless`, `browser_type`, `browser_launch_options` or ' - '`browser_new_context_options` arguments when `browser_pool` is provided.' + 'You cannot provide `headless`, `browser_type`, `browser_launch_options`, ' + '`browser_new_context_options` or `fingerprint_generator` arguments when `browser_pool` ' + 'is provided.' ) # If browser_pool is not provided, create a new instance of BrowserPool with specified arguments. @@ -116,7 +125,7 @@ def __init__( browser_type=browser_type, browser_launch_options=browser_launch_options, browser_new_context_options=browser_new_context_options, - **(browser_pool_options or {}), + fingerprint_generator=fingerprint_generator, ) self._browser_pool = browser_pool diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py index 69009ee308..535f65d058 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py @@ -203,9 +203,7 @@ async def test_custom_fingerprint_uses_generator_options(httpbin: URL) -> None: screen=ScreenOptions(min_width=min_width, max_width=max_width, min_height=min_height, max_height=max_height), ) - crawler = PlaywrightCrawler( - headless=True, browser_pool_options={'fingerprint_generator': DefaultFingerprintGenerator(fingerprint_options)} - ) + crawler = PlaywrightCrawler(headless=True, fingerprint_generator=DefaultFingerprintGenerator(fingerprint_options)) response_headers = dict[str, str]() fingerprints = dict[str, Any]() @@ -237,9 +235,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: async def test_custom_fingerprint_matches_header_user_agent(httpbin: URL) -> None: """Test that generated fingerprint and header have matching user agent.""" - crawler = PlaywrightCrawler( - headless=True, browser_pool_options={'fingerprint_generator': DefaultFingerprintGenerator()} - ) + crawler = PlaywrightCrawler(headless=True, fingerprint_generator=DefaultFingerprintGenerator()) response_headers = dict[str, str]() fingerprints = dict[str, str]() From 25aa4e2fea20622c72e714748e5b1aebe87b688e Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 14 Jan 2025 11:11:23 +0100 Subject: [PATCH 18/32] Revert unnecessary change to function doc string. --- .../_playwright_browser_controller.py | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index b1136b2f21..d4347826a3 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -146,12 +146,33 @@ async def new_page( return page + @override + async def close(self, *, force: bool = False) -> None: + """Close the browser. + + Args: + force: Whether to force close all open pages before closing the browser. + + Raises: + ValueError: If there are still open pages when trying to close the browser. + """ + if self.pages_count > 0 and not force: + raise ValueError('Cannot close the browser while there are open pages.') + + if self._browser_context: + await self._browser_context.close() + await self._browser.close() + + def _on_page_close(self, page: Page) -> None: + """Handle actions after a page is closed.""" + self._pages.remove(page) + async def _create_browser_context( self, browser_new_context_options: Mapping[str, Any] | None = None, proxy_info: ProxyInfo | None = None, ) -> BrowserContext: - """Set browser context. + """Create a new browser context with the specified proxy settings. Create context with fingerprints and headers using with `self._fingerprint_generator` if available. Create context without fingerprints, but with headers based on `self._header_generator` if available. @@ -161,7 +182,7 @@ async def _create_browser_context( browser_new_context_options = dict(browser_new_context_options) if browser_new_context_options else {} if proxy_info: - if browser_new_context_options['proxy']: + if browser_new_context_options.get('proxy'): logger.warning("browser_new_context_options['proxy'] overriden by explicit `proxy_info` argument.") browser_new_context_options['proxy'] = ProxySettings( @@ -189,24 +210,3 @@ async def _create_browser_context( ) return await self._browser.new_context(**browser_new_context_options) - - @override - async def close(self, *, force: bool = False) -> None: - """Close the browser. - - Args: - force: Whether to force close all open pages before closing the browser. - - Raises: - ValueError: If there are still open pages when trying to close the browser. - """ - if self.pages_count > 0 and not force: - raise ValueError('Cannot close the browser while there are open pages.') - - if self._browser_context: - await self._browser_context.close() - await self._browser.close() - - def _on_page_close(self, page: Page) -> None: - """Handle actions after a page is closed.""" - self._pages.remove(page) From 5e46b78eddfa6ca18f91cd0efb5d52e5d4568b4c Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 14 Jan 2025 11:26:45 +0100 Subject: [PATCH 19/32] Make test adapter-generic. (Hint about browserforge being just implementation detail and not core functionality.) --- .../fingerprint_suite/_fingerprint_generator.py | 10 +++++++--- tests/unit/fingerprint_suite/test_adapters.py | 16 +++++++++++----- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/crawlee/fingerprint_suite/_fingerprint_generator.py b/src/crawlee/fingerprint_suite/_fingerprint_generator.py index 5cdac40c88..775c06b86d 100644 --- a/src/crawlee/fingerprint_suite/_fingerprint_generator.py +++ b/src/crawlee/fingerprint_suite/_fingerprint_generator.py @@ -1,13 +1,17 @@ +from __future__ import annotations + from abc import ABC, abstractmethod +from typing import TYPE_CHECKING -from browserforge.fingerprints import Fingerprint +if TYPE_CHECKING: + from browserforge.fingerprints import Fingerprint -from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions + from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions class AbstractFingerprintGenerator(ABC): @abstractmethod - def __init__(self, options: FingerprintGeneratorOptions, *, strict: bool = False) -> None: + def __init__(self, options: FingerprintGeneratorOptions | None = None, *, strict: bool = False) -> None: """A default constructor. Args: diff --git a/tests/unit/fingerprint_suite/test_adapters.py b/tests/unit/fingerprint_suite/test_adapters.py index 793a223cdb..36a7131762 100644 --- a/tests/unit/fingerprint_suite/test_adapters.py +++ b/tests/unit/fingerprint_suite/test_adapters.py @@ -1,13 +1,18 @@ -from crawlee.fingerprint_suite._browserforge_adapter import FingerprintGenerator +import pytest + +from crawlee.fingerprint_suite import AbstractFingerprintGenerator +from crawlee.fingerprint_suite._browserforge_adapter import FingerprintGenerator as BrowserForgeAdapter from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions, HeaderGeneratorOptions, ScreenOptions -def test_fingerprint_generator_has_default() -> None: +@pytest.mark.parametrize('FingerprintGenerator', [pytest.param(BrowserForgeAdapter, id='browserforge')]) +def test_fingerprint_generator_has_default(FingerprintGenerator: type[AbstractFingerprintGenerator]) -> None: # noqa:N803 # Test is more readable if argument(class) is PascalCase """Test that header generator can work without any options.""" assert FingerprintGenerator().generate() -def test_fingerprint_generator_some_options() -> None: +@pytest.mark.parametrize('FingerprintGenerator', [pytest.param(BrowserForgeAdapter, id='browserforge')]) +def test_fingerprint_generator_some_options(FingerprintGenerator: type[AbstractFingerprintGenerator]) -> None: # noqa:N803 # Test is more readable if argument(class) is PascalCase """Test that header generator can work with only some options.""" options = FingerprintGeneratorOptions(screen=ScreenOptions(min_width=500), mock_web_rtc=True) @@ -17,7 +22,8 @@ def test_fingerprint_generator_some_options() -> None: assert fingerprint.screen.availWidth >= 500 -def test_fingerprint_generator_all_options() -> None: +@pytest.mark.parametrize('FingerprintGenerator', [pytest.param(BrowserForgeAdapter, id='browserforge')]) +def test_fingerprint_generator_all_options(FingerprintGenerator: type[AbstractFingerprintGenerator]) -> None: # noqa:N803 # Test is more readable if argument(class) is PascalCase """Test that header generator can work with all the options. Some most basic checks of fingerprint. Fingerprint generation option might have no effect if there is no fingerprint sample present in collected data. @@ -41,7 +47,7 @@ def test_fingerprint_generator_all_options() -> None: browsers=['firefox'], operating_systems=['windows'], devices=['mobile'], - locales=['en'], # This does not seem to generate any other values than `en-US` regardless of the input + locales=['en'], # Does not generate any other values than `en-US` regardless of the input in browserforge http_version='2', # Http1 does not work in browserforge ), ) From 69b697497f2080ab549e32a2667e597856021fc6 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 14 Jan 2025 11:36:57 +0100 Subject: [PATCH 20/32] Add types to __init__ if fingerprint_suite --- src/crawlee/fingerprint_suite/__init__.py | 1 + .../_playwright/test_playwright_crawler.py | 8 ++++++-- tests/unit/fingerprint_suite/test_adapters.py | 16 ++++++++++------ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/crawlee/fingerprint_suite/__init__.py b/src/crawlee/fingerprint_suite/__init__.py index f2acb2b6aa..1b93804055 100644 --- a/src/crawlee/fingerprint_suite/__init__.py +++ b/src/crawlee/fingerprint_suite/__init__.py @@ -1,3 +1,4 @@ from ._browserforge_adapter import FingerprintGenerator as DefaultFingerprintGenerator from ._fingerprint_generator import AbstractFingerprintGenerator from ._header_generator import HeaderGenerator +from ._types import FingerprintGeneratorOptions, HeaderGeneratorOptions, ScreenOptions diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py index 535f65d058..15db5c8300 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py @@ -11,7 +11,12 @@ from crawlee import Glob, Request from crawlee._types import EnqueueStrategy from crawlee.crawlers import PlaywrightCrawler -from crawlee.fingerprint_suite import DefaultFingerprintGenerator +from crawlee.fingerprint_suite import ( + DefaultFingerprintGenerator, + FingerprintGeneratorOptions, + HeaderGeneratorOptions, + ScreenOptions, +) from crawlee.fingerprint_suite._consts import ( PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA, PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_MOBILE, @@ -19,7 +24,6 @@ PW_CHROMIUM_HEADLESS_DEFAULT_USER_AGENT, PW_FIREFOX_HEADLESS_DEFAULT_USER_AGENT, ) -from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions, HeaderGeneratorOptions, ScreenOptions if TYPE_CHECKING: from yarl import URL diff --git a/tests/unit/fingerprint_suite/test_adapters.py b/tests/unit/fingerprint_suite/test_adapters.py index 36a7131762..f62a51d5ab 100644 --- a/tests/unit/fingerprint_suite/test_adapters.py +++ b/tests/unit/fingerprint_suite/test_adapters.py @@ -1,17 +1,21 @@ import pytest -from crawlee.fingerprint_suite import AbstractFingerprintGenerator -from crawlee.fingerprint_suite._browserforge_adapter import FingerprintGenerator as BrowserForgeAdapter -from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions, HeaderGeneratorOptions, ScreenOptions +from crawlee.fingerprint_suite import ( + AbstractFingerprintGenerator, + DefaultFingerprintGenerator, + FingerprintGeneratorOptions, + HeaderGeneratorOptions, + ScreenOptions, +) -@pytest.mark.parametrize('FingerprintGenerator', [pytest.param(BrowserForgeAdapter, id='browserforge')]) +@pytest.mark.parametrize('FingerprintGenerator', [pytest.param(DefaultFingerprintGenerator, id='browserforge')]) def test_fingerprint_generator_has_default(FingerprintGenerator: type[AbstractFingerprintGenerator]) -> None: # noqa:N803 # Test is more readable if argument(class) is PascalCase """Test that header generator can work without any options.""" assert FingerprintGenerator().generate() -@pytest.mark.parametrize('FingerprintGenerator', [pytest.param(BrowserForgeAdapter, id='browserforge')]) +@pytest.mark.parametrize('FingerprintGenerator', [pytest.param(DefaultFingerprintGenerator, id='browserforge')]) def test_fingerprint_generator_some_options(FingerprintGenerator: type[AbstractFingerprintGenerator]) -> None: # noqa:N803 # Test is more readable if argument(class) is PascalCase """Test that header generator can work with only some options.""" options = FingerprintGeneratorOptions(screen=ScreenOptions(min_width=500), mock_web_rtc=True) @@ -22,7 +26,7 @@ def test_fingerprint_generator_some_options(FingerprintGenerator: type[AbstractF assert fingerprint.screen.availWidth >= 500 -@pytest.mark.parametrize('FingerprintGenerator', [pytest.param(BrowserForgeAdapter, id='browserforge')]) +@pytest.mark.parametrize('FingerprintGenerator', [pytest.param(DefaultFingerprintGenerator, id='browserforge')]) def test_fingerprint_generator_all_options(FingerprintGenerator: type[AbstractFingerprintGenerator]) -> None: # noqa:N803 # Test is more readable if argument(class) is PascalCase """Test that header generator can work with all the options. Some most basic checks of fingerprint. From 27479be911ccbcdaf361d01a9edf31dfdd325386 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Mon, 20 Jan 2025 13:40:14 +0100 Subject: [PATCH 21/32] Remove FingerprintGeneratorOptions --- src/crawlee/fingerprint_suite/__init__.py | 2 +- .../_browserforge_adapter.py | 49 ++++++++++--------- .../_fingerprint_generator.py | 21 ++++++-- src/crawlee/fingerprint_suite/_types.py | 22 +-------- .../_playwright/test_playwright_crawler.py | 9 ++-- tests/unit/fingerprint_suite/test_adapters.py | 22 ++++----- 6 files changed, 63 insertions(+), 62 deletions(-) diff --git a/src/crawlee/fingerprint_suite/__init__.py b/src/crawlee/fingerprint_suite/__init__.py index 1b93804055..5d182a5a13 100644 --- a/src/crawlee/fingerprint_suite/__init__.py +++ b/src/crawlee/fingerprint_suite/__init__.py @@ -1,4 +1,4 @@ from ._browserforge_adapter import FingerprintGenerator as DefaultFingerprintGenerator from ._fingerprint_generator import AbstractFingerprintGenerator from ._header_generator import HeaderGenerator -from ._types import FingerprintGeneratorOptions, HeaderGeneratorOptions, ScreenOptions +from ._types import HeaderGeneratorOptions, ScreenOptions diff --git a/src/crawlee/fingerprint_suite/_browserforge_adapter.py b/src/crawlee/fingerprint_suite/_browserforge_adapter.py index 8ec460d7d8..8dbac55f5b 100644 --- a/src/crawlee/fingerprint_suite/_browserforge_adapter.py +++ b/src/crawlee/fingerprint_suite/_browserforge_adapter.py @@ -1,41 +1,46 @@ from __future__ import annotations from copy import deepcopy -from typing import Any +from typing import TYPE_CHECKING, Any from browserforge.fingerprints import Fingerprint as bf_Fingerprint from browserforge.fingerprints import FingerprintGenerator as bf_FingerprintGenerator from browserforge.fingerprints import Screen from typing_extensions import override -from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator -from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions +from ._fingerprint_generator import AbstractFingerprintGenerator +if TYPE_CHECKING: + from ._types import HeaderGeneratorOptions, ScreenOptions -class FingerprintGenerator(AbstractFingerprintGenerator): - def __init__(self, options: FingerprintGeneratorOptions | None = None, *, strict: bool = False) -> None: - self._options = FingerprintGenerator._prepare_options(options or FingerprintGeneratorOptions()) - self._strict = strict - - @override - def generate(self) -> bf_Fingerprint: - return bf_FingerprintGenerator().generate(**self._options) - @staticmethod - def _prepare_options(options: FingerprintGeneratorOptions) -> dict[Any, Any]: - """Adapt options for `browserforge.fingerprints.FingerprintGenerator`.""" - raw_options = options.model_dump() - bf_options = {} - if raw_options['header_options'] is None: +class FingerprintGenerator(AbstractFingerprintGenerator): + def __init__( + self, + *, + header_options: HeaderGeneratorOptions | None = None, + screen_options: ScreenOptions | None = None, + mock_web_rtc: bool | None = None, + slim: bool | None = None, + ) -> None: + bf_options: dict[str, Any] = {'mock_webrtc': mock_web_rtc, 'slim': slim} + + if header_options is None: bf_header_options = {} else: - bf_header_options = deepcopy(raw_options['header_options']) + bf_header_options = deepcopy(header_options.model_dump()) bf_header_options['browser'] = bf_header_options.pop('browsers', None) bf_header_options['os'] = bf_header_options.pop('operating_systems', None) bf_header_options['device'] = bf_header_options.pop('devices', None) bf_header_options['locale'] = bf_header_options.pop('locales', None) - bf_options['mock_webrtc'] = raw_options['mock_web_rtc'] - bf_options['screen'] = Screen(**(raw_options.get('screen') or {})) - bf_options['slim'] = raw_options['slim'] - return {**bf_options, **bf_header_options} + if screen_options is None: + bf_options['screen'] = Screen() + else: + bf_options['screen'] = Screen(**screen_options.model_dump()) + + self._options = {**bf_options, **bf_header_options} + + @override + def generate(self) -> bf_Fingerprint: + return bf_FingerprintGenerator().generate(**self._options) diff --git a/src/crawlee/fingerprint_suite/_fingerprint_generator.py b/src/crawlee/fingerprint_suite/_fingerprint_generator.py index 775c06b86d..70c54e83b9 100644 --- a/src/crawlee/fingerprint_suite/_fingerprint_generator.py +++ b/src/crawlee/fingerprint_suite/_fingerprint_generator.py @@ -6,16 +6,31 @@ if TYPE_CHECKING: from browserforge.fingerprints import Fingerprint - from crawlee.fingerprint_suite._types import FingerprintGeneratorOptions + from crawlee.fingerprint_suite._types import HeaderGeneratorOptions, ScreenOptions class AbstractFingerprintGenerator(ABC): @abstractmethod - def __init__(self, options: FingerprintGeneratorOptions | None = None, *, strict: bool = False) -> None: + def __init__( + self, + *, + header_options: HeaderGeneratorOptions | None = None, + screen_options: ScreenOptions | None = None, + mock_web_rtc: bool | None = None, + slim: bool | None = None, + strict: bool = False, + ) -> None: """A default constructor. + All generator options are optional. If any value is not specified, then `None` is set in the options. + Default values for options set to `None` are implementation detail of used fingerprint generator. + Specific default values should not be relied upon. Use explicit values if it matters for your use case. + Args: - options: Options used for generating fingerprints. + header_options: Collection of header related attributes that can be used by the fingerprint generator. + screen_options: Defines the screen constrains for the fingerprint generator. + mock_web_rtc: Whether to mock WebRTC when injecting the fingerprint. + slim: Disables performance-heavy evasions when injecting the fingerprint. strict: If set to `True`, it will raise error if it is not possible to generate fingerprints based on the `options`. Default behavior is relaxation of `options` until it is possible to generate a fingerprint. """ diff --git a/src/crawlee/fingerprint_suite/_types.py b/src/crawlee/fingerprint_suite/_types.py index 8617f76601..f7f5822ca1 100644 --- a/src/crawlee/fingerprint_suite/_types.py +++ b/src/crawlee/fingerprint_suite/_types.py @@ -11,7 +11,7 @@ class ScreenOptions(BaseModel): - """Defines the screen constrains for the fingerprint generator .""" + """Defines the screen constrains for the fingerprint generator.""" min_width: Annotated[float | None, Field(alias='minWidth')] = None max_width: Annotated[float | None, Field(alias='maxWidth')] = None @@ -49,23 +49,3 @@ class HeaderGeneratorOptions(BaseModel): class Config: extra = 'forbid' populate_by_name = True - - -class FingerprintGeneratorOptions(BaseModel): - """Collection of fingerprint related attributes that can be used by the fingerprint generator. - - All generator options are optional. If any value is not specified, then `None` is set in the options. - Default values for options set to `None` are implementation detail of used fingerprint generator. - Specific default values should not be relied upon. Use explicit values if it matters for your use case. - """ - - header_options: HeaderGeneratorOptions | None = None - screen: ScreenOptions | None = None - mock_web_rtc: Annotated[bool | None, Field(alias='mockWebRTC')] = None - """Whether to mock WebRTC when injecting the fingerprint.""" - slim: Annotated[bool | None, Field(alias='slim')] = None - """Disables performance-heavy evasions when injecting the fingerprint.""" - - class Config: - extra = 'forbid' - populate_by_name = True diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py index 15db5c8300..26b465f33d 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py @@ -13,7 +13,6 @@ from crawlee.crawlers import PlaywrightCrawler from crawlee.fingerprint_suite import ( DefaultFingerprintGenerator, - FingerprintGeneratorOptions, HeaderGeneratorOptions, ScreenOptions, ) @@ -202,12 +201,14 @@ async def test_custom_fingerprint_uses_generator_options(httpbin: URL) -> None: min_height = 500 max_height = 1200 - fingerprint_options = FingerprintGeneratorOptions( + fingerprint_generator = DefaultFingerprintGenerator( header_options=HeaderGeneratorOptions(browsers=['firefox'], operating_systems=['android']), - screen=ScreenOptions(min_width=min_width, max_width=max_width, min_height=min_height, max_height=max_height), + screen_options=ScreenOptions( + min_width=min_width, max_width=max_width, min_height=min_height, max_height=max_height + ), ) - crawler = PlaywrightCrawler(headless=True, fingerprint_generator=DefaultFingerprintGenerator(fingerprint_options)) + crawler = PlaywrightCrawler(headless=True, fingerprint_generator=fingerprint_generator) response_headers = dict[str, str]() fingerprints = dict[str, Any]() diff --git a/tests/unit/fingerprint_suite/test_adapters.py b/tests/unit/fingerprint_suite/test_adapters.py index f62a51d5ab..eb5cf27ad3 100644 --- a/tests/unit/fingerprint_suite/test_adapters.py +++ b/tests/unit/fingerprint_suite/test_adapters.py @@ -3,7 +3,6 @@ from crawlee.fingerprint_suite import ( AbstractFingerprintGenerator, DefaultFingerprintGenerator, - FingerprintGeneratorOptions, HeaderGeneratorOptions, ScreenOptions, ) @@ -18,9 +17,12 @@ def test_fingerprint_generator_has_default(FingerprintGenerator: type[AbstractFi @pytest.mark.parametrize('FingerprintGenerator', [pytest.param(DefaultFingerprintGenerator, id='browserforge')]) def test_fingerprint_generator_some_options(FingerprintGenerator: type[AbstractFingerprintGenerator]) -> None: # noqa:N803 # Test is more readable if argument(class) is PascalCase """Test that header generator can work with only some options.""" - options = FingerprintGeneratorOptions(screen=ScreenOptions(min_width=500), mock_web_rtc=True) - fingerprint = FingerprintGenerator(options=options).generate() + fingerprint = FingerprintGenerator( + mock_web_rtc=True, + screen_options=ScreenOptions(min_width=500), + header_options=HeaderGeneratorOptions(strict=True), + ).generate() assert fingerprint.mockWebRTC is True assert fingerprint.screen.availWidth >= 500 @@ -37,15 +39,15 @@ def test_fingerprint_generator_all_options(FingerprintGenerator: type[AbstractFi min_height = 400 max_height = 1200 - options = FingerprintGeneratorOptions( - screen=ScreenOptions( + fingerprint = FingerprintGenerator( + mock_web_rtc=True, + slim=True, + screen_options=ScreenOptions( min_width=min_width, max_width=max_width, min_height=min_height, max_height=max_height, ), - mock_web_rtc=True, - slim=False, header_options=HeaderGeneratorOptions( strict=True, browsers=['firefox'], @@ -54,9 +56,7 @@ def test_fingerprint_generator_all_options(FingerprintGenerator: type[AbstractFi locales=['en'], # Does not generate any other values than `en-US` regardless of the input in browserforge http_version='2', # Http1 does not work in browserforge ), - ) - - fingerprint = FingerprintGenerator(options=options).generate() + ).generate() assert fingerprint.screen.availWidth >= min_width assert fingerprint.screen.availWidth <= max_width @@ -64,7 +64,7 @@ def test_fingerprint_generator_all_options(FingerprintGenerator: type[AbstractFi assert fingerprint.screen.availHeight <= max_height assert fingerprint.mockWebRTC is True - assert fingerprint.slim is False + assert fingerprint.slim is True assert 'Firefox' in fingerprint.navigator.userAgent assert 'Win' in fingerprint.navigator.oscpu assert 'en-US' in fingerprint.navigator.languages From 1cbadb0655e48b2a2d425ac5ea858cf8969e62a1 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 23 Jan 2025 14:17:34 +0100 Subject: [PATCH 22/32] Review commnets --- src/crawlee/browsers/_browser_pool.py | 6 ++-- .../_playwright_browser_controller.py | 6 ++-- .../browsers/_playwright_browser_plugin.py | 6 ++-- .../_playwright/_playwright_crawler.py | 6 ++-- src/crawlee/fingerprint_suite/__init__.py | 4 +-- .../_browserforge_adapter.py | 18 +++++++++-- .../_fingerprint_generator.py | 30 +------------------ tests/unit/fingerprint_suite/test_adapters.py | 18 ++++------- 8 files changed, 37 insertions(+), 57 deletions(-) diff --git a/src/crawlee/browsers/_browser_pool.py b/src/crawlee/browsers/_browser_pool.py index 4fe331290b..f5c9f9754c 100644 --- a/src/crawlee/browsers/_browser_pool.py +++ b/src/crawlee/browsers/_browser_pool.py @@ -23,7 +23,7 @@ from types import TracebackType from crawlee.browsers._base_browser_plugin import BaseBrowserPlugin - from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator + from crawlee.fingerprint_suite._fingerprint_generator import FingerprintGenerator from crawlee.proxy_configuration import ProxyInfo logger = getLogger(__name__) @@ -104,7 +104,7 @@ def with_default_plugin( browser_launch_options: Mapping[str, Any] | None = None, browser_new_context_options: Mapping[str, Any] | None = None, headless: bool | None = None, - fingerprint_generator: AbstractFingerprintGenerator | None = None, + fingerprint_generator: FingerprintGenerator | None = None, **kwargs: Any, ) -> BrowserPool: """Create a new instance with a single `PlaywrightBrowserPlugin` configured with the provided options. @@ -118,7 +118,7 @@ def with_default_plugin( are provided directly to Playwright's `browser.new_context` method. For more details, refer to the Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context. headless: Whether to run the browser in headless mode. - fingerprint_generator: An optional instance of implementation of `AbstractFingerprintGenerator` that is used + fingerprint_generator: An optional instance of implementation of `FingerprintGenerator` that is used to generate browser fingerprints together with consistent headers. kwargs: Additional arguments for default constructor. """ diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index f465b9a839..1caa1a38d6 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -19,7 +19,7 @@ from playwright.async_api import Browser - from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator + from crawlee.fingerprint_suite._fingerprint_generator import FingerprintGenerator from crawlee.proxy_configuration import ProxyInfo from logging import getLogger @@ -44,7 +44,7 @@ def __init__( *, max_open_pages_per_browser: int = 20, header_generator: HeaderGenerator | None = _DEFAULT_HEADER_GENERATOR, - fingerprint_generator: AbstractFingerprintGenerator | None = None, + fingerprint_generator: FingerprintGenerator | None = None, ) -> None: """A default constructor. @@ -54,7 +54,7 @@ def __init__( header_generator: An optional `HeaderGenerator` instance used to generate and manage HTTP headers for requests made by the browser. By default, a predefined header generator is used. Set to `None` to disable automatic header modifications. - fingerprint_generator: An optional instance of implementation of `AbstractFingerprintGenerator` that is used + fingerprint_generator: An optional instance of implementation of `FingerprintGenerator` that is used to generate browser fingerprints together with consistent headers. """ if fingerprint_generator and header_generator is not self._DEFAULT_HEADER_GENERATOR: diff --git a/src/crawlee/browsers/_playwright_browser_plugin.py b/src/crawlee/browsers/_playwright_browser_plugin.py index b842bf5b4a..156196c2c8 100644 --- a/src/crawlee/browsers/_playwright_browser_plugin.py +++ b/src/crawlee/browsers/_playwright_browser_plugin.py @@ -19,7 +19,7 @@ from types import TracebackType from crawlee.browsers._types import BrowserType - from crawlee.fingerprint_suite._fingerprint_generator import AbstractFingerprintGenerator + from crawlee.fingerprint_suite._fingerprint_generator import FingerprintGenerator logger = getLogger(__name__) @@ -44,7 +44,7 @@ def __init__( browser_launch_options: dict[str, Any] | None = None, browser_new_context_options: dict[str, Any] | None = None, max_open_pages_per_browser: int = 20, - fingerprint_generator: AbstractFingerprintGenerator | None = None, + fingerprint_generator: FingerprintGenerator | None = None, ) -> None: """A default constructor. @@ -58,7 +58,7 @@ def __init__( Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context. max_open_pages_per_browser: The maximum number of pages that can be opened in a single browser instance. Once reached, a new browser instance will be launched to handle the excess. - fingerprint_generator: An optional instance of implementation of `AbstractFingerprintGenerator` that is used + fingerprint_generator: An optional instance of implementation of `FingerprintGenerator` that is used to generate browser fingerprints together with consistent headers. """ config = service_locator.get_configuration() diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py index 57a9c964bb..1c53ee601e 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py +++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py @@ -25,7 +25,7 @@ from crawlee._types import BasicCrawlingContext, EnqueueLinksKwargs from crawlee.browsers._types import BrowserType - from crawlee.fingerprint_suite import AbstractFingerprintGenerator + from crawlee.fingerprint_suite import FingerprintGenerator @docs_group('Classes') @@ -76,7 +76,7 @@ def __init__( browser_type: BrowserType | None = None, browser_launch_options: Mapping[str, Any] | None = None, browser_new_context_options: Mapping[str, Any] | None = None, - fingerprint_generator: AbstractFingerprintGenerator | None = None, + fingerprint_generator: FingerprintGenerator | None = None, headless: bool | None = None, **kwargs: Unpack[BasicCrawlerOptions[PlaywrightCrawlingContext]], ) -> None: @@ -94,7 +94,7 @@ def __init__( are provided directly to Playwright's `browser.new_context` method. For more details, refer to the [Playwright documentation](https://playwright.dev/python/docs/api/class-browser#browser-new-context). This option should not be used if `browser_pool` is provided. - fingerprint_generator: An optional instance of implementation of `AbstractFingerprintGenerator` that is used + fingerprint_generator: An optional instance of implementation of `FingerprintGenerator` that is used to generate browser fingerprints together with consistent headers. headless: Whether to run the browser in headless mode. This option should not be used if `browser_pool` is provided. diff --git a/src/crawlee/fingerprint_suite/__init__.py b/src/crawlee/fingerprint_suite/__init__.py index 5d182a5a13..35611eb676 100644 --- a/src/crawlee/fingerprint_suite/__init__.py +++ b/src/crawlee/fingerprint_suite/__init__.py @@ -1,4 +1,4 @@ -from ._browserforge_adapter import FingerprintGenerator as DefaultFingerprintGenerator -from ._fingerprint_generator import AbstractFingerprintGenerator +from ._browserforge_adapter import BrowserforgeFingerprintGenerator as DefaultFingerprintGenerator +from ._fingerprint_generator import FingerprintGenerator from ._header_generator import HeaderGenerator from ._types import HeaderGeneratorOptions, ScreenOptions diff --git a/src/crawlee/fingerprint_suite/_browserforge_adapter.py b/src/crawlee/fingerprint_suite/_browserforge_adapter.py index 8dbac55f5b..33d5a8e755 100644 --- a/src/crawlee/fingerprint_suite/_browserforge_adapter.py +++ b/src/crawlee/fingerprint_suite/_browserforge_adapter.py @@ -8,13 +8,13 @@ from browserforge.fingerprints import Screen from typing_extensions import override -from ._fingerprint_generator import AbstractFingerprintGenerator +from ._fingerprint_generator import FingerprintGenerator if TYPE_CHECKING: from ._types import HeaderGeneratorOptions, ScreenOptions -class FingerprintGenerator(AbstractFingerprintGenerator): +class BrowserforgeFingerprintGenerator(FingerprintGenerator): def __init__( self, *, @@ -23,6 +23,20 @@ def __init__( mock_web_rtc: bool | None = None, slim: bool | None = None, ) -> None: + """A default constructor. + + All generator options are optional. If any value is not specified, then `None` is set in the options. + Default values for options set to `None` are implementation detail of used fingerprint generator. + Specific default values should not be relied upon. Use explicit values if it matters for your use case. + + Args: + header_options: Collection of header related attributes that can be used by the fingerprint generator. + screen_options: Defines the screen constrains for the fingerprint generator. + mock_web_rtc: Whether to mock WebRTC when injecting the fingerprint. + slim: Disables performance-heavy evasions when injecting the fingerprint. + strict: If set to `True`, it will raise error if it is not possible to generate fingerprints based on the + `options`. Default behavior is relaxation of `options` until it is possible to generate a fingerprint. + """ bf_options: dict[str, Any] = {'mock_webrtc': mock_web_rtc, 'slim': slim} if header_options is None: diff --git a/src/crawlee/fingerprint_suite/_fingerprint_generator.py b/src/crawlee/fingerprint_suite/_fingerprint_generator.py index 70c54e83b9..491e97e05e 100644 --- a/src/crawlee/fingerprint_suite/_fingerprint_generator.py +++ b/src/crawlee/fingerprint_suite/_fingerprint_generator.py @@ -6,36 +6,8 @@ if TYPE_CHECKING: from browserforge.fingerprints import Fingerprint - from crawlee.fingerprint_suite._types import HeaderGeneratorOptions, ScreenOptions - - -class AbstractFingerprintGenerator(ABC): - @abstractmethod - def __init__( - self, - *, - header_options: HeaderGeneratorOptions | None = None, - screen_options: ScreenOptions | None = None, - mock_web_rtc: bool | None = None, - slim: bool | None = None, - strict: bool = False, - ) -> None: - """A default constructor. - - All generator options are optional. If any value is not specified, then `None` is set in the options. - Default values for options set to `None` are implementation detail of used fingerprint generator. - Specific default values should not be relied upon. Use explicit values if it matters for your use case. - - Args: - header_options: Collection of header related attributes that can be used by the fingerprint generator. - screen_options: Defines the screen constrains for the fingerprint generator. - mock_web_rtc: Whether to mock WebRTC when injecting the fingerprint. - slim: Disables performance-heavy evasions when injecting the fingerprint. - strict: If set to `True`, it will raise error if it is not possible to generate fingerprints based on the - `options`. Default behavior is relaxation of `options` until it is possible to generate a fingerprint. - """ - ... +class FingerprintGenerator(ABC): @abstractmethod def generate(self) -> Fingerprint: """Method that is capable of generating fingerprints. diff --git a/tests/unit/fingerprint_suite/test_adapters.py b/tests/unit/fingerprint_suite/test_adapters.py index eb5cf27ad3..b742b2a05e 100644 --- a/tests/unit/fingerprint_suite/test_adapters.py +++ b/tests/unit/fingerprint_suite/test_adapters.py @@ -1,24 +1,19 @@ -import pytest - from crawlee.fingerprint_suite import ( - AbstractFingerprintGenerator, DefaultFingerprintGenerator, HeaderGeneratorOptions, ScreenOptions, ) -@pytest.mark.parametrize('FingerprintGenerator', [pytest.param(DefaultFingerprintGenerator, id='browserforge')]) -def test_fingerprint_generator_has_default(FingerprintGenerator: type[AbstractFingerprintGenerator]) -> None: # noqa:N803 # Test is more readable if argument(class) is PascalCase +def test_fingerprint_generator_has_default() -> None: # Test is more readable if argument(class) is PascalCase """Test that header generator can work without any options.""" - assert FingerprintGenerator().generate() + assert DefaultFingerprintGenerator().generate() -@pytest.mark.parametrize('FingerprintGenerator', [pytest.param(DefaultFingerprintGenerator, id='browserforge')]) -def test_fingerprint_generator_some_options(FingerprintGenerator: type[AbstractFingerprintGenerator]) -> None: # noqa:N803 # Test is more readable if argument(class) is PascalCase +def test_fingerprint_generator_some_options() -> None: # Test is more readable if argument(class) is PascalCase """Test that header generator can work with only some options.""" - fingerprint = FingerprintGenerator( + fingerprint = DefaultFingerprintGenerator( mock_web_rtc=True, screen_options=ScreenOptions(min_width=500), header_options=HeaderGeneratorOptions(strict=True), @@ -28,8 +23,7 @@ def test_fingerprint_generator_some_options(FingerprintGenerator: type[AbstractF assert fingerprint.screen.availWidth >= 500 -@pytest.mark.parametrize('FingerprintGenerator', [pytest.param(DefaultFingerprintGenerator, id='browserforge')]) -def test_fingerprint_generator_all_options(FingerprintGenerator: type[AbstractFingerprintGenerator]) -> None: # noqa:N803 # Test is more readable if argument(class) is PascalCase +def test_fingerprint_generator_all_options() -> None: # Test is more readable if argument(class) is PascalCase """Test that header generator can work with all the options. Some most basic checks of fingerprint. Fingerprint generation option might have no effect if there is no fingerprint sample present in collected data. @@ -39,7 +33,7 @@ def test_fingerprint_generator_all_options(FingerprintGenerator: type[AbstractFi min_height = 400 max_height = 1200 - fingerprint = FingerprintGenerator( + fingerprint = DefaultFingerprintGenerator( mock_web_rtc=True, slim=True, screen_options=ScreenOptions( From 8e44acdd1ad7aad9ca8a542bca0e66a2a30e7928 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Fri, 24 Jan 2025 10:57:48 +0100 Subject: [PATCH 23/32] Handle inconsistent result from browserforge fingerprint generator --- .../_browserforge_adapter.py | 14 ++++++++++++- tests/unit/fingerprint_suite/test_adapters.py | 20 ++++++++++--------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/crawlee/fingerprint_suite/_browserforge_adapter.py b/src/crawlee/fingerprint_suite/_browserforge_adapter.py index 33d5a8e755..f4534d552c 100644 --- a/src/crawlee/fingerprint_suite/_browserforge_adapter.py +++ b/src/crawlee/fingerprint_suite/_browserforge_adapter.py @@ -54,7 +54,19 @@ def __init__( bf_options['screen'] = Screen(**screen_options.model_dump()) self._options = {**bf_options, **bf_header_options} + self._generator = bf_FingerprintGenerator() @override def generate(self) -> bf_Fingerprint: - return bf_FingerprintGenerator().generate(**self._options) + # browserforge fingerprint generation can be flaky + # https://github.com/daijro/browserforge/issues/22" + # During test runs around 10 % flakiness was detected. + # Max attempt set to 10 as (0.1)^10 is considered sufficiently low probability. + max_attempts = 10 + for attempt in range(max_attempts): + try: + return self._generator.generate(**self._options) + except ValueError: # noqa:PERF203 + if attempt == max_attempts: + raise + raise RuntimeError('Failed to generate fingerprint.') diff --git a/tests/unit/fingerprint_suite/test_adapters.py b/tests/unit/fingerprint_suite/test_adapters.py index b742b2a05e..2521230206 100644 --- a/tests/unit/fingerprint_suite/test_adapters.py +++ b/tests/unit/fingerprint_suite/test_adapters.py @@ -5,25 +5,27 @@ ) -def test_fingerprint_generator_has_default() -> None: # Test is more readable if argument(class) is PascalCase +def test_fingerprint_generator_has_default() -> None: """Test that header generator can work without any options.""" assert DefaultFingerprintGenerator().generate() -def test_fingerprint_generator_some_options() -> None: # Test is more readable if argument(class) is PascalCase - """Test that header generator can work with only some options.""" - - fingerprint = DefaultFingerprintGenerator( +def test_fingerprint_generator_some_options_stress_test() -> None: + """Test that header generator can work consistently.""" + fingerprint_generator = DefaultFingerprintGenerator( mock_web_rtc=True, screen_options=ScreenOptions(min_width=500), header_options=HeaderGeneratorOptions(strict=True), - ).generate() + ) - assert fingerprint.mockWebRTC is True - assert fingerprint.screen.availWidth >= 500 + for _ in range(20): + fingerprint = fingerprint_generator.generate() + + assert fingerprint.mockWebRTC is True + assert fingerprint.screen.availWidth > 500 -def test_fingerprint_generator_all_options() -> None: # Test is more readable if argument(class) is PascalCase +def test_fingerprint_generator_all_options() -> None: """Test that header generator can work with all the options. Some most basic checks of fingerprint. Fingerprint generation option might have no effect if there is no fingerprint sample present in collected data. From d8001e7122f92e5c7399f1a78432e90bb2852ac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20Proch=C3=A1zka?= Date: Mon, 27 Jan 2025 08:28:30 +0100 Subject: [PATCH 24/32] Apply suggestions from code review Co-authored-by: Vlada Dusek --- src/crawlee/browsers/_browser_pool.py | 2 +- src/crawlee/browsers/_playwright_browser_controller.py | 2 +- src/crawlee/browsers/_playwright_browser_plugin.py | 2 +- src/crawlee/fingerprint_suite/_fingerprint_generator.py | 1 - 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/crawlee/browsers/_browser_pool.py b/src/crawlee/browsers/_browser_pool.py index f5c9f9754c..268aa583fa 100644 --- a/src/crawlee/browsers/_browser_pool.py +++ b/src/crawlee/browsers/_browser_pool.py @@ -23,7 +23,7 @@ from types import TracebackType from crawlee.browsers._base_browser_plugin import BaseBrowserPlugin - from crawlee.fingerprint_suite._fingerprint_generator import FingerprintGenerator + from crawlee.fingerprint_suite import FingerprintGenerator from crawlee.proxy_configuration import ProxyInfo logger = getLogger(__name__) diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index 1caa1a38d6..a85805b58c 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -19,7 +19,7 @@ from playwright.async_api import Browser - from crawlee.fingerprint_suite._fingerprint_generator import FingerprintGenerator + from crawlee.fingerprint_suite import FingerprintGenerator from crawlee.proxy_configuration import ProxyInfo from logging import getLogger diff --git a/src/crawlee/browsers/_playwright_browser_plugin.py b/src/crawlee/browsers/_playwright_browser_plugin.py index 156196c2c8..f780b94520 100644 --- a/src/crawlee/browsers/_playwright_browser_plugin.py +++ b/src/crawlee/browsers/_playwright_browser_plugin.py @@ -19,7 +19,7 @@ from types import TracebackType from crawlee.browsers._types import BrowserType - from crawlee.fingerprint_suite._fingerprint_generator import FingerprintGenerator + from crawlee.fingerprint_suite import FingerprintGenerator logger = getLogger(__name__) diff --git a/src/crawlee/fingerprint_suite/_fingerprint_generator.py b/src/crawlee/fingerprint_suite/_fingerprint_generator.py index 491e97e05e..8d2c782c95 100644 --- a/src/crawlee/fingerprint_suite/_fingerprint_generator.py +++ b/src/crawlee/fingerprint_suite/_fingerprint_generator.py @@ -16,4 +16,3 @@ def generate(self) -> Fingerprint: Return type is temporarily set to `Fingerprint` from `browserforge`. This is subject to change and most likely it will change to custom `Fingerprint` class defined in this repo later. """ - ... From 07acbfa5c164b5ba20cd2e5729b6d7162d8af0bf Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Mon, 27 Jan 2025 10:04:21 +0100 Subject: [PATCH 25/32] Docs Update docstrings. Add example code + doc page. --- ...ight_crawler_with_fingerprint_generator.py | 40 +++++++++++++++++++ ...ght_crawler_with_fingerprint_generator.mdx | 17 ++++++++ .../_browserforge_adapter.py | 5 +++ .../_fingerprint_generator.py | 7 +++- 4 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 docs/examples/code/playwright_crawler_with_fingerprint_generator.py create mode 100644 docs/examples/playwright_crawler_with_fingerprint_generator.mdx diff --git a/docs/examples/code/playwright_crawler_with_fingerprint_generator.py b/docs/examples/code/playwright_crawler_with_fingerprint_generator.py new file mode 100644 index 0000000000..51c464a0fc --- /dev/null +++ b/docs/examples/code/playwright_crawler_with_fingerprint_generator.py @@ -0,0 +1,40 @@ +import asyncio + +from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext +from crawlee.fingerprint_suite import DefaultFingerprintGenerator, HeaderGeneratorOptions, ScreenOptions + + +async def main() -> None: + # Use default fingerprint generator with desired fingerprint options. + # Generator will try to generate real looking browser fingerprint based on the options. + # Unspecified fingerprint options will be automatically selected by the generator. + fingerprint_generator = DefaultFingerprintGenerator( + header_options=HeaderGeneratorOptions(browsers=['chromium']), + screen_options=ScreenOptions(min_width=400), + ) + + crawler = PlaywrightCrawler( + # Limit the crawl to max requests. Remove or increase it for crawling all links. + max_requests_per_crawl=10, + # Headless mode, set to False to see the browser in action. + headless=False, + # Browser types supported by Playwright. + browser_type='chromium', + # Fingerprint generator to be used. By default no fingerprint generation is done. + fingerprint_generator=fingerprint_generator, + ) + + # Define the default request handler, which will be called for every request. + @crawler.router.default_handler + async def request_handler(context: PlaywrightCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url} ...') + + # Find a link to the next page and enqueue it if it exists. + await context.enqueue_links(selector='.morelink') + + # Run the crawler with the initial list of URLs. + await crawler.run(['https://news.ycombinator.com/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/playwright_crawler_with_fingerprint_generator.mdx b/docs/examples/playwright_crawler_with_fingerprint_generator.mdx new file mode 100644 index 0000000000..00989501df --- /dev/null +++ b/docs/examples/playwright_crawler_with_fingerprint_generator.mdx @@ -0,0 +1,17 @@ +--- +id: playwright-crawler-with-fingeprint-generator +title: Playwright crawler with fingerprint generator +--- + +import ApiLink from '@site/src/components/ApiLink'; +import CodeBlock from '@theme/CodeBlock'; + +import PlaywrightCrawlerExample from '!!raw-loader!./code/playwright_crawler_with_fingerprint_generator.py'; + +This example demonstrates how to use `PlaywrightCrawler` together with `FingerprintGenerator` that will populate several browser attributes to mimic real browser fingerprint. To read more about fingerprints please see: https://docs.apify.com/academy/anti-scraping/techniques/fingerprinting. + +You can implement your own fingerprint generator or use `DefaultFingerprintGenerator`. To use the generator initialize it with the desired fingerprint options. The generator will try to create fingerprint based on those options. Unspecified options will be automatically selected by the generator from the set of reasonable values. If some option is important for you, do not rely on the default and explicitly define it. + + + {PlaywrightCrawlerExample} + diff --git a/src/crawlee/fingerprint_suite/_browserforge_adapter.py b/src/crawlee/fingerprint_suite/_browserforge_adapter.py index f4534d552c..3b4551115e 100644 --- a/src/crawlee/fingerprint_suite/_browserforge_adapter.py +++ b/src/crawlee/fingerprint_suite/_browserforge_adapter.py @@ -8,13 +8,18 @@ from browserforge.fingerprints import Screen from typing_extensions import override +from crawlee._utils.docs import docs_group + from ._fingerprint_generator import FingerprintGenerator if TYPE_CHECKING: from ._types import HeaderGeneratorOptions, ScreenOptions +@docs_group('Classes') class BrowserforgeFingerprintGenerator(FingerprintGenerator): + """`FingerprintGenerator` adapter for fingerprint generator from `browserforge`.""" + def __init__( self, *, diff --git a/src/crawlee/fingerprint_suite/_fingerprint_generator.py b/src/crawlee/fingerprint_suite/_fingerprint_generator.py index 8d2c782c95..f32f24935e 100644 --- a/src/crawlee/fingerprint_suite/_fingerprint_generator.py +++ b/src/crawlee/fingerprint_suite/_fingerprint_generator.py @@ -3,14 +3,19 @@ from abc import ABC, abstractmethod from typing import TYPE_CHECKING +from crawlee._utils.docs import docs_group + if TYPE_CHECKING: from browserforge.fingerprints import Fingerprint +@docs_group('Classes') class FingerprintGenerator(ABC): + """A class for creating browser fingerprints that mimic browser fingerprints of real users.""" + @abstractmethod def generate(self) -> Fingerprint: - """Method that is capable of generating fingerprints. + """Generate browser fingerprints. This is experimental feature. Return type is temporarily set to `Fingerprint` from `browserforge`. This is subject to change and most likely From 866fe98202329f7d12cd4a77839df5c1cca702e9 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Mon, 27 Jan 2025 10:30:33 +0100 Subject: [PATCH 26/32] Make sure browserforge files are downloaded before tests. (To avoid some import race conditions downloads when running pytest with multiple processes) --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 73f0d92029..89ce795859 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ install-dev: poetry install --all-extras poetry run pre-commit install poetry run playwright install + poetry run python -m browserforge update build: poetry build --no-interaction -vv From acc720f2fd888cd5ed4179561dbab5b81986c43b Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 28 Jan 2025 16:18:07 +0100 Subject: [PATCH 27/32] HeaderGenerator from browserforge WIP --- .../_browserforge_adapter.py | 27 +- src/crawlee/fingerprint_suite/_consts.py | 1010 +---------------- .../fingerprint_suite/_header_generator.py | 38 +- .../test_header_generator.py | 37 +- 4 files changed, 57 insertions(+), 1055 deletions(-) diff --git a/src/crawlee/fingerprint_suite/_browserforge_adapter.py b/src/crawlee/fingerprint_suite/_browserforge_adapter.py index 3b4551115e..14ea9b377f 100644 --- a/src/crawlee/fingerprint_suite/_browserforge_adapter.py +++ b/src/crawlee/fingerprint_suite/_browserforge_adapter.py @@ -6,14 +6,16 @@ from browserforge.fingerprints import Fingerprint as bf_Fingerprint from browserforge.fingerprints import FingerprintGenerator as bf_FingerprintGenerator from browserforge.fingerprints import Screen +from browserforge.headers.generator import HeaderGenerator as bf_HeaderGenerator from typing_extensions import override from crawlee._utils.docs import docs_group +from ._consts import BROWSER_TYPE_HEADER_KEYWORD from ._fingerprint_generator import FingerprintGenerator if TYPE_CHECKING: - from ._types import HeaderGeneratorOptions, ScreenOptions + from ._types import HeaderGeneratorOptions, ScreenOptions, SupportedBrowserType @docs_group('Classes') @@ -75,3 +77,26 @@ def generate(self) -> bf_Fingerprint: if attempt == max_attempts: raise raise RuntimeError('Failed to generate fingerprint.') + + +@docs_group('Classes') +class BrowserforgeHeaderGenerator: + + def __init__(self): + self._generator = bf_HeaderGenerator() + + def generate(self, browser_type: SupportedBrowserType) -> bf_Fingerprint: + # browserforge header generation can be flaky. Enforce basic QA on generated headers + max_attempts = 10 + + if browser_type=='webkit': + bf_browser_type = 'safari' + else: + bf_browser_type = browser_type + + for attempt in range(max_attempts): + generated_header = self._generator.generate(browser=bf_browser_type) + if any(keyword in generated_header['User-Agent'] for keyword in BROWSER_TYPE_HEADER_KEYWORD[browser_type]): + return generated_header + print(generated_header['User-Agent']) + raise RuntimeError('Failed to generate header.') diff --git a/src/crawlee/fingerprint_suite/_consts.py b/src/crawlee/fingerprint_suite/_consts.py index 788a4865b5..d510f11078 100644 --- a/src/crawlee/fingerprint_suite/_consts.py +++ b/src/crawlee/fingerprint_suite/_consts.py @@ -17,1006 +17,10 @@ ) PW_WEBKIT_HEADLESS_DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15' -# Random 1000 user agents from Apify fingerprint dataset. -USER_AGENT_POOL = [ - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.43', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv109.0) Gecko/20100101 Firefox/109.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.58', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv109.0) Gecko/20100101 Firefox/117.0', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.15 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv108.0) Gecko/20100101 Firefox/108.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.57', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv100.0) Gecko/20100101 Firefox/100.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv123.0) Gecko/20100101 Firefox/123.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 13; CPH2487) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv97.0) Gecko/20100101 Firefox/97.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv101.0) Gecko/20100101 Firefox/101.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.140', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv109.0) Gecko/20100101 Firefox/112.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36 Edg/105.0.1343.42', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.35', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv109.0) Gecko/20100101 Firefox/113.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 OPR/101.0.0.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Android 13; Mobile; rv121.0) Gecko/121.0 Firefox/121.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.58', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [LinkedInApp]/9.27.3917.3', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.88 Mobile Safari/537.36 +https//sitebulb.com', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.88 Mobile Safari/537.36 +https//sitebulb.com', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.2 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv109.0) Gecko/20100101 Firefox/116.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [LinkedInApp]/9.29.5438', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv108.0) Gecko/20100101 Firefox/108.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.115 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv124.0) Gecko/20100101 Firefox/124.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.15', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv109.0) Gecko/20100101 Firefox/119.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.63 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_1_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv125.0) Gecko/20100101 Firefox/125.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv101.0) Gecko/20100101 Firefox/101.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 OPR/93.0.0.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.3 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Linux; Android 13; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 13; SM-A716U1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.15', - 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.95 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.5672.126 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/97.0.4692.71 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 12; CPH2159) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.54', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.41', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.82', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Mobile/15E148 DuckDuckGo/7 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5.2 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.6.1 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Linux; Android 11; RMX3201) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.85 Mobile Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.62 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 CCleaner/116.0.0.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.15', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv107.0) Gecko/20100101 Firefox/107.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 9; SM-G955F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.87 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 9; RMX1805) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.162 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/21A360 [FBAN/FBIOS;FBAV/453.0.0.47.106;FBBV/570990458;FBDV/iPhone12,1;FBMD/iPhone;FBSN/iOS;FBSV/17.0.3;FBSS/2;FBID/phone;FBLC/en_GB;FBOP/5;FBRV/573792857]', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.68', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.81 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv109.0) Gecko/20100101 Firefox/115.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv109.0) Gecko/20100101 Firefox/117.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv109.0) Gecko/20100101 Firefox/109.0', - 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.88 Mobile Safari/537.36 +https//sitebulb.com', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv96.0) Gecko/20100101 Firefox/96.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5.1 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.98 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv109.0) Gecko/20100101 Firefox/116.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Safari/605.1.15', - 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/112.0.5615.46 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 Edg/96.0.1054.62', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_5_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5.1 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 OPR/106.0.0.0', - 'Mozilla/5.0 (Linux; Android 10; Infinix X688B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.98 Mobile Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv109.0) Gecko/20100101 Firefox/117.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/121.0.6167.138 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (X11; CrOS x86_64 14695.85.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.75 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv109.0) Gecko/20100101 Firefox/110.0', - 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv109.0) Gecko/20100101 Firefox/110.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/20H19 Instagram 303.3.0.24.111 (iPhone10,5; iOS 16_7; pt_BR; pt; scale=3.00; 1242x2208; 523000219)', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 36b1546a5700e52eb2972b3f92b314fa', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15', - 'Mozilla/5.0 (Linux; Android 12; SM-G998B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.3 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv101.0) Gecko/20100101 Firefox/101.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36 Edg/100.0.1185.44', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 13; Pixel 7 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36 EdgA/114.0.1823.74', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv109.0) Gecko/20100101 Firefox/117.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv99.0) Gecko/20100101 Firefox/99.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 OPR/101.0.0.0 (Edition std-1)', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 10; Infinix X682B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv125.0) Gecko/20100101 Firefox/125.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv108.0) Gecko/20100101 Firefox/108.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.35', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/21B5056e Instagram 303.3.0.24.111 (iPhone11,6; iOS 17_1; pt_BR; pt; scale=3.00; 1242x2688; 523000219)', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv109.0) Gecko/20100101 Firefox/109.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 12_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/100.0.4896.85 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.15', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.15', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv109.0) Gecko/20100101 Firefox/114.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv125.0) Gecko/20100101 Firefox/125.0', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_8 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'My browser', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_0_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 12; SM-G991U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.104 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 12; SAMSUNG SM-A528B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/16.0 Chrome/92.0.4515.166 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv109.0) Gecko/20100101 Firefox/112.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.70', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/96.0.4664.116 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv97.0) Gecko/20100101 Firefox/97.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 13; 2201116PG Build/TKQ1.221114.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/117.0.0.0 Mobile Safari/537.36 Instagram 304.0.0.24.106 Android (33/13; 440dpi; 1080x2180; Xiaomi/POCO; 2201116PG; veux; qcom; pt_BR; 524093855)', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.160 YaBrowser/22.5.1.985 Yowser/2.5 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64; rv100.0) Gecko/20100101 Firefox/100.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv125.0) Gecko/20100101 Firefox/125.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/109.0.0.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 12; RMX2155) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2.1 Safari/605.1.15', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64; rv109.0) Gecko/20100101 Firefox/115.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv97.0) Gecko/20100101 Firefox/97.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.6.1 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 10; HD1900 Build/QKQ1.190716.003; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/75.0.3770.156 Mobile Safari/537.36 aweme_230400 JsSdk/1.0 NetType/WIFI AppName/aweme app_version/23.4.0 ByteLocale/zh-CN Region/CN AppSkin/white AppTheme/light BytedanceWebview/d8a21c6 WebView/075113004008', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.5563.64 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64; rv108.0) Gecko/20100101 Firefox/108.0', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv95.0) Gecko/20100101 Firefox/95.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 13; sdk_gphone64_x86_64 Build/TE1A.220922.010; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/122.0.6261.105 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36 Edg/104.0.1293.70', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.63 Safari/537.36 Edg/102.0.1245.30', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36 Edg/97.0.1072.55', - 'Mozilla/5.0 (X11; Linux x86_64; rv109.0) Gecko/20100101 Firefox/115.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', -] + +BROWSER_TYPE_HEADER_KEYWORD = { + 'chromium':{'Chrome', 'CriOS'}, + 'firefox':{'Firefox', 'FxiOS'}, + 'edge':{'Edg', 'Edge', 'EdgA', 'EdgiOS'}, + 'webkit':{'Safari'}, +} diff --git a/src/crawlee/fingerprint_suite/_header_generator.py b/src/crawlee/fingerprint_suite/_header_generator.py index 5e821336c8..06de832064 100644 --- a/src/crawlee/fingerprint_suite/_header_generator.py +++ b/src/crawlee/fingerprint_suite/_header_generator.py @@ -1,20 +1,14 @@ from __future__ import annotations -import random from typing import TYPE_CHECKING from crawlee._types import HttpHeaders from crawlee._utils.docs import docs_group +from crawlee.fingerprint_suite._browserforge_adapter import BrowserforgeHeaderGenerator from crawlee.fingerprint_suite._consts import ( - COMMON_ACCEPT, - COMMON_ACCEPT_LANGUAGE, PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA, PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_MOBILE, PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_PLATFORM, - PW_CHROMIUM_HEADLESS_DEFAULT_USER_AGENT, - PW_FIREFOX_HEADLESS_DEFAULT_USER_AGENT, - PW_WEBKIT_HEADLESS_DEFAULT_USER_AGENT, - USER_AGENT_POOL, ) if TYPE_CHECKING: @@ -25,22 +19,23 @@ class HeaderGenerator: """Generates realistic looking or browser-like HTTP headers.""" + def __init__(self): + self._generator = BrowserforgeHeaderGenerator() + def get_common_headers(self) -> HttpHeaders: """Get common HTTP headers ("Accept", "Accept-Language"). We do not modify the "Accept-Encoding", "Connection" and other headers. They should be included and handled by the HTTP client or browser. """ - headers = { - 'Accept': COMMON_ACCEPT, - 'Accept-Language': COMMON_ACCEPT_LANGUAGE, - } - return HttpHeaders(headers) + all_headers = self._generator.generate() + return HttpHeaders({key:value for key, value in all_headers.items() if key in {'Accept', 'Accept-Language'}}) def get_random_user_agent_header(self) -> HttpHeaders: """Get a random User-Agent header.""" - headers = {'User-Agent': random.choice(USER_AGENT_POOL)} - return HttpHeaders(headers) + all_headers = self._generator.generate() + return HttpHeaders({'User-Agent':all_headers['User-Agent']}) + def get_user_agent_header( self, @@ -50,19 +45,10 @@ def get_user_agent_header( """Get the User-Agent header based on the browser type.""" headers = dict[str, str]() - if browser_type == 'chromium': - headers['User-Agent'] = PW_CHROMIUM_HEADLESS_DEFAULT_USER_AGENT - - elif browser_type == 'firefox': - headers['User-Agent'] = PW_FIREFOX_HEADLESS_DEFAULT_USER_AGENT - - elif browser_type == 'webkit': - headers['User-Agent'] = PW_WEBKIT_HEADLESS_DEFAULT_USER_AGENT - - else: + if browser_type not in {'chromium', 'firefox', 'webkit', 'edge'}: raise ValueError(f'Unsupported browser type: {browser_type}') - - return HttpHeaders(headers) + all_headers = self._generator.generate(browser_type=browser_type) + return HttpHeaders({'User-Agent': all_headers['User-Agent']}) def get_sec_ch_ua_headers( self, diff --git a/tests/unit/fingerprint_suite/test_header_generator.py b/tests/unit/fingerprint_suite/test_header_generator.py index 7720c65e47..532ad96a51 100644 --- a/tests/unit/fingerprint_suite/test_header_generator.py +++ b/tests/unit/fingerprint_suite/test_header_generator.py @@ -4,14 +4,12 @@ from crawlee.fingerprint_suite import HeaderGenerator from crawlee.fingerprint_suite._consts import ( + BROWSER_TYPE_HEADER_KEYWORD, PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA, PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_MOBILE, PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_PLATFORM, - PW_CHROMIUM_HEADLESS_DEFAULT_USER_AGENT, - PW_FIREFOX_HEADLESS_DEFAULT_USER_AGENT, - PW_WEBKIT_HEADLESS_DEFAULT_USER_AGENT, - USER_AGENT_POOL, ) +from crawlee.fingerprint_suite._types import SupportedBrowserType def test_get_common_headers() -> None: @@ -28,34 +26,23 @@ def test_get_random_user_agent_header() -> None: headers = header_generator.get_random_user_agent_header() assert 'User-Agent' in headers - assert headers['User-Agent'] in USER_AGENT_POOL + assert headers['User-Agent'] -def test_get_user_agent_header_chromium() -> None: - """Test that the User-Agent header is generated correctly for Chromium.""" - header_generator = HeaderGenerator() - headers = header_generator.get_user_agent_header(browser_type='chromium') - - assert 'User-Agent' in headers - assert headers['User-Agent'] == PW_CHROMIUM_HEADLESS_DEFAULT_USER_AGENT - - -def test_get_user_agent_header_firefox() -> None: - """Test that the User-Agent header is generated correctly for Firefox.""" - header_generator = HeaderGenerator() - headers = header_generator.get_user_agent_header(browser_type='firefox') - - assert 'User-Agent' in headers - assert headers['User-Agent'] == PW_FIREFOX_HEADLESS_DEFAULT_USER_AGENT +@pytest.mark.parametrize('_', range(100)) +@pytest.mark.parametrize('browser_type', [ + 'chromium','firefox','edge','webkit' +]) +def test_get_user_agent_header_stress_test(browser_type: SupportedBrowserType,_) -> None: + """Test that the User-Agent header is consistently generated correctly. -def test_get_user_agent_header_webkit() -> None: - """Test that the User-Agent header is generated correctly for WebKit.""" + (Very fast even when stress tested.)""" header_generator = HeaderGenerator() - headers = header_generator.get_user_agent_header(browser_type='webkit') + headers = header_generator.get_user_agent_header(browser_type=browser_type) assert 'User-Agent' in headers - assert headers['User-Agent'] == PW_WEBKIT_HEADLESS_DEFAULT_USER_AGENT + assert any(keyword in headers['User-Agent'] for keyword in BROWSER_TYPE_HEADER_KEYWORD[browser_type]) def test_get_user_agent_header_invalid_browser_type() -> None: From f3606033fcef0682d4f0811f0dd8036b4720fcb3 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 29 Jan 2025 09:33:57 +0100 Subject: [PATCH 28/32] Tests in progress --- .../fingerprint_suite/_browserforge_adapter.py | 13 ++++++++++++- src/crawlee/fingerprint_suite/_consts.py | 1 - src/crawlee/fingerprint_suite/_header_generator.py | 12 +++++++----- .../unit/fingerprint_suite/test_header_generator.py | 7 ++++++- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/crawlee/fingerprint_suite/_browserforge_adapter.py b/src/crawlee/fingerprint_suite/_browserforge_adapter.py index 14ea9b377f..dcd2c2830a 100644 --- a/src/crawlee/fingerprint_suite/_browserforge_adapter.py +++ b/src/crawlee/fingerprint_suite/_browserforge_adapter.py @@ -1,11 +1,14 @@ from __future__ import annotations +import json from copy import deepcopy from typing import TYPE_CHECKING, Any +from browserforge.bayesian_network import extract_json from browserforge.fingerprints import Fingerprint as bf_Fingerprint from browserforge.fingerprints import FingerprintGenerator as bf_FingerprintGenerator from browserforge.fingerprints import Screen +from browserforge.headers.generator import DATA_DIR from browserforge.headers.generator import HeaderGenerator as bf_HeaderGenerator from typing_extensions import override @@ -85,7 +88,7 @@ class BrowserforgeHeaderGenerator: def __init__(self): self._generator = bf_HeaderGenerator() - def generate(self, browser_type: SupportedBrowserType) -> bf_Fingerprint: + def generate(self, browser_type: SupportedBrowserType = "chromium") -> bf_Fingerprint: # browserforge header generation can be flaky. Enforce basic QA on generated headers max_attempts = 10 @@ -100,3 +103,11 @@ def generate(self, browser_type: SupportedBrowserType) -> bf_Fingerprint: return generated_header print(generated_header['User-Agent']) raise RuntimeError('Failed to generate header.') + + + +def get_user_agent_pool() -> set[str]: + """Get set of `User-Agent` strings available to browserforge.""" + header_network = extract_json((DATA_DIR / "header-network.zip")) + return set(header_network['nodes'][6]['possibleValues']) | set(header_network['nodes'][7]['possibleValues']) + diff --git a/src/crawlee/fingerprint_suite/_consts.py b/src/crawlee/fingerprint_suite/_consts.py index d510f11078..bf49ee91cf 100644 --- a/src/crawlee/fingerprint_suite/_consts.py +++ b/src/crawlee/fingerprint_suite/_consts.py @@ -17,7 +17,6 @@ ) PW_WEBKIT_HEADLESS_DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15' - BROWSER_TYPE_HEADER_KEYWORD = { 'chromium':{'Chrome', 'CriOS'}, 'firefox':{'Firefox', 'FxiOS'}, diff --git a/src/crawlee/fingerprint_suite/_header_generator.py b/src/crawlee/fingerprint_suite/_header_generator.py index 06de832064..c075001c81 100644 --- a/src/crawlee/fingerprint_suite/_header_generator.py +++ b/src/crawlee/fingerprint_suite/_header_generator.py @@ -22,6 +22,10 @@ class HeaderGenerator: def __init__(self): self._generator = BrowserforgeHeaderGenerator() + def _get_specific_headers(self, all_headers: dict[str,str], header_names: set[str]) -> HttpHeaders: + return HttpHeaders({key:value for key, value in all_headers.items() if key in header_names}) + + def get_common_headers(self) -> HttpHeaders: """Get common HTTP headers ("Accept", "Accept-Language"). @@ -29,12 +33,12 @@ def get_common_headers(self) -> HttpHeaders: by the HTTP client or browser. """ all_headers = self._generator.generate() - return HttpHeaders({key:value for key, value in all_headers.items() if key in {'Accept', 'Accept-Language'}}) + return self._get_specific_headers(all_headers, header_names={'Accept', 'Accept-Language'}) def get_random_user_agent_header(self) -> HttpHeaders: """Get a random User-Agent header.""" all_headers = self._generator.generate() - return HttpHeaders({'User-Agent':all_headers['User-Agent']}) + return self._get_specific_headers(all_headers, header_names={'User-Agent'}) def get_user_agent_header( @@ -43,12 +47,10 @@ def get_user_agent_header( browser_type: SupportedBrowserType = 'chromium', ) -> HttpHeaders: """Get the User-Agent header based on the browser type.""" - headers = dict[str, str]() - if browser_type not in {'chromium', 'firefox', 'webkit', 'edge'}: raise ValueError(f'Unsupported browser type: {browser_type}') all_headers = self._generator.generate(browser_type=browser_type) - return HttpHeaders({'User-Agent': all_headers['User-Agent']}) + return self._get_specific_headers(all_headers, header_names={'User-Agent'}) def get_sec_ch_ua_headers( self, diff --git a/tests/unit/fingerprint_suite/test_header_generator.py b/tests/unit/fingerprint_suite/test_header_generator.py index 532ad96a51..b818e56ab6 100644 --- a/tests/unit/fingerprint_suite/test_header_generator.py +++ b/tests/unit/fingerprint_suite/test_header_generator.py @@ -3,6 +3,7 @@ import pytest from crawlee.fingerprint_suite import HeaderGenerator +from crawlee.fingerprint_suite._browserforge_adapter import get_user_agent_pool from crawlee.fingerprint_suite._consts import ( BROWSER_TYPE_HEADER_KEYWORD, PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA, @@ -11,6 +12,9 @@ ) from crawlee.fingerprint_suite._types import SupportedBrowserType +@pytest.fixture(scope="session") +def user_agents_pool(): + return get_user_agent_pool() def test_get_common_headers() -> None: header_generator = HeaderGenerator() @@ -34,7 +38,7 @@ def test_get_random_user_agent_header() -> None: @pytest.mark.parametrize('browser_type', [ 'chromium','firefox','edge','webkit' ]) -def test_get_user_agent_header_stress_test(browser_type: SupportedBrowserType,_) -> None: +def test_get_user_agent_header_stress_test(browser_type: SupportedBrowserType,user_agents_pool,_) -> None: """Test that the User-Agent header is consistently generated correctly. (Very fast even when stress tested.)""" @@ -43,6 +47,7 @@ def test_get_user_agent_header_stress_test(browser_type: SupportedBrowserType,_) assert 'User-Agent' in headers assert any(keyword in headers['User-Agent'] for keyword in BROWSER_TYPE_HEADER_KEYWORD[browser_type]) + assert headers['User-Agent'] in user_agents_pool def test_get_user_agent_header_invalid_browser_type() -> None: From 11cc9139872d46bde559cb51ad3508a7b4675357 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 29 Jan 2025 16:03:36 +0100 Subject: [PATCH 29/32] Wait for Accept Language fix in browserforge --- .../_browserforge_adapter.py | 30 +++++----- src/crawlee/fingerprint_suite/_consts.py | 19 ++----- .../fingerprint_suite/_header_generator.py | 37 +++--------- tests/unit/conftest.py | 6 ++ .../_playwright/test_playwright_crawler.py | 39 ++++++------- .../test_header_generator.py | 57 +++++++++---------- tests/unit/http_clients/test_httpx.py | 7 ++- 7 files changed, 83 insertions(+), 112 deletions(-) diff --git a/src/crawlee/fingerprint_suite/_browserforge_adapter.py b/src/crawlee/fingerprint_suite/_browserforge_adapter.py index dcd2c2830a..39603615d0 100644 --- a/src/crawlee/fingerprint_suite/_browserforge_adapter.py +++ b/src/crawlee/fingerprint_suite/_browserforge_adapter.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json from copy import deepcopy from typing import TYPE_CHECKING, Any @@ -84,30 +83,31 @@ def generate(self) -> bf_Fingerprint: @docs_group('Classes') class BrowserforgeHeaderGenerator: + def __init__(self) -> None: + self._generator = bf_HeaderGenerator(locale=["en-Us", "en"]) - def __init__(self): - self._generator = bf_HeaderGenerator() - - def generate(self, browser_type: SupportedBrowserType = "chromium") -> bf_Fingerprint: + def generate(self, browser_type: SupportedBrowserType = 'chromium') -> dict[str, str]: # browserforge header generation can be flaky. Enforce basic QA on generated headers max_attempts = 10 - if browser_type=='webkit': - bf_browser_type = 'safari' - else: - bf_browser_type = browser_type + bf_browser_type = 'safari' if browser_type == 'webkit' else browser_type - for attempt in range(max_attempts): + for _attempt in range(max_attempts): generated_header = self._generator.generate(browser=bf_browser_type) if any(keyword in generated_header['User-Agent'] for keyword in BROWSER_TYPE_HEADER_KEYWORD[browser_type]): return generated_header - print(generated_header['User-Agent']) raise RuntimeError('Failed to generate header.') +def get_available_header_network() -> dict: + """Get header network that contains possible header values.""" + return extract_json(DATA_DIR / 'header-network.zip') -def get_user_agent_pool() -> set[str]: - """Get set of `User-Agent` strings available to browserforge.""" - header_network = extract_json((DATA_DIR / "header-network.zip")) - return set(header_network['nodes'][6]['possibleValues']) | set(header_network['nodes'][7]['possibleValues']) +def get_available_header_values(header_network: dict, node_name: str | set[str]) -> set[str]: + """Get set of possible header values from available header network.""" + node_names = {node_name} if isinstance(node_name, str) else node_name + for node in header_network['nodes']: + if node['name'] in node_names: + return set(node['possibleValues']) + return None diff --git a/src/crawlee/fingerprint_suite/_consts.py b/src/crawlee/fingerprint_suite/_consts.py index bf49ee91cf..b375c72ea9 100644 --- a/src/crawlee/fingerprint_suite/_consts.py +++ b/src/crawlee/fingerprint_suite/_consts.py @@ -6,20 +6,9 @@ COMMON_ACCEPT_LANGUAGE = 'en-US,en;q=0.9' -# Playwright default headers (user-agents and sec-ch) for headless browsers. -PW_CHROMIUM_HEADLESS_DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36' -PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA = '"Not=A?Brand";v="8", "Chromium";v="124", "Google Chrome";v="124"' -PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_MOBILE = '?0' -PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_PLATFORM = '"macOS"' - -PW_FIREFOX_HEADLESS_DEFAULT_USER_AGENT = ( - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv125.0) Gecko/20100101 Firefox/125.0' -) -PW_WEBKIT_HEADLESS_DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15' - BROWSER_TYPE_HEADER_KEYWORD = { - 'chromium':{'Chrome', 'CriOS'}, - 'firefox':{'Firefox', 'FxiOS'}, - 'edge':{'Edg', 'Edge', 'EdgA', 'EdgiOS'}, - 'webkit':{'Safari'}, + 'chromium': {'Chrome', 'CriOS'}, + 'firefox': {'Firefox', 'FxiOS'}, + 'edge': {'Edg', 'Edge', 'EdgA', 'EdgiOS'}, + 'webkit': {'Safari'}, } diff --git a/src/crawlee/fingerprint_suite/_header_generator.py b/src/crawlee/fingerprint_suite/_header_generator.py index c075001c81..0d8aaba3d1 100644 --- a/src/crawlee/fingerprint_suite/_header_generator.py +++ b/src/crawlee/fingerprint_suite/_header_generator.py @@ -5,11 +5,6 @@ from crawlee._types import HttpHeaders from crawlee._utils.docs import docs_group from crawlee.fingerprint_suite._browserforge_adapter import BrowserforgeHeaderGenerator -from crawlee.fingerprint_suite._consts import ( - PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA, - PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_MOBILE, - PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_PLATFORM, -) if TYPE_CHECKING: from crawlee.fingerprint_suite._types import SupportedBrowserType @@ -19,12 +14,11 @@ class HeaderGenerator: """Generates realistic looking or browser-like HTTP headers.""" - def __init__(self): + def __init__(self) -> None: self._generator = BrowserforgeHeaderGenerator() - def _get_specific_headers(self, all_headers: dict[str,str], header_names: set[str]) -> HttpHeaders: - return HttpHeaders({key:value for key, value in all_headers.items() if key in header_names}) - + def _get_specific_headers(self, all_headers: dict[str, str], header_names: set[str]) -> HttpHeaders: + return HttpHeaders({key: value for key, value in all_headers.items() if key in header_names}) def get_common_headers(self) -> HttpHeaders: """Get common HTTP headers ("Accept", "Accept-Language"). @@ -40,7 +34,6 @@ def get_random_user_agent_header(self) -> HttpHeaders: all_headers = self._generator.generate() return self._get_specific_headers(all_headers, header_names={'User-Agent'}) - def get_user_agent_header( self, *, @@ -57,22 +50,10 @@ def get_sec_ch_ua_headers( *, browser_type: SupportedBrowserType = 'chromium', ) -> HttpHeaders: - """Get the Sec-Ch-Ua headers based on the browser type.""" - headers = dict[str, str]() - - if browser_type == 'chromium': - # Currently, only Chromium uses Sec-Ch-Ua headers. - headers['Sec-Ch-Ua'] = PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA - headers['Sec-Ch-Ua-Mobile'] = PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_MOBILE - headers['Sec-Ch-Ua-Platform'] = PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_PLATFORM - - elif browser_type == 'firefox': # noqa: SIM114 - pass - - elif browser_type == 'webkit': - pass - - else: + """Get the sec-ch-ua headers based on the browser type.""" + if browser_type not in {'chromium', 'firefox', 'webkit', 'edge'}: raise ValueError(f'Unsupported browser type: {browser_type}') - - return HttpHeaders(headers) + all_headers = self._generator.generate(browser_type=browser_type) + return self._get_specific_headers( + all_headers, header_names={'sec-ch-ua', 'sec-ch-ua-mobile', 'sec-ch-ua-platform'} + ) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index f8ef842ab4..a736ac51f6 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -12,6 +12,7 @@ from crawlee import service_locator from crawlee.configuration import Configuration +from crawlee.fingerprint_suite._browserforge_adapter import get_available_header_network from crawlee.proxy_configuration import ProxyInfo from crawlee.storage_clients import MemoryStorageClient from crawlee.storages import _creation_management @@ -176,3 +177,8 @@ def memory_storage_client(tmp_path: Path) -> MemoryStorageClient: ) return MemoryStorageClient.from_config(config) + + +@pytest.fixture(scope='session') +def header_network() -> dict: + return get_available_header_network() diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py index 4c426097fb..ac596577c3 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py @@ -16,14 +16,9 @@ HeaderGeneratorOptions, ScreenOptions, ) -from crawlee.fingerprint_suite._consts import ( - PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA, - PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_MOBILE, - PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_PLATFORM, - PW_CHROMIUM_HEADLESS_DEFAULT_USER_AGENT, - PW_FIREFOX_HEADLESS_DEFAULT_USER_AGENT, -) +from crawlee.fingerprint_suite._consts import BROWSER_TYPE_HEADER_KEYWORD from crawlee.proxy_configuration import ProxyConfiguration +from crawlee.fingerprint_suite._browserforge_adapter import get_available_header_values if TYPE_CHECKING: from yarl import URL @@ -109,8 +104,9 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: assert handled_urls == set() -async def test_chromium_headless_headers(httpbin: URL) -> None: - crawler = PlaywrightCrawler(headless=True, browser_type='chromium') +async def test_chromium_headless_headers(httpbin: URL, header_network: dict) -> None: + browser_type = 'chromium' + crawler = PlaywrightCrawler(headless=True, browser_type=browser_type) headers = dict[str, str]() @crawler.router.default_handler @@ -123,22 +119,21 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: await crawler.run([str(httpbin / 'get')]) - assert 'User-Agent' in headers - assert 'Sec-Ch-Ua' in headers - assert 'Sec-Ch-Ua-Mobile' in headers - assert 'Sec-Ch-Ua-Platform' in headers + user_agent = headers.get('User-Agent') + assert user_agent in get_available_header_values(header_network, {'user-agent', 'User-Agent'}) + assert any(keyword in user_agent for keyword in BROWSER_TYPE_HEADER_KEYWORD[browser_type]) + + assert headers.get('Sec-Ch-Ua') in get_available_header_values(header_network, 'sec-ch-ua') + assert headers.get('Sec-Ch-Ua-Mobile') in get_available_header_values(header_network, 'sec-ch-ua-mobile') + assert headers.get('Sec-Ch-Ua-Platform') in get_available_header_values(header_network, 'sec-ch-ua-platform') assert 'headless' not in headers['Sec-Ch-Ua'].lower() assert 'headless' not in headers['User-Agent'].lower() - assert headers['Sec-Ch-Ua'] == PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA - assert headers['Sec-Ch-Ua-Mobile'] == PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_MOBILE - assert headers['Sec-Ch-Ua-Platform'] == PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_PLATFORM - assert headers['User-Agent'] == PW_CHROMIUM_HEADLESS_DEFAULT_USER_AGENT - -async def test_firefox_headless_headers(httpbin: URL) -> None: - crawler = PlaywrightCrawler(headless=True, browser_type='firefox') +async def test_firefox_headless_headers(httpbin: URL, header_network: dict) -> None: + browser_type = 'firefox' + crawler = PlaywrightCrawler(headless=True, browser_type=browser_type) headers = dict[str, str]() @crawler.router.default_handler @@ -158,7 +153,9 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: assert 'headless' not in headers['User-Agent'].lower() - assert headers['User-Agent'] == PW_FIREFOX_HEADLESS_DEFAULT_USER_AGENT + user_agent = headers.get('User-Agent') + assert user_agent in get_available_header_values(header_network, {'user-agent', 'User-Agent'}) + assert any(keyword in user_agent for keyword in BROWSER_TYPE_HEADER_KEYWORD[browser_type]) async def test_custom_headers(httpbin: URL) -> None: diff --git a/tests/unit/fingerprint_suite/test_header_generator.py b/tests/unit/fingerprint_suite/test_header_generator.py index b818e56ab6..bea5e837a1 100644 --- a/tests/unit/fingerprint_suite/test_header_generator.py +++ b/tests/unit/fingerprint_suite/test_header_generator.py @@ -1,20 +1,18 @@ from __future__ import annotations +from typing import TYPE_CHECKING + import pytest from crawlee.fingerprint_suite import HeaderGenerator -from crawlee.fingerprint_suite._browserforge_adapter import get_user_agent_pool +from crawlee.fingerprint_suite._browserforge_adapter import get_available_header_values from crawlee.fingerprint_suite._consts import ( BROWSER_TYPE_HEADER_KEYWORD, - PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA, - PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_MOBILE, - PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_PLATFORM, ) -from crawlee.fingerprint_suite._types import SupportedBrowserType -@pytest.fixture(scope="session") -def user_agents_pool(): - return get_user_agent_pool() +if TYPE_CHECKING: + from crawlee.fingerprint_suite._types import SupportedBrowserType + def test_get_common_headers() -> None: header_generator = HeaderGenerator() @@ -33,21 +31,18 @@ def test_get_random_user_agent_header() -> None: assert headers['User-Agent'] - -@pytest.mark.parametrize('_', range(100)) -@pytest.mark.parametrize('browser_type', [ - 'chromium','firefox','edge','webkit' -]) -def test_get_user_agent_header_stress_test(browser_type: SupportedBrowserType,user_agents_pool,_) -> None: +@pytest.mark.parametrize('browser_type', ['chromium', 'firefox', 'edge', 'webkit']) +def test_get_user_agent_header_stress_test(browser_type: SupportedBrowserType, header_network: dict) -> None: """Test that the User-Agent header is consistently generated correctly. (Very fast even when stress tested.)""" - header_generator = HeaderGenerator() - headers = header_generator.get_user_agent_header(browser_type=browser_type) + for _ in range(100): + header_generator = HeaderGenerator() + headers = header_generator.get_user_agent_header(browser_type=browser_type) - assert 'User-Agent' in headers - assert any(keyword in headers['User-Agent'] for keyword in BROWSER_TYPE_HEADER_KEYWORD[browser_type]) - assert headers['User-Agent'] in user_agents_pool + assert 'User-Agent' in headers + assert any(keyword in headers['User-Agent'] for keyword in BROWSER_TYPE_HEADER_KEYWORD[browser_type]) + assert headers['User-Agent'] in get_available_header_values(header_network, {'user-agent', 'User-Agent'}) def test_get_user_agent_header_invalid_browser_type() -> None: @@ -58,21 +53,23 @@ def test_get_user_agent_header_invalid_browser_type() -> None: header_generator.get_user_agent_header(browser_type='invalid_browser') # type: ignore[arg-type] -def test_get_sec_ch_ua_headers_chromium() -> None: - """Test that Sec-Ch-Ua headers are generated correctly for Chromium.""" +def test_get_sec_ch_ua_headers_chromium(header_network: dict) -> None: + """Test that sec-ch-ua headers are generated correctly for Chromium.""" header_generator = HeaderGenerator() headers = header_generator.get_sec_ch_ua_headers(browser_type='chromium') - assert 'Sec-Ch-Ua' in headers - assert headers['Sec-Ch-Ua'] == PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA - assert 'Sec-Ch-Ua-Mobile' in headers - assert headers['Sec-Ch-Ua-Mobile'] == PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_MOBILE - assert 'Sec-Ch-Ua-Platform' in headers - assert headers['Sec-Ch-Ua-Platform'] == PW_CHROMIUM_HEADLESS_DEFAULT_SEC_CH_UA_PLATFORM + assert 'sec-ch-ua' in headers + assert headers['sec-ch-ua'] in get_available_header_values(header_network, 'sec-ch-ua') + + assert 'sec-ch-ua-mobile' in headers + assert headers['sec-ch-ua-mobile'] in get_available_header_values(header_network, 'sec-ch-ua-mobile') + + assert 'sec-ch-ua-platform' in headers + assert headers['sec-ch-ua-platform'] in get_available_header_values(header_network, 'sec-ch-ua-platform') def test_get_sec_ch_ua_headers_firefox() -> None: - """Test that Sec-Ch-Ua headers are not generated for Firefox.""" + """Test that sec-ch-ua headers are not generated for Firefox.""" header_generator = HeaderGenerator() headers = header_generator.get_sec_ch_ua_headers(browser_type='firefox') @@ -80,7 +77,7 @@ def test_get_sec_ch_ua_headers_firefox() -> None: def test_get_sec_ch_ua_headers_webkit() -> None: - """Test that Sec-Ch-Ua headers are not generated for WebKit.""" + """Test that sec-ch-ua headers are not generated for WebKit.""" header_generator = HeaderGenerator() headers = header_generator.get_sec_ch_ua_headers(browser_type='webkit') @@ -88,7 +85,7 @@ def test_get_sec_ch_ua_headers_webkit() -> None: def test_get_sec_ch_ua_headers_invalid_browser_type() -> None: - """Test that an invalid browser type raises a ValueError for Sec-Ch-Ua headers.""" + """Test that an invalid browser type raises a ValueError for sec-ch-ua headers.""" header_generator = HeaderGenerator() with pytest.raises(ValueError, match='Unsupported browser type'): diff --git a/tests/unit/http_clients/test_httpx.py b/tests/unit/http_clients/test_httpx.py index 2224a0c146..f7ce86ae83 100644 --- a/tests/unit/http_clients/test_httpx.py +++ b/tests/unit/http_clients/test_httpx.py @@ -8,7 +8,8 @@ from crawlee import Request from crawlee.errors import ProxyError -from crawlee.fingerprint_suite._consts import COMMON_ACCEPT, COMMON_ACCEPT_LANGUAGE, USER_AGENT_POOL +from crawlee.fingerprint_suite._browserforge_adapter import get_available_header_values +from crawlee.fingerprint_suite._consts import COMMON_ACCEPT, COMMON_ACCEPT_LANGUAGE from crawlee.http_clients import HttpxHttpClient from crawlee.statistics import Statistics @@ -88,7 +89,7 @@ async def test_send_request_with_proxy_disabled( await http_client.send_request(url, proxy_info=disabled_proxy) -async def test_common_headers_and_user_agent(httpbin: URL) -> None: +async def test_common_headers_and_user_agent(httpbin: URL, header_network: dict) -> None: client = HttpxHttpClient() response = await client.send_request(str(httpbin / 'get')) @@ -104,4 +105,4 @@ async def test_common_headers_and_user_agent(httpbin: URL) -> None: # By default, HTTPX uses its own User-Agent, which should be replaced by the one from the header generator. assert 'User-Agent' in response_headers assert 'python-httpx' not in response_headers['User-Agent'] - assert response_headers['User-Agent'] in USER_AGENT_POOL + assert response_headers['User-Agent'] in get_available_header_values(header_network, {'User-Agent', 'user-agent'}) From 5f9192e23d5364de8c08c6d2453929438c8ed6f1 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 5 Feb 2025 09:05:10 +0100 Subject: [PATCH 30/32] Workaround for missing changes in upstream repo --- .../_browserforge_adapter.py | 58 +++++++++++++++++-- .../_playwright/test_playwright_crawler.py | 7 ++- 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/src/crawlee/fingerprint_suite/_browserforge_adapter.py b/src/crawlee/fingerprint_suite/_browserforge_adapter.py index 39603615d0..10be5413c2 100644 --- a/src/crawlee/fingerprint_suite/_browserforge_adapter.py +++ b/src/crawlee/fingerprint_suite/_browserforge_adapter.py @@ -17,9 +17,56 @@ from ._fingerprint_generator import FingerprintGenerator if TYPE_CHECKING: + from camoufox.utils import ListOrString + from ._types import HeaderGeneratorOptions, ScreenOptions, SupportedBrowserType +class PatchedHeaderGenerator(bf_HeaderGenerator): + """Browserforge `HeaderGenerator` that contains patches not accepted in upstream repo.""" + + def _get_accept_language_header(self, locales: ListOrString) -> str: + """Generates the Accept-Language header based on the given locales. + + Patched version due to PR of upstream repo not being merged: https://github.com/daijro/browserforge/pull/24 + + Parameters: + locales (ListOrString): Locale(s). + + Returns: + str: Accept-Language header string. + """ + # First locale does not include quality factor, q=1 is considered as implicit. + additional_locales = [f'{locale};q={0.9 - index * 0.1:.1f}' for index, locale in enumerate(locales[1:])] + return ','.join((locales[0], *additional_locales)) + + +class PatchedFingerprintGenerator(bf_FingerprintGenerator): + """Browserforge `FingerprintGenerator` that contains patches not accepted in upstream repo.""" + + def __init__( # type:ignore[no-untyped-def] # Upstream repo types missing. + self, + *, + screen: Screen | None = None, + strict: bool = False, + mock_webrtc: bool = False, + slim: bool = False, + **header_kwargs, # noqa:ANN003 # Upstream repo types missing. + ) -> None: + """Initializes the FingerprintGenerator with the given options. + + Parameters: + screen (Screen, optional): Screen constraints for the generated fingerprint. + strict (bool, optional): Whether to raise an exception if the constraints are too strict. Default is False. + mock_webrtc (bool, optional): Whether to mock WebRTC when injecting the fingerprint. Default is False. + slim (bool, optional): Disables performance-heavy evasions when injecting the fingerprint. Default is False. + **header_kwargs: Header generation options for HeaderGenerator + """ + super().__init__(screen=screen, strict=strict, mock_webrtc=mock_webrtc, slim=slim) + # Replace `self.header_generator` To make sure that we consistently use `PatchedHeaderGenerator` + self.header_generator = PatchedHeaderGenerator(**header_kwargs) + + @docs_group('Classes') class BrowserforgeFingerprintGenerator(FingerprintGenerator): """`FingerprintGenerator` adapter for fingerprint generator from `browserforge`.""" @@ -63,7 +110,7 @@ def __init__( bf_options['screen'] = Screen(**screen_options.model_dump()) self._options = {**bf_options, **bf_header_options} - self._generator = bf_FingerprintGenerator() + self._generator = PatchedFingerprintGenerator() @override def generate(self) -> bf_Fingerprint: @@ -81,10 +128,11 @@ def generate(self) -> bf_Fingerprint: raise RuntimeError('Failed to generate fingerprint.') -@docs_group('Classes') class BrowserforgeHeaderGenerator: + """`HeaderGenerator` adapter for fingerprint generator from `browserforge`.""" + def __init__(self) -> None: - self._generator = bf_HeaderGenerator(locale=["en-Us", "en"]) + self._generator = PatchedHeaderGenerator(locale=['en-US', 'en']) def generate(self, browser_type: SupportedBrowserType = 'chromium') -> dict[str, str]: # browserforge header generation can be flaky. Enforce basic QA on generated headers @@ -93,7 +141,7 @@ def generate(self, browser_type: SupportedBrowserType = 'chromium') -> dict[str, bf_browser_type = 'safari' if browser_type == 'webkit' else browser_type for _attempt in range(max_attempts): - generated_header = self._generator.generate(browser=bf_browser_type) + generated_header: dict[str, str] = self._generator.generate(browser=bf_browser_type) if any(keyword in generated_header['User-Agent'] for keyword in BROWSER_TYPE_HEADER_KEYWORD[browser_type]): return generated_header raise RuntimeError('Failed to generate header.') @@ -110,4 +158,4 @@ def get_available_header_values(header_network: dict, node_name: str | set[str]) for node in header_network['nodes']: if node['name'] in node_names: return set(node['possibleValues']) - return None + return set() diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py index ac596577c3..e36d19e7c0 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py @@ -16,13 +16,14 @@ HeaderGeneratorOptions, ScreenOptions, ) +from crawlee.fingerprint_suite._browserforge_adapter import get_available_header_values from crawlee.fingerprint_suite._consts import BROWSER_TYPE_HEADER_KEYWORD from crawlee.proxy_configuration import ProxyConfiguration -from crawlee.fingerprint_suite._browserforge_adapter import get_available_header_values if TYPE_CHECKING: from yarl import URL + from crawlee.browsers._types import BrowserType from crawlee.crawlers import PlaywrightCrawlingContext, PlaywrightPreNavCrawlingContext @@ -105,7 +106,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: async def test_chromium_headless_headers(httpbin: URL, header_network: dict) -> None: - browser_type = 'chromium' + browser_type: BrowserType = 'chromium' crawler = PlaywrightCrawler(headless=True, browser_type=browser_type) headers = dict[str, str]() @@ -132,7 +133,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: async def test_firefox_headless_headers(httpbin: URL, header_network: dict) -> None: - browser_type = 'firefox' + browser_type: BrowserType = 'firefox' crawler = PlaywrightCrawler(headless=True, browser_type=browser_type) headers = dict[str, str]() From b7d3427b135bb31a2ff6c90fb1867f8c4dff9e79 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 6 Feb 2025 09:21:12 +0100 Subject: [PATCH 31/32] Poetry lock from master --- poetry.lock | 545 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 447 insertions(+), 98 deletions(-) diff --git a/poetry.lock b/poetry.lock index d53122af33..074b8efbd0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -71,18 +71,18 @@ scrapy = ["scrapy (>=2.11.0)"] [[package]] name = "apify-client" -version = "1.8.1" +version = "1.9.0" description = "Apify API client for Python" optional = true python-versions = "<4.0,>=3.9" files = [ - {file = "apify_client-1.8.1-py3-none-any.whl", hash = "sha256:cfa6df3816c436204e37457fba28981a0ef6a7602cde372463f0f078eee64747"}, - {file = "apify_client-1.8.1.tar.gz", hash = "sha256:2be1be7879570655bddeebf126833efe94cabb95b3755592845e92c20c70c674"}, + {file = "apify_client-1.9.0-py3-none-any.whl", hash = "sha256:dd67093c570cea068ac5ad4100f67d57f1aeb217f6f887b369420f913ef597e7"}, + {file = "apify_client-1.9.0.tar.gz", hash = "sha256:f57ec6a2d6b978daa48ffc470e31cfa586ab82145a88acfd1fa79b5857013ddb"}, ] [package.dependencies] apify-shared = ">=1.1.2" -httpx = ">=0.25.0" +httpx = ">=0.25" more_itertools = ">=10.0.0" [[package]] @@ -135,17 +135,18 @@ test = ["astroid (>=2,<4)", "pytest", "pytest-cov", "pytest-xdist"] [[package]] name = "beautifulsoup4" -version = "4.12.3" +version = "4.13.3" description = "Screen-scraping library" optional = true -python-versions = ">=3.6.0" +python-versions = ">=3.7.0" files = [ - {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"}, - {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"}, + {file = "beautifulsoup4-4.13.3-py3-none-any.whl", hash = "sha256:99045d7d3f08f91f0d656bc9b7efbae189426cd913d830294a15eefa0ea4df16"}, + {file = "beautifulsoup4-4.13.3.tar.gz", hash = "sha256:1bd32405dacc920b42b83ba01644747ed77456a65760e285fbc47633ceddaf8b"}, ] [package.dependencies] soupsieve = ">1.2" +typing-extensions = ">=4.0.0" [package.extras] cchardet = ["cchardet"] @@ -405,13 +406,13 @@ cffi = ">=1.0.0" [[package]] name = "browserforge" -version = "1.2.1" +version = "1.2.3" description = "Intelligent browser header & fingerprint generator" optional = true python-versions = "<4.0,>=3.8" files = [ - {file = "browserforge-1.2.1-py3-none-any.whl", hash = "sha256:b2813b4de80b9c48c88700c93e3dfa6a64694d04f3263545e28bb03dd95df27e"}, - {file = "browserforge-1.2.1.tar.gz", hash = "sha256:7036d73fb066a4361a015b619079474c42d8b0ff415e1d874b62366de48d0b61"}, + {file = "browserforge-1.2.3-py3-none-any.whl", hash = "sha256:a6c71ed4688b2f1b0bee757ca82ddad0007cbba68a71eca66ca607dde382f132"}, + {file = "browserforge-1.2.3.tar.gz", hash = "sha256:d5bec6dffd4748b30fbac9f9c1ef33b26c01a23185240bf90011843e174b7ecc"}, ] [package.dependencies] @@ -448,13 +449,13 @@ virtualenv = ["virtualenv (>=20.0.35)"] [[package]] name = "certifi" -version = "2024.12.14" +version = "2025.1.31" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.12.14-py3-none-any.whl", hash = "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56"}, - {file = "certifi-2024.12.14.tar.gz", hash = "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"}, + {file = "certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe"}, + {file = "certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651"}, ] [[package]] @@ -931,20 +932,20 @@ files = [ [[package]] name = "deprecated" -version = "1.2.15" +version = "1.2.18" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" files = [ - {file = "Deprecated-1.2.15-py2.py3-none-any.whl", hash = "sha256:353bc4a8ac4bfc96800ddab349d89c25dec1079f65fd53acdcc1e0b975b21320"}, - {file = "deprecated-1.2.15.tar.gz", hash = "sha256:683e561a90de76239796e6b6feac66b99030d2dd3fcf61ef996330f14bbb9b0d"}, + {file = "Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec"}, + {file = "deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d"}, ] [package.dependencies] wrapt = ">=1.10,<2" [package.extras] -dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "jinja2 (>=3.0.3,<3.1.0)", "setuptools", "sphinx (<2)", "tox"] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools", "tox"] [[package]] name = "distlib" @@ -1199,18 +1200,18 @@ files = [ [[package]] name = "h2" -version = "4.1.0" -description = "HTTP/2 State-Machine based protocol implementation" +version = "4.2.0" +description = "Pure-Python HTTP/2 protocol implementation" optional = false -python-versions = ">=3.6.1" +python-versions = ">=3.9" files = [ - {file = "h2-4.1.0-py3-none-any.whl", hash = "sha256:03a46bcf682256c95b5fd9e9a99c1323584c3eec6440d379b9903d709476bc6d"}, - {file = "h2-4.1.0.tar.gz", hash = "sha256:a83aca08fbe7aacb79fec788c9c0bac936343560ed9ec18b82a13a12c28d2abb"}, + {file = "h2-4.2.0-py3-none-any.whl", hash = "sha256:479a53ad425bb29af087f3458a61d30780bc818e4ebcf01f0b536ba916462ed0"}, + {file = "h2-4.2.0.tar.gz", hash = "sha256:c8a52129695e88b1a0578d8d2cc6842bbd79128ac685463b887ee278126ad01f"}, ] [package.dependencies] -hpack = ">=4.0,<5" -hyperframe = ">=6.0,<7" +hpack = ">=4.1,<5" +hyperframe = ">=6.1,<7" [[package]] name = "hpack" @@ -1435,6 +1436,16 @@ qtconsole = ["qtconsole"] test = ["pickleshare", "pytest (<7.1)", "pytest-asyncio (<0.22)", "testpath"] test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.22)", "pandas", "pickleshare", "pytest (<7.1)", "pytest-asyncio (<0.22)", "testpath", "trio"] +[[package]] +name = "jaro-winkler" +version = "2.0.3" +description = "Original, standard and customisable versions of the Jaro-Winkler functions." +optional = true +python-versions = "*" +files = [ + {file = "jaro_winkler-2.0.3-py3-none-any.whl", hash = "sha256:9ad42a94eb110351e72dd5b9e0a0f1053b0760761d676f9be35da19ea80d511b"}, +] + [[package]] name = "jedi" version = "0.19.2" @@ -1496,6 +1507,17 @@ files = [ {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] +[[package]] +name = "joblib" +version = "1.4.2" +description = "Lightweight pipelining with Python functions" +optional = true +python-versions = ">=3.8" +files = [ + {file = "joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6"}, + {file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"}, +] + [[package]] name = "lazy-object-proxy" version = "1.10.0" @@ -1932,49 +1954,43 @@ typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} [[package]] name = "mypy" -version = "1.14.1" +version = "1.15.0" description = "Optional static typing for Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "mypy-1.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:52686e37cf13d559f668aa398dd7ddf1f92c5d613e4f8cb262be2fb4fedb0fcb"}, - {file = "mypy-1.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1fb545ca340537d4b45d3eecdb3def05e913299ca72c290326be19b3804b39c0"}, - {file = "mypy-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90716d8b2d1f4cd503309788e51366f07c56635a3309b0f6a32547eaaa36a64d"}, - {file = "mypy-1.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ae753f5c9fef278bcf12e1a564351764f2a6da579d4a81347e1d5a15819997b"}, - {file = "mypy-1.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e0fe0f5feaafcb04505bcf439e991c6d8f1bf8b15f12b05feeed96e9e7bf1427"}, - {file = "mypy-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:7d54bd85b925e501c555a3227f3ec0cfc54ee8b6930bd6141ec872d1c572f81f"}, - {file = "mypy-1.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f995e511de847791c3b11ed90084a7a0aafdc074ab88c5a9711622fe4751138c"}, - {file = "mypy-1.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d64169ec3b8461311f8ce2fd2eb5d33e2d0f2c7b49116259c51d0d96edee48d1"}, - {file = "mypy-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba24549de7b89b6381b91fbc068d798192b1b5201987070319889e93038967a8"}, - {file = "mypy-1.14.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:183cf0a45457d28ff9d758730cd0210419ac27d4d3f285beda038c9083363b1f"}, - {file = "mypy-1.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f2a0ecc86378f45347f586e4163d1769dd81c5a223d577fe351f26b179e148b1"}, - {file = "mypy-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:ad3301ebebec9e8ee7135d8e3109ca76c23752bac1e717bc84cd3836b4bf3eae"}, - {file = "mypy-1.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:30ff5ef8519bbc2e18b3b54521ec319513a26f1bba19a7582e7b1f58a6e69f14"}, - {file = "mypy-1.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cb9f255c18052343c70234907e2e532bc7e55a62565d64536dbc7706a20b78b9"}, - {file = "mypy-1.14.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b4e3413e0bddea671012b063e27591b953d653209e7a4fa5e48759cda77ca11"}, - {file = "mypy-1.14.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:553c293b1fbdebb6c3c4030589dab9fafb6dfa768995a453d8a5d3b23784af2e"}, - {file = "mypy-1.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fad79bfe3b65fe6a1efaed97b445c3d37f7be9fdc348bdb2d7cac75579607c89"}, - {file = "mypy-1.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:8fa2220e54d2946e94ab6dbb3ba0a992795bd68b16dc852db33028df2b00191b"}, - {file = "mypy-1.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:92c3ed5afb06c3a8e188cb5da4984cab9ec9a77ba956ee419c68a388b4595255"}, - {file = "mypy-1.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dbec574648b3e25f43d23577309b16534431db4ddc09fda50841f1e34e64ed34"}, - {file = "mypy-1.14.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8c6d94b16d62eb3e947281aa7347d78236688e21081f11de976376cf010eb31a"}, - {file = "mypy-1.14.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d4b19b03fdf54f3c5b2fa474c56b4c13c9dbfb9a2db4370ede7ec11a2c5927d9"}, - {file = "mypy-1.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0c911fde686394753fff899c409fd4e16e9b294c24bfd5e1ea4675deae1ac6fd"}, - {file = "mypy-1.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:8b21525cb51671219f5307be85f7e646a153e5acc656e5cebf64bfa076c50107"}, - {file = "mypy-1.14.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7084fb8f1128c76cd9cf68fe5971b37072598e7c31b2f9f95586b65c741a9d31"}, - {file = "mypy-1.14.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8f845a00b4f420f693f870eaee5f3e2692fa84cc8514496114649cfa8fd5e2c6"}, - {file = "mypy-1.14.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:44bf464499f0e3a2d14d58b54674dee25c031703b2ffc35064bd0df2e0fac319"}, - {file = "mypy-1.14.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c99f27732c0b7dc847adb21c9d47ce57eb48fa33a17bc6d7d5c5e9f9e7ae5bac"}, - {file = "mypy-1.14.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:bce23c7377b43602baa0bd22ea3265c49b9ff0b76eb315d6c34721af4cdf1d9b"}, - {file = "mypy-1.14.1-cp38-cp38-win_amd64.whl", hash = "sha256:8edc07eeade7ebc771ff9cf6b211b9a7d93687ff892150cb5692e4f4272b0837"}, - {file = "mypy-1.14.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3888a1816d69f7ab92092f785a462944b3ca16d7c470d564165fe703b0970c35"}, - {file = "mypy-1.14.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:46c756a444117c43ee984bd055db99e498bc613a70bbbc120272bd13ca579fbc"}, - {file = "mypy-1.14.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:27fc248022907e72abfd8e22ab1f10e903915ff69961174784a3900a8cba9ad9"}, - {file = "mypy-1.14.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:499d6a72fb7e5de92218db961f1a66d5f11783f9ae549d214617edab5d4dbdbb"}, - {file = "mypy-1.14.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:57961db9795eb566dc1d1b4e9139ebc4c6b0cb6e7254ecde69d1552bf7613f60"}, - {file = "mypy-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:07ba89fdcc9451f2ebb02853deb6aaaa3d2239a236669a63ab3801bbf923ef5c"}, - {file = "mypy-1.14.1-py3-none-any.whl", hash = "sha256:b66a60cc4073aeb8ae00057f9c1f64d49e90f918fbcef9a977eb121da8b8f1d1"}, - {file = "mypy-1.14.1.tar.gz", hash = "sha256:7ec88144fe9b510e8475ec2f5f251992690fcf89ccb4500b214b4226abcd32d6"}, + {file = "mypy-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:979e4e1a006511dacf628e36fadfecbcc0160a8af6ca7dad2f5025529e082c13"}, + {file = "mypy-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c4bb0e1bd29f7d34efcccd71cf733580191e9a264a2202b0239da95984c5b559"}, + {file = "mypy-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be68172e9fd9ad8fb876c6389f16d1c1b5f100ffa779f77b1fb2176fcc9ab95b"}, + {file = "mypy-1.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7be1e46525adfa0d97681432ee9fcd61a3964c2446795714699a998d193f1a3"}, + {file = "mypy-1.15.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2e2c2e6d3593f6451b18588848e66260ff62ccca522dd231cd4dd59b0160668b"}, + {file = "mypy-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:6983aae8b2f653e098edb77f893f7b6aca69f6cffb19b2cc7443f23cce5f4828"}, + {file = "mypy-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2922d42e16d6de288022e5ca321cd0618b238cfc5570e0263e5ba0a77dbef56f"}, + {file = "mypy-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2ee2d57e01a7c35de00f4634ba1bbf015185b219e4dc5909e281016df43f5ee5"}, + {file = "mypy-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:973500e0774b85d9689715feeffcc980193086551110fd678ebe1f4342fb7c5e"}, + {file = "mypy-1.15.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a95fb17c13e29d2d5195869262f8125dfdb5c134dc8d9a9d0aecf7525b10c2c"}, + {file = "mypy-1.15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1905f494bfd7d85a23a88c5d97840888a7bd516545fc5aaedff0267e0bb54e2f"}, + {file = "mypy-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:c9817fa23833ff189db061e6d2eff49b2f3b6ed9856b4a0a73046e41932d744f"}, + {file = "mypy-1.15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:aea39e0583d05124836ea645f412e88a5c7d0fd77a6d694b60d9b6b2d9f184fd"}, + {file = "mypy-1.15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2f2147ab812b75e5b5499b01ade1f4a81489a147c01585cda36019102538615f"}, + {file = "mypy-1.15.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce436f4c6d218a070048ed6a44c0bbb10cd2cc5e272b29e7845f6a2f57ee4464"}, + {file = "mypy-1.15.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8023ff13985661b50a5928fc7a5ca15f3d1affb41e5f0a9952cb68ef090b31ee"}, + {file = "mypy-1.15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1124a18bc11a6a62887e3e137f37f53fbae476dc36c185d549d4f837a2a6a14e"}, + {file = "mypy-1.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:171a9ca9a40cd1843abeca0e405bc1940cd9b305eaeea2dda769ba096932bb22"}, + {file = "mypy-1.15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93faf3fdb04768d44bf28693293f3904bbb555d076b781ad2530214ee53e3445"}, + {file = "mypy-1.15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:811aeccadfb730024c5d3e326b2fbe9249bb7413553f15499a4050f7c30e801d"}, + {file = "mypy-1.15.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98b7b9b9aedb65fe628c62a6dc57f6d5088ef2dfca37903a7d9ee374d03acca5"}, + {file = "mypy-1.15.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c43a7682e24b4f576d93072216bf56eeff70d9140241f9edec0c104d0c515036"}, + {file = "mypy-1.15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:baefc32840a9f00babd83251560e0ae1573e2f9d1b067719479bfb0e987c6357"}, + {file = "mypy-1.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:b9378e2c00146c44793c98b8d5a61039a048e31f429fb0eb546d93f4b000bedf"}, + {file = "mypy-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e601a7fa172c2131bff456bb3ee08a88360760d0d2f8cbd7a75a65497e2df078"}, + {file = "mypy-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:712e962a6357634fef20412699a3655c610110e01cdaa6180acec7fc9f8513ba"}, + {file = "mypy-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f95579473af29ab73a10bada2f9722856792a36ec5af5399b653aa28360290a5"}, + {file = "mypy-1.15.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f8722560a14cde92fdb1e31597760dc35f9f5524cce17836c0d22841830fd5b"}, + {file = "mypy-1.15.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1fbb8da62dc352133d7d7ca90ed2fb0e9d42bb1a32724c287d3c76c58cbaa9c2"}, + {file = "mypy-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:d10d994b41fb3497719bbf866f227b3489048ea4bbbb5015357db306249f7980"}, + {file = "mypy-1.15.0-py3-none-any.whl", hash = "sha256:5469affef548bd1895d86d3bf10ce2b44e33d86923c29e4d675b3e323437ea3e"}, + {file = "mypy-1.15.0.tar.gz", hash = "sha256:404534629d51d3efea5c800ee7c42b72a6554d6c400e6a79eafe15d11341fd43"}, ] [package.dependencies] @@ -2048,6 +2064,124 @@ files = [ deprecated = ">=1.2.0,<2.0.0" typing-extensions = ">=3.0.0" +[[package]] +name = "numpy" +version = "2.0.2" +description = "Fundamental package for array computing in Python" +optional = true +python-versions = ">=3.9" +files = [ + {file = "numpy-2.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:51129a29dbe56f9ca83438b706e2e69a39892b5eda6cedcb6b0c9fdc9b0d3ece"}, + {file = "numpy-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f15975dfec0cf2239224d80e32c3170b1d168335eaedee69da84fbe9f1f9cd04"}, + {file = "numpy-2.0.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8c5713284ce4e282544c68d1c3b2c7161d38c256d2eefc93c1d683cf47683e66"}, + {file = "numpy-2.0.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:becfae3ddd30736fe1889a37f1f580e245ba79a5855bff5f2a29cb3ccc22dd7b"}, + {file = "numpy-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2da5960c3cf0df7eafefd806d4e612c5e19358de82cb3c343631188991566ccd"}, + {file = "numpy-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:496f71341824ed9f3d2fd36cf3ac57ae2e0165c143b55c3a035ee219413f3318"}, + {file = "numpy-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a61ec659f68ae254e4d237816e33171497e978140353c0c2038d46e63282d0c8"}, + {file = "numpy-2.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d731a1c6116ba289c1e9ee714b08a8ff882944d4ad631fd411106a30f083c326"}, + {file = "numpy-2.0.2-cp310-cp310-win32.whl", hash = "sha256:984d96121c9f9616cd33fbd0618b7f08e0cfc9600a7ee1d6fd9b239186d19d97"}, + {file = "numpy-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:c7b0be4ef08607dd04da4092faee0b86607f111d5ae68036f16cc787e250a131"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:49ca4decb342d66018b01932139c0961a8f9ddc7589611158cb3c27cbcf76448"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:11a76c372d1d37437857280aa142086476136a8c0f373b2e648ab2c8f18fb195"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:807ec44583fd708a21d4a11d94aedf2f4f3c3719035c76a2bbe1fe8e217bdc57"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8cafab480740e22f8d833acefed5cc87ce276f4ece12fdaa2e8903db2f82897a"}, + {file = "numpy-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a15f476a45e6e5a3a79d8a14e62161d27ad897381fecfa4a09ed5322f2085669"}, + {file = "numpy-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13e689d772146140a252c3a28501da66dfecd77490b498b168b501835041f951"}, + {file = "numpy-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9ea91dfb7c3d1c56a0e55657c0afb38cf1eeae4544c208dc465c3c9f3a7c09f9"}, + {file = "numpy-2.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c1c9307701fec8f3f7a1e6711f9089c06e6284b3afbbcd259f7791282d660a15"}, + {file = "numpy-2.0.2-cp311-cp311-win32.whl", hash = "sha256:a392a68bd329eafac5817e5aefeb39038c48b671afd242710b451e76090e81f4"}, + {file = "numpy-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:286cd40ce2b7d652a6f22efdfc6d1edf879440e53e76a75955bc0c826c7e64dc"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:df55d490dea7934f330006d0f81e8551ba6010a5bf035a249ef61a94f21c500b"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8df823f570d9adf0978347d1f926b2a867d5608f434a7cff7f7908c6570dcf5e"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9a92ae5c14811e390f3767053ff54eaee3bf84576d99a2456391401323f4ec2c"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a842d573724391493a97a62ebbb8e731f8a5dcc5d285dfc99141ca15a3302d0c"}, + {file = "numpy-2.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05e238064fc0610c840d1cf6a13bf63d7e391717d247f1bf0318172e759e692"}, + {file = "numpy-2.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a"}, + {file = "numpy-2.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:96a55f64139912d61de9137f11bf39a55ec8faec288c75a54f93dfd39f7eb40c"}, + {file = "numpy-2.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec9852fb39354b5a45a80bdab5ac02dd02b15f44b3804e9f00c556bf24b4bded"}, + {file = "numpy-2.0.2-cp312-cp312-win32.whl", hash = "sha256:671bec6496f83202ed2d3c8fdc486a8fc86942f2e69ff0e986140339a63bcbe5"}, + {file = "numpy-2.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:cfd41e13fdc257aa5778496b8caa5e856dc4896d4ccf01841daee1d96465467a"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9059e10581ce4093f735ed23f3b9d283b9d517ff46009ddd485f1747eb22653c"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:423e89b23490805d2a5a96fe40ec507407b8ee786d66f7328be214f9679df6dd"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:2b2955fa6f11907cf7a70dab0d0755159bca87755e831e47932367fc8f2f2d0b"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:97032a27bd9d8988b9a97a8c4d2c9f2c15a81f61e2f21404d7e8ef00cb5be729"}, + {file = "numpy-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e795a8be3ddbac43274f18588329c72939870a16cae810c2b73461c40718ab1"}, + {file = "numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26b258c385842546006213344c50655ff1555a9338e2e5e02a0756dc3e803dd"}, + {file = "numpy-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fec9451a7789926bcf7c2b8d187292c9f93ea30284802a0ab3f5be8ab36865d"}, + {file = "numpy-2.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9189427407d88ff25ecf8f12469d4d39d35bee1db5d39fc5c168c6f088a6956d"}, + {file = "numpy-2.0.2-cp39-cp39-win32.whl", hash = "sha256:905d16e0c60200656500c95b6b8dca5d109e23cb24abc701d41c02d74c6b3afa"}, + {file = "numpy-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:a3f4ab0caa7f053f6797fcd4e1e25caee367db3112ef2b6ef82d749530768c73"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7f0a0c6f12e07fa94133c8a67404322845220c06a9e80e85999afe727f7438b8"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-macosx_14_0_x86_64.whl", hash = "sha256:312950fdd060354350ed123c0e25a71327d3711584beaef30cdaa93320c392d4"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26df23238872200f63518dd2aa984cfca675d82469535dc7162dc2ee52d9dd5c"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a46288ec55ebbd58947d31d72be2c63cbf839f0a63b49cb755022310792a3385"}, + {file = "numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78"}, +] + +[[package]] +name = "numpy" +version = "2.2.2" +description = "Fundamental package for array computing in Python" +optional = true +python-versions = ">=3.10" +files = [ + {file = "numpy-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7079129b64cb78bdc8d611d1fd7e8002c0a2565da6a47c4df8062349fee90e3e"}, + {file = "numpy-2.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2ec6c689c61df613b783aeb21f945c4cbe6c51c28cb70aae8430577ab39f163e"}, + {file = "numpy-2.2.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:40c7ff5da22cd391944a28c6a9c638a5eef77fcf71d6e3a79e1d9d9e82752715"}, + {file = "numpy-2.2.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:995f9e8181723852ca458e22de5d9b7d3ba4da3f11cc1cb113f093b271d7965a"}, + {file = "numpy-2.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b78ea78450fd96a498f50ee096f69c75379af5138f7881a51355ab0e11286c97"}, + {file = "numpy-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3fbe72d347fbc59f94124125e73fc4976a06927ebc503ec5afbfb35f193cd957"}, + {file = "numpy-2.2.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8e6da5cffbbe571f93588f562ed130ea63ee206d12851b60819512dd3e1ba50d"}, + {file = "numpy-2.2.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:09d6a2032faf25e8d0cadde7fd6145118ac55d2740132c1d845f98721b5ebcfd"}, + {file = "numpy-2.2.2-cp310-cp310-win32.whl", hash = "sha256:159ff6ee4c4a36a23fe01b7c3d07bd8c14cc433d9720f977fcd52c13c0098160"}, + {file = "numpy-2.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:64bd6e1762cd7f0986a740fee4dff927b9ec2c5e4d9a28d056eb17d332158014"}, + {file = "numpy-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:642199e98af1bd2b6aeb8ecf726972d238c9877b0f6e8221ee5ab945ec8a2189"}, + {file = "numpy-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6d9fc9d812c81e6168b6d405bf00b8d6739a7f72ef22a9214c4241e0dc70b323"}, + {file = "numpy-2.2.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:c7d1fd447e33ee20c1f33f2c8e6634211124a9aabde3c617687d8b739aa69eac"}, + {file = "numpy-2.2.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:451e854cfae0febe723077bd0cf0a4302a5d84ff25f0bfece8f29206c7bed02e"}, + {file = "numpy-2.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd249bc894af67cbd8bad2c22e7cbcd46cf87ddfca1f1289d1e7e54868cc785c"}, + {file = "numpy-2.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02935e2c3c0c6cbe9c7955a8efa8908dd4221d7755644c59d1bba28b94fd334f"}, + {file = "numpy-2.2.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a972cec723e0563aa0823ee2ab1df0cb196ed0778f173b381c871a03719d4826"}, + {file = "numpy-2.2.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6d6a0910c3b4368d89dde073e630882cdb266755565155bc33520283b2d9df8"}, + {file = "numpy-2.2.2-cp311-cp311-win32.whl", hash = "sha256:860fd59990c37c3ef913c3ae390b3929d005243acca1a86facb0773e2d8d9e50"}, + {file = "numpy-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:da1eeb460ecce8d5b8608826595c777728cdf28ce7b5a5a8c8ac8d949beadcf2"}, + {file = "numpy-2.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ac9bea18d6d58a995fac1b2cb4488e17eceeac413af014b1dd26170b766d8467"}, + {file = "numpy-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:23ae9f0c2d889b7b2d88a3791f6c09e2ef827c2446f1c4a3e3e76328ee4afd9a"}, + {file = "numpy-2.2.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3074634ea4d6df66be04f6728ee1d173cfded75d002c75fac79503a880bf3825"}, + {file = "numpy-2.2.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:8ec0636d3f7d68520afc6ac2dc4b8341ddb725039de042faf0e311599f54eb37"}, + {file = "numpy-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ffbb1acd69fdf8e89dd60ef6182ca90a743620957afb7066385a7bbe88dc748"}, + {file = "numpy-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0349b025e15ea9d05c3d63f9657707a4e1d471128a3b1d876c095f328f8ff7f0"}, + {file = "numpy-2.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:463247edcee4a5537841d5350bc87fe8e92d7dd0e8c71c995d2c6eecb8208278"}, + {file = "numpy-2.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9dd47ff0cb2a656ad69c38da850df3454da88ee9a6fde0ba79acceee0e79daba"}, + {file = "numpy-2.2.2-cp312-cp312-win32.whl", hash = "sha256:4525b88c11906d5ab1b0ec1f290996c0020dd318af8b49acaa46f198b1ffc283"}, + {file = "numpy-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:5acea83b801e98541619af398cc0109ff48016955cc0818f478ee9ef1c5c3dcb"}, + {file = "numpy-2.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b208cfd4f5fe34e1535c08983a1a6803fdbc7a1e86cf13dd0c61de0b51a0aadc"}, + {file = "numpy-2.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d0bbe7dd86dca64854f4b6ce2ea5c60b51e36dfd597300057cf473d3615f2369"}, + {file = "numpy-2.2.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:22ea3bb552ade325530e72a0c557cdf2dea8914d3a5e1fecf58fa5dbcc6f43cd"}, + {file = "numpy-2.2.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:128c41c085cab8a85dc29e66ed88c05613dccf6bc28b3866cd16050a2f5448be"}, + {file = "numpy-2.2.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:250c16b277e3b809ac20d1f590716597481061b514223c7badb7a0f9993c7f84"}, + {file = "numpy-2.2.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0c8854b09bc4de7b041148d8550d3bd712b5c21ff6a8ed308085f190235d7ff"}, + {file = "numpy-2.2.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b6fb9c32a91ec32a689ec6410def76443e3c750e7cfc3fb2206b985ffb2b85f0"}, + {file = "numpy-2.2.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:57b4012e04cc12b78590a334907e01b3a85efb2107df2b8733ff1ed05fce71de"}, + {file = "numpy-2.2.2-cp313-cp313-win32.whl", hash = "sha256:4dbd80e453bd34bd003b16bd802fac70ad76bd463f81f0c518d1245b1c55e3d9"}, + {file = "numpy-2.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:5a8c863ceacae696aff37d1fd636121f1a512117652e5dfb86031c8d84836369"}, + {file = "numpy-2.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:b3482cb7b3325faa5f6bc179649406058253d91ceda359c104dac0ad320e1391"}, + {file = "numpy-2.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:9491100aba630910489c1d0158034e1c9a6546f0b1340f716d522dc103788e39"}, + {file = "numpy-2.2.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:41184c416143defa34cc8eb9d070b0a5ba4f13a0fa96a709e20584638254b317"}, + {file = "numpy-2.2.2-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7dca87ca328f5ea7dafc907c5ec100d187911f94825f8700caac0b3f4c384b49"}, + {file = "numpy-2.2.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bc61b307655d1a7f9f4b043628b9f2b721e80839914ede634e3d485913e1fb2"}, + {file = "numpy-2.2.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fad446ad0bc886855ddf5909cbf8cb5d0faa637aaa6277fb4b19ade134ab3c7"}, + {file = "numpy-2.2.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:149d1113ac15005652e8d0d3f6fd599360e1a708a4f98e43c9c77834a28238cb"}, + {file = "numpy-2.2.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:106397dbbb1896f99e044efc90360d098b3335060375c26aa89c0d8a97c5f648"}, + {file = "numpy-2.2.2-cp313-cp313t-win32.whl", hash = "sha256:0eec19f8af947a61e968d5429f0bd92fec46d92b0008d0a6685b40d6adf8a4f4"}, + {file = "numpy-2.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:97b974d3ba0fb4612b77ed35d7627490e8e3dff56ab41454d9e8b23448940576"}, + {file = "numpy-2.2.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b0531f0b0e07643eb089df4c509d30d72c9ef40defa53e41363eca8a8cc61495"}, + {file = "numpy-2.2.2-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:e9e82dcb3f2ebbc8cb5ce1102d5f1c5ed236bf8a11730fb45ba82e2841ec21df"}, + {file = "numpy-2.2.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0d4142eb40ca6f94539e4db929410f2a46052a0fe7a2c1c59f6179c39938d2a"}, + {file = "numpy-2.2.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:356ca982c188acbfa6af0d694284d8cf20e95b1c3d0aefa8929376fea9146f60"}, + {file = "numpy-2.2.2.tar.gz", hash = "sha256:ed6906f61834d687738d25988ae117683705636936cc605be0bb208b23df4d8f"}, +] + [[package]] name = "packaging" version = "24.2" @@ -2135,23 +2269,23 @@ type = ["mypy (>=1.11.2)"] [[package]] name = "playwright" -version = "1.49.1" +version = "1.50.0" description = "A high-level API to automate web browsers" optional = true python-versions = ">=3.9" files = [ - {file = "playwright-1.49.1-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:1041ffb45a0d0bc44d698d3a5aa3ac4b67c9bd03540da43a0b70616ad52592b8"}, - {file = "playwright-1.49.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9f38ed3d0c1f4e0a6d1c92e73dd9a61f8855133249d6f0cec28648d38a7137be"}, - {file = "playwright-1.49.1-py3-none-macosx_11_0_universal2.whl", hash = "sha256:3be48c6d26dc819ca0a26567c1ae36a980a0303dcd4249feb6f59e115aaddfb8"}, - {file = "playwright-1.49.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:753ca90ee31b4b03d165cfd36e477309ebf2b4381953f2a982ff612d85b147d2"}, - {file = "playwright-1.49.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd9bc8dab37aa25198a01f555f0a2e2c3813fe200fef018ac34dfe86b34994b9"}, - {file = "playwright-1.49.1-py3-none-win32.whl", hash = "sha256:43b304be67f096058e587dac453ece550eff87b8fbed28de30f4f022cc1745bb"}, - {file = "playwright-1.49.1-py3-none-win_amd64.whl", hash = "sha256:47b23cb346283278f5b4d1e1990bcb6d6302f80c0aa0ca93dd0601a1400191df"}, + {file = "playwright-1.50.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:f36d754a6c5bd9bf7f14e8f57a2aea6fd08f39ca4c8476481b9c83e299531148"}, + {file = "playwright-1.50.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:40f274384591dfd27f2b014596250b2250c843ed1f7f4ef5d2960ecb91b4961e"}, + {file = "playwright-1.50.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:9922ef9bcd316995f01e220acffd2d37a463b4ad10fd73e388add03841dfa230"}, + {file = "playwright-1.50.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:8fc628c492d12b13d1f347137b2ac6c04f98197ff0985ef0403a9a9ee0d39131"}, + {file = "playwright-1.50.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcff35f72db2689a79007aee78f1b0621a22e6e3d6c1f58aaa9ac805bf4497c"}, + {file = "playwright-1.50.0-py3-none-win32.whl", hash = "sha256:3b906f4d351260016a8c5cc1e003bb341651ae682f62213b50168ed581c7558a"}, + {file = "playwright-1.50.0-py3-none-win_amd64.whl", hash = "sha256:1859423da82de631704d5e3d88602d755462b0906824c1debe140979397d2e8d"}, ] [package.dependencies] -greenlet = "3.1.1" -pyee = "12.0.0" +greenlet = ">=3.1.1,<4.0.0" +pyee = ">=12,<13" [[package]] name = "pluggy" @@ -2376,13 +2510,13 @@ files = [ [[package]] name = "pydantic" -version = "2.10.5" +version = "2.10.6" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.10.5-py3-none-any.whl", hash = "sha256:4dd4e322dbe55472cb7ca7e73f4b63574eecccf2835ffa2af9021ce113c83c53"}, - {file = "pydantic-2.10.5.tar.gz", hash = "sha256:278b38dbbaec562011d659ee05f63346951b3a248a6f3642e1bc68894ea2b4ff"}, + {file = "pydantic-2.10.6-py3-none-any.whl", hash = "sha256:427d664bf0b8a2b34ff5dd0f5a18df00591adcee7198fbd71981054cef37b584"}, + {file = "pydantic-2.10.6.tar.gz", hash = "sha256:ca5daa827cce33de7a42be142548b0096bf05a7e7b365aebfa5f8eeec7128236"}, ] [package.dependencies] @@ -2555,13 +2689,13 @@ yapf = ">=0.30.0" [[package]] name = "pyee" -version = "12.0.0" +version = "12.1.1" description = "A rough port of Node.js's EventEmitter to Python with a few tricks of its own" optional = false python-versions = ">=3.8" files = [ - {file = "pyee-12.0.0-py3-none-any.whl", hash = "sha256:7b14b74320600049ccc7d0e0b1becd3b4bd0a03c745758225e31a59f4095c990"}, - {file = "pyee-12.0.0.tar.gz", hash = "sha256:c480603f4aa2927d4766eb41fa82793fe60a82cbfdb8d688e0d08c55a534e145"}, + {file = "pyee-12.1.1-py3-none-any.whl", hash = "sha256:18a19c650556bb6b32b406d7f017c8f513aceed1ef7ca618fb65de7bd2d347ef"}, + {file = "pyee-12.1.1.tar.gz", hash = "sha256:bbc33c09e2ff827f74191e3e5bbc6be7da02f627b7ec30d86f5ce1a6fb2424a3"}, ] [package.dependencies] @@ -2619,13 +2753,13 @@ dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments [[package]] name = "pytest-asyncio" -version = "0.25.2" +version = "0.25.3" description = "Pytest support for asyncio" optional = false python-versions = ">=3.9" files = [ - {file = "pytest_asyncio-0.25.2-py3-none-any.whl", hash = "sha256:0d0bb693f7b99da304a0634afc0a4b19e49d5e0de2d670f38dc4bfa5727c5075"}, - {file = "pytest_asyncio-0.25.2.tar.gz", hash = "sha256:3f8ef9a98f45948ea91a0ed3dc4268b5326c0e7bce73892acc654df4262ad45f"}, + {file = "pytest_asyncio-0.25.3-py3-none-any.whl", hash = "sha256:9e89518e0f9bd08928f97a3482fdc4e244df17529460bc038291ccaf8f85c7c3"}, + {file = "pytest_asyncio-0.25.3.tar.gz", hash = "sha256:fc1da2cf9f125ada7e710b4ddad05518d4cee187ae9412e9ac9271003497f07a"}, ] [package.dependencies] @@ -2928,6 +3062,209 @@ files = [ [package.dependencies] xmod = "*" +[[package]] +name = "scikit-learn" +version = "1.5.2" +description = "A set of python modules for machine learning and data mining" +optional = true +python-versions = ">=3.9" +files = [ + {file = "scikit_learn-1.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:299406827fb9a4f862626d0fe6c122f5f87f8910b86fe5daa4c32dcd742139b6"}, + {file = "scikit_learn-1.5.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:2d4cad1119c77930b235579ad0dc25e65c917e756fe80cab96aa3b9428bd3fb0"}, + {file = "scikit_learn-1.5.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c412ccc2ad9bf3755915e3908e677b367ebc8d010acbb3f182814524f2e5540"}, + {file = "scikit_learn-1.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a686885a4b3818d9e62904d91b57fa757fc2bed3e465c8b177be652f4dd37c8"}, + {file = "scikit_learn-1.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:c15b1ca23d7c5f33cc2cb0a0d6aaacf893792271cddff0edbd6a40e8319bc113"}, + {file = "scikit_learn-1.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:03b6158efa3faaf1feea3faa884c840ebd61b6484167c711548fce208ea09445"}, + {file = "scikit_learn-1.5.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:1ff45e26928d3b4eb767a8f14a9a6efbf1cbff7c05d1fb0f95f211a89fd4f5de"}, + {file = "scikit_learn-1.5.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f763897fe92d0e903aa4847b0aec0e68cadfff77e8a0687cabd946c89d17e675"}, + {file = "scikit_learn-1.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8b0ccd4a902836493e026c03256e8b206656f91fbcc4fde28c57a5b752561f1"}, + {file = "scikit_learn-1.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:6c16d84a0d45e4894832b3c4d0bf73050939e21b99b01b6fd59cbb0cf39163b6"}, + {file = "scikit_learn-1.5.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f932a02c3f4956dfb981391ab24bda1dbd90fe3d628e4b42caef3e041c67707a"}, + {file = "scikit_learn-1.5.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:3b923d119d65b7bd555c73be5423bf06c0105678ce7e1f558cb4b40b0a5502b1"}, + {file = "scikit_learn-1.5.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f60021ec1574e56632be2a36b946f8143bf4e5e6af4a06d85281adc22938e0dd"}, + {file = "scikit_learn-1.5.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:394397841449853c2290a32050382edaec3da89e35b3e03d6cc966aebc6a8ae6"}, + {file = "scikit_learn-1.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:57cc1786cfd6bd118220a92ede80270132aa353647684efa385a74244a41e3b1"}, + {file = "scikit_learn-1.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9a702e2de732bbb20d3bad29ebd77fc05a6b427dc49964300340e4c9328b3f5"}, + {file = "scikit_learn-1.5.2-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:b0768ad641981f5d3a198430a1d31c3e044ed2e8a6f22166b4d546a5116d7908"}, + {file = "scikit_learn-1.5.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:178ddd0a5cb0044464fc1bfc4cca5b1833bfc7bb022d70b05db8530da4bb3dd3"}, + {file = "scikit_learn-1.5.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7284ade780084d94505632241bf78c44ab3b6f1e8ccab3d2af58e0e950f9c12"}, + {file = "scikit_learn-1.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:b7b0f9a0b1040830d38c39b91b3a44e1b643f4b36e36567b80b7c6bd2202a27f"}, + {file = "scikit_learn-1.5.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:757c7d514ddb00ae249832fe87100d9c73c6ea91423802872d9e74970a0e40b9"}, + {file = "scikit_learn-1.5.2-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:52788f48b5d8bca5c0736c175fa6bdaab2ef00a8f536cda698db61bd89c551c1"}, + {file = "scikit_learn-1.5.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:643964678f4b5fbdc95cbf8aec638acc7aa70f5f79ee2cdad1eec3df4ba6ead8"}, + {file = "scikit_learn-1.5.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca64b3089a6d9b9363cd3546f8978229dcbb737aceb2c12144ee3f70f95684b7"}, + {file = "scikit_learn-1.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:3bed4909ba187aca80580fe2ef370d9180dcf18e621a27c4cf2ef10d279a7efe"}, + {file = "scikit_learn-1.5.2.tar.gz", hash = "sha256:b4237ed7b3fdd0a4882792e68ef2545d5baa50aca3bb45aa7df468138ad8f94d"}, +] + +[package.dependencies] +joblib = ">=1.2.0" +numpy = ">=1.19.5" +scipy = ">=1.6.0" +threadpoolctl = ">=3.1.0" + +[package.extras] +benchmark = ["matplotlib (>=3.3.4)", "memory_profiler (>=0.57.0)", "pandas (>=1.1.5)"] +build = ["cython (>=3.0.10)", "meson-python (>=0.16.0)", "numpy (>=1.19.5)", "scipy (>=1.6.0)"] +docs = ["Pillow (>=7.1.2)", "matplotlib (>=3.3.4)", "memory_profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pydata-sphinx-theme (>=0.15.3)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)", "sphinx (>=7.3.7)", "sphinx-copybutton (>=0.5.2)", "sphinx-design (>=0.5.0)", "sphinx-design (>=0.6.0)", "sphinx-gallery (>=0.16.0)", "sphinx-prompt (>=1.4.0)", "sphinx-remove-toctrees (>=1.0.0.post1)", "sphinxcontrib-sass (>=0.3.4)", "sphinxext-opengraph (>=0.9.1)"] +examples = ["matplotlib (>=3.3.4)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)"] +install = ["joblib (>=1.2.0)", "numpy (>=1.19.5)", "scipy (>=1.6.0)", "threadpoolctl (>=3.1.0)"] +maintenance = ["conda-lock (==2.5.6)"] +tests = ["black (>=24.3.0)", "matplotlib (>=3.3.4)", "mypy (>=1.9)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pyamg (>=4.0.0)", "pyarrow (>=12.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.2.1)", "scikit-image (>=0.17.2)"] + +[[package]] +name = "scikit-learn" +version = "1.6.1" +description = "A set of python modules for machine learning and data mining" +optional = true +python-versions = ">=3.9" +files = [ + {file = "scikit_learn-1.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d056391530ccd1e501056160e3c9673b4da4805eb67eb2bdf4e983e1f9c9204e"}, + {file = "scikit_learn-1.6.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:0c8d036eb937dbb568c6242fa598d551d88fb4399c0344d95c001980ec1c7d36"}, + {file = "scikit_learn-1.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8634c4bd21a2a813e0a7e3900464e6d593162a29dd35d25bdf0103b3fce60ed5"}, + {file = "scikit_learn-1.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:775da975a471c4f6f467725dff0ced5c7ac7bda5e9316b260225b48475279a1b"}, + {file = "scikit_learn-1.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:8a600c31592bd7dab31e1c61b9bbd6dea1b3433e67d264d17ce1017dbdce8002"}, + {file = "scikit_learn-1.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:72abc587c75234935e97d09aa4913a82f7b03ee0b74111dcc2881cba3c5a7b33"}, + {file = "scikit_learn-1.6.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b3b00cdc8f1317b5f33191df1386c0befd16625f49d979fe77a8d44cae82410d"}, + {file = "scikit_learn-1.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc4765af3386811c3ca21638f63b9cf5ecf66261cc4815c1db3f1e7dc7b79db2"}, + {file = "scikit_learn-1.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25fc636bdaf1cc2f4a124a116312d837148b5e10872147bdaf4887926b8c03d8"}, + {file = "scikit_learn-1.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:fa909b1a36e000a03c382aade0bd2063fd5680ff8b8e501660c0f59f021a6415"}, + {file = "scikit_learn-1.6.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:926f207c804104677af4857b2c609940b743d04c4c35ce0ddc8ff4f053cddc1b"}, + {file = "scikit_learn-1.6.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:2c2cae262064e6a9b77eee1c8e768fc46aa0b8338c6a8297b9b6759720ec0ff2"}, + {file = "scikit_learn-1.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1061b7c028a8663fb9a1a1baf9317b64a257fcb036dae5c8752b2abef31d136f"}, + {file = "scikit_learn-1.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e69fab4ebfc9c9b580a7a80111b43d214ab06250f8a7ef590a4edf72464dd86"}, + {file = "scikit_learn-1.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:70b1d7e85b1c96383f872a519b3375f92f14731e279a7b4c6cfd650cf5dffc52"}, + {file = "scikit_learn-1.6.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2ffa1e9e25b3d93990e74a4be2c2fc61ee5af85811562f1288d5d055880c4322"}, + {file = "scikit_learn-1.6.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:dc5cf3d68c5a20ad6d571584c0750ec641cc46aeef1c1507be51300e6003a7e1"}, + {file = "scikit_learn-1.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c06beb2e839ecc641366000ca84f3cf6fa9faa1777e29cf0c04be6e4d096a348"}, + {file = "scikit_learn-1.6.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8ca8cb270fee8f1f76fa9bfd5c3507d60c6438bbee5687f81042e2bb98e5a97"}, + {file = "scikit_learn-1.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:7a1c43c8ec9fde528d664d947dc4c0789be4077a3647f232869f41d9bf50e0fb"}, + {file = "scikit_learn-1.6.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a17c1dea1d56dcda2fac315712f3651a1fea86565b64b48fa1bc090249cbf236"}, + {file = "scikit_learn-1.6.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6a7aa5f9908f0f28f4edaa6963c0a6183f1911e63a69aa03782f0d924c830a35"}, + {file = "scikit_learn-1.6.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0650e730afb87402baa88afbf31c07b84c98272622aaba002559b614600ca691"}, + {file = "scikit_learn-1.6.1-cp313-cp313t-win_amd64.whl", hash = "sha256:3f59fe08dc03ea158605170eb52b22a105f238a5d512c4470ddeca71feae8e5f"}, + {file = "scikit_learn-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6849dd3234e87f55dce1db34c89a810b489ead832aaf4d4550b7ea85628be6c1"}, + {file = "scikit_learn-1.6.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:e7be3fa5d2eb9be7d77c3734ff1d599151bb523674be9b834e8da6abe132f44e"}, + {file = "scikit_learn-1.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:44a17798172df1d3c1065e8fcf9019183f06c87609b49a124ebdf57ae6cb0107"}, + {file = "scikit_learn-1.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8b7a3b86e411e4bce21186e1c180d792f3d99223dcfa3b4f597ecc92fa1a422"}, + {file = "scikit_learn-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:7a73d457070e3318e32bdb3aa79a8d990474f19035464dfd8bede2883ab5dc3b"}, + {file = "scikit_learn-1.6.1.tar.gz", hash = "sha256:b4fc2525eca2c69a59260f583c56a7557c6ccdf8deafdba6e060f94c1c59738e"}, +] + +[package.dependencies] +joblib = ">=1.2.0" +numpy = ">=1.19.5" +scipy = ">=1.6.0" +threadpoolctl = ">=3.1.0" + +[package.extras] +benchmark = ["matplotlib (>=3.3.4)", "memory_profiler (>=0.57.0)", "pandas (>=1.1.5)"] +build = ["cython (>=3.0.10)", "meson-python (>=0.16.0)", "numpy (>=1.19.5)", "scipy (>=1.6.0)"] +docs = ["Pillow (>=7.1.2)", "matplotlib (>=3.3.4)", "memory_profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pydata-sphinx-theme (>=0.15.3)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)", "sphinx (>=7.3.7)", "sphinx-copybutton (>=0.5.2)", "sphinx-design (>=0.5.0)", "sphinx-design (>=0.6.0)", "sphinx-gallery (>=0.17.1)", "sphinx-prompt (>=1.4.0)", "sphinx-remove-toctrees (>=1.0.0.post1)", "sphinxcontrib-sass (>=0.3.4)", "sphinxext-opengraph (>=0.9.1)", "towncrier (>=24.8.0)"] +examples = ["matplotlib (>=3.3.4)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)"] +install = ["joblib (>=1.2.0)", "numpy (>=1.19.5)", "scipy (>=1.6.0)", "threadpoolctl (>=3.1.0)"] +maintenance = ["conda-lock (==2.5.6)"] +tests = ["black (>=24.3.0)", "matplotlib (>=3.3.4)", "mypy (>=1.9)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pyamg (>=4.0.0)", "pyarrow (>=12.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.5.1)", "scikit-image (>=0.17.2)"] + +[[package]] +name = "scipy" +version = "1.13.1" +description = "Fundamental algorithms for scientific computing in Python" +optional = true +python-versions = ">=3.9" +files = [ + {file = "scipy-1.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:20335853b85e9a49ff7572ab453794298bcf0354d8068c5f6775a0eabf350aca"}, + {file = "scipy-1.13.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d605e9c23906d1994f55ace80e0125c587f96c020037ea6aa98d01b4bd2e222f"}, + {file = "scipy-1.13.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cfa31f1def5c819b19ecc3a8b52d28ffdcc7ed52bb20c9a7589669dd3c250989"}, + {file = "scipy-1.13.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26264b282b9da0952a024ae34710c2aff7d27480ee91a2e82b7b7073c24722f"}, + {file = "scipy-1.13.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:eccfa1906eacc02de42d70ef4aecea45415f5be17e72b61bafcfd329bdc52e94"}, + {file = "scipy-1.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:2831f0dc9c5ea9edd6e51e6e769b655f08ec6db6e2e10f86ef39bd32eb11da54"}, + {file = "scipy-1.13.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:27e52b09c0d3a1d5b63e1105f24177e544a222b43611aaf5bc44d4a0979e32f9"}, + {file = "scipy-1.13.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:54f430b00f0133e2224c3ba42b805bfd0086fe488835effa33fa291561932326"}, + {file = "scipy-1.13.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e89369d27f9e7b0884ae559a3a956e77c02114cc60a6058b4e5011572eea9299"}, + {file = "scipy-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a78b4b3345f1b6f68a763c6e25c0c9a23a9fd0f39f5f3d200efe8feda560a5fa"}, + {file = "scipy-1.13.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:45484bee6d65633752c490404513b9ef02475b4284c4cfab0ef946def50b3f59"}, + {file = "scipy-1.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:5713f62f781eebd8d597eb3f88b8bf9274e79eeabf63afb4a737abc6c84ad37b"}, + {file = "scipy-1.13.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5d72782f39716b2b3509cd7c33cdc08c96f2f4d2b06d51e52fb45a19ca0c86a1"}, + {file = "scipy-1.13.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:017367484ce5498445aade74b1d5ab377acdc65e27095155e448c88497755a5d"}, + {file = "scipy-1.13.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:949ae67db5fa78a86e8fa644b9a6b07252f449dcf74247108c50e1d20d2b4627"}, + {file = "scipy-1.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de3ade0e53bc1f21358aa74ff4830235d716211d7d077e340c7349bc3542e884"}, + {file = "scipy-1.13.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2ac65fb503dad64218c228e2dc2d0a0193f7904747db43014645ae139c8fad16"}, + {file = "scipy-1.13.1-cp312-cp312-win_amd64.whl", hash = "sha256:cdd7dacfb95fea358916410ec61bbc20440f7860333aee6d882bb8046264e949"}, + {file = "scipy-1.13.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:436bbb42a94a8aeef855d755ce5a465479c721e9d684de76bf61a62e7c2b81d5"}, + {file = "scipy-1.13.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:8335549ebbca860c52bf3d02f80784e91a004b71b059e3eea9678ba994796a24"}, + {file = "scipy-1.13.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d533654b7d221a6a97304ab63c41c96473ff04459e404b83275b60aa8f4b7004"}, + {file = "scipy-1.13.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:637e98dcf185ba7f8e663e122ebf908c4702420477ae52a04f9908707456ba4d"}, + {file = "scipy-1.13.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a014c2b3697bde71724244f63de2476925596c24285c7a637364761f8710891c"}, + {file = "scipy-1.13.1-cp39-cp39-win_amd64.whl", hash = "sha256:392e4ec766654852c25ebad4f64e4e584cf19820b980bc04960bca0b0cd6eaa2"}, + {file = "scipy-1.13.1.tar.gz", hash = "sha256:095a87a0312b08dfd6a6155cbbd310a8c51800fc931b8c0b84003014b874ed3c"}, +] + +[package.dependencies] +numpy = ">=1.22.4,<2.3" + +[package.extras] +dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy", "pycodestyle", "pydevtool", "rich-click", "ruff", "types-psutil", "typing_extensions"] +doc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.12.0)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0)", "sphinx-design (>=0.4.0)"] +test = ["array-api-strict", "asv", "gmpy2", "hypothesis (>=6.30)", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] + +[[package]] +name = "scipy" +version = "1.15.1" +description = "Fundamental algorithms for scientific computing in Python" +optional = true +python-versions = ">=3.10" +files = [ + {file = "scipy-1.15.1-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:c64ded12dcab08afff9e805a67ff4480f5e69993310e093434b10e85dc9d43e1"}, + {file = "scipy-1.15.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:5b190b935e7db569960b48840e5bef71dc513314cc4e79a1b7d14664f57fd4ff"}, + {file = "scipy-1.15.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:4b17d4220df99bacb63065c76b0d1126d82bbf00167d1730019d2a30d6ae01ea"}, + {file = "scipy-1.15.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:63b9b6cd0333d0eb1a49de6f834e8aeaefe438df8f6372352084535ad095219e"}, + {file = "scipy-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f151e9fb60fbf8e52426132f473221a49362091ce7a5e72f8aa41f8e0da4f25"}, + {file = "scipy-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21e10b1dd56ce92fba3e786007322542361984f8463c6d37f6f25935a5a6ef52"}, + {file = "scipy-1.15.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5dff14e75cdbcf07cdaa1c7707db6017d130f0af9ac41f6ce443a93318d6c6e0"}, + {file = "scipy-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:f82fcf4e5b377f819542fbc8541f7b5fbcf1c0017d0df0bc22c781bf60abc4d8"}, + {file = "scipy-1.15.1-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:5bd8d27d44e2c13d0c1124e6a556454f52cd3f704742985f6b09e75e163d20d2"}, + {file = "scipy-1.15.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:be3deeb32844c27599347faa077b359584ba96664c5c79d71a354b80a0ad0ce0"}, + {file = "scipy-1.15.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:5eb0ca35d4b08e95da99a9f9c400dc9f6c21c424298a0ba876fdc69c7afacedf"}, + {file = "scipy-1.15.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:74bb864ff7640dea310a1377d8567dc2cb7599c26a79ca852fc184cc851954ac"}, + {file = "scipy-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:667f950bf8b7c3a23b4199db24cb9bf7512e27e86d0e3813f015b74ec2c6e3df"}, + {file = "scipy-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395be70220d1189756068b3173853029a013d8c8dd5fd3d1361d505b2aa58fa7"}, + {file = "scipy-1.15.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ce3a000cd28b4430426db2ca44d96636f701ed12e2b3ca1f2b1dd7abdd84b39a"}, + {file = "scipy-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:3fe1d95944f9cf6ba77aa28b82dd6bb2a5b52f2026beb39ecf05304b8392864b"}, + {file = "scipy-1.15.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c09aa9d90f3500ea4c9b393ee96f96b0ccb27f2f350d09a47f533293c78ea776"}, + {file = "scipy-1.15.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:0ac102ce99934b162914b1e4a6b94ca7da0f4058b6d6fd65b0cef330c0f3346f"}, + {file = "scipy-1.15.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:09c52320c42d7f5c7748b69e9f0389266fd4f82cf34c38485c14ee976cb8cb04"}, + {file = "scipy-1.15.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:cdde8414154054763b42b74fe8ce89d7f3d17a7ac5dd77204f0e142cdc9239e9"}, + {file = "scipy-1.15.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c9d8fc81d6a3b6844235e6fd175ee1d4c060163905a2becce8e74cb0d7554ce"}, + {file = "scipy-1.15.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fb57b30f0017d4afa5fe5f5b150b8f807618819287c21cbe51130de7ccdaed2"}, + {file = "scipy-1.15.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:491d57fe89927fa1aafbe260f4cfa5ffa20ab9f1435025045a5315006a91b8f5"}, + {file = "scipy-1.15.1-cp312-cp312-win_amd64.whl", hash = "sha256:900f3fa3db87257510f011c292a5779eb627043dd89731b9c461cd16ef76ab3d"}, + {file = "scipy-1.15.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:100193bb72fbff37dbd0bf14322314fc7cbe08b7ff3137f11a34d06dc0ee6b85"}, + {file = "scipy-1.15.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:2114a08daec64980e4b4cbdf5bee90935af66d750146b1d2feb0d3ac30613692"}, + {file = "scipy-1.15.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6b3e71893c6687fc5e29208d518900c24ea372a862854c9888368c0b267387ab"}, + {file = "scipy-1.15.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:837299eec3d19b7e042923448d17d95a86e43941104d33f00da7e31a0f715d3c"}, + {file = "scipy-1.15.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82add84e8a9fb12af5c2c1a3a3f1cb51849d27a580cb9e6bd66226195142be6e"}, + {file = "scipy-1.15.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:070d10654f0cb6abd295bc96c12656f948e623ec5f9a4eab0ddb1466c000716e"}, + {file = "scipy-1.15.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:55cc79ce4085c702ac31e49b1e69b27ef41111f22beafb9b49fea67142b696c4"}, + {file = "scipy-1.15.1-cp313-cp313-win_amd64.whl", hash = "sha256:c352c1b6d7cac452534517e022f8f7b8d139cd9f27e6fbd9f3cbd0bfd39f5bef"}, + {file = "scipy-1.15.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0458839c9f873062db69a03de9a9765ae2e694352c76a16be44f93ea45c28d2b"}, + {file = "scipy-1.15.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:af0b61c1de46d0565b4b39c6417373304c1d4f5220004058bdad3061c9fa8a95"}, + {file = "scipy-1.15.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:71ba9a76c2390eca6e359be81a3e879614af3a71dfdabb96d1d7ab33da6f2364"}, + {file = "scipy-1.15.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:14eaa373c89eaf553be73c3affb11ec6c37493b7eaaf31cf9ac5dffae700c2e0"}, + {file = "scipy-1.15.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f735bc41bd1c792c96bc426dece66c8723283695f02df61dcc4d0a707a42fc54"}, + {file = "scipy-1.15.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2722a021a7929d21168830790202a75dbb20b468a8133c74a2c0230c72626b6c"}, + {file = "scipy-1.15.1-cp313-cp313t-win_amd64.whl", hash = "sha256:bc7136626261ac1ed988dca56cfc4ab5180f75e0ee52e58f1e6aa74b5f3eacd5"}, + {file = "scipy-1.15.1.tar.gz", hash = "sha256:033a75ddad1463970c96a88063a1df87ccfddd526437136b6ee81ff0312ebdf6"}, +] + +[package.dependencies] +numpy = ">=1.23.5,<2.5" + +[package.extras] +dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy (==1.10.0)", "pycodestyle", "pydevtool", "rich-click", "ruff (>=0.0.292)", "types-psutil", "typing_extensions"] +doc = ["intersphinx_registry", "jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.16.5)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<8.0.0)", "sphinx-copybutton", "sphinx-design (>=0.4.0)"] +test = ["Cython", "array-api-strict (>=2.0,<2.1.1)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] + [[package]] name = "setuptools" version = "75.8.0" @@ -3062,6 +3399,17 @@ files = [ {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"}, ] +[[package]] +name = "threadpoolctl" +version = "3.5.0" +description = "threadpoolctl" +optional = true +python-versions = ">=3.8" +files = [ + {file = "threadpoolctl-3.5.0-py3-none-any.whl", hash = "sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467"}, + {file = "threadpoolctl-3.5.0.tar.gz", hash = "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107"}, +] + [[package]] name = "tldextract" version = "5.1.3" @@ -3152,13 +3500,13 @@ test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0, [[package]] name = "typeapi" -version = "2.2.3" +version = "2.2.4" description = "" optional = false -python-versions = "<4.0,>=3.8" +python-versions = ">=3.8" files = [ - {file = "typeapi-2.2.3-py3-none-any.whl", hash = "sha256:038062b473dd9bc182966469d7a37d81ba7fa5bb0c01f30b0604b5667b13a47b"}, - {file = "typeapi-2.2.3.tar.gz", hash = "sha256:61cf8c852c05471522fcf55ec37d0c37f0de6943cc8e4d58529f796881e32c08"}, + {file = "typeapi-2.2.4-py3-none-any.whl", hash = "sha256:bd6d5e5907fa47e0303bf254e7cc8712d4be4eb26d7ffaedb67c9e7844c53bb8"}, + {file = "typeapi-2.2.4.tar.gz", hash = "sha256:daa80767520c0957a320577e4f729c0ba6921c708def31f4c6fd8d611908fd7b"}, ] [package.dependencies] @@ -3183,13 +3531,13 @@ typing-extensions = ">=3.7.4.3" [[package]] name = "types-beautifulsoup4" -version = "4.12.0.20241020" +version = "4.12.0.20250204" description = "Typing stubs for beautifulsoup4" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "types-beautifulsoup4-4.12.0.20241020.tar.gz", hash = "sha256:158370d08d0cd448bd11b132a50ff5279237a5d4b5837beba074de152a513059"}, - {file = "types_beautifulsoup4-4.12.0.20241020-py3-none-any.whl", hash = "sha256:c95e66ce15a4f5f0835f7fbc5cd886321ae8294f977c495424eaf4225307fd30"}, + {file = "types_beautifulsoup4-4.12.0.20250204-py3-none-any.whl", hash = "sha256:57ce9e75717b63c390fd789c787d267a67eb01fa6d800a03b9bdde2e877ed1eb"}, + {file = "types_beautifulsoup4-4.12.0.20250204.tar.gz", hash = "sha256:f083d8edcbd01279f8c3995b56cfff2d01f1bb894c3b502ba118d36fbbc495bf"}, ] [package.dependencies] @@ -3289,13 +3637,13 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [[package]] name = "w3lib" -version = "2.2.1" +version = "2.3.1" description = "Library of web-related functions" optional = true -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "w3lib-2.2.1-py3-none-any.whl", hash = "sha256:e56d81c6a6bf507d7039e0c95745ab80abd24b465eb0f248af81e3eaa46eb510"}, - {file = "w3lib-2.2.1.tar.gz", hash = "sha256:756ff2d94c64e41c8d7c0c59fea12a5d0bc55e33a531c7988b4a163deb9b07dd"}, + {file = "w3lib-2.3.1-py3-none-any.whl", hash = "sha256:9ccd2ae10c8c41c7279cd8ad4fe65f834be894fe7bfdd7304b991fd69325847b"}, + {file = "w3lib-2.3.1.tar.gz", hash = "sha256:5c8ac02a3027576174c2b61eb9a2170ba1b197cae767080771b6f1febda249a4"}, ] [[package]] @@ -3799,7 +4147,8 @@ cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\ cffi = ["cffi (>=1.11)"] [extras] -all = ["beautifulsoup4", "browserforge", "curl-cffi", "html5lib", "lxml", "parsel", "playwright"] +adaptive-playwright = ["jaro-winkler", "playwright", "scikit-learn", "scikit-learn"] +all = ["beautifulsoup4", "browserforge", "curl-cffi", "html5lib", "jaro-winkler", "lxml", "parsel", "playwright", "scikit-learn", "scikit-learn"] beautifulsoup = ["beautifulsoup4", "html5lib", "lxml"] curl-impersonate = ["curl-cffi"] parsel = ["parsel"] @@ -3808,4 +4157,4 @@ playwright = ["browserforge", "playwright"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "83d1698ca3a2e6d67ef01adcd2975c572fd3a7e93f4c17e024a92c465fd92709" +content-hash = "c2d2e2b38eed54ed0867b36bfce3c46bc9018fcde338fa30656938da564fd555" From 6937d0f34848bcbc3e87e535be976a86e4ce647e Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 6 Feb 2025 09:58:50 +0100 Subject: [PATCH 32/32] Add sec headers constraint for chromium --- .../_browserforge_adapter.py | 23 +++++++++++++++---- .../test_header_generator.py | 10 ++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/crawlee/fingerprint_suite/_browserforge_adapter.py b/src/crawlee/fingerprint_suite/_browserforge_adapter.py index 08d0a33f63..6834ad2291 100644 --- a/src/crawlee/fingerprint_suite/_browserforge_adapter.py +++ b/src/crawlee/fingerprint_suite/_browserforge_adapter.py @@ -17,15 +17,13 @@ from ._fingerprint_generator import FingerprintGenerator if TYPE_CHECKING: - from camoufox.utils import ListOrString - from ._types import HeaderGeneratorOptions, ScreenOptions, SupportedBrowserType class PatchedHeaderGenerator(bf_HeaderGenerator): """Browserforge `HeaderGenerator` that contains patches not accepted in upstream repo.""" - def _get_accept_language_header(self, locales: ListOrString) -> str: + def _get_accept_language_header(self, locales: tuple[str, ...]) -> str: """Generates the Accept-Language header based on the given locales. Patched version due to PR of upstream repo not being merged: https://github.com/daijro/browserforge/pull/24 @@ -140,22 +138,37 @@ def __init__(self) -> None: def generate(self, browser_type: SupportedBrowserType = 'chromium') -> dict[str, str]: """Generate headers. - browser_type = `chromium` is not just Google Chrome, but also other chromium based browsers! + browser_type = `chromium` is in general sense not just Google Chrome, but also other chromium based browsers. For example this Safari user agent can be generated for `chromium` input: `Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/130.0.6723.90 Mobile/15E148 Safari/604.1` + To remain consistent with previous implementation only subset of `chromium` header will be allowed. """ # browserforge header generation can be flaky. Enforce basic QA on generated headers - max_attempts = 20 + max_attempts = 10 + + if browser_type == 'chromium': + # `BrowserForge` header generator considers `chromium` in general sense and therefore will generate also + # other `Chromium` based browser headers. This adapter desires only specific subset of `chromium` headers + # that contain all 'sec-ch-ua', 'sec-ch-ua-mobile', 'sec-ch-ua-platform' headers. + # Increase max attempts as from `BrowserForge` header generator perspective even `chromium` + # headers without `sec-...` headers are valid. + max_attempts += 50 bf_browser_type = 'safari' if browser_type == 'webkit' else browser_type for _attempt in range(max_attempts): generated_header: dict[str, str] = self._generator.generate(browser=bf_browser_type) if any(keyword in generated_header['User-Agent'] for keyword in BROWSER_TYPE_HEADER_KEYWORD[browser_type]): + if browser_type == 'chromium' and not self._contains_all_sec_headers(generated_header): + continue + return generated_header raise RuntimeError('Failed to generate header.') + def _contains_all_sec_headers(self, headers: dict[str, str]) -> bool: + return all(header_name in headers for header_name in ('sec-ch-ua', 'sec-ch-ua-mobile', 'sec-ch-ua-platform')) + def get_available_header_network() -> dict: """Get header network that contains possible header values.""" diff --git a/tests/unit/fingerprint_suite/test_header_generator.py b/tests/unit/fingerprint_suite/test_header_generator.py index 0217f0da93..540ac38055 100644 --- a/tests/unit/fingerprint_suite/test_header_generator.py +++ b/tests/unit/fingerprint_suite/test_header_generator.py @@ -53,6 +53,16 @@ def test_get_user_agent_header_invalid_browser_type() -> None: header_generator.get_user_agent_header(browser_type='invalid_browser') # type: ignore[arg-type] +def test_get_sec_ch_ua_headers_chromium(header_network: dict) -> None: + """Test that Sec-Ch-Ua headers are generated correctly for Chromium.""" + header_generator = HeaderGenerator() + headers = header_generator.get_sec_ch_ua_headers(browser_type='chromium') + + assert headers.get('sec-ch-ua') in get_available_header_values(header_network, 'sec-ch-ua') + assert headers.get('sec-ch-ua-mobile') in get_available_header_values(header_network, 'sec-ch-ua-mobile') + assert headers.get('sec-ch-ua-platform') in get_available_header_values(header_network, 'sec-ch-ua-platform') + + def test_get_sec_ch_ua_headers_firefox() -> None: """Test that sec-ch-ua headers are not generated for Firefox.""" header_generator = HeaderGenerator()