Merge branch 'master' into fix/NotImplementedError

bluet · Dec 10, 2023 · 1baa1c9 · 1baa1c9
2 parents 312517e + dc944d6
commit 1baa1c9
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 12 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -4,11 +4,15 @@
                 "aiodns",
                 "aiohttp",
                 "asyncio",
+                "fgroup",
+                "fparser",
+                "gparser",
                 "gunicorn",
                 "maxminddb",
                 "proxybroker",
                 "proxycontrol",
-                "Referer"
+                "Referer",
+                "uparser"
         ],
         "esbonio.sphinx.confDir": "${workspaceFolder}/docs/source"
 }
diff --git a/proxybroker/cli.py b/proxybroker/cli.py
@@ -7,9 +7,6 @@
 import sys
 from contextlib import contextmanager
 
-import sys
-import asyncio
-
 if sys.platform == 'win32':
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 

diff --git a/proxybroker/proxy.py b/proxybroker/proxy.py
@@ -72,6 +72,9 @@ def __init__(self, host=None, port=None, types=(), timeout=8, verify_ssl=False):
                 'Try Proxy.create() if the host is a domain'
             )
 
+        if port is None:
+            raise ValueError('The port of proxy cannot be None')
+
         self.port = int(port)
         if self.port > 65535:
             raise ValueError('The port of proxy cannot be greater than 65535')
@@ -197,7 +200,7 @@ def avgRespTime(self):
             Use :attr:`avg_resp_time` instead.
         """
         warnings.warn(
-            '`avgRespTime` property is deprecated, ' 'use `avg_resp_time` instead.',
+            '`avgRespTime` property is deprecated, use `avg_resp_time` instead.',
             DeprecationWarning,
         )
         return self.avg_resp_time

diff --git a/proxybroker/utils.py b/proxybroker/utils.py
@@ -22,6 +22,10 @@
     r'(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)'
 )
 
+IPv6Pattern = re.compile(
+    r'\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*'
+)
+
 IPPortPatternLine = re.compile(
     r'^.*?(?P<ip>(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)).*?(?P<port>\d{2,5}).*$',  # noqa
     flags=re.MULTILINE,
@@ -33,14 +37,9 @@
     flags=re.DOTALL,
 )
 
-# IsIpPattern = re.compile(
-#     r'^(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)$')
-
-
 def get_headers(rv=False):
     _rv = str(random.randint(1000, 9999)) if rv else ''
     headers = {
-        # 'User-Agent': 'Mozilla/5.0 (X11; U; Linux i386; ru-RU; rv:2.0) Gecko/20100625 Firefox/3.5.11',  # noqa
         'User-Agent': 'PxBroker/%s/%s' % (version, _rv),
         'Accept': '*/*',
         'Accept-Encoding': 'gzip, deflate',
@@ -53,8 +52,8 @@ def get_headers(rv=False):
 
 
 def get_all_ip(page):
-    # TODO: add IPv6 support
-    return set(IPPattern.findall(page))
+    # NOTE: IPv6 addresses support need to be tested
+    return set(IPPattern.findall(page) + IPv6Pattern.findall(page))
 
 
 def get_status_code(resp, start=9, stop=12):