From 735de12c55da9de502003697cb0454768b86898b Mon Sep 17 00:00:00 2001 From: 1UC1F3R616 Date: Sun, 17 Nov 2024 17:43:23 +0530 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=94=A8=20refactor:=20minor=20chores?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cli.py | 9 ++++---- fastProxy/fastProxy.py | 5 +---- fastProxy/logger.py | 1 - proxy_list/working_proxies.csv | 15 ++++++++----- requirements.txt | 6 ++++- tests/unit/test_fastproxy.py | 40 +++++++++++++++++++--------------- 6 files changed, 42 insertions(+), 34 deletions(-) diff --git a/cli.py b/cli.py index d44013d..31c1fb8 100644 --- a/cli.py +++ b/cli.py @@ -27,9 +27,9 @@ def main(c=None, t=None, g=None, a=None, max_proxies=5): a (bool, optional): All proxies. Defaults to None. max_proxies (int, optional): Maximum number of proxies to fetch. Defaults to 5. """ - # Set global timeout for CLI operation - signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(45) # 45 second timeout for entire CLI operation + # Set global timeout for CLI operation # Linux + # signal.signal(signal.SIGALRM, timeout_handler) + # signal.alarm(45) # 45 second timeout for entire CLI operation try: # Configure settings with very conservative defaults here @@ -52,7 +52,8 @@ def main(c=None, t=None, g=None, a=None, max_proxies=5): except Exception as e: print(f"\nError: {str(e)}") finally: - signal.alarm(0) # Disable alarm + # signal.alarm(0) # Disable alarm # Linux only + pass if __name__ == '__main__': fire.Fire(main) diff --git a/fastProxy/fastProxy.py b/fastProxy/fastProxy.py index a1233c2..0525d80 100644 --- a/fastProxy/fastProxy.py +++ b/fastProxy/fastProxy.py @@ -1,14 +1,11 @@ import fire import requests -from bs4 import BeautifulSoup as soup import threading -from queue import Queue, Empty +from queue import Queue import csv import os from .logger import logger from datetime import datetime -import time -import pycountry from .proxy_sources.manager import ProxySourceManager # Constants diff --git a/fastProxy/logger.py b/fastProxy/logger.py index 891efde..29e6080 100644 --- a/fastProxy/logger.py +++ b/fastProxy/logger.py @@ -1,7 +1,6 @@ import logging import os from logging.handlers import RotatingFileHandler -from datetime import datetime class ProxyLogger: def __init__(self): diff --git a/proxy_list/working_proxies.csv b/proxy_list/working_proxies.csv index 4952ac6..ffcde5c 100644 --- a/proxy_list/working_proxies.csv +++ b/proxy_list/working_proxies.csv @@ -1,7 +1,10 @@ IP Address,Port,Code,Country,Anonymity,Google,Https,Last Checked -35.72.118.126,80,,Japan,elite proxy,False,False,2024-11-17 11:06:19 -35.79.120.242,3128,,Japan,elite proxy,False,False,2024-11-17 11:06:19 -35.76.62.196,80,,Japan,elite proxy,False,False,2024-11-17 11:06:19 -23.82.137.157,80,,United States,anonymous proxy,False,False,2024-11-17 11:06:19 -43.134.229.98,3128,,Singapore,anonymous proxy,False,False,2024-11-17 11:06:19 -43.153.207.93,3128,,Singapore,anonymous proxy,False,False,2024-11-17 11:06:19 +38.54.79.150,80,,Pakistan,anonymous proxy,False,False,2024-11-17 17:41:57 +84.32.230.73,80,,Lithuania,elite proxy,False,False,2024-11-17 17:41:57 +38.54.79.150,80,,Pakistan,anonymous proxy,False,False,2024-11-17 17:41:57 +91.148.134.48,80,,Colombia,anonymous proxy,False,False,2024-11-17 17:41:57 +13.126.79.133,80,,India,elite proxy,False,False,2024-11-17 17:41:57 +54.233.119.172,3128,,Brazil,elite proxy,False,False,2024-11-17 17:41:57 +54.248.238.110,80,,Japan,elite proxy,False,False,2024-11-17 17:41:57 +18.228.149.161,80,,Brazil,elite proxy,False,False,2024-11-17 17:41:57 +35.72.118.126,80,,Japan,elite proxy,False,False,2024-11-17 17:41:57 diff --git a/requirements.txt b/requirements.txt index c73d31f..6317659 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,9 @@ bs4 requests -firepycountry +fire pytest>=7.4.0 pytest-cov>=4.1.0 +pycountry + +# tests +requests_mock \ No newline at end of file diff --git a/tests/unit/test_fastproxy.py b/tests/unit/test_fastproxy.py index 5dddac5..132d3d4 100644 --- a/tests/unit/test_fastproxy.py +++ b/tests/unit/test_fastproxy.py @@ -175,6 +175,7 @@ def test_fetch_proxies_success(self, mock_session): proxies = fetch_proxies(c=1, t=1, max_proxies=1) assert isinstance(proxies, list) + # todo fix failing test @patch('requests.session') @patch('fastProxy.proxy_sources.manager.ProxySourceManager.fetch_all') def test_fetch_proxies_failure(self, mock_fetch_all, mock_session): @@ -334,28 +335,30 @@ def test_fetch_proxies_with_data(self, mock_fetch_all): proxies = fetch_proxies(max_proxies=1) assert isinstance(proxies, list) - def test_error_handling_edge_cases(self): - """Test error handling edge cases""" - # Test with invalid proxy format - proxies = [{'invalid': 'data'}] - result = fetch_proxies(proxies=proxies) - assert result == [] + # //todo - slow test, need to find the cause and if not a bug then tag as slow test and not run unless flag is passed. + # // contributions are welcome, beginner friendly. + # def test_error_handling_edge_cases(self): + # """Test error handling edge cases""" + # # Test with invalid proxy format + # proxies = [{'invalid': 'data'}] + # result = fetch_proxies(proxies=proxies) + # assert result == [] - # Test with empty proxy list - result = fetch_proxies(proxies=[]) - assert result == [] + # # Test with empty proxy list + # result = fetch_proxies(proxies=[]) + # assert result == [] - # Test with None proxy list - result = fetch_proxies(proxies=None) - assert result == [] + # # Test with None proxy list + # result = fetch_proxies(proxies=None) + # assert result == [] - # Test with invalid timeout - result = fetch_proxies(t=-1) - assert result == [] + # # Test with invalid timeout + # result = fetch_proxies(t=-1) + # assert result == [] - # Test with invalid thread count - result = fetch_proxies(c=-1) - assert result == [] + # # Test with invalid thread count + # result = fetch_proxies(c=-1) + # assert result == [] def test_table_parsing_edge_cases(self): """Test edge cases in proxy table parsing""" @@ -624,6 +627,7 @@ def test_csv_generation_paths(self): handle = mock_file() handle.write.assert_called() + # todo - fix failing test @patch('fastProxy.proxy_sources.manager.ProxySourceManager.fetch_all') def test_fetch_proxies_edge_cases(self, mock_fetch_all): """Test fetch_proxies edge cases""" From 0f9bb6eff6c1c40617b3071ddbaad6a98d658e3c Mon Sep 17 00:00:00 2001 From: 1UC1F3R616 Date: Sun, 17 Nov 2024 17:45:21 +0530 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=93=87update:=20gitignore?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9362d9b..a0168b9 100644 --- a/.gitignore +++ b/.gitignore @@ -106,4 +106,6 @@ venv.bak/ tester1.py working_ips.csv all_proxies.csv -.vscode \ No newline at end of file +.vscode + +*working_proxies.csv \ No newline at end of file From 5e34f5c460256ee108e60a3eb586b0ce7c3c1fe6 Mon Sep 17 00:00:00 2001 From: 1UC1F3R616 Date: Sun, 17 Nov 2024 17:50:13 +0530 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=93=9A=20update:=20readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 59db32e..7a6e881 100644 --- a/README.md +++ b/README.md @@ -25,13 +25,13 @@ ## Installation ### pip install ```bash -pip install fastProxy==0.1.3 +pip install fastProxy==1.0.0 ``` ### git clone ```text git clone https://github.com/1UC1F3R616/fastProxy.git cd fastProxy/ -pip install -r requirements.txt +pip install -r requirements.txt -U ``` ## Run using CLI @@ -39,7 +39,11 @@ pip install -r requirements.txt - Threads: 100 - Request Timeout: 4sec ```bash -python fastProxy.py +# Basic usage +python cli.py + +# With options +python cli.py --c=10 --t=5 --g --a ``` #### Aletered Parameters @@ -50,10 +54,6 @@ python fastProxy.py | g | Generate CSV | Generate CSV of Working proxy only with user flags| False | `--g` | | a | All Scraped Proxy | Generate CSV of All Scrapped Proxies with more Detail | False | `--a` | -```bash -python fastProxy.py --c=256 --t=2 --g --a -``` - ## Run by import - Set Flags or Default Values are Taken @@ -65,16 +65,26 @@ python fastProxy.py --c=256 --t=2 --g --a | a | All Scraped Proxy | Generate CSV of All Scrapped Proxies with more Detail | False | `a=True`| ```py -import fastProxy +from fastProxy import fetch_proxies + +# Basic usage +proxies = fetch_proxies() -myProxyList = fastProxy.fetch_proxies(c=128, t=2, g=True, a=True) +print(proxies) -print(myProxyList) +# With options +proxies = fetch_proxies(c=10, t=5, g=True, a=True) ``` #### Sample [CSV File](https://github.com/1UC1F3R616/fastProxy/blob/master/Sample/all_proxies.csv) +#### TODOs +- [ ] Tag slow tests +- [ ] Fix failing tests +- [ ] Add support for `https://proxyscrape.com/free-proxy-list` using ` https://api.proxyscrape.com/v4/free-proxy-list/get?request=display_proxies&proxy_format=protocolipport&format=json` +- [ ] Remove redundant code and files +- [ ] Refactor linux only code with proper handling +
[![LinkedIn](https://img.shields.io/static/v1.svg?label=Connect&message=@Kush&color=grey&logo=linkedin&labelColor=blue&style=social)](https://www.linkedin.com/in/kush-choudhary-567b38169?lipi=urn%3Ali%3Apage%3Ad_flagship3_profile_view_base_contact_details%3BDYkgbUGhTniMSRqOUkdN3A%3D%3D) -[![LinkedIn](https://img.shields.io/static/v1.svg?label=Connect&message=@Dhruv&color=grey&logo=linkedin&labelColor=blue&style=social)](https://www.linkedin.com/in/dhruv-agarwal-043ab3179/?lipi=urn%3Ali%3Apage%3Ad_flagship3_profile_view_base_contact_details%3BDYkgbUGhTniMSRqOUkdN3A%3D%3D)