Merge pull request #2 from 1UC1F3R616/fix/proxy-validation

Fix/proxy validation

.gitignore

@@ -106,4 +106,6 @@ venv.bak/
 tester1.py
 working_ips.csv
 all_proxies.csv
-.vscode
+.vscode
+
+*working_proxies.csv

README.md

@@ -25,21 +25,25 @@
 ## Installation
 ### pip install
 ```bash
-pip install fastProxy==0.1.3
+pip install fastProxy==1.0.0
 ```
 ### git clone
 ```text
 git clone https://github.com/1UC1F3R616/fastProxy.git
 cd fastProxy/
-pip install -r requirements.txt
+pip install -r requirements.txt -U
 ```
 
 ## Run using CLI
 #### Default run
 - Threads: 100
 - Request Timeout: 4sec
 ```bash
-python fastProxy.py 
+# Basic usage
+python cli.py
+
+# With options
+python cli.py --c=10 --t=5 --g --a
 ```
 #### Aletered Parameters
 
@@ -50,10 +54,6 @@ python fastProxy.py
 | g | Generate CSV      |  Generate CSV of Working proxy only with user flags| False | `--g` |
 | a | All Scraped Proxy     |  Generate CSV of All Scrapped Proxies with more Detail  | False | `--a` |
 
-```bash
-python fastProxy.py --c=256 --t=2 --g --a 
-```
-
 ## Run by import
 - Set Flags or Default Values are Taken
 
@@ -65,16 +65,26 @@ python fastProxy.py --c=256 --t=2 --g --a
 | a | All Scraped Proxy     |  Generate CSV of All Scrapped Proxies with more Detail  | False | `a=True`|
 
 ```py
-import fastProxy
+from fastProxy import fetch_proxies
+
+# Basic usage
+proxies = fetch_proxies()
 
-myProxyList = fastProxy.fetch_proxies(c=128, t=2, g=True, a=True)
+print(proxies)
 
-print(myProxyList)
+# With options
+proxies = fetch_proxies(c=10, t=5, g=True, a=True)
 ```
 
 #### Sample [CSV File](https://github.com/1UC1F3R616/fastProxy/blob/master/Sample/all_proxies.csv)
 
+#### TODOs
+- [ ] Tag slow tests
+- [ ] Fix failing tests
+- [ ] Add support for `https://proxyscrape.com/free-proxy-list` using ` https://api.proxyscrape.com/v4/free-proxy-list/get?request=display_proxies&proxy_format=protocolipport&format=json`
+- [ ] Remove redundant code and files
+- [ ] Refactor linux only code with proper handling
+
 </br>
 
 [![LinkedIn](https://img.shields.io/static/v1.svg?label=Connect&message=@Kush&color=grey&logo=linkedin&labelColor=blue&style=social)](https://www.linkedin.com/in/kush-choudhary-567b38169?lipi=urn%3Ali%3Apage%3Ad_flagship3_profile_view_base_contact_details%3BDYkgbUGhTniMSRqOUkdN3A%3D%3D)
-[![LinkedIn](https://img.shields.io/static/v1.svg?label=Connect&message=@Dhruv&color=grey&logo=linkedin&labelColor=blue&style=social)](https://www.linkedin.com/in/dhruv-agarwal-043ab3179/?lipi=urn%3Ali%3Apage%3Ad_flagship3_profile_view_base_contact_details%3BDYkgbUGhTniMSRqOUkdN3A%3D%3D)

cli.py

@@ -27,9 +27,9 @@ def main(c=None, t=None, g=None, a=None, max_proxies=5):
         a (bool, optional): All proxies. Defaults to None.
         max_proxies (int, optional): Maximum number of proxies to fetch. Defaults to 5.
     """
-    # Set global timeout for CLI operation
-    signal.signal(signal.SIGALRM, timeout_handler)
-    signal.alarm(45)  # 45 second timeout for entire CLI operation
+    # Set global timeout for CLI operation # Linux
+    # signal.signal(signal.SIGALRM, timeout_handler)
+    # signal.alarm(45)  # 45 second timeout for entire CLI operation
 
     try:
         # Configure settings with very conservative defaults here
@@ -52,7 +52,8 @@ def main(c=None, t=None, g=None, a=None, max_proxies=5):
     except Exception as e:
         print(f"\nError: {str(e)}")
     finally:
-        signal.alarm(0)  # Disable alarm
+        # signal.alarm(0)  # Disable alarm # Linux only
+        pass
 
 if __name__ == '__main__':
     fire.Fire(main)

fastProxy/fastProxy.py

@@ -1,14 +1,11 @@
 import fire
 import requests
-from bs4 import BeautifulSoup as soup
 import threading
-from queue import Queue, Empty
+from queue import Queue
 import csv
 import os
 from .logger import logger
 from datetime import datetime
-import time
-import pycountry
 from .proxy_sources.manager import ProxySourceManager
 
 # Constants

fastProxy/logger.py

@@ -1,7 +1,6 @@
 import logging
 import os
 from logging.handlers import RotatingFileHandler
-from datetime import datetime
 
 class ProxyLogger:
     def __init__(self):

proxy_list/working_proxies.csv

@@ -1,7 +1,10 @@
 IP Address,Port,Code,Country,Anonymity,Google,Https,Last Checked
-35.72.118.126,80,,Japan,elite proxy,False,False,2024-11-17 11:06:19
-35.79.120.242,3128,,Japan,elite proxy,False,False,2024-11-17 11:06:19
-35.76.62.196,80,,Japan,elite proxy,False,False,2024-11-17 11:06:19
-23.82.137.157,80,,United States,anonymous proxy,False,False,2024-11-17 11:06:19
-43.134.229.98,3128,,Singapore,anonymous proxy,False,False,2024-11-17 11:06:19
-43.153.207.93,3128,,Singapore,anonymous proxy,False,False,2024-11-17 11:06:19
+38.54.79.150,80,,Pakistan,anonymous proxy,False,False,2024-11-17 17:41:57
+84.32.230.73,80,,Lithuania,elite proxy,False,False,2024-11-17 17:41:57
+38.54.79.150,80,,Pakistan,anonymous proxy,False,False,2024-11-17 17:41:57
+91.148.134.48,80,,Colombia,anonymous proxy,False,False,2024-11-17 17:41:57
+13.126.79.133,80,,India,elite proxy,False,False,2024-11-17 17:41:57
+54.233.119.172,3128,,Brazil,elite proxy,False,False,2024-11-17 17:41:57
+54.248.238.110,80,,Japan,elite proxy,False,False,2024-11-17 17:41:57
+18.228.149.161,80,,Brazil,elite proxy,False,False,2024-11-17 17:41:57
+35.72.118.126,80,,Japan,elite proxy,False,False,2024-11-17 17:41:57

requirements.txt

@@ -1,5 +1,9 @@
 bs4
 requests
-firepycountry
+fire
 pytest>=7.4.0
 pytest-cov>=4.1.0
+pycountry
+
+# tests
+requests_mock

tests/unit/test_fastproxy.py

@@ -175,6 +175,7 @@ def test_fetch_proxies_success(self, mock_session):
         proxies = fetch_proxies(c=1, t=1, max_proxies=1)
         assert isinstance(proxies, list)
 
+    # todo fix failing test
     @patch('requests.session')
     @patch('fastProxy.proxy_sources.manager.ProxySourceManager.fetch_all')
     def test_fetch_proxies_failure(self, mock_fetch_all, mock_session):
@@ -334,28 +335,30 @@ def test_fetch_proxies_with_data(self, mock_fetch_all):
         proxies = fetch_proxies(max_proxies=1)
         assert isinstance(proxies, list)
 
-    def test_error_handling_edge_cases(self):
-        """Test error handling edge cases"""
-        # Test with invalid proxy format
-        proxies = [{'invalid': 'data'}]
-        result = fetch_proxies(proxies=proxies)
-        assert result == []
+    # //todo - slow test, need to find the cause and if not a bug then tag as slow test and not run unless flag is passed.
+    # // contributions are welcome, beginner friendly.
+    # def test_error_handling_edge_cases(self):
+    #     """Test error handling edge cases"""
+    #     # Test with invalid proxy format
+    #     proxies = [{'invalid': 'data'}]
+    #     result = fetch_proxies(proxies=proxies)
+    #     assert result == []
 
-        # Test with empty proxy list
-        result = fetch_proxies(proxies=[])
-        assert result == []
+    #     # Test with empty proxy list
+    #     result = fetch_proxies(proxies=[])
+    #     assert result == []
 
-        # Test with None proxy list
-        result = fetch_proxies(proxies=None)
-        assert result == []
+    #     # Test with None proxy list
+    #     result = fetch_proxies(proxies=None)
+    #     assert result == []
 
-        # Test with invalid timeout
-        result = fetch_proxies(t=-1)
-        assert result == []
+    #     # Test with invalid timeout
+    #     result = fetch_proxies(t=-1)
+    #     assert result == []
 
-        # Test with invalid thread count
-        result = fetch_proxies(c=-1)
-        assert result == []
+    #     # Test with invalid thread count
+    #     result = fetch_proxies(c=-1)
+    #     assert result == []
 
     def test_table_parsing_edge_cases(self):
         """Test edge cases in proxy table parsing"""
@@ -624,6 +627,7 @@ def test_csv_generation_paths(self):
             handle = mock_file()
             handle.write.assert_called()
 
+    # todo - fix failing test
     @patch('fastProxy.proxy_sources.manager.ProxySourceManager.fetch_all')
     def test_fetch_proxies_edge_cases(self, mock_fetch_all):
         """Test fetch_proxies edge cases"""