Desc loader

arc53 · Oct 12, 2023 · 2cfb416 · 2cfb416
1 parent 50f07f9
commit 2cfb416
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/application/parser/remote/crawler_loader.py b/application/parser/remote/crawler_loader.py
@@ -7,6 +7,7 @@ class CrawlerLoader(BaseRemote):
     def __init__(self, limit=10):
         from langchain.document_loaders import WebBaseLoader
         self.loader = WebBaseLoader
+        #No pages scraped limit, set None for no limit
         self.limit = limit
 
     def load_data(self, url):