feat: add albania_public_procurement_commission_historical and base c…

…lass

docs/spiders.rst

@@ -136,6 +136,13 @@ Albania
 
    scrapy crawl albania_public_procurement_commission
 
+.. autoclass:: kingfisher_scrapy.spiders.albania_public_procurement_commission_historical.AlbaniaPublicProcurementCommissionHistorical
+   :no-members:
+
+.. code-block:: bash
+
+   scrapy crawl albania_public_procurement_commission_historical
+
 Argentina
 ~~~~~~~~~
 

kingfisher_scrapy/spiders/albania_public_procurement_commission.py

@@ -1,12 +1,7 @@
-import json
+from kingfisher_scrapy.spiders.albania_public_procurement_commission_base import AlbaniaPublicProcurementCommissionBase
 
-import scrapy
 
-from kingfisher_scrapy import util
-from kingfisher_scrapy.base_spiders import SimpleSpider
-
-
-class AlbaniaPublicProcurementCommission(SimpleSpider):
+class AlbaniaPublicProcurementCommission(AlbaniaPublicProcurementCommissionBase):
     """
     Domain
       Komisioni i Prokurimit Publik (KPP) (Public Procurement Commission)
@@ -23,24 +18,8 @@ class AlbaniaPublicProcurementCommission(SimpleSpider):
     name = 'albania_public_procurement_commission'
 
     # BaseSpider
-    date_format = 'year'
     default_from_date = '2021'
-    date_required = True
-    root_path = 'result'
-
-    # SimpleSpider
-    data_type = 'release_package'
 
-    def start_requests(self):
-        url = 'https://kpp.al/api/public/Decision/getBulkJsonByYear'
-        for year in util.date_range_by_year(self.from_date.year, self.until_date.year):
-            payload = {
-                "extraConditions": [
-                    ["decision_date", ">=", f"1/1/{year}"],
-                    ["decision_date", "<=", f"1/1/{year + 1}"],
-                ]
-            }
-            yield scrapy.Request(
-                url, meta={'file_name': f'{year}.json'}, method='POST', body=json.dumps(payload),
-                headers={'Accept': 'application/json', 'Content-Type': 'application/json'}
-            )
+    # AlbaniaPublicProcurementCommissionBase
+    base_url = 'https://kpp.al/api/public/Decision/getBulkJsonByYear'
+    date_param = 'decision_date'

kingfisher_scrapy/spiders/albania_public_procurement_commission_base.py

@@ -0,0 +1,29 @@
+import json
+
+import scrapy
+
+from kingfisher_scrapy import util
+from kingfisher_scrapy.base_spiders import SimpleSpider
+
+
+class AlbaniaPublicProcurementCommissionBase(SimpleSpider):
+    # BaseSpider
+    date_format = 'year'
+    date_required = True
+    root_path = 'result'
+
+    # SimpleSpider
+    data_type = 'release_package'
+
+    def start_requests(self):
+        for year in util.date_range_by_year(self.from_date.year, self.until_date.year):
+            payload = {
+                "extraConditions": [
+                    [self.date_param, ">=", f"1/1/{year}"],
+                    [self.date_param, "<=", f"1/1/{year + 1}"],
+                ]
+            }
+            yield scrapy.Request(
+                self.base_url, meta={'file_name': f'{year}.json'}, method='POST', body=json.dumps(payload),
+                headers={'Accept': 'application/json', 'Content-Type': 'application/json'}
+            )

kingfisher_scrapy/spiders/albania_public_procurement_commission_historical.py

@@ -0,0 +1,27 @@
+from kingfisher_scrapy.spiders.albania_public_procurement_commission_base import AlbaniaPublicProcurementCommissionBase
+
+
+class AlbaniaPublicProcurementCommissionHistorical(AlbaniaPublicProcurementCommissionBase):
+    """
+    Domain
+      Komisioni i Prokurimit Publik (KPP) (Public Procurement Commission)
+    Spider arguments
+      from_date
+        Download only data from this year onward (YYYY format). Defaults to '2010'.
+      until_date
+        Download only data until this year (YYYY format). Defaults to '2020'.
+
+    Bulk download documentation
+      https://kpp.al/en/HistorikVendimesh?objektProkurimi=&viti=&vitiId=&muaji=&muajiId=
+    """
+
+    name = 'albania_public_procurement_commission_historical'
+
+    # BaseSpider
+    default_from_date = '2010'
+    default_until_date = '2020'
+    skip_pluck = 'Already covered (see code for details)'  # albania_public_procurement_commission
+
+    # AlbaniaPublicProcurementCommissionBase
+    base_url = 'https://kpp.al/api/public/Decision/getHistoricBulkJsonByYear'
+    date_param = 'prot_date'