Faster tender scraping

Author: akariv

budgetkey_data_pipelines/pipelines/procurement/tenders/pipeline-spec.yaml

@@ -21,9 +21,9 @@ scraper-exemptions:
         input_resource: mr-gov-il-search-exemption-messages
         tender_type: exemptions
     - run: throttle
-      # parameters:
-      #   sleep-seconds: 0.5
-      #   log-interval-seconds: 2
+      parameters:
+        sleep-seconds: 0.25
+#         log-interval-seconds: 0
     # go over each publisher and get urls for all exemptions related to this publisher
     # also adds tender_type column - which is needed as all tenders are on same table
     - run: add_publisher_urls_resource
@@ -58,6 +58,8 @@ scraper-office:
         input_resource: mr-gov-il-search-office-tenders
         tender_type: office
     - run: throttle
+      parameters:
+        sleep-seconds: 0.25
     # go over each publisher and get urls for all office tenders related to this publisher
     # also adds tender_type column - which is needed as all tenders are on same table
     - run: add_publisher_urls_resource