Merge pull request #33 from nlnwa/timeout

Add option for setting timeout on crawlSeed request to Frontier
nlnwa · Oct 28, 2020 · ee3d429 · ee3d429
2 parents d38b8d6 + af9c582
commit ee3d429
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/protobuf/frontier/v1/frontier.proto b/protobuf/frontier/v1/frontier.proto
@@ -6,6 +6,7 @@ import "commons/v1/resources.proto";
 import "config/v1/resources.proto";
 import "frontier/v1/resources.proto";
 import "google/protobuf/empty.proto";
+import "google/protobuf/timestamp.proto";
 
 option go_package = "github.com/nlnwa/veidemann-api-go/frontier/v1;frontier";
 option java_multiple_files = true;
@@ -38,6 +39,8 @@ message CrawlSeedRequest {
     string job_execution_id = 1;
     veidemann.api.config.v1.ConfigObject job = 5;
     veidemann.api.config.v1.ConfigObject seed = 6;
+    // When this seed should stop crawling. Absence of this value indicates no timeout
+    google.protobuf.Timestamp timeout = 7;
 }
 
 // The execution id for a seed crawl

diff --git a/protobuf/frontier/v1/resources.proto b/protobuf/frontier/v1/resources.proto
@@ -72,8 +72,6 @@ message Cookie {
 message CrawlHostGroup {
     string id = 1; // SHA-1 hash of uri or crawlgroup config id
     string politeness_id = 2; // The politeness config this group is valid for
-    google.protobuf.Timestamp next_fetch_time = 3; // The earliest time a URI from this group might be fetched
-    bool busy = 4; // True if crawler is busy with fetching a URI from this group
     int64 queued_uri_count = 5; // The number of queued Uri's belonging to this CrawlHostGroup
 }