Merge branch 'develop'

3c7 · Jul 15, 2023 · d0f67b4 · d0f67b4
2 parents 1a7c44b + d4af246
commit d0f67b4
Show file tree

Hide file tree

Showing 5 changed files with 4,104 additions and 93 deletions.
diff --git a/common_osint_model/models/http.py b/common_osint_model/models/http.py
@@ -3,6 +3,7 @@
 
 import mmh3
 from pydantic import BaseModel
+from hhhash import hash_from_banner
 
 from common_osint_model.models import ShodanDataHandler, CensysDataHandler, BinaryEdgeDataHandler, Logger
 from common_osint_model.utils import hash_all
@@ -111,7 +112,7 @@ def from_shodan(cls, d: Dict):
 
         raw = d["http"]["securitytxt"].encode("utf-8")
         md5, sha1, sha256, murmur = hash_all(raw)
-        return HTTPComponentContentRobots(
+        return HTTPComponentContentSecurity(
             raw=raw,
             md5=md5,
             sha1=sha1,
@@ -219,6 +220,7 @@ class HTTPComponent(BaseModel, ShodanDataHandler, CensysDataHandler, BinaryEdgeD
     headers: Optional[Dict[str, str]]
     content: Optional[HTTPComponentContent]
     shodan_headers_hash: Optional[str]
+    hhhash: Optional[str]
 
     @classmethod
     def from_shodan(cls, d: Dict):
@@ -240,7 +242,8 @@ def from_shodan(cls, d: Dict):
         return HTTPComponent(
             headers=headers,
             content=content,
-            shodan_headers_hash=d.get("http", {}).get("headers_hash", None)
+            shodan_headers_hash=d.get("http", {}).get("headers_hash", None),
+            hhhash=hash_from_banner(banner)
         )
 
     @classmethod
@@ -267,7 +270,8 @@ def from_censys(cls, d: Dict):
         return HTTPComponent(
             headers=headers,
             content=HTTPComponentContent.from_censys(d),
-            shodan_headers_hash=headers_hash
+            shodan_headers_hash=headers_hash,
+            hhhash=hash_from_banner(d["banner"])
         )
 
     @classmethod