From 89d3b8cdfa69a9cd3e3eba94c3967cb82669281e Mon Sep 17 00:00:00 2001 From: 3c7 <3c7@posteo.de> Date: Tue, 15 Aug 2023 10:34:12 +0200 Subject: [PATCH 1/2] Removed old code & updated dependencies --- README.md | 16 -- common_osint_model/__init__.py | 10 - common_osint_model/censys/__init__.py | 0 common_osint_model/censys/v1.py | 297 -------------------------- common_osint_model/censys/v2.py | 169 --------------- common_osint_model/certificate.py | 126 ----------- common_osint_model/cli.py | 79 ------- common_osint_model/shodan.py | 273 ----------------------- common_osint_model/utils.py | 60 ------ poetry.lock | 152 +++++++------ pyproject.toml | 2 +- 11 files changed, 85 insertions(+), 1099 deletions(-) delete mode 100644 common_osint_model/censys/__init__.py delete mode 100644 common_osint_model/censys/v1.py delete mode 100644 common_osint_model/censys/v2.py delete mode 100644 common_osint_model/certificate.py delete mode 100644 common_osint_model/cli.py delete mode 100644 common_osint_model/shodan.py diff --git a/README.md b/README.md index bd94530..acf129c 100644 --- a/README.md +++ b/README.md @@ -96,22 +96,6 @@ for idx, service in enumerate(host.services): print("Exporting data as flattened json blob for further use, e.g. Elasticsearch indexing...") print(host.flattened_json()) - -# Pre-v0.4.0 (Dictionary based approach - DEPRECATED) -from common_osint_model import from_shodan, from_shodan_flattened, from_censys_ipv4, from_censys_ipv4_flattened, - from_x509_pem, from_x509_pem_flattened - -raw_shodan = get_my_shodan_data() -converted_s = from_shodan(raw_shodan) # Deprecated -flattened_s = from_shodan_flattened(raw_shodan) # Deprecated - -raw_censys_ipv4 = get_my_censys_ipv4_data() -converted_c = from_censys_ipv4(raw_censys_ipv4) -flattened_c = from_censys_ipv4_flattened(raw_censys_ipv4) - -raw_certificate = open('certificate.pem').read() -converted_certificate = from_x509_pem(raw_shodan) -flattened_certificate = from_x509_pem_flattened(raw_shodan) ``` ### Example output diff --git a/common_osint_model/__init__.py b/common_osint_model/__init__.py index cacc0eb..7cd294d 100644 --- a/common_osint_model/__init__.py +++ b/common_osint_model/__init__.py @@ -1,13 +1,3 @@ -from common_osint_model.shodan import from_shodan, from_shodan_flattened -from common_osint_model.censys.v1 import ( - from_censys_ipv4, - from_censys_ipv4_flattened, - from_censys_certificates, - from_censys_certificates_flattened -) -from common_osint_model.censys.v2 import from_censys, from_censys_flattened -from common_osint_model.certificate import from_x509_pem, from_x509_pem_flattened - from common_osint_model.models.host import Host from common_osint_model.models.domain import * from common_osint_model.models.service import Service diff --git a/common_osint_model/censys/__init__.py b/common_osint_model/censys/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/common_osint_model/censys/v1.py b/common_osint_model/censys/v1.py deleted file mode 100644 index 18df4a8..0000000 --- a/common_osint_model/censys/v1.py +++ /dev/null @@ -1,297 +0,0 @@ -import warnings -from datetime import datetime - -from DateTime import DateTime -from mmh3 import hash as mmh3_hash - -from common_osint_model.utils import flatten, unflatten, common_model_cn_extraction, sha256_from_body_string, \ - list_cleanup - - -def from_censys_ipv4(raw: dict) -> dict: - """ - Converts a Censys IPv4 dictionary into the common format - :param raw: Censys IPv4 dict - :return: Common format dict - """ - warnings.warn("This function was deprecated in v0.4.0. and will be removed in v0.5.0", DeprecationWarning) - flattened = False - for k in raw.keys(): - if "." in k: - flattened = True - break - elif k == "443" or k == "80" or k == "22" or k == "autonomous_system": - break - - if flattened: - raw = unflatten(raw) - - g = {} - ports = [] - g.update(censys_ipv4_meta_extraction(raw)) - for protocol in raw.get("protocols", []): - (port, protocol) = protocol.split("/") - ports.append(port) - g.update(censys_ipv4_service_extraction(raw, port, protocol)) - g.update(dict(ports=ports)) - g["domains"] = common_model_cn_extraction(g) - return g - - -def from_censys_ipv4_flattened(raw: dict) -> dict: - """ - Converts a Censys IPv4 dictionary into the common format - :param raw: Censys IPv4 dict - :return: Common format dict, flattened - """ - warnings.warn("This function was deprecated in v0.4.0. and will be removed in v0.5.0", DeprecationWarning) - return flatten(from_censys_ipv4(raw)) - - -def censys_ipv4_meta_extraction(raw: dict) -> dict: - """ - Extracts metadata from Censys IPv4 dicts - :param raw: Censys IPv4 dict - :return: Metadata part of common format dict - """ - _as = raw.get("autonomous_system", None) or dict() - return { - "ip": raw["ip"], # Program should fail if IP is not given - "as": { - "number": _as.get("asn", None), - "name": _as.get("name", None), - "location": _as.get("country_code", None), - "prefix": _as.get("routed_prefix", None), - }, - } - - -def censys_ipv4_service_extraction(raw: dict, port: str, protocol: str) -> dict: - """ - Extracts meta information and routes to correct method for service detail extraction. - :param raw: Censys IPv4 dict - :param port: Port as str - :param protocol: Protocol as str - :return: Service dictionary - """ - s = raw.get(port, {}).get(protocol, None) or dict() - timestamp = raw.get("updated_at", None) - service = { - "timestamp": int(DateTime(timestamp)) if timestamp else None, - "timestamp_readable": DateTime(timestamp).ISO8601() if timestamp else None, - } - keys = s.keys() - if "banner_decoded" in keys: - service.update(dict(banner=s["banner_decoded"])) - if "get" in keys: - http = censys_ipv4_http_extraction(s["get"]) - service.update({"http": http}) - if "tls" in keys: - tls = censys_ipv4_tls_extraction(s["tls"]) - service.update({"tls": tls}) - if protocol == 'ssh': - ssh = censys_ipv4_ssh_extraction(s) - service.update({"ssh": ssh}) - return {port: service} - - -def censys_ipv4_http_extraction(s: dict) -> dict: - """ - Extracts HTTP relevant data out ot service part of Censys IPv4 dict - :param s: Service part of a censys dict - :return: Dictionary with HTTP data - """ - headers = s.get("headers", {}) - for h in headers.get("unknown", []): - headers.update({h["key"].lower().replace("-", "_"): h["value"]}) - if "unknown" in headers.keys(): - del headers["unknown"] - return { - "headers": headers, - "content": { - "html": s.get("body", None), - "hash": { - "shodan": mmh3_hash(s.get("body", None) or ""), - "sha256": s.get("body_sha256", None) or sha256_from_body_string("") - }, - "favicon": {"shodan": None, "sha256": None}, - }, - } - - -def censys_ipv4_tls_extraction(s: dict) -> dict: - """ - Extracts TLS relevant data out ot service part of Censys IPv4 dict - :param s: Service part of a censys dict - :return: Dictionary with TLS data - """ - cert = s.get("certificate", {}).get("parsed", {}) - subject = cert.get("subject", None) or dict() - issuer = cert.get("issuer", None) or dict() - validity = cert.get("validity", None) or dict() - common_name = subject.get("common_name", []) - common_name.extend(cert.get("names", [])) - if len(common_name) == 0: - common_name = None - cert_issued = DateTime(validity.get("start", None)) - cert_expires = DateTime(validity.get("end", None)) - cert_length = validity.get("length", None) - return { - "certificate": { - "issuer_dn": cert.get("issuer_dn", None), - "subject_dn": cert.get("subject_dn", None), - "issuer": { - # Censys always uses lists for those kind of attributes - "common_name": issuer.get("common_name", [None])[0], - "country": issuer.get("country", [None])[0], - "locality": issuer.get("locality", [None])[0], - "province": issuer.get("province", [None])[0], - "organization": issuer.get("organization", [None])[0], - "organizational_unit": issuer.get("organizational_unit", [None])[0], - "email_address": issuer.get("email_address", [None])[0], - }, - "subject": { - # Censys always uses lists for those kind of attributes, multiple CNs are okay, though - "common_name": common_name, - "country": subject.get("country", [None])[0], - "locality": subject.get("locality", [None])[0], - "province": subject.get("province", [None])[0], - "organization": subject.get("organization", [None])[0], - "organizational_unit": subject.get("organizational_unit", [None])[0], - "email_address": subject.get("email_address", [None])[0], - }, - "validity": { - "start": int(cert_issued), - "start_readable": cert_issued.ISO8601(), - "end": int(cert_expires), - "end_readable": cert_expires.ISO8601(), - "length": cert_length - }, - "fingerprint": { - "sha1": cert.get("fingerprint_sha1", None), - "sha256": cert.get("fingerprint_sha256", None) - } - } - } - - -def censys_ipv4_ssh_extraction(s: dict) -> dict: - """ - Extracts SSH relevant data out ot service part of Censys IPv4 dict - :param s: Service part of a censys dict - :return: Dictionary with SSH data - """ - v2 = s.get("v2", None) or dict() - banner = v2.get("banner", None) or dict() - support = v2.get("support", None) or dict() - s2c = support.get("server_to_client", None) or dict() - shk = v2.get("server_host_key", None) or dict() - return { - "version": banner.get("raw", None), - "key_exchange": { - "algorithms": { - "compression": s2c.get("compressions", None), - "encryption": s2c.get("ciphers", None), - "key_exchange": support.get("kex_algorithms", None), - "mac": s2c.get("macs", None), - "key_algorithms": support.get("host_key_algorithms", None) - } - }, - "key": { - "hash": { - "sha256": shk.get("fingerprint_sha256", None) - }, - "type": shk.get("key_algorithm", None) - } - } - - -def from_censys_certificates(raw: dict) -> dict: - """ - Converts a censys certificates dictionary to the common model format - :param raw: Censys certificates dict - :return: Common model dict - """ - flattened = False - for k in raw.keys(): - if "." in k: - flattened = True - break - elif k == "parsed": - break - - if flattened: - raw = unflatten(raw) - - return dict(censys_certificates_parsed_extraction(raw["parsed"])) - - -def from_censys_certificates_flattened(raw: dict) -> dict: - """ - Converts a censys certificates dictionary to the flattened common model format - :param raw: Censys certificates dict - :return: Common model dict - """ - return flatten(from_censys_certificates(raw)) - - -def censys_certificates_parsed_extraction(parsed: dict) -> dict: - """ - Extracts the parsed certificate data - :param p: "Parsed" dictionary of censys data - :return: Dictionary with parsed certificate data in common model format - """ - issuer = {} - subject = {} - fingerprint = {} - validity = {} - for issuer_key, issuer_item in parsed.get("issuer", {}).items(): - issuer.update({ - issuer_key: issuer_item - }) - for subject_key, subject_item in parsed.get("subject", {}).items(): - subject.update({ - subject_key: subject_item - }) - for fp_key, fp_item in parsed.items(): - if "fingerprint_" not in fp_key: - continue - fingerprint.update({ - fp_key.replace("fingerprint_", ""): fp_item - }) - - for extension, content in parsed.get("extensions", {}).items(): - if extension == "subject_alt_name": - cn = subject.get("common_name", None) - altnames = content.get("dns_names", []) - if cn and cn in altnames or not cn: - subject["common_name"] = altnames - elif cn and cn not in altnames: - if isinstance(cn, str): - altnames.append(cn) - elif isinstance(cn, list): - for name in cn: - if name not in altnames: - altnames.append(name) - subject["common_name"] = altnames - - start = datetime.strptime(parsed["validity"]["start"], "%Y-%m-%dT%H:%M:%SZ") - end = datetime.strptime(parsed["validity"]["end"], "%Y-%m-%dT%H:%M:%SZ") - validity = { - "start": int(start.timestamp()), - "start_readable": start.isoformat(), - "end": int(end.timestamp()), - "end_readable": end.isoformat(), - "length": int((end - start).total_seconds()) - } - - p = { - "issuer": issuer, - "issuer_dn": parsed.get("issuer_dn", None), - "subject": subject, - "subject_dn": parsed.get("subject_dn", None), - "fingerprint": fingerprint, - "validity": validity, - "serial_number": parsed.get("serial_number", None) - } - return list_cleanup(p) diff --git a/common_osint_model/censys/v2.py b/common_osint_model/censys/v2.py deleted file mode 100644 index 53d8b38..0000000 --- a/common_osint_model/censys/v2.py +++ /dev/null @@ -1,169 +0,0 @@ -import warnings -from typing import Dict, List, Any - -from DateTime import DateTime -from mmh3 import hash as mmh3_hash - -from common_osint_model.utils import sha256_from_body_string, flatten - - -def from_censys(raw: Dict) -> Dict: - """ - Convert Censys data model to the common data model. - - :param raw: Censys Search 2.0 dictionary - """ - warnings.warn("This function was deprecated in v0.4.0. and will be removed in v0.5.0", DeprecationWarning) - common = {} - common.update( - censys_meta_extraction(raw) - ) - - for service in raw.get("services", []): - common.update(censys_extract_service(service)) - return common - - -def from_censys_flattened(raw: Dict) -> Dict: - warnings.warn("This function was deprecated in v0.4.0. and will be removed in v0.5.0", DeprecationWarning) - return flatten(from_censys(raw)) - - -def censys_meta_extraction(raw: Dict) -> Dict: - """ - Returns all metadata. - - :param raw: Censys Search 2.0 dictionary - """ - _as = raw.get("autonomous_system", {}) - return { - "ip": raw.get("ip", "None"), - "as": { - "name": _as.get("name", "None"), - "number": _as.get("asn", "None"), - "description": _as.get("description", "None"), - "location": _as.get("country_code", "None"), - "prefix": _as.get("bgp_prefix", "None") - }, - "location": { - **raw.get("location", {}) - }, - "ports": [service["port"] for service in raw.get("services", [])] - } - - -def censys_extract_service(service: Dict) -> Dict: - """ - Extracts relevant information from a service object/dict. - - :param service: Censys Search 2.0 service dictionary - """ - port = service["port"] - timestamp = service.get("observed_at", None) - if timestamp: - timestamp = DateTime(timestamp) - s_common = { - "banner": service.get("banner", None), - "timestamp": int(timestamp), - "timestamp_readable": timestamp.ISO8601() - } - if "http" in service: - s_common.update({"http": censys_extract_http_service(service)}) - if "tls" in service: - s_common.update({"tls": censys_extract_tls_service(service)}) - return { - port: s_common - } - - -def censys_extract_http_service(service: Dict) -> Dict: - """Extracts relevant http service fields. - - :param service: Censys Search 2.0 service dictionary - """ - s_http = {} - res = service.get("http", {}).get("response", None) - if not res: - return {} - - headers = res.get("headers", None) - if headers: - s_http["headers"] = {} - for k, v in headers.items(): - if k == "_encoding": - continue - - s_http["headers"][k.lower()] = v[0] - s_http["content"] = { - "html": res.get("body"), - "hash": { - "shodan": mmh3_hash(res.get("body", None) or ""), - "sha256": sha256_from_body_string(res.get("body", None) or ""), - "censys": res.get("body_hash", None) - } - } - return s_http - - -def censys_extract_tls_service(service: Dict) -> Dict: - """Extracts relevant tls service fields. - - :param service: Censys Search 2.0 service dictionary - """ - s_tls = {} - cert = service.get("tls", {}).get("certificates", {}).get("leaf_data", None) - c_issuer = cert.get("issuer", None) or dict() - c_subject = cert.get("subject", None) or dict() - common_name = c_subject.get("common_name", []) - common_name.extend(cert.get("names", [])) - if len(common_name) == 0: - common_name = None - else: - common_name = sorted(list(set(common_name))) - if not cert: - return {} - - s_tls["certificate"] = { - "issuer_dn": cert.get("issuer_dn", None), - "subject_dn": cert.get("subject_dn", None), - "issuer": { - "common_name": _first_or_none(c_issuer["common_name"]), - # MISSING! "country": _first_or_none(c_issuer["country"]), - "locality": _first_or_none(c_issuer["locality"]), - "province": _first_or_none(c_issuer["province"]), - "organization": _first_or_none(c_issuer["organization"]), - "organizational_unit": _first_or_none(c_issuer["organizational_unit"]), - # MISSING! "email_address": _first_or_none(c_issuer["email_address"]), - }, - "subject": { - "common_name": common_name, - # MISSING! "country": _first_or_none(c_issuer["country"]), - "locality": _first_or_none(c_subject["locality"]), - "province": _first_or_none(c_subject["province"]), - "organization": _first_or_none(c_subject["organization"]), - "organizational_unit": _first_or_none(c_subject["organizational_unit"]), - # MISSING! "email_address": _first_or_none(c_subject["email_address"]), - }, - "fingerprint": { - "sha256": cert.get("fingerprint", None) - } - } - return s_tls - - -def censys_extract_ssh(service: Dict) -> Dict: - """ - Extracts relevant ssh service fields. - - :param service: Censys Search 2.0 service dictionary - """ - s_ssh = {} - - -def _first_or_none(l: List) -> Any: - """Returns first element of list or none, if list is empty.""" - if not l: - return None - if len(l) > 0: - return l[0] - return None diff --git a/common_osint_model/certificate.py b/common_osint_model/certificate.py deleted file mode 100644 index 629bdd2..0000000 --- a/common_osint_model/certificate.py +++ /dev/null @@ -1,126 +0,0 @@ -from cryptography.x509 import load_pem_x509_certificate -from cryptography.x509.extensions import SubjectAlternativeName -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.hashes import MD5, SHA1, SHA256 -from cryptography.hazmat.backends.openssl.x509 import _Certificate as Certificate -from binascii import hexlify -from typing import Union -from datetime import timezone -from common_osint_model.utils import flatten, list_cleanup - - -def from_x509_pem(pem: Union[str, bytes]) -> dict: - """ - Converts a certificate in PEM format given as bytes or as string to a dictionary. - :param pem: PEM certificates as either string or bytes - :return: dictionary in common format - """ - g = {} - if isinstance(pem, str): - pem = pem.encode("ascii") - - certificate = load_pem_x509_certificate(pem, default_backend()) - g.update(certificate_dn_extraction(certificate)) - g.update(certificate_fingerprint_extraction(certificate)) - g.update(certificate_validity_extraction(certificate)) - g.update(dict(serial_number=certificate.serial_number)) - return list_cleanup(g) - - -def from_x509_pem_flattened(pem: Union[str, bytes]) -> dict: - """ - Wraps from_x509_pem and flattens the output dict - :param pem: PEM certificates as either string or bytes - :return: flattened dictionary in common format - """ - return flatten(from_x509_pem(pem)) - - -def certificate_dn_extraction(certificate: Certificate) -> dict: - """ - Extracts distinguished names of the given certificate - :param certificate: object of type cryptography.hazmat.backends.openssl.x509._Certificate - :return: dictionary containing issuer and subject DN - """ - dns = dict(subject={}, issuer={}, subject_dn=None, issuer_dn=None) - terms = dict( - CN='common_name', - C='country', - L='locality', - ST='province', - O='organization', - OU='organizational_unit', - email='email_address' - ) - - dns["issuer_dn"] = certificate.issuer.rfc4514_string() - for term in certificate.issuer.rfc4514_string().split(','): - k, v = term.split("=") - key = terms[k.strip()] - if key in dns["issuer"].keys(): - if isinstance(dns["issuer"][key], list): - dns["issuer"][key].append(v) - else: - dns["issuer"][key] = [dns["issuer"][key], v] - else: - dns["issuer"].update({ - terms[k.strip()]: v - }) - - dns["subject_dn"] = certificate.subject.rfc4514_string() - for term in certificate.subject.rfc4514_string().split(','): - k, v = term.split("=") - key = terms[k.strip()] - if key in dns["subject"].keys(): - if isinstance(dns["subject"][key], list): - dns["subject"][key].append(v) - else: - dns["subject"][key] = [dns["subject"][key], v] - else: - dns["subject"].update({ - terms[k.strip()]: v - }) - try: - subjectAltName = certificate.extensions.get_extension_for_oid(SubjectAlternativeName.oid) - except: - subjectAltName = None - - if subjectAltName: - dns["subject"]["common_name"] = [dns["subject"]["common_name"]] - for v in subjectAltName.value: - if v.value not in dns["subject"]["common_name"]: - dns["subject"]["common_name"].append(v.value) - - return dns - - -def certificate_fingerprint_extraction(certificate: Certificate) -> dict: - """ - Calculates certificate fingerprints as MD5, SHA1 and SHA256 - :param certificate: object of type cryptography.hazmat.backends.openssl.x509._Certificate - :return: dictionary containing all fingerprints - """ - return { - "fingerprint": { - "md5": hexlify(certificate.fingerprint(MD5())).decode("ascii"), - "sha1": hexlify(certificate.fingerprint(SHA1())).decode("ascii"), - "sha256": hexlify(certificate.fingerprint(SHA256())).decode("ascii"), - } - } - - -def certificate_validity_extraction(certificate: Certificate) -> dict: - """ - Extracts validity information of given certificate - :param certificate: object of type cryptography.hazmat.backends.openssl.x509._Certificate - :return: dictionary containing the validity timestamps - """ - return { - "validity": { - "start": int(certificate.not_valid_before.timestamp()), - "start_readable": certificate.not_valid_before.replace(tzinfo=timezone.utc, microsecond=0).isoformat(), - "end": int(certificate.not_valid_after.timestamp()), - "end_readable": certificate.not_valid_after.replace(tzinfo=timezone.utc, microsecond=0).isoformat(), - "length": int((certificate.not_valid_after - certificate.not_valid_before).total_seconds()) - } - } diff --git a/common_osint_model/cli.py b/common_osint_model/cli.py deleted file mode 100644 index 4624fa4..0000000 --- a/common_osint_model/cli.py +++ /dev/null @@ -1,79 +0,0 @@ -import io -import json -from argparse import ArgumentParser - -from common_osint_model import * - - -def convshodan() -> None: - """ - Converts a JSON file with shodan data to the common data model - """ - args = parse_args() - if args.flatten: - convert(args.filepath, from_shodan_flattened, args.indent) - else: - convert(args.filepath, from_shodan, args.indent) - - -def convcensys(): - """ - Converts a JSON file with shodan data to the common data model - """ - args = parse_args() - if args.flatten: - convert(args.filepath, from_censys_ipv4_flattened, args.indent) - else: - convert(args.filepath, from_censys_ipv4, args.indent) - - -def convcensyscert(): - """ - Converts a JSON file with censys certificate data to the common data model - """ - args = parse_args() - if args.flatten: - convert(args.filepath, from_censys_certificates_flattened, args.indent) - else: - convert(args.filepath, from_censys_certificates, args.indent) - - -def convcensys2(): - """ - Converts a JSON file with censys search 2.0 data to the common data model - """ - args = parse_args() - if args.flatten: - convert(args.filepath, from_censys_flattened, args.indent) - else: - convert(args.filepath, from_censys, args.indent) - - -def convcert(): - """ - Converts a certificate PEM file into the common data model - """ - args = parse_args() - if args.flatten: - d = from_x509_pem_flattened(open(args.filepath).read()) - else: - d = from_x509_pem(open(args.filepath).read()) - print(json.dumps(d, indent=args.indent)) - - -def parse_args(): - parser = ArgumentParser() - parser.add_argument("filepath", type=str, help="JSON file to read from") - parser.add_argument( - "-F", "--flatten", action="store_true", help="Flatten the output" - ) - parser.add_argument( - "-I", "--indent", type=int, default=4, help="Indent of the JSON output" - ) - return parser.parse_args() - - -def convert(filepath: str, f, indent: int = 4) -> None: - with io.open(filepath, "r") as file: - data = json.loads(file.read()) - print(json.dumps(f(data), indent=indent)) diff --git a/common_osint_model/shodan.py b/common_osint_model/shodan.py deleted file mode 100644 index a7df482..0000000 --- a/common_osint_model/shodan.py +++ /dev/null @@ -1,273 +0,0 @@ -import re -import warnings -from base64 import b64decode -from binascii import hexlify -from datetime import datetime -from hashlib import sha256 -from typing import Union - -from DateTime import DateTime - -from common_osint_model.utils import flatten, common_model_cn_extraction, sha256_from_body_string - - -def from_shodan(raw: Union[list, dict]) -> dict: - """ - Turn an expected Shodan dictionary into a generic format - :param raw: Shodan list or dictionary from host queries - :return: Generic host describing dictionary - """ - warnings.warn("This function was deprecated in v0.4.0. and will be removed in v0.5.0", DeprecationWarning) - g = {} - services = [] - if isinstance(raw, dict): - if isinstance(raw.get("data", ""), list): - services.extend(raw["data"]) - else: - services.append(raw) - elif isinstance(raw, list): - if len(raw) == 1: - raw = raw[0] - if isinstance(raw.get("data", ""), list): - services.extend(raw["data"]) - else: - services.append(raw) - else: - services.extend(raw) - else: - raise TypeError("Given parameter 'raw' must be either a list or a dict.") - # Get the meta data - g.update(shodan_meta_extraction(services)) - # Get data for every service - ports = [] - for s in services: - ports.append(s["port"]) - g.update(shodan_service_extraction(s)) - g.update(dict(ports=ports)) - g["domains"].extend(common_model_cn_extraction(g)) - return g - - -def from_shodan_flattened(raw: Union[list, dict]) -> dict: - """ - Turn an expected Shodan dictionary into a generic format - :param raw: Shodan list or dictionary from host queries - :return: Generic host describing dictionary, flattened - """ - warnings.warn("This function was deprecated in v0.4.0. and will be removed in v0.5.0", DeprecationWarning) - return flatten(from_shodan(raw)) - - -def shodan_meta_extraction(raw: Union[dict, list]) -> dict: - """ - Returns a dictionary containing all the meta information from a Shodan host or service - :param raw: Either a service dict or a host list object - :return: Dictionary containing all meta information - """ - g = {} - if isinstance(raw, dict): - o = raw - elif isinstance(raw, list): - o = raw[0] - else: - raise TypeError("Given parameter 'raw' must be either a list or a dict.") - g.update( - { - "as": { - "number": o.get("asn", None), - "name": o.get("isp", None), - "location": o.get("location", {}).get("country_code", None), - "prefix": None, # Todo: Check of routed prefix is given in censys output - }, - "domains": [], - "org": o.get("org", None), - "ip": o.get("ip_str"), - "location": { - "city": o.get("city", None), - "country": o.get("country_name", None), - "country_code": o.get("country_code", None), - "postal_code": o.get("postal_code", None), - "coordinates": { - "latitude": o.get("latitude", None), - "longitude": o.get("longitude", None) - } - } - } - ) - for domain in o.get("domains", []): - g["domains"].append({"type": "shodan-domain", "value": domain}) - for hostname in o.get("hostnames", []): - g["domains"].append({"type": "shodan-hostname", "value": hostname}) - return g - - -def shodan_service_extraction(s: dict) -> dict: - """ - Returns a dictionary containing a properly formatted service based on a shodan raw dictionary - :param s: Shodan service dictionary - :return: Porperly formatted dictionary - """ - g = {} - p = s["port"] - g.update( - { - p: { - "banner": s["data"], - "timestamp": int(DateTime(s["timestamp"])), - "timestamp_readable": DateTime(s["timestamp"]).ISO8601(), - } - } - ) - k = s.keys() - - if "http" in k or s["data"][:4] == "HTTP": - g[p].update(shodan_http_extraction(s)) - if "ssl" in k: - g[p].update(shodan_ssl_extraction(s)) - if "ssh" in k: - g[p].update(shodan_ssh_extraction(s)) - return g - - -def shodan_http_extraction(s: dict) -> dict: - """ - Extracts http(s) specific values from shodan service dict - :param s: Shodan service dictionary - :return: Properly formatted, service related dictionary - """ - http = s.get("http", None) or dict() - headers = {} - for line in s["data"].split("\n"): - line = line.strip() - if not ":" in line: - continue - h = line.split(":") - headers.update({h[0].lower().replace("-", "_"): ":".join(h[1:]).strip()}) - favicon = http.get("favicon", None) or dict() - favicon_data = favicon.get("data", None) - favicon_hash = favicon.get("hash", None) - favicon_sha256 = None - if favicon: - favicon_decoded = b64decode(favicon_data) - h = sha256() - h.update(favicon_decoded) - favicon_sha256 = hexlify(h.digest()).decode("ascii") - return { - "http": { - "headers": headers, - "content": { - "html": http.get("html", None), - "hash": { - "shodan": http.get("html_hash", None), - "sha256": sha256_from_body_string(http.get("html", None) or "") - }, - "favicon": {"shodan": favicon_hash, "sha256": favicon_sha256}, - }, - } - } - - -def shodan_ssl_extraction(s: dict) -> dict: - """ - Extracts ssl/tls specific values from shodan service dict - :param s: Shodan service dictionary - :return: Properly formatted, service related dictionary - """ - ssl = s.get("ssl", None) or dict() - cert = ssl.get("cert", {}) - subject = cert.get("subject", None) or dict() - issuer = cert.get("issuer", None) or dict() - fingerprint = cert.get("fingerprint", None) or dict() - dhparams = ssl.get("dhparams", None) or dict() - common_name = subject.get("CN", None) - common_names = set() - if common_name: - common_names.add(common_name) - for ext in ssl.get("extensions", []): - if ext["name"] == "subjectAltName": - for domain in re.split(r"(\\x[0-9a-z]{2}|\\[a-z^x]{1})", ext["date"]): - if "." in domain: - common_names.add(domain) - cert_issued = int(datetime.strptime(cert.get("issued", None), "%Y%m%d%H%M%SZ").timestamp()) - cert_expires = int(datetime.strptime(cert.get("expires", None), "%Y%m%d%H%M%SZ").timestamp()) - return { - "tls": { - "certificate": { - "subject": { - "country": subject.get("C", None), - "province": subject.get("ST", None), - "locality": subject.get("L", None), - "common_name": list(common_names), - "organization": subject.get("O", None), - "organizational_unit": subject.get("OU", None), - "email_address": subject.get("email", None), - }, - "issuer": { - "country": issuer.get("C", None), - "province": issuer.get("ST", None), - "locality": issuer.get("L", None), - "common_name": issuer.get("CN", None), - "organization": issuer.get("O", None), - "organizational_unit": issuer.get("OU", None), - "email_address": issuer.get("email", None), - }, - "fingerprint": { - "sha1": fingerprint.get("sha1", None), - "sha256": fingerprint.get("sha256", None), - }, - "serial_number": str(int(cert.get("serial", None))), - "validity": { - "start": cert_issued, - "start_readable": DateTime(cert_issued, 'UTC').ISO8601(), - "end": cert_expires, - "end_readable": DateTime(cert_expires, 'UTC').ISO8601(), - "length": cert_expires - cert_issued, - } - }, - "dhparam": { - "bits": dhparams.get("bits", None), - "generator": dhparams.get("generator", None), - }, - "cipher": { - "id": None, - "name": ssl.get("cipher", {}).get("name", None), - "bits": ssl.get("cipher", {}).get("bits", None), - }, - "ja3s": ssl.get("ja3s", None), - "jarm": ssl.get("jarm", None) - } - } - - -def shodan_ssh_extraction(s: dict) -> dict: - """ - Extracts ssh specific values from shodan service dict - :param s: Shodan service dictionary - :return: Properly formatted, service related dictionary - """ - ssh = s.get("ssh", None) or dict() - key_exchange = ssh.get("kex", None) or dict() - h = sha256() - h.update(b64decode(ssh.get("key", None))) - return { - "ssh": { - "version": s["data"].split("Key type")[0].strip(), - "key_exchange": { - "algorithms": { - "compression": key_exchange.get("compression_algorithms", None), - "encryption": key_exchange.get("encryption_algorithms", None), - "key_exchange": key_exchange.get("kex_algorithms", None), - "mac": key_exchange.get("mac_algorithms", None), - "key_algorithms": key_exchange.get( - "server_host_key_algorithms", None - ), - } - }, - "key": { - "hash": { - "sha256": hexlify(h.digest()).decode("ascii") - }, - "type": ssh.get("type", None), - }, - } - } diff --git a/common_osint_model/utils.py b/common_osint_model/utils.py index 17a6367..c2ecf42 100644 --- a/common_osint_model/utils.py +++ b/common_osint_model/utils.py @@ -1,5 +1,4 @@ import hashlib -from binascii import hexlify from typing import Tuple import mmh3 @@ -22,65 +21,6 @@ def flatten(d: dict, parent_key: str = "") -> dict: return dict(items) -def unflatten(flattened: dict) -> dict: - """ - Unflattens a dictionary - :param flattened: Flattened dictionary - :return: Unflattened dictionary - """ - unflattened = {} - for key, value in flattened.items(): - parts = key.split(".") - d = unflattened - for part in parts[:-1]: - if part not in d: - d[part] = dict() - d = d[part] - d[parts[-1]] = value - return unflattened - - -def common_model_cn_extraction(g: dict) -> list: - """ - Loops through all keys in the already converted model in order to find domains - :param g: dictionary of generic model - :return: list of domain objects - """ - domains = [] - for key, value in g.items(): - if not isinstance(value, dict): - continue - - if "tls" in value.keys(): - cns = value["tls"].get("certificate", {}).get("subject", {}).get("common_name", None) or list() - for cn in cns: - # Sloppy check if this is a real domain - if "." in cn: - domains.append({"type": "common_name", "value": cn}) - return domains - - -def sha256_from_body_string(b: str) -> str: - """ - Returns the sha256 hash of an html body given as string - :param b: html body as string - :return: hex digest of sha256 hash - """ - h = hashlib.sha256() - h.update(b.encode("utf8")) - return hexlify(h.digest()).decode("ascii") - - -def list_cleanup(d: dict) -> dict: - for k, v in d.items(): - if isinstance(v, dict): - d[k] = list_cleanup(v) - elif isinstance(v, list): - if len(v) == 1: - d[k] = v[0] - return d - - def hash_all(data: bytes) -> Tuple[str, str, str, str]: """ Helper function to create all hashes for data given. diff --git a/poetry.lock b/poetry.lock index 79e936d..aca5b60 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,20 +1,22 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "certifi" -version = "2023.5.7" +version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." +category = "main" optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, - {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, + {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, + {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, ] [[package]] name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." +category = "main" optional = false python-versions = "*" files = [ @@ -91,6 +93,7 @@ pycparser = "*" name = "charset-normalizer" version = "3.2.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -173,34 +176,35 @@ files = [ [[package]] name = "cryptography" -version = "41.0.2" +version = "41.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-41.0.2-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:01f1d9e537f9a15b037d5d9ee442b8c22e3ae11ce65ea1f3316a41c78756b711"}, - {file = "cryptography-41.0.2-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:079347de771f9282fbfe0e0236c716686950c19dee1b76240ab09ce1624d76d7"}, - {file = "cryptography-41.0.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:439c3cc4c0d42fa999b83ded80a9a1fb54d53c58d6e59234cfe97f241e6c781d"}, - {file = "cryptography-41.0.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f14ad275364c8b4e525d018f6716537ae7b6d369c094805cae45300847e0894f"}, - {file = "cryptography-41.0.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:84609ade00a6ec59a89729e87a503c6e36af98ddcd566d5f3be52e29ba993182"}, - {file = "cryptography-41.0.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:49c3222bb8f8e800aead2e376cbef687bc9e3cb9b58b29a261210456a7783d83"}, - {file = "cryptography-41.0.2-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:d73f419a56d74fef257955f51b18d046f3506270a5fd2ac5febbfa259d6c0fa5"}, - {file = "cryptography-41.0.2-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:2a034bf7d9ca894720f2ec1d8b7b5832d7e363571828037f9e0c4f18c1b58a58"}, - {file = "cryptography-41.0.2-cp37-abi3-win32.whl", hash = "sha256:d124682c7a23c9764e54ca9ab5b308b14b18eba02722b8659fb238546de83a76"}, - {file = "cryptography-41.0.2-cp37-abi3-win_amd64.whl", hash = "sha256:9c3fe6534d59d071ee82081ca3d71eed3210f76ebd0361798c74abc2bcf347d4"}, - {file = "cryptography-41.0.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a719399b99377b218dac6cf547b6ec54e6ef20207b6165126a280b0ce97e0d2a"}, - {file = "cryptography-41.0.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:182be4171f9332b6741ee818ec27daff9fb00349f706629f5cbf417bd50e66fd"}, - {file = "cryptography-41.0.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7a9a3bced53b7f09da251685224d6a260c3cb291768f54954e28f03ef14e3766"}, - {file = "cryptography-41.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f0dc40e6f7aa37af01aba07277d3d64d5a03dc66d682097541ec4da03cc140ee"}, - {file = "cryptography-41.0.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:674b669d5daa64206c38e507808aae49904c988fa0a71c935e7006a3e1e83831"}, - {file = "cryptography-41.0.2-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7af244b012711a26196450d34f483357e42aeddb04128885d95a69bd8b14b69b"}, - {file = "cryptography-41.0.2-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:9b6d717393dbae53d4e52684ef4f022444fc1cce3c48c38cb74fca29e1f08eaa"}, - {file = "cryptography-41.0.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:192255f539d7a89f2102d07d7375b1e0a81f7478925b3bc2e0549ebf739dae0e"}, - {file = "cryptography-41.0.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f772610fe364372de33d76edcd313636a25684edb94cee53fd790195f5989d14"}, - {file = "cryptography-41.0.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b332cba64d99a70c1e0836902720887fb4529ea49ea7f5462cf6640e095e11d2"}, - {file = "cryptography-41.0.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:9a6673c1828db6270b76b22cc696f40cde9043eb90373da5c2f8f2158957f42f"}, - {file = "cryptography-41.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:342f3767e25876751e14f8459ad85e77e660537ca0a066e10e75df9c9e9099f0"}, - {file = "cryptography-41.0.2.tar.gz", hash = "sha256:7d230bf856164de164ecb615ccc14c7fc6de6906ddd5b491f3af90d3514c925c"}, + {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507"}, + {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922"}, + {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81"}, + {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd"}, + {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47"}, + {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116"}, + {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c"}, + {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae"}, + {file = "cryptography-41.0.3-cp37-abi3-win32.whl", hash = "sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306"}, + {file = "cryptography-41.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574"}, + {file = "cryptography-41.0.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087"}, + {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858"}, + {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906"}, + {file = "cryptography-41.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e"}, + {file = "cryptography-41.0.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd"}, + {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207"}, + {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84"}, + {file = "cryptography-41.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7"}, + {file = "cryptography-41.0.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d"}, + {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de"}, + {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1"}, + {file = "cryptography-41.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4"}, + {file = "cryptography-41.0.3.tar.gz", hash = "sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34"}, ] [package.dependencies] @@ -220,6 +224,7 @@ test-randomorder = ["pytest-randomly"] name = "datetime" version = "4.9" description = "This package provides a DateTime data type, as known from Zope. Unless you need to communicate with Zope APIs, you're probably better off using Python's built-in datetime module." +category = "main" optional = false python-versions = "*" files = [ @@ -235,6 +240,7 @@ pytz = "*" name = "hhhash" version = "0.3" description = "HHHash library is calculate HHHash from HTTP servers." +category = "main" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -249,6 +255,7 @@ requests = ">=2.20.0,<3.0.0" name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -260,6 +267,7 @@ files = [ name = "mmh3" version = "3.1.0" description = "Python wrapper for MurmurHash (MurmurHash3), a set of fast and robust hash functions." +category = "main" optional = false python-versions = "*" files = [ @@ -304,6 +312,7 @@ files = [ name = "pycparser" version = "2.21" description = "C parser in Python" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -313,47 +322,48 @@ files = [ [[package]] name = "pydantic" -version = "1.10.11" +version = "1.10.12" description = "Data validation and settings management using python type hints" +category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ff44c5e89315b15ff1f7fdaf9853770b810936d6b01a7bcecaa227d2f8fe444f"}, - {file = "pydantic-1.10.11-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a6c098d4ab5e2d5b3984d3cb2527e2d6099d3de85630c8934efcfdc348a9760e"}, - {file = "pydantic-1.10.11-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16928fdc9cb273c6af00d9d5045434c39afba5f42325fb990add2c241402d151"}, - {file = "pydantic-1.10.11-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0588788a9a85f3e5e9ebca14211a496409cb3deca5b6971ff37c556d581854e7"}, - {file = "pydantic-1.10.11-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e9baf78b31da2dc3d3f346ef18e58ec5f12f5aaa17ac517e2ffd026a92a87588"}, - {file = "pydantic-1.10.11-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:373c0840f5c2b5b1ccadd9286782852b901055998136287828731868027a724f"}, - {file = "pydantic-1.10.11-cp310-cp310-win_amd64.whl", hash = "sha256:c3339a46bbe6013ef7bdd2844679bfe500347ac5742cd4019a88312aa58a9847"}, - {file = "pydantic-1.10.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:08a6c32e1c3809fbc49debb96bf833164f3438b3696abf0fbeceb417d123e6eb"}, - {file = "pydantic-1.10.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a451ccab49971af043ec4e0d207cbc8cbe53dbf148ef9f19599024076fe9c25b"}, - {file = "pydantic-1.10.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b02d24f7b2b365fed586ed73582c20f353a4c50e4be9ba2c57ab96f8091ddae"}, - {file = "pydantic-1.10.11-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f34739a89260dfa420aa3cbd069fbcc794b25bbe5c0a214f8fb29e363484b66"}, - {file = "pydantic-1.10.11-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e297897eb4bebde985f72a46a7552a7556a3dd11e7f76acda0c1093e3dbcf216"}, - {file = "pydantic-1.10.11-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d185819a7a059550ecb85d5134e7d40f2565f3dd94cfd870132c5f91a89cf58c"}, - {file = "pydantic-1.10.11-cp311-cp311-win_amd64.whl", hash = "sha256:4400015f15c9b464c9db2d5d951b6a780102cfa5870f2c036d37c23b56f7fc1b"}, - {file = "pydantic-1.10.11-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2417de68290434461a266271fc57274a138510dca19982336639484c73a07af6"}, - {file = "pydantic-1.10.11-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:331c031ba1554b974c98679bd0780d89670d6fd6f53f5d70b10bdc9addee1713"}, - {file = "pydantic-1.10.11-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8268a735a14c308923e8958363e3a3404f6834bb98c11f5ab43251a4e410170c"}, - {file = "pydantic-1.10.11-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:44e51ba599c3ef227e168424e220cd3e544288c57829520dc90ea9cb190c3248"}, - {file = "pydantic-1.10.11-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d7781f1d13b19700b7949c5a639c764a077cbbdd4322ed505b449d3ca8edcb36"}, - {file = "pydantic-1.10.11-cp37-cp37m-win_amd64.whl", hash = "sha256:7522a7666157aa22b812ce14c827574ddccc94f361237ca6ea8bb0d5c38f1629"}, - {file = "pydantic-1.10.11-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc64eab9b19cd794a380179ac0e6752335e9555d214cfcb755820333c0784cb3"}, - {file = "pydantic-1.10.11-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8dc77064471780262b6a68fe67e013298d130414d5aaf9b562c33987dbd2cf4f"}, - {file = "pydantic-1.10.11-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe429898f2c9dd209bd0632a606bddc06f8bce081bbd03d1c775a45886e2c1cb"}, - {file = "pydantic-1.10.11-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:192c608ad002a748e4a0bed2ddbcd98f9b56df50a7c24d9a931a8c5dd053bd3d"}, - {file = "pydantic-1.10.11-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ef55392ec4bb5721f4ded1096241e4b7151ba6d50a50a80a2526c854f42e6a2f"}, - {file = "pydantic-1.10.11-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:41e0bb6efe86281623abbeeb0be64eab740c865388ee934cd3e6a358784aca6e"}, - {file = "pydantic-1.10.11-cp38-cp38-win_amd64.whl", hash = "sha256:265a60da42f9f27e0b1014eab8acd3e53bd0bad5c5b4884e98a55f8f596b2c19"}, - {file = "pydantic-1.10.11-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:469adf96c8e2c2bbfa655fc7735a2a82f4c543d9fee97bd113a7fb509bf5e622"}, - {file = "pydantic-1.10.11-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e6cbfbd010b14c8a905a7b10f9fe090068d1744d46f9e0c021db28daeb8b6de1"}, - {file = "pydantic-1.10.11-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abade85268cc92dff86d6effcd917893130f0ff516f3d637f50dadc22ae93999"}, - {file = "pydantic-1.10.11-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e9738b0f2e6c70f44ee0de53f2089d6002b10c33264abee07bdb5c7f03038303"}, - {file = "pydantic-1.10.11-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:787cf23e5a0cde753f2eabac1b2e73ae3844eb873fd1f5bdbff3048d8dbb7604"}, - {file = "pydantic-1.10.11-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:174899023337b9fc685ac8adaa7b047050616136ccd30e9070627c1aaab53a13"}, - {file = "pydantic-1.10.11-cp39-cp39-win_amd64.whl", hash = "sha256:1954f8778489a04b245a1e7b8b22a9d3ea8ef49337285693cf6959e4b757535e"}, - {file = "pydantic-1.10.11-py3-none-any.whl", hash = "sha256:008c5e266c8aada206d0627a011504e14268a62091450210eda7c07fabe6963e"}, - {file = "pydantic-1.10.11.tar.gz", hash = "sha256:f66d479cf7eb331372c470614be6511eae96f1f120344c25f3f9bb59fb1b5528"}, + {file = "pydantic-1.10.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a1fcb59f2f355ec350073af41d927bf83a63b50e640f4dbaa01053a28b7a7718"}, + {file = "pydantic-1.10.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b7ccf02d7eb340b216ec33e53a3a629856afe1c6e0ef91d84a4e6f2fb2ca70fe"}, + {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fb2aa3ab3728d950bcc885a2e9eff6c8fc40bc0b7bb434e555c215491bcf48b"}, + {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:771735dc43cf8383959dc9b90aa281f0b6092321ca98677c5fb6125a6f56d58d"}, + {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ca48477862372ac3770969b9d75f1bf66131d386dba79506c46d75e6b48c1e09"}, + {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a5e7add47a5b5a40c49b3036d464e3c7802f8ae0d1e66035ea16aa5b7a3923ed"}, + {file = "pydantic-1.10.12-cp310-cp310-win_amd64.whl", hash = "sha256:e4129b528c6baa99a429f97ce733fff478ec955513630e61b49804b6cf9b224a"}, + {file = "pydantic-1.10.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b0d191db0f92dfcb1dec210ca244fdae5cbe918c6050b342d619c09d31eea0cc"}, + {file = "pydantic-1.10.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:795e34e6cc065f8f498c89b894a3c6da294a936ee71e644e4bd44de048af1405"}, + {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69328e15cfda2c392da4e713443c7dbffa1505bc9d566e71e55abe14c97ddc62"}, + {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2031de0967c279df0d8a1c72b4ffc411ecd06bac607a212892757db7462fc494"}, + {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ba5b2e6fe6ca2b7e013398bc7d7b170e21cce322d266ffcd57cca313e54fb246"}, + {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2a7bac939fa326db1ab741c9d7f44c565a1d1e80908b3797f7f81a4f86bc8d33"}, + {file = "pydantic-1.10.12-cp311-cp311-win_amd64.whl", hash = "sha256:87afda5539d5140cb8ba9e8b8c8865cb5b1463924d38490d73d3ccfd80896b3f"}, + {file = "pydantic-1.10.12-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:549a8e3d81df0a85226963611950b12d2d334f214436a19537b2efed61b7639a"}, + {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598da88dfa127b666852bef6d0d796573a8cf5009ffd62104094a4fe39599565"}, + {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba5c4a8552bff16c61882db58544116d021d0b31ee7c66958d14cf386a5b5350"}, + {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c79e6a11a07da7374f46970410b41d5e266f7f38f6a17a9c4823db80dadf4303"}, + {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab26038b8375581dc832a63c948f261ae0aa21f1d34c1293469f135fa92972a5"}, + {file = "pydantic-1.10.12-cp37-cp37m-win_amd64.whl", hash = "sha256:e0a16d274b588767602b7646fa05af2782576a6cf1022f4ba74cbb4db66f6ca8"}, + {file = "pydantic-1.10.12-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a9dfa722316f4acf4460afdf5d41d5246a80e249c7ff475c43a3a1e9d75cf62"}, + {file = "pydantic-1.10.12-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a73f489aebd0c2121ed974054cb2759af8a9f747de120acd2c3394cf84176ccb"}, + {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b30bcb8cbfccfcf02acb8f1a261143fab622831d9c0989707e0e659f77a18e0"}, + {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fcfb5296d7877af406ba1547dfde9943b1256d8928732267e2653c26938cd9c"}, + {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:2f9a6fab5f82ada41d56b0602606a5506aab165ca54e52bc4545028382ef1c5d"}, + {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dea7adcc33d5d105896401a1f37d56b47d443a2b2605ff8a969a0ed5543f7e33"}, + {file = "pydantic-1.10.12-cp38-cp38-win_amd64.whl", hash = "sha256:1eb2085c13bce1612da8537b2d90f549c8cbb05c67e8f22854e201bde5d98a47"}, + {file = "pydantic-1.10.12-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ef6c96b2baa2100ec91a4b428f80d8f28a3c9e53568219b6c298c1125572ebc6"}, + {file = "pydantic-1.10.12-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c076be61cd0177a8433c0adcb03475baf4ee91edf5a4e550161ad57fc90f523"}, + {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d5a58feb9a39f481eda4d5ca220aa8b9d4f21a41274760b9bc66bfd72595b86"}, + {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5f805d2d5d0a41633651a73fa4ecdd0b3d7a49de4ec3fadf062fe16501ddbf1"}, + {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1289c180abd4bd4555bb927c42ee42abc3aee02b0fb2d1223fb7c6e5bef87dbe"}, + {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5d1197e462e0364906cbc19681605cb7c036f2475c899b6f296104ad42b9f5fb"}, + {file = "pydantic-1.10.12-cp39-cp39-win_amd64.whl", hash = "sha256:fdbdd1d630195689f325c9ef1a12900524dceb503b00a987663ff4f58669b93d"}, + {file = "pydantic-1.10.12-py3-none-any.whl", hash = "sha256:b749a43aa51e32839c9d71dc67eb1e4221bb04af1033a32e3923d46f9effa942"}, + {file = "pydantic-1.10.12.tar.gz", hash = "sha256:0fe8a415cea8f340e7a9af9c54fc71a649b43e8ca3cc732986116b3cb135d303"}, ] [package.dependencies] @@ -367,6 +377,7 @@ email = ["email-validator (>=1.0.3)"] name = "pytz" version = "2023.3" description = "World timezone definitions, modern and historical" +category = "main" optional = false python-versions = "*" files = [ @@ -378,6 +389,7 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -399,6 +411,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "setuptools" version = "68.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -415,6 +428,7 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "typing-extensions" version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -424,13 +438,14 @@ files = [ [[package]] name = "urllib3" -version = "2.0.3" +version = "2.0.4" description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "urllib3-2.0.3-py3-none-any.whl", hash = "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1"}, - {file = "urllib3-2.0.3.tar.gz", hash = "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825"}, + {file = "urllib3-2.0.4-py3-none-any.whl", hash = "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"}, + {file = "urllib3-2.0.4.tar.gz", hash = "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11"}, ] [package.extras] @@ -443,6 +458,7 @@ zstd = ["zstandard (>=0.18.0)"] name = "zope-interface" version = "6.0" description = "Interfaces for Python" +category = "main" optional = false python-versions = ">=3.7" files = [ diff --git a/pyproject.toml b/pyproject.toml index 3607e6e..8eec9e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "common-osint-model" -version = "0.4.15" +version = "0.5.0" description = "Converting data from services like BinaryEdge, Censys and Shodan to a common data model." authors = ["3c7 <3c7@posteo.de>"] license = "MIT" From 856fce085b49405161d10d535e7754c3dfddd4ec Mon Sep 17 00:00:00 2001 From: 3c7 <3c7@posteo.de> Date: Tue, 15 Aug 2023 10:34:51 +0200 Subject: [PATCH 2/2] Beta instead of full release --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8eec9e6..cc31f9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "common-osint-model" -version = "0.5.0" +version = "0.5.0-beta1" description = "Converting data from services like BinaryEdge, Censys and Shodan to a common data model." authors = ["3c7 <3c7@posteo.de>"] license = "MIT"