From 9649fd4c87494cbbed492beed6b5528e66dbb1c9 Mon Sep 17 00:00:00 2001 From: "Gergely Daroczi (@daroczig)" Date: Fri, 29 Nov 2024 22:55:36 +0100 Subject: [PATCH 01/13] welcome upcloud regions --- pyproject.toml | 2 + src/sc_crawler/vendors/__init__.py | 2 +- src/sc_crawler/vendors/upcloud.py | 349 +++++++++++++++++++++++++++++ src/sc_crawler/vendors/vendors.py | 15 ++ 4 files changed, 367 insertions(+), 1 deletion(-) create mode 100644 src/sc_crawler/vendors/upcloud.py diff --git a/pyproject.toml b/pyproject.toml index 1f5ff4a1..9ee31db3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,11 +56,13 @@ aws = ["boto3"] hcloud = ["hcloud"] gcp = ["google-cloud", "google-cloud-compute", "google-cloud-billing"] azure = ["azure-identity", "azure-mgmt-resource", "azure-mgmt-compute"] +upcloud = ["upcloud-api"] vendors = [ "sparecores-crawler[aws]", "sparecores-crawler[hcloud]", "sparecores-crawler[gcp]", "sparecores-crawler[azure]", + "sparecores-crawler[upcloud]", ] all = ["sparecores-crawler[testing]", "sparecores-crawler[vendors]"] diff --git a/src/sc_crawler/vendors/__init__.py b/src/sc_crawler/vendors/__init__.py index 7d883310..c5e83d59 100644 --- a/src/sc_crawler/vendors/__init__.py +++ b/src/sc_crawler/vendors/__init__.py @@ -1,3 +1,3 @@ """Helper methods for each cloud compute resource provider.""" -from .vendors import aws, azure, gcp, hcloud # noqa: F401 +from .vendors import aws, azure, gcp, hcloud, upcloud # noqa: F401 diff --git a/src/sc_crawler/vendors/upcloud.py b/src/sc_crawler/vendors/upcloud.py new file mode 100644 index 00000000..9ab080a4 --- /dev/null +++ b/src/sc_crawler/vendors/upcloud.py @@ -0,0 +1,349 @@ +import os +from functools import cache + +from upcloud_api import CloudManager + +from ..lookup import map_compliance_frameworks_to_vendor + + +@cache +def _client() -> CloudManager: + """Authorized Hetzner Cloud client using the HCLOUD_TOKEN env var.""" + try: + username = os.environ["UPCLOUD_USERNAME"] + except KeyError: + raise KeyError("Missing environment variable: UPCLOUD_USERNAME") + try: + password = os.environ["UPCLOUD_PASSWORD"] + except KeyError: + raise KeyError("Missing environment variable: UPCLOUD_PASSWORD") + manager = CloudManager(username, password) + manager.authenticate() + return manager + + +# _client().get_prices() +# servers = _client().get_server_sizes() +# servers[1] + +# templates = _client().get_templates() +# templates[1] + +# _client().get_server_plans() + +# prices = _client().get_prices() +# prices["prices"]["zone"][1]["server_plan_HIMEM-24xCPU-512GB"] # EUR cent! + + +def inventory_compliance_frameworks(vendor): + """Manual list of known compliance frameworks at UpCloud. + + Data collected from their Security and Standards docs at + .""" + return map_compliance_frameworks_to_vendor( + vendor.vendor_id, + ["iso27001"], + ) + + +def inventory_regions(vendor): + """List all regions via API call. + + Data manually enriched from https://upcloud.com/data-centres.""" + manual_data = { + "au-syd1": { + "country_id": "AU", + "state": "New South Wales", + "city": "Sydney", + "founding_year": 2021, + "green_energy": False, + "lon": 151.189377, + "lat": -33.918251, + }, + "de-fra1": { + "country_id": "DE", + "state": "Hesse", + "city": "Frankfurt", + "founding_year": 2015, + "green_energy": True, + "lon": 8.735120, + "lat": 50.119190, + }, + "fi-hel1": { + "country_id": "FI", + "state": "Uusimaa", + "city": "Helsinki", + "founding_year": 2011, + "green_energy": True, + "lon": 24.778570, + "lat": 60.20323, + }, + "fi-hel2": { + "country_id": "FI", + "state": "Uusimaa", + "city": "Helsinki", + "founding_year": 2018, + "green_energy": True, + "lon": 24.876350, + "lat": 60.216209, + }, + "es-mad1": { + "country_id": "ES", + "state": "Madrid", + "city": "Madrid", + "founding_year": 2020, + "green_energy": True, + "lon": -3.6239873, + "lat": 40.4395019, + }, + "nl-ams1": { + "country_id": "NL", + "state": "Noord Holland", + "city": "Amsterdam", + "founding_year": 2017, + "green_energy": True, + "lon": 4.8400019, + "lat": 52.3998291, + }, + "pl-waw1": { + "country_id": "PL", + "state": "Mazowieckie", + "city": "Warsaw", + "founding_year": 2020, + "green_energy": True, + "lon": 20.9192823, + "lat": 52.1905901, + }, + "se-sto1": { + "country_id": "SE", + "state": "Stockholm", + "city": "Stockholm", + "founding_year": 2015, + "green_energy": True, + "lon": 18.102788, + "lat": 59.2636708, + }, + "sg-sin1": { + "country_id": "SG", + "state": "Singapore", + "city": "Singapore", + "founding_year": 2017, + "green_energy": True, + "lon": 103.7022636, + "lat": 1.3172304, + }, + "uk-lon1": { + "country_id": "GB", + "state": "London", + "city": "London", + "founding_year": 2012, + "green_energy": True, + # approximate .. probably business address + "lon": -0.1037341, + "lat": 51.5232232, + }, + "us-chi1": { + "country_id": "US", + "state": "Illinois", + "city": "Chicago", + "founding_year": 2014, + "green_energy": False, + "lon": -87.6342056, + "lat": 41.8761287, + }, + "us-nyc1": { + "country_id": "US", + "state": "New York", + "city": "New York", + "founding_year": 2020, + "green_energy": False, + "lon": -74.0645536, + "lat": 40.7834325, + }, + "us-sjo1": { + "country_id": "US", + "state": "California", + "city": "San Jose", + "founding_year": 2018, + "green_energy": False, + "lon": -121.9754458, + "lat": 37.3764769, + }, + } + items = [] + regions = _client().get_zones()["zones"]["zone"] + for region in regions: + if region["public"] == "yes": + if region["id"] not in manual_data: + raise ValueError(f"Missing manual data for {region['id']}") + region_data = manual_data[region["id"]] + items.append( + { + "vendor_id": vendor.vendor_id, + "region_id": region["id"], + "name": region["description"], + "api_reference": region["id"], + "display_name": ( + region["description"] + f" ({region_data['country_id']})" + ), + "aliases": [], + "country_id": region_data["country_id"], + "state": region_data["state"], + "city": region_data["city"], + "address_line": None, + "zip_code": None, + "lon": region_data["lon"], + "lat": region_data["lat"], + "founding_year": region_data["founding_year"], + "green_energy": region_data["green_energy"], + } + ) + return items + + +def inventory_zones(vendor): + items = [] + # for zone in []: + # items.append({ + # "vendor_id": vendor.vendor_id, + # "region_id": "", + # "zone_id": "", + # "name": "", + # }) + return items + + +def inventory_servers(vendor): + items = [] + # for server in []: + # items.append( + # { + # "vendor_id": vendor.vendor_id, + # "server_id": , + # "name": , + # "description": None, + # "vcpus": , + # "hypervisor": None, + # "cpu_allocation": CpuAllocation...., + # "cpu_cores": None, + # "cpu_speed": None, + # "cpu_architecture": CpuArchitecture...., + # "cpu_manufacturer": None, + # "cpu_family": None, + # "cpu_model": None, + # "cpu_l1_cache: None, + # "cpu_l2_cache: None, + # "cpu_l3_cache: None, + # "cpu_flags: [], + # "cpus": [], + # "memory_amount": , + # "memory_generation": None, + # "memory_speed": None, + # "memory_ecc": None, + # "gpu_count": 0, + # "gpu_memory_min": None, + # "gpu_memory_total": None, + # "gpu_manufacturer": None, + # "gpu_family": None, + # "gpu_model": None, + # "gpus": [], + # "storage_size": 0, + # "storage_type": None, + # "storages": [], + # "network_speed": None, + # "inbound_traffic": 0, + # "outbound_traffic": 0, + # "ipv4": 0, + # } + # ) + return items + + +def inventory_server_prices(vendor): + items = [] + # for server in []: + # items.append({ + # "vendor_id": , + # "region_id": , + # "zone_id": , + # "server_id": , + # "operating_system": , + # "allocation": Allocation...., + # "unit": PriceUnit.HOUR, + # "price": , + # "price_upfront": 0, + # "price_tiered": [], + # "currency": "USD", + # }) + return items + + +def inventory_server_prices_spot(vendor): + return [] + + +def inventory_storages(vendor): + items = [] + # for storage in []: + # items.append( + # { + # "storage_id": , + # "vendor_id": vendor.vendor_id, + # "name": , + # "description": None, + # "storage_type": StorageType...., + # "max_iops": None, + # "max_throughput": None, + # "min_size": None, + # "max_size": None, + # } + # ) + return items + + +def inventory_storage_prices(vendor): + items = [] + # for price in []: + # items.append( + # { + # "vendor_id": vendor.vendor_id, + # "region_id": , + # "storage_id": , + # "unit": PriceUnit.GB_MONTH, + # "price": , + # "currency": "USD", + # } + # ) + return items + + +def inventory_traffic_prices(vendor): + items = [] + # for price in []: + # items.append( + # { + # "vendor_id": vendor.vendor_id, + # "region_id": , + # "price": , + # "price_tiered": [], + # "currency": "USD", + # "unit": PriceUnit.GB_MONTH, + # "direction": TrafficDirection...., + # } + # ) + return items + + +def inventory_ipv4_prices(vendor): + items = [] + # for price in []: + # items.append( + # { + # "vendor_id": vendor.vendor_id, + # "region_id": , + # "price": , + # "currency": "USD", + # "unit": PriceUnit.HOUR, + # } + # ) + return items diff --git a/src/sc_crawler/vendors/vendors.py b/src/sc_crawler/vendors/vendors.py index 32be30e6..79eb7a18 100644 --- a/src/sc_crawler/vendors/vendors.py +++ b/src/sc_crawler/vendors/vendors.py @@ -66,3 +66,18 @@ status_page="https://azure.status.microsoft.com", ) """Microsoft Azure.""" + +upcloud = Vendor( + vendor_id="upcloud", + name="UpCloud", + logo="https://sparecores.com/assets/images/vendors/upcloud.svg", + homepage="https://upcloud.com", + country=countries["FI"], + state="Uusimaa", + city="Helsinki", + address_line="Aleksanterinkatu 15 B, 7th floor", + zip_code="00100", + founding_year=2012, + status_page="https://status.upcloud.com", +) +"""UpCloud.""" From 3ede9b1e380169d1bc41b073abe005f339464138 Mon Sep 17 00:00:00 2001 From: "Gergely Daroczi (@daroczig)" Date: Fri, 29 Nov 2024 22:55:57 +0100 Subject: [PATCH 02/13] note the need for api reference and display name --- docs/add_vendor.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/add_vendor.md b/docs/add_vendor.md index b12b516b..536c1251 100644 --- a/docs/add_vendor.md +++ b/docs/add_vendor.md @@ -72,6 +72,8 @@ def inventory_regions(vendor): # "vendor_id": vendor.vendor_id, # "region_id": "", # "name": "", + # "api_reference": "", + # "display_name": "", # "aliases": [], # "country_id": "", # "state": None, @@ -95,6 +97,8 @@ def inventory_zones(vendor): # "region_id": "", # "zone_id": "", # "name": "", + # "api_reference": "", + # "display_name": "", # }) return items @@ -107,6 +111,8 @@ def inventory_servers(vendor): # "vendor_id": vendor.vendor_id, # "server_id": , # "name": , + # "api_reference": , + # "display_name": , # "description": None, # "vcpus": , # "hypervisor": None, From e14b8ba2e7dec38745a0765e1e7e0062748b7ae3 Mon Sep 17 00:00:00 2001 From: "Gergely Daroczi (@daroczig)" Date: Fri, 29 Nov 2024 23:17:08 +0100 Subject: [PATCH 03/13] dummy zones --- src/sc_crawler/vendors/upcloud.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/sc_crawler/vendors/upcloud.py b/src/sc_crawler/vendors/upcloud.py index 9ab080a4..8400c885 100644 --- a/src/sc_crawler/vendors/upcloud.py +++ b/src/sc_crawler/vendors/upcloud.py @@ -202,14 +202,24 @@ def inventory_regions(vendor): def inventory_zones(vendor): + """List all regions as availability zones. + + There is no concept of having multiple availability zones withing + a region (virtual datacenter) at UpCloud, so creating 1-1 + dummy Zones reusing the Region id and name. + """ items = [] - # for zone in []: - # items.append({ - # "vendor_id": vendor.vendor_id, - # "region_id": "", - # "zone_id": "", - # "name": "", - # }) + for region in vendor.regions: + items.append( + { + "vendor_id": vendor.vendor_id, + "region_id": region.region_id, + "zone_id": region.region_id, + "name": region.name, + "api_reference": region.region_id, + "display_name": region.name, + } + ) return items From 3c4eae63ef40d84a3257f1ff13df25d9468b2784 Mon Sep 17 00:00:00 2001 From: "Gergely Daroczi (@daroczig)" Date: Sat, 30 Nov 2024 22:39:18 +0100 Subject: [PATCH 04/13] missed closing quotes --- docs/add_vendor.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/add_vendor.md b/docs/add_vendor.md index 536c1251..54f82b8b 100644 --- a/docs/add_vendor.md +++ b/docs/add_vendor.md @@ -123,10 +123,10 @@ def inventory_servers(vendor): # "cpu_manufacturer": None, # "cpu_family": None, # "cpu_model": None, - # "cpu_l1_cache: None, - # "cpu_l2_cache: None, - # "cpu_l3_cache: None, - # "cpu_flags: [], + # "cpu_l1_cache": None, + # "cpu_l2_cache": None, + # "cpu_l3_cache": None, + # "cpu_flags": [], # "cpus": [], # "memory_amount": , # "memory_generation": None, From b858e0aaa4d70b59b46f24b4c2c50dbe0e363018 Mon Sep 17 00:00:00 2001 From: "Gergely Daroczi (@daroczig)" Date: Sun, 1 Dec 2024 00:00:44 +0100 Subject: [PATCH 05/13] note missed serer family field --- docs/add_vendor.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/add_vendor.md b/docs/add_vendor.md index 54f82b8b..0b97160d 100644 --- a/docs/add_vendor.md +++ b/docs/add_vendor.md @@ -114,6 +114,7 @@ def inventory_servers(vendor): # "api_reference": , # "display_name": , # "description": None, + # "family": None, # "vcpus": , # "hypervisor": None, # "cpu_allocation": CpuAllocation...., From c62a33d3fa33c2db06f329a0aaf0b2efa7c67d12 Mon Sep 17 00:00:00 2001 From: "Gergely Daroczi (@daroczig)" Date: Sun, 1 Dec 2024 00:01:39 +0100 Subject: [PATCH 06/13] ingest servers --- src/sc_crawler/vendors/upcloud.py | 134 ++++++++++++++++++++---------- 1 file changed, 88 insertions(+), 46 deletions(-) diff --git a/src/sc_crawler/vendors/upcloud.py b/src/sc_crawler/vendors/upcloud.py index 8400c885..1b613de3 100644 --- a/src/sc_crawler/vendors/upcloud.py +++ b/src/sc_crawler/vendors/upcloud.py @@ -1,20 +1,25 @@ -import os from functools import cache +from os import environ +from re import compile as recompile from upcloud_api import CloudManager from ..lookup import map_compliance_frameworks_to_vendor +from ..table_fields import CpuAllocation, CpuArchitecture, StorageType + +# ############################################################################## +# Cached client wrappers @cache def _client() -> CloudManager: """Authorized Hetzner Cloud client using the HCLOUD_TOKEN env var.""" try: - username = os.environ["UPCLOUD_USERNAME"] + username = environ["UPCLOUD_USERNAME"] except KeyError: raise KeyError("Missing environment variable: UPCLOUD_USERNAME") try: - password = os.environ["UPCLOUD_PASSWORD"] + password = environ["UPCLOUD_PASSWORD"] except KeyError: raise KeyError("Missing environment variable: UPCLOUD_PASSWORD") manager = CloudManager(username, password) @@ -22,6 +27,33 @@ def _client() -> CloudManager: return manager +# ############################################################################## +# Internal helpers + + +def _parse_server_name(name): + """Extract server family and description from the server id.""" + name_pattern = recompile( + # optional leading ALLCAPS chars are the family name + r"((?P[A-Z]*)-)?" r"(?P[0-9]+)xCPU-" r"(?P[0-9]+)GB" + ) + name_match = name_pattern.match(name) + if not name_match: + raise ValueError(f"Server name '{name}' does not match the expected format.") + data = name_match.groupdict() + family_mapping = { + None: "General Purpose", + "DEV": "Developer", + "HICPU": "High CPU", + "HIMEM": "High Memory", + } + data["family"] = family_mapping.get(data["family"], data["family"]) + data["description"] = ( + f"{data['family']} {data['vcpus']} vCPUs, {data['memory']} GB RAM" + ) + return data + + # _client().get_prices() # servers = _client().get_server_sizes() # servers[1] @@ -29,11 +61,12 @@ def _client() -> CloudManager: # templates = _client().get_templates() # templates[1] -# _client().get_server_plans() - # prices = _client().get_prices() # prices["prices"]["zone"][1]["server_plan_HIMEM-24xCPU-512GB"] # EUR cent! +# ############################################################################## +# Public methods to fetch data + def inventory_compliance_frameworks(vendor): """Manual list of known compliance frameworks at UpCloud. @@ -224,48 +257,57 @@ def inventory_zones(vendor): def inventory_servers(vendor): + servers = _client().get_server_plans()["plans"]["plan"] items = [] - # for server in []: - # items.append( - # { - # "vendor_id": vendor.vendor_id, - # "server_id": , - # "name": , - # "description": None, - # "vcpus": , - # "hypervisor": None, - # "cpu_allocation": CpuAllocation...., - # "cpu_cores": None, - # "cpu_speed": None, - # "cpu_architecture": CpuArchitecture...., - # "cpu_manufacturer": None, - # "cpu_family": None, - # "cpu_model": None, - # "cpu_l1_cache: None, - # "cpu_l2_cache: None, - # "cpu_l3_cache: None, - # "cpu_flags: [], - # "cpus": [], - # "memory_amount": , - # "memory_generation": None, - # "memory_speed": None, - # "memory_ecc": None, - # "gpu_count": 0, - # "gpu_memory_min": None, - # "gpu_memory_total": None, - # "gpu_manufacturer": None, - # "gpu_family": None, - # "gpu_model": None, - # "gpus": [], - # "storage_size": 0, - # "storage_type": None, - # "storages": [], - # "network_speed": None, - # "inbound_traffic": 0, - # "outbound_traffic": 0, - # "ipv4": 0, - # } - # ) + for server in servers: + server_data = _parse_server_name(server["name"]) + items.append( + { + "vendor_id": vendor.vendor_id, + "server_id": server["name"], + "name": server["name"], + "api_reference": server["name"], + "display_name": server["name"], + "description": server_data["description"], + "family": server_data["family"], + "vcpus": server["core_number"], + # https://upcloud.com/docs/products/cloud-servers/features/cloud-server-system/#virtualisation + "hypervisor": "KVM", + # all servers comes with dedicated vCPU + "cpu_allocation": CpuAllocation.DEDICATED, + "cpu_cores": None, + "cpu_speed": None, + # no known ARM options + "cpu_architecture": CpuArchitecture.X86_64, + "cpu_manufacturer": None, + "cpu_family": None, + "cpu_model": None, + "cpu_l1_cache": None, + "cpu_l2_cache": None, + "cpu_l3_cache": None, + "cpu_flags": [], + "cpus": [], + "memory_amount": server["memory_amount"], + "memory_generation": None, + "memory_speed": None, + "memory_ecc": None, + # no GPU options + "gpu_count": 0, + "gpu_memory_min": None, + "gpu_memory_total": None, + "gpu_manufacturer": None, + "gpu_family": None, + "gpu_model": None, + "gpus": [], + "storage_size": server["storage_size"], + "storage_type": (StorageType.SSD if server["storage_tier"] else None), + "storages": [], + "network_speed": None, + "inbound_traffic": 0, + "outbound_traffic": server["public_traffic_out"], + "ipv4": 1, + } + ) return items From 7f88596b50630ac6c90a24ef8c3cb7a5fcfc27f9 Mon Sep 17 00:00:00 2001 From: "Gergely Daroczi (@daroczig)" Date: Thu, 5 Dec 2024 13:48:19 +0100 Subject: [PATCH 07/13] fix DEV-422 exclude servers with questionable availability --- src/sc_crawler/vendors/azure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sc_crawler/vendors/azure.py b/src/sc_crawler/vendors/azure.py index a3b38b3d..dca4fdd8 100644 --- a/src/sc_crawler/vendors/azure.py +++ b/src/sc_crawler/vendors/azure.py @@ -918,7 +918,7 @@ def inventory_servers(vendor): servers.pop(i) # servers randomly switching between active/inactive status # TODO review from time to time - if name == "Standard_M896ixds_32_v3": + if name in ["Standard_M896ixds_32_v3", "Standard_M64-32bds_1_v3"]: vendor.log(f"Excluding server with questionable availability: {name}") servers.pop(i) servers = preprocess_servers(servers, vendor, _standardize_server) From a28fcd3b2264518fd671d947ebaead4029b10bf7 Mon Sep 17 00:00:00 2001 From: "Gergely Daroczi (@daroczig)" Date: Thu, 5 Dec 2024 16:27:18 +0100 Subject: [PATCH 08/13] ingest server prices --- src/sc_crawler/vendors/upcloud.py | 36 +++++++++++++++++++------------ 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/sc_crawler/vendors/upcloud.py b/src/sc_crawler/vendors/upcloud.py index 1b613de3..10adb36e 100644 --- a/src/sc_crawler/vendors/upcloud.py +++ b/src/sc_crawler/vendors/upcloud.py @@ -313,20 +313,28 @@ def inventory_servers(vendor): def inventory_server_prices(vendor): items = [] - # for server in []: - # items.append({ - # "vendor_id": , - # "region_id": , - # "zone_id": , - # "server_id": , - # "operating_system": , - # "allocation": Allocation...., - # "unit": PriceUnit.HOUR, - # "price": , - # "price_upfront": 0, - # "price_tiered": [], - # "currency": "USD", - # }) + prices = _client().get_prices() + for zone_prices in prices["prices"]["zone"]: + zone_id = zone_prices["name"] + for k, v in zone_prices.items(): + if not k.startswith("server_plan"): + continue + server_plan = k[len("server_plan_") :] + items.append( + { + "vendor_id": vendor.vendor_id, + "region_id": zone_id, + "zone_id": zone_id, + "server_id": server_plan, + "operating_system": "Linux", + "allocation": Allocation.ONDEMAND, + "unit": PriceUnit.HOUR, + "price": v["price"], + "price_upfront": 0, + "price_tiered": [], + "currency": "EUR", + } + ) return items From 262dd4abd3d265f5f7853b29e8f6d6dfc8145ae0 Mon Sep 17 00:00:00 2001 From: "Gergely Daroczi (@daroczig)" Date: Thu, 5 Dec 2024 16:27:38 +0100 Subject: [PATCH 09/13] ingest storage types and prices --- src/sc_crawler/vendors/upcloud.py | 104 +++++++++++++++++++----------- 1 file changed, 67 insertions(+), 37 deletions(-) diff --git a/src/sc_crawler/vendors/upcloud.py b/src/sc_crawler/vendors/upcloud.py index 10adb36e..39234e5f 100644 --- a/src/sc_crawler/vendors/upcloud.py +++ b/src/sc_crawler/vendors/upcloud.py @@ -5,7 +5,13 @@ from upcloud_api import CloudManager from ..lookup import map_compliance_frameworks_to_vendor -from ..table_fields import CpuAllocation, CpuArchitecture, StorageType +from ..table_fields import ( + Allocation, + CpuAllocation, + CpuArchitecture, + PriceUnit, + StorageType, +) # ############################################################################## # Cached client wrappers @@ -13,7 +19,7 @@ @cache def _client() -> CloudManager: - """Authorized Hetzner Cloud client using the HCLOUD_TOKEN env var.""" + """Authorized UpCloud client using the UPCLOUD_USERNAME and UPCLOUD_PASSWORD env vars.""" try: username = environ["UPCLOUD_USERNAME"] except KeyError: @@ -27,6 +33,36 @@ def _client() -> CloudManager: return manager +UPCLOUD_STORAGES = [ + { + "id": "hdd", + "name": "Archive", + "description": "High-capacity data storage", + "storage_type": StorageType.HDD, + "min_size": 1, + "max_size": 4096, + "max_iops": 600, + }, + { + "id": "standard", + "name": "Standard", + "description": "General purpose data storage", + "storage_type": StorageType.SSD, + "min_size": 1, + "max_size": 4096, + "max_iops": 10000, + }, + { + "id": "maxiops", + "name": "MaxIOPS", + "description": "High-performance web servers and applications", + "storage_type": StorageType.SSD, + "min_size": 1, + "max_size": 4096, + "max_iops": 100000, + }, +] + # ############################################################################## # Internal helpers @@ -54,16 +90,6 @@ def _parse_server_name(name): return data -# _client().get_prices() -# servers = _client().get_server_sizes() -# servers[1] - -# templates = _client().get_templates() -# templates[1] - -# prices = _client().get_prices() -# prices["prices"]["zone"][1]["server_plan_HIMEM-24xCPU-512GB"] # EUR cent! - # ############################################################################## # Public methods to fetch data @@ -344,36 +370,40 @@ def inventory_server_prices_spot(vendor): def inventory_storages(vendor): items = [] - # for storage in []: - # items.append( - # { - # "storage_id": , - # "vendor_id": vendor.vendor_id, - # "name": , - # "description": None, - # "storage_type": StorageType...., - # "max_iops": None, - # "max_throughput": None, - # "min_size": None, - # "max_size": None, - # } - # ) + for storage in UPCLOUD_STORAGES: + items.append( + { + "storage_id": storage["id"], + "vendor_id": vendor.vendor_id, + "name": storage["name"], + "description": storage["description"], + "storage_type": storage["storage_type"], + "max_iops": storage["max_iops"], + "max_throughput": None, + "min_size": storage["min_size"], + "max_size": storage["max_size"], + } + ) return items def inventory_storage_prices(vendor): items = [] - # for price in []: - # items.append( - # { - # "vendor_id": vendor.vendor_id, - # "region_id": , - # "storage_id": , - # "unit": PriceUnit.GB_MONTH, - # "price": , - # "currency": "USD", - # } - # ) + prices = _client().get_prices() + for zone_prices in prices["prices"]["zone"]: + zone_id = zone_prices["name"] + for k, v in zone_prices.items(): + if k in ["storage_" + s["id"] for s in UPCLOUD_STORAGES]: + items.append( + { + "vendor_id": vendor.vendor_id, + "region_id": zone_id, + "storage_id": k[len("storage_") :], + "unit": PriceUnit.GB_MONTH, + "price": v["price"], + "currency": "EUR", + } + ) return items From 87c71c6c84536e55bf419064ed530dc5cfe6934b Mon Sep 17 00:00:00 2001 From: "Gergely Daroczi (@daroczig)" Date: Thu, 5 Dec 2024 16:35:50 +0100 Subject: [PATCH 10/13] ingest traffic prices --- src/sc_crawler/vendors/upcloud.py | 46 ++++++++++++++++++------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/src/sc_crawler/vendors/upcloud.py b/src/sc_crawler/vendors/upcloud.py index 39234e5f..200690b6 100644 --- a/src/sc_crawler/vendors/upcloud.py +++ b/src/sc_crawler/vendors/upcloud.py @@ -11,6 +11,7 @@ CpuArchitecture, PriceUnit, StorageType, + TrafficDirection, ) # ############################################################################## @@ -341,7 +342,6 @@ def inventory_server_prices(vendor): items = [] prices = _client().get_prices() for zone_prices in prices["prices"]["zone"]: - zone_id = zone_prices["name"] for k, v in zone_prices.items(): if not k.startswith("server_plan"): continue @@ -349,13 +349,13 @@ def inventory_server_prices(vendor): items.append( { "vendor_id": vendor.vendor_id, - "region_id": zone_id, - "zone_id": zone_id, + "region_id": zone_prices["name"], + "zone_id": zone_prices["name"], "server_id": server_plan, "operating_system": "Linux", "allocation": Allocation.ONDEMAND, "unit": PriceUnit.HOUR, - "price": v["price"], + "price": v["price"] / 100, "price_upfront": 0, "price_tiered": [], "currency": "EUR", @@ -391,16 +391,15 @@ def inventory_storage_prices(vendor): items = [] prices = _client().get_prices() for zone_prices in prices["prices"]["zone"]: - zone_id = zone_prices["name"] for k, v in zone_prices.items(): if k in ["storage_" + s["id"] for s in UPCLOUD_STORAGES]: items.append( { "vendor_id": vendor.vendor_id, - "region_id": zone_id, + "region_id": zone_prices["name"], "storage_id": k[len("storage_") :], "unit": PriceUnit.GB_MONTH, - "price": v["price"], + "price": v["price"] / 100, "currency": "EUR", } ) @@ -409,22 +408,31 @@ def inventory_storage_prices(vendor): def inventory_traffic_prices(vendor): items = [] - # for price in []: - # items.append( - # { - # "vendor_id": vendor.vendor_id, - # "region_id": , - # "price": , - # "price_tiered": [], - # "currency": "USD", - # "unit": PriceUnit.GB_MONTH, - # "direction": TrafficDirection...., - # } - # ) + prices = _client().get_prices() + for zone_prices in prices["prices"]["zone"]: + for k, v in zone_prices.items(): + if k == "public_ipv4_bandwidth_out": + for direction in [d for d in TrafficDirection]: + items.append( + { + "vendor_id": vendor.vendor_id, + "region_id": zone_prices["name"], + "price": ( + v["price"] / 100 + if direction == TrafficDirection.OUT + else 0 + ), + "price_tiered": [], + "currency": "EUR", + "unit": PriceUnit.GB_MONTH, + "direction": direction, + } + ) return items def inventory_ipv4_prices(vendor): + # ipv4_address items = [] # for price in []: # items.append( From b5664b54261d79a9de93e5d76461f0bc742f5b50 Mon Sep 17 00:00:00 2001 From: "Gergely Daroczi (@daroczig)" Date: Thu, 5 Dec 2024 16:38:31 +0100 Subject: [PATCH 11/13] ingest IPv4 prices --- docs/add_vendor.md | 2 +- src/sc_crawler/vendors/upcloud.py | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/add_vendor.md b/docs/add_vendor.md index 0b97160d..be66efb9 100644 --- a/docs/add_vendor.md +++ b/docs/add_vendor.md @@ -236,7 +236,7 @@ def inventory_ipv4_prices(vendor): # "region_id": , # "price": , # "currency": "USD", - # "unit": PriceUnit.HOUR, + # "unit": PriceUnit.MONTH, # } # ) return items diff --git a/src/sc_crawler/vendors/upcloud.py b/src/sc_crawler/vendors/upcloud.py index 200690b6..9cf20620 100644 --- a/src/sc_crawler/vendors/upcloud.py +++ b/src/sc_crawler/vendors/upcloud.py @@ -432,16 +432,18 @@ def inventory_traffic_prices(vendor): def inventory_ipv4_prices(vendor): - # ipv4_address items = [] - # for price in []: - # items.append( - # { - # "vendor_id": vendor.vendor_id, - # "region_id": , - # "price": , - # "currency": "USD", - # "unit": PriceUnit.HOUR, - # } - # ) + prices = _client().get_prices() + for zone_prices in prices["prices"]["zone"]: + for k, v in zone_prices.items(): + if k == "ipv4_address": + items.append( + { + "vendor_id": vendor.vendor_id, + "region_id": zone_prices["name"], + "price": v["price"] / 100, + "currency": "EUR", + "unit": PriceUnit.MONTH, + } + ) return items From 358082c21beb5bc136924f9d555a438fd99d04b7 Mon Sep 17 00:00:00 2001 From: "Gergely Daroczi (@daroczig)" Date: Thu, 5 Dec 2024 16:50:27 +0100 Subject: [PATCH 12/13] IPv4 price is per hour --- src/sc_crawler/vendors/upcloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sc_crawler/vendors/upcloud.py b/src/sc_crawler/vendors/upcloud.py index 9cf20620..db7a8de5 100644 --- a/src/sc_crawler/vendors/upcloud.py +++ b/src/sc_crawler/vendors/upcloud.py @@ -443,7 +443,7 @@ def inventory_ipv4_prices(vendor): "region_id": zone_prices["name"], "price": v["price"] / 100, "currency": "EUR", - "unit": PriceUnit.MONTH, + "unit": PriceUnit.HOUR, } ) return items From 421b5251a02864d5f4d7a4527828c781da1ff687 Mon Sep 17 00:00:00 2001 From: "Gergely Daroczi (@daroczig)" Date: Tue, 10 Dec 2024 23:09:25 +0100 Subject: [PATCH 13/13] note added support for UpCloud --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e75c0efb..402ec866 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## v0.3.x (development version) +New vendor(s): + +- UpCloud + New benchmark(s): - PassMark.