diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 939a0c9..9945dfd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest services: clickhouse: - image: clickhouse/clickhouse-server + image: clickhouse/clickhouse-server:22.8 ports: ["8123:8123"] steps: - uses: actions/checkout@v3 diff --git a/diamond_miner/generators/database.py b/diamond_miner/generators/database.py index 706d658..1f42417 100644 --- a/diamond_miner/generators/database.py +++ b/diamond_miner/generators/database.py @@ -16,6 +16,7 @@ from diamond_miner.queries import GetProbesDiff from diamond_miner.typing import FlowMapper, IPNetwork, Probe +max_probes = 0 def probe_generator_from_database( client: ClickHouseClient, @@ -45,6 +46,11 @@ def probe_generator_from_database( >>> (str(ip_address(probes[0][0])), *probes[0][1:]) ('::ffff:808:100', 24000, 33434, 1, 'icmp') """ + global max_probes + if max_probes == 0: + max_probes = 4095 # XXX make this a parameter + logger.info("capping the number of probes to send at %d", max_probes) + rows = GetProbesDiff( round_eq=round_, probe_ttl_geq=probe_ttl_geq, probe_ttl_leq=probe_ttl_leq ).execute_iter(client, measurement_id, subsets=subsets) @@ -60,8 +66,8 @@ def probe_generator_from_database( addr_offset, port_offset = mapper.offset(flow_id, dst_prefix_int) dst_addr = dst_prefix_int + addr_offset src_port = probe_src_port + port_offset - if src_port > (2**16 - 1): - # TEMP: Log prefixes that overflows the port number and skip prefix. - logger.warning("Port overflow for %s", row) + # Note that port_offset is actually the number of probes sent after having already sent 256 probes. + if port_offset > max_probes: + logger.warning("not probing %s after having already sent %d probes", row, max_probes+256) break yield dst_addr, src_port, probe_dst_port, ttl, protocol_str # type: ignore diff --git a/docs/dev.md b/docs/dev.md index c3f4718..5729138 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -31,7 +31,7 @@ poetry run bumpversion patch # or minor/major Most tests require a running instance of ClickHouse with pre-populated tables. To start a ClickHouse server and insert the test data: ```bash -docker run --rm -d -p 8123:8123 clickhouse/clickhouse-server:22.6 +docker run --rm -d -p 8123:8123 clickhouse/clickhouse-server:22.8 poetry run python tests/data/insert.py ``` diff --git a/docs/usage.md b/docs/usage.md index dea87e8..b9be993 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -12,7 +12,7 @@ These components can be pieced together to conduct various kind of topology meas To run the examples below, you need a running [ClickHouse](https://clickhouse.com) server: ```bash -docker run --rm -d -p 8123:8123 clickhouse/clickhouse-server:22.6 +docker run --rm -d -p 8123:8123 clickhouse/clickhouse-server:22.8 ``` You also need [`pycaracal`](https://github.com/dioptra-io/caracal) and [`pych-client`](https://github.com/dioptra-io/pych-client).