From e8b16611b582275bdb9815bc5a80e0559a811758 Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Sun, 6 Oct 2024 11:34:09 -0600 Subject: [PATCH] Polygon to cells experimental fuzzer (#800) This adds new fuzzers for the `polygonToCellsExperimental` function, based on the existing functions. Since #796 (this PR is based on that branch) adds containment modes, this updates the existing fuzzers to exercise those options too. The estimation functions are adjusted so that the fuzzers pass. This may reduce performance of the new algorithms somewhat, but avoids potential buffer problems. This can be revisited to get them back to how they were intended to work. --------- Co-authored-by: Nick Rabinowitz --- .gitignore | 1 + CMakeLists.txt | 4 + scripts/make_countries.js | 1 + src/apps/fuzzers/README.md | 5 +- src/apps/fuzzers/fuzzerPolygonToCells.c | 16 ++- .../fuzzerPolygonToCellsExperimental.c | 106 ++++++++++++++++++ .../fuzzerPolygonToCellsExperimentalNoHoles.c | 64 +++++++++++ .../fuzzers/fuzzerPolygonToCellsNoHoles.c | 6 +- .../testPolygonToCellsReportedExperimental.c | 30 +++++ src/h3lib/lib/polyfill.c | 30 ++++- src/h3lib/lib/polygon.c | 2 +- 11 files changed, 250 insertions(+), 15 deletions(-) create mode 100644 src/apps/fuzzers/fuzzerPolygonToCellsExperimental.c create mode 100644 src/apps/fuzzers/fuzzerPolygonToCellsExperimentalNoHoles.c diff --git a/.gitignore b/.gitignore index 5ecab9725..130f47983 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ /lib/ # Travis CI build directory /build/ +/build*/ # Local build directories /Debug/ /Release/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 61724da9c..302336bae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -285,7 +285,9 @@ set(OTHER_SOURCE_FILES src/apps/fuzzers/fuzzerDirectedEdge.c src/apps/fuzzers/fuzzerLocalIj.c src/apps/fuzzers/fuzzerPolygonToCells.c + src/apps/fuzzers/fuzzerPolygonToCellsExperimental.c src/apps/fuzzers/fuzzerPolygonToCellsNoHoles.c + src/apps/fuzzers/fuzzerPolygonToCellsExperimentalNoHoles.c src/apps/fuzzers/fuzzerCellToChildPos.c src/apps/fuzzers/fuzzerInternalAlgos.c src/apps/fuzzers/fuzzerInternalCoordIjk.c @@ -560,7 +562,9 @@ if(BUILD_FUZZERS) add_h3_fuzzer(fuzzerDirectedEdge src/apps/fuzzers/fuzzerDirectedEdge.c) add_h3_fuzzer(fuzzerLocalIj src/apps/fuzzers/fuzzerLocalIj.c) add_h3_fuzzer(fuzzerPolygonToCells src/apps/fuzzers/fuzzerPolygonToCells.c) + add_h3_fuzzer(fuzzerPolygonToCellsExperimental src/apps/fuzzers/fuzzerPolygonToCellsExperimental.c) add_h3_fuzzer(fuzzerPolygonToCellsNoHoles src/apps/fuzzers/fuzzerPolygonToCellsNoHoles.c) + add_h3_fuzzer(fuzzerPolygonToCellsExperimentalNoHoles src/apps/fuzzers/fuzzerPolygonToCellsExperimentalNoHoles.c) add_h3_fuzzer(fuzzerCellToChildPos src/apps/fuzzers/fuzzerCellToChildPos.c) if(ENABLE_REQUIRES_ALL_SYMBOLS) add_h3_fuzzer(fuzzerInternalAlgos src/apps/fuzzers/fuzzerInternalAlgos.c) diff --git a/scripts/make_countries.js b/scripts/make_countries.js index 8741de46e..cef1d8f49 100644 --- a/scripts/make_countries.js +++ b/scripts/make_countries.js @@ -170,6 +170,7 @@ for (int res = 0; res < MAX_RES + 1; res++) { H3_EXPORT(maxPolygonToCellsSizeExperimental)(&COUNTRIES[index], res, CONTAINMENT_CENTER, &numHexagons); hexagons = calloc(numHexagons, sizeof(H3Index)); H3_EXPORT(polygonToCellsExperimental)(&COUNTRIES[index], res, CONTAINMENT_FULL, hexagons); + free(hexagons); } }); diff --git a/src/apps/fuzzers/README.md b/src/apps/fuzzers/README.md index c7349e36f..2bb4eaa8b 100644 --- a/src/apps/fuzzers/README.md +++ b/src/apps/fuzzers/README.md @@ -11,8 +11,8 @@ such as the H3 core library. The public API of H3 is covered in the following fuzzers: -| Function | File or status -| -------- | -------------- +| Function | File +| -------- | ---- | areNeighborCells | [fuzzerDirectedEdge](./fuzzerDirectedEdge.c) | cellArea | [fuzzerCellArea](./fuzzerCellArea.c) | cellToBoundary | [fuzzerCellToLatLng](./fuzzerCellToLatLng.c) @@ -60,6 +60,7 @@ The public API of H3 is covered in the following fuzzers: | localIjToCell | [fuzzerLocalIj](./fuzzerLocalIj.c) | originToDirectedEdges | [fuzzerDirectedEdge](./fuzzerDirectedEdge.c) | polygonToCells | [fuzzerPoylgonToCells](./fuzzerPolygonToCells.c) +| polygonToCellsExperimental | [fuzzerPoylgonToCellsExperimental](./fuzzerPolygonToCellsExperimental.c) [fuzzerPoylgonToCellsExperimentalNoHoles](./fuzzerPolygonToCellsExperimentalNoHoles.c) | radsToDegs | Trivial | stringToH3 | [fuzzerIndexIO](./fuzzerIndexIO.c) | uncompactCells | [fuzzerCompact](./fuzzerCompact.c) diff --git a/src/apps/fuzzers/fuzzerPolygonToCells.c b/src/apps/fuzzers/fuzzerPolygonToCells.c index 2c5333f9a..84fe863b7 100644 --- a/src/apps/fuzzers/fuzzerPolygonToCells.c +++ b/src/apps/fuzzers/fuzzerPolygonToCells.c @@ -1,5 +1,5 @@ /* - * Copyright 2022 Uber Technologies, Inc. + * Copyright 2022-2024 Uber Technologies, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ #include "aflHarness.h" #include "h3api.h" +#include "polygon.h" #include "utility.h" typedef struct { @@ -71,7 +72,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { int res = args->res % (MAX_RES + 1); GeoPolygon geoPolygon; - geoPolygon.numHoles = args->numHoles % MAX_HOLES; + int originalNumHoles = args->numHoles % MAX_HOLES; + geoPolygon.numHoles = originalNumHoles; if (geoPolygon.numHoles < 0) { return 0; } @@ -88,10 +90,12 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { } } - // TODO: Fuzz the `flags` input as well when it has meaningful input - run(&geoPolygon, 0, res); - geoPolygon.numHoles = 0; - run(&geoPolygon, 0, res); + for (uint32_t flags = 0; flags < CONTAINMENT_INVALID; flags++) { + geoPolygon.numHoles = originalNumHoles; + run(&geoPolygon, 0, res); + geoPolygon.numHoles = 0; + run(&geoPolygon, 0, res); + } free(geoPolygon.holes); return 0; diff --git a/src/apps/fuzzers/fuzzerPolygonToCellsExperimental.c b/src/apps/fuzzers/fuzzerPolygonToCellsExperimental.c new file mode 100644 index 000000000..1a87dc4a0 --- /dev/null +++ b/src/apps/fuzzers/fuzzerPolygonToCellsExperimental.c @@ -0,0 +1,106 @@ +/* + * Copyright 2023-2024 Uber Technologies, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** @file + * @brief Fuzzer program for polygonToCells2 and related functions + */ + +#include "aflHarness.h" +#include "h3api.h" +#include "polyfill.h" +#include "polygon.h" +#include "utility.h" + +typedef struct { + int res; + int numHoles; + // repeating: num verts, verts + // We add a large fixed buffer so our test case generator for AFL + // knows how large to make the file. + uint8_t buffer[1024]; +} inputArgs; + +const int MAX_RES = 15; +const int MAX_SZ = 4000000; +const int MAX_HOLES = 100; + +int populateGeoLoop(GeoLoop *g, const uint8_t *data, size_t *offset, + size_t size) { + if (size < *offset + sizeof(int)) { + return 1; + } + int numVerts = *(const int *)(data + *offset); + *offset = *offset + sizeof(int); + g->numVerts = numVerts; + if (size < *offset + sizeof(LatLng) * numVerts) { + return 1; + } + g->verts = (LatLng *)(data + *offset); + *offset = *offset + sizeof(LatLng) * numVerts; + return 0; +} + +void run(GeoPolygon *geoPolygon, uint32_t flags, int res) { + int64_t sz; + H3Error err = H3_EXPORT(maxPolygonToCellsSizeExperimental)(geoPolygon, res, + flags, &sz); + if (!err && sz < MAX_SZ) { + H3Index *out = calloc(sz, sizeof(H3Index)); + H3_EXPORT(polygonToCellsExperimental)(geoPolygon, res, flags, out); + free(out); + } +} + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + // TODO: It is difficult for the fuzzer to generate inputs that are + // considered valid by this fuzzer. fuzzerPolygonToCellsNoHoles.c + // is a workaround for that. + if (size < sizeof(inputArgs)) { + return 0; + } + const inputArgs *args = (const inputArgs *)data; + int res = args->res % (MAX_RES + 1); + + GeoPolygon geoPolygon; + int originalNumHoles = args->numHoles % MAX_HOLES; + geoPolygon.numHoles = originalNumHoles; + if (geoPolygon.numHoles < 0) { + return 0; + } + geoPolygon.holes = calloc(geoPolygon.numHoles, sizeof(GeoLoop)); + size_t offset = sizeof(inputArgs) - sizeof(args->buffer); + if (populateGeoLoop(&geoPolygon.geoloop, data, &offset, size)) { + free(geoPolygon.holes); + return 0; + } + for (int i = 0; i < geoPolygon.numHoles; i++) { + if (populateGeoLoop(&geoPolygon.holes[i], data, &offset, size)) { + free(geoPolygon.holes); + return 0; + } + } + + for (uint32_t flags = 0; flags < CONTAINMENT_INVALID; flags++) { + geoPolygon.numHoles = originalNumHoles; + run(&geoPolygon, flags, res); + geoPolygon.numHoles = 0; + run(&geoPolygon, flags, res); + } + free(geoPolygon.holes); + + return 0; +} + +AFL_HARNESS_MAIN(sizeof(inputArgs)); diff --git a/src/apps/fuzzers/fuzzerPolygonToCellsExperimentalNoHoles.c b/src/apps/fuzzers/fuzzerPolygonToCellsExperimentalNoHoles.c new file mode 100644 index 000000000..a507e432c --- /dev/null +++ b/src/apps/fuzzers/fuzzerPolygonToCellsExperimentalNoHoles.c @@ -0,0 +1,64 @@ +/* + * Copyright 2023-2024 Uber Technologies, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** @file + * @brief Fuzzer program for polygonToCellsExperimental and related functions, + * without holes + */ + +#include "aflHarness.h" +#include "h3api.h" +#include "polyfill.h" +#include "polygon.h" +#include "utility.h" + +const int MAX_RES = 15; +const int MAX_SZ = 4000000; + +void run(GeoPolygon *geoPolygon, uint32_t flags, int res) { + int64_t sz; + H3Error err = H3_EXPORT(maxPolygonToCellsSizeExperimental)(geoPolygon, res, + flags, &sz); + if (!err && sz < MAX_SZ) { + H3Index *out = calloc(sz, sizeof(H3Index)); + H3_EXPORT(polygonToCellsExperimental)(geoPolygon, res, flags, out); + free(out); + } +} + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + if (size < sizeof(int)) { + return 0; + } + + uint8_t res = *data; + size_t vertsSize = size - 1; + int numVerts = vertsSize / sizeof(LatLng); + + GeoPolygon geoPolygon; + geoPolygon.numHoles = 0; + geoPolygon.holes = NULL; + geoPolygon.geoloop.numVerts = numVerts; + // Offset by 1 since *data was used for `res`, above. + geoPolygon.geoloop.verts = (LatLng *)(data + 1); + + for (uint32_t flags = 0; flags < CONTAINMENT_INVALID; flags++) { + run(&geoPolygon, flags, res); + } + + return 0; +} + +AFL_HARNESS_MAIN(sizeof(H3Index) * 1024); diff --git a/src/apps/fuzzers/fuzzerPolygonToCellsNoHoles.c b/src/apps/fuzzers/fuzzerPolygonToCellsNoHoles.c index 72558b51e..b4e1d47bf 100644 --- a/src/apps/fuzzers/fuzzerPolygonToCellsNoHoles.c +++ b/src/apps/fuzzers/fuzzerPolygonToCellsNoHoles.c @@ -19,6 +19,7 @@ #include "aflHarness.h" #include "h3api.h" +#include "polygon.h" #include "utility.h" const int MAX_RES = 15; @@ -50,8 +51,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { // Offset by 1 since *data was used for `res`, above. geoPolygon.geoloop.verts = (LatLng *)(data + 1); - // TODO: Fuzz the `flags` input as well when it has meaningful input - run(&geoPolygon, 0, res); + for (uint32_t flags = 0; flags < CONTAINMENT_INVALID; flags++) { + run(&geoPolygon, flags, res); + } return 0; } diff --git a/src/apps/testapps/testPolygonToCellsReportedExperimental.c b/src/apps/testapps/testPolygonToCellsReportedExperimental.c index b3fe8e5b8..e1cc62fea 100644 --- a/src/apps/testapps/testPolygonToCellsReportedExperimental.c +++ b/src/apps/testapps/testPolygonToCellsReportedExperimental.c @@ -28,6 +28,36 @@ // Tests for specific polygonToCells examples SUITE(polygonToCells_reported) { + // fuzzer crash due to inconsistent handling of CONTAINMENT_OVERLAPPING + TEST(fuzzer_crash) { + uint8_t data[] = { + 0xff, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa, 0x0, 0x0, 0xff, + 0xff, 0x0, 0x0, 0x0, 0xa, 0xa, 0xa, 0xa, 0xa, 0xff, + }; + + uint8_t res = 0; + size_t vertsSize = sizeof(data); + int numVerts = vertsSize / sizeof(LatLng); + + GeoPolygon geoPolygon; + geoPolygon.numHoles = 0; + geoPolygon.holes = NULL; + geoPolygon.geoloop.numVerts = numVerts; + // Offset by 1 since *data was used for `res`, above. + geoPolygon.geoloop.verts = (LatLng *)(data); + + uint32_t flags = CONTAINMENT_OVERLAPPING; + int64_t sz; + t_assertSuccess(H3_EXPORT(maxPolygonToCellsSizeExperimental)( + &geoPolygon, res, flags, &sz)); + t_assert(sz == 1, "Expected output count"); + H3Index *out = calloc(sz, sizeof(H3Index)); + t_assertSuccess(H3_EXPORT(polygonToCellsExperimental)(&geoPolygon, res, + flags, out)); + free(out); + } + // https://github.com/uber/h3-js/issues/76#issuecomment-561204505 TEST(entireWorld) { // TODO: Fails for a single worldwide polygon diff --git a/src/h3lib/lib/polyfill.c b/src/h3lib/lib/polyfill.c index 150dd2474..6269aaef3 100644 --- a/src/h3lib/lib/polyfill.c +++ b/src/h3lib/lib/polyfill.c @@ -433,7 +433,8 @@ void iterStepPolygonCompact(IterCellsPolygonCompact *iter) { // Target res: Do a fine-grained check if (cellRes == iter->_res) { - if (mode == CONTAINMENT_CENTER || mode == CONTAINMENT_OVERLAPPING) { + if (mode == CONTAINMENT_CENTER || mode == CONTAINMENT_OVERLAPPING || + mode == CONTAINMENT_OVERLAPPING_BBOX) { // Check if the cell center is inside the polygon LatLng center; H3Error centerErr = H3_EXPORT(cellToLatLng)(cell, ¢er); @@ -448,7 +449,8 @@ void iterStepPolygonCompact(IterCellsPolygonCompact *iter) { return; } } - if (mode == CONTAINMENT_OVERLAPPING) { + if (mode == CONTAINMENT_OVERLAPPING || + mode == CONTAINMENT_OVERLAPPING_BBOX) { // For overlapping, we need to do a quick check to determine // whether the polygon is wholly contained by the cell. We // check the first polygon vertex, which if it is contained @@ -477,7 +479,8 @@ void iterStepPolygonCompact(IterCellsPolygonCompact *iter) { } } } - if (mode == CONTAINMENT_FULL || mode == CONTAINMENT_OVERLAPPING) { + if (mode == CONTAINMENT_FULL || mode == CONTAINMENT_OVERLAPPING || + mode == CONTAINMENT_OVERLAPPING_BBOX) { CellBoundary boundary; H3Error boundaryErr = H3_EXPORT(cellToBoundary)(cell, &boundary); @@ -494,7 +497,8 @@ void iterStepPolygonCompact(IterCellsPolygonCompact *iter) { return; } // Check if the cell is fully contained by the polygon - if (mode == CONTAINMENT_FULL && + if ((mode == CONTAINMENT_FULL || + mode == CONTAINMENT_OVERLAPPING_BBOX) && cellBoundaryInsidePolygon(iter->_polygon, iter->_bboxes, &boundary, &bbox)) { // Set to next output @@ -692,9 +696,27 @@ void iterDestroyPolygon(IterCellsPolygon *iter) { H3Error H3_EXPORT(polygonToCellsExperimental)(const GeoPolygon *polygon, int res, uint32_t flags, H3Index *out) { +#ifdef H3_POLYGON_TO_CELLS_ASSERT + // TODO: This is incompatible with testH3Memory, since it will make more + // allocations. This is just for debugging that the algorithm is not + // exceeding its buffer size. + int64_t maxSize; + H3Error sizeError = H3_EXPORT(maxPolygonToCellsSizeExperimental)( + polygon, res, flags, &maxSize); + if (sizeError) { + return sizeError; + } +#endif + IterCellsPolygon iter = iterInitPolygon(polygon, res, flags); int64_t i = 0; for (; iter.cell; iterStepPolygon(&iter)) { +#ifdef H3_POLYGON_TO_CELLS_ASSERT + if (NEVER(i >= maxSize)) { + iterDestroyPolygon(&iter); + return E_FAILED; + } +#endif out[i++] = iter.cell; } return iter.error; diff --git a/src/h3lib/lib/polygon.c b/src/h3lib/lib/polygon.c index 6119ee7d7..18f5b858c 100644 --- a/src/h3lib/lib/polygon.c +++ b/src/h3lib/lib/polygon.c @@ -130,7 +130,7 @@ bool cellBoundaryInsidePolygon(const GeoPolygon *geoPolygon, const BBox *bboxes, // Check for line intersections with, or containment of, any hole for (int i = 0; i < geoPolygon->numHoles; i++) { - // If the hole has no verts, it is not possible to intersect + // If the hole has no verts, it is not possible to intersect with it. if (geoPolygon->holes[i].numVerts > 0 && (pointInsideGeoLoop(&boundaryLoop, boundaryBBox, &geoPolygon->holes[i].verts[0]) ||