From 1666da9b85762e22fd652576d0796d756f22ce5f Mon Sep 17 00:00:00 2001 From: afeher Date: Mon, 5 Aug 2024 14:50:53 -0400 Subject: [PATCH] bde_xt_cpp_splitter integrated --- BdeBuildSystem/BdeMetadataUtils.cmake | 12 +- BdeBuildSystem/BdeTargetUtils.cmake | 70 ++++++--- BdeBuildSystem/BdeTestDriverUtils.cmake | 147 ++++++++++++++++++- BdeBuildSystem/scripts/sim_cpp11_features.pl | 2 + docs/docs/bbs/tools/bbs_build_env.rst | 2 +- 5 files changed, 203 insertions(+), 30 deletions(-) diff --git a/BdeBuildSystem/BdeMetadataUtils.cmake b/BdeBuildSystem/BdeMetadataUtils.cmake index ea4a8adf..19034399 100644 --- a/BdeBuildSystem/BdeMetadataUtils.cmake +++ b/BdeBuildSystem/BdeMetadataUtils.cmake @@ -35,6 +35,7 @@ the parent scope: * _TEST_PCDEPS * _TEST_SOURCES * _GTEST_SOURCES + * _SPLIT_TEST_SOURCES * _METADATA_DIR .. code-block:: cmake @@ -65,6 +66,7 @@ variables in the parent scope: * _TEST_PCDEPS * _TEST_SOURCES * _GTEST_SOURCES + * _SPLIT_TEST_SOURCES * _METADATA_DIRS #]] function(bbs_read_metadata) @@ -168,7 +170,8 @@ macro(bbs_read_group_metadata group dir) bbs_read_package_metadata(${pkg} ${_SOURCE_DIR}/${pkg}) set(propagate_properties COMPONENTS INCLUDE_DIRS INCLUDE_FILES SOURCE_DIRS SOURCE_FILES - TEST_SOURCES GTEST_SOURCES METADATA_DIRS) + TEST_SOURCES GTEST_SOURCES SPLIT_TEST_SOURCES + METADATA_DIRS) # Private packages do not propagate their include files to the group if (${pkg} IN_LIST _PRIVATE_PACKAGES) @@ -190,7 +193,7 @@ macro(bbs_read_group_metadata group dir) foreach(var PACKAGES DEPENDS PCDEPS TEST_DEPENDS TEST_PCDEPS COMPONENTS INCLUDE_DIRS INCLUDE_FILES SOURCE_DIRS SOURCE_FILES - TEST_SOURCES GTEST_SOURCES METADATA_DIRS) + TEST_SOURCES GTEST_SOURCES SPLIT_TEST_SOURCES METADATA_DIRS) set(${group}_${var} ${${group}_${var}} PARENT_SCOPE) endforeach() endmacro() @@ -264,6 +267,9 @@ macro(_bbs_set_bde_component_lists dir package mems) list(APPEND ${package}_TEST_SOURCES ${dir}/${mem}.t.cpp) elseif(EXISTS ${dir}/${mem}.g.cpp) list(APPEND ${package}_GTEST_SOURCES ${dir}/${mem}.g.cpp) + elseif(EXISTS ${dir}/${mem}.xt.cpp) + message(TRACE "Split TD: ---> ${dir}/${mem}.xt.cpp <---") + list(APPEND ${package}_SPLIT_TEST_SOURCES ${dir}/${mem}.xt.cpp) endif() # finding numbered and forwarding header tests @@ -296,7 +302,7 @@ macro(_bbs_set_bde_component_lists dir package mems) # propagate the lists to the caller foreach(var COMPONENTS INCLUDE_DIRS INCLUDE_FILES SOURCE_DIRS SOURCE_FILES MAIN_SOURCE - TEST_SOURCES GTEST_SOURCES) + TEST_SOURCES GTEST_SOURCES SPLIT_TEST_SOURCES) set(${package}_${var} ${${package}_${var}} PARENT_SCOPE) endforeach() endmacro() diff --git a/BdeBuildSystem/BdeTargetUtils.cmake b/BdeBuildSystem/BdeTargetUtils.cmake index 08905116..fa759d5a 100644 --- a/BdeBuildSystem/BdeTargetUtils.cmake +++ b/BdeBuildSystem/BdeTargetUtils.cmake @@ -58,9 +58,9 @@ endif() if(NOT DEFINED CHECK_CYCLES) find_file(CHECK_CYCLES "check_cycles.py" - PATHS ${CMAKE_CURRENT_LIST_DIR}/scripts - NO_DEFAULT_PATH - ) + PATHS ${CMAKE_CURRENT_LIST_DIR}/scripts + NO_DEFAULT_PATH + ) if(CHECK_CYCLES) message(STATUS "Found check_cycles.py in ${CHECK_CYCLES}") @@ -159,7 +159,7 @@ function(bbs_configure_target_tests target) cmake_parse_arguments("" "" "" - "TEST_SOURCES;SOURCES;GTEST_SOURCES;TEST_DEPS;LABELS" + "TEST_SOURCES;SOURCES;GTEST_SOURCES;SPLIT_SOURCES;TEST_DEPS;LABELS" ${ARGN}) bbs_assert_no_unparsed_args("") @@ -184,6 +184,13 @@ function(bbs_configure_target_tests target) LABELS ${_LABELS}) set(${target}_TEST_TARGETS "${${target}_TEST_TARGETS}" PARENT_SCOPE) endif() + if (_SPLIT_SOURCES) + bbs_add_component_tests(${target} + SPLIT_SOURCES ${_SPLIT_SOURCES} + TEST_DEPS ${_TEST_DEPS} + LABELS ${_LABELS}) + set(${target}_TEST_TARGETS "${${target}_TEST_TARGETS}" PARENT_SCOPE) + endif() endfunction() #[[.rst: @@ -381,6 +388,13 @@ endfunction() # # Generate cpp03 source files. function (bbs_generate_cpp03_sources srcFiles) + cmake_parse_arguments(PARSE_ARGV 1 + "" + "IMMEDIATE" + "" + "") + bbs_assert_no_unparsed_args("") + if(SIM_CPP11) get_property(cmd_wrapper GLOBAL PROPERTY BBS_CMD_WRAPPER) @@ -397,9 +411,17 @@ function (bbs_generate_cpp03_sources srcFiles) string(REPLACE "_cpp03." "." cpp11SrcFile ${srcFile}) message(TRACE "sim_cpp11 ${cpp11Operation}: ${cpp11SrcFile} -> ${srcFile}") + set(command ${cmd_wrapper} "${PERL_EXECUTABLE}" "${SIM_CPP11}" ${cpp11VerifyOption} "${cpp11SrcFile}") + + if (_IMMEDIATE) + execute_process( + COMMAND ${command} + COMMAND_ERROR_IS_FATAL ANY) + endif() + add_custom_command( OUTPUT "${srcFile}" - COMMAND ${cmd_wrapper} "${PERL_EXECUTABLE}" "${SIM_CPP11}" ${cpp11VerifyOption} "${cpp11SrcFile}" + COMMAND ${command} DEPENDS "${cpp11SrcFile}") endif() endforeach() @@ -573,16 +595,18 @@ function(bbs_setup_target_uor target) bbs_generate_cpp03_sources("${${pkg}_INCLUDE_FILES}") bbs_generate_cpp03_sources("${${pkg}_SOURCE_FILES}") bbs_generate_cpp03_sources("${${pkg}_TEST_SOURCES}") + bbs_generate_cpp03_sources("${${pkg}_SPLIT_TEST_SOURCES}" IMMEDIATE) if (NOT _SKIP_TESTS) bbs_configure_target_tests(${pkg} - TEST_SOURCES ${${pkg}_TEST_SOURCES} - GTEST_SOURCES ${${pkg}_GTEST_SOURCES} - TEST_DEPS ${${pkg}_DEPENDS} - ${${pkg}_TEST_DEPENDS} - ${${uor_name}_PCDEPS} - ${${uor_name}_TEST_PCDEPS} - LABELS "all" ${target} ${pkg}) + TEST_SOURCES ${${pkg}_TEST_SOURCES} + GTEST_SOURCES ${${pkg}_GTEST_SOURCES} + SPLIT_SOURCES ${${pkg}_SPLIT_TEST_SOURCES} + TEST_DEPS ${${pkg}_DEPENDS} + ${${pkg}_TEST_DEPENDS} + ${${uor_name}_PCDEPS} + ${${uor_name}_TEST_PCDEPS} + LABELS "all" ${target} ${pkg}) endif() endif() endforeach() @@ -635,11 +659,12 @@ function(bbs_setup_target_uor target) bbs_import_target_dependencies(${target} ${${uor_name}_PCDEPS}) if (NOT _SKIP_TESTS) bbs_configure_target_tests(${target} - TEST_SOURCES ${${uor_name}_TEST_SOURCES} - GTEST_SOURCES ${${uor_name}_GTEST_SOURCES} - TEST_DEPS ${${uor_name}_PCDEPS} - ${${uor_name}_TEST_PCDEPS} - LABELS "all" ${target}) + TEST_SOURCES ${${uor_name}_TEST_SOURCES} + GTEST_SOURCES ${${uor_name}_GTEST_SOURCES} + SPLIT_SOURCES ${${uor_name}_SPLIT_TEST_SOURCES} + TEST_DEPS ${${uor_name}_PCDEPS} + ${${uor_name}_TEST_PCDEPS} + LABELS "all" ${target}) if (${target}_TEST_TARGETS) bbs_import_target_dependencies(${target} ${${uor_name}_TEST_PCDEPS}) endif() @@ -751,11 +776,12 @@ function(bbs_setup_target_uor target) # Set up tests and link against the private library if (NOT _SKIP_TESTS) bbs_configure_target_tests(${lib_target} - TEST_SOURCES ${${uor_name}_TEST_SOURCES} - GTEST_SOURCES ${${uor_name}_GTEST_SOURCES} - TEST_DEPS ${${uor_name}_PCDEPS} - ${${uor_name}_TEST_PCDEPS} - LABELS "all" ${target}) + TEST_SOURCES ${${uor_name}_TEST_SOURCES} + GTEST_SOURCES ${${uor_name}_GTEST_SOURCES} + SPLIT_SOURCES ${${uor_name}_SPLIT_TEST_SOURCES} + TEST_DEPS ${${uor_name}_PCDEPS} + ${${uor_name}_TEST_PCDEPS} + LABELS "all" ${target}) if (TARGET ${lib_target}.t) if (NOT TARGET ${target}.t) add_custom_target(${target}.t) diff --git a/BdeBuildSystem/BdeTestDriverUtils.cmake b/BdeBuildSystem/BdeTestDriverUtils.cmake index f76bbaa4..af0c2581 100644 --- a/BdeBuildSystem/BdeTestDriverUtils.cmake +++ b/BdeBuildSystem/BdeTestDriverUtils.cmake @@ -9,6 +9,8 @@ This module provide a set of function to generate BDE tests # On windows we will invoke the python script directly # On unix - the shell wrapper will find the interperter and # invoke the implementation file. +find_package(Python3 3.6 REQUIRED) + find_file(BBS_RUNTEST bbs_runtest.py PATHS "${CMAKE_CURRENT_LIST_DIR}/scripts") @@ -16,7 +18,6 @@ if (NOT BBS_RUNTEST) message(FATAL_ERROR "Failed to find bbs_runtest") endif() -find_package(Python3 3.6 REQUIRED) set(BBS_RUNTEST ${Python3_EXECUTABLE} ${BBS_RUNTEST}) if (BBS_USE_WAFSTYLEOUT) @@ -24,6 +25,21 @@ if (BBS_USE_WAFSTYLEOUT) set(BBS_RUNTEST ${cmd_wrapper} ${BBS_RUNTEST}) endif() +find_file(BBS_SPLIT_TEST bde_xt_cpp_splitter.py + PATHS "${CMAKE_CURRENT_LIST_DIR}/scripts/bde_xt_cpp_splitter") + +if (BBS_SPLIT_TEST) + message(STATUS "Found split test driver generator: ${BBS_SPLIT_TEST}") + set(BBS_SPLIT_TEST ${Python3_EXECUTABLE} ${BBS_SPLIT_TEST}) + + if (BBS_USE_WAFSTYLEOUT) + get_property(cmd_wrapper GLOBAL PROPERTY BBS_CMD_WRAPPER) + set(BBS_SPLIT_TEST ${cmd_wrapper} ${BBS_SPLIT_TEST}) + endif() +else() + message(FATAL_ERROR "Failed to find test split generator") +endif() + #[[.rst: .. command:: bbs_add_bde_style_test @@ -73,7 +89,9 @@ BDE tests and generate necessary build dependencies and test labels. .. code-block:: cmake - bbs_add_component_tests(target TEST_SOURCES source1.t.cpp [source2.t.cpp ...] + bbs_add_component_tests(target + TEST_SOURCES source1.t.cpp [source2.t.cpp ...] + SPLIT_SOURCES source3.xt.cpp [source4.xt.cpp] [ WORKING_DIRECTORY dir ] [ TEST_VERBOSITY verbosity ] [ EXTRA_ARGS test_arg ... ] @@ -88,10 +106,10 @@ function(bbs_add_component_tests target) "" "" "TEST_VERBOSITY;TEST_REGEX" - "EXTRA_ARGS;LABELS;TEST_SOURCES;TEST_DEPS") + "EXTRA_ARGS;LABELS;TEST_SOURCES;SPLIT_SOURCES;TEST_DEPS") bbs_assert_no_unparsed_args("") - if (NOT _TEST_SOURCES) + if (NOT _TEST_SOURCES AND NOT _SPLIT_SOURCES) message(FATAL_ERROR "No sources for the test ${target}") endif() @@ -160,5 +178,126 @@ function(bbs_add_component_tests target) endif() endforeach() + set(split_cpp03_test_srcs ${_SPLIT_SOURCES}) + list(FILTER split_cpp03_test_srcs INCLUDE REGEX "_cpp03\.") + + message(STATUS "Split source: ${_SPLIT_SOURCES}") + foreach(test_src ${_SPLIT_SOURCES}) + # Stripping all extentions from the test source ( including numbers + # from the numbered tests ) + get_filename_component(test_name ${test_src} NAME_WE) + if (BDE_TEST_REGEX AND NOT ${test_name} MATCHES "${BDE_TEST_REGEX}") + # Generate test target only for matching test regex, if any. + continue() + endif() + + # Stripping last 2 extentions from the test source (.xt.cpp) + get_filename_component(test_target_name ${test_src} NAME_WLE) + get_filename_component(test_target_name ${test_target_name} NAME_WLE) + + message(DEBUG "Running TD splitter for: ${test_src}") + + # Check if _cpp03 version of this test exists + if (${split_cpp03_test_srcs} MATCHES "${test_target_name}_cpp03\.xt\.cpp") + set(test_has_cpp03 TRUE) + else() + set(test_has_cpp03 FALSE) + endif() + + # Creating output folder + set(td_output_dir "${CMAKE_CURRENT_BINARY_DIR}/${test_name}_split") + + # remove temp file if it exists so that it will only contain what we generate + if(NOT EXISTS "${td_output_dir}") + file(MAKE_DIRECTORY "${td_output_dir}") + endif() + + set(stamp_file ${td_output_dir}/${test_name}.stamp) + + set(command ${BBS_SPLIT_TEST} -o ${td_output_dir} -s ${test_name}.stamp ${test_src}) + execute_process( + COMMAND ${command} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMAND_ERROR_IS_FATAL ANY) + + file(STRINGS ${stamp_file} td_cpp_files) + + set(outputs ${stamp_file}) + foreach(split_test ${td_cpp_files}) + list(APPEND outputs ${td_output_dir}/${split_test}) + endforeach() + + add_custom_command( + OUTPUT ${outputs} + COMMAND ${command} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + MAIN_DEPENDENCY ${test_src}) + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${stamp_file}) + + message(DEBUG "Generated files found: ${td_cpp_files}") + + if("${td_cpp_files}" STREQUAL "") + message(FATAL_ERROR "Test splitter generated no files") + endif() + + # Adding package test target + if (NOT TARGET ${test_name}.t) + add_custom_target(${test_name}.t) + endif() + + if (NOT TARGET ${target}.t) + add_custom_target(${target}.t) + endif() + + if (NOT TARGET all.t) + add_custom_target(all.t) + endif() + + # Processing individual tests and adding them to the target test + foreach(split_test ${td_cpp_files}) + get_filename_component(split_target_name ${split_test} NAME_WLE) + get_filename_component(split_target_name ${split_target_name} NAME_WLE) + + if (${test_name} MATCHES "_cpp03") + # Add a custom target for dependency handling, but no executable for _cpp03 splits + add_custom_target(${split_target_name}.t SOURCES ${td_output_dir}/${split_test}) + else() + message(DEBUG "add_executable(${split_target_name}.t EXCLUDE_FROM_ALL ${td_output_dir}/${split_test})") + add_executable(${split_target_name}.t EXCLUDE_FROM_ALL ${td_output_dir}/${split_test}) + + bbs_add_target_bde_flags(${split_target_name}.t PRIVATE) + bbs_add_target_thread_flags(${split_target_name}.t PRIVATE) + + message(DEBUG "target_link_libraries(${split_target_name}.t PUBLIC ${target} ${_TEST_DEPS})") + target_link_libraries(${split_target_name}.t PUBLIC ${target} ${_TEST_DEPS}) + + if (BDE_BUILD_TARGET_FUZZ) + target_link_libraries(${split_target_name}.t PRIVATE "-fsanitize=fuzzer") + endif() + + bbs_add_bde_style_test(${split_target_name}.t + WORKING_DIRECTORY "${_WORKING_DIRECTORY}" + TEST_VERBOSITY "${_TEST_VERBOSITY}" + EXTRA_ARGS "${_EXTRA_ARGS}" + LABELS "${_LABELS}" + "${test_name}") + + if (test_has_cpp03) + target_include_directories(${split_target_name}.t PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/${test_name}_cpp03_split") + endif() + endif() + + message(TRACE "Adding ${test_name}.t -> ${split_target_name}.t") + add_dependencies(${test_name}.t ${split_target_name}.t) + endforeach() + + add_dependencies(${target}.t ${test_name}.t) + add_dependencies(all.t ${test_name}.t) + + list(APPEND test_targets ${test_name}.t) + endforeach() + + message(STATUS "test_targets: ${test_targets}") + set(${target}_TEST_TARGETS "${test_targets}" PARENT_SCOPE) endfunction() diff --git a/BdeBuildSystem/scripts/sim_cpp11_features.pl b/BdeBuildSystem/scripts/sim_cpp11_features.pl index e7422cdd..a70a7caa 100755 --- a/BdeBuildSystem/scripts/sim_cpp11_features.pl +++ b/BdeBuildSystem/scripts/sim_cpp11_features.pl @@ -2275,6 +2275,8 @@ ($$) # Mapping of extensions to prefix and suffix text, in descending order of the # length of the extensions. my @extToBoilerplateMap = ( + [ qr{_cpp03\.xt\.cpp$} + , $cpp03TestDriverPrefix, $cpp03TestDriverSuffix ], [ qr{_cpp03\.[0-9]+\.t\.cpp$} , $cpp03TestDriverPrefix, $cpp03TestDriverSuffix ], [ qr{_cpp03\.t\.cpp$} , $cpp03TestDriverPrefix, $cpp03TestDriverSuffix ], diff --git a/docs/docs/bbs/tools/bbs_build_env.rst b/docs/docs/bbs/tools/bbs_build_env.rst index 37e05332..8bb8c4d7 100644 --- a/docs/docs/bbs/tools/bbs_build_env.rst +++ b/docs/docs/bbs/tools/bbs_build_env.rst @@ -81,7 +81,7 @@ Environment Variables Set by bbs_build_env ------------------------------------------ The following environment variables may be set by evaluating the output of -bde_setwafenv.py: +bbs_build_env.py: - ``CXX``