Merge pull request #343 from afeher/drqs-175992119-integrate-bde_xt_c…

…pp_splitter-into-bbs_build

bde_xt_cpp_splitter integrated

BdeBuildSystem/BdeMetadataUtils.cmake

@@ -35,6 +35,7 @@ the parent scope:
     * <package>_TEST_PCDEPS
     * <package>_TEST_SOURCES
     * <package>_GTEST_SOURCES
+    * <package>_SPLIT_TEST_SOURCES
     * <package>_METADATA_DIR
 
 .. code-block:: cmake
@@ -65,6 +66,7 @@ variables in the parent scope:
     * <group>_TEST_PCDEPS
     * <group>_TEST_SOURCES
     * <group>_GTEST_SOURCES
+    * <group>_SPLIT_TEST_SOURCES
     * <group>_METADATA_DIRS
 #]]
 function(bbs_read_metadata)
@@ -168,7 +170,8 @@ macro(bbs_read_group_metadata group dir)
             bbs_read_package_metadata(${pkg} ${_SOURCE_DIR}/${pkg})
             set(propagate_properties COMPONENTS INCLUDE_DIRS INCLUDE_FILES
                                      SOURCE_DIRS SOURCE_FILES
-                                     TEST_SOURCES GTEST_SOURCES METADATA_DIRS)
+                                     TEST_SOURCES GTEST_SOURCES SPLIT_TEST_SOURCES
+                                     METADATA_DIRS)
 
             # Private packages do not propagate their include files to the group
             if (${pkg} IN_LIST _PRIVATE_PACKAGES)
@@ -190,7 +193,7 @@ macro(bbs_read_group_metadata group dir)
 
     foreach(var PACKAGES DEPENDS PCDEPS TEST_DEPENDS TEST_PCDEPS COMPONENTS
                 INCLUDE_DIRS INCLUDE_FILES SOURCE_DIRS SOURCE_FILES
-                TEST_SOURCES GTEST_SOURCES METADATA_DIRS)
+                TEST_SOURCES GTEST_SOURCES SPLIT_TEST_SOURCES METADATA_DIRS)
         set(${group}_${var} ${${group}_${var}} PARENT_SCOPE)
     endforeach()
 endmacro()
@@ -264,6 +267,9 @@ macro(_bbs_set_bde_component_lists dir package mems)
                 list(APPEND ${package}_TEST_SOURCES ${dir}/${mem}.t.cpp)
             elseif(EXISTS ${dir}/${mem}.g.cpp)
                 list(APPEND ${package}_GTEST_SOURCES ${dir}/${mem}.g.cpp)
+            elseif(EXISTS ${dir}/${mem}.xt.cpp)
+                message(TRACE "Split TD: --->  ${dir}/${mem}.xt.cpp <---")
+                list(APPEND ${package}_SPLIT_TEST_SOURCES ${dir}/${mem}.xt.cpp)
             endif()
 
             # finding numbered and forwarding header tests
@@ -296,7 +302,7 @@ macro(_bbs_set_bde_component_lists dir package mems)
     # propagate the lists to the caller
     foreach(var COMPONENTS INCLUDE_DIRS INCLUDE_FILES
                 SOURCE_DIRS SOURCE_FILES MAIN_SOURCE
-                TEST_SOURCES GTEST_SOURCES)
+                TEST_SOURCES GTEST_SOURCES SPLIT_TEST_SOURCES)
         set(${package}_${var} ${${package}_${var}} PARENT_SCOPE)
     endforeach()
 endmacro()

BdeBuildSystem/BdeTargetUtils.cmake

@@ -58,9 +58,9 @@ endif()
 if(NOT DEFINED CHECK_CYCLES)
     find_file(CHECK_CYCLES
               "check_cycles.py"
-              PATHS ${CMAKE_CURRENT_LIST_DIR}/scripts
-              NO_DEFAULT_PATH
-              )
+               PATHS ${CMAKE_CURRENT_LIST_DIR}/scripts
+               NO_DEFAULT_PATH
+             )
 
     if(CHECK_CYCLES)
         message(STATUS "Found check_cycles.py in ${CHECK_CYCLES}")
@@ -159,7 +159,7 @@ function(bbs_configure_target_tests target)
     cmake_parse_arguments(""
                           ""
                           ""
-                          "TEST_SOURCES;SOURCES;GTEST_SOURCES;TEST_DEPS;LABELS"
+                          "TEST_SOURCES;SOURCES;GTEST_SOURCES;SPLIT_SOURCES;TEST_DEPS;LABELS"
                           ${ARGN})
     bbs_assert_no_unparsed_args("")
 
@@ -184,6 +184,13 @@ function(bbs_configure_target_tests target)
                                  LABELS        ${_LABELS})
         set(${target}_TEST_TARGETS "${${target}_TEST_TARGETS}" PARENT_SCOPE)
     endif()
+    if (_SPLIT_SOURCES)
+        bbs_add_component_tests(${target}
+                                SPLIT_SOURCES ${_SPLIT_SOURCES}
+                                TEST_DEPS     ${_TEST_DEPS}
+                                LABELS        ${_LABELS})
+        set(${target}_TEST_TARGETS "${${target}_TEST_TARGETS}" PARENT_SCOPE)
+    endif()
 endfunction()
 
 #[[.rst:
@@ -381,6 +388,13 @@ endfunction()
 #
 # Generate cpp03 source files.
 function (bbs_generate_cpp03_sources srcFiles)
+    cmake_parse_arguments(PARSE_ARGV 1
+                          ""
+                          "IMMEDIATE"
+                          ""
+                          "")
+    bbs_assert_no_unparsed_args("")
+
     if(SIM_CPP11)
         get_property(cmd_wrapper GLOBAL PROPERTY BBS_CMD_WRAPPER)
 
@@ -397,9 +411,17 @@ function (bbs_generate_cpp03_sources srcFiles)
                 string(REPLACE "_cpp03." "." cpp11SrcFile ${srcFile})
                 message(TRACE "sim_cpp11 ${cpp11Operation}: ${cpp11SrcFile} -> ${srcFile}")
 
+                set(command ${cmd_wrapper} "${PERL_EXECUTABLE}" "${SIM_CPP11}" ${cpp11VerifyOption} "${cpp11SrcFile}")
+
+                if (_IMMEDIATE)
+                    execute_process(
+                        COMMAND ${command}
+                        COMMAND_ERROR_IS_FATAL ANY)
+                endif()
+
                 add_custom_command(
                     OUTPUT    "${srcFile}"
-                    COMMAND   ${cmd_wrapper} "${PERL_EXECUTABLE}" "${SIM_CPP11}" ${cpp11VerifyOption} "${cpp11SrcFile}"
+                    COMMAND   ${command}
                     DEPENDS   "${cpp11SrcFile}")
             endif()
         endforeach()
@@ -573,16 +595,18 @@ function(bbs_setup_target_uor target)
                     bbs_generate_cpp03_sources("${${pkg}_INCLUDE_FILES}")
                     bbs_generate_cpp03_sources("${${pkg}_SOURCE_FILES}")
                     bbs_generate_cpp03_sources("${${pkg}_TEST_SOURCES}")
+                    bbs_generate_cpp03_sources("${${pkg}_SPLIT_TEST_SOURCES}" IMMEDIATE)
 
                     if (NOT _SKIP_TESTS)
                         bbs_configure_target_tests(${pkg}
-                                                   TEST_SOURCES   ${${pkg}_TEST_SOURCES}
-                                                   GTEST_SOURCES  ${${pkg}_GTEST_SOURCES}
-                                                   TEST_DEPS      ${${pkg}_DEPENDS}
-                                                                  ${${pkg}_TEST_DEPENDS}
-                                                                  ${${uor_name}_PCDEPS}
-                                                                  ${${uor_name}_TEST_PCDEPS}
-                                                   LABELS         "all" ${target} ${pkg})
+                                                   TEST_SOURCES  ${${pkg}_TEST_SOURCES}
+                                                   GTEST_SOURCES ${${pkg}_GTEST_SOURCES}
+                                                   SPLIT_SOURCES ${${pkg}_SPLIT_TEST_SOURCES}
+                                                   TEST_DEPS     ${${pkg}_DEPENDS}
+                                                                 ${${pkg}_TEST_DEPENDS}
+                                                                 ${${uor_name}_PCDEPS}
+                                                                 ${${uor_name}_TEST_PCDEPS}
+                                                   LABELS        "all" ${target} ${pkg})
                     endif()
                 endif()
             endforeach()
@@ -635,11 +659,12 @@ function(bbs_setup_target_uor target)
             bbs_import_target_dependencies(${target} ${${uor_name}_PCDEPS})
             if (NOT _SKIP_TESTS)
                 bbs_configure_target_tests(${target}
-                                           TEST_SOURCES   ${${uor_name}_TEST_SOURCES}
-                                           GTEST_SOURCES  ${${uor_name}_GTEST_SOURCES}
-                                           TEST_DEPS      ${${uor_name}_PCDEPS}
-                                                          ${${uor_name}_TEST_PCDEPS}
-                                           LABELS         "all" ${target})
+                                           TEST_SOURCES  ${${uor_name}_TEST_SOURCES}
+                                           GTEST_SOURCES ${${uor_name}_GTEST_SOURCES}
+                                           SPLIT_SOURCES ${${uor_name}_SPLIT_TEST_SOURCES}
+                                           TEST_DEPS     ${${uor_name}_PCDEPS}
+                                                         ${${uor_name}_TEST_PCDEPS}
+                                           LABELS        "all" ${target})
                 if (${target}_TEST_TARGETS)
                     bbs_import_target_dependencies(${target} ${${uor_name}_TEST_PCDEPS})
                 endif()
@@ -751,11 +776,12 @@ function(bbs_setup_target_uor target)
         # Set up tests and link against the private library
         if (NOT _SKIP_TESTS)
             bbs_configure_target_tests(${lib_target}
-                                       TEST_SOURCES   ${${uor_name}_TEST_SOURCES}
-                                       GTEST_SOURCES  ${${uor_name}_GTEST_SOURCES}
-                                       TEST_DEPS      ${${uor_name}_PCDEPS}
-                                                      ${${uor_name}_TEST_PCDEPS}
-                                       LABELS         "all" ${target})
+                                       TEST_SOURCES  ${${uor_name}_TEST_SOURCES}
+                                       GTEST_SOURCES ${${uor_name}_GTEST_SOURCES}
+                                       SPLIT_SOURCES ${${uor_name}_SPLIT_TEST_SOURCES}
+                                       TEST_DEPS     ${${uor_name}_PCDEPS}
+                                                     ${${uor_name}_TEST_PCDEPS}
+                                       LABELS        "all" ${target})
             if (TARGET ${lib_target}.t)
                 if (NOT TARGET ${target}.t)
                     add_custom_target(${target}.t)

BdeBuildSystem/BdeTestDriverUtils.cmake

@@ -9,21 +9,37 @@ This module provide a set of function to generate BDE tests
 # On windows we will invoke the python script directly
 # On unix - the shell wrapper will find the interperter and 
 # invoke the implementation file.
+find_package(Python3 3.6 REQUIRED)
+
 find_file(BBS_RUNTEST bbs_runtest.py
           PATHS "${CMAKE_CURRENT_LIST_DIR}/scripts")
 
 if (NOT BBS_RUNTEST)
     message(FATAL_ERROR "Failed to find bbs_runtest")
 endif()
 
-find_package(Python3 3.6 REQUIRED)
 set(BBS_RUNTEST ${Python3_EXECUTABLE} ${BBS_RUNTEST})
 
 if (BBS_USE_WAFSTYLEOUT)
     get_property(cmd_wrapper GLOBAL PROPERTY BBS_CMD_WRAPPER)
     set(BBS_RUNTEST ${cmd_wrapper} ${BBS_RUNTEST})
 endif()
 
+find_file(BBS_SPLIT_TEST bde_xt_cpp_splitter.py
+          PATHS "${CMAKE_CURRENT_LIST_DIR}/scripts/bde_xt_cpp_splitter")
+
+if (BBS_SPLIT_TEST)
+    message(STATUS "Found split test driver generator: ${BBS_SPLIT_TEST}")
+    set(BBS_SPLIT_TEST ${Python3_EXECUTABLE} ${BBS_SPLIT_TEST})
+
+    if (BBS_USE_WAFSTYLEOUT)
+        get_property(cmd_wrapper GLOBAL PROPERTY BBS_CMD_WRAPPER)
+        set(BBS_SPLIT_TEST ${cmd_wrapper} ${BBS_SPLIT_TEST})
+    endif()
+else()
+    message(FATAL_ERROR "Failed to find test split generator")
+endif()
+
 #[[.rst:
 .. command:: bbs_add_bde_style_test
 
@@ -73,7 +89,9 @@ BDE tests and generate necessary build dependencies and test labels.
 
 .. code-block:: cmake
 
-   bbs_add_component_tests(target TEST_SOURCES source1.t.cpp [source2.t.cpp ...]
+   bbs_add_component_tests(target 
+                           TEST_SOURCES source1.t.cpp [source2.t.cpp ...]
+                           SPLIT_SOURCES source3.xt.cpp [source4.xt.cpp]
                            [ WORKING_DIRECTORY      dir            ]
                            [ TEST_VERBOSITY         verbosity      ]
                            [ EXTRA_ARGS             test_arg   ... ]
@@ -88,10 +106,10 @@ function(bbs_add_component_tests target)
                           ""
                           ""
                           "TEST_VERBOSITY;TEST_REGEX"
-                          "EXTRA_ARGS;LABELS;TEST_SOURCES;TEST_DEPS")
+                          "EXTRA_ARGS;LABELS;TEST_SOURCES;SPLIT_SOURCES;TEST_DEPS")
     bbs_assert_no_unparsed_args("")
 
-    if (NOT _TEST_SOURCES)
+    if (NOT _TEST_SOURCES AND NOT _SPLIT_SOURCES)
         message(FATAL_ERROR "No sources for the test ${target}")
     endif()
 
@@ -160,5 +178,126 @@ function(bbs_add_component_tests target)
         endif()
     endforeach()
 
+    set(split_cpp03_test_srcs ${_SPLIT_SOURCES})
+    list(FILTER split_cpp03_test_srcs INCLUDE REGEX "_cpp03\.")
+
+    message(STATUS "Split source: ${_SPLIT_SOURCES}")
+    foreach(test_src ${_SPLIT_SOURCES})
+        # Stripping all extentions from the test source ( including numbers
+        # from the numbered tests )
+        get_filename_component(test_name ${test_src} NAME_WE)
+        if (BDE_TEST_REGEX AND NOT ${test_name} MATCHES "${BDE_TEST_REGEX}")
+            # Generate test target only for matching test regex, if any.
+            continue()
+        endif()
+
+        # Stripping last 2 extentions from the test source (.xt.cpp)
+        get_filename_component(test_target_name ${test_src} NAME_WLE)
+        get_filename_component(test_target_name ${test_target_name} NAME_WLE)
+
+        message(DEBUG "Running TD splitter for: ${test_src}")
+
+        # Check if _cpp03 version of this test exists
+        if (${split_cpp03_test_srcs} MATCHES "${test_target_name}_cpp03\.xt\.cpp")
+            set(test_has_cpp03 TRUE)
+        else()
+            set(test_has_cpp03 FALSE)
+        endif()
+
+        # Creating output folder
+        set(td_output_dir "${CMAKE_CURRENT_BINARY_DIR}/${test_name}_split")
+
+        # remove temp file if it exists so that it will only contain what we generate
+        if(NOT EXISTS "${td_output_dir}")
+            file(MAKE_DIRECTORY "${td_output_dir}")
+        endif()
+
+        set(stamp_file ${td_output_dir}/${test_name}.stamp)
+
+        set(command ${BBS_SPLIT_TEST} -o ${td_output_dir} -s ${test_name}.stamp ${test_src})
+        execute_process(
+            COMMAND ${command}
+            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+            COMMAND_ERROR_IS_FATAL ANY)
+
+        file(STRINGS ${stamp_file} td_cpp_files)
+
+        set(outputs ${stamp_file})
+        foreach(split_test ${td_cpp_files})
+            list(APPEND outputs ${td_output_dir}/${split_test})
+        endforeach()
+
+        add_custom_command(
+            OUTPUT ${outputs}
+            COMMAND ${command}
+            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+            MAIN_DEPENDENCY ${test_src})
+        set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${stamp_file})
+
+        message(DEBUG "Generated files found: ${td_cpp_files}")
+
+        if("${td_cpp_files}" STREQUAL "")
+            message(FATAL_ERROR "Test splitter generated no files")
+        endif()
+
+        # Adding package test target
+        if (NOT TARGET ${test_name}.t)
+            add_custom_target(${test_name}.t)
+        endif()
+
+        if (NOT TARGET ${target}.t)
+            add_custom_target(${target}.t)
+        endif()
+
+        if (NOT TARGET all.t)
+            add_custom_target(all.t)
+        endif()
+
+        # Processing individual tests and adding them to the target test
+        foreach(split_test ${td_cpp_files})
+            get_filename_component(split_target_name ${split_test} NAME_WLE)
+            get_filename_component(split_target_name ${split_target_name} NAME_WLE)
+
+            if (${test_name} MATCHES "_cpp03")
+                # Add a custom target for dependency handling, but no executable for _cpp03 splits
+                add_custom_target(${split_target_name}.t SOURCES ${td_output_dir}/${split_test})
+            else()
+                message(DEBUG "add_executable(${split_target_name}.t EXCLUDE_FROM_ALL ${td_output_dir}/${split_test})")
+                add_executable(${split_target_name}.t EXCLUDE_FROM_ALL ${td_output_dir}/${split_test})
+
+                bbs_add_target_bde_flags(${split_target_name}.t PRIVATE)
+                bbs_add_target_thread_flags(${split_target_name}.t PRIVATE)
+
+                message(DEBUG "target_link_libraries(${split_target_name}.t PUBLIC ${target} ${_TEST_DEPS})")
+                target_link_libraries(${split_target_name}.t PUBLIC ${target} ${_TEST_DEPS})
+
+                if (BDE_BUILD_TARGET_FUZZ)
+                    target_link_libraries(${split_target_name}.t PRIVATE "-fsanitize=fuzzer")
+                endif()
+
+                bbs_add_bde_style_test(${split_target_name}.t
+                                    WORKING_DIRECTORY "${_WORKING_DIRECTORY}"
+                                    TEST_VERBOSITY    "${_TEST_VERBOSITY}"
+                                    EXTRA_ARGS        "${_EXTRA_ARGS}"
+                                    LABELS            "${_LABELS}"
+                                                        "${test_name}")
+
+                if (test_has_cpp03)
+                    target_include_directories(${split_target_name}.t PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/${test_name}_cpp03_split")
+                endif()
+            endif()
+
+            message(TRACE "Adding ${test_name}.t -> ${split_target_name}.t")
+            add_dependencies(${test_name}.t ${split_target_name}.t)
+        endforeach()
+
+        add_dependencies(${target}.t ${test_name}.t)
+        add_dependencies(all.t ${test_name}.t)
+
+        list(APPEND test_targets ${test_name}.t)
+    endforeach()
+
+    message(STATUS "test_targets: ${test_targets}")
+
     set(${target}_TEST_TARGETS "${test_targets}" PARENT_SCOPE)
 endfunction()

BdeBuildSystem/scripts/sim_cpp11_features.pl

@@ -2275,6 +2275,8 @@ ($$)
 # Mapping of extensions to prefix and suffix text, in descending order of the
 # length of the extensions.
 my @extToBoilerplateMap = (
+    [ qr{_cpp03\.xt\.cpp$}
+                            , $cpp03TestDriverPrefix, $cpp03TestDriverSuffix ],
     [ qr{_cpp03\.[0-9]+\.t\.cpp$}
                             , $cpp03TestDriverPrefix, $cpp03TestDriverSuffix ],
     [ qr{_cpp03\.t\.cpp$}   , $cpp03TestDriverPrefix, $cpp03TestDriverSuffix ],

docs/docs/bbs/tools/bbs_build_env.rst

@@ -81,7 +81,7 @@ Environment Variables Set by bbs_build_env
 ------------------------------------------
 
 The following environment variables may be set by evaluating the output of
-bde_setwafenv.py:
+bbs_build_env.py:
 
 - ``CXX``