diff --git a/.gitignore b/.gitignore index 5890e7732..594759257 100644 --- a/.gitignore +++ b/.gitignore @@ -4,15 +4,6 @@ .clangd cscope* compile_commands.* -/components -/drivers -/bsp -examples/solutions -examples/wifi -examples/btble -examples/zigbee -tools/bflb_tools -tools/CI chiptest **/__pycache__ **/build @@ -26,6 +17,5 @@ chiptest *.pptx *.svd **/_build -tools/reg_parser/Peripherals/*_reg.h -tools/reg_parser/InitVals/*_register.c **/img_create_iot +docs/zh/_static/** \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 148a9d037..ae9f0a949 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,36 @@ cmake_minimum_required(VERSION 3.15) +# check VERSION file +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION") + # read info from VERSION file + file(READ "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION_FILE_CONTENT) + + string(REGEX MATCH "PROJECT_SDK_VERSION[ ]+\"([^\"]+)\"" _ ${VERSION_FILE_CONTENT}) + set(PROJECT_SDK_VERSION "${CMAKE_MATCH_1}") +else() + # get git tag + execute_process( + COMMAND git describe --abbrev=8 --tags --dirty --always + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE GIT_TAG + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + ) + + if(GIT_TAG) + set(PROJECT_SDK_VERSION ${GIT_TAG}) + else() + message(WARNING "No Valid version info found for SDK!") + set(PROJECT_SDK_VERSION "version-unknown-panic") + endif() +endif() +message(STATUS "Project SDK Version: ${PROJECT_SDK_VERSION}") + +# handle __FILE__ macro issue +set(SOURCE_PATH "${CMAKE_CURRENT_SOURCE_DIR}") +sdk_add_compile_options("-fmacro-prefix-map=${SOURCE_PATH}=.") +sdk_add_compile_options("-fdebug-prefix-map=${SOURCE_PATH}=.") + add_subdirectory(bsp) add_subdirectory(components) add_subdirectory(drivers/lhal) @@ -10,4 +41,4 @@ endif() sdk_add_subdirectory_ifdef(CONFIG_RF drivers/soc/${CHIP}/phyrf) if("${CHIP}" STREQUAL "bl616") sdk_add_subdirectory_ifdef(CONFIG_RF drivers/rfparam) -endif() \ No newline at end of file +endif() diff --git a/VERSION b/VERSION new file mode 100644 index 000000000..d280973b8 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +PROJECT_SDK_VERSION "2.0.0" \ No newline at end of file diff --git a/bsp/CMakeLists.txt b/bsp/CMakeLists.txt index b44089780..c45ba4425 100644 --- a/bsp/CMakeLists.txt +++ b/bsp/CMakeLists.txt @@ -1,2 +1,9 @@ +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/sdk_version.h.in" + "${CMAKE_BINARY_DIR}/generated/sdk_version.h" +) +target_include_directories(app PRIVATE "${CMAKE_BINARY_DIR}/generated") +target_sources(app PRIVATE sdk_version.c) + add_subdirectory(board) -add_subdirectory(common) \ No newline at end of file +add_subdirectory(common) diff --git a/bsp/board/bl602dk/builtin_imgs/boot2_bl602_debug_v8.1.1.bin b/bsp/board/bl602dk/builtin_imgs/boot2_bl602_debug_v8.1.1.bin deleted file mode 100644 index 5f5006de5..000000000 Binary files a/bsp/board/bl602dk/builtin_imgs/boot2_bl602_debug_v8.1.1.bin and /dev/null differ diff --git a/bsp/board/bl602dk/builtin_imgs/boot2_bl602_debug_v8.1.2.bin b/bsp/board/bl602dk/builtin_imgs/boot2_bl602_debug_v8.1.2.bin new file mode 100644 index 000000000..3a12ea87c Binary files /dev/null and b/bsp/board/bl602dk/builtin_imgs/boot2_bl602_debug_v8.1.2.bin differ diff --git a/bsp/board/bl602dk/builtin_imgs/boot2_bl602_release_v8.1.1.bin b/bsp/board/bl602dk/builtin_imgs/boot2_bl602_release_v8.1.1.bin deleted file mode 100644 index 430bc4d10..000000000 Binary files a/bsp/board/bl602dk/builtin_imgs/boot2_bl602_release_v8.1.1.bin and /dev/null differ diff --git a/bsp/board/bl602dk/builtin_imgs/boot2_bl602_release_v8.1.2.bin b/bsp/board/bl602dk/builtin_imgs/boot2_bl602_release_v8.1.2.bin new file mode 100644 index 000000000..0156e4b54 Binary files /dev/null and b/bsp/board/bl602dk/builtin_imgs/boot2_bl602_release_v8.1.2.bin differ diff --git a/bsp/board/bl602dk/builtin_imgs/mfg_bl602_gu_567b8d690_40m_autoboot_v2.89.bin b/bsp/board/bl602dk/builtin_imgs/mfg_bl602_gu_2e8b4aa47_40m_autoboot_v2.90.bin similarity index 51% rename from bsp/board/bl602dk/builtin_imgs/mfg_bl602_gu_567b8d690_40m_autoboot_v2.89.bin rename to bsp/board/bl602dk/builtin_imgs/mfg_bl602_gu_2e8b4aa47_40m_autoboot_v2.90.bin index 4cbae15c2..8a31c8a11 100644 Binary files a/bsp/board/bl602dk/builtin_imgs/mfg_bl602_gu_567b8d690_40m_autoboot_v2.89.bin and b/bsp/board/bl602dk/builtin_imgs/mfg_bl602_gu_2e8b4aa47_40m_autoboot_v2.90.bin differ diff --git a/bsp/board/bl602dk/config/mfg_bl602_gu_567b8d690_40m_v2.89.bin b/bsp/board/bl602dk/builtin_imgs/mfg_bl602_gu_2e8b4aa47_40m_v2.90.bin similarity index 51% rename from bsp/board/bl602dk/config/mfg_bl602_gu_567b8d690_40m_v2.89.bin rename to bsp/board/bl602dk/builtin_imgs/mfg_bl602_gu_2e8b4aa47_40m_v2.90.bin index 7026e23c0..fa1bfbe30 100644 Binary files a/bsp/board/bl602dk/config/mfg_bl602_gu_567b8d690_40m_v2.89.bin and b/bsp/board/bl602dk/builtin_imgs/mfg_bl602_gu_2e8b4aa47_40m_v2.90.bin differ diff --git a/bsp/board/bl602dk/config/boot2_bl602_release_v8.1.1.bin b/bsp/board/bl602dk/config/boot2_bl602_release_v8.1.1.bin deleted file mode 100644 index 430bc4d10..000000000 Binary files a/bsp/board/bl602dk/config/boot2_bl602_release_v8.1.1.bin and /dev/null differ diff --git a/bsp/board/bl602dk/config/boot2_bl602_release_v8.1.2.bin b/bsp/board/bl602dk/config/boot2_bl602_release_v8.1.2.bin new file mode 100644 index 000000000..0156e4b54 Binary files /dev/null and b/bsp/board/bl602dk/config/boot2_bl602_release_v8.1.2.bin differ diff --git a/bsp/board/bl602dk/builtin_imgs/mfg_bl602_gu_567b8d690_40m_v2.89.bin b/bsp/board/bl602dk/config/mfg_bl602_gu_2e8b4aa47_40m_v2.90.bin similarity index 51% rename from bsp/board/bl602dk/builtin_imgs/mfg_bl602_gu_567b8d690_40m_v2.89.bin rename to bsp/board/bl602dk/config/mfg_bl602_gu_2e8b4aa47_40m_v2.90.bin index 7026e23c0..fa1bfbe30 100644 Binary files a/bsp/board/bl602dk/builtin_imgs/mfg_bl602_gu_567b8d690_40m_v2.89.bin and b/bsp/board/bl602dk/config/mfg_bl602_gu_2e8b4aa47_40m_v2.90.bin differ diff --git a/bsp/board/bl616dk/CMakeLists.txt b/bsp/board/bl616dk/CMakeLists.txt index 4c514985c..3c916d780 100644 --- a/bsp/board/bl616dk/CMakeLists.txt +++ b/bsp/board/bl616dk/CMakeLists.txt @@ -7,7 +7,14 @@ if(CONFIG_RF) target_sources(app PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/board_rf.c) endif() +if(CONFIG_LINK_IN_PSRAM) +message("LINK IN PSRAM") +sdk_add_compile_definitions(-DCONFIG_PSRAM_COPY_CODE) +sdk_set_linker_script(bl616_chip_psram.ld) +else() +message("LINK IN FLASH") sdk_set_linker_script(bl616_flash.ld) +endif() if(CONFIG_PSRAM) sdk_add_compile_definitions(-DCONFIG_PSRAM) @@ -18,4 +25,4 @@ endif() sdk_add_compile_definitions_ifdef(CONFIG_PSRAM_COPY_CODE -DCONFIG_PSRAM_COPY_CODE) -sdk_add_link_options(-ufw_header) \ No newline at end of file +sdk_add_link_options(-ufw_header) diff --git a/examples/peripherals/sec_eng/sec_eng_aes_sw_key_costtime/bl616_flash.ld b/bsp/board/bl616dk/bl616_chip_psram.ld similarity index 74% rename from examples/peripherals/sec_eng/sec_eng_aes_sw_key_costtime/bl616_flash.ld rename to bsp/board/bl616dk/bl616_chip_psram.ld index ddeb24f3e..903959218 100644 --- a/examples/peripherals/sec_eng/sec_eng_aes_sw_key_costtime/bl616_flash.ld +++ b/bsp/board/bl616dk/bl616_chip_psram.ld @@ -16,24 +16,28 @@ ENTRY(__start) StackSize = 0x1000; /* 4KB */ HeapMinSize = 0x1000; /* 4KB */ -psram_min_size = 0x1000; - +psram_heap_min_size = 0x1000; +nocache_max_size = 20K; __EM_SIZE = DEFINED(btble_controller_init) ? 32K : 0K; +wifi_ram_max_size = DEFINED(wifi_main) ? (160K - __EM_SIZE): 32K; + +remain_wifi_ram = 160K - wifi_ram_max_size - __EM_SIZE; + __RFTLV_SIZE_OFFSET = 1K; __RFTLV_SIZE_HOLE = 2K; -__RFTLV_HEAD1_H = (0x46524C42); /* BLRF */ +__RFTLV_HEAD1_H = (0x46524C42); /* */ __RFTLV_HEAD1_L = (0x41524150); /* PAPA */ MEMORY { fw_header_memory (rx) : ORIGIN = 0xA0000000 - 0x1000, LENGTH = 4K xip_memory (rx) : ORIGIN = 0xA0000000, LENGTH = 4M - ram_psram (wxa) : ORIGIN = 0xA8000000, LENGTH = 4M - itcm_memory (rx) : ORIGIN = 0x62FC0000, LENGTH = 12K + ram_psram (wxa) : ORIGIN = 0xA8000000 + __RFTLV_SIZE_OFFSET, LENGTH = 4M - __RFTLV_SIZE_OFFSET + itcm_memory (rx) : ORIGIN = 0x62FC0400, LENGTH = 11K /* first 1K is for security */ dtcm_memory (rx) : ORIGIN = 0x62FC3000, LENGTH = 4K - nocache_ram_memory (!rx) : ORIGIN = 0x22FC4000, LENGTH = 64K - ram_memory (!rx) : ORIGIN = 0x62FD4000, LENGTH = 320K-12K-4K-64K - ram_wifi (wxa) : ORIGIN = 0x23010000, LENGTH = 160K - __EM_SIZE + nocache_ram_memory (!rx) : ORIGIN = 0x22FC4000, LENGTH = nocache_max_size + ram_memory (!rx) : ORIGIN = 0x62FC4000 + nocache_max_size, LENGTH = 320K - 12K - 4K - nocache_max_size + remain_wifi_ram + ram_wifi (wxa) : ORIGIN = 0x23038000 - wifi_ram_max_size - __EM_SIZE, LENGTH = wifi_ram_max_size } SECTIONS @@ -50,19 +54,29 @@ SECTIONS { KEEP (*(SORT_NONE(.init))) KEEP (*(SORT_NONE(.vector))) - + *system_bl616*.o*(.rodata*) + *system_bl616*.o*(.text*) + *start_load*.o*(.rodata*) + *start_load*.o*(.text*) } > xip_memory - .rftlv.tool : + . = ALIGN(4); + . = ORIGIN(xip_memory) + __RFTLV_SIZE_OFFSET; + __psram_load_addr = .; + .rftlv.tool : AT (__psram_load_addr) { - . = ORIGIN(xip_memory) + __RFTLV_SIZE_OFFSET; + . = ALIGN(4); + __psram_data_start__ = .; PROVIDE( _ld_symbol_rftlv_address = . ); LONG(__RFTLV_HEAD1_H); LONG(__RFTLV_HEAD1_L); - . = ORIGIN(xip_memory) + __RFTLV_SIZE_OFFSET + __RFTLV_SIZE_HOLE; - } > xip_memory + . = __psram_data_start__ + __RFTLV_SIZE_HOLE; + } > ram_psram + . = ORIGIN(xip_memory) + __RFTLV_SIZE_OFFSET + __RFTLV_SIZE_HOLE; + + __rftlv_load_addr = (__psram_load_addr + SIZEOF (.rftlv.tool)); - .text : + .psram_data_region : AT (__rftlv_load_addr) { . = ALIGN(4); __text_code_start__ = .; @@ -76,6 +90,12 @@ SECTIONS KEEP(*(FSymTab)) __fsymtab_end = .; + /* section information for components's version */ + . = ALIGN(4); + _version_info_section_start = .; + KEEP(*(.version.*)) + _version_info_section_end = .; + . = ALIGN(4); __vsymtab_start = .; KEEP(*(VSymTab)) @@ -102,18 +122,47 @@ SECTIONS *(.srodata) *(.srodata.*) + /* _dump_sections for bflb_coredump.c*/ + . = ALIGN(4); + PROVIDE( _dump_sections = ABSOLUTE(.)); + LONG(ORIGIN(itcm_memory)); + LONG(LENGTH(itcm_memory)); + LONG(ORIGIN(dtcm_memory)); + LONG(LENGTH(dtcm_memory)); + LONG(ORIGIN(ram_memory)); + LONG(LENGTH(ram_memory)); + LONG(ORIGIN(nocache_ram_memory)); + LONG(LENGTH(nocache_ram_memory)); + LONG(ORIGIN(ram_wifi)); + LONG(LENGTH(ram_wifi)); + /* + LONG(ORIGIN(ram_psram)); + LONG(LENGTH(ram_psram)); + */ + LONG(0x28010000); /* bt memory */ + LONG(__EM_SIZE); + LONG(0xffffffff); + LONG(0xffffffff); + /* _dump_sections for bl_coredump.c*/ + . = ALIGN(4); __text_code_end__ = .; - } > xip_memory - . = ALIGN(4); - __itcm_load_addr = .; + KEEP(*(.psram_data*)) + + . = ALIGN(4); + __psram_data_end__ = .; + } > ram_psram + + __etext_final = (__rftlv_load_addr + SIZEOF (.psram_data_region)); + ASSERT(__etext_final <= ORIGIN(xip_memory) + LENGTH(xip_memory), "code memory overflow") + + __itcm_load_addr = (__rftlv_load_addr + SIZEOF (.psram_data_region)); .itcm_region : AT (__itcm_load_addr) { . = ALIGN(4); __tcm_code_start__ = .; - *(.tcm_code.*) *(.tcm_const.*) *(.sclock_rlt_code.*) @@ -162,6 +211,8 @@ SECTIONS * size of stack_dummy section */ __StackTop = ORIGIN(dtcm_memory) + LENGTH(dtcm_memory); PROVIDE( __freertos_irq_stack_top = __StackTop); + PROVIDE( _sp_main = __StackTop); + PROVIDE( _sp_base = __StackTop - StackSize); __StackLimit = __StackTop - SIZEOF(.stack_dummy); /* Check if data + heap + stack exceeds RAM limit */ @@ -208,22 +259,6 @@ SECTIONS } > ram_memory - __psram_load_addr = (__ram_load_addr + SIZEOF (RAM_DATA)); - - .psram_data_region : AT (__psram_load_addr) - { - . = ALIGN(4); - __psram_data_start__ = .; - - KEEP(*(.psram_data*)) - - . = ALIGN(4); - __psram_data_end__ = .; - } > ram_psram - - __etext_final = (__psram_load_addr + SIZEOF (.psram_data_region)); - ASSERT(__etext_final <= ORIGIN(xip_memory) + LENGTH(xip_memory), "code memory overflow") - .bss (NOLOAD) : { . = ALIGN(4); @@ -296,8 +331,8 @@ SECTIONS } > ram_psram __psram_limit = ORIGIN(ram_psram) + LENGTH(ram_psram); - ASSERT(__psram_limit - __psram_heap_base >= psram_min_size, "psram heap region overflow") - + ASSERT(__psram_limit - __psram_heap_base >= psram_heap_min_size, "psram heap region overflow") + .wifibss (NOLOAD) : { PROVIDE( __wifi_bss_start = ADDR(.wifibss) ); @@ -322,6 +357,6 @@ SECTIONS PROVIDE( _heap_wifi_start = . ); PROVIDE( _heap_wifi_size = ORIGIN(ram_wifi) + LENGTH(ram_wifi) - _heap_wifi_start ); PROVIDE(__LD_CONFIG_EM_SEL = __EM_SIZE); - + ASSERT((wifi_ram_max_size + __EM_SIZE) <= 160K,"wifi ram overflow") } diff --git a/bsp/board/bl616dk/bl616_flash.ld b/bsp/board/bl616dk/bl616_flash.ld index e756aaf5e..63aa738b6 100644 --- a/bsp/board/bl616dk/bl616_flash.ld +++ b/bsp/board/bl616dk/bl616_flash.ld @@ -17,7 +17,6 @@ ENTRY(__start) StackSize = 0x1000; /* 4KB */ HeapMinSize = 0x1000; /* 4KB */ psram_heap_min_size = 0x1000; -nocache_max_size = 20K; __EM_SIZE = DEFINED(btble_controller_init) ? 32K : 0K; wifi_ram_max_size = DEFINED(wifi_main) ? (160K - __EM_SIZE): 32K; @@ -33,10 +32,8 @@ MEMORY fw_header_memory (rx) : ORIGIN = 0xA0000000 - 0x1000, LENGTH = 4K xip_memory (rx) : ORIGIN = 0xA0000000, LENGTH = 4M ram_psram (wxa) : ORIGIN = 0xA8000000, LENGTH = 4M - itcm_memory (rx) : ORIGIN = 0x62FC0400, LENGTH = 11K /* first 1K is for security */ - dtcm_memory (rx) : ORIGIN = 0x62FC3000, LENGTH = 4K - nocache_ram_memory (!rx) : ORIGIN = 0x22FC4000, LENGTH = nocache_max_size - ram_memory (!rx) : ORIGIN = 0x62FC4000 + nocache_max_size, LENGTH = 320K - 12K - 4K - nocache_max_size + remain_wifi_ram + ram_nocache (!rx) : ORIGIN = 0x22FC0400, LENGTH = 320K - 1K + remain_wifi_ram /* reserved 1k for security */ + ram_memory (!rx) : ORIGIN = 0x62FC0400, LENGTH = 320K - 1K + remain_wifi_ram /* reserved 1k for security */ ram_wifi (wxa) : ORIGIN = 0x23038000 - wifi_ram_max_size - __EM_SIZE, LENGTH = wifi_ram_max_size } @@ -80,6 +77,12 @@ SECTIONS KEEP(*(FSymTab)) __fsymtab_end = .; + /* section information for components's version */ + . = ALIGN(4); + _version_info_section_start = .; + KEEP(*(.version.*)) + _version_info_section_end = .; + . = ALIGN(4); __vsymtab_start = .; KEEP(*(VSymTab)) @@ -109,14 +112,10 @@ SECTIONS /* _dump_sections for bflb_coredump.c*/ . = ALIGN(4); PROVIDE( _dump_sections = ABSOLUTE(.)); - LONG(ORIGIN(itcm_memory)); - LONG(LENGTH(itcm_memory)); - LONG(ORIGIN(dtcm_memory)); - LONG(LENGTH(dtcm_memory)); LONG(ORIGIN(ram_memory)); LONG(LENGTH(ram_memory)); - LONG(ORIGIN(nocache_ram_memory)); - LONG(LENGTH(nocache_ram_memory)); + LONG(ORIGIN(ram_nocache)); + LONG(LENGTH(ram_nocache)); LONG(ORIGIN(ram_wifi)); LONG(LENGTH(ram_wifi)); /* @@ -134,7 +133,38 @@ SECTIONS } > xip_memory . = ALIGN(4); - __itcm_load_addr = .; + __nocache_ram_load_addr = .; + + .nocache_ram_region : AT (__nocache_ram_load_addr) + { + . = ALIGN(4); + __nocache_ram_data_start__ = .; + + *(.nocache_ram) + + . = ALIGN(4); + __nocache_ram_data_end__ = .; + } > ram_nocache + + .nocache_noinit_ram_region (NOLOAD) : + { + . = ALIGN(4); + __nocache_noinit_ram_data_start__ = .; + + *(.nocache_noinit_ram) + *(.noncacheable) + + . = ALIGN(4); + __nocache_noinit_ram_data_end__ = .; + } > ram_nocache + + /* Skip the nocache region */ + .ram_skip_nocache_region (NOLOAD) : + { + . += (__nocache_noinit_ram_data_end__ - ORIGIN(ram_nocache)); + } > ram_memory + + __itcm_load_addr = __nocache_ram_load_addr + SIZEOF(.nocache_ram_region); .itcm_region : AT (__itcm_load_addr) { @@ -158,7 +188,7 @@ SECTIONS . = ALIGN(4); __tcm_code_end__ = .; - } > itcm_memory + } > ram_memory __dtcm_load_addr = __itcm_load_addr + SIZEOF(.itcm_region); @@ -172,46 +202,9 @@ SECTIONS . = ALIGN(4); __tcm_data_end__ = .; - } > dtcm_memory - - /*************************************************************************/ - /* .stack_dummy section doesn't contains any symbols. It is only - * used for linker to calculate size of stack sections, and assign - * values to stack symbols later */ - .stack_dummy (NOLOAD): - { - . = ALIGN(0x4); - . = . + StackSize; - . = ALIGN(0x4); - } > dtcm_memory - - /* Set stack top to end of RAM, and stack limit move down by - * size of stack_dummy section */ - __StackTop = ORIGIN(dtcm_memory) + LENGTH(dtcm_memory); - PROVIDE( __freertos_irq_stack_top = __StackTop); - PROVIDE( _sp_main = __StackTop); - PROVIDE( _sp_base = __StackTop - StackSize); - __StackLimit = __StackTop - SIZEOF(.stack_dummy); - - /* Check if data + heap + stack exceeds RAM limit */ - ASSERT(__StackLimit >= __tcm_data_end__, "region RAM overflowed with stack") - - /*************************************************************************/ - - __nocache_ram_load_addr = __dtcm_load_addr + SIZEOF(.dtcm_region); - - .nocache_ram_region : AT (__nocache_ram_load_addr) - { - . = ALIGN(4); - __nocache_ram_data_start__ = .; - - *(.nocache_ram) - - . = ALIGN(4); - __nocache_ram_data_end__ = .; - } > nocache_ram_memory + } > ram_memory - __ram_load_addr = __nocache_ram_load_addr + SIZEOF(.nocache_ram_region); + __ram_load_addr = __dtcm_load_addr + SIZEOF(.dtcm_region); /* Data section */ RAM_DATA : AT (__ram_load_addr) @@ -228,6 +221,20 @@ SECTIONS *(.sdata2) *(.sdata2.*) + . = ALIGN(8); + *(._k_queue.static.*) + *(._k_sem.static.*) + *(._k_mutex.static.*) + _bt_gatt_service_static_list_start = .; + KEEP(*(SORT_BY_NAME("._bt_gatt_service_static.static.*"))) + _bt_gatt_service_static_list_end = .; + _bt_l2cap_fixed_chan_list_start = .; + KEEP(*(SORT_BY_NAME("._bt_l2cap_fixed_chan.static.*"))) + _bt_l2cap_fixed_chan_list_end = .; + _bt_l2cap_br_fixed_chan_list_start = .; + KEEP(*(SORT_BY_NAME("._bt_l2cap_br_fixed_chan.static.*"))) + _bt_l2cap_br_fixed_chan_list_end = .; + . = ALIGN(4); __bflog_tags_start__ = .; *(.bflog_tags_array) @@ -236,7 +243,6 @@ SECTIONS __ram_data_end__ = .; } > ram_memory - __psram_load_addr = (__ram_load_addr + SIZEOF (RAM_DATA)); .psram_data_region : AT (__psram_load_addr) @@ -253,6 +259,19 @@ SECTIONS __etext_final = (__psram_load_addr + SIZEOF (.psram_data_region)); ASSERT(__etext_final <= ORIGIN(xip_memory) + LENGTH(xip_memory), "code memory overflow") + .stack (NOLOAD): + { + __StackLimit = .; + . = ALIGN(16); + . = . + StackSize; + __StackTop = .; + __freertos_irq_stack_top = .; + } > ram_memory + + PROVIDE( __freertos_irq_stack_top = __StackTop); + PROVIDE( _sp_main = __StackTop); + PROVIDE( _sp_base = __StackLimit); + .bss (NOLOAD) : { . = ALIGN(4); @@ -277,18 +296,6 @@ SECTIONS __noinit_data_end__ = .; } > ram_memory - .nocache_noinit_ram_region (NOLOAD) : - { - . = ALIGN(4); - __nocache_noinit_ram_data_start__ = .; - - *(.nocache_noinit_ram) - *(.noncacheable) - - . = ALIGN(4); - __nocache_noinit_ram_data_end__ = .; - } > nocache_ram_memory - .heap (NOLOAD): { . = ALIGN(4); diff --git a/bsp/board/bl616dk/bl616_ram.ld b/bsp/board/bl616dk/bl616_ram.ld index 7403e4429..28feb57e4 100644 --- a/bsp/board/bl616dk/bl616_ram.ld +++ b/bsp/board/bl616dk/bl616_ram.ld @@ -28,7 +28,7 @@ MEMORY { fw_header_memory (rx) : ORIGIN = 0x62FC0000 - (256+16), LENGTH = 256+16 code_memory (rx) : ORIGIN = 0x62FC0000, LENGTH = 200K - psram (wxa) : ORIGIN = 0xA8000000, LENGTH = 4M + ram_psram (wxa) : ORIGIN = 0xA8000000, LENGTH = 4M itcm_memory (rx) : ORIGIN = 0x62FF2000, LENGTH = 0K dtcm_memory (rx) : ORIGIN = 0x62FF2000, LENGTH = 4K nocache_ram_memory (!rx) : ORIGIN = 0x22FF3000, LENGTH = 60K @@ -78,6 +78,12 @@ SECTIONS KEEP(*(FSymTab)) __fsymtab_end = .; + /* section information for components's version */ + . = ALIGN(4); + _version_info_section_start = .; + KEEP(*(.version.*)) + _version_info_section_end = .; + . = ALIGN(4); __vsymtab_start = .; KEEP(*(VSymTab)) @@ -182,7 +188,20 @@ SECTIONS __ram_data_end__ = .; } > ram_memory - __etext_final = (__ram_load_addr + SIZEOF (RAM_DATA)); + __psram_load_addr = (__ram_load_addr + SIZEOF (RAM_DATA)); + + .psram_data_region : AT (__psram_load_addr) + { + . = ALIGN(4); + __psram_data_start__ = .; + + KEEP(*(.psram_data*)) + + . = ALIGN(4); + __psram_data_end__ = .; + } > ram_psram + + __etext_final = (__psram_load_addr + SIZEOF (.psram_data_region)); ASSERT(__etext_final <= ORIGIN(code_memory) + LENGTH(code_memory), "code memory overflow") .bss (NOLOAD) : @@ -237,21 +256,6 @@ SECTIONS __HeapLimit = ORIGIN(ram_memory) + LENGTH(ram_memory); ASSERT(__HeapLimit - __HeapBase >= HeapMinSize, "heap region overflow") - .psmram_data (NOLOAD): - { - . = ALIGN(4); - __psram_data_start__ = .; - - /*__end__ = .;*/ - /*end = __end__;*/ - KEEP(*(.psram_data*)) - KEEP(*(.psram_noinit*)) - KEEP(*(.img_buf*)) - - . = ALIGN(4); - __psram_data_end__ = .; - } > psram - .psram_heap (NOLOAD): { . = ALIGN(4); @@ -259,9 +263,9 @@ SECTIONS KEEP(*(.psram_heap*)) . = ALIGN(4); __psram_heap_end = .; - } > psram + } > ram_psram - __psram_limit = ORIGIN(psram) + LENGTH(psram); + __psram_limit = ORIGIN(ram_psram) + LENGTH(ram_psram); ASSERT(__psram_limit - __psram_heap_base >= psram_min_size, "psram heap region overflow") .wifibss (NOLOAD) : diff --git a/bsp/board/bl616dk/board.c b/bsp/board/bl616dk/board.c index fa64e1b42..c6fb7e602 100644 --- a/bsp/board/bl616dk/board.c +++ b/bsp/board/bl616dk/board.c @@ -8,6 +8,7 @@ #include "bflb_rtc.h" #include "bflb_flash.h" #include "bflb_acomp.h" +#include "bflb_efuse.h" #include "board.h" #include "bl616_tzc_sec.h" #include "bl616_psram.h" @@ -15,8 +16,6 @@ #include "mem.h" -#define WB_4MB_PSRAM (1) - #ifdef CONFIG_BSP_SDH_SDCARD #include "sdh_sdcard.h" #endif @@ -161,6 +160,40 @@ void bl_show_log(void) printf("Copyright (c) 2022 Bouffalolab team\r\n"); } +static const char* bl_sys_version(const char ***ctx) +{ + extern uint8_t _version_info_section_start; + extern uint8_t _version_info_section_end; + const char ** const version_section_start = (const char**)&_version_info_section_start; + const char ** const version_section_end = (const char**)&_version_info_section_end; + const char *version_str; + + //init + if (NULL == (*ctx)) { + (*ctx) = version_section_start; + } + //check the end + if (version_section_end == (*ctx)) { + return NULL; + } + version_str = (**ctx); + *ctx = (*ctx) + 1; + return version_str; +} + +void bl_show_component_version(void) +{ + const char **ctx = NULL; + const char *version_str; + + puts("Version of used components:\r\n"); + while ((version_str = bl_sys_version(&ctx))) { + puts("\tVersion: "); + puts(version_str); + puts("\r\n"); + } +} + void bl_show_flashinfo(void) { spi_flash_cfg_type flashCfg; @@ -210,7 +243,7 @@ static void console_init() bflb_gpio_uart_init(gpio, GPIO_PIN_21, GPIO_UART_FUNC_UART0_TX); bflb_gpio_uart_init(gpio, GPIO_PIN_22, GPIO_UART_FUNC_UART0_RX); - struct bflb_uart_config_s cfg; + struct bflb_uart_config_s cfg = {0}; cfg.baudrate = 2000000; cfg.data_bits = UART_DATA_BITS_8; cfg.stop_bits = UART_STOP_BITS_1; @@ -218,6 +251,7 @@ static void console_init() cfg.flow_ctrl = 0; cfg.tx_fifo_threshold = 7; cfg.rx_fifo_threshold = 7; + cfg.bit_order = UART_LSB_FIRST; uart0 = bflb_device_get_by_name("uart0"); @@ -233,7 +267,7 @@ void board_init(void) size_t heap_len; flag = bflb_irq_save(); -#ifndef CONFIG_PSRAM_COPY_CODE +#ifndef CONFIG_BOARD_FLASH_INIT_SKIP ret = bflb_flash_init(); #endif system_clock_init(); @@ -242,25 +276,22 @@ void board_init(void) console_init(); -#ifdef CONFIG_PSRAM -#ifndef CONFIG_PSRAM_COPY_CODE - board_psram_x8_init(); - Tzc_Sec_PSRAMB_Access_Release(); -#endif - // extern uint32_t __psram_load_addr; - - // extern uint32_t __psram_data_start__; - // extern uint32_t __psram_data_end__; + bl_show_log(); + bl_show_component_version(); - // uint32_t *pSrc, *pDest; +#ifdef CONFIG_PSRAM + static bflb_efuse_device_info_type device_info; - // /* BF Add psram data copy */ - // pSrc = &__psram_load_addr; - // pDest = &__psram_data_start__; + bflb_efuse_get_device_info(&device_info); + if (device_info.psram_info == 0) { + printf("This chip has no psram, please disable CONFIG_PSRAM\r\n"); + while (1) {} + } - // for (; pDest < &__psram_data_end__;) { - // *pDest++ = *pSrc++; - // } + if (BL616_PSRAM_INIT_DONE == 0) { + board_psram_x8_init(); + Tzc_Sec_PSRAMB_Access_Release(); + } heap_len = ((size_t)&__psram_limit - (size_t)&__psram_heap_base); pmem_init((void *)&__psram_heap_base, heap_len); @@ -269,7 +300,6 @@ void board_init(void) heap_len = ((size_t)&__HeapLimit - (size_t)&__HeapBase); kmem_init((void *)&__HeapBase, heap_len); - bl_show_log(); if (ret != 0) { printf("flash init fail!!!\r\n"); } @@ -378,8 +408,6 @@ void board_adc_gpio_init() bflb_gpio_init(gpio, GPIO_PIN_0, GPIO_ANALOG | GPIO_SMT_EN | GPIO_DRV_0); /* ADC_CH10 */ bflb_gpio_init(gpio, GPIO_PIN_27, GPIO_ANALOG | GPIO_SMT_EN | GPIO_DRV_0); - /* ADC_CH11 */ - bflb_gpio_init(gpio, GPIO_PIN_28, GPIO_ANALOG | GPIO_SMT_EN | GPIO_DRV_0); } void board_dac_gpio_init() @@ -606,8 +634,8 @@ static void reboot_cmd(int argc, char **argv) } SHELL_CMD_EXPORT_ALIAS(reboot_cmd, reboot, reboot); -#define MFG_CONFIG_REG (0x2000F100) -#define MFG_CONFIG_VAL ("0mfg") +#define MFG_CONFIG_REG (0x2000F100) +#define MFG_CONFIG_VAL ("0mfg") void mfg_config(void) { @@ -618,7 +646,7 @@ void mfg_config(void) .byte = MFG_CONFIG_VAL, }; - *(volatile uint32_t*)(MFG_CONFIG_REG) = mfg.word; + *(volatile uint32_t *)(MFG_CONFIG_REG) = mfg.word; } static void mfg_cmd(int argc, char **argv) diff --git a/bsp/board/bl616dk/builtin_imgs/boot2_bl616_isp_debug_v8.1.1.bin b/bsp/board/bl616dk/builtin_imgs/boot2_bl616_isp_debug_v8.1.1.bin deleted file mode 100644 index 924c6bfa4..000000000 Binary files a/bsp/board/bl616dk/builtin_imgs/boot2_bl616_isp_debug_v8.1.1.bin and /dev/null differ diff --git a/bsp/board/bl616dk/builtin_imgs/boot2_bl616_isp_debug_v8.1.2.bin b/bsp/board/bl616dk/builtin_imgs/boot2_bl616_isp_debug_v8.1.2.bin new file mode 100644 index 000000000..322924726 Binary files /dev/null and b/bsp/board/bl616dk/builtin_imgs/boot2_bl616_isp_debug_v8.1.2.bin differ diff --git a/bsp/board/bl616dk/builtin_imgs/boot2_bl616_isp_release_v8.1.1.bin b/bsp/board/bl616dk/builtin_imgs/boot2_bl616_isp_release_v8.1.1.bin deleted file mode 100644 index 121006591..000000000 Binary files a/bsp/board/bl616dk/builtin_imgs/boot2_bl616_isp_release_v8.1.1.bin and /dev/null differ diff --git a/bsp/board/bl616dk/builtin_imgs/boot2_bl616_isp_release_v8.1.2.bin b/bsp/board/bl616dk/builtin_imgs/boot2_bl616_isp_release_v8.1.2.bin new file mode 100644 index 000000000..54974c017 Binary files /dev/null and b/bsp/board/bl616dk/builtin_imgs/boot2_bl616_isp_release_v8.1.2.bin differ diff --git a/bsp/board/bl616dk/builtin_imgs/mfg_bl616_gu_214e807d5_autoboot_v2.43.bin b/bsp/board/bl616dk/builtin_imgs/mfg_bl616_gu_214e807d5_autoboot_v2.43.bin deleted file mode 100644 index b493c7f39..000000000 Binary files a/bsp/board/bl616dk/builtin_imgs/mfg_bl616_gu_214e807d5_autoboot_v2.43.bin and /dev/null differ diff --git a/bsp/board/bl616dk/builtin_imgs/mfg_bl616_gu_214e807d5_v2.43.bin b/bsp/board/bl616dk/builtin_imgs/mfg_bl616_gu_214e807d5_v2.43.bin deleted file mode 100644 index 8dc29a493..000000000 Binary files a/bsp/board/bl616dk/builtin_imgs/mfg_bl616_gu_214e807d5_v2.43.bin and /dev/null differ diff --git a/bsp/board/bl616dk/builtin_imgs/mfg_bl616_gu_34c572ca0_autoboot_v2.49.bin b/bsp/board/bl616dk/builtin_imgs/mfg_bl616_gu_34c572ca0_autoboot_v2.49.bin new file mode 100644 index 000000000..3455d3abd Binary files /dev/null and b/bsp/board/bl616dk/builtin_imgs/mfg_bl616_gu_34c572ca0_autoboot_v2.49.bin differ diff --git a/bsp/board/bl616dk/builtin_imgs/mfg_bl616_gu_34c572ca0_v2.49.bin b/bsp/board/bl616dk/builtin_imgs/mfg_bl616_gu_34c572ca0_v2.49.bin new file mode 100644 index 000000000..267216613 Binary files /dev/null and b/bsp/board/bl616dk/builtin_imgs/mfg_bl616_gu_34c572ca0_v2.49.bin differ diff --git a/bsp/board/bl616dk/config/boot2_bl616_isp_release_v8.1.1.bin b/bsp/board/bl616dk/config/boot2_bl616_isp_release_v8.1.1.bin deleted file mode 100644 index 121006591..000000000 Binary files a/bsp/board/bl616dk/config/boot2_bl616_isp_release_v8.1.1.bin and /dev/null differ diff --git a/bsp/board/bl616dk/config/boot2_bl616_isp_release_v8.1.2.bin b/bsp/board/bl616dk/config/boot2_bl616_isp_release_v8.1.2.bin new file mode 100644 index 000000000..54974c017 Binary files /dev/null and b/bsp/board/bl616dk/config/boot2_bl616_isp_release_v8.1.2.bin differ diff --git a/bsp/board/bl616dk/config/mfg_bl616_gu_214e807d5_v2.43.bin b/bsp/board/bl616dk/config/mfg_bl616_gu_214e807d5_v2.43.bin deleted file mode 100644 index 8dc29a493..000000000 Binary files a/bsp/board/bl616dk/config/mfg_bl616_gu_214e807d5_v2.43.bin and /dev/null differ diff --git a/bsp/board/bl616dk/config/mfg_bl616_gu_34c572ca0_v2.49.bin b/bsp/board/bl616dk/config/mfg_bl616_gu_34c572ca0_v2.49.bin new file mode 100644 index 000000000..267216613 Binary files /dev/null and b/bsp/board/bl616dk/config/mfg_bl616_gu_34c572ca0_v2.49.bin differ diff --git a/bsp/board/bl616dk/fw_header.c b/bsp/board/bl616dk/fw_header.c index 375cec094..0358cbb72 100644 --- a/bsp/board/bl616dk/fw_header.c +++ b/bsp/board/bl616dk/fw_header.c @@ -143,11 +143,11 @@ __attribute__((section(".fw_header"))) struct bootheader_t fw_header = { /* cpu cfg */ .cpu_cfg.config_enable = 0x01, /* coinfig this cpu */ .cpu_cfg.halt_cpu = 0x0, /* halt this cpu */ - .cpu_cfg.cache_enable = 0x0, /* cache setting :only for BL Cache */ - .cpu_cfg.cache_wa = 0x0, /* cache setting :only for BL Cache*/ - .cpu_cfg.cache_wb = 0x0, /* cache setting :only for BL Cache*/ - .cpu_cfg.cache_wt = 0x0, /* cache setting :only for BL Cache*/ - .cpu_cfg.cache_way_dis = 0x0, /* cache setting :only for BL Cache*/ + .cpu_cfg.cache_enable = 0x0, /* cache setting :only for Bouffalo Cache */ + .cpu_cfg.cache_wa = 0x0, /* cache setting :only for Bouffalo Cache*/ + .cpu_cfg.cache_wb = 0x0, /* cache setting :only for Bouffalo Cache*/ + .cpu_cfg.cache_wt = 0x0, /* cache setting :only for Bouffalo Cache*/ + .cpu_cfg.cache_way_dis = 0x0, /* cache setting :only for Bouffalo Cache*/ .cpu_cfg.rsvd = 0x0, /* image_address_offset */ diff --git a/bsp/board/bl702dk/board.c b/bsp/board/bl702dk/board.c index ac4ef32b9..f1a94f1d3 100644 --- a/bsp/board/bl702dk/board.c +++ b/bsp/board/bl702dk/board.c @@ -82,7 +82,6 @@ struct spi_psram_cfg_type ap_memory1604 = { }; struct sf_ctrl_cmds_cfg cmds_cfg = { - .cmds_core_en = 1, .cmds_en = 1, .burst_toggle_en = 1, .cmds_wrap_mode = 0, diff --git a/bsp/board/bl702dk/builtin_imgs/boot2_bl702_debug_v8.1.1.bin b/bsp/board/bl702dk/builtin_imgs/boot2_bl702_debug_v8.1.1.bin deleted file mode 100644 index 8e58e1009..000000000 Binary files a/bsp/board/bl702dk/builtin_imgs/boot2_bl702_debug_v8.1.1.bin and /dev/null differ diff --git a/bsp/board/bl702dk/builtin_imgs/boot2_bl702_debug_v8.1.2.bin b/bsp/board/bl702dk/builtin_imgs/boot2_bl702_debug_v8.1.2.bin new file mode 100644 index 000000000..120e82ca7 Binary files /dev/null and b/bsp/board/bl702dk/builtin_imgs/boot2_bl702_debug_v8.1.2.bin differ diff --git a/bsp/board/bl702dk/builtin_imgs/boot2_bl702_release_v8.1.1.bin b/bsp/board/bl702dk/builtin_imgs/boot2_bl702_release_v8.1.1.bin deleted file mode 100644 index 756d59b30..000000000 Binary files a/bsp/board/bl702dk/builtin_imgs/boot2_bl702_release_v8.1.1.bin and /dev/null differ diff --git a/bsp/board/bl702dk/builtin_imgs/boot2_bl702_release_v8.1.2.bin b/bsp/board/bl702dk/builtin_imgs/boot2_bl702_release_v8.1.2.bin new file mode 100644 index 000000000..3bcb2423b Binary files /dev/null and b/bsp/board/bl702dk/builtin_imgs/boot2_bl702_release_v8.1.2.bin differ diff --git a/bsp/board/bl702dk/builtin_imgs/mfg_bl702_stdfw.bin b/bsp/board/bl702dk/builtin_imgs/mfg_bl702_stdfw.bin index cd374773f..c815ce665 100644 Binary files a/bsp/board/bl702dk/builtin_imgs/mfg_bl702_stdfw.bin and b/bsp/board/bl702dk/builtin_imgs/mfg_bl702_stdfw.bin differ diff --git a/bsp/board/bl702dk/config/boot2_bl702_release_v8.1.1.bin b/bsp/board/bl702dk/config/boot2_bl702_release_v8.1.1.bin deleted file mode 100644 index 756d59b30..000000000 Binary files a/bsp/board/bl702dk/config/boot2_bl702_release_v8.1.1.bin and /dev/null differ diff --git a/bsp/board/bl702dk/config/boot2_bl702_release_v8.1.2.bin b/bsp/board/bl702dk/config/boot2_bl702_release_v8.1.2.bin new file mode 100644 index 000000000..3bcb2423b Binary files /dev/null and b/bsp/board/bl702dk/config/boot2_bl702_release_v8.1.2.bin differ diff --git a/bsp/board/bl808dk/bl808_flash_m0.ld b/bsp/board/bl808dk/bl808_flash_m0.ld index c502beb23..d9054fad9 100644 --- a/bsp/board/bl808dk/bl808_flash_m0.ld +++ b/bsp/board/bl808dk/bl808_flash_m0.ld @@ -16,20 +16,20 @@ ENTRY(__start) StackSize = 0x0400; /* 1KB */ HeapMinSize = 0x1000; /* 4KB */ -psram_min_size = 0x1000; - +psram_heap_min_size = 0x1000; __EM_SIZE = DEFINED(btble_controller_init) ? 32K : 0K; +wifi_ram_max_size = 0K; + +remain_wifi_ram = 160K - wifi_ram_max_size - __EM_SIZE; MEMORY { fw_header_memory (rx) : ORIGIN = 0x58000000 - 0x1000, LENGTH = 4K xip_memory (rx) : ORIGIN = 0x58000000, LENGTH = 32M ram_psram (wxa) : ORIGIN = 0x50000000, LENGTH = 64M - itcm_memory (rx) : ORIGIN = 0x62020000, LENGTH = 20K - dtcm_memory (rx) : ORIGIN = 0x62025000, LENGTH = 4K - nocache_ram_memory (!rx) : ORIGIN = 0x22026000, LENGTH = 16K - ram_memory (!rx) : ORIGIN = 0x6202A000, LENGTH = 24K - ram_wifi (wxa) : ORIGIN = 0x22030000, LENGTH = 160K - __EM_SIZE + ram_nocache (!rx) : ORIGIN = 0x22020400, LENGTH = 64K - 1K + remain_wifi_ram /* reserved 1k for security */ + ram_memory (!rx) : ORIGIN = 0x62020400, LENGTH = 64K - 1K + remain_wifi_ram /* reserved 1k for security */ + ram_wifi (wxa) : ORIGIN = 0x23038000 - wifi_ram_max_size - __EM_SIZE, LENGTH = wifi_ram_max_size xram_memory (!rx) : ORIGIN = 0x40000000, LENGTH = 16K } @@ -94,14 +94,10 @@ SECTIONS /* _dump_sections for bflb_coredump.c*/ . = ALIGN(4); PROVIDE( _dump_sections = ABSOLUTE(.)); - LONG(ORIGIN(itcm_memory)); - LONG(LENGTH(itcm_memory)); - LONG(ORIGIN(dtcm_memory)); - LONG(LENGTH(dtcm_memory)); LONG(ORIGIN(ram_memory)); LONG(LENGTH(ram_memory)); - LONG(ORIGIN(nocache_ram_memory)); - LONG(LENGTH(nocache_ram_memory)); + LONG(ORIGIN(ram_nocache)); + LONG(LENGTH(ram_nocache)); LONG(ORIGIN(ram_wifi)); LONG(LENGTH(ram_wifi)); LONG(ORIGIN(xram_memory)); @@ -121,7 +117,38 @@ SECTIONS } > xip_memory . = ALIGN(4); - __itcm_load_addr = .; + __nocache_ram_load_addr = .; + + .nocache_ram_region : AT (__nocache_ram_load_addr) + { + . = ALIGN(4); + __nocache_ram_data_start__ = .; + + *(.nocache_ram) + + . = ALIGN(4); + __nocache_ram_data_end__ = .; + } > ram_nocache + + .nocache_noinit_ram_region (NOLOAD) : + { + . = ALIGN(4); + __nocache_noinit_ram_data_start__ = .; + + *(.nocache_noinit_ram) + *(.noncacheable) + + . = ALIGN(4); + __nocache_noinit_ram_data_end__ = .; + } > ram_nocache + + /* Skip the nocache region */ + .ram_skip_nocache_region (NOLOAD) : + { + . += (__nocache_noinit_ram_data_end__ - ORIGIN(ram_nocache)); + } > ram_memory + + __itcm_load_addr = __nocache_ram_load_addr + SIZEOF(.nocache_ram_region); .itcm_region : AT (__itcm_load_addr) { @@ -150,7 +177,7 @@ SECTIONS . = ALIGN(4); __tcm_code_end__ = .; - } > itcm_memory + } > ram_memory __dtcm_load_addr = __itcm_load_addr + SIZEOF(.itcm_region); @@ -164,44 +191,9 @@ SECTIONS . = ALIGN(4); __tcm_data_end__ = .; - } > dtcm_memory - - /*************************************************************************/ - /* .stack_dummy section doesn't contains any symbols. It is only - * used for linker to calculate size of stack sections, and assign - * values to stack symbols later */ - .stack_dummy (NOLOAD): - { - . = ALIGN(0x4); - . = . + StackSize; - . = ALIGN(0x4); - } > dtcm_memory - - /* Set stack top to end of RAM, and stack limit move down by - * size of stack_dummy section */ - __StackTop = ORIGIN(dtcm_memory) + LENGTH(dtcm_memory); - PROVIDE( __freertos_irq_stack_top = __StackTop); - PROVIDE( _sp_main = __StackTop); - PROVIDE( _sp_base = __StackTop - StackSize); - __StackLimit = __StackTop - SIZEOF(.stack_dummy); - - /* Check if data + heap + stack exceeds RAM limit */ - ASSERT(__StackLimit >= __tcm_data_end__, "region RAM overflowed with stack") - /*************************************************************************/ - __nocache_ram_load_addr = __dtcm_load_addr + SIZEOF(.dtcm_region); - - .nocache_ram_region : AT (__nocache_ram_load_addr) - { - . = ALIGN(4); - __nocache_ram_data_start__ = .; - - *(.nocache_ram) - - . = ALIGN(4); - __nocache_ram_data_end__ = .; - } > nocache_ram_memory + } > ram_memory - __ram_load_addr = __nocache_ram_load_addr + SIZEOF(.nocache_ram_region); + __ram_load_addr = __dtcm_load_addr + SIZEOF(.nocache_ram_region); /* Data section */ RAM_DATA : AT (__ram_load_addr) @@ -218,6 +210,20 @@ SECTIONS *(.sdata2) *(.sdata2.*) + . = ALIGN(8); + *(._k_queue.static.*) + *(._k_sem.static.*) + *(._k_mutex.static.*) + _bt_gatt_service_static_list_start = .; + KEEP(*(SORT_BY_NAME("._bt_gatt_service_static.static.*"))) + _bt_gatt_service_static_list_end = .; + _bt_l2cap_fixed_chan_list_start = .; + KEEP(*(SORT_BY_NAME("._bt_l2cap_fixed_chan.static.*"))) + _bt_l2cap_fixed_chan_list_end = .; + _bt_l2cap_br_fixed_chan_list_start = .; + KEEP(*(SORT_BY_NAME("._bt_l2cap_br_fixed_chan.static.*"))) + _bt_l2cap_br_fixed_chan_list_end = .; + . = ALIGN(4); __bflog_tags_start__ = .; *(.bflog_tags_array) @@ -242,6 +248,19 @@ SECTIONS __etext_final = (__psram_load_addr + SIZEOF (.psram_data_region)); ASSERT(__etext_final <= ORIGIN(xip_memory) + LENGTH(xip_memory), "code memory overflow") + .stack (NOLOAD): + { + __StackLimit = .; + . = ALIGN(16); + . = . + StackSize; + __StackTop = .; + __freertos_irq_stack_top = .; + } > ram_memory + + PROVIDE( __freertos_irq_stack_top = __StackTop); + PROVIDE( _sp_main = __StackTop); + PROVIDE( _sp_base = __StackLimit); + .bss (NOLOAD) : { . = ALIGN(4); @@ -266,18 +285,6 @@ SECTIONS __noinit_data_end__ = .; } > ram_memory - .nocache_noinit_ram_region (NOLOAD) : - { - . = ALIGN(4); - __nocache_noinit_ram_data_start__ = .; - - *(.nocache_noinit_ram) - *(.noncacheable) - - . = ALIGN(4); - __nocache_noinit_ram_data_end__ = .; - } > nocache_ram_memory - .heap (NOLOAD): { . = ALIGN(4); @@ -315,30 +322,8 @@ SECTIONS } > ram_psram __psram_limit = ORIGIN(ram_psram) + LENGTH(ram_psram); - ASSERT(__psram_limit - __psram_heap_base >= psram_min_size, "psram heap region overflow") + ASSERT(__psram_limit - __psram_heap_base >= psram_heap_min_size, "psram heap region overflow") - .wifibss (NOLOAD) : - { - PROVIDE( __wifi_bss_start = ADDR(.wifibss) ); - PROVIDE( __wifi_bss_end = ADDR(.wifibss) + SIZEOF(.wifibss) ); - *ipc_shared.o(COMMON) - *sdu_shared.o(COMMON) - *hal_desc.o(COMMON) - *txl_buffer_shared.o(COMMON) - *txl_frame_shared.o(COMMON) - *scan_shared.o(COMMON) - *scanu_shared.o(COMMON) - *mfp_bip.o(COMMON) - *me_mic.o(COMMON) - *bl_sta_mgmt_others.o(COMMON) - *bl_pmk_mgmt.o(COMMON) - *bl_pmk_mgmt_internal.o(COMMON) - *libwifi_drv.a:bl_utils.o(COMMON) - *libwifi_drv.a:bl_utils.o(.bss*) - *(.wifi_ram*) - . = ALIGN(16); - } > ram_wifi - PROVIDE(__LD_CONFIG_EM_SEL = __EM_SIZE); } diff --git a/bsp/board/bl808dk/board.c b/bsp/board/bl808dk/board.c index 904d04910..b2bfe91d8 100644 --- a/bsp/board/bl808dk/board.c +++ b/bsp/board/bl808dk/board.c @@ -3,10 +3,10 @@ #include "bflb_clock.h" #include "bflb_rtc.h" #include "bflb_flash.h" +#include "bflb_efuse.h" #include "bl808_glb.h" #include "bl808_psram_uhs.h" #include "bl808_tzc_sec.h" -#include "bl808_ef_cfg.h" #include "bl808_uhs_phy.h" #include "board.h" @@ -107,10 +107,10 @@ int uhs_psram_init(void) }; bflb_efuse_device_info_type chip_info; - bflb_ef_ctrl_get_device_info(&chip_info); - if (chip_info.psramInfo == UHS_32MB_PSRAM) { + bflb_efuse_get_device_info(&chip_info); + if (chip_info.psram_info == UHS_32MB_PSRAM) { psramDefaultCfg.psramMemSize = PSRAM_MEM_SIZE_32MB; - } else if (chip_info.psramInfo == UHS_64MB_PSRAM) { + } else if (chip_info.psram_info == UHS_64MB_PSRAM) { psramDefaultCfg.psramMemSize = PSRAM_MEM_SIZE_64MB; } else { return -1; diff --git a/bsp/board/bl808dk/builtin_imgs/boot2_bl808_debug_v8.1.1.bin b/bsp/board/bl808dk/builtin_imgs/boot2_bl808_debug_v8.1.1.bin deleted file mode 100644 index 9339a98f6..000000000 Binary files a/bsp/board/bl808dk/builtin_imgs/boot2_bl808_debug_v8.1.1.bin and /dev/null differ diff --git a/bsp/board/bl808dk/builtin_imgs/boot2_bl808_debug_v8.1.2.bin b/bsp/board/bl808dk/builtin_imgs/boot2_bl808_debug_v8.1.2.bin new file mode 100644 index 000000000..dbb1549ed Binary files /dev/null and b/bsp/board/bl808dk/builtin_imgs/boot2_bl808_debug_v8.1.2.bin differ diff --git a/bsp/board/bl808dk/builtin_imgs/boot2_bl808_release_v8.1.1.bin b/bsp/board/bl808dk/builtin_imgs/boot2_bl808_release_v8.1.1.bin deleted file mode 100644 index 0f6ae8c72..000000000 Binary files a/bsp/board/bl808dk/builtin_imgs/boot2_bl808_release_v8.1.1.bin and /dev/null differ diff --git a/bsp/board/bl808dk/builtin_imgs/boot2_bl808_release_v8.1.2.bin b/bsp/board/bl808dk/builtin_imgs/boot2_bl808_release_v8.1.2.bin new file mode 100644 index 000000000..8755b17c8 Binary files /dev/null and b/bsp/board/bl808dk/builtin_imgs/boot2_bl808_release_v8.1.2.bin differ diff --git a/bsp/board/bl808dk/config/boot2_bl808_release_v8.1.1.bin b/bsp/board/bl808dk/config/boot2_bl808_release_v8.1.1.bin deleted file mode 100644 index 0f6ae8c72..000000000 Binary files a/bsp/board/bl808dk/config/boot2_bl808_release_v8.1.1.bin and /dev/null differ diff --git a/bsp/board/bl808dk/config/boot2_bl808_release_v8.1.2.bin b/bsp/board/bl808dk/config/boot2_bl808_release_v8.1.2.bin new file mode 100644 index 000000000..8755b17c8 Binary files /dev/null and b/bsp/board/bl808dk/config/boot2_bl808_release_v8.1.2.bin differ diff --git a/bsp/common/ethernet/ethernetif.c b/bsp/common/ethernet/ethernetif.c index 49c3870fb..c5ddf3805 100644 --- a/bsp/common/ethernet/ethernetif.c +++ b/bsp/common/ethernet/ethernetif.c @@ -39,11 +39,12 @@ #include "bflb_common.h" + /* Private typedef -----------------------------------------------------------*/ /* Private define ------------------------------------------------------------*/ /* Network interface name */ -#define IFNAME0 'b' -#define IFNAME1 'l' +#define IFNAME0 'e' +#define IFNAME1 'x' #define ETH_DMA_TRANSMIT_TIMEOUT (20U) @@ -76,10 +77,10 @@ uint8_t DHCP_state = DHCP_OFF; struct bflb_device_s *emac0; struct bflb_emac_phy_cfg_s phy_cfg = { .auto_negotiation = 1, /*!< Speed and mode auto negotiation */ - .full_duplex = 0, /*!< Duplex mode */ - .speed = 0, /*!< Speed mode */ + .full_duplex = 1, /*!< Duplex mode */ + .speed = 100, /*!< Speed mode */ #ifdef PHY_8720 - .phy_address = 1, /*!< PHY address */ + .phy_address = 0x01, /*!< PHY address */ .phy_id = 0x7c0f0, /*!< PHY OUI, masked */ #else #ifdef PHY_8201F @@ -204,7 +205,7 @@ void low_level_init(struct netif *netif) /* phy module init */ ret = ethernet_phy_init(emac0, &phy_cfg); - printf("ETH PHY init ok!\r\n"); + printf("ETH PHY init done! %d\r\n", ret); ethernet_phy_status_get(); if (PHY_STATE_UP == phy_cfg.phy_state) { printf("PHY[%lx] @%d ready on %dMbps, %s duplex\n\r", phy_cfg.phy_id, phy_cfg.phy_address, phy_cfg.speed, phy_cfg.full_duplex ? "full" : "half"); @@ -476,7 +477,7 @@ void ethernet_link_status_updated(struct netif *netif) /* IP address default setting */ ethernet_set_static_ip(netif); uint8_t iptxt[20]; - sprintf((char *)iptxt, "%s", ip4addr_ntoa(netif_ip4_addr(netif))); + snprintf((char *)iptxt, sizeof(iptxt), "%s", ip4addr_ntoa(netif_ip4_addr(netif))); printf("Static IP address: %s\r\n", iptxt); #endif } else { @@ -498,47 +499,45 @@ void ethernet_link_check_state(struct netif *netif) emac_phy_status_t phy_state; uint32_t linkchanged = 0; - // uint32_t speed = 0, duplex = 0; + uint32_t speed = 0, duplex = 0; phy_state = ethernet_phy_status_get(); if (netif_is_link_up(netif) && (phy_state <= EMAC_PHY_STAT_LINK_DOWN)) { - printf("Link Down\r\n"); - bflb_emac_stop(emac0); + printf("Link Down!\r\n"); + // bflb_emac_stop(emac0); netif_set_down(netif); netif_set_link_down(netif); - } else if (!netif_is_link_up(netif) && (phy_state <= EMAC_PHY_STAT_LINK_DOWN)) { - printf("Reinit\r\n"); - ethernet_phy_init(emac0, &phy_cfg); } else if (!netif_is_link_up(netif) && (phy_state > EMAC_PHY_STAT_LINK_UP)) { - // switch (phy_state) { - // case EMAC_PHY_STAT_100MBITS_FULLDUPLEX: - // duplex = 1; - // speed = 100; - // linkchanged = 1; - // break; - - // case EMAC_PHY_STAT_100MBITS_HALFDUPLEX: - // duplex = 0; - // speed = 100; - // linkchanged = 1; - // break; - - // case EMAC_PHY_STAT_10MBITS_FULLDUPLEX: - // duplex = 1; - // speed = 10; - // linkchanged = 1; - // break; - - // case EMAC_PHY_STAT_10MBITS_HALFDUPLEX: - // duplex = 0; - // speed = 10; - // linkchanged = 1; - // break; - - // default: - // break; - // } + printf("Link Up!\r\n"); + switch (phy_state) { + case EMAC_PHY_STAT_100MBITS_FULLDUPLEX: + duplex = 1; + speed = 100; + linkchanged = 1; + break; + + case EMAC_PHY_STAT_100MBITS_HALFDUPLEX: + duplex = 0; + speed = 100; + linkchanged = 1; + break; + + case EMAC_PHY_STAT_10MBITS_FULLDUPLEX: + duplex = 1; + speed = 10; + linkchanged = 1; + break; + + case EMAC_PHY_STAT_10MBITS_HALFDUPLEX: + duplex = 0; + speed = 10; + linkchanged = 1; + break; + + default: + break; + } if (linkchanged) { /* Get MAC Config MAC */ @@ -565,6 +564,11 @@ void dhcp_thread(void const *argument) for (;;) { switch (DHCP_state) { + case DHCP_OFF: { + printf("DHCP OFF!\r\n"); + vTaskDelay(1000); + } + break; case DHCP_START: { ip_addr_set_zero_ip4(&netif->ip_addr); ip_addr_set_zero_ip4(&netif->netmask); @@ -576,7 +580,7 @@ void dhcp_thread(void const *argument) case DHCP_WAIT_ADDRESS: { if (dhcp_supplied_address(netif)) { DHCP_state = DHCP_ADDRESS_ASSIGNED; - sprintf((char *)iptxt, "%s", ip4addr_ntoa(netif_ip4_addr(netif))); + snprintf((char *)iptxt, sizeof(iptxt), "%s", ip4addr_ntoa(netif_ip4_addr(netif))); printf("IP address assigned by a DHCP server: %s\r\n", iptxt); } else { dhcp = (struct dhcp *)netif_get_client_data(netif, LWIP_NETIF_CLIENT_DATA_INDEX_DHCP); @@ -587,7 +591,7 @@ void dhcp_thread(void const *argument) /* Static address used */ ethernet_set_static_ip(netif); - sprintf((char *)iptxt, "%s", ip4addr_ntoa(netif_ip4_addr(netif))); + snprintf((char *)iptxt, sizeof(iptxt), "%s", ip4addr_ntoa(netif_ip4_addr(netif))); printf("DHCP Timeout !! \r\n"); printf("Static IP address: %s\r\n", iptxt); } @@ -596,6 +600,11 @@ void dhcp_thread(void const *argument) case DHCP_ADDRESS_ASSIGNED: { netif->state = DHCP_ADDRESS_ASSIGNED; } break; + case DHCP_TIMEOUT: { + printf("DHCP TIMEOUT!\r\n"); + netif->state = DHCP_LINK_DOWN; + } + break; case DHCP_LINK_DOWN: { DHCP_state = DHCP_OFF; printf("The network cable is not connected \r\n"); diff --git a/bsp/common/ethernet/phy_8720.c b/bsp/common/ethernet/phy_8720.c index 8896c04a6..48729aaf9 100644 --- a/bsp/common/ethernet/phy_8720.c +++ b/bsp/common/ethernet/phy_8720.c @@ -339,8 +339,20 @@ int phy_8720_init(struct bflb_device_s *emac, struct bflb_emac_phy_cfg_s *cfg) */ emac_phy_status_t phy_8720_status_get(void) { + uint16_t phy_BSR; CHECK_PARAM(NULL != phy_8720_cfg); + if (0 != bflb_emac_phy_reg_read(emac_dev, PHY_8720_BSR, &phy_BSR)) { + return -1; + } + + // printf("read BSR 0x01 =%x\r\n", phy_BSR); + if (phy_BSR & 0x4) { + phy_8720_cfg->phy_state = PHY_STATE_UP; + } else { + phy_8720_cfg->phy_state = PHY_STATE_DOWN; + } + if ((100 == phy_8720_cfg->speed) && (phy_8720_cfg->full_duplex) && (PHY_STATE_UP == phy_8720_cfg->phy_state)) { diff --git a/bsp/common/ethernet/phy_8720.h b/bsp/common/ethernet/phy_8720.h index 7515ee15d..e982333a8 100644 --- a/bsp/common/ethernet/phy_8720.h +++ b/bsp/common/ethernet/phy_8720.h @@ -61,6 +61,21 @@ #define PHY_8720_ADVERTISE_8023 (1 << 0) #define PHY_8720_LPA ((uint16_t)0x05U) /*!< Auto-negotiation link partner base page ability */ #define PHY_8720_EXPANSION ((uint16_t)0x06U) /*!< Auto-negotiation expansion */ + +// Special Modes Register +#define PHY_8720_SMR ((uint16_t)0x12U) /*!< Special Modes Register */ +#define PHY_8720_SMR_MODE_OFFSET (5) /*!< MODE 0x000 10H; 0x001 10F; 0x010 100H; 0x011 100F; 0x100 100HA; 0x101 100HAR; 0x110 PDM; 0x111 all capable Auto-negotiation */ +#define PHY_8720_SMR_MODE_MASK (0x7 << PHY_8720_SMR_MODE_OFFSET) +#define PHY_8720_SMR_MODE_10H_NA (0x0 << PHY_8720_SMR_MODE_OFFSET) +#define PHY_8720_SMR_MODE_10F_NA (0x1 << PHY_8720_SMR_MODE_OFFSET) +#define PHY_8720_SMR_MODE_100H_NA (0x2 << PHY_8720_SMR_MODE_OFFSET) +#define PHY_8720_SMR_MODE_100F_NA (0x3 << PHY_8720_SMR_MODE_OFFSET) +#define PHY_8720_SMR_MODE_100H_AO (0x4 << PHY_8720_SMR_MODE_OFFSET) +#define PHY_8720_SMR_MODE_100HR_AO (0x5 << PHY_8720_SMR_MODE_OFFSET) +#define PHY_8720_SMR_MODE_PDM (0x6 << PHY_8720_SMR_MODE_OFFSET) +#define PHY_8720_SMR_MODE_ALL_CAPABLE (0x7 << PHY_8720_SMR_MODE_OFFSET) +#define PHY_8720_SMR_PHYAD_OFFSET (0x0) + /* Section 4: Extended PHY Registers */ #define PHY_8720_CTRL_STATUS ((uint16_t)17U) /*!< PHY MODE CONTROL/STATUS REGISTER Offset */ #define PHY_8720_CTRL_STATUS_EDPWRDOWN (1 << 13) /*!< EDPWRDOWN */ diff --git a/bsp/common/lcd/spi/st7789v_spi.c b/bsp/common/lcd/spi/st7789v_spi.c index 5a9e0ba5c..c7eaf6cf7 100644 --- a/bsp/common/lcd/spi/st7789v_spi.c +++ b/bsp/common/lcd/spi/st7789v_spi.c @@ -1,288 +1,288 @@ -/** - * @file st7789v_spi.c - * @brief - * - * Copyright (c) 2021 Bouffalolab team - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The - * ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - */ - -#include "../lcd.h" - -#if defined(LCD_SPI_ST7789V) - -#if (LCD_SPI_INTERFACE_TYPE == 1) -#include "bl_spi_hard_4.h" - -#define lcd_spi_init lcd_spi_hard_4_init -#define lcd_spi_isbusy lcd_spi_hard_4_is_busy - -#define lcd_spi_transmit_cmd_para lcd_spi_hard_4_transmit_cmd_para -#define lcd_spi_transmit_cmd_pixel_sync lcd_spi_hard_4_transmit_cmd_pixel_sync -#define lcd_spi_transmit_cmd_pixel_fill_sync lcd_spi_hard_4_transmit_cmd_pixel_fill_sync - -#define lcd_spi_sync_callback_enable lcd_spi_hard_4_async_callback_enable -#define lcd_spi_async_callback_register lcd_spi_hard_4_async_callback_register -#define lcd_spi_transmit_cmd_pixel_async lcd_spi_hard_4_transmit_cmd_pixel_async -#define lcd_spi_transmit_cmd_pixel_fill_async lcd_spi_hard_4_transmit_cmd_pixel_fill_async - -static lcd_spi_hard_4_init_t spi_para = { - .clock_freq = 40 * 1000 * 1000, -#if (ST7789V_SPI_PIXEL_FORMAT == 1) - .pixel_format = LCD_SPI_LCD_PIXEL_FORMAT_RGB565, -#elif (ST7789V_SPI_PIXEL_FORMAT == 2) - .pixel_format = LCD_SPI_LCD_PIXEL_FORMAT_NRGB8888, -#endif -}; - -#else - -#error "Configuration error" - -#endif -const st7789v_spi_init_cmd_t st7789v_spi_init_cmds[] = { - { 0x01, NULL, 0 }, - { 0xFF, NULL, 10 }, - { 0x11, NULL, 0 }, /* Exit sleep */ - { 0xFF, NULL, 120 }, - - { 0x3A, "\x05", 1 }, - - { 0xB2, "\x0C\x0C\x00\x33\x33", 5 }, - { 0xB7, "\x35", 1 }, - { 0xBB, "\x32", 1 }, - { 0xC2, "\x01", 1 }, - { 0xC3, "\x15", 1 }, - { 0xC4, "\x20", 1 }, - { 0xC6, "\x0F", 1 }, - { 0xD0, "\xA4\xA1", 2 }, - { 0xE0, "\xD0\x0A\x10\x0C\x0C\x18\x35\x43\x4D\x39\x13\x13\x2D\x34", 14 }, - { 0xE1, "\xD0\x05\x0B\x06\x05\x02\x35\x43\x4D\x16\x15\x15\x2E\x32", 14 }, - - { 0x21, NULL, 0 }, - { 0x29, NULL, 0 }, - { 0x36, "\x00", 1 }, -}; - -/** - * @brief st7789v_spi_async_callback_enable - * - * @return - */ -void st7789v_spi_async_callback_enable(bool enable) -{ - lcd_spi_sync_callback_enable(enable); -} - -/** - * @brief st7789v_spi_async_callback_register - * - * @return - */ -void st7789v_spi_async_callback_register(void (*callback)(void)) -{ - lcd_spi_async_callback_register(callback); -} - -/** - * @brief st7789v_spi_draw_is_busy, After the call st7789v_spi_draw_picture_dma must check this, - * if st7789v_spi_draw_is_busy() == 1, Don't allow other draw !! - * can run in the DMA interrupt callback function. - * - * @return int 0:draw end; 1:Being draw - */ -int st7789v_spi_draw_is_busy(void) -{ - return lcd_spi_isbusy(); -} - -/** - * @brief st7789v_spi_init - * - * @return int - */ -int st7789v_spi_init() -{ - lcd_spi_init(&spi_para); - - for (uint16_t i = 0; i < (sizeof(st7789v_spi_init_cmds) / sizeof(st7789v_spi_init_cmds[0])); i++) { - if (st7789v_spi_init_cmds[i].cmd == 0xFF && st7789v_spi_init_cmds[i].data == NULL && st7789v_spi_init_cmds[i].databytes) { - bflb_mtimer_delay_ms(st7789v_spi_init_cmds[i].databytes); - } else { - lcd_spi_transmit_cmd_para(st7789v_spi_init_cmds[i].cmd, (void *)(st7789v_spi_init_cmds[i].data), st7789v_spi_init_cmds[i].databytes); - } - } - - return 0; -} - -/** - * @brief - * - * @param dir - * @param mir_flag - */ -int st7789v_spi_set_dir(uint8_t dir, uint8_t mir_flag) -{ - uint8_t param; - - switch (dir) { - case 0: - if (!mir_flag) - param = 0x00; - else - param = 0x40; - break; - case 1: - if (!mir_flag) - param = 0x20; - else - param = 0xA0; - break; - case 2: - if (!mir_flag) - param = 0x80; - else - param = 0xC0; - break; - case 3: - if (!mir_flag) - param = 0xE0; - else - param = 0x60; - - break; - default: - return -1; - break; - } - - lcd_spi_transmit_cmd_para(0x36, (void *)¶m, 1); - - return dir; -} - -/** - * @brief st7789v_spi_set_draw_window - * - * @param x1 - * @param y1 - * @param x2 - * @param y2 - */ -void st7789v_spi_set_draw_window(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2) -{ -#if ST7789V_SPI_OFFSET_X - x1 += ST7789V_SPI_OFFSET_X; - x2 += ST7789V_SPI_OFFSET_X; -#endif -#if ST7789V_SPI_OFFSET_Y - y1 += ST7789V_SPI_OFFSET_Y; - y2 += ST7789V_SPI_OFFSET_Y; -#endif - - int8_t param[4]; - - param[0] = (x1 >> 8) & 0xFF; - param[1] = x1 & 0xFF; - param[2] = (x2 >> 8) & 0xFF; - param[3] = x2 & 0xFF; - - lcd_spi_transmit_cmd_para(0x2A, (void *)param, 4); - - param[0] = (y1 >> 8) & 0xFF; - param[1] = y1 & 0xFF; - param[2] = (y2 >> 8) & 0xFF; - param[3] = y2 & 0xFF; - - lcd_spi_transmit_cmd_para(0x2B, (void *)param, 4); -} - -/** - * @brief st7789v_spi_draw_point - * - * @param x - * @param y - * @param color - */ -void st7789v_spi_draw_point(uint16_t x, uint16_t y, st7789v_spi_color_t color) -{ - /* set window */ - st7789v_spi_set_draw_window(x, y, x, y); - - lcd_spi_transmit_cmd_pixel_sync(0x2C, (void *)&color, 1); -} - -/** - * @brief st7789v_draw_area - * - * @param x1 - * @param y1 - * @param x2 - * @param y2 - * @param color - */ -void st7789v_spi_draw_area(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2, st7789v_spi_color_t color) -{ - uint32_t pixel_cnt = (x2 - x1 + 1) * (y2 - y1 + 1); - - /* set window */ - st7789v_spi_set_draw_window(x1, y1, x2, y2); - - lcd_spi_transmit_cmd_pixel_fill_sync(0x2C, (uint32_t)color, pixel_cnt); -} - -/** - * @brief st7789v_draw_picture_dma, Non-blocking! Using DMA acceleration, Not waiting for the draw end - * After the call, No other operations are allowed until (st7789v_draw_is_busy()==0) - * - * @param x1 - * @param y1 - * @param x2 - * @param y2 - * @param picture - */ -void st7789v_spi_draw_picture_nonblocking(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2, st7789v_spi_color_t *picture) -{ - size_t pixel_cnt = (x2 - x1 + 1) * (y2 - y1 + 1); - - /* set window */ - st7789v_spi_set_draw_window(x1, y1, x2, y2); - - lcd_spi_transmit_cmd_pixel_async(0x2C, (void *)picture, pixel_cnt); -} - -/** - * @brief st7789v_draw_picture,Blocking,Using DMA acceleration,Waiting for the draw end - * - * @param x1 - * @param y1 - * @param x2 - * @param y2 - * @param picture - */ -void st7789v_spi_draw_picture_blocking(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2, st7789v_spi_color_t *picture) -{ - size_t pixel_cnt = (x2 - x1 + 1) * (y2 - y1 + 1); - - /* set window */ - st7789v_spi_set_draw_window(x1, y1, x2, y2); - - lcd_spi_transmit_cmd_pixel_sync(0x2C, (void *)picture, pixel_cnt); -} - -#endif +/** + * @file st7789v_spi.c + * @brief + * + * Copyright (c) 2021 Bouffalolab team + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + */ + +#include "../lcd.h" + +#if defined(LCD_SPI_ST7789V) + +#if (LCD_SPI_INTERFACE_TYPE == 1) +#include "bl_spi_hard_4.h" + +#define lcd_spi_init lcd_spi_hard_4_init +#define lcd_spi_isbusy lcd_spi_hard_4_is_busy + +#define lcd_spi_transmit_cmd_para lcd_spi_hard_4_transmit_cmd_para +#define lcd_spi_transmit_cmd_pixel_sync lcd_spi_hard_4_transmit_cmd_pixel_sync +#define lcd_spi_transmit_cmd_pixel_fill_sync lcd_spi_hard_4_transmit_cmd_pixel_fill_sync + +#define lcd_spi_sync_callback_enable lcd_spi_hard_4_async_callback_enable +#define lcd_spi_async_callback_register lcd_spi_hard_4_async_callback_register +#define lcd_spi_transmit_cmd_pixel_async lcd_spi_hard_4_transmit_cmd_pixel_async +#define lcd_spi_transmit_cmd_pixel_fill_async lcd_spi_hard_4_transmit_cmd_pixel_fill_async + +static lcd_spi_hard_4_init_t spi_para = { + .clock_freq = 40 * 1000 * 1000, +#if (ST7789V_SPI_PIXEL_FORMAT == 1) + .pixel_format = LCD_SPI_LCD_PIXEL_FORMAT_RGB565, +#elif (ST7789V_SPI_PIXEL_FORMAT == 2) + .pixel_format = LCD_SPI_LCD_PIXEL_FORMAT_NRGB8888, +#endif +}; + +#else + +#error "Configuration error" + +#endif +const st7789v_spi_init_cmd_t st7789v_spi_init_cmds[] = { + { 0x01, NULL, 0 }, + { 0xFF, NULL, 10 }, + { 0x11, NULL, 0 }, /* Exit sleep */ + { 0xFF, NULL, 120 }, + + { 0x3A, "\x05", 1 }, + + { 0xB2, "\x0C\x0C\x00\x33\x33", 5 }, + { 0xB7, "\x35", 1 }, + { 0xBB, "\x32", 1 }, + { 0xC2, "\x01", 1 }, + { 0xC3, "\x15", 1 }, + { 0xC4, "\x20", 1 }, + { 0xC6, "\x0F", 1 }, + { 0xD0, "\xA4\xA1", 2 }, + { 0xE0, "\xD0\x0A\x10\x0C\x0C\x18\x35\x43\x4D\x39\x13\x13\x2D\x34", 14 }, + { 0xE1, "\xD0\x05\x0B\x06\x05\x02\x35\x43\x4D\x16\x15\x15\x2E\x32", 14 }, + + { 0x21, NULL, 0 }, + { 0x29, NULL, 0 }, + { 0x36, "\x00", 1 }, +}; + +/** + * @brief st7789v_spi_async_callback_enable + * + * @return + */ +void st7789v_spi_async_callback_enable(bool enable) +{ + lcd_spi_sync_callback_enable(enable); +} + +/** + * @brief st7789v_spi_async_callback_register + * + * @return + */ +void st7789v_spi_async_callback_register(void (*callback)(void)) +{ + lcd_spi_async_callback_register(callback); +} + +/** + * @brief st7789v_spi_draw_is_busy, After the call st7789v_spi_draw_picture_dma must check this, + * if st7789v_spi_draw_is_busy() == 1, Don't allow other draw !! + * can run in the DMA interrupt callback function. + * + * @return int 0:draw end; 1:Being draw + */ +int st7789v_spi_draw_is_busy(void) +{ + return lcd_spi_isbusy(); +} + +/** + * @brief st7789v_spi_init + * + * @return int + */ +int st7789v_spi_init() +{ + lcd_spi_init(&spi_para); + + for (uint16_t i = 0; i < (sizeof(st7789v_spi_init_cmds) / sizeof(st7789v_spi_init_cmds[0])); i++) { + if (st7789v_spi_init_cmds[i].cmd == 0xFF && st7789v_spi_init_cmds[i].data == NULL && st7789v_spi_init_cmds[i].databytes) { + bflb_mtimer_delay_ms(st7789v_spi_init_cmds[i].databytes); + } else { + lcd_spi_transmit_cmd_para(st7789v_spi_init_cmds[i].cmd, (void *)(st7789v_spi_init_cmds[i].data), st7789v_spi_init_cmds[i].databytes); + } + } + + return 0; +} + +/** + * @brief + * + * @param dir + * @param mir_flag + */ +int st7789v_spi_set_dir(uint8_t dir, uint8_t mir_flag) +{ + uint8_t param; + + switch (dir) { + case 0: + if (!mir_flag) + param = 0x00; + else + param = 0x40; + break; + case 1: + if (!mir_flag) + param = 0x20; + else + param = 0xA0; + break; + case 2: + if (!mir_flag) + param = 0x80; + else + param = 0xC0; + break; + case 3: + if (!mir_flag) + param = 0xE0; + else + param = 0x60; + + break; + default: + return -1; + break; + } + + lcd_spi_transmit_cmd_para(0x36, (void *)¶m, 1); + + return dir; +} + +/** + * @brief st7789v_spi_set_draw_window + * + * @param x1 + * @param y1 + * @param x2 + * @param y2 + */ +void st7789v_spi_set_draw_window(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2) +{ +#if ST7789V_SPI_OFFSET_X + x1 += ST7789V_SPI_OFFSET_X; + x2 += ST7789V_SPI_OFFSET_X; +#endif +#if ST7789V_SPI_OFFSET_Y + y1 += ST7789V_SPI_OFFSET_Y; + y2 += ST7789V_SPI_OFFSET_Y; +#endif + + int8_t param[4]; + + param[0] = (x1 >> 8) & 0xFF; + param[1] = x1 & 0xFF; + param[2] = (x2 >> 8) & 0xFF; + param[3] = x2 & 0xFF; + + lcd_spi_transmit_cmd_para(0x2A, (void *)param, 4); + + param[0] = (y1 >> 8) & 0xFF; + param[1] = y1 & 0xFF; + param[2] = (y2 >> 8) & 0xFF; + param[3] = y2 & 0xFF; + + lcd_spi_transmit_cmd_para(0x2B, (void *)param, 4); +} + +/** + * @brief st7789v_spi_draw_point + * + * @param x + * @param y + * @param color + */ +void st7789v_spi_draw_point(uint16_t x, uint16_t y, st7789v_spi_color_t color) +{ + /* set window */ + st7789v_spi_set_draw_window(x, y, x, y); + + lcd_spi_transmit_cmd_pixel_sync(0x2C, (void *)&color, 1); +} + +/** + * @brief st7789v_draw_area + * + * @param x1 + * @param y1 + * @param x2 + * @param y2 + * @param color + */ +void st7789v_spi_draw_area(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2, st7789v_spi_color_t color) +{ + uint32_t pixel_cnt = (x2 - x1 + 1) * (y2 - y1 + 1); + + /* set window */ + st7789v_spi_set_draw_window(x1, y1, x2, y2); + + lcd_spi_transmit_cmd_pixel_fill_sync(0x2C, (uint32_t)color, pixel_cnt); +} + +/** + * @brief st7789v_draw_picture_dma, Non-blocking! Using DMA acceleration, Not waiting for the draw end + * After the call, No other operations are allowed until (st7789v_draw_is_busy()==0) + * + * @param x1 + * @param y1 + * @param x2 + * @param y2 + * @param picture + */ +void st7789v_spi_draw_picture_nonblocking(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2, st7789v_spi_color_t *picture) +{ + size_t pixel_cnt = (x2 - x1 + 1) * (y2 - y1 + 1); + + /* set window */ + st7789v_spi_set_draw_window(x1, y1, x2, y2); + + lcd_spi_transmit_cmd_pixel_async(0x2C, (void *)picture, pixel_cnt); +} + +/** + * @brief st7789v_draw_picture,Blocking,Using DMA acceleration,Waiting for the draw end + * + * @param x1 + * @param y1 + * @param x2 + * @param y2 + * @param picture + */ +void st7789v_spi_draw_picture_blocking(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2, st7789v_spi_color_t *picture) +{ + size_t pixel_cnt = (x2 - x1 + 1) * (y2 - y1 + 1); + + /* set window */ + st7789v_spi_set_draw_window(x1, y1, x2, y2); + + lcd_spi_transmit_cmd_pixel_sync(0x2C, (void *)picture, pixel_cnt); +} + +#endif diff --git a/bsp/common/lcd/spi/st7789v_spi.h b/bsp/common/lcd/spi/st7789v_spi.h index 2aef4ed57..8a714531a 100644 --- a/bsp/common/lcd/spi/st7789v_spi.h +++ b/bsp/common/lcd/spi/st7789v_spi.h @@ -1,55 +1,55 @@ -/** - * @file st7789v.h - * @brief - * - * Copyright (c) 2021 Bouffalolab team - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The - * ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - */ - -#ifndef _ST7789V_SPI_H_ -#define _ST7789V_SPI_H_ - -#include "../lcd_conf.h" - -#if defined LCD_SPI_ST7789V - -/* Do not modify the following */ - -#define ST7789V_SPI_COLOR_DEPTH 16 - -typedef struct { - uint8_t cmd; /* 0xFF : delay(databytes)ms */ - const char *data; - uint8_t databytes; /* Num of data in data; or delay time */ -} st7789v_spi_init_cmd_t; - -typedef uint16_t st7789v_spi_color_t; - -int st7789v_spi_init(); -void st7789v_spi_async_callback_enable(bool enable); -void st7789v_spi_async_callback_register(void (*callback)(void)); -int st7789v_spi_set_dir(uint8_t dir, uint8_t mir_flag); -void st7789v_spi_set_draw_window(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2); -void st7789v_spi_draw_point(uint16_t x, uint16_t y, st7789v_spi_color_t color); -void st7789v_spi_draw_area(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2, st7789v_spi_color_t color); -void st7789v_spi_draw_picture_nonblocking(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2, st7789v_spi_color_t *picture); -void st7789v_spi_draw_picture_blocking(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2, st7789v_spi_color_t *picture); -int st7789v_spi_draw_is_busy(void); - -#endif +/** + * @file st7789v.h + * @brief + * + * Copyright (c) 2021 Bouffalolab team + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef _ST7789V_SPI_H_ +#define _ST7789V_SPI_H_ + +#include "../lcd_conf.h" + +#if defined LCD_SPI_ST7789V + +/* Do not modify the following */ + +#define ST7789V_SPI_COLOR_DEPTH 16 + +typedef struct { + uint8_t cmd; /* 0xFF : delay(databytes)ms */ + const char *data; + uint8_t databytes; /* Num of data in data; or delay time */ +} st7789v_spi_init_cmd_t; + +typedef uint16_t st7789v_spi_color_t; + +int st7789v_spi_init(); +void st7789v_spi_async_callback_enable(bool enable); +void st7789v_spi_async_callback_register(void (*callback)(void)); +int st7789v_spi_set_dir(uint8_t dir, uint8_t mir_flag); +void st7789v_spi_set_draw_window(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2); +void st7789v_spi_draw_point(uint16_t x, uint16_t y, st7789v_spi_color_t color); +void st7789v_spi_draw_area(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2, st7789v_spi_color_t color); +void st7789v_spi_draw_picture_nonblocking(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2, st7789v_spi_color_t *picture); +void st7789v_spi_draw_picture_blocking(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2, st7789v_spi_color_t *picture); +int st7789v_spi_draw_is_busy(void); + +#endif #endif \ No newline at end of file diff --git a/bsp/common/sdcard/sdh_sdcard.c b/bsp/common/sdcard/sdh_sdcard.c index 98f188f00..5cb1c4e75 100644 --- a/bsp/common/sdcard/sdh_sdcard.c +++ b/bsp/common/sdcard/sdh_sdcard.c @@ -63,7 +63,7 @@ static SemaphoreHandle_t SDH_DataWaitSem = NULL; static SDH_DMA_Cfg_Type SDH_DMA_Cfg_TypeInstance; /*causion: ADMA related variables must on OCRAM or shared ram*/ -static __attribute__((aligned(32), section(".noncacheable"))) SDH_ADMA2_Desc_Type adma2Entries[16]; +static __ALIGNED(32) ATTR_NOCACHE_RAM_SECTION SDH_ADMA2_Desc_Type adma2Entries[32]; /* Private function prototypes -----------------------------------------------*/ static void SD_DecodeCid(sd_card_t *card, uint32_t *rawCid); @@ -1485,7 +1485,13 @@ status_t SDH_ReadMultiBlocks(uint8_t *readbuff, uint32_t ReadAddr, uint16_t Bloc SDH_DMA_Cfg_TypeInstance.admaEntries = (uint32_t *)adma2Entries; SDH_DMA_Cfg_TypeInstance.maxEntries = sizeof(adma2Entries) / sizeof(adma2Entries[0]); +#if 0 + bflb_l1c_dcache_clean_range((void *)(readbuff), 1); + bflb_l1c_dcache_clean_range((void *)(readbuff + BlockSize * NumberOfBlocks), 1); bflb_l1c_dcache_invalidate_range((void *)(readbuff), BlockSize * NumberOfBlocks); +#else + bflb_l1c_dcache_clean_range((void *)(readbuff), BlockSize * NumberOfBlocks); +#endif errorstatus = SDH_CardTransferNonBlocking(&SDH_DMA_Cfg_TypeInstance, &SDH_Trans_Cfg_TypeInstance); diff --git a/bsp/sdk_version.c b/bsp/sdk_version.c new file mode 100644 index 000000000..9ee6892af --- /dev/null +++ b/bsp/sdk_version.c @@ -0,0 +1,3 @@ +#include + +static const char *version_sdk __attribute__((used, section(".version.sdk"))) = "component_version_sdk_" PROJECT_SDK_VERSION; diff --git a/bsp/sdk_version.h.in b/bsp/sdk_version.h.in new file mode 100644 index 000000000..3b89700bb --- /dev/null +++ b/bsp/sdk_version.h.in @@ -0,0 +1 @@ +#define PROJECT_SDK_VERSION "@PROJECT_SDK_VERSION@" diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index ccd3efcb2..988199d16 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -13,7 +13,4 @@ add_subdirectory(usb) add_subdirectory(utils) add_subdirectory(wireless) sdk_add_subdirectory_ifdef(CONFIG_MULTIMEDIA multimedia) -sdk_add_subdirectory_ifdef(CONFIG_HIBOOSTER hibooster) sdk_add_subdirectory_ifdef(CONFIG_EASYFLASH4 easyflash4) -sdk_add_subdirectory_ifdef(CONFIG_HYPERVISOR hypervisor) -sdk_add_subdirectory_ifdef(CONFIG_MEMTESTER memtester) diff --git a/components/crypto/mbedtls/.gitignore b/components/crypto/mbedtls/.gitignore deleted file mode 100644 index 940141ac0..000000000 --- a/components/crypto/mbedtls/.gitignore +++ /dev/null @@ -1 +0,0 @@ -mbedtls diff --git a/components/crypto/mbedtls/mbedtls/programs/hash/generic_sum.c b/components/crypto/mbedtls/mbedtls/programs/hash/generic_sum.c index f570e3eb9..71715b492 100644 --- a/components/crypto/mbedtls/mbedtls/programs/hash/generic_sum.c +++ b/components/crypto/mbedtls/mbedtls/programs/hash/generic_sum.c @@ -128,7 +128,7 @@ static int generic_check( const mbedtls_md_info_t *md_info, char *filename ) nb_tot2++; for( i = 0; i < mbedtls_md_get_size( md_info ); i++ ) - sprintf( buf + i * 2, "%02x", sum[i] ); + mbedtls_snprintf( buf + i * 2, sizeof(buf) - (i * 2), "%02x", sum[i] ); /* Use constant-time buffer comparison */ diff = 0; diff --git a/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_client1.c b/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_client1.c index c9f8b3c70..9c9862377 100644 --- a/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_client1.c +++ b/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_client1.c @@ -221,7 +221,7 @@ int main( void ) mbedtls_printf( " > Write to server:" ); fflush( stdout ); - len = sprintf( (char *) buf, GET_REQUEST ); + len = mbedtls_snprintf( (char *) buf, sizeof(buf), GET_REQUEST ); while( ( ret = mbedtls_ssl_write( &ssl, buf, len ) ) <= 0 ) { diff --git a/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_fork_server.c b/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_fork_server.c index 58c8749da..8dab881ed 100644 --- a/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_fork_server.c +++ b/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_fork_server.c @@ -354,7 +354,7 @@ int main( void ) mbedtls_printf( "pid %d: Start writing to client.\n", pid ); fflush( stdout ); - len = sprintf( (char *) buf, HTTP_RESPONSE, + len = mbedtls_snprintf( (char *) buf, sizeof(buf), HTTP_RESPONSE, mbedtls_ssl_get_ciphersuite( &ssl ) ); while( cnt++ < 100 ) diff --git a/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_mail_client.c b/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_mail_client.c index 534c68bdd..79b594fea 100644 --- a/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_mail_client.c +++ b/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_mail_client.c @@ -649,7 +649,7 @@ int main( int argc, char *argv[] ) fflush( stdout ); gethostname( hostname, 32 ); - len = sprintf( (char *) buf, "EHLO %s\r\n", hostname ); + len = mbedtls_snprintf( (char *) buf, sizeof(buf), "EHLO %s\r\n", hostname ); ret = write_ssl_and_get_response( &ssl, buf, len ); if( ret < 200 || ret > 299 ) { @@ -675,7 +675,7 @@ int main( int argc, char *argv[] ) fflush( stdout ); gethostname( hostname, 32 ); - len = sprintf( (char *) buf, "EHLO %s\r\n", hostname ); + len = mbedtls_snprintf( (char *) buf, sizeof(buf), "EHLO %s\r\n", hostname ); ret = write_and_get_response( &server_fd, buf, len ); if( ret < 200 || ret > 299 ) { @@ -689,7 +689,7 @@ int main( int argc, char *argv[] ) fflush( stdout ); gethostname( hostname, 32 ); - len = sprintf( (char *) buf, "STARTTLS\r\n" ); + len = mbedtls_snprintf( (char *) buf, sizeof(buf), "STARTTLS\r\n" ); ret = write_and_get_response( &server_fd, buf, len ); if( ret < 200 || ret > 299 ) { @@ -709,7 +709,7 @@ int main( int argc, char *argv[] ) mbedtls_printf( " > Write AUTH LOGIN to server:" ); fflush( stdout ); - len = sprintf( (char *) buf, "AUTH LOGIN\r\n" ); + len = mbedtls_snprintf( (char *) buf, sizeof(buf), "AUTH LOGIN\r\n" ); ret = write_ssl_and_get_response( &ssl, buf, len ); if( ret < 200 || ret > 399 ) { @@ -729,7 +729,7 @@ int main( int argc, char *argv[] ) mbedtls_printf( " failed\n ! mbedtls_base64_encode returned %d\n\n", ret ); goto exit; } - len = sprintf( (char *) buf, "%s\r\n", base ); + len = mbedtls_snprintf( (char *) buf, sizeof(buf), "%s\r\n", base ); ret = write_ssl_and_get_response( &ssl, buf, len ); if( ret < 300 || ret > 399 ) { @@ -749,7 +749,7 @@ int main( int argc, char *argv[] ) mbedtls_printf( " failed\n ! mbedtls_base64_encode returned %d\n\n", ret ); goto exit; } - len = sprintf( (char *) buf, "%s\r\n", base ); + len = mbedtls_snprintf( (char *) buf, sizeof(buf), "%s\r\n", base ); ret = write_ssl_and_get_response( &ssl, buf, len ); if( ret < 200 || ret > 399 ) { @@ -764,7 +764,7 @@ int main( int argc, char *argv[] ) mbedtls_printf( " > Write MAIL FROM to server:" ); fflush( stdout ); - len = sprintf( (char *) buf, "MAIL FROM:<%s>\r\n", opt.mail_from ); + len = mbedtls_snprintf( (char *) buf, sizeof(buf), "MAIL FROM:<%s>\r\n", opt.mail_from ); ret = write_ssl_and_get_response( &ssl, buf, len ); if( ret < 200 || ret > 299 ) { @@ -777,7 +777,7 @@ int main( int argc, char *argv[] ) mbedtls_printf( " > Write RCPT TO to server:" ); fflush( stdout ); - len = sprintf( (char *) buf, "RCPT TO:<%s>\r\n", opt.mail_to ); + len = mbedtls_snprintf( (char *) buf, sizeof(buf), "RCPT TO:<%s>\r\n", opt.mail_to ); ret = write_ssl_and_get_response( &ssl, buf, len ); if( ret < 200 || ret > 299 ) { @@ -790,7 +790,7 @@ int main( int argc, char *argv[] ) mbedtls_printf( " > Write DATA to server:" ); fflush( stdout ); - len = sprintf( (char *) buf, "DATA\r\n" ); + len = mbedtls_snprintf( (char *) buf, sizeof(buf), "DATA\r\n" ); ret = write_ssl_and_get_response( &ssl, buf, len ); if( ret < 300 || ret > 399 ) { @@ -803,14 +803,14 @@ int main( int argc, char *argv[] ) mbedtls_printf( " > Write content to server:" ); fflush( stdout ); - len = sprintf( (char *) buf, "From: %s\r\nSubject: mbed TLS Test mail\r\n\r\n" + len = mbedtls_snprintf( (char *) buf, sizeof(buf), "From: %s\r\nSubject: mbed TLS Test mail\r\n\r\n" "This is a simple test mail from the " "mbed TLS mail client example.\r\n" "\r\n" "Enjoy!", opt.mail_from ); ret = write_ssl_data( &ssl, buf, len ); - len = sprintf( (char *) buf, "\r\n.\r\n"); + len = mbedtls_snprintf( (char *) buf, sizeof(buf), "\r\n.\r\n"); ret = write_ssl_and_get_response( &ssl, buf, len ); if( ret < 200 || ret > 299 ) { diff --git a/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_pthread_server.c b/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_pthread_server.c index ca84c7a0a..15537cbe6 100644 --- a/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_pthread_server.c +++ b/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_pthread_server.c @@ -203,7 +203,7 @@ static void *handle_ssl_connection( void *data ) */ mbedtls_printf( " [ #%ld ] > Write to client:\n", thread_id ); - len = sprintf( (char *) buf, HTTP_RESPONSE, + len = mbedtls_snprintf( (char *) buf, sizeof(buf), HTTP_RESPONSE, mbedtls_ssl_get_ciphersuite( &ssl ) ); while( ( ret = mbedtls_ssl_write( &ssl, buf, len ) ) <= 0 ) diff --git a/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_server.c b/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_server.c index 956cebcb9..e7740abe5 100644 --- a/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_server.c +++ b/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_server.c @@ -316,7 +316,7 @@ int main( void ) mbedtls_printf( " > Write to client:" ); fflush( stdout ); - len = sprintf( (char *) buf, HTTP_RESPONSE, + len = mbedtls_snprintf( (char *) buf, sizeof(buf), HTTP_RESPONSE, mbedtls_ssl_get_ciphersuite( &ssl ) ); while( ( ret = mbedtls_ssl_write( &ssl, buf, len ) ) <= 0 ) diff --git a/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_server2.c b/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_server2.c index 4e9e499a9..8d7a27b7e 100644 --- a/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_server2.c +++ b/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_server2.c @@ -3720,7 +3720,7 @@ int main( int argc, char *argv[] ) /* If the format of the response changes, make sure there is enough * room in buf (buf_content_size calculation above). */ - len = sprintf( (char *) buf, HTTP_RESPONSE, + len = mbedtls_snprintf( (char *) buf, buf_content_size + 1, HTTP_RESPONSE, mbedtls_ssl_get_ciphersuite( &ssl ) ); /* Add padding to the response to reach opt.response_size in length */ diff --git a/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_test_common_source.c b/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_test_common_source.c index f5d868046..8c9d0556e 100644 --- a/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_test_common_source.c +++ b/components/crypto/mbedtls/mbedtls/programs/ssl/ssl_test_common_source.c @@ -77,24 +77,24 @@ int nss_keylog_export( void *p_expkey, ((void) server_random); ((void) tls_prf_type); - len += sprintf( nss_keylog_line + len, + len += mbedtls_snprintf( nss_keylog_line + len, sizeof(nss_keylog_line) - len, "%s", "CLIENT_RANDOM " ); for( j = 0; j < client_random_len; j++ ) { - len += sprintf( nss_keylog_line + len, + len += mbedtls_snprintf( nss_keylog_line + len, sizeof(nss_keylog_line) - len, "%02x", client_random[j] ); } - len += sprintf( nss_keylog_line + len, " " ); + len += mbedtls_snprintf( nss_keylog_line + len, sizeof(nss_keylog_line) - len, " " ); for( j = 0; j < master_secret_len; j++ ) { - len += sprintf( nss_keylog_line + len, + len += mbedtls_snprintf( nss_keylog_line + len, sizeof(nss_keylog_line) - len, "%02x", ms[j] ); } - len += sprintf( nss_keylog_line + len, "\n" ); + len += mbedtls_snprintf( nss_keylog_line + len, sizeof(nss_keylog_line) - len, "\n" ); nss_keylog_line[ len ] = '\0'; mbedtls_printf( "\n" ); diff --git a/components/crypto/mbedtls/port/hw_acc/bignum_ext.c b/components/crypto/mbedtls/port/hw_acc/bignum_ext.c index 0a95893cd..c27542893 100644 --- a/components/crypto/mbedtls/port/hw_acc/bignum_ext.c +++ b/components/crypto/mbedtls/port/hw_acc/bignum_ext.c @@ -17,7 +17,7 @@ void dump_mpi(const char *tag, const mbedtls_mpi *bn) if (bn) { mbedtls_mpi_write_string(bn, 16, buf, buf_len, &olen); } else { - strcpy(buf, ""); + strlcpy(buf, "", buf_len); } printf(buf); printf("\r\n"); diff --git a/components/easyflash4/src/easyflash.c b/components/easyflash4/src/easyflash.c index a9b4130d5..aeb93c8d5 100644 --- a/components/easyflash4/src/easyflash.c +++ b/components/easyflash4/src/easyflash.c @@ -56,6 +56,8 @@ #if !defined(EF_ERASE_MIN_SIZE) #error "Please configure minimum size of flash erasure (in ef_cfg.h)" #endif +/* initialize OK flag */ +static bool easyflash_init_ok = false; /** * EasyFlash system initialize. @@ -72,6 +74,10 @@ EfErrCode easyflash_init(void) { const ef_env *default_env_set; EfErrCode result = EF_NO_ERR; + if(easyflash_init_ok) + { + return EF_NO_ERR; + } result = ef_port_init(&default_env_set, &default_env_set_size); #ifdef EF_USING_ENV @@ -94,6 +100,7 @@ EfErrCode easyflash_init(void) { if (result == EF_NO_ERR) { EF_INFO("EasyFlash V%s is initialize success.\r\n", EF_SW_VERSION); + easyflash_init_ok = true; } else { EF_INFO("EasyFlash V%s is initialize fail.\r\n", EF_SW_VERSION); } diff --git a/components/easyflash4/src/ef_env.c b/components/easyflash4/src/ef_env.c index 3d2b8d931..d9f7b5396 100644 --- a/components/easyflash4/src/ef_env.c +++ b/components/easyflash4/src/ef_env.c @@ -1160,7 +1160,9 @@ static EfErrCode move_env(env_node_obj_t env) if (in_recovery_check) { struct env_node_obj env_bak; char name[EF_ENV_NAME_MAX + 1] = { 0 }; - strncpy(name, env->name, env->name_len); + if (strlcpy(name, env->name, sizeof(name)) >= sizeof(name)) { + EF_ASSERT(0); + } /* check the ENV in flash is already create success */ if (find_env_no_cache(name, &env_bak)) { /* already create success, don't need to duplicate */ diff --git a/components/fs/littlefs/.gitignore b/components/fs/littlefs/.gitignore deleted file mode 100644 index 84fea743e..000000000 --- a/components/fs/littlefs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -littlefs diff --git a/components/fs/littlefs/CMakeLists.txt b/components/fs/littlefs/CMakeLists.txt index 72105eade..87eed6ea4 100644 --- a/components/fs/littlefs/CMakeLists.txt +++ b/components/fs/littlefs/CMakeLists.txt @@ -22,4 +22,4 @@ endif() if(CONFIG_LITTLEFS_FLASH_ADDRESS) sdk_add_compile_definitions(-DCONFIG_LITTLEFS_FLASH_ADDRESS=${CONFIG_LITTLEFS_FLASH_ADDRESS}) -endif() \ No newline at end of file +endif() diff --git a/components/fs/littlefs/easyflash_port/easyflash.h b/components/fs/littlefs/easyflash_port/easyflash.h new file mode 100644 index 000000000..6d9a0539d --- /dev/null +++ b/components/fs/littlefs/easyflash_port/easyflash.h @@ -0,0 +1,55 @@ +/* + * This file is the header file that adapts LittleFS to EasyFlash4. + */ + +#ifndef EASYFLASH_H_ +#define EASYFLASH_H_ + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* EasyFlash error code */ +typedef enum { + EF_NO_ERR, + EF_ERASE_ERR, + EF_READ_ERR, + EF_WRITE_ERR, + EF_ENV_NAME_ERR, + EF_ENV_NAME_EXIST, + EF_ENV_FULL, + EF_ENV_INIT_FAILED, + EF_ENV_ARG_ERR, +} EfErrCode; + +/* easyflash.c */ +EfErrCode easyflash_init(void); + +/* only supported on ef_env.c */ +size_t ef_get_env_blob(const char *key, void *value_buf, size_t buf_len, size_t *saved_value_len); +size_t ef_get_env_blob_offset(const char *key, void *value_buf, size_t buf_len, size_t *saved_value_len, int offset); +EfErrCode ef_set_env_blob(const char *key, const void *value_buf, size_t buf_len); + +char *ef_get_env(const char *key) __attribute__((deprecated("Not recommended, use ef_get_env_blob instead"))); +EfErrCode ef_set_env(const char *key, const char *value); +EfErrCode ef_del_env(const char *key); + +EfErrCode ef_save_env(void); + +/* clear all env */ +EfErrCode ef_env_set_default(void); + +EfErrCode ef_set_and_save_env(const char *key, const char *value); +EfErrCode ef_del_and_save_env(const char *key); + +EfErrCode ef_print_env(void); + +#ifdef __cplusplus +} +#endif + +#endif /* EASYFLASH_H_ */ diff --git a/components/fs/littlefs/easyflash_port/lfs_easyflash.c b/components/fs/littlefs/easyflash_port/lfs_easyflash.c new file mode 100644 index 000000000..f41eeeee8 --- /dev/null +++ b/components/fs/littlefs/easyflash_port/lfs_easyflash.c @@ -0,0 +1,454 @@ +#include +#include +#include +#include +#include +#ifdef CONFIG_FREERTOS +#include +#include +#endif + +#define DBG_TAG "LFS" + +#include "log.h" + +#include "lfs.h" +#include "lfs_port.h" +#include "easyflash.h" + +static lfs_t *lfs = NULL; +static char path_buffer[64]; + +#ifndef LFS_EF_NAMESPACE +#define LFS_EF_NAMESPACE "/_ef4_kvs_" +#endif + +#ifdef CONFIG_FREERTOS +static SemaphoreHandle_t env_giant_lock = NULL; +#endif + +/* littlefs init */ +EfErrCode +easyflash_init(void) +{ + struct lfs_info stat; + int32_t ret; + + lfs = lfs_xip_init(); + if (lfs == NULL) { + LOG_E("littlefs backend init failed.\r\n"); + return EF_ENV_INIT_FAILED; + } + +#ifdef CONFIG_FREERTOS +#if configUSE_RECURSIVE_MUTEXES + env_giant_lock = xSemaphoreCreateRecursiveMutex(); +#else + env_giant_lock = xSemaphoreCreateMutex(); +#endif +#endif + + /* init namespace */ + ret = lfs_stat(lfs, LFS_EF_NAMESPACE, &stat); + if (ret == LFS_ERR_OK) { + if (stat.type == LFS_TYPE_DIR) { + return EF_NO_ERR; + } else if (stat.type == LFS_TYPE_REG) { + LOG_E("namespace directory conflicts with standard file.\r\n"); + return EF_ENV_INIT_FAILED; + } + } + + ret = lfs_mkdir(lfs, LFS_EF_NAMESPACE); + if (ret != LFS_ERR_OK) { + errno = -ret; + LOG_E("create namespace directory failed.\r\n"); + return EF_ENV_INIT_FAILED; + } + + return EF_NO_ERR; +} + +#ifdef CONFIG_FREERTOS +static void ef_giant_lock(void) +{ +#if configUSE_RECURSIVE_MUTEXES + xSemaphoreTakeRecursive(env_giant_lock, portMAX_DELAY); +#else + xSemaphoreTake(env_giant_lock, portMAX_DELAY); +#endif +} + +static void ef_giant_unlock(void) +{ +#if configUSE_RECURSIVE_MUTEXES + xSemaphoreGiveRecursive(env_giant_lock); +#else + xSemaphoreGive(env_giant_lock); +#endif +} + +#define EF_GIANT_LOCK() ef_giant_lock() +#define EF_GIANT_UNLOCK(ret) \ + ef_giant_unlock(); \ + return (ret) +#else +#define EF_GIANT_LOCK() +#define EF_GIANT_UNLOCK(ret) return (ret) +#endif + +static int +gen_kv_key_path(char *buf, size_t buf_len, const char *prefix, const char *path) +{ + int i, j = 0; + int prefix_len = strlen(prefix); + int path_len; + + i = prefix_len + 1; /* prefix + '/' */ + /* cal full path string length */ + for (j = 0; path[j] != 0; j++) { + switch (path[j]) { + case '}': + case '/': + i += 2; + break; + default: + i++; + } + } + path_len = j; + + /* oversize */ + if (i > buf_len - 1) { + return i; + } + + /* do strcat */ + memcpy(buf, prefix, prefix_len); + buf[prefix_len] = '/'; + + for (j = 0, i = 0; j < path_len; j++) { + switch (path[j]) { + case '}': + case '/': + (buf + prefix_len + 1)[i++] = '}'; + (buf + prefix_len + 1)[i++] = path[j] ^ 0x20; + break; + default: + (buf + prefix_len + 1)[i++] = path[j]; + } + } + + (buf + prefix_len + 1)[i] = 0; + return prefix_len + 1 + i; +} + +/* only supported on ef_env.c */ +size_t +ef_get_env_blob_offset(const char *key, void *value_buf, size_t buf_len, size_t *saved_value_len, int offset) +{ + lfs_file_t file; + int32_t ret, read_len; + + if (lfs == NULL) { + return 0; + } + + if (key == NULL || value_buf == NULL || buf_len == 0) { + return 0; + } + + EF_GIANT_LOCK(); + + ret = gen_kv_key_path(path_buffer, sizeof(path_buffer), LFS_EF_NAMESPACE, key); + if (ret >= sizeof(path_buffer)) { + LOG_E("key name is too long to truncated.\r\n"); + EF_GIANT_UNLOCK(0); + } + + ret = lfs_file_open(lfs, &file, path_buffer, LFS_O_RDONLY); + if (ret != LFS_ERR_OK) { + errno = -ret; + EF_GIANT_UNLOCK(0); + } + + ret = lfs_file_seek(lfs, &file, offset, LFS_SEEK_SET); + if (ret < 0) { + errno = -ret; + lfs_file_close(lfs, &file); + EF_GIANT_UNLOCK(0); + } + + ret = lfs_file_read(lfs, &file, value_buf, buf_len); + if (ret < 0) { + LOG_E("lfs_file_read failed with errno:%d.\r\n", ret); + errno = -ret; + lfs_file_close(lfs, &file); + EF_GIANT_UNLOCK(0); + } + + read_len = ret; + + if (saved_value_len != NULL) { + ret = lfs_file_size(lfs, &file); + if (ret < 0) { + LOG_E("lfs_file_size failed with errno:%d.\r\n", ret); + lfs_file_close(lfs, &file); + errno = -ret; + EF_GIANT_UNLOCK(0); + } + *saved_value_len = ret; + } + + lfs_file_close(lfs, &file); + + EF_GIANT_UNLOCK(read_len); +} + +size_t +ef_get_env_blob(const char *key, void *value_buf, size_t buf_len, size_t *saved_value_len) +{ + return ef_get_env_blob_offset(key, value_buf, buf_len, saved_value_len, 0); +} + +EfErrCode +ef_set_env_blob(const char *key, const void *value_buf, size_t buf_len) +{ + lfs_file_t file; + int32_t ret; + + if (lfs == NULL) { + return EF_ENV_INIT_FAILED; + } + + if (key == NULL || value_buf == NULL || buf_len == 0) { + return EF_ENV_ARG_ERR; + } + + EF_GIANT_LOCK(); + + ret = gen_kv_key_path(path_buffer, sizeof(path_buffer), LFS_EF_NAMESPACE, key); + if (ret >= sizeof(path_buffer)) { + LOG_E("key name is too long to truncated.\r\n"); + EF_GIANT_UNLOCK(EF_ENV_NAME_ERR); + } + + ret = lfs_file_open(lfs, &file, path_buffer, LFS_O_RDWR | LFS_O_CREAT); + if (ret != LFS_ERR_OK) { + errno = -ret; + LOG_E("lfs_file_open failed with errno:%d\r\n", ret); + EF_GIANT_UNLOCK(EF_WRITE_ERR); + } + + ret = lfs_file_write(lfs, &file, value_buf, buf_len); + if (ret != buf_len) { + errno = -ret; + LOG_E("lfs_file_write failed with errno:%d.\r\n", ret); + lfs_file_close(lfs, &file); + EF_GIANT_UNLOCK(EF_WRITE_ERR); + } + + ret = lfs_file_truncate(lfs, &file, buf_len); + if (ret != LFS_ERR_OK) { + errno = -ret; + LOG_E("lfs_file_truncate failed with errno:%d\r\n", ret); + lfs_file_close(lfs, &file); + EF_GIANT_UNLOCK(EF_WRITE_ERR); + } + + lfs_file_close(lfs, &file); + + EF_GIANT_UNLOCK(EF_NO_ERR); +} + +EfErrCode +ef_del_env(const char *key) +{ + int32_t ret; + + if (lfs == NULL) { + return EF_ENV_INIT_FAILED; + } + + if (key == NULL) { + return EF_ENV_ARG_ERR; + } + + EF_GIANT_LOCK(); + ret = gen_kv_key_path(path_buffer, sizeof(path_buffer), LFS_EF_NAMESPACE, key); + if (ret >= sizeof(path_buffer)) { + LOG_E("key name is too long to truncated.\r\n"); + EF_GIANT_UNLOCK(EF_ENV_NAME_ERR); + } + + lfs_remove(lfs, path_buffer); + EF_GIANT_UNLOCK(EF_NO_ERR); +} + +/**************************************************************************** + * DEPRECATED DEPRECATED-DEPRECATED-DEPRECATED-DEPRECATED-DEPRECATED + ***************************************************************************/ +static bool ef_is_str(uint8_t *value, size_t len) +{ +#define __is_print(ch) ((unsigned int)((ch) - ' ') < 127u - ' ') + size_t i; + + for (i = 0; i < len; i++) { + if (!__is_print(value[i])) { + return false; + } + } + return true; +} + +/**************************************************************************** + * DEPRECATED DEPRECATED-DEPRECATED-DEPRECATED-DEPRECATED-DEPRECATED + ***************************************************************************/ +char * +ef_get_env(const char *key) +{ +#ifndef EF_STR_ENV_VALUE_MAX_SIZE +#define EF_STR_ENV_VALUE_MAX_SIZE (3979) +#endif + static char value[EF_STR_ENV_VALUE_MAX_SIZE + 1]; + size_t get_size; + + printf("WARNING!!! ef_get_env is deprecated, use ef_get_env_blob instead.\r\n"); + + get_size = ef_get_env_blob(key, value, EF_STR_ENV_VALUE_MAX_SIZE, NULL); + value[get_size] = '\0'; + + get_size = get_size > 0 ? strlen(value) : get_size; + if (get_size > 0 && ef_is_str((uint8_t *)value, get_size)) { + value[get_size] = '\0'; + return value; + } else { + LOG_W("The ENV value isn't string. Could not be returned\r\n"); + return NULL; + } +} + +EfErrCode +ef_set_env(const char *key, const char *value) +{ + return ef_set_env_blob(key, value, strlen(value) + 1); +} + +EfErrCode +ef_save_env(void) +{ + return EF_NO_ERR; +} + +/* clear all env */ +EfErrCode +ef_env_set_default(void) +{ + /* clear all kv */ + lfs_dir_t dir = {}; + int ret; + + if (lfs == NULL) { + return EF_ENV_INIT_FAILED; + } + + EF_GIANT_LOCK(); + + ret = lfs_dir_open(lfs, &dir, LFS_EF_NAMESPACE); + if (ret != LFS_ERR_OK) { + errno = -ret; + EF_GIANT_UNLOCK(EF_READ_ERR); + } + + struct lfs_info info = {}; + + while (1) { + ret = lfs_dir_read(lfs, &dir, &info); + if (ret < 0) { + errno = -ret; + ret = EF_READ_ERR; + break; + } else if (ret == 0) { + ret = EF_NO_ERR; + break; + } + + if (strcmp(info.name, ".") == 0 || strcmp(info.name, "..") == 0) { + continue; + } + + if (info.type != LFS_TYPE_REG) { + LOG_E("Unexpected file type! name:%s, size: %d, type: %d\r\n", info.name, info.size, info.type); + ret = EF_ENV_NAME_ERR; + break; + } + + ret = snprintf(path_buffer, sizeof(path_buffer), "%s/%s", LFS_EF_NAMESPACE, info.name); + assert(ret <= sizeof(path_buffer) - 1); + + ret = lfs_remove(lfs, path_buffer); + if (ret < 0) { + errno = -ret; + ret = EF_WRITE_ERR; + break; + } + } + + lfs_dir_close(lfs, &dir); + EF_GIANT_UNLOCK(ret); +} + +EfErrCode +ef_set_and_save_env(const char *key, const char *value) +{ + return ef_set_env(key, value); +} + +EfErrCode +ef_del_and_save_env(const char *key) +{ + return ef_del_env(key); +} + +EfErrCode +ef_print_env(void) +{ + /* clear all kv */ + lfs_dir_t dir = {}; + int ret; + + if (lfs == NULL) { + return EF_ENV_INIT_FAILED; + } + + EF_GIANT_LOCK(); + + ret = lfs_dir_open(lfs, &dir, LFS_EF_NAMESPACE); + if (ret != LFS_ERR_OK) { + errno = -ret; + EF_GIANT_UNLOCK(EF_READ_ERR); + } + + struct lfs_info info = {}; + + while (1) { + ret = lfs_dir_read(lfs, &dir, &info); + if (ret < 0) { + errno = -ret; + ret = EF_READ_ERR; + break; + } else if (ret == 0) { + ret = EF_NO_ERR; + break; + } + + if (strcmp(info.name, ".") == 0 || strcmp(info.name, "..") == 0) { + continue; + } + + printf("key: %s, size: %d, type: %d\r\n", info.name, info.size, info.type); + } + + lfs_dir_close(lfs, &dir); + EF_GIANT_UNLOCK(EF_NO_ERR); +} diff --git a/components/fs/littlefs/littlefs/.gitattributes b/components/fs/littlefs/littlefs/.gitattributes new file mode 100644 index 000000000..26d042513 --- /dev/null +++ b/components/fs/littlefs/littlefs/.gitattributes @@ -0,0 +1,4 @@ +# GitHub really wants to mark littlefs as a python project, telling it to +# reclassify our test .toml files as C code (which they are 95% of anyways) +# remedies this +*.toml linguist-language=c diff --git a/components/fs/littlefs/littlefs/.github/workflows/post-release.yml b/components/fs/littlefs/littlefs/.github/workflows/post-release.yml index a44a675d1..6524124d1 100644 --- a/components/fs/littlefs/littlefs/.github/workflows/post-release.yml +++ b/components/fs/littlefs/littlefs/.github/workflows/post-release.yml @@ -4,9 +4,13 @@ on: branches: [master] types: [released] +defaults: + run: + shell: bash -euv -o pipefail {0} + jobs: post-release: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: # trigger post-release in dependency repo, this indirection allows the # dependency repo to be updated often without affecting this repo. At @@ -21,6 +25,7 @@ jobs: event_type: "post-release", client_payload: { repo: env.GITHUB_REPOSITORY, - version: "${{github.event.release.tag_name}}"}}' \ - | tee /dev/stderr)" + version: "${{github.event.release.tag_name}}", + }, + }' | tee /dev/stderr)" diff --git a/components/fs/littlefs/littlefs/.github/workflows/release.yml b/components/fs/littlefs/littlefs/.github/workflows/release.yml index c38b8de63..5ccc9e058 100644 --- a/components/fs/littlefs/littlefs/.github/workflows/release.yml +++ b/components/fs/littlefs/littlefs/.github/workflows/release.yml @@ -5,9 +5,13 @@ on: branches: [master] types: [completed] +defaults: + run: + shell: bash -euv -o pipefail {0} + jobs: release: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 # need to manually check for a couple things # - tests passed? @@ -31,8 +35,22 @@ jobs: with: workflow: ${{github.event.workflow_run.name}} run_id: ${{github.event.workflow_run.id}} - name: results - path: results + name: sizes + path: sizes + - uses: dawidd6/action-download-artifact@v2 + continue-on-error: true + with: + workflow: ${{github.event.workflow_run.name}} + run_id: ${{github.event.workflow_run.id}} + name: cov + path: cov + - uses: dawidd6/action-download-artifact@v2 + continue-on-error: true + with: + workflow: ${{github.event.workflow_run.name}} + run_id: ${{github.event.workflow_run.id}} + name: bench + path: bench - name: find-version run: | @@ -63,84 +81,126 @@ jobs: - name: find-prev-version continue-on-error: true run: | - LFS_PREV_VERSION="$(git describe --tags --abbrev=0 --match 'v*')" + LFS_PREV_VERSION="$( \ + git describe --tags --abbrev=0 --match 'v*' \ + || true)" echo "LFS_PREV_VERSION=$LFS_PREV_VERSION" echo "LFS_PREV_VERSION=$LFS_PREV_VERSION" >> $GITHUB_ENV # try to find results from tests - - name: collect-results + - name: create-table run: | # previous results to compare against? [ -n "$LFS_PREV_VERSION" ] && curl -sS \ - "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/` - `status/$LFS_PREV_VERSION?per_page=100" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/status/$LFS_PREV_VERSION` + `?per_page=100" \ | jq -re 'select(.sha != env.GITHUB_SHA) | .statuses[]' \ - >> prev-results.json \ + >> prev-status.json \ || true # build table for GitHub - echo "" >> results.txt - echo "" >> results.txt - echo "" >> results.txt - echo "" >> results.txt - for r in Code Stack Structs Coverage - do - echo "" >> results.txt - done - echo "" >> results.txt - echo "" >> results.txt + declare -A table - echo "" >> results.txt - for c in "" readonly threadsafe migrate error-asserts + # sizes table + i=0 + j=0 + for c in "" readonly threadsafe multiversion migrate error-asserts do - echo "" >> results.txt + # per-config results c_or_default=${c:-default} - echo "" >> results.txt - for r in code stack structs + c_camel=${c_or_default^} + table[$i,$j]=$c_camel + ((j+=1)) + + for s in code stack structs do - # per-config results - echo "" >> results.txt + select(.context == "'"sizes (thumb${c:+, $c}) / $s"'").description + | capture("(?[0-9∞]+)").prev' \ + prev-status.json || echo 0)" + ./scripts/summary.py $f --max=stack_limit -Y \ + | awk ' + NR==2 {$1=0; printf "%s B",$NF} + NR==2 && ENVIRON["PREV"]+0 != 0 { + printf " (%+.1f%%)",100*($NF-ENVIRON["PREV"])/ENVIRON["PREV"] + }' \ + | sed -e 's/ /\ /g') + ((j+=1)) done - # coverage results - if [ -z $c ] - then - echo "" >> results.txt - fi - echo "" >> results.txt + printf " (%+.1f%%)",$4-ENVIRON["PREV"] + }' \ + | sed -e 's/ /\ /g') + ((j=4, i+=1)) done - echo "" >> results.txt - echo "
Configuration$r
${c_or_default^}" >> results.txt - [ -e results/thumb${c:+-$c}.csv ] && ( \ + f=sizes/thumb${c:+-$c}.$s.csv + [ -e $f ] && table[$i,$j]=$( \ export PREV="$(jq -re ' - select(.context == "'"results (thumb${c:+, $c}) / $r"'").description - | capture("(?[0-9∞]+)").result' \ - prev-results.json || echo 0)" - ./scripts/summary.py results/thumb${c:+-$c}.csv -f $r -Y | awk ' - NR==2 {printf "%s B",$2} - NR==2 && ENVIRON["PREV"]+0 != 0 { - printf " (%+.1f%%)",100*($2-ENVIRON["PREV"])/ENVIRON["PREV"]} - NR==2 {printf "\n"}' \ - | sed -e 's/ /\ /g' \ - >> results.txt) - echo "" >> results.txt - [ -e results/coverage.csv ] && ( \ - export PREV="$(jq -re ' - select(.context == "results / coverage").description - | capture("(?[0-9\\.]+)").result' \ - prev-results.json || echo 0)" - ./scripts/coverage.py -u results/coverage.csv -Y | awk -F '[ /%]+' ' - NR==2 {printf "%.1f%% of %d lines",$4,$3} + ((j=0, i+=1)) + done + + # coverage table + i=0 + j=4 + for s in lines branches + do + table[$i,$j]=${s^} + ((j+=1)) + + f=cov/cov.csv + [ -e $f ] && table[$i,$j]=$( \ + export PREV="$(jq -re ' + select(.context == "'"cov / $s"'").description + | capture("(?[0-9]+)/(?[0-9]+)") + | 100*((.prev_a|tonumber) / (.prev_b|tonumber))' \ + prev-status.json || echo 0)" + ./scripts/cov.py -u $f -f$s -Y \ + | awk -F '[ /%]+' -v s=$s ' + NR==2 {$1=0; printf "%d/%d %s",$2,$3,s} NR==2 && ENVIRON["PREV"]+0 != 0 { - printf " (%+.1f%%)",$4-ENVIRON["PREV"]} - NR==2 {printf "\n"}' \ - | sed -e 's/ /\ /g' \ - >> results.txt) - echo "
" >> results.txt - cat results.txt + # benchmark table + i=3 + j=4 + for s in readed proged erased + do + table[$i,$j]=${s^} + ((j+=1)) + + f=bench/bench.csv + [ -e $f ] && table[$i,$j]=$( \ + export PREV="$(jq -re ' + select(.context == "'"bench / $s"'").description + | capture("(?[0-9]+)").prev' \ + prev-status.json || echo 0)" + ./scripts/summary.py $f -f$s=bench_$s -Y \ + | awk ' + NR==2 {$1=0; printf "%s B",$NF} + NR==2 && ENVIRON["PREV"]+0 != 0 { + printf " (%+.1f%%)",100*($NF-ENVIRON["PREV"])/ENVIRON["PREV"] + }' \ + | sed -e 's/ /\ /g') + ((j=4, i+=1)) + done + + # build the actual table + echo "| | Code | Stack | Structs | | Coverage |" >> table.txt + echo "|:--|-----:|------:|--------:|:--|---------:|" >> table.txt + for ((i=0; i<6; i++)) + do + echo -n "|" >> table.txt + for ((j=0; j<6; j++)) + do + echo -n " " >> table.txt + [[ i -eq 2 && j -eq 5 ]] && echo -n "**Benchmarks**" >> table.txt + echo -n "${table[$i,$j]:-}" >> table.txt + echo -n " |" >> table.txt + done + echo >> table.txt + done + + cat table.txt # find changes from history - - name: collect-changes + - name: create-changes run: | [ -n "$LFS_PREV_VERSION" ] || exit 0 # use explicit link to github commit so that release notes can @@ -164,7 +224,7 @@ jobs: git config user.email ${{secrets.BOT_EMAIL}} git fetch "https://github.com/$GITHUB_REPOSITORY.git" \ "v$LFS_VERSION_MAJOR-prefix" || true - ./scripts/prefix.py "lfs$LFS_VERSION_MAJOR" + ./scripts/changeprefix.py --git "lfs" "lfs$LFS_VERSION_MAJOR" git branch "v$LFS_VERSION_MAJOR-prefix" $( \ git commit-tree $(git write-tree) \ $(git rev-parse --verify -q FETCH_HEAD | sed -e 's/^/-p /') \ @@ -182,15 +242,19 @@ jobs: run: | # create release and patch version tag (vN.N.N) # only draft if not a patch release - [ -e results.txt ] && export RESULTS="$(cat results.txt)" - [ -e changes.txt ] && export CHANGES="$(cat changes.txt)" + touch release.txt + [ -e table.txt ] && cat table.txt >> release.txt + echo >> release.txt + [ -e changes.txt ] && cat changes.txt >> release.txt + cat release.txt + curl -sS -X POST -H "authorization: token ${{secrets.BOT_TOKEN}}" \ "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/releases" \ - -d "$(jq -n '{ + -d "$(jq -n --rawfile release release.txt '{ tag_name: env.LFS_VERSION, name: env.LFS_VERSION | rtrimstr(".0"), target_commitish: "${{github.event.workflow_run.head_sha}}", draft: env.LFS_VERSION | endswith(".0"), - body: [env.RESULTS, env.CHANGES | select(.)] | join("\n\n")}' \ - | tee /dev/stderr)" + body: $release, + }' | tee /dev/stderr)" diff --git a/components/fs/littlefs/littlefs/.github/workflows/status.yml b/components/fs/littlefs/littlefs/.github/workflows/status.yml index d28b17cc7..8bd3990cd 100644 --- a/components/fs/littlefs/littlefs/.github/workflows/status.yml +++ b/components/fs/littlefs/littlefs/.github/workflows/status.yml @@ -4,11 +4,15 @@ on: workflows: [test] types: [completed] +defaults: + run: + shell: bash -euv -o pipefail {0} + jobs: + # forward custom statuses status: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - # custom statuses? - uses: dawidd6/action-download-artifact@v2 continue-on-error: true with: @@ -50,6 +54,47 @@ jobs: state: env.STATE, context: env.CONTEXT, description: env.DESCRIPTION, - target_url: env.TARGET_URL}' \ - | tee /dev/stderr)" + target_url: env.TARGET_URL, + }' | tee /dev/stderr)" + done + + # forward custom pr-comments + comment: + runs-on: ubuntu-22.04 + + # only run on success (we don't want garbage comments!) + if: ${{github.event.workflow_run.conclusion == 'success'}} + + steps: + # generated comment? + - uses: dawidd6/action-download-artifact@v2 + continue-on-error: true + with: + workflow: ${{github.event.workflow_run.name}} + run_id: ${{github.event.workflow_run.id}} + name: comment + path: comment + - name: update-comment + continue-on-error: true + run: | + ls comment + for s in $(shopt -s nullglob ; echo comment/*.json) + do + export NUMBER="$(jq -er '.number' $s)" + export BODY="$(jq -er '.body' $s)" + + # check that the comment was from the most recent commit on the + # pull request + [ "$(curl -sS -H "authorization: token ${{secrets.BOT_TOKEN}}" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/pulls/$NUMBER" \ + | jq -er '.head.sha')" \ + == ${{github.event.workflow_run.head_sha}} ] || continue + + # update comment + curl -sS -X POST -H "authorization: token ${{secrets.BOT_TOKEN}}" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/issues/` + `$NUMBER/comments" \ + -d "$(jq -n '{ + body: env.BODY, + }' | tee /dev/stderr)" done diff --git a/components/fs/littlefs/littlefs/.github/workflows/test.yml b/components/fs/littlefs/littlefs/.github/workflows/test.yml index 81f00c1ef..ccb08fea9 100644 --- a/components/fs/littlefs/littlefs/.github/workflows/test.yml +++ b/components/fs/littlefs/littlefs/.github/workflows/test.yml @@ -1,14 +1,20 @@ name: test on: [push, pull_request] +defaults: + run: + shell: bash -euv -o pipefail {0} + env: CFLAGS: -Werror MAKEFLAGS: -j + TESTFLAGS: -k + BENCHFLAGS: jobs: # run tests test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -18,315 +24,564 @@ jobs: - uses: actions/checkout@v2 - name: install run: | - # need a few additional tools - # - # note this includes gcc-10, which is required for -fcallgraph-info=su + # need a few things sudo apt-get update -qq - sudo apt-get install -qq gcc-10 python3 python3-pip lcov - sudo pip3 install toml - echo "CC=gcc-10" >> $GITHUB_ENV - gcc-10 --version - lcov --version + sudo apt-get install -qq gcc python3 python3-pip + pip3 install toml + gcc --version python3 --version - # need newer lcov version for gcc-10 - #sudo apt-get remove lcov - #wget https://launchpad.net/ubuntu/+archive/primary/+files/lcov_1.15-1_all.deb - #sudo apt install ./lcov_1.15-1_all.deb - #lcov --version - #which lcov - #ls -lha /usr/bin/lcov - wget https://github.com/linux-test-project/lcov/releases/download/v1.15/lcov-1.15.tar.gz - tar xf lcov-1.15.tar.gz - sudo make -C lcov-1.15 install - - # setup a ram-backed disk to speed up reentrant tests - mkdir disks - sudo mount -t tmpfs -o size=100m tmpfs disks - TESTFLAGS="$TESTFLAGS --disk=disks/disk" - - # collect coverage - mkdir -p coverage - TESTFLAGS="$TESTFLAGS --coverage=` - `coverage/${{github.job}}-${{matrix.arch}}.info" - - echo "TESTFLAGS=$TESTFLAGS" >> $GITHUB_ENV - # cross-compile with ARM Thumb (32-bit, little-endian) - name: install-thumb if: ${{matrix.arch == 'thumb'}} run: | sudo apt-get install -qq \ - gcc-10-arm-linux-gnueabi \ + gcc-arm-linux-gnueabi \ libc6-dev-armel-cross \ qemu-user - echo "CC=arm-linux-gnueabi-gcc-10 -mthumb --static" >> $GITHUB_ENV + echo "CC=arm-linux-gnueabi-gcc -mthumb --static" >> $GITHUB_ENV echo "EXEC=qemu-arm" >> $GITHUB_ENV - arm-linux-gnueabi-gcc-10 --version + arm-linux-gnueabi-gcc --version qemu-arm -version # cross-compile with MIPS (32-bit, big-endian) - name: install-mips if: ${{matrix.arch == 'mips'}} run: | sudo apt-get install -qq \ - gcc-10-mips-linux-gnu \ + gcc-mips-linux-gnu \ libc6-dev-mips-cross \ qemu-user - echo "CC=mips-linux-gnu-gcc-10 --static" >> $GITHUB_ENV + echo "CC=mips-linux-gnu-gcc --static" >> $GITHUB_ENV echo "EXEC=qemu-mips" >> $GITHUB_ENV - mips-linux-gnu-gcc-10 --version + mips-linux-gnu-gcc --version qemu-mips -version # cross-compile with PowerPC (32-bit, big-endian) - name: install-powerpc if: ${{matrix.arch == 'powerpc'}} run: | sudo apt-get install -qq \ - gcc-10-powerpc-linux-gnu \ + gcc-powerpc-linux-gnu \ libc6-dev-powerpc-cross \ qemu-user - echo "CC=powerpc-linux-gnu-gcc-10 --static" >> $GITHUB_ENV + echo "CC=powerpc-linux-gnu-gcc --static" >> $GITHUB_ENV echo "EXEC=qemu-ppc" >> $GITHUB_ENV - powerpc-linux-gnu-gcc-10 --version + powerpc-linux-gnu-gcc --version qemu-ppc -version + # does littlefs compile? + - name: test-build + run: | + make clean + make build + # make sure example can at least compile - name: test-example run: | + make clean sed -n '/``` c/,/```/{/```/d; p}' README.md > test.c - make all CFLAGS+=" \ + CFLAGS="$CFLAGS \ -Duser_provided_block_device_read=NULL \ -Duser_provided_block_device_prog=NULL \ -Duser_provided_block_device_erase=NULL \ -Duser_provided_block_device_sync=NULL \ - -include stdio.h" + -include stdio.h" \ + make all rm test.c - # test configurations - # normal+reentrant tests - - name: test-default - run: | - make clean - make test TESTFLAGS+="-nrk" - # NOR flash: read/prog = 1 block = 4KiB - - name: test-nor + # run the tests! + - name: test run: | make clean - make test TESTFLAGS+="-nrk \ - -DLFS_READ_SIZE=1 -DLFS_BLOCK_SIZE=4096" - # SD/eMMC: read/prog = 512 block = 512 - - name: test-emmc + make test + + # collect coverage info + # + # Note the goal is to maximize coverage in the small, easy-to-run + # tests, so we intentionally exclude more aggressive powerloss testing + # from coverage results + - name: cov + if: ${{matrix.arch == 'x86_64'}} run: | - make clean - make test TESTFLAGS+="-nrk \ - -DLFS_READ_SIZE=512 -DLFS_BLOCK_SIZE=512" - # NAND flash: read/prog = 4KiB block = 32KiB - - name: test-nand + make lfs.cov.csv + ./scripts/cov.py -u lfs.cov.csv + mkdir -p cov + cp lfs.cov.csv cov/cov.csv + + # find compile-time measurements + - name: sizes run: | make clean - make test TESTFLAGS+="-nrk \ - -DLFS_READ_SIZE=4096 -DLFS_BLOCK_SIZE=\(32*1024\)" - # other extreme geometries that are useful for various corner cases - - name: test-no-intrinsics + CFLAGS="$CFLAGS \ + -DLFS_NO_ASSERT \ + -DLFS_NO_DEBUG \ + -DLFS_NO_WARN \ + -DLFS_NO_ERROR" \ + make lfs.code.csv lfs.data.csv lfs.stack.csv lfs.structs.csv + ./scripts/structs.py -u lfs.structs.csv + ./scripts/summary.py lfs.code.csv lfs.data.csv lfs.stack.csv \ + -bfunction \ + -fcode=code_size \ + -fdata=data_size \ + -fstack=stack_limit --max=stack_limit + mkdir -p sizes + cp lfs.code.csv sizes/${{matrix.arch}}.code.csv + cp lfs.data.csv sizes/${{matrix.arch}}.data.csv + cp lfs.stack.csv sizes/${{matrix.arch}}.stack.csv + cp lfs.structs.csv sizes/${{matrix.arch}}.structs.csv + - name: sizes-readonly run: | make clean - make test TESTFLAGS+="-nrk \ - -DLFS_NO_INTRINSICS" - - name: test-byte-writes - # it just takes too long to test byte-level writes when in qemu, - # should be plenty covered by the other configurations - if: ${{matrix.arch == 'x86_64'}} + CFLAGS="$CFLAGS \ + -DLFS_NO_ASSERT \ + -DLFS_NO_DEBUG \ + -DLFS_NO_WARN \ + -DLFS_NO_ERROR \ + -DLFS_READONLY" \ + make lfs.code.csv lfs.data.csv lfs.stack.csv lfs.structs.csv + ./scripts/structs.py -u lfs.structs.csv + ./scripts/summary.py lfs.code.csv lfs.data.csv lfs.stack.csv \ + -bfunction \ + -fcode=code_size \ + -fdata=data_size \ + -fstack=stack_limit --max=stack_limit + mkdir -p sizes + cp lfs.code.csv sizes/${{matrix.arch}}-readonly.code.csv + cp lfs.data.csv sizes/${{matrix.arch}}-readonly.data.csv + cp lfs.stack.csv sizes/${{matrix.arch}}-readonly.stack.csv + cp lfs.structs.csv sizes/${{matrix.arch}}-readonly.structs.csv + - name: sizes-threadsafe run: | make clean - make test TESTFLAGS+="-nrk \ - -DLFS_READ_SIZE=1 -DLFS_CACHE_SIZE=1" - - name: test-block-cycles + CFLAGS="$CFLAGS \ + -DLFS_NO_ASSERT \ + -DLFS_NO_DEBUG \ + -DLFS_NO_WARN \ + -DLFS_NO_ERROR \ + -DLFS_THREADSAFE" \ + make lfs.code.csv lfs.data.csv lfs.stack.csv lfs.structs.csv + ./scripts/structs.py -u lfs.structs.csv + ./scripts/summary.py lfs.code.csv lfs.data.csv lfs.stack.csv \ + -bfunction \ + -fcode=code_size \ + -fdata=data_size \ + -fstack=stack_limit --max=stack_limit + mkdir -p sizes + cp lfs.code.csv sizes/${{matrix.arch}}-threadsafe.code.csv + cp lfs.data.csv sizes/${{matrix.arch}}-threadsafe.data.csv + cp lfs.stack.csv sizes/${{matrix.arch}}-threadsafe.stack.csv + cp lfs.structs.csv sizes/${{matrix.arch}}-threadsafe.structs.csv + - name: sizes-multiversion run: | make clean - make test TESTFLAGS+="-nrk \ - -DLFS_BLOCK_CYCLES=1" - - name: test-odd-block-count + CFLAGS="$CFLAGS \ + -DLFS_NO_ASSERT \ + -DLFS_NO_DEBUG \ + -DLFS_NO_WARN \ + -DLFS_NO_ERROR \ + -DLFS_MULTIVERSION" \ + make lfs.code.csv lfs.data.csv lfs.stack.csv lfs.structs.csv + ./scripts/structs.py -u lfs.structs.csv + ./scripts/summary.py lfs.code.csv lfs.data.csv lfs.stack.csv \ + -bfunction \ + -fcode=code_size \ + -fdata=data_size \ + -fstack=stack_limit --max=stack_limit + mkdir -p sizes + cp lfs.code.csv sizes/${{matrix.arch}}-multiversion.code.csv + cp lfs.data.csv sizes/${{matrix.arch}}-multiversion.data.csv + cp lfs.stack.csv sizes/${{matrix.arch}}-multiversion.stack.csv + cp lfs.structs.csv sizes/${{matrix.arch}}-multiversion.structs.csv + - name: sizes-migrate run: | make clean - make test TESTFLAGS+="-nrk \ - -DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD_SIZE=256" - - name: test-odd-block-size + CFLAGS="$CFLAGS \ + -DLFS_NO_ASSERT \ + -DLFS_NO_DEBUG \ + -DLFS_NO_WARN \ + -DLFS_NO_ERROR \ + -DLFS_MIGRATE" \ + make lfs.code.csv lfs.data.csv lfs.stack.csv lfs.structs.csv + ./scripts/structs.py -u lfs.structs.csv + ./scripts/summary.py lfs.code.csv lfs.data.csv lfs.stack.csv \ + -bfunction \ + -fcode=code_size \ + -fdata=data_size \ + -fstack=stack_limit --max=stack_limit + mkdir -p sizes + cp lfs.code.csv sizes/${{matrix.arch}}-migrate.code.csv + cp lfs.data.csv sizes/${{matrix.arch}}-migrate.data.csv + cp lfs.stack.csv sizes/${{matrix.arch}}-migrate.stack.csv + cp lfs.structs.csv sizes/${{matrix.arch}}-migrate.structs.csv + - name: sizes-error-asserts run: | make clean - make test TESTFLAGS+="-nrk \ - -DLFS_READ_SIZE=11 -DLFS_BLOCK_SIZE=704" + CFLAGS="$CFLAGS \ + -DLFS_NO_DEBUG \ + -DLFS_NO_WARN \ + -DLFS_NO_ERROR \ + -D'LFS_ASSERT(test)=do {if(!(test)) {return -1;}} while(0)'" \ + make lfs.code.csv lfs.data.csv lfs.stack.csv lfs.structs.csv + ./scripts/structs.py -u lfs.structs.csv + ./scripts/summary.py lfs.code.csv lfs.data.csv lfs.stack.csv \ + -bfunction \ + -fcode=code_size \ + -fdata=data_size \ + -fstack=stack_limit --max=stack_limit + mkdir -p sizes + cp lfs.code.csv sizes/${{matrix.arch}}-error-asserts.code.csv + cp lfs.data.csv sizes/${{matrix.arch}}-error-asserts.data.csv + cp lfs.stack.csv sizes/${{matrix.arch}}-error-asserts.stack.csv + cp lfs.structs.csv sizes/${{matrix.arch}}-error-asserts.structs.csv - # upload coverage for later coverage - - name: upload-coverage + # create size statuses + - name: upload-sizes uses: actions/upload-artifact@v2 with: - name: coverage - path: coverage - retention-days: 1 - - # update results - - name: results + name: sizes + path: sizes + - name: status-sizes run: | - mkdir -p results - make clean - make lfs.csv \ - CFLAGS+=" \ - -DLFS_NO_ASSERT \ - -DLFS_NO_DEBUG \ - -DLFS_NO_WARN \ - -DLFS_NO_ERROR" - cp lfs.csv results/${{matrix.arch}}.csv - ./scripts/summary.py results/${{matrix.arch}}.csv - - name: results-readonly - run: | - mkdir -p results - make clean - make lfs.csv \ - CFLAGS+=" \ - -DLFS_NO_ASSERT \ - -DLFS_NO_DEBUG \ - -DLFS_NO_WARN \ - -DLFS_NO_ERROR \ - -DLFS_READONLY" - cp lfs.csv results/${{matrix.arch}}-readonly.csv - ./scripts/summary.py results/${{matrix.arch}}-readonly.csv - - name: results-threadsafe - run: | - mkdir -p results - make clean - make lfs.csv \ - CFLAGS+=" \ - -DLFS_NO_ASSERT \ - -DLFS_NO_DEBUG \ - -DLFS_NO_WARN \ - -DLFS_NO_ERROR \ - -DLFS_THREADSAFE" - cp lfs.csv results/${{matrix.arch}}-threadsafe.csv - ./scripts/summary.py results/${{matrix.arch}}-threadsafe.csv - - name: results-migrate - run: | - mkdir -p results - make clean - make lfs.csv \ - CFLAGS+=" \ - -DLFS_NO_ASSERT \ - -DLFS_NO_DEBUG \ - -DLFS_NO_WARN \ - -DLFS_NO_ERROR \ - -DLFS_MIGRATE" - cp lfs.csv results/${{matrix.arch}}-migrate.csv - ./scripts/summary.py results/${{matrix.arch}}-migrate.csv - - name: results-error-asserts - run: | - mkdir -p results - make clean - make lfs.csv \ - CFLAGS+=" \ - -DLFS_NO_DEBUG \ - -DLFS_NO_WARN \ - -DLFS_NO_ERROR \ - -D'LFS_ASSERT(test)=do {if(!(test)) {return -1;}} while(0)'" - cp lfs.csv results/${{matrix.arch}}-error-asserts.csv - ./scripts/summary.py results/${{matrix.arch}}-error-asserts.csv - - name: upload-results + mkdir -p status + for f in $(shopt -s nullglob ; echo sizes/*.csv) + do + # skip .data.csv as it should always be zero + [[ $f == *.data.csv ]] && continue + export STEP="sizes$(echo $f \ + | sed -n 's/[^-.]*-\([^.]*\)\..*csv/-\1/p')" + export CONTEXT="sizes (${{matrix.arch}}$(echo $f \ + | sed -n 's/[^-.]*-\([^.]*\)\..*csv/, \1/p')) / $(echo $f \ + | sed -n 's/[^.]*\.\(.*\)\.csv/\1/p')" + export PREV="$(curl -sS \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/status/master` + `?per_page=100" \ + | jq -re 'select(.sha != env.GITHUB_SHA) | .statuses[] + | select(.context == env.CONTEXT).description + | capture("(?[0-9∞]+)").prev' \ + || echo 0)" + export DESCRIPTION="$(./scripts/summary.py $f --max=stack_limit -Y \ + | awk ' + NR==2 {$1=0; printf "%s B",$NF} + NR==2 && ENVIRON["PREV"]+0 != 0 { + printf " (%+.1f%%)",100*($NF-ENVIRON["PREV"])/ENVIRON["PREV"] + }')" + jq -n '{ + state: "success", + context: env.CONTEXT, + description: env.DESCRIPTION, + target_job: "${{github.job}} (${{matrix.arch}})", + target_step: env.STEP, + }' | tee status/$(basename $f .csv).json + done + - name: upload-status-sizes uses: actions/upload-artifact@v2 with: - name: results - path: results + name: status + path: status + retention-days: 1 - # create statuses with results - - name: collect-status + # create cov statuses + - name: upload-cov + if: ${{matrix.arch == 'x86_64'}} + uses: actions/upload-artifact@v2 + with: + name: cov + path: cov + - name: status-cov + if: ${{matrix.arch == 'x86_64'}} run: | mkdir -p status - for f in $(shopt -s nullglob ; echo results/*.csv) + f=cov/cov.csv + for s in lines branches do - export STEP="results$( - echo $f | sed -n 's/[^-]*-\(.*\).csv/-\1/p')" - for r in code stack structs - do - export CONTEXT="results (${{matrix.arch}}$( - echo $f | sed -n 's/[^-]*-\(.*\).csv/, \1/p')) / $r" - export PREV="$(curl -sS \ - "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/status/master?per_page=100" \ - | jq -re 'select(.sha != env.GITHUB_SHA) | .statuses[] - | select(.context == env.CONTEXT).description - | capture("(?[0-9∞]+)").result' \ - || echo 0)" - export DESCRIPTION="$(./scripts/summary.py $f -f $r -Y | awk ' - NR==2 {printf "%s B",$2} + export STEP="cov" + export CONTEXT="cov / $s" + export PREV="$(curl -sS \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/status/master` + `?per_page=100" \ + | jq -re 'select(.sha != env.GITHUB_SHA) | .statuses[] + | select(.context == env.CONTEXT).description + | capture("(?[0-9]+)/(?[0-9]+)") + | 100*((.prev_a|tonumber) / (.prev_b|tonumber))' \ + || echo 0)" + export DESCRIPTION="$(./scripts/cov.py -u $f -f$s -Y \ + | awk -F '[ /%]+' -v s=$s ' + NR==2 {$1=0; printf "%d/%d %s",$2,$3,s} NR==2 && ENVIRON["PREV"]+0 != 0 { - printf " (%+.1f%%)",100*($2-ENVIRON["PREV"])/ENVIRON["PREV"]}')" - jq -n '{ - state: "success", - context: env.CONTEXT, - description: env.DESCRIPTION, - target_job: "${{github.job}} (${{matrix.arch}})", - target_step: env.STEP}' \ - | tee status/$r-${{matrix.arch}}$( - echo $f | sed -n 's/[^-]*-\(.*\).csv/-\1/p').json - done + printf " (%+.1f%%)",$4-ENVIRON["PREV"] + }')" + jq -n '{ + state: "success", + context: env.CONTEXT, + description: env.DESCRIPTION, + target_job: "${{github.job}} (${{matrix.arch}})", + target_step: env.STEP, + }' | tee status/$(basename $f .csv)-$s.json done - - name: upload-status + - name: upload-status-sizes + if: ${{matrix.arch == 'x86_64'}} uses: actions/upload-artifact@v2 with: name: status path: status retention-days: 1 - # run under Valgrind to check for memory errors - valgrind: - runs-on: ubuntu-20.04 + # run as many exhaustive tests as fits in GitHub's time limits + # + # this grows exponentially, so it doesn't turn out to be that many + test-pls: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + pls: [1, 2] + steps: - uses: actions/checkout@v2 - name: install run: | - # need toml, also pip3 isn't installed by default? + # need a few things sudo apt-get update -qq - sudo apt-get install -qq python3 python3-pip - sudo pip3 install toml - - name: install-valgrind + sudo apt-get install -qq gcc python3 python3-pip + pip3 install toml + gcc --version + python3 --version + - name: test-pls + if: ${{matrix.pls <= 1}} + run: | + TESTFLAGS="$TESTFLAGS -P${{matrix.pls}}" make test + # >=2pls takes multiple days to run fully, so we can only + # run a subset of tests, these are the most important + - name: test-limited-pls + if: ${{matrix.pls > 1}} + run: | + TESTFLAGS="$TESTFLAGS -P${{matrix.pls}} test_dirs test_relocations" \ + make test + + # run with LFS_NO_INTRINSICS to make sure that works + test-no-intrinsics: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + - name: install + run: | + # need a few things + sudo apt-get update -qq + sudo apt-get install -qq gcc python3 python3-pip + pip3 install toml + gcc --version + python3 --version + - name: test-no-intrinsics + run: | + CFLAGS="$CFLAGS -DLFS_NO_INTRINSICS" make test + + # run LFS_MULTIVERSION tests + test-multiversion: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + - name: install + run: | + # need a few things + sudo apt-get update -qq + sudo apt-get install -qq gcc python3 python3-pip + pip3 install toml + gcc --version + python3 --version + - name: test-multiversion run: | + CFLAGS="$CFLAGS -DLFS_MULTIVERSION" make test + + # run tests on the older version lfs2.0 + test-lfs2_0: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + - name: install + run: | + # need a few things sudo apt-get update -qq - sudo apt-get install -qq valgrind + sudo apt-get install -qq gcc python3 python3-pip + pip3 install toml + gcc --version + python3 --version + - name: test-lfs2_0 + run: | + CFLAGS="$CFLAGS -DLFS_MULTIVERSION" \ + TESTFLAGS="$TESTFLAGS -DDISK_VERSION=0x00020000" \ + make test + + # run under Valgrind to check for memory errors + test-valgrind: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + - name: install + run: | + # need a few things + sudo apt-get update -qq + sudo apt-get install -qq gcc python3 python3-pip valgrind + pip3 install toml + gcc --version + python3 --version valgrind --version - # normal tests, we don't need to test all geometries + # Valgrind takes a while with diminishing value, so only test + # on one geometry - name: test-valgrind - run: make test TESTFLAGS+="-k --valgrind" + run: | + TESTFLAGS="$TESTFLAGS --valgrind --context=1024 -Gdefault -Pnone" \ + make test # test that compilation is warning free under clang - clang: - runs-on: ubuntu-20.04 + # run with Clang, mostly to check for Clang-specific warnings + test-clang: + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 - name: install run: | - # need toml, also pip3 isn't installed by default? + # need a few things + sudo apt-get install -qq clang python3 python3-pip + pip3 install toml + clang --version + python3 --version + - name: test-clang + run: | + # override CFLAGS since Clang does not support -fcallgraph-info + # and -ftrack-macro-expansions + make \ + CC=clang \ + CFLAGS="$CFLAGS -MMD -g3 -I. -std=c99 -Wall -Wextra -pedantic" \ + test + + # run benchmarks + # + # note there's no real benefit to running these on multiple archs + bench: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + - name: install + run: | + # need a few things sudo apt-get update -qq - sudo apt-get install -qq python3 python3-pip - sudo pip3 install toml - - name: install-clang + sudo apt-get install -qq gcc python3 python3-pip valgrind + pip3 install toml + gcc --version + python3 --version + valgrind --version + - name: bench + run: | + make bench + + # find bench results + make lfs.bench.csv + ./scripts/summary.py lfs.bench.csv \ + -bsuite \ + -freaded=bench_readed \ + -fproged=bench_proged \ + -ferased=bench_erased + mkdir -p bench + cp lfs.bench.csv bench/bench.csv + + # find perfbd results + make lfs.perfbd.csv + ./scripts/perfbd.py -u lfs.perfbd.csv + mkdir -p bench + cp lfs.perfbd.csv bench/perfbd.csv + + # create bench statuses + - name: upload-bench + uses: actions/upload-artifact@v2 + with: + name: bench + path: bench + - name: status-bench + run: | + mkdir -p status + f=bench/bench.csv + for s in readed proged erased + do + export STEP="bench" + export CONTEXT="bench / $s" + export PREV="$(curl -sS \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/status/master` + `?per_page=100" \ + | jq -re 'select(.sha != env.GITHUB_SHA) | .statuses[] + | select(.context == env.CONTEXT).description + | capture("(?[0-9]+)").prev' \ + || echo 0)" + export DESCRIPTION="$(./scripts/summary.py $f -f$s=bench_$s -Y \ + | awk ' + NR==2 {$1=0; printf "%s B",$NF} + NR==2 && ENVIRON["PREV"]+0 != 0 { + printf " (%+.1f%%)",100*($NF-ENVIRON["PREV"])/ENVIRON["PREV"] + }')" + jq -n '{ + state: "success", + context: env.CONTEXT, + description: env.DESCRIPTION, + target_job: "${{github.job}}", + target_step: env.STEP, + }' | tee status/$(basename $f .csv)-$s.json + done + - name: upload-status-bench + uses: actions/upload-artifact@v2 + with: + name: status + path: status + retention-days: 1 + + # run compatibility tests using the current master as the previous version + test-compat: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + if: ${{github.event_name == 'pull_request'}} + # checkout the current pr target into lfsp + - uses: actions/checkout@v2 + if: ${{github.event_name == 'pull_request'}} + with: + ref: ${{github.event.pull_request.base.ref}} + path: lfsp + - name: install + if: ${{github.event_name == 'pull_request'}} run: | + # need a few things sudo apt-get update -qq - sudo apt-get install -qq clang - echo "CC=clang" >> $GITHUB_ENV - clang --version - # no reason to not test again - - name: test-clang - run: make test TESTFLAGS+="-k" + sudo apt-get install -qq gcc python3 python3-pip + pip3 install toml + gcc --version + python3 --version + # adjust prefix of lfsp + - name: changeprefix + if: ${{github.event_name == 'pull_request'}} + run: | + ./scripts/changeprefix.py lfs lfsp lfsp/*.h lfsp/*.c + - name: test-compat + if: ${{github.event_name == 'pull_request'}} + run: | + TESTS=tests/test_compat.toml \ + SRC="$(find . lfsp -name '*.c' -maxdepth 1 \ + -and -not -name '*.t.*' \ + -and -not -name '*.b.*')" \ + CFLAGS="-DLFSP=lfsp/lfsp.h" \ + make test # self-host with littlefs-fuse for a fuzz-like test fuse: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 if: ${{!endsWith(github.ref, '-prefix')}} steps: - uses: actions/checkout@v2 - name: install run: | - # need toml, also pip3 isn't installed by default? + # need a few things sudo apt-get update -qq - sudo apt-get install -qq python3 python3-pip libfuse-dev + sudo apt-get install -qq gcc python3 python3-pip libfuse-dev sudo pip3 install toml - fusermount -V gcc --version + python3 --version + fusermount -V - uses: actions/checkout@v2 with: repository: littlefs-project/littlefs-fuse @@ -359,22 +614,24 @@ jobs: cd mount/littlefs stat . ls -flh + make -B test-runner make -B test # test migration using littlefs-fuse migrate: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 if: ${{!endsWith(github.ref, '-prefix')}} steps: - uses: actions/checkout@v2 - name: install run: | - # need toml, also pip3 isn't installed by default? + # need a few things sudo apt-get update -qq - sudo apt-get install -qq python3 python3-pip libfuse-dev + sudo apt-get install -qq gcc python3 python3-pip libfuse-dev sudo pip3 install toml - fusermount -V gcc --version + python3 --version + fusermount -V - uses: actions/checkout@v2 with: repository: littlefs-project/littlefs-fuse @@ -414,6 +671,7 @@ jobs: cd mount/littlefs stat . ls -flh + make -B test-runner make -B test # attempt to migrate @@ -428,66 +686,185 @@ jobs: cd mount/littlefs stat . ls -flh + make -B test-runner make -B test - # collect coverage info - coverage: - runs-on: ubuntu-20.04 - needs: [test] + # status related tasks that run after tests + status: + runs-on: ubuntu-22.04 + needs: [test, bench] steps: - uses: actions/checkout@v2 + if: ${{github.event_name == 'pull_request'}} - name: install + if: ${{github.event_name == 'pull_request'}} run: | - sudo apt-get update -qq - sudo apt-get install -qq python3 python3-pip lcov - sudo pip3 install toml - # yes we continue-on-error nearly every step, continue-on-error - # at job level apparently still marks a job as failed, which isn't - # what we want + # need a few things + sudo apt-get install -qq gcc python3 python3-pip + pip3 install toml + gcc --version + python3 --version - uses: actions/download-artifact@v2 + if: ${{github.event_name == 'pull_request'}} continue-on-error: true with: - name: coverage - path: coverage - - name: results-coverage + name: sizes + path: sizes + - uses: actions/download-artifact@v2 + if: ${{github.event_name == 'pull_request'}} continue-on-error: true - run: | - mkdir -p results - lcov $(for f in coverage/*.info ; do echo "-a $f" ; done) \ - -o results/coverage.info - ./scripts/coverage.py results/coverage.info -o results/coverage.csv - - name: upload-results - uses: actions/upload-artifact@v2 with: - name: results - path: results - - name: collect-status - run: | - mkdir -p status - [ -e results/coverage.csv ] || exit 0 - export STEP="results-coverage" - export CONTEXT="results / coverage" - export PREV="$(curl -sS \ - "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/status/master?per_page=100" \ - | jq -re 'select(.sha != env.GITHUB_SHA) | .statuses[] - | select(.context == env.CONTEXT).description - | capture("(?[0-9\\.]+)").result' \ - || echo 0)" - export DESCRIPTION="$( - ./scripts/coverage.py -u results/coverage.csv -Y | awk -F '[ /%]+' ' - NR==2 {printf "%.1f%% of %d lines",$4,$3} - NR==2 && ENVIRON["PREV"]+0 != 0 { - printf " (%+.1f%%)",$4-ENVIRON["PREV"]}')" - jq -n '{ - state: "success", - context: env.CONTEXT, - description: env.DESCRIPTION, - target_job: "${{github.job}}", - target_step: env.STEP}' \ - | tee status/coverage.json - - name: upload-status + name: cov + path: cov + - uses: actions/download-artifact@v2 + if: ${{github.event_name == 'pull_request'}} + continue-on-error: true + with: + name: bench + path: bench + + # try to find results from tests + - name: create-table + if: ${{github.event_name == 'pull_request'}} + run: | + # compare against pull-request target + curl -sS \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/status/` + `${{github.event.pull_request.base.ref}}` + `?per_page=100" \ + | jq -re 'select(.sha != env.GITHUB_SHA) | .statuses[]' \ + >> prev-status.json \ + || true + + # build table for GitHub + declare -A table + + # sizes table + i=0 + j=0 + for c in "" readonly threadsafe multiversion migrate error-asserts + do + # per-config results + c_or_default=${c:-default} + c_camel=${c_or_default^} + table[$i,$j]=$c_camel + ((j+=1)) + + for s in code stack structs + do + f=sizes/thumb${c:+-$c}.$s.csv + [ -e $f ] && table[$i,$j]=$( \ + export PREV="$(jq -re ' + select(.context == "'"sizes (thumb${c:+, $c}) / $s"'").description + | capture("(?[0-9∞]+)").prev' \ + prev-status.json || echo 0)" + ./scripts/summary.py $f --max=stack_limit -Y \ + | awk ' + NR==2 {$1=0; printf "%s B",$NF} + NR==2 && ENVIRON["PREV"]+0 != 0 { + printf " (%+.1f%%)",100*($NF-ENVIRON["PREV"])/ENVIRON["PREV"] + }' \ + | sed -e 's/ /\ /g') + ((j+=1)) + done + ((j=0, i+=1)) + done + + # coverage table + i=0 + j=4 + for s in lines branches + do + table[$i,$j]=${s^} + ((j+=1)) + + f=cov/cov.csv + [ -e $f ] && table[$i,$j]=$( \ + export PREV="$(jq -re ' + select(.context == "'"cov / $s"'").description + | capture("(?[0-9]+)/(?[0-9]+)") + | 100*((.prev_a|tonumber) / (.prev_b|tonumber))' \ + prev-status.json || echo 0)" + ./scripts/cov.py -u $f -f$s -Y \ + | awk -F '[ /%]+' -v s=$s ' + NR==2 {$1=0; printf "%d/%d %s",$2,$3,s} + NR==2 && ENVIRON["PREV"]+0 != 0 { + printf " (%+.1f%%)",$4-ENVIRON["PREV"] + }' \ + | sed -e 's/ /\ /g') + ((j=4, i+=1)) + done + + # benchmark table + i=3 + j=4 + for s in readed proged erased + do + table[$i,$j]=${s^} + ((j+=1)) + + f=bench/bench.csv + [ -e $f ] && table[$i,$j]=$( \ + export PREV="$(jq -re ' + select(.context == "'"bench / $s"'").description + | capture("(?[0-9]+)").prev' \ + prev-status.json || echo 0)" + ./scripts/summary.py $f -f$s=bench_$s -Y \ + | awk ' + NR==2 {$1=0; printf "%s B",$NF} + NR==2 && ENVIRON["PREV"]+0 != 0 { + printf " (%+.1f%%)",100*($NF-ENVIRON["PREV"])/ENVIRON["PREV"] + }' \ + | sed -e 's/ /\ /g') + ((j=4, i+=1)) + done + + # build the actual table + echo "| | Code | Stack | Structs | | Coverage |" >> table.txt + echo "|:--|-----:|------:|--------:|:--|---------:|" >> table.txt + for ((i=0; i<6; i++)) + do + echo -n "|" >> table.txt + for ((j=0; j<6; j++)) + do + echo -n " " >> table.txt + [[ i -eq 2 && j -eq 5 ]] && echo -n "**Benchmarks**" >> table.txt + echo -n "${table[$i,$j]:-}" >> table.txt + echo -n " |" >> table.txt + done + echo >> table.txt + done + + cat table.txt + + # create a bot comment for successful runs on pull requests + - name: create-comment + if: ${{github.event_name == 'pull_request'}} + run: | + touch comment.txt + echo "
" >> comment.txt + echo "" >> comment.txt + echo "Tests passed ✓, ` + `Code: $(awk 'NR==3 {print $4}' table.txt || true), ` + `Stack: $(awk 'NR==3 {print $6}' table.txt || true), ` + `Structs: $(awk 'NR==3 {print $8}' table.txt || true)" \ + >> comment.txt + echo "" >> comment.txt + echo >> comment.txt + [ -e table.txt ] && cat table.txt >> comment.txt + echo >> comment.txt + echo "
" >> comment.txt + cat comment.txt + + mkdir -p comment + jq -n --rawfile comment comment.txt '{ + number: ${{github.event.number}}, + body: $comment, + }' | tee comment/comment.json + - name: upload-comment uses: actions/upload-artifact@v2 with: - name: status - path: status + name: comment + path: comment retention-days: 1 + diff --git a/components/fs/littlefs/littlefs/.gitignore b/components/fs/littlefs/littlefs/.gitignore index 3f7b860ed..09707c6bd 100644 --- a/components/fs/littlefs/littlefs/.gitignore +++ b/components/fs/littlefs/littlefs/.gitignore @@ -4,11 +4,31 @@ *.a *.ci *.csv +*.t.* +*.b.* +*.gcno +*.gcda +*.perf +lfs +liblfs.a # Testing things -blocks/ -lfs -test.c -tests/*.toml.* -scripts/__pycache__ +runners/test_runner +runners/bench_runner +lfs.code.csv +lfs.data.csv +lfs.stack.csv +lfs.structs.csv +lfs.cov.csv +lfs.perf.csv +lfs.perfbd.csv +lfs.test.csv +lfs.bench.csv + +# Misc +tags .gdb_history +scripts/__pycache__ + +# Historical, probably should remove at some point +tests/*.toml.* diff --git a/components/fs/littlefs/littlefs/DESIGN.md b/components/fs/littlefs/littlefs/DESIGN.md index 1d02ba3bf..9c9703a6c 100644 --- a/components/fs/littlefs/littlefs/DESIGN.md +++ b/components/fs/littlefs/littlefs/DESIGN.md @@ -59,7 +59,7 @@ This leaves us with three major requirements for an embedded filesystem. RAM to temporarily store filesystem metadata. For ROM, this means we need to keep our design simple and reuse code paths - were possible. For RAM we have a stronger requirement, all RAM usage is + where possible. For RAM we have a stronger requirement, all RAM usage is bounded. This means RAM usage does not grow as the filesystem changes in size or number of files. This creates a unique challenge as even presumably simple operations, such as traversing the filesystem, become surprisingly @@ -626,7 +626,7 @@ log₂_n_ pointers that skip to different preceding elements of the skip-list. The name comes from heavy use of the [CTZ instruction][wikipedia-ctz], which -lets us calculate the power-of-two factors efficiently. For a give block _n_, +lets us calculate the power-of-two factors efficiently. For a given block _n_, that block contains ctz(_n_)+1 pointers. ``` diff --git a/components/fs/littlefs/littlefs/Makefile b/components/fs/littlefs/littlefs/Makefile index 138793363..51e9f98ed 100644 --- a/components/fs/littlefs/littlefs/Makefile +++ b/components/fs/littlefs/littlefs/Makefile @@ -1,172 +1,585 @@ -ifdef BUILDDIR -# make sure BUILDDIR ends with a slash -override BUILDDIR := $(BUILDDIR)/ -# bit of a hack, but we want to make sure BUILDDIR directory structure -# is correct before any commands -$(if $(findstring n,$(MAKEFLAGS)),, $(shell mkdir -p \ - $(BUILDDIR) \ - $(BUILDDIR)bd \ - $(BUILDDIR)tests)) -endif - +# overrideable build dir, default is in-place +BUILDDIR ?= . # overridable target/src/tools/flags/etc ifneq ($(wildcard test.c main.c),) -TARGET ?= $(BUILDDIR)lfs +TARGET ?= $(BUILDDIR)/lfs else -TARGET ?= $(BUILDDIR)lfs.a +TARGET ?= $(BUILDDIR)/liblfs.a endif -CC ?= gcc -AR ?= ar -SIZE ?= size -CTAGS ?= ctags -NM ?= nm -OBJDUMP ?= objdump -LCOV ?= lcov +CC ?= gcc +AR ?= ar +SIZE ?= size +CTAGS ?= ctags +NM ?= nm +OBJDUMP ?= objdump +VALGRIND ?= valgrind +GDB ?= gdb +PERF ?= perf -SRC ?= $(wildcard *.c) -OBJ := $(SRC:%.c=$(BUILDDIR)%.o) -DEP := $(SRC:%.c=$(BUILDDIR)%.d) -ASM := $(SRC:%.c=$(BUILDDIR)%.s) -CGI := $(SRC:%.c=$(BUILDDIR)%.ci) +SRC ?= $(filter-out $(wildcard *.t.* *.b.*),$(wildcard *.c)) +OBJ := $(SRC:%.c=$(BUILDDIR)/%.o) +DEP := $(SRC:%.c=$(BUILDDIR)/%.d) +ASM := $(SRC:%.c=$(BUILDDIR)/%.s) +CI := $(SRC:%.c=$(BUILDDIR)/%.ci) +GCDA := $(SRC:%.c=$(BUILDDIR)/%.t.gcda) +TESTS ?= $(wildcard tests/*.toml) +TEST_SRC ?= $(SRC) \ + $(filter-out $(wildcard bd/*.t.* bd/*.b.*),$(wildcard bd/*.c)) \ + runners/test_runner.c +TEST_RUNNER ?= $(BUILDDIR)/runners/test_runner +TEST_A := $(TESTS:%.toml=$(BUILDDIR)/%.t.a.c) \ + $(TEST_SRC:%.c=$(BUILDDIR)/%.t.a.c) +TEST_C := $(TEST_A:%.t.a.c=%.t.c) +TEST_OBJ := $(TEST_C:%.t.c=%.t.o) +TEST_DEP := $(TEST_C:%.t.c=%.t.d) +TEST_CI := $(TEST_C:%.t.c=%.t.ci) +TEST_GCNO := $(TEST_C:%.t.c=%.t.gcno) +TEST_GCDA := $(TEST_C:%.t.c=%.t.gcda) +TEST_PERF := $(TEST_RUNNER:%=%.perf) +TEST_TRACE := $(TEST_RUNNER:%=%.trace) +TEST_CSV := $(TEST_RUNNER:%=%.csv) + +BENCHES ?= $(wildcard benches/*.toml) +BENCH_SRC ?= $(SRC) \ + $(filter-out $(wildcard bd/*.t.* bd/*.b.*),$(wildcard bd/*.c)) \ + runners/bench_runner.c +BENCH_RUNNER ?= $(BUILDDIR)/runners/bench_runner +BENCH_A := $(BENCHES:%.toml=$(BUILDDIR)/%.b.a.c) \ + $(BENCH_SRC:%.c=$(BUILDDIR)/%.b.a.c) +BENCH_C := $(BENCH_A:%.b.a.c=%.b.c) +BENCH_OBJ := $(BENCH_C:%.b.c=%.b.o) +BENCH_DEP := $(BENCH_C:%.b.c=%.b.d) +BENCH_CI := $(BENCH_C:%.b.c=%.b.ci) +BENCH_GCNO := $(BENCH_C:%.b.c=%.b.gcno) +BENCH_GCDA := $(BENCH_C:%.b.c=%.b.gcda) +BENCH_PERF := $(BENCH_RUNNER:%=%.perf) +BENCH_TRACE := $(BENCH_RUNNER:%=%.trace) +BENCH_CSV := $(BENCH_RUNNER:%=%.csv) + +CFLAGS += -fcallgraph-info=su +CFLAGS += -g3 +CFLAGS += -I. +CFLAGS += -std=c99 -Wall -Wextra -pedantic +CFLAGS += -Wmissing-prototypes +CFLAGS += -ftrack-macro-expansion=0 ifdef DEBUG -override CFLAGS += -O0 +CFLAGS += -O0 else -override CFLAGS += -Os +CFLAGS += -Os endif ifdef TRACE -override CFLAGS += -DLFS_YES_TRACE +CFLAGS += -DLFS_YES_TRACE +endif +ifdef YES_COV +CFLAGS += --coverage +endif +ifdef YES_PERF +CFLAGS += -fno-omit-frame-pointer +endif +ifdef YES_PERFBD +CFLAGS += -fno-omit-frame-pointer endif -override CFLAGS += -g3 -override CFLAGS += -I. -override CFLAGS += -std=c99 -Wall -Wextra -pedantic ifdef VERBOSE -override TESTFLAGS += -v -override CALLSFLAGS += -v -override CODEFLAGS += -v -override DATAFLAGS += -v -override STACKFLAGS += -v -override STRUCTSFLAGS += -v -override COVERAGEFLAGS += -v +CODEFLAGS += -v +DATAFLAGS += -v +STACKFLAGS += -v +STRUCTSFLAGS += -v +COVFLAGS += -v +PERFFLAGS += -v +PERFBDFLAGS += -v +endif +# forward -j flag +PERFFLAGS += $(filter -j%,$(MAKEFLAGS)) +PERFBDFLAGS += $(filter -j%,$(MAKEFLAGS)) +ifneq ($(NM),nm) +CODEFLAGS += --nm-path="$(NM)" +DATAFLAGS += --nm-path="$(NM)" +endif +ifneq ($(OBJDUMP),objdump) +CODEFLAGS += --objdump-path="$(OBJDUMP)" +DATAFLAGS += --objdump-path="$(OBJDUMP)" +STRUCTSFLAGS += --objdump-path="$(OBJDUMP)" +PERFFLAGS += --objdump-path="$(OBJDUMP)" +PERFBDFLAGS += --objdump-path="$(OBJDUMP)" +endif +ifneq ($(PERF),perf) +PERFFLAGS += --perf-path="$(PERF)" +endif + +TESTFLAGS += -b +BENCHFLAGS += -b +# forward -j flag +TESTFLAGS += $(filter -j%,$(MAKEFLAGS)) +BENCHFLAGS += $(filter -j%,$(MAKEFLAGS)) +ifdef YES_PERF +TESTFLAGS += -p $(TEST_PERF) +BENCHFLAGS += -p $(BENCH_PERF) +endif +ifdef YES_PERFBD +TESTFLAGS += -t $(TEST_TRACE) --trace-backtrace --trace-freq=100 +endif +ifndef NO_PERFBD +BENCHFLAGS += -t $(BENCH_TRACE) --trace-backtrace --trace-freq=100 +endif +ifdef YES_TESTMARKS +TESTFLAGS += -o $(TEST_CSV) +endif +ifndef NO_BENCHMARKS +BENCHFLAGS += -o $(BENCH_CSV) +endif +ifdef VERBOSE +TESTFLAGS += -v +TESTCFLAGS += -v +BENCHFLAGS += -v +BENCHCFLAGS += -v endif ifdef EXEC -override TESTFLAGS += --exec="$(EXEC)" +TESTFLAGS += --exec="$(EXEC)" +BENCHFLAGS += --exec="$(EXEC)" endif -ifdef COVERAGE -override TESTFLAGS += --coverage +ifneq ($(GDB),gdb) +TESTFLAGS += --gdb-path="$(GDB)" +BENCHFLAGS += --gdb-path="$(GDB)" endif -ifdef BUILDDIR -override TESTFLAGS += --build-dir="$(BUILDDIR:/=)" -override CALLSFLAGS += --build-dir="$(BUILDDIR:/=)" -override CODEFLAGS += --build-dir="$(BUILDDIR:/=)" -override DATAFLAGS += --build-dir="$(BUILDDIR:/=)" -override STACKFLAGS += --build-dir="$(BUILDDIR:/=)" -override STRUCTSFLAGS += --build-dir="$(BUILDDIR:/=)" -override COVERAGEFLAGS += --build-dir="$(BUILDDIR:/=)" +ifneq ($(VALGRIND),valgrind) +TESTFLAGS += --valgrind-path="$(VALGRIND)" +BENCHFLAGS += --valgrind-path="$(VALGRIND)" endif -ifneq ($(NM),nm) -override CODEFLAGS += --nm-tool="$(NM)" -override DATAFLAGS += --nm-tool="$(NM)" +ifneq ($(PERF),perf) +TESTFLAGS += --perf-path="$(PERF)" +BENCHFLAGS += --perf-path="$(PERF)" endif -ifneq ($(OBJDUMP),objdump) -override STRUCTSFLAGS += --objdump-tool="$(OBJDUMP)" + +# this is a bit of a hack, but we want to make sure the BUILDDIR +# directory structure is correct before we run any commands +ifneq ($(BUILDDIR),.) +$(if $(findstring n,$(MAKEFLAGS)),, $(shell mkdir -p \ + $(addprefix $(BUILDDIR)/,$(dir \ + $(SRC) \ + $(TESTS) \ + $(TEST_SRC) \ + $(BENCHES) \ + $(BENCH_SRC))))) endif # commands + +## Build littlefs .PHONY: all build all build: $(TARGET) +## Build assembly files .PHONY: asm asm: $(ASM) +## Find the total size .PHONY: size size: $(OBJ) $(SIZE) -t $^ +## Generate a ctags file .PHONY: tags tags: $(CTAGS) --totals --c-types=+p $(shell find -H -name '*.h') $(SRC) -.PHONY: calls -calls: $(CGI) - ./scripts/calls.py $^ $(CALLSFLAGS) - -.PHONY: test -test: - ./scripts/test.py $(TESTFLAGS) -.SECONDEXPANSION: -test%: tests/test$$(firstword $$(subst \#, ,%)).toml - ./scripts/test.py $@ $(TESTFLAGS) +## Show this help text +.PHONY: help +help: + @$(strip awk '/^## / { \ + sub(/^## /,""); \ + getline rule; \ + while (rule ~ /^(#|\.PHONY|ifdef|ifndef)/) getline rule; \ + gsub(/:.*/, "", rule); \ + printf " "" %-25s %s\n", rule, $$0 \ + }' $(MAKEFILE_LIST)) +## Find the per-function code size .PHONY: code -code: $(OBJ) - ./scripts/code.py $^ -S $(CODEFLAGS) +code: CODEFLAGS+=-S +code: $(OBJ) $(BUILDDIR)/lfs.code.csv + ./scripts/code.py $(OBJ) $(CODEFLAGS) + +## Compare per-function code size +.PHONY: code-diff +code-diff: $(OBJ) + ./scripts/code.py $^ $(CODEFLAGS) -d $(BUILDDIR)/lfs.code.csv +## Find the per-function data size .PHONY: data -data: $(OBJ) - ./scripts/data.py $^ -S $(DATAFLAGS) +data: DATAFLAGS+=-S +data: $(OBJ) $(BUILDDIR)/lfs.data.csv + ./scripts/data.py $(OBJ) $(DATAFLAGS) +## Compare per-function data size +.PHONY: data-diff +data-diff: $(OBJ) + ./scripts/data.py $^ $(DATAFLAGS) -d $(BUILDDIR)/lfs.data.csv + +## Find the per-function stack usage .PHONY: stack -stack: $(CGI) - ./scripts/stack.py $^ -S $(STACKFLAGS) +stack: STACKFLAGS+=-S +stack: $(CI) $(BUILDDIR)/lfs.stack.csv + ./scripts/stack.py $(CI) $(STACKFLAGS) + +## Compare per-function stack usage +.PHONY: stack-diff +stack-diff: $(CI) + ./scripts/stack.py $^ $(STACKFLAGS) -d $(BUILDDIR)/lfs.stack.csv +## Find function sizes +.PHONY: funcs +funcs: SUMMARYFLAGS+=-S +funcs: \ + $(BUILDDIR)/lfs.code.csv \ + $(BUILDDIR)/lfs.data.csv \ + $(BUILDDIR)/lfs.stack.csv + $(strip ./scripts/summary.py $^ \ + -bfunction \ + -fcode=code_size \ + -fdata=data_size \ + -fstack=stack_limit --max=stack \ + $(SUMMARYFLAGS)) + +## Compare function sizes +.PHONY: funcs-diff +funcs-diff: SHELL=/bin/bash +funcs-diff: $(OBJ) $(CI) + $(strip ./scripts/summary.py \ + <(./scripts/code.py $(OBJ) -q $(CODEFLAGS) -o-) \ + <(./scripts/data.py $(OBJ) -q $(DATAFLAGS) -o-) \ + <(./scripts/stack.py $(CI) -q $(STACKFLAGS) -o-) \ + -bfunction \ + -fcode=code_size \ + -fdata=data_size \ + -fstack=stack_limit --max=stack \ + $(SUMMARYFLAGS) -d <(./scripts/summary.py \ + $(BUILDDIR)/lfs.code.csv \ + $(BUILDDIR)/lfs.data.csv \ + $(BUILDDIR)/lfs.stack.csv \ + -q $(SUMMARYFLAGS) -o-)) + +## Find struct sizes .PHONY: structs -structs: $(OBJ) - ./scripts/structs.py $^ -S $(STRUCTSFLAGS) +structs: STRUCTSFLAGS+=-S +structs: $(OBJ) $(BUILDDIR)/lfs.structs.csv + ./scripts/structs.py $(OBJ) $(STRUCTSFLAGS) + +## Compare struct sizes +.PHONY: structs-diff +structs-diff: $(OBJ) + ./scripts/structs.py $^ $(STRUCTSFLAGS) -d $(BUILDDIR)/lfs.structs.csv + +## Find the line/branch coverage after a test run +.PHONY: cov +cov: COVFLAGS+=-s +cov: $(GCDA) $(BUILDDIR)/lfs.cov.csv + $(strip ./scripts/cov.py $(GCDA) \ + $(patsubst %,-F%,$(SRC)) \ + $(COVFLAGS)) + +## Compare line/branch coverage +.PHONY: cov-diff +cov-diff: $(GCDA) + $(strip ./scripts/cov.py $^ \ + $(patsubst %,-F%,$(SRC)) \ + $(COVFLAGS) -d $(BUILDDIR)/lfs.cov.csv) + +## Find the perf results after bench run with YES_PERF +.PHONY: perf +perf: PERFFLAGS+=-S +perf: $(BENCH_PERF) $(BUILDDIR)/lfs.perf.csv + $(strip ./scripts/perf.py $(BENCH_PERF) \ + $(patsubst %,-F%,$(SRC)) \ + $(PERFFLAGS)) + +## Compare perf results +.PHONY: perf-diff +perf-diff: $(BENCH_PERF) + $(strip ./scripts/perf.py $^ \ + $(patsubst %,-F%,$(SRC)) \ + $(PERFFLAGS) -d $(BUILDDIR)/lfs.perf.csv) + +## Find the perfbd results after a bench run +.PHONY: perfbd +perfbd: PERFBDFLAGS+=-S +perfbd: $(BENCH_TRACE) $(BUILDDIR)/lfs.perfbd.csv + $(strip ./scripts/perfbd.py $(BENCH_RUNNER) $(BENCH_TRACE) \ + $(patsubst %,-F%,$(SRC)) \ + $(PERFBDFLAGS)) + +## Compare perfbd results +.PHONY: perfbd-diff +perfbd-diff: $(BENCH_TRACE) + $(strip ./scripts/perfbd.py $(BENCH_RUNNER) $^ \ + $(patsubst %,-F%,$(SRC)) \ + $(PERFBDFLAGS) -d $(BUILDDIR)/lfs.perfbd.csv) + +## Find a summary of compile-time sizes +.PHONY: summary sizes +summary sizes: \ + $(BUILDDIR)/lfs.code.csv \ + $(BUILDDIR)/lfs.data.csv \ + $(BUILDDIR)/lfs.stack.csv \ + $(BUILDDIR)/lfs.structs.csv + $(strip ./scripts/summary.py $^ \ + -fcode=code_size \ + -fdata=data_size \ + -fstack=stack_limit --max=stack \ + -fstructs=struct_size \ + -Y $(SUMMARYFLAGS)) -.PHONY: coverage -coverage: - ./scripts/coverage.py $(BUILDDIR)tests/*.toml.info -s $(COVERAGEFLAGS) +## Compare compile-time sizes +.PHONY: summary-diff sizes-diff +summary-diff sizes-diff: SHELL=/bin/bash +summary-diff sizes-diff: $(OBJ) $(CI) + $(strip ./scripts/summary.py \ + <(./scripts/code.py $(OBJ) -q $(CODEFLAGS) -o-) \ + <(./scripts/data.py $(OBJ) -q $(DATAFLAGS) -o-) \ + <(./scripts/stack.py $(CI) -q $(STACKFLAGS) -o-) \ + <(./scripts/structs.py $(OBJ) -q $(STRUCTSFLAGS) -o-) \ + -fcode=code_size \ + -fdata=data_size \ + -fstack=stack_limit --max=stack \ + -fstructs=struct_size \ + -Y $(SUMMARYFLAGS) -d <(./scripts/summary.py \ + $(BUILDDIR)/lfs.code.csv \ + $(BUILDDIR)/lfs.data.csv \ + $(BUILDDIR)/lfs.stack.csv \ + $(BUILDDIR)/lfs.structs.csv \ + -q $(SUMMARYFLAGS) -o-)) + +## Build the test-runner +.PHONY: test-runner build-test +test-runner build-test: CFLAGS+=-Wno-missing-prototypes +ifndef NO_COV +test-runner build-test: CFLAGS+=--coverage +endif +ifdef YES_PERF +test-runner build-test: CFLAGS+=-fno-omit-frame-pointer +endif +ifdef YES_PERFBD +test-runner build-test: CFLAGS+=-fno-omit-frame-pointer +endif +# note we remove some binary dependent files during compilation, +# otherwise it's way to easy to end up with outdated results +test-runner build-test: $(TEST_RUNNER) +ifndef NO_COV + rm -f $(TEST_GCDA) +endif +ifdef YES_PERF + rm -f $(TEST_PERF) +endif +ifdef YES_PERFBD + rm -f $(TEST_TRACE) +endif + +## Run the tests, -j enables parallel tests +.PHONY: test +test: test-runner + ./scripts/test.py $(TEST_RUNNER) $(TESTFLAGS) + +## List the tests +.PHONY: test-list +test-list: test-runner + ./scripts/test.py $(TEST_RUNNER) $(TESTFLAGS) -l + +## Summarize the testmarks +.PHONY: testmarks +testmarks: SUMMARYFLAGS+=-spassed +testmarks: $(TEST_CSV) $(BUILDDIR)/lfs.test.csv + $(strip ./scripts/summary.py $(TEST_CSV) \ + -bsuite \ + -fpassed=test_passed \ + $(SUMMARYFLAGS)) + +## Compare testmarks against a previous run +.PHONY: testmarks-diff +testmarks-diff: $(TEST_CSV) + $(strip ./scripts/summary.py $^ \ + -bsuite \ + -fpassed=test_passed \ + $(SUMMARYFLAGS) -d $(BUILDDIR)/lfs.test.csv) + +## Build the bench-runner +.PHONY: bench-runner build-bench +bench-runner build-bench: CFLAGS+=-Wno-missing-prototypes +ifdef YES_COV +bench-runner build-bench: CFLAGS+=--coverage +endif +ifdef YES_PERF +bench-runner build-bench: CFLAGS+=-fno-omit-frame-pointer +endif +ifndef NO_PERFBD +bench-runner build-bench: CFLAGS+=-fno-omit-frame-pointer +endif +# note we remove some binary dependent files during compilation, +# otherwise it's way to easy to end up with outdated results +bench-runner build-bench: $(BENCH_RUNNER) +ifdef YES_COV + rm -f $(BENCH_GCDA) +endif +ifdef YES_PERF + rm -f $(BENCH_PERF) +endif +ifndef NO_PERFBD + rm -f $(BENCH_TRACE) +endif + +## Run the benchmarks, -j enables parallel benchmarks +.PHONY: bench +bench: bench-runner + ./scripts/bench.py $(BENCH_RUNNER) $(BENCHFLAGS) + +## List the benchmarks +.PHONY: bench-list +bench-list: bench-runner + ./scripts/bench.py $(BENCH_RUNNER) $(BENCHFLAGS) -l + +## Summarize the benchmarks +.PHONY: benchmarks +benchmarks: SUMMARYFLAGS+=-Serased -Sproged -Sreaded +benchmarks: $(BENCH_CSV) $(BUILDDIR)/lfs.bench.csv + $(strip ./scripts/summary.py $(BENCH_CSV) \ + -bsuite \ + -freaded=bench_readed \ + -fproged=bench_proged \ + -ferased=bench_erased \ + $(SUMMARYFLAGS)) + +## Compare benchmarks against a previous run +.PHONY: benchmarks-diff +benchmarks-diff: $(BENCH_CSV) + $(strip ./scripts/summary.py $^ \ + -bsuite \ + -freaded=bench_readed \ + -fproged=bench_proged \ + -ferased=bench_erased \ + $(SUMMARYFLAGS) -d $(BUILDDIR)/lfs.bench.csv) -.PHONY: summary -summary: $(BUILDDIR)lfs.csv - ./scripts/summary.py -Y $^ $(SUMMARYFLAGS) # rules -include $(DEP) +-include $(TEST_DEP) .SUFFIXES: +.SECONDARY: -$(BUILDDIR)lfs: $(OBJ) +$(BUILDDIR)/lfs: $(OBJ) $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ -$(BUILDDIR)lfs.a: $(OBJ) +$(BUILDDIR)/liblfs.a: $(OBJ) $(AR) rcs $@ $^ -$(BUILDDIR)lfs.csv: $(OBJ) $(CGI) - ./scripts/code.py $(OBJ) -q $(CODEFLAGS) -o $@ - ./scripts/data.py $(OBJ) -q -m $@ $(DATAFLAGS) -o $@ - ./scripts/stack.py $(CGI) -q -m $@ $(STACKFLAGS) -o $@ - ./scripts/structs.py $(OBJ) -q -m $@ $(STRUCTSFLAGS) -o $@ - $(if $(COVERAGE),\ - ./scripts/coverage.py $(BUILDDIR)tests/*.toml.info \ - -q -m $@ $(COVERAGEFLAGS) -o $@) +$(BUILDDIR)/lfs.code.csv: $(OBJ) + ./scripts/code.py $^ -q $(CODEFLAGS) -o $@ + +$(BUILDDIR)/lfs.data.csv: $(OBJ) + ./scripts/data.py $^ -q $(DATAFLAGS) -o $@ + +$(BUILDDIR)/lfs.stack.csv: $(CI) + ./scripts/stack.py $^ -q $(STACKFLAGS) -o $@ + +$(BUILDDIR)/lfs.structs.csv: $(OBJ) + ./scripts/structs.py $^ -q $(STRUCTSFLAGS) -o $@ + +$(BUILDDIR)/lfs.cov.csv: $(GCDA) + $(strip ./scripts/cov.py $^ \ + $(patsubst %,-F%,$(SRC)) \ + -q $(COVFLAGS) -o $@) + +$(BUILDDIR)/lfs.perf.csv: $(BENCH_PERF) + $(strip ./scripts/perf.py $^ \ + $(patsubst %,-F%,$(SRC)) \ + -q $(PERFFLAGS) -o $@) + +$(BUILDDIR)/lfs.perfbd.csv: $(BENCH_TRACE) + $(strip ./scripts/perfbd.py $(BENCH_RUNNER) $^ \ + $(patsubst %,-F%,$(SRC)) \ + -q $(PERFBDFLAGS) -o $@) + +$(BUILDDIR)/lfs.test.csv: $(TEST_CSV) + cp $^ $@ + +$(BUILDDIR)/lfs.bench.csv: $(BENCH_CSV) + cp $^ $@ -$(BUILDDIR)%.o: %.c - $(CC) -c -MMD $(CFLAGS) $< -o $@ +$(BUILDDIR)/runners/test_runner: $(TEST_OBJ) + $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ + +$(BUILDDIR)/runners/bench_runner: $(BENCH_OBJ) + $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ + +# our main build rule generates .o, .d, and .ci files, the latter +# used for stack analysis +$(BUILDDIR)/%.o $(BUILDDIR)/%.ci: %.c + $(CC) -c -MMD $(CFLAGS) $< -o $(BUILDDIR)/$*.o + +$(BUILDDIR)/%.o $(BUILDDIR)/%.ci: $(BUILDDIR)/%.c + $(CC) -c -MMD $(CFLAGS) $< -o $(BUILDDIR)/$*.o -$(BUILDDIR)%.s: %.c +$(BUILDDIR)/%.s: %.c $(CC) -S $(CFLAGS) $< -o $@ -# gcc depends on the output file for intermediate file names, so -# we can't omit to .o output. We also need to serialize with the -# normal .o rule because otherwise we can end up with multiprocess -# problems with two instances of gcc modifying the same .o -$(BUILDDIR)%.ci: %.c | $(BUILDDIR)%.o - $(CC) -c -MMD -fcallgraph-info=su $(CFLAGS) $< -o $| +$(BUILDDIR)/%.c: %.a.c + ./scripts/prettyasserts.py -p LFS_ASSERT $< -o $@ + +$(BUILDDIR)/%.c: $(BUILDDIR)/%.a.c + ./scripts/prettyasserts.py -p LFS_ASSERT $< -o $@ + +$(BUILDDIR)/%.t.a.c: %.toml + ./scripts/test.py -c $< $(TESTCFLAGS) -o $@ + +$(BUILDDIR)/%.t.a.c: %.c $(TESTS) + ./scripts/test.py -c $(TESTS) -s $< $(TESTCFLAGS) -o $@ + +$(BUILDDIR)/%.b.a.c: %.toml + ./scripts/bench.py -c $< $(BENCHCFLAGS) -o $@ + +$(BUILDDIR)/%.b.a.c: %.c $(BENCHES) + ./scripts/bench.py -c $(BENCHES) -s $< $(BENCHCFLAGS) -o $@ -# clean everything +## Clean everything .PHONY: clean clean: - rm -f $(BUILDDIR)lfs - rm -f $(BUILDDIR)lfs.a - rm -f $(BUILDDIR)lfs.csv + rm -f $(BUILDDIR)/lfs + rm -f $(BUILDDIR)/liblfs.a + rm -f $(BUILDDIR)/lfs.code.csv + rm -f $(BUILDDIR)/lfs.data.csv + rm -f $(BUILDDIR)/lfs.stack.csv + rm -f $(BUILDDIR)/lfs.structs.csv + rm -f $(BUILDDIR)/lfs.cov.csv + rm -f $(BUILDDIR)/lfs.perf.csv + rm -f $(BUILDDIR)/lfs.perfbd.csv + rm -f $(BUILDDIR)/lfs.test.csv + rm -f $(BUILDDIR)/lfs.bench.csv rm -f $(OBJ) - rm -f $(CGI) rm -f $(DEP) rm -f $(ASM) - rm -f $(BUILDDIR)tests/*.toml.* + rm -f $(CI) + rm -f $(TEST_RUNNER) + rm -f $(TEST_A) + rm -f $(TEST_C) + rm -f $(TEST_OBJ) + rm -f $(TEST_DEP) + rm -f $(TEST_CI) + rm -f $(TEST_GCNO) + rm -f $(TEST_GCDA) + rm -f $(TEST_PERF) + rm -f $(TEST_TRACE) + rm -f $(TEST_CSV) + rm -f $(BENCH_RUNNER) + rm -f $(BENCH_A) + rm -f $(BENCH_C) + rm -f $(BENCH_OBJ) + rm -f $(BENCH_DEP) + rm -f $(BENCH_CI) + rm -f $(BENCH_GCNO) + rm -f $(BENCH_GCDA) + rm -f $(BENCH_PERF) + rm -f $(BENCH_TRACE) + rm -f $(BENCH_CSV) diff --git a/components/fs/littlefs/littlefs/README.md b/components/fs/littlefs/littlefs/README.md index 32b3793f3..df7ee0032 100644 --- a/components/fs/littlefs/littlefs/README.md +++ b/components/fs/littlefs/littlefs/README.md @@ -226,6 +226,13 @@ License Identifiers that are here available: http://spdx.org/licenses/ to create images of the filesystem on your PC. Check if littlefs will fit your needs, create images for a later download to the target memory or inspect the content of a binary image of the target memory. + +- [littlefs2-rust] - A Rust wrapper for littlefs. This project allows you + to use littlefs in a Rust-friendly API, reaping the benefits of Rust's memory + safety and other guarantees. + +- [littlefs-disk-img-viewer] - A memory-efficient web application for viewing + littlefs disk images in your web browser. - [mklfs] - A command line tool built by the [Lua RTOS] guys for making littlefs images from a host PC. Supports Windows, Mac OS, and Linux. @@ -243,8 +250,16 @@ License Identifiers that are here available: http://spdx.org/licenses/ MCUs. It offers static wear-leveling and power-resilience with only a fixed _O(|address|)_ pointer structure stored on each block and in RAM. +- [ChaN's FatFs] - A lightweight reimplementation of the infamous FAT filesystem + for microcontroller-scale devices. Due to limitations of FAT it can't provide + power-loss resilience, but it does allow easy interop with PCs. + +- [chamelon] - A pure-OCaml implementation of (most of) littlefs, designed for + use with the MirageOS library operating system project. It is interoperable + with the reference implementation, with some caveats. [BSD-3-Clause]: https://spdx.org/licenses/BSD-3-Clause.html +[littlefs-disk-img-viewer]: https://github.com/tniessen/littlefs-disk-img-viewer [littlefs-fuse]: https://github.com/geky/littlefs-fuse [FUSE]: https://github.com/libfuse/libfuse [littlefs-js]: https://github.com/geky/littlefs-js @@ -255,4 +270,7 @@ License Identifiers that are here available: http://spdx.org/licenses/ [LittleFileSystem]: https://os.mbed.com/docs/mbed-os/latest/apis/littlefilesystem.html [SPIFFS]: https://github.com/pellepl/spiffs [Dhara]: https://github.com/dlbeer/dhara +[ChaN's FatFs]: http://elm-chan.org/fsw/ff/00index_e.html [littlefs-python]: https://pypi.org/project/littlefs-python/ +[littlefs2-rust]: https://crates.io/crates/littlefs2 +[chamelon]: https://github.com/yomimono/chamelon diff --git a/components/fs/littlefs/littlefs/SPEC.md b/components/fs/littlefs/littlefs/SPEC.md index 3663ea544..2370ea6d7 100644 --- a/components/fs/littlefs/littlefs/SPEC.md +++ b/components/fs/littlefs/littlefs/SPEC.md @@ -1,10 +1,10 @@ ## littlefs technical specification -This is the technical specification of the little filesystem. This document -covers the technical details of how the littlefs is stored on disk for -introspection and tooling. This document assumes you are familiar with the -design of the littlefs, for more info on how littlefs works check -out [DESIGN.md](DESIGN.md). +This is the technical specification of the little filesystem with on-disk +version lfs2.1. This document covers the technical details of how the littlefs +is stored on disk for introspection and tooling. This document assumes you are +familiar with the design of the littlefs, for more info on how littlefs works +check out [DESIGN.md](DESIGN.md). ``` | | | .---._____ @@ -133,12 +133,6 @@ tags XORed together, starting with `0xffffffff`. '-------------------' '-------------------' ``` -One last thing to note before we get into the details around tag encoding. Each -tag contains a valid bit used to indicate if the tag and containing commit is -valid. This valid bit is the first bit found in the tag and the commit and can -be used to tell if we've attempted to write to the remaining space in the -block. - Here's a more complete example of metadata block containing 4 entries: ``` @@ -191,6 +185,53 @@ Here's a more complete example of metadata block containing 4 entries: '---- most recent D ``` +Two things to note before we get into the details around tag encoding: + +1. Each tag contains a valid bit used to indicate if the tag and containing + commit is valid. After XORing, this bit should always be zero. + + At the end of each commit, the valid bit of the previous tag is XORed + with the lowest bit in the type field of the CRC tag. This allows + the CRC tag to force the next commit to fail the valid bit test if it + has not yet been written to. + +2. The valid bit alone is not enough info to know if the next commit has been + erased. We don't know the order bits will be programmed in a program block, + so it's possible that the next commit had an attempted program that left the + valid bit unchanged. + + To ensure we only ever program erased bytes, each commit can contain an + optional forward-CRC (FCRC). An FCRC contains a checksum of some amount of + bytes in the next commit at the time it was erased. + + ``` + .-------------------. \ \ + | revision count | | | + |-------------------| | | + | metadata | | | + | | +---. +-- current commit + | | | | | + |-------------------| | | | + | FCRC ---|-. | | + |-------------------| / | | | + | CRC -----|-' / + |-------------------| | + | padding | | padding (does't need CRC) + | | | + |-------------------| \ | \ + | erased? | +-' | + | | | | +-- next commit + | v | / | + | | / + | | + '-------------------' + ``` + + If the FCRC is missing or the checksum does not match, we must assume a + commit was attempted but failed due to power-loss. + + Note that end-of-block commits do not need an FCRC. + ## Metadata tags So in littlefs, 32-bit tags describe every type of metadata. And this means @@ -785,3 +826,41 @@ CRC fields: are made about the contents. --- +#### `0x5ff` LFS_TYPE_FCRC + +Added in lfs2.1, the optional FCRC tag contains a checksum of some amount of +bytes in the next commit at the time it was erased. This allows us to ensure +that we only ever program erased bytes, even if a previous commit failed due +to power-loss. + +When programming a commit, the FCRC size must be at least as large as the +program block size. However, the program block is not saved on disk, and can +change between mounts, so the FCRC size on disk may be different than the +current program block size. + +If the FCRC is missing or the checksum does not match, we must assume a +commit was attempted but failed due to power-loss. + +Layout of the FCRC tag: + +``` + tag data +[-- 32 --][-- 32 --|-- 32 --] +[1|- 11 -| 10 | 10 ][-- 32 --|-- 32 --] + ^ ^ ^ ^ ^- fcrc size ^- fcrc + | | | '- size (8) + | | '------ id (0x3ff) + | '------------ type (0x5ff) + '----------------- valid bit +``` + +FCRC fields: + +1. **FCRC size (32-bits)** - Number of bytes after this commit's CRC tag's + padding to include in the FCRC. + +2. **FCRC (32-bits)** - CRC of the bytes after this commit's CRC tag's padding + when erased. Like the CRC tag, this uses a CRC-32 with a polynomial of + `0x04c11db7` initialized with `0xffffffff`. + +--- diff --git a/components/fs/littlefs/littlefs/bd/lfs_emubd.c b/components/fs/littlefs/littlefs/bd/lfs_emubd.c new file mode 100644 index 000000000..c27ae3008 --- /dev/null +++ b/components/fs/littlefs/littlefs/bd/lfs_emubd.c @@ -0,0 +1,645 @@ +/* + * Emulating block device, wraps filebd and rambd while providing a bunch + * of hooks for testing littlefs in various conditions. + * + * Copyright (c) 2022, The littlefs authors. + * Copyright (c) 2017, Arm Limited. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 199309L +#endif + +#include "bd/lfs_emubd.h" + +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#endif + + +// access to lazily-allocated/copy-on-write blocks +// +// Note we can only modify a block if we have exclusive access to it (rc == 1) +// + +static lfs_emubd_block_t *lfs_emubd_incblock(lfs_emubd_block_t *block) { + if (block) { + block->rc += 1; + } + return block; +} + +static void lfs_emubd_decblock(lfs_emubd_block_t *block) { + if (block) { + block->rc -= 1; + if (block->rc == 0) { + free(block); + } + } +} + +static lfs_emubd_block_t *lfs_emubd_mutblock( + const struct lfs_config *cfg, + lfs_emubd_block_t **block) { + lfs_emubd_t *bd = cfg->context; + lfs_emubd_block_t *block_ = *block; + if (block_ && block_->rc == 1) { + // rc == 1? can modify + return block_; + + } else if (block_) { + // rc > 1? need to create a copy + lfs_emubd_block_t *nblock = malloc( + sizeof(lfs_emubd_block_t) + bd->cfg->erase_size); + if (!nblock) { + return NULL; + } + + memcpy(nblock, block_, + sizeof(lfs_emubd_block_t) + bd->cfg->erase_size); + nblock->rc = 1; + + lfs_emubd_decblock(block_); + *block = nblock; + return nblock; + + } else { + // no block? need to allocate + lfs_emubd_block_t *nblock = malloc( + sizeof(lfs_emubd_block_t) + bd->cfg->erase_size); + if (!nblock) { + return NULL; + } + + nblock->rc = 1; + nblock->wear = 0; + + // zero for consistency + memset(nblock->data, + (bd->cfg->erase_value != -1) ? bd->cfg->erase_value : 0, + bd->cfg->erase_size); + + *block = nblock; + return nblock; + } +} + + +// emubd create/destroy + +int lfs_emubd_create(const struct lfs_config *cfg, + const struct lfs_emubd_config *bdcfg) { + LFS_EMUBD_TRACE("lfs_emubd_create(%p {.context=%p, " + ".read=%p, .prog=%p, .erase=%p, .sync=%p}, " + "%p {.read_size=%"PRIu32", .prog_size=%"PRIu32", " + ".erase_size=%"PRIu32", .erase_count=%"PRIu32", " + ".erase_value=%"PRId32", .erase_cycles=%"PRIu32", " + ".badblock_behavior=%"PRIu8", .power_cycles=%"PRIu32", " + ".powerloss_behavior=%"PRIu8", .powerloss_cb=%p, " + ".powerloss_data=%p, .track_branches=%d})", + (void*)cfg, cfg->context, + (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, + (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, + (void*)bdcfg, + bdcfg->read_size, bdcfg->prog_size, bdcfg->erase_size, + bdcfg->erase_count, bdcfg->erase_value, bdcfg->erase_cycles, + bdcfg->badblock_behavior, bdcfg->power_cycles, + bdcfg->powerloss_behavior, (void*)(uintptr_t)bdcfg->powerloss_cb, + bdcfg->powerloss_data, bdcfg->track_branches); + lfs_emubd_t *bd = cfg->context; + bd->cfg = bdcfg; + + // allocate our block array, all blocks start as uninitialized + bd->blocks = malloc(bd->cfg->erase_count * sizeof(lfs_emubd_block_t*)); + if (!bd->blocks) { + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", LFS_ERR_NOMEM); + return LFS_ERR_NOMEM; + } + memset(bd->blocks, 0, bd->cfg->erase_count * sizeof(lfs_emubd_block_t*)); + + // setup testing things + bd->readed = 0; + bd->proged = 0; + bd->erased = 0; + bd->power_cycles = bd->cfg->power_cycles; + bd->disk = NULL; + + if (bd->cfg->disk_path) { + bd->disk = malloc(sizeof(lfs_emubd_disk_t)); + if (!bd->disk) { + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", LFS_ERR_NOMEM); + return LFS_ERR_NOMEM; + } + bd->disk->rc = 1; + bd->disk->scratch = NULL; + + #ifdef _WIN32 + bd->disk->fd = open(bd->cfg->disk_path, + O_RDWR | O_CREAT | O_BINARY, 0666); + #else + bd->disk->fd = open(bd->cfg->disk_path, + O_RDWR | O_CREAT, 0666); + #endif + if (bd->disk->fd < 0) { + int err = -errno; + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", err); + return err; + } + + // if we're emulating erase values, we can keep a block around in + // memory of just the erase state to speed up emulated erases + if (bd->cfg->erase_value != -1) { + bd->disk->scratch = malloc(bd->cfg->erase_size); + if (!bd->disk->scratch) { + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", LFS_ERR_NOMEM); + return LFS_ERR_NOMEM; + } + memset(bd->disk->scratch, + bd->cfg->erase_value, + bd->cfg->erase_size); + + // go ahead and erase all of the disk, otherwise the file will not + // match our internal representation + for (size_t i = 0; i < bd->cfg->erase_count; i++) { + ssize_t res = write(bd->disk->fd, + bd->disk->scratch, + bd->cfg->erase_size); + if (res < 0) { + int err = -errno; + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", err); + return err; + } + } + } + } + + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", 0); + return 0; +} + +int lfs_emubd_destroy(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_destroy(%p)", (void*)cfg); + lfs_emubd_t *bd = cfg->context; + + // decrement reference counts + for (lfs_block_t i = 0; i < bd->cfg->erase_count; i++) { + lfs_emubd_decblock(bd->blocks[i]); + } + free(bd->blocks); + + // clean up other resources + if (bd->disk) { + bd->disk->rc -= 1; + if (bd->disk->rc == 0) { + close(bd->disk->fd); + free(bd->disk->scratch); + free(bd->disk); + } + } + + LFS_EMUBD_TRACE("lfs_emubd_destroy -> %d", 0); + return 0; +} + + + +// block device API + +int lfs_emubd_read(const struct lfs_config *cfg, lfs_block_t block, + lfs_off_t off, void *buffer, lfs_size_t size) { + LFS_EMUBD_TRACE("lfs_emubd_read(%p, " + "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")", + (void*)cfg, block, off, buffer, size); + lfs_emubd_t *bd = cfg->context; + + // check if read is valid + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->read_size == 0); + LFS_ASSERT(size % bd->cfg->read_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); + + // get the block + const lfs_emubd_block_t *b = bd->blocks[block]; + if (b) { + // block bad? + if (bd->cfg->erase_cycles && b->wear >= bd->cfg->erase_cycles && + bd->cfg->badblock_behavior == LFS_EMUBD_BADBLOCK_READERROR) { + LFS_EMUBD_TRACE("lfs_emubd_read -> %d", LFS_ERR_CORRUPT); + return LFS_ERR_CORRUPT; + } + + // read data + memcpy(buffer, &b->data[off], size); + } else { + // zero for consistency + memset(buffer, + (bd->cfg->erase_value != -1) ? bd->cfg->erase_value : 0, + size); + } + + // track reads + bd->readed += size; + if (bd->cfg->read_sleep) { + int err = nanosleep(&(struct timespec){ + .tv_sec=bd->cfg->read_sleep/1000000000, + .tv_nsec=bd->cfg->read_sleep%1000000000}, + NULL); + if (err) { + err = -errno; + LFS_EMUBD_TRACE("lfs_emubd_read -> %d", err); + return err; + } + } + + LFS_EMUBD_TRACE("lfs_emubd_read -> %d", 0); + return 0; +} + +int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, + lfs_off_t off, const void *buffer, lfs_size_t size) { + LFS_EMUBD_TRACE("lfs_emubd_prog(%p, " + "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")", + (void*)cfg, block, off, buffer, size); + lfs_emubd_t *bd = cfg->context; + + // check if write is valid + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->prog_size == 0); + LFS_ASSERT(size % bd->cfg->prog_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); + + // get the block + lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]); + if (!b) { + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", LFS_ERR_NOMEM); + return LFS_ERR_NOMEM; + } + + // block bad? + if (bd->cfg->erase_cycles && b->wear >= bd->cfg->erase_cycles) { + if (bd->cfg->badblock_behavior == + LFS_EMUBD_BADBLOCK_PROGERROR) { + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", LFS_ERR_CORRUPT); + return LFS_ERR_CORRUPT; + } else if (bd->cfg->badblock_behavior == + LFS_EMUBD_BADBLOCK_PROGNOOP || + bd->cfg->badblock_behavior == + LFS_EMUBD_BADBLOCK_ERASENOOP) { + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", 0); + return 0; + } + } + + // were we erased properly? + if (bd->cfg->erase_value != -1) { + for (lfs_off_t i = 0; i < size; i++) { + LFS_ASSERT(b->data[off+i] == bd->cfg->erase_value); + } + } + + // prog data + memcpy(&b->data[off], buffer, size); + + // mirror to disk file? + if (bd->disk) { + off_t res1 = lseek(bd->disk->fd, + (off_t)block*bd->cfg->erase_size + (off_t)off, + SEEK_SET); + if (res1 < 0) { + int err = -errno; + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", err); + return err; + } + + ssize_t res2 = write(bd->disk->fd, buffer, size); + if (res2 < 0) { + int err = -errno; + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", err); + return err; + } + } + + // track progs + bd->proged += size; + if (bd->cfg->prog_sleep) { + int err = nanosleep(&(struct timespec){ + .tv_sec=bd->cfg->prog_sleep/1000000000, + .tv_nsec=bd->cfg->prog_sleep%1000000000}, + NULL); + if (err) { + err = -errno; + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", err); + return err; + } + } + + // lose power? + if (bd->power_cycles > 0) { + bd->power_cycles -= 1; + if (bd->power_cycles == 0) { + // simulate power loss + bd->cfg->powerloss_cb(bd->cfg->powerloss_data); + } + } + + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", 0); + return 0; +} + +int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block) { + LFS_EMUBD_TRACE("lfs_emubd_erase(%p, 0x%"PRIx32" (%"PRIu32"))", + (void*)cfg, block, ((lfs_emubd_t*)cfg->context)->cfg->erase_size); + lfs_emubd_t *bd = cfg->context; + + // check if erase is valid + LFS_ASSERT(block < bd->cfg->erase_count); + + // get the block + lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]); + if (!b) { + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", LFS_ERR_NOMEM); + return LFS_ERR_NOMEM; + } + + // block bad? + if (bd->cfg->erase_cycles) { + if (b->wear >= bd->cfg->erase_cycles) { + if (bd->cfg->badblock_behavior == + LFS_EMUBD_BADBLOCK_ERASEERROR) { + LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", LFS_ERR_CORRUPT); + return LFS_ERR_CORRUPT; + } else if (bd->cfg->badblock_behavior == + LFS_EMUBD_BADBLOCK_ERASENOOP) { + LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", 0); + return 0; + } + } else { + // mark wear + b->wear += 1; + } + } + + // emulate an erase value? + if (bd->cfg->erase_value != -1) { + memset(b->data, bd->cfg->erase_value, bd->cfg->erase_size); + + // mirror to disk file? + if (bd->disk) { + off_t res1 = lseek(bd->disk->fd, + (off_t)block*bd->cfg->erase_size, + SEEK_SET); + if (res1 < 0) { + int err = -errno; + LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", err); + return err; + } + + ssize_t res2 = write(bd->disk->fd, + bd->disk->scratch, + bd->cfg->erase_size); + if (res2 < 0) { + int err = -errno; + LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", err); + return err; + } + } + } + + // track erases + bd->erased += bd->cfg->erase_size; + if (bd->cfg->erase_sleep) { + int err = nanosleep(&(struct timespec){ + .tv_sec=bd->cfg->erase_sleep/1000000000, + .tv_nsec=bd->cfg->erase_sleep%1000000000}, + NULL); + if (err) { + err = -errno; + LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", err); + return err; + } + } + + // lose power? + if (bd->power_cycles > 0) { + bd->power_cycles -= 1; + if (bd->power_cycles == 0) { + // simulate power loss + bd->cfg->powerloss_cb(bd->cfg->powerloss_data); + } + } + + LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", 0); + return 0; +} + +int lfs_emubd_sync(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_sync(%p)", (void*)cfg); + + // do nothing + (void)cfg; + + LFS_EMUBD_TRACE("lfs_emubd_sync -> %d", 0); + return 0; +} + +/// Additional extended API for driving test features /// + +static int lfs_emubd_rawcrc(const struct lfs_config *cfg, + lfs_block_t block, uint32_t *crc) { + lfs_emubd_t *bd = cfg->context; + + // check if crc is valid + LFS_ASSERT(block < cfg->block_count); + + // crc the block + uint32_t crc_ = 0xffffffff; + const lfs_emubd_block_t *b = bd->blocks[block]; + if (b) { + crc_ = lfs_crc(crc_, b->data, cfg->block_size); + } else { + uint8_t erase_value = (bd->cfg->erase_value != -1) + ? bd->cfg->erase_value + : 0; + for (lfs_size_t i = 0; i < cfg->block_size; i++) { + crc_ = lfs_crc(crc_, &erase_value, 1); + } + } + *crc = 0xffffffff ^ crc_; + + return 0; +} + +int lfs_emubd_crc(const struct lfs_config *cfg, + lfs_block_t block, uint32_t *crc) { + LFS_EMUBD_TRACE("lfs_emubd_crc(%p, %"PRIu32", %p)", + (void*)cfg, block, crc); + int err = lfs_emubd_rawcrc(cfg, block, crc); + LFS_EMUBD_TRACE("lfs_emubd_crc -> %d", err); + return err; +} + +int lfs_emubd_bdcrc(const struct lfs_config *cfg, uint32_t *crc) { + LFS_EMUBD_TRACE("lfs_emubd_bdcrc(%p, %p)", (void*)cfg, crc); + + uint32_t crc_ = 0xffffffff; + for (lfs_block_t i = 0; i < cfg->block_count; i++) { + uint32_t i_crc; + int err = lfs_emubd_rawcrc(cfg, i, &i_crc); + if (err) { + LFS_EMUBD_TRACE("lfs_emubd_bdcrc -> %d", err); + return err; + } + + crc_ = lfs_crc(crc_, &i_crc, sizeof(uint32_t)); + } + *crc = 0xffffffff ^ crc_; + + LFS_EMUBD_TRACE("lfs_emubd_bdcrc -> %d", 0); + return 0; +} + +lfs_emubd_sio_t lfs_emubd_readed(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_readed(%p)", (void*)cfg); + lfs_emubd_t *bd = cfg->context; + LFS_EMUBD_TRACE("lfs_emubd_readed -> %"PRIu64, bd->readed); + return bd->readed; +} + +lfs_emubd_sio_t lfs_emubd_proged(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_proged(%p)", (void*)cfg); + lfs_emubd_t *bd = cfg->context; + LFS_EMUBD_TRACE("lfs_emubd_proged -> %"PRIu64, bd->proged); + return bd->proged; +} + +lfs_emubd_sio_t lfs_emubd_erased(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_erased(%p)", (void*)cfg); + lfs_emubd_t *bd = cfg->context; + LFS_EMUBD_TRACE("lfs_emubd_erased -> %"PRIu64, bd->erased); + return bd->erased; +} + +int lfs_emubd_setreaded(const struct lfs_config *cfg, lfs_emubd_io_t readed) { + LFS_EMUBD_TRACE("lfs_emubd_setreaded(%p, %"PRIu64")", (void*)cfg, readed); + lfs_emubd_t *bd = cfg->context; + bd->readed = readed; + LFS_EMUBD_TRACE("lfs_emubd_setreaded -> %d", 0); + return 0; +} + +int lfs_emubd_setproged(const struct lfs_config *cfg, lfs_emubd_io_t proged) { + LFS_EMUBD_TRACE("lfs_emubd_setproged(%p, %"PRIu64")", (void*)cfg, proged); + lfs_emubd_t *bd = cfg->context; + bd->proged = proged; + LFS_EMUBD_TRACE("lfs_emubd_setproged -> %d", 0); + return 0; +} + +int lfs_emubd_seterased(const struct lfs_config *cfg, lfs_emubd_io_t erased) { + LFS_EMUBD_TRACE("lfs_emubd_seterased(%p, %"PRIu64")", (void*)cfg, erased); + lfs_emubd_t *bd = cfg->context; + bd->erased = erased; + LFS_EMUBD_TRACE("lfs_emubd_seterased -> %d", 0); + return 0; +} + +lfs_emubd_swear_t lfs_emubd_wear(const struct lfs_config *cfg, + lfs_block_t block) { + LFS_EMUBD_TRACE("lfs_emubd_wear(%p, %"PRIu32")", (void*)cfg, block); + lfs_emubd_t *bd = cfg->context; + + // check if block is valid + LFS_ASSERT(block < bd->cfg->erase_count); + + // get the wear + lfs_emubd_wear_t wear; + const lfs_emubd_block_t *b = bd->blocks[block]; + if (b) { + wear = b->wear; + } else { + wear = 0; + } + + LFS_EMUBD_TRACE("lfs_emubd_wear -> %"PRIi32, wear); + return wear; +} + +int lfs_emubd_setwear(const struct lfs_config *cfg, + lfs_block_t block, lfs_emubd_wear_t wear) { + LFS_EMUBD_TRACE("lfs_emubd_setwear(%p, %"PRIu32", %"PRIi32")", + (void*)cfg, block, wear); + lfs_emubd_t *bd = cfg->context; + + // check if block is valid + LFS_ASSERT(block < bd->cfg->erase_count); + + // set the wear + lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]); + if (!b) { + LFS_EMUBD_TRACE("lfs_emubd_setwear -> %d", LFS_ERR_NOMEM); + return LFS_ERR_NOMEM; + } + b->wear = wear; + + LFS_EMUBD_TRACE("lfs_emubd_setwear -> %d", 0); + return 0; +} + +lfs_emubd_spowercycles_t lfs_emubd_powercycles( + const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_powercycles(%p)", (void*)cfg); + lfs_emubd_t *bd = cfg->context; + + LFS_EMUBD_TRACE("lfs_emubd_powercycles -> %"PRIi32, bd->power_cycles); + return bd->power_cycles; +} + +int lfs_emubd_setpowercycles(const struct lfs_config *cfg, + lfs_emubd_powercycles_t power_cycles) { + LFS_EMUBD_TRACE("lfs_emubd_setpowercycles(%p, %"PRIi32")", + (void*)cfg, power_cycles); + lfs_emubd_t *bd = cfg->context; + + bd->power_cycles = power_cycles; + + LFS_EMUBD_TRACE("lfs_emubd_powercycles -> %d", 0); + return 0; +} + +int lfs_emubd_copy(const struct lfs_config *cfg, lfs_emubd_t *copy) { + LFS_EMUBD_TRACE("lfs_emubd_copy(%p, %p)", (void*)cfg, (void*)copy); + lfs_emubd_t *bd = cfg->context; + + // lazily copy over our block array + copy->blocks = malloc(bd->cfg->erase_count * sizeof(lfs_emubd_block_t*)); + if (!copy->blocks) { + LFS_EMUBD_TRACE("lfs_emubd_copy -> %d", LFS_ERR_NOMEM); + return LFS_ERR_NOMEM; + } + + for (size_t i = 0; i < bd->cfg->erase_count; i++) { + copy->blocks[i] = lfs_emubd_incblock(bd->blocks[i]); + } + + // other state + copy->readed = bd->readed; + copy->proged = bd->proged; + copy->erased = bd->erased; + copy->power_cycles = bd->power_cycles; + copy->disk = bd->disk; + if (copy->disk) { + copy->disk->rc += 1; + } + copy->cfg = bd->cfg; + + LFS_EMUBD_TRACE("lfs_emubd_copy -> %d", 0); + return 0; +} + diff --git a/components/fs/littlefs/littlefs/bd/lfs_emubd.h b/components/fs/littlefs/littlefs/bd/lfs_emubd.h new file mode 100644 index 000000000..9049649f6 --- /dev/null +++ b/components/fs/littlefs/littlefs/bd/lfs_emubd.h @@ -0,0 +1,241 @@ +/* + * Emulating block device, wraps filebd and rambd while providing a bunch + * of hooks for testing littlefs in various conditions. + * + * Copyright (c) 2022, The littlefs authors. + * Copyright (c) 2017, Arm Limited. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ +#ifndef LFS_EMUBD_H +#define LFS_EMUBD_H + +#include "lfs.h" +#include "lfs_util.h" +#include "bd/lfs_rambd.h" +#include "bd/lfs_filebd.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + + +// Block device specific tracing +#ifndef LFS_EMUBD_TRACE +#ifdef LFS_EMUBD_YES_TRACE +#define LFS_EMUBD_TRACE(...) LFS_TRACE(__VA_ARGS__) +#else +#define LFS_EMUBD_TRACE(...) +#endif +#endif + +// Mode determining how "bad-blocks" behave during testing. This simulates +// some real-world circumstances such as progs not sticking (prog-noop), +// a readonly disk (erase-noop), and ECC failures (read-error). +// +// Not that read-noop is not allowed. Read _must_ return a consistent (but +// may be arbitrary) value on every read. +typedef enum lfs_emubd_badblock_behavior { + LFS_EMUBD_BADBLOCK_PROGERROR, + LFS_EMUBD_BADBLOCK_ERASEERROR, + LFS_EMUBD_BADBLOCK_READERROR, + LFS_EMUBD_BADBLOCK_PROGNOOP, + LFS_EMUBD_BADBLOCK_ERASENOOP, +} lfs_emubd_badblock_behavior_t; + +// Mode determining how power-loss behaves during testing. For now this +// only supports a noop behavior, leaving the data on-disk untouched. +typedef enum lfs_emubd_powerloss_behavior { + LFS_EMUBD_POWERLOSS_NOOP, +} lfs_emubd_powerloss_behavior_t; + +// Type for measuring read/program/erase operations +typedef uint64_t lfs_emubd_io_t; +typedef int64_t lfs_emubd_sio_t; + +// Type for measuring wear +typedef uint32_t lfs_emubd_wear_t; +typedef int32_t lfs_emubd_swear_t; + +// Type for tracking power-cycles +typedef uint32_t lfs_emubd_powercycles_t; +typedef int32_t lfs_emubd_spowercycles_t; + +// Type for delays in nanoseconds +typedef uint64_t lfs_emubd_sleep_t; +typedef int64_t lfs_emubd_ssleep_t; + +// emubd config, this is required for testing +struct lfs_emubd_config { + // Minimum size of a read operation in bytes. + lfs_size_t read_size; + + // Minimum size of a program operation in bytes. + lfs_size_t prog_size; + + // Size of an erase operation in bytes. + lfs_size_t erase_size; + + // Number of erase blocks on the device. + lfs_size_t erase_count; + + // 8-bit erase value to use for simulating erases. -1 does not simulate + // erases, which can speed up testing by avoiding the extra block-device + // operations to store the erase value. + int32_t erase_value; + + // Number of erase cycles before a block becomes "bad". The exact behavior + // of bad blocks is controlled by badblock_behavior. + uint32_t erase_cycles; + + // The mode determining how bad-blocks fail + lfs_emubd_badblock_behavior_t badblock_behavior; + + // Number of write operations (erase/prog) before triggering a power-loss. + // power_cycles=0 disables this. The exact behavior of power-loss is + // controlled by a combination of powerloss_behavior and powerloss_cb. + lfs_emubd_powercycles_t power_cycles; + + // The mode determining how power-loss affects disk + lfs_emubd_powerloss_behavior_t powerloss_behavior; + + // Function to call to emulate power-loss. The exact behavior of power-loss + // is up to the runner to provide. + void (*powerloss_cb)(void*); + + // Data for power-loss callback + void *powerloss_data; + + // True to track when power-loss could have occured. Note this involves + // heavy memory usage! + bool track_branches; + + // Path to file to use as a mirror of the disk. This provides a way to view + // the current state of the block device. + const char *disk_path; + + // Artificial delay in nanoseconds, there is no purpose for this other + // than slowing down the simulation. + lfs_emubd_sleep_t read_sleep; + + // Artificial delay in nanoseconds, there is no purpose for this other + // than slowing down the simulation. + lfs_emubd_sleep_t prog_sleep; + + // Artificial delay in nanoseconds, there is no purpose for this other + // than slowing down the simulation. + lfs_emubd_sleep_t erase_sleep; +}; + +// A reference counted block +typedef struct lfs_emubd_block { + uint32_t rc; + lfs_emubd_wear_t wear; + + uint8_t data[]; +} lfs_emubd_block_t; + +// Disk mirror +typedef struct lfs_emubd_disk { + uint32_t rc; + int fd; + uint8_t *scratch; +} lfs_emubd_disk_t; + +// emubd state +typedef struct lfs_emubd { + // array of copy-on-write blocks + lfs_emubd_block_t **blocks; + + // some other test state + lfs_emubd_io_t readed; + lfs_emubd_io_t proged; + lfs_emubd_io_t erased; + lfs_emubd_powercycles_t power_cycles; + lfs_emubd_disk_t *disk; + + const struct lfs_emubd_config *cfg; +} lfs_emubd_t; + + +/// Block device API /// + +// Create an emulating block device using the geometry in lfs_config +int lfs_emubd_create(const struct lfs_config *cfg, + const struct lfs_emubd_config *bdcfg); + +// Clean up memory associated with block device +int lfs_emubd_destroy(const struct lfs_config *cfg); + +// Read a block +int lfs_emubd_read(const struct lfs_config *cfg, lfs_block_t block, + lfs_off_t off, void *buffer, lfs_size_t size); + +// Program a block +// +// The block must have previously been erased. +int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, + lfs_off_t off, const void *buffer, lfs_size_t size); + +// Erase a block +// +// A block must be erased before being programmed. The +// state of an erased block is undefined. +int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block); + +// Sync the block device +int lfs_emubd_sync(const struct lfs_config *cfg); + + +/// Additional extended API for driving test features /// + +// A CRC of a block for debugging purposes +int lfs_emubd_crc(const struct lfs_config *cfg, + lfs_block_t block, uint32_t *crc); + +// A CRC of the entire block device for debugging purposes +int lfs_emubd_bdcrc(const struct lfs_config *cfg, uint32_t *crc); + +// Get total amount of bytes read +lfs_emubd_sio_t lfs_emubd_readed(const struct lfs_config *cfg); + +// Get total amount of bytes programmed +lfs_emubd_sio_t lfs_emubd_proged(const struct lfs_config *cfg); + +// Get total amount of bytes erased +lfs_emubd_sio_t lfs_emubd_erased(const struct lfs_config *cfg); + +// Manually set amount of bytes read +int lfs_emubd_setreaded(const struct lfs_config *cfg, lfs_emubd_io_t readed); + +// Manually set amount of bytes programmed +int lfs_emubd_setproged(const struct lfs_config *cfg, lfs_emubd_io_t proged); + +// Manually set amount of bytes erased +int lfs_emubd_seterased(const struct lfs_config *cfg, lfs_emubd_io_t erased); + +// Get simulated wear on a given block +lfs_emubd_swear_t lfs_emubd_wear(const struct lfs_config *cfg, + lfs_block_t block); + +// Manually set simulated wear on a given block +int lfs_emubd_setwear(const struct lfs_config *cfg, + lfs_block_t block, lfs_emubd_wear_t wear); + +// Get the remaining power-cycles +lfs_emubd_spowercycles_t lfs_emubd_powercycles( + const struct lfs_config *cfg); + +// Manually set the remaining power-cycles +int lfs_emubd_setpowercycles(const struct lfs_config *cfg, + lfs_emubd_powercycles_t power_cycles); + +// Create a copy-on-write copy of the state of this block device +int lfs_emubd_copy(const struct lfs_config *cfg, lfs_emubd_t *copy); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif diff --git a/components/fs/littlefs/littlefs/bd/lfs_filebd.c b/components/fs/littlefs/littlefs/bd/lfs_filebd.c index ee0c31e20..4ff25d448 100644 --- a/components/fs/littlefs/littlefs/bd/lfs_filebd.c +++ b/components/fs/littlefs/littlefs/bd/lfs_filebd.c @@ -15,19 +15,20 @@ #include #endif -int lfs_filebd_createcfg(const struct lfs_config *cfg, const char *path, +int lfs_filebd_create(const struct lfs_config *cfg, const char *path, const struct lfs_filebd_config *bdcfg) { - LFS_FILEBD_TRACE("lfs_filebd_createcfg(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " + LFS_FILEBD_TRACE("lfs_filebd_create(%p {.context=%p, " + ".read=%p, .prog=%p, .erase=%p, .sync=%p}, " "\"%s\", " - "%p {.erase_value=%"PRId32"})", + "%p {.read_size=%"PRIu32", .prog_size=%"PRIu32", " + ".erase_size=%"PRIu32", .erase_count=%"PRIu32"})", (void*)cfg, cfg->context, (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - path, (void*)bdcfg, bdcfg->erase_value); + path, + (void*)bdcfg, + bdcfg->read_size, bdcfg->prog_size, bdcfg->erase_size, + bdcfg->erase_count); lfs_filebd_t *bd = cfg->context; bd->cfg = bdcfg; @@ -40,31 +41,14 @@ int lfs_filebd_createcfg(const struct lfs_config *cfg, const char *path, if (bd->fd < 0) { int err = -errno; - LFS_FILEBD_TRACE("lfs_filebd_createcfg -> %d", err); + LFS_FILEBD_TRACE("lfs_filebd_create -> %d", err); return err; } - LFS_FILEBD_TRACE("lfs_filebd_createcfg -> %d", 0); + LFS_FILEBD_TRACE("lfs_filebd_create -> %d", 0); return 0; } -int lfs_filebd_create(const struct lfs_config *cfg, const char *path) { - LFS_FILEBD_TRACE("lfs_filebd_create(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " - "\"%s\")", - (void*)cfg, cfg->context, - (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, - (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - path); - static const struct lfs_filebd_config defaults = {.erase_value=-1}; - int err = lfs_filebd_createcfg(cfg, path, &defaults); - LFS_FILEBD_TRACE("lfs_filebd_create -> %d", err); - return err; -} - int lfs_filebd_destroy(const struct lfs_config *cfg) { LFS_FILEBD_TRACE("lfs_filebd_destroy(%p)", (void*)cfg); lfs_filebd_t *bd = cfg->context; @@ -86,18 +70,17 @@ int lfs_filebd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_filebd_t *bd = cfg->context; // check if read is valid - LFS_ASSERT(off % cfg->read_size == 0); - LFS_ASSERT(size % cfg->read_size == 0); - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->read_size == 0); + LFS_ASSERT(size % bd->cfg->read_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); // zero for reproducibility (in case file is truncated) - if (bd->cfg->erase_value != -1) { - memset(buffer, bd->cfg->erase_value, size); - } + memset(buffer, 0, size); // read off_t res1 = lseek(bd->fd, - (off_t)block*cfg->block_size + (off_t)off, SEEK_SET); + (off_t)block*bd->cfg->erase_size + (off_t)off, SEEK_SET); if (res1 < 0) { int err = -errno; LFS_FILEBD_TRACE("lfs_filebd_read -> %d", err); @@ -117,41 +100,20 @@ int lfs_filebd_read(const struct lfs_config *cfg, lfs_block_t block, int lfs_filebd_prog(const struct lfs_config *cfg, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size) { - LFS_FILEBD_TRACE("lfs_filebd_prog(%p, 0x%"PRIx32", %"PRIu32", %p, %"PRIu32")", + LFS_FILEBD_TRACE("lfs_filebd_prog(%p, " + "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")", (void*)cfg, block, off, buffer, size); lfs_filebd_t *bd = cfg->context; // check if write is valid - LFS_ASSERT(off % cfg->prog_size == 0); - LFS_ASSERT(size % cfg->prog_size == 0); - LFS_ASSERT(block < cfg->block_count); - - // check that data was erased? only needed for testing - if (bd->cfg->erase_value != -1) { - off_t res1 = lseek(bd->fd, - (off_t)block*cfg->block_size + (off_t)off, SEEK_SET); - if (res1 < 0) { - int err = -errno; - LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", err); - return err; - } - - for (lfs_off_t i = 0; i < size; i++) { - uint8_t c; - ssize_t res2 = read(bd->fd, &c, 1); - if (res2 < 0) { - int err = -errno; - LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", err); - return err; - } - - LFS_ASSERT(c == bd->cfg->erase_value); - } - } + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->prog_size == 0); + LFS_ASSERT(size % bd->cfg->prog_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); // program data off_t res1 = lseek(bd->fd, - (off_t)block*cfg->block_size + (off_t)off, SEEK_SET); + (off_t)block*bd->cfg->erase_size + (off_t)off, SEEK_SET); if (res1 < 0) { int err = -errno; LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", err); @@ -170,30 +132,15 @@ int lfs_filebd_prog(const struct lfs_config *cfg, lfs_block_t block, } int lfs_filebd_erase(const struct lfs_config *cfg, lfs_block_t block) { - LFS_FILEBD_TRACE("lfs_filebd_erase(%p, 0x%"PRIx32")", (void*)cfg, block); + LFS_FILEBD_TRACE("lfs_filebd_erase(%p, 0x%"PRIx32" (%"PRIu32"))", + (void*)cfg, block, ((lfs_file_t*)cfg->context)->cfg->erase_size); lfs_filebd_t *bd = cfg->context; // check if erase is valid - LFS_ASSERT(block < cfg->block_count); - - // erase, only needed for testing - if (bd->cfg->erase_value != -1) { - off_t res1 = lseek(bd->fd, (off_t)block*cfg->block_size, SEEK_SET); - if (res1 < 0) { - int err = -errno; - LFS_FILEBD_TRACE("lfs_filebd_erase -> %d", err); - return err; - } - - for (lfs_off_t i = 0; i < cfg->block_size; i++) { - ssize_t res2 = write(bd->fd, &(uint8_t){bd->cfg->erase_value}, 1); - if (res2 < 0) { - int err = -errno; - LFS_FILEBD_TRACE("lfs_filebd_erase -> %d", err); - return err; - } - } - } + LFS_ASSERT(block < bd->cfg->erase_count); + + // erase is a noop + (void)block; LFS_FILEBD_TRACE("lfs_filebd_erase -> %d", 0); return 0; @@ -201,6 +148,7 @@ int lfs_filebd_erase(const struct lfs_config *cfg, lfs_block_t block) { int lfs_filebd_sync(const struct lfs_config *cfg) { LFS_FILEBD_TRACE("lfs_filebd_sync(%p)", (void*)cfg); + // file sync lfs_filebd_t *bd = cfg->context; #ifdef _WIN32 diff --git a/components/fs/littlefs/littlefs/bd/lfs_filebd.h b/components/fs/littlefs/littlefs/bd/lfs_filebd.h index 1a9456c51..d7d2fd95a 100644 --- a/components/fs/littlefs/littlefs/bd/lfs_filebd.h +++ b/components/fs/littlefs/littlefs/bd/lfs_filebd.h @@ -18,18 +18,27 @@ extern "C" // Block device specific tracing +#ifndef LFS_FILEBD_TRACE #ifdef LFS_FILEBD_YES_TRACE #define LFS_FILEBD_TRACE(...) LFS_TRACE(__VA_ARGS__) #else #define LFS_FILEBD_TRACE(...) #endif +#endif -// filebd config (optional) +// filebd config struct lfs_filebd_config { - // 8-bit erase value to use for simulating erases. -1 does not simulate - // erases, which can speed up testing by avoiding all the extra block-device - // operations to store the erase value. - int32_t erase_value; + // Minimum size of a read operation in bytes. + lfs_size_t read_size; + + // Minimum size of a program operation in bytes. + lfs_size_t prog_size; + + // Size of an erase operation in bytes. + lfs_size_t erase_size; + + // Number of erase blocks on the device. + lfs_size_t erase_count; }; // filebd state @@ -39,9 +48,8 @@ typedef struct lfs_filebd { } lfs_filebd_t; -// Create a file block device using the geometry in lfs_config -int lfs_filebd_create(const struct lfs_config *cfg, const char *path); -int lfs_filebd_createcfg(const struct lfs_config *cfg, const char *path, +// Create a file block device +int lfs_filebd_create(const struct lfs_config *cfg, const char *path, const struct lfs_filebd_config *bdcfg); // Clean up memory associated with block device diff --git a/components/fs/littlefs/littlefs/bd/lfs_rambd.c b/components/fs/littlefs/littlefs/bd/lfs_rambd.c index 39bb81507..a6a05727a 100644 --- a/components/fs/littlefs/littlefs/bd/lfs_rambd.c +++ b/components/fs/littlefs/littlefs/bd/lfs_rambd.c @@ -7,18 +7,19 @@ */ #include "bd/lfs_rambd.h" -int lfs_rambd_createcfg(const struct lfs_config *cfg, +int lfs_rambd_create(const struct lfs_config *cfg, const struct lfs_rambd_config *bdcfg) { - LFS_RAMBD_TRACE("lfs_rambd_createcfg(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " - "%p {.erase_value=%"PRId32", .buffer=%p})", + LFS_RAMBD_TRACE("lfs_rambd_create(%p {.context=%p, " + ".read=%p, .prog=%p, .erase=%p, .sync=%p}, " + "%p {.read_size=%"PRIu32", .prog_size=%"PRIu32", " + ".erase_size=%"PRIu32", .erase_count=%"PRIu32", " + ".buffer=%p})", (void*)cfg, cfg->context, (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - (void*)bdcfg, bdcfg->erase_value, bdcfg->buffer); + (void*)bdcfg, + bdcfg->read_size, bdcfg->prog_size, bdcfg->erase_size, + bdcfg->erase_count, bdcfg->buffer); lfs_rambd_t *bd = cfg->context; bd->cfg = bdcfg; @@ -26,40 +27,20 @@ int lfs_rambd_createcfg(const struct lfs_config *cfg, if (bd->cfg->buffer) { bd->buffer = bd->cfg->buffer; } else { - bd->buffer = lfs_malloc(cfg->block_size * cfg->block_count); + bd->buffer = lfs_malloc(bd->cfg->erase_size * bd->cfg->erase_count); if (!bd->buffer) { - LFS_RAMBD_TRACE("lfs_rambd_createcfg -> %d", LFS_ERR_NOMEM); + LFS_RAMBD_TRACE("lfs_rambd_create -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } } - // zero for reproducibility? - if (bd->cfg->erase_value != -1) { - memset(bd->buffer, bd->cfg->erase_value, - cfg->block_size * cfg->block_count); - } else { - memset(bd->buffer, 0, cfg->block_size * cfg->block_count); - } + // zero for reproducibility + memset(bd->buffer, 0, bd->cfg->erase_size * bd->cfg->erase_count); - LFS_RAMBD_TRACE("lfs_rambd_createcfg -> %d", 0); + LFS_RAMBD_TRACE("lfs_rambd_create -> %d", 0); return 0; } -int lfs_rambd_create(const struct lfs_config *cfg) { - LFS_RAMBD_TRACE("lfs_rambd_create(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"})", - (void*)cfg, cfg->context, - (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, - (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count); - static const struct lfs_rambd_config defaults = {.erase_value=-1}; - int err = lfs_rambd_createcfg(cfg, &defaults); - LFS_RAMBD_TRACE("lfs_rambd_create -> %d", err); - return err; -} - int lfs_rambd_destroy(const struct lfs_config *cfg) { LFS_RAMBD_TRACE("lfs_rambd_destroy(%p)", (void*)cfg); // clean up memory @@ -79,12 +60,13 @@ int lfs_rambd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_rambd_t *bd = cfg->context; // check if read is valid - LFS_ASSERT(off % cfg->read_size == 0); - LFS_ASSERT(size % cfg->read_size == 0); - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->read_size == 0); + LFS_ASSERT(size % bd->cfg->read_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); // read data - memcpy(buffer, &bd->buffer[block*cfg->block_size + off], size); + memcpy(buffer, &bd->buffer[block*bd->cfg->erase_size + off], size); LFS_RAMBD_TRACE("lfs_rambd_read -> %d", 0); return 0; @@ -98,37 +80,28 @@ int lfs_rambd_prog(const struct lfs_config *cfg, lfs_block_t block, lfs_rambd_t *bd = cfg->context; // check if write is valid - LFS_ASSERT(off % cfg->prog_size == 0); - LFS_ASSERT(size % cfg->prog_size == 0); - LFS_ASSERT(block < cfg->block_count); - - // check that data was erased? only needed for testing - if (bd->cfg->erase_value != -1) { - for (lfs_off_t i = 0; i < size; i++) { - LFS_ASSERT(bd->buffer[block*cfg->block_size + off + i] == - bd->cfg->erase_value); - } - } + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->prog_size == 0); + LFS_ASSERT(size % bd->cfg->prog_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); // program data - memcpy(&bd->buffer[block*cfg->block_size + off], buffer, size); + memcpy(&bd->buffer[block*bd->cfg->erase_size + off], buffer, size); LFS_RAMBD_TRACE("lfs_rambd_prog -> %d", 0); return 0; } int lfs_rambd_erase(const struct lfs_config *cfg, lfs_block_t block) { - LFS_RAMBD_TRACE("lfs_rambd_erase(%p, 0x%"PRIx32")", (void*)cfg, block); + LFS_RAMBD_TRACE("lfs_rambd_erase(%p, 0x%"PRIx32" (%"PRIu32"))", + (void*)cfg, block, ((lfs_rambd_t*)cfg->context)->cfg->erase_size); lfs_rambd_t *bd = cfg->context; // check if erase is valid - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(block < bd->cfg->erase_count); - // erase, only needed for testing - if (bd->cfg->erase_value != -1) { - memset(&bd->buffer[block*cfg->block_size], - bd->cfg->erase_value, cfg->block_size); - } + // erase is a noop + (void)block; LFS_RAMBD_TRACE("lfs_rambd_erase -> %d", 0); return 0; @@ -136,8 +109,10 @@ int lfs_rambd_erase(const struct lfs_config *cfg, lfs_block_t block) { int lfs_rambd_sync(const struct lfs_config *cfg) { LFS_RAMBD_TRACE("lfs_rambd_sync(%p)", (void*)cfg); - // sync does nothing because we aren't backed by anything real + + // sync is a noop (void)cfg; + LFS_RAMBD_TRACE("lfs_rambd_sync -> %d", 0); return 0; } diff --git a/components/fs/littlefs/littlefs/bd/lfs_rambd.h b/components/fs/littlefs/littlefs/bd/lfs_rambd.h index 3a70bc6e1..86637026a 100644 --- a/components/fs/littlefs/littlefs/bd/lfs_rambd.h +++ b/components/fs/littlefs/littlefs/bd/lfs_rambd.h @@ -18,17 +18,27 @@ extern "C" // Block device specific tracing +#ifndef LFS_RAMBD_TRACE #ifdef LFS_RAMBD_YES_TRACE #define LFS_RAMBD_TRACE(...) LFS_TRACE(__VA_ARGS__) #else #define LFS_RAMBD_TRACE(...) #endif +#endif -// rambd config (optional) +// rambd config struct lfs_rambd_config { - // 8-bit erase value to simulate erasing with. -1 indicates no erase - // occurs, which is still a valid block device - int32_t erase_value; + // Minimum size of a read operation in bytes. + lfs_size_t read_size; + + // Minimum size of a program operation in bytes. + lfs_size_t prog_size; + + // Size of an erase operation in bytes. + lfs_size_t erase_size; + + // Number of erase blocks on the device. + lfs_size_t erase_count; // Optional statically allocated buffer for the block device. void *buffer; @@ -41,9 +51,8 @@ typedef struct lfs_rambd { } lfs_rambd_t; -// Create a RAM block device using the geometry in lfs_config -int lfs_rambd_create(const struct lfs_config *cfg); -int lfs_rambd_createcfg(const struct lfs_config *cfg, +// Create a RAM block device +int lfs_rambd_create(const struct lfs_config *cfg, const struct lfs_rambd_config *bdcfg); // Clean up memory associated with block device diff --git a/components/fs/littlefs/littlefs/bd/lfs_testbd.c b/components/fs/littlefs/littlefs/bd/lfs_testbd.c deleted file mode 100644 index 1f0877d43..000000000 --- a/components/fs/littlefs/littlefs/bd/lfs_testbd.c +++ /dev/null @@ -1,303 +0,0 @@ -/* - * Testing block device, wraps filebd and rambd while providing a bunch - * of hooks for testing littlefs in various conditions. - * - * Copyright (c) 2022, The littlefs authors. - * Copyright (c) 2017, Arm Limited. All rights reserved. - * SPDX-License-Identifier: BSD-3-Clause - */ -#include "bd/lfs_testbd.h" - -#include - - -int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, - const struct lfs_testbd_config *bdcfg) { - LFS_TESTBD_TRACE("lfs_testbd_createcfg(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " - "\"%s\", " - "%p {.erase_value=%"PRId32", .erase_cycles=%"PRIu32", " - ".badblock_behavior=%"PRIu8", .power_cycles=%"PRIu32", " - ".buffer=%p, .wear_buffer=%p})", - (void*)cfg, cfg->context, - (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, - (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - path, (void*)bdcfg, bdcfg->erase_value, bdcfg->erase_cycles, - bdcfg->badblock_behavior, bdcfg->power_cycles, - bdcfg->buffer, bdcfg->wear_buffer); - lfs_testbd_t *bd = cfg->context; - bd->cfg = bdcfg; - - // setup testing things - bd->persist = path; - bd->power_cycles = bd->cfg->power_cycles; - - if (bd->cfg->erase_cycles) { - if (bd->cfg->wear_buffer) { - bd->wear = bd->cfg->wear_buffer; - } else { - bd->wear = lfs_malloc(sizeof(lfs_testbd_wear_t)*cfg->block_count); - if (!bd->wear) { - LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM); - return LFS_ERR_NOMEM; - } - } - - memset(bd->wear, 0, sizeof(lfs_testbd_wear_t) * cfg->block_count); - } - - // create underlying block device - if (bd->persist) { - bd->u.file.cfg = (struct lfs_filebd_config){ - .erase_value = bd->cfg->erase_value, - }; - int err = lfs_filebd_createcfg(cfg, path, &bd->u.file.cfg); - LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", err); - return err; - } else { - bd->u.ram.cfg = (struct lfs_rambd_config){ - .erase_value = bd->cfg->erase_value, - .buffer = bd->cfg->buffer, - }; - int err = lfs_rambd_createcfg(cfg, &bd->u.ram.cfg); - LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", err); - return err; - } -} - -int lfs_testbd_create(const struct lfs_config *cfg, const char *path) { - LFS_TESTBD_TRACE("lfs_testbd_create(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " - "\"%s\")", - (void*)cfg, cfg->context, - (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, - (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - path); - static const struct lfs_testbd_config defaults = {.erase_value=-1}; - int err = lfs_testbd_createcfg(cfg, path, &defaults); - LFS_TESTBD_TRACE("lfs_testbd_create -> %d", err); - return err; -} - -int lfs_testbd_destroy(const struct lfs_config *cfg) { - LFS_TESTBD_TRACE("lfs_testbd_destroy(%p)", (void*)cfg); - lfs_testbd_t *bd = cfg->context; - if (bd->cfg->erase_cycles && !bd->cfg->wear_buffer) { - lfs_free(bd->wear); - } - - if (bd->persist) { - int err = lfs_filebd_destroy(cfg); - LFS_TESTBD_TRACE("lfs_testbd_destroy -> %d", err); - return err; - } else { - int err = lfs_rambd_destroy(cfg); - LFS_TESTBD_TRACE("lfs_testbd_destroy -> %d", err); - return err; - } -} - -/// Internal mapping to block devices /// -static int lfs_testbd_rawread(const struct lfs_config *cfg, lfs_block_t block, - lfs_off_t off, void *buffer, lfs_size_t size) { - lfs_testbd_t *bd = cfg->context; - if (bd->persist) { - return lfs_filebd_read(cfg, block, off, buffer, size); - } else { - return lfs_rambd_read(cfg, block, off, buffer, size); - } -} - -static int lfs_testbd_rawprog(const struct lfs_config *cfg, lfs_block_t block, - lfs_off_t off, const void *buffer, lfs_size_t size) { - lfs_testbd_t *bd = cfg->context; - if (bd->persist) { - return lfs_filebd_prog(cfg, block, off, buffer, size); - } else { - return lfs_rambd_prog(cfg, block, off, buffer, size); - } -} - -static int lfs_testbd_rawerase(const struct lfs_config *cfg, - lfs_block_t block) { - lfs_testbd_t *bd = cfg->context; - if (bd->persist) { - return lfs_filebd_erase(cfg, block); - } else { - return lfs_rambd_erase(cfg, block); - } -} - -static int lfs_testbd_rawsync(const struct lfs_config *cfg) { - lfs_testbd_t *bd = cfg->context; - if (bd->persist) { - return lfs_filebd_sync(cfg); - } else { - return lfs_rambd_sync(cfg); - } -} - -/// block device API /// -int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, - lfs_off_t off, void *buffer, lfs_size_t size) { - LFS_TESTBD_TRACE("lfs_testbd_read(%p, " - "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")", - (void*)cfg, block, off, buffer, size); - lfs_testbd_t *bd = cfg->context; - - // check if read is valid - LFS_ASSERT(off % cfg->read_size == 0); - LFS_ASSERT(size % cfg->read_size == 0); - LFS_ASSERT(block < cfg->block_count); - - // block bad? - if (bd->cfg->erase_cycles && bd->wear[block] >= bd->cfg->erase_cycles && - bd->cfg->badblock_behavior == LFS_TESTBD_BADBLOCK_READERROR) { - LFS_TESTBD_TRACE("lfs_testbd_read -> %d", LFS_ERR_CORRUPT); - return LFS_ERR_CORRUPT; - } - - // read - int err = lfs_testbd_rawread(cfg, block, off, buffer, size); - LFS_TESTBD_TRACE("lfs_testbd_read -> %d", err); - return err; -} - -int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, - lfs_off_t off, const void *buffer, lfs_size_t size) { - LFS_TESTBD_TRACE("lfs_testbd_prog(%p, " - "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")", - (void*)cfg, block, off, buffer, size); - lfs_testbd_t *bd = cfg->context; - - // check if write is valid - LFS_ASSERT(off % cfg->prog_size == 0); - LFS_ASSERT(size % cfg->prog_size == 0); - LFS_ASSERT(block < cfg->block_count); - - // block bad? - if (bd->cfg->erase_cycles && bd->wear[block] >= bd->cfg->erase_cycles) { - if (bd->cfg->badblock_behavior == - LFS_TESTBD_BADBLOCK_PROGERROR) { - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_CORRUPT); - return LFS_ERR_CORRUPT; - } else if (bd->cfg->badblock_behavior == - LFS_TESTBD_BADBLOCK_PROGNOOP || - bd->cfg->badblock_behavior == - LFS_TESTBD_BADBLOCK_ERASENOOP) { - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0); - return 0; - } - } - - // prog - int err = lfs_testbd_rawprog(cfg, block, off, buffer, size); - if (err) { - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); - return err; - } - - // lose power? - if (bd->power_cycles > 0) { - bd->power_cycles -= 1; - if (bd->power_cycles == 0) { - // sync to make sure we persist the last changes - LFS_ASSERT(lfs_testbd_rawsync(cfg) == 0); - // simulate power loss - exit(33); - } - } - - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0); - return 0; -} - -int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { - LFS_TESTBD_TRACE("lfs_testbd_erase(%p, 0x%"PRIx32")", (void*)cfg, block); - lfs_testbd_t *bd = cfg->context; - - // check if erase is valid - LFS_ASSERT(block < cfg->block_count); - - // block bad? - if (bd->cfg->erase_cycles) { - if (bd->wear[block] >= bd->cfg->erase_cycles) { - if (bd->cfg->badblock_behavior == - LFS_TESTBD_BADBLOCK_ERASEERROR) { - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", LFS_ERR_CORRUPT); - return LFS_ERR_CORRUPT; - } else if (bd->cfg->badblock_behavior == - LFS_TESTBD_BADBLOCK_ERASENOOP) { - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", 0); - return 0; - } - } else { - // mark wear - bd->wear[block] += 1; - } - } - - // erase - int err = lfs_testbd_rawerase(cfg, block); - if (err) { - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); - return err; - } - - // lose power? - if (bd->power_cycles > 0) { - bd->power_cycles -= 1; - if (bd->power_cycles == 0) { - // sync to make sure we persist the last changes - LFS_ASSERT(lfs_testbd_rawsync(cfg) == 0); - // simulate power loss - exit(33); - } - } - - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0); - return 0; -} - -int lfs_testbd_sync(const struct lfs_config *cfg) { - LFS_TESTBD_TRACE("lfs_testbd_sync(%p)", (void*)cfg); - int err = lfs_testbd_rawsync(cfg); - LFS_TESTBD_TRACE("lfs_testbd_sync -> %d", err); - return err; -} - - -/// simulated wear operations /// -lfs_testbd_swear_t lfs_testbd_getwear(const struct lfs_config *cfg, - lfs_block_t block) { - LFS_TESTBD_TRACE("lfs_testbd_getwear(%p, %"PRIu32")", (void*)cfg, block); - lfs_testbd_t *bd = cfg->context; - - // check if block is valid - LFS_ASSERT(bd->cfg->erase_cycles); - LFS_ASSERT(block < cfg->block_count); - - LFS_TESTBD_TRACE("lfs_testbd_getwear -> %"PRIu32, bd->wear[block]); - return bd->wear[block]; -} - -int lfs_testbd_setwear(const struct lfs_config *cfg, - lfs_block_t block, lfs_testbd_wear_t wear) { - LFS_TESTBD_TRACE("lfs_testbd_setwear(%p, %"PRIu32")", (void*)cfg, block); - lfs_testbd_t *bd = cfg->context; - - // check if block is valid - LFS_ASSERT(bd->cfg->erase_cycles); - LFS_ASSERT(block < cfg->block_count); - - bd->wear[block] = wear; - - LFS_TESTBD_TRACE("lfs_testbd_setwear -> %d", 0); - return 0; -} diff --git a/components/fs/littlefs/littlefs/bd/lfs_testbd.h b/components/fs/littlefs/littlefs/bd/lfs_testbd.h deleted file mode 100644 index 61679e5e0..000000000 --- a/components/fs/littlefs/littlefs/bd/lfs_testbd.h +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Testing block device, wraps filebd and rambd while providing a bunch - * of hooks for testing littlefs in various conditions. - * - * Copyright (c) 2022, The littlefs authors. - * Copyright (c) 2017, Arm Limited. All rights reserved. - * SPDX-License-Identifier: BSD-3-Clause - */ -#ifndef LFS_TESTBD_H -#define LFS_TESTBD_H - -#include "lfs.h" -#include "lfs_util.h" -#include "bd/lfs_rambd.h" -#include "bd/lfs_filebd.h" - -#ifdef __cplusplus -extern "C" -{ -#endif - - -// Block device specific tracing -#ifdef LFS_TESTBD_YES_TRACE -#define LFS_TESTBD_TRACE(...) LFS_TRACE(__VA_ARGS__) -#else -#define LFS_TESTBD_TRACE(...) -#endif - -// Mode determining how "bad blocks" behave during testing. This simulates -// some real-world circumstances such as progs not sticking (prog-noop), -// a readonly disk (erase-noop), and ECC failures (read-error). -// -// Not that read-noop is not allowed. Read _must_ return a consistent (but -// may be arbitrary) value on every read. -enum lfs_testbd_badblock_behavior { - LFS_TESTBD_BADBLOCK_PROGERROR, - LFS_TESTBD_BADBLOCK_ERASEERROR, - LFS_TESTBD_BADBLOCK_READERROR, - LFS_TESTBD_BADBLOCK_PROGNOOP, - LFS_TESTBD_BADBLOCK_ERASENOOP, -}; - -// Type for measuring wear -typedef uint32_t lfs_testbd_wear_t; -typedef int32_t lfs_testbd_swear_t; - -// testbd config, this is required for testing -struct lfs_testbd_config { - // 8-bit erase value to use for simulating erases. -1 does not simulate - // erases, which can speed up testing by avoiding all the extra block-device - // operations to store the erase value. - int32_t erase_value; - - // Number of erase cycles before a block becomes "bad". The exact behavior - // of bad blocks is controlled by the badblock_mode. - uint32_t erase_cycles; - - // The mode determining how bad blocks fail - uint8_t badblock_behavior; - - // Number of write operations (erase/prog) before forcefully killing - // the program with exit. Simulates power-loss. 0 disables. - uint32_t power_cycles; - - // Optional buffer for RAM block device. - void *buffer; - - // Optional buffer for wear - void *wear_buffer; -}; - -// testbd state -typedef struct lfs_testbd { - union { - struct { - lfs_filebd_t bd; - struct lfs_filebd_config cfg; - } file; - struct { - lfs_rambd_t bd; - struct lfs_rambd_config cfg; - } ram; - } u; - - bool persist; - uint32_t power_cycles; - lfs_testbd_wear_t *wear; - - const struct lfs_testbd_config *cfg; -} lfs_testbd_t; - - -/// Block device API /// - -// Create a test block device using the geometry in lfs_config -// -// Note that filebd is used if a path is provided, if path is NULL -// testbd will use rambd which can be much faster. -int lfs_testbd_create(const struct lfs_config *cfg, const char *path); -int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, - const struct lfs_testbd_config *bdcfg); - -// Clean up memory associated with block device -int lfs_testbd_destroy(const struct lfs_config *cfg); - -// Read a block -int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, - lfs_off_t off, void *buffer, lfs_size_t size); - -// Program a block -// -// The block must have previously been erased. -int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, - lfs_off_t off, const void *buffer, lfs_size_t size); - -// Erase a block -// -// A block must be erased before being programmed. The -// state of an erased block is undefined. -int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block); - -// Sync the block device -int lfs_testbd_sync(const struct lfs_config *cfg); - - -/// Additional extended API for driving test features /// - -// Get simulated wear on a given block -lfs_testbd_swear_t lfs_testbd_getwear(const struct lfs_config *cfg, - lfs_block_t block); - -// Manually set simulated wear on a given block -int lfs_testbd_setwear(const struct lfs_config *cfg, - lfs_block_t block, lfs_testbd_wear_t wear); - - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif diff --git a/components/fs/littlefs/littlefs/benches/bench_dir.toml b/components/fs/littlefs/littlefs/benches/bench_dir.toml new file mode 100644 index 000000000..5f8cb490c --- /dev/null +++ b/components/fs/littlefs/littlefs/benches/bench_dir.toml @@ -0,0 +1,270 @@ +[cases.bench_dir_open] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.N = 1024 +defines.FILE_SIZE = 8 +defines.CHUNK_SIZE = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + // first create the files + char name[256]; + uint8_t buffer[CHUNK_SIZE]; + for (lfs_size_t i = 0; i < N; i++) { + sprintf(name, "file%08x", i); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint32_t file_prng = i; + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + buffer[k] = BENCH_PRNG(&file_prng); + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + } + + // then read the files + BENCH_START(); + uint32_t prng = 42; + for (lfs_size_t i = 0; i < N; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (N-1-i) + : BENCH_PRNG(&prng) % N; + sprintf(name, "file%08x", i_); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, LFS_O_RDONLY) => 0; + + uint32_t file_prng = i_; + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + lfs_file_read(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + assert(buffer[k] == BENCH_PRNG(&file_prng)); + } + } + + lfs_file_close(&lfs, &file) => 0; + } + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_dir_creat] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.N = 1024 +defines.FILE_SIZE = 8 +defines.CHUNK_SIZE = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + BENCH_START(); + uint32_t prng = 42; + char name[256]; + uint8_t buffer[CHUNK_SIZE]; + for (lfs_size_t i = 0; i < N; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (N-1-i) + : BENCH_PRNG(&prng) % N; + sprintf(name, "file%08x", i_); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; + + uint32_t file_prng = i_; + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + buffer[k] = BENCH_PRNG(&file_prng); + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + } + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_dir_remove] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.N = 1024 +defines.FILE_SIZE = 8 +defines.CHUNK_SIZE = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + // first create the files + char name[256]; + uint8_t buffer[CHUNK_SIZE]; + for (lfs_size_t i = 0; i < N; i++) { + sprintf(name, "file%08x", i); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint32_t file_prng = i; + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + buffer[k] = BENCH_PRNG(&file_prng); + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + } + + // then remove the files + BENCH_START(); + uint32_t prng = 42; + for (lfs_size_t i = 0; i < N; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (N-1-i) + : BENCH_PRNG(&prng) % N; + sprintf(name, "file%08x", i_); + int err = lfs_remove(&lfs, name); + assert(!err || err == LFS_ERR_NOENT); + } + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_dir_read] +defines.N = 1024 +defines.FILE_SIZE = 8 +defines.CHUNK_SIZE = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + // first create the files + char name[256]; + uint8_t buffer[CHUNK_SIZE]; + for (lfs_size_t i = 0; i < N; i++) { + sprintf(name, "file%08x", i); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint32_t file_prng = i; + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + buffer[k] = BENCH_PRNG(&file_prng); + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + } + + // then read the directory + BENCH_START(); + lfs_dir_t dir; + lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; + lfs_dir_read(&lfs, &dir, &info) => 1; + assert(info.type == LFS_TYPE_DIR); + assert(strcmp(info.name, ".") == 0); + lfs_dir_read(&lfs, &dir, &info) => 1; + assert(info.type == LFS_TYPE_DIR); + assert(strcmp(info.name, "..") == 0); + for (int i = 0; i < N; i++) { + sprintf(name, "file%08x", i); + lfs_dir_read(&lfs, &dir, &info) => 1; + assert(info.type == LFS_TYPE_REG); + assert(strcmp(info.name, name) == 0); + } + lfs_dir_read(&lfs, &dir, &info) => 0; + lfs_dir_close(&lfs, &dir) => 0; + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_dir_mkdir] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.N = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + BENCH_START(); + uint32_t prng = 42; + char name[256]; + for (lfs_size_t i = 0; i < N; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (N-1-i) + : BENCH_PRNG(&prng) % N; + printf("hm %d\n", i); + sprintf(name, "dir%08x", i_); + int err = lfs_mkdir(&lfs, name); + assert(!err || err == LFS_ERR_EXIST); + } + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_dir_rmdir] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.N = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + // first create the dirs + char name[256]; + for (lfs_size_t i = 0; i < N; i++) { + sprintf(name, "dir%08x", i); + lfs_mkdir(&lfs, name) => 0; + } + + // then remove the dirs + BENCH_START(); + uint32_t prng = 42; + for (lfs_size_t i = 0; i < N; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (N-1-i) + : BENCH_PRNG(&prng) % N; + sprintf(name, "dir%08x", i_); + int err = lfs_remove(&lfs, name); + assert(!err || err == LFS_ERR_NOENT); + } + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + + diff --git a/components/fs/littlefs/littlefs/benches/bench_file.toml b/components/fs/littlefs/littlefs/benches/bench_file.toml new file mode 100644 index 000000000..168eaad87 --- /dev/null +++ b/components/fs/littlefs/littlefs/benches/bench_file.toml @@ -0,0 +1,95 @@ +[cases.bench_file_read] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.SIZE = '128*1024' +defines.CHUNK_SIZE = 64 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_size_t chunks = (SIZE+CHUNK_SIZE-1)/CHUNK_SIZE; + + // first write the file + lfs_file_t file; + uint8_t buffer[CHUNK_SIZE]; + lfs_file_open(&lfs, &file, "file", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + for (lfs_size_t i = 0; i < chunks; i++) { + uint32_t chunk_prng = i; + for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) { + buffer[j] = BENCH_PRNG(&chunk_prng); + } + + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + lfs_file_close(&lfs, &file) => 0; + + // then read the file + BENCH_START(); + lfs_file_open(&lfs, &file, "file", LFS_O_RDONLY) => 0; + + uint32_t prng = 42; + for (lfs_size_t i = 0; i < chunks; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (chunks-1-i) + : BENCH_PRNG(&prng) % chunks; + lfs_file_seek(&lfs, &file, i_*CHUNK_SIZE, LFS_SEEK_SET) + => i_*CHUNK_SIZE; + lfs_file_read(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + + uint32_t chunk_prng = i_; + for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) { + assert(buffer[j] == BENCH_PRNG(&chunk_prng)); + } + } + + lfs_file_close(&lfs, &file) => 0; + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_file_write] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.SIZE = '128*1024' +defines.CHUNK_SIZE = 64 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_size_t chunks = (SIZE+CHUNK_SIZE-1)/CHUNK_SIZE; + + BENCH_START(); + lfs_file_t file; + lfs_file_open(&lfs, &file, "file", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint8_t buffer[CHUNK_SIZE]; + uint32_t prng = 42; + for (lfs_size_t i = 0; i < chunks; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (chunks-1-i) + : BENCH_PRNG(&prng) % chunks; + uint32_t chunk_prng = i_; + for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) { + buffer[j] = BENCH_PRNG(&chunk_prng); + } + + lfs_file_seek(&lfs, &file, i_*CHUNK_SIZE, LFS_SEEK_SET) + => i_*CHUNK_SIZE; + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' diff --git a/components/fs/littlefs/littlefs/benches/bench_superblock.toml b/components/fs/littlefs/littlefs/benches/bench_superblock.toml new file mode 100644 index 000000000..37659d477 --- /dev/null +++ b/components/fs/littlefs/littlefs/benches/bench_superblock.toml @@ -0,0 +1,56 @@ +[cases.bench_superblocks_found] +# support benchmarking with files +defines.N = [0, 1024] +defines.FILE_SIZE = 8 +defines.CHUNK_SIZE = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + + // create files? + lfs_mount(&lfs, cfg) => 0; + char name[256]; + uint8_t buffer[CHUNK_SIZE]; + for (lfs_size_t i = 0; i < N; i++) { + sprintf(name, "file%08x", i); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + buffer[k] = i+j+k; + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + } + lfs_unmount(&lfs) => 0; + + BENCH_START(); + lfs_mount(&lfs, cfg) => 0; + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_superblocks_missing] +code = ''' + lfs_t lfs; + + BENCH_START(); + int err = lfs_mount(&lfs, cfg); + assert(err != 0); + BENCH_STOP(); +''' + +[cases.bench_superblocks_format] +code = ''' + lfs_t lfs; + + BENCH_START(); + lfs_format(&lfs, cfg) => 0; + BENCH_STOP(); +''' + diff --git a/components/fs/littlefs/littlefs/lfs.c b/components/fs/littlefs/littlefs/lfs.c index 26280fa89..338bb6021 100644 --- a/components/fs/littlefs/littlefs/lfs.c +++ b/components/fs/littlefs/littlefs/lfs.c @@ -46,8 +46,8 @@ static int lfs_bd_read(lfs_t *lfs, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) { uint8_t *data = buffer; - if (block >= lfs->cfg->block_count || - off+size > lfs->cfg->block_size) { + if (off+size > lfs->cfg->block_size + || (lfs->block_count && block >= lfs->block_count)) { return LFS_ERR_CORRUPT; } @@ -104,7 +104,7 @@ static int lfs_bd_read(lfs_t *lfs, } // load to cache, first condition can no longer fail - LFS_ASSERT(block < lfs->cfg->block_count); + LFS_ASSERT(!lfs->block_count || block < lfs->block_count); rcache->block = block; rcache->off = lfs_aligndown(off, lfs->cfg->read_size); rcache->size = lfs_min( @@ -135,14 +135,14 @@ static int lfs_bd_cmp(lfs_t *lfs, uint8_t dat[8]; diff = lfs_min(size-i, sizeof(dat)); - int res = lfs_bd_read(lfs, + int err = lfs_bd_read(lfs, pcache, rcache, hint-i, block, off+i, &dat, diff); - if (res) { - return res; + if (err) { + return err; } - res = memcmp(dat, data + i, diff); + int res = memcmp(dat, data + i, diff); if (res) { return res < 0 ? LFS_CMP_LT : LFS_CMP_GT; } @@ -151,11 +151,32 @@ static int lfs_bd_cmp(lfs_t *lfs, return LFS_CMP_EQ; } +static int lfs_bd_crc(lfs_t *lfs, + const lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_size_t hint, + lfs_block_t block, lfs_off_t off, lfs_size_t size, uint32_t *crc) { + lfs_size_t diff = 0; + + for (lfs_off_t i = 0; i < size; i += diff) { + uint8_t dat[8]; + diff = lfs_min(size-i, sizeof(dat)); + int err = lfs_bd_read(lfs, + pcache, rcache, hint-i, + block, off+i, &dat, diff); + if (err) { + return err; + } + + *crc = lfs_crc(*crc, &dat, diff); + } + + return 0; +} + #ifndef LFS_READONLY static int lfs_bd_flush(lfs_t *lfs, lfs_cache_t *pcache, lfs_cache_t *rcache, bool validate) { if (pcache->block != LFS_BLOCK_NULL && pcache->block != LFS_BLOCK_INLINE) { - LFS_ASSERT(pcache->block < lfs->cfg->block_count); + LFS_ASSERT(pcache->block < lfs->block_count); lfs_size_t diff = lfs_alignup(pcache->size, lfs->cfg->prog_size); int err = lfs->cfg->prog(lfs->cfg, pcache->block, pcache->off, pcache->buffer, diff); @@ -208,7 +229,7 @@ static int lfs_bd_prog(lfs_t *lfs, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size) { const uint8_t *data = buffer; - LFS_ASSERT(block == LFS_BLOCK_INLINE || block < lfs->cfg->block_count); + LFS_ASSERT(block == LFS_BLOCK_INLINE || block < lfs->block_count); LFS_ASSERT(off + size <= lfs->cfg->block_size); while (size > 0) { @@ -252,7 +273,7 @@ static int lfs_bd_prog(lfs_t *lfs, #ifndef LFS_READONLY static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) { - LFS_ASSERT(block < lfs->cfg->block_count); + LFS_ASSERT(block < lfs->block_count); int err = lfs->cfg->erase(lfs->cfg, block); LFS_ASSERT(err <= 0); return err; @@ -279,14 +300,12 @@ static inline int lfs_pair_cmp( paira[0] == pairb[1] || paira[1] == pairb[0]); } -#ifndef LFS_READONLY -static inline bool lfs_pair_sync( +static inline bool lfs_pair_issync( const lfs_block_t paira[2], const lfs_block_t pairb[2]) { return (paira[0] == pairb[0] && paira[1] == pairb[1]) || (paira[0] == pairb[1] && paira[1] == pairb[0]); } -#endif static inline void lfs_pair_fromle32(lfs_block_t pair[2]) { pair[0] = lfs_fromle32(pair[0]); @@ -325,6 +344,10 @@ static inline uint16_t lfs_tag_type1(lfs_tag_t tag) { return (tag & 0x70000000) >> 20; } +static inline uint16_t lfs_tag_type2(lfs_tag_t tag) { + return (tag & 0x78000000) >> 20; +} + static inline uint16_t lfs_tag_type3(lfs_tag_t tag) { return (tag & 0x7ff00000) >> 20; } @@ -386,7 +409,7 @@ static inline bool lfs_gstate_hasorphans(const lfs_gstate_t *a) { } static inline uint8_t lfs_gstate_getorphans(const lfs_gstate_t *a) { - return lfs_tag_size(a->tag); + return lfs_tag_size(a->tag) & 0x1ff; } static inline bool lfs_gstate_hasmove(const lfs_gstate_t *a) { @@ -394,6 +417,10 @@ static inline bool lfs_gstate_hasmove(const lfs_gstate_t *a) { } #endif +static inline bool lfs_gstate_needssuperblock(const lfs_gstate_t *a) { + return lfs_tag_size(a->tag) >> 9; +} + static inline bool lfs_gstate_hasmovehere(const lfs_gstate_t *a, const lfs_block_t *pair) { return lfs_tag_type1(a->tag) && lfs_pair_cmp(a->pair, pair) == 0; @@ -413,6 +440,24 @@ static inline void lfs_gstate_tole32(lfs_gstate_t *a) { } #endif +// operations on forward-CRCs used to track erased state +struct lfs_fcrc { + lfs_size_t size; + uint32_t crc; +}; + +static void lfs_fcrc_fromle32(struct lfs_fcrc *fcrc) { + fcrc->size = lfs_fromle32(fcrc->size); + fcrc->crc = lfs_fromle32(fcrc->crc); +} + +#ifndef LFS_READONLY +static void lfs_fcrc_tole32(struct lfs_fcrc *fcrc) { + fcrc->size = lfs_tole32(fcrc->size); + fcrc->crc = lfs_tole32(fcrc->crc); +} +#endif + // other endianness operations static void lfs_ctz_fromle32(struct lfs_ctz *ctz) { ctz->head = lfs_fromle32(ctz->head); @@ -473,6 +518,28 @@ static void lfs_mlist_append(lfs_t *lfs, struct lfs_mlist *mlist) { lfs->mlist = mlist; } +// some other filesystem operations +static uint32_t lfs_fs_disk_version(lfs_t *lfs) { + (void)lfs; +#ifdef LFS_MULTIVERSION + if (lfs->cfg->disk_version) { + return lfs->cfg->disk_version; + } else +#endif + { + return LFS_DISK_VERSION; + } +} + +static uint16_t lfs_fs_disk_version_major(lfs_t *lfs) { + return 0xffff & (lfs_fs_disk_version(lfs) >> 16); + +} + +static uint16_t lfs_fs_disk_version_minor(lfs_t *lfs) { + return 0xffff & (lfs_fs_disk_version(lfs) >> 0); +} + /// Internal operations predeclared here /// #ifndef LFS_READONLY @@ -500,6 +567,8 @@ static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t dir[2], static int lfs_fs_forceconsistency(lfs_t *lfs); #endif +static void lfs_fs_prepsuperblock(lfs_t *lfs, bool needssuperblock); + #ifdef LFS_MIGRATE static int lfs1_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); @@ -528,7 +597,7 @@ static int lfs_rawunmount(lfs_t *lfs); static int lfs_alloc_lookahead(void *p, lfs_block_t block) { lfs_t *lfs = (lfs_t*)p; lfs_block_t off = ((block - lfs->free.off) - + lfs->cfg->block_count) % lfs->cfg->block_count; + + lfs->block_count) % lfs->block_count; if (off < lfs->free.size) { lfs->free.buffer[off / 32] |= 1U << (off % 32); @@ -542,7 +611,7 @@ static int lfs_alloc_lookahead(void *p, lfs_block_t block) { // is to prevent blocks from being garbage collected in the middle of a // commit operation static void lfs_alloc_ack(lfs_t *lfs) { - lfs->free.ack = lfs->cfg->block_count; + lfs->free.ack = lfs->block_count; } // drop the lookahead buffer, this is done during mounting and failed @@ -553,6 +622,26 @@ static void lfs_alloc_drop(lfs_t *lfs) { lfs_alloc_ack(lfs); } +#ifndef LFS_READONLY +static int lfs_fs_rawgc(lfs_t *lfs) { + // Move free offset at the first unused block (lfs->free.i) + // lfs->free.i is equal lfs->free.size when all blocks are used + lfs->free.off = (lfs->free.off + lfs->free.i) % lfs->block_count; + lfs->free.size = lfs_min(8*lfs->cfg->lookahead_size, lfs->free.ack); + lfs->free.i = 0; + + // find mask of free blocks from tree + memset(lfs->free.buffer, 0, lfs->cfg->lookahead_size); + int err = lfs_fs_rawtraverse(lfs, lfs_alloc_lookahead, lfs, true); + if (err) { + lfs_alloc_drop(lfs); + return err; + } + + return 0; +} +#endif + #ifndef LFS_READONLY static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { while (true) { @@ -563,7 +652,7 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { if (!(lfs->free.buffer[off / 32] & (1U << (off % 32)))) { // found a free block - *block = (lfs->free.off + off) % lfs->cfg->block_count; + *block = (lfs->free.off + off) % lfs->block_count; // eagerly find next off so an alloc ack can // discredit old lookahead blocks @@ -585,16 +674,8 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { return LFS_ERR_NOSPC; } - lfs->free.off = (lfs->free.off + lfs->free.size) - % lfs->cfg->block_count; - lfs->free.size = lfs_min(8*lfs->cfg->lookahead_size, lfs->free.ack); - lfs->free.i = 0; - - // find mask of free blocks from tree - memset(lfs->free.buffer, 0, lfs->cfg->lookahead_size); - int err = lfs_fs_rawtraverse(lfs, lfs_alloc_lookahead, lfs, true); - if (err) { - lfs_alloc_drop(lfs); + int err = lfs_fs_rawgc(lfs); + if(err) { return err; } } @@ -808,7 +889,7 @@ static int lfs_dir_traverse(lfs_t *lfs, // iterate over directory and attrs lfs_tag_t tag; const void *buffer; - struct lfs_diskoff disk; + struct lfs_diskoff disk = {0}; while (true) { { if (off+lfs_tag_dsize(ptag) < dir->off) { @@ -998,7 +1079,8 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, // if either block address is invalid we return LFS_ERR_CORRUPT here, // otherwise later writes to the pair could fail - if (pair[0] >= lfs->cfg->block_count || pair[1] >= lfs->cfg->block_count) { + if (lfs->block_count + && (pair[0] >= lfs->block_count || pair[1] >= lfs->block_count)) { return LFS_ERR_CORRUPT; } @@ -1035,6 +1117,11 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, bool tempsplit = false; lfs_stag_t tempbesttag = besttag; + // assume not erased until proven otherwise + bool maybeerased = false; + bool hasfcrc = false; + struct lfs_fcrc fcrc; + dir->rev = lfs_tole32(dir->rev); uint32_t crc = lfs_crc(0xffffffff, &dir->rev, sizeof(dir->rev)); dir->rev = lfs_fromle32(dir->rev); @@ -1049,7 +1136,6 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, if (err) { if (err == LFS_ERR_CORRUPT) { // can't continue? - dir->erased = false; break; } return err; @@ -1058,19 +1144,19 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, crc = lfs_crc(crc, &tag, sizeof(tag)); tag = lfs_frombe32(tag) ^ ptag; - // next commit not yet programmed or we're not in valid range + // next commit not yet programmed? if (!lfs_tag_isvalid(tag)) { - dir->erased = (lfs_tag_type1(ptag) == LFS_TYPE_CRC && - dir->off % lfs->cfg->prog_size == 0); + // we only might be erased if the last tag was a crc + maybeerased = (lfs_tag_type2(ptag) == LFS_TYPE_CCRC); break; + // out of range? } else if (off + lfs_tag_dsize(tag) > lfs->cfg->block_size) { - dir->erased = false; break; } ptag = tag; - if (lfs_tag_type1(tag) == LFS_TYPE_CRC) { + if (lfs_tag_type2(tag) == LFS_TYPE_CCRC) { // check the crc attr uint32_t dcrc; err = lfs_bd_read(lfs, @@ -1078,7 +1164,6 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, dir->pair[0], off+sizeof(tag), &dcrc, sizeof(dcrc)); if (err) { if (err == LFS_ERR_CORRUPT) { - dir->erased = false; break; } return err; @@ -1086,7 +1171,6 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, dcrc = lfs_fromle32(dcrc); if (crc != dcrc) { - dir->erased = false; break; } @@ -1108,26 +1192,21 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, dir->tail[1] = temptail[1]; dir->split = tempsplit; - // reset crc + // reset crc, hasfcrc crc = 0xffffffff; continue; } // crc the entry first, hopefully leaving it in the cache - for (lfs_off_t j = sizeof(tag); j < lfs_tag_dsize(tag); j++) { - uint8_t dat; - err = lfs_bd_read(lfs, - NULL, &lfs->rcache, lfs->cfg->block_size, - dir->pair[0], off+j, &dat, 1); - if (err) { - if (err == LFS_ERR_CORRUPT) { - dir->erased = false; - break; - } - return err; + err = lfs_bd_crc(lfs, + NULL, &lfs->rcache, lfs->cfg->block_size, + dir->pair[0], off+sizeof(tag), + lfs_tag_dsize(tag)-sizeof(tag), &crc); + if (err) { + if (err == LFS_ERR_CORRUPT) { + break; } - - crc = lfs_crc(crc, &dat, 1); + return err; } // directory modification tags? @@ -1154,12 +1233,24 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, dir->pair[0], off+sizeof(tag), &temptail, 8); if (err) { if (err == LFS_ERR_CORRUPT) { - dir->erased = false; break; } return err; } lfs_pair_fromle32(temptail); + } else if (lfs_tag_type3(tag) == LFS_TYPE_FCRC) { + err = lfs_bd_read(lfs, + NULL, &lfs->rcache, lfs->cfg->block_size, + dir->pair[0], off+sizeof(tag), + &fcrc, sizeof(fcrc)); + if (err) { + if (err == LFS_ERR_CORRUPT) { + break; + } + } + + lfs_fcrc_fromle32(&fcrc); + hasfcrc = true; } // found a match for our fetcher? @@ -1168,7 +1259,6 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, dir->pair[0], off+sizeof(tag)}); if (res < 0) { if (res == LFS_ERR_CORRUPT) { - dir->erased = false; break; } return res; @@ -1190,35 +1280,67 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, } } - // consider what we have good enough - if (dir->off > 0) { - // synthetic move - if (lfs_gstate_hasmovehere(&lfs->gdisk, dir->pair)) { - if (lfs_tag_id(lfs->gdisk.tag) == lfs_tag_id(besttag)) { - besttag |= 0x80000000; - } else if (besttag != -1 && - lfs_tag_id(lfs->gdisk.tag) < lfs_tag_id(besttag)) { - besttag -= LFS_MKTAG(0, 1, 0); + // found no valid commits? + if (dir->off == 0) { + // try the other block? + lfs_pair_swap(dir->pair); + dir->rev = revs[(r+1)%2]; + continue; + } + + // did we end on a valid commit? we may have an erased block + dir->erased = false; + if (maybeerased && dir->off % lfs->cfg->prog_size == 0) { + #ifdef LFS_MULTIVERSION + // note versions < lfs2.1 did not have fcrc tags, if + // we're < lfs2.1 treat missing fcrc as erased data + // + // we don't strictly need to do this, but otherwise writing + // to lfs2.0 disks becomes very inefficient + if (lfs_fs_disk_version(lfs) < 0x00020001) { + dir->erased = true; + + } else + #endif + if (hasfcrc) { + // check for an fcrc matching the next prog's erased state, if + // this failed most likely a previous prog was interrupted, we + // need a new erase + uint32_t fcrc_ = 0xffffffff; + int err = lfs_bd_crc(lfs, + NULL, &lfs->rcache, lfs->cfg->block_size, + dir->pair[0], dir->off, fcrc.size, &fcrc_); + if (err && err != LFS_ERR_CORRUPT) { + return err; } - } - // found tag? or found best id? - if (id) { - *id = lfs_min(lfs_tag_id(besttag), dir->count); + // found beginning of erased part? + dir->erased = (fcrc_ == fcrc.crc); } + } - if (lfs_tag_isvalid(besttag)) { - return besttag; - } else if (lfs_tag_id(besttag) < dir->count) { - return LFS_ERR_NOENT; - } else { - return 0; + // synthetic move + if (lfs_gstate_hasmovehere(&lfs->gdisk, dir->pair)) { + if (lfs_tag_id(lfs->gdisk.tag) == lfs_tag_id(besttag)) { + besttag |= 0x80000000; + } else if (besttag != -1 && + lfs_tag_id(lfs->gdisk.tag) < lfs_tag_id(besttag)) { + besttag -= LFS_MKTAG(0, 1, 0); } } - // failed, try the other block? - lfs_pair_swap(dir->pair); - dir->rev = revs[(r+1)%2]; + // found tag? or found best id? + if (id) { + *id = lfs_min(lfs_tag_id(besttag), dir->count); + } + + if (lfs_tag_isvalid(besttag)) { + return besttag; + } else if (lfs_tag_id(besttag) < dir->count) { + return LFS_ERR_NOENT; + } else { + return 0; + } } LFS_ERROR("Corrupted dir pair at {0x%"PRIx32", 0x%"PRIx32"}", @@ -1256,7 +1378,7 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id, struct lfs_info *info) { if (id == 0x3ff) { // special case for root - strcpy(info->name, "/"); + strlcpy(info->name, "/", sizeof(info->name)); info->type = LFS_TYPE_DIR; return 0; } @@ -1492,9 +1614,15 @@ static int lfs_dir_commitattr(lfs_t *lfs, struct lfs_commit *commit, #endif #ifndef LFS_READONLY + static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) { // align to program units - const lfs_off_t end = lfs_alignup(commit->off + 2*sizeof(uint32_t), + // + // this gets a bit complex as we have two types of crcs: + // - 5-word crc with fcrc to check following prog (middle of block) + // - 2-word crc with no following prog (end of block) + const lfs_off_t end = lfs_alignup( + lfs_min(commit->off + 5*sizeof(uint32_t), lfs->cfg->block_size), lfs->cfg->prog_size); lfs_off_t off1 = 0; @@ -1504,89 +1632,128 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) { // padding is not crced, which lets fetches skip padding but // makes committing a bit more complicated while (commit->off < end) { - lfs_off_t off = commit->off + sizeof(lfs_tag_t); - lfs_off_t noff = lfs_min(end - off, 0x3fe) + off; + lfs_off_t noff = ( + lfs_min(end - (commit->off+sizeof(lfs_tag_t)), 0x3fe) + + (commit->off+sizeof(lfs_tag_t))); + // too large for crc tag? need padding commits if (noff < end) { - noff = lfs_min(noff, end - 2*sizeof(uint32_t)); + noff = lfs_min(noff, end - 5*sizeof(uint32_t)); } - // read erased state from next program unit - lfs_tag_t tag = 0xffffffff; - int err = lfs_bd_read(lfs, - NULL, &lfs->rcache, sizeof(tag), - commit->block, noff, &tag, sizeof(tag)); - if (err && err != LFS_ERR_CORRUPT) { - return err; - } + // space for fcrc? + uint8_t eperturb = (uint8_t)-1; + if (noff >= end && noff <= lfs->cfg->block_size - lfs->cfg->prog_size) { + // first read the leading byte, this always contains a bit + // we can perturb to avoid writes that don't change the fcrc + int err = lfs_bd_read(lfs, + NULL, &lfs->rcache, lfs->cfg->prog_size, + commit->block, noff, &eperturb, 1); + if (err && err != LFS_ERR_CORRUPT) { + return err; + } - // build crc tag - bool reset = ~lfs_frombe32(tag) >> 31; - tag = LFS_MKTAG(LFS_TYPE_CRC + reset, 0x3ff, noff - off); + #ifdef LFS_MULTIVERSION + // unfortunately fcrcs break mdir fetching < lfs2.1, so only write + // these if we're a >= lfs2.1 filesystem + if (lfs_fs_disk_version(lfs) <= 0x00020000) { + // don't write fcrc + } else + #endif + { + // find the expected fcrc, don't bother avoiding a reread + // of the eperturb, it should still be in our cache + struct lfs_fcrc fcrc = { + .size = lfs->cfg->prog_size, + .crc = 0xffffffff + }; + err = lfs_bd_crc(lfs, + NULL, &lfs->rcache, lfs->cfg->prog_size, + commit->block, noff, fcrc.size, &fcrc.crc); + if (err && err != LFS_ERR_CORRUPT) { + return err; + } + + lfs_fcrc_tole32(&fcrc); + err = lfs_dir_commitattr(lfs, commit, + LFS_MKTAG(LFS_TYPE_FCRC, 0x3ff, sizeof(struct lfs_fcrc)), + &fcrc); + if (err) { + return err; + } + } + } - // write out crc - uint32_t footer[2]; - footer[0] = lfs_tobe32(tag ^ commit->ptag); - commit->crc = lfs_crc(commit->crc, &footer[0], sizeof(footer[0])); - footer[1] = lfs_tole32(commit->crc); - err = lfs_bd_prog(lfs, + // build commit crc + struct { + lfs_tag_t tag; + uint32_t crc; + } ccrc; + lfs_tag_t ntag = LFS_MKTAG( + LFS_TYPE_CCRC + (((uint8_t)~eperturb) >> 7), 0x3ff, + noff - (commit->off+sizeof(lfs_tag_t))); + ccrc.tag = lfs_tobe32(ntag ^ commit->ptag); + commit->crc = lfs_crc(commit->crc, &ccrc.tag, sizeof(lfs_tag_t)); + ccrc.crc = lfs_tole32(commit->crc); + + int err = lfs_bd_prog(lfs, &lfs->pcache, &lfs->rcache, false, - commit->block, commit->off, &footer, sizeof(footer)); + commit->block, commit->off, &ccrc, sizeof(ccrc)); if (err) { return err; } // keep track of non-padding checksum to verify if (off1 == 0) { - off1 = commit->off + sizeof(uint32_t); + off1 = commit->off + sizeof(lfs_tag_t); crc1 = commit->crc; } - commit->off += sizeof(tag)+lfs_tag_size(tag); - commit->ptag = tag ^ ((lfs_tag_t)reset << 31); - commit->crc = 0xffffffff; // reset crc for next "commit" - } + commit->off = noff; + // perturb valid bit? + commit->ptag = ntag ^ ((0x80UL & ~eperturb) << 24); + // reset crc for next commit + commit->crc = 0xffffffff; - // flush buffers - int err = lfs_bd_sync(lfs, &lfs->pcache, &lfs->rcache, false); - if (err) { - return err; + // manually flush here since we don't prog the padding, this confuses + // the caching layer + if (noff >= end || noff >= lfs->pcache.off + lfs->cfg->cache_size) { + // flush buffers + int err = lfs_bd_sync(lfs, &lfs->pcache, &lfs->rcache, false); + if (err) { + return err; + } + } } // successful commit, check checksums to make sure + // + // note that we don't need to check padding commits, worst + // case if they are corrupted we would have had to compact anyways lfs_off_t off = commit->begin; - lfs_off_t noff = off1; - while (off < end) { - uint32_t crc = 0xffffffff; - for (lfs_off_t i = off; i < noff+sizeof(uint32_t); i++) { - // check against written crc, may catch blocks that - // become readonly and match our commit size exactly - if (i == off1 && crc != crc1) { - return LFS_ERR_CORRUPT; - } - - // leave it up to caching to make this efficient - uint8_t dat; - err = lfs_bd_read(lfs, - NULL, &lfs->rcache, noff+sizeof(uint32_t)-i, - commit->block, i, &dat, 1); - if (err) { - return err; - } + uint32_t crc = 0xffffffff; + int err = lfs_bd_crc(lfs, + NULL, &lfs->rcache, off1+sizeof(uint32_t), + commit->block, off, off1-off, &crc); + if (err) { + return err; + } - crc = lfs_crc(crc, &dat, 1); - } + // check non-padding commits against known crc + if (crc != crc1) { + return LFS_ERR_CORRUPT; + } - // detected write error? - if (crc != 0) { - return LFS_ERR_CORRUPT; - } + // make sure to check crc in case we happen to pick + // up an unrelated crc (frozen block?) + err = lfs_bd_crc(lfs, + NULL, &lfs->rcache, sizeof(uint32_t), + commit->block, off1, sizeof(uint32_t), &crc); + if (err) { + return err; + } - // skip padding - off = lfs_min(end - noff, 0x3fe) + noff; - if (off < end) { - off = lfs_min(off, end - 2*sizeof(uint32_t)); - } - noff = off + sizeof(uint32_t); + if (crc != 0) { + return LFS_ERR_CORRUPT; } return 0; @@ -1927,11 +2094,20 @@ static int lfs_dir_splittingcompact(lfs_t *lfs, lfs_mdir_t *dir, return err; } - // space is complicated, we need room for tail, crc, gstate, - // cleanup delete, and we cap at half a block to give room - // for metadata updates. + // space is complicated, we need room for: + // + // - tail: 4+2*4 = 12 bytes + // - gstate: 4+3*4 = 16 bytes + // - move delete: 4 = 4 bytes + // - crc: 4+4 = 8 bytes + // total = 40 bytes + // + // And we cap at half a block to avoid degenerate cases with + // nearly-full metadata blocks. + // if (end - split < 0xff - && size <= lfs_min(lfs->cfg->block_size - 36, + && size <= lfs_min( + lfs->cfg->block_size - 40, lfs_alignup( (lfs->cfg->metadata_max ? lfs->cfg->metadata_max @@ -1975,9 +2151,11 @@ static int lfs_dir_splittingcompact(lfs_t *lfs, lfs_mdir_t *dir, return size; } - // do we have extra space? littlefs can't reclaim this space - // by itself, so expand cautiously - if ((lfs_size_t)size < lfs->cfg->block_count/2) { + // littlefs cannot reclaim expanded superblocks, so expand cautiously + // + // if our filesystem is more than ~88% full, don't expand, this is + // somewhat arbitrary + if (lfs->block_count - size > lfs->block_count/8) { LFS_DEBUG("Expanding superblock at rev %"PRIu32, dir->rev); int err = lfs_dir_split(lfs, dir, attrs, attrcount, source, begin, end); @@ -2541,12 +2719,12 @@ static int lfs_dir_rawread(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { // special offset for '.' and '..' if (dir->pos == 0) { info->type = LFS_TYPE_DIR; - strcpy(info->name, "."); + strlcpy(info->name, ".", sizeof(info->name)); dir->pos += 1; return true; } else if (dir->pos == 1) { info->type = LFS_TYPE_DIR; - strcpy(info->name, ".."); + strlcpy(info->name, "..", sizeof(info->name)); dir->pos += 1; return true; } @@ -2595,11 +2773,6 @@ static int lfs_dir_rawseek(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off) { dir->id = (off > 0 && lfs_pair_cmp(dir->head, lfs->root) == 0); while (off > 0) { - int diff = lfs_min(dir->m.count - dir->id, off); - dir->id += diff; - dir->pos += diff; - off -= diff; - if (dir->id == dir->m.count) { if (!dir->m.split) { return LFS_ERR_INVAL; @@ -2612,6 +2785,11 @@ static int lfs_dir_rawseek(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off) { dir->id = 0; } + + int diff = lfs_min(dir->m.count - dir->id, off); + dir->id += diff; + dir->pos += diff; + off -= diff; } return 0; @@ -3348,7 +3526,7 @@ static lfs_ssize_t lfs_file_flushedwrite(lfs_t *lfs, lfs_file_t *file, // find out which block we're extending from int err = lfs_ctz_find(lfs, NULL, &file->cache, file->ctz.head, file->ctz.size, - file->pos-1, &file->block, &file->off); + file->pos-1, &file->block, &(lfs_off_t){0}); if (err) { file->flags |= LFS_F_ERRED; return err; @@ -3526,26 +3704,55 @@ static int lfs_file_rawtruncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) { lfs_off_t pos = file->pos; lfs_off_t oldsize = lfs_file_rawsize(lfs, file); if (size < oldsize) { - // need to flush since directly changing metadata - int err = lfs_file_flush(lfs, file); - if (err) { - return err; - } + // revert to inline file? + if (size <= lfs_min(0x3fe, lfs_min( + lfs->cfg->cache_size, + (lfs->cfg->metadata_max ? + lfs->cfg->metadata_max : lfs->cfg->block_size) / 8))) { + // flush+seek to head + lfs_soff_t res = lfs_file_rawseek(lfs, file, 0, LFS_SEEK_SET); + if (res < 0) { + return (int)res; + } - // lookup new head in ctz skip list - err = lfs_ctz_find(lfs, NULL, &file->cache, - file->ctz.head, file->ctz.size, - size, &file->block, &file->off); - if (err) { - return err; - } + // read our data into rcache temporarily + lfs_cache_drop(lfs, &lfs->rcache); + res = lfs_file_flushedread(lfs, file, + lfs->rcache.buffer, size); + if (res < 0) { + return (int)res; + } - // need to set pos/block/off consistently so seeking back to - // the old position does not get confused - file->pos = size; - file->ctz.head = file->block; - file->ctz.size = size; - file->flags |= LFS_F_DIRTY | LFS_F_READING; + file->ctz.head = LFS_BLOCK_INLINE; + file->ctz.size = size; + file->flags |= LFS_F_DIRTY | LFS_F_READING | LFS_F_INLINE; + file->cache.block = file->ctz.head; + file->cache.off = 0; + file->cache.size = lfs->cfg->cache_size; + memcpy(file->cache.buffer, lfs->rcache.buffer, size); + + } else { + // need to flush since directly changing metadata + int err = lfs_file_flush(lfs, file); + if (err) { + return err; + } + + // lookup new head in ctz skip list + err = lfs_ctz_find(lfs, NULL, &file->cache, + file->ctz.head, file->ctz.size, + size-1, &file->block, &(lfs_off_t){0}); + if (err) { + return err; + } + + // need to set pos/block/off consistently so seeking back to + // the old position does not get confused + file->pos = size; + file->ctz.head = file->block; + file->ctz.size = size; + file->flags |= LFS_F_DIRTY | LFS_F_READING; + } } else if (size > oldsize) { // flush+seek if not already at end lfs_soff_t res = lfs_file_rawseek(lfs, file, 0, LFS_SEEK_END); @@ -3903,8 +4110,24 @@ static int lfs_rawremoveattr(lfs_t *lfs, const char *path, uint8_t type) { /// Filesystem operations /// static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->cfg = cfg; + lfs->block_count = cfg->block_count; // May be 0 int err = 0; +#ifdef LFS_MULTIVERSION + // this driver only supports minor version < current minor version + LFS_ASSERT(!lfs->cfg->disk_version || ( + (0xffff & (lfs->cfg->disk_version >> 16)) + == LFS_DISK_VERSION_MAJOR + && (0xffff & (lfs->cfg->disk_version >> 0)) + <= LFS_DISK_VERSION_MINOR)); +#endif + + // check that bool is a truthy-preserving type + // + // note the most common reason for this failure is a before-c99 compiler, + // which littlefs currently does not support + LFS_ASSERT((bool)0x80000000); + // validate that the lfs-cfg sizes were initiated properly before // performing any arithmetic logics with them LFS_ASSERT(lfs->cfg->read_size != 0); @@ -3917,7 +4140,10 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { LFS_ASSERT(lfs->cfg->cache_size % lfs->cfg->prog_size == 0); LFS_ASSERT(lfs->cfg->block_size % lfs->cfg->cache_size == 0); - // check that the block size is large enough to fit ctz pointers + // check that the block size is large enough to fit all ctz pointers + LFS_ASSERT(lfs->cfg->block_size >= 128); + // this is the exact calculation for all ctz pointers, if this fails + // and the simpler assert above does not, math must be broken LFS_ASSERT(4*lfs_npw2(0xffffffff / (lfs->cfg->block_size-2*4)) <= lfs->cfg->block_size); @@ -4027,6 +4253,8 @@ static int lfs_deinit(lfs_t *lfs) { return 0; } + + #ifndef LFS_READONLY static int lfs_rawformat(lfs_t *lfs, const struct lfs_config *cfg) { int err = 0; @@ -4036,11 +4264,13 @@ static int lfs_rawformat(lfs_t *lfs, const struct lfs_config *cfg) { return err; } + LFS_ASSERT(cfg->block_count != 0); + // create free lookahead memset(lfs->free.buffer, 0, lfs->cfg->lookahead_size); lfs->free.off = 0; lfs->free.size = lfs_min(8*lfs->cfg->lookahead_size, - lfs->cfg->block_count); + lfs->block_count); lfs->free.i = 0; lfs_alloc_ack(lfs); @@ -4053,9 +4283,9 @@ static int lfs_rawformat(lfs_t *lfs, const struct lfs_config *cfg) { // write one superblock lfs_superblock_t superblock = { - .version = LFS_DISK_VERSION, + .version = lfs_fs_disk_version(lfs), .block_size = lfs->cfg->block_size, - .block_count = lfs->cfg->block_count, + .block_count = lfs->block_count, .name_max = lfs->name_max, .file_max = lfs->file_max, .attr_max = lfs->attr_max, @@ -4101,14 +4331,23 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { // scan directory blocks for superblock and any global updates lfs_mdir_t dir = {.tail = {0, 1}}; - lfs_block_t cycle = 0; + lfs_block_t tortoise[2] = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}; + lfs_size_t tortoise_i = 1; + lfs_size_t tortoise_period = 1; while (!lfs_pair_isnull(dir.tail)) { - if (cycle >= lfs->cfg->block_count/2) { - // loop detected + // detect cycles with Brent's algorithm + if (lfs_pair_issync(dir.tail, tortoise)) { + LFS_WARN("Cycle detected in tail list"); err = LFS_ERR_CORRUPT; goto cleanup; } - cycle += 1; + if (tortoise_i == tortoise_period) { + tortoise[0] = dir.tail[0]; + tortoise[1] = dir.tail[1]; + tortoise_i = 0; + tortoise_period *= 2; + } + tortoise_i += 1; // fetch next block in tail list lfs_stag_t tag = lfs_dir_fetchmatch(lfs, &dir, dir.tail, @@ -4142,14 +4381,33 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { // check version uint16_t major_version = (0xffff & (superblock.version >> 16)); uint16_t minor_version = (0xffff & (superblock.version >> 0)); - if ((major_version != LFS_DISK_VERSION_MAJOR || - minor_version > LFS_DISK_VERSION_MINOR)) { - LFS_ERROR("Invalid version v%"PRIu16".%"PRIu16, - major_version, minor_version); + if (major_version != lfs_fs_disk_version_major(lfs) + || minor_version > lfs_fs_disk_version_minor(lfs)) { + LFS_ERROR("Invalid version " + "v%"PRIu16".%"PRIu16" != v%"PRIu16".%"PRIu16, + major_version, + minor_version, + lfs_fs_disk_version_major(lfs), + lfs_fs_disk_version_minor(lfs)); err = LFS_ERR_INVAL; goto cleanup; } + // found older minor version? set an in-device only bit in the + // gstate so we know we need to rewrite the superblock before + // the first write + if (minor_version < lfs_fs_disk_version_minor(lfs)) { + LFS_DEBUG("Found older minor version " + "v%"PRIu16".%"PRIu16" < v%"PRIu16".%"PRIu16, + major_version, + minor_version, + lfs_fs_disk_version_major(lfs), + lfs_fs_disk_version_minor(lfs)); + // note this bit is reserved on disk, so fetching more gstate + // will not interfere here + lfs_fs_prepsuperblock(lfs, true); + } + // check superblock configuration if (superblock.name_max) { if (superblock.name_max > lfs->name_max) { @@ -4184,13 +4442,17 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { lfs->attr_max = superblock.attr_max; } - if (superblock.block_count != lfs->cfg->block_count) { + // this is where we get the block_count from disk if block_count=0 + if (lfs->cfg->block_count + && superblock.block_count != lfs->cfg->block_count) { LFS_ERROR("Invalid block count (%"PRIu32" != %"PRIu32")", superblock.block_count, lfs->cfg->block_count); err = LFS_ERR_INVAL; goto cleanup; } + lfs->block_count = superblock.block_count; + if (superblock.block_size != lfs->cfg->block_size) { LFS_ERROR("Invalid block size (%"PRIu32" != %"PRIu32")", superblock.block_size, lfs->cfg->block_size); @@ -4206,12 +4468,6 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { } } - // found superblock? - if (lfs_pair_isnull(lfs->root)) { - err = LFS_ERR_INVAL; - goto cleanup; - } - // update littlefs with gstate if (!lfs_gstate_iszero(&lfs->gstate)) { LFS_DEBUG("Found pending gstate 0x%08"PRIx32"%08"PRIx32"%08"PRIx32, @@ -4224,7 +4480,7 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { // setup free lookahead, to distribute allocations uniformly across // boots, we start the allocator at a random location - lfs->free.off = lfs->seed % lfs->cfg->block_count; + lfs->free.off = lfs->seed % lfs->block_count; lfs_alloc_drop(lfs); return 0; @@ -4240,6 +4496,46 @@ static int lfs_rawunmount(lfs_t *lfs) { /// Filesystem filesystem operations /// +static int lfs_fs_rawstat(lfs_t *lfs, struct lfs_fsinfo *fsinfo) { + // if the superblock is up-to-date, we must be on the most recent + // minor version of littlefs + if (!lfs_gstate_needssuperblock(&lfs->gstate)) { + fsinfo->disk_version = lfs_fs_disk_version(lfs); + + // otherwise we need to read the minor version on disk + } else { + // fetch the superblock + lfs_mdir_t dir; + int err = lfs_dir_fetch(lfs, &dir, lfs->root); + if (err) { + return err; + } + + lfs_superblock_t superblock; + lfs_stag_t tag = lfs_dir_get(lfs, &dir, LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock); + if (tag < 0) { + return tag; + } + lfs_superblock_fromle32(&superblock); + + // read the on-disk version + fsinfo->disk_version = superblock.version; + } + + // filesystem geometry + fsinfo->block_size = lfs->cfg->block_size; + fsinfo->block_count = lfs->block_count; + + // other on-disk configuration, we cache all of these for internal use + fsinfo->name_max = lfs->name_max; + fsinfo->file_max = lfs->file_max; + fsinfo->attr_max = lfs->attr_max; + + return 0; +} + int lfs_fs_rawtraverse(lfs_t *lfs, int (*cb)(void *data, lfs_block_t block), void *data, bool includeorphans) { @@ -4259,13 +4555,22 @@ int lfs_fs_rawtraverse(lfs_t *lfs, } #endif - lfs_block_t cycle = 0; + lfs_block_t tortoise[2] = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}; + lfs_size_t tortoise_i = 1; + lfs_size_t tortoise_period = 1; while (!lfs_pair_isnull(dir.tail)) { - if (cycle >= lfs->cfg->block_count/2) { - // loop detected + // detect cycles with Brent's algorithm + if (lfs_pair_issync(dir.tail, tortoise)) { + LFS_WARN("Cycle detected in tail list"); return LFS_ERR_CORRUPT; } - cycle += 1; + if (tortoise_i == tortoise_period) { + tortoise[0] = dir.tail[0]; + tortoise[1] = dir.tail[1]; + tortoise_i = 0; + tortoise_period *= 2; + } + tortoise_i += 1; for (int i = 0; i < 2; i++) { int err = cb(data, dir.tail[i]); @@ -4344,13 +4649,22 @@ static int lfs_fs_pred(lfs_t *lfs, // iterate over all directory directory entries pdir->tail[0] = 0; pdir->tail[1] = 1; - lfs_block_t cycle = 0; + lfs_block_t tortoise[2] = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}; + lfs_size_t tortoise_i = 1; + lfs_size_t tortoise_period = 1; while (!lfs_pair_isnull(pdir->tail)) { - if (cycle >= lfs->cfg->block_count/2) { - // loop detected + // detect cycles with Brent's algorithm + if (lfs_pair_issync(pdir->tail, tortoise)) { + LFS_WARN("Cycle detected in tail list"); return LFS_ERR_CORRUPT; } - cycle += 1; + if (tortoise_i == tortoise_period) { + tortoise[0] = pdir->tail[0]; + tortoise[1] = pdir->tail[1]; + tortoise_i = 0; + tortoise_period *= 2; + } + tortoise_i += 1; if (lfs_pair_cmp(pdir->tail, pair) == 0) { return 0; @@ -4400,13 +4714,22 @@ static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], // use fetchmatch with callback to find pairs parent->tail[0] = 0; parent->tail[1] = 1; - lfs_block_t cycle = 0; + lfs_block_t tortoise[2] = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}; + lfs_size_t tortoise_i = 1; + lfs_size_t tortoise_period = 1; while (!lfs_pair_isnull(parent->tail)) { - if (cycle >= lfs->cfg->block_count/2) { - // loop detected + // detect cycles with Brent's algorithm + if (lfs_pair_issync(parent->tail, tortoise)) { + LFS_WARN("Cycle detected in tail list"); return LFS_ERR_CORRUPT; } - cycle += 1; + if (tortoise_i == tortoise_period) { + tortoise[0] = parent->tail[0]; + tortoise[1] = parent->tail[1]; + tortoise_i = 0; + tortoise_period *= 2; + } + tortoise_i += 1; lfs_stag_t tag = lfs_dir_fetchmatch(lfs, parent, parent->tail, LFS_MKTAG(0x7ff, 0, 0x3ff), @@ -4423,9 +4746,15 @@ static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], } #endif +static void lfs_fs_prepsuperblock(lfs_t *lfs, bool needssuperblock) { + lfs->gstate.tag = (lfs->gstate.tag & ~LFS_MKTAG(0, 0, 0x200)) + | (uint32_t)needssuperblock << 9; +} + #ifndef LFS_READONLY static int lfs_fs_preporphans(lfs_t *lfs, int8_t orphans) { - LFS_ASSERT(lfs_tag_size(lfs->gstate.tag) > 0 || orphans >= 0); + LFS_ASSERT(lfs_tag_size(lfs->gstate.tag) > 0x000 || orphans >= 0); + LFS_ASSERT(lfs_tag_size(lfs->gstate.tag) < 0x1ff || orphans <= 0); lfs->gstate.tag += orphans; lfs->gstate.tag = ((lfs->gstate.tag & ~LFS_MKTAG(0x800, 0, 0)) | ((uint32_t)lfs_gstate_hasorphans(&lfs->gstate) << 31)); @@ -4444,6 +4773,45 @@ static void lfs_fs_prepmove(lfs_t *lfs, } #endif +#ifndef LFS_READONLY +static int lfs_fs_desuperblock(lfs_t *lfs) { + if (!lfs_gstate_needssuperblock(&lfs->gstate)) { + return 0; + } + + LFS_DEBUG("Rewriting superblock {0x%"PRIx32", 0x%"PRIx32"}", + lfs->root[0], + lfs->root[1]); + + lfs_mdir_t root; + int err = lfs_dir_fetch(lfs, &root, lfs->root); + if (err) { + return err; + } + + // write a new superblock + lfs_superblock_t superblock = { + .version = lfs_fs_disk_version(lfs), + .block_size = lfs->cfg->block_size, + .block_count = lfs->block_count, + .name_max = lfs->name_max, + .file_max = lfs->file_max, + .attr_max = lfs->attr_max, + }; + + lfs_superblock_tole32(&superblock); + err = lfs_dir_commit(lfs, &root, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock})); + if (err) { + return err; + } + + lfs_fs_prepsuperblock(lfs, false); + return 0; +} +#endif + #ifndef LFS_READONLY static int lfs_fs_demove(lfs_t *lfs) { if (!lfs_gstate_hasmove(&lfs->gdisk)) { @@ -4456,6 +4824,10 @@ static int lfs_fs_demove(lfs_t *lfs) { lfs->gdisk.pair[1], lfs_tag_id(lfs->gdisk.tag)); + // no other gstate is supported at this time, so if we found something else + // something most likely went wrong in gstate calculation + LFS_ASSERT(lfs_tag_type3(lfs->gdisk.tag) == LFS_TYPE_DELETE); + // fetch and delete the moved entry lfs_mdir_t movedir; int err = lfs_dir_fetch(lfs, &movedir, lfs->gdisk.pair); @@ -4482,12 +4854,20 @@ static int lfs_fs_deorphan(lfs_t *lfs, bool powerloss) { return 0; } - int8_t found = 0; -restart: - { + // Check for orphans in two separate passes: + // - 1 for half-orphans (relocations) + // - 2 for full-orphans (removes/renames) + // + // Two separate passes are needed as half-orphans can contain outdated + // references to full-orphans, effectively hiding them from the deorphan + // search. + // + int pass = 0; + while (pass < 2) { // Fix any orphans lfs_mdir_t pdir = {.split = true, .tail = {0, 1}}; lfs_mdir_t dir; + bool moreorphans = false; // iterate over all directory directory entries while (!lfs_pair_isnull(pdir.tail)) { @@ -4505,42 +4885,7 @@ static int lfs_fs_deorphan(lfs_t *lfs, bool powerloss) { return tag; } - // note we only check for full orphans if we may have had a - // power-loss, otherwise orphans are created intentionally - // during operations such as lfs_mkdir - if (tag == LFS_ERR_NOENT && powerloss) { - // we are an orphan - LFS_DEBUG("Fixing orphan {0x%"PRIx32", 0x%"PRIx32"}", - pdir.tail[0], pdir.tail[1]); - - // steal state - err = lfs_dir_getgstate(lfs, &dir, &lfs->gdelta); - if (err) { - return err; - } - - // steal tail - lfs_pair_tole32(dir.tail); - int state = lfs_dir_orphaningcommit(lfs, &pdir, LFS_MKATTRS( - {LFS_MKTAG(LFS_TYPE_TAIL + dir.split, 0x3ff, 8), - dir.tail})); - lfs_pair_fromle32(dir.tail); - if (state < 0) { - return state; - } - - found += 1; - - // did our commit create more orphans? - if (state == LFS_OK_ORPHANED) { - goto restart; - } - - // refetch tail - continue; - } - - if (tag != LFS_ERR_NOENT) { + if (pass == 0 && tag != LFS_ERR_NOENT) { lfs_block_t pair[2]; lfs_stag_t state = lfs_dir_get(lfs, &parent, LFS_MKTAG(0x7ff, 0x3ff, 0), tag, pair); @@ -4549,7 +4894,7 @@ static int lfs_fs_deorphan(lfs_t *lfs, bool powerloss) { } lfs_pair_fromle32(pair); - if (!lfs_pair_sync(pair, pdir.tail)) { + if (!lfs_pair_issync(pair, pdir.tail)) { // we have desynced LFS_DEBUG("Fixing half-orphan " "{0x%"PRIx32", 0x%"PRIx32"} " @@ -4579,33 +4924,69 @@ static int lfs_fs_deorphan(lfs_t *lfs, bool powerloss) { return state; } - found += 1; - // did our commit create more orphans? if (state == LFS_OK_ORPHANED) { - goto restart; + moreorphans = true; } // refetch tail continue; } } + + // note we only check for full orphans if we may have had a + // power-loss, otherwise orphans are created intentionally + // during operations such as lfs_mkdir + if (pass == 1 && tag == LFS_ERR_NOENT && powerloss) { + // we are an orphan + LFS_DEBUG("Fixing orphan {0x%"PRIx32", 0x%"PRIx32"}", + pdir.tail[0], pdir.tail[1]); + + // steal state + err = lfs_dir_getgstate(lfs, &dir, &lfs->gdelta); + if (err) { + return err; + } + + // steal tail + lfs_pair_tole32(dir.tail); + int state = lfs_dir_orphaningcommit(lfs, &pdir, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_TAIL + dir.split, 0x3ff, 8), + dir.tail})); + lfs_pair_fromle32(dir.tail); + if (state < 0) { + return state; + } + + // did our commit create more orphans? + if (state == LFS_OK_ORPHANED) { + moreorphans = true; + } + + // refetch tail + continue; + } } pdir = dir; } + + pass = moreorphans ? 0 : pass+1; } // mark orphans as fixed - return lfs_fs_preporphans(lfs, -lfs_min( - lfs_gstate_getorphans(&lfs->gstate), - found)); + return lfs_fs_preporphans(lfs, -lfs_gstate_getorphans(&lfs->gstate)); } #endif #ifndef LFS_READONLY static int lfs_fs_forceconsistency(lfs_t *lfs) { - int err = lfs_fs_demove(lfs); + int err = lfs_fs_desuperblock(lfs); + if (err) { + return err; + } + + err = lfs_fs_demove(lfs); if (err) { return err; } @@ -4619,6 +5000,36 @@ static int lfs_fs_forceconsistency(lfs_t *lfs) { } #endif +#ifndef LFS_READONLY +static int lfs_fs_rawmkconsistent(lfs_t *lfs) { + // lfs_fs_forceconsistency does most of the work here + int err = lfs_fs_forceconsistency(lfs); + if (err) { + return err; + } + + // do we have any pending gstate? + lfs_gstate_t delta = {0}; + lfs_gstate_xor(&delta, &lfs->gdisk); + lfs_gstate_xor(&delta, &lfs->gstate); + if (!lfs_gstate_iszero(&delta)) { + // lfs_dir_commit will implicitly write out any pending gstate + lfs_mdir_t root; + err = lfs_dir_fetch(lfs, &root, lfs->root); + if (err) { + return err; + } + + err = lfs_dir_commit(lfs, &root, NULL, 0); + if (err) { + return err; + } + } + + return 0; +} +#endif + static int lfs_fs_size_count(void *p, lfs_block_t block) { (void)block; lfs_size_t *size = p; @@ -4636,6 +5047,45 @@ static lfs_ssize_t lfs_fs_rawsize(lfs_t *lfs) { return size; } +#ifndef LFS_READONLY +static int lfs_fs_rawgrow(lfs_t *lfs, lfs_size_t block_count) { + // shrinking is not supported + LFS_ASSERT(block_count >= lfs->block_count); + + if (block_count > lfs->block_count) { + lfs->block_count = block_count; + + // fetch the root + lfs_mdir_t root; + int err = lfs_dir_fetch(lfs, &root, lfs->root); + if (err) { + return err; + } + + // update the superblock + lfs_superblock_t superblock; + lfs_stag_t tag = lfs_dir_get(lfs, &root, LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock); + if (tag < 0) { + return tag; + } + lfs_superblock_fromle32(&superblock); + + superblock.block_count = lfs->block_count; + + lfs_superblock_tole32(&superblock); + err = lfs_dir_commit(lfs, &root, LFS_MKATTRS( + {tag, &superblock})); + if (err) { + return err; + } + } + + return 0; +} +#endif + #ifdef LFS_MIGRATE ////// Migration from littelfs v1 below this ////// @@ -5059,6 +5509,10 @@ static int lfs1_unmount(lfs_t *lfs) { /// v1 migration /// static int lfs_rawmigrate(lfs_t *lfs, const struct lfs_config *cfg) { struct lfs1 lfs1; + + // Indeterminate filesystem size not allowed for migration. + LFS_ASSERT(cfg->block_count != 0); + int err = lfs1_mount(lfs, &lfs1, cfg); if (err) { return err; @@ -5755,6 +6209,20 @@ int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir) { return err; } +int lfs_fs_stat(lfs_t *lfs, struct lfs_fsinfo *fsinfo) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_fs_stat(%p, %p)", (void*)lfs, (void*)fsinfo); + + err = lfs_fs_rawstat(lfs, fsinfo); + + LFS_TRACE("lfs_fs_stat -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} + lfs_ssize_t lfs_fs_size(lfs_t *lfs) { int err = LFS_LOCK(lfs->cfg); if (err) { @@ -5784,6 +6252,54 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void *, lfs_block_t), void *data) { return err; } +#ifndef LFS_READONLY +int lfs_fs_gc(lfs_t *lfs) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_fs_gc(%p)", (void*)lfs); + + err = lfs_fs_rawgc(lfs); + + LFS_TRACE("lfs_fs_gc -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + +#ifndef LFS_READONLY +int lfs_fs_mkconsistent(lfs_t *lfs) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_fs_mkconsistent(%p)", (void*)lfs); + + err = lfs_fs_rawmkconsistent(lfs); + + LFS_TRACE("lfs_fs_mkconsistent -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + +#ifndef LFS_READONLY +int lfs_fs_grow(lfs_t *lfs, lfs_size_t block_count) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_fs_grow(%p, %"PRIu32")", (void*)lfs, block_count); + + err = lfs_fs_rawgrow(lfs, block_count); + + LFS_TRACE("lfs_fs_grow -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + #ifdef LFS_MIGRATE int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) { int err = LFS_LOCK(cfg); diff --git a/components/fs/littlefs/littlefs/lfs.h b/components/fs/littlefs/littlefs/lfs.h index 2bce17f5c..9eeab230e 100644 --- a/components/fs/littlefs/littlefs/lfs.h +++ b/components/fs/littlefs/littlefs/lfs.h @@ -8,8 +8,6 @@ #ifndef LFS_H #define LFS_H -#include -#include #include "lfs_util.h" #ifdef __cplusplus @@ -23,14 +21,14 @@ extern "C" // Software library version // Major (top-nibble), incremented on backwards incompatible changes // Minor (bottom-nibble), incremented on feature additions -#define LFS_VERSION 0x00020005 +#define LFS_VERSION 0x00020008 #define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16)) #define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >> 0)) // Version of On-disk data structures // Major (top-nibble), incremented on backwards incompatible changes // Minor (bottom-nibble), incremented on feature additions -#define LFS_DISK_VERSION 0x00020000 +#define LFS_DISK_VERSION 0x00020001 #define LFS_DISK_VERSION_MAJOR (0xffff & (LFS_DISK_VERSION >> 16)) #define LFS_DISK_VERSION_MINOR (0xffff & (LFS_DISK_VERSION >> 0)) @@ -114,6 +112,8 @@ enum lfs_type { LFS_TYPE_SOFTTAIL = 0x600, LFS_TYPE_HARDTAIL = 0x601, LFS_TYPE_MOVESTATE = 0x7ff, + LFS_TYPE_CCRC = 0x500, + LFS_TYPE_FCRC = 0x5ff, // internal chip sources LFS_FROM_NOOP = 0x000, @@ -263,6 +263,14 @@ struct lfs_config { // can help bound the metadata compaction time. Must be <= block_size. // Defaults to block_size when zero. lfs_size_t metadata_max; + +#ifdef LFS_MULTIVERSION + // On-disk version to use when writing in the form of 16-bit major version + // + 16-bit minor version. This limiting metadata to what is supported by + // older minor versions. Note that some features will be lost. Defaults to + // to the most recent minor version when zero. + uint32_t disk_version; +#endif }; // File info structure @@ -280,6 +288,27 @@ struct lfs_info { char name[LFS_NAME_MAX+1]; }; +// Filesystem info structure +struct lfs_fsinfo { + // On-disk version. + uint32_t disk_version; + + // Size of a logical block in bytes. + lfs_size_t block_size; + + // Number of logical blocks in filesystem. + lfs_size_t block_count; + + // Upper limit on the length of file names in bytes. + lfs_size_t name_max; + + // Upper limit on the size of files in bytes. + lfs_size_t file_max; + + // Upper limit on the size of custom attributes in bytes. + lfs_size_t attr_max; +}; + // Custom attribute structure, used to describe custom attributes // committed atomically during file writes. struct lfs_attr { @@ -410,6 +439,7 @@ typedef struct lfs { } free; const struct lfs_config *cfg; + lfs_size_t block_count; lfs_size_t name_max; lfs_size_t file_max; lfs_size_t attr_max; @@ -659,6 +689,12 @@ int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir); /// Filesystem-level filesystem operations +// Find on-disk info about the filesystem +// +// Fills out the fsinfo structure based on the filesystem found on-disk. +// Returns a negative error code on failure. +int lfs_fs_stat(lfs_t *lfs, struct lfs_fsinfo *fsinfo); + // Finds the current size of the filesystem // // Note: Result is best effort. If files share COW structures, the returned @@ -676,6 +712,40 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs); // Returns a negative error code on failure. int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); +// Attempt to proactively find free blocks +// +// Calling this function is not required, but may allowing the offloading of +// the expensive block allocation scan to a less time-critical code path. +// +// Note: littlefs currently does not persist any found free blocks to disk. +// This may change in the future. +// +// Returns a negative error code on failure. Finding no free blocks is +// not an error. +int lfs_fs_gc(lfs_t *lfs); + +#ifndef LFS_READONLY +// Attempt to make the filesystem consistent and ready for writing +// +// Calling this function is not required, consistency will be implicitly +// enforced on the first operation that writes to the filesystem, but this +// function allows the work to be performed earlier and without other +// filesystem changes. +// +// Returns a negative error code on failure. +int lfs_fs_mkconsistent(lfs_t *lfs); +#endif + +#ifndef LFS_READONLY +// Grows the filesystem to a new size, updating the superblock with the new +// block count. +// +// Note: This is irreversible. +// +// Returns a negative error code on failure. +int lfs_fs_grow(lfs_t *lfs, lfs_size_t block_count); +#endif + #ifndef LFS_READONLY #ifdef LFS_MIGRATE // Attempts to migrate a previous version of littlefs diff --git a/components/fs/littlefs/littlefs/runners/bench_runner.c b/components/fs/littlefs/littlefs/runners/bench_runner.c new file mode 100644 index 000000000..99e2a5b99 --- /dev/null +++ b/components/fs/littlefs/littlefs/runners/bench_runner.c @@ -0,0 +1,2055 @@ +/* + * Runner for littlefs benchmarks + * + * Copyright (c) 2022, The littlefs authors. + * SPDX-License-Identifier: BSD-3-Clause + */ +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 199309L +#endif + +#include "runners/bench_runner.h" +#include "bd/lfs_emubd.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +// some helpers + +// append to an array with amortized doubling +void *mappend(void **p, + size_t size, + size_t *count, + size_t *capacity) { + uint8_t *p_ = *p; + size_t count_ = *count; + size_t capacity_ = *capacity; + + count_ += 1; + if (count_ > capacity_) { + capacity_ = (2*capacity_ < 4) ? 4 : 2*capacity_; + + p_ = realloc(p_, capacity_*size); + if (!p_) { + return NULL; + } + } + + *p = p_; + *count = count_; + *capacity = capacity_; + return &p_[(count_-1)*size]; +} + +// a quick self-terminating text-safe varint scheme +static void leb16_print(uintmax_t x) { + // allow 'w' to indicate negative numbers + if ((intmax_t)x < 0) { + printf("w"); + x = -x; + } + + while (true) { + char nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0); + printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10); + if (x <= 0xf) { + break; + } + x >>= 4; + } +} + +static uintmax_t leb16_parse(const char *s, char **tail) { + bool neg = false; + uintmax_t x = 0; + if (tail) { + *tail = (char*)s; + } + + if (s[0] == 'w') { + neg = true; + s = s+1; + } + + size_t i = 0; + while (true) { + uintmax_t nibble = s[i]; + if (nibble >= '0' && nibble <= '9') { + nibble = nibble - '0'; + } else if (nibble >= 'a' && nibble <= 'v') { + nibble = nibble - 'a' + 10; + } else { + // invalid? + return 0; + } + + x |= (nibble & 0xf) << (4*i); + i += 1; + if (!(nibble & 0x10)) { + s = s + i; + break; + } + } + + if (tail) { + *tail = (char*)s; + } + return neg ? -x : x; +} + + + +// bench_runner types + +typedef struct bench_geometry { + const char *name; + bench_define_t defines[BENCH_GEOMETRY_DEFINE_COUNT]; +} bench_geometry_t; + +typedef struct bench_id { + const char *name; + const bench_define_t *defines; + size_t define_count; +} bench_id_t; + + +// bench suites are linked into a custom ld section +extern struct bench_suite __start__bench_suites; +extern struct bench_suite __stop__bench_suites; + +const struct bench_suite *bench_suites = &__start__bench_suites; +#define BENCH_SUITE_COUNT \ + ((size_t)(&__stop__bench_suites - &__start__bench_suites)) + + +// bench define management +typedef struct bench_define_map { + const bench_define_t *defines; + size_t count; +} bench_define_map_t; + +typedef struct bench_define_names { + const char *const *names; + size_t count; +} bench_define_names_t; + +intmax_t bench_define_lit(void *data) { + return (intptr_t)data; +} + +#define BENCH_CONST(x) {bench_define_lit, (void*)(uintptr_t)(x)} +#define BENCH_LIT(x) ((bench_define_t)BENCH_CONST(x)) + + +#define BENCH_DEF(k, v) \ + intmax_t bench_define_##k(void *data) { \ + (void)data; \ + return v; \ + } + + BENCH_IMPLICIT_DEFINES +#undef BENCH_DEF + +#define BENCH_DEFINE_MAP_OVERRIDE 0 +#define BENCH_DEFINE_MAP_EXPLICIT 1 +#define BENCH_DEFINE_MAP_PERMUTATION 2 +#define BENCH_DEFINE_MAP_GEOMETRY 3 +#define BENCH_DEFINE_MAP_IMPLICIT 4 +#define BENCH_DEFINE_MAP_COUNT 5 + +bench_define_map_t bench_define_maps[BENCH_DEFINE_MAP_COUNT] = { + [BENCH_DEFINE_MAP_IMPLICIT] = { + (const bench_define_t[BENCH_IMPLICIT_DEFINE_COUNT]) { + #define BENCH_DEF(k, v) \ + [k##_i] = {bench_define_##k, NULL}, + + BENCH_IMPLICIT_DEFINES + #undef BENCH_DEF + }, + BENCH_IMPLICIT_DEFINE_COUNT, + }, +}; + +#define BENCH_DEFINE_NAMES_SUITE 0 +#define BENCH_DEFINE_NAMES_IMPLICIT 1 +#define BENCH_DEFINE_NAMES_COUNT 2 + +bench_define_names_t bench_define_names[BENCH_DEFINE_NAMES_COUNT] = { + [BENCH_DEFINE_NAMES_IMPLICIT] = { + (const char *const[BENCH_IMPLICIT_DEFINE_COUNT]){ + #define BENCH_DEF(k, v) \ + [k##_i] = #k, + + BENCH_IMPLICIT_DEFINES + #undef BENCH_DEF + }, + BENCH_IMPLICIT_DEFINE_COUNT, + }, +}; + +intmax_t *bench_define_cache; +size_t bench_define_cache_count; +unsigned *bench_define_cache_mask; + +const char *bench_define_name(size_t define) { + // lookup in our bench names + for (size_t i = 0; i < BENCH_DEFINE_NAMES_COUNT; i++) { + if (define < bench_define_names[i].count + && bench_define_names[i].names + && bench_define_names[i].names[define]) { + return bench_define_names[i].names[define]; + } + } + + return NULL; +} + +bool bench_define_ispermutation(size_t define) { + // is this define specific to the permutation? + for (size_t i = 0; i < BENCH_DEFINE_MAP_IMPLICIT; i++) { + if (define < bench_define_maps[i].count + && bench_define_maps[i].defines[define].cb) { + return true; + } + } + + return false; +} + +intmax_t bench_define(size_t define) { + // is the define in our cache? + if (define < bench_define_cache_count + && (bench_define_cache_mask[define/(8*sizeof(unsigned))] + & (1 << (define%(8*sizeof(unsigned)))))) { + return bench_define_cache[define]; + } + + // lookup in our bench defines + for (size_t i = 0; i < BENCH_DEFINE_MAP_COUNT; i++) { + if (define < bench_define_maps[i].count + && bench_define_maps[i].defines[define].cb) { + intmax_t v = bench_define_maps[i].defines[define].cb( + bench_define_maps[i].defines[define].data); + + // insert into cache! + bench_define_cache[define] = v; + bench_define_cache_mask[define / (8*sizeof(unsigned))] + |= 1 << (define%(8*sizeof(unsigned))); + + return v; + } + } + + return 0; + + // not found? + const char *name = bench_define_name(define); + fprintf(stderr, "error: undefined define %s (%zd)\n", + name ? name : "(unknown)", + define); + assert(false); + exit(-1); +} + +void bench_define_flush(void) { + // clear cache between permutations + memset(bench_define_cache_mask, 0, + sizeof(unsigned)*( + (bench_define_cache_count+(8*sizeof(unsigned))-1) + / (8*sizeof(unsigned)))); +} + +// geometry updates +const bench_geometry_t *bench_geometry = NULL; + +void bench_define_geometry(const bench_geometry_t *geometry) { + bench_define_maps[BENCH_DEFINE_MAP_GEOMETRY] = (bench_define_map_t){ + geometry->defines, BENCH_GEOMETRY_DEFINE_COUNT}; +} + +// override updates +typedef struct bench_override { + const char *name; + const intmax_t *defines; + size_t permutations; +} bench_override_t; + +const bench_override_t *bench_overrides = NULL; +size_t bench_override_count = 0; + +bench_define_t *bench_override_defines = NULL; +size_t bench_override_define_count = 0; +size_t bench_override_define_permutations = 1; +size_t bench_override_define_capacity = 0; + +// suite/perm updates +void bench_define_suite(const struct bench_suite *suite) { + bench_define_names[BENCH_DEFINE_NAMES_SUITE] = (bench_define_names_t){ + suite->define_names, suite->define_count}; + + // make sure our cache is large enough + if (lfs_max(suite->define_count, BENCH_IMPLICIT_DEFINE_COUNT) + > bench_define_cache_count) { + // align to power of two to avoid any superlinear growth + size_t ncount = 1 << lfs_npw2( + lfs_max(suite->define_count, BENCH_IMPLICIT_DEFINE_COUNT)); + bench_define_cache = realloc(bench_define_cache, ncount*sizeof(intmax_t)); + bench_define_cache_mask = realloc(bench_define_cache_mask, + sizeof(unsigned)*( + (ncount+(8*sizeof(unsigned))-1) + / (8*sizeof(unsigned)))); + bench_define_cache_count = ncount; + } + + // map any overrides + if (bench_override_count > 0) { + // first figure out the total size of override permutations + size_t count = 0; + size_t permutations = 1; + for (size_t i = 0; i < bench_override_count; i++) { + for (size_t d = 0; + d < lfs_max( + suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + // define name match? + const char *name = bench_define_name(d); + if (name && strcmp(name, bench_overrides[i].name) == 0) { + count = lfs_max(count, d+1); + permutations *= bench_overrides[i].permutations; + break; + } + } + } + bench_override_define_count = count; + bench_override_define_permutations = permutations; + + // make sure our override arrays are big enough + if (count * permutations > bench_override_define_capacity) { + // align to power of two to avoid any superlinear growth + size_t ncapacity = 1 << lfs_npw2(count * permutations); + bench_override_defines = realloc( + bench_override_defines, + sizeof(bench_define_t)*ncapacity); + bench_override_define_capacity = ncapacity; + } + + // zero unoverridden defines + memset(bench_override_defines, 0, + sizeof(bench_define_t) * count * permutations); + + // compute permutations + size_t p = 1; + for (size_t i = 0; i < bench_override_count; i++) { + for (size_t d = 0; + d < lfs_max( + suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + // define name match? + const char *name = bench_define_name(d); + if (name && strcmp(name, bench_overrides[i].name) == 0) { + // scatter the define permutations based on already + // seen permutations + for (size_t j = 0; j < permutations; j++) { + bench_override_defines[j*count + d] = BENCH_LIT( + bench_overrides[i].defines[(j/p) + % bench_overrides[i].permutations]); + } + + // keep track of how many permutations we've seen so far + p *= bench_overrides[i].permutations; + break; + } + } + } + } +} + +void bench_define_perm( + const struct bench_suite *suite, + const struct bench_case *case_, + size_t perm) { + if (case_->defines) { + bench_define_maps[BENCH_DEFINE_MAP_PERMUTATION] = (bench_define_map_t){ + case_->defines + perm*suite->define_count, + suite->define_count}; + } else { + bench_define_maps[BENCH_DEFINE_MAP_PERMUTATION] = (bench_define_map_t){ + NULL, 0}; + } +} + +void bench_define_override(size_t perm) { + bench_define_maps[BENCH_DEFINE_MAP_OVERRIDE] = (bench_define_map_t){ + bench_override_defines + perm*bench_override_define_count, + bench_override_define_count}; +} + +void bench_define_explicit( + const bench_define_t *defines, + size_t define_count) { + bench_define_maps[BENCH_DEFINE_MAP_EXPLICIT] = (bench_define_map_t){ + defines, define_count}; +} + +void bench_define_cleanup(void) { + // bench define management can allocate a few things + free(bench_define_cache); + free(bench_define_cache_mask); + free(bench_override_defines); +} + + + +// bench state +extern const bench_geometry_t *bench_geometries; +extern size_t bench_geometry_count; + +const bench_id_t *bench_ids = (const bench_id_t[]) { + {NULL, NULL, 0}, +}; +size_t bench_id_count = 1; + +size_t bench_step_start = 0; +size_t bench_step_stop = -1; +size_t bench_step_step = 1; + +const char *bench_disk_path = NULL; +const char *bench_trace_path = NULL; +bool bench_trace_backtrace = false; +uint32_t bench_trace_period = 0; +uint32_t bench_trace_freq = 0; +FILE *bench_trace_file = NULL; +uint32_t bench_trace_cycles = 0; +uint64_t bench_trace_time = 0; +uint64_t bench_trace_open_time = 0; +lfs_emubd_sleep_t bench_read_sleep = 0.0; +lfs_emubd_sleep_t bench_prog_sleep = 0.0; +lfs_emubd_sleep_t bench_erase_sleep = 0.0; + +// this determines both the backtrace buffer and the trace printf buffer, if +// trace ends up interleaved or truncated this may need to be increased +#ifndef BENCH_TRACE_BACKTRACE_BUFFER_SIZE +#define BENCH_TRACE_BACKTRACE_BUFFER_SIZE 8192 +#endif +void *bench_trace_backtrace_buffer[ + BENCH_TRACE_BACKTRACE_BUFFER_SIZE / sizeof(void*)]; + +// trace printing +void bench_trace(const char *fmt, ...) { + if (bench_trace_path) { + // sample at a specific period? + if (bench_trace_period) { + if (bench_trace_cycles % bench_trace_period != 0) { + bench_trace_cycles += 1; + return; + } + bench_trace_cycles += 1; + } + + // sample at a specific frequency? + if (bench_trace_freq) { + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + uint64_t now = (uint64_t)t.tv_sec*1000*1000*1000 + + (uint64_t)t.tv_nsec; + if (now - bench_trace_time < (1000*1000*1000) / bench_trace_freq) { + return; + } + bench_trace_time = now; + } + + if (!bench_trace_file) { + // Tracing output is heavy and trying to open every trace + // call is slow, so we only try to open the trace file every + // so often. Note this doesn't affect successfully opened files + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + uint64_t now = (uint64_t)t.tv_sec*1000*1000*1000 + + (uint64_t)t.tv_nsec; + if (now - bench_trace_open_time < 100*1000*1000) { + return; + } + bench_trace_open_time = now; + + // try to open the trace file + int fd; + if (strcmp(bench_trace_path, "-") == 0) { + fd = dup(1); + if (fd < 0) { + return; + } + } else { + fd = open( + bench_trace_path, + O_WRONLY | O_CREAT | O_APPEND | O_NONBLOCK, + 0666); + if (fd < 0) { + return; + } + int err = fcntl(fd, F_SETFL, O_WRONLY | O_CREAT | O_APPEND); + assert(!err); + } + + FILE *f = fdopen(fd, "a"); + assert(f); + int err = setvbuf(f, NULL, _IOFBF, + BENCH_TRACE_BACKTRACE_BUFFER_SIZE); + assert(!err); + bench_trace_file = f; + } + + // print trace + va_list va; + va_start(va, fmt); + int res = vfprintf(bench_trace_file, fmt, va); + va_end(va); + if (res < 0) { + fclose(bench_trace_file); + bench_trace_file = NULL; + return; + } + + if (bench_trace_backtrace) { + // print backtrace + size_t count = backtrace( + bench_trace_backtrace_buffer, + BENCH_TRACE_BACKTRACE_BUFFER_SIZE); + // note we skip our own stack frame + for (size_t i = 1; i < count; i++) { + res = fprintf(bench_trace_file, "\tat %p\n", + bench_trace_backtrace_buffer[i]); + if (res < 0) { + fclose(bench_trace_file); + bench_trace_file = NULL; + return; + } + } + } + + // flush immediately + fflush(bench_trace_file); + } +} + + +// bench prng +uint32_t bench_prng(uint32_t *state) { + // A simple xorshift32 generator, easily reproducible. Keep in mind + // determinism is much more important than actual randomness here. + uint32_t x = *state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + *state = x; + return x; +} + + +// bench recording state +static struct lfs_config *bench_cfg = NULL; +static lfs_emubd_io_t bench_last_readed = 0; +static lfs_emubd_io_t bench_last_proged = 0; +static lfs_emubd_io_t bench_last_erased = 0; +lfs_emubd_io_t bench_readed = 0; +lfs_emubd_io_t bench_proged = 0; +lfs_emubd_io_t bench_erased = 0; + +void bench_reset(void) { + bench_readed = 0; + bench_proged = 0; + bench_erased = 0; + bench_last_readed = 0; + bench_last_proged = 0; + bench_last_erased = 0; +} + +void bench_start(void) { + assert(bench_cfg); + lfs_emubd_sio_t readed = lfs_emubd_readed(bench_cfg); + assert(readed >= 0); + lfs_emubd_sio_t proged = lfs_emubd_proged(bench_cfg); + assert(proged >= 0); + lfs_emubd_sio_t erased = lfs_emubd_erased(bench_cfg); + assert(erased >= 0); + + bench_last_readed = readed; + bench_last_proged = proged; + bench_last_erased = erased; +} + +void bench_stop(void) { + assert(bench_cfg); + lfs_emubd_sio_t readed = lfs_emubd_readed(bench_cfg); + assert(readed >= 0); + lfs_emubd_sio_t proged = lfs_emubd_proged(bench_cfg); + assert(proged >= 0); + lfs_emubd_sio_t erased = lfs_emubd_erased(bench_cfg); + assert(erased >= 0); + + bench_readed += readed - bench_last_readed; + bench_proged += proged - bench_last_proged; + bench_erased += erased - bench_last_erased; +} + + +// encode our permutation into a reusable id +static void perm_printid( + const struct bench_suite *suite, + const struct bench_case *case_) { + (void)suite; + // case[:permutation] + printf("%s:", case_->name); + for (size_t d = 0; + d < lfs_max( + suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + if (bench_define_ispermutation(d)) { + leb16_print(d); + leb16_print(BENCH_DEFINE(d)); + } + } +} + +// a quick trie for keeping track of permutations we've seen +typedef struct bench_seen { + struct bench_seen_branch *branches; + size_t branch_count; + size_t branch_capacity; +} bench_seen_t; + +struct bench_seen_branch { + intmax_t define; + struct bench_seen branch; +}; + +bool bench_seen_insert( + bench_seen_t *seen, + const struct bench_suite *suite, + const struct bench_case *case_) { + (void)case_; + bool was_seen = true; + + // use the currently set defines + for (size_t d = 0; + d < lfs_max( + suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + // treat unpermuted defines the same as 0 + intmax_t define = bench_define_ispermutation(d) ? BENCH_DEFINE(d) : 0; + + // already seen? + struct bench_seen_branch *branch = NULL; + for (size_t i = 0; i < seen->branch_count; i++) { + if (seen->branches[i].define == define) { + branch = &seen->branches[i]; + break; + } + } + + // need to create a new node + if (!branch) { + was_seen = false; + branch = mappend( + (void**)&seen->branches, + sizeof(struct bench_seen_branch), + &seen->branch_count, + &seen->branch_capacity); + branch->define = define; + branch->branch = (bench_seen_t){NULL, 0, 0}; + } + + seen = &branch->branch; + } + + return was_seen; +} + +void bench_seen_cleanup(bench_seen_t *seen) { + for (size_t i = 0; i < seen->branch_count; i++) { + bench_seen_cleanup(&seen->branches[i].branch); + } + free(seen->branches); +} + +// iterate through permutations in a bench case +static void case_forperm( + const struct bench_suite *suite, + const struct bench_case *case_, + const bench_define_t *defines, + size_t define_count, + void (*cb)( + void *data, + const struct bench_suite *suite, + const struct bench_case *case_), + void *data) { + // explicit permutation? + if (defines) { + bench_define_explicit(defines, define_count); + + for (size_t v = 0; v < bench_override_define_permutations; v++) { + // define override permutation + bench_define_override(v); + bench_define_flush(); + + cb(data, suite, case_); + } + + return; + } + + bench_seen_t seen = {NULL, 0, 0}; + + for (size_t k = 0; k < case_->permutations; k++) { + // define permutation + bench_define_perm(suite, case_, k); + + for (size_t v = 0; v < bench_override_define_permutations; v++) { + // define override permutation + bench_define_override(v); + + for (size_t g = 0; g < bench_geometry_count; g++) { + // define geometry + bench_define_geometry(&bench_geometries[g]); + bench_define_flush(); + + // have we seen this permutation before? + bool was_seen = bench_seen_insert(&seen, suite, case_); + if (!(k == 0 && v == 0 && g == 0) && was_seen) { + continue; + } + + cb(data, suite, case_); + } + } + } + + bench_seen_cleanup(&seen); +} + + +// how many permutations are there actually in a bench case +struct perm_count_state { + size_t total; + size_t filtered; +}; + +void perm_count( + void *data, + const struct bench_suite *suite, + const struct bench_case *case_) { + struct perm_count_state *state = data; + (void)suite; + (void)case_; + + state->total += 1; + + if (case_->filter && !case_->filter()) { + return; + } + + state->filtered += 1; +} + + +// operations we can do +static void summary(void) { + printf("%-23s %7s %7s %7s %11s\n", + "", "flags", "suites", "cases", "perms"); + size_t suites = 0; + size_t cases = 0; + bench_flags_t flags = 0; + struct perm_count_state perms = {0, 0}; + + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + cases += 1; + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_count, + &perms); + } + + suites += 1; + flags |= bench_suites[i].flags; + } + } + + char perm_buf[64]; + snprintf(perm_buf, sizeof(perm_buf), "%zu/%zu", perms.filtered, perms.total); + char flag_buf[64]; + snprintf(flag_buf, sizeof(flag_buf), "%s%s", + (flags & BENCH_REENTRANT) ? "r" : "", + (!flags) ? "-" : ""); + printf("%-23s %7s %7zu %7zu %11s\n", + "TOTAL", + flag_buf, + suites, + cases, + perm_buf); +} + +static void list_suites(void) { + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + size_t len = strlen(bench_suites[i].name); + if (len > name_width) { + name_width = len; + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + + printf("%-*s %7s %7s %11s\n", + name_width, "suite", "flags", "cases", "perms"); + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + size_t cases = 0; + struct perm_count_state perms = {0, 0}; + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + cases += 1; + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_count, + &perms); + } + + // no benches found? + if (!cases) { + continue; + } + + char perm_buf[64]; + snprintf(perm_buf, sizeof(perm_buf), "%zu/%zu", perms.filtered, perms.total); + char flag_buf[64]; + snprintf(flag_buf, sizeof(flag_buf), "%s%s", + (bench_suites[i].flags & BENCH_REENTRANT) ? "r" : "", + (!bench_suites[i].flags) ? "-" : ""); + printf("%-*s %7s %7zu %11s\n", + name_width, + bench_suites[i].name, + flag_buf, + cases, + perm_buf); + } + } +} + +static void list_cases(void) { + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + size_t len = strlen(bench_suites[i].cases[j].name); + if (len > name_width) { + name_width = len; + } + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + + printf("%-*s %7s %11s\n", name_width, "case", "flags", "perms"); + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + struct perm_count_state perms = {0, 0}; + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_count, + &perms); + + char perm_buf[64]; + snprintf(perm_buf, sizeof(perm_buf), "%zu/%zu", perms.filtered, perms.total); + char flag_buf[64]; + snprintf(flag_buf, sizeof(flag_buf), "%s%s", + (bench_suites[i].cases[j].flags & BENCH_REENTRANT) + ? "r" : "", + (!bench_suites[i].cases[j].flags) + ? "-" : ""); + printf("%-*s %7s %11s\n", + name_width, + bench_suites[i].cases[j].name, + flag_buf, + perm_buf); + } + } + } +} + +static void list_suite_paths(void) { + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + size_t len = strlen(bench_suites[i].name); + if (len > name_width) { + name_width = len; + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + + printf("%-*s %s\n", name_width, "suite", "path"); + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + size_t cases = 0; + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + + cases += 1; + } + } + + // no benches found? + if (!cases) { + continue; + } + + printf("%-*s %s\n", + name_width, + bench_suites[i].name, + bench_suites[i].path); + } + } +} + +static void list_case_paths(void) { + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + size_t len = strlen(bench_suites[i].cases[j].name); + if (len > name_width) { + name_width = len; + } + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + + printf("%-*s %s\n", name_width, "case", "path"); + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + printf("%-*s %s\n", + name_width, + bench_suites[i].cases[j].name, + bench_suites[i].cases[j].path); + } + } + } +} + +struct list_defines_define { + const char *name; + intmax_t *values; + size_t value_count; + size_t value_capacity; +}; + +struct list_defines_defines { + struct list_defines_define *defines; + size_t define_count; + size_t define_capacity; +}; + +static void list_defines_add( + struct list_defines_defines *defines, + size_t d) { + const char *name = bench_define_name(d); + intmax_t value = BENCH_DEFINE(d); + + // define already in defines? + for (size_t i = 0; i < defines->define_count; i++) { + if (strcmp(defines->defines[i].name, name) == 0) { + // value already in values? + for (size_t j = 0; j < defines->defines[i].value_count; j++) { + if (defines->defines[i].values[j] == value) { + return; + } + } + + *(intmax_t*)mappend( + (void**)&defines->defines[i].values, + sizeof(intmax_t), + &defines->defines[i].value_count, + &defines->defines[i].value_capacity) = value; + + return; + } + } + + // new define? + struct list_defines_define *define = mappend( + (void**)&defines->defines, + sizeof(struct list_defines_define), + &defines->define_count, + &defines->define_capacity); + define->name = name; + define->values = malloc(sizeof(intmax_t)); + define->values[0] = value; + define->value_count = 1; + define->value_capacity = 1; +} + +void perm_list_defines( + void *data, + const struct bench_suite *suite, + const struct bench_case *case_) { + struct list_defines_defines *defines = data; + (void)suite; + (void)case_; + + // collect defines + for (size_t d = 0; + d < lfs_max(suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + if (d < BENCH_IMPLICIT_DEFINE_COUNT + || bench_define_ispermutation(d)) { + list_defines_add(defines, d); + } + } +} + +void perm_list_permutation_defines( + void *data, + const struct bench_suite *suite, + const struct bench_case *case_) { + struct list_defines_defines *defines = data; + (void)suite; + (void)case_; + + // collect permutation_defines + for (size_t d = 0; + d < lfs_max(suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + if (bench_define_ispermutation(d)) { + list_defines_add(defines, d); + } + } +} + +extern const bench_geometry_t builtin_geometries[]; + +static void list_defines(void) { + struct list_defines_defines defines = {NULL, 0, 0}; + + // add defines + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_list_defines, + &defines); + } + } + } + + for (size_t i = 0; i < defines.define_count; i++) { + printf("%s=", defines.defines[i].name); + for (size_t j = 0; j < defines.defines[i].value_count; j++) { + printf("%jd", defines.defines[i].values[j]); + if (j != defines.defines[i].value_count-1) { + printf(","); + } + } + printf("\n"); + } + + for (size_t i = 0; i < defines.define_count; i++) { + free(defines.defines[i].values); + } + free(defines.defines); +} + +static void list_permutation_defines(void) { + struct list_defines_defines defines = {NULL, 0, 0}; + + // add permutation defines + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_list_permutation_defines, + &defines); + } + } + } + + for (size_t i = 0; i < defines.define_count; i++) { + printf("%s=", defines.defines[i].name); + for (size_t j = 0; j < defines.defines[i].value_count; j++) { + printf("%jd", defines.defines[i].values[j]); + if (j != defines.defines[i].value_count-1) { + printf(","); + } + } + printf("\n"); + } + + for (size_t i = 0; i < defines.define_count; i++) { + free(defines.defines[i].values); + } + free(defines.defines); +} + +static void list_implicit_defines(void) { + struct list_defines_defines defines = {NULL, 0, 0}; + + // yes we do need to define a suite, this does a bit of bookeeping + // such as setting up the define cache + bench_define_suite(&(const struct bench_suite){0}); + + // make sure to include builtin geometries here + extern const bench_geometry_t builtin_geometries[]; + for (size_t g = 0; builtin_geometries[g].name; g++) { + bench_define_geometry(&builtin_geometries[g]); + bench_define_flush(); + + // add implicit defines + for (size_t d = 0; d < BENCH_IMPLICIT_DEFINE_COUNT; d++) { + list_defines_add(&defines, d); + } + } + + for (size_t i = 0; i < defines.define_count; i++) { + printf("%s=", defines.defines[i].name); + for (size_t j = 0; j < defines.defines[i].value_count; j++) { + printf("%jd", defines.defines[i].values[j]); + if (j != defines.defines[i].value_count-1) { + printf(","); + } + } + printf("\n"); + } + + for (size_t i = 0; i < defines.define_count; i++) { + free(defines.defines[i].values); + } + free(defines.defines); +} + + + +// geometries to bench + +const bench_geometry_t builtin_geometries[] = { + {"default", {{0}, BENCH_CONST(16), BENCH_CONST(512), {0}}}, + {"eeprom", {{0}, BENCH_CONST(1), BENCH_CONST(512), {0}}}, + {"emmc", {{0}, {0}, BENCH_CONST(512), {0}}}, + {"nor", {{0}, BENCH_CONST(1), BENCH_CONST(4096), {0}}}, + {"nand", {{0}, BENCH_CONST(4096), BENCH_CONST(32768), {0}}}, + {NULL, {{0}, {0}, {0}, {0}}}, +}; + +const bench_geometry_t *bench_geometries = builtin_geometries; +size_t bench_geometry_count = 5; + +static void list_geometries(void) { + // at least size so that names fit + unsigned name_width = 23; + for (size_t g = 0; builtin_geometries[g].name; g++) { + size_t len = strlen(builtin_geometries[g].name); + if (len > name_width) { + name_width = len; + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + + // yes we do need to define a suite, this does a bit of bookeeping + // such as setting up the define cache + bench_define_suite(&(const struct bench_suite){0}); + + printf("%-*s %7s %7s %7s %7s %11s\n", + name_width, "geometry", "read", "prog", "erase", "count", "size"); + for (size_t g = 0; builtin_geometries[g].name; g++) { + bench_define_geometry(&builtin_geometries[g]); + bench_define_flush(); + printf("%-*s %7ju %7ju %7ju %7ju %11ju\n", + name_width, + builtin_geometries[g].name, + READ_SIZE, + PROG_SIZE, + ERASE_SIZE, + ERASE_COUNT, + ERASE_SIZE*ERASE_COUNT); + } +} + + + +// global bench step count +size_t bench_step = 0; + +void perm_run( + void *data, + const struct bench_suite *suite, + const struct bench_case *case_) { + (void)data; + + // skip this step? + if (!(bench_step >= bench_step_start + && bench_step < bench_step_stop + && (bench_step-bench_step_start) % bench_step_step == 0)) { + bench_step += 1; + return; + } + bench_step += 1; + + // filter? + if (case_->filter && !case_->filter()) { + printf("skipped "); + perm_printid(suite, case_); + printf("\n"); + return; + } + + // create block device and configuration + lfs_emubd_t bd; + + struct lfs_config cfg = { + .context = &bd, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .block_size = BLOCK_SIZE, + .block_count = BLOCK_COUNT, + .block_cycles = BLOCK_CYCLES, + .cache_size = CACHE_SIZE, + .lookahead_size = LOOKAHEAD_SIZE, + }; + + struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, + .erase_value = ERASE_VALUE, + .erase_cycles = ERASE_CYCLES, + .badblock_behavior = BADBLOCK_BEHAVIOR, + .disk_path = bench_disk_path, + .read_sleep = bench_read_sleep, + .prog_sleep = bench_prog_sleep, + .erase_sleep = bench_erase_sleep, + }; + + int err = lfs_emubd_create(&cfg, &bdcfg); + if (err) { + fprintf(stderr, "error: could not create block device: %d\n", err); + exit(-1); + } + + // run the bench + bench_cfg = &cfg; + bench_reset(); + printf("running "); + perm_printid(suite, case_); + printf("\n"); + + case_->run(&cfg); + + printf("finished "); + perm_printid(suite, case_); + printf(" %"PRIu64" %"PRIu64" %"PRIu64, + bench_readed, + bench_proged, + bench_erased); + printf("\n"); + + // cleanup + err = lfs_emubd_destroy(&cfg); + if (err) { + fprintf(stderr, "error: could not destroy block device: %d\n", err); + exit(-1); + } +} + +static void run(void) { + // ignore disconnected pipes + signal(SIGPIPE, SIG_IGN); + + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_run, + NULL); + } + } + } +} + + + +// option handling +enum opt_flags { + OPT_HELP = 'h', + OPT_SUMMARY = 'Y', + OPT_LIST_SUITES = 'l', + OPT_LIST_CASES = 'L', + OPT_LIST_SUITE_PATHS = 1, + OPT_LIST_CASE_PATHS = 2, + OPT_LIST_DEFINES = 3, + OPT_LIST_PERMUTATION_DEFINES = 4, + OPT_LIST_IMPLICIT_DEFINES = 5, + OPT_LIST_GEOMETRIES = 6, + OPT_DEFINE = 'D', + OPT_GEOMETRY = 'G', + OPT_STEP = 's', + OPT_DISK = 'd', + OPT_TRACE = 't', + OPT_TRACE_BACKTRACE = 7, + OPT_TRACE_PERIOD = 8, + OPT_TRACE_FREQ = 9, + OPT_READ_SLEEP = 10, + OPT_PROG_SLEEP = 11, + OPT_ERASE_SLEEP = 12, +}; + +const char *short_opts = "hYlLD:G:s:d:t:"; + +const struct option long_opts[] = { + {"help", no_argument, NULL, OPT_HELP}, + {"summary", no_argument, NULL, OPT_SUMMARY}, + {"list-suites", no_argument, NULL, OPT_LIST_SUITES}, + {"list-cases", no_argument, NULL, OPT_LIST_CASES}, + {"list-suite-paths", no_argument, NULL, OPT_LIST_SUITE_PATHS}, + {"list-case-paths", no_argument, NULL, OPT_LIST_CASE_PATHS}, + {"list-defines", no_argument, NULL, OPT_LIST_DEFINES}, + {"list-permutation-defines", + no_argument, NULL, OPT_LIST_PERMUTATION_DEFINES}, + {"list-implicit-defines", + no_argument, NULL, OPT_LIST_IMPLICIT_DEFINES}, + {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES}, + {"define", required_argument, NULL, OPT_DEFINE}, + {"geometry", required_argument, NULL, OPT_GEOMETRY}, + {"step", required_argument, NULL, OPT_STEP}, + {"disk", required_argument, NULL, OPT_DISK}, + {"trace", required_argument, NULL, OPT_TRACE}, + {"trace-backtrace", no_argument, NULL, OPT_TRACE_BACKTRACE}, + {"trace-period", required_argument, NULL, OPT_TRACE_PERIOD}, + {"trace-freq", required_argument, NULL, OPT_TRACE_FREQ}, + {"read-sleep", required_argument, NULL, OPT_READ_SLEEP}, + {"prog-sleep", required_argument, NULL, OPT_PROG_SLEEP}, + {"erase-sleep", required_argument, NULL, OPT_ERASE_SLEEP}, + {NULL, 0, NULL, 0}, +}; + +const char *const help_text[] = { + "Show this help message.", + "Show quick summary.", + "List bench suites.", + "List bench cases.", + "List the path for each bench suite.", + "List the path and line number for each bench case.", + "List all defines in this bench-runner.", + "List explicit defines in this bench-runner.", + "List implicit defines in this bench-runner.", + "List the available disk geometries.", + "Override a bench define.", + "Comma-separated list of disk geometries to bench.", + "Comma-separated range of bench permutations to run (start,stop,step).", + "Direct block device operations to this file.", + "Direct trace output to this file.", + "Include a backtrace with every trace statement.", + "Sample trace output at this period in cycles.", + "Sample trace output at this frequency in hz.", + "Artificial read delay in seconds.", + "Artificial prog delay in seconds.", + "Artificial erase delay in seconds.", +}; + +int main(int argc, char **argv) { + void (*op)(void) = run; + + size_t bench_override_capacity = 0; + size_t bench_geometry_capacity = 0; + size_t bench_id_capacity = 0; + + // parse options + while (true) { + int c = getopt_long(argc, argv, short_opts, long_opts, NULL); + switch (c) { + // generate help message + case OPT_HELP: { + printf("usage: %s [options] [bench_id]\n", argv[0]); + printf("\n"); + + printf("options:\n"); + size_t i = 0; + while (long_opts[i].name) { + size_t indent; + if (long_opts[i].has_arg == no_argument) { + if (long_opts[i].val >= '0' && long_opts[i].val < 'z') { + indent = printf(" -%c, --%s ", + long_opts[i].val, + long_opts[i].name); + } else { + indent = printf(" --%s ", + long_opts[i].name); + } + } else { + if (long_opts[i].val >= '0' && long_opts[i].val < 'z') { + indent = printf(" -%c %s, --%s %s ", + long_opts[i].val, + long_opts[i].name, + long_opts[i].name, + long_opts[i].name); + } else { + indent = printf(" --%s %s ", + long_opts[i].name, + long_opts[i].name); + } + } + + // a quick, hacky, byte-level method for text wrapping + size_t len = strlen(help_text[i]); + size_t j = 0; + if (indent < 24) { + printf("%*s %.80s\n", + (int)(24-1-indent), + "", + &help_text[i][j]); + j += 80; + } else { + printf("\n"); + } + + while (j < len) { + printf("%24s%.80s\n", "", &help_text[i][j]); + j += 80; + } + + i += 1; + } + + printf("\n"); + exit(0); + } + // summary/list flags + case OPT_SUMMARY: + op = summary; + break; + case OPT_LIST_SUITES: + op = list_suites; + break; + case OPT_LIST_CASES: + op = list_cases; + break; + case OPT_LIST_SUITE_PATHS: + op = list_suite_paths; + break; + case OPT_LIST_CASE_PATHS: + op = list_case_paths; + break; + case OPT_LIST_DEFINES: + op = list_defines; + break; + case OPT_LIST_PERMUTATION_DEFINES: + op = list_permutation_defines; + break; + case OPT_LIST_IMPLICIT_DEFINES: + op = list_implicit_defines; + break; + case OPT_LIST_GEOMETRIES: + op = list_geometries; + break; + // configuration + case OPT_DEFINE: { + // allocate space + bench_override_t *override = mappend( + (void**)&bench_overrides, + sizeof(bench_override_t), + &bench_override_count, + &bench_override_capacity); + + // parse into string key/intmax_t value, cannibalizing the + // arg in the process + char *sep = strchr(optarg, '='); + char *parsed = NULL; + if (!sep) { + goto invalid_define; + } + *sep = '\0'; + override->name = optarg; + optarg = sep+1; + + // parse comma-separated permutations + { + override->defines = NULL; + override->permutations = 0; + size_t override_capacity = 0; + while (true) { + optarg += strspn(optarg, " "); + + if (strncmp(optarg, "range", strlen("range")) == 0) { + // range of values + optarg += strlen("range"); + optarg += strspn(optarg, " "); + if (*optarg != '(') { + goto invalid_define; + } + optarg += 1; + + intmax_t start = strtoumax(optarg, &parsed, 0); + intmax_t stop = -1; + intmax_t step = 1; + // allow empty string for start=0 + if (parsed == optarg) { + start = 0; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != ')') { + goto invalid_define; + } + + if (*optarg == ',') { + optarg += 1; + stop = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=end + if (parsed == optarg) { + stop = -1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != ')') { + goto invalid_define; + } + + if (*optarg == ',') { + optarg += 1; + step = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=1 + if (parsed == optarg) { + step = 1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ')') { + goto invalid_define; + } + } + } else { + // single value = stop only + stop = start; + start = 0; + } + + if (*optarg != ')') { + goto invalid_define; + } + optarg += 1; + + // calculate the range of values + assert(step != 0); + for (intmax_t i = start; + (step < 0) + ? i > stop + : (uintmax_t)i < (uintmax_t)stop; + i += step) { + *(intmax_t*)mappend( + (void**)&override->defines, + sizeof(intmax_t), + &override->permutations, + &override_capacity) = i; + } + } else if (*optarg != '\0') { + // single value + intmax_t define = strtoimax(optarg, &parsed, 0); + if (parsed == optarg) { + goto invalid_define; + } + optarg = parsed + strspn(parsed, " "); + *(intmax_t*)mappend( + (void**)&override->defines, + sizeof(intmax_t), + &override->permutations, + &override_capacity) = define; + } else { + break; + } + + if (*optarg == ',') { + optarg += 1; + } + } + } + assert(override->permutations > 0); + break; + +invalid_define: + fprintf(stderr, "error: invalid define: %s\n", optarg); + exit(-1); + } + case OPT_GEOMETRY: { + // reset our geometry scenarios + if (bench_geometry_capacity > 0) { + free((bench_geometry_t*)bench_geometries); + } + bench_geometries = NULL; + bench_geometry_count = 0; + bench_geometry_capacity = 0; + + // parse the comma separated list of disk geometries + while (*optarg) { + // allocate space + bench_geometry_t *geometry = mappend( + (void**)&bench_geometries, + sizeof(bench_geometry_t), + &bench_geometry_count, + &bench_geometry_capacity); + + // parse the disk geometry + optarg += strspn(optarg, " "); + + // named disk geometry + size_t len = strcspn(optarg, " ,"); + for (size_t i = 0; builtin_geometries[i].name; i++) { + if (len == strlen(builtin_geometries[i].name) + && memcmp(optarg, + builtin_geometries[i].name, + len) == 0) { + *geometry = builtin_geometries[i]; + optarg += len; + goto geometry_next; + } + } + + // comma-separated read/prog/erase/count + if (*optarg == '{') { + lfs_size_t sizes[4]; + size_t count = 0; + + char *s = optarg + 1; + while (count < 4) { + char *parsed = NULL; + sizes[count] = strtoumax(s, &parsed, 0); + count += 1; + + s = parsed + strspn(parsed, " "); + if (*s == ',') { + s += 1; + continue; + } else if (*s == '}') { + s += 1; + break; + } else { + goto geometry_unknown; + } + } + + // allow implicit r=p and p=e for common geometries + memset(geometry, 0, sizeof(bench_geometry_t)); + if (count >= 3) { + geometry->defines[READ_SIZE_i] + = BENCH_LIT(sizes[0]); + geometry->defines[PROG_SIZE_i] + = BENCH_LIT(sizes[1]); + geometry->defines[ERASE_SIZE_i] + = BENCH_LIT(sizes[2]); + } else if (count >= 2) { + geometry->defines[PROG_SIZE_i] + = BENCH_LIT(sizes[0]); + geometry->defines[ERASE_SIZE_i] + = BENCH_LIT(sizes[1]); + } else { + geometry->defines[ERASE_SIZE_i] + = BENCH_LIT(sizes[0]); + } + if (count >= 4) { + geometry->defines[ERASE_COUNT_i] + = BENCH_LIT(sizes[3]); + } + optarg = s; + goto geometry_next; + } + + // leb16-encoded read/prog/erase/count + if (*optarg == ':') { + lfs_size_t sizes[4]; + size_t count = 0; + + char *s = optarg + 1; + while (true) { + char *parsed = NULL; + uintmax_t x = leb16_parse(s, &parsed); + if (parsed == s || count >= 4) { + break; + } + + sizes[count] = x; + count += 1; + s = parsed; + } + + // allow implicit r=p and p=e for common geometries + memset(geometry, 0, sizeof(bench_geometry_t)); + if (count >= 3) { + geometry->defines[READ_SIZE_i] + = BENCH_LIT(sizes[0]); + geometry->defines[PROG_SIZE_i] + = BENCH_LIT(sizes[1]); + geometry->defines[ERASE_SIZE_i] + = BENCH_LIT(sizes[2]); + } else if (count >= 2) { + geometry->defines[PROG_SIZE_i] + = BENCH_LIT(sizes[0]); + geometry->defines[ERASE_SIZE_i] + = BENCH_LIT(sizes[1]); + } else { + geometry->defines[ERASE_SIZE_i] + = BENCH_LIT(sizes[0]); + } + if (count >= 4) { + geometry->defines[ERASE_COUNT_i] + = BENCH_LIT(sizes[3]); + } + optarg = s; + goto geometry_next; + } + +geometry_unknown: + // unknown scenario? + fprintf(stderr, "error: unknown disk geometry: %s\n", + optarg); + exit(-1); + +geometry_next: + optarg += strspn(optarg, " "); + if (*optarg == ',') { + optarg += 1; + } else if (*optarg == '\0') { + break; + } else { + goto geometry_unknown; + } + } + break; + } + case OPT_STEP: { + char *parsed = NULL; + bench_step_start = strtoumax(optarg, &parsed, 0); + bench_step_stop = -1; + bench_step_step = 1; + // allow empty string for start=0 + if (parsed == optarg) { + bench_step_start = 0; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != '\0') { + goto step_unknown; + } + + if (*optarg == ',') { + optarg += 1; + bench_step_stop = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=end + if (parsed == optarg) { + bench_step_stop = -1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != '\0') { + goto step_unknown; + } + + if (*optarg == ',') { + optarg += 1; + bench_step_step = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=1 + if (parsed == optarg) { + bench_step_step = 1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != '\0') { + goto step_unknown; + } + } + } else { + // single value = stop only + bench_step_stop = bench_step_start; + bench_step_start = 0; + } + + break; +step_unknown: + fprintf(stderr, "error: invalid step: %s\n", optarg); + exit(-1); + } + case OPT_DISK: + bench_disk_path = optarg; + break; + case OPT_TRACE: + bench_trace_path = optarg; + break; + case OPT_TRACE_BACKTRACE: + bench_trace_backtrace = true; + break; + case OPT_TRACE_PERIOD: { + char *parsed = NULL; + bench_trace_period = strtoumax(optarg, &parsed, 0); + if (parsed == optarg) { + fprintf(stderr, "error: invalid trace-period: %s\n", optarg); + exit(-1); + } + break; + } + case OPT_TRACE_FREQ: { + char *parsed = NULL; + bench_trace_freq = strtoumax(optarg, &parsed, 0); + if (parsed == optarg) { + fprintf(stderr, "error: invalid trace-freq: %s\n", optarg); + exit(-1); + } + break; + } + case OPT_READ_SLEEP: { + char *parsed = NULL; + double read_sleep = strtod(optarg, &parsed); + if (parsed == optarg) { + fprintf(stderr, "error: invalid read-sleep: %s\n", optarg); + exit(-1); + } + bench_read_sleep = read_sleep*1.0e9; + break; + } + case OPT_PROG_SLEEP: { + char *parsed = NULL; + double prog_sleep = strtod(optarg, &parsed); + if (parsed == optarg) { + fprintf(stderr, "error: invalid prog-sleep: %s\n", optarg); + exit(-1); + } + bench_prog_sleep = prog_sleep*1.0e9; + break; + } + case OPT_ERASE_SLEEP: { + char *parsed = NULL; + double erase_sleep = strtod(optarg, &parsed); + if (parsed == optarg) { + fprintf(stderr, "error: invalid erase-sleep: %s\n", optarg); + exit(-1); + } + bench_erase_sleep = erase_sleep*1.0e9; + break; + } + // done parsing + case -1: + goto getopt_done; + // unknown arg, getopt prints a message for us + default: + exit(-1); + } + } +getopt_done: ; + + if (argc > optind) { + // reset our bench identifier list + bench_ids = NULL; + bench_id_count = 0; + bench_id_capacity = 0; + } + + // parse bench identifier, if any, cannibalizing the arg in the process + for (; argc > optind; optind++) { + bench_define_t *defines = NULL; + size_t define_count = 0; + + // parse name, can be suite or case + char *name = argv[optind]; + char *defines_ = strchr(name, ':'); + if (defines_) { + *defines_ = '\0'; + defines_ += 1; + } + + // remove optional path and .toml suffix + char *slash = strrchr(name, '/'); + if (slash) { + name = slash+1; + } + + size_t name_len = strlen(name); + if (name_len > 5 && strcmp(&name[name_len-5], ".toml") == 0) { + name[name_len-5] = '\0'; + } + + if (defines_) { + // parse defines + while (true) { + char *parsed; + size_t d = leb16_parse(defines_, &parsed); + intmax_t v = leb16_parse(parsed, &parsed); + if (parsed == defines_) { + break; + } + defines_ = parsed; + + if (d >= define_count) { + // align to power of two to avoid any superlinear growth + size_t ncount = 1 << lfs_npw2(d+1); + defines = realloc(defines, + ncount*sizeof(bench_define_t)); + memset(defines+define_count, 0, + (ncount-define_count)*sizeof(bench_define_t)); + define_count = ncount; + } + defines[d] = BENCH_LIT(v); + } + } + + // append to identifier list + *(bench_id_t*)mappend( + (void**)&bench_ids, + sizeof(bench_id_t), + &bench_id_count, + &bench_id_capacity) = (bench_id_t){ + .name = name, + .defines = defines, + .define_count = define_count, + }; + } + + // do the thing + op(); + + // cleanup (need to be done for valgrind benching) + bench_define_cleanup(); + if (bench_overrides) { + for (size_t i = 0; i < bench_override_count; i++) { + free((void*)bench_overrides[i].defines); + } + free((void*)bench_overrides); + } + if (bench_geometry_capacity) { + free((void*)bench_geometries); + } + if (bench_id_capacity) { + for (size_t i = 0; i < bench_id_count; i++) { + free((void*)bench_ids[i].defines); + } + free((void*)bench_ids); + } +} diff --git a/components/fs/littlefs/littlefs/runners/bench_runner.h b/components/fs/littlefs/littlefs/runners/bench_runner.h new file mode 100644 index 000000000..b072970ea --- /dev/null +++ b/components/fs/littlefs/littlefs/runners/bench_runner.h @@ -0,0 +1,137 @@ +/* + * Runner for littlefs benchmarks + * + * Copyright (c) 2022, The littlefs authors. + * SPDX-License-Identifier: BSD-3-Clause + */ +#ifndef BENCH_RUNNER_H +#define BENCH_RUNNER_H + + +// override LFS_TRACE +void bench_trace(const char *fmt, ...); + +#define LFS_TRACE_(fmt, ...) \ + bench_trace("%s:%d:trace: " fmt "%s\n", \ + __FILE__, \ + __LINE__, \ + __VA_ARGS__) +#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") +#define LFS_EMUBD_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") + +// provide BENCH_START/BENCH_STOP macros +void bench_start(void); +void bench_stop(void); + +#define BENCH_START() bench_start() +#define BENCH_STOP() bench_stop() + + +// note these are indirectly included in any generated files +#include "bd/lfs_emubd.h" +#include + +// give source a chance to define feature macros +#undef _FEATURES_H +#undef _STDIO_H + + +// generated bench configurations +struct lfs_config; + +enum bench_flags { + BENCH_REENTRANT = 0x1, +}; +typedef uint8_t bench_flags_t; + +typedef struct bench_define { + intmax_t (*cb)(void *data); + void *data; +} bench_define_t; + +struct bench_case { + const char *name; + const char *path; + bench_flags_t flags; + size_t permutations; + + const bench_define_t *defines; + + bool (*filter)(void); + void (*run)(struct lfs_config *cfg); +}; + +struct bench_suite { + const char *name; + const char *path; + bench_flags_t flags; + + const char *const *define_names; + size_t define_count; + + const struct bench_case *cases; + size_t case_count; +}; + + +// deterministic prng for pseudo-randomness in benches +uint32_t bench_prng(uint32_t *state); + +#define BENCH_PRNG(state) bench_prng(state) + + +// access generated bench defines +intmax_t bench_define(size_t define); + +#define BENCH_DEFINE(i) bench_define(i) + +// a few preconfigured defines that control how benches run + +#define READ_SIZE_i 0 +#define PROG_SIZE_i 1 +#define ERASE_SIZE_i 2 +#define ERASE_COUNT_i 3 +#define BLOCK_SIZE_i 4 +#define BLOCK_COUNT_i 5 +#define CACHE_SIZE_i 6 +#define LOOKAHEAD_SIZE_i 7 +#define BLOCK_CYCLES_i 8 +#define ERASE_VALUE_i 9 +#define ERASE_CYCLES_i 10 +#define BADBLOCK_BEHAVIOR_i 11 +#define POWERLOSS_BEHAVIOR_i 12 + +#define READ_SIZE bench_define(READ_SIZE_i) +#define PROG_SIZE bench_define(PROG_SIZE_i) +#define ERASE_SIZE bench_define(ERASE_SIZE_i) +#define ERASE_COUNT bench_define(ERASE_COUNT_i) +#define BLOCK_SIZE bench_define(BLOCK_SIZE_i) +#define BLOCK_COUNT bench_define(BLOCK_COUNT_i) +#define CACHE_SIZE bench_define(CACHE_SIZE_i) +#define LOOKAHEAD_SIZE bench_define(LOOKAHEAD_SIZE_i) +#define BLOCK_CYCLES bench_define(BLOCK_CYCLES_i) +#define ERASE_VALUE bench_define(ERASE_VALUE_i) +#define ERASE_CYCLES bench_define(ERASE_CYCLES_i) +#define BADBLOCK_BEHAVIOR bench_define(BADBLOCK_BEHAVIOR_i) +#define POWERLOSS_BEHAVIOR bench_define(POWERLOSS_BEHAVIOR_i) + +#define BENCH_IMPLICIT_DEFINES \ + BENCH_DEF(READ_SIZE, PROG_SIZE) \ + BENCH_DEF(PROG_SIZE, ERASE_SIZE) \ + BENCH_DEF(ERASE_SIZE, 0) \ + BENCH_DEF(ERASE_COUNT, (1024*1024)/BLOCK_SIZE) \ + BENCH_DEF(BLOCK_SIZE, ERASE_SIZE) \ + BENCH_DEF(BLOCK_COUNT, ERASE_COUNT/lfs_max(BLOCK_SIZE/ERASE_SIZE,1))\ + BENCH_DEF(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ + BENCH_DEF(LOOKAHEAD_SIZE, 16) \ + BENCH_DEF(BLOCK_CYCLES, -1) \ + BENCH_DEF(ERASE_VALUE, 0xff) \ + BENCH_DEF(ERASE_CYCLES, 0) \ + BENCH_DEF(BADBLOCK_BEHAVIOR, LFS_EMUBD_BADBLOCK_PROGERROR) \ + BENCH_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP) + +#define BENCH_GEOMETRY_DEFINE_COUNT 4 +#define BENCH_IMPLICIT_DEFINE_COUNT 13 + + +#endif diff --git a/components/fs/littlefs/littlefs/runners/test_runner.c b/components/fs/littlefs/littlefs/runners/test_runner.c new file mode 100644 index 000000000..1f0904845 --- /dev/null +++ b/components/fs/littlefs/littlefs/runners/test_runner.c @@ -0,0 +1,2798 @@ +/* + * Runner for littlefs tests + * + * Copyright (c) 2022, The littlefs authors. + * SPDX-License-Identifier: BSD-3-Clause + */ +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 199309L +#endif + +#include "runners/test_runner.h" +#include "bd/lfs_emubd.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +// some helpers + +// append to an array with amortized doubling +void *mappend(void **p, + size_t size, + size_t *count, + size_t *capacity) { + uint8_t *p_ = *p; + size_t count_ = *count; + size_t capacity_ = *capacity; + + count_ += 1; + if (count_ > capacity_) { + capacity_ = (2*capacity_ < 4) ? 4 : 2*capacity_; + + p_ = realloc(p_, capacity_*size); + if (!p_) { + return NULL; + } + } + + *p = p_; + *count = count_; + *capacity = capacity_; + return &p_[(count_-1)*size]; +} + +// a quick self-terminating text-safe varint scheme +static void leb16_print(uintmax_t x) { + // allow 'w' to indicate negative numbers + if ((intmax_t)x < 0) { + printf("w"); + x = -x; + } + + while (true) { + char nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0); + printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10); + if (x <= 0xf) { + break; + } + x >>= 4; + } +} + +static uintmax_t leb16_parse(const char *s, char **tail) { + bool neg = false; + uintmax_t x = 0; + if (tail) { + *tail = (char*)s; + } + + if (s[0] == 'w') { + neg = true; + s = s+1; + } + + size_t i = 0; + while (true) { + uintmax_t nibble = s[i]; + if (nibble >= '0' && nibble <= '9') { + nibble = nibble - '0'; + } else if (nibble >= 'a' && nibble <= 'v') { + nibble = nibble - 'a' + 10; + } else { + // invalid? + return 0; + } + + x |= (nibble & 0xf) << (4*i); + i += 1; + if (!(nibble & 0x10)) { + s = s + i; + break; + } + } + + if (tail) { + *tail = (char*)s; + } + return neg ? -x : x; +} + + + +// test_runner types + +typedef struct test_geometry { + const char *name; + test_define_t defines[TEST_GEOMETRY_DEFINE_COUNT]; +} test_geometry_t; + +typedef struct test_powerloss { + const char *name; + void (*run)( + const lfs_emubd_powercycles_t *cycles, + size_t cycle_count, + const struct test_suite *suite, + const struct test_case *case_); + const lfs_emubd_powercycles_t *cycles; + size_t cycle_count; +} test_powerloss_t; + +typedef struct test_id { + const char *name; + const test_define_t *defines; + size_t define_count; + const lfs_emubd_powercycles_t *cycles; + size_t cycle_count; +} test_id_t; + + +// test suites are linked into a custom ld section +extern struct test_suite __start__test_suites; +extern struct test_suite __stop__test_suites; + +const struct test_suite *test_suites = &__start__test_suites; +#define TEST_SUITE_COUNT \ + ((size_t)(&__stop__test_suites - &__start__test_suites)) + + +// test define management +typedef struct test_define_map { + const test_define_t *defines; + size_t count; +} test_define_map_t; + +typedef struct test_define_names { + const char *const *names; + size_t count; +} test_define_names_t; + +intmax_t test_define_lit(void *data) { + return (intptr_t)data; +} + +#define TEST_CONST(x) {test_define_lit, (void*)(uintptr_t)(x)} +#define TEST_LIT(x) ((test_define_t)TEST_CONST(x)) + + +#define TEST_DEF(k, v) \ + intmax_t test_define_##k(void *data) { \ + (void)data; \ + return v; \ + } + + TEST_IMPLICIT_DEFINES +#undef TEST_DEF + +#define TEST_DEFINE_MAP_OVERRIDE 0 +#define TEST_DEFINE_MAP_EXPLICIT 1 +#define TEST_DEFINE_MAP_PERMUTATION 2 +#define TEST_DEFINE_MAP_GEOMETRY 3 +#define TEST_DEFINE_MAP_IMPLICIT 4 +#define TEST_DEFINE_MAP_COUNT 5 + +test_define_map_t test_define_maps[TEST_DEFINE_MAP_COUNT] = { + [TEST_DEFINE_MAP_IMPLICIT] = { + (const test_define_t[TEST_IMPLICIT_DEFINE_COUNT]) { + #define TEST_DEF(k, v) \ + [k##_i] = {test_define_##k, NULL}, + + TEST_IMPLICIT_DEFINES + #undef TEST_DEF + }, + TEST_IMPLICIT_DEFINE_COUNT, + }, +}; + +#define TEST_DEFINE_NAMES_SUITE 0 +#define TEST_DEFINE_NAMES_IMPLICIT 1 +#define TEST_DEFINE_NAMES_COUNT 2 + +test_define_names_t test_define_names[TEST_DEFINE_NAMES_COUNT] = { + [TEST_DEFINE_NAMES_IMPLICIT] = { + (const char *const[TEST_IMPLICIT_DEFINE_COUNT]){ + #define TEST_DEF(k, v) \ + [k##_i] = #k, + + TEST_IMPLICIT_DEFINES + #undef TEST_DEF + }, + TEST_IMPLICIT_DEFINE_COUNT, + }, +}; + +intmax_t *test_define_cache; +size_t test_define_cache_count; +unsigned *test_define_cache_mask; + +const char *test_define_name(size_t define) { + // lookup in our test names + for (size_t i = 0; i < TEST_DEFINE_NAMES_COUNT; i++) { + if (define < test_define_names[i].count + && test_define_names[i].names + && test_define_names[i].names[define]) { + return test_define_names[i].names[define]; + } + } + + return NULL; +} + +bool test_define_ispermutation(size_t define) { + // is this define specific to the permutation? + for (size_t i = 0; i < TEST_DEFINE_MAP_IMPLICIT; i++) { + if (define < test_define_maps[i].count + && test_define_maps[i].defines[define].cb) { + return true; + } + } + + return false; +} + +intmax_t test_define(size_t define) { + // is the define in our cache? + if (define < test_define_cache_count + && (test_define_cache_mask[define/(8*sizeof(unsigned))] + & (1 << (define%(8*sizeof(unsigned)))))) { + return test_define_cache[define]; + } + + // lookup in our test defines + for (size_t i = 0; i < TEST_DEFINE_MAP_COUNT; i++) { + if (define < test_define_maps[i].count + && test_define_maps[i].defines[define].cb) { + intmax_t v = test_define_maps[i].defines[define].cb( + test_define_maps[i].defines[define].data); + + // insert into cache! + test_define_cache[define] = v; + test_define_cache_mask[define / (8*sizeof(unsigned))] + |= 1 << (define%(8*sizeof(unsigned))); + + return v; + } + } + + return 0; + + // not found? + const char *name = test_define_name(define); + fprintf(stderr, "error: undefined define %s (%zd)\n", + name ? name : "(unknown)", + define); + assert(false); + exit(-1); +} + +void test_define_flush(void) { + // clear cache between permutations + memset(test_define_cache_mask, 0, + sizeof(unsigned)*( + (test_define_cache_count+(8*sizeof(unsigned))-1) + / (8*sizeof(unsigned)))); +} + +// geometry updates +const test_geometry_t *test_geometry = NULL; + +void test_define_geometry(const test_geometry_t *geometry) { + test_define_maps[TEST_DEFINE_MAP_GEOMETRY] = (test_define_map_t){ + geometry->defines, TEST_GEOMETRY_DEFINE_COUNT}; +} + +// override updates +typedef struct test_override { + const char *name; + const intmax_t *defines; + size_t permutations; +} test_override_t; + +const test_override_t *test_overrides = NULL; +size_t test_override_count = 0; + +test_define_t *test_override_defines = NULL; +size_t test_override_define_count = 0; +size_t test_override_define_permutations = 1; +size_t test_override_define_capacity = 0; + +// suite/perm updates +void test_define_suite(const struct test_suite *suite) { + test_define_names[TEST_DEFINE_NAMES_SUITE] = (test_define_names_t){ + suite->define_names, suite->define_count}; + + // make sure our cache is large enough + if (lfs_max(suite->define_count, TEST_IMPLICIT_DEFINE_COUNT) + > test_define_cache_count) { + // align to power of two to avoid any superlinear growth + size_t ncount = 1 << lfs_npw2( + lfs_max(suite->define_count, TEST_IMPLICIT_DEFINE_COUNT)); + test_define_cache = realloc(test_define_cache, ncount*sizeof(intmax_t)); + test_define_cache_mask = realloc(test_define_cache_mask, + sizeof(unsigned)*( + (ncount+(8*sizeof(unsigned))-1) + / (8*sizeof(unsigned)))); + test_define_cache_count = ncount; + } + + // map any overrides + if (test_override_count > 0) { + // first figure out the total size of override permutations + size_t count = 0; + size_t permutations = 1; + for (size_t i = 0; i < test_override_count; i++) { + for (size_t d = 0; + d < lfs_max( + suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + // define name match? + const char *name = test_define_name(d); + if (name && strcmp(name, test_overrides[i].name) == 0) { + count = lfs_max(count, d+1); + permutations *= test_overrides[i].permutations; + break; + } + } + } + test_override_define_count = count; + test_override_define_permutations = permutations; + + // make sure our override arrays are big enough + if (count * permutations > test_override_define_capacity) { + // align to power of two to avoid any superlinear growth + size_t ncapacity = 1 << lfs_npw2(count * permutations); + test_override_defines = realloc( + test_override_defines, + sizeof(test_define_t)*ncapacity); + test_override_define_capacity = ncapacity; + } + + // zero unoverridden defines + memset(test_override_defines, 0, + sizeof(test_define_t) * count * permutations); + + // compute permutations + size_t p = 1; + for (size_t i = 0; i < test_override_count; i++) { + for (size_t d = 0; + d < lfs_max( + suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + // define name match? + const char *name = test_define_name(d); + if (name && strcmp(name, test_overrides[i].name) == 0) { + // scatter the define permutations based on already + // seen permutations + for (size_t j = 0; j < permutations; j++) { + test_override_defines[j*count + d] = TEST_LIT( + test_overrides[i].defines[(j/p) + % test_overrides[i].permutations]); + } + + // keep track of how many permutations we've seen so far + p *= test_overrides[i].permutations; + break; + } + } + } + } +} + +void test_define_perm( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm) { + if (case_->defines) { + test_define_maps[TEST_DEFINE_MAP_PERMUTATION] = (test_define_map_t){ + case_->defines + perm*suite->define_count, + suite->define_count}; + } else { + test_define_maps[TEST_DEFINE_MAP_PERMUTATION] = (test_define_map_t){ + NULL, 0}; + } +} + +void test_define_override(size_t perm) { + test_define_maps[TEST_DEFINE_MAP_OVERRIDE] = (test_define_map_t){ + test_override_defines + perm*test_override_define_count, + test_override_define_count}; +} + +void test_define_explicit( + const test_define_t *defines, + size_t define_count) { + test_define_maps[TEST_DEFINE_MAP_EXPLICIT] = (test_define_map_t){ + defines, define_count}; +} + +void test_define_cleanup(void) { + // test define management can allocate a few things + free(test_define_cache); + free(test_define_cache_mask); + free(test_override_defines); +} + + + +// test state +extern const test_geometry_t *test_geometries; +extern size_t test_geometry_count; + +extern const test_powerloss_t *test_powerlosses; +extern size_t test_powerloss_count; + +const test_id_t *test_ids = (const test_id_t[]) { + {NULL, NULL, 0, NULL, 0}, +}; +size_t test_id_count = 1; + +size_t test_step_start = 0; +size_t test_step_stop = -1; +size_t test_step_step = 1; + +const char *test_disk_path = NULL; +const char *test_trace_path = NULL; +bool test_trace_backtrace = false; +uint32_t test_trace_period = 0; +uint32_t test_trace_freq = 0; +FILE *test_trace_file = NULL; +uint32_t test_trace_cycles = 0; +uint64_t test_trace_time = 0; +uint64_t test_trace_open_time = 0; +lfs_emubd_sleep_t test_read_sleep = 0.0; +lfs_emubd_sleep_t test_prog_sleep = 0.0; +lfs_emubd_sleep_t test_erase_sleep = 0.0; + +// this determines both the backtrace buffer and the trace printf buffer, if +// trace ends up interleaved or truncated this may need to be increased +#ifndef TEST_TRACE_BACKTRACE_BUFFER_SIZE +#define TEST_TRACE_BACKTRACE_BUFFER_SIZE 8192 +#endif +void *test_trace_backtrace_buffer[ + TEST_TRACE_BACKTRACE_BUFFER_SIZE / sizeof(void*)]; + +// trace printing +void test_trace(const char *fmt, ...) { + if (test_trace_path) { + // sample at a specific period? + if (test_trace_period) { + if (test_trace_cycles % test_trace_period != 0) { + test_trace_cycles += 1; + return; + } + test_trace_cycles += 1; + } + + // sample at a specific frequency? + if (test_trace_freq) { + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + uint64_t now = (uint64_t)t.tv_sec*1000*1000*1000 + + (uint64_t)t.tv_nsec; + if (now - test_trace_time < (1000*1000*1000) / test_trace_freq) { + return; + } + test_trace_time = now; + } + + if (!test_trace_file) { + // Tracing output is heavy and trying to open every trace + // call is slow, so we only try to open the trace file every + // so often. Note this doesn't affect successfully opened files + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + uint64_t now = (uint64_t)t.tv_sec*1000*1000*1000 + + (uint64_t)t.tv_nsec; + if (now - test_trace_open_time < 100*1000*1000) { + return; + } + test_trace_open_time = now; + + // try to open the trace file + int fd; + if (strcmp(test_trace_path, "-") == 0) { + fd = dup(1); + if (fd < 0) { + return; + } + } else { + fd = open( + test_trace_path, + O_WRONLY | O_CREAT | O_APPEND | O_NONBLOCK, + 0666); + if (fd < 0) { + return; + } + int err = fcntl(fd, F_SETFL, O_WRONLY | O_CREAT | O_APPEND); + assert(!err); + } + + FILE *f = fdopen(fd, "a"); + assert(f); + int err = setvbuf(f, NULL, _IOFBF, + TEST_TRACE_BACKTRACE_BUFFER_SIZE); + assert(!err); + test_trace_file = f; + } + + // print trace + va_list va; + va_start(va, fmt); + int res = vfprintf(test_trace_file, fmt, va); + va_end(va); + if (res < 0) { + fclose(test_trace_file); + test_trace_file = NULL; + return; + } + + if (test_trace_backtrace) { + // print backtrace + size_t count = backtrace( + test_trace_backtrace_buffer, + TEST_TRACE_BACKTRACE_BUFFER_SIZE); + // note we skip our own stack frame + for (size_t i = 1; i < count; i++) { + res = fprintf(test_trace_file, "\tat %p\n", + test_trace_backtrace_buffer[i]); + if (res < 0) { + fclose(test_trace_file); + test_trace_file = NULL; + return; + } + } + } + + // flush immediately + fflush(test_trace_file); + } +} + + +// test prng +uint32_t test_prng(uint32_t *state) { + // A simple xorshift32 generator, easily reproducible. Keep in mind + // determinism is much more important than actual randomness here. + uint32_t x = *state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + *state = x; + return x; +} + + +// encode our permutation into a reusable id +static void perm_printid( + const struct test_suite *suite, + const struct test_case *case_, + const lfs_emubd_powercycles_t *cycles, + size_t cycle_count) { + (void)suite; + // case[:permutation[:powercycles]] + printf("%s:", case_->name); + for (size_t d = 0; + d < lfs_max( + suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + if (test_define_ispermutation(d)) { + leb16_print(d); + leb16_print(TEST_DEFINE(d)); + } + } + + // only print power-cycles if any occured + if (cycles) { + printf(":"); + for (size_t i = 0; i < cycle_count; i++) { + leb16_print(cycles[i]); + } + } +} + + +// a quick trie for keeping track of permutations we've seen +typedef struct test_seen { + struct test_seen_branch *branches; + size_t branch_count; + size_t branch_capacity; +} test_seen_t; + +struct test_seen_branch { + intmax_t define; + struct test_seen branch; +}; + +bool test_seen_insert( + test_seen_t *seen, + const struct test_suite *suite, + const struct test_case *case_) { + (void)case_; + bool was_seen = true; + + // use the currently set defines + for (size_t d = 0; + d < lfs_max( + suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + // treat unpermuted defines the same as 0 + intmax_t define = test_define_ispermutation(d) ? TEST_DEFINE(d) : 0; + + // already seen? + struct test_seen_branch *branch = NULL; + for (size_t i = 0; i < seen->branch_count; i++) { + if (seen->branches[i].define == define) { + branch = &seen->branches[i]; + break; + } + } + + // need to create a new node + if (!branch) { + was_seen = false; + branch = mappend( + (void**)&seen->branches, + sizeof(struct test_seen_branch), + &seen->branch_count, + &seen->branch_capacity); + branch->define = define; + branch->branch = (test_seen_t){NULL, 0, 0}; + } + + seen = &branch->branch; + } + + return was_seen; +} + +void test_seen_cleanup(test_seen_t *seen) { + for (size_t i = 0; i < seen->branch_count; i++) { + test_seen_cleanup(&seen->branches[i].branch); + } + free(seen->branches); +} + +static void run_powerloss_none( + const lfs_emubd_powercycles_t *cycles, + size_t cycle_count, + const struct test_suite *suite, + const struct test_case *case_); +static void run_powerloss_cycles( + const lfs_emubd_powercycles_t *cycles, + size_t cycle_count, + const struct test_suite *suite, + const struct test_case *case_); + +// iterate through permutations in a test case +static void case_forperm( + const struct test_suite *suite, + const struct test_case *case_, + const test_define_t *defines, + size_t define_count, + const lfs_emubd_powercycles_t *cycles, + size_t cycle_count, + void (*cb)( + void *data, + const struct test_suite *suite, + const struct test_case *case_, + const test_powerloss_t *powerloss), + void *data) { + // explicit permutation? + if (defines) { + test_define_explicit(defines, define_count); + + for (size_t v = 0; v < test_override_define_permutations; v++) { + // define override permutation + test_define_override(v); + test_define_flush(); + + // explicit powerloss cycles? + if (cycles) { + cb(data, suite, case_, &(test_powerloss_t){ + .run=run_powerloss_cycles, + .cycles=cycles, + .cycle_count=cycle_count}); + } else { + for (size_t p = 0; p < test_powerloss_count; p++) { + // skip non-reentrant tests when powerloss testing + if (test_powerlosses[p].run != run_powerloss_none + && !(case_->flags & TEST_REENTRANT)) { + continue; + } + + cb(data, suite, case_, &test_powerlosses[p]); + } + } + } + + return; + } + + test_seen_t seen = {NULL, 0, 0}; + + for (size_t k = 0; k < case_->permutations; k++) { + // define permutation + test_define_perm(suite, case_, k); + + for (size_t v = 0; v < test_override_define_permutations; v++) { + // define override permutation + test_define_override(v); + + for (size_t g = 0; g < test_geometry_count; g++) { + // define geometry + test_define_geometry(&test_geometries[g]); + test_define_flush(); + + // have we seen this permutation before? + bool was_seen = test_seen_insert(&seen, suite, case_); + if (!(k == 0 && v == 0 && g == 0) && was_seen) { + continue; + } + + if (cycles) { + cb(data, suite, case_, &(test_powerloss_t){ + .run=run_powerloss_cycles, + .cycles=cycles, + .cycle_count=cycle_count}); + } else { + for (size_t p = 0; p < test_powerloss_count; p++) { + // skip non-reentrant tests when powerloss testing + if (test_powerlosses[p].run != run_powerloss_none + && !(case_->flags & TEST_REENTRANT)) { + continue; + } + + cb(data, suite, case_, &test_powerlosses[p]); + } + } + } + } + } + + test_seen_cleanup(&seen); +} + + +// how many permutations are there actually in a test case +struct perm_count_state { + size_t total; + size_t filtered; +}; + +void perm_count( + void *data, + const struct test_suite *suite, + const struct test_case *case_, + const test_powerloss_t *powerloss) { + struct perm_count_state *state = data; + (void)suite; + (void)case_; + (void)powerloss; + + state->total += 1; + + if (case_->filter && !case_->filter()) { + return; + } + + state->filtered += 1; +} + + +// operations we can do +static void summary(void) { + printf("%-23s %7s %7s %7s %11s\n", + "", "flags", "suites", "cases", "perms"); + size_t suites = 0; + size_t cases = 0; + test_flags_t flags = 0; + struct perm_count_state perms = {0, 0}; + + for (size_t t = 0; t < test_id_count; t++) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + test_define_suite(&test_suites[i]); + + for (size_t j = 0; j < test_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (test_ids[t].name && !( + strcmp(test_ids[t].name, + test_suites[i].name) == 0 + || strcmp(test_ids[t].name, + test_suites[i].cases[j].name) == 0)) { + continue; + } + + cases += 1; + case_forperm( + &test_suites[i], + &test_suites[i].cases[j], + test_ids[t].defines, + test_ids[t].define_count, + test_ids[t].cycles, + test_ids[t].cycle_count, + perm_count, + &perms); + } + + suites += 1; + flags |= test_suites[i].flags; + } + } + + char perm_buf[64]; + snprintf(perm_buf, sizeof(perm_buf), "%zu/%zu", perms.filtered, perms.total); + char flag_buf[64]; + snprintf(flag_buf, sizeof(flag_buf), "%s%s", + (flags & TEST_REENTRANT) ? "r" : "", + (!flags) ? "-" : ""); + printf("%-23s %7s %7zu %7zu %11s\n", + "TOTAL", + flag_buf, + suites, + cases, + perm_buf); +} + +static void list_suites(void) { + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + size_t len = strlen(test_suites[i].name); + if (len > name_width) { + name_width = len; + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + + printf("%-*s %7s %7s %11s\n", + name_width, "suite", "flags", "cases", "perms"); + for (size_t t = 0; t < test_id_count; t++) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + test_define_suite(&test_suites[i]); + + size_t cases = 0; + struct perm_count_state perms = {0, 0}; + + for (size_t j = 0; j < test_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (test_ids[t].name && !( + strcmp(test_ids[t].name, + test_suites[i].name) == 0 + || strcmp(test_ids[t].name, + test_suites[i].cases[j].name) == 0)) { + continue; + } + + cases += 1; + case_forperm( + &test_suites[i], + &test_suites[i].cases[j], + test_ids[t].defines, + test_ids[t].define_count, + test_ids[t].cycles, + test_ids[t].cycle_count, + perm_count, + &perms); + } + + // no tests found? + if (!cases) { + continue; + } + + char perm_buf[64]; + snprintf(perm_buf, sizeof(perm_buf), "%zu/%zu", perms.filtered, perms.total); + char flag_buf[64]; + snprintf(flag_buf, sizeof(flag_buf), "%s%s", + (test_suites[i].flags & TEST_REENTRANT) ? "r" : "", + (!test_suites[i].flags) ? "-" : ""); + printf("%-*s %7s %7zu %11s\n", + name_width, + test_suites[i].name, + flag_buf, + cases, + perm_buf); + } + } +} + +static void list_cases(void) { + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + for (size_t j = 0; j < test_suites[i].case_count; j++) { + size_t len = strlen(test_suites[i].cases[j].name); + if (len > name_width) { + name_width = len; + } + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + + printf("%-*s %7s %11s\n", name_width, "case", "flags", "perms"); + for (size_t t = 0; t < test_id_count; t++) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + test_define_suite(&test_suites[i]); + + for (size_t j = 0; j < test_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (test_ids[t].name && !( + strcmp(test_ids[t].name, + test_suites[i].name) == 0 + || strcmp(test_ids[t].name, + test_suites[i].cases[j].name) == 0)) { + continue; + } + + struct perm_count_state perms = {0, 0}; + case_forperm( + &test_suites[i], + &test_suites[i].cases[j], + test_ids[t].defines, + test_ids[t].define_count, + test_ids[t].cycles, + test_ids[t].cycle_count, + perm_count, + &perms); + + char perm_buf[64]; + snprintf(perm_buf, sizeof(perm_buf), "%zu/%zu", perms.filtered, perms.total); + char flag_buf[64]; + snprintf(flag_buf, sizeof(flag_buf), "%s%s", + (test_suites[i].cases[j].flags & TEST_REENTRANT) + ? "r" : "", + (!test_suites[i].cases[j].flags) + ? "-" : ""); + printf("%-*s %7s %11s\n", + name_width, + test_suites[i].cases[j].name, + flag_buf, + perm_buf); + } + } + } +} + +static void list_suite_paths(void) { + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + size_t len = strlen(test_suites[i].name); + if (len > name_width) { + name_width = len; + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + + printf("%-*s %s\n", name_width, "suite", "path"); + for (size_t t = 0; t < test_id_count; t++) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + size_t cases = 0; + + for (size_t j = 0; j < test_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (test_ids[t].name && !( + strcmp(test_ids[t].name, + test_suites[i].name) == 0 + || strcmp(test_ids[t].name, + test_suites[i].cases[j].name) == 0)) { + continue; + } + + cases += 1; + } + + // no tests found? + if (!cases) { + continue; + } + + printf("%-*s %s\n", + name_width, + test_suites[i].name, + test_suites[i].path); + } + } +} + +static void list_case_paths(void) { + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + for (size_t j = 0; j < test_suites[i].case_count; j++) { + size_t len = strlen(test_suites[i].cases[j].name); + if (len > name_width) { + name_width = len; + } + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + + printf("%-*s %s\n", name_width, "case", "path"); + for (size_t t = 0; t < test_id_count; t++) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + for (size_t j = 0; j < test_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (test_ids[t].name && !( + strcmp(test_ids[t].name, + test_suites[i].name) == 0 + || strcmp(test_ids[t].name, + test_suites[i].cases[j].name) == 0)) { + continue; + } + + printf("%-*s %s\n", + name_width, + test_suites[i].cases[j].name, + test_suites[i].cases[j].path); + } + } + } +} + +struct list_defines_define { + const char *name; + intmax_t *values; + size_t value_count; + size_t value_capacity; +}; + +struct list_defines_defines { + struct list_defines_define *defines; + size_t define_count; + size_t define_capacity; +}; + +static void list_defines_add( + struct list_defines_defines *defines, + size_t d) { + const char *name = test_define_name(d); + intmax_t value = TEST_DEFINE(d); + + // define already in defines? + for (size_t i = 0; i < defines->define_count; i++) { + if (strcmp(defines->defines[i].name, name) == 0) { + // value already in values? + for (size_t j = 0; j < defines->defines[i].value_count; j++) { + if (defines->defines[i].values[j] == value) { + return; + } + } + + *(intmax_t*)mappend( + (void**)&defines->defines[i].values, + sizeof(intmax_t), + &defines->defines[i].value_count, + &defines->defines[i].value_capacity) = value; + + return; + } + } + + // new define? + struct list_defines_define *define = mappend( + (void**)&defines->defines, + sizeof(struct list_defines_define), + &defines->define_count, + &defines->define_capacity); + define->name = name; + define->values = malloc(sizeof(intmax_t)); + define->values[0] = value; + define->value_count = 1; + define->value_capacity = 1; +} + +void perm_list_defines( + void *data, + const struct test_suite *suite, + const struct test_case *case_, + const test_powerloss_t *powerloss) { + struct list_defines_defines *defines = data; + (void)suite; + (void)case_; + (void)powerloss; + + // collect defines + for (size_t d = 0; + d < lfs_max(suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + if (d < TEST_IMPLICIT_DEFINE_COUNT + || test_define_ispermutation(d)) { + list_defines_add(defines, d); + } + } +} + +void perm_list_permutation_defines( + void *data, + const struct test_suite *suite, + const struct test_case *case_, + const test_powerloss_t *powerloss) { + struct list_defines_defines *defines = data; + (void)suite; + (void)case_; + (void)powerloss; + + // collect permutation_defines + for (size_t d = 0; + d < lfs_max(suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + if (test_define_ispermutation(d)) { + list_defines_add(defines, d); + } + } +} + +extern const test_geometry_t builtin_geometries[]; + +static void list_defines(void) { + struct list_defines_defines defines = {NULL, 0, 0}; + + // add defines + for (size_t t = 0; t < test_id_count; t++) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + test_define_suite(&test_suites[i]); + + for (size_t j = 0; j < test_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (test_ids[t].name && !( + strcmp(test_ids[t].name, + test_suites[i].name) == 0 + || strcmp(test_ids[t].name, + test_suites[i].cases[j].name) == 0)) { + continue; + } + + case_forperm( + &test_suites[i], + &test_suites[i].cases[j], + test_ids[t].defines, + test_ids[t].define_count, + test_ids[t].cycles, + test_ids[t].cycle_count, + perm_list_defines, + &defines); + } + } + } + + for (size_t i = 0; i < defines.define_count; i++) { + printf("%s=", defines.defines[i].name); + for (size_t j = 0; j < defines.defines[i].value_count; j++) { + printf("%jd", defines.defines[i].values[j]); + if (j != defines.defines[i].value_count-1) { + printf(","); + } + } + printf("\n"); + } + + for (size_t i = 0; i < defines.define_count; i++) { + free(defines.defines[i].values); + } + free(defines.defines); +} + +static void list_permutation_defines(void) { + struct list_defines_defines defines = {NULL, 0, 0}; + + // add permutation defines + for (size_t t = 0; t < test_id_count; t++) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + test_define_suite(&test_suites[i]); + + for (size_t j = 0; j < test_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (test_ids[t].name && !( + strcmp(test_ids[t].name, + test_suites[i].name) == 0 + || strcmp(test_ids[t].name, + test_suites[i].cases[j].name) == 0)) { + continue; + } + + case_forperm( + &test_suites[i], + &test_suites[i].cases[j], + test_ids[t].defines, + test_ids[t].define_count, + test_ids[t].cycles, + test_ids[t].cycle_count, + perm_list_permutation_defines, + &defines); + } + } + } + + for (size_t i = 0; i < defines.define_count; i++) { + printf("%s=", defines.defines[i].name); + for (size_t j = 0; j < defines.defines[i].value_count; j++) { + printf("%jd", defines.defines[i].values[j]); + if (j != defines.defines[i].value_count-1) { + printf(","); + } + } + printf("\n"); + } + + for (size_t i = 0; i < defines.define_count; i++) { + free(defines.defines[i].values); + } + free(defines.defines); +} + +static void list_implicit_defines(void) { + struct list_defines_defines defines = {NULL, 0, 0}; + + // yes we do need to define a suite, this does a bit of bookeeping + // such as setting up the define cache + test_define_suite(&(const struct test_suite){0}); + + // make sure to include builtin geometries here + extern const test_geometry_t builtin_geometries[]; + for (size_t g = 0; builtin_geometries[g].name; g++) { + test_define_geometry(&builtin_geometries[g]); + test_define_flush(); + + // add implicit defines + for (size_t d = 0; d < TEST_IMPLICIT_DEFINE_COUNT; d++) { + list_defines_add(&defines, d); + } + } + + for (size_t i = 0; i < defines.define_count; i++) { + printf("%s=", defines.defines[i].name); + for (size_t j = 0; j < defines.defines[i].value_count; j++) { + printf("%jd", defines.defines[i].values[j]); + if (j != defines.defines[i].value_count-1) { + printf(","); + } + } + printf("\n"); + } + + for (size_t i = 0; i < defines.define_count; i++) { + free(defines.defines[i].values); + } + free(defines.defines); +} + + + +// geometries to test + +const test_geometry_t builtin_geometries[] = { + {"default", {{0}, TEST_CONST(16), TEST_CONST(512), {0}}}, + {"eeprom", {{0}, TEST_CONST(1), TEST_CONST(512), {0}}}, + {"emmc", {{0}, {0}, TEST_CONST(512), {0}}}, + {"nor", {{0}, TEST_CONST(1), TEST_CONST(4096), {0}}}, + {"nand", {{0}, TEST_CONST(4096), TEST_CONST(32768), {0}}}, + {NULL, {{0}, {0}, {0}, {0}}}, +}; + +const test_geometry_t *test_geometries = builtin_geometries; +size_t test_geometry_count = 5; + +static void list_geometries(void) { + // at least size so that names fit + unsigned name_width = 23; + for (size_t g = 0; builtin_geometries[g].name; g++) { + size_t len = strlen(builtin_geometries[g].name); + if (len > name_width) { + name_width = len; + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + + // yes we do need to define a suite, this does a bit of bookeeping + // such as setting up the define cache + test_define_suite(&(const struct test_suite){0}); + + printf("%-*s %7s %7s %7s %7s %11s\n", + name_width, "geometry", "read", "prog", "erase", "count", "size"); + for (size_t g = 0; builtin_geometries[g].name; g++) { + test_define_geometry(&builtin_geometries[g]); + test_define_flush(); + printf("%-*s %7ju %7ju %7ju %7ju %11ju\n", + name_width, + builtin_geometries[g].name, + READ_SIZE, + PROG_SIZE, + ERASE_SIZE, + ERASE_COUNT, + ERASE_SIZE*ERASE_COUNT); + } +} + + +// scenarios to run tests under power-loss + +static void run_powerloss_none( + const lfs_emubd_powercycles_t *cycles, + size_t cycle_count, + const struct test_suite *suite, + const struct test_case *case_) { + (void)cycles; + (void)cycle_count; + (void)suite; + + // create block device and configuration + lfs_emubd_t bd; + + struct lfs_config cfg = { + .context = &bd, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .block_size = BLOCK_SIZE, + .block_count = BLOCK_COUNT, + .block_cycles = BLOCK_CYCLES, + .cache_size = CACHE_SIZE, + .lookahead_size = LOOKAHEAD_SIZE, + #ifdef LFS_MULTIVERSION + .disk_version = DISK_VERSION, + #endif + }; + + struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, + .erase_value = ERASE_VALUE, + .erase_cycles = ERASE_CYCLES, + .badblock_behavior = BADBLOCK_BEHAVIOR, + .disk_path = test_disk_path, + .read_sleep = test_read_sleep, + .prog_sleep = test_prog_sleep, + .erase_sleep = test_erase_sleep, + }; + + int err = lfs_emubd_create(&cfg, &bdcfg); + if (err) { + fprintf(stderr, "error: could not create block device: %d\n", err); + exit(-1); + } + + // run the test + printf("running "); + perm_printid(suite, case_, NULL, 0); + printf("\n"); + + case_->run(&cfg); + + printf("finished "); + perm_printid(suite, case_, NULL, 0); + printf("\n"); + + // cleanup + err = lfs_emubd_destroy(&cfg); + if (err) { + fprintf(stderr, "error: could not destroy block device: %d\n", err); + exit(-1); + } +} + +static void powerloss_longjmp(void *c) { + jmp_buf *powerloss_jmp = c; + longjmp(*powerloss_jmp, 1); +} + +static void run_powerloss_linear( + const lfs_emubd_powercycles_t *cycles, + size_t cycle_count, + const struct test_suite *suite, + const struct test_case *case_) { + (void)cycles; + (void)cycle_count; + (void)suite; + + // create block device and configuration + lfs_emubd_t bd; + jmp_buf powerloss_jmp; + volatile lfs_emubd_powercycles_t i = 1; + + struct lfs_config cfg = { + .context = &bd, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .block_size = BLOCK_SIZE, + .block_count = BLOCK_COUNT, + .block_cycles = BLOCK_CYCLES, + .cache_size = CACHE_SIZE, + .lookahead_size = LOOKAHEAD_SIZE, + #ifdef LFS_MULTIVERSION + .disk_version = DISK_VERSION, + #endif + }; + + struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, + .erase_value = ERASE_VALUE, + .erase_cycles = ERASE_CYCLES, + .badblock_behavior = BADBLOCK_BEHAVIOR, + .disk_path = test_disk_path, + .read_sleep = test_read_sleep, + .prog_sleep = test_prog_sleep, + .erase_sleep = test_erase_sleep, + .power_cycles = i, + .powerloss_behavior = POWERLOSS_BEHAVIOR, + .powerloss_cb = powerloss_longjmp, + .powerloss_data = &powerloss_jmp, + }; + + int err = lfs_emubd_create(&cfg, &bdcfg); + if (err) { + fprintf(stderr, "error: could not create block device: %d\n", err); + exit(-1); + } + + // run the test, increasing power-cycles as power-loss events occur + printf("running "); + perm_printid(suite, case_, NULL, 0); + printf("\n"); + + while (true) { + if (!setjmp(powerloss_jmp)) { + // run the test + case_->run(&cfg); + break; + } + + // power-loss! + printf("powerloss "); + perm_printid(suite, case_, NULL, 0); + printf(":"); + for (lfs_emubd_powercycles_t j = 1; j <= i; j++) { + leb16_print(j); + } + printf("\n"); + + i += 1; + lfs_emubd_setpowercycles(&cfg, i); + } + + printf("finished "); + perm_printid(suite, case_, NULL, 0); + printf("\n"); + + // cleanup + err = lfs_emubd_destroy(&cfg); + if (err) { + fprintf(stderr, "error: could not destroy block device: %d\n", err); + exit(-1); + } +} + +static void run_powerloss_log( + const lfs_emubd_powercycles_t *cycles, + size_t cycle_count, + const struct test_suite *suite, + const struct test_case *case_) { + (void)cycles; + (void)cycle_count; + (void)suite; + + // create block device and configuration + lfs_emubd_t bd; + jmp_buf powerloss_jmp; + volatile lfs_emubd_powercycles_t i = 1; + + struct lfs_config cfg = { + .context = &bd, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .block_size = BLOCK_SIZE, + .block_count = BLOCK_COUNT, + .block_cycles = BLOCK_CYCLES, + .cache_size = CACHE_SIZE, + .lookahead_size = LOOKAHEAD_SIZE, + #ifdef LFS_MULTIVERSION + .disk_version = DISK_VERSION, + #endif + }; + + struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, + .erase_value = ERASE_VALUE, + .erase_cycles = ERASE_CYCLES, + .badblock_behavior = BADBLOCK_BEHAVIOR, + .disk_path = test_disk_path, + .read_sleep = test_read_sleep, + .prog_sleep = test_prog_sleep, + .erase_sleep = test_erase_sleep, + .power_cycles = i, + .powerloss_behavior = POWERLOSS_BEHAVIOR, + .powerloss_cb = powerloss_longjmp, + .powerloss_data = &powerloss_jmp, + }; + + int err = lfs_emubd_create(&cfg, &bdcfg); + if (err) { + fprintf(stderr, "error: could not create block device: %d\n", err); + exit(-1); + } + + // run the test, increasing power-cycles as power-loss events occur + printf("running "); + perm_printid(suite, case_, NULL, 0); + printf("\n"); + + while (true) { + if (!setjmp(powerloss_jmp)) { + // run the test + case_->run(&cfg); + break; + } + + // power-loss! + printf("powerloss "); + perm_printid(suite, case_, NULL, 0); + printf(":"); + for (lfs_emubd_powercycles_t j = 1; j <= i; j *= 2) { + leb16_print(j); + } + printf("\n"); + + i *= 2; + lfs_emubd_setpowercycles(&cfg, i); + } + + printf("finished "); + perm_printid(suite, case_, NULL, 0); + printf("\n"); + + // cleanup + err = lfs_emubd_destroy(&cfg); + if (err) { + fprintf(stderr, "error: could not destroy block device: %d\n", err); + exit(-1); + } +} + +static void run_powerloss_cycles( + const lfs_emubd_powercycles_t *cycles, + size_t cycle_count, + const struct test_suite *suite, + const struct test_case *case_) { + (void)suite; + + // create block device and configuration + lfs_emubd_t bd; + jmp_buf powerloss_jmp; + volatile size_t i = 0; + + struct lfs_config cfg = { + .context = &bd, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .block_size = BLOCK_SIZE, + .block_count = BLOCK_COUNT, + .block_cycles = BLOCK_CYCLES, + .cache_size = CACHE_SIZE, + .lookahead_size = LOOKAHEAD_SIZE, + #ifdef LFS_MULTIVERSION + .disk_version = DISK_VERSION, + #endif + }; + + struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, + .erase_value = ERASE_VALUE, + .erase_cycles = ERASE_CYCLES, + .badblock_behavior = BADBLOCK_BEHAVIOR, + .disk_path = test_disk_path, + .read_sleep = test_read_sleep, + .prog_sleep = test_prog_sleep, + .erase_sleep = test_erase_sleep, + .power_cycles = (i < cycle_count) ? cycles[i] : 0, + .powerloss_behavior = POWERLOSS_BEHAVIOR, + .powerloss_cb = powerloss_longjmp, + .powerloss_data = &powerloss_jmp, + }; + + int err = lfs_emubd_create(&cfg, &bdcfg); + if (err) { + fprintf(stderr, "error: could not create block device: %d\n", err); + exit(-1); + } + + // run the test, increasing power-cycles as power-loss events occur + printf("running "); + perm_printid(suite, case_, NULL, 0); + printf("\n"); + + while (true) { + if (!setjmp(powerloss_jmp)) { + // run the test + case_->run(&cfg); + break; + } + + // power-loss! + assert(i <= cycle_count); + printf("powerloss "); + perm_printid(suite, case_, cycles, i+1); + printf("\n"); + + i += 1; + lfs_emubd_setpowercycles(&cfg, + (i < cycle_count) ? cycles[i] : 0); + } + + printf("finished "); + perm_printid(suite, case_, NULL, 0); + printf("\n"); + + // cleanup + err = lfs_emubd_destroy(&cfg); + if (err) { + fprintf(stderr, "error: could not destroy block device: %d\n", err); + exit(-1); + } +} + +struct powerloss_exhaustive_state { + struct lfs_config *cfg; + + lfs_emubd_t *branches; + size_t branch_count; + size_t branch_capacity; +}; + +struct powerloss_exhaustive_cycles { + lfs_emubd_powercycles_t *cycles; + size_t cycle_count; + size_t cycle_capacity; +}; + +static void powerloss_exhaustive_branch(void *c) { + struct powerloss_exhaustive_state *state = c; + // append to branches + lfs_emubd_t *branch = mappend( + (void**)&state->branches, + sizeof(lfs_emubd_t), + &state->branch_count, + &state->branch_capacity); + if (!branch) { + fprintf(stderr, "error: exhaustive: out of memory\n"); + exit(-1); + } + + // create copy-on-write copy + int err = lfs_emubd_copy(state->cfg, branch); + if (err) { + fprintf(stderr, "error: exhaustive: could not create bd copy\n"); + exit(-1); + } + + // also trigger on next power cycle + lfs_emubd_setpowercycles(state->cfg, 1); +} + +static void run_powerloss_exhaustive_layer( + struct powerloss_exhaustive_cycles *cycles, + const struct test_suite *suite, + const struct test_case *case_, + struct lfs_config *cfg, + struct lfs_emubd_config *bdcfg, + size_t depth) { + (void)suite; + + struct powerloss_exhaustive_state state = { + .cfg = cfg, + .branches = NULL, + .branch_count = 0, + .branch_capacity = 0, + }; + + // run through the test without additional powerlosses, collecting possible + // branches as we do so + lfs_emubd_setpowercycles(state.cfg, depth > 0 ? 1 : 0); + bdcfg->powerloss_data = &state; + + // run the tests + case_->run(cfg); + + // aggressively clean up memory here to try to keep our memory usage low + int err = lfs_emubd_destroy(cfg); + if (err) { + fprintf(stderr, "error: could not destroy block device: %d\n", err); + exit(-1); + } + + // recurse into each branch + for (size_t i = 0; i < state.branch_count; i++) { + // first push and print the branch + lfs_emubd_powercycles_t *cycle = mappend( + (void**)&cycles->cycles, + sizeof(lfs_emubd_powercycles_t), + &cycles->cycle_count, + &cycles->cycle_capacity); + if (!cycle) { + fprintf(stderr, "error: exhaustive: out of memory\n"); + exit(-1); + } + *cycle = i+1; + + printf("powerloss "); + perm_printid(suite, case_, cycles->cycles, cycles->cycle_count); + printf("\n"); + + // now recurse + cfg->context = &state.branches[i]; + run_powerloss_exhaustive_layer(cycles, + suite, case_, + cfg, bdcfg, depth-1); + + // pop the cycle + cycles->cycle_count -= 1; + } + + // clean up memory + free(state.branches); +} + +static void run_powerloss_exhaustive( + const lfs_emubd_powercycles_t *cycles, + size_t cycle_count, + const struct test_suite *suite, + const struct test_case *case_) { + (void)cycles; + (void)suite; + + // create block device and configuration + lfs_emubd_t bd; + + struct lfs_config cfg = { + .context = &bd, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .block_size = BLOCK_SIZE, + .block_count = BLOCK_COUNT, + .block_cycles = BLOCK_CYCLES, + .cache_size = CACHE_SIZE, + .lookahead_size = LOOKAHEAD_SIZE, + #ifdef LFS_MULTIVERSION + .disk_version = DISK_VERSION, + #endif + }; + + struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, + .erase_value = ERASE_VALUE, + .erase_cycles = ERASE_CYCLES, + .badblock_behavior = BADBLOCK_BEHAVIOR, + .disk_path = test_disk_path, + .read_sleep = test_read_sleep, + .prog_sleep = test_prog_sleep, + .erase_sleep = test_erase_sleep, + .powerloss_behavior = POWERLOSS_BEHAVIOR, + .powerloss_cb = powerloss_exhaustive_branch, + .powerloss_data = NULL, + }; + + int err = lfs_emubd_create(&cfg, &bdcfg); + if (err) { + fprintf(stderr, "error: could not create block device: %d\n", err); + exit(-1); + } + + // run the test, increasing power-cycles as power-loss events occur + printf("running "); + perm_printid(suite, case_, NULL, 0); + printf("\n"); + + // recursively exhaust each layer of powerlosses + run_powerloss_exhaustive_layer( + &(struct powerloss_exhaustive_cycles){NULL, 0, 0}, + suite, case_, + &cfg, &bdcfg, cycle_count); + + printf("finished "); + perm_printid(suite, case_, NULL, 0); + printf("\n"); +} + + +const test_powerloss_t builtin_powerlosses[] = { + {"none", run_powerloss_none, NULL, 0}, + {"log", run_powerloss_log, NULL, 0}, + {"linear", run_powerloss_linear, NULL, 0}, + {"exhaustive", run_powerloss_exhaustive, NULL, SIZE_MAX}, + {NULL, NULL, NULL, 0}, +}; + +const char *const builtin_powerlosses_help[] = { + "Run with no power-losses.", + "Run with exponentially-decreasing power-losses.", + "Run with linearly-decreasing power-losses.", + "Run a all permutations of power-losses, this may take a while.", + "Run a all permutations of n power-losses.", + "Run a custom comma-separated set of power-losses.", + "Run a custom leb16-encoded set of power-losses.", +}; + +// default to -Pnone,linear, which provides a good heuristic while still +// running quickly +const test_powerloss_t *test_powerlosses = (const test_powerloss_t[]){ + {"none", run_powerloss_none, NULL, 0}, + {"linear", run_powerloss_linear, NULL, 0}, +}; +size_t test_powerloss_count = 2; + +static void list_powerlosses(void) { + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; builtin_powerlosses[i].name; i++) { + size_t len = strlen(builtin_powerlosses[i].name); + if (len > name_width) { + name_width = len; + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + + printf("%-*s %s\n", name_width, "scenario", "description"); + size_t i = 0; + for (; builtin_powerlosses[i].name; i++) { + printf("%-*s %s\n", + name_width, + builtin_powerlosses[i].name, + builtin_powerlosses_help[i]); + } + + // a couple more options with special parsing + printf("%-*s %s\n", name_width, "1,2,3", builtin_powerlosses_help[i+0]); + printf("%-*s %s\n", name_width, "{1,2,3}", builtin_powerlosses_help[i+1]); + printf("%-*s %s\n", name_width, ":1248g1", builtin_powerlosses_help[i+2]); +} + + +// global test step count +size_t test_step = 0; + +void perm_run( + void *data, + const struct test_suite *suite, + const struct test_case *case_, + const test_powerloss_t *powerloss) { + (void)data; + + // skip this step? + if (!(test_step >= test_step_start + && test_step < test_step_stop + && (test_step-test_step_start) % test_step_step == 0)) { + test_step += 1; + return; + } + test_step += 1; + + // filter? + if (case_->filter && !case_->filter()) { + printf("skipped "); + perm_printid(suite, case_, NULL, 0); + printf("\n"); + return; + } + + powerloss->run( + powerloss->cycles, powerloss->cycle_count, + suite, case_); +} + +static void run(void) { + // ignore disconnected pipes + signal(SIGPIPE, SIG_IGN); + + for (size_t t = 0; t < test_id_count; t++) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + test_define_suite(&test_suites[i]); + + for (size_t j = 0; j < test_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (test_ids[t].name && !( + strcmp(test_ids[t].name, + test_suites[i].name) == 0 + || strcmp(test_ids[t].name, + test_suites[i].cases[j].name) == 0)) { + continue; + } + + case_forperm( + &test_suites[i], + &test_suites[i].cases[j], + test_ids[t].defines, + test_ids[t].define_count, + test_ids[t].cycles, + test_ids[t].cycle_count, + perm_run, + NULL); + } + } + } +} + + + +// option handling +enum opt_flags { + OPT_HELP = 'h', + OPT_SUMMARY = 'Y', + OPT_LIST_SUITES = 'l', + OPT_LIST_CASES = 'L', + OPT_LIST_SUITE_PATHS = 1, + OPT_LIST_CASE_PATHS = 2, + OPT_LIST_DEFINES = 3, + OPT_LIST_PERMUTATION_DEFINES = 4, + OPT_LIST_IMPLICIT_DEFINES = 5, + OPT_LIST_GEOMETRIES = 6, + OPT_LIST_POWERLOSSES = 7, + OPT_DEFINE = 'D', + OPT_GEOMETRY = 'G', + OPT_POWERLOSS = 'P', + OPT_STEP = 's', + OPT_DISK = 'd', + OPT_TRACE = 't', + OPT_TRACE_BACKTRACE = 8, + OPT_TRACE_PERIOD = 9, + OPT_TRACE_FREQ = 10, + OPT_READ_SLEEP = 11, + OPT_PROG_SLEEP = 12, + OPT_ERASE_SLEEP = 13, +}; + +const char *short_opts = "hYlLD:G:P:s:d:t:"; + +const struct option long_opts[] = { + {"help", no_argument, NULL, OPT_HELP}, + {"summary", no_argument, NULL, OPT_SUMMARY}, + {"list-suites", no_argument, NULL, OPT_LIST_SUITES}, + {"list-cases", no_argument, NULL, OPT_LIST_CASES}, + {"list-suite-paths", no_argument, NULL, OPT_LIST_SUITE_PATHS}, + {"list-case-paths", no_argument, NULL, OPT_LIST_CASE_PATHS}, + {"list-defines", no_argument, NULL, OPT_LIST_DEFINES}, + {"list-permutation-defines", + no_argument, NULL, OPT_LIST_PERMUTATION_DEFINES}, + {"list-implicit-defines", + no_argument, NULL, OPT_LIST_IMPLICIT_DEFINES}, + {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES}, + {"list-powerlosses", no_argument, NULL, OPT_LIST_POWERLOSSES}, + {"define", required_argument, NULL, OPT_DEFINE}, + {"geometry", required_argument, NULL, OPT_GEOMETRY}, + {"powerloss", required_argument, NULL, OPT_POWERLOSS}, + {"step", required_argument, NULL, OPT_STEP}, + {"disk", required_argument, NULL, OPT_DISK}, + {"trace", required_argument, NULL, OPT_TRACE}, + {"trace-backtrace", no_argument, NULL, OPT_TRACE_BACKTRACE}, + {"trace-period", required_argument, NULL, OPT_TRACE_PERIOD}, + {"trace-freq", required_argument, NULL, OPT_TRACE_FREQ}, + {"read-sleep", required_argument, NULL, OPT_READ_SLEEP}, + {"prog-sleep", required_argument, NULL, OPT_PROG_SLEEP}, + {"erase-sleep", required_argument, NULL, OPT_ERASE_SLEEP}, + {NULL, 0, NULL, 0}, +}; + +const char *const help_text[] = { + "Show this help message.", + "Show quick summary.", + "List test suites.", + "List test cases.", + "List the path for each test suite.", + "List the path and line number for each test case.", + "List all defines in this test-runner.", + "List explicit defines in this test-runner.", + "List implicit defines in this test-runner.", + "List the available disk geometries.", + "List the available power-loss scenarios.", + "Override a test define.", + "Comma-separated list of disk geometries to test.", + "Comma-separated list of power-loss scenarios to test.", + "Comma-separated range of test permutations to run (start,stop,step).", + "Direct block device operations to this file.", + "Direct trace output to this file.", + "Include a backtrace with every trace statement.", + "Sample trace output at this period in cycles.", + "Sample trace output at this frequency in hz.", + "Artificial read delay in seconds.", + "Artificial prog delay in seconds.", + "Artificial erase delay in seconds.", +}; + +int main(int argc, char **argv) { + void (*op)(void) = run; + + size_t test_override_capacity = 0; + size_t test_geometry_capacity = 0; + size_t test_powerloss_capacity = 0; + size_t test_id_capacity = 0; + + // parse options + while (true) { + int c = getopt_long(argc, argv, short_opts, long_opts, NULL); + switch (c) { + // generate help message + case OPT_HELP: { + printf("usage: %s [options] [test_id]\n", argv[0]); + printf("\n"); + + printf("options:\n"); + size_t i = 0; + while (long_opts[i].name) { + size_t indent; + if (long_opts[i].has_arg == no_argument) { + if (long_opts[i].val >= '0' && long_opts[i].val < 'z') { + indent = printf(" -%c, --%s ", + long_opts[i].val, + long_opts[i].name); + } else { + indent = printf(" --%s ", + long_opts[i].name); + } + } else { + if (long_opts[i].val >= '0' && long_opts[i].val < 'z') { + indent = printf(" -%c %s, --%s %s ", + long_opts[i].val, + long_opts[i].name, + long_opts[i].name, + long_opts[i].name); + } else { + indent = printf(" --%s %s ", + long_opts[i].name, + long_opts[i].name); + } + } + + // a quick, hacky, byte-level method for text wrapping + size_t len = strlen(help_text[i]); + size_t j = 0; + if (indent < 24) { + printf("%*s %.80s\n", + (int)(24-1-indent), + "", + &help_text[i][j]); + j += 80; + } else { + printf("\n"); + } + + while (j < len) { + printf("%24s%.80s\n", "", &help_text[i][j]); + j += 80; + } + + i += 1; + } + + printf("\n"); + exit(0); + } + // summary/list flags + case OPT_SUMMARY: + op = summary; + break; + case OPT_LIST_SUITES: + op = list_suites; + break; + case OPT_LIST_CASES: + op = list_cases; + break; + case OPT_LIST_SUITE_PATHS: + op = list_suite_paths; + break; + case OPT_LIST_CASE_PATHS: + op = list_case_paths; + break; + case OPT_LIST_DEFINES: + op = list_defines; + break; + case OPT_LIST_PERMUTATION_DEFINES: + op = list_permutation_defines; + break; + case OPT_LIST_IMPLICIT_DEFINES: + op = list_implicit_defines; + break; + case OPT_LIST_GEOMETRIES: + op = list_geometries; + break; + case OPT_LIST_POWERLOSSES: + op = list_powerlosses; + break; + // configuration + case OPT_DEFINE: { + // allocate space + test_override_t *override = mappend( + (void**)&test_overrides, + sizeof(test_override_t), + &test_override_count, + &test_override_capacity); + + // parse into string key/intmax_t value, cannibalizing the + // arg in the process + char *sep = strchr(optarg, '='); + char *parsed = NULL; + if (!sep) { + goto invalid_define; + } + *sep = '\0'; + override->name = optarg; + optarg = sep+1; + + // parse comma-separated permutations + { + override->defines = NULL; + override->permutations = 0; + size_t override_capacity = 0; + while (true) { + optarg += strspn(optarg, " "); + + if (strncmp(optarg, "range", strlen("range")) == 0) { + // range of values + optarg += strlen("range"); + optarg += strspn(optarg, " "); + if (*optarg != '(') { + goto invalid_define; + } + optarg += 1; + + intmax_t start = strtoumax(optarg, &parsed, 0); + intmax_t stop = -1; + intmax_t step = 1; + // allow empty string for start=0 + if (parsed == optarg) { + start = 0; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != ')') { + goto invalid_define; + } + + if (*optarg == ',') { + optarg += 1; + stop = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=end + if (parsed == optarg) { + stop = -1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != ')') { + goto invalid_define; + } + + if (*optarg == ',') { + optarg += 1; + step = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=1 + if (parsed == optarg) { + step = 1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ')') { + goto invalid_define; + } + } + } else { + // single value = stop only + stop = start; + start = 0; + } + + if (*optarg != ')') { + goto invalid_define; + } + optarg += 1; + + // calculate the range of values + assert(step != 0); + for (intmax_t i = start; + (step < 0) + ? i > stop + : (uintmax_t)i < (uintmax_t)stop; + i += step) { + *(intmax_t*)mappend( + (void**)&override->defines, + sizeof(intmax_t), + &override->permutations, + &override_capacity) = i; + } + } else if (*optarg != '\0') { + // single value + intmax_t define = strtoimax(optarg, &parsed, 0); + if (parsed == optarg) { + goto invalid_define; + } + optarg = parsed + strspn(parsed, " "); + *(intmax_t*)mappend( + (void**)&override->defines, + sizeof(intmax_t), + &override->permutations, + &override_capacity) = define; + } else { + break; + } + + if (*optarg == ',') { + optarg += 1; + } + } + } + assert(override->permutations > 0); + break; + +invalid_define: + fprintf(stderr, "error: invalid define: %s\n", optarg); + exit(-1); + } + case OPT_GEOMETRY: { + // reset our geometry scenarios + if (test_geometry_capacity > 0) { + free((test_geometry_t*)test_geometries); + } + test_geometries = NULL; + test_geometry_count = 0; + test_geometry_capacity = 0; + + // parse the comma separated list of disk geometries + while (*optarg) { + // allocate space + test_geometry_t *geometry = mappend( + (void**)&test_geometries, + sizeof(test_geometry_t), + &test_geometry_count, + &test_geometry_capacity); + + // parse the disk geometry + optarg += strspn(optarg, " "); + + // named disk geometry + size_t len = strcspn(optarg, " ,"); + for (size_t i = 0; builtin_geometries[i].name; i++) { + if (len == strlen(builtin_geometries[i].name) + && memcmp(optarg, + builtin_geometries[i].name, + len) == 0) { + *geometry = builtin_geometries[i]; + optarg += len; + goto geometry_next; + } + } + + // comma-separated read/prog/erase/count + if (*optarg == '{') { + lfs_size_t sizes[4]; + size_t count = 0; + + char *s = optarg + 1; + while (count < 4) { + char *parsed = NULL; + sizes[count] = strtoumax(s, &parsed, 0); + count += 1; + + s = parsed + strspn(parsed, " "); + if (*s == ',') { + s += 1; + continue; + } else if (*s == '}') { + s += 1; + break; + } else { + goto geometry_unknown; + } + } + + // allow implicit r=p and p=e for common geometries + memset(geometry, 0, sizeof(test_geometry_t)); + if (count >= 3) { + geometry->defines[READ_SIZE_i] + = TEST_LIT(sizes[0]); + geometry->defines[PROG_SIZE_i] + = TEST_LIT(sizes[1]); + geometry->defines[ERASE_SIZE_i] + = TEST_LIT(sizes[2]); + } else if (count >= 2) { + geometry->defines[PROG_SIZE_i] + = TEST_LIT(sizes[0]); + geometry->defines[ERASE_SIZE_i] + = TEST_LIT(sizes[1]); + } else { + geometry->defines[ERASE_SIZE_i] + = TEST_LIT(sizes[0]); + } + if (count >= 4) { + geometry->defines[ERASE_COUNT_i] + = TEST_LIT(sizes[3]); + } + optarg = s; + goto geometry_next; + } + + // leb16-encoded read/prog/erase/count + if (*optarg == ':') { + lfs_size_t sizes[4]; + size_t count = 0; + + char *s = optarg + 1; + while (true) { + char *parsed = NULL; + uintmax_t x = leb16_parse(s, &parsed); + if (parsed == s || count >= 4) { + break; + } + + sizes[count] = x; + count += 1; + s = parsed; + } + + // allow implicit r=p and p=e for common geometries + memset(geometry, 0, sizeof(test_geometry_t)); + if (count >= 3) { + geometry->defines[READ_SIZE_i] + = TEST_LIT(sizes[0]); + geometry->defines[PROG_SIZE_i] + = TEST_LIT(sizes[1]); + geometry->defines[ERASE_SIZE_i] + = TEST_LIT(sizes[2]); + } else if (count >= 2) { + geometry->defines[PROG_SIZE_i] + = TEST_LIT(sizes[0]); + geometry->defines[ERASE_SIZE_i] + = TEST_LIT(sizes[1]); + } else { + geometry->defines[ERASE_SIZE_i] + = TEST_LIT(sizes[0]); + } + if (count >= 4) { + geometry->defines[ERASE_COUNT_i] + = TEST_LIT(sizes[3]); + } + optarg = s; + goto geometry_next; + } + +geometry_unknown: + // unknown scenario? + fprintf(stderr, "error: unknown disk geometry: %s\n", + optarg); + exit(-1); + +geometry_next: + optarg += strspn(optarg, " "); + if (*optarg == ',') { + optarg += 1; + } else if (*optarg == '\0') { + break; + } else { + goto geometry_unknown; + } + } + break; + } + case OPT_POWERLOSS: { + // reset our powerloss scenarios + if (test_powerloss_capacity > 0) { + free((test_powerloss_t*)test_powerlosses); + } + test_powerlosses = NULL; + test_powerloss_count = 0; + test_powerloss_capacity = 0; + + // parse the comma separated list of power-loss scenarios + while (*optarg) { + // allocate space + test_powerloss_t *powerloss = mappend( + (void**)&test_powerlosses, + sizeof(test_powerloss_t), + &test_powerloss_count, + &test_powerloss_capacity); + + // parse the power-loss scenario + optarg += strspn(optarg, " "); + + // named power-loss scenario + size_t len = strcspn(optarg, " ,"); + for (size_t i = 0; builtin_powerlosses[i].name; i++) { + if (len == strlen(builtin_powerlosses[i].name) + && memcmp(optarg, + builtin_powerlosses[i].name, + len) == 0) { + *powerloss = builtin_powerlosses[i]; + optarg += len; + goto powerloss_next; + } + } + + // comma-separated permutation + if (*optarg == '{') { + lfs_emubd_powercycles_t *cycles = NULL; + size_t cycle_count = 0; + size_t cycle_capacity = 0; + + char *s = optarg + 1; + while (true) { + char *parsed = NULL; + *(lfs_emubd_powercycles_t*)mappend( + (void**)&cycles, + sizeof(lfs_emubd_powercycles_t), + &cycle_count, + &cycle_capacity) + = strtoumax(s, &parsed, 0); + + s = parsed + strspn(parsed, " "); + if (*s == ',') { + s += 1; + continue; + } else if (*s == '}') { + s += 1; + break; + } else { + goto powerloss_unknown; + } + } + + *powerloss = (test_powerloss_t){ + .run = run_powerloss_cycles, + .cycles = cycles, + .cycle_count = cycle_count, + }; + optarg = s; + goto powerloss_next; + } + + // leb16-encoded permutation + if (*optarg == ':') { + lfs_emubd_powercycles_t *cycles = NULL; + size_t cycle_count = 0; + size_t cycle_capacity = 0; + + char *s = optarg + 1; + while (true) { + char *parsed = NULL; + uintmax_t x = leb16_parse(s, &parsed); + if (parsed == s) { + break; + } + + *(lfs_emubd_powercycles_t*)mappend( + (void**)&cycles, + sizeof(lfs_emubd_powercycles_t), + &cycle_count, + &cycle_capacity) = x; + s = parsed; + } + + *powerloss = (test_powerloss_t){ + .run = run_powerloss_cycles, + .cycles = cycles, + .cycle_count = cycle_count, + }; + optarg = s; + goto powerloss_next; + } + + // exhaustive permutations + { + char *parsed = NULL; + size_t count = strtoumax(optarg, &parsed, 0); + if (parsed == optarg) { + goto powerloss_unknown; + } + *powerloss = (test_powerloss_t){ + .run = run_powerloss_exhaustive, + .cycles = NULL, + .cycle_count = count, + }; + optarg = (char*)parsed; + goto powerloss_next; + } + +powerloss_unknown: + // unknown scenario? + fprintf(stderr, "error: unknown power-loss scenario: %s\n", + optarg); + exit(-1); + +powerloss_next: + optarg += strspn(optarg, " "); + if (*optarg == ',') { + optarg += 1; + } else if (*optarg == '\0') { + break; + } else { + goto powerloss_unknown; + } + } + break; + } + case OPT_STEP: { + char *parsed = NULL; + test_step_start = strtoumax(optarg, &parsed, 0); + test_step_stop = -1; + test_step_step = 1; + // allow empty string for start=0 + if (parsed == optarg) { + test_step_start = 0; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != '\0') { + goto step_unknown; + } + + if (*optarg == ',') { + optarg += 1; + test_step_stop = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=end + if (parsed == optarg) { + test_step_stop = -1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != '\0') { + goto step_unknown; + } + + if (*optarg == ',') { + optarg += 1; + test_step_step = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=1 + if (parsed == optarg) { + test_step_step = 1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != '\0') { + goto step_unknown; + } + } + } else { + // single value = stop only + test_step_stop = test_step_start; + test_step_start = 0; + } + + break; +step_unknown: + fprintf(stderr, "error: invalid step: %s\n", optarg); + exit(-1); + } + case OPT_DISK: + test_disk_path = optarg; + break; + case OPT_TRACE: + test_trace_path = optarg; + break; + case OPT_TRACE_BACKTRACE: + test_trace_backtrace = true; + break; + case OPT_TRACE_PERIOD: { + char *parsed = NULL; + test_trace_period = strtoumax(optarg, &parsed, 0); + if (parsed == optarg) { + fprintf(stderr, "error: invalid trace-period: %s\n", optarg); + exit(-1); + } + break; + } + case OPT_TRACE_FREQ: { + char *parsed = NULL; + test_trace_freq = strtoumax(optarg, &parsed, 0); + if (parsed == optarg) { + fprintf(stderr, "error: invalid trace-freq: %s\n", optarg); + exit(-1); + } + break; + } + case OPT_READ_SLEEP: { + char *parsed = NULL; + double read_sleep = strtod(optarg, &parsed); + if (parsed == optarg) { + fprintf(stderr, "error: invalid read-sleep: %s\n", optarg); + exit(-1); + } + test_read_sleep = read_sleep*1.0e9; + break; + } + case OPT_PROG_SLEEP: { + char *parsed = NULL; + double prog_sleep = strtod(optarg, &parsed); + if (parsed == optarg) { + fprintf(stderr, "error: invalid prog-sleep: %s\n", optarg); + exit(-1); + } + test_prog_sleep = prog_sleep*1.0e9; + break; + } + case OPT_ERASE_SLEEP: { + char *parsed = NULL; + double erase_sleep = strtod(optarg, &parsed); + if (parsed == optarg) { + fprintf(stderr, "error: invalid erase-sleep: %s\n", optarg); + exit(-1); + } + test_erase_sleep = erase_sleep*1.0e9; + break; + } + // done parsing + case -1: + goto getopt_done; + // unknown arg, getopt prints a message for us + default: + exit(-1); + } + } +getopt_done: ; + + if (argc > optind) { + // reset our test identifier list + test_ids = NULL; + test_id_count = 0; + test_id_capacity = 0; + } + + // parse test identifier, if any, cannibalizing the arg in the process + for (; argc > optind; optind++) { + test_define_t *defines = NULL; + size_t define_count = 0; + lfs_emubd_powercycles_t *cycles = NULL; + size_t cycle_count = 0; + + // parse name, can be suite or case + char *name = argv[optind]; + char *defines_ = strchr(name, ':'); + if (defines_) { + *defines_ = '\0'; + defines_ += 1; + } + + // remove optional path and .toml suffix + char *slash = strrchr(name, '/'); + if (slash) { + name = slash+1; + } + + size_t name_len = strlen(name); + if (name_len > 5 && strcmp(&name[name_len-5], ".toml") == 0) { + name[name_len-5] = '\0'; + } + + if (defines_) { + // parse defines + char *cycles_ = strchr(defines_, ':'); + if (cycles_) { + *cycles_ = '\0'; + cycles_ += 1; + } + + while (true) { + char *parsed; + size_t d = leb16_parse(defines_, &parsed); + intmax_t v = leb16_parse(parsed, &parsed); + if (parsed == defines_) { + break; + } + defines_ = parsed; + + if (d >= define_count) { + // align to power of two to avoid any superlinear growth + size_t ncount = 1 << lfs_npw2(d+1); + defines = realloc(defines, + ncount*sizeof(test_define_t)); + memset(defines+define_count, 0, + (ncount-define_count)*sizeof(test_define_t)); + define_count = ncount; + } + defines[d] = TEST_LIT(v); + } + + if (cycles_) { + // parse power cycles + size_t cycle_capacity = 0; + while (*cycles_ != '\0') { + char *parsed = NULL; + *(lfs_emubd_powercycles_t*)mappend( + (void**)&cycles, + sizeof(lfs_emubd_powercycles_t), + &cycle_count, + &cycle_capacity) + = leb16_parse(cycles_, &parsed); + if (parsed == cycles_) { + fprintf(stderr, "error: " + "could not parse test cycles: %s\n", + cycles_); + exit(-1); + } + cycles_ = parsed; + } + } + } + + // append to identifier list + *(test_id_t*)mappend( + (void**)&test_ids, + sizeof(test_id_t), + &test_id_count, + &test_id_capacity) = (test_id_t){ + .name = name, + .defines = defines, + .define_count = define_count, + .cycles = cycles, + .cycle_count = cycle_count, + }; + } + + // do the thing + op(); + + // cleanup (need to be done for valgrind testing) + test_define_cleanup(); + if (test_overrides) { + for (size_t i = 0; i < test_override_count; i++) { + free((void*)test_overrides[i].defines); + } + free((void*)test_overrides); + } + if (test_geometry_capacity) { + free((void*)test_geometries); + } + if (test_powerloss_capacity) { + for (size_t i = 0; i < test_powerloss_count; i++) { + free((void*)test_powerlosses[i].cycles); + } + free((void*)test_powerlosses); + } + if (test_id_capacity) { + for (size_t i = 0; i < test_id_count; i++) { + free((void*)test_ids[i].defines); + free((void*)test_ids[i].cycles); + } + free((void*)test_ids); + } +} diff --git a/components/fs/littlefs/littlefs/runners/test_runner.h b/components/fs/littlefs/littlefs/runners/test_runner.h new file mode 100644 index 000000000..4be72e42f --- /dev/null +++ b/components/fs/littlefs/littlefs/runners/test_runner.h @@ -0,0 +1,133 @@ +/* + * Runner for littlefs tests + * + * Copyright (c) 2022, The littlefs authors. + * SPDX-License-Identifier: BSD-3-Clause + */ +#ifndef TEST_RUNNER_H +#define TEST_RUNNER_H + + +// override LFS_TRACE +void test_trace(const char *fmt, ...); + +#define LFS_TRACE_(fmt, ...) \ + test_trace("%s:%d:trace: " fmt "%s\n", \ + __FILE__, \ + __LINE__, \ + __VA_ARGS__) +#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") +#define LFS_EMUBD_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") + + +// note these are indirectly included in any generated files +#include "bd/lfs_emubd.h" +#include + +// give source a chance to define feature macros +#undef _FEATURES_H +#undef _STDIO_H + + +// generated test configurations +struct lfs_config; + +enum test_flags { + TEST_REENTRANT = 0x1, +}; +typedef uint8_t test_flags_t; + +typedef struct test_define { + intmax_t (*cb)(void *data); + void *data; +} test_define_t; + +struct test_case { + const char *name; + const char *path; + test_flags_t flags; + size_t permutations; + + const test_define_t *defines; + + bool (*filter)(void); + void (*run)(struct lfs_config *cfg); +}; + +struct test_suite { + const char *name; + const char *path; + test_flags_t flags; + + const char *const *define_names; + size_t define_count; + + const struct test_case *cases; + size_t case_count; +}; + + +// deterministic prng for pseudo-randomness in testes +uint32_t test_prng(uint32_t *state); + +#define TEST_PRNG(state) test_prng(state) + + +// access generated test defines +intmax_t test_define(size_t define); + +#define TEST_DEFINE(i) test_define(i) + +// a few preconfigured defines that control how tests run + +#define READ_SIZE_i 0 +#define PROG_SIZE_i 1 +#define ERASE_SIZE_i 2 +#define ERASE_COUNT_i 3 +#define BLOCK_SIZE_i 4 +#define BLOCK_COUNT_i 5 +#define CACHE_SIZE_i 6 +#define LOOKAHEAD_SIZE_i 7 +#define BLOCK_CYCLES_i 8 +#define ERASE_VALUE_i 9 +#define ERASE_CYCLES_i 10 +#define BADBLOCK_BEHAVIOR_i 11 +#define POWERLOSS_BEHAVIOR_i 12 +#define DISK_VERSION_i 13 + +#define READ_SIZE TEST_DEFINE(READ_SIZE_i) +#define PROG_SIZE TEST_DEFINE(PROG_SIZE_i) +#define ERASE_SIZE TEST_DEFINE(ERASE_SIZE_i) +#define ERASE_COUNT TEST_DEFINE(ERASE_COUNT_i) +#define BLOCK_SIZE TEST_DEFINE(BLOCK_SIZE_i) +#define BLOCK_COUNT TEST_DEFINE(BLOCK_COUNT_i) +#define CACHE_SIZE TEST_DEFINE(CACHE_SIZE_i) +#define LOOKAHEAD_SIZE TEST_DEFINE(LOOKAHEAD_SIZE_i) +#define BLOCK_CYCLES TEST_DEFINE(BLOCK_CYCLES_i) +#define ERASE_VALUE TEST_DEFINE(ERASE_VALUE_i) +#define ERASE_CYCLES TEST_DEFINE(ERASE_CYCLES_i) +#define BADBLOCK_BEHAVIOR TEST_DEFINE(BADBLOCK_BEHAVIOR_i) +#define POWERLOSS_BEHAVIOR TEST_DEFINE(POWERLOSS_BEHAVIOR_i) +#define DISK_VERSION TEST_DEFINE(DISK_VERSION_i) + +#define TEST_IMPLICIT_DEFINES \ + TEST_DEF(READ_SIZE, PROG_SIZE) \ + TEST_DEF(PROG_SIZE, ERASE_SIZE) \ + TEST_DEF(ERASE_SIZE, 0) \ + TEST_DEF(ERASE_COUNT, (1024*1024)/ERASE_SIZE) \ + TEST_DEF(BLOCK_SIZE, ERASE_SIZE) \ + TEST_DEF(BLOCK_COUNT, ERASE_COUNT/lfs_max(BLOCK_SIZE/ERASE_SIZE,1)) \ + TEST_DEF(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ + TEST_DEF(LOOKAHEAD_SIZE, 16) \ + TEST_DEF(BLOCK_CYCLES, -1) \ + TEST_DEF(ERASE_VALUE, 0xff) \ + TEST_DEF(ERASE_CYCLES, 0) \ + TEST_DEF(BADBLOCK_BEHAVIOR, LFS_EMUBD_BADBLOCK_PROGERROR) \ + TEST_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP) \ + TEST_DEF(DISK_VERSION, 0) + +#define TEST_GEOMETRY_DEFINE_COUNT 4 +#define TEST_IMPLICIT_DEFINE_COUNT 14 + + +#endif diff --git a/components/fs/littlefs/littlefs/scripts/bench.py b/components/fs/littlefs/littlefs/scripts/bench.py new file mode 100644 index 000000000..f22841eac --- /dev/null +++ b/components/fs/littlefs/littlefs/scripts/bench.py @@ -0,0 +1,1430 @@ +#!/usr/bin/env python3 +# +# Script to compile and runs benches. +# +# Example: +# ./scripts/bench.py runners/bench_runner -b +# +# Copyright (c) 2022, The littlefs authors. +# SPDX-License-Identifier: BSD-3-Clause +# + +import collections as co +import csv +import errno +import glob +import itertools as it +import math as m +import os +import pty +import re +import shlex +import shutil +import signal +import subprocess as sp +import threading as th +import time +import toml + + +RUNNER_PATH = './runners/bench_runner' +HEADER_PATH = 'runners/bench_runner.h' + +GDB_PATH = ['gdb'] +VALGRIND_PATH = ['valgrind'] +PERF_SCRIPT = ['./scripts/perf.py'] + + +def openio(path, mode='r', buffering=-1): + # allow '-' for stdin/stdout + if path == '-': + if mode == 'r': + return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering) + else: + return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering) + else: + return open(path, mode, buffering) + +class BenchCase: + # create a BenchCase object from a config + def __init__(self, config, args={}): + self.name = config.pop('name') + self.path = config.pop('path') + self.suite = config.pop('suite') + self.lineno = config.pop('lineno', None) + self.if_ = config.pop('if', None) + if isinstance(self.if_, bool): + self.if_ = 'true' if self.if_ else 'false' + self.code = config.pop('code') + self.code_lineno = config.pop('code_lineno', None) + self.in_ = config.pop('in', + config.pop('suite_in', None)) + + # figure out defines and build possible permutations + self.defines = set() + self.permutations = [] + + # defines can be a dict or a list or dicts + suite_defines = config.pop('suite_defines', {}) + if not isinstance(suite_defines, list): + suite_defines = [suite_defines] + defines = config.pop('defines', {}) + if not isinstance(defines, list): + defines = [defines] + + def csplit(v): + # split commas but only outside of parens + parens = 0 + i_ = 0 + for i in range(len(v)): + if v[i] == ',' and parens == 0: + yield v[i_:i] + i_ = i+1 + elif v[i] in '([{': + parens += 1 + elif v[i] in '}])': + parens -= 1 + if v[i_:].strip(): + yield v[i_:] + + def parse_define(v): + # a define entry can be a list + if isinstance(v, list): + for v_ in v: + yield from parse_define(v_) + # or a string + elif isinstance(v, str): + # which can be comma-separated values, with optional + # range statements. This matches the runtime define parser in + # the runner itself. + for v_ in csplit(v): + m = re.search(r'\brange\b\s*\(' + '(?P[^,\s]*)' + '\s*(?:,\s*(?P[^,\s]*)' + '\s*(?:,\s*(?P[^,\s]*)\s*)?)?\)', + v_) + if m: + start = (int(m.group('start'), 0) + if m.group('start') else 0) + stop = (int(m.group('stop'), 0) + if m.group('stop') else None) + step = (int(m.group('step'), 0) + if m.group('step') else 1) + if m.lastindex <= 1: + start, stop = 0, start + for x in range(start, stop, step): + yield from parse_define('%s(%d)%s' % ( + v_[:m.start()], x, v_[m.end():])) + else: + yield v_ + # or a literal value + elif isinstance(v, bool): + yield 'true' if v else 'false' + else: + yield v + + # build possible permutations + for suite_defines_ in suite_defines: + self.defines |= suite_defines_.keys() + for defines_ in defines: + self.defines |= defines_.keys() + self.permutations.extend(dict(perm) for perm in it.product(*( + [(k, v) for v in parse_define(vs)] + for k, vs in sorted((suite_defines_ | defines_).items())))) + + for k in config.keys(): + print('%swarning:%s in %s, found unused key %r' % ( + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + self.name, + k), + file=sys.stderr) + + +class BenchSuite: + # create a BenchSuite object from a toml file + def __init__(self, path, args={}): + self.path = path + self.name = os.path.basename(path) + if self.name.endswith('.toml'): + self.name = self.name[:-len('.toml')] + + # load toml file and parse bench cases + with open(self.path) as f: + # load benches + config = toml.load(f) + + # find line numbers + f.seek(0) + case_linenos = [] + code_linenos = [] + for i, line in enumerate(f): + match = re.match( + '(?P\[\s*cases\s*\.\s*(?P\w+)\s*\])' + '|' '(?Pcode\s*=)', + line) + if match and match.group('case'): + case_linenos.append((i+1, match.group('name'))) + elif match and match.group('code'): + code_linenos.append(i+2) + + # sort in case toml parsing did not retain order + case_linenos.sort() + + cases = config.pop('cases') + for (lineno, name), (nlineno, _) in it.zip_longest( + case_linenos, case_linenos[1:], + fillvalue=(float('inf'), None)): + code_lineno = min( + (l for l in code_linenos if l >= lineno and l < nlineno), + default=None) + cases[name]['lineno'] = lineno + cases[name]['code_lineno'] = code_lineno + + self.if_ = config.pop('if', None) + if isinstance(self.if_, bool): + self.if_ = 'true' if self.if_ else 'false' + + self.code = config.pop('code', None) + self.code_lineno = min( + (l for l in code_linenos + if not case_linenos or l < case_linenos[0][0]), + default=None) + + # a couple of these we just forward to all cases + defines = config.pop('defines', {}) + in_ = config.pop('in', None) + + self.cases = [] + for name, case in sorted(cases.items(), + key=lambda c: c[1].get('lineno')): + self.cases.append(BenchCase(config={ + 'name': name, + 'path': path + (':%d' % case['lineno'] + if 'lineno' in case else ''), + 'suite': self.name, + 'suite_defines': defines, + 'suite_in': in_, + **case}, + args=args)) + + # combine per-case defines + self.defines = set.union(*( + set(case.defines) for case in self.cases)) + + for k in config.keys(): + print('%swarning:%s in %s, found unused key %r' % ( + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + self.name, + k), + file=sys.stderr) + + + +def compile(bench_paths, **args): + # find .toml files + paths = [] + for path in bench_paths: + if os.path.isdir(path): + path = path + '/*.toml' + + for path in glob.glob(path): + paths.append(path) + + if not paths: + print('no bench suites found in %r?' % bench_paths) + sys.exit(-1) + + # load the suites + suites = [BenchSuite(path, args) for path in paths] + suites.sort(key=lambda s: s.name) + + # check for name conflicts, these will cause ambiguity problems later + # when running benches + seen = {} + for suite in suites: + if suite.name in seen: + print('%swarning:%s conflicting suite %r, %s and %s' % ( + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + suite.name, + suite.path, + seen[suite.name].path), + file=sys.stderr) + seen[suite.name] = suite + + for case in suite.cases: + # only allow conflicts if a case and its suite share a name + if case.name in seen and not ( + isinstance(seen[case.name], BenchSuite) + and seen[case.name].cases == [case]): + print('%swarning:%s conflicting case %r, %s and %s' % ( + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + case.name, + case.path, + seen[case.name].path), + file=sys.stderr) + seen[case.name] = case + + # we can only compile one bench suite at a time + if not args.get('source'): + if len(suites) > 1: + print('more than one bench suite for compilation? (%r)' % bench_paths) + sys.exit(-1) + + suite = suites[0] + + # write generated bench source + if 'output' in args: + with openio(args['output'], 'w') as f: + _write = f.write + def write(s): + f.lineno += s.count('\n') + _write(s) + def writeln(s=''): + f.lineno += s.count('\n') + 1 + _write(s) + _write('\n') + f.lineno = 1 + f.write = write + f.writeln = writeln + + f.writeln("// Generated by %s:" % sys.argv[0]) + f.writeln("//") + f.writeln("// %s" % ' '.join(sys.argv)) + f.writeln("//") + f.writeln() + + # include bench_runner.h in every generated file + f.writeln("#include \"%s\"" % args['include']) + f.writeln() + + # write out generated functions, this can end up in different + # files depending on the "in" attribute + # + # note it's up to the specific generated file to declare + # the bench defines + def write_case_functions(f, suite, case): + # create case define functions + if case.defines: + # deduplicate defines by value to try to reduce the + # number of functions we generate + define_cbs = {} + for i, defines in enumerate(case.permutations): + for k, v in sorted(defines.items()): + if v not in define_cbs: + name = ('__bench__%s__%s__%d' + % (case.name, k, i)) + define_cbs[v] = name + f.writeln('intmax_t %s(' + '__attribute__((unused)) ' + 'void *data) {' % name) + f.writeln(4*' '+'return %s;' % v) + f.writeln('}') + f.writeln() + f.writeln('const bench_define_t ' + '__bench__%s__defines[][' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d] = {' + % (case.name, len(suite.defines))) + for defines in case.permutations: + f.writeln(4*' '+'{') + for k, v in sorted(defines.items()): + f.writeln(8*' '+'[%-24s] = {%s, NULL},' % ( + k+'_i', define_cbs[v])) + f.writeln(4*' '+'},') + f.writeln('};') + f.writeln() + + # create case filter function + if suite.if_ is not None or case.if_ is not None: + f.writeln('bool __bench__%s__filter(void) {' + % (case.name)) + f.writeln(4*' '+'return %s;' + % ' && '.join('(%s)' % if_ + for if_ in [suite.if_, case.if_] + if if_ is not None)) + f.writeln('}') + f.writeln() + + # create case run function + f.writeln('void __bench__%s__run(' + '__attribute__((unused)) struct lfs_config *cfg) {' + % (case.name)) + f.writeln(4*' '+'// bench case %s' % case.name) + if case.code_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (case.code_lineno, suite.path)) + f.write(case.code) + if case.code_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (f.lineno+1, args['output'])) + f.writeln('}') + f.writeln() + + if not args.get('source'): + if suite.code is not None: + if suite.code_lineno is not None: + f.writeln('#line %d "%s"' + % (suite.code_lineno, suite.path)) + f.write(suite.code) + if suite.code_lineno is not None: + f.writeln('#line %d "%s"' + % (f.lineno+1, args['output'])) + f.writeln() + + if suite.defines: + for i, define in enumerate(sorted(suite.defines)): + f.writeln('#ifndef %s' % define) + f.writeln('#define %-24s ' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d' % (define+'_i', i)) + f.writeln('#define %-24s ' + 'BENCH_DEFINE(%s)' % (define, define+'_i')) + f.writeln('#endif') + f.writeln() + + # create case functions + for case in suite.cases: + if case.in_ is None: + write_case_functions(f, suite, case) + else: + if case.defines: + f.writeln('extern const bench_define_t ' + '__bench__%s__defines[][' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d];' + % (case.name, len(suite.defines))) + if suite.if_ is not None or case.if_ is not None: + f.writeln('extern bool __bench__%s__filter(' + 'void);' + % (case.name)) + f.writeln('extern void __bench__%s__run(' + 'struct lfs_config *cfg);' + % (case.name)) + f.writeln() + + # create suite struct + # + # note we place this in the custom bench_suites section with + # minimum alignment, otherwise GCC ups the alignment to + # 32-bytes for some reason + f.writeln('__attribute__((section("_bench_suites"), ' + 'aligned(1)))') + f.writeln('const struct bench_suite __bench__%s__suite = {' + % suite.name) + f.writeln(4*' '+'.name = "%s",' % suite.name) + f.writeln(4*' '+'.path = "%s",' % suite.path) + f.writeln(4*' '+'.flags = 0,') + if suite.defines: + # create suite define names + f.writeln(4*' '+'.define_names = (const char *const[' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d]){' % ( + len(suite.defines))) + for k in sorted(suite.defines): + f.writeln(8*' '+'[%-24s] = "%s",' % (k+'_i', k)) + f.writeln(4*' '+'},') + f.writeln(4*' '+'.define_count = ' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d,' % len(suite.defines)) + f.writeln(4*' '+'.cases = (const struct bench_case[]){') + for case in suite.cases: + # create case structs + f.writeln(8*' '+'{') + f.writeln(12*' '+'.name = "%s",' % case.name) + f.writeln(12*' '+'.path = "%s",' % case.path) + f.writeln(12*' '+'.flags = 0,') + f.writeln(12*' '+'.permutations = %d,' + % len(case.permutations)) + if case.defines: + f.writeln(12*' '+'.defines ' + '= (const bench_define_t*)__bench__%s__defines,' + % (case.name)) + if suite.if_ is not None or case.if_ is not None: + f.writeln(12*' '+'.filter = __bench__%s__filter,' + % (case.name)) + f.writeln(12*' '+'.run = __bench__%s__run,' + % (case.name)) + f.writeln(8*' '+'},') + f.writeln(4*' '+'},') + f.writeln(4*' '+'.case_count = %d,' % len(suite.cases)) + f.writeln('};') + f.writeln() + + else: + # copy source + f.writeln('#line 1 "%s"' % args['source']) + with open(args['source']) as sf: + shutil.copyfileobj(sf, f) + f.writeln() + + # write any internal benches + for suite in suites: + for case in suite.cases: + if (case.in_ is not None + and os.path.normpath(case.in_) + == os.path.normpath(args['source'])): + # write defines, but note we need to undef any + # new defines since we're in someone else's file + if suite.defines: + for i, define in enumerate( + sorted(suite.defines)): + f.writeln('#ifndef %s' % define) + f.writeln('#define %-24s ' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d' % ( + define+'_i', i)) + f.writeln('#define %-24s ' + 'BENCH_DEFINE(%s)' % ( + define, define+'_i')) + f.writeln('#define ' + '__BENCH__%s__NEEDS_UNDEF' % ( + define)) + f.writeln('#endif') + f.writeln() + + write_case_functions(f, suite, case) + + if suite.defines: + for define in sorted(suite.defines): + f.writeln('#ifdef __BENCH__%s__NEEDS_UNDEF' + % define) + f.writeln('#undef __BENCH__%s__NEEDS_UNDEF' + % define) + f.writeln('#undef %s' % define) + f.writeln('#undef %s' % (define+'_i')) + f.writeln('#endif') + f.writeln() + +def find_runner(runner, **args): + cmd = runner.copy() + + # run under some external command? + if args.get('exec'): + cmd[:0] = args['exec'] + + # run under valgrind? + if args.get('valgrind'): + cmd[:0] = args['valgrind_path'] + [ + '--leak-check=full', + '--track-origins=yes', + '--error-exitcode=4', + '-q'] + + # run under perf? + if args.get('perf'): + cmd[:0] = args['perf_script'] + list(filter(None, [ + '-R', + '--perf-freq=%s' % args['perf_freq'] + if args.get('perf_freq') else None, + '--perf-period=%s' % args['perf_period'] + if args.get('perf_period') else None, + '--perf-events=%s' % args['perf_events'] + if args.get('perf_events') else None, + '--perf-path=%s' % args['perf_path'] + if args.get('perf_path') else None, + '-o%s' % args['perf']])) + + # other context + if args.get('geometry'): + cmd.append('-G%s' % args['geometry']) + if args.get('disk'): + cmd.append('-d%s' % args['disk']) + if args.get('trace'): + cmd.append('-t%s' % args['trace']) + if args.get('trace_backtrace'): + cmd.append('--trace-backtrace') + if args.get('trace_period'): + cmd.append('--trace-period=%s' % args['trace_period']) + if args.get('trace_freq'): + cmd.append('--trace-freq=%s' % args['trace_freq']) + if args.get('read_sleep'): + cmd.append('--read-sleep=%s' % args['read_sleep']) + if args.get('prog_sleep'): + cmd.append('--prog-sleep=%s' % args['prog_sleep']) + if args.get('erase_sleep'): + cmd.append('--erase-sleep=%s' % args['erase_sleep']) + + # defines? + if args.get('define'): + for define in args.get('define'): + cmd.append('-D%s' % define) + + return cmd + +def list_(runner, bench_ids=[], **args): + cmd = find_runner(runner, **args) + bench_ids + if args.get('summary'): cmd.append('--summary') + if args.get('list_suites'): cmd.append('--list-suites') + if args.get('list_cases'): cmd.append('--list-cases') + if args.get('list_suite_paths'): cmd.append('--list-suite-paths') + if args.get('list_case_paths'): cmd.append('--list-case-paths') + if args.get('list_defines'): cmd.append('--list-defines') + if args.get('list_permutation_defines'): + cmd.append('--list-permutation-defines') + if args.get('list_implicit_defines'): + cmd.append('--list-implicit-defines') + if args.get('list_geometries'): cmd.append('--list-geometries') + + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + return sp.call(cmd) + + +def find_perms(runner_, ids=[], **args): + case_suites = {} + expected_case_perms = co.defaultdict(lambda: 0) + expected_perms = 0 + total_perms = 0 + + # query cases from the runner + cmd = runner_ + ['--list-cases'] + ids + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + pattern = re.compile( + '^(?P[^\s]+)' + '\s+(?P[^\s]+)' + '\s+(?P\d+)/(?P\d+)') + # skip the first line + for line in it.islice(proc.stdout, 1, None): + m = pattern.match(line) + if m: + filtered = int(m.group('filtered')) + perms = int(m.group('perms')) + expected_case_perms[m.group('case')] += filtered + expected_perms += filtered + total_perms += perms + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + # get which suite each case belongs to via paths + cmd = runner_ + ['--list-case-paths'] + ids + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + pattern = re.compile( + '^(?P[^\s]+)' + '\s+(?P[^:]+):(?P\d+)') + # skip the first line + for line in it.islice(proc.stdout, 1, None): + m = pattern.match(line) + if m: + path = m.group('path') + # strip path/suffix here + suite = os.path.basename(path) + if suite.endswith('.toml'): + suite = suite[:-len('.toml')] + case_suites[m.group('case')] = suite + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + # figure out expected suite perms + expected_suite_perms = co.defaultdict(lambda: 0) + for case, suite in case_suites.items(): + expected_suite_perms[suite] += expected_case_perms[case] + + return ( + case_suites, + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) + +def find_path(runner_, id, **args): + path = None + # query from runner + cmd = runner_ + ['--list-case-paths', id] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + pattern = re.compile( + '^(?P[^\s]+)' + '\s+(?P[^:]+):(?P\d+)') + # skip the first line + for line in it.islice(proc.stdout, 1, None): + m = pattern.match(line) + if m and path is None: + path_ = m.group('path') + lineno = int(m.group('lineno')) + path = (path_, lineno) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + return path + +def find_defines(runner_, id, **args): + # query permutation defines from runner + cmd = runner_ + ['--list-permutation-defines', id] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + defines = co.OrderedDict() + pattern = re.compile('^(?P\w+)=(?P.+)') + for line in proc.stdout: + m = pattern.match(line) + if m: + define = m.group('define') + value = m.group('value') + defines[define] = value + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + return defines + + +# Thread-safe CSV writer +class BenchOutput: + def __init__(self, path, head=None, tail=None): + self.f = openio(path, 'w+', 1) + self.lock = th.Lock() + self.head = head or [] + self.tail = tail or [] + self.writer = csv.DictWriter(self.f, self.head + self.tail) + self.rows = [] + + def close(self): + self.f.close() + + def __enter__(self): + return self + + def __exit__(self, *_): + self.f.close() + + def writerow(self, row): + with self.lock: + self.rows.append(row) + if all(k in self.head or k in self.tail for k in row.keys()): + # can simply append + self.writer.writerow(row) + else: + # need to rewrite the file + self.head.extend(row.keys() - (self.head + self.tail)) + self.f.seek(0) + self.f.truncate() + self.writer = csv.DictWriter(self.f, self.head + self.tail) + self.writer.writeheader() + for row in self.rows: + self.writer.writerow(row) + +# A bench failure +class BenchFailure(Exception): + def __init__(self, id, returncode, stdout, assert_=None): + self.id = id + self.returncode = returncode + self.stdout = stdout + self.assert_ = assert_ + +def run_stage(name, runner_, ids, stdout_, trace_, output_, **args): + # get expected suite/case/perm counts + (case_suites, + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) = find_perms(runner_, ids, **args) + + passed_suite_perms = co.defaultdict(lambda: 0) + passed_case_perms = co.defaultdict(lambda: 0) + passed_perms = 0 + readed = 0 + proged = 0 + erased = 0 + failures = [] + killed = False + + pattern = re.compile('^(?:' + '(?Prunning|finished|skipped|powerloss)' + ' (?P(?P[^:]+)[^\s]*)' + '(?: (?P\d+))?' + '(?: (?P\d+))?' + '(?: (?P\d+))?' + '|' '(?P[^:]+):(?P\d+):(?Passert):' + ' *(?P.*)' + ')$') + locals = th.local() + children = set() + + def run_runner(runner_, ids=[]): + nonlocal passed_suite_perms + nonlocal passed_case_perms + nonlocal passed_perms + nonlocal readed + nonlocal proged + nonlocal erased + nonlocal locals + + # run the benches! + cmd = runner_ + ids + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + + mpty, spty = pty.openpty() + proc = sp.Popen(cmd, stdout=spty, stderr=spty, close_fds=False) + os.close(spty) + children.add(proc) + mpty = os.fdopen(mpty, 'r', 1) + + last_id = None + last_stdout = co.deque(maxlen=args.get('context', 5) + 1) + last_assert = None + try: + while True: + # parse a line for state changes + try: + line = mpty.readline() + except OSError as e: + if e.errno != errno.EIO: + raise + break + if not line: + break + last_stdout.append(line) + if stdout_: + try: + stdout_.write(line) + stdout_.flush() + except BrokenPipeError: + pass + + m = pattern.match(line) + if m: + op = m.group('op') or m.group('op_') + if op == 'running': + locals.seen_perms += 1 + last_id = m.group('id') + last_stdout.clear() + last_assert = None + elif op == 'finished': + case = m.group('case') + suite = case_suites[case] + readed_ = int(m.group('readed')) + proged_ = int(m.group('proged')) + erased_ = int(m.group('erased')) + passed_suite_perms[suite] += 1 + passed_case_perms[case] += 1 + passed_perms += 1 + readed += readed_ + proged += proged_ + erased += erased_ + if output_: + # get defines and write to csv + defines = find_defines( + runner_, m.group('id'), **args) + output_.writerow({ + 'suite': suite, + 'case': case, + 'bench_readed': readed_, + 'bench_proged': proged_, + 'bench_erased': erased_, + **defines}) + elif op == 'skipped': + locals.seen_perms += 1 + elif op == 'assert': + last_assert = ( + m.group('path'), + int(m.group('lineno')), + m.group('message')) + # go ahead and kill the process, aborting takes a while + if args.get('keep_going'): + proc.kill() + except KeyboardInterrupt: + raise BenchFailure(last_id, 1, list(last_stdout)) + finally: + children.remove(proc) + mpty.close() + + proc.wait() + if proc.returncode != 0: + raise BenchFailure( + last_id, + proc.returncode, + list(last_stdout), + last_assert) + + def run_job(runner_, ids=[], start=None, step=None): + nonlocal failures + nonlocal killed + nonlocal locals + + start = start or 0 + step = step or 1 + while start < total_perms: + job_runner = runner_.copy() + if args.get('isolate') or args.get('valgrind'): + job_runner.append('-s%s,%s,%s' % (start, start+step, step)) + else: + job_runner.append('-s%s,,%s' % (start, step)) + + try: + # run the benches + locals.seen_perms = 0 + run_runner(job_runner, ids) + assert locals.seen_perms > 0 + start += locals.seen_perms*step + + except BenchFailure as failure: + # keep track of failures + if output_: + case, _ = failure.id.split(':', 1) + suite = case_suites[case] + # get defines and write to csv + defines = find_defines(runner_, failure.id, **args) + output_.writerow({ + 'suite': suite, + 'case': case, + **defines}) + + # race condition for multiple failures? + if failures and not args.get('keep_going'): + break + + failures.append(failure) + + if args.get('keep_going') and not killed: + # resume after failed bench + assert locals.seen_perms > 0 + start += locals.seen_perms*step + continue + else: + # stop other benches + killed = True + for child in children.copy(): + child.kill() + break + + + # parallel jobs? + runners = [] + if 'jobs' in args: + for job in range(args['jobs']): + runners.append(th.Thread( + target=run_job, args=(runner_, ids, job, args['jobs']), + daemon=True)) + else: + runners.append(th.Thread( + target=run_job, args=(runner_, ids, None, None), + daemon=True)) + + def print_update(done): + if not args.get('verbose') and (args['color'] or done): + sys.stdout.write('%s%srunning %s%s:%s %s%s' % ( + '\r\x1b[K' if args['color'] else '', + '\x1b[?7l' if not done else '', + ('\x1b[34m' if not failures else '\x1b[31m') + if args['color'] else '', + name, + '\x1b[m' if args['color'] else '', + ', '.join(filter(None, [ + '%d/%d suites' % ( + sum(passed_suite_perms[k] == v + for k, v in expected_suite_perms.items()), + len(expected_suite_perms)) + if (not args.get('by_suites') + and not args.get('by_cases')) else None, + '%d/%d cases' % ( + sum(passed_case_perms[k] == v + for k, v in expected_case_perms.items()), + len(expected_case_perms)) + if not args.get('by_cases') else None, + '%d/%d perms' % (passed_perms, expected_perms), + '%s%d/%d failures%s' % ( + '\x1b[31m' if args['color'] else '', + len(failures), + expected_perms, + '\x1b[m' if args['color'] else '') + if failures else None])), + '\x1b[?7h' if not done else '\n')) + sys.stdout.flush() + + for r in runners: + r.start() + + try: + while any(r.is_alive() for r in runners): + time.sleep(0.01) + print_update(False) + except KeyboardInterrupt: + # this is handled by the runner threads, we just + # need to not abort here + killed = True + finally: + print_update(True) + + for r in runners: + r.join() + + return ( + expected_perms, + passed_perms, + readed, + proged, + erased, + failures, + killed) + + +def run(runner, bench_ids=[], **args): + # query runner for benches + runner_ = find_runner(runner, **args) + print('using runner: %s' % ' '.join(shlex.quote(c) for c in runner_)) + (_, + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) = find_perms(runner_, bench_ids, **args) + print('found %d suites, %d cases, %d/%d permutations' % ( + len(expected_suite_perms), + len(expected_case_perms), + expected_perms, + total_perms)) + print() + + # automatic job detection? + if args.get('jobs') == 0: + args['jobs'] = len(os.sched_getaffinity(0)) + + # truncate and open logs here so they aren't disconnected between benches + stdout = None + if args.get('stdout'): + stdout = openio(args['stdout'], 'w', 1) + trace = None + if args.get('trace'): + trace = openio(args['trace'], 'w', 1) + output = None + if args.get('output'): + output = BenchOutput(args['output'], + ['suite', 'case'], + ['bench_readed', 'bench_proged', 'bench_erased']) + + # measure runtime + start = time.time() + + # spawn runners + expected = 0 + passed = 0 + readed = 0 + proged = 0 + erased = 0 + failures = [] + for by in (bench_ids if bench_ids + else expected_case_perms.keys() if args.get('by_cases') + else expected_suite_perms.keys() if args.get('by_suites') + else [None]): + # spawn jobs for stage + (expected_, + passed_, + readed_, + proged_, + erased_, + failures_, + killed) = run_stage( + by or 'benches', + runner_, + [by] if by is not None else [], + stdout, + trace, + output, + **args) + # collect passes/failures + expected += expected_ + passed += passed_ + readed += readed_ + proged += proged_ + erased += erased_ + failures.extend(failures_) + if (failures and not args.get('keep_going')) or killed: + break + + stop = time.time() + + if stdout: + try: + stdout.close() + except BrokenPipeError: + pass + if trace: + try: + trace.close() + except BrokenPipeError: + pass + if output: + output.close() + + # show summary + print() + print('%sdone:%s %s' % ( + ('\x1b[34m' if not failures else '\x1b[31m') + if args['color'] else '', + '\x1b[m' if args['color'] else '', + ', '.join(filter(None, [ + '%d readed' % readed, + '%d proged' % proged, + '%d erased' % erased, + 'in %.2fs' % (stop-start)])))) + print() + + # print each failure + for failure in failures: + assert failure.id is not None, '%s broken? %r' % ( + ' '.join(shlex.quote(c) for c in runner_), + failure) + + # get some extra info from runner + path, lineno = find_path(runner_, failure.id, **args) + defines = find_defines(runner_, failure.id, **args) + + # show summary of failure + print('%s%s:%d:%sfailure:%s %s%s failed' % ( + '\x1b[01m' if args['color'] else '', + path, lineno, + '\x1b[01;31m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + failure.id, + ' (%s)' % ', '.join('%s=%s' % (k,v) for k,v in defines.items()) + if defines else '')) + + if failure.stdout: + stdout = failure.stdout + if failure.assert_ is not None: + stdout = stdout[:-1] + for line in stdout[-args.get('context', 5):]: + sys.stdout.write(line) + + if failure.assert_ is not None: + path, lineno, message = failure.assert_ + print('%s%s:%d:%sassert:%s %s' % ( + '\x1b[01m' if args['color'] else '', + path, lineno, + '\x1b[01;31m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + message)) + with open(path) as f: + line = next(it.islice(f, lineno-1, None)).strip('\n') + print(line) + print() + + # drop into gdb? + if failures and (args.get('gdb') + or args.get('gdb_case') + or args.get('gdb_main')): + failure = failures[0] + cmd = runner_ + [failure.id] + + if args.get('gdb_main'): + # we don't really need the case breakpoint here, but it + # can be helpful + path, lineno = find_path(runner_, failure.id, **args) + cmd[:0] = args['gdb_path'] + [ + '-ex', 'break main', + '-ex', 'break %s:%d' % (path, lineno), + '-ex', 'run', + '--args'] + elif args.get('gdb_case'): + path, lineno = find_path(runner_, failure.id, **args) + cmd[:0] = args['gdb_path'] + [ + '-ex', 'break %s:%d' % (path, lineno), + '-ex', 'run', + '--args'] + elif failure.assert_ is not None: + cmd[:0] = args['gdb_path'] + [ + '-ex', 'run', + '-ex', 'frame function raise', + '-ex', 'up 2', + '--args'] + else: + cmd[:0] = args['gdb_path'] + [ + '-ex', 'run', + '--args'] + + # exec gdb interactively + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + os.execvp(cmd[0], cmd) + + return 1 if failures else 0 + + +def main(**args): + # figure out what color should be + if args.get('color') == 'auto': + args['color'] = sys.stdout.isatty() + elif args.get('color') == 'always': + args['color'] = True + else: + args['color'] = False + + if args.get('compile'): + return compile(**args) + elif (args.get('summary') + or args.get('list_suites') + or args.get('list_cases') + or args.get('list_suite_paths') + or args.get('list_case_paths') + or args.get('list_defines') + or args.get('list_permutation_defines') + or args.get('list_implicit_defines') + or args.get('list_geometries')): + return list_(**args) + else: + return run(**args) + + +if __name__ == "__main__": + import argparse + import sys + argparse.ArgumentParser._handle_conflict_ignore = lambda *_: None + argparse._ArgumentGroup._handle_conflict_ignore = lambda *_: None + parser = argparse.ArgumentParser( + description="Build and run benches.", + allow_abbrev=False, + conflict_handler='ignore') + parser.add_argument( + '-v', '--verbose', + action='store_true', + help="Output commands that run behind the scenes.") + parser.add_argument( + '--color', + choices=['never', 'always', 'auto'], + default='auto', + help="When to use terminal colors. Defaults to 'auto'.") + + # bench flags + bench_parser = parser.add_argument_group('bench options') + bench_parser.add_argument( + 'runner', + nargs='?', + type=lambda x: x.split(), + help="Bench runner to use for benching. Defaults to %r." % RUNNER_PATH) + bench_parser.add_argument( + 'bench_ids', + nargs='*', + help="Description of benches to run.") + bench_parser.add_argument( + '-Y', '--summary', + action='store_true', + help="Show quick summary.") + bench_parser.add_argument( + '-l', '--list-suites', + action='store_true', + help="List bench suites.") + bench_parser.add_argument( + '-L', '--list-cases', + action='store_true', + help="List bench cases.") + bench_parser.add_argument( + '--list-suite-paths', + action='store_true', + help="List the path for each bench suite.") + bench_parser.add_argument( + '--list-case-paths', + action='store_true', + help="List the path and line number for each bench case.") + bench_parser.add_argument( + '--list-defines', + action='store_true', + help="List all defines in this bench-runner.") + bench_parser.add_argument( + '--list-permutation-defines', + action='store_true', + help="List explicit defines in this bench-runner.") + bench_parser.add_argument( + '--list-implicit-defines', + action='store_true', + help="List implicit defines in this bench-runner.") + bench_parser.add_argument( + '--list-geometries', + action='store_true', + help="List the available disk geometries.") + bench_parser.add_argument( + '-D', '--define', + action='append', + help="Override a bench define.") + bench_parser.add_argument( + '-G', '--geometry', + help="Comma-separated list of disk geometries to bench.") + bench_parser.add_argument( + '-d', '--disk', + help="Direct block device operations to this file.") + bench_parser.add_argument( + '-t', '--trace', + help="Direct trace output to this file.") + bench_parser.add_argument( + '--trace-backtrace', + action='store_true', + help="Include a backtrace with every trace statement.") + bench_parser.add_argument( + '--trace-period', + help="Sample trace output at this period in cycles.") + bench_parser.add_argument( + '--trace-freq', + help="Sample trace output at this frequency in hz.") + bench_parser.add_argument( + '-O', '--stdout', + help="Direct stdout to this file. Note stderr is already merged here.") + bench_parser.add_argument( + '-o', '--output', + help="CSV file to store results.") + bench_parser.add_argument( + '--read-sleep', + help="Artificial read delay in seconds.") + bench_parser.add_argument( + '--prog-sleep', + help="Artificial prog delay in seconds.") + bench_parser.add_argument( + '--erase-sleep', + help="Artificial erase delay in seconds.") + bench_parser.add_argument( + '-j', '--jobs', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Number of parallel runners to run. 0 runs one runner per core.") + bench_parser.add_argument( + '-k', '--keep-going', + action='store_true', + help="Don't stop on first error.") + bench_parser.add_argument( + '-i', '--isolate', + action='store_true', + help="Run each bench permutation in a separate process.") + bench_parser.add_argument( + '-b', '--by-suites', + action='store_true', + help="Step through benches by suite.") + bench_parser.add_argument( + '-B', '--by-cases', + action='store_true', + help="Step through benches by case.") + bench_parser.add_argument( + '--context', + type=lambda x: int(x, 0), + default=5, + help="Show this many lines of stdout on bench failure. " + "Defaults to 5.") + bench_parser.add_argument( + '--gdb', + action='store_true', + help="Drop into gdb on bench failure.") + bench_parser.add_argument( + '--gdb-case', + action='store_true', + help="Drop into gdb on bench failure but stop at the beginning " + "of the failing bench case.") + bench_parser.add_argument( + '--gdb-main', + action='store_true', + help="Drop into gdb on bench failure but stop at the beginning " + "of main.") + bench_parser.add_argument( + '--gdb-path', + type=lambda x: x.split(), + default=GDB_PATH, + help="Path to the gdb executable, may include flags. " + "Defaults to %r." % GDB_PATH) + bench_parser.add_argument( + '--exec', + type=lambda e: e.split(), + help="Run under another executable.") + bench_parser.add_argument( + '--valgrind', + action='store_true', + help="Run under Valgrind to find memory errors. Implicitly sets " + "--isolate.") + bench_parser.add_argument( + '--valgrind-path', + type=lambda x: x.split(), + default=VALGRIND_PATH, + help="Path to the Valgrind executable, may include flags. " + "Defaults to %r." % VALGRIND_PATH) + bench_parser.add_argument( + '-p', '--perf', + help="Run under Linux's perf to sample performance counters, writing " + "samples to this file.") + bench_parser.add_argument( + '--perf-freq', + help="perf sampling frequency. This is passed directly to the perf " + "script.") + bench_parser.add_argument( + '--perf-period', + help="perf sampling period. This is passed directly to the perf " + "script.") + bench_parser.add_argument( + '--perf-events', + help="perf events to record. This is passed directly to the perf " + "script.") + bench_parser.add_argument( + '--perf-script', + type=lambda x: x.split(), + default=PERF_SCRIPT, + help="Path to the perf script to use. Defaults to %r." % PERF_SCRIPT) + bench_parser.add_argument( + '--perf-path', + type=lambda x: x.split(), + help="Path to the perf executable, may include flags. This is passed " + "directly to the perf script") + + # compilation flags + comp_parser = parser.add_argument_group('compilation options') + comp_parser.add_argument( + 'bench_paths', + nargs='*', + help="Description of *.toml files to compile. May be a directory " + "or a list of paths.") + comp_parser.add_argument( + '-c', '--compile', + action='store_true', + help="Compile a bench suite or source file.") + comp_parser.add_argument( + '-s', '--source', + help="Source file to compile, possibly injecting internal benches.") + comp_parser.add_argument( + '--include', + default=HEADER_PATH, + help="Inject this header file into every compiled bench file. " + "Defaults to %r." % HEADER_PATH) + comp_parser.add_argument( + '-o', '--output', + help="Output file.") + + # runner/bench_paths overlap, so need to do some munging here + args = parser.parse_intermixed_args() + args.bench_paths = [' '.join(args.runner or [])] + args.bench_ids + args.runner = args.runner or [RUNNER_PATH] + + sys.exit(main(**{k: v + for k, v in vars(args).items() + if v is not None})) diff --git a/components/fs/littlefs/littlefs/scripts/changeprefix.py b/components/fs/littlefs/littlefs/scripts/changeprefix.py new file mode 100644 index 000000000..381a45687 --- /dev/null +++ b/components/fs/littlefs/littlefs/scripts/changeprefix.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# +# Change prefixes in files/filenames. Useful for creating different versions +# of a codebase that don't conflict at compile time. +# +# Example: +# $ ./scripts/changeprefix.py lfs lfs3 +# +# Copyright (c) 2022, The littlefs authors. +# Copyright (c) 2019, Arm Limited. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause +# + +import glob +import itertools +import os +import os.path +import re +import shlex +import shutil +import subprocess +import tempfile + +GIT_PATH = ['git'] + + +def openio(path, mode='r', buffering=-1): + # allow '-' for stdin/stdout + if path == '-': + if mode == 'r': + return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering) + else: + return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering) + else: + return open(path, mode, buffering) + +def changeprefix(from_prefix, to_prefix, line): + line, count1 = re.subn( + '\\b'+from_prefix, + to_prefix, + line) + line, count2 = re.subn( + '\\b'+from_prefix.upper(), + to_prefix.upper(), + line) + line, count3 = re.subn( + '\\B-D'+from_prefix.upper(), + '-D'+to_prefix.upper(), + line) + return line, count1+count2+count3 + +def changefile(from_prefix, to_prefix, from_path, to_path, *, + no_replacements=False): + # rename any prefixes in file + count = 0 + + # create a temporary file to avoid overwriting ourself + if from_path == to_path and to_path != '-': + to_path_temp = tempfile.NamedTemporaryFile('w', delete=False) + to_path = to_path_temp.name + else: + to_path_temp = None + + with openio(from_path) as from_f: + with openio(to_path, 'w') as to_f: + for line in from_f: + if not no_replacements: + line, n = changeprefix(from_prefix, to_prefix, line) + count += n + to_f.write(line) + + if from_path != '-' and to_path != '-': + shutil.copystat(from_path, to_path) + + if to_path_temp: + os.rename(to_path, from_path) + elif from_path != '-': + os.remove(from_path) + + # Summary + print('%s: %d replacements' % ( + '%s -> %s' % (from_path, to_path) if not to_path_temp else from_path, + count)) + +def main(from_prefix, to_prefix, paths=[], *, + verbose=False, + output=None, + no_replacements=False, + no_renames=False, + git=False, + no_stage=False, + git_path=GIT_PATH): + if not paths: + if git: + cmd = git_path + ['ls-tree', '-r', '--name-only', 'HEAD'] + if verbose: + print(' '.join(shlex.quote(c) for c in cmd)) + paths = subprocess.check_output(cmd, encoding='utf8').split() + else: + print('no paths?', file=sys.stderr) + sys.exit(1) + + for from_path in paths: + # rename filename? + if output: + to_path = output + elif no_renames: + to_path = from_path + else: + to_path = os.path.join( + os.path.dirname(from_path), + changeprefix(from_prefix, to_prefix, + os.path.basename(from_path))[0]) + + # rename contents + changefile(from_prefix, to_prefix, from_path, to_path, + no_replacements=no_replacements) + + # stage? + if git and not no_stage: + if from_path != to_path: + cmd = git_path + ['rm', '-q', from_path] + if verbose: + print(' '.join(shlex.quote(c) for c in cmd)) + subprocess.check_call(cmd) + cmd = git_path + ['add', to_path] + if verbose: + print(' '.join(shlex.quote(c) for c in cmd)) + subprocess.check_call(cmd) + + +if __name__ == "__main__": + import argparse + import sys + parser = argparse.ArgumentParser( + description="Change prefixes in files/filenames. Useful for creating " + "different versions of a codebase that don't conflict at compile " + "time.", + allow_abbrev=False) + parser.add_argument( + 'from_prefix', + help="Prefix to replace.") + parser.add_argument( + 'to_prefix', + help="Prefix to replace with.") + parser.add_argument( + 'paths', + nargs='*', + help="Files to operate on.") + parser.add_argument( + '-v', '--verbose', + action='store_true', + help="Output commands that run behind the scenes.") + parser.add_argument( + '-o', '--output', + help="Output file.") + parser.add_argument( + '-N', '--no-replacements', + action='store_true', + help="Don't change prefixes in files") + parser.add_argument( + '-R', '--no-renames', + action='store_true', + help="Don't rename files") + parser.add_argument( + '--git', + action='store_true', + help="Use git to find/update files.") + parser.add_argument( + '--no-stage', + action='store_true', + help="Don't stage changes with git.") + parser.add_argument( + '--git-path', + type=lambda x: x.split(), + default=GIT_PATH, + help="Path to git executable, may include flags. " + "Defaults to %r." % GIT_PATH) + sys.exit(main(**{k: v + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/components/fs/littlefs/littlefs/scripts/code.py b/components/fs/littlefs/littlefs/scripts/code.py old mode 100755 new mode 100644 index b394e9cda..ba8bd1e0d --- a/components/fs/littlefs/littlefs/scripts/code.py +++ b/components/fs/littlefs/littlefs/scripts/code.py @@ -1,42 +1,188 @@ #!/usr/bin/env python3 # -# Script to find code size at the function level. Basically just a bit wrapper +# Script to find code size at the function level. Basically just a big wrapper # around nm with some extra conveniences for comparing builds. Heavily inspired # by Linux's Bloat-O-Meter. # +# Example: +# ./scripts/code.py lfs.o lfs_util.o -Ssize +# +# Copyright (c) 2022, The littlefs authors. +# Copyright (c) 2020, Arm Limited. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause +# -import os -import glob +import collections as co +import csv +import difflib import itertools as it -import subprocess as sp -import shlex +import math as m +import os import re -import csv -import collections as co +import shlex +import subprocess as sp + + +NM_PATH = ['nm'] +NM_TYPES = 'tTrRdD' +OBJDUMP_PATH = ['objdump'] + + +# integer fields +class Int(co.namedtuple('Int', 'x')): + __slots__ = () + def __new__(cls, x=0): + if isinstance(x, Int): + return x + if isinstance(x, str): + try: + x = int(x, 0) + except ValueError: + # also accept +-∞ and +-inf + if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): + x = m.inf + elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): + x = -m.inf + else: + raise + assert isinstance(x, int) or m.isinf(x), x + return super().__new__(cls, x) + + def __str__(self): + if self.x == m.inf: + return '∞' + elif self.x == -m.inf: + return '-∞' + else: + return str(self.x) + + def __int__(self): + assert not m.isinf(self.x) + return self.x + + def __float__(self): + return float(self.x) + + none = '%7s' % '-' + def table(self): + return '%7s' % (self,) + + diff_none = '%7s' % '-' + diff_table = table + + def diff_diff(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + diff = new - old + if diff == +m.inf: + return '%7s' % '+∞' + elif diff == -m.inf: + return '%7s' % '-∞' + else: + return '%+7d' % diff + + def ratio(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + if m.isinf(new) and m.isinf(old): + return 0.0 + elif m.isinf(new): + return +m.inf + elif m.isinf(old): + return -m.inf + elif not old and not new: + return 0.0 + elif not old: + return 1.0 + else: + return (new-old) / old + + def __add__(self, other): + return self.__class__(self.x + other.x) + + def __sub__(self, other): + return self.__class__(self.x - other.x) + + def __mul__(self, other): + return self.__class__(self.x * other.x) +# code size results +class CodeResult(co.namedtuple('CodeResult', [ + 'file', 'function', + 'size'])): + _by = ['file', 'function'] + _fields = ['size'] + _sort = ['size'] + _types = {'size': Int} -OBJ_PATHS = ['*.o'] + __slots__ = () + def __new__(cls, file='', function='', size=0): + return super().__new__(cls, file, function, + Int(size)) -def collect(paths, **args): - results = co.defaultdict(lambda: 0) - pattern = re.compile( + def __add__(self, other): + return CodeResult(self.file, self.function, + self.size + other.size) + + +def openio(path, mode='r', buffering=-1): + # allow '-' for stdin/stdout + if path == '-': + if mode == 'r': + return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering) + else: + return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering) + else: + return open(path, mode, buffering) + +def collect(obj_paths, *, + nm_path=NM_PATH, + nm_types=NM_TYPES, + objdump_path=OBJDUMP_PATH, + sources=None, + everything=False, + **args): + size_pattern = re.compile( '^(?P[0-9a-fA-F]+)' + - ' (?P[%s])' % re.escape(args['type']) + + ' (?P[%s])' % re.escape(nm_types) + ' (?P.+?)$') - for path in paths: - # note nm-tool may contain extra args - cmd = args['nm_tool'] + ['--size-sort', path] + line_pattern = re.compile( + '^\s+(?P[0-9]+)' + '(?:\s+(?P[0-9]+))?' + '\s+.*' + '\s+(?P[^\s]+)$') + info_pattern = re.compile( + '^(?:.*(?PDW_TAG_[a-z_]+).*' + '|.*DW_AT_name.*:\s*(?P[^:\s]+)\s*' + '|.*DW_AT_decl_file.*:\s*(?P[0-9]+)\s*)$') + + results = [] + for path in obj_paths: + # guess the source, if we have debug-info we'll replace this later + file = re.sub('(\.o)?$', '.c', path, 1) + + # find symbol sizes + results_ = [] + # note nm-path may contain extra args + cmd = nm_path + ['--size-sort', path] if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE if not args.get('verbose') else None, universal_newlines=True, - errors='replace') + errors='replace', + close_fds=False) for line in proc.stdout: - m = pattern.match(line) + m = size_pattern.match(line) if m: - results[(path, m.group('func'))] += int(m.group('size'), 16) + func = m.group('func') + # discard internal functions + if not everything and func.startswith('__'): + continue + results_.append(CodeResult( + file, func, + int(m.group('size'), 16))) proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -44,241 +190,518 @@ def collect(paths, **args): sys.stdout.write(line) sys.exit(-1) - flat_results = [] - for (file, func), size in results.items(): - # map to source files - if args.get('build_dir'): - file = re.sub('%s/*' % re.escape(args['build_dir']), '', file) - # replace .o with .c, different scripts report .o/.c, we need to - # choose one if we want to deduplicate csv files - file = re.sub('\.o$', '.c', file) - # discard internal functions - if not args.get('everything'): - if func.startswith('__'): - continue - # discard .8449 suffixes created by optimizer - func = re.sub('\.[0-9]+', '', func) - - flat_results.append((file, func, size)) - - return flat_results - -def main(**args): - def openio(path, mode='r'): - if path == '-': - if 'r' in mode: - return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + + # try to figure out the source file if we have debug-info + dirs = {} + files = {} + # note objdump-path may contain extra args + cmd = objdump_path + ['--dwarf=rawline', path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + for line in proc.stdout: + # note that files contain references to dirs, which we + # dereference as soon as we see them as each file table follows a + # dir table + m = line_pattern.match(line) + if m: + if not m.group('dir'): + # found a directory entry + dirs[int(m.group('no'))] = m.group('path') + else: + # found a file entry + dir = int(m.group('dir')) + if dir in dirs: + files[int(m.group('no'))] = os.path.join( + dirs[dir], + m.group('path')) + else: + files[int(m.group('no'))] = m.group('path') + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + # do nothing on error, we don't need objdump to work, source files + # may just be inaccurate + pass + + defs = {} + is_func = False + f_name = None + f_file = None + # note objdump-path may contain extra args + cmd = objdump_path + ['--dwarf=info', path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + for line in proc.stdout: + # state machine here to find definitions + m = info_pattern.match(line) + if m: + if m.group('tag'): + if is_func: + defs[f_name] = files.get(f_file, '?') + is_func = (m.group('tag') == 'DW_TAG_subprogram') + elif m.group('name'): + f_name = m.group('name') + elif m.group('file'): + f_file = int(m.group('file')) + if is_func: + defs[f_name] = files.get(f_file, '?') + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + # do nothing on error, we don't need objdump to work, source files + # may just be inaccurate + pass + + for r in results_: + # find best matching debug symbol, this may be slightly different + # due to optimizations + if defs: + # exact match? avoid difflib if we can for speed + if r.function in defs: + file = defs[r.function] + else: + _, file = max( + defs.items(), + key=lambda d: difflib.SequenceMatcher(None, + d[0], + r.function, False).ratio()) else: - return os.fdopen(os.dup(sys.stdout.fileno()), 'w') - else: - return open(path, mode) + file = r.file - # find sizes - if not args.get('use', None): - # find .o files - paths = [] - for path in args['obj_paths']: - if os.path.isdir(path): - path = path + '/*.o' + # ignore filtered sources + if sources is not None: + if not any( + os.path.abspath(file) == os.path.abspath(s) + for s in sources): + continue + else: + # default to only cwd + if not everything and not os.path.commonpath([ + os.getcwd(), + os.path.abspath(file)]) == os.getcwd(): + continue + + # simplify path + if os.path.commonpath([ + os.getcwd(), + os.path.abspath(file)]) == os.getcwd(): + file = os.path.relpath(file) + else: + file = os.path.abspath(file) + + results.append(r._replace(file=file)) + + return results - for path in glob.glob(path): - paths.append(path) - if not paths: - print('no .obj files found in %r?' % args['obj_paths']) +def fold(Result, results, *, + by=None, + defines=None, + **_): + if by is None: + by = Result._by + + for k in it.chain(by or [], (k for k, _ in defines or [])): + if k not in Result._by and k not in Result._fields: + print("error: could not find field %r?" % k) sys.exit(-1) - results = collect(paths, **args) - else: - with openio(args['use']) as f: - r = csv.DictReader(f) - results = [ - ( result['file'], - result['name'], - int(result['code_size'])) - for result in r - if result.get('code_size') not in {None, ''}] - - total = 0 - for _, _, size in results: - total += size + # filter by matching defines + if defines is not None: + results_ = [] + for r in results: + if all(getattr(r, k) in vs for k, vs in defines): + results_.append(r) + results = results_ - # find previous results? - if args.get('diff'): - try: - with openio(args['diff']) as f: - r = csv.DictReader(f) - prev_results = [ - ( result['file'], - result['name'], - int(result['code_size'])) - for result in r - if result.get('code_size') not in {None, ''}] - except FileNotFoundError: - prev_results = [] + # organize results into conflicts + folding = co.OrderedDict() + for r in results: + name = tuple(getattr(r, k) for k in by) + if name not in folding: + folding[name] = [] + folding[name].append(r) - prev_total = 0 - for _, _, size in prev_results: - prev_total += size + # merge conflicts + folded = [] + for name, rs in folding.items(): + folded.append(sum(rs[1:], start=rs[0])) - # write results to CSV - if args.get('output'): - merged_results = co.defaultdict(lambda: {}) - other_fields = [] + return folded - # merge? - if args.get('merge'): - try: - with openio(args['merge']) as f: - r = csv.DictReader(f) - for result in r: - file = result.pop('file', '') - func = result.pop('name', '') - result.pop('code_size', None) - merged_results[(file, func)] = result - other_fields = result.keys() - except FileNotFoundError: - pass - - for file, func, size in results: - merged_results[(file, func)]['code_size'] = size +def table(Result, results, diff_results=None, *, + by=None, + fields=None, + sort=None, + summary=False, + all=False, + percent=False, + **_): + all_, all = all, __builtins__.all - with openio(args['output'], 'w') as f: - w = csv.DictWriter(f, ['file', 'name', *other_fields, 'code_size']) - w.writeheader() - for (file, func), result in sorted(merged_results.items()): - w.writerow({'file': file, 'name': func, **result}) - - # print results - def dedup_entries(results, by='name'): - entries = co.defaultdict(lambda: 0) - for file, func, size in results: - entry = (file if by == 'file' else func) - entries[entry] += size - return entries - - def diff_entries(olds, news): - diff = co.defaultdict(lambda: (0, 0, 0, 0)) - for name, new in news.items(): - diff[name] = (0, new, new, 1.0) - for name, old in olds.items(): - _, new, _, _ = diff[name] - diff[name] = (old, new, new-old, (new-old)/old if old else 1.0) - return diff - - def sorted_entries(entries): - if args.get('size_sort'): - return sorted(entries, key=lambda x: (-x[1], x)) - elif args.get('reverse_size_sort'): - return sorted(entries, key=lambda x: (+x[1], x)) - else: - return sorted(entries) + if by is None: + by = Result._by + if fields is None: + fields = Result._fields + types = Result._types - def sorted_diff_entries(entries): - if args.get('size_sort'): - return sorted(entries, key=lambda x: (-x[1][1], x)) - elif args.get('reverse_size_sort'): - return sorted(entries, key=lambda x: (+x[1][1], x)) - else: - return sorted(entries, key=lambda x: (-x[1][3], x)) + # fold again + results = fold(Result, results, by=by) + if diff_results is not None: + diff_results = fold(Result, diff_results, by=by) - def print_header(by=''): - if not args.get('diff'): - print('%-36s %7s' % (by, 'size')) - else: - print('%-36s %7s %7s %7s' % (by, 'old', 'new', 'diff')) + # organize by name + table = { + ','.join(str(getattr(r, k) or '') for k in by): r + for r in results} + diff_table = { + ','.join(str(getattr(r, k) or '') for k in by): r + for r in diff_results or []} + names = list(table.keys() | diff_table.keys()) + + # sort again, now with diff info, note that python's sort is stable + names.sort() + if diff_results is not None: + names.sort(key=lambda n: tuple( + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), + reverse=True) + if sort: + for k, reverse in reversed(sort): + names.sort( + key=lambda n: tuple( + (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None else () + for k in ([k] if k else [ + k for k in Result._sort if k in fields])), + reverse=reverse ^ (not k or k in Result._fields)) - def print_entry(name, size): - print("%-36s %7d" % (name, size)) - def print_diff_entry(name, old, new, diff, ratio): - print("%-36s %7s %7s %+7d%s" % (name, - old or "-", - new or "-", - diff, - ' (%+.1f%%)' % (100*ratio) if ratio else '')) + # build up our lines + lines = [] - def print_entries(by='name'): - entries = dedup_entries(results, by=by) + # header + header = [] + header.append('%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else '') + if diff_results is None: + for k in fields: + header.append(k) + elif percent: + for k in fields: + header.append(k) + else: + for k in fields: + header.append('o'+k) + for k in fields: + header.append('n'+k) + for k in fields: + header.append('d'+k) + header.append('') + lines.append(header) - if not args.get('diff'): - print_header(by=by) - for name, size in sorted_entries(entries.items()): - print_entry(name, size) + def table_entry(name, r, diff_r=None, ratios=[]): + entry = [] + entry.append(name) + if diff_results is None: + for k in fields: + entry.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) + elif percent: + for k in fields: + entry.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - prev_entries = dedup_entries(prev_results, by=by) - diff = diff_entries(prev_entries, entries) - print_header(by='%s (%d added, %d removed)' % (by, - sum(1 for old, _, _, _ in diff.values() if not old), - sum(1 for _, new, _, _ in diff.values() if not new))) - for name, (old, new, diff, ratio) in sorted_diff_entries( - diff.items()): - if ratio or args.get('all'): - print_diff_entry(name, old, new, diff, ratio) - - def print_totals(): - if not args.get('diff'): - print_entry('TOTAL', total) + for k in fields: + entry.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + entry.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + entry.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + entry.append('') + elif percent: + entry.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) else: - ratio = (0.0 if not prev_total and not total - else 1.0 if not prev_total - else (total-prev_total)/prev_total) - print_diff_entry('TOTAL', - prev_total, total, - total-prev_total, - ratio) - - if args.get('quiet'): - pass - elif args.get('summary'): - print_header() - print_totals() - elif args.get('files'): - print_entries(by='file') - print_totals() + entry.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + return entry + + # entries + if not summary: + for name in names: + r = table.get(name) + if diff_results is None: + diff_r = None + ratios = None + else: + diff_r = diff_table.get(name) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + if not all_ and not any(ratios): + continue + lines.append(table_entry(name, r, diff_r, ratios)) + + # total + r = next(iter(fold(Result, results, by=[])), None) + if diff_results is None: + diff_r = None + ratios = None else: - print_entries(by='name') - print_totals() + diff_r = next(iter(fold(Result, diff_results, by=[])), None) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + lines.append(table_entry('TOTAL', r, diff_r, ratios)) + + # find the best widths, note that column 0 contains the names and column -1 + # the ratios, so those are handled a bit differently + widths = [ + ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1 + for w, i in zip( + it.chain([23], it.repeat(7)), + range(len(lines[0])-1))] + + # print our table + for line in lines: + print('%-*s %s%s' % ( + widths[0], line[0], + ' '.join('%*s' % (w, x) + for w, x in zip(widths[1:], line[1:-1])), + line[-1])) + + +def main(obj_paths, *, + by=None, + fields=None, + defines=None, + sort=None, + **args): + # find sizes + if not args.get('use', None): + results = collect(obj_paths, **args) + else: + results = [] + with openio(args['use']) as f: + reader = csv.DictReader(f, restval='') + for r in reader: + if not any('code_'+k in r and r['code_'+k].strip() + for k in CodeResult._fields): + continue + try: + results.append(CodeResult( + **{k: r[k] for k in CodeResult._by + if k in r and r[k].strip()}, + **{k: r['code_'+k] for k in CodeResult._fields + if 'code_'+k in r and r['code_'+k].strip()})) + except TypeError: + pass + + # fold + results = fold(CodeResult, results, by=by, defines=defines) + + # sort, note that python's sort is stable + results.sort() + if sort: + for k, reverse in reversed(sort): + results.sort( + key=lambda r: tuple( + (getattr(r, k),) if getattr(r, k) is not None else () + for k in ([k] if k else CodeResult._sort)), + reverse=reverse ^ (not k or k in CodeResult._fields)) + + # write results to CSV + if args.get('output'): + with openio(args['output'], 'w') as f: + writer = csv.DictWriter(f, + (by if by is not None else CodeResult._by) + + ['code_'+k for k in ( + fields if fields is not None else CodeResult._fields)]) + writer.writeheader() + for r in results: + writer.writerow( + {k: getattr(r, k) for k in ( + by if by is not None else CodeResult._by)} + | {'code_'+k: getattr(r, k) for k in ( + fields if fields is not None else CodeResult._fields)}) + + # find previous results? + if args.get('diff'): + diff_results = [] + try: + with openio(args['diff']) as f: + reader = csv.DictReader(f, restval='') + for r in reader: + if not any('code_'+k in r and r['code_'+k].strip() + for k in CodeResult._fields): + continue + try: + diff_results.append(CodeResult( + **{k: r[k] for k in CodeResult._by + if k in r and r[k].strip()}, + **{k: r['code_'+k] for k in CodeResult._fields + if 'code_'+k in r and r['code_'+k].strip()})) + except TypeError: + pass + except FileNotFoundError: + pass + + # fold + diff_results = fold(CodeResult, diff_results, by=by, defines=defines) + + # print table + if not args.get('quiet'): + table(CodeResult, results, + diff_results if args.get('diff') else None, + by=by if by is not None else ['function'], + fields=fields, + sort=sort, + **args) + if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Find code size at the function level.") - parser.add_argument('obj_paths', nargs='*', default=OBJ_PATHS, - help="Description of where to find *.o files. May be a directory \ - or a list of paths. Defaults to %r." % OBJ_PATHS) - parser.add_argument('-v', '--verbose', action='store_true', + description="Find code size at the function level.", + allow_abbrev=False) + parser.add_argument( + 'obj_paths', + nargs='*', + help="Input *.o files.") + parser.add_argument( + '-v', '--verbose', + action='store_true', help="Output commands that run behind the scenes.") - parser.add_argument('-q', '--quiet', action='store_true', + parser.add_argument( + '-q', '--quiet', + action='store_true', help="Don't show anything, useful with -o.") - parser.add_argument('-o', '--output', + parser.add_argument( + '-o', '--output', help="Specify CSV file to store results.") - parser.add_argument('-u', '--use', - help="Don't compile and find code sizes, instead use this CSV file.") - parser.add_argument('-d', '--diff', - help="Specify CSV file to diff code size against.") - parser.add_argument('-m', '--merge', - help="Merge with an existing CSV file when writing to output.") - parser.add_argument('-a', '--all', action='store_true', - help="Show all functions, not just the ones that changed.") - parser.add_argument('-A', '--everything', action='store_true', + parser.add_argument( + '-u', '--use', + help="Don't parse anything, use this CSV file.") + parser.add_argument( + '-d', '--diff', + help="Specify CSV file to diff against.") + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") + parser.add_argument( + '-p', '--percent', + action='store_true', + help="Only show percentage change, not a full diff.") + parser.add_argument( + '-b', '--by', + action='append', + choices=CodeResult._by, + help="Group by this field.") + parser.add_argument( + '-f', '--field', + dest='fields', + action='append', + choices=CodeResult._fields, + help="Show this field.") + parser.add_argument( + '-D', '--define', + dest='defines', + action='append', + type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), + help="Only include results where this field is this value.") + class AppendSort(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.sort is None: + namespace.sort = [] + namespace.sort.append((value, True if option == '-S' else False)) + parser.add_argument( + '-s', '--sort', + nargs='?', + action=AppendSort, + help="Sort by this field.") + parser.add_argument( + '-S', '--reverse-sort', + nargs='?', + action=AppendSort, + help="Sort by this field, but backwards.") + parser.add_argument( + '-Y', '--summary', + action='store_true', + help="Only show the total.") + parser.add_argument( + '-F', '--source', + dest='sources', + action='append', + help="Only consider definitions in this file. Defaults to anything " + "in the current directory.") + parser.add_argument( + '--everything', + action='store_true', help="Include builtin and libc specific symbols.") - parser.add_argument('-s', '--size-sort', action='store_true', - help="Sort by size.") - parser.add_argument('-S', '--reverse-size-sort', action='store_true', - help="Sort by size, but backwards.") - parser.add_argument('-F', '--files', action='store_true', - help="Show file-level code sizes. Note this does not include padding! " - "So sizes may differ from other tools.") - parser.add_argument('-Y', '--summary', action='store_true', - help="Only show the total code size.") - parser.add_argument('--type', default='tTrRdD', + parser.add_argument( + '--nm-types', + default=NM_TYPES, help="Type of symbols to report, this uses the same single-character " - "type-names emitted by nm. Defaults to %(default)r.") - parser.add_argument('--nm-tool', default=['nm'], type=lambda x: x.split(), - help="Path to the nm tool to use.") - parser.add_argument('--build-dir', - help="Specify the relative build directory. Used to map object files \ - to the correct source files.") - sys.exit(main(**vars(parser.parse_args()))) + "type-names emitted by nm. Defaults to %r." % NM_TYPES) + parser.add_argument( + '--nm-path', + type=lambda x: x.split(), + default=NM_PATH, + help="Path to the nm executable, may include flags. " + "Defaults to %r." % NM_PATH) + parser.add_argument( + '--objdump-path', + type=lambda x: x.split(), + default=OBJDUMP_PATH, + help="Path to the objdump executable, may include flags. " + "Defaults to %r." % OBJDUMP_PATH) + sys.exit(main(**{k: v + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/components/fs/littlefs/littlefs/scripts/cov.py b/components/fs/littlefs/littlefs/scripts/cov.py new file mode 100644 index 000000000..b61b2e524 --- /dev/null +++ b/components/fs/littlefs/littlefs/scripts/cov.py @@ -0,0 +1,828 @@ +#!/usr/bin/env python3 +# +# Script to find coverage info after running tests. +# +# Example: +# ./scripts/cov.py \ +# lfs.t.a.gcda lfs_util.t.a.gcda \ +# -Flfs.c -Flfs_util.c -slines +# +# Copyright (c) 2022, The littlefs authors. +# Copyright (c) 2020, Arm Limited. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause +# + +import collections as co +import csv +import itertools as it +import json +import math as m +import os +import re +import shlex +import subprocess as sp + +# TODO use explode_asserts to avoid counting assert branches? +# TODO use dwarf=info to find functions for inline functions? + +GCOV_PATH = ['gcov'] + + +# integer fields +class Int(co.namedtuple('Int', 'x')): + __slots__ = () + def __new__(cls, x=0): + if isinstance(x, Int): + return x + if isinstance(x, str): + try: + x = int(x, 0) + except ValueError: + # also accept +-∞ and +-inf + if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): + x = m.inf + elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): + x = -m.inf + else: + raise + assert isinstance(x, int) or m.isinf(x), x + return super().__new__(cls, x) + + def __str__(self): + if self.x == m.inf: + return '∞' + elif self.x == -m.inf: + return '-∞' + else: + return str(self.x) + + def __int__(self): + assert not m.isinf(self.x) + return self.x + + def __float__(self): + return float(self.x) + + none = '%7s' % '-' + def table(self): + return '%7s' % (self,) + + diff_none = '%7s' % '-' + diff_table = table + + def diff_diff(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + diff = new - old + if diff == +m.inf: + return '%7s' % '+∞' + elif diff == -m.inf: + return '%7s' % '-∞' + else: + return '%+7d' % diff + + def ratio(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + if m.isinf(new) and m.isinf(old): + return 0.0 + elif m.isinf(new): + return +m.inf + elif m.isinf(old): + return -m.inf + elif not old and not new: + return 0.0 + elif not old: + return 1.0 + else: + return (new-old) / old + + def __add__(self, other): + return self.__class__(self.x + other.x) + + def __sub__(self, other): + return self.__class__(self.x - other.x) + + def __mul__(self, other): + return self.__class__(self.x * other.x) + +# fractional fields, a/b +class Frac(co.namedtuple('Frac', 'a,b')): + __slots__ = () + def __new__(cls, a=0, b=None): + if isinstance(a, Frac) and b is None: + return a + if isinstance(a, str) and b is None: + a, b = a.split('/', 1) + if b is None: + b = a + return super().__new__(cls, Int(a), Int(b)) + + def __str__(self): + return '%s/%s' % (self.a, self.b) + + def __float__(self): + return float(self.a) + + none = '%11s %7s' % ('-', '-') + def table(self): + t = self.a.x/self.b.x if self.b.x else 1.0 + return '%11s %7s' % ( + self, + '∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%.1f%%' % (100*t)) + + diff_none = '%11s' % '-' + def diff_table(self): + return '%11s' % (self,) + + def diff_diff(self, other): + new_a, new_b = self if self else (Int(0), Int(0)) + old_a, old_b = other if other else (Int(0), Int(0)) + return '%11s' % ('%s/%s' % ( + new_a.diff_diff(old_a).strip(), + new_b.diff_diff(old_b).strip())) + + def ratio(self, other): + new_a, new_b = self if self else (Int(0), Int(0)) + old_a, old_b = other if other else (Int(0), Int(0)) + new = new_a.x/new_b.x if new_b.x else 1.0 + old = old_a.x/old_b.x if old_b.x else 1.0 + return new - old + + def __add__(self, other): + return self.__class__(self.a + other.a, self.b + other.b) + + def __sub__(self, other): + return self.__class__(self.a - other.a, self.b - other.b) + + def __mul__(self, other): + return self.__class__(self.a * other.a, self.b + other.b) + + def __lt__(self, other): + self_t = self.a.x/self.b.x if self.b.x else 1.0 + other_t = other.a.x/other.b.x if other.b.x else 1.0 + return (self_t, self.a.x) < (other_t, other.a.x) + + def __gt__(self, other): + return self.__class__.__lt__(other, self) + + def __le__(self, other): + return not self.__gt__(other) + + def __ge__(self, other): + return not self.__lt__(other) + +# coverage results +class CovResult(co.namedtuple('CovResult', [ + 'file', 'function', 'line', + 'calls', 'hits', 'funcs', 'lines', 'branches'])): + _by = ['file', 'function', 'line'] + _fields = ['calls', 'hits', 'funcs', 'lines', 'branches'] + _sort = ['funcs', 'lines', 'branches', 'hits', 'calls'] + _types = { + 'calls': Int, 'hits': Int, + 'funcs': Frac, 'lines': Frac, 'branches': Frac} + + __slots__ = () + def __new__(cls, file='', function='', line=0, + calls=0, hits=0, funcs=0, lines=0, branches=0): + return super().__new__(cls, file, function, int(Int(line)), + Int(calls), Int(hits), Frac(funcs), Frac(lines), Frac(branches)) + + def __add__(self, other): + return CovResult(self.file, self.function, self.line, + max(self.calls, other.calls), + max(self.hits, other.hits), + self.funcs + other.funcs, + self.lines + other.lines, + self.branches + other.branches) + + +def openio(path, mode='r', buffering=-1): + # allow '-' for stdin/stdout + if path == '-': + if mode == 'r': + return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering) + else: + return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering) + else: + return open(path, mode, buffering) + +def collect(gcda_paths, *, + gcov_path=GCOV_PATH, + sources=None, + everything=False, + **args): + results = [] + for path in gcda_paths: + # get coverage info through gcov's json output + # note, gcov-path may contain extra args + cmd = GCOV_PATH + ['-b', '-t', '--json-format', path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + data = json.load(proc.stdout) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + # collect line/branch coverage + for file in data['files']: + # ignore filtered sources + if sources is not None: + if not any( + os.path.abspath(file['file']) == os.path.abspath(s) + for s in sources): + continue + else: + # default to only cwd + if not everything and not os.path.commonpath([ + os.getcwd(), + os.path.abspath(file['file'])]) == os.getcwd(): + continue + + # simplify path + if os.path.commonpath([ + os.getcwd(), + os.path.abspath(file['file'])]) == os.getcwd(): + file_name = os.path.relpath(file['file']) + else: + file_name = os.path.abspath(file['file']) + + for func in file['functions']: + func_name = func.get('name', '(inlined)') + # discard internal functions (this includes injected test cases) + if not everything: + if func_name.startswith('__'): + continue + + # go ahead and add functions, later folding will merge this if + # there are other hits on this line + results.append(CovResult( + file_name, func_name, func['start_line'], + func['execution_count'], 0, + Frac(1 if func['execution_count'] > 0 else 0, 1), + 0, + 0)) + + for line in file['lines']: + func_name = line.get('function_name', '(inlined)') + # discard internal function (this includes injected test cases) + if not everything: + if func_name.startswith('__'): + continue + + # go ahead and add lines, later folding will merge this if + # there are other hits on this line + results.append(CovResult( + file_name, func_name, line['line_number'], + 0, line['count'], + 0, + Frac(1 if line['count'] > 0 else 0, 1), + Frac( + sum(1 if branch['count'] > 0 else 0 + for branch in line['branches']), + len(line['branches'])))) + + return results + + +def fold(Result, results, *, + by=None, + defines=None, + **_): + if by is None: + by = Result._by + + for k in it.chain(by or [], (k for k, _ in defines or [])): + if k not in Result._by and k not in Result._fields: + print("error: could not find field %r?" % k) + sys.exit(-1) + + # filter by matching defines + if defines is not None: + results_ = [] + for r in results: + if all(getattr(r, k) in vs for k, vs in defines): + results_.append(r) + results = results_ + + # organize results into conflicts + folding = co.OrderedDict() + for r in results: + name = tuple(getattr(r, k) for k in by) + if name not in folding: + folding[name] = [] + folding[name].append(r) + + # merge conflicts + folded = [] + for name, rs in folding.items(): + folded.append(sum(rs[1:], start=rs[0])) + + return folded + +def table(Result, results, diff_results=None, *, + by=None, + fields=None, + sort=None, + summary=False, + all=False, + percent=False, + **_): + all_, all = all, __builtins__.all + + if by is None: + by = Result._by + if fields is None: + fields = Result._fields + types = Result._types + + # fold again + results = fold(Result, results, by=by) + if diff_results is not None: + diff_results = fold(Result, diff_results, by=by) + + # organize by name + table = { + ','.join(str(getattr(r, k) or '') for k in by): r + for r in results} + diff_table = { + ','.join(str(getattr(r, k) or '') for k in by): r + for r in diff_results or []} + names = list(table.keys() | diff_table.keys()) + + # sort again, now with diff info, note that python's sort is stable + names.sort() + if diff_results is not None: + names.sort(key=lambda n: tuple( + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), + reverse=True) + if sort: + for k, reverse in reversed(sort): + names.sort( + key=lambda n: tuple( + (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None else () + for k in ([k] if k else [ + k for k in Result._sort if k in fields])), + reverse=reverse ^ (not k or k in Result._fields)) + + + # build up our lines + lines = [] + + # header + header = [] + header.append('%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else '') + if diff_results is None: + for k in fields: + header.append(k) + elif percent: + for k in fields: + header.append(k) + else: + for k in fields: + header.append('o'+k) + for k in fields: + header.append('n'+k) + for k in fields: + header.append('d'+k) + header.append('') + lines.append(header) + + def table_entry(name, r, diff_r=None, ratios=[]): + entry = [] + entry.append(name) + if diff_results is None: + for k in fields: + entry.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) + elif percent: + for k in fields: + entry.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + else: + for k in fields: + entry.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + entry.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + entry.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + entry.append('') + elif percent: + entry.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + entry.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + return entry + + # entries + if not summary: + for name in names: + r = table.get(name) + if diff_results is None: + diff_r = None + ratios = None + else: + diff_r = diff_table.get(name) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + if not all_ and not any(ratios): + continue + lines.append(table_entry(name, r, diff_r, ratios)) + + # total + r = next(iter(fold(Result, results, by=[])), None) + if diff_results is None: + diff_r = None + ratios = None + else: + diff_r = next(iter(fold(Result, diff_results, by=[])), None) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + lines.append(table_entry('TOTAL', r, diff_r, ratios)) + + # find the best widths, note that column 0 contains the names and column -1 + # the ratios, so those are handled a bit differently + widths = [ + ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1 + for w, i in zip( + it.chain([23], it.repeat(7)), + range(len(lines[0])-1))] + + # print our table + for line in lines: + print('%-*s %s%s' % ( + widths[0], line[0], + ' '.join('%*s' % (w, x) + for w, x in zip(widths[1:], line[1:-1])), + line[-1])) + + +def annotate(Result, results, *, + annotate=False, + lines=False, + branches=False, + **args): + # if neither branches/lines specified, color both + if annotate and not lines and not branches: + lines, branches = True, True + + for path in co.OrderedDict.fromkeys(r.file for r in results).keys(): + # flatten to line info + results = fold(Result, results, by=['file', 'line']) + table = {r.line: r for r in results if r.file == path} + + # calculate spans to show + if not annotate: + spans = [] + last = None + func = None + for line, r in sorted(table.items()): + if ((lines and int(r.hits) == 0) + or (branches and r.branches.a < r.branches.b)): + if last is not None and line - last.stop <= args['context']: + last = range( + last.start, + line+1+args['context']) + else: + if last is not None: + spans.append((last, func)) + last = range( + line-args['context'], + line+1+args['context']) + func = r.function + if last is not None: + spans.append((last, func)) + + with open(path) as f: + skipped = False + for i, line in enumerate(f): + # skip lines not in spans? + if not annotate and not any(i+1 in s for s, _ in spans): + skipped = True + continue + + if skipped: + skipped = False + print('%s@@ %s:%d: %s @@%s' % ( + '\x1b[36m' if args['color'] else '', + path, + i+1, + next(iter(f for _, f in spans)), + '\x1b[m' if args['color'] else '')) + + # build line + if line.endswith('\n'): + line = line[:-1] + + if i+1 in table: + r = table[i+1] + line = '%-*s // %s hits%s' % ( + args['width'], + line, + r.hits, + ', %s branches' % (r.branches,) + if int(r.branches.b) else '') + + if args['color']: + if lines and int(r.hits) == 0: + line = '\x1b[1;31m%s\x1b[m' % line + elif branches and r.branches.a < r.branches.b: + line = '\x1b[35m%s\x1b[m' % line + + print(line) + + +def main(gcda_paths, *, + by=None, + fields=None, + defines=None, + sort=None, + hits=False, + **args): + # figure out what color should be + if args.get('color') == 'auto': + args['color'] = sys.stdout.isatty() + elif args.get('color') == 'always': + args['color'] = True + else: + args['color'] = False + + # find sizes + if not args.get('use', None): + results = collect(gcda_paths, **args) + else: + results = [] + with openio(args['use']) as f: + reader = csv.DictReader(f, restval='') + for r in reader: + if not any('cov_'+k in r and r['cov_'+k].strip() + for k in CovResult._fields): + continue + try: + results.append(CovResult( + **{k: r[k] for k in CovResult._by + if k in r and r[k].strip()}, + **{k: r['cov_'+k] + for k in CovResult._fields + if 'cov_'+k in r + and r['cov_'+k].strip()})) + except TypeError: + pass + + # fold + results = fold(CovResult, results, by=by, defines=defines) + + # sort, note that python's sort is stable + results.sort() + if sort: + for k, reverse in reversed(sort): + results.sort( + key=lambda r: tuple( + (getattr(r, k),) if getattr(r, k) is not None else () + for k in ([k] if k else CovResult._sort)), + reverse=reverse ^ (not k or k in CovResult._fields)) + + # write results to CSV + if args.get('output'): + with openio(args['output'], 'w') as f: + writer = csv.DictWriter(f, + (by if by is not None else CovResult._by) + + ['cov_'+k for k in ( + fields if fields is not None else CovResult._fields)]) + writer.writeheader() + for r in results: + writer.writerow( + {k: getattr(r, k) for k in ( + by if by is not None else CovResult._by)} + | {'cov_'+k: getattr(r, k) for k in ( + fields if fields is not None else CovResult._fields)}) + + # find previous results? + if args.get('diff'): + diff_results = [] + try: + with openio(args['diff']) as f: + reader = csv.DictReader(f, restval='') + for r in reader: + if not any('cov_'+k in r and r['cov_'+k].strip() + for k in CovResult._fields): + continue + try: + diff_results.append(CovResult( + **{k: r[k] for k in CovResult._by + if k in r and r[k].strip()}, + **{k: r['cov_'+k] + for k in CovResult._fields + if 'cov_'+k in r + and r['cov_'+k].strip()})) + except TypeError: + pass + except FileNotFoundError: + pass + + # fold + diff_results = fold(CovResult, diff_results, + by=by, defines=defines) + + # print table + if not args.get('quiet'): + if (args.get('annotate') + or args.get('lines') + or args.get('branches')): + # annotate sources + annotate(CovResult, results, **args) + else: + # print table + table(CovResult, results, + diff_results if args.get('diff') else None, + by=by if by is not None else ['function'], + fields=fields if fields is not None + else ['lines', 'branches'] if not hits + else ['calls', 'hits'], + sort=sort, + **args) + + # catch lack of coverage + if args.get('error_on_lines') and any( + r.lines.a < r.lines.b for r in results): + sys.exit(2) + elif args.get('error_on_branches') and any( + r.branches.a < r.branches.b for r in results): + sys.exit(3) + + +if __name__ == "__main__": + import argparse + import sys + parser = argparse.ArgumentParser( + description="Find coverage info after running tests.", + allow_abbrev=False) + parser.add_argument( + 'gcda_paths', + nargs='*', + help="Input *.gcda files.") + parser.add_argument( + '-v', '--verbose', + action='store_true', + help="Output commands that run behind the scenes.") + parser.add_argument( + '-q', '--quiet', + action='store_true', + help="Don't show anything, useful with -o.") + parser.add_argument( + '-o', '--output', + help="Specify CSV file to store results.") + parser.add_argument( + '-u', '--use', + help="Don't parse anything, use this CSV file.") + parser.add_argument( + '-d', '--diff', + help="Specify CSV file to diff against.") + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") + parser.add_argument( + '-p', '--percent', + action='store_true', + help="Only show percentage change, not a full diff.") + parser.add_argument( + '-b', '--by', + action='append', + choices=CovResult._by, + help="Group by this field.") + parser.add_argument( + '-f', '--field', + dest='fields', + action='append', + choices=CovResult._fields, + help="Show this field.") + parser.add_argument( + '-D', '--define', + dest='defines', + action='append', + type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), + help="Only include results where this field is this value.") + class AppendSort(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.sort is None: + namespace.sort = [] + namespace.sort.append((value, True if option == '-S' else False)) + parser.add_argument( + '-s', '--sort', + nargs='?', + action=AppendSort, + help="Sort by this field.") + parser.add_argument( + '-S', '--reverse-sort', + nargs='?', + action=AppendSort, + help="Sort by this field, but backwards.") + parser.add_argument( + '-Y', '--summary', + action='store_true', + help="Only show the total.") + parser.add_argument( + '-F', '--source', + dest='sources', + action='append', + help="Only consider definitions in this file. Defaults to anything " + "in the current directory.") + parser.add_argument( + '--everything', + action='store_true', + help="Include builtin and libc specific symbols.") + parser.add_argument( + '--hits', + action='store_true', + help="Show total hits instead of coverage.") + parser.add_argument( + '-A', '--annotate', + action='store_true', + help="Show source files annotated with coverage info.") + parser.add_argument( + '-L', '--lines', + action='store_true', + help="Show uncovered lines.") + parser.add_argument( + '-B', '--branches', + action='store_true', + help="Show uncovered branches.") + parser.add_argument( + '-c', '--context', + type=lambda x: int(x, 0), + default=3, + help="Show n additional lines of context. Defaults to 3.") + parser.add_argument( + '-W', '--width', + type=lambda x: int(x, 0), + default=80, + help="Assume source is styled with this many columns. Defaults to 80.") + parser.add_argument( + '--color', + choices=['never', 'always', 'auto'], + default='auto', + help="When to use terminal colors. Defaults to 'auto'.") + parser.add_argument( + '-e', '--error-on-lines', + action='store_true', + help="Error if any lines are not covered.") + parser.add_argument( + '-E', '--error-on-branches', + action='store_true', + help="Error if any branches are not covered.") + parser.add_argument( + '--gcov-path', + default=GCOV_PATH, + type=lambda x: x.split(), + help="Path to the gcov executable, may include paths. " + "Defaults to %r." % GCOV_PATH) + sys.exit(main(**{k: v + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/components/fs/littlefs/littlefs/scripts/coverage.py b/components/fs/littlefs/littlefs/scripts/coverage.py deleted file mode 100755 index b3a90ed23..000000000 --- a/components/fs/littlefs/littlefs/scripts/coverage.py +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env python3 -# -# Parse and report coverage info from .info files generated by lcov -# -import os -import glob -import csv -import re -import collections as co -import bisect as b - - -INFO_PATHS = ['tests/*.toml.info'] - -def collect(paths, **args): - file = None - funcs = [] - lines = co.defaultdict(lambda: 0) - pattern = re.compile( - '^(?PSF:/?(?P.*))$' - '|^(?PFN:(?P[0-9]*),(?P.*))$' - '|^(?PDA:(?P[0-9]*),(?P[0-9]*))$') - for path in paths: - with open(path) as f: - for line in f: - m = pattern.match(line) - if m and m.group('file'): - file = m.group('file_name') - elif m and file and m.group('func'): - funcs.append((file, int(m.group('func_lineno')), - m.group('func_name'))) - elif m and file and m.group('line'): - lines[(file, int(m.group('line_lineno')))] += ( - int(m.group('line_hits'))) - - # map line numbers to functions - funcs.sort() - def func_from_lineno(file, lineno): - i = b.bisect(funcs, (file, lineno)) - if i and funcs[i-1][0] == file: - return funcs[i-1][2] - else: - return None - - # reduce to function info - reduced_funcs = co.defaultdict(lambda: (0, 0)) - for (file, line_lineno), line_hits in lines.items(): - func = func_from_lineno(file, line_lineno) - if not func: - continue - hits, count = reduced_funcs[(file, func)] - reduced_funcs[(file, func)] = (hits + (line_hits > 0), count + 1) - - results = [] - for (file, func), (hits, count) in reduced_funcs.items(): - # discard internal/testing functions (test_* injected with - # internal testing) - if not args.get('everything'): - if func.startswith('__') or func.startswith('test_'): - continue - # discard .8449 suffixes created by optimizer - func = re.sub('\.[0-9]+', '', func) - results.append((file, func, hits, count)) - - return results - - -def main(**args): - def openio(path, mode='r'): - if path == '-': - if 'r' in mode: - return os.fdopen(os.dup(sys.stdin.fileno()), 'r') - else: - return os.fdopen(os.dup(sys.stdout.fileno()), 'w') - else: - return open(path, mode) - - # find coverage - if not args.get('use'): - # find *.info files - paths = [] - for path in args['info_paths']: - if os.path.isdir(path): - path = path + '/*.gcov' - - for path in glob.glob(path): - paths.append(path) - - if not paths: - print('no .info files found in %r?' % args['info_paths']) - sys.exit(-1) - - results = collect(paths, **args) - else: - with openio(args['use']) as f: - r = csv.DictReader(f) - results = [ - ( result['file'], - result['name'], - int(result['coverage_hits']), - int(result['coverage_count'])) - for result in r - if result.get('coverage_hits') not in {None, ''} - if result.get('coverage_count') not in {None, ''}] - - total_hits, total_count = 0, 0 - for _, _, hits, count in results: - total_hits += hits - total_count += count - - # find previous results? - if args.get('diff'): - try: - with openio(args['diff']) as f: - r = csv.DictReader(f) - prev_results = [ - ( result['file'], - result['name'], - int(result['coverage_hits']), - int(result['coverage_count'])) - for result in r - if result.get('coverage_hits') not in {None, ''} - if result.get('coverage_count') not in {None, ''}] - except FileNotFoundError: - prev_results = [] - - prev_total_hits, prev_total_count = 0, 0 - for _, _, hits, count in prev_results: - prev_total_hits += hits - prev_total_count += count - - # write results to CSV - if args.get('output'): - merged_results = co.defaultdict(lambda: {}) - other_fields = [] - - # merge? - if args.get('merge'): - try: - with openio(args['merge']) as f: - r = csv.DictReader(f) - for result in r: - file = result.pop('file', '') - func = result.pop('name', '') - result.pop('coverage_hits', None) - result.pop('coverage_count', None) - merged_results[(file, func)] = result - other_fields = result.keys() - except FileNotFoundError: - pass - - for file, func, hits, count in results: - merged_results[(file, func)]['coverage_hits'] = hits - merged_results[(file, func)]['coverage_count'] = count - - with openio(args['output'], 'w') as f: - w = csv.DictWriter(f, ['file', 'name', *other_fields, 'coverage_hits', 'coverage_count']) - w.writeheader() - for (file, func), result in sorted(merged_results.items()): - w.writerow({'file': file, 'name': func, **result}) - - # print results - def dedup_entries(results, by='name'): - entries = co.defaultdict(lambda: (0, 0)) - for file, func, hits, count in results: - entry = (file if by == 'file' else func) - entry_hits, entry_count = entries[entry] - entries[entry] = (entry_hits + hits, entry_count + count) - return entries - - def diff_entries(olds, news): - diff = co.defaultdict(lambda: (0, 0, 0, 0, 0, 0, 0)) - for name, (new_hits, new_count) in news.items(): - diff[name] = ( - 0, 0, - new_hits, new_count, - new_hits, new_count, - (new_hits/new_count if new_count else 1.0) - 1.0) - for name, (old_hits, old_count) in olds.items(): - _, _, new_hits, new_count, _, _, _ = diff[name] - diff[name] = ( - old_hits, old_count, - new_hits, new_count, - new_hits-old_hits, new_count-old_count, - ((new_hits/new_count if new_count else 1.0) - - (old_hits/old_count if old_count else 1.0))) - return diff - - def sorted_entries(entries): - if args.get('coverage_sort'): - return sorted(entries, key=lambda x: (-(x[1][0]/x[1][1] if x[1][1] else -1), x)) - elif args.get('reverse_coverage_sort'): - return sorted(entries, key=lambda x: (+(x[1][0]/x[1][1] if x[1][1] else -1), x)) - else: - return sorted(entries) - - def sorted_diff_entries(entries): - if args.get('coverage_sort'): - return sorted(entries, key=lambda x: (-(x[1][2]/x[1][3] if x[1][3] else -1), x)) - elif args.get('reverse_coverage_sort'): - return sorted(entries, key=lambda x: (+(x[1][2]/x[1][3] if x[1][3] else -1), x)) - else: - return sorted(entries, key=lambda x: (-x[1][6], x)) - - def print_header(by=''): - if not args.get('diff'): - print('%-36s %19s' % (by, 'hits/line')) - else: - print('%-36s %19s %19s %11s' % (by, 'old', 'new', 'diff')) - - def print_entry(name, hits, count): - print("%-36s %11s %7s" % (name, - '%d/%d' % (hits, count) - if count else '-', - '%.1f%%' % (100*hits/count) - if count else '-')) - - def print_diff_entry(name, - old_hits, old_count, - new_hits, new_count, - diff_hits, diff_count, - ratio): - print("%-36s %11s %7s %11s %7s %11s%s" % (name, - '%d/%d' % (old_hits, old_count) - if old_count else '-', - '%.1f%%' % (100*old_hits/old_count) - if old_count else '-', - '%d/%d' % (new_hits, new_count) - if new_count else '-', - '%.1f%%' % (100*new_hits/new_count) - if new_count else '-', - '%+d/%+d' % (diff_hits, diff_count), - ' (%+.1f%%)' % (100*ratio) if ratio else '')) - - def print_entries(by='name'): - entries = dedup_entries(results, by=by) - - if not args.get('diff'): - print_header(by=by) - for name, (hits, count) in sorted_entries(entries.items()): - print_entry(name, hits, count) - else: - prev_entries = dedup_entries(prev_results, by=by) - diff = diff_entries(prev_entries, entries) - print_header(by='%s (%d added, %d removed)' % (by, - sum(1 for _, old, _, _, _, _, _ in diff.values() if not old), - sum(1 for _, _, _, new, _, _, _ in diff.values() if not new))) - for name, ( - old_hits, old_count, - new_hits, new_count, - diff_hits, diff_count, ratio) in sorted_diff_entries( - diff.items()): - if ratio or args.get('all'): - print_diff_entry(name, - old_hits, old_count, - new_hits, new_count, - diff_hits, diff_count, - ratio) - - def print_totals(): - if not args.get('diff'): - print_entry('TOTAL', total_hits, total_count) - else: - ratio = ((total_hits/total_count - if total_count else 1.0) - - (prev_total_hits/prev_total_count - if prev_total_count else 1.0)) - print_diff_entry('TOTAL', - prev_total_hits, prev_total_count, - total_hits, total_count, - total_hits-prev_total_hits, total_count-prev_total_count, - ratio) - - if args.get('quiet'): - pass - elif args.get('summary'): - print_header() - print_totals() - elif args.get('files'): - print_entries(by='file') - print_totals() - else: - print_entries(by='name') - print_totals() - -if __name__ == "__main__": - import argparse - import sys - parser = argparse.ArgumentParser( - description="Parse and report coverage info from .info files \ - generated by lcov") - parser.add_argument('info_paths', nargs='*', default=INFO_PATHS, - help="Description of where to find *.info files. May be a directory \ - or list of paths. *.info files will be merged to show the total \ - coverage. Defaults to %r." % INFO_PATHS) - parser.add_argument('-v', '--verbose', action='store_true', - help="Output commands that run behind the scenes.") - parser.add_argument('-o', '--output', - help="Specify CSV file to store results.") - parser.add_argument('-u', '--use', - help="Don't do any work, instead use this CSV file.") - parser.add_argument('-d', '--diff', - help="Specify CSV file to diff code size against.") - parser.add_argument('-m', '--merge', - help="Merge with an existing CSV file when writing to output.") - parser.add_argument('-a', '--all', action='store_true', - help="Show all functions, not just the ones that changed.") - parser.add_argument('-A', '--everything', action='store_true', - help="Include builtin and libc specific symbols.") - parser.add_argument('-s', '--coverage-sort', action='store_true', - help="Sort by coverage.") - parser.add_argument('-S', '--reverse-coverage-sort', action='store_true', - help="Sort by coverage, but backwards.") - parser.add_argument('-F', '--files', action='store_true', - help="Show file-level coverage.") - parser.add_argument('-Y', '--summary', action='store_true', - help="Only show the total coverage.") - parser.add_argument('-q', '--quiet', action='store_true', - help="Don't show anything, useful with -o.") - parser.add_argument('--build-dir', - help="Specify the relative build directory. Used to map object files \ - to the correct source files.") - sys.exit(main(**vars(parser.parse_args()))) diff --git a/components/fs/littlefs/littlefs/scripts/data.py b/components/fs/littlefs/littlefs/scripts/data.py old mode 100755 new mode 100644 index 4b8e00da6..e9770aa1e --- a/components/fs/littlefs/littlefs/scripts/data.py +++ b/components/fs/littlefs/littlefs/scripts/data.py @@ -1,42 +1,188 @@ #!/usr/bin/env python3 # -# Script to find data size at the function level. Basically just a bit wrapper +# Script to find data size at the function level. Basically just a big wrapper # around nm with some extra conveniences for comparing builds. Heavily inspired # by Linux's Bloat-O-Meter. # +# Example: +# ./scripts/data.py lfs.o lfs_util.o -Ssize +# +# Copyright (c) 2022, The littlefs authors. +# Copyright (c) 2020, Arm Limited. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause +# -import os -import glob +import collections as co +import csv +import difflib import itertools as it -import subprocess as sp -import shlex +import math as m +import os import re -import csv -import collections as co +import shlex +import subprocess as sp + + +NM_PATH = ['nm'] +NM_TYPES = 'dDbB' +OBJDUMP_PATH = ['objdump'] + + +# integer fields +class Int(co.namedtuple('Int', 'x')): + __slots__ = () + def __new__(cls, x=0): + if isinstance(x, Int): + return x + if isinstance(x, str): + try: + x = int(x, 0) + except ValueError: + # also accept +-∞ and +-inf + if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): + x = m.inf + elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): + x = -m.inf + else: + raise + assert isinstance(x, int) or m.isinf(x), x + return super().__new__(cls, x) + + def __str__(self): + if self.x == m.inf: + return '∞' + elif self.x == -m.inf: + return '-∞' + else: + return str(self.x) + + def __int__(self): + assert not m.isinf(self.x) + return self.x + + def __float__(self): + return float(self.x) + + none = '%7s' % '-' + def table(self): + return '%7s' % (self,) + + diff_none = '%7s' % '-' + diff_table = table + + def diff_diff(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + diff = new - old + if diff == +m.inf: + return '%7s' % '+∞' + elif diff == -m.inf: + return '%7s' % '-∞' + else: + return '%+7d' % diff + + def ratio(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + if m.isinf(new) and m.isinf(old): + return 0.0 + elif m.isinf(new): + return +m.inf + elif m.isinf(old): + return -m.inf + elif not old and not new: + return 0.0 + elif not old: + return 1.0 + else: + return (new-old) / old + + def __add__(self, other): + return self.__class__(self.x + other.x) + + def __sub__(self, other): + return self.__class__(self.x - other.x) + + def __mul__(self, other): + return self.__class__(self.x * other.x) +# data size results +class DataResult(co.namedtuple('DataResult', [ + 'file', 'function', + 'size'])): + _by = ['file', 'function'] + _fields = ['size'] + _sort = ['size'] + _types = {'size': Int} -OBJ_PATHS = ['*.o'] + __slots__ = () + def __new__(cls, file='', function='', size=0): + return super().__new__(cls, file, function, + Int(size)) -def collect(paths, **args): - results = co.defaultdict(lambda: 0) - pattern = re.compile( + def __add__(self, other): + return DataResult(self.file, self.function, + self.size + other.size) + + +def openio(path, mode='r', buffering=-1): + # allow '-' for stdin/stdout + if path == '-': + if mode == 'r': + return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering) + else: + return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering) + else: + return open(path, mode, buffering) + +def collect(obj_paths, *, + nm_path=NM_PATH, + nm_types=NM_TYPES, + objdump_path=OBJDUMP_PATH, + sources=None, + everything=False, + **args): + size_pattern = re.compile( '^(?P[0-9a-fA-F]+)' + - ' (?P[%s])' % re.escape(args['type']) + + ' (?P[%s])' % re.escape(nm_types) + ' (?P.+?)$') - for path in paths: - # note nm-tool may contain extra args - cmd = args['nm_tool'] + ['--size-sort', path] + line_pattern = re.compile( + '^\s+(?P[0-9]+)' + '(?:\s+(?P[0-9]+))?' + '\s+.*' + '\s+(?P[^\s]+)$') + info_pattern = re.compile( + '^(?:.*(?PDW_TAG_[a-z_]+).*' + '|.*DW_AT_name.*:\s*(?P[^:\s]+)\s*' + '|.*DW_AT_decl_file.*:\s*(?P[0-9]+)\s*)$') + + results = [] + for path in obj_paths: + # guess the source, if we have debug-info we'll replace this later + file = re.sub('(\.o)?$', '.c', path, 1) + + # find symbol sizes + results_ = [] + # note nm-path may contain extra args + cmd = nm_path + ['--size-sort', path] if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE if not args.get('verbose') else None, universal_newlines=True, - errors='replace') + errors='replace', + close_fds=False) for line in proc.stdout: - m = pattern.match(line) + m = size_pattern.match(line) if m: - results[(path, m.group('func'))] += int(m.group('size'), 16) + func = m.group('func') + # discard internal functions + if not everything and func.startswith('__'): + continue + results_.append(DataResult( + file, func, + int(m.group('size'), 16))) proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -44,240 +190,515 @@ def collect(paths, **args): sys.stdout.write(line) sys.exit(-1) - flat_results = [] - for (file, func), size in results.items(): - # map to source files - if args.get('build_dir'): - file = re.sub('%s/*' % re.escape(args['build_dir']), '', file) - # replace .o with .c, different scripts report .o/.c, we need to - # choose one if we want to deduplicate csv files - file = re.sub('\.o$', '.c', file) - # discard internal functions - if not args.get('everything'): - if func.startswith('__'): - continue - # discard .8449 suffixes created by optimizer - func = re.sub('\.[0-9]+', '', func) - flat_results.append((file, func, size)) - - return flat_results - -def main(**args): - def openio(path, mode='r'): - if path == '-': - if 'r' in mode: - return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + + # try to figure out the source file if we have debug-info + dirs = {} + files = {} + # note objdump-path may contain extra args + cmd = objdump_path + ['--dwarf=rawline', path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + for line in proc.stdout: + # note that files contain references to dirs, which we + # dereference as soon as we see them as each file table follows a + # dir table + m = line_pattern.match(line) + if m: + if not m.group('dir'): + # found a directory entry + dirs[int(m.group('no'))] = m.group('path') + else: + # found a file entry + dir = int(m.group('dir')) + if dir in dirs: + files[int(m.group('no'))] = os.path.join( + dirs[dir], + m.group('path')) + else: + files[int(m.group('no'))] = m.group('path') + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + # do nothing on error, we don't need objdump to work, source files + # may just be inaccurate + pass + + defs = {} + is_func = False + f_name = None + f_file = None + # note objdump-path may contain extra args + cmd = objdump_path + ['--dwarf=info', path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + for line in proc.stdout: + # state machine here to find definitions + m = info_pattern.match(line) + if m: + if m.group('tag'): + if is_func: + defs[f_name] = files.get(f_file, '?') + is_func = (m.group('tag') == 'DW_TAG_subprogram') + elif m.group('name'): + f_name = m.group('name') + elif m.group('file'): + f_file = int(m.group('file')) + if is_func: + defs[f_name] = files.get(f_file, '?') + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + # do nothing on error, we don't need objdump to work, source files + # may just be inaccurate + pass + + for r in results_: + # find best matching debug symbol, this may be slightly different + # due to optimizations + if defs: + # exact match? avoid difflib if we can for speed + if r.function in defs: + file = defs[r.function] + else: + _, file = max( + defs.items(), + key=lambda d: difflib.SequenceMatcher(None, + d[0], + r.function, False).ratio()) else: - return os.fdopen(os.dup(sys.stdout.fileno()), 'w') - else: - return open(path, mode) + file = r.file - # find sizes - if not args.get('use', None): - # find .o files - paths = [] - for path in args['obj_paths']: - if os.path.isdir(path): - path = path + '/*.o' + # ignore filtered sources + if sources is not None: + if not any( + os.path.abspath(file) == os.path.abspath(s) + for s in sources): + continue + else: + # default to only cwd + if not everything and not os.path.commonpath([ + os.getcwd(), + os.path.abspath(file)]) == os.getcwd(): + continue + + # simplify path + if os.path.commonpath([ + os.getcwd(), + os.path.abspath(file)]) == os.getcwd(): + file = os.path.relpath(file) + else: + file = os.path.abspath(file) + + results.append(r._replace(file=file)) + + return results - for path in glob.glob(path): - paths.append(path) - if not paths: - print('no .obj files found in %r?' % args['obj_paths']) +def fold(Result, results, *, + by=None, + defines=None, + **_): + if by is None: + by = Result._by + + for k in it.chain(by or [], (k for k, _ in defines or [])): + if k not in Result._by and k not in Result._fields: + print("error: could not find field %r?" % k) sys.exit(-1) - results = collect(paths, **args) - else: - with openio(args['use']) as f: - r = csv.DictReader(f) - results = [ - ( result['file'], - result['name'], - int(result['data_size'])) - for result in r - if result.get('data_size') not in {None, ''}] - - total = 0 - for _, _, size in results: - total += size + # filter by matching defines + if defines is not None: + results_ = [] + for r in results: + if all(getattr(r, k) in vs for k, vs in defines): + results_.append(r) + results = results_ - # find previous results? - if args.get('diff'): - try: - with openio(args['diff']) as f: - r = csv.DictReader(f) - prev_results = [ - ( result['file'], - result['name'], - int(result['data_size'])) - for result in r - if result.get('data_size') not in {None, ''}] - except FileNotFoundError: - prev_results = [] + # organize results into conflicts + folding = co.OrderedDict() + for r in results: + name = tuple(getattr(r, k) for k in by) + if name not in folding: + folding[name] = [] + folding[name].append(r) - prev_total = 0 - for _, _, size in prev_results: - prev_total += size + # merge conflicts + folded = [] + for name, rs in folding.items(): + folded.append(sum(rs[1:], start=rs[0])) - # write results to CSV - if args.get('output'): - merged_results = co.defaultdict(lambda: {}) - other_fields = [] + return folded - # merge? - if args.get('merge'): - try: - with openio(args['merge']) as f: - r = csv.DictReader(f) - for result in r: - file = result.pop('file', '') - func = result.pop('name', '') - result.pop('data_size', None) - merged_results[(file, func)] = result - other_fields = result.keys() - except FileNotFoundError: - pass - - for file, func, size in results: - merged_results[(file, func)]['data_size'] = size +def table(Result, results, diff_results=None, *, + by=None, + fields=None, + sort=None, + summary=False, + all=False, + percent=False, + **_): + all_, all = all, __builtins__.all - with openio(args['output'], 'w') as f: - w = csv.DictWriter(f, ['file', 'name', *other_fields, 'data_size']) - w.writeheader() - for (file, func), result in sorted(merged_results.items()): - w.writerow({'file': file, 'name': func, **result}) - - # print results - def dedup_entries(results, by='name'): - entries = co.defaultdict(lambda: 0) - for file, func, size in results: - entry = (file if by == 'file' else func) - entries[entry] += size - return entries - - def diff_entries(olds, news): - diff = co.defaultdict(lambda: (0, 0, 0, 0)) - for name, new in news.items(): - diff[name] = (0, new, new, 1.0) - for name, old in olds.items(): - _, new, _, _ = diff[name] - diff[name] = (old, new, new-old, (new-old)/old if old else 1.0) - return diff - - def sorted_entries(entries): - if args.get('size_sort'): - return sorted(entries, key=lambda x: (-x[1], x)) - elif args.get('reverse_size_sort'): - return sorted(entries, key=lambda x: (+x[1], x)) - else: - return sorted(entries) + if by is None: + by = Result._by + if fields is None: + fields = Result._fields + types = Result._types - def sorted_diff_entries(entries): - if args.get('size_sort'): - return sorted(entries, key=lambda x: (-x[1][1], x)) - elif args.get('reverse_size_sort'): - return sorted(entries, key=lambda x: (+x[1][1], x)) - else: - return sorted(entries, key=lambda x: (-x[1][3], x)) + # fold again + results = fold(Result, results, by=by) + if diff_results is not None: + diff_results = fold(Result, diff_results, by=by) - def print_header(by=''): - if not args.get('diff'): - print('%-36s %7s' % (by, 'size')) - else: - print('%-36s %7s %7s %7s' % (by, 'old', 'new', 'diff')) + # organize by name + table = { + ','.join(str(getattr(r, k) or '') for k in by): r + for r in results} + diff_table = { + ','.join(str(getattr(r, k) or '') for k in by): r + for r in diff_results or []} + names = list(table.keys() | diff_table.keys()) + + # sort again, now with diff info, note that python's sort is stable + names.sort() + if diff_results is not None: + names.sort(key=lambda n: tuple( + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), + reverse=True) + if sort: + for k, reverse in reversed(sort): + names.sort( + key=lambda n: tuple( + (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None else () + for k in ([k] if k else [ + k for k in Result._sort if k in fields])), + reverse=reverse ^ (not k or k in Result._fields)) - def print_entry(name, size): - print("%-36s %7d" % (name, size)) - def print_diff_entry(name, old, new, diff, ratio): - print("%-36s %7s %7s %+7d%s" % (name, - old or "-", - new or "-", - diff, - ' (%+.1f%%)' % (100*ratio) if ratio else '')) + # build up our lines + lines = [] - def print_entries(by='name'): - entries = dedup_entries(results, by=by) + # header + header = [] + header.append('%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else '') + if diff_results is None: + for k in fields: + header.append(k) + elif percent: + for k in fields: + header.append(k) + else: + for k in fields: + header.append('o'+k) + for k in fields: + header.append('n'+k) + for k in fields: + header.append('d'+k) + header.append('') + lines.append(header) - if not args.get('diff'): - print_header(by=by) - for name, size in sorted_entries(entries.items()): - print_entry(name, size) + def table_entry(name, r, diff_r=None, ratios=[]): + entry = [] + entry.append(name) + if diff_results is None: + for k in fields: + entry.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) + elif percent: + for k in fields: + entry.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - prev_entries = dedup_entries(prev_results, by=by) - diff = diff_entries(prev_entries, entries) - print_header(by='%s (%d added, %d removed)' % (by, - sum(1 for old, _, _, _ in diff.values() if not old), - sum(1 for _, new, _, _ in diff.values() if not new))) - for name, (old, new, diff, ratio) in sorted_diff_entries( - diff.items()): - if ratio or args.get('all'): - print_diff_entry(name, old, new, diff, ratio) - - def print_totals(): - if not args.get('diff'): - print_entry('TOTAL', total) + for k in fields: + entry.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + entry.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + entry.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + entry.append('') + elif percent: + entry.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) else: - ratio = (0.0 if not prev_total and not total - else 1.0 if not prev_total - else (total-prev_total)/prev_total) - print_diff_entry('TOTAL', - prev_total, total, - total-prev_total, - ratio) - - if args.get('quiet'): - pass - elif args.get('summary'): - print_header() - print_totals() - elif args.get('files'): - print_entries(by='file') - print_totals() + entry.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + return entry + + # entries + if not summary: + for name in names: + r = table.get(name) + if diff_results is None: + diff_r = None + ratios = None + else: + diff_r = diff_table.get(name) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + if not all_ and not any(ratios): + continue + lines.append(table_entry(name, r, diff_r, ratios)) + + # total + r = next(iter(fold(Result, results, by=[])), None) + if diff_results is None: + diff_r = None + ratios = None else: - print_entries(by='name') - print_totals() + diff_r = next(iter(fold(Result, diff_results, by=[])), None) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + lines.append(table_entry('TOTAL', r, diff_r, ratios)) + + # find the best widths, note that column 0 contains the names and column -1 + # the ratios, so those are handled a bit differently + widths = [ + ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1 + for w, i in zip( + it.chain([23], it.repeat(7)), + range(len(lines[0])-1))] + + # print our table + for line in lines: + print('%-*s %s%s' % ( + widths[0], line[0], + ' '.join('%*s' % (w, x) + for w, x in zip(widths[1:], line[1:-1])), + line[-1])) + + +def main(obj_paths, *, + by=None, + fields=None, + defines=None, + sort=None, + **args): + # find sizes + if not args.get('use', None): + results = collect(obj_paths, **args) + else: + results = [] + with openio(args['use']) as f: + reader = csv.DictReader(f, restval='') + for r in reader: + try: + results.append(DataResult( + **{k: r[k] for k in DataResult._by + if k in r and r[k].strip()}, + **{k: r['data_'+k] for k in DataResult._fields + if 'data_'+k in r and r['data_'+k].strip()})) + except TypeError: + pass + + # fold + results = fold(DataResult, results, by=by, defines=defines) + + # sort, note that python's sort is stable + results.sort() + if sort: + for k, reverse in reversed(sort): + results.sort( + key=lambda r: tuple( + (getattr(r, k),) if getattr(r, k) is not None else () + for k in ([k] if k else DataResult._sort)), + reverse=reverse ^ (not k or k in DataResult._fields)) + + # write results to CSV + if args.get('output'): + with openio(args['output'], 'w') as f: + writer = csv.DictWriter(f, + (by if by is not None else DataResult._by) + + ['data_'+k for k in ( + fields if fields is not None else DataResult._fields)]) + writer.writeheader() + for r in results: + writer.writerow( + {k: getattr(r, k) for k in ( + by if by is not None else DataResult._by)} + | {'data_'+k: getattr(r, k) for k in ( + fields if fields is not None else DataResult._fields)}) + + # find previous results? + if args.get('diff'): + diff_results = [] + try: + with openio(args['diff']) as f: + reader = csv.DictReader(f, restval='') + for r in reader: + if not any('data_'+k in r and r['data_'+k].strip() + for k in DataResult._fields): + continue + try: + diff_results.append(DataResult( + **{k: r[k] for k in DataResult._by + if k in r and r[k].strip()}, + **{k: r['data_'+k] for k in DataResult._fields + if 'data_'+k in r and r['data_'+k].strip()})) + except TypeError: + pass + except FileNotFoundError: + pass + + # fold + diff_results = fold(DataResult, diff_results, by=by, defines=defines) + + # print table + if not args.get('quiet'): + table(DataResult, results, + diff_results if args.get('diff') else None, + by=by if by is not None else ['function'], + fields=fields, + sort=sort, + **args) + if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Find data size at the function level.") - parser.add_argument('obj_paths', nargs='*', default=OBJ_PATHS, - help="Description of where to find *.o files. May be a directory \ - or a list of paths. Defaults to %r." % OBJ_PATHS) - parser.add_argument('-v', '--verbose', action='store_true', + description="Find data size at the function level.", + allow_abbrev=False) + parser.add_argument( + 'obj_paths', + nargs='*', + help="Input *.o files.") + parser.add_argument( + '-v', '--verbose', + action='store_true', help="Output commands that run behind the scenes.") - parser.add_argument('-q', '--quiet', action='store_true', + parser.add_argument( + '-q', '--quiet', + action='store_true', help="Don't show anything, useful with -o.") - parser.add_argument('-o', '--output', + parser.add_argument( + '-o', '--output', help="Specify CSV file to store results.") - parser.add_argument('-u', '--use', - help="Don't compile and find data sizes, instead use this CSV file.") - parser.add_argument('-d', '--diff', - help="Specify CSV file to diff data size against.") - parser.add_argument('-m', '--merge', - help="Merge with an existing CSV file when writing to output.") - parser.add_argument('-a', '--all', action='store_true', - help="Show all functions, not just the ones that changed.") - parser.add_argument('-A', '--everything', action='store_true', + parser.add_argument( + '-u', '--use', + help="Don't parse anything, use this CSV file.") + parser.add_argument( + '-d', '--diff', + help="Specify CSV file to diff against.") + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") + parser.add_argument( + '-p', '--percent', + action='store_true', + help="Only show percentage change, not a full diff.") + parser.add_argument( + '-b', '--by', + action='append', + choices=DataResult._by, + help="Group by this field.") + parser.add_argument( + '-f', '--field', + dest='fields', + action='append', + choices=DataResult._fields, + help="Show this field.") + parser.add_argument( + '-D', '--define', + dest='defines', + action='append', + type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), + help="Only include results where this field is this value.") + class AppendSort(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.sort is None: + namespace.sort = [] + namespace.sort.append((value, True if option == '-S' else False)) + parser.add_argument( + '-s', '--sort', + nargs='?', + action=AppendSort, + help="Sort by this field.") + parser.add_argument( + '-S', '--reverse-sort', + nargs='?', + action=AppendSort, + help="Sort by this field, but backwards.") + parser.add_argument( + '-Y', '--summary', + action='store_true', + help="Only show the total.") + parser.add_argument( + '-F', '--source', + dest='sources', + action='append', + help="Only consider definitions in this file. Defaults to anything " + "in the current directory.") + parser.add_argument( + '--everything', + action='store_true', help="Include builtin and libc specific symbols.") - parser.add_argument('-s', '--size-sort', action='store_true', - help="Sort by size.") - parser.add_argument('-S', '--reverse-size-sort', action='store_true', - help="Sort by size, but backwards.") - parser.add_argument('-F', '--files', action='store_true', - help="Show file-level data sizes. Note this does not include padding! " - "So sizes may differ from other tools.") - parser.add_argument('-Y', '--summary', action='store_true', - help="Only show the total data size.") - parser.add_argument('--type', default='dDbB', + parser.add_argument( + '--nm-types', + default=NM_TYPES, help="Type of symbols to report, this uses the same single-character " - "type-names emitted by nm. Defaults to %(default)r.") - parser.add_argument('--nm-tool', default=['nm'], type=lambda x: x.split(), - help="Path to the nm tool to use.") - parser.add_argument('--build-dir', - help="Specify the relative build directory. Used to map object files \ - to the correct source files.") - sys.exit(main(**vars(parser.parse_args()))) + "type-names emitted by nm. Defaults to %r." % NM_TYPES) + parser.add_argument( + '--nm-path', + type=lambda x: x.split(), + default=NM_PATH, + help="Path to the nm executable, may include flags. " + "Defaults to %r." % NM_PATH) + parser.add_argument( + '--objdump-path', + type=lambda x: x.split(), + default=OBJDUMP_PATH, + help="Path to the objdump executable, may include flags. " + "Defaults to %r." % OBJDUMP_PATH) + sys.exit(main(**{k: v + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/components/fs/littlefs/littlefs/scripts/explode_asserts.py b/components/fs/littlefs/littlefs/scripts/explode_asserts.py deleted file mode 100755 index 8a8e5b1c0..000000000 --- a/components/fs/littlefs/littlefs/scripts/explode_asserts.py +++ /dev/null @@ -1,383 +0,0 @@ -#!/usr/bin/env python3 - -import re -import sys - -PATTERN = ['LFS_ASSERT', 'assert'] -PREFIX = 'LFS' -MAXWIDTH = 16 - -ASSERT = "__{PREFIX}_ASSERT_{TYPE}_{COMP}" -FAIL = """ -__attribute__((unused)) -static void __{prefix}_assert_fail_{type}( - const char *file, int line, const char *comp, - {ctype} lh, size_t lsize, - {ctype} rh, size_t rsize) {{ - printf("%s:%d:assert: assert failed with ", file, line); - __{prefix}_assert_print_{type}(lh, lsize); - printf(", expected %s ", comp); - __{prefix}_assert_print_{type}(rh, rsize); - printf("\\n"); - fflush(NULL); - raise(SIGABRT); -}} -""" - -COMP = { - '==': 'eq', - '!=': 'ne', - '<=': 'le', - '>=': 'ge', - '<': 'lt', - '>': 'gt', -} - -TYPE = { - 'int': { - 'ctype': 'intmax_t', - 'fail': FAIL, - 'print': """ - __attribute__((unused)) - static void __{prefix}_assert_print_{type}({ctype} v, size_t size) {{ - (void)size; - printf("%"PRIiMAX, v); - }} - """, - 'assert': """ - #define __{PREFIX}_ASSERT_{TYPE}_{COMP}(file, line, lh, rh) - do {{ - __typeof__(lh) _lh = lh; - __typeof__(lh) _rh = (__typeof__(lh))rh; - if (!(_lh {op} _rh)) {{ - __{prefix}_assert_fail_{type}(file, line, "{comp}", - (intmax_t)_lh, 0, (intmax_t)_rh, 0); - }} - }} while (0) - """ - }, - 'bool': { - 'ctype': 'bool', - 'fail': FAIL, - 'print': """ - __attribute__((unused)) - static void __{prefix}_assert_print_{type}({ctype} v, size_t size) {{ - (void)size; - printf("%s", v ? "true" : "false"); - }} - """, - 'assert': """ - #define __{PREFIX}_ASSERT_{TYPE}_{COMP}(file, line, lh, rh) - do {{ - bool _lh = !!(lh); - bool _rh = !!(rh); - if (!(_lh {op} _rh)) {{ - __{prefix}_assert_fail_{type}(file, line, "{comp}", - _lh, 0, _rh, 0); - }} - }} while (0) - """ - }, - 'mem': { - 'ctype': 'const void *', - 'fail': FAIL, - 'print': """ - __attribute__((unused)) - static void __{prefix}_assert_print_{type}({ctype} v, size_t size) {{ - const uint8_t *s = v; - printf("\\\""); - for (size_t i = 0; i < size && i < {maxwidth}; i++) {{ - if (s[i] >= ' ' && s[i] <= '~') {{ - printf("%c", s[i]); - }} else {{ - printf("\\\\x%02x", s[i]); - }} - }} - if (size > {maxwidth}) {{ - printf("..."); - }} - printf("\\\""); - }} - """, - 'assert': """ - #define __{PREFIX}_ASSERT_{TYPE}_{COMP}(file, line, lh, rh, size) - do {{ - const void *_lh = lh; - const void *_rh = rh; - if (!(memcmp(_lh, _rh, size) {op} 0)) {{ - __{prefix}_assert_fail_{type}(file, line, "{comp}", - _lh, size, _rh, size); - }} - }} while (0) - """ - }, - 'str': { - 'ctype': 'const char *', - 'fail': FAIL, - 'print': """ - __attribute__((unused)) - static void __{prefix}_assert_print_{type}({ctype} v, size_t size) {{ - __{prefix}_assert_print_mem(v, size); - }} - """, - 'assert': """ - #define __{PREFIX}_ASSERT_{TYPE}_{COMP}(file, line, lh, rh) - do {{ - const char *_lh = lh; - const char *_rh = rh; - if (!(strcmp(_lh, _rh) {op} 0)) {{ - __{prefix}_assert_fail_{type}(file, line, "{comp}", - _lh, strlen(_lh), _rh, strlen(_rh)); - }} - }} while (0) - """ - } -} - -def mkdecls(outf, maxwidth=16): - outf.write("#include \n") - outf.write("#include \n") - outf.write("#include \n") - outf.write("#include \n") - outf.write("#include \n") - - for type, desc in sorted(TYPE.items()): - format = { - 'type': type.lower(), 'TYPE': type.upper(), - 'ctype': desc['ctype'], - 'prefix': PREFIX.lower(), 'PREFIX': PREFIX.upper(), - 'maxwidth': maxwidth, - } - outf.write(re.sub('\s+', ' ', - desc['print'].strip().format(**format))+'\n') - outf.write(re.sub('\s+', ' ', - desc['fail'].strip().format(**format))+'\n') - - for op, comp in sorted(COMP.items()): - format.update({ - 'comp': comp.lower(), 'COMP': comp.upper(), - 'op': op, - }) - outf.write(re.sub('\s+', ' ', - desc['assert'].strip().format(**format))+'\n') - -def mkassert(type, comp, lh, rh, size=None): - format = { - 'type': type.lower(), 'TYPE': type.upper(), - 'comp': comp.lower(), 'COMP': comp.upper(), - 'prefix': PREFIX.lower(), 'PREFIX': PREFIX.upper(), - 'lh': lh.strip(' '), - 'rh': rh.strip(' '), - 'size': size, - } - if size: - return ((ASSERT + '(__FILE__, __LINE__, {lh}, {rh}, {size})') - .format(**format)) - else: - return ((ASSERT + '(__FILE__, __LINE__, {lh}, {rh})') - .format(**format)) - - -# simple recursive descent parser -LEX = { - 'ws': [r'(?:\s|\n|#.*?\n|//.*?\n|/\*.*?\*/)+'], - 'assert': PATTERN, - 'string': [r'"(?:\\.|[^"])*"', r"'(?:\\.|[^'])\'"], - 'arrow': ['=>'], - 'paren': ['\(', '\)'], - 'op': ['strcmp', 'memcmp', '->'], - 'comp': ['==', '!=', '<=', '>=', '<', '>'], - 'logic': ['\&\&', '\|\|'], - 'sep': [':', ';', '\{', '\}', ','], -} - -class ParseFailure(Exception): - def __init__(self, expected, found): - self.expected = expected - self.found = found - - def __str__(self): - return "expected %r, found %s..." % ( - self.expected, repr(self.found)[:70]) - -class Parse: - def __init__(self, inf, lexemes): - p = '|'.join('(?P<%s>%s)' % (n, '|'.join(l)) - for n, l in lexemes.items()) - p = re.compile(p, re.DOTALL) - data = inf.read() - tokens = [] - while True: - m = p.search(data) - if m: - if m.start() > 0: - tokens.append((None, data[:m.start()])) - tokens.append((m.lastgroup, m.group())) - data = data[m.end():] - else: - tokens.append((None, data)) - break - self.tokens = tokens - self.off = 0 - - def lookahead(self, *pattern): - if self.off < len(self.tokens): - token = self.tokens[self.off] - if token[0] in pattern or token[1] in pattern: - self.m = token[1] - return self.m - self.m = None - return self.m - - def accept(self, *patterns): - m = self.lookahead(*patterns) - if m is not None: - self.off += 1 - return m - - def expect(self, *patterns): - m = self.accept(*patterns) - if not m: - raise ParseFailure(patterns, self.tokens[self.off:]) - return m - - def push(self): - return self.off - - def pop(self, state): - self.off = state - -def passert(p): - def pastr(p): - p.expect('assert') ; p.accept('ws') ; p.expect('(') ; p.accept('ws') - p.expect('strcmp') ; p.accept('ws') ; p.expect('(') ; p.accept('ws') - lh = pexpr(p) ; p.accept('ws') - p.expect(',') ; p.accept('ws') - rh = pexpr(p) ; p.accept('ws') - p.expect(')') ; p.accept('ws') - comp = p.expect('comp') ; p.accept('ws') - p.expect('0') ; p.accept('ws') - p.expect(')') - return mkassert('str', COMP[comp], lh, rh) - - def pamem(p): - p.expect('assert') ; p.accept('ws') ; p.expect('(') ; p.accept('ws') - p.expect('memcmp') ; p.accept('ws') ; p.expect('(') ; p.accept('ws') - lh = pexpr(p) ; p.accept('ws') - p.expect(',') ; p.accept('ws') - rh = pexpr(p) ; p.accept('ws') - p.expect(',') ; p.accept('ws') - size = pexpr(p) ; p.accept('ws') - p.expect(')') ; p.accept('ws') - comp = p.expect('comp') ; p.accept('ws') - p.expect('0') ; p.accept('ws') - p.expect(')') - return mkassert('mem', COMP[comp], lh, rh, size) - - def paint(p): - p.expect('assert') ; p.accept('ws') ; p.expect('(') ; p.accept('ws') - lh = pexpr(p) ; p.accept('ws') - comp = p.expect('comp') ; p.accept('ws') - rh = pexpr(p) ; p.accept('ws') - p.expect(')') - return mkassert('int', COMP[comp], lh, rh) - - def pabool(p): - p.expect('assert') ; p.accept('ws') ; p.expect('(') ; p.accept('ws') - lh = pexprs(p) ; p.accept('ws') - p.expect(')') - return mkassert('bool', 'eq', lh, 'true') - - def pa(p): - return p.expect('assert') - - state = p.push() - lastf = None - for pa in [pastr, pamem, paint, pabool, pa]: - try: - return pa(p) - except ParseFailure as f: - p.pop(state) - lastf = f - else: - raise lastf - -def pexpr(p): - res = [] - while True: - if p.accept('('): - res.append(p.m) - while True: - res.append(pexprs(p)) - if p.accept('sep'): - res.append(p.m) - else: - break - res.append(p.expect(')')) - elif p.lookahead('assert'): - res.append(passert(p)) - elif p.accept('assert', 'ws', 'string', 'op', None): - res.append(p.m) - else: - return ''.join(res) - -def pexprs(p): - res = [] - while True: - res.append(pexpr(p)) - if p.accept('comp', 'logic', ','): - res.append(p.m) - else: - return ''.join(res) - -def pstmt(p): - ws = p.accept('ws') or '' - lh = pexprs(p) - if p.accept('=>'): - rh = pexprs(p) - return ws + mkassert('int', 'eq', lh, rh) - else: - return ws + lh - - -def main(args): - inf = open(args.input, 'r') if args.input else sys.stdin - outf = open(args.output, 'w') if args.output else sys.stdout - - lexemes = LEX.copy() - if args.pattern: - lexemes['assert'] = args.pattern - p = Parse(inf, lexemes) - - # write extra verbose asserts - mkdecls(outf, maxwidth=args.maxwidth) - if args.input: - outf.write("#line %d \"%s\"\n" % (1, args.input)) - - # parse and write out stmt at a time - try: - while True: - outf.write(pstmt(p)) - if p.accept('sep'): - outf.write(p.m) - else: - break - except ParseFailure as f: - pass - - for i in range(p.off, len(p.tokens)): - outf.write(p.tokens[i][1]) - -if __name__ == "__main__": - import argparse - parser = argparse.ArgumentParser( - description="Cpp step that increases assert verbosity") - parser.add_argument('input', nargs='?', - help="Input C file after cpp.") - parser.add_argument('-o', '--output', required=True, - help="Output C file.") - parser.add_argument('-p', '--pattern', action='append', - help="Patterns to search for starting an assert statement.") - parser.add_argument('--maxwidth', default=MAXWIDTH, type=int, - help="Maximum number of characters to display for strcmp and memcmp.") - main(parser.parse_args()) diff --git a/components/fs/littlefs/littlefs/scripts/perf.py b/components/fs/littlefs/littlefs/scripts/perf.py new file mode 100644 index 000000000..2ee006c0b --- /dev/null +++ b/components/fs/littlefs/littlefs/scripts/perf.py @@ -0,0 +1,1344 @@ +#!/usr/bin/env python3 +# +# Script to aggregate and report Linux perf results. +# +# Example: +# ./scripts/perf.py -R -obench.perf ./runners/bench_runner +# ./scripts/perf.py bench.perf -j -Flfs.c -Flfs_util.c -Scycles +# +# Copyright (c) 2022, The littlefs authors. +# SPDX-License-Identifier: BSD-3-Clause +# + +import bisect +import collections as co +import csv +import errno +import fcntl +import functools as ft +import itertools as it +import math as m +import multiprocessing as mp +import os +import re +import shlex +import shutil +import subprocess as sp +import tempfile +import zipfile + +# TODO support non-zip perf results? + + +PERF_PATH = ['perf'] +PERF_EVENTS = 'cycles,branch-misses,branches,cache-misses,cache-references' +PERF_FREQ = 100 +OBJDUMP_PATH = ['objdump'] +THRESHOLD = (0.5, 0.85) + + +# integer fields +class Int(co.namedtuple('Int', 'x')): + __slots__ = () + def __new__(cls, x=0): + if isinstance(x, Int): + return x + if isinstance(x, str): + try: + x = int(x, 0) + except ValueError: + # also accept +-∞ and +-inf + if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): + x = m.inf + elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): + x = -m.inf + else: + raise + assert isinstance(x, int) or m.isinf(x), x + return super().__new__(cls, x) + + def __str__(self): + if self.x == m.inf: + return '∞' + elif self.x == -m.inf: + return '-∞' + else: + return str(self.x) + + def __int__(self): + assert not m.isinf(self.x) + return self.x + + def __float__(self): + return float(self.x) + + none = '%7s' % '-' + def table(self): + return '%7s' % (self,) + + diff_none = '%7s' % '-' + diff_table = table + + def diff_diff(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + diff = new - old + if diff == +m.inf: + return '%7s' % '+∞' + elif diff == -m.inf: + return '%7s' % '-∞' + else: + return '%+7d' % diff + + def ratio(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + if m.isinf(new) and m.isinf(old): + return 0.0 + elif m.isinf(new): + return +m.inf + elif m.isinf(old): + return -m.inf + elif not old and not new: + return 0.0 + elif not old: + return 1.0 + else: + return (new-old) / old + + def __add__(self, other): + return self.__class__(self.x + other.x) + + def __sub__(self, other): + return self.__class__(self.x - other.x) + + def __mul__(self, other): + return self.__class__(self.x * other.x) + +# perf results +class PerfResult(co.namedtuple('PerfResult', [ + 'file', 'function', 'line', + 'cycles', 'bmisses', 'branches', 'cmisses', 'caches', + 'children'])): + _by = ['file', 'function', 'line'] + _fields = ['cycles', 'bmisses', 'branches', 'cmisses', 'caches'] + _sort = ['cycles', 'bmisses', 'cmisses', 'branches', 'caches'] + _types = { + 'cycles': Int, + 'bmisses': Int, 'branches': Int, + 'cmisses': Int, 'caches': Int} + + __slots__ = () + def __new__(cls, file='', function='', line=0, + cycles=0, bmisses=0, branches=0, cmisses=0, caches=0, + children=[]): + return super().__new__(cls, file, function, int(Int(line)), + Int(cycles), Int(bmisses), Int(branches), Int(cmisses), Int(caches), + children) + + def __add__(self, other): + return PerfResult(self.file, self.function, self.line, + self.cycles + other.cycles, + self.bmisses + other.bmisses, + self.branches + other.branches, + self.cmisses + other.cmisses, + self.caches + other.caches, + self.children + other.children) + + +def openio(path, mode='r', buffering=-1): + # allow '-' for stdin/stdout + if path == '-': + if mode == 'r': + return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering) + else: + return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering) + else: + return open(path, mode, buffering) + +# run perf as a subprocess, storing measurements into a zip file +def record(command, *, + output=None, + perf_freq=PERF_FREQ, + perf_period=None, + perf_events=PERF_EVENTS, + perf_path=PERF_PATH, + **args): + # create a temporary file for perf to write to, as far as I can tell + # this is strictly needed because perf's pipe-mode only works with stdout + with tempfile.NamedTemporaryFile('rb') as f: + # figure out our perf invocation + perf = perf_path + list(filter(None, [ + 'record', + '-F%s' % perf_freq + if perf_freq is not None + and perf_period is None else None, + '-c%s' % perf_period + if perf_period is not None else None, + '-B', + '-g', + '--all-user', + '-e%s' % perf_events, + '-o%s' % f.name])) + + # run our command + try: + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in perf + command)) + err = sp.call(perf + command, close_fds=False) + + except KeyboardInterrupt: + err = errno.EOWNERDEAD + + # synchronize access + z = os.open(output, os.O_RDWR | os.O_CREAT) + fcntl.flock(z, fcntl.LOCK_EX) + + # copy measurements into our zip file + with os.fdopen(z, 'r+b') as z: + with zipfile.ZipFile(z, 'a', + compression=zipfile.ZIP_DEFLATED, + compresslevel=1) as z: + with z.open('perf.%d' % os.getpid(), 'w') as g: + shutil.copyfileobj(f, g) + + # forward the return code + return err + + +# try to only process each dso onceS +# +# note this only caches with the non-keyword arguments +def multiprocessing_cache(f): + local_cache = {} + manager = mp.Manager() + global_cache = manager.dict() + lock = mp.Lock() + + def multiprocessing_cache(*args, **kwargs): + # check local cache? + if args in local_cache: + return local_cache[args] + # check global cache? + with lock: + if args in global_cache: + v = global_cache[args] + local_cache[args] = v + return v + # fall back to calling the function + v = f(*args, **kwargs) + global_cache[args] = v + local_cache[args] = v + return v + + return multiprocessing_cache + +@multiprocessing_cache +def collect_syms_and_lines(obj_path, *, + objdump_path=None, + **args): + symbol_pattern = re.compile( + '^(?P[0-9a-fA-F]+)' + '\s+.*' + '\s+(?P[0-9a-fA-F]+)' + '\s+(?P[^\s]+)\s*$') + line_pattern = re.compile( + '^\s+(?:' + # matches dir/file table + '(?P[0-9]+)' + '(?:\s+(?P[0-9]+))?' + '\s+.*' + '\s+(?P[^\s]+)' + # matches line opcodes + '|' '\[[^\]]*\]\s+' + '(?:' + '(?PSpecial)' + '|' '(?PCopy)' + '|' '(?PEnd of Sequence)' + '|' 'File .*?to (?:entry )?(?P\d+)' + '|' 'Line .*?to (?P[0-9]+)' + '|' '(?:Address|PC) .*?to (?P[0x0-9a-fA-F]+)' + '|' '.' ')*' + ')$', re.IGNORECASE) + + # figure out symbol addresses and file+line ranges + syms = {} + sym_at = [] + cmd = objdump_path + ['-t', obj_path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + for line in proc.stdout: + m = symbol_pattern.match(line) + if m: + name = m.group('name') + addr = int(m.group('addr'), 16) + size = int(m.group('size'), 16) + # ignore zero-sized symbols + if not size: + continue + # note multiple symbols can share a name + if name not in syms: + syms[name] = set() + syms[name].add((addr, size)) + sym_at.append((addr, name, size)) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + # assume no debug-info on failure + pass + + # sort and keep largest/first when duplicates + sym_at.sort(key=lambda x: (x[0], -x[2], x[1])) + sym_at_ = [] + for addr, name, size in sym_at: + if len(sym_at_) == 0 or sym_at_[-1][0] != addr: + sym_at_.append((addr, name, size)) + sym_at = sym_at_ + + # state machine for dwarf line numbers, note that objdump's + # decodedline seems to have issues with multiple dir/file + # tables, which is why we need this + lines = [] + line_at = [] + dirs = {} + files = {} + op_file = 1 + op_line = 1 + op_addr = 0 + cmd = objdump_path + ['--dwarf=rawline', obj_path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + for line in proc.stdout: + m = line_pattern.match(line) + if m: + if m.group('no') and not m.group('dir'): + # found a directory entry + dirs[int(m.group('no'))] = m.group('path') + elif m.group('no'): + # found a file entry + dir = int(m.group('dir')) + if dir in dirs: + files[int(m.group('no'))] = os.path.join( + dirs[dir], + m.group('path')) + else: + files[int(m.group('no'))] = m.group('path') + else: + # found a state machine update + if m.group('op_file'): + op_file = int(m.group('op_file'), 0) + if m.group('op_line'): + op_line = int(m.group('op_line'), 0) + if m.group('op_addr'): + op_addr = int(m.group('op_addr'), 0) + + if (m.group('op_special') + or m.group('op_copy') + or m.group('op_end')): + file = os.path.abspath(files.get(op_file, '?')) + lines.append((file, op_line, op_addr)) + line_at.append((op_addr, file, op_line)) + + if m.group('op_end'): + op_file = 1 + op_line = 1 + op_addr = 0 + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + # assume no debug-info on failure + pass + + # sort and keep first when duplicates + lines.sort() + lines_ = [] + for file, line, addr in lines: + if len(lines_) == 0 or lines_[-1][0] != file or lines[-1][1] != line: + lines_.append((file, line, addr)) + lines = lines_ + + # sort and keep first when duplicates + line_at.sort() + line_at_ = [] + for addr, file, line in line_at: + if len(line_at_) == 0 or line_at_[-1][0] != addr: + line_at_.append((addr, file, line)) + line_at = line_at_ + + return syms, sym_at, lines, line_at + + +def collect_decompressed(path, *, + perf_path=PERF_PATH, + sources=None, + everything=False, + propagate=0, + depth=1, + **args): + sample_pattern = re.compile( + '(?P\w+)' + '\s+(?P\w+)' + '\s+(?P