diff --git a/docker/CI/cli.sh b/docker/CI/cli.sh deleted file mode 100755 index 658324b8f7a..00000000000 --- a/docker/CI/cli.sh +++ /dev/null @@ -1 +0,0 @@ -docker-compose exec server-0 /opt/byconity/bin/clickhouse client --port 52145 --host 127.0.0.1 diff --git a/docker/CI/docker-compose-nexusfs.yml b/docker/CI/docker-compose-nexusfs.yml deleted file mode 100644 index c4dda2deba8..00000000000 --- a/docker/CI/docker-compose-nexusfs.yml +++ /dev/null @@ -1,174 +0,0 @@ -version: "3" - -services: - # After upgrade to docker-compose v2, we could use `include` instead of `extend`. - hdfs-namenode: - extends: - file: ./common/hdfs.yml - service: hdfs-namenode - hdfs-datanode: - extends: - file: ./common/hdfs.yml - service: hdfs-datanode - fdb: - extends: - file: ./common/fdb.yml - service: fdb - my_mysql: - extends: - file: ./common/mysql.yml - service: my_mysql - tso: - image: hub.byted.org/bytehouse/debian.bullseye.fdb.udf:0.1 - command: bash -c "fdbcli -C /config/fdb.cluster --exec \"configure new single ssd\"; tso-server --config-file /config/tso.yml" - depends_on: - - fdb - - hdfs-namenode - volumes: - - ${CNCH_BINARY_PATH}/:/opt/byconity/bin/:ro - - ${CNCH_LIBRARY_PATH}/:/opt/byconity/lib/:ro - - ./nexusfs/:/config/:ro - - ./test_output/tso/:/var/log/byconity/:rw - environment: &env - LD_LIBRARY_PATH: /opt/byconity/lib - PATH: /opt/byconity/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - ASAN_OPTIONS: - TSAN_OPTIONS: - IS_CI_ENV: 1 - CI_PIPELINE_NAME: CI - cap_add: - - SYS_PTRACE - healthcheck: - test: ["CMD", "curl", "localhost:18845"] - interval: 5s - - server-0: - image: hub.byted.org/bytehouse/debian.bullseye.fdb.udf:0.1 - command: bash -c "(udf-manager --config-file /config/server.yml & clickhouse-server --config-file /config/server.yml)" - depends_on: - tso: - condition: service_healthy - ports: - - "9000:52145" - - "127.0.0.1:8123:21557" - - "127.0.0.1:9004:9004" - environment: - <<: *env - SERVER_ID: server-0 - volumes: - - ${CNCH_BINARY_PATH}/:/opt/byconity/bin/:ro - - ${CNCH_LIBRARY_PATH}/:/opt/byconity/lib/:ro - - ./nexusfs/:/config/:ro - - ./test_output/server-0/:/var/log/byconity/:rw - - ./queries/:/opt/byconity/queries/:ro - cap_add: - - SYS_PTRACE - healthcheck: - test: ["CMD", "curl", "localhost:21557"] - interval: 5s - - server-1: - image: hub.byted.org/bytehouse/debian.bullseye.fdb.udf:0.1 - command: bash -c "(udf-manager --config-file /config/server.yml & clickhouse-server --config-file /config/server.yml)" - depends_on: - tso: - condition: service_healthy - ports: - - "9001:52145" - - "127.0.0.1:8124:21557" - environment: - <<: *env - SERVER_ID: server-1 - volumes: - - ${CNCH_BINARY_PATH}/:/opt/byconity/bin/:ro - - ${CNCH_LIBRARY_PATH}/:/opt/byconity/lib/:ro - - ./nexusfs/:/config/:ro - - ./test_output/server-1/:/var/log/byconity/:rw - - ./queries/:/opt/byconity/queries/:ro - cap_add: - - SYS_PTRACE - healthcheck: - test: ["CMD", "curl", "localhost:52145"] - interval: 5s - - worker-write: - image: hub.byted.org/bytehouse/debian.bullseye.fdb.udf:0.1 - command: bash -c "clickhouse-server --config-file /config/worker.yml" - depends_on: - - server-0 - - server-1 - ports: - - "52149:52145" - environment: - <<: *env - WORKER_GROUP_ID: wg_write - VIRTUAL_WAREHOUSE_ID: vw_write - WORKER_ID: w0 - volumes: - - ${CNCH_BINARY_PATH}/:/opt/byconity/bin/:ro - - ${CNCH_LIBRARY_PATH}/:/opt/byconity/lib/:ro - - ./nexusfs/:/config/:ro - - ./test_output/worker-write/:/var/log/byconity/:rw - - ./queries/:/opt/byconity/queries/:ro - cap_add: - - SYS_PTRACE - worker-default: - image: hub.byted.org/bytehouse/debian.bullseye.fdb.udf:0.1 - command: bash -c "(udf-manager --config-file /config/worker.yml & clickhouse-server --config-file /config/worker.yml)" - depends_on: - - server-0 - - server-1 - environment: - <<: *env - WORKER_GROUP_ID: wg_default - VIRTUAL_WAREHOUSE_ID: vw_default - WORKER_ID: r0 - volumes: - - ${CNCH_BINARY_PATH}/:/opt/byconity/bin/:ro - - ${CNCH_LIBRARY_PATH}/:/opt/byconity/lib/:ro - - ./nexusfs/:/config/:ro - - ./test_output/worker-default/:/var/log/byconity/:rw - - ./queries/:/opt/byconity/queries/:ro - cap_add: - - SYS_PTRACE - daemon-manager: - image: hub.byted.org/bytehouse/debian.bullseye.fdb.udf:0.1 - command: bash -c "daemon-manager --config-file ./config/daemon-manager.yml" - depends_on: - server-0: - condition: service_healthy - server-1: - condition: service_healthy - environment: - <<: *env - volumes: - - ${CNCH_BINARY_PATH}/:/opt/byconity/bin/:ro - - ${CNCH_LIBRARY_PATH}/:/opt/byconity/lib/:ro - - ./nexusfs/:/config/:ro - - ./test_output/daemon-manager/:/var/log/byconity/:rw - cap_add: - - SYS_PTRACE - restart: always - - resource-manager: - image: hub.byted.org/bytehouse/debian.bullseye.fdb.udf:0.1 - command: bash -c "resource-manager --config-file /config/resource-manager.yml" - depends_on: - - tso - volumes: - - ${CNCH_BINARY_PATH}/:/opt/byconity/bin/:ro - - ${CNCH_LIBRARY_PATH}/:/opt/byconity/lib/:ro - - ./nexusfs/:/config/:ro - - ./test_output/rm/:/var/log/byconity/:rw - environment: - <<: *env - cap_add: - - SYS_PTRACE - -volumes: - fdb-data: - external: false - hdfs-namenode: - external: false - hdfs-datanode: - external: false diff --git a/docker/CI/multi-servers/server.yml b/docker/CI/multi-servers/server.yml deleted file mode 100644 index 3c5d5ad9480..00000000000 --- a/docker/CI/multi-servers/server.yml +++ /dev/null @@ -1,228 +0,0 @@ -logger: - level: trace - log: /var/log/byconity/out.log - errorlog: /var/log/byconity/err.log - testlog: /var/log/byconity/test.log - size: 1000M - count: 10 - console: true -additional_services: - GIS: 1 - VectorSearch: 1 - FullTextSearch: 1 -http_port: 21557 -rpc_port: 30605 -tcp_port: 52145 -ha_tcp_port: 26247 -exchange_port: 47447 -exchange_status_port: 60611 -interserver_http_port: 30491 -mysql_port: 9004 -listen_host: "0.0.0.0" -prometheus: - endpoint: "/metrics" - port: 0 - metrics: true - events: true - asynchronous_metrics: true - part_metrics: false -cnch_type: server -max_connections: 4096 -keep_alive_timeout: 3 -max_concurrent_queries: 200 -uncompressed_cache_size: 8589934592 -mark_cache_size: 5368709120 -path: /var/byconity/ -tmp_path: /var/byconity/tmp_data/ -users_config: /config/users.yml -default_profile: default -default_database: default -timezone: Europe/Moscow -mlock_executable: false -enable_tenant_systemdb: false -macros: - "-incl": macros - "-optional": true -builtin_dictionaries_reload_interval: 3600 -max_session_timeout: 3600 -default_session_timeout: 60 -dictionaries_config: "*_dictionary.xml" -format_schema_path: /var/byconity/format_schemas/ -perQuery: 1 -storage_configuration: - disks: - hdfs_disk: - path: /user/clickhouse/ - type: bytehdfs - local_disk: - path: /var/byconity/data/ - type: local - policies: - default: - volumes: - hdfs: - default: hdfs_disk - disk: hdfs_disk - local: - default: local_disk - disk: local_disk -cnch_kafka_log: - database: cnch_system - table: cnch_kafka_log - flush_max_row_count: 10000 - flush_interval_milliseconds: 7500 -cnch_unique_table_log: - database: cnch_system - table: cnch_unique_table_log - flush_max_row_count: 10000 - flush_interval_milliseconds: 7500 -cnch_query_log: - database: cnch_system - table: cnch_query_log - flush_max_row_count: 10000 - flush_interval_milliseconds: 7500 -query_log: - database: system - table: query_log - flush_interval_milliseconds: 15000 - partition_by: event_date -part_allocation_algorithm: 1 -consistent_hash_ring: - num_replicas: 16 - num_probes: 21 - load_factor: 1.3 -service_discovery: - mode: local - cluster: default - disable_cache: false - cache_timeout: 5 - server: - psm: data.cnch.server - node: - - host: server-0 - hostname: server-0 - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - - host: server-1 - hostname: server-1 - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - tso: - psm: data.cnch.tso - node: - host: tso-0 - hostname: tso - ports: - port: - - name: PORT0 - value: 18845 - - name: PORT2 - value: 9181 - resource_manager: - psm: data.cnch.resource_manager - node: - host: resource-manager-0 - hostname: resource-manager-0 - ports: - port: - name: PORT0 - value: 28989 - daemon_manager: - psm: data.cnch.daemon_manager - node: - host: daemon-manager-0 - hostname: daemon-manager - ports: - port: - name: PORT0 - value: 17553 - vw_psm: data.cnch.vw - vw: - psm: data.cnch.vw - node: - - host: worker-write-0 - hostname: worker-write - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - vw_name: vw_write - - host: worker-default-0 - hostname: worker-default - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - vw_name: vw_default -catalog: - name_space: default -catalog_service: - type: fdb - fdb: - cluster_file: /config/fdb.cluster -hdfs_addr: hdfs://hdfs-namenode:9000 -udf_path: /var/byconity/data/user_defined -udf_manager_server: - timeout_ms: 20000 - max_retry: 1 -udf_processor: - count: 3 - uds_path: /dev/shm/udf_processor_server - timeout_ms: 10000 - max_retry: 1 -custom_settings_prefixes: SQL_ -restrict_tenanted_users_to_whitelist_settings: false -restrict_tenanted_users_to_privileged_operations: false -sensitive_permission_tenants: 1234 diff --git a/docker/CI/multi-servers/worker.yml b/docker/CI/multi-servers/worker.yml deleted file mode 100644 index 314f12df597..00000000000 --- a/docker/CI/multi-servers/worker.yml +++ /dev/null @@ -1,202 +0,0 @@ -logger: - level: trace - log: /var/log/byconity/out.log - errorlog: /var/log/byconity/err.log - testlog: /var/log/byconity/test.log - size: 1000M - count: 10 -http_port: 21557 -rpc_port: 30605 -tcp_port: 52145 -ha_tcp_port: 26247 -exchange_port: 47447 -exchange_status_port: 60611 -interserver_http_port: 30491 -listen_host: "0.0.0.0" -cnch_type: worker -vw_name: vw_default -max_connections: 4096 -keep_alive_timeout: 3 -max_concurrent_queries: 200 -uncompressed_cache_size: 8589934592 -mark_cache_size: 5368709120 -path: /var/byconity/ -tmp_path: /var/byconity/tmp_data/ -users_config: /config/users.yml -default_profile: default -default_database: default -timezone: Europe/Moscow -mlock_executable: false -enable_tenant_systemdb: false -macros: - "-incl": macros - "-optional": true -builtin_dictionaries_reload_interval: 3600 -max_session_timeout: 3600 -default_session_timeout: 60 -dictionaries_config: "*_dictionary.xml" -format_schema_path: /var/byconity/format_schemas/ -perQuery: 1 -storage_configuration: - disks: - hdfs_disk: - path: /user/clickhouse/ - type: bytehdfs - local_disk: - path: /var/byconity/data/ - type: local - policies: - default: - volumes: - hdfs: - default: hdfs_disk - disk: hdfs_disk - local: - default: local_disk - disk: local_disk -hdfs_addr: "hdfs://hdfs-namenode:9000" -cnch_unique_table_log: - database: cnch_system - table: cnch_unique_table_log - flush_max_row_count: 10000 - flush_interval_milliseconds: 7500 -query_log: - database: system - table: query_log - flush_interval_milliseconds: 15000 - partition_by: event_date -service_discovery: - mode: local - cluster: default - disable_cache: false - cache_timeout: 5 - server: - psm: data.cnch.server - node: - - host: server-0 - hostname: server-0 - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - - host: server-1 - hostname: server-1 - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - tso: - psm: data.cnch.tso - node: - host: tso-0 - hostname: tso - ports: - port: - - name: PORT0 - value: 18845 - - name: PORT2 - value: 9181 - resource_manager: - psm: data.cnch.resource_manager - node: - host: resource-manager-0 - hostname: resource-manager-0 - ports: - port: - name: PORT0 - value: 28989 - daemon_manager: - psm: data.cnch.daemon_manager - node: - host: daemon-manager-0 - hostname: daemon-manager - ports: - port: - name: PORT0 - value: 17553 - vw_psm: data.cnch.vw - vw: - psm: data.cnch.vw - node: - - host: worker-write-0 - hostname: worker-write - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - vw_name: vw_write - - host: worker-default-0 - hostname: worker-default - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - vw_name: vw_default -catalog: - name_space: default -catalog_service: - type: fdb - fdb: - cluster_file: /config/fdb.cluster -udf_path: /var/byconity/data/user_defined -udf_manager_server: - timeout_ms: 20000 - max_retry: 1 -udf_processor: - count: 3 - uds_path: /dev/shm/udf_processor_worker - timeout_ms: 10000 - max_retry: 1 -restrict_tenanted_users_to_whitelist_settings: false -restrict_tenanted_users_to_privileged_operations: false -additional_services: - FullTextSearch: true -sensitive_permission_tenants: 1234 diff --git a/docker/CI/multi-workers/server.yml b/docker/CI/multi-workers/server.yml deleted file mode 100644 index 109b3dd933e..00000000000 --- a/docker/CI/multi-workers/server.yml +++ /dev/null @@ -1,252 +0,0 @@ -logger: - level: trace - log: /var/log/byconity/out.log - errorlog: /var/log/byconity/err.log - testlog: /var/log/byconity/test.log - size: 1000M - count: 10 -additional_services: - GIS: 1 - VectorSearch: 1 - FullTextSearch: 1 -http_port: 21557 -rpc_port: 30605 -tcp_port: 52145 -ha_tcp_port: 26247 -exchange_port: 47447 -exchange_status_port: 60611 -interserver_http_port: 30491 -mysql_port: 9004 -listen_host: "0.0.0.0" -prometheus: - endpoint: "/metrics" - port: 0 - metrics: true - events: true - asynchronous_metrics: true - part_metrics: false -cnch_type: server -max_connections: 4096 -keep_alive_timeout: 3 -max_concurrent_queries: 200 -uncompressed_cache_size: 8589934592 -mark_cache_size: 5368709120 -path: /var/byconity/ -tmp_path: /var/byconity/tmp_data/ -users_config: /config/users.yml -default_profile: default -default_database: default -timezone: Europe/Moscow -mlock_executable: false -enable_tenant_systemdb: false -macros: - "-incl": macros - "-optional": true -builtin_dictionaries_reload_interval: 3600 -max_session_timeout: 3600 -default_session_timeout: 60 -dictionaries_config: "*_dictionary.xml" -format_schema_path: /var/byconity/format_schemas/ -perQuery: 1 -storage_configuration: - disks: - hdfs_disk: - path: /user/clickhouse/ - type: bytehdfs - local_disk: - path: /var/byconity/data/ - type: local - policies: - default: - volumes: - hdfs: - default: hdfs_disk - disk: hdfs_disk - local: - default: local_disk - disk: local_disk -cnch_kafka_log: - database: cnch_system - table: cnch_kafka_log - flush_max_row_count: 10000 - flush_interval_milliseconds: 7500 -cnch_unique_table_log: - database: cnch_system - table: cnch_unique_table_log - flush_max_row_count: 10000 - flush_interval_milliseconds: 7500 -cnch_query_log: - database: cnch_system - table: cnch_query_log - flush_max_row_count: 10000 - flush_interval_milliseconds: 7500 -query_log: - database: system - table: query_log - flush_interval_milliseconds: 15000 - partition_by: event_date -part_allocation_algorithm: 1 -consistent_hash_ring: - num_replicas: 16 - num_probes: 21 - load_factor: 1.3 -service_discovery: - mode: local - cluster: default - disable_cache: false - cache_timeout: 5 - server: - psm: data.cnch.server - node: - - host: server-0 - hostname: server-0 - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - - host: server-1 - hostname: server-1 - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - resource_manager: - psm: data.cnch.resource_manager - node: - host: resource-manager-0 - hostname: resource-manager-0 - ports: - port: - name: PORT0 - value: 28989 - daemon_manager: - psm: data.cnch.daemon_manager - node: - host: daemon-manager-0 - hostname: daemon-manager-0 - ports: - port: - name: PORT0 - value: 17553 - tso: - psm: data.cnch.tso - node: - host: tso-0 - hostname: tso-0 - ports: - port: - - name: PORT0 - value: 18845 - - name: PORT2 - value: 9181 - vw_psm: data.cnch.vw - vw: - psm: data.cnch.vw - node: - vw_name: vw_write - host: worker-write - hostname: worker-write - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - node: - vw_name: vw_default - host: worker-default-0 - hostname: worker-default-0 - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - node: - vw_name: vw_default - host: worker-default-1 - hostname: worker-default-1 - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 -catalog: - name_space: default -catalog_service: - type: fdb - fdb: - cluster_file: /config/fdb.cluster -external_catalog_mgr: - type: fdb - fdb: - cluster_file: /config/fdb/cluster -hdfs_addr: "hdfs://hdfs-namenode:9000" -udf_path: /var/byconity/data/user_defined -udf_manager_server: - timeout_ms: 20000 - max_retry: 1 -udf_processor: - count: 3 - uds_path: /dev/shm/udf_processor_server - timeout_ms: 10000 - max_retry: 1 -custom_settings_prefixes: SQL_ -restrict_tenanted_users_to_whitelist_settings: false -restrict_tenanted_users_to_privileged_operations: false -sensitive_permission_tenants: 1234 diff --git a/docker/CI/nexusfs/conf.d/catalog.yml b/docker/CI/nexusfs/conf.d/catalog.yml deleted file mode 100644 index 7ddd7231874..00000000000 --- a/docker/CI/nexusfs/conf.d/catalog.yml +++ /dev/null @@ -1,6 +0,0 @@ -catalog: - name_space: default -catalog_service: - type: fdb - fdb: - cluster_file: /config/fdb.cluster diff --git a/docker/CI/nexusfs/conf.d/service_discovery.yml b/docker/CI/nexusfs/conf.d/service_discovery.yml deleted file mode 100644 index 7627487161e..00000000000 --- a/docker/CI/nexusfs/conf.d/service_discovery.yml +++ /dev/null @@ -1,115 +0,0 @@ -service_discovery: - mode: local - cluster: default - disable_cache: false - cache_timeout: 5 - server: - psm: data.cnch.server - node: - - host: server-0 - hostname: server-0 - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - - host: server-1 - hostname: server-1 - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - tso: - psm: data.cnch.tso - node: - host: tso - hostname: tso - ports: - port: - - name: PORT0 - value: 18845 - - name: PORT2 - value: 9181 - resource_manager: - psm: data.cnch.resource_manager - node: - host: resource-manager - hostname: resource-manager - ports: - port: - name: PORT0 - value: 28989 - daemon_manager: - psm: data.cnch.daemon_manager - node: - host: daemon-manager - hostname: daemon-manager - ports: - port: - name: PORT0 - value: 17553 - vw_psm: data.cnch.vw - vw: - psm: data.cnch.vw - node: - - host: worker-write - hostname: worker-write - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - vw_name: vw_write - - host: worker-default - hostname: worker-default - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - vw_name: vw_default diff --git a/docker/CI/nexusfs/conf.d/storage.yml b/docker/CI/nexusfs/conf.d/storage.yml deleted file mode 100644 index 020132e7b53..00000000000 --- a/docker/CI/nexusfs/conf.d/storage.yml +++ /dev/null @@ -1,18 +0,0 @@ -hdfs_addr: hdfs://hdfs-namenode:9000 -storage_configuration: - disks: - hdfs_disk: - path: /user/clickhouse/ - type: bytehdfs - local_disk: - path: /var/byconity/data/ - type: local - policies: - default: - volumes: - hdfs: - default: hdfs_disk - disk: hdfs_disk - local: - default: local_disk - disk: local_disk diff --git a/docker/CI/nexusfs/daemon-manager.yml b/docker/CI/nexusfs/daemon-manager.yml deleted file mode 100644 index c4cbe3dcbf3..00000000000 --- a/docker/CI/nexusfs/daemon-manager.yml +++ /dev/null @@ -1,63 +0,0 @@ -logger: - level: trace - log: /var/log/byconity/out.log - errorlog: /var/log/byconity/err.log - testlog: /var/log/byconity/test.log - size: 1000M - count: 10 -http_port: 21557 -rpc_port: 30605 -tcp_port: 52145 -ha_tcp_port: 26247 -exchange_port: 47447 -exchange_status_port: 60611 -interserver_http_port: 30491 -listen_host: "0.0.0.0" -cnch_type: server -max_connections: 4096 -keep_alive_timeout: 3 -max_concurrent_queries: 200 -uncompressed_cache_size: 8589934592 -mark_cache_size: 5368709120 -path: /var/byconity/ -tmp_path: /var/byconity/tmp_data/ -users_config: /config/users.yml -default_profile: default -default_database: default -timezone: Europe/Moscow -mlock_executable: false -macros: - "-incl": macros - "-optional": true -builtin_dictionaries_reload_interval: 3600 -max_session_timeout: 3600 -default_session_timeout: 60 -dictionaries_config: "*_dictionary.xml" -format_schema_path: /var/byconity/format_schemas/ -perQuery: 1 -daemon_manager: - port: 17553 - daemon_jobs: - job: - - name: PART_GC - interval: 10000 - disable: 0 - - name: PART_MERGE - interval: 10000 - disable: 0 - - name: CONSUMER - interval: 10000 - disable: 0 - - name: GLOBAL_GC - interval: 5000 - disable: 1 - - name: PART_CLUSTERING - interval: 30000 - disable: 0 - - name: DEDUP_WORKER - interval: 3000 - disable: 0 - # Increasing the frequency of recycling in a test environment - - name: TXN_GC - interval: 3000 - disable: 0 diff --git a/docker/CI/nexusfs/fdb.cluster b/docker/CI/nexusfs/fdb.cluster deleted file mode 100644 index b04f02bc3b5..00000000000 --- a/docker/CI/nexusfs/fdb.cluster +++ /dev/null @@ -1 +0,0 @@ -docker:docker@fdb:4550 diff --git a/docker/CI/nexusfs/resource-manager.yml b/docker/CI/nexusfs/resource-manager.yml deleted file mode 100644 index b53233f1d0f..00000000000 --- a/docker/CI/nexusfs/resource-manager.yml +++ /dev/null @@ -1,29 +0,0 @@ -logger: - level: trace - log: /var/log/byconity/out.log - errorlog: /var/log/byconity/err.log - testlog: /var/log/byconity/test.log - size: 1000M - count: 10 -listen_host: "0.0.0.0" -path: /var/byconity/ -timezone: Europe/Moscow -perQuery: 1 -resource_manager: - port: 28989 - vws: - vw: - - name: vw_default - type: default - num_workers: 1 - worker_groups: - worker_group: - name: wg_default - type: Physical - - name: vw_write - type: write - num_workers: 1 - worker_groups: - worker_group: - name: wg_write - type: Physical diff --git a/docker/CI/nexusfs/server.yml b/docker/CI/nexusfs/server.yml deleted file mode 100644 index f03178bd0e2..00000000000 --- a/docker/CI/nexusfs/server.yml +++ /dev/null @@ -1,105 +0,0 @@ -logger: - level: trace - log: /var/log/byconity/out.log - errorlog: /var/log/byconity/err.log - testlog: /var/log/byconity/test.log - size: 1000M - count: 10 - console: true -additional_services: - GIS: 1 - VectorSearch: 1 - FullTextSearch: 1 -http_port: 21557 -rpc_port: 30605 -tcp_port: 52145 -ha_tcp_port: 26247 -exchange_port: 47447 -exchange_status_port: 60611 -interserver_http_port: 30491 -mysql_port: 9004 -listen_host: "0.0.0.0" -prometheus: - endpoint: "/metrics" - port: 0 - metrics: true - events: true - asynchronous_metrics: true - part_metrics: false -cnch_type: server -max_connections: 4096 -keep_alive_timeout: 3 -max_concurrent_queries: 200 -uncompressed_cache_size: 8589934592 -mark_cache_size: 5368709120 -path: /var/byconity/ -tmp_path: /var/byconity/tmp_data/ -users_config: /config/users.yml -default_profile: default -default_database: default -timezone: Europe/Moscow -mlock_executable: false -enable_tenant_systemdb: false -macros: - "-incl": macros - "-optional": true -builtin_dictionaries_reload_interval: 3600 -max_session_timeout: 3600 -default_session_timeout: 60 -dictionaries_config: "*_dictionary.xml" -format_schema_path: /var/byconity/format_schemas/ -perQuery: 1 -nexus_fs: - enable: 1 - use_memory_device: 0 - enable_async_io: 0 - cache_size: 5368709120 - region_size: 4194304 - segment_size: 524288 - enable_memory_buffer: 1 - memory_buffer_size: 1073741824 - clean_regions_pool: 16 - clean_region_threads: 4 - num_in_mem_buffers: 32 - reader_threads: 32 -merge_tree: - reorganize_marks_data_layout: 1 - enable_nexus_fs: 1 -cnch_kafka_log: - database: cnch_system - table: cnch_kafka_log - flush_max_row_count: 10000 - flush_interval_milliseconds: 7500 -cnch_unique_table_log: - database: cnch_system - table: cnch_unique_table_log - flush_max_row_count: 10000 - flush_interval_milliseconds: 7500 -cnch_query_log: - database: cnch_system - table: cnch_query_log - flush_max_row_count: 10000 - flush_interval_milliseconds: 7500 -query_log: - database: system - table: query_log - flush_interval_milliseconds: 15000 - partition_by: event_date -part_allocation_algorithm: 1 -consistent_hash_ring: - num_replicas: 16 - num_probes: 21 - load_factor: 1.3 -udf_path: /var/byconity/data/user_defined -udf_manager_server: - timeout_ms: 20000 - max_retry: 1 -udf_processor: - count: 3 - uds_path: /dev/shm/udf_processor_server - timeout_ms: 10000 - max_retry: 1 -custom_settings_prefixes: SQL_ -restrict_tenanted_users_to_whitelist_settings: false -restrict_tenanted_users_to_privileged_operations: false -sensitive_permission_tenants: 1234 diff --git a/docker/CI/nexusfs/tso.yml b/docker/CI/nexusfs/tso.yml deleted file mode 100644 index 095eb2ebe7e..00000000000 --- a/docker/CI/nexusfs/tso.yml +++ /dev/null @@ -1,22 +0,0 @@ -logger: - level: trace - log: /var/log/byconity/tso.log - errorlog: /var/log/byconity/tso.err.log - testlog: /var/log/byconity/tso.test.log - size: 1000M - count: 10 - console: false -listen_host: "0.0.0.0" -path: /var/byconity/tso -tmp_path: /var/byconity/tmp -tso_service: - type: fdb - fdb: - cluster_file: /config/fdb.cluster - port: 18845 - http: - port: 9181 - receive_timeout: 1800 - send_timeout: 1800 - tso_window_ms: 3000 - tso_get_leader_info_interval_ms: 0 diff --git a/docker/CI/nexusfs/users.yml b/docker/CI/nexusfs/users.yml deleted file mode 100644 index 61e2e5a63d0..00000000000 --- a/docker/CI/nexusfs/users.yml +++ /dev/null @@ -1,38 +0,0 @@ -profiles: - default: - load_balancing: random - log_queries: 1 - max_execution_time: 180 - exchange_timeout_ms: 300000 - enable_nexus_fs: 1 - -users: - default: - networks: - ip: ::/0 - password: "" - profile: default - quota: default - access_management: 1 - server: - networks: - ip: ::/0 - password: "" - profile: default - quota: default - probe: - networks: - ip: ::/0 - password: "" - profile: default - quota: default - -quotas: - default: - interval: - duration: 3600 - queries: 0 - errors: 0 - result_rows: 0 - read_rows: 0 - execution_time: 0 \ No newline at end of file diff --git a/docker/CI/nexusfs/worker.yml b/docker/CI/nexusfs/worker.yml deleted file mode 100644 index a97e011eb56..00000000000 --- a/docker/CI/nexusfs/worker.yml +++ /dev/null @@ -1,82 +0,0 @@ -logger: - level: trace - log: /var/log/byconity/out.log - errorlog: /var/log/byconity/err.log - testlog: /var/log/byconity/test.log - size: 1000M - count: 10 -http_port: 21557 -rpc_port: 30605 -tcp_port: 52145 -ha_tcp_port: 26247 -exchange_port: 47447 -exchange_status_port: 60611 -interserver_http_port: 30491 -listen_host: "0.0.0.0" -cnch_type: worker -vw_name: vw_default -max_connections: 4096 -keep_alive_timeout: 3 -max_concurrent_queries: 200 -uncompressed_cache_size: 8589934592 -mark_cache_size: 5368709120 -path: /var/byconity/ -tmp_path: /var/byconity/tmp_data/ -users_config: /config/users.yml -default_profile: default -default_database: default -timezone: Europe/Moscow -mlock_executable: false -enable_tenant_systemdb: false -macros: - "-incl": macros - "-optional": true -builtin_dictionaries_reload_interval: 3600 -max_session_timeout: 3600 -default_session_timeout: 60 -dictionaries_config: "*_dictionary.xml" -format_schema_path: /var/byconity/format_schemas/ -perQuery: 1 -nexus_fs: - enable: 1 - use_memory_device: 0 - enable_async_io: 0 - cache_size: 5368709120 - region_size: 4194304 - segment_size: 524288 - enable_memory_buffer: 1 - memory_buffer_size: 1073741824 - clean_regions_pool: 16 - clean_region_threads: 4 - num_in_mem_buffers: 32 - reader_threads: 32 -merge_tree: - reorganize_marks_data_layout: 1 - enable_nexus_fs: 1 -cnch_unique_table_log: - database: cnch_system - table: cnch_unique_table_log - flush_max_row_count: 10000 - flush_interval_milliseconds: 7500 -query_log: - database: system - table: query_log - flush_interval_milliseconds: 15000 - partition_by: event_date -udf_path: /var/byconity/data/user_defined -udf_manager_server: - timeout_ms: 20000 - max_retry: 1 -udf_processor: - count: 3 - uds_path: /dev/shm/udf_processor_worker - timeout_ms: 10000 - max_retry: 1 -restrict_tenanted_users_to_system_tables: false -restrict_tenanted_users_to_whitelist_settings: false -restrict_tenanted_users_to_privileged_operations: false -additional_services: - FullTextSearch: true - VectorSearch: true - GIS: true -sensitive_permission_tenants: 1234 diff --git a/docker/CI/s3/server.yml b/docker/CI/s3/server.yml deleted file mode 100644 index 6d53ac43097..00000000000 --- a/docker/CI/s3/server.yml +++ /dev/null @@ -1,236 +0,0 @@ -# Auto-generated! Please do not modify this file directly. Refer to 'convert-hdfs-configs-to-s3.sh'. -logger: - level: trace - log: /var/log/byconity/out.log - errorlog: /var/log/byconity/err.log - testlog: /var/log/byconity/test.log - size: 1000M - count: 10 - console: true -additional_services: - GIS: 1 - VectorSearch: 1 - FullTextSearch: 1 -http_port: 21557 -rpc_port: 30605 -tcp_port: 52145 -ha_tcp_port: 26247 -exchange_port: 47447 -exchange_status_port: 60611 -interserver_http_port: 30491 -mysql_port: 9004 -listen_host: "0.0.0.0" -prometheus: - endpoint: "/metrics" - port: 0 - metrics: true - events: true - asynchronous_metrics: true - part_metrics: false -cnch_type: server -max_connections: 4096 -keep_alive_timeout: 3 -max_concurrent_queries: 200 -uncompressed_cache_size: 8589934592 -mark_cache_size: 5368709120 -path: /var/byconity/ -tmp_path: /var/byconity/tmp_data/ -users_config: /config/users.yml -default_profile: default -default_database: default -timezone: Europe/Moscow -mlock_executable: false -enable_tenant_systemdb: false -macros: - "-incl": macros - "-optional": true -builtin_dictionaries_reload_interval: 3600 -max_session_timeout: 3600 -default_session_timeout: 60 -dictionaries_config: "*_dictionary.xml" -format_schema_path: /var/byconity/format_schemas/ -perQuery: 1 -storage_configuration: - disks: - local_disk: - path: /var/byconity/data/ - type: local - s3_disk: - path: data123/ - type: s3 - endpoint: http://minio:9000 - bucket: cnch - ak_id: minio - ak_secret: minio123 - policies: - default: - volumes: - local: - default: local_disk - disk: local_disk - cnch_default_hdfs: - volumes: - s3: - default: s3_disk - disk: s3_disk - # To avoid break hard-coded test cases. - cnch_default_policy: cnch_default_hdfs -cnch_kafka_log: - database: cnch_system - table: cnch_kafka_log - flush_max_row_count: 10000 - flush_interval_milliseconds: 7500 -cnch_unique_table_log: - database: cnch_system - table: cnch_unique_table_log - flush_max_row_count: 10000 - flush_interval_milliseconds: 7500 -cnch_query_log: - database: cnch_system - table: cnch_query_log - flush_max_row_count: 10000 - flush_interval_milliseconds: 7500 -query_log: - database: system - table: query_log - flush_interval_milliseconds: 15000 - partition_by: event_date -part_allocation_algorithm: 1 -consistent_hash_ring: - num_replicas: 16 - num_probes: 21 - load_factor: 1.3 -service_discovery: - mode: local - cluster: default - disable_cache: false - cache_timeout: 5 - server: - psm: data.cnch.server - node: - - host: server-0 - hostname: server-0 - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - - host: server-1 - hostname: server-1 - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - tso: - psm: data.cnch.tso - node: - host: tso-0 - hostname: tso - ports: - port: - - name: PORT0 - value: 18845 - - name: PORT2 - value: 9181 - resource_manager: - psm: data.cnch.resource_manager - node: - host: resource-manager-0 - hostname: resource-manager-0 - ports: - port: - name: PORT0 - value: 28989 - daemon_manager: - psm: data.cnch.daemon_manager - node: - host: daemon-manager-0 - hostname: daemon-manager - ports: - port: - name: PORT0 - value: 17553 - vw_psm: data.cnch.vw - vw: - psm: data.cnch.vw - node: - - host: worker-write-0 - hostname: worker-write - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - vw_name: vw_write - - host: worker-default-0 - hostname: worker-default - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - vw_name: vw_default -catalog: - name_space: default -catalog_service: - type: fdb - fdb: - cluster_file: /config/fdb.cluster -udf_path: /var/byconity/data/user_defined -udf_manager_server: - timeout_ms: 20000 - max_retry: 1 -udf_processor: - count: 3 - uds_path: /dev/shm/udf_processor_server - timeout_ms: 10000 - max_retry: 1 -custom_settings_prefixes: SQL_ -restrict_tenanted_users_to_whitelist_settings: false -restrict_tenanted_users_to_privileged_operations: false -sensitive_permission_tenants: 1234 diff --git a/docker/CI/s3/worker.yml b/docker/CI/s3/worker.yml deleted file mode 100644 index 503691fab5b..00000000000 --- a/docker/CI/s3/worker.yml +++ /dev/null @@ -1,210 +0,0 @@ -# Auto-generated! Please do not modify this file directly. Refer to 'convert-hdfs-configs-to-s3.sh'. -logger: - level: trace - log: /var/log/byconity/out.log - errorlog: /var/log/byconity/err.log - testlog: /var/log/byconity/test.log - size: 1000M - count: 10 -http_port: 21557 -rpc_port: 30605 -tcp_port: 52145 -ha_tcp_port: 26247 -exchange_port: 47447 -exchange_status_port: 60611 -interserver_http_port: 30491 -listen_host: "0.0.0.0" -cnch_type: worker -vw_name: vw_default -max_connections: 4096 -keep_alive_timeout: 3 -max_concurrent_queries: 200 -uncompressed_cache_size: 8589934592 -mark_cache_size: 5368709120 -path: /var/byconity/ -tmp_path: /var/byconity/tmp_data/ -users_config: /config/users.yml -default_profile: default -default_database: default -timezone: Europe/Moscow -mlock_executable: false -enable_tenant_systemdb: false -macros: - "-incl": macros - "-optional": true -builtin_dictionaries_reload_interval: 3600 -max_session_timeout: 3600 -default_session_timeout: 60 -dictionaries_config: "*_dictionary.xml" -format_schema_path: /var/byconity/format_schemas/ -perQuery: 1 -storage_configuration: - disks: - local_disk: - path: /var/byconity/data/ - type: local - s3_disk: - path: data123/ - type: s3 - endpoint: http://minio:9000 - bucket: cnch - ak_id: minio - ak_secret: minio123 - policies: - default: - volumes: - local: - default: local_disk - disk: local_disk - cnch_default_hdfs: - volumes: - s3: - default: s3_disk - disk: s3_disk - # To avoid break hard-coded test cases. - cnch_default_policy: cnch_default_hdfs -cnch_unique_table_log: - database: cnch_system - table: cnch_unique_table_log - flush_max_row_count: 10000 - flush_interval_milliseconds: 7500 -query_log: - database: system - table: query_log - flush_interval_milliseconds: 15000 - partition_by: event_date -service_discovery: - mode: local - cluster: default - disable_cache: false - cache_timeout: 5 - server: - psm: data.cnch.server - node: - - host: server-0 - hostname: server-0 - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - - host: server-1 - hostname: server-1 - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - tso: - psm: data.cnch.tso - node: - host: tso-0 - hostname: tso - ports: - port: - - name: PORT0 - value: 18845 - - name: PORT2 - value: 9181 - resource_manager: - psm: data.cnch.resource_manager - node: - host: resource-manager-0 - hostname: resource-manager-0 - ports: - port: - name: PORT0 - value: 28989 - daemon_manager: - psm: data.cnch.daemon_manager - node: - host: daemon-manager-0 - hostname: daemon-manager - ports: - port: - name: PORT0 - value: 17553 - vw_psm: data.cnch.vw - vw: - psm: data.cnch.vw - node: - - host: worker-write-0 - hostname: worker-write - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - vw_name: vw_write - - host: worker-default-0 - hostname: worker-default - ports: - port: - - name: PORT2 - value: 21557 - - name: PORT1 - value: 30605 - - name: PORT0 - value: 52145 - - name: PORT4 - value: 27651 - - name: PORT3 - value: 45443 - - name: PORT5 - value: 47447 - - name: PORT6 - value: 60611 - vw_name: vw_default -catalog: - name_space: default -catalog_service: - type: fdb - fdb: - cluster_file: /config/fdb.cluster -udf_path: /var/byconity/data/user_defined -udf_manager_server: - timeout_ms: 20000 - max_retry: 1 -udf_processor: - count: 3 - uds_path: /dev/shm/udf_processor_worker - timeout_ms: 10000 - max_retry: 1 -restrict_tenanted_users_to_whitelist_settings: false -restrict_tenanted_users_to_privileged_operations: false -additional_services: - FullTextSearch: true -sensitive_permission_tenants: 1234 diff --git a/docker/ci-deploy/config/users.yml b/docker/ci-deploy/config/users.yml index 773b115f3f9..2d745378055 100644 --- a/docker/ci-deploy/config/users.yml +++ b/docker/ci-deploy/config/users.yml @@ -4,7 +4,14 @@ profiles: log_queries: 1 max_execution_time: 180 exchange_timeout_ms: 300000 - cnch_max_cached_storage : 50000 + point_lookup: + max_threads: 1 + exchange_source_pipeline_threads: 1 + enable_plan_cache: true + query_worker_fault_tolerance: false + send_cacheable_table_definitions: true + optimize_skip_unused_shards: true + enable_prune_source_plan_segment: true readonly: readonly: 1 diff --git a/docker/docker-compose/byconity-multi-cluster/users.yml b/docker/docker-compose/byconity-multi-cluster/users.yml index 800ccc4b5c9..10f91465665 100644 --- a/docker/docker-compose/byconity-multi-cluster/users.yml +++ b/docker/docker-compose/byconity-multi-cluster/users.yml @@ -4,7 +4,15 @@ profiles: log_queries: 1 max_execution_time: 180 exchange_timeout_ms: 300000 - + enable_auto_query_forwarding: true + point_lookup: + max_threads: 1 + exchange_source_pipeline_threads: 1 + enable_plan_cache: true + query_worker_fault_tolerance: false + send_cacheable_table_definitions: true + optimize_skip_unused_shards: true + enable_prune_source_plan_segment: true users: default: networks: @@ -35,4 +43,4 @@ quotas: result_rows: 0 read_rows: 0 execution_time: 0 -cnch_config: "/config/cnch-config.yml" \ No newline at end of file +cnch_config: "/config/cnch-config.yml" diff --git a/docker/docker-compose/byconity-multiworkers-cluster/users.yml b/docker/docker-compose/byconity-multiworkers-cluster/users.yml index 800ccc4b5c9..0d24e8d9080 100644 --- a/docker/docker-compose/byconity-multiworkers-cluster/users.yml +++ b/docker/docker-compose/byconity-multiworkers-cluster/users.yml @@ -4,6 +4,14 @@ profiles: log_queries: 1 max_execution_time: 180 exchange_timeout_ms: 300000 + point_lookup: + max_threads: 1 + exchange_source_pipeline_threads: 1 + enable_plan_cache: true + query_worker_fault_tolerance: false + send_cacheable_table_definitions: true + optimize_skip_unused_shards: true + enable_prune_source_plan_segment: true users: default: @@ -35,4 +43,4 @@ quotas: result_rows: 0 read_rows: 0 execution_time: 0 -cnch_config: "/config/cnch-config.yml" \ No newline at end of file +cnch_config: "/config/cnch-config.yml" diff --git a/docker/docker-compose/byconity-simple-cluster/users.yml b/docker/docker-compose/byconity-simple-cluster/users.yml index af9df8a1bc1..d1ec3f918d5 100644 --- a/docker/docker-compose/byconity-simple-cluster/users.yml +++ b/docker/docker-compose/byconity-simple-cluster/users.yml @@ -4,7 +4,14 @@ profiles: log_queries: 1 max_execution_time: 180 exchange_timeout_ms: 300000 - + point_lookup: + max_threads: 1 + exchange_source_pipeline_threads: 1 + enable_plan_cache: true + query_worker_fault_tolerance: false + send_cacheable_table_definitions: true + optimize_skip_unused_shards: true + enable_prune_source_plan_segment: true users: default: networks: diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 423471290b6..661469e6901 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -1337,144 +1337,3 @@ Result: │ 2,"good" │ └───────────────────────────────────────────┘ ``` - -## snowflakeToDateTime {#snowflakeToDateTime} - -extract time from snowflake id as DateTime format. - -**Syntax** - -``` sql -snowflakeToDateTime(value [, time_zone]) -``` - -**Parameters** - -- `value` — `snowflake id`, Int64 value. -- `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../../sql-reference/data-types/string.md). - -**Returned value** - -- value converted to the `DateTime` data type. - -**Example** - -Query: - -``` sql -SELECT snowflakeToDateTime(CAST('1426860702823350272', 'Int64'), 'UTC'); -``` - -Result: - -``` text -┌─snowflakeToDateTime(CAST('1426860702823350272', 'Int64'), 'UTC')─┐ -│ 2021-08-15 10:57:56 │ -└──────────────────────────────────────────────────────────────────┘ -``` - -## snowflakeToDateTime64 {#snowflakeToDateTime64} - -extract time from snowflake id as DateTime64 format. - -**Syntax** - -``` sql -snowflakeToDateTime64(value [, time_zone]) -``` - -**Parameters** - -- `value` — `snowflake id`, Int64 value. -- `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../../sql-reference/data-types/string.md). - -**Returned value** - -- value converted to the `DateTime64` data type. - -**Example** - -Query: - -``` sql -SELECT snowflakeToDateTime64(CAST('1426860802823350272', 'Int64'), 'UTC'); -``` - -Result: - -``` text -┌─snowflakeToDateTime64(CAST('1426860802823350272', 'Int64'), 'UTC')─┐ -│ 2021-08-15 10:58:19.841 │ -└────────────────────────────────────────────────────────────────────┘ -``` - -## dateTimeToSnowflake {#dateTimeToSnowflake} - -convert DateTime to the first snowflake id at the giving time. - -**Syntax** - -``` sql -dateTimeToSnowflake(value) -``` - -**Parameters** - -- `value` — Date and time. [DateTime](../../sql-reference/data-types/datetime.md). - - -**Returned value** - -- `value` converted to the `Int64` data type as the first snowflake id at that time. - -**Example** - -Query: - -``` sql -SELECT dateTimeToSnowflake(CAST('2021-08-15 18:57:56', 'DateTime')); -``` - -Result: - -``` text -┌─dateTimeToSnowflake(CAST('2021-08-15 18:57:56', 'DateTime'))─┐ -│ 1426860702823350272 │ -└──────────────────────────────────────────────────────────────┘ -``` - - -## dateTime64ToSnowflake {#dateTime64ToSnowflake} - -convert DateTime64 to the first snowflake id at the giving time. - -**Syntax** - -``` sql -dateTime64ToSnowflake(value) -``` - -**Parameters** - -- `value` — Date and time. [DateTime64](../../sql-reference/data-types/datetime64.md). - - -**Returned value** - -- `value` converted to the `Int64` data type as the first snowflake id at that time. - -**Example** - -Query: - -``` sql -SELECT dateTime64ToSnowflake(CAST('2021-08-15 18:57:56.073', 'DateTime64')); -``` - -Result: - -``` text -┌─dateTime64ToSnowflake(CAST('2021-08-15 18:57:56.073', 'DateTime64'))─┐ -│ 1426860703129534464 │ -└──────────────────────────────────────────────────────────────────────┘ -``` \ No newline at end of file diff --git a/docs/en/sql-reference/functions/uuid-functions.md b/docs/en/sql-reference/functions/uuid-functions.md new file mode 100644 index 00000000000..6bed681c896 --- /dev/null +++ b/docs/en/sql-reference/functions/uuid-functions.md @@ -0,0 +1,925 @@ +--- +toc_priority: 53 +toc_title: UUID +--- + +# Functions for Working with UUIDs + +## generateUUIDv4 + +Generates a [version 4](https://tools.ietf.org/html/rfc4122#section-4.4) [UUID](../data-types/uuid.md). + +**Syntax** + +``` sql +generateUUIDv4([expr]) +``` + +**Arguments** + +- `expr` — An arbitrary [expression](../syntax.md#syntax-expressions) used to bypass [common subexpression elimination](../functions/index.md#common-subexpression-elimination) if the function is called multiple times in a query. The value of the expression has no effect on the returned UUID. Optional. + +**Returned value** + +A value of type UUIDv4. + +**Example** + +First, create a table with a column of type UUID, then insert a generated UUIDv4 into the table. + +``` sql +CREATE TABLE tab (uuid UUID) ENGINE = Memory; + +INSERT INTO tab SELECT generateUUIDv4(); + +SELECT * FROM tab; +``` + +Result: + +```response +┌─────────────────────────────────uuid─┐ +│ f4bf890f-f9dc-4332-ad5c-0c18e73f28e9 │ +└──────────────────────────────────────┘ +``` + +**Example with multiple UUIDs generated per row** + +```sql +SELECT generateUUIDv4(1), generateUUIDv4(2); + +┌─generateUUIDv4(1)────────────────────┬─generateUUIDv4(2)────────────────────┐ +│ 2d49dc6e-ddce-4cd0-afb8-790956df54c1 │ 8abf8c13-7dea-4fdf-af3e-0e18767770e6 │ +└──────────────────────────────────────┴──────────────────────────────────────┘ +``` + +## generateUUIDv7 {#generateUUIDv7} + +Generates a [version 7](https://datatracker.ietf.org/doc/html/draft-peabody-dispatch-new-uuid-format-04) [UUID](../data-types/uuid.md). + +The generated UUID contains the current Unix timestamp in milliseconds (48 bits), followed by version "7" (4 bits), a counter (42 bit) to distinguish UUIDs within a millisecond (including a variant field "2", 2 bit), and a random field (32 bits). +For any given timestamp (unix_ts_ms), the counter starts at a random value and is incremented by 1 for each new UUID until the timestamp changes. +In case the counter overflows, the timestamp field is incremented by 1 and the counter is reset to a random new start value. + +Function `generateUUIDv7` guarantees that the counter field within a timestamp increments monotonically across all function invocations in concurrently running threads and queries. + +``` + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +├─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┤ +| unix_ts_ms | +├─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┤ +| unix_ts_ms | ver | counter_high_bits | +├─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┤ +|var| counter_low_bits | +├─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┤ +| rand_b | +└─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┘ +``` + +:::note +As of April 2024, version 7 UUIDs are in draft status and their layout may change in future. +::: + +**Syntax** + +``` sql +generateUUIDv7([expr]) +``` + +**Arguments** + +- `expr` — An arbitrary [expression](../syntax.md#syntax-expressions) used to bypass [common subexpression elimination](../functions/index.md#common-subexpression-elimination) if the function is called multiple times in a query. The value of the expression has no effect on the returned UUID. Optional. + +**Returned value** + +A value of type UUIDv7. + +**Example** + +First, create a table with a column of type UUID, then insert a generated UUIDv7 into the table. + +``` sql +CREATE TABLE tab (uuid UUID) ENGINE = Memory; + +INSERT INTO tab SELECT generateUUIDv7(); + +SELECT * FROM tab; +``` + +Result: + +```response +┌─────────────────────────────────uuid─┐ +│ 018f05af-f4a8-778f-beee-1bedbc95c93b │ +└──────────────────────────────────────┘ +``` + +**Example with multiple UUIDs generated per row** + +```sql +SELECT generateUUIDv7(1), generateUUIDv7(2); + +┌─generateUUIDv7(1)────────────────────┬─generateUUIDv7(2)────────────────────┐ +│ 018f05c9-4ab8-7b86-b64e-c9f03fbd45d1 │ 018f05c9-4ab8-7b86-b64e-c9f12efb7e16 │ +└──────────────────────────────────────┴──────────────────────────────────────┘ +``` + +## empty + +Checks whether the input UUID is empty. + +**Syntax** + +```sql +empty(UUID) +``` + +The UUID is considered empty if it contains all zeros (zero UUID). + +The function also works for [Arrays](array-functions.md#function-empty) and [Strings](string-functions.md#empty). + +**Arguments** + +- `x` — A UUID. [UUID](../data-types/uuid.md). + +**Returned value** + +- Returns `1` for an empty UUID or `0` for a non-empty UUID. [UInt8](../data-types/int-uint.md). + +**Example** + +To generate the UUID value, ClickHouse provides the [generateUUIDv4](#generateuuidv4) function. + +Query: + +```sql +SELECT empty(generateUUIDv4()); +``` + +Result: + +```response +┌─empty(generateUUIDv4())─┐ +│ 0 │ +└─────────────────────────┘ +``` + +## notEmpty + +Checks whether the input UUID is non-empty. + +**Syntax** + +```sql +notEmpty(UUID) +``` + +The UUID is considered empty if it contains all zeros (zero UUID). + +The function also works for [Arrays](array-functions.md#function-notempty) or [Strings](string-functions.md#notempty). + +**Arguments** + +- `x` — A UUID. [UUID](../data-types/uuid.md). + +**Returned value** + +- Returns `1` for a non-empty UUID or `0` for an empty UUID. [UInt8](../data-types/int-uint.md). + +**Example** + +To generate the UUID value, ClickHouse provides the [generateUUIDv4](#generateuuidv4) function. + +Query: + +```sql +SELECT notEmpty(generateUUIDv4()); +``` + +Result: + +```response +┌─notEmpty(generateUUIDv4())─┐ +│ 1 │ +└────────────────────────────┘ +``` + +## toUUID + +Converts a value of type String to a UUID. + +``` sql +toUUID(string) +``` + +**Returned value** + +The UUID type value. + +**Usage example** + +``` sql +SELECT toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0') AS uuid +``` + +Result: + +```response +┌─────────────────────────────────uuid─┐ +│ 61f0c404-5cb3-11e7-907b-a6006ad3dba0 │ +└──────────────────────────────────────┘ +``` + +## toUUIDOrDefault + +**Arguments** + +- `string` — String of 36 characters or FixedString(36). [String](../syntax.md#string). +- `default` — UUID to be used as the default if the first argument cannot be converted to a UUID type. [UUID](../data-types/uuid.md). + +**Returned value** + +UUID + +``` sql +toUUIDOrDefault(string, default) +``` + +**Returned value** + +The UUID type value. + +**Usage examples** + +This first example returns the first argument converted to a UUID type as it can be converted: + +``` sql +SELECT toUUIDOrDefault('61f0c404-5cb3-11e7-907b-a6006ad3dba0', cast('59f0c404-5cb3-11e7-907b-a6006ad3dba0' as UUID)); +``` + +Result: + +```response +┌─toUUIDOrDefault('61f0c404-5cb3-11e7-907b-a6006ad3dba0', CAST('59f0c404-5cb3-11e7-907b-a6006ad3dba0', 'UUID'))─┐ +│ 61f0c404-5cb3-11e7-907b-a6006ad3dba0 │ +└───────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +This second example returns the second argument (the provided default UUID) as the first argument cannot be converted to a UUID type: + +```sql +SELECT toUUIDOrDefault('-----61f0c404-5cb3-11e7-907b-a6006ad3dba0', cast('59f0c404-5cb3-11e7-907b-a6006ad3dba0' as UUID)); +``` + +Result: + +```response +┌─toUUIDOrDefault('-----61f0c404-5cb3-11e7-907b-a6006ad3dba0', CAST('59f0c404-5cb3-11e7-907b-a6006ad3dba0', 'UUID'))─┐ +│ 59f0c404-5cb3-11e7-907b-a6006ad3dba0 │ +└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +## toUUIDOrNull + +Takes an argument of type String and tries to parse it into UUID. If failed, returns NULL. + +``` sql +toUUIDOrNull(string) +``` + +**Returned value** + +The Nullable(UUID) type value. + +**Usage example** + +``` sql +SELECT toUUIDOrNull('61f0c404-5cb3-11e7-907b-a6006ad3dba0T') AS uuid +``` + +Result: + +```response +┌─uuid─┐ +│ ᴺᵁᴸᴸ │ +└──────┘ +``` + +## toUUIDOrZero + +It takes an argument of type String and tries to parse it into UUID. If failed, returns zero UUID. + +``` sql +toUUIDOrZero(string) +``` + +**Returned value** + +The UUID type value. + +**Usage example** + +``` sql +SELECT toUUIDOrZero('61f0c404-5cb3-11e7-907b-a6006ad3dba0T') AS uuid +``` + +Result: + +```response +┌─────────────────────────────────uuid─┐ +│ 00000000-0000-0000-0000-000000000000 │ +└──────────────────────────────────────┘ +``` + +## UUIDStringToNum + +Accepts `string` containing 36 characters in the format `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`, and returns a [FixedString(16)](../data-types/fixedstring.md) as its binary representation, with its format optionally specified by `variant` (`Big-endian` by default). + +**Syntax** + +``` sql +UUIDStringToNum(string[, variant = 1]) +``` + +**Arguments** + +- `string` — A [String](../syntax.md#syntax-string-literal) of 36 characters or [FixedString](../syntax.md#syntax-string-literal) +- `variant` — Integer, representing a variant as specified by [RFC4122](https://datatracker.ietf.org/doc/html/rfc4122#section-4.1.1). 1 = `Big-endian` (default), 2 = `Microsoft`. + +**Returned value** + +FixedString(16) + +**Usage examples** + +``` sql +SELECT + '612f3c40-5d3b-217e-707b-6a546a3d7b29' AS uuid, + UUIDStringToNum(uuid) AS bytes +``` + +Result: + +```response +┌─uuid─────────────────────────────────┬─bytes────────────┐ +│ 612f3c40-5d3b-217e-707b-6a546a3d7b29 │ a/<@];!~p{jTj={) │ +└──────────────────────────────────────┴──────────────────┘ +``` + +``` sql +SELECT + '612f3c40-5d3b-217e-707b-6a546a3d7b29' AS uuid, + UUIDStringToNum(uuid, 2) AS bytes +``` + +Result: + +```response +┌─uuid─────────────────────────────────┬─bytes────────────┐ +│ 612f3c40-5d3b-217e-707b-6a546a3d7b29 │ @ clickhouse_applications[] = {"storage-tools", mainEntryStorageTools}, {"storage_tools", mainEntryStorageTools}, #endif +#if ENABLE_CLICKHOUSE_SCHEMA_ADVISOR + {"schema-advisor", mainEntryClickHouseSchemaAdvisor}, +#endif }; diff --git a/programs/schema-advisor/CMakeLists.txt b/programs/schema-advisor/CMakeLists.txt new file mode 100644 index 00000000000..c8327d5e751 --- /dev/null +++ b/programs/schema-advisor/CMakeLists.txt @@ -0,0 +1,29 @@ +set(CLICKHOUSE_SCHEMA_ADVISOR_SOURCES + SchemaAdvisor.cpp + CodecAdvisor.cpp + TypeAdvisor.cpp + IndexAdvisor.cpp + PrewhereAdvisor.cpp + SampleColumnReader.cpp + Statistics.cpp + CompressedStatisticsCollectBuffer.cpp + ColumnUsageExtractor.cpp + MockGlobalContext.cpp + MockEnvironment.cpp +) + +set(CLICKHOUSE_SCHEMA_ADVISOR_LINK + PRIVATE + boost::program_options + clickhouse_functions + clickhouse_aggregate_functions + clickhouse_parsers + dbms + clickhouse_storages_system +) + +if (CLICKHOUSE_SPLIT_BINARY) + list(APPEND CLICKHOUSE_SCHEMA_ADVISOR_LINK $) +endif() + +clickhouse_program_add(schema-advisor) diff --git a/programs/schema-advisor/CodecAdvisor.cpp b/programs/schema-advisor/CodecAdvisor.cpp new file mode 100644 index 00000000000..f0b3bac38be --- /dev/null +++ b/programs/schema-advisor/CodecAdvisor.cpp @@ -0,0 +1,212 @@ +#include "CodecAdvisor.h" +#include "CompressedStatisticsCollectBuffer.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +CodecAdvisor::CodecAdvisor( + const po::variables_map & options, + const ColumnsDescription & column_descs, + std::string absolute_part_path_, + size_t sample_row_number_, + size_t max_threads_) + : absolute_part_path(std::move(absolute_part_path_)) + , sample_row_number(sample_row_number_) + , max_threads(max_threads_) +{ + parseCodecCandidates(options); + setSamplingColumnFiles(absolute_part_path, column_descs); +} + +void CodecAdvisor::parseCodecCandidates(const po::variables_map & options) +{ + block_size = options["block-size"].as(); + + bool use_lz4hc = options.count("hc"); + bool use_zstd = options.count("zstd"); + std::vector combi_codec; + if (options.count("codec")) + combi_codec = options["codec"].as>(); + + if (!use_lz4hc && !use_zstd && combi_codec.empty()) + throw Exception( + "Missing options, either --hc or --zstd or --codec options is required", ErrorCodes::BAD_ARGUMENTS); + if ((use_lz4hc || use_zstd) && !combi_codec.empty()) + throw Exception( + "Wrong options, codec flags like --zstd and --codec options are mutually exclusive", ErrorCodes::BAD_ARGUMENTS); + if (!combi_codec.empty() && options.count("level")) + throw Exception("Wrong options, --level is not compatible with --codec list", ErrorCodes::BAD_ARGUMENTS); + + std::string method_family; + if (use_lz4hc) + method_family = "LZ4HC"; + else if (use_zstd) + method_family = "ZSTD"; + + std::optional level = std::nullopt; + if (options.count("level")) + level = options["level"].as(); + + CompressionCodecPtr codec; + if (!combi_codec.empty()) + { + ParserCodec codec_parser; + std::string combi_codec_line = boost::algorithm::join(combi_codec, ","); + auto ast = parseQuery(codec_parser, "(" + combi_codec_line + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + codec = CompressionCodecFactory::instance().get(ast, nullptr); + } + else + codec = CompressionCodecFactory::instance().get(method_family, level); + + codecs_to_compare.push_back(codec); +} + +/// Select column files to sample and estimate profit +void CodecAdvisor::setSamplingColumnFiles(const std::string & part_path, const ColumnsDescription & column_descs) +{ + Poco::DirectoryIterator end; + for (Poco::DirectoryIterator it(part_path); it != end; ++it) + { + if (it->isFile() && endsWith(it->path(), ".bin")) + { + std::string file_path = it->path(); + std::string file_name = it.name(); + std::string column_name; + if (isMapImplicitKey(file_name) && !isMapBaseFile(file_name)) + column_name = parseMapNameFromImplicitFileName(file_name); + else if (endsWith(it->path(), ".null.bin")) + column_name = unescapeForFileName(file_name.substr(0, file_name.size() - 9)); + else + column_name = unescapeForFileName(file_name.substr(0, file_name.size() - 4)); + + if (column_descs.has(column_name)) + column_files_to_sample.push_back(std::make_shared(file_path, column_name)); + } + } +} + +void CodecAdvisor::execute() +{ + size_t part_row_count; + std::string part_count_path = absolute_part_path + "/count.txt"; + { + ReadBufferFromFile in(part_count_path, METADATA_FILE_BUFFER_SIZE); + readIntText(part_row_count, in); + assertEOF(in); + } + + auto run_estimate_task = [&](const SamplingColumnFilePtr & column_file_to_sample) { + std::string file_path = column_file_to_sample->file_path; + column_file_to_sample->origin_file_size = std::filesystem::file_size(file_path) * sample_row_number / part_row_count; + + CompressedReadBufferFromFile from(std::make_unique(file_path), true, 0, column_file_to_sample->origin_file_size, true); + CompressedStatisticsCollectBuffer to(codecs_to_compare[0], block_size); /// TODO(weiping.qw): support comparing multiple codecs after FSST is imported. + copyData(from, to); + + column_file_to_sample->optimized_file_size = to.getCompressedBytes(); + }; + + ExceptionHandler exception_handler; + ///make queue size large enough to hold all tasks. + ThreadPool pool(max_threads, max_threads, 100000); + + for (const auto & file : column_files_to_sample) + { + pool.trySchedule( + createExceptionHandledJob( + [&, column_file_to_sample = file]() { run_estimate_task(column_file_to_sample); } + , exception_handler + ) + ); + } + pool.wait(); + /// throw if exception during collecting compression info. + exception_handler.throwIfException(); +} + +void CodecAdvisor::serializeJson(WriteBuffer & buf, bool verbose) +{ + size_t total_origin_file_size = 0; + size_t total_optimized_file_size = 0; + + std::unordered_map column_origin_file_sizes; + std::unordered_map column_optimized_file_sizes; + for (const auto & file : column_files_to_sample) + { + /// skip column without potential compression profit + if (file->origin_file_size <= file->optimized_file_size) + continue; + + total_origin_file_size += file->origin_file_size; + total_optimized_file_size += file->optimized_file_size; + if (verbose) + { + if (column_origin_file_sizes.find(file->column_name) == column_origin_file_sizes.end()) + { + column_origin_file_sizes.emplace(file->column_name, file->origin_file_size); + column_optimized_file_sizes.emplace(file->column_name, file->optimized_file_size); + } + else + { + column_origin_file_sizes[file->column_name] += file->origin_file_size; + column_optimized_file_sizes[file->column_name] += file->optimized_file_size; + } + } + } + + if (verbose) + { + bool first = true; + writeString("\"columns\":[", buf); + for (const auto & entry : column_origin_file_sizes) + { + if (first) + first = false; + else + writeString(",", buf); + std::string column_name = entry.first; + writeString("{\"name\":\"", buf); + writeString(column_name, buf); + writeString("\",", buf); + size_t column_origin_file_size = entry.second; + size_t column_optimized_file_size = column_optimized_file_sizes[column_name]; + double column_estimated_profit = + (column_origin_file_size == 0 || column_origin_file_size <= column_optimized_file_size) + ? 0 : (column_origin_file_size - column_optimized_file_size) * 100.0 / column_origin_file_size; + writeString("\"codec\":{\"", buf); + writeString(queryToString(codecs_to_compare[0]->getCodecDesc()), buf); + writeString("\":{\"compression ratio\":", buf); + writeFloatText(column_estimated_profit, buf); + writeString("}}}", buf); + } + writeString("],", buf); + } + + double estimated_profit = (total_origin_file_size - total_optimized_file_size) * 100.0 / total_origin_file_size; + + writeString("\"codec\":{\"compression ratio\":", buf); + writeFloatText(estimated_profit, buf); + writeString("}", buf); +} + +} diff --git a/programs/schema-advisor/CodecAdvisor.h b/programs/schema-advisor/CodecAdvisor.h new file mode 100644 index 00000000000..610e5df12ab --- /dev/null +++ b/programs/schema-advisor/CodecAdvisor.h @@ -0,0 +1,43 @@ +#pragma once + +#include + +#include "SchemaAdvisorHelpers.h" + +#include +#include +#include + +namespace DB +{ + +namespace po = boost::program_options; + +class CodecAdvisor +{ +private: + SamplingColumnFiles column_files_to_sample; + Codecs codecs_to_compare; + const std::string absolute_part_path; + const size_t sample_row_number; + const size_t max_threads; + unsigned block_size; + + void parseCodecCandidates(const po::variables_map & options); + void setSamplingColumnFiles(const std::string & part_path, const ColumnsDescription & column_descs); + +public: + CodecAdvisor( + const po::variables_map & options, + const ColumnsDescription & column_descs, + std::string absolute_part_path, + size_t sample_row_number, + size_t max_threads); + + virtual ~CodecAdvisor() = default; + + void execute(); + void serializeJson(WriteBuffer & buf, bool verbose = false); +}; + +} diff --git a/programs/schema-advisor/ColumnUsageExtractor.cpp b/programs/schema-advisor/ColumnUsageExtractor.cpp new file mode 100644 index 00000000000..f080d09bbc6 --- /dev/null +++ b/programs/schema-advisor/ColumnUsageExtractor.cpp @@ -0,0 +1,223 @@ +#include "ColumnUsageExtractor.h" +#include "SchemaAdvisorHelpers.h" +#include "MockEnvironment.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace +{ + WorkloadQueries buildWorkloadQueriesCollectException(const std::vector & queries, + ContextPtr from_context, + ThreadPool & query_thread_pool, + MessageCollector & collector) + { + WorkloadQueries res(queries.size()); + for (size_t i = 0; i < queries.size(); ++i) + { + query_thread_pool.scheduleOrThrowOnError([&, i] { + setThreadName("BuildQuery"); + try + { + res[i] = WorkloadQuery::build("q" + std::to_string(i), queries[i], from_context); + } + catch (...) + { + std::string msg = "failed to build query " + std::to_string(i) + "\nreason: " + getCurrentExceptionMessage(true) + + "\nsql: " + queries[i] + "\n"; + collector.collect(std::move(msg)); + } + }); + } + query_thread_pool.wait(); + res.erase(std::remove(res.begin(), res.end(), nullptr), res.end()); + return res; + } +} + +ColumnUsages ColumnUsageExtractor::extractColumnUsages(const std::vector & queries) const +{ + ThreadPool query_thread_pool{std::min(max_threads, queries.size())}; + MessageCollector collector; + WorkloadQueries workload_queries = buildWorkloadQueriesCollectException(queries, context, query_thread_pool, collector); + if (queries.empty()) + throw Exception("No valid query has been extracted", ErrorCodes::BAD_ARGUMENTS); + + LOG_DEBUG(getLogger("ColumnUsageExtractor"), "Successfully planed {} / {} queries", workload_queries.size(), queries.size()); + collector.logCollectedError(); + + return buildColumnUsages(workload_queries); +} + +ColumnUsageExtractor::ColumnToScannedUsages ColumnUsageExtractor::extractUsageForLowCardinality(const ColumnUsages & column_usages) const +{ + ColumnToScannedUsages res; + for (const auto & [column, info] : column_usages) + { + if (MockEnvironment::isPrimaryKey(column, context)) + { + LOG_DEBUG(getLogger("ColumnUsageExtractor"), "Column {} skipped because it is a primary key", column.getFullName()); + continue; + } + + auto scanned = info.getUsages(ColumnUsageType::SCANNED, /*only_source_table=*/false); + if (scanned.empty()) + continue; + + res.emplace(column, scanned.size()); + } + + return res; +} + +ColumnUsageExtractor::ColumnToEqualityAndInUsages ColumnUsageExtractor::extractUsageForSkipIndex(const ColumnUsages & column_usages) const +{ + /// if only interested in a specific table, do it here + // std::erase_if(column_usages, [&](const auto & pair) { return pair.first.database != database || pair.first.table != table;}); + + ColumnToEqualityAndInUsages res; + for (const auto & [column, info] : column_usages) + { + if (MockEnvironment::isPrimaryKey(column, context)) + { + LOG_DEBUG(getLogger("ColumnUsageExtractor"), "Column {} skipped because it is a primary key", column.getFullName()); + continue; + } + + size_t arraysetfunc_count = info.getFrequency(ColumnUsageType::ARRAY_SET_FUNCTION, /*only_source_table=*/false); + size_t others_count = info.getFrequency(ColumnUsageType::OTHER_PREDICATE, /*only_source_table=*/false); + + if (arraysetfunc_count) + { + auto arraysetfuncs = info.getUsages(ColumnUsageType::ARRAY_SET_FUNCTION, /*only_source_table=*/false); + size_t total_count = arraysetfuncs.size() + others_count; + /// TODO: Optimize the ColumnToEqualityAndInUsages struct? + res.emplace(column, std::make_tuple(std::move(arraysetfuncs), std::vector{}, total_count)); + + continue; + } + + auto equalities = info.getUsages(ColumnUsageType::EQUALITY_PREDICATE, /*only_source_table=*/false); + auto ins = info.getUsages(ColumnUsageType::IN_PREDICATE, /*only_source_table=*/false); + size_t ranges_count = info.getFrequency(ColumnUsageType::RANGE_PREDICATE, /*only_source_table=*/false); + + size_t total_count = equalities.size() + ins.size() + ranges_count + others_count; + if (total_count == 0) + { + LOG_DEBUG( + getLogger("ColumnUsageExtractor"), + "Column {} skipped, total count: {}", + column.getFullName(), + total_count); + continue; + } + + // Keep the set size threshold limit on + // Remove in lists whose in set size is larger than IN_LIST_SIZE_UPPER_BOUND + std::erase_if(ins, [](const ColumnUsage & usage) { + if (auto func = dynamic_pointer_cast(usage.expression); func && func->name == "in") + if (auto expr_list = dynamic_pointer_cast(func->arguments); expr_list && expr_list->children.size() == 2) + if (auto tuple = dynamic_pointer_cast(expr_list->children[1]); tuple && tuple->name == "tuple") + if (auto tuple_expr = dynamic_pointer_cast(tuple->arguments)) + return tuple_expr->children.size() > IN_LIST_SIZE_UPPER_BOUND; + return true; + }); + + size_t eq_in_count = equalities.size() + ins.size(); + if (eq_in_count == 0) + { + LOG_DEBUG( + getLogger("ColumnUsageExtractor"), + "Column {} skipped, eq & in count: {}, total count: {}", + column.getFullName(), + eq_in_count, + total_count); + continue; + } + + // Temply loosen the restrictions of in+equals proportion + if (eq_in_count * 1.0 < total_count * EQUALITY_AND_IN_PREDICATE_THRESHOLD) + { + LOG_DEBUG( + getLogger("ColumnUsageExtractor"), + "Column {} maybe skipped, eq & in count: {}, total count: {}", + column.getFullName(), + eq_in_count, + total_count); + continue; + } + + LOG_DEBUG( + getLogger("ColumnUsageExtractor"), + "Column {} added, eq & in count: {}, total count: {}", + column.getFullName(), + eq_in_count, + total_count); + + res.emplace(column, std::make_tuple(std::move(equalities), std::move(ins), total_count)); + } + return res; +} + +ColumnUsageExtractor::ColumnToPrewherePredicateUsages ColumnUsageExtractor::extractUsageForPrewhere(const ColumnUsages & column_usages) const +{ + /// if only interested in a specific table, do it here + // std::erase_if(column_usages, [&](const auto & pair) { return pair.first.database != database || pair.first.table != table;}); + + ColumnToPrewherePredicateUsages res; + for (const auto & [column, info] : column_usages) + { + if (MockEnvironment::isPrimaryKey(column, context)) + { + LOG_DEBUG(getLogger("ColumnUsageExtractor"), "Column {} skipped because it is a primary key", column.getFullName()); + continue; + } + + auto equalities = info.getUsages(ColumnUsageType::EQUALITY_PREDICATE, /*only_source_table=*/false); + auto ins = info.getUsages(ColumnUsageType::IN_PREDICATE, /*only_source_table=*/false); + auto ranges = info.getUsages(ColumnUsageType::RANGE_PREDICATE, /*only_source_table=*/false); + auto others = info.getUsages(ColumnUsageType::OTHER_PREDICATE, /*only_source_table=*/false); + + size_t total_count = equalities.size() + ins.size() + ranges.size() + others.size(); + + if (total_count == 0) + continue; + + // Keep the set size threshold limit on + // Remove in lists whose in set size is larger than IN_LIST_SIZE_UPPER_BOUND + std::erase_if(ins, [](const ColumnUsage & usage) { + if (auto func = dynamic_pointer_cast(usage.expression); func && func->name == "in") + if (auto expr_list = dynamic_pointer_cast(func->arguments); expr_list && expr_list->children.size() == 2) + if (auto tuple = dynamic_pointer_cast(expr_list->children[1]); tuple && tuple->name == "tuple") + if (auto tuple_expr = dynamic_pointer_cast(tuple->arguments)) + return tuple_expr->children.size() > IN_LIST_SIZE_UPPER_BOUND; + return true; + }); + + res.emplace(column, std::make_tuple(std::move(equalities), std::move(ins), std::move(ranges), std::move(others), total_count)); + } + return res; +} + +} // DB diff --git a/programs/schema-advisor/ColumnUsageExtractor.h b/programs/schema-advisor/ColumnUsageExtractor.h new file mode 100644 index 00000000000..a29633ffca7 --- /dev/null +++ b/programs/schema-advisor/ColumnUsageExtractor.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace DB +{ + +class ColumnUsageExtractor +{ +public: + // EqualityAndInUsages: + // for skip index: equality_usages, in_usages, total_predicates + // for bitmap index: arraysetfunc_usages, {}, total_predicates + using EqualityAndInUsages = std::tuple, std::vector, size_t>; + using ColumnToEqualityAndInUsages = std::unordered_map; + using ColumnToScannedUsages = std::unordered_map; + + // for prewhere: equality_usages, in_usages, range_usages, other_usages, total_predicates + using PrewherePredicateUsages = std::tuple, std::vector, std::vector, std::vector, size_t>; + using ColumnToPrewherePredicateUsages = std::unordered_map; + + + explicit ColumnUsageExtractor(ContextMutablePtr _context, size_t _max_threads): context(_context), max_threads(_max_threads) {} + + ColumnUsages extractColumnUsages(const std::vector & queries) const; + ColumnToEqualityAndInUsages extractUsageForSkipIndex(const ColumnUsages & column_usages) const; + ColumnToPrewherePredicateUsages extractUsageForPrewhere(const ColumnUsages & column_usages) const; + + ColumnToScannedUsages extractUsageForLowCardinality(const ColumnUsages & column_usages) const; + +private: + ContextMutablePtr context; + size_t max_threads; + // which "in" filters are considered interesting + static constexpr size_t IN_LIST_SIZE_UPPER_BOUND = 10; + static constexpr float EQUALITY_AND_IN_PREDICATE_THRESHOLD = 0.5; +}; + +} // DB diff --git a/programs/schema-advisor/CompressedStatisticsCollectBuffer.cpp b/programs/schema-advisor/CompressedStatisticsCollectBuffer.cpp new file mode 100644 index 00000000000..d052fb5ceeb --- /dev/null +++ b/programs/schema-advisor/CompressedStatisticsCollectBuffer.cpp @@ -0,0 +1,54 @@ +#include +#include + +#include +#include + +#include "CompressedStatisticsCollectBuffer.h" +#include + +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ +} + +static constexpr auto CHECKSUM_SIZE{sizeof(CityHash_v1_0_2::uint128)}; + +void CompressedStatisticsCollectBuffer::nextImpl() +{ + if (!offset()) + return; + + size_t decompressed_size = offset(); + UInt32 compressed_reserve_size = codec->getCompressedReserveSize(decompressed_size); + compressed_buffer.resize(compressed_reserve_size); + UInt32 compressed_size = codec->compress(working_buffer.begin(), decompressed_size, compressed_buffer.data()); + + // FIXME remove this after fixing msan report in lz4. + // Almost always reproduces on stateless tests, the exact test unknown. + __msan_unpoison(compressed_buffer.data(), compressed_size); + + total_compressed_size += CHECKSUM_SIZE + compressed_size; +} + + +CompressedStatisticsCollectBuffer::CompressedStatisticsCollectBuffer( + CompressionCodecPtr codec_, + size_t buf_size) + : BufferWithOwnMemory(buf_size), codec(std::move(codec_)) +{ +} + +CompressedStatisticsCollectBuffer::~CompressedStatisticsCollectBuffer() +{ + /// FIXME move final flush into the caller + next(); +} + +} diff --git a/programs/schema-advisor/CompressedStatisticsCollectBuffer.h b/programs/schema-advisor/CompressedStatisticsCollectBuffer.h new file mode 100644 index 00000000000..84f0b31c36b --- /dev/null +++ b/programs/schema-advisor/CompressedStatisticsCollectBuffer.h @@ -0,0 +1,53 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include + + +namespace DB +{ + +class CompressedStatisticsCollectBuffer : public BufferWithOwnMemory +{ +private: + CompressionCodecPtr codec; + PODArray compressed_buffer; + size_t total_compressed_size = 0; + + void nextImpl() override; + +public: + CompressedStatisticsCollectBuffer( + CompressionCodecPtr codec_ = CompressionCodecFactory::instance().getDefaultCodec(), + size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE); + + /// The amount of compressed data + size_t getCompressedBytes() + { + next(); + return total_compressed_size; + } + + /// How many uncompressed bytes were written to the buffer + size_t getUncompressedBytes() + { + return count(); + } + + /// How many bytes are in the buffer (not yet compressed) + size_t getRemainingBytes() + { + nextIfAtEnd(); + return offset(); + } + + ~CompressedStatisticsCollectBuffer() override; +}; + +} diff --git a/programs/schema-advisor/IndexAdvisor.cpp b/programs/schema-advisor/IndexAdvisor.cpp new file mode 100644 index 00000000000..e60f186a989 --- /dev/null +++ b/programs/schema-advisor/IndexAdvisor.cpp @@ -0,0 +1,274 @@ +#include "IndexAdvisor.h" +#include "ColumnUsageExtractor.h" +#include "IO/WriteIntText.h" +#include "SampleColumnReader.h" +#include "Statistics.h" +#include "SchemaAdvisorHelpers.h" +#include "PotentialColumn.h" + +#include +#include +#include + +#include "Common/Exception.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +static constexpr double ADVISOR_HIGH_CARDINALITY_NDV_THRESHOLD = 0.33; + +IndexAdvisor::IndexAdvisor( + MockEnvironment & env_, + const po::variables_map & options_, + size_t sample_row_number_, + size_t max_threads_) + : env(env_) + , options(options_) + , sample_row_number(sample_row_number_) + , max_threads(max_threads_) +{ +} + +// High_Cardinality_Threshold: +// 20w / 65536 = sample_row_number / ndv => ndv ～ 1/3 sample_row_number +// For skip index, ndv > High_Cardinality_Threshold +// For bitmap index, 10 < ndv <= High_Cardinality_Threshold +bool checkColumnCardinality(String column_name, size_t ndv, size_t sample_row_number, PotentialIndexType & type) +{ + bool basic_cardinality = ndv > 10; + bool high_cardinality = ndv > ADVISOR_HIGH_CARDINALITY_NDV_THRESHOLD * sample_row_number; + + auto get_ndv_check_msg = [&]() -> String + { + if (type == PotentialIndexType::BITMAP_INDEX) + { + if (!basic_cardinality) + return fmt::format("Column {} skipped because the ndv ({}) is less than 10", column_name, ndv); + if (high_cardinality) + type = PotentialIndexType::SEGMENT_BITMAP_INDEX; + } + if (type == PotentialIndexType::BLOOM_FILTER) + { + if (!high_cardinality) + return fmt::format("Column {} skipped because of the array ndv({}) / sample_rows({}) < threshold({})", column_name, ndv, sample_row_number, ADVISOR_HIGH_CARDINALITY_NDV_THRESHOLD); + } + return ""; + }; + + auto check_ndv_msg = get_ndv_check_msg(); + if (!check_ndv_msg.empty()) + { + LOG_DEBUG(getLogger("ColumnUsageExtractor"), check_ndv_msg); + return false; + } + return true; +}; + +void IndexAdvisor::execute() +{ + auto context = createContext(options, env); + auto queries = loadQueries(options); + + LOG_DEBUG(getLogger("ColumnUsageExtractor"), "++++++++++ begin to executor index advisor ++++++++++"); + + ColumnUsageExtractor extractor(context, max_threads); + auto column_usages = extractor.extractColumnUsages(queries); + auto skip_index_usages = extractor.extractUsageForSkipIndex(column_usages); + + auto make_predicate_info = [&](PredicateInfos & predicate_infos, ColumnUsageType predicate_type, const std::vector & column_usages_) { + PredicateExpressions predicate_expressions; + size_t total_predicate_expression = 0; + for (const auto & equality_usage : column_usages_) + { + auto & count = predicate_expressions[equality_usage.expression]; + ++count; + total_predicate_expression++; + } + predicate_infos.insert({predicate_type, {predicate_expressions, total_predicate_expression}}); + }; + + UniExtract uniq_extract; + for (const auto & index_usage : skip_index_usages) + { + auto column_info = index_usage.first; + + auto storage = MockEnvironment::tryGetLocalTable(column_info.database, column_info.table, context); + if (!storage) + throw Exception(column_info.database + "(" + column_info.table + "): can not find local table.", ErrorCodes::NOT_FOUND_EXPECTED_DATA_PART); + + auto metadata = storage->getInMemoryMetadataCopy(); + auto column_and_type = metadata.getColumns().tryGetColumn(GetColumnsOptions::Kind::AllPhysical, column_info.column); + if (!column_and_type) + continue; + + auto column_type = column_and_type->type; + + bool check_bitmap_index = false; + bool already_bitmap_index = false; + if (isArray(column_type)) + { + if (column_type->isBitmapIndex() || column_type->isSegmentBitmapIndex()) + { + LOG_DEBUG(getLogger("ColumnUsageExtractor"), "Column " + column_info.column + " skipped because has already been a bitmap index column"); + // continue; + already_bitmap_index = true; + } + check_bitmap_index = true; + } + + std::vector data_path_list; + // if (options.count("path")) + // { + // std::string path = options["path"].as(); + // if (!endsWith(path, "/")) + // path.append("/"); + // data_path_list.emplace_back(path); + // } + // else + // { + boost::split(data_path_list, options["data-path-list"].as(), boost::is_any_of(" ,")); + for (auto & i : data_path_list) + { + if (!endsWith(i, "/")) + i = i.append("/"); + } + // } + + std::string absolute_part_path; + try + { + absolute_part_path = selectPartPath(options, data_path_list, storage->getStorageID().getDatabaseName(), storage->getStorageID().getTableName(), sample_row_number); + } + catch (Exception & e) + { + if (e.code() == ErrorCodes::NOT_FOUND_EXPECTED_DATA_PART) + { + LOG_DEBUG( + getLogger("ColumnUsageExtractor"), + "Can't find suitable part for table " + column_info.database + "." + column_info.table + + ", maybe because of the total part rows < " + std::to_string(sample_row_number)); + continue; + } + else + throw e; + } + + SampleColumnReader reader(absolute_part_path + "/", 0, sample_row_number); + ColumnPtr column; + try + { + column = reader.readColumn({index_usage.first.column, column_type}); + } + catch (...) + { + // Just skip the column if it can't be read + LOG_DEBUG( + getLogger("ColumnUsageExtractor"), + "Can't read column file " + index_usage.first.column + " from table " + column_info.database + "." + column_info.table + + ", error message: " + + getCurrentExceptionMessage(true)); + continue; + } + + if (check_bitmap_index) + { + size_t ndv = uniq_extract.executeOnColumnArray(column, column_type).get(); + auto bitmap_index_type = already_bitmap_index ? PotentialIndexType::ALREADY_BITMAP_INDEX : PotentialIndexType::BITMAP_INDEX; + if (!checkColumnCardinality(column_info.getFullName(), ndv, sample_row_number, bitmap_index_type)) + continue; + + StatisticInfo statistic_info{ndv, sample_row_number, std::get<2>(index_usage.second)}; + + PredicateInfos predicate_infos; + make_predicate_info(predicate_infos, ColumnUsageType::ARRAY_SET_FUNCTION, std::get<0>(index_usage.second)); + + PotentialColumnInfo potential_column{bitmap_index_type, statistic_info, predicate_infos}; + potential_columns.insert({std::move(column_info), std::move(potential_column)}); + + continue; + } + + // All following: check skip index + size_t ndv = uniq_extract.executeOnColumn(column, column_type).get(); + auto skip_index_type = PotentialIndexType::BLOOM_FILTER; + if (!checkColumnCardinality(column_info.getFullName(), ndv, sample_row_number, skip_index_type)) + continue; + + StatisticInfo statistic_info{ndv, sample_row_number, std::get<2>(index_usage.second)}; + + PredicateInfos predicate_infos; + make_predicate_info(predicate_infos, ColumnUsageType::EQUALITY_PREDICATE, std::get<0>(index_usage.second)); + make_predicate_info(predicate_infos, ColumnUsageType::IN_PREDICATE, std::get<1>(index_usage.second)); + + PotentialColumnInfo potential_column{skip_index_type, statistic_info, predicate_infos}; + potential_columns.insert({std::move(column_info), std::move(potential_column)}); + } + + LOG_DEBUG(getLogger("ColumnUsageExtractor"), "Extracted {} column usages", potential_columns.size()); + for ([[maybe_unused]] auto & [column_info, potential_column] : potential_columns) + { + std::stringstream ss; + ss << column_info.getFullName() << "\tindex_type:" << toString(potential_column.index_type) + << "\tsample_ndv:" << potential_column.statistic_info.sample_ndv + << "\tsample_row_num:" << potential_column.statistic_info.sample_row_num << "\ttarget_expression_cnt:" + << potential_column.predicate_infos[ColumnUsageType::EQUALITY_PREDICATE].total + + potential_column.predicate_infos[ColumnUsageType::IN_PREDICATE].total + + potential_column.predicate_infos[ColumnUsageType::ARRAY_SET_FUNCTION].total + << "\ttotal_expr count:" << potential_column.statistic_info.total_predicates; + + LOG_DEBUG(getLogger("ColumnUsageExtractor"), ss.str()); + } +} + +/// TODO: seperate two indices +void IndexAdvisor::serializeJson(WriteBuffer & buf, bool /* verbose */) +{ + bool first = true; + writeString("\"index\":[", buf); + for (auto & [column_info, potential_column] : potential_columns) + { + if (first) + first = false; + else + writeString(",", buf); + + writeString(R"({"db":")", buf); + writeString(column_info.database, buf); + writeString(R"(","table":")", buf); + writeString(column_info.table, buf); + writeString(R"(","column_name":")", buf); + writeString(column_info.column, buf); + + writeString(R"(","index_type":")", buf); + writeString(toString(potential_column.index_type), buf); + + writeString(R"(","sample_ndv":")", buf); + writeIntText(potential_column.statistic_info.sample_ndv, buf); + writeString(R"(","sample_row_num":")", buf); + writeIntText(potential_column.statistic_info.sample_row_num, buf); + + // The usage type (EQUALITY_PREDICATE + IN_PREDICATE) and (ARRAY_SET_FUNCTION) + // will not appear at the same time, so we can simply add the cnt + size_t target_expression_cnt = potential_column.predicate_infos[ColumnUsageType::EQUALITY_PREDICATE].total + + potential_column.predicate_infos[ColumnUsageType::IN_PREDICATE].total + + potential_column.predicate_infos[ColumnUsageType::ARRAY_SET_FUNCTION].total; + writeString(R"(","target_expression_cnt":")", buf); + writeIntText(target_expression_cnt, buf); + writeString(R"(","total_expression_cnt":")", buf); + writeIntText(potential_column.statistic_info.total_predicates, buf); + writeString("\"}", buf); + } + writeString("]", buf); +} + +} diff --git a/programs/schema-advisor/IndexAdvisor.h b/programs/schema-advisor/IndexAdvisor.h new file mode 100644 index 00000000000..de259fa9a63 --- /dev/null +++ b/programs/schema-advisor/IndexAdvisor.h @@ -0,0 +1,39 @@ +#pragma once + +#include "MockEnvironment.h" +#include "PotentialColumn.h" + +#include + +#include +#include + +namespace DB +{ + +namespace po = boost::program_options; + +class IndexAdvisor +{ +private: + MockEnvironment & env; + po::variables_map options; + const size_t sample_row_number; + const size_t max_threads; + + PotentialColumns potential_columns; + +public: + IndexAdvisor( + MockEnvironment & env_, + const po::variables_map & options_, + size_t sample_row_number, + size_t max_threads); + + virtual ~IndexAdvisor() = default; + + void execute(); + void serializeJson(WriteBuffer & buf, bool verbose = false); +}; + +} diff --git a/programs/schema-advisor/MockEnvironment.cpp b/programs/schema-advisor/MockEnvironment.cpp new file mode 100644 index 00000000000..211c34a7956 --- /dev/null +++ b/programs/schema-advisor/MockEnvironment.cpp @@ -0,0 +1,330 @@ +#include "MockEnvironment.h" +#include "MockGlobalContext.h" +#include "SchemaAdvisorHelpers.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace +{ + std::string readString(const std::string & file_path) + { + std::ifstream fin(file_path); + std::stringstream buffer; + buffer << fin.rdbuf(); + return buffer.str(); + } +} + +MockEnvironment::MockEnvironment(const std::string & path, size_t max_threads) + : session_context(MockGlobalContext::instance().createSessionContext()) + , actual_folder(path) + , mock_folder(std::filesystem::path{"/tmp"} / ("advisor_tool_" + toString(UUIDHelpers::generateV4()))) +{ + session_context->setPath(mock_folder.string() + '/'); + session_context->setMetastorePath((mock_folder / METASTORE).string() + '/'); + + SettingsChanges setting_changes; + setting_changes.emplace_back("max_threads", max_threads); + setting_changes.emplace_back("enable_memory_catalog", true); + session_context->applySettingsChanges(setting_changes); + std::filesystem::remove_all(mock_folder); + std::filesystem::create_directories(mock_folder); + std::filesystem::create_directories(mock_folder / METASTORE); + std::filesystem::create_directories(mock_folder / METADATA); + std::filesystem::create_directories(mock_folder / DATA); + + registerFunctions(); + registerFormats(); + registerStorages(); + registerAggregateFunctions(); + registerHints(); + registerDisks(); + Statistics::CacheManager::initialize(session_context); + + // make system database + DatabasePtr system_database = DatabaseCatalog::instance().tryGetDatabase(DatabaseCatalog::SYSTEM_DATABASE, session_context); + if (!system_database) + { + system_database = std::make_shared(DatabaseCatalog::SYSTEM_DATABASE, session_context); + DatabaseCatalog::instance().attachDatabase(DatabaseCatalog::SYSTEM_DATABASE, system_database); + attachSystemTablesLocal(*system_database); + } +} + +MockEnvironment::~MockEnvironment() +{ + for (const auto & [name, database] : DatabaseCatalog::instance().getDatabases(session_context)) + { + for (auto it = database->getTablesIterator(session_context); it->isValid(); it->next()) + { + database->dropTable(session_context, it->name(), /*no_delay=*/true); + } + database->drop(session_context); + } + + std::filesystem::remove_all(mock_folder); +} + +std::vector MockEnvironment::listDatabases() +{ + std::vector res; + auto meta_path = actual_folder / METADATA; + if (!std::filesystem::exists(meta_path)) + throw Exception("cannot find metadata", ErrorCodes::CANNOT_OPEN_FILE); + for (const auto & file : std::filesystem::directory_iterator{meta_path}) + { + const std::filesystem::path & fullname = file.path(); + if (fullname.extension() == ".sql") + res.emplace_back(fullname.stem().string()); + } + return res; +} + +std::vector MockEnvironment::listTables(const std::string & database) +{ + std::vector res; + auto meta_path = actual_folder / METADATA / database; + if (!std::filesystem::exists(meta_path)) + throw Exception("cannot find metadata", ErrorCodes::CANNOT_OPEN_FILE); + for (const auto & file : std::filesystem::directory_iterator{meta_path}) + { + const std::filesystem::path & fullname = file.path(); + if (fullname.extension() == ".sql") + res.emplace_back(fullname.stem().string()); + } + return res; +} + +bool MockEnvironment::containsDatabase(const std::string & database) +{ + return std::filesystem::exists(actual_folder / METADATA / (database + ".sql")); +} + +bool MockEnvironment::containsTable(const std::string & database, const std::string & table) +{ + return std::filesystem::exists(actual_folder / METADATA / database / (table + ".sql")); +} + +std::string MockEnvironment::getCreateDatabaseSql(const std::string & database) +{ + if (!containsDatabase(database)) + throw Exception("cannot find database " + database, ErrorCodes::CANNOT_OPEN_FILE); + return readString(actual_folder / METADATA / (database + ".sql")); +} + +std::string MockEnvironment::getCreateTableSql(const std::string & database, const std::string & table) +{ + if (!containsTable(database, table)) + throw Exception("cannot find table " + database + "." + table, ErrorCodes::CANNOT_OPEN_FILE); + return readString(actual_folder / METADATA / database / (table + ".sql")); +} + +ColumnsDescription MockEnvironment::getColumnsDescription(const std::string & database, const std::string & table) +{ + std::string create_table = getCreateTableSql(database, table); + ContextMutablePtr context = createQueryContext(); + auto ast = parse(create_table, context)->as(); + return InterpreterCreateQuery::getColumnsDescription(*ast.columns_list->columns, context, ast.attach, false); +} + +ContextMutablePtr MockEnvironment::createQueryContext() +{ + ContextMutablePtr query_context = Context::createCopy(session_context); + query_context->createPlanNodeIdAllocator(); + query_context->createSymbolAllocator(); + query_context->makeQueryContext(); + return query_context; +} + +ASTPtr MockEnvironment::parse(std::string_view sql, ContextPtr query_context) +{ + const char * begin = sql.data(); + const char * end = begin + sql.size(); + ParserQuery parser(end, ParserSettings::valueOf(query_context->getSettingsRef())); + return parseQuery( + parser, begin, end, "", + query_context->getSettingsRef().max_query_size, + query_context->getSettingsRef().max_parser_depth); +} + +QueryPlanPtr MockEnvironment::plan(std::string_view sql, ContextMutablePtr query_context) +{ + ASTPtr ast = parse(sql, query_context); + ast = QueryRewriter().rewrite(ast, query_context); + AnalysisPtr analysis = QueryAnalyzer::analyze(ast, query_context); + QueryPlanPtr query_plan = QueryPlanner().plan(ast, *analysis, query_context); + return query_plan; +} + +void MockEnvironment::execute(const std::string & sql, ContextMutablePtr query_context) +{ + executeQuery(sql, query_context, /*internal=*/true); +} + +void MockEnvironment::createMockDatabase(const std::string & database) +{ + if (DatabaseCatalog::instance().isDatabaseExist(database, session_context)) + return; + ContextMutablePtr query_context = createQueryContext(); + std::string sql = getCreateDatabaseSql(database); + // the sql is "attach _ ..." in metadata, we revert it + auto ast = dynamic_pointer_cast(parse(sql, query_context)); + if (!ast) + throw Exception("failed to create database " + database + ", invalid sql: " + sql, ErrorCodes::BAD_ARGUMENTS); + ast->attach = false; + ast->database = database; + ast->uuid = UUIDHelpers::Nil; + // there are some problems with destructing an Atomic database, so we force to memory + if (ast->storage && ast->storage->engine) + ast->storage->engine->name = "Memory"; + ast->cluster = ""; + execute(serializeAST(*ast), query_context); +} + +void MockEnvironment::createMockTable(const std::string & database, const std::string & table) +{ + createMockDatabase(database); + if (DatabaseCatalog::instance().getDatabase(database, session_context)->isTableExist(table, session_context)) + return; + ContextMutablePtr query_context = createQueryContext(); + SettingsChanges setting_changes; + setting_changes.emplace_back("enable_constraint_check", false); + setting_changes.emplace_back("allow_nullable_key", true); + query_context->applySettingsChanges(setting_changes); + + std::string sql = getCreateTableSql(database, table); + // the sql is "attach _ ..." in metadata, we revert it + auto ast = dynamic_pointer_cast(parse(sql, query_context)); + if (!ast) + throw Exception("failed to create table " + database + "." + table + ", invalid sql: " + sql, ErrorCodes::BAD_ARGUMENTS); + ast->attach = false; + ast->database = database; + ast->table = table; + ast->uuid = UUIDHelpers::Nil; + ast->cluster = ""; + + if (ast->storage && ast->storage->engine) + { + auto engine_name = ast->storage->engine->name; + if (engine_name == "Distributed") + ast->storage->engine->arguments->children[0] = std::make_shared(MockGlobalContext::ADVISOR_SHARD); + else if (engine_name.starts_with("Ha")) + { + // HaUniqueMergeTree, HaMergeTree require zookeeper + engine_name = engine_name.substr(2, engine_name.length()); + ASTPtr mock_engine = makeASTFunction(engine_name); + ast->storage->set(ast->storage->engine, mock_engine); + } + + if (engine_name == "MergeTree") + { + ASTSetQuery * settings = ast->storage->settings; + if (!settings) + ast->storage->set(settings, std::make_shared()); + settings->is_standalone = false; + settings->changes.emplace_back("enable_metastore", false); + } + + if (engine_name == "UniqueMergeTree") + { + ASTSetQuery * settings = ast->storage->settings; + if (!settings) + ast->storage->set(settings, std::make_shared()); + settings->is_standalone = false; + settings->changes.emplace_back("part_writer_flag", true); + settings->changes.emplace_back("enable_metastore", false); + } + } + + std::string create_sql = serializeAST(*ast); + try + { + execute(std::move(create_sql), query_context); + } + catch (...) + { + LOG_ERROR(getLogger("MockEnvironment"), "Create table {} failed: {}", table, getCurrentExceptionMessage(true)); + } +} + +bool MockEnvironment::isPrimaryKey(const QualifiedColumnName & column, ContextPtr context) +{ + StoragePtr table = tryGetLocalTable(column.database, column.table, context); + + if (!table) + return false; + + auto metadata = table->getInMemoryMetadataCopy(); + std::optional primary_key = std::nullopt; + if (metadata.isPrimaryKeyDefined()) + primary_key = metadata.getPrimaryKey(); + // From CH: By default the primary key is the same as the sorting key (which is specified by the ORDER BY clause). + // Thus in most cases it is unnecessary to specify a separate PRIMARY KEY clause. + else if (auto merge_tree = dynamic_pointer_cast(table); merge_tree && metadata.isSortingKeyDefined()) + primary_key = metadata.getSortingKey(); + + if (!primary_key) + return false; + + const auto & primary_key_columns = primary_key.value().expression->getRequiredColumns(); + return std::find(primary_key_columns.begin(), primary_key_columns.end(), column.column) != primary_key_columns.end(); +} + +StoragePtr MockEnvironment::tryGetLocalTable(const std::string & database_name, const std::string & table_name, ContextPtr context) +{ + StoragePtr table; + + if (DatabasePtr database = DatabaseCatalog::instance().tryGetDatabase(database_name, context)) + table = database->tryGetTable(table_name, context); + + if (auto distributed = dynamic_pointer_cast(table)) + if (auto remote_database = DatabaseCatalog::instance().tryGetDatabase(distributed->getRemoteDatabaseName(), context)) + if (auto remote_table = remote_database->tryGetTable(distributed->getRemoteTableName(), context)) + table = remote_table; + + return table; +} + + + +} // DB diff --git a/programs/schema-advisor/MockEnvironment.h b/programs/schema-advisor/MockEnvironment.h new file mode 100644 index 00000000000..71f89614b58 --- /dev/null +++ b/programs/schema-advisor/MockEnvironment.h @@ -0,0 +1,63 @@ +#pragma once + +#include "Interpreters/StorageID.h" +#include "MockGlobalContext.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace DB +{ + +class MockEnvironment +{ +public: + explicit MockEnvironment(const std::string & path, size_t max_threads); + MockEnvironment(const MockEnvironment &other) = delete; + ~MockEnvironment(); + + // list the available databases or tables under the give path + std::vector listDatabases(); + std::vector listTables(const std::string & database); + bool containsDatabase(const std::string & database); + bool containsTable(const std::string & database, const std::string & table); + + // get the create-table/database sql + std::string getCreateDatabaseSql(const std::string & database); + std::string getCreateTableSql(const std::string & database, const std::string & table); + ColumnsDescription getColumnsDescription(const std::string & database, const std::string & table); + + // mock the query execution environment + ContextMutablePtr createQueryContext(); + ASTPtr parse(std::string_view sql, ContextPtr query_context); + QueryPlanPtr plan(std::string_view sql, ContextMutablePtr query_context); // no optimize + void execute(const std::string & sql, ContextMutablePtr query_context); // supposed to execute ddl only + + void createMockDatabase(const std::string & database); + void createMockTable(const std::string & database, const std::string & table); + + static bool isPrimaryKey(const QualifiedColumnName & column, ContextPtr context); + static StoragePtr tryGetLocalTable(const std::string & database_name, const std::string & table_name, ContextPtr context); + +private: + ContextMutablePtr session_context; + const std::filesystem::path actual_folder; + const std::filesystem::path mock_folder; + static constexpr const char * METADATA = "metadata"; + static constexpr const char * METASTORE = "metastore"; + static constexpr const char * DATA = "data"; +}; + + +} // DB diff --git a/programs/schema-advisor/MockGlobalContext.cpp b/programs/schema-advisor/MockGlobalContext.cpp new file mode 100644 index 00000000000..aadc30e5b1a --- /dev/null +++ b/programs/schema-advisor/MockGlobalContext.cpp @@ -0,0 +1,68 @@ +#include "MockGlobalContext.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +ContextMutablePtr MockGlobalContext::createSessionContext() +{ + ContextMutablePtr session_context = Context::createCopy(context); + session_context->makeSessionContext(); + return session_context; +} + +MockGlobalContext::MockGlobalContext() +{ + shared_context = Context::createShared(); + context = Context::createGlobal(shared_context.get()); + context->makeGlobalContext(); + ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(MockGlobalContext::mockConfig())); + context->setConfig(configuration); +} + +XMLDocumentPtr MockGlobalContext::mockConfig() +{ + XMLDocumentPtr document = new Poco::XML::Document(); + Poco::AutoPtr yandex = document->createElement("yandex"); + Poco::AutoPtr remote_servers = document->createElement("remote_servers"); + Poco::AutoPtr advisor_shard = document->createElement(ADVISOR_SHARD); + Poco::AutoPtr shard = document->createElement("shard"); + Poco::AutoPtr replica = document->createElement("replica"); + + Poco::AutoPtr host = document->createElement("host"); + Poco::AutoPtr host_text = document->createTextNode("localhost"); + host->appendChild(host_text); + replica->appendChild(host); + + Poco::AutoPtr port = document->createElement("port"); + Poco::AutoPtr port_text = document->createTextNode("9000"); + port->appendChild(port_text); + replica->appendChild(port); + + Poco::AutoPtr exchange_port = document->createElement("exchange_port"); + Poco::AutoPtr exchange_port_text = document->createTextNode("9300"); + exchange_port->appendChild(exchange_port_text); + replica->appendChild(exchange_port); + + Poco::AutoPtr exchange_status_port = document->createElement("exchange_status_port"); + Poco::AutoPtr exchange_status_port_text = document->createTextNode("9400"); + exchange_status_port->appendChild(exchange_status_port_text); + replica->appendChild(exchange_status_port); + + shard->appendChild(replica); + advisor_shard->appendChild(shard); + remote_servers->appendChild(advisor_shard); + yandex->appendChild(remote_servers); + document->appendChild(yandex); + + return document; +} + +} diff --git a/programs/schema-advisor/MockGlobalContext.h b/programs/schema-advisor/MockGlobalContext.h new file mode 100644 index 00000000000..e04ff33aacf --- /dev/null +++ b/programs/schema-advisor/MockGlobalContext.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ +class MockGlobalContext +{ +public: + static constexpr const char * ADVISOR_SHARD = "advisor_shard"; + + static MockGlobalContext & instance() + { + static MockGlobalContext mock_context; + return mock_context; + } + + ContextMutablePtr createSessionContext(); + +private: + explicit MockGlobalContext(); + static XMLDocumentPtr mockConfig(); + + SharedContextHolder shared_context; + ContextMutablePtr context; +}; + +} diff --git a/programs/schema-advisor/PotentialColumn.h b/programs/schema-advisor/PotentialColumn.h new file mode 100644 index 00000000000..a89ba79cc33 --- /dev/null +++ b/programs/schema-advisor/PotentialColumn.h @@ -0,0 +1,101 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +/** + * StatisticInfo: + * - sample_ndv: the ndv for a column in the particular part + * - total_rows: the total number of rows for the particular part + * - total_predicates: The total number of predicates involving the specified column, including in and equals and others + */ +struct StatisticInfo +{ + size_t sample_ndv{}; + size_t sample_row_num{}; + size_t total_predicates{}; +}; + +/** + * PredicateInfo: + * - PredicateExpressions: predicate_ast_expression - count + * - total: The total number of occurrences of a ColumnUsageType + */ +using PredicateExpressions = std::unordered_map; +struct PredicateInfo +{ + PredicateExpressions expressions; + size_t total{}; + + PredicateInfo() = default; + PredicateInfo(PredicateExpressions & expressions_, size_t total_): expressions(std::move(expressions_)), total(total_) {} +}; +using PredicateInfos = std::unordered_map; + +enum class PotentialIndexType +{ + BLOOM_FILTER, // just support bloom_filter for skip index + BITMAP_INDEX, + SEGMENT_BITMAP_INDEX, // For high cordinality + ALREADY_BITMAP_INDEX, // used in test, to find the column already has bitmap index +}; + +inline std::string toString(PotentialIndexType indexType) +{ + switch (indexType) + { + case PotentialIndexType::BLOOM_FILTER: + return "BLOOM_FILTER"; + case PotentialIndexType::BITMAP_INDEX: + return "BITMAP_INDEX"; + case PotentialIndexType::SEGMENT_BITMAP_INDEX: + return "SEGMENT_BITMAP_INDEX"; + case PotentialIndexType::ALREADY_BITMAP_INDEX: + return "ALREADY_BITMAP_INDEX"; + default: + return "Unknown"; + } +} + +struct PotentialColumnInfo +{ + PotentialIndexType index_type; + StatisticInfo statistic_info; + PredicateInfos predicate_infos; +}; +using PotentialColumns = std::unordered_map; + +using PotentialPrewhereColumns = std::unordered_map, QualifiedColumnNameHash>; + +struct IndexOverhead +{ + size_t hashs; + size_t bits_per_rows; + size_t uncompressed_index_size; +}; + +using IndexSelectors = std::vector; +struct IndexEffect +{ + IndexSelectors index_selectors; + size_t total_expressions; +}; + +struct PotentialIndex +{ + QualifiedColumnName column; + PotentialIndexType index_type; + Float32 false_positive_rate; + + IndexOverhead index_overhead; + IndexEffect index_effect; + + StatisticInfo statistic_info; + PredicateInfos predicate_infos; +}; + +} diff --git a/programs/schema-advisor/PrewhereAdvisor.cpp b/programs/schema-advisor/PrewhereAdvisor.cpp new file mode 100644 index 00000000000..468fa7ea156 --- /dev/null +++ b/programs/schema-advisor/PrewhereAdvisor.cpp @@ -0,0 +1,238 @@ +#include "PrewhereAdvisor.h" +#include "ColumnUsageExtractor.h" +#include "Columns/IColumn.h" +#include +#include +#include "Core/Field.h" +#include "IO/WriteIntText.h" +#include "QueryPlan/PlanSerDerHelper.h" +#include "SampleColumnReader.h" +#include "Statistics.h" +#include "SchemaAdvisorHelpers.h" +#include "PotentialColumn.h" + +#include +#include +#include +#include +#include + +#include "Common/Exception.h" +#include "common/types.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +PrewhereAdvisor::PrewhereAdvisor( + MockEnvironment & env_, + const po::variables_map & options_, + size_t sample_row_number_, + size_t max_threads_, + Float64 mark_filter_threshold_, + Float64 top_3_mark_filter_threshold_) + : env(env_) + , options(options_) + , sample_row_number(sample_row_number_) + , max_threads(max_threads_) + , mark_filter_threshold(mark_filter_threshold_) + , top_3_mark_filter_threshold(top_3_mark_filter_threshold_) + , sample_mark_number(static_cast(sample_row_number_) / 8192) + , column_size_threshold(128 * sample_row_number_) +{ +} + +Float64 PrewhereAdvisor::calcMarkFilterRatio(const Field & field) const +{ + const auto & array_field = DB::safeGet(field); + + size_t total_marks = 0; + for (const auto & tuple_field : array_field) + { + total_marks += DB::safeGet(DB::safeGet(tuple_field)[1]); + } + + Float64 avg_mark_occurrence_rate = static_cast(total_marks) / array_field.size(); + + return avg_mark_occurrence_rate / sample_mark_number; +} + +std::pair PrewhereAdvisor::calcMarkFilterRatio(const ColumnPtr & column) const +{ + const auto * array_column = typeid_cast(column.get()); + const auto * tuple_column = typeid_cast((array_column->getDataPtr()).get()); + const auto * mark_count_column = typeid_cast(tuple_column->getColumns()[1].get()); + + size_t total_marks = 0; + std::priority_queue, std::greater> top_3_mark_pq; + for (size_t i = 0; i < mark_count_column->size(); i++) + { + auto current_mark = mark_count_column->get64(i); + total_marks += current_mark; + + if (top_3_mark_pq.size() < 3) + { + top_3_mark_pq.push(current_mark); + } + else if (current_mark > top_3_mark_pq.top()) + { + top_3_mark_pq.pop(); + top_3_mark_pq.push(current_mark); + } + } + size_t queue_size = top_3_mark_pq.size(); + + size_t top_3_mark_sum = 0; + while(!top_3_mark_pq.empty()) + { + top_3_mark_sum += top_3_mark_pq.top(); + top_3_mark_pq.pop(); + } + + Float64 avg_mark_occurrence_rate = static_cast(total_marks) / mark_count_column->size(); + Float64 top_3_mark_occurrence_rate = static_cast(top_3_mark_sum) / queue_size; + + return std::make_pair(avg_mark_occurrence_rate / sample_mark_number, top_3_mark_occurrence_rate / sample_mark_number); +} + +void PrewhereAdvisor::execute() +{ + auto context = createContext(options, env); + auto queries = loadQueries(options); + + LOG_DEBUG(getLogger("PrewhereAdvisor"), "++++++++++ begin to executor prewhere advisor ++++++++++"); + + ColumnUsageExtractor extractor(context, max_threads); + auto column_usages = extractor.extractColumnUsages(queries); + auto prewhere_usages = extractor.extractUsageForPrewhere(column_usages); + + LOG_DEBUG(getLogger("PrewhereAdvisor"), "Extracted {} prewhere_usages usages for prewhere", prewhere_usages.size()); + + CountByGranularity count_by_granularity; + + for (const auto & prewhere_usage : prewhere_usages) + { + auto column_info = prewhere_usage.first; + + auto storage = MockEnvironment::tryGetLocalTable(column_info.database, column_info.table, context); + if (!storage) + throw Exception(column_info.database + "(" + column_info.table + "): can not find local table.", ErrorCodes::NOT_FOUND_EXPECTED_DATA_PART); + + auto metadata = storage->getInMemoryMetadataCopy(); + auto column_and_type = metadata.getColumns().tryGetColumn(GetColumnsOptions::Kind::AllPhysical, column_info.column); + if (!column_and_type) + continue; + + auto column_type = column_and_type->type; + + if (isArray(column_type)) + continue; + + std::vector data_path_list; + + boost::split(data_path_list, options["data-path-list"].as(), boost::is_any_of(" ,")); + for (auto & path : data_path_list) + { + if (!endsWith(path, "/")) + path = path.append("/"); + } + + std::string absolute_part_path; + try + { + absolute_part_path = selectPartPath(options, data_path_list, storage->getStorageID().getDatabaseName(), storage->getStorageID().getTableName(), sample_row_number); + } + catch (Exception & e) + { + if (e.code() == ErrorCodes::NOT_FOUND_EXPECTED_DATA_PART) + { + LOG_DEBUG( + getLogger("PrewhereAdvisor"), + "Can't find suitable part for table " + column_info.database + "." + column_info.table + + ", maybe because of the total part rows < " + std::to_string(sample_row_number)); + continue; + } + else + throw e; + } + + SampleColumnReader reader(absolute_part_path + "/", 0, sample_row_number); + ColumnPtr column; + try + { + column = reader.readColumn({prewhere_usage.first.column, column_type}); + } + catch (...) + { + // Just skip the column if it can't be read + LOG_DEBUG( + getLogger("PrewhereAdvisor"), + "Can't read column file " + prewhere_usage.first.column + " from table " + column_info.database + "." + column_info.table + + ", error message: " + + getCurrentExceptionMessage(true)); + continue; + } + + std::pair mark_filter_pair; + try + { + mark_filter_pair = calcMarkFilterRatio(count_by_granularity.executeOnColumn(column, column_type)); + } + catch (Exception & e) + { + if (e.code() == ErrorCodes::BAD_ARGUMENTS) + { + LOG_DEBUG( + getLogger("PrewhereAdvisor"), "Error while calculate mark filter ratio, error message: " + e.message()); + continue; + } + else + throw e; + } + + LOG_DEBUG(getLogger("PrewhereAdvisor"), "Column {} mark filter ratio is {}, top-3 mark filter ratio is {}, column size is {} MB", column_info.column, mark_filter_pair.first, mark_filter_pair.second, column->byteSize()/1000000); + + if (((mark_filter_pair.first <= mark_filter_threshold && mark_filter_pair.second <= top_3_mark_filter_threshold) || (mark_filter_pair.first < 0.1 && mark_filter_pair.second < 0.76)) && column->byteSize() < column_size_threshold) + potential_columns.insert({std::move(column_info), mark_filter_pair}); + } +} + +/// TODO: seperate two indices +void PrewhereAdvisor::serializeJson(WriteBuffer & buf, bool /* verbose */) +{ + bool first = true; + writeString("\"prewhere\":[", buf); + for (auto & [column_info, mark_filter_ratio] : potential_columns) + { + if (first) + first = false; + else + writeString(",", buf); + + writeString(R"({"db":")", buf); + writeString(column_info.database, buf); + writeString(R"(","table":")", buf); + writeString(column_info.table, buf); + writeString(R"(","column_name":")", buf); + writeString(column_info.column, buf); + + writeString(R"(","mark_filter_ratio":")", buf); + writeString(toString(mark_filter_ratio.first), buf); + writeString(R"(","top_3_mark_filter_ratio":")", buf); + writeString(toString(mark_filter_ratio.second), buf); + + writeString("\"}", buf); + } + writeString("]", buf); +} + +} diff --git a/programs/schema-advisor/PrewhereAdvisor.h b/programs/schema-advisor/PrewhereAdvisor.h new file mode 100644 index 00000000000..48e3d2cf67e --- /dev/null +++ b/programs/schema-advisor/PrewhereAdvisor.h @@ -0,0 +1,48 @@ +#pragma once + +#include "MockEnvironment.h" +#include "PotentialColumn.h" + +#include + +#include +#include + +namespace DB +{ + +namespace po = boost::program_options; + +class PrewhereAdvisor +{ +private: + MockEnvironment & env; + po::variables_map options; + const size_t sample_row_number; + const size_t max_threads; + const Float64 mark_filter_threshold; + const Float64 top_3_mark_filter_threshold; + const Float64 sample_mark_number; + // We do not pushdown the column if the field size > 128 bytes + const size_t column_size_threshold; + + PotentialPrewhereColumns potential_columns; + + Float64 calcMarkFilterRatio(const Field & field) const; + std::pair calcMarkFilterRatio(const ColumnPtr & column) const; +public: + PrewhereAdvisor( + MockEnvironment & env_, + const po::variables_map & options_, + size_t sample_row_number_, + size_t max_threads_, + Float64 mark_filter_threshold_, + Float64 top_3_mark_filter_threshold_); + + virtual ~PrewhereAdvisor() = default; + + void execute(); + void serializeJson(WriteBuffer & buf, bool verbose = false); +}; + +} diff --git a/programs/schema-advisor/SampleColumnReader.cpp b/programs/schema-advisor/SampleColumnReader.cpp new file mode 100644 index 00000000000..a5cd4eedbbb --- /dev/null +++ b/programs/schema-advisor/SampleColumnReader.cpp @@ -0,0 +1,341 @@ +#include "SampleColumnReader.h" + +#include +#include +#include +#include +#include +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ARGUMENT_OUT_OF_BOUND; + extern const int BAD_ARGUMENTS; + extern const int CANNOT_SEEK_THROUGH_FILE; + extern const int CORRUPTED_DATA; + extern const int NO_FILE_IN_DATA_PART; + extern const int UNKNOWN_PART_TYPE; +} + +SampleColumnIndexGranularityInfo::SampleColumnIndexGranularityInfo(const String & path_to_part) +{ + auto mrk_ext = getMarksExtensionFromFilesystem(path_to_part); + if (*mrk_ext == getNonAdaptiveMrkExtension()) + { + is_adaptive = false; + part_type = MergeTreeDataPartType::WIDE; + marks_file_extension = *mrk_ext; + } + else if (*mrk_ext == getAdaptiveMrkExtension(MergeTreeDataPartType::WIDE)) + { + is_adaptive = true; + part_type = MergeTreeDataPartType::WIDE; + marks_file_extension = *mrk_ext; + } + else + { + throw Exception("Can't determine part type, because of unsupported mark extension " + *mrk_ext, ErrorCodes::UNKNOWN_PART_TYPE); + } +} + +std::optional SampleColumnIndexGranularityInfo::getMarksExtensionFromFilesystem(const String & path_to_part) +{ + if (std::filesystem::exists(path_to_part)) + { + Poco::DirectoryIterator end; + for (Poco::DirectoryIterator it(path_to_part); it != end; ++it) + { + const auto & ext = std::filesystem::path(it->path()).extension(); + if (ext == getNonAdaptiveMrkExtension() + || ext == getAdaptiveMrkExtension(MergeTreeDataPartType::WIDE) + || ext == getAdaptiveMrkExtension(MergeTreeDataPartType::COMPACT)) + return ext; + } + } + return {}; +} + +std::string SampleColumnIndexGranularityInfo::getAdaptiveMrkExtension(MergeTreeDataPartType part_type_) +{ + if (part_type_ == MergeTreeDataPartType::WIDE) + return ".mrk2"; + else if (part_type_ == MergeTreeDataPartType::COMPACT) + return ".mrk3"; + else if (part_type_ == MergeTreeDataPartType::IN_MEMORY) + return ""; + else + throw Exception("Unknown part type", ErrorCodes::UNKNOWN_PART_TYPE); +} + +size_t SampleColumnIndexGranularityInfo::getMarkSizeInBytes() const +{ + if (part_type == MergeTreeDataPartType::WIDE) + return is_adaptive ? getAdaptiveMrkSizeWide() : getNonAdaptiveMrkSizeWide(); + else + throw Exception("Unsupported type: " + part_type.toString(), ErrorCodes::UNKNOWN_PART_TYPE); +} + +size_t SampleColumnIndexGranularityInfo::getMarksCount(const String & path_prefix) const +{ + std::string marks_file_path = getMarksFilePath(path_prefix); + if (!std::filesystem::exists(marks_file_path)) + throw Exception("Marks file '" + marks_file_path + "' doesn't exist", ErrorCodes::NO_FILE_IN_DATA_PART); + + size_t marks_file_size = std::filesystem::file_size(marks_file_path); + return marks_file_size / getMarkSizeInBytes(); +} + +size_t SampleColumnIndexGranularityInfo::getMarksTotalSizeInBytes(const String & path_prefix) const +{ + std::string marks_file_path = getMarksFilePath(path_prefix); + if (!std::filesystem::exists(marks_file_path)) + throw Exception("Marks file '" + marks_file_path + "' doesn't exist", ErrorCodes::NO_FILE_IN_DATA_PART); + + return std::filesystem::file_size(marks_file_path); +} + +SampleColumnMarksLoader::SampleColumnMarksLoader( + const String & path_prefix_, + const String & stream_name_, + size_t marks_count_, + const SampleColumnIndexGranularityInfo & index_granularity_info_, + off_t mark_file_offset_, + size_t mark_file_size_) + : mrk_path(index_granularity_info_.getMarksFilePath(path_prefix_)) + , stream_name(stream_name_) + , marks_count(marks_count_) + , mark_file_offset(mark_file_offset_) + , mark_file_size(mark_file_size_) + , index_granularity_info(index_granularity_info_) {} + +const MarkInCompressedFile & SampleColumnMarksLoader::getMark(size_t row_index) +{ + if (!marks) + loadMarks(); + + return (*marks)[row_index]; +} + +SampleColumnMarksLoader::MarksPtr SampleColumnMarksLoader::loadMarksImpl() +{ + /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. + MemoryTrackerBlockerInThread temporarily_disable_memory_tracker; + + size_t mark_size = index_granularity_info.getMarkSizeInBytes(); + size_t expected_file_size = mark_size * marks_count; + + if (expected_file_size != mark_file_size) + throw Exception( + "Bad size of marks file '" + mrk_path + "' for stream '" + stream_name + "': " + std::to_string(mark_file_size) + ", must be: " + std::to_string(expected_file_size), + ErrorCodes::CORRUPTED_DATA); + + auto res = std::make_shared(marks_count); + + if (!index_granularity_info.is_adaptive) + { + /// Read directly to marks. + auto buffer = std::make_unique(mrk_path); + if (buffer->seek(mark_file_offset, SEEK_SET) != mark_file_offset) + throw Exception("Cannot seek to mark file " + mrk_path + " for stream " + stream_name, ErrorCodes::CANNOT_SEEK_THROUGH_FILE); + + if (buffer->eof() || buffer->buffer().size() != mark_file_size) + throw Exception("Cannot read all marks from file " + mrk_path + ", eof: " + std::to_string(buffer->eof()) + + ", buffer size: " + std::to_string(buffer->buffer().size()) + ", file size: " + std::to_string(mark_file_size), ErrorCodes::CANNOT_READ_ALL_DATA); + + buffer->readStrict(reinterpret_cast(res->data()), mark_file_size); + } + else + { + auto buffer = std::make_unique(mrk_path); + if (buffer->seek(mark_file_offset, SEEK_SET) != mark_file_offset) + throw Exception("Cannot seek to mark file " + mrk_path + " for stream " + stream_name, ErrorCodes::CANNOT_SEEK_THROUGH_FILE); + + size_t i = 0; + off_t limit_offset_in_file = mark_file_offset + mark_file_size; + while (buffer->getPosition() < limit_offset_in_file) + { + res->read(*buffer, i, 1); + buffer->seek(sizeof(size_t), SEEK_CUR); + ++i; + } + + if (i * mark_size != mark_file_size) + throw Exception("Cannot read all marks from file " + mrk_path, ErrorCodes::CANNOT_READ_ALL_DATA); + } + res->protect(); + return res; +} + +void SampleColumnMarksLoader::loadMarks() +{ + String mrk_name = index_granularity_info.getMarksFilePath(stream_name); + marks = loadMarksImpl(); + + if (!marks) + throw Exception("Failed to load marks: " + mrk_name + " from path:" + mrk_path, ErrorCodes::LOGICAL_ERROR); +} + +SampleColumnReaderStream::SampleColumnReaderStream( + const String & path_prefix_, const String & stream_name_, const String & data_file_extension_, + const SampleColumnIndexGranularityInfo * index_granularity_info_, + size_t max_rows_to_read_) + : path_prefix(path_prefix_) + , max_rows_to_read(max_rows_to_read_) + , marks_loader( + path_prefix_ + , stream_name_ + , index_granularity_info_->getMarksCount(path_prefix_) + , *index_granularity_info_ + , mark_file_offset + , index_granularity_info_->getMarksTotalSizeInBytes(path_prefix_)) +{ + std::string data_file_path = path_prefix_ + data_file_extension_; + /// Initialize the objects that shall be used to perform read operations. + auto buffer = std::make_unique( + std::make_unique(data_file_path), + /* allow_different_codecs = */true, + data_file_offset, + std::filesystem::file_size(data_file_path), + /* is_limit = */true); + + /* if (!settings.checksum_on_read) */ + buffer->disableChecksumming(); + + non_cached_buffer = std::move(buffer); + data_buffer = non_cached_buffer.get(); +} + +void SampleColumnReaderStream::seekToMark(size_t index) +{ + MarkInCompressedFile mark = marks_loader.getMark(index); + + try + { + non_cached_buffer->seek(mark.offset_in_compressed_file + data_file_offset, mark.offset_in_decompressed_block); + } + catch (Exception & e) + { + /// Better diagnostics. + if (e.code() == ErrorCodes::ARGUMENT_OUT_OF_BOUND) + e.addMessage("(while seeking to mark " + toString(index) + + " of column " + path_prefix + "; offsets are: " + + toString(mark.offset_in_compressed_file + data_file_offset) + " " + + toString(mark.offset_in_decompressed_block) + ")"); + + throw; + } +} + +void SampleColumnReaderStream::seekToStart() +{ + try + { + non_cached_buffer->seek(data_file_offset, 0); +#ifdef ENABLE_QPL_COMPRESSION + if (non_cached_async_buffer) + non_cached_async_buffer->seek(data_file_offset, 0); +#endif + } + catch (Exception & e) + { + /// Better diagnostics. + if (e.code() == ErrorCodes::ARGUMENT_OUT_OF_BOUND) + e.addMessage("(while seeking to start of column " + path_prefix + ")"); + + throw; + } +} + +SampleColumnReader::SampleColumnReader( + std::string path_to_part_, size_t from_mark_, size_t max_rows_to_read_) + : path_to_part(std::move(path_to_part_)) + , from_mark(from_mark_) + , max_rows_to_read(max_rows_to_read_) {} + +ReadBuffer * SampleColumnReader::getStream( + [[maybe_unused]] bool stream_for_prefix, + const ISerialization::SubstreamPath & substream_path, + const NameAndTypePair & name_and_type, + size_t from_mark_) +{ + String stream_name = ISerialization::getFileNameForStream(name_and_type, substream_path); + + auto it = streams.find(stream_name); + if (it == streams.end()) + return nullptr; + + SampleColumnReaderStream & stream = *it->second; + + if (stream_for_prefix) + stream.seekToStart(); + else + stream.seekToMark(from_mark_); + + return stream.data_buffer; +} + + +ColumnPtr SampleColumnReader::readColumn(const NameAndTypePair & name_and_type) +{ + SampleColumnIndexGranularityInfo index_granularity_info(path_to_part); + + ISerialization::StreamCallback callback = [&](const ISerialization::SubstreamPath & substream_path) { + String stream_name = ISerialization::getFileNameForStream(name_and_type, substream_path); + + if (streams.count(stream_name)) + return; +/* + auto check_validity + = [&](String & stream_name_) -> bool { return data_part->getChecksums()->files.count(stream_name_ + DATA_FILE_EXTENSION); }; + + // If data file is missing then we will not try to open it. + // It is necessary since it allows to add new column to structure of the table without creating new files for old parts. + // + if ((!name_and_type.type->isKVMap() && !check_validity(stream_name)) + || (name_and_type.type->isKVMap() && !tryConvertToValidKVStreamName(stream_name, check_validity))) + return; +*/ + std::string path_prefix = path_to_part + stream_name; + streams.emplace( + stream_name, + std::make_unique( + path_prefix, + stream_name, + DATA_FILE_EXTENSION, + &index_granularity_info, + max_rows_to_read + )); + }; + + auto serialization = name_and_type.type->getDefaultSerialization(); + serialization->enumerateStreams(callback); + + ColumnPtr column = name_and_type.type->createColumn(); + //double & avg_value_size_hint = avg_value_size_hints[name_and_type.name]; + ISerialization::DeserializeBinaryBulkSettings deserialize_settings; + // deserialize_settings.avg_value_size_hint = avg_value_size_hint; + + const auto & name = name_and_type.name; + + if (deserialize_binary_bulk_state_map.count(name) == 0) + { + deserialize_settings.getter = [&](const ISerialization::SubstreamPath & substream_path) + { + return getStream(true, substream_path, name_and_type, from_mark); + }; + serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, deserialize_binary_bulk_state_map[name]); + } + + deserialize_settings.getter = [&](const ISerialization::SubstreamPath & substream_path) + { + return getStream(false, substream_path, name_and_type, from_mark); + }; + deserialize_settings.continuous_reading = 0; + auto & deserialize_state = deserialize_binary_bulk_state_map[name]; + + serialization->deserializeBinaryBulkWithMultipleStreams(column, max_rows_to_read, deserialize_settings, deserialize_state, nullptr); + return column; +} + +} diff --git a/programs/schema-advisor/SampleColumnReader.h b/programs/schema-advisor/SampleColumnReader.h new file mode 100644 index 00000000000..ecb6126f830 --- /dev/null +++ b/programs/schema-advisor/SampleColumnReader.h @@ -0,0 +1,133 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace DB +{ + +struct SampleColumnIndexGranularityInfo +{ +public: + /// Marks file extension '.mrk' or '.mrk2' + String marks_file_extension; + + /// Is stride in rows between marks non fixed? + bool is_adaptive = false; + + SampleColumnIndexGranularityInfo(const String & path_to_part); + + String getMarksFilePath(const String & path_prefix) const + { + return path_prefix + marks_file_extension; + } + + size_t getMarkSizeInBytes() const; + size_t getMarksCount(const String & path_prefix) const; + size_t getMarksTotalSizeInBytes(const String & path_prefix) const; + +private: + MergeTreeDataPartType part_type; + std::optional getMarksExtensionFromFilesystem(const String & path_to_part); + std::string getAdaptiveMrkExtension(MergeTreeDataPartType part_type); +}; + +class SampleColumnMarksLoader +{ +public: + using MarksPtr = std::shared_ptr; + + SampleColumnMarksLoader( + const String & path_prefix_, + const String & stream_name_, + size_t marks_count_, + const SampleColumnIndexGranularityInfo & index_granularity_info_, + off_t mark_file_offset_, + size_t mark_file_size_); + + const MarkInCompressedFile & getMark(size_t row_index); + + bool initialized() const { return marks != nullptr; } + +private: + String mrk_path; + String stream_name; // for compacted map + size_t marks_count; + + off_t mark_file_offset; + size_t mark_file_size; + + SampleColumnIndexGranularityInfo index_granularity_info; + + MarksPtr marks; + + void loadMarks(); + MarksPtr loadMarksImpl(); +}; + +class SampleColumnReaderStream +{ +public: + SampleColumnReaderStream( + const String & path_prefix_, const String & stream_name_, const String & data_file_extension_, + const SampleColumnIndexGranularityInfo * index_granularity_info_, + size_t max_rows_to_read_); + + virtual ~SampleColumnReaderStream() = default; + + void seekToMark(size_t index); + + void seekToStart(); + + ReadBuffer * data_buffer; + +private: + std::string path_prefix; + off_t data_file_offset = 0; + off_t mark_file_offset = 0; + [[maybe_unused]] size_t max_rows_to_read; + + std::unique_ptr non_cached_buffer; + + SampleColumnMarksLoader marks_loader; +}; + +using SampleFileStreams = std::map>; +using DeserializeBinaryBulkStateMap = std::map; + +class SampleColumnReader +{ +private: + const std::string path_to_part; + size_t from_mark; + size_t max_rows_to_read; + + SampleFileStreams streams; + + /// Stores states for IDataType::deserializeBinaryBulk + DeserializeBinaryBulkStateMap deserialize_binary_bulk_state_map; + +public: + SampleColumnReader( + std::string path_to_part_, + size_t from_mark_, + size_t max_rows_to_read_); + + virtual ~SampleColumnReader() = default; + + ReadBuffer * getStream( + bool stream_for_prefix, + const ISerialization::SubstreamPath & substream_path, + const NameAndTypePair & name_and_type, + size_t from_mark); + + ColumnPtr readColumn(const NameAndTypePair & name_and_type); +}; + +} diff --git a/programs/schema-advisor/SchemaAdvisor.cpp b/programs/schema-advisor/SchemaAdvisor.cpp new file mode 100644 index 00000000000..2dda2ab4e5b --- /dev/null +++ b/programs/schema-advisor/SchemaAdvisor.cpp @@ -0,0 +1,243 @@ +#include "Advisor/Advisor.h" +#include "CodecAdvisor.h" +#include "ColumnUsageExtractor.h" +#include "IndexAdvisor.h" +#include "MockEnvironment.h" +#include "SchemaAdvisorHelpers.h" +#include "TypeAdvisor.h" +#include "PrewhereAdvisor.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + +static constexpr size_t DEFAULT_SAMPLE_ROW_NUMBER = 1000000; +static constexpr size_t DEFAULT_MAX_THREADS = 8; +static constexpr Float64 MARK_FILTER_THRESHOLD = 0.35; +static constexpr Float64 TOP_3_MARK_FILTER_THRESHOLD = 0.65; + +} + +int mainEntryClickHouseSchemaAdvisor(int argc, char ** argv) +{ + using namespace DB; + namespace po = boost::program_options; + + po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); + desc.add_options()("help,h", "produce help message") + /// mandatory + ("db", po::value()->value_name("DATABASE"), "db name")( + "table", po::value()->value_name("TABLE"), "table name")( + "mode", po::value()->value_name("MODE"), + "mode: type, codec, type-codec, skip-index, projection, materialized-view, projection, order-by-key, sharding-key")( + "path", po::value()->value_name("PATH"), "main path")( + "meta-path", po::value()->value_name("META PATH"), "meta path")( + "data-path-list", po::value()->value_name("DATA PATH LIST"), "data path list, format: path1,path2")( + "settings", po::value()->default_value(""), "set settings, format: key1=value,key2=value2")( + "log-level", po::value()->default_value(""), + "log level: trace, debug, information, notice, warning, error. Disable if emtpy.") + /// optional + ("part", po::value()->value_name("PART"), "sample part name")( + "max-threads", po::value()->default_value(DEFAULT_MAX_THREADS), "max threads for schema advisor")( + "sample-number", po::value()->default_value(DEFAULT_SAMPLE_ROW_NUMBER), "sample row number")( + "verbose", "print column compression gain ratio") + /// codec + ("fsst", "codec mode: use FSST instead of LZ4")("zstd", "codec mode: use ZSTD instead of LZ4")( + "level", po::value(), "codec mode: compression level for codecs specified via flags")( + "codec", po::value>()->multitoken(), "codec mode: use codecs combination instead of LZ4")( + "hc", "codec mode: use LZ4HC instead of LZ4")("none", "codec mode: use no compression instead of LZ4")( + "block-size,b", + po::value()->default_value(DBMS_DEFAULT_BUFFER_SIZE), + "codec mode: compress in blocks of specified size") + /// skip-index + ("query-file", po::value()->value_name("QUERIES"), "absolute path to the query file seperated by newline")( + "query-file-delimiter", po::value()->value_name("DELIMITER"), "query file delimiter, default is new line.") + /// tos + ("tos-ak", po::value()->value_name("TOS AK"), "tos access key")( + "vetos-endpoint", po::value()->value_name("VETOS ENDPOINT"), "ve tos endpoint")( + "vetos-region", po::value()->value_name("VETOS REGION"), "ve tos region")( + "vetos-ak", po::value()->value_name("VETOS AK"), "ve tos access key")( + "vetos-sk", po::value()->value_name("VETOS SK"), "ve tos secret key") + /// prewhere + ("mark_filter_threshold", po::value()->default_value(MARK_FILTER_THRESHOLD), "threshold for mark filter ratio") ("top_3_mark_filter_threshold", po::value()->default_value(TOP_3_MARK_FILTER_THRESHOLD), "threshold for mark filter ratio") ("low-cardinality", "recommend low-cardinality only in type advisor") ( + "scanned_count_threshold_for_lc", po::value()->default_value(0.035), "recommend low-cardinality only scanned count > scan_count_threshold_for_lc") ( + "cardinality_ratio_threshold_for_lc", po::value()->default_value(0.05), "recommend low-cardinality only cardinality < sample_row_number * cardinality_ratio_threshold_for_lc"); + + WriteBufferFromOwnString str_buf; + std::unique_ptr stdout_buf = std::make_unique(STDOUT_FILENO); + bool verbose = false; + + try + { + po::variables_map options; + po::store(po::command_line_parser(argc, argv).options(desc).run(), options); + + if (options.count("help")) + { + std::cout << "Usage: " << argv[0] << " [options] < INPUT > OUTPUT" << std::endl; + std::cout << "Usage: " << argv[0] << " [options] INPUT OUTPUT" << std::endl; + std::cout << desc << std::endl; + return 0; + } + + if (!options.count("db") || !options.count("mode")) + throw Exception("Missing option, 'db' or 'mode' is missing", ErrorCodes::BAD_ARGUMENTS); + + // if (options.count("path") == options.count("meta-path") || options.count("meta-path") != options.count("data-path-list")) + // throw Exception("Missing option, either single 'path' argument or both meta path and data path list arguments are allowed", ErrorCodes::BAD_ARGUMENTS); + + std::string db_name = options["db"].as(); + std::string advisor_mode = options["mode"].as(); + + std::string meta_path; + std::vector data_path_list; + // if (options.count("path")) + // { + // std::string path = options["path"].as(); + // if (!endsWith(path, "/")) + // path.append("/"); + // meta_path = path; + // data_path_list.emplace_back(path); + // } + // else + // { + meta_path = options["meta-path"].as(); + if (!endsWith(meta_path, "/")) + meta_path.append("/"); + boost::split(data_path_list, options["data-path-list"].as(), boost::is_any_of(" ,")); + for(auto & path : data_path_list) + { + if (!endsWith(path, "/")) + path = path.append("/"); + } + // } + + size_t sample_row_number = options["sample-number"].as(); + size_t max_threads = options["max-threads"].as(); + Float64 mark_filter_threshold = options["mark_filter_threshold"].as(); + Float64 top_3_mark_filter_threshold = options["top_3_mark_filter_threshold"].as(); + verbose = options.count("verbose"); + + + if (auto log_level = options["log-level"].as(); !log_level.empty()) + setupLogging(log_level); + + // prepare mock env + MockEnvironment env(meta_path, max_threads); + + if (advisor_mode == "codec") + { + if (!options.count("table")) + throw Exception("Missing option, 'table' is missing", ErrorCodes::BAD_ARGUMENTS); + + std::string table_name = options["table"].as(); + std::string absolute_part_path = selectPartPath(options, data_path_list, db_name, table_name, sample_row_number); + serializeJsonPrefix(str_buf, db_name, table_name, absolute_part_path, verbose); + ColumnsDescription columns = env.getColumnsDescription(db_name, table_name); + + CodecAdvisor codec_advisor(options, columns, absolute_part_path, sample_row_number, max_threads); + codec_advisor.execute(); + codec_advisor.serializeJson(str_buf, verbose); + serializeJsonSuffix(str_buf); + } + else if (advisor_mode == "type") + { + if (!options.count("table")) + throw Exception("Missing option, 'table' is missing", ErrorCodes::BAD_ARGUMENTS); + + auto lc_only = options.count("low-cardinality"); + auto scanned_count_threshold_for_lc = options["scanned_count_threshold_for_lc"].as(); + auto cardinality_ratio_threshold_for_lc = options["cardinality_ratio_threshold_for_lc"].as(); + + std::string table_name = options["table"].as(); + std::string absolute_part_path = selectPartPath(options, data_path_list, db_name, table_name, sample_row_number); + serializeJsonPrefix(str_buf, db_name, table_name, absolute_part_path, verbose); + ColumnsDescription columns = env.getColumnsDescription(db_name, table_name); + + TypeAdvisor type_advisor(env, options, columns, absolute_part_path, sample_row_number, max_threads, lc_only, scanned_count_threshold_for_lc, cardinality_ratio_threshold_for_lc); + type_advisor.execute(); + type_advisor.serializeJson(str_buf, verbose); + serializeJsonSuffix(str_buf); + } + else if (advisor_mode == "skip-index") // currently extracts all usages for the database + { + serializeJsonPrefixWithDB(str_buf, db_name); + IndexAdvisor index_advisor(env, options, sample_row_number, max_threads); + index_advisor.execute(); + index_advisor.serializeJson(str_buf, verbose); + serializeJsonSuffix(str_buf); + } + else if (advisor_mode == "prewhere") // currently extracts all usages for the database + { + serializeJsonPrefixWithDB(str_buf, db_name); + PrewhereAdvisor prewhere_advisor(env, options, sample_row_number, max_threads, mark_filter_threshold, top_3_mark_filter_threshold); + prewhere_advisor.execute(); + prewhere_advisor.serializeJson(str_buf, verbose); + serializeJsonSuffix(str_buf); + } + else if (advisor_mode == "materialized-view") + { + Advisor advisor{ASTAdviseQuery::AdvisorType::MATERIALIZED_VIEW}; + WorkloadAdvises advises = advisor.analyze(loadQueries(options), createContext(options, env)); + serializeJson("materialized-view", "ddl", db_name, advises, str_buf, verbose); + } + else if (advisor_mode == "projection") + { + Advisor advisor{ASTAdviseQuery::AdvisorType::PROJECTION}; + WorkloadAdvises advises = advisor.analyze(loadQueries(options), createContext(options, env)); + serializeJson("projection", "ddl", db_name, advises, str_buf, verbose); + } + else if (advisor_mode == "order-by-key") + { + Advisor advisor{ASTAdviseQuery::AdvisorType::ORDER_BY}; + WorkloadAdvises advises = advisor.analyze(loadQueries(options), createContext(options, env)); + serializeJson(advisor_mode, "candidate", db_name, advises, str_buf, verbose); + } + else if (advisor_mode == "cluster-key") + { + Advisor advisor{ASTAdviseQuery::AdvisorType::CLUSTER_BY}; + WorkloadAdvises advises = advisor.analyze(loadQueries(options), createContext(options, env)); + serializeJson(advisor_mode, "candidate", db_name, advises, str_buf, verbose); + } + else if (advisor_mode == "column-usage") + { + Advisor advisor{ASTAdviseQuery::AdvisorType::COLUMN_USAGE}; + WorkloadAdvises advises = advisor.analyze(loadQueries(options), createContext(options, env)); + serializeJson(advisor_mode, "usage", db_name, advises, str_buf, verbose); + } + else + { + throw Exception("Unsupported advisor mode: " + advisor_mode, ErrorCodes::BAD_ARGUMENTS); + } + } + catch (...) + { + serializeException(*stdout_buf, getCurrentExceptionMessage(verbose)); + return getCurrentExceptionCode(); + } + writeString(str_buf.str(), *stdout_buf); + + return 0; +} diff --git a/programs/schema-advisor/SchemaAdvisorHelpers.h b/programs/schema-advisor/SchemaAdvisorHelpers.h new file mode 100644 index 00000000000..02546c075a0 --- /dev/null +++ b/programs/schema-advisor/SchemaAdvisorHelpers.h @@ -0,0 +1,418 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "MockEnvironment.h" + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NOT_FOUND_EXPECTED_DATA_PART; + extern const int NO_FILE_IN_DATA_PART; +} + +namespace po = boost::program_options; + +[[maybe_unused]] static void setupLogging(const std::string & log_level) +{ + Poco::AutoPtr channel(new Poco::ConsoleChannel); + Poco::AutoPtr formatter(new Poco::PatternFormatter); + formatter->setProperty("pattern", "%L%Y-%m-%d %H:%M:%S.%i <%p> %s: %t"); + Poco::AutoPtr formatting_channel(new Poco::FormattingChannel(formatter, channel)); + Poco::Logger::root().setChannel(formatting_channel); + Poco::Logger::root().setLevel(log_level); +} + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int NOT_FOUND_EXPECTED_DATA_PART; + extern const int NETWORK_ERROR; +} + +namespace po = boost::program_options; + +static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768; +static constexpr size_t DEFAULT_MAX_SAMPLE_CANDIDATE_NUM = 20; +static constexpr auto DEFAULT_TOS_PSM = "toutiao.tos.tosapi"; + +struct SamplingColumnFile +{ + SamplingColumnFile(std::string file_path_, std::string column_name_) + : file_path(std::move(file_path_)), column_name(std::move(column_name_)) + { + } + + std::string file_path; + std::string column_name; + size_t origin_file_size = 0; + size_t optimized_file_size = 0; +}; + +using SamplingColumnFilePtr = std::shared_ptr; +using SamplingColumnFiles = std::vector; + +// a thread-safe implementation +class MessageCollector +{ +public: + void collect(std::string && msg) + { + std::lock_guard lock(mutex); + messages.emplace_back(std::move(msg)); + } + + void logCollectedError() + { + for (const auto & msg : messages) + LOG_ERROR(getLogger("MessageCollector"), "{}", msg); + messages.clear(); + } + +private: + std::vector messages; + bthread::Mutex mutex; +}; + +static std::string readSqlFile(String source_uri, [[maybe_unused]]const po::variables_map & options) +{ + // std::string uri_prefix = source_uri.substr(0, source_uri.find_last_of('/')); + // Poco::URI uri(uri_prefix); + // const String& scheme = uri.getScheme(); + + + // if (scheme == "tos") // tos on cloud, url like "tos://bucket/key" + // { + // if (!options.count("tos-ak")) + // throw Exception("Option tos-ak is missing for tos uri", ErrorCodes::BAD_ARGUMENTS); + // std::string tos_ak = options["tos-ak"].as(); + + // Poco::URI tos_uri(source_uri); + // auto host = tos_uri.getHost(); + // auto port = tos_uri.getPort(); + // std::string tos_psm = DEFAULT_TOS_PSM; + // std::string tos_server; + + // if (host.empty() || port == 0) + // { + // auto tos_servers = ServiceDiscovery::lookup(DEFAULT_TOS_PSM, std::pair("cluster", "default")); + // if (tos_servers.empty()) + // throw Exception("Can not find tos servers with PSM: " + tos_psm, ErrorCodes::NETWORK_ERROR); + // auto generator = std::mt19937(std::random_device{}()); // mt19937 engine + // std::uniform_int_distribution distribution(0, tos_servers.size() - 1); + // tos_server = tos_servers.at(distribution(generator)); + // } + // else + // { + // tos_server = normalizeHost(host) + ":" + toString(port); + // } + + // ConnectionTimeouts timeouts( + // {DEFAULT_HTTP_READ_BUFFER_CONNECTION_TIMEOUT, 0}, + // {DEFAULT_HTTP_READ_BUFFER_TIMEOUT, 0}, + // {DEFAULT_HTTP_READ_BUFFER_TIMEOUT, 0}); + + // std::string tos_http_uri_str = fmt::format( + // "http://{}{}?timeout={}s", tos_server, tos_uri.getPath(), DBMS_DEFAULT_CONNECT_TIMEOUT_SEC); + // Poco::URI tos_http_uri = Poco::URI(tos_http_uri_str); + // HTTPSessionPtr session = makeHTTPSession(tos_http_uri, timeouts); + + // Poco::Net::HTTPRequest request{Poco::Net::HTTPRequest::HTTP_GET, tos_http_uri.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1}; + // request.set("X-Tos-Access", tos_ak); + // request.setHost(tos_http_uri.getHost()); + // request.setChunkedTransferEncoding(false); + + // session->sendRequest(request); + // Poco::Net::HTTPResponse response; + // std::istream * response_body = receiveResponse(*session, request, response, false); + // Poco::StreamCopier::copyToString(*response_body, res); + // } + // #if USE_VE_TOS + // else if (scheme == "vetos") // tos on volcano engine, url like "vetos://bucket/key" + // { + // Poco::URI vetos_uri(source_uri); + // vetos_uri.getPath(); + // if(vetos_uri.getPath().empty() || vetos_uri.getHost().empty()) + // { + // throw Exception("Invalid ve-tos path.", ErrorCodes::LOGICAL_ERROR); + // } + // const String& bucket = vetos_uri.getHost(); + // size_t size = vetos_uri.getPath().size(); + // String key = vetos_uri.getPath().substr(1, size - 1); + // if (!options.count("vetos-endpoint")) + // throw Exception("Option vetos-endpoint is missing for ve tos uri", ErrorCodes::BAD_ARGUMENTS); + // if (!options.count("vetos-region")) + // throw Exception("Option vetos-region is missing for ve tos uri", ErrorCodes::BAD_ARGUMENTS); + // if (!options.count("vetos-ak")) + // throw Exception("Option vetos-ak is missing for ve tos uri", ErrorCodes::BAD_ARGUMENTS); + // if (!options.count("vetos-sk")) + // throw Exception("Option vetos-sk is missing for ve tos uri", ErrorCodes::BAD_ARGUMENTS); + // std::string ve_tos_endpoint = options["vetos-endpoint"].as(); + // std::string ve_tos_region = options["vetos-region"].as(); + // std::string ve_tos_ak = options["vetos-ak"].as(); + // std::string ve_tos_sk = options["vetos-sk"].as(); + + // std::unique_ptr read_buf = + // std::make_unique(ve_tos_endpoint, ve_tos_region, ve_tos_ak, ve_tos_sk, bucket, key); + + // readStringUntilEOF(res, *read_buf); + // } + // #endif // USE_VE_TOS + // else // absolute file path on local file system + // { + + std::string res; + + std::ifstream fin(source_uri); + std::stringstream buffer; + buffer << fin.rdbuf(); + res = buffer.str(); + // } + + return res; +} + + +/// Select the target part according to specific rule if 'part' option is not specified +[[maybe_unused]] static std::string selectPartPath(const po::variables_map & options, const std::vector & data_path_list, const std::string & db_name, const std::string & table_name, size_t sample_row_number) +{ + for (const auto & path : data_path_list) + { + if (options.count("part")) + { + std::string part = options["part"].as(); + if (endsWith(part, "/")) + part.pop_back(); + return path + "metadata/" + escapeForFileName(db_name) + "/" + escapeForFileName(table_name) + "/" + part; + } + + std::string table_data_path = path + "data/" + escapeForFileName(db_name) + "/" + escapeForFileName(table_name) + "/"; + if (!std::filesystem::exists(table_data_path)) + continue; + + std::multimap parts_by_timestamp; + Poco::DirectoryIterator end; + for (Poco::DirectoryIterator it(table_data_path); it != end; ++it) + { + if (it->isDirectory() + && it.name() != "detached" + && it.name() != "log" + && it.name() != "catalog.db" + && it.name() != "manifest" + && !startsWith(it.name(), "tmp-fetch") + && !startsWith(it.name(), "tmp_") + && !startsWith(it.name(), "delete_tmp")) + { + size_t part_row_count; + std::string part_count_path = it->path() + "/count.txt"; + { + ReadBufferFromFile in(part_count_path, METADATA_FILE_BUFFER_SIZE); + readIntText(part_row_count, in); + assertEOF(in); + } + if (part_row_count >= sample_row_number) + { + parts_by_timestamp.emplace(it->getLastModified().epochTime(), it->path()); + if (parts_by_timestamp.size() > DEFAULT_MAX_SAMPLE_CANDIDATE_NUM) + break; + } + } + } + if (!parts_by_timestamp.empty()) + return parts_by_timestamp.begin()->second; + } + + throw Exception(db_name + "(" + table_name + "): failed to find qualified sample part.", ErrorCodes::NOT_FOUND_EXPECTED_DATA_PART); +} + +/// Generate output prefix in JSON format +[[maybe_unused]] static void serializeJsonPrefix(WriteBuffer & buf, std::string db_name, std::string table_name, std::string absolute_part_path, bool verbose) +{ + writeString(R"({"recommendation":{"db":")", buf); + writeString(db_name, buf); + writeString("\",", buf); + writeString(R"("table":")", buf); + writeString(table_name, buf); + writeString("\",", buf); + if (verbose) + { + writeString(R"("part selected":")", buf); + writeString(absolute_part_path, buf); + writeString("\",", buf); + } +} + +[[maybe_unused]] static void serializeJsonPrefixWithDB(WriteBuffer & buf, std::string db_name) +{ + writeString(R"({"recommendation":{"db":")", buf); + writeString(db_name, buf); + writeString("\",", buf); +} + +/// Generate output suffix in JSON format +[[maybe_unused]] static void serializeJsonSuffix(WriteBuffer & buf) +{ + writeString("}}", buf); +} + +/// Generate exception in JSON format +[[maybe_unused]] static void serializeException(WriteBuffer & buf, std::string error_msg) +{ + writeString(R"({"exception":")", buf); + writeString(error_msg, buf); + writeString("\"}", buf); +} + +[[maybe_unused]] static std::vector loadQueries(po::variables_map & options) +{ + std::string query_file = options["query-file"].as(); + + std::vector splits; + if (Poco::toLower(query_file).ends_with(".json")) + { + PlanReproducer reproducer{query_file, nullptr}; + for (const auto & name : reproducer.getQueries()->getNames()) + splits.emplace_back(reproducer.getQuery(name).query); + return splits; + } + + std::string query_content = readSqlFile(query_file, options); + std::string delimiter = "\n"; + if (options.count("query-file-delimiter")) + delimiter = options["query-file-delimiter"].as(); + + size_t last = 0; + size_t next; + while ((next = query_content.find(delimiter, last)) != std::string::npos) + { + auto query = query_content.substr(last, next - last); + boost::replace_all(query, "\\r", "\r"); + boost::replace_all(query, "\\n", "\n"); + boost::replace_all(query, "\\t", "\t"); + boost::replace_all(query, "\\\"", "\""); + boost::replace_all(query, "\\'", "'"); + splits.push_back(query); + last = next + 1; + } + if (splits.empty()) + throw Poco::Exception("'" + query_file + "' is empty?"); + return splits; +} + +[[maybe_unused]] static ContextMutablePtr createContext(po::variables_map & options, MockEnvironment & env) +{ + if (options["db"].empty()) + throw Exception("argument db is requried", ErrorCodes::BAD_ARGUMENTS); + + std::string db_name = options["db"].as(); + std::vector db_list; + boost::algorithm::split(db_list, db_name, boost::is_any_of(","), boost::token_compress_on); + + if (db_list.empty()) + throw Exception("argument db is requried", ErrorCodes::BAD_ARGUMENTS); + + for (const auto & db : db_list) + { + env.createMockDatabase(db); + // todo: currently we create all tables in the db + for (const auto & table : env.listTables(db)) + env.createMockTable(db, table); + } + + auto context = env.createQueryContext(); + context->setCurrentDatabase(db_list[0]); + + std::string settings = options["settings"].as(); + if (!settings.empty()) + { + ParserSetQuery parser{true}; + ASTPtr ast = parseQuery(parser, settings, 0, 0); + context->applySettingsChanges(ast->as()->changes); + } + + return context; +} + +[[maybe_unused]] static void serializeJson(const std::string & advise_type, const String & advise_name, const String & db, const WorkloadAdvises & advises, WriteBuffer & buf, bool) +{ + Poco::JSON::Array advises_array; + for (const auto & advise : advises) + { + Poco::JSON::Object advise_object; + advise_object.set("db", advise->getTable().database); + advise_object.set("table", advise->getTable().table); + if (advise->getColumnName().has_value()) { + advise_object.set("column", advise->getColumnName().value()); + } + + if (!advise->getCandidates().empty()) + { + Poco::JSON::Array candidates; + for (const auto & item : advise->getCandidates()) + { + Poco::JSON::Object candidate_object; + candidate_object.set(advise_name, item.first); + candidate_object.set("benefit", item.second); + candidates.add(candidate_object); + } + advise_object.set("candidates", candidates); + } + else + { + advise_object.set(advise_name, advise->getOptimizedValue()); + advise_object.set("benefit", advise->getBenefit()); + } + + if (!advise->getRelatedQueries().empty()) + { + Poco::JSON::Array related_queries; + for (const auto & query : advise->getRelatedQueries()) + related_queries.add(query); + advise_object.set("relatedQueries", related_queries); + } + + advises_array.add(advise_object); + } + + Poco::JSON::Object advises_object; + advises_object.set(advise_type, advises_array); + + Poco::JSON::Object recommendation_object; + recommendation_object.set("db", db); + recommendation_object.set(advise_type, advises_object); + + Poco::JSON::Object res; + res.set("recommendation", recommendation_object); + std::ostringstream oss; + Poco::JSON::Stringifier::condense(res, oss); + writeString(oss.str(), buf); +} +} diff --git a/programs/schema-advisor/Statistics.cpp b/programs/schema-advisor/Statistics.cpp new file mode 100644 index 00000000000..3581bb17d61 --- /dev/null +++ b/programs/schema-advisor/Statistics.cpp @@ -0,0 +1,453 @@ +#include "Statistics.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "Columns/IColumn.h" + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CANNOT_ALLOCATE_MEMORY; + extern const int BAD_ARGUMENTS; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +template typename Data, template typename DataForVariadic, bool is_able_to_parallelize_merge> +AggregateFunctionPtr +createAggregateFunctionUniq(const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *) +{ + assertNoParameters(name, params); + + if (argument_types.empty()) + throw Exception("Incorrect number of arguments for aggregate function " + name, + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + /// We use exact hash function if the user wants it; + /// or if the arguments are not contiguous in memory, because only exact hash function have support for this case. + /// bool use_exact_hash_function = is_exact || !isAllArgumentsContiguousInMemory(argument_types); + + if (argument_types.size() == 1) + { + const IDataType & argument_type = *argument_types[0]; + + AggregateFunctionPtr res(createWithNumericType(*argument_types[0], argument_types)); + + WhichDataType which(argument_type); + if (res) + return res; + else if (which.isDate()) + return std::make_shared>>(argument_types); + else if (which.isDate32()) + return std::make_shared>>(argument_types); + else if (which.isDateTime()) + return std::make_shared>>(argument_types); + else if (which.isStringOrFixedString()) + return std::make_shared>>(argument_types); + else if (which.isUUID()) + return std::make_shared>>(argument_types); + else if (which.isTuple()) + { + /* + if (use_exact_hash_function) + return std::make_shared>>(argument_types); + else + return std::make_shared>>(argument_types); + */ + throw Exception("Unsupported tuple data type for uniqExtract", ErrorCodes::BAD_ARGUMENTS); + } + } + + /* "Variadic" method also works as a fallback generic case for single argument. + if (use_exact_hash_function) + return std::make_shared>>(argument_types); + else + return std::make_shared>>(argument_types); + */ + throw Exception("Unsupported arguments size " + std::to_string(argument_types.size()), ErrorCodes::BAD_ARGUMENTS); +} + +DataTypes transformArguments(const DataTypes & arguments) +{ + size_t size = arguments.size(); + DataTypes res(size); + for (size_t i = 0; i < size; ++i) + res[i] = removeNullable(arguments[i]); + return res; +} + +Field UniExtract::executeOnColumn(const ColumnPtr & column, const DataTypePtr & type) +{ + String name = "uniqExact"; + DataTypes argument_types(1); + argument_types[0] = type; + Array parameters; + + AggregateFunctionPtr nested_function = + createAggregateFunctionUniq + (name, transformArguments(argument_types), parameters, nullptr); + AggregateFunctionPtr aggregate_function = type->isNullable() + ? std::make_shared>(nested_function, argument_types, parameters) + : nested_function; + + size_t total_size_of_aggregate_states = 0; /// The total size of the row from the aggregate functions. + // add info to track alignment requirement + // If there are states whose alignment are v1, ..vn, align_aggregate_states will be max(v1, ... vn) + size_t align_aggregate_states = 1; + total_size_of_aggregate_states = aggregate_function->sizeOfData(); + align_aggregate_states = std::max(align_aggregate_states, aggregate_function->alignOfData()); + + std::shared_ptr aggregates_pool = std::make_shared(); /// The pool that is currently used for allocation. + AggregateDataPtr place = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + try + { + /** An exception may occur if there is a shortage of memory. + * In order that then everything is properly destroyed, we "roll back" some of the created states. + * The code is not very convenient. + */ + aggregate_function->create(place); + } + catch (...) + { + aggregate_function->destroy(place); + throw Exception("Cannot allocate memory", ErrorCodes::CANNOT_ALLOCATE_MEMORY); + } + size_t rows = column->size(); + ColumnRawPtrs column_ptrs; + column_ptrs.emplace_back(column.get()); + const IColumn ** batch_arguments = column_ptrs.data(); + + aggregate_function->addBatchSinglePlace(rows, place, batch_arguments, nullptr); + + DataTypePtr result_type = std::make_shared(); + ColumnPtr result_column = result_type->createColumn(); + MutableColumnPtr mutable_column = result_column->assumeMutable(); + aggregate_function->insertResultInto(place, *mutable_column, nullptr); + return (*result_column)[0]; +} + +Field UniExtract::executeOnColumnArray(const ColumnPtr & column, const DataTypePtr & type) +{ + if (!isArray(type)) + return 0; + + const auto * array_type = checkAndGetDataType(type.get()); + const auto& nested_type = array_type->getNestedType(); + + String inner_func_name = "uniqExact"; + String combinator_suffix = "Array"; + + DataTypes nested_argument_types{nested_type}; + DataTypes argument_types{type}; + Array parameters; + + // For inner func uniqExact + AggregateFunctionPtr nested_function = + createAggregateFunctionUniq + (inner_func_name, transformArguments(nested_argument_types), parameters, nullptr); + AggregateFunctionPtr uniq_exact_function = type->isNullable() + ? std::make_shared>(nested_function, nested_argument_types, parameters) + : nested_function; + + // For combinator -Array + AggregateFunctionCombinatorPtr array_combinator = AggregateFunctionCombinatorFactory::instance().tryFindSuffix(combinator_suffix); + AggregateFunctionPtr uniq_exact_array_function = array_combinator->transformAggregateFunction( + uniq_exact_function, {}, argument_types, parameters); + + size_t total_size_of_aggregate_states = 0; + size_t align_aggregate_states = 1; + total_size_of_aggregate_states = uniq_exact_array_function->sizeOfData(); + align_aggregate_states = std::max(align_aggregate_states, uniq_exact_array_function->alignOfData()); + + std::shared_ptr aggregates_pool = std::make_shared(); + AggregateDataPtr place = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + try + { + uniq_exact_array_function->create(place); + } + catch (...) + { + uniq_exact_array_function->destroy(place); + throw Exception("Cannot allocate memory", ErrorCodes::CANNOT_ALLOCATE_MEMORY); + } + size_t rows = column->size(); + ColumnRawPtrs column_ptrs; + column_ptrs.emplace_back(column.get()); + const IColumn ** batch_arguments = column_ptrs.data(); + + uniq_exact_array_function->addBatchSinglePlace(rows, place, batch_arguments, nullptr); + + DataTypePtr result_type = std::make_shared(); + ColumnPtr result_column = result_type->createColumn(); + MutableColumnPtr mutable_column = result_column->assumeMutable(); + uniq_exact_array_function->insertResultInto(place, *mutable_column, nullptr); + + return (*result_column)[0]; +} + +AggregateFunctionPtr createAggregateFunctionMin( + const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings) +{ + return AggregateFunctionPtr(createAggregateFunctionSingleValue(name, argument_types, parameters, settings)); +} + +Field Min::executeOnColumn(const ColumnPtr & column, const DataTypePtr & type) +{ + String name = "min"; + DataTypes argument_types(1); + argument_types[0] = type; + Array parameters; + + AggregateFunctionPtr nested_function = createAggregateFunctionMin(name, transformArguments(argument_types), parameters, nullptr); + AggregateFunctionPtr aggregate_function = type->isNullable() + ? std::make_shared>(nested_function, argument_types, parameters) + : nested_function; + + size_t total_size_of_aggregate_states = 0; /// The total size of the row from the aggregate functions. + // add info to track alignment requirement + // If there are states whose alignment are v1, ..vn, align_aggregate_states will be max(v1, ... vn) + size_t align_aggregate_states = 1; + total_size_of_aggregate_states = aggregate_function->sizeOfData(); + align_aggregate_states = std::max(align_aggregate_states, aggregate_function->alignOfData()); + + std::shared_ptr aggregates_pool = std::make_shared(); /// The pool that is currently used for allocation. + AggregateDataPtr place = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + try + { + /** An exception may occur if there is a shortage of memory. + * In order that then everything is properly destroyed, we "roll back" some of the created states. + * The code is not very convenient. + */ + aggregate_function->create(place); + } + catch (...) + { + aggregate_function->destroy(place); + throw Exception("Cannot allocate memory", ErrorCodes::CANNOT_ALLOCATE_MEMORY); + } + size_t rows = column->size(); + ColumnRawPtrs column_ptrs; + column_ptrs.emplace_back(column.get()); + const IColumn ** batch_arguments = column_ptrs.data(); + + aggregate_function->addBatchSinglePlace(rows, place, batch_arguments, nullptr); + + ColumnPtr result_column = type->createColumn(); + MutableColumnPtr mutable_column = result_column->assumeMutable(); + aggregate_function->insertResultInto(place, *mutable_column, nullptr); + return (*result_column)[0]; +} + +AggregateFunctionPtr createAggregateFunctionMax( + const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings) +{ + return AggregateFunctionPtr(createAggregateFunctionSingleValue(name, argument_types, parameters, settings)); +} + +Field Max::executeOnColumn(const ColumnPtr & column, const DataTypePtr & type) +{ + String name = "max"; + DataTypes argument_types(1); + argument_types[0] = type; + Array parameters; + + AggregateFunctionPtr nested_function = createAggregateFunctionMax(name, transformArguments(argument_types), parameters, nullptr); + AggregateFunctionPtr aggregate_function = type->isNullable() + ? std::make_shared>(nested_function, argument_types, parameters) + : nested_function; + + size_t total_size_of_aggregate_states = 0; /// The total size of the row from the aggregate functions. + // add info to track alignment requirement + // If there are states whose alignment are v1, ..vn, align_aggregate_states will be max(v1, ... vn) + size_t align_aggregate_states = 1; + total_size_of_aggregate_states = aggregate_function->sizeOfData(); + align_aggregate_states = std::max(align_aggregate_states, aggregate_function->alignOfData()); + + std::shared_ptr aggregates_pool = std::make_shared(); /// The pool that is currently used for allocation. + AggregateDataPtr place = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + try + { + /** An exception may occur if there is a shortage of memory. + * In order that then everything is properly destroyed, we "roll back" some of the created states. + * The code is not very convenient. + */ + aggregate_function->create(place); + } + catch (...) + { + aggregate_function->destroy(place); + throw Exception("Cannot allocate memory", ErrorCodes::CANNOT_ALLOCATE_MEMORY); + } + size_t rows = column->size(); + ColumnRawPtrs column_ptrs; + column_ptrs.emplace_back(column.get()); + const IColumn ** batch_arguments = column_ptrs.data(); + + aggregate_function->addBatchSinglePlace(rows, place, batch_arguments, nullptr); + + ColumnPtr result_column = type->createColumn(); + MutableColumnPtr mutable_column = result_column->assumeMutable(); + aggregate_function->insertResultInto(place, *mutable_column, nullptr); + return (*result_column)[0]; +} + +AggregateFunctionPtr createAggregateFunctionCountByGranularity( + const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *) +{ + if (argument_types.size() != 1) + throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const IDataType & argument_type = *argument_types[0]; + WhichDataType which(argument_type); + + if (which.isNothing() || which.isArray() || which.isFunction() || which.isAggregateFunction() || which.isMap() || which.isBitmap64() + || which.isSet() || which.isTuple() || which.isInterval() || which.isDecimal() || which.isInt128() || which.isUInt128() || which.isDateOrDateTime()) + { + throw Exception( + "argument of " + name + + " can not be " + "(Nothing,Array,Function," + "AggregateFunction,Map,Bitmap64," + "Set,Tuple,Interval," + "Decimal,Int128,UInt128, DateOrDateTime)", + ErrorCodes::BAD_ARGUMENTS); + } + else if (which.isStringOrFixedString()) + { + //auto a =AggregateFunctionCountByGranularity(argument_types, params); + return std::make_shared>(argument_types, params); + } + else if (which.isInt8()) + { + auto a = AggregateFunctionCountByGranularity(argument_types, params); + return std::make_shared>(argument_types, params); + } + else if (which.isUInt8() || which.isEnum8()) + { + return std::make_shared>(argument_types, params); + } + else if (which.isInt16()) + { + return std::make_shared>(argument_types, params); + } + else if (which.isUInt16() || which.isEnum16()) + { + return std::make_shared>(argument_types, params); + } + else if (which.isInt32()) + { + return std::make_shared>(argument_types, params); + } + else if (which.isUInt32() || which.isDateTime()) + { + return std::make_shared>(argument_types, params); + } + else if (which.isInt64()) + { + return std::make_shared>(argument_types, params); + } + else if (which.isUInt64()) + { + return std::make_shared>(argument_types, params); + } + // TODO can't support Int128 for now + // else if (which.isInt128()) + // { + // return std::make_shared>(argument_types, params); + // } + else if (which.isUInt128()) + { + return std::make_shared>(argument_types, params); + } + else if (which.isFloat32()) + { + return std::make_shared>(argument_types, params); + } + else if (which.isFloat64()) + { + return std::make_shared>(argument_types, params); + } + // TODO can't support Decimal for now + // else if (which.isDecimal32()) + // { + // return std::make_shared>(argument_types, params); + // } + // else if (which.isDecimal64() || which.isDateTime64()) + // { + // return std::make_shared>(argument_types, params); + // } + // else if (which.isDecimal128()) + // { + // return std::make_shared>(argument_types, params); + // } + else + { + return std::make_shared>(argument_types, params); + } + + __builtin_unreachable(); +} + +ColumnPtr CountByGranularity::executeOnColumn(const ColumnPtr & column, const DataTypePtr & type) +{ + String name = "countByGranularity"; + DataTypes argument_types(1); + argument_types[0] = recursiveRemoveLowCardinality(type); + Array parameters; + + AggregateFunctionPtr nested_function = createAggregateFunctionCountByGranularity(name, transformArguments(argument_types), parameters, nullptr); + AggregateFunctionPtr aggregate_function = argument_types[0]->isNullable() ? std::make_shared>(nested_function, argument_types, parameters) + : nested_function; + + size_t total_size_of_aggregate_states = 0; /// The total size of the row from the aggregate functions. + // add info to track alignment requirement + // If there are states whose alignment are v1, ..vn, align_aggregate_states will be max(v1, ... vn) + size_t align_aggregate_states = 1; + total_size_of_aggregate_states = aggregate_function->sizeOfData(); + align_aggregate_states = std::max(align_aggregate_states, aggregate_function->alignOfData()); + + std::shared_ptr aggregates_pool = std::make_shared(); /// The pool that is currently used for allocation. + AggregateDataPtr place = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + try + { + /** An exception may occur if there is a shortage of memory. + * In order that then everything is properly destroyed, we "roll back" some of the created states. + * The code is not very convenient. + */ + aggregate_function->create(place); + } + catch (...) + { + aggregate_function->destroy(place); + throw Exception("Cannot allocate memory", ErrorCodes::CANNOT_ALLOCATE_MEMORY); + } + size_t rows = column->size(); + ColumnRawPtrs column_ptrs; + column_ptrs.emplace_back(recursiveRemoveLowCardinality(column).get()); + const IColumn ** batch_arguments = column_ptrs.data(); + + aggregate_function->addBatchSinglePlace(rows, place, batch_arguments, nullptr); + + ColumnPtr result_column = nested_function->getReturnType()->createColumn(); + MutableColumnPtr mutable_column = result_column->assumeMutable(); + aggregate_function->insertResultInto(place, *mutable_column, nullptr); + + return result_column; +} + +} diff --git a/programs/schema-advisor/Statistics.h b/programs/schema-advisor/Statistics.h new file mode 100644 index 00000000000..70ee858bd08 --- /dev/null +++ b/programs/schema-advisor/Statistics.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + +namespace DB +{ +class UniExtract +{ +public: + Field executeOnColumn(const ColumnPtr & column, const DataTypePtr & type); + Field executeOnColumnArray(const ColumnPtr & column, const DataTypePtr & type); +}; + +class Min +{ +public: + Field executeOnColumn(const ColumnPtr & column, const DataTypePtr & type); +}; + +class Max +{ +public: + Field executeOnColumn(const ColumnPtr & column, const DataTypePtr & type); +}; + +class CountByGranularity +{ +public: + ColumnPtr executeOnColumn(const ColumnPtr & column, const DataTypePtr & type); +}; + +} diff --git a/programs/schema-advisor/TypeAdvisor.cpp b/programs/schema-advisor/TypeAdvisor.cpp new file mode 100644 index 00000000000..004425a9a0a --- /dev/null +++ b/programs/schema-advisor/TypeAdvisor.cpp @@ -0,0 +1,237 @@ +#include "TypeAdvisor.h" +#include "ColumnUsageExtractor.h" +#include "Core/Types.h" +#include "SampleColumnReader.h" +#include "Statistics.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +TypeAdvisor::TypeAdvisor( + MockEnvironment & env_, + const po::variables_map & options_, + const ColumnsDescription & column_descs_, + std::string absolute_part_path_, + size_t sample_row_number_, + size_t max_threads_, + bool lc_only_, + Float64 scanned_count_threshold_for_lc_, + Float64 cardinality_ratio_threshold_for_lc_) + : env(env_) + , options(options_) + , column_descs(column_descs_) + , absolute_part_path(absolute_part_path_ + "/") + , sample_row_number(sample_row_number_) + , max_threads(max_threads_) + , lc_only(lc_only_) + , scanned_count_threshold_for_lc(scanned_count_threshold_for_lc_) + , cardinality_ratio_threshold_for_lc(cardinality_ratio_threshold_for_lc_) +{ + parseCodecCandidates(); +} + +void TypeAdvisor::parseCodecCandidates() +{ + +} + +DataTypePtr decayDataType(DataTypePtr type) +{ + if (type->isNullable()) + return dynamic_cast(type.get())->getNestedType(); + return type; +} + +TypeAdvisor::TypeRecommendation buildTypeRecommendation(std::string column_name, std::string origin_type, bool is_type_nullable, std::string optimized_type) +{ + return {column_name, is_type_nullable ? "Nullable(" + origin_type + ")" : origin_type, is_type_nullable ? "Nullable(" + optimized_type + ")" : optimized_type}; +} + +void TypeAdvisor::adviseLowCardinality() +{ + auto context = createContext(options, env); + auto queries = loadQueries(options); + + ColumnUsageExtractor extractor(context, max_threads); + auto column_usages = extractor.extractColumnUsages(queries); + auto type_usages = extractor.extractUsageForLowCardinality(column_usages); + + LOG_DEBUG(getLogger("TypeAdvisor"), "Extract {} candidate columns, {}, {}", type_usages.size(), scanned_count_threshold_for_lc, cardinality_ratio_threshold_for_lc); + + UniExtract uniq_extract; + for (const auto & type_usage : type_usages) + { + if (type_usage.second < queries.size() * scanned_count_threshold_for_lc) + { + LOG_DEBUG(getLogger("TypeAdvisor"), "Do not Recommend lowcardinality column {}, scanned count is {}", type_usage.first.column, type_usage.second); + continue; + } + + auto column_info = type_usage.first; + if (isMapImplicitKey(column_info.column)) + continue; + + auto storage = MockEnvironment::tryGetLocalTable(column_info.database, column_info.table, context); + if (!storage) + throw Exception(column_info.database + "(" + column_info.table + "): can not find local table.", ErrorCodes::NOT_FOUND_EXPECTED_DATA_PART); + + auto metadata = storage->getInMemoryMetadataCopy(); + auto column_and_type = metadata.getColumns().tryGetColumn(GetColumnsOptions::Kind::AllPhysical, column_info.column); + if (!column_and_type) + continue; + + auto column_type = column_and_type->type; + if (column_type->getTypeId() == TypeIndex::LowCardinality || !isString(decayDataType(column_type))) + continue; + + SampleColumnReader reader(absolute_part_path + "/", 0, sample_row_number); + ColumnPtr column; + try + { + column = reader.readColumn({type_usage.first.column, column_type}); + } + catch (...) + { + // Just skip the column if it can't be read + LOG_DEBUG( + getLogger("TypeAdvisor"), + "Can't read column file " + type_usage.first.column + " from table " + column_info.database + "." + column_info.table + + ", error message: " + + getCurrentExceptionMessage(true)); + continue; + } + + // All following: check skip index + size_t ndv = uniq_extract.executeOnColumn(column, column_type).get(); + + if (ndv > sample_row_number * cardinality_ratio_threshold_for_lc) + { + LOG_DEBUG(getLogger("TypeAdvisor"), "Do not Recommend lowcardinality column {}, scanned count is {}, ndv is {}", type_usage.first.column, type_usage.second, ndv); + continue; + } + + LOG_DEBUG(getLogger("TypeAdvisor"), "Recommend lowcardinality column {}, scanned count is {}, ndv is {}", type_usage.first.column, type_usage.second, ndv); + + type_recommendations.push_back({column_and_type->name + , column_and_type->type->isNullable() ? "Nullable(String)" : "String" + , column_and_type->type->isNullable() ? "LowCardinality(Nullable(String))" : "LowCardinality(String)"});} + +} + +void TypeAdvisor::execute() +{ + if (lc_only) + return adviseLowCardinality(); + + UniExtract uniqExtractFunc; + Max maxFunc; + Min minFunc; + SampleColumnReader reader(absolute_part_path, 0, sample_row_number); + for (const NameAndTypePair & name_and_type : column_descs.getOrdinary()) + { + auto decayed_type = decayDataType(name_and_type.type); + + bool is_string = decayed_type->getTypeId() == TypeIndex::String; + bool is_float_64 = decayed_type->getTypeId() == TypeIndex::Float64; + bool is_unsigned_integer = decayed_type->isValueRepresentedByUnsignedInteger() && decayed_type->isSummable(); + bool is_integer = decayed_type->isValueRepresentedByInteger() && decayed_type->isSummable(); + + if (is_string) + { + ColumnPtr column = reader.readColumn(name_and_type); + auto ndv = uniqExtractFunc.executeOnColumn(column, name_and_type.type).get(); + if (ndv < ADVISOR_LOW_CARDINALITY_NDV_THRESHOLD) + type_recommendations.push_back({name_and_type.name + , name_and_type.type->isNullable() ? "Nullable(String)" : "String" + , name_and_type.type->isNullable() ? "LowCardinality(Nullable(String))" : "LowCardinality(String)"}); + } + else if (is_float_64) + { + ColumnPtr column = reader.readColumn(name_and_type); + auto max = maxFunc.executeOnColumn(column, name_and_type.type).get(); + auto min = minFunc.executeOnColumn(column, name_and_type.type).get(); + if (min >= std::numeric_limits::min() && max <= std::numeric_limits::max()) + type_recommendations.push_back({name_and_type.name + , name_and_type.type->isNullable() ? "Nullable(Float64)" : "Float64" + , name_and_type.type->isNullable() ? "Nullable(Float32)" : "Float32"}); + } + else if (is_unsigned_integer) + { + if (decayed_type->getTypeId() == TypeIndex::UInt8) /// skip UInt8 + continue; + + ColumnPtr column = reader.readColumn(name_and_type); + auto max = maxFunc.executeOnColumn(column, name_and_type.type).get(); + if (max <= std::numeric_limits::max()) + type_recommendations.push_back(buildTypeRecommendation(name_and_type.name, decayed_type->getName(), name_and_type.type->isNullable(), "UInt8")); + else if (max <= std::numeric_limits::max()) + type_recommendations.push_back(buildTypeRecommendation(name_and_type.name, decayed_type->getName(), name_and_type.type->isNullable(), "UInt16")); + else if (max <= std::numeric_limits::max()) + type_recommendations.push_back(buildTypeRecommendation(name_and_type.name, decayed_type->getName(), name_and_type.type->isNullable(), "UInt32")); + } + else if (is_integer) + { + if (decayed_type->getTypeId() == TypeIndex::Int8) /// skip Int8 + continue; + + ColumnPtr column = reader.readColumn(name_and_type); + auto max = maxFunc.executeOnColumn(column, name_and_type.type).get(); + auto min = minFunc.executeOnColumn(column, name_and_type.type).get(); + if (min >= std::numeric_limits::min() && max <= std::numeric_limits::max()) + type_recommendations.push_back(buildTypeRecommendation(name_and_type.name, decayed_type->getName(), name_and_type.type->isNullable(), "Int8")); + else if (min >= std::numeric_limits::min() && max <= std::numeric_limits::max()) + type_recommendations.push_back(buildTypeRecommendation(name_and_type.name, decayed_type->getName(), name_and_type.type->isNullable(), "Int16")); + else if (min >= std::numeric_limits::min() && max <= std::numeric_limits::max()) + type_recommendations.push_back(buildTypeRecommendation(name_and_type.name, decayed_type->getName(), name_and_type.type->isNullable(), "Int32")); + } + /// TODO(weiping.qw): add more rules + } +} + +void TypeAdvisor::serializeJson(WriteBuffer & buf, [[maybe_unused]] bool verbose) +{ + bool first = true; + writeString("\"type\":[", buf); + for (const auto & entry : type_recommendations) + { + if (first) + first = false; + else + writeString(",", buf); + std::string column_name = entry.column_name; + writeString("{\"name\":\"", buf); + writeString(column_name, buf); + writeString("\",", buf); + std::string column_origin_type = entry.origin_type; + std::string column_optimized_type = entry.optimized_type; + writeString("\"origin\":\"", buf); + writeString(column_origin_type, buf); + writeString("\",", buf); + writeString("\"optimized\":\"", buf); + writeString(column_optimized_type, buf); + writeString("\"}", buf); + } + writeString("]", buf); +} + +} diff --git a/programs/schema-advisor/TypeAdvisor.h b/programs/schema-advisor/TypeAdvisor.h new file mode 100644 index 00000000000..88e10d60b3b --- /dev/null +++ b/programs/schema-advisor/TypeAdvisor.h @@ -0,0 +1,71 @@ +#pragma once + +#include + +#include "SchemaAdvisorHelpers.h" + +#include +#include +#include + +namespace DB +{ + +namespace po = boost::program_options; + +class TypeAdvisor +{ +public: + struct TypeRecommendation + { + TypeRecommendation( + std::string column_name_, + std::string origin_type_, + std::string optimized_type_ + ) : column_name(column_name_) + , origin_type(origin_type_) + , optimized_type(optimized_type_) {} + + std::string column_name; + std::string origin_type; + std::string optimized_type; + }; + +private: + static constexpr const size_t ADVISOR_LOW_CARDINALITY_NDV_THRESHOLD = 65535; + + MockEnvironment & env; + po::variables_map options; + const ColumnsDescription column_descs; + Codecs codecs_to_compare; + std::string absolute_part_path; + const size_t sample_row_number; + [[maybe_unused]] const size_t max_threads; + std::vector type_recommendations; + const bool lc_only; + const Float64 scanned_count_threshold_for_lc; + const Float64 cardinality_ratio_threshold_for_lc; + + void parseCodecCandidates(); + + void adviseLowCardinality(); + +public: + TypeAdvisor( + MockEnvironment & env_, + const po::variables_map & options_, + const ColumnsDescription & column_descs_, + std::string absolute_part_path_, + size_t sample_row_number_, + size_t max_threads_, + bool lc_only_, + Float64 scanned_count_threshold_for_lc_, + Float64 cardinality_ratio_threshold_for_lc_); + + virtual ~TypeAdvisor() = default; + + void execute(); + void serializeJson(WriteBuffer & buf, bool verbose = false); +}; + +} diff --git a/programs/schema-advisor/clickhouse-schema-advisor.cpp b/programs/schema-advisor/clickhouse-schema-advisor.cpp new file mode 100644 index 00000000000..556fd3a1bdf --- /dev/null +++ b/programs/schema-advisor/clickhouse-schema-advisor.cpp @@ -0,0 +1,6 @@ +int mainEntryClickHouseSchemaAdvisor(int argc, char ** argv); + +int main(int argc_, char ** argv_) +{ + return mainEntryClickHouseSchemaAdvisor(argc_, argv_); +} diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 849af98506b..c01e8dddeca 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1183,6 +1183,17 @@ int Server::main(const std::vector & /*args*/) global_context->setVWCustomizedSettings(std::make_shared(config)); } + if (global_context->getIsRestrictSettingsToWhitelist()) + { + auto setting_names = getMultipleValuesFromConfig(*config, "tenant_whitelist_settings", "name"); + std::unordered_set setting_names_set; + for (auto& setting : setting_names) + { + setting_names_set.emplace(setting); + } + global_context->setExtraRestrictSettingsToWhitelist(std::move(setting_names_set)); + } + if (auto catalog = global_context->tryGetCnchCatalog()) catalog->loadFromConfig("catalog_service", *config); }, diff --git a/src/Advisor/Advisor.cpp b/src/Advisor/Advisor.cpp index b04696ff8f1..50e15e47da5 100644 --- a/src/Advisor/Advisor.cpp +++ b/src/Advisor/Advisor.cpp @@ -1,8 +1,10 @@ #include #include -#include +#include +#include #include +#include #include #include #include @@ -26,19 +28,25 @@ WorkloadAdvisors Advisor::getAdvisors(ASTAdviseQuery::AdvisorType type) { case ASTAdviseQuery::AdvisorType::ALL: return { - std::make_shared(), + std::make_shared(), std::make_shared(), + std::make_shared(), std::make_shared(MaterializedViewAdvisor::OutputType::PROJECTION, true, true), - std::make_shared(MaterializedViewAdvisor::OutputType::MATERIALIZED_VIEW, true, false)}; + std::make_shared(MaterializedViewAdvisor::OutputType::MATERIALIZED_VIEW, true, true)}; + case ASTAdviseQuery::AdvisorType::ORDER_BY: - return {std::make_shared()}; - case ASTAdviseQuery::AdvisorType::DISTRIBUTED_BY: + return {std::make_shared()}; + case ASTAdviseQuery::AdvisorType::CLUSTER_BY: return {std::make_shared()}; + case ASTAdviseQuery::AdvisorType::DATA_TYPE: + return {std::make_shared()}; case ASTAdviseQuery::AdvisorType::MATERIALIZED_VIEW: - return {std::make_shared(MaterializedViewAdvisor::OutputType::MATERIALIZED_VIEW, true, false)}; + return {std::make_shared(MaterializedViewAdvisor::OutputType::MATERIALIZED_VIEW, true, true)}; case ASTAdviseQuery::AdvisorType::PROJECTION: return {std::make_shared(MaterializedViewAdvisor::OutputType::PROJECTION, true, true)}; - } + case ASTAdviseQuery::AdvisorType::COLUMN_USAGE: + return {std::make_shared()}; + } } WorkloadAdvises Advisor::analyze(const std::vector & queries_, ContextPtr context_) diff --git a/src/Advisor/Advisor.h b/src/Advisor/Advisor.h index 184dd8c0a2e..2172906ef00 100644 --- a/src/Advisor/Advisor.h +++ b/src/Advisor/Advisor.h @@ -21,7 +21,7 @@ class Advisor } WorkloadAdvises analyze(const std::vector & queries, ContextPtr context); - private: +private: static WorkloadAdvisors getAdvisors(ASTAdviseQuery::AdvisorType type); ASTAdviseQuery::AdvisorType type; diff --git a/src/Advisor/AdvisorContext.cpp b/src/Advisor/AdvisorContext.cpp index 67faebe3188..32c4d1e56aa 100644 --- a/src/Advisor/AdvisorContext.cpp +++ b/src/Advisor/AdvisorContext.cpp @@ -19,7 +19,7 @@ AdvisorContext AdvisorContext::buildFrom(ContextMutablePtr session_context, Work ColumnUsages column_usages = buildColumnUsages(queries); SignatureUsages signature_usages = buildSignatureUsages(queries, session_context); -std::unordered_map query_id_to_query; + std::unordered_map query_id_to_query; for (const auto & query : queries) query_id_to_query[query->getQueryId()] = query; @@ -27,7 +27,7 @@ std::unordered_map query_id_to_query; session_context, tables, queries, -std::move(query_id_to_query), + std::move(query_id_to_query), query_thread_pool, std::move(column_usages), std::move(signature_usages)); diff --git a/src/Advisor/AdvisorContext.h b/src/Advisor/AdvisorContext.h index 6fe854acd8b..c5c0c2332d1 100644 --- a/src/Advisor/AdvisorContext.h +++ b/src/Advisor/AdvisorContext.h @@ -19,7 +19,7 @@ class AdvisorContext ContextMutablePtr session_context; WorkloadTables & tables; WorkloadQueries & queries; -std::unordered_map query_id_to_query; + std::unordered_map query_id_to_query; ThreadPool & query_thread_pool; const ColumnUsages column_usages; const SignatureUsages signature_usages; @@ -34,16 +34,16 @@ std::unordered_map query_id_to_query; private: AdvisorContext( ContextMutablePtr _session_context, - WorkloadTables & _tables, - WorkloadQueries & _queries, -std::unordered_map _query_id_to_query, - ThreadPool & _query_thread_pool, - ColumnUsages _column_usages, - SignatureUsages _signature_usages) + WorkloadTables & _tables, + WorkloadQueries & _queries, + std::unordered_map _query_id_to_query, + ThreadPool & _query_thread_pool, + ColumnUsages _column_usages, + SignatureUsages _signature_usages) : session_context(_session_context) , tables(_tables) , queries(_queries) -, query_id_to_query(std::move(_query_id_to_query)) + , query_id_to_query(std::move(_query_id_to_query)) , query_thread_pool(_query_thread_pool) , column_usages(std::move(_column_usages)) , signature_usages(std::move(_signature_usages)) diff --git a/src/Advisor/ColumnUsage.cpp b/src/Advisor/ColumnUsage.cpp index 9cb58a98221..5799879d4e6 100644 --- a/src/Advisor/ColumnUsage.cpp +++ b/src/Advisor/ColumnUsage.cpp @@ -3,10 +3,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -52,13 +54,23 @@ namespace return function.name == "in" && function.arguments && function.arguments->children.size() == 2; } - std::optional> extractPredicateUsage(ConstASTPtr expression) + ASTPtr unwarpMonotonicFunction(ASTPtr expr) { - auto fun = dynamic_pointer_cast(expression); + if (auto * function = expr->as()) + { + if (function->arguments->children.size() == 1) + return unwarpMonotonicFunction(function->arguments->children[0]); + } + return expr; + }; + + std::optional> extractPredicateUsage(ConstASTPtr predicate) + { + auto fun = dynamic_pointer_cast(predicate); if (!fun || !fun->arguments || fun->arguments->children.size() != 2) return std::nullopt; - auto identifier = dynamic_pointer_cast(fun->arguments->children[0]); - if (!identifier) + auto left = unwarpMonotonicFunction(fun->arguments->children[0]); + auto identifier = dynamic_pointer_cast(left); if (!identifier) return std::nullopt; const std::string & symbol = identifier->name(); @@ -119,6 +131,9 @@ class ColumnUsageVisitor : public PlanNodeVisitor void visitAggregatingNode(AggregatingNode & node, ColumnUsages & column_usages) override; void visitCTERefNode(CTERefNode & node, ColumnUsages & column_usages) override; + void extractFilterUsages(ConstASTPtr expr, PlanNodePtr, ColumnUsages & column_usages); + void extractArraySetFunctions(ConstASTPtr expression, const PlanNodePtr & node, ColumnUsages & column_usages); + private: std::unordered_map symbol_to_table_column_map; std::unordered_set visited_ctes; @@ -154,6 +169,19 @@ size_t ColumnUsageInfo::getFrequency(ColumnUsageType type, bool only_source_tabl return freq; } +std::unordered_map ColumnUsageInfo::getFrequencies(bool only_source_table) const { + std::unordered_map res; + for (const auto & item : usages_only_source_table) { + res[item.first] += 1; + } + if (!only_source_table) { + for (const auto & item : usages_non_source_table) { + res[item.first] += 1; + } + } + return res; +} + std::vector ColumnUsageInfo::getUsages(ColumnUsageType type, bool only_source_table) const { std::vector res{}; @@ -195,11 +223,34 @@ void ColumnUsageVisitor::visitTableScanNode(TableScanNode & node, ColumnUsages & auto table_step = dynamic_pointer_cast(node.getStep()); const StorageID & storage_id = table_step->getStorageID(); + std::unordered_map table_columns; + + for (const auto & column_name : table_step->getRequiredColumns()) + { + QualifiedColumnName column{storage_id.getDatabaseName(), storage_id.getTableName(), column_name}; + table_columns.insert_or_assign(column_name, ColumnNameWithSourceTableFlag{column, true}); + } + + // extract usages + symbol_to_table_column_map.swap(table_columns); + for (const auto & column_name : table_step->getRequiredColumns()) + addUsage(column_usages, column_name, ColumnUsageType::SCANNED, node.shared_from_this()); + + if (table_step->getPrewhere()) + extractFilterUsages(table_step->getPrewhere(), node.shared_from_this(), column_usages); + + // for (auto [output, expr] : table_step->getIndexExpressions()) + // extractFilterUsages(expr, node.shared_from_this(), column_usages); + + for (auto [output, expr] : table_step->getInlineExpressions()) + extractFilterUsages(expr, node.shared_from_this(), column_usages); + + symbol_to_table_column_map.swap(table_columns); + for (const auto & [column_name, alias] : table_step->getColumnAlias()) { QualifiedColumnName column{storage_id.getDatabaseName(), storage_id.getTableName(), column_name}; - symbol_to_table_column_map.emplace(alias, ColumnNameWithSourceTableFlag{column, true}); - addUsage(column_usages, alias, ColumnUsageType::SCANNED, node.shared_from_this()); + symbol_to_table_column_map.insert_or_assign(alias, ColumnNameWithSourceTableFlag{column, true}); } } @@ -207,12 +258,44 @@ void ColumnUsageVisitor::visitFilterNode(FilterNode & node, ColumnUsages & colum { processChildren(node, column_usages); auto filter_step = dynamic_pointer_cast(node.getStep()); - for (const ConstASTPtr & expression : PredicateUtils::extractConjuncts(filter_step->getFilter())) + extractFilterUsages(filter_step->getFilter(), node.shared_from_this(), column_usages); +} + +void ColumnUsageVisitor::extractFilterUsages(ConstASTPtr expr, PlanNodePtr node, ColumnUsages & column_usages) +{ + for (const auto & expression : PredicateUtils::extractConjuncts(expr)) { auto usage_opt = extractPredicateUsage(expression); if (usage_opt.has_value()) - addUsage(column_usages, usage_opt.value().first, usage_opt.value().second, node.shared_from_this(), expression); + addUsage(column_usages, usage_opt.value().first, usage_opt.value().second, node, expression); + else + { + auto names = SymbolsExtractor::extract(expression); + for (const auto & name : names) + { + addUsage(column_usages, name, ColumnUsageType::OTHER_PREDICATE, node, expression); + } + } + } + extractArraySetFunctions(expr, node, column_usages); +} + +void ColumnUsageVisitor::extractArraySetFunctions(ConstASTPtr expression, const PlanNodePtr & node, ColumnUsages & column_usages) +{ + auto function = dynamic_pointer_cast(expression); + if (const auto * func = expression->as()) + { + if (!func->arguments || func->arguments->children.empty()) return; + auto * ident = func->arguments->children[0]->as(); + if (ident && BitmapIndexHelper::isArraySetFunctions(func->name)) + { + addUsage(column_usages, ident->name(), ColumnUsageType::ARRAY_SET_FUNCTION, node, expression); + return; + } } + + for (const auto & child : expression->children) + extractArraySetFunctions(child, node, column_usages); } void ColumnUsageVisitor::visitJoinNode(JoinNode & node, ColumnUsages & column_usages) @@ -256,8 +339,10 @@ void ColumnUsageVisitor::visitProjectionNode(ProjectionNode & node, ColumnUsages { auto it = symbol_to_table_column_map.find(identifier->name()); if (it != symbol_to_table_column_map.end()) - symbol_to_table_column_map.emplace(out_symbol, it->second); + symbol_to_table_column_map.insert_or_assign(out_symbol, it->second); } + + extractArraySetFunctions(in_ast, node.shared_from_this(), column_usages); } } @@ -286,4 +371,27 @@ void ColumnUsageVisitor::visitCTERefNode(CTERefNode & node, ColumnUsages & colum VisitorUtil::accept(cte_info.getCTEs().at(cte_id), *this, column_usages); } +String toString(ColumnUsageType type) +{ + switch (type) { + case ColumnUsageType::SCANNED: + return "Scanned"; + case ColumnUsageType::EQUI_JOIN: + return "EquiJoin"; + case ColumnUsageType::NON_EQUI_JOIN: + return "NonEquiJoin"; + case ColumnUsageType::GROUP_BY: + return "GroupBy"; + case ColumnUsageType::EQUALITY_PREDICATE: + return "EqualityPredicate"; + case ColumnUsageType::IN_PREDICATE: + return "InPredicate"; + case ColumnUsageType::RANGE_PREDICATE: + return "RangePredicate"; + case ColumnUsageType::ARRAY_SET_FUNCTION: + return "ArraySetFunction"; + case ColumnUsageType::OTHER_PREDICATE: + return "OtherPredicate"; + } +} } diff --git a/src/Advisor/ColumnUsage.h b/src/Advisor/ColumnUsage.h index 3c3e670febc..27f097a53d0 100644 --- a/src/Advisor/ColumnUsage.h +++ b/src/Advisor/ColumnUsage.h @@ -23,9 +23,12 @@ enum class ColumnUsageType EQUALITY_PREDICATE, // columns in "= literal" filters IN_PREDICATE, // columns in "in list" filters RANGE_PREDICATE, // columns in "> literal" or "< literal" filters + ARRAY_SET_FUNCTION, // columns in "has" or "arraySetCheck" OTHER_PREDICATE, // columns in "column ???" filters }; +String toString(ColumnUsageType type); + struct ColumnUsage { ColumnUsageType type; @@ -42,6 +45,7 @@ class ColumnUsageInfo void update(ColumnUsage usage, bool is_source_table); size_t getFrequency(ColumnUsageType type, bool only_source_table = false) const; + std::unordered_map getFrequencies(bool only_source_table = false) const; std::vector getUsages(ColumnUsageType type, bool only_source_table = false) const; private: diff --git a/src/Advisor/Rules/ColumnUsageAdvise.cpp b/src/Advisor/Rules/ColumnUsageAdvise.cpp new file mode 100644 index 00000000000..3fcc62a3441 --- /dev/null +++ b/src/Advisor/Rules/ColumnUsageAdvise.cpp @@ -0,0 +1,73 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace DB +{ + +class ColumnUsageAdvise : public IWorkloadAdvise +{ +public: + ColumnUsageAdvise(QualifiedTableName table_, String column_, std::vector> candidates_) + : table(std::move(table_)), column(std::move(column_)), candidates(std::move(candidates_)) + { + } + + String apply(WorkloadTables &) override { return "not implement"; } + + QualifiedTableName getTable() override { return table; } + std::optional getColumnName() override { return column; } + String getAdviseType() override { return "Column Usage"; } + String getOriginalValue() override { return ""; } + String getOptimizedValue() override { return ""; } + double getBenefit() override { return 0; } + std::vector> getCandidates() override { return candidates; } + +private: + QualifiedTableName table; + String column; + std::vector> candidates; +}; + +WorkloadAdvises ColumnUsageAdvisor::analyze(AdvisorContext & context) const +{ + std::map> column_usage_by_table; + for (const auto & [qualified_column, metrics] : context.column_usages) + { + for (const auto & [type, count] : metrics.getFrequencies(true)) + { + column_usage_by_table[qualified_column][toString(type)] += count; + } + } + + WorkloadAdvises res; + for (const auto & [table, column_freq] : column_usage_by_table) + { + std::vector> sorted_freq{column_freq.begin(), column_freq.end()}; + std::sort(sorted_freq.begin(), sorted_freq.end(), [](const auto & p1, const auto & p2) { + // enforce unique ordering + if (p1.second == p2.second) + return p1.first < p2.first; + return p1.second < p2.second; + }); + + res.emplace_back(std::make_shared(table.getQualifiedTable(), table.column, sorted_freq)); + } + return res; +} + +} diff --git a/src/Advisor/Rules/ColumnUsageAdvise.h b/src/Advisor/Rules/ColumnUsageAdvise.h new file mode 100644 index 00000000000..18159501406 --- /dev/null +++ b/src/Advisor/Rules/ColumnUsageAdvise.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace DB +{ + +class ColumnUsageAdvisor : public IWorkloadAdvisor +{ +public: + String getName() const override { return "ColumnUsageAdvisor"; } + WorkloadAdvises analyze(AdvisorContext & context) const override; + +private: + // Poco::Logger * log = getLogger("OrderByKeyAdvisor"); +}; + +} diff --git a/src/Advisor/Rules/DataTypeAdvise.cpp b/src/Advisor/Rules/DataTypeAdvise.cpp new file mode 100644 index 00000000000..d14512374e5 --- /dev/null +++ b/src/Advisor/Rules/DataTypeAdvise.cpp @@ -0,0 +1,174 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "Interpreters/StorageID.h" +#include +#include +#include + +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int UNKNOWN_TABLE; +} + +WorkloadAdvises DataTypeAdvisor::analyze(AdvisorContext & context) const +{ + WorkloadAdvises res; + for (auto & [table_name, workload_table] : context.tables.getTables()) + { + auto basic_stats = workload_table->getStats().getBasicStats(); + if (!basic_stats.get()) + throw Exception("Empty statistics when analyzing data types for table " + table_name.getFullName(), ErrorCodes::LOGICAL_ERROR); + + auto storage = DatabaseCatalog::instance().getTable(StorageID{table_name.database, table_name.table}, context.session_context); + auto columns = storage->getInMemoryMetadataPtr()->getColumns().getAll(); + auto extended_stats + = workload_table->getStats().collectExtendedStats(context.session_context, table_name.database, table_name.table, columns); + + if (!extended_stats.get()) + throw Exception("Empty extended statistics when analyzing data types for table " + table_name.getFullName(), ErrorCodes::LOGICAL_ERROR); + + const auto local_table = workload_table->getTablePtr(); + if (!dynamic_cast(local_table.get())) + throw Exception("Table " + table_name.getFullName() + " is not merge tree table", ErrorCodes::UNKNOWN_TABLE); + + UInt64 row_count = basic_stats->getRowCount(); + auto & table_stats = basic_stats->getSymbolStatistics(); + + for (auto & [column_name, symbol_stats] : table_stats) + { + if (symbol_stats->getNullsCount() == row_count) /// all nulls + continue; + + const auto & column_type = local_table->getInMemoryMetadataPtr()->getColumns().getPhysical(column_name).type; + auto decayed_type = Statistics::decayDataType(column_type); + + bool is_string = decayed_type->getTypeId() == TypeIndex::String || decayed_type->getTypeId() == TypeIndex::FixedString; + bool is_unsigned_integer = decayed_type->isValueRepresentedByUnsignedInteger() && decayed_type->isSummable(); + bool is_integer = decayed_type->isValueRepresentedByInteger() && decayed_type->isSummable(); + + String optimized_type; + if ((is_string && string_type_advisor->checkAndApply(local_table, symbol_stats, extended_stats->at(column_name), row_count, optimized_type)) + || (is_unsigned_integer && integer_type_advisor->checkAndApply(local_table, symbol_stats, decayed_type, true, optimized_type)) + || (is_integer && integer_type_advisor->checkAndApply(local_table, symbol_stats, decayed_type, false, optimized_type))) + { + res.emplace_back(std::make_shared(table_name, column_name, column_type->getName(), optimized_type)); + } + } + } + return res; +} + +bool DataTypeAdvisor::StringTypeAdvisor::checkAndApply(const StoragePtr & local_table, const SymbolStatisticsPtr & symbol_stats, WorkloadExtendedStat & extended_symbol_stats, UInt64 row_count, String & optimized_type) +{ + const auto & nulls_count = symbol_stats->getNullsCount(); + + /// check date + const Field & count_to_date = extended_symbol_stats[WorkloadExtendedStatsType::COUNT_TO_DATE_OR_NULL]; + bool all_date = !count_to_date.isNull() ? count_to_date.get() + nulls_count == row_count : false; + if (all_date) + { + optimized_type = nulls_count > 0 ? "Nullable(Date)" : "Date"; + return true; + } + + /// check date time + const Field & count_to_date_time = extended_symbol_stats[WorkloadExtendedStatsType::COUNT_TO_DATE_TIME_OR_NULL]; + bool all_date_time = !count_to_date_time.isNull() ? count_to_date_time.get() + nulls_count == row_count : false; + if (all_date_time) + { + optimized_type = nulls_count > 0 ? "Nullable(DateTime)" : "DateTime"; + return true; + } + + /// check uint32 + const Field & count_to_uint32 = extended_symbol_stats[WorkloadExtendedStatsType::COUNT_TO_UINT32_OR_NULL]; + bool all_unsigned_integer = !count_to_uint32.isNull() ? count_to_uint32.get() + nulls_count == row_count : false; + if (all_unsigned_integer) + { + optimized_type = nulls_count > 0 ? "Nullable(UInt32)" : "UInt32"; + return true; + } + + /// check float32 + const Field & count_to_float32 = extended_symbol_stats[WorkloadExtendedStatsType::COUNT_TO_FLOAT32_OR_NULL]; + bool all_float32 = !count_to_float32.isNull() ? count_to_float32.get() + nulls_count == row_count : false; + if (all_float32) + { + optimized_type = nulls_count > 0 ? "Nullable(Float32)" : "Float32"; + return true; + } + + /// check (global) low cardinality + const auto & ndv = symbol_stats->getNdv(); + const auto * merge_tree_storage = dynamic_cast(local_table.get()); + bool can_be_inside_low_cardinality = ndv < merge_tree_storage->getSettings()->low_cardinality_ndv_threshold && ndv + nulls_count != row_count; + if (can_be_inside_low_cardinality) + { + String nested_type = nulls_count > 0 ? "Nullable(String)" : "String"; + optimized_type = "LowCardinality(" + nested_type + ")"; + return true; + } + + /// check fixed string + const auto & avg_len = symbol_stats->getAvg(); + bool is_fixed_size = false; /// TODO + if (is_fixed_size) + { + optimized_type = nulls_count > 0 ? "Nullable(FixedString("+ toString(avg_len) +"))" : "FixedString(" + toString(avg_len) + ")"; + return true; + } + + return false; +} + +bool DataTypeAdvisor::IntegerTypeAdvisor::checkAndApply( + [[maybe_unused]] const StoragePtr & local_table, + const SymbolStatisticsPtr & symbol_stats, + const DataTypePtr & decayed_original_type, + bool is_unsigned_type, + String & optimized_type) +{ + const auto & nulls_count = symbol_stats->getNullsCount(); + const auto & max = symbol_stats->getMax(); + + DataTypePtr new_type = nullptr; + if (is_unsigned_type) + { + if (max <= std::numeric_limits::max()) new_type = std::make_shared(); + else if (max <= std::numeric_limits::max()) new_type = std::make_shared(); + else if (max <= std::numeric_limits::max()) new_type = std::make_shared(); + } + else + { + if (max <= std::numeric_limits::max()) new_type = std::make_shared(); + else if (max <= std::numeric_limits::max()) new_type = std::make_shared(); + else if (max <= std::numeric_limits::max()) new_type = std::make_shared(); + } + + if (new_type && new_type->getTypeId() < decayed_original_type->getTypeId()) + { + optimized_type = nulls_count > 0 ? "Nullable(" + new_type->getName() + ")" : new_type->getName(); + return true; + } + + return false; +} + +} diff --git a/src/Advisor/Rules/DataTypeAdvise.h b/src/Advisor/Rules/DataTypeAdvise.h new file mode 100644 index 00000000000..6fa686f90ce --- /dev/null +++ b/src/Advisor/Rules/DataTypeAdvise.h @@ -0,0 +1,77 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +class DataTypeAdvisor : public IWorkloadAdvisor +{ +public: + DataTypeAdvisor() + { + string_type_advisor = std::unique_ptr(); + integer_type_advisor = std::unique_ptr(); + } + String getName() const override { return "DataTypeAdvisor"; } + WorkloadAdvises analyze(AdvisorContext & context) const override; + +private: + class StringTypeAdvisor + { + public: + bool checkAndApply(const StoragePtr & local_table, const SymbolStatisticsPtr & symbol_stats, WorkloadExtendedStat & extended_symbol_stats, UInt64 row_count, String & optimized_type); + }; + + class IntegerTypeAdvisor + { + public: + bool checkAndApply(const StoragePtr & local_table, const SymbolStatisticsPtr & symbol_stats, const DataTypePtr & decayed_original_type, bool is_unsigned_type, String & optimized_type); + }; + + std::unique_ptr string_type_advisor; + std::unique_ptr integer_type_advisor; +}; + +class DataTypeAdvise : public IWorkloadAdvise +{ +public: + DataTypeAdvise( + const QualifiedTableName & table_, const String & column_name_, const String & original_type_, const String & new_type_) + : table(table_), column_name(column_name_), original_type(original_type_), new_type(new_type_) + { + } + + String apply([[maybe_unused]] WorkloadTables & tables) override + { + /// TODO: modify ddl + return ""; + } + + QualifiedTableName getTable() override { return table; } + std::optional getColumnName() override { return {column_name}; } + String getAdviseType() override { return "Data Type"; } + String getOriginalValue() override { return original_type; } + String getOptimizedValue() override { return new_type; } + +private: + QualifiedTableName table; + String column_name; + String original_type; + String new_type; + + const LoggerPtr log = getLogger("DataTypeAdvise"); +}; + +} diff --git a/src/Advisor/Rules/ClusterKeyAdvise.cpp b/src/Advisor/Rules/OrderByKeyAdvise.cpp similarity index 51% rename from src/Advisor/Rules/ClusterKeyAdvise.cpp rename to src/Advisor/Rules/OrderByKeyAdvise.cpp index c5f7c68cc86..fbad81bdc0e 100644 --- a/src/Advisor/Rules/ClusterKeyAdvise.cpp +++ b/src/Advisor/Rules/OrderByKeyAdvise.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -13,20 +13,28 @@ #include #include #include +#include +#include namespace DB { -class ClusterKeyAdvise : public IWorkloadAdvise +class OrderByKeyAdvise : public IWorkloadAdvise { public: - ClusterKeyAdvise(QualifiedTableName table_, - ASTPtr original_order_by_, - String original_column_, - String new_column_, - double benefit_) - : table(std::move(table_)), original_order_by(original_order_by_) - , original_column(std::move(original_column_)), new_column(std::move(new_column_)), benefit(benefit_) + OrderByKeyAdvise( + QualifiedTableName table_, + ASTPtr original_order_by_, + String original_column_, + String new_column_, + double benefit_, + std::vector> candidates_) + : table(std::move(table_)) + , original_order_by(original_order_by_) + , original_column(std::move(original_column_)) + , new_column(std::move(new_column_)) + , benefit(benefit_) + , candidates(std::move(candidates_)) { } @@ -52,6 +60,7 @@ class ClusterKeyAdvise : public IWorkloadAdvise String getOriginalValue() override { return original_column; } String getOptimizedValue() override { return new_column; } double getBenefit() override { return benefit; } + std::vector> getCandidates() override { return candidates; } private: QualifiedTableName table; @@ -59,15 +68,18 @@ class ClusterKeyAdvise : public IWorkloadAdvise String original_column; String new_column; double benefit; + std::vector> candidates; }; -WorkloadAdvises ClusterKeyAdvisor::analyze(AdvisorContext & context) const +WorkloadAdvises OrderByKeyAdvisor::analyze(AdvisorContext & context) const { - std::unordered_map> column_usage_by_table; + std::unordered_map> column_usage_by_table; for (const auto & [qualified_column, metrics] : context.column_usages) { auto predicate_freq = metrics.getFrequency(ColumnUsageType::EQUALITY_PREDICATE, /*only_source_table=*/true) - + metrics.getFrequency(ColumnUsageType::RANGE_PREDICATE, /*only_source_table=*/true); + + metrics.getFrequency(ColumnUsageType::IN_PREDICATE, /*only_source_table=*/true) + + metrics.getFrequency(ColumnUsageType::RANGE_PREDICATE, /*only_source_table=*/true) + + metrics.getFrequency(ColumnUsageType::EQUI_JOIN, /*only_source_table=*/true) /* runtime_filter*/; if (predicate_freq > 0 && isValidColumn(qualified_column, context)) column_usage_by_table[qualified_column.getQualifiedTable()][qualified_column.column] += predicate_freq; } @@ -75,31 +87,33 @@ WorkloadAdvises ClusterKeyAdvisor::analyze(AdvisorContext & context) const WorkloadAdvises res{}; for (const auto & [table, column_freq] : column_usage_by_table) { - auto max_column_freq = *std::max_element(column_freq.begin(), column_freq.end(), - [](const auto & p1, const auto & p2) { - // enforce unique ordering - if (p1.second == p2.second) - return p1.first < p2.first; - return p1.second < p2.second; - }); + std::vector> sorted_freq{column_freq.begin(), column_freq.end()}; + std::sort(sorted_freq.begin(), sorted_freq.end(), [](const auto & p1, const auto & p2) { + // enforce unique ordering + if (p1.second == p2.second) + return p1.first > p2.first; + return p1.second > p2.second; + }); + if (sorted_freq.size() > 3) + sorted_freq.resize(3); auto optimized_table = context.tables.tryGetTable(table); auto order_by = optimized_table ? optimized_table->getOrderBy() : nullptr; auto original_column = (order_by) ? serializeAST(*order_by) : String{}; - res.emplace_back(std::make_shared(table, order_by, original_column, max_column_freq.first, max_column_freq.second)); + res.emplace_back( + std::make_shared(table, order_by, original_column, sorted_freq[0].first, sorted_freq[0].second, sorted_freq)); } return res; } -bool ClusterKeyAdvisor::isValidColumn(const QualifiedColumnName & column, AdvisorContext & context) const +bool OrderByKeyAdvisor::isValidColumn(const QualifiedColumnName & /*column*/, AdvisorContext & /*context*/) const { - auto column_type = context.getColumnType(column); - if (!column_type || !column_type->isComparable()) // sharding key only accepts integers - { - LOG_DEBUG(log, "Column {}.{}.{} is not a valid order by key, because it is not comparable", - column.database, column.table, column.column); - return false; - } + // auto column_type = context.getColumnType(column); + // if (!column_type || !column_type->isValueRepresentedByInteger()) // sharding key only accepts integers + // { + // LOG_DEBUG(log, "Column {}.{}.{} is not a valid sharding key, because it is not an integer type", column.database, column.table, column.column); + // return false; + // } return true; } diff --git a/src/Advisor/Rules/ClusterKeyAdvise.h b/src/Advisor/Rules/OrderByKeyAdvise.h similarity index 80% rename from src/Advisor/Rules/ClusterKeyAdvise.h rename to src/Advisor/Rules/OrderByKeyAdvise.h index 7a95002b19d..b27a488edb8 100644 --- a/src/Advisor/Rules/ClusterKeyAdvise.h +++ b/src/Advisor/Rules/OrderByKeyAdvise.h @@ -10,10 +10,10 @@ namespace DB { -class ClusterKeyAdvisor : public IWorkloadAdvisor +class OrderByKeyAdvisor : public IWorkloadAdvisor { public: - String getName() const override { return "ClusterKeyAdvisor"; } + String getName() const override { return "OrderByKeyAdvisor"; } WorkloadAdvises analyze(AdvisorContext & context) const override; private: diff --git a/src/Advisor/Rules/WorkloadAdvisor.h b/src/Advisor/Rules/WorkloadAdvisor.h index 9fb4bd44ffa..79a87bbc15f 100644 --- a/src/Advisor/Rules/WorkloadAdvisor.h +++ b/src/Advisor/Rules/WorkloadAdvisor.h @@ -78,7 +78,11 @@ class IWorkloadAdvise virtual String getOriginalValue() = 0; virtual String getOptimizedValue() = 0; virtual double getBenefit() { return 0.0; } - virtual std::vector getRelatedQueries() { return {}; } + virtual std::vector> getCandidates() { return {}; } + virtual std::vector getRelatedQueries() + { + return {}; + } }; } diff --git a/src/Advisor/SignatureUsage.cpp b/src/Advisor/SignatureUsage.cpp index fde4000cdb7..788f9ce2042 100644 --- a/src/Advisor/SignatureUsage.cpp +++ b/src/Advisor/SignatureUsage.cpp @@ -17,7 +17,7 @@ SignatureUsages buildSignatureUsages(const WorkloadQueries & queries, ContextPtr SignatureUsages signature_usages; for (const auto & query : queries) { - const auto & plan = query->getPlan(); + const auto & plan = query->getPlanBeforeCascades(); PlanSignatureProvider provider(plan->getCTEInfo(), context); auto plan_signatures = provider.computeSignatures(plan->getPlanNode()); for (const auto & [plan_node, signature] : plan_signatures) diff --git a/src/Advisor/WorkloadQuery.cpp b/src/Advisor/WorkloadQuery.cpp index bd161345b42..54f947edc1d 100644 --- a/src/Advisor/WorkloadQuery.cpp +++ b/src/Advisor/WorkloadQuery.cpp @@ -92,12 +92,23 @@ WorkloadQueryPtr WorkloadQuery::build(const std::string & query_id, const std::s context->applySettingsChanges( {DB::SettingChange("enable_sharding_optimize", "true"), // for colocated join DB::SettingChange("enable_runtime_filter", "false"), // for calculating signature - DB::SettingChange("enable_optimzier", "true")}); + DB::SettingChange("enable_optimzier", "true"), + DB::SettingChange("cte_mode", "INLINED")}); // for materialized view context->createPlanNodeIdAllocator(); context->createSymbolAllocator(); context->createOptimizerMetrics(); context->makeQueryContext(); + if (context->getSettingsRef().print_graphviz) + { + std::stringstream path; + path << context->getSettingsRef().graphviz_path.toString(); + path << "/" << query_id << ".sql"; + std::ofstream out(path.str()); + out << query; + out.close(); + } + // parse and plan const char * begin = query.data(); const char * end = begin + query.size(); @@ -120,6 +131,7 @@ WorkloadQueryPtr WorkloadQuery::build(const std::string & query_id, const std::s CardinalityEstimator::estimate(*query_plan, context); PlanCostMap costs = calculateCost(*query_plan, *context); + return std::make_unique( context, query_id, query, std::move(query_plan), std::move(plan_before_cascades), std::move(query_tables), std::move(costs)); } @@ -140,11 +152,10 @@ WorkloadQueries WorkloadQuery::build(const std::vector & queries, c { WorkloadQueryPtr workload_query = build("q" + std::to_string(i), query, from_context); res[i] = std::move(workload_query); - } catch (Exception & e) + } catch (...) { - LOG_WARNING(getLogger("WorkloadQuery"), - "failed to build query, reason: {}, sql: {}", - e.message(), query); + LOG_WARNING(getLogger("WorkloadQuery"),"failed to build query, reason: {}, sql: {}", + getCurrentExceptionMessage(true), query); } }); } @@ -156,7 +167,7 @@ WorkloadQueries WorkloadQuery::build(const std::vector & queries, c double WorkloadQuery::getOptimalCost(const TableLayout & table_layout) { -if (!root_group) + if (!root_group) { cascades_context = std::make_shared( query_context, @@ -180,6 +191,7 @@ if (!root_group) GroupId root_group_id = root_group->getGroupId(); CascadesOptimizer::optimize(root_group_id, *cascades_context, required_property); auto res = cascades_context->getMemo().getGroupById(root_group_id)->getBestExpression(required_property)->getCost(); + GraphvizPrinter::printMemo(cascades_context->getMemo(), root_group_id, query_context, "CascadesOptimizer-Memo-Graph"); return res; } diff --git a/src/Advisor/WorkloadQuery.h b/src/Advisor/WorkloadQuery.h index 71ea5a91a8c..88d8a7bb7e9 100644 --- a/src/Advisor/WorkloadQuery.h +++ b/src/Advisor/WorkloadQuery.h @@ -49,6 +49,7 @@ class WorkloadQuery : private boost::noncopyable } const std::string & getSQL() const { return sql; } const QueryPlanPtr & getPlan() const { return plan; } + const QueryPlanPtr & getPlanBeforeCascades() const { return plan_before_cascades; } const PlanCostMap & getCosts() const { return costs; } /* diff --git a/src/Advisor/WorkloadTable.h b/src/Advisor/WorkloadTable.h index 61137442608..13f306afbe3 100644 --- a/src/Advisor/WorkloadTable.h +++ b/src/Advisor/WorkloadTable.h @@ -48,6 +48,8 @@ class WorkloadTable { } + StoragePtr getTablePtr() const { return storage; } + ASTPtr getDDL() const { return create_table_ddl; } bool isOptimized() const { return optimized; } diff --git a/src/Advisor/WorkloadTableStats.h b/src/Advisor/WorkloadTableStats.h index 52baba82480..e02ccdd8f1e 100644 --- a/src/Advisor/WorkloadTableStats.h +++ b/src/Advisor/WorkloadTableStats.h @@ -37,8 +37,7 @@ class WorkloadTableStats const String & table_name, const NamesAndTypesList & columns); - PlanNodeStatisticsPtr basic_stats; - WorkloadExtendedStatsPtr extended_stats; + PlanNodeStatisticsPtr getBasicStats() { return basic_stats; } private: explicit WorkloadTableStats(PlanNodeStatisticsPtr basic_stats_) @@ -47,6 +46,9 @@ class WorkloadTableStats { } + PlanNodeStatisticsPtr basic_stats; + WorkloadExtendedStatsPtr extended_stats; + static const char * getStatsAggregation(const WorkloadExtendedStatsType & type) { switch (type) diff --git a/src/Advisor/tests/gtest_column_usage.cpp b/src/Advisor/tests/gtest_column_usage.cpp index e900eb31ca9..04b21868c9e 100644 --- a/src/Advisor/tests/gtest_column_usage.cpp +++ b/src/Advisor/tests/gtest_column_usage.cpp @@ -19,14 +19,16 @@ class ColumnUsageTest : public ::testing::Test " deptno UInt32 not null," " name Nullable(String)," " salary Nullable(Float64)," - " commission Nullable(UInt32)" - ") ENGINE=CnchMergeTree() order by empid;"); + " commission Nullable(UInt32)," + " history Array(UInt32)" + ") ENGINE=Memory();"); tester->execute("CREATE TABLE IF NOT EXISTS depts(" " deptno UInt32 not null," " name Nullable(String)" - ") ENGINE=CnchMergeTree() order by deptno;"); + ") ENGINE=Memory();"); } + static void TearDownTestCase() { tester.reset(); } ColumnUsages buildColumnUsagesFromSQL(std::initializer_list sql_list) { @@ -72,7 +74,7 @@ TEST_F(ColumnUsageTest, testSelect) "select empid from emps"}); auto select_usages = getColumnFrequencies(column_usages, ColumnUsageType::SCANNED, true); auto empid_column = QualifiedColumnName{tester->getDatabaseName(), "emps", "empid"}; - EXPECT_EQ(select_usages.size(), 5); + EXPECT_EQ(select_usages.size(), 6); ASSERT_TRUE(select_usages.contains(empid_column)); EXPECT_EQ(select_usages[empid_column], 2); } @@ -106,9 +108,9 @@ TEST_F(ColumnUsageTest, testNestedJoin) TEST_F(ColumnUsageTest, testNestedJoinCountAll) { - tester->execute("CREATE TABLE IF NOT EXISTS A(a UInt32 not null, b UInt32 not null) ENGINE=CnchMergeTree() order by tuple();"); - tester->execute("CREATE TABLE IF NOT EXISTS B(b UInt32 not null, c UInt32 not null) ENGINE=CnchMergeTree() order by tuple();"); - tester->execute("CREATE TABLE IF NOT EXISTS C(c UInt32 not null, d UInt32 not null) ENGINE=CnchMergeTree() order by tuple();"); + tester->execute("CREATE TABLE IF NOT EXISTS A(a UInt32 not null, b UInt32 not null) ENGINE=Memory();"); + tester->execute("CREATE TABLE IF NOT EXISTS B(b UInt32 not null, c UInt32 not null) ENGINE=Memory();"); + tester->execute("CREATE TABLE IF NOT EXISTS C(c UInt32 not null, d UInt32 not null) ENGINE=Memory();"); auto column_usages = buildColumnUsagesFromSQL({"select * from A, B, C where A.b = B.b and B.c = C.c"}); @@ -161,4 +163,27 @@ TEST_F(ColumnUsageTest, testInFilter) EXPECT_EQ(in_usages[empid_column], 1); } +TEST_F(ColumnUsageTest, testArraySetFnction) +{ + auto column_usages = buildColumnUsagesFromSQL({"select if(arraySetCheck(history, (9000)), 'hint', 'miss') from emps " + "where has(history, 9000) and arraySetCheck(history, (9000)) = 1"}); + auto history_column = QualifiedColumnName{tester->getDatabaseName(), "emps", "history"}; + + auto array_set_usages = getColumnFrequencies(column_usages, ColumnUsageType::ARRAY_SET_FUNCTION, true); + ASSERT_TRUE(array_set_usages.contains(history_column)); + EXPECT_GE(array_set_usages[history_column], 2); +} + +TEST_F(ColumnUsageTest, testPrewhere) +{ + auto column_usages = buildColumnUsagesFromSQL({"select empid from emps " + "prewhere arraySetCheck(history, (9000)) = 1 where empid in (1,2,3)"}); + auto history_column = QualifiedColumnName{tester->getDatabaseName(), "emps", "history"}; + + auto array_set_usages = getColumnFrequencies(column_usages, ColumnUsageType::ARRAY_SET_FUNCTION, true); + ASSERT_TRUE(array_set_usages.contains(history_column)); + EXPECT_GE(array_set_usages[history_column], 1); +} + + } // namespace DB diff --git a/src/Advisor/tests/gtest_cluster_key.cpp b/src/Advisor/tests/gtest_order_by_key.cpp similarity index 73% rename from src/Advisor/tests/gtest_cluster_key.cpp rename to src/Advisor/tests/gtest_order_by_key.cpp index 228fe81c40b..bada3439f80 100644 --- a/src/Advisor/tests/gtest_cluster_key.cpp +++ b/src/Advisor/tests/gtest_order_by_key.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -11,7 +11,7 @@ using namespace DB; -class ClusterKeyTest : public ::testing::Test +class OrderByKeyTest : public ::testing::Test { public: static void SetUpTestSuite() @@ -23,24 +23,24 @@ class ClusterKeyTest : public ::testing::Test " name Nullable(String)," " salary Nullable(Float64)," " commission Nullable(UInt32)" - ") ENGINE=CnchMergeTree() order by empid;"); + ") ENGINE=Memory();"); tester->execute("CREATE TABLE IF NOT EXISTS depts(" " deptno UInt32 not null," " name Nullable(String)" - ") ENGINE=CnchMergeTree() order by deptno;"); + ") ENGINE=Memory();"); } static void TearDownTestCase() { tester.reset(); } - + static std::shared_ptr tester; }; -std::shared_ptr ClusterKeyTest::tester; +std::shared_ptr OrderByKeyTest::tester; -TEST_F(ClusterKeyTest, testSimple) +TEST_F(OrderByKeyTest, testSimple) { auto context = tester->createQueryContext(); std::vector sqls( @@ -49,17 +49,17 @@ TEST_F(ClusterKeyTest, testSimple) WorkloadQueries queries = WorkloadQuery::build(sqls, context, query_thread_pool); WorkloadTables tables(context); AdvisorContext advisor_context = AdvisorContext::buildFrom(context, tables, queries, query_thread_pool); - auto advise = ClusterKeyAdvisor().analyze(advisor_context); + auto advise = OrderByKeyAdvisor().analyze(advisor_context); EXPECT_EQ(advise.size(), 1); QualifiedTableName emps{tester->getDatabaseName(), "emps"}; EXPECT_EQ(advise[0]->getTable(), emps); EXPECT_EQ(advise[0]->getOptimizedValue(), "empid"); } -TEST_F(ClusterKeyTest, testUpdateOrderBy) +TEST_F(OrderByKeyTest, testUpdateOrderBy) { std::string database = tester->getDatabaseName(); - std::string create_table_ddl = "CREATE TABLE IF NOT EXISTS " + database + std::string table_ddl = "CREATE TABLE IF NOT EXISTS " + database + ".emps(" " empid UInt32 not null," " deptno UInt32 not null," @@ -70,13 +70,14 @@ TEST_F(ClusterKeyTest, testUpdateOrderBy) "order by deptno;"; auto query_context = tester->createQueryContext(); - auto create_ast = tester->parse(create_table_ddl, query_context); + query_context->applySettingsChanges({DB::SettingChange("dialect_type", "CLICKHOUSE")}); + auto create_ast = tester->parse(table_ddl, query_context); WorkloadTable table(nullptr, create_ast, WorkloadTableStats::build(query_context, tester->getDatabaseName(), "emps")); table.updateOrderBy(std::make_shared("empid")); - std::string optimal_ddl = serializeAST(*table.getDDL()); - std::cout << optimal_ddl << std::endl; - EXPECT_TRUE(optimal_ddl.find("ORDER BY deptno") == std::string::npos); - EXPECT_TRUE(optimal_ddl.find("ORDER BY empid") != std::string::npos); + std::string local_ddl = serializeAST(*table.getDDL()); + std::cout << local_ddl << std::endl; + EXPECT_TRUE(local_ddl.find("ORDER BY deptno") == std::string::npos); + EXPECT_TRUE(local_ddl.find("ORDER BY empid") != std::string::npos); } diff --git a/src/AggregateFunctions/AggregateBitmapExpressionCommon.h b/src/AggregateFunctions/AggregateBitmapExpressionCommon.h index d3e5a34195a..2b51f1ca21f 100644 --- a/src/AggregateFunctions/AggregateBitmapExpressionCommon.h +++ b/src/AggregateFunctions/AggregateBitmapExpressionCommon.h @@ -18,12 +18,16 @@ #include #include +#include #include #include #include +#include #include +#include +#include namespace DB @@ -33,16 +37,140 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int BAD_ARGUMENTS; } -template || std::is_same_v > > -struct AggregateFunctionBitMapData +#define AGGREGATE_INTEGRAL_BITMAP_KEY_INTERNAL_START (std::numeric_limits::min()+10000) +#define AGGREGATE_STRING_BITMAP_KEY_INTERNAL_PREFIX "BITMAP*AGG*KEY" + +template +inline T trans_global_index_to_bitmap_key(int arg) +{ + return AGGREGATE_INTEGRAL_BITMAP_KEY_INTERNAL_START+arg; +} + +template<> +inline String trans_global_index_to_bitmap_key(int arg) +{ + return AGGREGATE_STRING_BITMAP_KEY_INTERNAL_PREFIX+toString(arg); +} + +template +inline String trans_global_index_to_bitmap_string_key(int arg) +{ + return toString(AGGREGATE_INTEGRAL_BITMAP_KEY_INTERNAL_START+arg); +} + +template<> +inline String trans_global_index_to_bitmap_string_key(int arg) +{ + return AGGREGATE_STRING_BITMAP_KEY_INTERNAL_PREFIX+toString(arg); +} + +template +inline bool check_is_internal_bitmap_key(T key) +{ + return (static_cast(key) < AGGREGATE_INTEGRAL_BITMAP_KEY_INTERNAL_START); +} + +template<> +inline bool check_is_internal_bitmap_key(String key) +{ + return 0 == key.compare(0, strlen(AGGREGATE_STRING_BITMAP_KEY_INTERNAL_PREFIX), AGGREGATE_STRING_BITMAP_KEY_INTERNAL_PREFIX); +} + +inline bool existBitengineExpressionKeyword(const String& str) +{ + if (str.empty()) + return false; + + for (const char & keyword : BITENGINE_EXPRESSION_KEYWORDS) + { + if (str.find(keyword) != std::string::npos) + { + return true; + } + } + + if (!str.empty() && str[0] == BITENGINE_SPECIAL_KEYWORD) + { + throw Exception("Input tag starts with '_' is not allowd. your tag: " + str + , ErrorCodes::BAD_ARGUMENTS); + } + + return false; +} + +template +void checkIntegerExpression(const String & expression) +{ + auto is_integer = [&]() { + try + { + std::size_t pos; + [[maybe_unused]] UInt64 result = std::stoull(expression, &pos); + + return pos == expression.size(); + } + catch (...) + { + return false; + } + }; + + if constexpr (std::is_integral_v) + { + if (is_integer()) + { + return; + } + + if (expression.find('-') != expression.npos) + { + throw Exception( + "The tag (or bitmap key): " + expression + " has character '-', " + + "you mean to calculate the difference set? If so, you should use '~', but not '-'.", + ErrorCodes::BAD_ARGUMENTS); + } + else if (expression.find("～") != expression.npos) + { + throw Exception( + "The tag (or bitmap key): " + expression + " has Chinese character '～', " + + "you mean to calculate the difference set? If so, you should use English character '~'.", + ErrorCodes::BAD_ARGUMENTS); + } + else if (expression.find("，") != expression.npos) + { + throw Exception( + "The tag (or bitmap key): " + expression + " has Chinese character '，', " + + "you mean to calculate the union set? If so, you should use English character ','.", + ErrorCodes::BAD_ARGUMENTS); + } + else if (expression.find("｜") != expression.npos) + { + throw Exception( + "The tag (or bitmap key): " + expression + " has Chinese character '｜', " + + "you mean to calculate the union set? If so, you should use English character '|'.", + ErrorCodes::BAD_ARGUMENTS); + } + else + { + throw Exception( + "The tag (or bitmap key): " + expression + " is illegal, " + "and it should be an integer.", ErrorCodes::BAD_ARGUMENTS); + } + } +} + +template || std::is_same_v) && (std::is_same_v || std::is_same_v) > > +struct AggregateFunctionBitMapDataImpl { - std::unordered_map bitmap_map; + std::unordered_map bitmap_map; bool is_finished = false; - AggregateFunctionBitMapData() = default; + AggregateFunctionBitMapDataImpl() = default; - void add(const T key, const BitMap64 & bitmap) + void add(const T key, const BitmapType & bitmap) { auto it = bitmap_map.find(key); if (it == bitmap_map.end()) { @@ -52,9 +180,9 @@ struct AggregateFunctionBitMapData } } - void merge(AggregateFunctionBitMapData && rhs) + void merge(AggregateFunctionBitMapDataImpl && rhs) { - std::unordered_map & rhs_map = rhs.bitmap_map; + std::unordered_map & rhs_map = rhs.bitmap_map; for (auto it = rhs_map.begin(); it != rhs_map.end(); ++it) { auto jt = bitmap_map.find(it->first); @@ -68,9 +196,9 @@ struct AggregateFunctionBitMapData is_finished = false; } - bool empty() { return bitmap_map.empty(); } + bool empty() const { return bitmap_map.empty(); } - UInt64 getCardinality(const T key) + UInt64 getCardinality(const T & key) { auto it = bitmap_map.find(key); if (it != bitmap_map.end()) @@ -108,7 +236,7 @@ struct AggregateFunctionBitMapData readVarUInt(bytes_size, buf); PODArray buffer(bytes_size); buf.readStrict(buffer.data(), bytes_size); - BitMap64 bitmap = BitMap64::readSafe(buffer.data(), bytes_size); + BitmapType bitmap = BitmapType::readSafe(buffer.data(), bytes_size); bitmap_map.emplace(key, std::move(bitmap)); } @@ -116,6 +244,11 @@ struct AggregateFunctionBitMapData } }; + +template || std::is_same_v > > +using AggregateFunctionBitMapData = AggregateFunctionBitMapDataImpl; + + template || std::is_same_v > > struct BitMapExpressionNode { @@ -130,7 +263,7 @@ struct BitMapExpressionNode BitMapExpressionNode(String && left_, String && op_, String && right_, const T res_, bool replicated_ = false) : left(std::move(left_)), op(std::move(op_)), right(std::move(right_)), res(res_), replicated(replicated_) {} - String toString() + String toString() const { std::ostringstream oss; oss << left << " " << op << " " << right << " = " << res << " REPLICATED: " << replicated << "\n"; @@ -138,8 +271,10 @@ struct BitMapExpressionNode } }; -template > > -struct BitMapExpressionAnalyzer +template || std::is_same_v) && (std::is_same_v || std::is_same_v) > > +struct BitMapExpressionAnalyzerImpl { using BitMapExpressions = std::vector>; String original_expression; @@ -149,14 +284,14 @@ struct BitMapExpressionAnalyzer bool only_or = true; NameSet or_expressions; - BitMapExpressionAnalyzer(const String & expression) + explicit BitMapExpressionAnalyzerImpl(const String & expression) : original_expression(expression) { analyze(); } - BitMapExpressionAnalyzer() = default; + BitMapExpressionAnalyzerImpl() = default; - void subExpression(std::stack & expression_stack, T & global_index, String & right) + void subExpression(std::stack & expression_stack, int & global_index, String & right) { while (!expression_stack.empty() && (expression_stack.top() == "&" || expression_stack.top() == "|" @@ -172,19 +307,19 @@ struct BitMapExpressionAnalyzer String operation = expression_stack.top(); expression_stack.pop(); if (expression_stack.empty()) - throw Exception("Invalid expression " + operation + " for BitMap: " + original_expression, ErrorCodes::LOGICAL_ERROR); + throw Exception("Invalid expression " + operation + " for BitMap: " + original_expression, ErrorCodes::BAD_ARGUMENTS); String left = expression_stack.top(); expression_stack.pop(); // Optimize the case which right is equal to left. // If the operation is "~", add a no-exists result to expression_stack so that we can get an empty bitmap if (right == left && operation == "~") { - T res = global_index--; - right = toString(res); + int res = global_index--; + right = trans_global_index_to_bitmap_string_key(res); } else if (right != left) { - T res = global_index--; + int res = global_index--; bool replicated = false; auto left_it = replicated_keys.find(left); auto right_it = replicated_keys.find(right); @@ -198,8 +333,9 @@ struct BitMapExpressionAnalyzer if (right_it->second > 1) replicated = true; } - expression_actions.emplace_back(std::move(left), std::move(operation), std::move(right), res, replicated); - right = toString(res); + expression_actions.emplace_back( + std::move(left), std::move(operation), std::move(right), trans_global_index_to_bitmap_key(res), replicated); + right = trans_global_index_to_bitmap_string_key(res); } } } @@ -217,6 +353,10 @@ struct BitMapExpressionAnalyzer || expression[i] == ')' || expression[i] == '#' || expression[i] == '~' || expression[i] == ' ') { if (number_index != expression_size) { String number = expression.substr(number_index, (i - number_index)); + if constexpr (std::is_integral_v) + { + checkIntegerExpression(number); + } replicated_keys[number] += 1; or_expressions.insert(number); expression_vector.push_back(std::move(number)); @@ -260,35 +400,35 @@ struct BitMapExpressionAnalyzer if (only_or) { - final_key = -1; + final_key = trans_global_index_to_bitmap_key(-1); return; } - T global_index = -1; + int global_index = -1; - for (size_t i = 0; i < expression_vector.size(); i++) + for (const auto & i : expression_vector) { - if (expression_vector[i] == "(" || expression_vector[i] == "&" - || expression_vector[i] == "|" || expression_vector[i] == "," - || expression_vector[i] == "~") + if (i == "(" || i == "&" + || i == "|" || i == "," + || i == "~") { - expression_stack.push(expression_vector[i]); + expression_stack.push(i); } - else if (expression_vector[i] == ")") + else if (i == ")") { if (expression_stack.empty()) - throw Exception("Invalid expression " + expression_vector[i] + " for BitMap: " + original_expression, ErrorCodes::LOGICAL_ERROR); + throw Exception("Invalid expression " + i + " for BitMap: " + original_expression, ErrorCodes::BAD_ARGUMENTS); String number = expression_stack.top(); expression_stack.pop(); if (expression_stack.empty()) - throw Exception("Invalid expression " + number + " for BitMap: " + original_expression, ErrorCodes::LOGICAL_ERROR); + throw Exception("Invalid expression " + number + " for BitMap: " + original_expression, ErrorCodes::BAD_ARGUMENTS); expression_stack.pop(); subExpression(expression_stack, global_index, number); expression_stack.push(number); } else { - String right = expression_vector[i]; + String right = i; // If there are replicated number, we cannot use some optimization strategy to execute expression subExpression(expression_stack, global_index, right); expression_stack.push(right); @@ -300,11 +440,11 @@ struct BitMapExpressionAnalyzer std::istringstream iss(res); iss >> final_key; } else { - throw Exception("Invalid expression for BitMap: " + original_expression, ErrorCodes::LOGICAL_ERROR); + throw Exception("Invalid expression for BitMap: " + original_expression, ErrorCodes::BAD_ARGUMENTS); } } - void executeExpressionImpl(String left, String operation, String right, T res, bool replicated, const AggregateFunctionBitMapData& data) const + void executeExpressionImpl(String left, String operation, String right, T res, bool replicated, const AggregateFunctionBitMapDataImpl& data) const { std::istringstream iss(left); T left_key; @@ -313,12 +453,12 @@ struct BitMapExpressionAnalyzer T right_key; iss2 >> right_key; - auto& bitmap_map = const_cast&>(data.bitmap_map); + auto& bitmap_map = const_cast&>(data.bitmap_map); auto left_iter = bitmap_map.find(left_key); auto right_iter = bitmap_map.find(right_key); if (left_iter == bitmap_map.end()) { - BitMap64 temp_bitmap; + BitmapType temp_bitmap; auto res_pair = bitmap_map.emplace(left_key, std::move(temp_bitmap)); if (res_pair.second) left_iter = res_pair.first; @@ -326,7 +466,7 @@ struct BitMapExpressionAnalyzer throw Exception("Existing empty BitMap64 when inserting empty BitMap64", ErrorCodes::LOGICAL_ERROR); } if (right_iter == bitmap_map.end()) { - BitMap64 temp_bitmap; + BitmapType temp_bitmap; auto res_pair = bitmap_map.emplace(right_key, std::move(temp_bitmap)); if (res_pair.second) right_iter = res_pair.first; @@ -371,7 +511,7 @@ struct BitMapExpressionAnalyzer } } - void executeExpressionOnlyOr(const AggregateFunctionBitMapData & data) const + void executeExpressionOnlyOr(const AggregateFunctionBitMapDataImpl & data) const { std::set key_set; for (const auto & expression : or_expressions) @@ -382,14 +522,14 @@ struct BitMapExpressionAnalyzer key_set.insert(key); } - auto& bitmap_map = const_cast&>(data.bitmap_map); + auto& bitmap_map = const_cast&>(data.bitmap_map); if (key_set.size() == 1) { T key = *key_set.begin(); auto it = bitmap_map.find(key); if (it == bitmap_map.end()) { - BitMap64 temp_bitmap; + BitmapType temp_bitmap; auto res_pair = bitmap_map.emplace(key, std::move(temp_bitmap)); if (res_pair.second) it = res_pair.first; @@ -402,34 +542,50 @@ struct BitMapExpressionAnalyzer return; } - std::map> roaring_map; - for (const auto & key: key_set) + if constexpr(std::is_same_v) { - auto it = bitmap_map.find(key); - if (it == bitmap_map.end()) - continue; - std::map & inner_roaring = const_cast &>(it->second.getRoarings()); - for (auto jt = inner_roaring.begin(); jt != inner_roaring.end(); ++jt) + std::map> roaring_map; + for (const auto & key: key_set) { - if (roaring_map.find(jt->first) == roaring_map.end()) - roaring_map.emplace(jt->first, std::vector()); - roaring_map[jt->first].emplace_back(&jt->second); + auto it = bitmap_map.find(key); + if (it == bitmap_map.end()) + continue; + std::map & inner_roaring = const_cast &>(it->second.getRoarings()); + for (auto & jt : inner_roaring) + { + if (roaring_map.find(jt.first) == roaring_map.end()) + roaring_map.emplace(jt.first, std::vector()); + roaring_map[jt.first].emplace_back(&jt.second); + } } - } - BitMap64 res_roaring; + BitMap64 res_roaring; - for (auto it = roaring_map.begin(); it != roaring_map.end(); ++it) + for (auto & it : roaring_map) + { + roaring::Roaring result = roaring::Roaring::fastunion(it.second.size(), static_cast(&(*(it.second.begin())))); + const_cast &>(res_roaring.getRoarings()).emplace(it.first, std::move(result)); + } + + bitmap_map[final_key] = std::move(res_roaring); + } + else if constexpr (std::is_same_v) { - roaring::Roaring result - = roaring::Roaring::fastunion(it->second.size(), static_cast(&(*(it->second.begin())))); - const_cast &>(res_roaring.getRoarings()).emplace(it->first, std::move(result)); + std::vector vec; + for (const auto & key: key_set) + { + auto it = bitmap_map.find(key); + if (it == bitmap_map.end()) + continue; + vec.emplace_back(reinterpret_cast(&(it->second))); + } + roaring::Roaring res_roaring = roaring::Roaring::fastunion(vec.size(), &(*(vec.begin()))); + bitmap_map[final_key].loadBitmap(std::move(res_roaring)); } - bitmap_map[final_key] = std::move(res_roaring); } - void executeExpression(const AggregateFunctionBitMapData & data) const + void executeExpression(const AggregateFunctionBitMapDataImpl & data) const { if (only_or) { @@ -444,20 +600,23 @@ struct BitMapExpressionAnalyzer } } }; +template || std::is_same_v > > +using BitMapExpressionAnalyzer = BitMapExpressionAnalyzerImpl; + template || std::is_same_v > > struct BitMapExpressionMultiAnalyzer { using BitMapExpressions = std::vector>; std::vector original_expressions; - T global_index = -1; + int global_index = -1; std::vector final_keys; std::vector expression_only_ors; std::vector expression_actions_vector; std::unordered_map replicated_keys; std::vector or_expressions; - BitMapExpressionMultiAnalyzer(const std::vector & expressions) + explicit BitMapExpressionMultiAnalyzer(const std::vector & expressions) : original_expressions(expressions) { analyze(); @@ -481,20 +640,19 @@ struct BitMapExpressionMultiAnalyzer String operation = expression_stack.top(); expression_stack.pop(); if (expression_stack.empty()) - throw Exception("Invalid expression " + operation + " for BitMap: " + original_expressions[index], ErrorCodes::LOGICAL_ERROR); + throw Exception("Invalid expression " + operation + " for BitMap: " + original_expressions[index], ErrorCodes::BAD_ARGUMENTS); String left = expression_stack.top(); expression_stack.pop(); // Optimize the case which right is equal to left. // If the operation is "~", add a no-exists result to expression_stack so that we can get an empty bitmap if (right == left && operation == "~") { - T res = global_index--; + int res = global_index--; right = toString(res); } else if (right != left) { - T res = global_index--; - + T res = trans_global_index_to_bitmap_key(global_index--); expression_actions_vector[index].emplace_back(std::move(left), std::move(operation), std::move(right), res, false); right = toString(res); } @@ -566,45 +724,60 @@ struct BitMapExpressionMultiAnalyzer // replace number with final key if (number.size() > 1 && number[0] == '_') { - auto res_index = std::stoi(number.substr(1)); - if (res_index <= 0 || res_index > static_cast(index)) + size_t res_index{0}; + + try { - throw Exception("Invalid expression " + number + " for BitMap: " + original_expression, ErrorCodes::LOGICAL_ERROR); + res_index = std::stoi(number.substr(1)); + } + catch (std::exception &e) + { + throw Exception("Bad cast number to position: " + number + ", reason: " + String(e.what()), + ErrorCodes::BAD_ARGUMENTS); + } + + if (res_index <= 0 || res_index > index) + { + throw Exception("Invalid expression " + number + " for BitMap: " + original_expression, ErrorCodes::BAD_ARGUMENTS); } number = toString(final_keys[res_index - 1]); } + if constexpr (std::is_integral_v) + { + checkIntegerExpression(number); + } or_expression.insert(number); - expression_vector.push_back(std::move(number)); + expression_vector.emplace_back(std::move(number)); number_index = expression_size; } switch (expression[i]) { case '(': - expression_vector.push_back("("); + expression_vector.emplace_back("("); break; case '&': { - expression_vector.push_back("&"); + expression_vector.emplace_back("&"); only_or = false; break; } case '|': - expression_vector.push_back("|"); + expression_vector.emplace_back("|"); break; case ')': - expression_vector.push_back(")"); + expression_vector.emplace_back(")"); break; case ',': - expression_vector.push_back(","); + expression_vector.emplace_back(","); break; case '~': { - expression_vector.push_back("~"); + expression_vector.emplace_back("~"); only_or = false; break; } case '#': - expression_vector.push_back("#"); + expression_vector.emplace_back("#"); break; } } else { @@ -619,33 +792,34 @@ struct BitMapExpressionMultiAnalyzer expression_only_ors.emplace_back(only_or); if (only_or) { - final_keys.emplace_back(global_index--); + T current_key = trans_global_index_to_bitmap_key(global_index--); + final_keys.emplace_back(current_key); return; } - for (size_t i = 0; i < expression_vector.size(); i++) + for (const auto & i : expression_vector) { - if (expression_vector[i] == "(" || expression_vector[i] == "&" - || expression_vector[i] == "|" || expression_vector[i] == "," - || expression_vector[i] == "~") + if (i == "(" || i == "&" + || i == "|" || i == "," + || i == "~") { - expression_stack.push(expression_vector[i]); + expression_stack.push(i); } - else if (expression_vector[i] == ")") + else if (i == ")") { if (expression_stack.empty()) - throw Exception("Invalid expression " + expression_vector[i] + " for BitMap: " + original_expression, ErrorCodes::LOGICAL_ERROR); + throw Exception("Invalid expression " + i + " for BitMap: " + original_expression, ErrorCodes::BAD_ARGUMENTS); String number = expression_stack.top(); expression_stack.pop(); if (expression_stack.empty()) - throw Exception("Invalid expression " + number + " for BitMap: " + original_expression, ErrorCodes::LOGICAL_ERROR); + throw Exception("Invalid expression " + number + " for BitMap: " + original_expression, ErrorCodes::BAD_ARGUMENTS); expression_stack.pop(); subExpression(expression_stack, number, index); expression_stack.push(number); } else { - String right = expression_vector[i]; + String right = i; // If there are replicated number, we cannot use some optimization strategy to execute expression subExpression(expression_stack, right, index); expression_stack.push(right); @@ -659,7 +833,7 @@ struct BitMapExpressionMultiAnalyzer iss >> temp_final_key; final_keys.emplace_back(temp_final_key); } else { - throw Exception("Invalid expression for BitMap: " + original_expression, ErrorCodes::LOGICAL_ERROR); + throw Exception("Invalid expression for BitMap: " + original_expression, ErrorCodes::BAD_ARGUMENTS); } } @@ -740,7 +914,7 @@ struct BitMapExpressionMultiAnalyzer iss >> key; if (toString(key) != expression) throw Exception("expression is not fully parsed! parsed: " + toString(key) + ", but your input: " + expression - + ", please check function name and expression type", ErrorCodes::LOGICAL_ERROR); + + ", please check function name and expression type", ErrorCodes::BAD_ARGUMENTS); key_set.insert(key); } @@ -774,28 +948,27 @@ struct BitMapExpressionMultiAnalyzer return; } - std::map> roaring_map; + std::map> roaring_map; for (const auto & key: key_set) { auto it = bitmap_map.find(key); if (it == bitmap_map.end()) continue; std::map & inner_roaring = const_cast &>(it->second.getRoarings()); - for (auto jt = inner_roaring.begin(); jt != inner_roaring.end(); ++jt) + for (auto & jt : inner_roaring) { - if (roaring_map.find(jt->first) == roaring_map.end()) - roaring_map.emplace(jt->first, std::vector()); - roaring_map[jt->first].emplace_back(&jt->second); + if (roaring_map.find(jt.first) == roaring_map.end()) + roaring_map.emplace(jt.first, std::vector()); + roaring_map[jt.first].emplace_back(&jt.second); } } BitMap64 res_roaring; - for (auto it = roaring_map.begin(); it != roaring_map.end(); ++it) + for (auto & it : roaring_map) { - roaring::Roaring result - = roaring::Roaring::fastunion(it->second.size(), static_cast(&(*(it->second.begin())))); - const_cast &>(res_roaring.getRoarings()).emplace(it->first, std::move(result)); + roaring::Roaring result = roaring::Roaring::fastunion(it.second.size(), static_cast(&(*(it.second.begin())))); + const_cast &>(res_roaring.getRoarings()).emplace(it.first, std::move(result)); } bitmap_map[final_keys[index]] = std::move(res_roaring); @@ -817,4 +990,405 @@ struct BitMapExpressionMultiAnalyzer } }; + +struct BitMapExpressionWithDateMultiAnalyzer +{ + using BitMapExpressions = std::vector>; + // original_expressions is a list extracted from AggFunction's first parameter. + // for SQL below: + // * Select BitmapMultiCountWithDateV2('conjunct1','conjunct2')(p_date, tag, uid) From ... * + // The original_expressions is ['conjunct1', 'conjunct2'] + std::vector original_expressions; + Int64 global_index = -1; + std::unordered_set keys_without_date; + std::vector final_keys; + std::vector expression_only_ors; + std::vector expression_actions_vector; + std::unordered_map replicated_keys; + std::vector or_expressions; + std::unordered_set interested_tokens; + + explicit BitMapExpressionWithDateMultiAnalyzer(const std::vector & expressions) + : original_expressions(expressions) + { + analyze(); + } + + BitMapExpressionWithDateMultiAnalyzer() = default; + + void subExpression(std::stack & expression_stack, String & right, size_t index) + { + while (!expression_stack.empty() && + (expression_stack.top() == "&" || expression_stack.top() == "|" + || expression_stack.top() == "," || expression_stack.top() == "~" || right == "#")) + { + if (right == "#") + { + right = expression_stack.top(); + expression_stack.pop(); + } + if (expression_stack.empty()) + break; + String operation = expression_stack.top(); + expression_stack.pop(); + if (expression_stack.empty()) + throw Exception("Invalid expression " + operation + " for BitMap: " + original_expressions[index], ErrorCodes::BAD_ARGUMENTS); + String left = expression_stack.top(); + expression_stack.pop(); + // Optimize the case which right is equal to left. + // If the operation is "~", add a no-exists result to expression_stack so that we can get an empty bitmap + if (right == left && operation == "~") + { + Int64 res = global_index--; + right = std::to_string(res); + } + else + { + Int64 res = global_index--; + + expression_actions_vector[index].emplace_back(std::move(left), std::move(operation), std::move(right), std::to_string(res), false); + right = std::to_string(res); + } + } + } + + void analyze() + { + for (size_t i = 0; i < original_expressions.size(); i++) + analyzeExpression(original_expressions[i], i); + for (auto & expression_actions: expression_actions_vector) + { + for (BitMapExpressionNode & expression_action: expression_actions) + { + replicated_keys[expression_action.left] += 1; + replicated_keys[expression_action.right] += 1; + } + } + for (const NameSet& or_expression: or_expressions) + { + for (const String& or_expression_item: or_expression) + { + replicated_keys[or_expression_item] += 1; + } + } + + for (auto & expression_actions: expression_actions_vector) + { + for (BitMapExpressionNode & expression_action: expression_actions) + { + auto left_it = replicated_keys.find(expression_action.left); + do { + if (left_it == replicated_keys.end()) + break; + if (left_it->second <= 1) + break; + + expression_action.replicated = true; + } while(false); + } + } + } + void analyzeExpression(String& original_expression, size_t index) + { + bool only_or = true; + NameSet or_expression; + String expression = original_expression + "#"; + std::stack expression_stack; + size_t expression_size = expression.size(); + std::vector expression_vector; + size_t number_index = expression_size; + for (size_t i = 0; i < expression_size; i++) + { + if (expression[i] == '(' || expression[i] == '&' || expression[i] == '|' || expression[i] == ',' + || expression[i] == ')' || expression[i] == '#' || expression[i] == '~' || expression[i] == ' ') { + if (number_index != expression_size) { + String number = expression.substr(number_index, (i - number_index)); + // check mistakable characters + if (number == "-") + { + throw Exception( + "The tag (or bitmap key): " + number + " has character '-', " + + "you mean to calculate the difference set? If so, you should use '~', but not '-'.", + ErrorCodes::BAD_ARGUMENTS); + } + else if (number.find("～") != number.npos) + { + throw Exception( + "The tag (or bitmap key): " + number + " has Chinese character '～', " + + "you mean to calculate the difference set? If so, you should use English character '~'.", + ErrorCodes::BAD_ARGUMENTS); + } + else if (number.find("，") != number.npos) + { + throw Exception( + "The tag (or bitmap key): " + number + " has Chinese character '，', " + + "you mean to calculate the union set? If so, you should use English character ','.", + ErrorCodes::BAD_ARGUMENTS); + } + else if (number.find("｜") != number.npos) + { + throw Exception( + "The tag (or bitmap key): " + number + " has Chinese character '｜', " + + "you mean to calculate the union set? If so, you should use English character '|'.", + ErrorCodes::BAD_ARGUMENTS); + } + // replace number with final key + if (number.size() > 1 && number[0] == '_') + { + size_t res_index{0}; + + try { + res_index = std::stoi(number.substr(1)); + } catch (std::exception &e) { + throw Exception("Bad cast number to position: " + number + ", reason: " + String(e.what()), + ErrorCodes::BAD_ARGUMENTS); + } + + if (res_index <= 0 || res_index > index) + { + throw Exception("Invalid expression " + number + " for BitMap: " + original_expression, ErrorCodes::BAD_ARGUMENTS); + } + number = final_keys[res_index - 1]; + } + else if (!number.empty()) { + if (number.find('_') == std::string::npos) + { + keys_without_date.emplace(number); + } + else + { + // The expression has '_' in it, So the expression can be a key_date pair. + if ('_' != number[0]){ + // We don't want those _1,_2,... pairs + interested_tokens.emplace(number); + } + } + } + + or_expression.insert(number); + expression_vector.push_back(std::move(number)); + number_index = expression_size; + } + switch (expression[i]) { + case '(': + expression_vector.push_back("("); + break; + case '&': + { + expression_vector.push_back("&"); + only_or = false; + break; + } + case '|': + expression_vector.push_back("|"); + break; + case ')': + expression_vector.push_back(")"); + break; + case ',': + expression_vector.push_back(","); + break; + case '~': + { + expression_vector.push_back("~"); + only_or = false; + break; + } + case '#': + expression_vector.push_back("#"); + break; + } + } else { + if (number_index == expression_size) { + number_index = i; + } + } + } + BitMapExpressions expressions; + expression_actions_vector.emplace_back(std::move(expressions)); + or_expressions.emplace_back(std::move(or_expression)); + expression_only_ors.emplace_back(only_or); + if (only_or) + { + final_keys.emplace_back(std::to_string(global_index--)); + return; + } + + for (const auto & i : expression_vector) + { + if (i == "(" || i == "&" + || i == "|" || i == "," + || i == "~") + { + expression_stack.push(i); + } + else if (i == ")") + { + if (expression_stack.empty()) + throw Exception("Invalid expression " + i + " for BitMap: " + original_expression, ErrorCodes::BAD_ARGUMENTS); + String number = expression_stack.top(); + expression_stack.pop(); + if (expression_stack.empty()) + throw Exception("Invalid expression " + number + " for BitMap: " + original_expression, ErrorCodes::BAD_ARGUMENTS); + expression_stack.pop(); + subExpression(expression_stack, number, index); + expression_stack.push(number); + } + else + { + String right = i; + // If there are replicated number, we cannot use some optimization strategy to execute expression + subExpression(expression_stack, right, index); + expression_stack.push(right); + } + } + + if (expression_stack.size() == 1) { + const String & res = expression_stack.top(); + final_keys.emplace_back(res); + } else { + throw Exception("Invalid expression for BitMap: " + original_expression, ErrorCodes::BAD_ARGUMENTS); + } + } + + static void executeExpressionImpl(String left_key, String operation, String right_key, String res, bool replicated, const AggregateFunctionBitMapData& data) + { + auto& bitmap_map = const_cast&>(data.bitmap_map); + auto left_iter = bitmap_map.find(left_key); + auto right_iter = bitmap_map.find(right_key); + + if (left_iter == bitmap_map.end()) { + BitMap64 temp_bitmap; + auto res_pair = bitmap_map.emplace(left_key, std::move(temp_bitmap)); + if (res_pair.second) + left_iter = res_pair.first; + else + throw Exception("Existing empty BitMap64 when inserting empty BitMap64", ErrorCodes::LOGICAL_ERROR); + } + if (right_iter == bitmap_map.end()) { + BitMap64 temp_bitmap; + auto res_pair = bitmap_map.emplace(right_key, std::move(temp_bitmap)); + if (res_pair.second) + right_iter = res_pair.first; + else + throw Exception("Existing empty BitMap64 when inserting empty BitMap64", ErrorCodes::LOGICAL_ERROR); + } + if (!replicated) + { + if (operation == "|" || operation == ",") { + left_iter->second |= right_iter->second; + auto left_item = bitmap_map.extract(left_iter->first); + left_item.key() = res; + bitmap_map.insert(std::move(left_item)); + } + else if (operation == "&") { + left_iter->second &= right_iter->second; + auto left_item = bitmap_map.extract(left_iter->first); + left_item.key() = res; + bitmap_map.insert(std::move(left_item)); + } + else if (operation == "~") { + left_iter->second -= right_iter->second; + auto left_item = bitmap_map.extract(left_iter->first); + left_item.key() = res; + bitmap_map.insert(std::move(left_item)); + } + } + else + { + if (operation == "|" || operation == ",") { + bitmap_map[res] = left_iter->second; + bitmap_map[res] |= right_iter->second; + } + else if (operation == "&") { + bitmap_map[res] = left_iter->second; + bitmap_map[res] &= right_iter->second; + } + else if (operation == "~") { + bitmap_map[res] = left_iter->second; + bitmap_map[res] -= right_iter->second; + } + } + } + + void executeExpressionOnlyOr(const AggregateFunctionBitMapData & data, size_t index) const + { + std::set key_set; + for (const auto & expression : or_expressions[index]) + { + key_set.insert(expression); + } + + auto& bitmap_map = const_cast&>(data.bitmap_map); + + if (key_set.size() == 1) + { + String key = *key_set.begin(); + auto it = bitmap_map.find(key); + if (it == bitmap_map.end()) { + BitMap64 temp_bitmap; + auto res_pair = bitmap_map.emplace(key, std::move(temp_bitmap)); + if (res_pair.second) + it = res_pair.first; + else + throw Exception("Existing empty BitMap64 when inserting empty BitMap64", ErrorCodes::LOGICAL_ERROR); + } + auto or_it = replicated_keys.find(key); + if (or_it == replicated_keys.end()) + return; + else if (or_it->second > 1) + { + bitmap_map[final_keys[index]] = it->second; + } + else + { + auto it_final_item = bitmap_map.extract(it->first); + it_final_item.key() = final_keys[index]; + bitmap_map.insert(std::move(it_final_item)); + } + return; + } + + std::map> roaring_map; + for (const auto & key: key_set) + { + auto it = bitmap_map.find(key); + if (it == bitmap_map.end()) + continue; + std::map & inner_roaring = const_cast &>(it->second.getRoarings()); + for (auto & jt : inner_roaring) + { + if (roaring_map.find(jt.first) == roaring_map.end()) + roaring_map.emplace(jt.first, std::vector()); + roaring_map[jt.first].emplace_back(&jt.second); + } + } + + BitMap64 res_roaring; + + for (auto & it : roaring_map) + { + roaring::Roaring result = roaring::Roaring::fastunion(it.second.size(), &(*(it.second.begin()))); + const_cast &>(res_roaring.getRoarings()).emplace(it.first, std::move(result)); + } + + bitmap_map[final_keys[index]] = std::move(res_roaring); + } + + void executeExpression(const AggregateFunctionBitMapData & data, size_t index) const + { + if (expression_only_ors[index]) + { + executeExpressionOnlyOr(data, index); + } + else + { + for (const auto & action : expression_actions_vector[index]) + { + executeExpressionImpl(action.left, action.op, action.right, action.res, action.replicated, data); + } + } + } +}; + } diff --git a/src/AggregateFunctions/AggregateFunctionBitMapJoin.cpp b/src/AggregateFunctions/AggregateFunctionBitMapJoin.cpp index 698fbfdca2e..fb06d214a1d 100644 --- a/src/AggregateFunctions/AggregateFunctionBitMapJoin.cpp +++ b/src/AggregateFunctions/AggregateFunctionBitMapJoin.cpp @@ -22,6 +22,12 @@ namespace DB { + namespace ErrorCodes + { + extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + } + /// Expected format is 'P.I' or 'I', and P means the position, /// as well as the I means the index of argument PositionIndexPair parsePositionAndIndex(String & input) @@ -52,15 +58,15 @@ namespace DB for (size_t i = 0; i < arr.size(); ++i) { if (arr.at(i).safeGet().empty()) - throw Exception("AggregateFunction " + name + ": empty string in parameter is invalid", ErrorCodes::LOGICAL_ERROR); + throw Exception("AggregateFunction " + name + ": empty string in parameter is invalid", ErrorCodes::BAD_ARGUMENTS); UInt64 pos = 0, idx = 0; std::tie(pos, idx) = parsePositionAndIndex(arr.at(i).safeGet()); if (pos == 0 || ((pos^0xFF) && pos > union_num+1)) { - throw Exception("AggregateFunction " + name + ": wrong value of keys postion identifier, which starts from 1", ErrorCodes::LOGICAL_ERROR); + throw Exception("AggregateFunction " + name + ": wrong value of keys postion identifier, which starts from 1", ErrorCodes::BAD_ARGUMENTS); } if (idx < 3 || idx > argument_num) - throw Exception("AggregateFunction " + name + ": wrong value of key index, which starts from 3", ErrorCodes::LOGICAL_ERROR); + throw Exception("AggregateFunction " + name + ": wrong value of key index, which starts from 3", ErrorCodes::BAD_ARGUMENTS); to.emplace_back(pos, idx); } } @@ -70,7 +76,7 @@ namespace DB { UInt64 idx = arr.at(i).safeGet(); if (idx < 3 || idx > argument_num) - throw Exception("AggregateFunction " + name + ": wrong value of key index", ErrorCodes::LOGICAL_ERROR); + throw Exception("AggregateFunction " + name + ": wrong value of key index", ErrorCodes::BAD_ARGUMENTS); to.emplace_back(0xFF, idx); } } @@ -93,11 +99,11 @@ namespace /// 6 params are: (union_num, [join_keys], [group_by_keys], bitmap_op, join_type, thread_number, 0), the last 0 mean result is cardinality /// 7 params are: (union_num, [join_keys], [group_by_keys], bitmap_op, join_type, thread_number, result_type) result_type: 0->cardinality, 1->raw bitmap if (parameters.size() != 3 && parameters.size() != 5 && parameters.size() != 6 && parameters.size() != 7) - throw Exception("AggregateFunction " + name + " needs 3, 5, 6 or 7 parameters", ErrorCodes::LOGICAL_ERROR); + throw Exception("AggregateFunction " + name + " needs 3, 5, 6 or 7 parameters", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); UInt64 union_num = parameters[0].safeGet(); if (union_num != 1) - throw Exception("AggregateFunction " + name + " can only support one JOIN now, set 1 please", ErrorCodes::LOGICAL_ERROR); + throw Exception("AggregateFunction " + name + " can only support one JOIN now, set 1 please", ErrorCodes::BAD_ARGUMENTS); Array join_arr = parameters[1].safeGet(); Array group_by_arr = parameters[2].safeGet(); @@ -110,7 +116,7 @@ namespace keys_set.emplace(jk.second); } if (keys_set.size() != join_keys_idx.size()) - throw Exception("AggregateFunction " + name + ": duplicated join key index, only one is ok", ErrorCodes::LOGICAL_ERROR); + throw Exception("AggregateFunction " + name + ": duplicated join key index, only one is ok", ErrorCodes::BAD_ARGUMENTS); getParameterOfPositionAndIndex(group_by_arr, name, union_num, argument_types.size(), group_by_keys_idx); @@ -125,12 +131,12 @@ namespace if (group_by_keys_idx[i] == group_by_keys_idx[j] || (group_by_keys_idx[i].second == group_by_keys_idx[j].second && (group_by_keys_idx[i].first == 0xFF || group_by_keys_idx[j].first == 0xFF))) - throw Exception("AggregateFunction " + name + ": duplicated group by index", ErrorCodes::LOGICAL_ERROR); + throw Exception("AggregateFunction " + name + ": duplicated group by index", ErrorCodes::BAD_ARGUMENTS); } } String logic_str, join_str; - if (parameters.size() == 5 || parameters.size() == 6) + if (parameters.size() >= 5) { logic_str = parameters[3].safeGet(); join_str = parameters[4].safeGet(); @@ -140,16 +146,16 @@ namespace if (!logic_op.isValid()) throw Exception( "AggregateFunction " + name + " only support logic operation: AND, OR, XOR, besides empty string is also ok", - DB::ErrorCodes::LOGICAL_ERROR); + DB::ErrorCodes::BAD_ARGUMENTS); JoinOperation join_op(join_str); if (!join_op.isValid()) throw Exception( "AggregateFunction " + name + " only support join type: INNER, LEFT. And empty string means INNER JOIN", - DB::ErrorCodes::LOGICAL_ERROR); + DB::ErrorCodes::BAD_ARGUMENTS); UInt64 thread_num = 32; - if (parameters.size() == 6) + if (parameters.size() >= 6) { thread_num = parameters[5].safeGet(); } @@ -160,19 +166,19 @@ namespace result_type = parameters[6].safeGet(); } if (result_type != 0 && result_type != 1) - throw Exception("AggregateFunction " + name + " only support result_type: 0, 1", ErrorCodes::LOGICAL_ERROR); + throw Exception("AggregateFunction " + name + " only support result_type: 0, 1", ErrorCodes::BAD_ARGUMENTS); if (!WhichDataType(argument_types[0]).isUInt8()) - throw Exception("AggregateFunction " + name + " needs Int type for its first argument", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("AggregateFunction " + name + " needs Int type for its first argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (!isBitmap64(argument_types[1])) throw Exception( - "AggregateFunction " + name + " needs BitMap64 type for its second argument", ErrorCodes::NOT_IMPLEMENTED); + "AggregateFunction " + name + " needs BitMap64 type for its second argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); for (size_t i = 2; i < argument_types.size(); ++i) { if (!isString(argument_types[i])) - throw Exception("AggregateFunction " + name + " needs String type", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("AggregateFunction " + name + " needs String type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } return std::make_shared(argument_types, union_num, join_keys_idx, group_by_keys_idx, logic_op, join_op, thread_num, result_type); diff --git a/src/AggregateFunctions/AggregateFunctionBitMapJoin.h b/src/AggregateFunctions/AggregateFunctionBitMapJoin.h index 645ba0dc8af..06e98ca0285 100644 --- a/src/AggregateFunctions/AggregateFunctionBitMapJoin.h +++ b/src/AggregateFunctions/AggregateFunctionBitMapJoin.h @@ -45,7 +45,6 @@ namespace DB { namespace ErrorCodes { -extern const int LOGICAL_ERROR; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } @@ -62,34 +61,34 @@ enum LogicOperationType struct LogicOperation { - LogicOperation() : logicOp(LogicOperationType::NONE) {} - LogicOperation(String operation) + LogicOperation() : logic_op(LogicOperationType::NONE) {} + explicit LogicOperation(String operation) { std::transform(operation.begin(), operation.end(), operation.begin(), ::toupper); if (operation == "NONE" || operation.empty()) - logicOp = LogicOperationType::NONE; + logic_op = LogicOperationType::NONE; else if (operation == "AND") - logicOp = LogicOperationType::AND; + logic_op = LogicOperationType::AND; else if (operation == "OR") - logicOp = LogicOperationType::OR; + logic_op = LogicOperationType::OR; else if (operation == "XOR") - logicOp = LogicOperationType::XOR; + logic_op = LogicOperationType::XOR; else if (operation == "ANDNOT") - logicOp = LogicOperationType::ANDNOT; + logic_op = LogicOperationType::ANDNOT; else if (operation == "RANDNOT" || operation == "REVERSEANDNOT") - logicOp = LogicOperationType::REVERSEANDNOT; + logic_op = LogicOperationType::REVERSEANDNOT; else - logicOp = LogicOperationType::UNDEFINED; + logic_op = LogicOperationType::UNDEFINED; } LogicOperation(const LogicOperation & rhs) { - this->logicOp = rhs.logicOp; + this->logic_op = rhs.logic_op; } - bool isValid() { return logicOp < LogicOperationType::UNDEFINED; } + bool isValid() const { return logic_op < LogicOperationType::UNDEFINED; } - LogicOperationType logicOp; + LogicOperationType logic_op; }; enum JoinType @@ -101,37 +100,39 @@ enum JoinType struct JoinOperation { - JoinOperation() : joinOp(JoinType::INNER) {} - JoinOperation(String operation) + JoinOperation() : join_op(JoinType::INNER) {} + explicit JoinOperation(String operation) { std::transform(operation.begin(), operation.end(), operation.begin(), ::toupper); if (operation.empty() || operation == "INNER") - joinOp = JoinType::INNER; + join_op = JoinType::INNER; else if (operation == "LEFT") - joinOp = JoinType::LEFT; + join_op = JoinType::LEFT; else - joinOp = JoinType::INVALID; + join_op = JoinType::INVALID; } - bool isValid() { return joinOp < JoinType::INVALID; } + bool isValid() const { return join_op < JoinType::INVALID; } - JoinType joinOp; + JoinType join_op; }; using JoinKeys = Strings; using GroupByKeys = Strings; using Position = UInt8; using BitMapPtr = std::shared_ptr; -using JoinTuple = std::tuple; +using JoinKeysPtr = std::shared_ptr; +using GroupByKeysPtr = std::shared_ptr; +using JoinTuple = std::tuple; using JoinTuplePtr = std::shared_ptr; using JoinTuplePtrs = std::vector; using PositionIndexPair = std::pair; -void writeStrings(const Strings & data, WriteBuffer & buf) +void writeStrings(const std::shared_ptr & data, WriteBuffer & buf) { - size_t size = data.size(); + size_t size = data->size(); writeVarUInt(size, buf); - for (auto & key : data) + for (auto & key : *data) writeString(key.data(), key.size(), buf); } @@ -151,21 +152,24 @@ void readStrings(Strings & data, ReadBuffer & buf) // The key used to hash the join keys or group by keys struct StringsMapKey { - Strings keys; + std::shared_ptr keys; StringsMapKey() = default; - StringsMapKey(String & key_) : keys{key_} {} - StringsMapKey(Strings && keys_) : keys(std::move(keys_)) {} - StringsMapKey(const Strings && keys_) : keys(std::move(keys_)) {} + explicit StringsMapKey(String & key_) + { + Strings strs{ key_ }; + keys = std::make_shared(std::move(strs)); + } + explicit StringsMapKey(std::shared_ptr && keyPtr) : keys(std::move(keyPtr)) {} bool operator==(const StringsMapKey & rhs) const { - if (keys.size() != rhs.keys.size()) + if (keys->size() != rhs.keys->size()) return false; - for (size_t i = 0; i < keys.size(); ++i) + for (size_t i = 0; i < keys->size(); ++i) { - if (keys.at(i) != rhs.keys.at(i)) + if (keys->at(i) != rhs.keys->at(i)) return false; } return true; @@ -176,12 +180,12 @@ struct HashStringsMapKey { size_t operator()(const StringsMapKey & one) const { - if (one.keys.empty()) + if (one.keys->empty()) return std::hash()(""); - size_t res = std::hash()(one.keys.at(0)); - for (size_t i = 1; i < one.keys.size(); ++i) - res ^= std::hash()(one.keys.at(i)) >> i; + size_t res = std::hash()(one.keys->at(0)); + for (size_t i = 1; i < one.keys->size(); ++i) + res ^= std::hash()(one.keys->at(i)) >> i; return res; } @@ -225,14 +229,14 @@ class KVBigLock { // Here is no lock, we just do this in a single thread void getAllKeyValueByResultType(ColumnTuple & tuple_in_array, size_t result_type) { - for (auto it = m_map.begin(); it != m_map.end(); ++it) + for (auto & it : m_map) { - BitMapPtr bitmap_ptr = std::get<2>(*(it->second.at(0))); - size_t key_size = it->first.keys.size(); + BitMapPtr bitmap_ptr = std::get<2>(*(it.second.at(0))); + size_t key_size = it.first.keys->size(); for (size_t i = 0; i < key_size; ++i) { auto & column_group_by = static_cast(tuple_in_array.getColumn(i)); - column_group_by.insert(it->first.keys.at(i)); + column_group_by.insert(it.first.keys->at(i)); } if (result_type == 0) { @@ -256,42 +260,47 @@ class KVBigLock { class KVSharded { public: - KVSharded(size_t num_shard) : m_mask(num_shard - 1), m_shards(num_shard) + explicit KVSharded(size_t num_shard) : m_mask(num_shard - 1), m_shards(num_shard) { if ((num_shard & m_mask) != 0) - throw Exception("num_shard should be a power of two", ErrorCodes::LOGICAL_ERROR); + throw Exception("num_shard should be a power of two", ErrorCodes::BAD_ARGUMENTS); } KVSharded(KVSharded && rhs) : m_mask(std::move(rhs.m_mask)), m_shards(std::move(rhs.m_shards)) {} - void operator=(KVSharded && rhs) + KVSharded& operator=(KVSharded && rhs) { - m_shards = std::move(rhs.m_shards); + if (this != &rhs) // Optional: Check for self-assignment + { + m_mask = std::move(rhs.m_mask); + m_shards = std::move(rhs.m_shards); + } + return *this; } void put(const StringsMapKey & key, const JoinTuplePtrs & value) { - get_shard(key).emplaceKVOrAddValue(std::move(key), std::move(value)); + getShard(key).emplaceKVOrAddValue(std::move(key), std::move(value)); } std::optional get(const StringsMapKey & key) { - return get_shard(key).get(key); + return getShard(key).get(key); } /// It's used in insertIntoResult function, by a single thread void writeResultOfKeyAndValue(ColumnTuple & tuple_in_array, size_t result_type) { - for (auto it = m_shards.begin(); it != m_shards.end(); ++it) + for (auto & m_shard : m_shards) { - it->getAllKeyValueByResultType(tuple_in_array, result_type); + m_shard.getAllKeyValueByResultType(tuple_in_array, result_type); } } private: - const size_t m_mask; + size_t m_mask; std::vector m_shards; - KVBigLock & get_shard(const StringsMapKey & key) + KVBigLock & getShard(const StringsMapKey & key) { HashStringsMapKey hash_fn; size_t h = hash_fn(key); @@ -300,31 +309,37 @@ class KVSharded }; /// It's used to accommodate user input data, and data is grouped by join keys -struct PositionTuples +struct JoinPositionTuples { Position position; HashedStringsKeyTuples tuples; // The key used here is join key - PositionTuples() = default; - PositionTuples(Position pos) : position(pos) {} - PositionTuples(const PositionTuples & rhs) : position(rhs.position), tuples(rhs.tuples) {} - PositionTuples(PositionTuples && rhs) : position(rhs.position), tuples(std::move(rhs.tuples)) {} - PositionTuples(Position && pos, StringsMapKey && join_keys, JoinTuplePtr && val) + JoinPositionTuples() = default; + explicit JoinPositionTuples(Position pos) : position(pos) {} + JoinPositionTuples(const JoinPositionTuples & rhs) = default; + JoinPositionTuples(JoinPositionTuples && rhs) : position(rhs.position), tuples(std::move(rhs.tuples)) {} + JoinPositionTuples(Position && pos, StringsMapKey && join_keys, JoinTuplePtr && val) : position(std::move(pos)), tuples{{std::move(join_keys), JoinTuplePtrs{val}}} {} - void operator=(const PositionTuples & rhs) + JoinPositionTuples& operator=(const JoinPositionTuples & rhs) { - this->position = rhs.position; - this->tuples = rhs.tuples; + if (this != &rhs) { // Check for self-assignment + this->position = rhs.position; + this->tuples = rhs.tuples; + } + return *this; } - void operator=(const PositionTuples && rhs) + JoinPositionTuples& operator=(JoinPositionTuples && rhs) { - this->position = std::move(rhs.position); - this->tuples = std::move(rhs.tuples); + if (this != &rhs) { // Check for self-assignment + this->position = std::move(rhs.position); + this->tuples = std::move(rhs.tuples); + } + return *this; } - void emplace_back(StringsMapKey && join_key, JoinTuplePtrs && value) + void emplaceBack(StringsMapKey && join_key, JoinTuplePtrs && value) { auto it = this->tuples.find(join_key); if (it == this->tuples.end()) @@ -337,16 +352,16 @@ struct PositionTuples std::make_move_iterator(value.end())); } - void emplace_back(StringsMapKey && join_key, JoinTuplePtr && value) + void emplaceBack(StringsMapKey && join_key, JoinTuplePtr && value) { - this->emplace_back(std::move(join_key), JoinTuplePtrs{value}); + this->emplaceBack(std::move(join_key), JoinTuplePtrs{value}); } - void insert(PositionTuples && rhs) + void insert(JoinPositionTuples && rhs) { - for (auto rt = rhs.tuples.begin(); rt != rhs.tuples.end(); ++rt) + for (auto & tuple : rhs.tuples) { - this->emplace_back(std::move(const_cast(rt->first)), std::move(rt->second)); + this->emplaceBack(std::move(const_cast(tuple.first)), std::move(tuple.second)); } } @@ -355,22 +370,21 @@ struct PositionTuples writeVarUInt(position, buf); size_t map_size = tuples.size(); writeVarUInt(map_size, buf); - - for (auto it = tuples.begin(); it != tuples.end(); ++it) + for (const auto & tuple : tuples) { - writeStrings(it->first.keys, buf); + writeStrings(tuple.first.keys, buf); - size_t tuples_num = it->second.size(); + size_t tuples_num = tuple.second.size(); writeVarUInt(tuples_num, buf); - for (auto jt = it->second.begin(); jt != it->second.end(); ++jt) + for (const auto & jt : tuple.second) { - JoinKeys join_key; - GroupByKeys group_by; + JoinKeysPtr join_key_ptr; + GroupByKeysPtr group_by_ptr; BitMapPtr bitmap_ptr; - std::tie(join_key, group_by, bitmap_ptr) = *(*jt); + std::tie(join_key_ptr, group_by_ptr, bitmap_ptr) = *jt; - writeStrings(join_key, buf); - writeStrings(group_by, buf); + writeStrings(const_cast(join_key_ptr), buf); + writeStrings(const_cast(group_by_ptr), buf); size_t bytes_size = (*bitmap_ptr).getSizeInBytes(); writeVarUInt(bytes_size, buf); @@ -415,14 +429,18 @@ struct PositionTuples buf.readStrict(buffer.data(), bytes_size); BitMap64 bitmap = BitMap64::readSafe(buffer.data(), bytes_size); - tmp_tuple = std::make_tuple(std::move(join_key), - std::move(group_by), - std::make_shared(bitmap)); + JoinKeysPtr join_key_ptr = make_shared(join_key); + GroupByKeysPtr group_by_ptr = make_shared(group_by); + + tmp_tuple = std::make_tuple(std::move(join_key_ptr), + std::move(group_by_ptr), + std::make_shared(bitmap)); tuples_ptrs.emplace_back(std::make_shared(tmp_tuple)); } - this->emplace_back(StringsMapKey(std::move(key)), std::move(tuples_ptrs)); + std::shared_ptr key_ptr = std::make_shared(std::move(key)); + this->emplaceBack(StringsMapKey(std::move(key_ptr)), std::move(tuples_ptrs)); } } }; @@ -431,22 +449,23 @@ struct AggregateFunctionBitMapJoinData { AggregateFunctionBitMapJoinData() = default; - std::vector join_tuples_by_position; + std::vector join_tuples_by_position; - void add(const Position & pos, const BitMapPtr bitmap_ptr, const JoinKeys & join_keys, GroupByKeys & group_bys, size_t union_num) + void add(const Position & pos, const BitMapPtr bitmap_ptr, JoinKeysPtr & join_keys, GroupByKeysPtr & group_bys, size_t union_num) { if (pos > union_num+1) throw Exception("AggregateFunction BitMapJoin: Wrong position value. Position starts from 1 and ends with join_num+1 ", - DB::ErrorCodes::LOGICAL_ERROR); + DB::ErrorCodes::BAD_ARGUMENTS); - StringsMapKey key(std::move(join_keys)); + JoinKeysPtr cpy(join_keys); + StringsMapKey key(std::move(cpy)); JoinTuplePtr tuple_ptr{std::make_shared(std::make_tuple(std::move(join_keys), std::move(group_bys), std::move(bitmap_ptr)))}; for (auto & pos_tuples : join_tuples_by_position) // Position value is in a small range, just compare one by one { if (pos-1 == pos_tuples.position) // position starts from 0, but pos from user starts from 1 { - pos_tuples.emplace_back(std::move(key), std::move(tuple_ptr)); + pos_tuples.emplaceBack(std::move(key), std::move(tuple_ptr)); return; } } @@ -457,7 +476,7 @@ struct AggregateFunctionBitMapJoinData void merge (const AggregateFunctionBitMapJoinData & rhs) { auto & lhs_tuples_by_position = this->join_tuples_by_position; - auto & rhs_tuples_by_position = const_cast &>(rhs.join_tuples_by_position); + auto & rhs_tuples_by_position = const_cast &>(rhs.join_tuples_by_position); if (rhs_tuples_by_position.empty()) return; @@ -468,20 +487,20 @@ struct AggregateFunctionBitMapJoinData } // Position value is in a small range, just compare one by one - for (auto rt = rhs_tuples_by_position.begin(); rt != rhs_tuples_by_position.end(); ++rt) + for (auto & rt : rhs_tuples_by_position) { bool pos_exists = false; - for (auto lt = lhs_tuples_by_position.begin(); lt != lhs_tuples_by_position.end(); ++lt) + for (auto & lt : lhs_tuples_by_position) { - if (lt->position == rt->position) + if (lt.position == rt.position) { - lt->insert(std::move(*rt)); + lt.insert(std::move(rt)); pos_exists = true; } } if (!pos_exists) { - lhs_tuples_by_position.emplace_back(std::move(*rt)); + lhs_tuples_by_position.emplace_back(std::move(rt)); } } } @@ -490,10 +509,9 @@ struct AggregateFunctionBitMapJoinData { size_t position_num = join_tuples_by_position.size(); writeVarUInt(position_num, buf); - for (auto it = join_tuples_by_position.begin(); - it != join_tuples_by_position.end(); ++it) + for (const auto & it : join_tuples_by_position) { - it->serialize(buf); + it.serialize(buf); } } @@ -504,7 +522,7 @@ struct AggregateFunctionBitMapJoinData for (size_t i = 0; i < position_num; ++i) { - PositionTuples pos_tuple; + JoinPositionTuples pos_tuple; pos_tuple.deserialize(buf); join_tuples_by_position.emplace_back(std::move(pos_tuple)); } @@ -563,12 +581,15 @@ class AggregateFunctionBitMapJoin final : public IAggregateFunctionDataHelper(pos) && columns_str.at(pi.second - 3) == "#-1#") - throw Exception("The column you identified for group by is invalid, where data is '#-1#'", ErrorCodes::LOGICAL_ERROR); + throw Exception("The column you identified for group by is invalid, where data is '#-1#'", ErrorCodes::BAD_ARGUMENTS); group_by_keys.emplace_back(columns_str.at(pi.second - 3)); } - this->data(place).add(pos, bitmap_ptr, join_keys, group_by_keys, union_num); + auto join_keys_ptr = make_shared(join_keys); + auto group_by_keys_ptr = make_shared(group_by_keys); + + this->data(place).add(pos, bitmap_ptr, join_keys_ptr, group_by_keys_ptr, union_num); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr __restrict rhs, Arena *) const override @@ -588,13 +609,12 @@ class AggregateFunctionBitMapJoin final : public IAggregateFunctionDataHelper &>(this->data(place).join_tuples_by_position); + auto & this_join_tuples = const_cast &>(this->data(place).join_tuples_by_position); if (this_join_tuples.size() < 2) return; - // throw Exception("AggregateFunction " + getName() + ": at least one position has no data actually", ErrorCodes::LOGICAL_ERROR); sort(this_join_tuples.begin(), this_join_tuples.end(), - [](const PositionTuples & left, const PositionTuples & right) -> bool { + [](const JoinPositionTuples & left, const JoinPositionTuples & right) -> bool { return left.position < right.position; }); @@ -619,31 +639,34 @@ class AggregateFunctionBitMapJoin final : public IAggregateFunctionDataHelperfirst; - auto & left = gt->second; // left JoinTuplePtrs + auto & key = gt.first; + auto & left = gt.second; // left JoinTuplePtrs + + if (left.empty()) + continue; auto rjt = rhs_data.find(key); if (rjt == rhs_data.end()) // key is not matched { - switch (join_operation.joinOp) + switch (join_operation.join_op) { case JoinType::INNER : // INNER JOIN continue; case JoinType::LEFT : // ALL LEFT JOIN { - for (auto it = left.begin(); it != left.end(); ++it) + for (auto & it : left) { - Strings group_by_keys = std::get<1>(*(*it)); - result.put(StringsMapKey(std::move(group_by_keys)), {*it}); + auto group_by_keys = std::get<1>(*it); + result.put(StringsMapKey(std::move(group_by_keys)), {it}); } } continue; @@ -653,39 +676,39 @@ class AggregateFunctionBitMapJoin final : public IAggregateFunctionDataHelpersecond; // right JoinTuplePtrs - for (auto lt = left.begin(); lt != left.end(); ++lt) + for (auto & lt : left) { - for (auto rt = right.cbegin(); rt != right.cend(); ++rt) + for (const auto & rt : right) { - Strings join_keys; - Strings lt_group_bys, rt_group_bys; + JoinKeysPtr join_keys_ptr; + GroupByKeysPtr lt_group_bys, rt_group_bys; BitMapPtr lt_bitmap_ptr, rt_bitmap_ptr; - std::tie(join_keys, lt_group_bys, lt_bitmap_ptr) = *(*lt); - std::tie(std::ignore, rt_group_bys, rt_bitmap_ptr) = *(*rt); + std::tie(join_keys_ptr, lt_group_bys, lt_bitmap_ptr) = *lt; + std::tie(std::ignore, rt_group_bys, rt_bitmap_ptr) = *rt; Strings group_bys; for (size_t i = 0; i < group_by_keys_idx.size(); ++i) { if (group_by_keys_idx[i].first == 0xFF) // If no position identifier { - if (lt_group_bys.at(i) != "#-1#") // left subquery has a group by key - group_bys.emplace_back(std::move(lt_group_bys.at(i))); + if (lt_group_bys->at(i) != "#-1#") // left subquery has a group by key + group_bys.emplace_back(std::move(lt_group_bys->at(i))); else - group_bys.emplace_back(std::move(rt_group_bys.at(i))); + group_bys.emplace_back(std::move(rt_group_bys->at(i))); } else { if (group_by_keys_idx[i].first == 1) - group_bys.emplace_back(std::move(lt_group_bys.at(i))); + group_bys.emplace_back(std::move(lt_group_bys->at(i))); else if (group_by_keys_idx[i].first == 2) - group_bys.emplace_back(std::move(rt_group_bys.at(i))); + group_bys.emplace_back(std::move(rt_group_bys->at(i))); } } BitMap64 bitmap(*lt_bitmap_ptr); - switch (logic_operation.logicOp) + switch (logic_operation.logic_op) { case DB::LogicOperationType::NONE : { @@ -712,10 +735,12 @@ class AggregateFunctionBitMapJoin final : public IAggregateFunctionDataHelper(std::move(bitmap)))}; + auto group_by_ptr = make_shared(group_bys); - result.put(std::move(StringsMapKey(std::move(group_bys))), + JoinTuple tmp_tuple{std::make_tuple(join_keys_ptr, group_by_ptr, + std::make_shared(std::move(bitmap)))}; + + result.put(std::move(StringsMapKey(std::move(group_by_ptr))), std::move(JoinTuplePtrs{std::make_shared(tmp_tuple)})); } } @@ -723,34 +748,44 @@ class AggregateFunctionBitMapJoin final : public IAggregateFunctionDataHelper threadPool = std::make_unique(thread_num_); + std::unique_ptr thread_pool = std::make_unique(thread_num_); for (size_t i = 0; i < thread_num_; ++i) { - auto joinAndFunc = std::bind(runJoin, i); - threadPool->scheduleOrThrowOnError(joinAndFunc); + auto join_and_func = [i, &run_join]() { run_join(i); }; + thread_pool->scheduleOrThrowOnError(join_and_func); } - threadPool->wait(); + thread_pool->wait(); } - KVSharded doJoinWithLogicOperation(std::vector & this_join_tuples) const + KVSharded doJoinWithLogicOperation(std::vector & this_join_tuples) const { HashedStringsKeyTuples & left_join_tuples = this_join_tuples.at(0).tuples; HashedStringsKeyTuples & right_join_tuples = this_join_tuples.at(1).tuples; // split the map to several vector - std::vector pair_vector_buckets(thread_num); + std::vector pair_vector_buckets; size_t idx = 0; - for (auto key_tuple_it = left_join_tuples.begin(); key_tuple_it != left_join_tuples.end(); ++key_tuple_it) + auto key_tuple_it = left_join_tuples.begin(); + for (; key_tuple_it != left_join_tuples.end(); ++key_tuple_it) + { + Pairs p{{key_tuple_it->first, key_tuple_it->second}}; + pair_vector_buckets.emplace_back(p); + ++idx; + } + + /// processing remaing data + for (; key_tuple_it != left_join_tuples.end(); ++key_tuple_it) { - pair_vector_buckets.at(idx % thread_num).emplace_back(std::move(*key_tuple_it)); - left_join_tuples.erase(key_tuple_it); - idx++; + pair_vector_buckets.at(idx % thread_num).emplace_back(key_tuple_it->first, key_tuple_it->second); + ++idx; } + left_join_tuples.clear(); KVSharded result(128); - joinMultiThreads(result, pair_vector_buckets, right_join_tuples, thread_num); + size_t actual_thread_num = std::min(thread_num, pair_vector_buckets.size()); + joinMultiThreads(result, pair_vector_buckets, right_join_tuples, actual_thread_num); return result; } diff --git a/src/AggregateFunctions/AggregateFunctionBitMapJoinAndCard.cpp b/src/AggregateFunctions/AggregateFunctionBitMapJoinAndCard.cpp index c3fa425e78a..7ffdca95438 100644 --- a/src/AggregateFunctions/AggregateFunctionBitMapJoinAndCard.cpp +++ b/src/AggregateFunctions/AggregateFunctionBitMapJoinAndCard.cpp @@ -21,19 +21,26 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + namespace { AggregateFunctionPtr createAggregateFunctionBitMapJoinAndCard(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) { if (argument_types.size() < 4) - throw Exception("AggregateFunction " + name + " needs at least four arguments", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("AggregateFunction " + name + " needs at least four arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); Int32 union_num = 0; UInt64 thread_num = 0; UInt64 limit_bitmap_number = 0; - if (parameters.size() == 0) - throw Exception("AggregateFunction " + name + " needs two parameters (join_num, thread_num)", ErrorCodes::NOT_IMPLEMENTED); + if (parameters.empty()) + throw Exception("AggregateFunction " + name + " needs two parameters (join_num, thread_num)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); else { union_num = static_cast(parameters[0].safeGet()); @@ -44,7 +51,7 @@ AggregateFunctionPtr createAggregateFunctionBitMapJoinAndCard(const std::string } if (union_num == 0 || union_num > 8) // a continuos 8 join is meaningless, 1 join is mostly used. - throw Exception("AggregateFunction " + name + " join_number is in range [1,8]", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("AggregateFunction " + name + " join_number is in range [1,8]", ErrorCodes::BAD_ARGUMENTS); if (thread_num == 0) thread_num = 16; if (thread_num > 48) // Several Storage-C machine only have 48 cores, besides 48 threads is large enough @@ -53,23 +60,23 @@ AggregateFunctionPtr createAggregateFunctionBitMapJoinAndCard(const std::string limit_bitmap_number = 100000000; // 100 million if (!isBitmap64(argument_types[0])) - throw Exception("AggregateFunction " + name + " needs BitMap64 type for its first argument", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("AggregateFunction " + name + " needs BitMap64 type for its first argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (!WhichDataType(argument_types[1]).isUInt8()) - throw Exception("AggregateFunction " + name + " needs Int type for its second argument", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("AggregateFunction " + name + " needs Int type for its second argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (!WhichDataType(argument_types[2]).isInt32()) - throw Exception("AggregateFunction " + name + " needs Int32 type for its third argument", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("AggregateFunction " + name + " needs Int32 type for its third argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - DataTypePtr attr_val_type = argument_types[3]; + const DataTypePtr& attr_val_type = argument_types[3]; if (!isString(*attr_val_type)) - throw Exception("AggregateFunction " + name + " needs String type for its fourth argument", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("AggregateFunction " + name + " needs String type for its fourth argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); for (size_t i = 4; i < argument_types.size(); ++i) { if (!isString(argument_types[i])) - throw Exception("AggregateFunction " + name + " needs String type for args...", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("AggregateFunction " + name + " needs String type for args...", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } return std::make_shared(argument_types, union_num, thread_num, limit_bitmap_number); @@ -78,13 +85,13 @@ AggregateFunctionPtr createAggregateFunctionBitMapJoinAndCard(const std::string AggregateFunctionPtr createAggregateFunctionBitMapJoinAndCard2(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) { if (argument_types.size() < 4) - throw Exception("AggregateFunction " + name + " needs at least four arguments", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("AggregateFunction " + name + " needs at least four arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); Int32 union_num = 0; UInt64 thread_num = 0; UInt64 limit_bitmap_number = 0; - if (parameters.size() == 0) - throw Exception("AggregateFunction " + name + " needs two parameters (join_num, thread_num)", ErrorCodes::NOT_IMPLEMENTED); + if (parameters.empty()) + throw Exception("AggregateFunction " + name + " needs two parameters (join_num, thread_num)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); else { union_num = static_cast(parameters[0].safeGet()); @@ -95,7 +102,7 @@ AggregateFunctionPtr createAggregateFunctionBitMapJoinAndCard2(const std::string } if (union_num == 0 || union_num > 8) // a continuos 8 join is meaningless, 1 join is mostly used. - throw Exception("AggregateFunction " + name + " join_number is in range [1,8]", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("AggregateFunction " + name + " join_number is in range [1,8]", ErrorCodes::BAD_ARGUMENTS); if (thread_num == 0) thread_num = 16; if (thread_num > 48) // Several Storage-C machine only have 48 cores, and 48 threads is large enough @@ -104,23 +111,23 @@ AggregateFunctionPtr createAggregateFunctionBitMapJoinAndCard2(const std::string limit_bitmap_number = 100000000; // 100 million if (!isBitmap64(argument_types[0])) - throw Exception("AggregateFunction " + name + " needs BitMap64 type for its first argument", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("AggregateFunction " + name + " needs BitMap64 type for its first argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (!WhichDataType(argument_types[1]).isUInt8()) - throw Exception("AggregateFunction " + name + " needs Int type for its second argument", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("AggregateFunction " + name + " needs Int type for its second argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (!WhichDataType(argument_types[2]).isInt32()) - throw Exception("AggregateFunction " + name + " needs Int32 type for its third argument", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("AggregateFunction " + name + " needs Int32 type for its third argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - DataTypePtr attr_val_type = argument_types[3]; + const DataTypePtr& attr_val_type = argument_types[3]; if (!isString(*attr_val_type)) - throw Exception("AggregateFunction " + name + " needs String type for its fourth argument", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("AggregateFunction " + name + " needs String type for its fourth argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); for (size_t i = 4; i < argument_types.size(); ++i) { if (!isString(argument_types[i])) - throw Exception("AggregateFunction " + name + " needs String type for args...", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("AggregateFunction " + name + " needs String type for args...", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } return std::make_shared(argument_types, union_num, thread_num, limit_bitmap_number); diff --git a/src/AggregateFunctions/AggregateFunctionBitMapJoinAndCard.h b/src/AggregateFunctions/AggregateFunctionBitMapJoinAndCard.h index 9f74cc911ed..0a6fb63869e 100644 --- a/src/AggregateFunctions/AggregateFunctionBitMapJoinAndCard.h +++ b/src/AggregateFunctions/AggregateFunctionBitMapJoinAndCard.h @@ -52,7 +52,6 @@ namespace DB namespace ErrorCodes { - extern const int LOGICAL_ERROR; extern const int NUMBER_OF_ARGUMENTS_DOES_NOT_MATCH; extern const int TOO_MANY_ROWS; } @@ -68,7 +67,7 @@ struct PositionTuples JoinTuplePtrs tuples; PositionTuples() = default; - PositionTuples(Int32 pos_):position(pos_) {} + explicit PositionTuples(Int32 pos_):position(pos_) {} PositionTuples(Int32 pos_, JoinTuplePtrs && tuples_) : position(pos_), tuples(std::move(tuples_)) {} void addTuple(const JoinTuple & tup) @@ -89,7 +88,7 @@ struct JoinTupleMapKey DB::String attr_val; DB::Strings args; - JoinTupleMapKey() { } + JoinTupleMapKey() = default; JoinTupleMapKey(const Int32 pos_, const DB::String & attr_val_, const DB::Strings & args_) : pos(pos_), attr_val(attr_val_), args(args_) { } bool operator==(const JoinTupleMapKey & rhs) const @@ -104,7 +103,7 @@ struct HashJoinTupleMapKey { size_t res = std::hash()(key.pos); res ^= std::hash()(key.attr_val); - for (auto a : key.args) + for (const auto& a : key.args) { res ^= std::hash()(a); } @@ -121,7 +120,7 @@ struct AggregateFunctionBitMapJoinAndCardData void add(const BitMapPtr & bitmap_ptr, const Int32 & pos, const JoinKey & join_key, const String & attr_val, const Strings & args, Int32 union_num) { if (pos <= 0 || pos > union_num+1) - throw Exception("AggregateFunction BitMapJoinAndCard: Wrong position value. Position starts from 1 and ends with union_num+1 ", DB::ErrorCodes::LOGICAL_ERROR); + throw Exception("AggregateFunction BitMapJoinAndCard: Wrong position value. Position starts from 1 and ends with union_num+1 ", DB::ErrorCodes::BAD_ARGUMENTS); Strings attr_vals(union_num+1); attr_vals[pos-1] = attr_val; @@ -140,15 +139,15 @@ struct AggregateFunctionBitMapJoinAndCardData void merge(const AggregateFunctionBitMapJoinAndCardData & rhs) { - for (auto rt = rhs.join_tuple_map.begin(); rt != rhs.join_tuple_map.end(); ++rt) + for (const auto & rt : rhs.join_tuple_map) { - auto it = join_tuple_map.find(rt->first); + auto it = join_tuple_map.find(rt.first); if (it == join_tuple_map.end()) - join_tuple_map.emplace(std::move(rt->first), std::move(rt->second)); + join_tuple_map.emplace(std::move(rt.first), std::move(rt.second)); else { - *std::get<0>((it->second)) |= *std::get<0>((rt->second)); + *std::get<0>((it->second)) |= *std::get<0>((rt.second)); } } } @@ -157,14 +156,14 @@ struct AggregateFunctionBitMapJoinAndCardData { size_t map_size = join_tuple_map.size(); writeVarUInt(map_size, buf); - for (auto it = join_tuple_map.begin(); it != join_tuple_map.end(); ++it) + for (const auto & it : join_tuple_map) { BitMapPtr bitmap_ptr; Int32 pos; JoinKey joinkey; Strings attr_vals; Strings args; - std::tie(bitmap_ptr, pos, joinkey, attr_vals, args) = it->second; + std::tie(bitmap_ptr, pos, joinkey, attr_vals, args) = it.second; size_t bytes_size = (*bitmap_ptr).getSizeInBytes(); writeVarUInt(bytes_size, buf); @@ -176,13 +175,13 @@ struct AggregateFunctionBitMapJoinAndCardData writeVarInt(joinkey, buf); writeVarUInt(attr_vals.size(), buf); - for (auto str : attr_vals) + for (const auto& str : attr_vals) { writeString(str, buf); } writeVarUInt((args).size(), buf); - for (auto a : args) + for (const auto& a : args) { writeString(a, buf); } @@ -256,7 +255,7 @@ class AggregateFunctionBitMapJoinAndCard final : public IAggregateFunctionDataHe auto bitmap_ptr = std::make_shared(std::move(const_cast(bitmap))); const auto & col_position = static_cast(*columns[1]); - const Int32 & positionInUnion = static_cast(col_position.getElement(row_num)); + const Int32 & position_in_union = static_cast(col_position.getElement(row_num)); const auto & col_joinkey = static_cast(*columns[2]); const JoinKey & join_key = col_joinkey.getElement(row_num); @@ -271,7 +270,7 @@ class AggregateFunctionBitMapJoinAndCard final : public IAggregateFunctionDataHe args.emplace_back(col_arg.getDataAt(row_num).toString()); } - this->data(place).add(bitmap_ptr, positionInUnion, join_key, attr_val, args, union_num); + this->data(place).add(bitmap_ptr, position_in_union, join_key, attr_val, args, union_num); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr __restrict rhs, Arena *) const override @@ -304,20 +303,20 @@ class AggregateFunctionBitMapJoinAndCard final : public IAggregateFunctionDataHe void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { auto & tuples_map = this->data(place).join_tuple_map; - std::vector tuplesByPosition; + std::vector tuples_by_position; for (size_t i = 0; i < union_num + 1; ++i) { - tuplesByPosition.emplace_back(i, JoinTuplePtrs()); + tuples_by_position.emplace_back(i, JoinTuplePtrs()); } //partition all input tuples by position - for (auto p = tuples_map.begin(); p != tuples_map.end(); ++p) + for (auto & p : tuples_map) { - Int32 pos = p->first.pos; - tuplesByPosition.at(pos-1).addTuple(p->second); + Int32 pos = p.first.pos; + tuples_by_position.at(pos-1).addTuple(p.second); } - const auto res = calcJoin(tuplesByPosition); + const auto res = calcJoin(tuples_by_position); auto & col = static_cast(to); auto &col_offsets = static_cast(col.getOffsetsColumn()); @@ -329,16 +328,16 @@ class AggregateFunctionBitMapJoinAndCard final : public IAggregateFunctionDataHe size_t args_num = arguments_num - 4; - for (auto & p : res) + for (const auto & p : res) { - for (auto rt = p.begin(); rt != p.end(); ++rt) + for (const auto & rt : p) { UInt64 bitmap_cardinality; JoinKey joinkey; Strings attr_vals; Strings args; - std::tie(bitmap_cardinality, std::ignore, joinkey, attr_vals, args) = std::move(*rt); + std::tie(bitmap_cardinality, std::ignore, joinkey, attr_vals, args) = std::move(rt); col_bitmap_card.insert(bitmap_cardinality); col_joinkey.insert(joinkey); @@ -358,24 +357,24 @@ class AggregateFunctionBitMapJoinAndCard final : public IAggregateFunctionDataHe } private: - std::vector> - calcJoinMultiThreads(std::shared_ptr> & res_ptr, const std::shared_ptr & rhs, size_t thread_num_, const bool is_last_join) const + static std::vector> + calcJoinMultiThreads(std::shared_ptr> & res_ptr, const std::shared_ptr & rhs, size_t thread_num_, const bool is_last_join) { std::vector intermediate_tuples_bucktes(thread_num_, JoinTuplePtrs()); // It store the intermediate JOIN result, and it's used for next JOIN std::vector> res_tuples_buckets(thread_num_, std::vector()); // It store the final result of the last JOIN ThreadGroupStatusPtr thread_group = CurrentThread::getGroup(); - auto runJoinAndCard = [&] (size_t index) + auto run_join_and_card = [&] (size_t index) { - setThreadName("bitmapJoinAndCard"); + setThreadName("JoinAndCard"); CurrentThread::attachToIfDetached(thread_group); JoinTuplePtrs tuples_tmp; std::vector res_tuples_in_a_thread; auto & left = res_ptr->at(index); - for (auto rt = rhs->tuples.begin(); rt != rhs->tuples.end(); ++rt) + for (auto & rt : rhs->tuples) { - for (auto lt = left.begin(); lt != left.end(); ++lt) + for (auto & lt : left) { BitMapPtr bitmap_ptr, rt_bitmap_ptr; Int32 pos, rt_pos; @@ -383,8 +382,8 @@ class AggregateFunctionBitMapJoinAndCard final : public IAggregateFunctionDataHe Strings attr_vals, rt_attr_vals; Strings args, rt_args; - std::tie(bitmap_ptr, pos, joinkey, attr_vals, args) = *(*lt); - std::tie(rt_bitmap_ptr, rt_pos, std::ignore, rt_attr_vals, rt_args) = *(*rt); + std::tie(bitmap_ptr, pos, joinkey, attr_vals, args) = *lt; + std::tie(rt_bitmap_ptr, rt_pos, std::ignore, rt_attr_vals, rt_args) = *rt; BitMap64 bitmap(*bitmap_ptr); bitmap &= *rt_bitmap_ptr; @@ -416,15 +415,15 @@ class AggregateFunctionBitMapJoinAndCard final : public IAggregateFunctionDataHe res_tuples_buckets[index] = std::move(res_tuples_in_a_thread); }; - std::unique_ptr threadPool = std::make_unique(thread_num_); + std::unique_ptr thread_pool = std::make_unique(thread_num_); for (size_t i = 0; i < thread_num_; ++i) { - auto joinAndCardFunc = std::bind(runJoinAndCard, i); - threadPool->scheduleOrThrowOnError(joinAndCardFunc); + auto join_and_card_func = [&run_join_and_card, i]() { run_join_and_card(i); }; + thread_pool->scheduleOrThrowOnError(join_and_card_func); } - threadPool->wait(); + thread_pool->wait(); res_ptr = std::make_shared>(std::move(intermediate_tuples_bucktes)); // For intermediate JOIN, a empty object returned, @@ -436,7 +435,7 @@ class AggregateFunctionBitMapJoinAndCard final : public IAggregateFunctionDataHe { //partition the entire position tuples into several parts if (position_tuples.empty()) - throw Exception("BitMapJoinAndCard::calcJoin: empty input data!", DB::ErrorCodes::LOGICAL_ERROR); + throw Exception("BitMapJoinAndCard::calcJoin: empty input data!", DB::ErrorCodes::BAD_ARGUMENTS); // look up for the largest parts size_t max_size = 0; diff --git a/src/AggregateFunctions/AggregateFunctionBitMapJoinAndCard2.h b/src/AggregateFunctions/AggregateFunctionBitMapJoinAndCard2.h index 96b87d0d8e7..3f0620ad8d6 100644 --- a/src/AggregateFunctions/AggregateFunctionBitMapJoinAndCard2.h +++ b/src/AggregateFunctions/AggregateFunctionBitMapJoinAndCard2.h @@ -19,6 +19,7 @@ #include #include +#include "Common/formatIPv6.h" #include #include #include @@ -50,7 +51,6 @@ namespace DB namespace ErrorCodes { - extern const int LOGICAL_ERROR; extern const int NUMBER_OF_ARGUMENTS_DOES_NOT_MATCH; } @@ -65,7 +65,7 @@ struct AggregateFunctionBitMapJoinAndCard2Data void add(const BitMapPtr & bitmap_ptr, const Int32 & pos, const JoinKey & join_key, const String & attr_val, const Strings & args, Int32 union_num) { if (pos <= 0 || pos > union_num+1) - throw Exception("AggregateFunction BitMapJoinAndCard2: Wrong position value. Position starts from 1 and ends with join_num+1, please check", DB::ErrorCodes::LOGICAL_ERROR); + throw Exception("AggregateFunction BitMapJoinAndCard2: Wrong position value. Position starts from 1 and ends with join_num+1, please check", DB::ErrorCodes::BAD_ARGUMENTS); Strings attr_vals(union_num+1); attr_vals[pos-1] = attr_val; @@ -84,14 +84,14 @@ struct AggregateFunctionBitMapJoinAndCard2Data size_t input_tuples_size = input_tuples.size(); writeVarUInt(input_tuples_size, buf); - for (auto it = input_tuples.begin(); it != input_tuples.end(); ++it) + for (const auto & input_tuple : input_tuples) { BitMapPtr bitmap_ptr; Int32 pos; JoinKey joinkey; Strings attr_vals; Strings args; - std::tie(bitmap_ptr, pos, joinkey, attr_vals, args) = *it; + std::tie(bitmap_ptr, pos, joinkey, attr_vals, args) = input_tuple; size_t bytes_size = (*bitmap_ptr).getSizeInBytes(); writeVarUInt(bytes_size, buf); @@ -103,13 +103,13 @@ struct AggregateFunctionBitMapJoinAndCard2Data writeVarInt(joinkey, buf); writeVarUInt(attr_vals.size(), buf); - for (auto str: attr_vals) + for (const auto& str: attr_vals) { writeString(str, buf); } writeVarUInt((args).size(), buf); - for (auto a: args) + for (const auto& a: args) { writeString(a, buf); } @@ -183,7 +183,7 @@ class AggregateFunctionBitMapJoinAndCard2 final : public IAggregateFunctionDataH auto bitmap_ptr = std::make_shared(std::move(const_cast(bitmap))); const auto & col_position = static_cast(*columns[1]); - const Int32 & positionInUnion = static_cast(col_position.getElement(row_num)); + const Int32 & position_in_union = static_cast(col_position.getElement(row_num)); const auto & col_joinkey = static_cast(*columns[2]); const JoinKey & join_key = col_joinkey.getElement(row_num); @@ -198,7 +198,7 @@ class AggregateFunctionBitMapJoinAndCard2 final : public IAggregateFunctionDataH args.emplace_back(col_arg.getDataAt(row_num).toString()); } - this->data(place).add(bitmap_ptr, positionInUnion, join_key, attr_val, args, union_num); + this->data(place).add(bitmap_ptr, position_in_union, join_key, attr_val, args, union_num); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr __restrict rhs, Arena *) const override @@ -232,20 +232,20 @@ class AggregateFunctionBitMapJoinAndCard2 final : public IAggregateFunctionDataH { auto & input_tuples = this->data(place).input_tuples; - std::vector tuplesByPosition; + std::vector tuples_by_position; for (size_t i = 0; i < union_num + 1; ++i) { - tuplesByPosition.emplace_back(i, JoinTuplePtrs()); + tuples_by_position.emplace_back(i, JoinTuplePtrs()); } //partition all input tuples by position for (auto & p : input_tuples) { Int32 pos = std::get<1>(p); - tuplesByPosition.at(pos-1).addTuple(p); + tuples_by_position.at(pos-1).addTuple(p); } - const auto res = calcJoin(tuplesByPosition); + const auto res = calcJoin(tuples_by_position); auto & col = static_cast(to); auto &col_offsets = static_cast(col.getOffsetsColumn()); @@ -257,16 +257,16 @@ class AggregateFunctionBitMapJoinAndCard2 final : public IAggregateFunctionDataH size_t args_num = arguments_num - 4; - for (auto & p : res) + for (const auto & p : res) { - for (auto rt = p.begin(); rt != p.end(); ++rt) + for (const auto & rt : p) { UInt64 bitmap_cardinality; JoinKey joinkey; Strings attr_vals; Strings args; - std::tie(bitmap_cardinality, std::ignore, joinkey, attr_vals, args) = std::move(*rt); + std::tie(bitmap_cardinality, std::ignore, joinkey, attr_vals, args) = std::move(rt); col_bitmap_card.insert(bitmap_cardinality); col_joinkey.insert(joinkey); @@ -293,17 +293,17 @@ class AggregateFunctionBitMapJoinAndCard2 final : public IAggregateFunctionDataH std::vector> res_tuples_buckets(thread_num_, std::vector()); // It store the final result of the last JOIN ThreadGroupStatusPtr thread_group = CurrentThread::getGroup(); - auto runJoinAndCard = [&] (size_t index) + auto run_join_and_card = [&] (size_t index) { - setThreadName("bitmapJoinAndCard"); + setThreadName("JoinAndCard2"); CurrentThread::attachToIfDetached(thread_group); JoinTuplePtrs tuples_tmp; std::vector res_tuples_in_a_thread; auto & left = res_ptr->at(index); - for (auto rt = rhs->tuples.begin(); rt != rhs->tuples.end(); ++rt) + for (auto & rt : rhs->tuples) { - for (auto lt = left.begin(); lt != left.end(); ++lt) + for (auto & lt : left) { BitMapPtr bitmap_ptr, rt_bitmap_ptr; Int32 pos, rt_pos; @@ -311,8 +311,8 @@ class AggregateFunctionBitMapJoinAndCard2 final : public IAggregateFunctionDataH Strings attr_vals, rt_attr_vals; Strings args, rt_args; - std::tie(bitmap_ptr, pos, joinkey, attr_vals, args) = *(*lt); - std::tie(rt_bitmap_ptr, rt_pos, std::ignore, rt_attr_vals, rt_args) = *(*rt); + std::tie(bitmap_ptr, pos, joinkey, attr_vals, args) = *lt; + std::tie(rt_bitmap_ptr, rt_pos, std::ignore, rt_attr_vals, rt_args) = *rt; BitMap64 bitmap(*bitmap_ptr); bitmap &= *rt_bitmap_ptr; @@ -344,15 +344,15 @@ class AggregateFunctionBitMapJoinAndCard2 final : public IAggregateFunctionDataH res_tuples_buckets[index] = std::move(res_tuples_in_a_thread); }; - std::unique_ptr threadPool = std::make_unique(thread_num_); + std::unique_ptr thread_pool = std::make_unique(thread_num_); for (size_t i = 0; i < thread_num; ++i) { - auto joinAndCardFunc = std::bind(runJoinAndCard, i); - threadPool->scheduleOrThrowOnError(joinAndCardFunc); + auto join_and_card_func = [&run_join_and_card, i]() { run_join_and_card(i); }; + thread_pool->scheduleOrThrowOnError(join_and_card_func); } - threadPool->wait(); + thread_pool->wait(); res_ptr = std::make_shared>(std::move(intermediate_tuples_bucktes)); // For intermediate JOIN, a empty object returned, @@ -364,7 +364,7 @@ class AggregateFunctionBitMapJoinAndCard2 final : public IAggregateFunctionDataH { //partition the entire position tuples into several parts if (position_tuples.empty()) - throw Exception("BitMapJoinAndCard::calcJoin: empty input data!", DB::ErrorCodes::LOGICAL_ERROR); + throw Exception("BitMapJoinAndCard::calcJoin: empty input data!", DB::ErrorCodes::BAD_ARGUMENTS); //look up for the largest parts size_t max_size = 0; diff --git a/src/AggregateFunctions/AggregateFunctionBitmapColumnDiff.cpp b/src/AggregateFunctions/AggregateFunctionBitmapColumnDiff.cpp index dad1c81af24..af43a47bd32 100644 --- a/src/AggregateFunctions/AggregateFunctionBitmapColumnDiff.cpp +++ b/src/AggregateFunctions/AggregateFunctionBitmapColumnDiff.cpp @@ -28,33 +28,33 @@ AggregateFunctionPtr createAggregateFunctionBitmapColumnDiff(const std::string & if (argument_types.size() != 2) throw Exception("AggregateFunction " + name + " need only two arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - UInt64 return_type_{0}, diff_step_{1}; + UInt64 return_type{0}, diff_step{1}; String diff_direction_str{"forward"}; if (!parameters.empty() && parameters.size() != 3) throw Exception("AggregateFunction " + name + " need three parameters", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); if (!parameters.empty()) { - parameters[0].tryGet(return_type_); + parameters[0].tryGet(return_type); parameters[1].tryGet(diff_direction_str); - parameters[2].tryGet(diff_step_); + parameters[2].tryGet(diff_step); } if (!isBitmap64(argument_types[1])) throw Exception("AggregateFunction " + name + " need BitMap64 type for its second argument", ErrorCodes::NOT_IMPLEMENTED); - DataTypePtr data_type_0 = argument_types[0]; + const DataTypePtr& data_type_0 = argument_types[0]; if (!WhichDataType(data_type_0).isDate() && !WhichDataType(data_type_0).isUInt() && !WhichDataType(data_type_0).isInt() && !WhichDataType(data_type_0).isString()) throw Exception("AggregateFunction " + name + " need Date/Int/UInt/String type for its first argument, for order sorting.", ErrorCodes::NOT_IMPLEMENTED); if (WhichDataType(data_type_0).isDate()) - return std::make_shared>(argument_types, return_type_, diff_direction_str, diff_step_, true); + return std::make_shared>(argument_types, return_type, diff_direction_str, diff_step, true); else if (WhichDataType(data_type_0).isString()) - return std::make_shared>(argument_types, return_type_, diff_direction_str, diff_step_); + return std::make_shared>(argument_types, return_type, diff_direction_str, diff_step); else { AggregateFunctionPtr res; - res.reset(createWithNumericType(*data_type_0, argument_types, return_type_, diff_direction_str, diff_step_)); + res.reset(createWithNumericType(*data_type_0, argument_types, return_type, diff_direction_str, diff_step)); return res; } } diff --git a/src/AggregateFunctions/AggregateFunctionBitmapColumnDiff.h b/src/AggregateFunctions/AggregateFunctionBitmapColumnDiff.h index f6077abb366..18d36812f4e 100644 --- a/src/AggregateFunctions/AggregateFunctionBitmapColumnDiff.h +++ b/src/AggregateFunctions/AggregateFunctionBitmapColumnDiff.h @@ -35,7 +35,6 @@ namespace DB { namespace ErrorCodes { - extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; @@ -56,10 +55,12 @@ struct AggregateFunctionBitMapColumnDiffData void add(const T key, const BitMap64 & bitmap) { - auto [it, inserted] = data.try_emplace(key, std::make_unique(std::move(const_cast(bitmap)))); - if (!inserted) { + auto it = data.find(key); + + if (it != data.end()) *(it->second) |= bitmap; - } + else + data.emplace(key, std::make_unique(const_cast(bitmap))); } void merge(AggregateFunctionBitMapColumnDiffData & rhs) @@ -133,7 +134,7 @@ enum DiffDirection struct DiffDirectionOp { DiffDirectionOp() : diff_direc(DiffDirection::FORWARD) {} - DiffDirectionOp(String diff_dir_op) + explicit DiffDirectionOp(String diff_dir_op) { std::transform(diff_dir_op.begin(), diff_dir_op.end(), diff_dir_op.begin(), ::tolower); if (diff_dir_op.empty() || diff_dir_op == "forward") @@ -227,7 +228,7 @@ using DiffPair = typename std::pair; return; if (diff_step >= input_data.size()) - throw Exception(getName() + ": the step " + std::to_string(diff_step) + " is larger than data size", ErrorCodes::LOGICAL_ERROR); + throw Exception(getName() + ": the step " + std::to_string(diff_step) + " is larger than data size", ErrorCodes::BAD_ARGUMENTS); std::vector all_data; std::unordered_map> intermediate_res; diff --git a/src/AggregateFunctions/AggregateFunctionBitmapExpressionCalculation.cpp b/src/AggregateFunctions/AggregateFunctionBitmapExpressionCalculation.cpp index 12b63ca2c3e..6ea7ae7c6e7 100644 --- a/src/AggregateFunctions/AggregateFunctionBitmapExpressionCalculation.cpp +++ b/src/AggregateFunctions/AggregateFunctionBitmapExpressionCalculation.cpp @@ -17,30 +17,37 @@ #include #include #include - -#pragma GCC diagnostic ignored "-Wunused" -#pragma GCC diagnostic ignored "-Wunused-parameter" +#include namespace DB { struct Settings; +namespace ErrorCodes +{ + const extern int TYPE_MISMATCH; + const extern int SIZES_OF_COLUMNS_DOESNT_MATCH; + const extern int AGGREGATE_FUNCTION_THROW; +} + namespace { template