diff --git a/README.md b/README.md index 64215cb..6265a80 100644 --- a/README.md +++ b/README.md @@ -36,21 +36,24 @@ The meaning and format of the variables used are specified in the table below. | --------- | ------- | | `ansible_python_interpreter` | The correct path to the Python interpreter on the YDB cluster hosts. | | `ydb_dir` | Path of the YDB software installation directory to be created on the hosts. | -| `libidn_archive` | Enable the installation of custom-built libidn for RHEL, AlmaLinux or Rocky Linux. | | `ydb_archive` | YDB server binary package in .tar.gz format | +| `ydb_version` | YDB version label used to define the installation directory path for the particular version of YDB binary package. Typically depends oin the `ydb_archive` value | | `ydb_unpack_options` | Extra flags to be passed to `tar` for unpacking the YDB server binaries, typically should contain the value `--strip-component=1` | | `ydb_tls_dir` | Path to the local directory with the TLS certificates and keys, as generated by the [sample script](https://github.com/ydb-platform/ydb/tree/main/ydb/deploy/tls_cert_gen), or following the filename convention used by the sample script | | `ydb_config` | The name of the cluster configuration file within the `files` subdirectory (**without** the `actor_system_config` snippet!) | | `ydb_domain` | The name of the root domain hosting the databases, value `Root` is used in the YDB documentation | +| `ydb_dbname` | Database name which is used for database creation, database nodes deployment and database nodes rolling restart | | `ydb_disks` | Disk layout of storage nodes, defined as `ydbd_static` in the hosts file. Defined as list of structures having the following fields:
`name` - physical device name (like `/dev/sdb` or `/dev/vdb`);
`label` - the desired YDB data partition label, as used in the cluster configuration file (like `ydb_disk_1`) | -| `ydb_dynnodes` | Set of dynamic nodes to be ran on each host listed as `ydbd_dynamic` in the hosts file. Defined as list of structures having the following fields:
`dbname` - name of the YDB database handled by the corresponding dynamic node;
`instance` - dynamic node service instance name, allowing to distinguish between multiple dynamic nodes for the same database running in the same host;
`offset` - integer number `0-N`, used as the offset for the standard network port numbers (`0` means using the standard ports). | -| `ydb_brokers` | List of host names running the YDB static nodes, exactly 3 (three) host names must be specified | +| `ydb_dynnodes` | Set of dynamic nodes to be ran on each host listed as `ydbd_dynamic` in the hosts file. Defined as list of structures having the following fields:
`dbname` - name of the YDB database handled by the corresponding dynamic node;
`instance` - dynamic node service instance name, allowing to distinguish between multiple dynamic nodes for the same database running in the same host;
`offset` - integer number `0-N`, used as the offset for the standard network port numbers (`0` means using the standard ports);
`affinity` - CPU core mask or range to be used for the service instance, optional. | +| `ydb_affinity_static` | CPU core mask or range to be used by the storage nodes services, optional | +| `ydb_brokers` | List of host names running the YDB static nodes, normally three host names must be specified | | `ydb_cores_static` | Number of cores to be used by thread pools of the static nodes | | `ydb_cores_dynamic` | Number of cores to be used by thread pools of the dynamic nodes | -| `ydb_dbname` | Database name, for database creation, dynamic nodes deployment and dynamic nodes rolling restart | | `ydb_pool_kind` | YDB default storage pool kind, as specified in the static nodes configuration file in the `storage_pool_types.kind` field | | `ydb_default_groups` | Initial number of storage groups in the newly created database | | `dynnode_restart_sleep_seconds` | Number of seconds to sleep after startup of each dynamic node during the rolling restart. | +| `ydb_congestion_setting` | TCP congestion control setting, recommended value is `htcp`. Optional | +| `ydb_congestion_module` | TCP congestion control kernel loadable module name, recommended value is `tcp_htcp`. Optional | ## Installing the YDB cluster using the Ansible playbooks diff --git a/binaries-all.yaml b/binaries-all.yaml new file mode 100644 index 0000000..a7089b2 --- /dev/null +++ b/binaries-all.yaml @@ -0,0 +1,3 @@ +- hosts: all + roles: + - role: ydbd diff --git a/binaries-dynamic.yaml b/binaries-dynamic.yaml index e7c839d..e79fe62 100644 --- a/binaries-dynamic.yaml +++ b/binaries-dynamic.yaml @@ -1,3 +1,4 @@ - hosts: ydbd_dynamic roles: - role: ydbd + - role: ydbd_dynamic diff --git a/binaries-static.yaml b/binaries-static.yaml index 9c8c8dc..822221a 100644 --- a/binaries-static.yaml +++ b/binaries-static.yaml @@ -1,3 +1,4 @@ - hosts: ydbd_static roles: - role: ydbd + - role: ydbd_static diff --git a/files/config-3nodes-extra.yaml b/files/config-3nodes-extra.yaml new file mode 100644 index 0000000..6d2cc22 --- /dev/null +++ b/files/config-3nodes-extra.yaml @@ -0,0 +1,191 @@ +storage_config_generation: 0 +static_erasure: mirror-3-dc +host_configs: # the list of available host configurations in the cluster. +- drive: + - path: /dev/disk/by-partlabel/ydb_disk_1 + type: SSD + - path: /dev/disk/by-partlabel/ydb_disk_2 + type: SSD + - path: /dev/disk/by-partlabel/ydb_disk_3 + type: SSD + host_config_id: 1 +hosts: +- host: zeit-1 + host_config_id: 1 + walle_location: + body: 1 + data_center: 'zone-a' + rack: '1' +- host: zeit-2 + host_config_id: 1 + walle_location: + body: 2 + data_center: 'zone-b' + rack: '2' +- host: zeit-3 + host_config_id: 1 + walle_location: + body: 3 + data_center: 'zone-c' + rack: '3' +domains_config: + # There can be only one root domain in a cluster. Domain name prefixes all scheme objects names, e.g. full name of a table table1 in database db1. + # in a cluster with domains_config.domain.name parameter set to Root would be equal to /Root/db1/table1 + domain: + - name: Domain0 + domain_id: 1 + storage_pool_types: + - kind: ssd + pool_config: + box_id: 1 + # fault tolerance mode name - none, block-4-2, or mirror-3-dc.. + # See docs for more details https://ydb.tech/en/docs/deploy/configuration/config#domains-blob + erasure_species: mirror-3-dc + kind: ssd + geometry: + realm_level_begin: 10 + realm_level_end: 20 + domain_level_begin: 10 + domain_level_end: 256 + pdisk_filter: + - property: + - type: SSD # device type to match host_configs.drive.type + vdisk_kind: Default + state_storage: + - ring: + node: [1, 2, 3] + nto_select: 3 + ssid: 1 + security_config: + enforce_user_token_requirement: true + monitoring_allowed_sids: + - "root" + - "ydb-admin1@ldap" + - "CN=ydb-admins,OU=demo1,DC=ydb,DC=demo@ldap" + administration_allowed_sids: + - "root" + - "ydb-admin1@ldap" + - "CN=ydb-admins,OU=demo1,DC=ydb,DC=demo@ldap" + viewer_allowed_sids: + - "root" + - "ydb-admin1@ldap" + - "CN=ydb-admins,OU=demo1,DC=ydb,DC=demo@ldap" + +blob_storage_config: # configuration of static blobstorage group. + # YDB uses this group to store system tablets' data, like SchemeShard + service_set: + groups: + - erasure_species: mirror-3-dc # fault tolerance mode name for the static group + rings: # in mirror-3-dc must have exactly 3 rings or availability zones + - fail_domains: # first record: fail domains of the static group describe where each vdisk of the static group should be located. + - vdisk_locations: + - node_id: zeit-1 + pdisk_category: SSD + path: /dev/disk/by-partlabel/ydb_disk_1 + - vdisk_locations: + - node_id: zeit-1 + pdisk_category: SSD + path: /dev/disk/by-partlabel/ydb_disk_2 + - vdisk_locations: + - node_id: zeit-1 + pdisk_category: SSD + path: /dev/disk/by-partlabel/ydb_disk_3 + - fail_domains: # second ring: fail domains of the static group describe where each vdisk of the static group should be located. + - vdisk_locations: + - node_id: zeit-2 + pdisk_category: SSD + path: /dev/disk/by-partlabel/ydb_disk_1 + - vdisk_locations: + - node_id: zeit-2 + pdisk_category: SSD + path: /dev/disk/by-partlabel/ydb_disk_2 + - vdisk_locations: + - node_id: zeit-2 + pdisk_category: SSD + path: /dev/disk/by-partlabel/ydb_disk_3 + - fail_domains: # third ring: fail domains of the static group describe where each vdisk of the static group should be located. + - vdisk_locations: + - node_id: zeit-3 + pdisk_category: SSD + path: /dev/disk/by-partlabel/ydb_disk_1 + - vdisk_locations: + - node_id: zeit-3 + pdisk_category: SSD + path: /dev/disk/by-partlabel/ydb_disk_2 + - vdisk_locations: + - node_id: zeit-3 + pdisk_category: SSD + path: /dev/disk/by-partlabel/ydb_disk_3 + +channel_profile_config: + profile: + - channel: + - erasure_species: mirror-3-dc + pdisk_category: 1 # 0=ROT, 1=SSD, 2=NVME + storage_pool_kind: ssd + - erasure_species: mirror-3-dc + pdisk_category: 1 + storage_pool_kind: ssd + - erasure_species: mirror-3-dc + pdisk_category: 1 + storage_pool_kind: ssd + profile_id: 0 + +interconnect_config: + start_tcp: true + encryption_mode: OPTIONAL + path_to_certificate_file: "/opt/ydb/certs/node.crt" + path_to_private_key_file: "/opt/ydb/certs/node.key" + path_to_ca_file: "/opt/ydb/certs/ca.crt" + +grpc_config: + cert: "/opt/ydb/certs/node.crt" + key: "/opt/ydb/certs/node.key" + ca: "/opt/ydb/certs/ca.crt" + services_enabled: + - legacy + +auth_config: + use_builtin_auth: true + ldap_authentication: + host: "dc2016.ydb.demo" + port: 389 + base_dn: "OU=demo1,DC=ydb,DC=demo" + bind_dn: "ydb-admin1@ydb.demo" + bind_password: "P@$$w0rd+" + search_filter: "sAMAccountName=$username" + use_tls: + enable: true + ca_cert_file: "/etc/ssl/certs/ca-certificates.crt" + cert_require: DEMAND + ldap_authentication_domain: "ldap" + refresh_time: "10m" + +table_service_config: + sql_version: 1 + +bootstrap_config: + shared_cache_config: {memory_limit: '4294967296'} + +feature_flags: + enable_grpc_audit: false + enable_get_node_labels: true + enable_views: true + +log_config: + default_level: 4 + uaclient_config: + uri: 'localhost:16400' + log_name: main + +audit_config: + unified_agent_backend: + format: JSON + log_name: audit + +# Actor system config is appended to the configuration file by Ansible task. +# Provided here in the commented form for reference +#actor_system_config: +# use_auto_config: true +# node_type: STORAGE # STORAGE or COMPUTE +# cpu_count: 6 diff --git a/files/config-3nodes.example b/files/config-3nodes.example index 128b211..67e517e 100644 --- a/files/config-3nodes.example +++ b/files/config-3nodes.example @@ -147,7 +147,7 @@ bootstrap_config: audit_config: file_backend: format: JSON - file_path: "/opt/ydb/audit/ydb-audit.log" + file_path: "ydb-audit.log" # Actor system config is appended to the configuration file by Ansible task. # Provided here in the commented form for reference #actor_system_config: diff --git a/files/config-9nodes.yaml.example b/files/config-9nodes.yaml.example index b86eee4..a40dee1 100644 --- a/files/config-9nodes.yaml.example +++ b/files/config-9nodes.yaml.example @@ -168,7 +168,7 @@ bootstrap_config: audit_config: file_backend: format: JSON - file_path: "/opt/ydb/audit/ydb-audit.log" + file_path: "ydb-audit.log" # Actor system config is appended to the configuration file by Ansible task. # Provided here in the commented form for reference #actor_system_config: diff --git a/files/example-dynamic-config.yaml b/files/example-dynamic-config.yaml new file mode 100644 index 0000000..6527cf6 --- /dev/null +++ b/files/example-dynamic-config.yaml @@ -0,0 +1,16 @@ +metadata: + kind: MainConfig + cluster: "" + version: 6 +config: + yaml_config_enabled: true + log_config: + default_level: 4 +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: [] diff --git a/files/modules-ydbd.conf b/files/modules-ydbd.conf deleted file mode 100644 index 8f3d659..0000000 --- a/files/modules-ydbd.conf +++ /dev/null @@ -1 +0,0 @@ -tcp_htcp diff --git a/group_vars/all.example b/group_vars/all.example index 00e545e..8b889e6 100644 --- a/group_vars/all.example +++ b/group_vars/all.example @@ -2,16 +2,9 @@ ansible_python_interpreter: /usr/bin/python3 -# libidn compiled code. -# Required for RHEL 8, RHEL 9 and similar (AlmaLinux, etc) -# libidn_archive: libidn.tar.gz - # YDB cluster domain (aka cluster identifier), Root in the documentation ydb_domain: Domain0 -# YDB cluster name -ydb_cluster: cluster1 - # YDB software installation directory ydb_dir: /opt/ydb @@ -19,10 +12,10 @@ ydb_dir: /opt/ydb ydb_tls_dir: ./tls/CA/certs/2023-03-09_16-49-39 # YDB server software archive -ydb_archive: yasubd-24.1.18.1-linux-amd64.tar.gz +ydb_archive: yasubd-24.1.18.2-linux-amd64.tar.gz # YDB server software release -ydb_version: 24.1.18.1 +ydb_version: 24.1.18.2 # Extra options passed to tar for unpack, typically "--strip-component=1" ydb_unpack_options: "--strip-component=1" @@ -48,7 +41,8 @@ ydb_cores_dynamic: 14 ydb_disks: - name: /dev/vdb label: ydb_disk_1 -# Configuration settings for dynamic YDB cluster nodes + - name: /dev/vdc + label: ydb_disk_2 # The list of exactly three node brokers (static nodes) ydb_brokers: @@ -73,4 +67,14 @@ ydb_dynnodes: ydb_affinity_static: '0-31' # Extra sleep time to allow dynamic nodes to settle up -dynnode_restart_sleep_seconds: 30 +dynnode_restart_sleep_seconds: 10 + +# Source path to Unified Agent binary +unified_agent_binary: ../unified_agent +# Unified Agent installation root +unified_agent_home: /opt/ydb/unified_agent + +# Optional network congestion control setting, htcp recommended +ydb_congestion_setting: htcp +# Optional module to load the congestion control algorithm +ydb_congestion_module: tcp_htcp diff --git a/install-static.yaml b/install-static.yaml index d3ee936..420ed2d 100644 --- a/install-static.yaml +++ b/install-static.yaml @@ -4,3 +4,4 @@ - role: ydbd - role: ydbd_config - role: ydbd_static + - role: ydbd_init diff --git a/roles/ydbd/tasks/main.yml b/roles/ydbd/tasks/main.yml index c96cd50..35843e2 100644 --- a/roles/ydbd/tasks/main.yml +++ b/roles/ydbd/tasks/main.yml @@ -39,33 +39,6 @@ name: jq state: present -- name: Deploy custom libidn - block: - - name: Unpacking custom libidn - unarchive: creates=/usr/local/lib/libidn.so.11 dest=/usr/local/lib group=bin owner=root src={{ libidn_archive }} - - name: Symlinking libidn.so.11 to /lib64 - ansible.builtin.file: - src: /usr/local/lib/libidn.so.11 - dest: /lib64/libidn.so.11 - owner: root - group: bin - state: link - - name: Symlinking libidn.so.11 to /usr/lib - ansible.builtin.file: - src: /usr/local/lib/libidn.so.11 - dest: /usr/lib/libidn.so.11 - owner: root - group: bin - state: link - - name: Symlinking libidn.so.12 to /usr/lib - ansible.builtin.file: - src: /usr/local/lib/libidn.so.12 - dest: /usr/lib/libidn.so.12 - owner: root - group: bin - state: link - when: libidn_archive is defined - - name: Copy the thp-config.sh script copy: src=thp-config.sh dest=/usr/local/bin/ydbd-thp-config.sh @@ -84,20 +57,25 @@ state: started name: ydb-hugepages -- name: Ensure kernel-modules-extra is installed on RHEL variants - ansible.builtin.package: - name: kernel-modules-extra - state: present - when: ansible_distribution == 'CentOS' or ansible_distribution == 'AlmaLinux' - -- name: Copy the kernel module customization for htcp congestion - copy: src=modules-ydbd.conf dest=/etc/modules-load.d/50-ydbd.conf - -- name: Copy the network sysctl adjustments - copy: src=sysctl-ydbd.conf dest=/etc/sysctl.d/50-ydbd.conf - -- name: Load the tcp_htcp module - command: "/sbin/modprobe tcp_htcp" +- name: Congestion control kernel module setup + block: + - name: Ensure kernel-modules-extra is installed on RHEL variants + ansible.builtin.package: + name: kernel-modules-extra + state: present + when: ansible_distribution == 'CentOS' or ansible_distribution == 'AlmaLinux' + - name: Build the kernel module customization for congestion control + template: + src: network-modules.j2 + dest: /etc/modules-load.d/50-ydbd.conf + - name: Load the {{ ydb_congestion_module }} module + command: "/sbin/modprobe {{ ydb_congestion_module }}" + when: ydb_congestion_module is defined + +- name: Build the kernel settings adjustments file + template: + src: kernel-settings.j2 + dest: /etc/sysctl.d/50-ydbd.conf - name: Refresh the current sysctl settings command: "/sbin/sysctl --system" @@ -117,19 +95,19 @@ owner: ydb - name: Create the YDB release directory - file: state=directory path={{ ydb_dir }}/release group=bin owner=root mode=755 + file: state=directory path={{ ydb_dir }}/release group=bin owner=root mode='755' - name: Create the YDB configuration directory - file: state=directory path={{ ydb_dir }}/cfg group=bin owner=root mode=755 + file: state=directory path={{ ydb_dir }}/cfg group=bin owner=root mode='755' -- name: Create the YDB audit directory - file: state=directory path={{ ydb_dir }}/audit group=ydb owner=ydb mode=700 +- name: Create the YDB audit base directory + file: state=directory path={{ ydb_dir }}/audit group=ydb owner=ydb mode='700' - name: Create the YDB certs directory - file: state=directory path={{ ydb_dir }}/certs group=ydb owner=ydb mode=700 + file: state=directory path={{ ydb_dir }}/certs group=ydb owner=ydb mode='700' - name: Create the YDB configuration backup directory - file: state=directory path={{ ydb_dir }}/reserve group=ydb owner=ydb mode=700 + file: state=directory path={{ ydb_dir }}/reserve group=ydb owner=ydb mode='700' - name: Create the YDB server binary directory file: state=directory @@ -160,7 +138,7 @@ force=yes - name: Create the YDB CLI default binary directory - file: state=directory path={{ ydb_dir }}/home/ydb/bin recurse=true group=ydb owner=ydb mode=700 + file: state=directory path={{ ydb_dir }}/home/ydb/bin recurse=true group=ydb owner=ydb mode='700' - name: Disable YDB CLI version checks copy: src=ydb-cli-config.json dest={{ ydb_dir }}/home/ydb/bin/config.json group=ydb owner=ydb diff --git a/files/sysctl-ydbd.conf b/roles/ydbd/templates/kernel-settings.j2 similarity index 72% rename from files/sysctl-ydbd.conf rename to roles/ydbd/templates/kernel-settings.j2 index 026dd43..6c954d1 100644 --- a/files/sysctl-ydbd.conf +++ b/roles/ydbd/templates/kernel-settings.j2 @@ -1,7 +1,5 @@ -net.ipv4.tcp_allowed_congestion_control = reno cubic htcp -net.ipv4.tcp_congestion_control = htcp - -net.ipv4.tcp_max_orphans=262144 +#jinja2: trim_blocks:False +net.ipv4.tcp_max_orphans = 262144 net.core.rmem_max = 26214400 net.core.rmem_default = 6250000 @@ -33,3 +31,8 @@ net.ipv4.tcp_fin_timeout = 9 net.ipv4.tcp_keepalive_intvl = 30 net.ipv4.tcp_keepalive_probes = 3 net.ipv4.tcp_keepalive_time = 300 + +{%- if ydb_congestion_setting is defined %} +net.ipv4.tcp_allowed_congestion_control = reno cubic {{ ydb_congestion_setting }} +net.ipv4.tcp_congestion_control = {{ ydb_congestion_setting }} +{%- endif %} diff --git a/roles/ydbd/templates/network-modules.j2 b/roles/ydbd/templates/network-modules.j2 new file mode 100644 index 0000000..eb33df1 --- /dev/null +++ b/roles/ydbd/templates/network-modules.j2 @@ -0,0 +1 @@ +{{ ydb_congestion_module }} diff --git a/roles/ydbd_config/tasks/main.yml b/roles/ydbd_config/tasks/main.yml index 9fe08c4..8c74d72 100644 --- a/roles/ydbd_config/tasks/main.yml +++ b/roles/ydbd_config/tasks/main.yml @@ -2,35 +2,67 @@ # ydbd configuration - name: Add the configuration backup script - template: src=backup_config.j2 dest={{ ydb_dir }}/home/backup_config.sh mode=755 + template: src=backup_config.j2 dest={{ ydb_dir }}/home/backup_config.sh mode='755' - name: Add the configuration file updater script - template: src=update_config_file.j2 dest={{ ydb_dir }}/home/update_config_file.sh mode=755 + template: src=update_config_file.j2 dest={{ ydb_dir }}/home/update_config_file.sh mode='755' # ydb_config_backup is passed via extra vars, generated as "date '+%Y-%m-%d_%H-%M-%S'" - name: Backup the existing configuration files command: "{{ ydb_dir }}/home/backup_config.sh {{ ydb_config_backup }}" become: true - become_user: ydb + become_user: root when: ydb_config_backup is defined - name: Copy the TLS ca.crt - copy: src={{ ydb_tls_dir }}/ca.crt dest={{ ydb_dir }}/certs/ca.crt + copy: + src: "{{ ydb_tls_dir }}/ca.crt" + dest: "{{ ydb_dir }}/certs/ca.crt" + group: ydb + owner: root + mode: '440' -- name: Copy the TLS node.crt - copy: src={{ ydb_tls_dir }}/{{ inventory_hostname_short }}/node.crt dest={{ ydb_dir }}/certs/node.crt +- name: Public copy of ca.crt + copy: + src: "{{ ydb_tls_dir }}/ca.crt" + dest: "{{ ydb_dir }}/cfg/ca.crt" + group: root + owner: ydb + mode: '444' -- name: Copy the TLS node.key - copy: src={{ ydb_tls_dir }}/{{ inventory_hostname_short }}/node.key dest={{ ydb_dir }}/certs/node.key +- name: Identify if short hostnames should be used + set_fact: + ydb_tls_host: "{{ inventory_hostname_short if lookup('fileglob', ydb_tls_dir+'/'+inventory_hostname_short+'/node.crt', errors='warn') else inventory_hostname }}" +- name: Copy the TLS node.crt + copy: + src: "{{ ydb_tls_dir }}/{{ ydb_tls_host }}/node.crt" + dest: "{{ ydb_dir }}/certs/node.crt" + group: ydb + owner: root + mode: '440' +- name: Copy the TLS node.key + copy: + src: "{{ ydb_tls_dir }}/{{ ydb_tls_host }}/node.key" + dest: "{{ ydb_dir }}/certs/node.key" + group: ydb + owner: root + mode: '440' - name: Copy the TLS web.pem - copy: src={{ ydb_tls_dir }}/{{ inventory_hostname_short }}/web.pem dest={{ ydb_dir }}/certs/web.pem - -- name: Public copy of ca.crt - copy: src={{ ydb_tls_dir }}/ca.crt dest={{ ydb_dir }}/cfg/ca.crt + copy: + src: "{{ ydb_tls_dir }}/{{ ydb_tls_host }}/web.pem" + dest: "{{ ydb_dir }}/certs/web.pem" + group: ydb + owner: root + mode: '440' - name: Copy cluster configuration file - copy: src={{ ydb_config }} dest={{ ydb_dir }}/cfg/ydbd-config.yaml group=ydb owner=root mode=440 + copy: + src: "{{ ydb_config }}" + dest: "{{ ydb_dir }}/cfg/ydbd-config.yaml" + group: ydb + owner: root + mode: '440' when: ydb_config is defined - name: Build dynamic node configuration file diff --git a/roles/ydbd_config/templates/backup_config.j2 b/roles/ydbd_config/templates/backup_config.j2 index fec6f88..923dba7 100644 --- a/roles/ydbd_config/templates/backup_config.j2 +++ b/roles/ydbd_config/templates/backup_config.j2 @@ -35,5 +35,7 @@ preserve ${YDB_DIR}/certs/web.pem preserve ${YDB_DIR}/cfg/ydbd-config.yaml preserve ${YDB_DIR}/cfg/ydbd-static.yaml preserve ${YDB_DIR}/cfg/ydbd-dynamic.yaml +chown ydb:ydb ${BACKUP_DIR}/* +chmod 400 ${BACKUP_DIR}/* # End Of File \ No newline at end of file diff --git a/roles/ydbd_config_dynamic/tasks/main.yml b/roles/ydbd_config_dynamic/tasks/main.yml index a570e8e..ad76e74 100644 --- a/roles/ydbd_config_dynamic/tasks/main.yml +++ b/roles/ydbd_config_dynamic/tasks/main.yml @@ -2,7 +2,7 @@ # ydbd dynamic nodes configuration - name: Copy cluster configuration file - copy: src={{ ydb_config }} dest={{ ydb_dir }}/cfg/ydbd-config.yaml group=ydb owner=root mode=440 + copy: src={{ ydb_config }} dest={{ ydb_dir }}/cfg/ydbd-config.yaml group=ydb owner=root mode='440' when: ydb_config is defined - name: Build dynamic node configuration file diff --git a/roles/ydbd_config_static/tasks/main.yml b/roles/ydbd_config_static/tasks/main.yml index 2c599e2..278141d 100644 --- a/roles/ydbd_config_static/tasks/main.yml +++ b/roles/ydbd_config_static/tasks/main.yml @@ -2,7 +2,7 @@ # ydbd static nodes configuration - name: Copy cluster configuration file - copy: src={{ ydb_config }} dest={{ ydb_dir }}/cfg/ydbd-config.yaml group=ydb owner=root mode=440 + copy: src={{ ydb_config }} dest={{ ydb_dir }}/cfg/ydbd-config.yaml group=ydb owner=root mode='440' when: ydb_config is defined - name: Build static node configuration file diff --git a/roles/ydbd_dynamic/tasks/main.yml b/roles/ydbd_dynamic/tasks/main.yml index e04dfe0..9828a88 100644 --- a/roles/ydbd_dynamic/tasks/main.yml +++ b/roles/ydbd_dynamic/tasks/main.yml @@ -2,7 +2,11 @@ # ydbd dynamic node deployment - name: Create the dynamic node startup script - template: src=wait_dynamic.j2 dest={{ ydb_dir }}/home/wait_dynamic.sh mode=755 + template: src=wait_dynamic.j2 dest={{ ydb_dir }}/home/wait_dynamic.sh mode='755' + +- name: Create the YDB audit directories for database nodes + file: state=directory path={{ ydb_dir }}/audit/{{ ydb_dbname }}-{{ item.instance }} group=ydb owner=ydb mode='700' + loop: "{{ ydb_dynnodes }}" - name: Generate the dynamic node service files template: @@ -31,3 +35,9 @@ - name: Cleanup the transferred secrets file: state=absent path={{ ydb_dir }}/certs/secret + +- name: Enable the dynamic nodes for automatic startup + ansible.builtin.systemd: + enabled: true + name: "ydbd-{{ ydb_dbname }}-{{ item.instance }}" + loop: "{{ ydb_dynnodes }}" diff --git a/roles/ydbd_dynamic/templates/dynnode-service.j2 b/roles/ydbd_dynamic/templates/dynnode-service.j2 index 77fe200..b534cca 100644 --- a/roles/ydbd_dynamic/templates/dynnode-service.j2 +++ b/roles/ydbd_dynamic/templates/dynnode-service.j2 @@ -10,16 +10,15 @@ StartLimitBurst=15 Restart=always RestartSec=1 User=ydb +UMask=077 PermissionsStartOnly=true -StandardOutput=syslog -StandardError=syslog SyslogIdentifier=ydbd SyslogFacility=daemon SyslogLevel=err +WorkingDirectory={{ ydb_dir }}/audit/{{ ydb_dbname }}-{{ item.instance }} Environment=LD_LIBRARY_PATH={{ ydb_dir }}/lib -ExecStart={{ ydb_dir }}/bin/ydbd server --cluster-name "{{ ydb_cluster }}" \ - --yaml-config {{ ydb_dir }}/cfg/ydbd-dynamic.yaml --ca {{ ydb_dir }}/certs/ca.crt \ - --grpc-ca {{ ydb_dir }}/certs/ca.crt --mon-cert {{ ydb_dir }}/certs/web.pem \ +ExecStart={{ ydb_dir }}/bin/ydbd server --tenant /{{ ydb_domain }}/{{ ydb_dbname }} --tcp \ + --yaml-config {{ ydb_dir }}/cfg/ydbd-dynamic.yaml \ {%- if 'offset' in item %} --grpcs-port {{ 2136 + item.offset }} --ic-port {{ 19002 + item.offset }} --mon-port {{ 8766 + item.offset }} \ {%- else %} @@ -29,9 +28,10 @@ ExecStart={{ ydb_dir }}/bin/ydbd server --cluster-name "{{ ydb_cluster }}" \ --node-broker grpcs://{{ brk }}:2135 \ {%- endfor %} {%- if ydb_node_dc is defined %} - --data-center {{ ydb_node_dc }} --rack {{ ydb_node_rack }} --body {{ ydb_node_body }} + --data-center {{ ydb_node_dc }} --rack {{ ydb_node_rack }} --body {{ ydb_node_body }} \ {%- endif %} - --tenant /{{ ydb_domain }}/{{ ydb_dbname }} + --ca {{ ydb_dir }}/certs/ca.crt --grpc-ca {{ ydb_dir }}/certs/ca.crt \ + --mon-cert {{ ydb_dir }}/certs/web.pem LimitNOFILE=65536 LimitCORE=0 LimitMEMLOCK=32212254720 diff --git a/roles/ydbd_init/tasks/main.yml b/roles/ydbd_init/tasks/main.yml new file mode 100644 index 0000000..1693569 --- /dev/null +++ b/roles/ydbd_init/tasks/main.yml @@ -0,0 +1,44 @@ +--- +# ydbd storage initialization + +- name: Execute disk formatting + command: "{{ ydb_dir }}/home/safe_format.sh {{ item['name'] }} {{ item['label'] }}" + with_items: "{{ ydb_disks }}" + +- name: Start the storage nodes + ansible.builtin.systemd: + state: started + name: ydbd-storage + any_errors_fatal: true + +- name: Wait for storage node startup + command: "{{ ydb_dir }}/home/wait_initial.sh {{ inventory_hostname }}" + become: true + become_user: ydb + any_errors_fatal: true + +- name: Initial YDB cluster setup + block: + - name: Initialize the YDB storage + command: "{{ ydb_dir }}/home/init_storage.sh {{ inventory_hostname }}" + become: true + become_user: ydb + - name: Wait for storage bootstrap to complete + command: "{{ ydb_dir }}/home/wait_normal.sh {{ inventory_hostname }}" + become: true + become_user: ydb + - name: Transfer the secrets + copy: src=secret dest={{ ydb_dir }}/certs/secret + - name: Set the initial cluster password + command: "{{ ydb_dir }}/home/init_password.sh {{ inventory_hostname }}" + become: true + become_user: ydb + run_once: true + +- name: Cleanup the transferred secrets + file: state=absent path={{ ydb_dir }}/certs/secret + +- name: Enable the storage nodes for automatic startup + ansible.builtin.systemd: + enabled: true + name: ydbd-storage diff --git a/roles/ydbd_static/tasks/main.yml b/roles/ydbd_static/tasks/main.yml index a756a48..0171764 100644 --- a/roles/ydbd_static/tasks/main.yml +++ b/roles/ydbd_static/tasks/main.yml @@ -1,66 +1,32 @@ --- # ydbd static node deployment +- name: Create the YDB audit directory for the storage node + file: state=directory path={{ ydb_dir }}/audit/storage group=ydb owner=ydb mode='700' + - name: Generate the static node service files template: src: static-service.j2 dest: "/etc/systemd/system/ydbd-storage.service" - name: Add the disk formatting script - template: src=safe_format.j2 dest={{ ydb_dir }}/home/safe_format.sh mode=755 + template: src=safe_format.j2 dest={{ ydb_dir }}/home/safe_format.sh mode='755' - name: Add the static node startup wait script - template: src=wait_initial.j2 dest={{ ydb_dir }}/home/wait_initial.sh mode=755 + template: src=wait_initial.j2 dest={{ ydb_dir }}/home/wait_initial.sh mode='755' - name: Add the storage recovery wait script - template: src=wait_normal.j2 dest={{ ydb_dir }}/home/wait_normal.sh mode=755 + template: src=wait_normal.j2 dest={{ ydb_dir }}/home/wait_normal.sh mode='755' - name: Add the storage initialization script - template: src=init_storage.j2 dest={{ ydb_dir }}/home/init_storage.sh mode=755 + template: src=init_storage.j2 dest={{ ydb_dir }}/home/init_storage.sh mode='755' - name: Add the password initialization script - template: src=init_password.j2 dest={{ ydb_dir }}/home/init_password.sh mode=755 + template: src=init_password.j2 dest={{ ydb_dir }}/home/init_password.sh mode='755' - name: Add the database creation script - template: src=create_database.j2 dest={{ ydb_dir }}/home/create_database.sh mode=755 - -- name: Execute disk formatting - command: "{{ ydb_dir }}/home/safe_format.sh {{ item['name'] }} {{ item['label'] }}" - with_items: "{{ ydb_disks }}" + template: src=create_database.j2 dest={{ ydb_dir }}/home/create_database.sh mode='755' - name: Refresh systemd services configuration ansible.builtin.systemd: daemon_reload: true - -- name: Start the storage node - ansible.builtin.systemd: - state: started - name: ydbd-storage - any_errors_fatal: true - -- name: Wait for storage node startup - command: "{{ ydb_dir }}/home/wait_initial.sh {{ inventory_hostname }}" - become: true - become_user: ydb - any_errors_fatal: true - -- name: Initial YDB cluster setup - block: - - name: Initialize the YDB storage - command: "{{ ydb_dir }}/home/init_storage.sh {{ inventory_hostname }}" - become: true - become_user: ydb - - name: Wait for storage bootstrap to complete - command: "{{ ydb_dir }}/home/wait_normal.sh {{ inventory_hostname }}" - become: true - become_user: ydb - - name: Transfer the secrets - copy: src=secret dest={{ ydb_dir }}/certs/secret - - name: Set the initial cluster password - command: "{{ ydb_dir }}/home/init_password.sh {{ inventory_hostname }}" - become: true - become_user: ydb - run_once: true - -- name: Cleanup the transferred secrets - file: state=absent path={{ ydb_dir }}/certs/secret diff --git a/roles/ydbd_static/templates/static-service.j2 b/roles/ydbd_static/templates/static-service.j2 index be34b2c..2dd9b36 100644 --- a/roles/ydbd_static/templates/static-service.j2 +++ b/roles/ydbd_static/templates/static-service.j2 @@ -10,13 +10,15 @@ StartLimitBurst=15 Restart=always RestartSec=1 User=ydb +UMask=077 PermissionsStartOnly=true SyslogIdentifier=ydbd SyslogFacility=daemon SyslogLevel=err +WorkingDirectory={{ ydb_dir }}/audit/storage Environment=LD_LIBRARY_PATH={{ ydb_dir }}/lib -ExecStart={{ ydb_dir }}/bin/ydbd server --node static --log-level 3 --syslog --tcp \ - --cluster-name "{{ ydb_cluster }}" --yaml-config {{ ydb_dir }}/cfg/ydbd-static.yaml \ +ExecStart={{ ydb_dir }}/bin/ydbd server --node static --tcp \ + --yaml-config {{ ydb_dir }}/cfg/ydbd-static.yaml \ --grpcs-port 2135 --ic-port 19001 --mon-port 8765 --mon-cert {{ ydb_dir }}/certs/web.pem LimitNOFILE=65536 LimitCORE=0 diff --git a/roles/ydbd_static/templates/wait_initial.j2 b/roles/ydbd_static/templates/wait_initial.j2 index fdd3c6d..6ddb9f6 100644 --- a/roles/ydbd_static/templates/wait_initial.j2 +++ b/roles/ydbd_static/templates/wait_initial.j2 @@ -4,7 +4,11 @@ set +e set +u -DB_ENDPOINT=grpcs://"$1":2135 +if [ -z "$1" ]; then + DB_ENDPOINT=grpcs://`hostname -f`:2135 +else + DB_ENDPOINT=grpcs://"$1":2135 +fi DB_DOMAIN=/{{ ydb_domain }} CAFILE={{ ydb_dir }}/certs/ca.crt PASSFILE={{ ydb_dir }}/certs/secret diff --git a/roles/ydbd_static/templates/wait_normal.j2 b/roles/ydbd_static/templates/wait_normal.j2 index 49f8274..74e9dd6 100644 --- a/roles/ydbd_static/templates/wait_normal.j2 +++ b/roles/ydbd_static/templates/wait_normal.j2 @@ -4,7 +4,11 @@ set +e set +u -DB_ENDPOINT=grpcs://"$1":2135 +if [ -z "$1" ]; then + DB_ENDPOINT=grpcs://`hostname -f`:2135 +else + DB_ENDPOINT=grpcs://"$1":2135 +fi DB_DOMAIN=/{{ ydb_domain }} CAFILE={{ ydb_dir }}/certs/ca.crt PASSFILE={{ ydb_dir }}/certs/secret diff --git a/roles/ydbd_unified_agent/tasks/main.yml b/roles/ydbd_unified_agent/tasks/main.yml new file mode 100644 index 0000000..624138e --- /dev/null +++ b/roles/ydbd_unified_agent/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# unified agent installation + +- name: Create the unified agent base directory + file: state=directory path={{ unified_agent_home }} group=bin owner=root mode='755' + +- name: Create the unified agent storage directory + file: state=directory path={{ unified_agent_home }}/storage group=ydb owner=ydb mode='700' + +- name: Create the technical logs directory + file: state=directory path={{ ydb_dir }}/logs group=ydb owner=ydb mode='750' + +- name: Copy the unified agent binary + copy: src={{ unified_agent_binary }} dest={{ unified_agent_home }}/unified_agent group=bin owner=root mode='555' + +- name: Generate the unified agent service files + template: + src: unified-agent-service.j2 + dest: "/etc/systemd/system/ydb-unified-agent.service" + +- name: Generate the unified agent configuration + template: + src: unified-agent-config.j2 + dest: "{{ unified_agent_home }}/unified_agent.yaml" + +- name: Generate the log rotation configuration + template: + src: ydb-logs-rotation.j2 + dest: /etc/logrotate.d/ydbd + +- name: Refresh systemd services configuration + ansible.builtin.systemd: + daemon_reload: true + +- name: Activate the unified agent + ansible.builtin.systemd: + enabled: true + state: restarted + name: ydb-unified-agent diff --git a/roles/ydbd_unified_agent/templates/unified-agent-config.j2 b/roles/ydbd_unified_agent/templates/unified-agent-config.j2 new file mode 100644 index 0000000..e11751b --- /dev/null +++ b/roles/ydbd_unified_agent/templates/unified-agent-config.j2 @@ -0,0 +1,72 @@ +#jinja2: trim_blocks:False +status: + port: 16301 + +storages: + - name: main + plugin: fs + config: + directory: {{ unified_agent_home }}/storage + max_partition_size: 50mb + max_segment_size: 10mb + +routes: + - input: + plugin: grpc + config: + uri: localhost:16400 + max_receive_message_size: 2mb + flow_control: + new_sessions_rate_limit: 50 + channel: + pipe: + - storage_ref: + name: main + - filter: + plugin: assign + config: + message: + - _payload: "{$short_host_name} ydbd[{_pid}]: {_payload}" + fanout: + - channel: + pipe: + - filter: + plugin: match + config: + session: + _log_name: audit + - filter: + plugin: batch + config: + delimiter: "\n" + flush_period: 500ms + limit: + count: 100 + output: + id: output_audit + plugin: debug + config: + file_name: {{ ydb_dir }}/audit/ydb-audit.log + delimiter: "\n" + _test: {register_test_handlers: true} + - channel: + pipe: + - filter: + plugin: match + config: + session: + _log_name: main + - filter: + plugin: batch + config: + delimiter: "\n" + flush_period: 500ms + limit: + count: 100 + output: + id: output_main + plugin: debug + config: + file_name: {{ ydb_dir }}/logs/ydbd-main.log + delimiter: "\n" + _test: {register_test_handlers: true} diff --git a/roles/ydbd_unified_agent/templates/unified-agent-service.j2 b/roles/ydbd_unified_agent/templates/unified-agent-service.j2 new file mode 100644 index 0000000..94c7ecb --- /dev/null +++ b/roles/ydbd_unified_agent/templates/unified-agent-service.j2 @@ -0,0 +1,22 @@ +#jinja2: trim_blocks:False +[Unit] +Description=YDB unified agent +After=network-online.target rc-local.service +Wants=network-online.target +StartLimitInterval=10 +StartLimitBurst=15 + +[Service] +Restart=always +RestartSec=1 +User=ydb +PermissionsStartOnly=true +SyslogIdentifier=ydb-unified-agent +SyslogFacility=daemon +SyslogLevel=err +ExecStart={{ unified_agent_home }}/unified_agent --config {{ unified_agent_home }}/unified_agent.yaml +LimitNOFILE=65536 +LimitCORE=0 + +[Install] +WantedBy=multi-user.target diff --git a/roles/ydbd_unified_agent/templates/ydb-logs-rotation.j2 b/roles/ydbd_unified_agent/templates/ydb-logs-rotation.j2 new file mode 100644 index 0000000..043dd37 --- /dev/null +++ b/roles/ydbd_unified_agent/templates/ydb-logs-rotation.j2 @@ -0,0 +1,25 @@ +{{ ydb_dir }}/audit/ydb-audit.log { + daily + missingok + maxsize 500M + compress + delaycompress + rotate 10 + notifempty + postrotate + curl http://localhost:16301/output_audit/reopen_file + endscript +} + +{{ ydb_dir }}/logs/ydbd-main.log { + daily + missingok + maxsize 500M + compress + delaycompress + rotate 10 + notifempty + postrotate + curl http://localhost:16301/output_main/reopen_file + endscript +} diff --git a/run-binaries-all.sh b/run-binaries-all.sh new file mode 100755 index 0000000..049263a --- /dev/null +++ b/run-binaries-all.sh @@ -0,0 +1,3 @@ +#! /bin/sh + +ansible-playbook -b -i hosts -f 20 binaries-all.yaml diff --git a/run-unified-agent.sh b/run-unified-agent.sh new file mode 100755 index 0000000..69bb70d --- /dev/null +++ b/run-unified-agent.sh @@ -0,0 +1,3 @@ +#! /bin/sh + +ansible-playbook -b -i hosts -f 20 unified-agent.yaml diff --git a/unified-agent.yaml b/unified-agent.yaml new file mode 100644 index 0000000..8276284 --- /dev/null +++ b/unified-agent.yaml @@ -0,0 +1,3 @@ +- hosts: all + roles: + - role: ydbd_unified_agent