From 41a4ba2a11a0170a288a21f22b2a8da2e3f3325c Mon Sep 17 00:00:00 2001 From: Dipankar Das <65275144+dipankardas011@users.noreply.github.com> Date: Tue, 30 Apr 2024 14:26:36 +0530 Subject: [PATCH 01/44] feat: added install script Signed-off-by: Dipankar Das <65275144+dipankardas011@users.noreply.github.com> --- scripts/install.sh | 313 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 313 insertions(+) create mode 100755 scripts/install.sh diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 00000000..99759a9c --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,313 @@ +#!/bin/bash +PS4='+\[\033[0;33m\](\[\033[0;36m\]${BASH_SOURCE##*/}:${LINENO}\[\033[0;33m\])\[\033[0m\] ' + +set -xe + +print_in_color() { + local color="$1" + shift + if [[ -z ${NO_COLOR+x} ]]; then + printf "$color%b\e[0m\n" "$*" + else + printf "%b\n" "$*" + fi +} + +red() { print_in_color "\e[31m" "$*"; } +green() { print_in_color "\e[32m" "$*"; } +yellow() { print_in_color "\e[33m" "$*"; } +blue() { print_in_color "\e[34m" "$*"; } +magenta() { print_in_color "\e[35m" "$*"; } +cyan() { print_in_color "\e[36m" "$*"; } +bold() { print_in_color "\e[1m" "$*"; } +underlined() { print_in_color "\e[4m" "$*"; } + +info_blue(){ + echo -e $'\U0001F6A7' "$(blue "$*")" +} + +info_green(){ + echo "$(cyan '=>') $(green "$*")" +} + +info_red(){ + echo "$(red 'X') $(red "$*")" +} + +isRoot() { + if [ "$(id -u)" -eq 0 ]; then + info_red "You are running as root." + exit 69 + fi +} + +has_docker_compose(){ + declare desc="return 0 if we have docker compose command" + if [[ "$(dockexr compose version 2>&1 || true)" = *"docker: 'compose' is not a docker command."* ]]; then + return 1 + else + return 0 + fi +} + +has_pyenv(){ + declare desc="return 0 if we have docker compose command" + if [[ "$(pyenv --version 2>&1 || true)" = *"pyenv: command not found"* ]]; then + return 1 + else + return 0 + fi +} + +has_tty() { + declare desc="return 0 if we have a tty" + if [[ "$(/usr/bin/tty || true)" == "not a tty" ]]; then + return 1 + else + return 0 + fi +} +if has_tty; then + if ! [[ "${INTERACTIVE:-}" ]]; then + export DEBIAN_FRONTEND=noninteractive + fi +else + export DEBIAN_FRONTEND=noninteractive +fi + +if [[ "${DEBUG:-}" -eq 1 ]]; then + set -x +fi + +# Function to install Homebrew on macOS +install_homebrew() { + if ! type brew > /dev/null 2>&1; then + NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" else + else + info_green "brew is already installed." + fi +} + +# Function to check and install Docker Desktop on macOS +install_docker_macos() { + if ! type docker > /dev/null 2>&1; then + info_blue "Installing Docker Desktop for macOS..." + install_homebrew + brew install --cask docker + # The Docker app needs to be opened to complete the installation, which can't be fully automated + info_green "Docker Desktop installed." + else + info_green "Docker Desktop is already installed." + fi +} + +# Function to check and install Docker Engine and Docker Compose on Ubuntu +install_docker_ubuntu() { + ARCH=$(dpkg --print-architecture) # Detects the architecture (amd64, arm64, etc.) + if command -v docker > /dev/null; then + info_green "Docker is already installed." + else + info_blue "Installing Docker Engine for Ubuntu..." + + # Add Docker's official GPG key: + sudo DEBIAN_FRONTEND=noninteractive apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y ca-certificates curl + sudo DEBIAN_FRONTEND=noninteractive install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo DEBIAN_FRONTEND=noninteractive chmod a+r /etc/apt/keyrings/docker.asc + + # Add the repository to Apt sources: + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + + sudo DEBIAN_FRONTEND=noninteractive apt-get update + + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin + + info_green "Docker Engine installed" + fi + + # Check if the docker group exists, create it if not + if ! getent group docker > /dev/null; then + info_green "Docker group does not exist. Creating docker group..." + sudo groupadd docker + fi + + # Check if $USER is in the docker group + if id -nG "$USER" | grep -qw docker; then + info_green "$USER is already a member of the docker group." + else + info_blue "$USER is not a member of the docker group. Adding $USER to the docker group..." + sudo usermod -aG docker "$USER" + info_green "$USER has been added to the docker group." + fi + + if ! has_docker_compose; then + info_blue "Installing Docker Compose..." + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose-plugin + info_green "Docker Compose installed." + else + info_green "Docker Compose is already installed." + fi + + sudo systemctl enable docker.service + sudo systemctl start docker.service +} + +install_pyenv_python(){ + + if ! type python3 > /dev/null 2>&1 || ! python3 -c 'import sys; assert sys.version_info >= (3,10)' 2>/dev/null; then + LATEST_PYTHON=$(pyenv install --list | grep -v - | grep -v b | grep -v a | grep -v rc | grep -E '^\s*3' | tail -1 | tr -d '[:space:]') + if [ -z "$LATEST_PYTHON" ]; then + info_red "Could not find the latest Python version." + exit 1 + fi + info_blue "Latest Python version available is: $LATEST_PYTHON" + + # Check if the latest version is already installed + if pyenv versions | grep -q "$LATEST_PYTHON"; then + info_green "Python $LATEST_PYTHON is already installed." + else + info_blue "Installing Python $LATEST_PYTHON..." + # Install the latest Python version. Adjust this line if you need custom configuration or want to handle output differently. + pyenv install "${LATEST_PYTHON}" + pyenv global "${LATEST_PYTHON}" + info_green "Python $LATEST_PYTHON installed." + fi + fi + + if ! type pip3 > /dev/null 2>&1; then + info_blue "Installing pip3..." + python -m ensurepip --upgrade + info_green "Installed pip3" + fi +} + +# Function to install Python and frappe-manager on Ubuntu +install_python_and_frappe_ubuntu() { + if ! has_pyenv; then + if ! type python3 > /dev/null 2>&1 || ! python3 -c 'import sys; assert sys.version_info >= (3,10)' 2>/dev/null; then + # is pyenv installed + info_blue "Using $(yellow 'apt') for installing python3..." + sudo DEBIAN_FRONTEND=noninteractive apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y python3 + info_green "Installed python3" + else + info_green "python 3.10 or higher is already installed." + fi + + if ! type pip3 > /dev/null 2>&1; then + info_blue "Using $(yellow 'apt') for installing pip3..." + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y python3-pip + info_green "Installed pip3" + else + info_green "pip3 already installed." + fi + else + info_blue "Pyenv detected. Using it to install python" + install_pyenv_python + fi + + info_blue "Installing frappe-manager..." + pip3 install --user --upgrade frappe-manager + info_green "$(bold 'fm' $(pip3 list | grep frappe-manager | awk '{print $2}')) installed." +} + +# Function to install Python and frappe-manager on macOS +install_python_and_frappe_macos() { + if ! has_pyenv; then + if ! type python3 > /dev/null 2>&1 || ! python3 -c 'import sys; assert sys.version_info >= (3,10)' 2>/dev/null; then + info_blue "Using $(yellow 'brew') for installing python..." + brew install python + info_green "$(python3 -V) installed" + else + info_green "python 3.10 or higher is already installed." + fi + + if ! type pip3 > /dev/null 2>&1; then + info_blue "Using $(yellow 'brew') for installing pip3..." + python -m ensurepip --upgrade + info_green "Installed pip3" + else + info_green "pip3 already installed." + fi + else + info_blue "Pyenv detected. Using it to install python" + install_pyenv_python + fi + + info_blue "Installing frappe-manager..." + pip3 install --user frappe-manager + info_green "$(bold 'fm' $(pip3 list | grep frappe-manager | awk '{print $2}')) installed." +} + +handle_shell(){ + local shellrc + + if ! has_pyenv; then + if [[ "${SHELL:-}" = *"bash"* ]]; then + shellrc="${HOME}/.bashrc" + elif [[ "${SHELL:-}" = *"zsh"* ]]; then + shellrc="${HOME}/.zshrc" + fi + + if [[ "${shellrc:-}" ]]; then + info_blue "Checking default pip3 dir in PATH" + if ! [[ "$PATH" = *"$HOME/.local/bin"* ]];then + export PATH="$HOME/.local/bin:$PATH" + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$shellrc" + info_green "Added $HOME/.local/bin to PATH using $shellrc file." + else + info_green "$HOME/.local/bin already in PATH." + fi + fi + info_blue "Installing fm shell completion." + $HOME/.local/bin/fm --install-completion + else + info_blue "Installing fm shell completion." + export PATH="$(pyenv root)/shims:$PATH" + fm --install-completion + fi + +} + + +# Detect OS and call the respective functions +OS="$(uname)" +if [ "$OS" == "Darwin" ]; then + install_docker_macos + install_python_and_frappe_macos + handle_shell + info_yellow "🔴 Please complete docker desktop setup before using fm." + osascript -e 'display notification "Please complete docker desktop setup before using fm." with title "🔴 fm - complete docker desktop setup."' + # start docker app + open -na /Applications/Docker.app + info_green "Script execution completed." + +elif [ "$OS" == "Linux" ]; then + . /etc/os-release + if [ "$NAME" == "Ubuntu" ]; then + isRoot + install_docker_ubuntu + install_python_and_frappe_ubuntu + handle_shell + + # Function to be executed upon script exit + function remind_logout() { + info_green "Please log out and log back from the current shell for linux group changes to take effect." + } + # Trap the EXIT signal and call remind_logout function + trap remind_logout EXIT + + info_green "Script execution completed." + sudo su - "$USER" + else + info_red "Unsupported Linux distribution. This script supports macOS and Ubuntu." + exit 1 + fi +else + info_red "Unsupported operating system. This script supports macOS and Ubuntu." + exit 1 +fi From 282225438dd309b4990d749cc2306563df2ee7e0 Mon Sep 17 00:00:00 2001 From: Dipankar Das <65275144+dipankardas011@users.noreply.github.com> Date: Tue, 30 Apr 2024 14:33:09 +0530 Subject: [PATCH 02/44] feat: added install readme Signed-off-by: Dipankar Das <65275144+dipankardas011@users.noreply.github.com> --- scripts/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 scripts/README.md diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..1bdeda62 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,14 @@ +## Scripts + +### Install + +#### Linux +```bash +bash <(curl -s https://raw.githubusercontent.com/rtCamp/Frappe-Manager/automation/scripts/install.sh) +``` + +#### MacOS +```bash +zsh <(curl -s https://raw.githubusercontent.com/rtCamp/Frappe-Manager/automation/scripts/install.sh) +``` + From 6d393627432d3b95501ff091095adaff01682dc9 Mon Sep 17 00:00:00 2001 From: Dipankar Das <65275144+dipankardas011@users.noreply.github.com> Date: Tue, 30 Apr 2024 14:50:04 +0530 Subject: [PATCH 03/44] fix: missing yello info logging Signed-off-by: Dipankar Das <65275144+dipankardas011@users.noreply.github.com> --- scripts/install.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/install.sh b/scripts/install.sh index 99759a9c..8ebd9e6b 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -30,6 +30,11 @@ info_green(){ echo "$(cyan '=>') $(green "$*")" } + +info_yellow(){ + echo "$(cyan '=>') $(yellow "$*")" +} + info_red(){ echo "$(red 'X') $(red "$*")" } From 844e4b32445971c3b742d3d31d49281238a89e27 Mon Sep 17 00:00:00 2001 From: Dipankar Das <65275144+dipankardas011@users.noreply.github.com> Date: Tue, 30 Apr 2024 14:54:33 +0530 Subject: [PATCH 04/44] patch: added the upgrade of frappe pkg Signed-off-by: Dipankar Das <65275144+dipankardas011@users.noreply.github.com> --- scripts/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install.sh b/scripts/install.sh index 8ebd9e6b..c5411a4d 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -244,7 +244,7 @@ install_python_and_frappe_macos() { fi info_blue "Installing frappe-manager..." - pip3 install --user frappe-manager + pip3 install --user --upgrade frappe-manager info_green "$(bold 'fm' $(pip3 list | grep frappe-manager | awk '{print $2}')) installed." } From 4d0748c9de40e1e65a0620bcb305bb297d49efdd Mon Sep 17 00:00:00 2001 From: Dipankar Das <65275144+dipankardas011@users.noreply.github.com> Date: Tue, 30 Apr 2024 15:30:51 +0530 Subject: [PATCH 05/44] fix: added root check for macos Signed-off-by: Dipankar Das <65275144+dipankardas011@users.noreply.github.com> --- scripts/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/install.sh b/scripts/install.sh index c5411a4d..15c57583 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -282,6 +282,7 @@ handle_shell(){ # Detect OS and call the respective functions OS="$(uname)" if [ "$OS" == "Darwin" ]; then + isRoot install_docker_macos install_python_and_frappe_macos handle_shell From c6273b7af26d24a884c705359a6990b8e80eb7b5 Mon Sep 17 00:00:00 2001 From: Dipankar Das <65275144+dipankardas011@users.noreply.github.com> Date: Tue, 30 Apr 2024 15:33:52 +0530 Subject: [PATCH 06/44] patch: moved the root check Signed-off-by: Dipankar Das <65275144+dipankardas011@users.noreply.github.com> --- scripts/install.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 15c57583..7d4cf402 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -281,8 +281,8 @@ handle_shell(){ # Detect OS and call the respective functions OS="$(uname)" +isRoot if [ "$OS" == "Darwin" ]; then - isRoot install_docker_macos install_python_and_frappe_macos handle_shell @@ -295,7 +295,6 @@ if [ "$OS" == "Darwin" ]; then elif [ "$OS" == "Linux" ]; then . /etc/os-release if [ "$NAME" == "Ubuntu" ]; then - isRoot install_docker_ubuntu install_python_and_frappe_ubuntu handle_shell From 872a5525c53c3c1371334ce57ad894b3f3b7c431 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 30 May 2024 03:06:34 +0530 Subject: [PATCH 07/44] Add dynamic examples for the cli root commands --- frappe_manager/__init__.py | 23 ++++ frappe_manager/commands.py | 17 --- frappe_manager/utils/cli_examples.py | 37 +++++++ frappe_manager/utils/examples.json | 154 +++++++++++++++++++++++++++ 4 files changed, 214 insertions(+), 17 deletions(-) create mode 100644 frappe_manager/utils/cli_examples.py create mode 100644 frappe_manager/utils/examples.json diff --git a/frappe_manager/__init__.py b/frappe_manager/__init__.py index 618348cf..78a825d7 100644 --- a/frappe_manager/__init__.py +++ b/frappe_manager/__init__.py @@ -1,5 +1,28 @@ from pathlib import Path from enum import Enum +import rich +from typer.core import TyperCommand +from frappe_manager.utils.cli_examples import get_examples_from_toml +import typer.rich_utils as ut +from rich.panel import Panel + +# patch rich_format_help to display examples Panel +rich_format_help_original = ut.rich_format_help +def print_fm_examples(*,obj, ctx, markup_mode): + rich_format_help_original(obj=obj,ctx=ctx,markup_mode=markup_mode) + new_doc = get_examples_from_toml(obj.name, frappe_version=STABLE_APP_BRANCH_MAPPING_LIST['frappe']) + from rich import inspect + inspect(obj) + if isinstance(obj,TyperCommand): + rich.print(Panel( + new_doc, + padding=ut.STYLE_OPTIONS_TABLE_PADDING, + border_style=ut.STYLE_OPTIONS_PANEL_BORDER, + title='Examples', + title_align=ut.ALIGN_OPTIONS_PANEL, + )) + +ut.rich_format_help = print_fm_examples # TODO configure this using config # sites_dir = Path().home() / __name__.split(".")[0] diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index dcb3f563..17ffe4e2 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -141,7 +141,6 @@ def app_callback( ctx.obj["verbose"] = verbose ctx.obj['fm_config_manager'] = fm_config_manager - @app.command(no_args_is_help=True) def create( ctx: typer.Context, @@ -187,22 +186,6 @@ def create( # TODO Create markdown table for the below help """ Create a new bench. - - Frappe\[version-15] will be installed by default. - - [bold white on black]Examples:[/bold white on black] - - [bold]# Install frappe\[version-15][/bold] - $ [blue]fm create example[/blue] - - [bold]# Install frappe\[develop][/bold] - $ [blue]fm create example --frappe-branch develop[/blue] - - [bold]# Install frappe\[version-15], erpnext\[version-15] and hrms\[version-15][/bold] - $ [blue]fm create example --apps erpnext:version-15 --apps hrms:version-15[/blue] - - [bold]# Install frappe\[version-15], erpnext\[version-14] and hrms\[version-14][/bold] - $ [blue]fm create example --frappe-branch version-14 --apps erpnext:version-14 --apps hrms:version-14[/blue] """ services_manager: ServicesManager = ctx.obj["services"] diff --git a/frappe_manager/utils/cli_examples.py b/frappe_manager/utils/cli_examples.py new file mode 100644 index 00000000..9028f385 --- /dev/null +++ b/frappe_manager/utils/cli_examples.py @@ -0,0 +1,37 @@ +from typing import Dict, List +import json +from pathlib import Path +import importlib.resources as pkg_resources + +import rich + +def get_frappe_manager_own_files(file_path: str): + return Path(str(pkg_resources.files("frappe_manager").joinpath(file_path))) + +def get_examples_from_toml(command_name: str, frappe_version: str, toml_path: Path = get_frappe_manager_own_files('./utils/examples.json')): + file_data = toml_path.read_bytes() + data: Dict[str,List[Dict[str,str]]] = json.loads(file_data) + + example_data ={ + 'current_command' : command_name, + 'benchname' : 'example.com', + 'default_version' : frappe_version + } + + from rich.table import Table + + examples_table = Table(padding=(0,0),title=None,show_header=False,show_lines=False, box=None) + + if command_name in data: + + for element in data[command_name]: + desc = element.get('desc', 'None') + code = element.get('code', 'None') + + element_table = Table(box=None,show_lines=False) + + element_table.add_row(f"[bold cyan]{desc.format(**example_data)}[/bold cyan]") + element_table.add_row(f"[blue]:play_button:[/blue] {code.format(**example_data)}") + examples_table.add_row(element_table) + + return examples_table diff --git a/frappe_manager/utils/examples.json b/frappe_manager/utils/examples.json new file mode 100644 index 00000000..c1513e10 --- /dev/null +++ b/frappe_manager/utils/examples.json @@ -0,0 +1,154 @@ +{ + "create": [ + { + "desc": "Install Frappe with the stable {default_version} branch.", + "code": "fm {current_command} {benchname}" + }, + { + "desc": "Install Frappe with the develop branch.", + "code": "fm {current_command} {benchname} --frappe-branch develop" + }, + { + "desc": "Install Frappe, ERPNext, and HRMS with the stable {default_version} branch.", + "code": "fm {current_command} {benchname} --apps erpnext --apps hrms" + }, + { + "desc": "Install Frappe[{default_version}], ERPNext[{default_version}], and HRMS[develop].", + "code": "fm {current_command} {benchname} --apps erpnext --apps hrms:develop" + }, + { + "desc": "Use Frappe production mode based bench.", + "code": "fm {current_command} {benchname} --env prod" + }, + { + "desc": "Enable Admin Tools.", + "code": "fm {current_command} {benchname} --admin-tools enable" + }, + { + "desc": "Enable HTTPS using HTTP01 Let's Encrypt challenge certificate.", + "code": "fm {current_command} {benchname} --ssl letsencrypt --letsencrypt-preferred-challenge http01 --letsencrypt-email cloudflare@example.com" + }, + { + "desc": "Enable HTTPS using DNS01 Let's Encrypt challenge certificate.", + "code": "fm {current_command} {benchname} --ssl letsencrypt --letsencrypt-preferred-challenge dns01" + } + ], + "delete": [ + { + "desc": "Delete bench {benchname}", + "code": "fm {current_command} {benchname}" + } + ], + "stop": [ + { + "desc": "Stop bench {benchname}", + "code": "fm {current_command} {benchname}" + } + ], + "info": [ + { + "desc": "Show information about bench {benchname}", + "code": "fm {current_command} {benchname}" + } + ], + "list": [ + { + "desc": "List all available benches", + "code": "fm {current_command}" + } + ], + "code": [ + { + "desc": "Open the bench {benchname} in VSCode", + "code": "fm {current_command} {benchname}" + }, + { + "desc": "Open the bench {benchname} with force start", + "code": "fm {current_command} {benchname} -f" + }, + { + "desc": "Add custom extension other than default available", + "code": "fm {current_command} {benchname} -e vscodevim.vim" + }, + { + "desc": "Sync vscode frapep debugger configuration", + "code": "fm {current_command} {benchname} -d" + }, + { + "desc": "Use different workdir in vscode", + "code": "fm {current_command} {benchname} -w /workspace" + } + ], + "logs": [ + { + "desc": "Show logs of Frappe server", + "code": "fm {current_command} {benchname}" + }, + { + "desc": "Show logs of Frappe server and follow", + "code": "fm {current_command} {benchname} --follow" + }, + { + "desc": "Show logs of NGINX container and follow", + "code": "fm {current_command} {benchname} --service nginx --follow" + } + ], + "shell": [ + { + "desc": "Spawn shell for bench {benchname}, user - 'frappe', service - 'frappe'", + "code": "fm {current_command} {benchname}" + }, + { + "desc": "Spawn shell for bench {benchname}, user - 'root', service - 'frappe'", + "code": "fm {current_command} {benchname} --user root" + }, + { + "desc": "Spawn shell for bench {benchname}, user - 'root', service - 'nginx'", + "code": "fm {current_command} {benchname} --service nginx --user nginx" + } + ], + "start": [ + { + "desc": "Start bench {benchname}", + "code": "fm {current_command} {benchname}" + } + ], + "update": [ + { + "desc": "Enable SSL.", + "code": "fm update {benchname} --ssl letsencrypt" + }, + { + "desc": "Enable HTTPS using HTTP01 Let's Encrypt challenge certificate.", + "code": "fm {current_command} {benchname} --ssl letsencrypt --letsencrypt-preferred-challenge http01 --letsencrypt-email cloudflare@example.com" + }, + { + "desc": "Enable HTTPS using DNS01 Let's Encrypt challenge certificate.", + "code": "fm {current_command} {benchname} --ssl letsencrypt --letsencrypt-preferred-challenge dns01" + }, + { + "desc": "Toggle admin-tools enable.", + "code": "fm {current_command} {benchname} --admin-tools enable" + }, + { + "desc": "Toggle admin-tools disable.", + "code": "fm {current_command} {benchname} --admin-tools disable" + }, + { + "desc": "Switch to frappe production environment.", + "code": "fm {current_command} {benchname} --env prod" + }, + { + "desc": "Switch to frappe development environment.", + "code": "fm {current_command} {benchname} --environment dev" + }, + { + "desc": "Enable frappe developer mode.", + "code": "fm {current_command} {benchname} --developer-mode enable" + }, + { + "desc": "Disable frappe developer mode.", + "code": "fm {current_command} {benchname} --developer-mode disable" + } + ] +} From 53518829dfdbfcf21c3640192fa3b9ce9241a28b Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 3 Jun 2024 19:35:21 +0530 Subject: [PATCH 08/44] Update examples.json with all commands examples --- frappe_manager/__init__.py | 73 +++-- frappe_manager/utils/cli_examples.py | 49 +++- frappe_manager/utils/examples.json | 400 +++++++++++++++++---------- 3 files changed, 337 insertions(+), 185 deletions(-) diff --git a/frappe_manager/__init__.py b/frappe_manager/__init__.py index 78a825d7..94917289 100644 --- a/frappe_manager/__init__.py +++ b/frappe_manager/__init__.py @@ -1,26 +1,49 @@ from pathlib import Path from enum import Enum -import rich +from typing import Optional from typer.core import TyperCommand from frappe_manager.utils.cli_examples import get_examples_from_toml import typer.rich_utils as ut from rich.panel import Panel # patch rich_format_help to display examples Panel +# save the function so that recurssion doesn't occur rich_format_help_original = ut.rich_format_help -def print_fm_examples(*,obj, ctx, markup_mode): - rich_format_help_original(obj=obj,ctx=ctx,markup_mode=markup_mode) - new_doc = get_examples_from_toml(obj.name, frappe_version=STABLE_APP_BRANCH_MAPPING_LIST['frappe']) - from rich import inspect - inspect(obj) - if isinstance(obj,TyperCommand): - rich.print(Panel( - new_doc, - padding=ut.STYLE_OPTIONS_TABLE_PADDING, - border_style=ut.STYLE_OPTIONS_PANEL_BORDER, - title='Examples', - title_align=ut.ALIGN_OPTIONS_PANEL, - )) + + +def print_fm_examples(*, obj, ctx, markup_mode): + # utilising the original saved function + rich_format_help_original(obj=obj, ctx=ctx, markup_mode=markup_mode) + + print(ctx.command_path.split(' ')) + if not hasattr(ctx.parent, 'info_name'): + return + + command = ctx.parent.info_name + + sub_command: Optional[str] = None + + if command == 'fm': + command = ctx.info_name + else: + sub_command = ctx.info_name + + + new_doc = get_examples_from_toml(command=command, frappe_version=STABLE_APP_BRANCH_MAPPING_LIST["frappe"], sub_command=sub_command) + + if new_doc: + if isinstance(obj, TyperCommand): + # printing the examples at the end of the help + import rich + rich.print( + Panel( + new_doc, + padding=ut.STYLE_OPTIONS_TABLE_PADDING, + border_style=ut.STYLE_OPTIONS_PANEL_BORDER, + title="Examples", + title_align=ut.ALIGN_OPTIONS_PANEL, + ) + ) ut.rich_format_help = print_fm_examples @@ -29,14 +52,14 @@ def print_fm_examples(*,obj, ctx, markup_mode): CLI_DIR = Path.home() / "frappe" CLI_FM_CONFIG_PATH = CLI_DIR / "fm_config.toml" CLI_SITES_ARCHIVE = CLI_DIR / "archived" -CLI_LOG_DIRECTORY = CLI_DIR / 'logs' -CLI_BENCHES_DIRECTORY = CLI_DIR / 'sites' -CLI_SERVICES_DIRECTORY = CLI_DIR / 'services' +CLI_LOG_DIRECTORY = CLI_DIR / "logs" +CLI_BENCHES_DIRECTORY = CLI_DIR / "sites" +CLI_SERVICES_DIRECTORY = CLI_DIR / "services" -CLI_SERVICES_NGINX_PROXY_DIR = CLI_SERVICES_DIRECTORY / 'nginx-proxy' -CLI_SERVICES_NGINX_PROXY_SSL_DIR = CLI_SERVICES_NGINX_PROXY_DIR / 'ssl' +CLI_SERVICES_NGINX_PROXY_DIR = CLI_SERVICES_DIRECTORY / "nginx-proxy" +CLI_SERVICES_NGINX_PROXY_SSL_DIR = CLI_SERVICES_NGINX_PROXY_DIR / "ssl" -CLI_BENCH_CONFIG_FILE_NAME = 'bench_config.toml' +CLI_BENCH_CONFIG_FILE_NAME = "bench_config.toml" SSL_RENEW_BEFORE_DAYS = 30 @@ -64,12 +87,12 @@ class SiteServicesEnum(str, Enum): STABLE_APP_BRANCH_MAPPING_LIST = { - "frappe": 'version-15', - "erpnext": 'version-15', - "hrms": 'version-15', + "frappe": "version-15", + "erpnext": "version-15", + "hrms": "version-15", } class EnableDisableOptionsEnum(str, Enum): - enable = 'enable' - disable = 'disable' + enable = "enable" + disable = "disable" diff --git a/frappe_manager/utils/cli_examples.py b/frappe_manager/utils/cli_examples.py index 9028f385..893684b1 100644 --- a/frappe_manager/utils/cli_examples.py +++ b/frappe_manager/utils/cli_examples.py @@ -1,20 +1,22 @@ -from typing import Dict, List +from copy import deepcopy +from typing import Dict, List, Optional import json from pathlib import Path import importlib.resources as pkg_resources -import rich - def get_frappe_manager_own_files(file_path: str): return Path(str(pkg_resources.files("frappe_manager").joinpath(file_path))) -def get_examples_from_toml(command_name: str, frappe_version: str, toml_path: Path = get_frappe_manager_own_files('./utils/examples.json')): +def get_examples_from_toml(command: str, frappe_version: str, toml_path: Path = get_frappe_manager_own_files('./utils/examples.json'), sub_command: Optional[str] = None ): file_data = toml_path.read_bytes() data: Dict[str,List[Dict[str,str]]] = json.loads(file_data) + bench_name = 'example.com' + example_data ={ - 'current_command' : command_name, - 'benchname' : 'example.com', + 'command' : command, + 'sub_command' : sub_command, + 'benchname' : bench_name, 'default_version' : frappe_version } @@ -22,16 +24,33 @@ def get_examples_from_toml(command_name: str, frappe_version: str, toml_path: Pa examples_table = Table(padding=(0,0),title=None,show_header=False,show_lines=False, box=None) - if command_name in data: + if command in data: + examples_data = data[command] + + if isinstance(examples_data, dict): + if not sub_command: + if not 'examples'in examples_data: + return None + else: + examples_data = examples_data[sub_command] + sub_command = f" {sub_command} " + else: + sub_command = ' ' + + + if examples_data: + element_example_data = deepcopy(example_data) - for element in data[command_name]: - desc = element.get('desc', 'None') - code = element.get('code', 'None') + for element in examples_data: + desc = element.get('desc', 'None') + code = element.get('code', 'None') - element_table = Table(box=None,show_lines=False) + if 'benchname' in element: + element_example_data['benchname'] = element['benchname'] - element_table.add_row(f"[bold cyan]{desc.format(**example_data)}[/bold cyan]") - element_table.add_row(f"[blue]:play_button:[/blue] {code.format(**example_data)}") - examples_table.add_row(element_table) + element_table = Table(box=None,show_lines=False) - return examples_table + element_table.add_row(f"[bold cyan]{desc.format(**example_data)}[/bold cyan]") + element_table.add_row(f"[blue]:play_button:[/blue] fm {command}{sub_command}{element_example_data['benchname']}{code.format(**element_example_data)}") + examples_table.add_row(element_table) + return examples_table diff --git a/frappe_manager/utils/examples.json b/frappe_manager/utils/examples.json index c1513e10..7dc7e0be 100644 --- a/frappe_manager/utils/examples.json +++ b/frappe_manager/utils/examples.json @@ -1,154 +1,264 @@ { - "create": [ - { - "desc": "Install Frappe with the stable {default_version} branch.", - "code": "fm {current_command} {benchname}" + "create": { + "examples": [ + { + "desc": "Install Frappe with the stable {default_version} branch.", + "code": "" + }, + { + "desc": "Install Frappe with the develop branch.", + "code": " --frappe-branch develop" + }, + { + "desc": "Install Frappe, ERPNext, and HRMS with the stable {default_version} branch.", + "code": " --apps erpnext --apps hrms" + }, + { + "desc": "Install Frappe[{default_version}], ERPNext[{default_version}], and HRMS[develop].", "code": " --apps erpnext --apps hrms:develop" + }, + { + "desc": "Use Frappe production mode based bench.", + "code": " --env prod" + }, + { + "desc": "Enable Admin Tools.", + "code": " --admin-tools enable" + }, + { + "desc": "Enable HTTPS using HTTP01 Let's Encrypt challenge certificate.", + "code": " --ssl letsencrypt --letsencrypt-preferred-challenge http01 --letsencrypt-email cloudflare@example.com" + }, + { + "desc": "Enable HTTPS using DNS01 Let's Encrypt challenge certificate.", + "code": " --ssl letsencrypt --letsencrypt-preferred-challenge dns01" + } + ] + }, + "delete": { + "examples": [ + { + "desc": "Delete bench {benchname}", + "code": "" + } + ] + }, + "stop": { + "examples": [ + { + "desc": "Stop bench {benchname}", + "code": "" + } + ] + }, + "info": { + "examples": [ + { + "desc": "Show information about bench {benchname}", + "code": "" + } + ] + }, + "list": { + "examples": [ + { + "desc": "List all available benches", + "code": "fm {current_command}" + } + ] + }, + "code": { + "examples": [ + { + "desc": "Open the bench {benchname} in VSCode", + "code": "" + }, + { + "desc": "Open the bench {benchname} with force start", + "code": " -f" + }, + { + "desc": "Add custom extension other than default available", + "code": " -e vscodevim.vim" + }, + { + "desc": "Sync vscode frapep debugger configuration", + "code": " -d" + }, + { + "desc": "Use different workdir in vscode", + "code": " -w /workspace" + } + ] + }, + "logs": { + "examples": [ + { + "desc": "Show logs of Frappe server", + "code": "" + }, + { + "desc": "Show logs of Frappe server and follow", + "code": " --follow" + }, + { + "desc": "Show logs of NGINX container and follow", + "code": " --service nginx --follow" + } + ] + }, + "shell": { + "examples": [ + { + "desc": "Spawn shell for bench {benchname}, user - 'frappe', service - 'frappe'", + "code": "" + }, + { + "desc": "Spawn shell for bench {benchname}, user - 'root', service - 'frappe'", + "code": " --user root" + }, + { + "desc": "Spawn shell for bench {benchname}, user - 'root', service - 'nginx'", + "code": " --service nginx --user nginx" + } + ] + }, + "start": { + "examples": [ + { + "desc": "Start bench {benchname}", + "code": "" + } + ] + }, + "update": { + "examples": [ + { + "desc": "Enable SSL.", + "code": " --ssl letsencrypt" + }, + { + "desc": "Enable HTTPS using HTTP01 Let's Encrypt challenge certificate.", + "code": " --ssl letsencrypt --letsencrypt-preferred-challenge http01 --letsencrypt-email cloudflare@example.com" + }, + { + "desc": "Enable HTTPS using DNS01 Let's Encrypt challenge certificate.", + "code": " --ssl letsencrypt --letsencrypt-preferred-challenge dns01" + }, + { + "desc": "Toggle admin-tools enable.", + "code": " --admin-tools enable" + }, + { + "desc": "Toggle admin-tools disable.", + "code": " --admin-tools disable" + }, + { + "desc": "Switch to frappe production environment.", + "code": " --env prod" + }, + { + "desc": "Switch to frappe development environment.", + "code": " --environment dev" + }, + { + "desc": "Enable frappe developer mode.", + "code": " --developer-mode enable" + }, + { + "desc": "Disable frappe developer mode.", + "code": " --developer-mode disable" + } + ] + }, + "services": { + "start": { + "examples": [ + { + "desc": "Start global-db only.", + "code": " global-db" + }, + { + "desc": "Start all global services", + "code": " all" + } + ] }, - { - "desc": "Install Frappe with the develop branch.", - "code": "fm {current_command} {benchname} --frappe-branch develop" + "restart": { + "examples": [ + { + "desc": "Restart global-db only.", + "code": " global-db" + }, + { + "desc": "Restart all global services", + "code": " all" + } + ] }, - { - "desc": "Install Frappe, ERPNext, and HRMS with the stable {default_version} branch.", - "code": "fm {current_command} {benchname} --apps erpnext --apps hrms" + "shell": { + "examples": [ + { + "desc": "Shell global-db", + "code": " global-db" + }, + { + "desc": "Shell global-nginx-proxy", + "code": " global-nginx-proxy" + } + ] }, - { - "desc": "Install Frappe[{default_version}], ERPNext[{default_version}], and HRMS[develop].", - "code": "fm {current_command} {benchname} --apps erpnext --apps hrms:develop" - }, - { - "desc": "Use Frappe production mode based bench.", - "code": "fm {current_command} {benchname} --env prod" - }, - { - "desc": "Enable Admin Tools.", - "code": "fm {current_command} {benchname} --admin-tools enable" - }, - { - "desc": "Enable HTTPS using HTTP01 Let's Encrypt challenge certificate.", - "code": "fm {current_command} {benchname} --ssl letsencrypt --letsencrypt-preferred-challenge http01 --letsencrypt-email cloudflare@example.com" - }, - { - "desc": "Enable HTTPS using DNS01 Let's Encrypt challenge certificate.", - "code": "fm {current_command} {benchname} --ssl letsencrypt --letsencrypt-preferred-challenge dns01" - } - ], - "delete": [ - { - "desc": "Delete bench {benchname}", - "code": "fm {current_command} {benchname}" - } - ], - "stop": [ - { - "desc": "Stop bench {benchname}", - "code": "fm {current_command} {benchname}" - } - ], - "info": [ - { - "desc": "Show information about bench {benchname}", - "code": "fm {current_command} {benchname}" - } - ], - "list": [ - { - "desc": "List all available benches", - "code": "fm {current_command}" - } - ], - "code": [ - { - "desc": "Open the bench {benchname} in VSCode", - "code": "fm {current_command} {benchname}" - }, - { - "desc": "Open the bench {benchname} with force start", - "code": "fm {current_command} {benchname} -f" - }, - { - "desc": "Add custom extension other than default available", - "code": "fm {current_command} {benchname} -e vscodevim.vim" - }, - { - "desc": "Sync vscode frapep debugger configuration", - "code": "fm {current_command} {benchname} -d" - }, - { - "desc": "Use different workdir in vscode", - "code": "fm {current_command} {benchname} -w /workspace" + "stop": { + "examples": [ + { + "desc": "Stop global-db", + "code": " global-db" + }, + { + "desc": "Stop all services.", + "code": " all" + } + ] } - ], - "logs": [ - { - "desc": "Show logs of Frappe server", - "code": "fm {current_command} {benchname}" + }, + "ssl": { + "delete": { + "examples": [ + { + "desc": "Delete {benchname}'s ssl certificate.", + "code": "" + } + ] }, - { - "desc": "Show logs of Frappe server and follow", - "code": "fm {current_command} {benchname} --follow" - }, - { - "desc": "Show logs of NGINX container and follow", - "code": "fm {current_command} {benchname} --service nginx --follow" - } - ], - "shell": [ - { - "desc": "Spawn shell for bench {benchname}, user - 'frappe', service - 'frappe'", - "code": "fm {current_command} {benchname}" - }, - { - "desc": "Spawn shell for bench {benchname}, user - 'root', service - 'frappe'", - "code": "fm {current_command} {benchname} --user root" - }, - { - "desc": "Spawn shell for bench {benchname}, user - 'root', service - 'nginx'", - "code": "fm {current_command} {benchname} --service nginx --user nginx" + "renew": { + "examples": [ + { + "desc": "Renew all certificates.", + "code": " --all", + "benchname": "" + }, + { + "desc": "Renew specific {benchname} ssl certificate.", + "code": "" + } + ] } - ], - "start": [ - { - "desc": "Start bench {benchname}", - "code": "fm {current_command} {benchname}" - } - ], - "update": [ - { - "desc": "Enable SSL.", - "code": "fm update {benchname} --ssl letsencrypt" - }, - { - "desc": "Enable HTTPS using HTTP01 Let's Encrypt challenge certificate.", - "code": "fm {current_command} {benchname} --ssl letsencrypt --letsencrypt-preferred-challenge http01 --letsencrypt-email cloudflare@example.com" - }, - { - "desc": "Enable HTTPS using DNS01 Let's Encrypt challenge certificate.", - "code": "fm {current_command} {benchname} --ssl letsencrypt --letsencrypt-preferred-challenge dns01" - }, - { - "desc": "Toggle admin-tools enable.", - "code": "fm {current_command} {benchname} --admin-tools enable" - }, - { - "desc": "Toggle admin-tools disable.", - "code": "fm {current_command} {benchname} --admin-tools disable" - }, - { - "desc": "Switch to frappe production environment.", - "code": "fm {current_command} {benchname} --env prod" - }, - { - "desc": "Switch to frappe development environment.", - "code": "fm {current_command} {benchname} --environment dev" - }, - { - "desc": "Enable frappe developer mode.", - "code": "fm {current_command} {benchname} --developer-mode enable" - }, - { - "desc": "Disable frappe developer mode.", - "code": "fm {current_command} {benchname} --developer-mode disable" + }, + "self": { + "update": { + "examples": [ + { + "desc": "Update fm to the latest version available on pypi", + "code": "" + } + ], + "images": { + "examples": [ + { + "desc": "Update all frappe required docker images.", + "code": "" + } + ] + } } - ] + } } From 6e30036c5b2845783e73da0c878f195a771d52d0 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 3 Jun 2024 20:25:50 +0530 Subject: [PATCH 09/44] Update examples show for all subcommands and commands --- frappe_manager/__init__.py | 36 ++++++------------ frappe_manager/utils/cli_examples.py | 56 ++++++++++++++-------------- 2 files changed, 38 insertions(+), 54 deletions(-) diff --git a/frappe_manager/__init__.py b/frappe_manager/__init__.py index 94917289..df5fbba9 100644 --- a/frappe_manager/__init__.py +++ b/frappe_manager/__init__.py @@ -15,35 +15,21 @@ def print_fm_examples(*, obj, ctx, markup_mode): # utilising the original saved function rich_format_help_original(obj=obj, ctx=ctx, markup_mode=markup_mode) - print(ctx.command_path.split(' ')) - if not hasattr(ctx.parent, 'info_name'): - return + commands_stack = ctx.command_path.split(' ')[1:] - command = ctx.parent.info_name - - sub_command: Optional[str] = None - - if command == 'fm': - command = ctx.info_name - else: - sub_command = ctx.info_name - - - new_doc = get_examples_from_toml(command=command, frappe_version=STABLE_APP_BRANCH_MAPPING_LIST["frappe"], sub_command=sub_command) + new_doc = get_examples_from_toml(commands_stack=commands_stack, frappe_version=STABLE_APP_BRANCH_MAPPING_LIST["frappe"]) if new_doc: - if isinstance(obj, TyperCommand): - # printing the examples at the end of the help - import rich - rich.print( - Panel( - new_doc, - padding=ut.STYLE_OPTIONS_TABLE_PADDING, - border_style=ut.STYLE_OPTIONS_PANEL_BORDER, - title="Examples", - title_align=ut.ALIGN_OPTIONS_PANEL, - ) + import rich + rich.print( + Panel( + new_doc, + padding=ut.STYLE_OPTIONS_TABLE_PADDING, + border_style=ut.STYLE_OPTIONS_PANEL_BORDER, + title="Examples", + title_align=ut.ALIGN_OPTIONS_PANEL, ) + ) ut.rich_format_help = print_fm_examples diff --git a/frappe_manager/utils/cli_examples.py b/frappe_manager/utils/cli_examples.py index 893684b1..0888a53b 100644 --- a/frappe_manager/utils/cli_examples.py +++ b/frappe_manager/utils/cli_examples.py @@ -1,5 +1,5 @@ from copy import deepcopy -from typing import Dict, List, Optional +from typing import Dict, List import json from pathlib import Path import importlib.resources as pkg_resources @@ -7,50 +7,48 @@ def get_frappe_manager_own_files(file_path: str): return Path(str(pkg_resources.files("frappe_manager").joinpath(file_path))) -def get_examples_from_toml(command: str, frappe_version: str, toml_path: Path = get_frappe_manager_own_files('./utils/examples.json'), sub_command: Optional[str] = None ): +def get_examples_from_toml(commands_stack: List[str], frappe_version: str, toml_path: Path = get_frappe_manager_own_files('./utils/examples.json')): file_data = toml_path.read_bytes() data: Dict[str,List[Dict[str,str]]] = json.loads(file_data) bench_name = 'example.com' example_data ={ - 'command' : command, - 'sub_command' : sub_command, 'benchname' : bench_name, 'default_version' : frappe_version } - from rich.table import Table + examples_data = deepcopy(data) + + for command in commands_stack: + if not command in examples_data: + return None + + examples_data = examples_data[command] - examples_table = Table(padding=(0,0),title=None,show_header=False,show_lines=False, box=None) - if command in data: - examples_data = data[command] + if not 'examples' in examples_data: + return None - if isinstance(examples_data, dict): - if not sub_command: - if not 'examples'in examples_data: - return None - else: - examples_data = examples_data[sub_command] - sub_command = f" {sub_command} " - else: - sub_command = ' ' + examples_data = examples_data['examples'] + from rich.table import Table + + examples_table = Table(padding=(0,0),title=None,show_header=False,show_lines=False, box=None) - if examples_data: - element_example_data = deepcopy(example_data) + if examples_data: + element_example_data = deepcopy(example_data) - for element in examples_data: - desc = element.get('desc', 'None') - code = element.get('code', 'None') + for element in examples_data: + desc = element.get('desc', 'None') + code = element.get('code', 'None') - if 'benchname' in element: - element_example_data['benchname'] = element['benchname'] + if 'benchname' in element: + element_example_data['benchname'] = element['benchname'] - element_table = Table(box=None,show_lines=False) + element_table = Table(box=None,show_lines=False) - element_table.add_row(f"[bold cyan]{desc.format(**example_data)}[/bold cyan]") - element_table.add_row(f"[blue]:play_button:[/blue] fm {command}{sub_command}{element_example_data['benchname']}{code.format(**element_example_data)}") - examples_table.add_row(element_table) - return examples_table + element_table.add_row(f"[bold cyan]{desc.format(**example_data)}[/bold cyan]") + element_table.add_row(f"[blue]:play_button:[/blue] fm {' '.join(commands_stack)} {element_example_data['benchname']}{code.format(**element_example_data)}") + examples_table.add_row(element_table) + return examples_table From 2df105dda65cffec3cae74a8d4372e23db66a13d Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 3 Jun 2024 20:31:05 +0530 Subject: [PATCH 10/44] Update examples.json --- frappe_manager/utils/examples.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe_manager/utils/examples.json b/frappe_manager/utils/examples.json index 7dc7e0be..6233ffae 100644 --- a/frappe_manager/utils/examples.json +++ b/frappe_manager/utils/examples.json @@ -248,14 +248,16 @@ "examples": [ { "desc": "Update fm to the latest version available on pypi", - "code": "" + "code": "", + "benchname": "" } ], "images": { "examples": [ { "desc": "Update all frappe required docker images.", - "code": "" + "code": "", + "benchname": "" } ] } From 871c718fedd5e524de3963f03761e1d757d91154 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 4 Jun 2024 15:20:42 +0530 Subject: [PATCH 11/44] fix: correct tag push for docker images --- .github/workflows/bake-images.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/bake-images.yml b/.github/workflows/bake-images.yml index 05e04308..ca9efdea 100644 --- a/.github/workflows/bake-images.yml +++ b/.github/workflows/bake-images.yml @@ -78,6 +78,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 with: + ref: main fetch-depth: 0 - name: Set up tag From a15f382ceee837131dac9ae47ebe08c0420a8cd5 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 14 Jun 2024 16:17:55 +0530 Subject: [PATCH 12/44] fix: fm list help not showing --- frappe_manager/utils/examples.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe_manager/utils/examples.json b/frappe_manager/utils/examples.json index 6233ffae..1009b197 100644 --- a/frappe_manager/utils/examples.json +++ b/frappe_manager/utils/examples.json @@ -62,7 +62,7 @@ "examples": [ { "desc": "List all available benches", - "code": "fm {current_command}" + "code": "" } ] }, From aafc01a636de88bdcd7d05c6c7f68b8b4e55c3d8 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 17 Jun 2024 15:28:48 +0530 Subject: [PATCH 13/44] fix: add file command for `bench restore` --- Docker/frappe/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Docker/frappe/Dockerfile b/Docker/frappe/Dockerfile index 6dac0231..68551fce 100644 --- a/Docker/frappe/Dockerfile +++ b/Docker/frappe/Dockerfile @@ -73,6 +73,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --no-instal gosu \ fonts-powerline \ zsh \ + file \ && rm -rf /var/lib/apt/lists/* RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ From b5b75c0a1c83adea3d7f78fef67eec8b640d4e7d Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 17 Jun 2024 20:15:55 +0530 Subject: [PATCH 14/44] add: stop nginx-proxy from overriding acme challenge location --- frappe_manager/templates/docker-compose.services.tmpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe_manager/templates/docker-compose.services.tmpl b/frappe_manager/templates/docker-compose.services.tmpl index ca603ab5..a1e1f4a7 100644 --- a/frappe_manager/templates/docker-compose.services.tmpl +++ b/frappe_manager/templates/docker-compose.services.tmpl @@ -28,6 +28,8 @@ services: container_name: fm_global-nginx-proxy user: REPLACE_WITH_CURRENT_USER:REPLACE_WITH_CURRENT_USER_GROUP image: jwilder/nginx-proxy + environment: + ACME_HTTP_CHALLENGE_LOCATION: false ports: - "80:80" - "443:443" From 4ac5ecf3c721643fa5db91135b2099a003b01ac2 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 17 Jun 2024 20:19:35 +0530 Subject: [PATCH 15/44] Update images tags to specific version --- frappe_manager/templates/docker-compose.admin-tools.tmpl | 2 +- frappe_manager/templates/docker-compose.services.tmpl | 2 +- frappe_manager/templates/docker-compose.tmpl | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe_manager/templates/docker-compose.admin-tools.tmpl b/frappe_manager/templates/docker-compose.admin-tools.tmpl index f55bc7fd..6d613dbf 100644 --- a/frappe_manager/templates/docker-compose.admin-tools.tmpl +++ b/frappe_manager/templates/docker-compose.admin-tools.tmpl @@ -9,7 +9,7 @@ services: site-network: adminer: - image: adminer:latest + image: adminer:4 container_name: REPLACE_ME_WITH_CONTAINER_NAME environment: ADMINER_DEFAULT_SERVER: global-db diff --git a/frappe_manager/templates/docker-compose.services.tmpl b/frappe_manager/templates/docker-compose.services.tmpl index a1e1f4a7..be3e20e2 100644 --- a/frappe_manager/templates/docker-compose.services.tmpl +++ b/frappe_manager/templates/docker-compose.services.tmpl @@ -27,7 +27,7 @@ services: global-nginx-proxy: container_name: fm_global-nginx-proxy user: REPLACE_WITH_CURRENT_USER:REPLACE_WITH_CURRENT_USER_GROUP - image: jwilder/nginx-proxy + image: jwilder/nginx-proxy:1.6 environment: ACME_HTTP_CHALLENGE_LOCATION: false ports: diff --git a/frappe_manager/templates/docker-compose.tmpl b/frappe_manager/templates/docker-compose.tmpl index 17938a2a..6252d99d 100644 --- a/frappe_manager/templates/docker-compose.tmpl +++ b/frappe_manager/templates/docker-compose.tmpl @@ -87,7 +87,7 @@ services: global-backend-network: redis-cache: - image: redis:alpine + image: redis:6.2-alpine container_name: REPLACE_ME_WITH_CONTAINER_NAME volumes: - redis-cache-data:/data @@ -97,7 +97,7 @@ services: site-network: redis-queue: - image: redis:alpine + image: redis:6.2-alpine container_name: REPLACE_ME_WITH_CONTAINER_NAME volumes: - redis-queue-data:/data @@ -107,7 +107,7 @@ services: site-network: redis-socketio: - image: redis:alpine + image: redis:6.2-alpine container_name: REPLACE_ME_WITH_CONTAINER_NAME volumes: - redis-socketio-data:/data From c887dd715ab07d2d174ab9eca942f74dccb39a9b Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 11 Jun 2024 20:45:32 +0530 Subject: [PATCH 16/44] feat: move bench operations to cli from docker entrypoint --- Docker/frappe/entrypoint.sh | 11 +- Docker/frappe/frappe-dev.conf | 2 +- Docker/frappe/helper-function.sh | 26 ++ Docker/frappe/user-script.sh | 175 +--------- frappe_manager/__init__.py | 1 - frappe_manager/commands.py | 50 ++- frappe_manager/docker_wrapper/DockerClient.py | 20 +- .../docker_wrapper/DockerCompose.py | 1 + .../migration_manager/backup_manager.py | 18 +- frappe_manager/site_manager/bench_config.py | 33 +- .../site_manager/bench_site_manager.py | 324 ++++++++++++++++++ frappe_manager/site_manager/site.py | 108 ++---- .../site_manager/site_exceptions.py | 218 +++++++++++- .../workers_manager/SiteWorker.py | 33 +- frappe_manager/templates/docker-compose.tmpl | 14 +- frappe_manager/utils/callbacks.py | 6 +- frappe_manager/utils/helpers.py | 6 +- pyproject.toml | 2 +- 18 files changed, 734 insertions(+), 314 deletions(-) create mode 100644 frappe_manager/site_manager/bench_site_manager.py diff --git a/Docker/frappe/entrypoint.sh b/Docker/frappe/entrypoint.sh index 53ec18e1..b41d3bdf 100755 --- a/Docker/frappe/entrypoint.sh +++ b/Docker/frappe/entrypoint.sh @@ -26,7 +26,11 @@ update_uid_gid "${USERID}" "${USERGROUP}" "frappe" "frappe" mkdir -p /opt/user/conf.d +start_time=$(date +%s.%N) chown -R "$USERID":"$USERGROUP" /opt +end_time=$(date +%s.%N) +execution_time=$(awk "BEGIN {print $end_time - $start_time}") +echo "Time taken for chown /opt : $execution_time seconds" if [[ ! -d "/workspace/.oh-my-zsh" ]]; then cp -pr /opt/user/.oh-my-zsh /workspace/ @@ -40,9 +44,12 @@ if [[ ! -f "/workspace/.profile" ]]; then cp -p /opt/user/.profile /workspace/ fi -chown "$USERID":"$USERGROUP" /workspace /workspace/frappe-bench -ls -pA /workspace | xargs -I{} chown -R "$USERID":"$USERGROUP" /workspace/{} & +start_time=$(date +%s.%N) +chown -R "$USERID":"$USERGROUP" /workspace +end_time=$(date +%s.%N) +execution_time=$(awk "BEGIN {print $end_time - $start_time}") +echo "Time taken for chown /workspace : $execution_time seconds" if [ "$#" -gt 0 ]; then gosu "$USERID":"$USERGROUP" "/scripts/$@" & diff --git a/Docker/frappe/frappe-dev.conf b/Docker/frappe/frappe-dev.conf index 125c9aad..a23008e8 100644 --- a/Docker/frappe/frappe-dev.conf +++ b/Docker/frappe/frappe-dev.conf @@ -13,7 +13,7 @@ stopsignal=QUIT [program:frappe-bench-frappe-watch] command=/opt/user/bench-dev-watch.sh priority=4 -autostart=true +autostart=false autorestart=false stdout_logfile=/workspace/frappe-bench/logs/watch.dev.log redirect_stderr=true diff --git a/Docker/frappe/helper-function.sh b/Docker/frappe/helper-function.sh index 5d29457a..e6f2466d 100755 --- a/Docker/frappe/helper-function.sh +++ b/Docker/frappe/helper-function.sh @@ -1,5 +1,31 @@ #!/usr/bin/bash +# Function: chown directory and files +# Parameters: +# - user +# - group +chown_directory_and_files(){ + local user; user="$1" + local group; group="$2" + local dir; dir="$3" + + user_not_owned_files=$(find "$dir" ! -user "$user" -type f -exec realpath {} + | sort -u) + group_not_owned_files=$(find "$dir" ! -group "$group" -type f -exec realpath {} + | sort -u) + + user_not_owned_dirs=$(find "$dir" ! -user "$user" -type d -exec realpath {} + | sort -u) + group_not_owned_dirs=$(find "$dir" ! -group "$group" -type d -exec realpath {} + | sort -u) + + # Concatenate both lists, sort, and remove duplicates + not_owned_files=$(echo -e "$user_not_owned_files\n$group_not_owned_files" | sort -u) + not_owned_dirs=$(echo -e "$user_not_owned_dirs\n$group_not_owned_dirs" | sort -u) + + cpu_cores=$(nproc) + + echo "$not_owned_files" | xargs -P "$cpu_cores" -I{} bash -c "if [ -f {} ]; then chown ${user}:${group} {};fi" + + echo "$not_owned_dirs" | xargs -P "$cpu_cores" -I{} bash -c "if [ -d {} ]; then chown -R ${user}:${group} {};fi" +} + # Function: update_common_site_config # Description: Updates the common site config file with the provided key-value pair. # Parameters: diff --git a/Docker/frappe/user-script.sh b/Docker/frappe/user-script.sh index 72c0d524..883dacc3 100755 --- a/Docker/frappe/user-script.sh +++ b/Docker/frappe/user-script.sh @@ -1,8 +1,5 @@ #!/bin/bash -source /scripts/helper-function.sh -source /prebake_info - set -e cleanup() { @@ -24,170 +21,12 @@ emer() { exit 1 } -# handle previous logs location symlink -if [[ -d 'logs' ]]; then - if [[ -f 'logs/bench-start.log' ]]; then - mv logs/bench-start.log logs/bench-start.log.bak - fi - ln -sfn ../frappe-bench/logs/web.dev.log logs/bench-start.log -fi - -REDIS_SOCKETIO_PORT=80 -WEB_PORT=80 - -if [[ ! "${MARIADB_HOST:-}" ]]; then - MARIADB_HOST='global-db' -fi - -if [[ "${DEBUG:-}" ]]; then - set -x -fi - -if [[ ! "${MARIADB_ROOT_PASS:-}" ]]; then - MARIADB_ROOT_PASS='root' -fi - -BENCH_COMMAND='/opt/.pyenv/shims/bench' - -[[ "${ENVIRONMENT:-}" ]] || emer "[ERROR] ENVIRONMENT env not found. Please provide ENVIRONMENT env." -[[ "${WEB_PORT:-}" ]] || emer "[ERROR] WEB_PORT env not found. Please provide WEB_PORT env." -[[ "${SITENAME:-}" ]] || emer "[ERROR] SITENAME env not found. Please provide SITENAME env." - -configure_common_site_config() { - # start_time=$(date +%s.%N) - - update_common_site_config db_host "$MARIADB_HOST" - update_common_site_config db_port 3307 - update_common_site_config redis_cache "redis://${CONTAINER_NAME_PREFIX}-redis-cache:6379" - update_common_site_config redis_queue "redis://${CONTAINER_NAME_PREFIX}-redis-queue:6379" - update_common_site_config redis_socketio "redis://${CONTAINER_NAME_PREFIX}-redis-socketio:6379" - update_common_site_config webserver_port "$WEB_PORT" - update_common_site_config socketio_port "$REDIS_SOCKETIO_PORT" - update_common_site_config restart_supervisor_on_update 0 - update_common_site_config developer_mode "$DEVELOPER_MODE" - - # end_time=$(date +%s.%N) - # execution_time=$(awk "BEGIN {print $end_time - $start_time}") - # echo "Execution time for set-config : $execution_time seconds" -} - -# check if the site is created -if [[ ! -d "/workspace/frappe-bench/sites/$SITENAME" ]]; then - - [[ "${REDIS_SOCKETIO_PORT:-}" ]] || emer "[ERROR] REDIS_SOCKETIO_PORT env not found. Please provide REDIS_SOCKETIO_PORT env." - [[ "${DEVELOPER_MODE:-}" ]] || emer "[ERROR] DEVELOPER_MODE env not found. Please provide DEVELOPER_MODE env." - [[ "${MARIADB_ROOT_PASS:-}" ]] || emer "[ERROR] MARIADB_ROOT_PASS env not found. Please provide MARIADB_ROOT_PASS env." - [[ "${MARIADB_HOST:-}" ]] || emer "[ERROR] MARIADB_HOST env not found. Please provide MARIADB_HOST env." - [[ "${ADMIN_PASS:-}" ]] || emer "[ERROR] ADMIN_PASS env not found. Please provide ADMIN_PASS env." - [[ "${DB_NAME:-}" ]] || emer "[ERROR] DB_NAME env not found. Please provide DB_NAME env." - [[ "${CONTAINER_NAME_PREFIX:-}" ]] || emer "[ERROR] CONTAINER_NAME_PREFIX env not found. Please provide CONTAINER_NAME_PREFIX env." - - # setting configuration - wait-for-it -t 120 "$MARIADB_HOST":3306 - wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-cache":6379 - wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-queue":6379 - wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-socketio":6379 - - cd frappe-bench - - configure_common_site_config - - # HANDLE Frappe - if [[ ! "${FRAPPE_BRANCH}" = "${PREBAKE_FRAPPE_BRANCH}" ]]; then - bench get-app --overwrite --branch "${FRAPPE_BRANCH}" frappe - fi - - install_apps "$APPS_LIST" "$PREBAKE_APPS" - - rm -rf archived - - $BENCH_COMMAND setup supervisor --skip-redis --skip-supervisord --yes --user "$USER" - /scripts/divide-supervisor-conf.py config/supervisor.conf - - echo "Environment: ${ENVIRONMENT}" - echo "Configuring frappe server" - bench_serve_help_output=$($BENCH_COMMAND serve --help) - host_changed=$(echo "$bench_serve_help_output" | grep -c 'host' || true) - - # Addresses the introduction of the --host flag in bench serve command for compatibility with Frappe version updates. - if [[ "$host_changed" -ge 1 ]]; then - awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--host 0.0.0.0 --port "a); print}' /opt/user/bench-dev-server >file.tmp && mv file.tmp /opt/user/bench-dev-server.sh - else - awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--port "a); print}' /opt/user/bench-dev-server >file.tmp && mv file.tmp /opt/user/bench-dev-server.sh - fi - - if [[ "$DEVELOPER_MODE" == "true" ]]; then - bench setup requirements --dev - fi - - chmod +x /opt/user/bench-dev-server.sh - - $BENCH_COMMAND build & - $BENCH_COMMAND new-site --db-root-password $(cat $MARIADB_ROOT_PASS) --db-name "$DB_NAME" --db-host "$MARIADB_HOST" --admin-password "$ADMIN_PASS" --db-port 3306 --verbose --no-mariadb-socket "$SITENAME" - $BENCH_COMMAND use "$SITENAME" - $BENCH_COMMAND --site "$SITENAME" scheduler enable - - wait - - if [[ "${ENVIRONMENT}" = "dev" ]]; then - cp /opt/user/frappe-dev.conf /opt/user/conf.d/frappe-dev.conf - else - ln -sfn /workspace/frappe-bench/config/frappe-bench-frappe-web.fm.supervisor.conf /opt/user/conf.d/frappe-bench-frappe-web.fm.supervisor.conf - fi - - if [[ -n "$BENCH_START_OFF" ]]; then - tail -f /dev/null - else - supervisord -c /opt/user/supervisord.conf & - supervisord_pid=$! - wait $supervisord_pid - fi - +# fi +if [[ -n "$BENCH_START_OFF" ]]; then + tail -f /dev/null else - wait-for-it -t 120 "$MARIADB_HOST":3306 - wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-cache":6379 - wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-queue":6379 - wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-socketio":6379 - - cd frappe-bench - - echo "Environment: ${ENVIRONMENT}" - echo "Configuring frappe dev server" - # Addresses the introduction of the --host flag in bench serve command for compatibility with Frappe version updates. - bench_serve_help_output=$($BENCH_COMMAND serve --help) - - host_changed=$(echo "$bench_serve_help_output" | grep -c 'host' || true) - - $BENCH_COMMAND setup supervisor --skip-redis --skip-supervisord --yes --user "$USER" - /scripts/divide-supervisor-conf.py config/supervisor.conf - - # Addresses the introduction of the --host flag in bench serve command for compatibility with Frappe version updates. - if [[ "$host_changed" -ge 1 ]]; then - awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--host 0.0.0.0 --port "a); print}' /opt/user/bench-dev-server >file.tmp && mv file.tmp /opt/user/bench-dev-server.sh - else - awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--port "a); print}' /opt/user/bench-dev-server >file.tmp && mv file.tmp /opt/user/bench-dev-server.sh - fi - - chmod +x /opt/user/bench-dev-server.sh - - # if [[ "${ENVIRONMENT}" = "dev" ]]; then - # cp /opt/user/frappe-dev.conf /opt/user/conf.d/frappe-dev.conf - # else - # if [[ -f '/workspace/frappe-bench/config/frappe-bench-frappe-web.fm.supervisor.conf' ]]; then - - # ln -sfn /workspace/frappe-bench/config/frappe-bench-frappe-web.fm.supervisor.conf /opt/user/conf.d/frappe-bench-frappe-web.fm.supervisor.conf - # else - # emer 'Not able to start the server. /workspace/frappe-bench/config/frappe-bench-frappe-web.fm.supervisor.conf not found.' - # fi - # fi - - if [[ -n "$BENCH_START_OFF" ]]; then - tail -f /dev/null - else - echo "Starting supervisor.." - supervisord -c /opt/user/supervisord.conf & - supervisord_pid=$! - wait $supervisord_pid - fi - + echo "Starting supervisor.." + supervisord -c /opt/user/supervisord.conf & + supervisord_pid=$! + wait $supervisord_pid fi diff --git a/frappe_manager/__init__.py b/frappe_manager/__init__.py index df5fbba9..5c04a6de 100644 --- a/frappe_manager/__init__.py +++ b/frappe_manager/__init__.py @@ -10,7 +10,6 @@ # save the function so that recurssion doesn't occur rich_format_help_original = ut.rich_format_help - def print_fm_examples(*, obj, ctx, markup_mode): # utilising the original saved function rich_format_help_original(obj=obj, ctx=ctx, markup_mode=markup_mode) diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index 17ffe4e2..f27ab7c1 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -246,19 +246,16 @@ def create( admin_tools=True if environment == FMBenchEnvType.dev else False, admin_pass=admin_pass, # TODO get this info from services, maybe ? - mariadb_host=services_manager.database_manager.database_server_info.host, - mariadb_root_pass='/run/secrets/db_root_password', environment_type=environment, root_path=bench_config_path, ssl=ssl_certificate, ) compose_path = bench_path / 'docker-compose.yml' - bench_workers = BenchWorkers(benchname, bench_path) compose_file_manager = ComposeFile(compose_path) compose_project = ComposeProject(compose_file_manager, verbose) - bench: Bench = Bench(bench_path, benchname, bench_config, compose_project, bench_workers, services_manager) + bench: Bench = Bench(bench_path, benchname, bench_config, compose_project, services_manager) benches.add_bench(bench) benches.create_benches(is_template_bench=template) @@ -286,7 +283,6 @@ def delete( bench_compose_path = bench_path / 'docker-compose.yml' compose_file_manager = ComposeFile(bench_compose_path) bench_compose_project = ComposeProject(compose_file_manager) - bench_workers = BenchWorkers(benchname, bench_path) bench_config_path = bench_path / CLI_BENCH_CONFIG_FILE_NAME # try using bench object if not then create bench @@ -306,24 +302,26 @@ def delete( # TODO do something about this forcefully delete maybe admin_tools=False, admin_pass='pass', - mariadb_host='global-db', - mariadb_root_pass="/run/secrets/db_root_password", environment_type=FMBenchEnvType.dev, ssl=SSLCertificate(domain=benchname, ssl_type=SUPPORTED_SSL_TYPES.none), root_path=bench_config_path, ) + bench = Bench( bench_path, benchname, fake_config, bench_compose_project, - bench_workers, services=services_manager, workers_check=False, ) else: - bench = Bench.get_object(benchname, services_manager, workers_check=False, admin_tools_check=False) + bench = Bench.get_object( + benchname, + services=services_manager, + workers_check=False, + admin_tools_check=False) benches.add_bench(bench) benches.remove_benches() @@ -627,3 +625,37 @@ def update( if bench_config_save: bench.save_bench_config() + +@app.command() +def test( + ctx: typer.Context, + benchname: Annotated[ + Optional[str], + typer.Argument( + help="Name of the bench.", autocompletion=sites_autocompletion_callback, callback=sitename_callback + ), + ] = None, + apps: Annotated[ + List[str], + typer.Option( + "--apps", + "-a", + help="FrappeVerse apps to install. App should be specified in format : or .", + callback=apps_list_validation_callback, + show_default=False, + ), + ] = [], +): + """Shows information about given bench.""" + + services_manager = ctx.obj["services"] + verbose = ctx.obj['verbose'] + bench = Bench.get_object(benchname, services_manager) + from frappe_manager.site_manager.bench_site_manager import BenchOperations + + print(STABLE_APP_BRANCH_MAPPING_LIST.keys()) + # benchops = BenchOperations(bench) + # benchops.bench_install_app_env('crmxx') + + # benchops.is_required_services_available() + # benchops.change_frappeverse_prebaked_app_branch('frappe', 'version-14') diff --git a/frappe_manager/docker_wrapper/DockerClient.py b/frappe_manager/docker_wrapper/DockerClient.py index a56d9152..4ea554b4 100644 --- a/frappe_manager/docker_wrapper/DockerClient.py +++ b/frappe_manager/docker_wrapper/DockerClient.py @@ -1,10 +1,12 @@ import json import shlex +from sys import exception from typing import Literal, Optional, List from pathlib import Path from frappe_manager.docker_wrapper.DockerCompose import DockerComposeWrapper from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager.docker_wrapper.DockerException import DockerException from frappe_manager.utils.docker import ( SubprocessOutput, is_current_user_in_group, @@ -50,9 +52,14 @@ def version(self) -> dict: ver_cmd += parameters_to_options(parameters) - output: SubprocessOutput = run_command_with_exit_code(self.docker_cmd + ver_cmd, stream=False) - version: dict = json.loads(" ".join(output.stdout)) - return version + try: + output: SubprocessOutput = run_command_with_exit_code(self.docker_cmd + ver_cmd, stream=False) + version: dict = json.loads(" ".join(output.stdout)) + return version + except DockerException as e: + version: dict = json.loads(" ".join(e.output.stdout)) + return version + def server_running(self) -> bool: """ @@ -62,12 +69,15 @@ def server_running(self) -> bool: bool: True if the Docker server is running, False otherwise. """ docker_info = self.version() + if "Server" in docker_info: - return True + if docker_info['Server']: + return True + else: + return False else: # check if the current user in the docker group and notify the user is_current_user_in_group("docker") - return False def cp( diff --git a/frappe_manager/docker_wrapper/DockerCompose.py b/frappe_manager/docker_wrapper/DockerCompose.py index 601dd1b3..4d18ee8b 100644 --- a/frappe_manager/docker_wrapper/DockerCompose.py +++ b/frappe_manager/docker_wrapper/DockerCompose.py @@ -181,6 +181,7 @@ def exec( "env", "use_shlex_split", "capture_output", + "no_tty" ] exec_cmd += parameters_to_options(parameters, exclude=remove_parameters) diff --git a/frappe_manager/migration_manager/backup_manager.py b/frappe_manager/migration_manager/backup_manager.py index e0301c3e..1864a502 100644 --- a/frappe_manager/migration_manager/backup_manager.py +++ b/frappe_manager/migration_manager/backup_manager.py @@ -51,16 +51,23 @@ def exists(self): current_migration_timestamp = f"{datetime.now().strftime('%d-%b-%y--%H-%M-%S')}" -CLI_MIGARATIONS_DIR = CLI_DIR / 'backups' / 'migrations' / current_migration_timestamp +CLI_MIGARATIONS_DIR = CLI_DIR / 'backups' / 'migrations' class BackupManager: - def __init__(self, name, benches_dir=CLI_BENCHES_DIRECTORY, backup_dir: Path = CLI_MIGARATIONS_DIR): + def __init__( + self, + name: str, + backup_group_name: str = 'migrations', + benches_dir: Path = CLI_BENCHES_DIRECTORY, + backup_dir: Path = CLI_MIGARATIONS_DIR, + ): self.name = name - self.root_backup_dir: Path = backup_dir + self.backup_group_name = backup_group_name + self.root_backup_dir: Path = backup_dir / backup_group_name / current_migration_timestamp self.benches_dir: Path = benches_dir self.backup_dir: Path = self.root_backup_dir / self.name - self.bench_backup_dir: Path = Path('backups') / 'migrations' / current_migration_timestamp + self.bench_backup_dir: Path = Path('backups') / backup_group_name / current_migration_timestamp self.backups = [] self.logger = log.get_logger() @@ -83,6 +90,9 @@ def backup( if bench_name: dest: Path = self.benches_dir / bench_name / self.bench_backup_dir / self.name / src.name + if self.name == self.backup_group_name: + dest: Path = self.benches_dir / bench_name / self.bench_backup_dir / src.name + backup_data = BackupData(src, dest, bench=bench_name, allow_restore=allow_restore) if not backup_data.real_dest.parent.exists(): diff --git a/frappe_manager/site_manager/bench_config.py b/frappe_manager/site_manager/bench_config.py index f1373b4a..1caaa7b9 100644 --- a/frappe_manager/site_manager/bench_config.py +++ b/frappe_manager/site_manager/bench_config.py @@ -1,8 +1,9 @@ from enum import Enum import os +from frappe_manager.services_manager.database_service_manager import DatabaseServerServiceInfo import tomlkit from pathlib import Path -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional from pydantic import BaseModel, Field, model_validator, validator from frappe_manager import STABLE_APP_BRANCH_MAPPING_LIST from frappe_manager.metadata_manager import FMConfigManager, FMLetsencryptConfig @@ -44,9 +45,7 @@ class BenchConfig(BaseModel): ) admin_pass: str = Field('admin', description="The admin password") root_path: Path = Field(..., description="The root path") - mariadb_host: str = Field('global-db', description="The host for MariaDB") - mariadb_root_pass: str = Field(default='/run/secrets/db_root_password', description="The root password for MariaDB") - apps_list: List[str] = Field(default=[], description="List of apps") + apps_list: List[Dict[str,Optional[str]]] = Field(default=[], description="List of apps") userid: int = Field(default_factory=os.getuid, description="The user ID of the current process") usergroup: int = Field(default_factory=os.getgid, description="The group ID of the current process") @@ -163,21 +162,27 @@ def import_from_toml(cls, path: Path) -> "BenchConfig": return bench_config_instance + def get_commmon_site_config_data(self, db_server_info: DatabaseServerServiceInfo) -> Dict[str, Any]: + common_site_config_data = { + "install_apps": [], + "db_host" : db_server_info.host, + "db_port" : db_server_info.port, + "redis_cache" : f"redis://{self.container_name_prefix}-redis-cache:6379", + "redis_queue" : f"redis://{self.container_name_prefix}-redis-queue:6379", + "redis_socketio" : f"redis://{self.container_name_prefix}-redis-socketio:6379", + "webserver_port" : 80, + "socketio_port" : 80, + "restart_supervisor_on_update" : 0, + "developer_mode" : self.developer_mode, + } + + return common_site_config_data + def export_to_compose_inputs(self): environment = { "frappe": { "USERID": self.userid, "USERGROUP": self.usergroup, - "APPS_LIST": ",".join(self.apps_list) if self.apps_list else None, - "FRAPPE_BRANCH": self.frappe_branch, - "DEVELOPER_MODE": self.developer_mode, - "ADMIN_PASS": self.admin_pass, - "DB_NAME": self.db_name, - "SITENAME": self.name, - "MARIADB_HOST": self.mariadb_host, - "MARIADB_ROOT_PASS": self.mariadb_root_pass, - "CONTAINER_NAME_PREFIX": self.container_name_prefix, - "ENVIRONMENT": self.environment_type.value, }, "nginx": { "SITENAME": self.name, diff --git a/frappe_manager/site_manager/bench_site_manager.py b/frappe_manager/site_manager/bench_site_manager.py new file mode 100644 index 00000000..85697007 --- /dev/null +++ b/frappe_manager/site_manager/bench_site_manager.py @@ -0,0 +1,324 @@ +from collections.abc import Iterable +from pathlib import Path +from typing import Dict, List, Optional, Tuple +from frappe_manager import STABLE_APP_BRANCH_MAPPING_LIST +from frappe_manager.docker_wrapper.DockerException import DockerException +from frappe_manager.docker_wrapper.subprocess_output import SubprocessOutput +from frappe_manager.site_manager.site_exceptions import ( + BenchOperationBenchAppInSiteFailed, + BenchOperationBenchBuildFailed, + BenchOperationBenchInstallAppInPythonEnvFailed, + BenchOperationBenchRemoveAppFromPythonEnvFailed, + BenchOperationBenchSiteCreateFailed, + BenchOperationException, + BenchOperationFrappeBranchChangeFailed, + BenchOperationWaitForRequiredServiceFailed, +) +from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager.utils.docker import parameters_to_options + + +class BenchOperations: + def __init__(self, bench) -> None: + self.bench = bench + self.bench_cli_cmd = ["/opt/.pyenv/shims/bench"] + self.frappe_bench_dir = self.bench.path / "workspace" / "frappe-bench" + + def create_fm_bench(self): + richprint.change_head("Configuring common_site_config.json") + common_site_config_data = self.bench.bench_config.get_commmon_site_config_data( + self.bench.services.database_manager.database_server_info + ) + self.bench.common_bench_config_set(common_site_config_data) + richprint.print("Configured common_site_config.json") + + richprint.change_head("Configuring frappe server") + self.setup_frappe_server_config() + richprint.print("Configured frappe server") + + self.setup_supervisor(force=True) + + self.change_frappeverse_prebaked_app_branch(app="frappe", branch=self.bench.bench_config.frappe_branch) + + self.is_required_services_available() + + self.bench_install_apps(self.bench.bench_config.apps_list) + + self.frappe_container_run( + "rm -rf /workspace/frappe-bench/archived", + BenchOperationException(self.bench.name, "Failed to remove /workspace/frappe-bench/archived directory."), + ) + + richprint.change_head(f"Creating bench site {self.bench.name}") + self.create_bench_site() + richprint.print(f"Created bench site {self.bench.name}") + + self.bench_install_apps_site() + + def create_bench_site(self): + new_site_command = self.bench_cli_cmd + ["new-site"] + new_site_command += ["--db-root-password", self.bench.services.database_manager.database_server_info.password] + new_site_command += ["--db-name", self.bench.bench_config.db_name] + new_site_command += ["--db-host", self.bench.services.database_manager.database_server_info.host] + new_site_command += ["--admin-password", self.bench.bench_config.admin_pass] + new_site_command += ["--db-port", str(self.bench.services.database_manager.database_server_info.port)] + new_site_command += ["--verbose", "--no-mariadb-socket"] + new_site_command += [self.bench.name] + + new_site_command = " ".join(new_site_command) + + self.frappe_container_run( + new_site_command, raise_exception_obj=BenchOperationBenchSiteCreateFailed(self.bench.name) + ) + + self.frappe_container_run( + " ".join(self.bench_cli_cmd + [f"use {self.bench.name}"]), + raise_exception_obj=BenchOperationException( + self.bench.name, f"Failed to set {self.bench.name} as default site." + ), + ) + + self.frappe_container_run( + " ".join(self.bench_cli_cmd + [f"--site {self.bench.name} scheduler enable"]), + raise_exception_obj=BenchOperationException( + self.bench.name, f"Failed to enable {self.bench.name}'s scheduler." + ), + ) + + def is_required_services_available(self): + richprint.change_head("Checking if required services are available.") + required_services = { + self.bench.services.database_manager.database_server_info.host: self.bench.services.database_manager.database_server_info.port, + f"{self.bench.bench_config.container_name_prefix}-redis-cache": 6379, + f"{self.bench.bench_config.container_name_prefix}-redis-queue": 6379, + f"{self.bench.bench_config.container_name_prefix}-redis-socketio": 6379, + } + for service, port in required_services.items(): + output = self.wait_for_required_service(host=service, port=port) + if output.combined: + richprint.print(output.combined[-1].replace('wait-for-it: ', '')) + + def frappe_container_run( + self, + command: str, + raise_exception_obj: Optional[BenchOperationException] = None, + capture_output: bool = False, + user: str = "frappe", + workdir="/workspace/frappe-bench", + ): + try: + if capture_output: + output: SubprocessOutput = self.bench.compose_project.docker.compose.exec( + service="frappe", command=command, user=user, workdir=workdir, stream=not capture_output + ) + return output + else: + output: Iterable[Tuple[str, bytes]] = self.bench.compose_project.docker.compose.exec( + service="frappe", command=command, workdir=workdir, user=user, stream=not capture_output + ) + richprint.live_lines(output) + + except DockerException as e: + if raise_exception_obj: + raise_exception_obj.set_output(e.output) + raise raise_exception_obj + raise e + + def change_frappeverse_prebaked_app_branch(self, app: str, branch: str): + richprint.change_head(f"Configuring {app} app's branch -> {branch}") + + if not branch == STABLE_APP_BRANCH_MAPPING_LIST[app]: + richprint.change_head( + f"Changing prebaked {app} app's branch {STABLE_APP_BRANCH_MAPPING_LIST[app]} -> {self.bench.bench_config.frappe_branch}" + ) + change_frappe_branch_command = self.bench_cli_cmd + [f"get-app --overwrite --branch {branch} frappe"] + change_frappe_branch_command = " ".join(change_frappe_branch_command) + + exception = BenchOperationFrappeBranchChangeFailed( + bench_name=self.bench.name, app=app, branch=self.bench.bench_config.frappe_branch + ) + + self.frappe_container_run(command=change_frappe_branch_command, raise_exception_obj=exception) + + richprint.print(f"Configured {app} app's branch -> {self.bench.bench_config.frappe_branch}") + + def setup_supervisor(self, force: bool = False): + config_dir_path: Path = self.frappe_bench_dir / "config" + supervisor_conf_path: Path = config_dir_path / "supervisor.conf" + + richprint.change_head("Checking supervisor configuration") + if not supervisor_conf_path.exists() or force: + richprint.change_head("Setting up supervisor configs") + + bench_setup_supervisor_command = self.bench_cli_cmd + [ + "setup supervisor --skip-redis --skip-supervisord --yes --user frappe" + ] + + bench_setup_supervisor_command = " ".join(bench_setup_supervisor_command) + bench_setup_supervisor_exception = BenchOperationException(self.bench.name, "Failed to setup supervisor.") + self.frappe_container_run(bench_setup_supervisor_command, bench_setup_supervisor_exception) + self.split_supervisor_config() + richprint.print("Setuped supervisor configs") + + def split_supervisor_config(self): + import configparser + + supervisor_conf_path: Path = self.frappe_bench_dir / "config" / "supervisor.conf" + config = configparser.ConfigParser(allow_no_value=True, strict=False, interpolation=None) + config.read_string(supervisor_conf_path.read_text()) + + for section_name in config.sections(): + if not "group:" in section_name: + section_config = configparser.ConfigParser(interpolation=None) + section_config.add_section(section_name) + for key, value in config.items(section_name): + if "frappe-bench-frappe-web" in section_name: + if key == "command": + value = value.replace("127.0.0.1:80", "0.0.0.0:80") + section_config.set(section_name, key, value) + if "worker" in section_name: + file_name = f"{section_name.replace('program:','')}.workers.fm.supervisor.conf" + else: + file_name = f"{section_name.replace('program:','')}.fm.supervisor.conf" + + new_file: Path = supervisor_conf_path.parent / file_name + + with open(new_file, "w") as section_file: + section_config.write(section_file) + + self.bench.logger.info(f"Split supervisor conf {section_name} => {file_name}") + + def setup_frappe_server_config(self): + bench_serve_help_output: Optional[SubprocessOutput] = self.frappe_container_run( + " ".join(self.bench_cli_cmd + ["serve --help"]), capture_output=True + ) + bench_dev_server_script_output = self.frappe_container_run( + "cat /opt/user/bench-dev-server", capture_output=True + ) + import re + + if "host" in " ".join(bench_serve_help_output.combined): + new_bench_dev_server_script = re.sub( + r"--port \d+", "--host 0.0.0.0 --port 80", " ".join(bench_dev_server_script_output.combined) + ) + else: + new_bench_dev_server_script = re.sub( + r"--port \d+", "--port 80", " ".join(bench_dev_server_script_output.combined) + ) + + self.frappe_container_run(f'echo "{new_bench_dev_server_script}" > /opt/user/bench-dev-server.sh') + self.frappe_container_run("chmod +x /opt/user/bench-dev-server.sh", user='root') + + def bench_install_apps(self, apps_lists, already_installed_apps: Dict = STABLE_APP_BRANCH_MAPPING_LIST): + to_install_apps = [x["app"] for x in apps_lists] + + for app, branch in already_installed_apps.items(): + if app == 'frappe': + continue + + if app not in to_install_apps: + richprint.change_head(f"Removing prebaked app {app} from python env.") + self.bench_rm_app_env(app) + richprint.print(f"Remove prebaked app {app}") + + for app_info in apps_lists: + app = app_info["app"] + branch = app_info["branch"] + + status_txt = f"Building and Installing app {app} in env." + + if branch: + status_txt = f"Building and Installing app {app} -> {branch}." + + richprint.change_head(status_txt) + + if app in already_installed_apps.keys(): + if already_installed_apps[app] == branch: + richprint.print(f"Skipped installation of prebaked app [blue]{app} -> {branch}[/blue].") + continue + + if not branch: + branch = already_installed_apps[app] + + self.bench_install_app_env(app, branch) + + richprint.print(f"Builded and Installed app [blue]{app}{' -> ' + branch if branch else ''}[/blue] in env.") + + def bench_install_apps_site(self): + apps_dir = self.frappe_bench_dir / 'apps' + apps_dirs: List[Path] = [item for item in apps_dir.iterdir() if item.is_dir()] + + for app in apps_dirs: + richprint.change_head(f"Installing app {app.name} in site.") + self.bench_install_app_site(app.name) + richprint.print(f"Installed app {app.name} in site.") + + def bench_build(self, app_list: Optional[List[str]] = None): + build_cmd = self.bench_cli_cmd + ["build"] + + if app_list is not None: + for app in app_list: + build_cmd += ["--app"] + [app] + + build_exception = BenchOperationBenchBuildFailed(bench_name=self.bench.name, apps=app_list) + + build_cmd = " ".join(build_cmd) + self.frappe_container_run(build_cmd, build_exception) + + def bench_install_app_env( + self, app: str, branch: Optional[str] = None, overwrite: bool = True, skip_assets: bool = False + ): + parameters: Dict = locals() + + remove_parameters = ["app"] + app_install_env_command = self.bench_cli_cmd + ["get-app"] + app_install_env_command += parameters_to_options(parameters, exclude=remove_parameters) + app_install_env_command += [app] + + app_install_env_command = " ".join(app_install_env_command) + app_install_exception = BenchOperationBenchInstallAppInPythonEnvFailed(bench_name=self.bench.name, app_name=app) + + self.frappe_container_run( + app_install_env_command, + raise_exception_obj=app_install_exception, + ) + + def bench_rm_app_env(self, app: str, no_backup: bool = True, force: bool = True): + parameters: dict = locals() + + remove_parameters = ["app"] + app_rm_env_command = self.bench_cli_cmd + ["rm"] + app_rm_env_command += parameters_to_options(parameters, exclude=remove_parameters) + app_rm_env_command += [app] + + app_rm_env_command = " ".join(app_rm_env_command) + + self.frappe_container_run( + app_rm_env_command, + raise_exception_obj=BenchOperationBenchRemoveAppFromPythonEnvFailed( + bench_name=self.bench.name, app_name=app + ), + ) + + def bench_install_app_site(self, app: str): + app_install_site_command = self.bench_cli_cmd + ["--site", self.bench.name] + app_install_site_command += ["install-app", app] + app_install_site_command = " ".join(app_install_site_command) + + self.frappe_container_run( + app_install_site_command, + raise_exception_obj=BenchOperationBenchAppInSiteFailed(bench_name=self.bench.name, app_name=app), + ) + + def is_bench_site_exists(self, bench_site_name: str): + site_path: Path = self.frappe_bench_dir / "sites" / bench_site_name + return site_path.exists() + + def wait_for_required_service(self, host: str, port: int, timeout: int = 120): + return self.frappe_container_run( + f"wait-for-it -t {timeout} {host}:{port}", + raise_exception_obj=BenchOperationWaitForRequiredServiceFailed( + bench_name=self.bench.name, host=host, port=port, timeout=timeout + ), + capture_output=True, + ) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 52fab987..c3a3a07e 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -8,6 +8,7 @@ import subprocess from typing import Any, Dict, List, Optional from pathlib import Path +from frappe_manager.site_manager.bench_site_manager import BenchOperations from rich.table import Table from frappe_manager.compose_project.compose_project import ComposeProject from frappe_manager.docker_wrapper.DockerException import DockerException @@ -25,6 +26,7 @@ BenchFailedToRemoveDevPackages, BenchFrappeServiceSupervisorNotRunning, BenchNotRunning, + BenchOperationException, BenchRemoveDirectoryError, BenchSSLCertificateAlreadyIssued, BenchSSLCertificateNotIssued, @@ -61,7 +63,6 @@ def __init__( name: str, bench_config: BenchConfig, compose_project: ComposeProject, - bench_workers: BenchWorkers, services: ServicesManager, workers_check: bool = True, admin_tools_check: bool = True, @@ -74,7 +75,6 @@ def __init__( self.backup_path = self.path / 'backups' self.bench_config: BenchConfig = bench_config self.compose_project: ComposeProject = compose_project - self.workers = bench_workers self.logger = log.get_logger() self.proxy_manager: NginxProxyManager = NginxProxyManager('nginx', self.compose_project) self.admin_tools: AdminTools = AdminTools(self.name, self.path, self.proxy_manager) @@ -84,6 +84,8 @@ def __init__( webroot_dir=self.proxy_manager.dirs.html.host, proxy_manager=services.proxy_manager, ) + self.benchops = BenchOperations(self) + self.workers = BenchWorkers(self, not verbose) if workers_check: self.ensure_workers_running_if_available() @@ -110,7 +112,6 @@ def get_object( compose_file_manager = ComposeFile(bench_path / "docker-compose.yml") compose_project: ComposeProject = ComposeProject(compose_file_manager, verbose=verbose) - workers = BenchWorkers(bench_name, bench_path, not verbose) bench_config: BenchConfig = BenchConfig.import_from_toml(bench_config_path) @@ -119,7 +120,6 @@ def get_object( 'path': bench_path, 'bench_config': bench_config, 'compose_project': compose_project, - 'bench_workers': workers, 'services': services, 'workers_check': workers_check, 'admin_tools_check': admin_tools_check, @@ -191,7 +191,6 @@ def create(self, is_template_bench: bool = False): self.create_compose_dirs() if is_template_bench: - self.remove_attached_secrets() global_db_info = self.services.database_manager.database_server_info self.sync_bench_common_site_config(global_db_info.host, global_db_info.port) self.save_bench_config() @@ -203,15 +202,17 @@ def create(self, is_template_bench: bool = False): richprint.print(f"Started bench services.") richprint.change_head("Creating bench and bench site.") - self.frappe_logs_till_start() - richprint.print("Bench and bench site created.") + self.benchops.create_fm_bench() + self.sync_bench_config_configuration() richprint.change_head("Configuring bench workers.") self.sync_workers_compose(force_recreate=True) richprint.change_head("Configuring bench workers.") richprint.update_live() - richprint.change_head(f"Commencing site status check on bench site.") + self.save_bench_config() + + richprint.change_head(f"Commencing site status check") # check if bench is created if not self.is_bench_created(): @@ -219,8 +220,6 @@ def create(self, is_template_bench: bool = False): richprint.print("Bench site is active and responding.") - self.remove_attached_secrets() - self.logger.info(f"{self.name}: Bench site is active and responding.") self.info() @@ -405,9 +404,12 @@ def start(self, force: bool = False): global_db_info = self.services.database_manager.database_server_info self.sync_bench_common_site_config(global_db_info.host, global_db_info.port) + self.sync_workers_compose() + richprint.change_head("Starting bench services") self.admin_tools.remove_nginx_location_config() self.compose_project.start_service(force_recreate=force) + self.benchops.is_required_services_available() self.sync_bench_config_configuration() self.save_bench_config() richprint.print("Started bench services.") @@ -418,12 +420,8 @@ def start(self, force: bool = False): self.workers.compose_project.start_service(force_recreate=force) richprint.print("Started bench workers services.") - richprint.change_head('Starting frappe server') - self.frappe_logs_till_start() richprint.print('Started frappe server.') - self.sync_workers_compose() - def frappe_logs_till_start(self): """ Retrieves and prints the logs of the 'frappe' service until site supervisor starts. @@ -558,12 +556,19 @@ def is_bench_created(self, retry=60, interval=1) -> bool: return False def sync_workers_compose(self, force_recreate: bool = False): - self.regenerate_workers_supervisor_conf() - are_workers_not_changed = self.workers.is_expected_worker_same_as_template() + workers_backup_manager = self.backup_workers_supervisor_conf() + try: + self.benchops.setup_supervisor(force=True) + except BenchOperationException as e: + self.backup_restore_workers_supervisor(workers_backup_manager) + + are_workers_not_changed = self.workers.is_new_workers_added() + if are_workers_not_changed: richprint.print("Workers configuration remains unchanged.") return + self.backup_workers_supervisor_conf() self.workers.generate_compose() self.workers.compose_project.start_service(force_recreate=force_recreate) @@ -574,11 +579,10 @@ def backup_restore_workers_supervisor(self, backup_manager: BackupManager): def backup_workers_supervisor_conf(self): # TODO this can be given to woker class ? - backup_workers_manager = BackupManager('workers', self.backup_path) - + backup_workers_manager = BackupManager(name='workers', backup_group_name='workers') # TODO use backup manager # take backup - backup_workers_manager.backup(self.workers.supervisor_config_path) + backup_workers_manager.backup(self.workers.supervisor_config_path, bench_name=self.name) if self.workers.supervisor_config_path.exists(): for file_path in self.workers.config_dir.iterdir(): @@ -589,37 +593,11 @@ def backup_workers_supervisor_conf(self): from_path = file_path to_path = file_path.parent / f"{file_path.name}.bak" backup_workers_manager.backup(from_path) + file_path.unlink() return backup_workers_manager def regenerate_workers_supervisor_conf(self): - richprint.change_head("Regenerating supervisor.conf.") self.backup_workers_supervisor_conf() - # generate the supervisor.conf - try: - bench_setup_supervisor_command = ( - "bench setup supervisor --skip-redis --skip-supervisord --yes --user frappe" - ) - output = self.compose_project.docker.compose.exec( - service="frappe", - command=bench_setup_supervisor_command, - user="frappe", - workdir="/workspace/frappe-bench", - stream=True, - ) - richprint.live_lines(output, padding=(0, 0, 0, 2)) - generate_split_config_command = "/scripts/divide-supervisor-conf.py config/supervisor.conf" - output = self.compose_project.docker.compose.exec( - service="frappe", - command=generate_split_config_command, - user="frappe", - workdir="/workspace/frappe-bench", - stream=True, - ) - richprint.live_lines(output, padding=(0, 0, 0, 2)) - return True - except DockerException: - richprint.print("Rolling back to previous workers configuration.") - raise BenchWorkersSupervisorConfigurtionGenerateError(self.name) def get_bench_installed_apps_list(self): apps_json_file = self.path / "workspace" / "frappe-bench" / "sites" / "apps.json" @@ -634,28 +612,6 @@ def get_bench_installed_apps_list(self): def get_db_connection_info(self): return get_bench_db_connection_info(self.name, self.path) - def remove_attached_secrets(self): - richprint.change_head(f"Removing bench attached secrets") - running = False - - if self.compose_project.running: - running = True - self.compose_project.stop_service(services=['frappe']) - if self.workers.compose_project.compose_file_manager.exists(): - self.workers.compose_project.stop_service() - - self.compose_project.compose_file_manager.remove_secrets_from_container("frappe") - self.compose_project.compose_file_manager.remove_root_secrets_compose() - self.compose_project.compose_file_manager.write_to_file() - - if running: - self.start() - richprint.change_head("Waiting till bench server has created.") - self.frappe_logs_till_start() - richprint.print("Bench server has created.") - - richprint.print(f"Removed bench attached secrets.") - def create_certificate(self): self.certificate_manager.generate_certificate() self.save_bench_config() @@ -800,6 +756,11 @@ def shell(self, compose_service: str, user: str | None): """ richprint.change_head(f"Spawning shell") + import sys + + if sys.stdin.isatty() is False: + print('stdin ', sys.stdin.readlines()) + if compose_service == "frappe" and not user: user = "frappe" @@ -994,7 +955,10 @@ def attach_to_bench(self, user: str, extensions: List[str], workdir: str, debugg # install black in env try: self.compose_project.docker.compose.exec( - service="frappe", command="/workspace/frappe-bench/env/bin/pip install black", stream=True + service="frappe", + command="/workspace/frappe-bench/env/bin/pip install black", + user='frappe', + stream=True, ) except DockerException as e: self.logger.error(f"black installation exception: {capture_and_format_exception()}") @@ -1095,7 +1059,7 @@ def restart_frappe_server(self): restart_command = 'supervisorctl -c /opt/user/supervisord.conf restart all' try: - self.compose_project.docker.compose.exec('frappe', restart_command, stream=False) + self.compose_project.docker.compose.exec('frappe', restart_command, user='frappe', stream=False) except DockerException as e: raise BenchException("frappe", "Faild to restart frappe server.") richprint.print("Restarted frappe server.") @@ -1134,7 +1098,7 @@ def remove_dev_packages(self): dev_packages = self.get_apps_dev_requirements() remove_command = '/workspace/frappe-bench/env/bin/python -m pip uninstall --yes ' + " ".join(dev_packages) try: - self.compose_project.docker.compose.exec('frappe', command=remove_command, stream=False) + self.compose_project.docker.compose.exec('frappe', command=remove_command, user='frappe', stream=False) except DockerException as e: raise BenchFailedToRemoveDevPackages(self.name) richprint.print("Removed dev packages from env.") @@ -1146,7 +1110,7 @@ def install_dev_packages(self): dev_packages ) try: - self.compose_project.docker.compose.exec('frappe', command=install_command, stream=False) + self.compose_project.docker.compose.exec('frappe', command=install_command, user='frappe', stream=False) except DockerException as e: raise BenchFailedToRemoveDevPackages(self.name) richprint.print("Installed dev packages in env.") diff --git a/frappe_manager/site_manager/site_exceptions.py b/frappe_manager/site_manager/site_exceptions.py index 75f4de36..5d758793 100644 --- a/frappe_manager/site_manager/site_exceptions.py +++ b/frappe_manager/site_manager/site_exceptions.py @@ -1,6 +1,12 @@ +from builtins import len from pathlib import Path +from typing import List, Optional -from pydantic import config +from frappe_manager.docker_wrapper.subprocess_output import SubprocessOutput +from frappe_manager.utils import helpers +from rich.box import Box +from rich.style import Style +from typer import Option class BenchException(Exception): @@ -18,7 +24,7 @@ def __init__( self, bench_name: str, path: Path, - message: str = 'Compose file not found at {}. Aborting operation.', + message: str = "Compose file not found at {}. Aborting operation.", ): self.bench_name = bench_name self.path = path @@ -31,7 +37,7 @@ def __init__( self, bench_name: str, service: str, - message: str = 'Service {} not running.', + message: str = "Service {} not running.", ): self.bench_name = bench_name self.service = service @@ -44,7 +50,7 @@ def __init__( self, bench_name: str, path: Path, - message: str = 'Bench not found at {}.', + message: str = "Bench not found at {}.", ): self.bench_name = bench_name self.path = path @@ -57,7 +63,7 @@ def __init__( self, bench_name: str, path: Path, - message: str = 'Remove dirs failed at {}.', + message: str = "Remove dirs failed at {}.", ): self.bench_name = bench_name self.path = path @@ -70,7 +76,7 @@ def __init__( self, bench_name: str, path: Path, - message: str = 'Log file not found at {}.', + message: str = "Log file not found at {}.", ): self.bench_name = bench_name self.path = path @@ -82,7 +88,7 @@ class BenchWorkersStartError(BenchException): def __init__( self, bench_name: str, - message: str = 'Workers not able to start.', + message: str = "Workers not able to start.", ): self.bench_name = bench_name self.message = message @@ -93,7 +99,7 @@ class BenchWorkersSupervisorConfigurtionGenerateError(BenchException): def __init__( self, bench_name: str, - message: str = 'Failed to configure workers.', + message: str = "Failed to configure workers.", ): self.bench_name = bench_name self.message = message @@ -105,7 +111,7 @@ def __init__( self, bench_name: str, config_dir: str, - message: str = 'Superviosrd workers configuration not found in {}.', + message: str = "Superviosrd workers configuration not found in {}.", ): self.bench_name = bench_name self.config_dir = config_dir @@ -114,7 +120,7 @@ def __init__( class BenchConfigFileNotFound(BenchException): - def __init__(self, bench_name, config_path, message='Config file not found at {}.'): + def __init__(self, bench_name, config_path, message="Config file not found at {}."): self.bench_name = bench_name self.config_path = config_path self.message = message.format(config_path) @@ -122,7 +128,7 @@ def __init__(self, bench_name, config_path, message='Config file not found at {} class BenchConfigValidationError(BenchException): - def __init__(self, bench_name, config_path, message='FM bench config not valid at {}'): + def __init__(self, bench_name, config_path, message="FM bench config not valid at {}"): self.bench_name = bench_name self.conig_path = config_path self.message = message.format(self.conig_path) @@ -177,3 +183,193 @@ def __init__(self, bench_name, message="Supervisorctl is not running in frappe s self.bench_name = bench_name self.message = message super().__init__(self.bench_name, self.message) + + +class BenchOperationException(BenchException): + def __init__(self, bench_name, message: str, print_combined: bool = True, print_stdout: bool = False, print_stderr: bool = False): + self.bench_name = bench_name + self.message = message + self.print_stdout = print_stdout + self.print_stderr = print_stderr + self.print_combined = print_combined + self.output = None + super().__init__(self.bench_name, self.message) + + def set_output(self, output: SubprocessOutput): + self.output = output + from rich.panel import Panel + + import typer.rich_utils as ut + + to_print = [] + + box: Box = Box("╭ \n" " \n" " ── \n" "│ \n" " \n" " \n" " | \n" " \n", ascii=True) + if self.print_stdout: + panel = Panel.fit( + "\n".join(self.output.stdout), + box=box, + padding=(0, 1), + border_style="dim", + title="Error command stdout", + title_align="left", + ) + to_print.append(helpers.rich_object_to_string(panel)) + + if self.print_combined: + panel = Panel.fit( + "\n".join(self.output.combined), + box=box, + padding=(0, 1), + border_style="dim", + title="Error command output", + title_align="left", + ) + to_print.append(helpers.rich_object_to_string(panel)) + + if self.print_stderr: + panel = Panel.fit( + "\n".join(self.output.stderr), + box=box, + padding=(0, 1), + border_style="dim", + title="Error command stderr", + title_align="left", + ) + to_print.append(helpers.rich_object_to_string(panel)) + + self.message = self.message + "\n" + "\n".join(to_print) + super().__init__(self.bench_name, self.message) + + +class BenchOperationFrappeBranchChangeFailed(BenchException): + def __init__(self, bench_name, app: str, branch: str, message: str = "Failed to change {} app branch to {}."): + self.bench_name = bench_name + self.app = app + self.branch = branch + self.message = message.format(app, branch) + super().__init__(self.bench_name, self.message) + + +class BenchOperationWaitForRequiredServiceFailed(BenchOperationException): + def __init__( + self, + bench_name, + host: str, + port: str, + timeout: int, + message: str = "Waiting for service {}:{} timed out. {}", + print_combined: bool = True, + print_stdout: bool = False, + print_stderr: bool = False, + ): + self.bench_name = bench_name + self.host = host + self.port = port + self.timeout = timeout + self.print_stdout = print_stdout + self.print_stderr = print_stderr + self.print_combined = print_combined + self.message = message.format(host, port, timeout) + + super().__init__(self.bench_name, self.message, self.print_combined, self.print_stdout, self.print_stderr) + + +class BenchOperationBenchSiteCreateFailed(BenchOperationException): + def __init__( + self, + bench_name, + print_combined: bool = True, + print_stdout: bool = False, + print_stderr: bool = False, + message: str = "Failed to create site {}.", + ): + self.bench_name = bench_name + self.message = message.format(bench_name) + self.print_stdout = print_stdout + self.print_stderr = print_stderr + self.print_combined = print_combined + super().__init__(self.bench_name, self.message, self.print_combined, self.print_stdout, self.print_stderr) + + +class BenchOperationBenchInstallAppInPythonEnvFailed(BenchOperationException): + def __init__( + self, + bench_name, + app_name: str, + message: str = "Failed to install app {} in python env.", + print_combined: bool = True, + print_stdout: bool = False, + print_stderr: bool = False, + ): + self.bench_name = bench_name + self.app_name = app_name + self.message = message.format(app_name) + self.print_stdout = print_stdout + self.print_stderr = print_stderr + self.print_combined = print_combined + + super().__init__(self.bench_name, self.message, self.print_combined, self.print_stdout, self.print_stderr) + + +class BenchOperationBenchRemoveAppFromPythonEnvFailed(BenchOperationException): + def __init__( + self, + bench_name, + app_name: str, + message: str = "Failed to remove app {} from python env.", + print_combined: bool = True, + print_stdout: bool = False, + print_stderr: bool = False, + ): + self.bench_name = bench_name + self.app_name = app_name + self.message = message.format(app_name) + self.print_stdout = print_stdout + self.print_stderr = print_stderr + self.print_combined = print_combined + + super().__init__(self.bench_name, self.message, self.print_combined, self.print_stdout, self.print_stderr) + + +class BenchOperationBenchAppInSiteFailed(BenchOperationException): + def __init__( + self, + bench_name, + app_name: str, + message: str = "Failed to install app {} in site {}.", + print_combined: bool = True, + print_stdout: bool = False, + print_stderr: bool = False, + ): + self.bench_name = bench_name + self.app_name = app_name + self.message = message.format(app_name, self.bench_name) + self.print_stdout = print_stdout + self.print_stderr = print_stderr + self.print_combined = print_combined + super().__init__(self.bench_name, self.message, self.print_combined, self.print_stdout, self.print_stderr) + + +class BenchOperationBenchBuildFailed(BenchOperationException): + def __init__( + self, + bench_name, + apps: Optional[List[str]] = None, + message: str = "Failed to build", + print_combined: bool = True, + print_stdout: bool = False, + print_stderr: bool = False, + ): + self.bench_name = bench_name + self.apps = apps + if apps: + message = message + " app" + if len(apps) > 1: + message = message + " apps" + for app in apps: + message += f" {app}" + self.message = message + self.print_stdout = print_stdout + self.print_stderr = print_stderr + self.print_combined = print_combined + super().__init__(self.bench_name, self.message, self.print_combined, self.print_stdout, self.print_stderr) diff --git a/frappe_manager/site_manager/workers_manager/SiteWorker.py b/frappe_manager/site_manager/workers_manager/SiteWorker.py index b5ba747e..f1f377aa 100644 --- a/frappe_manager/site_manager/workers_manager/SiteWorker.py +++ b/frappe_manager/site_manager/workers_manager/SiteWorker.py @@ -1,26 +1,32 @@ from copy import deepcopy from pathlib import Path +from typing import List, TYPE_CHECKING from frappe_manager.compose_manager.ComposeFile import ComposeFile from frappe_manager.compose_project.compose_project import ComposeProject from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.site_manager.site_exceptions import BenchWorkersSupervisorConfigurtionNotFoundError from frappe_manager.utils.helpers import get_container_name_prefix, get_current_fm_version +if TYPE_CHECKING: + from frappe_manager.site_manager.site import Bench + class BenchWorkers: - def __init__(self, bench_name: str, bench_path: Path, verbose: bool = True): - self.compose_path = bench_path / "docker-compose.workers.yml" - self.config_dir = bench_path / "workspace" / "frappe-bench" / "config" + def __init__(self, bench: 'Bench', verbose: bool = True): + self.bench = bench + self.compose_path = self.bench.path / "docker-compose.workers.yml" + self.config_dir = self.bench.path / "workspace" / "frappe-bench" / "config" self.supervisor_config_path = self.config_dir / "supervisor.conf" - self.bench_name = bench_name self.quiet = not verbose self.compose_project = ComposeProject( ComposeFile(self.compose_path, template_name='docker-compose.workers.tmpl') ) - def get_expected_workers(self) -> list[str]: + def get_expected_workers(self) -> List[str]: richprint.change_head("Checking workers info.") + workers_supervisor_conf_paths = [] + for file_path in self.config_dir.iterdir(): file_path_abs = str(file_path.absolute()) if file_path.is_file(): @@ -28,7 +34,7 @@ def get_expected_workers(self) -> list[str]: workers_supervisor_conf_paths.append(file_path) if len(workers_supervisor_conf_paths) == 0: - raise BenchWorkersSupervisorConfigurtionNotFoundError(self.bench_name, self.config_dir) + raise BenchWorkersSupervisorConfigurtionNotFoundError(self.bench.name, self.config_dir) workers_expected_service_names = [] @@ -42,11 +48,20 @@ def get_expected_workers(self) -> list[str]: return workers_expected_service_names - def is_expected_worker_same_as_template(self) -> bool: + def is_new_workers_added(self) -> bool: if not self.compose_project.compose_file_manager.is_template_loaded: prev_workers = self.compose_project.compose_file_manager.get_services_list() prev_workers.sort() expected_workers = self.get_expected_workers() + # get custom workers from common_site_config.json + # common_site_config_data = self.bench.get_common_bench_config() + + # if 'workers' in common_site_config_data: + # custom_workers: List[str] = common_site_config_data['workers'].keys() + # for worker in custom_workers: + # worker = f'{worker}-worker' + # if worker not in prev_workers: + # return False return prev_workers == expected_workers else: return False @@ -83,12 +98,12 @@ def generate_compose(self): self.compose_project.compose_file_manager.yml["services"][worker] = worker_config - self.compose_project.compose_file_manager.set_container_names(get_container_name_prefix(self.bench_name)) + self.compose_project.compose_file_manager.set_container_names(get_container_name_prefix(self.bench.name)) self.compose_project.compose_file_manager.set_version(get_current_fm_version()) # set network name self.compose_project.compose_file_manager.yml["networks"]["site-network"]["name"] = ( - self.bench_name.replace(".", "") + f"-network" + self.bench.name.replace(".", "") + f"-network" ) self.compose_project.compose_file_manager.write_to_file() diff --git a/frappe_manager/templates/docker-compose.tmpl b/frappe_manager/templates/docker-compose.tmpl index 6252d99d..fbaf500e 100644 --- a/frappe_manager/templates/docker-compose.tmpl +++ b/frappe_manager/templates/docker-compose.tmpl @@ -3,18 +3,8 @@ services: image: ghcr.io/rtcamp/frappe-manager-frappe:v0.13.0 container_name: REPLACE_ME_WITH_CONTAINER_NAME environment: - ADMIN_PASS: REPLACE_me_with_frappe_web_admin_pass - # apps are defined as :, if branch name not given then default github branch will be used. - APPS_LIST: REPLACE_ME_APPS_LIST - DB_NAME: REPLACE_ME_WITH_DB_NAME_TO_CREATE - # DEVERLOPER_MODE bool -> true/false - DEVELOPER_MODE: REPLACE_ME_WITH_DEVELOPER_MODE_TOGGLE - FRAPPE_BRANCH: REPLACE_ME_WITH_BRANCH_OF_FRAPPE - MARIADB_ROOT_PASS: REPLACE_ME_WITH_DB_ROOT_PASSWORD - SITENAME: REPLACE_ME_WITH_THE_SITE_NAME - USERGROUP: REPLACE_ME_WITH_CURRENT_USER_GROUP USERID: REPLACE_ME_WITH_CURRENT_USER - MARIADB_HOST: REPLACE_ME_WITH_DB_HOST + USERGROUP: REPLACE_ME_WITH_CURRENT_USER_GROUP volumes: - ./workspace:/workspace:cached expose: @@ -24,8 +14,6 @@ services: networks: site-network: global-backend-network: - secrets: - - db_root_password nginx: image: ghcr.io/rtcamp/frappe-manager-nginx:v0.13.0 diff --git a/frappe_manager/utils/callbacks.py b/frappe_manager/utils/callbacks.py index 700a78f9..24ca9acd 100644 --- a/frappe_manager/utils/callbacks.py +++ b/frappe_manager/utils/callbacks.py @@ -31,6 +31,7 @@ def apps_list_validation_callback(value: List[str] | None): raise typer.BadParameter("'frappe' should not be included here.") if "https:" in app or "http:" in app: + temp_appx = appx appx = [":".join(appx[:2])] @@ -70,7 +71,10 @@ def apps_list_validation_callback(value: List[str] | None): richprint.stop() raise typer.BadParameter(f"Invaid branch '{appx[1]}' for '{appx[0]}'.") - appx = ":".join(appx) + appx = { + 'app' : appx[0], + 'branch' : appx[1] if len(appx) > 1 else None , + } apps_list.append(appx) return apps_list diff --git a/frappe_manager/utils/helpers.py b/frappe_manager/utils/helpers.py index 32d30c44..62abed85 100644 --- a/frappe_manager/utils/helpers.py +++ b/frappe_manager/utils/helpers.py @@ -388,14 +388,14 @@ def get_frappe_manager_own_files(file_path: str): return Path(str(pkg_resources.files("frappe_manager").joinpath(file_path))) -def rich_traceback_to_string(traceback: Traceback) -> str: +def rich_object_to_string(obj) -> str: """Convert a rich Traceback object to a string.""" # Initialize a 'fake' console with StringIO to capture output capture_buffer = StringIO() fake_console = Console(force_terminal=False, file=capture_buffer) - fake_console.print(traceback, crop=False, overflow='ignore') + fake_console.print(obj, crop=False, overflow='ignore') captured_str = capture_buffer.getvalue() # Retrieve the captured output as a string capture_buffer.close() @@ -413,7 +413,7 @@ def capture_and_format_exception(traceback_max_frames: int = 100) -> str: ) # Convert the Traceback object to a formatted string - formatted_traceback = rich_traceback_to_string(traceback) + formatted_traceback = rich_object_to_string(traceback) return formatted_traceback diff --git a/pyproject.toml b/pyproject.toml index d7b274b4..f761189d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "frappe-manager" -version = "0.14.0" +version = "0.15.0" license = "MIT" repository = "https://github.com/rtcamp/frappe-manager" description = "A CLI tool based on Docker Compose to easily manage Frappe based projects. As of now, only suitable for development in local machines running on Mac and Linux based OS." From 674be2b43aa1d3437c30675e7ac0fdb131d7f2ab Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 14 Jun 2024 15:45:40 +0530 Subject: [PATCH 17/44] fix: remove duplicate backup of workers --- frappe_manager/site_manager/site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index c3a3a07e..dd1c878e 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -557,6 +557,7 @@ def is_bench_created(self, retry=60, interval=1) -> bool: def sync_workers_compose(self, force_recreate: bool = False): workers_backup_manager = self.backup_workers_supervisor_conf() + try: self.benchops.setup_supervisor(force=True) except BenchOperationException as e: @@ -568,7 +569,6 @@ def sync_workers_compose(self, force_recreate: bool = False): richprint.print("Workers configuration remains unchanged.") return - self.backup_workers_supervisor_conf() self.workers.generate_compose() self.workers.compose_project.start_service(force_recreate=force_recreate) From 774ae814667ece97db5a32d90be4044a53348bd6 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 18 Jun 2024 12:59:03 +0530 Subject: [PATCH 18/44] remove secrects attribute --- frappe_manager/templates/docker-compose.tmpl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frappe_manager/templates/docker-compose.tmpl b/frappe_manager/templates/docker-compose.tmpl index fbaf500e..54ce5b87 100644 --- a/frappe_manager/templates/docker-compose.tmpl +++ b/frappe_manager/templates/docker-compose.tmpl @@ -118,7 +118,3 @@ networks: global-backend-network: name: fm-global-backend-network external: true - -secrets: - db_root_password: - file: REPLACE_ME_WITH_SECERETS_PATH From af325d56b6b09fa77f475be6b5d977fcab1a0cc6 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 18 Jun 2024 15:22:24 +0530 Subject: [PATCH 19/44] rename file --- .../site_manager/{bench_site_manager.py => bench_operations.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename frappe_manager/site_manager/{bench_site_manager.py => bench_operations.py} (100%) diff --git a/frappe_manager/site_manager/bench_site_manager.py b/frappe_manager/site_manager/bench_operations.py similarity index 100% rename from frappe_manager/site_manager/bench_site_manager.py rename to frappe_manager/site_manager/bench_operations.py From 352bf3ce79bc48f892783ca5ebee474c6a790cf0 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 19 Jun 2024 15:05:59 +0530 Subject: [PATCH 20/44] cleanup --- frappe_manager/commands.py | 13 ++-- frappe_manager/site_manager/bench_config.py | 24 ++++---- .../site_manager/bench_operations.py | 7 ++- frappe_manager/site_manager/site.py | 59 ++++++++----------- .../site_manager/site_exceptions.py | 12 +++- .../ssl_manager/letsencrypt_certificate.py | 2 +- frappe_manager/utils/site.py | 29 +++++---- 7 files changed, 75 insertions(+), 71 deletions(-) diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index f27ab7c1..858ec3d0 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -1,4 +1,5 @@ from pathlib import Path +from frappe_manager.site_manager import bench_operations from frappe_manager.site_manager.site_exceptions import BenchNotRunning from frappe_manager.utils.site import pull_docker_images import typer @@ -24,7 +25,6 @@ from frappe_manager.services_manager.services import ServicesManager from frappe_manager.migration_manager.migration_executor import MigrationExecutor from frappe_manager.site_manager.site import Bench -from frappe_manager.site_manager.workers_manager.SiteWorker import BenchWorkers from frappe_manager.ssl_manager import LETSENCRYPT_PREFERRED_CHALLENGE, SUPPORTED_SSL_TYPES from frappe_manager.ssl_manager.certificate import SSLCertificate from frappe_manager.ssl_manager.letsencrypt_certificate import LetsencryptSSLCertificate @@ -75,7 +75,7 @@ def app_callback( if not help_called: first_time_install = False - richprint.start(f"Working") + richprint.start("Working") if not CLI_DIR.exists(): # creating the sites dir @@ -105,7 +105,6 @@ def app_callback( fm_config_manager: FMConfigManager = FMConfigManager.import_from_toml() - # docker pull if first_time_install: if not fm_config_manager.root_path.exists(): @@ -141,6 +140,7 @@ def app_callback( ctx.obj["verbose"] = verbose ctx.obj['fm_config_manager'] = fm_config_manager + @app.command(no_args_is_help=True) def create( ctx: typer.Context, @@ -317,11 +317,7 @@ def delete( ) else: - bench = Bench.get_object( - benchname, - services=services_manager, - workers_check=False, - admin_tools_check=False) + bench = Bench.get_object(benchname, services=services_manager, workers_check=False, admin_tools_check=False) benches.add_bench(bench) benches.remove_benches() @@ -626,6 +622,7 @@ def update( if bench_config_save: bench.save_bench_config() + @app.command() def test( ctx: typer.Context, diff --git a/frappe_manager/site_manager/bench_config.py b/frappe_manager/site_manager/bench_config.py index 1caaa7b9..dc957046 100644 --- a/frappe_manager/site_manager/bench_config.py +++ b/frappe_manager/site_manager/bench_config.py @@ -4,9 +4,9 @@ import tomlkit from pathlib import Path from typing import Any, Dict, List, Optional -from pydantic import BaseModel, Field, model_validator, validator +from pydantic import BaseModel, Field from frappe_manager import STABLE_APP_BRANCH_MAPPING_LIST -from frappe_manager.metadata_manager import FMConfigManager, FMLetsencryptConfig +from frappe_manager.metadata_manager import FMConfigManager from frappe_manager.ssl_manager import LETSENCRYPT_PREFERRED_CHALLENGE, SUPPORTED_SSL_TYPES from frappe_manager.ssl_manager.certificate import SSLCertificate from frappe_manager.ssl_manager.letsencrypt_certificate import LetsencryptSSLCertificate @@ -45,7 +45,7 @@ class BenchConfig(BaseModel): ) admin_pass: str = Field('admin', description="The admin password") root_path: Path = Field(..., description="The root path") - apps_list: List[Dict[str,Optional[str]]] = Field(default=[], description="List of apps") + apps_list: List[Dict[str, Optional[str]]] = Field(default=[], description="List of apps") userid: int = Field(default_factory=os.getuid, description="The user ID of the current process") usergroup: int = Field(default_factory=os.getgid, description="The group ID of the current process") @@ -165,15 +165,15 @@ def import_from_toml(cls, path: Path) -> "BenchConfig": def get_commmon_site_config_data(self, db_server_info: DatabaseServerServiceInfo) -> Dict[str, Any]: common_site_config_data = { "install_apps": [], - "db_host" : db_server_info.host, - "db_port" : db_server_info.port, - "redis_cache" : f"redis://{self.container_name_prefix}-redis-cache:6379", - "redis_queue" : f"redis://{self.container_name_prefix}-redis-queue:6379", - "redis_socketio" : f"redis://{self.container_name_prefix}-redis-socketio:6379", - "webserver_port" : 80, - "socketio_port" : 80, - "restart_supervisor_on_update" : 0, - "developer_mode" : self.developer_mode, + "db_host": db_server_info.host, + "db_port": db_server_info.port, + "redis_cache": f"redis://{self.container_name_prefix}-redis-cache:6379", + "redis_queue": f"redis://{self.container_name_prefix}-redis-queue:6379", + "redis_socketio": f"redis://{self.container_name_prefix}-redis-socketio:6379", + "webserver_port": 80, + "socketio_port": 80, + "restart_supervisor_on_update": 0, + "developer_mode": self.developer_mode, } return common_site_config_data diff --git a/frappe_manager/site_manager/bench_operations.py b/frappe_manager/site_manager/bench_operations.py index 85697007..feb081cd 100644 --- a/frappe_manager/site_manager/bench_operations.py +++ b/frappe_manager/site_manager/bench_operations.py @@ -12,10 +12,12 @@ BenchOperationBenchSiteCreateFailed, BenchOperationException, BenchOperationFrappeBranchChangeFailed, + BenchOperationRequiredDockerImagesNotAvailable, BenchOperationWaitForRequiredServiceFailed, ) from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.utils.docker import parameters_to_options +from frappe_manager.utils.site import get_all_docker_images class BenchOperations: @@ -25,6 +27,7 @@ def __init__(self, bench) -> None: self.frappe_bench_dir = self.bench.path / "workspace" / "frappe-bench" def create_fm_bench(self): + richprint.change_head("Configuring common_site_config.json") common_site_config_data = self.bench.bench_config.get_commmon_site_config_data( self.bench.services.database_manager.database_server_info @@ -94,7 +97,7 @@ def is_required_services_available(self): f"{self.bench.bench_config.container_name_prefix}-redis-socketio": 6379, } for service, port in required_services.items(): - output = self.wait_for_required_service(host=service, port=port) + output: SubprocessOutput = self.wait_for_required_service(host=service, port=port) if output.combined: richprint.print(output.combined[-1].replace('wait-for-it: ', '')) @@ -168,7 +171,7 @@ def split_supervisor_config(self): config.read_string(supervisor_conf_path.read_text()) for section_name in config.sections(): - if not "group:" in section_name: + if "group:" not in section_name: section_config = configparser.ConfigParser(interpolation=None) section_config.add_section(section_name) for key, value in config.items(section_name): diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index dd1c878e..852ffa7e 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -8,7 +8,7 @@ import subprocess from typing import Any, Dict, List, Optional from pathlib import Path -from frappe_manager.site_manager.bench_site_manager import BenchOperations +from frappe_manager.site_manager.bench_operations import BenchOperations from rich.table import Table from frappe_manager.compose_project.compose_project import ComposeProject from frappe_manager.docker_wrapper.DockerException import DockerException @@ -31,10 +31,9 @@ BenchSSLCertificateAlreadyIssued, BenchSSLCertificateNotIssued, BenchServiceNotRunning, - BenchWorkersSupervisorConfigurtionGenerateError, ) from frappe_manager.site_manager.workers_manager.SiteWorker import BenchWorkers -from frappe_manager.ssl_manager import LETSENCRYPT_PREFERRED_CHALLENGE, SUPPORTED_SSL_TYPES +from frappe_manager.ssl_manager import SUPPORTED_SSL_TYPES from frappe_manager.ssl_manager.certificate import SSLCertificate from frappe_manager.ssl_manager.nginxproxymanager import NginxProxyManager from frappe_manager.ssl_manager.ssl_certificate_manager import SSLCertificateManager @@ -50,7 +49,6 @@ CLI_BENCH_CONFIG_FILE_NAME, CLI_BENCHES_DIRECTORY, CLI_DIR, - EnableDisableOptionsEnum, SiteServicesEnum, ) from frappe_manager.utils.site import domain_level, generate_services_table, get_bench_db_connection_info @@ -183,10 +181,10 @@ def create(self, is_template_bench: bool = False): None """ try: - richprint.change_head(f"Creating Bench Directory") + richprint.change_head("Creating Bench Directory") self.path.mkdir(parents=True, exist_ok=True) - richprint.change_head(f"Generating bench compose") + richprint.change_head("Generating bench compose") self.generate_compose(self.bench_config.export_to_compose_inputs()) self.create_compose_dirs() @@ -197,9 +195,9 @@ def create(self, is_template_bench: bool = False): richprint.print(f"Created template bench: {self.name}", emoji_code=":white_check_mark:") return - richprint.change_head(f"Starting bench services") + richprint.change_head("Starting bench services") self.compose_project.start_service(force_recreate=True) - richprint.print(f"Started bench services.") + richprint.print("Started bench services.") richprint.change_head("Creating bench and bench site.") self.benchops.create_fm_bench() @@ -212,7 +210,7 @@ def create(self, is_template_bench: bool = False): self.save_bench_config() - richprint.change_head(f"Commencing site status check") + richprint.change_head("Commencing site status check") # check if bench is created if not self.is_bench_created(): @@ -224,9 +222,9 @@ def create(self, is_template_bench: bool = False): self.info() - if not ".localhost" in self.name: + if ".localhost" not in self.name: richprint.print( - f"Please note that You will have to add a host entry to your system's hosts file to access the bench locally." + "Please note that You will have to add a host entry to your system's hosts file to access the bench locally." ) except Exception as e: @@ -313,10 +311,6 @@ def generate_compose(self, inputs: dict) -> None: self.compose_project.compose_file_manager.set_network_alias("nginx", "site-network", [self.name]) self.compose_project.compose_file_manager.set_container_names(get_container_name_prefix(self.name)) - self.compose_project.compose_file_manager.set_secret_file_path( - "db_root_password", str(self.services.database_manager.database_server_info.secret_path.absolute()) - ) - self.compose_project.compose_file_manager.set_version(get_current_fm_version()) self.compose_project.compose_file_manager.set_top_networks_name( "site-network", get_container_name_prefix(self.name) @@ -464,9 +458,9 @@ def stop(self) -> bool: Returns: bool: True if the site is successfully stopped, False otherwise. """ - richprint.change_head(f"Stopping bench services") + richprint.change_head("Stopping bench services") self.compose_project.stop_service() - richprint.print(f"Stopped bench services.") + richprint.print("Stopped bench services.") if self.workers.compose_project.compose_file_manager.exists(): richprint.change_head("Starting bench workers services") @@ -509,7 +503,7 @@ def remove_containers_and_dirs(self): else: richprint.warning('Bench admin tools compose file not found. Skipping containers removal.') - richprint.change_head(f"Removing all bench files and directories.") + richprint.change_head("Removing all bench files and directories.") try: shutil.rmtree(self.path) except PermissionError: @@ -529,7 +523,7 @@ def remove_containers_and_dirs(self): except Exception: raise BenchRemoveDirectoryError(self.name, self.path) - richprint.print(f"Removed all bench files and directories.") + richprint.print("Removed all bench files and directories.") def is_bench_created(self, retry=60, interval=1) -> bool: curl_command = 'curl -I --max-time {retry} --connect-timeout {retry} {headers} {url}' @@ -591,7 +585,6 @@ def backup_workers_supervisor_conf(self): continue if file_path_abs.endswith(".fm.supervisor.conf"): from_path = file_path - to_path = file_path.parent / f"{file_path.name}.bak" backup_workers_manager.backup(from_path) file_path.unlink() return backup_workers_manager @@ -662,7 +655,7 @@ def info(self): this information using the richprint library. """ - richprint.change_head(f"Getting bench info") + richprint.change_head("Getting bench info") bench_db_info = self.get_db_connection_info() db_user = bench_db_info["name"] @@ -692,9 +685,11 @@ def info(self): "DB User": db_user, "DB Password": db_pass, "Environment": self.bench_config.environment_type.value, - "HTTPS": f'{ssl_service_type.upper()} ({format_ssl_certificate_time_remaining(self.certificate_manager.get_certficate_expiry())})' - if self.has_certificate() - else 'Not Enabled', + "HTTPS": ( + f'{ssl_service_type.upper()} ({format_ssl_certificate_time_remaining(self.certificate_manager.get_certficate_expiry())})' + if self.has_certificate() + else 'Not Enabled' + ), } if not self.bench_config.admin_tools: @@ -754,12 +749,7 @@ def shell(self, compose_service: str, user: str | None): user (str | None): The name of the user. If None, defaults to "frappe". """ - richprint.change_head(f"Spawning shell") - - import sys - - if sys.stdin.isatty() is False: - print('stdin ', sys.stdin.readlines()) + richprint.change_head("Spawning shell") if compose_service == "frappe" and not user: user = "frappe" @@ -788,6 +778,7 @@ def shell(self, compose_service: str, user: str | None): try: self.compose_project.docker.compose.exec(**exec_args) + except DockerException as e: richprint.warning(f"Shell exited with error code: {e.output.exit_code}") @@ -844,7 +835,7 @@ def logs(self, follow: bool, service: Optional[SiteServicesEnum] = None): follow (bool): Whether to continuously follow the logs or not. service (str, optional): The name of the service to display logs for. If not provided, logs for the entire site will be displayed. """ - richprint.change_head(f"Showing logs") + richprint.change_head("Showing logs") try: if not service: self.handle_frappe_server_file_logs(follow=follow) @@ -916,10 +907,10 @@ def attach_to_bench(self, user: str, extensions: List[str], workdir: str, debugg extensions_previous = [] if not extensions_previous == extensions or not user == user: - richprint.change_head(f"Configuration changed, regenerating label in bench compose") + richprint.change_head("Configuration changed, regenerating label in bench compose") self.compose_project.compose_file_manager.set_labels("frappe", labels) self.compose_project.compose_file_manager.write_to_file() - richprint.print(f"Regenerated bench compose.") + richprint.print("Regenerated bench compose.") self.compose_project.start_service(['frappe']) # sync debugger files @@ -972,7 +963,7 @@ def attach_to_bench(self, user: str, extensions: List[str], workdir: str, debugg if output.returncode != 0: raise BenchAttachTocontainerFailed(self.name, 'frappe') - richprint.print(f"Attached to frappe service container.") + richprint.print("Attached to frappe service container.") def remove_database_and_user(self): """ diff --git a/frappe_manager/site_manager/site_exceptions.py b/frappe_manager/site_manager/site_exceptions.py index 5d758793..e7dcd985 100644 --- a/frappe_manager/site_manager/site_exceptions.py +++ b/frappe_manager/site_manager/site_exceptions.py @@ -186,7 +186,14 @@ def __init__(self, bench_name, message="Supervisorctl is not running in frappe s class BenchOperationException(BenchException): - def __init__(self, bench_name, message: str, print_combined: bool = True, print_stdout: bool = False, print_stderr: bool = False): + def __init__( + self, + bench_name, + message: str, + print_combined: bool = True, + print_stdout: bool = False, + print_stderr: bool = False, + ): self.bench_name = bench_name self.message = message self.print_stdout = print_stdout @@ -199,11 +206,10 @@ def set_output(self, output: SubprocessOutput): self.output = output from rich.panel import Panel - import typer.rich_utils as ut - to_print = [] box: Box = Box("╭ \n" " \n" " ── \n" "│ \n" " \n" " \n" " | \n" " \n", ascii=True) + if self.print_stdout: panel = Panel.fit( "\n".join(self.output.stdout), diff --git a/frappe_manager/ssl_manager/letsencrypt_certificate.py b/frappe_manager/ssl_manager/letsencrypt_certificate.py index 87432950..88c8e49c 100644 --- a/frappe_manager/ssl_manager/letsencrypt_certificate.py +++ b/frappe_manager/ssl_manager/letsencrypt_certificate.py @@ -1,5 +1,5 @@ from typing import Optional, List, Self -from pydantic import EmailStr, Field, model_validator, validator +from pydantic import EmailStr, Field, model_validator from frappe_manager.ssl_manager import LETSENCRYPT_PREFERRED_CHALLENGE from frappe_manager.ssl_manager.certificate import SSLCertificate from frappe_manager.ssl_manager.certificate_exceptions import SSLDNSChallengeCredentailsNotFound diff --git a/frappe_manager/utils/site.py b/frappe_manager/utils/site.py index 07148ff6..9f265f07 100644 --- a/frappe_manager/utils/site.py +++ b/frappe_manager/utils/site.py @@ -7,6 +7,7 @@ from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.site_manager.site_exceptions import BenchException + def generate_services_table(services_status: dict): # running site services status services_table = Table( @@ -18,14 +19,10 @@ def generate_services_table(services_status: dict): box=None, ) - services_table.add_column( - "Service Status", ratio=1, no_wrap=True, width=None, min_width=20 - ) - services_table.add_column( - "Service Status", ratio=1, no_wrap=True, width=None, min_width=20 - ) + services_table.add_column("Service Status", ratio=1, no_wrap=True, width=None, min_width=20) + services_table.add_column("Service Status", ratio=1, no_wrap=True, width=None, min_width=20) - for index in range(0,len(services_status),2): + for index in range(0, len(services_status), 2): first_service_table = None second_service_table = None @@ -36,7 +33,7 @@ def generate_services_table(services_status: dict): pass try: - second_service = list(services_status.keys())[index+1] + second_service = list(services_status.keys())[index + 1] second_service_table = create_service_element(second_service, services_status[second_service]) except IndexError: pass @@ -45,6 +42,7 @@ def generate_services_table(services_status: dict): return services_table + def create_service_element(service, running_status): service_table = Table( show_lines=False, @@ -62,7 +60,8 @@ def create_service_element(service, running_status): ) return service_table -def parse_docker_volume(volume_string: str, root_volumes:dict, compose_path: Path): + +def parse_docker_volume(volume_string: str, root_volumes: dict, compose_path: Path): string_parts = volume_string.split(':') @@ -84,10 +83,11 @@ def parse_docker_volume(volume_string: str, root_volumes:dict, compose_path: Pat if not is_bind_mount: volume_type = DockerVolumeType.volume - docker_volume = DockerVolumeMount(src,dest,volume_type, compose_path) + docker_volume = DockerVolumeMount(src, dest, volume_type, compose_path) return docker_volume + def is_fqdn(hostname: str) -> bool: """ https://en.m.wikipedia.org/wiki/Fully_qualified_domain_name @@ -111,6 +111,7 @@ def is_fqdn(hostname: str) -> bool: # Check that all labels match that pattern. return all(fqdn.match(label) for label in labels) + def is_wildcard_fqdn(hostname: str) -> bool: """ Check if the hostname is a fully qualified domain name (FQDN) with optional wildcard. @@ -142,6 +143,7 @@ def is_wildcard_fqdn(hostname: str) -> bool: # Check the first label for wildcard pattern, then check all labels for standard pattern return status + def domain_level(domain): # Split the domain name into individual parts parts = domain.split('.') @@ -149,6 +151,7 @@ def domain_level(domain): # Return the number of parts minus 1 (excluding the TLD) return len(parts) - 1 + def validate_sitename(sitename: str) -> str: match = is_fqdn(sitename) @@ -156,10 +159,14 @@ def validate_sitename(sitename: str) -> str: sitename = sitename + ".localhost" if not match: - richprint.error(f"The {sitename} must follow Fully Qualified Domain Name (FQDN) format.", exception=BenchException(sitename,f"Valid FQDN site name not provided.")) + richprint.error( + f"The {sitename} must follow Fully Qualified Domain Name (FQDN) format.", + exception=BenchException(sitename, f"Valid FQDN site name not provided."), + ) return sitename + def get_bench_db_connection_info(bench_name: str, bench_path: Path): db_info = {} site_config_file = bench_path / "workspace" / "frappe-bench" / "sites" / bench_name / "site_config.json" From 37fa3865a8cc790603b76faae0ee3a4178bbcf2d Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 19 Jun 2024 15:23:23 +0530 Subject: [PATCH 21/44] check docker images before starting services --- frappe_manager/docker_wrapper/DockerClient.py | 59 +++++++++++-------- .../site_manager/bench_operations.py | 30 ++++++++++ frappe_manager/site_manager/site.py | 4 ++ .../site_manager/site_exceptions.py | 12 ++++ frappe_manager/utils/site.py | 17 ++++-- 5 files changed, 93 insertions(+), 29 deletions(-) diff --git a/frappe_manager/docker_wrapper/DockerClient.py b/frappe_manager/docker_wrapper/DockerClient.py index 4ea554b4..2cc41b02 100644 --- a/frappe_manager/docker_wrapper/DockerClient.py +++ b/frappe_manager/docker_wrapper/DockerClient.py @@ -20,10 +20,10 @@ class DockerClient: This class provide one to one mapping to the docker command. Only this args have are different use case. - stream (bool, optional): A boolean flag indicating whether to stream the output of the command as it runs. - If set to True, the output will be displayed in real-time. If set to False, the output will be + stream (bool, optional): A boolean flag indicating whether to stream the output of the command as it runs. + If set to True, the output will be displayed in real-time. If set to False, the output will be displayed after the command completes. Defaults to False. - stream_only_exit_code (bool, optional): A boolean flag indicating whether to only stream the exit code of the + stream_only_exit_code (bool, optional): A boolean flag indicating whether to only stream the exit code of the command. Defaults to False. """ @@ -60,7 +60,6 @@ def version(self) -> dict: version: dict = json.loads(" ".join(e.output.stdout)) return version - def server_running(self) -> bool: """ Checks if the Docker server is running. @@ -112,10 +111,7 @@ def cp( cp_cmd += [f"{source}"] cp_cmd += [f"{destination}"] - iterator = run_command_with_exit_code( - self.docker_cmd + cp_cmd, - stream=stream - ) + iterator = run_command_with_exit_code(self.docker_cmd + cp_cmd, stream=stream) return iterator def kill( @@ -132,10 +128,7 @@ def kill( kill_cmd += parameters_to_options(parameters, exclude=remove_parameters) kill_cmd += [f"{container}"] - iterator = run_command_with_exit_code( - self.docker_cmd + kill_cmd, - stream=stream - ) + iterator = run_command_with_exit_code(self.docker_cmd + kill_cmd, stream=stream) return iterator def rm( @@ -154,10 +147,7 @@ def rm( rm_cmd += parameters_to_options(parameters, exclude=remove_parameters) rm_cmd += [f"{container}"] - iterator = run_command_with_exit_code( - self.docker_cmd + rm_cmd, - stream=stream - ) + iterator = run_command_with_exit_code(self.docker_cmd + rm_cmd, stream=stream) return iterator def run( @@ -176,13 +166,13 @@ def run( parameters: dict = locals() run_cmd: list = ["run"] - remove_parameters = ["stream", "command", "image","use_shlex_split","env"] + remove_parameters = ["stream", "command", "image", "use_shlex_split", "env"] run_cmd += parameters_to_options(parameters, exclude=remove_parameters) - if type(env) == list: + if isinstance(env, list): for i in env: - run_cmd+= ["--env", i] + run_cmd += ["--env", i] run_cmd += [f"{image}"] @@ -192,10 +182,7 @@ def run( else: run_cmd += [command] - iterator = run_command_with_exit_code( - self.docker_cmd + run_cmd, - stream=stream - ) + iterator = run_command_with_exit_code(self.docker_cmd + run_cmd, stream=stream) return iterator def pull( @@ -209,7 +196,7 @@ def pull( pull_cmd: list[str] = ["pull"] - remove_parameters = ["stream","container_name"] + remove_parameters = ["stream", "container_name"] pull_cmd += parameters_to_options(parameters, exclude=remove_parameters) pull_cmd += [container_name] @@ -219,3 +206,27 @@ def pull( stream=stream, ) return iterator + + def images( + self, + format: Literal['json'] = 'json', + ): + parameters: dict = locals() + + images_cmd: list[str] = ["images"] + remove_parameters = [] + + images_cmd += parameters_to_options(parameters, exclude=remove_parameters) + + output: SubprocessOutput = run_command_with_exit_code( + self.docker_cmd + images_cmd, + stream=False, + ) + + images = [] + + if output.stdout: + for image in output.stdout: + images.append(json.loads(image)) + + return images diff --git a/frappe_manager/site_manager/bench_operations.py b/frappe_manager/site_manager/bench_operations.py index feb081cd..1f108119 100644 --- a/frappe_manager/site_manager/bench_operations.py +++ b/frappe_manager/site_manager/bench_operations.py @@ -325,3 +325,33 @@ def wait_for_required_service(self, host: str, port: int, timeout: int = 120): ), capture_output=True, ) + + def check_required_docker_images_available(self): + richprint.change_head("Checking required docker images availability") + fm_images = get_all_docker_images() + system_available_images = self.bench.compose_project.docker.images() + + not_available_images = [] + + for key, value in fm_images.items(): + name = value['name'] + tag = value['tag'] + + found = False + + for item in system_available_images: + if item.get('Repository') == name and item.get('Tag') == tag: + found = True + break + + if not found: + image = f"{name}:{tag}" + not_available_images.append(image) + + # remove duplicates + not_available_images = list(dict.fromkeys(not_available_images)) + + if not_available_images: + for image in not_available_images: + richprint.error(f"Docker image '{image}' is not available locally") + raise BenchOperationRequiredDockerImagesNotAvailable(self.bench.name, 'fm self update images') diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 852ffa7e..3eeede10 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -180,6 +180,8 @@ def create(self, is_template_bench: bool = False): Returns: None """ + self.benchops.check_required_docker_images_available() + try: richprint.change_head("Creating Bench Directory") self.path.mkdir(parents=True, exist_ok=True) @@ -394,6 +396,8 @@ def start(self, force: bool = False): Starts the bench. """ + self.benchops.check_required_docker_images_available() + # Should be done in site manager ? global_db_info = self.services.database_manager.database_server_info self.sync_bench_common_site_config(global_db_info.host, global_db_info.port) diff --git a/frappe_manager/site_manager/site_exceptions.py b/frappe_manager/site_manager/site_exceptions.py index e7dcd985..5179ddeb 100644 --- a/frappe_manager/site_manager/site_exceptions.py +++ b/frappe_manager/site_manager/site_exceptions.py @@ -256,6 +256,18 @@ def __init__(self, bench_name, app: str, branch: str, message: str = "Failed to super().__init__(self.bench_name, self.message) +class BenchOperationRequiredDockerImagesNotAvailable(BenchException): + def __init__( + self, + bench_name, + pull_command, + message: str = "Required docker images not available. Pull all required images using command '{}'.", + ): + self.bench_name = bench_name + self.message = message.format(pull_command) + super().__init__(self.bench_name, self.message) + + class BenchOperationWaitForRequiredServiceFailed(BenchOperationException): def __init__( self, diff --git a/frappe_manager/utils/site.py b/frappe_manager/utils/site.py index 9f265f07..c94ba3e3 100644 --- a/frappe_manager/utils/site.py +++ b/frappe_manager/utils/site.py @@ -182,13 +182,10 @@ def get_bench_db_connection_info(bench_name: str, bench_path: Path): db_info["password"] = None return db_info -def pull_docker_images() -> bool: + +def get_all_docker_images(): from frappe_manager.compose_manager.ComposeFile import ComposeFile - from frappe_manager.docker_wrapper.DockerClient import DockerClient - from frappe_manager.docker_wrapper.DockerException import DockerException - images_list = [] - docker = DockerClient() temp_bench_compose_file_manager = ComposeFile(loadfile=Path('/dev/null/docker-compose.yml')) services_manager_compose_file_manager = ComposeFile( loadfile=Path('/dev/null/docker-compose.yml'), template_name='docker-compose.services.tmpl' @@ -200,6 +197,16 @@ def pull_docker_images() -> bool: images = temp_bench_compose_file_manager.get_all_images() images.update(services_manager_compose_file_manager.get_all_images()) images.update(admin_tools_manager_compose_file_manager.get_all_images()) + return images + + +def pull_docker_images() -> bool: + from frappe_manager.docker_wrapper.DockerException import DockerException + from frappe_manager.docker_wrapper.DockerClient import DockerClient + + docker = DockerClient() + images = get_all_docker_images() + images_list = [] for service, image_info in images.items(): image = f"{image_info['name']}:{image_info['tag']}" From 7af01a0e6f9f96a82c76ef1b1680ab8a06a37006 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 19 Jun 2024 17:43:27 +0530 Subject: [PATCH 22/44] Update status messages --- frappe_manager/commands.py | 3 ++- frappe_manager/display_manager/DisplayManager.py | 4 ++-- frappe_manager/site_manager/bench_operations.py | 2 +- frappe_manager/site_manager/site.py | 10 +++++----- frappe_manager/site_manager/site_exceptions.py | 10 ++++++++-- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index 858ec3d0..e319573f 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -111,9 +111,10 @@ def app_callback( richprint.print("It seems like the first installation. Pulling docker images...️", "🔍") completed_status = pull_docker_images() + if not completed_status: shutil.rmtree(CLI_DIR) - richprint.exit("Aborting. [bold][blue]fm[/blue][/bold] will not be able to work without images. 🖼️") + richprint.exit("Aborting. Not able to pull all required Docker images.") current_version = Version(get_current_fm_version()) fm_config_manager.version = current_version diff --git a/frappe_manager/display_manager/DisplayManager.py b/frappe_manager/display_manager/DisplayManager.py index a7b80ea6..06b14b2f 100644 --- a/frappe_manager/display_manager/DisplayManager.py +++ b/frappe_manager/display_manager/DisplayManager.py @@ -89,7 +89,7 @@ def exit(self, text: str, emoji_code: str = ":x:", os_exit=False, error_msg=None raise typer.Exit(1) - def print(self, text: str, emoji_code: str = ":white_check_mark:", prefix: Optional[str] = None): + def print(self, text: str, emoji_code: str = ":white_check_mark:", prefix: Optional[str] = None, **kwargs): """ Prints the given text with an optional emoji code. @@ -102,7 +102,7 @@ def print(self, text: str, emoji_code: str = ":white_check_mark:", prefix: Optio if prefix: msg = f"{emoji_code} {prefix} {text}" - self.stdout.print(msg) + self.stdout.print(msg, **kwargs) def update_head(self, text: str): """ diff --git a/frappe_manager/site_manager/bench_operations.py b/frappe_manager/site_manager/bench_operations.py index 1f108119..a1be733a 100644 --- a/frappe_manager/site_manager/bench_operations.py +++ b/frappe_manager/site_manager/bench_operations.py @@ -99,7 +99,7 @@ def is_required_services_available(self): for service, port in required_services.items(): output: SubprocessOutput = self.wait_for_required_service(host=service, port=port) if output.combined: - richprint.print(output.combined[-1].replace('wait-for-it: ', '')) + richprint.print(output.combined[-1].replace('wait-for-it: ', ''), highlight=False) def frappe_container_run( self, diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 3eeede10..8aa08c94 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -232,7 +232,7 @@ def create(self, is_template_bench: bool = False): except Exception as e: richprint.stop() - richprint.error(f"[red][bold]Error Occured : {e}[/bold][/red]") + richprint.error(f"[red][bold]Error Occured: [/bold][/red]{e}") exception_traceback_str = capture_and_format_exception() @@ -418,8 +418,6 @@ def start(self, force: bool = False): self.workers.compose_project.start_service(force_recreate=force) richprint.print("Started bench workers services.") - richprint.print('Started frappe server.') - def frappe_logs_till_start(self): """ Retrieves and prints the logs of the 'frappe' service until site supervisor starts. @@ -1134,7 +1132,8 @@ def switch_bench_env(self): start_command = 'supervisorctl -c /opt/user/supervisord.conf start all' self.frappe_service_run_command(start_command) - richprint.print(f"Started {self.bench_config.environment_type.value} services.") + + richprint.print(f"Configured and Started {self.bench_config.environment_type.value} services.") elif self.bench_config.environment_type == FMBenchEnvType.prod: self.remove_dev_packages() @@ -1157,7 +1156,8 @@ def switch_bench_env(self): start_command = 'supervisorctl -c /opt/user/supervisord.conf start all' self.frappe_service_run_command(start_command) - richprint.print(f"Started {self.bench_config.environment_type.value} services.") + + richprint.print(f"Configured and Started {self.bench_config.environment_type.value} services.") def is_supervisord_running(self, interval: int = 2, timeout: int = 30): for i in range(timeout): diff --git a/frappe_manager/site_manager/site_exceptions.py b/frappe_manager/site_manager/site_exceptions.py index 5179ddeb..fe901bfc 100644 --- a/frappe_manager/site_manager/site_exceptions.py +++ b/frappe_manager/site_manager/site_exceptions.py @@ -14,8 +14,13 @@ def __init__( self, bench_name: str, message: str, + prefix_bench_name: bool = True, ): - self.message = f"[blue][bold]{bench_name} :[/bold][/blue] {message}" + self.message = message + + if prefix_bench_name: + self.message = f"[blue][bold]{bench_name} :[/bold][/blue] {message}" + super().__init__(self.message) @@ -244,7 +249,8 @@ def set_output(self, output: SubprocessOutput): to_print.append(helpers.rich_object_to_string(panel)) self.message = self.message + "\n" + "\n".join(to_print) - super().__init__(self.bench_name, self.message) + + super().__init__(self.bench_name, self.message, prefix_bench_name=False) class BenchOperationFrappeBranchChangeFailed(BenchException): From d4de05141324ceb5f85a67b537d8f061a6fa45db Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 19 Jun 2024 17:46:51 +0530 Subject: [PATCH 23/44] Update frappe docker image tag --- Docker/images-tag.json | 2 +- frappe_manager/templates/docker-compose.tmpl | 6 +++--- frappe_manager/templates/docker-compose.workers.tmpl | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Docker/images-tag.json b/Docker/images-tag.json index defdf3a5..92e32eab 100644 --- a/Docker/images-tag.json +++ b/Docker/images-tag.json @@ -1,5 +1,5 @@ { - "frappe": "v0.13.0", + "frappe": "v0.15.0", "nginx": "v0.13.0", "mailhog": "v0.8.3" } diff --git a/frappe_manager/templates/docker-compose.tmpl b/frappe_manager/templates/docker-compose.tmpl index 54ce5b87..012101ff 100644 --- a/frappe_manager/templates/docker-compose.tmpl +++ b/frappe_manager/templates/docker-compose.tmpl @@ -1,6 +1,6 @@ services: frappe: - image: ghcr.io/rtcamp/frappe-manager-frappe:v0.13.0 + image: ghcr.io/rtcamp/frappe-manager-frappe:v0.15.0 container_name: REPLACE_ME_WITH_CONTAINER_NAME environment: USERID: REPLACE_ME_WITH_CURRENT_USER @@ -40,7 +40,7 @@ services: global-frontend-network: socketio: - image: ghcr.io/rtcamp/frappe-manager-frappe:v0.13.0 + image: ghcr.io/rtcamp/frappe-manager-frappe:v0.15.0 container_name: REPLACE_ME_WITH_CONTAINER_NAME environment: TIMEOUT: 60000 @@ -58,7 +58,7 @@ services: site-network: schedule: - image: ghcr.io/rtcamp/frappe-manager-frappe:v0.13.0 + image: ghcr.io/rtcamp/frappe-manager-frappe:v0.15.0 container_name: REPLACE_ME_WITH_CONTAINER_NAME environment: TIMEOUT: 60000 diff --git a/frappe_manager/templates/docker-compose.workers.tmpl b/frappe_manager/templates/docker-compose.workers.tmpl index 316d3872..fa927c77 100644 --- a/frappe_manager/templates/docker-compose.workers.tmpl +++ b/frappe_manager/templates/docker-compose.workers.tmpl @@ -1,6 +1,6 @@ services: worker-name: - image: ghcr.io/rtcamp/frappe-manager-frappe:v0.13.0 + image: ghcr.io/rtcamp/frappe-manager-frappe:v0.15.0 environment: TIMEOUT: 6000 CHANGE_DIR: /workspace/frappe-bench From 0773c7b444fbf4e0fc35ae2732abd126606774f4 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 19 Jun 2024 17:47:20 +0530 Subject: [PATCH 24/44] Update black and ruff configs --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f761189d..c9e7e883 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,11 @@ documentation = "https://github.com/rtcamp/frappe-manager/wiki" readme = "README.md" [tool.black] -line-length = 150 +line-length = 120 +skip-string-normalization = true + +[tool.ruff] +ignore = ["F841"] [tool.poetry.urls] "Bug Tracker" = "https://github.com/rtcamp/frappe-manager/issues" From a5d18d38d560050545928e5b8bfe726638ac67e5 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 19 Jun 2024 18:45:07 +0530 Subject: [PATCH 25/44] Remove test command --- frappe_manager/commands.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index e319573f..d66d45da 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -622,38 +622,3 @@ def update( if bench_config_save: bench.save_bench_config() - - -@app.command() -def test( - ctx: typer.Context, - benchname: Annotated[ - Optional[str], - typer.Argument( - help="Name of the bench.", autocompletion=sites_autocompletion_callback, callback=sitename_callback - ), - ] = None, - apps: Annotated[ - List[str], - typer.Option( - "--apps", - "-a", - help="FrappeVerse apps to install. App should be specified in format : or .", - callback=apps_list_validation_callback, - show_default=False, - ), - ] = [], -): - """Shows information about given bench.""" - - services_manager = ctx.obj["services"] - verbose = ctx.obj['verbose'] - bench = Bench.get_object(benchname, services_manager) - from frappe_manager.site_manager.bench_site_manager import BenchOperations - - print(STABLE_APP_BRANCH_MAPPING_LIST.keys()) - # benchops = BenchOperations(bench) - # benchops.bench_install_app_env('crmxx') - - # benchops.is_required_services_available() - # benchops.change_frappeverse_prebaked_app_branch('frappe', 'version-14') From 2b1cea897ea9be2e854c4605f262156c9b39aa85 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 19 Jun 2024 20:46:26 +0530 Subject: [PATCH 26/44] add site name in zsh shell prompt --- Docker/frappe/Dockerfile | 1 + Docker/frappe/entrypoint.sh | 1 + Docker/frappe/fm.zsh-theme | 57 +++++++++++++++++++++++++++++++++++++ Docker/frappe/zshrc | 2 +- 4 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 Docker/frappe/fm.zsh-theme diff --git a/Docker/frappe/Dockerfile b/Docker/frappe/Dockerfile index 68551fce..04952673 100644 --- a/Docker/frappe/Dockerfile +++ b/Docker/frappe/Dockerfile @@ -120,6 +120,7 @@ RUN sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master RUN unset ZSH COPY --chown=frappe:frappe ./zshrc /opt/user/.zshrc +COPY --chown=frappe:frappe ./fm.zsh-theme /opt/user/fm.zsh-theme RUN git clone --depth 1 https://github.com/pyenv/pyenv.git .pyenv \ && pyenv install $PYTHON_VERSION \ diff --git a/Docker/frappe/entrypoint.sh b/Docker/frappe/entrypoint.sh index b41d3bdf..eff47eb6 100755 --- a/Docker/frappe/entrypoint.sh +++ b/Docker/frappe/entrypoint.sh @@ -34,6 +34,7 @@ echo "Time taken for chown /opt : $execution_time seconds" if [[ ! -d "/workspace/.oh-my-zsh" ]]; then cp -pr /opt/user/.oh-my-zsh /workspace/ + cp -p /opt/user/fm.zsh-theme /workspace/.oh-my-zsh/custom/themes/ fi if [[ ! -f "/workspace/.zshrc" ]]; then diff --git a/Docker/frappe/fm.zsh-theme b/Docker/frappe/fm.zsh-theme new file mode 100644 index 00000000..19e3295e --- /dev/null +++ b/Docker/frappe/fm.zsh-theme @@ -0,0 +1,57 @@ +# Based on gnsh theme + +setopt prompt_subst + +() { + +local PR_USER PR_USER_OP PR_PROMPT PR_HOST + +# Check the UID +if [[ $UID -ne 0 ]]; then # normal user + PR_USER='%F{green}%n%f' + PR_USER_OP='%F{green}%#%f' + PR_PROMPT='%f➤ %f' +else # root + PR_USER='%F{red}%n%f' + PR_USER_OP='%F{red}%#%f' + PR_PROMPT='%F{red}➤ %f' +fi + +FM_HOST() { + local default_hostname="%${1}" + local common_site_config="/workspace/frappe-bench/sites/common_site_config.json" + if [[ -f "$common_site_config" ]]; then + fm_hostname=$(jq -r .default_site "$common_site_config") + if [[ "$fm_hostname" != "null" ]]; then + echo "$fm_hostname" + else + echo "$default_hostname" # Default to hostname + fi + else + echo "$default_hostname" # Default to hostname + fi +} + +# Check if we are on SSH or not +if [[ -n "$SSH_CLIENT" || -n "$SSH2_CLIENT" ]]; then + PR_HOST='%F{red}$(FM_HOST M)%f' # SSH +else + PR_HOST='%F{green}$(FM_HOST m)%f' # no SSH +fi + +local return_code="%(?..%F{red}%? ↵%f)" + +local user_host="${PR_USER}%F{cyan}@${PR_HOST}" +local current_dir="%B%F{blue}%~%f%b" +local git_branch='$(git_prompt_info)' + +PROMPT="╭─${user_host} ${current_dir} \$(ruby_prompt_info) ${git_branch} +╰─$PR_PROMPT " +RPROMPT="${return_code}" + +ZSH_THEME_GIT_PROMPT_PREFIX="%F{yellow}‹" +ZSH_THEME_GIT_PROMPT_SUFFIX="› %f" +ZSH_THEME_RUBY_PROMPT_PREFIX="%F{red}‹" +ZSH_THEME_RUBY_PROMPT_SUFFIX="›%f" + +} diff --git a/Docker/frappe/zshrc b/Docker/frappe/zshrc index bf6c7f16..f970b210 100644 --- a/Docker/frappe/zshrc +++ b/Docker/frappe/zshrc @@ -5,7 +5,7 @@ export PATH=$HOME/.local/bin:$PATH # Path to your oh-my-zsh installation. export ZSH=$HOME/.oh-my-zsh -ZSH_THEME="gnzh" +ZSH_THEME="fm" plugins=(git z) From 1b1cf3e7f2613ff8ef3294175e1eab4e4cd30a0f Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 20 Jun 2024 17:29:02 +0530 Subject: [PATCH 27/44] Update services osx template with specific services version --- frappe_manager/templates/docker-compose.services.osx.tmpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe_manager/templates/docker-compose.services.osx.tmpl b/frappe_manager/templates/docker-compose.services.osx.tmpl index 053e83c6..48a689d2 100644 --- a/frappe_manager/templates/docker-compose.services.osx.tmpl +++ b/frappe_manager/templates/docker-compose.services.osx.tmpl @@ -27,7 +27,9 @@ services: global-nginx-proxy: container_name: fm_global-nginx-proxy user: REPLACE_WITH_CURRENT_USER:REPLACE_WITH_CURRENT_USER_GROUP - image: jwilder/nginx-proxy + image: jwilder/nginx-proxy:1.6 + environment: + ACME_HTTP_CHALLENGE_LOCATION: false ports: - "80:80" - "443:443" From bb22067b66fec08481689b544dfb7b4f8e1c1ad3 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 20 Jun 2024 17:30:24 +0530 Subject: [PATCH 28/44] fix: supervisor double backup --- frappe_manager/site_manager/bench_operations.py | 10 ++++++---- frappe_manager/site_manager/site.py | 16 ++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/frappe_manager/site_manager/bench_operations.py b/frappe_manager/site_manager/bench_operations.py index a1be733a..1718862e 100644 --- a/frappe_manager/site_manager/bench_operations.py +++ b/frappe_manager/site_manager/bench_operations.py @@ -151,17 +151,19 @@ def setup_supervisor(self, force: bool = False): richprint.change_head("Checking supervisor configuration") if not supervisor_conf_path.exists() or force: - richprint.change_head("Setting up supervisor configs") + richprint.change_head("Configuring supervisor configs") bench_setup_supervisor_command = self.bench_cli_cmd + [ "setup supervisor --skip-redis --skip-supervisord --yes --user frappe" ] bench_setup_supervisor_command = " ".join(bench_setup_supervisor_command) - bench_setup_supervisor_exception = BenchOperationException(self.bench.name, "Failed to setup supervisor.") + bench_setup_supervisor_exception = BenchOperationException( + self.bench.name, "Failed to configure supervisor." + ) self.frappe_container_run(bench_setup_supervisor_command, bench_setup_supervisor_exception) self.split_supervisor_config() - richprint.print("Setuped supervisor configs") + richprint.print("Configured supervisor configs") def split_supervisor_config(self): import configparser @@ -222,7 +224,7 @@ def bench_install_apps(self, apps_lists, already_installed_apps: Dict = STABLE_A if app not in to_install_apps: richprint.change_head(f"Removing prebaked app {app} from python env.") self.bench_rm_app_env(app) - richprint.print(f"Remove prebaked app {app}") + richprint.print(f"Removed prebaked app {app}") for app_info in apps_lists: app = app_info["app"] diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 8aa08c94..84fbf571 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -206,7 +206,7 @@ def create(self, is_template_bench: bool = False): self.sync_bench_config_configuration() richprint.change_head("Configuring bench workers.") - self.sync_workers_compose(force_recreate=True) + self.sync_workers_compose(force_recreate=True, setup_supervisor=False) richprint.change_head("Configuring bench workers.") richprint.update_live() @@ -551,13 +551,13 @@ def is_bench_created(self, retry=60, interval=1) -> bool: time.sleep(interval) return False - def sync_workers_compose(self, force_recreate: bool = False): - workers_backup_manager = self.backup_workers_supervisor_conf() - - try: - self.benchops.setup_supervisor(force=True) - except BenchOperationException as e: - self.backup_restore_workers_supervisor(workers_backup_manager) + def sync_workers_compose(self, force_recreate: bool = False, setup_supervisor: bool = True): + if setup_supervisor: + workers_backup_manager = self.backup_workers_supervisor_conf() + try: + self.benchops.setup_supervisor(force=True) + except BenchOperationException as e: + self.backup_restore_workers_supervisor(workers_backup_manager) are_workers_not_changed = self.workers.is_new_workers_added() From ce829eda10cfa3a283562ce04751830336b29666 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 20 Jun 2024 18:55:42 +0530 Subject: [PATCH 29/44] fix: backup manager in migrations --- frappe_manager/migration_manager/backup_manager.py | 2 +- frappe_manager/migration_manager/migration_base.py | 2 +- .../migration_manager/migrations/migrate_0_10_0.py | 2 +- .../migration_manager/migrations/migrate_0_11_0.py | 2 +- .../migration_manager/migrations/migrate_0_12_0.py | 2 +- .../migration_manager/migrations/migrate_0_13_0.py | 13 +++---------- .../migration_manager/migrations/migrate_0_13_1.py | 2 +- .../migration_manager/migrations/migrate_0_14_0.py | 2 +- .../migration_manager/migrations/migrate_0_9_0.py | 2 +- 9 files changed, 11 insertions(+), 18 deletions(-) diff --git a/frappe_manager/migration_manager/backup_manager.py b/frappe_manager/migration_manager/backup_manager.py index 1864a502..ab4dcba1 100644 --- a/frappe_manager/migration_manager/backup_manager.py +++ b/frappe_manager/migration_manager/backup_manager.py @@ -51,7 +51,7 @@ def exists(self): current_migration_timestamp = f"{datetime.now().strftime('%d-%b-%y--%H-%M-%S')}" -CLI_MIGARATIONS_DIR = CLI_DIR / 'backups' / 'migrations' +CLI_MIGARATIONS_DIR = CLI_DIR / 'backups' class BackupManager: diff --git a/frappe_manager/migration_manager/migration_base.py b/frappe_manager/migration_manager/migration_base.py index e5d3adad..184f5b23 100644 --- a/frappe_manager/migration_manager/migration_base.py +++ b/frappe_manager/migration_manager/migration_base.py @@ -25,7 +25,7 @@ class MigrationBase(ABC): logger: Logger = log.get_logger() def init(self): - self.backup_manager = BackupManager(str(self.version), self.benches_dir) + self.backup_manager = BackupManager(name=str(self.version), benches_dir=self.benches_dir) self.benches_manager = MigrationBenches(self.benches_dir) self.services_manager: MigrationServicesManager = MigrationServicesManager(services_path=CLI_DIR / 'services') diff --git a/frappe_manager/migration_manager/migrations/migrate_0_10_0.py b/frappe_manager/migration_manager/migrations/migrate_0_10_0.py index b11b3576..f2116621 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_10_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_10_0.py @@ -40,7 +40,7 @@ class MigrationV0100(MigrationBase): def init(self): self.benches_dir = CLI_DIR / "sites" - self.backup_manager = BackupManager(str(self.version), self.benches_dir) + self.backup_manager = BackupManager(name=str(self.version), benches_dir=self.benches_dir) self.string_timestamp = datetime.now().strftime("%d-%b-%y--%H-%M-%S") self.benches_manager = MigrationBenches(self.benches_dir) self.services_manager: MigrationServicesManager = MigrationServicesManager() diff --git a/frappe_manager/migration_manager/migrations/migrate_0_11_0.py b/frappe_manager/migration_manager/migrations/migrate_0_11_0.py index f0759a01..c7c18e6f 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_11_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_11_0.py @@ -17,7 +17,7 @@ class MigrationV0110(MigrationBase): def init(self): self.cli_dir: Path = Path.home() / 'frappe' self.benches_dir = self.cli_dir / "sites" - self.backup_manager = BackupManager(str(self.version), self.benches_dir) + self.backup_manager = BackupManager(name=str(self.version), benches_dir=self.benches_dir) self.benches_manager = MigrationBenches(self.benches_dir) self.services_manager: MigrationServicesManager = MigrationServicesManager( services_path=self.cli_dir / 'services' diff --git a/frappe_manager/migration_manager/migrations/migrate_0_12_0.py b/frappe_manager/migration_manager/migrations/migrate_0_12_0.py index b3362811..0dd4f03b 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_12_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_12_0.py @@ -19,7 +19,7 @@ class MigrationV0120(MigrationBase): def init(self): self.cli_dir: Path = Path.home() / 'frappe' self.benches_dir = self.cli_dir / "sites" - self.backup_manager = BackupManager(str(self.version), self.benches_dir) + self.backup_manager = BackupManager(name=str(self.version), benches_dir=self.benches_dir) self.benches_manager = MigrationBenches(self.benches_dir) self.services_manager: MigrationServicesManager = MigrationServicesManager( services_path=self.cli_dir / 'services' diff --git a/frappe_manager/migration_manager/migrations/migrate_0_13_0.py b/frappe_manager/migration_manager/migrations/migrate_0_13_0.py index 207fd1c6..6ffb5012 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_13_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_13_0.py @@ -26,26 +26,18 @@ class MigrationV0130(MigrationBase): def init(self): self.cli_dir: Path = Path.home() / 'frappe' self.benches_dir = self.cli_dir / "sites" - self.backup_manager = BackupManager(str(self.version), self.benches_dir) + self.backup_manager = BackupManager(name=str(self.version), benches_dir=self.benches_dir) self.benches_manager = MigrationBenches(self.benches_dir) self.services_manager: MigrationServicesManager = MigrationServicesManager( services_path=self.cli_dir / 'services' ) def migrate_services(self): - # backup services compose - if not self.services_manager.compose_project.compose_file_manager.exists(): - raise MigrationExceptionInBench( - f"Services compose at {self.services_manager.compose_project.compose_file_manager} not found." - ) - - self.backup_manager.backup(self.services_manager.compose_project.compose_file_manager.compose_path) - # remove version from services yml try: del self.services_manager.compose_project.compose_file_manager.yml['version'] except KeyError: - self.logger.warning(f"[services]: 'version' attribute not found in compose file.") + self.logger.warning("[services]: 'version' attribute not found in compose file.") pass # include new volume info @@ -207,6 +199,7 @@ def migrate_bench_compose(self, bench: MigrationBench): bench.compose_project.compose_file_manager.write_to_file() richprint.print(f"Migrated [blue]{bench.name}[/blue] compose file.") + self.migrate_workers_compose(bench) def migrate_workers_compose(self, bench: MigrationBench): if bench.workers_compose_project.compose_file_manager.compose_path.exists(): diff --git a/frappe_manager/migration_manager/migrations/migrate_0_13_1.py b/frappe_manager/migration_manager/migrations/migrate_0_13_1.py index e7f3b9c0..0efb3f1e 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_13_1.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_13_1.py @@ -16,7 +16,7 @@ class MigrationV0131(MigrationBase): def init(self): self.cli_dir: Path = Path.home() / 'frappe' self.benches_dir = self.cli_dir / "sites" - self.backup_manager = BackupManager(str(self.version), self.benches_dir) + self.backup_manager = BackupManager(name=str(self.version), benches_dir=self.benches_dir) self.benches_manager = MigrationBenches(self.benches_dir) self.services_manager: MigrationServicesManager = MigrationServicesManager( services_path=self.cli_dir / 'services' diff --git a/frappe_manager/migration_manager/migrations/migrate_0_14_0.py b/frappe_manager/migration_manager/migrations/migrate_0_14_0.py index cdd50511..ad38090b 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_14_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_14_0.py @@ -13,7 +13,7 @@ class MigrationV0140(MigrationBase): def init(self): self.cli_dir: Path = Path.home() / 'frappe' self.benches_dir = self.cli_dir / "sites" - self.backup_manager = BackupManager(str(self.version), self.benches_dir) + self.backup_manager = BackupManager(name=str(self.version), benches_dir=self.benches_dir) self.benches_manager = MigrationBenches(self.benches_dir) self.services_manager: MigrationServicesManager = MigrationServicesManager( services_path=self.cli_dir / 'services' diff --git a/frappe_manager/migration_manager/migrations/migrate_0_9_0.py b/frappe_manager/migration_manager/migrations/migrate_0_9_0.py index 72204c00..0e5d8248 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_9_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_9_0.py @@ -12,7 +12,7 @@ class MigrationV090(MigrationBase): def init(self): self.benches_dir = CLI_DIR / "sites" - self.backup_manager = BackupManager(str(self.version)) + self.backup_manager = BackupManager(name=str(self.version), benches_dir=self.benches_dir) if self.benches_dir.exists(): self.skip = True From 3e95ae9265089bae9238e5a71708fe3267183af5 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 20 Jun 2024 18:56:40 +0530 Subject: [PATCH 30/44] add v0.15.0 migration --- .../migrations/migrate_0_15_0.py | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 frappe_manager/migration_manager/migrations/migrate_0_15_0.py diff --git a/frappe_manager/migration_manager/migrations/migrate_0_15_0.py b/frappe_manager/migration_manager/migrations/migrate_0_15_0.py new file mode 100644 index 00000000..b83d1f6b --- /dev/null +++ b/frappe_manager/migration_manager/migrations/migrate_0_15_0.py @@ -0,0 +1,140 @@ +from pathlib import Path +from frappe_manager.compose_manager.ComposeFile import ComposeFile +from frappe_manager.compose_project.compose_project import ComposeProject +from frappe_manager.docker_wrapper.DockerClient import DockerClient +from frappe_manager.migration_manager.migration_base import MigrationBase +from frappe_manager.migration_manager.migration_exections import MigrationExceptionInBench +from frappe_manager.migration_manager.migration_helpers import ( + MigrationBench, + MigrationBenches, + MigrationServicesManager, +) +from frappe_manager.migration_manager.version import Version +from frappe_manager.migration_manager.backup_manager import BackupManager +from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager.utils.helpers import get_container_name_prefix + + +class MigrationV0150(MigrationBase): + version = Version("0.15.0") + + def init(self): + self.cli_dir: Path = Path.home() / 'frappe' + self.benches_dir = self.cli_dir / "sites" + self.backup_manager = BackupManager(name=str(self.version), benches_dir=self.benches_dir) + self.benches_manager = MigrationBenches(self.benches_dir) + self.services_manager: MigrationServicesManager = MigrationServicesManager( + services_path=self.cli_dir / 'services' + ) + self.pulled_images_list = [] + + def migrate_services(self): + images_info = self.services_manager.compose_project.compose_file_manager.get_all_images() + images_info['global-nginx-proxy'] = {'name': 'jwilder/nginx-proxy', 'tag': '1.6'} + + # pulling nginx proxy image + pull_image = f"{images_info['global-nginx-proxy']['name']}:{images_info['global-nginx-proxy']['tag']}" + richprint.change_head(f"Pulling Image {pull_image}") + output = DockerClient().pull(container_name=pull_image, stream=True) + richprint.live_lines(output, padding=(0, 0, 0, 2)) + richprint.print(f"Image pulled [blue]{pull_image}[/blue]") + + # add ACME_HTTP_CHALLENGE_LOCATION: false env + envs = self.services_manager.compose_project.compose_file_manager.get_all_envs() + envs['global-nginx-proxy'] = {'ACME_HTTP_CHALLENGE_LOCATION': False} + + self.services_manager.compose_project.compose_file_manager.set_all_envs(envs) + self.services_manager.compose_project.compose_file_manager.set_version(str(self.version)) + self.services_manager.compose_project.compose_file_manager.write_to_file() + + def migrate_bench(self, bench: MigrationBench): + bench.compose_project.down_service(volumes=True) + richprint.change_head("Migrating bench compose") + + if not bench.compose_project.compose_file_manager.exists(): + richprint.print(f"Failed to migrate {bench.name} compose file.") + raise MigrationExceptionInBench(f"{bench.compose_project.compose_file_manager.compose_path} not found.") + + images_info = bench.compose_project.compose_file_manager.get_all_images() + + frappe_image_info = images_info['frappe'] + frappe_image_info['tag'] = self.version.version_string() + + redis_image_info = images_info['redis-cache'] + redis_image_info['tag'] = '6.2-alpine' + + # change image frappe, socketio, schedule + images_info['frappe'] = frappe_image_info + images_info['socketio'] = frappe_image_info + images_info['schedule'] = frappe_image_info + + # change image for redis + images_info['redis-cache'] = redis_image_info + images_info['redis-queue'] = redis_image_info + images_info['redis-socketio'] = redis_image_info + + for image in [frappe_image_info, redis_image_info]: + pull_image = f"{image['name']}:{image['tag']}" + if pull_image not in self.pulled_images_list: + richprint.change_head(f"Pulling Image {pull_image}") + output = DockerClient().pull(container_name=pull_image, stream=True) + richprint.live_lines(output, padding=(0, 0, 0, 2)) + richprint.print(f"Image pulled [blue]{pull_image}[/blue]") + self.pulled_images_list.append(pull_image) + + bench.compose_project.compose_file_manager.set_all_images(images_info) + bench.compose_project.compose_file_manager.set_version(str(self.version)) + bench.compose_project.compose_file_manager.write_to_file() + + self.migrate_workers_compose(bench) + self.migrate_admin_tools_compose(bench) + + def migrate_workers_compose(self, bench: MigrationBench): + if bench.workers_compose_project.compose_file_manager.compose_path.exists(): + richprint.print("Migrating workers compose") + workers_image_info = bench.workers_compose_project.compose_file_manager.get_all_images() + + for worker in workers_image_info.keys(): + workers_image_info[worker]['tag'] = self.version.version_string() + + bench.workers_compose_project.compose_file_manager.set_top_networks_name( + "site-network", get_container_name_prefix(bench.name) + ) + bench.workers_compose_project.compose_file_manager.set_container_names( + get_container_name_prefix(bench.name) + ) + bench.workers_compose_project.compose_file_manager.set_all_images(workers_image_info) + bench.workers_compose_project.compose_file_manager.set_version(str(self.version)) + bench.workers_compose_project.compose_file_manager.write_to_file() + richprint.print(f"Migrated [blue]{bench.name}[/blue] workers compose file.") + + def migrate_admin_tools_compose(self, bench: MigrationBench): + admin_tool_compose_file = bench.path / 'docker-compose.admin-tools.yml' + if admin_tool_compose_file.exists(): + admin_tool_compose_file_manager = ComposeFile(admin_tool_compose_file, 'docker-compose.admin-tools.tmpl') + admin_tool_compose_project = ComposeProject(admin_tool_compose_file_manager) + + richprint.print("Migrating admin-tools compose") + + admin_tools_image_info = admin_tool_compose_project.compose_file_manager.get_all_images() + admin_tools_image_info['adminer']['tag'] = '4' + + for image in [admin_tools_image_info['adminer']]: + pull_image = f"{image['name']}:{image['tag']}" + + if pull_image not in self.pulled_images_list: + richprint.change_head(f"Pulling Image {pull_image}") + output = DockerClient().pull(container_name=pull_image, stream=True) + richprint.live_lines(output, padding=(0, 0, 0, 2)) + richprint.print(f"Image pulled [blue]{pull_image}[/blue]") + self.pulled_images_list.append(pull_image) + + admin_tool_compose_project.compose_file_manager.set_top_networks_name( + "site-network", get_container_name_prefix(bench.name) + ) + admin_tool_compose_project.compose_file_manager.set_all_images(admin_tools_image_info) + admin_tool_compose_project.compose_file_manager.set_container_names(get_container_name_prefix(bench.name)) + admin_tool_compose_project.compose_file_manager.set_version(str(self.version)) + admin_tool_compose_project.compose_file_manager.write_to_file() + + richprint.print(f"Migrated [blue]{bench.name}[/blue] admin-tools compose file.") From e039e62eb05fee3667468e5a25d6539f493191f2 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 20 Jun 2024 19:37:33 +0530 Subject: [PATCH 31/44] fix: bench restart in prod environment --- Docker/frappe/bench-wrapper.sh | 4 ++-- Docker/frappe/frappe-dev.conf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Docker/frappe/bench-wrapper.sh b/Docker/frappe/bench-wrapper.sh index e8bfe4e1..c5839b85 100644 --- a/Docker/frappe/bench-wrapper.sh +++ b/Docker/frappe/bench-wrapper.sh @@ -1,9 +1,9 @@ #!/bin/bash restart_command() { - supervisorctl -c /opt/user/supervisord.conf restart frappe-bench-dev: + supervisorctl -c /opt/user/supervisord.conf restart all } status_command() { - supervisorctl -c /opt/user/supervisord.conf status frappe-bench-dev: + supervisorctl -c /opt/user/supervisord.conf status all } if [[ "$@" =~ ^restart[[:space:]]* ]]; then diff --git a/Docker/frappe/frappe-dev.conf b/Docker/frappe/frappe-dev.conf index a23008e8..125c9aad 100644 --- a/Docker/frappe/frappe-dev.conf +++ b/Docker/frappe/frappe-dev.conf @@ -13,7 +13,7 @@ stopsignal=QUIT [program:frappe-bench-frappe-watch] command=/opt/user/bench-dev-watch.sh priority=4 -autostart=false +autostart=true autorestart=false stdout_logfile=/workspace/frappe-bench/logs/watch.dev.log redirect_stderr=true From 8cefbfe6d156c232bf92dea09fdfec3c06a6e457 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 21 Jun 2024 12:06:24 +0530 Subject: [PATCH 32/44] Update status msgs --- .../migration_manager/migrations/migrate_0_15_0.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frappe_manager/migration_manager/migrations/migrate_0_15_0.py b/frappe_manager/migration_manager/migrations/migrate_0_15_0.py index b83d1f6b..0d803f54 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_15_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_15_0.py @@ -39,6 +39,8 @@ def migrate_services(self): richprint.live_lines(output, padding=(0, 0, 0, 2)) richprint.print(f"Image pulled [blue]{pull_image}[/blue]") + richprint.change_head("Migrating services compose") + # add ACME_HTTP_CHALLENGE_LOCATION: false env envs = self.services_manager.compose_project.compose_file_manager.get_all_envs() envs['global-nginx-proxy'] = {'ACME_HTTP_CHALLENGE_LOCATION': False} @@ -46,6 +48,7 @@ def migrate_services(self): self.services_manager.compose_project.compose_file_manager.set_all_envs(envs) self.services_manager.compose_project.compose_file_manager.set_version(str(self.version)) self.services_manager.compose_project.compose_file_manager.write_to_file() + richprint.print("Migrated services compose") def migrate_bench(self, bench: MigrationBench): bench.compose_project.down_service(volumes=True) @@ -91,7 +94,7 @@ def migrate_bench(self, bench: MigrationBench): def migrate_workers_compose(self, bench: MigrationBench): if bench.workers_compose_project.compose_file_manager.compose_path.exists(): - richprint.print("Migrating workers compose") + richprint.change_head("Migrating workers compose") workers_image_info = bench.workers_compose_project.compose_file_manager.get_all_images() for worker in workers_image_info.keys(): @@ -114,7 +117,7 @@ def migrate_admin_tools_compose(self, bench: MigrationBench): admin_tool_compose_file_manager = ComposeFile(admin_tool_compose_file, 'docker-compose.admin-tools.tmpl') admin_tool_compose_project = ComposeProject(admin_tool_compose_file_manager) - richprint.print("Migrating admin-tools compose") + richprint.change_head("Migrating admin-tools compose") admin_tools_image_info = admin_tool_compose_project.compose_file_manager.get_all_images() admin_tools_image_info['adminer']['tag'] = '4' From 66e58b56c219d341fa1251f5da73121b95c9d31a Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 21 Jun 2024 12:07:11 +0530 Subject: [PATCH 33/44] fix: sync workers after migrations --- frappe_manager/site_manager/site.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 84fbf571..d4fd0660 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -402,12 +402,11 @@ def start(self, force: bool = False): global_db_info = self.services.database_manager.database_server_info self.sync_bench_common_site_config(global_db_info.host, global_db_info.port) - self.sync_workers_compose() - richprint.change_head("Starting bench services") self.admin_tools.remove_nginx_location_config() self.compose_project.start_service(force_recreate=force) self.benchops.is_required_services_available() + self.sync_workers_compose() self.sync_bench_config_configuration() self.save_bench_config() richprint.print("Started bench services.") From ac07e9cd5cd6ff29a2cc628939b1d63a4fa2b18c Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 21 Jun 2024 12:23:47 +0530 Subject: [PATCH 34/44] fix: check db start after services migration --- .../migrations/migrate_0_15_0.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/frappe_manager/migration_manager/migrations/migrate_0_15_0.py b/frappe_manager/migration_manager/migrations/migrate_0_15_0.py index 0d803f54..18a4eee1 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_15_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_15_0.py @@ -12,6 +12,11 @@ from frappe_manager.migration_manager.version import Version from frappe_manager.migration_manager.backup_manager import BackupManager from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager.services_manager.database_service_manager import ( + DatabaseServerServiceInfo, + DatabaseServiceManager, + MariaDBManager, +) from frappe_manager.utils.helpers import get_container_name_prefix @@ -49,6 +54,17 @@ def migrate_services(self): self.services_manager.compose_project.compose_file_manager.set_version(str(self.version)) self.services_manager.compose_project.compose_file_manager.write_to_file() richprint.print("Migrated services compose") + richprint.change_head("Restarting services") + self.services_manager.compose_project.start_service(force_recreate=True) + + services_database_manager: DatabaseServiceManager = MariaDBManager( + DatabaseServerServiceInfo.import_from_compose_file('global-db', self.services_manager.compose_project), + self.services_manager.compose_project, + ) + # wait till db starts + services_database_manager.wait_till_db_start() + + richprint.print("Restarted services") def migrate_bench(self, bench: MigrationBench): bench.compose_project.down_service(volumes=True) From 1f895a2e49417a252314e0474504c8ccbf8ce5ab Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 21 Jun 2024 17:36:08 +0530 Subject: [PATCH 35/44] remove supervisor conf divide script --- Docker/frappe/Dockerfile | 2 -- Docker/frappe/divide-supervisor-conf.py | 41 ------------------------- 2 files changed, 43 deletions(-) delete mode 100755 Docker/frappe/divide-supervisor-conf.py diff --git a/Docker/frappe/Dockerfile b/Docker/frappe/Dockerfile index 04952673..9232f78f 100644 --- a/Docker/frappe/Dockerfile +++ b/Docker/frappe/Dockerfile @@ -163,7 +163,6 @@ COPY --chown=frappe:frappe --chmod=0755 ./bench-wrapper.sh /opt/user/.bin/bench COPY --chmod=0755 ./prebake.sh /scripts/ COPY --chmod=0755 ./helper-function.sh /scripts/ -COPY --chmod=0755 ./divide-supervisor-conf.py /scripts/ RUN ls -lah /workspace && /scripts/prebake.sh && mv /workspace/frappe-bench/apps/* /workspace/ @@ -183,7 +182,6 @@ RUN mkdir -p /scripts COPY --chmod=0755 ./entrypoint.sh / COPY --chmod=0755 ./user-script.sh /scripts/ COPY --chmod=0755 ./launch.sh /scripts/ -COPY --chmod=0755 ./divide-supervisor-conf.py /scripts/ COPY --chmod=0755 ./helper-function.sh /scripts/ RUN rm -rf /opt && mkdir -p /workspace /opt diff --git a/Docker/frappe/divide-supervisor-conf.py b/Docker/frappe/divide-supervisor-conf.py deleted file mode 100755 index 26086043..00000000 --- a/Docker/frappe/divide-supervisor-conf.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import configparser -from pathlib import Path - -print(sys.argv) -conf_file_path= Path(sys.argv[1]) -conf_file_absolute_path = conf_file_path.absolute() - -if not len(sys.argv) == 2: - print(f"Generates individual program conf from supervisor.conf.\nUSAGE: {sys.argv[0]} SUPERVISOR_CONF_PATH\n\n SUPERVISOR_CONF_PATH -> Absolute path to supervisor.conf") - sys.exit(0) - -config = configparser.ConfigParser(allow_no_value=True,strict=False,interpolation=None) - -superconf = open(conf_file_absolute_path,'r+') - -config.read_file(superconf) - -print(f"Divided {conf_file_absolute_path} into ") - -for section_name in config.sections(): - if not 'group:' in section_name: - - section_config = configparser.ConfigParser(interpolation=None) - section_config.add_section(section_name) - for key, value in config.items(section_name): - if 'frappe-bench-frappe-web' in section_name: - if key == 'command': - value = value.replace("127.0.0.1:80","0.0.0.0:80") - section_config.set(section_name, key, value) - - if 'worker' in section_name: - file_name = f"{section_name.replace('program:','')}.workers.fm.supervisor.conf" - else: - file_name = f"{section_name.replace('program:','')}.fm.supervisor.conf" - - with open(conf_file_path.parent / file_name, 'w') as section_file: - section_config.write(section_file) - print(f" - {section_name} => {file_name}") From 5a7212bf71d4824a0ef5ff6bcf68750ae866c367 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 21 Jun 2024 18:03:19 +0530 Subject: [PATCH 36/44] remove chown permissions for /workspace --- Docker/frappe/entrypoint.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Docker/frappe/entrypoint.sh b/Docker/frappe/entrypoint.sh index eff47eb6..ebefd947 100755 --- a/Docker/frappe/entrypoint.sh +++ b/Docker/frappe/entrypoint.sh @@ -46,11 +46,11 @@ if [[ ! -f "/workspace/.profile" ]]; then fi -start_time=$(date +%s.%N) -chown -R "$USERID":"$USERGROUP" /workspace -end_time=$(date +%s.%N) -execution_time=$(awk "BEGIN {print $end_time - $start_time}") -echo "Time taken for chown /workspace : $execution_time seconds" +# start_time=$(date +%s.%N) +# chown -R "$USERID":"$USERGROUP" /workspace +# end_time=$(date +%s.%N) +# execution_time=$(awk "BEGIN {print $end_time - $start_time}") +# echo "Time taken for chown /workspace : $execution_time seconds" if [ "$#" -gt 0 ]; then gosu "$USERID":"$USERGROUP" "/scripts/$@" & From e84dc72df3bc4eeae898c8b938fed3492a4f963c Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 24 Jun 2024 23:16:08 +0530 Subject: [PATCH 37/44] fix: create default.conf only when not available --- Docker/nginx/entrypoint.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Docker/nginx/entrypoint.sh b/Docker/nginx/entrypoint.sh index a73cd8f3..040d5950 100755 --- a/Docker/nginx/entrypoint.sh +++ b/Docker/nginx/entrypoint.sh @@ -9,7 +9,9 @@ cleanup() { trap cleanup SIGTERM -/config/jinja2 -D SITENAME="$SITENAME" /config/template.conf > /etc/nginx/conf.d/default.conf +if ! [[ -f "/etc/nginx/conf.d/default.conf" ]]; then + /config/jinja2 -D SITENAME="$SITENAME" /config/template.conf > /etc/nginx/conf.d/default.conf +fi nginx -g 'daemon off;' & nginx_pid=$! From b320c8cb184b60966adbcdd1232617cab7e520ce Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 24 Jun 2024 23:19:32 +0530 Subject: [PATCH 38/44] Update nginx docker image version --- Docker/images-tag.json | 2 +- frappe_manager/templates/docker-compose.tmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Docker/images-tag.json b/Docker/images-tag.json index 92e32eab..42405335 100644 --- a/Docker/images-tag.json +++ b/Docker/images-tag.json @@ -1,5 +1,5 @@ { "frappe": "v0.15.0", - "nginx": "v0.13.0", + "nginx": "v0.15.0", "mailhog": "v0.8.3" } diff --git a/frappe_manager/templates/docker-compose.tmpl b/frappe_manager/templates/docker-compose.tmpl index 012101ff..06ceb999 100644 --- a/frappe_manager/templates/docker-compose.tmpl +++ b/frappe_manager/templates/docker-compose.tmpl @@ -16,7 +16,7 @@ services: global-backend-network: nginx: - image: ghcr.io/rtcamp/frappe-manager-nginx:v0.13.0 + image: ghcr.io/rtcamp/frappe-manager-nginx:v0.15.0 container_name: REPLACE_ME_WITH_CONTAINER_NAME user: REPLACE_ME_WITH_CURRENT_USER:REPLACE_ME_WITH_CURRENT_USER_GROUP environment: From d75bde4d941cfed65767edc83c3512752cbf721e Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 24 Jun 2024 23:19:51 +0530 Subject: [PATCH 39/44] Add migration for updating nginx image --- .../migration_manager/migrations/migrate_0_15_0.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frappe_manager/migration_manager/migrations/migrate_0_15_0.py b/frappe_manager/migration_manager/migrations/migrate_0_15_0.py index 18a4eee1..742302a1 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_15_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_15_0.py @@ -82,6 +82,12 @@ def migrate_bench(self, bench: MigrationBench): redis_image_info = images_info['redis-cache'] redis_image_info['tag'] = '6.2-alpine' + nginx_image_info = images_info['nginx'] + nginx_image_info['tag'] = self.version.version_string() + + # change image nginx + images_info['nginx'] = nginx_image_info + # change image frappe, socketio, schedule images_info['frappe'] = frappe_image_info images_info['socketio'] = frappe_image_info @@ -92,7 +98,7 @@ def migrate_bench(self, bench: MigrationBench): images_info['redis-queue'] = redis_image_info images_info['redis-socketio'] = redis_image_info - for image in [frappe_image_info, redis_image_info]: + for image in [frappe_image_info, redis_image_info, nginx_image_info]: pull_image = f"{image['name']}:{image['tag']}" if pull_image not in self.pulled_images_list: richprint.change_head(f"Pulling Image {pull_image}") From 12799b330ca83d793fef406ebeea3376465279ab Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 24 Jun 2024 23:50:51 +0530 Subject: [PATCH 40/44] fix: don't run entrypoint script when copying from docker image --- frappe_manager/utils/docker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe_manager/utils/docker.py b/frappe_manager/utils/docker.py index 8b9dde99..dea1a444 100644 --- a/frappe_manager/utils/docker.py +++ b/frappe_manager/utils/docker.py @@ -257,6 +257,7 @@ def host_run_cp(image: str, source: str, destination: str, docker): name=source_container_name, detach=True, stream=False, + entrypoint='bash', command="tail -f /dev/null", ) except DockerException as e: From f06103406837a7b838a63e9d2e8cb24f9fe2b9ed Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 24 Jun 2024 23:52:25 +0530 Subject: [PATCH 41/44] remove nginx provided default.conf --- Docker/nginx/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/nginx/Dockerfile b/Docker/nginx/Dockerfile index a2eb549f..f590c279 100644 --- a/Docker/nginx/Dockerfile +++ b/Docker/nginx/Dockerfile @@ -15,6 +15,6 @@ COPY --from=builder /dist/ /config/ COPY template.conf /config/ COPY ./entrypoint.sh / RUN chmod +x /entrypoint.sh -RUN mkdir -p /etc/nginx/custom +RUN mkdir -p /etc/nginx/custom && rm -rf /etc/nginx/conf.d/default.conf ENTRYPOINT ["/bin/bash","/entrypoint.sh"] From c16e7b677b4946d7d2d3433253591d492335e10b Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 25 Jun 2024 16:41:20 +0530 Subject: [PATCH 42/44] remove nginx conf during migrations --- .../migration_manager/migrations/migrate_0_15_0.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frappe_manager/migration_manager/migrations/migrate_0_15_0.py b/frappe_manager/migration_manager/migrations/migrate_0_15_0.py index 742302a1..02fcb05d 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_15_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_15_0.py @@ -98,6 +98,12 @@ def migrate_bench(self, bench: MigrationBench): images_info['redis-queue'] = redis_image_info images_info['redis-socketio'] = redis_image_info + # remove default.conf file from nginx + nginx_default_conf_path = bench.path / 'configs' / 'nginx' / 'conf' / 'conf.d' / 'default.conf' + + if nginx_default_conf_path.exists(): + nginx_default_conf_path.unlink() + for image in [frappe_image_info, redis_image_info, nginx_image_info]: pull_image = f"{image['name']}:{image['tag']}" if pull_image not in self.pulled_images_list: From 87732ebc90d5eba80f2efdabcfcf48774ae980e8 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 25 Jun 2024 17:50:10 +0530 Subject: [PATCH 43/44] Update shortflag of --environment --- frappe_manager/commands.py | 4 ++-- frappe_manager/utils/examples.json | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index d66d45da..1dbbb2c7 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -157,7 +157,7 @@ def create( ), ] = [], environment: Annotated[ - FMBenchEnvType, typer.Option("--environment", "--env", help="Select bench environment type.") + FMBenchEnvType, typer.Option("--environment", "-e", help="Select bench environment type.") ] = FMBenchEnvType.dev, letsencrypt_preferred_challenge: Annotated[ Optional[LETSENCRYPT_PREFERRED_CHALLENGE], @@ -500,7 +500,7 @@ def update( ] = None, environment: Annotated[ Optional[FMBenchEnvType], - typer.Option("--environment", "--env", help="Switch bench environment.", show_default=False), + typer.Option("--environment", "-e", help="Switch bench environment.", show_default=False), ] = None, developer_mode: Annotated[ Optional[EnableDisableOptionsEnum], diff --git a/frappe_manager/utils/examples.json b/frappe_manager/utils/examples.json index 1009b197..045625db 100644 --- a/frappe_manager/utils/examples.json +++ b/frappe_manager/utils/examples.json @@ -14,11 +14,12 @@ "code": " --apps erpnext --apps hrms" }, { - "desc": "Install Frappe[{default_version}], ERPNext[{default_version}], and HRMS[develop].", "code": " --apps erpnext --apps hrms:develop" + "desc": "Install Frappe[{default_version}], ERPNext[{default_version}], and HRMS[develop].", + "code": " --apps erpnext --apps hrms:develop" }, { "desc": "Use Frappe production mode based bench.", - "code": " --env prod" + "code": " -e prod" }, { "desc": "Enable Admin Tools.", @@ -154,7 +155,7 @@ }, { "desc": "Switch to frappe production environment.", - "code": " --env prod" + "code": " -e prod" }, { "desc": "Switch to frappe development environment.", From 6607abd961ee43d7773bbebe9a5a4857268463ed Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 25 Jun 2024 18:07:36 +0530 Subject: [PATCH 44/44] fix: backup of workers in bench --- frappe_manager/site_manager/site.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index d4fd0660..e13eed38 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -573,10 +573,7 @@ def backup_restore_workers_supervisor(self, backup_manager: BackupManager): backup_manager.restore(backup, force=True) def backup_workers_supervisor_conf(self): - # TODO this can be given to woker class ? backup_workers_manager = BackupManager(name='workers', backup_group_name='workers') - # TODO use backup manager - # take backup backup_workers_manager.backup(self.workers.supervisor_config_path, bench_name=self.name) if self.workers.supervisor_config_path.exists(): @@ -586,7 +583,7 @@ def backup_workers_supervisor_conf(self): continue if file_path_abs.endswith(".fm.supervisor.conf"): from_path = file_path - backup_workers_manager.backup(from_path) + backup_workers_manager.backup(from_path, bench_name=self.name) file_path.unlink() return backup_workers_manager