diff --git a/package.json b/package.json index d75c91baa..92c4f1330 100644 --- a/package.json +++ b/package.json @@ -1,17 +1,6 @@ { "scripts": { - "vm:provision": "multipass launch --name umbrel-dev --cpus 4 --memory 8G --disk 50G 22.04 && npm run vm:stop && multipass mount --type native $PWD umbrel-dev:/opt/umbrel-mount && multipass exec umbrel-dev -- /opt/umbrel-mount/scripts/vm provision", - "vm:shell": "multipass shell umbrel-dev", - "vm:exec": "multipass exec --working-directory /home/ubuntu umbrel-dev --", - "vm:logs": "multipass exec umbrel-dev -- journalctl --unit umbreld-production --unit umbreld --unit ui --follow --lines 100 --output cat", - "vm:enable-development": "multipass exec umbrel-dev -- /opt/umbrel-mount/scripts/vm enable-development", - "vm:enable-production": "multipass exec umbrel-dev -- /opt/umbrel-mount/scripts/vm enable-production", - "vm:install-deps": "multipass exec umbrel-dev -- /opt/umbrel-mount/scripts/vm install-deps", - "vm:trpc": "npm run --silent vm:exec -- UMBREL_DATA_DIR=./data UMBREL_TRPC_ENDPOINT=http://localhost/trpc npm --prefix /home/ubuntu/umbrel/packages/umbreld run start -- client", - "vm:start": "multipass start umbrel-dev", - "vm:stop": "multipass stop umbrel-dev", - "vm:restart": "multipass restart umbrel-dev", - "vm:destroy": "multipass delete umbrel-dev && multipass purge", - "vm:remount": "multipass mount . umbrel-dev:/opt/umbrel-mount" + "dev": "./scripts/umbrel-dev", + "dev:help": "npm run dev help" } } diff --git a/packages/umbreld/package.json b/packages/umbreld/package.json index be32d0bbb..f3c55291e 100644 --- a/packages/umbreld/package.json +++ b/packages/umbreld/package.json @@ -14,7 +14,6 @@ "private": true, "scripts": { "start": "./source/cli.ts", - "start:vm": "sudo FORCE_COLOR=1 npm run start -- --data-directory ./data --port 80 --log-level verbose", "client": "UMBREL_DATA_DIR=./data UMBREL_TRPC_ENDPOINT=http://localhost:3001/trpc npm run start -- client", "format": "prettier --write .", "format:check": "prettier --check .", @@ -26,9 +25,8 @@ "test:integration": "npm run test -- integration", "test:coverage": "open ./coverage/index.html", "test-everything": "npm run format && npm run test -- --run && npm run lint", - "watch": "NODE_ENV=development nodemon --ext js,json,ts --watch source --exec npm run", - "dev": "UMBREL_UI_PROXY=http://localhost:3000 npm run watch -- start -- -- --data-directory ./data --port 3001 --log-level verbose", - "dev:vm": "sudo FORCE_COLOR=1 npm run dev -- --port 80", + "watch": "NODE_ENV=development nodemon --legacy-watch --ext js,json,ts --watch source --exec npm run", + "dev": "FORCE_COLOR=1 UMBREL_UI_PROXY=http://localhost:3000 npm run watch -- start -- -- --data-directory /home/umbrel/umbrel --log-level verbose", "build": "tsx scripts/build.ts", "prepare-release": "tsx scripts/prepare-release.ts", "timestamp-release": "ots-cli.js stamp release/SHA256SUMS", diff --git a/packages/umbreld/source/modules/cli-client.ts b/packages/umbreld/source/modules/cli-client.ts index d02e52b53..5ced5f183 100644 --- a/packages/umbreld/source/modules/cli-client.ts +++ b/packages/umbreld/source/modules/cli-client.ts @@ -1,5 +1,4 @@ import process from 'node:process' -import os from 'node:os' import {createTRPCProxyClient, httpLink} from '@trpc/client' import fse from 'fs-extra' @@ -9,7 +8,7 @@ import * as jwt from './jwt.js' import type {AppRouter} from './server/trpc/index.js' // TODO: Maybe just read the endpoint from the data dir -const dataDir = process.env.UMBREL_DATA_DIR ?? `${os.homedir()}/umbrel` +const dataDir = process.env.UMBREL_DATA_DIR ?? '/home/umbrel/umbrel' const trpcEndpoint = process.env.UMBREL_TRPC_ENDPOINT ?? `http://localhost/trpc` async function signJwt() { diff --git a/packages/umbreld/umbreld b/packages/umbreld/umbreld index aaa025b0d..44c1551b1 100755 --- a/packages/umbreld/umbreld +++ b/packages/umbreld/umbreld @@ -3,6 +3,15 @@ # We need to add this shim as the main umbreld entrypoint so we can sets up the environmnet we need # like adding node_modules/.bin to the PATH so we have access to tsx. + +# Hook to run development mode +if [[ -d "/umbrel-dev" ]] +then + echo "Running in development mode" + cd /umbrel-dev + exec npm run dev container-init +fi + # Find the project directory and follow symlinks if necessary project_directory="$(dirname $(readlink -f "${BASH_SOURCE[0]}"))" diff --git a/scripts/umbrel-dev b/scripts/umbrel-dev new file mode 100755 index 000000000..baf5d1eb0 --- /dev/null +++ b/scripts/umbrel-dev @@ -0,0 +1,296 @@ +#!/usr/bin/env bash +set -euo pipefail + +# The instance id is used to namespace the dev environment to allow for multiple instances to run +# without conflicts. e.g: +# npm run dev start +# UMBREL_DEV_INSTANCE='apps' npm run dev start +# +# Will spin up two separate umbrel-dev instances accessible at: +# http://umbrel-dev.local +# http://umbrel-dev-apps.local +INSTANCE_ID_PREFIX="umbrel-dev" +INSTANCE_ID="${INSTANCE_ID_PREFIX}${UMBREL_DEV_INSTANCE:+-$UMBREL_DEV_INSTANCE}" + +show_help() { + cat << EOF +umbrel-dev + +Automatically initialize and manage an umbrelOS development environment. + +Usage: npm run dev [-- ] + +Commands: + help Show this help message + start Either start an existing dev environment or create and start a new one + logs Stream umbreld logs + shell Get a shell inside the running dev environment + exec -- Execute a command inside the running dev environment + client -- [] Query the umbreld RPC server via a CLI client + rebuild Rebuild the operating system image from source and reboot the dev environment into it + restart Restart the dev environment + stop Stop the dev environment + reset Reset the dev environment to a fresh state + destroy Destroy the dev environment + +Environment Variables: + UMBREL_DEV_INSTANCE The instance id of the dev environment. Allows running multiple instances of + umbrel-dev in different namespaces. + +Note: umbrel-dev requires a Docker environment that exposes container IPs to the host. This is how Docker +natively works on Linux and can be done with OrbStack on macOS. On Windows this should work with WSL 2. + +EOF +} + +build_os_image() { + docker buildx build --load --file packages/os/umbrelos.Dockerfile --tag "${INSTANCE_ID}" . +} + +create_instance() { + # --privileged is needed for systemd to work inside the container. + # + # We mount a named volume namespaced to the instance id at /data to immitate + # the data partition of a physical install. + # + # We mount the monorepo inside the container at /umbrel-dev as readonly. We + # setup a writeable fs overlay later to allow the container to install dependencies + # without modifying the hosts source code dir. + # + # --label "dev.orbstack.http-port=80" stops OrbStack from trying to guess which port + # we're trying to expose which causes some weirdness since it often gets it wrong. + # + # --label "dev.orbstack.domains=${INSTANCE_ID}.local" makes the instance accessble at + # umbrel-dev.local on OrbStack installs. + # + # /sbin/init kicks of systemd as the container entrypoint. + docker run \ + --detach \ + --interactive \ + --tty \ + --privileged \ + --name "${INSTANCE_ID}" \ + --hostname "${INSTANCE_ID}" \ + --volume "${INSTANCE_ID}:/data" \ + --volume "${PWD}:/umbrel-dev:ro" \ + --label "dev.orbstack.http-port=80" \ + --label "dev.orbstack.domains=${INSTANCE_ID}.local" \ + "${INSTANCE_ID}" \ + /sbin/init +} + +start_instance() { + docker start "${INSTANCE_ID}" +} + +exec_in_instance() { + docker exec --interactive --tty "${INSTANCE_ID}" "${@}" +} + +stop_instance() { + # We first need to execute poweroff inside the instance so systemd gracefully stops services before we kill the container + exec_in_instance poweroff + docker stop "${INSTANCE_ID}" +} + +remove_instance() { + docker rm --force "${INSTANCE_ID}" +} + +remove_volume() { + docker volume rm "${INSTANCE_ID}" +} + +get_instance_ip() { + docker inspect --format '{{ .NetworkSettings.IPAddress }}' "${INSTANCE_ID}" +} + +# Get the command +if [ -z ${1+x} ]; then + command="" +else + command="$1" +fi + +if [[ "${command}" = "start" ]] || [[ "${command}" = "" ]] +then + echo "Starting umbrel-dev instance..." + if ! start_instance > /dev/null + then + echo "Instance not found, creating a new one..." + if ! docker image inspect "${INSTANCE_ID}" > /dev/null + then + build_os_image + fi + create_instance + fi + echo + echo "umbrel-dev instance is booting up..." + + # Stream systemd logs until boot has completed + docker logs --tail 100 --follow "${INSTANCE_ID}" 2> /dev/null & + logs_pid=$! + exec_in_instance systemctl is-active --wait multi-user.target > /dev/null|| true + sleep 2 + kill "${logs_pid}" || true + wait + + # Stream umbreld logs until web server is up + docker exec "${INSTANCE_ID}" journalctl --unit umbrel --follow --lines 100 --output cat 2> /dev/null & + logs_pid=$! + docker exec "${INSTANCE_ID}" curl --silent --retry 300 --retry-delay 1 --retry-connrefused http://localhost > /dev/null 2>&1 || true + sleep 0.1 + kill "${logs_pid}" || true + wait + + # Done! + cat << 'EOF' + + + ,;###GGGGGGGGGGl#Sp + ,##GGGlW""^' '`""%GGGG#S, + ,#GGG" "lGG#o + #GGl^ '$GG# + ,#GGb \GGG, + lGG" "GGG + #GGGlGGGl##p,,p##lGGl##p,,p###ll##GGGG + !GGGlW"""*GGGGGGG#""""WlGGGGG#W""*WGGGGS + "" "^ '" "" + +EOF + echo " Your umbrel-dev instance is ready at:" + echo + echo " http://${INSTANCE_ID}.local" + echo " http://$(get_instance_ip)" + + exit +fi + +if [[ "${command}" = "help" ]] +then + show_help + + exit +fi + +if [[ "${command}" = "shell" ]] +then + exec_in_instance bash + + exit +fi + +if [[ "${command}" = "exec" ]] +then + shift + exec_in_instance "${@}" + + exit +fi + +if [[ "${command}" = "logs" ]] +then + exec_in_instance journalctl --unit umbrel --follow --lines 100 --output cat + + exit +fi + +if [[ "${command}" = "client" ]] +then + shift + exec_in_instance npm --prefix /umbrel-dev/packages/umbreld run start -- client ${@} + + exit +fi + +if [[ "${command}" = "rebuild" ]] +then + echo "Rebuilding the operating system image from source..." + build_os_image + echo "Restarting the dev environment with the new image..." + stop_instance || true + remove_instance || true + create_instance + + exit +fi + +if [[ "${command}" = "destroy" ]] +then + echo "Destroying the dev environment..." + remove_instance || true + remove_volume || true + + exit +fi + +if [[ "${command}" = "reset" ]] +then + echo "Resetting the dev environment state..." + stop_instance || true + remove_instance || true + remove_volume || true + create_instance + + exit +fi + +if [[ "${command}" = "restart" ]] +then + echo "Restarting the dev environment..." + stop_instance + start_instance + + exit +fi + +if [[ "${command}" = "stop" ]] +then + echo "Stopping the dev environment..." + stop_instance + + exit +fi + +# This is a special command that runs directly inside the container to setup the environment +# It is not intended to be run on the host machine! +if [[ "${command}" = "container-init" ]] +then + # Check if this is the first boot + first_boot=false + if [[ ! -d "/data/umbrel-dev-overlay" ]] + then + first_boot=true + fi + + # Setup fs overlay so we can write to the source code dir without modifying it on the host + echo "Setting up fs overlay..." + mkdir -p /data/umbrel-dev-overlay/upperdir + mkdir -p /data/umbrel-dev-overlay/workdir + mount -t overlay overlay -o lowerdir=/umbrel-dev,upperdir=/data/umbrel-dev-overlay/upperdir,workdir=/data/umbrel-dev-overlay/workdir /umbrel-dev || true + + # If this is the first boot we should nuke node_modules if they exist so we get fresh Linux deps instead + # of trying to reuse deps installed from the host. (causes issues with macos native deps) + if [[ "${first_boot}" = true ]] + then + echo "Nuking node_modules inherited from host..." + rm -rf /umbrel-dev/packages/ui/node_modules || true + rm -rf /umbrel-dev/packages/umbreld/node_modules || true + fi + + # Install dependencies + echo "Installing dependencies..." + npm --prefix /umbrel-dev/packages/umbreld install + npm --prefix /umbrel-dev/packages/ui install + + # Run umbreld and ui + echo "Starting umbreld and ui..." + npm --prefix /umbrel-dev/packages/umbreld run dev & + CHOKIDAR_USEPOLLING=true npm --prefix /umbrel-dev/packages/ui run dev & + wait + + exit +fi + +show_help +exit \ No newline at end of file diff --git a/scripts/vm b/scripts/vm deleted file mode 100755 index af4920501..000000000 --- a/scripts/vm +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -MOUNT_DIR="/opt/umbrel-mount" -VM_DIR="/home/ubuntu/umbrel" - -function sync_umbrel_source() { - sudo mkdir -p "${VM_DIR}" - sudo chown ubuntu:ubuntu "${VM_DIR}" - rsync -avh \ - --exclude packages/os/ \ - --exclude packages/umbreld/data/ \ - --exclude packages/ui/public/generated-tabler-icons/ \ - --exclude packages/ui/dist/ \ - --exclude packages/ui/dist-app-auth/ \ - --exclude node_modules/ \ - --exclude .git/ \ - --exclude .pnpm-store/ \ - "${MOUNT_DIR}/." "${VM_DIR}" -} - -function install_deps() { - echo installing os deps - curl -fsSL https://deb.nodesource.com/setup_18.x | sudo DEBIAN_FRONTEND=noninteractive bash - - sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes nodejs build-essential - - echo installing umbreld deps - pushd "${VM_DIR}/packages/umbreld" - npm install - sudo DEBIAN_FRONTEND=noninteractive npm run start provision-os - popd - - echo installing ui deps - pushd "${VM_DIR}/packages/ui" - sudo npm install -g pnpm@8 - pnpm install - popd -} - -function install_services() { - - services=( - "sync" - "ui" - "umbreld" - "umbreld-production" - ) - - for service in "${services[@]}" - do - echo " -[Unit] -Description=${service} - -[Service] -ExecStart="${MOUNT_DIR}/scripts/vm" ${service} -Restart=always -StartLimitInterval=0 - -[Install] -WantedBy=multi-user.target" | sudo tee "/etc/systemd/system/${service}.service" - - sudo systemctl daemon-reload - if [[ "${service}" != "umbreld-production" ]] - then - sudo systemctl enable "${service}" - sudo systemctl start "${service}" - fi - done -} - -command="${1}" - -if [[ "${command}" = "provision" ]] -then - sync_umbrel_source - install_deps - install_services - echo - echo " ☂️ VM setup complete" - echo - echo " http://umbrel-dev.local" - - exit -fi - -if [[ "${command}" = "sync" ]] -then - while true - do - sync_umbrel_source - sleep 0.1 - done - exit -fi - -if [[ "${command}" = "ui" ]] -then - cd "${VM_DIR}/packages/ui" - exec npm run dev -fi - -if [[ "${command}" = "umbreld" ]] -then - cd "${VM_DIR}/packages/umbreld" - exec npm run dev:vm -fi - -if [[ "${command}" = "umbreld-production" ]] -then - cd "${VM_DIR}/packages/ui" - pnpm run build - rm -rf "${VM_DIR}/packages/umbreld/ui" || true - mv "${VM_DIR}/packages/ui/dist" "${VM_DIR}/packages/umbreld/ui" - cd "${VM_DIR}/packages/umbreld" - exec npm run start:vm -fi - -if [[ "${command}" = "enable-production" ]] -then - echo "Enabling production services" - sudo systemctl stop umbreld ui - sudo systemctl disable umbreld ui - sudo systemctl enable umbreld-production - sudo systemctl restart umbreld-production -fi - -if [[ "${command}" = "enable-development" ]] -then - echo "Enabling development services" - sudo systemctl stop umbreld-production - sudo systemctl disable umbreld-production - sudo systemctl enable umbreld ui - sudo systemctl start umbreld ui -fi - -if [[ "${command}" = "install-deps" ]] -then - echo installing umbreld deps - cd "${VM_DIR}/packages/umbreld" - npm install - - echo installing ui deps - cd "${VM_DIR}/packages/ui" - pnpm install - exit -fi \ No newline at end of file