diff --git a/proxy/nginx.sh b/proxy/nginx.sh index c89863c06..8aa26a555 100644 --- a/proxy/nginx.sh +++ b/proxy/nginx.sh @@ -1,12 +1,19 @@ #!/bin/sh set -eu -SUBST_VARS=$(grep -o '\${[A-Z_0-9]*}' /opt/nginx/nginx.conf.template | sort -u | tr '\n' ' ') -NGINX_CONFIG=/opt/nginx/nginx.conf +if [ -z "${OMEGA_AUTH_SECRET:-}" ] && [ -n "${OMEGACLAW_AUTH_SECRET:-}" ]; then + echo "OMEGACLAW_AUTH_SECRET is deprecated, rename it to OMEGA_AUTH_SECRET" >&2 + OMEGA_AUTH_SECRET=${OMEGACLAW_AUTH_SECRET} + export OMEGA_AUTH_SECRET +fi + +NGINX_TEMPLATE=${NGINX_TEMPLATE:-/opt/nginx/nginx.conf.template} +NGINX_CONFIG=${NGINX_CONFIG:-/opt/nginx/nginx.conf} +SUBST_VARS=$(grep -o '\${[A-Z_0-9]*}' "${NGINX_TEMPLATE}" | sort -u | tr '\n' ' ') touch "${NGINX_CONFIG}" chmod 0600 "${NGINX_CONFIG}" envsubst "$SUBST_VARS" \ - < /opt/nginx/nginx.conf.template \ + < "${NGINX_TEMPLATE}" \ > "${NGINX_CONFIG}" nginx -c "${NGINX_CONFIG}" diff --git a/scripts/omega b/scripts/omega index f2d8845f0..15954614a 100755 --- a/scripts/omega +++ b/scripts/omega @@ -850,6 +850,82 @@ options() { fi } +migrate_omegaclaw_memory() { + local old_container="omegaclaw" + local old_volume="omegaclaw-memory" + local new_volume="omega-memory" + local memory_path="/PeTTa/repos/Omega/memory" + local marker=".migrated-to-omega" + local started_marker=".migration-started" + local stopped_old_container=0 + local old_state + + if [ -n "${memory_import_file}" ] \ + || ! docker volume inspect "${old_volume}" >/dev/null 2>&1; then + return 0 + fi + if ! old_state=$(docker run --rm --entrypoint sh --volume "${old_volume}:/from:ro" "${image}" \ + -c "if [ -e /from/${marker} ]; then echo migrated; elif [ -e /from/${started_marker} ]; then echo interrupted; fi"); then + echo "Could not read the migration markers in ${old_volume}" >&2 + return 1 + fi + case "${old_state}" in + migrated) + return 0 + ;; + interrupted) + echo "The last copy from ${old_volume} did not finish, copying again" + if docker volume inspect "${new_volume}" >/dev/null 2>&1 \ + && ! docker volume rm "${new_volume}" >/dev/null; then + echo "Could not remove the partly copied volume ${new_volume}" >&2 + return 1 + fi + ;; + *) + if docker volume inspect "${new_volume}" >/dev/null 2>&1; then + return 0 + fi + ;; + esac + + echo "Copying memory from volume ${old_volume} to ${new_volume}" + if [ "$(docker inspect -f '{{.State.Running}}' "${old_container}" 2>/dev/null)" = "true" ]; then + if ! docker stop "${old_container}" >/dev/null; then + echo "Could not stop container ${old_container}, memory was not copied" >&2 + return 1 + fi + stopped_old_container=1 + echo "Stopped container ${old_container}" + fi + + if docker run --rm --entrypoint sh --volume "${old_volume}:/from" "${image}" \ + -c "date -u > /from/${started_marker}" \ + && docker volume create "${new_volume}" >/dev/null \ + && docker run --rm --entrypoint sh --volume "${new_volume}:${memory_path}" "${image}" \ + -c "test -f ${memory_path}/prompt.txt" \ + && docker run --rm --entrypoint sh \ + --volume "${old_volume}:/from:ro" --volume "${new_volume}:/to" "${image}" \ + -c 'list=$(mktemp) && cd /from && find . -mindepth 1 -maxdepth 1 ! -name "prompt*.txt" ! -name tg_prompt.txt ! -name .migration-started > "$list" && while IFS= read -r entry; do rm -rf "/to/$entry" && cp -a "$entry" /to/ || exit 1; done < "$list"' \ + && docker run --rm --entrypoint sh --volume "${old_volume}:/from" "${image}" \ + -c "date -u > /from/${marker}"; then + echo "Memory copied. Carrying memory over is experimental, so keep the volume ${old_volume} until you have checked that the agent remembers what it knew. Then remove the old installation with 'docker rm ${old_container}' and 'docker volume rm ${old_volume}'" + return 0 + fi + + echo "Could not copy memory from ${old_volume}, the old installation is left as it was" >&2 + if docker volume rm "${new_volume}" >/dev/null 2>&1 \ + || ! docker volume inspect "${new_volume}" >/dev/null 2>&1; then + docker run --rm --entrypoint sh --volume "${old_volume}:/from" "${image}" \ + -c "rm -f /from/${started_marker}" >/dev/null 2>&1 || true + else + echo "Could not remove ${new_volume}, the next start will copy the memory again" >&2 + fi + if [ "${stopped_old_container}" = "1" ]; then + docker start "${old_container}" >/dev/null 2>&1 || true + fi + return 1 +} + prepare_memory_transfer_dir() { python3 "$tmp_memory_transfer_validator" "${memory_transfer_dir}" "${memory_transfer_gid}" } @@ -911,6 +987,8 @@ start() { fi fi + migrate_omegaclaw_memory || return 1 + docker rm -f omega 2>/dev/null || true docker_cmd=( diff --git a/tests/test_omega_launcher_migration.py b/tests/test_omega_launcher_migration.py new file mode 100644 index 000000000..c67cd6418 --- /dev/null +++ b/tests/test_omega_launcher_migration.py @@ -0,0 +1,430 @@ +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] +LAUNCHER = REPO_ROOT / "scripts" / "omega" +IMAGE = "test/omega:new" + +FAKE_DOCKER = r'''#!/usr/bin/env python3 +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +MEMORY_PATH = "/PeTTa/repos/Omega/memory" +root = Path(os.environ["FAKE_DOCKER_ROOT"]) +volumes = root / "volumes" +containers = root / "containers" +args = sys.argv[1:] + +with open(root / "calls.log", "a", encoding="utf-8") as log: + log.write(" ".join(args) + "\n") + + +def volume(name): + return volumes / name + + +if args[:2] == ["volume", "inspect"]: + sys.exit(0 if volume(args[2]).is_dir() else 1) +if args[:2] == ["volume", "create"]: + volume(args[2]).mkdir(parents=True, exist_ok=True) + print(args[2]) + sys.exit(0) +if args[:2] == ["volume", "rm"]: + if os.environ.get("FAKE_DOCKER_FAIL_VOLUME_RM") or not volume(args[2]).is_dir(): + sys.exit(1) + shutil.rmtree(volume(args[2])) + sys.exit(0) +if args[:1] == ["inspect"]: + state = containers / args[-1] + if not state.exists(): + sys.exit(1) + print("true" if state.read_text().strip() == "running" else "false") + sys.exit(0) +if args[:1] in (["stop"], ["start"]): + state = containers / args[-1] + if not state.exists(): + sys.exit(1) + state.write_text("running" if args[0] == "start" else "exited") + sys.exit(0) +if args[:1] == ["run"]: + mounts = {} + entrypoint = None + index = 1 + while index < len(args): + option = args[index] + if option in ("--volume", "-v"): + source, target = args[index + 1].split(":")[:2] + mounts[target] = source + index += 2 + elif option in ("--entrypoint", "--user", "--group-add", "-e", "--name", "--security-opt", "--tmpfs"): + if option == "--entrypoint": + entrypoint = args[index + 1] + index += 2 + elif option.startswith("-"): + index += 1 + else: + break + command = args[index + 1:] + sources = set(mounts.values()) + if os.environ.get("FAKE_DOCKER_FAIL_MARKER_CHECK") and "echo migrated" in " ".join(command): + sys.exit(1) + if os.environ.get("FAKE_DOCKER_FAIL_COPY") and {"omegaclaw-memory", "omega-memory"} <= sources: + sys.exit(1) + for target, source in mounts.items(): + if "/" in source: + continue + volume(source).mkdir(parents=True, exist_ok=True) + if target == MEMORY_PATH and not any(volume(source).iterdir()): + shutil.copytree(root / "image-memory", volume(source), dirs_exist_ok=True) + if entrypoint == "sh" and command[:1] == ["-c"]: + script = command[1] + for target in sorted(mounts, key=len, reverse=True): + script = re.sub(re.escape(target) + r"(?=[/\"'\s;]|$)", str(volume(mounts[target])), script) + environment = os.environ.copy() + environment["TMPDIR"] = str(root) + if os.environ.get("FAKE_DOCKER_CP_FAILS") and {"omegaclaw-memory", "omega-memory"} <= sources: + failing_bin = root / "failing-bin" + failing_bin.mkdir(exist_ok=True) + failing_cp = failing_bin / "cp" + failing_cp.write_text( + "#!/bin/sh\n" + f"if [ ! -e '{root / 'cp-failed'}' ]; then\n" + f" : > '{root / 'cp-failed'}'\n" + " echo 'cp: No space left on device' >&2\n" + " exit 1\n" + "fi\n" + f"exec '{shutil.which('cp')}' \"$@\"\n" + ) + failing_cp.chmod(0o755) + environment["PATH"] = f"{failing_bin}{os.pathsep}{environment['PATH']}" + sys.exit(subprocess.run(["sh", "-c", script], env=environment).returncode) +sys.exit(0) +''' + + +@pytest.fixture +def docker_root(tmp_path): + root = tmp_path / "docker" + (root / "volumes").mkdir(parents=True) + (root / "containers").mkdir() + image_memory = root / "image-memory" + (image_memory / "chroma_db").mkdir(parents=True) + (image_memory / "history.metta").write_text("") + (image_memory / "prompt.txt").write_text("omega prompt\n") + (image_memory / "prompt_ASICloud.txt").write_text("omega asicloud prompt\n") + (image_memory / "tg_prompt.txt").write_text("omega telegram prompt\n") + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + docker = bin_dir / "docker" + docker.write_text(FAKE_DOCKER, encoding="utf-8") + docker.chmod(0o755) + return root + + +def _install_omegaclaw(root, running=True): + old = root / "volumes" / "omegaclaw-memory" + (old / "chroma_db").mkdir(parents=True) + (old / "chroma_db" / "chroma.sqlite3").write_text("old long-term memory") + (old / ".channel").mkdir() + (old / ".channel" / "authenticated-user.json").write_text('{"user": "owner"}') + (old / "history.metta").write_text("(old history)\n") + (old / "prompt.txt").write_text("omegaclaw prompt\n") + (old / "prompt_ASICloud.txt").write_text("omegaclaw asicloud prompt\n") + (old / "saved note.txt").write_text("kept by the agent") + (root / "containers" / "omegaclaw").write_text("running" if running else "exited") + return old + + +def _snapshot(directory): + return { + str(path.relative_to(directory)): path.read_text() if path.is_file() else None + for path in sorted(directory.rglob("*")) + } + + +def _launcher(root, *arguments, fail_copy=False, fail_cp=False, fail_volume_rm=False, fail_marker_check=False): + environment = os.environ.copy() + environment["PATH"] = f"{root.parent / 'bin'}{os.pathsep}{environment['PATH']}" + environment["FAKE_DOCKER_ROOT"] = str(root) + environment["ASI_API_KEY"] = "test-token" + if fail_copy: + environment["FAKE_DOCKER_FAIL_COPY"] = "1" + if fail_cp: + environment["FAKE_DOCKER_CP_FAILS"] = "1" + if fail_volume_rm: + environment["FAKE_DOCKER_FAIL_VOLUME_RM"] = "1" + if fail_marker_check: + environment["FAKE_DOCKER_FAIL_MARKER_CHECK"] = "1" + return subprocess.run( + [str(LAUNCHER), *arguments], + cwd=REPO_ROOT, + env=environment, + capture_output=True, + text=True, + check=False, + ) + + +def _calls(root): + log = root / "calls.log" + return log.read_text(encoding="utf-8").splitlines() if log.exists() else [] + + +def _started_agent(root): + return any(call.startswith("run -d") for call in _calls(root)) + + +def _read(path): + return path.read_text() if path.is_file() else None + + +def _container_state(root, name): + return (root / "containers" / name).read_text() + + +def test_first_start_copies_old_memory_except_prompts(docker_root): + _install_omegaclaw(docker_root) + + result = _launcher(docker_root, "start", "-d", IMAGE) + + assert result.returncode == 0, result.stderr + new = docker_root / "volumes" / "omega-memory" + assert _read(new / "history.metta") == "(old history)\n" + assert _read(new / "chroma_db" / "chroma.sqlite3") == "old long-term memory" + assert _read(new / ".channel" / "authenticated-user.json") == '{"user": "owner"}' + assert _read(new / "saved note.txt") == "kept by the agent" + assert _read(new / "prompt.txt") == "omega prompt\n" + assert _read(new / "prompt_ASICloud.txt") == "omega asicloud prompt\n" + assert _read(new / "tg_prompt.txt") == "omega telegram prompt\n" + assert _started_agent(docker_root) + + +def test_first_start_keeps_old_volume_contents(docker_root): + old = _install_omegaclaw(docker_root) + before = _snapshot(old) + + result = _launcher(docker_root, "start", "-d", IMAGE) + + assert result.returncode == 0, result.stderr + after = _snapshot(old) + assert {path: after[path] for path in before} == before + + +def test_first_start_stops_running_old_container_before_copy(docker_root): + _install_omegaclaw(docker_root, running=True) + + result = _launcher(docker_root, "start", "-d", IMAGE) + + assert result.returncode == 0, result.stderr + assert _container_state(docker_root, "omegaclaw") == "exited" + calls = _calls(docker_root) + assert calls.index("stop omegaclaw") < calls.index("volume create omega-memory") + + +def test_stopped_old_container_stays_stopped(docker_root): + _install_omegaclaw(docker_root, running=False) + + result = _launcher(docker_root, "start", "-d", IMAGE) + + assert result.returncode == 0, result.stderr + assert _container_state(docker_root, "omegaclaw") == "exited" + assert _read(docker_root / "volumes" / "omega-memory" / "history.metta") == "(old history)\n" + + +def test_start_after_clean_does_not_bring_old_memory_back(docker_root): + _install_omegaclaw(docker_root) + assert _launcher(docker_root, "start", "-d", IMAGE).returncode == 0 + assert _launcher(docker_root, "clean").returncode == 0 + + result = _launcher(docker_root, "start", "-d", IMAGE) + + assert result.returncode == 0, result.stderr + assert _read(docker_root / "volumes" / "omega-memory" / "history.metta") == "" + + +def test_existing_new_volume_is_left_untouched(docker_root): + _install_omegaclaw(docker_root) + new = docker_root / "volumes" / "omega-memory" + new.mkdir() + (new / "history.metta").write_text("(new history)\n") + + result = _launcher(docker_root, "start", "-d", IMAGE) + + assert result.returncode == 0, result.stderr + assert _snapshot(new) == {"history.metta": "(new history)\n"} + assert _container_state(docker_root, "omegaclaw") == "running" + + +def test_without_old_volume_agent_starts_on_image_memory(docker_root): + result = _launcher(docker_root, "start", "-d", IMAGE) + + assert result.returncode == 0, result.stderr + assert _started_agent(docker_root) + assert _snapshot(docker_root / "volumes" / "omega-memory") == _snapshot(docker_root / "image-memory") + assert not (docker_root / "volumes" / "omegaclaw-memory").exists() + + +def test_memory_import_skips_migration(docker_root, tmp_path): + _install_omegaclaw(docker_root) + transfer = tmp_path / "transfer" + transfer.mkdir() + transfer.chmod(0o2770) + (transfer / "memory.tar.gz").touch() + python3 = docker_root.parent / "bin" / "python3" + python3.write_text( + "#!/bin/sh\n" + "case \"$1\" in\n" + " *omega-memory-transfer-validator*) exit 0 ;;\n" + "esac\n" + f"exec {sys.executable!s} \"$@\"\n", + encoding="utf-8", + ) + python3.chmod(0o755) + + result = _launcher( + docker_root, + "start", + "-d", + IMAGE, + "--memory-transfer-dir", + str(transfer), + "--memory-transfer-gid", + str(transfer.stat().st_gid), + "--memory-import", + "memory.tar.gz", + ) + + assert result.returncode == 0, result.stderr + assert _read(docker_root / "volumes" / "omega-memory" / "history.metta") == "" + assert _container_state(docker_root, "omegaclaw") == "running" + + +def test_failed_copy_restores_previous_state(docker_root): + old = _install_omegaclaw(docker_root, running=True) + before = _snapshot(old) + + result = _launcher(docker_root, "start", "-d", IMAGE, fail_copy=True) + + assert result.returncode != 0 + assert not (docker_root / "volumes" / "omega-memory").exists() + assert _container_state(docker_root, "omegaclaw") == "running" + assert _snapshot(old) == before + assert not _started_agent(docker_root) + + +def test_failed_file_copy_restores_previous_state(docker_root): + old = _install_omegaclaw(docker_root, running=True) + before = _snapshot(old) + + result = _launcher(docker_root, "start", "-d", IMAGE, fail_cp=True) + + assert result.returncode != 0 + assert not (docker_root / "volumes" / "omega-memory").exists() + assert _container_state(docker_root, "omegaclaw") == "running" + assert _snapshot(old) == before + assert not _started_agent(docker_root) + + +def test_telegram_prompt_comes_from_image(docker_root): + old = _install_omegaclaw(docker_root) + (old / "tg_prompt.txt").write_text("dev build telegram prompt\n") + + result = _launcher(docker_root, "start", "-d", IMAGE) + + assert result.returncode == 0, result.stderr + assert _read(docker_root / "volumes" / "omega-memory" / "tg_prompt.txt") == "omega telegram prompt\n" + + +def test_image_without_omega_memory_layout_aborts(docker_root): + old = _install_omegaclaw(docker_root, running=True) + before = _snapshot(old) + shutil.rmtree(docker_root / "image-memory") + (docker_root / "image-memory").mkdir() + + result = _launcher(docker_root, "start", "-d", IMAGE) + + assert result.returncode != 0 + assert not (docker_root / "volumes" / "omega-memory").exists() + assert _container_state(docker_root, "omegaclaw") == "running" + assert _snapshot(old) == before + assert not _started_agent(docker_root) + + +def _interrupted_migration(root, image_files): + old = _install_omegaclaw(root, running=False) + (old / ".migration-started").write_text("") + new = root / "volumes" / "omega-memory" + new.mkdir() + if image_files: + shutil.copytree(root / "image-memory", new, dirs_exist_ok=True) + return old, new + + +def test_interrupted_copy_is_redone(docker_root): + old, new = _interrupted_migration(docker_root, image_files=True) + + result = _launcher(docker_root, "start", "-d", IMAGE) + + assert result.returncode == 0, result.stderr + assert _read(new / "history.metta") == "(old history)\n" + assert _read(new / "chroma_db" / "chroma.sqlite3") == "old long-term memory" + assert _read(new / "prompt.txt") == "omega prompt\n" + assert (old / ".migrated-to-omega").exists() + assert _started_agent(docker_root) + + +def test_run_interrupted_before_the_image_files_is_redone(docker_root): + old, new = _interrupted_migration(docker_root, image_files=False) + + result = _launcher(docker_root, "start", "-d", IMAGE) + + assert result.returncode == 0, result.stderr + assert _read(new / "history.metta") == "(old history)\n" + assert _read(new / "prompt.txt") == "omega prompt\n" + assert _started_agent(docker_root) + + +def test_start_marker_stays_out_of_new_volume(docker_root): + _install_omegaclaw(docker_root) + + result = _launcher(docker_root, "start", "-d", IMAGE) + + assert result.returncode == 0, result.stderr + assert not (docker_root / "volumes" / "omega-memory" / ".migration-started").exists() + + +def test_copy_is_redone_after_new_volume_could_not_be_removed(docker_root): + old = _install_omegaclaw(docker_root, running=True) + + failed = _launcher(docker_root, "start", "-d", IMAGE, fail_copy=True, fail_volume_rm=True) + + assert failed.returncode != 0 + assert (old / ".migration-started").exists() + assert not _started_agent(docker_root) + + result = _launcher(docker_root, "start", "-d", IMAGE) + + assert result.returncode == 0, result.stderr + assert _read(docker_root / "volumes" / "omega-memory" / "history.metta") == "(old history)\n" + assert _started_agent(docker_root) + + +def test_start_stops_when_markers_cannot_be_read(docker_root): + _, new = _interrupted_migration(docker_root, image_files=True) + before = _snapshot(new) + + result = _launcher(docker_root, "start", "-d", IMAGE, fail_marker_check=True) + + assert result.returncode != 0 + assert not _started_agent(docker_root) + assert _snapshot(new) == before diff --git a/tests/test_proxy_auth_secret.py b/tests/test_proxy_auth_secret.py new file mode 100644 index 000000000..d02c542c7 --- /dev/null +++ b/tests/test_proxy_auth_secret.py @@ -0,0 +1,59 @@ +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] +NGINX_SCRIPT = REPO_ROOT / "proxy" / "nginx.sh" +TEMPLATE = REPO_ROOT / "proxy" / "nginx.conf.template" + +pytestmark = pytest.mark.skipif(shutil.which("envsubst") is None, reason="envsubst is not installed") + + +def _render(tmp_path, **secrets): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + nginx = bin_dir / "nginx" + nginx.write_text("#!/bin/sh\nexit 0\n") + nginx.chmod(0o755) + config = tmp_path / "nginx.conf" + environment = {name: value for name, value in os.environ.items() if not name.endswith("AUTH_SECRET")} + environment.update(secrets) + environment["PATH"] = f"{bin_dir}{os.pathsep}{environment['PATH']}" + environment["NGINX_TEMPLATE"] = str(TEMPLATE) + environment["NGINX_CONFIG"] = str(config) + result = subprocess.run( + ["sh", str(NGINX_SCRIPT)], + env=environment, + capture_output=True, + text=True, + check=False, + ) + return result, config.read_text() if config.exists() else None + + +def test_old_secret_name_still_enables_auth(tmp_path): + result, config = _render(tmp_path, OMEGACLAW_AUTH_SECRET="4242") + + assert result.returncode == 0, result.stderr + assert 'set $auth_secret_value "4242";' in config + assert '$http_x_auth_token = "4242"' in config + assert "OMEGACLAW_AUTH_SECRET" in result.stderr + + +def test_new_secret_name_wins_over_old(tmp_path): + result, config = _render(tmp_path, OMEGA_AUTH_SECRET="1111", OMEGACLAW_AUTH_SECRET="4242") + + assert result.returncode == 0, result.stderr + assert 'set $auth_secret_value "1111";' in config + assert "4242" not in config + + +def test_without_secret_auth_stays_off(tmp_path): + result, config = _render(tmp_path) + + assert result.returncode == 0, result.stderr + assert 'set $auth_secret_value "";' in config