From c1980c7718f7bdbde6610f971b698ddac42f52b8 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 29 Sep 2026 19:34:20 +0800 Subject: [PATCH] fix: stage monitor dependencies without changing the active environment Co-Authored-By: Codex --- ops/quant-monitor/README.md | 8 ++ .../scripts/setup_vps_runtime.sh | 120 +++++++++++++++--- .../tests/test_deploy_scripts.py | 98 ++++++++++++++ 3 files changed, 206 insertions(+), 20 deletions(-) diff --git a/ops/quant-monitor/README.md b/ops/quant-monitor/README.md index 7ad1fbf1..a27ac168 100644 --- a/ops/quant-monitor/README.md +++ b/ops/quant-monitor/README.md @@ -192,4 +192,12 @@ Linux x86_64 / glibc ≥ 2.34** 上实际需要的直接与传递依赖（含既 锁 setuptools/wheel，不隐式下载 build deps；最后 `pip check`。pip 成功不等于业 务验收。 +需要单独准备 venv 时，可显式设置 `QUANT_MONITOR_VENV` 指向一个新的绝对目录；其 +父目录必须已存在，目标不能已存在，也不能与 AAB 源码、monitor/data、现用 `.venv`、 +`QUANT_PROJECTS_ROOT`、`LIFECYCLE_LOCAL_ROOT` 或 QPK 镜像重叠。此模式只从现有 QPK +镜像读取精确 pin 的 Git archive，不 fetch、 +checkout 或改写共享镜像；本地缺少该 commit 时会在创建 venv 前失败。未设置该变量 +时仍安装到 `$QUANT_MONITOR_ROOT/.venv`，默认流程不变。暂存不会修改 systemd 配置； +服务是否使用该路径须由独立的配置变更明确决定。 + 生产 venv 是否已按此 lock 迁移须另做安装/读回；本说明不声称生产已切换。 diff --git a/ops/quant-monitor/scripts/setup_vps_runtime.sh b/ops/quant-monitor/scripts/setup_vps_runtime.sh index 08cb5bc8..83da82e9 100755 --- a/ops/quant-monitor/scripts/setup_vps_runtime.sh +++ b/ops/quant-monitor/scripts/setup_vps_runtime.sh @@ -121,33 +121,113 @@ if [[ ! "$QPK_RUNTIME_SHA" =~ ^[0-9a-f]{40}$ ]]; then exit 1 fi -VENV="$ROOT/.venv" QPK_ROOT="${QUANT_PLATFORM_KIT_ROOT:?QuantPlatformKit not found}" +QPK_ARCHIVE_REF=HEAD +if [[ "${QUANT_MONITOR_VENV+x}" == x ]]; then + VENV_REQUESTED="$QUANT_MONITOR_VENV" -qpk_created=0 -if [[ ! -d "$QPK_ROOT/.git" ]]; then - echo "[setup] cloning QuantPlatformKit into $QPK_ROOT" >&2 - mkdir -p "$(dirname "$QPK_ROOT")" - git clone --no-checkout https://github.com/QuantStrategyLab/QuantPlatformKit.git "$QPK_ROOT" - qpk_created=1 -fi + canonical_new_directory_path() { + local requested="$1" parent basename canonical_parent + [[ "$requested" == /* && "$requested" != *$'\n'* && "$requested" != *$'\r'* ]] || return 1 + case "/$requested/" in */../*) return 1 ;; esac + [[ ! -e "$requested" && ! -L "$requested" ]] || return 1 + parent="${requested%/*}" + basename="${requested##*/}" + [[ -n "$basename" && "$basename" != "." && "$basename" != ".." && -d "$parent" ]] || return 1 + canonical_parent="$(cd -P "$parent" && pwd -P)" || return 1 + CANONICAL_PATH="${canonical_parent%/}/$basename" + } + + canonical_existing_or_new_path() { + local requested="$1" existing suffix="" basename canonical_parent + [[ "$requested" == /* ]] || return 1 + if [[ -d "$requested" ]]; then + CANONICAL_PATH="$(cd -P "$requested" && pwd -P)" + return + fi + existing="$requested" + while [[ "$existing" != "/" && "$existing" == */ ]]; do + existing="${existing%/}" + done + while [[ ! -e "$existing" && ! -L "$existing" ]]; do + basename="${existing##*/}" + [[ -n "$basename" ]] || return 1 + suffix="/$basename$suffix" + existing="${existing%/*}" + [[ -n "$existing" ]] || existing="/" + done + [[ -d "$existing" ]] || return 1 + canonical_parent="$(cd -P "$existing" && pwd -P)" || return 1 + if [[ "$canonical_parent" == "/" ]]; then + CANONICAL_PATH="/${suffix#/}" + else + CANONICAL_PATH="${canonical_parent}${suffix}" + fi + } -git -C "$QPK_ROOT" fetch origin "$QPK_RUNTIME_SHA" --quiet -# Fresh --no-checkout clones report every tracked path as deleted until the first -# checkout; only existing mirrors must refuse unknown/staged edits first. -if [[ "$qpk_created" -eq 0 ]]; then - if [[ -n "$(git -C "$QPK_ROOT" status --porcelain --untracked-files=no)" ]]; then - echo "[setup] QPK tracked changes require mirror synchronization/review" >&2 + paths_overlap() { + [[ "$1" == "$2" || "$1" == "$2/"* || "$2" == "$1/"* ]] + } + + if ! canonical_new_directory_path "$VENV_REQUESTED"; then + echo "[setup] staging venv must be a new absolute path with an existing parent and no '..' segments" >&2 + exit 1 + fi + VENV="$CANONICAL_PATH" + PROTECTED_PATHS=() + for protected_path in \ + "$AAB_ROOT" "$ROOT" "$ROOT/data" "$ROOT/.venv" \ + "$QUANT_PROJECTS_ROOT" "$LIFECYCLE_LOCAL_ROOT" "$QPK_ROOT"; do + if ! canonical_existing_or_new_path "$protected_path"; then + echo "[setup] unable to resolve protected runtime path" >&2 + exit 1 + fi + PROTECTED_PATHS+=("$CANONICAL_PATH") + done + for protected_path in "${PROTECTED_PATHS[@]}"; do + if paths_overlap "$VENV" "$protected_path"; then + echo "[setup] staging venv overlaps a protected source, data, live venv, or QPK path" >&2 + exit 1 + fi + done + + if [[ ! -d "$QPK_ROOT/.git" && ! -f "$QPK_ROOT/.git" ]] || \ + ! git -C "$QPK_ROOT" cat-file -e "${QPK_RUNTIME_SHA}^{commit}" 2>/dev/null; then + echo "[setup] pinned QuantPlatformKit commit is unavailable in the existing mirror; staging refused without mirror changes" >&2 + exit 1 + fi + QPK_ARCHIVE_REF="$QPK_RUNTIME_SHA" +else + VENV="$ROOT/.venv" + qpk_created=0 + if [[ ! -d "$QPK_ROOT/.git" ]]; then + echo "[setup] cloning QuantPlatformKit into $QPK_ROOT" >&2 + mkdir -p "$(dirname "$QPK_ROOT")" + git clone --no-checkout https://github.com/QuantStrategyLab/QuantPlatformKit.git "$QPK_ROOT" + qpk_created=1 + fi + + git -C "$QPK_ROOT" fetch origin "$QPK_RUNTIME_SHA" --quiet + # Fresh --no-checkout clones report every tracked path as deleted until the first + # checkout; only existing mirrors must refuse unknown/staged edits first. + if [[ "$qpk_created" -eq 0 ]]; then + if [[ -n "$(git -C "$QPK_ROOT" status --porcelain --untracked-files=no)" ]]; then + echo "[setup] QPK tracked changes require mirror synchronization/review" >&2 + exit 1 + fi + fi + git -C "$QPK_ROOT" checkout --detach --quiet "$QPK_RUNTIME_SHA" + ACTUAL_QPK_SHA="$(git -C "$QPK_ROOT" rev-parse HEAD)" + if [[ "$ACTUAL_QPK_SHA" != "$QPK_RUNTIME_SHA" ]]; then + echo "[setup] QuantPlatformKit checkout does not match pinned SHA" >&2 exit 1 fi fi -git -C "$QPK_ROOT" checkout --detach --quiet "$QPK_RUNTIME_SHA" -ACTUAL_QPK_SHA="$(git -C "$QPK_ROOT" rev-parse HEAD)" -if [[ "$ACTUAL_QPK_SHA" != "$QPK_RUNTIME_SHA" ]]; then - echo "[setup] QuantPlatformKit checkout does not match pinned SHA" >&2 + +if [[ "${QUANT_MONITOR_VENV+x}" == x ]] && ! mkdir -- "$VENV"; then + echo "[setup] unable to claim the staging venv path" >&2 exit 1 fi - python3 -m venv "$VENV" # Locked third-party tree only; no unbounded pip/wheel upgrade and no free-floating # numpy/pandas/google-cloud-storage installs. @@ -160,7 +240,7 @@ fi # Reuse locked setuptools/wheel; do not download build dependencies. build_root=$(mktemp -d "${TMPDIR:-/tmp}/quant-monitor-qpk.XXXXXX") trap 'rm -rf -- "$build_root"' EXIT -git -C "$QPK_ROOT" archive HEAD | tar -x -C "$build_root" +git -C "$QPK_ROOT" archive "$QPK_ARCHIVE_REF" | tar -x -C "$build_root" if ! "$VENV/bin/python" -m pip install --no-deps --no-build-isolation "$build_root"; then echo "[setup] QuantPlatformKit install failed" >&2 exit 1 diff --git a/ops/quant-monitor/tests/test_deploy_scripts.py b/ops/quant-monitor/tests/test_deploy_scripts.py index ed715157..b4a9be74 100644 --- a/ops/quant-monitor/tests/test_deploy_scripts.py +++ b/ops/quant-monitor/tests/test_deploy_scripts.py @@ -371,6 +371,12 @@ def test_setup_fresh_qpk_clone_checks_out_pin_and_existing_dirty_mirror_still_re qpk_parent = root / "mirrors" qpk_parent.mkdir() qpk = qpk_parent / "QuantPlatformKit" + lifecycle_root = root / "lifecycle-store" + lifecycle_root.mkdir() + qpk_alias = root / "mirror-root-alias" + qpk_alias.symlink_to(qpk_parent, target_is_directory=True) + lifecycle_alias = root / "lifecycle-root-alias" + lifecycle_alias.symlink_to(lifecycle_root, target_is_directory=True) bin_dir = root / "bin" bin_dir.mkdir() real_git = shutil.which("git") @@ -392,12 +398,15 @@ def test_setup_fresh_qpk_clone_checks_out_pin_and_existing_dirty_mirror_still_re pip_log = root / "pip.log" pip_code = ( f"#!{sys.executable}\n" + "import os\n" "import sys\n" "from pathlib import Path\n" f"log = Path({str(pip_log)!r})\n" "prev = log.read_text() if log.exists() else ''\n" "log.write_text(prev + ' '.join(sys.argv[1:]) + '\\n')\n" "args = sys.argv[1:]\n" + "if os.environ.get('SETUP_FAIL_PIP') == '1' and 'install' in args:\n" + " raise SystemExit(17)\n" "if '-e' in args:\n" " raise SystemExit(9)\n" "if 'install' in args and '-U' in args:\n" @@ -466,9 +475,11 @@ def test_setup_fresh_qpk_clone_checks_out_pin_and_existing_dirty_mirror_still_re "QUANT_PLATFORM_KIT_ROOT": str(qpk), "AIAUDIT_BRIDGE_ROOT": str(aab), "QUANT_PROJECTS_ROOT": str(qpk_parent), + "LIFECYCLE_LOCAL_ROOT": str(lifecycle_root), "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1", } + env.pop("QUANT_MONITOR_VENV", None) self.assertFalse(qpk.exists()) fresh = subprocess.run( ["bash", str(ROOT / "scripts" / "setup_vps_runtime.sh"), aab_sha, str(aab)], @@ -490,6 +501,93 @@ def test_setup_fresh_qpk_clone_checks_out_pin_and_existing_dirty_mirror_still_re self.assertIn("check", pip_calls) self.assertNotIn(" -U ", f" {pip_calls} ") + live_venv_marker = monitor / ".venv" / "keep-existing-env.txt" + live_venv_marker.write_text("preserve\n", encoding="utf-8") + stage_parent = root / "isolated-stage" + stage_parent.mkdir() + staged_venv = stage_parent / ".venv" + staged_qpk = Path(f"{staged_venv}.qpk") + qpk_status_before_stage = self._git(qpk, "status", "--porcelain", "--untracked-files=all") + staged = subprocess.run( + ["bash", str(ROOT / "scripts" / "setup_vps_runtime.sh"), aab_sha, str(aab)], + text=True, + capture_output=True, + check=False, + env={**env, "QUANT_MONITOR_VENV": str(staged_venv)}, + ) + self.assertEqual(staged.returncode, 0, staged.stderr) + self.assertTrue((staged_venv / "bin" / "python").is_file()) + self.assertFalse(staged_qpk.exists()) + self.assertEqual(self._git(qpk, "rev-parse", "HEAD"), pin) + self.assertEqual( + self._git(qpk, "status", "--porcelain", "--untracked-files=all"), + qpk_status_before_stage, + ) + self.assertEqual((qpk / "module.py").read_text(encoding="utf-8"), "VERSION = 1\n") + self.assertEqual(live_venv_marker.read_text(encoding="utf-8"), "preserve\n") + + alias_parent = root / "live-venv-alias" + alias_parent.symlink_to(monitor / ".venv", target_is_directory=True) + invalid_stages = ( + monitor / ".venv", + aab / "new-stage", + alias_parent / "staged", + qpk_parent / "new-stage", + lifecycle_root / "new-stage", + qpk_alias / "new-stage", + lifecycle_alias / "new-stage", + root / "missing-parent" / ".venv", + stage_parent / ".." / "escaped-stage", + ) + for invalid_stage in invalid_stages: + refused = subprocess.run( + ["bash", str(ROOT / "scripts" / "setup_vps_runtime.sh"), aab_sha, str(aab)], + text=True, + capture_output=True, + check=False, + env={**env, "QUANT_MONITOR_VENV": str(invalid_stage)}, + ) + self.assertNotEqual(refused.returncode, 0, str(invalid_stage)) + self.assertEqual(self._git(qpk, "rev-parse", "HEAD"), pin) + self.assertEqual((qpk / "module.py").read_text(encoding="utf-8"), "VERSION = 1\n") + self.assertEqual(live_venv_marker.read_text(encoding="utf-8"), "preserve\n") + + relative_protected_root = subprocess.run( + ["bash", str(ROOT / "scripts" / "setup_vps_runtime.sh"), aab_sha, str(aab)], + text=True, + capture_output=True, + check=False, + timeout=5, + env={ + **env, + "QUANT_MONITOR_VENV": str(stage_parent / "relative-root-stage"), + "LIFECYCLE_LOCAL_ROOT": "missing-lifecycle-root", + }, + ) + self.assertNotEqual(relative_protected_root.returncode, 0) + self.assertIn("unable to resolve protected runtime path", relative_protected_root.stderr) + self.assertFalse((stage_parent / "relative-root-stage").exists()) + + failed_stage = root / "failed-stage" / ".venv" + failed_stage.parent.mkdir() + failed = subprocess.run( + ["bash", str(ROOT / "scripts" / "setup_vps_runtime.sh"), aab_sha, str(aab)], + text=True, + capture_output=True, + check=False, + env={ + **env, + "QUANT_MONITOR_VENV": str(failed_stage), + "SETUP_FAIL_PIP": "1", + }, + ) + self.assertNotEqual(failed.returncode, 0) + self.assertIn("locked dependency install failed", failed.stderr) + self.assertTrue(live_venv_marker.is_file()) + self.assertEqual(live_venv_marker.read_text(encoding="utf-8"), "preserve\n") + self.assertEqual(self._git(qpk, "rev-parse", "HEAD"), pin) + self.assertEqual((qpk / "module.py").read_text(encoding="utf-8"), "VERSION = 1\n") + (qpk / "module.py").write_text("unknown local edit\n", encoding="utf-8") blocked = subprocess.run( ["bash", str(ROOT / "scripts" / "setup_vps_runtime.sh"), aab_sha, str(aab)],