From 9445e8d4430b28ee59fb8cecf88774432c6132ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D1=96=D1=82=D0=B0=D0=BB=D1=96=D0=B9=20=D0=9B=D0=B8?= =?UTF-8?q?=D1=82=D0=BE=D0=B2=D1=87=D0=B5=D0=BD=D0=BA=D0=BE?= Date: Sat, 21 Nov 2026 18:59:19 +0000 Subject: [PATCH 1/2] Harden AutoMap authentication, media processing, and OAuth --- .github/workflows/python-tests.yml | 17 +- AutoMap-agent/README.md | 10 +- AutoMap-agent/agent/automap_client.py | 6 +- AutoMap-agent/agent/db.py | 9 +- AutoMap-agent/agent/media_pool.py | 5 + AutoMap-agent/agent/private_files.py | 40 +++++ AutoMap-agent/agent/runner.py | 4 +- AutoMap-agent/agent/youtube_intel.py | 4 +- AutoMap-agent/docker-compose.local.yml | 17 ++ AutoMap-agent/docker-compose.yml | 3 +- AutoMap-agent/example.env | 3 +- AutoMap-agent/requirements.txt | 2 +- AutoMap-agent/templates/agent.html | 51 ++---- AutoMap-agent/tests/test_agent_optimizer.py | 2 +- AutoMap-agent/tests/test_agent_web_api.py | 44 +++++ AutoMap-agent/web.py | 117 +++++++++---- Dockerfile.ytserver | 16 +- docker-compose.yml | 18 +- example.env | 8 +- main.py | 38 ++--- map/auto_map.py | 4 +- media_inputs.py | 50 ++++++ overview.md | 2 +- pyproject.toml | 12 +- readme.md | 16 +- requirements.txt | 7 +- server_runtime.py | 6 +- tests/browser/test_cookie_file_upload.py | 5 +- tests/conftest.py | 7 +- tests/frontend/topic-preview.test.cjs | 3 + tests/test_audio_upload.py | 2 +- tests/test_server_runtime.py | 16 +- tests/test_web.py | 2 +- tests/test_web_auth.py | 26 ++- tests/test_web_auth_security.py | 22 ++- tests/test_worker.py | 59 ++++--- tests/test_worker_audit.py | 57 +++++-- tests/test_youtube_upload.py | 2 +- video_render.py | 3 +- video_stickers.py | 15 +- web_auth.py | 25 ++- yt-dlp_serv.py | 176 ++++++++++---------- 42 files changed, 624 insertions(+), 307 deletions(-) create mode 100644 AutoMap-agent/agent/private_files.py create mode 100644 AutoMap-agent/docker-compose.local.yml create mode 100644 media_inputs.py diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 0249bf7..935bcd4 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -16,9 +16,12 @@ name: Run Pytest on: push: - branches: ["main"] + branches: ["main", "dev"] pull_request: - branches: ["main"] + branches: ["main", "dev"] + +permissions: + contents: read jobs: test: @@ -26,6 +29,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v4 + with: + persist-credentials: false - name: Set up Python uses: actions/setup-python@v5 with: @@ -40,6 +45,14 @@ jobs: run: python -m playwright install --with-deps chromium - name: Run tests and measure coverage run: python -m pytest --cov --cov-report=term-missing --cov-report=xml + - name: Run Agent tests + env: + PYTHONPATH: AutoMap-agent + run: | + python -m pip install -r AutoMap-agent/requirements.txt + python -m pytest AutoMap-agent/tests + - name: Run frontend tests + run: node --test tests/frontend/*.test.cjs - name: Upload coverage report if: always() uses: actions/upload-artifact@v4 diff --git a/AutoMap-agent/README.md b/AutoMap-agent/README.md index 99bf9e4..b2c205c 100644 --- a/AutoMap-agent/README.md +++ b/AutoMap-agent/README.md @@ -2,12 +2,17 @@ Autonomous 24/7 YouTube Shorts agent separated from `AutoMap-cli`. -## Local Docker (connects to local AutoMap on `http://localhost:3000`) +## Local Docker ```bash +# From the AutoMap-cli root docker compose up -d --build +cd AutoMap-agent +docker compose -f docker-compose.yml -f docker-compose.local.yml up -d --build ``` +The agent uses the private `automap-agent-link` network to reach AutoMap. The browser opens AutoMap at `http://localhost:3000`. + Open **http://localhost:4000**. ## Server / VPS Deployment (connects to remote AutoMap server) @@ -16,10 +21,9 @@ Open **http://localhost:4000**. 2. Set: - `AGENT_SERVER=TRUE` - `AGENT_WEB_USERNAME` & `AGENT_WEB_PASSWORD` - - `AGENT_SECRET_KEY` + - Optional `AGENT_SECRET_KEY` to override the secret persisted in `instance/` - `AUTOMAP_URL=https://your-automap-domain.com` - `AUTOMAP_PUBLIC_URL=https://your-automap-domain.com` - `WEB_LOGIN` & `WEB_PASSWORD` (if your AutoMap server has login enabled) - `AGENT_YOUTUBE_REDIRECT_URI=https://your-agent-domain.com/api/agent/youtube/callback` - - `OAUTHLIB_INSECURE_TRANSPORT=0` 3. Run `docker compose up -d --build`. diff --git a/AutoMap-agent/agent/automap_client.py b/AutoMap-agent/agent/automap_client.py index 61c6ae6..852a594 100644 --- a/AutoMap-agent/agent/automap_client.py +++ b/AutoMap-agent/agent/automap_client.py @@ -40,15 +40,15 @@ def __init__( session: requests.Session | None = None, ) -> None: self.base_url = (base_url or os.environ.get("AUTOMAP_URL") or "http://host.docker.internal:3000").rstrip("/") - self.login_user = login if login is not None else os.environ.get("WEB_LOGIN", "deltaaa") - self.login_password = password if password is not None else os.environ.get("WEB_PASSWORD", "deltaaa00!") + self.login_user = login if login is not None else os.environ.get("WEB_LOGIN", "") + self.login_password = password if password is not None else os.environ.get("WEB_PASSWORD", "") self.timeout = float(timeout) self.http = session or requests.Session() self._logged_in = False def ensure_auth(self) -> None: """Sign in via /login when the web service runs with SERVER=TRUE.""" - if self._logged_in: + if self._logged_in or not (self.login_user and self.login_password): return try: resp = self.http.post( diff --git a/AutoMap-agent/agent/db.py b/AutoMap-agent/agent/db.py index b11147f..7763736 100644 --- a/AutoMap-agent/agent/db.py +++ b/AutoMap-agent/agent/db.py @@ -52,7 +52,13 @@ class AgentDB: def __init__(self, db_path: str | Path | None = None) -> None: self.path = Path(db_path) if db_path is not None else default_db_path() - self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + try: + database = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + except FileExistsError: + pass + else: + os.close(database) self.init_schema() @contextmanager @@ -152,6 +158,7 @@ def init_schema(self) -> None: ); """ ) + conn.execute("BEGIN IMMEDIATE") self._ensure_column(conn, "competitor_videos", "likes", "INTEGER NOT NULL DEFAULT 0") self._ensure_column(conn, "competitor_videos", "comments", "INTEGER NOT NULL DEFAULT 0") self._ensure_column(conn, "experiments", "avg_view_duration_sec", "REAL") diff --git a/AutoMap-agent/agent/media_pool.py b/AutoMap-agent/agent/media_pool.py index ecb13c1..e0c0d67 100644 --- a/AutoMap-agent/agent/media_pool.py +++ b/AutoMap-agent/agent/media_pool.py @@ -134,6 +134,8 @@ def probe_video_duration(video_path: Path) -> float | None: "ffprobe", "-v", "error", + "-protocol_whitelist", "file", + "-format_whitelist", "mov,matroska,webm", "-show_entries", "format=duration", "-of", @@ -187,6 +189,9 @@ def split_background_video( "-y", "-ss", f"{start:.2f}", + "-nostdin", + "-protocol_whitelist", "file", + "-format_whitelist", "mov,matroska,webm", "-i", str(source_path), "-t", diff --git a/AutoMap-agent/agent/private_files.py b/AutoMap-agent/agent/private_files.py new file mode 100644 index 0000000..cc0fbe0 --- /dev/null +++ b/AutoMap-agent/agent/private_files.py @@ -0,0 +1,40 @@ +"""Write credentials with private permissions and atomic replacement.""" + +import os +import secrets +from pathlib import Path +from tempfile import NamedTemporaryFile + + +def write_private_text(path: Path, content: str, *, create_only: bool = False) -> None: + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + with NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent, delete=False) as stream: + temporary = Path(stream.name) + try: + stream.write(content) + except BaseException: + stream.close() + temporary.unlink(missing_ok=True) + raise + try: + if create_only: + try: + os.link(temporary, path) + except FileExistsError: + pass + else: + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +def session_secret(path: Path) -> str: + configured = os.environ.get("AGENT_SECRET_KEY", "").strip() + if configured: + return configured + if not path.is_file(): + write_private_text(path, secrets.token_hex(32), create_only=True) + value = path.read_text(encoding="utf-8").strip() + if not value: + raise RuntimeError("The agent session secret file is empty.") + return value diff --git a/AutoMap-agent/agent/runner.py b/AutoMap-agent/agent/runner.py index 0bfd9bc..392c6c0 100644 --- a/AutoMap-agent/agent/runner.py +++ b/AutoMap-agent/agent/runner.py @@ -377,9 +377,7 @@ def _is_retryable_model_error(err: AutoMapClientError) -> bool: log.warning("Could not render clean reveal base map for %s", zoom_key, exc_info=True) if effective_reveal and not reveal_clean_map_file: effective_reveal = False - effective_zoom = False - effective_zoom_out = False - log.warning("Skipping zoom/reveal for %s because its clean map is unavailable", zoom_key) + log.warning("Skipping reveal for %s because its clean map is unavailable", zoom_key) duration = max(9, min(15, int(cfg.get("video_duration") or 11))) render_extra: dict[str, Any] = { diff --git a/AutoMap-agent/agent/youtube_intel.py b/AutoMap-agent/agent/youtube_intel.py index fe64175..27d57ca 100644 --- a/AutoMap-agent/agent/youtube_intel.py +++ b/AutoMap-agent/agent/youtube_intel.py @@ -13,6 +13,8 @@ import requests +from .private_files import write_private_text + from .db import AgentDB, utc_now_iso from .optimizer import compute_video_score @@ -211,7 +213,7 @@ def _load_oauth_credentials(self): creds = Credentials.from_authorized_user_info(info) if creds.expired and creds.refresh_token: creds.refresh(Request()) - self.token_path.write_text(creds.to_json(), encoding="utf-8") + write_private_text(self.token_path, creds.to_json()) return creds def has_upload_credentials(self) -> bool: diff --git a/AutoMap-agent/docker-compose.local.yml b/AutoMap-agent/docker-compose.local.yml new file mode 100644 index 0000000..686e7e3 --- /dev/null +++ b/AutoMap-agent/docker-compose.local.yml @@ -0,0 +1,17 @@ +services: + agent-web: + environment: + AUTOMAP_URL: http://automap-web:5000 + networks: + - default + - automap + agent-runner: + environment: + AUTOMAP_URL: http://automap-web:5000 + networks: + - default + - automap +networks: + automap: + external: true + name: automap-agent-link diff --git a/AutoMap-agent/docker-compose.yml b/AutoMap-agent/docker-compose.yml index c648ef9..2a37d0f 100644 --- a/AutoMap-agent/docker-compose.yml +++ b/AutoMap-agent/docker-compose.yml @@ -5,7 +5,7 @@ services: dockerfile: Dockerfile command: ["gunicorn", "--bind", "0.0.0.0:4000", "--workers", "2", "--threads", "4", "--timeout", "120", "web:app"] ports: - - "${AGENT_PORT:-4000}:4000" + - "${AGENT_BIND_ADDRESS:-127.0.0.1}:${AGENT_PORT:-4000}:4000" env_file: - path: .env required: false @@ -15,7 +15,6 @@ services: - AGENT_DB_PATH=/app/instance/agent.db - AGENT_MEDIA_DIR=/app/agent_media - AGENT_YOUTUBE_TOKEN_PATH=/app/instance/agent_youtube_token.json - - OAUTHLIB_INSECURE_TRANSPORT=${OAUTHLIB_INSECURE_TRANSPORT:-1} extra_hosts: - "host.docker.internal:host-gateway" volumes: diff --git a/AutoMap-agent/example.env b/AutoMap-agent/example.env index d7b348b..b38fb3d 100644 --- a/AutoMap-agent/example.env +++ b/AutoMap-agent/example.env @@ -5,7 +5,6 @@ AGENT_PORT=4000 AUTOMAP_URL=http://host.docker.internal:3000 AUTOMAP_PUBLIC_URL=http://localhost:3000 -OAUTHLIB_INSECURE_TRANSPORT=1 # --- Server / VPS deployment mode --- # Set AGENT_SERVER=TRUE and AGENT_WEB_PASSWORD to protect the agent dashboard with login. @@ -18,7 +17,7 @@ OAUTHLIB_INSECURE_TRANSPORT=1 # WEB_LOGIN=your-automap-login # WEB_PASSWORD=your-automap-password # AGENT_YOUTUBE_REDIRECT_URI=https://your-agent-domain.com/api/agent/youtube/callback -# OAUTHLIB_INSECURE_TRANSPORT=0 +# AGENT_BIND_ADDRESS=127.0.0.1 # Optional env fallbacks (you can also enter all of these directly in the web UI): # TELEGRAM_BOT_TOKEN= diff --git a/AutoMap-agent/requirements.txt b/AutoMap-agent/requirements.txt index 96d6204..8403640 100644 --- a/AutoMap-agent/requirements.txt +++ b/AutoMap-agent/requirements.txt @@ -1,4 +1,4 @@ -Flask>=3.0,<4.0 +Flask>=3.1,<4.0 gunicorn>=22.0,<24.0 python-dotenv>=1.0,<2.0 requests>=2.31,<3.0 diff --git a/AutoMap-agent/templates/agent.html b/AutoMap-agent/templates/agent.html index 109f488..5e2438c 100644 --- a/AutoMap-agent/templates/agent.html +++ b/AutoMap-agent/templates/agent.html @@ -468,14 +468,19 @@

document.getElementById('secTavilyApiKeyBadge').classList.toggle('hidden', !data.tavily_api_configured); if (document.activeElement?.id !== 'secTgChatId' && data.telegram_chat_id && !document.getElementById('secTgChatId').value) { document.getElementById('secTgChatId').value = data.telegram_chat_id; - saveLocalSecret('secTgChatId', data.telegram_chat_id); } const renderFileList = (elId, kind, list) => { const el = document.getElementById(elId); if (!el) return; el.innerHTML = (list && list.length) - ? list.map(name => `
  • ${esc(name)}${kind === 'background' ? `` : ''}
  • `).join('') + ? list.map(name => `
  • ${esc(name)}${kind === 'background' ? `` : ''}
  • `).join('') : '
  • Empty
  • '; + for (const button of el.querySelectorAll('button[data-filename]')) { + button.addEventListener('click', () => { + if (button.dataset.action === 'split') splitMedia(button.dataset.filename); + else deleteMedia(kind, button.dataset.filename); + }); + } }; renderFileList('bgFilesList', 'background', pool.backgrounds || []); renderFileList('musicFilesList', 'music', pool.music || []); @@ -714,37 +719,14 @@

    loadOverview(); } - const SECRET_STORAGE_KEYS = { - secTgToken: 'automap-agent-tg-token', - secTgChatId: 'automap-agent-tg-chat-id', - secYtApiKey: 'automap-agent-yt-api-key', - secGeminiApiKey: 'automap-gemini-api-key', - secDeepseekApiKey: 'automap-deepseek-api-key', - secTavilyApiKey: 'automap-tavily-api-key', - }; - - function saveLocalSecret(id, val) { + function clearLegacyAgentSecrets() { try { - const key = SECRET_STORAGE_KEYS[id]; - if (!key) return; - if (val) localStorage.setItem(key, val); - else localStorage.removeItem(key); + for (const key of ['automap-agent-tg-token', 'automap-agent-tg-chat-id', 'automap-agent-yt-api-key']) { + localStorage.removeItem(key); + } } catch (_) {} } - function initLocalSecrets() { - Object.entries(SECRET_STORAGE_KEYS).forEach(([id, storageKey]) => { - const el = document.getElementById(id); - if (!el) return; - try { - const saved = localStorage.getItem(storageKey); - if (saved) el.value = saved; - } catch (_) {} - el.addEventListener('input', () => saveLocalSecret(id, el.value.trim())); - el.addEventListener('change', () => saveLocalSecret(id, el.value.trim())); - }); - } - function toggleSecretVisibility(inputId, iconId) { const input = document.getElementById(inputId); const icon = document.getElementById(iconId); @@ -776,12 +758,9 @@

    }); const data = await res.json(); if (res.ok) { - if (tok) saveLocalSecret('secTgToken', tok); - if (cid) saveLocalSecret('secTgChatId', cid); - if (ytk) saveLocalSecret('secYtApiKey', ytk); - if (gmk) saveLocalSecret('secGeminiApiKey', gmk); - if (dsk) saveLocalSecret('secDeepseekApiKey', dsk); - if (tvk) saveLocalSecret('secTavilyApiKey', tvk); + for (const id of ['secTgToken', 'secYtApiKey', 'secGeminiApiKey', 'secDeepseekApiKey', 'secTavilyApiKey']) { + document.getElementById(id).value = ''; + } showAgentToast('Keys saved! Agent will pick them up automatically.'); } else { showAgentToast(data.error || 'Failed to save keys', true); @@ -882,7 +861,7 @@

    history.replaceState(null, '', location.pathname); } - initLocalSecrets(); + clearLegacyAgentSecrets(); loadOverview(); setInterval(loadOverview, 15000); diff --git a/AutoMap-agent/tests/test_agent_optimizer.py b/AutoMap-agent/tests/test_agent_optimizer.py index c2089c3..a95d4c3 100644 --- a/AutoMap-agent/tests/test_agent_optimizer.py +++ b/AutoMap-agent/tests/test_agent_optimizer.py @@ -18,7 +18,7 @@ def test_score_and_suffix_inference(): assert compute_video_score(1000, None, 80.0) == 800.0 - assert compute_video_score(1000, 5000, 90.0, likes_72h=250) == 5625.0 + assert compute_video_score(1000, 5000, 90.0, likes_72h=250) == 5400.0 assert infer_label_suffix("How tall is the tallest mountain in your country?", "number") == "m" assert infer_label_suffix("Forest coverage percentage in Europe", "number") == "%" assert infer_label_suffix("Most popular drink in Europe", "text") == "" diff --git a/AutoMap-agent/tests/test_agent_web_api.py b/AutoMap-agent/tests/test_agent_web_api.py index afdedcb..37063a7 100644 --- a/AutoMap-agent/tests/test_agent_web_api.py +++ b/AutoMap-agent/tests/test_agent_web_api.py @@ -110,3 +110,47 @@ def test_agent_dashboard_and_api_endpoints(tmp_path, monkeypatch): assert up_resp.status_code == 200 and "beat.mp3" in up_resp.get_json()["saved"] del_resp = client.post("/api/agent/media/delete", json={"kind": "music", "filename": "beat.mp3"}) assert del_resp.status_code == 200 + + +def test_oauth_callback_requires_the_browser_that_started_it(tmp_path, monkeypatch): + import io + import json + from urllib.parse import parse_qs, urlsplit + + monkeypatch.setitem(web.app.config, "AGENT_DB_PATH", str(tmp_path / "agent.db")) + monkeypatch.setattr(web, "AUTH_ENABLED", False) + oauth = { + "web": { + "client_id": "test-client", + "client_secret": "test-secret", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + }, + } + client = web.app.test_client() + + def connect(): + return client.post("/api/agent/youtube/connect", data={ + "client_secret": (io.BytesIO(json.dumps(oauth).encode()), "client_secret.json"), + }) + + response = connect() + assert response.status_code == 200 + params = parse_qs(urlsplit(response.json["authorization_url"]).query) + assert params["code_challenge_method"] == ["S256"] + callback = f"/api/agent/youtube/callback?state={params['state'][0]}&error=access_denied" + assert web.app.test_client().get(callback).location == "/?youtube=failed" + assert client.get(callback).location == "/?youtube=denied" + assert client.get(callback).location == "/?youtube=failed" + + oauth["web"]["token_uri"] = "https://example.test/token" + assert connect().status_code == 400 + + +def test_cross_origin_agent_mutations_are_rejected(tmp_path, monkeypatch): + monkeypatch.setitem(web.app.config, "AGENT_DB_PATH", str(tmp_path / "agent.db")) + monkeypatch.setattr(web, "AUTH_ENABLED", False) + response = web.app.test_client().post( + "/api/agent/youtube/disconnect", headers={"Origin": "https://other.example"}, + ) + assert response.status_code == 403 diff --git a/AutoMap-agent/web.py b/AutoMap-agent/web.py index f86aebe..7acabc9 100644 --- a/AutoMap-agent/web.py +++ b/AutoMap-agent/web.py @@ -2,10 +2,12 @@ from __future__ import annotations +import hashlib import hmac import json import os import secrets +import time from pathlib import Path from typing import Any from urllib.parse import urlsplit @@ -25,6 +27,7 @@ from agent.db import AgentDB from agent.media_pool import MAX_MEDIA_BYTES, MAX_MEDIA_MB, MediaPool, split_background_video from agent.optimizer import AgentOptimizer +from agent.private_files import session_secret, write_private_text from agent.telegram_bot import TelegramBot from agent.youtube_intel import AGENT_YOUTUBE_SCOPES @@ -59,19 +62,39 @@ def parse_oauth_client_config(raw_obj: object) -> dict[str, Any]: raise ValueError("Invalid client_secret.json format.") for section in ("web", "installed"): block = raw_obj.get(section) - if isinstance(block, dict) and block.get("client_id") and block.get("client_secret"): - return {section: block} + if not isinstance(block, dict): + continue + if not all(isinstance(block.get(key), str) and block[key] for key in ("client_id", "client_secret")): + continue + if (block.get("auth_uri") != "https://accounts.google.com/o/oauth2/auth" + or block.get("token_uri") != "https://oauth2.googleapis.com/token"): + raise ValueError("The client secret file must use the standard Google OAuth endpoints.") + return {section: block} raise ValueError("client_secret.json must contain a 'web' or 'installed' OAuth client.") app = Flask(__name__, template_folder=str(BASE_DIR / "templates")) -app.secret_key = (os.environ.get("AGENT_SECRET_KEY") or secrets.token_hex(32)).strip() +instance_dir = Path(os.environ["AGENT_DB_PATH"]).parent if os.environ.get("AGENT_DB_PATH") else Path(app.instance_path) +app.secret_key = session_secret(instance_dir / "agent-session-secret") app.config["MAX_CONTENT_LENGTH"] = 250 * 1024 * 1024 SERVER_MODE = os.environ.get("AGENT_SERVER", "FALSE").strip().upper() in ("1", "TRUE", "YES", "ON") AUTH_USER = os.environ.get("AGENT_WEB_USERNAME", "admin").strip() AUTH_PASS = os.environ.get("AGENT_WEB_PASSWORD", "").strip() -AUTH_ENABLED = SERVER_MODE and bool(AUTH_PASS) +if SERVER_MODE and (not AUTH_USER or not AUTH_PASS): + raise RuntimeError("Set AGENT_WEB_USERNAME and AGENT_WEB_PASSWORD before enabling AGENT_SERVER.") +AUTH_ENABLED = SERVER_MODE +AUTH_IDENTITY = hmac.new(app.secret_key.encode(), (AUTH_USER + "\0" + AUTH_PASS).encode(), hashlib.sha256).hexdigest() +app.config.update( + SESSION_COOKIE_NAME="automap_agent", + SESSION_COOKIE_HTTPONLY=True, + SESSION_COOKIE_SAMESITE="Lax", + SESSION_COOKIE_SECURE=SERVER_MODE, +) +if not SERVER_MODE: + allowed_hosts = os.environ.get("AGENT_TRUSTED_HOSTS", "") + app.config["TRUSTED_HOSTS"] = ["localhost", "127.0.0.1", "[::1]"] + app.config["TRUSTED_HOSTS"].extend(host.strip() for host in allowed_hosts.split(",") if host.strip()) def _get_db() -> AgentDB: @@ -111,11 +134,20 @@ def _public_automap_url(cfg: dict[str, Any]) -> str: @app.before_request def _require_auth(): - if not AUTH_ENABLED: - return None - if request.endpoint in ("login_page", "logout_page", "api_agent_youtube_callback", "healthz"): + if request.method not in ("GET", "HEAD", "OPTIONS"): + origin = request.headers.get("Origin") + try: + allowed_origin = not origin or urlsplit(origin).netloc == request.host + except ValueError: + allowed_origin = False + if not allowed_origin or request.headers.get("Sec-Fetch-Site") in ("cross-site", "same-site"): + return jsonify(status="error", error="Cross-origin requests are forbidden."), 403 + if request.path.startswith("/api/") and request.is_json: + if not isinstance(request.get_json(silent=True), dict): + return jsonify(status="error", error="Request body must be a JSON object."), 400 + if not AUTH_ENABLED or request.endpoint in ("login_page", "healthz"): return None - if session.get("agent_authed") is True: + if session.get("agent_identity") == AUTH_IDENTITY: return None if request.path.startswith("/api/"): return jsonify(status="error", error="Authentication required."), 401 @@ -135,8 +167,9 @@ def login_page(): if request.method == "POST": user = str(request.form.get("username") or "").strip() pwd = str(request.form.get("password") or "") - if hmac.compare_digest(user, AUTH_USER) and hmac.compare_digest(pwd, AUTH_PASS): - session["agent_authed"] = True + if hmac.compare_digest(user.encode(), AUTH_USER.encode()) and hmac.compare_digest(pwd.encode(), AUTH_PASS.encode()): + session.clear() + session["agent_identity"] = AUTH_IDENTITY return redirect(url_for("agent_home")) error = "Invalid username or password." return render_template("login.html", error=error) @@ -144,7 +177,7 @@ def login_page(): @app.post("/logout") def logout_page(): - session.pop("agent_authed", None) + session.clear() return redirect(url_for("login_page")) @@ -248,7 +281,9 @@ def api_agent_youtube_connect(): uploaded = request.files.get("client_secret") if uploaded is None: return jsonify(status="error", error="Choose a client_secret.json file first."), 400 - raw = uploaded.read(128 * 1024) + raw = uploaded.read(128 * 1024 + 1) + if len(raw) > 128 * 1024: + return jsonify(status="error", error="OAuth client file exceeds the size limit."), 400 try: cfg_obj = parse_oauth_client_config(json.loads(raw.decode("utf-8"))) except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc: @@ -256,13 +291,17 @@ def api_agent_youtube_connect(): redirect_uri = _oauth_redirect_uri() host = (urlsplit(redirect_uri).hostname or "").lower() - if "installed" in cfg_obj and host not in ("localhost", "127.0.0.1"): + if "installed" in cfg_obj and host not in ("localhost", "127.0.0.1", "::1"): return jsonify( status="error", error="Desktop app credentials require localhost. For a hosted site use a Web application client with the displayed Redirect URI.", ), 400 - flow = Flow.from_client_config(cfg_obj, scopes=list(AGENT_YOUTUBE_SCOPES), redirect_uri=redirect_uri) + if urlsplit(redirect_uri).scheme != "https" and host not in ("localhost", "127.0.0.1", "::1"): + return jsonify(status="error", error="Hosted Google authorization requires HTTPS."), 400 + flow = Flow.from_client_config( + cfg_obj, scopes=list(AGENT_YOUTUBE_SCOPES), redirect_uri=redirect_uri, autogenerate_code_verifier=True, + ) state = secrets.token_urlsafe(24) auth_url, _ = flow.authorization_url( access_type="offline", @@ -270,40 +309,49 @@ def api_agent_youtube_connect(): prompt="consent", state=state, ) - pending_path = _get_db().path.parent / "agent_oauth_pending.json" - pending_path.write_text( + old_state = session.get("agent_oauth_state") + if old_state: + _pending_oauth_path(old_state).unlink(missing_ok=True) + pending_path = _pending_oauth_path(state) + write_private_text( + pending_path, json.dumps( { "state": state, "config": cfg_obj, "redirect_uri": redirect_uri, - "code_verifier": getattr(flow, "code_verifier", None), + "code_verifier": flow.code_verifier, + "created": time.time(), } ), - encoding="utf-8", ) session["agent_oauth_state"] = state return jsonify(status="success", authorization_url=auth_url) +def _pending_oauth_path(state: str) -> Path: + key = hashlib.sha256(state.encode()).hexdigest() + return _get_db().path.parent / "agent_oauth_pending" / f"{key}.json" + + @app.get("/api/agent/youtube/callback") def api_agent_youtube_callback(): """Complete Google OAuth flow for the agent and persist its credentials token.""" from google_auth_oauthlib.flow import Flow - if request.args.get("error"): - return redirect("/?youtube=denied") - state = str(request.args.get("state") or "") - code = str(request.args.get("code") or "") - pending_path = _get_db().path.parent / "agent_oauth_pending.json" - if not code or not state or not pending_path.is_file(): + state = request.args.get("state", "") + expected = session.get("agent_oauth_state") + if not expected or not state or not hmac.compare_digest(state.encode(), expected.encode()): return redirect("/?youtube=failed") + session.pop("agent_oauth_state", None) + pending_path = _pending_oauth_path(expected) try: pending = json.loads(pending_path.read_text(encoding="utf-8")) - if pending.get("state") != state or ( - session.get("agent_oauth_state") and session.get("agent_oauth_state") != state - ): + pending_path.unlink() + if time.time() - pending["created"] > 600: return redirect("/?youtube=failed") + if request.args.get("error") or not request.args.get("code"): + return redirect("/?youtube=denied") flow = Flow.from_client_config( pending["config"], scopes=list(AGENT_YOUTUBE_SCOPES), @@ -312,12 +360,10 @@ def api_agent_youtube_callback(): ) if pending.get("code_verifier"): flow.code_verifier = pending["code_verifier"] - flow.fetch_token(code=code) - tok_path = _token_path() - tok_path.parent.mkdir(parents=True, exist_ok=True) - tok_path.write_text(flow.credentials.to_json(), encoding="utf-8") - pending_path.unlink(missing_ok=True) - session.pop("agent_oauth_state", None) + flow.fetch_token(code=request.args["code"]) + if not flow.credentials.refresh_token: + return redirect("/?youtube=failed") + write_private_text(_token_path(), flow.credentials.to_json()) return redirect("/?youtube=connected") except Exception: return redirect("/?youtube=failed") @@ -328,6 +374,9 @@ def api_agent_youtube_disconnect(): """Remove persisted agent YouTube OAuth token.""" tok_path = _token_path() tok_path.unlink(missing_ok=True) + state = session.pop("agent_oauth_state", None) + if state: + _pending_oauth_path(state).unlink(missing_ok=True) return jsonify(status="success", message="Agent YouTube OAuth disconnected.") @@ -567,4 +616,4 @@ def api_agent_video(exp_id: int): if __name__ == "__main__": port = int(os.environ.get("PORT", "4000")) - app.run(host="0.0.0.0", port=port, debug=False) + app.run(host=os.environ.get("AGENT_HOST", "127.0.0.1"), port=port, debug=False) diff --git a/Dockerfile.ytserver b/Dockerfile.ytserver index b54ab7c..cd95e5e 100644 --- a/Dockerfile.ytserver +++ b/Dockerfile.ytserver @@ -12,22 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +FROM denoland/deno:bin@sha256:b507583d3c68ba69f3ed2ce5969ae5b8b70f43421c781279c7aea06435263543 AS deno + FROM rust:1.83-slim AS rust-builder WORKDIR /build -RUN apt-get update && apt-get install -y python3 python3-pip && rm -rf /var/lib/apt/lists/* \ +RUN apt-get update && apt-get install -y --no-install-recommends python3 python3-pip && rm -rf /var/lib/apt/lists/* \ && pip3 install --no-cache-dir --break-system-packages "maturin>=1.7,<2.0" COPY automap_rs/ ./automap_rs/ -RUN cd automap_rs && maturin build --release --interpreter python3 --out /wheels +RUN cd automap_rs && maturin build --locked --release --interpreter python3 --out /wheels FROM python:3.12-slim -RUN apt-get update && apt-get install -y \ - ffmpeg \ - curl \ - unzip \ - && curl -fsSL https://deno.land/install.sh | DENO_INSTALL=/usr/local sh \ +RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg \ && rm -rf /var/lib/apt/lists/* +COPY --from=deno /deno /usr/local/bin/deno +ENV FFMPEG_BINARY=/usr/bin/ffmpeg WORKDIR /app @@ -39,7 +39,7 @@ COPY --from=rust-builder /wheels/*.whl /tmp/ RUN pip install --no-cache-dir /tmp/*.whl && rm -f /tmp/*.whl -COPY yt-dlp_serv.py sticker_themes.py extra_stickers.py server_runtime.py render_progress.py video_text.py video_stickers.py video_render.py ./ +COPY media_inputs.py yt-dlp_serv.py sticker_themes.py extra_stickers.py server_runtime.py render_progress.py video_text.py video_stickers.py video_render.py ./ COPY resources/fonts/ resources/fonts/ EXPOSE 8080 diff --git a/docker-compose.yml b/docker-compose.yml index 74ecdef..1c8c589 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,12 +19,15 @@ services: context: . dockerfile: Dockerfile.web ports: - - "3000:5000" + - "${AUTOMAP_BIND_ADDRESS:-127.0.0.1}:3000:5000" environment: PORT: "5000" + BIND_HOST: "0.0.0.0" SERVER: "${SERVER:-FALSE}" - WEB_LOGIN: "${WEB_LOGIN:-deltaaa}" - WEB_PASSWORD: "${WEB_PASSWORD:-deltaaa00!}" + WEB_LOGIN: "${WEB_LOGIN:-}" + WEB_PASSWORD: "${WEB_PASSWORD:-}" + WEB_TRUSTED_HOSTS: "${WEB_TRUSTED_HOSTS:-}" + WEB_COOKIE_SECURE: "${WEB_COOKIE_SECURE:-}" WEB_SECRET_KEY: "${WEB_SECRET_KEY:-}" GEMINI_API_KEY: "${GEMINI_API_KEY:-}" DEEPSEEK_API_KEY: "${DEEPSEEK_API_KEY:-}" @@ -37,7 +40,10 @@ services: volumes: - automap-session:/app/instance networks: - - app-network + app-network: + agent-link: + aliases: + - automap-web yt-dlp-worker: restart: unless-stopped @@ -46,6 +52,8 @@ services: dockerfile: Dockerfile.ytserver environment: - PORT=8080 + - BIND_HOST=0.0.0.0 + - AUTOMAP_STORAGE_DIR=/tmp/automap_storage - SERVER=${SERVER:-FALSE} volumes: - automap-media:/tmp/automap_storage @@ -53,6 +61,8 @@ services: - app-network networks: + agent-link: + name: automap-agent-link app-network: driver: bridge diff --git a/example.env b/example.env index a999fde..d73fe4a 100644 --- a/example.env +++ b/example.env @@ -20,8 +20,12 @@ DEEPSEEK_API_KEY=your_deepseek_api_key # BRAVE_SEARCH_API_KEY=your_brave_search_api_key # Local mode is the default. Set TRUE only in the server deployment .env. SERVER=FALSE -WEB_LOGIN=deltaaa -WEB_PASSWORD=deltaaa00! +# WEB_LOGIN=your-username +# WEB_PASSWORD=choose-a-unique-password +# Optional host binding for Docker. Loopback is the default. +# AUTOMAP_BIND_ADDRESS=127.0.0.1 +# Server mode uses Secure cookies by default. For trusted HTTP-only deployments: +# WEB_COOKIE_SECURE=FALSE # Optional stable session signing secret; otherwise generated in instance/. # WEB_SECRET_KEY=replace-with-a-long-random-secret diff --git a/main.py b/main.py index 765ee67..8aedf05 100644 --- a/main.py +++ b/main.py @@ -39,13 +39,6 @@ BASE_DIR = Path(__file__).parent os.chdir(BASE_DIR) -STORAGE_DIR = "/tmp/automap_storage" -DOWNLOADS = os.path.join(STORAGE_DIR, "downloads") -CUT_DIR = os.path.join(STORAGE_DIR, "cut") - -os.makedirs(STORAGE_DIR, exist_ok=True) -os.makedirs(CUT_DIR, exist_ok=True) - from dotenv import dotenv_values from map.auto_map import ( auto_map, @@ -65,6 +58,7 @@ ) from flask import Flask, request, jsonify, send_file, render_template, Response, session +from media_inputs import open_upload_image, youtube_video_url from web_ai_cache import ai_cache FONTS_DIR = BASE_DIR / "resources" / "fonts" @@ -445,9 +439,9 @@ def _inspect_uploaded_map_image( if not decoded: raise ValueError("Decoded image payload is empty.") try: - with Image.open(io.BytesIO(decoded)) as opened: + with open_upload_image(io.BytesIO(decoded)) as opened: img = opened.convert("RGBA") - except (UnidentifiedImageError, OSError, ValueError) as exc: + except (UnidentifiedImageError, OSError, Image.DecompressionBombError) as exc: raise ValueError("Uploaded file is not a valid image.") from exc width, height = img.size @@ -899,11 +893,9 @@ def download_batch_zip(): continue try: image_bytes = base64.b64decode(image_url.split(',', 1)[1], validate=True) - with Image.open(io.BytesIO(image_bytes)) as image: - if image.format != 'PNG': - continue + with open_upload_image(io.BytesIO(image_bytes), ('PNG',)) as image: image.verify() - except (ValueError, OSError): + except (ValueError, OSError, Image.DecompressionBombError): continue base = re.sub(r"[^\w\-]+", "_", prompt.strip().replace(" ", "_")).strip("_") if prompt else f"map_{idx:02d}" @@ -1288,11 +1280,14 @@ def music_download(): """ data = request.get_json(silent=True) or {} - url = (data.get('url') or '').strip() - if not url: - return jsonify({"status": "error", "error": "Missing 'url' in request body."}), 400 + try: + url = youtube_video_url(data.get('url')) + except ValueError as error: + return jsonify(status='error', error=str(error)), 400 - cookies = str(data.get('cookies') or '').strip('\r\n') + cookies = data.get('cookies') or '' + if not isinstance(cookies, str) or len(cookies.encode('utf-8')) > 256 * 1024: + return jsonify(status='error', error='Invalid YouTube cookies file.'), 400 forward_payload = {"url": url, "cookies": cookies} if cookies else {"url": url} # Another service downloads music from YouTube and converts it to mp3. @@ -1457,7 +1452,7 @@ def _region_mask_from_index( pix_mask[flat[ids == region_idx]] = 255 if not pix_mask.any(): return None - return Image.fromarray(pix_mask.reshape((h, w)), mode="L") + return Image.fromarray(pix_mask.reshape((h, w))) def _world_country_floodfill_mask(seeds: list[tuple[int, int]]) -> Image.Image | None: @@ -1498,7 +1493,7 @@ def _world_country_floodfill_mask(seeds: list[tuple[int, int]]) -> Image.Image | break if filled is None or not filled.any(): return None - return Image.fromarray((filled.astype(np.uint8) * 255), mode="L") + return Image.fromarray(filled.astype(np.uint8) * 255) def _extract_country_mask_image( @@ -1605,6 +1600,8 @@ def _extract_country_reveal_mask( mask.save(buf, format="PNG") b64_mask = base64.b64encode(buf.getvalue()).decode("ascii") with _ZOOM_CACHE_LOCK: + if cache_key not in _REVEAL_MASK_CACHE and len(_REVEAL_MASK_CACHE) >= 64: + _REVEAL_MASK_CACHE.pop(next(iter(_REVEAL_MASK_CACHE))) _REVEAL_MASK_CACHE[cache_key] = b64_mask return b64_mask @@ -1637,7 +1634,7 @@ def _decode_image_size(image_data: object) -> tuple[int, int] | None: if not decoded: return None try: - with Image.open(io.BytesIO(decoded)) as opened: + with open_upload_image(io.BytesIO(decoded)) as opened: return opened.size except (UnidentifiedImageError, OSError, Image.DecompressionBombError, ValueError): return None @@ -1788,6 +1785,7 @@ def make_video() -> tuple[object, int]: "title_other_lines_color": data.get('title_other_lines_color', '#ffe735'), "title_uppercase": data.get('title_uppercase', True), "visual_effects": data.get('visual_effects', True), + **({"draft": data["draft"]} if "draft" in data else {}), "watermark_image": data.get('watermark_image', ''), "watermark_text": data.get('watermark_text', ''), "watermark_color": data.get('watermark_color', '#a4e9ee'), diff --git a/map/auto_map.py b/map/auto_map.py index 8b8e279..91caee1 100644 --- a/map/auto_map.py +++ b/map/auto_map.py @@ -765,7 +765,7 @@ def _build_extrema_highlight_layer( pix_mask = np.zeros(w * h, dtype=np.uint8) pix_mask[flat[ids == region_idx]] = 255 if pix_mask.any(): - mask = Image.fromarray(pix_mask.reshape((h, w)), mode="L") + mask = Image.fromarray(pix_mask.reshape((h, w))) elif wc_interior_crisp is not None and 1 <= region_idx <= len(coords_list): max_px = int(0.08 * w * h) filled = _floodfill_bool_mask(wc_interior_crisp, coords_list[region_idx - 1], max_px) @@ -776,7 +776,7 @@ def _build_extrema_highlight_layer( if filled is not None and filled.any(): break if filled is not None and filled.any(): - mask = Image.fromarray((filled.astype(np.uint8) * 255), mode="L") + mask = Image.fromarray(filled.astype(np.uint8) * 255) if mask is None: continue diff --git a/media_inputs.py b/media_inputs.py new file mode 100644 index 0000000..bd2e45e --- /dev/null +++ b/media_inputs.py @@ -0,0 +1,50 @@ +"""Limits for untrusted images and YouTube audio URLs.""" + +import os +import re +from pathlib import Path +from urllib.parse import parse_qs, urlsplit + +from PIL import Image + + +MEDIA_STORAGE_DIR = os.environ.get("AUTOMAP_STORAGE_DIR") or str(Path(__file__).resolve().parent / "instance" / "media") +IMAGE_FORMATS = ('PNG', 'JPEG', 'WEBP', 'GIF', 'BMP') +MAX_IMAGE_PIXELS = 20_000_000 +_YOUTUBE_HOSTS = {'youtube.com', 'www.youtube.com', 'm.youtube.com', 'music.youtube.com'} +_VIDEO_ID = re.compile(r'[A-Za-z0-9_-]{11}') + + +def open_upload_image(stream, formats=IMAGE_FORMATS): + """Read an image header with an explicit format and pixel limit.""" + image = Image.open(stream, formats=formats) + if image.width * image.height > MAX_IMAGE_PIXELS: + image.close() + raise ValueError('Image must be at most 20 megapixels.') + return image + + +def youtube_video_url(value): + """Return a canonical, single-video HTTPS URL for the YouTube extractor.""" + if not isinstance(value, str) or len(value) > 2048: + raise ValueError('Enter a YouTube video URL.') + try: + url = urlsplit(value.strip()) + if (url.scheme not in ('https', 'http') or url.username is not None + or url.password is not None or url.port is not None): + raise ValueError + host = url.hostname + parts = url.path.strip('/').split('/') + if host == 'youtu.be' and len(parts) == 1: + video_id = parts[0] + elif host in _YOUTUBE_HOSTS and url.path == '/watch': + video_id, = parse_qs(url.query, max_num_fields=32).get('v', []) + elif host in _YOUTUBE_HOSTS and len(parts) == 2 and parts[0] in ('shorts', 'live', 'embed'): + video_id = parts[1] + else: + raise ValueError + if not _VIDEO_ID.fullmatch(video_id): + raise ValueError + except ValueError: + raise ValueError('Enter a YouTube video URL.') from None + return f'https://www.youtube.com/watch?v={video_id}' diff --git a/overview.md b/overview.md index 9bd2601..a41d0b3 100644 --- a/overview.md +++ b/overview.md @@ -95,7 +95,7 @@ working key; media operations require additional tools and integration checks. - The worker uses yt-dlp, ffmpeg, pydub, and MoviePy. Its Dockerfile installs ffmpeg, curl, and Node.js. Software rendering defaults to `libx264`; hardware codecs require compatible host support. -- Persistent media paths are under `/tmp/automap_storage/{downloads,cut,render}`. +- The worker stores media under `instance/media` for native runs and `/tmp/automap_storage` in Compose. Downloads use a separate temporary directory per request before storing MP3s. Music-download requests trigger cleanup of files older than 30 minutes. - Cutting accepts `MM:SS` and produces an eight-second clip. Track titles are diff --git a/pyproject.toml b/pyproject.toml index 12857e5..bada915 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ build-backend = "setuptools.build_meta" name = "AutoMap" version = "0.1.0" dependencies = [ - "pillow>=9.2.0,<12.0.0", + "pillow>=13.0.0,<14.0.0", "click>=8.0.0", "google-genai>=1.56.0", "cryptography>=43.0.0", @@ -31,10 +31,10 @@ dependencies = [ "rich>=13.0.0", "flask>=3.1.0", "waitress>=3.0.2,<4.0.0", - "yt-dlp>=2024.12.0", + "yt-dlp[default]>=2026.10.26", "requests>=2.31.0", "pydub>=0.25.1", - "moviepy>=2.0.0", + "moviepy @ https://github.com/Zulko/moviepy/archive/211e4b15f6ce4f34a6a9efbfff40590e43a68f77.tar.gz#sha256=8a4e08332e9ad5080ffca374a6f7081cfa65189acf5a1f93a65bb2fb227bab8a", "proglog>=0.1.10,<1.0.0", ] @@ -43,10 +43,12 @@ automap = "cli:main" [tool.setuptools] py-modules = ["cli"] -packages = ["map"] +packages = ["map", "resources"] +include-package-data = false [tool.setuptools.package-data] -map = ["filling.json", "filling.template.json"] +map = ["filling.template.json"] +resources = ["fonts/*.ttf", "maps/*.png"] [tool.pytest.ini_options] pythonpath = "." diff --git a/readme.md b/readme.md index b530f9c..77f5383 100644 --- a/readme.md +++ b/readme.md @@ -73,7 +73,7 @@ WEB_PASSWORD=your_password docker compose up --build -d ``` -Open [http://localhost:3000](http://localhost:3000). With `SERVER=TRUE`, sign in using the credentials from `.env`. +Open [http://localhost:3000](http://localhost:3000). For local HTTP testing with `SERVER=TRUE`, set `WEB_COOKIE_SECURE=FALSE` if your browser requires it. For an HTTPS deployment, keep the default Secure cookie setting. Useful commands: @@ -83,7 +83,7 @@ docker compose up --build -d # Apply code, dependency, or configuration chang docker compose down # Stop and remove containers; retain named volumes. ``` -Compose publishes only the web application (`3000` on the host to `5000` in the container). The media worker listens on `8080` inside the private Compose network. Both services use `restart: unless-stopped`. +Compose binds the web application to `127.0.0.1:3000` by default. Set `AUTOMAP_BIND_ADDRESS` if your reverse proxy or deployment needs a different interface. The media worker listens on `8080` inside the private Compose network. Both services use `restart: unless-stopped`. The `automap-session` volume retains the generated session secret. The `automap-media` volume stores worker media, which is still subject to expiry cleanup. Removing a volume removes its contents. @@ -194,7 +194,7 @@ minute while it is open. **Clear History** removes the saved history as well. 1. Generate a map or select a map image in Video Studio. 2. Supply background footage and a headline. -3. Optionally download a music track from a supported URL or upload MP3, WAV, M4A, AAC, OGG, or FLAC audio. Uploaded music is converted to MP3. +3. Optionally download audio from a YouTube video link or upload MP3, WAV, M4A, AAC, OGG, or FLAC audio. Uploaded music is converted to MP3. 4. Use a full track or trim a **1–30 second** segment from an `MM:SS` start time. 5. Configure title font, size and colors, watermarks, stickers, and other overlays, then render and download the MP4. @@ -350,8 +350,12 @@ if png is not None: | `GEMINI_API_KEY` | Server key for Gemini/Gemma models | | `DEEPSEEK_API_KEY` | Server key for DeepSeek models | | `SERVER` | Defaults to local mode (`FALSE`). Exact `TRUE` enables Waitress and web authentication | -| `WEB_LOGIN` | Login name; default `deltaaa` | -| `WEB_PASSWORD` | Login password; default `deltaaa00!` | +| `WEB_LOGIN` | Login name, required in server mode | +| `WEB_PASSWORD` | Unique login password, required in server mode | +| `WEB_COOKIE_SECURE` | Defaults to `TRUE` in server mode. Set `FALSE` only for a trusted HTTP-only setup | +| `WEB_TRUSTED_HOSTS` | Additional comma-separated hostnames accepted in unauthenticated local mode | +| `BIND_HOST` | Native listening address, default `127.0.0.1` | +| `AUTOMAP_BIND_ADDRESS` | Compose host listening address, default `127.0.0.1` | | `WEB_SECRET_KEY` | Optional stable session-signing secret | | `WEB_AUTH_SECRET_FILE` | Native deployment override for the generated secret file | | `PORT` | Native service port; web default `5000`, worker default `8080` | @@ -365,7 +369,7 @@ The server keeps `SERVER=TRUE` and its public `YOUTUBE_REDIRECT_URI` in its own Login sessions are valid for up to a year. Changing the credentials or signing secret invalidates existing sessions. Without `WEB_SECRET_KEY`, a secret is generated in `instance/web-session-secret`; keep that file across native restarts or retain the Compose session volume. For Compose, add an explicit environment entry to use `WEB_AUTH_SECRET_FILE`. Compose already fixes `YTDLP_URL` to the private worker address; edit that entry only if you change the service layout. -Use your own credentials and HTTPS for a public deployment. Keep the media worker private; the web application provides the login gate. After changing Compose environment values, run `docker compose up -d` to recreate affected containers: `docker compose restart` retains the old values. +Use your own credentials and HTTPS for a public deployment. If you used the old built-in password, choose a new one before upgrading. Keep the media worker private; the web application provides the login gate. After changing Compose environment values, run `docker compose up -d` to recreate affected containers: `docker compose restart` retains the old values. ## Tests diff --git a/requirements.txt b/requirements.txt index 1219c23..4692022 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ # limitations under the License. # Runtime dependencies for the CLI, web app, and media worker. -pillow>=9.2.0,<12.0.0 +pillow>=13.0.0,<14.0.0 click>=8.0.0 google-genai>=1.56.0 python-dotenv>=1.0.0 @@ -21,9 +21,10 @@ flask>=3.1.0 waitress>=3.0.2,<4.0.0 requests>=2.31.0 rich>=13.0.0 -yt-dlp>=2024.12.0 +yt-dlp[default]>=2026.10.26 pydub>=0.25.1 -moviepy>=2.0.0 +# Released MoviePy 2.2.1 caps Pillow below the security fixes. +moviepy @ https://github.com/Zulko/moviepy/archive/211e4b15f6ce4f34a6a9efbfff40590e43a68f77.tar.gz#sha256=8a4e08332e9ad5080ffca374a6f7081cfa65189acf5a1f93a65bb2fb227bab8a proglog>=0.1.10,<1.0.0 # YouTube Data API uploads and OAuth authorization. diff --git a/server_runtime.py b/server_runtime.py index d6632fb..b9c694d 100644 --- a/server_runtime.py +++ b/server_runtime.py @@ -28,11 +28,13 @@ def setting(name, default): return os.environ.get(name, file_settings.get(name) or default) port = int(setting("PORT", str(default_port))) + host = setting("BIND_HOST", "127.0.0.1") production = setting("SERVER", "FALSE") == "TRUE" if production: from waitress import serve app.debug = False - serve(app, host="0.0.0.0", port=port, threads=8, channel_timeout=600) + serve(app, host=host, port=port, threads=8, channel_timeout=600, + max_request_body_size=app.config.get("MAX_CONTENT_LENGTH") or 64 * 1024 * 1024) else: - app.run(host="0.0.0.0", port=port, debug=True) + app.run(host=host, port=port, debug=False) diff --git a/tests/browser/test_cookie_file_upload.py b/tests/browser/test_cookie_file_upload.py index 002cea0..adf3f64 100644 --- a/tests/browser/test_cookie_file_upload.py +++ b/tests/browser/test_cookie_file_upload.py @@ -15,6 +15,7 @@ """Cookies remain parseable across file import, browser storage and both APIs.""" from http.cookiejar import MozillaCookieJar from pathlib import Path +from subprocess import CompletedProcess import pytest @@ -45,10 +46,10 @@ def download(command, **kwargs): received.append({cookie.name: cookie.value for cookie in jar}) output = command[command.index('-o') + 1].replace('%(ext)s', 'mp3') Path(output).write_bytes(b'test-mp3') + return CompletedProcess(command, 0, stdout=b'"Test song"') monkeypatch.setattr(worker.subprocess, 'run', download) - monkeypatch.setattr(worker, '_get_song_title', lambda url: 'Test song') - page.locator('#musicUrlInput').fill('https://www.youtube.com/watch?v=test') + page.locator('#musicUrlInput').fill('https://www.youtube.com/watch?v=videoId0001') page.evaluate('handleMusicDownload({preventDefault(){}})') page.wait_for_function("!document.getElementById('musicBtn').disabled") assert received == [{'SID': 'fake-test-value', 'EMPTY': ''}] diff --git a/tests/conftest.py b/tests/conftest.py index bceac21..b3a81b6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,8 +32,8 @@ class AuthenticatedWebClient(FlaskClient): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) response = self.post('/login', data={ - 'login': os.environ.get('WEB_LOGIN', 'deltaaa'), - 'password': os.environ.get('WEB_PASSWORD', 'deltaaa00!'), + 'login': os.environ.get('WEB_LOGIN', 'test-admin'), + 'password': os.environ.get('WEB_PASSWORD', 'test-only-password'), }) assert response.status_code == 302 @@ -77,6 +77,9 @@ def web(monkeypatch, tmp_path): monkeypatch.setattr(dotenv, 'dotenv_values', lambda *args, **kwargs: {}) monkeypatch.delenv('GEMINI_API_KEY', raising=False) monkeypatch.setenv('SERVER', 'TRUE') + monkeypatch.setenv('WEB_LOGIN', 'test-admin') + monkeypatch.setenv('WEB_PASSWORD', 'test-only-password') + monkeypatch.setenv('WEB_COOKIE_SECURE', 'FALSE') monkeypatch.setenv('WEB_AUTH_SECRET_FILE', str(tmp_path / 'session-secret')) monkeypatch.chdir(ROOT) spec = importlib.util.spec_from_file_location('web_app', ROOT / 'main.py') diff --git a/tests/frontend/topic-preview.test.cjs b/tests/frontend/topic-preview.test.cjs index 958e450..6f74d80 100644 --- a/tests/frontend/topic-preview.test.cjs +++ b/tests/frontend/topic-preview.test.cjs @@ -10,6 +10,9 @@ test('typing coalesces preview work, composition waits, blur flushes the final d setTimeout(fn) { callback = fn; return 1; }, clearTimeout() { callback = undefined; }, updateCommandPreview() { updates++; }, + saveBrowserValue() {}, + MAP_TOPIC_STORAGE_KEY: 'topic', + document: {getElementById() { return {value: 'Topic'}; }}, }); vm.runInContext(html.slice(html.indexOf(' let topicPreviewTimer;'), html.indexOf(' // Flush a pending draft')), context); diff --git a/tests/test_audio_upload.py b/tests/test_audio_upload.py index 1e26dc1..6630168 100644 --- a/tests/test_audio_upload.py +++ b/tests/test_audio_upload.py @@ -105,7 +105,7 @@ def test_converted_audio_limit_and_subprocess_timeout(worker, monkeypatch): monkeypatch.setattr(worker, 'MAX_MATERIAL_BYTES', 8) def conversion(command, **kwargs): assert kwargs['timeout'] == 30 - assert command[command.index('-protocol_whitelist') + 1] == 'file,pipe' + assert command[command.index('-protocol_whitelist') + 1] == 'file' Path(command[-1]).write_bytes(b'x' * 9) return Mock(returncode=0) monkeypatch.setattr(worker.subprocess, 'run', conversion) diff --git a/tests/test_server_runtime.py b/tests/test_server_runtime.py index 36e4e0e..54a0511 100644 --- a/tests/test_server_runtime.py +++ b/tests/test_server_runtime.py @@ -30,7 +30,7 @@ def runtime(monkeypatch): monkeypatch.delenv("PORT", raising=False) serve = Mock() monkeypatch.setitem(sys.modules, "waitress", SimpleNamespace(serve=serve)) - return SimpleNamespace(debug=True, run=Mock()), serve + return SimpleNamespace(debug=True, config={"MAX_CONTENT_LENGTH": 1024}, run=Mock()), serve def test_production_uses_waitress(runtime, monkeypatch): @@ -40,13 +40,13 @@ def test_production_uses_waitress(runtime, monkeypatch): server_runtime.run_app(app) assert app.debug is False app.run.assert_not_called() - serve.assert_called_once_with(app, host="0.0.0.0", port=5051, threads=8, channel_timeout=600) + serve.assert_called_once_with(app, host="127.0.0.1", port=5051, threads=8, channel_timeout=600, max_request_body_size=1024) def test_development_uses_flask(runtime): app, serve = runtime server_runtime.run_app(app) - app.run.assert_called_once_with(host="0.0.0.0", port=5000, debug=True) + app.run.assert_called_once_with(host="127.0.0.1", port=5000, debug=False) serve.assert_not_called() @@ -55,7 +55,7 @@ def test_only_exact_true_enables_production(runtime, monkeypatch, value): app, serve = runtime monkeypatch.setenv("SERVER", value) server_runtime.run_app(app) - app.run.assert_called_once_with(host="0.0.0.0", port=5000, debug=True) + app.run.assert_called_once_with(host="127.0.0.1", port=5000, debug=False) serve.assert_not_called() @@ -73,5 +73,11 @@ def test_environment_overrides_dotenv(runtime, monkeypatch): monkeypatch.setenv("SERVER", "FALSE") monkeypatch.setenv("PORT", "5001") server_runtime.run_app(app) - app.run.assert_called_once_with(host="0.0.0.0", port=5001, debug=True) + app.run.assert_called_once_with(host="127.0.0.1", port=5001, debug=False) serve.assert_not_called() + +def test_bind_host_can_be_configured(runtime, monkeypatch): + app, _serve = runtime + monkeypatch.setenv("BIND_HOST", "0.0.0.0") + server_runtime.run_app(app) + app.run.assert_called_once_with(host="0.0.0.0", port=5000, debug=False) diff --git a/tests/test_web.py b/tests/test_web.py index 4b90077..354b27a 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -107,7 +107,7 @@ def test_generate_server_key_does_not_require_form_key(web, monkeypatch, generat PROXIES = [ - ('/api/music_download', {'url': 'https://example.test/song'}, 150), + ('/api/music_download', {'url': 'https://www.youtube.com/watch?v=videoId0001'}, 150), ('/api/music_cut', {'title': 'song', 'start_time': '00:01'}, 40), ('/api/render_video', {'map_file': 'map', 'background_video': 'bg', 'video_title': 'title'}, 300), ] diff --git a/tests/test_web_auth.py b/tests/test_web_auth.py index c067ce7..8de6e79 100644 --- a/tests/test_web_auth.py +++ b/tests/test_web_auth.py @@ -24,8 +24,10 @@ @pytest.fixture def auth_app(tmp_path, monkeypatch): monkeypatch.setenv('SERVER', 'TRUE') - for name in ('WEB_LOGIN', 'WEB_PASSWORD', 'WEB_SECRET_KEY'): - monkeypatch.delenv(name, raising=False) + monkeypatch.setenv('WEB_LOGIN', 'test-admin') + monkeypatch.setenv('WEB_PASSWORD', 'test-only-password') + monkeypatch.delenv('WEB_SECRET_KEY', raising=False) + monkeypatch.delenv('WEB_COOKIE_SECURE', raising=False) monkeypatch.setenv('WEB_AUTH_SECRET_FILE', str(tmp_path / 'secret')) def create(settings=None): @@ -61,18 +63,19 @@ def test_unauthenticated_pages_and_api(auth_app): assert page.status_code == 200 assert b'' in page.data assert b'Sign in' in page.data and b'Username' in page.data and b'Password' in page.data - failure = client.post('/login', data={'login': 'deltaaa', 'password': 'wrong'}) + failure = client.post('/login', data={'login': 'test-admin', 'password': 'wrong'}) assert failure.status_code == 401 assert b'Invalid username or password.' in failure.data assert client.get('/api/status').status_code == 401 -def test_default_login_cookie_survives_new_client_and_restart(auth_app): +def test_login_cookie_survives_new_client_and_restart(auth_app): first = auth_app().test_client() - response = first.post('/login', data={'login': 'deltaaa', 'password': 'deltaaa00!', 'next': '/studio'}) + response = first.post('/login', data={'login': 'test-admin', 'password': 'test-only-password', 'next': '/studio'}) assert response.location == '/studio' cookie_header = response.headers['Set-Cookie'] assert 'Expires=' in cookie_header and 'HttpOnly' in cookie_header and 'SameSite=Lax' in cookie_header + assert 'Secure' in cookie_header cookie = first.get_cookie('session') restarted = auth_app().test_client() restarted.set_cookie('session', cookie.value) @@ -88,14 +91,14 @@ def test_default_login_cookie_survives_new_client_and_restart(auth_app): def test_environment_credentials_and_rotation(auth_app, monkeypatch): first = auth_app().test_client() - first.post('/login', data={'login': 'deltaaa', 'password': 'deltaaa00!'}) + first.post('/login', data={'login': 'test-admin', 'password': 'test-only-password'}) old_cookie = first.get_cookie('session').value monkeypatch.setenv('WEB_LOGIN', 'owner') monkeypatch.setenv('WEB_PASSWORD', 'новий-secret') client = auth_app().test_client() client.set_cookie('session', old_cookie) assert client.get('/api/status').status_code == 401 - assert client.post('/login', data={'login': 'deltaaa', 'password': 'deltaaa00!'}).status_code == 401 + assert client.post('/login', data={'login': 'test-admin', 'password': 'test-only-password'}).status_code == 401 assert client.post('/login', data={'login': 'owner', 'password': 'новий-secret'}).status_code == 302 assert client.get('/api/status').status_code == 200 @@ -103,14 +106,21 @@ def test_environment_credentials_and_rotation(auth_app, monkeypatch): @pytest.mark.parametrize('target', ['https://evil.example', '//evil.example', '/\\evil.example']) def test_login_rejects_external_redirects(auth_app, target): response = auth_app().test_client().post('/login', data={ - 'login': 'deltaaa', 'password': 'deltaaa00!', 'next': target, + 'login': 'test-admin', 'password': 'test-only-password', 'next': target, }) assert response.location == '/' def test_dotenv_credentials_and_env_precedence(auth_app, monkeypatch): monkeypatch.setenv('WEB_LOGIN', 'environment-user') + monkeypatch.delenv('WEB_PASSWORD') client = auth_app({'WEB_LOGIN': 'dotenv-user', 'WEB_PASSWORD': 'dotenv-password'}).test_client() assert client.post('/login', data={ 'login': 'environment-user', 'password': 'dotenv-password', }).status_code == 302 + +@pytest.mark.parametrize('password', ['', 'deltaaa00!']) +def test_server_rejects_missing_and_legacy_passwords(auth_app, monkeypatch, password): + monkeypatch.setenv('WEB_PASSWORD', password) + with pytest.raises(RuntimeError, match='unique WEB_PASSWORD'): + auth_app() diff --git a/tests/test_web_auth_security.py b/tests/test_web_auth_security.py index 48387aa..97ad5c4 100644 --- a/tests/test_web_auth_security.py +++ b/tests/test_web_auth_security.py @@ -18,7 +18,7 @@ def test_unicode_logout_token_is_rejected(auth_app): client = auth_app().test_client() - client.post('/login', data={'login': 'deltaaa', 'password': 'deltaaa00!'}) + client.post('/login', data={'login': 'test-admin', 'password': 'test-only-password'}) assert client.post('/logout', data={'csrf_token': 'невірний-токен'}).status_code == 400 assert client.get('/api/status').status_code == 200 @@ -26,7 +26,25 @@ def test_unicode_logout_token_is_rejected(auth_app): def test_control_characters_in_next_do_not_crash_login(auth_app): client = auth_app().test_client() response = client.post('/login', data={ - 'login': 'deltaaa', 'password': 'deltaaa00!', 'next': '/\t/[invalid-ipv6', + 'login': 'test-admin', 'password': 'test-only-password', 'next': '/\t/[invalid-ipv6', }) assert response.status_code == 302 assert response.location == '/' + + +def test_cross_origin_login_does_not_set_session(auth_app): + client = auth_app().test_client() + credentials = {'login': 'test-admin', 'password': 'test-only-password'} + response = client.post('/login', data=credentials, headers={'Origin': 'https://other.example'}) + assert response.status_code == 403 + assert client.get('/api/status').status_code == 401 + response = client.post('/login', data=credentials, headers={'Origin': 'http://localhost'}) + assert response.status_code == 302 + assert client.get('/api/status').status_code == 200 + + +def test_local_mode_accepts_only_configured_hosts(auth_app, monkeypatch): + monkeypatch.setenv('SERVER', 'FALSE') + client = auth_app().test_client() + assert client.get('http://localhost/').status_code == 200 + assert client.get('http://external.example/').status_code == 400 diff --git a/tests/test_worker.py b/tests/test_worker.py index 46b3ed3..124ef91 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -13,27 +13,21 @@ # limitations under the License. import base64 +import json +import io import os from pathlib import Path import subprocess from unittest.mock import MagicMock, Mock import pytest -import requests +from PIL import Image -@pytest.mark.parametrize('html,expected', [ - ('Other', 'My song'), - ('Fallback song', 'Fallback song'), ('', None), -]) -def test_song_title_metadata(worker, monkeypatch, html, expected): - monkeypatch.setattr(requests, 'get', Mock(return_value=Mock(text=html))) - assert worker._get_song_title('https://example.test') == expected - - -def test_song_title_network_error(worker, monkeypatch): - monkeypatch.setattr(requests, 'get', Mock(side_effect=requests.Timeout())) - assert worker._get_song_title('https://example.test') is None +def sample_map(): + output = io.BytesIO() + Image.new('RGBA', (8, 8)).save(output, format='PNG') + return base64.b64encode(output.getvalue()).decode() def test_cleanup_removes_only_expired_files(worker, monkeypatch): @@ -60,11 +54,14 @@ def test_download_stores_mp3_and_cleans_temporary_files(worker, monkeypatch, tmp download = tmp_path / 'temporary-download' download.mkdir() monkeypatch.setattr(worker.tempfile, 'mkdtemp', lambda **kwargs: str(download)) - run = Mock(side_effect=lambda *args, **kwargs: (download / 'track.mp3').write_bytes(b'mp3-data')) + def run_download(*args, **kwargs): + (download / 'track.mp3').write_bytes(b'mp3-data') + return Mock(stdout=json.dumps(title).encode()) + + run = Mock(side_effect=run_download) monkeypatch.setattr(worker.subprocess, 'run', run) - monkeypatch.setattr(worker, '_get_song_title', lambda url: title) monkeypatch.setattr(worker.uuid, 'uuid4', lambda: Mock(hex='0123456789abcdef')) - response = worker.app.test_client().post('/api/music_download/', json={'url': ' https://example.test/song '}) + response = worker.app.test_client().post('/api/music_download/', json={'url': ' https://www.youtube.com/watch?v=videoId0001 '}) assert response.status_code == 200, response.json expected_title = ('A_nice___song_' if title else 'downloaded_audio') + '_0123456789ab' assert response.json['title'] == expected_title @@ -73,7 +70,7 @@ def test_download_stores_mp3_and_cleans_temporary_files(worker, monkeypatch, tmp assert not download.exists() command = run.call_args.args[0] assert command[0] == 'yt-dlp' and '--no-playlist' in command - assert command[-1] == 'https://example.test/song' + assert command[-1] == 'https://www.youtube.com/watch?v=videoId0001' @pytest.mark.parametrize('failure', [ @@ -85,7 +82,7 @@ def test_download_errors_cleanup(worker, monkeypatch, tmp_path, failure): folder.mkdir() monkeypatch.setattr(worker.tempfile, 'mkdtemp', lambda **kwargs: str(folder)) monkeypatch.setattr(worker.subprocess, 'run', Mock(side_effect=failure)) - response = worker.app.test_client().post('/api/music_download', json={'url': 'https://example.test'}) + response = worker.app.test_client().post('/api/music_download', json={'url': 'https://www.youtube.com/watch?v=videoId0001'}) assert response.status_code == 500 and response.json['status'] == 'error' assert not folder.exists() @@ -102,7 +99,7 @@ def test_download_requires_url(worker, payload): def test_download_explains_youtube_authentication_failure(worker, monkeypatch, stderr, expected): failure = subprocess.CalledProcessError(1, 'yt-dlp', stderr=stderr) monkeypatch.setattr(worker.subprocess, 'run', Mock(side_effect=failure)) - response = worker.app.test_client().post('/api/music_download', json={'url': 'https://www.youtube.com/watch?v=test'}) + response = worker.app.test_client().post('/api/music_download', json={'url': 'https://www.youtube.com/watch?v=videoId0001'}) assert response.status_code == 500 assert expected in response.json['error'] @@ -113,11 +110,13 @@ def test_download_moves_audio_across_filesystems(worker, monkeypatch, tmp_path): download = tmp_path / 'temporary-download' download.mkdir() monkeypatch.setattr(worker.tempfile, 'mkdtemp', lambda **kwargs: str(download)) - monkeypatch.setattr(worker.subprocess, 'run', Mock( - side_effect=lambda *args, **kwargs: (download / 'track.mp3').write_bytes(b'audio-data'))) - monkeypatch.setattr(worker, '_get_song_title', lambda url: 'Selected_song') + def run_download(*args, **kwargs): + (download / 'track.mp3').write_bytes(b'audio-data') + return Mock(stdout=b'"Selected_song"') + + monkeypatch.setattr(worker.subprocess, 'run', Mock(side_effect=run_download)) monkeypatch.setattr(worker.os, 'rename', Mock(side_effect=OSError(errno.EXDEV, 'Invalid cross-device link'))) - response = worker.app.test_client().post('/api/music_download', json={'url': 'https://example.test/song'}) + response = worker.app.test_client().post('/api/music_download', json={'url': 'https://www.youtube.com/watch?v=videoId0001'}) assert response.status_code == 200, response.json assert response.json['title'].startswith('Selected_song_') assert (Path(worker.DOWNLOADS) / (response.json['title'] + '.mp3')).read_bytes() == b'audio-data' @@ -173,9 +172,9 @@ def test_render_validation(worker, changes): @pytest.mark.parametrize('field', ['map_file', 'background_video', 'watermark_image']) def test_render_material_limit(worker, monkeypatch, field): - monkeypatch.setattr(worker, 'MAX_MATERIAL_BYTES', 3) - data = dict(map_file='eA==', background_video='eA==', video_title='test') - data[field] = base64.b64encode(b'1234').decode() + monkeypatch.setattr(worker, 'MAX_MATERIAL_BYTES', 256) + data = dict(map_file=sample_map(), background_video='eA==', video_title='test') + data[field] = base64.b64encode(b'x' * 257).decode() response = worker.app.test_client().post('/api/render_video', json=data) assert response.status_code == 400 and '15 MB' in response.json['error'] @@ -188,7 +187,7 @@ def test_render_material_limit(worker, monkeypatch, field): {'watermark_image': base64.b64encode(b'not an image').decode()}, ]) def test_invalid_watermark_rejected_before_render(worker, settings): - data = dict(map_file='eA==', background_video='eA==', video_title='test', **settings) + data = dict(map_file=sample_map(), background_video='eA==', video_title='test', **settings) response = worker.app.test_client().post('/api/render_video', json=data) assert response.status_code == 400, response.json @@ -202,10 +201,10 @@ def temporary(**kwargs): return file monkeypatch.setattr(worker.tempfile, 'NamedTemporaryFile', temporary) response = worker.app.test_client().post('/api/render_video', json={ - 'map_file': 'data:image/png;base64,eA==', 'background_video': 'eA==', 'video_title': 'test', + 'map_file': sample_map(), 'background_video': 'eA==', 'video_title': 'test', }) assert response.status_code == 500 - assert len(created) == 2 and all(not Path(path).exists() for path in created) + assert len(created) == 3 and all(not Path(path).exists() for path in created) @pytest.mark.parametrize('title', ['', 'a' * 1001]) @@ -288,7 +287,7 @@ def test_cut_rejects_start_beyond_track(worker, monkeypatch): def test_render_requires_audio_when_music_is_enabled(worker): response = worker.app.test_client().post('/api/render_video', json={ - 'map_file': 'eA==', 'background_video': 'eA==', 'video_title': 'Headline', + 'map_file': sample_map(), 'background_video': 'eA==', 'video_title': 'Headline', 'audio_title': '', 'include_audio': True, }) assert response.status_code == 400 diff --git a/tests/test_worker_audit.py b/tests/test_worker_audit.py index 601c376..79dfc13 100644 --- a/tests/test_worker_audit.py +++ b/tests/test_worker_audit.py @@ -14,6 +14,7 @@ """Regressions found while checking long-lived server deployments.""" import base64 +import json from pathlib import Path import subprocess from unittest.mock import Mock @@ -22,8 +23,10 @@ @pytest.mark.parametrize('value', [123, ['https://example.test'], '--exec=bad', - 'file:///etc/passwd', 'https://[bad', 'https:///missing-host']) -def test_download_rejects_non_http_urls_before_running_process(worker, monkeypatch, value): + 'file:///example.mp3', 'https://[bad', 'https:///missing-host', + 'https://example.test/audio', 'https://www.youtube.com/redirect', + 'https://www.youtube.com:123/watch?v=videoId0001']) +def test_download_rejects_unsupported_urls_before_running_process(worker, monkeypatch, value): run = Mock() monkeypatch.setattr(worker.subprocess, 'run', run) response = worker.app.test_client().post('/api/music_download', json={'url': value}) @@ -37,11 +40,11 @@ def test_download_timeout_releases_temporary_storage(worker, monkeypatch, tmp_pa monkeypatch.setattr(worker.tempfile, 'mkdtemp', lambda **_: str(folder)) run = Mock(side_effect=subprocess.TimeoutExpired('yt-dlp', 120)) monkeypatch.setattr(worker.subprocess, 'run', run) - response = worker.app.test_client().post('/api/music_download', json={'url': 'https://example.test/song'}) + response = worker.app.test_client().post('/api/music_download', json={'url': 'https://www.youtube.com/watch?v=videoId0001'}) assert response.status_code == 504 assert not folder.exists() assert run.call_args.kwargs['timeout'] == 120 - assert run.call_args.args[0][-2:] == ['--', 'https://example.test/song'] + assert run.call_args.args[0][-2:] == ['--', 'https://www.youtube.com/watch?v=videoId0001'] @pytest.mark.parametrize('title', ['a' * 1000, 'Найкрасивіші країни Європи ' * 20, '😀' * 1000]) @@ -49,10 +52,12 @@ def test_long_titles_fit_download_and_render_filenames(worker, monkeypatch, tmp_ folder = tmp_path / 'download' folder.mkdir() monkeypatch.setattr(worker.tempfile, 'mkdtemp', lambda **_: str(folder)) - monkeypatch.setattr(worker.subprocess, 'run', Mock( - side_effect=lambda *args, **kwargs: (folder / 'track.mp3').write_bytes(b'mp3'))) - monkeypatch.setattr(worker, '_get_song_title', lambda _: title) - response = worker.app.test_client().post('/api/music_download', json={'url': 'https://example.test/song'}) + def download(*args, **kwargs): + (folder / 'track.mp3').write_bytes(b'mp3') + return Mock(stdout=json.dumps(title).encode()) + + monkeypatch.setattr(worker.subprocess, 'run', Mock(side_effect=download)) + response = worker.app.test_client().post('/api/music_download', json={'url': 'https://www.youtube.com/watch?v=videoId0001'}) assert response.status_code == 200, response.json assert base64.b64decode(response.json['file']) == b'mp3' assert (Path(worker.DOWNLOADS) / (response.json['title'] + '.mp3')).exists() @@ -91,23 +96,24 @@ def test_download_accepts_storage_relative_filename(worker): ]) def test_download_identifiers_preserve_tracks_and_cuts_on_title_collisions(worker, monkeypatch, titles): tracks = iter((b'first-track', b'second-track')) + titles = iter(titles) temporary_folders = [] def download(command, **kwargs): output = Path(command[command.index('-o') + 1].replace('%(ext)s', 'mp3')) temporary_folders.append(output.parent) output.write_bytes(next(tracks)) + return Mock(stdout=json.dumps(next(titles)).encode()) monkeypatch.setattr(worker.subprocess, 'run', Mock(side_effect=download)) - monkeypatch.setattr(worker, '_get_song_title', Mock(side_effect=titles)) client = worker.app.test_client() - first = client.post('/api/music_download', json={'url': 'https://example.test/first'}) + first = client.post('/api/music_download', json={'url': 'https://www.youtube.com/watch?v=videoId0001'}) assert first.status_code == 200, first.json first_title = first.json['title'] first_cut = Path(worker.CUT_DIR) / f'{first_title}_cut.mp3' first_cut.write_bytes(b'first-cut') - second = client.post('/api/music_download', json={'url': 'https://example.test/second'}) + second = client.post('/api/music_download', json={'url': 'https://www.youtube.com/watch?v=videoId0002'}) assert second.status_code == 200, second.json second_title = second.json['title'] assert first_title != second_title @@ -129,9 +135,7 @@ def download(command, **kwargs): output.write_bytes(b'x' * size) monkeypatch.setattr(worker.subprocess, 'run', Mock(side_effect=download)) - title_lookup = Mock(return_value='Track') - monkeypatch.setattr(worker, '_get_song_title', title_lookup) - response = worker.app.test_client().post('/api/music_download', json={'url': 'https://example.test/song'}) + response = worker.app.test_client().post('/api/music_download', json={'url': 'https://www.youtube.com/watch?v=videoId0001'}) if size == 3: assert response.status_code == 200, response.json assert worker._decode_audio_payload(response.json['file']) == b'xxx' @@ -143,5 +147,28 @@ def download(command, **kwargs): assert 'shorter track' in response.json['error'] assert 'file' not in response.json assert not list(Path(worker.DOWNLOADS).iterdir()) - title_lookup.assert_not_called() assert all(not folder.exists() for folder in temporary_folders) + + +@pytest.mark.parametrize('url', [ + 'https://youtu.be/videoId0001?si=share-token', + 'http://music.youtube.com/watch?v=videoId0001', + 'https://www.youtube.com/shorts/videoId0001', +]) +def test_download_uses_only_the_canonical_youtube_video(worker, monkeypatch, url): + run = Mock(side_effect=subprocess.TimeoutExpired('yt-dlp', 120)) + monkeypatch.setattr(worker.subprocess, 'run', run) + assert worker.app.test_client().post('/api/music_download', json={'url': url}).status_code == 504 + command = run.call_args.args[0] + assert command[-1] == 'https://www.youtube.com/watch?v=videoId0001' + assert command[command.index('--use-extractors') + 1] == 'youtube' + assert '--ignore-config' in command + + +def test_worker_rejects_non_object_and_oversized_json(worker): + client = worker.app.test_client() + assert client.post('/api/render_video', json=['test']).status_code == 400 + worker.app.config['MAX_CONTENT_LENGTH'] = 16 + response = client.post('/api/music_upload', json={'audio_file': 'a' * 16}) + assert response.status_code == 413 + assert response.is_json diff --git a/tests/test_youtube_upload.py b/tests/test_youtube_upload.py index 6c455d1..086cd40 100644 --- a/tests/test_youtube_upload.py +++ b/tests/test_youtube_upload.py @@ -106,7 +106,7 @@ def test_desktop_client_requires_localhost(web, tmp_path): 'client_secret': (io.BytesIO(json.dumps({'installed': CLIENT['web']}).encode()), 'secret.json')}) # Session cookies are scoped to localhost; host change also invalidates CSRF. assert response.status_code == 401 - client.post('/login', base_url='https://host.test', data={'login': 'deltaaa', 'password': 'deltaaa00!'}) + client.post('/login', base_url='https://host.test', data={'login': 'test-admin', 'password': 'test-only-password'}) token = client.get('/api/youtube/status', base_url='https://host.test').json['csrf_token'] response = client.post('/api/youtube/connect', base_url='https://host.test', headers={'X-YouTube-CSRF': token}, data={ 'client_secret': (io.BytesIO(json.dumps({'installed': CLIENT['web']}).encode()), 'secret.json')}) diff --git a/video_render.py b/video_render.py index ce44f66..023aeaa 100644 --- a/video_render.py +++ b/video_render.py @@ -78,7 +78,8 @@ def _prepare_background(src: str, dst: str, duration: float, size=(1080, 1920), f'scale={w}:{h}:flags=bicubic'] filters.append('format=yuv420p') command = [FFMPEG_BINARY, '-hide_banner', '-loglevel', 'error', '-nostdin', '-y', - '-i', src, '-t', f'{duration:.3f}', '-an', '-vf', ','.join(filters), + '-protocol_whitelist', 'file', '-format_whitelist', 'mov,matroska,webm', + '-i', src, '-map', '0:v:0', '-t', f'{duration:.3f}', '-an', '-vf', ','.join(filters), '-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '10', '-threads', str(_render_threads()), dst] result = subprocess.run(command, capture_output=True, text=True, timeout=600) diff --git a/video_stickers.py b/video_stickers.py index 1061170..6c5288f 100644 --- a/video_stickers.py +++ b/video_stickers.py @@ -18,7 +18,7 @@ from video_text import TITLE_FONT_PATH, _with_shadow -STORAGE_DIR = "/tmp/automap_storage" +from media_inputs import MEDIA_STORAGE_DIR, open_upload_image from sticker_themes import ( ALLOWED_STICKERS, @@ -294,17 +294,20 @@ def _sticker_image(kind: str, size: int = 140, effects: bool = True, allow_remot # Library stickers (without a gen_ prefix) use Twemoji cache/CDN when available. codepoint = None if gen_style else TWEMOJI_CODEPOINTS.get(kind) if codepoint: - stickers_dir = os.path.join(STORAGE_DIR, 'stickers') + stickers_dir = os.path.join(MEDIA_STORAGE_DIR, 'stickers') cache_path = os.path.join(stickers_dir, f'{kind}.png') if not os.path.isfile(cache_path) and allow_remote: try: import requests os.makedirs(stickers_dir, exist_ok=True) url = f'https://cdn.jsdelivr.net/gh/twitter/twemoji@14.0.2/assets/72x72/{codepoint}.png' - resp = requests.get(url, timeout=1.5) - if resp.status_code == 200 and resp.content: - with Image.open(io.BytesIO(resp.content)) as fetched: - fetched.convert('RGBA').save(cache_path, format='PNG') + with requests.get(url, timeout=1.5, stream=True, allow_redirects=False) as response: + if response.status_code == 200: + data = response.raw.read(128 * 1024 + 1, decode_content=True) + if len(data) <= 128 * 1024: + with open_upload_image(io.BytesIO(data), ('PNG',)) as fetched: + if fetched.size == (72, 72): + fetched.convert('RGBA').save(cache_path, format='PNG') except Exception: pass if os.path.isfile(cache_path): diff --git a/web_auth.py b/web_auth.py index b9a926e..100dd75 100644 --- a/web_auth.py +++ b/web_auth.py @@ -64,8 +64,15 @@ def init_web_auth(app, env_values=None): settings = env_values or {} server_mode = os.environ.get('SERVER', settings.get('SERVER') or 'FALSE') == 'TRUE' app.config['WEB_AUTH_ENABLED'] = server_mode - login = os.environ.get('WEB_LOGIN', settings.get('WEB_LOGIN') or 'deltaaa') - password = os.environ.get('WEB_PASSWORD', settings.get('WEB_PASSWORD') or 'deltaaa00!') + if not server_mode: + allowed_hosts = os.environ.get('WEB_TRUSTED_HOSTS', settings.get('WEB_TRUSTED_HOSTS') or '') + app.config['TRUSTED_HOSTS'] = ['localhost', '127.0.0.1', '[::1]', 'automap-web', 'host.docker.internal'] + app.config['TRUSTED_HOSTS'].extend(host.strip() for host in allowed_hosts.split(',') if host.strip()) + login = os.environ.get('WEB_LOGIN', settings.get('WEB_LOGIN') or '') + password = os.environ.get('WEB_PASSWORD', settings.get('WEB_PASSWORD') or '') + if server_mode and (not login.strip() or not password.strip() or password == 'deltaaa00!'): + raise RuntimeError('Set WEB_LOGIN and a unique WEB_PASSWORD before enabling SERVER=TRUE.') + cookie_secure = os.environ.get('WEB_COOKIE_SECURE', settings.get('WEB_COOKIE_SECURE')) app.secret_key = _session_secret(app, settings) app.config.update( PERMANENT_SESSION_LIFETIME=timedelta(days=365), @@ -74,6 +81,7 @@ def init_web_auth(app, env_values=None): SESSION_REFRESH_EACH_REQUEST=False, SESSION_COOKIE_HTTPONLY=True, SESSION_COOKIE_SAMESITE='Lax', + SESSION_COOKIE_SECURE=(cookie_secure or str(server_mode)).lower() == 'true', ) identity = hmac.new(str(app.secret_key).encode(), (login + '\0' + password).encode(), hashlib.sha256).hexdigest() @@ -82,6 +90,19 @@ def authenticated(): value = session.get('web_identity', '') return isinstance(value, str) and hmac.compare_digest(value, identity) + @app.before_request + def reject_cross_origin_writes(): + if request.method in ('GET', 'HEAD', 'OPTIONS'): + return None + origin = request.headers.get('Origin') + fetch_site = request.headers.get('Sec-Fetch-Site') + try: + allowed_origin = not origin or urlsplit(origin).netloc == request.host + except ValueError: + allowed_origin = False + if not allowed_origin or fetch_site in ('cross-site', 'same-site'): + return jsonify(error='Cross-origin requests are forbidden.'), 403 + @app.before_request def require_web_login(): if not server_mode: diff --git a/yt-dlp_serv.py b/yt-dlp_serv.py index 8cab4d3..7ecf110 100644 --- a/yt-dlp_serv.py +++ b/yt-dlp_serv.py @@ -15,6 +15,7 @@ import base64 import importlib import io +import json import logging import math import os @@ -26,12 +27,28 @@ import time import uuid from functools import lru_cache -from urllib.parse import urlencode, urlsplit +from urllib.parse import urlencode from flask import Flask, request, jsonify, send_file +from PIL import Image, ImageOps, UnidentifiedImageError +from media_inputs import MEDIA_STORAGE_DIR, open_upload_image, youtube_video_url from render_progress import RenderProgress app = Flask(__name__) +app.config['MAX_CONTENT_LENGTH'] = 88 * 1024 * 1024 + + +@app.before_request +def validate_worker_json(): + if request.method == 'POST' and request.is_json: + if not isinstance(request.get_json(silent=True), dict): + return jsonify(status='error', error='Request body must be a JSON object.'), 400 + + +@app.errorhandler(413) +def request_too_large(_error): + return jsonify(status='error', error='Upload exceeds the total request size limit.'), 413 + render_progress = RenderProgress() @@ -43,12 +60,12 @@ def get_render_progress(render_id): response.headers['Cache-Control'] = 'no-store' return response, 200 if result else 404 -STORAGE_DIR = "/tmp/automap_storage" +STORAGE_DIR = MEDIA_STORAGE_DIR DOWNLOADS = os.path.join(STORAGE_DIR, "downloads") CUT_DIR = os.path.join(STORAGE_DIR, "cut") RENDER_DIR = os.path.join(STORAGE_DIR, "render") -os.makedirs(STORAGE_DIR, exist_ok=True) +os.makedirs(STORAGE_DIR, mode=0o700, exist_ok=True) os.makedirs(DOWNLOADS, exist_ok=True) os.makedirs(CUT_DIR, exist_ok=True) os.makedirs(RENDER_DIR, exist_ok=True) @@ -61,8 +78,6 @@ def get_render_progress(render_id): # h264_nvenc - NVIDIA GPUs # h264_videotoolbox - Apple Silicon chips ALLOWED_CODECS = ("libx264", "h264_nvenc", "h264_videotoolbox") -_OG_TITLE_RE = re.compile(r'(.*?)') _SAFE_TITLE_RE = re.compile(r'[^\w\-]') _AUDIO_TITLE_RE = re.compile(r'[\w\-]+') @@ -104,7 +119,8 @@ def _convert_uploaded_audio(raw, start=None, duration=None): with open(source, 'wb') as stream: stream.write(raw) command = [FFMPEG_BINARY, '-hide_banner', '-loglevel', 'error', '-nostdin', - '-protocol_whitelist', 'file,pipe'] + '-protocol_whitelist', 'file', + '-format_whitelist', 'mp3,wav,mov,aac,ogg,flac,matroska,webm,aiff'] if start is not None: command += ['-ss', str(start)] command += ['-i', source, '-map', '0:a:0', '-vn'] @@ -194,31 +210,9 @@ def cleanup_old_files(max_age_seconds=1800): except FileNotFoundError: pass # Another request may have removed or renamed the file. -def _get_song_title(url: str) -> str | None: - """Try to extract the song title from the given URL.""" - import requests - - try: - response = requests.get(url, timeout=10) - response.raise_for_status() - - match = _OG_TITLE_RE.search(response.text) - if match: - return match.group(1) - - # Fall back to the page title when Open Graph metadata is unavailable. - match_title = _HTML_TITLE_RE.search(response.text) - if match_title: - return match_title.group(1) - - except requests.RequestException: - return None - - @app.route('/api/music_download/', methods=['POST']) @app.route('/api/music_download', methods=['POST']) def music_download(): - cleanup_old_files() # Clean up old files before processing the new request """Download a video's audio, convert it to mp3, and return it as base64. Request JSON: @@ -231,47 +225,40 @@ def music_download(): { "status": "error", "error": "" } """ data = request.get_json(silent=True) or {} - raw_url = data.get('url') or '' - if not isinstance(raw_url, str): - return jsonify(status='error', error='URL must be an HTTP or HTTPS address.'), 400 - url = raw_url.strip() - if not url: - return jsonify({"status": "error", "error": "Missing 'url' in request body."}), 400 try: - parsed_url = urlsplit(url) - valid_url = parsed_url.scheme in ('http', 'https') and bool(parsed_url.hostname) - except ValueError: - valid_url = False - if not valid_url: - return jsonify(status='error', error='URL must be an HTTP or HTTPS address.'), 400 - - # yt-dlp with `-x` writes to a temp file, so pipe through the temp dir and - # read the produced .mp3 back. `-o -` is unreliable when extracting audio. + url = youtube_video_url(data.get('url')) + except ValueError as error: + return jsonify(status='error', error=str(error)), 400 + cookies = data.get('cookies') or '' + if not isinstance(cookies, str) or len(cookies.encode('utf-8')) > 256 * 1024: + return jsonify(status='error', error='Invalid YouTube cookies file.'), 400 - cookies = str(data.get('cookies') or '').strip('\r\n') + cleanup_old_files() download_dir = tempfile.mkdtemp(prefix="automap_download_") - cookie_args = [] - if cookies: - cookie_path = os.path.join(download_dir, "cookies.txt") - with open(cookie_path, "w", encoding="utf-8") as cf: - cf.write(cookies + "\n") - cookie_args = ["--cookies", cookie_path] - command = [ - "yt-dlp", - "--remote-components", "ejs:github", - *cookie_args, - "-f", "ba[ext=m4a]/ba", - "-x", - "--audio-format", "mp3", - "--audio-quality", "0", - "--no-playlist", - "-o", os.path.join(download_dir, "track.%(ext)s"), - "--", - url, - ] - try: - subprocess.run(command, capture_output=True, check=True, timeout=120) + cookie_args = [] + if cookies: + cookie_path = os.path.join(download_dir, "cookies.txt") + with open(cookie_path, "w", encoding="utf-8") as cookie_file: + cookie_file.write(cookies.strip('\r\n') + "\n") + cookie_args = ["--cookies", cookie_path] + command = [ + "yt-dlp", + "--ignore-config", + "--use-extractors", "youtube", + "--no-simulate", "--print", "after_move:%(title)j", + "--max-filesize", str(MAX_MATERIAL_BYTES), + *cookie_args, + "-f", "ba[ext=m4a]/ba", + "-x", + "--audio-format", "mp3", + "--audio-quality", "0", + "--no-playlist", + "-o", os.path.join(download_dir, "track.%(ext)s"), + "--", + url, + ] + result = subprocess.run(command, capture_output=True, check=True, timeout=120) # Find the produced mp3 inside the temp dir. mp3_path = None @@ -296,7 +283,12 @@ def music_download(): # Page titles are display labels, not unique track identifiers. Keep each # download independent from previous tracks and their saved cuts. - title = f"{_filename_stem(_get_song_title(url) or 'downloaded_audio')}_{uuid.uuid4().hex[:12]}" + try: + source_title = json.loads(result.stdout.decode('utf-8').strip()) + except (ValueError, AttributeError): + source_title = None + stem = _filename_stem(source_title) if isinstance(source_title, str) and source_title else 'downloaded_audio' + title = f"{stem}_{uuid.uuid4().hex[:12]}" base64_audio = base64.b64encode(audio).decode("utf-8") new_mp3_path = os.path.join(DOWNLOADS, f"{title}.mp3") # Temporary downloads and persistent storage may be separate mounts. @@ -323,6 +315,8 @@ def music_download(): 'Upload fresh YouTube account cookies. If they are already fresh, ' 'YouTube may also be challenging the server connection; ' 'a correctly formatted cookies file alone cannot guarantee access.') + else: + error_message = 'Unable to download this YouTube video. Check that it is available and within the size limit.' return jsonify({"status": "error", "error": f"yt-dlp ERROR: {error_message}"}), 500 @@ -330,8 +324,9 @@ def music_download(): return jsonify({"status": "error", "error": "yt-dlp not found. Please ensure yt-dlp is installed."}), 500 - except Exception as e: # noqa: BLE001 - surface any failure to the client - return jsonify({"status": "error", "error": f"Unexpected error: {str(e)}"}), 500 + except Exception: + app.logger.exception("Audio download failed") + return jsonify(status="error", error="Audio download failed."), 500 finally: shutil.rmtree(download_dir, ignore_errors=True) @@ -564,9 +559,7 @@ def _decode_reveal_mask(raw_mask: object, box_w: int, box_h: int): if not raw_bytes or len(raw_bytes) > MAX_MATERIAL_BYTES: return None try: - with Image.open(io.BytesIO(raw_bytes)) as mask_img: - if mask_img.width * mask_img.height > 20_000_000: - return None + with open_upload_image(io.BytesIO(raw_bytes), ('PNG',)) as mask_img: mask_resized = mask_img.convert('L').resize( (box_w, box_h), Image.Resampling.BILINEAR, @@ -939,7 +932,10 @@ def _decode_material(value: str, label: str) -> bytes: try: map_bytes = _decode_material(map_b64, "map_file") bg_bytes = _decode_material(background_b64, "background_video") - from PIL import Image, ImageOps, UnidentifiedImageError + with open_upload_image(io.BytesIO(map_bytes)) as map_image: + if min(map_image.size) < 2 or max(map_image.size) > 16 * min(map_image.size): + raise ValueError('Map image proportions are unsupported.') + map_image = map_image.convert('RGBA') def _decode_optional_image(field: str, label: str): raw_val = data.get(field) @@ -947,9 +943,7 @@ def _decode_optional_image(field: str, label: str): return None raw_bytes = _decode_material(str(raw_val), field) try: - with Image.open(io.BytesIO(raw_bytes)) as uploaded: - if uploaded.width * uploaded.height > 20_000_000: - raise ValueError(f'{label} image must be at most 20 megapixels.') + with open_upload_image(io.BytesIO(raw_bytes)) as uploaded: return uploaded.convert('RGBA') except (UnidentifiedImageError, OSError, Image.DecompressionBombError): raise ValueError(f'{label} must be a valid image.') @@ -960,6 +954,8 @@ def _decode_optional_image(field: str, label: str): top_left_custom_img = _decode_optional_image('top_left_sticker_image', 'Top-left sticker') top_right_custom_img = _decode_optional_image('top_right_sticker_image', 'Top-right sticker') reveal_clean_map_img = _decode_optional_image('reveal_clean_map_file', 'Reveal clean map') + except (UnidentifiedImageError, OSError, Image.DecompressionBombError): + return jsonify(status='error', error='Map must be a valid image.'), 400 except ValueError as ve: return jsonify({"status": "error", "error": str(ve)}), 400 @@ -988,6 +984,7 @@ def _decode_optional_image(field: str, label: str): map_path = bg_path = audio_temp_path = prepared_bg_path = None media_resources = [] + render_complete = False try: from moviepy import (VideoFileClip, CompositeVideoClip, ImageClip, AudioFileClip) @@ -996,8 +993,8 @@ def _decode_optional_image(field: str, label: str): # Write the decoded materials to temp files on disk. with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as map_file_temp: - map_file_temp.write(map_bytes) map_path = map_file_temp.name + map_image.save(map_file_temp, format="PNG") with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as bg_video_temp: bg_video_temp.write(bg_bytes) @@ -1005,8 +1002,8 @@ def _decode_optional_image(field: str, label: str): if audio_bytes: with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as audio_temp: - audio_temp.write(audio_bytes) audio_temp_path = audio_temp.name + audio_temp.write(_convert_uploaded_audio(audio_bytes, duration=30)) audio_path = audio_temp_path # Target clip length: 1..30 seconds (default 8.0), never longer than the source clips. @@ -1016,20 +1013,16 @@ def _decode_optional_image(field: str, label: str): TARGET = 8.0 TARGET = max(1.0, min(30.0, TARGET)) - source_video = VideoFileClip(bg_path, audio=False) - source_duration = source_video.duration or TARGET - source_video.close() - bg_duration = min(TARGET, source_duration) - # Fill the portrait frame without stretching the source footage; FFmpeg # scales, crops and softens natively instead of per frame in Python. canvas_w, canvas_h = 1080, 1920 effects = data.get('visual_effects', True) is not False with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as prepared_temp: prepared_bg_path = prepared_temp.name - _prepare_background(bg_path, prepared_bg_path, bg_duration, (canvas_w, canvas_h), effects) + _prepare_background(bg_path, prepared_bg_path, TARGET, (canvas_w, canvas_h), effects) background_video = VideoFileClip(prepared_bg_path, audio=False) media_resources.append(background_video) + bg_duration = min(TARGET, background_video.duration or TARGET) background_video = background_video.with_duration(bg_duration) dynamic_layers = {id(background_video)} dynamic_fade_timing = {} @@ -1476,10 +1469,14 @@ def _zoom_full_frame(get_frame, t: float) -> np.ndarray: } if rust_warning: payload["warning"] = rust_warning + render_complete = True return jsonify(payload), 200 - - except Exception as e: - return jsonify({"status": "error", "error": f"Error processing video: {str(e)}"}), 500 + + except ValueError as error: + return jsonify(status='error', error=str(error)), 400 + except Exception: + app.logger.exception("Video rendering failed") + return jsonify(status='error', error='Error processing video.'), 500 finally: # Release FFmpeg readers before removing their input files. @@ -1488,8 +1485,9 @@ def _zoom_full_frame(get_frame, t: float) -> np.ndarray: resource.close() except Exception: pass - # Always clean up the temp input files. - for p in (map_path, bg_path, audio_temp_path, prepared_bg_path): + # A failed render must not leave a partially encoded file available. + incomplete_output = None if render_complete else output_path + for p in (map_path, bg_path, audio_temp_path, prepared_bg_path, incomplete_output): if p and os.path.exists(p): try: os.remove(p) @@ -1520,7 +1518,7 @@ def download_video(): render_root = os.path.realpath(RENDER_DIR) if os.path.commonpath([candidate, render_root]) != render_root: return jsonify({"status": "error", "error": "Invalid path."}), 400 - if not os.path.isfile(candidate): + if not candidate.lower().endswith('.mp4') or not os.path.isfile(candidate): return jsonify({"status": "error", "error": "File not found."}), 404 return send_file( From 43801d3684f6ab26bae14b5025feb19249f6cadb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D1=96=D1=82=D0=B0=D0=BB=D1=96=D0=B9=20=D0=9B=D0=B8?= =?UTF-8?q?=D1=82=D0=BE=D0=B2=D1=87=D0=B5=D0=BD=D0=BA=D0=BE?= Date: Sat, 21 Nov 2026 19:01:39 +0000 Subject: [PATCH 2/2] Verify the deploy server SSH host key --- .github/workflows/deploy.yml | 14 +++++++++++++- readme.md | 5 +++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 152b4b2..6189486 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -19,16 +19,28 @@ on: branches: - main +permissions: + contents: read + jobs: deploy: runs-on: ubuntu-latest steps: + - name: Require the trusted SSH host fingerprint + env: + SSH_HOST_FINGERPRINT: ${{ secrets.SSH_HOST_FINGERPRINT }} + run: | + if [[ ! "$SSH_HOST_FINGERPRINT" =~ ^SHA256:[A-Za-z0-9+/]{43}$ ]]; then + printf '%s\n' 'Set SSH_HOST_FINGERPRINT to the trusted server host key fingerprint.' >&2 + exit 1 + fi - name: Deploy via SSH - uses: appleboy/ssh-action@v1.2.0 + uses: appleboy/ssh-action@7eaf76671a0d7eec5d98ee897acda4f968735a17 # v1.2.0 with: host: 152.70.8.95 username: ubuntu key: ${{ secrets.SSH_PRIVATE_KEY }} + fingerprint: ${{ secrets.SSH_HOST_FINGERPRINT }} script: | cd ~/AutoMap-cli git fetch origin main diff --git a/readme.md b/readme.md index 77f5383..e766afd 100644 --- a/readme.md +++ b/readme.md @@ -371,6 +371,11 @@ Login sessions are valid for up to a year. Changing the credentials or signing s Use your own credentials and HTTPS for a public deployment. If you used the old built-in password, choose a new one before upgrading. Keep the media worker private; the web application provides the login gate. After changing Compose environment values, run `docker compose up -d` to recreate affected containers: `docker compose restart` retains the old values. +GitHub Actions deployment requires the repository secret `SSH_HOST_FINGERPRINT`. +Get the SHA-256 fingerprint of the SSH host key from the trusted server console +and store its `SHA256:...` value before deploying. For an Ed25519 host key, use +`ssh-keygen -E sha256 -lf /etc/ssh/ssh_host_ed25519_key.pub`. + ## Tests The suite mocks external AI/HTTP downloads and does not require real credentials. It covers map rendering, decimal ranges and extrema, CLI/CSV behavior, provider/key isolation, authentication, service proxies, media handling, and browser interaction. Real software-encoded video rendering is included; live model/YouTube availability and GPU encoders are not tested.