Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,28 @@ on:
branches:
- main

permissions:
contents: read

jobs:
deploy:
runs-on: ubuntu-latest
steps:
- name: Require the trusted SSH host fingerprint
env:
SSH_HOST_FINGERPRINT: ${{ secrets.SSH_HOST_FINGERPRINT }}
run: |
if [[ ! "$SSH_HOST_FINGERPRINT" =~ ^SHA256:[A-Za-z0-9+/]{43}$ ]]; then
printf '%s\n' 'Set SSH_HOST_FINGERPRINT to the trusted server host key fingerprint.' >&2
exit 1
fi
- name: Deploy via SSH
uses: appleboy/ssh-action@v1.2.0
uses: appleboy/ssh-action@7eaf76671a0d7eec5d98ee897acda4f968735a17 # v1.2.0
with:
host: 152.70.8.95
username: ubuntu
key: ${{ secrets.SSH_PRIVATE_KEY }}
fingerprint: ${{ secrets.SSH_HOST_FINGERPRINT }}
script: |
cd ~/AutoMap-cli
git fetch origin main
Expand Down
17 changes: 15 additions & 2 deletions .github/workflows/python-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,21 @@ name: Run Pytest

on:
push:
branches: ["main"]
branches: ["main", "dev"]
pull_request:
branches: ["main"]
branches: ["main", "dev"]

permissions:
contents: read

jobs:
test:
runs-on: ubuntu-latest
steps:
- name: Check out repository
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@v5
with:
Expand All @@ -40,6 +45,14 @@ jobs:
run: python -m playwright install --with-deps chromium
- name: Run tests and measure coverage
run: python -m pytest --cov --cov-report=term-missing --cov-report=xml
- name: Run Agent tests
env:
PYTHONPATH: AutoMap-agent
run: |
python -m pip install -r AutoMap-agent/requirements.txt
python -m pytest AutoMap-agent/tests
- name: Run frontend tests
run: node --test tests/frontend/*.test.cjs
- name: Upload coverage report
if: always()
uses: actions/upload-artifact@v4
Expand Down
10 changes: 7 additions & 3 deletions AutoMap-agent/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,17 @@

Autonomous 24/7 YouTube Shorts agent separated from `AutoMap-cli`.

## Local Docker (connects to local AutoMap on `http://localhost:3000`)
## Local Docker

```bash
# From the AutoMap-cli root
docker compose up -d --build
cd AutoMap-agent
docker compose -f docker-compose.yml -f docker-compose.local.yml up -d --build
```

The agent uses the private `automap-agent-link` network to reach AutoMap. The browser opens AutoMap at `http://localhost:3000`.

Open **http://localhost:4000**.

## Server / VPS Deployment (connects to remote AutoMap server)
Expand All @@ -16,10 +21,9 @@ Open **http://localhost:4000**.
2. Set:
- `AGENT_SERVER=TRUE`
- `AGENT_WEB_USERNAME` & `AGENT_WEB_PASSWORD`
- `AGENT_SECRET_KEY`
- Optional `AGENT_SECRET_KEY` to override the secret persisted in `instance/`
- `AUTOMAP_URL=https://your-automap-domain.com`
- `AUTOMAP_PUBLIC_URL=https://your-automap-domain.com`
- `WEB_LOGIN` & `WEB_PASSWORD` (if your AutoMap server has login enabled)
- `AGENT_YOUTUBE_REDIRECT_URI=https://your-agent-domain.com/api/agent/youtube/callback`
- `OAUTHLIB_INSECURE_TRANSPORT=0`
3. Run `docker compose up -d --build`.
6 changes: 3 additions & 3 deletions AutoMap-agent/agent/automap_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,15 @@ def __init__(
session: requests.Session | None = None,
) -> None:
self.base_url = (base_url or os.environ.get("AUTOMAP_URL") or "http://host.docker.internal:3000").rstrip("/")
self.login_user = login if login is not None else os.environ.get("WEB_LOGIN", "deltaaa")
self.login_password = password if password is not None else os.environ.get("WEB_PASSWORD", "deltaaa00!")
self.login_user = login if login is not None else os.environ.get("WEB_LOGIN", "")
self.login_password = password if password is not None else os.environ.get("WEB_PASSWORD", "")
self.timeout = float(timeout)
self.http = session or requests.Session()
self._logged_in = False

def ensure_auth(self) -> None:
"""Sign in via /login when the web service runs with SERVER=TRUE."""
if self._logged_in:
if self._logged_in or not (self.login_user and self.login_password):
return
try:
resp = self.http.post(
Expand Down
9 changes: 8 additions & 1 deletion AutoMap-agent/agent/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,13 @@ class AgentDB:

def __init__(self, db_path: str | Path | None = None) -> None:
self.path = Path(db_path) if db_path is not None else default_db_path()
self.path.parent.mkdir(parents=True, exist_ok=True)
self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
try:
database = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
except FileExistsError:
pass
else:
os.close(database)
self.init_schema()

@contextmanager
Expand Down Expand Up @@ -152,6 +158,7 @@ def init_schema(self) -> None:
);
"""
)
conn.execute("BEGIN IMMEDIATE")
self._ensure_column(conn, "competitor_videos", "likes", "INTEGER NOT NULL DEFAULT 0")
self._ensure_column(conn, "competitor_videos", "comments", "INTEGER NOT NULL DEFAULT 0")
self._ensure_column(conn, "experiments", "avg_view_duration_sec", "REAL")
Expand Down
5 changes: 5 additions & 0 deletions AutoMap-agent/agent/media_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,8 @@ def probe_video_duration(video_path: Path) -> float | None:
"ffprobe",
"-v",
"error",
"-protocol_whitelist", "file",
"-format_whitelist", "mov,matroska,webm",
"-show_entries",
"format=duration",
"-of",
Expand Down Expand Up @@ -187,6 +189,9 @@ def split_background_video(
"-y",
"-ss",
f"{start:.2f}",
"-nostdin",
"-protocol_whitelist", "file",
"-format_whitelist", "mov,matroska,webm",
"-i",
str(source_path),
"-t",
Expand Down
40 changes: 40 additions & 0 deletions AutoMap-agent/agent/private_files.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
"""Write credentials with private permissions and atomic replacement."""

import os
import secrets
from pathlib import Path
from tempfile import NamedTemporaryFile


def write_private_text(path: Path, content: str, *, create_only: bool = False) -> None:
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
with NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent, delete=False) as stream:
temporary = Path(stream.name)
try:
stream.write(content)
except BaseException:
stream.close()
temporary.unlink(missing_ok=True)
raise
try:
if create_only:
try:
os.link(temporary, path)
except FileExistsError:
pass
else:
temporary.replace(path)
finally:
temporary.unlink(missing_ok=True)


def session_secret(path: Path) -> str:
configured = os.environ.get("AGENT_SECRET_KEY", "").strip()
if configured:
return configured
if not path.is_file():
write_private_text(path, secrets.token_hex(32), create_only=True)
value = path.read_text(encoding="utf-8").strip()
if not value:
raise RuntimeError("The agent session secret file is empty.")
return value
4 changes: 1 addition & 3 deletions AutoMap-agent/agent/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -377,9 +377,7 @@ def _is_retryable_model_error(err: AutoMapClientError) -> bool:
log.warning("Could not render clean reveal base map for %s", zoom_key, exc_info=True)
if effective_reveal and not reveal_clean_map_file:
effective_reveal = False
effective_zoom = False
effective_zoom_out = False
log.warning("Skipping zoom/reveal for %s because its clean map is unavailable", zoom_key)
log.warning("Skipping reveal for %s because its clean map is unavailable", zoom_key)

duration = max(9, min(15, int(cfg.get("video_duration") or 11)))
render_extra: dict[str, Any] = {
Expand Down
4 changes: 3 additions & 1 deletion AutoMap-agent/agent/youtube_intel.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@

import requests

from .private_files import write_private_text

from .db import AgentDB, utc_now_iso
from .optimizer import compute_video_score

Expand Down Expand Up @@ -211,7 +213,7 @@ def _load_oauth_credentials(self):
creds = Credentials.from_authorized_user_info(info)
if creds.expired and creds.refresh_token:
creds.refresh(Request())
self.token_path.write_text(creds.to_json(), encoding="utf-8")
write_private_text(self.token_path, creds.to_json())
return creds

def has_upload_credentials(self) -> bool:
Expand Down
17 changes: 17 additions & 0 deletions AutoMap-agent/docker-compose.local.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
services:
agent-web:
environment:
AUTOMAP_URL: http://automap-web:5000
networks:
- default
- automap
agent-runner:
environment:
AUTOMAP_URL: http://automap-web:5000
networks:
- default
- automap
networks:
automap:
external: true
name: automap-agent-link
3 changes: 1 addition & 2 deletions AutoMap-agent/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ services:
dockerfile: Dockerfile
command: ["gunicorn", "--bind", "0.0.0.0:4000", "--workers", "2", "--threads", "4", "--timeout", "120", "web:app"]
ports:
- "${AGENT_PORT:-4000}:4000"
- "${AGENT_BIND_ADDRESS:-127.0.0.1}:${AGENT_PORT:-4000}:4000"
env_file:
- path: .env
required: false
Expand All @@ -15,7 +15,6 @@ services:
- AGENT_DB_PATH=/app/instance/agent.db
- AGENT_MEDIA_DIR=/app/agent_media
- AGENT_YOUTUBE_TOKEN_PATH=/app/instance/agent_youtube_token.json
- OAUTHLIB_INSECURE_TRANSPORT=${OAUTHLIB_INSECURE_TRANSPORT:-1}
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
Expand Down
3 changes: 1 addition & 2 deletions AutoMap-agent/example.env
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
AGENT_PORT=4000
AUTOMAP_URL=http://host.docker.internal:3000
AUTOMAP_PUBLIC_URL=http://localhost:3000
OAUTHLIB_INSECURE_TRANSPORT=1

# --- Server / VPS deployment mode ---
# Set AGENT_SERVER=TRUE and AGENT_WEB_PASSWORD to protect the agent dashboard with login.
Expand All @@ -18,7 +17,7 @@ OAUTHLIB_INSECURE_TRANSPORT=1
# WEB_LOGIN=your-automap-login
# WEB_PASSWORD=your-automap-password
# AGENT_YOUTUBE_REDIRECT_URI=https://your-agent-domain.com/api/agent/youtube/callback
# OAUTHLIB_INSECURE_TRANSPORT=0
# AGENT_BIND_ADDRESS=127.0.0.1

# Optional env fallbacks (you can also enter all of these directly in the web UI):
# TELEGRAM_BOT_TOKEN=
Expand Down
2 changes: 1 addition & 1 deletion AutoMap-agent/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Flask>=3.0,<4.0
Flask>=3.1,<4.0
gunicorn>=22.0,<24.0
python-dotenv>=1.0,<2.0
requests>=2.31,<3.0
Expand Down
51 changes: 15 additions & 36 deletions AutoMap-agent/templates/agent.html
Original file line number Diff line number Diff line change
Expand Up @@ -468,14 +468,19 @@ <h2 class="text-sm font-bold text-white flex items-center gap-2">
document.getElementById('secTavilyApiKeyBadge').classList.toggle('hidden', !data.tavily_api_configured);
if (document.activeElement?.id !== 'secTgChatId' && data.telegram_chat_id && !document.getElementById('secTgChatId').value) {
document.getElementById('secTgChatId').value = data.telegram_chat_id;
saveLocalSecret('secTgChatId', data.telegram_chat_id);
}
const renderFileList = (elId, kind, list) => {
const el = document.getElementById(elId);
if (!el) return;
el.innerHTML = (list && list.length)
? list.map(name => `<li class="flex items-center justify-between gap-1 bg-slate-900/70 px-2 py-1 rounded"><span class="truncate">${esc(name)}</span><span class="flex items-center gap-0.5 shrink-0">${kind === 'background' ? `<button type="button" onclick="splitMedia('${esc(name)}')" class="text-sky-400 hover:text-sky-200 px-1" title="Split long background into 15s clips"><i class="fa-solid fa-scissors text-[10px]"></i></button>` : ''}<button type="button" onclick="deleteMedia('${kind}', '${esc(name)}')" class="text-rose-400 hover:text-rose-200 px-1">&times;</button></span></li>`).join('')
? list.map(name => `<li class="flex items-center justify-between gap-1 bg-slate-900/70 px-2 py-1 rounded"><span class="truncate">${esc(name)}</span><span class="flex items-center gap-0.5 shrink-0">${kind === 'background' ? `<button type="button" data-action="split" data-filename="${esc(name)}" class="text-sky-400 hover:text-sky-200 px-1" title="Split long background into 15s clips"><i class="fa-solid fa-scissors text-[10px]"></i></button>` : ''}<button type="button" data-action="delete" data-filename="${esc(name)}" class="text-rose-400 hover:text-rose-200 px-1">&times;</button></span></li>`).join('')
: '<li class="text-slate-500">Empty</li>';
for (const button of el.querySelectorAll('button[data-filename]')) {
button.addEventListener('click', () => {
if (button.dataset.action === 'split') splitMedia(button.dataset.filename);
else deleteMedia(kind, button.dataset.filename);
});
}
};
renderFileList('bgFilesList', 'background', pool.backgrounds || []);
renderFileList('musicFilesList', 'music', pool.music || []);
Expand Down Expand Up @@ -714,37 +719,14 @@ <h2 class="text-sm font-bold text-white flex items-center gap-2">
loadOverview();
}

const SECRET_STORAGE_KEYS = {
secTgToken: 'automap-agent-tg-token',
secTgChatId: 'automap-agent-tg-chat-id',
secYtApiKey: 'automap-agent-yt-api-key',
secGeminiApiKey: 'automap-gemini-api-key',
secDeepseekApiKey: 'automap-deepseek-api-key',
secTavilyApiKey: 'automap-tavily-api-key',
};

function saveLocalSecret(id, val) {
function clearLegacyAgentSecrets() {
try {
const key = SECRET_STORAGE_KEYS[id];
if (!key) return;
if (val) localStorage.setItem(key, val);
else localStorage.removeItem(key);
for (const key of ['automap-agent-tg-token', 'automap-agent-tg-chat-id', 'automap-agent-yt-api-key']) {
localStorage.removeItem(key);
}
} catch (_) {}
}

function initLocalSecrets() {
Object.entries(SECRET_STORAGE_KEYS).forEach(([id, storageKey]) => {
const el = document.getElementById(id);
if (!el) return;
try {
const saved = localStorage.getItem(storageKey);
if (saved) el.value = saved;
} catch (_) {}
el.addEventListener('input', () => saveLocalSecret(id, el.value.trim()));
el.addEventListener('change', () => saveLocalSecret(id, el.value.trim()));
});
}

function toggleSecretVisibility(inputId, iconId) {
const input = document.getElementById(inputId);
const icon = document.getElementById(iconId);
Expand Down Expand Up @@ -776,12 +758,9 @@ <h2 class="text-sm font-bold text-white flex items-center gap-2">
});
const data = await res.json();
if (res.ok) {
if (tok) saveLocalSecret('secTgToken', tok);
if (cid) saveLocalSecret('secTgChatId', cid);
if (ytk) saveLocalSecret('secYtApiKey', ytk);
if (gmk) saveLocalSecret('secGeminiApiKey', gmk);
if (dsk) saveLocalSecret('secDeepseekApiKey', dsk);
if (tvk) saveLocalSecret('secTavilyApiKey', tvk);
for (const id of ['secTgToken', 'secYtApiKey', 'secGeminiApiKey', 'secDeepseekApiKey', 'secTavilyApiKey']) {
document.getElementById(id).value = '';
}
showAgentToast('Keys saved! Agent will pick them up automatically.');
} else {
showAgentToast(data.error || 'Failed to save keys', true);
Expand Down Expand Up @@ -882,7 +861,7 @@ <h2 class="text-sm font-bold text-white flex items-center gap-2">
history.replaceState(null, '', location.pathname);
}

initLocalSecrets();
clearLegacyAgentSecrets();
loadOverview();
setInterval(loadOverview, 15000);
</script>
Expand Down
2 changes: 1 addition & 1 deletion AutoMap-agent/tests/test_agent_optimizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

def test_score_and_suffix_inference():
assert compute_video_score(1000, None, 80.0) == 800.0
assert compute_video_score(1000, 5000, 90.0, likes_72h=250) == 5625.0
assert compute_video_score(1000, 5000, 90.0, likes_72h=250) == 5400.0
assert infer_label_suffix("How tall is the tallest mountain in your country?", "number") == "m"
assert infer_label_suffix("Forest coverage percentage in Europe", "number") == "%"
assert infer_label_suffix("Most popular drink in Europe", "text") == ""
Expand Down
Loading
Loading