diff --git a/Dockerfile.ytserver b/Dockerfile.ytserver index 22a03a4..c1fc6b0 100644 --- a/Dockerfile.ytserver +++ b/Dockerfile.ytserver @@ -14,7 +14,7 @@ COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -COPY yt-dlp_serv.py sticker_themes.py server_runtime.py ./ +COPY yt-dlp_serv.py sticker_themes.py extra_stickers.py server_runtime.py render_progress.py ./ COPY resources/fonts/ resources/fonts/ EXPOSE 8080 diff --git a/benchmarks/bench_video_render.py b/benchmarks/bench_video_render.py new file mode 100644 index 0000000..b5bf5d6 --- /dev/null +++ b/benchmarks/bench_video_render.py @@ -0,0 +1,81 @@ +"""Render benchmark for the video worker. + +Usage: python benchmarks/bench_video_render.py [--seconds 3] [--module yt-dlp_serv.py] + +Renders a short synthetic clip through /api/render_video with visual effects +on and off (plus a draft run) and prints wall-clock timings. ``--module`` lets +you time another revision of the worker file for before/after comparisons. +""" +import argparse +import base64 +import importlib.util +import os +import sys +import tempfile +import time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def load_worker(path): + sys.path.insert(0, ROOT) + spec = importlib.util.spec_from_file_location('bench_worker', path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--seconds', type=float, default=3.0) + parser.add_argument('--module', default=os.path.join(ROOT, 'yt-dlp_serv.py')) + parser.add_argument('--repeat', type=int, default=1) + args = parser.parse_args() + + import numpy as np + from moviepy import VideoClip + from PIL import Image + + worker = load_worker(args.module) + tmp = tempfile.mkdtemp() + worker.RENDER_DIR = tmp + background = os.path.join(tmp, 'bg.mp4') + # Moving gradient at 1280x720 so scaling, cropping and blur do real work. + xs = np.linspace(0, 255, 1280)[None, :, None] + + def frame(t): + base = (xs + 120 * t) % 255 + return np.broadcast_to(base, (720, 1280, 3)).astype(np.uint8) + + VideoClip(frame, duration=args.seconds).write_videofile( + background, fps=30, codec='libx264', logger=None) + map_png = os.path.join(tmp, 'map.png') + Image.new('RGBA', (1600, 1200), (40, 160, 90, 255)).save(map_png) + material = { + 'map_file': base64.b64encode(open(map_png, 'rb').read()).decode(), + 'background_video': base64.b64encode(open(background, 'rb').read()).decode(), + 'video_title': 'Benchmark title\nSecond line', + 'duration': args.seconds, + 'watermark_text': 'automap', 'watermark_stars': True, + 'subscribe_badge': True, 'subscribe_delay': 0.5, + 'top_left_sticker': 'star', 'top_right_sticker': 'heart', + 'allow_remote_stickers': False, + } + client = worker.app.test_client() + cases = [('effects on', {'visual_effects': True}), + ('effects off', {'visual_effects': False}), + ('draft, effects on', {'visual_effects': True, 'draft': True})] + for name, extra in cases: + best = None + for _ in range(args.repeat): + start = time.perf_counter() + response = client.post('/api/render_video', json={**material, **extra}) + elapsed = time.perf_counter() - start + assert response.status_code == 200, response.json + best = elapsed if best is None else min(best, elapsed) + os.remove(response.json['file_path']) + print(f'{name:20s} {best:6.2f}s for {args.seconds:g}s clip') + + +if __name__ == '__main__': + main() diff --git a/cli.py b/cli.py index 867b5fd..4044681 100644 --- a/cli.py +++ b/cli.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Command-line interface (`automap make` and `automap batch`) for AutoMap.""" + import os from pathlib import Path @@ -33,6 +35,9 @@ "gemini-3-flash-preview", "gemma-4-31b-it", "gemma-4-26b-a4b-it", + # Current DeepSeek API model identifiers: https://api-docs.deepseek.com/ + "deepseek-flash", + "deepseek-v4-pro", ] @@ -98,7 +103,7 @@ def main(): @click.option("--key", "api_key", default=None, - help="Gemini API key for this run (overrides GEMINI_API_KEY env var).") + help="API key for the selected model provider (overrides its environment key).") @click.option("--temperature", type=click.FloatRange(0, 2), default=None, help="AI sampling randomness, from 0 to 2. Omit to use the model default.") def make(prompt, mode, map_type, size_mod, optional_feature, model, font_name, show_result, verbose, api_key, temperature): @@ -110,7 +115,7 @@ def make(prompt, mode, map_type, size_mod, optional_feature, model, font_name, s """ # Lazy import so `automap --help` stays fast and doesn't require API keys. from map import log - from map.functions import AIServiceError + from map.functions import AIServiceError, provider_for_model, provider_key_name from map.auto_map import auto_map ai_options = {} @@ -129,7 +134,7 @@ def make(prompt, mode, map_type, size_mod, optional_feature, model, font_name, s # A key passed on the command line overrides the env var for this run only. if api_key: - os.environ["GEMINI_API_KEY"] = api_key + os.environ[provider_key_name(provider_for_model(model))] = api_key # Click accepts mixed-case choices; the renderer compares lowercase names. try: @@ -201,17 +206,17 @@ def make(prompt, mode, map_type, size_mod, optional_feature, model, font_name, s @click.option("--key", "api_key", default=None, - help="Gemini API key for this run (overrides GEMINI_API_KEY env var).") + help="API key for the selected model provider (overrides its environment key).") def batch(ideas_file, output_dir, mode, map_type, size_mod, optional_feature, model, font_name, verbose, api_key): """Generate multiple maps from ideas listed in a .csv IDEAS_FILE.""" from map import log - from map.functions import AIServiceError + from map.functions import AIServiceError, provider_for_model, provider_key_name from map.auto_map import auto_map from map.functions import load_ideas_from_file, slugify_prompt log.set_verbose(verbose) if api_key: - os.environ["GEMINI_API_KEY"] = api_key + os.environ[provider_key_name(provider_for_model(model))] = api_key try: ideas = load_ideas_from_file(ideas_file) @@ -253,7 +258,7 @@ def batch(ideas_file, output_dir, mode, map_type, size_mod, optional_feature, mo font_name=item_font, show_result=False, io_res=True, - api_key=api_key, + api_key=api_key if provider_for_model(item_model) == provider_for_model(model) else None, **style_options, ) except AIServiceError as error: diff --git a/docker-compose.yml b/docker-compose.yml index 56bd1ce..7718c25 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,7 @@ services: WEB_PASSWORD: "${WEB_PASSWORD:-deltaaa00!}" WEB_SECRET_KEY: "${WEB_SECRET_KEY:-}" GEMINI_API_KEY: "${GEMINI_API_KEY:-}" + DEEPSEEK_API_KEY: "${DEEPSEEK_API_KEY:-}" YTDLP_URL: http://yt-dlp-worker:8080 volumes: - automap-session:/app/instance diff --git a/example.env b/example.env index af606c7..f550408 100644 --- a/example.env +++ b/example.env @@ -1,4 +1,5 @@ GEMINI_API_KEY=your_google_ai_studio_key +DEEPSEEK_API_KEY=your_deepseek_api_key SERVER=TRUE WEB_LOGIN=deltaaa WEB_PASSWORD=deltaaa00! diff --git a/extra_stickers.py b/extra_stickers.py new file mode 100644 index 0000000..ca141b1 --- /dev/null +++ b/extra_stickers.py @@ -0,0 +1,115 @@ +"""Offline vector artwork for the extended sticker library.""" + + +def draw_extra_sticker(kind, draw, s, outline, ow, star_polygon_fn=None): + """Draw an extended procedural sticker onto a Pillow ImageDraw canvas. + + Returns True if `kind` is handled by this library, False otherwise. + """ + extras = { + 'mountain', 'volcano', 'island', 'rainbow', 'moon', 'rain', 'robot', + 'satellite', 'laptop', 'graduation', 'medal', 'cat', 'dog', 'fish', 'burger', 'train', + } + if kind not in extras: + return False + + def coords(values): + return tuple(value * s for value in values) + + def ellipse(box, color): + draw.ellipse(coords(box), fill=color, outline=outline, width=ow) + + def rect(box, color): + draw.rounded_rectangle(coords(box), radius=ow * 2, fill=color, outline=outline, width=ow) + + def poly(points, color): + draw.polygon([coords(point) for point in points], fill=color, outline=outline, width=ow) + + def line(points, color, width=None): + draw.line([coords(point) for point in points], fill=color, width=width or ow) + + blue, gold, white = '#38bdf8', '#fbbf24', '#f8fafc' + if kind in ('mountain', 'volcano'): + if kind == 'volcano': + ellipse((.35, .06, .61, .3), '#94a3b8') + poly([(.08, .9), (.48, .2), (.9, .9)], '#78716c' if kind == 'volcano' else '#64748b') + poly([(.32, .49), (.48, .2), (.66, .49), (.53, .42), (.43, .51)], '#fb923c' if kind == 'volcano' else white) + if kind == 'volcano': + line([(.49, .42), (.6, .65), (.55, .86)], '#ef4444', ow * 2) + elif kind == 'island': + ellipse((.07, .65, .94, .94), blue) + ellipse((.19, .66, .84, .87), gold) + line([(.48, .74), (.6, .3)], '#92400e', ow * 2) + for tip in [(.23, .3), (.38, .1), (.82, .16), (.91, .39), (.7, .5)]: + poly([(.58, .31), tip, (.55, .4)], '#22c55e') + elif kind == 'rainbow': + for index, color in enumerate(['#ef4444', '#fb923c', gold, '#22c55e', '#38bdf8', '#a78bfa']): + inset = index * .055 + draw.arc(coords((.08 + inset, .13 + inset, .92 - inset, .97 - inset)), 180, 360, fill=color, width=ow * 2) + elif kind == 'moon': + ellipse((.15, .08, .85, .89), gold) + draw.ellipse(coords((.37, .01, .99, .7)), fill=(0, 0, 0, 0)) + elif kind == 'rain': + for box in [(.12, .25, .57, .61), (.31, .12, .75, .61), (.56, .29, .9, .63)]: + ellipse(box, '#cbd5e1') + for x in (.27, .5, .73): + line([(x, .7), (x - .06, .89)], blue, ow * 2) + elif kind == 'robot': + line([(.5, .1), (.5, .25)], '#94a3b8') + ellipse((.44, .05, .56, .18), '#ef4444') + rect((.14, .25, .86, .86), '#94a3b8') + for x in (.27, .6): + ellipse((x, .39, x + .13, .55), blue) + rect((.29, .65, .71, .77), white) + line([(.43, .66), (.43, .76)], outline) + line([(.57, .66), (.57, .76)], outline) + elif kind == 'satellite': + rect((.05, .3, .31, .73), '#2563eb') + rect((.69, .3, .95, .73), '#2563eb') + line([(.29, .52), (.72, .52)], gold, ow * 2) + rect((.35, .25, .65, .78), '#cbd5e1') + poly([(.41, .24), (.64, .07), (.76, .28)], white) + elif kind == 'laptop': + rect((.18, .16, .82, .69), '#475569') + rect((.25, .24, .75, .6), blue) + poly([(.18, .69), (.82, .69), (.96, .86), (.04, .86)], '#cbd5e1') + elif kind == 'graduation': + rect((.25, .4, .74, .75), '#1e293b') + poly([(.07, .34), (.5, .13), (.94, .34), (.5, .55)], '#334155') + line([(.51, .34), (.83, .42), (.83, .78)], gold, ow * 2) + elif kind == 'medal': + poly([(.25, .09), (.47, .09), (.61, .51), (.36, .55)], '#ef4444') + poly([(.54, .09), (.76, .09), (.64, .55), (.4, .51)], '#2563eb') + ellipse((.2, .4, .81, .96), gold) + if star_polygon_fn is not None: + draw.polygon(star_polygon_fn(s * .5, s * .68, s * .18, s * .08), fill=white) + elif kind in ('cat', 'dog'): + color = '#fb923c' if kind == 'cat' else '#b45309' + if kind == 'cat': + poly([(.18, .43), (.18, .12), (.47, .35)], color) + poly([(.53, .35), (.83, .12), (.83, .43)], color) + else: + ellipse((.04, .19, .34, .72), color) + ellipse((.67, .19, .96, .72), color) + ellipse((.18, .27, .83, .91), color) + for x in (.33, .62): + ellipse((x, .47, x + .06, .55), outline) + ellipse((.36, .61, .66, .81), '#ffedd5') + poly([(.44, .64), (.59, .64), (.51, .73)], outline) + elif kind == 'fish': + poly([(.59, .51), (.92, .27), (.92, .79)], '#fb923c') + ellipse((.08, .26, .74, .8), blue) + ellipse((.22, .41, .29, .48), outline) + line([(.47, .39), (.41, .54), (.49, .65)], '#2563eb') + elif kind == 'burger': + ellipse((.13, .13, .88, .62), gold) + for y, color in [(.44, '#22c55e'), (.56, '#ef4444'), (.68, '#78350f'), (.78, gold)]: + rect((.12, y, .89, y + .12), color) + elif kind == 'train': + rect((.21, .09, .8, .8), '#ef4444') + rect((.28, .22, .72, .48), blue) + for x in (.3, .61): + ellipse((x, .59, x + .1, .69), gold) + line([(.27, .83), (.14, .95)], outline, ow * 2) + line([(.74, .83), (.87, .95)], outline, ow * 2) + return True diff --git a/main.py b/main.py index df20abb..ffb0fec 100644 --- a/main.py +++ b/main.py @@ -15,10 +15,9 @@ """Flask web interface for AutoMap. Serves the single-page UI in ``templates/index.html`` and exposes a small JSON -API used by that UI. The API key handling is the interesting part: if a ``.env`` -file with a real ``GEMINI_API_KEY`` is present, the web UI never asks for a key. -If it is missing, ``/api/status`` reports it and the UI shows a key input, whose -value is forwarded to the generation request only. +API used by that UI. Gemini and DeepSeek use separate keys from the environment +or ``.env``. If the selected provider has no configured key, ``/api/status`` +reports it and the UI uses that provider's browser-saved key for the request. """ import base64 @@ -29,7 +28,7 @@ from pathlib import Path from PIL import Image, UnidentifiedImageError -import requests +import requests as _requests_module # auto_map() loads fonts/maps via *relative* paths ("resources/fonts", ...), # so the process must run from the AutoMap-cli project directory. Anchor the @@ -55,9 +54,13 @@ ) from map.constants import europe_coords, world_coords from cli import AVAILABLE_MODELS -from map.functions import AIServiceError, parse_ideas_content, slugify_prompt, validate_temperature +from map.functions import ( + AIServiceError, configured_api_key, list_deepseek_models, parse_ideas_content, + provider_for_model, provider_key_name, slugify_prompt, validate_temperature, +) -from flask import Flask, request, jsonify, send_file, render_template, Response +from flask import Flask, request, jsonify, send_file, render_template, Response, session +from web_ai_cache import ai_cache FONTS_DIR = BASE_DIR / "resources" / "fonts" @@ -66,6 +69,37 @@ if not key and env.is_file(): key = (dotenv_values(env).get("GEMINI_API_KEY") or "").strip() needs_api_key = not key or key.startswith("your_") + +class _PooledRequests: + """``requests``-compatible facade that reuses one pooled Session. + + Keeps keep-alive connections to the worker instead of opening a new TCP + connection per proxied call. Attribute access (exceptions, etc.) falls + through to the ``requests`` module, and tests can still patch + ``main.requests.get``/``post``. + """ + + def __init__(self): + self._session = _requests_module.Session() + self._orig_get = _requests_module.get + self._orig_post = _requests_module.post + + def get(self, *args, **kwargs): + if _requests_module.get is not self._orig_get: + return _requests_module.get(*args, **kwargs) + return self._session.get(*args, **kwargs) + + def post(self, *args, **kwargs): + if _requests_module.post is not self._orig_post: + return _requests_module.post(*args, **kwargs) + return self._session.post(*args, **kwargs) + + def __getattr__(self, name): + return getattr(_requests_module, name) + + +requests = _PooledRequests() + WORKER_URL = (os.environ.get("YTDLP_URL") or "http://yt-dlp-worker:8080").rstrip('/') @@ -99,7 +133,7 @@ def validate_api_json_body(): data = request.get_json(silent=True) if not isinstance(data, dict): return jsonify(status='error', error='Request body must be a valid JSON object.'), 400 - for name in ('api_key', 'url'): + for name in ('api_key', 'url', 'provider', 'model'): if data.get(name) is not None and not isinstance(data[name], str): return jsonify(status='error', error=f'{name} must be a string.'), 400 @@ -110,6 +144,7 @@ def request_too_large(error): @app.route('/') def main(): + """Serve the main single-page map generator UI.""" return render_template("index.html") @@ -121,7 +156,7 @@ def studio(): @app.route('/api/video_title_preview', methods=['POST']) def video_title_preview(): - import requests + """Proxy title/watermark/sticker preview requests to the yt-dlp render worker.""" data = request.get_json(silent=True) or {} allowed_keys = ( @@ -169,7 +204,6 @@ def api_fonts(): @app.route('/api/codecs') def api_codecs(): """Ask the render worker which encoders its FFmpeg and hardware can use.""" - import requests try: response = requests.get(f'{WORKER_URL}/api/codecs', timeout=20) @@ -180,12 +214,91 @@ def api_codecs(): @app.route('/api/status') def api_status(): - """Report whether the server already has a usable Gemini API key. + """Report whether the selected provider already has a server-side key. Response: { "needs_api_key": true } -> the UI must ask the user for a key """ - return jsonify({"needs_api_key": needs_api_key}) + provider = request.args.get('provider', 'gemini') + if provider not in ('gemini', 'deepseek'): + return jsonify(status='error', error='Unknown AI provider.'), 400 + required = needs_api_key if provider == 'gemini' else not _server_api_key(provider) + return jsonify({"needs_api_key": required}) + + +def _server_api_key(provider): + """Return the server-configured API key for ``provider``, if any.""" + return configured_api_key(provider, dotenv_values(env) if env.is_file() else {}) + + +def _request_provider(data): + """Resolve and validate the AI provider from ``provider`` or ``model``.""" + inferred = provider_for_model(data.get('model') or '') + provider = data.get('provider') or inferred + if provider not in ('gemini', 'deepseek'): + raise ValueError('Unknown AI provider.') + if data.get('model') and inferred != provider: + raise ValueError('The selected model belongs to a different AI provider.') + return provider + + +def _request_api_key(data, provider): + """Resolve the API key for ``provider`` from server config or the request payload.""" + supplied = str(data.get('api_key') or '').strip() or None + if provider == 'deepseek': + resolved = _server_api_key(provider) or supplied + else: + resolved = supplied + if not needs_api_key: + return resolved + if not resolved: + raise ValueError(f'API key required. Provide {provider_key_name(provider)} via .env or the web form.') + return resolved + + +_TRUTHY = {'1', 'true', 'yes', 'on'} + + +def _wants_refresh(data) -> bool: + """Return True if the request asks to bypass the AI response cache.""" + value = data.get('refresh_ai', data.get('refresh', False)) + return value is True or str(value).strip().lower() in _TRUTHY + + +def _auto_map_with_ai_cache(*, refresh: bool, provider: str, api_key, **kwargs): + """Call auto_map, reusing a cached AI answer for identical data requests. + + Only request-affecting inputs form the key, so font/style/palette changes + reuse the answer. API keys only contribute a hash to the user scope. + Returns (buffer, cache_hit). + """ + temperature = kwargs.get('temperature') + identity = '' + try: + identity = str(session.get('web_identity', '') or '') + except RuntimeError: + identity = '' + cache_key = ai_cache.make_key( + ai_cache.scope(identity, api_key), + prompt=str(kwargs.get('prompt') or ''), + map_type=str(kwargs.get('map_type') or ''), + model=str(kwargs.get('model') or ''), + provider=provider, + temperature=temperature, + short_form=kwargs.get('optional_feature') == 'short_form', + web_search=bool(kwargs.get('web_search', False)), + max_search_requests=kwargs.get('max_search_requests', 5), + ) + cached = None if refresh else ai_cache.get(cache_key) + if cached is not None: + kwargs['ai_answer_override'] = cached + buffer = auto_map(api_key=api_key, **kwargs) + if cached is None and buffer is not None: + meta = getattr(buffer, 'map_meta', None) + answer = meta.get('ai_answer') if isinstance(meta, dict) else None + if isinstance(answer, dict): + ai_cache.put(cache_key, answer) + return buffer, cached is not None def _build_analysis_from_ai( @@ -423,10 +536,11 @@ def generate(): data = request.get_json(silent=True) or {} # Optional user-supplied key (used only when .env has none). - api_key = (data.get('api_key') or '').strip() or None - if needs_api_key and not api_key: - return jsonify({"status": "error", - "error": "API key required. Provide GEMINI_API_KEY via .env or the web form."}), 400 + try: + provider = _request_provider(data) + api_key = _request_api_key(data, provider) + except ValueError as error: + return jsonify(status='error', error=str(error)), 400 # Validate the requested font against what is actually on disk, otherwise # auto_map would raise FileNotFoundError when loading the TrueType file. @@ -461,7 +575,9 @@ def generate(): style_kwargs[kw_name] = data[style_key] try: - buffer = auto_map( + buffer, ai_cache_hit = _auto_map_with_ai_cache( + refresh=_wants_refresh(data), + provider=provider, prompt=data['topic'], mode=data['mode'], map_type=data['map_type'], @@ -473,6 +589,7 @@ def generate(): io_res=True, api_key=api_key, **temperature_options, + **({"web_search": True, "max_search_requests": max(1, min(10, int(data.get("max_search_requests") or 5)))} if str(data.get("web_search") or "").strip().lower() in ("1", "true", "yes", "on") else {}), **style_kwargs, ) except AIServiceError as error: @@ -496,6 +613,7 @@ def generate(): "status": "success", "command": command, "image_url": f"data:image/png;base64,{encoded}", + "ai_cached": ai_cache_hit, } render_meta = getattr(buffer, "map_meta", None) ai_vals = render_meta.get("ai_answer") if isinstance(render_meta, dict) else None @@ -514,10 +632,22 @@ def generate(): @app.route('/api/models', methods=['POST']) def list_available_models(): - """Return Gemini/Gemma models available for the active API key.""" + """Return models available for the selected provider's API key.""" data = request.get_json(silent=True) or {} + try: + provider = _request_provider(data) + except ValueError as error: + return jsonify(status='error', error=str(error), models=[]), 400 + if provider == 'deepseek': + try: + models = list_deepseek_models(_request_api_key(data, provider)) + return jsonify(status='success', models=models) + except ValueError as error: + return jsonify(status='error', error=str(error), models=[]), 400 + except AIServiceError as error: + return jsonify(status='error', error=str(error), models=[]), error.status_code req_key = str(data.get("api_key") or "").strip() - env_key = (os.getenv("GEMINI_API_KEY") or "").strip() + env_key = _server_api_key('gemini') or '' key = req_key or env_key if not key: return jsonify({"status": "error", "error": "API key required.", "models": []}), 400 @@ -563,6 +693,83 @@ def download_batch_template(): ) +@app.route('/api/download_batch_zip', methods=['POST']) +def download_batch_zip(): + """Bundle generated batch maps into a single ZIP archive named by topic.""" + import zipfile + + data = request.get_json(silent=True) or {} + items = data.get("items") or [] + if not isinstance(items, list) or not items: + return jsonify({"status": "error", "error": "No maps provided."}), 400 + + buf = io.BytesIO() + used_names: set[str] = set() + added = 0 + with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for idx, item in enumerate(items, start=1): + if not isinstance(item, dict): + continue + image_url = str(item.get("image_url") or "").strip() + prompt = str(item.get("prompt") or "").strip() + if not image_url.startswith('data:image/png;base64,'): + continue + try: + image_bytes = base64.b64decode(image_url.split(',', 1)[1], validate=True) + with Image.open(io.BytesIO(image_bytes)) as image: + if image.format != 'PNG': + continue + image.verify() + except (ValueError, OSError): + continue + + base = re.sub(r"[^\w\-]+", "_", prompt.strip().replace(" ", "_")).strip("_") if prompt else f"map_{idx:02d}" + if not base: + base = f"map_{idx:02d}" + count = 1 + arc_name = f"{base}.png" + while arc_name in used_names: + count += 1 + arc_name = f"{base}_{count}.png" + used_names.add(arc_name) + zf.writestr(arc_name, image_bytes) + added += 1 + + if added == 0: + return jsonify({"status": "error", "error": "No valid map files found to archive."}), 404 + + buf.seek(0) + return send_file( + buf, + mimetype="application/zip", + as_attachment=True, + download_name="automap_batch.zip", + ) + + +@app.route('/api/parse_batch_csv', methods=['POST']) +def parse_batch_csv(): + """Parse an uploaded .csv ideas file and return the normalized idea rows.""" + if request.files and "file" in request.files: + uploaded = request.files["file"] + filename = uploaded.filename or "" + ideas_text = uploaded.read().decode("utf-8", errors="replace") + else: + data = request.get_json(silent=True) or {} + filename = str(data.get("filename") or "") + ideas_text = str(data.get("ideas_text") or "") + + try: + ideas = parse_ideas_content(ideas_text, filename=filename) + except ValueError as err: + return jsonify({"status": "error", "error": str(err)}), 400 + + if not ideas: + return jsonify({"status": "error", "error": "No valid map ideas found in the uploaded file."}), 400 + + return jsonify({"status": "success", "ideas": ideas, "total": len(ideas)}) + + @app.route('/api/generate_batch/', methods=['POST']) @app.route('/api/generate_batch', methods=['POST']) def generate_batch(): @@ -577,12 +784,11 @@ def generate_batch(): filename = str(data.get("filename") or "") ideas_text = str(data.get("ideas_text") or "") - api_key = str(data.get("api_key") or "").strip() or None - if needs_api_key and not api_key: - return jsonify({ - "status": "error", - "error": "API key required. Provide GEMINI_API_KEY via .env or the web form.", - }), 400 + try: + provider = _request_provider(data) + api_key = _request_api_key(data, provider) + except ValueError as error: + return jsonify(status='error', error=str(error)), 400 try: ideas = parse_ideas_content(ideas_text, filename=filename) @@ -595,7 +801,7 @@ def generate_batch(): default_font, default_eop, default_gsm = _parse_render_params(data) default_mode = str(data.get("mode") or "text").lower() default_map_type = str(data.get("map_type") or "default").lower() - default_model = str(data.get("model") or AVAILABLE_MODELS[0]) + default_model = str(data.get("model") or ('deepseek-flash' if provider == 'deepseek' else AVAILABLE_MODELS[0])) available_fonts = set(_available_fonts()) temperature_options: dict[str, object] = {} @@ -641,7 +847,13 @@ def generate_batch(): out_filename += ".png" try: - buffer = auto_map( + item_provider = provider_for_model(item_model) + item_api_key = api_key if item_provider == provider else _server_api_key(item_provider) + if not item_api_key and (item_provider == 'deepseek' or needs_api_key): + raise ValueError(f'API key required for {item_provider}. Use one provider per CSV or configure its server key.') + buffer, _hit = _auto_map_with_ai_cache( + refresh=_wants_refresh(data), + provider=item_provider, prompt=prompt, mode=item_mode, map_type=item_map_type, @@ -651,8 +863,9 @@ def generate_batch(): font_name=item_font, show_result=False, io_res=True, - api_key=api_key, + api_key=item_api_key, **temperature_options, + **({"web_search": True, "max_search_requests": max(1, min(10, int(data.get("max_search_requests") or 5)))} if str(data.get("web_search") or "").strip().lower() in ("1", "true", "yes", "on") else {}), **item_style_kwargs, ) except Exception as err: # noqa: BLE001 @@ -687,6 +900,7 @@ def generate_batch(): "gsm": item_gsm, "extra_option": item_eop or "none", "model": item_model, + "provider": item_provider, "font": item_font, "palette_theme": item_style_kwargs.get("palette_theme", "auto"), } @@ -864,7 +1078,6 @@ def music_download(): { "status": "success", "file": "", "title": "" } { "status": "error", "error": "" } """ - import requests data = request.get_json(silent=True) or {} url = (data.get('url') or '').strip() @@ -922,7 +1135,6 @@ def cut_audio() -> tuple[object, int]: { "status": "success", "file": "", "title": "" } { "status": "error", "error": "" } """ - import requests data: dict[str, object] = request.get_json(silent=True) or {} start_time = str(data.get('start_time') or '').strip() @@ -963,6 +1175,32 @@ def cut_audio() -> tuple[object, int]: return jsonify(payload), response.status_code +def _render_video_request_data(): + """Return render settings from JSON or from a binary multipart upload. + + Multipart form: a ``payload`` field holding the JSON settings object plus + file parts named like the JSON fields (``map_file``, ``background_video``, + ``audio_file``, ...). Files are converted to the data URLs the worker + expects, so the browser avoids base64 encoding and the ~33% upload overhead. + """ + if not request.files and 'payload' not in request.form: + return request.get_json(silent=True) or {} + import json + try: + data = json.loads(request.form.get('payload') or '{}') + except ValueError: + return None + if not isinstance(data, dict): + return None + for field, storage in request.files.items(): + raw = storage.read() + if not raw: + continue + mime = storage.mimetype or 'application/octet-stream' + data[field] = f"data:{mime};base64,{base64.b64encode(raw).decode('ascii')}" + return data + + @app.route('/api/render_video', methods=['POST']) @app.route('/api/render_video/', methods=['POST']) def make_video() -> tuple[object, int]: @@ -981,9 +1219,10 @@ def make_video() -> tuple[object, int]: { "status": "success", "file_path": "...", "download_url": "..." } { "status": "error", "error": "" } """ - import requests - data: dict[str, object] = request.get_json(silent=True) or {} + data = _render_video_request_data() + if data is None: + return jsonify(status='error', error='Invalid multipart payload field.'), 400 # The frontend sends base64 strings; keep them as-is (default to empty string). map_b64 = data.get('map_file') or '' @@ -1026,6 +1265,7 @@ def make_video() -> tuple[object, int]: "subscribe_text": data.get('subscribe_text', 'Subscribe!'), "subscribe_delay": data.get('subscribe_delay', 2.5), "intro_fade": data.get('intro_fade', False), + **({'render_id': data['render_id']} if data.get('render_id') is not None else {}), }, # Video rendering (moviepy/ffmpeg) can take a while - allow more time. timeout=300, @@ -1045,6 +1285,22 @@ def make_video() -> tuple[object, int]: return jsonify(payload), response.status_code +@app.get('/api/render_progress/') +def video_render_progress(render_id): + try: + response = requests.get(f'{WORKER_URL}/api/render_progress/{render_id}', timeout=5) + except requests.RequestException: + return jsonify(error='Render progress is temporarily unavailable.'), 502 + try: + result = jsonify(response.json()) + result.headers['Cache-Control'] = 'no-store' + return result, response.status_code + except ValueError: + return jsonify(error='Render service returned invalid progress.'), 502 + finally: + response.close() + + @app.route('/api/download_video', methods=['GET']) @app.route('/api/download_video/', methods=['GET']) def download_video(): @@ -1054,7 +1310,6 @@ def download_video(): lives in the worker container's storage, so we proxy the download to the worker and stream its response back as an attachment. """ - import requests from flask import Response raw_path = (request.args.get('path') or '').strip() diff --git a/map/auto_map.py b/map/auto_map.py index 5f5134b..5e6b732 100644 --- a/map/auto_map.py +++ b/map/auto_map.py @@ -13,13 +13,18 @@ # limitations under the License. import ast +import hashlib +import os import random import io +import tempfile +import threading import math import re from collections import Counter from pathlib import Path +import numpy as np from PIL import Image, ImageDraw, ImageFont from .constants import europe_coords, world_coords @@ -166,7 +171,176 @@ } +# --------------------------------------------------------------------------- +# Per-process caches for decoded artwork, prepared overlays and region-ID maps. +# Cached images are shared; callers must copy them before mutating. +# --------------------------------------------------------------------------- + +_REGION_CACHE_VERSION = 1 +_CACHE_LOCK = threading.Lock() +_IMAGE_CACHE: dict[tuple, Image.Image] = {} +_RELIEF_CACHE: dict[tuple, Image.Image] = {} +_REGION_CACHE: dict[tuple, tuple[np.ndarray, np.ndarray]] = {} + + +def _file_key(path: Path) -> tuple | None: + """Identity of an artwork file; None when it cannot be stat'ed (no caching).""" + try: + st = os.stat(path) + except OSError: + return None + return (str(Path(path).resolve()), st.st_mtime_ns, st.st_size) + + +def _load_rgba(path: Path) -> Image.Image: + """Decode ``path`` as RGBA once per process. Returned image is shared: copy before mutating.""" + key = _file_key(path) + if key is not None: + with _CACHE_LOCK: + cached = _IMAGE_CACHE.get(key) + if cached is not None: + return cached + img = Image.open(path).convert("RGBA") + if key is not None: + img.load() + with _CACHE_LOCK: + _IMAGE_CACHE[key] = img + return img + + +def _prepared_relief(path: Path, size: tuple[int, int]) -> Image.Image: + """Relief overlay resized to ``size`` with reduced opacity (shared, do not mutate).""" + key = _file_key(path) + ckey = None if key is None else (key, tuple(size)) + if ckey is not None: + with _CACHE_LOCK: + cached = _RELIEF_CACHE.get(ckey) + if cached is not None: + return cached + relief = _load_rgba(path) + relief = relief.resize(size) if relief.size != tuple(size) else relief.copy() + _, _, _, a = relief.split() + relief.putalpha(a.point(_RELIEF_ALPHA_LUT)) + if ckey is not None: + with _CACHE_LOCK: + _RELIEF_CACHE[ckey] = relief + return relief + + +def _region_thresh(name: str) -> int: + """Return the flood-fill color tolerance for a region.""" + # Tolerances are tuned to the shading in the source map images. + if name == "south_america": + return 15 + if name == "asia": + return 20 + return 30 + + +def _region_cache_dir() -> Path: + """Return the directory used to cache precomputed region-ID maps on disk.""" + env = os.environ.get("AUTOMAP_REGION_CACHE_DIR") + return Path(env) if env else Path(tempfile.gettempdir()) / "automap-region-cache" + + +def _compute_region_ids(base: Image.Image, coords: dict) -> np.ndarray: + """Flood fill every seed once, on a scratch copy, with a unique sentinel per region. + + Fills run in the same order and with the same tolerances as the original + per-render flood fill. Sentinels use alpha 128 so they differ by more than + any tolerance from both opaque and transparent artwork, which keeps + already-claimed regions acting as boundaries for later fills. + Returns an array of region indices (1-based, 0 = unpainted). + """ + work = base.copy() + for idx, (name, points) in enumerate(coords.items(), start=1): + sentinel = (idx & 0xFF, (idx >> 8) & 0xFF, 77, 128) + thresh = _region_thresh(name) + for coord in points: + ImageDraw.floodfill(work, xy=coord, value=sentinel, thresh=thresh) + orig = np.asarray(base) + new = np.asarray(work) + changed = np.any(orig != new, axis=2) + ids = new[..., 0].astype(np.uint16) | (new[..., 1].astype(np.uint16) << 8) + ids[~changed] = 0 + return ids + + +def _region_index(map_path: Path, base: Image.Image, coords: dict) -> tuple[np.ndarray, np.ndarray]: + """Return (flat pixel indices, region ids) for every pixel belonging to a region. + + Cached in memory per (artwork file, seeds) and on disk keyed by a hash of the + decoded pixels, seeds, and tolerances, so artwork or seed edits regenerate it. + """ + seeds = tuple((str(n), _region_thresh(n), tuple(tuple(map(int, c)) for c in pts)) + for n, pts in coords.items()) + fkey = _file_key(map_path) + mkey = None if fkey is None else (fkey, base.size, seeds) + if mkey is not None: + with _CACHE_LOCK: + hit = _REGION_CACHE.get(mkey) + if hit is not None: + return hit + + disk_path = None + if mkey is not None: + h = hashlib.sha256() + h.update(f"v{_REGION_CACHE_VERSION}|{base.size}|{seeds!r}|".encode()) + h.update(base.tobytes()) + disk_path = _region_cache_dir() / f"{h.hexdigest()}.npz" + try: + with np.load(disk_path) as data: + result = (data["flat"], data["ids"]) + log.info("Loaded cached region map") + except (OSError, KeyError, ValueError): + result = None + if result is not None: + with _CACHE_LOCK: + _REGION_CACHE[mkey] = result + return result + + log.info("Building region map (one-time per base map)") + ids = _compute_region_ids(base, coords).ravel() + flat = np.flatnonzero(ids) + id_dtype = np.uint8 if len(coords) < 256 else np.uint16 + result = (flat.astype(np.uint32), ids[flat].astype(id_dtype)) + + if mkey is not None: + with _CACHE_LOCK: + _REGION_CACHE[mkey] = result + try: + disk_path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=disk_path.parent, suffix=".npz.tmp") + with os.fdopen(fd, "wb") as fh: + np.savez(fh, flat=result[0], ids=result[1]) + os.replace(tmp, disk_path) + except OSError: + pass + return result + + +def _paint_regions(img: Image.Image, flat: np.ndarray, ids: np.ndarray, + colors: dict[int, tuple[int, int, int, int]]) -> Image.Image: + """Return a copy of ``img`` with region ``i`` painted ``colors[i]``.""" + if not colors: + return img.copy() + n = int(max(colors)) + 1 + if ids.size: + n = max(n, int(ids.max()) + 1) + lut = np.zeros((n, 4), dtype=np.uint8) + use = np.zeros(n, dtype=bool) + for i, c in colors.items(): + lut[i] = c + use[i] = True + arr = np.array(img, dtype=np.uint8) + pix = arr.reshape(-1, 4) + sel = use[ids] + pix[flat[sel]] = lut[ids[sel]] + return Image.fromarray(arr) + + def _resolve_theme_name(palette_theme: str = "auto", prompt: str = "", mode: str = "text") -> str: + """Pick a palette theme name from an explicit choice, prompt keywords, or mode.""" theme = (palette_theme or "auto").strip().lower() if theme in _PALETTE_THEMES and theme != "auto": return theme @@ -183,6 +357,7 @@ def _resolve_theme_name(palette_theme: str = "auto", prompt: str = "", mode: str def _hex_to_rgba(val: object, default: tuple[int, int, int, int]) -> tuple[int, int, int, int]: + """Parse a ``#RRGGBB`` hex string into an opaque RGBA tuple, or return ``default``.""" if not isinstance(val, str): return default clean = val.strip().lstrip("#") @@ -192,6 +367,7 @@ def _hex_to_rgba(val: object, default: tuple[int, int, int, int]) -> tuple[int, def _to_rgba_tuple(val: object) -> tuple[int, int, int, int] | None: + """Coerce an RGB/RGBA sequence or ``#RRGGBB`` hex string into a clamped RGBA tuple.""" if isinstance(val, (list, tuple)) and len(val) in (3, 4): try: r, g, b = int(val[0]), int(val[1]), int(val[2]) @@ -208,6 +384,7 @@ def _parse_filling_overrides( raw_txt: object, raw_num: object, ) -> tuple[dict[str, tuple[int, int, int, int]], dict[tuple[float, float], tuple[int, int, int, int]]]: + """Normalize optional text and numeric filling overrides into RGBA lookup dicts.""" txt_out: dict[str, tuple[int, int, int, int]] = {} num_out: dict[tuple[float, float], tuple[int, int, int, int]] = {} @@ -370,7 +547,9 @@ def auto_map(prompt: str, mode: str, map_type: str, size_mod: int, auto_style: bool | None = None, palette_theme: str = "auto", ai_answer_override: dict[str, object] | None = None, - temperature: float | None = None): + temperature: float | None = None, + web_search: bool = False, + max_search_requests: int = 5): """Create a map from an AI query and save it as ``result.png``. CLI options: @@ -419,6 +598,12 @@ def get_font(size): except (ValueError, TypeError, OverflowError): label_stroke_width = 15 label_auto_fit = bool(ls.get("auto_fit", False)) if "auto_fit" in ls else False + try: + raw_offset = ls.get('offset_x', 0) + offset = float(raw_offset) if not isinstance(raw_offset, bool) else 0 + label_offset_x = max(-500, min(500, round(offset))) if math.isfinite(offset) else 0 + except (TypeError, ValueError, OverflowError): + label_offset_x = 0 # Most least and short form conflict solution # @TODO Make most/least and short form compatible @@ -464,7 +649,7 @@ def get_font(size): # opening selected map log.step(f"Loading base map: {path.name}") - img = Image.open(path).convert("RGBA") + img = _load_rgba(path) # shared cached copy; never mutated in place log.info(f"Base map size: {img.size[0]}x{img.size[1]}") # Keep labels separate so later flood fills and overlays cannot paint over them. @@ -477,6 +662,9 @@ def get_font(size): # Requesting information from AI log.step("Requesting data from AI") ai_options = {} if temperature is None else {'temperature': temperature} + if web_search: + ai_options["web_search"] = True + ai_options["max_search_requests"] = max_search_requests ai_answer = ai_request(prompt, map_type, model, api_key, **ai_options) if ai_answer is None: @@ -551,6 +739,8 @@ def get_font(size): if "oceania" in ai_answer_norm and "australia" not in ai_answer_norm: ai_answer_norm["australia"] = ai_answer_norm["oceania"] + region_colors: dict[int, tuple[int, int, int, int]] = {} + for index, (name, points) in enumerate(dict_.items(), start=1): name_key = str(name).strip().lower().replace(" ", "_").replace("-", "_") '''Number mode''' @@ -616,20 +806,13 @@ def get_font(size): # Flag maps retain their original artwork. Other maps use every seed point # to fill disconnected pieces of a region, such as islands. + # Painting is applied after the loop from a cached region-ID map. if map_type == 'default' or map_type == 'world': - for coord in points: - # Tolerances are tuned to the shading in the source map images. - if name == "south_america": - thresh = 15 - elif name == "asia": - thresh = 20 - else: - thresh = 30 - - ImageDraw.floodfill(img, xy=coord, value=color, thresh=thresh) + region_colors[index] = tuple(int(c) for c in color) # The first fill seed also anchors the region's single label. x, y = points[0] + x += label_offset_x info = str(info) @@ -703,28 +886,27 @@ def get_font(size): "stroke_color": f"#{label_stroke_fill[0]:02x}{label_stroke_fill[1]:02x}{label_stroke_fill[2]:02x}", "stroke_width": label_stroke_width, "auto_fit": label_auto_fit, + "offset_x": label_offset_x, "recommended_font": font_name or ("Bips.ttf" if mode == "number" else "DIN Condensed.ttf"), "recommended_gsm": size_mod, }, } + if map_type == 'default' or map_type == 'world': + flat, ids = _region_index(path, img, dict_) + img = _paint_regions(img, flat, ids, region_colors) + # adding relief map log.step("Compositing layers") if map_type in ['default', 'flag']: log.info("Adding relief + borders overlay") - relief = Image.open(Path(__file__).parent.parent / "./resources/maps/map_relief.png").convert("RGBA") - relief = relief.resize(img.size) - - # Reduce relief opacity to preserve the generated region colors beneath it. - _, _, _, a = relief.split() - relief.putalpha(a.point(_RELIEF_ALPHA_LUT)) + # Relief is resized and its opacity reduced once per process. + relief = _prepared_relief(Path(__file__).parent.parent / "./resources/maps/map_relief.png", img.size) result = Image.alpha_composite(img, relief) # Restore borders above the relief, then put readable labels on top. - borders = Image.open( - Path(__file__).parent.parent / "./resources/maps/default_map_blured.png" - ).convert("RGBA") + borders = _load_rgba(Path(__file__).parent.parent / "./resources/maps/default_map_blured.png") result = Image.alpha_composite(result, borders) result = Image.alpha_composite(result, text_layer) @@ -735,17 +917,11 @@ def get_font(size): # merging layers elif map_type == 'world': log.info("Adding world relief + borders overlay") - relief = Image.open(Path(__file__).parent.parent / "./resources/maps/world_map_relief.png").convert("RGBA") - if relief.size != img.size: - relief = relief.resize(img.size) - - # Reduce relief opacity to preserve the generated region colors beneath it. - _, _, _, a = relief.split() - relief.putalpha(a.point(_RELIEF_ALPHA_LUT)) + relief = _prepared_relief(Path(__file__).parent.parent / "./resources/maps/world_map_relief.png", img.size) result = Image.alpha_composite(img, relief) - borders = Image.open(Path(__file__).parent.parent / "./resources/maps/world_map_blured.png").convert("RGBA") + borders = _load_rgba(Path(__file__).parent.parent / "./resources/maps/world_map_blured.png") result = Image.alpha_composite(result, borders) result = Image.alpha_composite(result, text_layer) diff --git a/map/functions.py b/map/functions.py index ed8e689..c21e7f1 100644 --- a/map/functions.py +++ b/map/functions.py @@ -2,8 +2,10 @@ import io import json import math +import os import re import sys +import requests try: import httpx2 as httpx except ImportError: # Older google-genai releases use the original transport. @@ -20,6 +22,24 @@ load_dotenv(".env") # loading .env file AI_TIMEOUT_MS = 45_000 +DEEPSEEK_API_URL = 'https://api.deepseek.com' + + +def provider_for_model(model): + """DeepSeek models use their own endpoint and credentials.""" + return 'deepseek' if str(model).lower().startswith('deepseek-') else 'gemini' + + +def provider_key_name(provider): + """Return the environment/settings key name for the given AI provider.""" + return 'DEEPSEEK_API_KEY' if provider == 'deepseek' else 'GEMINI_API_KEY' + + +def configured_api_key(provider, settings=None): + """Return the configured API key for ``provider``, ignoring empty or ``your_*`` placeholders.""" + name = provider_key_name(provider) + value = (os.environ.get(name) or (settings or {}).get(name) or '').strip() + return value if value and not value.lower().startswith('your_') else None class AIServiceError(RuntimeError): @@ -30,6 +50,166 @@ def __init__(self, message, status_code=503): self.status_code = status_code +def _deepseek_json(path, api_key, *, payload=None, timeout=45): + """Call only the fixed DeepSeek origin; never expose upstream credential errors.""" + if not api_key: + raise AIServiceError('DeepSeek API key required. Set DEEPSEEK_API_KEY or enter it in the web form.', 400) + headers = {'Authorization': f'Bearer {api_key}', 'Accept': 'application/json'} + try: + if payload is None: + response = requests.get(f'{DEEPSEEK_API_URL}/{path}', headers=headers, + timeout=(10, timeout), allow_redirects=False) + else: + response = requests.post(f'{DEEPSEEK_API_URL}/{path}', headers=headers, json=payload, + timeout=(10, timeout), allow_redirects=False) + except requests.Timeout as error: + raise AIServiceError(f'DeepSeek did not respond within {timeout} seconds. Try again or select another model.', 504) from error + except requests.RequestException as error: + raise AIServiceError('Could not connect to DeepSeek. Please try again later.', 503) from error + try: + status = response.status_code + if status in (401, 403): + raise AIServiceError('DeepSeek rejected the API key. Check your key and account access.', 400) + if status == 402: + raise AIServiceError('Your DeepSeek account has insufficient balance. Top up your DeepSeek account.', 402) + if status == 429: + raise AIServiceError('DeepSeek request limit reached. Please try again later.', 429) + if status >= 500: + raise AIServiceError('DeepSeek is overloaded or temporarily unavailable. Try again shortly or select another model.', 503) + if status != 200: + raise AIServiceError('DeepSeek could not process the request. Check the selected model and API access.', 400) + try: + result = response.json() + except ValueError as error: + raise AIServiceError('DeepSeek returned an invalid response. Please try again.', 502) from error + if not isinstance(result, dict) or result.get('error'): + raise AIServiceError('DeepSeek could not complete the request. Please try again.', 502) + return result + finally: + response.close() + + +def list_deepseek_models(api_key): + """Return sorted ``deepseek-*`` model IDs available for ``api_key``.""" + result = _deepseek_json('models', api_key, timeout=15) + models = result.get('data') + if not isinstance(models, list): + raise AIServiceError('DeepSeek returned an invalid model list.', 502) + return sorted({model['id'] for model in models if isinstance(model, dict) + and isinstance(model.get('id'), str) and model['id'].startswith('deepseek-')}) + + +def _web_search_ddg(query, max_results=5): + """Fetch top web search snippets from Wikipedia's public search API without external dependencies.""" + # Use MediaWiki's JSON search API instead of scraping HTML search pages that return HTTP 202 bot challenges. + try: + resp = requests.get( + 'https://en.wikipedia.org/w/api.php', + params={ + 'action': 'query', + 'list': 'search', + 'srsearch': str(query), + 'srlimit': int(max_results), + 'format': 'json', + 'utf8': 1, + }, + headers={'User-Agent': 'AutoMap-cli/1.0 (https://github.com/deltaaa00/AutoMap-cli)'}, + timeout=(5, 10), + ) + if resp.status_code != 200: + return f'Search failed (HTTP {resp.status_code}).' + data = resp.json() + except (requests.RequestException, ValueError) as err: + return f'Search error: {err}' + + results = data.get('query', {}).get('search') or [] + lines = [] + for item in results[:max_results]: + title = str(item.get('title') or '').strip() + snippet = re.sub(r'<[^>]+>', '', str(item.get('snippet') or '')).replace('"', '"').replace('&', '&').strip() + if title or snippet: + lines.append(f'- {title}: {snippet}' if title and snippet else f'- {title or snippet}') + return '\n'.join(lines) if lines else 'No search results found.' + + +def _deepseek_request(prompt, model, api_key=None, *, temperature=None, web_search=False, max_search_requests=5): + """Send a chat-completion request to DeepSeek, optionally running a bounded web_search tool loop.""" + sys_msg = 'Return only a JSON object mapping region names to values. Example: {"ukraine": 42}.' + if web_search: + sys_msg += ' You have access to the web_search tool—use it to look up accurate, up-to-date facts and statistics before answering.' + messages = [ + {'role': 'system', 'content': sys_msg}, + {'role': 'user', 'content': prompt}, + ] + payload = { + 'model': model, + 'messages': messages, + 'response_format': {'type': 'json_object'}, + 'thinking': {'type': 'disabled'}, + 'max_tokens': 4096, + 'stream': False, + } + if web_search: + payload['tools'] = [{ + 'type': 'function', + 'function': { + 'name': 'web_search', + 'description': 'Search the web for up-to-date facts, rankings, and statistics.', + 'parameters': { + 'type': 'object', + 'properties': {'query': {'type': 'string', 'description': 'Search query'}}, + 'required': ['query'], + }, + }, + }] + if temperature is not None: + payload['temperature'] = validate_temperature(temperature) + + # Track remaining search calls so the model cannot loop indefinitely on tool_calls. + budget = max(1, min(10, int(max_search_requests or 5))) + key = api_key or configured_api_key('deepseek') + result = _deepseek_json('chat/completions', key, payload=payload) + + while web_search and budget > 0: + try: + choice = result['choices'][0] + msg = choice['message'] + except (KeyError, IndexError, TypeError) as error: + raise AIServiceError('DeepSeek returned no map data. Please try again.', 502) from error + tool_calls = msg.get('tool_calls') or [] + if choice.get('finish_reason') != 'tool_calls' or not tool_calls: + break + + # Append the assistant's tool-call turn, then answer each tool call in order. + messages.append(msg) + for tc in tool_calls: + if budget <= 0: + messages.append({'role': 'tool', 'tool_call_id': tc['id'], 'content': 'Search budget reached. Return final JSON now.'}) + continue + budget -= 1 + try: + args = json.loads(tc.get('function', {}).get('arguments') or '{}') + q = args.get('query') or prompt + except ValueError: + q = prompt + messages.append({'role': 'tool', 'tool_call_id': tc['id'], 'content': _web_search_ddg(q)}) + + # Once the search budget is spent, strip `tools` so DeepSeek must return the final JSON. + if budget <= 0: + payload.pop('tools', None) + result = _deepseek_json('chat/completions', key, payload=payload) + try: + choice = result['choices'][0] + content = choice['message']['content'] + except (KeyError, IndexError, TypeError) as error: + raise AIServiceError('DeepSeek returned no map data. Please try again.', 502) from error + if choice.get('finish_reason') == 'length': + raise AIServiceError('DeepSeek returned an incomplete map. Try a shorter prompt.', 502) + if not isinstance(content, str) or not content.strip(): + raise AIServiceError('DeepSeek returned no map data. Please try again.', 502) + return content + + def _error(title: str, message: str = ""): """Print an error message to stderr (CLI-safe replacement for tkinter showerror).""" log.error(f"{title}: {message}" if message else title) @@ -52,7 +232,7 @@ def validate_temperature(value: object) -> float: return temperature -def _ai_request(prompt, mode, ai_model, api_key=None, *, temperature=None) -> str | None: +def _ai_request(prompt, mode, ai_model, api_key=None, *, temperature=None, web_search=False, max_search_requests=5) -> str | None: """Send a request to the AI model and return the response as a string.""" if mode == 'default' or mode == 'flag': @@ -76,6 +256,12 @@ def _ai_request(prompt, mode, ai_model, api_key=None, *, temperature=None) -> st config = {} if temperature is None else { 'config': types.GenerateContentConfig(temperature=validate_temperature(temperature)), } + if provider_for_model(ai_model) == 'deepseek': + with log.spinner('Waiting for DeepSeek response...'): + return _deepseek_request( + prompt_to_ai, ai_model, api_key, + temperature=temperature, web_search=web_search, max_search_requests=max_search_requests, + ) try: with log.spinner("Waiting for AI response..."): # If a key was supplied (e.g. from the web UI when .env is missing), @@ -123,12 +309,15 @@ def _ai_request(prompt, mode, ai_model, api_key=None, *, temperature=None) -> st def ai_request(prompt: str, mode: str, ai_model: str, api_key: str | None = None, _ai_answer = None, - *, temperature: float | None = None) -> dict | None: + *, temperature: float | None = None, web_search: bool = False, max_search_requests: int = 5) -> dict | None: "handle the AI request and parse the response into a dictionary." if _ai_answer is not None: request = _ai_answer else: options = {} if temperature is None else {'temperature': temperature} + if web_search: + options['web_search'] = True + options['max_search_requests'] = max_search_requests request = _ai_request(prompt, mode, ai_model, api_key, **options) # The AI request can fail and return None (the error is already logged). @@ -167,6 +356,11 @@ def ai_request(prompt: str, mode: str, ai_model: str, api_key: str | None = None def load_filling(): + """Load text and numeric color-filling rules from ``filling.json`` (or ``filling.template.json``). + + Returns ``(filling_txt, filling_num)`` where ``filling_txt`` maps label strings + to RGBA tuples and ``filling_num`` maps ``(low, high)`` numeric bounds to RGBA tuples. + """ path = Path(__file__).parent / "filling.json" if not path.is_file(): diff --git a/pyproject.toml b/pyproject.toml index 76e61e9..b519aeb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "requests>=2.31.0", "pydub>=0.25.1", "moviepy>=2.0.0", + "proglog>=0.1.10,<1.0.0", ] [project.scripts] diff --git a/readme.md b/readme.md index fe7e519..3e710df 100644 --- a/readme.md +++ b/readme.md @@ -1,229 +1,316 @@ -# AutoMap-cli (AutoMap v2.0) +# AutoMap -![Logo](resources/demo/banner.png) +![AutoMap](resources/demo/banner.png) -**AutoMap** is a map-generating tool. Describe what you want in plain language and AutoMap uses AI to collect the data, then paints it onto a map for you — a coloured, labelled map in seconds. +AutoMap turns a natural-language topic into a colored, labeled map and can place that map in a short portrait video. It includes a web interface, a command-line interface, and a Python renderer. -It ships with three parts: +Choose **Google Gemini** or **DeepSeek** for map data. Customize the palette and labels, generate several maps from a CSV file, or open **Video Studio** to combine a map with background footage, a headline, branding, and optional music. -- a **CLI** (`cli.py` / the `automap` command), -- a **web UI** (a small Flask app in `main.py`), -- and a Python API (`map.auto_map.auto_map`). +AI-generated values can be approximate or incorrect. Check the underlying data before publishing a map; a prompt asking for current information does not guarantee verified sources or live web research. -## Features +## What you can make -AutoMap can render three map types: +| Map type | Coverage | Appearance | +| --- | --- | --- | +| `default` | European countries, with the regions included in the bundled artwork | Filled regions with relief and borders | +| `flag` | The same European layout | Flag artwork with labels | +| `world` | Continents | Filled continent regions with relief and borders | -| Map type | Description | -| --------- | ------------------------------- | -| `default` | Europe (default map) | -| `flag` | Europe with flag-style borders | -| `world` | World map grouped by continent | +Maps support categorical **text** values and **number** values. Numeric palettes preserve decimal values and interval boundaries. Short numeric strings such as `10k`, `2.5M`, and `1B` are understood by numeric coloring and Most/Least calculations. -Data can be shown in two modes: +The web interface provides: -- **`text`** — categorical values (e.g. `Coffee` → a specific colour). -- **`number`** — numeric values, coloured by value ranges. +- Separate **Gemini** and **DeepSeek** sections in the model selector, with independent API keys. +- Single-map generation and CSV batch generation. +- **Theme & Style** in the left panel, opening Color Filling Studio for palettes, category rules, numeric intervals, and label styling. +- Repainting from an existing map's values without another AI request. +- Palette inspection from an uploaded image. This samples colors; it does not recover the image's underlying country data. +- PNG downloads, run history, and transfer of the latest generated map to Video Studio. +- Video Studio at `/studio`, with title previews, custom overlays, music download/upload and trimming, and MP4 rendering. -Optional extras: +![Example map](resources/demo/map_example.png) -- `show_most_least` — highlight the country/continent with the most and least of a value. -- `short_form` — shorten large numbers (e.g. `1000000` → `1M`). +## Quick start with Docker -Here is an example (*default map*): +Docker Compose starts both required services: the web application and the media worker. The worker image includes FFmpeg, Node.js, and the Python media dependencies. -![Map example](resources/demo/map_example.png) +```sh +git clone https://github.com/deltaaa00/AutoMap-cli.git +cd AutoMap-cli +git switch dev +cp example.env .env +``` + +Edit `.env`. Configure either API key, both keys, or leave their placeholder values and enter a key in the browser instead. Set your own login credentials before exposing the app to other users. -## Stack +```dotenv +GEMINI_API_KEY=your_google_ai_studio_key +DEEPSEEK_API_KEY=your_deepseek_api_key +SERVER=TRUE +WEB_LOGIN=your_username +WEB_PASSWORD=your_password +# Optional: use a stable, random signing secret instead of the generated file. +# WEB_SECRET_KEY=your_long_random_secret +``` + +```sh +docker compose up --build -d +``` + +Open [http://localhost:3000](http://localhost:3000). With `SERVER=TRUE`, sign in using the credentials from `.env`. + +Useful commands: + +```sh +docker compose logs -f web yt-dlp-worker +docker compose up --build -d # Apply code, dependency, or configuration changes. +docker compose down # Stop and remove containers; retain named volumes. +``` -Libraries AutoMap uses today: +Compose publishes only the web application (`3000` on the host to `5000` in the container). The media worker listens on `8080` inside the private Compose network. Both services use `restart: unless-stopped`. -- **AI** - - [google-genai](https://github.com/googleapis/python-genai) -- **Drawing** - - [pillow](https://github.com/python-pillow/Pillow) -- **Web UI** - - [flask](https://github.com/pallets/flask) -- **CLI** - - [click](https://github.com/pallets/click) - - [rich](https://github.com/Textualize/rich) -- **Other** - - [python-dotenv](https://github.com/theskumar/python-dotenv) +The `automap-session` volume retains the generated session secret. The `automap-media` volume stores worker media, which is still subject to expiry cleanup. Removing a volume removes its contents. -## Installation +## API keys and model selection -To get started with **AutoMap**, follow these steps: +Get a key from [Google AI Studio](https://aistudio.google.com/apikey) or the [DeepSeek platform](https://platform.deepseek.com/). -1. **Clone the repository:** - ```bash - git clone https://github.com/deltaaa00/auto_map.git - cd automap - ``` +| Provider section | Server setting | Built-in default or fallback | +| --- | --- | --- | +| Gemini, including available Gemma models | `GEMINI_API_KEY` | Default: `gemini-3.8-flash` | +| DeepSeek | `DEEPSEEK_API_KEY` | `deepseek-flash`, `deepseek-v4-pro` | -2. **Set up a virtual environment:** - ```bash - python -m venv venv - # Windows: - venv\Scripts\activate - # macOS / Linux: - source venv/bin/activate - ``` +The web app requests the models available for the active provider's key and keeps built-in options if discovery is unavailable. Model availability, quota, and billing depend on the provider and account. Use `python cli.py make --help` to see the CLI's supported model choices. -3. **Install dependencies:** - ```bash - pip install -r requirements.txt - ``` +Keys can come from the process environment or the project-root `.env` file. Environment settings take precedence over file settings. Compose forwards both provider keys from its launching environment or `.env`. - For testing, install `requirements-test.txt` instead. It includes runtime - dependencies and test tools. See [tests/README.md](tests/README.md) for test commands. +In the browser: -4. **Configure your API key:** - AutoMap reads `GEMINI_API_KEY` from a `.env` file (see `example.env`). - - Copy `example.env` to `.env`. - - Get a key from [Google AI Studio](https://aistudio.google.com/apikey). - - Set it in `.env`: `GEMINI_API_KEY=your_key_here`. +1. Choose a model from the **Gemini** or **DeepSeek** section. +2. If that provider has no configured server key, enter its key in the displayed field. +3. The shared key field switches with the selected model. Keys are saved separately in this browser for the next visit: switching providers restores the matching key, and clearing the current provider's field leaves the other saved key untouched. - On Windows/macOS you can also use the helper scripts in `util/` (`key_man.bat`, `key_man.command`) to manage the key. +Browser-supplied keys are sent to the application server for the active provider's requests. Server-configured keys are not returned to the browser. Browser persistence requires storage to remain enabled; it is specific to the browser profile and site origin. -5. **Optionally customize your filling configuration:** - The bundled template is used automatically when `map/filling.json` is absent. - Copy it to override the default rules: - ```bash - cp map/filling.template.json map/filling.json - ``` +DeepSeek uses its official Chat Completions API with JSON output and thinking disabled for map generation. No separate OpenAI SDK is required. API requests have a 45-second timeout and no application-level automatic retry. Map drawing takes additional time. Provider overload, quota limits, invalid credentials, insufficient DeepSeek balance, and timeouts are reported in the interface. -## Usage +## Run from the source checkout -### Web UI +Use **Python 3.12**, matching the Docker images and test workflow. Keep the repository's `resources/` directory available: the renderer uses its map artwork and fonts. -Start the Flask server: +```sh +python3.12 -m venv .venv +source .venv/bin/activate +# Windows PowerShell: .venv\Scripts\Activate.ps1 +python -m pip install -r requirements.txt +``` + +For map generation through the CLI, this is enough once an API key is configured. To use the `automap` entry point while keeping assets in the checkout: + +```sh +python -m pip install -e . +``` + +For native Video Studio operation, also install FFmpeg and make its executable available on `PATH`. URL audio downloads use the `yt-dlp` executable installed by the requirements; Node.js is also included in the Docker worker for supported download workflows. Docker is the simplest way to run the complete media stack. + +To run the web app and worker outside Compose, start them in separate terminals from the project root. Activate the virtual environment in each terminal. + +```sh +# Terminal 1: media worker, default port 8080. +python yt-dlp_serv.py +``` -```bash +```sh +# Terminal 2: web app, default port 5000. +export YTDLP_URL=http://127.0.0.1:8080 python main.py ``` -Then open in your browser. If no key is found in `.env`, the UI prompts you to paste one; otherwise it uses the key from `.env` automatically. +On PowerShell, set the worker URL with `$env:YTDLP_URL = 'http://127.0.0.1:8080'`. Open [http://localhost:5000](http://localhost:5000). -With `SERVER=TRUE`, sign in with `deltaaa` / `deltaaa00!`, or set `WEB_LOGIN` and `WEB_PASSWORD` in the environment or `.env` to use your own credentials. Environment values take precedence. The browser remembers your sign-in for up to a year. Changing the credentials invalidates existing sessions. In development mode, pages and APIs open without login. +Only the exact value **`SERVER=TRUE`** enables production mode: Waitress, debugging off, and web authentication. Other values use Flask's development server with debugging enabled and no login gate. The supplied `example.env` enables production mode. -The entered Google API key, last map topic, and palette are remembered in the same browser. Browser storage must remain enabled to keep these preferences between visits. +## Web workflow -Open **Theme & Style** in the left panel to choose a map palette in Color Filling -Studio. Selecting a batch result restores that map's generation settings before -you repaint it. +### Generate and style a map -The model list is refreshed for your API key, with an updated offline fallback -list shared by the web interface and CLI. The default is `gemini-3.8-flash`. -AI generation uses a 45-second request timeout without automatic retries. If a -model is overloaded, rate-limited, or times out, the page displays the reason and -lets you retry or choose another model. Drawing the map takes additional time. +Choose a model, enter a topic, and select text or numeric mode and a map type. Adjust font size, font, temperature, and the optional feature, then generate. -In Video Studio, use **Or upload your own music** to select an MP3, WAV, M4A, AAC, OGG or FLAC file up to 10 MB. The track is converted to MP3 and can be previewed, cut or used directly in the rendered video. The source track and selected cut are saved in this browser so they remain available after reopening the studio. +Open **Theme & Style** from the left panel to choose a palette or edit category colors, numeric ranges, and label appearance. **Apply & Re-Paint Current Map** reuses the generated values, so changing colors does not require another AI call. Selecting a batch result restores its generation parameters before repainting it. -Music downloaded from a URL must also fit the 10 MiB MP3 limit. Larger tracks -are rejected immediately; select a shorter track. Each download has its own ID, -so matching song titles cannot overwrite another track or reuse an older cut. +The **Global label position** slider in the main left panel moves all country or +continent labels horizontally, from −500 to +500 pixels. Negative values move +left; positive values move right. The value is remembered in this browser. Use +**Apply & Re-Paint Current Map**, next to the generation controls, to apply the +shift without another AI request. The slider's **Reset** restores zero offset. -### Running on a server +The generated map's topic travels with its image into Video Studio and fills the +video headline automatically. Batch selection and restored history use the topic +of the selected map. Headlines you edit manually are kept when returning to the +studio or selecting another generated map. -Install the updated dependencies and set `SERVER=TRUE` in the environment or `.env`, then run `python main.py`. This starts the production Waitress server with debugging disabled. Without `SERVER=TRUE`, the Flask development server runs with debugging enabled. `PORT` defaults to `5000` for the web service and `8080` for the media worker. +The temperature slider accepts `0`–`2` and defaults to `0.75` in the web UI. It controls sampling where the selected model supports it; it does not ensure factual accuracy, and some models manage sampling themselves. -For Docker, set `SERVER=TRUE` in `.env`, run `docker compose up --build -d`, and open port `3000`. Compose forwards this switch to both services and keeps the media worker private to the Docker network. It passes `WEB_LOGIN`, `WEB_PASSWORD`, `WEB_SECRET_KEY`, and `GEMINI_API_KEY` from your environment or `.env` to the web container. Only the exact value `TRUE` activates production mode. +The browser remembers API keys, the last topic, and palette preferences. Video Studio also stores selected materials and settings locally. Storage limits or disabled storage can prevent persistence; download important outputs rather than treating browser storage as an archive. -You can use the server's exported environment variables directly; no `.env` file or code changes are required: +**Run History** is saved in this browser's IndexedDB with map images, values, and +generation settings, but without API keys. It survives reloads and reopening the +site. Runs older than 24 hours are removed when the page opens and checked every +minute while it is open. **Clear History** removes the saved history as well. -```bash -export SERVER=TRUE -export WEB_LOGIN='your-username' -export WEB_PASSWORD='your-password' -# Optional: export GEMINI_API_KEY='your-key' -docker compose up -d --build -``` +### Create a portrait video + +1. Generate a map or select a map image in Video Studio. +2. Supply background footage and a headline. +3. Optionally download a music track from a supported URL or upload MP3, WAV, M4A, AAC, OGG, or FLAC audio. Uploaded music is converted to MP3. +4. Use a full track or trim a **1–30 second** segment from an `MM:SS` start time. +5. Configure title font, size and colors, watermarks, stickers, and other overlays, then render and download the MP4. + +The worker renders a **1080 × 1920** portrait frame. Video duration is limited to **1–30 seconds** and to the available source media. Without selected music, rendering can proceed without audio; background footage's original audio is removed. -Compose reads the environment of the process that launches it. Export these variables in the launching shell, or supply them through the service manager running Compose. Environment values override `.env`. After changing variables, run `docker compose up -d` again to recreate affected containers; `docker compose restart` keeps the old values. Direct `python main.py` launches read the same exported variables. +The CPU encoder is `libx264`. NVIDIA `h264_nvenc` and macOS `h264_videotoolbox` are selectable only when detected by the worker. Installing an encoder does not guarantee compatible hardware or drivers. -Both containers restart automatically after a failure or host reboot, unless manually stopped. For a deployment outside Compose, set `YTDLP_URL` to the worker address (for example `http://127.0.0.1:8080`); its default is `http://yt-dlp-worker:8080`. +During rendering, a progress bar shows the worker's audio/video encoding progress +and current phase. It reaches 100% only after the finished MP4 is ready. The +sticker library also includes mountains, volcanoes, islands, weather, robots, +satellites, laptops, graduation caps, medals, cats, dogs, fish, burgers, and trains; +these have offline artwork as well as the existing sticker styles. -Session signing uses a generated secret in `instance/web-session-secret`; retain this file across restarts. Compose stores it in the `automap-session` volume. Alternatively, set a stable, random `WEB_SECRET_KEY` on every instance. Removing the secret file or volume, or changing the secret, requires signing in again. Serve the public site over HTTPS to protect the login and API key in transit. +Maps, background footage, audio, and custom overlay materials have a **10 MiB per-material limit**. URL downloads are also rejected if the resulting MP3 exceeds that limit. A long track may therefore need to be replaced with a shorter source. Each audio download has a unique identifier, independent of its display title. -### CLI +Worker files older than approximately **30 minutes** are cleaned up when relevant operations run. Download completed videos promptly. Browser-restored inline audio can keep a selected track usable after worker files have expired, subject to browser storage availability. -Generate a map straight from the terminal: +## Command-line usage -```bash +Run commands from the repository root: + +```sh python cli.py make "Population of European countries" -m number -mp default +python cli.py make "Most popular hot drink by country" -m text --model deepseek-flash +python cli.py make "GDP per capita by country" -m number -eop show_most_least --temperature 0.4 ``` -Flags: +`make` saves `result.png` in the repository root, replacing an existing result. `--show` opens the image after generation; it is off by default. + +| Option | Values or purpose | Default | +| --- | --- | --- | +| `-m`, `--mode` | `text`, `number` | `text` | +| `-mp`, `--map-type` | `default`, `flag`, `world` | `default` | +| `-gsm`, `--global-size-modifier` | Integer font-size adjustment | `15` | +| `-eop`, `--extra-option` | `none`, `show_most_least`, `short_form` | `none` | +| `--model` | Supported model ID; see command help | `gemini-3.8-flash` | +| `--font` | Bundled `.ttf` or `.otf` filename | `Bips.ttf` | +| `--key` | Key for the selected model's provider | Provider environment/file key | +| `--temperature` | Sampling value from `0` to `2`; `make` only | Provider default when omitted | +| `--show` / `--no-show` | Open the generated image; `make` only | `--no-show` | +| `-v`, `--verbose` | Detailed progress output | Off | -- `-m, --mode` — `text` (default) or `number` -- `-mp, --map-type` — `default` (default), `flag` or `world` -- `-gsm, --global-size-modifier` — global font size modifier (default `15`) -- `-eop, --extra-option` — `none` (default), `show_most_least` or `short_form` -- `--model` — AI model to use -- `--temperature` — response randomness from `0` to `2`; omit for the model default. The web slider defaults to `0.75` and remembers your selection. This controls sampling, not factual accuracy. -- `--font` — font file name from `resources/fonts` -- `--show/--no-show` — open the result image after generating -- `-v, --verbose` — print detailed progress output +`show_most_least` is supported in **numeric mode on Europe maps** (`default` or `flag`). `short_form` asks the model for shortened values such as `1k` or `2M` and is supported in **text mode**. These are mutually exclusive extra options. -After installing the package (`pip install -e .`) you can also use the `automap` command: +### Batch generation -```bash -automap make "Population of European countries" -m number -mp default +Start with [batch_template.csv](batch_template.csv), or create a UTF-8 CSV: + +```csv +prompt,mode,map_type,gsm,extra_option,model,font,palette_theme,output +"Population density in Europe",number,default,15,none,gemini-3.8-flash,Bips.ttf,viridis,europe_density.png +"Most popular drink in Europe",text,default,15,none,deepseek-flash,Bips.ttf,categorical,europe_drinks.png ``` -The result is saved as `result.png`. +```sh +python cli.py batch batch_template.csv -o output_maps +``` -## Fonts +Only `prompt` is required. Optional columns are `mode`, `map_type`, `gsm`, `extra_option`, `model`, `font`, `palette_theme`, and `output`. Unknown or duplicate columns and invalid values are rejected. CSV `gsm` must be an integer from `1` to `100`. Blank optional cells use command or web-form defaults. -AutoMap ships with a few default fonts. You can drop your own `.ttf` / `.otf` files into `resources/fonts` and select them on the map. +The batch command accepts the shared map/model/font/key/verbosity options above, plus `-o` / `--output-dir` (default: `output_maps`). It does not expose `--temperature` or `--show`. Rows run sequentially. Without `output`, filenames use a row number and a slug of the prompt; an explicit filename is confined to the output directory and given a `.png` extension if needed. -![Fonts](resources/demo/fonts.png) +Mixed-provider CSV files need credentials for each provider. A key entered in the browser or passed through CLI `--key` is applied only to its selected provider; it is not forwarded to the other provider. Configure both server/environment keys for mixed-provider batches. -## Filling +The web interface also accepts CSV ideas files and provides individual PNG downloads. Batch results remain available in the current page's result list and history. -You can define colours for your data in `map/filling.json`. It contains two dictionaries: +## Filling configuration and Python API -- **`filling_txt`** — colours for `text` mode. For example, if the data is `Tea`, the colour is `(246, 255, 0, 255)`. -- **`filling_num`** — colours for `number` mode, keyed by value ranges. For example, `(100,200)` means *value > 100 and value <= 200* → `(235, 40, 33, 255)`. +If `map/filling.json` is absent, AutoMap reads the bundled `map/filling.template.json`. To maintain your own default rules: -> AutoMap uses **RGBA**. The last number must be `255`. +```sh +cp map/filling.template.json map/filling.json +``` -```json -{ - "filling_txt": { - "Tea": [246, 255, 0, 255], - "Coffee": [54, 4, 4, 255] - }, - "filling_num": { - "(100,200)": [235, 40, 33, 255] - } -} +`filling_txt` maps text labels to RGBA arrays. `filling_num` maps stringified `(low, high)` pairs to RGBA arrays, using **`low < value <= high`**. Fractional interval boundaries are supported. The web editor can export a filling configuration. + +For Python use, the renderer accepts model/key settings, style overrides, and optional supplied values: + +```python +from map.auto_map import auto_map + +png = auto_map( + prompt="Example values", + mode="number", + map_type="default", + size_mod=15, + optional_feature="show_most_least", + model="gemini-3.8-flash", + font_name="Bips.ttf", + show_result=False, + io_res=True, + auto_style=True, + palette_theme="viridis", + ai_answer_override={"ukraine": "2M", "poland": 100000}, +) +if png is not None: + with open("example.png", "wb") as output: + output.write(png.getvalue()) ``` -## Deployment (Vercel) +`ai_answer_override` bypasses AI entirely. With `io_res=True`, a successful call returns a PNG `BytesIO` buffer; otherwise it saves and returns the path to `result.png`. Omitted countries or continents are not populated by this example. -The web UI can be deployed to Vercel. The repository root contains a `vercel.json` that builds `AutoMap-cli/main.py` with the Python runtime. +## Server configuration -1. Set `GEMINI_API_KEY` under **Project → Settings → Environment Variables** (do **not** commit a `.env`). -2. Deploy. If no key is configured, the UI simply prompts the visitor for one. +| Variable | Purpose | +| --- | --- | +| `GEMINI_API_KEY` | Server key for Gemini/Gemma models | +| `DEEPSEEK_API_KEY` | Server key for DeepSeek models | +| `SERVER` | Exact `TRUE` enables Waitress and web authentication | +| `WEB_LOGIN` | Login name; default `deltaaa` | +| `WEB_PASSWORD` | Login password; default `deltaaa00!` | +| `WEB_SECRET_KEY` | Optional stable session-signing secret | +| `WEB_AUTH_SECRET_FILE` | Native deployment override for the generated secret file | +| `PORT` | Native service port; web default `5000`, worker default `8080` | +| `YTDLP_URL` | Web-to-worker URL; default `http://yt-dlp-worker:8080` | -Note that AI + image generation can be slow, and Vercel's Hobby plan has short function time limits — a paid plan may be needed for larger requests. +Login sessions are valid for up to a year. Changing the credentials or signing secret invalidates existing sessions. Without `WEB_SECRET_KEY`, a secret is generated in `instance/web-session-secret`; keep that file across native restarts or retain the Compose session volume. For Compose, add an explicit environment entry to use `WEB_AUTH_SECRET_FILE`. Compose already fixes `YTDLP_URL` to the private worker address; edit that entry only if you change the service layout. -## License +Use your own credentials and HTTPS for a public deployment. Keep the media worker private; the web application provides the login gate. After changing Compose environment values, run `docker compose up -d` to recreate affected containers: `docker compose restart` retains the old values. -This project is licensed under the **Apache License 2.0**. +## Tests -Copyright © 2025-2026 **deltaaa00**. +The suite mocks external AI/HTTP downloads and does not require real credentials. It covers map rendering, decimal ranges and extrema, CLI/CSV behavior, provider/key isolation, authentication, service proxies, media handling, and browser interaction. Real software-encoded video rendering is included; live model/YouTube availability and GPU encoders are not tested. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +```sh +python -m pip install -r requirements-test.txt +python -m playwright install chromium +python -m pytest --cov --cov-report=term-missing +``` + +On Linux, install browser system dependencies with `python -m playwright install --with-deps chromium`. On macOS, browser tests use installed Google Chrome when available, otherwise Playwright Chromium. + +```sh +python -m pytest -m 'not browser' # Python checks without launching a browser. +python -m pytest -m browser # Browser interaction checks only. +``` - http://www.apache.org/licenses/LICENSE-2.0 +The coverage configuration requires **90%** coverage. See [tests/README.md](tests/README.md) for isolation details and test setup. The GitHub Actions workflow runs on pushes and pull requests targeting `main`. + +## Troubleshooting + +- **AI overload, timeout, or quota error:** retry later or select another available model. Drawing and encoding still take time after the AI response; AutoMap does not guarantee a fixed generation time. +- **Key input appears for one provider only:** that provider has no server key. Gemini and DeepSeek credentials are independent. +- **Video or music requests cannot reach the worker:** check `YTDLP_URL` for native launches and inspect `docker compose logs yt-dlp-worker` for Compose. +- **Audio download or conversion fails:** verify FFmpeg/yt-dlp availability, the source URL's accessibility, and the resulting file size. External sites may change or restrict downloads. +- **Files or preferences disappear after reopening:** browser storage may be disabled, cleared, or full; worker files also expire. Keep downloaded copies of important material. +- **A custom font or map asset is missing:** run from the source checkout and retain `resources/fonts/` and `resources/maps/`. + +## License -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +[Apache License 2.0](LICENSE). diff --git a/render_progress.py b/render_progress.py new file mode 100644 index 0000000..f230c42 --- /dev/null +++ b/render_progress.py @@ -0,0 +1,100 @@ +"""Thread-safe per-render progress for the single-process media worker.""" +from functools import wraps +from threading import Lock +from time import monotonic +from uuid import UUID + +from flask import current_app, g, jsonify, request +from proglog import ProgressBarLogger + + +class RenderProgress: + """Thread-safe in-memory store for tracking active and recent video renders.""" + + def __init__(self): + self.records = {} + self.lock = Lock() + + def update(self, render_id, *, progress=None, phase=None, status=None, error=None): + """Update fields for an existing render record; ignores unknown IDs.""" + with self.lock: + record = self.records.get(render_id) + if record is None: + return + if progress is not None: + # Clamp to [0, 100] and never regress progress on out-of-order callbacks. + record['progress'] = max(record['progress'], min(100, progress)) + for key, value in (('phase', phase), ('status', status), ('error', error)): + if value is not None: + record[key] = value + record['updated_at'] = monotonic() + + def snapshot(self, render_id): + """Return a copy of the public progress fields for ``render_id``, or ``None``.""" + with self.lock: + record = self.records.get(render_id) + return {key: value for key, value in record.items() if key != 'updated_at'} if record else None + + def track(self, function): + """Flask route decorator that registers, tracks, and finalizes a ``render_id``.""" + @wraps(function) + def tracked(*args, **kwargs): + data = request.get_json(silent=True) or {} + raw_id = data.get('render_id') + if raw_id is None: + return function(*args, **kwargs) + try: + render_id = str(UUID(str(raw_id))) + except ValueError: + return jsonify(status='error', error='Invalid render ID.'), 400 + with self.lock: + now = monotonic() + # Evict stale records older than 1 hour before checking capacity/duplicates. + self.records = {key: value for key, value in self.records.items() + if now - value['updated_at'] < 3600} + if render_id in self.records: + return jsonify(status='error', error='This render ID has already been used.'), 409 + if len(self.records) >= 256: + return jsonify(status='error', error='Render service is busy. Try again later.'), 503 + self.records[render_id] = dict(status='rendering', progress=0, + phase='Preparing materials', updated_at=now) + g.render_progress_id = render_id + try: + response = current_app.make_response(function(*args, **kwargs)) + payload = response.get_json(silent=True) or {} + if response.status_code < 400 and payload.get('status') == 'success': + self.update(render_id, status='success', progress=100, phase='Complete') + else: + self.update(render_id, status='error', phase='Render failed', + error=payload.get('error') or 'Video render failed.') + return response + except Exception: + self.update(render_id, status='error', phase='Render failed', error='Video render failed.') + raise + return tracked + + def logger(self): + """Return a Proglog logger bound to the current request's ``render_progress_id``.""" + render_id = getattr(g, 'render_progress_id', None) + if render_id is None: + return None + tracker = self + + class Logger(ProgressBarLogger): + """Bridge MoviePy/Proglog bar updates to ``RenderProgress`` percentage ranges.""" + + def bars_callback(self, bar, attr, value, old_value=None): + if attr != 'index': + return + total = self.bars[bar].get('total') or 0 + if total <= 0: + return + fraction = max(0, min(1, value / total)) + # Map audio chunking to 5-15% and video frame encoding to 15-95%. + if bar == 'chunk': + tracker.update(render_id, progress=round(5 + 10 * fraction), phase='Encoding audio') + elif bar == 'frame_index': + tracker.update(render_id, progress=round(15 + 80 * fraction), + phase='Finalizing video' if fraction == 1 else 'Encoding video') + + return Logger() diff --git a/requirements.txt b/requirements.txt index 4ae01cb..e81ca4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ rich>=13.0.0 yt-dlp>=2024.12.0 pydub>=0.25.1 moviepy>=2.0.0 +proglog>=0.1.10,<1.0.0 diff --git a/static/studio-storage.js b/static/studio-storage.js index df635d4..3958fbe 100644 --- a/static/studio-storage.js +++ b/static/studio-storage.js @@ -4,8 +4,14 @@ window.StudioStorage = (() => { function open() { if (!connection) { connection = new Promise((resolve, reject) => { - const request = indexedDB.open('automap-studio', 1); - request.onupgradeneeded = () => request.result.createObjectStore('assets'); + const request = indexedDB.open('automap-studio', 2); + request.onupgradeneeded = () => { + const db = request.result; + if (!db.objectStoreNames.contains('assets')) db.createObjectStore('assets'); + const history = db.objectStoreNames.contains('history') + ? request.transaction.objectStore('history') : db.createObjectStore('history', {keyPath: 'id'}); + if (!history.indexNames.contains('createdAt')) history.createIndex('createdAt', 'createdAt'); + }; request.onsuccess = () => { const db = request.result; db.onversionchange = () => { db.close(); connection = null; }; @@ -26,6 +32,7 @@ window.StudioStorage = (() => { const record = request.result; resolve(record ? { updatedAt: record.updatedAt, + topic: typeof record.topic === 'string' ? record.topic : '', file: record.blob ? new File([record.blob], record.name, { type: record.type, lastModified: record.lastModified, }) : null, @@ -35,11 +42,12 @@ window.StudioStorage = (() => { }); } - async function setAsset(key, file) { + async function setAsset(key, file, metadata = {}) { const db = await open(); const record = { blob: file, name: file?.name, type: file?.type, lastModified: file?.lastModified, updatedAt: Date.now(), + topic: typeof metadata.topic === 'string' ? metadata.topic.trim().slice(0, 1000) : '', }; return new Promise((resolve, reject) => { const transaction = db.transaction('assets', 'readwrite'); @@ -50,5 +58,50 @@ window.StudioStorage = (() => { }); } - return {getAsset, setAsset}; + const HISTORY_RETENTION_MS = 24 * 60 * 60 * 1000; + + /* Run a read/write history transaction, pruning entries older than 24 hours. */ + async function historyTransaction(action) { + const db = await open(); + return new Promise((resolve, reject) => { + const transaction = db.transaction('history', 'readwrite'); + const store = transaction.objectStore('history'); + let entries = []; + if (action?.clear) { + store.clear(); + } else { + const cutoff = Date.now() - HISTORY_RETENTION_MS; + const cursorRequest = store.index('createdAt').openCursor(IDBKeyRange.upperBound(cutoff, true)); + cursorRequest.onsuccess = () => { + const cursor = cursorRequest.result; + if (cursor) { + cursor.delete(); + cursor.continue(); + } else if (action?.entry) { + store.put(action.entry); + } else if (!action?.prune) { + const all = store.getAll(); + all.onsuccess = () => { + entries = all.result.filter(entry => { + const valid = Number.isFinite(entry.createdAt) && entry.createdAt >= cutoff && + entry.state && typeof entry.state === 'object' && !Array.isArray(entry.state); + if (!valid) store.delete(entry.id); + return valid; + }); + }; + } + }; + } + transaction.oncomplete = () => resolve(entries.sort((a, b) => b.createdAt - a.createdAt)); + transaction.onabort = () => reject(transaction.error || new Error('History could not be saved.')); + transaction.onerror = () => reject(transaction.error); + }); + } + + return {getAsset, setAsset, HISTORY_RETENTION_MS, + getHistory: () => historyTransaction(), + pruneHistory: () => historyTransaction({prune: true}), + addHistory: entry => historyTransaction({entry}), + clearHistory: () => historyTransaction({clear: true}), + }; })(); diff --git a/sticker_themes.py b/sticker_themes.py index a0528eb..325b9d6 100644 --- a/sticker_themes.py +++ b/sticker_themes.py @@ -1,5 +1,6 @@ """English-only sticker catalog and keyword dictionary for automatic video sticker selection.""" +# Valid sticker identifiers accepted by the API; 'none' and 'auto' are control modes. ALLOWED_STICKERS: tuple[str, ...] = ( 'none', 'auto', 'santa', 'gift', 'snowflake', 'pumpkin', 'globe', 'compass', 'pin', 'plane', @@ -9,8 +10,12 @@ 'bear', 'paw', 'tree', 'wolf', 'beer', 'coffee', 'wine', 'pizza', 'soccer', 'car', 'brain', 'book', 'sun', + 'mountain', 'volcano', 'island', 'rainbow', 'moon', 'rain', + 'robot', 'satellite', 'laptop', 'graduation', 'medal', + 'cat', 'dog', 'fish', 'burger', 'train', ) +# Mapping from sticker name to lowercase hex Twemoji codepoint for SVG/PNG asset lookup. TWEMOJI_CODEPOINTS: dict[str, str] = { 'santa': '1f385', 'gift': '1f381', @@ -46,8 +51,14 @@ 'brain': '1f9e0', 'book': '1f4da', 'sun': '2600', + 'mountain': '26f0', 'volcano': '1f30b', 'island': '1f3dd', 'rainbow': '1f308', + 'moon': '1f319', 'rain': '1f327', 'robot': '1f916', 'satellite': '1f6f0', + 'laptop': '1f4bb', 'graduation': '1f393', 'medal': '1f3c5', 'cat': '1f431', + 'dog': '1f436', 'fish': '1f41f', 'burger': '1f354', 'train': '1f686', } +# Ordered priority rules: ((keywords...), (left_sticker, right_sticker)). +# First matching theme wins when resolving 'auto' stickers from a video title. AUTO_STICKER_THEMES: tuple[tuple[tuple[str, ...], tuple[str, str]], ...] = ( ( ( @@ -181,6 +192,8 @@ ), ) +# Short keywords that must match whole words only (e.g. 'ai', 'car', 'top') +# to avoid false substring matches inside unrelated words. EXACT_WORD_KEYWORDS: set[str] = { 'ai', 'gdp', 'nato', 'top', 'iq', 'car', 'sun', 'pet', 'dog', 'cat', 'bar', 'pub', 'tea', 'oil', 'gas', 'bus', 'gym', 'ice', 'hot', 'god', diff --git a/templates/index.html b/templates/index.html index a5f8216..c0d4991 100644 --- a/templates/index.html +++ b/templates/index.html @@ -282,7 +282,7 @@

Saved in browser @@ -353,7 +353,11 @@

+
@@ -369,7 +373,7 @@

Controls response randomness, not factual accuracy. Lower values do not prevent AI hallucinations.

+ +
+ + + 5 +

@@ -643,11 +669,29 @@

+
+ + 0 px +
+ +
← LeftRight →
+

Moves every country label horizontally. Click Re-Paint to apply without another AI request.

+

+ + + @@ -736,6 +780,13 @@

Cartograp
+
@@ -764,7 +815,7 @@

Cartograp

Synthesizing Cartographic Layer
-
Querying Gemini model and rendering map
+
Querying AI model and rendering map
@@ -863,8 +914,9 @@

Ready to Synthesize
[Info] Select your desired flags -m, -mp, -gsm, -eop, --model, --font and click "Run CLI & Generate Map".

- +