Repository navigation
Expand file tree
/
Copy pathscrape_usage.py
More file actions
594 lines (532 loc) · 30.1 KB
/
Copy pathscrape_usage.py
File metadata and controls
594 lines (532 loc) · 30.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
# -*- coding: utf-8 -*-
"""
opencode.ai 使用历史爬取脚本 · Console 版(2026-10 改版后)
改版事实(以真实登录后的页面与接口为准):
- 旧站是 SolidStart,取数走 POST /_server 的 usage.list server function,登录入口在首页;
现在 /workspace、/usage 全部 404,/_server 返回 302 到 /console/login,旧接口彻底失效。
- 新站是 /console 单页应用:登录页 /console/login(带 hCaptcha,只能人工通过),会话 cookie
为 __Host-console_session,业务接口在 /console/api/* 且必须带 x-org-id 请求头;组织 ID 由
GET /console/api/orgs 获取(与旧的 wrk_... 工作区 ID 不是同一个值)。
- 逐条请求记录在 "Logs" 页(页面原话:Every inference request in this organization, newest
first. Kept for 30 days.),数据源 GET /console/api/request-logs?category=inference,
字段 inputTokens / cacheReadTokens / cacheWriteTokens / outputTokens / reasoningTokens /
cost(美元原值) / sessionID / serviceAPIKeyID / startedAt(毫秒) 与旧版一一对应。
- 该接口只保留 30 天,更早的记录只能沿用本地库里已有的历史;翻页游标 nextCursor 是 JSON 字符串。
- 多账号(实测):GET /console/api/user 返回 {"id":"acc_...","email":"..."},是登录态对应的权威身份;
会话 cookie __Host-console_session 一个身份一条,Playwright 每次 launch 都是全新上下文,
所以"一个账号一份 storage_state 文件"就能并存且互不覆盖。取数前先取身份,用它给每条记录打标。
用法:
python scrape_usage.py # 拉最近 30 天逐条记录,按 账号+记录ID 写进本地库
python scrape_usage.py --account all # 遍历注册表里的全部账号(默认即全部)
python scrape_usage.py --account <账号id> # 只更新某个账号
python scrape_usage.py --days 7 # 只回溯 7 天
python scrape_usage.py --incremental # 每个账号各自翻到早于自己在库里的最新记录即停止
python scrape_usage.py --start 2026-10-01 --end 2026-10-09 # 只入库这个时间段的记录
python scrape_usage.py --force-login # 忽略缓存会话,重新走一次浏览器登录
只写 opencode_usage.db 这一份存储;需要表格时起服务走 GET /api/export.xlsx,从库里现生成。
库只增不删:时间窗只筛本次入库哪些记录,不会清空窗口外的历史。
"""
import argparse
import json
import os
import sys
import time as _time
from datetime import datetime, timedelta, timezone
from pathlib import Path
import httpx
import accounts # 多账号注册表:登录态文件、权威身份、org 缓存都在这一处
import usage_db # 唯一数据存储:抓到的记录只写 SQLite,需要表格时再从库里导出
# ============ 配置 ============
BASE_URL = "https://opencode.ai"
CONSOLE_API = f"{BASE_URL}/console/api"
# 代理直接从环境变量读取;未设置则直连(不使用代理)
PROXY = (os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY") or
os.environ.get("ALL_PROXY") or "").strip()
OUT_DIR = Path(__file__).parent
# 登录页实测的三个入口(/console/login 上依次是 Google / GitHub / email)
LOGIN_LABELS = {"github": "Continue with GitHub", "google": "Continue with Google",
"email": "Continue with email"}
LOCAL_TZ = timezone(timedelta(hours=8)) # 页面显示时区 +08:00
RETENTION_DAYS = 30 # request-logs 服务端保留窗口
PAGE_LIMIT = 100 # request-logs 每页条数
LOGIN_EXPIRED_MSG = "登录态已失效,请在设置里填写账号密码并重新登录(--force-login --email ... --password ...)"
# =============================
def cookie_header(state_file: Path) -> str:
"""从 storage_state 读取 opencode.ai 域的 cookie,拼成 Cookie header。"""
if not state_file.exists():
return ""
try:
state = json.loads(state_file.read_text(encoding="utf-8"))
except Exception:
return ""
return "; ".join(f"{c['name']}={c['value']}" for c in state.get("cookies", [])
if "opencode.ai" in c.get("domain", ""))
def make_client(state_file: Path, org: str = "") -> httpx.Client:
"""构造带登录态 cookie、x-org-id 与代理的 HTTP 客户端。"""
headers = {
"User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"),
"Accept": "application/json",
"Accept-Language": "en-US,en;q=0.9",
"Referer": f"{BASE_URL}/console/",
"Origin": BASE_URL,
}
ck = cookie_header(state_file)
if ck:
headers["Cookie"] = ck
if org:
headers["x-org-id"] = org
if not PROXY:
return httpx.Client(headers=headers, timeout=60)
try:
return httpx.Client(headers=headers, proxy=PROXY, timeout=60)
except TypeError:
return httpx.Client(headers=headers, proxies={"http://": PROXY, "https://": PROXY}, timeout=60)
def _check_auth(resp) -> None:
if resp.status_code in (401, 403) or '"Unauthorized"' in resp.text:
raise RuntimeError(LOGIN_EXPIRED_MSG)
def whoami(client: httpx.Client) -> dict:
"""当前登录态对应的账号身份(实测返回 {"id":"acc_...","email":"...","role":"user"})。
多账号必须以此为准而不是以用户填写的邮箱为准:填写邮箱只用于登录,
数据归属要用服务端身份,否则同一份会话登进了谁就记到谁名下。
"""
resp = client.get(f"{CONSOLE_API}/user")
_check_auth(resp)
if resp.status_code != 200:
raise RuntimeError(f"请求 /console/api/user 失败: HTTP {resp.status_code} {resp.text[:200]}")
data = resp.json() or {}
identity = (data.get("email") or data.get("id") or "").strip()
if not identity:
raise RuntimeError("登录态未返回账号身份(/console/api/user 无 email/id),无法归属数据")
return {"identity": identity, "accountId": (data.get("id") or "").strip()}
def ensure_org(client: httpx.Client) -> str:
"""取当前会话的组织 ID(/console/api/orgs 不需要 x-org-id)。"""
resp = client.get(f"{CONSOLE_API}/orgs")
_check_auth(resp)
if resp.status_code != 200:
raise RuntimeError(f"请求 /console/api/orgs 失败: HTTP {resp.status_code} {resp.text[:200]}")
orgs = resp.json()
if not isinstance(orgs, list) or not orgs:
raise RuntimeError("该账号下没有任何组织(org),无法取数")
return orgs[0]["id"]
# ============ 取数(逐条推理请求日志) ============
def fetch_usage_records(client: httpx.Client, days: int, floor_ms: int = 0) -> list:
"""按游标翻页拉取 inference 请求日志,返回原始记录列表。
停止条件:翻完 / 触达回溯窗口 / 增量模式下遇到早于 floor_ms 的记录。
"""
cutoff_ms = int((datetime.now(timezone.utc) - timedelta(days=days)).timestamp() * 1000)
records = []
cursor = None
pages = 0
while True:
params = {"category": "inference", "limit": str(PAGE_LIMIT)}
if cursor:
params["cursor"] = cursor
resp = client.get(f"{CONSOLE_API}/request-logs", params=params)
_check_auth(resp)
if resp.status_code != 200:
raise RuntimeError(f"请求 request-logs 失败: HTTP {resp.status_code} {resp.text[:200]}")
data = resp.json()
items = data.get("items") or []
pages += 1
reached_floor = False
for it in items:
started = int(it.get("startedAt") or 0)
if started and started < cutoff_ms:
reached_floor = True
continue
if floor_ms and started and started <= floor_ms:
reached_floor = True
continue
records.append(it)
print(f" 第{pages}页: {len(items)} 条(累计保留 {len(records)})", flush=True)
cursor = data.get("nextCursor")
if reached_floor or not items or not cursor:
break
return records
def to_int(v) -> int:
try:
return int(float(v))
except (TypeError, ValueError):
return 0
def ms_to_local(ms) -> str:
try:
return datetime.fromtimestamp(int(ms) / 1000, tz=timezone.utc).astimezone(LOCAL_TZ).strftime("%Y-%m-%d %H:%M:%S")
except (TypeError, ValueError):
return ""
def record_to_row(r: dict, identity: str = "") -> dict:
"""逐条请求记录 → 统一行口径(与 usage_db 入库、导出报表共用同一套列名)。
多账号靠"账号"这一列落地:值就是服务端返回的身份,SQLite 侧的 account_key
由它哈希得出,所以库和报表都只多这一列即可无损归属。
"""
dt = ms_to_local(r.get("startedAt"))
cost = r.get("cost")
try:
cost = round(float(cost), 8) if cost is not None else 0
except (TypeError, ValueError):
cost = 0
inp = to_int(r.get("inputTokens"))
cache = to_int(r.get("cacheReadTokens"))
out = to_int(r.get("outputTokens"))
reason = to_int(r.get("reasoningTokens"))
return {
"日期时间": dt,
"月份": dt[:7],
"账号": identity or "",
"模型": r.get("model", "") or "",
"提供方": r.get("provider", "") or "",
"输入 Input": inp,
"缓存读取 Cache Read": cache,
"输入合计": inp + cache,
"输出 Output": out,
"推理 Reasoning": reason,
"缓存写入 Cache Write": to_int(r.get("cacheWriteTokens")),
"输出合计": out + reason,
"状态": r.get("outcome", "") or "",
"耗时(ms)": to_int(r.get("durationMs")),
"成本($)": cost,
"成本显示": f"USD (${cost:.4f})",
"Session": r.get("sessionID", "") or "",
"Key ID": r.get("serviceAPIKeyID", "") or "",
"计划": r.get("product", "") or "",
"记录ID": r.get("id", "") or r.get("requestID", "") or "",
}
# ============ 时间段过滤 ============
def parse_local(s):
try:
return datetime.fromisoformat(str(s).replace(" ", "T"))
except Exception:
return None
def filter_rows(rows: list, months: int, start: str, end: str) -> list:
"""按月份数 / 指定时间段过滤本次要入库的记录(基于本地时间)。
只筛"这次写进去哪些",不删库里已有的记录——库是唯一存储,按时间窗更新不等于清空历史。
"""
if not months and not start and not end:
return rows
cutoff = (datetime.now(LOCAL_TZ).replace(tzinfo=None) - timedelta(days=365.25 / 12 * months)) if months else None
start_dt = datetime.fromisoformat(start) if start else None
end_dt = datetime.fromisoformat(end + "T23:59:59") if end else None
kept = []
for r in rows:
t = parse_local(r.get("日期时间") or "")
if t is None:
kept.append(r)
continue
if cutoff and t < cutoff:
continue
if start_dt and t < start_dt:
continue
if end_dt and t > end_dt:
continue
kept.append(r)
return kept
# ============ 登录(仅当需要时使用 Playwright 走一遍网页登录) ============
def ensure_login(email=None, password=None, force_login=False, method="github",
state_file=None) -> None:
"""打开新版 Console 登录页完成登录(GitHub / Google OAuth,或邮箱验证链接),并保存该账号的 storage_state。
登录成功判据:浏览器回到 /console(非 login/signup),且 /console/api/orgs 能取到组织。
state_file 由调用方按账号给出——新账号首次登录时它还不存在,浏览器上下文就是干净的,
不会带上别的账号的会话 cookie(Playwright 每次 launch 都是全新上下文,会话只来自这份文件)。
"""
from playwright.async_api import async_playwright # 延迟导入:平时爬取不加载
state_file = Path(state_file) if state_file else accounts.state_file({"id": method})
if force_login and state_file.exists():
state_file.unlink()
login_email = email or ""
login_password = password or ""
import asyncio
async def wait_logged_in(page, label, timeout_ms=300000) -> bool:
"""等待 OAuth 完成并回到 Console。
hCaptcha 与 2FA 只能由用户在窗口内人工完成(脚本不绕过人机验证);脚本负责识别页型:
关联账号会话失效时代填账号密码,GitHub 授权页代点 Authorize。"""
deadline = _time.time() + timeout_ms / 1000
said_captcha = said_2fa = filled_provider = clicked_authorize = False
while _time.time() < deadline:
low = page.url.lower()
if already_signed_in(page):
return True
if not said_captcha and any("hcaptcha" in f.url.lower() for f in page.frames):
said_captcha = True
print("请在浏览器窗口中完成 hCaptcha 安全验证,脚本会继续等待…", flush=True)
# 三层恢复:仅当 provider 侧真的出现登录表单才填账号密码
if not filled_provider and "github.com" in low and await page.locator("#login_field").count() > 0:
if not login_email or not login_password:
raise RuntimeError("关联账号(GitHub)会话已失效,需要账号密码登录(请在设置页填写账号密码)")
await page.fill("#login_field", login_email)
await page.click("input[type=submit]")
await page.wait_for_timeout(3000)
if await page.locator("#password").count() > 0:
await page.fill("#password", login_password)
await page.click("input[type=submit]")
filled_provider = True
continue
if not filled_provider and "accounts.google.com" in low and await page.locator('input[type="email"]').count() > 0:
if not login_email or not login_password:
raise RuntimeError("关联账号(Google)会话已失效,需要账号密码登录(请在设置页填写账号密码)")
await page.fill('input[type="email"]', login_email)
await page.click("#identifierNext")
await page.wait_for_timeout(4000)
if await page.locator('input[type="password"]').count() > 0:
await page.fill('input[type="password"]', login_password)
await page.click("#passwordNext")
filled_provider = True
continue
if not clicked_authorize and "github.com" in low and "authorize" in low:
clicked_authorize = True
try:
await page.click("input[value='Authorize'], button:has-text('Authorize')", timeout=8000)
except Exception:
pass
continue
if not said_2fa and any(k in low for k in ("two-factor", "login/device", "verify",
"challenge", "enter the code")):
said_2fa = True
print("检测到账号安全验证,请在浏览器窗口中完成验证码 / 设备码授权…", flush=True)
await page.wait_for_timeout(1500)
return False
def already_signed_in(page) -> bool:
low = page.url.lower()
return "opencode.ai/console" in low and "/console/login" not in low and "/console/signup" not in low
async def _do_login():
async with async_playwright() as p:
# 无代理时绝不能传 proxy={"server": ""}:Playwright 会直接抛
# BrowserType.launch: Invalid URL,登录流程还没开始就 rc=1
launch_kwargs = {"headless": False, # 可见模式:OAuth 与 hCaptcha 对用户可见,人工完成后脚本继续
"args": ["--disable-blink-features=AutomationControlled"]}
if PROXY:
launch_kwargs["proxy"] = {"server": PROXY}
browser = await p.chromium.launch(**launch_kwargs)
ctx = await browser.new_context(
viewport={"width": 1440, "height": 900}, locale="en-US",
storage_state=str(state_file) if state_file.exists() else None,
)
page = await ctx.new_page()
page.set_default_timeout(45000)
label = LOGIN_LABELS.get(method) or LOGIN_LABELS["github"]
await page.goto(f"{BASE_URL}/console/login?next=%2Fconsole%2F",
wait_until="domcontentloaded", timeout=60000)
# 登录页是 SPA:要么出现登录按钮(需要走登录),要么被直接跳进 Console(缓存会话仍有效)。
# 只等按钮会在"已登录"时超时并误报 rc=1。
button_seen = False
try:
await page.wait_for_selector(f"text={label}", timeout=20000)
button_seen = True
except Exception:
pass
if not button_seen and not already_signed_in(page):
try:
await page.wait_for_url(lambda u: "/console/login" not in u.lower(), timeout=20000)
except Exception:
pass
if already_signed_in(page):
print("已处于登录态,直接进入 Console", flush=True)
elif not button_seen:
raise RuntimeError(f"登录页未出现“{label}”入口:{page.url}")
else:
print(f"请在浏览器窗口中完成登录与安全验证(点击“{label}”,hCaptcha 需人工)…", flush=True)
try:
await page.click(f"text={label}", timeout=10000)
except Exception:
# 按钮可能被 hCaptcha 安全校验拦住,交由用户在窗口内手动完成
pass
if method == "email":
# 实测:点开后是页内联邮箱表单(input[type=email] + “Continue”),
# 提交后 opencode 往该邮箱发验证链接/验证码;hCaptcha 与收信只能人工完成,
# 脚本只负责代填并继续等会话落地(wait_logged_in 认的是 URL + /console/api/orgs)
await page.wait_for_timeout(1500)
box = page.locator('input[type="email"]')
if await box.count() > 0:
if login_email:
try:
await box.first.fill(login_email)
await page.click('text="Continue"', timeout=8000)
except Exception:
pass
print("已提交邮箱,请在浏览器窗口里完成 hCaptcha,并到邮箱点验证链接 / 填验证码…", flush=True)
if not await wait_logged_in(page, label):
raise RuntimeError("登录超时:hCaptcha 安全验证或授权未在浏览器窗口中完成")
# 以服务端为准:能取到组织才算真的登录成功
await page.goto(f"{BASE_URL}/console/", wait_until="domcontentloaded", timeout=60000)
await page.wait_for_timeout(2000)
orgs = await page.evaluate(
"async () => { const r = await fetch('/console/api/orgs',{headers:{'accept':'application/json'}});"
" return {status:r.status, body: await r.text()}; }")
if orgs.get("status") != 200:
raise RuntimeError(f"登录后仍取不到组织信息: HTTP {orgs.get('status')} {str(orgs.get('body'))[:160]}")
await ctx.storage_state(path=str(state_file))
print("登录完成,会话已保存", flush=True)
await browser.close()
asyncio.run(_do_login())
# ============ 多账号取数 ============
def resolve_target_accounts(args) -> list:
"""本次要更新的账号条目列表。
--account <id> 指定单个;--account all 或省略 = 注册表里的全部账号(多账号就是一次跑完)。
给了 --email 且注册表里还没有它 → 登记为新账号(命令行首次登录也能纳管)。
"""
reg = accounts.load()
sel = (args.account or "all").strip()
if sel and sel != "all":
entry = accounts.find(sel)
if not entry:
raise RuntimeError(f"未找到账号条目 {sel}(--account 用注册表里的 id)")
return [entry]
for a in reg["accounts"]:
ident = accounts.account_identity(a).lower()
if args.email and (ident == args.email.lower() or (a.get("email") or "").lower() == args.email.lower()):
if args.password and args.password != a.get("password"):
a = accounts.upsert(email=args.email, password=args.password,
login_method=args.login_method, entry_id_=a["id"])
return [a]
if reg["accounts"] and not args.email:
return list(reg["accounts"])
return [accounts.upsert(email=args.email, password=args.password,
login_method=args.login_method, label=args.label)]
def discover_org(client, entry: dict, args) -> str:
"""组织 ID 每次都以服务端为准(改版后它会变),账号自己的缓存只作兜底。
旧版把 org 存在全局 opencode_env.json 里,多账号会互相覆盖,所以缓存改到账号条目上。
"""
override = (os.environ.get("OPENCODE_ORG_ID", "").strip() or (args.workspace or "").strip())
if override:
return override
cached = (entry.get("org") or "").strip()
try:
return ensure_org(client)
except RuntimeError as e:
if "登录态已失效" in str(e):
raise
if not cached:
raise
print(f"组织发现失败,沿用该账号缓存 org={cached}", flush=True)
return cached
def incremental_floor(args, conn, identity: str) -> int:
"""增量起点 = 该账号自己在库里的最新记录时间(库是唯一存储,不再回读任何表格文件)。
按账号取而不是全库取:全库最新那条可能属于另一个账号,用它当起点会让本账号的
历史缺口永远补不上。
"""
if not args.incremental:
return 0
latest = usage_db.latest_started(conn, accounts.identity_key(identity))
if not latest:
print(f"增量起点:{identity} 在库里还没有记录,按完整窗口抓取", flush=True)
return 0
dt = parse_local(latest)
if dt is None:
print(f"增量起点:{identity} 的最新记录时间无法解析({latest}),按完整窗口抓取", flush=True)
return 0
print(f"增量起点({identity} 在库里的最新记录): {latest}", flush=True)
return int(dt.replace(tzinfo=LOCAL_TZ).astimezone(timezone.utc).timestamp() * 1000)
def fetch_for_account(entry: dict, args, conn) -> tuple:
"""单账号完整流程:登录态 → 服务端身份 → 组织 → 翻页取数。返回 (身份, 原始记录)。"""
state = accounts.state_file(entry)
email = entry.get("email") or args.email
password = entry.get("password") or args.password
method = entry.get("loginMethod") or args.login_method
if not state.exists() or args.force_login:
print(f"需要登录 / 重新授权 opencode.ai({accounts.account_label(entry)})...", flush=True)
ensure_login(email, password, args.force_login, method, state)
auth_attempts = 0
net_retries = 0
while True:
try:
with make_client(state) as client:
who = whoami(client)
# 以服务端身份为准换绑:登录时填的邮箱只是凭据,数据归属看 /console/api/user
entry = accounts.rekey(entry["id"], who["identity"], who["accountId"])
state = accounts.state_file(entry)
org = discover_org(client, entry, args)
client.headers["x-org-id"] = org
print(f"账号: {who['identity']} 组织 ID: {org} 回溯窗口: {args.days} 天", flush=True)
floor_ms = incremental_floor(args, conn, who["identity"])
fetched = fetch_usage_records(client, max(1, args.days), floor_ms)
accounts.mark_synced(entry["id"], identity=who["identity"],
account_id=who["accountId"], org=org)
return who["identity"], fetched
except RuntimeError as e:
# 登录态失效 → 打开浏览器重新授权(hCaptcha 由人工完成),然后重试
if "登录态已失效" in str(e) and auth_attempts < 2:
auth_attempts += 1
print(f"opencode 登录态失效,按该账号的登录方式({method})重新授权...", flush=True)
ensure_login(email, password, False, method, state)
continue
raise
except Exception as e:
# 网络 / SSL 偶发错误 → 短暂重试,避免一次抖动导致整个更新失败
if net_retries < 3:
net_retries += 1
print(f"[retry] 网络异常: {type(e).__name__}: {e},第 {net_retries}/3 次重试...", flush=True)
_time.sleep(2)
continue
raise RuntimeError(f"网络异常持续: {type(e).__name__}: {e}")
# ============ 主流程 ============
def main() -> None:
parser = argparse.ArgumentParser(description="爬取 opencode.ai 使用历史(Console 版逐条请求日志)")
parser.add_argument("--days", type=int, default=RETENTION_DAYS,
help=f"回溯天数(服务端最多保留 {RETENTION_DAYS} 天,默认全部窗口)")
parser.add_argument("--account", type=str, default="all",
help="要更新的账号条目 id;all=注册表全部账号(默认),留空同 all")
parser.add_argument("--email", type=str, default="", help="登录账号(留空用缓存登录态)")
parser.add_argument("--password", type=str, default="", help="登录密码")
parser.add_argument("--label", type=str, default="", help="账号显示名(仅新登记账号时生效)")
parser.add_argument("--login-method", type=str, default="github", choices=["github", "google", "email"],
help="登录方式(github / google / email 魔法链接)")
parser.add_argument("--workspace", type=str, default="",
help="兼容旧参数:组织 ID 现由 /console/api/orgs 自动获取,此项仅作覆盖用")
parser.add_argument("--force-login", action="store_true", help="强制重新登录(忽略缓存登录态)")
parser.add_argument("--start", type=str, default="", help="本次入库的时间段起点 YYYY-MM-DD(本地时间)")
parser.add_argument("--end", type=str, default="", help="本次入库的时间段终点 YYYY-MM-DD(本地时间,含当天)")
parser.add_argument("--months", type=int, default=0, help="只入库最近 N 个月的记录(0=不过滤;窗口外的既有历史不动)")
parser.add_argument("--incremental", action="store_true",
help="增量:每个账号各自翻到早于自己在库里的最新记录即停止")
args = parser.parse_args()
# 唯一存储是 SQLite:抓到的记录直接入库,不再产出/回读第二份表格文件。
# 老安装若库里还没有数据,先用它遗留的历史表格做一次性的迁移导入。
conn = usage_db.connect()
try:
usage_db.init(conn, accounts.default_identity())
migrated = usage_db.import_excel_if_needed(conn, OUT_DIR / "opencode_usage_history.xlsx",
accounts.legacy_identity())
if migrated.startswith("已从历史"):
print(f"历史迁移: {migrated}", flush=True)
# 旧表格里没有"账号"列的历史行属于谁:只有升级前那一个账号能把它们算作自己的历史
legacy_identity = accounts.legacy_identity()
targets = resolve_target_accounts(args)
print(f"待更新账号: {len(targets)} 个({', '.join(accounts.account_label(a) or a['id'] for a in targets)})",
flush=True)
failures = []
stored_total = 0
for entry in targets:
label = accounts.account_label(entry) or entry["id"]
print(f"\n===== 账号 {label} =====", flush=True)
try:
identity, fetched = fetch_for_account(entry, args, conn)
except Exception as e:
# 一个账号失败(比如 hCaptcha 没人工通过)不能带走其它账号的更新
failures.append(f"{label}: {type(e).__name__}: {e}")
print(f"[fail] 账号 {label} 更新失败: {type(e).__name__}: {e}", flush=True)
continue
rows = [record_to_row(r, identity) for r in fetched]
kept = filter_rows(rows, args.months, args.start, args.end)
dup = len(usage_db.existing_ids(conn, accounts.identity_key(identity),
[str(r.get("记录ID") or "") for r in kept]))
n = usage_db.upsert_rows(conn, kept, legacy_identity)
stored_total += n - dup
print(f"本次抓取: {len(fetched)} 条 · 入库 {n} 条(新增 {n - dup},重复覆盖 {dup})(账号 {identity})",
flush=True)
print(f"\n本次新增: {stored_total} 条", flush=True)
if args.start or args.end:
print(f"时间窗 {args.start or '不限'} ~ {args.end or '不限'} 之外的既有历史保持不变")
elif args.months:
print(f"最近 {args.months} 个月之外的既有历史保持不变")
print(f"总计记录: {usage_db.count_records(conn)}")
print("按账号统计: " + " · ".join(
f"{a or '未知'} {c} 条" for a, c in conn.execute(
"SELECT account_label, COUNT(*) FROM usage_records GROUP BY account_label ORDER BY account_label")),
flush=True)
finally:
conn.close()
if failures:
for f in failures:
print(f"[fail] {f}", flush=True)
sys.exit(1)
if __name__ == "__main__":
main()