diff --git a/.oxlintrc.json b/.oxlintrc.json index 3091ea0b..3eab2b4e 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -39,6 +39,11 @@ ], "unicorn/prefer-global-this": "off", "eslint/no-new": "off", + "eslint/no-unused-vars": [ + "error", + { "argsIgnorePattern": "^_", "varsIgnorePattern": "^_", "caughtErrorsIgnorePattern": "^_" } + ], + "eslint/no-unused-private-class-members": "error", "eslint/sort-imports": "off", "eslint/no-duplicate-imports": ["warn", { "allowSeparateTypeImports": true }], "eslint/new-cap": "off", diff --git a/README.md b/README.md index 29cc81e4..d3738bfd 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ Sync Engine is a vault syncing plugin to **synchronize vault files in multiple d **Features**: - [Client-side encryption](https://sync.consensia.cc/deep-dive/modules/encryption) +- [Realtime collaboration](https://sync.consensia.cc/deep-dive/modules/collaboration) over encrypted MQTT notifications - Granular sync strategy configuration (bidirectional / mirror remote / mirror local / don't sync) using Glob rules. - Startup / periodic / sync-on-focus / save-on-change syncing. - Conflict resolve strategies ([smart merge](https://sync.consensia.cc/deep-dive/modules/smart-merge) / keep both / latest survive / keep remote / keep local / skip). diff --git a/README.zh.md b/README.zh.md index 5917abd7..297f061a 100644 --- a/README.zh.md +++ b/README.zh.md @@ -54,6 +54,7 @@ Sync Engine 是一款用于**多设备间同步 Vault 笔记文件**的插件, **核心特性**: - [客户端端到端加密](https://sync.consensia.cc/deep-dive/modules/encryption) +- [实时协作](https://sync.consensia.cc/deep-dive/modules/collaboration),通过加密的 MQTT 通知同步 - 使用 Glob 规则进行精细化的同步策略配置(双向同步 / 镜像远程 / 镜像本地 / 不同步)。 - 支持启动时同步、定时同步、焦点同步以及变更时同步。 - 灵活的冲突解决方案([智能合并](https://sync.consensia.cc/deep-dive/modules/smart-merge) / 保留两者 / 保留最新版 / 覆盖为远端 / 覆盖为本地 / 直接跳过)。 diff --git a/bun.lock b/bun.lock index fd513d25..21efc1a0 100644 --- a/bun.lock +++ b/bun.lock @@ -32,6 +32,16 @@ "vue-tsc": "^3.3.14", }, }, + "packages/collaboration": { + "name": "collaboration", + "version": "0.1.0", + "devDependencies": { + "@hesprs/sync-engine-sdk": "workspace:*", + "@repo/shared": "workspace:*", + "mqtt": "^5.16.1", + "uni-kv": "catalog:", + }, + }, "packages/dropbox": { "name": "dropbox", "version": "0.0.1", @@ -60,6 +70,16 @@ "uni-kv": "catalog:", }, }, + "packages/history": { + "name": "history", + "version": "0.0.1", + "devDependencies": { + "@hesprs/sync-engine-sdk": "workspace:*", + "@repo/shared": "workspace:*", + "fflate": "^0.8.2", + "uni-kv": "catalog:", + }, + }, "packages/i18n": { "name": "i18n", "version": "0.0.1", @@ -572,6 +592,8 @@ "@types/node": ["@types/node@26.6.6", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-MzQj+8Kx3XI40OmUAE6Qoh58ZMNstNNMwn2nfd3qWsfe7nDNSqJNzes4njhz6AHwrSHOffSLwXU6yb474Q4aog=="], + "@types/readable-stream": ["@types/readable-stream@4.0.25", "", { "dependencies": { "@types/node": "*" } }, "sha512-7GsxL8Wd5X3jh73/qgW7WCnbBvGSdD8U3E4Jiuus3/8I+PN20DtlkJtwFhr/sImXfDDQ2Z03i/9wNiIF3TVSsw=="], + "@types/retry": ["@types/retry@0.12.0", "", {}, "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA=="], "@types/tern": ["@types/tern@0.23.9", "", { "dependencies": { "@types/estree": "*" } }, "sha512-ypzHFE/wBzh+BlH6rrBgS5I/Z7RD21pGhZ2rltb/+ZrVM1awdZwjx7hE5XfuYgHWk9uvV5HLZN3SloevCAp3Bw=="], @@ -582,7 +604,9 @@ "@types/web-bluetooth": ["@types/web-bluetooth@0.0.21", "", {}, "sha512-oIQLCGWtcFZy2JW77j9k8nHzAOpqMHLQejDA48XXMWH6tjCQHz5RCFz1bzsmROyL6PUm+LLnUiI4BCn221inxA=="], - "@ungap/structured-clone": ["@ungap/structured-clone@1.4.0", "", {}, "sha512-1mEZtMKPM09vDmQt5y7YvmN2+DFTP7Tg0EWXdic8/C6VRnpb33e4ghisCIE3WZjsE2N8mf+QV1Zqh7ZFYLWInQ=="], + "@types/ws": ["@types/ws@8.18.2", "", { "dependencies": { "@types/node": "*" } }, "sha512-67MQl+fpWKVTT1NYdnmo3U4sc/xPo/zQBncVnI74qmQa0z/b+1g6iYqNmGCPbxO+zz2aklb08a0oHfegiVd0/w=="], + + "@ungap/structured-clone": ["@ungap/structured-clone@1.4.0", "https://registry.npmmirror.com/@ungap/structured-clone/-/structured-clone-1.4.0.tgz", {}, "sha512-1mEZtMKPM09vDmQt5y7YvmN2+DFTP7Tg0EWXdic8/C6VRnpb33e4ghisCIE3WZjsE2N8mf+QV1Zqh7ZFYLWInQ=="], "@unocss/autocomplete": ["@unocss/autocomplete@66.12.0", "", { "dependencies": { "fzf": "^0.5.2", "lru-cache": "^11.5.3" } }, "sha512-klClL63gwOr3L6FAnOiicbzwWypK/qYkn0m1UyrRqMCzy673Xl7mRt3Y1/mijP1xWd8WwYsW1KZEak+qtk5vDg=="], @@ -690,6 +714,8 @@ "@yuku-toolchain/win32-x64": ["@yuku-toolchain/win32-x64@0.27.0", "", { "os": "win32", "cpu": "x64" }, "sha512-7LSviI8HtHga0g7G2oT7C/SP2IoLmrg8LYPeWZkp4ljFtaWKJoTRF0Iv0cVt0dR4rehoN/+TjN4r8SoArCKvlg=="], + "abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="], + "agent-base": ["agent-base@9.0.0", "", {}, "sha512-TQf59BsZnytt8GdJKLPfUZ54g/iaUL2OWDSFCCvMOhsHduDQxO8xC4PNeyIkVcA5KwL2phPSv0douC0fgWzmnA=="], "alien-signals": ["alien-signals@3.2.1", "", {}, "sha512-I8FjmltrfnDFoZedi5CG8DghVYNhzb/Ijluz7tCSJH0xpd0484Kowhbb1XDYOxfJpU1p5wnM2X54dA+IfGyD1g=="], @@ -708,17 +734,25 @@ "birpc": ["birpc@2.9.0", "", {}, "sha512-KrayHS5pBi69Xi9JmvoqrIgYGDkD6mcSe/i6YKi3w5kekCLzrX4+nawcXqrj2tIp50Kw/mT/s3p+GVK0A0sKxw=="], + "bl": ["bl@6.1.6", "", { "dependencies": { "@types/readable-stream": "^4.0.0", "buffer": "^6.0.3", "inherits": "^2.0.4", "readable-stream": "^4.2.0" } }, "sha512-jLsPgN/YSvPUg9UX0Kd73CXpm2Psg9FxMeCSXnk3WBO3CMT10JMwijubhGfHCnFu6TPn1ei3b975dxv7K2pWVg=="], + "boolbase": ["boolbase@2.0.0", "", {}, "sha512-DkVaaQHymRhpYEYo9x1oo7Q7B0Y6KJUsjm3c9eTyFDby4MHLBTwZ6ZDWBel5zrYxj1WsZgC5oLpiz+93MluXeA=="], "bowser": ["bowser@2.15.0", "", {}, "sha512-xic6KSZglg/I4ymp45fdM4GWfYpBK79Fxd/TOvJktlaPgQqqDxvmGwOPZl2V0dyeS3Lj5c180bLhXF43iyAjZQ=="], "brace-expansion": ["brace-expansion@5.0.12", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ=="], - "browserslist": ["browserslist@4.29.5", "", { "dependencies": { "baseline-browser-mapping": "^2.11.33", "caniuse-lite": "^1.0.30001819", "electron-to-chromium": "^1.5.464", "node-releases": "^2.0.60", "update-browserslist-db": "^1.3.4" }, "bin": { "browserslist": "cli.js" } }, "sha512-khTmKu5NGYTotqcYo8Bew2k4RAE9PuXRSFRXG2lQBrCEQHyeQ/vX6gKMxfokwywBsLiXZMnHsmWMG1pnO8J7uQ=="], + "broker-factory": ["broker-factory@3.1.15", "", { "dependencies": { "@babel/runtime": "^7.29.7", "fast-unique-numbers": "^9.0.27", "tslib": "^2.8.1", "worker-factory": "^7.0.50" } }, "sha512-ko+aWvgNuP49meGrdjUu7rC+Y+Wai3cCPxP3xWwHsHfehFjOh5ZQM2yC4gEB2UddeZ/YXhm0K1eG/L6fxym2Og=="], + + "browserslist": ["browserslist@4.29.3", "https://registry.npmmirror.com/browserslist/-/browserslist-4.29.3.tgz", { "dependencies": { "baseline-browser-mapping": "^2.11.26", "caniuse-lite": "^1.0.30001813", "electron-to-chromium": "^1.5.439", "node-releases": "^2.0.57", "update-browserslist-db": "^1.3.3" }, "bin": { "browserslist": "cli.js" } }, "sha512-1R4kiYKXGViqEN0CnoDrXc1StD9niAwu+j2dukWzrD4bJgsD4lDmEp0CRbc6E/vYJIfTHwPmwyaKtVSudICdPA=="], + + "buffer": ["buffer@6.0.3", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" } }, "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA=="], "buffer-equal-constant-time": ["buffer-equal-constant-time@1.0.1", "", {}, "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="], - "bun-types": ["bun-types@1.4.4", "", { "dependencies": { "@types/node": "*", "undici-types": "*" } }, "sha512-yjrJh5oGX7mMPKGd19YXyIjmEuj4G4zNC30bs7PM9Az7NQ3ndF3j6hbeRdvJTRQEhJ/FYQwW0a2moweEi2hMxQ=="], + "buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="], + + "bun-types": ["bun-types@1.4.4", "https://registry.npmmirror.com/bun-types/-/bun-types-1.4.4.tgz", { "dependencies": { "@types/node": "*", "undici-types": "*" } }, "sha512-yjrJh5oGX7mMPKGd19YXyIjmEuj4G4zNC30bs7PM9Az7NQ3ndF3j6hbeRdvJTRQEhJ/FYQwW0a2moweEi2hMxQ=="], "cac": ["cac@7.1.0", "", {}, "sha512-LMDU/U2yA4IPIY8KT0NsNPWEYZvAMkbgyspxJ6btJaYV+WMDOHAZCmB0lsTObiHaRS1V/ztKAl7+TSkTqXyeoA=="], @@ -736,13 +770,19 @@ "chokidar": ["chokidar@5.0.0", "", { "dependencies": { "readdirp": "^5.0.0" } }, "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw=="], - "colorette": ["colorette@2.0.20", "", {}, "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w=="], + "collaboration": ["collaboration@workspace:packages/collaboration"], + + "colorette": ["colorette@2.0.20", "https://registry.npmmirror.com/colorette/-/colorette-2.0.20.tgz", {}, "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w=="], "comma-separated-tokens": ["comma-separated-tokens@2.0.3", "", {}, "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg=="], "commander": ["commander@12.1.0", "", {}, "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA=="], - "consola": ["consola@3.4.2", "", {}, "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA=="], + "commist": ["commist@3.2.0", "", {}, "sha512-4PIMoPniho+LqXmpS5d3NuGYncG6XWlkBSVGiWycL22dd42OYdUGil2CWuzklaJoNxyxUSpO4MKIBU94viWNAw=="], + + "concat-stream": ["concat-stream@2.0.0", "", { "dependencies": { "buffer-from": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.0.2", "typedarray": "^0.0.6" } }, "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A=="], + + "consola": ["consola@3.4.2", "https://registry.npmmirror.com/consola/-/consola-3.4.2.tgz", {}, "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA=="], "convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="], @@ -814,14 +854,22 @@ "estree-walker": ["estree-walker@2.0.2", "", {}, "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="], + "event-target-shim": ["event-target-shim@5.0.1", "", {}, "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="], + + "events": ["events@3.3.0", "", {}, "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="], + "extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="], "fast-sha256": ["fast-sha256@1.3.0", "", {}, "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ=="], - "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], + "fast-unique-numbers": ["fast-unique-numbers@9.0.27", "", { "dependencies": { "@babel/runtime": "^7.29.2", "tslib": "^2.8.1" } }, "sha512-nDA9ADeINN8SA2u2wCtU+siWFTTDqQR37XvgPIDDmboWQeExz7X0mImxuaN+kJddliIqy2FpVRmnvRZ+j8i1/A=="], + + "fdir": ["fdir@6.5.0", "https://registry.npmmirror.com/fdir/-/fdir-6.5.0.tgz", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], "fetch-blob": ["fetch-blob@3.2.0", "", { "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" } }, "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ=="], + "fflate": ["fflate@0.8.3", "", {}, "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA=="], + "focus-trap": ["focus-trap@8.2.4", "", { "dependencies": { "tabbable": "^6.5.0" } }, "sha512-oy6dj7WZYgMbbpoOA29K5W++473F1zEzdI1oJgPb6x7yWT36YaWedUe6i6YPWEt5quSh0pRMohVyHLWXUxtdxg=="], "formdata-polyfill": ["formdata-polyfill@4.0.10", "", { "dependencies": { "fetch-blob": "^3.1.2" } }, "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g=="], @@ -858,8 +906,12 @@ "hast-util-whitespace": ["hast-util-whitespace@3.0.0", "", { "dependencies": { "@types/hast": "^3.0.0" } }, "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw=="], + "help-me": ["help-me@5.0.0", "", {}, "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg=="], + "highlight.js": ["highlight.js@10.7.3", "", {}, "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A=="], + "history": ["history@workspace:packages/history"], + "hookable": ["hookable@6.1.2", "", {}, "sha512-+abwxtiEA52GCVIsQqut3S/uKTbUwYIp4Pe/vv+6py5XiXBCqMZHg6pA6Y5qhgLSEys0/cYuPbO0z1QFj5ZCmg=="], "hosted-git-info": ["hosted-git-info@9.0.3", "", { "dependencies": { "lru-cache": "^11.1.0" } }, "sha512-Hc+ghLoSt6QaYZUv0WBiIvmMDZuZZ7oaDvdH8MbfOO4lOsxdXLEvuC6ePoGs9H1X9oCLyq6+NVN0MKqD+ydxyg=="], @@ -880,13 +932,19 @@ "idb": ["idb@8.0.4", "", {}, "sha512-jd0ITCrN+Zah4J7R/Z2jnR3W7A4BXDAjmo62MwWuSjxCiueUAf7X5rbodEdidWzeknwXHmLpdZkoiiCppNi30Q=="], + "ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="], + "ignore": ["ignore@7.0.8", "", {}, "sha512-YYNsSlXBjMk92SKnkwvB5LOVSa6OznlFUGcsvrFgNJbJCd0M1XKeFVRc8ZByeCqz32FivYNHJVooLmdqrmvp/Q=="], "import-meta-resolve": ["import-meta-resolve@4.2.0", "", {}, "sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg=="], "import-without-cache": ["import-without-cache@0.4.1", "", {}, "sha512-vXoV9PjKHEednCUu01e98TkImxy67e3BJXbTIOmIq4Hyzw3IAwYRcuPkfeTzJzwyyts4kjuA73x+pWJLc4f86A=="], - "is-what": ["is-what@5.5.7", "", {}, "sha512-b8A+OE0RcNFy/+qmts8Uv+vumkUZHwIa80YMZBfh6+b75Oq2J8qR3v+/F5vKq4TNNiurrgrE7+XsOa8+1y4lYw=="], + "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], + + "ip-address": ["ip-address@10.7.6", "", {}, "sha512-00LYkM6byILa9eQIx6qoaXCM40xTLd56AZ4vyaPW3YTyb49VZYPo5xsiAq4tuQ54GH4R7gVdYScJ1B7B77PuTg=="], + + "is-what": ["is-what@5.5.7", "https://registry.npmmirror.com/is-what/-/is-what-5.5.7.tgz", {}, "sha512-b8A+OE0RcNFy/+qmts8Uv+vumkUZHwIa80YMZBfh6+b75Oq2J8qR3v+/F5vKq4TNNiurrgrE7+XsOa8+1y4lYw=="], "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], @@ -938,7 +996,7 @@ "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], - "lru-cache": ["lru-cache@11.5.3", "", {}, "sha512-U4N8FgzmWxc8k1VH8Kr6lQg18U7Fjvby6wXHVRX/ZZ7IwWbRMgrRbP0Wrb5q5NVinryp4SQampHKdvtecItxUg=="], + "lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], "magic-string": ["magic-string@1.4.5", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.6.0" } }, "sha512-oLLC2+YvQUfAa+vxoufE/yc1icb87M9iXIRqAoR/2EhE/8cb3SF/+Gi0c2By1B+5Vz2e2vQtUvHiu1+63Znh+w=="], @@ -968,11 +1026,17 @@ "minimatch": ["minimatch@11.0.3", "", { "dependencies": { "@isaacs/nonrecursive-glob-match": "^1.0.3", "brace-expansion": "^5.0.12" } }, "sha512-iny/40PEGkNJQ5/Pv6z3nm6FbRAby32GeZRjCUJk2WnNDbgPSTcdI+DDvLqsgBdD4cD5sMtMheqPPnehCCa8Nw=="], - "minisearch": ["minisearch@7.2.0", "", {}, "sha512-dqT2XBYUOZOiC5t2HRnwADjhNS2cecp9u+TJRiJ1Qp/f5qjkeT5APcGPjHw+bz89Ms8Jp+cG4AlE+QZ/QnDglg=="], + "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], + + "minisearch": ["minisearch@7.2.0", "https://registry.npmmirror.com/minisearch/-/minisearch-7.2.0.tgz", {}, "sha512-dqT2XBYUOZOiC5t2HRnwADjhNS2cecp9u+TJRiJ1Qp/f5qjkeT5APcGPjHw+bz89Ms8Jp+cG4AlE+QZ/QnDglg=="], "moment": ["moment@2.31.0", "", {}, "sha512-0acOTfMiWOheYS4eoWb80yYMb/JLvVv9SHbs2PehaDzfUG0Bw855SKyk0IKTnPGa5+U2bmi3W68l1+sGLX/pvw=="], - "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "mqtt": ["mqtt@5.16.1", "", { "dependencies": { "@types/readable-stream": "^4.0.21", "@types/ws": "^8.18.1", "commist": "^3.2.0", "concat-stream": "^2.0.0", "debug": "^4.4.1", "help-me": "^5.0.0", "lru-cache": "^10.4.3", "minimist": "^1.2.8", "mqtt-packet": "^9.0.2", "readable-stream": "^4.7.0", "rfdc": "^1.4.1", "socks": "^2.8.6", "split2": "^4.2.0", "worker-timers": "^8.0.23", "ws": "^8.18.3" }, "bin": { "mqtt": "build/bin/mqtt.js", "mqtt_pub": "build/bin/pub.js", "mqtt_sub": "build/bin/sub.js" } }, "sha512-9aEaiR/mLblsMAiynD8quhcGTaINdta2bdyknAtfVzESfoe6Ncdh/uEs5p+3zqcTEtgYErwL4pNk0QkAynXxlg=="], + + "mqtt-packet": ["mqtt-packet@9.0.2", "", { "dependencies": { "bl": "^6.0.8", "debug": "^4.3.4", "process-nextick-args": "^2.0.1" } }, "sha512-MvIY0B8/qjq7bKxdN1eD+nrljoeaai+qjLJgfRn3TiMuz0pamsIWY2bFODPZMSNmabsLANXsLl4EMoWvlaTZWA=="], + + "ms": ["ms@2.1.3", "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], "muggle-string": ["muggle-string@0.4.1", "", {}, "sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ=="], @@ -1042,6 +1106,10 @@ "postcss-minify-selectors": ["postcss-minify-selectors@9.0.8", "", { "dependencies": { "browserslist": "^4.29.3", "caniuse-api": "^4.0.0", "cssnano-utils": "^9.0.0" }, "peerDependencies": { "postcss": "^8.5.29" } }, "sha512-qMeQvr3ODymnvoDSwml4uTWufma0T2FhRxce4htQ+45otzsm1QyNAg98DXGrawVpGooU2vElaBd8qI+YCwF0UQ=="], + "process": ["process@0.11.10", "", {}, "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A=="], + + "process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="], + "proper-lockfile": ["proper-lockfile@4.1.2", "", { "dependencies": { "graceful-fs": "^4.2.4", "retry": "^0.12.0", "signal-exit": "^3.0.2" } }, "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA=="], "property-information": ["property-information@7.2.0", "", {}, "sha512-IAtzIB6sUiWaJYrX9smp3V46pBGbBeLFRGdh25kg1334VcBlD8HzhPeNIWQH9zhGmo2itIe25EHt9dQP7G5hmg=="], @@ -1056,7 +1124,9 @@ "rclone-s3-timestamp": ["rclone-s3-timestamp@workspace:packages/rclone-s3-timestamp"], - "readdirp": ["readdirp@5.1.1", "", {}, "sha512-Kko+Y5XQ6fM+Ce3dq3m9YGxnacYZYl9cA1wZjaF3Vbry2L3i1qVg8+CAgNPsXRArPMUMCaOR7oa9Nqntc43JKA=="], + "readable-stream": ["readable-stream@4.8.0", "", { "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", "events": "^3.3.0", "process": "^0.11.10", "string_decoder": "^1.3.0" } }, "sha512-2jhTTU3Eilp/2DMT71gbcoqC35eQHasMJH0RHkYxwlxIv3+FR6Y5xAFZIwEGLxKkjZYEFlsiMNIdwUqjK2lLHQ=="], + + "readdirp": ["readdirp@5.1.1", "https://registry.npmmirror.com/readdirp/-/readdirp-5.1.1.tgz", {}, "sha512-Kko+Y5XQ6fM+Ce3dq3m9YGxnacYZYl9cA1wZjaF3Vbry2L3i1qVg8+CAgNPsXRArPMUMCaOR7oa9Nqntc43JKA=="], "regex": ["regex@6.1.0", "", { "dependencies": { "regex-utilities": "^2.3.0" } }, "sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg=="], @@ -1068,7 +1138,9 @@ "retry": ["retry@0.12.0", "", {}, "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow=="], - "rolldown": ["rolldown@1.3.1", "", { "dependencies": { "@oxc-project/types": "=0.157.0", "@rolldown/pluginutils": "^1.0.0" }, "optionalDependencies": { "@rolldown/binding-android-arm-eabi": "1.3.1", "@rolldown/binding-android-arm64": "1.3.1", "@rolldown/binding-darwin-arm64": "1.3.1", "@rolldown/binding-darwin-x64": "1.3.1", "@rolldown/binding-freebsd-x64": "1.3.1", "@rolldown/binding-linux-arm-gnueabihf": "1.3.1", "@rolldown/binding-linux-arm64-gnu": "1.3.1", "@rolldown/binding-linux-arm64-musl": "1.3.1", "@rolldown/binding-linux-ppc64-gnu": "1.3.1", "@rolldown/binding-linux-s390x-gnu": "1.3.1", "@rolldown/binding-linux-x64-gnu": "1.3.1", "@rolldown/binding-linux-x64-musl": "1.3.1", "@rolldown/binding-openharmony-arm64": "1.3.1", "@rolldown/binding-wasm32-wasi": "1.3.1", "@rolldown/binding-win32-arm64-msvc": "1.3.1", "@rolldown/binding-win32-x64-msvc": "1.3.1" }, "bin": { "rolldown": "./bin/cli.mjs" } }, "sha512-RAIazzQpeoYB0r8jWuNDX+fT1NJ+9Cqdt0pSi9jc9zwX21k2x2IUPuYJmZxDB2k8ZisNv4gNIPj+HwRCTJG4uw=="], + "rfdc": ["rfdc@1.4.1", "", {}, "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA=="], + + "rolldown": ["rolldown@1.3.0", "", { "dependencies": { "@oxc-project/types": "=0.157.0", "@rolldown/pluginutils": "^1.0.0" }, "optionalDependencies": { "@rolldown/binding-android-arm-eabi": "1.3.0", "@rolldown/binding-android-arm64": "1.3.0", "@rolldown/binding-darwin-arm64": "1.3.0", "@rolldown/binding-darwin-x64": "1.3.0", "@rolldown/binding-freebsd-x64": "1.3.0", "@rolldown/binding-linux-arm-gnueabihf": "1.3.0", "@rolldown/binding-linux-arm64-gnu": "1.3.0", "@rolldown/binding-linux-arm64-musl": "1.3.0", "@rolldown/binding-linux-ppc64-gnu": "1.3.0", "@rolldown/binding-linux-s390x-gnu": "1.3.0", "@rolldown/binding-linux-x64-gnu": "1.3.0", "@rolldown/binding-linux-x64-musl": "1.3.0", "@rolldown/binding-openharmony-arm64": "1.3.0", "@rolldown/binding-wasm32-wasi": "1.3.0", "@rolldown/binding-win32-arm64-msvc": "1.3.0", "@rolldown/binding-win32-x64-msvc": "1.3.0" }, "bin": { "rolldown": "./bin/cli.mjs" } }, "sha512-7j1jdVlzWzbRat9EDMsB/QmyMcc6hOO0vbcma8LGjBTmKlpS5Hem+5fpntc/HoRV5tLrG+Qo6I1ZHV/ZuL5S8g=="], "rolldown-plugin-dts": ["rolldown-plugin-dts@0.30.0", "", { "dependencies": { "dts-resolver": "^3.0.0", "get-tsconfig": "5.0.0-beta.6", "obug": "^3.0.0", "yuku": "^0.27.0" }, "peerDependencies": { "@volar/typescript": "~2.4.0", "@vue/language-core": "~3.2.0 || ~3.3.0", "rolldown": "^1.2.0", "tsrs": "*", "typescript": "^5.0.0 || ^6.0.0 || ~7.0.0 || ~7.1.0-0", "vue-tsc": "~3.2.0 || ~3.3.0" }, "optionalPeers": ["@volar/typescript", "@vue/language-core", "tsrs", "typescript", "vue-tsc"] }, "sha512-5FNezfYv5alHvrYw5Xoj4RNqbZ9F+/1jETFSohhYfRiama8tUN4mcVURmBqbZQba45t5hRdW7aSOnV6WYIj0MA=="], @@ -1092,8 +1164,12 @@ "signal-exit": ["signal-exit@3.0.7", "", {}, "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="], + "smart-buffer": ["smart-buffer@4.2.0", "", {}, "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg=="], + "smart-merge": ["smart-merge@workspace:packages/smart-merge"], + "socks": ["socks@2.8.10", "", { "dependencies": { "ip-address": "^10.1.1", "smart-buffer": "^4.2.0" } }, "sha512-e0VyvkVTwVYViNovRkZ9aodhxVlyoMn7eJhVUPxZ+eK9P/7CBkxvvsBOHqFPEH416726W8tLXXXjKwqgTErrCQ=="], + "solid-js": ["solid-js@1.9.18", "", { "dependencies": { "csstype": "^3.1.0", "seroval": "~1.6.11", "seroval-plugins": "~1.6.8" } }, "sha512-CHGO5YFk6Y9m/1YSxVtKnXfrW7JD3Tmof8HmR4z6TmCPsS8efo8LnzIR8CQh3cUvffqYm6DWyBFfk8dQw1CNQQ=="], "solid-refresh": ["solid-refresh@0.7.8", "", { "dependencies": { "@babel/generator": "^7.29.1", "@babel/types": "^7.29.0" }, "peerDependencies": { "solid-js": "^1.3" } }, "sha512-iG442T3HXJp5jEebCy8okETrWnmX7CnxNOKrJQVeufh6WXSn+xtLXaptUXXMHTWcRqTJOJwQ8UwzxrRtVFSIzA=="], @@ -1102,11 +1178,15 @@ "space-separated-tokens": ["space-separated-tokens@2.0.2", "", {}, "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q=="], - "srvx": ["srvx@1.0.8", "", { "bin": { "srvx": "./bin/srvx.mjs" } }, "sha512-ngFS/J6St4PJfHRHGjr78xkPPtbyXtdMOLNE4x/XZunEK3OhV7aGUWMHf9WTUEy6wm1U0W4okkd82ZFFWRVQTw=="], + "split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="], + + "srvx": ["srvx@1.0.5", "https://registry.npmmirror.com/srvx/-/srvx-1.0.5.tgz", { "bin": { "srvx": "./bin/srvx.mjs" } }, "sha512-KvSKRpgPG/oaq3cyT614OQ2bAa7DynuGamivCZeoZzIUULOdbQz6wUtnnVuC4+DCx2Zglzo8v5gBYmWIWYhDxA=="], "standardwebhooks": ["standardwebhooks@1.1.1", "", { "dependencies": { "@stablelib/base64": "^1.0.0", "fast-sha256": "^1.3.0" } }, "sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ=="], - "stringify-entities": ["stringify-entities@4.0.4", "", { "dependencies": { "character-entities-html4": "^2.0.0", "character-entities-legacy": "^3.0.0" } }, "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg=="], + "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], + + "stringify-entities": ["stringify-entities@4.0.4", "https://registry.npmmirror.com/stringify-entities/-/stringify-entities-4.0.4.tgz", { "dependencies": { "character-entities-html4": "^2.0.0", "character-entities-legacy": "^3.0.0" } }, "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg=="], "style-mod": ["style-mod@4.1.4", "", {}, "sha512-XXWIQt633/EpAFx8aZDOTjBzrCaGmhvEQlQo6MVPfa2OzO2cWo+4hV9h+6UkHYlXGfy+ODXKUdP7Pthmcu5ATw=="], @@ -1138,7 +1218,9 @@ "typebox": ["typebox@1.4.6", "", {}, "sha512-Yh8Mi4hzD7gqcuv9ZIk6HK1KVJKubTlfxaNxkrYK35HCvxoCjCuWVr8cBjQ/ChfA7yliPteJEBWHGFVGWDuF5Q=="], - "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], + "typedarray": ["typedarray@0.0.6", "", {}, "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA=="], + + "typescript": ["typescript@6.0.3", "https://registry.npmmirror.com/typescript/-/typescript-6.0.3.tgz", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], "ufo": ["ufo@1.6.4", "", {}, "sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA=="], @@ -1174,7 +1256,9 @@ "update-browserslist-db": ["update-browserslist-db@1.3.4", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-t8wonIOdQyOjtPd2VktlBu6QjTVTks6v8bSi7kd0r2Vll8TK1JYDZtRRdbmCHOAMphkZgrzK+L+vXK4pyZ8t4A=="], - "verkit": ["verkit@0.5.0", "", {}, "sha512-u4J1hr6Y+C/hit34GwL3eIcUf3wngT8TlYUjK7tk0jxydKLAq3sYvifjdTDA6U3KCKRcQCQDGf/srjxfZF3l7g=="], + "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], + + "verkit": ["verkit@0.5.0", "https://registry.npmmirror.com/verkit/-/verkit-0.5.0.tgz", {}, "sha512-u4J1hr6Y+C/hit34GwL3eIcUf3wngT8TlYUjK7tk0jxydKLAq3sYvifjdTDA6U3KCKRcQCQDGf/srjxfZF3l7g=="], "vfile": ["vfile@6.0.3", "", { "dependencies": { "@types/unist": "^3.0.0", "vfile-message": "^4.0.0" } }, "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q=="], @@ -1214,7 +1298,15 @@ "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], - "ws": ["ws@8.22.2", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-/rEZ2ejown2N5gxVTHUu9bVVC7ttu0/Tgzf5hbTOixR3Fs7mbbXgn2DbdbcKgJOmpQMtrIUWivtYuVMnp/W/yA=="], + "worker-factory": ["worker-factory@7.0.50", "", { "dependencies": { "@babel/runtime": "^7.29.7", "fast-unique-numbers": "^9.0.27", "tslib": "^2.8.1" } }, "sha512-hhwc0G+sFwM4qBuhJIUBn2p1Jf8v/FwmLUANBf/Q+Lt2uI8mfIZQhXaZQACodQD4R7Zp6cn/6702bIvNn2puJQ=="], + + "worker-timers": ["worker-timers@8.0.34", "", { "dependencies": { "@babel/runtime": "^7.29.7", "tslib": "^2.8.1", "worker-timers-broker": "^8.0.18", "worker-timers-worker": "^9.0.15" } }, "sha512-WXL+Dqsm0G6dnC66rQsvM3tPT2adhbqSirWUCZGglkALOr8ocS0KlpU0iuKbjflJOlu3ydZikMEVrYnHOM9suw=="], + + "worker-timers-broker": ["worker-timers-broker@8.0.18", "", { "dependencies": { "@babel/runtime": "^7.29.7", "broker-factory": "^3.1.15", "fast-unique-numbers": "^9.0.27", "tslib": "^2.8.1", "worker-timers-worker": "^9.0.15" } }, "sha512-FrjzDVX1wKfZN0gRbCFqv8VHuTncG4sbI/WGEg4tSSQeIsnwqg4YBYWMAHYLJtUDEYYmiK65UKFrVMVk2irDSg=="], + + "worker-timers-worker": ["worker-timers-worker@9.0.15", "", { "dependencies": { "@babel/runtime": "^7.29.7", "tslib": "^2.8.1", "worker-factory": "^7.0.50" } }, "sha512-KKUe7lZ/Aignr51H6hOUik8LwTnIgojH/1lwhli8A8qIEIyewogZTpNpMW5B6BF7nmwBOkUoTYgf1H3QShcjSA=="], + + "ws": ["ws@8.22.1", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-oyL6OPYme/CmiFvNuGzeoxCJ4nOrZE2O1FwM9MWM1AsGri6nLXB2pg+lNA8aLeXMSav0J6eIVS1j6vUPzUtK9g=="], "yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], @@ -1240,7 +1332,7 @@ "@earendil-works/pi-tui/marked": ["marked@18.0.11", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-HnslJfsZkRPBDJRHvVtAaWlZHEpSu7u8LgQuJCELjRKuWR+hpq4A7sLq3p8HaI9ypVoXDXxV34CsQJEe1+J5Aw=="], - "@types/node/undici-types": ["undici-types@8.9.0", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="], + "@unocss/autocomplete/lru-cache": ["lru-cache@11.5.3", "https://registry.npmmirror.com/lru-cache/-/lru-cache-11.5.3.tgz", {}, "sha512-U4N8FgzmWxc8k1VH8Kr6lQg18U7Fjvby6wXHVRX/ZZ7IwWbRMgrRbP0Wrb5q5NVinryp4SQampHKdvtecItxUg=="], "@vue/compiler-sfc/magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], @@ -1248,6 +1340,8 @@ "babel-plugin-jsx-dom-expressions/@babel/helper-module-imports": ["@babel/helper-module-imports@7.18.6", "", { "dependencies": { "@babel/types": "^7.18.6" } }, "sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA=="], + "concat-stream/readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], + "dom-serializer/domelementtype": ["domelementtype@3.0.0", "", {}, "sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg=="], "dom-serializer/entities": ["entities@8.1.0", "", {}, "sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA=="], @@ -1258,6 +1352,8 @@ "gaxios/https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="], + "hosted-git-info/lru-cache": ["lru-cache@11.5.3", "https://registry.npmmirror.com/lru-cache/-/lru-cache-11.5.3.tgz", {}, "sha512-U4N8FgzmWxc8k1VH8Kr6lQg18U7Fjvby6wXHVRX/ZZ7IwWbRMgrRbP0Wrb5q5NVinryp4SQampHKdvtecItxUg=="], + "htmlparser2/domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="], "htmlparser2/domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="], diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index a0e5a99e..08c852e4 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -164,6 +164,11 @@ const localeConfig = configGenerator((t) => { link: `${deepDive}/modules/rclone-s3-timestamp`, text: t('rcloneS3Timestamp'), }, + { + link: `${deepDive}/modules/collaboration`, + text: t('collaboration'), + }, + { link: `${deepDive}/modules/history`, text: t('history') }, ], text: t('modules'), }, diff --git a/docs/.vitepress/i18n.ts b/docs/.vitepress/i18n.ts index 98aef3cb..9d136375 100644 --- a/docs/.vitepress/i18n.ts +++ b/docs/.vitepress/i18n.ts @@ -9,6 +9,7 @@ const en = { benchmarking: 'Benchmarking', claims: 'Claims', code: 'en-US', + collaboration: 'Collaboration', contributing: 'Contributing', copyright: 'Copyright', debugAndTesting: 'Debug & Testing', @@ -27,6 +28,7 @@ const en = { fileTree: 'File Tree', folder: '', gdrive: 'Google Drive', + history: 'History Versions', home: 'Home', licenseMessage: 'All content licensed under the CC BY 4.0 License.', diff --git a/docs/src/pages/en/deep-dive/modules/collaboration.md b/docs/src/pages/en/deep-dive/modules/collaboration.md new file mode 100644 index 00000000..c9e4e587 --- /dev/null +++ b/docs/src/pages/en/deep-dive/modules/collaboration.md @@ -0,0 +1,73 @@ +# Collaboration Module + +Collaboration is an optional realtime notification layer. It does not replace the configured remote backend: after a sync changes remote state, a device publishes a small encrypted _pulse_ over MQTT. The pulse lists only the file keys that changed in that sync, so peers can refresh just those keys against the backend instead of walking the whole vault on every notification. + +The module registers a remote file system wrapper, a sync trigger, settings, and a conflict resolver that routes concurrently overwritten files into the configured merge strategy. The pulse carries no file content, UIDs, sizes, or rules — only the changed keys, inside the encrypted payload. The vault's full structure never travels over the broker. + +## Room Code and Keys + +A room code is generated once and stored in Obsidian's secret storage. It has the form `sec1..`, where the room ID is 16 random bytes and the secret is 32 random bytes, both encoded as base64url. + +Every client derives the same keys from the room code: + +- `PBKDF2-SHA-256` with 210,000 iterations derives a master secret from the room secret. +- The salt is `SHA-256("sync-engine-collaboration/v1:")`. +- `HKDF-SHA-256` derives a 32-byte encryption key and a 32-byte topic key from the master secret. +- The MQTT topic is `sync-engine/collaboration/v1/`. + +The room ID is therefore public inside the room code, while the topic key and encryption key are not. A client that knows the topic cannot decrypt messages without the room secret. + +## Message Flow + +Each client tracks a `dirty` flag and a set of changed keys per remote backend, both set whenever a local sync writes, moves, deletes, or creates something on the remote backend. A publish happens only when that flag is set, the transport is connected, and a remote UID is known. The flag and the key set are cleared after the pulse is sent, guarded by a revision counter so a write made during publishing is not lost. + +The encrypted payload contains a timestamp and, when it fits, the changed keys. The key set is bounded by both a maximum count and a byte budget (the ciphertext must stay under 4096 bytes). A batch that exceeds either bound degrades to a keyless pulse, which asks peers to do a full remote walk instead. All framing metadata lives in the MQTT 5 PUBLISH variable header as User Properties — `v`, `client_id`, `batch_id`, and `sent_at` — so the PUBLISH payload carries only `nonce` and `ciphertext`, and the changed keys stay inside the encrypted bytes. Brokers can identify the sender without touching the encrypted bytes. Credentials for authenticated brokers are read from Obsidian's secret storage at connect time. + +A pulse is a single small message sealed with `AES-GCM-256`. The additional authenticated data binds the topic, batch ID, client ID, and timestamp; because the User Properties feed that AAD, tampering with them makes the message fail to decrypt. There is no compression, chunking, or reassembly. + +Messages outside a two-minute clock window are rejected. Duplicate `batchId` values are ignored, and the receiver drops pulses whose `client_id` matches its own. Message size depends on the size of the change batch, not on the total number of files in the vault. + +## Applying a Pulse + +When a pulse arrives, the sync bridge compares its timestamp and batch ID with any pending pulse and keeps the newer one. It requests a collaboration-triggered sync. + +The sync core needs a _complete_ remote listing: a key missing from it is treated as a remote deletion and would remove the local file. So the trigger supplies a `remoteLister` built from the cached remote snapshot that the core's context wrapper already maintains in memory. Starting from that complete snapshot, it re-stats only the keys named in the pulse against the real backend and patches the snapshot, then returns the full array. This keeps the network cost proportional to the number of changed keys while the deciders still see the whole remote state. + +A full remote `list('/')` is still used when the local snapshot has never been populated (for example just after startup) or when the receiver gets a keyless pulse. That walk also initializes file system wrappers such as [asymmetric storage](../asymmetric-storage) before any download and refreshes the snapshot. Scheduled and startup syncs do the same, so a peer that was offline and missed several pulses heals its snapshot through the next full walk rather than relying on a continuous pulse sequence. Because both peers read the same backend, no stat list needs to be shipped. + +## Race Detection + +A concurrent upload can overwrite a file this device just uploaded. The module does not try to prevent that race; it detects it and lets the configured conflict resolver merge the divergent versions. Every edit is already captured durably by the [history versions](./history) module, so no in-memory ledger or replay is needed. + +The module journals recent uploads. During a collaboration-triggered sync, the wrapped decider checks the real remote listing: a changed UID for a recently uploaded key within the race window forces a conflict. That forced record routes the key into the configured conflict resolver, which merges both sides instead of silently dropping one. Rules whose strategy is `none` or already wrapped by the module are left unchanged. + +## Settings + +| Setting | Behavior | +| --- | --- | +| Enable collaboration | Connects to the broker and starts publishing and accepting pulses. | +| Broker URL | MQTT WebSocket URL. The default is the public EMQX broker. | +| Broker username | Optional username for authenticated brokers, stored in plain text. | +| Broker password | Optional. Selects a secret from Obsidian's secret storage for brokers that require authentication. | +| Client ID prefix | Prefilled with the Obsidian device type on first run and editable. A random suffix is added for each session so copied settings do not collide. | +| Room code | Selects a secret from Obsidian's secret storage. Generate and Copy actions are provided. | +| Topic | Copies the room topic to the clipboard with the button; no topic text is shown. | + +The status row reports the current transport state and a broker error when one is available. Reconnect applies the current settings immediately. + +## Security and Privacy + +The broker sees the topic, packet timing, encrypted packet sizes, and the plaintext User Properties (`client_id`, `batch_id`, `sent_at`, `v`). It cannot read the changed-key names: they sit inside the `AES-GCM-256` ciphertext, not in the User Properties, and it sees no UIDs, sizes, rules, or file content. A holder of the room secret can decrypt a captured pulse and learn the names of the files changed in that batch, but never the vault's full structure or any file content. This is the deliberate trade-off against a keyless design: the broker itself learns nothing about files, while a leaked room code exposes the changed names of the pulses an attacker managed to capture, rather than only packet timing. The topic is derived from the room secret, so an observer cannot discover it from the room ID alone. + +The room secret is the only credential. Anyone with the room code can join, decrypt pulses, and publish them. The module does not provide member management, key rotation, revocation, forward secrecy, or protection against a member replaying an old pulse within the freshness window. + +The public default broker is suitable for testing and for metadata that users accept exposing to the broker. Users who need a private relay should configure their own MQTT WebSocket endpoint. + +## Limits and Compatibility + +- MQTT 5 over WebSocket is required. +- A pulse is a single small message; its size depends on the change batch, not on the vault size. An oversized batch degrades to a keyless pulse and a full remote walk on the receiver. There is no chunking or reassembly. +- Incremental sync relies on the in-memory remote snapshot. A receiver with an unpopulated snapshot (cold start) or a keyless pulse falls back to a full remote walk, which refreshes the snapshot. Missed pulses while offline are healed by such a walk, not by a replayed sequence. +- Collaboration assumes every device uses the same remote backend and comparable sync rules. Different backends can produce different UIDs and are not a supported sharing setup. +- The module uses the browser build of `mqtt`; no Node.js APIs are used. +- This is file-level race detection, not character-level collaborative editing. Concurrent edits go through the configured conflict resolver. diff --git a/docs/src/pages/en/deep-dive/modules/history.md b/docs/src/pages/en/deep-dive/modules/history.md new file mode 100644 index 00000000..a794ced6 --- /dev/null +++ b/docs/src/pages/en/deep-dive/modules/history.md @@ -0,0 +1,95 @@ +# History Version Module + +The History Version Module stores every edit of a note as an immutable, content-addressed commit and restores any historical version with a bounded number of remote requests. It is an optional module built on the SDK and backed only by passive object storage: it needs no server-side script, no distributed lock, and no atomic append. It uses a dedicated handle below the [encryption](./encryption) wrapper so that remote object names stay literal content hashes; when encryption is enabled the object contents are encrypted at rest. + +The design is CDC + a content-addressed object pool + sealed pack files. The newest state and the history are separate channels, so second-level editing never waits on history I/O. + +## Goals and Non-Goals + +Goals are second-level perceived sync of the newest state, zero-loss capture of every micro-edit (including after a hard process kill), immutable remote writes that never let concurrent devices overwrite each other, restore of any version in O(1) remote round trips, and chunk-level deduplication across versions, notes, and devices. + +Character-level collaborative editing is not a goal: the vault `modify` event yields whole-file snapshots, so the module works at file and chunk granularity. It does not replace the configured remote backend or the record store (see [Sync](../sync) and [File System](../file-system)), and does not perform server-side garbage collection. When [encryption](./encryption) is enabled, history **contents** are encrypted (see [Encryption Compatibility](#encryption-compatibility)); object names stay literal. + +## Design Invariants + +Four invariants carry the whole design. + +1. **Every remote object name is self-certifying.** A name contains either a content hash or a device-scoped monotone sequence plus a payload hash. Two devices never target the same name, so multi-device safety needs no `If-Match`, ETag preconditions, or bucket versioning — none of which the `RootFs` interface exposes (see [File System Wrappers](../file-system-wrappers)). +2. **The local WAL is the source of truth until a pack is acknowledged.** An edit is durable only after its WAL entry is persisted; no in-memory staging is authoritative. +3. **The newest state and the history have separate lifecycles.** Neither gates the other: a failed history flush never delays an upload, and a failed fast upload never loses a WAL entry. +4. **A commit is a chunk list, not a patch.** Restore reads one commit and the referenced blobs; delta efficiency comes from chunk reuse, not from replaying textual diffs. + +## Dual-Channel Data Flow + +On every vault `modify`, the module awaits a WAL append keyed by a monotone sequence and stamped with a Lamport clock (`max(local, seen) + 1`). Independently, a roughly one-second debounced fast path asks the sync core to upload the live note, and the [Collaboration](./collaboration) pulse tells peers to pull. A flush predicate — idle for 30 seconds, 20 revisions, 256 KiB uncompressed, or lifecycle exit — triggers the slow history channel. The fast path reuses the existing upload task, so mtime/ctime metadata and the record store stay consistent; no remote listing happens on the save path. + +A flush snapshots the pending WAL into an immutable batch, splits content, deduplicates chunks against the local chunk index, assembles commits sorted by `(lamport, deviceId, keyHash)`, and writes one pack. The write ordering is **blobs → pack → head → WAL commit**: blob and pack writes are content-addressed and idempotent, the head log has a single writer, and WAL entries are deleted last so every earlier step is replayable. Replaying the same batch reproduces identical bytes and the same pack name. + +On cold start the module replays any pending WAL, reads the head logs (one listing), and reconciles orphan packs from a per-device pack listing (one request per device). A pack that was written but whose head log was not yet rewritten is therefore promoted to the index without reading any object: the filename already carries its sequence and payload hash. This bounded repair path satisfies the ban on scanning the full remote. + +## Local Persistence and Remote Layout + +Six `uni-kv` stores are namespaced by the vault/remote pair: a pending WAL, a chunk-to-location CAS index, a version-to-index lookup, a mirror of remote head logs, a confirmed-blob set, and the persisted Lamport clock. Each WAL entry carries its own Lamport value, so a single `store.batch` on the pending store is the atomic durability boundary for both the edit and its clock bump. Observed remote peaks are folded back into the clock during reconciliation. + +Remote objects live under a literal `sync-engine-history//` prefix with three subfolders: `packs//p--.pack`, `blobs/.blob`, and `heads/.log`. There is deliberately no `objects/` directory: chunk objects exist only as byte ranges inside packs. Each device is the sole writer of its own head log, which stays small by carrying only a recent window of pack references and a compact chunk map. + +## Chunking and Deduplication + +At WAL ingestion, content is routed by size. Notes below 4 KiB skip CDC and are stored inline as whole text with a whole-file hash; larger notes are split by FastCDC (min 512 B, average 1–2 KiB, max 4 KiB); binary attachments above 5 MiB become standalone blobs without CDC. The gear hash is hand-written in `packages/shared` and stays dependency-free and mobile-safe. All content hashing uses WebCrypto SHA-256; the 32-bit FNV-1a namespace hash is never used for content addressing. + +Chunk identity is the full SHA-256 of the chunk bytes. Before a chunk is appended to a pack, its hash is looked up in the chunk index; only unseen bytes are written, giving cross-version, cross-note, and cross-device dedup. The concatenated payload is compressed with `fflate`, not per chunk, so shared chunks stay byte-identical and verifiable. + +## Immutable Packfile Format + +A pack is a single binary file written once and never modified: a fixed 128-byte header, a length-delimited sorted index of commit entries, and a deflated payload holding small-file texts, chunk bytes, and inline blobs. The header carries the index and payload offsets and lengths, so restore can parse the index without inflating the payload. The pack hash is SHA-256 over the header fields (excluding the hash), the index, and the payload; the same value appears in the filename, making the object self-certifying at listing time. A checkpoint entry stores a full chunk list with every chunk byte present in the same pack, emitted periodically so restore never needs to resolve chunks from more than a small bounded set of older packs. + +## Multi-Device DAG and Deterministic Merge + +Each commit names its parents. A device that edits while offline produces a chain whose parent is the last commit it saw; when it later reads another device's head log, the two chains form a DAG rather than overwriting one another. Ordering is total and deterministic by `lamport`, then `deviceId`, then `keyHash`, so all devices that have seen the same commits linearize identically with no negotiation or lock. + +When two tips edit the same key, the merge is two-stage. First, chunk-sequence alignment treats each side's chunk-list as a token sequence and computes the LCS against the common ancestor with the ONP kernel extracted into `packages/shared`; unchanged chunk runs are reused directly. Second, the divergent runs are handed to a text-level three-way merge (see [Smart Merge](./smart-merge)). Restoring a multi-parent commit writes the merged text as a new WAL commit with both tips as parents; a fresh CDC pass then stores only its new chunks on the next flush. Conflict markers are still committed; nothing is silently discarded. + +## Restore + +A version pick resolves the commit location locally in O(1), fetches the pack, reads its index entry, resolves any missing chunks through the chunk index by fetching only the packs that hold them, skips attachments already confirmed present locally and fetches the rest, concatenates chunk bytes in order, inflates, and decodes UTF-8. A multi-parent target goes through the DAG merge above. Remote cost is one pack read, a small number of supplemental chunk-pack reads, and one GET per attachment, independent of how old the version is. A local index miss falls back to a head refresh, bounded per-device pack repair, and finally a full rebuild. Because the SDK `Fs` exposes whole-file reads but no Range reads, restore currently reads full packs; Range requests are a future optimization. + +## User Interface + +History is browsed per file, not from the global settings page. Right-clicking a Markdown file in the file explorer adds a **Sync Engine: Show history** item (prefixed so it cannot collide with other plugins) that opens a version browser for that file only. The browser lists the file's versions newest first, each labelled by its timestamp — the global Lamport clock is an internal ordering key and is never exposed. The newest version is preselected so its read-only preview is immediately visible. Cancel and Restore share a single row; the first Restore click reveals a confirmation line and the second writes back to the current file. Settings hold only global options — retention days and a manual garbage-collection button. + +## Storage Isolation and Registration + +History keys must stay literal so content hashes deduplicate correctly. The [Asymmetric Storage](../asymmetric-storage) wrapper flattens keys with anchors and the encryption wrapper obfuscates them, so the module does not register a file system wrapper into the sync stack. At `syncInitialized` it receives the run's `remoteFs`, walks to the root with `digOriginal`, and applies `prefixWrapper` on the history prefix. The resulting handle skips the asymmetric, context, optimizer, and encryption wrappers while still passing through the backend's request middleware (so S3 and WebDAV requests are signed correctly), and stays invisible to the sync core's listing so history objects never trigger deletion decisions. Registration is the standard set of settings, i18n, a vault `modify` listener, a workspace `file-menu` listener, and a `syncInitialized` subscription; no core type or `RootFs` signature changes are required (see [Extensibility](../extensibility)). Skipping the encryption wrapper is safe because object contents are encrypted independently, as described in [Encryption Compatibility](#encryption-compatibility). + +## Encryption Compatibility + +The dedicated handle deliberately bypasses the encryption wrapper, so history would otherwise be stored as plaintext while vault files are encrypted. To avoid that, the module detects whether [encryption](./encryption) is enabled (via the encryption module's settings and the password in Obsidian secret storage) and derives an independent AES-GCM key from that password using PBKDF2 with a module-specific salt. The key derivation is intentionally separate from the encryption module's own argon2id/HKDF derivation, so the same password never produces a reused key across two protocols. + +Every object written through the handle — packs, blobs and head logs — is encrypted at the PUT boundary and decrypted at the GET boundary. Remote object **names remain literal** (`p--.pack`, `.blob`, `.log`), preserving self-certifying names, content-hash deduplication and cross-device orphan discovery. Encrypted objects carry a magic prefix (`SEHPC1`); any payload without this header fails fast with an explicit error. If encryption is enabled but its password is missing, the module keeps the remote handle unset so no plaintext history can be written at all. Changing the encryption password makes previously encrypted objects undecipherable; the module reports this as an explicit error rather than returning garbage. Object names are not obfuscated, so the set of stored content hashes remains visible metadata — note contents never are. + +## Failure Matrix + +| Failure | Recovery | +| --- | --- | +| Kill before WAL persist | Unavoidable at this boundary; the persist is awaited before capture is acknowledged | +| Kill after WAL persist | Non-empty WAL at cold start is replayed; the pack PUT is idempotent | +| Pack PUT done, head rewrite failed | Per-device pack listing promotes the orphan without object reads | +| Torn or stale head log | Integrity mismatch or generation regression triggers re-read and listing fallback | +| Concurrent devices write the same key | Different device path segments; DAG merge on read | +| Duplicate flush replay | Identical pack name and bytes | +| Missing blob or local index miss | Refetch or head refresh → bounded repair → full rebuild | +| Encryption password changed | Encrypted objects fail to decrypt with an explicit error; no garbage content is returned | +| Encryption enabled without a password | The remote handle stays unset, so no plaintext history is written | +| Fast-channel upload failure | Independent of WAL; retried on next sync or pulse | + +## Retention and Garbage Collection + +Retention is client-side and explicit, governed by both a maximum version count per note (`maxVersions`, default 100), an age cutoff (`retentionDays`, default 90), and an intelligent tiered retention policy (`tieredRetention`, default true). To prevent rapid, continuous editing sessions from crowding out previous days' milestones, the tiered policy retains fine-grained versions for recent work (< 1 hour), at most one version per hour for the past 24 hours, at most one version per day for the past 30 days, and at most one version per week beyond that. Compaction rewrites live chunks into a new immutable pack with checkpoint entries, publishes a new head log, prunes obsolete entries from the local index, and deletes packs that contain no live versions. A pack is never mutated in place, and the operation is idempotent. + +## Collaboration + +The module captures every edit durably, so the volatile in-memory edit ledger and automatic replay in [Collaboration](./collaboration) are no longer needed: a concurrent overwrite becomes a DAG branch rather than data loss, and automatic replay would write an already-archived version back onto the mainline. Collaboration keeps its pulse, race detection, and forced-conflict routing into the configured merge strategy, while this module owns where versions live and how they are retrieved. + +## Implementation Discipline + +The module uses `undefined`, not `null`, takes `Context` structurally, keeps module-specific behavior out of plugin core, and uses no Node.js APIs (see [Project Discipline](../discipline)). Data is normalized once at ingestion: keys NFC-normalized, sizes in bytes, timestamps in Unix milliseconds. diff --git a/docs/src/pages/en/usage/permissions.md b/docs/src/pages/en/usage/permissions.md index 9aafbf4f..e8092257 100644 --- a/docs/src/pages/en/usage/permissions.md +++ b/docs/src/pages/en/usage/permissions.md @@ -22,12 +22,13 @@ Sync Engine also persists its modules in `/.obsidian/plugins/sync- ## Network Requests -Sync Engine only makes network requests for the two purposes below: +Sync Engine only makes network requests for the purposes below: - performs authorization and syncs files to the configured backend - fetches module sources and downloads modules +- connects to the MQTT broker configured in the [Collaboration](../deep-dive/modules/collaboration) module while collaboration is enabled -Requests made for syncing purpose only happen during sync runs. +Requests made for backend syncing only happen during sync runs. MQTT broker traffic happens while Collaboration is enabled. Module sources are fetched only when automatic module update starts or user opens the module management page. Sync Engine only fetches module sources defined in the "Module sources" setting. Modules are only downloaded when the user manually downloads a module or during module auto update. @@ -45,6 +46,7 @@ Obsidian secret storage is shown as "Keychain" in Obsidian settings. Sync Engine - [Google Drive](../deep-dive/modules/gdrive) module: manages a secret named `sync-engine-gdrive-refresh-token` automatically and uses that for authorizing connection checks and sync operations. - [Dropbox](../deep-dive/modules/dropbox) module: manages a secret named `sync-engine-dropbox-refresh-token` automatically and uses that for authorizing connection checks and sync operations. - [Encryption](../deep-dive/modules/encryption) module: reads the configured encryption password secret at the start of a sync run for later encryption and decryption +- [Collaboration](../deep-dive/modules/collaboration) module: reads the configured room code secret when connecting and writes a new room code only when the user requests one Sync Engine and official modules never read beyond what is provided by the user. diff --git a/modules.json b/modules.json index b623cb34..a986e4b4 100644 --- a/modules.json +++ b/modules.json @@ -86,6 +86,16 @@ "minPluginVersion": "3.4.0", "readme": "https://sync.consensia.cc/deep-dive/modules/smart-merge" }, + { + "id": "history", + "name": "History Versions", + "version": "0.1.0", + "description": "Stores every edit as an immutable, content-addressed commit and restores any version with a bounded number of remote requests.", + "icon": "history", + "main": "https://sync.consensia.cc/modules/history.js", + "minPluginVersion": "3.4.0", + "readme": "https://sync.consensia.cc/deep-dive/modules/history" + }, { "id": "rclone-s3-timestamp", "name": "Rclone S3 Timestamp", @@ -95,5 +105,15 @@ "main": "https://sync.consensia.cc/modules/rclone-s3-timestamp.js", "minPluginVersion": "3.3.0", "readme": "https://sync.consensia.cc/deep-dive/modules/rclone-s3-timestamp" + }, + { + "id": "collaboration", + "name": "Collaboration", + "version": "0.1.0", + "description": "Zero-cost realtime collaboration over encrypted MQTT messages.", + "icon": "users", + "main": "https://sync.consensia.cc/modules/collaboration.js", + "minPluginVersion": "3.4.0", + "readme": "https://sync.consensia.cc/deep-dive/modules/collaboration" } ] diff --git a/package.json b/package.json index 57865463..45ee5a39 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,8 @@ "compile": "turbo run compile", "ver": "bun scripts/version-bump.ts", "fix": "turbo run fix", - "check": "turbo run check", + "check:unused": "bun scripts/check-unused-exports.ts", + "check": "turbo run check && bun scripts/check-unused-exports.ts", "tests": "turbo run tests" }, "devDependencies": { diff --git a/packages/collaboration/package.json b/packages/collaboration/package.json new file mode 100644 index 00000000..e0f593ff --- /dev/null +++ b/packages/collaboration/package.json @@ -0,0 +1,26 @@ +{ + "name": "collaboration", + "version": "0.1.0", + "private": true, + "license": "MIT", + "contributors": [ + { + "name": "Hēsperus", + "github": "hesprs" + } + ], + "type": "module", + "scripts": { + "dev": "MODE=dev bun --bun tsdown", + "compile": "bun --bun tsdown", + "fix": "oxlint --silent --fix && oxfmt", + "check": "oxlint && oxfmt --check", + "tests": "bun --preload=./test/mocks test" + }, + "devDependencies": { + "@hesprs/sync-engine-sdk": "workspace:*", + "@repo/shared": "workspace:*", + "mqtt": "^5.16.1", + "uni-kv": "catalog:" + } +} diff --git a/packages/collaboration/src/constants.ts b/packages/collaboration/src/constants.ts new file mode 100644 index 00000000..eee45adf --- /dev/null +++ b/packages/collaboration/src/constants.ts @@ -0,0 +1,111 @@ +/** + * Collaboration Module Constants + * + * Centralizes timing intervals, debounce windows, TTL windows, payload limits, and cryptographic + * parameters for realtime collaboration pulse signaling. + * + * Design Invariant: Storage is the single source of truth. Collaboration operates exclusively as an + * ephemeral signaling (pulse) layer over MQTT, never carrying file content. + */ + +// ========================================== +// Timing, Debounce & Life-cycle Constants +// ========================================== + +/** + * Debounce delay (in ms) for processing incoming pulse messages in SyncBridge before triggering an + * incremental sync against remote storage. + */ +export const DEFAULT_PULSE_DEBOUNCE_MS = 1500; + +/** + * Time-to-live (in ms) for deduplication of received pulse message IDs. Messages older than 5 + * minutes are discarded to prevent replay attacks. + */ +export const SEEN_TTL = 300_000; + +/** + * Maximum clock skew tolerance (in ms) when evaluating timestamp freshness. Messages stamped + * farther in the past or future than 2 minutes are dropped. + */ +export const MAX_CLOCK_SKEW = 120_000; + +/** + * Time window (in ms) to track recent remote uploads in RaceJournal. Used to detect concurrent + * upload races from peers and avoid false overwrites. + */ +export const RACE_WINDOW = 30_000; + +/** Maximum time (in ms) to wait for an MQTT connection / subscription to settle. */ +export const MQTT_CONNECT_TIMEOUT_MS = 15_000; + +/** Maximum time (in ms) to wait for an MQTT publish acknowledgment before timing out. */ +export const MQTT_PUBLISH_TIMEOUT_MS = 15_000; + +// ========================================== +// Protocol, Buffer & Payload Limits +// ========================================== + +/** Wire protocol version for collaboration envelopes. */ +export const PROTOCOL_VERSION = 1; + +/** Default secret ID for storing collaboration room secrets in the credential store. */ +export const DEFAULT_SECRET_ID = 'sync-engine-collaboration'; + +/** Maximum entries retained in the deduplication cache for seen message IDs. */ +export const MAX_SEEN_CACHE = 8192; + +/** Maximum entries retained in the concurrent upload race journal. */ +export const MAX_JOURNAL_ENTRIES = 4096; + +/** Prefix applied to dynamic deciders registered by the collaboration module. */ +export const WRAPPER_PREFIX = 'collaboration:'; + +/** + * Maximum ciphertext size (in bytes) for a pulse envelope. A pulse is a single small ciphertext; + * this bounds absurd messages from the broker. + */ +export const MAX_CIPHERTEXT_BYTES = 4096; + +/** + * Maximum plaintext size (in bytes) for a pulse envelope payload. AES-GCM adds a 16-byte tag; the + * rest is reserved for JSON framing. + */ +export const MAX_PLAINTEXT_BYTES = MAX_CIPHERTEXT_BYTES - 16 - 64; + +/** Upper bound on changed keys carried in one pulse envelope, independent of the byte budget. */ +export const MAX_CHANGED_KEYS = 512; + +// ========================================== +// Cryptographic Parameters +// ========================================== + +/** Prefix for serialized human-shareable room codes. */ +export const ROOM_CODE_PREFIX = 'sec1'; + +/** Byte length of the room ID random entropy. */ +export const ROOM_ID_BYTES = 16; + +/** Byte length of the room secret shared key. */ +export const ROOM_SECRET_BYTES = 32; + +/** Byte length of AES-GCM Initialization Vector (nonce). */ +export const NONCE_BYTES = 12; + +/** Byte length of derived message encryption key. */ +export const MESSAGE_KEY_BYTES = 32; + +/** Byte length of derived topic authentication key. */ +export const TOPIC_KEY_BYTES = 32; + +/** PBKDF2 iteration count for deriving master room key from secret. */ +export const KEY_DERIVATION_ITERATIONS = 210_000; + +/** Domain-separation salt for key derivation. */ +export const CRYPTO_SALT = 'sync-engine-collaboration/v1'; + +/** HKDF / PBKDF2 info string for message encryption key. */ +export const ENCRYPTION_INFO = 'message-encryption'; + +/** HKDF / PBKDF2 info string for topic derivation key. */ +export const TOPIC_INFO = 'topic'; diff --git a/packages/collaboration/src/crypto.ts b/packages/collaboration/src/crypto.ts new file mode 100644 index 00000000..b8e5097f --- /dev/null +++ b/packages/collaboration/src/crypto.ts @@ -0,0 +1,144 @@ +import { + CRYPTO_SALT, + ENCRYPTION_INFO, + KEY_DERIVATION_ITERATIONS, + MAX_CLOCK_SKEW, + MESSAGE_KEY_BYTES, + NONCE_BYTES, + ROOM_CODE_PREFIX, + ROOM_ID_BYTES, + ROOM_SECRET_BYTES, + TOPIC_INFO, + TOPIC_KEY_BYTES, +} from '@/constants'; +import { decodeBase64Url, encodeBase64Url, toArrayBuffer } from '@repo/shared/binary'; +import type { Binary } from '@hesprs/sync-engine-sdk'; + +export { decodeBase64, decodeBase64Url, encodeBase64, encodeBase64Url } from '@repo/shared/binary'; +export { + KEY_DERIVATION_ITERATIONS, + MAX_CLOCK_SKEW, + MESSAGE_KEY_BYTES, + NONCE_BYTES, + ROOM_CODE_PREFIX, + ROOM_ID_BYTES, + ROOM_SECRET_BYTES, + TOPIC_KEY_BYTES, +} from '@/constants'; + +const textEncoder = new TextEncoder(); + +export type RoomCode = { roomId: Binary; secret: Binary }; +export type RoomKeys = { encryptionKey: CryptoKey; topicKey: CryptoKey; roomId: string }; +export type ChunkAAD = { batchId: string; clientId: string; sentAt: number; topic: string }; + +export function generateRoomCode(): string { + const roomId = crypto.getRandomValues(new Uint8Array(ROOM_ID_BYTES)); + const secret = crypto.getRandomValues(new Uint8Array(ROOM_SECRET_BYTES)); + return `${ROOM_CODE_PREFIX}.${encodeBase64Url(roomId)}.${encodeBase64Url(secret)}`; +} + +export function parseRoomCode(value: string): RoomCode { + const [prefix, roomIdPart, secretPart, ...rest] = value.trim().split('.'); + if (prefix !== ROOM_CODE_PREFIX || !roomIdPart || !secretPart || rest.length !== 0) + throw new Error('Invalid collaboration room code'); + const roomId = decodeBase64Url(roomIdPart); + const secret = decodeBase64Url(secretPart); + if (roomId.byteLength !== ROOM_ID_BYTES || secret.byteLength !== ROOM_SECRET_BYTES) + throw new Error('Invalid collaboration room code'); + return { roomId, secret }; +} + +export function createChunkAAD({ batchId, clientId, sentAt, topic }: ChunkAAD): Binary { + return textEncoder.encode( + [ROOM_CODE_PREFIX, topic, batchId, clientId, String(sentAt)].join('\0'), + ); +} + +export async function deriveRoomKeys(room: RoomCode): Promise { + const roomId = encodeBase64Url(room.roomId); + const salt = await crypto.subtle.digest( + 'SHA-256', + textEncoder.encode(`${CRYPTO_SALT}:${roomId}`), + ); + const secretKey = await crypto.subtle.importKey( + 'raw', + toArrayBuffer(room.secret), + 'PBKDF2', + false, + ['deriveBits'], + ); + const master = await crypto.subtle.deriveBits( + { hash: 'SHA-256', iterations: KEY_DERIVATION_ITERATIONS, name: 'PBKDF2', salt }, + secretKey, + MESSAGE_KEY_BYTES * 8, + ); + const [encryptionKey, topicKey] = await Promise.all([ + deriveHkdfKey(master, ENCRYPTION_INFO, salt), + deriveHkdfKey(master, TOPIC_INFO, salt), + ]); + return { + encryptionKey: await crypto.subtle.importKey('raw', encryptionKey, 'AES-GCM', false, [ + 'encrypt', + 'decrypt', + ]), + roomId, + topicKey: await crypto.subtle.importKey( + 'raw', + topicKey, + { hash: 'SHA-256', name: 'HMAC' }, + false, + ['sign'], + ), + }; +} + +async function deriveHkdfKey( + master: BufferSource, + info: string, + salt: BufferSource, +): Promise { + const key = await crypto.subtle.importKey('raw', master, 'HKDF', false, ['deriveBits']); + return crypto.subtle.deriveBits( + { hash: 'SHA-256', info: textEncoder.encode(info), name: 'HKDF', salt }, + key, + TOPIC_KEY_BYTES * 8, + ); +} + +export async function deriveTopic(keys: RoomKeys): Promise { + const digest = await crypto.subtle.sign('HMAC', keys.topicKey, textEncoder.encode(keys.roomId)); + return `sync-engine/collaboration/v1/${encodeBase64Url(digest)}`; +} + +export async function encryptChunk( + key: CryptoKey, + value: BufferSource, + aad: BufferSource, +): Promise<{ ciphertext: Binary; nonce: Binary }> { + const nonce = crypto.getRandomValues(new Uint8Array(NONCE_BYTES)); + const ciphertext = await crypto.subtle.encrypt( + { additionalData: aad, iv: nonce, name: 'AES-GCM' }, + key, + value, + ); + return { ciphertext: new Uint8Array(ciphertext), nonce }; +} + +export async function decryptChunk( + key: CryptoKey, + ciphertext: BufferSource, + nonce: BufferSource, + aad: BufferSource, +): Promise { + const value = await crypto.subtle.decrypt( + { additionalData: aad, iv: nonce, name: 'AES-GCM' }, + key, + ciphertext, + ); + return new Uint8Array(value); +} + +export function isFresh(sentAt: number, now = Date.now(), tolerance = MAX_CLOCK_SKEW): boolean { + return Number.isFinite(sentAt) && Math.abs(now - sentAt) <= tolerance; +} diff --git a/packages/collaboration/src/i18n.ts b/packages/collaboration/src/i18n.ts new file mode 100644 index 00000000..0ef2fe24 --- /dev/null +++ b/packages/collaboration/src/i18n.ts @@ -0,0 +1,242 @@ +export type CollaborationTranslations = { + brokerConnectionError: (message: string) => string; + brokerUrl: string; + brokerUrlDescription: string; + brokerUrlPlaceholder: string; + clientIdPrefix: string; + clientIdPrefixDescription: string; + collaboration: string; + copyRoomCode: string; + copyTopic: string; + enableCollaboration: string; + enableCollaborationDescription: string; + generateRoomCode: string; + invalidMessage: string; + mobileStatusIcon: string; + mobileStatusIconDescription: string; + mqttErrorSuffix: string; + password: string; + passwordDescription: string; + publishFailed: (message: string) => string; + raceDetected: (count: number) => string; + reconnect: string; + roomCode: string; + roomCodeCopied: string; + roomCodeDescription: string; + roomCodeInvalid: string; + roomCodeMissing: string; + staleMessage: string; + status: string; + statusConnected: string; + statusConnecting: string; + statusDisabled: string; + statusError: string; + statusMissingRoomCode: string; + statusReconnecting: string; + topic: string; + topicCopied: string; + topicDescription: string; + username: string; + usernameDescription: string; + usernamePlaceholder: string; +}; + +const en: CollaborationTranslations = { + brokerConnectionError: (message) => `Collaboration broker connection failed: ${message}`, + brokerUrl: 'Broker URL', + brokerUrlDescription: + 'MQTT broker WebSocket URL. The default is a free public EMQX broker; do not use it for messages you are not willing to expose to the broker.', + brokerUrlPlaceholder: 'wss://broker.emqx.io:8084/mqtt', + clientIdPrefix: 'Client ID prefix', + clientIdPrefixDescription: + 'Prefilled with your device type on first run. Each session appends a random suffix so clients do not disconnect each other.', + collaboration: 'Collaboration', + copyRoomCode: 'Copy', + copyTopic: 'Copy topic', + enableCollaboration: 'Enable collaboration', + enableCollaborationDescription: + 'Connect to the broker and exchange encrypted change notifications with devices using the same room code. Peers then list the remote backend themselves.', + generateRoomCode: 'Generate', + invalidMessage: 'Ignored an invalid or untrusted collaboration message.', + mobileStatusIcon: 'Show connection status icon on mobile', + mobileStatusIconDescription: + 'Display live collaboration connection status icon in the top navigation bar on mobile devices.', + mqttErrorSuffix: ' (MQTT error)', + password: 'Broker password', + passwordDescription: + 'Select a secret from Obsidian secret storage for brokers that require authentication.', + publishFailed: (message) => `Collaboration notification failed: ${message}`, + raceDetected: (count) => + `Detected ${count} concurrently overwritten file(s); routing to the configured conflict resolver.`, + reconnect: 'Reconnect', + roomCode: 'Room code', + roomCodeCopied: 'Room code copied.', + roomCodeDescription: + 'Create one room code and add it to Obsidian secret storage on every device. Share it only through a trusted channel.', + roomCodeInvalid: 'The collaboration room code is invalid.', + roomCodeMissing: 'No collaboration room code is configured.', + staleMessage: + 'Ignored a collaboration message because the device clocks differ by more than 2 minutes.', + status: 'Status', + statusConnected: 'Connected', + statusConnecting: 'Connecting', + statusDisabled: 'Disabled', + statusError: 'Error', + statusMissingRoomCode: 'Room code missing', + statusReconnecting: 'Reconnecting', + topic: 'Topic', + topicCopied: 'Topic copied.', + topicDescription: 'Copy the topic used by this room to the clipboard.', + username: 'Broker username', + usernameDescription: 'Username for brokers that require authentication. Stored in plain text.', + usernamePlaceholder: 'username', +}; + +const zh: CollaborationTranslations = { + brokerConnectionError: (message) => `协作 broker 连接失败:${message}`, + brokerUrl: 'Broker 地址', + brokerUrlDescription: + 'MQTT broker 的 WebSocket 地址。默认为免费的公共 EMQX broker;公共 broker 会看到消息元数据。', + brokerUrlPlaceholder: 'wss://broker.emqx.io:8084/mqtt', + clientIdPrefix: '客户端 ID 前缀', + clientIdPrefixDescription: + '首次启用时自动填入本设备类型,可自定义。每次启动会在其后附加随机后缀,避免客户端互相踢下线。', + collaboration: '实时协作', + copyRoomCode: '复制', + copyTopic: '复制主题', + enableCollaboration: '启用实时协作', + enableCollaborationDescription: + '连接 broker,并与使用相同房间码的设备交换加密的变更通知,由各端自行遍历远端。', + generateRoomCode: '生成', + invalidMessage: '已忽略无效或不可信的协作消息。', + mobileStatusIcon: '移动端顶部显示连接状态图标', + mobileStatusIconDescription: '在移动端笔记顶部操作栏显示实时协作连接状态图标(WiFi 图标)。', + mqttErrorSuffix: ' (MQTT 异常)', + password: 'Broker 密码', + passwordDescription: '从 Obsidian 密钥存储中选择一个密钥,用于需要认证的 broker。', + publishFailed: (message) => `协作通知发送失败:${message}`, + raceDetected: (count) => `检测到 ${count} 个并发覆盖的文件,正在交由配置的冲突解决器处理。`, + reconnect: '重新连接', + roomCode: '房间码', + roomCodeCopied: '房间码已复制。', + roomCodeDescription: + '生成一次房间码,并在每台设备上加入 Obsidian 密钥存储。请只通过可信渠道分享。', + roomCodeInvalid: '协作房间码无效。', + roomCodeMissing: '尚未配置协作房间码。', + staleMessage: '已忽略一条协作消息:设备时钟相差超过 2 分钟。', + status: '状态', + statusConnected: '已连接', + statusConnecting: '正在连接', + statusDisabled: '已禁用', + statusError: '错误', + statusMissingRoomCode: '缺少房间码', + statusReconnecting: '正在重连', + topic: '主题', + topicCopied: '主题已复制。', + topicDescription: '复制此房间使用的主题到剪贴板。', + username: 'Broker 用户名', + usernameDescription: '需要认证的 broker 的用户名,明文保存。', + usernamePlaceholder: '用户名', +}; + +const zhTW: CollaborationTranslations = { + brokerConnectionError: (message) => `協作 broker 連線失敗:${message}`, + brokerUrl: 'Broker 位址', + brokerUrlDescription: + 'MQTT broker 的 WebSocket 位址。預設為免費的公共 EMQX broker;公共 broker 會看到訊息中繼資料。', + brokerUrlPlaceholder: 'wss://broker.emqx.io:8084/mqtt', + clientIdPrefix: '用戶端 ID 前綴', + clientIdPrefixDescription: + '首次啟用時自動填入本裝置類型,可自訂。每次啟動會在其後附加隨機後綴,避免用戶端互相中斷連線。', + collaboration: '即時協作', + copyRoomCode: '複製', + copyTopic: '複製主題', + enableCollaboration: '啟用即時協作', + enableCollaborationDescription: + '連線 broker,並與使用相同房間碼的裝置交換加密的變更通知,由各端自行遍歷遠端。', + generateRoomCode: '產生', + invalidMessage: '已忽略無效或不可信的協作訊息。', + mobileStatusIcon: '行動端頂部顯示連線狀態圖示', + mobileStatusIconDescription: '在行動端筆記頂部操作列顯示即時協作連線狀態圖示(WiFi 圖示)。', + mqttErrorSuffix: ' (MQTT 異常)', + password: 'Broker 密碼', + passwordDescription: '從 Obsidian 密鑰儲存中選擇一個密鑰,用於需要認證的 broker。', + publishFailed: (message) => `協作通知傳送失敗:${message}`, + raceDetected: (count) => `偵測到 ${count} 個並行覆寫的檔案,正在交由設定的衝突解決器處理。`, + reconnect: '重新連線', + roomCode: '房間碼', + roomCodeCopied: '房間碼已複製。', + roomCodeDescription: + '產生一次房間碼,並在每台裝置上加入 Obsidian 密鑰儲存。請只透過可信管道分享。', + roomCodeInvalid: '協作房間碼無效。', + roomCodeMissing: '尚未設定協作房間碼。', + staleMessage: '已忽略一則協作訊息:裝置時鐘相差超過 2 分鐘。', + status: '狀態', + statusConnected: '已連線', + statusConnecting: '正在連線', + statusDisabled: '已停用', + statusError: '錯誤', + statusMissingRoomCode: '缺少房間碼', + statusReconnecting: '正在重新連線', + topic: '主題', + topicCopied: '主題已複製。', + topicDescription: '複製此房間使用的主題到剪貼簿。', + username: 'Broker 使用者名稱', + usernameDescription: '需要認證的 broker 的使用者名稱,以明文儲存。', + usernamePlaceholder: '使用者名稱', +}; + +const ru: CollaborationTranslations = { + brokerConnectionError: (message) => + `Не удалось подключиться к брокеру совместной работы: ${message}`, + brokerUrl: 'URL брокера', + brokerUrlDescription: + 'WebSocket-адрес брокера MQTT. По умолчанию используется бесплатный публичный EMQX; брокер видит метаданные сообщений.', + brokerUrlPlaceholder: 'wss://broker.emqx.io:8084/mqtt', + clientIdPrefix: 'Префикс ID клиента', + clientIdPrefixDescription: + 'При первом запуске подставляется тип устройства; значение можно изменить. К префиксу добавляется случайный суффикс на каждый сеанс, чтобы клиенты не отключали друг друга.', + collaboration: 'Совместная работа', + copyRoomCode: 'Копировать', + copyTopic: 'Копировать топик', + enableCollaboration: 'Включить совместную работу', + enableCollaborationDescription: + 'Подключиться к брокеру и обмениваться зашифрованными уведомлениями с устройствами, использующими тот же код комнаты.', + generateRoomCode: 'Создать', + invalidMessage: 'Недопустимое или недоверенное сообщение совместной работы проигнорировано.', + mobileStatusIcon: 'Значок статуса на мобильных устройствах', + mobileStatusIconDescription: + 'Отображать значок статуса подключения совместной работы на верхней панели мобильных устройств.', + mqttErrorSuffix: ' (Ошибка MQTT)', + password: 'Пароль брокера', + passwordDescription: + 'Выберите секрет из хранилища секретов Obsidian для брокеров, требующих аутентификации.', + publishFailed: (message) => `Не удалось отправить уведомление совместной работы: ${message}`, + raceDetected: (count) => + `Обнаружено файлов с одновременной перезаписью: ${count}. Передаём настроенному обработчику конфликтов.`, + reconnect: 'Переподключить', + roomCode: 'Код комнаты', + roomCodeCopied: 'Код комнаты скопирован.', + roomCodeDescription: + 'Создайте один код комнаты и добавьте его в хранилище секретов Obsidian на каждом устройстве. Передавайте его только по доверенному каналу.', + roomCodeInvalid: 'Неверный код комнаты.', + roomCodeMissing: 'Код комнаты не настроен.', + staleMessage: + 'Сообщение совместной работы проигнорировано: разница часов устройств превышает 2 минуты.', + status: 'Статус', + statusConnected: 'Подключено', + statusConnecting: 'Подключение', + statusDisabled: 'Отключено', + statusError: 'Ошибка', + statusMissingRoomCode: 'Нет кода комнаты', + statusReconnecting: 'Переподключение', + topic: 'Топик', + topicCopied: 'Топик скопирован.', + topicDescription: 'Скопировать топик этой комнаты в буфер обмена.', + username: 'Имя пользователя брокера', + usernameDescription: + 'Имя пользователя для брокеров, требующих аутентификации. Хранится в открытом виде.', + usernamePlaceholder: 'имя пользователя', +}; + +export { en, ru, zh, zhTW }; diff --git a/packages/collaboration/src/index.ts b/packages/collaboration/src/index.ts new file mode 100644 index 00000000..c2763412 --- /dev/null +++ b/packages/collaboration/src/index.ts @@ -0,0 +1,437 @@ +import { DEFAULT_SECRET_ID, MAX_SEEN_CACHE, SEEN_TTL } from '@/constants'; +import { + deriveRoomKeys, + deriveTopic, + encodeBase64Url, + generateRoomCode as createRoomCode, + isFresh, + parseRoomCode, +} from '@/crypto'; +import { en, ru, zh, zhTW } from '@/i18n'; +import { openPulse, parseEnvelope, preparePulse, sealPulse } from '@/protocol'; +import { RaceJournal } from '@/race'; +import captureRemoteState from '@/remote-wrapper'; +import collaborationSetting from '@/setting'; +import RemoteStateCache from '@/state-cache'; +import SyncBridge from '@/sync-bridge'; +import CollaborationTransport from '@/transport'; +import deviceType, { + buildClientId, + cleanupMobileHeaderIcons, + readSecret, + text, + toError, +} from '@/utils'; +import type { Envelope, UserProperties } from '@/protocol'; +import type { CollaborationSettings, CollaborationStatus } from '@/setting'; +import type { CollaborationContext } from '@/types'; + +export default class Collaboration { + private readonly cleanup: Array<() => void> = []; + private readonly instanceId = encodeBase64Url(crypto.getRandomValues(new Uint8Array(9))); + private readonly journal = new RaceJournal(); + private readonly seen = new Map(); + private readonly stateCache = new RemoteStateCache(); + private bridge?: SyncBridge; + private clientId = ''; + private connectionGeneration = 0; + private keys?: Awaited>; + private lastErrorLog = 0; + private lastInvalidLog = 0; + private lastRemoteUid?: string; + private lastStaleLog = 0; + private publishQueued = false; + private publishing = false; + private roomTopic?: string; + private status: CollaborationStatus = 'disabled'; + private statusError?: Error; + private topic?: string; + private transport?: CollaborationTransport; + + moduleSettings: CollaborationSettings = { + brokerUrl: 'wss://broker.emqx.io:8084/mqtt', + // Empty means "not set yet"; start() fills in the detected device type. + clientIdPrefix: '', + enabled: false, + passwordSecretId: '', + roomSecretId: '', + username: '', + }; + + constructor(private readonly ctx: CollaborationContext) { + for (const [locale, res] of [ + ['en', en], + ['zh', zh], + ['zh-TW', zhTW], + ['ru', ru], + ] as const) + ctx.registerI18n(locale, res); + } + + readonly start = () => { + const { ctx } = this; + if (!this.moduleSettings.clientIdPrefix) { + this.moduleSettings.clientIdPrefix = deviceType(); + void ctx.saveSettings(); + } + this.bridge = new SyncBridge( + { + deciderRegistry: ctx.deciderRegistry, + dispatch: ctx.dispatch, + getConflictResolver: ctx.getConflictResolver, + memoryDB: ctx.memoryDB, + on: ctx.on, + registerDecider: ctx.registerDecider, + requestSync: ctx.requestSync, + translate: ctx.translate, + }, + { + fastMode: () => Boolean(this.ctx.settings.realtimeSyncFastMode), + journal: this.journal, + rules: () => this.ctx.settings.syncStrategy, + }, + ); + this.cleanup.push( + ctx.registerRemoteFsWrapper({ + apply: (fs) => { + if (!this.moduleSettings.enabled || !this.bridge) return; + return captureRemoteState(fs, { + activeUploads: this.bridge.activeUploadKeys, + enabled: () => this.moduleSettings.enabled, + journal: this.journal, + stateCache: this.stateCache, + }); + }, + priority: 30_100, + }), + ctx.registerTrigger('collaboration', { + options: () => ({ + needConfirmDeletion: this.ctx.settings.confirmDeleteInAutoSync, + ...this.bridge?.triggerOptions(), + }), + priority: 1500, + }), + ctx.registerTrigger('realtime', { + options: () => ({ + needConfirmDeletion: this.ctx.settings.confirmDeleteInAutoSync, + ...this.bridge?.triggerOptions(), + }), + priority: 1000, + }), + ctx.registerSetting({ + apply: collaborationSetting( + { + app: ctx.app, + copyRoomCode: this.copyRoomCode, + copyTopic: this.copyTopic, + generateRoomCode: this.generateRoomCode, + getStatus: this.getStatus, + getTopic: () => this.topic, + reconnect: this.reconnect, + refreshSettingTab: ctx.refreshSettingTab, + rerenderSettingTab: ctx.rerenderSettingTab, + saveSettings: ctx.saveSettings, + translate: ctx.translate, + }, + this.moduleSettings, + ), + priority: 3050, + }), + ctx.on('syncInitialized', ({ remoteFs }) => { + this.lastRemoteUid = remoteFs.getUid(); + }), + ctx.on('syncTerminated', () => { + void this.publishPulse(); + }), + ); + if (typeof window !== 'undefined') { + const onOnline = () => { + if (this.moduleSettings.enabled && this.status !== 'connected') + void this.reconnect(); + }; + const onVisibility = () => { + if ( + document.visibilityState === 'visible' && + this.moduleSettings.enabled && + this.status !== 'connected' + ) + void this.reconnect(); + }; + window.addEventListener('online', onOnline); + document.addEventListener('visibilitychange', onVisibility); + this.cleanup.push(() => { + window.removeEventListener('online', onOnline); + document.removeEventListener('visibilitychange', onVisibility); + }); + } + cleanupMobileHeaderIcons(); + this.updateModuleStatus(); + void this.reconnect(); + }; + + readonly dispose = () => { + this.connectionGeneration++; + this.transport?.disconnect(); + this.transport = undefined; + this.bridge?.dispose(); + this.bridge = undefined; + this.cleanup.splice(0).forEach((cleanup) => cleanup()); + this.resetRoomState(); + this.keys = undefined; + this.topic = undefined; + this.roomTopic = undefined; + this.status = 'disabled'; + this.updateModuleStatus(); + cleanupMobileHeaderIcons(); + }; + + private readonly getStatus = () => ({ error: this.statusError, status: this.status }); + + private readonly setStatus = (status: CollaborationStatus, error?: Error) => { + this.status = status; + this.statusError = error; + this.updateModuleStatus(); + this.ctx.rerenderSettingTab(); + }; + + private updateModuleStatus(): void { + if (!this.moduleSettings.enabled || this.status === 'disabled') { + this.ctx.setModuleStatus('collaboration'); + return; + } + if (this.status === 'connected') { + this.ctx.setModuleStatus('collaboration', { level: 'ok' }); + return; + } + const isError = this.status === 'error'; + const key = isError + ? 'statusError' + : this.status === 'reconnecting' + ? 'statusReconnecting' + : this.status === 'connecting' + ? 'statusConnecting' + : 'statusMissingRoomCode'; + const msg = + isError && this.statusError?.message + ? this.statusError.message + : text(this.ctx.translate(key)); + this.ctx.setModuleStatus('collaboration', { + level: isError ? 'error' : 'warning', + message: `MQTT: ${msg}`, + }); + } + + private readonly resetRoomState = () => { + this.seen.clear(); + this.journal.clear(); + this.stateCache.clearAll(); + this.bridge?.reset(); + this.publishQueued = false; + }; + + private readonly reconnect = async () => { + const generation = ++this.connectionGeneration; + const previousTopic = this.roomTopic; + this.transport?.disconnect(); + this.transport = undefined; + this.keys = undefined; + this.topic = undefined; + this.publishQueued = false; + if (!this.moduleSettings.enabled) { + this.setStatus('disabled'); + return; + } + const secret = readSecret( + this.ctx.app, + this.moduleSettings.roomSecretId || DEFAULT_SECRET_ID, + ); + if (!secret) { + this.setStatus('missing-room'); + this.ctx.dispatch('logGeneral', text(this.ctx.translate('roomCodeMissing'))); + return; + } + let room: ReturnType; + try { + room = parseRoomCode(secret); + } catch { + const error = new Error(text(this.ctx.translate('roomCodeInvalid'))); + this.setStatus('error', error); + this.ctx.dispatch('errorGeneral', error); + return; + } + let keys: Awaited>; + let topic: string; + try { + keys = await deriveRoomKeys(room); + topic = await deriveTopic(keys); + } catch (error) { + const normalized = toError(error); + this.setStatus('error', normalized); + this.ctx.dispatch('errorGeneral', normalized); + return; + } + if (generation !== this.connectionGeneration) return; + if (previousTopic !== undefined && previousTopic !== topic) this.resetRoomState(); + this.keys = keys; + this.roomTopic = topic; + this.topic = topic; + this.clientId = buildClientId(this.moduleSettings.clientIdPrefix, this.instanceId); + const transport = new CollaborationTransport({ + clientId: this.clientId, + onMessage: this.onMessage, + onStatus: this.onStatus, + password: readSecret(this.ctx.app, this.moduleSettings.passwordSecretId), + topic, + url: this.moduleSettings.brokerUrl, + username: this.moduleSettings.username || undefined, + }); + this.transport = transport; + await transport.connect(); + if (generation !== this.connectionGeneration) transport.disconnect(); + }; + + private readonly onStatus = ( + status: 'connected' | 'connecting' | 'disabled' | 'error' | 'reconnecting', + error?: Error, + ) => { + this.setStatus(status, error ?? (status === 'reconnecting' ? this.statusError : undefined)); + const now = Date.now(); + if (error && now - this.lastErrorLog > 5000) { + this.lastErrorLog = now; + this.ctx.dispatch( + 'errorGeneral', + new Error(text(this.ctx.translate('brokerConnectionError', error.message))), + ); + } + if (status === 'connected') void this.publishPulse(); + }; + + private readonly onMessage = (raw: string, properties: UserProperties) => { + if (!this.keys || !this.topic) return; + let envelope: Envelope; + try { + envelope = parseEnvelope(raw, properties); + } catch { + this.logInvalidMessage(); + return; + } + if (envelope.clientId === this.clientId) return; + if (!isFresh(envelope.sentAt)) { + this.logStaleMessage(); + return; + } + if (this.seen.has(envelope.batchId)) return; + this.seen.set(envelope.batchId, Date.now()); + this.pruneReceived(); + void this.finishPulse(envelope); + }; + + private readonly finishPulse = async (envelope: Envelope) => { + const keys = this.keys; + const topic = this.topic; + if (!keys || !topic) return; + try { + const pulse = await openPulse({ envelope, keys, topic }); + if (!isFresh(pulse.sentAt) || this.topic !== topic) return; + this.bridge?.accept(pulse); + } catch { + this.logInvalidMessage(); + } + }; + + private readonly pruneReceived = () => { + const now = Date.now(); + for (const [key, timestamp] of this.seen) + if (now - timestamp > SEEN_TTL) this.seen.delete(key); + while (this.seen.size > MAX_SEEN_CACHE) { + const oldest = this.seen.keys().next().value; + if (oldest === undefined) break; + this.seen.delete(oldest); + } + }; + + private readonly logInvalidMessage = () => { + const now = Date.now(); + if (now - this.lastInvalidLog < 60_000) return; + this.lastInvalidLog = now; + this.ctx.dispatch('logGeneral', text(this.ctx.translate('invalidMessage'))); + }; + + private readonly logStaleMessage = () => { + const now = Date.now(); + if (now - this.lastStaleLog < 60_000) return; + this.lastStaleLog = now; + this.ctx.dispatch('logGeneral', text(this.ctx.translate('staleMessage'))); + }; + + private readonly publishPulse = async () => { + if (this.publishing) { + this.publishQueued = true; + return; + } + if ( + !this.moduleSettings.enabled || + !this.keys || + !this.topic || + !this.transport || + this.status !== 'connected' || + !this.lastRemoteUid + ) + return; + const uid = this.lastRemoteUid; + if (!this.stateCache.isDirty(uid)) return; + const revision = this.stateCache.getRevision(uid); + const changedKeys = this.stateCache.takeChanged(uid); + const generation = this.connectionGeneration; + const keys = this.keys; + const topic = this.topic; + const transport = this.transport; + this.publishing = true; + try { + const prepared = preparePulse({ keys: changedKeys, sentAt: Date.now() }); + if (generation !== this.connectionGeneration || transport !== this.transport) return; + const envelopes = await sealPulse({ clientId: this.clientId, keys, prepared, topic }); + if (generation !== this.connectionGeneration || transport !== this.transport) return; + await transport.publish(envelopes); + if (generation !== this.connectionGeneration || transport !== this.transport) return; + this.stateCache.clearDirty(uid, revision); + } catch (error) { + const normalized = toError(error); + this.ctx.dispatch( + 'errorGeneral', + new Error(text(this.ctx.translate('publishFailed', normalized.message))), + ); + } finally { + this.publishing = false; + if (this.publishQueued) { + this.publishQueued = false; + void this.publishPulse(); + } + } + }; + + private readonly generateRoomCode = async (): Promise => { + const code = createRoomCode(); + const id = this.moduleSettings.roomSecretId || DEFAULT_SECRET_ID; + this.ctx.app.secretStorage.setSecret(id, code); + this.moduleSettings.roomSecretId = id; + await this.ctx.saveSettings(); + void this.reconnect(); + return this.copyRoomCode(); + }; + + private readonly copyRoomCode = async (): Promise => { + const id = this.moduleSettings.roomSecretId; + const code = id ? this.ctx.app.secretStorage.getSecret(id) : undefined; + if (!code || !navigator.clipboard) return false; + await navigator.clipboard.writeText(code); + return true; + }; + + private readonly copyTopic = async (): Promise => { + const topic = this.topic; + if (!topic || !navigator.clipboard) return false; + await navigator.clipboard.writeText(topic); + return true; + }; +} diff --git a/packages/collaboration/src/mqtt.d.ts b/packages/collaboration/src/mqtt.d.ts new file mode 100644 index 00000000..3af156d2 --- /dev/null +++ b/packages/collaboration/src/mqtt.d.ts @@ -0,0 +1,6 @@ +declare module 'mqtt/dist/mqtt.esm' { + import type * as mqtt from 'mqtt'; + + const value: typeof mqtt; + export default value; +} diff --git a/packages/collaboration/src/protocol.ts b/packages/collaboration/src/protocol.ts new file mode 100644 index 00000000..556848b8 --- /dev/null +++ b/packages/collaboration/src/protocol.ts @@ -0,0 +1,205 @@ +import { + MAX_CHANGED_KEYS, + MAX_CIPHERTEXT_BYTES, + MAX_PLAINTEXT_BYTES, + PROTOCOL_VERSION, +} from '@/constants'; +import { + createChunkAAD, + decodeBase64Url, + decryptChunk, + encodeBase64Url, + encryptChunk, +} from '@/crypto'; +import type { RoomKeys } from '@/crypto'; +import type { Binary } from '@hesprs/sync-engine-sdk'; + +export { + MAX_CHANGED_KEYS, + MAX_CIPHERTEXT_BYTES, + MAX_PLAINTEXT_BYTES, + PROTOCOL_VERSION, + SEEN_TTL, +} from '@/constants'; + +const textEncoder = new TextEncoder(); +const textDecoder = new TextDecoder(); + +/** + * Transport metadata carried in MQTT 5 User Properties (plaintext, visible to the broker) instead + * of the PUBLISH payload. Every value is a string; numbers are stringified. These values are also + * bound into the AAD, so tampering with them makes the pulse fail to decrypt. + */ +export type WireMetadata = { batchId: string; clientId: string; sentAt: number; v: 1 }; + +/** The encrypted payload carries a timestamp and, when it fits, the changed file keys. */ +export type PulsePayload = { sentAt: number; keys?: Array }; + +/** Internal, fully assembled message. The wire payload only carries nonce + ciphertext. */ +export type Envelope = WireMetadata & { nonce: string; ciphertext: string }; + +/** The decrypted, validated pulse handed to the sync bridge. */ +export type Pulse = { batchId: string; clientId: string; sentAt: number; keys?: Array }; + +export type PreparedPulse = { data: Binary; sentAt: number }; + +const WIRE_PROPERTIES = { + batchId: 'batch_id', + clientId: 'client_id', + sentAt: 'sent_at', + v: 'v', +} as const; + +export type UserProperties = Record>; + +export function wireProperties(metadata: WireMetadata): UserProperties { + return { + [WIRE_PROPERTIES.v]: String(metadata.v), + [WIRE_PROPERTIES.clientId]: metadata.clientId, + [WIRE_PROPERTIES.batchId]: metadata.batchId, + [WIRE_PROPERTIES.sentAt]: String(metadata.sentAt), + }; +} + +function readProperty(properties: UserProperties, key: string): string | undefined { + const value = properties[key]; + return typeof value === 'string' ? value : undefined; +} + +function parseWireMetadata(properties: UserProperties): WireMetadata { + const v = readProperty(properties, WIRE_PROPERTIES.v); + const batchId = readProperty(properties, WIRE_PROPERTIES.batchId); + const clientId = readProperty(properties, WIRE_PROPERTIES.clientId); + const sentAt = Number(readProperty(properties, WIRE_PROPERTIES.sentAt)); + if ( + v !== String(PROTOCOL_VERSION) || + !isString(batchId) || + batchId.length > 128 || + !isString(clientId) || + clientId.length > 128 || + !isFiniteNumber(sentAt) + ) + throw new Error('Invalid collaboration envelope'); + return { batchId, clientId, sentAt, v: PROTOCOL_VERSION }; +} + +export function preparePulse({ + sentAt, + keys, +}: { + sentAt: number; + keys?: Array; +}): PreparedPulse { + let payload: PulsePayload = { sentAt }; + if (keys && keys.length > 0 && keys.length <= MAX_CHANGED_KEYS) payload = { keys, sentAt }; + let data = textEncoder.encode(JSON.stringify(payload)); + if ('keys' in payload && data.byteLength > MAX_PLAINTEXT_BYTES) { + payload = { sentAt }; + data = textEncoder.encode(JSON.stringify(payload)); + } + return { data, sentAt }; +} + +/** Seals a pulse into one envelope. A large change batch degrades to a keyless pulse. */ +export async function sealPulse({ + prepared, + keys, + topic, + clientId, +}: { + prepared: PreparedPulse; + keys: RoomKeys; + topic: string; + clientId: string; +}): Promise> { + const { data, sentAt } = prepared; + const batchId = encodeBase64Url(crypto.getRandomValues(new Uint8Array(12))); + const { ciphertext, nonce } = await encryptChunk( + keys.encryptionKey, + data, + createChunkAAD({ batchId, clientId, sentAt, topic }), + ); + return [ + { + batchId, + ciphertext: encodeBase64Url(ciphertext), + clientId, + nonce: encodeBase64Url(nonce), + sentAt, + v: PROTOCOL_VERSION, + }, + ]; +} + +export async function openPulse({ + envelope, + keys, + topic, +}: { + envelope: Envelope; + keys: RoomKeys; + topic: string; +}): Promise { + const { batchId, clientId, sentAt } = envelope; + const nonce = decodeBase64Url(envelope.nonce); + const ciphertext = decodeBase64Url(envelope.ciphertext); + if (nonce.byteLength !== 12 || ciphertext.byteLength > MAX_CIPHERTEXT_BYTES) + throw new Error('Invalid collaboration pulse'); + const data = await decryptChunk( + keys.encryptionKey, + ciphertext, + nonce, + createChunkAAD({ batchId, clientId, sentAt, topic }), + ); + const payload = parsePulsePayload(data); + if (payload.sentAt !== sentAt) throw new Error('Collaboration message metadata mismatch'); + return { batchId, clientId, sentAt, ...(payload.keys ? { keys: payload.keys } : {}) }; +} + +function parsePulsePayload(data: Binary): PulsePayload { + const parsed: unknown = JSON.parse(textDecoder.decode(data)); + if (!isRecord(parsed)) throw new Error('Invalid collaboration payload'); + const { sentAt, keys } = parsed; + if (!isFiniteNumber(sentAt)) throw new Error('Invalid collaboration payload'); + if (keys === undefined) return { sentAt }; + if ( + !Array.isArray(keys) || + keys.length > MAX_CHANGED_KEYS || + !keys.every((key) => isString(key) && key.length > 0 && key.length <= 1024) + ) + throw new Error('Invalid collaboration payload'); + return { keys: keys as Array, sentAt }; +} + +/** Reassembles the internal envelope from the wire payload and the MQTT 5 User Properties. */ +export function parseEnvelope(value: string, properties: UserProperties = {}): Envelope { + const metadata = parseWireMetadata(properties); + const parsed: unknown = JSON.parse(value); + if (!isRecord(parsed)) throw new Error('Invalid collaboration envelope'); + const { nonce, ciphertext } = parsed; + if (!isString(nonce) || !isString(ciphertext)) + throw new Error('Invalid collaboration envelope'); + if ( + decodeBase64Url(nonce).byteLength !== 12 || + decodeBase64Url(ciphertext).byteLength > MAX_CIPHERTEXT_BYTES + ) + throw new Error('Invalid collaboration envelope'); + return { ...metadata, ciphertext, nonce }; +} + +/** Only the encrypted bytes go on the wire; all framing metadata is in User Properties. */ +export function stringifyEnvelope(envelope: Envelope): string { + return JSON.stringify({ ciphertext: envelope.ciphertext, nonce: envelope.nonce }); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && Boolean(value); +} + +function isString(value: unknown): value is string { + return typeof value === 'string'; +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value); +} diff --git a/packages/collaboration/src/race.ts b/packages/collaboration/src/race.ts new file mode 100644 index 00000000..5be2bae3 --- /dev/null +++ b/packages/collaboration/src/race.ts @@ -0,0 +1,120 @@ +import { MAX_JOURNAL_ENTRIES, RACE_WINDOW } from '@/constants'; +import type { + ConflictResolver, + DeciderInput, + RecordStatsMap, + StatsMap, +} from '@hesprs/sync-engine-sdk'; + +export { MAX_JOURNAL_ENTRIES, RACE_WINDOW } from '@/constants'; + +export type UploadEpoch = { key: string; localUid: string; remoteUid: string; uploadedAt: number }; + +export class RaceJournal { + private readonly entries = new Map(); + + record(epoch: UploadEpoch): void { + this.delete(epoch.key); + this.entries.set(epoch.key, epoch); + while (this.entries.size > MAX_JOURNAL_ENTRIES) { + const oldest = this.entries.keys().next().value; + if (oldest === undefined) break; + this.delete(oldest); + } + } + + get(key: string): UploadEpoch | undefined { + return this.entries.get(key); + } + + all(): IterableIterator<[string, UploadEpoch]> { + return this.entries.entries(); + } + + prune(now = Date.now()): void { + for (const [key, epoch] of this.entries) + if (now - epoch.uploadedAt > RACE_WINDOW) this.delete(key); + } + + delete(key: string): void { + this.entries.delete(key); + } + + clear(): void { + this.entries.clear(); + } +} + +/** + * Finds uploads that a peer overwrote, using the REAL remote listing (DeciderInput.remoteStats) + * rather than a peer-supplied stat list. Only journal keys are examined, so cost is proportional to + * recent uploads, not to the size of the vault. The time window itself is enforced by + * forceRaceConflicts with Date.now(). + */ +export function findRaceCandidates({ + journal, + remoteStats, + window = RACE_WINDOW, +}: { + journal: RaceJournal; + remoteStats: StatsMap; + window?: number; +}): Map { + const candidates = new Map(); + for (const [key, epoch] of journal.all()) { + const remote = remoteStats.get(key); + if (!remote || remote.isDir) continue; + if (remote.uid === epoch.remoteUid) continue; + if (remote.mtime > 0 && Math.abs(remote.mtime - epoch.uploadedAt) > window) continue; + candidates.set(key, epoch); + } + return candidates; +} + +export function forceRaceConflicts( + input: DeciderInput, + candidates: ReadonlyMap, + now = Date.now(), +): { forcedKeys: Set; records: RecordStatsMap } { + const records = new Map(input.records); + const forcedKeys = new Set(); + for (const [key, epoch] of candidates) { + if (now - epoch.uploadedAt > RACE_WINDOW) continue; + const record = input.records.get(key); + const local = input.localStats.get(key); + const remote = input.remoteStats.get(key); + if ( + !record || + record.isDir || + !local || + local.isDir || + !remote || + remote.isDir || + record.local !== epoch.localUid || + record.remote !== epoch.remoteUid || + local.uid !== epoch.localUid || + remote.uid === epoch.remoteUid + ) + continue; + records.set(key, { + isDir: false, + local: `\0collaboration-race:${key}:${epoch.uploadedAt}`, + remote: record.remote, + }); + forcedKeys.add(key); + } + return { forcedKeys, records }; +} + +/** + * A forced conflict means this device's recent upload was overwritten by a peer. The edit is + * durably captured by the history module, so there is nothing to replay here: the forced record + * routes the key into the configured conflict resolver, which merges the divergent versions. + */ +export function createRaceAwareResolver({ + getConfiguredResolver, +}: { + getConfiguredResolver: () => ConflictResolver; +}): ConflictResolver { + return async (payload) => getConfiguredResolver()(payload); +} diff --git a/packages/collaboration/src/remote-wrapper.ts b/packages/collaboration/src/remote-wrapper.ts new file mode 100644 index 00000000..53df2adb --- /dev/null +++ b/packages/collaboration/src/remote-wrapper.ts @@ -0,0 +1,103 @@ +import type { RaceJournal } from '@/race'; +import type RemoteStateCache from '@/state-cache'; +import type { + Binary, + FileStat, + FolderStat, + Fs, + ListReporter, + WrappedFs, +} from '@hesprs/sync-engine-sdk'; + +export type CaptureOptions = { + activeUploads: ReadonlySet; + enabled: () => boolean; + journal: RaceJournal; + onUpload?: (key: string, remoteUid: string, stat: FileStat) => void; + stateCache: RemoteStateCache; +}; + +class CapturingRemoteFs implements WrappedFs { + private readonly uid: string; + + constructor( + readonly original: Fs, + private readonly options: CaptureOptions, + ) { + this.uid = original.getUid(); + } + + getUid(): string { + return this.uid; + } + + async read(key: string, stat: FileStat) { + return this.original.read(key, stat); + } + + readStream(key: string, stat: FileStat) { + return this.original.readStream(key, stat); + } + + async write(key: string, value: Binary, stat: FileStat): Promise { + const capture = this.shouldCapture(key, stat); + const uid = await this.original.write(key, value, stat); + this.options.stateCache.onWrite(this.uid, key); + if (capture) this.captureUpload(key, uid, stat); + return uid; + } + + async writeStream(key: string, value: ReadableStream, stat: FileStat): Promise { + const capture = this.shouldCapture(key, stat); + const uid = await this.original.writeStream(key, value, stat); + this.options.stateCache.onWrite(this.uid, key); + if (capture) this.captureUpload(key, uid, stat); + return uid; + } + + async delete(key: string): Promise { + await this.original.delete(key); + this.options.stateCache.onDelete(this.uid, key); + this.options.journal.delete(key); + } + + async move(oldKey: string, newKey: string): Promise { + await this.original.move(oldKey, newKey); + this.options.stateCache.onMove(this.uid, oldKey, newKey); + const epoch = this.options.journal.get(oldKey); + if (epoch) { + this.options.journal.delete(oldKey); + this.options.journal.record({ ...epoch, key: newKey }); + } + } + + async mkdir(key: string, stat: FolderStat, recursive?: boolean): Promise { + await this.original.mkdir(key, stat, recursive); + this.options.stateCache.onMkdir(this.uid, key); + } + + stat(key: string) { + return this.original.stat(key); + } + + exists(key: string) { + return this.original.exists(key); + } + + list(key: string, reporter: ListReporter) { + return this.original.list(key, reporter); + } + + private shouldCapture(key: string, stat: FileStat): boolean { + return this.options.enabled() && this.options.activeUploads.has(key) && !stat.isDir; + } + + private captureUpload(key: string, remoteUid: string, stat: FileStat): void { + this.options.journal.record({ key, localUid: stat.uid, remoteUid, uploadedAt: Date.now() }); + this.options.onUpload?.(key, remoteUid, stat); + } +} + +export default function captureRemoteState(original: Fs, options: CaptureOptions): WrappedFs { + return new CapturingRemoteFs(original, options); +} diff --git a/packages/collaboration/src/setting.ts b/packages/collaboration/src/setting.ts new file mode 100644 index 00000000..d33ceda9 --- /dev/null +++ b/packages/collaboration/src/setting.ts @@ -0,0 +1,244 @@ +import deviceType from '@/utils'; +import { settingEntry as s } from '@hesprs/sync-engine-sdk'; +import { Notice, SecretComponent } from 'obsidian'; +import type { CollaborationTranslations } from '@/i18n'; +import type { CallableOrObjectTree, MaybePromise, Translate } from '@hesprs/sync-engine-sdk'; +import type { App, SettingGroupItem, TextComponent } from 'obsidian'; + +export type CollaborationSettings = { + brokerUrl: string; + clientIdPrefix: string; + enabled: boolean; + passwordSecretId: string; + roomSecretId: string; + showMobileStatusIcon?: boolean; + username: string; +}; +export type CollaborationStatus = + | 'connected' + | 'connecting' + | 'disabled' + | 'error' + | 'missing-room' + | 'reconnecting'; + +export default function collaborationSetting( + ctx: { + app: App; + copyRoomCode: () => MaybePromise; + copyTopic: () => MaybePromise; + generateRoomCode: () => MaybePromise; + getStatus: () => { error?: Error; status: CollaborationStatus }; + getTopic: () => string | undefined; + reconnect: () => MaybePromise; + refreshSettingTab: () => void; + rerenderSettingTab: () => void; + saveSettings: () => MaybePromise; + translate: Translate; + }, + settings: CollaborationSettings, +): CallableOrObjectTree { + const { translate, saveSettings } = ctx; + const saveAndReconnect = () => { + void saveSettings(); + void ctx.reconnect(); + }; + const textField = + ( + key: 'brokerUrl' | 'clientIdPrefix' | 'username', + placeholder: string, + sanitize: (value: string) => string | undefined, + ) => + (text: TextComponent) => { + text.setValue(settings[key]) + .setPlaceholder(placeholder) + .inputEl.addEventListener('blur', () => { + const value = sanitize(text.getValue()); + if (value === undefined) { + text.inputEl.addClass('sync-engine-invalid-input'); + return; + } + text.inputEl.removeClass('sync-engine-invalid-input'); + if (settings[key] === value) return; + settings[key] = value; + text.setValue(value); + saveAndReconnect(); + }); + }; + const statusText = () => { + const { status, error } = ctx.getStatus(); + const text = + status === 'connected' + ? translate('statusConnected') + : status === 'connecting' + ? translate('statusConnecting') + : status === 'reconnecting' + ? translate('statusReconnecting') + : status === 'disabled' + ? translate('statusDisabled') + : status === 'missing-room' + ? translate('statusMissingRoomCode') + : translate('statusError'); + return error ? `${text}: ${error.message}` : text; + }; + return { + 3050: s( + (self) => ({ + heading: translate('collaboration'), + items: Object.values(self).map((node) => node(node) as SettingGroupItem), + type: 'group', + }), + { + 1000: s(() => ({ + desc: translate('enableCollaborationDescription'), + name: translate('enableCollaboration'), + render: (setting) => { + setting.addToggle((toggle) => + toggle.setValue(settings.enabled).onChange((enabled) => { + settings.enabled = enabled; + saveAndReconnect(); + }), + ); + }, + })), + 2000: s(() => ({ + desc: statusText(), + name: translate('status'), + render: (setting) => { + setting.addButton((button) => + button.setButtonText(translate('reconnect')).onClick(() => { + void Promise.resolve(ctx.reconnect()).finally(() => + ctx.rerenderSettingTab(), + ); + }), + ); + }, + })), + 2100: s(() => ({ + desc: translate('topicDescription'), + name: translate('topic'), + render: (setting) => { + setting.addButton((button) => + button + .setButtonText(translate('copyTopic')) + .setDisabled(ctx.getTopic() === undefined) + .onClick(() => { + void Promise.resolve(ctx.copyTopic()) + .then((copied) => { + if (copied) new Notice(translate('topicCopied')); + }) + .catch((error: unknown) => { + new Notice( + error instanceof Error + ? error.message + : String(error), + ); + }); + }), + ); + }, + })), + 3000: s(() => ({ + desc: translate('brokerUrlDescription'), + name: translate('brokerUrl'), + render: (setting) => { + setting.addText( + textField('brokerUrl', translate('brokerUrlPlaceholder'), (value) => { + try { + const url = new URL(value.trim()); + if (url.protocol !== 'ws:' && url.protocol !== 'wss:') return; + return url.toString(); + } catch { + // Invalid broker URLs keep the previous value. + } + }), + ); + }, + })), + 3100: s(() => ({ + desc: translate('usernameDescription'), + name: translate('username'), + render: (setting) => { + setting.addText( + textField('username', translate('usernamePlaceholder'), (value) => + value.trim(), + ), + ); + }, + })), + 3200: s(() => ({ + desc: translate('passwordDescription'), + name: translate('password'), + render: (setting) => { + setting.addComponent((element) => + new SecretComponent(ctx.app, element) + .setValue(settings.passwordSecretId) + .onChange((value) => { + settings.passwordSecretId = value ?? ''; + saveAndReconnect(); + }), + ); + }, + })), + 4000: s(() => ({ + desc: translate('clientIdPrefixDescription'), + name: translate('clientIdPrefix'), + render: (setting) => { + setting.addText( + textField('clientIdPrefix', deviceType(), (value) => { + const normalized = value.trim().replaceAll(/[^A-Za-z0-9_-]/gu, ''); + return normalized || deviceType(); + }), + ); + }, + })), + 5000: s(() => ({ + desc: translate('roomCodeDescription'), + name: translate('roomCode'), + render: (setting) => { + setting + .addComponent((element) => + new SecretComponent(ctx.app, element) + .setValue(settings.roomSecretId) + .onChange((value) => { + settings.roomSecretId = value ?? ''; + saveAndReconnect(); + }), + ) + .addButton((button) => + button.setButtonText(translate('generateRoomCode')).onClick(() => { + void Promise.resolve(ctx.generateRoomCode()) + .then((copied) => { + if (copied) new Notice(translate('roomCodeCopied')); + ctx.rerenderSettingTab(); + }) + .catch((error: unknown) => { + new Notice( + error instanceof Error + ? error.message + : String(error), + ); + }); + }), + ) + .addButton((button) => + button.setButtonText(translate('copyRoomCode')).onClick(() => { + void Promise.resolve(ctx.copyRoomCode()) + .then((copied) => { + if (copied) new Notice(translate('roomCodeCopied')); + }) + .catch((error: unknown) => { + new Notice( + error instanceof Error + ? error.message + : String(error), + ); + }); + }), + ); + }, + })), + }, + ), + }; +} diff --git a/packages/collaboration/src/state-cache.ts b/packages/collaboration/src/state-cache.ts new file mode 100644 index 00000000..17f7168e --- /dev/null +++ b/packages/collaboration/src/state-cache.ts @@ -0,0 +1,80 @@ +/** + * Tracks whether a local sync changed the remote state since the last published pulse, and which + * keys changed. Peers use the key list to patch their cached remote snapshot; a pulse that exceeds + * the size budget degrades to a keyless one and peers fall back to a full remote walk. + */ +export default class RemoteStateCache { + private readonly dirty = new Set(); + private readonly revisions = new Map(); + private readonly changed = new Map>(); + + private addChanged(uid: string, key: string): void { + if (key === '/') return; + let keys = this.changed.get(uid); + if (!keys) { + keys = new Set(); + this.changed.set(uid, keys); + } + keys.add(key); + } + + private markDirty(uid: string): void { + this.revisions.set(uid, (this.revisions.get(uid) ?? 0) + 1); + this.dirty.add(uid); + } + + clear(uid: string): void { + this.dirty.delete(uid); + this.revisions.delete(uid); + this.changed.delete(uid); + } + + clearAll(): void { + this.dirty.clear(); + this.revisions.clear(); + this.changed.clear(); + } + + onWrite(uid: string, key: string): void { + this.markDirty(uid); + this.addChanged(uid, key); + } + + onMkdir(uid: string, key: string): void { + this.markDirty(uid); + this.addChanged(uid, key); + } + + onDelete(uid: string, key: string): void { + this.markDirty(uid); + this.addChanged(uid, key); + } + + onMove(uid: string, oldKey: string, newKey: string): void { + this.markDirty(uid); + this.addChanged(uid, oldKey); + this.addChanged(uid, newKey); + } + + getRevision(uid: string): number { + return this.revisions.get(uid) ?? 0; + } + + isDirty(uid: string): boolean { + return this.dirty.has(uid); + } + + /** Returns the accumulated changed keys, or `undefined` when there are none. */ + takeChanged(uid: string): Array | undefined { + const keys = this.changed.get(uid); + if (!keys || keys.size === 0) return undefined; + return [...keys].sort(); + } + + /** Clears dirty only if no newer write arrived while the pulse was being published. */ + clearDirty(uid: string, revision?: number): void { + if (revision !== undefined && this.getRevision(uid) > revision) return; + this.dirty.delete(uid); + this.changed.delete(uid); + } +} diff --git a/packages/collaboration/src/sync-bridge.ts b/packages/collaboration/src/sync-bridge.ts new file mode 100644 index 00000000..0a3db0dd --- /dev/null +++ b/packages/collaboration/src/sync-bridge.ts @@ -0,0 +1,307 @@ +import { DEFAULT_PULSE_DEBOUNCE_MS, WRAPPER_PREFIX } from '@/constants'; +import { createRaceAwareResolver, findRaceCandidates, forceRaceConflicts } from '@/race'; +import type { CollaborationTranslations } from '@/i18n'; +import type { Pulse } from '@/protocol'; +import type { RaceJournal, UploadEpoch } from '@/race'; +import type { GlobStrategy } from '@/types'; +import type { + DeciderEntry, + Dispatch, + Events, + ExistingMemoryDB, + Fs, + On, + Stat, + StoreSync, + SyncOptions, + SyncTerminateReason, + Translate, +} from '@hesprs/sync-engine-sdk'; + +type BridgeContext = { + deciderRegistry: Map; + dispatch: Dispatch; + getConflictResolver: () => NonNullable; + memoryDB: ExistingMemoryDB; + on: On; + registerDecider: (id: string, entry: DeciderEntry) => () => boolean; + requestSync: (trigger: string) => Promise; + translate: Translate; +}; + +/** + * Builds a remote lister that returns a complete stat array from the cached remote snapshot, + * patching only the keys changed since the last full walk. Empty cache or a keyless pulse returns + * `undefined` so the sync core falls back to a full remote walk. + */ +function makeIncrementalLister( + keys: Array | undefined, + memoryDB: ExistingMemoryDB, + fastModeFallback = false, + bridge?: SyncBridge, +): NonNullable | undefined { + const store = memoryDB.getStore('remoteContext20000'); + if (store.keys().length === 0) return undefined; + if (!keys || keys.length === 0) { + if (!fastModeFallback) return undefined; + return async ({ reporter }) => { + const entries = store.values(); + const filtered: Array = []; + await Promise.all( + entries.map(async (stat, index) => { + if ( + (await reporter({ + completed: index + 1, + current: stat.key, + total: entries.length, + })) === 'exclude' + ) + return; + filtered.push(stat); + }), + ); + return filtered; + }; + } + return async ({ remoteFs, reporter }) => { + try { + await patchChangedStats(remoteFs, store, keys, bridge); + } catch { + return remoteFs.list('/', reporter); + } + const entries = store.values(); + const filtered: Array = []; + await Promise.all( + entries.map(async (stat, index) => { + if ( + (await reporter({ + completed: index + 1, + current: stat.key, + total: entries.length, + })) === 'exclude' + ) + return; + filtered.push(stat); + }), + ); + return filtered; + }; +} + +/** Re-stats changed keys against the real backend and refreshes the cached snapshot in place. */ +async function patchChangedStats( + remoteFs: Fs, + store: StoreSync, + keys: Array, + bridge?: SyncBridge, +): Promise { + await Promise.all( + keys.map(async (key) => { + if (!(await remoteFs.exists(key))) { + store.delete(key); + return; + } + const stat = await remoteFs.stat(key); + store.set(key, stat); + if (!stat.isDir && bridge) bridge.recordCommittedPeer(key, stat.uid); + }), + ); +} + +export default class SyncBridge { + private readonly cleanupCallbacks: Array<() => void> = []; + private readonly deciderCleanups: Array<() => boolean> = []; + private readonly activeUploads = new Set(); + private readonly committedPeers = new Map(); + /** + * Shared with the race-aware resolver by reference. buildOptions() runs before the wrapped + * decider, which runs before any ResolveConflict task — so the decider must MUTATE this map + * (never reassign it) for the resolver to see the candidates computed from the real listing. + */ + private readonly activeCandidates = new Map(); + private pending?: Pulse; + private syncTimer?: ReturnType; + + constructor( + private readonly ctx: BridgeContext, + private readonly options: { + fastMode?: () => boolean; + journal: RaceJournal; + rules: () => Array; + }, + ) { + this.cleanupCallbacks.push( + ctx.on('syncStarted', () => { + this.activeUploads.clear(); + this.activeCandidates.clear(); + }), + ctx.on('executionStarted', (tasks) => { + this.activeUploads.clear(); + for (const task of tasks) { + if (task.name !== 'upload') continue; + if (!task.local?.isDir) this.activeUploads.add(task.key); + } + }), + ctx.on('syncTerminated', () => { + this.activeUploads.clear(); + this.activeCandidates.clear(); + this.committedPeers.clear(); + if (this.pending) void this.ctx.requestSync('collaboration'); + }), + ); + } + + get triggerOptions(): () => SyncOptions | undefined { + return () => this.buildOptions(); + } + + accept(pulse: Pulse, debounceMs = DEFAULT_PULSE_DEBOUNCE_MS): void { + const current = this.pending; + if ( + current && + (pulse.sentAt < current.sentAt || + (pulse.sentAt === current.sentAt && pulse.batchId < current.batchId)) + ) + return; + this.pending = pulse; + this.options.journal.prune(); + if (this.syncTimer) clearTimeout(this.syncTimer); + if (debounceMs <= 0) { + void this.ctx.requestSync('collaboration'); + return; + } + this.syncTimer = setTimeout(() => { + this.syncTimer = undefined; + void this.ctx.requestSync('collaboration'); + }, debounceMs); + } + + private buildOptions(): SyncOptions | undefined { + const fastMode = Boolean(this.options.fastMode?.()); + if (!this.pending) { + const remoteLister = fastMode + ? makeIncrementalLister(undefined, this.ctx.memoryDB, true, this) + : undefined; + if (!remoteLister) return; + const rules = this.options.rules(); + const syncStrategy = rules.map((rule) => ({ + expr: rule.expr, + strategy: this.wrappedStrategy(rule.strategy), + })); + const conflictResolver = createRaceAwareResolver({ + getConfiguredResolver: this.ctx.getConflictResolver, + }); + return { conflictResolver, remoteLister, syncStrategy }; + } + const { keys } = this.pending; + this.pending = undefined; + this.activeCandidates.clear(); + const remoteLister = makeIncrementalLister(keys, this.ctx.memoryDB, fastMode, this); + const options: SyncOptions = remoteLister ? { remoteLister } : {}; + const rules = this.options.rules(); + const syncStrategy = rules.map((rule) => ({ + expr: rule.expr, + strategy: this.wrappedStrategy(rule.strategy), + })); + const conflictResolver = createRaceAwareResolver({ + getConfiguredResolver: this.ctx.getConflictResolver, + }); + return { ...options, conflictResolver, syncStrategy }; + } + + private wrappedStrategy(strategy: string): string { + if (strategy === 'none' || strategy.startsWith(WRAPPER_PREFIX)) return strategy; + const base = this.ctx.deciderRegistry.get(strategy); + if (!base) return strategy; + const id = `${WRAPPER_PREFIX}${strategy}`; + if (this.ctx.deciderRegistry.has(id)) return id; + this.deciderCleanups.push( + this.ctx.registerDecider(id, { + decider: (input) => { + // Input.remoteStats is the real remote listing, so candidates are computed + // Against current backend state rather than a peer-supplied stat list. + this.activeCandidates.clear(); + for (const [key, epoch] of findRaceCandidates({ + journal: this.options.journal, + remoteStats: input.remoteStats, + })) + this.activeCandidates.set(key, epoch); + const { forcedKeys, records } = forceRaceConflicts( + input, + this.activeCandidates, + ); + for (const [key, committedRemoteUid] of this.committedPeers) { + const remote = input.remoteStats.get(key); + const local = input.localStats.get(key); + if ( + remote && + !remote.isDir && + remote.uid === committedRemoteUid && + local && + !local.isDir + ) { + const existing = input.records.get(key); + records.set(key, { + isDir: false, + local: local.uid, + remote: existing && !existing.isDir ? existing.remote : '', + }); + } + } + if (forcedKeys.size) + this.ctx.dispatch( + 'logSync', + text(this.ctx.translate('raceDetected', forcedKeys.size)), + ); + const tasks = base.decider({ ...input, records }); + for (const [key, committedRemoteUid] of this.committedPeers) { + const remote = input.remoteStats.get(key); + const local = input.localStats.get(key); + if ( + remote && + !remote.isDir && + remote.uid === committedRemoteUid && + local && + !local.isDir + ) { + const hasTask = tasks.some((t) => t.key === key); + if (!hasTask) + tasks.push(input.taskFactory('download', { key, remote })); + } + } + return tasks; + }, + }), + ); + return id; + } + + recordCommittedPeer(key: string, remoteUid: string): void { + this.committedPeers.set(key, remoteUid); + } + + reset(): void { + if (this.syncTimer) { + clearTimeout(this.syncTimer); + this.syncTimer = undefined; + } + this.pending = undefined; + this.activeCandidates.clear(); + this.activeUploads.clear(); + this.committedPeers.clear(); + } + + dispose(): void { + this.reset(); + this.cleanupCallbacks.splice(0).forEach((cleanup) => cleanup()); + this.deciderCleanups.splice(0).forEach((cleanup) => cleanup()); + } + + get activeUploadKeys(): ReadonlySet { + return this.activeUploads; + } +} + +function text(value: unknown): string { + return typeof value === 'string' ? value : String(value); +} diff --git a/packages/collaboration/src/transport.ts b/packages/collaboration/src/transport.ts new file mode 100644 index 00000000..9667ea91 --- /dev/null +++ b/packages/collaboration/src/transport.ts @@ -0,0 +1,169 @@ +import { MQTT_CONNECT_TIMEOUT_MS, MQTT_PUBLISH_TIMEOUT_MS } from '@/constants'; +import { stringifyEnvelope, wireProperties } from '@/protocol'; +import { uint8ArrayToText } from '@repo/shared/binary'; +import mqtt from 'mqtt/dist/mqtt.esm'; +import type { Envelope, UserProperties } from '@/protocol'; +import type { IClientOptions, IClientPublishOptions, MqttClient } from 'mqtt'; + +export type TransportStatus = 'connected' | 'connecting' | 'disabled' | 'error' | 'reconnecting'; + +export type TransportOptions = { + clientId: string; + onMessage: (value: string, properties: UserProperties) => void; + onStatus: (status: TransportStatus, error?: Error) => void; + password?: string; + topic: string; + url: string; + username?: string; +}; + +/** MQTT 5 CONNECT options. Credentials are omitted entirely when unset. */ +export function connectionOptions({ + clientId, + password, + username, +}: TransportOptions): IClientOptions { + return { + clean: true, + clientId, + connectTimeout: 10_000, + forceNativeWebSocket: true, + keepalive: 30, + protocolVersion: 5, + reconnectPeriod: 5000, + resubscribe: true, + ...(username ? { username } : {}), + ...(password ? { password } : {}), + }; +} + +/** + * MQTT 5 PUBLISH options. All framing metadata rides in the variable header as User Properties, so + * the PUBLISH payload carries only the encrypted bytes. + */ +export function publishOptions(envelope: Envelope): IClientPublishOptions { + return { properties: { userProperties: wireProperties(envelope) }, qos: 1, retain: false }; +} + +export default class CollaborationTransport { + private client?: MqttClient; + private generation = 0; + + constructor(private readonly options: TransportOptions) {} + + connect(): Promise { + this.disconnect(); + const generation = ++this.generation; + const { onStatus, topic, url } = this.options; + onStatus('connecting'); + return new Promise((resolve) => { + let client: MqttClient; + try { + client = mqtt.connect(url, connectionOptions(this.options)); + } catch (error) { + onStatus('error', error instanceof Error ? error : new Error(String(error))); + resolve(); + return; + } + this.client = client; + let connected = false; + let settled = false; + const settle = () => { + if (settled) return; + settled = true; + clearTimeout(timeout); + resolve(); + }; + const timeout = setTimeout(() => { + if (generation === this.generation) + onStatus('error', new Error('MQTT connection timed out')); + settle(); + }, MQTT_CONNECT_TIMEOUT_MS); + client.on('connect', () => { + if (generation !== this.generation) return; + connected = true; + client.subscribe(topic, { qos: 1 }, (error) => { + if (generation !== this.generation) return; + if (error) { + onStatus('error', error); + settle(); + return; + } + onStatus('connected'); + settle(); + }); + }); + client.on('reconnect', () => { + if (generation === this.generation) onStatus('reconnecting'); + }); + client.on('close', () => { + if (generation !== this.generation) return; + if (!connected) { + if (settled) onStatus('reconnecting'); + else { + onStatus( + 'error', + new Error('MQTT connection closed before it was established'), + ); + settle(); + } + return; + } + onStatus('reconnecting'); + }); + client.on('error', (error) => { + if (generation === this.generation) onStatus('error', error); + settle(); + }); + client.on('message', (receivedTopic, payload, packet) => { + if (generation !== this.generation || receivedTopic !== topic) return; + try { + this.options.onMessage( + typeof payload === 'string' + ? payload + : uint8ArrayToText(new Uint8Array(payload)), + packet.properties?.userProperties ?? {}, + ); + } catch (error) { + this.options.onStatus( + 'error', + error instanceof Error ? error : new Error(String(error)), + ); + } + }); + }); + } + + async publish(envelopes: ReadonlyArray): Promise { + const client = this.client; + if (!client?.connected) throw new Error('MQTT client is not connected'); + await Promise.all( + envelopes.map( + (envelope) => + new Promise((resolve, reject) => { + const timeout = setTimeout( + () => reject(new Error('MQTT publish timed out')), + MQTT_PUBLISH_TIMEOUT_MS, + ); + client.publish( + this.options.topic, + stringifyEnvelope(envelope), + publishOptions(envelope), + (error) => { + clearTimeout(timeout); + if (error) reject(error); + else resolve(); + }, + ); + }), + ), + ); + } + + disconnect(): void { + this.generation++; + const client = this.client; + this.client = undefined; + if (client) client.end(true); + } +} diff --git a/packages/collaboration/src/types.ts b/packages/collaboration/src/types.ts new file mode 100644 index 00000000..4366f7b8 --- /dev/null +++ b/packages/collaboration/src/types.ts @@ -0,0 +1,48 @@ +import type { CollaborationTranslations } from '@/i18n'; +import type { + ConflictResolver, + DeciderEntry, + Dispatch, + Events, + ExistingMemoryDB, + FsWrapperEntry, + ModuleStatus, + ObsidianLanguageCode, + On, + SelectFromContext, + SettingEntry, + SyncTerminateReason, + Translate, + TranslationResource, + TriggerEntry, +} from '@hesprs/sync-engine-sdk'; +import type { App } from 'obsidian'; + +export type GlobStrategy = { expr: string; strategy: string }; +export type TogglableValue = { enabled: boolean; value: T }; + +export type CollaborationContext = SelectFromContext<{ + app: App; + deciderRegistry: Map; + dispatch: Dispatch; + getConflictResolver: () => ConflictResolver; + memoryDB: ExistingMemoryDB; + on: On; + refreshSettingTab: () => void; + registerDecider: (id: string, entry: DeciderEntry) => () => boolean; + registerI18n: (locale: ObsidianLanguageCode, resource: TranslationResource) => void; + registerRemoteFsWrapper: (entry: FsWrapperEntry) => () => boolean; + registerSetting: (entry: SettingEntry) => () => boolean; + registerTrigger: (key: string, entry: TriggerEntry) => () => boolean; + requestSync: (trigger: string) => Promise; + rerenderSettingTab: () => void; + saveSettings: () => Promise; + setModuleStatus: (id: string, status?: ModuleStatus) => () => void; + settings: { + confirmDeleteInAutoSync?: boolean; + realtimeSync?: TogglableValue; + realtimeSyncFastMode?: boolean; + syncStrategy: Array; + }; + translate: Translate; +}>; diff --git a/packages/collaboration/src/utils.ts b/packages/collaboration/src/utils.ts new file mode 100644 index 00000000..01e275e0 --- /dev/null +++ b/packages/collaboration/src/utils.ts @@ -0,0 +1,20 @@ +import deviceType from '@repo/shared/device-type'; + +export { default } from '@repo/shared/device-type'; +export { toError } from '@repo/shared/error'; +export { readSecret } from '@repo/shared/secret'; + +export function text(value: unknown): string { + return typeof value === 'string' ? value : String(value); +} + +/** Full MQTT client ID: the configured (or detected) prefix plus a per-session random suffix. */ +export function buildClientId(prefix: string, instanceId: string): string { + const normalized = prefix.replaceAll(/[^A-Za-z0-9_-]/gu, '').slice(0, 64) || deviceType(); + return `${normalized}-${instanceId}`; +} + +export function cleanupMobileHeaderIcons(): void { + if (typeof document === 'undefined') return; + document.querySelectorAll('.sync-engine-collab-action').forEach((el) => el.remove()); +} diff --git a/packages/collaboration/test/broker.integration.test.ts b/packages/collaboration/test/broker.integration.test.ts new file mode 100644 index 00000000..173d0f09 --- /dev/null +++ b/packages/collaboration/test/broker.integration.test.ts @@ -0,0 +1,73 @@ +import { deriveRoomKeys, deriveTopic, generateRoomCode, parseRoomCode } from '@/crypto'; +import { openPulse, parseEnvelope, preparePulse, sealPulse } from '@/protocol'; +import CollaborationTransport from '@/transport'; +import { expect, test } from 'bun:test'; +import type { UserProperties } from '@/protocol'; + +const url = process.env.MQTT_TEST_URL; +const prefix = process.env.MQTT_TEST_CLIENT_ID ?? 'sync-engine-test'; + +function timeout(promise: Promise, milliseconds: number): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('MQTT test timed out')), milliseconds); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error: unknown) => { + clearTimeout(timer); + reject(error instanceof Error ? error : new Error(String(error))); + }, + ); + }); +} + +test.skipIf(!url)( + 'publishes and receives an encrypted pulse', + async () => { + if (!url) return; + const keys = await deriveRoomKeys(parseRoomCode(generateRoomCode())); + const topic = await deriveTopic(keys); + const suffix = crypto.randomUUID().slice(0, 8); + let receiver: CollaborationTransport | undefined; + let sender: CollaborationTransport | undefined; + try { + type Received = { raw: string; properties: UserProperties }; + const received = new Promise((resolve) => { + receiver = new CollaborationTransport({ + clientId: `${prefix}-${suffix}-b`, + onMessage: (raw, properties) => resolve({ properties, raw }), + onStatus: () => {}, + topic, + url, + }); + }); + const senderClientId = `${prefix}-${suffix}-a`; + sender = new CollaborationTransport({ + clientId: senderClientId, + onMessage: () => {}, + onStatus: () => {}, + topic, + url, + }); + await Promise.all([sender.connect(), receiver?.connect()]); + const sentAt = Date.now(); + const prepared = preparePulse({ sentAt }); + const envelopes = await sealPulse({ clientId: senderClientId, keys, prepared, topic }); + await sender.publish(envelopes); + const { raw, properties } = await timeout(received, 15_000); + const pulse = await openPulse({ + envelope: parseEnvelope(raw, properties), + keys, + topic, + }); + expect(pulse).toMatchObject({ clientId: senderClientId, sentAt }); + expect(pulse).not.toHaveProperty('stats'); + } finally { + sender?.disconnect(); + receiver?.disconnect(); + } + }, + 20_000, +); diff --git a/packages/collaboration/test/crypto.test.ts b/packages/collaboration/test/crypto.test.ts new file mode 100644 index 00000000..e6bfeda8 --- /dev/null +++ b/packages/collaboration/test/crypto.test.ts @@ -0,0 +1,83 @@ +import { + createChunkAAD, + decodeBase64Url, + deriveRoomKeys, + deriveTopic, + encodeBase64Url, + encryptChunk, + decryptChunk, + generateRoomCode, + isFresh, + parseRoomCode, +} from '@/crypto'; +import { beforeAll, expect, test } from 'bun:test'; +import type { RoomKeys } from '@/crypto'; + +const roomCode = generateRoomCode(); +let keys: RoomKeys; +let topic: string; + +beforeAll(async () => { + keys = await deriveRoomKeys(parseRoomCode(roomCode)); + topic = await deriveTopic(keys); +}); + +test('generates and parses room codes', () => { + const first = generateRoomCode(); + const second = generateRoomCode(); + expect(first).not.toBe(second); + expect(parseRoomCode(first).roomId.byteLength).toBe(16); + expect(parseRoomCode(first).secret.byteLength).toBe(32); +}); + +test('rejects malformed room codes', () => { + expect(() => parseRoomCode('nope')).toThrow(); + expect(() => parseRoomCode('sec1.abc.def')).toThrow(); + expect(() => parseRoomCode('sec2.abc.def')).toThrow(); +}); + +test('round trips base64url values', () => { + const value = crypto.getRandomValues(new Uint8Array(257)); + expect(decodeBase64Url(encodeBase64Url(value))).toStrictEqual(value); +}); + +test('derives a stable topic and encrypts chunks', async () => { + expect(await deriveTopic(keys)).toBe(topic); + const aad = createChunkAAD({ batchId: 'batch', clientId: 'sender', sentAt: Date.now(), topic }); + const { ciphertext, nonce } = await encryptChunk( + keys.encryptionKey, + new TextEncoder().encode('secret payload'), + aad, + ); + expect( + new TextDecoder().decode(await decryptChunk(keys.encryptionKey, ciphertext, nonce, aad)), + ).toBe('secret payload'); +}); + +test('rejects a different key and tampered ciphertext', async () => { + const other = await deriveRoomKeys(parseRoomCode(generateRoomCode())); + const aad = createChunkAAD({ batchId: 'batch', clientId: 'sender', sentAt: Date.now(), topic }); + const { ciphertext, nonce } = await encryptChunk( + keys.encryptionKey, + new TextEncoder().encode('secret payload'), + aad, + ); + const tampered = new Uint8Array(ciphertext); + tampered[0] ^= 1; + const wrongKey = await decryptChunk(other.encryptionKey, ciphertext, nonce, aad).catch( + (error: unknown) => error, + ); + const tamperedResult = await decryptChunk(keys.encryptionKey, tampered, nonce, aad).catch( + (error: unknown) => error, + ); + expect(wrongKey).toBeInstanceOf(Error); + expect(tamperedResult).toBeInstanceOf(Error); +}); + +test('rejects stale and future timestamps', () => { + const now = Date.now(); + expect(isFresh(now)).toBe(true); + expect(isFresh(now - 60_000)).toBe(true); + expect(isFresh(now - 121_000)).toBe(false); + expect(isFresh(now + 121_000)).toBe(false); +}); diff --git a/packages/collaboration/test/mocks.ts b/packages/collaboration/test/mocks.ts new file mode 100644 index 00000000..9a27a793 --- /dev/null +++ b/packages/collaboration/test/mocks.ts @@ -0,0 +1,4 @@ +import ObsidianMock from '@repo/shared/obsidian-mock'; +import { mock } from 'bun:test'; + +void mock.module('obsidian', () => ObsidianMock); diff --git a/packages/collaboration/test/protocol.test.ts b/packages/collaboration/test/protocol.test.ts new file mode 100644 index 00000000..93088964 --- /dev/null +++ b/packages/collaboration/test/protocol.test.ts @@ -0,0 +1,112 @@ +import { deriveRoomKeys, deriveTopic, generateRoomCode, parseRoomCode } from '@/crypto'; +import { + MAX_CHANGED_KEYS, + openPulse, + parseEnvelope, + preparePulse, + sealPulse, + stringifyEnvelope, + wireProperties, +} from '@/protocol'; +import { beforeAll, expect, test } from 'bun:test'; +import type { RoomKeys } from '@/crypto'; + +const clientId = 'windows-abc123'; +let keys: RoomKeys; +let topic: string; + +beforeAll(async () => { + keys = await deriveRoomKeys(parseRoomCode(generateRoomCode())); + topic = await deriveTopic(keys); +}); + +async function sealed(sentAt = Date.now(), changedKeys?: Array) { + const prepared = preparePulse({ keys: changedKeys, sentAt }); + const envelopes = await sealPulse({ clientId, keys, prepared, topic }); + return { envelopes, sentAt }; +} + +test('round trips a keyless pulse', async () => { + const { envelopes, sentAt } = await sealed(); + expect(envelopes).toHaveLength(1); + const pulse = await openPulse({ envelope: envelopes[0], keys, topic }); + expect(pulse).toStrictEqual({ batchId: envelopes[0].batchId, clientId, sentAt }); +}); + +test('round trips a pulse carrying changed keys', async () => { + const changedKeys = ['a.md', 'folder/b.md', 'c.txt']; + const { envelopes, sentAt } = await sealed(Date.now(), changedKeys); + const pulse = await openPulse({ envelope: envelopes[0], keys, topic }); + expect(pulse).toStrictEqual({ + batchId: envelopes[0].batchId, + clientId, + keys: changedKeys, + sentAt, + }); +}); + +test('a large batch of keys degrades to a keyless pulse', async () => { + const changedKeys = Array.from( + { length: 100 }, + (_, index) => `folder-${index}/${'x'.repeat(60)}.md`, + ); + const { envelopes, sentAt } = await sealed(Date.now(), changedKeys); + const pulse = await openPulse({ envelope: envelopes[0], keys, topic }); + expect(pulse).toStrictEqual({ batchId: envelopes[0].batchId, clientId, sentAt }); + expect(pulse).not.toHaveProperty('keys'); +}); + +test('more than the maximum number of keys degrades to a keyless pulse', async () => { + const changedKeys = Array.from({ length: MAX_CHANGED_KEYS + 1 }, (_, index) => `${index}.md`); + const { envelopes, sentAt } = await sealed(Date.now(), changedKeys); + const pulse = await openPulse({ envelope: envelopes[0], keys, topic }); + expect(pulse).toStrictEqual({ batchId: envelopes[0].batchId, clientId, sentAt }); +}); + +test('the wire payload carries only encrypted bytes', async () => { + const { envelopes } = await sealed(Date.now(), ['note.md']); + const parsed: unknown = JSON.parse(stringifyEnvelope(envelopes[0])); + if (typeof parsed !== 'object' || parsed === null) throw new Error('Not an object'); + expect(Object.keys(parsed).sort()).toStrictEqual(['ciphertext', 'nonce']); +}); + +test('User Properties carry only framing metadata', async () => { + const { envelopes } = await sealed(); + const properties = wireProperties(envelopes[0]); + expect(Object.keys(properties).sort()).toStrictEqual(['batch_id', 'client_id', 'sent_at', 'v']); + expect(properties.client_id).toBe(clientId); + expect(properties.batch_id).toBe(envelopes[0].batchId); + expect(properties.sent_at).toBe(String(envelopes[0].sentAt)); + expect(properties).not.toHaveProperty('index'); + expect(properties).not.toHaveProperty('count'); + expect(properties).not.toHaveProperty('stats'); +}); + +test('parseEnvelope reassembles metadata from User Properties', async () => { + const { envelopes } = await sealed(); + const parsed = parseEnvelope(stringifyEnvelope(envelopes[0]), wireProperties(envelopes[0])); + expect(parsed).toStrictEqual(envelopes[0]); +}); + +test('rejects a tampered client_id and a wrong room', async () => { + const { envelopes } = await sealed(); + const other = await deriveRoomKeys(parseRoomCode(generateRoomCode())); + const wrongRoom = await openPulse({ envelope: envelopes[0], keys: other, topic }).catch( + (error: unknown) => error, + ); + expect(wrongRoom).toBeInstanceOf(Error); + + const tampered = { ...envelopes[0], clientId: 'other' }; + const tamperedResult = await openPulse({ envelope: tampered, keys, topic }).catch( + (error: unknown) => error, + ); + expect(tamperedResult).toBeInstanceOf(Error); +}); + +test('rejects malformed envelopes', () => { + expect(() => parseEnvelope('{}', {})).toThrow(); + expect(() => parseEnvelope('{"nonce":"***","ciphertext":"***"}', {})).toThrow(); + expect(() => + parseEnvelope('{}', wireProperties({ batchId: 'x', clientId: 'x', sentAt: 1, v: 1 })), + ).toThrow(); +}); diff --git a/packages/collaboration/test/race.test.ts b/packages/collaboration/test/race.test.ts new file mode 100644 index 00000000..374ba4ef --- /dev/null +++ b/packages/collaboration/test/race.test.ts @@ -0,0 +1,107 @@ +import { + createRaceAwareResolver, + findRaceCandidates, + forceRaceConflicts, + RaceJournal, +} from '@/race'; +import { testKit } from '@hesprs/sync-engine-sdk/dev'; +import { expect, test } from 'bun:test'; +import type { UploadEpoch } from '@/race'; +import type { + ConflictResolver, + DeciderInput, + RecordStatsMap, + StatsMap, +} from '@hesprs/sync-engine-sdk'; + +const { file } = testKit; + +function epoch(overrides: Partial = {}): UploadEpoch { + return { + key: 'note.md', + localUid: 'local-u', + remoteUid: 'remote-old', + uploadedAt: Date.now(), + ...overrides, + }; +} + +test('finds only real-listing overwrites with a different remote uid', () => { + const journal = new RaceJournal(); + const item = epoch(); + journal.record(item); + const now = item.uploadedAt + 1000; + const remoteStats = (uid: string, mtime = now): StatsMap => + new Map([['note.md', file('note.md', { mtime, size: 20, uid })]]); + + expect(findRaceCandidates({ journal, remoteStats: remoteStats('remote-new') }).size).toBe(1); + // The peer still advertises the uid we uploaded: no race. + expect(findRaceCandidates({ journal, remoteStats: remoteStats('remote-old') }).size).toBe(0); + // A stale listing outside the mtime window is not a race. + expect( + findRaceCandidates({ journal, remoteStats: remoteStats('remote-new', now + 60_000) }).size, + ).toBe(0); + // Only journal keys are examined, no matter how large the vault is. + const large: StatsMap = new Map(remoteStats('remote-new')); + for (let index = 0; index < 100; index++) { + const key = `other-${index}.md`; + large.set(key, file(key, { size: 1, uid: `u${index}` })); + } + expect(findRaceCandidates({ journal, remoteStats: large }).size).toBe(1); +}); + +test('forces a conflict only when record and local stat still match the upload', () => { + const item = epoch(); + const localStats: StatsMap = new Map([ + ['note.md', file('note.md', { size: 14, uid: 'local-u' })], + ]); + const remoteStats: StatsMap = new Map([ + ['note.md', file('note.md', { size: 14, uid: 'remote-new' })], + ]); + const records: RecordStatsMap = new Map([ + ['note.md', { isDir: false, local: 'local-u', remote: 'remote-old' }], + ]); + const input = { + localStats, + records, + remoteStats, + taskFactory: (() => {}) as never, + } satisfies DeciderInput; + const result = forceRaceConflicts(input, new Map([['note.md', item]])); + + expect(result.forcedKeys.has('note.md')).toBe(true); + expect(result.records.get('note.md')?.isDir).toBe(false); + expect(records.get('note.md')).toStrictEqual({ + isDir: false, + local: 'local-u', + remote: 'remote-old', + }); + + const changed = forceRaceConflicts( + { + ...input, + records: new Map([['note.md', { isDir: false, local: 'other', remote: 'remote-old' }]]), + }, + new Map([['note.md', item]]), + ); + expect(changed.forcedKeys.size).toBe(0); +}); + +test('createRaceAwareResolver delegates every conflict to the configured resolver', async () => { + let called = 0; + const configured: ConflictResolver = (payload) => { + called++; + expect(payload.key).toBe('note.md'); + return Promise.resolve(); + }; + const resolver = createRaceAwareResolver({ getConfiguredResolver: () => configured }); + await resolver({ + key: 'note.md', + local: file('note.md', { size: 11, uid: 'local-u' }), + localFs: {} as never, + record: {} as never, + remote: file('note.md', { size: 7, uid: 'remote-new' }), + remoteFs: {} as never, + }); + expect(called).toBe(1); +}); diff --git a/packages/collaboration/test/remote-wrapper.test.ts b/packages/collaboration/test/remote-wrapper.test.ts new file mode 100644 index 00000000..b8d48e33 --- /dev/null +++ b/packages/collaboration/test/remote-wrapper.test.ts @@ -0,0 +1,69 @@ +import { RaceJournal } from '@/race'; +import captureRemoteState from '@/remote-wrapper'; +import RemoteStateCache from '@/state-cache'; +import { testKit } from '@hesprs/sync-engine-sdk/dev'; +import { expect, test } from 'bun:test'; + +const { bytes, file, fs } = testKit; + +function setup() { + const remote = fs({ control: { read: () => bytes('base'), write: () => 'uploaded-uid' } }); + const stateCache = new RemoteStateCache(); + const journal = new RaceJournal(); + const activeUploads = new Set(['note.md']); + const wrapped = captureRemoteState(remote.fs, { + activeUploads, + enabled: () => true, + journal, + stateCache, + }); + return { activeUploads, journal, remote, stateCache, wrapped }; +} + +test('journals the upload epoch', async () => { + const { journal, remote, wrapped } = setup(); + + await wrapped.write('note.md', bytes('local'), file('note.md', { size: 5, uid: 'local-uid' })); + + // No base read is performed; the edit is durably captured by the history module instead. + expect(remote.calls.read).toHaveLength(0); + expect(journal.get('note.md')).toMatchObject({ + key: 'note.md', + localUid: 'local-uid', + remoteUid: 'uploaded-uid', + }); +}); + +test('does not journal writes outside an upload task', async () => { + const { activeUploads, journal, wrapped } = setup(); + activeUploads.clear(); + + await wrapped.write('note.md', bytes('local'), file('note.md', { size: 5, uid: 'local-uid' })); + + expect(journal.get('note.md')).toBeUndefined(); +}); + +test('marks the local state dirty after writes and moves', async () => { + const { stateCache, wrapped } = setup(); + expect(stateCache.isDirty('uid')).toBe(false); + + await wrapped.write('other.md', bytes('x'), file('other.md', { size: 1, uid: 'local' })); + expect(stateCache.isDirty('uid')).toBe(true); + stateCache.clearDirty('uid'); + + await wrapped.move('other.md', 'moved.md'); + expect(stateCache.isDirty('uid')).toBe(true); +}); + +test('collects changed keys on every mutating remote operation', async () => { + const { stateCache, wrapped } = setup(); + const stat = file('note.md', { size: 1, uid: 'local' }); + + await wrapped.write('a.md', bytes('x'), stat); + await wrapped.writeStream('b.md', new ReadableStream(), stat); + await wrapped.mkdir('folder/', { isDir: true, key: 'folder/', meta: () => ({}) }); + await wrapped.move('a.md', 'c.md'); + await wrapped.delete('b.md'); + + expect(stateCache.takeChanged('uid')).toStrictEqual(['a.md', 'b.md', 'c.md', 'folder/']); +}); diff --git a/packages/collaboration/test/setting.test.ts b/packages/collaboration/test/setting.test.ts new file mode 100644 index 00000000..b173c8fd --- /dev/null +++ b/packages/collaboration/test/setting.test.ts @@ -0,0 +1,73 @@ +import collaborationSetting from '@/setting'; +import { describe, expect, mock, test } from 'bun:test'; +import type { CollaborationSettings } from '@/setting'; +import type { App } from 'obsidian'; + +describe('CollaborationSettings', () => { + const createMockCtx = (secretValue?: string) => { + const secrets = new Map(); + if (secretValue) secrets.set('sec-123', secretValue); + + const mockApp = { + secretStorage: { getSecret: (id: string) => secrets.get(id) }, + } as unknown as App; + + return { + app: mockApp, + copyRoomCode: mock(() => Promise.resolve(true)), + copyTopic: mock(() => Promise.resolve(true)), + generateRoomCode: mock(() => Promise.resolve(true)), + getStatus: () => ({ status: 'disabled' as const }), + getTopic: () => 'test-topic', + reconnect: mock(() => Promise.resolve()), + refreshSettingTab: mock(() => {}), + rerenderSettingTab: mock(() => {}), + saveSettings: mock(() => Promise.resolve()), + translate: ((key: string) => key) as never, + }; + }; + + test('renders collaboration enable toggle and toggles setting', () => { + const ctx = createMockCtx(); + const settings: CollaborationSettings = { + brokerUrl: 'wss://example.com', + clientIdPrefix: 'desktop', + enabled: false, + passwordSecretId: '', + roomSecretId: '', + username: '', + }; + + const tree = collaborationSetting(ctx, settings) as unknown as Record< + number, + Record< + number, + (self: unknown) => { + render: (setting: { + addToggle: (cb: (toggle: unknown) => void) => void; + }) => void; + } + > + >; + + let onChangeCb: ((enabled: boolean) => void) | undefined; + const mockToggle = { + onChange: (cb: (enabled: boolean) => void) => { + onChangeCb = cb; + return mockToggle; + }, + setValue: () => mockToggle, + }; + + const node1000 = tree[3050][1000]; + expect(node1000).toBeDefined(); + + const def = node1000(node1000); + def.render({ addToggle: (cb) => cb(mockToggle) }); + + expect(onChangeCb).toBeDefined(); + onChangeCb?.(true); + expect(settings.enabled).toBe(true); + expect(ctx.reconnect).toHaveBeenCalled(); + }); +}); diff --git a/packages/collaboration/test/state-cache.test.ts b/packages/collaboration/test/state-cache.test.ts new file mode 100644 index 00000000..fffbce29 --- /dev/null +++ b/packages/collaboration/test/state-cache.test.ts @@ -0,0 +1,76 @@ +import RemoteStateCache from '@/state-cache'; +import { expect, test } from 'bun:test'; + +test('marks dirty on every mutating remote operation', () => { + const cache = new RemoteStateCache(); + expect(cache.isDirty('uid')).toBe(false); + cache.onWrite('uid', 'note.md'); + expect(cache.isDirty('uid')).toBe(true); + cache.clearDirty('uid'); + cache.onMkdir('uid', 'folder/'); + expect(cache.isDirty('uid')).toBe(true); + cache.clearDirty('uid'); + cache.onDelete('uid', 'note.md'); + expect(cache.isDirty('uid')).toBe(true); + cache.clearDirty('uid'); + cache.onMove('uid', 'a.md', 'b.md'); + expect(cache.isDirty('uid')).toBe(true); +}); + +test('collects changed keys and returns them sorted', () => { + const cache = new RemoteStateCache(); + expect(cache.takeChanged('uid')).toBeUndefined(); + cache.onWrite('uid', 'z.md'); + cache.onWrite('uid', 'a.md'); + cache.onWrite('uid', 'z.md'); + cache.onMkdir('uid', 'dir/'); + expect(cache.takeChanged('uid')).toStrictEqual(['a.md', 'dir/', 'z.md']); +}); + +test('a move records both the old and new key', () => { + const cache = new RemoteStateCache(); + cache.onMove('uid', 'old.md', 'new.md'); + expect(cache.takeChanged('uid')).toStrictEqual(['new.md', 'old.md']); +}); + +test('the root key is never collected', () => { + const cache = new RemoteStateCache(); + cache.onMkdir('uid', '/'); + expect(cache.takeChanged('uid')).toBeUndefined(); +}); + +test('clearing dirty also clears collected keys', () => { + const cache = new RemoteStateCache(); + cache.onWrite('uid', 'note.md'); + cache.clearDirty('uid'); + expect(cache.takeChanged('uid')).toBeUndefined(); +}); + +test('revision guard keeps a write made during publish dirty, with its keys', () => { + const cache = new RemoteStateCache(); + cache.onWrite('uid', 'first.md'); + const revision = cache.getRevision('uid'); + // A newer write arrives while the pulse is being published. + cache.onWrite('uid', 'second.md'); + cache.clearDirty('uid', revision); + expect(cache.isDirty('uid')).toBe(true); + expect(cache.takeChanged('uid')).toStrictEqual(['first.md', 'second.md']); + cache.clearDirty('uid'); + expect(cache.isDirty('uid')).toBe(false); + expect(cache.takeChanged('uid')).toBeUndefined(); +}); + +test('clear and clearAll reset the dirty state and collected keys', () => { + const cache = new RemoteStateCache(); + cache.onWrite('one', 'a.md'); + cache.onWrite('two', 'b.md'); + cache.clear('one'); + expect(cache.isDirty('one')).toBe(false); + expect(cache.takeChanged('one')).toBeUndefined(); + expect(cache.isDirty('two')).toBe(true); + expect(cache.takeChanged('two')).toStrictEqual(['b.md']); + cache.clearAll(); + expect(cache.isDirty('two')).toBe(false); + expect(cache.takeChanged('two')).toBeUndefined(); + expect(cache.getRevision('two')).toBe(0); +}); diff --git a/packages/collaboration/test/sync-bridge.test.ts b/packages/collaboration/test/sync-bridge.test.ts new file mode 100644 index 00000000..8d831a3b --- /dev/null +++ b/packages/collaboration/test/sync-bridge.test.ts @@ -0,0 +1,353 @@ +import { RaceJournal } from '@/race'; +import SyncBridge from '@/sync-bridge'; +import { testKit } from '@hesprs/sync-engine-sdk/dev'; +import { expect, test } from 'bun:test'; +import { openMemoryDB } from 'uni-kv'; +import type { Pulse } from '@/protocol'; +import type { GlobStrategy } from '@/types'; +import type { + DatabaseSync, + DeciderEntry, + Dispatch, + Events, + Fs, + On, + Stat, +} from '@hesprs/sync-engine-sdk'; + +const { file } = testKit; +const rules: Array = [{ expr: '*', strategy: 'bidirectional' }]; + +type TestDB = DatabaseSync<{ remoteContext20000: Stat }, Record>; + +function pulse(overrides: Partial = {}): Pulse { + return { batchId: 'batch', clientId: 'peer', sentAt: Date.now(), ...overrides }; +} + +function setup() { + const listeners = new Map void>>(); + const on = ((event: keyof Events, listener: (payload: never) => void) => { + let set = listeners.get(event); + if (!set) { + set = new Set(); + listeners.set(event, set); + } + set.add(listener); + return () => set.delete(listener); + }) as On; + const dispatched: Array<{ event: string; payload: unknown }> = []; + const dispatch = ((event: string, payload?: unknown) => { + dispatched.push({ event, payload }); + }) as Dispatch; + const registry = new Map([ + [ + 'bidirectional', + { + decider: (input) => { + const local = input.localStats.get('note.md'); + const remote = input.remoteStats.get('note.md'); + if (!local || !remote || local.isDir || remote.isDir) return []; + return [ + input.taskFactory('resolveConflict', { key: 'note.md', local, remote }), + ]; + }, + }, + ], + ]); + const journal = new RaceJournal(); + const memoryDB = openMemoryDB(`collab-bridge-${crypto.randomUUID()}`) as TestDB; + const bridge = new SyncBridge( + { + deciderRegistry: registry, + dispatch, + getConflictResolver: () => () => {}, + memoryDB, + on, + registerDecider: (id, entry) => { + registry.set(id, entry); + return () => registry.delete(id); + }, + requestSync: () => Promise.resolve({ result: 'noop' }), + translate: ((key: string) => key) as never, + }, + { journal, rules: () => rules }, + ); + return { bridge, dispatched, journal, memoryDB, registry }; +} + +/** Stands in for the wrapped remote Fs: exists()/stat() read the current backend state. */ +function fakeRemoteFs(current: Map): Fs { + return { + exists: (key: string) => Promise.resolve(current.has(key)), + stat: (key: string) => { + const stat = current.get(key); + if (!stat) return Promise.reject(new Error(`Missing ${key}`)); + return Promise.resolve(stat); + }, + } as Fs; +} + +test('a keyless pulse falls back to the full remote walk', () => { + const { bridge } = setup(); + bridge.accept(pulse()); + const options = bridge.triggerOptions(); + expect(options?.remoteLister).toBeUndefined(); + expect(options?.syncStrategy).toStrictEqual([ + { expr: '*', strategy: 'collaboration:bidirectional' }, + ]); + expect(options?.conflictResolver).toBeDefined(); + bridge.dispose(); +}); + +test('a keyless pulse with fastMode enabled uses the cached snapshot instead of a full walk', () => { + const { memoryDB, registry } = setup(); + memoryDB.getStore('remoteContext20000').set('cached.md', file('cached.md')); + const bridge = new SyncBridge( + { + deciderRegistry: registry, + dispatch: () => {}, + getConflictResolver: () => () => {}, + memoryDB, + on: () => () => {}, + registerDecider: () => () => true, + requestSync: () => Promise.resolve({ result: 'noop' }), + translate: ((key: string) => key) as never, + }, + { + fastMode: () => true, + journal: new RaceJournal(), + rules: () => [{ expr: '*', strategy: 'bidirectional' }], + }, + ); + bridge.accept(pulse()); + const options = bridge.triggerOptions(); + expect(options?.remoteLister).toBeDefined(); + bridge.dispose(); +}); + +test('changed keys with an empty snapshot fall back to the full remote walk', () => { + const { bridge } = setup(); + bridge.accept(pulse({ keys: ['note.md'] })); + const options = bridge.triggerOptions(); + expect(options?.remoteLister).toBeUndefined(); + bridge.dispose(); +}); + +test('changed keys with a populated snapshot produce an incremental lister', () => { + const { bridge, memoryDB } = setup(); + memoryDB.getStore('remoteContext20000').set('note.md', file('note.md')); + bridge.accept(pulse({ keys: ['note.md'] })); + const options = bridge.triggerOptions(); + expect(options?.remoteLister).toBeDefined(); + bridge.dispose(); +}); + +test('the incremental lister returns a complete stat array', async () => { + const { bridge, memoryDB } = setup(); + const store = memoryDB.getStore('remoteContext20000'); + store.set('unchanged.md', file('unchanged.md', { uid: 'u1' })); + store.set('changed.md', file('changed.md', { uid: 'old' })); + store.set('deleted.md', file('deleted.md', { uid: 'd1' })); + const current = new Map([ + ['unchanged.md', file('unchanged.md', { uid: 'u1' })], + ['changed.md', file('changed.md', { uid: 'new' })], + ['new.md', file('new.md', { uid: 'n1' })], + ]); + const remoteFs = fakeRemoteFs(current); + bridge.accept(pulse({ keys: ['changed.md', 'deleted.md', 'new.md'] })); + const lister = bridge.triggerOptions()?.remoteLister; + expect(lister).toBeDefined(); + const stats = await lister?.({ + localFs: {} as Fs, + record: {} as never, + remoteFs, + reporter: () => 'include', + }); + const byKey = new Map(stats?.map((stat) => [stat.key, stat])); + // Unchanged keys survive, changed keys are refreshed, new keys appear, deletes are removed. + expect(byKey.has('unchanged.md')).toBe(true); + expect(byKey.get('changed.md')).toMatchObject({ uid: 'new' }); + expect(byKey.has('new.md')).toBe(true); + expect(byKey.has('deleted.md')).toBe(false); + bridge.dispose(); +}); + +test('the incremental lister honors reporter exclusions', async () => { + const { bridge, memoryDB } = setup(); + const store = memoryDB.getStore('remoteContext20000'); + store.set('keep.md', file('keep.md')); + store.set('drop.md', file('drop.md')); + const remoteFs = fakeRemoteFs(new Map([['keep.md', file('keep.md')]])); + bridge.accept(pulse({ keys: ['keep.md'] })); + const lister = bridge.triggerOptions()?.remoteLister; + const stats = await lister?.({ + localFs: {} as Fs, + record: {} as never, + remoteFs, + reporter: ({ current }) => (current === 'drop.md' ? 'exclude' : 'include'), + }); + expect(stats?.map((stat) => stat.key)).toStrictEqual(['keep.md']); + bridge.dispose(); +}); + +test('the incremental lister falls back to remoteFs.list on patch error', async () => { + const { bridge, memoryDB } = setup(); + const store = memoryDB.getStore('remoteContext20000'); + store.set('note.md', file('note.md')); + const remoteFs = { + exists: () => Promise.reject(new Error('backend network failure')), + list: () => Promise.resolve([file('fallback.md')]), + } as unknown as Fs; + bridge.accept(pulse({ keys: ['note.md'] })); + const lister = bridge.triggerOptions()?.remoteLister; + expect(lister).toBeDefined(); + const stats = await lister?.({ + localFs: {} as Fs, + record: {} as never, + remoteFs, + reporter: () => 'include', + }); + expect(stats?.map((s) => s.key)).toStrictEqual(['fallback.md']); + bridge.dispose(); +}); + +test('race forcing is unconditional: the incremental lister and sync strategy are both present', () => { + const { bridge, memoryDB } = setup(); + memoryDB.getStore('remoteContext20000').set('note.md', file('note.md')); + const fresh = new SyncBridge( + { + deciderRegistry: new Map(), + dispatch: () => {}, + getConflictResolver: () => () => {}, + memoryDB, + on: () => () => false, + registerDecider: () => () => false, + requestSync: () => Promise.resolve({ result: 'noop' }), + translate: ((key: string) => key) as never, + }, + { journal: new RaceJournal(), rules: () => rules }, + ); + fresh.accept(pulse({ keys: ['note.md'] })); + const options = fresh.triggerOptions(); + expect(options?.remoteLister).toBeDefined(); + expect(options?.syncStrategy).toStrictEqual([{ expr: '*', strategy: 'bidirectional' }]); + expect(options?.conflictResolver).toBeDefined(); + fresh.dispose(); + bridge.dispose(); +}); + +test('drops pending collaboration state on reset', () => { + const { bridge } = setup(); + bridge.accept(pulse()); + expect(bridge.triggerOptions()).toBeDefined(); + bridge.accept(pulse()); + bridge.reset(); + expect(bridge.triggerOptions()).toBeUndefined(); + bridge.dispose(); +}); + +test('keeps only the newest pulse by sentAt then batchId', () => { + const { bridge } = setup(); + const now = Date.now(); + bridge.accept(pulse({ batchId: 'newer', sentAt: now })); + bridge.accept(pulse({ batchId: 'older', sentAt: now - 1000 })); + const options = bridge.triggerOptions(); + expect(options).toBeDefined(); + // A stale pulse must not leave a second pending sync behind. + expect(bridge.triggerOptions()).toBeUndefined(); + bridge.dispose(); +}); + +test('the wrapped decider fills candidates from the real remote listing', () => { + const { bridge, dispatched, journal, registry } = setup(); + const now = Date.now(); + journal.record({ + key: 'note.md', + localUid: 'local-uid', + remoteUid: 'remote-old', + uploadedAt: now, + }); + bridge.accept(pulse()); + const options = bridge.triggerOptions(); + const wrapped = registry.get('collaboration:bidirectional'); + expect(wrapped).toBeDefined(); + const local = file('note.md', { size: 20, uid: 'local-uid' }); + const remote = file('note.md', { mtime: now, size: 20, uid: 'remote-new' }); + wrapped?.decider({ + localStats: new Map([['note.md', local]]), + records: new Map([['note.md', { isDir: false, local: 'local-uid', remote: 'remote-old' }]]), + remoteStats: new Map([['note.md', remote]]), + taskFactory: ((name: string, taskOptions: object) => ({ name, ...taskOptions })) as never, + }); + // The resolver closes over this very map instance, so seeing the candidate here proves the + // Decider -> resolver hand-off works despite buildOptions() running before decideTasks(). + expect( + (bridge as unknown as { activeCandidates: Map }).activeCandidates.size, + ).toBe(1); + expect(dispatched.some((entry) => entry.event === 'logSync')).toBe(true); + expect(options?.conflictResolver).toBeDefined(); + bridge.dispose(); +}); + +test('the wrapped decider reconciles committed peer files to prevent false conflicts by downloading', () => { + const { bridge, registry } = setup(); + bridge.recordCommittedPeer('synced.md', 'remote-committed-uid'); + bridge.accept(pulse()); + bridge.triggerOptions(); + const wrapped = registry.get('collaboration:bidirectional'); + expect(wrapped).toBeDefined(); + + const local = file('synced.md', { size: 50, uid: 'local-synced-uid' }); + const remote = file('synced.md', { mtime: Date.now(), size: 50, uid: 'remote-committed-uid' }); + const tasks = wrapped?.decider({ + localStats: new Map([['synced.md', local]]), + records: new Map([ + ['synced.md', { isDir: false, local: 'old-local', remote: 'old-remote' }], + ]), + remoteStats: new Map([['synced.md', remote]]), + taskFactory: ((name: string, taskOptions: object) => ({ name, ...taskOptions })) as never, + }); + + expect(tasks).toBeDefined(); + expect(tasks?.length).toBe(1); + const task = tasks?.[0]; + expect(task?.name).toBe('download'); + expect((task as unknown as { key: string })?.key).toBe('synced.md'); + bridge.dispose(); +}); + +test('patchChangedStats automatically records committed peers from remote stats during incremental listing', async () => { + const { bridge, registry, memoryDB } = setup(); + memoryDB.getStore('remoteContext20000').set('peer.md', file('peer.md', { uid: 'old-remote' })); + const remoteStat = file('peer.md', { mtime: Date.now(), size: 3, uid: 'new-remote' }); + const remoteFs = fakeRemoteFs(new Map([['peer.md', remoteStat]])); + + bridge.accept(pulse({ keys: ['peer.md'] })); + const options = bridge.triggerOptions(); + expect(options?.remoteLister).toBeDefined(); + + await options?.remoteLister?.({ + localFs: {} as never, + record: {} as never, + remoteFs, + reporter: () => 'include', + }); + + const wrapped = registry.get('collaboration:bidirectional'); + expect(wrapped).toBeDefined(); + + const local = file('peer.md', { size: 3, uid: 'local-stat-uid' }); + const tasks = wrapped?.decider({ + localStats: new Map([['peer.md', local]]), + records: new Map([['peer.md', { isDir: false, local: 'old-local', remote: 'old-remote' }]]), + remoteStats: new Map([['peer.md', remoteStat]]), + taskFactory: ((name: string, taskOptions: object) => ({ name, ...taskOptions })) as never, + }); + + expect(tasks).toBeDefined(); + expect(tasks?.length).toBe(1); + const task = tasks?.[0]; + expect(task?.name).toBe('download'); + expect((task as unknown as { key: string })?.key).toBe('peer.md'); + bridge.dispose(); +}); diff --git a/packages/collaboration/test/transport.test.ts b/packages/collaboration/test/transport.test.ts new file mode 100644 index 00000000..7e0f218a --- /dev/null +++ b/packages/collaboration/test/transport.test.ts @@ -0,0 +1,63 @@ +import { wireProperties } from '@/protocol'; +import { connectionOptions, publishOptions } from '@/transport'; +import { expect, test } from 'bun:test'; +import type { Envelope } from '@/protocol'; +import type { TransportOptions } from '@/transport'; + +function options(overrides: Partial = {}): TransportOptions { + return { + clientId: 'windows-abc123', + onMessage: () => {}, + onStatus: () => {}, + topic: 'sync-engine/collaboration/v1/room', + url: 'wss://broker.example/mqtt', + ...overrides, + }; +} + +function envelope(): Envelope { + return { + batchId: 'batch-1', + ciphertext: 'Y2lwaGVy', + clientId: 'windows-abc123', + nonce: 'bm9uY2U', + sentAt: 1234, + v: 1, + }; +} + +test('connect uses MQTT 5 and passes credentials only when configured', () => { + const bare = connectionOptions(options()); + expect(bare.protocolVersion).toBe(5); + expect(bare.clientId).toBe('windows-abc123'); + expect(bare).not.toHaveProperty('username'); + expect(bare).not.toHaveProperty('password'); + + const authenticated = connectionOptions(options({ password: 's3cr3t', username: 'paid-user' })); + expect(authenticated.username).toBe('paid-user'); + expect(authenticated.password).toBe('s3cr3t'); +}); + +test('empty credentials are treated as unset', () => { + const result = connectionOptions(options({ password: '', username: '' })); + expect(result).not.toHaveProperty('username'); + expect(result).not.toHaveProperty('password'); +}); + +test('publish carries all framing metadata as User Properties', () => { + const result = publishOptions(envelope()); + expect(result.properties?.userProperties).toStrictEqual(wireProperties(envelope())); + expect(result.properties?.userProperties?.client_id).toBe('windows-abc123'); + expect(result.qos).toBe(1); + expect(result.retain).toBe(false); +}); + +test('publish User Properties are plaintext strings and carry no file data', () => { + const properties = wireProperties(envelope()); + for (const value of Object.values(properties)) expect(typeof value).toBe('string'); + expect(Object.keys(properties).sort()).toStrictEqual(['batch_id', 'client_id', 'sent_at', 'v']); + expect(properties).not.toHaveProperty('index'); + expect(properties).not.toHaveProperty('count'); + expect(properties).not.toHaveProperty('stats'); + expect(properties).not.toHaveProperty('fingerprint'); +}); diff --git a/packages/collaboration/test/utils.test.ts b/packages/collaboration/test/utils.test.ts new file mode 100644 index 00000000..0ec8a7c2 --- /dev/null +++ b/packages/collaboration/test/utils.test.ts @@ -0,0 +1,57 @@ +import deviceType from '@/utils'; +import { afterEach, expect, test } from 'bun:test'; +import { Platform } from 'obsidian'; + +const defaults = { ...Platform }; + +afterEach(() => { + Object.assign(Platform, defaults); +}); + +function set(flags: Partial): void { + Object.assign(Platform, defaults, flags); +} + +test('falls back to desktop when no OS flag is set', () => { + set({}); + expect(deviceType()).toBe('desktop'); +}); + +test('returns desktop OS names', () => { + set({ isWin: true }); + expect(deviceType()).toBe('windows'); + set({ isLinux: true, isWin: false }); + expect(deviceType()).toBe('linux'); + set({ isLinux: false, isMacOS: true }); + expect(deviceType()).toBe('macos'); +}); + +test('prefers mobile app flags over desktop OS flags', () => { + set({ isIosApp: true, isMacOS: true }); + expect(deviceType()).toBe('ios'); + set({ isAndroidApp: true, isLinux: true }); + expect(deviceType()).toBe('android'); +}); + +test('distinguishes tablets by the same precedence as the plugin OS map', () => { + set({ isAndroidApp: true, isTablet: true }); + expect(deviceType()).toBe('android-tablet'); + set({ isMacOS: true, isTablet: true }); + expect(deviceType()).toBe('ipados'); +}); + +test('output is always safe for an MQTT client ID', () => { + for (const flags of [ + {}, + { isWin: true }, + { isLinux: true }, + { isMacOS: true }, + { isIosApp: true }, + { isAndroidApp: true }, + { isAndroidApp: true, isTablet: true }, + { isMacOS: true, isTablet: true }, + ]) { + set(flags); + expect(deviceType()).toMatch(/^[a-z-]+$/u); + } +}); diff --git a/packages/collaboration/tsconfig.json b/packages/collaboration/tsconfig.json new file mode 100644 index 00000000..ac4e13f2 --- /dev/null +++ b/packages/collaboration/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { "paths": { "@/*": ["./src/*"], "@": ["./src"] } }, + "include": ["src/**/*.ts", "test/**/*.ts", "tsdown.config.ts"] +} diff --git a/packages/collaboration/tsdown.config.ts b/packages/collaboration/tsdown.config.ts new file mode 100644 index 00000000..8c72782b --- /dev/null +++ b/packages/collaboration/tsdown.config.ts @@ -0,0 +1,22 @@ +import syncEngineModule from '@hesprs/sync-engine-sdk/tsdown-plugin'; +import moduleCatalog from '@repo/shared/module-catalog'; +import { defineConfig } from 'tsdown'; + +export default defineConfig({ + clean: process.env.MODE !== 'dev', + dts: false, + entry: { collaboration: 'src/index.ts' }, + inputOptions: { + resolve: { + aliasFields: [['browser']], + conditionNames: ['browser'], + mainFields: ['browser', 'module', 'main'], + }, + }, + minify: true, + outExtensions: () => ({ js: '.js' }), + outputOptions: { codeSplitting: false }, + platform: 'browser', + plugins: [syncEngineModule(moduleCatalog)], + target: 'es2024', +}); diff --git a/packages/dropbox/src/dropbox/auth.ts b/packages/dropbox/src/dropbox/auth.ts index 9b39c1f8..a614d394 100644 --- a/packages/dropbox/src/dropbox/auth.ts +++ b/packages/dropbox/src/dropbox/auth.ts @@ -2,13 +2,14 @@ import { API_RPC, OAUTH_AUTHORIZE_URL, OAUTH_TOKEN_URL } from './api'; import { encodeBase64Url, textToUint8Array } from '@repo/shared/binary'; import { getStatus } from '@repo/shared/error'; import { compileQuery } from '@repo/shared/path'; +import { sha256Digest } from '@repo/shared/sha256-bytes'; import { requestUrl } from 'obsidian'; -import type { Binary, Request, StringKV } from '@hesprs/sync-engine-sdk'; +import type { Request, StringKV } from '@hesprs/sync-engine-sdk'; import type { ObsidianProtocolData, SecretStorage } from 'obsidian'; export const APP_KEY = atob(process.env.DROPBOX_APP_KEY ?? ''); -export const REDIRECT_URI = 'obsidian://sync-engine-dropbox-auth'; -export const OAUTH_SCOPE = 'files.content.read files.content.write files.metadata.read'; +const REDIRECT_URI = 'obsidian://sync-engine-dropbox-auth'; +const OAUTH_SCOPE = 'files.content.read files.content.write files.metadata.read'; const KEYCHAIN_SECRET_ID = 'sync-engine-dropbox-refresh-token'; const FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded'; const TOKEN_EXPIRY_MARGIN = 60_000; @@ -38,11 +39,7 @@ function describeAuthError(data: TokenResponse, status: number): string { return data.error_description ?? data.error ?? `HTTP ${status}`; } -async function sha256(text: string): Promise { - return new Uint8Array(await crypto.subtle.digest('SHA-256', textToUint8Array(text))); -} - -export function buildAuthorizeUrl(state: string, codeChallenge: string): string { +function buildAuthorizeUrl(state: string, codeChallenge: string): string { return `${OAUTH_AUTHORIZE_URL}?${compileQuery({ client_id: APP_KEY, code_challenge: codeChallenge, @@ -60,7 +57,10 @@ export async function startAuthorization(): Promise { const state = encodeBase64Url(crypto.getRandomValues(new Uint8Array(16))); return { state, - url: buildAuthorizeUrl(state, encodeBase64Url(await sha256(verifier))), + url: buildAuthorizeUrl( + state, + encodeBase64Url(await sha256Digest(textToUint8Array(verifier))), + ), verifier, }; } diff --git a/packages/encryption/src/wrapper/shared.ts b/packages/encryption/src/wrapper/shared.ts index 48b923a2..acb6f4ec 100644 --- a/packages/encryption/src/wrapper/shared.ts +++ b/packages/encryption/src/wrapper/shared.ts @@ -1,4 +1,5 @@ import { concatBinary, textToUint8Array, toUint8Array } from '@repo/shared/binary'; +import { sha256Digest } from '@repo/shared/sha256-bytes'; import type { Binary } from '@hesprs/sync-engine-sdk'; const EMPTY_SALT: Binary = new Uint8Array(0); @@ -8,7 +9,7 @@ export const DECRYPTION_ERROR_MESSAGE = export const MASTER_KEY_LENGTH = 32; export const MASTER_SALT_LENGTH = 16; export const FILE_SALT_LENGTH = 16; -export const AES_GCM_TAG_LENGTH = 16; +const AES_GCM_TAG_LENGTH = 16; export const CONTENT_CHUNK_SIZE = 128 * 1024; const ENCRYPTED_CONTENT_CHUNK_SIZE = CONTENT_CHUNK_SIZE + AES_GCM_TAG_LENGTH; const FILE_KEY_INFO = 'file-key-v1'; @@ -91,7 +92,7 @@ export function encodeUInt96(value: number): Binary { return encodeUInt(value, 12); } -export function encodeUInt64(value: number): Binary { +function encodeUInt64(value: number): Binary { return encodeUInt(value, 8); } @@ -107,6 +108,4 @@ function encodeUInt(value: number, byteLength: number): Binary { return result; } -export function sha256Digest(data: BufferSource): Promise { - return crypto.subtle.digest('SHA-256', data); -} +export { sha256Digest } from '@repo/shared/sha256-bytes'; diff --git a/packages/gdrive/src/gdrive/auth.ts b/packages/gdrive/src/gdrive/auth.ts index eaea7e97..658b2b69 100644 --- a/packages/gdrive/src/gdrive/auth.ts +++ b/packages/gdrive/src/gdrive/auth.ts @@ -10,8 +10,8 @@ import { getStatus } from '@repo/shared/error'; import { Platform, requestUrl, SecretStorage } from 'obsidian'; import type { Request, StringKV } from '@hesprs/sync-engine-sdk'; -export const CLIENT_ID = atob(process.env.CLIENT_ID ?? ''); -export const CLIENT_SECRET = atob(process.env.CLIENT_SECRET ?? ''); // Not really a secret +const CLIENT_ID = atob(process.env.CLIENT_ID ?? ''); +const CLIENT_SECRET = atob(process.env.CLIENT_SECRET ?? ''); // Not really a secret const KEYCHAIN_SECRET_ID = 'sync-engine-gdrive-refresh-token'; // Secret storage id under which the Google refresh token is stored. type TokenResponse = { diff --git a/packages/history/package.json b/packages/history/package.json new file mode 100644 index 00000000..d0675040 --- /dev/null +++ b/packages/history/package.json @@ -0,0 +1,26 @@ +{ + "name": "history", + "version": "0.0.1", + "private": true, + "license": "MIT", + "contributors": [ + { + "name": "Hēsperus", + "github": "hesprs" + } + ], + "type": "module", + "scripts": { + "dev": "MODE=dev bun --bun tsdown", + "compile": "bun --bun tsdown", + "fix": "oxlint --silent --fix && oxfmt", + "check": "oxlint && oxfmt --check", + "tests": "bun --preload=./test/mocks test" + }, + "devDependencies": { + "@hesprs/sync-engine-sdk": "workspace:*", + "@repo/shared": "workspace:*", + "fflate": "^0.8.2", + "uni-kv": "catalog:" + } +} diff --git a/packages/history/src/chunker.ts b/packages/history/src/chunker.ts new file mode 100644 index 00000000..94d0cca2 --- /dev/null +++ b/packages/history/src/chunker.ts @@ -0,0 +1,61 @@ +import { TEXT_EXTS } from '@/constants'; +import { BIG_BLOB_LIMIT, chunkContent, SMALL_FILE_LIMIT } from '@repo/shared/cdc'; +import { sha256Bytes, sha256Text } from '@repo/shared/sha256-bytes'; +import type { HistoryStores } from '@/store'; +import type { BlobRef } from '@/types'; + +export { TEXT_EXTS } from '@/constants'; +export { assembleChunksToText as assembleChunks } from '@repo/shared/cdc'; + +const decoder = new TextDecoder(); + +export type ChunkedContent = { + text?: string; + chunkHashes?: Array; + newChunks?: Array<{ hash: string; bytes: Uint8Array }>; + blobs?: Array; + blobData?: Array<{ hash: string; bytes: Uint8Array }>; +}; + +export const isTextFile = (path: string): boolean => + TEXT_EXTS.some((ext) => path.toLowerCase().endsWith(ext)); + +/** Hashes a unified key for `IndexEntry.keyHash`, normalized to NFC. */ +export function keyHash(key: string): Promise { + return sha256Text(key.normalize('NFC')); +} + +/** + * Routes content at WAL ingestion. Small notes are stored inline; larger notes are split by FastCDC + * and only unseen chunk bytes are kept; attachments and big notes become standalone blobs. + */ +export async function captureContent( + stores: HistoryStores, + key: string, + data: Uint8Array, +): Promise { + const normalizedKey = key.normalize('NFC'); + if (!isTextFile(normalizedKey) || data.byteLength >= BIG_BLOB_LIMIT) { + const hash = await sha256Bytes(data); + return { + blobData: [{ bytes: new Uint8Array(data), hash }], + blobs: [{ hash, key: normalizedKey, size: data.byteLength }], + }; + } + if (data.byteLength < SMALL_FILE_LIMIT) return { text: decoder.decode(data) }; + + const rawChunks = await chunkContent(data); + const chunkHashes = rawChunks.map((c) => c.hash); + const uniqueChunks = new Map(); + for (const { bytes, hash } of rawChunks) + if (!uniqueChunks.has(hash)) uniqueChunks.set(hash, bytes); + + const newChunks: Array<{ hash: string; bytes: Uint8Array }> = []; + await Promise.all( + [...uniqueChunks.entries()].map(async ([hash, bytes]) => { + const existing = await stores.chunks.get(hash); + if (existing === undefined) newChunks.push({ bytes: new Uint8Array(bytes), hash }); + }), + ); + return { chunkHashes, newChunks }; +} diff --git a/packages/history/src/constants.ts b/packages/history/src/constants.ts new file mode 100644 index 00000000..2211c78e --- /dev/null +++ b/packages/history/src/constants.ts @@ -0,0 +1,134 @@ +/** + * History Module Constants + * + * Centralizes time units, ingestion debounces, packfile assembly thresholds, sync triggers, storage + * paths, binary framing headers, and tiered retention/GC policies. + */ + +// ========================================== +// Time Units (milliseconds) +// ========================================== + +const SECOND_MS = 1000; +const MINUTE_MS = 60 * SECOND_MS; +const HOUR_MS = 60 * MINUTE_MS; +export const DAY_MS = 24 * HOUR_MS; +export const WEEK_MS = 7 * DAY_MS; + +// ========================================== +// Local WAL & Edit Ingestion Cadence +// ========================================== + +/** + * Debounce delay (in ms) after user stops typing before capturing a draft into WAL. 15 seconds + * allows natural pauses while coalescing typing bursts into coherent versions. + */ +export const TEXT_DEBOUNCE_MS = 15_000; + +/** + * Maximum wait time (in ms) during continuous typing before forcing a WAL commit. Prevents long + * marathon editing sessions from leaving no recovery milestones for hours. + */ +export const TEXT_MAX_WAIT_MS = 120_000; + +/** Debounce delay (in ms) for binary/attachment files before recording. */ +export const NON_TEXT_DEBOUNCE_MS = 1000; + +/** Maximum wait time (in ms) for binary/attachment files before forced commit. */ +export const NON_TEXT_MAX_WAIT_MS = 5000; + +// ========================================== +// Fast Channel & Remote Flush Thresholds +// ========================================== + +/** + * Minimum fast channel debounce floor (in ms). When negotiating with the main plugin's realtimeSync + * setting, fast channel sync requests are delayed by at least Math.max(FAST_SYNC_DEBOUNCE_MS, + * realtimeSync.value) to prevent prematurely interrupting the user's configured sync cadence. If + * realtimeSync is disabled, the fast channel stays silent. + */ +export const FAST_SYNC_DEBOUNCE_MS = 1000; + +/** + * Base idle time (in ms) before uncommitted WAL entries are packed into an immutable .pack file. + * Negotiates with realtimeSync so pack assembly aligns with longer sync intervals when configured. + */ +export const PACK_IDLE_MS = 30_000; + +/** Maximum uncommitted revisions accumulated in WAL before immediately triggering a pack flush. */ +export const MAX_PACK_REVISIONS = 20; + +/** Maximum uncommitted payload volume (256 KB) before immediately triggering a pack flush. */ +export const MAX_PACK_BYTES = 256 * 1024; + +/** Maximum rolling pack window referenced directly in the device's head log. */ +export const MAX_PACK_WINDOW = 32; + +// ========================================== +// Retention, Settings & GC Policies +// ========================================== + +/** Default maximum number of retained versions per note. */ +export const DEFAULT_MAX_VERSIONS = 100; + +/** Default maximum retention duration (in days) before older versions are purged. */ +export const DEFAULT_RETENTION_DAYS = 90; + +/** Minimum cooldown interval (24 hours) between automatic background GC runs. */ +export const AUTO_GC_COOLDOWN_MS = DAY_MS; + +/** + * Tier 1 cutoff: Edits younger than 1 hour are preserved fine-grained to allow full undo of recent + * work. + */ +export const TIER_RECENT_CUTOFF_MS = HOUR_MS; + +/** + * Tier 2 cutoff: Edits between 1 hour and 24 hours are thinned to at most 1 version per calendar + * hour. + */ +export const TIER_HOURLY_CUTOFF_MS = 24 * HOUR_MS; + +/** + * Tier 3 cutoff: Edits between 1 day and 30 days are thinned to at most 1 version per calendar day + * (capping monthly history to ~30 daily milestones). Edits older than 30 days are thinned to at + * most 1 version per week. + */ +export const TIER_DAILY_CUTOFF_MS = 30 * DAY_MS; + +// ========================================== +// Storage Keys, Prefixes & Packfile Framing +// ========================================== + +/** Prefix for all remote history objects stored in S3/WebDAV. */ +export const HISTORY_PREFIX = 'sync-engine-history/'; + +/** File extensions treated as mergeable plain text by the chunker. */ +export const TEXT_EXTS = ['.md', '.canvas', '.txt', '.json', '.base']; + +/** Key used in history database for persistent Lamport logical clock. */ +export const LAMPORT_KEY = 'clock'; + +/** Key used in history database for the unique local device ID. */ +export const DEVICE_ID_KEY = 'id'; + +/** Key used in history database for recording the last successful GC run timestamp. */ +export const GC_TIME_KEY = 'time'; + +/** Packfile magic 4-byte header identifier (SEHP: 0x53454850). */ +export const PACK_MAGIC = 0x53_45_48_50; + +/** Packfile binary format version. */ +export const PACK_VERSION = 1; + +/** Fixed byte size of the packfile binary header. */ +export const PACK_HEADER_BYTES = 128; + +/** Flag bit in IndexEntry indicating a small inline note payload. */ +export const FLAG_SMALL_FILE = 1; + +/** Flag bit in IndexEntry indicating a file deletion tombstone. */ +export const FLAG_DELETION = 2; + +/** Flag bit in IndexEntry indicating a major milestone/checkpoint. */ +export const FLAG_CHECKPOINT = 4; diff --git a/packages/history/src/crypto.ts b/packages/history/src/crypto.ts new file mode 100644 index 00000000..4f6818d5 --- /dev/null +++ b/packages/history/src/crypto.ts @@ -0,0 +1,101 @@ +import { readSecret } from '@repo/shared/secret'; +import type { App } from 'obsidian'; + +/** Prefix marking an encrypted history object. */ +const MAGIC = new Uint8Array([0x53, 0x45, 0x48, 0x50, 0x43, 0x31]); // "SEHPC1" +const IV_BYTES = 12; +// Module-specific salt: the derived key is intentionally independent from the encryption module's +// own rootFileKey (which uses argon2id + HKDF). Reusing the password but not the key derivation +// avoids cross-protocol key reuse while still requiring no second password from the user. +const KEY_SALT = new TextEncoder().encode('sync-engine-history/content/v1'); +const PBKDF2_ITERATIONS = 100_000; + +export type HistoryCipher = { + encrypt: (data: Uint8Array) => Promise; + decrypt: (data: Uint8Array) => Promise; +}; + +function hasMagic(data: Uint8Array): boolean { + if (data.byteLength < MAGIC.byteLength) return false; + for (let index = 0; index < MAGIC.byteLength; index++) + if (data[index] !== MAGIC[index]) return false; + return true; +} + +/** Concatenation over Uint8Array without the BufferSource variance of concatBinary. */ +function concat(...parts: Array): Uint8Array { + let total = 0; + for (const part of parts) total += part.byteLength; + const out = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.byteLength; + } + return out; +} + +function concrete(data: Uint8Array): Uint8Array { + return new Uint8Array(data); +} + +export async function createCipher(password: string): Promise { + const baseKey = await crypto.subtle.importKey( + 'raw', + concrete(new TextEncoder().encode(password)).buffer, + 'PBKDF2', + false, + ['deriveKey'], + ); + const key = await crypto.subtle.deriveKey( + { hash: 'SHA-256', iterations: PBKDF2_ITERATIONS, name: 'PBKDF2', salt: KEY_SALT.buffer }, + baseKey, + { length: 256, name: 'AES-GCM' }, + false, + ['encrypt', 'decrypt'], + ); + return { + async decrypt(data) { + if (!hasMagic(data)) + throw new Error('Cannot decrypt history: missing encryption magic header.'); + const iv = data.subarray(MAGIC.byteLength, MAGIC.byteLength + IV_BYTES); + const ciphertext = data.subarray(MAGIC.byteLength + IV_BYTES); + try { + return new Uint8Array( + await crypto.subtle.decrypt( + { iv: concrete(iv).buffer, name: 'AES-GCM' }, + key, + concrete(ciphertext).buffer, + ), + ); + } catch { + throw new Error( + 'Cannot decrypt history: the encryption password changed or data is corrupted.', + ); + } + }, + async encrypt(data) { + const iv = crypto.getRandomValues(new Uint8Array(IV_BYTES)); + const ciphertext = new Uint8Array( + await crypto.subtle.encrypt({ iv, name: 'AES-GCM' }, key, concrete(data).buffer), + ); + return concat(MAGIC, iv, ciphertext); + }, + }; +} + +/** + * Detects whether the encryption module is active and derives a content cipher from its password. + * Returns undefined when encryption is off (plaintext). Throws when encryption is on but no + * password is configured, because writing plaintext in that state would leak note contents. + */ +export async function createHistoryCipher( + modules: Record, + app: App, +): Promise { + const encryption = modules.encryption as { enabled?: boolean; password?: string } | undefined; + if (!encryption?.enabled) return undefined; + const password = readSecret(app, encryption.password); + if (!password) throw new Error('Encryption is enabled but its password is not set.'); + return createCipher(password); +} diff --git a/packages/history/src/dag.ts b/packages/history/src/dag.ts new file mode 100644 index 00000000..3ea41586 --- /dev/null +++ b/packages/history/src/dag.ts @@ -0,0 +1,102 @@ +import Onp from '@repo/shared/onp'; +import type { IndexEntry } from '@/types'; + +/** Deterministic total order: all devices that saw the same commits linearize identically. */ +export function compareCommits(a: IndexEntry, b: IndexEntry): number { + return ( + a.lamport - b.lamport || + a.deviceId.localeCompare(b.deviceId) || + a.keyHash.localeCompare(b.keyHash) + ); +} + +export function linearize(entries: ReadonlyArray): Array { + return [...entries].sort(compareCommits); +} + +/** Extracts the common (unchanged) token runs between two sequences via the shared ONP kernel. */ +function commonRuns(o: ReadonlyArray, side: ReadonlyArray): Array { + const hunks = new Onp(o, side).compose(); + const common: Array = []; + let cursor = 0; + for (const hunk of hunks) { + const [start, length] = hunk.file1; + for (const token of o.slice(cursor, start)) common.push(token); + cursor = start + length; + } + for (const token of o.slice(cursor)) common.push(token); + return common; +} + +export type MergeResult = { tokens: Array; conflicted: boolean }; + +/** + * Two-stage merge. Stage 1 aligns the two chunk-hash sequences against the common ancestor with a + * shared ONP LCS so unchanged chunk runs are reused by hash. Stage 2 hands only the divergent runs + * to `mergeText`, producing merged tokens that a fresh CDC pass will store as new chunks. + */ +export function mergeChunkSequences( + ancestor: ReadonlyArray, + a: ReadonlyArray, + b: ReadonlyArray, + mergeText: (o: string, a: string, b: string) => { text: string; conflicted: boolean }, +): MergeResult { + const commonA = commonRuns(ancestor, a); + const commonB = commonRuns(ancestor, b); + const commonSet = new Set(commonA.filter((token) => commonB.includes(token))); + const changedA = a.filter((token) => !commonSet.has(token)); + const changedB = b.filter((token) => !commonSet.has(token)); + const merged = mergeText(ancestor.join('\n'), changedA.join('\n'), changedB.join('\n')); + const divergent = merged.text.split('\n').filter((line) => line.length > 0); + const tokens = [...commonA]; + for (const token of divergent) if (!tokens.includes(token)) tokens.push(token); + return { conflicted: merged.conflicted, tokens }; +} + +/** + * Line-level three-way merge built on the same ONP kernel (not a copy of Smart Merge's diff3). + * Conflict markers are preserved so nothing is silently discarded. + */ +export function mergeTextLines( + o: string, + a: string, + b: string, +): { text: string; conflicted: boolean } { + const base = o.split('\n'); + const left = a.split('\n'); + const right = b.split('\n'); + if (left.join('\n') === right.join('\n')) return { conflicted: false, text: a }; + if (left.join('\n') === base.join('\n')) return { conflicted: false, text: b }; + if (right.join('\n') === base.join('\n')) return { conflicted: false, text: a }; + const common = commonRuns(base, left); + const commonRight = new Set(commonRuns(base, right)); + const stable = common.filter((line) => commonRight.has(line)); + const localOnly = left.filter((line) => !stable.includes(line)); + const remoteOnly = right.filter((line) => !stable.includes(line)); + const parts = [...stable]; + if (localOnly.length && remoteOnly.length) { + parts.push('<<<<<<< ours', ...localOnly, '=======', ...remoteOnly, '>>>>>>> theirs'); + return { conflicted: true, text: parts.join('\n') }; + } + parts.push(...localOnly, ...remoteOnly); + return { conflicted: false, text: parts.join('\n') }; +} + +/** A merge commit has both tips as parents; the result is committed even when conflicted. */ +export function mergeCommit(options: { + a: IndexEntry; + b: IndexEntry; + deviceId: string; + key: string; + lamport: number; +}): Pick { + const { a, b, deviceId, key, lamport } = options; + return { + at: Date.now(), + deviceId, + key, + keyHash: a.keyHash, + lamport, + parents: [`${a.key}@${a.lamport}`, `${b.key}@${b.lamport}`], + }; +} diff --git a/packages/history/src/diff.ts b/packages/history/src/diff.ts new file mode 100644 index 00000000..6e5361a1 --- /dev/null +++ b/packages/history/src/diff.ts @@ -0,0 +1,77 @@ +import Onp from '@repo/shared/onp'; + +export type DiffLine = { text: string; type: 'added' | 'removed' | 'unchanged' }; + +export function computeLineDiff(historical: string, current: string): Array { + const cleanHist = historical.replaceAll('\r\n', '\n'); + const cleanCurr = current.replaceAll('\r\n', '\n'); + const a = cleanHist ? cleanHist.split('\n') : []; + const b = cleanCurr ? cleanCurr.split('\n') : []; + const hunks = new Onp(a, b).compose(); + const lines: Array = []; + let curA = 0; + let curB = 0; + for (const hunk of hunks) { + const [startA, lenA] = hunk.file1; + const [startB, lenB] = hunk.file2; + while (curA < startA && curB < startB) { + lines.push({ text: a[curA], type: 'unchanged' }); + curA++; + curB++; + } + for (let i = 0; i < lenA; i++) lines.push({ text: a[startA + i], type: 'removed' }); + for (let i = 0; i < lenB; i++) lines.push({ text: b[startB + i], type: 'added' }); + curA = startA + lenA; + curB = startB + lenB; + } + while (curA < a.length && curB < b.length) { + lines.push({ text: a[curA], type: 'unchanged' }); + curA++; + curB++; + } + return lines; +} + +export function renderDiffLines(previewEl: HTMLElement, diffLines: Array): void { + for (const line of diffLines) { + const row = previewEl.createEl('div'); + row.style.display = 'flex'; + row.style.alignItems = 'flex-start'; + row.style.padding = '2px 10px'; + row.style.fontFamily = 'var(--font-monospace, monospace)'; + row.style.fontSize = '12px'; + row.style.lineHeight = '1.6'; + row.style.whiteSpace = 'pre-wrap'; + row.style.wordBreak = 'break-word'; + + const sign = row.createSpan(); + sign.style.width = '18px'; + sign.style.flexShrink = '0'; + sign.style.userSelect = 'none'; + sign.style.fontWeight = 'bold'; + + const textSpan = row.createSpan(); + textSpan.style.flex = '1'; + textSpan.style.whiteSpace = 'pre-wrap'; + textSpan.style.wordBreak = 'break-word'; + textSpan.style.color = 'var(--text-normal)'; + textSpan.setText(line.text); + + if (line.type === 'added') { + row.style.backgroundColor = 'rgba(46, 160, 67, 0.15)'; + row.style.borderLeft = '3px solid var(--text-success, #2ea043)'; + sign.style.color = 'var(--text-success, #2ea043)'; + sign.setText('+'); + } else if (line.type === 'removed') { + row.style.backgroundColor = 'rgba(248, 81, 73, 0.15)'; + row.style.borderLeft = '3px solid var(--text-error, #f85149)'; + sign.style.color = 'var(--text-error, #f85149)'; + sign.setText('-'); + } else { + row.style.backgroundColor = 'transparent'; + row.style.borderLeft = '3px solid transparent'; + sign.style.color = 'var(--text-muted, #8b949e)'; + sign.setText(' '); + } + } +} diff --git a/packages/history/src/flush.ts b/packages/history/src/flush.ts new file mode 100644 index 00000000..dd741104 --- /dev/null +++ b/packages/history/src/flush.ts @@ -0,0 +1,435 @@ +import { keyHash } from '@/chunker'; +import { + FAST_SYNC_DEBOUNCE_MS, + MAX_PACK_BYTES, + MAX_PACK_REVISIONS, + MAX_PACK_WINDOW, + PACK_IDLE_MS, +} from '@/constants'; +import { buildCommitLog } from '@/head-log'; +import { markChunks, packRefKey, putCommitLocation } from '@/index-store'; +import { decodePack, encodePack, packFileName, textBytes } from '@/pack'; +import { blobPresent, bumpLamport, clearPending, markBlobs, readPending } from '@/store'; +import { FLAG_DELETION, FLAG_SMALL_FILE } from '@/types'; +import { isNotFound } from '@repo/shared/error'; +import { sha256Bytes, sha256Text } from '@repo/shared/sha256-bytes'; +import type { RemoteHistory } from '@/remote'; +import type { HistoryStores } from '@/store'; +import type { BlobRef, ChunkLocation, CommitLog, IndexEntry, PackRef, WalEntry } from '@/types'; + +export type FlushContext = { + getRealtimeSyncConfig?: () => { enabled: boolean; value: number } | undefined; + onFlushError?: (error: Error) => void; + remote: RemoteHistory; + requestSync: (trigger: string) => void; + stores: HistoryStores; +}; + +type Assembled = { + entries: Array; + payload: Uint8Array; + packRef: PackRef; + wal: Array; + chunkLocations: Array<{ hash: string; location: ChunkLocation }>; + blobBytes: Array<{ hash: string; bytes: Uint8Array }>; + blobRefs: Array; +}; + +function isMissingError(error: unknown): boolean { + if (isNotFound(error)) return true; + const msg = String(error).toLowerCase(); + return msg.includes('nosuchkey') || msg.includes('missing') || msg.includes('not found'); +} + +function uint8Concat(parts: Array): Uint8Array { + let total = 0; + for (const part of parts) total += part.byteLength; + const out = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.byteLength; + } + return out; +} + +/** Assembles one immutable pack from a snapshot of pending WAL entries. */ +async function assemble(ctx: FlushContext, wal: Array, seq: number): Promise { + const { deviceId } = ctx.remote; + const payloadParts: Array = []; + const entries: Array = []; + const chunkLocations: Array<{ hash: string; location: ChunkLocation }> = []; + const blobBytes: Array<{ hash: string; bytes: Uint8Array }> = []; + const blobRefs: Array = []; + const packRefBase = { deviceId, fromLamport: 0, hash: '', seq, toLamport: 0 }; + let offset = 0; + + for (const entry of wal) { + const indexEntry: IndexEntry = { + at: entry.at, + blobs: entry.blobs ?? [], + chunks: [], + deviceId, + flags: entry.flags ?? 0, + key: entry.key, + keyHash: await keyHash(entry.key), + lamport: entry.lamport, + parents: entry.parents ?? (entry.parent ? [entry.parent] : []), + }; + if (entry.text !== undefined) { + const bytes = textBytes(entry.text); + payloadParts.push(bytes); + indexEntry.inlineOffset = offset; + indexEntry.inlineLength = bytes.byteLength; + indexEntry.flags |= FLAG_SMALL_FILE; + indexEntry.hash = await sha256Bytes(bytes); + offset += bytes.byteLength; + } else if (entry.chunkHashes) { + indexEntry.hash = await sha256Text(entry.chunkHashes.join(',')); + const newByHash = new Map((entry.newChunks ?? []).map((c) => [c.hash, c.bytes])); + for (const hash of entry.chunkHashes) { + const bytes = newByHash.get(hash); + if (bytes) { + payloadParts.push(bytes); + indexEntry.chunks.push({ hash, length: bytes.byteLength, offset }); + chunkLocations.push({ + hash, + location: { length: bytes.byteLength, offset, pack: packRefBase }, + }); + offset += bytes.byteLength; + } else + // Already present remotely; restore resolves it through the chunk index by hash. + indexEntry.chunks.push({ hash, length: 0, offset: -1 }); + } + } else if (entry.blobs && entry.blobs.length > 0) + indexEntry.hash = await sha256Text(entry.blobs.map((b) => b.hash).join(',')); + else if ((entry.flags ?? 0) & FLAG_DELETION) indexEntry.hash = ''; + for (const blob of entry.blobData ?? []) blobBytes.push(blob); + for (const blob of entry.blobs ?? []) blobRefs.push(blob); + entries.push(indexEntry); + } + + const lamports = wal.map((entry) => entry.lamport); + const packRef: PackRef = { + ...packRefBase, + fromLamport: Math.min(...lamports), + toLamport: Math.max(...lamports), + }; + // Fill the real hash into every chunk location and the pack ref after encodePack returns it. + return { + blobBytes, + blobRefs, + chunkLocations, + entries, + packRef: { ...packRef, hash: '' }, + payload: uint8Concat(payloadParts), + wal, + }; +} + +async function currentLog(ctx: FlushContext): Promise { + const local = await ctx.stores.heads.get(ctx.remote.deviceId); + if (local) return local; + const remote = await ctx.remote.readHead(ctx.remote.deviceId); + if (remote) return remote; + return buildCommitLog({ + chunkIndex: {}, + deviceId: ctx.remote.deviceId, + generation: 0, + heads: [], + ns: ctx.remote.ns, + packs: [], + peakLamport: 0, + seq: 0, + updatedAt: Date.now(), + }); +} + +/** Replays a pending batch into one pack. Ordering is blobs → pack → head → WAL commit. */ +export async function flushPending(ctx: FlushContext, reason: string): Promise { + const wal = await readPending(ctx.stores); + return flushWal(ctx, wal, reason); +} + +/** Bounded flush for lifecycle exit: caps the batch at 256 KiB and leaves the remainder queued. */ +export async function flushPendingBatch( + ctx: FlushContext, + reason: string, + maxBytes = MAX_PACK_BYTES, +): Promise { + const all = await readPending(ctx.stores); + const wal: Array = []; + let bytes = 0; + for (const entry of all) { + const size = + (entry.text?.length ?? 0) + + (entry.newChunks?.length ?? 0) + + (entry.blobData?.reduce((acc, b) => acc + b.bytes.byteLength, 0) ?? 0); + if (wal.length > 0 && bytes + size > maxBytes) break; + wal.push(entry); + bytes += size; + } + if (wal.length === 0) return false; + return flushWal(ctx, wal, reason); +} + +async function flushWal( + ctx: FlushContext, + wal: Array, + _reason: string, +): Promise { + if (wal.length === 0) return false; + const log = await currentLog(ctx); + const seq = log.seq + 1; + const assembled = await assemble(ctx, wal, seq); + + const encoded = await encodePack( + { + baseLamport: assembled.packRef.fromLamport, + // Deterministic: replaying the same batch must reproduce identical bytes. + createdAt: Math.max(...wal.map((entry) => entry.at)), + deviceId: ctx.remote.deviceId, + ns: ctx.remote.ns, + peakLamport: assembled.packRef.toLamport, + seq, + }, + assembled.entries, + assembled.payload, + ); + // `hash` is only known after encode; patch the pack ref and chunk locations. + const realRef: PackRef = { ...assembled.packRef, hash: encoded.header.hash }; + for (const location of assembled.chunkLocations) location.location.pack = realRef; + + // 1. Blobs (content-addressed, idempotent). Skip ones already confirmed present locally. + for (const blob of assembled.blobBytes) + if (blob.bytes.byteLength && !(await blobPresent(ctx.stores, blob.hash))) + await ctx.remote.putBlob(blob.hash, blob.bytes); + // 2. Pack (immutable, idempotent; identical bytes on replay). + await ctx.remote.putPack(seq, encoded.header.hash, encoded.bytes); + // 3. Head log (single writer; only this device writes its own log). + const nextLog = await buildCommitLog({ + chunkIndex: { ...log.chunkIndex }, + deviceId: ctx.remote.deviceId, + generation: log.generation + 1, + heads: assembled.entries.map((entry) => `${entry.key}@${entry.lamport}`), + ns: ctx.remote.ns, + packs: [...log.packs, realRef].slice(-MAX_PACK_WINDOW), + peakLamport: Math.max(log.peakLamport, assembled.packRef.toLamport), + seq, + updatedAt: Date.now(), + }); + for (const location of assembled.chunkLocations) + nextLog.chunkIndex[location.hash] = packRefKey(realRef); + await ctx.remote.writeHead(nextLog); + await ctx.stores.heads.set(ctx.remote.deviceId, nextLog); + // 4. Local index + WAL commit (last; everything before is replayable). + for (const entry of assembled.entries) + await putCommitLocation(ctx.stores, `${entry.key}@${entry.lamport}`, { + indexLength: encoded.header.indexLength, + indexOffset: encoded.header.indexOffset, + pack: realRef, + }); + await markChunks(ctx.stores, assembled.chunkLocations); + await markBlobs(ctx.stores, assembled.blobRefs); + await clearPending(ctx.stores, wal); + return true; +} + +/** Rebuilds the version and chunk indexes from one decoded pack. */ +async function indexPack( + ctx: FlushContext, + ref: PackRef, + header: { indexLength: number; indexOffset: number }, + entries: Array, +): Promise { + for (const entry of entries) + await putCommitLocation(ctx.stores, `${entry.key}@${entry.lamport}`, { + indexLength: header.indexLength, + indexOffset: header.indexOffset, + pack: ref, + }); + const locations = entries.flatMap((entry) => + entry.chunks + .filter((chunk) => chunk.offset >= 0) + .map((chunk) => ({ + hash: chunk.hash, + location: { length: chunk.length, offset: chunk.offset, pack: ref }, + })), + ); + await markChunks(ctx.stores, locations); +} + +/** Reads a pack by its reference and indexes it. */ +async function indexPackByRef(ctx: FlushContext, ref: PackRef): Promise { + const bytes = await ctx.remote.getObject( + `${ctx.remote.ns}/packs/${ref.deviceId}/${packFileName(ref.deviceId, ref.seq, ref.hash)}`, + ); + const decoded = await decodePack(bytes); + await indexPack(ctx, ref, decoded.header, decoded.entries); +} + +/** + * Cold-start reconciliation. + * + * 1. Replay any pending WAL. + * 2. LIST heads/ once, read each head log, mirror it locally, and fold its peak Lamport into the local + * clock so future edits are ordered after every observed remote commit. + * 3. For each device with newer commits, LIST packs// once and promote orphan packs (written + * but never recorded in the head log) using the self-certifying seq+hash in the filename. + * 4. Rebuild the index and chunk stores only from newly observed packs, skipping packs that were + * already indexed under previous generations. + */ +const inFlightReconciles = new Map>(); + +export async function reconcile(ctx: FlushContext): Promise { + const key = `${ctx.remote.ns}:${ctx.remote.deviceId}`; + const active = inFlightReconciles.get(key); + if (active) return active; + const promise = runReconcile(ctx).finally(() => inFlightReconciles.delete(key)); + inFlightReconciles.set(key, promise); + return promise; +} + +async function runReconcile(ctx: FlushContext): Promise { + await flushPending(ctx, 'cold-start'); + const headStats = await ctx.remote.listHeads(); + let seenLamport = 0; + const logsToReconcile: Array<{ cached?: CommitLog; log: CommitLog }> = []; + for (const head of headStats) { + const headId = typeof head === 'string' ? head : head.deviceId; + const uid = typeof head === 'string' ? undefined : head.uid; + const cached = await ctx.stores.heads.get(headId); + if (headId === ctx.remote.deviceId && cached) { + seenLamport = Math.max(seenLamport, cached.peakLamport); + continue; + } + + if (cached && uid !== undefined && cached.remoteUid === uid) { + seenLamport = Math.max(seenLamport, cached.peakLamport); + continue; + } + + const log = await ctx.remote.readHead(headId); + if (!log) continue; + if (uid !== undefined) log.remoteUid = uid; + seenLamport = Math.max(seenLamport, log.peakLamport); + if (cached && cached.generation >= log.generation) { + if (uid !== undefined && cached.remoteUid !== uid) { + cached.remoteUid = uid; + await ctx.stores.heads.set(headId, cached); + } + continue; + } + + logsToReconcile.push({ cached, log }); + } + if (seenLamport > 0) await bumpLamport(ctx.stores, seenLamport); + + for (const { cached, log } of logsToReconcile) { + const cachedPackKeys = new Set( + cached?.packs.map((pack) => `${pack.deviceId}:${pack.seq}:${pack.hash}`), + ); + const known = new Set( + log.packs.map((pack) => packFileName(pack.deviceId, pack.seq, pack.hash)), + ); + for (const fileName of await ctx.remote.listPacks(log.deviceId)) { + if (known.has(fileName)) continue; + // An orphan pack is self-certifying: seq + hash in the name are enough to index it. + try { + const bytes = await ctx.remote.getObject( + `${ctx.remote.ns}/packs/${log.deviceId}/${fileName}`, + ); + const decoded = await decodePack(bytes); + const orphanRef: PackRef = { + deviceId: decoded.header.deviceId, + fromLamport: decoded.header.baseLamport, + hash: decoded.header.hash, + seq: decoded.header.seq, + toLamport: decoded.header.peakLamport, + }; + await indexPack(ctx, orphanRef, decoded.header, decoded.entries); + } catch (error) { + if (isMissingError(error)) continue; + throw error; + } + } + for (const ref of log.packs) { + if (cachedPackKeys.has(`${ref.deviceId}:${ref.seq}:${ref.hash}`)) continue; + try { + await indexPackByRef(ctx, ref); + } catch (error) { + if (isMissingError(error)) continue; + throw error; + } + } + await ctx.stores.heads.set(log.deviceId, log); + } +} + +export class FlushScheduler { + private pendingBytes = 0; + private pendingRevisions = 0; + private idleTimer?: ReturnType; + private fastTimer?: ReturnType; + private flushing = false; + + constructor(private readonly ctx: FlushContext) {} + + /** Called on every captured edit. Fast channel and flush predicates negotiate with realtimeSync. */ + onEdit(size: number): void { + this.pendingBytes += size; + this.pendingRevisions += 1; + + if (this.fastTimer) clearTimeout(this.fastTimer); + const realtimeConfig = this.ctx.getRealtimeSyncConfig?.(); + if (realtimeConfig === undefined) + this.fastTimer = setTimeout( + () => this.ctx.requestSync('realtime'), + FAST_SYNC_DEBOUNCE_MS, + ); + else if (realtimeConfig.enabled) { + const fastDebounceMs = Math.max(FAST_SYNC_DEBOUNCE_MS, realtimeConfig.value); + this.fastTimer = setTimeout(() => this.ctx.requestSync('realtime'), fastDebounceMs); + } + + if (this.idleTimer) clearTimeout(this.idleTimer); + const packIdleMs = realtimeConfig?.enabled + ? Math.max(PACK_IDLE_MS, realtimeConfig.value) + : PACK_IDLE_MS; + this.idleTimer = setTimeout(() => void this.trigger('idle'), packIdleMs); + + if (this.pendingRevisions >= MAX_PACK_REVISIONS || this.pendingBytes >= MAX_PACK_BYTES) + void this.trigger(this.pendingRevisions >= MAX_PACK_REVISIONS ? 'count' : 'volume'); + } + + async trigger(reason: string): Promise { + if (this.flushing) return; + this.flushing = true; + try { + await flushPending(this.ctx, reason); + this.pendingBytes = 0; + this.pendingRevisions = 0; + } catch (error) { + // WAL entries stay pending; the next trigger replays the same bytes idempotently. + this.ctx.onFlushError?.(error instanceof Error ? error : new Error(String(error))); + } finally { + this.flushing = false; + } + } + + /** Best-effort last chance before exit; caps the batch and leaves the remainder queued. */ + async onUnload(): Promise { + if (this.fastTimer) clearTimeout(this.fastTimer); + if (this.idleTimer) clearTimeout(this.idleTimer); + try { + await flushPendingBatch(this.ctx, 'lifecycle'); + } catch (error) { + this.ctx.onFlushError?.(error instanceof Error ? error : new Error(String(error))); + } + } + + dispose(): void { + if (this.fastTimer) clearTimeout(this.fastTimer); + if (this.idleTimer) clearTimeout(this.idleTimer); + } +} diff --git a/packages/history/src/gc.ts b/packages/history/src/gc.ts new file mode 100644 index 00000000..9118dce7 --- /dev/null +++ b/packages/history/src/gc.ts @@ -0,0 +1,340 @@ +import { + DAY_MS, + DEFAULT_MAX_VERSIONS, + DEFAULT_RETENTION_DAYS, + MAX_PACK_WINDOW, + TIER_DAILY_CUTOFF_MS, + TIER_HOURLY_CUTOFF_MS, + TIER_RECENT_CUTOFF_MS, + WEEK_MS, +} from '@/constants'; +import { buildCommitLog } from '@/head-log'; +import { allChunkLocations, allCommitLocations, markChunks, packRefKey } from '@/index-store'; +import { decodePack, encodePack, packFileName } from '@/pack'; +import { FLAG_CHECKPOINT } from '@/types'; +import { isNotFound } from '@repo/shared/error'; +import type { DecodedPack } from '@/pack'; +import type { RemoteHistory } from '@/remote'; +import type { HistoryStores } from '@/store'; +import type { ChunkLocation, CommitLog, IndexEntry, PackRef } from '@/types'; + +export type GcContext = { remote: RemoteHistory; stores: HistoryStores }; +export type GcOptions = { + maxVersions?: number; + now?: number; + retentionDays?: number; + tiered?: boolean; +}; + +function calendarDayKey(timestamp: number): string { + const d = new Date(timestamp); + return `${d.getFullYear()}-${d.getMonth() + 1}-${d.getDate()}`; +} + +function calendarHourKey(timestamp: number): string { + const d = new Date(timestamp); + return `${calendarDayKey(timestamp)}-${d.getHours()}`; +} + +function weekKey(timestamp: number): string { + return `w-${Math.floor(timestamp / WEEK_MS)}`; +} + +function selectLiveEntries( + fileEntries: Array<{ entry: IndexEntry; packKey: string }>, + options: { cutoff: number; maxVersions: number; now?: number; tiered?: boolean }, +): { + kept: Array<{ entry: IndexEntry; packKey: string }>; + pruned: Array<{ entry: IndexEntry; packKey: string }>; +} { + const { cutoff, maxVersions, now = Date.now(), tiered = true } = options; + fileEntries.sort((a, b) => b.entry.lamport - a.entry.lamport || b.entry.at - a.entry.at); + + if (!tiered) { + const kept: Array<{ entry: IndexEntry; packKey: string }> = []; + const pruned: Array<{ entry: IndexEntry; packKey: string }> = []; + let count = 0; + for (const item of fileEntries) + if (item.entry.at >= cutoff && count < maxVersions) { + count++; + kept.push(item); + } else pruned.push(item); + return { kept, pruned }; + } + + const valid: Array<{ entry: IndexEntry; packKey: string }> = []; + const pruned: Array<{ entry: IndexEntry; packKey: string }> = []; + for (const item of fileEntries) + if (item.entry.at >= cutoff) valid.push(item); + else pruned.push(item); + + const seenBuckets = new Set(); + const tieredCandidates: Array<{ entry: IndexEntry; packKey: string }> = []; + const tieredPruned: Array<{ entry: IndexEntry; packKey: string }> = []; + + for (const item of valid) { + const age = Math.max(0, now - item.entry.at); + if (age < TIER_RECENT_CUTOFF_MS) tieredCandidates.push(item); + else if (age < TIER_HOURLY_CUTOFF_MS) { + const bucket = `h-${calendarHourKey(item.entry.at)}`; + if (seenBuckets.has(bucket)) tieredPruned.push(item); + else { + seenBuckets.add(bucket); + tieredCandidates.push(item); + } + } else if (age < TIER_DAILY_CUTOFF_MS) { + const bucket = `d-${calendarDayKey(item.entry.at)}`; + if (seenBuckets.has(bucket)) tieredPruned.push(item); + else { + seenBuckets.add(bucket); + tieredCandidates.push(item); + } + } else { + const bucket = weekKey(item.entry.at); + if (seenBuckets.has(bucket)) tieredPruned.push(item); + else { + seenBuckets.add(bucket); + tieredCandidates.push(item); + } + } + } + + if (tieredCandidates.length <= maxVersions) + return { kept: tieredCandidates, pruned: [...pruned, ...tieredPruned] }; + + const historical: Array<{ entry: IndexEntry; packKey: string }> = []; + const recent: Array<{ entry: IndexEntry; packKey: string }> = []; + for (const item of tieredCandidates) + if (Math.max(0, now - item.entry.at) >= TIER_HOURLY_CUTOFF_MS) historical.push(item); + else recent.push(item); + + let kept: Array<{ entry: IndexEntry; packKey: string }>; + let extraPruned: Array<{ entry: IndexEntry; packKey: string }>; + + if (historical.length < maxVersions) { + const remainingQuota = maxVersions - historical.length; + const keptRecent = recent.slice(0, remainingQuota); + const droppedRecent = recent.slice(remainingQuota); + kept = [...keptRecent, ...historical]; + extraPruned = droppedRecent; + } else { + kept = tieredCandidates.slice(0, maxVersions); + extraPruned = tieredCandidates.slice(maxVersions); + } + + return { kept, pruned: [...pruned, ...tieredPruned, ...extraPruned] }; +} + +function isMissingError(error: unknown): boolean { + if (isNotFound(error)) return true; + const msg = String(error).toLowerCase(); + return msg.includes('nosuchkey') || msg.includes('missing') || msg.includes('not found'); +} + +function packObjectKey(remote: RemoteHistory, pack: PackRef): string { + return `${remote.ns}/packs/${pack.deviceId}/p-${String(pack.seq).padStart(8, '0')}-${pack.hash}.pack`; +} + +/** Collects every chunk hash still referenced by commit entries newer than the retention cutoff. */ +async function liveSet( + ctx: GcContext, + options: GcOptions, +): Promise<{ + decodedPacks: Map; + entries: Array; + hashes: Set; + obsoletePacks: Map; +}> { + const retentionDays = options.retentionDays ?? DEFAULT_RETENTION_DAYS; + const maxVersions = options.maxVersions ?? DEFAULT_MAX_VERSIONS; + const cutoff = Date.now() - retentionDays * DAY_MS; + + const packMap = new Map }>(); + for (const [key, location] of await allCommitLocations(ctx.stores)) { + const pKey = packRefKey(location.pack); + let item = packMap.get(pKey); + if (!item) { + item = { pack: location.pack, versionKeys: [] }; + packMap.set(pKey, item); + } + item.versionKeys.push(key); + } + + const decodedPacks = new Map(); + for (const [pKey, item] of packMap) + try { + const bytes = await ctx.remote.getObject(packObjectKey(ctx.remote, item.pack)); + const decoded = await decodePack(bytes); + decodedPacks.set(pKey, decoded); + } catch (error) { + if (isMissingError(error)) { + for (const deadKey of item.versionKeys) await ctx.stores.index.delete(deadKey); + continue; + } + throw error; + } + + const byKey = new Map>(); + for (const [pKey, decoded] of decodedPacks) + for (const entry of decoded.entries) { + const list = byKey.get(entry.key) ?? []; + list.push({ entry, packKey: pKey }); + byKey.set(entry.key, list); + } + + const hashes = new Set(); + const entries: Array = []; + const livePackKeys = new Set(); + const prunedEntries: Array = []; + + for (const [, fileEntries] of byKey) { + const { kept, pruned } = selectLiveEntries(fileEntries, { + cutoff, + maxVersions, + now: options.now, + tiered: options.tiered !== false, + }); + for (const item of kept) { + entries.push(item.entry); + livePackKeys.add(item.packKey); + for (const chunk of item.entry.chunks) hashes.add(chunk.hash); + } + for (const item of pruned) prunedEntries.push(item.entry); + } + + const obsoletePacks = new Map(); + for (const [pKey, item] of packMap) + if (!livePackKeys.has(pKey)) obsoletePacks.set(pKey, item.pack); + + for (const entry of prunedEntries) + await ctx.stores.index.delete(`${entry.key}@${entry.lamport}`); + + for (const [key, location] of await allCommitLocations(ctx.stores)) + if (obsoletePacks.has(packRefKey(location.pack))) await ctx.stores.index.delete(key); + + return { decodedPacks, entries, hashes, obsoletePacks }; +} + +/** + * Client-side compaction. Live chunks are rewritten into a new immutable pack (checkpoint entries), + * then obsolete packs are deleted. A pack is never mutated in place. Re-running is idempotent. + */ +export async function compact(ctx: GcContext, options: number | GcOptions = {}): Promise { + const opts: GcOptions = typeof options === 'number' ? { retentionDays: options } : options; + const { decodedPacks, entries, hashes, obsoletePacks } = await liveSet(ctx, opts); + if (entries.length === 0) return 0; + + const locations = new Map( + (await allChunkLocations(ctx.stores)).map(([hash, loc]) => [hash, loc]), + ); + const payloadParts: Array = []; + const chunkLocations: Array<{ hash: string; location: ChunkLocation }> = []; + const log = (await ctx.stores.heads.get(ctx.remote.deviceId)) ?? (await freshLog(ctx)); + const seq = log.seq + 1; + const newRefBase: PackRef = { + deviceId: ctx.remote.deviceId, + fromLamport: Math.min(...entries.map((e) => e.lamport)), + hash: '', + seq, + toLamport: Math.max(...entries.map((e) => e.lamport)), + }; + let offset = 0; + for (const hash of hashes) { + const location = locations.get(hash); + if (!location) continue; + const pKey = packRefKey(location.pack); + let decoded = decodedPacks.get(pKey); + if (!decoded) + try { + const bytes = await ctx.remote.getObject(packObjectKey(ctx.remote, location.pack)); + decoded = await decodePack(bytes); + decodedPacks.set(pKey, decoded); + } catch (error) { + if (isMissingError(error)) continue; + throw error; + } + const chunk = decoded.payload.subarray(location.offset, location.offset + location.length); + payloadParts.push(new Uint8Array(chunk)); + chunkLocations.push({ + hash, + location: { length: chunk.byteLength, offset, pack: newRefBase }, + }); + offset += chunk.byteLength; + } + const payload = concat(payloadParts); + const checkpointEntries: Array = []; + for (const entry of entries) { + const chunks = []; + for (const chunk of entry.chunks) chunks.push({ ...chunk }); + checkpointEntries.push({ ...entry, chunks, flags: entry.flags | FLAG_CHECKPOINT }); + } + const encoded = await encodePack( + { + baseLamport: newRefBase.fromLamport, + createdAt: Date.now(), + deviceId: ctx.remote.deviceId, + ns: ctx.remote.ns, + peakLamport: newRefBase.toLamport, + seq, + }, + checkpointEntries, + payload, + ); + const realRef: PackRef = { ...newRefBase, hash: encoded.header.hash }; + for (const location of chunkLocations) location.location.pack = realRef; + await ctx.remote.putPack(seq, encoded.header.hash, encoded.bytes); + const nextLog = await buildCommitLog({ + chunkIndex: {}, + deviceId: ctx.remote.deviceId, + generation: log.generation + 1, + heads: log.heads, + ns: ctx.remote.ns, + packs: [realRef].slice(-MAX_PACK_WINDOW), + peakLamport: realRef.toLamport, + seq, + updatedAt: Date.now(), + }); + for (const location of chunkLocations) nextLog.chunkIndex[location.hash] = packRefKey(realRef); + await ctx.remote.writeHead(nextLog); + await ctx.stores.heads.set(ctx.remote.deviceId, nextLog); + await markChunks(ctx.stores, chunkLocations); + let removed = 0; + for (const pack of obsoletePacks.values()) + try { + await ctx.remote.deletePack( + pack.deviceId, + packFileName(pack.deviceId, pack.seq, pack.hash), + ); + removed++; + } catch (error) { + if (isMissingError(error)) continue; + throw error; + } + return removed; +} + +async function freshLog(ctx: GcContext): Promise { + return buildCommitLog({ + chunkIndex: {}, + deviceId: ctx.remote.deviceId, + generation: 0, + heads: [], + ns: ctx.remote.ns, + packs: [], + peakLamport: 0, + seq: 0, + updatedAt: Date.now(), + }); +} + +function concat(parts: Array): Uint8Array { + let total = 0; + for (const part of parts) total += part.byteLength; + const out = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.byteLength; + } + return out; +} diff --git a/packages/history/src/head-log.ts b/packages/history/src/head-log.ts new file mode 100644 index 00000000..593293ab --- /dev/null +++ b/packages/history/src/head-log.ts @@ -0,0 +1,43 @@ +import { sha256Text } from '@repo/shared/sha256-bytes'; +import type { CommitLog } from '@/types'; + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +type LogWithoutIntegrity = Omit; + +function canonical(log: LogWithoutIntegrity): string { + return JSON.stringify({ + chunkIndex: log.chunkIndex, + deviceId: log.deviceId, + generation: log.generation, + heads: log.heads, + ns: log.ns, + packs: log.packs, + peakLamport: log.peakLamport, + seq: log.seq, + updatedAt: log.updatedAt, + }); +} + +/** Builds a head log with a fresh integrity hash. */ +export async function buildCommitLog(log: LogWithoutIntegrity): Promise { + return { ...log, integrity: await sha256Text(canonical(log)) }; +} + +export function encodeHeadLog(log: CommitLog): Uint8Array { + return encoder.encode(JSON.stringify(log)); +} + +/** Decodes and verifies a head log. Returns undefined when the integrity hash does not match. */ +export async function decodeHeadLog(bytes: Uint8Array): Promise { + let log: CommitLog; + try { + log = JSON.parse(decoder.decode(bytes)) as CommitLog; + } catch { + return undefined; + } + const { integrity, ...rest } = log; + const expected = await sha256Text(canonical(rest)); + return expected === integrity ? log : undefined; +} diff --git a/packages/history/src/i18n.ts b/packages/history/src/i18n.ts new file mode 100644 index 00000000..cda00650 --- /dev/null +++ b/packages/history/src/i18n.ts @@ -0,0 +1,224 @@ +export type HistoryTranslations = { + autoGc: string; + autoGcDescription: string; + cancel: string; + confirmRestore: (label: string) => string; + confirmRestoreButton: string; + currentVersionBadge: string; + gc: string; + gcDescription: string; + gcDone: (count: number) => string; + history: string; + historyTitle: (file: string) => string; + loadFailed: (message: string) => string; + maxVersions: string; + maxVersionsDescription: string; + noDeletedNotes: string; + noVersions: string; + previewHint: string; + previewHintMobile: string; + protectExcludedNotes: string; + protectExcludedNotesDescription: string; + recoverDeleted: string; + recoverDeletedTitle: string; + restore: string; + restoreFailed: (message: string) => string; + restored: (label: string) => string; + retentionDays: string; + retentionDaysDescription: string; + runGc: string; + showHistory: string; + tieredRetention: string; + tieredRetentionDescription: string; + trackAttachments: string; + trackAttachmentsDescription: string; + version: (args: { at: number }) => string; + viewDiff: string; + viewDiffShort: string; + viewFull: string; + viewFullShort: string; +}; + +const en: HistoryTranslations = { + autoGc: 'Automatic garbage collection', + autoGcDescription: + 'Periodically clean up obsolete packs and pruned versions in the background.', + cancel: 'Cancel', + confirmRestore: (label) => + `Replace the current file with the version from ${label}? This cannot be undone.`, + confirmRestoreButton: 'Confirm restore', + currentVersionBadge: '(current)', + gc: 'Garbage collection', + gcDescription: + 'Rewrite live chunks into a new pack and remove obsolete packs. Packs are never modified in place.', + gcDone: (count) => `Removed ${count} obsolete pack(s).`, + history: 'History versions', + historyTitle: (file) => `Sync Engine history — ${file}`, + loadFailed: (message) => `Could not load this version: ${message}`, + maxVersions: 'Maximum saved versions', + maxVersionsDescription: + 'Maximum number of recent versions to keep per file during garbage collection.', + noDeletedNotes: 'No deleted notes with historical versions found.', + noVersions: 'No versions captured for this file yet.', + previewHint: 'Select a version on the left to preview its content.', + previewHintMobile: 'Select a version above to preview its content.', + protectExcludedNotes: 'Protect excluded notes', + protectExcludedNotesDescription: + 'Track history versions for notes excluded from sync by sync rules (does not apply to attachments).', + recoverDeleted: 'Recover deleted note', + recoverDeletedTitle: 'Recover deleted notes', + restore: 'Restore this version', + restoreFailed: (message) => `Restore failed: ${message}`, + restored: (label) => `Restored the version from ${label}.`, + retentionDays: 'Retention days', + retentionDaysDescription: + 'Versions older than this many days are dropped during garbage collection.', + runGc: 'Run garbage collection', + showHistory: 'Sync Engine: Show history', + tieredRetention: 'Tiered retention', + tieredRetentionDescription: + 'Gradually thin older versions (frequent recent versions, daily for past month, weekly beyond) so rapid editing does not crowd out earlier history.', + trackAttachments: 'Track attachment history', + trackAttachmentsDescription: + 'Save historical versions for synced attachments and binary files (increases remote storage usage).', + version: ({ at }) => (at > 0 ? new Date(at).toLocaleString() : 'Time unknown'), + viewDiff: 'Diff with current', + viewDiffShort: 'Diff', + viewFull: 'Full text', + viewFullShort: 'Full', +}; + +const zh: HistoryTranslations = { + autoGc: '自动垃圾回收', + autoGcDescription: '定期在后台自动清理过时包和已淘汰版本。', + cancel: '取消', + confirmRestore: (label) => `用 ${label} 的版本覆盖当前文件?此操作无法撤销。`, + confirmRestoreButton: '确认恢复', + currentVersionBadge: '(当前版本)', + gc: '垃圾回收', + gcDescription: '将仍在使用的块重写进新包并删除过时包,绝不原地修改已有包。', + gcDone: (count) => `已清理 ${count} 个过时包。`, + history: '历史版本', + historyTitle: (file) => `Sync Engine 历史 — ${file}`, + loadFailed: (message) => `无法加载该版本:${message}`, + maxVersions: '最大保存版本数', + maxVersionsDescription: '垃圾回收时每个文件保留的最新版本数量上限。', + noDeletedNotes: '未找到含有历史版本的已删除笔记。', + noVersions: '该文件尚未捕获任何版本。', + previewHint: '在左侧选择一个版本以预览内容。', + previewHintMobile: '在上方选择一个版本以预览内容。', + protectExcludedNotes: '保护被排除同步的笔记', + protectExcludedNotesDescription: '为被同步规则排除的文本笔记保留历史版本(不包含附件)。', + recoverDeleted: '恢复已删除笔记', + recoverDeletedTitle: '恢复已删除笔记', + restore: '恢复此版本', + restoreFailed: (message) => `恢复失败:${message}`, + restored: (label) => `已恢复 ${label} 的版本。`, + retentionDays: '保留天数', + retentionDaysDescription: '垃圾回收会丢弃超过该天数的旧版本。', + runGc: '运行垃圾回收', + showHistory: 'Sync Engine:查看历史', + tieredRetention: '分层保留策略', + tieredRetentionDescription: + '按时间逐步稀疏化保留旧版本(近期高频、单月内每天一个、往后每周一个),防止短时间连续编辑挤占早期历史。', + trackAttachments: '记录附件历史版本', + trackAttachmentsDescription: + '为库内正常同步的附件及二进制文件保存历史版本(会增加云端存储占用)。', + version: ({ at }) => (at > 0 ? new Date(at).toLocaleString() : '时间未知'), + viewDiff: '对比当前版本', + viewDiffShort: '对比', + viewFull: '查看全文', + viewFullShort: '全文', +}; + +const zhTW: HistoryTranslations = { + autoGc: '自動垃圾回收', + autoGcDescription: '定期在背景自動清理過時套件與已淘汰版本。', + cancel: '取消', + confirmRestore: (label) => `用 ${label} 的版本覆寫目前檔案?此動作無法復原。`, + confirmRestoreButton: '確認還原', + currentVersionBadge: '(目前版本)', + gc: '垃圾回收', + gcDescription: '將仍在使用的區塊重寫進新套件並刪除過時套件,絕不原地修改已有套件。', + gcDone: (count) => `已清理 ${count} 個過時套件。`, + history: '歷史版本', + historyTitle: (file) => `Sync Engine 歷史 — ${file}`, + loadFailed: (message) => `無法載入該版本:${message}`, + maxVersions: '最大儲存版本數', + maxVersionsDescription: '垃圾回收時每個檔案保留的最新版本數量上限。', + noDeletedNotes: '未找到含有歷史版本的已刪除筆記。', + noVersions: '該檔案尚未擷取任何版本。', + previewHint: '在左側選擇一個版本以預覽內容。', + previewHintMobile: '在上方選擇一個版本以預覽內容。', + protectExcludedNotes: '保護被排除同步的筆記', + protectExcludedNotesDescription: '為被同步規則排除的文字筆記保留歷史版本(不包含附件)。', + recoverDeleted: '還原已刪除筆記', + recoverDeletedTitle: '還原已刪除筆記', + restore: '還原此版本', + restoreFailed: (message) => `還原失敗:${message}`, + restored: (label) => `已還原 ${label} 的版本。`, + retentionDays: '保留天數', + retentionDaysDescription: '垃圾回收會捨棄超過此天數的舊版本。', + runGc: '執行垃圾回收', + showHistory: 'Sync Engine:查看歷史', + tieredRetention: '分層保留策略', + tieredRetentionDescription: + '按時間逐步稀疏化保留舊版本(近期高頻、單月內每天一個、往後每週一個),防止短時間連續編輯擠佔早期歷史。', + trackAttachments: '記錄附件歷史版本', + trackAttachmentsDescription: + '為庫內正常同步的附件及二進位檔案儲存歷史版本(會增加雲端儲存空間佔用)。', + version: ({ at }) => (at > 0 ? new Date(at).toLocaleString() : '時間未知'), + viewDiff: '對比目前版本', + viewDiffShort: '對比', + viewFull: '檢視全文', + viewFullShort: '全文', +}; + +const ru: HistoryTranslations = { + autoGc: 'Автоматическая сборка мусора', + autoGcDescription: 'Периодически очищать устаревшие пакеты в фоновом режиме.', + cancel: 'Отмена', + confirmRestore: (label) => + `Заменить текущий файл версией от ${label}? Это действие нельзя отменить.`, + confirmRestoreButton: 'Подтвердить восстановление', + currentVersionBadge: '(текущая)', + gc: 'Сборка мусора', + gcDescription: + 'Перезаписать используемые блоки в новый пакет и удалить устаревшие. Пакеты никогда не изменяются на месте.', + gcDone: (count) => `Удалено устаревших пакетов: ${count}.`, + history: 'История версий', + historyTitle: (file) => `Sync Engine: история — ${file}`, + loadFailed: (message) => `Не удалось загрузить версию: ${message}`, + maxVersions: 'Максимальное количество версий', + maxVersionsDescription: + 'Максимальное число последних версий для каждого файла, сохраняемых при сборке мусора.', + noDeletedNotes: 'Удаленных заметок с историей версий не найдено.', + noVersions: 'Для этого файла ещё нет версий.', + previewHint: 'Выберите версию слева, чтобы просмотреть содержимое.', + previewHintMobile: 'Выберите версию выше, чтобы просмотреть содержимое.', + protectExcludedNotes: 'Защита исключенных заметок', + protectExcludedNotesDescription: + 'Сохранять историю версий для текстовых заметок, исключенных из синхронизации (не распространяется на вложения).', + recoverDeleted: 'Восстановить удаленную заметку', + recoverDeletedTitle: 'Восстановление удаленных заметок', + restore: 'Восстановить эту версию', + restoreFailed: (message) => `Не удалось восстановить: ${message}`, + restored: (label) => `Восстановлена версия от ${label}.`, + retentionDays: 'Срок хранения (дни)', + retentionDaysDescription: 'Версии старше этого числа дней удаляются при сборке мусора.', + runGc: 'Запустить сборку мусора', + showHistory: 'Sync Engine: показать историю', + tieredRetention: 'Многоуровневое хранение', + tieredRetentionDescription: + 'Постепенно прореживать старые версии (частые недавние, ежедневные за последний месяц, еженедельные ранее), чтобы частое редактирование не вытесняло историю.', + trackAttachments: 'История версий для вложений', + trackAttachmentsDescription: + 'Сохранять историю версий для синхронизируемых вложений и бинарных файлов (увеличивает объем используемого хранилища).', + version: ({ at }) => (at > 0 ? new Date(at).toLocaleString() : 'Время неизвестно'), + viewDiff: 'Сравнить с текущей', + viewDiffShort: 'Дифф', + viewFull: 'Полный текст', + viewFullShort: 'Текст', +}; + +export { en, ru, zh, zhTW }; diff --git a/packages/history/src/index-store.ts b/packages/history/src/index-store.ts new file mode 100644 index 00000000..650b8cc6 --- /dev/null +++ b/packages/history/src/index-store.ts @@ -0,0 +1,56 @@ +import type { HistoryStores } from '@/store'; +import type { ChunkLocation, CommitLocation, PackRef, VersionRef } from '@/types'; + +export function versionKey(ref: VersionRef): string { + return ref.commitHash ?? `${ref.key}@${ref.lamport ?? ''}`; +} + +/** Records where a commit lives so restore is an O(1) local lookup. */ +export async function putCommitLocation( + stores: HistoryStores, + ref: string, + location: CommitLocation, +): Promise { + await stores.index.set(ref, location); +} + +export async function lookupCommit( + stores: HistoryStores, + ref: VersionRef, +): Promise { + return stores.index.get(versionKey(ref)); +} + +/** Confirms that a chunk is present remotely, enabling cross-version/cross-note dedup. */ +export async function markChunks( + stores: HistoryStores, + chunks: Array<{ hash: string; location: ChunkLocation }>, +): Promise { + await stores.chunks.batch( + chunks.map(({ hash, location }) => ({ key: hash, type: 'set' as const, value: location })), + ); +} + +export async function chunkLocation( + stores: HistoryStores, + hash: string, +): Promise { + return stores.chunks.get(hash); +} + +/** Reads every known commit location (used by GC and the version picker). */ +export async function allCommitLocations( + stores: HistoryStores, +): Promise> { + return stores.index.entries(); +} + +export async function allChunkLocations( + stores: HistoryStores, +): Promise> { + return stores.chunks.entries(); +} + +export function packRefKey(pack: PackRef): string { + return `${pack.deviceId}:${pack.seq}:${pack.hash}`; +} diff --git a/packages/history/src/index.ts b/packages/history/src/index.ts new file mode 100644 index 00000000..6fae3088 --- /dev/null +++ b/packages/history/src/index.ts @@ -0,0 +1,465 @@ +import { isTextFile } from '@/chunker'; +import { + AUTO_GC_COOLDOWN_MS, + DEFAULT_MAX_VERSIONS, + DEFAULT_RETENTION_DAYS, + NON_TEXT_DEBOUNCE_MS, + NON_TEXT_MAX_WAIT_MS, + TEXT_DEBOUNCE_MS, + TEXT_MAX_WAIT_MS, +} from '@/constants'; +import { createHistoryCipher } from '@/crypto'; +import { FlushScheduler, reconcile } from '@/flush'; +import { compact } from '@/gc'; +import { en, ru, zh, zhTW } from '@/i18n'; +import { allCommitLocations } from '@/index-store'; +import HistoryModal, { RecoverDeletedModal } from '@/modal'; +import { openRemoteHistory } from '@/remote'; +import { listVersions, restoreVersion } from '@/restore'; +import historySetting from '@/setting'; +import { getOrCreateDeviceId, openHistoryStores, readLastGcTime, saveLastGcTime } from '@/store'; +import { + findDeletedKeys, + isRemoteWriteTask, + namespaceOf, + readCurrentText, + readFileBytes, + text, + writeVaultFile, +} from '@/vault'; +import { ingestDeletion, ingestEdit } from '@/wal'; +import { NONE_STRATEGY, prepareGlobMatch } from '@repo/shared/glob-match'; +import { sha256Bytes } from '@repo/shared/sha256-bytes'; +import { Notice } from 'obsidian'; +import type { HistoryCipher } from '@/crypto'; +import type { RemoteHistory } from '@/remote'; +import type { HistorySettings } from '@/setting'; +import type { HistoryStores } from '@/store'; +import type { HistoryContext, PendingDebounce } from '@/types'; +import type { Fs } from '@hesprs/sync-engine-sdk'; +import type { Menu, TAbstractFile, TFile } from 'obsidian'; + +export default class HistoryModule { + private readonly cleanup: Array<() => void> = []; + private readonly pendingDebounces = new Map(); + private readonly lastIngestedHash = new Map(); + private readonly syncWrittenFiles = new Set(); + private commitQueue = Promise.resolve(); + private cachedRulesKey = ''; + private compiledMatcher?: (path: string) => { strategy: string }; + private stores?: HistoryStores; + private storesNamespace?: string; + private remote?: RemoteHistory; + private remoteFsUid?: string; + private scheduler?: FlushScheduler; + private deviceId = ''; + + moduleSettings: HistorySettings = { + autoGc: true, + maxVersions: DEFAULT_MAX_VERSIONS, + protectExcludedNotes: false, + retentionDays: DEFAULT_RETENTION_DAYS, + tieredRetention: true, + trackAttachments: false, + }; + + constructor(private readonly ctx: HistoryContext) { + ctx.registerI18n('en', en); + ctx.registerI18n('zh', zh); + ctx.registerI18n('zh-TW', zhTW); + ctx.registerI18n('ru', ru); + } + + readonly start = () => { + const { ctx } = this; + // No namespace yet means no backend: defer opening stores to syncInitialized so no empty-id store is created. + const namespace = namespaceOf(ctx); + if (namespace !== undefined) this.openStores(namespace); + ctx.registerEvent(ctx.app.vault.on('create', this.onVaultCreate)); + ctx.registerEvent(ctx.app.vault.on('modify', this.onVaultModify)); + ctx.registerEvent(ctx.app.vault.on('delete', this.onVaultDelete)); + ctx.registerEvent(ctx.app.vault.on('rename', this.onVaultRename)); + ctx.registerEvent(ctx.app.workspace.on('file-menu', this.onFileMenu)); + ctx.addCommand({ + callback: () => void this.openRecoverDeletedModal().catch(() => {}), + id: 'history-recover-deleted', + name: ctx.translate('recoverDeleted'), + }); + this.cleanup.push( + ctx.on('syncStarted', () => { + this.syncWrittenFiles.clear(); + void this.flushPendingDebounces(); + }), + ctx.on('executionStarted', (tasks) => { + this.syncWrittenFiles.clear(); + for (const task of tasks) + if (!task.local?.isDir && isRemoteWriteTask(task.name)) + this.syncWrittenFiles.add(task.key); + }), + ctx.on('syncInitialized', ({ remoteFs }) => { + void this.initializeRemote(remoteFs); + }), + ctx.on('syncTerminated', () => { + const hadRemoteWrites = this.syncWrittenFiles.size > 0; + this.syncWrittenFiles.clear(); + if (hadRemoteWrites && this.remote && this.stores) + void reconcile({ + remote: this.remote, + requestSync: () => {}, + stores: this.stores, + }).catch(() => {}); + + void this.checkAutoGc(); + }), + ctx.registerSetting({ + apply: historySetting( + { saveSettings: ctx.saveSettings, translate: ctx.translate }, + this.moduleSettings, + { runGc: async () => this.runGc() }, + ), + priority: 3055, + }), + ); + const onVisibility = () => { + if (document.visibilityState === 'hidden') + void this.flushPendingDebounces().then(() => this.scheduler?.onUnload()); + }; + if (typeof window !== 'undefined') { + window.addEventListener('visibilitychange', onVisibility); + const onBeforeUnload = () => void this.flushPendingDebounces(); + window.addEventListener('beforeunload', onBeforeUnload); + this.cleanup.push(() => { + window.removeEventListener('visibilitychange', onVisibility); + window.removeEventListener('beforeunload', onBeforeUnload); + }); + } + }; + + /** Opens stores for a namespace once and migrates legacy settings into them. */ + private openStores(namespace: string): HistoryStores { + if (this.stores && this.storesNamespace === namespace) return this.stores; + const stores = openHistoryStores(this.ctx.indexedDB, namespace); + this.stores = stores; + this.storesNamespace = namespace; + void getOrCreateDeviceId(stores, this.moduleSettings.deviceId).then((id) => { + this.deviceId = id; + }); + const { deviceId, lastGcTime } = this.moduleSettings; + if (deviceId !== undefined || lastGcTime !== undefined) { + delete this.moduleSettings.deviceId; + delete this.moduleSettings.lastGcTime; + if (lastGcTime) void saveLastGcTime(stores, lastGcTime); + void this.ctx.saveSettings(); + } + return stores; + } + + /** + * Derives a content cipher when the encryption module is active, then opens the isolated + * history handle. If encryption is on but its password is missing we deliberately keep + * `this.remote` unset so no plaintext history can be written. + */ + private async initializeRemote(remoteFs: Fs): Promise { + const fsUid = remoteFs.getUid(); + if (this.remote && this.remoteFsUid === fsUid) return; + const { ctx } = this; + const namespace = ctx.getNamespace(undefined, remoteFs); + const stores = this.openStores(namespace); + let cipher: HistoryCipher | undefined; + try { + cipher = await createHistoryCipher(ctx.settings.modules, ctx.app); + } catch (error) { + new Notice(`History encryption: ${String(error)}`); + return; + } + const deviceId = + this.deviceId || (await getOrCreateDeviceId(stores, this.moduleSettings.deviceId)); + this.deviceId = deviceId; + const remote = openRemoteHistory(remoteFs, namespace, deviceId, cipher); + if (!remote) return; + this.remote = remote; + this.remoteFsUid = fsUid; + this.scheduler?.dispose(); + this.scheduler = new FlushScheduler({ + getRealtimeSyncConfig: () => ctx.settings.realtimeSync, + remote, + requestSync: (trigger) => void ctx.requestSync(trigger), + stores, + }); + void reconcile({ remote, requestSync: () => {}, stores }).catch(() => {}); + void this.checkAutoGc(); + } + + private isSyncIgnored(filePath: string): boolean { + const syncStrategy = this.ctx.settings.syncStrategy; + if (!syncStrategy || syncStrategy.length === 0) return false; + const rulesKey = syncStrategy.map((rule) => `${rule.expr}:${rule.strategy}`).join('|'); + if (this.cachedRulesKey !== rulesKey || !this.compiledMatcher) { + this.cachedRulesKey = rulesKey; + this.compiledMatcher = prepareGlobMatch(syncStrategy); + } + return this.compiledMatcher(filePath).strategy === NONE_STRATEGY; + } + + private isHistoryIgnored(filePath: string): boolean { + const isText = isTextFile(filePath); + if (this.isSyncIgnored(filePath)) + return !(isText && this.moduleSettings.protectExcludedNotes); + return !isText && !this.moduleSettings.trackAttachments; + } + + /** Adds a per-file "Show history" item to the file explorer context menu. */ + private readonly onFileMenu = (menu: Menu, file: TAbstractFile) => { + if ('children' in file || this.isHistoryIgnored(file.path)) return; + menu.addItem((item) => { + item.setTitle(this.ctx.translate('showHistory')) + .setIcon('history') + .onClick(() => void this.openHistoryForPath(file.path)); + }); + }; + + private async openHistoryForPath(filePath: string): Promise { + if (!this.remote || !this.stores || this.isHistoryIgnored(filePath)) return; + const ctx = { remote: this.remote, requestSync: () => {}, stores: this.stores }; + let versions; + try { + await this.flushPendingForPath(filePath); + await this.scheduler?.trigger('openHistory'); + await reconcile(ctx); + versions = await listVersions(ctx, filePath); + } catch (error) { + new Notice(text(this.ctx.translate('loadFailed', String(error)))); + return; + } + const currentText = await readCurrentText(this.ctx.app, filePath); + const isText = isTextFile(filePath); + + new HistoryModal( + { app: this.ctx.app, translate: this.ctx.translate }, + { path: filePath }, + versions, + { + currentText, + isBinary: !isText, + load: async (key, lamport) => { + const result = await restoreVersion(ctx, { key, lamport }); + return { binary: result.binary, text: result.text }; + }, + restore: (textContent, binaryContent) => + writeVaultFile(this.ctx.app, filePath, textContent, binaryContent), + }, + ).open(); + } + + private async openRecoverDeletedModal(): Promise { + if (!this.stores || !this.remote) return; + await this.flushPendingDebounces(); + await reconcile({ remote: this.remote, requestSync: () => {}, stores: this.stores }); + const locations = await allCommitLocations(this.stores); + const deletedKeys = findDeletedKeys(this.ctx.app, locations, (key) => + this.isHistoryIgnored(key), + ); + new RecoverDeletedModal( + { app: this.ctx.app, translate: this.ctx.translate }, + deletedKeys, + (key) => { + void this.openHistoryForPath(key); + }, + ).open(); + } + + private commitEdit(key: string, data: Uint8Array, hash: string): Promise { + const next = this.commitQueue + .then(async () => { + if (!this.stores) return; + this.lastIngestedHash.set(key, hash); + await this.stores.hashes.set(key, hash); + const entry = await ingestEdit(this.stores, key, data); + this.scheduler?.onEdit( + entry.text?.length ?? + entry.newChunks?.length ?? + entry.blobData?.reduce((acc, b) => acc + b.bytes.byteLength, 0) ?? + 0, + ); + }) + .catch(() => {}); + this.commitQueue = next; + return next; + } + + private async flushPendingForPath(filePath: string): Promise { + if (!this.stores || this.isHistoryIgnored(filePath)) return; + const pending = this.pendingDebounces.get(filePath); + if (!pending) return; + clearTimeout(pending.timer); + this.pendingDebounces.delete(filePath); + const hash = await sha256Bytes(pending.data); + await this.commitEdit(filePath, pending.data, hash); + } + + private async flushPendingDebounces(): Promise { + if (!this.stores || this.pendingDebounces.size === 0) return; + const entries = [...this.pendingDebounces.entries()]; + this.pendingDebounces.clear(); + for (const [index, [key, pending]] of entries.entries()) { + clearTimeout(pending.timer); + if (this.isHistoryIgnored(key)) continue; + const hash = await sha256Bytes(pending.data); + void this.commitEdit(key, pending.data, hash); + if ((index + 1) % 10 === 0) + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); + } + await this.commitQueue; + } + + private queueIngest(filePath: string, data: Uint8Array): void { + if (!this.stores || this.isHistoryIgnored(filePath)) return; + const isText = isTextFile(filePath); + const debounceMs = isText ? TEXT_DEBOUNCE_MS : NON_TEXT_DEBOUNCE_MS; + const maxWaitMs = isText ? TEXT_MAX_WAIT_MS : NON_TEXT_MAX_WAIT_MS; + void sha256Bytes(data).then(async (hash) => { + if (!this.stores) return; + let lastHash = this.lastIngestedHash.get(filePath); + if (lastHash === undefined) { + lastHash = await this.stores.hashes.get(filePath); + if (lastHash !== undefined) this.lastIngestedHash.set(filePath, lastHash); + } + if (lastHash === hash) return; + const now = Date.now(); + const existing = this.pendingDebounces.get(filePath); + if (existing) { + clearTimeout(existing.timer); + if (now - existing.firstScheduledAt >= maxWaitMs) { + this.pendingDebounces.delete(filePath); + void this.commitEdit(filePath, data, hash); + return; + } + const timer = setTimeout(() => { + this.pendingDebounces.delete(filePath); + void this.commitEdit(filePath, data, hash); + }, debounceMs); + this.pendingDebounces.set(filePath, { + data, + firstScheduledAt: existing.firstScheduledAt, + timer, + }); + } else { + const timer = setTimeout(() => { + this.pendingDebounces.delete(filePath); + void this.commitEdit(filePath, data, hash); + }, debounceMs); + this.pendingDebounces.set(filePath, { data, firstScheduledAt: now, timer }); + } + }); + } + + private shouldIgnoreFile(file: TAbstractFile): boolean { + if ( + !this.stores || + 'children' in file || + this.isHistoryIgnored(file.path) || + this.syncWrittenFiles.has(file.path) + ) { + this.syncWrittenFiles.delete(file.path); + return true; + } + return false; + } + + private readonly onVaultCreate = (file: TAbstractFile) => { + if (this.shouldIgnoreFile(file)) return; + const filePath = file.path; + void readFileBytes(this.ctx.app, file as TFile).then( + (data) => { + if (!this.stores || data.byteLength === 0) return; + this.queueIngest(filePath, data); + }, + () => {}, + ); + }; + + private readonly onVaultModify = (file: TAbstractFile) => { + if (this.shouldIgnoreFile(file)) return; + const filePath = file.path; + void readFileBytes(this.ctx.app, file as TFile).then( + (data) => { + if (!this.stores) return; + this.queueIngest(filePath, data); + }, + () => {}, + ); + }; + + private readonly onVaultDelete = (file: TAbstractFile) => { + const { stores } = this; + if (!stores || this.shouldIgnoreFile(file)) return; + const filePath = file.path; + const pending = this.pendingDebounces.get(filePath); + if (pending) { + clearTimeout(pending.timer); + this.pendingDebounces.delete(filePath); + } + this.lastIngestedHash.delete(filePath); + void stores.hashes.delete(filePath); + void ingestDeletion(stores, filePath).then(() => { + this.scheduler?.onEdit(0); + }); + }; + + private readonly onVaultRename = (file: TAbstractFile, oldPath: string) => { + if (!this.stores || 'children' in file) return; + if (this.syncWrittenFiles.has(file.path)) { + this.syncWrittenFiles.delete(file.path); + return; + } + const pendingOld = this.pendingDebounces.get(oldPath); + if (pendingOld) { + clearTimeout(pendingOld.timer); + this.pendingDebounces.delete(oldPath); + } + this.lastIngestedHash.delete(oldPath); + void this.stores.hashes.delete(oldPath); + if (!this.isHistoryIgnored(file.path)) this.onVaultModify(file); + }; + + private async checkAutoGc(): Promise { + if (this.moduleSettings.autoGc === false || !this.remote || !this.stores) return; + const now = Date.now(); + const lastGc = await readLastGcTime(this.stores); + if (now - lastGc < AUTO_GC_COOLDOWN_MS) return; + try { + await this.runGc(); + } catch { + // Background auto GC error is non-fatal + } + } + + private async runGc(): Promise { + if (!this.remote || !this.stores) return 0; + await this.flushPendingDebounces(); + await reconcile({ remote: this.remote, requestSync: () => {}, stores: this.stores }); + const { maxVersions, retentionDays, tieredRetention } = this.moduleSettings; + const result = await compact( + { remote: this.remote, stores: this.stores }, + { maxVersions, retentionDays, tiered: tieredRetention !== false }, + ); + await saveLastGcTime(this.stores, Date.now()); + return result; + } + + readonly dispose = () => { + void this.flushPendingDebounces(); + void this.scheduler?.onUnload(); + for (const pending of this.pendingDebounces.values()) clearTimeout(pending.timer); + this.pendingDebounces.clear(); + this.syncWrittenFiles.clear(); + this.cleanup.splice(0).forEach((fn) => fn()); + this.scheduler?.dispose(); + this.stores = undefined; + this.storesNamespace = undefined; + this.remote = undefined; + this.remoteFsUid = undefined; + }; +} diff --git a/packages/history/src/modal.ts b/packages/history/src/modal.ts new file mode 100644 index 00000000..48bdc85f --- /dev/null +++ b/packages/history/src/modal.ts @@ -0,0 +1,468 @@ +import { computeLineDiff, renderDiffLines } from './diff'; +import { createMobileRoller } from './roller'; +import { Modal, Notice, Platform, Setting, TFile } from 'obsidian'; +import type { HistoryTranslations } from './i18n'; +import type { MobileRoller } from './roller'; +import type { VersionSummary } from './types'; +import type { Translate } from '@hesprs/sync-engine-sdk'; +import type { App, ButtonComponent } from 'obsidian'; + +export { computeLineDiff, type DiffLine } from './diff'; +export { default as RecoverDeletedModal } from './recover-deleted-modal'; + +export type HistoryModalActions = { + currentText?: string; + isBinary?: boolean; + load: (key: string, lamport: number) => Promise<{ binary?: Uint8Array; text: string }>; + restore: (text: string, binary?: Uint8Array) => Promise; +}; + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(2)} MB`; +} + +/** Translate returns DocumentFragment for function-valued keys; our translations yield strings. */ +function text(value: unknown): string { + return typeof value === 'string' ? value : String(value); +} + +/** Per-file history browser: a timestamp list, a visual diff / preview, and a confirmed restore. */ +export default class HistoryModal extends Modal { + private selected?: VersionSummary; + private selectedText = ''; + private selectedBinary?: Uint8Array; + private objectUrl?: string; + private currentText = ''; + private viewMode: 'diff' | 'full' = 'diff'; + private previewEl?: HTMLElement; + private confirmationEl?: HTMLElement; + private restoreButton?: ButtonComponent; + private fullBtn?: HTMLButtonElement; + private diffBtn?: HTMLButtonElement; + private diffStatsEl?: HTMLElement; + private listEl?: HTMLElement; + private mobileRoller?: MobileRoller; + private confirming = false; + + constructor( + private readonly ctx: { app: App; translate: Translate }, + private readonly file: TFile | { path: string }, + private readonly versions: Array, + private readonly actions: HistoryModalActions, + ) { + super(ctx.app); + this.setTitle(text(ctx.translate('historyTitle', file.path))); + if (actions.currentText !== undefined) this.currentText = actions.currentText; + if (actions.isBinary) this.viewMode = 'full'; + } + + onOpen(): void { + const { contentEl } = this; + contentEl.empty(); + contentEl.style.display = 'flex'; + contentEl.style.flexDirection = 'column'; + contentEl.style.gap = '12px'; + contentEl.style.minHeight = Platform.isMobile ? '0' : '420px'; + + if (!this.currentText) { + const abstractFile = this.ctx.app.vault.getAbstractFileByPath(this.file.path); + if (abstractFile instanceof TFile) + void this.ctx.app.vault.read(abstractFile).then((current) => { + this.currentText = current; + if (this.selected) this.renderPreview(); + }); + } + + if (this.versions.length === 0) { + contentEl.createEl('p', { text: text(this.ctx.translate('noVersions')) }); + return; + } + + let previewWrap: HTMLElement; + let firstItem: HTMLButtonElement | undefined; + const isMobile = Platform.isMobile; + + if (isMobile) { + previewWrap = contentEl.createEl('div'); + previewWrap.style.display = 'flex'; + previewWrap.style.flexDirection = 'column'; + previewWrap.style.flex = '1'; + previewWrap.style.minWidth = '0'; + previewWrap.style.minHeight = '0'; + } else { + const layout = contentEl.createEl('div'); + layout.style.display = 'flex'; + layout.style.flex = '1'; + layout.style.gap = '16px'; + layout.style.minHeight = '300px'; + + const list = layout.createEl('div'); + this.listEl = list; + list.style.display = 'flex'; + list.style.flexDirection = 'column'; + list.style.gap = '6px'; + list.style.width = '220px'; + list.style.flexShrink = '0'; + list.style.overflowY = 'auto'; + list.style.maxHeight = '55vh'; + list.style.paddingRight = '4px'; + + for (const version of this.versions) { + const item = list.createEl('button'); + item.style.width = '100%'; + item.style.textAlign = 'left'; + item.style.padding = '8px 12px'; + item.style.borderRadius = '6px'; + item.style.fontSize = '13px'; + item.style.cursor = 'pointer'; + item.style.border = '1px solid var(--background-modifier-border)'; + item.style.backgroundColor = 'var(--background-secondary)'; + item.style.color = 'var(--text-normal)'; + + item.createSpan({ text: this.label(version) }); + if (version.lamport === this.versions[0]?.lamport) { + const badge = item.createSpan({ + text: ` ${text(this.ctx.translate('currentVersionBadge'))}`, + }); + badge.style.fontSize = '11px'; + badge.style.color = 'var(--interactive-accent)'; + badge.style.fontWeight = 'normal'; + } + + item.addEventListener('click', () => void this.select(version, item, list)); + firstItem ??= item; + } + + previewWrap = layout.createEl('div'); + previewWrap.style.display = 'flex'; + previewWrap.style.flexDirection = 'column'; + previewWrap.style.flex = '1'; + previewWrap.style.minWidth = '0'; + } + + const toolbar = previewWrap.createEl('div'); + toolbar.style.display = 'flex'; + toolbar.style.alignItems = 'center'; + toolbar.style.gap = '8px'; + toolbar.style.marginBottom = '8px'; + + if (isMobile) + this.mobileRoller = createMobileRoller({ + container: toolbar, + currentBadgeText: text(this.ctx.translate('currentVersionBadge')), + label: (v) => this.label(v), + onSelect: (target) => void this.select(target), + onStep: (delta) => void this.stepVersion(delta), + versions: this.versions, + }); + + if (!this.actions.isBinary) { + const btnGroup = isMobile ? toolbar.createEl('div') : toolbar; + if (isMobile) { + btnGroup.style.display = 'flex'; + btnGroup.style.borderRadius = '6px'; + btnGroup.style.overflow = 'hidden'; + btnGroup.style.border = '1px solid var(--background-modifier-border)'; + btnGroup.style.flexShrink = '0'; + } + + this.diffBtn = btnGroup.createEl('button', { + text: text(this.ctx.translate(isMobile ? 'viewDiffShort' : 'viewDiff')), + }); + this.fullBtn = btnGroup.createEl('button', { + text: text(this.ctx.translate(isMobile ? 'viewFullShort' : 'viewFull')), + }); + + this.diffStatsEl = toolbar.createEl('div'); + this.diffStatsEl.style.fontSize = '12px'; + this.diffStatsEl.style.marginLeft = isMobile ? '0' : 'auto'; + this.diffStatsEl.style.fontFamily = 'var(--font-monospace, monospace)'; + this.diffStatsEl.style.flexShrink = '0'; + + this.updateToolbarButtons(); + + this.diffBtn.addEventListener('click', () => { + this.viewMode = 'diff'; + this.updateToolbarButtons(); + this.renderPreview(); + }); + this.fullBtn.addEventListener('click', () => { + this.viewMode = 'full'; + this.updateToolbarButtons(); + this.renderPreview(); + }); + } + + this.previewEl = previewWrap.createEl('div'); + this.previewEl.style.flex = '1'; + this.previewEl.style.minHeight = isMobile ? '0' : '240px'; + this.previewEl.style.maxHeight = isMobile ? 'none' : '55vh'; + this.previewEl.style.overflow = 'auto'; + this.previewEl.style.border = '1px solid var(--background-modifier-border)'; + this.previewEl.style.borderRadius = '6px'; + this.previewEl.style.backgroundColor = 'var(--background-primary)'; + this.previewEl.style.padding = '8px 0'; + this.previewEl.setText( + text(this.ctx.translate(isMobile ? 'previewHintMobile' : 'previewHint')), + ); + + this.confirmationEl = contentEl.createEl('div'); + this.confirmationEl.style.display = 'none'; + this.confirmationEl.style.padding = '8px 12px'; + this.confirmationEl.style.borderRadius = '4px'; + this.confirmationEl.style.backgroundColor = + 'var(--background-modifier-error, rgba(248, 81, 73, 0.15))'; + this.confirmationEl.style.color = 'var(--text-error, #f85149)'; + this.confirmationEl.style.border = '1px solid var(--text-error, #f85149)'; + this.confirmationEl.style.fontSize = '13px'; + + const actions = new Setting(contentEl).setClass('border-none'); + actions.addButton((button) => + button.setButtonText(text(this.ctx.translate('cancel'))).onClick(() => { + if (this.confirming) this.resetConfirmation(); + else this.close(); + }), + ); + actions.addButton((button) => { + button + .setButtonText(text(this.ctx.translate('restore'))) + .setCta() + .setDisabled(true); + button.onClick(() => void this.onRestoreClick()); + this.restoreButton = button; + }); + + const newest = this.versions[0]; + if (newest) void this.select(newest, firstItem, this.listEl); + } + + private updateToolbarButtons(): void { + const isDiff = this.viewMode === 'diff'; + const isMobile = Platform.isMobile; + const applyBtnStyle = (btn: HTMLButtonElement | undefined, active: boolean) => { + if (!btn) return; + btn.style.padding = isMobile ? '0 10px' : '4px 12px'; + btn.style.height = isMobile ? '32px' : 'auto'; + btn.style.fontSize = '12px'; + btn.style.borderRadius = isMobile ? '0' : '4px'; + btn.style.cursor = 'pointer'; + btn.style.border = isMobile ? 'none' : '1px solid var(--background-modifier-border)'; + if (active) { + btn.style.backgroundColor = 'var(--interactive-accent)'; + btn.style.color = 'var(--text-on-accent)'; + if (!isMobile) btn.style.borderColor = 'var(--interactive-accent)'; + } else { + btn.style.backgroundColor = 'var(--background-secondary)'; + btn.style.color = 'var(--text-muted)'; + } + }; + applyBtnStyle(this.diffBtn, isDiff); + applyBtnStyle(this.fullBtn, !isDiff); + } + + private label(version: VersionSummary): string { + return text(this.ctx.translate('version', { at: version.at })); + } + + onClose(): void { + if (this.objectUrl) { + URL.revokeObjectURL(this.objectUrl); + this.objectUrl = undefined; + } + } + + private renderPreview(): void { + if (!this.previewEl) return; + this.previewEl.empty(); + if (this.objectUrl) { + URL.revokeObjectURL(this.objectUrl); + this.objectUrl = undefined; + } + if (!this.selectedText && !this.selectedBinary && !this.selected) { + if (this.diffStatsEl) this.diffStatsEl.empty(); + this.previewEl.setText( + text(this.ctx.translate(Platform.isMobile ? 'previewHintMobile' : 'previewHint')), + ); + return; + } + + if (this.selectedBinary) { + if (this.diffStatsEl) this.diffStatsEl.empty(); + const wrap = this.previewEl.createEl('div'); + wrap.style.display = 'flex'; + wrap.style.flexDirection = 'column'; + wrap.style.alignItems = 'center'; + wrap.style.justifyContent = 'center'; + wrap.style.padding = '16px'; + wrap.style.gap = '12px'; + + const ext = this.file.path.split('.').pop()?.toLowerCase() ?? ''; + const imageExts = new Set(['png', 'jpg', 'jpeg', 'gif', 'svg', 'webp', 'bmp', 'ico']); + const audioExts = new Set(['mp3', 'wav', 'm4a', 'ogg']); + + const sizeLabel = formatBytes(this.selectedBinary.byteLength); + const info = wrap.createEl('div'); + info.style.fontSize = '12px'; + info.style.color = 'var(--text-muted)'; + info.style.fontFamily = 'var(--font-monospace, monospace)'; + info.setText(`${ext ? `.${ext.toUpperCase()}` : 'Binary'} • ${sizeLabel}`); + + if (imageExts.has(ext)) { + const mime = + ext === 'svg' ? 'image/svg+xml' : `image/${ext === 'jpg' ? 'jpeg' : ext}`; + const blob = new Blob([this.selectedBinary.buffer as ArrayBuffer], { type: mime }); + this.objectUrl = URL.createObjectURL(blob); + const img = wrap.createEl('img'); + img.src = this.objectUrl; + img.style.maxWidth = '100%'; + img.style.maxHeight = '40vh'; + img.style.objectFit = 'contain'; + img.style.borderRadius = '4px'; + img.style.border = '1px solid var(--background-modifier-border)'; + } else if (audioExts.has(ext)) { + const blob = new Blob([this.selectedBinary.buffer as ArrayBuffer]); + this.objectUrl = URL.createObjectURL(blob); + const audio = wrap.createEl('audio'); + audio.src = this.objectUrl; + audio.controls = true; + } + return; + } + + if (this.viewMode === 'full') { + if (this.diffStatsEl) this.diffStatsEl.empty(); + const pre = this.previewEl.createEl('pre'); + pre.style.margin = '0'; + pre.style.padding = '10px 14px'; + pre.style.whiteSpace = 'pre-wrap'; + pre.style.wordBreak = 'break-word'; + pre.style.fontFamily = 'var(--font-monospace, monospace)'; + pre.style.fontSize = '12px'; + pre.style.lineHeight = '1.6'; + pre.style.color = 'var(--text-normal)'; + pre.setText(this.selectedText || '(empty note)'); + return; + } + + const diffLines = computeLineDiff(this.selectedText, this.currentText); + let addedCount = 0; + let removedCount = 0; + for (const line of diffLines) + if (line.type === 'added') addedCount++; + else if (line.type === 'removed') removedCount++; + if (this.diffStatsEl) { + this.diffStatsEl.empty(); + const addSpan = this.diffStatsEl.createSpan({ text: `+${addedCount}` }); + addSpan.style.color = 'var(--text-success, #2ea043)'; + addSpan.style.fontWeight = 'bold'; + addSpan.style.marginRight = '8px'; + const remSpan = this.diffStatsEl.createSpan({ text: `-${removedCount}` }); + remSpan.style.color = 'var(--text-error, #f85149)'; + remSpan.style.fontWeight = 'bold'; + } + + renderDiffLines(this.previewEl, diffLines); + } + + private async select( + version: VersionSummary, + item?: HTMLButtonElement, + list?: HTMLElement, + ): Promise { + if (this.objectUrl) { + URL.revokeObjectURL(this.objectUrl); + this.objectUrl = undefined; + } + this.selected = version; + this.selectedText = ''; + this.selectedBinary = undefined; + this.resetConfirmation(); + const listContainer = list ?? this.listEl; + const index = this.versions.findIndex((v) => v.lamport === version.lamport); + if (listContainer) + for (const [idx, child] of [...listContainer.children].entries()) { + const btn = child as HTMLElement; + const isCurrent = idx === index; + btn.style.backgroundColor = isCurrent + ? 'var(--background-modifier-hover)' + : 'var(--background-secondary)'; + btn.style.borderColor = isCurrent + ? 'var(--interactive-accent)' + : 'var(--background-modifier-border)'; + btn.style.fontWeight = isCurrent ? 'bold' : 'normal'; + } + + if (item) { + item.style.backgroundColor = 'var(--background-modifier-hover)'; + item.style.borderColor = 'var(--interactive-accent)'; + item.style.fontWeight = 'bold'; + } + if (this.mobileRoller) this.mobileRoller.update(version, index); + + if (this.previewEl) { + this.previewEl.empty(); + this.previewEl.setText( + text(this.ctx.translate(Platform.isMobile ? 'previewHintMobile' : 'previewHint')), + ); + } + if (this.restoreButton) this.restoreButton.setDisabled(true); + try { + const loaded = await this.actions.load(this.file.path, version.lamport); + this.selectedText = loaded.text; + this.selectedBinary = loaded.binary; + this.renderPreview(); + if (this.restoreButton) this.restoreButton.setDisabled(false); + } catch (error) { + const message = text(this.ctx.translate('loadFailed', String(error))); + if (this.previewEl) { + this.previewEl.empty(); + this.previewEl.setText(message); + } + new Notice(message); + } + } + + private async stepVersion(delta: number): Promise { + if (!this.selected) return; + const currentIndex = this.versions.findIndex((v) => v.lamport === this.selected?.lamport); + if (currentIndex === -1) return; + const nextIndex = currentIndex + delta; + if (nextIndex < 0 || nextIndex >= this.versions.length) return; + const target = this.versions[nextIndex]; + if (target) await this.select(target); + } + + private async onRestoreClick(): Promise { + if (!this.selected || !this.restoreButton) return; + const label = this.label(this.selected); + if (!this.confirming) { + this.confirming = true; + if (this.confirmationEl) { + this.confirmationEl.textContent = text(this.ctx.translate('confirmRestore', label)); + this.confirmationEl.style.display = 'block'; + } + this.restoreButton.setButtonText(text(this.ctx.translate('confirmRestoreButton'))); + return; + } + try { + await this.actions.restore(this.selectedText, this.selectedBinary); + new Notice(text(this.ctx.translate('restored', label))); + this.close(); + } catch (error) { + new Notice(text(this.ctx.translate('restoreFailed', String(error)))); + this.resetConfirmation(); + } + } + + private resetConfirmation(): void { + this.confirming = false; + if (this.confirmationEl) { + this.confirmationEl.style.display = 'none'; + this.confirmationEl.textContent = ''; + } + if (this.restoreButton) + this.restoreButton.setButtonText(text(this.ctx.translate('restore'))); + } +} diff --git a/packages/history/src/pack.ts b/packages/history/src/pack.ts new file mode 100644 index 00000000..f33946de --- /dev/null +++ b/packages/history/src/pack.ts @@ -0,0 +1,253 @@ +import { PACK_MAGIC, PACK_VERSION } from '@/types'; +import { textToUint8Array } from '@repo/shared/binary'; +import { sha256Bytes } from '@repo/shared/sha256-bytes'; +// The /browser subpath avoids fflate's `node` export condition, which pulls in `module` and +// worker_threads. Those cannot be resolved when Obsidian loads the module as browser ESM. +import { deflateSync, inflateSync } from 'fflate/browser'; +import type { Codec, IndexEntry, PackHeader } from '@/types'; +import type { Binary } from '@repo/shared/binary'; + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +/** Minimal concatenation over Uint8Array (avoids the BufferSource variance of concatBinary). */ +function concat(...parts: Array): Uint8Array { + let total = 0; + for (const part of parts) total += part.byteLength; + const out = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.byteLength; + } + return out; +} + +export type PackInput = { + baseLamport: number; + createdAt: number; + deviceId: string; + ns: string; + peakLamport: number; + seq: number; +}; + +export type EncodedPack = { bytes: Uint8Array; header: PackHeader; entries: Array }; +export type DecodedPack = { header: PackHeader; entries: Array; payload: Uint8Array }; + +function u32(value: number): Uint8Array { + const out = new Uint8Array(4); + new DataView(out.buffer).setUint32(0, value, false); + return out; +} + +function readU32(bytes: Uint8Array, offset: number): number { + return new DataView(bytes.buffer, bytes.byteOffset + offset, 4).getUint32(0, false); +} + +function jsonBytes(value: unknown): Uint8Array { + return encoder.encode(JSON.stringify(value)); +} + +/** + * A fixed 128-byte reserved prefix carrying the numeric fields; ids live in the length-delimited + * meta. + */ +function encodeReservedPrefix(header: PackHeader): Uint8Array { + const out = new Uint8Array(128); + const view = new DataView(out.buffer); + let offset = 0; + view.setUint32(offset, PACK_MAGIC, false); + offset += 4; + view.setUint16(offset, PACK_VERSION, false); + offset += 2; + view.setUint16(offset, header.codec === 'deflate' ? 1 : 0, false); + offset += 2; + const numbers = [ + header.seq, + header.baseLamport, + header.peakLamport, + header.createdAt, + header.commitCount, + header.indexOffset, + header.indexLength, + header.payloadOffset, + header.payloadLength, + ]; + for (const number of numbers) { + view.setFloat64(offset, number, false); + offset += 8; + } + return out; +} + +function decodeReservedPrefix(bytes: Uint8Array): { codec: Codec; numbers: Array } { + const view = new DataView(bytes.buffer, bytes.byteOffset, 128); + if (view.getUint32(0, false) !== PACK_MAGIC) throw new Error('Invalid pack magic'); + if (view.getUint16(4, false) !== PACK_VERSION) throw new Error('Unsupported pack version'); + const codec: Codec = view.getUint16(6, false) === 1 ? 'deflate' : 'raw'; + const numbers: Array = []; + for (let index = 0; index < 9; index++) numbers.push(view.getFloat64(8 + index * 8, false)); + return { codec, numbers }; +} + +/** Canonical, hash-stable serialization of the header fields (identical in encode and decode). */ +function canonicalHeader(h: Omit): Uint8Array { + return jsonBytes({ + baseLamport: h.baseLamport, + codec: h.codec, + commitCount: h.commitCount, + createdAt: h.createdAt, + deviceId: h.deviceId, + indexLength: h.indexLength, + indexOffset: h.indexOffset, + magic: h.magic, + ns: h.ns, + payloadLength: h.payloadLength, + payloadOffset: h.payloadOffset, + peakLamport: h.peakLamport, + seq: h.seq, + version: h.version, + }); +} + +function sortEntries(entries: Array): Array { + return [...entries].sort( + (a, b) => + a.lamport - b.lamport || + a.deviceId.localeCompare(b.deviceId) || + a.keyHash.localeCompare(b.keyHash), + ); +} + +/** Encodes the three regions and returns the self-certifying header (hash is embedded in the name). */ +export async function encodePack( + partial: PackInput, + entries: Array, + payload: Uint8Array, +): Promise { + const sorted = sortEntries(entries); + const codec: Codec = payload.byteLength > 0 ? 'deflate' : 'raw'; + const compressed = codec === 'deflate' ? deflateSync(payload) : payload; + const meta = { deviceId: partial.deviceId, ns: partial.ns }; + const metaBytes = concat(u32(metaBytesLength(meta)), jsonBytes(meta)); + const indexBytes = encodeIndex(sorted); + const reserved = 128; + const indexOffset = reserved + metaBytes.byteLength; + const payloadOffset = indexOffset + indexBytes.byteLength; + const headerBase: Omit = { + ...partial, + codec, + commitCount: sorted.length, + indexLength: indexBytes.byteLength, + indexOffset, + magic: PACK_MAGIC, + payloadLength: compressed.byteLength, + payloadOffset, + version: PACK_VERSION, + }; + const hash = await sha256Bytes(concat(canonicalHeader(headerBase), indexBytes, compressed)); + const header: PackHeader = { ...headerBase, hash }; + const bytes = concat(encodeReservedPrefix(header), metaBytes, indexBytes, compressed); + return { bytes, entries: sorted, header }; +} + +function metaBytesLength(meta: { deviceId: string; ns: string }): number { + return jsonBytes(meta).byteLength; +} + +function encodeIndex(entries: Array): Uint8Array { + const parts: Array = [u32(entries.length)]; + for (const entry of entries) { + const bytes = jsonBytes(entry); + parts.push(u32(bytes.byteLength), bytes); + } + return concat(...parts); +} + +function decodeIndex(bytes: Uint8Array, offset: number, length: number): Array { + const region = bytes.subarray(offset, offset + length); + const count = readU32(region, 0); + let cursor = 4; + const entries: Array = []; + for (let index = 0; index < count; index++) { + const size = readU32(region, cursor); + cursor += 4; + entries.push( + JSON.parse(decoder.decode(region.subarray(cursor, cursor + size))) as IndexEntry, + ); + cursor += size; + } + return entries; +} + +function decodeMeta(bytes: Uint8Array): { deviceId: string; ns: string } { + const length = readU32(bytes, 128); + return JSON.parse(decoder.decode(bytes.subarray(132, 132 + length))) as { + deviceId: string; + ns: string; + }; +} + +/** Parses header + index only, without inflating the payload. */ +export async function readIndexOnly( + bytes: Uint8Array, +): Promise<{ header: PackHeader; entries: Array }> { + const { codec, numbers } = decodeReservedPrefix(bytes); + const meta = decodeMeta(bytes); + const [ + seq, + baseLamport, + peakLamport, + createdAt, + commitCount, + indexOffset, + indexLength, + payloadOffset, + payloadLength, + ] = numbers; + const entries = decodeIndex(bytes, indexOffset, indexLength); + const headerBase: Omit = { + baseLamport, + codec, + commitCount, + createdAt, + deviceId: meta.deviceId, + indexLength, + indexOffset, + magic: PACK_MAGIC, + ns: meta.ns, + payloadLength, + payloadOffset, + peakLamport, + seq, + version: PACK_VERSION, + }; + const hash = await sha256Bytes( + concat( + canonicalHeader(headerBase), + bytes.subarray(indexOffset, indexOffset + indexLength), + bytes.subarray(payloadOffset, payloadOffset + payloadLength), + ), + ); + return { entries, header: { ...headerBase, hash } }; +} + +/** Full decode: verifies magic, recomputes the hash, inflates the payload. */ +export async function decodePack(bytes: Uint8Array): Promise { + const { header, entries } = await readIndexOnly(bytes); + const compressed = bytes.subarray( + header.payloadOffset, + header.payloadOffset + header.payloadLength, + ); + const payload = header.codec === 'deflate' ? inflateSync(compressed) : compressed; + return { entries, header, payload }; +} + +export function packFileName(deviceId: string, seq: number, hash: string): string { + return `p-${String(seq).padStart(8, '0')}-${hash}.pack`; +} + +export function textBytes(text: string): Binary { + return textToUint8Array(text); +} diff --git a/packages/history/src/recover-deleted-modal.ts b/packages/history/src/recover-deleted-modal.ts new file mode 100644 index 00000000..7537ff6f --- /dev/null +++ b/packages/history/src/recover-deleted-modal.ts @@ -0,0 +1,56 @@ +import { Modal } from 'obsidian'; +import type { HistoryTranslations } from './i18n'; +import type { Translate } from '@hesprs/sync-engine-sdk'; +import type { App } from 'obsidian'; + +function text(value: unknown): string { + return typeof value === 'string' ? value : String(value); +} + +/** Modal to select and recover deleted notes. */ +export default class RecoverDeletedModal extends Modal { + constructor( + private readonly ctx: { app: App; translate: Translate }, + private readonly deletedKeys: Array, + private readonly onSelect: (key: string) => void, + ) { + super(ctx.app); + this.setTitle(text(ctx.translate('recoverDeletedTitle'))); + } + + onOpen(): void { + const { contentEl } = this; + contentEl.empty(); + contentEl.style.display = 'flex'; + contentEl.style.flexDirection = 'column'; + contentEl.style.gap = '12px'; + if (this.deletedKeys.length === 0) { + contentEl.createEl('p', { text: text(this.ctx.translate('noDeletedNotes')) }); + return; + } + const list = contentEl.createEl('div'); + list.style.display = 'flex'; + list.style.flexDirection = 'column'; + list.style.gap = '6px'; + list.style.maxHeight = '60vh'; + list.style.overflowY = 'auto'; + list.style.paddingRight = '4px'; + + for (const key of this.deletedKeys) { + const item = list.createEl('button', { text: key }); + item.style.width = '100%'; + item.style.textAlign = 'left'; + item.style.padding = '8px 12px'; + item.style.borderRadius = '4px'; + item.style.fontSize = '13px'; + item.style.cursor = 'pointer'; + item.style.border = '1px solid var(--background-modifier-border)'; + item.style.backgroundColor = 'var(--background-secondary)'; + item.style.color = 'var(--text-normal)'; + item.addEventListener('click', () => { + this.close(); + this.onSelect(key); + }); + } + } +} diff --git a/packages/history/src/remote.ts b/packages/history/src/remote.ts new file mode 100644 index 00000000..80f70086 --- /dev/null +++ b/packages/history/src/remote.ts @@ -0,0 +1,159 @@ +import { HISTORY_PREFIX } from '@/constants'; +import { decodeHeadLog, encodeHeadLog } from '@/head-log'; +import { packFileName } from '@/pack'; +import { digOriginal, prefixWrapper } from '@hesprs/sync-engine-sdk'; +import { getStatus, isNotFound } from '@repo/shared/error'; +import { dirname } from '@repo/shared/path'; +import type { HistoryCipher } from '@/crypto'; +import type { CommitLog } from '@/types'; +import type { Binary, FileStat, Fs, ListOptions, Stat } from '@hesprs/sync-engine-sdk'; + +export type RemoteHeadStat = { deviceId: string; mtime?: number; uid?: string }; + +export type RemoteHistory = { + fs: Fs; + ns: string; + deviceId: string; + putPack: (seq: number, hash: string, bytes: Uint8Array) => Promise; + putBlob: (hash: string, bytes: Uint8Array) => Promise; + getObject: (key: string) => Promise; + readHead: (deviceId: string) => Promise; + writeHead: (log: CommitLog) => Promise; + listHeads: () => Promise>; + listPacks: (deviceId: string) => Promise>; + deletePack: (deviceId: string, fileName: string) => Promise; +}; + +function fileStat(key: string, size: number, mtime = Date.now()): FileStat { + return { isDir: false, key, meta: () => ({}), mtime, size, uid: `history-${mtime}` }; +} + +const includeAll = (): ListOptions => 'include'; + +function remotePath(ns: string, ...parts: Array): string { + return [ns, ...parts].join('/'); +} + +function headKey(ns: string, deviceId: string): string { + return remotePath(ns, 'heads', `${deviceId}.log`); +} + +function packKey(ns: string, deviceId: string, fileName: string): string { + return remotePath(ns, 'packs', deviceId, fileName); +} + +function blobKey(ns: string, hash: string): string { + return remotePath(ns, 'blobs', `${hash}.blob`); +} + +/** + * Builds a dedicated history handle from the run's remoteFs. digOriginal walks past every sync + * wrapper, so history never pollutes the sync core's remote snapshot. When encryption is active, + * object contents are AES-GCM encrypted here at the PUT/GET boundary while remote object names stay + * literal (content hashes), preserving self-certifying names, dedup and orphan discovery. + */ +export function openRemoteHistory( + remoteFs: Fs, + namespace: string | Error, + deviceId: string, + cipher?: HistoryCipher, +): RemoteHistory | undefined { + if (namespace instanceof Error || namespace === '') return undefined; + const fs = prefixWrapper(digOriginal(remoteFs), HISTORY_PREFIX); + const ns = namespace; + + /** Writes bytes, encrypting at rest when a cipher is active. */ + async function put(key: string, bytes: Uint8Array): Promise { + const payload = cipher ? await cipher.encrypt(bytes) : bytes; + const stat = fileStat(key, payload.byteLength); + try { + await fs.write(key, payload as Binary, stat); + } catch (error) { + const status = getStatus(error); + if (status === 409 || status === 404) { + const dir = dirname(key); + if (dir && dir !== '/') { + await fs.mkdir(dir, { isDir: true, key: dir, meta: () => ({}) }, true); + await fs.write(key, payload as Binary, stat); + return; + } + } + throw error; + } + } + + /** + * Reads raw (possibly encrypted) bytes, without decrypting. History always decodes a whole + * object in memory, so a single full read is used instead of the size-based streaming path. + */ + async function readBytes(key: string): Promise { + return fs.read(key, fileStat(key, 0)); + } + + async function get(key: string): Promise { + const raw = await readBytes(key); + return cipher ? cipher.decrypt(raw) : raw; + } + + async function listHeadStats(): Promise> { + let stats: Array; + try { + stats = await fs.list(remotePath(ns, 'heads'), includeAll); + } catch { + return []; + } + return stats + .filter((stat): stat is FileStat => !stat.isDir && stat.key.endsWith('.log')) + .map((stat) => ({ + deviceId: (stat.key.split('/').at(-1) ?? stat.key).slice(0, -4), + mtime: stat.mtime, + uid: stat.uid, + })); + } + + async function listPackNames(packDeviceId: string): Promise> { + // Object stores have no real folder object, so exists(folder) is unreliable. List directly + // and treat a missing folder as an empty set. + let stats: Array; + try { + stats = await fs.list(remotePath(ns, 'packs', packDeviceId), includeAll); + } catch { + return []; + } + return stats + .filter((stat) => !stat.isDir && stat.key.endsWith('.pack')) + .map((stat) => stat.key.split('/').at(-1) ?? stat.key); + } + + return { + async deletePack(packDeviceId, fileName) { + await fs.delete(packKey(ns, packDeviceId, fileName)); + }, + deviceId, + fs, + getObject: get, + listHeads: listHeadStats, + listPacks: listPackNames, + ns, + async putBlob(hash, bytes) { + await put(blobKey(ns, hash), bytes); + }, + async putPack(seq, hash, bytes) { + await put(packKey(ns, deviceId, packFileName(deviceId, seq, hash)), bytes); + }, + async readHead(headDeviceId) { + const key = headKey(ns, headDeviceId); + try { + const bytes = await get(key); + if (!bytes || bytes.byteLength === 0) return; + return await decodeHeadLog(bytes); + } catch (error) { + if (isNotFound(error) || String(error).toLowerCase().includes('missing')) return; + throw error; + } + }, + async writeHead(log) { + await put(headKey(ns, log.deviceId), encodeHeadLog(log)); + }, + }; +} diff --git a/packages/history/src/restore.ts b/packages/history/src/restore.ts new file mode 100644 index 00000000..53bde07a --- /dev/null +++ b/packages/history/src/restore.ts @@ -0,0 +1,292 @@ +import { assembleChunks, captureContent, isTextFile } from '@/chunker'; +import { mergeTextLines } from '@/dag'; +import { reconcile } from '@/flush'; +import { chunkLocation, lookupCommit, versionKey } from '@/index-store'; +import { decodePack, readIndexOnly } from '@/pack'; +import { appendWal, blobPresent } from '@/store'; +import { FLAG_DELETION, FLAG_SMALL_FILE } from '@/types'; +import { isNotFound } from '@repo/shared/error'; +import { sha256Bytes } from '@repo/shared/sha256-bytes'; +import type { RemoteHistory } from '@/remote'; +import type { HistoryStores } from '@/store'; +import type { ChunkRef, IndexEntry, RestoreResult, VersionRef, VersionSummary } from '@/types'; + +export type RestoreContext = { remote: RemoteHistory; stores: HistoryStores }; + +function packObjectKey(remote: RemoteHistory, fileName: string, deviceId: string): string { + return `${remote.ns}/packs/${deviceId}/${fileName}`; +} + +function packFileNameFromRef(deviceId: string, seq: number, hash: string): string { + return `p-${String(seq).padStart(8, '0')}-${hash}.pack`; +} + +async function fetchPack( + ctx: RestoreContext, + pack: { deviceId: string; seq: number; hash: string }, +): Promise { + return ctx.remote.getObject( + packObjectKey( + ctx.remote, + packFileNameFromRef(pack.deviceId, pack.seq, pack.hash), + pack.deviceId, + ), + ); +} + +/** Resolves an ordered ChunkRef[] into bytes, fetching only packs that hold missing chunks. */ +async function resolveChunks( + ctx: RestoreContext, + chunks: Array, + ownPayload?: Uint8Array, +): Promise> { + const out: Array = []; + for (const chunk of chunks) { + if (chunk.offset >= 0 && ownPayload) { + out.push(ownPayload.subarray(chunk.offset, chunk.offset + chunk.length)); + continue; + } + const location = await chunkLocation(ctx.stores, chunk.hash); + if (!location) throw new Error(`Missing chunk ${chunk.hash}`); + const bytes = await fetchPack(ctx, location.pack); + const decoded = await decodePack(bytes); + out.push(decoded.payload.subarray(location.offset, location.offset + location.length)); + } + return out; +} + +/** O(1) local lookup; on a miss, refresh heads and rebuild the indexes before giving up. */ +async function lookupWithRepair(ctx: RestoreContext, ref: VersionRef) { + const hit = await lookupCommit(ctx.stores, ref); + if (hit) return hit; + // Head refresh → bounded per-device pack repair → full index rebuild, then retry. + await reconcile({ remote: ctx.remote, requestSync: () => {}, stores: ctx.stores }); + const repaired = await lookupCommit(ctx.stores, ref); + if (!repaired) throw new Error(`No commit location for ${ref.key}`); + return repaired; +} + +/** Restores one version with a bounded number of remote reads. */ +export async function restoreVersion(ctx: RestoreContext, ref: VersionRef): Promise { + const location = await lookupWithRepair(ctx, ref); + let bytes: Uint8Array; + try { + bytes = await fetchPack(ctx, location.pack); + } catch (error) { + if (isNotFound(error) || String(error).toLowerCase().includes('missing')) + await ctx.stores.index.delete(versionKey(ref)); + throw error; + } + const { entries } = await readIndexOnly(bytes); + const entry = entries.find( + (candidate) => candidate.key === ref.key && candidate.lamport === ref.lamport, + ); + // Never fall back to another commit in the pack: a pack batches edits from many files, so + // returning entries[0] would silently restore a different file's content. + if (!entry) { + await ctx.stores.index.delete(versionKey(ref)); + throw new Error(`Version ${ref.key}@${ref.lamport} is missing from its pack`); + } + + if (entry.blobs && entry.blobs.length > 0) { + const blob = entry.blobs[0]; + if (!blob) throw new Error(`Empty blob entry for ${ref.key}`); + const blobBytes = await ctx.remote.getObject(`${ctx.remote.ns}/blobs/${blob.hash}.blob`); + const isText = isTextFile(entry.key); + return { + at: entry.at, + binary: isText ? undefined : blobBytes, + key: entry.key, + lamport: entry.lamport, + merged: false, + text: isText ? new TextDecoder().decode(blobBytes) : '', + }; + } + + const decoded = await decodePack(bytes); + let text: string; + if (entry.flags & FLAG_SMALL_FILE && entry.inlineOffset !== undefined) + text = new TextDecoder().decode( + decoded.payload.subarray( + entry.inlineOffset, + entry.inlineOffset + (entry.inlineLength ?? 0), + ), + ); + else { + const parts = await resolveChunks(ctx, entry.chunks, decoded.payload); + text = assembleChunks(parts); + } + + let merged = false; + if (entry.parents.length > 1) { + // A DAG fork: align the two tips against their common ancestor and merge divergent runs. + const tips = await Promise.all( + entry.parents.slice(0, 2).map(async (parent) => findEntry(ctx, parent)), + ); + const ancestor = await findAncestor(ctx, tips[0], tips[1]); + if (tips[0] && tips[1]) { + const [text0, text1, textAncestor] = await Promise.all([ + readEntryText(ctx, tips[0]), + readEntryText(ctx, tips[1]), + readEntryText(ctx, ancestor), + ]); + const mergeResult = mergeTextLines(textAncestor, text0, text1); + text = mergeResult.text; + merged = true; + // Conflict markers are still committed; nothing is silently discarded. The merged text + // becomes a new commit with both tips as parents, then a fresh CDC pass persists only + // its new chunks (queued in the WAL for the next flush). + const content = await captureContent( + ctx.stores, + entry.key, + new TextEncoder().encode(text), + ); + await appendWal(ctx.stores, { + at: Date.now(), + key: entry.key, + parent: '', + parents: entry.parents.slice(0, 2), + ...content, + }); + } + } + + for (const blob of entry.blobs) + if (!(await blobPresent(ctx.stores, blob.hash))) + await ctx.remote.getObject(`${ctx.remote.ns}/blobs/${blob.hash}.blob`); + return { at: entry.at, key: entry.key, lamport: entry.lamport, merged, text }; +} + +async function findEntry(ctx: RestoreContext, refKey: string): Promise { + const [key, lamport] = refKey.split('@'); + const location = await lookupCommit(ctx.stores, { key, lamport: Number(lamport) }); + if (!location) return undefined; + const bytes = await fetchPack(ctx, location.pack); + const { entries } = await readIndexOnly(bytes); + return entries.find((entry) => entry.key === key && entry.lamport === Number(lamport)); +} + +async function findAncestor( + ctx: RestoreContext, + a?: IndexEntry, + b?: IndexEntry, +): Promise { + if (!a || !b) return undefined; + for (const parent of a.parents) { + const entry = await findEntry(ctx, parent); + if (entry && b.parents.includes(`${entry.key}@${entry.lamport}`)) return entry; + } + return undefined; +} + +async function readEntryText(ctx: RestoreContext, entry?: IndexEntry): Promise { + if (!entry) return ''; + const location = await lookupCommit(ctx.stores, { key: entry.key, lamport: entry.lamport }); + if (!location) return ''; + const bytes = await fetchPack(ctx, location.pack); + const decoded = await decodePack(bytes); + if (entry.flags & FLAG_SMALL_FILE && entry.inlineOffset !== undefined) + return new TextDecoder().decode( + decoded.payload.subarray( + entry.inlineOffset, + entry.inlineOffset + (entry.inlineLength ?? 0), + ), + ); + const parts = await resolveChunks(ctx, entry.chunks, decoded.payload); + return assembleChunks(parts); +} + +async function resolveFingerprint( + entry: IndexEntry, + getPayload: () => Promise, +): Promise { + if (entry.hash) return entry.hash; + if (entry.flags & FLAG_DELETION) return ''; + if (entry.chunks.length > 0) return entry.chunks.map((c) => c.hash).join(','); + if (entry.blobs.length > 0) return entry.blobs.map((b) => b.hash).join(','); + if ( + entry.flags & FLAG_SMALL_FILE && + entry.inlineOffset !== undefined && + entry.inlineLength !== undefined + ) { + const payload = await getPayload(); + const slice = payload.subarray(entry.inlineOffset, entry.inlineOffset + entry.inlineLength); + return sha256Bytes(slice); + } + return undefined; +} + +/** Lists a file's versions with their timestamps, reading each pack's index at most once. */ +export async function listVersions( + ctx: RestoreContext, + key: string, +): Promise> { + const prefix = `${key}@`; + const refs = (await ctx.stores.index.entries()).filter(([ref]) => ref.startsWith(prefix)); + const byPack = new Map< + string, + { deviceId: string; hash: string; seq: number; lamports: Array } + >(); + for (const [ref, location] of refs) { + const { deviceId, hash, seq } = location.pack; + const packKey = `${deviceId}:${seq}:${hash}`; + let group = byPack.get(packKey); + if (!group) { + group = { deviceId, hash, lamports: [], seq }; + byPack.set(packKey, group); + } + group.lamports.push(Number(ref.slice(prefix.length))); + } + type VersionWithFingerprint = VersionSummary & { fingerprint?: string }; + const summaries: Array = []; + for (const group of byPack.values()) { + let bytes: Uint8Array; + try { + bytes = await fetchPack(ctx, group); + } catch (error) { + if (isNotFound(error) || String(error).toLowerCase().includes('missing')) + for (const lamport of group.lamports) + await ctx.stores.index.delete(`${key}@${lamport}`); + continue; + } + const { entries } = await readIndexOnly(bytes); + let decodedPayload: Uint8Array | undefined; + const getPayload = async () => { + if (!decodedPayload) { + const decoded = await decodePack(bytes); + decodedPayload = decoded.payload; + } + return decodedPayload; + }; + for (const lamport of group.lamports) { + const entry = entries.find( + (candidate) => candidate.lamport === lamport && candidate.key === key, + ); + if (!entry) { + await ctx.stores.index.delete(`${key}@${lamport}`); + continue; + } + const fp = await resolveFingerprint(entry, getPayload); + summaries.push({ at: entry.at, fingerprint: fp, index: 0, key, lamport }); + } + } + // Versions are per file: number them 1..N newest-first so the UI never exposes the global + // Lamport clock (which is shared across every file and therefore has gaps). + summaries.sort((a, b) => b.lamport - a.lamport); + const distinct: Array = []; + let lastFp: string | undefined; + for (const summary of summaries) { + if (summary.fingerprint !== undefined && summary.fingerprint === lastFp) continue; + lastFp = summary.fingerprint; + distinct.push(summary); + } + const numbered: Array = []; + for (const [position, summary] of distinct.entries()) + numbered.push({ + at: summary.at, + index: position + 1, + key: summary.key, + lamport: summary.lamport, + }); + return numbered; +} diff --git a/packages/history/src/roller.ts b/packages/history/src/roller.ts new file mode 100644 index 00000000..4d7cbc26 --- /dev/null +++ b/packages/history/src/roller.ts @@ -0,0 +1,185 @@ +import type { VersionSummary } from './types'; + +export type MobileRoller = { + readonly container: HTMLElement; + update: (selected: VersionSummary, index: number) => void; +}; + +export function createMobileRoller(options: { + container: HTMLElement; + versions: Array; + label: (version: VersionSummary) => string; + currentBadgeText: string; + onSelect: (version: VersionSummary) => void; + onStep: (delta: number) => void; +}): MobileRoller { + const { container, versions, label, currentBadgeText, onSelect, onStep } = options; + + const roller = container.createEl('div'); + roller.style.display = 'flex'; + roller.style.alignItems = 'center'; + roller.style.height = '32px'; + roller.style.borderRadius = '6px'; + roller.style.border = '1px solid var(--background-modifier-border)'; + roller.style.backgroundColor = 'var(--background-secondary)'; + roller.style.flex = '1'; + roller.style.minWidth = '0'; + roller.style.overflow = 'hidden'; + roller.style.position = 'relative'; + + const labelWrap = roller.createEl('div'); + labelWrap.style.display = 'flex'; + labelWrap.style.alignItems = 'center'; + labelWrap.style.justifyContent = 'space-between'; + labelWrap.style.gap = '6px'; + labelWrap.style.flex = '1'; + labelWrap.style.minWidth = '0'; + labelWrap.style.padding = '0 8px'; + labelWrap.style.height = '100%'; + labelWrap.style.position = 'relative'; + + const labelEl = labelWrap.createEl('span'); + labelEl.style.fontSize = '12px'; + labelEl.style.whiteSpace = 'nowrap'; + labelEl.style.overflow = 'hidden'; + labelEl.style.textOverflow = 'ellipsis'; + labelEl.style.color = 'var(--text-normal)'; + + const countEl = labelWrap.createEl('span'); + countEl.style.fontSize = '11px'; + countEl.style.color = 'var(--text-muted)'; + countEl.style.flexShrink = '0'; + + const selectEl = labelWrap.createEl('select'); + selectEl.style.position = 'absolute'; + selectEl.style.inset = '0'; + selectEl.style.opacity = '0'; + selectEl.style.width = '100%'; + selectEl.style.height = '100%'; + selectEl.style.cursor = 'pointer'; + + for (const [idx, v] of versions.entries()) { + const opt = selectEl.createEl('option', { value: String(idx) }); + const badge = idx === 0 ? ` ${currentBadgeText}` : ''; + opt.text = `${label(v)}${badge}`; + } + + selectEl.addEventListener('change', () => { + const idx = Number.parseInt(selectEl.value, 10); + const target = versions[idx]; + if (target) onSelect(target); + }); + + let touchStartY = 0; + let touchStartX = 0; + labelWrap.addEventListener( + 'touchstart', + (e) => { + const touch = e.touches[0]; + if (touch) { + touchStartY = touch.clientY; + touchStartX = touch.clientX; + } + }, + { passive: true }, + ); + + labelWrap.addEventListener( + 'touchend', + (e) => { + const touch = e.changedTouches[0]; + if (!touch) return; + const deltaY = touch.clientY - touchStartY; + const deltaX = touch.clientX - touchStartX; + if (Math.abs(deltaY) > 20 && Math.abs(deltaY) > Math.abs(deltaX)) { + e.preventDefault(); + selectEl.blur(); + if (deltaY > 0) onStep(-1); + else onStep(1); + } + }, + { passive: false }, + ); + + roller.addEventListener( + 'wheel', + (e) => { + e.preventDefault(); + if (e.deltaY > 0) onStep(1); + else if (e.deltaY < 0) onStep(-1); + }, + { passive: false }, + ); + + const stepper = roller.createEl('div'); + stepper.style.display = 'flex'; + stepper.style.flexDirection = 'column'; + stepper.style.width = '24px'; + stepper.style.height = '100%'; + stepper.style.borderLeft = '1px solid var(--background-modifier-border)'; + stepper.style.flexShrink = '0'; + + const prevBtn = stepper.createEl('button'); + prevBtn.style.flex = '1'; + prevBtn.style.padding = '0'; + prevBtn.style.border = 'none'; + prevBtn.style.background = 'transparent'; + prevBtn.style.display = 'flex'; + prevBtn.style.alignItems = 'center'; + prevBtn.style.justifyContent = 'center'; + prevBtn.style.fontSize = '8px'; + prevBtn.style.cursor = 'pointer'; + prevBtn.style.color = 'var(--text-muted)'; + prevBtn.setText('▲'); + prevBtn.addEventListener('click', (e) => { + e.stopPropagation(); + onStep(-1); + }); + + const nextBtn = stepper.createEl('button'); + nextBtn.style.flex = '1'; + nextBtn.style.padding = '0'; + nextBtn.style.border = 'none'; + nextBtn.style.borderTop = '1px solid var(--background-modifier-border)'; + nextBtn.style.background = 'transparent'; + nextBtn.style.display = 'flex'; + nextBtn.style.alignItems = 'center'; + nextBtn.style.justifyContent = 'center'; + nextBtn.style.fontSize = '8px'; + nextBtn.style.cursor = 'pointer'; + nextBtn.style.color = 'var(--text-muted)'; + nextBtn.setText('▼'); + nextBtn.addEventListener('click', (e) => { + e.stopPropagation(); + onStep(1); + }); + + return { + container: roller, + update: (selected, index) => { + const isCurrent = selected.lamport === versions[0]?.lamport; + const badge = isCurrent ? ` ${currentBadgeText}` : ''; + labelEl.textContent = `${label(selected)}${badge}`; + labelEl.style.transition = 'opacity 0.15s ease'; + labelEl.style.opacity = '0.5'; + setTimeout(() => { + labelEl.style.opacity = '1'; + }, 50); + + if (index !== -1) { + countEl.textContent = `${index + 1}/${versions.length}`; + selectEl.selectedIndex = index; + } + + const prevDisabled = index <= 0; + prevBtn.disabled = prevDisabled; + prevBtn.style.opacity = prevDisabled ? '0.3' : '1'; + prevBtn.style.cursor = prevDisabled ? 'default' : 'pointer'; + + const nextDisabled = index === -1 || index >= versions.length - 1; + nextBtn.disabled = nextDisabled; + nextBtn.style.opacity = nextDisabled ? '0.3' : '1'; + nextBtn.style.cursor = nextDisabled ? 'default' : 'pointer'; + }, + }; +} diff --git a/packages/history/src/setting.ts b/packages/history/src/setting.ts new file mode 100644 index 00000000..2d325999 --- /dev/null +++ b/packages/history/src/setting.ts @@ -0,0 +1,138 @@ +import { settingEntry as s } from '@hesprs/sync-engine-sdk'; +import { Notice } from 'obsidian'; +import type { HistoryTranslations } from '@/i18n'; +import type { CallableOrObjectTree, Translate } from '@hesprs/sync-engine-sdk'; +import type { SettingGroupItem } from 'obsidian'; + +export type HistorySettings = { + autoGc?: boolean; + deviceId?: string; + lastGcTime?: number; + maxVersions: number; + protectExcludedNotes?: boolean; + retentionDays: number; + tieredRetention?: boolean; + trackAttachments?: boolean; +}; + +export type HistorySettingActions = { runGc: () => Promise }; + +export default function historySetting( + { + saveSettings, + translate, + }: { saveSettings: () => Promise; translate: Translate }, + settings: HistorySettings, + actions: HistorySettingActions, +): CallableOrObjectTree { + return { + 3055: s( + (self) => ({ + heading: translate('history'), + items: Object.values(self).map((node) => node(node) as SettingGroupItem), + type: 'group', + }), + { + 1000: s(() => ({ + desc: translate('retentionDaysDescription'), + name: translate('retentionDays'), + render: (setting) => { + setting.addText((text) => { + text.setValue(String(settings.retentionDays)).onChange((value) => { + const parsed = Number(value); + if (Number.isFinite(parsed) && parsed >= 0) + settings.retentionDays = parsed; + void saveSettings(); + }); + }); + }, + })), + 2000: s(() => ({ + desc: translate('maxVersionsDescription'), + name: translate('maxVersions'), + render: (setting) => { + setting.addText((text) => { + text.setValue(String(settings.maxVersions)).onChange((value) => { + const parsed = Number(value); + if (Number.isFinite(parsed) && parsed >= 0) + settings.maxVersions = parsed; + void saveSettings(); + }); + }); + }, + })), + 2100: s(() => ({ + desc: translate('tieredRetentionDescription'), + name: translate('tieredRetention'), + render: (setting) => { + setting.addToggle((toggle) => { + toggle + .setValue(settings.tieredRetention !== false) + .onChange((value) => { + settings.tieredRetention = value; + void saveSettings(); + }); + }); + }, + })), + 2500: s(() => ({ + desc: translate('autoGcDescription'), + name: translate('autoGc'), + render: (setting) => { + setting.addToggle((toggle) => { + toggle.setValue(settings.autoGc !== false).onChange((value) => { + settings.autoGc = value; + void saveSettings(); + }); + }); + }, + })), + 2600: s(() => ({ + desc: translate('protectExcludedNotesDescription'), + name: translate('protectExcludedNotes'), + render: (setting) => { + setting.addToggle((toggle) => { + toggle + .setValue(settings.protectExcludedNotes === true) + .onChange((value) => { + settings.protectExcludedNotes = value; + void saveSettings(); + }); + }); + }, + })), + 2700: s(() => ({ + desc: translate('trackAttachmentsDescription'), + name: translate('trackAttachments'), + render: (setting) => { + setting.addToggle((toggle) => { + toggle + .setValue(settings.trackAttachments === true) + .onChange((value) => { + settings.trackAttachments = value; + void saveSettings(); + }); + }); + }, + })), + 3000: s(() => ({ + desc: translate('gcDescription'), + name: translate('gc'), + render: (setting) => { + setting.addButton((button) => + button.setButtonText(translate('runGc')).onClick(() => { + void actions + .runGc() + .then((count) => new Notice(translate('gcDone', count))) + .catch( + (error: unknown) => + new Notice(translate('restoreFailed', String(error))), + ); + }), + ); + }, + })), + }, + ), + }; +} diff --git a/packages/history/src/store.ts b/packages/history/src/store.ts new file mode 100644 index 00000000..417fa21f --- /dev/null +++ b/packages/history/src/store.ts @@ -0,0 +1,144 @@ +import { DEVICE_ID_KEY, GC_TIME_KEY, LAMPORT_KEY } from '@/constants'; +import type { BlobRef, ChunkLocation, CommitLocation, CommitLog, WalEntry } from '@/types'; +import type { DatabaseAsync, DatabaseSync, StoreAsync } from '@hesprs/sync-engine-sdk'; + +export type HistoryStoreSchema = Record; +export type HistoryDatabase = DatabaseAsync | DatabaseSync; + +export type HistoryStores = { + blobs: StoreAsync; + chunks: StoreAsync; + hashes: StoreAsync; + heads: StoreAsync; + index: StoreAsync; + meta: StoreAsync; + pending: StoreAsync; +}; + +function store( + db: HistoryDatabase, + name: `${string}-history-${S}`, +): HistoryStores[S] { + return db.getStore(name) as HistoryStores[S]; +} + +/** + * Opens the WAL/CAS/head stores for one vault/remote pair. The namespace is the leading id so every + * store of a pair shares a prefix and can be enumerated, migrated or deleted together. Single-value + * metadata (Lamport clock, device id, last GC time) shares one `meta` store. + */ +export function openHistoryStores(db: HistoryDatabase, namespace: string): HistoryStores { + return { + blobs: store(db, `${namespace}-history-blobs`), + chunks: store(db, `${namespace}-history-chunks`), + hashes: store(db, `${namespace}-history-hashes`), + heads: store(db, `${namespace}-history-heads`), + index: store(db, `${namespace}-history-index`), + meta: store(db, `${namespace}-history-meta`), + pending: store(db, `${namespace}-history-pending`), + }; +} + +const deviceIdInFlight = new Map, Promise>(); + +/** Reads the persisted Lamport clock, falling back to the WAL's own lamport fields. */ +export async function readLamport(stores: HistoryStores, seen = 0): Promise { + const clock = await stores.meta.get(LAMPORT_KEY); + const wal = await stores.pending.values(); + let fromWal = 0; + for (const entry of wal) fromWal = Math.max(fromWal, entry.lamport); + return Math.max(typeof clock === 'number' ? clock : 0, fromWal, seen); +} + +export async function bumpLamport(stores: HistoryStores, seen = 0): Promise { + const current = (await readLamport(stores, seen)) + 1; + await stores.meta.set(LAMPORT_KEY, current); + return current; +} + +/** Retrieves or initializes the local device identifier for this history store. */ +export function getOrCreateDeviceId( + stores: HistoryStores, + legacyDeviceId?: string, +): Promise { + const inFlight = deviceIdInFlight.get(stores.meta); + if (inFlight) return inFlight; + const promise = (async () => { + const stored = await stores.meta.get(DEVICE_ID_KEY); + if (typeof stored === 'string' && stored.length > 0) return stored; + const id = + typeof legacyDeviceId === 'string' && legacyDeviceId.length > 0 + ? legacyDeviceId + : crypto.randomUUID(); + await stores.meta.set(DEVICE_ID_KEY, id); + return id; + })().finally(() => { + deviceIdInFlight.delete(stores.meta); + }); + deviceIdInFlight.set(stores.meta, promise); + return promise; +} + +export async function readDeviceId(stores: HistoryStores): Promise { + const value = await stores.meta.get(DEVICE_ID_KEY); + return typeof value === 'string' ? value : undefined; +} + +/** Reads the timestamp of the last history garbage collection. */ +export async function readLastGcTime(stores: HistoryStores): Promise { + const value = await stores.meta.get(GC_TIME_KEY); + return typeof value === 'number' ? value : 0; +} + +/** Persists the timestamp of the last history garbage collection. */ +export async function saveLastGcTime(stores: HistoryStores, time: number): Promise { + await stores.meta.set(GC_TIME_KEY, time); +} + +/** + * Appends a WAL entry and its Lamport bump together. The entry itself carries the lamport value, so + * a single pending-store batch is the atomic durability boundary: a kill before it resolves loses + * the edit (unavoidable), a kill after it resolves never loses it. + */ +export async function appendWal( + stores: HistoryStores, + entry: Omit, + seenLamport = 0, +): Promise { + const lamport = await bumpLamport(stores, seenLamport); + const keys = await stores.pending.keys(); + let seq = 0; + for (const value of keys) seq = Math.max(seq, Number(value) + 1); + const full: WalEntry = { ...entry, lamport, seq }; + await stores.pending.batch([{ key: String(seq), type: 'set', value: full }]); + return full; +} + +export async function readPending(stores: HistoryStores): Promise> { + const entries = await stores.pending.values(); + return entries.sort((a, b) => a.seq - b.seq); +} + +/** Deletes WAL entries only after the head log rewrite succeeds. */ +export async function clearPending( + stores: HistoryStores, + entries: ReadonlyArray, +): Promise { + await stores.pending.batch( + entries.map((entry) => ({ key: String(entry.seq), type: 'delete' as const })), + ); +} + +/** Marks attachments as confirmed present so restore can skip refetching them. */ +export async function markBlobs( + stores: HistoryStores, + blobs: ReadonlyArray, +): Promise { + await stores.blobs.batch( + blobs.map((blob) => ({ key: blob.hash, type: 'set' as const, value: blob })), + ); +} + +export async function blobPresent(stores: HistoryStores, hash: string): Promise { + return (await stores.blobs.get(hash)) !== undefined; +} diff --git a/packages/history/src/types.ts b/packages/history/src/types.ts new file mode 100644 index 00000000..9497be04 --- /dev/null +++ b/packages/history/src/types.ts @@ -0,0 +1,160 @@ +import type { HistoryTranslations } from '@/i18n'; +import type { HistoryDatabase } from '@/store'; +import type { + Context, + Events, + ObsidianLanguageCode, + On, + SelectFromContext, + SettingEntry, + SyncTerminateReason, + Translate, + TranslationResource, +} from '@hesprs/sync-engine-sdk'; +import type { GlobStrategy } from '@repo/shared/glob-match'; +import type { App, Command, EventRef } from 'obsidian'; + +export type ChunkRef = { + hash: string; // Full SHA-256 of chunk bytes + offset: number; // Byte offset into the inflated payload + length: number; +}; + +export type BlobRef = { + hash: string; + key: string; // Unified key, NFC normalized + size: number; +}; + +export type PackRef = { + deviceId: string; + seq: number; + hash: string; + fromLamport: number; + toLamport: number; +}; + +/** Fixed prefix of every .pack. `hash` is also embedded in the filename. */ +export type PackHeader = { + magic: number; + version: number; + ns: string; + deviceId: string; + seq: number; + baseLamport: number; + peakLamport: number; + createdAt: number; // Unix ms + commitCount: number; + indexOffset: number; + indexLength: number; + payloadOffset: number; + payloadLength: number; + codec: Codec; + hash: string; +}; + +export type Codec = 'deflate' | 'raw'; + +/** One logical version. The chunk list IS the delta: unchanged chunks are references. */ +export type IndexEntry = { + keyHash: string; // SHA-256 of the normalized unified key + key: string; + lamport: number; + deviceId: string; + at: number; // Unix ms + parents: Array; // DAG parent commit hashes, empty for a root + chunks: Array; // Ordered; empty for a small-file or deletion + blobs: Array; + inlineOffset?: number; // Set when the note is stored as whole text + inlineLength?: number; + flags: number; // Bit 0 small-file, bit 1 deletion, bit 2 checkpoint + hash?: string; // Content fingerprint for fast zero-cost version deduplication +}; + +/** Per-device head log. Single writer; `generation` detects stale mirrors. */ +export type CommitLog = { + deviceId: string; + ns: string; + seq: number; // Last published pack seq + peakLamport: number; + generation: number; + packs: Array; // Recent window, oldest first + chunkIndex: Record; // Chunk hash → packRef key, recent window + heads: Array; // Current DAG tip commit hashes + updatedAt: number; + integrity: string; // SHA-256 of the log minus this field + remoteUid?: string; // Remote storage UID/ETag for zero-cost reconcile short-circuit +}; + +/** WAL record. Persisted before an edit is considered captured. */ +export type WalEntry = { + seq: number; + key: string; + lamport: number; + at: number; + parent: string; + parents?: Array; // Set for a DAG merge commit (multiple tips) + text?: string; // < 4 KiB note + chunkHashes?: Array; // >= 4 KiB note + newChunks?: Array<{ hash: string; bytes: Uint8Array }>; + blobs?: Array; + blobData?: Array<{ hash: string; bytes: Uint8Array }>; + flushedPack?: string; // Set only after head rewrite succeeds + flags?: number; +}; + +export type CommitLocation = { pack: PackRef; indexOffset: number; indexLength: number }; +export type ChunkLocation = { pack: PackRef; offset: number; length: number }; + +export type FlushReason = 'idle' | 'count' | 'volume' | 'lifecycle'; +export type VersionRef = { key: string; lamport?: number; commitHash?: string }; +export type RestoreResult = { + at: number; + binary?: Uint8Array; + key: string; + lamport: number; + merged: boolean; + text: string; +}; + +/** + * UI-facing version descriptor. `index` is per-file (1 = newest); `lamport` is the internal global + * ordering key. + */ +export type VersionSummary = { key: string; lamport: number; at: number; index: number }; + +export { + FLAG_CHECKPOINT, + FLAG_DELETION, + FLAG_SMALL_FILE, + PACK_HEADER_BYTES, + PACK_MAGIC, + PACK_VERSION, +} from '@/constants'; + +export type TogglableValue = { enabled: boolean; value: T }; + +export type HistoryContext = SelectFromContext<{ + addCommand: (command: Command) => Command; + app: App; + getNamespace: Context['getNamespace']; + indexedDB: HistoryDatabase; + on: On; + registerEvent: (ref: EventRef) => void; + registerI18n: (locale: ObsidianLanguageCode, resource: TranslationResource) => void; + registerSetting: (entry: SettingEntry) => () => boolean; + requestSync: (trigger: string) => Promise; + saveSettings: () => Promise; + settings: { + modules: Record; + realtimeSync?: TogglableValue; + syncStrategy?: Array; + }; + translate: Translate; +}>; + +export type PendingDebounce = { + data: Uint8Array; + firstScheduledAt: number; + timer: ReturnType; +}; diff --git a/packages/history/src/vault.ts b/packages/history/src/vault.ts new file mode 100644 index 00000000..cf7c5ca5 --- /dev/null +++ b/packages/history/src/vault.ts @@ -0,0 +1,79 @@ +import { isTextFile } from '@/chunker'; +import { TFile } from 'obsidian'; +import type { App } from 'obsidian'; + +export async function readFileBytes(app: App, file: TFile): Promise { + if (typeof app.vault.readBinary === 'function') { + const buf = await app.vault.readBinary(file); + return new Uint8Array(buf); + } + const content = await app.vault.read(file); + return new TextEncoder().encode(content); +} + +export async function readCurrentText(app: App, filePath: string): Promise { + if (!isTextFile(filePath)) return ''; + const existing = app.vault.getAbstractFileByPath(filePath); + if (existing instanceof TFile) + try { + return await app.vault.read(existing); + } catch { + return ''; + } + return ''; +} + +export async function writeVaultFile( + app: App, + filePath: string, + content: string, + binary?: Uint8Array, +): Promise { + const target = app.vault.getAbstractFileByPath(filePath); + const isText = isTextFile(filePath); + if (isText || !binary) + await (target instanceof TFile + ? app.vault.modify(target, content) + : app.vault.create(filePath, content)); + else if ( + typeof app.vault.modifyBinary === 'function' && + typeof app.vault.createBinary === 'function' + ) + await (target instanceof TFile + ? app.vault.modifyBinary(target, binary.buffer as ArrayBuffer) + : app.vault.createBinary(filePath, binary.buffer as ArrayBuffer)); + else { + const decoded = new TextDecoder().decode(binary); + await (target instanceof TFile + ? app.vault.modify(target, decoded) + : app.vault.create(filePath, decoded)); + } +} + +export function findDeletedKeys( + app: App, + locations: Array<[string, unknown]>, + isIgnored: (key: string) => boolean, +): Array { + return [ + ...new Set( + locations + .map(([key]) => key.split('@')[0]) + .filter((key) => !app.vault.getAbstractFileByPath(key) && !isIgnored(key)), + ), + ]; +} + +export const isRemoteWriteTask = (name: string): boolean => + name === 'download' || + name === 'resolveConflict' || + name === 'moveLocal' || + name === 'removeLocal'; + +export const namespaceOf = (ctx: { getNamespace: () => string | Error }): string | undefined => { + const ns = ctx.getNamespace(); + return ns instanceof Error ? undefined : ns; +}; + +/** Translate types function-valued keys as DocumentFragment; our translations return strings. */ +export const text = (value: unknown): string => (typeof value === 'string' ? value : String(value)); diff --git a/packages/history/src/wal.ts b/packages/history/src/wal.ts new file mode 100644 index 00000000..1262ec79 --- /dev/null +++ b/packages/history/src/wal.ts @@ -0,0 +1,57 @@ +import { captureContent } from '@/chunker'; +import { appendWal } from '@/store'; +import { FLAG_DELETION } from '@/types'; +import type { HistoryStores } from '@/store'; +import type { WalEntry } from '@/types'; + +/** Reads the current DAG tip for a key; the parent of a new local edit. */ +function parentOf(entries: ReadonlyArray, key: string): string { + for (let index = entries.length - 1; index >= 0; index--) { + const entry = entries[index]; + if (entry.key === key) return `${entry.key}@${entry.lamport}`; + } + return ''; +} + +/** + * Captures one whole-file snapshot into the WAL. The batch persist is awaited before the edit is + * considered captured, so a hard process kill after this resolves never loses the version. + */ +export async function ingestEdit( + stores: HistoryStores, + key: string, + data: Uint8Array, + seenLamport = 0, +): Promise { + const normalizedKey = key.normalize('NFC'); + const pending = await stores.pending.values(); + const parent = parentOf( + pending.sort((a, b) => a.seq - b.seq), + normalizedKey, + ); + const content = await captureContent(stores, normalizedKey, data); + return appendWal( + stores, + { at: Date.now(), key: normalizedKey, parent, ...content }, + seenLamport, + ); +} + +/** Captures a file deletion into the WAL. */ +export async function ingestDeletion( + stores: HistoryStores, + key: string, + seenLamport = 0, +): Promise { + const normalizedKey = key.normalize('NFC'); + const pending = await stores.pending.values(); + const parent = parentOf( + pending.sort((a, b) => a.seq - b.seq), + normalizedKey, + ); + return appendWal( + stores, + { at: Date.now(), flags: FLAG_DELETION, key: normalizedKey, parent }, + seenLamport, + ); +} diff --git a/packages/history/test/chunker.test.ts b/packages/history/test/chunker.test.ts new file mode 100644 index 00000000..a80e6a7f --- /dev/null +++ b/packages/history/test/chunker.test.ts @@ -0,0 +1,48 @@ +import { assembleChunks, captureContent } from '@/chunker'; +import { openHistoryStores } from '@/store'; +import { expect, test } from 'bun:test'; +import { openMemoryDB } from 'uni-kv'; +import type { HistoryDatabase } from '@/store'; + +const encoder = new TextEncoder(); + +function setup() { + const db = openMemoryDB(`history-chunker-${crypto.randomUUID()}`) as never as HistoryDatabase; + return openHistoryStores(db, 'ns-test'); +} + +test('assembleChunks concatenates ordered chunk bytes back into text', () => { + const parts = [encoder.encode('# Head'), encoder.encode('\n'), encoder.encode('Body')]; + expect(assembleChunks(parts)).toBe('# Head\nBody'); +}); + +test('a single-byte edit keeps downstream chunks identical', async () => { + const stores = setup(); + const original = new Uint8Array(32 * 1024); + let state = 3; + for (let i = 0; i < original.byteLength; i++) { + state = (Math.imul(state, 1_103_515_245) + 12_345) >>> 0; + original[i] = state & 0xff; + } + const edited = Uint8Array.from(original); + edited[8000] ^= 0xff; + const a = await captureContent(stores, 'note.md', original); + const b = await captureContent(stores, 'note.md', edited); + const aHashes = a.chunkHashes ?? []; + const bHashes = b.chunkHashes ?? []; + const shared = bHashes.filter((hash) => aHashes.includes(hash)); + expect(shared.length / bHashes.length).toBeGreaterThan(0.7); +}); + +test('captureContent treats non-text files as standalone blobs regardless of size', async () => { + const stores = setup(); + const smallImage = new Uint8Array([1, 2, 3, 4]); + const result = await captureContent(stores, 'photo.jpg', smallImage); + expect(result.text).toBeUndefined(); + expect(result.chunkHashes).toBeUndefined(); + expect(result.blobs).toHaveLength(1); + expect(result.blobs?.[0].key).toBe('photo.jpg'); + expect(result.blobs?.[0].size).toBe(4); + expect(result.blobData).toHaveLength(1); + expect(result.blobData?.[0].bytes).toStrictEqual(smallImage); +}); diff --git a/packages/history/test/dag.test.ts b/packages/history/test/dag.test.ts new file mode 100644 index 00000000..72253700 --- /dev/null +++ b/packages/history/test/dag.test.ts @@ -0,0 +1,56 @@ +import { compareCommits, linearize, mergeChunkSequences, mergeCommit, mergeTextLines } from '@/dag'; +import { expect, test } from 'bun:test'; +import type { IndexEntry } from '@/types'; + +function entry(overrides: Partial = {}): IndexEntry { + return { + at: 1000, + blobs: [], + chunks: [], + deviceId: 'a', + flags: 0, + key: 'note.md', + keyHash: 'kh', + lamport: 1, + parents: [], + ...overrides, + }; +} + +test('compareCommits gives a deterministic total order', () => { + const a = entry({ deviceId: 'a', keyHash: 'k', lamport: 1 }); + const b = entry({ deviceId: 'b', keyHash: 'k', lamport: 1 }); + const c = entry({ deviceId: 'a', keyHash: 'k', lamport: 2 }); + expect(compareCommits(a, b)).toBeLessThan(0); + expect(compareCommits(a, c)).toBeLessThan(0); + expect(linearize([c, b, a]).map((e) => e.lamport)).toStrictEqual([1, 1, 2]); +}); + +test('mergeTextLines takes a clean side when the other is unchanged', () => { + const o = 'a\nb\nc'; + expect(mergeTextLines(o, 'a\nB\nc', o)).toStrictEqual({ conflicted: false, text: 'a\nB\nc' }); + expect(mergeTextLines(o, o, 'a\nb\nC')).toStrictEqual({ conflicted: false, text: 'a\nb\nC' }); +}); + +test('mergeTextLines emits conflict markers when both sides diverge', () => { + const result = mergeTextLines('a\nb\nc', 'a\nL\nc', 'a\nR\nc'); + expect(result.conflicted).toBe(true); + expect(result.text).toContain('<<<<<<< ours'); + expect(result.text).toContain('>>>>>>> theirs'); +}); + +test('mergeChunkSequences reuses common chunk hashes', () => { + const o = ['c1', 'c2', 'c3']; + const a = ['c1', 'cX', 'c3']; + const b = ['c1', 'c2', 'c3']; + const result = mergeChunkSequences(o, a, b, mergeTextLines); + expect(result.tokens).toContain('c1'); + expect(result.tokens).toContain('c3'); +}); + +test('mergeCommit names both tips as parents', () => { + const a = entry({ deviceId: 'a', lamport: 2 }); + const b = entry({ deviceId: 'b', lamport: 3 }); + const merged = mergeCommit({ a, b, deviceId: 'a', key: 'note.md', lamport: 4 }); + expect(merged.parents).toStrictEqual(['note.md@2', 'note.md@3']); +}); diff --git a/packages/history/test/encryption.test.ts b/packages/history/test/encryption.test.ts new file mode 100644 index 00000000..144970c7 --- /dev/null +++ b/packages/history/test/encryption.test.ts @@ -0,0 +1,191 @@ +import { createCipher, createHistoryCipher } from '@/crypto'; +import { buildCommitLog } from '@/head-log'; +import { openRemoteHistory } from '@/remote'; +import { expect, test } from 'bun:test'; +import type { RemoteHistory } from '@/remote'; +import type { Binary, FileStat, FolderStat, Fs, ListReporter, Stat } from '@hesprs/sync-engine-sdk'; +import type { App } from 'obsidian'; + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +function mustRemote( + fs: Fs, + namespace: string | Error, + deviceId: string, + cipher?: Awaited>, +): RemoteHistory { + const remote = openRemoteHistory(fs, namespace, deviceId, cipher); + if (!remote) throw new Error('expected a remote history handle'); + return remote; +} + +function fakeApp(secrets: Record): App { + return { secretStorage: { getSecret: (id: string) => secrets[id] } } as unknown as App; +} + +test('a cipher round-trips bytes and the ciphertext does not contain plaintext', async () => { + const cipher = await createCipher('correct horse battery staple'); + const plaintext = encoder.encode('# Secret note\nprivate content'); + const encrypted = await cipher.encrypt(plaintext); + expect(decoder.decode(encrypted)).not.toContain('private content'); + expect(await cipher.decrypt(encrypted)).toStrictEqual(plaintext); +}); + +test('decrypting unencrypted data (no magic) throws an error', async () => { + const cipher = await createCipher('pw'); + const plaintext = encoder.encode('plaintext without magic'); + expect(cipher.decrypt(plaintext)).rejects.toThrow('missing encryption magic'); +}); + +test('a wrong password fails loudly instead of producing garbage', async () => { + const cipher = await createCipher('password-a'); + const other = await createCipher('password-b'); + const encrypted = await cipher.encrypt(encoder.encode('secret')); + let message = ''; + try { + await other.decrypt(encrypted); + } catch (error) { + message = (error as Error).message; + } + expect(message).toContain('password changed'); +}); + +test('encryption off yields no cipher; on with a password yields a cipher', async () => { + expect( + await createHistoryCipher({ encryption: { enabled: false } }, fakeApp({})), + ).toBeUndefined(); + const modules = { encryption: { enabled: true, password: 'secret-id' } }; + const cipher = await createHistoryCipher(modules, fakeApp({ 'secret-id': 'pw' })); + expect(cipher).toBeDefined(); +}); + +test('encryption on without a password throws rather than writing plaintext', async () => { + const modules = { encryption: { enabled: true, password: 'missing' } }; + let threw = false; + try { + await createHistoryCipher(modules, fakeApp({})); + } catch { + threw = true; + } + expect(threw).toBe(true); +}); + +/** Minimal in-memory RootFs: enough for digOriginal + prefixWrapper + read/write/list. */ +function fakeFs() { + const store = new Map(); + const listStats = (key: string): Array => + [...store.entries()] + .filter(([name]) => name.startsWith(key)) + .map(([name, value]) => ({ + isDir: false as const, + key: name, + meta: () => ({}), + mtime: 0, + size: value.byteLength, + uid: 'u', + })); + const fs: Fs = { + delete(key) { + store.delete(key); + return Promise.resolve(); + }, + exists: (key) => Promise.resolve(store.has(key)), + getUid: () => 'root', + list: (key: string, _reporter: ListReporter) => Promise.resolve(listStats(key)), + mkdir: (_key: string, _stat: FolderStat) => Promise.resolve(), + move(oldKey, newKey) { + const value = store.get(oldKey); + if (value) { + store.set(newKey, value); + store.delete(oldKey); + } + return Promise.resolve(); + }, + read: (key: string) => Promise.resolve((store.get(key) ?? new Uint8Array()) as Binary), + readStream(key: string) { + return Promise.resolve( + new ReadableStream({ + start(controller) { + controller.enqueue((store.get(key) ?? new Uint8Array()) as Binary); + controller.close(); + }, + }), + ); + }, + stat: (key: string): Promise => { + const value = store.get(key); + if (!value) return Promise.reject(new Error(`missing ${key}`)); + return Promise.resolve({ + isDir: false, + key, + meta: () => ({}), + mtime: 0, + size: value.byteLength, + uid: 'u', + }); + }, + write: async (key: string, value) => { + store.set(key, new Uint8Array(await value)); + return 'uid'; + }, + async writeStream(key: string, value) { + const reader = (await value).getReader(); + const parts: Array = []; + for (;;) { + const { done, value: chunk } = await reader.read(); + if (done) break; + parts.push(chunk); + } + let total = 0; + for (const part of parts) total += part.byteLength; + const joined = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + joined.set(part, offset); + offset += part.byteLength; + } + store.set(key, joined); + return 'uid'; + }, + }; + return { fs, store }; +} + +test('without a cipher the remote handle writes plaintext (backward compatible)', async () => { + const { fs, store } = fakeFs(); + const remote = mustRemote(fs, 'ns', 'device-a'); + await remote.putBlob('hash1', encoder.encode('plaintext blob')); + const stored = [...store.values()][0]; + expect(decoder.decode(stored)).toContain('plaintext blob'); +}); + +test('with a cipher the stored object is encrypted but reads back as plaintext', async () => { + const { fs, store } = fakeFs(); + const cipher = await createCipher('pw'); + const remote = mustRemote(fs, 'ns', 'device-a', cipher); + const blob = encoder.encode('secret attachment content'); + await remote.putBlob('hash2', blob); + const stored = [...store.values()][0]; + expect(decoder.decode(stored)).not.toContain('secret attachment content'); + expect(await remote.getObject('ns/blobs/hash2.blob')).toStrictEqual(blob); +}); + +test('head logs survive encryption (write then read)', async () => { + const { fs } = fakeFs(); + const cipher = await createCipher('pw'); + const remote = mustRemote(fs, 'ns', 'device-a', cipher); + const log = await buildCommitLog({ + chunkIndex: {}, + deviceId: 'device-a', + generation: 1, + heads: ['note.md@1'], + ns: 'ns', + packs: [], + peakLamport: 1, + seq: 1, + updatedAt: 1000, + }); + await remote.writeHead(log); + expect(await remote.readHead('device-a')).toStrictEqual(log); +}); diff --git a/packages/history/test/flush.test.ts b/packages/history/test/flush.test.ts new file mode 100644 index 00000000..fcb8a46b --- /dev/null +++ b/packages/history/test/flush.test.ts @@ -0,0 +1,482 @@ +import { flushPending, flushPendingBatch, FlushScheduler, reconcile } from '@/flush'; +import { decodeHeadLog } from '@/head-log'; +import { readLamport, openHistoryStores, readPending } from '@/store'; +import { ingestEdit } from '@/wal'; +import { expect, test } from 'bun:test'; +import { openMemoryDB } from 'uni-kv'; +import type { FlushContext } from '@/flush'; +import type { RemoteHistory } from '@/remote'; +import type { HistoryDatabase } from '@/store'; +import type { CommitLog, WalEntry } from '@/types'; + +const encoder = new TextEncoder(); + +async function expectAsyncThrow(promise: Promise, message: string): Promise { + try { + await promise; + } catch (error) { + expect((error as Error).message).toBe(message); + return; + } + throw new Error(`Expected rejection: ${message}`); +} + +type Recorded = { kind: string; key: string }; + +function fakeRemote() { + const written: Array = []; + const objects = new Map(); + const heads = new Map(); + const remote: RemoteHistory = { + deletePack(_deviceId, fileName) { + written.push({ key: fileName, kind: 'delete' }); + objects.delete(`ns/packs/device-a/${fileName}`); + return Promise.resolve(); + }, + deviceId: 'device-a', + fs: {} as never, + getObject(key) { + const value = objects.get(key); + return value ? Promise.resolve(value) : Promise.reject(new Error(`missing ${key}`)); + }, + listHeads() { + return Promise.resolve([...heads.keys()].map((deviceId) => ({ deviceId }))); + }, + listPacks(deviceId = 'device-a') { + return Promise.resolve( + [...objects.keys()] + .filter( + (key) => key.startsWith(`ns/packs/${deviceId}/`) && key.endsWith('.pack'), + ) + .map((key) => key.split('/').at(-1) ?? key), + ); + }, + ns: 'ns', + putBlob(hash, bytes) { + written.push({ key: hash, kind: 'blob' }); + objects.set(`ns/blobs/${hash}.blob`, bytes); + return Promise.resolve(); + }, + putPack(seq, hash, bytes) { + written.push({ key: `p-${seq}-${hash}.pack`, kind: 'pack' }); + objects.set(`ns/packs/device-a/p-${seq}-${hash}.pack`, bytes); + return Promise.resolve(); + }, + readHead(id): Promise { + const bytes = heads.get(id); + return bytes + ? decodeHeadLog(bytes) + : Promise.resolve(undefined as CommitLog | undefined); + }, + writeHead(log: CommitLog) { + written.push({ key: log.deviceId, kind: 'head' }); + heads.set(log.deviceId, new TextEncoder().encode(JSON.stringify(log))); + return Promise.resolve(); + }, + }; + return { heads, objects, remote, written }; +} + +function setup() { + const db = openMemoryDB(`history-flush-${crypto.randomUUID()}`) as never as HistoryDatabase; + const stores = openHistoryStores(db, 'ns'); + const fake = fakeRemote(); + const ctx: FlushContext = { remote: fake.remote, requestSync: () => {}, stores }; + return { ...fake, ctx, stores }; +} + +test('flush writes in the order blobs → pack → head → WAL commit', async () => { + const { ctx, written, stores } = setup(); + await ingestEdit(stores, 'note.md', encoder.encode('one')); + await ingestEdit(stores, 'note.md', encoder.encode('two')); + expect(await flushPending(ctx, 'count')).toBe(true); + expect(written.map((w) => w.kind)).toStrictEqual(['pack', 'head']); + expect(await readPending(stores)).toStrictEqual([]); + expect(await readLamport(stores)).toBe(2); +}); + +test('a successful flush records commit locations for O(1) restore', async () => { + const { ctx, stores } = setup(); + await ingestEdit(stores, 'note.md', encoder.encode('one')); + await flushPending(ctx, 'idle'); + expect(await stores.index.get('note.md@1')).toBeDefined(); +}); + +test('a head-write failure keeps the WAL pending for idempotent replay', async () => { + const { ctx, remote, stores } = setup(); + await ingestEdit(stores, 'note.md', encoder.encode('one')); + remote.writeHead = () => Promise.reject(new Error('network')); + await expectAsyncThrow(flushPending(ctx, 'idle'), 'network'); + expect(await readPending(stores)).toHaveLength(1); +}); + +test('replaying the same WAL batch reproduces the identical pack name (idempotent)', async () => { + const { ctx, written, stores, remote } = setup(); + await ingestEdit(stores, 'note.md', encoder.encode('same')); + const realWriteHead = remote.writeHead; + remote.writeHead = () => Promise.reject(new Error('crash before head')); + await expectAsyncThrow(flushPending(ctx, 'idle'), 'crash before head'); + remote.writeHead = realWriteHead; + // The WAL was kept, so the replay is byte-identical and targets the same pack name. + await flushPending(ctx, 'idle'); + const packNames = written.filter((w) => w.kind === 'pack').map((w) => w.key); + expect(packNames).toHaveLength(2); + expect(packNames[1]).toBe(packNames[0]); +}); + +test('reconcile promotes an orphan pack into the commit index', async () => { + const { ctx, objects, remote, stores } = setup(); + await ingestEdit(stores, 'note.md', encoder.encode('orphan')); + // Pack PUT succeeds but head rewrite never happens. + const { encodePack } = await import('@/pack'); + const { keyHash } = await import('@/chunker'); + const encoded = await encodePack( + { baseLamport: 1, createdAt: 1000, deviceId: 'device-a', ns: 'ns', peakLamport: 1, seq: 1 }, + [ + { + at: 1000, + blobs: [], + chunks: [], + deviceId: 'device-a', + flags: 1, + inlineLength: 6, + inlineOffset: 0, + key: 'note.md', + keyHash: await keyHash('note.md'), + lamport: 1, + parents: [], + }, + ], + encoder.encode('orphan'), + ); + objects.set(`ns/packs/device-a/p-1-${encoded.header.hash}.pack`, encoded.bytes); + const pending: Array = await readPending(stores); + for (const entry of pending) await stores.pending.delete(String(entry.seq)); + // Seed a head log whose pack window does not yet list the orphan pack. + const { buildCommitLog } = await import('@/head-log'); + const log = await buildCommitLog({ + chunkIndex: {}, + deviceId: 'device-a', + generation: 0, + heads: [], + ns: 'ns', + packs: [], + peakLamport: 0, + seq: 0, + updatedAt: 1000, + }); + await remote.writeHead(log); + await reconcile(ctx); + expect(await stores.index.get('note.md@1')).toBeDefined(); +}); + +test('a bounded flush caps the batch and leaves the remainder queued', async () => { + const { ctx, stores } = setup(); + for (let index = 0; index < 3; index++) + await ingestEdit(stores, 'note.md', encoder.encode('x'.repeat(100))); + expect(await readPending(stores)).toHaveLength(3); + // A 150-byte cap fits one ~100-byte entry; the rest must stay in the WAL. + await flushPendingBatch(ctx, 'lifecycle', 150); + expect(await readPending(stores)).toHaveLength(2); +}); + +test('onUnload performs a best-effort bounded flush without clearing all edits', async () => { + const { ctx, stores } = setup(); + await ingestEdit(stores, 'note.md', encoder.encode('one')); + const scheduler = new FlushScheduler(ctx); + scheduler.onEdit(3); + expect(await scheduler.onUnload()).toBeUndefined(); + expect(await readPending(stores)).toHaveLength(0); + scheduler.dispose(); +}); + +test('reconcile folds a remote head peak Lamport into the local clock', async () => { + const { ctx, remote, stores } = setup(); + const { buildCommitLog } = await import('@/head-log'); + const log = await buildCommitLog({ + chunkIndex: {}, + deviceId: 'device-b', + generation: 2, + heads: [], + ns: 'ns', + packs: [], + peakLamport: 42, + seq: 5, + updatedAt: 1000, + }); + await remote.writeHead(log); + await reconcile(ctx); + const next = await ingestEdit(stores, 'note.md', encoder.encode('after')); + expect(next.lamport).toBeGreaterThan(42); +}); + +test('reconcile skips downloading packs when cached head generation is up to date', async () => { + const { ctx, objects, remote } = setup(); + const { encodePack, packFileName } = await import('@/pack'); + const { keyHash } = await import('@/chunker'); + const { buildCommitLog } = await import('@/head-log'); + + const encoded = await encodePack( + { baseLamport: 1, createdAt: 1000, deviceId: 'device-b', ns: 'ns', peakLamport: 1, seq: 1 }, + [ + { + at: 1000, + blobs: [], + chunks: [], + deviceId: 'device-b', + flags: 1, + inlineLength: 4, + inlineOffset: 0, + key: 'test.md', + keyHash: await keyHash('test.md'), + lamport: 1, + parents: [], + }, + ], + encoder.encode('test'), + ); + const fileName = packFileName('device-b', 1, encoded.header.hash); + objects.set(`ns/packs/device-b/${fileName}`, encoded.bytes); + + const log = await buildCommitLog({ + chunkIndex: {}, + deviceId: 'device-b', + generation: 1, + heads: [], + ns: 'ns', + packs: [ + { + deviceId: 'device-b', + fromLamport: 1, + hash: encoded.header.hash, + seq: 1, + toLamport: 1, + }, + ], + peakLamport: 1, + seq: 1, + updatedAt: 1000, + }); + await remote.writeHead(log); + + let getObjectCalls = 0; + const origGetObject = remote.getObject; + remote.getObject = (key: string) => { + getObjectCalls++; + return origGetObject(key); + }; + + // First reconcile: indexes pack 1 + await reconcile(ctx); + expect(getObjectCalls).toBe(1); + + // Second reconcile with same generation: must NOT download pack again + await reconcile(ctx); + expect(getObjectCalls).toBe(1); +}); + +test('reconcile downloads only newly added packs when generation advances', async () => { + const { ctx, objects, remote } = setup(); + const { encodePack, packFileName } = await import('@/pack'); + const { keyHash } = await import('@/chunker'); + const { buildCommitLog } = await import('@/head-log'); + + const pack1 = await encodePack( + { baseLamport: 1, createdAt: 1000, deviceId: 'device-b', ns: 'ns', peakLamport: 1, seq: 1 }, + [ + { + at: 1000, + blobs: [], + chunks: [], + deviceId: 'device-b', + flags: 1, + inlineLength: 4, + inlineOffset: 0, + key: 'test.md', + keyHash: await keyHash('test.md'), + lamport: 1, + parents: [], + }, + ], + encoder.encode('one'), + ); + const fileName1 = packFileName('device-b', 1, pack1.header.hash); + objects.set(`ns/packs/device-b/${fileName1}`, pack1.bytes); + + const pack2 = await encodePack( + { baseLamport: 2, createdAt: 2000, deviceId: 'device-b', ns: 'ns', peakLamport: 2, seq: 2 }, + [ + { + at: 2000, + blobs: [], + chunks: [], + deviceId: 'device-b', + flags: 1, + inlineLength: 4, + inlineOffset: 0, + key: 'test.md', + keyHash: await keyHash('test.md'), + lamport: 2, + parents: [], + }, + ], + encoder.encode('two'), + ); + const fileName2 = packFileName('device-b', 2, pack2.header.hash); + + const log1 = await buildCommitLog({ + chunkIndex: {}, + deviceId: 'device-b', + generation: 1, + heads: [], + ns: 'ns', + packs: [ + { deviceId: 'device-b', fromLamport: 1, hash: pack1.header.hash, seq: 1, toLamport: 1 }, + ], + peakLamport: 1, + seq: 1, + updatedAt: 1000, + }); + await remote.writeHead(log1); + + const downloadedKeys: Array = []; + const origGetObject = remote.getObject; + remote.getObject = (key: string) => { + downloadedKeys.push(key); + return origGetObject(key); + }; + + await reconcile(ctx); + expect(downloadedKeys).toHaveLength(1); + expect(downloadedKeys[0]).toContain(fileName1); + + // Now remote writes pack 2 and advances generation to 2 + objects.set(`ns/packs/device-b/${fileName2}`, pack2.bytes); + const log2 = await buildCommitLog({ + chunkIndex: {}, + deviceId: 'device-b', + generation: 2, + heads: [], + ns: 'ns', + packs: [ + { deviceId: 'device-b', fromLamport: 1, hash: pack1.header.hash, seq: 1, toLamport: 1 }, + { deviceId: 'device-b', fromLamport: 2, hash: pack2.header.hash, seq: 2, toLamport: 2 }, + ], + peakLamport: 2, + seq: 2, + updatedAt: 2000, + }); + await remote.writeHead(log2); + + downloadedKeys.length = 0; + await reconcile(ctx); + // Should only download pack 2, NOT pack 1! + expect(downloadedKeys).toHaveLength(1); + expect(downloadedKeys[0]).toContain(fileName2); +}); + +test('reconcile deduplicates concurrent calls', async () => { + const { ctx, remote } = setup(); + let listHeadsCalls = 0; + const origListHeads = remote.listHeads; + remote.listHeads = async () => { + listHeadsCalls++; + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); + return origListHeads(); + }; + + await Promise.all([reconcile(ctx), reconcile(ctx)]); + expect(listHeadsCalls).toBe(1); +}); + +test('FlushScheduler negotiates with realtimeSync config', async () => { + const { ctx, stores } = setup(); + await ingestEdit(stores, 'note.md', encoder.encode('one')); + + let syncTrigger = ''; + const requestSync = (trigger: string) => { + syncTrigger = trigger; + }; + + // 1. When realtimeSync is explicitly disabled, fast channel stays silent + const disabledCtx = { + ...ctx, + getRealtimeSyncConfig: () => ({ enabled: false, value: 30_000 }), + requestSync, + }; + const disabledScheduler = new FlushScheduler(disabledCtx); + disabledScheduler.onEdit(3); + + // Wait 50ms - should not fire fastTimer + await new Promise((resolve) => { + setTimeout(resolve, 50); + }); + expect(syncTrigger).toBe(''); + disabledScheduler.dispose(); + + // 2. When realtimeSync is enabled with a value, respects it + syncTrigger = ''; + const enabledCtx = { + ...ctx, + getRealtimeSyncConfig: () => ({ enabled: true, value: 50 }), + requestSync, + }; + const enabledScheduler = new FlushScheduler(enabledCtx); + enabledScheduler.onEdit(3); + + // Wait 70ms (> 50ms test value, but FAST_SYNC_DEBOUNCE_MS floor is 1000ms) + // So at 70ms it should NOT have fired because Math.max(1000, 50) = 1000ms! + await new Promise((resolve) => { + setTimeout(resolve, 70); + }); + expect(syncTrigger).toBe(''); + enabledScheduler.dispose(); +}); + +test('reconcile skips readHead when remote head uid matches cached remoteUid', async () => { + const { ctx, remote, stores } = setup(); + const { buildCommitLog } = await import('@/head-log'); + + const log = await buildCommitLog({ + chunkIndex: {}, + deviceId: 'device-b', + generation: 1, + heads: [], + ns: 'ns', + packs: [], + peakLamport: 5, + seq: 1, + updatedAt: 1000, + }); + await remote.writeHead(log); + + let currentUid: string | undefined = 'etag-v1'; + remote.listHeads = () => Promise.resolve([{ deviceId: 'device-b', uid: currentUid }]); + + let readHeadCalls = 0; + const origReadHead = remote.readHead; + remote.readHead = (id: string) => { + readHeadCalls++; + return origReadHead(id); + }; + + // 1. First reconcile: reads head, caches remoteUid + await reconcile(ctx); + expect(readHeadCalls).toBe(1); + const cached1 = await stores.heads.get('device-b'); + expect(cached1?.remoteUid).toBe('etag-v1'); + expect(await readLamport(stores)).toBeGreaterThanOrEqual(5); + + // 2. Second reconcile with identical uid: skips readHead completely! + await reconcile(ctx); + expect(readHeadCalls).toBe(1); + + // 3. Third reconcile with updated uid: reads head and updates remoteUid + currentUid = 'etag-v2'; + await reconcile(ctx); + expect(readHeadCalls).toBe(2); + const cached2 = await stores.heads.get('device-b'); + expect(cached2?.remoteUid).toBe('etag-v2'); +}); diff --git a/packages/history/test/gc.test.ts b/packages/history/test/gc.test.ts new file mode 100644 index 00000000..66f37809 --- /dev/null +++ b/packages/history/test/gc.test.ts @@ -0,0 +1,259 @@ +import { keyHash } from '@/chunker'; +import { compact } from '@/gc'; +import { buildCommitLog } from '@/head-log'; +import { encodePack, packFileName } from '@/pack'; +import { openHistoryStores } from '@/store'; +import { expect, test } from 'bun:test'; +import { openMemoryDB } from 'uni-kv'; +import type { RemoteHistory } from '@/remote'; +import type { HistoryDatabase } from '@/store'; + +const encoder = new TextEncoder(); + +function setup() { + const db = openMemoryDB(`history-gc-${crypto.randomUUID()}`) as never as HistoryDatabase; + const stores = openHistoryStores(db, 'ns'); + const objects = new Map(); + const deleted: Array = []; + const deletedCalls: Array<{ deviceId: string; fileName: string }> = []; + const put: Array = []; + const remote: RemoteHistory = { + deletePack: (deviceId, fileName) => { + deleted.push(fileName); + deletedCalls.push({ deviceId, fileName }); + return Promise.resolve(); + }, + deviceId: 'device-a', + fs: {} as never, + getObject: (key) => { + const value = objects.get(key); + return value ? Promise.resolve(value) : Promise.reject(new Error(`missing ${key}`)); + }, + listHeads: () => Promise.resolve([]), + listPacks: () => Promise.resolve([]), + ns: 'ns', + putBlob: () => Promise.resolve(), + putPack: (_seq, hash) => { + put.push(hash); + return Promise.resolve(); + }, + readHead: () => Promise.resolve(undefined as never), + writeHead: () => Promise.resolve(), + }; + return { deleted, deletedCalls, objects, put, remote, stores }; +} + +async function seed( + setupReturn: ReturnType, + text: string, + at: number, + options: { deviceId?: string; key?: string; lamport?: number; seq?: number } = {}, +) { + const { objects, stores } = setupReturn; + const deviceId = options.deviceId ?? 'device-a'; + const key = options.key ?? 'note.md'; + const lamport = options.lamport ?? 1; + const seq = options.seq ?? 1; + const bytes = encoder.encode(text); + const encoded = await encodePack( + { baseLamport: lamport, createdAt: at, deviceId, ns: 'ns', peakLamport: lamport, seq }, + [ + { + at, + blobs: [], + chunks: [], + deviceId, + flags: 1, + inlineLength: bytes.byteLength, + inlineOffset: 0, + key, + keyHash: await keyHash(key), + lamport, + parents: [], + }, + ], + bytes, + ); + const fileName = packFileName(deviceId, seq, encoded.header.hash); + objects.set(`ns/packs/${deviceId}/${fileName}`, encoded.bytes); + await stores.index.set(`${key}@${lamport}`, { + indexLength: encoded.header.indexLength, + indexOffset: encoded.header.indexOffset, + pack: { + deviceId, + fromLamport: lamport, + hash: encoded.header.hash, + seq, + toLamport: lamport, + }, + }); + const log = await buildCommitLog({ + chunkIndex: {}, + deviceId, + generation: 0, + heads: [], + ns: 'ns', + packs: [], + peakLamport: lamport, + seq, + updatedAt: Date.now(), + }); + await stores.heads.set(deviceId, log); + return { fileName, hash: encoded.header.hash }; +} + +test('compact rewrites live versions into a new pack and deletes obsolete packs', async () => { + const setupReturn = setup(); + await seed(setupReturn, 'live text', Date.now()); + const { deleted, put, remote, stores } = setupReturn; + const removed = await compact({ remote, stores }, 90); + expect(put).toHaveLength(1); + expect(removed).toBe(0); // The seeded pack is still live (within retention). + expect(deleted).toStrictEqual([]); +}); + +test('compact never mutates an existing pack in place', async () => { + const setupReturn = setup(); + const { hash } = await seed(setupReturn, 'live text', Date.now()); + const { put } = setupReturn; + await compact({ remote: setupReturn.remote, stores: setupReturn.stores }, 90); + expect(put[0]).not.toBe(hash); +}); + +test('compact enforces maxVersions per note and removes packs with only pruned versions', async () => { + const setupReturn = setup(); + const now = Date.now(); + const p1 = await seed(setupReturn, 'v1', now - 3000, { lamport: 1, seq: 1 }); + await seed(setupReturn, 'v2', now - 2000, { lamport: 2, seq: 2 }); + await seed(setupReturn, 'v3', now - 1000, { lamport: 3, seq: 3 }); + const { deleted, remote, stores } = setupReturn; + const removed = await compact({ remote, stores }, { maxVersions: 2, retentionDays: 90 }); + expect(removed).toBe(1); + expect(deleted).toContain(p1.fileName); + expect(await stores.index.get('note.md@1')).toBeUndefined(); + expect(await stores.index.get('note.md@2')).toBeDefined(); + expect(await stores.index.get('note.md@3')).toBeDefined(); +}); + +test('compact supports multi-device packs and deletes with the correct deviceId', async () => { + const setupReturn = setup(); + const now = Date.now(); + const p1 = await seed(setupReturn, 'v1', now - 3000, { + deviceId: 'device-b', + lamport: 1, + seq: 1, + }); + await seed(setupReturn, 'v2', now - 2000, { deviceId: 'device-a', lamport: 2, seq: 2 }); + await seed(setupReturn, 'v3', now - 1000, { deviceId: 'device-a', lamport: 3, seq: 3 }); + const { deletedCalls, remote, stores } = setupReturn; + const removed = await compact({ remote, stores }, { maxVersions: 2, retentionDays: 90 }); + expect(removed).toBe(1); + expect(deletedCalls).toContainEqual({ deviceId: 'device-b', fileName: p1.fileName }); + expect(await stores.index.get('note.md@1')).toBeUndefined(); + expect(await stores.index.get('note.md@2')).toBeDefined(); +}); + +test('compact tolerates remote missing packs (NoSuchKey) and cleans up stale index entries', async () => { + const setupReturn = setup(); + const now = Date.now(); + const ghost = await seed(setupReturn, 'ghost', now - 3000, { + key: 'ghost.md', + lamport: 1, + seq: 1, + }); + setupReturn.objects.delete(`ns/packs/device-a/${ghost.fileName}`); + await seed(setupReturn, 'live', now - 1000, { key: 'live.md', lamport: 2, seq: 2 }); + const { remote, stores } = setupReturn; + const removed = await compact({ remote, stores }, { maxVersions: 10, retentionDays: 90 }); + expect(removed).toBe(1); + expect(await stores.index.get('ghost.md@1')).toBeUndefined(); + expect(await stores.index.get('live.md@2')).toBeDefined(); +}); + +test('compact tolerates deletePack failure (NoSuchKey) when pack is already deleted', async () => { + const setupReturn = setup(); + const now = Date.now(); + await seed(setupReturn, 'v1', now - 3000, { lamport: 1, seq: 1 }); + await seed(setupReturn, 'v2', now - 2000, { lamport: 2, seq: 2 }); + await seed(setupReturn, 'v3', now - 1000, { lamport: 3, seq: 3 }); + const { remote, stores } = setupReturn; + remote.deletePack = () => Promise.reject(new Error('S3 NoSuchKey: missing key')); + const removed = await compact({ remote, stores }, { maxVersions: 2, retentionDays: 90 }); + expect(removed).toBe(0); +}); + +test('compact with tiered retention preserves older daily versions even when same-day burst exceeds maxVersions', async () => { + const setupReturn = setup(); + const now = Date.now(); + const dayMs = 86_400_000; + // 1 version 10 days ago + await seed(setupReturn, 'v-10d', now - 10 * dayMs, { lamport: 1, seq: 1 }); + // 1 version yesterday + await seed(setupReturn, 'v-yesterday', now - dayMs, { lamport: 2, seq: 2 }); + // 5 rapid edits today (within last 10 minutes) + for (let i = 0; i < 5; i++) + await seed(setupReturn, `v-today-${i}`, now - 600_000 + i * 1000, { + lamport: 3 + i, + seq: 3 + i, + }); + + const { remote, stores } = setupReturn; + await compact({ remote, stores }, { maxVersions: 3, now, retentionDays: 90, tiered: true }); + + // Historical versions MUST survive despite the burst of edits today + expect(await stores.index.get('note.md@1')).toBeDefined(); + expect(await stores.index.get('note.md@2')).toBeDefined(); + // The newest version from today is kept + expect(await stores.index.get('note.md@7')).toBeDefined(); + // Older versions from today are pruned to respect maxVersions budget + expect(await stores.index.get('note.md@3')).toBeUndefined(); + expect(await stores.index.get('note.md@4')).toBeUndefined(); + expect(await stores.index.get('note.md@5')).toBeUndefined(); + expect(await stores.index.get('note.md@6')).toBeUndefined(); +}); + +test('compact with tiered retention thins multiple edits on past days down to 1 milestone per day', async () => { + const setupReturn = setup(); + const now = Date.now(); + const dayMs = 86_400_000; + // 3 versions from 3 days ago (e.g. morning, afternoon, evening) + await seed(setupReturn, 'v-3d-morning', now - 3 * dayMs - 3_600_000, { lamport: 1, seq: 1 }); + await seed(setupReturn, 'v-3d-afternoon', now - 3 * dayMs - 1_800_000, { lamport: 2, seq: 2 }); + await seed(setupReturn, 'v-3d-evening', now - 3 * dayMs, { lamport: 3, seq: 3 }); + // 1 version today + await seed(setupReturn, 'v-today', now - 1000, { lamport: 4, seq: 4 }); + + const { remote, stores } = setupReturn; + await compact({ remote, stores }, { maxVersions: 10, now, retentionDays: 90, tiered: true }); + + // The latest version of 3 days ago is kept as the daily milestone + expect(await stores.index.get('note.md@3')).toBeDefined(); + // Intermediate edits from 3 days ago are thinned out + expect(await stores.index.get('note.md@1')).toBeUndefined(); + expect(await stores.index.get('note.md@2')).toBeUndefined(); + // Today's version is kept + expect(await stores.index.get('note.md@4')).toBeDefined(); +}); + +test('compact with tiered: false falls back to flat newest-first pruning', async () => { + const setupReturn = setup(); + const now = Date.now(); + const dayMs = 86_400_000; + await seed(setupReturn, 'v-10d', now - 10 * dayMs, { lamport: 1, seq: 1 }); + await seed(setupReturn, 'v-yesterday', now - dayMs, { lamport: 2, seq: 2 }); + for (let i = 0; i < 5; i++) + await seed(setupReturn, `v-today-${i}`, now - 600_000 + i * 1000, { + lamport: 3 + i, + seq: 3 + i, + }); + + const { remote, stores } = setupReturn; + await compact({ remote, stores }, { maxVersions: 3, now, retentionDays: 90, tiered: false }); + + // In flat mode, the 3 newest versions are kept, crowding out older history + expect(await stores.index.get('note.md@1')).toBeUndefined(); + expect(await stores.index.get('note.md@2')).toBeUndefined(); + expect(await stores.index.get('note.md@5')).toBeDefined(); + expect(await stores.index.get('note.md@6')).toBeDefined(); + expect(await stores.index.get('note.md@7')).toBeDefined(); +}); diff --git a/packages/history/test/head-log.test.ts b/packages/history/test/head-log.test.ts new file mode 100644 index 00000000..c68630ff --- /dev/null +++ b/packages/history/test/head-log.test.ts @@ -0,0 +1,38 @@ +import { buildCommitLog, decodeHeadLog, encodeHeadLog } from '@/head-log'; +import { expect, test } from 'bun:test'; + +async function log() { + return buildCommitLog({ + chunkIndex: { abc: 'd:1:h' }, + deviceId: 'device-a', + generation: 0, + heads: ['note.md@3'], + ns: 'ns', + packs: [], + peakLamport: 3, + seq: 1, + updatedAt: 1000, + }); +} + +test('a head log round-trips and verifies its integrity hash', async () => { + const value = await log(); + const decoded = await decodeHeadLog(encodeHeadLog(value)); + expect(decoded).toStrictEqual(value); +}); + +test('a tampered log fails integrity verification', async () => { + const value = await log(); + const tampered = { ...value, peakLamport: 999 }; + expect(await decodeHeadLog(encodeHeadLog(tampered))).toBeUndefined(); +}); + +test('generation bump produces a different integrity hash', async () => { + const a = await log(); + const b = await buildCommitLog({ ...a, generation: 1 }); + expect(b.integrity).not.toBe(a.integrity); +}); + +test('malformed bytes decode to undefined', async () => { + expect(await decodeHeadLog(new TextEncoder().encode('not json'))).toBeUndefined(); +}); diff --git a/packages/history/test/i18n.test.ts b/packages/history/test/i18n.test.ts new file mode 100644 index 00000000..dfda8a8b --- /dev/null +++ b/packages/history/test/i18n.test.ts @@ -0,0 +1,18 @@ +import { en, ru, zh, zhTW } from '@/i18n'; +import { expect, test } from 'bun:test'; + +test('every locale prefixes the file-menu item with Sync Engine', () => { + for (const locale of [en, zh, zhTW, ru]) + expect(locale.showHistory.toLowerCase()).toContain('sync engine'); +}); + +test('version labels are timestamps, not global version numbers', () => { + const at = Date.UTC(2026, 10, 10, 13, 29, 39); + const label = en.version({ at }); + expect(label).not.toMatch(/version\s+\d/iu); + expect(label).toContain('2026'); +}); + +test('a missing timestamp degrades to an unknown-time label', () => { + expect(en.version({ at: 0 })).toBe('Time unknown'); +}); diff --git a/packages/history/test/index.test.ts b/packages/history/test/index.test.ts new file mode 100644 index 00000000..725ad755 --- /dev/null +++ b/packages/history/test/index.test.ts @@ -0,0 +1,472 @@ +import HistoryModule from '@/index'; +import { getOrCreateDeviceId, openHistoryStores, readLastGcTime } from '@/store'; +import { expect, test } from 'bun:test'; +import { openMemoryDB } from 'uni-kv'; +import type { WalEntry } from '@/types'; +import type { GlobStrategy } from '@repo/shared/glob-match'; + +const sleep = (ms: number) => + new Promise((resolve) => { + setTimeout(resolve, ms); + }); + +function fakeContext(namespace: string | Error = 'ns') { + const i18n: Array = []; + const settings: Array = []; + const events: Array = []; + const commands: Array = []; + const listeners = new Map void>>(); + const menuHandlers: Array<(menu: unknown, file: unknown) => void> = []; + const vaultHandlers = new Map void>(); + const localStorageMap = new Map(); + const files = new Map(); + const memDb = openMemoryDB(`history-${crypto.randomUUID()}`); + const ctx = { + addCommand: (command: unknown) => { + commands.push(command); + return command; + }, + app: { + loadLocalStorage: (key: string) => localStorageMap.get(key), + saveLocalStorage: (key: string, data: unknown) => { + if (data === undefined) localStorageMap.delete(key); + else localStorageMap.set(key, data); + }, + vault: { + getAbstractFileByPath: (path: string) => (files.has(path) ? { path } : undefined), + on: (event: string, handler: (file: unknown, oldPath?: string) => void) => { + vaultHandlers.set(event, handler); + return { id: `vault-${event}` }; + }, + read: (file: { path: string }) => Promise.resolve(files.get(file.path) ?? ''), + }, + workspace: { + on: (event: string, handler: (menu: unknown, file: unknown) => void) => { + if (event === 'file-menu') menuHandlers.push(handler); + return { id: event }; + }, + }, + }, + getNamespace: () => namespace, + indexedDB: memDb, + on: (event: string, listener: (payload?: unknown) => void) => { + const list = listeners.get(event) ?? []; + list.push(listener); + listeners.set(event, list); + return () => { + const idx = list.indexOf(listener); + if (idx !== -1) list.splice(idx, 1); + }; + }, + registerEvent: (ref: unknown) => events.push(ref), + registerI18n: (locale: string) => i18n.push(locale), + registerSetting: (entry: unknown) => { + settings.push(entry); + return () => true; + }, + requestSync: () => Promise.resolve({ result: 'noop' }), + saveSettings: () => Promise.resolve(), + settings: { modules: {}, syncStrategy: [] as Array }, + translate: (key: string) => key, + }; + return { + commands, + ctx, + events, + files, + i18n, + listeners, + memDb, + menuHandlers, + settings, + vaultHandlers, + }; +} + +test('the constructor registers all four locales', () => { + const { ctx, i18n } = fakeContext(); + new HistoryModule(ctx as never); + expect(i18n).toStrictEqual(['en', 'zh', 'zh-TW', 'ru']); +}); + +test('start initializes deviceId in store and removes legacy deviceId from moduleSettings', async () => { + const { ctx, memDb } = fakeContext(); + const module = new HistoryModule(ctx as never); + module.moduleSettings.deviceId = 'legacy-device-id-99'; + module.start(); + expect(module.moduleSettings.deviceId).toBeUndefined(); + const stores = openHistoryStores(memDb, 'ns'); + const storedId = await getOrCreateDeviceId(stores); + expect(storedId).toBe('legacy-device-id-99'); + module.dispose(); +}); + +test('start initializes deviceId in store when none exists', async () => { + const { ctx, memDb } = fakeContext(); + const module = new HistoryModule(ctx as never); + module.start(); + const stores = openHistoryStores(memDb, 'ns'); + const storedId = await getOrCreateDeviceId(stores); + expect(typeof storedId).toBe('string'); + expect(storedId.length).toBeGreaterThan(10); + module.dispose(); +}); + +test('start creates no stores when no backend is configured (namespace is an Error)', async () => { + const { ctx, memDb, vaultHandlers } = fakeContext(new Error('no backend')); + const module = new HistoryModule(ctx as never); + module.moduleSettings.deviceId = 'legacy-device-id-99'; + module.start(); + // Vault events and the legacy device id must not create empty-id stores. + const created = vaultHandlers.get('create'); + if (created) created({ path: 'note.md' }); + await sleep(30); + expect(memDb.getStoreNames()).toStrictEqual([]); + module.dispose(); +}); + +test('start registers a setting, a vault listener and a file-menu handler', () => { + const { commands, ctx, events, menuHandlers, settings } = fakeContext(); + const module = new HistoryModule(ctx as never); + expect(() => module.start()).not.toThrow(); + expect(settings).toHaveLength(1); + expect(events).toHaveLength(5); // Vault create, modify, delete, rename + workspace file-menu + expect(commands).toHaveLength(1); + expect(menuHandlers).toHaveLength(1); + module.dispose(); +}); + +test('the file-menu handler adds a Show history item only for supported files', () => { + const { ctx, menuHandlers } = fakeContext(); + const module = new HistoryModule(ctx as never); + module.start(); + const handler = menuHandlers[0]; + const added: Array = []; + const menu = { + addItem: (cb: (item: unknown) => void) => { + const item = { + onClick: () => item, + setIcon: () => item, + setTitle: (title: string) => { + added.push(title); + return item; + }, + }; + cb(item); + }, + }; + // By default (trackAttachments: false), only supported text files show history + handler(menu, { path: 'note.md' }); + handler(menu, { path: 'board.canvas' }); + handler(menu, { children: [], path: 'folder' }); + handler(menu, { path: 'image.png' }); + expect(added).toHaveLength(2); + expect(added[0]).toBe('showHistory'); + expect(added[1]).toBe('showHistory'); + + // When trackAttachments is true, attachments also show history + module.moduleSettings.trackAttachments = true; + handler(menu, { path: 'image.png' }); + expect(added).toHaveLength(3); + expect(added[2]).toBe('showHistory'); + module.dispose(); +}); + +test('onVaultCreate queues non-empty files with debounce, captures attachments only when trackAttachments is true', async () => { + const { ctx, files, listeners, memDb, vaultHandlers } = fakeContext(); + const module = new HistoryModule(ctx as never); + module.start(); + + const onCreate = vaultHandlers.get('create'); + expect(onCreate).toBeDefined(); + + // 1. Empty file create should be ignored + files.set('empty.md', ''); + onCreate?.({ path: 'empty.md' }); + await sleep(50); + const pendingEmpty = memDb.getStore('ns-history-pending').values(); + expect(pendingEmpty).toHaveLength(0); + + // 2. Folder should be ignored + onCreate?.({ children: [], path: 'my-folder' }); + await sleep(50); + const pendingFolder = memDb.getStore('ns-history-pending').values(); + expect(pendingFolder).toHaveLength(0); + + // 3. By default (trackAttachments: false), attachments are ignored + files.set('photo-ignored.png', 'binary-data'); + onCreate?.({ path: 'photo-ignored.png' }); + await sleep(50); + const syncStartedListeners = listeners.get('syncStarted') ?? []; + for (const fn of syncStartedListeners) fn(); + await sleep(50); + let pending = memDb.getStore('ns-history-pending').values() as Array; + expect(pending).toHaveLength(0); + + // 4. When trackAttachments is enabled, attachments are captured as standalone blobs + module.moduleSettings.trackAttachments = true; + files.set('photo.png', 'binary-data'); + onCreate?.({ path: 'photo.png' }); + await sleep(50); + for (const fn of syncStartedListeners) fn(); + await sleep(50); + pending = memDb.getStore('ns-history-pending').values() as Array; + expect(pending).toHaveLength(1); + expect(pending[0].key).toBe('photo.png'); + expect(pending[0].blobs).toBeDefined(); + expect(pending[0].blobs?.[0].key).toBe('photo.png'); + expect(pending[0].blobData).toBeDefined(); + + // 5. Non-empty markdown file is debounced and ingested upon flush/sync + files.set('note.md', '# Initial Created Content'); + onCreate?.({ path: 'note.md' }); + await sleep(50); + + for (const fn of syncStartedListeners) fn(); + await sleep(50); + + pending = memDb.getStore('ns-history-pending').values() as Array; + expect(pending).toHaveLength(2); + expect(pending[1].key).toBe('note.md'); + expect(pending[1].text).toBe('# Initial Created Content'); + + module.dispose(); +}); + +test('sync downloaded files are ignored and not captured by History', async () => { + const { ctx, files, listeners, memDb, vaultHandlers } = fakeContext(); + const module = new HistoryModule(ctx as never); + module.start(); + + const executionStartedListeners = listeners.get('executionStarted') ?? []; + const onCreate = vaultHandlers.get('create'); + const onModify = vaultHandlers.get('modify'); + + // Simulate sync engine downloading 2 files + const mockTasks = [ + { key: 'remote-1.md', name: 'download' }, + { key: 'remote-2.md', name: 'download' }, + ]; + for (const fn of executionStartedListeners) fn(mockTasks); + + files.set('remote-1.md', '# Downloaded from Remote 1'); + files.set('remote-2.md', '# Downloaded from Remote 2'); + onCreate?.({ path: 'remote-1.md' }); + onModify?.({ path: 'remote-2.md' }); + await sleep(50); + + // Trigger syncStarted or flush + const syncStartedListeners = listeners.get('syncStarted') ?? []; + for (const fn of syncStartedListeners) fn(); + await sleep(50); + + // Downloaded files should not be captured in history + const pending = memDb.getStore('ns-history-pending').values() as Array; + expect(pending).toHaveLength(0); + + module.dispose(); +}); + +test('syncStarted flushes pending debounces', async () => { + const { ctx, files, listeners, memDb, vaultHandlers } = fakeContext(); + const module = new HistoryModule(ctx as never); + module.start(); + + const onModify = vaultHandlers.get('modify'); + expect(onModify).toBeDefined(); + + files.set('debounced.md', '# Content to debounce'); + onModify?.({ path: 'debounced.md' }); + + // Allow read and hash to complete + await sleep(50); + // Still in debounce queue, not committed yet + let pending = memDb.getStore('ns-history-pending').values() as Array; + expect(pending).toHaveLength(0); + + // Trigger syncStarted + const syncStartedListeners = listeners.get('syncStarted') ?? []; + for (const fn of syncStartedListeners) fn(); + + await sleep(50); + pending = memDb.getStore('ns-history-pending').values() as Array; + expect(pending).toHaveLength(1); + expect(pending[0].key).toBe('debounced.md'); + + module.dispose(); +}); + +test('files with none sync strategy are completely ignored by history', async () => { + const { ctx, files, listeners, memDb, menuHandlers, vaultHandlers } = fakeContext(); + ctx.settings.syncStrategy = [ + { expr: '*', strategy: 'bidirectional' }, + { expr: 'ignored/**', strategy: 'none' }, + { expr: 'secret.md', strategy: 'none' }, + ]; + + const module = new HistoryModule(ctx as never); + module.start(); + + // 1. Context menu: ignored files should not have "showHistory" option + const menuHandler = menuHandlers[0]; + const added: Array = []; + const menu = { + addItem: (cb: (item: unknown) => void) => { + const item = { + onClick: () => item, + setIcon: () => item, + setTitle: (title: string) => { + added.push(title); + return item; + }, + }; + cb(item); + }, + }; + + menuHandler(menu, { path: 'regular.md' }); + menuHandler(menu, { path: 'ignored/file.md' }); + menuHandler(menu, { path: 'secret.md' }); + expect(added).toHaveLength(1); + expect(added[0]).toBe('showHistory'); + + // 2. Vault create: ignored file should not be captured into pending WAL + const onCreate = vaultHandlers.get('create'); + files.set('ignored/doc.md', '# Ignored Note'); + files.set('regular.md', '# Regular Note'); + onCreate?.({ path: 'ignored/doc.md' }); + onCreate?.({ path: 'regular.md' }); + + await sleep(50); + const syncStartedListeners = listeners.get('syncStarted') ?? []; + for (const fn of syncStartedListeners) fn(); + await sleep(50); + + const pending = memDb.getStore('ns-history-pending').values() as Array; + expect(pending).toHaveLength(1); + expect(pending[0].key).toBe('regular.md'); + + // 3. Vault modify: modifying ignored file should not be captured + const onModify = vaultHandlers.get('modify'); + files.set('ignored/doc.md', '# Ignored Note Modified'); + onModify?.({ path: 'ignored/doc.md' }); + await sleep(50); + for (const fn of syncStartedListeners) fn(); + await sleep(50); + + const pendingAfterModify = memDb.getStore('ns-history-pending').values() as Array; + expect(pendingAfterModify).toHaveLength(1); + expect(pendingAfterModify[0].key).toBe('regular.md'); + + // 4. Vault delete: deleting ignored file should not add WAL tombstone + const onDelete = vaultHandlers.get('delete'); + onDelete?.({ path: 'ignored/doc.md' }); + await sleep(50); + + const pendingAfterDelete = memDb.getStore('ns-history-pending').values() as Array; + expect(pendingAfterDelete).toHaveLength(1); + + // 5. Vault rename: renaming to an ignored path should not capture the new path + const onRename = vaultHandlers.get('rename'); + files.set('ignored/renamed.md', '# Renamed into ignored folder'); + onRename?.({ path: 'ignored/renamed.md' }, 'regular.md'); + await sleep(50); + for (const fn of syncStartedListeners) fn(); + await sleep(50); + + const pendingAfterRename = memDb.getStore('ns-history-pending').values() as Array; + // Only the previous regular.md entry exists, ignored/renamed.md was never captured + expect(pendingAfterRename.map((e) => e.key)).toEqual(['regular.md']); + + module.dispose(); +}); + +test('checkAutoGc persists lastGcTime in store and does not modify moduleSettings', async () => { + const { ctx, memDb } = fakeContext(); + let saveSettingsCalled = false; + ctx.saveSettings = () => { + saveSettingsCalled = true; + return Promise.resolve(); + }; + + const module = new HistoryModule(ctx as never); + module.start(); + + const mockRemote = { + deletePack: () => Promise.resolve(), + deviceId: 'device-a', + getObject: () => Promise.resolve(new Uint8Array()), + listHeads: () => Promise.resolve([]), + listPacks: () => Promise.resolve([]), + ns: 'ns', + putBlob: () => Promise.resolve(), + putHead: () => Promise.resolve(), + putPack: () => Promise.resolve(), + readHead: () => Promise.resolve(), + }; + (module as unknown as { remote: unknown }).remote = mockRemote; + + await (module as unknown as { checkAutoGc: () => Promise }).checkAutoGc(); + + const stores = openHistoryStores(memDb, 'ns'); + const stored = await readLastGcTime(stores); + expect(typeof stored).toBe('number'); + expect(stored).toBeGreaterThan(0); + expect((module.moduleSettings as Record).lastGcTime).toBeUndefined(); + expect(saveSettingsCalled).toBe(false); + + module.dispose(); +}); + +test('protectExcludedNotes allows excluded text files to be tracked while attachments remain ignored', async () => { + const { ctx, files, listeners, memDb, menuHandlers, vaultHandlers } = fakeContext(); + ctx.settings.syncStrategy = [ + { expr: '*', strategy: 'bidirectional' }, + { expr: 'excluded/**', strategy: 'none' }, + ]; + + const module = new HistoryModule(ctx as never); + module.moduleSettings.protectExcludedNotes = true; + module.start(); + + const menuHandler = menuHandlers[0]; + const added: Array = []; + const menu = { + addItem: (cb: (item: unknown) => void) => { + const item = { + onClick: () => item, + setIcon: () => item, + setTitle: (title: string) => { + added.push(title); + return item; + }, + }; + cb(item); + }, + }; + + // Context menu: excluded note should have showHistory, excluded attachment should not + menuHandler(menu, { path: 'excluded/note.md' }); + menuHandler(menu, { path: 'excluded/image.png' }); + expect(added).toHaveLength(1); + expect(added[0]).toBe('showHistory'); + + // Vault create: excluded note is captured into pending WAL, excluded attachment is ignored + const onCreate = vaultHandlers.get('create'); + files.set('excluded/note.md', '# Protected Note Content'); + files.set('excluded/image.png', 'binary-content'); + onCreate?.({ path: 'excluded/note.md' }); + onCreate?.({ path: 'excluded/image.png' }); + + await sleep(50); + const syncStartedListeners = listeners.get('syncStarted') ?? []; + for (const fn of syncStartedListeners) fn(); + await sleep(50); + + const pending = memDb.getStore('ns-history-pending').values() as Array; + expect(pending).toHaveLength(1); + expect(pending[0].key).toBe('excluded/note.md'); + expect(pending[0].text).toBe('# Protected Note Content'); + + module.dispose(); +}); diff --git a/packages/history/test/mocks.ts b/packages/history/test/mocks.ts new file mode 100644 index 00000000..9a27a793 --- /dev/null +++ b/packages/history/test/mocks.ts @@ -0,0 +1,4 @@ +import ObsidianMock from '@repo/shared/obsidian-mock'; +import { mock } from 'bun:test'; + +void mock.module('obsidian', () => ObsidianMock); diff --git a/packages/history/test/modal.test.ts b/packages/history/test/modal.test.ts new file mode 100644 index 00000000..fd686a7c --- /dev/null +++ b/packages/history/test/modal.test.ts @@ -0,0 +1,56 @@ +import { computeLineDiff } from '@/modal'; +import { expect, test } from 'bun:test'; + +test('computeLineDiff produces unchanged lines for identical texts', () => { + const lines = computeLineDiff('hello\nworld', 'hello\nworld'); + expect(lines).toStrictEqual([ + { text: 'hello', type: 'unchanged' }, + { text: 'world', type: 'unchanged' }, + ]); +}); + +test('computeLineDiff identifies additions and removals', () => { + const historical = 'line 1\nline 2\nline 3'; + const current = 'line 1\nmodified line 2\nline 3\nline 4'; + const lines = computeLineDiff(historical, current); + expect(lines).toStrictEqual([ + { text: 'line 1', type: 'unchanged' }, + { text: 'line 2', type: 'removed' }, + { text: 'modified line 2', type: 'added' }, + { text: 'line 3', type: 'unchanged' }, + { text: 'line 4', type: 'added' }, + ]); +}); + +test('83-line file with 1 line added produces exactly 1 line diff', () => { + const historical = Array.from({ length: 83 }, (_, i) => `Line ${i + 1}`).join('\n'); + const current = `${historical}\nLine 84`; + const lines = computeLineDiff(historical, current); + let added = 0; + let removed = 0; + let unchanged = 0; + for (const l of lines) + if (l.type === 'added') added++; + else if (l.type === 'removed') removed++; + else unchanged++; + + expect(added).toBe(1); + expect(removed).toBe(0); + expect(unchanged).toBe(83); +}); + +test('computeLineDiff handles empty strings cleanly without phantom line changes', () => { + expect(computeLineDiff('', 'hello')).toStrictEqual([{ text: 'hello', type: 'added' }]); + expect(computeLineDiff('hello', '')).toStrictEqual([{ text: 'hello', type: 'removed' }]); + expect(computeLineDiff('', '')).toStrictEqual([]); +}); + +test('computeLineDiff normalizes CRLF and LF line endings', () => { + const historical = 'line 1\r\nline 2\r\n'; + const current = 'line 1\nline 2\nline 3\n'; + const lines = computeLineDiff(historical, current); + const added = lines.filter((l) => l.type === 'added'); + const removed = lines.filter((l) => l.type === 'removed'); + expect(added).toHaveLength(1); + expect(removed).toHaveLength(0); +}); diff --git a/packages/history/test/pack.test.ts b/packages/history/test/pack.test.ts new file mode 100644 index 00000000..3ba35360 --- /dev/null +++ b/packages/history/test/pack.test.ts @@ -0,0 +1,89 @@ +import { decodePack, encodePack, packFileName, readIndexOnly, textBytes } from '@/pack'; +import { expect, test } from 'bun:test'; +import type { IndexEntry } from '@/types'; + +function entry(overrides: Partial = {}): IndexEntry { + return { + at: 1000, + blobs: [], + chunks: [], + deviceId: 'device-a', + flags: 0, + key: 'note.md', + keyHash: 'kh', + lamport: 1, + parents: [], + ...overrides, + }; +} + +const baseHeader = { + baseLamport: 1, + createdAt: 1000, + deviceId: 'device-a', + ns: 'ns', + peakLamport: 3, + seq: 7, +}; + +test('encodePack/decodePack round-trips entries and inflated payload', async () => { + const payload = textBytes('# header\nbody text'); + const encoded = await encodePack( + baseHeader, + [entry(), entry({ key: 'b.md', lamport: 3 })], + payload, + ); + const decoded = await decodePack(encoded.bytes); + expect(decoded.entries).toHaveLength(2); + expect(decoded.header.hash).toBe(encoded.header.hash); + expect(new TextDecoder().decode(decoded.payload)).toBe('# header\nbody text'); + expect(decoded.header.codec).toBe('deflate'); +}); + +test('the hash is self-certifying: readIndexOnly recomputes the same value', async () => { + const encoded = await encodePack(baseHeader, [entry()], textBytes('data')); + const parsed = await readIndexOnly(encoded.bytes); + expect(parsed.header.hash).toBe(encoded.header.hash); + expect(parsed.entries[0].key).toBe('note.md'); +}); + +test('entries are sorted by (lamport, deviceId, keyHash)', async () => { + const encoded = await encodePack( + baseHeader, + [ + entry({ deviceId: 'z', keyHash: 'a', lamport: 5 }), + entry({ deviceId: 'a', keyHash: 'z', lamport: 1 }), + entry({ deviceId: 'a', keyHash: 'a', lamport: 1 }), + ], + textBytes('x'), + ); + expect(encoded.entries.map((e) => `${e.lamport}:${e.deviceId}:${e.keyHash}`)).toStrictEqual([ + '1:a:a', + '1:a:z', + '5:z:a', + ]); +}); + +test('a corrupt magic is rejected', async () => { + const encoded = await encodePack(baseHeader, [entry()], textBytes('x')); + const corrupted = Uint8Array.from(encoded.bytes); + corrupted[0] = 0; + let message = ''; + try { + await decodePack(corrupted); + } catch (error) { + message = (error as Error).message; + } + expect(message).toBe('Invalid pack magic'); +}); + +test('an empty payload uses the raw codec', async () => { + const encoded = await encodePack(baseHeader, [entry()], new Uint8Array(0)); + expect(encoded.header.codec).toBe('raw'); + const decoded = await decodePack(encoded.bytes); + expect(decoded.payload.byteLength).toBe(0); +}); + +test('packFileName embeds seq and hash', () => { + expect(packFileName('d', 18, 'ab12')).toBe('p-00000018-ab12.pack'); +}); diff --git a/packages/history/test/restore.test.ts b/packages/history/test/restore.test.ts new file mode 100644 index 00000000..1d903e03 --- /dev/null +++ b/packages/history/test/restore.test.ts @@ -0,0 +1,276 @@ +import { keyHash } from '@/chunker'; +import { flushPending } from '@/flush'; +import { encodePack, packFileName } from '@/pack'; +import { listVersions, restoreVersion } from '@/restore'; +import { openHistoryStores, readPending } from '@/store'; +import { ingestEdit } from '@/wal'; +import { expect, test } from 'bun:test'; +import { openMemoryDB } from 'uni-kv'; +import type { RemoteHistory } from '@/remote'; +import type { HistoryDatabase } from '@/store'; + +const encoder = new TextEncoder(); + +function fakeRemote() { + const objects = new Map(); + const remote: RemoteHistory = { + deletePack: () => Promise.resolve(), + deviceId: 'device-a', + fs: {} as never, + getObject: (key) => { + const value = objects.get(key); + return value ? Promise.resolve(value) : Promise.reject(new Error(`missing ${key}`)); + }, + listHeads: () => Promise.resolve([]), + listPacks: () => Promise.resolve([]), + ns: 'ns', + putBlob: (hash, bytes) => { + objects.set(`ns/blobs/${hash}.blob`, bytes); + return Promise.resolve(); + }, + putPack: (seq, hash, bytes) => { + objects.set(`ns/packs/device-a/${packFileName('device-a', seq, hash)}`, bytes); + return Promise.resolve(); + }, + readHead: () => Promise.resolve(undefined as never), + writeHead: () => Promise.resolve(), + }; + return { objects, remote }; +} + +async function seedVersion(text: string, lamport = 1) { + const db = openMemoryDB(`history-restore-${crypto.randomUUID()}`) as never as HistoryDatabase; + const stores = openHistoryStores(db, 'ns'); + const { objects, remote } = fakeRemote(); + const bytes = encoder.encode(text); + const encoded = await encodePack( + { + baseLamport: lamport, + createdAt: 1000, + deviceId: 'device-a', + ns: 'ns', + peakLamport: lamport, + seq: 1, + }, + [ + { + at: 1000, + blobs: [], + chunks: [], + deviceId: 'device-a', + flags: 1, + inlineLength: bytes.byteLength, + inlineOffset: 0, + key: 'note.md', + keyHash: await keyHash('note.md'), + lamport, + parents: [], + }, + ], + bytes, + ); + const fileName = packFileName('device-a', 1, encoded.header.hash); + objects.set(`ns/packs/device-a/${fileName}`, encoded.bytes); + await stores.index.set(`note.md@${lamport}`, { + indexLength: encoded.header.indexLength, + indexOffset: encoded.header.indexOffset, + pack: { + deviceId: 'device-a', + fromLamport: lamport, + hash: encoded.header.hash, + seq: 1, + toLamport: lamport, + }, + }); + return { remote, stores }; +} + +test('restoreVersion reads the pack and returns the exact text', async () => { + const { remote, stores } = await seedVersion('# Restored text'); + const result = await restoreVersion({ remote, stores }, { key: 'note.md', lamport: 1 }); + expect(result.text).toBe('# Restored text'); + expect(result.key).toBe('note.md'); + expect(result.merged).toBe(false); +}); + +test('listVersions returns the captured lamports for a key', async () => { + const db = openMemoryDB(`history-list-${crypto.randomUUID()}`) as never as HistoryDatabase; + const stores = openHistoryStores(db, 'ns'); + const { remote } = fakeRemote(); + const ctx = { remote, requestSync: () => {}, stores }; + await ingestEdit(stores, 'note.md', encoder.encode('one')); + await ingestEdit(stores, 'note.md', encoder.encode('two')); + await flushPending(ctx, 'count'); + const versions = await listVersions({ remote, stores }, 'note.md'); + expect(versions.map((version) => version.lamport)).toStrictEqual([2, 1]); +}); + +test('listVersions prunes dead index entries on 404 pack and does not return unknown time versions', async () => { + const { remote, stores } = await seedVersion('one', 1); + // Point note.md@2 at a non-existent pack + await stores.index.set('note.md@2', { + indexLength: 10, + indexOffset: 10, + pack: { + deviceId: 'device-missing', + fromLamport: 2, + hash: 'deadbeef', + seq: 99, + toLamport: 2, + }, + }); + const versions = await listVersions({ remote, stores }, 'note.md'); + // Only the valid version 1 should be returned, no dummy versions with at: 0 ("时间未知") + expect(versions).toHaveLength(1); + expect(versions[0].lamport).toBe(1); + expect(versions[0].at).toBeGreaterThan(0); + // The dead reference should be auto-pruned from index + expect(await stores.index.get('note.md@2')).toBeUndefined(); +}); + +test('consecutive unmodified versions are collapsed into a single version', async () => { + const db = openMemoryDB(`history-dedup-${crypto.randomUUID()}`) as never as HistoryDatabase; + const stores = openHistoryStores(db, 'ns'); + const { remote } = fakeRemote(); + const ctx = { remote, requestSync: () => {}, stores }; + // Three versions: 'version A', 'version B', 'version B' (duplicate of B) + await ingestEdit(stores, 'note.md', encoder.encode('version A')); + await flushPending(ctx, 'count'); + await ingestEdit(stores, 'note.md', encoder.encode('version B')); + await flushPending(ctx, 'count'); + await ingestEdit(stores, 'note.md', encoder.encode('version B')); + await flushPending(ctx, 'count'); + + const versions = await listVersions({ remote, stores }, 'note.md'); + // The duplicate 'version B' should be collapsed; only 2 distinct versions displayed + expect(versions).toHaveLength(2); + expect(versions.map((v) => v.index)).toStrictEqual([1, 2]); + expect(versions.map((v) => v.lamport)).toStrictEqual([3, 1]); +}); + +test('restoreVersion refreshes heads on an index miss before giving up', async () => { + const { remote, stores } = await seedVersion('one', 1); + let headRefreshes = 0; + const original = remote.listHeads; + remote.listHeads = () => { + headRefreshes++; + return original(); + }; + let threw = false; + try { + await restoreVersion({ remote, stores }, { key: 'missing.md', lamport: 1 }); + } catch { + threw = true; + } + expect(threw).toBe(true); + expect(headRefreshes).toBeGreaterThan(0); +}); + +test('a DAG fork restores into a new merge commit with both tips as parents', async () => { + const db = openMemoryDB(`history-dag-${crypto.randomUUID()}`) as never as HistoryDatabase; + const stores = openHistoryStores(db, 'ns'); + const { objects, remote } = fakeRemote(); + const kh = await keyHash('note.md'); + const make = (lamport: number, parents: Array) => ({ + at: 1000, + blobs: [], + chunks: [], + deviceId: 'device-a', + flags: 1 as const, + inlineLength: 0, + inlineOffset: 0, + key: 'note.md', + keyHash: kh, + lamport, + parents, + }); + const entries = [make(1, []), make(2, []), make(3, ['note.md@1', 'note.md@2'])]; + const encoded = await encodePack( + { baseLamport: 1, createdAt: 1000, deviceId: 'device-a', ns: 'ns', peakLamport: 3, seq: 1 }, + entries, + new Uint8Array(0), + ); + const pack = { + deviceId: 'device-a', + fromLamport: 1, + hash: encoded.header.hash, + seq: 1, + toLamport: 3, + }; + objects.set(`ns/packs/device-a/${packFileName('device-a', 1, pack.hash)}`, encoded.bytes); + for (const lamport of [1, 2, 3]) + await stores.index.set(`note.md@${lamport}`, { + indexLength: encoded.header.indexLength, + indexOffset: encoded.header.indexOffset, + pack, + }); + const result = await restoreVersion({ remote, stores }, { key: 'note.md', lamport: 3 }); + expect(result.merged).toBe(true); + const pending = await readPending(stores); + expect(pending).toHaveLength(1); + expect(pending[0].parents).toStrictEqual(['note.md@1', 'note.md@2']); +}); + +test('versions are numbered per file even when a pack batches edits from several files', async () => { + const db = openMemoryDB(`history-perfile-${crypto.randomUUID()}`) as never as HistoryDatabase; + const stores = openHistoryStores(db, 'ns'); + const { remote } = fakeRemote(); + const ctx = { remote, requestSync: () => {}, stores }; + // Global Lamport clock: 1 (abcd), 2 (other), 3 (abcd). UI must not show 3 and 1 as "versions". + await ingestEdit(stores, 'abcd.md', encoder.encode('abcd v1')); + await ingestEdit(stores, 'other.md', encoder.encode('other v1')); + await ingestEdit(stores, 'abcd.md', encoder.encode('abcd v2')); + await flushPending(ctx, 'count'); + const abcd = await listVersions({ remote, stores }, 'abcd.md'); + const other = await listVersions({ remote, stores }, 'other.md'); + // Per-file indices are contiguous starting at 1 (newest first), independent of global lamport. + expect(abcd.map((version) => version.index)).toStrictEqual([1, 2]); + expect(abcd.map((version) => version.lamport)).toStrictEqual([3, 1]); + expect(other.map((version) => version.index)).toStrictEqual([1]); + expect(other[0].lamport).toBe(2); + // Restoring never returns another file's content from the same pack. + const abcdLatest = await restoreVersion({ remote, stores }, { key: 'abcd.md', lamport: 3 }); + const otherOnly = await restoreVersion({ remote, stores }, { key: 'other.md', lamport: 2 }); + expect(abcdLatest.text).toBe('abcd v2'); + expect(otherOnly.text).toBe('other v1'); +}); + +test('a missing version throws instead of falling back to another commit in the pack', async () => { + const db = openMemoryDB(`history-missing-${crypto.randomUUID()}`) as never as HistoryDatabase; + const stores = openHistoryStores(db, 'ns'); + const { remote } = fakeRemote(); + await ingestEdit(stores, 'a.md', encoder.encode('a')); + const ctx = { remote, requestSync: () => {}, stores }; + await flushPending(ctx, 'count'); + // Point a lookup at a lamport that is not in the pack; the old entries[0] fallback would + // silently return a.md's content. It must now fail loudly. + const location = await stores.index.get('a.md@1'); + if (location) await stores.index.set('a.md@999', location); + let threw = false; + try { + await restoreVersion({ remote, stores }, { key: 'a.md', lamport: 999 }); + } catch { + threw = true; + } + expect(threw).toBe(true); +}); + +test('restoreVersion restores standalone blob for attachment files without decoding text', async () => { + const db = openMemoryDB(`history-blob-${crypto.randomUUID()}`) as never as HistoryDatabase; + const stores = openHistoryStores(db, 'ns'); + const { remote } = fakeRemote(); + const ctx = { remote, requestSync: () => {}, stores }; + const rawBytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + await ingestEdit(stores, 'image.png', rawBytes); + await flushPending(ctx, 'count'); + + const versions = await listVersions(ctx, 'image.png'); + expect(versions).toHaveLength(1); + + const restored = await restoreVersion(ctx, { key: 'image.png', lamport: versions[0].lamport }); + expect(restored.key).toBe('image.png'); + expect(restored.binary).toBeDefined(); + expect(restored.binary).toStrictEqual(rawBytes); + expect(restored.text).toBe(''); + expect(restored.merged).toBe(false); +}); diff --git a/packages/history/test/setting.test.ts b/packages/history/test/setting.test.ts new file mode 100644 index 00000000..b83c59f6 --- /dev/null +++ b/packages/history/test/setting.test.ts @@ -0,0 +1,73 @@ +import { en } from '@/i18n'; +import historySetting from '@/setting'; +import { describe, expect, mock, test } from 'bun:test'; +import type { HistorySettings } from '@/setting'; + +describe('historySetting', () => { + test('tieredRetention toggle defaults to true and updates setting', () => { + const saveSettings = mock(() => Promise.resolve()); + const settings: HistorySettings = { + autoGc: true, + maxVersions: 100, + protectExcludedNotes: false, + retentionDays: 90, + tieredRetention: true, + trackAttachments: false, + }; + const actions = { runGc: mock(() => Promise.resolve(0)) }; + const tree = historySetting( + { saveSettings, translate: ((key: keyof typeof en) => en[key]) as never }, + settings, + actions, + ) as unknown as Record< + number, + Record< + number, + (self: unknown) => { + render: (setting: { + addToggle: ( + cb: (toggle: { + onChange: (fn: (v: boolean) => void) => void; + setValue: (v: boolean) => unknown; + }) => void, + ) => void; + }) => void; + } + > + >; + + const group = tree[3055]; + expect(group).toBeDefined(); + const tieredItem = group[2100]; + expect(tieredItem).toBeDefined(); + + let toggleVal = false; + let onChangeHandler: ((v: boolean) => void) | undefined; + const mockSetting = { + addToggle: ( + cb: (toggle: { + onChange: (fn: (v: boolean) => void) => void; + setValue: (v: boolean) => unknown; + }) => void, + ) => { + const mockToggle = { + onChange: (fn: (v: boolean) => void) => { + onChangeHandler = fn; + }, + setValue: (v: boolean) => { + toggleVal = v; + return mockToggle; + }, + }; + cb(mockToggle); + }, + }; + + tieredItem(tieredItem).render(mockSetting); + expect(toggleVal).toBe(true); + + onChangeHandler?.(false); + expect(settings.tieredRetention).toBe(false); + expect(saveSettings).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/history/test/store.test.ts b/packages/history/test/store.test.ts new file mode 100644 index 00000000..5992e491 --- /dev/null +++ b/packages/history/test/store.test.ts @@ -0,0 +1,91 @@ +import { + bumpLamport, + getOrCreateDeviceId, + openHistoryStores, + readDeviceId, + readLamport, + readLastGcTime, + saveLastGcTime, +} from '@/store'; +import { expect, test } from 'bun:test'; +import { openMemoryDB } from 'uni-kv'; +import type { HistoryDatabase } from '@/store'; + +function setup(namespace = 'ns-test') { + const db = openMemoryDB(`history-${crypto.randomUUID()}`) as never as HistoryDatabase; + return openHistoryStores(db, namespace); +} + +test('getOrCreateDeviceId generates and persists a fresh UUID when empty', async () => { + const stores = setup(); + expect(await readDeviceId(stores)).toBeUndefined(); + + const id = await getOrCreateDeviceId(stores); + expect(typeof id).toBe('string'); + expect(id.length).toBeGreaterThan(10); + + // Subsequent reads and calls return the same ID + expect(await readDeviceId(stores)).toBe(id); + expect(await getOrCreateDeviceId(stores)).toBe(id); +}); + +test('getOrCreateDeviceId migrates legacy deviceId when none exists', async () => { + const stores = setup(); + const id = await getOrCreateDeviceId(stores, 'legacy-uuid-123'); + expect(id).toBe('legacy-uuid-123'); + expect(await readDeviceId(stores)).toBe('legacy-uuid-123'); + + // Ignores legacy parameter once an ID is already stored + expect(await getOrCreateDeviceId(stores, 'ignored-legacy')).toBe('legacy-uuid-123'); +}); + +test('readLastGcTime defaults to 0 and saveLastGcTime persists the timestamp', async () => { + const stores = setup(); + expect(await readLastGcTime(stores)).toBe(0); + + const timestamp = 1_700_000_000_000; + await saveLastGcTime(stores, timestamp); + expect(await readLastGcTime(stores)).toBe(timestamp); +}); + +test('stores for different namespaces are isolated', async () => { + const db = openMemoryDB(`history-${crypto.randomUUID()}`) as never as HistoryDatabase; + const storesA = openHistoryStores(db, 'ns-a'); + const storesB = openHistoryStores(db, 'ns-b'); + + await getOrCreateDeviceId(storesA, 'device-a'); + await saveLastGcTime(storesA, 111); + + expect(await readDeviceId(storesA)).toBe('device-a'); + expect(await readLastGcTime(storesA)).toBe(111); + + expect(await readDeviceId(storesB)).toBeUndefined(); + expect(await readLastGcTime(storesB)).toBe(0); +}); + +test('clock, device id and GC time coexist in the shared meta store', async () => { + const stores = setup(); + await bumpLamport(stores, 10); + await getOrCreateDeviceId(stores, 'device-x'); + await saveLastGcTime(stores, 999); + + expect(await readLamport(stores)).toBe(11); + expect(await readDeviceId(stores)).toBe('device-x'); + expect(await readLastGcTime(stores)).toBe(999); + expect(await stores.meta.keys()).toStrictEqual(['clock', 'id', 'time']); +}); + +test('store names lead with the namespace id so a pair is manageable by prefix', () => { + const db = openMemoryDB(`history-${crypto.randomUUID()}`); + openHistoryStores(db as never as HistoryDatabase, 'ns-a'); + const names = db.getStoreNames(); + expect(names).toStrictEqual([ + 'ns-a-history-blobs', + 'ns-a-history-chunks', + 'ns-a-history-hashes', + 'ns-a-history-heads', + 'ns-a-history-index', + 'ns-a-history-meta', + 'ns-a-history-pending', + ]); +}); diff --git a/packages/history/test/wal.test.ts b/packages/history/test/wal.test.ts new file mode 100644 index 00000000..96628b73 --- /dev/null +++ b/packages/history/test/wal.test.ts @@ -0,0 +1,99 @@ +import { captureContent, keyHash } from '@/chunker'; +import { blobPresent, bumpLamport, markBlobs, openHistoryStores, readLamport } from '@/store'; +import { FLAG_DELETION } from '@/types'; +import { ingestDeletion, ingestEdit } from '@/wal'; +import { expect, test } from 'bun:test'; +import { openMemoryDB } from 'uni-kv'; +import type { HistoryDatabase } from '@/store'; + +const encoder = new TextEncoder(); + +function setup() { + const db = openMemoryDB(`history-${crypto.randomUUID()}`) as never as HistoryDatabase; + return openHistoryStores(db, 'ns-test'); +} + +test('a small note is captured inline and busts the Lamport clock', async () => { + const stores = setup(); + const entry = await ingestEdit(stores, 'note.md', encoder.encode('# Hello')); + expect(entry.text).toBe('# Hello'); + expect(entry.chunkHashes).toBeUndefined(); + expect(entry.lamport).toBe(1); + expect(await readLamport(stores)).toBe(1); +}); + +test('every edit is durable and the Lamport clock is strictly monotone', async () => { + const stores = setup(); + const a = await ingestEdit(stores, 'note.md', encoder.encode('one')); + const b = await ingestEdit(stores, 'note.md', encoder.encode('two')); + const c = await ingestEdit(stores, 'other.md', encoder.encode('three')); + expect([a.lamport, b.lamport, c.lamport]).toStrictEqual([1, 2, 3]); + expect(a.parent).toBe(''); + expect(b.parent).toBe('note.md@1'); + expect(c.parent).toBe(''); + const pending = await stores.pending.values(); + expect(pending).toHaveLength(3); +}); + +test('a large note is split by FastCDC and only unseen chunk bytes are kept', async () => { + const stores = setup(); + const data = new Uint8Array(20 * 1024); + for (let i = 0; i < data.byteLength; i++) data[i] = i & 0xff; + const entry = await ingestEdit(stores, 'big.md', data); + expect(entry.text).toBeUndefined(); + expect(entry.chunkHashes?.length).toBeGreaterThan(5); + expect(entry.newChunks?.length).toBe(new Set(entry.chunkHashes).size); + // Once a flush confirms the chunks in the CAS index, a re-ingest produces no new bytes. + for (const chunk of entry.newChunks ?? []) + await stores.chunks.set(chunk.hash, { + length: chunk.bytes.byteLength, + offset: 0, + pack: { deviceId: 'd', fromLamport: 0, hash: 'h', seq: 0, toLamport: 0 }, + }); + const again = await ingestEdit(stores, 'big.md', data); + expect(again.newChunks).toStrictEqual([]); +}); + +test('a big attachment is stored as a standalone blob', async () => { + const stores = setup(); + const data = new Uint8Array(5 * 1024 * 1024 + 1); + const content = await captureContent(stores, 'image.png', data); + expect(content.blobs).toHaveLength(1); + expect(content.blobs?.[0].hash).toHaveLength(64); + expect(content.chunkHashes).toBeUndefined(); +}); + +test('keyHash is NFC-normalized and content-addressed', async () => { + const a = await keyHash('café.md'); + const b = await keyHash('cafe\u0301.md'); + expect(a).toBe(b); + expect(a).toHaveLength(64); +}); + +test('confirmed-present blob set is persisted and queryable', async () => { + const stores = setup(); + const blob = { hash: 'abc123', key: 'image.png', size: 10 }; + expect(await blobPresent(stores, blob.hash)).toBe(false); + await markBlobs(stores, [blob]); + expect(await blobPresent(stores, blob.hash)).toBe(true); +}); + +test('the Lamport clock folds in observed remote lamports', async () => { + const stores = setup(); + await ingestEdit(stores, 'note.md', encoder.encode('one')); + expect(await readLamport(stores)).toBe(1); + // A remote head log reports a higher peak; the next local edit must be ordered after it. + await bumpLamport(stores, 50); + expect(await readLamport(stores)).toBeGreaterThan(50); + const next = await ingestEdit(stores, 'note.md', encoder.encode('two')); + expect(next.lamport).toBeGreaterThan(50); +}); + +test('ingestDeletion captures a deletion with FLAG_DELETION and bumps Lamport', async () => { + const stores = setup(); + await ingestEdit(stores, 'deleted.md', encoder.encode('before delete')); + const deletion = await ingestDeletion(stores, 'deleted.md'); + expect(deletion.flags).toBe(FLAG_DELETION); + expect(deletion.parent).toBe('deleted.md@1'); + expect(deletion.lamport).toBe(2); +}); diff --git a/packages/history/tsconfig.json b/packages/history/tsconfig.json new file mode 100644 index 00000000..ac4e13f2 --- /dev/null +++ b/packages/history/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { "paths": { "@/*": ["./src/*"], "@": ["./src"] } }, + "include": ["src/**/*.ts", "test/**/*.ts", "tsdown.config.ts"] +} diff --git a/packages/history/tsdown.config.ts b/packages/history/tsdown.config.ts new file mode 100644 index 00000000..24bd3edf --- /dev/null +++ b/packages/history/tsdown.config.ts @@ -0,0 +1,13 @@ +import syncEngineModule from '@hesprs/sync-engine-sdk/tsdown-plugin'; +import moduleCatalog from '@repo/shared/module-catalog'; +import { defineConfig } from 'tsdown'; + +export default defineConfig({ + clean: process.env.MODE !== 'dev', + dts: false, + entry: { history: 'src/index.ts' }, + minify: true, + outExtensions: () => ({ js: '.js' }), + outputOptions: { codeSplitting: false }, + plugins: [syncEngineModule(moduleCatalog)], +}); diff --git a/packages/plugin/dist/dev.spec.d.ts b/packages/plugin/dist/dev.spec.d.ts index 7f137b6f..efb81ecd 100644 --- a/packages/plugin/dist/dev.spec.d.ts +++ b/packages/plugin/dist/dev.spec.d.ts @@ -1,4 +1,6 @@ -import { Bt as Binary, Ft as RecordStat, It as RecordStatsMap, K as TaskNames, Lt as Stat, Mt as FolderStat, Nt as MaybePromise, Ot as RootFs, Rt as StatsMap, d as RequestResponse, jt as FileStat, kt as WrappedFs, l as Request, u as RequestParam, yt as Fs, z as Decider, zt as StringKV } from "./index-r0HV0XqQ.spec.js"; +import { At as RootFs, Bt as StatsMap, Ft as MaybePromise, Ht as Binary, J as TaskNames, Lt as RecordStat, Nt as FileStat, Pt as FolderStat, Rt as RecordStatsMap, V as Decider, Vt as StringKV, d as RequestResponse, jt as WrappedFs, l as Request, u as RequestParam, xt as Fs, zt as Stat } from "./index-BcHUWKEe.spec.js"; +/** Full SHA-256 over UTF-8 text or raw bytes, lowercase hex. */ +export declare function sha256(input: string | Uint8Array): Promise; declare function debugWrapper(original: Fs, log: (content: string) => void): WrappedFs; type FsCalls = { delete: Array; @@ -80,5 +82,4 @@ declare const testKit: { taskKeys: typeof taskKeys; taskNames: typeof taskNames; }; -declare function sha256(input: string): Promise; -export { debugWrapper, sha256, testKit }; \ No newline at end of file +export { debugWrapper, testKit }; \ No newline at end of file diff --git a/packages/plugin/dist/index-r0HV0XqQ.spec.d.ts b/packages/plugin/dist/index-BcHUWKEe.spec.d.ts similarity index 94% rename from packages/plugin/dist/index-r0HV0XqQ.spec.d.ts rename to packages/plugin/dist/index-BcHUWKEe.spec.d.ts index 4e2c3ef3..cde38b1d 100644 --- a/packages/plugin/dist/index-r0HV0XqQ.spec.d.ts +++ b/packages/plugin/dist/index-BcHUWKEe.spec.d.ts @@ -705,6 +705,73 @@ declare function pipe({ from, to, stat, key }: { }): MaybePromise; declare function readWithSize(fs: Fs, key: string, stat: FileStat): Promise>; declare function writeWithSize(fs: Fs, key: string, value: MaybePromise>, stat: FileStat): MaybePromise; +type SyncStage = 'none' | 'walkingRemote' | 'awaitingConfirmation' | 'executing' | 'completed' | 'completedNoop' | 'cancelled' | 'failed'; +type StatusLevel = 'ok' | 'warning' | 'error'; +type ModuleStatus = { + level: StatusLevel; + message?: string; +}; +declare class Observability { + private readonly ctx; + private lastSyncTime; + private lastFailure?; + private readonly sinceLastSyncText; + private readonly syncStage; + private readonly walkProgress; + private readonly executionProgress; + private readonly cleanupCallbacks; + private readonly moduleStatuses; + private startIconEl?; + private statusIconEl?; + private statusEl?; + private readonly t; + private readonly progressText; + readonly settings: { + noticeStatusOnMobile: boolean; + exportLogsDirectory: string; + }; + readonly i18n: { + startSync: string; + startNonInteractiveSync: string; + stopSync: string; + showProgress: string; + exportLogsToFile: string; + exportLogsFailed: string; + idle: string; + }; + constructor(ctx: { + addStatusBarItem: () => HTMLElement; + on: On; + translate: Translate; + isIdle: Ref; + dispatch: Dispatch; + requestSync: (trigger: string) => Promise; + showProgress: () => void; + addCommand: (command: Command) => Command; + addRibbonIcon: Plugin['addRibbonIcon']; + getLogs: () => string; + app: App; + }); + readonly setModuleStatus: (id: string, status?: ModuleStatus) => (() => void); + readonly getModuleStatus: (id: string) => ModuleStatus | undefined; + private computeBadge; + private readonly updateAllIcons; + private readonly cleanupMobileHeaderIcons; + private readonly updateMobileHeaderIcons; + readonly start: () => void; + private readonly setupStatus; + private readonly setupCommands; + private readonly exportLogs; + readonly dispose: () => void; + root: { + executionProgress: Ref>; + exportLogs: () => Promise; + getModuleStatus: (id: string) => ModuleStatus | undefined; + setModuleStatus: (id: string, status?: ModuleStatus) => (() => void); + syncStage: Ref; + walkProgress: Ref; + }; +} declare function digOriginal(wrapped: Fs): RootFs; type SelectFromContext = Context extends O ? O : never; type HeadSettingTranslations = { @@ -841,6 +908,7 @@ declare class Bootstrap { dontSync: string; } & ControlsSettingTranslations & DevelopmentSettingTranslations & FeaturesSettingTranslations & HeadSettingTranslations & MiscellaneousSettingTranslations & UntrustedModuleTranslations & FileTreeTranslations & ModulesTranslations; readonly settings: { + maxMemoryConsumption: TogglableValue; maxRequestConcurrency: TogglableValue; minRequestInterval: TogglableValue; realtimeSyncFastMode: boolean; @@ -872,56 +940,6 @@ declare class Bootstrap { readonly start: () => void; readonly dispose: () => void; } -type SyncStage = 'none' | 'walkingRemote' | 'awaitingConfirmation' | 'executing' | 'completed' | 'completedNoop' | 'cancelled' | 'failed'; -declare class Observability { - private readonly ctx; - private lastSyncTime; - private lastFailure?; - private readonly sinceLastSyncText; - private readonly syncStage; - private readonly walkProgress; - private readonly executionProgress; - private readonly cleanupCallbacks; - private readonly t; - private readonly progressText; - readonly settings: { - noticeStatusOnMobile: boolean; - exportLogsDirectory: string; - }; - readonly i18n: { - startSync: string; - startNonInteractiveSync: string; - stopSync: string; - showProgress: string; - exportLogsToFile: string; - exportLogsFailed: string; - idle: string; - }; - constructor(ctx: { - addStatusBarItem: () => HTMLElement; - on: On; - translate: Translate; - isIdle: Ref; - dispatch: Dispatch; - requestSync: (trigger: string) => Promise; - showProgress: () => void; - addCommand: (command: Command) => Command; - addRibbonIcon: Plugin['addRibbonIcon']; - getLogs: () => string; - app: App; - }); - readonly start: () => void; - private readonly setupStatus; - private readonly setupCommands; - private readonly exportLogs; - readonly dispose: () => void; - root: { - executionProgress: Ref>; - exportLogs: () => Promise; - syncStage: Ref; - walkProgress: Ref; - }; -} type DeleteConfirmReturn = { delete: Array; reupload: Array; @@ -1113,6 +1131,7 @@ declare class Sync { }; readonly settings: { maxFileSize: TogglableValue; + maxMemoryConsumption?: TogglableValue; syncStrategy: Array; }; readonly i18n: { @@ -1304,4 +1323,4 @@ type VaultRequest = (key: string, params?: T) => Promise; type TrashOption = 'local' | 'system' | 'permanent'; -export { MoveLocal as $, setNeedMigration as A, WriteAtom as At, DeciderInput as B, Binary as Bt, REGRET_ERROR_MESSAGE as C, MkdirAtom as Ct, readWithSize as D, OutputAtom as Dt, pipe as E, OptimizerOutput as Et, CallableOrObjectTree as F, RecordStat as Ft, Task as G, AddRecord as H, SettingEntry as I, RecordStatsMap as It, ResolveConflict as J, TaskNames as K, AugmentedModuleMeta as L, Stat as Lt, generateEditableList as M, FolderStat as Mt, reactivelyValidate as N, MaybePromise as Nt, writeWithSize as O, RootFs as Ot, settingEntry as P, Progress as Pt, MoveRemote as Q, ModuleMeta as R, StatsMap as Rt, digOriginal as S, ListReporter as St, concurrency as T, OptimizerInput as Tt, ConflictResolver as U, TaskFactory as V, ConflictResolverPayload as W, RemoveRecord as X, RemoveRemote as Y, RemoveLocal as Z, Events as _, CustomAtom as _t, LocalRequestMiddlewareEntry as a, ObsidianLanguageCode as at, ExistingMemoryDB as b, InputAtom as bt, RemoteRequestMiddlewareEntry as c, TranslationResource as ct, RequestResponse as d, DatabaseAsync as dt, Download as et, TriggerEntry as f, DatabaseSync as ft, Context as g, BatchOptimizer as gt, SyncTerminateReason as h, StoreSync as ht, FsWrapperEntry as i, Fragment as it, LabelDefinition as j, FileStat as jt, prefixWrapper as k, WrappedFs as kt, Request as l, Dispatch as lt, SyncOptions as m, StoreOperations as mt, ConflictResolverEntry as n, CreateLocalDir as nt, OptimizerEntry as o, Snippet as ot, RemoteLister as p, StoreAsync as pt, Upload as q, DeciderEntry as r, RecordStore as rt, RemoteFsEntry as s, Translate as st, VaultRequest as t, CreateRemoteDir as tt, RequestParam as u, On as ut, Settings as v, DeleteAtom as vt, chunkSize as w, MoveAtom as wt, SelectFromContext as x, ListOptions as xt, Translations as y, Fs as yt, Decider as z, StringKV as zt }; \ No newline at end of file +export { RemoveLocal as $, writeWithSize as A, RootFs as At, ModuleMeta as B, StatsMap as Bt, ModuleStatus as C, ListOptions as Ct, concurrency as D, OptimizerInput as Dt, chunkSize as E, MoveAtom as Et, reactivelyValidate as F, MaybePromise as Ft, ConflictResolver as G, DeciderInput as H, Binary as Ht, settingEntry as I, Progress as It, TaskNames as J, ConflictResolverPayload as K, CallableOrObjectTree as L, RecordStat as Lt, setNeedMigration as M, WriteAtom as Mt, LabelDefinition as N, FileStat as Nt, pipe as O, OptimizerOutput as Ot, generateEditableList as P, FolderStat as Pt, RemoveRecord as Q, SettingEntry as R, RecordStatsMap as Rt, digOriginal as S, InputAtom as St, REGRET_ERROR_MESSAGE as T, MkdirAtom as Tt, TaskFactory as U, Decider as V, StringKV as Vt, AddRecord as W, ResolveConflict as X, Upload as Y, RemoveRemote as Z, Events as _, StoreSync as _t, LocalRequestMiddlewareEntry as a, RecordStore as at, ExistingMemoryDB as b, DeleteAtom as bt, RemoteRequestMiddlewareEntry as c, Snippet as ct, RequestResponse as d, Dispatch as dt, MoveRemote as et, TriggerEntry as f, On as ft, Context as g, StoreOperations as gt, SyncTerminateReason as h, StoreAsync as ht, FsWrapperEntry as i, CreateLocalDir as it, prefixWrapper as j, WrappedFs as jt, readWithSize as k, OutputAtom as kt, Request as l, Translate as lt, SyncOptions as m, DatabaseSync as mt, ConflictResolverEntry as n, Download as nt, OptimizerEntry as o, Fragment as ot, RemoteLister as p, DatabaseAsync as pt, Task as q, DeciderEntry as r, CreateRemoteDir as rt, RemoteFsEntry as s, ObsidianLanguageCode as st, VaultRequest as t, MoveLocal as tt, RequestParam as u, TranslationResource as ut, Settings as v, BatchOptimizer as vt, StatusLevel as w, ListReporter as wt, SelectFromContext as x, Fs as xt, Translations as y, CustomAtom as yt, AugmentedModuleMeta as z, Stat as zt }; \ No newline at end of file diff --git a/packages/plugin/dist/index.spec.d.ts b/packages/plugin/dist/index.spec.d.ts index 62c5ee87..eb88f235 100644 --- a/packages/plugin/dist/index.spec.d.ts +++ b/packages/plugin/dist/index.spec.d.ts @@ -1,2 +1,2 @@ -import { $ as MoveLocal, A as setNeedMigration, At as WriteAtom, B as DeciderInput, Bt as Binary, C as REGRET_ERROR_MESSAGE, Ct as MkdirAtom, D as readWithSize, Dt as OutputAtom, E as pipe, Et as OptimizerOutput, F as CallableOrObjectTree, Ft as RecordStat, G as Task, H as AddRecord, I as SettingEntry, It as RecordStatsMap, J as ResolveConflict, K as TaskNames, L as AugmentedModuleMeta, Lt as Stat, M as generateEditableList, Mt as FolderStat, N as reactivelyValidate, Nt as MaybePromise, O as writeWithSize, Ot as RootFs, P as settingEntry, Pt as Progress, Q as MoveRemote, R as ModuleMeta, Rt as StatsMap, S as digOriginal, St as ListReporter, T as concurrency, Tt as OptimizerInput, U as ConflictResolver, V as TaskFactory, W as ConflictResolverPayload, X as RemoveRecord, Y as RemoveRemote, Z as RemoveLocal, _ as Events, _t as CustomAtom, a as LocalRequestMiddlewareEntry, at as ObsidianLanguageCode, b as ExistingMemoryDB, bt as InputAtom, c as RemoteRequestMiddlewareEntry, ct as TranslationResource, d as RequestResponse, dt as DatabaseAsync, et as Download, f as TriggerEntry, ft as DatabaseSync, g as Context, gt as BatchOptimizer, h as SyncTerminateReason, ht as StoreSync, i as FsWrapperEntry, it as Fragment, j as LabelDefinition, jt as FileStat, k as prefixWrapper, kt as WrappedFs, l as Request, lt as Dispatch, m as SyncOptions, mt as StoreOperations, n as ConflictResolverEntry, nt as CreateLocalDir, o as OptimizerEntry, ot as Snippet, p as RemoteLister, pt as StoreAsync, q as Upload, r as DeciderEntry, rt as RecordStore, s as RemoteFsEntry, st as Translate, t as VaultRequest, tt as CreateRemoteDir, u as RequestParam, ut as On, v as Settings, vt as DeleteAtom, w as chunkSize, wt as MoveAtom, x as SelectFromContext, xt as ListOptions, y as Translations, yt as Fs, z as Decider, zt as StringKV } from "./index-r0HV0XqQ.spec.js"; -export { type AddRecord, type AugmentedModuleMeta, type BatchOptimizer, type Binary, type CallableOrObjectTree, type ConflictResolver, type ConflictResolverEntry, type ConflictResolverPayload, type Context, type CreateLocalDir, type CreateRemoteDir, type CustomAtom, type DatabaseAsync, type DatabaseSync, type Decider, type DeciderEntry, type DeciderInput, type DeleteAtom, type Dispatch, type Download, type Events, type ExistingMemoryDB, type FileStat, type FolderStat, type Fragment, type Fs, type FsWrapperEntry, type InputAtom, type LabelDefinition, type ListOptions, type ListReporter, type LocalRequestMiddlewareEntry, type MaybePromise, type MkdirAtom, type ModuleMeta, type MoveAtom, type MoveLocal, type MoveRemote, type ObsidianLanguageCode, type On, type OptimizerEntry, type OptimizerInput, type OptimizerOutput, type OutputAtom, type Progress, REGRET_ERROR_MESSAGE, type RecordStat, type RecordStatsMap, type RecordStore, type RemoteFsEntry, type RemoteLister, type RemoteRequestMiddlewareEntry, type RemoveLocal, type RemoveRecord, type RemoveRemote, type Request, type RequestParam, type RequestResponse, type ResolveConflict, type RootFs, SelectFromContext, type SettingEntry, type Settings, type Snippet, type Stat, type StatsMap, type StoreAsync, type StoreOperations, type StoreSync, type StringKV, type SyncOptions, type SyncTerminateReason, type Task, type TaskFactory, type TaskNames, type Translate, type TranslationResource, type Translations, type TriggerEntry, type Upload, type VaultRequest, type WrappedFs, type WriteAtom, chunkSize, concurrency, digOriginal, generateEditableList, pipe, prefixWrapper, reactivelyValidate, readWithSize, setNeedMigration, settingEntry, writeWithSize }; \ No newline at end of file +import { $ as RemoveLocal, A as writeWithSize, At as RootFs, B as ModuleMeta, Bt as StatsMap, C as ModuleStatus, Ct as ListOptions, D as concurrency, Dt as OptimizerInput, E as chunkSize, Et as MoveAtom, F as reactivelyValidate, Ft as MaybePromise, G as ConflictResolver, H as DeciderInput, Ht as Binary, I as settingEntry, It as Progress, J as TaskNames, K as ConflictResolverPayload, L as CallableOrObjectTree, Lt as RecordStat, M as setNeedMigration, Mt as WriteAtom, N as LabelDefinition, Nt as FileStat, O as pipe, Ot as OptimizerOutput, P as generateEditableList, Pt as FolderStat, Q as RemoveRecord, R as SettingEntry, Rt as RecordStatsMap, S as digOriginal, St as InputAtom, T as REGRET_ERROR_MESSAGE, Tt as MkdirAtom, U as TaskFactory, V as Decider, Vt as StringKV, W as AddRecord, X as ResolveConflict, Y as Upload, Z as RemoveRemote, _ as Events, _t as StoreSync, a as LocalRequestMiddlewareEntry, at as RecordStore, b as ExistingMemoryDB, bt as DeleteAtom, c as RemoteRequestMiddlewareEntry, ct as Snippet, d as RequestResponse, dt as Dispatch, et as MoveRemote, f as TriggerEntry, ft as On, g as Context, gt as StoreOperations, h as SyncTerminateReason, ht as StoreAsync, i as FsWrapperEntry, it as CreateLocalDir, j as prefixWrapper, jt as WrappedFs, k as readWithSize, kt as OutputAtom, l as Request, lt as Translate, m as SyncOptions, mt as DatabaseSync, n as ConflictResolverEntry, nt as Download, o as OptimizerEntry, ot as Fragment, p as RemoteLister, pt as DatabaseAsync, q as Task, r as DeciderEntry, rt as CreateRemoteDir, s as RemoteFsEntry, st as ObsidianLanguageCode, t as VaultRequest, tt as MoveLocal, u as RequestParam, ut as TranslationResource, v as Settings, vt as BatchOptimizer, w as StatusLevel, wt as ListReporter, x as SelectFromContext, xt as Fs, y as Translations, yt as CustomAtom, z as AugmentedModuleMeta, zt as Stat } from "./index-BcHUWKEe.spec.js"; +export { type AddRecord, type AugmentedModuleMeta, type BatchOptimizer, type Binary, type CallableOrObjectTree, type ConflictResolver, type ConflictResolverEntry, type ConflictResolverPayload, type Context, type CreateLocalDir, type CreateRemoteDir, type CustomAtom, type DatabaseAsync, type DatabaseSync, type Decider, type DeciderEntry, type DeciderInput, type DeleteAtom, type Dispatch, type Download, type Events, type ExistingMemoryDB, type FileStat, type FolderStat, type Fragment, type Fs, type FsWrapperEntry, type InputAtom, type LabelDefinition, type ListOptions, type ListReporter, type LocalRequestMiddlewareEntry, type MaybePromise, type MkdirAtom, type ModuleMeta, type ModuleStatus, type MoveAtom, type MoveLocal, type MoveRemote, type ObsidianLanguageCode, type On, type OptimizerEntry, type OptimizerInput, type OptimizerOutput, type OutputAtom, type Progress, REGRET_ERROR_MESSAGE, type RecordStat, type RecordStatsMap, type RecordStore, type RemoteFsEntry, type RemoteLister, type RemoteRequestMiddlewareEntry, type RemoveLocal, type RemoveRecord, type RemoveRemote, type Request, type RequestParam, type RequestResponse, type ResolveConflict, type RootFs, SelectFromContext, type SettingEntry, type Settings, type Snippet, type Stat, type StatsMap, type StatusLevel, type StoreAsync, type StoreOperations, type StoreSync, type StringKV, type SyncOptions, type SyncTerminateReason, type Task, type TaskFactory, type TaskNames, type Translate, type TranslationResource, type Translations, type TriggerEntry, type Upload, type VaultRequest, type WrappedFs, type WriteAtom, chunkSize, concurrency, digOriginal, generateEditableList, pipe, prefixWrapper, reactivelyValidate, readWithSize, setNeedMigration, settingEntry, writeWithSize }; \ No newline at end of file diff --git a/packages/plugin/src/components/ModuleEditorModal.ts b/packages/plugin/src/components/ModuleEditorModal.ts index 4e14eda6..fb9a3e47 100644 --- a/packages/plugin/src/components/ModuleEditorModal.ts +++ b/packages/plugin/src/components/ModuleEditorModal.ts @@ -1,6 +1,6 @@ import { reactivelyValidate } from '@/settings/utils'; -import sha256 from '@/utils/sha-256'; import { formatUrl, parseUrl } from '@repo/shared/path'; +import { sha256 } from '@repo/shared/sha256-bytes'; import { App, Modal, Setting } from 'obsidian'; import type { AugmentedModuleMeta } from '@/modules/Extensibility'; import type { Fragment, Translate } from '@/modules/I18n'; diff --git a/packages/plugin/src/constants.ts b/packages/plugin/src/constants.ts new file mode 100644 index 00000000..ee5d7926 --- /dev/null +++ b/packages/plugin/src/constants.ts @@ -0,0 +1,66 @@ +/** + * Sync Engine Plugin Core Constants + * + * Centralizes default synchronization cadences, debounce delays, memory and concurrency budgets, + * file size limits, and UI update intervals. + */ + +// ========================================== +// Sync Cadence & Timing Defaults +// ========================================== + +/** + * Default delay (in ms) for startup sync after Obsidian's workspace layout is ready. 5 seconds + * allows initial index and plugins to settle before initiating network sync. + */ +export const DEFAULT_STARTUP_SYNC_DELAY_MS = 5000; + +/** + * Default debounce delay (in ms) for realtime sync after vault file modifications. 5 seconds + * provides a gentle batching window for multi-file operations. + */ +export const DEFAULT_REALTIME_SYNC_DELAY_MS = 5000; + +/** + * Default interval (in ms) for scheduled periodic sync. 5 minutes provides regular synchronization + * for background changes. + */ +export const DEFAULT_SCHEDULED_SYNC_INTERVAL_MS = 300_000; + +/** + * Default minimum cooldown gap (in ms) required between focus sync runs. Prevents rapid switching + * between apps/tabs from triggering redundant syncs. + */ +export const DEFAULT_FOCUS_SYNC_GAP_MS = 300_000; + +// ========================================== +// Resource, Concurrency & Memory Budgets +// ========================================== + +/** + * Default maximum file size (in bytes) eligible for automatic synchronization (30 MB). Files + * exceeding this threshold require explicit user opt-in or larger limit. + */ +export const DEFAULT_MAX_FILE_SIZE_BYTES = 31_457_280; + +/** + * Default maximum memory consumption budget (in bytes) for task execution (64 MB). Enforces + * streaming and backpressure when operating on large vaults or files. + */ +export const DEFAULT_MAX_MEMORY_BYTES = 64 * 1024 ** 2; + +/** Default maximum concurrent network requests permitted across the worker pool. */ +export const DEFAULT_MAX_REQUEST_CONCURRENCY = 50; + +// ========================================== +// UI, Observability & Logging Limits +// ========================================== + +/** Delay (in ms) before auto-hiding the mobile sync completion notice. */ +export const MOBILE_SYNC_NOTICE_HIDE_DELAY_MS = 2000; + +/** Interval (in ms) for updating the "since last sync" UI relative timestamp text. */ +export const SINCE_LAST_SYNC_UPDATE_INTERVAL_MS = 60_000; + +/** Maximum number of sync log records retained in the circular log buffer. */ +export const MAX_SYNC_LOGS = 100; diff --git a/packages/plugin/src/fs/middlewares/retry.ts b/packages/plugin/src/fs/middlewares/retry.ts index be7835e9..f553e8e5 100644 --- a/packages/plugin/src/fs/middlewares/retry.ts +++ b/packages/plugin/src/fs/middlewares/retry.ts @@ -70,9 +70,17 @@ const RETRYABLE_MESSAGE_PATTERNS = [ /\bsocket hang up\b/iu, /\bconnection closed\b/iu, /\bconnection reset\b/iu, - /\bconnection aborted\b/iu, + /\bconnection abort(?:ed)?\b/iu, /\bconnection refused\b/iu, + /\bstream closed\b/iu, + /\bstream was reset\b/iu, + /\bunexpected end of stream\b/iu, + /\bbroken pipe\b/iu, + /\bsocket closed\b/iu, + /\bnetwork is unreachable\b/iu, + /\bfailed to connect\b/iu, /\btemporarily unavailable\b/iu, + /\btime-?out\b/iu, /\btimed out\b/iu, ]; diff --git a/packages/plugin/src/index.ts b/packages/plugin/src/index.ts index f5d89c2a..a91ec0f0 100644 --- a/packages/plugin/src/index.ts +++ b/packages/plugin/src/index.ts @@ -1,5 +1,13 @@ +import { + DEFAULT_FOCUS_SYNC_GAP_MS, + DEFAULT_MAX_FILE_SIZE_BYTES, + DEFAULT_MAX_MEMORY_BYTES, + DEFAULT_MAX_REQUEST_CONCURRENCY, + DEFAULT_REALTIME_SYNC_DELAY_MS, + DEFAULT_SCHEDULED_SYNC_INTERVAL_MS, + DEFAULT_STARTUP_SYNC_DELAY_MS, +} from '@/constants'; import '@/global.css'; -import { NONE_STRATEGY } from './utils/glob-match'; import Bootstrap from '@/modules/Bootstrap'; import EventBus from '@/modules/EventBus'; import Extensibility, { OFFICIAL_SOURCE } from '@/modules/Extensibility'; @@ -11,6 +19,7 @@ import Scheduler from '@/modules/Scheduler'; import Setting from '@/modules/Setting'; import Storage from '@/modules/Storage'; import Sync from '@/modules/Sync'; +import { NONE_STRATEGY } from '@repo/shared/glob-match'; import { Plugin } from 'obsidian'; import { createContext } from 'synthkernel'; import type { App, PluginManifest } from 'obsidian'; @@ -80,21 +89,20 @@ export default class SyncEngine extends Plugin { conflictResolver: 'renameAndKeepBoth', customHeaders: [], exportLogsDirectory: 'Sync Engine Logs/', - focusSync: { enabled: false, value: 300_000 }, - maxFileSize: { enabled: false, value: 31_457_280 }, - // See Bootstrap TODO - // maxMemoryConsumption: { enabled: true, value: 100 * 1024 ** 2 }, - maxRequestConcurrency: { enabled: true, value: 50 }, + focusSync: { enabled: false, value: DEFAULT_FOCUS_SYNC_GAP_MS }, + maxFileSize: { enabled: false, value: DEFAULT_MAX_FILE_SIZE_BYTES }, + maxMemoryConsumption: { enabled: false, value: DEFAULT_MAX_MEMORY_BYTES }, + maxRequestConcurrency: { enabled: true, value: DEFAULT_MAX_REQUEST_CONCURRENCY }, minRequestInterval: { enabled: false, value: 0 }, moduleAutoUpdate: true, moduleSources: [OFFICIAL_SOURCE], modules: {}, noticeStatusOnMobile: true, - realtimeSync: { enabled: false, value: 5000 }, + realtimeSync: { enabled: false, value: DEFAULT_REALTIME_SYNC_DELAY_MS }, realtimeSyncFastMode: true, remoteFs: '', - scheduledSync: { enabled: false, value: 300_000 }, - startupSync: { enabled: false, value: 5000 }, + scheduledSync: { enabled: false, value: DEFAULT_SCHEDULED_SYNC_INTERVAL_MS }, + startupSync: { enabled: false, value: DEFAULT_STARTUP_SYNC_DELAY_MS }, syncStrategy: [ { expr: '*', strategy: 'bidirectional' }, ...[ diff --git a/packages/plugin/src/modules/Bootstrap.ts b/packages/plugin/src/modules/Bootstrap.ts index 0a7adffe..8bb0c9e5 100644 --- a/packages/plugin/src/modules/Bootstrap.ts +++ b/packages/plugin/src/modules/Bootstrap.ts @@ -20,7 +20,7 @@ import { renameAndKeepBothResolver, pholderDecider, } from '@/sync'; -import { NONE_STRATEGY } from '@/utils/glob-match'; +import { NONE_STRATEGY } from '@repo/shared/glob-match'; import type { Dispatch, On } from './EventBus'; import type { ObsidianLanguageCode, Translate, TranslationResource } from './I18n'; import type { @@ -88,8 +88,7 @@ export default class Bootstrap { FileTreeTranslations & ModulesTranslations; declare readonly settings: { - // See below - // maxMemoryConsumption: TogglableValue; + maxMemoryConsumption: TogglableValue; maxRequestConcurrency: TogglableValue; minRequestInterval: TogglableValue; realtimeSyncFastMode: boolean; @@ -178,6 +177,32 @@ export default class Bootstrap { }), priority: 1000, }); + registerTrigger('collaboration', { + options: () => ({ + needConfirmDeletion: getDeletionConfirm(), + remoteLister: + this.settings.realtimeSyncFastMode && context20000.keys().length + ? ({ reporter }) => { + const entries = context20000.values(); + const filtered: Array = []; + return Promise.all( + entries.map(async (stat, index) => { + if ( + (await reporter({ + completed: index + 1, + current: stat.key, + total: entries.length, + })) === 'exclude' + ) + return; + filtered.push(stat); + }), + ).then(() => filtered); + } + : undefined, + }), + priority: 1500, + }); registerTrigger('interval', { options: () => ({ needConfirmDeletion: getDeletionConfirm() }), priority: 2000, diff --git a/packages/plugin/src/modules/EventBus.ts b/packages/plugin/src/modules/EventBus.ts index e543b72b..c78cac79 100644 --- a/packages/plugin/src/modules/EventBus.ts +++ b/packages/plugin/src/modules/EventBus.ts @@ -1,3 +1,4 @@ +import { MAX_SYNC_LOGS } from '@/constants'; import formatDateTime from '@/utils/format-date'; import { formatTime } from '@/utils/unit-converter'; import { describeError } from '@repo/shared/error'; @@ -17,7 +18,6 @@ export const OS = { macOS: Platform.isMacOS, Windows: Platform.isWin, }; -const MAX_SYNC_LOGS = 100; export type Dispatch = ( ...[key, payload]: undefined extends O[K] ? [K] : [K, O[K]] diff --git a/packages/plugin/src/modules/Extensibility.ts b/packages/plugin/src/modules/Extensibility.ts index efb6310a..386851ee 100644 --- a/packages/plugin/src/modules/Extensibility.ts +++ b/packages/plugin/src/modules/Extensibility.ts @@ -1,10 +1,10 @@ import UntrustedModuleModal from '@/components/UntrustedModuleModal'; -import sha256 from '@/utils/sha-256'; import untilTrue from '@/utils/until-true'; import hash from '@repo/shared/crypto'; import { importCode } from '@repo/shared/e2e-utils.spec'; import { describeError, toError } from '@repo/shared/error'; import { parseUrl } from '@repo/shared/path'; +import { sha256 } from '@repo/shared/sha256-bytes'; import obsidian, { Notice, requestUrl } from 'obsidian'; import { compare } from 'verkit'; import type { Dispatch } from './EventBus'; diff --git a/packages/plugin/src/modules/Observability.ts b/packages/plugin/src/modules/Observability.ts index 519ef53d..adcbe4c8 100644 --- a/packages/plugin/src/modules/Observability.ts +++ b/packages/plugin/src/modules/Observability.ts @@ -1,3 +1,4 @@ +import { MOBILE_SYNC_NOTICE_HIDE_DELAY_MS, SINCE_LAST_SYNC_UPDATE_INTERVAL_MS } from '@/constants'; import roundPercent from '@/utils/round-percent'; import { formatTime } from '@/utils/unit-converter'; import { describeError, toError } from '@repo/shared/error'; @@ -8,7 +9,7 @@ import type { Translate } from './I18n'; import type { SyncTerminateReason, TaskInfo } from './Sync'; import type { Events, Translations } from '@'; import type { Progress } from '@/types'; -import type { App, Command, DataAdapter, Plugin } from 'obsidian'; +import type { App, Command, DataAdapter, Plugin, WorkspaceLeaf } from 'obsidian'; import type { Ref } from 'synthkernel'; export type SyncStage = @@ -21,7 +22,72 @@ export type SyncStage = | 'cancelled' | 'failed'; -const MOBILE_SYNC_NOTICE_HIDE_DELAY = 2000; +export type StatusLevel = 'ok' | 'warning' | 'error'; + +export type ModuleStatus = { level: StatusLevel; message?: string }; + +function renderSyncIcon( + container: HTMLElement, + badge?: 'warning' | 'error', + spinning = false, +): void { + container.empty(); + container.style.position = 'relative'; + container.style.display = 'inline-flex'; + container.style.alignItems = 'center'; + container.style.justifyContent = 'center'; + + const mainWrapper = container.createSpan({ cls: 'sync-engine-main-icon' }); + mainWrapper.style.display = 'inline-flex'; + mainWrapper.style.alignItems = 'center'; + mainWrapper.style.justifyContent = 'center'; + setIcon(mainWrapper, 'refresh-cw'); + if (spinning) mainWrapper.addClass('animate-spin'); + const mainSvg = mainWrapper.firstElementChild as HTMLElement | undefined; + if (mainSvg && spinning) mainSvg.addClass('animate-spin'); + + if (badge === 'warning') { + const badgeWrapper = container.createSpan({ cls: 'sync-engine-badge-icon' }); + badgeWrapper.style.position = 'absolute'; + badgeWrapper.style.bottom = '-2px'; + badgeWrapper.style.right = '-2px'; + badgeWrapper.style.width = '60%'; + badgeWrapper.style.height = '60%'; + badgeWrapper.style.display = 'inline-flex'; + badgeWrapper.style.alignItems = 'center'; + badgeWrapper.style.justifyContent = 'center'; + badgeWrapper.style.pointerEvents = 'none'; + setIcon(badgeWrapper, 'alert-circle'); + const badgeSvg = badgeWrapper.firstElementChild as HTMLElement | null; + if (badgeSvg) { + badgeSvg.style.width = '100%'; + badgeSvg.style.height = '100%'; + badgeSvg.style.stroke = 'var(--text-warning, #f59e0b)'; + badgeSvg.style.fill = 'var(--background-primary, #1e1e1e)'; + badgeSvg.style.strokeWidth = '2.5'; + } + } else if (badge === 'error') { + const badgeWrapper = container.createSpan({ cls: 'sync-engine-badge-icon' }); + badgeWrapper.style.position = 'absolute'; + badgeWrapper.style.bottom = '-2px'; + badgeWrapper.style.right = '-2px'; + badgeWrapper.style.width = '60%'; + badgeWrapper.style.height = '60%'; + badgeWrapper.style.display = 'inline-flex'; + badgeWrapper.style.alignItems = 'center'; + badgeWrapper.style.justifyContent = 'center'; + badgeWrapper.style.pointerEvents = 'none'; + setIcon(badgeWrapper, 'x-circle'); + const badgeSvg = badgeWrapper.firstElementChild as HTMLElement | null; + if (badgeSvg) { + badgeSvg.style.width = '100%'; + badgeSvg.style.height = '100%'; + badgeSvg.style.stroke = 'var(--text-error, #f85149)'; + badgeSvg.style.fill = 'var(--background-primary, #1e1e1e)'; + badgeSvg.style.strokeWidth = '2.5'; + } + } +} export default class Observability { private lastSyncTime = 0; @@ -31,6 +97,10 @@ export default class Observability { private readonly walkProgress = ref({ completed: 0, total: 1 }); private readonly executionProgress = ref>({ completed: 0, total: 0 }); private readonly cleanupCallbacks: Array<() => void> = []; + private readonly moduleStatuses = new Map(); + private startIconEl?: HTMLElement; + private statusIconEl?: HTMLElement; + private statusEl?: HTMLElement; private readonly t: Translate; private readonly progressText = computed( () => { @@ -89,6 +159,121 @@ export default class Observability { this.t = ctx.translate; } + readonly setModuleStatus = (id: string, status?: ModuleStatus): (() => void) => { + if (!status || status.level === 'ok') this.moduleStatuses.delete(id); + else this.moduleStatuses.set(id, status); + this.updateAllIcons(); + return () => { + this.moduleStatuses.delete(id); + this.updateAllIcons(); + }; + }; + + readonly getModuleStatus = (id: string): ModuleStatus | undefined => + this.moduleStatuses.get(id); + + private computeBadge(): { badge?: 'warning' | 'error'; message?: string } { + if (this.syncStage() === 'failed' && this.lastFailure) + return { badge: 'error', message: `${this.t('failed')}: ${this.lastFailure}` }; + let hasWarning = false; + let warningMessage: string | undefined; + for (const [id, status] of this.moduleStatuses) { + if (status.level === 'error') return { badge: 'error', message: status.message ?? id }; + if (status.level === 'warning') { + hasWarning = true; + warningMessage ??= status.message ?? id; + } + } + if (hasWarning) return { badge: 'warning', message: warningMessage }; + return {}; + } + + private readonly updateAllIcons = () => { + const isIdle = this.ctx.isIdle(); + const spinning = !isIdle; + const { badge, message } = this.computeBadge(); + + if (this.startIconEl) { + renderSyncIcon(this.startIconEl, badge, spinning); + const baseTitle = this.t('startSync'); + const title = message ? `${baseTitle} (${message})` : baseTitle; + this.startIconEl.setAttribute('aria-label', title); + this.startIconEl.setAttribute('title', title); + } + + if (this.statusIconEl) { + renderSyncIcon(this.statusIconEl, badge, spinning); + if (this.statusEl) + if (message) this.statusEl.setAttribute('title', message); + else this.statusEl.removeAttribute('title'); + } + + this.updateMobileHeaderIcons(); + }; + + private readonly cleanupMobileHeaderIcons = () => { + if (typeof document !== 'undefined') { + document.querySelectorAll('.sync-engine-mobile-action').forEach((el) => el.remove()); + document.querySelectorAll('.sync-engine-collab-action').forEach((el) => el.remove()); + } + if (this.ctx.app.workspace?.iterateRootLeaves) + this.ctx.app.workspace.iterateRootLeaves((leaf: WorkspaceLeaf) => { + const container = leaf.view?.containerEl; + if (!container) return; + const viewActions = + container.querySelector('.view-actions') ?? + container.closest('.workspace-leaf-content')?.querySelector('.view-actions'); + if (!viewActions) return; + viewActions + .querySelectorAll('.sync-engine-mobile-action') + .forEach((el) => el.remove()); + viewActions + .querySelectorAll('.sync-engine-collab-action') + .forEach((el) => el.remove()); + }); + }; + + private readonly updateMobileHeaderIcons = () => { + if (!Platform.isMobile) return; + if (!this.ctx.app.workspace?.iterateRootLeaves) return; + + const isIdle = this.ctx.isIdle(); + const spinning = !isIdle; + const { badge, message } = this.computeBadge(); + const baseTitle = this.t('startSync'); + const title = message ? `${baseTitle} (${message})` : baseTitle; + + this.ctx.app.workspace.iterateRootLeaves((leaf: WorkspaceLeaf) => { + const container = leaf.view?.containerEl; + if (!container) return; + const viewActions = + container.querySelector('.view-actions') ?? + container.closest('.workspace-leaf-content')?.querySelector('.view-actions'); + if (!viewActions) return; + + viewActions.querySelectorAll('.sync-engine-collab-action').forEach((el) => el.remove()); + + let btn = viewActions.querySelector('.sync-engine-mobile-action'); + if (!btn) { + btn = viewActions.createEl('button', { + cls: 'clickable-icon view-action sync-engine-mobile-action', + }); + viewActions.prepend(btn); + btn.addEventListener('click', (e) => { + e.stopPropagation(); + const current = this.computeBadge(); + if (current.message) new Notice(current.message); + if (this.ctx.isIdle()) void this.ctx.requestSync('manual'); + else this.ctx.showProgress(); + }); + } + + renderSyncIcon(btn, badge, spinning); + btn.setAttribute('aria-label', title); + btn.setAttribute('title', title); + }); + }; + readonly start = () => { const { ctx, @@ -116,6 +301,8 @@ export default class Observability { cleanupCallbacks.push( on('syncStarted', () => { syncStage('walkingRemote'); + this.lastFailure = undefined; + this.updateAllIcons(); window.clearInterval(updateInterval); sinceLastSyncText(''); if (settings.noticeStatusOnMobile && Platform.isMobile) { @@ -142,14 +329,14 @@ export default class Observability { noticeTimeout = window.setTimeout(() => { mobileSyncNotice?.hide(); mobileSyncNotice = undefined; - }, MOBILE_SYNC_NOTICE_HIDE_DELAY); + }, MOBILE_SYNC_NOTICE_HIDE_DELAY_MS); this.lastSyncTime = Date.now(); const setUpdateInterval = () => (updateInterval = window.setInterval(() => { const sinceNow = Date.now() - this.lastSyncTime; const time = formatTime(sinceNow); sinceLastSyncText(` ${time} ago`); - }, 60_000)); + }, SINCE_LAST_SYNC_UPDATE_INTERVAL_MS)); if (result === 'cancelled') syncStage('cancelled'); else if (result === 'completed') { syncStage('completed'); @@ -161,6 +348,8 @@ export default class Observability { this.lastFailure = reason.error.message; syncStage('failed'); } + if (result !== 'failed') this.lastFailure = undefined; + this.updateAllIcons(); walkProgress({ completed: 0, total: 1 }); executionProgress({ completed: 0, total: 0 }); }), @@ -174,45 +363,48 @@ export default class Observability { const startIcon = addRibbonIcon('refresh-cw', t('startSync'), () => { if (isIdle()) void requestSync('manual'); }); + this.startIconEl = startIcon; const stopIcon = addRibbonIcon('square', t('stopSync'), () => dispatch('syncCanceled')); cleanupCallbacks.push( isIdle.subscribe( (idle) => { - const svgIcon = startIcon.firstElementChild; - if (!svgIcon) return; if (idle) { startIcon.removeAttribute('aria-disabled'); - svgIcon.removeClass('animate-spin'); stopIcon.addClass('hidden'); } else { startIcon.setAttr('aria-disabled', 'true'); - svgIcon.addClass('animate-spin'); stopIcon.removeClass('hidden'); } + this.updateAllIcons(); }, { immediate: true }, ), ); + if (Platform.isMobile) { + const onLeafChange = () => { + this.updateMobileHeaderIcons(); + }; + const leafRef = this.ctx.app.workspace?.on?.('active-leaf-change', onLeafChange); + const layoutRef = this.ctx.app.workspace?.on?.('layout-change', onLeafChange); + cleanupCallbacks.push(() => { + if (leafRef) this.ctx.app.workspace?.offref?.(leafRef); + if (layoutRef) this.ctx.app.workspace?.offref?.(layoutRef); + this.cleanupMobileHeaderIcons(); + }); + this.updateMobileHeaderIcons(); + } }; private readonly setupStatus = () => { const { ctx, t, progressText } = this; - const { isIdle, addStatusBarItem } = ctx; + const { addStatusBarItem } = ctx; const statusEl = addStatusBarItem(); - setIcon(statusEl, 'refresh-cw'); + this.statusEl = statusEl; + const iconWrap = statusEl.createSpan({ cls: 'sync-engine-status-icon-wrap' }); + this.statusIconEl = iconWrap; const status = statusEl.createSpan({ cls: 'ml-1 max-w-250px truncate', text: t('idle') }); - this.cleanupCallbacks.push( - isIdle.subscribe( - (idle) => { - const icon = statusEl.firstElementChild; - if (!icon) return; - if (idle) icon.removeClass('animate-spin'); - else icon.addClass('animate-spin'); - }, - { immediate: true }, - ), - progressText.subscribe((text) => status.setText(text)), - ); + this.updateAllIcons(); + this.cleanupCallbacks.push(progressText.subscribe((text) => status.setText(text))); }; private readonly setupCommands = () => @@ -299,6 +491,8 @@ export default class Observability { root = { executionProgress: this.executionProgress, exportLogs: this.exportLogs, + getModuleStatus: this.getModuleStatus, + setModuleStatus: this.setModuleStatus, syncStage: this.syncStage, walkProgress: this.walkProgress, }; diff --git a/packages/plugin/src/modules/Scheduler.ts b/packages/plugin/src/modules/Scheduler.ts index a5c2a7f9..7aa7df80 100644 --- a/packages/plugin/src/modules/Scheduler.ts +++ b/packages/plugin/src/modules/Scheduler.ts @@ -1,5 +1,5 @@ -import { NONE_STRATEGY, prepareGlobMatch } from '@/utils/glob-match'; import untilTrue from '@/utils/until-true'; +import { NONE_STRATEGY, prepareGlobMatch } from '@repo/shared/glob-match'; import type { Dispatch } from './EventBus'; import type { SyncStage } from './Observability'; import type { SyncOptions, SyncTerminateReason } from './Sync'; diff --git a/packages/plugin/src/modules/Storage.ts b/packages/plugin/src/modules/Storage.ts index 5e659865..fb7510c0 100644 --- a/packages/plugin/src/modules/Storage.ts +++ b/packages/plugin/src/modules/Storage.ts @@ -6,8 +6,6 @@ import type { DatabaseAsync, StoreAsync } from 'uni-kv'; export type IndexedDBSchema = Record; export type RecordStore = StoreAsync; -export const SYNC_STATE_STORE_NAME = 'sync-state'; - export default class Storage { private readonly memoryDB; private readonly indexedDB; diff --git a/packages/plugin/src/modules/Sync.ts b/packages/plugin/src/modules/Sync.ts index bda930b4..8b06645f 100644 --- a/packages/plugin/src/modules/Sync.ts +++ b/packages/plugin/src/modules/Sync.ts @@ -5,12 +5,13 @@ import { AddRecord, RemoveRecord, convertMoves, + executeTasks, + getTaskRegion, syncCancelledError, taskMap, } from '@/sync'; -import { NONE_STRATEGY, prepareGlobMatch } from '@/utils/glob-match'; -import { REGRET_ERROR_MESSAGE } from '@/utils/pipe'; import { toError } from '@repo/shared/error'; +import { NONE_STRATEGY, prepareGlobMatch } from '@repo/shared/glob-match'; import { isSub } from '@repo/shared/path'; import { ref } from 'synthkernel'; import type { Dispatch, On } from './EventBus'; @@ -30,7 +31,7 @@ import type { StatsMap, TogglableValue, } from '@/types'; -import type { GlobMatchResult } from '@/utils/glob-match'; +import type { GlobMatchResult } from '@repo/shared/glob-match'; import type { Ref } from 'synthkernel'; export type SyncTerminateReason = @@ -75,7 +76,11 @@ export default class Sync { taskFailed: FailedTaskInfo; executionStarted: Array; }; - declare readonly settings: { maxFileSize: TogglableValue; syncStrategy: Array }; + declare readonly settings: { + maxFileSize: TogglableValue; + maxMemoryConsumption?: TogglableValue; + syncStrategy: Array; + }; declare readonly i18n: { xOperationsFailed: Snippet }; private readonly postProcess = (stats: Array) => { @@ -128,7 +133,6 @@ export default class Sync { this; const { on, dispatch, initializeSync, getConflictResolver, translate, decideTasks } = ctx; const isCancelled = ref(false); - let failedCount = 0; let tasks: Array; let terminateReason!: SyncTerminateReason; const cleanup = on('syncCanceled', () => isCancelled(true)); @@ -232,24 +236,17 @@ export default class Sync { if (isCancelled()) throw syncCancelledError; dispatch('executionStarted', tasks); - await Promise.all( - tasks.map(async (task) => { - const complete = () => dispatch('taskCompleted', toTaskInfo(task)); - try { - await task.exec(); - complete(); - } catch (error) { - if (isCancelled()) return; - const err = toError(error); - if (err.message === REGRET_ERROR_MESSAGE) { - complete(); - return; - } - failedCount++; - dispatch('taskFailed', { ...toTaskInfo(task), error: err }); - } - }), - ); + const userMemoryBudget = this.settings.maxMemoryConsumption?.enabled + ? this.settings.maxMemoryConsumption.value + : undefined; + const { failedCount } = await executeTasks(tasks, { + isCancelled, + maxMemory: userMemoryBudget, + onTaskComplete: (task) => dispatch('taskCompleted', toTaskInfo(task)), + onTaskFailed: (task, error) => { + dispatch('taskFailed', { ...toTaskInfo(task), error }); + }, + }); terminateReason = isCancelled() ? { result: 'cancelled' } @@ -325,15 +322,9 @@ function toTaskInfo({ key, name, local, remote }: Task): TaskInfo { } function sortTasks(tasks: Array) { - const region = (task: Task) => { - const isFolder = task.local?.isDir === true || task.remote?.isDir === true; - if (task.name === 'removeLocal' || task.name === 'removeRemote') return isFolder ? 3 : 0; - if (task.name === 'createLocalDir' || task.name === 'createRemoteDir') return 1; - return task.name === 'moveLocal' || task.name === 'moveRemote' ? 2 : 4; - }; tasks.sort((a, b) => { - const aRegion = region(a); - const bRegion = region(b); + const aRegion = getTaskRegion(a); + const bRegion = getTaskRegion(b); if (aRegion !== bRegion) return aRegion - bRegion; if (aRegion === 3) return b.key.length - a.key.length; if (aRegion === 1 || aRegion === 2) return a.key.length - b.key.length; diff --git a/packages/plugin/src/sdk/dev.ts b/packages/plugin/src/sdk/dev.ts index 66c1f5d7..366874bc 100644 --- a/packages/plugin/src/sdk/dev.ts +++ b/packages/plugin/src/sdk/dev.ts @@ -1,3 +1,3 @@ export { default as debugWrapper } from './debug-wrapper'; export { default as testKit } from '$/test-kit'; -export { default as sha256 } from '@/utils/sha-256'; +export { sha256 } from '@repo/shared/sha256-bytes'; diff --git a/packages/plugin/src/sdk/index.ts b/packages/plugin/src/sdk/index.ts index 8450d02a..eb153df2 100644 --- a/packages/plugin/src/sdk/index.ts +++ b/packages/plugin/src/sdk/index.ts @@ -79,6 +79,7 @@ export type { RecordStore } from '@/modules/Storage'; export type { ModuleMeta, AugmentedModuleMeta } from '@/modules/Extensibility'; export type { SyncTerminateReason, SyncOptions, RemoteLister } from '@/modules/Sync'; export type { ExistingMemoryDB } from '@/modules/Bootstrap'; +export type { ModuleStatus, StatusLevel } from '@/modules/Observability'; export type * from '@/fs/interface'; export type { VaultRequest } from '@/fs'; export type SelectFromContext = Context extends O ? O : never; diff --git a/packages/plugin/src/settings/controls.ts b/packages/plugin/src/settings/controls.ts index 1e1c17d4..3afd4ab0 100644 --- a/packages/plugin/src/settings/controls.ts +++ b/packages/plugin/src/settings/controls.ts @@ -74,19 +74,18 @@ export default function controlsSettings({ type: 'time', }), })), - // See bootstrap TODO - // 4000: s(() => ({ - // desc: translate('maxMemoryConsumptionDescription'), - // labels: [speedLabel()], - // name: translate('maxMemoryConsumption'), - // render: renderTogglableValue({ - // field: settings.maxMemoryConsumption, - // placeholder: translate('maxMemoryConsumptionPlaceholder'), - // rejectZero: true, - // saveSettings, - // type: 'fileSize', - // }), - // })), + 4000: s(() => ({ + desc: translate('maxMemoryConsumptionDescription'), + labels: [speedLabel()], + name: translate('maxMemoryConsumption'), + render: renderTogglableValue({ + field: settings.maxMemoryConsumption, + placeholder: translate('maxMemoryConsumptionPlaceholder'), + rejectZero: true, + saveSettings, + type: 'fileSize', + }), + })), }, ), }; diff --git a/packages/plugin/src/settings/head.ts b/packages/plugin/src/settings/head.ts index 2061bea9..116f2cc5 100644 --- a/packages/plugin/src/settings/head.ts +++ b/packages/plugin/src/settings/head.ts @@ -1,7 +1,7 @@ import ModuleManagement from './module-management'; import { generateEditableList, reactivelyValidate, settingEntry as s } from './utils'; -import { normalizeGlob } from '@/utils/glob-match'; import { describeError, toError } from '@repo/shared/error'; +import { normalizeGlob } from '@repo/shared/glob-match'; import { ExtraButtonComponent, Notice, PluginSettingTab, setTooltip } from 'obsidian'; import type { AugmentedSettingDefinitionItem, LabelDefinition } from './utils'; import type { Context, Events, Settings } from '@'; diff --git a/packages/plugin/src/sync/executor.ts b/packages/plugin/src/sync/executor.ts new file mode 100644 index 00000000..4bd6263f --- /dev/null +++ b/packages/plugin/src/sync/executor.ts @@ -0,0 +1,234 @@ +import { REGRET_ERROR_MESSAGE } from '@/utils/pipe'; +import { toError } from '@repo/shared/error'; +import { Platform } from 'obsidian'; +import type { Task } from './tasks/interface'; + +export const FAST_LANE_THRESHOLD = 64 * 1024; // 64 KiB + +export function getTaskRegion(task: Task): number { + const isFolder = task.local?.isDir === true || task.remote?.isDir === true; + if (task.name === 'removeLocal' || task.name === 'removeRemote') return isFolder ? 3 : 0; + if (task.name === 'createLocalDir' || task.name === 'createRemoteDir') return 1; + return task.name === 'moveLocal' || task.name === 'moveRemote' ? 2 : 4; +} + +export function getTaskTransferSize(task: Task): number { + if (task.name === 'upload') return task.local?.isDir ? 0 : (task.local?.size ?? 0); + if (task.name === 'download') return task.remote?.isDir ? 0 : (task.remote?.size ?? 0); + if (task.name === 'resolveConflict') { + const localSize = task.local?.isDir ? 0 : (task.local?.size ?? 0); + const remoteSize = task.remote?.isDir ? 0 : (task.remote?.size ?? 0); + return Math.max(localSize, remoteSize); + } + return 0; +} + +export function getDefaultMaxMemory(): number { + if (Platform.isMobile) { + const nav = typeof navigator === 'undefined' ? undefined : navigator; + const lowMem = + (nav && + 'deviceMemory' in nav && + typeof nav.deviceMemory === 'number' && + nav.deviceMemory <= 4) || + (nav && typeof nav.hardwareConcurrency === 'number' && nav.hardwareConcurrency <= 4); + return lowMem ? 8 * 1024 * 1024 : 16 * 1024 * 1024; + } + return 64 * 1024 * 1024; +} + +export type TaskExecutorOptions = { + isCancelled: () => boolean; + onTaskComplete: (task: Task) => void; + onTaskFailed: (task: Task, error: Error) => void; + maxMemory?: number; + maxConcurrency?: number; + maxFastLane?: number; +}; + +async function runTaskBatch( + batch: Array, + concurrency: number, + runTask: (task: Task) => Promise, + isCancelled: () => boolean, +): Promise { + if (batch.length === 0 || isCancelled()) return; + let index = 0; + const workers = Array.from({ length: Math.min(concurrency, batch.length) }, async () => { + while (index < batch.length && !isCancelled()) { + const task = batch[index++]; + await runTask(task); + } + }); + await Promise.all(workers); +} + +async function runTransferTasks( + tasks: Array, + options: TaskExecutorOptions, + runTask: (task: Task) => Promise, +): Promise { + if (tasks.length === 0 || options.isCancelled()) return; + + const maxMemory = options.maxMemory ?? getDefaultMaxMemory(); + const maxConcurrency = options.maxConcurrency ?? (Platform.isMobile ? 6 : 12); + const maxFastLane = options.maxFastLane ?? (Platform.isMobile ? 2 : 4); + + const fastQueue: Array = []; + const heavyQueue: Array = []; + + for (const task of tasks) { + const size = getTaskTransferSize(task); + if (size <= FAST_LANE_THRESHOLD) fastQueue.push(task); + else heavyQueue.push(task); + } + + let inFlightBytes = 0; + let activeCount = 0; + let activeFastLaneCount = 0; + + return new Promise((resolve) => { + let isSettled = false; + + const checkFinished = () => { + if (isSettled) return; + if (options.isCancelled() && activeCount === 0) { + isSettled = true; + resolve(); + return; + } + if (fastQueue.length === 0 && heavyQueue.length === 0 && activeCount === 0) { + isSettled = true; + resolve(); + } + }; + + const schedule = () => { + if (isSettled) return; + if (options.isCancelled()) { + checkFinished(); + return; + } + + while (!options.isCancelled() && activeCount < maxConcurrency) { + // Try scheduling a heavy task first if memory allows + if (heavyQueue.length > 0) { + const nextHeavy = heavyQueue[0]; + const heavySize = getTaskTransferSize(nextHeavy); + const canStartHeavy = + inFlightBytes === 0 || + inFlightBytes + heavySize <= maxMemory || + (inFlightBytes <= FAST_LANE_THRESHOLD * maxFastLane && + activeCount === activeFastLaneCount); + + if (canStartHeavy) { + heavyQueue.shift(); + startTask(nextHeavy, heavySize); + continue; + } + } + + // Try scheduling a fast-lane task + if (fastQueue.length > 0) { + const nextFast = fastQueue[0]; + const fastSize = getTaskTransferSize(nextFast); + const canStartFast = + inFlightBytes + fastSize <= maxMemory || activeFastLaneCount < maxFastLane; + + if (canStartFast) { + fastQueue.shift(); + startTask(nextFast, fastSize); + continue; + } + } + + break; + } + + checkFinished(); + }; + + const startTask = (task: Task, size: number) => { + activeCount++; + inFlightBytes += size; + const isFast = size <= FAST_LANE_THRESHOLD; + if (isFast) activeFastLaneCount++; + + void (async () => { + try { + await runTask(task); + } finally { + activeCount--; + inFlightBytes = Math.max(0, inFlightBytes - size); + if (isFast) activeFastLaneCount = Math.max(0, activeFastLaneCount - 1); + schedule(); + } + })(); + }; + + schedule(); + }); +} + +export async function executeTasks( + tasks: Array, + options: TaskExecutorOptions, +): Promise<{ failedCount: number }> { + let failedCount = 0; + const { isCancelled, onTaskComplete, onTaskFailed } = options; + + const runTask = async (task: Task) => { + if (isCancelled()) return; + const complete = () => onTaskComplete(task); + try { + await task.exec(); + complete(); + } catch (error) { + if (isCancelled()) return; + const err = toError(error); + if (err.message === REGRET_ERROR_MESSAGE) { + complete(); + return; + } + failedCount++; + onTaskFailed(task, err); + } + }; + + const region0Tasks: Array = []; // File removals + const region1Tasks: Array = []; // Dir creations (short to long) + const region2Tasks: Array = []; // Moves (short to long) + const region3Tasks: Array = []; // Dir removals (long to short) + const region4Tasks: Array = []; // Transfers & records + + for (const task of tasks) { + const r = getTaskRegion(task); + if (r === 0) region0Tasks.push(task); + else if (r === 1) region1Tasks.push(task); + else if (r === 2) region2Tasks.push(task); + else if (r === 3) region3Tasks.push(task); + else region4Tasks.push(task); + } + + // Phase 1: Structural & Metadata Operations + // 1.1 Remove files (bounded concurrency) + await runTaskBatch(region0Tasks, Platform.isMobile ? 4 : 8, runTask, isCancelled); + if (isCancelled()) return { failedCount }; + + // 1.2 Create directories (sequential to guarantee parent before child) + await runTaskBatch(region1Tasks, 1, runTask, isCancelled); + if (isCancelled()) return { failedCount }; + + // 1.3 Moves (sequential) + await runTaskBatch(region2Tasks, 1, runTask, isCancelled); + if (isCancelled()) return { failedCount }; + + // 1.4 Remove empty directories (sequential to guarantee child before parent) + await runTaskBatch(region3Tasks, 1, runTask, isCancelled); + if (isCancelled()) return { failedCount }; + + // Phase 2: Data Transfers & Records (Weighted Byte-Budget & Fast-Lane Pool) + await runTransferTasks(region4Tasks, options, runTask); + + return { failedCount }; +} diff --git a/packages/plugin/src/sync/index.ts b/packages/plugin/src/sync/index.ts index 45355452..a1792995 100644 --- a/packages/plugin/src/sync/index.ts +++ b/packages/plugin/src/sync/index.ts @@ -28,4 +28,12 @@ export { default as keepRemoteResolver } from './conflict-resolve/keep-remote'; export { default as keepLocalResolver } from './conflict-resolve/keep-local'; export { default as latestSurviveResolver } from './conflict-resolve/latest-survive'; export { default as renameAndKeepBothResolver } from './conflict-resolve/rename-and-keep-both'; +export { + executeTasks, + getTaskRegion, + getTaskTransferSize, + FAST_LANE_THRESHOLD, + getDefaultMaxMemory, +} from './executor'; +export type { TaskExecutorOptions } from './executor'; export const syncCancelledError = new Error('Sync cancelled by user.'); diff --git a/packages/plugin/src/utils/pipe.ts b/packages/plugin/src/utils/pipe.ts index 1e76edea..1e5a6e40 100644 --- a/packages/plugin/src/utils/pipe.ts +++ b/packages/plugin/src/utils/pipe.ts @@ -1,9 +1,8 @@ import { STREAM_RESERVATION_SIZE } from '@/fs/wrappers/memory-control'; -import { getStatus } from '@repo/shared/error'; +import { isNotFound } from '@repo/shared/error'; import { Platform } from 'obsidian'; import type { Fs } from '@/fs'; import type { Binary, FileStat, MaybePromise } from '@/types'; -import type { ErrorLike } from '@repo/shared/error'; export const REGRET_ERROR_MESSAGE = '=== Sync Engine Operation Regretted ==='; export const REGRET_ERROR = new Error(REGRET_ERROR_MESSAGE); @@ -34,7 +33,7 @@ export async function readWithSize(fs: Fs, key: string, stat: FileStat) { if (stat.size >= streamThreshold) return await fs.readStream(key, stat); return await fs.read(key, stat); } catch (error) { - if (isNonExistent(error)) throw REGRET_ERROR; + if (isNotFound(error)) throw REGRET_ERROR; throw error; } } @@ -49,11 +48,3 @@ export function writeWithSize( return fs.writeStream(key, value as MaybePromise>, stat); return fs.write(key, value as MaybePromise, stat); } - -// Swallow TOCTOU -function isNonExistent(error: unknown) { - return ( - getStatus(error) === 404 || - (((error as ErrorLike).message as string) ?? String(error)).toUpperCase().includes('ENOENT') - ); -} diff --git a/packages/plugin/src/utils/sha-256.ts b/packages/plugin/src/utils/sha-256.ts deleted file mode 100644 index 0a6f1721..00000000 --- a/packages/plugin/src/utils/sha-256.ts +++ /dev/null @@ -1,12 +0,0 @@ -const encoder = new TextEncoder(); - -export default async function sha256(input: string): Promise { - const data = encoder.encode(input); - const hashBuffer = await crypto.subtle.digest('SHA-256', data); - const hex = '0123456789abcdef'; - const lookup = Array.from({ length: 256 }, (_, i) => hex[i >> 4] + hex[i & 0xf]); - const view = new DataView(hashBuffer); - let output = ''; - for (let i = 0; i < 32; i++) output += lookup[view.getUint8(i)]; - return output; -} diff --git a/packages/plugin/test/observability.test.ts b/packages/plugin/test/observability.test.ts new file mode 100644 index 00000000..74424dfb --- /dev/null +++ b/packages/plugin/test/observability.test.ts @@ -0,0 +1,321 @@ +import Observability from '@/modules/Observability'; +import { expect, test } from 'bun:test'; +import { Platform } from 'obsidian'; +import { ref } from 'synthkernel'; + +type MockElement = { + addEventListener: ( + event: string, + handler: (e: { stopPropagation: () => void }) => void, + ) => void; + addClass: (...cls: Array) => void; + append: (...nodes: Array) => void; + children: Array; + closest: (sel: string) => MockElement | undefined; + createEl: (tag: string, options?: { cls?: string; text?: string }) => MockElement; + createSpan: (options?: { cls?: string; text?: string }) => MockElement; + empty: () => void; + firstElementChild: MockElement | undefined; + getAttribute: (name: string) => string | undefined; + hasClass: (cls: string) => boolean; + parent?: MockElement; + prepend: (...nodes: Array) => void; + querySelector: (sel: string) => MockElement | undefined; + querySelectorAll: (sel: string) => Array; + remove: () => void; + removeAttribute: (name: string) => void; + removeClass: (...cls: Array) => void; + setAttr: (name: string, val: string) => void; + setAttribute: (name: string, val: string) => void; + setText: (text: string) => void; + style: Record; + tag: string; + trigger: (event: string, e?: unknown) => void; +}; + +function fakeElement(tag = 'div'): MockElement { + const children: Array = []; + const attrs = new Map(); + const classes = new Set(); + const style: Record = {}; + const listeners = new Map void }) => void>>(); + + const el: MockElement = { + addClass: (...cls) => cls.forEach((c) => classes.add(c)), + addEventListener: (event, handler) => { + const list = listeners.get(event) ?? []; + list.push(handler); + listeners.set(event, list); + }, + append: (...nodes) => { + for (const node of nodes as Array) { + node.parent = el; + children.push(node); + } + }, + children, + closest: (sel) => { + if (sel.startsWith('.')) { + const cls = sel.slice(1); + let cur: MockElement | undefined = el; + while (cur) { + if (cur.hasClass(cls)) return cur; + cur = cur.parent; + } + } + return; + }, + createEl: (t, options) => { + const child = fakeElement(t); + if (options?.cls) child.addClass(...options.cls.split(' ').filter(Boolean)); + if (options?.text) child.setText(options.text); + child.parent = el; + children.push(child); + return child; + }, + createSpan: (options) => { + const span = fakeElement('span'); + if (options?.cls) span.addClass(...options.cls.split(' ').filter(Boolean)); + if (options?.text) span.setText(options.text); + span.parent = el; + children.push(span); + return span; + }, + empty: () => { + children.length = 0; + }, + get firstElementChild() { + return children[0]; + }, + getAttribute: (name) => attrs.get(name) ?? undefined, + hasClass: (cls) => classes.has(cls), + prepend: (...nodes) => { + for (const node of (nodes as Array).reverse()) { + node.parent = el; + children.unshift(node); + } + }, + querySelector: (sel) => { + if (sel.startsWith('.')) { + const cls = sel.slice(1); + const find = (node: MockElement): MockElement | undefined => { + if (node.hasClass(cls)) return node; + for (const child of node.children) { + const res = find(child); + if (res) return res; + } + return undefined; + }; + for (const child of el.children) { + const res = find(child); + if (res) return res; + } + } + return; + }, + querySelectorAll: (sel) => { + const result: Array = []; + if (sel.startsWith('.')) { + const cls = sel.slice(1); + const find = (node: MockElement) => { + if (node.hasClass(cls)) result.push(node); + for (const child of node.children) find(child); + }; + for (const child of el.children) find(child); + } + return result; + }, + remove: () => { + if (el.parent) { + const idx = el.parent.children.indexOf(el); + if (idx !== -1) el.parent.children.splice(idx, 1); + } + }, + removeAttribute: (name) => { + attrs.delete(name); + }, + removeClass: (...cls) => cls.forEach((c) => classes.delete(c)), + setAttr: (name, val) => { + attrs.set(name, val); + }, + setAttribute: (name, val) => { + attrs.set(name, val); + }, + setText: () => {}, + style, + tag, + trigger: (event, e = { stopPropagation: () => {} }) => { + const list = listeners.get(event) ?? []; + for (const handler of list) handler(e as { stopPropagation: () => void }); + }, + }; + return el; +} + +function createFakeObservability(options?: { + isMobile?: boolean; + rootLeaves?: Array<{ view?: { containerEl?: MockElement } }>; +}) { + const callbacks = new Map) => void>>(); + const workspaceCallbacks = new Map void>>(); + const isIdle = ref(true); + const startIconEl = fakeElement('div'); + const statusEl = fakeElement('div'); + let syncRequested = false; + let progressShown = false; + + const rootLeaves = options?.rootLeaves ?? []; + + const ctx = { + addCommand: () => ({}) as never, + addRibbonIcon: () => startIconEl as unknown as HTMLElement, + addStatusBarItem: () => statusEl as unknown as HTMLElement, + app: { + workspace: { + iterateRootLeaves: (cb: (leaf: unknown) => void) => { + rootLeaves.forEach(cb); + }, + offref: () => {}, + on: (event: string, cb: () => void) => { + const list = workspaceCallbacks.get(event) ?? []; + list.push(cb); + workspaceCallbacks.set(event, list); + return {} as never; + }, + }, + } as never, + dispatch: () => {}, + getLogs: () => '', + isIdle, + on: (event: string, cb: (...args: Array) => void) => { + const list = callbacks.get(event) ?? []; + list.push(cb); + callbacks.set(event, list); + return () => { + const current = callbacks.get(event) ?? []; + callbacks.set( + event, + current.filter((item) => item !== cb), + ); + }; + }, + requestSync: () => { + syncRequested = true; + return Promise.resolve({ result: 'completed' as const }); + }, + showProgress: () => { + progressShown = true; + }, + translate: (key: string) => key, + }; + + if (options?.isMobile !== undefined) Platform.isMobile = options.isMobile; + + const obs = new Observability(ctx as never); + (obs as unknown as { settings: object }).settings = { + exportLogsDirectory: '', + noticeStatusOnMobile: false, + }; + obs.start(); + return { + callbacks, + isIdle, + isProgressShown: () => progressShown, + isSyncRequested: () => syncRequested, + obs, + startIconEl, + statusEl, + workspaceCallbacks, + }; +} + +test('setModuleStatus manages module statuses and updates badge', () => { + Platform.isMobile = false; + const { obs, startIconEl } = createFakeObservability(); + expect(obs.getModuleStatus('collaboration')).toBeUndefined(); + + const unregister = obs.setModuleStatus('collaboration', { + level: 'warning', + message: 'MQTT: Reconnecting', + }); + expect(obs.getModuleStatus('collaboration')?.level).toBe('warning'); + expect(startIconEl.querySelector('.sync-engine-badge-icon')).toBeDefined(); + expect(startIconEl.getAttribute('aria-label')).toContain('MQTT: Reconnecting'); + + obs.setModuleStatus('collaboration', { level: 'error', message: 'MQTT: Connection failed' }); + expect(obs.getModuleStatus('collaboration')?.level).toBe('error'); + expect(startIconEl.getAttribute('aria-label')).toContain('MQTT: Connection failed'); + + unregister(); + expect(obs.getModuleStatus('collaboration')).toBeUndefined(); + expect(startIconEl.querySelector('.sync-engine-badge-icon')).toBeUndefined(); +}); + +test('sync failure produces error badge on the sync icon', () => { + Platform.isMobile = false; + const { callbacks, startIconEl } = createFakeObservability(); + const terminatedHandlers = callbacks.get('syncTerminated') ?? []; + for (const handler of terminatedHandlers) + handler({ error: new Error('Network error'), result: 'failed' }); + + expect(startIconEl.querySelector('.sync-engine-badge-icon')).toBeDefined(); + expect(startIconEl.getAttribute('aria-label')).toContain('Network error'); + + const startedHandlers = callbacks.get('syncStarted') ?? []; + for (const handler of startedHandlers) handler(); + + expect(startIconEl.querySelector('.sync-engine-badge-icon')).toBeUndefined(); +}); + +test('mobile header icon renders in view-actions and responds to status changes and clicks', () => { + const leafContainer = fakeElement('div'); + const viewActions = leafContainer.createEl('div', { cls: 'view-actions' }); + viewActions.createEl('button', { cls: 'sync-engine-collab-action' }); + + const fake = createFakeObservability({ + isMobile: true, + rootLeaves: [{ view: { containerEl: leafContainer } }], + }); + + // Legacy button removed and new action prepended + expect(viewActions.querySelector('.sync-engine-collab-action')).toBeUndefined(); + const mobileBtn = viewActions.querySelector('.sync-engine-mobile-action'); + expect(mobileBtn).toBeDefined(); + expect(mobileBtn?.querySelector('.sync-engine-main-icon')).toBeDefined(); + expect(mobileBtn?.querySelector('.sync-engine-badge-icon')).toBeUndefined(); + + // Set module warning + fake.obs.setModuleStatus('collaboration', { level: 'warning', message: 'MQTT: Reconnecting' }); + expect(mobileBtn?.querySelector('.sync-engine-badge-icon')).toBeDefined(); + expect(mobileBtn?.getAttribute('aria-label')).toContain('MQTT: Reconnecting'); + + // Set module error + fake.obs.setModuleStatus('collaboration', { + level: 'error', + message: 'MQTT: Connection failed', + }); + expect(mobileBtn?.querySelector('.sync-engine-badge-icon')).toBeDefined(); + expect(mobileBtn?.getAttribute('aria-label')).toContain('MQTT: Connection failed'); + + // Click while idle triggers manual sync + mobileBtn?.trigger('click'); + expect(fake.isSyncRequested()).toBe(true); + + // When syncing, spinning class is present + fake.isIdle(false); + const mainIcon = mobileBtn?.querySelector('.sync-engine-main-icon'); + expect(mainIcon?.hasClass('animate-spin')).toBe(true); + + // Click while syncing shows progress modal + mobileBtn?.trigger('click'); + expect(fake.isProgressShown()).toBe(true); + + // Switching leaves via active-leaf-change updates actions + const onLeafChangeHandlers = fake.workspaceCallbacks.get('active-leaf-change') ?? []; + for (const handler of onLeafChangeHandlers) handler(); + expect(viewActions.querySelector('.sync-engine-mobile-action')).toBeDefined(); + + fake.obs.dispose(); + expect(viewActions.querySelector('.sync-engine-mobile-action')).toBeUndefined(); + Platform.isMobile = false; +}); diff --git a/packages/plugin/test/retry-middleware.test.ts b/packages/plugin/test/retry-middleware.test.ts index 5e1c933c..17cd418e 100644 --- a/packages/plugin/test/retry-middleware.test.ts +++ b/packages/plugin/test/retry-middleware.test.ts @@ -124,3 +124,32 @@ test('retry middleware returns retryable status response after exhausting retrie expect(harness.calls).toHaveLength(3); expect(sleepSpy).toHaveBeenCalledTimes(2); }); + +test('retry middleware retries mobile OkHttp stream closed and connection errors', async () => { + const errors = [ + 'Request failed: java.io.IOException: stream closed', + 'Request failed: IOException: stream closed', + 'Request failed: java.io.IOException: unexpected end of stream', + 'Request failed: java.io.IOException: stream was reset: PROTOCOL_ERROR', + 'java.net.SocketException: Software caused connection abort', + 'java.net.SocketException: Broken pipe', + 'java.net.SocketException: Socket closed', + 'java.net.SocketTimeoutException: timeout', + ]; + + for (const msg of errors) { + sleepSpy.mockClear(); + let attempts = 0; + const harness = request(() => { + attempts += 1; + if (attempts < 2) throw new Error(msg); + return {}; + }); + const wrapped = retryMiddleware(harness.request, { maxRetry: 2, retryDelay: () => 25 }); + + const res = await wrapped('test.md'); + expect(res).toMatchObject({ status: 200 }); + expect(harness.calls).toHaveLength(2); + expect(sleepSpy).toHaveBeenCalledTimes(1); + } +}); diff --git a/packages/plugin/test/task-executor.test.ts b/packages/plugin/test/task-executor.test.ts new file mode 100644 index 00000000..c5a5fd70 --- /dev/null +++ b/packages/plugin/test/task-executor.test.ts @@ -0,0 +1,269 @@ +import { executeTasks, getTaskRegion, getTaskTransferSize } from '@/sync'; +import { REGRET_ERROR } from '@/utils/pipe'; +import { expect, test } from 'bun:test'; +import type { Task } from '@/sync'; + +function mockTask( + name: Task['name'], + key: string, + options: { isDir?: boolean; size?: number; exec?: () => Promise | void } = {}, +): Task { + const isDir = options.isDir ?? false; + const size = options.size ?? 0; + const stat = isDir + ? { isDir: true as const, key, meta: () => ({}) } + : { isDir: false as const, key, meta: () => ({}), mtime: 0, size, uid: 'uid' }; + + return { + exec: options.exec ?? (() => Promise.resolve()), + key, + local: stat, + localFs: {} as unknown as Task['localFs'], + name, + options: {} as unknown as Task['options'], + record: {} as unknown as Task['record'], + remote: stat, + remoteFs: {} as unknown as Task['remoteFs'], + } as unknown as Task; +} + +test('getTaskRegion identifies regions correctly', () => { + expect(getTaskRegion(mockTask('removeLocal', 'a.md'))).toBe(0); + expect(getTaskRegion(mockTask('removeRemote', 'a.md'))).toBe(0); + expect(getTaskRegion(mockTask('createLocalDir', 'a/', { isDir: true }))).toBe(1); + expect(getTaskRegion(mockTask('createRemoteDir', 'a/', { isDir: true }))).toBe(1); + expect(getTaskRegion(mockTask('moveLocal', 'a.md'))).toBe(2); + expect(getTaskRegion(mockTask('moveRemote', 'a.md'))).toBe(2); + expect(getTaskRegion(mockTask('removeLocal', 'a/', { isDir: true }))).toBe(3); + expect(getTaskRegion(mockTask('removeRemote', 'a/', { isDir: true }))).toBe(3); + expect(getTaskRegion(mockTask('upload', 'a.md'))).toBe(4); + expect(getTaskRegion(mockTask('download', 'a.md'))).toBe(4); + expect(getTaskRegion(mockTask('addRecord', 'a.md'))).toBe(4); +}); + +test('getTaskTransferSize returns sizes correctly', () => { + expect(getTaskTransferSize(mockTask('upload', 'a.md', { size: 1024 }))).toBe(1024); + expect(getTaskTransferSize(mockTask('download', 'a.md', { size: 2048 }))).toBe(2048); + expect(getTaskTransferSize(mockTask('createRemoteDir', 'a/', { isDir: true }))).toBe(0); + expect(getTaskTransferSize(mockTask('removeLocal', 'a.md', { size: 500 }))).toBe(0); +}); + +test('executeTasks respects phase order: Region 0 -> 1 -> 2 -> 3 -> 4', async () => { + const order: Array = []; + const completed: Array = []; + + const t0 = mockTask('removeLocal', 'old.md', { + exec: () => { + order.push('r0'); + }, + }); + const t1 = mockTask('createRemoteDir', 'folder/', { + exec: () => { + order.push('r1'); + }, + isDir: true, + }); + const t2 = mockTask('moveRemote', 'renamed.md', { + exec: () => { + order.push('r2'); + }, + }); + const t3 = mockTask('removeRemote', 'old-folder/', { + exec: () => { + order.push('r3'); + }, + isDir: true, + }); + const t4 = mockTask('upload', 'new.md', { + exec: () => { + order.push('r4'); + }, + size: 100, + }); + + const { failedCount } = await executeTasks([t4, t3, t2, t1, t0], { + isCancelled: () => false, + onTaskComplete: (t) => completed.push(t.key), + onTaskFailed: () => {}, + }); + + expect(failedCount).toBe(0); + expect(order).toEqual(['r0', 'r1', 'r2', 'r3', 'r4']); + expect(completed).toEqual(['old.md', 'folder/', 'renamed.md', 'old-folder/', 'new.md']); +}); + +test('executeTasks enforces sequential order for directory creations and deletions', async () => { + const dirCreations: Array = []; + const dirDeletions: Array = []; + + const tParentDir = mockTask('createRemoteDir', 'parent/', { + exec: async () => { + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); + dirCreations.push('parent/'); + }, + isDir: true, + }); + const tChildDir = mockTask('createRemoteDir', 'parent/child/', { + exec: () => { + dirCreations.push('parent/child/'); + }, + isDir: true, + }); + + const tChildDel = mockTask('removeRemote', 'parent/child/', { + exec: async () => { + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); + dirDeletions.push('parent/child/'); + }, + isDir: true, + }); + const tParentDel = mockTask('removeRemote', 'parent/', { + exec: () => { + dirDeletions.push('parent/'); + }, + isDir: true, + }); + + await executeTasks([tParentDir, tChildDir, tChildDel, tParentDel], { + isCancelled: () => false, + onTaskComplete: () => {}, + onTaskFailed: () => {}, + }); + + expect(dirCreations).toEqual(['parent/', 'parent/child/']); + expect(dirDeletions).toEqual(['parent/child/', 'parent/']); +}); + +test('executeTasks serializes heavy tasks to prevent memory explosion', async () => { + let activeHeavyCount = 0; + let maxConcurrentHeavy = 0; + + const createHeavy = (key: string) => + mockTask('upload', key, { + exec: async () => { + activeHeavyCount++; + maxConcurrentHeavy = Math.max(maxConcurrentHeavy, activeHeavyCount); + await new Promise((resolve) => { + setTimeout(resolve, 20); + }); + activeHeavyCount--; + }, + size: 10 * 1024 * 1024, // 10 MiB + }); + + const tasks = [createHeavy('1.pdf'), createHeavy('2.pdf'), createHeavy('3.pdf')]; + + const { failedCount } = await executeTasks(tasks, { + isCancelled: () => false, + maxMemory: 12 * 1024 * 1024, // Budget 12 MiB -> only 1 heavy task (10 MiB) can fit at a time! + onTaskComplete: () => {}, + onTaskFailed: () => {}, + }); + + expect(failedCount).toBe(0); + expect(maxConcurrentHeavy).toBe(1); +}); + +test('executeTasks enables fast-lane for small files even when budget is occupied', async () => { + let heavyActive = false; + const smallCompletedWhileHeavyRunning: Array = []; + + const heavy = mockTask('upload', 'huge.iso', { + exec: async () => { + heavyActive = true; + await new Promise((resolve) => { + setTimeout(resolve, 50); + }); + heavyActive = false; + }, + size: 20 * 1024 * 1024, // 20 MiB + }); + + const small1 = mockTask('upload', 'quick1.md', { + exec: () => { + if (heavyActive) smallCompletedWhileHeavyRunning.push('quick1.md'); + }, + size: 1024, // 1 KiB + }); + + const small2 = mockTask('upload', 'quick2.md', { + exec: () => { + if (heavyActive) smallCompletedWhileHeavyRunning.push('quick2.md'); + }, + size: 2048, // 2 KiB + }); + + await executeTasks([heavy, small1, small2], { + isCancelled: () => false, + maxFastLane: 2, + maxMemory: 16 * 1024 * 1024, // 16 MiB budget, occupied by 20 MiB heavy task + onTaskComplete: () => {}, + onTaskFailed: () => {}, + }); + + expect(smallCompletedWhileHeavyRunning.length).toBe(2); +}); + +test('executeTasks handles failed tasks and regret errors gracefully', async () => { + const failedKeys: Array = []; + const completedKeys: Array = []; + + const tFail = mockTask('upload', 'fail.md', { + exec: () => { + throw new Error('Network error'); + }, + size: 100, + }); + + const tRegret = mockTask('upload', 'regret.md', { + exec: () => { + throw REGRET_ERROR; + }, + size: 100, + }); + + const tSuccess = mockTask('upload', 'ok.md', { exec: () => {}, size: 100 }); + + const { failedCount } = await executeTasks([tFail, tRegret, tSuccess], { + isCancelled: () => false, + onTaskComplete: (t) => completedKeys.push(t.key), + onTaskFailed: (t) => failedKeys.push(t.key), + }); + + expect(failedCount).toBe(1); + expect(failedKeys).toEqual(['fail.md']); + expect(completedKeys).toEqual(['regret.md', 'ok.md']); +}); + +test('executeTasks stops scheduling when cancelled', async () => { + let cancelled = false; + const executed: Array = []; + + const t1 = mockTask('upload', '1.md', { + exec: () => { + executed.push('1.md'); + cancelled = true; // Trigger cancel during first task + }, + size: 100, + }); + + const t2 = mockTask('upload', '2.md', { + exec: () => { + executed.push('2.md'); + }, + size: 100, + }); + + await executeTasks([t1, t2], { + isCancelled: () => cancelled, + maxConcurrency: 1, + onTaskComplete: () => {}, + onTaskFailed: () => {}, + }); + + expect(executed).toEqual(['1.md']); +}); diff --git a/packages/s3/src/s3/sigv4.ts b/packages/s3/src/s3/sigv4.ts index 426d0c06..62b1a6cd 100644 --- a/packages/s3/src/s3/sigv4.ts +++ b/packages/s3/src/s3/sigv4.ts @@ -1,5 +1,6 @@ import { textToUint8Array } from '@repo/shared/binary'; import { compileQuery } from '@repo/shared/path'; +import { sha256Text, toHex } from '@repo/shared/sha256-bytes'; import { md5 } from 'hash-wasm'; import type { S3DB } from '..'; import type { Binary, Request, RequestParam, StringKV } from '@hesprs/sync-engine-sdk'; @@ -16,18 +17,6 @@ export type SigV4Options = { const encoder = new TextEncoder(); -function toHex(bytes: Binary): string { - return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join(''); -} - -async function sha256Hex(data: Binary): Promise { - const digest = await crypto.subtle.digest( - 'SHA-256', - data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength), - ); - return toHex(new Uint8Array(digest)); -} - async function hmac(key: Binary, message: string): Promise { const keyData = key.buffer.slice(key.byteOffset, key.byteOffset + key.byteLength); const cryptoKey = await crypto.subtle.importKey( @@ -84,7 +73,7 @@ function buildCanonicalHeaders(headers: StringKV): { return { canonicalHeaders: canonicalHeadersStr, signedHeaders }; } -export async function signRequest( +async function signRequest( { method, url, @@ -122,7 +111,7 @@ export async function signRequest( 'AWS4-HMAC-SHA256', amzDate, credentialScope, - await sha256Hex(encoder.encode(canonicalRequest)), + await sha256Text(canonicalRequest), ].join('\n'); let kSigning: Binary; diff --git a/packages/shared/package.json b/packages/shared/package.json index 7efd055a..12cfb33e 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -13,6 +13,6 @@ "scripts": { "fix": "oxlint --silent --fix && oxfmt", "check": "oxlint && oxfmt --check", - "tests": "bun test ./test" + "tests": "bun --preload=./test/mocks test" } } diff --git a/packages/shared/src/binary.ts b/packages/shared/src/binary.ts index b364f74a..2c2112ce 100644 --- a/packages/shared/src/binary.ts +++ b/packages/shared/src/binary.ts @@ -2,6 +2,7 @@ const encoder = new TextEncoder(); const decoder = new TextDecoder(); export type Binary = Uint8Array; +export { toHex } from './sha256-bytes'; export function toUint8Array(buffer: ArrayBuffer) { return new Uint8Array(buffer); @@ -97,19 +98,29 @@ export class PartBuffer { } } -export function encodeBase64(bytes: Binary): string { - return btoa(Array.from(bytes, (byte) => String.fromCodePoint(byte)).join('')); +export function encodeBase64(value: BufferSource): string { + const bytes = + value instanceof ArrayBuffer + ? new Uint8Array(value) + : new Uint8Array(value.buffer, value.byteOffset, value.byteLength); + let binary = ''; + const chunkSize = 0x80_00; + for (let offset = 0; offset < bytes.length; offset += chunkSize) + binary += String.fromCodePoint(...bytes.subarray(offset, offset + chunkSize)); + return btoa(binary); } export function decodeBase64(value: string): Binary { - return Uint8Array.from(atob(value), (char) => char.codePointAt(0) as number); + const binary = atob(value); + return Uint8Array.from(binary, (character) => character.codePointAt(0) as number); } -export function encodeBase64Url(value: Binary): string { +export function encodeBase64Url(value: BufferSource): string { return encodeBase64(value).replaceAll('+', '-').replaceAll('/', '_').replaceAll('=', ''); } export function decodeBase64Url(value: string): Binary { + if (!/^[A-Za-z0-9_-]*$/u.test(value)) throw new Error('Invalid base64url value'); const padding = value.length % 4; const normalized = value.replaceAll('-', '+').replaceAll('_', '/') + diff --git a/packages/shared/src/cdc.ts b/packages/shared/src/cdc.ts new file mode 100644 index 00000000..0305439f --- /dev/null +++ b/packages/shared/src/cdc.ts @@ -0,0 +1,178 @@ +import { textToUint8Array } from './binary'; +import { fastCdc } from './fastcdc'; +import { sha256Bytes } from './sha256-bytes'; +import type { FastCdcOptions } from './fastcdc'; + +export { BIG_BLOB_LIMIT, fastCdc, SMALL_FILE_LIMIT } from './fastcdc'; +export type { ChunkSpan, FastCdcOptions } from './fastcdc'; + +const textDecoder = new TextDecoder(); + +export type ChunkItem = { bytes: Uint8Array; hash: string }; + +export type ChunkDiff = { chunkHashes: Array; newChunks: Map }; + +/** + * Returns adaptive FastCDC parameters according to input byte length. Keeps chunk count and + * manifest size bounded even on multi-megabyte files. + */ +export function adaptiveCdcOptions(byteLength: number): FastCdcOptions { + if (byteLength <= 1024 * 1024) return { avg: 1024, max: 4096, min: 512 }; + if (byteLength <= 8 * 1024 * 1024) return { avg: 4096, max: 16_384, min: 2048 }; + return { avg: 16_384, max: 65_536, min: 4096 }; +} + +/** + * Splits arbitrary byte buffer into content-defined chunks and computes SHA-256 hash for each chunk + * in parallel. + */ +export async function chunkContent( + data: Uint8Array, + options?: FastCdcOptions, +): Promise> { + const spans = fastCdc(data, options); + return Promise.all( + spans.map(async (span) => { + const bytes = data.subarray(span.offset, span.offset + span.length); + return { bytes, hash: await sha256Bytes(bytes) }; + }), + ); +} + +/** Splits UTF-8 text into content-defined chunks. */ +export function chunkText(text: string, options?: FastCdcOptions): Promise> { + return chunkContent(textToUint8Array(text), options); +} + +/** + * Concatenates ordered chunk byte arrays into a single Uint8Array. Safe across arbitrary chunk + * boundaries (including multi-byte UTF-8 split points). + */ +export function assembleChunks(chunks: Array): Uint8Array { + let total = 0; + for (const chunk of chunks) total += chunk.byteLength; + const joined = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + joined.set(chunk, offset); + offset += chunk.byteLength; + } + return joined; +} + +/** Concatenates ordered chunk byte arrays and decodes them into UTF-8 text. */ +export function assembleChunksToText(chunks: Array): string { + return textDecoder.decode(assembleChunks(chunks)); +} + +/** + * Compares chunk list against known hashes to produce a full hash manifest and a map of only + * unseen/new chunk payloads. + */ +export function computeChunkDiff( + chunks: Array, + isKnown: (hash: string) => boolean, +): ChunkDiff { + const chunkHashes: Array = []; + const newChunks = new Map(); + for (const { bytes, hash } of chunks) { + chunkHashes.push(hash); + if (!isKnown(hash) && !newChunks.has(hash)) newChunks.set(hash, bytes); + } + return { chunkHashes, newChunks }; +} + +/** + * Reassembles raw bytes from a chunk hash list using a chunk resolver. Returns undefined if any + * chunk hash cannot be resolved (missing chunk). + */ +function reassembleFromChunks( + chunkHashes: Array, + getChunk: (hash: string) => Uint8Array | undefined, +): Uint8Array | undefined { + const pieces: Array = []; + for (const hash of chunkHashes) { + const chunk = getChunk(hash); + if (chunk === undefined) return undefined; + pieces.push(chunk); + } + return assembleChunks(pieces); +} + +/** + * Reassembles UTF-8 text from a chunk hash list using a chunk resolver. Returns undefined if any + * chunk hash cannot be resolved. + */ +export function reassembleTextFromChunks( + chunkHashes: Array, + getChunk: (hash: string) => Uint8Array | undefined, +): string | undefined { + const bytes = reassembleFromChunks(chunkHashes, getChunk); + return bytes ? textDecoder.decode(bytes) : undefined; +} + +/** + * Lightweight, memory-bounded in-memory LRU chunk cache. Designed for mobile platforms to prevent + * unbounded memory growth while keeping recent chunks immediately available for incremental diff + * and reassembly. + */ +export class LRUChunkCache { + private readonly cache = new Map(); + private currentBytes = 0; + + constructor( + private readonly maxEntries = 4096, + private readonly maxBytes = 16 * 1024 * 1024, + ) {} + + get(hash: string): Uint8Array | undefined { + const chunk = this.cache.get(hash); + if (chunk !== undefined) { + this.cache.delete(hash); + this.cache.set(hash, chunk); + } + return chunk; + } + + has(hash: string): boolean { + return this.cache.has(hash); + } + + set(hash: string, bytes: Uint8Array): void { + if (this.cache.has(hash)) this.cache.delete(hash); + else this.currentBytes += bytes.byteLength; + + this.cache.set(hash, bytes); + this.evict(); + } + + putAll(chunks: Iterable<{ bytes: Uint8Array; hash: string }>): void { + for (const { bytes, hash } of chunks) this.set(hash, bytes); + } + + clear(): void { + this.cache.clear(); + this.currentBytes = 0; + } + + get size(): number { + return this.cache.size; + } + + get byteLength(): number { + return this.currentBytes; + } + + private evict(): void { + while ( + (this.cache.size > this.maxEntries || this.currentBytes > this.maxBytes) && + this.cache.size > 0 + ) { + const oldestKey = this.cache.keys().next().value; + if (oldestKey === undefined) break; + const chunk = this.cache.get(oldestKey); + if (chunk !== undefined) this.currentBytes -= chunk.byteLength; + this.cache.delete(oldestKey); + } + } +} diff --git a/packages/shared/src/device-id.ts b/packages/shared/src/device-id.ts new file mode 100644 index 00000000..8de266a3 --- /dev/null +++ b/packages/shared/src/device-id.ts @@ -0,0 +1,41 @@ +import type { App } from 'obsidian'; + +export const DEVICE_ID_STORAGE_KEY = 'sync-engine-history-device-id'; + +/** + * Retrieves or initializes a local, per-device unique identifier. Stored in vault-specific + * localStorage or browser localStorage to prevent device ID syncing and collisions across synced + * data.json configurations. + */ +export function getOrCreateDeviceId(app: App, legacyDeviceId?: string): string { + try { + const loaded = + typeof app.loadLocalStorage === 'function' + ? app.loadLocalStorage(DEVICE_ID_STORAGE_KEY) + : undefined; + const stored = + (loaded === null ? undefined : loaded) ?? + (typeof localStorage === 'undefined' + ? undefined + : localStorage.getItem(DEVICE_ID_STORAGE_KEY)); + if (typeof stored === 'string' && stored.length > 0) return stored; + } catch { + // Ignore storage errors (private mode, sandboxes, etc.) + } + + const id = + typeof legacyDeviceId === 'string' && legacyDeviceId.length > 0 + ? legacyDeviceId + : crypto.randomUUID(); + + try { + if (typeof app.saveLocalStorage === 'function') + app.saveLocalStorage(DEVICE_ID_STORAGE_KEY, id); + else if (typeof localStorage !== 'undefined') + localStorage.setItem(DEVICE_ID_STORAGE_KEY, id); + } catch { + // Ignore storage errors + } + + return id; +} diff --git a/packages/shared/src/device-type.ts b/packages/shared/src/device-type.ts new file mode 100644 index 00000000..d5df484c --- /dev/null +++ b/packages/shared/src/device-type.ts @@ -0,0 +1,17 @@ +import { Platform } from 'obsidian'; + +/** + * Device label derived from Obsidian's platform flags, using the same precedence as the plugin + * core's OS map. Values are already safe for an MQTT client ID or device identifier (lowercase, + * hyphens, no spaces). + */ +export default function deviceType(): string { + if (Platform.isTablet && Platform.isAndroidApp) return 'android-tablet'; + if (Platform.isTablet && Platform.isMacOS) return 'ipados'; + if (Platform.isAndroidApp) return 'android'; + if (Platform.isIosApp) return 'ios'; + if (Platform.isLinux) return 'linux'; + if (Platform.isMacOS) return 'macos'; + if (Platform.isWin) return 'windows'; + return 'desktop'; +} diff --git a/packages/shared/src/error.ts b/packages/shared/src/error.ts index 9e9064d5..0878b780 100644 --- a/packages/shared/src/error.ts +++ b/packages/shared/src/error.ts @@ -24,3 +24,13 @@ export function describeError(error: Error, msg: string, newObj?: boolean) { e.message = `${msg}: \`${error.message}\``; return e; } + +export function isNotFound(error: unknown): boolean { + const message = (((error as ErrorLike).message as string) ?? String(error)).toUpperCase(); + return ( + getStatus(error) === 404 || + message.includes('ENOENT') || + message.includes('NOSUCHKEY') || + message.includes('NOT FOUND') + ); +} diff --git a/packages/shared/src/fastcdc.ts b/packages/shared/src/fastcdc.ts new file mode 100644 index 00000000..eb95ddd0 --- /dev/null +++ b/packages/shared/src/fastcdc.ts @@ -0,0 +1,68 @@ +export const SMALL_FILE_LIMIT = 4096; +export const BIG_BLOB_LIMIT = 5 * 1024 * 1024; + +export type ChunkSpan = { offset: number; length: number }; +export type FastCdcOptions = { min?: number; avg?: number; max?: number }; + +// Deterministic 256-entry gear table. No external RNG so chunk boundaries are reproducible +// across devices, which is required for content-addressed deduplication. +const GEAR = (() => { + const table = new Uint32Array(256); + let state = 0x9e_37_79_b9; + for (let i = 0; i < 256; i++) { + state = (Math.imul(state, 1_664_525) + 1_013_904_223) >>> 0; + table[i] = state; + } + return table; +})(); + +function maskFor(avg: number, min: number, max: number): { small: number; large: number } { + // FastCDC uses two masks: a harder one until the minimum, then an easier one to bound the tail. + let bits = 0; + let value = 1; + while (value < avg) { + value *= 2; + bits++; + } + const easy = (1 << bits) - 1; + const hard = (1 << (bits + 2)) - 1; + // Clamp so the min/max guards remain meaningful for unusual options. + return { large: Math.min(easy, max - min), small: Math.min(hard, max - min) }; +} + +/** + * Content-defined chunking over a gear hash. A rolling 32-bit hash hits a normalization mask to + * pick boundaries, so a local edit only shifts one or two chunks and leaves the rest intact. + */ +export function fastCdc(bytes: Uint8Array, options: FastCdcOptions = {}): Array { + const min = options.min ?? 512; + const avg = options.avg ?? 1024; + const max = options.max ?? 4096; + const { small, large } = maskFor(avg, min, max); + const spans: Array = []; + if (bytes.byteLength === 0) return spans; + + let start = 0; + while (start < bytes.byteLength) { + const remaining = bytes.byteLength - start; + if (remaining <= min) { + spans.push({ length: remaining, offset: start }); + break; + } + const limit = Math.min(max, remaining); + let hash = 0; + let cut = limit; + for (let i = 0; i < limit; i++) { + hash = ((hash << 1) + GEAR[bytes[start + i]]) >>> 0; + if (i + 1 < min) continue; + const mask = i + 1 < avg ? small : large; + if ((hash & mask) === 0) { + cut = i + 1; + break; + } + } + spans.push({ length: cut, offset: start }); + start += cut; + } + return spans; +} diff --git a/packages/plugin/src/utils/glob-match.ts b/packages/shared/src/glob-match.ts similarity index 98% rename from packages/plugin/src/utils/glob-match.ts rename to packages/shared/src/glob-match.ts index afcde15b..9d38196b 100644 --- a/packages/plugin/src/utils/glob-match.ts +++ b/packages/shared/src/glob-match.ts @@ -1,6 +1,6 @@ -import { isFolder } from '@repo/shared/path'; -import type { GlobStrategy } from '@/types'; +import { isFolder } from './path'; +export type GlobStrategy = { expr: string; strategy: string }; export type GlobMatchResult = { strategy: string; advance?: boolean }; export const NONE_STRATEGY = 'none'; diff --git a/packages/shared/src/obsidian-mock.spec.ts b/packages/shared/src/obsidian-mock.spec.ts index 2d1c2ed5..0b3d735c 100644 --- a/packages/shared/src/obsidian-mock.spec.ts +++ b/packages/shared/src/obsidian-mock.spec.ts @@ -3,7 +3,16 @@ export async function requestUrl() { return { headers: {}, status: 200, text: '' }; } -export const Platform = { isAndroidApp: false, isDesktop: true, isMobile: false }; +export const Platform = { + isAndroidApp: false, + isDesktop: true, + isIosApp: false, + isLinux: false, + isMacOS: false, + isMobile: false, + isTablet: false, + isWin: false, +}; export function normalizePath(path: string) { return path.replaceAll('\\', '/').replaceAll(/\/+/gu, '/'); @@ -16,8 +25,20 @@ export class Notice { export class Vault {} export class TFolder {} export class TFile {} +export class MarkdownView {} export class Plugin {} -export class App {} +export class App { + private readonly localStorageStore = new Map(); + + loadLocalStorage(key: string): unknown { + return this.localStorageStore.get(key); + } + + saveLocalStorage(key: string, data: unknown): void { + if (data === undefined || data === null) this.localStorageStore.delete(key); + else this.localStorageStore.set(key, data); + } +} export class Modal {} export class Setting {} export class PluginSettingTab {} diff --git a/packages/smart-merge/src/diff3/onp.ts b/packages/shared/src/onp.ts similarity index 100% rename from packages/smart-merge/src/diff3/onp.ts rename to packages/shared/src/onp.ts diff --git a/packages/shared/src/secret.ts b/packages/shared/src/secret.ts new file mode 100644 index 00000000..4857e4be --- /dev/null +++ b/packages/shared/src/secret.ts @@ -0,0 +1,8 @@ +import type { App } from 'obsidian'; + +/** Reads a secret from Obsidian's secret storage by its id. */ +export default function readSecret(app: App, id?: string): string | undefined { + return (id ? app.secretStorage.getSecret(id) : undefined) ?? undefined; +} + +export { readSecret }; diff --git a/packages/shared/src/sha256-bytes.ts b/packages/shared/src/sha256-bytes.ts new file mode 100644 index 00000000..a75d2174 --- /dev/null +++ b/packages/shared/src/sha256-bytes.ts @@ -0,0 +1,35 @@ +const encoder = new TextEncoder(); +const hex = '0123456789abcdef'; +const lookup = Array.from({ length: 256 }, (_, i) => hex[i >> 4] + hex[i & 0xf]); + +export function toHex(buffer: ArrayBuffer | Uint8Array): string { + const view = + buffer instanceof ArrayBuffer + ? new DataView(buffer) + : new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength); + let output = ''; + for (let i = 0; i < view.byteLength; i++) output += lookup[view.getUint8(i)]; + return output; +} + +/** Computes SHA-256 digest returning raw ArrayBuffer. */ +export function sha256Digest(data: BufferSource): Promise { + return crypto.subtle.digest('SHA-256', data); +} + +/** Content-addressing primitive. Full SHA-256 over arbitrary bytes, lowercase hex. */ +export async function sha256Bytes(data: Uint8Array): Promise { + // A fresh copy has a concrete ArrayBuffer, satisfying the WebCrypto BufferSource type. + const buffer = new Uint8Array(data).buffer; + return toHex(await crypto.subtle.digest('SHA-256', buffer)); +} + +/** Full SHA-256 over UTF-8 text, lowercase hex. */ +export async function sha256Text(input: string): Promise { + return toHex(await crypto.subtle.digest('SHA-256', encoder.encode(input))); +} + +/** Full SHA-256 over UTF-8 text or raw bytes, lowercase hex. */ +export async function sha256(input: string | Uint8Array): Promise { + return typeof input === 'string' ? sha256Text(input) : sha256Bytes(input); +} diff --git a/packages/shared/test/cdc.test.ts b/packages/shared/test/cdc.test.ts new file mode 100644 index 00000000..e1a73d2e --- /dev/null +++ b/packages/shared/test/cdc.test.ts @@ -0,0 +1,113 @@ +import { textToUint8Array } from '@/binary'; +import { + adaptiveCdcOptions, + assembleChunks, + assembleChunksToText, + chunkContent, + chunkText, + computeChunkDiff, + LRUChunkCache, + reassembleTextFromChunks, +} from '@/cdc'; +import { expect, test } from 'bun:test'; + +test('chunkContent and assembleChunks roundtrip arbitrary bytes', async () => { + const text = 'Hello world! '.repeat(500); // ~6.5 KB + const bytes = textToUint8Array(text); + const chunks = await chunkContent(bytes); + + expect(chunks.length).toBeGreaterThan(1); + for (const chunk of chunks) { + expect(chunk.hash).toMatch(/^[0-9a-f]{64}$/u); + expect(chunk.bytes.byteLength).toBeGreaterThan(0); + } + + const assembled = assembleChunks(chunks.map((c) => c.bytes)); + expect(assembled).toEqual(bytes); + expect(assembleChunksToText(chunks.map((c) => c.bytes))).toBe(text); +}); + +test('chunkText preserves multibyte UTF-8 characters across boundaries', async () => { + const unicode = '你好,世界!这是一段包含多字节字符的中文测试文本。'.repeat(100); + const chunks = await chunkText(unicode); + expect(chunks.length).toBeGreaterThan(1); + + const restored = assembleChunksToText(chunks.map((c) => c.bytes)); + expect(restored).toBe(unicode); +}); + +test('adaptiveCdcOptions selects larger bounds for larger files', () => { + const small = adaptiveCdcOptions(500 * 1024); + expect(small.avg).toBe(1024); + expect(small.max).toBe(4096); + + const medium = adaptiveCdcOptions(4 * 1024 * 1024); + expect(medium.avg).toBe(4096); + expect(medium.max).toBe(16_384); + + const huge = adaptiveCdcOptions(10 * 1024 * 1024); + expect(huge.avg).toBe(16_384); + expect(huge.max).toBe(65_536); +}); + +test('computeChunkDiff detects new vs known chunks accurately', async () => { + const base = `Stable header\n${'A'.repeat(10_000)}\nStable footer`; + const baseChunks = await chunkText(base); + const knownHashes = new Set(baseChunks.map((c) => c.hash)); + + // Modify one line in the middle + const modified = `Stable header\n${'A'.repeat(5000)}MODIFIED${'A'.repeat(4992)}\nStable footer`; + const modifiedChunks = await chunkText(modified); + + const diff = computeChunkDiff(modifiedChunks, (hash) => knownHashes.has(hash)); + expect(diff.chunkHashes.length).toBe(modifiedChunks.length); + // Only a small window of chunks around the edit change, the rest remain identical + expect(diff.newChunks.size).toBeLessThanOrEqual(4); + expect(diff.newChunks.size).toBeGreaterThan(0); +}); + +test('reassembleTextFromChunks returns undefined if any chunk is missing', async () => { + const text = 'Line 1\n'.repeat(1000); + const chunks = await chunkText(text); + const chunkMap = new Map(chunks.map((c) => [c.hash, c.bytes])); + + // 1. All chunks available + const success = reassembleTextFromChunks( + chunks.map((c) => c.hash), + (h) => chunkMap.get(h), + ); + expect(success).toBe(text); + + // 2. One chunk missing + const firstHash = chunks[0].hash; + chunkMap.delete(firstHash); + const failed = reassembleTextFromChunks( + chunks.map((c) => c.hash), + (h) => chunkMap.get(h), + ); + expect(failed).toBeUndefined(); +}); + +test('LRUChunkCache evicts oldest entries when capacity or bytes exceed limit', () => { + const cache = new LRUChunkCache(3, 100); + const b1 = new Uint8Array([1, 2, 3]); + const b2 = new Uint8Array([4, 5, 6]); + const b3 = new Uint8Array([7, 8, 9]); + const b4 = new Uint8Array([10, 11, 12]); + + cache.set('h1', b1); + cache.set('h2', b2); + cache.set('h3', b3); + expect(cache.size).toBe(3); + + // Access h1 to make it most recently used + cache.get('h1'); + + // Insert h4 -> should evict h2 (oldest) + cache.set('h4', b4); + expect(cache.size).toBe(3); + expect(cache.has('h2')).toBe(false); + expect(cache.has('h1')).toBe(true); + expect(cache.has('h3')).toBe(true); + expect(cache.has('h4')).toBe(true); +}); diff --git a/packages/shared/test/device-id.test.ts b/packages/shared/test/device-id.test.ts new file mode 100644 index 00000000..d1a1a265 --- /dev/null +++ b/packages/shared/test/device-id.test.ts @@ -0,0 +1,42 @@ +import { DEVICE_ID_STORAGE_KEY, getOrCreateDeviceId } from '@/device-id'; +import { App } from '@/obsidian-mock.spec'; +import { beforeEach, expect, test } from 'bun:test'; + +beforeEach(() => { + if (typeof localStorage !== 'undefined') localStorage.clear(); +}); + +test('returns existing deviceId from app.loadLocalStorage', () => { + const app = new App(); + app.saveLocalStorage(DEVICE_ID_STORAGE_KEY, 'stored-device-id'); + const id = getOrCreateDeviceId(app as never); + expect(id).toBe('stored-device-id'); +}); + +test('migrates legacy deviceId into app storage when none exists', () => { + const app = new App(); + const id = getOrCreateDeviceId(app as never, 'legacy-uuid'); + expect(id).toBe('legacy-uuid'); + expect(app.loadLocalStorage(DEVICE_ID_STORAGE_KEY)).toBe('legacy-uuid'); +}); + +test('generates and persists a fresh UUID when no prior ID exists', () => { + const app = new App(); + const id = getOrCreateDeviceId(app as never); + expect(id).toBeDefined(); + expect(typeof id).toBe('string'); + expect(id.length).toBeGreaterThan(10); + expect(app.loadLocalStorage(DEVICE_ID_STORAGE_KEY)).toBe(id); + + // Subsequent calls return the same ID + const second = getOrCreateDeviceId(app as never); + expect(second).toBe(id); +}); + +test('falls back to window.localStorage when app methods are missing', () => { + const bareApp = {} as never; + const id = getOrCreateDeviceId(bareApp, 'browser-local-id'); + expect(id).toBe('browser-local-id'); + if (typeof localStorage !== 'undefined') + expect(localStorage.getItem(DEVICE_ID_STORAGE_KEY)).toBe('browser-local-id'); +}); diff --git a/packages/shared/test/device-type.test.ts b/packages/shared/test/device-type.test.ts new file mode 100644 index 00000000..c32a6a84 --- /dev/null +++ b/packages/shared/test/device-type.test.ts @@ -0,0 +1,57 @@ +import deviceType from '@/device-type'; +import { afterEach, expect, test } from 'bun:test'; +import { Platform } from 'obsidian'; + +const defaults = { ...Platform }; + +afterEach(() => { + Object.assign(Platform, defaults); +}); + +function set(flags: Partial): void { + Object.assign(Platform, defaults, flags); +} + +test('falls back to desktop when no OS flag is set', () => { + set({}); + expect(deviceType()).toBe('desktop'); +}); + +test('returns desktop OS names', () => { + set({ isWin: true }); + expect(deviceType()).toBe('windows'); + set({ isLinux: true, isWin: false }); + expect(deviceType()).toBe('linux'); + set({ isLinux: false, isMacOS: true }); + expect(deviceType()).toBe('macos'); +}); + +test('prefers mobile app flags over desktop OS flags', () => { + set({ isIosApp: true, isMacOS: true }); + expect(deviceType()).toBe('ios'); + set({ isAndroidApp: true, isLinux: true }); + expect(deviceType()).toBe('android'); +}); + +test('distinguishes tablets by the same precedence as the plugin OS map', () => { + set({ isAndroidApp: true, isTablet: true }); + expect(deviceType()).toBe('android-tablet'); + set({ isMacOS: true, isTablet: true }); + expect(deviceType()).toBe('ipados'); +}); + +test('output is always safe for an MQTT client ID or device identifier', () => { + for (const flags of [ + {}, + { isWin: true }, + { isLinux: true }, + { isMacOS: true }, + { isIosApp: true }, + { isAndroidApp: true }, + { isAndroidApp: true, isTablet: true }, + { isMacOS: true, isTablet: true }, + ]) { + set(flags); + expect(deviceType()).toMatch(/^[a-z-]+$/u); + } +}); diff --git a/packages/shared/test/fastcdc.test.ts b/packages/shared/test/fastcdc.test.ts new file mode 100644 index 00000000..5997db6c --- /dev/null +++ b/packages/shared/test/fastcdc.test.ts @@ -0,0 +1,58 @@ +import { BIG_BLOB_LIMIT, fastCdc, SMALL_FILE_LIMIT } from '@/fastcdc'; +import { expect, test } from 'bun:test'; + +function bytes(length: number, seed = 7): Uint8Array { + const out = new Uint8Array(length); + let state = seed; + for (let i = 0; i < length; i++) { + state = (Math.imul(state, 1_103_515_245) + 12_345) >>> 0; + out[i] = state & 0xff; + } + return out; +} + +test('constants match the design routing thresholds', () => { + expect(SMALL_FILE_LIMIT).toBe(4096); + expect(BIG_BLOB_LIMIT).toBe(5 * 1024 * 1024); +}); + +test('empty and small inputs return a single trailing chunk', () => { + expect(fastCdc(new Uint8Array(0))).toStrictEqual([]); + const small = bytes(100); + const spans = fastCdc(small); + expect(spans).toHaveLength(1); + expect(spans[0]).toStrictEqual({ length: 100, offset: 0 }); +}); + +test('chunking is deterministic across runs', () => { + const data = bytes(64 * 1024); + expect(fastCdc(data)).toStrictEqual(fastCdc(data)); +}); + +test('every chunk respects min/avg/max bounds and covers the input exactly', () => { + const data = bytes(64 * 1024, 31); + const spans = fastCdc(data, { avg: 1024, max: 4096, min: 512 }); + let cursor = 0; + for (const span of spans) { + expect(span.offset).toBe(cursor); + expect(span.length).toBeGreaterThan(0); + expect(span.length).toBeLessThanOrEqual(4096); + cursor += span.length; + } + expect(cursor).toBe(data.byteLength); + // A 64 KiB input with a 1 KiB target must produce many chunks. + expect(spans.length).toBeGreaterThan(10); +}); + +test('a single-byte edit preserves downstream chunk boundaries (localized change)', () => { + const original = bytes(32 * 1024, 5); + const edited = Uint8Array.from(original); + edited[8000] ^= 0xff; + const a = fastCdc(original); + const b = fastCdc(edited); + const aHashes = new Set(a.map((s) => `${s.offset}:${s.length}`)); + const bHashes = b.map((s) => `${s.offset}:${s.length}`); + // Most chunks are byte-for-byte identical and keep the same offsets. + const shared = bHashes.filter((h) => aHashes.has(h)); + expect(shared.length / b.length).toBeGreaterThan(0.7); +}); diff --git a/packages/plugin/test/glob-match.test.ts b/packages/shared/test/glob-match.test.ts similarity index 98% rename from packages/plugin/test/glob-match.test.ts rename to packages/shared/test/glob-match.test.ts index b8bce64f..b58eb044 100644 --- a/packages/plugin/test/glob-match.test.ts +++ b/packages/shared/test/glob-match.test.ts @@ -1,7 +1,6 @@ -import { NONE_STRATEGY, normalizeGlob, prepareGlobMatch } from '@/utils/glob-match'; +import { NONE_STRATEGY, normalizeGlob, prepareGlobMatch } from '../src/glob-match'; import { expect, test } from 'bun:test'; -import type { GlobStrategy } from '@/types'; -import type { GlobMatchResult } from '@/utils/glob-match'; +import type { GlobMatchResult, GlobStrategy } from '../src/glob-match'; const rule = (expr: string, strategy: string): GlobStrategy => ({ expr, strategy }); diff --git a/packages/shared/test/mocks.ts b/packages/shared/test/mocks.ts new file mode 100644 index 00000000..de35caa8 --- /dev/null +++ b/packages/shared/test/mocks.ts @@ -0,0 +1,4 @@ +import ObsidianMock from '@/obsidian-mock'; +import { mock } from 'bun:test'; + +void mock.module('obsidian', () => ObsidianMock); diff --git a/packages/shared/test/onp.test.ts b/packages/shared/test/onp.test.ts new file mode 100644 index 00000000..edc5a465 --- /dev/null +++ b/packages/shared/test/onp.test.ts @@ -0,0 +1,65 @@ +import Onp from '@/onp'; +import { expect, test } from 'bun:test'; +import type { DiffHunk } from '@/onp'; + +function spansCover(hunks: Array, a: ReadonlyArray, b: ReadonlyArray) { + let x = 0; + let y = 0; + for (const hunk of hunks) { + expect(hunk.file1[0]).toBeGreaterThanOrEqual(x); + expect(hunk.file2[0]).toBeGreaterThanOrEqual(y); + x = hunk.file1[0] + hunk.file1[1]; + y = hunk.file2[0] + hunk.file2[1]; + expect(x).toBeLessThanOrEqual(a.length); + expect(y).toBeLessThanOrEqual(b.length); + } +} + +test('identical sequences produce no change hunks', () => { + const a = ['a', 'b', 'c', 'd']; + const hunks = new Onp(a, ['a', 'b', 'c', 'd']).compose(); + expect(hunks).toStrictEqual([]); +}); + +test('a single insertion yields one localized change hunk', () => { + const a = ['a', 'b', 'c']; + const b = ['a', 'b', 'X', 'c']; + const hunks = new Onp(a, b).compose(); + expect(hunks).toHaveLength(1); + spansCover(hunks, a, b); + expect(hunks[0].file1[1]).toBe(0); + expect(hunks[0].file2[1]).toBe(1); +}); + +test('a single deletion yields one localized change hunk', () => { + const a = ['a', 'b', 'X', 'c']; + const b = ['a', 'b', 'c']; + const hunks = new Onp(a, b).compose(); + expect(hunks).toHaveLength(1); + spansCover(hunks, a, b); + expect(hunks[0].file1[1]).toBe(1); + expect(hunks[0].file2[1]).toBe(0); +}); + +test('a replacement keeps surrounding runs as common', () => { + const a = ['1', 'L', '3', '4', '5']; + const b = ['1', 'R', '3', '4', '6']; + const hunks = new Onp(a, b).compose(); + spansCover(hunks, a, b); + // The divergent middle and the divergent tail are the only changed regions. + expect(hunks.length).toBeGreaterThanOrEqual(2); +}); + +test('reversed input (m < n) swaps sequences and still produces valid hunks', () => { + const a = ['a', 'b']; + const b = ['a', 'b', 'c', 'd', 'e']; + const hunks = new Onp(a, b).compose(); + spansCover(hunks, a, b); + expect(hunks.some((h) => h.file2[1] > 0 && h.file1[1] === 0)).toBe(true); +}); + +test('empty sequences are handled without error', () => { + expect(new Onp([], []).compose()).toStrictEqual([]); + expect(new Onp([], ['x']).compose()).toHaveLength(1); + expect(new Onp(['x'], []).compose()).toHaveLength(1); +}); diff --git a/packages/shared/test/sha256-bytes.test.ts b/packages/shared/test/sha256-bytes.test.ts new file mode 100644 index 00000000..7ad42e4b --- /dev/null +++ b/packages/shared/test/sha256-bytes.test.ts @@ -0,0 +1,26 @@ +import { sha256, sha256Bytes, sha256Digest, sha256Text, toHex } from '../src/sha256-bytes'; +import { expect, test } from 'bun:test'; + +test('sha256Text and sha256Bytes produce consistent lowercase hex hash', async () => { + const str = 'hello world'; + const bytes = new TextEncoder().encode(str); + const expected = 'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9'; + + expect(await sha256Text(str)).toBe(expected); + expect(await sha256Bytes(bytes)).toBe(expected); + expect(await sha256(str)).toBe(expected); + expect(await sha256(bytes)).toBe(expected); +}); + +test('sha256Digest returns raw ArrayBuffer', async () => { + const bytes = new TextEncoder().encode('hello world'); + const digest = await sha256Digest(bytes); + expect(digest instanceof ArrayBuffer).toBe(true); + expect(toHex(digest)).toBe('b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9'); +}); + +test('toHex converts both Uint8Array and ArrayBuffer', () => { + const bytes = new Uint8Array([0xde, 0xad, 0xbe, 0xef]); + expect(toHex(bytes)).toBe('deadbeef'); + expect(toHex(bytes.buffer)).toBe('deadbeef'); +}); diff --git a/packages/smart-merge/src/diff3/index.ts b/packages/smart-merge/src/diff3/index.ts index eff91275..4d21f2d8 100644 --- a/packages/smart-merge/src/diff3/index.ts +++ b/packages/smart-merge/src/diff3/index.ts @@ -23,8 +23,8 @@ // CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -import Onp from './onp'; -import type { DiffHunk } from './onp'; +import Onp from '@repo/shared/onp'; +import type { DiffHunk } from '@repo/shared/onp'; export type Conflict = { a: Array; o: Array; b: Array }; diff --git a/scripts/check-unused-exports.ts b/scripts/check-unused-exports.ts new file mode 100644 index 00000000..5bb07e3e --- /dev/null +++ b/scripts/check-unused-exports.ts @@ -0,0 +1,184 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import ts from 'typescript'; + +function collectFiles(dir: string): Array { + if (!fs.existsSync(dir)) return []; + const entries = fs.readdirSync(dir, { withFileTypes: true }); + const files: Array = []; + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name !== 'node_modules' && entry.name !== 'dist' && !entry.name.startsWith('.')) + files.push(...collectFiles(fullPath)); + } else if ( + entry.isFile() && + (fullPath.endsWith('.ts') || fullPath.endsWith('.tsx')) && + !fullPath.endsWith('.d.ts') + ) + files.push(fullPath); + } + return files; +} + +const rootDir = process.cwd(); +const allFiles = [ + ...collectFiles(path.join(rootDir, 'packages')), + ...collectFiles(path.join(rootDir, 'scripts')), + ...collectFiles(path.join(rootDir, 'docs')), +]; + +// Entry points whose exports are intended for external consumers (Obsidian runtime, SDK users, etc.) +const entryFiles = new Set(); + +const packagesDir = path.join(rootDir, 'packages'); +for (const p of fs.readdirSync(packagesDir)) { + const pkgPath = path.join(packagesDir, p); + if (!fs.statSync(pkgPath).isDirectory()) continue; + const mainIndex = path.normalize(path.join(pkgPath, 'src/index.ts')); + if (fs.existsSync(mainIndex)) entryFiles.add(mainIndex); +} + +// SDK entries +entryFiles.add(path.normalize(path.join(packagesDir, 'plugin/src/sdk/index.ts'))); +entryFiles.add(path.normalize(path.join(packagesDir, 'plugin/src/sdk/dev.ts'))); +entryFiles.add(path.normalize(path.join(packagesDir, 'plugin/src/sdk/tsdown-plugin.ts'))); + +// Step 1: Collect all used symbol names across all files +const usedNames = new Set(); + +for (const filePath of allFiles) { + const content = fs.readFileSync(filePath, 'utf8'); + const sourceFile = ts.createSourceFile(filePath, content, ts.ScriptTarget.Latest, true); + + const namespaces = new Set(); + + ts.forEachChild(sourceFile, function visit(node) { + // Import declarations: import { a, b as c } from '...' + if (ts.isImportDeclaration(node)) { + const clause = node.importClause; + if (clause) { + if (clause.name) usedNames.add(clause.name.text); + if (clause.namedBindings) { + if (ts.isNamedImports(clause.namedBindings)) { + for (const element of clause.namedBindings.elements) { + // If aliased (import { a as b }), the original export is element.propertyName + const importedName = element.propertyName + ? element.propertyName.text + : element.name.text; + usedNames.add(importedName); + } + } else if (ts.isNamespaceImport(clause.namedBindings)) { + namespaces.add(clause.namedBindings.name.text); + } + } + } + } + + // Re-export declarations: export { a } from '...' + if (ts.isExportDeclaration(node)) { + if (node.exportClause && ts.isNamedExports(node.exportClause)) { + for (const element of node.exportClause.elements) { + if (node.moduleSpecifier) { + const exportedName = element.propertyName + ? element.propertyName.text + : element.name.text; + usedNames.add(exportedName); + } + } + } + } + + // Dynamic imports / destructuring: const { a, b } = await import(...) + if (ts.isVariableDeclaration(node) && ts.isObjectBindingPattern(node.name)) { + for (const element of node.name.elements) { + if (ts.isIdentifier(element.name)) { + const propName = element.propertyName && ts.isIdentifier(element.propertyName) + ? element.propertyName.text + : element.name.text; + usedNames.add(propName); + } + } + } + + // Check property accesses on namespace imports (e.g. C.SECOND_MS) + if (ts.isPropertyAccessExpression(node)) { + if (ts.isIdentifier(node.expression) && namespaces.has(node.expression.text)) + usedNames.add(node.name.text); + } + + ts.forEachChild(node, visit); + }); +} + +// Step 2: Check non-entry internal files in packages/*/src for unused function/variable/class exports +type DeadExport = { + file: string; + kind: string; + line: number; + name: string; +}; + +const deadExports: Array = []; + +for (const filePath of allFiles) { + const normalized = path.normalize(filePath); + if (entryFiles.has(normalized)) continue; + if ( + normalized.includes('/test/') || + normalized.includes('.spec.') || + normalized.includes('/scripts/') || + normalized.includes('/docs/') + ) + continue; + if (!normalized.startsWith(path.join(rootDir, 'packages'))) continue; + + const content = fs.readFileSync(filePath, 'utf8'); + const sourceFile = ts.createSourceFile(filePath, content, ts.ScriptTarget.Latest, true); + + ts.forEachChild(sourceFile, (node) => { + const isExported = node.modifiers?.some((m) => m.kind === ts.SyntaxKind.ExportKeyword); + const isDefault = node.modifiers?.some((m) => m.kind === ts.SyntaxKind.DefaultKeyword); + if (!isExported || isDefault) return; + + // Check leading comments for @allow-unused or @public + const commentRanges = ts.getLeadingCommentRanges(content, node.pos) ?? []; + const hasAllow = commentRanges.some((range) => { + const text = content.slice(range.pos, range.end); + return text.includes('@allow-unused') || text.includes('@public'); + }); + if (hasAllow) return; + + const verify = (name: string, pos: number, kind: string) => { + if (!usedNames.has(name)) { + const { line } = sourceFile.getLineAndCharacterOfPosition(pos); + const relPath = path.relative(rootDir, filePath); + deadExports.push({ file: relPath, kind, line: line + 1, name }); + } + }; + + if (ts.isFunctionDeclaration(node) && node.name) { + verify(node.name.text, node.name.pos, 'function'); + } else if (ts.isVariableStatement(node)) { + for (const decl of node.declarationList.declarations) { + if (ts.isIdentifier(decl.name)) verify(decl.name.text, decl.name.pos, 'variable/constant'); + } + } else if (ts.isClassDeclaration(node) && node.name) { + verify(node.name.text, node.name.pos, 'class'); + } + }); +} + +if (deadExports.length > 0) { + console.error( + `\n❌ Found ${deadExports.length} unused exported function(s) / variable(s) across packages:\n`, + ); + for (const item of deadExports) + console.error(` - ${item.file}:${item.line} -> [${item.kind}] '${item.name}'`); + console.error( + '\nFix: remove "export" if only used internally, delete if dead code, or annotate with "/** @allow-unused */" if public.\n', + ); + process.exit(1); +} else { + console.log('✔ All exported functions and variables are referenced.'); +} diff --git a/tsconfig.json b/tsconfig.json index 7ed2dc5a..ff700f64 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,6 +9,7 @@ "skipLibCheck": true, "forceConsistentCasingInFileNames": true, "strictNullChecks": true, + "noUnusedLocals": true, "resolveJsonModule": true, "noEmit": true, "verbatimModuleSyntax": true,