From 817209af40b9948de23fc30881f8a8b580f50db2 Mon Sep 17 00:00:00 2001 From: Jared Hance Date: Tue, 17 Nov 2026 04:11:11 +0000 Subject: [PATCH 1/2] Trust the fine-grained cache for files not known to have changed A daemon started with --use-fine-grained-cache looks at every source on its first check: validate_meta stats each file (and hashes it when the cache's mtime does not match, which a downloaded cache's never does), and the fswatcher then stats them all again to find what changed since the cache. For a large repository on a network or FUSE filesystem, that is the whole cost of a cold start, though a tool that knows what changed - git, or a filesystem that keeps a journal - could say so in one step. --changed-since-cache-file names that file: every path that may differ from the files the cache was generated from, one per line. For any other source whose cache metadata names the same path, validate_meta accepts the metadata without a stat, State creation does not ask whether it is a directory, and the first fine-grained update looks only at the named files and at sources the cache has nothing for, through the same update_changed path dmypy recheck --update takes. A missing file is warned about and ignored, so every file is looked at as before. Like --quickstart-file, the option is hidden: its user is a wrapper that knows how the cache was made. --- mypy/build.py | 45 ++++++++++++++++++++++++++++++++++++-- mypy/config_parser.py | 1 + mypy/dmypy_server.py | 15 ++++++++++++- mypy/main.py | 1 + mypy/options.py | 6 +++++ test-data/unit/daemon.test | 26 ++++++++++++++++++++++ 6 files changed, 91 insertions(+), 3 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 38615f6ddbc14..5e5059f5babfa 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -955,6 +955,7 @@ def __init__( if self.verbosity() >= 2: self.trace(f"Plugins snapshot (fresh) {json.dumps(self.plugins_snapshot)}") self.quickstart_state = read_quickstart_file(options, self.stdout) + self.changed_since_cache = read_changed_since_cache_file(options, self.stdout) # Fine grained targets (module top levels and top level functions) processed by # the semantic analyzer, used only for testing. Currently used only by the new # semantic analyzer. Tuple of module and target name. @@ -1143,6 +1144,18 @@ def post_parse_all(self, states: list[State]) -> None: def use_fine_grained_cache(self) -> bool: return self.cache_enabled and self.options.use_fine_grained_cache + def trusts_cache_for(self, path: str) -> bool: + """Is the file at path known to match its fine-grained cache metadata? + + True only if --changed-since-cache-file names the files that may differ, and + it does not name this one. + """ + return ( + self.changed_since_cache is not None + and self.use_fine_grained_cache() + and path not in self.changed_since_cache + ) + def maybe_swap_for_shadow_path(self, path: str) -> str: if not self.shadow_map: return path @@ -1738,6 +1751,19 @@ def read_plugins_snapshot(manager: BuildManager) -> dict[str, str] | None: return snapshot +def read_changed_since_cache_file(options: Options, stdout: TextIO) -> set[str] | None: + """The paths that may differ from those the cache was generated from, if known.""" + if not options.changed_since_cache_file: + return None + try: + with open(options.changed_since_cache_file) as f: + return {os.path.normpath(line.rstrip("\n")) for line in f if line.strip()} + except OSError as e: + # Without it, every file is looked at, as without the option. + print(f"Warning: Failed to load changed-since-cache file: {str(e)}\n", file=stdout) + return None + + def read_quickstart_file( options: Options, stdout: TextIO ) -> dict[str, tuple[float, int, str]] | None: @@ -2194,6 +2220,10 @@ def validate_meta( # Normalize path under bazel to make sure it isn't absolute path = normpath(path, manager.options) + if path == meta.path and manager.trusts_cache_for(path): + manager.log(f"Metadata trusted for {id}: {path} is unchanged since the cache") + return meta + st = manager.get_stat(path) if st is None: return None @@ -2777,8 +2807,19 @@ def new_state( local_partial_types = meta.options["local_partial_types"] if options.local_partial_types != local_partial_types: options = options.apply_changes({"local_partial_types": local_partial_types}) - if path and source is None and manager.fscache.isdir(path): - source = "" + if path and source is None: + if ( + meta is not None + and meta.path == path + and path.endswith((".py", ".pyi")) + and manager.trusts_cache_for(path) + ): + # A file, as when the cache was made: no need to look. + is_dir = False + else: + is_dir = manager.fscache.isdir(path) + if is_dir: + source = "" if manager.stats_enabled: t0 = time.time() diff --git a/mypy/config_parser.py b/mypy/config_parser.py index 14e85d8dcef24..ab29f0e37de83 100644 --- a/mypy/config_parser.py +++ b/mypy/config_parser.py @@ -201,6 +201,7 @@ def split_commas(value: str) -> list[str]: "mypy_path": lambda s: [expand_path(p.strip()) for p in re.split("[,:]", s)], "files": split_and_match_files, "quickstart_file": expand_path, + "changed_since_cache_file": expand_path, "junit_xml": expand_path, "junit_format": check_junit_format, "follow_imports": check_follow_imports, diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index 066fbf9bed2d6..aeac31dce1960 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -505,7 +505,20 @@ def initialize_fine_grained( FileData(st_mtime=float(meta.mtime), st_size=meta.size, hash=meta.hash), ) - changed, removed = self.find_changed(sources) + changed_since_cache = self.fine_grained_manager.manager.changed_since_cache + if changed_since_cache is not None: + # Look only at the files that may differ from the cache, and at sources + # it has nothing for, which the build deferred. + graph = self.fine_grained_manager.graph + update = [ + source.path + for source in sources + if source.path + and (source.path in changed_since_cache or source.module not in graph) + ] + changed, removed = self.update_changed(sources, [], update) + else: + changed, removed = self.find_changed(sources) changed += self.find_added_suppressed( self.fine_grained_manager.graph, set(), diff --git a/mypy/main.py b/mypy/main.py index 9982d7a6eace5..3a14d78dda8f6 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1231,6 +1231,7 @@ def add_invertible_flag( misc_group = parser.add_argument_group(title="Miscellaneous") misc_group.add_argument("--quickstart-file", help=argparse.SUPPRESS) + misc_group.add_argument("--changed-since-cache-file", help=argparse.SUPPRESS) misc_group.add_argument( "--junit-xml", metavar="JUNIT_XML_OUTPUT_FILE", diff --git a/mypy/options.py b/mypy/options.py index 40fa56bd1b3ba..a177f038a74f8 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -288,6 +288,12 @@ def __init__(self) -> None: # source hashes as often. self.quickstart_file: str | None = None + # A file naming, one path per line, every file that may differ from those + # the fine-grained cache was generated from. With --use-fine-grained-cache, + # every other file is trusted to match its cache metadata without being + # looked at, so the first build costs what changed rather than what exists. + self.changed_since_cache_file: str | None = None + # A comma-separated list of files/directories for mypy to type check; # supports globbing self.files: list[str] | None = None diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index 448af5d080029..871a81bab2993 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -343,6 +343,32 @@ $ {python} -c "import sys; sys.stdout.write(open('log').read())" -- whether the source file got rehashed, which we don't want it to have been. $ {python} -c "x = open('.mypy_cache/3.11/bar.meta.ff', 'rb').read(); y = open('asdf.ff', 'rb').read(); assert x == y" +-- Only files the changed-since-cache file names are looked at: bar changes too, but is +-- trusted to match the cache until a recheck names it. baz has no cache, so it is checked. +[case testDaemonChangedSinceCache] +$ {python} -c "print('x = 1')" >foo.py +$ {python} -c "print('y = 1')" >bar.py +$ mypy --local-partial-types --cache-fine-grained --follow-imports=error --no-sqlite-cache --python-version=3.11 -- foo.py bar.py +Success: no issues found in 2 source files +$ {python} -c "print('x = lol')" >foo.py +$ {python} -c "print('y = lol')" >bar.py +$ {python} -c "print('z = lol')" >baz.py +$ {python} -c "print('foo.py')" >changed.txt +$ dmypy run -- foo.py bar.py baz.py --follow-imports=error --use-fine-grained-cache --no-sqlite-cache --python-version=3.11 --changed-since-cache-file changed.txt +Daemon started +baz.py:1: error: Name "lol" is not defined +foo.py:1: error: Name "lol" is not defined +Found 2 errors in 2 files (checked 3 source files) +== Return code: 1 +$ dmypy recheck --update bar.py +baz.py:1: error: Name "lol" is not defined +foo.py:1: error: Name "lol" is not defined +bar.py:1: error: Name "lol" is not defined +Found 3 errors in 3 files (checked 3 source files) +== Return code: 1 +$ dmypy stop +Daemon stopped + [case testDaemonSuggest] $ dmypy start --log-file log.txt -- --follow-imports=error --no-error-summary Daemon started From a4dd7ef4eed2c2829b0224b98a7bd3c328bed88b Mon Sep 17 00:00:00 2001 From: Jared Hance Date: Tue, 17 Nov 2026 17:20:35 +0000 Subject: [PATCH 2/2] Take trusted files' modules from the cache instead of crawling Before its first check, dmypy turns each source path into a module by crawling up its directories for __init__ files. With the cache trusted for files not named in --changed-since-cache-file, that is the one look left at each source's directories, and over a network or FUSE filesystem the largest part of a cold start. The cache already says it: each module's metadata has its path and id, and a module's components are the directories above the file, so its base directory follows without looking. A file's module is as when the cache was made unless the file, or an __init__ in a directory above it, changed since, so for every other file the first check takes its module from the cache; create_source_list takes these as known_modules, and crawls for the rest and for any path whose directories do not match its module. --- mypy/build.py | 26 +++++++++++++++++++++++ mypy/dmypy_server.py | 39 +++++++++++++++++++++++++++++++++- mypy/find_sources.py | 34 ++++++++++++++++++++++++++++- mypy/test/test_find_sources.py | 37 +++++++++++++++++++++++++++++++- test-data/unit/daemon.test | 23 ++++++++++++++++++++ 5 files changed, 156 insertions(+), 3 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 5e5059f5babfa..8e4a283aa1a3d 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1764,6 +1764,32 @@ def read_changed_since_cache_file(options: Options, stdout: TextIO) -> set[str] return None +def read_cached_modules(options: Options) -> dict[str, str]: + """Map the path of each module in the cache to the module's id.""" + metastore = create_metastore(options, parallel_worker=False) + modules = {} + try: + for name in metastore.list_all(): + if name.startswith("@") or not name.endswith((".meta.ff", ".meta.json")): + continue + try: + data = metastore.read(name) + except OSError: + continue + meta: CacheMeta | None + if name.endswith(".meta.ff"): + if data[0] != cache_version() or data[1] != CACHE_VERSION: + continue + meta = CacheMeta.read(ReadBuffer(data[2:]), data_file="") + else: + meta = CacheMeta.deserialize(json_loads(data), data_file="") + if meta is not None: + modules[meta.path] = meta.id + finally: + metastore.close() + return modules + + def read_quickstart_file( options: Options, stdout: TextIO ) -> dict[str, tuple[float, int, str]] | None: diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index aeac31dce1960..5ad0530ba7073 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -364,11 +364,48 @@ def cmd_check( ) -> dict[str, object]: """Check a list of files.""" try: - sources = create_source_list(files, self.options, self.fscache) + sources = create_source_list( + files, self.options, self.fscache, known_modules=self.trusted_cached_modules() + ) except InvalidSourceList as err: return {"out": "", "err": str(err), "status": 2} return self.check(sources, export_types, is_tty, terminal_width) + def trusted_cached_modules(self) -> dict[str, str] | None: + """The module of each file a first check trusts the cache for. + + A file's module is as when the cache was made unless the file, or an __init__ in a + directory above it, has changed since: so the first check of a daemon trusting its + cache need not crawl up each file's directories to find its module. + """ + if self.fine_grained_manager or not self.options.use_fine_grained_cache: + return None + # The build warns, if need be, of a file it cannot read. + changed = mypy.build.read_changed_since_cache_file(self.options, io.StringIO()) + if changed is None: + return None + changed_packages = { + os.path.dirname(path) + for path in changed + if os.path.basename(path) in ("__init__.py", "__init__.pyi") + } + + def under_changed_package(path: str) -> bool: + dir = os.path.dirname(path) + while True: + if dir in changed_packages: + return True + parent = os.path.dirname(dir) + if parent == dir: + return False + dir = parent + + return { + path: module + for path, module in mypy.build.read_cached_modules(self.options).items() + if path not in changed and not under_changed_package(path) + } + def cmd_recheck( self, is_tty: bool, diff --git a/mypy/find_sources.py b/mypy/find_sources.py index e1edebe90985b..c00930b9c0bb8 100644 --- a/mypy/find_sources.py +++ b/mypy/find_sources.py @@ -4,7 +4,7 @@ import functools import os -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import Final from mypy.fscache import FileSystemCache @@ -29,9 +29,13 @@ def create_source_list( options: Options, fscache: FileSystemCache | None = None, allow_empty_dir: bool = False, + known_modules: Mapping[str, str] | None = None, ) -> list[BuildSource]: """From a list of source files/directories, makes a list of BuildSources. + known_modules maps files to their modules, known already, as from a cache: such a + file's module and base directory come from it, without crawling up its directories. + Raises InvalidSourceList on errors. """ fscache = fscache or FileSystemCache() @@ -41,6 +45,11 @@ def create_source_list( for path in paths: path = os.path.normpath(path) if path.endswith(PY_EXTENSIONS): + module = known_modules.get(path) if known_modules else None + base_dir = base_dir_of(path, module) if module is not None else None + if module is not None and base_dir is not None: + sources.append(BuildSource(path, module, None, base_dir)) + continue # Can raise InvalidSourceList if a directory doesn't have a valid module name. name, base_dir = finder.crawl_up(path) sources.append(BuildSource(path, name, None, base_dir)) @@ -55,6 +64,29 @@ def create_source_list( return sources +def base_dir_of(path: str, module: str) -> str | None: + """The base directory crawl_up would find for a file holding module, without looking. + + Each component of a module but its last is a directory above the file, so the base + directory is as many directories up. None if the path's directories do not match the + module's components. + """ + dir = os.path.dirname(os.path.abspath(path)) + components = module.split(".") + stem = strip_py(os.path.basename(path)) + if stem == "__init__": + packages = components + elif components[-1] == stem: + packages = components[:-1] + else: + return None + for package in reversed(packages): + dir, name = os.path.split(dir) + if name.removesuffix("-stubs") != package: + return None + return dir + + def keyfunc(name: str) -> tuple[bool, int, str]: """Determines sort order for directory listing. diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index 321f3405e9991..462a81cd20d63 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -7,7 +7,7 @@ import pytest -from mypy.find_sources import InvalidSourceList, SourceFinder, create_source_list +from mypy.find_sources import InvalidSourceList, SourceFinder, base_dir_of, create_source_list from mypy.fscache import FileSystemCache from mypy.modulefinder import BuildSource from mypy.options import Options @@ -374,3 +374,38 @@ def test_find_sources_exclude(self) -> None: } fscache = FakeFSCache(files) assert len(find_sources(["."], options, fscache)) == len(files) + + def test_base_dir_of_matches_crawl_up(self) -> None: + files = { + "/a/pkg/__init__.py", + "/a/pkg/sub/__init__.py", + "/a/pkg/sub/mod.py", + "/a/pkg/mod.pyi", + "/a/top.py", + "/a/foo-stubs/__init__.pyi", + "/a/foo-stubs/bar.pyi", + } + finder = SourceFinder(FakeFSCache(files), Options()) + for f in files: + module, base_dir = finder.crawl_up(f) + assert base_dir_of(f, module) == base_dir, f + + def test_base_dir_of_mismatch(self) -> None: + assert base_dir_of("/a/pkg/mod.py", "pkg.other") is None + assert base_dir_of("/a/pkg/mod.py", "other.mod") is None + assert base_dir_of("/a/pkg/__init__.py", "other") is None + + def test_known_modules_need_no_crawl(self) -> None: + class NoFS(FakeFSCache): + def isfile(self, path: str) -> bool: + raise AssertionError(f"looked at {path}") + + sources = create_source_list( + ["/a/pkg/sub/mod.py"], + Options(), + NoFS(set()), + known_modules={"/a/pkg/sub/mod.py": "pkg.sub.mod"}, + ) + assert [(s.module, normalise_path(s.base_dir or "")) for s in sources] == [ + ("pkg.sub.mod", "/a") + ] diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index 871a81bab2993..d924a2fb1593d 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -369,6 +369,29 @@ Found 3 errors in 3 files (checked 3 source files) $ dmypy stop Daemon stopped +-- The first check takes the modules of files it trusts the cache for from the cache, but not +-- of those under an __init__ that changed: without pkg/__init__.py, pkg/a.py is module a. +[case testDaemonChangedSinceCacheModules] +$ {python} -c "import os; os.makedirs('pkg'); os.makedirs('other')" +$ {python} -c "print('')" >pkg/__init__.py +$ {python} -c "print('x = 1')" >pkg/a.py +$ {python} -c "print('')" >other/__init__.py +$ {python} -c "print('y = 1')" >other/b.py +$ {python} -c "print('from pkg.a import x; from other.b import y')" >c.py +$ mypy --local-partial-types --cache-fine-grained --follow-imports=error --no-sqlite-cache --python-version=3.11 -- pkg/__init__.py pkg/a.py other/__init__.py other/b.py c.py +Success: no issues found in 5 source files +$ {python} -c "import os; os.remove('pkg/__init__.py')" +$ {python} -c "print('pkg/__init__.py')" >changed.txt +$ dmypy start -- --local-partial-types --follow-imports=error --use-fine-grained-cache --no-sqlite-cache --python-version=3.11 --changed-since-cache-file changed.txt +Daemon started +$ dmypy check pkg/a.py other/__init__.py other/b.py c.py +c.py:1: error: Import of "pkg.a" ignored [misc] +c.py:1: note: (Using --follow-imports=error, module not passed on command line) +Found 1 error in 1 file (checked 4 source files) +== Return code: 1 +$ dmypy stop +Daemon stopped + [case testDaemonSuggest] $ dmypy start --log-file log.txt -- --follow-imports=error --no-error-summary Daemon started