Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 69 additions & 2 deletions mypy/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -955,6 +955,7 @@ def __init__(
if self.verbosity() >= 2:
self.trace(f"Plugins snapshot (fresh) {json.dumps(self.plugins_snapshot)}")
self.quickstart_state = read_quickstart_file(options, self.stdout)
self.changed_since_cache = read_changed_since_cache_file(options, self.stdout)
# Fine grained targets (module top levels and top level functions) processed by
# the semantic analyzer, used only for testing. Currently used only by the new
# semantic analyzer. Tuple of module and target name.
Expand Down Expand Up @@ -1143,6 +1144,18 @@ def post_parse_all(self, states: list[State]) -> None:
def use_fine_grained_cache(self) -> bool:
return self.cache_enabled and self.options.use_fine_grained_cache

def trusts_cache_for(self, path: str) -> bool:
"""Is the file at path known to match its fine-grained cache metadata?

True only if --changed-since-cache-file names the files that may differ, and
it does not name this one.
"""
return (
self.changed_since_cache is not None
and self.use_fine_grained_cache()
and path not in self.changed_since_cache
)

def maybe_swap_for_shadow_path(self, path: str) -> str:
if not self.shadow_map:
return path
Expand Down Expand Up @@ -1738,6 +1751,45 @@ def read_plugins_snapshot(manager: BuildManager) -> dict[str, str] | None:
return snapshot


def read_changed_since_cache_file(options: Options, stdout: TextIO) -> set[str] | None:
"""The paths that may differ from those the cache was generated from, if known."""
if not options.changed_since_cache_file:
return None
try:
with open(options.changed_since_cache_file) as f:
return {os.path.normpath(line.rstrip("\n")) for line in f if line.strip()}
except OSError as e:
# Without it, every file is looked at, as without the option.
print(f"Warning: Failed to load changed-since-cache file: {str(e)}\n", file=stdout)
return None


def read_cached_modules(options: Options) -> dict[str, str]:
"""Map the path of each module in the cache to the module's id."""
metastore = create_metastore(options, parallel_worker=False)
modules = {}
try:
for name in metastore.list_all():
if name.startswith("@") or not name.endswith((".meta.ff", ".meta.json")):
continue
try:
data = metastore.read(name)
except OSError:
continue
meta: CacheMeta | None
if name.endswith(".meta.ff"):
if data[0] != cache_version() or data[1] != CACHE_VERSION:
continue
meta = CacheMeta.read(ReadBuffer(data[2:]), data_file="")
else:
meta = CacheMeta.deserialize(json_loads(data), data_file="")
if meta is not None:
modules[meta.path] = meta.id
finally:
metastore.close()
return modules


def read_quickstart_file(
options: Options, stdout: TextIO
) -> dict[str, tuple[float, int, str]] | None:
Expand Down Expand Up @@ -2194,6 +2246,10 @@ def validate_meta(
# Normalize path under bazel to make sure it isn't absolute
path = normpath(path, manager.options)

if path == meta.path and manager.trusts_cache_for(path):
manager.log(f"Metadata trusted for {id}: {path} is unchanged since the cache")
return meta

st = manager.get_stat(path)
if st is None:
return None
Expand Down Expand Up @@ -2777,8 +2833,19 @@ def new_state(
local_partial_types = meta.options["local_partial_types"]
if options.local_partial_types != local_partial_types:
options = options.apply_changes({"local_partial_types": local_partial_types})
if path and source is None and manager.fscache.isdir(path):
source = ""
if path and source is None:
if (
meta is not None
and meta.path == path
and path.endswith((".py", ".pyi"))
and manager.trusts_cache_for(path)
):
# A file, as when the cache was made: no need to look.
is_dir = False
else:
is_dir = manager.fscache.isdir(path)
if is_dir:
source = ""

if manager.stats_enabled:
t0 = time.time()
Expand Down
1 change: 1 addition & 0 deletions mypy/config_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,7 @@ def split_commas(value: str) -> list[str]:
"mypy_path": lambda s: [expand_path(p.strip()) for p in re.split("[,:]", s)],
"files": split_and_match_files,
"quickstart_file": expand_path,
"changed_since_cache_file": expand_path,
"junit_xml": expand_path,
"junit_format": check_junit_format,
"follow_imports": check_follow_imports,
Expand Down
54 changes: 52 additions & 2 deletions mypy/dmypy_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -364,11 +364,48 @@ def cmd_check(
) -> dict[str, object]:
"""Check a list of files."""
try:
sources = create_source_list(files, self.options, self.fscache)
sources = create_source_list(
files, self.options, self.fscache, known_modules=self.trusted_cached_modules()
)
except InvalidSourceList as err:
return {"out": "", "err": str(err), "status": 2}
return self.check(sources, export_types, is_tty, terminal_width)

def trusted_cached_modules(self) -> dict[str, str] | None:
"""The module of each file a first check trusts the cache for.

A file's module is as when the cache was made unless the file, or an __init__ in a
directory above it, has changed since: so the first check of a daemon trusting its
cache need not crawl up each file's directories to find its module.
"""
if self.fine_grained_manager or not self.options.use_fine_grained_cache:
return None
# The build warns, if need be, of a file it cannot read.
changed = mypy.build.read_changed_since_cache_file(self.options, io.StringIO())
if changed is None:
return None
changed_packages = {
os.path.dirname(path)
for path in changed
if os.path.basename(path) in ("__init__.py", "__init__.pyi")
}

def under_changed_package(path: str) -> bool:
dir = os.path.dirname(path)
while True:
if dir in changed_packages:
return True
parent = os.path.dirname(dir)
if parent == dir:
return False
dir = parent

return {
path: module
for path, module in mypy.build.read_cached_modules(self.options).items()
if path not in changed and not under_changed_package(path)
}

def cmd_recheck(
self,
is_tty: bool,
Expand Down Expand Up @@ -505,7 +542,20 @@ def initialize_fine_grained(
FileData(st_mtime=float(meta.mtime), st_size=meta.size, hash=meta.hash),
)

changed, removed = self.find_changed(sources)
changed_since_cache = self.fine_grained_manager.manager.changed_since_cache
if changed_since_cache is not None:
# Look only at the files that may differ from the cache, and at sources
# it has nothing for, which the build deferred.
graph = self.fine_grained_manager.graph
update = [
source.path
for source in sources
if source.path
and (source.path in changed_since_cache or source.module not in graph)
]
changed, removed = self.update_changed(sources, [], update)
else:
changed, removed = self.find_changed(sources)
changed += self.find_added_suppressed(
self.fine_grained_manager.graph,
set(),
Expand Down
34 changes: 33 additions & 1 deletion mypy/find_sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import functools
import os
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from typing import Final

from mypy.fscache import FileSystemCache
Expand All @@ -29,9 +29,13 @@ def create_source_list(
options: Options,
fscache: FileSystemCache | None = None,
allow_empty_dir: bool = False,
known_modules: Mapping[str, str] | None = None,
) -> list[BuildSource]:
"""From a list of source files/directories, makes a list of BuildSources.

known_modules maps files to their modules, known already, as from a cache: such a
file's module and base directory come from it, without crawling up its directories.

Raises InvalidSourceList on errors.
"""
fscache = fscache or FileSystemCache()
Expand All @@ -41,6 +45,11 @@ def create_source_list(
for path in paths:
path = os.path.normpath(path)
if path.endswith(PY_EXTENSIONS):
module = known_modules.get(path) if known_modules else None
base_dir = base_dir_of(path, module) if module is not None else None
if module is not None and base_dir is not None:
sources.append(BuildSource(path, module, None, base_dir))
continue
# Can raise InvalidSourceList if a directory doesn't have a valid module name.
name, base_dir = finder.crawl_up(path)
sources.append(BuildSource(path, name, None, base_dir))
Expand All @@ -55,6 +64,29 @@ def create_source_list(
return sources


def base_dir_of(path: str, module: str) -> str | None:
"""The base directory crawl_up would find for a file holding module, without looking.

Each component of a module but its last is a directory above the file, so the base
directory is as many directories up. None if the path's directories do not match the
module's components.
"""
dir = os.path.dirname(os.path.abspath(path))
components = module.split(".")
stem = strip_py(os.path.basename(path))
if stem == "__init__":
packages = components
elif components[-1] == stem:
packages = components[:-1]
else:
return None
for package in reversed(packages):
dir, name = os.path.split(dir)
if name.removesuffix("-stubs") != package:
return None
return dir


def keyfunc(name: str) -> tuple[bool, int, str]:
"""Determines sort order for directory listing.

Expand Down
1 change: 1 addition & 0 deletions mypy/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -1231,6 +1231,7 @@ def add_invertible_flag(

misc_group = parser.add_argument_group(title="Miscellaneous")
misc_group.add_argument("--quickstart-file", help=argparse.SUPPRESS)
misc_group.add_argument("--changed-since-cache-file", help=argparse.SUPPRESS)
misc_group.add_argument(
"--junit-xml",
metavar="JUNIT_XML_OUTPUT_FILE",
Expand Down
6 changes: 6 additions & 0 deletions mypy/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,12 @@ def __init__(self) -> None:
# source hashes as often.
self.quickstart_file: str | None = None

# A file naming, one path per line, every file that may differ from those
# the fine-grained cache was generated from. With --use-fine-grained-cache,
# every other file is trusted to match its cache metadata without being
# looked at, so the first build costs what changed rather than what exists.
self.changed_since_cache_file: str | None = None

# A comma-separated list of files/directories for mypy to type check;
# supports globbing
self.files: list[str] | None = None
Expand Down
37 changes: 36 additions & 1 deletion mypy/test/test_find_sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import pytest

from mypy.find_sources import InvalidSourceList, SourceFinder, create_source_list
from mypy.find_sources import InvalidSourceList, SourceFinder, base_dir_of, create_source_list
from mypy.fscache import FileSystemCache
from mypy.modulefinder import BuildSource
from mypy.options import Options
Expand Down Expand Up @@ -374,3 +374,38 @@ def test_find_sources_exclude(self) -> None:
}
fscache = FakeFSCache(files)
assert len(find_sources(["."], options, fscache)) == len(files)

def test_base_dir_of_matches_crawl_up(self) -> None:
files = {
"/a/pkg/__init__.py",
"/a/pkg/sub/__init__.py",
"/a/pkg/sub/mod.py",
"/a/pkg/mod.pyi",
"/a/top.py",
"/a/foo-stubs/__init__.pyi",
"/a/foo-stubs/bar.pyi",
}
finder = SourceFinder(FakeFSCache(files), Options())
for f in files:
module, base_dir = finder.crawl_up(f)
assert base_dir_of(f, module) == base_dir, f

def test_base_dir_of_mismatch(self) -> None:
assert base_dir_of("/a/pkg/mod.py", "pkg.other") is None
assert base_dir_of("/a/pkg/mod.py", "other.mod") is None
assert base_dir_of("/a/pkg/__init__.py", "other") is None

def test_known_modules_need_no_crawl(self) -> None:
class NoFS(FakeFSCache):
def isfile(self, path: str) -> bool:
raise AssertionError(f"looked at {path}")

sources = create_source_list(
["/a/pkg/sub/mod.py"],
Options(),
NoFS(set()),
known_modules={"/a/pkg/sub/mod.py": "pkg.sub.mod"},
)
assert [(s.module, normalise_path(s.base_dir or "")) for s in sources] == [
("pkg.sub.mod", "/a")
]
49 changes: 49 additions & 0 deletions test-data/unit/daemon.test
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,55 @@ $ {python} -c "import sys; sys.stdout.write(open('log').read())"
-- whether the source file got rehashed, which we don't want it to have been.
$ {python} -c "x = open('.mypy_cache/3.11/bar.meta.ff', 'rb').read(); y = open('asdf.ff', 'rb').read(); assert x == y"

-- Only files the changed-since-cache file names are looked at: bar changes too, but is
-- trusted to match the cache until a recheck names it. baz has no cache, so it is checked.
[case testDaemonChangedSinceCache]
$ {python} -c "print('x = 1')" >foo.py
$ {python} -c "print('y = 1')" >bar.py
$ mypy --local-partial-types --cache-fine-grained --follow-imports=error --no-sqlite-cache --python-version=3.11 -- foo.py bar.py
Success: no issues found in 2 source files
$ {python} -c "print('x = lol')" >foo.py
$ {python} -c "print('y = lol')" >bar.py
$ {python} -c "print('z = lol')" >baz.py
$ {python} -c "print('foo.py')" >changed.txt
$ dmypy run -- foo.py bar.py baz.py --follow-imports=error --use-fine-grained-cache --no-sqlite-cache --python-version=3.11 --changed-since-cache-file changed.txt
Daemon started
baz.py:1: error: Name "lol" is not defined
foo.py:1: error: Name "lol" is not defined
Found 2 errors in 2 files (checked 3 source files)
== Return code: 1
$ dmypy recheck --update bar.py
baz.py:1: error: Name "lol" is not defined
foo.py:1: error: Name "lol" is not defined
bar.py:1: error: Name "lol" is not defined
Found 3 errors in 3 files (checked 3 source files)
== Return code: 1
$ dmypy stop
Daemon stopped

-- The first check takes the modules of files it trusts the cache for from the cache, but not
-- of those under an __init__ that changed: without pkg/__init__.py, pkg/a.py is module a.
[case testDaemonChangedSinceCacheModules]
$ {python} -c "import os; os.makedirs('pkg'); os.makedirs('other')"
$ {python} -c "print('')" >pkg/__init__.py
$ {python} -c "print('x = 1')" >pkg/a.py
$ {python} -c "print('')" >other/__init__.py
$ {python} -c "print('y = 1')" >other/b.py
$ {python} -c "print('from pkg.a import x; from other.b import y')" >c.py
$ mypy --local-partial-types --cache-fine-grained --follow-imports=error --no-sqlite-cache --python-version=3.11 -- pkg/__init__.py pkg/a.py other/__init__.py other/b.py c.py
Success: no issues found in 5 source files
$ {python} -c "import os; os.remove('pkg/__init__.py')"
$ {python} -c "print('pkg/__init__.py')" >changed.txt
$ dmypy start -- --local-partial-types --follow-imports=error --use-fine-grained-cache --no-sqlite-cache --python-version=3.11 --changed-since-cache-file changed.txt
Daemon started
$ dmypy check pkg/a.py other/__init__.py other/b.py c.py
c.py:1: error: Import of "pkg.a" ignored [misc]
c.py:1: note: (Using --follow-imports=error, module not passed on command line)
Found 1 error in 1 file (checked 4 source files)
== Return code: 1
$ dmypy stop
Daemon stopped

[case testDaemonSuggest]
$ dmypy start --log-file log.txt -- --follow-imports=error --no-error-summary
Daemon started
Expand Down
Loading