Skip to content

Commit 479a0ce

Browse files
committed
feat(schema)!: structured decorators on callables and classes (#128)
`decorators` was a list of raw `ast.unparse` output on callables only. The callee, its arguments and its location were fused into one opaque string, and `PyClass` had no field at all -- so `@dataclass` was dropped outright. It was not recoverable from anything else either: `ClassDef.lineno` points at the `class` keyword, so decorator lines fall outside `PyClass.span` and `module.source[span.bytes]` cannot reach them. Adds `PyDecorator` -- `name`, `qualified_name`, `positional_arguments`, `keyword_arguments`, `expression`, `span` -- and carries it on `PyCallable` and `PyClass`. `expression` keeps the full unparsed source so decorators too complex to decompose lose nothing. `qualified_name` is plumbing, not new analysis: Jedi already resolved these and the result was being discarded (`accessed_symbols` on a decorated callable already carried `functools.lru_cache`). Resolution infers at the LAST identifier of the callee so `@a.b.c` resolves `c` rather than `a`, and is best-effort -- dynamic and conditional decorators stay `None`, and a failure never aborts the symbol table. Neo4j: `:PyDecorator` merges on the resolved `qualified_name` where there is one, so `@lru_cache` and `@lru_cache(maxsize=128)` stop being two unrelated nodes. Per-application facts (arguments, expression) move onto `PY_DECORATED_BY`, since `:PyDecorator` is project-shared and never pruned -- anything application- specific on the node would accumulate across every project in the database. `PY_DECORATED_BY` now accepts `PyClass` as a start label. `PyClassAttribute` and `PyCallableParameter` get the field for cross-language parity but no plumbing: Python has no decorator syntax for either, so there is nothing to populate and the `[]` default is the whole implementation. No cache-guard change is needed, contrary to what the issue assumed: an old-shape cache fails Pydantic validation (`2 validation errors for PyCallable`) and core.py already catches that and rebuilds. The shape change is self-invalidating. BREAKING CHANGE: `decorators` elements change from `str` to an object. Consumers reading `decorators[0]` as a string must read `.name` or `.qualified_name`. `docs/handoff/` is deliberately untouched -- it is a frozen bundle pinned to 1.0.1.
1 parent 6587099 commit 479a0ce

6 files changed

Lines changed: 258 additions & 12 deletions

File tree

codeanalyzer/neo4j/project.py

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747
PyModule,
4848
PyVariableDeclaration,
4949
)
50-
from codeanalyzer.schema.py_schema import PyCallsite
50+
from codeanalyzer.schema.py_schema import PyCallsite, PyDecorator
5151

5252

5353
def project(app: PyApplication, app_name: str, sig_to_id: dict,
@@ -369,6 +369,9 @@ def _project_class(
369369
)
370370
b.edge(parent_rel, parent, ref)
371371

372+
for d in cl.decorators or []:
373+
_project_decorator(b, ref, d)
374+
372375
for base in cl.base_classes or []:
373376
if base:
374377
b.edge_to_symbol("PY_EXTENDS", ref, _symbol_ref(base, externals, sig_to_id))
@@ -439,9 +442,36 @@ def _project_variable(
439442
b.edge("PY_DECLARES_VAR", owner, ref)
440443

441444

442-
def _project_decorator(b: RowBuilder, on: NodeRef, decorator: str) -> None:
443-
dec = b.node(["PyDecorator"], "name", decorator, {"name": decorator})
444-
b.edge("PY_DECORATED_BY", on, dec)
445+
def _project_decorator(b: RowBuilder, on: NodeRef, decorator: PyDecorator) -> None:
446+
"""Project one decorator application (#128).
447+
448+
The merge key is the resolved ``qualified_name`` when Jedi supplies one, so
449+
``@lru_cache`` and ``@lru_cache(maxsize=128)`` land on one node instead of two,
450+
and two spellings of one decorator stop being separate nodes. Unresolved
451+
decorators fall back to the written spelling. Per-application facts (the
452+
arguments) ride on the relationship, not the shared node -- ``:PyDecorator``
453+
has no ``_module`` and is never pruned, so anything application-specific on it
454+
would accumulate across every project in the database.
455+
"""
456+
key = decorator.qualified_name or decorator.name
457+
dec = b.node(
458+
["PyDecorator"],
459+
"name",
460+
key,
461+
{"name": key, "qualified_name": decorator.qualified_name or ""},
462+
)
463+
b.edge(
464+
"PY_DECORATED_BY",
465+
on,
466+
dec,
467+
{
468+
"expression": decorator.expression or "",
469+
"positional_arguments": list(decorator.positional_arguments or []),
470+
"keyword_arguments_json": json.dumps(
471+
dict(decorator.keyword_arguments or {}), sort_keys=True
472+
),
473+
},
474+
)
445475

446476

447477
# ----------------------------------------------------------------------------------------------
@@ -482,6 +512,7 @@ def _class_props(cl: PyClass, file_key: str, source: str) -> Props:
482512
"name": cl.name,
483513
"code": _span_code(source, cl.span),
484514
"base_classes": list(cl.base_classes or []),
515+
"decorators": [d.qualified_name or d.name for d in (cl.decorators or [])],
485516
"docstring": _docstring_of(cl.comments),
486517
"start_line": cl.start_line,
487518
"end_line": cl.end_line,
@@ -504,7 +535,7 @@ def _callable_props(c: PyCallable, file_key: str, source: str) -> Props:
504535
"start_line": c.start_line,
505536
"end_line": c.end_line,
506537
"docstring": _docstring_of(c.comments),
507-
"decorators": list(c.decorators or []),
538+
"decorators": [d.qualified_name or d.name for d in (c.decorators or [])],
508539
"parameters_json": _stringify_if(c.parameters),
509540
"accessed_symbols_json": _stringify_if(c.accessed_symbols),
510541
"_module": file_key,

codeanalyzer/neo4j/schema.py

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ class RelType:
101101
"name": "string",
102102
"code": "string",
103103
"base_classes": "string[]",
104+
"decorators": "string[]",
104105
"docstring": "string",
105106
**_SPAN,
106107
"_module": "string",
@@ -138,7 +139,7 @@ class RelType:
138139
"PyDecorator",
139140
"PyDecorator",
140141
"name",
141-
{"name": "string"},
142+
{"name": "string", "qualified_name": "string"},
142143
),
143144
NodeLabel(
144145
"PyCallSite",
@@ -234,7 +235,16 @@ class RelType:
234235
["PyModule", "PyPackage"],
235236
{"spellings": "string[]", "imported_names": "string[]", "aliases": "string[]"},
236237
),
237-
RelType("PY_DECORATED_BY", ["PyCallable"], ["PyDecorator"]),
238+
RelType(
239+
"PY_DECORATED_BY",
240+
["PyCallable", "PyClass"],
241+
["PyDecorator"],
242+
{
243+
"expression": "string",
244+
"positional_arguments": "string[]",
245+
"keyword_arguments_json": "string",
246+
},
247+
),
238248
# Level-3 CPG overlay (-a 3 only): the cross-language dataflow vocabulary,
239249
# PY_-namespaced so per-language SDK backends can scope their queries.
240250
RelType("PY_HAS_CFG_NODE", ["PyCallable"], ["PyCFGNode"]),

codeanalyzer/schema/py_schema.py

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,13 +217,32 @@ class PyVariableDeclaration(BaseModel):
217217
end_column: int = -1
218218

219219

220+
@builder
221+
class PyDecorator(BaseModel):
222+
"""One decorator application, structured rather than a source string (#128).
223+
224+
``name`` is the spelling as written (``lru_cache``, ``builtins.staticmethod``);
225+
``qualified_name`` is Jedi's resolution of it (``functools.lru_cache``) and is
226+
absent when it cannot be resolved. ``expression`` keeps the full unparsed source
227+
so nothing is lost for decorators too complex to decompose.
228+
"""
229+
230+
name: str
231+
qualified_name: Optional[str] = None
232+
positional_arguments: List[str] = []
233+
keyword_arguments: Dict[str, str] = {}
234+
expression: str = ""
235+
span: Optional[Span] = None
236+
237+
220238
@builder
221239
class PyCallableParameter(BaseModel):
222240
"""Represents a parameter of a Python callable (function/method)."""
223241

224242
name: str
225243
type: Optional[str] = None
226244
default_value: Optional[str] = None
245+
decorators: List[PyDecorator] = []
227246
start_line: int = -1
228247
end_line: int = -1
229248
start_column: int = -1
@@ -271,7 +290,7 @@ class PyCallable(BaseModel):
271290
kind: str = "function"
272291
span: Optional[Span] = None
273292
comments: List[PyComment] = []
274-
decorators: List[str] = []
293+
decorators: List[PyDecorator] = []
275294
parameters: List[PyCallableParameter] = []
276295
return_type: Optional[str] = None
277296
start_line: int = -1
@@ -304,6 +323,7 @@ class PyClassAttribute(BaseModel):
304323
type: Optional[str] = None
305324
initializer: Optional[str] = None
306325
comments: List[PyComment] = []
326+
decorators: List[PyDecorator] = []
307327
start_line: int = -1
308328
end_line: int = -1
309329

@@ -319,6 +339,7 @@ class PyClass(BaseModel):
319339
span: Optional[Span] = None
320340
comments: List[PyComment] = []
321341
base_classes: List[str] = []
342+
decorators: List[PyDecorator] = []
322343
callables: Dict[str, PyCallable] = {} # methods, keystone containment name
323344
attributes: Dict[str, PyClassAttribute] = {}
324345
types: Dict[str, "PyClass"] = {} # inner classes, keystone containment name

codeanalyzer/syntactic_analysis/symbol_table_builder.py

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
PyCallableParameter,
1717
PyCallArgument,
1818
PyCallsite,
19+
PyDecorator,
1920
PyClass,
2021
PyClassAttribute,
2122
PyComment,
@@ -295,6 +296,7 @@ def _add_class(self, node: AST, script: Script, source: str, prefix: str = "") -
295296
.name(class_name)
296297
.signature(signature)
297298
.span(span)
299+
.decorators(self._decorators(child, script, source))
298300
.start_line(start_line)
299301
.end_line(end_line)
300302
.comments(self._pycomments(child, code))
@@ -331,7 +333,7 @@ def _callables(self, node: AST, script: Script, source: str, prefix: str = "") -
331333
getattr(child, "end_lineno", child.lineno),
332334
getattr(child, "end_col_offset", child.col_offset)),
333335
)
334-
decorators = [ast.unparse(d) for d in child.decorator_list]
336+
decorators = self._decorators(child, script, source)
335337

336338
if prefix:
337339
# We're in a nested context - build signature with prefix
@@ -590,6 +592,68 @@ def build_param(
590592

591593
return params
592594

595+
def _decorators(
596+
self, node: ast.AST, script: Optional[Script], source: str = ""
597+
) -> List[PyDecorator]:
598+
"""Structure each entry of ``node.decorator_list`` (#128).
599+
600+
``name`` is the spelling as written and ``qualified_name`` is Jedi's
601+
resolution of it, inferred at the last identifier of the callee so that
602+
``@a.b.c`` resolves ``c`` rather than ``a``. Resolution is best-effort:
603+
dynamic, conditional and re-exported decorators stay unresolved, and a
604+
failure here must never abort the symbol table.
605+
"""
606+
out: List[PyDecorator] = []
607+
for dec in getattr(node, "decorator_list", []) or []:
608+
callee = dec.func if isinstance(dec, ast.Call) else dec
609+
positional: List[str] = []
610+
keyword: Dict[str, str] = {}
611+
if isinstance(dec, ast.Call):
612+
positional = [ast.unparse(a) for a in dec.args]
613+
for kw in dec.keywords:
614+
# ``**kwargs`` has no arg name; keep it addressable rather
615+
# than dropping it.
616+
key = kw.arg if kw.arg is not None else f"**{ast.unparse(kw.value)}"
617+
keyword[key] = ast.unparse(kw.value)
618+
span = Span(
619+
start=(dec.lineno, dec.col_offset),
620+
end=(getattr(dec, "end_lineno", dec.lineno),
621+
getattr(dec, "end_col_offset", dec.col_offset)),
622+
bytes=byte_offsets(source, dec.lineno, dec.col_offset,
623+
getattr(dec, "end_lineno", dec.lineno),
624+
getattr(dec, "end_col_offset", dec.col_offset)),
625+
) if source else None
626+
out.append(
627+
PyDecorator.builder()
628+
.name(ast.unparse(callee))
629+
.qualified_name(self._decorator_qualified_name(callee, script))
630+
.positional_arguments(positional)
631+
.keyword_arguments(keyword)
632+
.expression(ast.unparse(dec))
633+
.span(span)
634+
.build()
635+
)
636+
return out
637+
638+
@staticmethod
639+
def _decorator_qualified_name(
640+
callee: ast.AST, script: Optional[Script]
641+
) -> Optional[str]:
642+
"""Jedi's full name for a decorator's callee, or ``None``."""
643+
if script is None:
644+
return None
645+
line = getattr(callee, "end_lineno", getattr(callee, "lineno", None))
646+
col = getattr(callee, "end_col_offset", None)
647+
if line is None or col is None:
648+
return None
649+
try:
650+
d = SymbolTableBuilder._first_definition(
651+
script.infer(line=line, column=max(col - 1, 0))
652+
)
653+
except Exception:
654+
return None
655+
return getattr(d, "full_name", None) if d is not None else None
656+
593657
def _accessed_symbols(
594658
self, fn_node: ast.FunctionDef, script: Script
595659
) -> List[PySymbol]:

schema.neo4j.json

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
"name": "string",
4242
"code": "string",
4343
"base_classes": "string[]",
44+
"decorators": "string[]",
4445
"docstring": "string",
4546
"start_line": "integer",
4647
"end_line": "integer",
@@ -92,7 +93,8 @@
9293
"merge_label": "PyDecorator",
9394
"key": "name",
9495
"properties": {
95-
"name": "string"
96+
"name": "string",
97+
"qualified_name": "string"
9698
}
9799
},
98100
{
@@ -280,12 +282,17 @@
280282
{
281283
"type": "PY_DECORATED_BY",
282284
"from": [
283-
"PyCallable"
285+
"PyCallable",
286+
"PyClass"
284287
],
285288
"to": [
286289
"PyDecorator"
287290
],
288-
"properties": {}
291+
"properties": {
292+
"expression": "string",
293+
"positional_arguments": "string[]",
294+
"keyword_arguments_json": "string"
295+
}
289296
},
290297
{
291298
"type": "PY_HAS_CFG_NODE",

0 commit comments

Comments
 (0)