Merge pull request #136 from codellm-devkit/feat/issue-128-structured-decorators

feat(schema)!: structured decorators on callables and classes

Author: rahlk

codeanalyzer/neo4j/project.py

@@ -47,7 +47,7 @@
     PyModule,
     PyVariableDeclaration,
 )
-from codeanalyzer.schema.py_schema import PyCallsite
+from codeanalyzer.schema.py_schema import PyCallsite, PyDecorator
 
 
 def project(app: PyApplication, app_name: str, sig_to_id: dict,
@@ -369,6 +369,9 @@ def _project_class(
     )
     b.edge(parent_rel, parent, ref)
 
+    for d in cl.decorators or []:
+        _project_decorator(b, ref, d)
+
     for base in cl.base_classes or []:
         if base:
             b.edge_to_symbol("PY_EXTENDS", ref, _symbol_ref(base, externals, sig_to_id))
@@ -439,9 +442,36 @@ def _project_variable(
     b.edge("PY_DECLARES_VAR", owner, ref)
 
 
-def _project_decorator(b: RowBuilder, on: NodeRef, decorator: str) -> None:
-    dec = b.node(["PyDecorator"], "name", decorator, {"name": decorator})
-    b.edge("PY_DECORATED_BY", on, dec)
+def _project_decorator(b: RowBuilder, on: NodeRef, decorator: PyDecorator) -> None:
+    """Project one decorator application (#128).
+
+    The merge key is the resolved ``qualified_name`` when Jedi supplies one, so
+    ``@lru_cache`` and ``@lru_cache(maxsize=128)`` land on one node instead of two,
+    and two spellings of one decorator stop being separate nodes. Unresolved
+    decorators fall back to the written spelling. Per-application facts (the
+    arguments) ride on the relationship, not the shared node -- ``:PyDecorator``
+    has no ``_module`` and is never pruned, so anything application-specific on it
+    would accumulate across every project in the database.
+    """
+    key = decorator.qualified_name or decorator.name
+    dec = b.node(
+        ["PyDecorator"],
+        "name",
+        key,
+        {"name": key, "qualified_name": decorator.qualified_name or ""},
+    )
+    b.edge(
+        "PY_DECORATED_BY",
+        on,
+        dec,
+        {
+            "expression": decorator.expression or "",
+            "positional_arguments": list(decorator.positional_arguments or []),
+            "keyword_arguments_json": json.dumps(
+                dict(decorator.keyword_arguments or {}), sort_keys=True
+            ),
+        },
+    )
 
 
 # ----------------------------------------------------------------------------------------------
@@ -482,6 +512,7 @@ def _class_props(cl: PyClass, file_key: str, source: str) -> Props:
             "name": cl.name,
             "code": _span_code(source, cl.span),
             "base_classes": list(cl.base_classes or []),
+            "decorators": [d.qualified_name or d.name for d in (cl.decorators or [])],
             "docstring": _docstring_of(cl.comments),
             "start_line": cl.start_line,
             "end_line": cl.end_line,
@@ -504,7 +535,7 @@ def _callable_props(c: PyCallable, file_key: str, source: str) -> Props:
             "start_line": c.start_line,
             "end_line": c.end_line,
             "docstring": _docstring_of(c.comments),
-            "decorators": list(c.decorators or []),
+            "decorators": [d.qualified_name or d.name for d in (c.decorators or [])],
             "parameters_json": _stringify_if(c.parameters),
             "accessed_symbols_json": _stringify_if(c.accessed_symbols),
             "_module": file_key,

codeanalyzer/neo4j/schema.py

@@ -101,6 +101,7 @@ class RelType:
             "name": "string",
             "code": "string",
             "base_classes": "string[]",
+            "decorators": "string[]",
             "docstring": "string",
             **_SPAN,
             "_module": "string",
@@ -138,7 +139,7 @@ class RelType:
         "PyDecorator",
         "PyDecorator",
         "name",
-        {"name": "string"},
+        {"name": "string", "qualified_name": "string"},
     ),
     NodeLabel(
         "PyCallSite",
@@ -234,7 +235,16 @@ class RelType:
         ["PyModule", "PyPackage"],
         {"spellings": "string[]", "imported_names": "string[]", "aliases": "string[]"},
     ),
-    RelType("PY_DECORATED_BY", ["PyCallable"], ["PyDecorator"]),
+    RelType(
+        "PY_DECORATED_BY",
+        ["PyCallable", "PyClass"],
+        ["PyDecorator"],
+        {
+            "expression": "string",
+            "positional_arguments": "string[]",
+            "keyword_arguments_json": "string",
+        },
+    ),
     # Level-3 CPG overlay (-a 3 only): the cross-language dataflow vocabulary,
     # PY_-namespaced so per-language SDK backends can scope their queries.
     RelType("PY_HAS_CFG_NODE", ["PyCallable"], ["PyCFGNode"]),

codeanalyzer/schema/py_schema.py

@@ -217,13 +217,32 @@ class PyVariableDeclaration(BaseModel):
     end_column: int = -1
 
 
+@builder
+class PyDecorator(BaseModel):
+    """One decorator application, structured rather than a source string (#128).
+
+    ``name`` is the spelling as written (``lru_cache``, ``builtins.staticmethod``);
+    ``qualified_name`` is Jedi's resolution of it (``functools.lru_cache``) and is
+    absent when it cannot be resolved. ``expression`` keeps the full unparsed source
+    so nothing is lost for decorators too complex to decompose.
+    """
+
+    name: str
+    qualified_name: Optional[str] = None
+    positional_arguments: List[str] = []
+    keyword_arguments: Dict[str, str] = {}
+    expression: str = ""
+    span: Optional[Span] = None
+
+
 @builder
 class PyCallableParameter(BaseModel):
     """Represents a parameter of a Python callable (function/method)."""
 
     name: str
     type: Optional[str] = None
     default_value: Optional[str] = None
+    decorators: List[PyDecorator] = []
     start_line: int = -1
     end_line: int = -1
     start_column: int = -1
@@ -271,7 +290,7 @@ class PyCallable(BaseModel):
     kind: str = "function"
     span: Optional[Span] = None
     comments: List[PyComment] = []
-    decorators: List[str] = []
+    decorators: List[PyDecorator] = []
     parameters: List[PyCallableParameter] = []
     return_type: Optional[str] = None
     start_line: int = -1
@@ -304,6 +323,7 @@ class PyClassAttribute(BaseModel):
     type: Optional[str] = None
     initializer: Optional[str] = None
     comments: List[PyComment] = []
+    decorators: List[PyDecorator] = []
     start_line: int = -1
     end_line: int = -1
 
@@ -319,6 +339,7 @@ class PyClass(BaseModel):
     span: Optional[Span] = None
     comments: List[PyComment] = []
     base_classes: List[str] = []
+    decorators: List[PyDecorator] = []
     callables: Dict[str, PyCallable] = {}  # methods, keystone containment name
     attributes: Dict[str, PyClassAttribute] = {}
     types: Dict[str, "PyClass"] = {}  # inner classes, keystone containment name

codeanalyzer/syntactic_analysis/symbol_table_builder.py

@@ -16,6 +16,7 @@
     PyCallableParameter,
     PyCallArgument,
     PyCallsite,
+    PyDecorator,
     PyClass,
     PyClassAttribute,
     PyComment,
@@ -295,6 +296,7 @@ def _add_class(self, node: AST, script: Script, source: str, prefix: str = "") -
                 .name(class_name)
                 .signature(signature)
                 .span(span)
+                .decorators(self._decorators(child, script, source))
                 .start_line(start_line)
                 .end_line(end_line)
                 .comments(self._pycomments(child, code))
@@ -331,7 +333,7 @@ def _callables(self, node: AST, script: Script, source: str, prefix: str = "") -
                                        getattr(child, "end_lineno", child.lineno),
                                        getattr(child, "end_col_offset", child.col_offset)),
                 )
-                decorators = [ast.unparse(d) for d in child.decorator_list]
+                decorators = self._decorators(child, script, source)
 
                 if prefix:
                     # We're in a nested context - build signature with prefix
@@ -590,6 +592,68 @@ def build_param(
 
         return params
 
+    def _decorators(
+        self, node: ast.AST, script: Optional[Script], source: str = ""
+    ) -> List[PyDecorator]:
+        """Structure each entry of ``node.decorator_list`` (#128).
+
+        ``name`` is the spelling as written and ``qualified_name`` is Jedi's
+        resolution of it, inferred at the last identifier of the callee so that
+        ``@a.b.c`` resolves ``c`` rather than ``a``. Resolution is best-effort:
+        dynamic, conditional and re-exported decorators stay unresolved, and a
+        failure here must never abort the symbol table.
+        """
+        out: List[PyDecorator] = []
+        for dec in getattr(node, "decorator_list", []) or []:
+            callee = dec.func if isinstance(dec, ast.Call) else dec
+            positional: List[str] = []
+            keyword: Dict[str, str] = {}
+            if isinstance(dec, ast.Call):
+                positional = [ast.unparse(a) for a in dec.args]
+                for kw in dec.keywords:
+                    # ``**kwargs`` has no arg name; keep it addressable rather
+                    # than dropping it.
+                    key = kw.arg if kw.arg is not None else f"**{ast.unparse(kw.value)}"
+                    keyword[key] = ast.unparse(kw.value)
+            span = Span(
+                start=(dec.lineno, dec.col_offset),
+                end=(getattr(dec, "end_lineno", dec.lineno),
+                     getattr(dec, "end_col_offset", dec.col_offset)),
+                bytes=byte_offsets(source, dec.lineno, dec.col_offset,
+                                   getattr(dec, "end_lineno", dec.lineno),
+                                   getattr(dec, "end_col_offset", dec.col_offset)),
+            ) if source else None
+            out.append(
+                PyDecorator.builder()
+                .name(ast.unparse(callee))
+                .qualified_name(self._decorator_qualified_name(callee, script))
+                .positional_arguments(positional)
+                .keyword_arguments(keyword)
+                .expression(ast.unparse(dec))
+                .span(span)
+                .build()
+            )
+        return out
+
+    @staticmethod
+    def _decorator_qualified_name(
+        callee: ast.AST, script: Optional[Script]
+    ) -> Optional[str]:
+        """Jedi's full name for a decorator's callee, or ``None``."""
+        if script is None:
+            return None
+        line = getattr(callee, "end_lineno", getattr(callee, "lineno", None))
+        col = getattr(callee, "end_col_offset", None)
+        if line is None or col is None:
+            return None
+        try:
+            d = SymbolTableBuilder._first_definition(
+                script.infer(line=line, column=max(col - 1, 0))
+            )
+        except Exception:
+            return None
+        return getattr(d, "full_name", None) if d is not None else None
+
     def _accessed_symbols(
         self, fn_node: ast.FunctionDef, script: Script
     ) -> List[PySymbol]:

schema.neo4j.json

@@ -41,6 +41,7 @@
         "name": "string",
         "code": "string",
         "base_classes": "string[]",
+        "decorators": "string[]",
         "docstring": "string",
         "start_line": "integer",
         "end_line": "integer",
@@ -92,7 +93,8 @@
       "merge_label": "PyDecorator",
       "key": "name",
       "properties": {
-        "name": "string"
+        "name": "string",
+        "qualified_name": "string"
       }
     },
     {
@@ -280,12 +282,17 @@
     {
       "type": "PY_DECORATED_BY",
       "from": [
-        "PyCallable"
+        "PyCallable",
+        "PyClass"
       ],
       "to": [
         "PyDecorator"
       ],
-      "properties": {}
+      "properties": {
+        "expression": "string",
+        "positional_arguments": "string[]",
+        "keyword_arguments_json": "string"
+      }
     },
     {
       "type": "PY_HAS_CFG_NODE",