1- """Config-key flatteners (#152). Pure text-in/records-out, mirroring
1+ """Config-key flatteners (#152, #165 ). Pure text-in/records-out, mirroring
22`artifacts/parsers.py`'s idiom: one dispatcher over per-format internals,
33never raising.
44
55Namespace dispatch: an env-family basename (`.env`, `.env.*`, `.flaskenv`)
66always wins, regardless of the artifact's declared `format`; otherwise the
7- `format` field selects yaml/json/toml/ini/properties. Any other format
8- extracts nothing (not a failure -- there's just nothing to flatten).
9-
10- Span precision differs by shape: env/properties/ini are line-oriented, so
11- the parse itself knows the exact defining line. yaml/json/toml are
12- tree-shaped -- span recovery falls back to a best-effort search for the
13- final dotted-key segment on its own line (see `_find_key_span`), which can
14- bind the wrong line when the same leaf name recurs at another nesting level,
15- or find nothing at all (`span=None`) for a minified/single-line file.
7+ `format` field selects yaml/json/toml/ini/properties/dockerfile. Any other
8+ format extracts nothing (not a failure -- there's just nothing to flatten).
9+
10+ Deployment-env namespaces (#165): a `dockerfile`-format artifact mints TWO
11+ namespaces from one file -- `ENV K=v` directives mint namespace `env` (the
12+ whole point is that `os.environ`/`os.getenv` reads, whose detector rule
13+ prefers namespace `env`, bind to them), `ARG K[=default]` directives mint
14+ namespace `dockerfile` (build-time only, deliberately not bindable by the
15+ env detectors). Both share the bare var name as `key`, so an `ARG X` later
16+ promoted via `ENV X=$X` -- a common idiom -- would collide on `id` (`key`
17+ alone determines it) if both used the plain id shape; the `dockerfile`
18+ mint's id is disambiguated with an internal `arg.` prefix (the `key` FIELD
19+ stays the bare name either way, since nothing about resolution or the
20+ issue's contract cares how the id looks).
21+
22+ A `yaml`-format artifact ALSO gets a supplementary recognition pass after
23+ the normal dotted-path flattening: well-known compose (`services.<name>.
24+ environment` map/list) and k8s (`...env.<idx>.name`/`.value`, matched as a
25+ dotted-path shape at any nesting depth, not schema-anchored) shapes mint
26+ ADDITIONAL namespace-`env` keys keyed on the bare var name, alongside the
27+ normal namespace-`yaml` dotted-path ones -- dual-minting is intentional
28+ (so `os.environ`/`os.getenv` reads bind to compose/k8s-declared vars too),
29+ never deduped away. Ids never collide with the plain yaml mint: the env
30+ mint's key is always the bare var name while the yaml mint's key always
31+ carries the full dotted path, so the two differ by construction for every
32+ shape this pass recognizes. This pass is shape-based, not filename/role
33+ gated -- any yaml artifact whose content happens to match mints the extra
34+ keys, matching this module's general overlay posture (permissive, never a
35+ schema validator).
36+
37+ Span precision differs by shape: env/properties/ini/dockerfile are
38+ line-oriented, so the parse itself knows the exact defining line. yaml/
39+ json/toml are tree-shaped -- span recovery falls back to a best-effort
40+ search for the final dotted-key segment on its own line (see
41+ `_find_key_span`), which can bind the wrong line when the same leaf name
42+ recurs at another nesting level, or find nothing at all (`span=None`) for
43+ a minified/single-line file. The compose/k8s env-recognition mint reuses
44+ this same best-effort search keyed on the bare var name: it finds the
45+ defining line for compose's map/list forms (`KEY:`/`KEY=` is the var's own
46+ line) but not k8s's name/value list shape (the var name is a VALUE on a
47+ `name:` line, not a key label there) -- k8s env-mint spans are `None`,
48+ an accepted extension of the existing best-effort gap.
1649"""
1750from __future__ import annotations
1851
@@ -94,9 +127,46 @@ def _parse_toml(text: str, lines: List[str]) -> List[_Entry]:
94127 return _flatten_structured (tomllib .loads (text ), text , lines )
95128
96129
130+ # --- compose/k8s env recognition (#165): supplemental namespace="env" mint
131+ # over the SAME flattened (dotted_key, value) pairs the "yaml" namespace
132+ # uses -- see the module docstring for why dual-minting is safe and
133+ # intentional. Shape-matched on the dotted path string, not the yaml tree,
134+ # so both recognizers stay simple regex-over-strings, symmetric with the
135+ # rest of this module's line/path-oriented parsing. -------------------------
136+
137+ _K8S_ENV_NAME = re .compile (r'(?:^|\.)env\.\d+\.name$' )
138+ _COMPOSE_ENV = re .compile (r'^services\.[^.]+\.environment\.(.+)$' )
139+
140+
141+ def _recognize_env_shapes (flat : List [Tuple [str , object ]]) -> List [Tuple [str , object ]]:
142+ """`(key, value)` pairs for every compose/k8s env shape found in `flat`
143+ (the same `_flatten(data)` pairs the "yaml" namespace flattens) --
144+ NOT dotted paths, the bare var name, matching `env` namespace semantics
145+ (`config_use.py` resolves that namespace by exact `key ==` match)."""
146+ by_path = dict (flat )
147+ out : List [Tuple [str , object ]] = []
148+ for dotted_key , value in flat :
149+ if _K8S_ENV_NAME .search (dotted_key ):
150+ sibling = dotted_key [: - len ("name" )] + "value" # ...env.<idx>.value
151+ out .append ((_stringify (value ), by_path .get (sibling )))
152+ continue
153+ m = _COMPOSE_ENV .match (dotted_key )
154+ if not m :
155+ continue
156+ tail = m .group (1 )
157+ if _ENV_KEY_NAME .match (tail ): # map form: tail IS the var name
158+ out .append ((tail , value ))
159+ elif tail .isdigit (): # list form: leaf is "KEY=val" or bare "KEY"
160+ key , sep , val = _stringify (value ).partition ("=" )
161+ if _ENV_KEY_NAME .match (key ):
162+ out .append ((key , val if sep else None ))
163+ return out
164+
165+
97166# --- env: KEY=value, `#` comments, `export ` prefix, quote stripping -------
98167
99168_ENV_LINE = re .compile (r'^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$' )
169+ _ENV_KEY_NAME = re .compile (r'^[A-Za-z_][A-Za-z0-9_]*$' ) # shared: dockerfile + compose/k8s recognition
100170
101171
102172def _env_value (raw : str ) -> str :
@@ -133,6 +203,116 @@ def _parse_env(text: str, lines: List[str]) -> List[_Entry]:
133203 return out
134204
135205
206+ # --- dockerfile: `ENV`/`ARG` directives (#165). Line-based, case-insensitive
207+ # instruction keywords (Dockerfile convention is uppercase, the spec itself
208+ # is not case-sensitive); no BuildKit heredoc awareness in v1 -- a heredoc
209+ # body line is just another line that doesn't match `_DOCKER_ENV`/`_DOCKER_ARG`
210+ # and is silently skipped, same as any other unparseable line (overlay
211+ # posture). Multi-stage `FROM ... AS x` scoping is not modeled -- every
212+ # ENV/ARG in the file is scanned regardless of which stage it's in. --------
213+
214+ _DOCKER_ENV = re .compile (r'^ENV\s+(.*)$' , re .IGNORECASE )
215+ _DOCKER_ARG = re .compile (r'^ARG\s+(.*)$' , re .IGNORECASE )
216+
217+
218+ def _join_continuations (lines : List [str ], start_i : int ) -> Tuple [str , int ]:
219+ """From `lines[start_i]`, join any backslash-continued following lines
220+ into one logical instruction line -- same trailing-backslash-drop +
221+ leading/trailing-whitespace-strip join `_parse_properties` already uses
222+ for its own continuations. Returns `(joined_text, index of the LAST
223+ line consumed)`."""
224+ i , n = start_i , len (lines )
225+ parts = [lines [i ].strip ()]
226+ while parts [- 1 ].endswith ("\\ " ) and i + 1 < n :
227+ parts [- 1 ] = parts [- 1 ][:- 1 ] # drop just the continuation backslash
228+ i += 1
229+ parts .append (lines [i ].strip ())
230+ return "" .join (parts ), i
231+
232+
233+ def _split_ws_respecting_quotes (s : str ) -> List [str ]:
234+ """Whitespace-split `s`, except inside a matching `'`/`"` span (a quoted
235+ value may contain spaces) -- quote characters stay IN the returned
236+ tokens, stripped afterward by `_env_value` so there is one quote-
237+ stripping implementation, not two."""
238+ tokens : List [str ] = []
239+ buf : List [str ] = []
240+ quote : Optional [str ] = None
241+ for ch in s :
242+ if quote :
243+ buf .append (ch )
244+ if ch == quote :
245+ quote = None
246+ elif ch in "'\" " :
247+ quote = ch
248+ buf .append (ch )
249+ elif ch .isspace ():
250+ if buf :
251+ tokens .append ("" .join (buf ))
252+ buf = []
253+ else :
254+ buf .append (ch )
255+ if buf :
256+ tokens .append ("" .join (buf ))
257+ return tokens
258+
259+
260+ def _dockerfile_env_entries (text : str , lines : List [str ]) -> List [_Entry ]:
261+ """`ENV` directives -> `(KEY, value, span)`. Handles `ENV K=v`, multi-key
262+ `ENV a=1 b=2`, and the legacy single-key `ENV K v` space form (Docker's
263+ own disambiguation rule: the token right after `ENV` decides the form --
264+ a `=` in it means one-or-more `key=value` pairs; no `=` means the
265+ legacy form, where the key is the first word and the REST of the line,
266+ verbatim, is the value)."""
267+ out : List [_Entry ] = []
268+ i , n = 0 , len (lines )
269+ while i < n :
270+ stripped = lines [i ].strip ()
271+ if not stripped or stripped .startswith ("#" ) or not _DOCKER_ENV .match (stripped ):
272+ i += 1
273+ continue
274+ start_lineno = i + 1
275+ joined , i = _join_continuations (lines , i )
276+ rest = _DOCKER_ENV .match (joined ).group (1 ).strip ()
277+ span = _line_span (text , lines , start_lineno )
278+ first_token = rest .split (None , 1 )[0 ] if rest else ""
279+ if "=" in first_token :
280+ for tok in _split_ws_respecting_quotes (rest ):
281+ key , sep , raw_val = tok .partition ("=" )
282+ if sep and _ENV_KEY_NAME .match (key ):
283+ out .append ((key , _env_value (raw_val ), span ))
284+ else :
285+ key , sep , raw_val = rest .partition (" " )
286+ if sep and _ENV_KEY_NAME .match (key ):
287+ out .append ((key , _env_value (raw_val .strip ()), span ))
288+ i += 1
289+ return out
290+
291+
292+ def _dockerfile_arg_entries (text : str , lines : List [str ]) -> List [_Entry ]:
293+ """`ARG KEY[=default]` -> `(KEY, value, span)`; no `=default` means
294+ `value=None` (#165's ARG semantics -- distinct from `_stringify`'s "" for
295+ a modeled null elsewhere in this module: an ARG default that is simply
296+ ABSENT is not the same fact as a key explicitly set to an empty value)."""
297+ out : List [_Entry ] = []
298+ i , n = 0 , len (lines )
299+ while i < n :
300+ stripped = lines [i ].strip ()
301+ if not stripped or stripped .startswith ("#" ) or not _DOCKER_ARG .match (stripped ):
302+ i += 1
303+ continue
304+ start_lineno = i + 1
305+ joined , i = _join_continuations (lines , i )
306+ rest = _DOCKER_ARG .match (joined ).group (1 ).strip ()
307+ span = _line_span (text , lines , start_lineno )
308+ key , sep , raw_default = rest .partition ("=" )
309+ key = key .strip ()
310+ if _ENV_KEY_NAME .match (key ):
311+ out .append ((key , _env_value (raw_default ) if sep else None , span ))
312+ i += 1
313+ return out
314+
315+
136316# --- properties: key=value / key: value, `\` continuations, `!`/`#` comments
137317
138318_PROPS_KV = re .compile (r'^(?P<key>[^=:\s]+)\s*[:=]\s*(?P<value>.*)$' )
@@ -259,16 +439,50 @@ def _stringify(value: object) -> str:
259439 return str (value )
260440
261441
442+ def _build_keys (
443+ artifact_id : str , namespace : str , entries : List [_Entry ], capture_value : bool ,
444+ * , raw_value : bool = False , id_key : Optional [Callable [[str ], str ]] = None ,
445+ ) -> List [PyConfigKey ]:
446+ """Coalesce `entries` (last dotted-key occurrence in file order wins --
447+ the existing L1 duplicate-key precedent, e.g. a redefined env var) into
448+ `PyConfigKey` records for one namespace.
449+
450+ `raw_value=True` (dockerfile) passes the parsed value straight through
451+ instead of `_stringify`-ing it, so an ARG's absent default surfaces as
452+ `value=None` rather than `_stringify`'s "" for a modeled null -- dockerfile
453+ values are already plain parsed text/`None`, never a yaml/json/toml
454+ bool/None that needs that coercion.
455+
456+ `id_key` remaps `dotted_key` for ID CONSTRUCTION only -- the `.key` FIELD
457+ always stays the bare `dotted_key`. Used solely so a Dockerfile ARG's id
458+ can't collide with an ENV of the same name minting the same bare-name id
459+ in the "env" namespace (`ARG X` then `ENV X=$X` is a common promotion
460+ idiom); every other namespace omits it, preserving today's id shape."""
461+ coalesced : Dict [str , Tuple [object , Optional [Span ]]] = {}
462+ for dotted_key , value , span in entries :
463+ coalesced [dotted_key ] = (value , span )
464+ keys = []
465+ for dotted_key , (value , span ) in coalesced .items ():
466+ text_value = value if raw_value else _stringify (value )
467+ keys .append (PyConfigKey (
468+ id = config_key_id (artifact_id , id_key (dotted_key ) if id_key else dotted_key ),
469+ key = dotted_key , namespace = namespace ,
470+ value = text_value if capture_value else None ,
471+ span = span , references = _find_references (text_value or "" ),
472+ ))
473+ return keys
474+
475+
262476# --- public API -----------------------------------------------------------
263477
264478def is_config_eligible (artifact : PyArtifact ) -> bool :
265479 """Whether `artifact` is worth extracting config keys from: an env-family
266- basename (`.env`/`.env.*`/`.flaskenv`, regardless of declared format), or
267- a namespace-bearing format (yaml/json/toml/ini/properties). A binary
268- artifact is never eligible -- there is no decodable text to flatten, and
269- a rule-matched-but-undecodable file downgrades to `format="binary"`
270- regardless of its basename (see discovery.py), so the binary check wins
271- even over an env-family name.
480+ basename (`.env`/`.env.*`/`.flaskenv`, regardless of declared format), a
481+ `dockerfile`- format artifact (#165), or a namespace-bearing format
482+ (yaml/json/toml/ini/properties). A binary artifact is never eligible --
483+ there is no decodable text to flatten, and a rule-matched-but-undecodable
484+ file downgrades to `format="binary"` regardless of its basename (see
485+ discovery.py), so the binary check wins even over an env-family name.
272486
273487 Callers (core.py's wiring) use this to skip the on-disk read + parse
274488 attempt entirely on artifacts that can never yield config keys, rather
@@ -277,7 +491,11 @@ def is_config_eligible(artifact: PyArtifact) -> bool:
277491 if artifact .format == "binary" :
278492 return False
279493 basename = artifact .path .rsplit ("/" , 1 )[- 1 ]
280- return _is_env_family (basename ) or artifact .format in _NAMESPACE_PARSERS
494+ return (
495+ _is_env_family (basename )
496+ or artifact .format == "dockerfile"
497+ or artifact .format in _NAMESPACE_PARSERS
498+ )
281499
282500
283501def extract_config_keys (
@@ -294,39 +512,46 @@ def extract_config_keys(
294512 `False` only when parsing raised. Never raises: every code path below is
295513 covered by one `try`/`except`, so a malformed file degrades to `([],
296514 False)` instead of an exception escaping to the caller. `keys` is always
297- sorted by `key` (L1 determinism).
515+ sorted by `key` (L1 determinism) -- a dockerfile or dual-minted yaml
516+ artifact concatenates its namespace groups before this one sort, and
517+ Python's sort is stable, so a same-`key` tie across namespaces still
518+ resolves deterministically (env before dockerfile; yaml before env).
298519
299520 `value` is populated only when `capture_value` is True; `key`,
300521 `namespace`, `span`, and `references` are extracted unconditionally
301522 either way (references are recognized in the raw leaf value regardless
302523 of whether that value is exposed)."""
303524 basename = artifact .path .rsplit ("/" , 1 )[- 1 ]
304- if _is_env_family (basename ):
305- namespace , parser = "env" , _parse_env
306- else :
307- parser = _NAMESPACE_PARSERS .get (artifact .format )
308- if parser is None :
309- return [], True
310- namespace = artifact .format
311-
312525 try :
313526 lines = full_text .splitlines ()
314- entries = parser (full_text , lines )
315- # Last occurrence wins on a duplicate dotted key (env/properties can
316- # legally redefine a key later in the file; a repeat would otherwise
317- # collide on `id`, which is derived from `key` alone).
318- coalesced : Dict [str , Tuple [object , Optional [Span ]]] = {}
319- for dotted_key , value , span in entries :
320- coalesced [dotted_key ] = (value , span )
321- keys = [
322- PyConfigKey (
323- id = config_key_id (artifact .id , dotted_key ), key = dotted_key ,
324- namespace = namespace ,
325- value = _stringify (value ) if capture_value else None ,
326- span = span , references = _find_references (_stringify (value )),
527+ if _is_env_family (basename ):
528+ keys = _build_keys (artifact .id , "env" , _parse_env (full_text , lines ), capture_value )
529+ elif artifact .format == "dockerfile" :
530+ keys = _build_keys (
531+ artifact .id , "env" , _dockerfile_env_entries (full_text , lines ), capture_value ,
532+ raw_value = True ,
533+ ) + _build_keys (
534+ artifact .id , "dockerfile" , _dockerfile_arg_entries (full_text , lines ), capture_value ,
535+ raw_value = True , id_key = lambda k : f"arg.{ k } " ,
536+ )
537+ elif artifact .format == "yaml" :
538+ # Two independent parses (like every other format here -- each
539+ # piece parses what IT needs, no shared-state shortcut): `_parse_
540+ # yaml` for the plain dotted-path entries, a second `safe_load`
541+ # for the raw tree the env-shape recognizer walks.
542+ flat = list (_flatten (yaml .safe_load (full_text ) or {}))
543+ env_entries = [
544+ (k , v , _find_key_span (full_text , lines , k )) for k , v in _recognize_env_shapes (flat )
545+ ]
546+ keys = (
547+ _build_keys (artifact .id , "yaml" , _parse_yaml (full_text , lines ), capture_value )
548+ + _build_keys (artifact .id , "env" , env_entries , capture_value )
327549 )
328- for dotted_key , (value , span ) in coalesced .items ()
329- ]
550+ else :
551+ parser = _NAMESPACE_PARSERS .get (artifact .format )
552+ if parser is None :
553+ return [], True
554+ keys = _build_keys (artifact .id , artifact .format , parser (full_text , lines ), capture_value )
330555 keys .sort (key = lambda k : k .key )
331556 return keys , True
332557 except Exception :
0 commit comments