diff --git a/docs/opentofu-migration.md b/docs/opentofu-migration.md index f7fe5c5..505147c 100644 --- a/docs/opentofu-migration.md +++ b/docs/opentofu-migration.md @@ -162,6 +162,27 @@ and deleting `campuses.tf` would plan the destroy of every campus. So `ct export tf --env prod --only group --no-versions` can run against the real `tofu/` directory: it writes `groups.tf` and `imports.tf` and touches nothing else. +### A re-keyed resource gets a `moved` block + +When a resource keeps its id but gets a new key, its HCL address changes. The +export compares each id with the label the id map recorded for it, and when they +differ it writes a `moved` block at the top of `imports.tf`, next to the import +for the new address. Without it, `tofu plan` would propose destroying the old +address, or fail on `prevent_destroy`. tofu moves the resource first, and the +import of the same id then does nothing. If the old address never reached tofu's +state, the move does nothing and the import brings the resource in. + +The id map is the only record of the old label. An export that writes the map +(the default) records the new label, so a **second** export loses the `moved` +block. Apply `imports.tf` on each host before re-exporting, or export with +`--no-ids` and refresh the map from tofu afterwards (`ct ids sync`). + +No `moved` block is written when the old address is still in use in the same +export, because two keys swapped or a new resource took over the old key. tofu +rejects such a move, and leaving it out would update the wrong object in place. +The export warns instead (`EXPORT_MOVE_CONFLICT`). Move the state by hand with +`tofu state mv` before applying. + ## `ct auth token`: a credential helper (#179) The provider needs credentials, and its `token` attribute wants a ChurchTools diff --git a/src/application/operations/export-tf.ts b/src/application/operations/export-tf.ts index 72185ab..ba0322d 100644 --- a/src/application/operations/export-tf.ts +++ b/src/application/operations/export-tf.ts @@ -2,7 +2,7 @@ import { mkdir, writeFile } from "node:fs/promises"; import { isAbsolute, join } from "node:path"; import { assertLabelsUnique, hclLabel, hclType, REFERENCE_FIELDS, renderResource } from "../../export/hcl.js"; -import { renderImports, type ImportTarget } from "../../export/imports.js"; +import { renderImports, renderMoved, type ImportTarget, type MovedTarget } from "../../export/imports.js"; import { EXPORTABLE_TYPES, fileForType } from "../../export/layout.js"; import { renderVersions } from "../../export/provider.js"; import { CATALOG_DIR } from "../../permissions/catalog-store.js"; @@ -40,6 +40,8 @@ export interface ExportTfRequest extends ProjectRequest { export interface ExportTfValue { files: string[]; imported: number; + /** Resources whose id the id map knows under another label, written as `moved` blocks (#199). */ + moved: MovedTarget[]; /** Keys that could not be used verbatim as an HCL identifier. */ relabelled: { key: string; label: string }[]; /** Managed types the provider has no mapping for yet, with how many were skipped. */ @@ -102,33 +104,41 @@ async function carryOverUncoveredIds( * resource it points at. * * ct's own state first, then the committed id map — which, after a tier's cutover, is the only - * place ct can still learn that campus 6 is `churchtools_campus.nord` on this host. The map is - * loaded only when state cannot answer, so an export that needs no reference never reads it (and a - * corrupt map fails only the export that depends on it). + * place ct can still learn that campus 6 is `churchtools_campus.nord` on this host. * * A miss is collected, not guessed at: falling back to the number would export an HCL file that is * right on this host and silently wrong on the other one. */ -function referenceResolver(rows: readonly ManagedResource[], loadMap: () => Promise) { +function referenceResolver( + rows: readonly ManagedResource[], + labelsFromMap: () => Promise>, +) { const fromState = new Map(); for (const r of rows) fromState.set(`${r.type}\u0000${r.id}`, r.key); - let map: Promise> | null = null; - const fromMap = () => - (map ??= loadMap().then((loaded) => { - const byId = new Map(); - // The label the map recorded is the address tofu actually has; re-deriving it from the key - // would agree today, but only for as long as hclLabel never changes. - for (const e of loaded?.entries ?? []) byId.set(`${e.type}\u0000${e.id}`, e.label ?? hclLabel(e.key)); - return byId; - })); return async (type: string, id: number): Promise => { const fromStateKey = fromState.get(`${type}\u0000${id}`); const label = - fromStateKey !== undefined ? hclLabel(fromStateKey) : (await fromMap()).get(`${type}\u0000${id}`); + fromStateKey !== undefined ? hclLabel(fromStateKey) : (await labelsFromMap()).get(`${type}\u0000${id}`); return label === undefined ? null : `${hclType(type)}.${label}.id`; }; } +/** + * `type\0id` → the HCL label the id map recorded, loaded once and only when asked for. + * + * The label the map recorded is the address tofu actually has; re-deriving it from the key would + * agree today, but only for as long as hclLabel never changes. + */ +function mapLabels(loadMap: () => Promise): () => Promise> { + let labels: Promise> | null = null; + return () => + (labels ??= loadMap().then((loaded) => { + const byId = new Map(); + for (const e of loaded?.entries ?? []) byId.set(`${e.type}\u0000${e.id}`, e.label ?? hclLabel(e.key)); + return byId; + })); +} + export async function runExportTf(request: ExportTfRequest): Promise { // resolveProject gives the env profile's state path AND the host the state // must belong to; loadState asserts that pairing, so an export can never @@ -164,7 +174,8 @@ export async function runExportTf(request: ExportTfRequest): Promise ({ type, count })); const catalogDir = join(project.cwd, CATALOG_DIR); - const resolveReference = referenceResolver(rows, () => loadIdMap(project.host, catalogDir)); + const labelsFromMap = mapLabels(() => loadIdMap(project.host, catalogDir)); + const resolveReference = referenceResolver(rows, labelsFromMap); const unresolved: string[] = []; // A group adopted before campus was a managed field has no `campusId` in state at all. The export // can only write "no campus" for it, and if the live group HAS a campus, `tofu plan` then proposes @@ -175,6 +186,17 @@ export async function runExportTf(request: ExportTfRequest): Promise(); const imports: ImportTarget[] = []; + const moved: MovedTarget[] = []; + const writtenLabels = new Set(); + // The moves only ever add a block, so a map that does not load costs the moves and a warning, not + // the export: `--no-ids` is how a broken map gets rebuilt. A reference that needs the map still + // fails on it, as it always did. + let mapUnreadable: string | null = null; + const previousLabels = () => + labelsFromMap().catch((err: unknown) => { + mapUnreadable = (err as Error).message; + return new Map(); + }); const relabelled: { key: string; label: string }[] = []; const idEntries: IdMapEntry[] = []; @@ -211,6 +233,14 @@ export async function runExportTf(request: ExportTfRequest): Promise(); + for (const m of moved) + sources.set(`${m.type}\u0000${m.from}`, (sources.get(`${m.type}\u0000${m.from}`) ?? 0) + 1); + const blockedMoves = moved.filter( + (m) => writtenLabels.has(`${m.type}\u0000${m.from}`) || sources.get(`${m.type}\u0000${m.from}`)! > 1, + ); + if (blockedMoves.length > 0) { + const blocked = new Set(blockedMoves); + moved.splice(0, moved.length, ...moved.filter((m) => !blocked.has(m))); + } + if (unresolved.length > 0) { throw new Error( `export: ${unresolved.length} reference(s) cannot be written: the id is not numeric, or is neither in ct's state nor in ` + @@ -239,7 +284,13 @@ export async function runExportTf(request: ExportTfRequest): Promise part !== "").join("\n"), + "utf8", + ); written.push("imports.tf"); const writeVersions = request.writeVersions ?? true; if (writeVersions) { @@ -291,6 +342,28 @@ export async function runExportTf(request: ExportTfRequest): Promise 0) { + const pairs = blockedMoves.map((m) => `${hclType(m.type)}.${m.from} -> ${m.to}`); + warnings.push({ + code: "EXPORT_MOVE_CONFLICT", + message: + `${blockedMoves.length} re-keyed resource(s) got no moved block, because the old address is still in ` + + `use in this export: ${pairs.join(", ")}. On a host whose tofu state has the old address, move it by ` + + `hand (\`tofu state mv\`) before applying, or the plan updates the wrong object in place.`, + details: { moves: blockedMoves.map((m) => ({ type: m.type, from: m.from, to: m.to })) }, + }); + } + + if (mapUnreadable !== null) { + warnings.push({ + code: "EXPORT_MOVES_SKIPPED", + message: + `The id map could not be read, so re-keyed resources got no moved block: ${mapUnreadable} ` + + `Refresh it from tofu (\`tofu state pull | ct ids sync --tofu-state -\`) and export again.`, + details: {}, + }); + } + if (idMapKept) { warnings.push({ code: "IDS_KEPT", @@ -305,7 +378,7 @@ export async function runExportTf(request: ExportTfRequest): Promise "${r.label}" (HCL references must be identifiers)`); } + for (const m of value.moved) { + info(`moved "${m.from}" -> "${m.to}" (${m.type} re-keyed; see imports.tf)`); + } if (value.idMapPath) info(`wrote ${relative(process.cwd(), value.idMapPath)}`); info(`${value.imported} resources exported`); // Printed last, after the success line, so the gap is the final thing on diff --git a/src/export/imports.ts b/src/export/imports.ts index bad94a0..c21274b 100644 --- a/src/export/imports.ts +++ b/src/export/imports.ts @@ -21,3 +21,34 @@ export function renderImports(targets: readonly ImportTarget[]): string { ) .join("\n"); } + +export interface MovedTarget { + type: string; + /** The HCL label tofu has the resource under now. */ + from: string; + /** The label this export writes it under. */ + to: string; +} + +/** + * A resource that kept its id but got a new ct key gets a new address. Without a `moved` block the + * old address loses its config and `tofu plan` proposes destroying it (#199). + * + * The import block for the new address stays: if the old address never reached tofu's state (an + * export that was never applied), the move is a no-op and the import is what brings the resource in. + * If it did, tofu moves first and the import of the same id is then a no-op. Both verified with + * OpenTofu 1.12. + */ +export function renderMoved(targets: readonly MovedTarget[]): string { + return targets + .map((t) => + [ + "moved {", + ` from = ${hclType(t.type)}.${t.from}`, + ` to = ${hclType(t.type)}.${t.to}`, + "}", + "", + ].join("\n"), + ) + .join("\n"); +} diff --git a/tests/export/export-tf.test.ts b/tests/export/export-tf.test.ts index 36dd677..125d68f 100644 --- a/tests/export/export-tf.test.ts +++ b/tests/export/export-tf.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { runExportTf } from "../../src/application/operations/export-tf.js"; -import { writeIdMap } from "../../src/resolve/idMap.js"; +import { idMapPath, writeIdMap } from "../../src/resolve/idMap.js"; const HOST = "https://example.church.tools"; @@ -392,3 +392,85 @@ describe("runExportTf — groups", () => { expect(map.group).toEqual({ jugend: { id: 99 } }); }); }); + +/** + * A resource that keeps its id but gets a new key gets a new HCL address. Without a `moved` block, + * tofu plans to destroy the old address (#199). + */ +describe("runExportTf — re-keyed resources", () => { + it("writes a moved block from the label the id map recorded, and keeps the import", async () => { + await exportInto({ jugend: row("group", "jugend", 99, { groupStatusId: 1 }) }); + const result = await exportInto({ jugend_nord: row("group", "jugend_nord", 99, { groupStatusId: 1 }) }); + + expect(result.value.moved).toEqual([{ type: "group", from: "jugend", to: "jugend_nord" }]); + const imports = await readFile(join(dir, "tofu", "imports.tf"), "utf8"); + expect(imports).toBe( + [ + "moved {", + " from = churchtools_group.jugend", + " to = churchtools_group.jugend_nord", + "}", + "", + "import {", + " to = churchtools_group.jugend_nord", + ' id = "99"', + "}", + "", + ].join("\n"), + ); + }); + + it("moves from the recorded label, not one re-derived from the old key", async () => { + await writeIdMap( + HOST, + [{ type: "campus", key: "3_nord", id: 0, label: "standort_nord" }], + join(dir, ".ct"), + ); + const result = await exportInto({ nord: row("campus", "nord", 0) }); + expect(result.value.moved).toEqual([{ type: "campus", from: "standort_nord", to: "nord" }]); + }); + + it("writes no moved block when the label is unchanged or the id is new", async () => { + await exportInto({ a: row("campus", "a", 1) }); + const result = await exportInto({ a: row("campus", "a", 1), b: row("campus", "b", 2) }); + expect(result.value.moved).toEqual([]); + expect(await readFile(join(dir, "tofu", "imports.tf"), "utf8")).not.toContain("moved"); + }); + + it("does not take an id of another type for a move", async () => { + await exportInto({ a: row("campus", "a", 7) }); + const result = await exportInto({ a: row("campus", "a", 7), t: row("group-type", "t", 7) }); + expect(result.value.moved).toEqual([]); + }); + + it("writes no moved block when two keys swapped, and warns instead", async () => { + await exportInto({ x: row("campus", "x", 1), y: row("campus", "y", 2) }); + const result = await exportInto({ y: row("campus", "y", 1), x: row("campus", "x", 2) }); + expect(result.value.moved).toEqual([]); + expect(await readFile(join(dir, "tofu", "imports.tf"), "utf8")).not.toContain("moved"); + expect(result.warnings.map((w) => w.code)).toContain("EXPORT_MOVE_CONFLICT"); + }); + + it("writes no moved block when a new resource took over the old key", async () => { + await exportInto({ a: row("campus", "a", 1) }); + const result = await exportInto({ c: row("campus", "c", 1), a: row("campus", "a", 3) }); + expect(result.value.moved).toEqual([]); + expect(result.warnings.find((w) => w.code === "EXPORT_MOVE_CONFLICT")?.message).toContain( + "churchtools_campus.a -> c", + ); + }); + + it("keeps a --no-ids export working on an unreadable id map, and says the moves were skipped", async () => { + await mkdir(join(dir, ".ct"), { recursive: true }); + await writeFile(idMapPath(HOST, join(dir, ".ct")), "{ not json", "utf8"); + await stateWith({ a: row("campus", "a", 1) }); + const result = await runExportTf({ + cwd: dir, + statePath: "ct-state.json", + outDir: "tofu", + writeIds: false, + }); + expect(result.value.imported).toBe(1); + expect(result.warnings.map((w) => w.code)).toContain("EXPORT_MOVES_SKIPPED"); + }); +});