Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
252 changes: 252 additions & 0 deletions tests/integration/charm/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,16 @@
import json
import logging
import random
import subprocess
import tempfile
import time
import tomllib
import zipfile
from datetime import datetime
from enum import Enum

import jubilant
import psycopg2
import requests
import yaml
from lightkube import Client
Expand All @@ -30,15 +33,20 @@
from tests.integration.helpers.juju import (
get_app_leader,
get_app_units,
get_credentials,
get_secret_by_label,
get_unit_address,
wait_for_apps_status,
)
from tests.integration.helpers.postgresql import db_connect, execute_query
from tests.integration.types import Substrate

logger = logging.getLogger(__name__)

DATA_INTEGRATOR_APP_NAME = "data-integrator"

RELATION_ENDPOINT = "postgresql"

MINUTE_SECS = 60

# The resume-refresh action is a one-shot charm_refresh state-machine probe: it fails while
Expand All @@ -57,6 +65,40 @@ class RoleAttributeValue(Enum):
ALL_DATABASES = 3


def consumer_alias_exists(juju: jubilant.Juju, app_name: str, label: str) -> bool:
"""Whether *app*'s leader holds a consumer-side alias for *label*.

A consumed-secret alias is not listed by ``secret-ids`` (which returns only
owned secrets), so ``get_async_secret_labels`` can't see it. Probe it directly:
``secret-get --label`` returns content when the alias exists and errors with
``consumer label "<label>" not found`` when it was never registered.
"""
leader = get_app_leader(juju, app_name)
try:
juju.cli("exec", "--unit", leader, "--", "secret-get", f"--label={label}")
return True
except jubilant.CLIError as error:
haystack = f"{error} {getattr(error, 'stderr', '')} {getattr(error, 'stdout', '')}".lower()
if "not found" in haystack:
return False
raise


def get_async_secret_labels(juju: jubilant.Juju, app_name: str) -> set[str]:
"""Return the async-replication secret labels owned by *app*'s leader unit."""
leader = get_app_leader(juju, app_name)
labels: set[str] = set()
for secret_id in juju.cli("exec", "--unit", leader, "--", "secret-ids").split():
info = juju.cli("exec", "--unit", leader, "--", "secret-info-get", secret_id)
for line in info.splitlines():
stripped = line.strip()
if stripped.startswith("label:"):
label = stripped.split(":", 1)[1].strip()
if "async-replication" in label:
labels.add(label)
return labels


def get_password(juju: jubilant.Juju, app_name: str, user: str = "operator") -> str | None:
"""Return the password of a PostgreSQL system user from the charm's app secret."""
secret = get_secret_by_label(juju, f"{PEER_RELATION}.{app_name}.app")
Expand Down Expand Up @@ -538,6 +580,15 @@ def get_db_max_written_value(
connection.close()


def start_continuous_writes(model: jubilant.Juju, test_app: str) -> None:
"""Start continuous writes through the test application (retry through transient errors)."""
for attempt in Retrying(stop=stop_after_attempt(10), reraise=True):
with attempt:
model.run(
unit=get_app_leader(model, test_app), action="start-continuous-writes"
).raise_on_failure()


def check_db_units_writes_increment(
juju: jubilant.Juju,
app_name: str,
Expand Down Expand Up @@ -836,3 +887,204 @@ def roles_attributes(predefined_roles: dict, combination: str) -> dict:
"write-data": write_data,
},
}


def get_lxd_machine_name(status, unit_name: str, app_name: str) -> str:
"""Get the LXD machine/container name for a given unit.

Args:
status: Juju status object
unit_name: Full unit name (e.g., "postgresql/0")
app_name: name of the app

Returns:
LXD machine/container name (instance_id)
"""
unit_info = status.get_units(app_name).get(unit_name)
if not unit_info:
raise RuntimeError(f"Unable to find unit {unit_name} in status")

machine_id = getattr(unit_info, "machine", None)
if not machine_id:
raise RuntimeError(f"Unable to find machine ID for unit {unit_name}")

machine_obj = (
getattr(status, "machines", {}).get(machine_id) if hasattr(status, "machines") else None
)
if not machine_obj:
raise RuntimeError(f"Unable to find machine object for machine {machine_id}")

machine_name = getattr(machine_obj, "instance_id", None)
if not machine_name:
raise RuntimeError(f"Unable to find instance_id for machine {machine_id}")

return machine_name


def verify_leader_active(status, unit_name: str, app_name: str) -> None:
"""Verify that a unit is the active leader.

Args:
status: Juju status object
unit_name: Full unit name to verify
app_name: name of the app

Raises:
AssertionError if unit is not active leader
"""
unit = status.get_units(app_name).get(unit_name)
assert unit is not None, f"Unit {unit_name} not found in status"
assert unit.leader, f"Unit {unit_name} is not the leader"
assert unit.workload_status.current == "active", (
f"Unit {unit_name} is not active. "
f"Status: {unit.workload_status.current}, "
f"Message: {unit.workload_status.message}"
)


def force_leader_election(juju: jubilant.Juju, original_leader: str, app_name: str) -> str:
"""Force a leader election by stopping the current leader's juju agent.

Args:
juju: Juju client
original_leader: Current leader unit name
app_name: name of the app

Returns:
Name of the newly elected leader unit

Raises:
RuntimeError: If no new leader is elected within timeout
"""
logger.info(f"Stopping juju agent on {original_leader} to force leader election")
status = juju.status()
machine_name = get_lxd_machine_name(status, original_leader, app_name)

# Get the machine ID from the status
unit_info = status.get_units(app_name).get(original_leader)
machine_id = getattr(unit_info, "machine", None)
if not machine_id:
raise RuntimeError(f"Unable to find machine ID for unit {original_leader}")

jujud_service = f"jujud-machine-{machine_id}"
logger.info(f"Stopping {jujud_service} service")

subprocess.check_call([
"lxc",
"exec",
machine_name,
"--",
"systemctl",
"stop",
jujud_service,
])

# Allow time for agent shutdown to propagate before polling
logger.info("Waiting for agent shutdown to propagate")
time.sleep(5)

# Wait for a new leader to be elected
logger.info("Waiting for new leader election")
new_leader = None
for _ in range(60): # Wait up to 60 seconds
time.sleep(1)
try:
status = juju.status()
for unit_name, unit_status in status.get_units(app_name).items():
if unit_status.leader and unit_name != original_leader:
new_leader = unit_name
break
if new_leader:
break
except Exception as e:
logger.debug(f"Error checking leader status: {e}")
continue

if new_leader is None:
# Restart the original leader's agent in case election failed
logger.info(f"Restarting {jujud_service} service after failed election")
subprocess.check_call([
"lxc",
"exec",
machine_name,
"--",
"systemctl",
"start",
jujud_service,
])
raise RuntimeError("No new leader elected within timeout")

logger.info(f"New leader elected: {new_leader}")

# Restart the original leader's agent so the cluster is healthy
logger.info(f"Restarting {jujud_service} service on {original_leader}")
subprocess.check_call([
"lxc",
"exec",
machine_name,
"--",
"systemctl",
"start",
jujud_service,
])

return new_leader


def verify_temp_table_creation(juju: jubilant.Juju) -> None:
"""Test that temporary tables can be created successfully."""
creds = get_credentials(juju, f"{DATA_INTEGRATOR_APP_NAME}/0")
uri = creds[RELATION_ENDPOINT]["uris"]

connection = None
try:
connection = psycopg2.connect(uri)
connection.autocommit = True
with connection.cursor() as cur:
cur.execute("CREATE TEMPORARY TABLE test (lines TEXT);")
logger.info("Successfully created temporary table")
finally:
if connection is not None:
connection.close()


def check_for_fix_log_message(juju: jubilant.Juju, unit_name: str) -> bool:
"""Check if the library fix log message appears in the unit's logs.

Args:
juju: Juju client
unit_name: Unit name to check logs for

Returns:
True if the log message was found, False otherwise
"""
logger.info("Checking debug logs for the library fix log message")
result = subprocess.run(
[
"juju",
"debug-log",
"--replay",
"--no-tail",
"-m",
juju.model,
"--include",
unit_name,
],
capture_output=True,
text=True,
)

expected_message = (
"Fixed permissions on temp tablespace directory at /var/snap/charmed-postgresql/common/data/temp/16/main "
"(persistent storage), existing tablespace remains valid"
)

if expected_message in result.stdout:
logger.info(f"✓ Found expected log message in {unit_name} logs")
return True

logger.warning(
f"Expected log message not found in {unit_name} logs. "
"This may indicate the code path was not triggered or permissions were already correct."
)
return False
Loading
Loading