Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .isort.cfg
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[settings]
line_length=1000
known_third_party=requests,ruamel,yaml,pytest,rapidfuzz,opensource,colorama,progressbar,progressbar2
known_third_party=requests,ruamel,pytest,rapidfuzz,opensource,colorama,progressbar,progressbar2
multi_line_output=3
include_trailing_comma=True
force_grid_wrap=0
Expand Down
192 changes: 72 additions & 120 deletions pixi.lock

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ classifiers = [

dependencies = [
"catkin_pkg >=0.4.16",
"ruamel.yaml >=0.16.6,<0.18.0",
"ruamel.yaml >=0.19.1,<0.20.0",
"rosdistro >=0.8.0",
"empy >=3.3.4,<4.0.0",
"requests >=2.24.0",
Expand Down
17 changes: 11 additions & 6 deletions vinca/generate_gha.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import argparse
import glob
import io
import os
import re
import sys
Expand All @@ -9,8 +10,8 @@
from typing import Any

import networkx as nx
import yaml
from rich import print
from ruamel.yaml import YAML

from vinca import config
from vinca.main import get_conda_subdir, read_vinca_yaml
Expand All @@ -22,7 +23,7 @@
get_skip_existing,
)
from vinca.utils import (
NoAliasDumper,
NoAliasRepresenter,
add_test_requirements,
build_requirement_graph,
built_for_distro,
Expand Down Expand Up @@ -136,7 +137,7 @@ def add_additional_recipes(args: Namespace):
additional_recipes = []
for recipe_path in glob.glob(additional_recipes_path + "/**/recipe.yaml"):
with open(recipe_path, encoding="utf-8") as recipe:
additional_recipe = yaml.safe_load(recipe)
additional_recipe = YAML(typ="safe").load(recipe)

name, version, bnumber = (
additional_recipe["package"]["name"],
Expand Down Expand Up @@ -182,8 +183,12 @@ def add_additional_recipes(args: Namespace):


def dump_for_gha(doc, f) -> None:
s = yaml.dump(doc, sort_keys=False, Dumper=NoAliasDumper)
s = s.replace("'on':", "on:")
yaml = YAML(typ="safe")
yaml.Representer = NoAliasRepresenter
yaml.default_flow_style = False
stream = io.StringIO()
yaml.dump(doc, stream)
s = stream.getvalue().replace("'on':", "on:")
with open(f, "w", encoding="utf-8") as fo:
fo.write(s)

Expand Down Expand Up @@ -572,7 +577,7 @@ def main() -> None:
# so all its possible requirements order the stages.
metas.append(
evaluate_target_selectors(
yaml.safe_load(fi.read()), target_platform=args.platform
YAML(typ="safe").load(fi.read()), target_platform=args.platform
)
)

Expand Down
14 changes: 5 additions & 9 deletions vinca/generate_gitlab.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,10 @@
from typing import Any

import networkx as nx
import yaml
from ruamel.yaml import YAML

from vinca.utils import extract_dependency_names

try:
from yaml import CDumper as Dumper
from yaml import CLoader as Loader
except ImportError:
from yaml import Dumper, Loader

# def setup_yaml():
# """ https://stackoverflow.com/a/8661021 """
# represent_dict_order = lambda self, data: self.represent_mapping('tag:yaml.org,2002:map', data.items())
Expand All @@ -36,12 +30,14 @@


def main() -> None:
yaml = YAML(typ="safe")
yaml.default_flow_style = False
metas = []

for f in glob.glob(os.path.join(sys.argv[1], "*.yaml")):
print(f)
with open(f, encoding="utf-8") as fi:
metas.append(yaml.load(fi.read(), Loader=Loader))
metas.append(yaml.load(fi.read()))

requirements = {}

Expand Down Expand Up @@ -103,4 +99,4 @@ def main() -> None:
gitlab_template["stages"] = stage_names

with open(".gitlab-ci.yml", "w", encoding="utf-8") as fo:
fo.write(yaml.dump(gitlab_template, Dumper=Dumper))
yaml.dump(gitlab_template, fo)
2 changes: 1 addition & 1 deletion vinca/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -505,7 +505,7 @@ def parse_package(pkg, distro: Distro, vinca_conf, path):
# parse as yaml
text = test.read_text(encoding="utf-8")
test_content = evaluate_distro_selectors(
ruamel.yaml.safe_load(text), ros_distro=config.ros_distro
ruamel.yaml.YAML(typ="safe").load(text), ros_distro=config.ros_distro
)
recipe["test"] = test_content

Expand Down
3 changes: 1 addition & 2 deletions vinca/migrate.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@

import networkx as nx
import ruamel.yaml
import yaml

from vinca import config
from vinca.distro import Distro
Expand Down Expand Up @@ -213,7 +212,7 @@ def main() -> None:

mfile = os.path.join(args.dir + "/migration.yaml")
with open(mfile, encoding="utf-8") as fi:
migration = yaml.safe_load(fi)
migration = ruamel.yaml.YAML(typ="safe").load(fi)
print(migration)
create_migration_instructions(
args.platform, migration.get("packages", []), args.trigger_branch
Expand Down
4 changes: 2 additions & 2 deletions vinca/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from pathlib import Path
from typing import Any

import yaml
from ruamel.yaml import YAML

from vinca.utils import get_repodata

Expand Down Expand Up @@ -152,7 +152,7 @@ def batch_stages(
timings = replace(timings, recipes=recipes)

with Path(config_path).open(encoding="utf-8") as stream:
vinca_conf = yaml.safe_load(stream) or {}
vinca_conf = YAML(typ="safe").load(stream) or {}
build_individually = set(
vinca_conf.get("build_in_own_stage")
# Accepted for compatibility with configurations written for the Azure generator.
Expand Down
9 changes: 6 additions & 3 deletions vinca/snapshot.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import argparse
import datetime

import yaml
from rosdistro import (
DistributionCache,
get_cached_distribution,
Expand All @@ -10,6 +9,7 @@
get_index_url,
)
from rosdistro.loader import load_url
from ruamel.yaml import YAML

from .distro import Distro

Expand All @@ -26,14 +26,15 @@ def load_distribution_cache(distro_name, rosdistro_ref=None) -> DistributionCach
reused from the published cache.
"""
cache_yaml = get_distribution_cache_string(get_index(get_index_url()), distro_name)
cache = DistributionCache(distro_name, yaml.safe_load(cache_yaml))
yaml = YAML(typ="safe")
cache = DistributionCache(distro_name, yaml.load(cache_yaml))
if not rosdistro_ref:
return cache

index = get_index(f"{ROSDISTRO_RAW_URL}/{rosdistro_ref}/index-v4.yaml")
urls = index.distributions[distro_name]["distribution"]
urls = urls if isinstance(urls, list) else [urls]
data = [yaml.safe_load(load_url(url)) for url in urls]
data = [yaml.load(load_url(url)) for url in urls]
cache.update_distribution(data)

# fetch the package.xml files the update dropped (only changed releases)
Expand Down Expand Up @@ -159,4 +160,6 @@ def main() -> None:
f.write(
f"# Snapshot generated by vinca-snapshot on {utc_time} UTC for distro {args.distro}{source}\n"
)
yaml = YAML(typ="safe")
yaml.default_flow_style = False
yaml.dump(output, f)
2 changes: 1 addition & 1 deletion vinca/template.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ def write_recipe(source, outputs, vinca_conf, distro, single_file: bool = True)
print("Using test: ", test)
text = test.read_text(encoding="utf-8")
test_content = evaluate_distro_selectors(
yaml.safe_load(text), ros_distro=config.ros_distro
yaml.YAML(typ="safe").load(text), ros_distro=config.ros_distro
)
meta["tests"] = list(meta.get("tests") or []) + test_content["tests"]

Expand Down
24 changes: 12 additions & 12 deletions vinca/test_generate_gha.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import sys

import pytest
import yaml
from ruamel.yaml import YAML

from vinca import config, generate_gha
from vinca.generate_gha import (
Expand All @@ -14,6 +14,8 @@
get_stage_name,
)

yaml = YAML(typ="safe")


@pytest.fixture(autouse=True)
def rolling_distro():
Expand Down Expand Up @@ -56,7 +58,7 @@ def test_unix_pipeline_sets_up_pixi(tmp_path):
target="linux-64",
)

workflow = pytest.importorskip("yaml").safe_load(outfile.read_text())
workflow = yaml.load(outfile.read_text())
steps = workflow["jobs"]["stage_0_job_0"]["steps"]
assert steps[1]["name"] == "Disable git auto-maintenance"
assert "maintenance.auto false" in steps[1]["run"]
Expand Down Expand Up @@ -114,7 +116,7 @@ def test_win_pipeline_calls_the_repository_build_script(tmp_path, monkeypatch):

build_win_pipeline([[["ros2-rclcpp"]]], "buildbranch_win", outfile=outfile)

workflow = pytest.importorskip("yaml").safe_load(outfile.read_text())
workflow = yaml.load(outfile.read_text())
steps = workflow["jobs"]["stage_0_job_0"]["steps"]
assert steps[-1]["run"] == "call .scripts\\build_win.bat"
assert steps[-1]["env"]["CURRENT_RECIPES"] == "ros2-rclcpp"
Expand Down Expand Up @@ -145,7 +147,6 @@ def test_recipe_requirements_are_host_and_run_names():


def test_stages_are_derived_from_the_generated_recipes(tmp_path, monkeypatch):
yaml = pytest.importorskip("yaml")
monkeypatch.chdir(tmp_path)
(tmp_path / "vinca.yaml").write_text("{}\n")
recipes = {
Expand All @@ -160,7 +161,7 @@ def test_stages_are_derived_from_the_generated_recipes(tmp_path, monkeypatch):
"package": {"name": name, "version": "1.0.0"},
"requirements": {"host": deps, "run": deps},
}
(tmp_path / "recipes" / name / "recipe.yaml").write_text(yaml.safe_dump(recipe))
yaml.dump(recipe, tmp_path / "recipes" / name / "recipe.yaml")
# no resolution of the ROS distribution: only the recipes are read
monkeypatch.setattr(generate_gha, "load_configuration", lambda: None)
monkeypatch.setattr(
Expand All @@ -172,7 +173,7 @@ def test_stages_are_derived_from_the_generated_recipes(tmp_path, monkeypatch):

generate_gha.main()

jobs = yaml.safe_load((tmp_path / "linux.yml").read_text())["jobs"]
jobs = yaml.load((tmp_path / "linux.yml").read_text())["jobs"]
assert [job["steps"][-1]["env"]["CURRENT_RECIPES"] for job in jobs.values()] == [
"ros2-a",
"ros2-b",
Expand Down Expand Up @@ -212,7 +213,7 @@ def test_stages_follow_the_requirements_of_the_platform(
for name, recipe in recipes.items():
(tmp_path / "recipes" / name).mkdir(parents=True)
recipe = {"package": {"name": name, "version": "1.0.0"}, **recipe}
(tmp_path / "recipes" / name / "recipe.yaml").write_text(yaml.safe_dump(recipe))
yaml.dump(recipe, tmp_path / "recipes" / name / "recipe.yaml")
monkeypatch.setattr(generate_gha, "load_configuration", lambda: None)
monkeypatch.setattr(
sys,
Expand All @@ -222,7 +223,7 @@ def test_stages_follow_the_requirements_of_the_platform(

generate_gha.main()

jobs = yaml.safe_load((tmp_path / outfile).read_text())["jobs"].values()
jobs = yaml.load((tmp_path / outfile).read_text())["jobs"].values()
gmock = next(
job
for job in jobs
Expand All @@ -236,13 +237,12 @@ def test_only_later_stages_declare_needs(tmp_path, monkeypatch):
(tmp_path / ".scripts").mkdir()
(tmp_path / ".scripts" / "build_win.bat").write_text("echo build\n")
stages = [[["ros2-a"], ["ros2-b"]], [["ros2-c"]]]
yaml = pytest.importorskip("yaml")

build_unix_pipeline(stages, "branch", outfile=tmp_path / "linux.yml")
build_win_pipeline(stages, "branch", outfile=tmp_path / "win.yml")

for outfile in ("linux.yml", "win.yml"):
jobs = yaml.safe_load((tmp_path / outfile).read_text())["jobs"]
jobs = yaml.load((tmp_path / outfile).read_text())["jobs"]
# actionlint rejects `needs: []`
assert "needs" not in jobs["stage_0_job_0"]
assert "needs" not in jobs["stage_0_job_1"]
Expand All @@ -269,7 +269,7 @@ def test_keep_file_adds_an_artifact_step(tmp_path, monkeypatch):
[[["ros-rolling-rclcpp"]]], "buildbranch_linux", outfile=str(outfile)
)

step = yaml.safe_load(outfile.read_text())["jobs"]["stage_0_job_0"]["steps"][-1]
step = yaml.load(outfile.read_text())["jobs"]["stage_0_job_0"]["steps"][-1]
assert step["uses"].startswith("actions/upload-artifact@")
assert step["if"] == "always()" and step["with"]["path"] == "work/times.jsonl"

Expand All @@ -284,5 +284,5 @@ def test_no_artifact_step_without_keep_file(tmp_path, monkeypatch):
[[["ros-rolling-rclcpp"]]], "buildbranch_linux", outfile=str(outfile)
)

steps = yaml.safe_load(outfile.read_text())["jobs"]["stage_0_job_0"]["steps"]
steps = yaml.load(outfile.read_text())["jobs"]["stage_0_job_0"]["steps"]
assert not any("upload-artifact" in step.get("uses", "") for step in steps)
6 changes: 4 additions & 2 deletions vinca/test_mutex_build_number.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,14 @@

from unittest.mock import Mock

import yaml
from ruamel.yaml import YAML

from vinca.main import expected_build_number
from vinca.mutex import generate_mutex_package_recipe
from vinca.template import write_recipe

yaml = YAML(typ="safe")


def _conf(**extra):
conf = {
Expand Down Expand Up @@ -49,7 +51,7 @@ def test_mutex_recipe_keeps_its_build_number(tmp_path, monkeypatch):
distro.name = "lyrical"
recipe = generate_mutex_package_recipe(conf, distro)
write_recipe({}, [recipe], conf, distro, single_file=False)
meta = yaml.safe_load(
meta = yaml.load(
(tmp_path / "recipes" / "ros2-distro-mutex" / "recipe.yaml").read_text()
)
assert meta["build"]["number"] == 28
Expand Down
6 changes: 4 additions & 2 deletions vinca/test_snapshot_main.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import sys

import yaml
from ruamel.yaml import YAML

from vinca import snapshot

yaml = YAML(typ="safe")


class FakeDistro:
"""Two released packages; the release repository of `gone` was removed."""
Expand Down Expand Up @@ -40,7 +42,7 @@ def test_snapshot_skips_package_without_package_xml(tmp_path, monkeypatch, capsy

snapshot.main()

written = yaml.safe_load(output.read_text(encoding="utf-8"))
written = yaml.load(output.read_text(encoding="utf-8"))
assert set(written) == {"kept"}
assert written["kept"]["dependencies"] == ["rclcpp"]
assert "Package 'gone' has no package.xml" in capsys.readouterr().out
Loading
Loading