From d3d8908cb12480aa81edbdbf1afc63441bae7812 Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Sun, 22 Feb 2026 21:36:02 +0300 Subject: [PATCH 01/29] feat: add logic parametrization test --- src/imgtests/database/database.py | 32 +- src/imgtests/planning/__init__.py | 21 + src/imgtests/planning/executor.py | 677 ++++++++++++++++++++++++++ src/imgtests/planning/models.py | 109 +++++ src/imgtests/planning/plan_builder.py | 169 +++++++ src/imgtests/planning/profiles.py | 144 ++++++ src/imgtests/reporting/__init__.py | 3 + src/imgtests/reporting/html_report.py | 291 +++++++++++ tests/image/run_profiled_plan.py | 271 +++++++++++ 9 files changed, 1709 insertions(+), 8 deletions(-) create mode 100644 src/imgtests/planning/__init__.py create mode 100644 src/imgtests/planning/executor.py create mode 100644 src/imgtests/planning/models.py create mode 100644 src/imgtests/planning/plan_builder.py create mode 100644 src/imgtests/planning/profiles.py create mode 100644 src/imgtests/reporting/__init__.py create mode 100644 src/imgtests/reporting/html_report.py create mode 100644 tests/image/run_profiled_plan.py diff --git a/src/imgtests/database/database.py b/src/imgtests/database/database.py index 3dbea4f5..90b6d943 100644 --- a/src/imgtests/database/database.py +++ b/src/imgtests/database/database.py @@ -20,6 +20,15 @@ Table = Literal["configurations", "experiments", "loaders", "observers"] +def _clip_db_str(value: str | None, limit: int = 200) -> str | None: + if value is None: + return None + s = str(value) + if len(s) <= limit: + return s + return s[: limit - 3] + "..." + + class ImgtestsDatabase: def __init__(self, database: str = "postgres") -> None: if database == "postgres": @@ -99,8 +108,8 @@ def insert_experiment( experiment_object = ExperimentBase( config_id=config_id, - description=description, - type=experiment_type, + description=_clip_db_str(description), + type=_clip_db_str(experiment_type), started_at=started_at, ended_at=ended_at, ) @@ -128,9 +137,9 @@ def insert_loader( # noqa: PLR0913 loader_object = LoaderBase( experiment_id=experiment_id, - command=command, + command=_clip_db_str(command) or "", result=result, - description=description, + description=_clip_db_str(description), started_at=started_at, ended_at=ended_at, ) @@ -158,9 +167,9 @@ def insert_observer( # noqa: PLR0913 observer_object = ObserverBase( experiment_id=experiment_id, - command=command, + command=_clip_db_str(command) or "", result=result, - description=description, + description=_clip_db_str(description), started_at=started_at, ended_at=ended_at, ) @@ -172,11 +181,18 @@ def insert_observer( # noqa: PLR0913 session.refresh(observer_object) return observer_object - def update_experiment_ended_at(self, experiment_id: int) -> None: + def update_experiment_ended_at( + self, + experiment_id: int, + ended_at: datetime | None = None, + ) -> None: + if ended_at is None: + ended_at = datetime.now(tz=ZoneInfo("UTC")) + self._check_session() with self.session() as session: experiment = session.query(ExperimentBase).filter_by(experiment_id=experiment_id).one() - experiment.ended_at = datetime.now(tz=ZoneInfo("UTC")) + experiment.ended_at = ended_at session.commit() def return_table(self, table_name: Table) -> list[Any]: diff --git a/src/imgtests/planning/__init__.py b/src/imgtests/planning/__init__.py new file mode 100644 index 00000000..58b92623 --- /dev/null +++ b/src/imgtests/planning/__init__.py @@ -0,0 +1,21 @@ +from imgtests.planning.models import ( + LoadPattern, + LoadTask, + PlanRequest, + PlanStage, + Subsystem, + TestKind, + TestPlan, +) +from imgtests.planning.plan_builder import build_plan + +__all__ = [ + "LoadPattern", + "LoadTask", + "PlanRequest", + "PlanStage", + "Subsystem", + "TestKind", + "TestPlan", + "build_plan", +] diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py new file mode 100644 index 00000000..8031d24b --- /dev/null +++ b/src/imgtests/planning/executor.py @@ -0,0 +1,677 @@ +from __future__ import annotations + +import json +import logging +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import suppress +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any + +from imgtests.exec.loaders.fio import Fio +from imgtests.exec.loaders.stress_ng import StressNg +from imgtests.sysrep import get_system_info + +if TYPE_CHECKING: + from pathlib import Path + + from imgtests.database.database import ImgtestsDatabase + from imgtests.exec.exec import SSHClient + from imgtests.planning.models import LoadTask, PlanStage, TestPlan + +logger = logging.getLogger(__name__) + +_DEFAULT_FIO_WORKDIR = "/var/lib/imgtests-fio" + +_DF_AVAIL_COLUMN_INDEX = 3 +_DF_MIN_COLUMNS = 4 +_STRESS_RETRY_RETURN_CODE = 3 + + +@dataclass(frozen=True) +class MetricSample: + stage_name: str + subsystem: str + metric_name: str + value: float + + +@dataclass(frozen=True) +class TaskRunResult: + task: LoadTask + started_at: datetime + ended_at: datetime + command: tuple[str, ...] + returncode: int + stdout: str + stderr: str + summary: dict[str, Any] | None + metrics: tuple[MetricSample, ...] + + +@dataclass(frozen=True) +class StageRunResult: + stage_name: str + started_at: datetime + ended_at: datetime + tasks: tuple[TaskRunResult, ...] + + +@dataclass(frozen=True) +class PlanExecutionResult: + experiment_id: int + plan_path: Path + started_at: datetime + ended_at: datetime + stage_runs: tuple[StageRunResult, ...] + metrics: tuple[MetricSample, ...] + + +def _truncate(text: str, max_len: int = 8000) -> str: + if len(text) <= max_len: + return text + return text[:max_len] + "\n...[truncated]..." + + +def _safe_float(value: Any) -> float | None: + try: + if value is None: + return None + return float(value) + except (TypeError, ValueError): + return None + + +def _safe_int(value: Any, default: int) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _try_parse_json(text: str) -> dict[str, Any]: + if not text or not str(text).strip(): + return {} + try: + data = json.loads(str(text).strip()) + return data if isinstance(data, dict) else {} + except json.JSONDecodeError: + return {} + + +def _parse_size_to_bytes(s: str) -> int | None: + v = str(s).strip() + if not v: + return None + if v.endswith("%"): + return None + + mul = 1 + last = v[-1].lower() + num = v + + if last in {"k", "m", "g", "t"}: + num = v[:-1] + if last == "k": + mul = 1024 + elif last == "m": + mul = 1024**2 + elif last == "g": + mul = 1024**3 + elif last == "t": + mul = 1024**4 + + try: + return int(float(num) * mul) + except (TypeError, ValueError): + return None + + +def _bytes_to_mib_str(b: int) -> str: + mib = max(1, b // (1024**2)) + return f"{mib}M" + + +def _remote_df_avail_bytes(client: SSHClient, path: str) -> int | None: + with suppress(Exception): + res = client(("df", "-PB1", path)) + if getattr(res, "returncode", 1) != 0: + return None + + out = (res.stdout or "").strip().splitlines() + if not out: + return None + + last = out[-1].split() + if len(last) < _DF_MIN_COLUMNS: + return None + + return int(last[_DF_AVAIL_COLUMN_INDEX]) + + return None + + +def _halve_positive_int(value: Any, fallback: int = 1) -> int: + try: + return max(1, int(value) // 2) + except (TypeError, ValueError): + return fallback + + +def _build_stress_retry_args(args: dict[str, Any]) -> dict[str, Any]: + retry_args = dict(args) + + if "vm" in retry_args: + retry_args["vm"] = _halve_positive_int(retry_args["vm"], fallback=1) + retry_args["vm_bytes"] = "15%" + for key in ("vm_populate", "vm_flip", "vm_mmap", "vm_hugepage"): + retry_args.pop(key, None) + + if "cpu" in retry_args: + cpu_value = _safe_int(retry_args["cpu"], default=1) + retry_args["cpu"] = 1 if cpu_value == 0 else max(1, min(cpu_value, 2)) + + if "sock" in retry_args: + retry_args["sock"] = _halve_positive_int(retry_args["sock"], fallback=1) + if "sock_ops" in retry_args: + try: + retry_args["sock_ops"] = max(10_000, int(retry_args["sock_ops"]) // 3) + except (TypeError, ValueError): + retry_args.pop("sock_ops", None) + + if "syscall" in retry_args: + retry_args["syscall"] = _halve_positive_int(retry_args["syscall"], fallback=1) + + return retry_args + + +def _stress_metric_samples( + stage_name: str, + subsystem: str, + metrics: list[Any], +) -> list[MetricSample]: + samples: list[MetricSample] = [] + + for metric in metrics: + base_metrics = ( + ("stress.bogo_ops", float(metric.bogo_ops)), + ("stress.real_time_secs", float(metric.real_time_secs)), + ("stress.usr_time_secs", float(metric.usr_time_secs)), + ("stress.sys_time_secs", float(metric.sys_time_secs)), + ("stress.bogo_ops_s_real_time", float(metric.bogo_ops_s_real_time)), + ("stress.bogo_ops_s_usr_sys_time", float(metric.bogo_ops_s_usr_sys_time)), + ("stress.cpu_used_per_instance", float(metric.cpu_used_per_instance)), + ) + for metric_name, value in base_metrics: + samples.append(MetricSample(stage_name, subsystem, metric_name, value)) + + if metric.rss_max_kb is not None: + samples.append( + MetricSample( + stage_name, + subsystem, + "stress.rss_max_kb", + float(metric.rss_max_kb), + ) + ) + + if metric.top10_slowest: + samples.append( + MetricSample( + stage_name, + subsystem, + "stress.syscall_slowest_avg_ns", + float(metric.top10_slowest[0].avg_ns), + ) + ) + + return samples + + +def _fio_op_samples( + stage_name: str, + subsystem: str, + op: str, + op_data: dict[str, Any], + wanted_p: dict[str, int], +) -> list[MetricSample]: + out: list[MetricSample] = [] + + iops = _safe_float(op_data.get("iops")) + bw = _safe_float(op_data.get("bw")) + runtime_ms = _safe_float(op_data.get("runtime")) + + clat = op_data.get("clat_ns") or {} + clat_mean = _safe_float(clat.get("mean")) if isinstance(clat, dict) else None + + if iops is not None: + out.append(MetricSample(stage_name, subsystem, f"fio.{op}.iops", iops)) + if bw is not None: + out.append(MetricSample(stage_name, subsystem, f"fio.{op}.bw_kib_s", bw)) + if runtime_ms is not None: + out.append(MetricSample(stage_name, subsystem, f"fio.{op}.runtime_ms", runtime_ms)) + if clat_mean is not None: + out.append(MetricSample(stage_name, subsystem, f"fio.{op}.clat_mean_ns", clat_mean)) + + pct = clat.get("percentile") if isinstance(clat, dict) else None + if isinstance(pct, dict): + for key, p_int in wanted_p.items(): + fv = _safe_float(pct.get(key)) + if fv is not None: + out.append(MetricSample(stage_name, subsystem, f"fio.{op}.clat_p{p_int}_ns", fv)) + + return out + + +class PlanExecutor: + def __init__( + self, + client: SSHClient, + db: ImgtestsDatabase, + results_dir: Path, + config_id: int | None = None, + experiment_description: str | None = None, + ) -> None: + self.client = client + self.db = db + self.results_dir = results_dir + self.config_id = config_id + self.experiment_description = experiment_description or "Auto-generated load test plan" + self.fio_workdir = _DEFAULT_FIO_WORKDIR + + def execute(self, plan: TestPlan) -> PlanExecutionResult: + self.results_dir.mkdir(parents=True, exist_ok=True) + started_at = datetime.now(UTC) + + plan_path = self.results_dir / f"plan_{plan.plan_id}.json" + plan_path.write_text( + json.dumps(plan.to_dict(), ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + cfg_id = self._ensure_config_id() + + experiment = self.db.insert_experiment( + config_id=cfg_id, + description=self.experiment_description, + experiment_type=getattr(plan.test_kind, "value", str(plan.test_kind)), + started_at=started_at, + ended_at=started_at, + ) + experiment_id = int(experiment.experiment_id) + + self.db.insert_loader( + experiment_id=experiment_id, + command="plan.json", + result=plan.to_dict(), + description="Generated execution plan", + started_at=started_at, + ended_at=started_at, + ) + + stage_runs: list[StageRunResult] = [] + collected_metrics: list[MetricSample] = [] + + for stage in plan.stages: + self._wait_for_stage_offset( + plan_started_at=started_at, + stage_offset_sec=stage.start_offset_sec, + ) + stage_started_at = datetime.now(UTC) + + self.db.insert_loader( + experiment_id=experiment_id, + command="plan-stage", + result={ + "stage_name": stage.name, + "start_offset_sec": stage.start_offset_sec, + "duration_sec": stage.duration_sec, + "pattern": getattr(stage.pattern, "value", str(stage.pattern)), + "tasks": [t.to_dict() for t in stage.tasks], + }, + description="Planned stage", + started_at=stage_started_at, + ended_at=stage_started_at, + ) + + task_runs = self._run_stage(stage) + stage_ended_at = datetime.now(UTC) + + for task_run in task_runs: + full_cmd = " ".join(task_run.command) if task_run.command else "" + subsystem_value = getattr( + task_run.task.subsystem, + "value", + str(task_run.task.subsystem), + ) + + self.db.insert_loader( + experiment_id=experiment_id, + command=full_cmd, + result={ + "stage_name": stage.name, + "subsystem": subsystem_value, + "tool": task_run.task.tool, + "returncode": task_run.returncode, + "command_full": full_cmd, + "stdout": _truncate(task_run.stdout), + "stderr": _truncate(task_run.stderr), + "summary": task_run.summary, + }, + description=f"Task result for stage={stage.name}", + started_at=task_run.started_at, + ended_at=task_run.ended_at, + ) + + for sample in task_run.metrics: + collected_metrics.append(sample) + self.db.insert_observer( + experiment_id=experiment_id, + command=f"{task_run.task.tool}:{sample.metric_name}", + result={ + "stage_name": sample.stage_name, + "subsystem": sample.subsystem, + "metric_name": sample.metric_name, + "value": sample.value, + }, + description="Observed numeric metric", + started_at=task_run.started_at, + ended_at=task_run.ended_at, + ) + + stage_runs.append( + StageRunResult( + stage_name=stage.name, + started_at=stage_started_at, + ended_at=stage_ended_at, + tasks=tuple(task_runs), + ) + ) + + ended_at = datetime.now(UTC) + self.db.update_experiment_ended_at( + experiment_id=experiment_id, + ended_at=ended_at, + ) + + return PlanExecutionResult( + experiment_id=experiment_id, + plan_path=plan_path, + started_at=started_at, + ended_at=ended_at, + stage_runs=tuple(stage_runs), + metrics=tuple(collected_metrics), + ) + + def _ensure_config_id(self) -> int: + if self.config_id is not None: + return int(self.config_id) + sys_info = get_system_info(self.client) + cfg = self.db.insert_from_system_info(sys_info) + return int(cfg.config_id) + + def _wait_for_stage_offset( + self, + plan_started_at: datetime, + stage_offset_sec: int, + ) -> None: + target_ts = plan_started_at.timestamp() + max(0, int(stage_offset_sec)) + now_ts = datetime.now(UTC).timestamp() + wait_sec = target_ts - now_ts + if wait_sec > 0: + time.sleep(wait_sec) + + def _run_stage(self, stage: PlanStage) -> list[TaskRunResult]: + if not stage.tasks: + return [] + + results_by_idx: dict[int, TaskRunResult] = {} + max_workers = max(1, len(stage.tasks)) + + with ThreadPoolExecutor(max_workers=max_workers) as pool: + future_map = { + pool.submit(self._run_task, stage, task): (idx, task) + for idx, task in enumerate(stage.tasks) + } + + for future in as_completed(future_map): + idx, task = future_map[future] + try: + results_by_idx[idx] = future.result() + except Exception as exc: + now = datetime.now(UTC) + logger.exception("Task failed with exception.") + results_by_idx[idx] = TaskRunResult( + task=task, + started_at=now, + ended_at=now, + command=("internal-error",), + returncode=1, + stdout="", + stderr=str(exc), + summary={"error": str(exc)}, + metrics=(), + ) + + return [results_by_idx[i] for i in range(len(stage.tasks))] + + def _run_task(self, stage: PlanStage, task: LoadTask) -> TaskRunResult: + tool = (task.tool or "").strip().lower().replace("_", "-") + started_at = datetime.now(UTC) + subsystem_value = getattr(task.subsystem, "value", str(task.subsystem)) + + logger.info( + "[PLAN] run stage=%s tool=%s subsystem=%s dur=%ss", + stage.name, + tool, + subsystem_value, + stage.duration_sec, + ) + + if tool in {"stress-ng", "stressng"}: + return self._run_stress_ng(stage, task, started_at) + + if tool == "fio": + return self._run_fio(stage, task, started_at) + + now = datetime.now(UTC) + return TaskRunResult( + task=task, + started_at=started_at, + ended_at=now, + command=("unknown-tool", tool), + returncode=1, + stdout="", + stderr=f"Unknown tool: {tool}", + summary={"error": f"Unknown tool: {tool}"}, + metrics=(), + ) + + def _retry_stress_run_if_needed( + self, + stress: StressNg, + stage_name: str, + timeout_sec: int, + args: dict[str, Any], + exec_res: Any, + ) -> tuple[Any, list[Any], Any] | None: + if exec_res.returncode != _STRESS_RETRY_RETURN_CODE: + return None + + retry_args = _build_stress_retry_args(args) + retry_timeout = max(5, int(timeout_sec * 0.5)) + + logger.info( + "[PLAN] stress-ng retry stage=%s rc=%s timeout=%s->%s", + stage_name, + _STRESS_RETRY_RETURN_CODE, + timeout_sec, + retry_timeout, + ) + + exec_res2, (metrics2, summary2) = stress.run( + timeout_sec=retry_timeout, + **retry_args, + ) + if exec_res2.returncode == 0: + return exec_res2, metrics2, summary2 + return None + + def _run_stress_ng( + self, + stage: PlanStage, + task: LoadTask, + started_at: datetime, + ) -> TaskRunResult: + stress = StressNg(self.client) + + args = dict(task.args or {}) + timeout_sec = int(args.pop("timeout_sec", stage.duration_sec)) + args.pop("verify", None) + + exec_res, (metrics, summary) = stress.run(timeout_sec=timeout_sec, **args) + + retry_result = self._retry_stress_run_if_needed( + stress=stress, + stage_name=stage.name, + timeout_sec=timeout_sec, + args=args, + exec_res=exec_res, + ) + if retry_result is not None: + exec_res, metrics, summary = retry_result + + ended_at = datetime.now(UTC) + subsystem = getattr(task.subsystem, "value", str(task.subsystem)) + samples = _stress_metric_samples(stage.name, subsystem, metrics) + + summary_dict = summary._asdict() if summary else None + logger.info( + "[PLAN] done stage=%s tool=stress-ng rc=%s", + stage.name, + exec_res.returncode, + ) + return TaskRunResult( + task=task, + started_at=started_at, + ended_at=ended_at, + command=exec_res.cmd, + returncode=exec_res.returncode, + stdout=exec_res.stdout, + stderr=exec_res.stderr, + summary=summary_dict, + metrics=tuple(samples), + ) + + def _run_fio( + self, + stage: PlanStage, + task: LoadTask, + started_at: datetime, + ) -> TaskRunResult: + fio = Fio(self.client) + self.client(("mkdir", "-p", self.fio_workdir)) + + args = dict(task.args or {}) + runtime_sec = int(args.pop("runtime_sec", stage.duration_sec)) + + created_filename: str | None = None + if "filename" not in args and "directory" not in args: + subsystem = getattr(task.subsystem, "value", str(task.subsystem)) + created_filename = f"{self.fio_workdir}/{stage.name}-{subsystem}.dat" + args["filename"] = created_filename + + if "size" in args: + req = _parse_size_to_bytes(str(args["size"])) + avail = _remote_df_avail_bytes(self.client, self.fio_workdir) + if req is not None and avail is not None: + cap = max(64 * 1024**2, int(avail * 0.25)) + safe = min(req, cap) + if safe < req: + args["size"] = _bytes_to_mib_str(safe) + + fio_name = args.pop( + "name", + f"{stage.name}-{getattr(task.subsystem, 'value', str(task.subsystem))}", + ) + + try: + result = fio.run( + name=fio_name, + runtime=runtime_sec, + time_based=True, + group_reporting=True, + direct=1, + ioengine=args.pop("ioengine", "libaio"), + **args, + ) + finally: + if created_filename: + with suppress(Exception): + self.client(("rm", "-f", created_filename)) + + ended_at = datetime.now(UTC) + + payload = _try_parse_json(result.stdout) + samples = self._fio_samples( + payload=payload, + stage_name=stage.name, + subsystem=getattr(task.subsystem, "value", str(task.subsystem)), + ) + + summary = None + if payload: + summary = {"jobs_count": len(payload.get("jobs", []))} + + logger.info("[PLAN] done stage=%s tool=fio rc=%s", stage.name, result.returncode) + return TaskRunResult( + task=task, + started_at=started_at, + ended_at=ended_at, + command=result.cmd if result.cmd else ("fio",), + returncode=result.returncode, + stdout=result.stdout, + stderr=result.stderr, + summary=summary, + metrics=tuple(samples), + ) + + def _fio_samples( + self, + payload: dict[str, Any], + stage_name: str, + subsystem: str, + ) -> list[MetricSample]: + jobs = payload.get("jobs", []) + if not isinstance(jobs, list): + return [] + + wanted_p = { + "50.000000": 50, + "90.000000": 90, + "95.000000": 95, + "99.000000": 99, + "99.900000": 999, + } + + out: list[MetricSample] = [] + for job in jobs: + if not isinstance(job, dict): + continue + + for op in ("read", "write", "trim"): + op_data = job.get(op, {}) + if not isinstance(op_data, dict): + continue + out.extend( + _fio_op_samples( + stage_name=stage_name, + subsystem=subsystem, + op=op, + op_data=op_data, + wanted_p=wanted_p, + ) + ) + + return out diff --git a/src/imgtests/planning/models.py b/src/imgtests/planning/models.py new file mode 100644 index 00000000..52f359c9 --- /dev/null +++ b/src/imgtests/planning/models.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime +from enum import Enum +from typing import Any +from uuid import uuid4 + + +class Subsystem(str, Enum): + CPU = "cpu" + MEMORY = "memory" + DISK = "disk" + NETWORK = "network" + SYSCALL = "syscall" + + +class TestKind(str, Enum): + LOAD = "load" + STRESS = "stress" + STABILITY = "stability" + SCALABILITY = "scalability" + VOLUME = "volume" + ISOLATED = "isolated" + SPIKE = "spike" + + +class LoadPattern(str, Enum): + SOFT = "soft" + BALANCED = "balanced" + INTENSE = "intense" + EXTREME = "extreme" + SPIKE = "spike" + + +@dataclass(frozen=True) +class LoadTask: + subsystem: Subsystem + tool: str + args: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + return { + "subsystem": self.subsystem.value, + "tool": self.tool, + "args": self.args, + } + + +@dataclass(frozen=True) +class PlanStage: + name: str + start_offset_sec: int + duration_sec: int + pattern: LoadPattern + tasks: tuple[LoadTask, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "start_offset_sec": self.start_offset_sec, + "duration_sec": self.duration_sec, + "pattern": self.pattern.value, + "tasks": [task.to_dict() for task in self.tasks], + } + + +@dataclass(frozen=True) +class PlanRequest: + duration_sec: int + subsystems: tuple[Subsystem, ...] + test_kind: TestKind + pattern: LoadPattern | None = None + + +@dataclass(frozen=True) +class TestPlan: + plan_id: str + created_at: datetime + duration_sec: int + subsystems: tuple[Subsystem, ...] + test_kind: TestKind + stages: tuple[PlanStage, ...] + + @staticmethod + def new( + duration_sec: int, + subsystems: tuple[Subsystem, ...], + test_kind: TestKind, + stages: tuple[PlanStage, ...], + ) -> TestPlan: + return TestPlan( + plan_id=uuid4().hex[:10], + created_at=datetime.now(UTC), + duration_sec=duration_sec, + subsystems=subsystems, + test_kind=test_kind, + stages=stages, + ) + + def to_dict(self) -> dict[str, Any]: + return { + "plan_id": self.plan_id, + "created_at": self.created_at.isoformat(), + "duration_sec": self.duration_sec, + "subsystems": [x.value for x in self.subsystems], + "test_kind": self.test_kind.value, + "stages": [stage.to_dict() for stage in self.stages], + } diff --git a/src/imgtests/planning/plan_builder.py b/src/imgtests/planning/plan_builder.py new file mode 100644 index 00000000..09a00e95 --- /dev/null +++ b/src/imgtests/planning/plan_builder.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from imgtests.planning.models import ( + LoadPattern, + PlanRequest, + PlanStage, + Subsystem, + TestKind, + TestPlan, +) +from imgtests.planning.profiles import PROFILE_LAYOUTS, build_task + +if TYPE_CHECKING: + from collections.abc import Iterable + +_ALLOC_GUARD_LIMIT = 100_000 + + +def build_plan(request: PlanRequest) -> TestPlan: + if request.duration_sec <= 0: + err = f"duration_sec must be > 0, got {request.duration_sec}" + raise ValueError(err) + + if not request.subsystems: + msg = "At least one subsystem must be provided." + raise ValueError(msg) + + stages: list[PlanStage] = [] + + if request.pattern is not None: + if request.test_kind == TestKind.ISOLATED: + stages = _build_isolated_stages( + request.duration_sec, + request.subsystems, + pattern=request.pattern, + ) + else: + tasks = tuple( + build_task(s, request.pattern, request.duration_sec) for s in request.subsystems + ) + stages = [ + PlanStage( + name=f"{request.test_kind.value}_{request.pattern.value}", + start_offset_sec=0, + duration_sec=request.duration_sec, + pattern=request.pattern, + tasks=tasks, + ) + ] + elif request.test_kind == TestKind.ISOLATED: + stages = _build_isolated_stages(request.duration_sec, request.subsystems) + else: + templates = PROFILE_LAYOUTS[request.test_kind] + durations = _allocate_durations( + request.duration_sec, + [tpl.weight for tpl in templates], + ) + + offset = 0 + for tpl, dur in zip(templates, durations, strict=True): + tasks = tuple(build_task(s, tpl.pattern, dur) for s in request.subsystems) + stages.append( + PlanStage( + name=tpl.name, + start_offset_sec=offset, + duration_sec=dur, + pattern=tpl.pattern, + tasks=tasks, + ) + ) + offset += dur + + return TestPlan.new( + duration_sec=request.duration_sec, + subsystems=request.subsystems, + test_kind=request.test_kind, + stages=tuple(stages), + ) + + +def _build_isolated_stages( + duration_sec: int, + subsystems: Iterable[Subsystem], + pattern: LoadPattern = LoadPattern.BALANCED, +) -> list[PlanStage]: + subsystems_list = list(subsystems) + durations = _allocate_durations(duration_sec, [1.0] * len(subsystems_list)) + stages: list[PlanStage] = [] + offset = 0 + + for subsystem, stage_duration in zip(subsystems_list, durations, strict=True): + task = build_task(subsystem, pattern, stage_duration) + stages.append( + PlanStage( + name=f"isolated_{subsystem.value}", + start_offset_sec=offset, + duration_sec=stage_duration, + pattern=pattern, + tasks=(task,), + ) + ) + offset += stage_duration + + return stages + + +def _allocate_durations(total_sec: int, weights: list[float]) -> list[int]: + _validate_allocation_inputs(total_sec, weights) + + weight_sum = sum(weights) + raw = [total_sec * (w / weight_sum) for w in weights] + base = [int(x) for x in raw] + base = [max(1, value) for value in base] + + diff = total_sec - sum(base) + if diff > 0: + _distribute_positive_diff(raw, base, diff) + elif diff < 0: + _distribute_negative_diff(raw, base, diff) + + if sum(base) != total_sec: + base[-1] += total_sec - sum(base) + + return base + + +def _validate_allocation_inputs(total_sec: int, weights: list[float]) -> None: + if total_sec <= 0: + msg = "total_sec must be > 0." + raise ValueError(msg) + + if not weights: + msg = "weights must not be empty." + raise ValueError(msg) + + if sum(weights) <= 0: + msg = "weights sum must be > 0." + raise ValueError(msg) + + +def _distribute_positive_diff(raw: list[float], base: list[int], diff: int) -> None: + frac_idx = sorted(range(len(raw)), key=lambda i: raw[i] - int(raw[i]), reverse=True) + idx = 0 + remaining = diff + + while remaining > 0: + base[frac_idx[idx % len(frac_idx)]] += 1 + remaining -= 1 + idx += 1 + + +def _distribute_negative_diff(raw: list[float], base: list[int], diff: int) -> None: + frac_idx = sorted(range(len(raw)), key=lambda i: raw[i] - int(raw[i])) + idx = 0 + guard = 0 + remaining = diff + + while remaining < 0 and guard < _ALLOC_GUARD_LIMIT: + target_idx = frac_idx[idx % len(frac_idx)] + if base[target_idx] > 1: + base[target_idx] -= 1 + remaining += 1 + idx += 1 + guard += 1 + + if remaining < 0: + base[-1] = max(1, base[-1] + remaining) diff --git a/src/imgtests/planning/profiles.py b/src/imgtests/planning/profiles.py new file mode 100644 index 00000000..003cc233 --- /dev/null +++ b/src/imgtests/planning/profiles.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from imgtests.planning.models import LoadPattern, LoadTask, Subsystem, TestKind + + +@dataclass(frozen=True) +class StageTemplate: + name: str + pattern: LoadPattern + weight: float + + +PROFILE_LAYOUTS: dict[TestKind, tuple[StageTemplate, ...]] = { + TestKind.LOAD: ( + StageTemplate("warmup", LoadPattern.SOFT, 0.20), + StageTemplate("main", LoadPattern.BALANCED, 0.60), + StageTemplate("cooldown", LoadPattern.SOFT, 0.20), + ), + TestKind.STRESS: ( + StageTemplate("ramp", LoadPattern.BALANCED, 0.15), + StageTemplate("peak", LoadPattern.EXTREME, 0.70), + StageTemplate("cooldown", LoadPattern.INTENSE, 0.15), + ), + TestKind.STABILITY: (StageTemplate("soak", LoadPattern.BALANCED, 1.00),), + TestKind.SCALABILITY: ( + StageTemplate("step_soft", LoadPattern.SOFT, 0.25), + StageTemplate("step_balanced", LoadPattern.BALANCED, 0.25), + StageTemplate("step_intense", LoadPattern.INTENSE, 0.25), + StageTemplate("step_extreme", LoadPattern.EXTREME, 0.25), + ), + TestKind.VOLUME: ( + StageTemplate("prefill", LoadPattern.BALANCED, 0.25), + StageTemplate("bulk", LoadPattern.INTENSE, 0.75), + ), + TestKind.SPIKE: ( + StageTemplate("baseline_1", LoadPattern.SOFT, 0.45), + StageTemplate("spike", LoadPattern.SPIKE, 0.10), + StageTemplate("baseline_2", LoadPattern.SOFT, 0.45), + ), + TestKind.ISOLATED: (StageTemplate("tools_smoke", LoadPattern.SOFT, 1.00),), +} + +_STRESS_ARGS: dict[Subsystem, dict[LoadPattern, dict[str, Any]]] = { + Subsystem.CPU: { + LoadPattern.SOFT: {"cpu": 1, "cpu_method": "matrixprod"}, + LoadPattern.BALANCED: {"cpu": 0, "cpu_method": "all"}, + LoadPattern.INTENSE: {"cpu": 0, "cpu_method": "all"}, + LoadPattern.EXTREME: {"cpu": 0, "cpu_method": "all"}, + LoadPattern.SPIKE: {"cpu": 0, "cpu_method": "all"}, + }, + Subsystem.MEMORY: { + LoadPattern.SOFT: {"vm": 1, "vm_method": "all", "vm_bytes": "10%"}, + LoadPattern.BALANCED: {"vm": 2, "vm_method": "all", "vm_bytes": "20%"}, + LoadPattern.INTENSE: {"vm": 4, "vm_method": "all", "vm_bytes": "35%"}, + LoadPattern.EXTREME: { + "vm": 8, + "vm_method": "all", + "vm_bytes": "50%", + "vm_populate": True, + }, + LoadPattern.SPIKE: { + "vm": 8, + "vm_method": "all", + "vm_bytes": "55%", + "vm_populate": True, + }, + }, + Subsystem.NETWORK: { + LoadPattern.SOFT: {"sock": 1, "sock_ops": 50_000}, + LoadPattern.BALANCED: {"sock": 2, "sock_ops": 150_000}, + LoadPattern.INTENSE: {"sock": 4, "sock_ops": 250_000}, + LoadPattern.EXTREME: {"sock": 6, "sock_ops": 350_000}, + LoadPattern.SPIKE: {"sock": 8, "sock_ops": 500_000}, + }, + Subsystem.SYSCALL: { + LoadPattern.SOFT: {"syscall": 1}, + LoadPattern.BALANCED: {"syscall": 2}, + LoadPattern.INTENSE: {"syscall": 4}, + LoadPattern.EXTREME: {"syscall": 6}, + LoadPattern.SPIKE: {"syscall": 8}, + }, +} + +_FIO_ARGS: dict[LoadPattern, dict[str, Any]] = { + LoadPattern.SOFT: { + "readwrite": "read", + "bs": "128k", + "numjobs": 1, + "iodepth": 1, + "size": "256M", + }, + LoadPattern.BALANCED: { + "readwrite": "randrw", + "bs": "16k", + "numjobs": 2, + "iodepth": 4, + "size": "512M", + }, + LoadPattern.INTENSE: { + "readwrite": "randrw", + "bs": "4k", + "numjobs": 4, + "iodepth": 16, + "size": "768M", + }, + LoadPattern.EXTREME: { + "readwrite": "randwrite", + "bs": "4k", + "numjobs": 6, + "iodepth": 24, + "size": "1G", + }, + LoadPattern.SPIKE: { + "readwrite": "randwrite", + "bs": "4k", + "numjobs": 6, + "iodepth": 24, + "size": "1G", + }, +} + + +def build_task(subsystem: Subsystem, pattern: LoadPattern, stage_duration_sec: int) -> LoadTask: + if subsystem == Subsystem.DISK: + args = dict(_FIO_ARGS[pattern]) + args["runtime_sec"] = stage_duration_sec + return LoadTask(subsystem=subsystem, tool="fio", args=args) + + subsystem_args = _STRESS_ARGS.get(subsystem, {}) + args = dict(subsystem_args.get(pattern, {})) + args["timeout_sec"] = stage_duration_sec + return LoadTask(subsystem=subsystem, tool="stress-ng", args=args) + + +def build_stage_tasks( + _test_kind: TestKind, + subsystems: tuple[Subsystem, ...], + pattern: LoadPattern, + stage_duration_sec: int, +) -> tuple[LoadTask, ...]: + return tuple(build_task(ss, pattern, stage_duration_sec) for ss in subsystems) diff --git a/src/imgtests/reporting/__init__.py b/src/imgtests/reporting/__init__.py new file mode 100644 index 00000000..dc83c72f --- /dev/null +++ b/src/imgtests/reporting/__init__.py @@ -0,0 +1,3 @@ +from imgtests.reporting.html_report import generate_html_report + +__all__ = ["generate_html_report"] diff --git a/src/imgtests/reporting/html_report.py b/src/imgtests/reporting/html_report.py new file mode 100644 index 00000000..a3e39b39 --- /dev/null +++ b/src/imgtests/reporting/html_report.py @@ -0,0 +1,291 @@ +from __future__ import annotations + +import html +import re +import statistics +from collections import defaultdict +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import matplotlib.pyplot as plt + +if TYPE_CHECKING: + from pathlib import Path + + from imgtests.planning.executor import MetricSample, PlanExecutionResult + from imgtests.planning.models import TestPlan + + +@dataclass(frozen=True) +class StatsRow: + stage_name: str + subsystem: str + metric_name: str + count: int + mean: float + variance: float + q25: float + q50: float + q75: float + q95: float + min_v: float + max_v: float + + +def generate_html_report(plan: TestPlan, execution: PlanExecutionResult, out_dir: Path) -> Path: + out_dir.mkdir(parents=True, exist_ok=True) + plots_dir = out_dir / "plots" + plots_dir.mkdir(parents=True, exist_ok=True) + + overall_rows = _compute_stats(list(execution.metrics), by_stage=False) + per_stage_rows = _compute_stats(list(execution.metrics), by_stage=True) + plot_files = _build_boxplots(list(execution.metrics), plots_dir) + + stage_run_map = {x.stage_name: x for x in execution.stage_runs} + total_tasks = sum(len(x.tasks) for x in execution.stage_runs) + failed_tasks = sum(1 for s in execution.stage_runs for t in s.tasks if t.returncode != 0) + + html_lines: list[str] = [] + html_lines.append("") + html_lines.append("") + html_lines.append("") + html_lines.append("") + html_lines.append("Load Test Report") + html_lines.append( + "" + ) + html_lines.append("") + + html_lines.append("

Load/Stress Test Report

") + html_lines.append( + f"

Experiment ID: {execution.experiment_id} | " + f"Plan ID: {html.escape(plan.plan_id)}

" + ) + html_lines.append( + f"

Started: {execution.started_at.isoformat()} | " + f"Ended: {execution.ended_at.isoformat()}

" + ) + html_lines.append( + f"

Total tasks: {total_tasks} | " + f"Failed: {failed_tasks}

" + ) + html_lines.append(f"

Plan file: {html.escape(str(execution.plan_path))}

") + + html_lines.append("

1) Plan Timeline

") + html_lines.append("") + html_lines.append( + "" + "" + "" + "" + "" + ) + for stage in plan.stages: + run = stage_run_map.get(stage.name) + actual_started = run.started_at.isoformat() if run else "-" + actual_ended = run.ended_at.isoformat() if run else "-" + actual_duration = f"{(run.ended_at - run.started_at).total_seconds():.2f}" if run else "-" + failures = sum(1 for t in (run.tasks if run else []) if t.returncode != 0) + html_lines.append( + "" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + "" + ) + html_lines.append("
StagePatternPlanned start, secPlanned duration, secActual startedActual endedActual duration, secTasksFailures
{html.escape(stage.name)}{html.escape(stage.pattern.value)}{stage.start_offset_sec}{stage.duration_sec}{html.escape(actual_started)}{html.escape(actual_ended)}{actual_duration}{len(stage.tasks)}{failures}
") + + html_lines.append("

2) Aggregated Statistics (Overall)

") + html_lines.append( + "

" + "Mean, variance, quartiles and 95th percentile for collected numeric metrics " + "(all stages merged)." + "

" + ) + html_lines.append(_render_stats_table(overall_rows, include_stage=False)) + + html_lines.append("

3) Statistics by Stage

") + html_lines.append("

Same stats, but computed separately for each stage.

") + html_lines.append(_render_stats_table(per_stage_rows, include_stage=True)) + + html_lines.append("

4) Boxplot Visualizations

") + if not plot_files: + html_lines.append("

No plots were generated (not enough numeric samples).

") + else: + for plot in plot_files: + rel = plot.relative_to(out_dir) + html_lines.append(f"

{html.escape(plot.stem)}

") + html_lines.append(f"{html.escape(plot.stem)}") + + html_lines.append("") + + report_path = out_dir / "report.html" + report_path.write_text("\n".join(html_lines), encoding="utf-8") + return report_path + + +def _render_stats_table(rows: list[StatsRow], include_stage: bool) -> str: + lines: list[str] = [] + lines.append("") + if include_stage: + lines.append( + "" + "" + "" + "" + ) + else: + lines.append( + "" + "" + "" + "" + ) + + for row in rows: + if include_stage: + lines.append( + "" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + "" + ) + else: + lines.append( + "" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + "" + ) + lines.append("
StageSubsystemMetricNMeanVarianceQ25Q50Q75Q95MinMax
SubsystemMetricNMeanVarianceQ25Q50Q75Q95MinMax
{html.escape(row.stage_name)}{html.escape(row.subsystem)}{html.escape(row.metric_name)}{row.count}{row.mean:.6g}{row.variance:.6g}{row.q25:.6g}{row.q50:.6g}{row.q75:.6g}{row.q95:.6g}{row.min_v:.6g}{row.max_v:.6g}
{html.escape(row.subsystem)}{html.escape(row.metric_name)}{row.count}{row.mean:.6g}{row.variance:.6g}{row.q25:.6g}{row.q50:.6g}{row.q75:.6g}{row.q95:.6g}{row.min_v:.6g}{row.max_v:.6g}
") + return "\n".join(lines) + + +def _compute_stats(samples: list[MetricSample], by_stage: bool) -> list[StatsRow]: + grouped: dict[tuple[str, str, str], list[float]] = defaultdict(list) + for sample in samples: + stage = sample.stage_name if by_stage else "ALL" + grouped[(stage, sample.subsystem, sample.metric_name)].append(float(sample.value)) + + rows: list[StatsRow] = [] + for (stage, subsystem, metric), values in sorted(grouped.items()): + values_sorted = sorted(values) + cnt = len(values_sorted) + if cnt == 0: + continue + + mean_val = statistics.fmean(values_sorted) + variance_val = statistics.pvariance(values_sorted) if cnt > 1 else 0.0 + + rows.append( + StatsRow( + stage_name=stage, + subsystem=subsystem, + metric_name=metric, + count=cnt, + mean=mean_val, + variance=variance_val, + q25=_quantile(values_sorted, 0.25), + q50=_quantile(values_sorted, 0.50), + q75=_quantile(values_sorted, 0.75), + q95=_quantile(values_sorted, 0.95), + min_v=values_sorted[0], + max_v=values_sorted[-1], + ) + ) + return rows + + +def _quantile(sorted_values: list[float], q: float) -> float: + n = len(sorted_values) + if n == 0: + return 0.0 + if n == 1: + return sorted_values[0] + + pos = (n - 1) * q + lo = int(pos) + hi = min(lo + 1, n - 1) + frac = pos - lo + return sorted_values[lo] + (sorted_values[hi] - sorted_values[lo]) * frac + + +def _build_boxplots(samples: list[MetricSample], plots_dir: Path) -> list[Path]: + metric_to_subsys: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list)) + for sample in samples: + metric_to_subsys[sample.metric_name][sample.subsystem].append(float(sample.value)) + + out_paths: list[Path] = [] + + for metric_name in sorted(metric_to_subsys): + by_sub = metric_to_subsys[metric_name] + labels: list[str] = [] + values: list[list[float]] = [] + + for subsystem in sorted(by_sub): + vals = by_sub[subsystem] + if vals: + labels.append(subsystem) + values.append(vals) + + if not values: + continue + + fig, ax = plt.subplots(figsize=(10, 4)) + ax.boxplot(values, labels=labels, showmeans=True) + ax.set_title(metric_name) + ax.set_xlabel("Subsystem") + ax.set_ylabel("Value") + ax.grid(visible=True, axis="y", alpha=0.3) + + out_path = plots_dir / f"{_safe_filename(metric_name)}.png" + fig.tight_layout() + fig.savefig(out_path) + plt.close(fig) + + out_paths.append(out_path) + + return out_paths + + +def _safe_filename(name: str) -> str: + safe = re.sub(r"[^a-zA-Z0-9_.-]+", "_", name).strip("_") + if not safe: + return "metric" + return safe diff --git a/tests/image/run_profiled_plan.py b/tests/image/run_profiled_plan.py new file mode 100644 index 00000000..e516f12c --- /dev/null +++ b/tests/image/run_profiled_plan.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +import logging +import os +import sys +from contextlib import suppress +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import paramiko.ssh_exception + +from imgtests.database.database import ImgtestsDatabase +from imgtests.exec.exec import wait_remote +from imgtests.logger import set_handlers +from imgtests.planning import LoadPattern, PlanRequest, Subsystem, TestKind, build_plan +from imgtests.planning.executor import PlanExecutor +from imgtests.reporting import generate_html_report + +logger = logging.getLogger() +set_handlers(logger, Path("processing.log")) + +yocto_conf = ( + "SSH_YOCTO_ADDR", + "SSH_YOCTO_USER", + "SSH_YOCTO_PASS", + "SSH_YOCTO_PORT", +) + +PROFILE_ORDER: tuple[TestKind, ...] = ( + TestKind.LOAD, + TestKind.STRESS, + TestKind.STABILITY, + TestKind.SCALABILITY, + TestKind.VOLUME, + TestKind.ISOLATED, + TestKind.SPIKE, +) + + +@dataclass(frozen=True) +class RunOneParams: + profile: TestKind + duration_sec: int + subsystems: tuple[Subsystem, ...] + pattern: LoadPattern | None + results_root: Path + + +def parse_bool_env(name: str, default: bool) -> bool: + raw = os.getenv(name) + if raw is None: + return default + return raw.strip().lower() in {"1", "true", "yes", "on"} + + +def parse_subsystems(raw: str) -> tuple[Subsystem, ...]: + value = raw.strip().lower() + if value == "all": + return tuple(Subsystem) + + mapping: dict[str, Subsystem] = { + "cpu": Subsystem.CPU, + "memory": Subsystem.MEMORY, + "disk": Subsystem.DISK, + "network": Subsystem.NETWORK, + "syscall": Subsystem.SYSCALL, + } + + out: list[Subsystem] = [] + seen: set[Subsystem] = set() + + for part in [x.strip().lower() for x in value.split(",") if x.strip()]: + if part not in mapping: + allowed = ", ".join(mapping) + msg = f"Unknown subsystem '{part}'. Allowed: {allowed}" + raise ValueError(msg) + + subsystem = mapping[part] + if subsystem not in seen: + out.append(subsystem) + seen.add(subsystem) + + if not out: + msg = "No subsystems provided." + raise ValueError(msg) + + return tuple(out) + + +def parse_profile(raw: str) -> TestKind: + try: + return TestKind(raw.strip().lower()) + except ValueError as exc: + allowed = ", ".join(x.value for x in TestKind) + msg = f"Unknown profile '{raw}'. Allowed: {allowed}" + raise ValueError(msg) from exc + + +def parse_profiles(raw: str) -> tuple[TestKind, ...]: + value = raw.strip().lower() + if value in {"", "all"}: + return PROFILE_ORDER + + out: list[TestKind] = [] + seen: set[TestKind] = set() + + for part in [x.strip() for x in value.split(",") if x.strip()]: + profile = parse_profile(part) + if profile not in seen: + out.append(profile) + seen.add(profile) + + if not out: + msg = "No profiles provided." + raise ValueError(msg) + + return tuple(out) + + +def parse_pattern(raw: str | None) -> LoadPattern | None: + if raw is None: + return None + + value = raw.strip().lower() + if value in {"", "auto"}: + return None + + try: + return LoadPattern(value) + except ValueError as exc: + allowed = ", ".join(x.value for x in LoadPattern) + msg = f"Unknown pattern '{raw}'. Allowed: {allowed}" + raise ValueError(msg) from exc + + +def run_one( + *, + client: Any, + db: ImgtestsDatabase, + params: RunOneParams, +) -> tuple[int, Path]: + ts = datetime.now(UTC).strftime("%Y%m%d_%H%M%S") + run_name = f"{ts}_{params.profile.value}" + if params.pattern is not None: + run_name += f"_{params.pattern.value}" + + run_dir = params.results_root / run_name + run_dir.mkdir(parents=True, exist_ok=True) + + req = PlanRequest( + duration_sec=params.duration_sec, + subsystems=params.subsystems, + test_kind=params.profile, + pattern=params.pattern, + ) + plan = build_plan(req) + + executor = PlanExecutor( + client=client, + db=db, + results_dir=run_dir, + experiment_description=f"Profiled plan: {params.profile.value}", + ) + execution = executor.execute(plan) + + report_dir = run_dir / "html" + report_path = generate_html_report(plan, execution, report_dir) + + failures = sum(1 for s in execution.stage_runs for t in s.tasks if t.returncode != 0) + + logger.info( + "[PROFILED] DONE profile=%s pattern=%s duration=%ss failures=%d experiment_id=%s", + params.profile.value, + params.pattern.value if params.pattern else "auto", + params.duration_sec, + failures, + execution.experiment_id, + ) + logger.info("[PROFILED] plan=%s", execution.plan_path) + logger.info("[PROFILED] report=%s", report_path) + + return failures, report_path + + +def main() -> None: + client = None + + try: + subsystems = parse_subsystems( + os.getenv("PLAN_SUBSYSTEMS", "cpu,memory,disk,network,syscall") + ) + results_root = Path(os.getenv("PLAN_RESULTS_DIR", "results/profiled")) + run_matrix = parse_bool_env("PLAN_RUN_MATRIX", default=False) + + raw_pattern = os.getenv("PLAN_PATTERN") + pattern = parse_pattern(raw_pattern) + + client = wait_remote(*yocto_conf) + if client is None: + logger.error("Failed to connect to Yocto host via SSH.") + sys.exit(1) + + db = ImgtestsDatabase() + + total_failures = 0 + last_report: Path | None = None + + if run_matrix: + profiles = parse_profiles(os.getenv("PLAN_MATRIX_PROFILES", "all")) + default_duration = int(os.getenv("PLAN_DURATION_SEC", "120")) + + for profile in profiles: + env_key = f"PLAN_DURATION_{profile.value.upper()}" + raw = os.getenv(env_key) + duration_sec = int(raw) if raw is not None else default_duration + + failures, report_path = run_one( + client=client, + db=db, + params=RunOneParams( + profile=profile, + duration_sec=duration_sec, + subsystems=subsystems, + pattern=pattern, + results_root=results_root, + ), + ) + total_failures += failures + last_report = report_path + else: + profile = parse_profile(os.getenv("PLAN_PROFILE", "load")) + duration_sec = int(os.getenv("PLAN_DURATION_SEC", "120")) + + failures, report_path = run_one( + client=client, + db=db, + params=RunOneParams( + profile=profile, + duration_sec=duration_sec, + subsystems=subsystems, + pattern=pattern, + results_root=results_root, + ), + ) + total_failures = failures + last_report = report_path + + if last_report is not None: + logger.info("[PROFILED] Last report path: %s", last_report) + + sys.exit(1 if total_failures else 0) + + except ValueError: + logger.exception("Invalid config") + sys.exit(2) + except paramiko.ssh_exception.SSHException: + logger.exception("SSH error during profiled execution.") + sys.exit(1) + except Exception: + logger.exception("Unexpected error during profiled execution.") + sys.exit(1) + finally: + if client is not None: + with suppress(Exception): + client.close() + + +if __name__ == "__main__": + main() From 266106b9f17fc34dec9962c315ba9a1b8a063817 Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Sun, 1 Mar 2026 19:20:17 +0300 Subject: [PATCH 02/29] fix remarks --- src/imgtests/database/database.py | 24 ++++++------- tests/image/run_profiled_plan.py | 59 ++++++++++++++++++------------- 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/src/imgtests/database/database.py b/src/imgtests/database/database.py index 61b5bb27..0aa5dba8 100644 --- a/src/imgtests/database/database.py +++ b/src/imgtests/database/database.py @@ -21,15 +21,6 @@ ExperimentType = Literal["performance", "endurance", "all"] -def _clip_db_str(value: str | None, limit: int = 200) -> str | None: - if value is None: - return None - s = str(value) - if len(s) <= limit: - return s - return s[: limit - 3] + "..." - - class ImgtestsDatabase: def __init__(self, database: str = "postgres") -> None: if database == "postgres": @@ -138,9 +129,9 @@ def insert_loader( # noqa: PLR0913 loader_object = LoaderBase( experiment_id=experiment_id, - command=_clip_db_str(command) or "", + command=_clip_db_str(command), result=result, - description=_clip_db_str(description), + description=_clip_db_str(description) if description is not None else None, started_at=started_at, ended_at=ended_at, ) @@ -168,9 +159,9 @@ def insert_observer( # noqa: PLR0913 observer_object = ObserverBase( experiment_id=experiment_id, - command=_clip_db_str(command) or "", + command=_clip_db_str(command), result=result, - description=_clip_db_str(description), + description=_clip_db_str(description) if description is not None else None, started_at=started_at, ended_at=ended_at, ) @@ -216,3 +207,10 @@ def return_table(self, table_name: Table) -> list[Any]: logger.error("Table '%s' doesn't exist.", table_name) return session.query(models[table_name]).all() + + +def _clip_db_str(value: str, limit: int = 200) -> str: + s = str(value) + if len(s) <= limit: + return s + return s[: limit - 3] + "..." diff --git a/tests/image/run_profiled_plan.py b/tests/image/run_profiled_plan.py index e516f12c..e303087f 100644 --- a/tests/image/run_profiled_plan.py +++ b/tests/image/run_profiled_plan.py @@ -1,13 +1,17 @@ +# tests/image/run_profiled_plan.py from __future__ import annotations import logging import os import sys -from contextlib import suppress +from contextlib import contextmanager, suppress from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Iterator import paramiko.ssh_exception @@ -184,24 +188,29 @@ def run_one( return failures, report_path -def main() -> None: - client = None - +@contextmanager +def _safe_close_client(client: Any) -> Iterator[Any]: try: - subsystems = parse_subsystems( - os.getenv("PLAN_SUBSYSTEMS", "cpu,memory,disk,network,syscall") - ) - results_root = Path(os.getenv("PLAN_RESULTS_DIR", "results/profiled")) - run_matrix = parse_bool_env("PLAN_RUN_MATRIX", default=False) + yield client + finally: + with suppress(Exception): + client.close() + + +def _run_main() -> int: + subsystems = parse_subsystems(os.getenv("PLAN_SUBSYSTEMS", "cpu,memory,disk,network,syscall")) + results_root = Path(os.getenv("PLAN_RESULTS_DIR", "results/profiled")) + run_matrix = parse_bool_env("PLAN_RUN_MATRIX", default=False) - raw_pattern = os.getenv("PLAN_PATTERN") - pattern = parse_pattern(raw_pattern) + raw_pattern = os.getenv("PLAN_PATTERN") + pattern = parse_pattern(raw_pattern) - client = wait_remote(*yocto_conf) - if client is None: - logger.error("Failed to connect to Yocto host via SSH.") - sys.exit(1) + client = wait_remote(*yocto_conf) + if client is None: + logger.error("Failed to connect to Yocto host via SSH.") + return 1 + with _safe_close_client(client): db = ImgtestsDatabase() total_failures = 0 @@ -250,21 +259,23 @@ def main() -> None: if last_report is not None: logger.info("[PROFILED] Last report path: %s", last_report) - sys.exit(1 if total_failures else 0) + return 1 if total_failures else 0 + +def main() -> None: + try: + exit_code = _run_main() except ValueError: logger.exception("Invalid config") - sys.exit(2) + exit_code = 2 except paramiko.ssh_exception.SSHException: logger.exception("SSH error during profiled execution.") - sys.exit(1) + exit_code = 1 except Exception: logger.exception("Unexpected error during profiled execution.") - sys.exit(1) - finally: - if client is not None: - with suppress(Exception): - client.close() + exit_code = 1 + + sys.exit(exit_code) if __name__ == "__main__": From edb6de213bd7d4e66d88a210e6edbd4239529108 Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Sun, 1 Mar 2026 19:21:41 +0300 Subject: [PATCH 03/29] delete unnecessary comment --- tests/image/run_profiled_plan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/image/run_profiled_plan.py b/tests/image/run_profiled_plan.py index e303087f..4220dfb3 100644 --- a/tests/image/run_profiled_plan.py +++ b/tests/image/run_profiled_plan.py @@ -1,4 +1,3 @@ -# tests/image/run_profiled_plan.py from __future__ import annotations import logging From 6d0adef9e439f1240823ae354ddaff0fa1ca97c1 Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Sun, 8 Mar 2026 17:14:11 +0300 Subject: [PATCH 04/29] adding a commom insertion interface to thr DB --- src/imgtests/database/database.py | 173 +++++++++++++++++++++++++-- src/imgtests/planning/executor.py | 93 +++++++------- tests/image/performance/fio_disks.py | 109 ++++++++++++++++- 3 files changed, 320 insertions(+), 55 deletions(-) diff --git a/src/imgtests/database/database.py b/src/imgtests/database/database.py index 0aa5dba8..b892995f 100644 --- a/src/imgtests/database/database.py +++ b/src/imgtests/database/database.py @@ -1,6 +1,9 @@ import logging import os +from collections.abc import Mapping, Sequence +from dataclasses import asdict, dataclass, is_dataclass from datetime import datetime +from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, get_args from zoneinfo import ZoneInfo @@ -19,6 +22,29 @@ logger = logging.getLogger(__name__) Table = Literal["configurations", "experiments", "loaders", "observers"] ExperimentType = Literal["performance", "endurance", "all"] +CommandValue = str | Sequence[str] + + +@dataclass(frozen=True) +class UtilityMetricRecord: + metric_name: str + value: float + context: dict[str, Any] | None = None + description: str | None = None + command: CommandValue | None = None + + +@dataclass(frozen=True) +class UtilityResultRecord: + experiment_id: int + utility: str + command: CommandValue + result: Any + started_at: datetime + ended_at: datetime + description: str | None = None + context: dict[str, Any] | None = None + metrics: tuple[UtilityMetricRecord, ...] = () class ImgtestsDatabase: @@ -100,8 +126,8 @@ def insert_experiment( experiment_object = ExperimentBase( config_id=config_id, - description=_clip_db_str(description), - type=_clip_db_str(experiment_type), + description=_validate_db_str(description), + type=_validate_db_str(experiment_type), started_at=started_at, ended_at=ended_at, ) @@ -129,9 +155,9 @@ def insert_loader( # noqa: PLR0913 loader_object = LoaderBase( experiment_id=experiment_id, - command=_clip_db_str(command), + command=_validate_db_str(command), result=result, - description=_clip_db_str(description) if description is not None else None, + description=_validate_db_str(description) if description is not None else None, started_at=started_at, ended_at=ended_at, ) @@ -159,9 +185,9 @@ def insert_observer( # noqa: PLR0913 observer_object = ObserverBase( experiment_id=experiment_id, - command=_clip_db_str(command), + command=_validate_db_str(command), result=result, - description=_clip_db_str(description) if description is not None else None, + description=_validate_db_str(description) if description is not None else None, started_at=started_at, ended_at=ended_at, ) @@ -173,6 +199,73 @@ def insert_observer( # noqa: PLR0913 session.refresh(observer_object) return observer_object + def insert_metric_observation( + self, + experiment_id: int, + utility: str, + metric: UtilityMetricRecord, + started_at: datetime, + ended_at: datetime, + ) -> ObserverBase: + payload: dict[str, Any] = { + "utility": utility, + "metric_name": metric.metric_name, + "value": float(metric.value), + } + if metric.context: + payload.update(_normalize_db_mapping(metric.context)) + + if metric.command is not None: + payload["command"] = _normalize_command_json(metric.command) + + command_label = ( + _command_db_label(metric.command, fallback=utility) + if metric.command is not None + else f"{utility}:{metric.metric_name}" + ) + + return self.insert_observer( + experiment_id=experiment_id, + command=command_label, + result=payload, + description=metric.description or "Observed numeric metric", + started_at=started_at, + ended_at=ended_at, + ) + + def insert_utility_result( + self, + record: UtilityResultRecord, + ) -> tuple[LoaderBase, tuple[ObserverBase, ...]]: + result_payload = _coerce_db_payload(record.result) + result_payload.setdefault("utility", record.utility) + result_payload.setdefault("command", _normalize_command_json(record.command)) + + if record.context: + for key, value in _normalize_db_mapping(record.context).items(): + result_payload.setdefault(key, value) + + loader = self.insert_loader( + experiment_id=record.experiment_id, + command=_command_db_label(record.command, fallback=record.utility), + result=result_payload, + description=record.description, + started_at=record.started_at, + ended_at=record.ended_at, + ) + + observers = tuple( + self.insert_metric_observation( + experiment_id=record.experiment_id, + utility=record.utility, + metric=metric, + started_at=record.started_at, + ended_at=record.ended_at, + ) + for metric in record.metrics + ) + return loader, observers + def update_experiment_ended_at( self, experiment_id: int, @@ -209,8 +302,68 @@ def return_table(self, table_name: Table) -> list[Any]: return session.query(models[table_name]).all() -def _clip_db_str(value: str, limit: int = 200) -> str: +def _validate_db_str(value: str, limit: int = 200) -> str: s = str(value) - if len(s) <= limit: - return s - return s[: limit - 3] + "..." + if len(s) > limit: + err_msg = f"Value is too long for DB field: {len(s)} > {limit}. Text: {s}" + raise ValueError(err_msg) + return s + + +def _normalize_command(command: CommandValue) -> str: + if isinstance(command, str): + return command + return " ".join(str(part) for part in command) + + +def _normalize_command_json(command: CommandValue) -> str | list[str]: + if isinstance(command, str): + return command + return [str(part) for part in command] + + +def _command_db_label(command: CommandValue, fallback: str) -> str: + if isinstance(command, str): + parts = command.strip().split() + return parts[0] if parts else fallback + + for part in command: + text = str(part).strip() + if text: + return text + + return fallback + + +def _normalize_db_mapping(mapping: Mapping[str, Any]) -> dict[str, Any]: + return {str(key): _normalize_db_value(value) for key, value in mapping.items()} + + +def _coerce_db_payload(value: Any) -> dict[str, Any]: + normalized = _normalize_db_value(value) + if isinstance(normalized, dict): + return normalized + if isinstance(normalized, list): + return {"items": normalized} + return {"value": normalized} + + +def _normalize_db_value(value: Any) -> Any: + normalized: Any + + if value is None or isinstance(value, (str, int, float, bool)): + normalized = value + elif isinstance(value, Path): + normalized = str(value) + elif isinstance(value, dict): + normalized = {str(key): _normalize_db_value(val) for key, val in value.items()} + elif is_dataclass(value): + normalized = _normalize_db_value(asdict(value)) + elif hasattr(value, "_asdict"): + normalized = _normalize_db_value(value._asdict()) + elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + normalized = [_normalize_db_value(item) for item in value] + else: + normalized = str(value) + + return normalized diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py index 8031d24b..94ae5f97 100644 --- a/src/imgtests/planning/executor.py +++ b/src/imgtests/planning/executor.py @@ -9,6 +9,7 @@ from datetime import UTC, datetime from typing import TYPE_CHECKING, Any +from imgtests.database.database import UtilityMetricRecord, UtilityResultRecord from imgtests.exec.loaders.fio import Fio from imgtests.exec.loaders.stress_ng import StressNg from imgtests.sysrep import get_system_info @@ -339,46 +340,8 @@ def execute(self, plan: TestPlan) -> PlanExecutionResult: stage_ended_at = datetime.now(UTC) for task_run in task_runs: - full_cmd = " ".join(task_run.command) if task_run.command else "" - subsystem_value = getattr( - task_run.task.subsystem, - "value", - str(task_run.task.subsystem), - ) - - self.db.insert_loader( - experiment_id=experiment_id, - command=full_cmd, - result={ - "stage_name": stage.name, - "subsystem": subsystem_value, - "tool": task_run.task.tool, - "returncode": task_run.returncode, - "command_full": full_cmd, - "stdout": _truncate(task_run.stdout), - "stderr": _truncate(task_run.stderr), - "summary": task_run.summary, - }, - description=f"Task result for stage={stage.name}", - started_at=task_run.started_at, - ended_at=task_run.ended_at, - ) - - for sample in task_run.metrics: - collected_metrics.append(sample) - self.db.insert_observer( - experiment_id=experiment_id, - command=f"{task_run.task.tool}:{sample.metric_name}", - result={ - "stage_name": sample.stage_name, - "subsystem": sample.subsystem, - "metric_name": sample.metric_name, - "value": sample.value, - }, - description="Observed numeric metric", - started_at=task_run.started_at, - ended_at=task_run.ended_at, - ) + self._insert_task_run(experiment_id, stage.name, task_run) + collected_metrics.extend(task_run.metrics) stage_runs.append( StageRunResult( @@ -456,6 +419,56 @@ def _run_stage(self, stage: PlanStage) -> list[TaskRunResult]: return [results_by_idx[i] for i in range(len(stage.tasks))] + def _insert_task_run( + self, + experiment_id: int, + stage_name: str, + task_run: TaskRunResult, + ) -> None: + subsystem_value = getattr(task_run.task.subsystem, "value", str(task_run.task.subsystem)) + command = task_run.command if task_run.command else (task_run.task.tool,) + + metrics = tuple( + UtilityMetricRecord( + metric_name=sample.metric_name, + value=sample.value, + context={ + "stage_name": sample.stage_name, + "subsystem": sample.subsystem, + "tool": task_run.task.tool, + }, + description="Observed numeric metric", + command=command, + ) + for sample in task_run.metrics + ) + + self.db.insert_utility_result( + UtilityResultRecord( + experiment_id=experiment_id, + utility=task_run.task.tool, + command=command, + result={ + "stage_name": stage_name, + "subsystem": subsystem_value, + "tool": task_run.task.tool, + "returncode": task_run.returncode, + "stdout": _truncate(task_run.stdout), + "stderr": _truncate(task_run.stderr), + "summary": task_run.summary, + }, + description=f"Task result for stage={stage_name}", + started_at=task_run.started_at, + ended_at=task_run.ended_at, + context={ + "stage_name": stage_name, + "subsystem": subsystem_value, + "tool": task_run.task.tool, + }, + metrics=metrics, + ) + ) + def _run_task(self, stage: PlanStage, task: LoadTask) -> TaskRunResult: tool = (task.tool or "").strip().lower().replace("_", "-") started_at = datetime.now(UTC) diff --git a/tests/image/performance/fio_disks.py b/tests/image/performance/fio_disks.py index 3d50b0ef..dbaf52e4 100644 --- a/tests/image/performance/fio_disks.py +++ b/tests/image/performance/fio_disks.py @@ -1,9 +1,12 @@ import logging +from datetime import UTC, datetime from pathlib import Path from typing import TYPE_CHECKING +from imgtests.database.database import ImgtestsDatabase, UtilityResultRecord from imgtests.runner import AbstractRunnableTimeLimitedTest from imgtests.suites.drive.fio import FioSuite, FioSuiteConfig, FioWorkload +from imgtests.sysrep import get_system_info if TYPE_CHECKING: from concurrent.futures import ThreadPoolExecutor @@ -40,8 +43,21 @@ class FioDisksScalingTest(AbstractRunnableTimeLimitedTest): """Test that runs fio on a disk with scaling workloads.""" - def __init__(self, timeout: int) -> None: + def __init__( + self, + timeout: int, + db: ImgtestsDatabase | None = None, + config_id: int | None = None, + experiment_description: str | None = None, + write_to_db: bool = True, + ) -> None: super().__init__("Scaling load drives with fio.", {"file"}, timeout=timeout) + if write_to_db and db is None: + err_msg = "ImgtestsDatabase must be provided when write_to_db=True." + raise ValueError(err_msg) + self.db = db if write_to_db else None + self.config_id = config_id + self.experiment_description = experiment_description def _run( self, @@ -49,21 +65,69 @@ def _run( client: SSHClient | None, timeout: int, ) -> None: + results_dir = Path().home() / "fio" cfg = FioSuiteConfig( suite="scaling", duration_sec=timeout, - results_dir=Path().home() / "fio", + results_dir=results_dir, workloads=SCALING_WORKLOADS, ) + started_at = datetime.now(UTC) out = FioSuite(client, cfg).run() + ended_at = datetime.now(UTC) + + if self.db is not None: + if self.config_id is None: + cfg_record = self.db.insert_from_system_info(get_system_info(client)) + self.config_id = int(cfg_record.config_id) + + experiment = self.db.insert_experiment( + config_id=int(self.config_id), + description=self.experiment_description or "fio suite 'scaling' run", + experiment_type="performance", + started_at=started_at, + ended_at=ended_at, + ) + self.db.insert_utility_result( + UtilityResultRecord( + experiment_id=int(experiment.experiment_id), + utility="fio", + command=("fio-suite", "scaling"), + result={ + "suite": "scaling", + "timeout_sec": timeout, + "results_dir": results_dir, + "workloads": SCALING_WORKLOADS, + "result": out, + }, + description="fio suite result: scaling", + started_at=started_at, + ended_at=ended_at, + context={"test_name": type(self).__name__}, + ) + ) + self.logger.info("FIO scaling PASSED: %s", out) class FioDisksNightly(AbstractRunnableTimeLimitedTest): """Tests that run fio on a disk with nightly workloads.""" - def __init__(self, timeout: int) -> None: + def __init__( + self, + timeout: int, + db: ImgtestsDatabase | None = None, + config_id: int | None = None, + experiment_description: str | None = None, + write_to_db: bool = True, + ) -> None: super().__init__("Nightly load drives with fio.", {"file"}, timeout) + if write_to_db and db is None: + err_msg = "ImgtestsDatabase must be provided when write_to_db=True." + raise ValueError(err_msg) + self.db = db if write_to_db else None + self.config_id = config_id + self.experiment_description = experiment_description def _run( self, @@ -71,11 +135,46 @@ def _run( client: SSHClient | None, timeout: int, ) -> None: + results_dir = Path().home() / "fio" cfg = FioSuiteConfig( suite="nightly", duration_sec=timeout, - results_dir=Path().home() / "fio", + results_dir=results_dir, workloads=NIGHTLY_WORKLOADS, ) + started_at = datetime.now(UTC) out = FioSuite(client, cfg).run() - logger.info("FIO nightly PASSED: %s", out) + ended_at = datetime.now(UTC) + + if self.db is not None: + if self.config_id is None: + cfg_record = self.db.insert_from_system_info(get_system_info(client)) + self.config_id = int(cfg_record.config_id) + + experiment = self.db.insert_experiment( + config_id=int(self.config_id), + description=self.experiment_description or "fio suite 'nightly' run", + experiment_type="performance", + started_at=started_at, + ended_at=ended_at, + ) + self.db.insert_utility_result( + UtilityResultRecord( + experiment_id=int(experiment.experiment_id), + utility="fio", + command=("fio-suite", "nightly"), + result={ + "suite": "nightly", + "timeout_sec": timeout, + "results_dir": results_dir, + "workloads": NIGHTLY_WORKLOADS, + "result": out, + }, + description="fio suite result: nightly", + started_at=started_at, + ended_at=ended_at, + context={"test_name": type(self).__name__}, + ) + ) + + self.logger.info("FIO nightly PASSED: %s", out) From 438357d7d1b786bc06234e1ef115f8f0ba95c42d Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Sun, 8 Mar 2026 17:19:36 +0300 Subject: [PATCH 05/29] delete clip --- src/imgtests/database/database.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/imgtests/database/database.py b/src/imgtests/database/database.py index 0aa5dba8..ec743cbb 100644 --- a/src/imgtests/database/database.py +++ b/src/imgtests/database/database.py @@ -100,8 +100,8 @@ def insert_experiment( experiment_object = ExperimentBase( config_id=config_id, - description=_clip_db_str(description), - type=_clip_db_str(experiment_type), + description=description, + type=experiment_type, started_at=started_at, ended_at=ended_at, ) @@ -129,9 +129,9 @@ def insert_loader( # noqa: PLR0913 loader_object = LoaderBase( experiment_id=experiment_id, - command=_clip_db_str(command), + command=command, result=result, - description=_clip_db_str(description) if description is not None else None, + description=description if description is not None else None, started_at=started_at, ended_at=ended_at, ) @@ -159,9 +159,9 @@ def insert_observer( # noqa: PLR0913 observer_object = ObserverBase( experiment_id=experiment_id, - command=_clip_db_str(command), + command=command, result=result, - description=_clip_db_str(description) if description is not None else None, + description=description if description is not None else None, started_at=started_at, ended_at=ended_at, ) @@ -207,10 +207,3 @@ def return_table(self, table_name: Table) -> list[Any]: logger.error("Table '%s' doesn't exist.", table_name) return session.query(models[table_name]).all() - - -def _clip_db_str(value: str, limit: int = 200) -> str: - s = str(value) - if len(s) <= limit: - return s - return s[: limit - 3] + "..." From 4f14d95174ef6f4b6e09903c6e90b261d0308b4a Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Wed, 11 Mar 2026 19:48:29 +0300 Subject: [PATCH 06/29] fix remarks --- src/imgtests/database/database.py | 13 +++++++++---- src/imgtests/planning/executor.py | 23 ++++++++++++++++++++--- src/imgtests/planning/models.py | 11 +++-------- src/imgtests/planning/profiles.py | 17 ++++++++++++----- tests/image/run_profiled_plan.py | 12 ++++++++---- 5 files changed, 52 insertions(+), 24 deletions(-) diff --git a/src/imgtests/database/database.py b/src/imgtests/database/database.py index ec743cbb..b6442807 100644 --- a/src/imgtests/database/database.py +++ b/src/imgtests/database/database.py @@ -118,7 +118,7 @@ def insert_loader( # noqa: PLR0913 experiment_id: int, command: str, result: dict[str, Any], - description: str | None = None, + description: str, started_at: datetime | None = None, ended_at: datetime | None = None, ) -> LoaderBase: @@ -131,7 +131,7 @@ def insert_loader( # noqa: PLR0913 experiment_id=experiment_id, command=command, result=result, - description=description if description is not None else None, + description=description, started_at=started_at, ended_at=ended_at, ) @@ -148,7 +148,7 @@ def insert_observer( # noqa: PLR0913 experiment_id: int, command: str, result: dict[str, Any], - description: str | None = None, + description: str, started_at: datetime | None = None, ended_at: datetime | None = None, ) -> ObserverBase: @@ -161,7 +161,7 @@ def insert_observer( # noqa: PLR0913 experiment_id=experiment_id, command=command, result=result, - description=description if description is not None else None, + description=description, started_at=started_at, ended_at=ended_at, ) @@ -207,3 +207,8 @@ def return_table(self, table_name: Table) -> list[Any]: logger.error("Table '%s' doesn't exist.", table_name) return session.query(models[table_name]).all() + + def _check_session(self) -> None: + if not hasattr(self, "session") or self.session is None: + error_message = "Database session not initialized." + raise RuntimeError(error_message) diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py index 8031d24b..b1313b91 100644 --- a/src/imgtests/planning/executor.py +++ b/src/imgtests/planning/executor.py @@ -11,12 +11,13 @@ from imgtests.exec.loaders.fio import Fio from imgtests.exec.loaders.stress_ng import StressNg +from imgtests.runner import Subsystem from imgtests.sysrep import get_system_info if TYPE_CHECKING: from pathlib import Path - from imgtests.database.database import ImgtestsDatabase + from imgtests.database.database import ExperimentType, ImgtestsDatabase from imgtests.exec.exec import SSHClient from imgtests.planning.models import LoadTask, PlanStage, TestPlan @@ -165,7 +166,7 @@ def _build_stress_retry_args(args: dict[str, Any]) -> dict[str, Any]: if "vm" in retry_args: retry_args["vm"] = _halve_positive_int(retry_args["vm"], fallback=1) retry_args["vm_bytes"] = "15%" - for key in ("vm_populate", "vm_flip", "vm_mmap", "vm_hugepage"): + for key in ("vm-populate", "vm-flip", "vm-mmap", "vm-hugepage"): retry_args.pop(key, None) if "cpu" in retry_args: @@ -183,6 +184,10 @@ def _build_stress_retry_args(args: dict[str, Any]) -> dict[str, Any]: if "syscall" in retry_args: retry_args["syscall"] = _halve_positive_int(retry_args["syscall"], fallback=1) + for key in ("mq", "pipe", "sem", "shm"): + if key in retry_args: + retry_args[key] = _halve_positive_int(retry_args[key], fallback=1) + return retry_args @@ -295,7 +300,7 @@ def execute(self, plan: TestPlan) -> PlanExecutionResult: experiment = self.db.insert_experiment( config_id=cfg_id, description=self.experiment_description, - experiment_type=getattr(plan.test_kind, "value", str(plan.test_kind)), + experiment_type=_resolve_experiment_type(plan), started_at=started_at, ended_at=started_at, ) @@ -675,3 +680,15 @@ def _fio_samples( ) return out + + +def _resolve_experiment_type(plan: TestPlan) -> ExperimentType: + test_kind = getattr(plan.test_kind, "value", str(plan.test_kind)) + + if set(plan.subsystems) == set(Subsystem): + return "all" + + if test_kind == "stability": + return "endurance" + + return "performance" diff --git a/src/imgtests/planning/models.py b/src/imgtests/planning/models.py index 52f359c9..d010c3e2 100644 --- a/src/imgtests/planning/models.py +++ b/src/imgtests/planning/models.py @@ -3,16 +3,11 @@ from dataclasses import dataclass from datetime import UTC, datetime from enum import Enum -from typing import Any +from typing import TYPE_CHECKING, Any from uuid import uuid4 - -class Subsystem(str, Enum): - CPU = "cpu" - MEMORY = "memory" - DISK = "disk" - NETWORK = "network" - SYSCALL = "syscall" +if TYPE_CHECKING: + from imgtests.runner import Subsystem class TestKind(str, Enum): diff --git a/src/imgtests/planning/profiles.py b/src/imgtests/planning/profiles.py index 003cc233..b1aa30ea 100644 --- a/src/imgtests/planning/profiles.py +++ b/src/imgtests/planning/profiles.py @@ -44,13 +44,20 @@ class StageTemplate: } _STRESS_ARGS: dict[Subsystem, dict[LoadPattern, dict[str, Any]]] = { - Subsystem.CPU: { + Subsystem.SYSTEM: { LoadPattern.SOFT: {"cpu": 1, "cpu_method": "matrixprod"}, LoadPattern.BALANCED: {"cpu": 0, "cpu_method": "all"}, LoadPattern.INTENSE: {"cpu": 0, "cpu_method": "all"}, LoadPattern.EXTREME: {"cpu": 0, "cpu_method": "all"}, LoadPattern.SPIKE: {"cpu": 0, "cpu_method": "all"}, }, + Subsystem.IPC: { + LoadPattern.SOFT: {"mq": 1, "pipe": 1, "sem": 1, "shm": 1}, + LoadPattern.BALANCED: {"mq": 2, "pipe": 2, "sem": 2, "shm": 2}, + LoadPattern.INTENSE: {"mq": 4, "pipe": 4, "sem": 4, "shm": 4}, + LoadPattern.EXTREME: {"mq": 6, "pipe": 6, "sem": 6, "shm": 6}, + LoadPattern.SPIKE: {"mq": 8, "pipe": 8, "sem": 8, "shm": 8}, + }, Subsystem.MEMORY: { LoadPattern.SOFT: {"vm": 1, "vm_method": "all", "vm_bytes": "10%"}, LoadPattern.BALANCED: {"vm": 2, "vm_method": "all", "vm_bytes": "20%"}, @@ -59,13 +66,13 @@ class StageTemplate: "vm": 8, "vm_method": "all", "vm_bytes": "50%", - "vm_populate": True, + "vm-populate": True, }, LoadPattern.SPIKE: { "vm": 8, "vm_method": "all", "vm_bytes": "55%", - "vm_populate": True, + "vm-populate": True, }, }, Subsystem.NETWORK: { @@ -75,7 +82,7 @@ class StageTemplate: LoadPattern.EXTREME: {"sock": 6, "sock_ops": 350_000}, LoadPattern.SPIKE: {"sock": 8, "sock_ops": 500_000}, }, - Subsystem.SYSCALL: { + Subsystem.SYSCALLS: { LoadPattern.SOFT: {"syscall": 1}, LoadPattern.BALANCED: {"syscall": 2}, LoadPattern.INTENSE: {"syscall": 4}, @@ -124,7 +131,7 @@ class StageTemplate: def build_task(subsystem: Subsystem, pattern: LoadPattern, stage_duration_sec: int) -> LoadTask: - if subsystem == Subsystem.DISK: + if subsystem == Subsystem.FILE: args = dict(_FIO_ARGS[pattern]) args["runtime_sec"] = stage_duration_sec return LoadTask(subsystem=subsystem, tool="fio", args=args) diff --git a/tests/image/run_profiled_plan.py b/tests/image/run_profiled_plan.py index 4220dfb3..aad02248 100644 --- a/tests/image/run_profiled_plan.py +++ b/tests/image/run_profiled_plan.py @@ -64,11 +64,15 @@ def parse_subsystems(raw: str) -> tuple[Subsystem, ...]: return tuple(Subsystem) mapping: dict[str, Subsystem] = { - "cpu": Subsystem.CPU, + "system": Subsystem.SYSTEM, + "cpu": Subsystem.SYSTEM, + "file": Subsystem.FILE, + "disk": Subsystem.FILE, + "ipc": Subsystem.IPC, "memory": Subsystem.MEMORY, - "disk": Subsystem.DISK, "network": Subsystem.NETWORK, - "syscall": Subsystem.SYSCALL, + "syscalls": Subsystem.SYSCALLS, + "syscall": Subsystem.SYSCALLS, } out: list[Subsystem] = [] @@ -197,7 +201,7 @@ def _safe_close_client(client: Any) -> Iterator[Any]: def _run_main() -> int: - subsystems = parse_subsystems(os.getenv("PLAN_SUBSYSTEMS", "cpu,memory,disk,network,syscall")) + subsystems = parse_subsystems(os.getenv("PLAN_SUBSYSTEMS", "all")) results_root = Path(os.getenv("PLAN_RESULTS_DIR", "results/profiled")) run_matrix = parse_bool_env("PLAN_RUN_MATRIX", default=False) From 108d2194fbafe6722c9b13f10ffcf7b9957c01e6 Mon Sep 17 00:00:00 2001 From: Artanias <43622365+Artanias@users.noreply.github.com> Date: Sun, 15 Mar 2026 23:22:50 +0300 Subject: [PATCH 07/29] fix: removes repeated _check_session method. --- src/imgtests/database/database.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/imgtests/database/database.py b/src/imgtests/database/database.py index b6442807..35ee72f3 100644 --- a/src/imgtests/database/database.py +++ b/src/imgtests/database/database.py @@ -40,11 +40,6 @@ def initialize_postgres(self) -> None: self.session = sessionmaker(self.engine) Base.metadata.create_all(self.engine) - def _check_session(self) -> None: - if not hasattr(self, "session") or self.session is None: - error_message = "Database session not initialized." - raise RuntimeError(error_message) - def insert_from_system_info(self, sys_info: SystemInfo) -> ConfigurationBase: db_os = str(sys_info.os_info) db_cinfo = str(sys_info.uname_info) From cc02caa76f2ec58fbf2d97a887cd83e9dcb62d40 Mon Sep 17 00:00:00 2001 From: Artanias <43622365+Artanias@users.noreply.github.com> Date: Sun, 15 Mar 2026 23:26:18 +0300 Subject: [PATCH 08/29] refactor: now description for the observers, loaders and experiments is required. --- src/imgtests/database/models/experiment.py | 2 +- src/imgtests/database/models/loader.py | 2 +- src/imgtests/database/models/observer.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/imgtests/database/models/experiment.py b/src/imgtests/database/models/experiment.py index 04092769..376d613b 100644 --- a/src/imgtests/database/models/experiment.py +++ b/src/imgtests/database/models/experiment.py @@ -14,7 +14,7 @@ class ExperimentBase(Base): experiment_id: Mapped[int] = mapped_column(primary_key=True) config_id: Mapped[int] = mapped_column(ForeignKey("configuration.config_id")) - description: Mapped[str | None] = mapped_column(String(100)) + description: Mapped[str] = mapped_column(String(100)) type: Mapped[str | None] = mapped_column(String(20)) started_at: Mapped[datetime | None] = mapped_column(DateTime) ended_at: Mapped[datetime | None] = mapped_column(DateTime) diff --git a/src/imgtests/database/models/loader.py b/src/imgtests/database/models/loader.py index bc54e03f..39fbb3a5 100644 --- a/src/imgtests/database/models/loader.py +++ b/src/imgtests/database/models/loader.py @@ -17,7 +17,7 @@ class LoaderBase(Base): experiment_id: Mapped[int] = mapped_column(ForeignKey("experiment.experiment_id")) command: Mapped[str] = mapped_column(String(300)) result: Mapped[dict] = mapped_column(JSON) - description: Mapped[str | None] = mapped_column(String(100)) + description: Mapped[str] = mapped_column(String(100)) started_at: Mapped[datetime | None] = mapped_column(DateTime) ended_at: Mapped[datetime | None] = mapped_column(DateTime) experiment: Mapped["ExperimentBase"] = relationship("ExperimentBase", back_populates="loaders") # noqa: UP037 diff --git a/src/imgtests/database/models/observer.py b/src/imgtests/database/models/observer.py index 714d13e0..031a3d44 100644 --- a/src/imgtests/database/models/observer.py +++ b/src/imgtests/database/models/observer.py @@ -17,7 +17,7 @@ class ObserverBase(Base): experiment_id: Mapped[int] = mapped_column(ForeignKey("experiment.experiment_id")) command: Mapped[str] = mapped_column(String(300)) result: Mapped[dict] = mapped_column(JSON) - description: Mapped[str | None] = mapped_column(String(100)) + description: Mapped[str] = mapped_column(String(100)) started_at: Mapped[datetime | None] = mapped_column(DateTime) ended_at: Mapped[datetime | None] = mapped_column(DateTime) experiment: Mapped["ExperimentBase"] = relationship( # noqa: UP037 From b88d932c9ec7d1c432f088b76820a377bb2d14e6 Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Wed, 18 Mar 2026 15:53:13 +0300 Subject: [PATCH 09/29] delete html report --- src/imgtests/reporting/__init__.py | 3 - src/imgtests/reporting/html_report.py | 291 -------------------------- tests/image/run_profiled_plan.py | 17 +- 3 files changed, 3 insertions(+), 308 deletions(-) delete mode 100644 src/imgtests/reporting/__init__.py delete mode 100644 src/imgtests/reporting/html_report.py diff --git a/src/imgtests/reporting/__init__.py b/src/imgtests/reporting/__init__.py deleted file mode 100644 index dc83c72f..00000000 --- a/src/imgtests/reporting/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from imgtests.reporting.html_report import generate_html_report - -__all__ = ["generate_html_report"] diff --git a/src/imgtests/reporting/html_report.py b/src/imgtests/reporting/html_report.py deleted file mode 100644 index a3e39b39..00000000 --- a/src/imgtests/reporting/html_report.py +++ /dev/null @@ -1,291 +0,0 @@ -from __future__ import annotations - -import html -import re -import statistics -from collections import defaultdict -from dataclasses import dataclass -from typing import TYPE_CHECKING - -import matplotlib.pyplot as plt - -if TYPE_CHECKING: - from pathlib import Path - - from imgtests.planning.executor import MetricSample, PlanExecutionResult - from imgtests.planning.models import TestPlan - - -@dataclass(frozen=True) -class StatsRow: - stage_name: str - subsystem: str - metric_name: str - count: int - mean: float - variance: float - q25: float - q50: float - q75: float - q95: float - min_v: float - max_v: float - - -def generate_html_report(plan: TestPlan, execution: PlanExecutionResult, out_dir: Path) -> Path: - out_dir.mkdir(parents=True, exist_ok=True) - plots_dir = out_dir / "plots" - plots_dir.mkdir(parents=True, exist_ok=True) - - overall_rows = _compute_stats(list(execution.metrics), by_stage=False) - per_stage_rows = _compute_stats(list(execution.metrics), by_stage=True) - plot_files = _build_boxplots(list(execution.metrics), plots_dir) - - stage_run_map = {x.stage_name: x for x in execution.stage_runs} - total_tasks = sum(len(x.tasks) for x in execution.stage_runs) - failed_tasks = sum(1 for s in execution.stage_runs for t in s.tasks if t.returncode != 0) - - html_lines: list[str] = [] - html_lines.append("") - html_lines.append("") - html_lines.append("") - html_lines.append("") - html_lines.append("Load Test Report") - html_lines.append( - "" - ) - html_lines.append("") - - html_lines.append("

Load/Stress Test Report

") - html_lines.append( - f"

Experiment ID: {execution.experiment_id} | " - f"Plan ID: {html.escape(plan.plan_id)}

" - ) - html_lines.append( - f"

Started: {execution.started_at.isoformat()} | " - f"Ended: {execution.ended_at.isoformat()}

" - ) - html_lines.append( - f"

Total tasks: {total_tasks} | " - f"Failed: {failed_tasks}

" - ) - html_lines.append(f"

Plan file: {html.escape(str(execution.plan_path))}

") - - html_lines.append("

1) Plan Timeline

") - html_lines.append("") - html_lines.append( - "" - "" - "" - "" - "" - ) - for stage in plan.stages: - run = stage_run_map.get(stage.name) - actual_started = run.started_at.isoformat() if run else "-" - actual_ended = run.ended_at.isoformat() if run else "-" - actual_duration = f"{(run.ended_at - run.started_at).total_seconds():.2f}" if run else "-" - failures = sum(1 for t in (run.tasks if run else []) if t.returncode != 0) - html_lines.append( - "" - f"" - f"" - f"" - f"" - f"" - f"" - f"" - f"" - f"" - "" - ) - html_lines.append("
StagePatternPlanned start, secPlanned duration, secActual startedActual endedActual duration, secTasksFailures
{html.escape(stage.name)}{html.escape(stage.pattern.value)}{stage.start_offset_sec}{stage.duration_sec}{html.escape(actual_started)}{html.escape(actual_ended)}{actual_duration}{len(stage.tasks)}{failures}
") - - html_lines.append("

2) Aggregated Statistics (Overall)

") - html_lines.append( - "

" - "Mean, variance, quartiles and 95th percentile for collected numeric metrics " - "(all stages merged)." - "

" - ) - html_lines.append(_render_stats_table(overall_rows, include_stage=False)) - - html_lines.append("

3) Statistics by Stage

") - html_lines.append("

Same stats, but computed separately for each stage.

") - html_lines.append(_render_stats_table(per_stage_rows, include_stage=True)) - - html_lines.append("

4) Boxplot Visualizations

") - if not plot_files: - html_lines.append("

No plots were generated (not enough numeric samples).

") - else: - for plot in plot_files: - rel = plot.relative_to(out_dir) - html_lines.append(f"

{html.escape(plot.stem)}

") - html_lines.append(f"{html.escape(plot.stem)}") - - html_lines.append("") - - report_path = out_dir / "report.html" - report_path.write_text("\n".join(html_lines), encoding="utf-8") - return report_path - - -def _render_stats_table(rows: list[StatsRow], include_stage: bool) -> str: - lines: list[str] = [] - lines.append("") - if include_stage: - lines.append( - "" - "" - "" - "" - ) - else: - lines.append( - "" - "" - "" - "" - ) - - for row in rows: - if include_stage: - lines.append( - "" - f"" - f"" - f"" - f"" - f"" - f"" - f"" - f"" - f"" - f"" - f"" - f"" - "" - ) - else: - lines.append( - "" - f"" - f"" - f"" - f"" - f"" - f"" - f"" - f"" - f"" - f"" - f"" - "" - ) - lines.append("
StageSubsystemMetricNMeanVarianceQ25Q50Q75Q95MinMax
SubsystemMetricNMeanVarianceQ25Q50Q75Q95MinMax
{html.escape(row.stage_name)}{html.escape(row.subsystem)}{html.escape(row.metric_name)}{row.count}{row.mean:.6g}{row.variance:.6g}{row.q25:.6g}{row.q50:.6g}{row.q75:.6g}{row.q95:.6g}{row.min_v:.6g}{row.max_v:.6g}
{html.escape(row.subsystem)}{html.escape(row.metric_name)}{row.count}{row.mean:.6g}{row.variance:.6g}{row.q25:.6g}{row.q50:.6g}{row.q75:.6g}{row.q95:.6g}{row.min_v:.6g}{row.max_v:.6g}
") - return "\n".join(lines) - - -def _compute_stats(samples: list[MetricSample], by_stage: bool) -> list[StatsRow]: - grouped: dict[tuple[str, str, str], list[float]] = defaultdict(list) - for sample in samples: - stage = sample.stage_name if by_stage else "ALL" - grouped[(stage, sample.subsystem, sample.metric_name)].append(float(sample.value)) - - rows: list[StatsRow] = [] - for (stage, subsystem, metric), values in sorted(grouped.items()): - values_sorted = sorted(values) - cnt = len(values_sorted) - if cnt == 0: - continue - - mean_val = statistics.fmean(values_sorted) - variance_val = statistics.pvariance(values_sorted) if cnt > 1 else 0.0 - - rows.append( - StatsRow( - stage_name=stage, - subsystem=subsystem, - metric_name=metric, - count=cnt, - mean=mean_val, - variance=variance_val, - q25=_quantile(values_sorted, 0.25), - q50=_quantile(values_sorted, 0.50), - q75=_quantile(values_sorted, 0.75), - q95=_quantile(values_sorted, 0.95), - min_v=values_sorted[0], - max_v=values_sorted[-1], - ) - ) - return rows - - -def _quantile(sorted_values: list[float], q: float) -> float: - n = len(sorted_values) - if n == 0: - return 0.0 - if n == 1: - return sorted_values[0] - - pos = (n - 1) * q - lo = int(pos) - hi = min(lo + 1, n - 1) - frac = pos - lo - return sorted_values[lo] + (sorted_values[hi] - sorted_values[lo]) * frac - - -def _build_boxplots(samples: list[MetricSample], plots_dir: Path) -> list[Path]: - metric_to_subsys: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list)) - for sample in samples: - metric_to_subsys[sample.metric_name][sample.subsystem].append(float(sample.value)) - - out_paths: list[Path] = [] - - for metric_name in sorted(metric_to_subsys): - by_sub = metric_to_subsys[metric_name] - labels: list[str] = [] - values: list[list[float]] = [] - - for subsystem in sorted(by_sub): - vals = by_sub[subsystem] - if vals: - labels.append(subsystem) - values.append(vals) - - if not values: - continue - - fig, ax = plt.subplots(figsize=(10, 4)) - ax.boxplot(values, labels=labels, showmeans=True) - ax.set_title(metric_name) - ax.set_xlabel("Subsystem") - ax.set_ylabel("Value") - ax.grid(visible=True, axis="y", alpha=0.3) - - out_path = plots_dir / f"{_safe_filename(metric_name)}.png" - fig.tight_layout() - fig.savefig(out_path) - plt.close(fig) - - out_paths.append(out_path) - - return out_paths - - -def _safe_filename(name: str) -> str: - safe = re.sub(r"[^a-zA-Z0-9_.-]+", "_", name).strip("_") - if not safe: - return "metric" - return safe diff --git a/tests/image/run_profiled_plan.py b/tests/image/run_profiled_plan.py index aad02248..460c1dcd 100644 --- a/tests/image/run_profiled_plan.py +++ b/tests/image/run_profiled_plan.py @@ -19,7 +19,6 @@ from imgtests.logger import set_handlers from imgtests.planning import LoadPattern, PlanRequest, Subsystem, TestKind, build_plan from imgtests.planning.executor import PlanExecutor -from imgtests.reporting import generate_html_report logger = logging.getLogger() set_handlers(logger, Path("processing.log")) @@ -172,9 +171,6 @@ def run_one( ) execution = executor.execute(plan) - report_dir = run_dir / "html" - report_path = generate_html_report(plan, execution, report_dir) - failures = sum(1 for s in execution.stage_runs for t in s.tasks if t.returncode != 0) logger.info( @@ -186,9 +182,8 @@ def run_one( execution.experiment_id, ) logger.info("[PROFILED] plan=%s", execution.plan_path) - logger.info("[PROFILED] report=%s", report_path) - return failures, report_path + return failures @contextmanager @@ -217,7 +212,6 @@ def _run_main() -> int: db = ImgtestsDatabase() total_failures = 0 - last_report: Path | None = None if run_matrix: profiles = parse_profiles(os.getenv("PLAN_MATRIX_PROFILES", "all")) @@ -228,7 +222,7 @@ def _run_main() -> int: raw = os.getenv(env_key) duration_sec = int(raw) if raw is not None else default_duration - failures, report_path = run_one( + failures = run_one( client=client, db=db, params=RunOneParams( @@ -240,12 +234,11 @@ def _run_main() -> int: ), ) total_failures += failures - last_report = report_path else: profile = parse_profile(os.getenv("PLAN_PROFILE", "load")) duration_sec = int(os.getenv("PLAN_DURATION_SEC", "120")) - failures, report_path = run_one( + failures = run_one( client=client, db=db, params=RunOneParams( @@ -257,10 +250,6 @@ def _run_main() -> int: ), ) total_failures = failures - last_report = report_path - - if last_report is not None: - logger.info("[PROFILED] Last report path: %s", last_report) return 1 if total_failures else 0 From 02f946312b0ee965ec8884f7b4356cc672b69cca Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Wed, 18 Mar 2026 16:55:39 +0300 Subject: [PATCH 10/29] fix remarks --- src/imgtests/exec/loaders/stress_ng.py | 45 +++++++++++++ src/imgtests/planning/executor.py | 91 ++------------------------ src/imgtests/planning/fio_sizing.py | 32 +++++++++ 3 files changed, 84 insertions(+), 84 deletions(-) create mode 100644 src/imgtests/planning/fio_sizing.py diff --git a/src/imgtests/exec/loaders/stress_ng.py b/src/imgtests/exec/loaders/stress_ng.py index 56fb8a8d..5b835172 100644 --- a/src/imgtests/exec/loaders/stress_ng.py +++ b/src/imgtests/exec/loaders/stress_ng.py @@ -505,3 +505,48 @@ def metrics_to_bmf(metrics: StressNGMetrics) -> dict[str, dict[str, Any]]: } return result + + +def _safe_int(value: Any, default: int) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _halve_positive_int(value: Any, fallback: int = 1) -> int: + try: + return max(1, int(value) // 2) + except (TypeError, ValueError): + return fallback + + +def build_stress_retry_args(args: dict[str, Any]) -> dict[str, Any]: + retry_args = dict(args) + + if "vm" in retry_args: + retry_args["vm"] = _halve_positive_int(retry_args["vm"], fallback=1) + retry_args["vm_bytes"] = "15%" + for key in ("vm-populate", "vm-flip", "vm-mmap", "vm-hugepage"): + retry_args.pop(key, None) + + if "cpu" in retry_args: + cpu_value = _safe_int(retry_args["cpu"], default=1) + retry_args["cpu"] = 1 if cpu_value == 0 else max(1, min(cpu_value, 2)) + + if "sock" in retry_args: + retry_args["sock"] = _halve_positive_int(retry_args["sock"], fallback=1) + if "sock_ops" in retry_args: + try: + retry_args["sock_ops"] = max(10_000, int(retry_args["sock_ops"]) // 3) + except (TypeError, ValueError): + retry_args.pop("sock_ops", None) + + if "syscall" in retry_args: + retry_args["syscall"] = _halve_positive_int(retry_args["syscall"], fallback=1) + + for key in ("mq", "pipe", "sem", "shm"): + if key in retry_args: + retry_args[key] = _halve_positive_int(retry_args[key], fallback=1) + + return retry_args diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py index b1313b91..59a72fdd 100644 --- a/src/imgtests/planning/executor.py +++ b/src/imgtests/planning/executor.py @@ -10,7 +10,8 @@ from typing import TYPE_CHECKING, Any from imgtests.exec.loaders.fio import Fio -from imgtests.exec.loaders.stress_ng import StressNg +from imgtests.exec.loaders.stress_ng import StressNg, build_stress_retry_args +from imgtests.planning.fio_sizing import bytes_to_mib_str, parse_size_to_bytes from imgtests.runner import Subsystem from imgtests.sysrep import get_system_info @@ -76,21 +77,14 @@ def _truncate(text: str, max_len: int = 8000) -> str: def _safe_float(value: Any) -> float | None: + if value is None: + return None try: - if value is None: - return None return float(value) except (TypeError, ValueError): return None -def _safe_int(value: Any, default: int) -> int: - try: - return int(value) - except (TypeError, ValueError): - return default - - def _try_parse_json(text: str) -> dict[str, Any]: if not text or not str(text).strip(): return {} @@ -101,39 +95,6 @@ def _try_parse_json(text: str) -> dict[str, Any]: return {} -def _parse_size_to_bytes(s: str) -> int | None: - v = str(s).strip() - if not v: - return None - if v.endswith("%"): - return None - - mul = 1 - last = v[-1].lower() - num = v - - if last in {"k", "m", "g", "t"}: - num = v[:-1] - if last == "k": - mul = 1024 - elif last == "m": - mul = 1024**2 - elif last == "g": - mul = 1024**3 - elif last == "t": - mul = 1024**4 - - try: - return int(float(num) * mul) - except (TypeError, ValueError): - return None - - -def _bytes_to_mib_str(b: int) -> str: - mib = max(1, b // (1024**2)) - return f"{mib}M" - - def _remote_df_avail_bytes(client: SSHClient, path: str) -> int | None: with suppress(Exception): res = client(("df", "-PB1", path)) @@ -153,44 +114,6 @@ def _remote_df_avail_bytes(client: SSHClient, path: str) -> int | None: return None -def _halve_positive_int(value: Any, fallback: int = 1) -> int: - try: - return max(1, int(value) // 2) - except (TypeError, ValueError): - return fallback - - -def _build_stress_retry_args(args: dict[str, Any]) -> dict[str, Any]: - retry_args = dict(args) - - if "vm" in retry_args: - retry_args["vm"] = _halve_positive_int(retry_args["vm"], fallback=1) - retry_args["vm_bytes"] = "15%" - for key in ("vm-populate", "vm-flip", "vm-mmap", "vm-hugepage"): - retry_args.pop(key, None) - - if "cpu" in retry_args: - cpu_value = _safe_int(retry_args["cpu"], default=1) - retry_args["cpu"] = 1 if cpu_value == 0 else max(1, min(cpu_value, 2)) - - if "sock" in retry_args: - retry_args["sock"] = _halve_positive_int(retry_args["sock"], fallback=1) - if "sock_ops" in retry_args: - try: - retry_args["sock_ops"] = max(10_000, int(retry_args["sock_ops"]) // 3) - except (TypeError, ValueError): - retry_args.pop("sock_ops", None) - - if "syscall" in retry_args: - retry_args["syscall"] = _halve_positive_int(retry_args["syscall"], fallback=1) - - for key in ("mq", "pipe", "sem", "shm"): - if key in retry_args: - retry_args[key] = _halve_positive_int(retry_args[key], fallback=1) - - return retry_args - - def _stress_metric_samples( stage_name: str, subsystem: str, @@ -504,7 +427,7 @@ def _retry_stress_run_if_needed( if exec_res.returncode != _STRESS_RETRY_RETURN_CODE: return None - retry_args = _build_stress_retry_args(args) + retry_args = build_stress_retry_args(args) retry_timeout = max(5, int(timeout_sec * 0.5)) logger.info( @@ -588,13 +511,13 @@ def _run_fio( args["filename"] = created_filename if "size" in args: - req = _parse_size_to_bytes(str(args["size"])) + req = parse_size_to_bytes(str(args["size"])) avail = _remote_df_avail_bytes(self.client, self.fio_workdir) if req is not None and avail is not None: cap = max(64 * 1024**2, int(avail * 0.25)) safe = min(req, cap) if safe < req: - args["size"] = _bytes_to_mib_str(safe) + args["size"] = bytes_to_mib_str(safe) fio_name = args.pop( "name", diff --git a/src/imgtests/planning/fio_sizing.py b/src/imgtests/planning/fio_sizing.py new file mode 100644 index 00000000..282568c7 --- /dev/null +++ b/src/imgtests/planning/fio_sizing.py @@ -0,0 +1,32 @@ +from __future__ import annotations + + +def parse_size_to_bytes(value: str) -> int | None: + normalized = str(value).strip() + if not normalized or normalized.endswith("%"): + return None + + multiplier = 1 + suffix = normalized[-1].lower() + number = normalized + + if suffix in {"k", "m", "g", "t"}: + number = normalized[:-1] + if suffix == "k": + multiplier = 1024 + elif suffix == "m": + multiplier = 1024**2 + elif suffix == "g": + multiplier = 1024**3 + elif suffix == "t": + multiplier = 1024**4 + + try: + return int(float(number) * multiplier) + except (TypeError, ValueError): + return None + + +def bytes_to_mib_str(size_bytes: int) -> str: + mib = max(1, size_bytes // (1024**2)) + return f"{mib}M" From 51ab4677312f3c84fae8f8d2fb28d5b550d725fa Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Wed, 18 Mar 2026 17:16:07 +0300 Subject: [PATCH 11/29] fix remarks --- src/imgtests/exec/loaders/stress_ng.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/imgtests/exec/loaders/stress_ng.py b/src/imgtests/exec/loaders/stress_ng.py index 5a7edec8..e2619eff 100644 --- a/src/imgtests/exec/loaders/stress_ng.py +++ b/src/imgtests/exec/loaders/stress_ng.py @@ -518,6 +518,7 @@ def metrics_to_json(metrics: StressNGResult) -> Any: "stress_ng_summary": metrics.summary._asdict() if metrics.summary else None, } + def _safe_int(value: Any, default: int) -> int: try: return int(value) From 4fb04e6ee1ed6242fc72a81e473ddd494b7b53f8 Mon Sep 17 00:00:00 2001 From: Artanias <43622365+Artanias@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:39:22 +0300 Subject: [PATCH 12/29] refactor: moves stress_ng helper function into the bottom of the module. --- src/imgtests/exec/loaders/stress_ng.py | 38 +++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/imgtests/exec/loaders/stress_ng.py b/src/imgtests/exec/loaders/stress_ng.py index e2619eff..379d1859 100644 --- a/src/imgtests/exec/loaders/stress_ng.py +++ b/src/imgtests/exec/loaders/stress_ng.py @@ -519,35 +519,21 @@ def metrics_to_json(metrics: StressNGResult) -> Any: } -def _safe_int(value: Any, default: int) -> int: - try: - return int(value) - except (TypeError, ValueError): - return default - - -def _halve_positive_int(value: Any, fallback: int = 1) -> int: - try: - return max(1, int(value) // 2) - except (TypeError, ValueError): - return fallback - - def build_stress_retry_args(args: dict[str, Any]) -> dict[str, Any]: retry_args = dict(args) if "vm" in retry_args: - retry_args["vm"] = _halve_positive_int(retry_args["vm"], fallback=1) + retry_args["vm"] = __half_positive_int(retry_args["vm"], fallback=1) retry_args["vm_bytes"] = "15%" for key in ("vm-populate", "vm-flip", "vm-mmap", "vm-hugepage"): retry_args.pop(key, None) if "cpu" in retry_args: - cpu_value = _safe_int(retry_args["cpu"], default=1) + cpu_value = __safe_int(retry_args["cpu"], default=1) retry_args["cpu"] = 1 if cpu_value == 0 else max(1, min(cpu_value, 2)) if "sock" in retry_args: - retry_args["sock"] = _halve_positive_int(retry_args["sock"], fallback=1) + retry_args["sock"] = __half_positive_int(retry_args["sock"], fallback=1) if "sock_ops" in retry_args: try: retry_args["sock_ops"] = max(10_000, int(retry_args["sock_ops"]) // 3) @@ -555,10 +541,24 @@ def build_stress_retry_args(args: dict[str, Any]) -> dict[str, Any]: retry_args.pop("sock_ops", None) if "syscall" in retry_args: - retry_args["syscall"] = _halve_positive_int(retry_args["syscall"], fallback=1) + retry_args["syscall"] = __half_positive_int(retry_args["syscall"], fallback=1) for key in ("mq", "pipe", "sem", "shm"): if key in retry_args: - retry_args[key] = _halve_positive_int(retry_args[key], fallback=1) + retry_args[key] = __half_positive_int(retry_args[key], fallback=1) return retry_args + + +def __safe_int(value: Any, default: int) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def __half_positive_int(value: Any, fallback: int = 1) -> int: + try: + return max(1, int(value) // 2) + except (TypeError, ValueError): + return fallback From a08a626a4dee800704d2d3c9cf6ff264cebeeb98 Mon Sep 17 00:00:00 2001 From: Artanias <43622365+Artanias@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:53:59 +0300 Subject: [PATCH 13/29] refactor: simplify parse_size_to_bytes. --- src/imgtests/planning/fio_sizing.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/imgtests/planning/fio_sizing.py b/src/imgtests/planning/fio_sizing.py index 282568c7..80ca38a3 100644 --- a/src/imgtests/planning/fio_sizing.py +++ b/src/imgtests/planning/fio_sizing.py @@ -6,20 +6,19 @@ def parse_size_to_bytes(value: str) -> int | None: if not normalized or normalized.endswith("%"): return None - multiplier = 1 suffix = normalized[-1].lower() - number = normalized - - if suffix in {"k", "m", "g", "t"}: - number = normalized[:-1] - if suffix == "k": + match suffix: + case "k": multiplier = 1024 - elif suffix == "m": + case "m": multiplier = 1024**2 - elif suffix == "g": + case "g": multiplier = 1024**3 - elif suffix == "t": + case "t": multiplier = 1024**4 + case _: + multiplier = 1 + number = normalized[:-1] try: return int(float(number) * multiplier) From e61036e29fef4127875cf90b40192dc0709d0fbb Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Fri, 20 Mar 2026 21:37:10 +0300 Subject: [PATCH 14/29] fix remarks --- src/imgtests/exec/loaders/stress_ng.py | 45 ------------------- src/imgtests/planning/executor.py | 29 ++++-------- .../{planning/fio_sizing.py => sizing.py} | 6 ++- 3 files changed, 13 insertions(+), 67 deletions(-) rename src/imgtests/{planning/fio_sizing.py => sizing.py} (79%) diff --git a/src/imgtests/exec/loaders/stress_ng.py b/src/imgtests/exec/loaders/stress_ng.py index 379d1859..4facaf8c 100644 --- a/src/imgtests/exec/loaders/stress_ng.py +++ b/src/imgtests/exec/loaders/stress_ng.py @@ -517,48 +517,3 @@ def metrics_to_json(metrics: StressNGResult) -> Any: "stress_ng_metrics": [metric._asdict() for metric in metrics.metrics], "stress_ng_summary": metrics.summary._asdict() if metrics.summary else None, } - - -def build_stress_retry_args(args: dict[str, Any]) -> dict[str, Any]: - retry_args = dict(args) - - if "vm" in retry_args: - retry_args["vm"] = __half_positive_int(retry_args["vm"], fallback=1) - retry_args["vm_bytes"] = "15%" - for key in ("vm-populate", "vm-flip", "vm-mmap", "vm-hugepage"): - retry_args.pop(key, None) - - if "cpu" in retry_args: - cpu_value = __safe_int(retry_args["cpu"], default=1) - retry_args["cpu"] = 1 if cpu_value == 0 else max(1, min(cpu_value, 2)) - - if "sock" in retry_args: - retry_args["sock"] = __half_positive_int(retry_args["sock"], fallback=1) - if "sock_ops" in retry_args: - try: - retry_args["sock_ops"] = max(10_000, int(retry_args["sock_ops"]) // 3) - except (TypeError, ValueError): - retry_args.pop("sock_ops", None) - - if "syscall" in retry_args: - retry_args["syscall"] = __half_positive_int(retry_args["syscall"], fallback=1) - - for key in ("mq", "pipe", "sem", "shm"): - if key in retry_args: - retry_args[key] = __half_positive_int(retry_args[key], fallback=1) - - return retry_args - - -def __safe_int(value: Any, default: int) -> int: - try: - return int(value) - except (TypeError, ValueError): - return default - - -def __half_positive_int(value: Any, fallback: int = 1) -> int: - try: - return max(1, int(value) // 2) - except (TypeError, ValueError): - return fallback diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py index 59a72fdd..7c10b628 100644 --- a/src/imgtests/planning/executor.py +++ b/src/imgtests/planning/executor.py @@ -10,9 +10,9 @@ from typing import TYPE_CHECKING, Any from imgtests.exec.loaders.fio import Fio -from imgtests.exec.loaders.stress_ng import StressNg, build_stress_retry_args -from imgtests.planning.fio_sizing import bytes_to_mib_str, parse_size_to_bytes +from imgtests.exec.loaders.stress_ng import StressNg from imgtests.runner import Subsystem +from imgtests.sizing import bytes_to_mib_str, parse_size_to_bytes from imgtests.sysrep import get_system_info if TYPE_CHECKING: @@ -70,12 +70,6 @@ class PlanExecutionResult: metrics: tuple[MetricSample, ...] -def _truncate(text: str, max_len: int = 8000) -> str: - if len(text) <= max_len: - return text - return text[:max_len] + "\n...[truncated]..." - - def _safe_float(value: Any) -> float | None: if value is None: return None @@ -278,16 +272,10 @@ def execute(self, plan: TestPlan) -> PlanExecutionResult: experiment_id=experiment_id, command=full_cmd, result={ - "stage_name": stage.name, - "subsystem": subsystem_value, - "tool": task_run.task.tool, "returncode": task_run.returncode, - "command_full": full_cmd, - "stdout": _truncate(task_run.stdout), - "stderr": _truncate(task_run.stderr), "summary": task_run.summary, }, - description=f"Task result for stage={stage.name}", + description=f"Task result stage={stage.name} subsystem={subsystem_value}", started_at=task_run.started_at, ended_at=task_run.ended_at, ) @@ -298,12 +286,12 @@ def execute(self, plan: TestPlan) -> PlanExecutionResult: experiment_id=experiment_id, command=f"{task_run.task.tool}:{sample.metric_name}", result={ - "stage_name": sample.stage_name, - "subsystem": sample.subsystem, - "metric_name": sample.metric_name, "value": sample.value, }, - description="Observed numeric metric", + description=( + f"Observed numeric metric stage={sample.stage_name} " + f"subsystem={sample.subsystem}" + ), started_at=task_run.started_at, ended_at=task_run.ended_at, ) @@ -427,7 +415,6 @@ def _retry_stress_run_if_needed( if exec_res.returncode != _STRESS_RETRY_RETURN_CODE: return None - retry_args = build_stress_retry_args(args) retry_timeout = max(5, int(timeout_sec * 0.5)) logger.info( @@ -440,7 +427,7 @@ def _retry_stress_run_if_needed( exec_res2, (metrics2, summary2) = stress.run( timeout_sec=retry_timeout, - **retry_args, + **args, ) if exec_res2.returncode == 0: return exec_res2, metrics2, summary2 diff --git a/src/imgtests/planning/fio_sizing.py b/src/imgtests/sizing.py similarity index 79% rename from src/imgtests/planning/fio_sizing.py rename to src/imgtests/sizing.py index 80ca38a3..be3ebb6e 100644 --- a/src/imgtests/planning/fio_sizing.py +++ b/src/imgtests/sizing.py @@ -10,15 +10,19 @@ def parse_size_to_bytes(value: str) -> int | None: match suffix: case "k": multiplier = 1024 + number = normalized[:-1] case "m": multiplier = 1024**2 + number = normalized[:-1] case "g": multiplier = 1024**3 + number = normalized[:-1] case "t": multiplier = 1024**4 + number = normalized[:-1] case _: multiplier = 1 - number = normalized[:-1] + number = normalized try: return int(float(number) * multiplier) From aad2bd304d8b0af440a43027305d5b35d517aa0a Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Mon, 23 Mar 2026 21:02:07 +0300 Subject: [PATCH 15/29] fix remarks --- src/imgtests/environment.py | 6 +- src/imgtests/exec/loaders/fio.py | 38 +++- src/imgtests/planning/executor.py | 74 ++----- src/imgtests/planning/models.py | 8 +- src/imgtests/planning/plan_builder.py | 9 +- src/imgtests/planning/profiles.py | 7 +- src/imgtests/runner.py | 263 +++++++++++++++++++++++- src/imgtests/sizing.py | 15 +- tests/image/run_profiled_plan.py | 274 -------------------------- tests/image/runner.py | 12 +- 10 files changed, 352 insertions(+), 354 deletions(-) delete mode 100644 tests/image/run_profiled_plan.py diff --git a/src/imgtests/environment.py b/src/imgtests/environment.py index d5707d8d..05721b69 100644 --- a/src/imgtests/environment.py +++ b/src/imgtests/environment.py @@ -2,9 +2,13 @@ from pathlib import Path from typing import cast +_MISSING = object() -def env_var_to_type[T](variable: str, val_type: type[T]) -> T: + +def env_var_to_type[T](variable: str, val_type: type[T], default: T | object = _MISSING) -> T: if variable not in os.environ: + if default is not _MISSING: + return cast("T", default) err_msg = f"Environment variable '{variable}' not found." raise ValueError(err_msg) diff --git a/src/imgtests/exec/loaders/fio.py b/src/imgtests/exec/loaders/fio.py index 54e34445..26b11570 100644 --- a/src/imgtests/exec/loaders/fio.py +++ b/src/imgtests/exec/loaders/fio.py @@ -1,7 +1,8 @@ +from contextlib import suppress from typing import TYPE_CHECKING, Any, Literal from imgtests.exec.base_util import GenericUtil -from imgtests.exec.exec import ExecResult, SSHClient +from imgtests.exec.exec import ExecResult, SSHClient, common_run_command from imgtests.exec.pkgmgrs.mixin import PkgMgrMixin from imgtests.exec.pkgmgrs.pip3 import Pip3 from imgtests.exec.utils import create_opt @@ -28,8 +29,13 @@ # fmt: on Direct = Literal[1] | None +_DF_AVAIL_COLUMN_INDEX = 3 +_DF_MIN_COLUMNS = 4 + class Fio(PkgMgrMixin, GenericUtil): + DEFAULT_WORKDIR = "/var/lib/imgtests-fio" + def __init__(self, ssh_client: SSHClient | None = None) -> None: super().__init__("fio", ssh_client) @@ -53,6 +59,36 @@ def ioengines(self) -> tuple[str, ...] | None: lines = lines[1:] return tuple(line.strip() for line in lines) + def ensure_default_workdir(self) -> str: + common_run_command(["mkdir", "-p", self.DEFAULT_WORKDIR], self.ssh_client) + return self.DEFAULT_WORKDIR + + def default_filename(self, filename: str) -> str: + return f"{self.ensure_default_workdir()}/{filename}" + + def available_bytes(self, path: str | None = None) -> int | None: + target_path = path or self.ensure_default_workdir() + with suppress(Exception): + res = common_run_command(["df", "-PB1", target_path], self.ssh_client) + if res.returncode != 0: + return None + + out = (res.stdout or "").strip().splitlines() + if not out: + return None + + last = out[-1].split() + if len(last) < _DF_MIN_COLUMNS: + return None + + return int(last[_DF_AVAIL_COLUMN_INDEX]) + + return None + + def cleanup_file(self, path: str) -> None: + with suppress(Exception): + common_run_command(["rm", "-f", path], self.ssh_client) + def run( # noqa: PLR0913 self, name: str | None = None, diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py index 7c10b628..126f8e00 100644 --- a/src/imgtests/planning/executor.py +++ b/src/imgtests/planning/executor.py @@ -4,16 +4,14 @@ import logging import time from concurrent.futures import ThreadPoolExecutor, as_completed -from contextlib import suppress from dataclasses import dataclass from datetime import UTC, datetime from typing import TYPE_CHECKING, Any from imgtests.exec.loaders.fio import Fio from imgtests.exec.loaders.stress_ng import StressNg -from imgtests.runner import Subsystem +from imgtests.runner import BaseRunner, Subsystem from imgtests.sizing import bytes_to_mib_str, parse_size_to_bytes -from imgtests.sysrep import get_system_info if TYPE_CHECKING: from pathlib import Path @@ -24,10 +22,6 @@ logger = logging.getLogger(__name__) -_DEFAULT_FIO_WORKDIR = "/var/lib/imgtests-fio" - -_DF_AVAIL_COLUMN_INDEX = 3 -_DF_MIN_COLUMNS = 4 _STRESS_RETRY_RETURN_CODE = 3 @@ -89,25 +83,6 @@ def _try_parse_json(text: str) -> dict[str, Any]: return {} -def _remote_df_avail_bytes(client: SSHClient, path: str) -> int | None: - with suppress(Exception): - res = client(("df", "-PB1", path)) - if getattr(res, "returncode", 1) != 0: - return None - - out = (res.stdout or "").strip().splitlines() - if not out: - return None - - last = out[-1].split() - if len(last) < _DF_MIN_COLUMNS: - return None - - return int(last[_DF_AVAIL_COLUMN_INDEX]) - - return None - - def _stress_metric_samples( stage_name: str, subsystem: str, @@ -186,38 +161,38 @@ def _fio_op_samples( return out -class PlanExecutor: +class PlanExecutor(BaseRunner): def __init__( self, client: SSHClient, db: ImgtestsDatabase, - results_dir: Path, - config_id: int | None = None, - experiment_description: str | None = None, ) -> None: self.client = client self.db = db - self.results_dir = results_dir - self.config_id = config_id - self.experiment_description = experiment_description or "Auto-generated load test plan" - self.fio_workdir = _DEFAULT_FIO_WORKDIR - def execute(self, plan: TestPlan) -> PlanExecutionResult: - self.results_dir.mkdir(parents=True, exist_ok=True) + def execute( + self, + plan: TestPlan, + *, + results_dir: Path, + experiment_description: str, + config_id: int | None = None, + ) -> PlanExecutionResult: + results_dir.mkdir(parents=True, exist_ok=True) started_at = datetime.now(UTC) - plan_path = self.results_dir / f"plan_{plan.plan_id}.json" + plan_path = results_dir / f"plan_{plan.plan_id}.json" plan_path.write_text( json.dumps(plan.to_dict(), ensure_ascii=False, indent=2), encoding="utf-8", ) - cfg_id = self._ensure_config_id() - - experiment = self.db.insert_experiment( - config_id=cfg_id, - description=self.experiment_description, + experiment = self.start_experiment( + client=self.client, + database=self.db, + description=experiment_description, experiment_type=_resolve_experiment_type(plan), + config_id=config_id, started_at=started_at, ended_at=started_at, ) @@ -320,13 +295,6 @@ def execute(self, plan: TestPlan) -> PlanExecutionResult: metrics=tuple(collected_metrics), ) - def _ensure_config_id(self) -> int: - if self.config_id is not None: - return int(self.config_id) - sys_info = get_system_info(self.client) - cfg = self.db.insert_from_system_info(sys_info) - return int(cfg.config_id) - def _wait_for_stage_offset( self, plan_started_at: datetime, @@ -486,7 +454,6 @@ def _run_fio( started_at: datetime, ) -> TaskRunResult: fio = Fio(self.client) - self.client(("mkdir", "-p", self.fio_workdir)) args = dict(task.args or {}) runtime_sec = int(args.pop("runtime_sec", stage.duration_sec)) @@ -494,12 +461,12 @@ def _run_fio( created_filename: str | None = None if "filename" not in args and "directory" not in args: subsystem = getattr(task.subsystem, "value", str(task.subsystem)) - created_filename = f"{self.fio_workdir}/{stage.name}-{subsystem}.dat" + created_filename = fio.default_filename(f"{stage.name}-{subsystem}.dat") args["filename"] = created_filename if "size" in args: req = parse_size_to_bytes(str(args["size"])) - avail = _remote_df_avail_bytes(self.client, self.fio_workdir) + avail = fio.available_bytes() if created_filename is not None else None if req is not None and avail is not None: cap = max(64 * 1024**2, int(avail * 0.25)) safe = min(req, cap) @@ -523,8 +490,7 @@ def _run_fio( ) finally: if created_filename: - with suppress(Exception): - self.client(("rm", "-f", created_filename)) + fio.cleanup_file(created_filename) ended_at = datetime.now(UTC) diff --git a/src/imgtests/planning/models.py b/src/imgtests/planning/models.py index d010c3e2..6c788cad 100644 --- a/src/imgtests/planning/models.py +++ b/src/imgtests/planning/models.py @@ -63,7 +63,7 @@ def to_dict(self) -> dict[str, Any]: @dataclass(frozen=True) class PlanRequest: duration_sec: int - subsystems: tuple[Subsystem, ...] + subsystems: frozenset[Subsystem] test_kind: TestKind pattern: LoadPattern | None = None @@ -73,14 +73,14 @@ class TestPlan: plan_id: str created_at: datetime duration_sec: int - subsystems: tuple[Subsystem, ...] + subsystems: frozenset[Subsystem] test_kind: TestKind stages: tuple[PlanStage, ...] @staticmethod def new( duration_sec: int, - subsystems: tuple[Subsystem, ...], + subsystems: frozenset[Subsystem], test_kind: TestKind, stages: tuple[PlanStage, ...], ) -> TestPlan: @@ -98,7 +98,7 @@ def to_dict(self) -> dict[str, Any]: "plan_id": self.plan_id, "created_at": self.created_at.isoformat(), "duration_sec": self.duration_sec, - "subsystems": [x.value for x in self.subsystems], + "subsystems": [x.value for x in sorted(self.subsystems, key=lambda item: item.value)], "test_kind": self.test_kind.value, "stages": [stage.to_dict() for stage in self.stages], } diff --git a/src/imgtests/planning/plan_builder.py b/src/imgtests/planning/plan_builder.py index 09a00e95..e4b4a2a2 100644 --- a/src/imgtests/planning/plan_builder.py +++ b/src/imgtests/planning/plan_builder.py @@ -27,18 +27,19 @@ def build_plan(request: PlanRequest) -> TestPlan: msg = "At least one subsystem must be provided." raise ValueError(msg) + ordered_subsystems = tuple(sorted(request.subsystems, key=lambda item: item.value)) stages: list[PlanStage] = [] if request.pattern is not None: if request.test_kind == TestKind.ISOLATED: stages = _build_isolated_stages( request.duration_sec, - request.subsystems, + ordered_subsystems, pattern=request.pattern, ) else: tasks = tuple( - build_task(s, request.pattern, request.duration_sec) for s in request.subsystems + build_task(s, request.pattern, request.duration_sec) for s in ordered_subsystems ) stages = [ PlanStage( @@ -50,7 +51,7 @@ def build_plan(request: PlanRequest) -> TestPlan: ) ] elif request.test_kind == TestKind.ISOLATED: - stages = _build_isolated_stages(request.duration_sec, request.subsystems) + stages = _build_isolated_stages(request.duration_sec, ordered_subsystems) else: templates = PROFILE_LAYOUTS[request.test_kind] durations = _allocate_durations( @@ -60,7 +61,7 @@ def build_plan(request: PlanRequest) -> TestPlan: offset = 0 for tpl, dur in zip(templates, durations, strict=True): - tasks = tuple(build_task(s, tpl.pattern, dur) for s in request.subsystems) + tasks = tuple(build_task(s, tpl.pattern, dur) for s in ordered_subsystems) stages.append( PlanStage( name=tpl.name, diff --git a/src/imgtests/planning/profiles.py b/src/imgtests/planning/profiles.py index b1aa30ea..e17abaab 100644 --- a/src/imgtests/planning/profiles.py +++ b/src/imgtests/planning/profiles.py @@ -144,8 +144,11 @@ def build_task(subsystem: Subsystem, pattern: LoadPattern, stage_duration_sec: i def build_stage_tasks( _test_kind: TestKind, - subsystems: tuple[Subsystem, ...], + subsystems: frozenset[Subsystem], pattern: LoadPattern, stage_duration_sec: int, ) -> tuple[LoadTask, ...]: - return tuple(build_task(ss, pattern, stage_duration_sec) for ss in subsystems) + return tuple( + build_task(ss, pattern, stage_duration_sec) + for ss in sorted(subsystems, key=lambda item: item.value) + ) diff --git a/src/imgtests/runner.py b/src/imgtests/runner.py index ea0717b4..7d93c445 100644 --- a/src/imgtests/runner.py +++ b/src/imgtests/runner.py @@ -1,18 +1,22 @@ +from __future__ import annotations + import logging from abc import ABC, abstractmethod from concurrent.futures import ThreadPoolExecutor from datetime import datetime from enum import Enum +from pathlib import Path from threading import Event, Thread -from typing import TYPE_CHECKING, Any, NamedTuple +from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple from zoneinfo import ZoneInfo import paramiko import paramiko.ssh_exception from imgtests.constant import LIB_NAME -from imgtests.database.database import ImgtestsDatabase -from imgtests.exec.exec import common_run_command +from imgtests.database.database import ExperimentType, ImgtestsDatabase +from imgtests.environment import env_var_to_type +from imgtests.exec.exec import SSHClient, common_run_command from imgtests.exec.observers.journalctl import Journalctl from imgtests.exec.observers.systemctl import Systemctl from imgtests.sysrep import get_system_info @@ -20,9 +24,9 @@ if TYPE_CHECKING: from collections.abc import Iterable - from imgtests.database.database import ExperimentType + from imgtests.database.models.experiment import ExperimentBase from imgtests.exec.base_util import BaseTestUtil - from imgtests.exec.exec import SSHClient + from imgtests.planning import LoadPattern, TestKind class Subsystem(str, Enum): @@ -153,7 +157,44 @@ class TestsRunnerConfig(NamedTuple): install_dependencies: bool = False -class TestsRunner: +class BaseRunner: + __slots__ = () + + @staticmethod + def resolve_config_id( + client: SSHClient | None, + database: ImgtestsDatabase, + config_id: int | None = None, + ) -> int: + if config_id is not None: + return int(config_id) + + result = get_system_info(client) + configuration_record = database.insert_from_system_info(result) + return int(configuration_record.config_id) + + @classmethod + def start_experiment( # noqa: PLR0913 + cls, + *, + client: SSHClient | None, + database: ImgtestsDatabase, + description: str, + experiment_type: ExperimentType, + config_id: int | None = None, + started_at: datetime | None = None, + ended_at: datetime | None = None, + ) -> ExperimentBase: + return database.insert_experiment( + config_id=cls.resolve_config_id(client, database, config_id), + description=description, + experiment_type=experiment_type, + started_at=started_at, + ended_at=ended_at, + ) + + +class TestsRunner(BaseRunner): __slots__ = ("__client", "__database", "__executor", "__test_config", "logger") def __init__(self, client: SSHClient | None, test_config: TestsRunnerConfig) -> None: @@ -167,10 +208,9 @@ def run(self) -> None: test_completed_event = Event() if self.__test_config.install_dependencies: self.install_dependencies() - result = get_system_info(self.__client) - configuration_record = self.__database.insert_from_system_info(result) - experiment = self.__database.insert_experiment( - config_id=configuration_record.config_id, + experiment = self.start_experiment( + client=self.__client, + database=self.__database, description=self.__test_config.description, experiment_type=self.__test_config.experiment_type, ) @@ -304,3 +344,206 @@ def _collect_system_errors(self, experiment_id: int, since: datetime, until: dat ended_at=until, result=journalctl.metrics_to_json(sstmd_err_m), ) + + +class ProfiledPlanRunner(BaseRunner): + __slots__ = ("client", "db", "executor") + + _SUBSYSTEM_ALIASES: ClassVar[dict[str, Subsystem]] = { + "cpu": Subsystem.SYSTEM, + "disk": Subsystem.FILE, + "ipc": Subsystem.IPC, + } + + def __init__( + self, + client: SSHClient, + db: ImgtestsDatabase, + ) -> None: + from imgtests.planning.executor import PlanExecutor # noqa: PLC0415 + + self.client = client + self.db = db + self.executor = PlanExecutor(client=client, db=db) + + def run_from_env(self) -> int: + subsystems = self._parse_subsystems(env_var_to_type("PLAN_SUBSYSTEMS", str, "all")) + results_root = env_var_to_type("PLAN_RESULTS_DIR", Path, Path("results/profiled")) + pattern = self._parse_pattern(env_var_to_type("PLAN_PATTERN", str, "")) + config_id = self.resolve_config_id(self.client, self.db) + + if env_var_to_type("PLAN_RUN_MATRIX", bool, default=False): + return self._run_matrix( + subsystems=subsystems, + results_root=results_root, + pattern=pattern, + config_id=config_id, + ) + + failures = self._run_one( + profile=self._parse_profile(env_var_to_type("PLAN_PROFILE", str, "load")), + duration_sec=env_var_to_type("PLAN_DURATION_SEC", int, 120), + subsystems=subsystems, + results_root=results_root, + pattern=pattern, + config_id=config_id, + ) + return 1 if failures else 0 + + def _run_matrix( + self, + *, + subsystems: frozenset[Subsystem], + results_root: Path, + pattern: LoadPattern | None, + config_id: int, + ) -> int: + total_failures = 0 + default_duration = env_var_to_type("PLAN_DURATION_SEC", int, 120) + + for index, profile in enumerate( + self._parse_profiles(env_var_to_type("PLAN_MATRIX_PROFILES", str, "all")) + ): + if index > 0: + self.client.reconnect() + + duration_sec = env_var_to_type( + f"PLAN_DURATION_{profile.value.upper()}", + int, + default_duration, + ) + total_failures += self._run_one( + profile=profile, + duration_sec=duration_sec, + subsystems=subsystems, + results_root=results_root, + pattern=pattern, + config_id=config_id, + ) + + return 1 if total_failures else 0 + + def _run_one( # noqa: PLR0913 + self, + *, + profile: TestKind, + duration_sec: int, + subsystems: frozenset[Subsystem], + results_root: Path, + pattern: LoadPattern | None, + config_id: int, + ) -> int: + from imgtests.planning import PlanRequest, build_plan # noqa: PLC0415 + + execution = self.executor.execute( + build_plan( + PlanRequest( + duration_sec=duration_sec, + subsystems=subsystems, + test_kind=profile, + pattern=pattern, + ) + ), + results_dir=results_root / self._build_run_name(profile, pattern), + experiment_description=f"Profiled plan: {profile.value}", + config_id=config_id, + ) + failures = sum( + 1 for stage in execution.stage_runs for task in stage.tasks if task.returncode != 0 + ) + + logging.getLogger(__name__).info( + "[PROFILED] DONE profile=%s pattern=%s duration=%ss failures=%d experiment_id=%s", + profile.value, + pattern.value if pattern else "auto", + duration_sec, + failures, + execution.experiment_id, + ) + logging.getLogger(__name__).info("[PROFILED] plan=%s", execution.plan_path) + return failures + + @staticmethod + def _build_run_name(profile: TestKind, pattern: LoadPattern | None) -> str: + run_name = datetime.now(tz=ZoneInfo("UTC")).strftime("%Y%m%d_%H%M%S") + run_name = f"{run_name}_{profile.value}" + if pattern is not None: + run_name += f"_{pattern.value}" + return run_name + + @classmethod + def _parse_subsystems(cls, raw: str) -> frozenset[Subsystem]: + value = raw.strip().lower() + if value == "all": + return frozenset(Subsystem) + + subsystems: set[Subsystem] = set() + for part in [item.strip().lower() for item in value.split(",") if item.strip()]: + if part in cls._SUBSYSTEM_ALIASES: + subsystems.add(cls._SUBSYSTEM_ALIASES[part]) + continue + + try: + subsystems.add(Subsystem(part)) + except ValueError as exc: + allowed = ", ".join( + sorted( + [subsystem.value for subsystem in Subsystem] + list(cls._SUBSYSTEM_ALIASES), + ) + ) + msg = f"Unknown subsystem '{part}'. Allowed: {allowed}" + raise ValueError(msg) from exc + + if not subsystems: + msg = "No subsystems provided." + raise ValueError(msg) + + return frozenset(subsystems) + + @staticmethod + def _parse_profile(raw: str) -> TestKind: + from imgtests.planning import TestKind # noqa: PLC0415 + + try: + return TestKind(raw.strip().lower()) + except ValueError as exc: + allowed = ", ".join(item.value for item in TestKind) + msg = f"Unknown profile '{raw}'. Allowed: {allowed}" + raise ValueError(msg) from exc + + @classmethod + def _parse_profiles(cls, raw: str) -> tuple[TestKind, ...]: + from imgtests.planning import TestKind # noqa: PLC0415 + + value = raw.strip().lower() + if value in {"", "all"}: + return tuple(TestKind) + + profiles: list[TestKind] = [] + seen: set[TestKind] = set() + for part in [item.strip() for item in value.split(",") if item.strip()]: + profile = cls._parse_profile(part) + if profile not in seen: + profiles.append(profile) + seen.add(profile) + + if not profiles: + msg = "No profiles provided." + raise ValueError(msg) + + return tuple(profiles) + + @staticmethod + def _parse_pattern(raw: str) -> LoadPattern | None: + from imgtests.planning import LoadPattern # noqa: PLC0415 + + value = raw.strip().lower() + if value in {"", "auto"}: + return None + + try: + return LoadPattern(value) + except ValueError as exc: + allowed = ", ".join(item.value for item in LoadPattern) + msg = f"Unknown pattern '{raw}'. Allowed: {allowed}" + raise ValueError(msg) from exc diff --git a/src/imgtests/sizing.py b/src/imgtests/sizing.py index be3ebb6e..24ff92df 100644 --- a/src/imgtests/sizing.py +++ b/src/imgtests/sizing.py @@ -1,11 +1,19 @@ from __future__ import annotations +import re + +_SIZE_RE = re.compile(r"^(?P\d+(?:\.\d+)?)(?P[kKmMgGtT]?)$") + def parse_size_to_bytes(value: str) -> int | None: normalized = str(value).strip() - if not normalized or normalized.endswith("%"): + if not normalized: return None + if _SIZE_RE.fullmatch(normalized) is None: + err_msg = f"Invalid size value '{value}'." + raise ValueError(err_msg) + suffix = normalized[-1].lower() match suffix: case "k": @@ -26,8 +34,9 @@ def parse_size_to_bytes(value: str) -> int | None: try: return int(float(number) * multiplier) - except (TypeError, ValueError): - return None + except (TypeError, ValueError) as exc: + err_msg = f"Invalid size value '{value}'." + raise ValueError(err_msg) from exc def bytes_to_mib_str(size_bytes: int) -> str: diff --git a/tests/image/run_profiled_plan.py b/tests/image/run_profiled_plan.py deleted file mode 100644 index 460c1dcd..00000000 --- a/tests/image/run_profiled_plan.py +++ /dev/null @@ -1,274 +0,0 @@ -from __future__ import annotations - -import logging -import os -import sys -from contextlib import contextmanager, suppress -from dataclasses import dataclass -from datetime import UTC, datetime -from pathlib import Path -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from collections.abc import Iterator - -import paramiko.ssh_exception - -from imgtests.database.database import ImgtestsDatabase -from imgtests.exec.exec import wait_remote -from imgtests.logger import set_handlers -from imgtests.planning import LoadPattern, PlanRequest, Subsystem, TestKind, build_plan -from imgtests.planning.executor import PlanExecutor - -logger = logging.getLogger() -set_handlers(logger, Path("processing.log")) - -yocto_conf = ( - "SSH_YOCTO_ADDR", - "SSH_YOCTO_USER", - "SSH_YOCTO_PASS", - "SSH_YOCTO_PORT", -) - -PROFILE_ORDER: tuple[TestKind, ...] = ( - TestKind.LOAD, - TestKind.STRESS, - TestKind.STABILITY, - TestKind.SCALABILITY, - TestKind.VOLUME, - TestKind.ISOLATED, - TestKind.SPIKE, -) - - -@dataclass(frozen=True) -class RunOneParams: - profile: TestKind - duration_sec: int - subsystems: tuple[Subsystem, ...] - pattern: LoadPattern | None - results_root: Path - - -def parse_bool_env(name: str, default: bool) -> bool: - raw = os.getenv(name) - if raw is None: - return default - return raw.strip().lower() in {"1", "true", "yes", "on"} - - -def parse_subsystems(raw: str) -> tuple[Subsystem, ...]: - value = raw.strip().lower() - if value == "all": - return tuple(Subsystem) - - mapping: dict[str, Subsystem] = { - "system": Subsystem.SYSTEM, - "cpu": Subsystem.SYSTEM, - "file": Subsystem.FILE, - "disk": Subsystem.FILE, - "ipc": Subsystem.IPC, - "memory": Subsystem.MEMORY, - "network": Subsystem.NETWORK, - "syscalls": Subsystem.SYSCALLS, - "syscall": Subsystem.SYSCALLS, - } - - out: list[Subsystem] = [] - seen: set[Subsystem] = set() - - for part in [x.strip().lower() for x in value.split(",") if x.strip()]: - if part not in mapping: - allowed = ", ".join(mapping) - msg = f"Unknown subsystem '{part}'. Allowed: {allowed}" - raise ValueError(msg) - - subsystem = mapping[part] - if subsystem not in seen: - out.append(subsystem) - seen.add(subsystem) - - if not out: - msg = "No subsystems provided." - raise ValueError(msg) - - return tuple(out) - - -def parse_profile(raw: str) -> TestKind: - try: - return TestKind(raw.strip().lower()) - except ValueError as exc: - allowed = ", ".join(x.value for x in TestKind) - msg = f"Unknown profile '{raw}'. Allowed: {allowed}" - raise ValueError(msg) from exc - - -def parse_profiles(raw: str) -> tuple[TestKind, ...]: - value = raw.strip().lower() - if value in {"", "all"}: - return PROFILE_ORDER - - out: list[TestKind] = [] - seen: set[TestKind] = set() - - for part in [x.strip() for x in value.split(",") if x.strip()]: - profile = parse_profile(part) - if profile not in seen: - out.append(profile) - seen.add(profile) - - if not out: - msg = "No profiles provided." - raise ValueError(msg) - - return tuple(out) - - -def parse_pattern(raw: str | None) -> LoadPattern | None: - if raw is None: - return None - - value = raw.strip().lower() - if value in {"", "auto"}: - return None - - try: - return LoadPattern(value) - except ValueError as exc: - allowed = ", ".join(x.value for x in LoadPattern) - msg = f"Unknown pattern '{raw}'. Allowed: {allowed}" - raise ValueError(msg) from exc - - -def run_one( - *, - client: Any, - db: ImgtestsDatabase, - params: RunOneParams, -) -> tuple[int, Path]: - ts = datetime.now(UTC).strftime("%Y%m%d_%H%M%S") - run_name = f"{ts}_{params.profile.value}" - if params.pattern is not None: - run_name += f"_{params.pattern.value}" - - run_dir = params.results_root / run_name - run_dir.mkdir(parents=True, exist_ok=True) - - req = PlanRequest( - duration_sec=params.duration_sec, - subsystems=params.subsystems, - test_kind=params.profile, - pattern=params.pattern, - ) - plan = build_plan(req) - - executor = PlanExecutor( - client=client, - db=db, - results_dir=run_dir, - experiment_description=f"Profiled plan: {params.profile.value}", - ) - execution = executor.execute(plan) - - failures = sum(1 for s in execution.stage_runs for t in s.tasks if t.returncode != 0) - - logger.info( - "[PROFILED] DONE profile=%s pattern=%s duration=%ss failures=%d experiment_id=%s", - params.profile.value, - params.pattern.value if params.pattern else "auto", - params.duration_sec, - failures, - execution.experiment_id, - ) - logger.info("[PROFILED] plan=%s", execution.plan_path) - - return failures - - -@contextmanager -def _safe_close_client(client: Any) -> Iterator[Any]: - try: - yield client - finally: - with suppress(Exception): - client.close() - - -def _run_main() -> int: - subsystems = parse_subsystems(os.getenv("PLAN_SUBSYSTEMS", "all")) - results_root = Path(os.getenv("PLAN_RESULTS_DIR", "results/profiled")) - run_matrix = parse_bool_env("PLAN_RUN_MATRIX", default=False) - - raw_pattern = os.getenv("PLAN_PATTERN") - pattern = parse_pattern(raw_pattern) - - client = wait_remote(*yocto_conf) - if client is None: - logger.error("Failed to connect to Yocto host via SSH.") - return 1 - - with _safe_close_client(client): - db = ImgtestsDatabase() - - total_failures = 0 - - if run_matrix: - profiles = parse_profiles(os.getenv("PLAN_MATRIX_PROFILES", "all")) - default_duration = int(os.getenv("PLAN_DURATION_SEC", "120")) - - for profile in profiles: - env_key = f"PLAN_DURATION_{profile.value.upper()}" - raw = os.getenv(env_key) - duration_sec = int(raw) if raw is not None else default_duration - - failures = run_one( - client=client, - db=db, - params=RunOneParams( - profile=profile, - duration_sec=duration_sec, - subsystems=subsystems, - pattern=pattern, - results_root=results_root, - ), - ) - total_failures += failures - else: - profile = parse_profile(os.getenv("PLAN_PROFILE", "load")) - duration_sec = int(os.getenv("PLAN_DURATION_SEC", "120")) - - failures = run_one( - client=client, - db=db, - params=RunOneParams( - profile=profile, - duration_sec=duration_sec, - subsystems=subsystems, - pattern=pattern, - results_root=results_root, - ), - ) - total_failures = failures - - return 1 if total_failures else 0 - - -def main() -> None: - try: - exit_code = _run_main() - except ValueError: - logger.exception("Invalid config") - exit_code = 2 - except paramiko.ssh_exception.SSHException: - logger.exception("SSH error during profiled execution.") - exit_code = 1 - except Exception: - logger.exception("Unexpected error during profiled execution.") - exit_code = 1 - - sys.exit(exit_code) - - -if __name__ == "__main__": - main() diff --git a/tests/image/runner.py b/tests/image/runner.py index 0a7e6a73..d3e6c044 100644 --- a/tests/image/runner.py +++ b/tests/image/runner.py @@ -24,9 +24,10 @@ StressNgParallelLoadTest, ) from image.performance.system import PTSSystemTest +from imgtests.database.database import ImgtestsDatabase from imgtests.exec.exec import wait_remote from imgtests.logger import set_handlers -from imgtests.runner import TestsRunner, TestsRunnerConfig +from imgtests.runner import ProfiledPlanRunner, TestsRunner, TestsRunnerConfig from imgtests.suites.general.joint_bench import JointBench from imgtests.suites.system import SystemLoadTimeTest, SystemSlowServicesTest @@ -85,6 +86,15 @@ def main() -> None: ) yocto_runner.run() + client = wait_remote(*yocto_conf) or sys.exit(1) + exit_code = ProfiledPlanRunner( + client=client, + db=ImgtestsDatabase(), + ).run_from_env() + client.close() + + sys.exit(exit_code) + if __name__ == "__main__": main() From b6acd98c72c010330fcba0ca4977b12340fda3a7 Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Sat, 28 Mar 2026 16:48:36 +0300 Subject: [PATCH 16/29] fix remarks --- src/imgtests/exec/loaders/fio.py | 18 +++++------------- src/imgtests/planning/executor.py | 5 ++++- src/imgtests/sizing.py | 2 +- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/imgtests/exec/loaders/fio.py b/src/imgtests/exec/loaders/fio.py index 26b11570..f24b141f 100644 --- a/src/imgtests/exec/loaders/fio.py +++ b/src/imgtests/exec/loaders/fio.py @@ -29,9 +29,6 @@ # fmt: on Direct = Literal[1] | None -_DF_AVAIL_COLUMN_INDEX = 3 -_DF_MIN_COLUMNS = 4 - class Fio(PkgMgrMixin, GenericUtil): DEFAULT_WORKDIR = "/var/lib/imgtests-fio" @@ -69,7 +66,10 @@ def default_filename(self, filename: str) -> str: def available_bytes(self, path: str | None = None) -> int | None: target_path = path or self.ensure_default_workdir() with suppress(Exception): - res = common_run_command(["df", "-PB1", target_path], self.ssh_client) + res = common_run_command( + ["df", "--output=avail", "--block-size=1", target_path], + self.ssh_client, + ) if res.returncode != 0: return None @@ -77,18 +77,10 @@ def available_bytes(self, path: str | None = None) -> int | None: if not out: return None - last = out[-1].split() - if len(last) < _DF_MIN_COLUMNS: - return None - - return int(last[_DF_AVAIL_COLUMN_INDEX]) + return int(out[-1].strip()) return None - def cleanup_file(self, path: str) -> None: - with suppress(Exception): - common_run_command(["rm", "-f", path], self.ssh_client) - def run( # noqa: PLR0913 self, name: str | None = None, diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py index 126f8e00..3ec90b61 100644 --- a/src/imgtests/planning/executor.py +++ b/src/imgtests/planning/executor.py @@ -4,12 +4,14 @@ import logging import time from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import suppress from dataclasses import dataclass from datetime import UTC, datetime from typing import TYPE_CHECKING, Any from imgtests.exec.loaders.fio import Fio from imgtests.exec.loaders.stress_ng import StressNg +from imgtests.exec.exec import common_run_command from imgtests.runner import BaseRunner, Subsystem from imgtests.sizing import bytes_to_mib_str, parse_size_to_bytes @@ -490,7 +492,8 @@ def _run_fio( ) finally: if created_filename: - fio.cleanup_file(created_filename) + with suppress(Exception): + common_run_command(["rm", "-f", created_filename], self.client) ended_at = datetime.now(UTC) diff --git a/src/imgtests/sizing.py b/src/imgtests/sizing.py index 24ff92df..b3d04a3d 100644 --- a/src/imgtests/sizing.py +++ b/src/imgtests/sizing.py @@ -41,4 +41,4 @@ def parse_size_to_bytes(value: str) -> int | None: def bytes_to_mib_str(size_bytes: int) -> str: mib = max(1, size_bytes // (1024**2)) - return f"{mib}M" + return f"{mib}MiB" From 1055b746d404b8226e7133ad4a9285b5123e569c Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Sat, 28 Mar 2026 16:52:54 +0300 Subject: [PATCH 17/29] fix pre-commit-check --- src/imgtests/planning/executor.py | 2 +- src/imgtests/runner.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py index 3ec90b61..a87e28e3 100644 --- a/src/imgtests/planning/executor.py +++ b/src/imgtests/planning/executor.py @@ -9,9 +9,9 @@ from datetime import UTC, datetime from typing import TYPE_CHECKING, Any +from imgtests.exec.exec import common_run_command from imgtests.exec.loaders.fio import Fio from imgtests.exec.loaders.stress_ng import StressNg -from imgtests.exec.exec import common_run_command from imgtests.runner import BaseRunner, Subsystem from imgtests.sizing import bytes_to_mib_str, parse_size_to_bytes diff --git a/src/imgtests/runner.py b/src/imgtests/runner.py index db0a9c3c..507764eb 100644 --- a/src/imgtests/runner.py +++ b/src/imgtests/runner.py @@ -4,8 +4,8 @@ from abc import ABC, abstractmethod from concurrent.futures import ThreadPoolExecutor from datetime import datetime -from pathlib import Path from enum import Enum, auto +from pathlib import Path from threading import Event, Thread from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple from zoneinfo import ZoneInfo From 2a16ef90aeee31128d425f2c6b4b1da2d1df8c0d Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Sat, 28 Mar 2026 16:57:06 +0300 Subject: [PATCH 18/29] fix pre-commit-check --- tests/image/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/image/runner.py b/tests/image/runner.py index c803cae7..a35e8352 100644 --- a/tests/image/runner.py +++ b/tests/image/runner.py @@ -104,7 +104,7 @@ def main() -> None: ) yocto_runner.run() - client = wait_remote(*yocto_conf) or sys.exit(1) + client = wait_remote(*YOCTO_CONF) or sys.exit(1) exit_code = ProfiledPlanRunner( client=client, db=ImgtestsDatabase(), From 36425397c225f8b06effb42e360a0415016fddc7 Mon Sep 17 00:00:00 2001 From: Artanias <43622365+Artanias@users.noreply.github.com> Date: Sat, 28 Mar 2026 21:52:43 +0300 Subject: [PATCH 19/29] refactor: more checks in the available_bytes function. --- src/imgtests/exec/loaders/fio.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/imgtests/exec/loaders/fio.py b/src/imgtests/exec/loaders/fio.py index f24b141f..ee0fa591 100644 --- a/src/imgtests/exec/loaders/fio.py +++ b/src/imgtests/exec/loaders/fio.py @@ -70,14 +70,17 @@ def available_bytes(self, path: str | None = None) -> int | None: ["df", "--output=avail", "--block-size=1", target_path], self.ssh_client, ) - if res.returncode != 0: + if res.returncode: return None out = (res.stdout or "").strip().splitlines() if not out: return None - return int(out[-1].strip()) + try: + return int(out[-1].strip()) + except (ValueError, IndexError): + return None return None From a480d1bf6c221011426b837afa2b3070538999bb Mon Sep 17 00:00:00 2001 From: Artanias <43622365+Artanias@users.noreply.github.com> Date: Sat, 28 Mar 2026 21:58:24 +0300 Subject: [PATCH 20/29] refactor: moves get_available_bytes functions from Fio class. --- src/imgtests/exec/loaders/fio.py | 42 +++++++++++++++---------------- src/imgtests/planning/executor.py | 8 ++++-- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/imgtests/exec/loaders/fio.py b/src/imgtests/exec/loaders/fio.py index ee0fa591..d309e6cb 100644 --- a/src/imgtests/exec/loaders/fio.py +++ b/src/imgtests/exec/loaders/fio.py @@ -30,6 +30,27 @@ Direct = Literal[1] | None +def get_available_bytes(client: SSHClient, path: str | None) -> int | None: + with suppress(Exception): + res = common_run_command( + ["df", "--output=avail", "--block-size=1", str(path)], + client, + ) + if res.returncode: + return None + + out = (res.stdout or "").strip().splitlines() + if not out: + return None + + try: + return int(out[-1].strip()) + except (ValueError, IndexError): + return None + + return None + + class Fio(PkgMgrMixin, GenericUtil): DEFAULT_WORKDIR = "/var/lib/imgtests-fio" @@ -63,27 +84,6 @@ def ensure_default_workdir(self) -> str: def default_filename(self, filename: str) -> str: return f"{self.ensure_default_workdir()}/{filename}" - def available_bytes(self, path: str | None = None) -> int | None: - target_path = path or self.ensure_default_workdir() - with suppress(Exception): - res = common_run_command( - ["df", "--output=avail", "--block-size=1", target_path], - self.ssh_client, - ) - if res.returncode: - return None - - out = (res.stdout or "").strip().splitlines() - if not out: - return None - - try: - return int(out[-1].strip()) - except (ValueError, IndexError): - return None - - return None - def run( # noqa: PLR0913 self, name: str | None = None, diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py index a87e28e3..44ef30eb 100644 --- a/src/imgtests/planning/executor.py +++ b/src/imgtests/planning/executor.py @@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Any from imgtests.exec.exec import common_run_command -from imgtests.exec.loaders.fio import Fio +from imgtests.exec.loaders.fio import Fio, get_available_bytes from imgtests.exec.loaders.stress_ng import StressNg from imgtests.runner import BaseRunner, Subsystem from imgtests.sizing import bytes_to_mib_str, parse_size_to_bytes @@ -468,7 +468,11 @@ def _run_fio( if "size" in args: req = parse_size_to_bytes(str(args["size"])) - avail = fio.available_bytes() if created_filename is not None else None + avail = ( + get_available_bytes(self.client, created_filename) + if created_filename is not None + else None + ) if req is not None and avail is not None: cap = max(64 * 1024**2, int(avail * 0.25)) safe = min(req, cap) From 183890312aff50305a355dac44f0841e4d9c261f Mon Sep 17 00:00:00 2001 From: Artanias <43622365+Artanias@users.noreply.github.com> Date: Sat, 28 Mar 2026 22:10:15 +0300 Subject: [PATCH 21/29] refactor: removes unnecessary logic. --- src/imgtests/exec/loaders/fio.py | 14 +++++++------- src/imgtests/planning/executor.py | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/imgtests/exec/loaders/fio.py b/src/imgtests/exec/loaders/fio.py index d309e6cb..b8eb9cf8 100644 --- a/src/imgtests/exec/loaders/fio.py +++ b/src/imgtests/exec/loaders/fio.py @@ -1,4 +1,5 @@ from contextlib import suppress +from pathlib import Path from typing import TYPE_CHECKING, Any, Literal from imgtests.exec.base_util import GenericUtil @@ -8,8 +9,6 @@ from imgtests.exec.utils import create_opt if TYPE_CHECKING: - from pathlib import Path - from imgtests.types import Version IOPattern = Literal[ @@ -30,7 +29,7 @@ Direct = Literal[1] | None -def get_available_bytes(client: SSHClient, path: str | None) -> int | None: +def get_available_bytes(client: SSHClient, path: str | Path) -> int | None: with suppress(Exception): res = common_run_command( ["df", "--output=avail", "--block-size=1", str(path)], @@ -52,7 +51,7 @@ def get_available_bytes(client: SSHClient, path: str | None) -> int | None: class Fio(PkgMgrMixin, GenericUtil): - DEFAULT_WORKDIR = "/var/lib/imgtests-fio" + DEFAULT_WORKDIR = Path("/var/lib/imgtests-fio") def __init__(self, ssh_client: SSHClient | None = None) -> None: super().__init__("fio", ssh_client) @@ -77,12 +76,13 @@ def ioengines(self) -> tuple[str, ...] | None: lines = lines[1:] return tuple(line.strip() for line in lines) - def ensure_default_workdir(self) -> str: - common_run_command(["mkdir", "-p", self.DEFAULT_WORKDIR], self.ssh_client) + @property + def workdir(self) -> Path: + common_run_command(["mkdir", "-p", str(self.DEFAULT_WORKDIR)], self.ssh_client) return self.DEFAULT_WORKDIR def default_filename(self, filename: str) -> str: - return f"{self.ensure_default_workdir()}/{filename}" + return f"{self.workdir}/{filename}" def run( # noqa: PLR0913 self, diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py index 44ef30eb..d5486001 100644 --- a/src/imgtests/planning/executor.py +++ b/src/imgtests/planning/executor.py @@ -460,11 +460,11 @@ def _run_fio( args = dict(task.args or {}) runtime_sec = int(args.pop("runtime_sec", stage.duration_sec)) - created_filename: str | None = None + created_filename: Path | None = None if "filename" not in args and "directory" not in args: subsystem = getattr(task.subsystem, "value", str(task.subsystem)) - created_filename = fio.default_filename(f"{stage.name}-{subsystem}.dat") - args["filename"] = created_filename + created_filename = fio.workdir / f"{stage.name}-{subsystem}.dat" + args["filename"] = str(created_filename) if "size" in args: req = parse_size_to_bytes(str(args["size"])) @@ -497,7 +497,7 @@ def _run_fio( finally: if created_filename: with suppress(Exception): - common_run_command(["rm", "-f", created_filename], self.client) + common_run_command(["rm", "-f", str(created_filename)], self.client) ended_at = datetime.now(UTC) From 0d5da2b9d1626d567e29ca3651789292b54b1905 Mon Sep 17 00:00:00 2001 From: Artanias <43622365+Artanias@users.noreply.github.com> Date: Sun, 29 Mar 2026 13:36:55 +0300 Subject: [PATCH 22/29] refactor: moves stress-ng error code 3 into StressNg class. --- src/imgtests/exec/loaders/stress_ng.py | 2 ++ src/imgtests/planning/executor.py | 6 ++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/imgtests/exec/loaders/stress_ng.py b/src/imgtests/exec/loaders/stress_ng.py index 977fc7ea..34704acd 100644 --- a/src/imgtests/exec/loaders/stress_ng.py +++ b/src/imgtests/exec/loaders/stress_ng.py @@ -69,6 +69,8 @@ class StressNGResult(NamedTuple): class StressNg(PkgMgrMixin, GenericUtil): + INITIALIZATION_FAILED_CODE = 3 # ENOMEM, ENOSPC, missing or unimplemented system call. + def __init__(self, ssh_client: SSHClient | None = None) -> None: super().__init__("stress-ng", ssh_client) diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py index d5486001..f7590ded 100644 --- a/src/imgtests/planning/executor.py +++ b/src/imgtests/planning/executor.py @@ -24,8 +24,6 @@ logger = logging.getLogger(__name__) -_STRESS_RETRY_RETURN_CODE = 3 - @dataclass(frozen=True) class MetricSample: @@ -382,7 +380,7 @@ def _retry_stress_run_if_needed( args: dict[str, Any], exec_res: Any, ) -> tuple[Any, list[Any], Any] | None: - if exec_res.returncode != _STRESS_RETRY_RETURN_CODE: + if exec_res.returncode != stress.INITIALIZATION_FAILED_CODE: return None retry_timeout = max(5, int(timeout_sec * 0.5)) @@ -390,7 +388,7 @@ def _retry_stress_run_if_needed( logger.info( "[PLAN] stress-ng retry stage=%s rc=%s timeout=%s->%s", stage_name, - _STRESS_RETRY_RETURN_CODE, + exec_res.returncode, timeout_sec, retry_timeout, ) From d4910cd414b6072ff433f5ad1eecf04f9acb302a Mon Sep 17 00:00:00 2001 From: Artanias <43622365+Artanias@users.noreply.github.com> Date: Sun, 29 Mar 2026 19:38:44 +0300 Subject: [PATCH 23/29] refactor: renames bytes_to_mib function. --- src/imgtests/planning/executor.py | 4 ++-- src/imgtests/sizing.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py index f7590ded..a80d39b1 100644 --- a/src/imgtests/planning/executor.py +++ b/src/imgtests/planning/executor.py @@ -13,7 +13,7 @@ from imgtests.exec.loaders.fio import Fio, get_available_bytes from imgtests.exec.loaders.stress_ng import StressNg from imgtests.runner import BaseRunner, Subsystem -from imgtests.sizing import bytes_to_mib_str, parse_size_to_bytes +from imgtests.sizing import parse_size_to_bytes, round_bytes_to_mib_str if TYPE_CHECKING: from pathlib import Path @@ -475,7 +475,7 @@ def _run_fio( cap = max(64 * 1024**2, int(avail * 0.25)) safe = min(req, cap) if safe < req: - args["size"] = bytes_to_mib_str(safe) + args["size"] = round_bytes_to_mib_str(safe) fio_name = args.pop( "name", diff --git a/src/imgtests/sizing.py b/src/imgtests/sizing.py index b3d04a3d..83fea0fd 100644 --- a/src/imgtests/sizing.py +++ b/src/imgtests/sizing.py @@ -39,6 +39,6 @@ def parse_size_to_bytes(value: str) -> int | None: raise ValueError(err_msg) from exc -def bytes_to_mib_str(size_bytes: int) -> str: +def round_bytes_to_mib_str(size_bytes: int) -> str: mib = max(1, size_bytes // (1024**2)) return f"{mib}MiB" From c83091a20f4222b8ff0a8a159db2713715869991 Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Mon, 30 Mar 2026 11:53:21 +0300 Subject: [PATCH 24/29] fix remarks --- src/imgtests/exec/loaders/fio.py | 85 ++++++++++++ src/imgtests/exec/loaders/stress_ng.py | 44 ++++++ src/imgtests/exec/metrics.py | 9 ++ src/imgtests/planning/executor.py | 180 +++---------------------- src/imgtests/runner.py | 1 - 5 files changed, 156 insertions(+), 163 deletions(-) create mode 100644 src/imgtests/exec/metrics.py diff --git a/src/imgtests/exec/loaders/fio.py b/src/imgtests/exec/loaders/fio.py index b8eb9cf8..7b0fb804 100644 --- a/src/imgtests/exec/loaders/fio.py +++ b/src/imgtests/exec/loaders/fio.py @@ -4,6 +4,7 @@ from imgtests.exec.base_util import GenericUtil from imgtests.exec.exec import ExecResult, SSHClient, common_run_command +from imgtests.exec.metrics import MetricSample from imgtests.exec.pkgmgrs.mixin import PkgMgrMixin from imgtests.exec.pkgmgrs.pip3 import Pip3 from imgtests.exec.utils import create_opt @@ -206,3 +207,87 @@ def version(self) -> Version | None: if package.name == self.name: return package.version return None + + +def fio_metrics_to_samples( + payload: dict[str, Any], + stage_name: str, + subsystem: str, +) -> list[MetricSample]: + jobs = payload.get("jobs", []) + if not isinstance(jobs, list): + return [] + + wanted_p = { + "50.000000": 50, + "90.000000": 90, + "95.000000": 95, + "99.000000": 99, + "99.900000": 999, + } + + out: list[MetricSample] = [] + for job in jobs: + if not isinstance(job, dict): + continue + + for op in ("read", "write", "trim"): + op_data = job.get(op, {}) + if not isinstance(op_data, dict): + continue + out.extend( + _fio_op_samples( + stage_name=stage_name, + subsystem=subsystem, + op=op, + op_data=op_data, + wanted_p=wanted_p, + ) + ) + + return out + + +def _fio_op_samples( + stage_name: str, + subsystem: str, + op: str, + op_data: dict[str, Any], + wanted_p: dict[str, int], +) -> list[MetricSample]: + out: list[MetricSample] = [] + + iops = _safe_float(op_data.get("iops")) + bw = _safe_float(op_data.get("bw")) + runtime_ms = _safe_float(op_data.get("runtime")) + + clat = op_data.get("clat_ns") or {} + clat_mean = _safe_float(clat.get("mean")) if isinstance(clat, dict) else None + + if iops is not None: + out.append(MetricSample(stage_name, subsystem, f"fio.{op}.iops", iops)) + if bw is not None: + out.append(MetricSample(stage_name, subsystem, f"fio.{op}.bw_kib_s", bw)) + if runtime_ms is not None: + out.append(MetricSample(stage_name, subsystem, f"fio.{op}.runtime_ms", runtime_ms)) + if clat_mean is not None: + out.append(MetricSample(stage_name, subsystem, f"fio.{op}.clat_mean_ns", clat_mean)) + + pct = clat.get("percentile") if isinstance(clat, dict) else None + if isinstance(pct, dict): + for key, p_int in wanted_p.items(): + fv = _safe_float(pct.get(key)) + if fv is not None: + out.append(MetricSample(stage_name, subsystem, f"fio.{op}.clat_p{p_int}_ns", fv)) + + return out + + +def _safe_float(value: Any) -> float | None: + if value is None: + return None + + try: + return float(value) + except (TypeError, ValueError): + return None diff --git a/src/imgtests/exec/loaders/stress_ng.py b/src/imgtests/exec/loaders/stress_ng.py index bcdb2040..bb204e4e 100644 --- a/src/imgtests/exec/loaders/stress_ng.py +++ b/src/imgtests/exec/loaders/stress_ng.py @@ -4,6 +4,7 @@ from imgtests.exec.base_util import GenericUtil from imgtests.exec.exec import ExecResult +from imgtests.exec.metrics import MetricSample from imgtests.exec.pkgmgrs.mixin import PkgMgrMixin from imgtests.exec.utils import add_flag, create_opt @@ -573,3 +574,46 @@ def metrics_to_json(metrics: StressNGResult) -> Any: "stress_ng_metrics": [metric._asdict() for metric in metrics.metrics], "stress_ng_summary": metrics.summary._asdict() if metrics.summary else None, } + + +def stress_metrics_to_samples( + stage_name: str, + subsystem: str, + metrics: list[StressNGMetrics], +) -> list[MetricSample]: + samples: list[MetricSample] = [] + + for metric in metrics: + base_metrics = ( + ("stress.bogo_ops", float(metric.bogo_ops)), + ("stress.real_time_secs", float(metric.real_time_secs)), + ("stress.usr_time_secs", float(metric.usr_time_secs)), + ("stress.sys_time_secs", float(metric.sys_time_secs)), + ("stress.bogo_ops_s_real_time", float(metric.bogo_ops_s_real_time)), + ("stress.bogo_ops_s_usr_sys_time", float(metric.bogo_ops_s_usr_sys_time)), + ("stress.cpu_used_per_instance", float(metric.cpu_used_per_instance)), + ) + for metric_name, value in base_metrics: + samples.append(MetricSample(stage_name, subsystem, metric_name, value)) + + if metric.rss_max_kb is not None: + samples.append( + MetricSample( + stage_name, + subsystem, + "stress.rss_max_kb", + float(metric.rss_max_kb), + ) + ) + + if metric.top10_slowest: + samples.append( + MetricSample( + stage_name, + subsystem, + "stress.syscall_slowest_avg_ns", + float(metric.top10_slowest[0].avg_ns), + ) + ) + + return samples diff --git a/src/imgtests/exec/metrics.py b/src/imgtests/exec/metrics.py new file mode 100644 index 00000000..d18fd293 --- /dev/null +++ b/src/imgtests/exec/metrics.py @@ -0,0 +1,9 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class MetricSample: + stage_name: str + subsystem: str + metric_name: str + value: float diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py index a80d39b1..ac957224 100644 --- a/src/imgtests/planning/executor.py +++ b/src/imgtests/planning/executor.py @@ -10,8 +10,9 @@ from typing import TYPE_CHECKING, Any from imgtests.exec.exec import common_run_command -from imgtests.exec.loaders.fio import Fio, get_available_bytes -from imgtests.exec.loaders.stress_ng import StressNg +from imgtests.exec.loaders.fio import Fio, fio_metrics_to_samples, get_available_bytes +from imgtests.exec.loaders.stress_ng import StressNg, stress_metrics_to_samples +from imgtests.exec.metrics import MetricSample from imgtests.runner import BaseRunner, Subsystem from imgtests.sizing import parse_size_to_bytes, round_bytes_to_mib_str @@ -25,14 +26,6 @@ logger = logging.getLogger(__name__) -@dataclass(frozen=True) -class MetricSample: - stage_name: str - subsystem: str - metric_name: str - value: float - - @dataclass(frozen=True) class TaskRunResult: task: LoadTask @@ -57,110 +50,12 @@ class StageRunResult: @dataclass(frozen=True) class PlanExecutionResult: experiment_id: int - plan_path: Path started_at: datetime ended_at: datetime stage_runs: tuple[StageRunResult, ...] metrics: tuple[MetricSample, ...] -def _safe_float(value: Any) -> float | None: - if value is None: - return None - try: - return float(value) - except (TypeError, ValueError): - return None - - -def _try_parse_json(text: str) -> dict[str, Any]: - if not text or not str(text).strip(): - return {} - try: - data = json.loads(str(text).strip()) - return data if isinstance(data, dict) else {} - except json.JSONDecodeError: - return {} - - -def _stress_metric_samples( - stage_name: str, - subsystem: str, - metrics: list[Any], -) -> list[MetricSample]: - samples: list[MetricSample] = [] - - for metric in metrics: - base_metrics = ( - ("stress.bogo_ops", float(metric.bogo_ops)), - ("stress.real_time_secs", float(metric.real_time_secs)), - ("stress.usr_time_secs", float(metric.usr_time_secs)), - ("stress.sys_time_secs", float(metric.sys_time_secs)), - ("stress.bogo_ops_s_real_time", float(metric.bogo_ops_s_real_time)), - ("stress.bogo_ops_s_usr_sys_time", float(metric.bogo_ops_s_usr_sys_time)), - ("stress.cpu_used_per_instance", float(metric.cpu_used_per_instance)), - ) - for metric_name, value in base_metrics: - samples.append(MetricSample(stage_name, subsystem, metric_name, value)) - - if metric.rss_max_kb is not None: - samples.append( - MetricSample( - stage_name, - subsystem, - "stress.rss_max_kb", - float(metric.rss_max_kb), - ) - ) - - if metric.top10_slowest: - samples.append( - MetricSample( - stage_name, - subsystem, - "stress.syscall_slowest_avg_ns", - float(metric.top10_slowest[0].avg_ns), - ) - ) - - return samples - - -def _fio_op_samples( - stage_name: str, - subsystem: str, - op: str, - op_data: dict[str, Any], - wanted_p: dict[str, int], -) -> list[MetricSample]: - out: list[MetricSample] = [] - - iops = _safe_float(op_data.get("iops")) - bw = _safe_float(op_data.get("bw")) - runtime_ms = _safe_float(op_data.get("runtime")) - - clat = op_data.get("clat_ns") or {} - clat_mean = _safe_float(clat.get("mean")) if isinstance(clat, dict) else None - - if iops is not None: - out.append(MetricSample(stage_name, subsystem, f"fio.{op}.iops", iops)) - if bw is not None: - out.append(MetricSample(stage_name, subsystem, f"fio.{op}.bw_kib_s", bw)) - if runtime_ms is not None: - out.append(MetricSample(stage_name, subsystem, f"fio.{op}.runtime_ms", runtime_ms)) - if clat_mean is not None: - out.append(MetricSample(stage_name, subsystem, f"fio.{op}.clat_mean_ns", clat_mean)) - - pct = clat.get("percentile") if isinstance(clat, dict) else None - if isinstance(pct, dict): - for key, p_int in wanted_p.items(): - fv = _safe_float(pct.get(key)) - if fv is not None: - out.append(MetricSample(stage_name, subsystem, f"fio.{op}.clat_p{p_int}_ns", fv)) - - return out - - class PlanExecutor(BaseRunner): def __init__( self, @@ -181,12 +76,6 @@ def execute( results_dir.mkdir(parents=True, exist_ok=True) started_at = datetime.now(UTC) - plan_path = results_dir / f"plan_{plan.plan_id}.json" - plan_path.write_text( - json.dumps(plan.to_dict(), ensure_ascii=False, indent=2), - encoding="utf-8", - ) - experiment = self.start_experiment( client=self.client, database=self.db, @@ -198,15 +87,6 @@ def execute( ) experiment_id = int(experiment.experiment_id) - self.db.insert_loader( - experiment_id=experiment_id, - command="plan.json", - result=plan.to_dict(), - description="Generated execution plan", - started_at=started_at, - ended_at=started_at, - ) - stage_runs: list[StageRunResult] = [] collected_metrics: list[MetricSample] = [] @@ -288,7 +168,6 @@ def execute( return PlanExecutionResult( experiment_id=experiment_id, - plan_path=plan_path, started_at=started_at, ended_at=ended_at, stage_runs=tuple(stage_runs), @@ -427,7 +306,7 @@ def _run_stress_ng( ended_at = datetime.now(UTC) subsystem = getattr(task.subsystem, "value", str(task.subsystem)) - samples = _stress_metric_samples(stage.name, subsystem, metrics) + samples = stress_metrics_to_samples(stage.name, subsystem, metrics) summary_dict = summary._asdict() if summary else None logger.info( @@ -500,7 +379,7 @@ def _run_fio( ended_at = datetime.now(UTC) payload = _try_parse_json(result.stdout) - samples = self._fio_samples( + samples = fio_metrics_to_samples( payload=payload, stage_name=stage.name, subsystem=getattr(task.subsystem, "value", str(task.subsystem)), @@ -523,44 +402,21 @@ def _run_fio( metrics=tuple(samples), ) - def _fio_samples( - self, - payload: dict[str, Any], - stage_name: str, - subsystem: str, - ) -> list[MetricSample]: - jobs = payload.get("jobs", []) - if not isinstance(jobs, list): - return [] - wanted_p = { - "50.000000": 50, - "90.000000": 90, - "95.000000": 95, - "99.000000": 99, - "99.900000": 999, - } - - out: list[MetricSample] = [] - for job in jobs: - if not isinstance(job, dict): - continue - - for op in ("read", "write", "trim"): - op_data = job.get(op, {}) - if not isinstance(op_data, dict): - continue - out.extend( - _fio_op_samples( - stage_name=stage_name, - subsystem=subsystem, - op=op, - op_data=op_data, - wanted_p=wanted_p, - ) - ) +def _try_parse_json(text: str) -> dict[str, Any]: + if not text: + return {} + + raw = str(text).strip() + if not raw: + return {} + + try: + data = json.loads(raw) + except json.JSONDecodeError: + return {} - return out + return data if isinstance(data, dict) else {} def _resolve_experiment_type(plan: TestPlan) -> ExperimentType: diff --git a/src/imgtests/runner.py b/src/imgtests/runner.py index f160aa1f..c517bf39 100644 --- a/src/imgtests/runner.py +++ b/src/imgtests/runner.py @@ -524,7 +524,6 @@ def _run_one( # noqa: PLR0913 failures, execution.experiment_id, ) - logging.getLogger(__name__).info("[PROFILED] plan=%s", execution.plan_path) return failures @staticmethod From 5eab76d9da8be6a994bcdc858fbf9012365c7115 Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Mon, 30 Mar 2026 11:56:16 +0300 Subject: [PATCH 25/29] fix pre-commit remarks --- src/imgtests/planning/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py index ac957224..61f629b0 100644 --- a/src/imgtests/planning/executor.py +++ b/src/imgtests/planning/executor.py @@ -12,7 +12,6 @@ from imgtests.exec.exec import common_run_command from imgtests.exec.loaders.fio import Fio, fio_metrics_to_samples, get_available_bytes from imgtests.exec.loaders.stress_ng import StressNg, stress_metrics_to_samples -from imgtests.exec.metrics import MetricSample from imgtests.runner import BaseRunner, Subsystem from imgtests.sizing import parse_size_to_bytes, round_bytes_to_mib_str @@ -21,6 +20,7 @@ from imgtests.database.database import ExperimentType, ImgtestsDatabase from imgtests.exec.exec import SSHClient + from imgtests.exec.metrics import MetricSample from imgtests.planning.models import LoadTask, PlanStage, TestPlan logger = logging.getLogger(__name__) From ad8a4f019c2de094a8b80e7d33f9acb513084080 Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Mon, 30 Mar 2026 15:53:56 +0300 Subject: [PATCH 26/29] fix remarks --- src/imgtests/planning/executor.py | 2 +- src/imgtests/planning/models.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py index 61f629b0..fbd99f46 100644 --- a/src/imgtests/planning/executor.py +++ b/src/imgtests/planning/executor.py @@ -346,7 +346,7 @@ def _run_fio( if "size" in args: req = parse_size_to_bytes(str(args["size"])) avail = ( - get_available_bytes(self.client, created_filename) + get_available_bytes(self.client, created_filename.parent) if created_filename is not None else None ) diff --git a/src/imgtests/planning/models.py b/src/imgtests/planning/models.py index 6c788cad..34845e00 100644 --- a/src/imgtests/planning/models.py +++ b/src/imgtests/planning/models.py @@ -3,11 +3,10 @@ from dataclasses import dataclass from datetime import UTC, datetime from enum import Enum -from typing import TYPE_CHECKING, Any +from typing import Any from uuid import uuid4 -if TYPE_CHECKING: - from imgtests.runner import Subsystem +from imgtests.runner import Subsystem # noqa: TC001 class TestKind(str, Enum): From d1b885f074024bf6b91d6ab002a37c1a34823da0 Mon Sep 17 00:00:00 2001 From: Artanias <43622365+Artanias@users.noreply.github.com> Date: Mon, 30 Mar 2026 22:14:44 +0300 Subject: [PATCH 27/29] refactor: moves Subsystem into the types module. --- src/imgtests/exec/loaders/fio.py | 6 +++--- src/imgtests/planning/executor.py | 5 +++-- src/imgtests/planning/models.py | 5 +++-- src/imgtests/planning/plan_builder.py | 3 ++- src/imgtests/planning/profiles.py | 3 ++- src/imgtests/runner.py | 11 +---------- src/imgtests/suites/general/joint_bench.py | 3 ++- src/imgtests/suites/system.py | 3 ++- src/imgtests/types.py | 9 +++++++++ tests/image/endurance/cpu.py | 3 ++- tests/image/endurance/disks.py | 3 ++- tests/image/endurance/memory.py | 3 ++- tests/image/endurance/network.py | 3 ++- tests/image/endurance/syscalls.py | 3 ++- tests/image/performance/cpu.py | 3 ++- tests/image/performance/fio_disks.py | 4 ++-- tests/image/performance/ipc.py | 3 ++- tests/image/performance/memory.py | 3 ++- tests/image/performance/network.py | 3 ++- tests/image/performance/std_utils.py | 3 ++- tests/image/performance/stress_ng_general.py | 3 ++- tests/image/performance/syscalls.py | 3 ++- tests/image/performance/system.py | 3 ++- 23 files changed, 55 insertions(+), 36 deletions(-) diff --git a/src/imgtests/exec/loaders/fio.py b/src/imgtests/exec/loaders/fio.py index 7b0fb804..2fe651b6 100644 --- a/src/imgtests/exec/loaders/fio.py +++ b/src/imgtests/exec/loaders/fio.py @@ -10,7 +10,7 @@ from imgtests.exec.utils import create_opt if TYPE_CHECKING: - from imgtests.types import Version + from imgtests.types import Subsystem, Version IOPattern = Literal[ "read", "write", "trim", "randread", "randwrite", "randtrim", "readwrite", "randrw", "trimwrite" @@ -212,7 +212,7 @@ def version(self) -> Version | None: def fio_metrics_to_samples( payload: dict[str, Any], stage_name: str, - subsystem: str, + subsystem: Subsystem, ) -> list[MetricSample]: jobs = payload.get("jobs", []) if not isinstance(jobs, list): @@ -238,7 +238,7 @@ def fio_metrics_to_samples( out.extend( _fio_op_samples( stage_name=stage_name, - subsystem=subsystem, + subsystem=subsystem.value, op=op, op_data=op_data, wanted_p=wanted_p, diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py index fbd99f46..aae5439b 100644 --- a/src/imgtests/planning/executor.py +++ b/src/imgtests/planning/executor.py @@ -12,8 +12,9 @@ from imgtests.exec.exec import common_run_command from imgtests.exec.loaders.fio import Fio, fio_metrics_to_samples, get_available_bytes from imgtests.exec.loaders.stress_ng import StressNg, stress_metrics_to_samples -from imgtests.runner import BaseRunner, Subsystem +from imgtests.runner import BaseRunner from imgtests.sizing import parse_size_to_bytes, round_bytes_to_mib_str +from imgtests.types import Subsystem if TYPE_CHECKING: from pathlib import Path @@ -382,7 +383,7 @@ def _run_fio( samples = fio_metrics_to_samples( payload=payload, stage_name=stage.name, - subsystem=getattr(task.subsystem, "value", str(task.subsystem)), + subsystem=task.subsystem, ) summary = None diff --git a/src/imgtests/planning/models.py b/src/imgtests/planning/models.py index 34845e00..9a8ca813 100644 --- a/src/imgtests/planning/models.py +++ b/src/imgtests/planning/models.py @@ -3,10 +3,11 @@ from dataclasses import dataclass from datetime import UTC, datetime from enum import Enum -from typing import Any +from typing import TYPE_CHECKING, Any from uuid import uuid4 -from imgtests.runner import Subsystem # noqa: TC001 +if TYPE_CHECKING: + from imgtests.types import Subsystem class TestKind(str, Enum): diff --git a/src/imgtests/planning/plan_builder.py b/src/imgtests/planning/plan_builder.py index e4b4a2a2..e096a1d3 100644 --- a/src/imgtests/planning/plan_builder.py +++ b/src/imgtests/planning/plan_builder.py @@ -6,7 +6,6 @@ LoadPattern, PlanRequest, PlanStage, - Subsystem, TestKind, TestPlan, ) @@ -15,6 +14,8 @@ if TYPE_CHECKING: from collections.abc import Iterable + from imgtests.types import Subsystem + _ALLOC_GUARD_LIMIT = 100_000 diff --git a/src/imgtests/planning/profiles.py b/src/imgtests/planning/profiles.py index e17abaab..9129dc47 100644 --- a/src/imgtests/planning/profiles.py +++ b/src/imgtests/planning/profiles.py @@ -3,7 +3,8 @@ from dataclasses import dataclass from typing import Any -from imgtests.planning.models import LoadPattern, LoadTask, Subsystem, TestKind +from imgtests.planning.models import LoadPattern, LoadTask, TestKind +from imgtests.types import Subsystem @dataclass(frozen=True) diff --git a/src/imgtests/runner.py b/src/imgtests/runner.py index c517bf39..5d161849 100644 --- a/src/imgtests/runner.py +++ b/src/imgtests/runner.py @@ -20,7 +20,7 @@ from imgtests.exec.observers.journalctl import Journalctl from imgtests.exec.observers.systemctl import Systemctl from imgtests.sysrep import get_system_info -from imgtests.types import TestsCounts +from imgtests.types import Subsystem, TestsCounts if TYPE_CHECKING: from collections.abc import Iterable @@ -30,15 +30,6 @@ from imgtests.planning import LoadPattern, TestKind -class Subsystem(str, Enum): - FILE = "file" - IPC = "IPC" - MEMORY = "memory" - NETWORK = "network" - SYSCALLS = "syscalls" - SYSTEM = "system" - - class TestStatus(Enum): PASSED = auto() FAILED = auto() diff --git a/src/imgtests/suites/general/joint_bench.py b/src/imgtests/suites/general/joint_bench.py index 8c597435..583cba63 100644 --- a/src/imgtests/suites/general/joint_bench.py +++ b/src/imgtests/suites/general/joint_bench.py @@ -5,7 +5,8 @@ from imgtests.exec.loaders.perf import Perf from imgtests.exec.loaders.pts import PhoronixTestSuite -from imgtests.runner import AbstractRunnableManyTimesTest, Subsystem, TestResult, TestStatus +from imgtests.runner import AbstractRunnableManyTimesTest, TestResult, TestStatus +from imgtests.types import Subsystem if TYPE_CHECKING: from collections.abc import Callable, Iterable diff --git a/src/imgtests/suites/system.py b/src/imgtests/suites/system.py index d7cb05ed..f49d6a10 100644 --- a/src/imgtests/suites/system.py +++ b/src/imgtests/suites/system.py @@ -2,7 +2,8 @@ from typing import TYPE_CHECKING from imgtests.exec.observers.systemd_analyze import SystemdAnalyze -from imgtests.runner import AbstractRunnableManyTimesTest, Subsystem, TestResult, TestStatus +from imgtests.runner import AbstractRunnableManyTimesTest, TestResult, TestStatus +from imgtests.types import Subsystem if TYPE_CHECKING: import logging diff --git a/src/imgtests/types.py b/src/imgtests/types.py index 35379f99..8673f72d 100644 --- a/src/imgtests/types.py +++ b/src/imgtests/types.py @@ -70,3 +70,12 @@ class TestsCounts(NamedTuple): passed_count: int = 0 failed_count: int = 0 skip_count: int = 0 + + +class Subsystem(str, Enum): + FILE = "file" + IPC = "IPC" + MEMORY = "memory" + NETWORK = "network" + SYSCALLS = "syscalls" + SYSTEM = "system" diff --git a/tests/image/endurance/cpu.py b/tests/image/endurance/cpu.py index b7b11041..2b2bfeed 100644 --- a/tests/image/endurance/cpu.py +++ b/tests/image/endurance/cpu.py @@ -1,14 +1,15 @@ from typing import TYPE_CHECKING from imgtests.exec.loaders import StressNg -from imgtests.runner import Subsystem, TestResult from imgtests.suites.general.stress_ng import StressNgTest +from imgtests.types import Subsystem if TYPE_CHECKING: from collections.abc import Iterable from concurrent.futures import ThreadPoolExecutor from imgtests.exec.exec import SSHClient + from imgtests.runner import TestResult class StressNgEnduranceCpuTest(StressNgTest): diff --git a/tests/image/endurance/disks.py b/tests/image/endurance/disks.py index a1408cb2..d0736c46 100644 --- a/tests/image/endurance/disks.py +++ b/tests/image/endurance/disks.py @@ -1,14 +1,15 @@ from typing import TYPE_CHECKING from imgtests.exec.loaders import StressNg -from imgtests.runner import Subsystem, TestResult from imgtests.suites.general.stress_ng import StressNgTest +from imgtests.types import Subsystem if TYPE_CHECKING: from collections.abc import Iterable from concurrent.futures import ThreadPoolExecutor from imgtests.exec.exec import SSHClient + from imgtests.runner import TestResult class StressNgEnduranceDisksTest(StressNgTest): diff --git a/tests/image/endurance/memory.py b/tests/image/endurance/memory.py index 681039af..90c6e5f2 100644 --- a/tests/image/endurance/memory.py +++ b/tests/image/endurance/memory.py @@ -1,14 +1,15 @@ from typing import TYPE_CHECKING, Any from imgtests.exec.loaders import StressNg -from imgtests.runner import Subsystem, TestResult from imgtests.suites.general.stress_ng import StressNgTest +from imgtests.types import Subsystem if TYPE_CHECKING: from collections.abc import Iterable from concurrent.futures import ThreadPoolExecutor from imgtests.exec.exec import SSHClient + from imgtests.runner import TestResult tests: list[dict[str, Any]] = [ # General memory stress test with mmap() callings diff --git a/tests/image/endurance/network.py b/tests/image/endurance/network.py index 2bcc2265..992ccbb4 100644 --- a/tests/image/endurance/network.py +++ b/tests/image/endurance/network.py @@ -6,8 +6,9 @@ from imgtests.exec.exec import SSHClient, common_run_command from imgtests.exec.loaders import StressNg -from imgtests.runner import AbstractRunnableManyTimesTest, Subsystem, TestResult, TestStatus +from imgtests.runner import AbstractRunnableManyTimesTest, TestResult, TestStatus from imgtests.suites.general.stress_ng import StressNgTest +from imgtests.types import Subsystem if TYPE_CHECKING: from collections.abc import Iterable diff --git a/tests/image/endurance/syscalls.py b/tests/image/endurance/syscalls.py index 66c759d5..84636ca4 100644 --- a/tests/image/endurance/syscalls.py +++ b/tests/image/endurance/syscalls.py @@ -3,8 +3,9 @@ from zoneinfo import ZoneInfo from imgtests.exec.loaders import Kirk, StressNg -from imgtests.runner import AbstractRunnableManyTimesTest, Subsystem, TestResult, TestStatus +from imgtests.runner import AbstractRunnableManyTimesTest, TestResult, TestStatus from imgtests.suites.general.stress_ng import StressNgTest +from imgtests.types import Subsystem if TYPE_CHECKING: from collections.abc import Iterable diff --git a/tests/image/performance/cpu.py b/tests/image/performance/cpu.py index 985bffad..d547984e 100644 --- a/tests/image/performance/cpu.py +++ b/tests/image/performance/cpu.py @@ -3,8 +3,9 @@ from zoneinfo import ZoneInfo from imgtests.exec.loaders import Chaosblade, StressNg -from imgtests.runner import AbstractRunnableTimeLimitedTest, Subsystem, TestResult, TestStatus +from imgtests.runner import AbstractRunnableTimeLimitedTest, TestResult, TestStatus from imgtests.suites.general.stress_ng import StressNgTest +from imgtests.types import Subsystem if TYPE_CHECKING: from collections.abc import Iterable diff --git a/tests/image/performance/fio_disks.py b/tests/image/performance/fio_disks.py index 46a0e71e..ca4531ce 100644 --- a/tests/image/performance/fio_disks.py +++ b/tests/image/performance/fio_disks.py @@ -4,9 +4,9 @@ from imgtests.exec.loaders.dmsetup import DeviceMapperSetup, setup_block_device from imgtests.exec.osinfo import get_os_release -from imgtests.runner import AbstractRunnableTimeLimitedTest, Subsystem, TestResult, TestStatus +from imgtests.runner import AbstractRunnableTimeLimitedTest, TestResult, TestStatus from imgtests.suites.drive.fio import FioSuite, FioSuiteConfig, FioWorkload -from imgtests.types import Distro +from imgtests.types import Distro, Subsystem if TYPE_CHECKING: from collections.abc import Iterable diff --git a/tests/image/performance/ipc.py b/tests/image/performance/ipc.py index 89048a74..abf3e10e 100644 --- a/tests/image/performance/ipc.py +++ b/tests/image/performance/ipc.py @@ -3,7 +3,8 @@ from zoneinfo import ZoneInfo from imgtests.exec.loaders import Perf -from imgtests.runner import AbstractRunnableManyTimesTest, Subsystem, TestResult, TestStatus +from imgtests.runner import AbstractRunnableManyTimesTest, TestResult, TestStatus +from imgtests.types import Subsystem if TYPE_CHECKING: from collections.abc import Iterable diff --git a/tests/image/performance/memory.py b/tests/image/performance/memory.py index 90af3dce..79cf3a21 100644 --- a/tests/image/performance/memory.py +++ b/tests/image/performance/memory.py @@ -7,8 +7,9 @@ from imgtests.exec.exec import common_run_command from imgtests.exec.loaders import StressNg from imgtests.exec.observers import Sar -from imgtests.runner import AbstractRunnableTimeLimitedTest, Subsystem, TestResult, TestStatus +from imgtests.runner import AbstractRunnableTimeLimitedTest, TestResult, TestStatus from imgtests.suites.general.stress_ng import StressNgTest +from imgtests.types import Subsystem if TYPE_CHECKING: from collections.abc import Iterable diff --git a/tests/image/performance/network.py b/tests/image/performance/network.py index be1613d3..17710043 100644 --- a/tests/image/performance/network.py +++ b/tests/image/performance/network.py @@ -4,7 +4,8 @@ from zoneinfo import ZoneInfo from imgtests.exec.loaders import Iperf3 -from imgtests.runner import AbstractRunnableTimeLimitedTest, Subsystem, TestResult, TestStatus +from imgtests.runner import AbstractRunnableTimeLimitedTest, TestResult, TestStatus +from imgtests.types import Subsystem if TYPE_CHECKING: from collections.abc import Iterable diff --git a/tests/image/performance/std_utils.py b/tests/image/performance/std_utils.py index 692e489d..d8b878f7 100644 --- a/tests/image/performance/std_utils.py +++ b/tests/image/performance/std_utils.py @@ -6,7 +6,8 @@ from zoneinfo import ZoneInfo from imgtests.exec.exec import common_run_command -from imgtests.runner import AbstractRunnableManyTimesTest, Subsystem, TestResult, TestStatus +from imgtests.runner import AbstractRunnableManyTimesTest, TestResult, TestStatus +from imgtests.types import Subsystem if TYPE_CHECKING: from collections.abc import Iterable diff --git a/tests/image/performance/stress_ng_general.py b/tests/image/performance/stress_ng_general.py index 76879323..517eab76 100644 --- a/tests/image/performance/stress_ng_general.py +++ b/tests/image/performance/stress_ng_general.py @@ -2,14 +2,15 @@ from typing import TYPE_CHECKING, Any from imgtests.exec.loaders import StressNg -from imgtests.runner import Subsystem, TestResult from imgtests.suites.general.stress_ng import StressNgTest +from imgtests.types import Subsystem if TYPE_CHECKING: from collections.abc import Iterable from concurrent.futures import ThreadPoolExecutor from imgtests.exec.exec import SSHClient + from imgtests.runner import TestResult tests: list[dict[str, Any]] = [ diff --git a/tests/image/performance/syscalls.py b/tests/image/performance/syscalls.py index af96355e..c03befaf 100644 --- a/tests/image/performance/syscalls.py +++ b/tests/image/performance/syscalls.py @@ -4,8 +4,9 @@ from zoneinfo import ZoneInfo from imgtests.exec.loaders import Kirk, Perf, StressNg -from imgtests.runner import AbstractRunnableTimeLimitedTest, Subsystem, TestResult, TestStatus +from imgtests.runner import AbstractRunnableTimeLimitedTest, TestResult, TestStatus from imgtests.suites.general.stress_ng import StressNgTest +from imgtests.types import Subsystem if TYPE_CHECKING: from collections.abc import Iterable diff --git a/tests/image/performance/system.py b/tests/image/performance/system.py index 77bbebcf..c6b7b8b0 100644 --- a/tests/image/performance/system.py +++ b/tests/image/performance/system.py @@ -3,7 +3,8 @@ from zoneinfo import ZoneInfo from imgtests.exec.loaders import PhoronixTestSuite -from imgtests.runner import AbstractRunnableManyTimesTest, Subsystem, TestResult, TestStatus +from imgtests.runner import AbstractRunnableManyTimesTest, TestResult, TestStatus +from imgtests.types import Subsystem if TYPE_CHECKING: from collections.abc import Iterable From a3f2ec45516567cd6ac509088846127e58ea20c1 Mon Sep 17 00:00:00 2001 From: Artanias <43622365+Artanias@users.noreply.github.com> Date: Mon, 30 Mar 2026 23:21:32 +0300 Subject: [PATCH 28/29] refactor: moves MetricSample into the types module. --- src/imgtests/exec/loaders/fio.py | 2 +- src/imgtests/exec/loaders/stress_ng.py | 2 +- src/imgtests/exec/metrics.py | 9 --------- src/imgtests/planning/executor.py | 3 +-- src/imgtests/types.py | 9 +++++++++ 5 files changed, 12 insertions(+), 13 deletions(-) delete mode 100644 src/imgtests/exec/metrics.py diff --git a/src/imgtests/exec/loaders/fio.py b/src/imgtests/exec/loaders/fio.py index 2fe651b6..06521759 100644 --- a/src/imgtests/exec/loaders/fio.py +++ b/src/imgtests/exec/loaders/fio.py @@ -4,10 +4,10 @@ from imgtests.exec.base_util import GenericUtil from imgtests.exec.exec import ExecResult, SSHClient, common_run_command -from imgtests.exec.metrics import MetricSample from imgtests.exec.pkgmgrs.mixin import PkgMgrMixin from imgtests.exec.pkgmgrs.pip3 import Pip3 from imgtests.exec.utils import create_opt +from imgtests.types import MetricSample if TYPE_CHECKING: from imgtests.types import Subsystem, Version diff --git a/src/imgtests/exec/loaders/stress_ng.py b/src/imgtests/exec/loaders/stress_ng.py index bb204e4e..1fa2fb7a 100644 --- a/src/imgtests/exec/loaders/stress_ng.py +++ b/src/imgtests/exec/loaders/stress_ng.py @@ -4,9 +4,9 @@ from imgtests.exec.base_util import GenericUtil from imgtests.exec.exec import ExecResult -from imgtests.exec.metrics import MetricSample from imgtests.exec.pkgmgrs.mixin import PkgMgrMixin from imgtests.exec.utils import add_flag, create_opt +from imgtests.types import MetricSample if TYPE_CHECKING: from imgtests.exec.exec import SSHClient diff --git a/src/imgtests/exec/metrics.py b/src/imgtests/exec/metrics.py deleted file mode 100644 index d18fd293..00000000 --- a/src/imgtests/exec/metrics.py +++ /dev/null @@ -1,9 +0,0 @@ -from dataclasses import dataclass - - -@dataclass(frozen=True) -class MetricSample: - stage_name: str - subsystem: str - metric_name: str - value: float diff --git a/src/imgtests/planning/executor.py b/src/imgtests/planning/executor.py index aae5439b..2e99c9e4 100644 --- a/src/imgtests/planning/executor.py +++ b/src/imgtests/planning/executor.py @@ -14,14 +14,13 @@ from imgtests.exec.loaders.stress_ng import StressNg, stress_metrics_to_samples from imgtests.runner import BaseRunner from imgtests.sizing import parse_size_to_bytes, round_bytes_to_mib_str -from imgtests.types import Subsystem +from imgtests.types import MetricSample, Subsystem if TYPE_CHECKING: from pathlib import Path from imgtests.database.database import ExperimentType, ImgtestsDatabase from imgtests.exec.exec import SSHClient - from imgtests.exec.metrics import MetricSample from imgtests.planning.models import LoadTask, PlanStage, TestPlan logger = logging.getLogger(__name__) diff --git a/src/imgtests/types.py b/src/imgtests/types.py index 8673f72d..2bf91745 100644 --- a/src/imgtests/types.py +++ b/src/imgtests/types.py @@ -1,5 +1,6 @@ import re from contextlib import suppress +from dataclasses import dataclass from enum import Enum from functools import total_ordering from typing import NamedTuple @@ -79,3 +80,11 @@ class Subsystem(str, Enum): NETWORK = "network" SYSCALLS = "syscalls" SYSTEM = "system" + + +@dataclass(frozen=True) +class MetricSample: + stage_name: str + subsystem: str + metric_name: str + value: float From 9bfe99e1548ed69cc37be2a85f9a59ba81dc1129 Mon Sep 17 00:00:00 2001 From: alwaysunhappy Date: Tue, 31 Mar 2026 20:53:38 +0300 Subject: [PATCH 29/29] fix remarks --- src/imgtests/runner.py | 8 ++++---- src/imgtests/sizing.py | 5 +---- tests/image/runner.py | 4 ++-- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/imgtests/runner.py b/src/imgtests/runner.py index 85aed46c..018973be 100644 --- a/src/imgtests/runner.py +++ b/src/imgtests/runner.py @@ -422,7 +422,7 @@ def __init__( self.db = db self.executor = PlanExecutor(client=client, db=db) - def run_from_env(self) -> int: + def run_from_env(self) -> bool: subsystems = self._parse_subsystems(env_var_to_type("PLAN_SUBSYSTEMS", str, "all")) results_root = env_var_to_type("PLAN_RESULTS_DIR", Path, Path("results/profiled")) pattern = self._parse_pattern(env_var_to_type("PLAN_PATTERN", str, "")) @@ -444,7 +444,7 @@ def run_from_env(self) -> int: pattern=pattern, config_id=config_id, ) - return 1 if failures else 0 + return failures > 0 def _run_matrix( self, @@ -453,7 +453,7 @@ def _run_matrix( results_root: Path, pattern: LoadPattern | None, config_id: int, - ) -> int: + ) -> bool: total_failures = 0 default_duration = env_var_to_type("PLAN_DURATION_SEC", int, 120) @@ -477,7 +477,7 @@ def _run_matrix( config_id=config_id, ) - return 1 if total_failures else 0 + return total_failures > 0 def _run_one( # noqa: PLR0913 self, diff --git a/src/imgtests/sizing.py b/src/imgtests/sizing.py index 83fea0fd..d746d3c0 100644 --- a/src/imgtests/sizing.py +++ b/src/imgtests/sizing.py @@ -15,19 +15,16 @@ def parse_size_to_bytes(value: str) -> int | None: raise ValueError(err_msg) suffix = normalized[-1].lower() + number = normalized[:-1] match suffix: case "k": multiplier = 1024 - number = normalized[:-1] case "m": multiplier = 1024**2 - number = normalized[:-1] case "g": multiplier = 1024**3 - number = normalized[:-1] case "t": multiplier = 1024**4 - number = normalized[:-1] case _: multiplier = 1 number = normalized diff --git a/tests/image/runner.py b/tests/image/runner.py index a2025601..8e4ef410 100644 --- a/tests/image/runner.py +++ b/tests/image/runner.py @@ -122,13 +122,13 @@ def main() -> None: yocto_runner.run() client = wait_remote(*YOCTO_CONF) or sys.exit(1) - exit_code = ProfiledPlanRunner( + has_failures = ProfiledPlanRunner( client=client, db=ImgtestsDatabase(), ).run_from_env() client.close() - sys.exit(exit_code) + sys.exit(1 if has_failures else 0) if __name__ == "__main__":