From 4ae040a1f7622887c31c9da7f4afc591b948ad1b Mon Sep 17 00:00:00 2001 From: YokiiroBW Date: Sun, 5 Jul 2026 15:39:33 +0800 Subject: [PATCH] feat: implement core foundation modules --- core/event_bus.py | 64 +++++++++++++++++- core/fsm.py | 80 +++++++++++++++++++++- core/model_router.py | 84 ++++++++++++++++++++++- core/prompt_pipeline.py | 94 +++++++++++++++++++++++++- core/queue_manager.py | 113 ++++++++++++++++++++++++++++++- core/secret_service.py | 56 ++++++++++++++- tests/test_core_modules.py | 135 +++++++++++++++++++++++++++++++++++++ 7 files changed, 620 insertions(+), 6 deletions(-) create mode 100644 tests/test_core_modules.py diff --git a/core/event_bus.py b/core/event_bus.py index 2f3ccf8..b509916 100644 --- a/core/event_bus.py +++ b/core/event_bus.py @@ -1 +1,63 @@ -"""In-process event bus placeholder.""" +"""Small in-process event bus for ImageStudio services.""" + +from __future__ import annotations + +from collections import defaultdict, deque +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Callable, DefaultDict +from uuid import uuid4 + + +EventHandler = Callable[["Event"], None] + + +@dataclass(frozen=True) +class Event: + event_type: str + payload: dict[str, Any] = field(default_factory=dict) + event_id: str = field(default_factory=lambda: f"evt_{uuid4().hex}") + created_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) + + def to_dict(self) -> dict[str, Any]: + return { + "event_id": self.event_id, + "event_type": self.event_type, + "payload": self.payload, + "created_at": self.created_at, + } + + +class EventBus: + def __init__(self, *, history_limit: int = 200): + self._handlers: DefaultDict[str, list[EventHandler]] = defaultdict(list) + self._history: deque[Event] = deque(maxlen=max(int(history_limit), 1)) + + def subscribe(self, event_type: str, handler: EventHandler) -> None: + normalized = _normalize_event_type(event_type) + if handler not in self._handlers[normalized]: + self._handlers[normalized].append(handler) + + def unsubscribe(self, event_type: str, handler: EventHandler) -> None: + normalized = _normalize_event_type(event_type) + self._handlers[normalized] = [item for item in self._handlers[normalized] if item is not handler] + + def publish(self, event_type: str, payload: dict[str, Any] | None = None) -> Event: + event = Event(event_type=_normalize_event_type(event_type), payload=dict(payload or {})) + self._history.append(event) + for handler in [*self._handlers.get(event.event_type, []), *self._handlers.get("*", [])]: + handler(event) + return event + + def history(self, event_type: str | None = None) -> list[Event]: + if event_type is None: + return list(self._history) + normalized = _normalize_event_type(event_type) + return [event for event in self._history if event.event_type == normalized] + + +def _normalize_event_type(event_type: str) -> str: + normalized = str(event_type or "").strip() + if not normalized: + raise ValueError("event_type is required") + return normalized diff --git a/core/fsm.py b/core/fsm.py index cf02f12..9907122 100644 --- a/core/fsm.py +++ b/core/fsm.py @@ -1 +1,79 @@ -"""Workflow finite-state-machine helpers placeholder.""" +"""Finite-state-machine helpers for workflows, nodes, and tasks.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +WORKFLOW_TRANSITIONS: dict[str, set[str]] = { + "draft": {"running", "archived"}, + "running": {"completed", "failed", "cancelled", "archived"}, + "completed": {"archived"}, + "failed": {"running", "archived"}, + "cancelled": {"running", "archived"}, + "archived": set(), +} + +NODE_TRANSITIONS: dict[str, set[str]] = { + "pending": {"queued", "running", "completed", "failed", "skipped"}, + "queued": {"running", "completed", "failed", "cancelled"}, + "running": {"completed", "failed", "cancelled"}, + "completed": {"queued"}, + "failed": {"queued", "skipped"}, + "cancelled": {"queued"}, + "skipped": {"queued"}, +} + +TASK_TRANSITIONS: dict[str, set[str]] = { + "created": {"queued", "running", "cancelled"}, + "queued": {"running", "cancelled", "failed"}, + "running": {"completed", "succeeded", "failed", "cancelled"}, + "completed": set(), + "succeeded": set(), + "failed": {"queued", "dead_letter"}, + "cancelled": {"queued"}, + "dead_letter": {"queued"}, +} + + +@dataclass(frozen=True) +class TransitionResult: + allowed: bool + current: str + target: str + reason: str = "" + + +def can_transition(current: str, target: str, transitions: dict[str, set[str]]) -> bool: + normalized_current = _normalize_state(current) + normalized_target = _normalize_state(target) + return normalized_target == normalized_current or normalized_target in transitions.get(normalized_current, set()) + + +def check_transition(current: str, target: str, transitions: dict[str, set[str]]) -> TransitionResult: + normalized_current = _normalize_state(current) + normalized_target = _normalize_state(target) + if normalized_current not in transitions: + return TransitionResult(False, normalized_current, normalized_target, "unknown current state") + if can_transition(normalized_current, normalized_target, transitions): + return TransitionResult(True, normalized_current, normalized_target) + return TransitionResult(False, normalized_current, normalized_target, "transition is not allowed") + + +def workflow_transition(current: str, target: str) -> TransitionResult: + return check_transition(current, target, WORKFLOW_TRANSITIONS) + + +def node_transition(current: str, target: str) -> TransitionResult: + return check_transition(current, target, NODE_TRANSITIONS) + + +def task_transition(current: str, target: str) -> TransitionResult: + return check_transition(current, target, TASK_TRANSITIONS) + + +def _normalize_state(value: str) -> str: + normalized = str(value or "").strip().lower() + if not normalized: + raise ValueError("state is required") + return normalized diff --git a/core/model_router.py b/core/model_router.py index 6632eff..f73b7f5 100644 --- a/core/model_router.py +++ b/core/model_router.py @@ -1 +1,83 @@ -"""Model route selection placeholder.""" +"""Model route selection helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +SUPPORTED = "confirmed_supported" +UNSUPPORTED = "confirmed_unsupported" +UNCONFIRMED = "unconfirmed" + + +@dataclass(frozen=True) +class ModelRoute: + provider_id: str + model_id: str + route_id: str | None = None + status: str = UNCONFIRMED + capabilities: set[str] = field(default_factory=set) + priority: int = 100 + + @property + def key(self) -> str: + return self.route_id or f"{self.provider_id}/{self.model_id}" + + def to_dict(self) -> dict[str, Any]: + return { + "route_id": self.route_id, + "provider_id": self.provider_id, + "model_id": self.model_id, + "status": self.status, + "capabilities": sorted(self.capabilities), + "priority": self.priority, + } + + +@dataclass(frozen=True) +class RouteSelection: + selected: ModelRoute | None + status: str + reason: str = "" + candidates: list[ModelRoute] = field(default_factory=list) + + @property + def valid(self) -> bool: + return self.selected is not None and self.status == SUPPORTED + + +def select_model_route( + routes: list[ModelRoute], + *, + required_capabilities: list[str] | None = None, + preferred_route_id: str | None = None, +) -> RouteSelection: + required = set(required_capabilities or []) + candidates = [route for route in routes if required.issubset(route.capabilities)] + if preferred_route_id: + preferred = [route for route in candidates if route.key == preferred_route_id or route.route_id == preferred_route_id] + if preferred: + candidates = preferred + candidates = sorted(candidates, key=lambda route: (route.status != SUPPORTED, route.priority, route.key)) + if not candidates: + return RouteSelection(None, UNCONFIRMED, "no route advertises required capabilities", []) + selected = candidates[0] + if selected.status == SUPPORTED: + return RouteSelection(selected, SUPPORTED, candidates=candidates) + if selected.status == UNSUPPORTED: + return RouteSelection(None, UNSUPPORTED, "best matching route is unsupported", candidates) + return RouteSelection(None, UNCONFIRMED, "best matching route is unconfirmed", candidates) + + +def route_from_capability_profile(profile: dict[str, Any]) -> ModelRoute: + capabilities = profile.get("capabilities") or {} + supports = capabilities.get("supports", capabilities if isinstance(capabilities, list) else []) + return ModelRoute( + provider_id=str(profile.get("provider_id") or ""), + model_id=str(profile.get("model_id") or ""), + route_id=profile.get("route_id"), + status=str(profile.get("status") or UNCONFIRMED), + capabilities={str(item) for item in supports}, + priority=int(profile.get("priority") or 100), + ) diff --git a/core/prompt_pipeline.py b/core/prompt_pipeline.py index 341ff0b..5da5ca7 100644 --- a/core/prompt_pipeline.py +++ b/core/prompt_pipeline.py @@ -1 +1,93 @@ -"""Prompt composition pipeline placeholder.""" +"""Prompt composition helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Mapping + + +@dataclass(frozen=True) +class PromptPart: + text: str + weight: float = 1.0 + enabled: bool = True + + def render(self) -> str: + text = self.text.strip() + if not text: + return "" + if self.weight == 1: + return text + return f"({text}:{self.weight:g})" + + +@dataclass(frozen=True) +class PromptResult: + prompt: str + negative_prompt: str + parts: list[str] + negative_parts: list[str] + + def to_dict(self) -> dict[str, Any]: + return { + "prompt": self.prompt, + "negative_prompt": self.negative_prompt, + "parts": self.parts, + "negative_parts": self.negative_parts, + } + + +def compose_prompt( + parts: list[str | PromptPart], + *, + negative_parts: list[str | PromptPart] | None = None, + variables: Mapping[str, Any] | None = None, +) -> PromptResult: + rendered_parts = _render_parts(parts, variables or {}) + rendered_negative = _render_parts(negative_parts or [], variables or {}) + return PromptResult( + prompt=", ".join(rendered_parts), + negative_prompt=", ".join(rendered_negative), + parts=rendered_parts, + negative_parts=rendered_negative, + ) + + +def merge_prompt_sections(*sections: Mapping[str, Any]) -> PromptResult: + parts: list[str | PromptPart] = [] + negative_parts: list[str | PromptPart] = [] + variables: dict[str, Any] = {} + for section in sections: + variables.update(section.get("variables") or {}) + parts.extend(_as_list(section.get("prompt") or section.get("parts"))) + negative_parts.extend(_as_list(section.get("negative_prompt") or section.get("negative_parts"))) + return compose_prompt(parts, negative_parts=negative_parts, variables=variables) + + +def _render_parts(parts: list[str | PromptPart], variables: Mapping[str, Any]) -> list[str]: + rendered: list[str] = [] + seen: set[str] = set() + for part in parts: + prompt_part = part if isinstance(part, PromptPart) else PromptPart(str(part)) + if not prompt_part.enabled: + continue + text = _substitute(prompt_part.render(), variables) + if text and text not in seen: + rendered.append(text) + seen.add(text) + return rendered + + +def _substitute(text: str, variables: Mapping[str, Any]) -> str: + rendered = text + for key, value in variables.items(): + rendered = rendered.replace("{" + str(key) + "}", str(value)) + return rendered.strip() + + +def _as_list(value: Any) -> list[Any]: + if value is None: + return [] + if isinstance(value, list): + return value + return [value] diff --git a/core/queue_manager.py b/core/queue_manager.py index 4272978..47b9a00 100644 --- a/core/queue_manager.py +++ b/core/queue_manager.py @@ -1 +1,112 @@ -"""Generation queue manager placeholder.""" +"""In-memory generation queue primitives.""" + +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any +from uuid import uuid4 + +from core.fsm import task_transition + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +@dataclass +class QueueItem: + task_id: str + payload: dict[str, Any] + status: str = "queued" + idempotency_key: str | None = None + attempts: int = 0 + error_message: str | None = None + created_at: str = field(default_factory=_now) + updated_at: str = field(default_factory=_now) + + def to_dict(self) -> dict[str, Any]: + return { + "task_id": self.task_id, + "payload": self.payload, + "status": self.status, + "idempotency_key": self.idempotency_key, + "attempts": self.attempts, + "error_message": self.error_message, + "created_at": self.created_at, + "updated_at": self.updated_at, + } + + +class QueueManager: + def __init__(self): + self._pending: deque[str] = deque() + self._items: dict[str, QueueItem] = {} + self._idempotency_index: dict[str, str] = {} + + def enqueue(self, payload: dict[str, Any], *, idempotency_key: str | None = None) -> QueueItem: + if idempotency_key and idempotency_key in self._idempotency_index: + return self._items[self._idempotency_index[idempotency_key]] + item = QueueItem( + task_id=str(payload.get("task_id") or f"task_{uuid4().hex}"), + payload=dict(payload), + idempotency_key=idempotency_key, + ) + self._items[item.task_id] = item + if idempotency_key: + self._idempotency_index[idempotency_key] = item.task_id + self._pending.append(item.task_id) + return item + + def dequeue(self) -> QueueItem | None: + while self._pending: + task_id = self._pending.popleft() + item = self._items.get(task_id) + if item and item.status == "queued": + self._set_status(item, "running") + item.attempts += 1 + return item + return None + + def complete(self, task_id: str, result: dict[str, Any] | None = None) -> QueueItem: + item = self.get(task_id) + item.payload = {**item.payload, "result": result or {}} + self._set_status(item, "succeeded") + item.error_message = None + return item + + def fail(self, task_id: str, error_message: str, *, dead_letter: bool = False) -> QueueItem: + item = self.get(task_id) + self._set_status(item, "failed") + item.error_message = error_message + if dead_letter: + self._set_status(item, "dead_letter") + return item + + def retry(self, task_id: str) -> QueueItem: + item = self.get(task_id) + self._set_status(item, "queued") + item.error_message = None + if task_id not in self._pending: + self._pending.append(task_id) + return item + + def get(self, task_id: str) -> QueueItem: + item = self._items.get(task_id) + if item is None: + raise KeyError(task_id) + return item + + def summary(self) -> dict[str, int]: + counts = {"queued": 0, "running": 0, "succeeded": 0, "failed": 0, "dead_letter": 0} + for item in self._items.values(): + counts[item.status] = counts.get(item.status, 0) + 1 + return counts + + def _set_status(self, item: QueueItem, status: str) -> None: + transition = task_transition(item.status, status) + if not transition.allowed: + raise ValueError(transition.reason) + item.status = transition.target + item.updated_at = _now() diff --git a/core/secret_service.py b/core/secret_service.py index 010c70e..62bf5b2 100644 --- a/core/secret_service.py +++ b/core/secret_service.py @@ -1 +1,55 @@ -"""Secret management placeholder.""" +"""Secret redaction and environment reference helpers.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any, Mapping + + +MASK = "***MASKED***" +SECRET_KEY_PARTS = ("api_key", "apikey", "token", "secret", "password", "dsn", "credential") + + +@dataclass(frozen=True) +class SecretRef: + name: str + is_set: bool + + def to_dict(self) -> dict[str, Any]: + return {"name": self.name, "is_set": self.is_set, "value": MASK if self.is_set else None} + + +def is_secret_key(path: str) -> bool: + lowered = path.lower() + if lowered.endswith("_set"): + return False + return any(part in lowered for part in SECRET_KEY_PARTS) + + +def redact(value: Any, path: str = "") -> Any: + if isinstance(value, Mapping): + redacted: dict[str, Any] = {} + for key, child in value.items(): + child_path = f"{path}.{key}" if path else str(key) + redacted[str(key)] = MASK if is_secret_key(child_path) and child else redact(child, child_path) + return redacted + if isinstance(value, list): + return [redact(item, path) for item in value] + return value + + +def env_ref(name: str) -> SecretRef: + normalized = str(name or "").strip() + if not normalized: + raise ValueError("secret env name is required") + return SecretRef(name=normalized, is_set=bool(os.environ.get(normalized))) + + +def resolve_env_value(reference: str) -> str | None: + normalized = str(reference or "").strip() + if normalized.startswith("env:"): + normalized = normalized[4:] + if not normalized: + return None + return os.environ.get(normalized) diff --git a/tests/test_core_modules.py b/tests/test_core_modules.py new file mode 100644 index 0000000..4ad01b6 --- /dev/null +++ b/tests/test_core_modules.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import os +import unittest + +from core.event_bus import EventBus +from core.fsm import node_transition, task_transition, workflow_transition +from core.model_router import ModelRoute, route_from_capability_profile, select_model_route +from core.prompt_pipeline import PromptPart, compose_prompt, merge_prompt_sections +from core.queue_manager import QueueManager +from core.secret_service import MASK, env_ref, redact, resolve_env_value + + +class EventBusTests(unittest.TestCase): + def test_publish_notifies_specific_and_wildcard_handlers(self): + bus = EventBus() + received = [] + + bus.subscribe("workflow.started", lambda event: received.append(("specific", event.event_type))) + bus.subscribe("*", lambda event: received.append(("wildcard", event.payload["workflow_id"]))) + + event = bus.publish("workflow.started", {"workflow_id": "workflow_001"}) + + self.assertEqual(event.event_type, "workflow.started") + self.assertEqual(received, [("specific", "workflow.started"), ("wildcard", "workflow_001")]) + self.assertEqual(bus.history("workflow.started")[0].event_id, event.event_id) + + +class FsmTests(unittest.TestCase): + def test_known_state_transitions_are_checked(self): + self.assertTrue(workflow_transition("draft", "running").allowed) + self.assertFalse(workflow_transition("archived", "running").allowed) + self.assertTrue(node_transition("failed", "queued").allowed) + self.assertFalse(task_transition("succeeded", "queued").allowed) + + +class QueueManagerTests(unittest.TestCase): + def test_queue_is_idempotent_and_tracks_retry(self): + queue = QueueManager() + first = queue.enqueue({"prompt": "portrait"}, idempotency_key="idem_001") + second = queue.enqueue({"prompt": "portrait"}, idempotency_key="idem_001") + + self.assertEqual(first.task_id, second.task_id) + running = queue.dequeue() + self.assertIsNotNone(running) + self.assertEqual(running.status, "running") + + failed = queue.fail(running.task_id, "mock failure") + self.assertEqual(failed.status, "failed") + retried = queue.retry(running.task_id) + self.assertEqual(retried.status, "queued") + self.assertEqual(queue.summary()["queued"], 1) + + +class PromptPipelineTests(unittest.TestCase): + def test_compose_prompt_dedupes_and_substitutes_variables(self): + result = compose_prompt( + ["portrait of {name}", "portrait of {name}", PromptPart("soft light", weight=1.2)], + negative_parts=["low quality"], + variables={"name": "Nova"}, + ) + + self.assertEqual(result.prompt, "portrait of Nova, (soft light:1.2)") + self.assertEqual(result.negative_prompt, "low quality") + + def test_merge_prompt_sections_combines_positive_and_negative(self): + result = merge_prompt_sections( + {"prompt": "anime", "negative_prompt": "blur"}, + {"parts": ["clean line"], "negative_parts": ["noise"]}, + ) + + self.assertEqual(result.parts, ["anime", "clean line"]) + self.assertEqual(result.negative_parts, ["blur", "noise"]) + + +class ModelRouterTests(unittest.TestCase): + def test_unconfirmed_route_is_not_treated_as_supported(self): + selection = select_model_route( + [ModelRoute("local", "draft", status="unconfirmed", capabilities={"text_to_image"})], + required_capabilities=["text_to_image"], + ) + + self.assertFalse(selection.valid) + self.assertEqual(selection.status, "unconfirmed") + + def test_supported_route_wins_by_priority(self): + selection = select_model_route( + [ + ModelRoute("slow", "model", status="confirmed_supported", capabilities={"text_to_image"}, priority=20), + ModelRoute("fast", "model", status="confirmed_supported", capabilities={"text_to_image"}, priority=5), + ], + required_capabilities=["text_to_image"], + ) + + self.assertTrue(selection.valid) + self.assertEqual(selection.selected.provider_id, "fast") + + def test_route_can_be_built_from_capability_profile(self): + route = route_from_capability_profile( + { + "provider_id": "p", + "model_id": "m", + "status": "confirmed_supported", + "capabilities": {"supports": ["image_edit"]}, + } + ) + + self.assertEqual(route.capabilities, {"image_edit"}) + + +class SecretServiceTests(unittest.TestCase): + def test_redact_masks_nested_secret_values(self): + redacted = redact( + { + "database": {"dsn": "postgresql://secret"}, + "security": {"integration_token_set": True, "integration_token": "abc"}, + } + ) + + self.assertEqual(redacted["database"]["dsn"], MASK) + self.assertTrue(redacted["security"]["integration_token_set"]) + self.assertEqual(redacted["security"]["integration_token"], MASK) + + def test_env_ref_reports_presence_without_value(self): + os.environ["IMAGESTUDIO_TEST_SECRET"] = "secret-value" + try: + reference = env_ref("IMAGESTUDIO_TEST_SECRET") + self.assertEqual(reference.to_dict()["value"], MASK) + self.assertEqual(resolve_env_value("env:IMAGESTUDIO_TEST_SECRET"), "secret-value") + finally: + os.environ.pop("IMAGESTUDIO_TEST_SECRET", None) + + +if __name__ == "__main__": + unittest.main()