S-03 下载器核心迁移

这个提交存在于:
Codex 2026-07-07 00:13:26 +08:00
父节点 eb0535de6f
当前提交 ce1ab6db1b
共有 4 个文件被更改,包括 213 次插入 和 42 次删除

查看文件

@ -35,6 +35,21 @@
"type": "int",
"default": 3600
},
"download_retry_count": {
"description": "下载失败或未发现图片时的重试次数。",
"type": "int",
"default": 2
},
"progress_interval_seconds": {
"description": "下载进度提示间隔,单位秒;设为 0 可关闭进度提示。",
"type": "int",
"default": 30
},
"max_concurrent_tasks": {
"description": "同时运行的下载任务数。建议保持 1,避免 JM 源站或本机资源压力过高。",
"type": "int",
"default": 1
},
"jm_batch_count": {
"description": "jmcomic 图片下载批次大小。",
"type": "int",

60
main.py
查看文件

@ -1,5 +1,6 @@
from __future__ import annotations
import asyncio
from pathlib import Path
from astrbot.api import AstrBotConfig, logger
@ -8,7 +9,7 @@ import astrbot.api.message_components as Comp
from astrbot.api.star import Context, Star, StarTools
from .services.archive import ArchiveResult, ArchiveService
from .services.download import JMDownloadService
from .services.download import DownloadProgress, JMDownloadService
from .services.komga import KomgaClient, KomgaScanResult
from .services.settings import PluginSettings
@ -23,6 +24,7 @@ class JMDownloadPlugin(Star):
self.downloader = JMDownloadService(self.settings)
self.archiver = ArchiveService(self.settings)
self.komga = KomgaClient(self.settings.komga)
self.task_semaphore = asyncio.Semaphore(self.settings.max_concurrent_tasks)
async def initialize(self) -> None:
logger.info(
@ -53,15 +55,20 @@ class JMDownloadPlugin(Star):
event.stop_event()
return
await event.send(event.plain_result(f"Starting JM download: {comic_id}"))
await event.send(event.plain_result(f"开始下载 JM {comic_id},任务已进入队列。"))
try:
downloaded = await self.downloader.download(comic_id)
archive_result = await self.archiver.archive(downloaded)
self.downloader.cleanup(downloaded)
scan_result = await self.komga.scan_library()
async with self.task_semaphore:
await event.send(event.plain_result(f"开始执行 JM {comic_id} 下载任务。"))
downloaded = await self.downloader.download(
comic_id,
progress_callback=lambda progress: self._send_progress(event, progress),
)
archive_result = await self.archiver.archive(downloaded)
self.downloader.cleanup(downloaded)
scan_result = await self.komga.scan_library()
except Exception as exc:
logger.exception(f"JMDownload failed for {comic_id}")
yield event.plain_result(f"JMDownload failed: {exc}")
yield event.plain_result(f"JMDownload 失败:{exc}")
event.stop_event()
return
@ -80,12 +87,15 @@ class JMDownloadPlugin(Star):
async def status(self, event: AstrMessageEvent):
"""Show current JMDownload archive and Komga settings."""
lines = [
"JMDownload status",
f"Archive root: {self.settings.archive_root}",
f"Archive format: {self.settings.archive_format}",
f"Send archive file: {self.settings.send_archive_file}",
f"Komga enabled: {self.settings.komga.enabled}",
f"Komga can scan: {self.settings.komga.can_scan}",
"JMDownload 状态",
f"归档目录:{self.settings.archive_root}",
f"归档格式:{self.settings.archive_format}",
f"并发任务数:{self.settings.max_concurrent_tasks}",
f"下载重试次数:{self.settings.download_retry_count}",
f"进度提示间隔:{self.settings.progress_interval_seconds} 秒",
f"发送归档文件:{self.settings.send_archive_file}",
f"Komga 自动扫描:{self.settings.komga.enabled}",
f"Komga 配置完整:{self.settings.komga.can_scan}",
]
yield event.plain_result("\n".join(lines))
event.stop_event()
@ -110,17 +120,31 @@ class JMDownloadPlugin(Star):
archive_result: ArchiveResult,
scan_result: KomgaScanResult,
) -> str:
existed = "existing archive reused" if archive_result.existed else "archive created"
existed = "复用已有归档" if archive_result.existed else "已创建归档"
return "\n".join(
[
f"JMDownload finished: {existed}",
f"Series: {archive_result.series_name}",
f"Pages: {archive_result.page_count}",
f"File: {archive_result.path}",
f"JMDownload 完成:{existed}",
f"系列:{archive_result.series_name}",
f"页数:{archive_result.page_count}",
f"文件:{archive_result.path}",
f"Komga: {scan_result.message}",
],
)
async def _send_progress(
self,
event: AstrMessageEvent,
progress: DownloadProgress,
) -> None:
await event.send(
event.plain_result(
"下载进度:"
f"JM {progress.comic_id} "
f"第 {progress.attempt}/{progress.max_attempts} 次尝试,"
f"已发现 {progress.image_count} 张图片"
),
)
def _get_data_dir(self) -> Path:
try:
return StarTools.get_data_dir("astrbot_plugin_jmdownload")

查看文件

@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import shutil
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
@ -23,6 +24,18 @@ class DownloadedComic:
image_files: list[Path]
@dataclass(frozen=True)
class DownloadProgress:
comic_id: str
attempt: int
max_attempts: int
image_count: int
message: str
ProgressCallback = Callable[[DownloadProgress], Awaitable[None]]
class DownloadError(Exception):
pass
@ -31,35 +44,81 @@ class JMDownloadService:
def __init__(self, settings: PluginSettings) -> None:
self.settings = settings
async def download(self, comic_id: str) -> DownloadedComic:
job_name = datetime.now().strftime(f"{comic_id}-%Y%m%d-%H%M%S")
async def download(
self,
comic_id: str,
progress_callback: ProgressCallback | None = None,
) -> DownloadedComic:
job_name = datetime.now().strftime(f"{comic_id}-%Y%m%d-%H%M%S-%f")
work_dir = self.settings.work_root / job_name
download_dir = work_dir / "downloads"
config_path = work_dir / "jmcomic.yml"
download_dir.mkdir(parents=True, exist_ok=True)
self._write_jmcomic_config(config_path, download_dir)
max_attempts = self.settings.download_retry_count + 1
last_error = ""
try:
title = await asyncio.wait_for(
asyncio.to_thread(self._download_sync, comic_id, config_path),
timeout=self.settings.download_timeout_seconds,
for attempt in range(1, max_attempts + 1):
attempt_dir = work_dir / f"attempt-{attempt}"
download_dir = attempt_dir / "downloads"
config_path = attempt_dir / "jmcomic.yml"
download_dir.mkdir(parents=True, exist_ok=True)
self._write_jmcomic_config(config_path, download_dir)
progress_task = self._create_progress_task(
comic_id,
attempt,
max_attempts,
download_dir,
progress_callback,
)
except TimeoutError as exc:
raise DownloadError(f"download timed out after {self.settings.download_timeout_seconds}s") from exc
except Exception as exc:
raise DownloadError(str(exc)) from exc
try:
title = await asyncio.wait_for(
asyncio.to_thread(self._download_sync, comic_id, config_path),
timeout=self.settings.download_timeout_seconds,
)
image_files = collect_image_files(download_dir)
if not image_files:
raise DownloadError("download finished but no image files were found")
image_files = collect_image_files(download_dir)
if not image_files:
raise DownloadError("download finished but no image files were found")
await self._emit_progress(
progress_callback,
DownloadProgress(
comic_id=comic_id,
attempt=attempt,
max_attempts=max_attempts,
image_count=len(image_files),
message="download completed",
),
)
return DownloadedComic(
comic_id=comic_id,
title=title or infer_album_title(download_dir, comic_id),
work_dir=work_dir,
download_dir=download_dir,
image_files=image_files,
)
except (TimeoutError, asyncio.TimeoutError) as exc:
last_error = (
f"download timed out after {self.settings.download_timeout_seconds}s"
)
logger.warning(
f"JMDownload: {comic_id} attempt {attempt} timed out; "
"not retrying because the underlying jmcomic thread may still be running",
)
raise DownloadError(last_error) from exc
except Exception as exc:
last_error = str(exc)
logger.warning(
f"JMDownload: {comic_id} attempt {attempt}/{max_attempts} failed: {last_error}",
)
if attempt >= max_attempts:
raise DownloadError(last_error) from exc
finally:
if progress_task:
progress_task.cancel()
try:
await progress_task
except asyncio.CancelledError:
pass
return DownloadedComic(
comic_id=comic_id,
title=title or infer_album_title(download_dir, comic_id),
work_dir=work_dir,
download_dir=download_dir,
image_files=image_files,
)
raise DownloadError(last_error or "download failed")
def cleanup(self, downloaded: DownloadedComic) -> None:
if not self.settings.delete_work_dir_after_archive:
@ -78,6 +137,64 @@ class JMDownloadService:
result = jmcomic.download_album(comic_id, option=option, check_exception=True)
return self._extract_album_title(result)
def _create_progress_task(
self,
comic_id: str,
attempt: int,
max_attempts: int,
download_dir: Path,
progress_callback: ProgressCallback | None,
) -> asyncio.Task | None:
if not progress_callback or self.settings.progress_interval_seconds <= 0:
return None
return asyncio.create_task(
self._report_progress(comic_id, attempt, max_attempts, download_dir, progress_callback),
)
async def _report_progress(
self,
comic_id: str,
attempt: int,
max_attempts: int,
download_dir: Path,
progress_callback: ProgressCallback,
) -> None:
try:
while True:
await asyncio.sleep(self.settings.progress_interval_seconds)
image_count = len(collect_image_files(download_dir))
message = (
"fetching album metadata"
if image_count == 0
else f"downloaded {image_count} images"
)
await self._emit_progress(
progress_callback,
DownloadProgress(
comic_id=comic_id,
attempt=attempt,
max_attempts=max_attempts,
image_count=image_count,
message=message,
),
)
except asyncio.CancelledError:
raise
except Exception as exc:
logger.warning(f"JMDownload: progress reporter failed: {exc}")
async def _emit_progress(
self,
progress_callback: ProgressCallback | None,
progress: DownloadProgress,
) -> None:
if not progress_callback:
return
try:
await progress_callback(progress)
except Exception as exc:
logger.warning(f"JMDownload: progress callback failed: {exc}")
def _extract_album_title(self, result: object) -> str:
album = getattr(result, "album", None)
if album is None and isinstance(result, tuple) and result:

查看文件

@ -51,6 +51,9 @@ class PluginSettings:
overwrite_existing: bool = False
delete_work_dir_after_archive: bool = True
download_timeout_seconds: int = 3600
download_retry_count: int = 2
progress_interval_seconds: int = 30
max_concurrent_tasks: int = 1
jm_batch_count: int = 20
jm_domains: list[str] | None = None
komga: KomgaSettings = KomgaSettings()
@ -91,6 +94,18 @@ class PluginSettings:
60,
int(config.get("download_timeout_seconds", 3600) or 3600),
),
download_retry_count=max(
0,
int(config.get("download_retry_count", 2) or 0),
),
progress_interval_seconds=max(
0,
int(config.get("progress_interval_seconds", 30) or 0),
),
max_concurrent_tasks=max(
1,
int(config.get("max_concurrent_tasks", 1) or 1),
),
jm_batch_count=max(1, int(config.get("jm_batch_count", 20) or 20)),
jm_domains=[str(domain).strip() for domain in jm_domains if str(domain).strip()],
komga=KomgaSettings.from_config(komga_config),