S-03 下载器核心迁移
这个提交存在于:
父节点
eb0535de6f
当前提交
ce1ab6db1b
共有 4 个文件被更改,包括 213 次插入 和 42 次删除
|
|
@ -35,6 +35,21 @@
|
|||
"type": "int",
|
||||
"default": 3600
|
||||
},
|
||||
"download_retry_count": {
|
||||
"description": "下载失败或未发现图片时的重试次数。",
|
||||
"type": "int",
|
||||
"default": 2
|
||||
},
|
||||
"progress_interval_seconds": {
|
||||
"description": "下载进度提示间隔,单位秒;设为 0 可关闭进度提示。",
|
||||
"type": "int",
|
||||
"default": 30
|
||||
},
|
||||
"max_concurrent_tasks": {
|
||||
"description": "同时运行的下载任务数。建议保持 1,避免 JM 源站或本机资源压力过高。",
|
||||
"type": "int",
|
||||
"default": 1
|
||||
},
|
||||
"jm_batch_count": {
|
||||
"description": "jmcomic 图片下载批次大小。",
|
||||
"type": "int",
|
||||
|
|
|
|||
60
main.py
60
main.py
|
|
@ -1,5 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from astrbot.api import AstrBotConfig, logger
|
||||
|
|
@ -8,7 +9,7 @@ import astrbot.api.message_components as Comp
|
|||
from astrbot.api.star import Context, Star, StarTools
|
||||
|
||||
from .services.archive import ArchiveResult, ArchiveService
|
||||
from .services.download import JMDownloadService
|
||||
from .services.download import DownloadProgress, JMDownloadService
|
||||
from .services.komga import KomgaClient, KomgaScanResult
|
||||
from .services.settings import PluginSettings
|
||||
|
||||
|
|
@ -23,6 +24,7 @@ class JMDownloadPlugin(Star):
|
|||
self.downloader = JMDownloadService(self.settings)
|
||||
self.archiver = ArchiveService(self.settings)
|
||||
self.komga = KomgaClient(self.settings.komga)
|
||||
self.task_semaphore = asyncio.Semaphore(self.settings.max_concurrent_tasks)
|
||||
|
||||
async def initialize(self) -> None:
|
||||
logger.info(
|
||||
|
|
@ -53,15 +55,20 @@ class JMDownloadPlugin(Star):
|
|||
event.stop_event()
|
||||
return
|
||||
|
||||
await event.send(event.plain_result(f"Starting JM download: {comic_id}"))
|
||||
await event.send(event.plain_result(f"开始下载 JM {comic_id},任务已进入队列。"))
|
||||
try:
|
||||
downloaded = await self.downloader.download(comic_id)
|
||||
archive_result = await self.archiver.archive(downloaded)
|
||||
self.downloader.cleanup(downloaded)
|
||||
scan_result = await self.komga.scan_library()
|
||||
async with self.task_semaphore:
|
||||
await event.send(event.plain_result(f"开始执行 JM {comic_id} 下载任务。"))
|
||||
downloaded = await self.downloader.download(
|
||||
comic_id,
|
||||
progress_callback=lambda progress: self._send_progress(event, progress),
|
||||
)
|
||||
archive_result = await self.archiver.archive(downloaded)
|
||||
self.downloader.cleanup(downloaded)
|
||||
scan_result = await self.komga.scan_library()
|
||||
except Exception as exc:
|
||||
logger.exception(f"JMDownload failed for {comic_id}")
|
||||
yield event.plain_result(f"JMDownload failed: {exc}")
|
||||
yield event.plain_result(f"JMDownload 失败:{exc}")
|
||||
event.stop_event()
|
||||
return
|
||||
|
||||
|
|
@ -80,12 +87,15 @@ class JMDownloadPlugin(Star):
|
|||
async def status(self, event: AstrMessageEvent):
|
||||
"""Show current JMDownload archive and Komga settings."""
|
||||
lines = [
|
||||
"JMDownload status",
|
||||
f"Archive root: {self.settings.archive_root}",
|
||||
f"Archive format: {self.settings.archive_format}",
|
||||
f"Send archive file: {self.settings.send_archive_file}",
|
||||
f"Komga enabled: {self.settings.komga.enabled}",
|
||||
f"Komga can scan: {self.settings.komga.can_scan}",
|
||||
"JMDownload 状态",
|
||||
f"归档目录:{self.settings.archive_root}",
|
||||
f"归档格式:{self.settings.archive_format}",
|
||||
f"并发任务数:{self.settings.max_concurrent_tasks}",
|
||||
f"下载重试次数:{self.settings.download_retry_count}",
|
||||
f"进度提示间隔:{self.settings.progress_interval_seconds} 秒",
|
||||
f"发送归档文件:{self.settings.send_archive_file}",
|
||||
f"Komga 自动扫描:{self.settings.komga.enabled}",
|
||||
f"Komga 配置完整:{self.settings.komga.can_scan}",
|
||||
]
|
||||
yield event.plain_result("\n".join(lines))
|
||||
event.stop_event()
|
||||
|
|
@ -110,17 +120,31 @@ class JMDownloadPlugin(Star):
|
|||
archive_result: ArchiveResult,
|
||||
scan_result: KomgaScanResult,
|
||||
) -> str:
|
||||
existed = "existing archive reused" if archive_result.existed else "archive created"
|
||||
existed = "复用已有归档" if archive_result.existed else "已创建归档"
|
||||
return "\n".join(
|
||||
[
|
||||
f"JMDownload finished: {existed}",
|
||||
f"Series: {archive_result.series_name}",
|
||||
f"Pages: {archive_result.page_count}",
|
||||
f"File: {archive_result.path}",
|
||||
f"JMDownload 完成:{existed}",
|
||||
f"系列:{archive_result.series_name}",
|
||||
f"页数:{archive_result.page_count}",
|
||||
f"文件:{archive_result.path}",
|
||||
f"Komga: {scan_result.message}",
|
||||
],
|
||||
)
|
||||
|
||||
async def _send_progress(
|
||||
self,
|
||||
event: AstrMessageEvent,
|
||||
progress: DownloadProgress,
|
||||
) -> None:
|
||||
await event.send(
|
||||
event.plain_result(
|
||||
"下载进度:"
|
||||
f"JM {progress.comic_id} "
|
||||
f"第 {progress.attempt}/{progress.max_attempts} 次尝试,"
|
||||
f"已发现 {progress.image_count} 张图片"
|
||||
),
|
||||
)
|
||||
|
||||
def _get_data_dir(self) -> Path:
|
||||
try:
|
||||
return StarTools.get_data_dir("astrbot_plugin_jmdownload")
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
import shutil
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
|
@ -23,6 +24,18 @@ class DownloadedComic:
|
|||
image_files: list[Path]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DownloadProgress:
|
||||
comic_id: str
|
||||
attempt: int
|
||||
max_attempts: int
|
||||
image_count: int
|
||||
message: str
|
||||
|
||||
|
||||
ProgressCallback = Callable[[DownloadProgress], Awaitable[None]]
|
||||
|
||||
|
||||
class DownloadError(Exception):
|
||||
pass
|
||||
|
||||
|
|
@ -31,35 +44,81 @@ class JMDownloadService:
|
|||
def __init__(self, settings: PluginSettings) -> None:
|
||||
self.settings = settings
|
||||
|
||||
async def download(self, comic_id: str) -> DownloadedComic:
|
||||
job_name = datetime.now().strftime(f"{comic_id}-%Y%m%d-%H%M%S")
|
||||
async def download(
|
||||
self,
|
||||
comic_id: str,
|
||||
progress_callback: ProgressCallback | None = None,
|
||||
) -> DownloadedComic:
|
||||
job_name = datetime.now().strftime(f"{comic_id}-%Y%m%d-%H%M%S-%f")
|
||||
work_dir = self.settings.work_root / job_name
|
||||
download_dir = work_dir / "downloads"
|
||||
config_path = work_dir / "jmcomic.yml"
|
||||
download_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._write_jmcomic_config(config_path, download_dir)
|
||||
max_attempts = self.settings.download_retry_count + 1
|
||||
last_error = ""
|
||||
|
||||
try:
|
||||
title = await asyncio.wait_for(
|
||||
asyncio.to_thread(self._download_sync, comic_id, config_path),
|
||||
timeout=self.settings.download_timeout_seconds,
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
attempt_dir = work_dir / f"attempt-{attempt}"
|
||||
download_dir = attempt_dir / "downloads"
|
||||
config_path = attempt_dir / "jmcomic.yml"
|
||||
download_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._write_jmcomic_config(config_path, download_dir)
|
||||
|
||||
progress_task = self._create_progress_task(
|
||||
comic_id,
|
||||
attempt,
|
||||
max_attempts,
|
||||
download_dir,
|
||||
progress_callback,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
raise DownloadError(f"download timed out after {self.settings.download_timeout_seconds}s") from exc
|
||||
except Exception as exc:
|
||||
raise DownloadError(str(exc)) from exc
|
||||
try:
|
||||
title = await asyncio.wait_for(
|
||||
asyncio.to_thread(self._download_sync, comic_id, config_path),
|
||||
timeout=self.settings.download_timeout_seconds,
|
||||
)
|
||||
image_files = collect_image_files(download_dir)
|
||||
if not image_files:
|
||||
raise DownloadError("download finished but no image files were found")
|
||||
|
||||
image_files = collect_image_files(download_dir)
|
||||
if not image_files:
|
||||
raise DownloadError("download finished but no image files were found")
|
||||
await self._emit_progress(
|
||||
progress_callback,
|
||||
DownloadProgress(
|
||||
comic_id=comic_id,
|
||||
attempt=attempt,
|
||||
max_attempts=max_attempts,
|
||||
image_count=len(image_files),
|
||||
message="download completed",
|
||||
),
|
||||
)
|
||||
return DownloadedComic(
|
||||
comic_id=comic_id,
|
||||
title=title or infer_album_title(download_dir, comic_id),
|
||||
work_dir=work_dir,
|
||||
download_dir=download_dir,
|
||||
image_files=image_files,
|
||||
)
|
||||
except (TimeoutError, asyncio.TimeoutError) as exc:
|
||||
last_error = (
|
||||
f"download timed out after {self.settings.download_timeout_seconds}s"
|
||||
)
|
||||
logger.warning(
|
||||
f"JMDownload: {comic_id} attempt {attempt} timed out; "
|
||||
"not retrying because the underlying jmcomic thread may still be running",
|
||||
)
|
||||
raise DownloadError(last_error) from exc
|
||||
except Exception as exc:
|
||||
last_error = str(exc)
|
||||
logger.warning(
|
||||
f"JMDownload: {comic_id} attempt {attempt}/{max_attempts} failed: {last_error}",
|
||||
)
|
||||
if attempt >= max_attempts:
|
||||
raise DownloadError(last_error) from exc
|
||||
finally:
|
||||
if progress_task:
|
||||
progress_task.cancel()
|
||||
try:
|
||||
await progress_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
return DownloadedComic(
|
||||
comic_id=comic_id,
|
||||
title=title or infer_album_title(download_dir, comic_id),
|
||||
work_dir=work_dir,
|
||||
download_dir=download_dir,
|
||||
image_files=image_files,
|
||||
)
|
||||
raise DownloadError(last_error or "download failed")
|
||||
|
||||
def cleanup(self, downloaded: DownloadedComic) -> None:
|
||||
if not self.settings.delete_work_dir_after_archive:
|
||||
|
|
@ -78,6 +137,64 @@ class JMDownloadService:
|
|||
result = jmcomic.download_album(comic_id, option=option, check_exception=True)
|
||||
return self._extract_album_title(result)
|
||||
|
||||
def _create_progress_task(
|
||||
self,
|
||||
comic_id: str,
|
||||
attempt: int,
|
||||
max_attempts: int,
|
||||
download_dir: Path,
|
||||
progress_callback: ProgressCallback | None,
|
||||
) -> asyncio.Task | None:
|
||||
if not progress_callback or self.settings.progress_interval_seconds <= 0:
|
||||
return None
|
||||
return asyncio.create_task(
|
||||
self._report_progress(comic_id, attempt, max_attempts, download_dir, progress_callback),
|
||||
)
|
||||
|
||||
async def _report_progress(
|
||||
self,
|
||||
comic_id: str,
|
||||
attempt: int,
|
||||
max_attempts: int,
|
||||
download_dir: Path,
|
||||
progress_callback: ProgressCallback,
|
||||
) -> None:
|
||||
try:
|
||||
while True:
|
||||
await asyncio.sleep(self.settings.progress_interval_seconds)
|
||||
image_count = len(collect_image_files(download_dir))
|
||||
message = (
|
||||
"fetching album metadata"
|
||||
if image_count == 0
|
||||
else f"downloaded {image_count} images"
|
||||
)
|
||||
await self._emit_progress(
|
||||
progress_callback,
|
||||
DownloadProgress(
|
||||
comic_id=comic_id,
|
||||
attempt=attempt,
|
||||
max_attempts=max_attempts,
|
||||
image_count=image_count,
|
||||
message=message,
|
||||
),
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.warning(f"JMDownload: progress reporter failed: {exc}")
|
||||
|
||||
async def _emit_progress(
|
||||
self,
|
||||
progress_callback: ProgressCallback | None,
|
||||
progress: DownloadProgress,
|
||||
) -> None:
|
||||
if not progress_callback:
|
||||
return
|
||||
try:
|
||||
await progress_callback(progress)
|
||||
except Exception as exc:
|
||||
logger.warning(f"JMDownload: progress callback failed: {exc}")
|
||||
|
||||
def _extract_album_title(self, result: object) -> str:
|
||||
album = getattr(result, "album", None)
|
||||
if album is None and isinstance(result, tuple) and result:
|
||||
|
|
|
|||
|
|
@ -51,6 +51,9 @@ class PluginSettings:
|
|||
overwrite_existing: bool = False
|
||||
delete_work_dir_after_archive: bool = True
|
||||
download_timeout_seconds: int = 3600
|
||||
download_retry_count: int = 2
|
||||
progress_interval_seconds: int = 30
|
||||
max_concurrent_tasks: int = 1
|
||||
jm_batch_count: int = 20
|
||||
jm_domains: list[str] | None = None
|
||||
komga: KomgaSettings = KomgaSettings()
|
||||
|
|
@ -91,6 +94,18 @@ class PluginSettings:
|
|||
60,
|
||||
int(config.get("download_timeout_seconds", 3600) or 3600),
|
||||
),
|
||||
download_retry_count=max(
|
||||
0,
|
||||
int(config.get("download_retry_count", 2) or 0),
|
||||
),
|
||||
progress_interval_seconds=max(
|
||||
0,
|
||||
int(config.get("progress_interval_seconds", 30) or 0),
|
||||
),
|
||||
max_concurrent_tasks=max(
|
||||
1,
|
||||
int(config.get("max_concurrent_tasks", 1) or 1),
|
||||
),
|
||||
jm_batch_count=max(1, int(config.get("jm_batch_count", 20) or 20)),
|
||||
jm_domains=[str(domain).strip() for domain in jm_domains if str(domain).strip()],
|
||||
komga=KomgaSettings.from_config(komga_config),
|
||||
|
|
|
|||
正在加载…
添加表格
在新工单中引用