S-03 下载器核心迁移
这个提交存在于:
父节点
eb0535de6f
当前提交
ce1ab6db1b
共有 4 个文件被更改,包括 213 次插入 和 42 次删除
|
|
@ -35,6 +35,21 @@
|
||||||
"type": "int",
|
"type": "int",
|
||||||
"default": 3600
|
"default": 3600
|
||||||
},
|
},
|
||||||
|
"download_retry_count": {
|
||||||
|
"description": "下载失败或未发现图片时的重试次数。",
|
||||||
|
"type": "int",
|
||||||
|
"default": 2
|
||||||
|
},
|
||||||
|
"progress_interval_seconds": {
|
||||||
|
"description": "下载进度提示间隔,单位秒;设为 0 可关闭进度提示。",
|
||||||
|
"type": "int",
|
||||||
|
"default": 30
|
||||||
|
},
|
||||||
|
"max_concurrent_tasks": {
|
||||||
|
"description": "同时运行的下载任务数。建议保持 1,避免 JM 源站或本机资源压力过高。",
|
||||||
|
"type": "int",
|
||||||
|
"default": 1
|
||||||
|
},
|
||||||
"jm_batch_count": {
|
"jm_batch_count": {
|
||||||
"description": "jmcomic 图片下载批次大小。",
|
"description": "jmcomic 图片下载批次大小。",
|
||||||
"type": "int",
|
"type": "int",
|
||||||
|
|
|
||||||
60
main.py
60
main.py
|
|
@ -1,5 +1,6 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from astrbot.api import AstrBotConfig, logger
|
from astrbot.api import AstrBotConfig, logger
|
||||||
|
|
@ -8,7 +9,7 @@ import astrbot.api.message_components as Comp
|
||||||
from astrbot.api.star import Context, Star, StarTools
|
from astrbot.api.star import Context, Star, StarTools
|
||||||
|
|
||||||
from .services.archive import ArchiveResult, ArchiveService
|
from .services.archive import ArchiveResult, ArchiveService
|
||||||
from .services.download import JMDownloadService
|
from .services.download import DownloadProgress, JMDownloadService
|
||||||
from .services.komga import KomgaClient, KomgaScanResult
|
from .services.komga import KomgaClient, KomgaScanResult
|
||||||
from .services.settings import PluginSettings
|
from .services.settings import PluginSettings
|
||||||
|
|
||||||
|
|
@ -23,6 +24,7 @@ class JMDownloadPlugin(Star):
|
||||||
self.downloader = JMDownloadService(self.settings)
|
self.downloader = JMDownloadService(self.settings)
|
||||||
self.archiver = ArchiveService(self.settings)
|
self.archiver = ArchiveService(self.settings)
|
||||||
self.komga = KomgaClient(self.settings.komga)
|
self.komga = KomgaClient(self.settings.komga)
|
||||||
|
self.task_semaphore = asyncio.Semaphore(self.settings.max_concurrent_tasks)
|
||||||
|
|
||||||
async def initialize(self) -> None:
|
async def initialize(self) -> None:
|
||||||
logger.info(
|
logger.info(
|
||||||
|
|
@ -53,15 +55,20 @@ class JMDownloadPlugin(Star):
|
||||||
event.stop_event()
|
event.stop_event()
|
||||||
return
|
return
|
||||||
|
|
||||||
await event.send(event.plain_result(f"Starting JM download: {comic_id}"))
|
await event.send(event.plain_result(f"开始下载 JM {comic_id},任务已进入队列。"))
|
||||||
try:
|
try:
|
||||||
downloaded = await self.downloader.download(comic_id)
|
async with self.task_semaphore:
|
||||||
archive_result = await self.archiver.archive(downloaded)
|
await event.send(event.plain_result(f"开始执行 JM {comic_id} 下载任务。"))
|
||||||
self.downloader.cleanup(downloaded)
|
downloaded = await self.downloader.download(
|
||||||
scan_result = await self.komga.scan_library()
|
comic_id,
|
||||||
|
progress_callback=lambda progress: self._send_progress(event, progress),
|
||||||
|
)
|
||||||
|
archive_result = await self.archiver.archive(downloaded)
|
||||||
|
self.downloader.cleanup(downloaded)
|
||||||
|
scan_result = await self.komga.scan_library()
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.exception(f"JMDownload failed for {comic_id}")
|
logger.exception(f"JMDownload failed for {comic_id}")
|
||||||
yield event.plain_result(f"JMDownload failed: {exc}")
|
yield event.plain_result(f"JMDownload 失败:{exc}")
|
||||||
event.stop_event()
|
event.stop_event()
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
@ -80,12 +87,15 @@ class JMDownloadPlugin(Star):
|
||||||
async def status(self, event: AstrMessageEvent):
|
async def status(self, event: AstrMessageEvent):
|
||||||
"""Show current JMDownload archive and Komga settings."""
|
"""Show current JMDownload archive and Komga settings."""
|
||||||
lines = [
|
lines = [
|
||||||
"JMDownload status",
|
"JMDownload 状态",
|
||||||
f"Archive root: {self.settings.archive_root}",
|
f"归档目录:{self.settings.archive_root}",
|
||||||
f"Archive format: {self.settings.archive_format}",
|
f"归档格式:{self.settings.archive_format}",
|
||||||
f"Send archive file: {self.settings.send_archive_file}",
|
f"并发任务数:{self.settings.max_concurrent_tasks}",
|
||||||
f"Komga enabled: {self.settings.komga.enabled}",
|
f"下载重试次数:{self.settings.download_retry_count}",
|
||||||
f"Komga can scan: {self.settings.komga.can_scan}",
|
f"进度提示间隔:{self.settings.progress_interval_seconds} 秒",
|
||||||
|
f"发送归档文件:{self.settings.send_archive_file}",
|
||||||
|
f"Komga 自动扫描:{self.settings.komga.enabled}",
|
||||||
|
f"Komga 配置完整:{self.settings.komga.can_scan}",
|
||||||
]
|
]
|
||||||
yield event.plain_result("\n".join(lines))
|
yield event.plain_result("\n".join(lines))
|
||||||
event.stop_event()
|
event.stop_event()
|
||||||
|
|
@ -110,17 +120,31 @@ class JMDownloadPlugin(Star):
|
||||||
archive_result: ArchiveResult,
|
archive_result: ArchiveResult,
|
||||||
scan_result: KomgaScanResult,
|
scan_result: KomgaScanResult,
|
||||||
) -> str:
|
) -> str:
|
||||||
existed = "existing archive reused" if archive_result.existed else "archive created"
|
existed = "复用已有归档" if archive_result.existed else "已创建归档"
|
||||||
return "\n".join(
|
return "\n".join(
|
||||||
[
|
[
|
||||||
f"JMDownload finished: {existed}",
|
f"JMDownload 完成:{existed}",
|
||||||
f"Series: {archive_result.series_name}",
|
f"系列:{archive_result.series_name}",
|
||||||
f"Pages: {archive_result.page_count}",
|
f"页数:{archive_result.page_count}",
|
||||||
f"File: {archive_result.path}",
|
f"文件:{archive_result.path}",
|
||||||
f"Komga: {scan_result.message}",
|
f"Komga: {scan_result.message}",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
async def _send_progress(
|
||||||
|
self,
|
||||||
|
event: AstrMessageEvent,
|
||||||
|
progress: DownloadProgress,
|
||||||
|
) -> None:
|
||||||
|
await event.send(
|
||||||
|
event.plain_result(
|
||||||
|
"下载进度:"
|
||||||
|
f"JM {progress.comic_id} "
|
||||||
|
f"第 {progress.attempt}/{progress.max_attempts} 次尝试,"
|
||||||
|
f"已发现 {progress.image_count} 张图片"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
def _get_data_dir(self) -> Path:
|
def _get_data_dir(self) -> Path:
|
||||||
try:
|
try:
|
||||||
return StarTools.get_data_dir("astrbot_plugin_jmdownload")
|
return StarTools.get_data_dir("astrbot_plugin_jmdownload")
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import shutil
|
import shutil
|
||||||
|
from collections.abc import Awaitable, Callable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
@ -23,6 +24,18 @@ class DownloadedComic:
|
||||||
image_files: list[Path]
|
image_files: list[Path]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class DownloadProgress:
|
||||||
|
comic_id: str
|
||||||
|
attempt: int
|
||||||
|
max_attempts: int
|
||||||
|
image_count: int
|
||||||
|
message: str
|
||||||
|
|
||||||
|
|
||||||
|
ProgressCallback = Callable[[DownloadProgress], Awaitable[None]]
|
||||||
|
|
||||||
|
|
||||||
class DownloadError(Exception):
|
class DownloadError(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
@ -31,35 +44,81 @@ class JMDownloadService:
|
||||||
def __init__(self, settings: PluginSettings) -> None:
|
def __init__(self, settings: PluginSettings) -> None:
|
||||||
self.settings = settings
|
self.settings = settings
|
||||||
|
|
||||||
async def download(self, comic_id: str) -> DownloadedComic:
|
async def download(
|
||||||
job_name = datetime.now().strftime(f"{comic_id}-%Y%m%d-%H%M%S")
|
self,
|
||||||
|
comic_id: str,
|
||||||
|
progress_callback: ProgressCallback | None = None,
|
||||||
|
) -> DownloadedComic:
|
||||||
|
job_name = datetime.now().strftime(f"{comic_id}-%Y%m%d-%H%M%S-%f")
|
||||||
work_dir = self.settings.work_root / job_name
|
work_dir = self.settings.work_root / job_name
|
||||||
download_dir = work_dir / "downloads"
|
max_attempts = self.settings.download_retry_count + 1
|
||||||
config_path = work_dir / "jmcomic.yml"
|
last_error = ""
|
||||||
download_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
self._write_jmcomic_config(config_path, download_dir)
|
|
||||||
|
|
||||||
try:
|
for attempt in range(1, max_attempts + 1):
|
||||||
title = await asyncio.wait_for(
|
attempt_dir = work_dir / f"attempt-{attempt}"
|
||||||
asyncio.to_thread(self._download_sync, comic_id, config_path),
|
download_dir = attempt_dir / "downloads"
|
||||||
timeout=self.settings.download_timeout_seconds,
|
config_path = attempt_dir / "jmcomic.yml"
|
||||||
|
download_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._write_jmcomic_config(config_path, download_dir)
|
||||||
|
|
||||||
|
progress_task = self._create_progress_task(
|
||||||
|
comic_id,
|
||||||
|
attempt,
|
||||||
|
max_attempts,
|
||||||
|
download_dir,
|
||||||
|
progress_callback,
|
||||||
)
|
)
|
||||||
except TimeoutError as exc:
|
try:
|
||||||
raise DownloadError(f"download timed out after {self.settings.download_timeout_seconds}s") from exc
|
title = await asyncio.wait_for(
|
||||||
except Exception as exc:
|
asyncio.to_thread(self._download_sync, comic_id, config_path),
|
||||||
raise DownloadError(str(exc)) from exc
|
timeout=self.settings.download_timeout_seconds,
|
||||||
|
)
|
||||||
|
image_files = collect_image_files(download_dir)
|
||||||
|
if not image_files:
|
||||||
|
raise DownloadError("download finished but no image files were found")
|
||||||
|
|
||||||
image_files = collect_image_files(download_dir)
|
await self._emit_progress(
|
||||||
if not image_files:
|
progress_callback,
|
||||||
raise DownloadError("download finished but no image files were found")
|
DownloadProgress(
|
||||||
|
comic_id=comic_id,
|
||||||
|
attempt=attempt,
|
||||||
|
max_attempts=max_attempts,
|
||||||
|
image_count=len(image_files),
|
||||||
|
message="download completed",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return DownloadedComic(
|
||||||
|
comic_id=comic_id,
|
||||||
|
title=title or infer_album_title(download_dir, comic_id),
|
||||||
|
work_dir=work_dir,
|
||||||
|
download_dir=download_dir,
|
||||||
|
image_files=image_files,
|
||||||
|
)
|
||||||
|
except (TimeoutError, asyncio.TimeoutError) as exc:
|
||||||
|
last_error = (
|
||||||
|
f"download timed out after {self.settings.download_timeout_seconds}s"
|
||||||
|
)
|
||||||
|
logger.warning(
|
||||||
|
f"JMDownload: {comic_id} attempt {attempt} timed out; "
|
||||||
|
"not retrying because the underlying jmcomic thread may still be running",
|
||||||
|
)
|
||||||
|
raise DownloadError(last_error) from exc
|
||||||
|
except Exception as exc:
|
||||||
|
last_error = str(exc)
|
||||||
|
logger.warning(
|
||||||
|
f"JMDownload: {comic_id} attempt {attempt}/{max_attempts} failed: {last_error}",
|
||||||
|
)
|
||||||
|
if attempt >= max_attempts:
|
||||||
|
raise DownloadError(last_error) from exc
|
||||||
|
finally:
|
||||||
|
if progress_task:
|
||||||
|
progress_task.cancel()
|
||||||
|
try:
|
||||||
|
await progress_task
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
pass
|
||||||
|
|
||||||
return DownloadedComic(
|
raise DownloadError(last_error or "download failed")
|
||||||
comic_id=comic_id,
|
|
||||||
title=title or infer_album_title(download_dir, comic_id),
|
|
||||||
work_dir=work_dir,
|
|
||||||
download_dir=download_dir,
|
|
||||||
image_files=image_files,
|
|
||||||
)
|
|
||||||
|
|
||||||
def cleanup(self, downloaded: DownloadedComic) -> None:
|
def cleanup(self, downloaded: DownloadedComic) -> None:
|
||||||
if not self.settings.delete_work_dir_after_archive:
|
if not self.settings.delete_work_dir_after_archive:
|
||||||
|
|
@ -78,6 +137,64 @@ class JMDownloadService:
|
||||||
result = jmcomic.download_album(comic_id, option=option, check_exception=True)
|
result = jmcomic.download_album(comic_id, option=option, check_exception=True)
|
||||||
return self._extract_album_title(result)
|
return self._extract_album_title(result)
|
||||||
|
|
||||||
|
def _create_progress_task(
|
||||||
|
self,
|
||||||
|
comic_id: str,
|
||||||
|
attempt: int,
|
||||||
|
max_attempts: int,
|
||||||
|
download_dir: Path,
|
||||||
|
progress_callback: ProgressCallback | None,
|
||||||
|
) -> asyncio.Task | None:
|
||||||
|
if not progress_callback or self.settings.progress_interval_seconds <= 0:
|
||||||
|
return None
|
||||||
|
return asyncio.create_task(
|
||||||
|
self._report_progress(comic_id, attempt, max_attempts, download_dir, progress_callback),
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _report_progress(
|
||||||
|
self,
|
||||||
|
comic_id: str,
|
||||||
|
attempt: int,
|
||||||
|
max_attempts: int,
|
||||||
|
download_dir: Path,
|
||||||
|
progress_callback: ProgressCallback,
|
||||||
|
) -> None:
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
await asyncio.sleep(self.settings.progress_interval_seconds)
|
||||||
|
image_count = len(collect_image_files(download_dir))
|
||||||
|
message = (
|
||||||
|
"fetching album metadata"
|
||||||
|
if image_count == 0
|
||||||
|
else f"downloaded {image_count} images"
|
||||||
|
)
|
||||||
|
await self._emit_progress(
|
||||||
|
progress_callback,
|
||||||
|
DownloadProgress(
|
||||||
|
comic_id=comic_id,
|
||||||
|
attempt=attempt,
|
||||||
|
max_attempts=max_attempts,
|
||||||
|
image_count=image_count,
|
||||||
|
message=message,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(f"JMDownload: progress reporter failed: {exc}")
|
||||||
|
|
||||||
|
async def _emit_progress(
|
||||||
|
self,
|
||||||
|
progress_callback: ProgressCallback | None,
|
||||||
|
progress: DownloadProgress,
|
||||||
|
) -> None:
|
||||||
|
if not progress_callback:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
await progress_callback(progress)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(f"JMDownload: progress callback failed: {exc}")
|
||||||
|
|
||||||
def _extract_album_title(self, result: object) -> str:
|
def _extract_album_title(self, result: object) -> str:
|
||||||
album = getattr(result, "album", None)
|
album = getattr(result, "album", None)
|
||||||
if album is None and isinstance(result, tuple) and result:
|
if album is None and isinstance(result, tuple) and result:
|
||||||
|
|
|
||||||
|
|
@ -51,6 +51,9 @@ class PluginSettings:
|
||||||
overwrite_existing: bool = False
|
overwrite_existing: bool = False
|
||||||
delete_work_dir_after_archive: bool = True
|
delete_work_dir_after_archive: bool = True
|
||||||
download_timeout_seconds: int = 3600
|
download_timeout_seconds: int = 3600
|
||||||
|
download_retry_count: int = 2
|
||||||
|
progress_interval_seconds: int = 30
|
||||||
|
max_concurrent_tasks: int = 1
|
||||||
jm_batch_count: int = 20
|
jm_batch_count: int = 20
|
||||||
jm_domains: list[str] | None = None
|
jm_domains: list[str] | None = None
|
||||||
komga: KomgaSettings = KomgaSettings()
|
komga: KomgaSettings = KomgaSettings()
|
||||||
|
|
@ -91,6 +94,18 @@ class PluginSettings:
|
||||||
60,
|
60,
|
||||||
int(config.get("download_timeout_seconds", 3600) or 3600),
|
int(config.get("download_timeout_seconds", 3600) or 3600),
|
||||||
),
|
),
|
||||||
|
download_retry_count=max(
|
||||||
|
0,
|
||||||
|
int(config.get("download_retry_count", 2) or 0),
|
||||||
|
),
|
||||||
|
progress_interval_seconds=max(
|
||||||
|
0,
|
||||||
|
int(config.get("progress_interval_seconds", 30) or 0),
|
||||||
|
),
|
||||||
|
max_concurrent_tasks=max(
|
||||||
|
1,
|
||||||
|
int(config.get("max_concurrent_tasks", 1) or 1),
|
||||||
|
),
|
||||||
jm_batch_count=max(1, int(config.get("jm_batch_count", 20) or 20)),
|
jm_batch_count=max(1, int(config.get("jm_batch_count", 20) or 20)),
|
||||||
jm_domains=[str(domain).strip() for domain in jm_domains if str(domain).strip()],
|
jm_domains=[str(domain).strip() for domain in jm_domains if str(domain).strip()],
|
||||||
komga=KomgaSettings.from_config(komga_config),
|
komga=KomgaSettings.from_config(komga_config),
|
||||||
|
|
|
||||||
正在加载…
添加表格
在新工单中引用