272 行
8.4 KiB
TypeScript
272 行
8.4 KiB
TypeScript
|
|
import { invoke } from "@tauri-apps/api/core";
|
|||
|
|
import { useCallback, useEffect, useRef, useState } from "react";
|
|||
|
|
|
|||
|
|
import { errorMessage } from "../../lib/errorMessage";
|
|||
|
|
import { astrbotAttachmentSchema } from "../chat/types";
|
|||
|
|
import {
|
|||
|
|
screenReadingStatusSchema,
|
|||
|
|
windowFrameSchema,
|
|||
|
|
windowTextSummarySchema,
|
|||
|
|
type ScreenReadingStatus,
|
|||
|
|
} from "./types";
|
|||
|
|
|
|||
|
|
/// The window the user is asked to bring forward before the capture starts.
|
|||
|
|
const FOCUS_DELAY_SECONDS = 5;
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Screen and window awareness, per ADR-0008.
|
|||
|
|
*
|
|||
|
|
* The switch, the session, and the rate limit all live in Rust; this panel only
|
|||
|
|
* asks and reports. In particular it never remembers whether it thinks a session
|
|||
|
|
* is live — every action re-reads the status Rust returns, so a session that
|
|||
|
|
* expired or was dropped by the switch cannot leave the UI claiming otherwise.
|
|||
|
|
*/
|
|||
|
|
export function ScreenAwarenessPanel() {
|
|||
|
|
const [status, setStatus] = useState<ScreenReadingStatus | null>(null);
|
|||
|
|
const [error, setError] = useState<string | null>(null);
|
|||
|
|
const [notice, setNotice] = useState<string | null>(null);
|
|||
|
|
const [busy, setBusy] = useState(false);
|
|||
|
|
const [countdown, setCountdown] = useState(0);
|
|||
|
|
const countdownRef = useRef<number | null>(null);
|
|||
|
|
|
|||
|
|
const refresh = useCallback(async () => {
|
|||
|
|
try {
|
|||
|
|
const value = await invoke<unknown>("screen_reading_status");
|
|||
|
|
setStatus(screenReadingStatusSchema.parse(value));
|
|||
|
|
setError(null);
|
|||
|
|
} catch (reason) {
|
|||
|
|
setError(errorMessage(reason));
|
|||
|
|
}
|
|||
|
|
}, []);
|
|||
|
|
|
|||
|
|
useEffect(() => {
|
|||
|
|
void refresh();
|
|||
|
|
}, [refresh]);
|
|||
|
|
|
|||
|
|
useEffect(
|
|||
|
|
() => () => {
|
|||
|
|
if (countdownRef.current !== null) {
|
|||
|
|
window.clearInterval(countdownRef.current);
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
[],
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
const toggle = async (enabled: boolean) => {
|
|||
|
|
setBusy(true);
|
|||
|
|
try {
|
|||
|
|
const value = await invoke<unknown>("set_screen_reading_enabled", {
|
|||
|
|
enabled,
|
|||
|
|
});
|
|||
|
|
setStatus(screenReadingStatusSchema.parse(value));
|
|||
|
|
setError(null);
|
|||
|
|
setNotice(
|
|||
|
|
enabled
|
|||
|
|
? "已允许读取窗口。仍需为每个窗口单独授权。"
|
|||
|
|
: "已关闭,进行中的会话也一并结束。",
|
|||
|
|
);
|
|||
|
|
} catch (reason) {
|
|||
|
|
setError(errorMessage(reason));
|
|||
|
|
} finally {
|
|||
|
|
setBusy(false);
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const share = async () => {
|
|||
|
|
setBusy(true);
|
|||
|
|
setError(null);
|
|||
|
|
setNotice(null);
|
|||
|
|
setCountdown(FOCUS_DELAY_SECONDS);
|
|||
|
|
countdownRef.current = window.setInterval(() => {
|
|||
|
|
setCountdown((current) => (current > 0 ? current - 1 : 0));
|
|||
|
|
}, 1000);
|
|||
|
|
try {
|
|||
|
|
// Rust waits out the same delay before reading the foreground window, so
|
|||
|
|
// this counter is a display of that wait rather than a second one.
|
|||
|
|
const value = await invoke<unknown>("begin_window_capture_session", {
|
|||
|
|
slot: "companion",
|
|||
|
|
focusDelaySeconds: FOCUS_DELAY_SECONDS,
|
|||
|
|
});
|
|||
|
|
const session =
|
|||
|
|
screenReadingStatusSchema.shape.sessions.element.parse(value);
|
|||
|
|
setNotice(`已授权读取「${session.windowTitle}」。`);
|
|||
|
|
await refresh();
|
|||
|
|
} catch (reason) {
|
|||
|
|
setError(errorMessage(reason));
|
|||
|
|
} finally {
|
|||
|
|
if (countdownRef.current !== null) {
|
|||
|
|
window.clearInterval(countdownRef.current);
|
|||
|
|
countdownRef.current = null;
|
|||
|
|
}
|
|||
|
|
setCountdown(0);
|
|||
|
|
setBusy(false);
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const stop = async (sessionToken: string) => {
|
|||
|
|
setBusy(true);
|
|||
|
|
try {
|
|||
|
|
const value = await invoke<unknown>("end_window_capture_session", {
|
|||
|
|
sessionToken,
|
|||
|
|
});
|
|||
|
|
setStatus(screenReadingStatusSchema.parse(value));
|
|||
|
|
setNotice("已结束该窗口的授权。");
|
|||
|
|
setError(null);
|
|||
|
|
} catch (reason) {
|
|||
|
|
setError(errorMessage(reason));
|
|||
|
|
} finally {
|
|||
|
|
setBusy(false);
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
/// Sends what the window says, as text. Cheaper and far less revealing than a
|
|||
|
|
/// picture, and enough for "what am I looking at".
|
|||
|
|
const sendText = async (sessionToken: string) => {
|
|||
|
|
setBusy(true);
|
|||
|
|
setError(null);
|
|||
|
|
try {
|
|||
|
|
const summary = windowTextSummarySchema.parse(
|
|||
|
|
await invoke<unknown>("read_window_text", { sessionToken }),
|
|||
|
|
);
|
|||
|
|
if (!summary.text.trim()) {
|
|||
|
|
setNotice("这个窗口没有可读的文本。");
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
await invoke<unknown>("send_connection_slot_message", {
|
|||
|
|
request: {
|
|||
|
|
slot: "companion",
|
|||
|
|
message: `我正在看的窗口内容:\n\n${summary.text}`,
|
|||
|
|
attachments: [],
|
|||
|
|
},
|
|||
|
|
});
|
|||
|
|
setNotice(
|
|||
|
|
`已发送窗口文本(${summary.visitedNodes} 个节点${summary.truncated ? ",已截断" : ""})。`,
|
|||
|
|
);
|
|||
|
|
} catch (reason) {
|
|||
|
|
setError(errorMessage(reason));
|
|||
|
|
} finally {
|
|||
|
|
setBusy(false);
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const sendFrame = async (sessionToken: string) => {
|
|||
|
|
setBusy(true);
|
|||
|
|
setError(null);
|
|||
|
|
try {
|
|||
|
|
const frame = windowFrameSchema.parse(
|
|||
|
|
await invoke<unknown>("capture_window_frame", { sessionToken }),
|
|||
|
|
);
|
|||
|
|
const attachment = astrbotAttachmentSchema.parse(
|
|||
|
|
await invoke<unknown>("upload_connection_slot_file", {
|
|||
|
|
request: {
|
|||
|
|
slot: "companion",
|
|||
|
|
fileName: `window-${frame.width}x${frame.height}.png`,
|
|||
|
|
mimeType: frame.mimeType,
|
|||
|
|
bytes: frame.bytes,
|
|||
|
|
},
|
|||
|
|
}),
|
|||
|
|
);
|
|||
|
|
await invoke<unknown>("send_connection_slot_message", {
|
|||
|
|
request: {
|
|||
|
|
slot: "companion",
|
|||
|
|
message: "这是我现在的窗口画面。",
|
|||
|
|
attachments: [attachment],
|
|||
|
|
},
|
|||
|
|
});
|
|||
|
|
setNotice(`已发送画面(${frame.width}×${frame.height})。`);
|
|||
|
|
} catch (reason) {
|
|||
|
|
setError(errorMessage(reason));
|
|||
|
|
} finally {
|
|||
|
|
setBusy(false);
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const sessions = status?.sessions ?? [];
|
|||
|
|
|
|||
|
|
return (
|
|||
|
|
<section className="awareness-panel">
|
|||
|
|
<div>
|
|||
|
|
<span className="section-kicker">Screen awareness</span>
|
|||
|
|
<h2>窗口读取</h2>
|
|||
|
|
<p>
|
|||
|
|
读取一个你指定的窗口,用来告诉陪伴实例你正在做什么。只读取单个窗口,
|
|||
|
|
不读取整个桌面;画面与文本不写入磁盘;只发送给陪伴槽位。
|
|||
|
|
</p>
|
|||
|
|
</div>
|
|||
|
|
|
|||
|
|
<label className="awareness-switch">
|
|||
|
|
<input
|
|||
|
|
type="checkbox"
|
|||
|
|
checked={status?.enabled ?? false}
|
|||
|
|
disabled={busy || !status}
|
|||
|
|
onChange={(event) => void toggle(event.target.checked)}
|
|||
|
|
/>
|
|||
|
|
<span>允许本应用读取窗口</span>
|
|||
|
|
</label>
|
|||
|
|
|
|||
|
|
{status ? (
|
|||
|
|
<p className="awareness-limits">
|
|||
|
|
画面最快每 {Math.round(status.pixelIntervalMs / 1000)} 秒一次,
|
|||
|
|
文本最快每 {status.textIntervalMs} 毫秒一次, 每次授权{" "}
|
|||
|
|
{Math.round(status.sessionTtlSeconds / 60)} 分钟后失效。
|
|||
|
|
</p>
|
|||
|
|
) : null}
|
|||
|
|
|
|||
|
|
<div className="awareness-actions">
|
|||
|
|
<button
|
|||
|
|
type="button"
|
|||
|
|
disabled={busy || !status?.enabled}
|
|||
|
|
onClick={() => void share()}
|
|||
|
|
>
|
|||
|
|
{countdown > 0 ? `请切换到目标窗口…${countdown}` : "授权一个窗口"}
|
|||
|
|
</button>
|
|||
|
|
</div>
|
|||
|
|
|
|||
|
|
{sessions.length ? (
|
|||
|
|
<ul className="awareness-sessions">
|
|||
|
|
{sessions.map((session) => (
|
|||
|
|
<li key={session.sessionToken}>
|
|||
|
|
<strong>{session.windowTitle || "(无标题窗口)"}</strong>
|
|||
|
|
<span className="awareness-session-meta">
|
|||
|
|
PID {session.processId}
|
|||
|
|
</span>
|
|||
|
|
<div className="awareness-session-actions">
|
|||
|
|
<button
|
|||
|
|
type="button"
|
|||
|
|
disabled={busy}
|
|||
|
|
onClick={() => void sendText(session.sessionToken)}
|
|||
|
|
>
|
|||
|
|
发送窗口文本
|
|||
|
|
</button>
|
|||
|
|
<button
|
|||
|
|
type="button"
|
|||
|
|
disabled={busy}
|
|||
|
|
onClick={() => void sendFrame(session.sessionToken)}
|
|||
|
|
>
|
|||
|
|
发送窗口画面
|
|||
|
|
</button>
|
|||
|
|
<button
|
|||
|
|
type="button"
|
|||
|
|
disabled={busy}
|
|||
|
|
onClick={() => void stop(session.sessionToken)}
|
|||
|
|
>
|
|||
|
|
结束授权
|
|||
|
|
</button>
|
|||
|
|
</div>
|
|||
|
|
</li>
|
|||
|
|
))}
|
|||
|
|
</ul>
|
|||
|
|
) : (
|
|||
|
|
<p className="awareness-empty">当前没有已授权的窗口。</p>
|
|||
|
|
)}
|
|||
|
|
|
|||
|
|
{notice ? <p className="awareness-notice">{notice}</p> : null}
|
|||
|
|
{error ? (
|
|||
|
|
<p className="error-banner" role="alert">
|
|||
|
|
{error}
|
|||
|
|
</p>
|
|||
|
|
) : null}
|
|||
|
|
</section>
|
|||
|
|
);
|
|||
|
|
}
|