Wires the window-awareness modules to the product for the first time. uia.rs and capture.rs were fully written and had no Tauri command at all; their only caller anywhere was an #[ignore]d smoke test, so nothing in a shipped build could reach them. Six commands, all behind the ADR-0008 policy: the privacy switch, the session list, start and stop, a window frame, and the window's UI Automation text. The read takes a closure so the window handle never leaves the policy module — there is no accessor that skips the switch, the session lookup, the expiry check, or the rate limit. Starting a session has an interaction problem worth naming: reading the foreground window is the only selection mechanism these modules have, and at the moment the user clicks the button, the foreground window is this application. So the button gives them a few bounded seconds to bring the target forward, and a window belonging to this process is refused rather than captured — which is the most likely outcome if the delay elapses while they are still on the settings page. The panel sends either the window's text or its picture to the companion slot. Text first in the UI order, deliberately: it is far less revealing than a screenshot and usually enough to answer "what am I looking at". The indicator in the shell is ADR-0008 item 5, which no project document required. Reading someone's screen with no visible sign is indistinguishable from spyware from the outside, and a stop control the user has to go looking for is not a stop control. It polls rather than waiting to be told, because an authorization can lapse on its own — and when it cannot read the status it renders nothing, claiming neither that a capture is live nor that one is not. The app's own commands need no capability entries: only plugin commands go through the ACL, which the existing commands working under `core:default` alone already demonstrates. Rust tests 110 -> 113, frontend 43 -> 47. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
271 行
8.4 KiB
TypeScript
271 行
8.4 KiB
TypeScript
import { invoke } from "@tauri-apps/api/core";
|
||
import { useCallback, useEffect, useRef, useState } from "react";
|
||
|
||
import { errorMessage } from "../../lib/errorMessage";
|
||
import { astrbotAttachmentSchema } from "../chat/types";
|
||
import {
|
||
screenReadingStatusSchema,
|
||
windowFrameSchema,
|
||
windowTextSummarySchema,
|
||
type ScreenReadingStatus,
|
||
} from "./types";
|
||
|
||
/// The window the user is asked to bring forward before the capture starts.
|
||
const FOCUS_DELAY_SECONDS = 5;
|
||
|
||
/**
|
||
* Screen and window awareness, per ADR-0008.
|
||
*
|
||
* The switch, the session, and the rate limit all live in Rust; this panel only
|
||
* asks and reports. In particular it never remembers whether it thinks a session
|
||
* is live — every action re-reads the status Rust returns, so a session that
|
||
* expired or was dropped by the switch cannot leave the UI claiming otherwise.
|
||
*/
|
||
export function ScreenAwarenessPanel() {
|
||
const [status, setStatus] = useState<ScreenReadingStatus | null>(null);
|
||
const [error, setError] = useState<string | null>(null);
|
||
const [notice, setNotice] = useState<string | null>(null);
|
||
const [busy, setBusy] = useState(false);
|
||
const [countdown, setCountdown] = useState(0);
|
||
const countdownRef = useRef<number | null>(null);
|
||
|
||
const refresh = useCallback(async () => {
|
||
try {
|
||
const value = await invoke<unknown>("screen_reading_status");
|
||
setStatus(screenReadingStatusSchema.parse(value));
|
||
setError(null);
|
||
} catch (reason) {
|
||
setError(errorMessage(reason));
|
||
}
|
||
}, []);
|
||
|
||
useEffect(() => {
|
||
void refresh();
|
||
}, [refresh]);
|
||
|
||
useEffect(
|
||
() => () => {
|
||
if (countdownRef.current !== null) {
|
||
window.clearInterval(countdownRef.current);
|
||
}
|
||
},
|
||
[],
|
||
);
|
||
|
||
const toggle = async (enabled: boolean) => {
|
||
setBusy(true);
|
||
try {
|
||
const value = await invoke<unknown>("set_screen_reading_enabled", {
|
||
enabled,
|
||
});
|
||
setStatus(screenReadingStatusSchema.parse(value));
|
||
setError(null);
|
||
setNotice(
|
||
enabled
|
||
? "已允许读取窗口。仍需为每个窗口单独授权。"
|
||
: "已关闭,进行中的会话也一并结束。",
|
||
);
|
||
} catch (reason) {
|
||
setError(errorMessage(reason));
|
||
} finally {
|
||
setBusy(false);
|
||
}
|
||
};
|
||
|
||
const share = async () => {
|
||
setBusy(true);
|
||
setError(null);
|
||
setNotice(null);
|
||
setCountdown(FOCUS_DELAY_SECONDS);
|
||
countdownRef.current = window.setInterval(() => {
|
||
setCountdown((current) => (current > 0 ? current - 1 : 0));
|
||
}, 1000);
|
||
try {
|
||
// Rust waits out the same delay before reading the foreground window, so
|
||
// this counter is a display of that wait rather than a second one.
|
||
const value = await invoke<unknown>("begin_window_capture_session", {
|
||
slot: "companion",
|
||
focusDelaySeconds: FOCUS_DELAY_SECONDS,
|
||
});
|
||
const session =
|
||
screenReadingStatusSchema.shape.sessions.element.parse(value);
|
||
setNotice(`已授权读取「${session.windowTitle}」。`);
|
||
await refresh();
|
||
} catch (reason) {
|
||
setError(errorMessage(reason));
|
||
} finally {
|
||
if (countdownRef.current !== null) {
|
||
window.clearInterval(countdownRef.current);
|
||
countdownRef.current = null;
|
||
}
|
||
setCountdown(0);
|
||
setBusy(false);
|
||
}
|
||
};
|
||
|
||
const stop = async (sessionToken: string) => {
|
||
setBusy(true);
|
||
try {
|
||
const value = await invoke<unknown>("end_window_capture_session", {
|
||
sessionToken,
|
||
});
|
||
setStatus(screenReadingStatusSchema.parse(value));
|
||
setNotice("已结束该窗口的授权。");
|
||
setError(null);
|
||
} catch (reason) {
|
||
setError(errorMessage(reason));
|
||
} finally {
|
||
setBusy(false);
|
||
}
|
||
};
|
||
|
||
/// Sends what the window says, as text. Cheaper and far less revealing than a
|
||
/// picture, and enough for "what am I looking at".
|
||
const sendText = async (sessionToken: string) => {
|
||
setBusy(true);
|
||
setError(null);
|
||
try {
|
||
const summary = windowTextSummarySchema.parse(
|
||
await invoke<unknown>("read_window_text", { sessionToken }),
|
||
);
|
||
if (!summary.text.trim()) {
|
||
setNotice("这个窗口没有可读的文本。");
|
||
return;
|
||
}
|
||
await invoke<unknown>("send_connection_slot_message", {
|
||
request: {
|
||
slot: "companion",
|
||
message: `我正在看的窗口内容:\n\n${summary.text}`,
|
||
attachments: [],
|
||
},
|
||
});
|
||
setNotice(
|
||
`已发送窗口文本(${summary.visitedNodes} 个节点${summary.truncated ? ",已截断" : ""})。`,
|
||
);
|
||
} catch (reason) {
|
||
setError(errorMessage(reason));
|
||
} finally {
|
||
setBusy(false);
|
||
}
|
||
};
|
||
|
||
const sendFrame = async (sessionToken: string) => {
|
||
setBusy(true);
|
||
setError(null);
|
||
try {
|
||
const frame = windowFrameSchema.parse(
|
||
await invoke<unknown>("capture_window_frame", { sessionToken }),
|
||
);
|
||
const attachment = astrbotAttachmentSchema.parse(
|
||
await invoke<unknown>("upload_connection_slot_file", {
|
||
request: {
|
||
slot: "companion",
|
||
fileName: `window-${frame.width}x${frame.height}.png`,
|
||
mimeType: frame.mimeType,
|
||
bytes: frame.bytes,
|
||
},
|
||
}),
|
||
);
|
||
await invoke<unknown>("send_connection_slot_message", {
|
||
request: {
|
||
slot: "companion",
|
||
message: "这是我现在的窗口画面。",
|
||
attachments: [attachment],
|
||
},
|
||
});
|
||
setNotice(`已发送画面(${frame.width}×${frame.height})。`);
|
||
} catch (reason) {
|
||
setError(errorMessage(reason));
|
||
} finally {
|
||
setBusy(false);
|
||
}
|
||
};
|
||
|
||
const sessions = status?.sessions ?? [];
|
||
|
||
return (
|
||
<section className="awareness-panel">
|
||
<div>
|
||
<span className="section-kicker">Screen awareness</span>
|
||
<h2>窗口读取</h2>
|
||
<p>
|
||
读取一个你指定的窗口,用来告诉陪伴实例你正在做什么。只读取单个窗口,
|
||
不读取整个桌面;画面与文本不写入磁盘;只发送给陪伴槽位。
|
||
</p>
|
||
</div>
|
||
|
||
<label className="awareness-switch">
|
||
<input
|
||
type="checkbox"
|
||
checked={status?.enabled ?? false}
|
||
disabled={busy || !status}
|
||
onChange={(event) => void toggle(event.target.checked)}
|
||
/>
|
||
<span>允许本应用读取窗口</span>
|
||
</label>
|
||
|
||
{status ? (
|
||
<p className="awareness-limits">
|
||
画面最快每 {Math.round(status.pixelIntervalMs / 1000)} 秒一次,
|
||
文本最快每 {status.textIntervalMs} 毫秒一次, 每次授权{" "}
|
||
{Math.round(status.sessionTtlSeconds / 60)} 分钟后失效。
|
||
</p>
|
||
) : null}
|
||
|
||
<div className="awareness-actions">
|
||
<button
|
||
type="button"
|
||
disabled={busy || !status?.enabled}
|
||
onClick={() => void share()}
|
||
>
|
||
{countdown > 0 ? `请切换到目标窗口…${countdown}` : "授权一个窗口"}
|
||
</button>
|
||
</div>
|
||
|
||
{sessions.length ? (
|
||
<ul className="awareness-sessions">
|
||
{sessions.map((session) => (
|
||
<li key={session.sessionToken}>
|
||
<strong>{session.windowTitle || "(无标题窗口)"}</strong>
|
||
<span className="awareness-session-meta">
|
||
PID {session.processId}
|
||
</span>
|
||
<div className="awareness-session-actions">
|
||
<button
|
||
type="button"
|
||
disabled={busy}
|
||
onClick={() => void sendText(session.sessionToken)}
|
||
>
|
||
发送窗口文本
|
||
</button>
|
||
<button
|
||
type="button"
|
||
disabled={busy}
|
||
onClick={() => void sendFrame(session.sessionToken)}
|
||
>
|
||
发送窗口画面
|
||
</button>
|
||
<button
|
||
type="button"
|
||
disabled={busy}
|
||
onClick={() => void stop(session.sessionToken)}
|
||
>
|
||
结束授权
|
||
</button>
|
||
</div>
|
||
</li>
|
||
))}
|
||
</ul>
|
||
) : (
|
||
<p className="awareness-empty">当前没有已授权的窗口。</p>
|
||
)}
|
||
|
||
{notice ? <p className="awareness-notice">{notice}</p> : null}
|
||
{error ? (
|
||
<p className="error-banner" role="alert">
|
||
{error}
|
||
</p>
|
||
) : null}
|
||
</section>
|
||
);
|
||
}
|