AstrBot-Desktop/src/features/awareness/ScreenAwarenessPanel.tsx
YokiiroBW b6a21a70ec
一些检测仍在等待运行
ci / Portable quality checks (push) Waiting to run
ci / Windows Tauri build (push) Waiting to run
feat(awareness): let the companion read one authorized window
Wires the window-awareness modules to the product for the first time.
uia.rs and capture.rs were fully written and had no Tauri command at all;
their only caller anywhere was an #[ignore]d smoke test, so nothing in a
shipped build could reach them.

Six commands, all behind the ADR-0008 policy: the privacy switch, the
session list, start and stop, a window frame, and the window's UI
Automation text. The read takes a closure so the window handle never
leaves the policy module — there is no accessor that skips the switch, the
session lookup, the expiry check, or the rate limit.

Starting a session has an interaction problem worth naming: reading the
foreground window is the only selection mechanism these modules have, and
at the moment the user clicks the button, the foreground window is this
application. So the button gives them a few bounded seconds to bring the
target forward, and a window belonging to this process is refused rather
than captured — which is the most likely outcome if the delay elapses
while they are still on the settings page.

The panel sends either the window's text or its picture to the companion
slot. Text first in the UI order, deliberately: it is far less revealing
than a screenshot and usually enough to answer "what am I looking at".

The indicator in the shell is ADR-0008 item 5, which no project document
required. Reading someone's screen with no visible sign is
indistinguishable from spyware from the outside, and a stop control the
user has to go looking for is not a stop control. It polls rather than
waiting to be told, because an authorization can lapse on its own — and
when it cannot read the status it renders nothing, claiming neither that a
capture is live nor that one is not.

The app's own commands need no capability entries: only plugin commands go
through the ACL, which the existing commands working under `core:default`
alone already demonstrates.

Rust tests 110 -> 113, frontend 43 -> 47.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-07-27 06:29:54 +08:00

271 行
8.4 KiB
TypeScript

此文件含有模棱两可的 Unicode 字符

此文件含有可能会与其他字符混淆的 Unicode 字符。 如果您是想特意这样的,可以安全地忽略该警告。 使用 Escape 按钮显示他们。

import { invoke } from "@tauri-apps/api/core";
import { useCallback, useEffect, useRef, useState } from "react";
import { errorMessage } from "../../lib/errorMessage";
import { astrbotAttachmentSchema } from "../chat/types";
import {
screenReadingStatusSchema,
windowFrameSchema,
windowTextSummarySchema,
type ScreenReadingStatus,
} from "./types";
/// The window the user is asked to bring forward before the capture starts.
const FOCUS_DELAY_SECONDS = 5;
/**
* Screen and window awareness, per ADR-0008.
*
* The switch, the session, and the rate limit all live in Rust; this panel only
* asks and reports. In particular it never remembers whether it thinks a session
* is live — every action re-reads the status Rust returns, so a session that
* expired or was dropped by the switch cannot leave the UI claiming otherwise.
*/
export function ScreenAwarenessPanel() {
const [status, setStatus] = useState<ScreenReadingStatus | null>(null);
const [error, setError] = useState<string | null>(null);
const [notice, setNotice] = useState<string | null>(null);
const [busy, setBusy] = useState(false);
const [countdown, setCountdown] = useState(0);
const countdownRef = useRef<number | null>(null);
const refresh = useCallback(async () => {
try {
const value = await invoke<unknown>("screen_reading_status");
setStatus(screenReadingStatusSchema.parse(value));
setError(null);
} catch (reason) {
setError(errorMessage(reason));
}
}, []);
useEffect(() => {
void refresh();
}, [refresh]);
useEffect(
() => () => {
if (countdownRef.current !== null) {
window.clearInterval(countdownRef.current);
}
},
[],
);
const toggle = async (enabled: boolean) => {
setBusy(true);
try {
const value = await invoke<unknown>("set_screen_reading_enabled", {
enabled,
});
setStatus(screenReadingStatusSchema.parse(value));
setError(null);
setNotice(
enabled
? "已允许读取窗口。仍需为每个窗口单独授权。"
: "已关闭,进行中的会话也一并结束。",
);
} catch (reason) {
setError(errorMessage(reason));
} finally {
setBusy(false);
}
};
const share = async () => {
setBusy(true);
setError(null);
setNotice(null);
setCountdown(FOCUS_DELAY_SECONDS);
countdownRef.current = window.setInterval(() => {
setCountdown((current) => (current > 0 ? current - 1 : 0));
}, 1000);
try {
// Rust waits out the same delay before reading the foreground window, so
// this counter is a display of that wait rather than a second one.
const value = await invoke<unknown>("begin_window_capture_session", {
slot: "companion",
focusDelaySeconds: FOCUS_DELAY_SECONDS,
});
const session =
screenReadingStatusSchema.shape.sessions.element.parse(value);
setNotice(`已授权读取「${session.windowTitle}」。`);
await refresh();
} catch (reason) {
setError(errorMessage(reason));
} finally {
if (countdownRef.current !== null) {
window.clearInterval(countdownRef.current);
countdownRef.current = null;
}
setCountdown(0);
setBusy(false);
}
};
const stop = async (sessionToken: string) => {
setBusy(true);
try {
const value = await invoke<unknown>("end_window_capture_session", {
sessionToken,
});
setStatus(screenReadingStatusSchema.parse(value));
setNotice("已结束该窗口的授权。");
setError(null);
} catch (reason) {
setError(errorMessage(reason));
} finally {
setBusy(false);
}
};
/// Sends what the window says, as text. Cheaper and far less revealing than a
/// picture, and enough for "what am I looking at".
const sendText = async (sessionToken: string) => {
setBusy(true);
setError(null);
try {
const summary = windowTextSummarySchema.parse(
await invoke<unknown>("read_window_text", { sessionToken }),
);
if (!summary.text.trim()) {
setNotice("这个窗口没有可读的文本。");
return;
}
await invoke<unknown>("send_connection_slot_message", {
request: {
slot: "companion",
message: `我正在看的窗口内容:\n\n${summary.text}`,
attachments: [],
},
});
setNotice(
`已发送窗口文本(${summary.visitedNodes} 个节点${summary.truncated ? ",已截断" : ""})。`,
);
} catch (reason) {
setError(errorMessage(reason));
} finally {
setBusy(false);
}
};
const sendFrame = async (sessionToken: string) => {
setBusy(true);
setError(null);
try {
const frame = windowFrameSchema.parse(
await invoke<unknown>("capture_window_frame", { sessionToken }),
);
const attachment = astrbotAttachmentSchema.parse(
await invoke<unknown>("upload_connection_slot_file", {
request: {
slot: "companion",
fileName: `window-${frame.width}x${frame.height}.png`,
mimeType: frame.mimeType,
bytes: frame.bytes,
},
}),
);
await invoke<unknown>("send_connection_slot_message", {
request: {
slot: "companion",
message: "这是我现在的窗口画面。",
attachments: [attachment],
},
});
setNotice(`已发送画面(${frame.width}×${frame.height})。`);
} catch (reason) {
setError(errorMessage(reason));
} finally {
setBusy(false);
}
};
const sessions = status?.sessions ?? [];
return (
<section className="awareness-panel">
<div>
<span className="section-kicker">Screen awareness</span>
<h2>窗口读取</h2>
<p>
读取一个你指定的窗口,用来告诉陪伴实例你正在做什么。只读取单个窗口,
不读取整个桌面;画面与文本不写入磁盘;只发送给陪伴槽位。
</p>
</div>
<label className="awareness-switch">
<input
type="checkbox"
checked={status?.enabled ?? false}
disabled={busy || !status}
onChange={(event) => void toggle(event.target.checked)}
/>
<span>允许本应用读取窗口</span>
</label>
{status ? (
<p className="awareness-limits">
画面最快每 {Math.round(status.pixelIntervalMs / 1000)} 秒一次,
文本最快每 {status.textIntervalMs} 毫秒一次, 每次授权{" "}
{Math.round(status.sessionTtlSeconds / 60)} 分钟后失效。
</p>
) : null}
<div className="awareness-actions">
<button
type="button"
disabled={busy || !status?.enabled}
onClick={() => void share()}
>
{countdown > 0 ? `请切换到目标窗口…${countdown}` : "授权一个窗口"}
</button>
</div>
{sessions.length ? (
<ul className="awareness-sessions">
{sessions.map((session) => (
<li key={session.sessionToken}>
<strong>{session.windowTitle || "(无标题窗口)"}</strong>
<span className="awareness-session-meta">
PID {session.processId}
</span>
<div className="awareness-session-actions">
<button
type="button"
disabled={busy}
onClick={() => void sendText(session.sessionToken)}
>
发送窗口文本
</button>
<button
type="button"
disabled={busy}
onClick={() => void sendFrame(session.sessionToken)}
>
发送窗口画面
</button>
<button
type="button"
disabled={busy}
onClick={() => void stop(session.sessionToken)}
>
结束授权
</button>
</div>
</li>
))}
</ul>
) : (
<p className="awareness-empty">当前没有已授权的窗口。</p>
)}
{notice ? <p className="awareness-notice">{notice}</p> : null}
{error ? (
<p className="error-banner" role="alert">
{error}
</p>
) : null}
</section>
);
}