import { invoke } from "@tauri-apps/api/core"; import { useCallback, useEffect, useRef, useState } from "react"; import { errorMessage } from "../../lib/errorMessage"; import { astrbotAttachmentSchema } from "../chat/types"; import { screenReadingStatusSchema, windowFrameSchema, windowTextSummarySchema, type ScreenReadingStatus, } from "./types"; /// The window the user is asked to bring forward before the capture starts. const FOCUS_DELAY_SECONDS = 5; /** * Screen and window awareness, per ADR-0008. * * The switch, the session, and the rate limit all live in Rust; this panel only * asks and reports. In particular it never remembers whether it thinks a session * is live — every action re-reads the status Rust returns, so a session that * expired or was dropped by the switch cannot leave the UI claiming otherwise. */ export function ScreenAwarenessPanel() { const [status, setStatus] = useState(null); const [error, setError] = useState(null); const [notice, setNotice] = useState(null); const [busy, setBusy] = useState(false); const [countdown, setCountdown] = useState(0); const countdownRef = useRef(null); const refresh = useCallback(async () => { try { const value = await invoke("screen_reading_status"); setStatus(screenReadingStatusSchema.parse(value)); setError(null); } catch (reason) { setError(errorMessage(reason)); } }, []); useEffect(() => { void refresh(); }, [refresh]); useEffect( () => () => { if (countdownRef.current !== null) { window.clearInterval(countdownRef.current); } }, [], ); const toggle = async (enabled: boolean) => { setBusy(true); try { const value = await invoke("set_screen_reading_enabled", { enabled, }); setStatus(screenReadingStatusSchema.parse(value)); setError(null); setNotice( enabled ? "已允许读取窗口。仍需为每个窗口单独授权。" : "已关闭,进行中的会话也一并结束。", ); } catch (reason) { setError(errorMessage(reason)); } finally { setBusy(false); } }; const share = async () => { setBusy(true); setError(null); setNotice(null); setCountdown(FOCUS_DELAY_SECONDS); countdownRef.current = window.setInterval(() => { setCountdown((current) => (current > 0 ? current - 1 : 0)); }, 1000); try { // Rust waits out the same delay before reading the foreground window, so // this counter is a display of that wait rather than a second one. const value = await invoke("begin_window_capture_session", { slot: "companion", focusDelaySeconds: FOCUS_DELAY_SECONDS, }); const session = screenReadingStatusSchema.shape.sessions.element.parse(value); setNotice(`已授权读取「${session.windowTitle}」。`); await refresh(); } catch (reason) { setError(errorMessage(reason)); } finally { if (countdownRef.current !== null) { window.clearInterval(countdownRef.current); countdownRef.current = null; } setCountdown(0); setBusy(false); } }; const stop = async (sessionToken: string) => { setBusy(true); try { const value = await invoke("end_window_capture_session", { sessionToken, }); setStatus(screenReadingStatusSchema.parse(value)); setNotice("已结束该窗口的授权。"); setError(null); } catch (reason) { setError(errorMessage(reason)); } finally { setBusy(false); } }; /// Sends what the window says, as text. Cheaper and far less revealing than a /// picture, and enough for "what am I looking at". const sendText = async (sessionToken: string) => { setBusy(true); setError(null); try { const summary = windowTextSummarySchema.parse( await invoke("read_window_text", { sessionToken }), ); if (!summary.text.trim()) { setNotice("这个窗口没有可读的文本。"); return; } await invoke("send_connection_slot_message", { request: { slot: "companion", message: `我正在看的窗口内容:\n\n${summary.text}`, attachments: [], }, }); setNotice( `已发送窗口文本(${summary.visitedNodes} 个节点${summary.truncated ? ",已截断" : ""})。`, ); } catch (reason) { setError(errorMessage(reason)); } finally { setBusy(false); } }; const sendFrame = async (sessionToken: string) => { setBusy(true); setError(null); try { const frame = windowFrameSchema.parse( await invoke("capture_window_frame", { sessionToken }), ); const attachment = astrbotAttachmentSchema.parse( await invoke("upload_connection_slot_file", { request: { slot: "companion", fileName: `window-${frame.width}x${frame.height}.png`, mimeType: frame.mimeType, bytes: frame.bytes, }, }), ); await invoke("send_connection_slot_message", { request: { slot: "companion", message: "这是我现在的窗口画面。", attachments: [attachment], }, }); setNotice(`已发送画面(${frame.width}×${frame.height})。`); } catch (reason) { setError(errorMessage(reason)); } finally { setBusy(false); } }; const sessions = status?.sessions ?? []; return (
Screen awareness

窗口读取

读取一个你指定的窗口,用来告诉陪伴实例你正在做什么。只读取单个窗口, 不读取整个桌面;画面与文本不写入磁盘;只发送给陪伴槽位。

{status ? (

画面最快每 {Math.round(status.pixelIntervalMs / 1000)} 秒一次, 文本最快每 {status.textIntervalMs} 毫秒一次, 每次授权{" "} {Math.round(status.sessionTtlSeconds / 60)} 分钟后失效。

) : null}
{sessions.length ? (
    {sessions.map((session) => (
  • {session.windowTitle || "(无标题窗口)"} PID {session.processId}
  • ))}
) : (

当前没有已授权的窗口。

)} {notice ?

{notice}

: null} {error ? (

{error}

) : null}
); }