From b2ecd1f3a4448229a2f7cc6f6c2128988dedd7b9 Mon Sep 17 00:00:00 2001 From: YokiiroBW Date: Mon, 27 Jul 2026 00:11:19 +0800 Subject: [PATCH] fix(net): probe connection slots outside the runtime lock activate_slot held the single mutex that covers both slots across health() and probe(). Those go through the REST client, whose total timeout is 120s, so one unreachable address froze every runtime command on both slots for that long: status polling, sending, cancelling and disconnecting all blocked on the same lock. The cancellation token was also only created after a successful probe, so the activation itself could not be interrupted. Split the function into three phases. Validation and client construction happen first with no lock held; a short critical section then bumps the generation, installs the cancellation token up front and publishes the Connecting snapshot; the probe runs outside the lock inside a biased select against that token; and a final critical section writes the result back only if the slot still belongs to this activation. The listener receives the same token that phase one installed. Creating a second one there would leave deactivate_slot cancelling a token no listener observes, so the desktop-v1 reconnect loop would outlive the disconnect. This also fixes a stuck-slot bug the queue did not name. Validation used to run after the Connecting snapshot had been written, so a rejected request tore down a working connection and left the slot in Connecting forever, with no client and no last_error to explain it. The three new tests were checked against a mutation that re-acquires the lock across the probe; the two concurrency tests fail under it. Refs: F-14 (audit P1-4). Co-Authored-By: Claude Opus 5 --- src-tauri/src/networking/runtime.rs | 380 ++++++++++++++++++++++------ 1 file changed, 301 insertions(+), 79 deletions(-) diff --git a/src-tauri/src/networking/runtime.rs b/src-tauri/src/networking/runtime.rs index 580cd17..b7aaab4 100644 --- a/src-tauri/src/networking/runtime.rs +++ b/src-tauri/src/networking/runtime.rs @@ -126,6 +126,44 @@ impl RuntimeCommandError { message, } } + + pub fn cancelled(message: &'static str) -> Self { + Self { + code: "cancelled", + message, + } + } +} + +/// A client that has been constructed but not yet published into its slot. +/// +/// Building the client is pure, so it happens before the slot lock is taken. +enum PendingClient { + Desktop(GatewayClient, String), + AstrBot(AstrBotClient), +} + +/// The result of the out-of-lock reachability probe. +enum ProbeOutcome { + Connected, + AuthFailed(&'static str), + Unavailable(&'static str), + Cancelled, +} + +/// Rejects a write-back that belongs to a superseded activation. +/// +/// Every activation and deactivation bumps `generation`, so a stale task that +/// resumes after the slot was reset must not publish its result over the new +/// occupant's state. +fn ensure_generation(runtime: &SlotRuntime, expected: u64) -> Result<(), RuntimeCommandError> { + if runtime.generation == expected { + Ok(()) + } else { + Err(RuntimeCommandError::cancelled( + "The request was cancelled because the slot changed.", + )) + } } impl DualSlotRuntime { @@ -194,32 +232,13 @@ impl DualSlotRuntime { )); } - let mut guard = self.inner.lock().await; - let runtime = slot_mut(&mut guard, request.slot); - if let Some(cancellation) = runtime.connection_cancellation.take() { - cancellation.cancel(); - } - if let Some(cancellation) = runtime.message_cancellation.take() { - cancellation.cancel(); - } - let generation = runtime.generation.wrapping_add(1); - *runtime = SlotRuntime { - snapshot: SlotRuntimeSnapshot { - slot: request.slot, - profile_id: Some(request.profile_id.clone()), - transport: Some(request.transport), - state: ConnectionState::Connecting, - server_instance_id: None, - active_session_id: request.session_id.clone(), - last_error: None, - }, - client: None, - connection_cancellation: None, - message_cancellation: None, - generation, - }; - - match request.transport { + // Phase 0: validate the request and build the client with no lock held. + // + // This used to run *after* the slot had already been overwritten with a + // `Connecting` snapshot, so a rejected request tore down a working + // connection and then parked the slot in `Connecting` forever, with no + // client and no `last_error`. + let pending = match request.transport { TransportKind::DesktopV1 => { let expected = request .expected_server_instance_id @@ -240,29 +259,7 @@ impl DualSlotRuntime { .map_err(|_| { RuntimeCommandError::unavailable("Desktop gateway client failed to start.") })?; - match client.health().await { - Ok(_) => { - runtime.snapshot.state = ConnectionState::Connected; - runtime.snapshot.server_instance_id = Some(expected); - runtime.client = Some(ActiveClient::Desktop(client)); - } - Err(GatewayError::HttpStatus(status)) if status.as_u16() == 401 => { - runtime.snapshot.state = ConnectionState::AuthExpired; - runtime.snapshot.last_error = - Some("Desktop gateway authentication failed.".into()); - return Err(RuntimeCommandError::unavailable( - "Desktop gateway authentication failed.", - )); - } - Err(_) => { - runtime.snapshot.state = ConnectionState::ServerUnavailable; - runtime.snapshot.last_error = - Some("Desktop gateway health check failed.".into()); - return Err(RuntimeCommandError::unavailable( - "Desktop gateway health check failed.", - )); - } - } + PendingClient::Desktop(client, expected) } TransportKind::AstrBotOpenApi => { let api_key = request.api_key.clone().ok_or_else(|| { @@ -295,42 +292,125 @@ impl DualSlotRuntime { }, _ => RuntimeCommandError::unavailable("AstrBot client failed to start."), })?; - match client.probe().await { - Ok(_) => { - runtime.snapshot.state = ConnectionState::Connected; - runtime.snapshot.server_instance_id = - Some(client.server_instance_id_blocking()); - runtime.client = Some(ActiveClient::AstrBot(client)); + PendingClient::AstrBot(client) + } + }; + + // Phase 1: claim the slot. One critical section, no await inside it. + // + // The cancellation token is installed here rather than after a + // successful probe, so an activation against an unreachable address can + // be interrupted by `deactivate_slot`. + let (generation, cancellation) = { + let mut guard = self.inner.lock().await; + let runtime = slot_mut(&mut guard, request.slot); + if let Some(cancellation) = runtime.connection_cancellation.take() { + cancellation.cancel(); + } + if let Some(cancellation) = runtime.message_cancellation.take() { + cancellation.cancel(); + } + let generation = runtime.generation.wrapping_add(1); + let cancellation = Arc::new(tokio_util::sync::CancellationToken::new()); + *runtime = SlotRuntime { + snapshot: SlotRuntimeSnapshot { + slot: request.slot, + profile_id: Some(request.profile_id.clone()), + transport: Some(request.transport), + state: ConnectionState::Connecting, + server_instance_id: None, + active_session_id: request.session_id.clone(), + last_error: None, + }, + client: None, + connection_cancellation: Some(Arc::clone(&cancellation)), + message_cancellation: None, + generation, + }; + (generation, cancellation) + }; + + // Phase 2: probe outside the lock. + // + // The probe uses the REST client, whose total timeout is 120s. Holding + // the runtime lock across it blocked every command on *both* slots — + // status polling, sending, cancelling, disconnecting — for as long as + // one unreachable address took to time out. + let outcome = match &pending { + PendingClient::Desktop(client, _) => tokio::select! { + biased; + _ = cancellation.cancelled() => ProbeOutcome::Cancelled, + result = client.health() => match result { + Ok(_) => ProbeOutcome::Connected, + Err(GatewayError::HttpStatus(status)) if status.as_u16() == 401 => { + ProbeOutcome::AuthFailed("Desktop gateway authentication failed.") } - Err(error) => { - runtime.snapshot.state = match error { - super::AstrBotClientError::HttpStatus(status) - if status.as_u16() == 401 || status.as_u16() == 403 => - { - ConnectionState::AuthExpired - } - _ => ConnectionState::ServerUnavailable, - }; - runtime.snapshot.last_error = Some("AstrBot probe failed.".into()); - return Err(RuntimeCommandError::unavailable("AstrBot probe failed.")); + Err(_) => ProbeOutcome::Unavailable("Desktop gateway health check failed."), + }, + }, + PendingClient::AstrBot(client) => tokio::select! { + biased; + _ = cancellation.cancelled() => ProbeOutcome::Cancelled, + result = client.probe() => match result { + Ok(_) => ProbeOutcome::Connected, + Err(AstrBotClientError::HttpStatus(status)) + if status.as_u16() == 401 || status.as_u16() == 403 => + { + ProbeOutcome::AuthFailed("AstrBot probe failed.") } + Err(_) => ProbeOutcome::Unavailable("AstrBot probe failed."), + }, + }, + }; + + // Phase 3: publish the result, but only if this activation still owns + // the slot. + let (snapshot, listener_client) = { + let mut guard = self.inner.lock().await; + let runtime = slot_mut(&mut guard, request.slot); + ensure_generation(runtime, generation)?; + match outcome { + ProbeOutcome::Cancelled => { + *runtime = SlotRuntime::empty_with_generation(request.slot, generation); + return Err(RuntimeCommandError::cancelled( + "The connection attempt was cancelled.", + )); + } + ProbeOutcome::AuthFailed(message) => { + runtime.snapshot.state = ConnectionState::AuthExpired; + runtime.snapshot.last_error = Some(message.to_owned()); + runtime.connection_cancellation = None; + return Err(RuntimeCommandError::unavailable(message)); + } + ProbeOutcome::Unavailable(message) => { + runtime.snapshot.state = ConnectionState::ServerUnavailable; + runtime.snapshot.last_error = Some(message.to_owned()); + runtime.connection_cancellation = None; + return Err(RuntimeCommandError::unavailable(message)); + } + ProbeOutcome::Connected => { + let (server_instance_id, client) = match pending { + PendingClient::Desktop(client, expected) => { + (expected, ActiveClient::Desktop(client)) + } + PendingClient::AstrBot(client) => ( + client.server_instance_id_blocking(), + ActiveClient::AstrBot(client), + ), + }; + runtime.snapshot.state = ConnectionState::Connected; + runtime.snapshot.server_instance_id = Some(server_instance_id); + runtime.client = Some(client); + (runtime.snapshot.clone(), runtime.client.clone()) } } - } - - let listener_client = runtime.client.clone(); - let connection_cancellation = Arc::new(tokio_util::sync::CancellationToken::new()); - runtime.connection_cancellation = Some(connection_cancellation.clone()); - let snapshot = runtime.snapshot.clone(); - drop(guard); + }; if let Some(client) = listener_client { - self.start_connection_listener( - request.slot, - generation, - client, - connection_cancellation, - ); + // The same token installed in phase 1. Creating a second one here + // would leave `deactivate_slot` cancelling a token that no listener + // observes, so the reconnect loop would outlive the disconnect. + self.start_connection_listener(request.slot, generation, client, cancellation); } Ok(snapshot) @@ -763,6 +843,148 @@ fn slot_mut(inner: &mut DualSlotRuntimeInner, slot: SlotKind) -> &mut SlotRuntim mod tests { use super::*; + const PROFILE_A: &str = "11111111-1111-1111-1111-111111111111"; + + /// Accepts connections and never answers, so a probe stays in flight. + /// + /// Deliberately hand-rolled rather than reusing the fake server in + /// `astrbot_client`'s transport tests: those belong to the R2 batch and + /// must not be reshaped to serve a different module. + async fn spawn_unresponsive_server() -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind a loopback listener"); + let address = listener.local_addr().expect("read the bound address"); + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((stream, _)) = listener.accept().await { + held.push(stream); + } + }); + format!("http://{address}") + } + + fn astrbot_request(slot: SlotKind, profile_id: &str, base_url: &str) -> SlotActivationRequest { + SlotActivationRequest { + profile_id: profile_id.into(), + slot, + base_url: base_url.into(), + transport: TransportKind::AstrBotOpenApi, + api_key: Some(SecretString::from("abk_test".to_owned())), + bearer_token: None, + remote_username: Some("tester".into()), + remote_display_name: None, + authentication_type: desktop_protocol::AuthenticationType::DeviceToken, + session_id: None, + expected_server_instance_id: None, + } + } + + async fn wait_for_connecting(runtime: &DualSlotRuntime, slot: SlotKind) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let status = runtime.status().await; + let snapshot = match slot { + SlotKind::Companion => status.companion, + SlotKind::Operations => status.operations, + }; + if snapshot.state == ConnectionState::Connecting { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the connecting snapshot must be published before the probe finishes"); + } + + #[tokio::test] + async fn activation_publishes_connecting_without_holding_the_slot_lock() { + let base_url = spawn_unresponsive_server().await; + let runtime = Arc::new(DualSlotRuntime::new()); + + let activation = { + let runtime = Arc::clone(&runtime); + tokio::spawn(async move { + runtime + .activate_slot(astrbot_request(SlotKind::Companion, PROFILE_A, &base_url)) + .await + }) + }; + + wait_for_connecting(&runtime, SlotKind::Companion).await; + + // The probe is still in flight. Before this fix the runtime lock was + // held across it, so both slots were frozen for up to 120 seconds. + tokio::time::timeout( + Duration::from_secs(5), + runtime.deactivate_slot(SlotKind::Operations), + ) + .await + .expect("an in-flight probe must not block the other slot") + .expect("deactivating the idle slot should succeed"); + + tokio::time::timeout(Duration::from_secs(5), runtime.status()) + .await + .expect("an in-flight probe must not block status reads"); + + activation.abort(); + } + + #[tokio::test] + async fn activation_is_cancelled_by_disconnect() { + let base_url = spawn_unresponsive_server().await; + let runtime = Arc::new(DualSlotRuntime::new()); + + let activation = { + let runtime = Arc::clone(&runtime); + tokio::spawn(async move { + runtime + .activate_slot(astrbot_request(SlotKind::Companion, PROFILE_A, &base_url)) + .await + }) + }; + + wait_for_connecting(&runtime, SlotKind::Companion).await; + + runtime + .deactivate_slot(SlotKind::Companion) + .await + .expect("disconnect during activation should succeed"); + + let error = tokio::time::timeout(Duration::from_secs(5), activation) + .await + .expect("the activation must end once it is cancelled") + .expect("the activation task should not panic") + .expect_err("a cancelled activation must not report success"); + assert_eq!(error.code, "cancelled"); + + // The write-back must not resurrect the slot the disconnect cleared. + let companion = runtime.status().await.companion; + assert_eq!(companion.state, ConnectionState::Disconnected); + assert!(companion.profile_id.is_none()); + } + + #[tokio::test] + async fn invalid_activation_leaves_the_slot_untouched() { + let runtime = DualSlotRuntime::new(); + let mut request = astrbot_request(SlotKind::Companion, PROFILE_A, "http://127.0.0.1:9"); + request.api_key = None; + + let error = runtime + .activate_slot(request) + .await + .expect_err("an activation without an API key must be rejected"); + assert_eq!(error.code, "validationFailed"); + + // Validation now runs before the slot is claimed. It used to run after + // the `Connecting` snapshot had been written, which left the slot stuck + // in `Connecting` with no client and no error to explain it. + let companion = runtime.status().await.companion; + assert_eq!(companion.state, ConnectionState::Disconnected); + assert!(companion.profile_id.is_none()); + } + #[tokio::test] async fn rejects_astrbot_activation_without_username_or_key() { let runtime = DualSlotRuntime::new();