diff --git a/.github/workflows/mitzo-provider-refresh.yml b/.github/workflows/mitzo-provider-refresh.yml new file mode 100644 index 0000000000..7759cfb14d --- /dev/null +++ b/.github/workflows/mitzo-provider-refresh.yml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Mitzo provider refresh regressions + +on: + push: + branches: [fix/warm-provider-refresh] + +permissions: + contents: read + +jobs: + provider-refresh: + runs-on: ubuntu-latest + timeout-minutes: 40 + env: + CARGO_BUILD_JOBS: "2" + OPENSHELL_TELEMETRY_ENABLED: "false" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install the pinned Rust toolchain + run: rustup toolchain install 1.95.0 --profile minimal --component rustfmt --component clippy + - name: Check formatting + run: cargo fmt --all -- --check + - name: Verify credential snapshot and notification behavior + run: cargo test --locked -p openshell-core --no-default-features provider_credentials::tests -- --test-threads=1 + - name: Verify policy reconciliation and request-driven wake-up + run: cargo test --locked -p openshell-sandbox --lib -- --test-threads=1 + - name: Verify the real SSH handler with synthetic transports + run: cargo test --locked -p openshell-supervisor-process ssh::tests -- --test-threads=1 + - name: Check the changed Rust crates + run: cargo clippy --locked -p openshell-core -p openshell-sandbox -p openshell-supervisor-process --all-targets -- -D warnings diff --git a/architecture/README.md b/architecture/README.md index 0d814a7a22..85d8a46cb7 100644 --- a/architecture/README.md +++ b/architecture/README.md @@ -151,6 +151,14 @@ supervisor. The supervisor validates and applies what can change at runtime, keeps last-known-good config when refresh fails, and leaves static isolation controls in place until the sandbox is recreated. +In the monolithic supervisor, authenticated SSH exec requests also wake this +existing reconciliation owner. Requests coalesce and are limited to one +request-driven poll per 250 ms; idle polling keeps its existing interval. The +wake-up carries no credential values and does not acknowledge readiness. New +children receive only the currently verified snapshot, so applications still +check their required environment before starting a model or replaying work. +The process-only sidecar topology retains its existing snapshot delivery path. + Live operations use the same authenticated gateway-supervisor relationship. Config refresh, policy updates, credential delivery, log push, connect, exec, file sync, and relay setup are multiplexed over supervisor sessions. If a diff --git a/crates/openshell-core/src/provider_credentials.rs b/crates/openshell-core/src/provider_credentials.rs index 677c716829..1590ea1408 100644 --- a/crates/openshell-core/src/provider_credentials.rs +++ b/crates/openshell-core/src/provider_credentials.rs @@ -12,6 +12,7 @@ use crate::{ use std::collections::{HashMap, HashSet, VecDeque}; use std::fmt; use std::sync::{Arc, RwLock}; +use tokio::sync::Notify; const MAX_RETAINED_CREDENTIAL_GENERATIONS: usize = 8; @@ -58,6 +59,7 @@ struct CompiledStaticCredentialEndpointBinding { #[derive(Debug, Clone)] pub struct ProviderCredentialState { inner: Arc>, + refresh_requested: Arc, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -74,6 +76,18 @@ impl fmt::Display for StaticCredentialBindingError { impl std::error::Error for StaticCredentialBindingError {} impl ProviderCredentialState { + /// Wake the existing gateway reconciliation owner before a fresh child. + /// This carries no credentials and does not acknowledge provider readiness. + pub fn request_refresh(&self) { + self.refresh_requested.notify_one(); + } + + /// A single owner consumes coalesced requests, including requests made + /// before it starts waiting. Credential installation remains unchanged. + pub async fn refresh_requested(&self) { + self.refresh_requested.notified().await; + } + pub fn from_environment( revision: u64, env: HashMap, @@ -96,6 +110,7 @@ impl ProviderCredentialState { let combined_resolver = merge_resolvers(&generations, current_resolver.as_ref()); Self { + refresh_requested: Arc::new(Notify::new()), inner: Arc::new(RwLock::new(ProviderCredentialStateInner { current: snapshot, generations, @@ -148,6 +163,7 @@ impl ProviderCredentialState { ); Ok(Self { + refresh_requested: Arc::new(Notify::new()), inner: Arc::new(RwLock::new(ProviderCredentialStateInner { current: snapshot, generations, @@ -178,6 +194,7 @@ impl ProviderCredentialState { }); Self { + refresh_requested: Arc::new(Notify::new()), inner: Arc::new(RwLock::new(ProviderCredentialStateInner { current: snapshot, generations: VecDeque::new(), @@ -799,6 +816,31 @@ mod tests { use super::*; use crate::google_cloud; + #[tokio::test] + async fn child_refresh_requests_wake_the_shared_owner_and_coalesce_without_changing_credentials() + { + let state = ProviderCredentialState::from_child_env_snapshot(7, HashMap::new()); + let child = state.clone(); + child.request_refresh(); + child.request_refresh(); + tokio::time::timeout( + std::time::Duration::from_millis(50), + state.refresh_requested(), + ) + .await + .expect("a request before polling must wake the owner"); + assert!( + tokio::time::timeout( + std::time::Duration::from_millis(10), + state.refresh_requested() + ) + .await + .is_err() + ); + assert_eq!(state.snapshot().revision, 7); + assert!(state.child_env_with_gcp_resolved().is_empty()); + } + fn binding(host: &str, port: u32, path: &str) -> StaticCredentialBinding { StaticCredentialBinding { endpoints: vec![StaticCredentialEndpointBinding { diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index b1c226cebd..1c20667e2c 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -3713,6 +3713,7 @@ async fn run_policy_poll_loop_with_client( // seed the applied-state trackers before OPA actually loads it. let mut pending_result = None; + let mut last_poll_started = tokio::time::Instant::now(); // Initialize revision from the first poll and acknowledge the initial // policy revision the supervisor actually loaded. A mismatched result is // reconciled below instead of being recorded as already applied. @@ -3780,7 +3781,18 @@ async fn run_policy_poll_loop_with_client( let result = if let Some(result) = pending_result.take() { result } else { - tokio::time::sleep(next_poll_delay(&ctx.extension_credentials, interval)).await; + tokio::select! { + () = tokio::time::sleep(next_poll_delay(&ctx.extension_credentials, interval)) => {} + () = ctx.provider_credentials.refresh_requested() => { + // Fresh SSH children can need newly attached providers. + // Coalesce bursts and bound request-driven polling without + // increasing the ordinary idle reconciliation rate. + tokio::time::sleep_until( + last_poll_started + Duration::from_millis(250), + ).await; + } + } + last_poll_started = tokio::time::Instant::now(); match client.poll_settings(&ctx.sandbox_id).await { Ok(result) => { let _ = ctx.workspace_tx.send(client.workspace()); @@ -5066,6 +5078,41 @@ network_policies: handle.abort(); } + #[tokio::test] + async fn child_environment_request_wakes_poll_without_waiting_for_the_periodic_interval() { + let mut v1 = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + v1.policy_hash = "same-policy".to_string(); + let mut v2 = v1.clone(); + v2.version = 2; + v2.config_revision = 200; + let engine = Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).unwrap()); + let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); + let mut ctx = policy_poll_test_context( + engine.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_revision), + has_last_valid_policy: true, + }, + default_middleware_connector(), + ); + ctx.interval_secs = 60; + let credentials = ctx.provider_credentials.clone(); + let (client, polls, mut reports) = scripted_policy_gateway(); + polls.send(v1).unwrap(); + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + expect_policy_report(&mut reports, 1).await; + polls.send(v2).unwrap(); + expect_no_policy_report(&mut reports).await; + credentials.request_refresh(); + expect_policy_report(&mut reports, 2).await; + assert_eq!(engine.current_generation(), 0); + handle.abort(); + } + #[tokio::test] async fn poll_rejects_first_tcp_expansion_and_reports_previous_policy_active() { let v1 = settings_poll_result( diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 893967b2ac..0c37215186 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -714,6 +714,10 @@ impl russh::server::Handler for SshHandler { if command.is_empty() { return Ok(()); } + // Request prompt reconciliation through the existing credential owner. + // This is not a readiness receipt: the child still receives only the + // currently verified environment, and launchers must gate their needs. + self.provider_credentials.request_refresh(); self.start_shell(channel, session.handle(), Some(command))?; Ok(()) } @@ -2428,6 +2432,17 @@ mod tests { /// namespace. async fn authenticated_test_client_with_main( main_session: Arc, + ) -> russh::client::Handle { + authenticated_test_client_with_credentials( + main_session, + ProviderCredentialState::from_child_env_snapshot(0, HashMap::new()), + ) + .await + } + + async fn authenticated_test_client_with_credentials( + main_session: Arc, + provider_credentials: ProviderCredentialState, ) -> russh::client::Handle { // Scoped so the `!Send` ThreadRng is dropped before the first await. let host_key = { @@ -2446,7 +2461,7 @@ mod tests { None, None, None, - ProviderCredentialState::from_child_env_snapshot(0, HashMap::new()), + provider_credentials, HashMap::new(), ResolvedProcessIdentity::default(), ProcessEnforcementMode::NetworkOnly, @@ -2486,6 +2501,24 @@ mod tests { authenticated_test_client_with_main(MainSession::inert()).await } + #[tokio::test] + async fn authenticated_exec_requests_a_provider_refresh_without_modifying_the_snapshot() { + let credentials = ProviderCredentialState::from_child_env_snapshot(7, HashMap::new()); + let client = + authenticated_test_client_with_credentials(MainSession::inert(), credentials.clone()) + .await; + let channel = client.channel_open_session().await.expect("open session"); + channel + .exec(true, "/usr/bin/true") + .await + .expect("send exec"); + tokio::time::timeout(Duration::from_secs(1), credentials.refresh_requested()) + .await + .expect("authenticated exec should wake the existing refresh owner"); + assert_eq!(credentials.snapshot().revision, 7); + assert!(credentials.child_env_with_gcp_resolved().is_empty()); + } + #[tokio::test] async fn abrupt_transport_drop_releases_main_input_lease() { let main_session = MainSession::inert(); diff --git a/docs/sandboxes/manage-providers.mdx b/docs/sandboxes/manage-providers.mdx index bc25e32ecb..d31d41ae69 100644 --- a/docs/sandboxes/manage-providers.mdx +++ b/docs/sandboxes/manage-providers.mdx @@ -303,6 +303,13 @@ for runtime attach and detach behavior. +For runtime attachment in the monolithic supervisor, a new authenticated SSH +exec request asks the supervisor to refresh provider state promptly. Applications +still need to check their required environment before initializing a client: +the attachment response confirms the gateway change, while the supervisor +validates and installs its local snapshot separately. Process-only sidecars +retain their existing snapshot delivery path. + ### Auto-Discovery Shortcut When `providers_v2_enabled=false` and the trailing command in