Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions .github/workflows/mitzo-provider-refresh.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

name: Mitzo provider refresh regressions

on:
push:
branches: [fix/warm-provider-refresh]

permissions:
contents: read

jobs:
provider-refresh:
runs-on: ubuntu-latest
timeout-minutes: 40
env:
CARGO_BUILD_JOBS: "2"
OPENSHELL_TELEMETRY_ENABLED: "false"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install the pinned Rust toolchain
run: rustup toolchain install 1.95.0 --profile minimal --component rustfmt --component clippy
- name: Check formatting
run: cargo fmt --all -- --check
- name: Verify credential snapshot and notification behavior
run: cargo test --locked -p openshell-core --no-default-features provider_credentials::tests -- --test-threads=1
- name: Verify policy reconciliation and request-driven wake-up
run: cargo test --locked -p openshell-sandbox --lib -- --test-threads=1
- name: Verify the real SSH handler with synthetic transports
run: cargo test --locked -p openshell-supervisor-process ssh::tests -- --test-threads=1
- name: Check the changed Rust crates
run: cargo clippy --locked -p openshell-core -p openshell-sandbox -p openshell-supervisor-process --all-targets -- -D warnings
8 changes: 8 additions & 0 deletions architecture/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,14 @@ supervisor. The supervisor validates and applies what can change at runtime,
keeps last-known-good config when refresh fails, and leaves static isolation
controls in place until the sandbox is recreated.

In the monolithic supervisor, authenticated SSH exec requests also wake this
existing reconciliation owner. Requests coalesce and are limited to one
request-driven poll per 250 ms; idle polling keeps its existing interval. The
wake-up carries no credential values and does not acknowledge readiness. New
children receive only the currently verified snapshot, so applications still
check their required environment before starting a model or replaying work.
The process-only sidecar topology retains its existing snapshot delivery path.

Live operations use the same authenticated gateway-supervisor relationship.
Config refresh, policy updates, credential delivery, log push, connect, exec,
file sync, and relay setup are multiplexed over supervisor sessions. If a
Expand Down
42 changes: 42 additions & 0 deletions crates/openshell-core/src/provider_credentials.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ use crate::{
use std::collections::{HashMap, HashSet, VecDeque};
use std::fmt;
use std::sync::{Arc, RwLock};
use tokio::sync::Notify;

const MAX_RETAINED_CREDENTIAL_GENERATIONS: usize = 8;

Expand Down Expand Up @@ -58,6 +59,7 @@ struct CompiledStaticCredentialEndpointBinding {
#[derive(Debug, Clone)]
pub struct ProviderCredentialState {
inner: Arc<RwLock<ProviderCredentialStateInner>>,
refresh_requested: Arc<Notify>,
}

#[derive(Debug, Clone, PartialEq, Eq)]
Expand All @@ -74,6 +76,18 @@ impl fmt::Display for StaticCredentialBindingError {
impl std::error::Error for StaticCredentialBindingError {}

impl ProviderCredentialState {
/// Wake the existing gateway reconciliation owner before a fresh child.
/// This carries no credentials and does not acknowledge provider readiness.
pub fn request_refresh(&self) {
self.refresh_requested.notify_one();
}

/// A single owner consumes coalesced requests, including requests made
/// before it starts waiting. Credential installation remains unchanged.
pub async fn refresh_requested(&self) {
self.refresh_requested.notified().await;
}

pub fn from_environment(
revision: u64,
env: HashMap<String, String>,
Expand All @@ -96,6 +110,7 @@ impl ProviderCredentialState {
let combined_resolver = merge_resolvers(&generations, current_resolver.as_ref());

Self {
refresh_requested: Arc::new(Notify::new()),
inner: Arc::new(RwLock::new(ProviderCredentialStateInner {
current: snapshot,
generations,
Expand Down Expand Up @@ -148,6 +163,7 @@ impl ProviderCredentialState {
);

Ok(Self {
refresh_requested: Arc::new(Notify::new()),
inner: Arc::new(RwLock::new(ProviderCredentialStateInner {
current: snapshot,
generations,
Expand Down Expand Up @@ -178,6 +194,7 @@ impl ProviderCredentialState {
});

Self {
refresh_requested: Arc::new(Notify::new()),
inner: Arc::new(RwLock::new(ProviderCredentialStateInner {
current: snapshot,
generations: VecDeque::new(),
Expand Down Expand Up @@ -799,6 +816,31 @@ mod tests {
use super::*;
use crate::google_cloud;

#[tokio::test]
async fn child_refresh_requests_wake_the_shared_owner_and_coalesce_without_changing_credentials()
{
let state = ProviderCredentialState::from_child_env_snapshot(7, HashMap::new());
let child = state.clone();
child.request_refresh();
child.request_refresh();
tokio::time::timeout(
std::time::Duration::from_millis(50),
state.refresh_requested(),
)
.await
.expect("a request before polling must wake the owner");
assert!(
tokio::time::timeout(
std::time::Duration::from_millis(10),
state.refresh_requested()
)
.await
.is_err()
);
assert_eq!(state.snapshot().revision, 7);
assert!(state.child_env_with_gcp_resolved().is_empty());
}

fn binding(host: &str, port: u32, path: &str) -> StaticCredentialBinding {
StaticCredentialBinding {
endpoints: vec![StaticCredentialEndpointBinding {
Expand Down
49 changes: 48 additions & 1 deletion crates/openshell-sandbox/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3713,6 +3713,7 @@ async fn run_policy_poll_loop_with_client<C: PolicyGatewayClient>(
// seed the applied-state trackers before OPA actually loads it.
let mut pending_result = None;

let mut last_poll_started = tokio::time::Instant::now();
// Initialize revision from the first poll and acknowledge the initial
// policy revision the supervisor actually loaded. A mismatched result is
// reconciled below instead of being recorded as already applied.
Expand Down Expand Up @@ -3780,7 +3781,18 @@ async fn run_policy_poll_loop_with_client<C: PolicyGatewayClient>(
let result = if let Some(result) = pending_result.take() {
result
} else {
tokio::time::sleep(next_poll_delay(&ctx.extension_credentials, interval)).await;
tokio::select! {
() = tokio::time::sleep(next_poll_delay(&ctx.extension_credentials, interval)) => {}
() = ctx.provider_credentials.refresh_requested() => {
// Fresh SSH children can need newly attached providers.
// Coalesce bursts and bound request-driven polling without
// increasing the ordinary idle reconciliation rate.
tokio::time::sleep_until(
last_poll_started + Duration::from_millis(250),
).await;
}
}
last_poll_started = tokio::time::Instant::now();
match client.poll_settings(&ctx.sandbox_id).await {
Ok(result) => {
let _ = ctx.workspace_tx.send(client.workspace());
Expand Down Expand Up @@ -5066,6 +5078,41 @@ network_policies:
handle.abort();
}

#[tokio::test]
async fn child_environment_request_wakes_poll_without_waiting_for_the_periodic_interval() {
let mut v1 = settings_poll_result(
Some(proto_policy_fixture()),
1,
openshell_core::proto::PolicySource::Sandbox,
);
v1.policy_hash = "same-policy".to_string();
let mut v2 = v1.clone();
v2.version = 2;
v2.config_revision = 200;
let engine = Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).unwrap());
let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1);
let mut ctx = policy_poll_test_context(
engine.clone(),
LoadedPolicyOrigin::Gateway {
revision: Some(loaded_revision),
has_last_valid_policy: true,
},
default_middleware_connector(),
);
ctx.interval_secs = 60;
let credentials = ctx.provider_credentials.clone();
let (client, polls, mut reports) = scripted_policy_gateway();
polls.send(v1).unwrap();
let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client));
expect_policy_report(&mut reports, 1).await;
polls.send(v2).unwrap();
expect_no_policy_report(&mut reports).await;
credentials.request_refresh();
expect_policy_report(&mut reports, 2).await;
assert_eq!(engine.current_generation(), 0);
handle.abort();
}

#[tokio::test]
async fn poll_rejects_first_tcp_expansion_and_reports_previous_policy_active() {
let v1 = settings_poll_result(
Expand Down
35 changes: 34 additions & 1 deletion crates/openshell-supervisor-process/src/ssh.rs
Original file line number Diff line number Diff line change
Expand Up @@ -714,6 +714,10 @@ impl russh::server::Handler for SshHandler {
if command.is_empty() {
return Ok(());
}
// Request prompt reconciliation through the existing credential owner.
// This is not a readiness receipt: the child still receives only the
// currently verified environment, and launchers must gate their needs.
self.provider_credentials.request_refresh();
self.start_shell(channel, session.handle(), Some(command))?;
Ok(())
}
Expand Down Expand Up @@ -2428,6 +2432,17 @@ mod tests {
/// namespace.
async fn authenticated_test_client_with_main(
main_session: Arc<MainSession>,
) -> russh::client::Handle<AcceptAnyServerKey> {
authenticated_test_client_with_credentials(
main_session,
ProviderCredentialState::from_child_env_snapshot(0, HashMap::new()),
)
.await
}

async fn authenticated_test_client_with_credentials(
main_session: Arc<MainSession>,
provider_credentials: ProviderCredentialState,
) -> russh::client::Handle<AcceptAnyServerKey> {
// Scoped so the `!Send` ThreadRng is dropped before the first await.
let host_key = {
Expand All @@ -2446,7 +2461,7 @@ mod tests {
None,
None,
None,
ProviderCredentialState::from_child_env_snapshot(0, HashMap::new()),
provider_credentials,
HashMap::new(),
ResolvedProcessIdentity::default(),
ProcessEnforcementMode::NetworkOnly,
Expand Down Expand Up @@ -2486,6 +2501,24 @@ mod tests {
authenticated_test_client_with_main(MainSession::inert()).await
}

#[tokio::test]
async fn authenticated_exec_requests_a_provider_refresh_without_modifying_the_snapshot() {
let credentials = ProviderCredentialState::from_child_env_snapshot(7, HashMap::new());
let client =
authenticated_test_client_with_credentials(MainSession::inert(), credentials.clone())
.await;
let channel = client.channel_open_session().await.expect("open session");
channel
.exec(true, "/usr/bin/true")
.await
.expect("send exec");
tokio::time::timeout(Duration::from_secs(1), credentials.refresh_requested())
.await
.expect("authenticated exec should wake the existing refresh owner");
assert_eq!(credentials.snapshot().revision, 7);
assert!(credentials.child_env_with_gcp_resolved().is_empty());
}

#[tokio::test]
async fn abrupt_transport_drop_releases_main_input_lease() {
let main_session = MainSession::inert();
Expand Down
7 changes: 7 additions & 0 deletions docs/sandboxes/manage-providers.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,13 @@ for runtime attach and detach behavior.

</Warning>

For runtime attachment in the monolithic supervisor, a new authenticated SSH
exec request asks the supervisor to refresh provider state promptly. Applications
still need to check their required environment before initializing a client:
the attachment response confirms the gateway change, while the supervisor
validates and installs its local snapshot separately. Process-only sidecars
retain their existing snapshot delivery path.

### Auto-Discovery Shortcut

When `providers_v2_enabled=false` and the trailing command in
Expand Down
Loading