Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .changeset/cli-migrate-cfg-gate.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
---
---

Gate Unix-only migration APIs and directory permissions behind cfg(unix) (#1121)
17 changes: 17 additions & 0 deletions .github/workflows/checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,23 @@ jobs:
- name: Release build
run: cargo build --workspace --all-features --release

windows-cli-check:
name: Windows CLI Check
runs-on: windows-latest
needs: changes
if: needs.changes.outputs.rust == 'true'
steps:
- name: Checkout
uses: actions/checkout@v4.4.0
with:
persist-credentials: false
- name: Install Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1.17.0
- name: CLI builds on Windows
run: cargo build -p nostream-cli --no-default-features
- name: CLI refuses the migration on Windows
run: cargo test -p nostream-cli --no-default-features --test migrate_cli_unsupported

changeset-check:
name: Changeset Required
runs-on: ubuntu-latest
Expand Down
30 changes: 26 additions & 4 deletions crates/nostream-cli/src/commands/migrate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,13 @@
use std::fs;
use std::io;
use std::io::Write;
#[cfg(unix)]
use std::os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt};
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};

use nostream_storage::migration::{MigratedTable, MigrationRow, MigrationSource, MigrationTarget};
use nostream_storage::{LmdbBackend, PostgresBackend};

Check failure on line 22 in crates/nostream-cli/src/commands/migrate.rs

View workflow job for this annotation

GitHub Actions / Windows CLI Check

unused import: `PostgresBackend`
use sha2::{Digest, Sha256};

use crate::config::CommandOutput;
Expand All @@ -42,6 +43,7 @@
}

/// `migrate pg-to-lmdb`: the copy, or the verification pass when `verify` is set.
#[cfg(unix)]
pub fn run(options: MigrateOptions, out: &mut dyn Write) -> Result<CommandOutput, CliError> {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
Expand Down Expand Up @@ -110,6 +112,16 @@
})
}

#[cfg(not(unix))]
pub fn run(_options: MigrateOptions, out: &mut dyn Write) -> Result<CommandOutput, CliError> {
writeln!(out, "migrate pg-to-lmdb is unsupported on this platform")
.map_err(|e| CliError::Migrate(e.to_string()))?;
Ok(CommandOutput {
lines: Vec::new(),
exit_code: 1,
})
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
}

/// The copy over an injected source, target and writer, which is how the whole run is tested
/// without a process and without a database.
///
Expand Down Expand Up @@ -359,7 +371,7 @@
/// a table finished. A manifest that records none of that cannot have written a row (R7),
/// so a fresh store beside it is honest; one that records progress is only meaningful
/// beside the store the progress went into.
fn records_progress(&self) -> bool {

Check failure on line 374 in crates/nostream-cli/src/commands/migrate.rs

View workflow job for this annotation

GitHub Actions / Windows CLI Check

method `records_progress` is never used
self.tables
.values()
.any(|entry| entry.rows != 0 || !entry.after.is_empty() || entry.complete)
Expand Down Expand Up @@ -575,10 +587,11 @@
// Without the file sync, a power loss can leave the new name pointing at a truncated
// manifest; `read_manifest` then refuses it, and the nonempty-target rule refuses to
// migrate over the store it describes, so the operator's only way forward is a discard.
let mut file = fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
let mut open_options = fs::OpenOptions::new();
open_options.write(true).create_new(true);
#[cfg(unix)]
open_options.mode(0o600);
let mut file = open_options
.open(&temporary)
.map_err(|error| CliError::Migrate(format!("{}: {error}", temporary.display())))?;
file.write_all(&bytes)
Expand Down Expand Up @@ -607,7 +620,7 @@
/// The source the run reads, as the setting that named it: `db.internal:5432/nostream (from
/// DB_URI)`. An unconfigured source is refused here, because `from_env` would otherwise fall
/// back to a local server's `postgres` database and copy whatever it holds.
fn describe_source() -> Result<String, CliError> {

Check failure on line 623 in crates/nostream-cli/src/commands/migrate.rs

View workflow job for this annotation

GitHub Actions / Windows CLI Check

function `describe_source` is never used
if let Some(uri) = std::env::var("DB_URI")
.ok()
.filter(|uri| !uri.trim().is_empty())
Expand Down Expand Up @@ -655,7 +668,7 @@
/// socket forms `postgres:///?host=/var/run/postgresql` and `postgres:///?hostaddr=127.0.0.1`,
/// whose authority is empty by design. A `?` inside a `#` fragment is not a query, because sqlx
/// ignores the fragment - `postgres://#?host=/sock` has no host at all.
fn is_url_with_host(uri: &str) -> bool {

Check failure on line 671 in crates/nostream-cli/src/commands/migrate.rs

View workflow job for this annotation

GitHub Actions / Windows CLI Check

function `is_url_with_host` is never used
let Some((scheme, rest)) = uri.split_once("://") else {
return false;
};
Expand Down Expand Up @@ -701,7 +714,7 @@
/// no `://` is redacted too. Matching credential-shaped keys instead would miss a
/// percent-encoded name, and when a permission error is the problem the server's own message
/// names the role.
fn without_credentials(uri: &str) -> String {

Check failure on line 717 in crates/nostream-cli/src/commands/migrate.rs

View workflow job for this annotation

GitHub Actions / Windows CLI Check

function `without_credentials` is never used
let uri = uri.split('?').next().unwrap_or(uri);
let uri = uri.split('#').next().unwrap_or(uri);
let (scheme, rest) = match uri.split_once("://") {
Expand All @@ -722,7 +735,7 @@
/// `DB_ACQUIRE_CONNECTION_TIMEOUT` is what the shipped `.env.example` sets, and a probe that
/// inherited it would be a minute of silence on a mistyped host: the budget is that setting
/// capped at `MIGRATION_CONNECT_CAP_MS`, so the ceiling holds whatever the pool is told.
fn probe_budget() -> Duration {

Check failure on line 738 in crates/nostream-cli/src/commands/migrate.rs

View workflow job for this annotation

GitHub Actions / Windows CLI Check

function `probe_budget` is never used
let cap = Duration::from_millis(nostream_storage::postgres::MIGRATION_CONNECT_CAP_MS);
std::env::var("DB_ACQUIRE_CONNECTION_TIMEOUT")
.ok()
Expand All @@ -739,6 +752,7 @@
/// sends the copy somewhere the operator did not name. Both are refused before anything is
/// opened. A target that does not exist yet cannot be judged here, so `create_target` creates it
/// before the store opens and runs this check again over what is then on disk.
#[cfg(unix)]
fn refuse_unsafe_target(options: &MigrateOptions) -> Result<(), CliError> {
refuse_unsafe_target_for(options, effective_uid())
}
Expand All @@ -747,12 +761,14 @@
///
/// `std::fs` exposes a file's owner but not the caller's, and reading it here costs no
/// dependency: this crate runs on Linux, where the kernel publishes it in one line.
#[cfg(unix)]
fn effective_uid() -> Option<u32> {
let status = fs::read_to_string("/proc/self/status").ok()?;
let line = status.lines().find(|line| line.starts_with("Uid:"))?;
line.split_whitespace().nth(2)?.parse().ok()
}

#[cfg(unix)]
fn refuse_unsafe_target_for(options: &MigrateOptions, caller: Option<u32>) -> Result<(), CliError> {
for path in [&options.target, &store_path(options)] {
let metadata = match fs::symlink_metadata(path) {
Expand Down Expand Up @@ -849,6 +865,7 @@
/// the entry itself, so a group- or world-writable ancestor lets another user move this target
/// aside and put their own there before the store opens. The sticky bit is the exception that
/// makes `/tmp` usable: it refuses the rename of an entry its writer does not own.
#[cfg(unix)]
fn refuse_swappable_ancestry(target: &Path) -> Result<(), CliError> {
let mut path = match target.parent() {
Some(parent) if parent.as_os_str().is_empty() => PathBuf::from("."),
Expand Down Expand Up @@ -893,6 +910,7 @@
/// by `create_dir_all` without a word. Creating it here, under the mode the store itself uses,
/// and checking again closes the window: after this call the target is a directory this process
/// made, or the run has ended.
#[cfg(unix)]
fn create_target(options: &MigrateOptions) -> Result<(), CliError> {
fs::DirBuilder::new()
.recursive(true)
Expand All @@ -913,7 +931,7 @@
}

/// The line and exit code a verify answers when the target has a manifest but no store.
fn no_store(options: &MigrateOptions, out: &mut dyn Write) -> Result<CommandOutput, CliError> {

Check failure on line 934 in crates/nostream-cli/src/commands/migrate.rs

View workflow job for this annotation

GitHub Actions / Windows CLI Check

function `no_store` is never used
writeln!(
out,
"No store at {}: a re-run resumes once the store is back. To start over, use a fresh \
Expand All @@ -935,7 +953,7 @@

/// Whether the target already holds an LMDB environment, asked the way R8 asks it: `<target>/
/// lmdb/data.mdb` is a file. A directory alone is not a store, and neither is a lock file.
fn store_present(options: &MigrateOptions) -> bool {

Check failure on line 956 in crates/nostream-cli/src/commands/migrate.rs

View workflow job for this annotation

GitHub Actions / Windows CLI Check

function `store_present` is never used
nostream_storage::backend::store_exists(&options.target)
}

Expand Down Expand Up @@ -979,7 +997,7 @@
/// process then answers `BadOpenOptions` — so a probe here would make a resume fail exactly
/// when its store is present. `tests::the_resume_check_leaves_the_store_openable` drives both
/// steps for that reason.
fn check_resume_store(

Check failure on line 1000 in crates/nostream-cli/src/commands/migrate.rs

View workflow job for this annotation

GitHub Actions / Windows CLI Check

function `check_resume_store` is never used
existing: Option<&Manifest>,
options: &MigrateOptions,
) -> Result<(), CliError> {
Expand All @@ -995,7 +1013,7 @@
}

/// Opens the store a copy writes, creating it when it is not there.
async fn open_target(root: &Path) -> Result<LmdbBackend, CliError> {

Check failure on line 1016 in crates/nostream-cli/src/commands/migrate.rs

View workflow job for this annotation

GitHub Actions / Windows CLI Check

function `open_target` is never used
LmdbBackend::open(root)
.await
.map_err(|error| CliError::Migrate(error.to_string()))
Expand Down Expand Up @@ -1038,6 +1056,7 @@
/// A link where the data file belongs is followed by `is_file()` and by LMDB both, so the
/// copy would read or write through it and heed would put `lock.mdb` beside the link's
/// target. The predicate over the file's own metadata is what AC18 asks for.
#[cfg(unix)]
#[test]
#[allow(clippy::expect_used)]
fn a_symlinked_data_file_is_refused() {
Expand All @@ -1063,6 +1082,7 @@
/// checked against the caller rather than the mode. Measured as if a different uid were
/// running, because a directory this process owns cannot be made to belong to another user
/// without root.
#[cfg(unix)]
#[test]
#[allow(clippy::expect_used)]
fn a_target_owned_by_another_user_is_refused() {
Expand Down Expand Up @@ -1091,6 +1111,7 @@
/// needs no ownership of the target to do it: a rename takes write permission on the holder.
/// Without the sticky bit that permission is refused; `/tmp` is exactly this mode plus the
/// sticky bit, which is why the bit, and not the mode alone, is the question.
#[cfg(unix)]
#[test]
#[allow(clippy::expect_used)]
fn a_target_under_a_swappable_directory_is_refused() {
Expand Down Expand Up @@ -1121,6 +1142,7 @@
/// target is created and then judged. Both halves are needed: the creation gives the second
/// check something to see, and the second check is what refuses a directory another user
/// planted while the source was being probed.
#[cfg(unix)]
#[test]
#[allow(clippy::expect_used)]
fn a_missing_target_is_created_and_then_judged() {
Expand Down
3 changes: 3 additions & 0 deletions crates/nostream-cli/tests/migrate_cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@
//! These targets may unwrap and assert by panicking: a failed assertion is a failed test.

#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
// The copy drives the Unix-only half of the command; the refusal on other platforms is pinned in
// `migrate_cli_unsupported.rs`, which is the only file that compiles there.
#![cfg(unix)]

use std::collections::BTreeMap;
use std::process::Command;
Expand Down
29 changes: 29 additions & 0 deletions crates/nostream-cli/tests/migrate_cli_unsupported.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
//! The `migrate pg-to-lmdb` command where the copy cannot run: every part of it needs Unix file
//! modes, so the command still exists but refuses, and this file pins what it answers.
//!
//! The copy, the manifest, resume and verify live in `migrate_cli.rs`, which compiles only on
//! Unix. CI runs this one from the `windows-latest` job.

#![cfg(not(unix))]
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]

use std::process::Command;

/// The refusal names the command, says the platform is the reason, and exits non-zero, so an
/// operator reading a pipeline log learns why rather than seeing a compile error or a hang.
#[test]
fn the_command_is_refused_with_a_named_reason_and_exit_1() {
let directory = tempfile::tempdir().expect("a temporary directory");
let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli"))
.args(["migrate", "pg-to-lmdb", "--target"])
.arg(directory.path())
.output()
.expect("the binary runs");

assert_eq!(output.status.code(), Some(1), "the refusal exits 1");
let stdout = String::from_utf8_lossy(&output.stdout);
assert!(
stdout.contains("migrate pg-to-lmdb is unsupported on this platform"),
"the refusal names the command and the reason: {stdout}"
);
}
2 changes: 1 addition & 1 deletion crates/nostream-config/src/load.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
//! validate never aborts the process. It is reported as [`Issue`] values and the caller gets
//! the shipped defaults, so a broken `settings.yaml` cannot stop a relay from starting.

use std::ffi::OsStr;
use std::fs;
use std::path::{Path, PathBuf};

Expand Down Expand Up @@ -70,6 +69,7 @@ pub fn load_document(path: &Path) -> Result<Value, ConfigError> {
/// and non-UTF-8 paths are handled because the work is on `OsStr` bytes, not a `to_str`.
#[cfg(unix)]
fn trim_trailing_separators(path: &Path) -> &Path {
use std::ffi::OsStr;
use std::os::unix::ffi::OsStrExt;

let bytes = path.as_os_str().as_bytes();
Expand Down
9 changes: 8 additions & 1 deletion crates/nostream-storage/src/backend.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Backend selection and the LMDB environment. Mirrors `src/utils/event-store.ts` (the
//! selection half) and the storage boundary in `src/@types/adapters.ts`.

#[cfg(feature = "lmdb")]
#[cfg(all(feature = "lmdb", unix))]
use std::os::unix::fs::DirBuilderExt;
use std::path::{Path, PathBuf};

Expand Down Expand Up @@ -99,6 +99,7 @@ impl LmdbEnv {
// host whose umask is 002 `create_dir_all` leaves it group-writable, which is both a
// wider audience than the store's own 0600 files and something the migration's target
// check refuses on the next run.
#[cfg(unix)]
std::fs::DirBuilder::new()
.recursive(true)
.mode(0o700)
Expand All @@ -107,6 +108,12 @@ impl LmdbEnv {
path: path.clone(),
source,
})?;
#[cfg(not(unix))]
std::fs::create_dir_all(&path).map_err(|source| StorageError::CreateDirectory {
path: path.clone(),
source,
})?;

let env = Self::open_with(&path, map_size, EnvFlags::empty())?;
Ok(Self { env, path })
}
Expand Down
Loading