From 13e876b965fa0f8366d4b0d464df57fe1f9f860a Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Thu, 11 Dec 2025 15:58:11 +0100 Subject: [PATCH 1/6] snapshot-editor: add TSC CLI helpers Add set/clear TSC CLI subcommands and wire them into snapshot-editor. Signed-off-by: Tomas Srnka --- src/snapshot-editor/Cargo.toml | 3 + src/snapshot-editor/src/edit_vmstate.rs | 6 +- src/snapshot-editor/src/main.rs | 7 ++ src/snapshot-editor/src/tsc.rs | 114 ++++++++++++++++++++++++ src/snapshot-editor/src/utils.rs | 7 +- 5 files changed, 131 insertions(+), 6 deletions(-) create mode 100644 src/snapshot-editor/src/tsc.rs diff --git a/src/snapshot-editor/Cargo.toml b/src/snapshot-editor/Cargo.toml index c8740a2ff8e..2aae4dd7610 100644 --- a/src/snapshot-editor/Cargo.toml +++ b/src/snapshot-editor/Cargo.toml @@ -27,5 +27,8 @@ vmm-sys-util = "0.15.0" [target.'cfg(target_arch = "aarch64")'.dependencies] clap-num = "1.2.0" +[target.'cfg(target_arch = "x86_64")'.dependencies] +kvm-ioctls = "0.19.1" + [lints] workspace = true diff --git a/src/snapshot-editor/src/edit_vmstate.rs b/src/snapshot-editor/src/edit_vmstate.rs index e538156814a..326f7fe72fe 100644 --- a/src/snapshot-editor/src/edit_vmstate.rs +++ b/src/snapshot-editor/src/edit_vmstate.rs @@ -51,9 +51,9 @@ fn edit( output_path: &PathBuf, f: impl Fn(MicrovmState) -> Result, ) -> Result<(), EditVmStateError> { - let snapshot = open_vmstate(vmstate_path)?; - let microvm_state = f(snapshot.data)?; - save_vmstate(microvm_state, output_path)?; + let mut snapshot = open_vmstate(vmstate_path)?; + snapshot.data = f(snapshot.data)?; + save_vmstate(&snapshot, output_path)?; Ok(()) } diff --git a/src/snapshot-editor/src/main.rs b/src/snapshot-editor/src/main.rs index fbc4a2ab883..e769a75c776 100644 --- a/src/snapshot-editor/src/main.rs +++ b/src/snapshot-editor/src/main.rs @@ -7,12 +7,14 @@ mod edit_memory; #[cfg(target_arch = "aarch64")] mod edit_vmstate; mod info; +mod tsc; mod utils; use edit_memory::{EditMemoryError, EditMemorySubCommand, edit_memory_command}; #[cfg(target_arch = "aarch64")] use edit_vmstate::{EditVmStateError, EditVmStateSubCommand, edit_vmstate_command}; use info::{InfoVmStateError, InfoVmStateSubCommand, info_vmstate_command}; +use tsc::{TscCommandError, TscSubCommand, tsc_command}; #[derive(Debug, thiserror::Error, displaydoc::Display)] enum SnapEditorError { @@ -23,6 +25,8 @@ enum SnapEditorError { EditVmState(#[from] EditVmStateError), /// Error during getting info from a vmstate file: {0} InfoVmState(#[from] InfoVmStateError), + /// Error during updating TSC metadata: {0} + EditTsc(#[from] TscCommandError), } #[derive(Debug, Parser)] @@ -41,6 +45,8 @@ enum Command { EditVmstate(EditVmStateSubCommand), #[command(subcommand)] InfoVmstate(InfoVmStateSubCommand), + #[command(subcommand)] + Tsc(TscSubCommand), } fn main_exec() -> Result<(), SnapEditorError> { @@ -51,6 +57,7 @@ fn main_exec() -> Result<(), SnapEditorError> { #[cfg(target_arch = "aarch64")] Command::EditVmstate(command) => edit_vmstate_command(command)?, Command::InfoVmstate(command) => info_vmstate_command(command)?, + Command::Tsc(command) => tsc_command(command)?, } Ok(()) diff --git a/src/snapshot-editor/src/tsc.rs b/src/snapshot-editor/src/tsc.rs new file mode 100644 index 00000000000..ee0ab85c76e --- /dev/null +++ b/src/snapshot-editor/src/tsc.rs @@ -0,0 +1,114 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::path::PathBuf; + +use clap::{Args, Subcommand}; + +use crate::utils::{open_vmstate, save_vmstate, UtilsError}; + +#[derive(Debug, thiserror::Error)] +pub enum TscCommandError { + #[error("{0}")] + Utils(#[from] UtilsError), + #[error("Missing --tsc-khz value; provide a target frequency in kHz.")] + MissingFrequency, + #[cfg(target_arch = "x86_64")] + #[error("Failed to open /dev/kvm: {0}")] + DetectOpenKvm(kvm_ioctls::Error), + #[cfg(target_arch = "x86_64")] + #[error("Failed to create KVM VM: {0}")] + DetectCreateVm(kvm_ioctls::Error), + #[cfg(target_arch = "x86_64")] + #[error("Failed to create KVM vCPU: {0}")] + DetectCreateVcpu(kvm_ioctls::Error), + #[cfg(target_arch = "x86_64")] + #[error("Failed to query TSC frequency from KVM: {0}")] + DetectQueryTsc(kvm_ioctls::Error), +} + +#[derive(Debug, Subcommand)] +pub enum TscSubCommand { + /// Set the saved TSC frequency (in kHz) for every vCPU in the vmstate file. + Set(SetTscArgs), + /// Remove the saved TSC frequency so Firecracker skips scaling on restore. + Clear(ClearTscArgs), +} + +#[derive(Debug, Args)] +pub struct SetTscArgs { + /// Path to the vmstate file to update. + #[arg(long)] + pub vmstate_path: PathBuf, + /// Optional output path; defaults to overwriting the input file. + #[arg(long)] + pub output_path: Option, + /// TSC frequency in kHz to embed in the vmstate snapshot. + #[arg(long, value_parser = clap::value_parser!(u32))] + pub tsc_khz: Option, +} + +#[derive(Debug, Args)] +pub struct ClearTscArgs { + /// Path to the vmstate file to update. + #[arg(long)] + pub vmstate_path: PathBuf, + /// Optional output path; defaults to overwriting the input file. + #[arg(long)] + pub output_path: Option, +} + +pub fn tsc_command(command: TscSubCommand) -> Result<(), TscCommandError> { + match command { + TscSubCommand::Set(args) => set_tsc(args), + TscSubCommand::Clear(args) => clear_tsc(args), + } +} + +fn set_tsc(args: SetTscArgs) -> Result<(), TscCommandError> { + #[cfg(target_arch = "x86_64")] + let freq = match args.tsc_khz { + Some(freq) => freq, + None => detect_host_tsc_khz()?, + }; + #[cfg(not(target_arch = "x86_64"))] + let freq = args.tsc_khz.ok_or(TscCommandError::MissingFrequency)?; + + let output_path = args + .output_path + .clone() + .unwrap_or(args.vmstate_path.clone()); + + let mut snapshot = open_vmstate(&args.vmstate_path)?; + for vcpu in &mut snapshot.data.vcpu_states { + vcpu.tsc_khz = Some(freq); + } + save_vmstate(&snapshot, &output_path)?; + Ok(()) +} + +#[cfg(target_arch = "x86_64")] +fn detect_host_tsc_khz() -> Result { + use kvm_ioctls::Kvm; + + let kvm = Kvm::new().map_err(TscCommandError::DetectOpenKvm)?; + let vm = kvm.create_vm().map_err(TscCommandError::DetectCreateVm)?; + let vcpu = vm + .create_vcpu(0) + .map_err(TscCommandError::DetectCreateVcpu)?; + vcpu.get_tsc_khz().map_err(TscCommandError::DetectQueryTsc) +} + +fn clear_tsc(args: ClearTscArgs) -> Result<(), TscCommandError> { + let output_path = args + .output_path + .clone() + .unwrap_or(args.vmstate_path.clone()); + + let mut snapshot = open_vmstate(&args.vmstate_path)?; + for vcpu in &mut snapshot.data.vcpu_states { + vcpu.tsc_khz = None; + } + save_vmstate(&snapshot, &output_path)?; + Ok(()) +} diff --git a/src/snapshot-editor/src/utils.rs b/src/snapshot-editor/src/utils.rs index 1d3d053d26a..99144875ac7 100644 --- a/src/snapshot-editor/src/utils.rs +++ b/src/snapshot-editor/src/utils.rs @@ -27,16 +27,17 @@ pub fn open_vmstate(snapshot_path: &PathBuf) -> Result, U Snapshot::load(&mut snapshot_reader).map_err(UtilsError::VmStateLoad) } -// This method is used only in aarch64 code so far #[allow(unused)] -pub fn save_vmstate(microvm_state: MicrovmState, output_path: &PathBuf) -> Result<(), UtilsError> { +pub fn save_vmstate( + snapshot: &Snapshot, + output_path: &PathBuf, +) -> Result<(), UtilsError> { let mut output_file = OpenOptions::new() .create(true) .write(true) .truncate(true) .open(output_path) .map_err(UtilsError::OutputFileOpen)?; - let mut snapshot = Snapshot::new(microvm_state); snapshot .save(&mut output_file) .map_err(UtilsError::VmStateSave)?; From fad1de639081fcac57b267e5492797236570c2a9 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Thu, 11 Dec 2025 16:17:51 +0100 Subject: [PATCH 2/6] docs: add snapshot-editor TSC CLI usage Document snapshot-editor TSC set/clear commands with usage examples. Signed-off-by: Tomas Srnka --- CHANGELOG.md | 2 ++ docs/snapshotting/snapshot-editor.md | 39 ++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5558cdf3795..8f6ed77f4f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to ## [Unreleased] ### Added +- Added snapshot-editor `tsc set/clear` commands to stamp or remove TSC + frequencies in vmstate snapshots (auto-detects host TSC on x86_64). ### Changed diff --git a/docs/snapshotting/snapshot-editor.md b/docs/snapshotting/snapshot-editor.md index 2c5e801796b..f44c0b7c530 100644 --- a/docs/snapshotting/snapshot-editor.md +++ b/docs/snapshotting/snapshot-editor.md @@ -136,3 +136,42 @@ Firecracker snapshot consists of 2 files: > ```bash > ./snapshot-editor info-vmstate vm-state --vmstate-path ./vmstate_file > ``` + +### `tsc` command + +#### `set` subcommand + +> Set the saved TSC frequency (in kHz) for every vCPU in a vmstate file. +> +> Arguments: +> +> - `VMSTATE_PATH` - path to the `vmstate` file +> - `OUTPUT_PATH` - optional output path; defaults to overwriting the input file +> - `--tsc-khz` - explicit TSC frequency (required on non-x86_64); on x86_64 it +> is auto-detected from the host if omitted +> +> Usage: +> +> ```bash +> snapshot-editor tsc set \ +> --vmstate-path ./vmstate_file \ +> --output-path ./new_vmstate_file \ +> --tsc-khz 2500000 +> ``` + +#### `clear` subcommand + +> Remove the saved TSC frequency so Firecracker skips scaling on restore. +> +> Arguments: +> +> - `VMSTATE_PATH` - path to the `vmstate` file +> - `OUTPUT_PATH` - optional output path; defaults to overwriting the input file +> +> Usage: +> +> ```bash +> snapshot-editor tsc clear \ +> --vmstate-path ./vmstate_file \ +> --output-path ./new_vmstate_file +> ``` From cef0905893bc4e72e2f7d8ccd33e5e4901df3004 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Thu, 11 Dec 2025 17:38:11 +0100 Subject: [PATCH 3/6] snapshot-editor: make tsc commands in-place Rewrite TSC set/clear to always modify the input vmstate file in-place. Update docs and changelog to reflect in-place behavior. Signed-off-by: Tomas Srnka --- CHANGELOG.md | 2 +- docs/snapshotting/snapshot-editor.md | 6 +----- src/snapshot-editor/src/tsc.rs | 22 +++------------------- 3 files changed, 5 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f6ed77f4f8..76cf886c827 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to ### Added - Added snapshot-editor `tsc set/clear` commands to stamp or remove TSC - frequencies in vmstate snapshots (auto-detects host TSC on x86_64). + frequencies in vmstate snapshots in place (auto-detects host TSC on x86_64). ### Changed diff --git a/docs/snapshotting/snapshot-editor.md b/docs/snapshotting/snapshot-editor.md index f44c0b7c530..42f5e75381c 100644 --- a/docs/snapshotting/snapshot-editor.md +++ b/docs/snapshotting/snapshot-editor.md @@ -146,7 +146,6 @@ Firecracker snapshot consists of 2 files: > Arguments: > > - `VMSTATE_PATH` - path to the `vmstate` file -> - `OUTPUT_PATH` - optional output path; defaults to overwriting the input file > - `--tsc-khz` - explicit TSC frequency (required on non-x86_64); on x86_64 it > is auto-detected from the host if omitted > @@ -155,7 +154,6 @@ Firecracker snapshot consists of 2 files: > ```bash > snapshot-editor tsc set \ > --vmstate-path ./vmstate_file \ -> --output-path ./new_vmstate_file \ > --tsc-khz 2500000 > ``` @@ -166,12 +164,10 @@ Firecracker snapshot consists of 2 files: > Arguments: > > - `VMSTATE_PATH` - path to the `vmstate` file -> - `OUTPUT_PATH` - optional output path; defaults to overwriting the input file > > Usage: > > ```bash > snapshot-editor tsc clear \ -> --vmstate-path ./vmstate_file \ -> --output-path ./new_vmstate_file +> --vmstate-path ./vmstate_file > ``` diff --git a/src/snapshot-editor/src/tsc.rs b/src/snapshot-editor/src/tsc.rs index ee0ab85c76e..3d11145dde2 100644 --- a/src/snapshot-editor/src/tsc.rs +++ b/src/snapshot-editor/src/tsc.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use clap::{Args, Subcommand}; -use crate::utils::{open_vmstate, save_vmstate, UtilsError}; +use crate::utils::{UtilsError, open_vmstate, save_vmstate}; #[derive(Debug, thiserror::Error)] pub enum TscCommandError { @@ -40,9 +40,6 @@ pub struct SetTscArgs { /// Path to the vmstate file to update. #[arg(long)] pub vmstate_path: PathBuf, - /// Optional output path; defaults to overwriting the input file. - #[arg(long)] - pub output_path: Option, /// TSC frequency in kHz to embed in the vmstate snapshot. #[arg(long, value_parser = clap::value_parser!(u32))] pub tsc_khz: Option, @@ -53,9 +50,6 @@ pub struct ClearTscArgs { /// Path to the vmstate file to update. #[arg(long)] pub vmstate_path: PathBuf, - /// Optional output path; defaults to overwriting the input file. - #[arg(long)] - pub output_path: Option, } pub fn tsc_command(command: TscSubCommand) -> Result<(), TscCommandError> { @@ -74,16 +68,11 @@ fn set_tsc(args: SetTscArgs) -> Result<(), TscCommandError> { #[cfg(not(target_arch = "x86_64"))] let freq = args.tsc_khz.ok_or(TscCommandError::MissingFrequency)?; - let output_path = args - .output_path - .clone() - .unwrap_or(args.vmstate_path.clone()); - let mut snapshot = open_vmstate(&args.vmstate_path)?; for vcpu in &mut snapshot.data.vcpu_states { vcpu.tsc_khz = Some(freq); } - save_vmstate(&snapshot, &output_path)?; + save_vmstate(&snapshot, &args.vmstate_path)?; Ok(()) } @@ -100,15 +89,10 @@ fn detect_host_tsc_khz() -> Result { } fn clear_tsc(args: ClearTscArgs) -> Result<(), TscCommandError> { - let output_path = args - .output_path - .clone() - .unwrap_or(args.vmstate_path.clone()); - let mut snapshot = open_vmstate(&args.vmstate_path)?; for vcpu in &mut snapshot.data.vcpu_states { vcpu.tsc_khz = None; } - save_vmstate(&snapshot, &output_path)?; + save_vmstate(&snapshot, &args.vmstate_path)?; Ok(()) } From 7ca7df46eb69dddb5e50d8b6f93783dd7fe6f3ee Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Sun, 14 Dec 2025 20:10:02 +0100 Subject: [PATCH 4/6] snapshot-editor: add in-place tsc test Add a unit test that sets and clears TSC values in-place. Verify vCPU metadata updates when reloading the vmstate. Signed-off-by: Tomas Srnka --- src/snapshot-editor/src/tsc.rs | 66 ++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/snapshot-editor/src/tsc.rs b/src/snapshot-editor/src/tsc.rs index 3d11145dde2..b987ae5dcf3 100644 --- a/src/snapshot-editor/src/tsc.rs +++ b/src/snapshot-editor/src/tsc.rs @@ -96,3 +96,69 @@ fn clear_tsc(args: ClearTscArgs) -> Result<(), TscCommandError> { save_vmstate(&snapshot, &args.vmstate_path)?; Ok(()) } + +#[cfg(test)] +mod tests { + use std::time::{SystemTime, UNIX_EPOCH}; + + use vmm::persist::MicrovmState; + use vmm::snapshot::Snapshot; + + use super::*; + use crate::utils::save_vmstate; + + fn temp_vmstate_path() -> PathBuf { + let mut path = std::env::temp_dir(); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + path.push(format!( + "snapshot-editor-tsc-{}-{}.bin", + std::process::id(), + nanos + )); + path + } + + #[test] + fn test_tsc_set_and_clear_in_place() { + let vmstate_path = temp_vmstate_path(); + + // Start from a valid vmstate snapshot. + let snapshot = Snapshot::new(MicrovmState::default()); + save_vmstate(&snapshot, &vmstate_path).expect("save initial vmstate"); + + let set_freq = 123_456u32; + set_tsc(SetTscArgs { + vmstate_path: vmstate_path.clone(), + tsc_khz: Some(set_freq), + }) + .expect("tsc set should succeed"); + + let snapshot = open_vmstate(&vmstate_path).expect("vmstate after set"); + assert!( + snapshot + .data + .vcpu_states + .iter() + .all(|vcpu| vcpu.tsc_khz == Some(set_freq)) + ); + + clear_tsc(ClearTscArgs { + vmstate_path: vmstate_path.clone(), + }) + .expect("tsc clear should succeed"); + + let snapshot = open_vmstate(&vmstate_path).expect("vmstate after clear"); + assert!( + snapshot + .data + .vcpu_states + .iter() + .all(|vcpu| vcpu.tsc_khz.is_none()) + ); + + let _ = std::fs::remove_file(vmstate_path); + } +} From 625432b38f733c1951ee66ca3c701cad47812ab0 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Sun, 14 Dec 2025 20:21:18 +0100 Subject: [PATCH 5/6] snapshot-editor: silence missing freq warning on x86_64 Allow MissingFrequency to be dead_code on x86_64 to avoid warnings. Signed-off-by: Tomas Srnka --- src/snapshot-editor/src/tsc.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/snapshot-editor/src/tsc.rs b/src/snapshot-editor/src/tsc.rs index b987ae5dcf3..dacdb70fd3d 100644 --- a/src/snapshot-editor/src/tsc.rs +++ b/src/snapshot-editor/src/tsc.rs @@ -11,6 +11,7 @@ use crate::utils::{UtilsError, open_vmstate, save_vmstate}; pub enum TscCommandError { #[error("{0}")] Utils(#[from] UtilsError), + #[cfg_attr(target_arch = "x86_64", allow(dead_code))] #[error("Missing --tsc-khz value; provide a target frequency in kHz.")] MissingFrequency, #[cfg(target_arch = "x86_64")] From 4f28b32821dc93a9f6b7a04385ddbfbd7dd981ac Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Sun, 14 Dec 2025 21:16:58 +0100 Subject: [PATCH 6/6] changelog: add spacing after heading Insert a blank line after the Added heading so mdformat style passes. Signed-off-by: Tomas Srnka --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76cf886c827..3c8e4049403 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to ## [Unreleased] ### Added + - Added snapshot-editor `tsc set/clear` commands to stamp or remove TSC frequencies in vmstate snapshots in place (auto-detects host TSC on x86_64).