Skip to content

Turbopack: Add a custom serde_self_describing Serializer/Deserializer #86885

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bgw wants to merge 2 commits into
base: canary
Choose a base branch
from
+1,196 −122
Open

Turbopack: Add a custom serde_self_describing Serializer/Deserializer #86885

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
662c1b5
Turbopack: Add a custom serde_self_describing Serializer/Deserializer
bgw Dec 3, 2025
24a2ad6
Use the new self-describing format, remove serde_json wrapper
bgw Dec 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 2 additions & 14 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

38 changes: 19 additions & 19 deletions crates/next-core/src/next_config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ pub struct NextConfig {
cache_handler: Option<RcStr>,
#[bincode(with_serde)]
cache_handlers: Option<FxIndexMap<RcStr, RcStr>>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
env: FxIndexMap<String, JsonValue>,
experimental: ExperimentalConfig,
images: ImageConfig,
Expand All @@ -101,12 +101,12 @@ pub struct NextConfig {
react_production_profiling: Option<bool>,
react_strict_mode: Option<bool>,
transpile_packages: Option<Vec<RcStr>>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
modularize_imports: Option<FxIndexMap<String, ModularizeImportPackageConfig>>,
dist_dir: RcStr,
dist_dir_root: RcStr,
deployment_id: Option<RcStr>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
sass_options: Option<serde_json::Value>,
trailing_slash: Option<bool>,
asset_prefix: Option<RcStr>,
Expand All @@ -119,9 +119,9 @@ pub struct NextConfig {
output: Option<OutputType>,
turbopack: Option<TurbopackConfig>,
production_browser_source_maps: bool,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
output_file_tracing_includes: Option<serde_json::Value>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
output_file_tracing_excludes: Option<serde_json::Value>,
// TODO: This option is not respected, it uses Turbopack's root instead.
output_file_tracing_root: Option<RcStr>,
Expand Down Expand Up @@ -154,9 +154,9 @@ pub struct NextConfig {
http_agent_options: HttpAgentConfig,
on_demand_entries: OnDemandEntriesConfig,
powered_by_header: bool,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
public_runtime_config: FxIndexMap<String, serde_json::Value>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
server_runtime_config: FxIndexMap<String, serde_json::Value>,
static_page_generation_timeout: f64,
target: Option<String>,
Expand Down Expand Up @@ -686,9 +686,9 @@ pub enum RemotePatternProtocol {
)]
#[serde(rename_all = "camelCase")]
pub struct TurbopackConfig {
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
pub rules: Option<FxIndexMap<RcStr, RuleConfigCollection>>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
pub resolve_alias: Option<FxIndexMap<RcStr, JsonValue>>,
pub resolve_extensions: Option<Vec<RcStr>>,
pub debug_ids: Option<bool>,
Expand Down Expand Up @@ -955,7 +955,7 @@ pub struct ExperimentalConfig {
/// @see [api reference](https://nextjs.org/docs/app/api-reference/next-config-js/mdxRs)
mdx_rs: Option<MdxRsOptions>,
strict_next_head: Option<bool>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
swc_plugins: Option<Vec<(RcStr, serde_json::Value)>>,
external_middleware_rewrites_resolve: Option<bool>,
scroll_restoration: Option<bool>,
Expand All @@ -964,7 +964,7 @@ pub struct ExperimentalConfig {
middleware_prefetch: Option<MiddlewarePrefetchType>,
/// optimizeCss can be boolean or critters' option object
/// Use Record<string, unknown> as critters doesn't export its Option type ([link](https://github.com/GoogleChromeLabs/critters/blob/a590c05f9197b656d2aeaae9369df2483c26b072/packages/critters/src/index.d.ts))
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
optimize_css: Option<serde_json::Value>,
next_script_workers: Option<bool>,
web_vitals_attribution: Option<Vec<RcStr>>,
Expand All @@ -982,15 +982,15 @@ pub struct ExperimentalConfig {
adjust_font_fallbacks_with_size_adjust: Option<bool>,
after: Option<bool>,
app_document_preloading: Option<bool>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
cache_life: Option<FxIndexMap<String, CacheLifeProfile>>,
case_sensitive_routes: Option<bool>,
cpus: Option<f64>,
cra_compat: Option<bool>,
disable_optimized_loading: Option<bool>,
disable_postcss_preset_env: Option<bool>,
esm_externals: Option<EsmExternals>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
extension_alias: Option<serde_json::Value>,
external_dir: Option<bool>,
/// If set to `false`, webpack won't fall back to polyfill Node.js modules
Expand All @@ -1005,7 +1005,7 @@ pub struct ExperimentalConfig {
instrumentation_hook: Option<bool>,
client_trace_metadata: Option<Vec<String>>,
large_page_data_bytes: Option<f64>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
logging: Option<serde_json::Value>,
memory_based_workers_count: Option<bool>,
/// Optimize React APIs for server builds.
Expand All @@ -1024,7 +1024,7 @@ pub struct ExperimentalConfig {
/// @internal Used by the Next.js internals only.
trust_host_header: Option<bool>,

#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
url_imports: Option<serde_json::Value>,
/// This option is to enable running the Webpack build in a worker thread
/// (doesn't apply to Turbopack).
Expand Down Expand Up @@ -1395,7 +1395,7 @@ pub struct ResolveExtensions(Option<Vec<RcStr>>);

#[turbo_tasks::value(transparent)]
pub struct SwcPlugins(
#[bincode(with = "turbo_bincode::serde_json")] Vec<(RcStr, serde_json::Value)>,
#[bincode(with = "turbo_bincode::serde_self_describing")] Vec<(RcStr, serde_json::Value)>,
);

#[turbo_tasks::value(transparent)]
Expand All @@ -1413,7 +1413,7 @@ pub struct OptionServerActions(Option<ServerActions>);

#[turbo_tasks::value(transparent)]
pub struct OptionJsonValue(
#[bincode(with = "turbo_bincode::serde_json")] pub Option<serde_json::Value>,
#[bincode(with = "turbo_bincode::serde_self_describing")] pub Option<serde_json::Value>,
);

fn turbopack_config_documentation_link() -> RcStr {
Expand Down Expand Up @@ -1466,7 +1466,7 @@ impl Issue for InvalidLoaderRuleRenameAsIssue {

#[turbo_tasks::value(shared)]
struct InvalidLoaderRuleConditionIssue {
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
condition: ConfigConditionItem,
config_file_path: FileSystemPath,
}
Expand Down Expand Up @@ -2207,7 +2207,7 @@ impl NextConfig {
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, Encode, Decode)]
#[serde(rename_all = "camelCase")]
pub struct JsConfig {
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
compiler_options: Option<serde_json::Value>,
}

Expand Down
1 change: 0 additions & 1 deletion turbopack/crates/turbo-bincode/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,5 @@ indexmap = { workspace = true }
mime = { workspace = true }
ringmap = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
smallvec = { workspace = true }
unty = { workspace = true }
71 changes: 1 addition & 70 deletions turbopack/crates/turbo-bincode/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#[doc(hidden)]
pub mod macro_helpers;
pub mod serde_self_describing;

use std::{any::Any, ptr::copy_nonoverlapping};

Expand Down Expand Up @@ -472,76 +473,6 @@ pub mod mime_option {
}
}

/// Encode/decode as a serialized string encoded using `serde_json`.
///
/// This encodes less efficiently than `#[bincode(with_serde)]` would, but avoids [bincode's known
/// compatibility issues][serde-issues]. Use this for infrequently-serialized types and when you're
/// unsure if the underlying type may trigger a serde compatibility issue.
///
/// In the future this could be replaced with a more efficient serde-compatible self-describing
/// format with a compact binary representation (e.g. pot or MessagePack), but `serde_json` is
/// convenient because it avoids introducing additional dependencies.
///
/// [serde-issues]: https://docs.rs/bincode/latest/bincode/serde/index.html#known-issues
pub mod serde_json {
use super::*;

pub fn encode<E: Encoder, T: serde::Serialize>(
value: &T,
encoder: &mut E,
) -> Result<(), EncodeError> {
let json_str =
::serde_json::to_string(value).map_err(|e| EncodeError::OtherString(e.to_string()))?;
Encode::encode(&json_str, encoder)
}

pub fn decode<Context, D: Decoder<Context = Context>, T: serde::de::DeserializeOwned>(
decoder: &mut D,
) -> Result<T, DecodeError> {
let json_str: String = Decode::decode(decoder)?;
::serde_json::from_str(&json_str).map_err(|e| DecodeError::OtherString(e.to_string()))
}

pub fn borrow_decode<
'de,
Context,
D: BorrowDecoder<'de, Context = Context>,
T: serde::de::Deserialize<'de>,
>(
decoder: &mut D,
) -> Result<T, DecodeError> {
let json_str: &str = BorrowDecode::borrow_decode(decoder)?;
::serde_json::from_str(json_str).map_err(|e| DecodeError::OtherString(e.to_string()))
}

#[cfg(test)]
mod tests {
use ::serde_json::{Value, json};
use bincode::{decode_from_slice, encode_to_vec};

use super::*;

#[test]
fn test_roundtrip() {
let cfg = bincode::config::standard();

#[derive(Encode, Decode)]
struct Wrapper(#[bincode(with = "crate::serde_json")] Value);

let value1 = Wrapper(json!({
"key1": [1, 2, 3],
"key2": [4, 5, 6]
}));

let value2: Wrapper = decode_from_slice(&encode_to_vec(&value1, cfg).unwrap(), cfg)
.unwrap()
.0;

assert_eq!(value1.0, value2.0);
}
}
}

pub mod either {
use ::either::Either;

Expand Down
Loading
Loading