[Turbopack] Implement support for webpack’s stats.json (#70996)

This allows bundle analyzer tools to visualize and explore the output of
Next.js apps built with Turbopack.

Like Turbopack’s support for manifest files, this creates individual
stats files per entry point and merges them into a single stats file for
the app.

To do:

- [x] Make opt-in and build-only
- [x] Apply to pages router

Test Plan: `TURBOPACK_BUILD=1 TURBOPACK=1 pnpm build && pnpm dlx
webpack-bundle-analyzer .next/server/webpack-stats.
json .`

crates/next-api/Cargo.toml

@@ -17,6 +17,7 @@ anyhow = { workspace = true, features = ["backtrace"] }
 futures = { workspace = true }
 indexmap = { workspace = true }
 next-core = { workspace = true }
+regex = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 shadow-rs = { workspace = true }

crates/next-api/src/app.rs

@@ -1,5 +1,5 @@
 use anyhow::{Context, Result};
-use indexmap::{IndexMap, IndexSet};
+use indexmap::{indexset, IndexMap, IndexSet};
 use next_core::{
     all_assets_from_entries,
     app_segment_config::NextSegmentConfig,
@@ -81,6 +81,7 @@ use crate::{
     project::Project,
     route::{AppPageRoute, Endpoint, Route, Routes, WrittenEndpoint},
     server_actions::create_server_actions_manifest,
+    webpack_stats::generate_webpack_stats,
 };
 
 #[turbo_tasks::value]
@@ -831,8 +832,8 @@ impl AppEndpoint {
 
         let server_path = node_root.join("server".into());
 
-        let mut server_assets = vec![];
-        let mut client_assets = vec![];
+        let mut server_assets = indexset![];
+        let mut client_assets = indexset![];
         // assets to add to the middleware manifest (to be loaded in the edge runtime).
         let mut middleware_assets = vec![];
 
@@ -854,7 +855,7 @@ impl AppEndpoint {
 
                 let mut client_shared_chunks_paths = vec![];
                 for chunk in client_shared_chunk_group.assets.await?.iter().copied() {
-                    client_assets.push(chunk);
+                    client_assets.insert(chunk);
 
                     let chunk_path = chunk.ident().path().await?;
                     if chunk_path.extension_ref() == Some("js") {
@@ -997,7 +998,7 @@ impl AppEndpoint {
                         File::from(serde_json::to_string_pretty(&app_build_manifest)?).into(),
                     ),
                 ));
-                server_assets.push(app_build_manifest_output);
+                server_assets.insert(app_build_manifest_output);
 
                 // polyfill-nomodule.js is a pre-compiled asset distributed as part of next,
                 // load it as a RawModule.
@@ -1014,7 +1015,29 @@ impl AppEndpoint {
                     .context("failed to resolve client-relative path to polyfill")?
                     .into();
                 let polyfill_client_paths = vec![polyfill_client_path];
-                client_assets.push(Vc::upcast(polyfill_output_asset));
+                client_assets.insert(Vc::upcast(polyfill_output_asset));
+
+                if *this
+                    .app_project
+                    .project()
+                    .should_create_webpack_stats()
+                    .await?
+                {
+                    let webpack_stats =
+                        generate_webpack_stats(app_entry.original_name.clone(), &client_assets)
+                            .await?;
+                    let stats_output: Vc<Box<dyn OutputAsset>> =
+                        Vc::upcast(VirtualOutputAsset::new(
+                            node_root.join(
+                                format!("server/app{manifest_path_prefix}/webpack-stats.json",)
+                                    .into(),
+                            ),
+                            AssetContent::file(
+                                File::from(serde_json::to_string_pretty(&webpack_stats)?).into(),
+                            ),
+                        ));
+                    server_assets.insert(Vc::upcast(stats_output));
+                }
 
                 let build_manifest = BuildManifest {
                     root_main_files: client_shared_chunks_paths,
@@ -1029,7 +1052,7 @@ impl AppEndpoint {
                         File::from(serde_json::to_string_pretty(&build_manifest)?).into(),
                     ),
                 ));
-                server_assets.push(build_manifest_output);
+                server_assets.insert(build_manifest_output);
 
                 let entry_manifest = ClientReferenceManifest::build_output(
                     node_root,
@@ -1042,7 +1065,7 @@ impl AppEndpoint {
                     this.app_project.project().next_config(),
                     runtime,
                 );
-                server_assets.push(entry_manifest);
+                server_assets.insert(entry_manifest);
 
                 if runtime == NextRuntime::Edge {
                     middleware_assets.push(entry_manifest);
@@ -1095,7 +1118,7 @@ impl AppEndpoint {
             )))
         }
 
-        let client_assets = OutputAssets::new(client_assets);
+        let client_assets = OutputAssets::new(client_assets.iter().cloned().collect::<Vec<_>>());
 
         let next_font_manifest_output = create_font_manifest(
             this.app_project.project().client_root(),
@@ -1108,7 +1131,7 @@ impl AppEndpoint {
             true,
         )
         .await?;
-        server_assets.push(next_font_manifest_output);
+        server_assets.insert(next_font_manifest_output);
 
         let endpoint_output = match runtime {
             NextRuntime::Edge => {
@@ -1139,7 +1162,7 @@ impl AppEndpoint {
                         Vc::upcast(chunking_context),
                     )
                     .await?;
-                    server_assets.push(manifest);
+                    server_assets.insert(manifest);
                     evaluatable_assets.push(loader);
                 }
 
@@ -1230,12 +1253,12 @@ impl AppEndpoint {
                         .cell(),
                     ),
                 ));
-                server_assets.push(middleware_manifest_v2);
+                server_assets.insert(middleware_manifest_v2);
 
                 // create app paths manifest
                 let app_paths_manifest_output =
                     create_app_paths_manifest(node_root, &app_entry.original_name, entry_file)?;
-                server_assets.push(app_paths_manifest_output);
+                server_assets.insert(app_paths_manifest_output);
 
                 // create react-loadable-manifest for next/dynamic
                 let mut dynamic_import_modules = collect_next_dynamic_imports(
@@ -1267,7 +1290,7 @@ impl AppEndpoint {
 
                 AppEndpointOutput::Edge {
                     files,
-                    server_assets: Vc::cell(server_assets),
+                    server_assets: Vc::cell(server_assets.iter().cloned().collect::<Vec<_>>()),
                     client_assets,
                 }
             }
@@ -1292,7 +1315,7 @@ impl AppEndpoint {
                         Vc::upcast(chunking_context),
                     )
                     .await?;
-                    server_assets.push(manifest);
+                    server_assets.insert(manifest);
                     evaluatable_assets.push(loader);
                 }
 
@@ -1377,7 +1400,7 @@ impl AppEndpoint {
                 }
                 .instrument(tracing::trace_span!("server node entrypoint"))
                 .await?);
-                server_assets.push(rsc_chunk);
+                server_assets.insert(rsc_chunk);
 
                 let app_paths_manifest_output = create_app_paths_manifest(
                     node_root,
@@ -1388,7 +1411,7 @@ impl AppEndpoint {
                         .context("RSC chunk path should be within app paths manifest directory")?
                         .into(),
                 )?;
-                server_assets.push(app_paths_manifest_output);
+                server_assets.insert(app_paths_manifest_output);
 
                 // create react-loadable-manifest for next/dynamic
                 let availability_info = Value::new(AvailabilityInfo::Root);
@@ -1422,7 +1445,7 @@ impl AppEndpoint {
 
                 AppEndpointOutput::NodeJs {
                     rsc_chunk,
-                    server_assets: Vc::cell(server_assets),
+                    server_assets: Vc::cell(server_assets.iter().cloned().collect::<Vec<_>>()),
                     client_assets,
                 }
             }

crates/next-api/src/lib.rs

@@ -18,6 +18,7 @@ pub mod project;
 pub mod route;
 mod server_actions;
 mod versioned_content_map;
+mod webpack_stats;
 
 // Declare build-time information variables generated in build.rs
 shadow_rs::shadow!(build);

crates/next-api/src/pages.rs

@@ -71,6 +71,7 @@ use crate::{
     },
     project::Project,
     route::{Endpoint, Route, Routes, WrittenEndpoint},
+    webpack_stats::generate_webpack_stats,
 };
 
 #[turbo_tasks::value]
@@ -1050,23 +1051,43 @@ impl PageEndpoint {
         };
 
         let pathname = this.pathname.await?;
-        let original_name = this.original_name.await?;
+        let original_name = &*this.original_name.await?;
 
         let client_assets = OutputAssets::new(client_assets);
 
+        let manifest_path_prefix = get_asset_prefix_from_pathname(&pathname);
+        let node_root = this.pages_project.project().node_root();
         let next_font_manifest_output = create_font_manifest(
             this.pages_project.project().client_root(),
-            this.pages_project.project().node_root(),
+            node_root,
             this.pages_project.pages_dir(),
-            &original_name,
-            &get_asset_prefix_from_pathname(&pathname),
+            original_name,
+            &manifest_path_prefix,
             &pathname,
             client_assets,
             false,
         )
         .await?;
         server_assets.push(next_font_manifest_output);
 
+        if *this
+            .pages_project
+            .project()
+            .should_create_webpack_stats()
+            .await?
+        {
+            let webpack_stats =
+                generate_webpack_stats(original_name.to_owned(), &client_assets.await?).await?;
+            let stats_output: Vc<Box<dyn OutputAsset>> = Vc::upcast(VirtualOutputAsset::new(
+                node_root
+                    .join(format!("server/pages{manifest_path_prefix}/webpack-stats.json",).into()),
+                AssetContent::file(
+                    File::from(serde_json::to_string_pretty(&webpack_stats)?).into(),
+                ),
+            ));
+            server_assets.push(Vc::upcast(stats_output));
+        }
+
         let page_output = match *ssr_chunk.await? {
             SsrChunk::NodeJs {
                 entry,

crates/next-api/src/project.rs

@@ -618,6 +618,13 @@ impl Project {
         self.js_config
     }
 
+    #[turbo_tasks::function]
+    pub(super) async fn should_create_webpack_stats(&self) -> Result<Vc<bool>> {
+        Ok(Vc::cell(
+            self.env.read("TURBOPACK_STATS".into()).await?.is_some(),
+        ))
+    }
+
     #[turbo_tasks::function]
     pub(super) async fn execution_context(self: Vc<Self>) -> Result<Vc<ExecutionContext>> {
         let node_root = self.node_root();