chore: write resources to disk to optimize loading time (#45)

* chore: write resources to disk to optimize loading time

* chore: update readme and test

README.md

@@ -15,10 +15,11 @@ Started in 68ms and updated in 1ms for a demo react project as below.
 ![img](./assets/performance.png)
 
 **Features**:
-* 🔥 **Super Fast**: Start a react / vue(incoming) project in milliseconds.
-* ⚡ **"1ms" HMR**: Finish a HMR within 10ms for the most situations.
-* 🧰 **Fully Pluggable**: Support both rust plugins and js plugins.
-* ⚙️ **Native Web Assets Compiling Supported**: Support support compiling JS/TS/JSX/TSX, css, html natively.
+
+- 🔥 **Super Fast**: Start a react / vue(incoming) project in milliseconds.
+- ⚡ **"1ms" HMR**: Finish a HMR within 10ms for the most situations.
+- 🧰 **Fully Pluggable**: Support both rust plugins and js plugins.
+- ⚙️ **Native Web Assets Compiling Supported**: Support support compiling JS/TS/JSX/TSX, css, html natively.
 
 <br/>
 
@@ -32,7 +33,9 @@ Started in 68ms and updated in 1ms for a demo react project as below.
 <br/>
 
 ## Getting Started
+
 Install Farm Cli:
+
 ```sh
 npm install -g @farmfe/cli
 ```
@@ -42,3 +45,113 @@ We provided a experience react project for now. Using `farm create` to create a
 ```sh
 farm create && cd farm-react && npm i && npm start
 ```
+
+## Configuring
+
+> Official docs site is on the way...
+
+Farm load configuration file from `farm.config.ts`. The available config as below:
+
+```ts
+export interface UserConfig {
+  /** current root of this project, default to current working directory */
+  root?: string;
+  /** js plugin(which is a javascript object) and rust plugin(which is string refer to a .farm file or a package) */
+  plugins?: (RustPlugin | JsPlugin)[];
+  /** config related to compilation */
+  compilation?: {
+    input?: Record<string, string>;
+    output?: {
+      filename?: string;
+      path?: string;
+      publicPath?: string;
+    };
+    resolve?: {
+      extensions?: string[];
+      alias?: Record<string, string>;
+      mainFields?: string[];
+      conditions?: string[];
+      symlinks: boolean;
+    };
+    external?: string[];
+    mode?: 'development' | 'production';
+    root?: string;
+    runtime?: {
+      path: string;
+      plugins?: string[];
+      swcHelpersPath?: string;
+    };
+    script?: {
+      // specify target es version
+      target?:
+        | 'es3'
+        | 'es5'
+        | 'es2015'
+        | 'es2016'
+        | 'es2017'
+        | 'es2018'
+        | 'es2019'
+        | 'es2020'
+        | 'es2021'
+        | 'es2022';
+      // config swc parser
+      parser?: {
+        esConfig?: {
+          jsx?: boolean;
+          fnBind: boolean;
+          // Enable decorators.
+          decorators: boolean;
+
+          // babel: `decorators.decoratorsBeforeExport`
+          //
+          // Effective only if `decorator` is true.
+          decoratorsBeforeExport: boolean;
+          exportDefaultFrom: boolean;
+          // Stage 3.
+          importAssertions: boolean;
+          privateInObject: boolean;
+          allowSuperOutsideMethod: boolean;
+          allowReturnOutsideFunction: boolean;
+        };
+        tsConfig?: {
+          tsx: boolean;
+          decorators: boolean;
+          /// `.d.ts`
+          dts: boolean;
+          noEarlyErrors: boolean;
+        };
+      };
+    };
+    partialBundling?: {
+      moduleBuckets?: {
+        name: string;
+        test: string[];
+      }[];
+    };
+    lazyCompilation?: boolean;
+  };
+  /** config related to dev server */
+  server?: UserServerConfig;
+}
+
+export type RustPlugin =
+  | string
+  | [
+      string,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      Record<string, any>
+    ];
+
+export interface JsPlugin {
+  resolve: JsPluginHook<
+    {
+      importers: string[];
+      specifiers: string[];
+    },
+    PluginResolveHookParam,
+    PluginResolveHookResult
+  >;
+
+  // load: JsPluginHook<{ filters: { ids: string[] }}>;
+}
+```

crates/compiler/src/generate/finalize_resources.rs

@@ -1,17 +1,76 @@
-use std::sync::Arc;
+use std::{
+  fs::{create_dir_all, read_dir, remove_file, File},
+  io::Write,
+  path::{Path, PathBuf},
+  sync::Arc,
+};
 
-use farmfe_core::context::CompilationContext;
+use farmfe_core::{context::CompilationContext, relative_path::RelativePath};
 use farmfe_toolkit::tracing;
 
 #[tracing::instrument(skip_all)]
 pub fn finalize_resources(context: &Arc<CompilationContext>) -> farmfe_core::error::Result<()> {
   tracing::trace!("Staring finalize_resources...");
-  let mut resources_map = context.resources_map.lock();
 
-  context
-    .plugin_driver
-    .finalize_resources(&mut *resources_map, context)?;
+  {
+    let mut resources_map = context.resources_map.lock();
+
+    context
+      .plugin_driver
+      .finalize_resources(&mut *resources_map, context)?;
+  }
+
+  write_resources(context);
 
   tracing::trace!("Finished finalize_resources.");
   Ok(())
 }
+
+pub fn write_resources(context: &Arc<CompilationContext>) {
+  let resources = context.resources_map.lock();
+  let output_dir = if Path::new(&context.config.output.path).is_absolute() {
+    PathBuf::from(&context.config.output.path)
+  } else {
+    RelativePath::new(&context.config.output.path).to_logical_path(&context.config.root)
+  };
+
+  if !output_dir.exists() {
+    create_dir_all(output_dir.clone()).unwrap();
+  }
+
+  // Remove useless resources
+  let existing_resources = read_dir(output_dir.clone())
+    .unwrap()
+    .map(|entry| {
+      let entry = entry.unwrap();
+      let path = entry.path();
+      let file_name = path.file_name().unwrap().to_str().unwrap().to_string();
+
+      file_name
+    })
+    .collect::<Vec<String>>();
+
+  for pre_resource in &existing_resources {
+    let file_path = RelativePath::new(pre_resource).to_logical_path(&output_dir);
+    // always remove html file
+    if pre_resource.ends_with(".html") {
+      remove_file(file_path).unwrap();
+      continue;
+    }
+
+    if !resources.contains_key(pre_resource) && file_path.exists() {
+      remove_file(file_path).unwrap();
+    }
+  }
+
+  // add new resources
+  for resource in resources.values() {
+    let file_path = RelativePath::new(&resource.name).to_logical_path(&output_dir);
+    // only write expose non-emitted resource
+    if !resource.emitted && !file_path.exists() {
+      let mut file = File::create(file_path).unwrap();
+
+      file.write_all(&resource.bytes).unwrap();
+    }
+  }
+}

crates/node/src/lib.rs

@@ -1,6 +1,11 @@
 #![deny(clippy::all)]
 
-use std::{collections::HashMap, sync::Arc};
+use std::{
+  collections::HashMap,
+  fs::remove_dir_all,
+  path::{Path, PathBuf},
+  sync::Arc,
+};
 
 use farmfe_compiler::{update::UpdateType, Compiler};
 
@@ -9,6 +14,7 @@ pub mod plugin_adapters;
 use farmfe_core::{
   config::{Config, Mode},
   module::ModuleId,
+  relative_path::RelativePath,
 };
 use farmfe_toolkit::tracing_subscriber::{self, fmt, prelude::*, EnvFilter};
 use napi::{bindgen_prelude::FromNapiValue, Env, JsObject, NapiRaw, Status};
@@ -108,19 +114,34 @@ impl JsCompiler {
   /// TODO: usage example
   #[napi]
   pub async fn compile(&self) -> napi::Result<()> {
+    let context = self.compiler.context();
+    let output_dir = if Path::new(&context.config.output.path).is_absolute() {
+      PathBuf::from(&context.config.output.path)
+    } else {
+      RelativePath::new(&context.config.output.path).to_logical_path(&context.config.root)
+    };
+
+    if output_dir.exists() {
+      remove_dir_all(&output_dir).map_err(|e| {
+        napi::Error::new(
+          Status::GenericFailure,
+          format!("remove output dir error: {}", e),
+        )
+      })?;
+    }
+
     self
       .compiler
       .compile()
-      .map_err(|e| napi::Error::new(Status::GenericFailure, format!("{}", e)))
+      .map_err(|e| napi::Error::new(Status::GenericFailure, format!("{}", e)))?;
+
+    Ok(())
   }
 
   /// sync compile
   #[napi]
   pub fn compile_sync(&self) -> napi::Result<()> {
-    self
-      .compiler
-      .compile()
-      .map_err(|e| napi::Error::new(Status::GenericFailure, format!("{}", e)))
+    unimplemented!("sync compile is not supported yet")
   }
 
   /// async update, return promise
@@ -179,23 +200,6 @@ impl JsCompiler {
     unimplemented!("sync update");
   }
 
-  #[napi]
-  pub fn resources(&self) -> HashMap<String, Vec<u8>> {
-    let context = self.compiler.context();
-    let resources = context.resources_map.lock();
-
-    let mut result = HashMap::new();
-
-    for resource in resources.values() {
-      // only write expose non-emitted resource
-      if !resource.emitted {
-        result.insert(resource.name.clone(), resource.bytes.clone());
-      }
-    }
-
-    result
-  }
-
   #[napi]
   pub fn has_module(&self, resolved_path: String) -> bool {
     let context = self.compiler.context();

crates/plugin_partial_bundling/src/lib.rs

@@ -152,18 +152,6 @@ impl Plugin for FarmPluginPartialBundling {
           .as_bytes(),
           8,
         );
-        // let id = format!(
-        //   "{}-{}-{}-{}",
-        //   module_bucket.id.to_string(),
-        //   module_type.to_string(),
-        //   module_ids
-        //     .iter()
-        //     .map(|m| m.to_string())
-        //     .collect::<Vec<_>>()
-        //     .join("_"),
-        //   immutable
-        // )
-        // .replace("/", "+");
         let mut resource_pot = ResourcePot::new(ResourcePotId::new(id), module_type.into());
 
         resource_pot.immutable = immutable;

packages/core/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @farmfe/core
 
+## 0.3.2
+
+### Patch Changes
+
+- write resources to disk to optimize resources loading time
+- Updated dependencies
+  - @farmfe/runtime-plugin-hmr@3.0.2
+  - @farmfe/runtime@0.3.2
+
 ## 0.3.1
 
 ### Patch Changes

packages/core/binding/binding.d.ts

@@ -41,6 +41,5 @@ export class Compiler {
   update(paths: Array<string>): Promise<JsUpdateResult>;
   /** sync update */
   updateSync(paths: Array<string>): JsUpdateResult;
-  resources(): Record<string, Array<number>>;
   hasModule(resolvedPath: string): boolean;
 }

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@farmfe/core",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",
@@ -61,8 +61,8 @@
     "type-check": "tsc -p tsconfig.build.json --noEmit"
   },
   "dependencies": {
-    "@farmfe/runtime": "workspace:^0.3.1",
-    "@farmfe/runtime-plugin-hmr": "workspace:^3.0.1",
+    "@farmfe/runtime": "workspace:^0.3.2",
+    "@farmfe/runtime-plugin-hmr": "workspace:^3.0.2",
     "@swc/helpers": "^0.4.9",
     "boxen": "^7.0.1",
     "chalk": "^5.2.0",