refactor(core): definition of "ExtensionFileSource" (#17823)

This commit changes definition of "ExtensionFileSource", by changing
"code" field to being "ExtensionFileSourceCode" enum. Currently the enum
has only a single variant "IncludedInBinary". It is done in preparation
to allow embedders to decide if they want to include the source code in the
binary when snapshotting (in most cases they shouldn't do that).

In the follow up commit we'll add more variants to
"ExtensionFileSourceCode".

"include_js_files_dir!" macro was removed in favor "include_js_files!"
macro which can now accept "dir" option.

12 files changed, 103 insertions, 56 deletions

diff --git a/bench_util/benches/utf8.rs b/bench_util/benches/utf8.rs
index 7a9066d1e..a8d6e20eb 100644
--- a/bench_util/benches/utf8.rs
+++ b/bench_util/benches/utf8.rs
@@ -7,12 +7,14 @@ use deno_bench_util::bencher::Bencher;
 use deno_bench_util::BenchOptions;
 use deno_core::Extension;
 use deno_core::ExtensionFileSource;
+use deno_core::ExtensionFileSourceCode;
 
 fn setup() -> Vec<Extension> {
   vec![Extension::builder("bench_setup")
     .js(vec![ExtensionFileSource {
       specifier: "setup.js".to_string(),
-      code: r#"
+      code: ExtensionFileSourceCode::IncludedInBinary(
+        r#"
       const hello = "hello world\n";
       const hello1k = hello.repeat(1e3);
       const hello1m = hello.repeat(1e6);
@@ -20,6 +22,7 @@ fn setup() -> Vec<Extension> {
       const hello1kEncoded = Deno.core.encode(hello1k);
       const hello1mEncoded = Deno.core.encode(hello1m);
       "#,
+      ),
     }])
     .build()]
 }
diff --git a/cli/build.rs b/cli/build.rs
index 84d8fd549..807009a5d 100644
--- a/cli/build.rs
+++ b/cli/build.rs
@@ -4,10 +4,11 @@ use std::env;
 use std::path::Path;
 use std::path::PathBuf;
 
-use deno_core::include_js_files_dir;
+use deno_core::include_js_files;
 use deno_core::snapshot_util::*;
 use deno_core::Extension;
 use deno_core::ExtensionFileSource;
+use deno_core::ExtensionFileSourceCode;
 use deno_runtime::deno_cache::SqliteBackedCache;
 use deno_runtime::permissions::PermissionsContainer;
 use deno_runtime::*;
@@ -17,7 +18,6 @@ mod ts {
   use crate::deno_webgpu_get_declaration;
   use deno_core::error::custom_error;
   use deno_core::error::AnyError;
-  use deno_core::include_js_files_dir;
   use deno_core::op;
   use deno_core::OpState;
   use deno_runtime::deno_node::SUPPORTED_BUILTIN_NODE_MODULES;
@@ -260,7 +260,7 @@ mod ts {
         op_load::decl(),
         op_script_version::decl(),
       ])
-      .js(include_js_files_dir! {
+      .js(include_js_files! {
         dir "tsc",
         "00_typescript.js",
         "99_main_compiler.js",
@@ -354,13 +354,15 @@ fn create_cli_snapshot(snapshot_path: PathBuf) {
     deno_flash::init::<PermissionsContainer>(false), // No --unstable
   ];
 
-  let mut esm_files = include_js_files_dir!(
+  let mut esm_files = include_js_files!(
     dir "js",
     "40_testing.js",
   );
   esm_files.push(ExtensionFileSource {
     specifier: "runtime/js/99_main.js".to_string(),
-    code: deno_runtime::js::SOURCE_CODE_FOR_99_MAIN_JS,
+    code: ExtensionFileSourceCode::IncludedInBinary(
+      deno_runtime::js::SOURCE_CODE_FOR_99_MAIN_JS,
+    ),
   });
   let extensions_with_js = vec![Extension::builder("cli")
     // FIXME(bartlomieju): information about which extensions were
diff --git a/core/extensions.rs b/core/extensions.rs
index e497b8003..ab686d868 100644
--- a/core/extensions.rs
+++ b/core/extensions.rs
@@ -7,9 +7,20 @@ use std::task::Context;
 use v8::fast_api::FastFunction;
 
 #[derive(Clone, Debug)]
+pub enum ExtensionFileSourceCode {
+  /// Source code is included in the binary produced. Either by being defined
+  /// inline, or included using `include_str!()`. If you are snapshotting, this
+  /// will result in two copies of the source code being included - one in the
+  /// snapshot, the other the static string in the `Extension`.
+  IncludedInBinary(&'static str),
+  // TODO(bartlomieju): add more variants that allow to read file from the disk,
+  // and not include it in the binary.
+}
+
+#[derive(Clone, Debug)]
 pub struct ExtensionFileSource {
   pub specifier: String,
-  pub code: &'static str,
+  pub code: ExtensionFileSourceCode,
 }
 pub type OpFnRef = v8::FunctionCallback;
 pub type OpMiddlewareFn = dyn Fn(OpDecl) -> OpDecl;
@@ -180,6 +191,9 @@ impl ExtensionBuilder {
 
   pub fn js(&mut self, js_files: Vec<ExtensionFileSource>) -> &mut Self {
     let js_files =
+      // TODO(bartlomieju): if we're automatically remapping here, then we should
+      // use a different result struct that `ExtensionFileSource` as it's confusing
+      // when (and why) the remapping happens.
       js_files.into_iter().map(|file_source| ExtensionFileSource {
         specifier: format!("internal:{}/{}", self.name, file_source.specifier),
         code: file_source.code,
@@ -189,16 +203,15 @@ impl ExtensionBuilder {
   }
 
   pub fn esm(&mut self, esm_files: Vec<ExtensionFileSource>) -> &mut Self {
-    let esm_files =
-      esm_files
-        .into_iter()
-        .map(|file_source| ExtensionFileSource {
-          specifier: format!(
-            "internal:{}/{}",
-            self.name, file_source.specifier
-          ),
-          code: file_source.code,
-        });
+    let esm_files = esm_files
+      .into_iter()
+      // TODO(bartlomieju): if we're automatically remapping here, then we should
+      // use a different result struct that `ExtensionFileSource` as it's confusing
+      // when (and why) the remapping happens.
+      .map(|file_source| ExtensionFileSource {
+        specifier: format!("internal:{}/{}", self.name, file_source.specifier),
+        code: file_source.code,
+      });
     self.esm.extend(esm_files);
     self
   }
@@ -259,48 +272,53 @@ impl ExtensionBuilder {
 }
 
 /// Helps embed JS files in an extension. Returns a vector of
-/// `ExtensionFileSource`, that represent the filename and source code.
+/// `ExtensionFileSource`, that represent the filename and source code. All
+/// specified files are rewritten into "internal:<extension_name>/<file_name>".
 ///
-/// Example:
+/// An optional "dir" option can be specified to prefix all files with a
+/// directory name.
+///
+/// Example (for "my_extension"):
 /// ```ignore
 /// include_js_files!(
 ///   "01_hello.js",
 ///   "02_goodbye.js",
 /// )
-/// ```
-#[macro_export]
-macro_rules! include_js_files {
-  ($($file:literal,)+) => {
-    vec![
-      $($crate::ExtensionFileSource {
-        specifier: $file.to_string(),
-        code: include_str!($file),
-      },)+
-    ]
-  };
-}
-
-/// Helps embed JS files in an extension. Returns a vector of
-/// `ExtensionFileSource`, that represent the filename and source code.
-/// Additional "dir" option is required, that specifies which directory in the
-/// crate root contains the listed files. "dir" option will be prepended to
-/// each file name.
+/// // Produces following specifiers:
+/// - "internal:my_extension/01_hello.js"
+/// - "internal:my_extension/02_goodbye.js"
 ///
-/// Example:
+/// /// Example with "dir" option (for "my_extension"):
 /// ```ignore
-/// include_js_files_dir!(
-///   dir "example",
+/// include_js_files!(
+///   dir "js",
 ///   "01_hello.js",
 ///   "02_goodbye.js",
 /// )
+/// // Produces following specifiers:
+/// - "internal:my_extension/js/01_hello.js"
+/// - "internal:my_extension/js/02_goodbye.js"
 /// ```
 #[macro_export]
-macro_rules! include_js_files_dir {
+macro_rules! include_js_files {
   (dir $dir:literal, $($file:literal,)+) => {
     vec![
       $($crate::ExtensionFileSource {
         specifier: concat!($dir, "/", $file).to_string(),
-        code: include_str!(concat!($dir, "/", $file)),
+        code: $crate::ExtensionFileSourceCode::IncludedInBinary(
+          include_str!(concat!($dir, "/", $file)
+        )),
+      },)+
+    ]
+  };
+
+  ($($file:literal,)+) => {
+    vec![
+      $($crate::ExtensionFileSource {
+        specifier: $file.to_string(),
+        code: $crate::ExtensionFileSourceCode::IncludedInBinary(
+          include_str!($file)
+        ),
       },)+
     ]
   };
diff --git a/core/lib.rs b/core/lib.rs
index a77701339..51a03493d 100644
--- a/core/lib.rs
+++ b/core/lib.rs
@@ -55,6 +55,7 @@ pub use crate::async_cell::RcRef;
 pub use crate::extensions::Extension;
 pub use crate::extensions::ExtensionBuilder;
 pub use crate::extensions::ExtensionFileSource;
+pub use crate::extensions::ExtensionFileSourceCode;
 pub use crate::extensions::OpDecl;
 pub use crate::extensions::OpMiddlewareFn;
 pub use crate::flags::v8_set_flags;
diff --git a/core/modules.rs b/core/modules.rs
index 1f427508a..43dabf411 100644
--- a/core/modules.rs
+++ b/core/modules.rs
@@ -3,6 +3,7 @@
 use crate::bindings;
 use crate::error::generic_error;
 use crate::extensions::ExtensionFileSource;
+use crate::extensions::ExtensionFileSourceCode;
 use crate::module_specifier::ModuleSpecifier;
 use crate::resolve_import;
 use crate::resolve_url;
@@ -402,7 +403,11 @@ impl ModuleLoader for InternalModuleLoader {
       let result = if let Some(load_callback) = &self.maybe_load_callback {
         load_callback(file_source)
       } else {
-        Ok(file_source.code.to_string())
+        match file_source.code {
+          ExtensionFileSourceCode::IncludedInBinary(code) => {
+            Ok(code.to_string())
+          }
+        }
       };
 
       return async move {
diff --git a/core/runtime.rs b/core/runtime.rs
index 25a09e85e..ae2b0489b 100644
--- a/core/runtime.rs
+++ b/core/runtime.rs
@@ -21,6 +21,7 @@ use crate::source_map::SourceMapCache;
 use crate::source_map::SourceMapGetter;
 use crate::Extension;
 use crate::ExtensionFileSource;
+use crate::ExtensionFileSourceCode;
 use crate::NoopModuleLoader;
 use crate::OpMiddlewareFn;
 use crate::OpResult;
@@ -868,11 +869,14 @@ impl JsRuntime {
       {
         let js_files = ext.get_js_sources();
         for file_source in js_files {
+          let ExtensionFileSourceCode::IncludedInBinary(code) =
+            file_source.code;
+
           // TODO(@AaronO): use JsRuntime::execute_static() here to move src off heap
           realm.execute_script(
             self.v8_isolate(),
             &file_source.specifier,
-            file_source.code,
+            code,
           )?;
         }
       }
diff --git a/ext/node/lib.rs b/ext/node/lib.rs
index 43861ef46..697100c8f 100644
--- a/ext/node/lib.rs
+++ b/ext/node/lib.rs
@@ -2,7 +2,6 @@
 
 use deno_core::error::AnyError;
 use deno_core::include_js_files;
-use deno_core::include_js_files_dir;
 use deno_core::located_script_name;
 use deno_core::op;
 use deno_core::Extension;
@@ -96,7 +95,7 @@ fn op_node_build_os() -> String {
 }
 
 pub fn init_polyfill() -> Extension {
-  let esm_files = include_js_files_dir!(
+  let esm_files = include_js_files!(
     dir "polyfills",
     "_core.ts",
     "_crypto/crypto_browserify/asn1.js/base/buffer.js",
diff --git a/ext/url/benches/url_ops.rs b/ext/url/benches/url_ops.rs
index 001b5de92..828b02297 100644
--- a/ext/url/benches/url_ops.rs
+++ b/ext/url/benches/url_ops.rs
@@ -7,6 +7,7 @@ use deno_bench_util::bencher::Bencher;
 
 use deno_core::Extension;
 use deno_core::ExtensionFileSource;
+use deno_core::ExtensionFileSourceCode;
 
 fn setup() -> Vec<Extension> {
   vec![
@@ -15,9 +16,11 @@ fn setup() -> Vec<Extension> {
     Extension::builder("bench_setup")
       .esm(vec![ExtensionFileSource {
         specifier: "internal:setup".to_string(),
-        code: r#"import { URL } from "internal:deno_url/00_url.js";
+        code: ExtensionFileSourceCode::IncludedInBinary(
+          r#"import { URL } from "internal:deno_url/00_url.js";
         globalThis.URL = URL;
         "#,
+        ),
       }])
       .build(),
   ]
diff --git a/ext/web/benches/encoding.rs b/ext/web/benches/encoding.rs
index bb297a1bf..bfae07937 100644
--- a/ext/web/benches/encoding.rs
+++ b/ext/web/benches/encoding.rs
@@ -6,6 +6,7 @@ use deno_bench_util::bencher::benchmark_group;
 use deno_bench_util::bencher::Bencher;
 use deno_core::Extension;
 use deno_core::ExtensionFileSource;
+use deno_core::ExtensionFileSourceCode;
 use deno_web::BlobStore;
 
 struct Permissions;
@@ -32,11 +33,13 @@ fn setup() -> Vec<Extension> {
     Extension::builder("bench_setup")
       .esm(vec![ExtensionFileSource {
         specifier: "internal:setup".to_string(),
-        code: r#"
+        code: ExtensionFileSourceCode::IncludedInBinary(
+          r#"
         import { TextDecoder } from "internal:deno_web/08_text_encoding.js";
         globalThis.TextDecoder = TextDecoder;
         globalThis.hello12k = Deno.core.encode("hello world\n".repeat(1e3));
         "#,
+        ),
       }])
       .state(|state| {
         state.put(Permissions {});
diff --git a/ext/web/benches/timers_ops.rs b/ext/web/benches/timers_ops.rs
index f01b4c532..657082df4 100644
--- a/ext/web/benches/timers_ops.rs
+++ b/ext/web/benches/timers_ops.rs
@@ -6,6 +6,7 @@ use deno_bench_util::bencher::benchmark_group;
 use deno_bench_util::bencher::Bencher;
 use deno_core::Extension;
 use deno_core::ExtensionFileSource;
+use deno_core::ExtensionFileSourceCode;
 use deno_web::BlobStore;
 
 struct Permissions;
@@ -32,11 +33,11 @@ fn setup() -> Vec<Extension> {
     .esm(vec![
       ExtensionFileSource {
         specifier: "internal:setup".to_string(), 
-        code: r#"
+        code: ExtensionFileSourceCode::IncludedInBinary(r#"
       import { setTimeout, handleTimerMacrotask } from "internal:deno_web/02_timers.js";
       globalThis.setTimeout = setTimeout;
       Deno.core.setMacrotaskCallback(handleTimerMacrotask);
-      "#
+      "#)
       },
     ])
     .state(|state| {
diff --git a/ext/webidl/benches/dict.rs b/ext/webidl/benches/dict.rs
index 00bef4935..df0844fce 100644
--- a/ext/webidl/benches/dict.rs
+++ b/ext/webidl/benches/dict.rs
@@ -7,6 +7,7 @@ use deno_bench_util::bencher::Bencher;
 
 use deno_core::Extension;
 use deno_core::ExtensionFileSource;
+use deno_core::ExtensionFileSourceCode;
 
 fn setup() -> Vec<Extension> {
   vec![
@@ -14,7 +15,9 @@ fn setup() -> Vec<Extension> {
     Extension::builder("deno_webidl_bench")
       .esm(vec![ExtensionFileSource {
         specifier: "internal:setup".to_string(),
-        code: include_str!("dict.js"),
+        code: ExtensionFileSourceCode::IncludedInBinary(include_str!(
+          "dict.js"
+        )),
       }])
       .build(),
   ]
diff --git a/runtime/build.rs b/runtime/build.rs
index bbdec8f57..a813d63cb 100644
--- a/runtime/build.rs
+++ b/runtime/build.rs
@@ -1,6 +1,6 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
-use deno_core::include_js_files_dir;
+use deno_core::include_js_files;
 use std::env;
 use std::path::PathBuf;
 
@@ -18,6 +18,7 @@ mod not_docs {
   use deno_ast::SourceTextInfo;
   use deno_core::error::AnyError;
   use deno_core::ExtensionFileSource;
+  use deno_core::ExtensionFileSourceCode;
 
   fn transpile_ts_for_snapshotting(
     file_source: &ExtensionFileSource,
@@ -34,13 +35,15 @@ mod not_docs {
       ),
     };
 
+    let ExtensionFileSourceCode::IncludedInBinary(code) = file_source.code;
+
     if !should_transpile {
-      return Ok(file_source.code.to_string());
+      return Ok(code.to_string());
     }
 
     let parsed = deno_ast::parse_module(ParseParams {
       specifier: file_source.specifier.to_string(),
-      text_info: SourceTextInfo::from_string(file_source.code.to_string()),
+      text_info: SourceTextInfo::from_string(code.to_string()),
       media_type,
       capture_tokens: false,
       scope_analysis: false,
@@ -188,7 +191,7 @@ mod not_docs {
         "deno_http",
         "deno_flash",
       ])
-      .esm(include_js_files_dir!(
+      .esm(include_js_files!(
         dir "js",
         "01_build.js",
         "01_errors.js",
@@ -283,7 +286,9 @@ mod not_docs {
           .dependencies(vec!["runtime"])
           .esm(vec![ExtensionFileSource {
             specifier: "js/99_main.js".to_string(),
-            code: include_str!("js/99_main.js"),
+            code: ExtensionFileSourceCode::IncludedInBinary(include_str!(
+              "js/99_main.js"
+            )),
           }])
           .build(),
       );