fix(cli): add support for non-UTF8 source files (#6789)

Fixes: #5542

1 files changed, 273 insertions, 73 deletions

diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs
index 7d72d5cd4..63743f5ab 100644
--- a/cli/file_fetcher.rs
+++ b/cli/file_fetcher.rs
@@ -7,10 +7,12 @@ use crate::http_util::FetchOnceResult;
 use crate::msg;
 use crate::op_error::OpError;
 use crate::permissions::Permissions;
+use crate::text_encoding;
 use deno_core::ErrBox;
 use deno_core::ModuleSpecifier;
 use futures::future::FutureExt;
 use log::info;
+use std::borrow::Cow;
 use std::collections::HashMap;
 use std::fs;
 use std::future::Future;
@@ -24,6 +26,47 @@ use std::sync::Arc;
 use std::sync::Mutex;
 use url::Url;
 
+/// Structure representing a text document.
+#[derive(Debug, Clone)]
+pub struct TextDocument {
+  bytes: Vec<u8>,
+  charset: Cow<'static, str>,
+}
+
+impl TextDocument {
+  pub fn new(
+    bytes: Vec<u8>,
+    charset: Option<impl Into<Cow<'static, str>>>,
+  ) -> TextDocument {
+    let charset = charset
+      .map(|cs| cs.into())
+      .unwrap_or_else(|| text_encoding::detect_charset(&bytes).into());
+    TextDocument { bytes, charset }
+  }
+
+  pub fn as_bytes(&self) -> &Vec<u8> {
+    &self.bytes
+  }
+
+  pub fn into_bytes(self) -> Vec<u8> {
+    self.bytes
+  }
+
+  pub fn to_str(&self) -> Result<Cow<str>, std::io::Error> {
+    text_encoding::convert_to_utf8(&self.bytes, &self.charset)
+  }
+
+  pub fn to_string(&self) -> Result<String, std::io::Error> {
+    self.to_str().map(String::from)
+  }
+}
+
+impl From<Vec<u8>> for TextDocument {
+  fn from(bytes: Vec<u8>) -> Self {
+    TextDocument::new(bytes, Option::<&str>::None)
+  }
+}
+
 /// Structure representing local or remote file.
 ///
 /// In case of remote file `url` might be different than originally requested URL, if so
@@ -34,7 +77,7 @@ pub struct SourceFile {
   pub filename: PathBuf,
   pub types_header: Option<String>,
   pub media_type: msg::MediaType,
-  pub source_code: Vec<u8>,
+  pub source_code: TextDocument,
 }
 
 /// Simple struct implementing in-process caching to prevent multiple
@@ -180,8 +223,9 @@ impl SourceFileFetcher {
     match result {
       Ok(mut file) => {
         // TODO: move somewhere?
-        if file.source_code.starts_with(b"#!") {
-          file.source_code = filter_shebang(file.source_code);
+        if file.source_code.bytes.starts_with(b"#!") {
+          file.source_code =
+            filter_shebang(&file.source_code.to_str().unwrap()[..]).into();
         }
 
         // Cache in-process for subsequent access.
@@ -313,12 +357,12 @@ impl SourceFileFetcher {
       Err(e) => return Err(e.into()),
     };
 
-    let media_type = map_content_type(&filepath, None);
+    let (media_type, charset) = map_content_type(&filepath, None);
     Ok(SourceFile {
       url: module_url.clone(),
       filename: filepath,
       media_type,
-      source_code,
+      source_code: TextDocument::new(source_code, charset),
       types_header: None,
     })
   }
@@ -380,7 +424,7 @@ impl SourceFileFetcher {
 
     let cache_filename = self.http_cache.get_cache_filename(module_url);
     let fake_filepath = PathBuf::from(module_url.path());
-    let media_type = map_content_type(
+    let (media_type, charset) = map_content_type(
       &fake_filepath,
       headers.get("content-type").map(|e| e.as_str()),
     );
@@ -389,7 +433,7 @@ impl SourceFileFetcher {
       url: module_url.clone(),
       filename: cache_filename,
       media_type,
-      source_code,
+      source_code: TextDocument::new(source_code, charset),
       types_header,
     }))
   }
@@ -490,7 +534,7 @@ impl SourceFileFetcher {
           let cache_filepath = dir.http_cache.get_cache_filename(&module_url);
           // Used to sniff out content type from file extension - probably to be removed
           let fake_filepath = PathBuf::from(module_url.path());
-          let media_type = map_content_type(
+          let (media_type, charset) = map_content_type(
             &fake_filepath,
             headers.get("content-type").map(String::as_str),
           );
@@ -502,7 +546,7 @@ impl SourceFileFetcher {
             url: module_url.clone(),
             filename: cache_filepath,
             media_type,
-            source_code: source,
+            source_code: TextDocument::new(source, charset),
             types_header,
           };
 
@@ -532,16 +576,19 @@ pub fn map_file_extension(path: &Path) -> msg::MediaType {
   }
 }
 
-// convert a ContentType string into a enumerated MediaType
-fn map_content_type(path: &Path, content_type: Option<&str>) -> msg::MediaType {
+// convert a ContentType string into a enumerated MediaType + optional charset
+fn map_content_type(
+  path: &Path,
+  content_type: Option<&str>,
+) -> (msg::MediaType, Option<String>) {
   match content_type {
     Some(content_type) => {
-      // sometimes there is additional data after the media type in
+      // Sometimes there is additional data after the media type in
       // Content-Type so we have to do a bit of manipulation so we are only
-      // dealing with the actual media type
-      let ct_vector: Vec<&str> = content_type.split(';').collect();
-      let ct: &str = ct_vector.first().unwrap();
-      match ct.to_lowercase().as_ref() {
+      // dealing with the actual media type.
+      let mut ct_iter = content_type.split(';');
+      let ct = ct_iter.next().unwrap();
+      let media_type = match ct.to_lowercase().as_ref() {
         "application/typescript"
         | "text/typescript"
         | "video/vnd.dlna.mpeg-tts"
@@ -565,9 +612,16 @@ fn map_content_type(path: &Path, content_type: Option<&str>) -> msg::MediaType {
           debug!("unknown content type: {}", content_type);
           msg::MediaType::Unknown
         }
-      }
+      };
+
+      let charset = ct_iter
+        .map(str::trim)
+        .find_map(|s| s.strip_prefix("charset="))
+        .map(String::from);
+
+      (media_type, charset)
     }
-    None => map_file_extension(path),
+    None => (map_file_extension(path), None),
   }
 }
 
@@ -586,8 +640,7 @@ fn map_js_like_extension(
   }
 }
 
-fn filter_shebang(bytes: Vec<u8>) -> Vec<u8> {
-  let string = str::from_utf8(&bytes).unwrap();
+fn filter_shebang(string: &str) -> Vec<u8> {
   if let Some(i) = string.find('\n') {
     let (_, rest) = string.split_at(i);
     rest.as_bytes().to_owned()
@@ -767,7 +820,7 @@ mod tests {
     assert!(result.is_ok());
     let r = result.unwrap();
     assert_eq!(
-      r.source_code,
+      r.source_code.bytes,
       &b"export { printHello } from \"./print_hello.ts\";\n"[..]
     );
     assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
@@ -794,7 +847,7 @@ mod tests {
     assert!(result2.is_ok());
     let r2 = result2.unwrap();
     assert_eq!(
-      r2.source_code,
+      r2.source_code.bytes,
       &b"export { printHello } from \"./print_hello.ts\";\n"[..]
     );
     // If get_source_file does not call remote, this should be JavaScript
@@ -823,7 +876,7 @@ mod tests {
     assert!(result3.is_ok());
     let r3 = result3.unwrap();
     assert_eq!(
-      r3.source_code,
+      r3.source_code.bytes,
       &b"export { printHello } from \"./print_hello.ts\";\n"[..]
     );
     // If get_source_file does not call remote, this should be JavaScript
@@ -850,7 +903,7 @@ mod tests {
     assert!(result4.is_ok());
     let r4 = result4.unwrap();
     let expected4 = &b"export { printHello } from \"./print_hello.ts\";\n"[..];
-    assert_eq!(r4.source_code, expected4);
+    assert_eq!(r4.source_code.bytes, expected4);
     // Resolved back to TypeScript
     assert_eq!(&(r4.media_type), &msg::MediaType::TypeScript);
 
@@ -880,7 +933,7 @@ mod tests {
     assert!(result.is_ok());
     let r = result.unwrap();
     let expected = b"export const loaded = true;\n";
-    assert_eq!(r.source_code, expected);
+    assert_eq!(r.source_code.bytes, expected);
     assert_eq!(&(r.media_type), &msg::MediaType::JavaScript);
     let (_, headers) = fetcher.http_cache.get(&module_url).unwrap();
     assert_eq!(headers.get("content-type").unwrap(), "text/javascript");
@@ -906,7 +959,7 @@ mod tests {
     assert!(result2.is_ok());
     let r2 = result2.unwrap();
     let expected2 = b"export const loaded = true;\n";
-    assert_eq!(r2.source_code, expected2);
+    assert_eq!(r2.source_code.bytes, expected2);
     // If get_source_file does not call remote, this should be TypeScript
     // as we modified before! (we do not overwrite .headers.json due to no http
     // fetch)
@@ -932,7 +985,7 @@ mod tests {
     assert!(result3.is_ok());
     let r3 = result3.unwrap();
     let expected3 = b"export const loaded = true;\n";
-    assert_eq!(r3.source_code, expected3);
+    assert_eq!(r3.source_code.bytes, expected3);
     // Now the old .headers.json file should be overwritten back to JavaScript!
     // (due to http fetch)
     assert_eq!(&(r3.media_type), &msg::MediaType::JavaScript);
@@ -1352,7 +1405,7 @@ mod tests {
       .await;
     assert!(result.is_ok());
     let r = result.unwrap();
-    assert_eq!(r.source_code, b"export const loaded = true;\n");
+    assert_eq!(r.source_code.bytes, b"export const loaded = true;\n");
     assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
 
     // Modify .metadata.json, make sure read from local
@@ -1368,7 +1421,7 @@ mod tests {
     let result2 = fetcher.fetch_cached_remote_source(&module_url, 1);
     assert!(result2.is_ok());
     let r2 = result2.unwrap().unwrap();
-    assert_eq!(r2.source_code, b"export const loaded = true;\n");
+    assert_eq!(r2.source_code.bytes, b"export const loaded = true;\n");
     // Not MediaType::TypeScript due to .headers.json modification
     assert_eq!(&(r2.media_type), &msg::MediaType::JavaScript);
 
@@ -1392,7 +1445,7 @@ mod tests {
       .await;
     assert!(result.is_ok());
     let r = result.unwrap();
-    assert_eq!(r.source_code, b"export const loaded = true;\n");
+    assert_eq!(r.source_code.bytes, b"export const loaded = true;\n");
     assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
     let (_, headers) = fetcher.http_cache.get(module_url).unwrap();
     assert_eq!(headers.get("content-type").unwrap(), "text/typescript");
@@ -1417,7 +1470,7 @@ mod tests {
       .await;
     assert!(result.is_ok());
     let r2 = result.unwrap();
-    assert_eq!(r2.source_code, b"export const loaded = true;\n");
+    assert_eq!(r2.source_code.bytes, b"export const loaded = true;\n");
     assert_eq!(&(r2.media_type), &msg::MediaType::JavaScript);
     let (_, headers) = fetcher.http_cache.get(module_url).unwrap();
     assert_eq!(headers.get("content-type").unwrap(), "text/javascript");
@@ -1442,7 +1495,7 @@ mod tests {
       .await;
     assert!(result.is_ok());
     let r3 = result.unwrap();
-    assert_eq!(r3.source_code, b"export const loaded = true;\n");
+    assert_eq!(r3.source_code.bytes, b"export const loaded = true;\n");
     assert_eq!(&(r3.media_type), &msg::MediaType::TypeScript);
     let (_, headers) = fetcher.http_cache.get(module_url).unwrap();
     assert_eq!(headers.get("content-type").unwrap(), "text/typescript");
@@ -1523,6 +1576,63 @@ mod tests {
     }
   }
 
+  async fn test_fetch_source_file_from_disk_nonstandard_encoding(
+    charset: &str,
+    expected_content: String,
+  ) {
+    let (_temp_dir, fetcher) = test_setup();
+
+    let p = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+      .join(format!("tests/encoding/{}.ts", charset));
+    let specifier =
+      ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap()).unwrap();
+    let r = fetcher
+      .fetch_source_file(&specifier, None, Permissions::allow_all())
+      .await;
+    assert!(r.is_ok());
+    let fetched_file = r.unwrap();
+    let source_code = fetched_file.source_code.to_str();
+    assert!(source_code.is_ok());
+    let actual = source_code.unwrap();
+    assert_eq!(expected_content, actual);
+  }
+
+  #[tokio::test]
+  async fn test_fetch_source_file_from_disk_utf_16_be() {
+    test_fetch_source_file_from_disk_nonstandard_encoding(
+      "utf-16be",
+      String::from_utf8(
+        b"\xEF\xBB\xBFconsole.log(\"Hello World\");\x0A".to_vec(),
+      )
+      .unwrap(),
+    )
+    .await;
+  }
+
+  #[tokio::test]
+  async fn test_fetch_source_file_from_disk_utf_16_le() {
+    test_fetch_source_file_from_disk_nonstandard_encoding(
+      "utf-16le",
+      String::from_utf8(
+        b"\xEF\xBB\xBFconsole.log(\"Hello World\");\x0A".to_vec(),
+      )
+      .unwrap(),
+    )
+    .await;
+  }
+
+  #[tokio::test]
+  async fn test_fetch_source_file_from_disk_utf_8_with_bom() {
+    test_fetch_source_file_from_disk_nonstandard_encoding(
+      "utf-8",
+      String::from_utf8(
+        b"\xEF\xBB\xBFconsole.log(\"Hello World\");\x0A".to_vec(),
+      )
+      .unwrap(),
+    )
+    .await;
+  }
+
   #[test]
   fn test_map_file_extension() {
     assert_eq!(
@@ -1571,43 +1681,43 @@ mod tests {
   fn test_map_content_type_extension_only() {
     // Extension only
     assert_eq!(
-      map_content_type(Path::new("foo/bar.ts"), None),
+      map_content_type(Path::new("foo/bar.ts"), None).0,
       msg::MediaType::TypeScript
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar.tsx"), None),
+      map_content_type(Path::new("foo/bar.tsx"), None).0,
       msg::MediaType::TSX
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar.d.ts"), None),
+      map_content_type(Path::new("foo/bar.d.ts"), None).0,
       msg::MediaType::TypeScript
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar.js"), None),
+      map_content_type(Path::new("foo/bar.js"), None).0,
       msg::MediaType::JavaScript
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar.txt"), None),
+      map_content_type(Path::new("foo/bar.txt"), None).0,
       msg::MediaType::Unknown
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar.jsx"), None),
+      map_content_type(Path::new("foo/bar.jsx"), None).0,
       msg::MediaType::JSX
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar.json"), None),
+      map_content_type(Path::new("foo/bar.json"), None).0,
       msg::MediaType::Json
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar.wasm"), None),
+      map_content_type(Path::new("foo/bar.wasm"), None).0,
       msg::MediaType::Wasm
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar.cjs"), None),
+      map_content_type(Path::new("foo/bar.cjs"), None).0,
       msg::MediaType::JavaScript
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar"), None),
+      map_content_type(Path::new("foo/bar"), None).0,
       msg::MediaType::Unknown
     );
   }
@@ -1616,140 +1726,154 @@ mod tests {
   fn test_map_content_type_media_type_with_no_extension() {
     // Media Type
     assert_eq!(
-      map_content_type(Path::new("foo/bar"), Some("application/typescript")),
+      map_content_type(Path::new("foo/bar"), Some("application/typescript")).0,
       msg::MediaType::TypeScript
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar"), Some("text/typescript")),
+      map_content_type(Path::new("foo/bar"), Some("text/typescript")).0,
       msg::MediaType::TypeScript
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar"), Some("video/vnd.dlna.mpeg-tts")),
+      map_content_type(Path::new("foo/bar"), Some("video/vnd.dlna.mpeg-tts")).0,
       msg::MediaType::TypeScript
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar"), Some("video/mp2t")),
+      map_content_type(Path::new("foo/bar"), Some("video/mp2t")).0,
       msg::MediaType::TypeScript
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar"), Some("application/x-typescript")),
+      map_content_type(Path::new("foo/bar"), Some("application/x-typescript"))
+        .0,
       msg::MediaType::TypeScript
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar"), Some("application/javascript")),
+      map_content_type(Path::new("foo/bar"), Some("application/javascript")).0,
       msg::MediaType::JavaScript
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar"), Some("text/javascript")),
+      map_content_type(Path::new("foo/bar"), Some("text/javascript")).0,
       msg::MediaType::JavaScript
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar"), Some("application/ecmascript")),
+      map_content_type(Path::new("foo/bar"), Some("application/ecmascript")).0,
       msg::MediaType::JavaScript
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar"), Some("text/ecmascript")),
+      map_content_type(Path::new("foo/bar"), Some("text/ecmascript")).0,
       msg::MediaType::JavaScript
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar"), Some("application/x-javascript")),
+      map_content_type(Path::new("foo/bar"), Some("application/x-javascript"))
+        .0,
       msg::MediaType::JavaScript
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar"), Some("application/json")),
+      map_content_type(Path::new("foo/bar"), Some("application/json")).0,
       msg::MediaType::Json
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar"), Some("application/node")),
+      map_content_type(Path::new("foo/bar"), Some("application/node")).0,
       msg::MediaType::JavaScript
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar"), Some("text/json")),
+      map_content_type(Path::new("foo/bar"), Some("text/json")).0,
       msg::MediaType::Json
     );
+    assert_eq!(
+      map_content_type(Path::new("foo/bar"), Some("text/json; charset=utf-8 ")),
+      (msg::MediaType::Json, Some("utf-8".to_owned()))
+    );
   }
 
   #[test]
   fn test_map_file_extension_media_type_with_extension() {
     assert_eq!(
-      map_content_type(Path::new("foo/bar.ts"), Some("text/plain")),
+      map_content_type(Path::new("foo/bar.ts"), Some("text/plain")).0,
       msg::MediaType::TypeScript
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar.ts"), Some("foo/bar")),
+      map_content_type(Path::new("foo/bar.ts"), Some("foo/bar")).0,
       msg::MediaType::Unknown
     );
     assert_eq!(
       map_content_type(
         Path::new("foo/bar.tsx"),
         Some("application/typescript"),
-      ),
+      )
+      .0,
       msg::MediaType::TSX
     );
     assert_eq!(
       map_content_type(
         Path::new("foo/bar.tsx"),
         Some("application/javascript"),
-      ),
+      )
+      .0,
       msg::MediaType::TSX
     );
     assert_eq!(
       map_content_type(
         Path::new("foo/bar.tsx"),
         Some("application/x-typescript"),
-      ),
+      )
+      .0,
       msg::MediaType::TSX
     );
     assert_eq!(
       map_content_type(
         Path::new("foo/bar.tsx"),
         Some("video/vnd.dlna.mpeg-tts"),
-      ),
+      )
+      .0,
       msg::MediaType::TSX
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar.tsx"), Some("video/mp2t")),
+      map_content_type(Path::new("foo/bar.tsx"), Some("video/mp2t")).0,
       msg::MediaType::TSX
     );
     assert_eq!(
       map_content_type(
         Path::new("foo/bar.jsx"),
         Some("application/javascript"),
-      ),
+      )
+      .0,
       msg::MediaType::JSX
     );
     assert_eq!(
       map_content_type(
         Path::new("foo/bar.jsx"),
         Some("application/x-typescript"),
-      ),
+      )
+      .0,
       msg::MediaType::JSX
     );
     assert_eq!(
       map_content_type(
         Path::new("foo/bar.jsx"),
         Some("application/ecmascript"),
-      ),
+      )
+      .0,
       msg::MediaType::JSX
     );
     assert_eq!(
-      map_content_type(Path::new("foo/bar.jsx"), Some("text/ecmascript")),
+      map_content_type(Path::new("foo/bar.jsx"), Some("text/ecmascript")).0,
       msg::MediaType::JSX
     );
     assert_eq!(
       map_content_type(
         Path::new("foo/bar.jsx"),
         Some("application/x-javascript"),
-      ),
+      )
+      .0,
       msg::MediaType::JSX
     );
   }
 
   #[test]
   fn test_filter_shebang() {
-    assert_eq!(filter_shebang(b"#!"[..].to_owned()), b"");
-    assert_eq!(filter_shebang(b"#!\n\n"[..].to_owned()), b"\n\n");
-    let code = b"#!/usr/bin/env deno\nconsole.log('hello');\n"[..].to_owned();
+    assert_eq!(filter_shebang("#!"), b"");
+    assert_eq!(filter_shebang("#!\n\n"), b"\n\n");
+    let code = "#!/usr/bin/env deno\nconsole.log('hello');\n";
     assert_eq!(filter_shebang(code), b"\nconsole.log('hello');\n");
   }
 
@@ -1771,7 +1895,7 @@ mod tests {
       .await;
     assert!(source.is_ok());
     let source = source.unwrap();
-    assert_eq!(source.source_code, b"console.log('etag')");
+    assert_eq!(source.source_code.bytes, b"console.log('etag')");
     assert_eq!(&(source.media_type), &msg::MediaType::TypeScript);
 
     let (_, headers) = fetcher.http_cache.get(&module_url).unwrap();
@@ -1798,7 +1922,7 @@ mod tests {
       )
       .await
       .unwrap();
-    assert_eq!(cached_source.source_code, b"changed content");
+    assert_eq!(cached_source.source_code.bytes, b"changed content");
 
     let modified2 = metadata_path.metadata().unwrap().modified().unwrap();
 
@@ -1825,7 +1949,7 @@ mod tests {
       .await;
     assert!(source.is_ok());
     let source = source.unwrap();
-    assert_eq!(source.source_code, b"export const foo = 'foo';");
+    assert_eq!(source.source_code.bytes, b"export const foo = 'foo';");
     assert_eq!(&(source.media_type), &msg::MediaType::JavaScript);
     assert_eq!(
       source.types_header,
@@ -1833,4 +1957,80 @@ mod tests {
     );
     drop(http_server_guard);
   }
+
+  #[tokio::test]
+  async fn test_fetch_source_file_from_net_utf16_le() {
+    let content =
+      std::str::from_utf8(b"\xEF\xBB\xBFconsole.log(\"Hello World\");\x0A")
+        .unwrap();
+    test_fetch_non_utf8_source_file_from_net(
+      "utf-16le",
+      "utf-16le.ts",
+      content,
+    )
+    .await;
+  }
+
+  #[tokio::test]
+  async fn test_fetch_source_file_from_net_utf16_be() {
+    let content =
+      std::str::from_utf8(b"\xEF\xBB\xBFconsole.log(\"Hello World\");\x0A")
+        .unwrap();
+    test_fetch_non_utf8_source_file_from_net(
+      "utf-16be",
+      "utf-16be.ts",
+      content,
+    )
+    .await;
+  }
+
+  #[tokio::test]
+  async fn test_fetch_source_file_from_net_windows_1255() {
+    let content = "console.log(\"\u{5E9}\u{5DC}\u{5D5}\u{5DD} \
+                   \u{5E2}\u{5D5}\u{5DC}\u{5DD}\");\u{A}";
+    test_fetch_non_utf8_source_file_from_net(
+      "windows-1255",
+      "windows-1255",
+      content,
+    )
+    .await;
+  }
+
+  async fn test_fetch_non_utf8_source_file_from_net(
+    charset: &str,
+    file_name: &str,
+    expected_content: &str,
+  ) {
+    let http_server_guard = test_util::http_server();
+    let (_temp_dir, fetcher) = test_setup();
+    let module_url = Url::parse(&format!(
+      "http://127.0.0.1:4545/cli/tests/encoding/{}",
+      file_name
+    ))
+    .unwrap();
+
+    let source = fetcher
+      .fetch_remote_source(
+        &module_url,
+        false,
+        false,
+        1,
+        &Permissions::allow_all(),
+      )
+      .await;
+    assert!(source.is_ok());
+    let source = source.unwrap();
+    assert_eq!(&source.source_code.charset.to_lowercase()[..], charset);
+    let text = &source.source_code.to_str().unwrap();
+    assert_eq!(text, expected_content);
+    assert_eq!(&(source.media_type), &msg::MediaType::TypeScript);
+
+    let (_, headers) = fetcher.http_cache.get(&module_url).unwrap();
+    assert_eq!(
+      headers.get("content-type").unwrap(),
+      &format!("application/typescript;charset={}", charset)
+    );
+
+    drop(http_server_guard);
+  }
 }