From 9d63772fe5bacc8fa1e0a8cbb152a2f107ae268f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Mon, 18 May 2020 12:59:29 +0200 Subject: refactor: rewrite TS dependency analysis in Rust (#5029) This commit completely overhauls how module analysis is performed in TS compiler by moving the logic to Rust. In the current setup module analysis is performed using "ts.preProcessFile" API in a special TS compiler worker running on a separate thread. "ts.preProcessFile" allowed us to build a lot of functionality in CLI including X-TypeScript-Types header support and @deno-types directive support. Unfortunately at the same time complexity of the ops required to perform supporting tasks exploded and caused some hidden permission escapes. This PR introduces "ModuleGraphLoader" which can parse source and load recursively all dependent source files; as well as declaration files. All dependencies used in TS compiler and now fetched and collected upfront in Rust before spinning up TS compiler. To achieve feature parity with existing APIs this commit includes a lot of changes: * add "ModuleGraphLoader" - can fetch local and remote sources - parses source code using SWC and extracts imports, exports, file references, special headers - this struct inherited all of the hidden complexity and cruft from TS version and requires several follow up PRs * rewrite cli/tsc.rs to perform module analysis upfront and send all required source code to TS worker in one message * remove op_resolve_modules and op_fetch_source_files from cli/ops/compiler.rs * run TS worker on the same thread --- cli/tests/020_json_modules.ts.out | 10 +++++----- cli/tests/037_fetch_multiple.out | 4 ++-- cli/tests/error_004_missing_module.ts.out | 9 +-------- cli/tests/error_005_missing_dynamic_import.ts.out | 9 +-------- cli/tests/error_006_import_ext_failure.ts.out | 9 +-------- cli/tests/error_011_bad_module_specifier.ts.out | 10 +--------- cli/tests/error_012_bad_dynamic_import_specifier.ts.out | 10 +--------- cli/tests/error_local_static_import_from_remote.ts.out | 9 +-------- cli/tests/error_type_definitions.ts.out | 11 +---------- cli/tests/integration_tests.rs | 2 +- cli/tests/type_definitions/bar.js | 5 +++++ 11 files changed, 20 insertions(+), 68 deletions(-) create mode 100644 cli/tests/type_definitions/bar.js (limited to 'cli/tests') diff --git a/cli/tests/020_json_modules.ts.out b/cli/tests/020_json_modules.ts.out index cda28038e..4369639eb 100644 --- a/cli/tests/020_json_modules.ts.out +++ b/cli/tests/020_json_modules.ts.out @@ -2,8 +2,8 @@ error: Uncaught TypeError: Cannot resolve extension for "[WILDCARD]config.json" with mediaType "Json". at getExtension ($deno$/compiler.ts:[WILDCARD]) at new SourceFile ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async compile ($deno$/compiler.ts:[WILDCARD]) - at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) - at async workerMessageRecvCallback ($deno$/runtime_worker.ts:[WILDCARD]) + at Function.addToCache ($deno$/compiler.ts:[WILDCARD]) + at buildSourceFileCache ($deno$/compiler.ts:[WILDCARD]) + at compile ($deno$/compiler.ts:[WILDCARD]) + at tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) +[WILDCARD] \ No newline at end of file diff --git a/cli/tests/037_fetch_multiple.out b/cli/tests/037_fetch_multiple.out index cdb6fe2ba..1a703a10c 100644 --- a/cli/tests/037_fetch_multiple.out +++ b/cli/tests/037_fetch_multiple.out @@ -1,5 +1,5 @@ -Compile [WILDCARD]/fetch/test.ts Download http://localhost:4545/cli/tests/subdir/mod2.ts Download http://localhost:4545/cli/tests/subdir/print_hello.ts -Compile [WILDCARD]/fetch/other.ts +Compile [WILDCARD]/fetch/test.ts Download http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts +Compile [WILDCARD]/fetch/other.ts diff --git a/cli/tests/error_004_missing_module.ts.out b/cli/tests/error_004_missing_module.ts.out index b93f2e613..d851882eb 100644 --- a/cli/tests/error_004_missing_module.ts.out +++ b/cli/tests/error_004_missing_module.ts.out @@ -1,8 +1 @@ -[WILDCARD]error: Uncaught NotFound: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_004_missing_module.ts" - at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at Object.sendAsync ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async compile ($deno$/compiler.ts:[WILDCARD]) - at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) - at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) +[WILDCARD]error: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_004_missing_module.ts" diff --git a/cli/tests/error_005_missing_dynamic_import.ts.out b/cli/tests/error_005_missing_dynamic_import.ts.out index e8c8f8cd0..0f0e449c4 100644 --- a/cli/tests/error_005_missing_dynamic_import.ts.out +++ b/cli/tests/error_005_missing_dynamic_import.ts.out @@ -1,8 +1 @@ -[WILDCARD]error: Uncaught NotFound: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_005_missing_dynamic_import.ts" - at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at Object.sendAsync ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async compile ($deno$/compiler.ts:[WILDCARD]) - at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) - at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) +[WILDCARD]error: Uncaught TypeError: Cannot resolve module "[WILDCARD]/bad-module.ts" diff --git a/cli/tests/error_006_import_ext_failure.ts.out b/cli/tests/error_006_import_ext_failure.ts.out index 5c89adeff..c44d5e746 100644 --- a/cli/tests/error_006_import_ext_failure.ts.out +++ b/cli/tests/error_006_import_ext_failure.ts.out @@ -1,8 +1 @@ -[WILDCARD]error: Uncaught NotFound: Cannot resolve module "[WILDCARD]/non-existent" from "[WILDCARD]/error_006_import_ext_failure.ts" - at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at Object.sendAsync ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async compile ($deno$/compiler.ts:[WILDCARD]) - at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) - at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) +[WILDCARD]error: Cannot resolve module "[WILDCARD]/non-existent" from "[WILDCARD]/error_006_import_ext_failure.ts" diff --git a/cli/tests/error_011_bad_module_specifier.ts.out b/cli/tests/error_011_bad_module_specifier.ts.out index 39726d5c6..e6f9b2321 100644 --- a/cli/tests/error_011_bad_module_specifier.ts.out +++ b/cli/tests/error_011_bad_module_specifier.ts.out @@ -1,9 +1 @@ -[WILDCARD]error: Uncaught URIError: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_011_bad_module_specifier.ts" - at unwrapResponse ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at Object.sendSync ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at resolveModules ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at async compile ($deno$/compiler.ts:[WILDCARD]) - at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) - at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) +[WILDCARD]error: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_011_bad_module_specifier.ts" diff --git a/cli/tests/error_012_bad_dynamic_import_specifier.ts.out b/cli/tests/error_012_bad_dynamic_import_specifier.ts.out index 900b8f52d..57e4003ce 100644 --- a/cli/tests/error_012_bad_dynamic_import_specifier.ts.out +++ b/cli/tests/error_012_bad_dynamic_import_specifier.ts.out @@ -1,9 +1 @@ -[WILDCARD]error: Uncaught URIError: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_012_bad_dynamic_import_specifier.ts" - at unwrapResponse ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at Object.sendSync ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at resolveModules ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at async compile ($deno$/compiler.ts:[WILDCARD]) - at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) - at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) +[WILDCARD]error: Uncaught TypeError: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_012_bad_dynamic_import_specifier.ts" diff --git a/cli/tests/error_local_static_import_from_remote.ts.out b/cli/tests/error_local_static_import_from_remote.ts.out index af3c8852f..4e6a9d4e7 100644 --- a/cli/tests/error_local_static_import_from_remote.ts.out +++ b/cli/tests/error_local_static_import_from_remote.ts.out @@ -1,9 +1,2 @@ [WILDCARD] -error: Uncaught PermissionDenied: Remote module are not allowed to statically import local modules. Use dynamic import instead. - at unwrapResponse ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at Object.sendAsync ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async compile ($deno$/compiler.ts:[WILDCARD]) - at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) - at async workerMessageRecvCallback ($deno$/runtime_worker.ts:[WILDCARD]) +error: Remote module are not allowed to statically import local modules. Use dynamic import instead. diff --git a/cli/tests/error_type_definitions.ts.out b/cli/tests/error_type_definitions.ts.out index bacca4f80..32c3c9b52 100644 --- a/cli/tests/error_type_definitions.ts.out +++ b/cli/tests/error_type_definitions.ts.out @@ -1,10 +1 @@ -[WILDCARD]error: Uncaught URIError: relative import path "baz" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/type_definitions/bar.d.ts" - at unwrapResponse ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at Object.sendSync ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at resolveModules ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async compile ($deno$/compiler.ts:[WILDCARD]) - at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) - at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) +[WILDCARD]error: relative import path "baz" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/type_definitions/bar.d.ts" diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 758cf1007..476a08224 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -1353,7 +1353,7 @@ itest!(error_004_missing_module { }); itest!(error_005_missing_dynamic_import { - args: "run --reload error_005_missing_dynamic_import.ts", + args: "run --reload --allow-read error_005_missing_dynamic_import.ts", exit_code: 1, output: "error_005_missing_dynamic_import.ts.out", }); diff --git a/cli/tests/type_definitions/bar.js b/cli/tests/type_definitions/bar.js new file mode 100644 index 000000000..e9c2e5193 --- /dev/null +++ b/cli/tests/type_definitions/bar.js @@ -0,0 +1,5 @@ +export class Bar { + constructor() { + this.baz = "baz"; + } +} -- cgit v1.2.3