diff options
| -rw-r--r-- | Makefile | 1 | ||||
| -rw-r--r-- | main.go | 151 | ||||
| -rw-r--r-- | main.ts | 6 | ||||
| -rw-r--r-- | msg.proto | 16 | ||||
| -rw-r--r-- | os.ts | 14 | ||||
| -rw-r--r-- | runtime.ts | 102 |
6 files changed, 177 insertions, 113 deletions
@@ -49,5 +49,6 @@ fmt: node_modules test: deno node test.js + go test .PHONY: test lint clean distclean @@ -5,8 +5,10 @@ import ( "encoding/hex" "github.com/golang/protobuf/proto" "github.com/ry/v8worker2" + "io" "io/ioutil" "net/http" + "net/url" "os" "path" "runtime" @@ -15,6 +17,10 @@ import ( "time" ) +var DenoDir string +var CompileDir string +var SrcDir string + var wg sync.WaitGroup var resChan chan *Msg @@ -30,59 +36,126 @@ func CacheFileName(filename string, sourceCodeBuf []byte) string { return path.Join(CompileDir, cacheKey+".js") } -func IsRemotePath(filename string) bool { - return strings.HasPrefix(filename, "/$remote$/") +func IsRemote(filename string) bool { + u, err := url.Parse(filename) + check(err) + return u.IsAbs() +} + +// Fetches a remoteUrl but also caches it to the localFilename. +func FetchRemoteSource(remoteUrl string, localFilename string) ([]byte, error) { + Assert(strings.HasPrefix(localFilename, SrcDir), localFilename) + var sourceReader io.Reader + + file, err := os.Open(localFilename) + if os.IsNotExist(err) { + // Fetch from HTTP. + res, err := http.Get(remoteUrl) + if err != nil { + return nil, err + } + defer res.Body.Close() + + err = os.MkdirAll(path.Dir(localFilename), 0700) + if err != nil { + return nil, err + } + + // Write to to file. Need to reopen it for writing. + file, err = os.OpenFile(localFilename, os.O_RDWR|os.O_CREATE, 0700) + if err != nil { + return nil, err + } + sourceReader = io.TeeReader(res.Body, file) // Fancy! + + } else if err != nil { + return nil, err + } else { + sourceReader = file + } + defer file.Close() + return ioutil.ReadAll(sourceReader) } -func FetchRemoteSource(remotePath string) (buf []byte, err error) { - url := strings.Replace(remotePath, "/$remote$/", "http://", 1) - // println("FetchRemoteSource", url) - res, err := http.Get(url) +func ResolveModule(moduleSpecifier string, containingFile string) ( + moduleName string, filename string, err error) { + moduleUrl, err := url.Parse(moduleSpecifier) if err != nil { return } - buf, err = ioutil.ReadAll(res.Body) - //println("FetchRemoteSource", err.Error()) - res.Body.Close() + baseUrl, err := url.Parse(containingFile) + if err != nil { + return + } + resolved := baseUrl.ResolveReference(moduleUrl) + moduleName = resolved.String() + if moduleUrl.IsAbs() { + filename = path.Join(SrcDir, resolved.Host, resolved.Path) + } else { + filename = resolved.Path + } return } -func HandleSourceCodeFetch(filename string) []byte { +const assetPrefix string = "/$asset$/" + +func HandleSourceCodeFetch(moduleSpecifier string, containingFile string) (out []byte) { res := &Msg{} var sourceCodeBuf []byte var err error - if IsRemotePath(filename) { - sourceCodeBuf, err = FetchRemoteSource(filename) - } else { - sourceCodeBuf, err = Asset("dist/" + filename) + + defer func() { if err != nil { - sourceCodeBuf, err = ioutil.ReadFile(filename) + res.Error = err.Error() } - } + out, err = proto.Marshal(res) + check(err) + }() + + moduleName, filename, err := ResolveModule(moduleSpecifier, containingFile) if err != nil { - res.Error = err.Error() + return + } + + if IsRemote(moduleName) { + sourceCodeBuf, err = FetchRemoteSource(moduleName, filename) + } else if strings.HasPrefix(moduleName, assetPrefix) { + f := strings.TrimPrefix(moduleName, assetPrefix) + sourceCodeBuf, err = Asset("dist/" + f) } else { - cacheFn := CacheFileName(filename, sourceCodeBuf) - outputCodeBuf, err := ioutil.ReadFile(cacheFn) - var outputCode string - if os.IsNotExist(err) { - outputCode = "" - } else if err != nil { - res.Error = err.Error() - } else { - outputCode = string(outputCodeBuf) - } + Assert(moduleName == filename, + "if a module isn't remote, it should have the same filename") + sourceCodeBuf, err = ioutil.ReadFile(moduleName) + } + if err != nil { + return + } - res.Payload = &Msg_SourceCodeFetchRes{ - SourceCodeFetchRes: &SourceCodeFetchResMsg{ - SourceCode: string(sourceCodeBuf), - OutputCode: outputCode, - }, - } + outputCode, err := LoadOutputCodeCache(filename, sourceCodeBuf) + if err != nil { + return } - out, err := proto.Marshal(res) - check(err) - return out + + res.Payload = &Msg_SourceCodeFetchRes{ + SourceCodeFetchRes: &SourceCodeFetchResMsg{ + ModuleName: moduleName, + Filename: filename, + SourceCode: string(sourceCodeBuf), + OutputCode: outputCode, + }, + } + return +} + +func LoadOutputCodeCache(filename string, sourceCodeBuf []byte) (outputCode string, err error) { + cacheFn := CacheFileName(filename, sourceCodeBuf) + outputCodeBuf, err := ioutil.ReadFile(cacheFn) + if os.IsNotExist(err) { + err = nil // Ignore error if we can't load the cache. + } else if err != nil { + outputCode = string(outputCodeBuf) + } + return } func HandleSourceCodeCache(filename string, sourceCode string, @@ -134,10 +207,6 @@ func loadAsset(w *v8worker2.Worker, path string) { check(err) } -var DenoDir string -var CompileDir string -var SrcDir string - func createDirs() { DenoDir = path.Join(UserHomeDir(), ".deno") CompileDir = path.Join(DenoDir, "compile") @@ -164,7 +233,7 @@ func recv(buf []byte) []byte { os.Exit(int(payload.Code)) case *Msg_SourceCodeFetch: payload := msg.GetSourceCodeFetch() - return HandleSourceCodeFetch(payload.Filename) + return HandleSourceCodeFetch(payload.ModuleSpecifier, payload.ContainingFile) case *Msg_SourceCodeCache: payload := msg.GetSourceCodeCache() return HandleSourceCodeCache(payload.Filename, payload.SourceCode, @@ -2,14 +2,12 @@ import { main as pb } from "./msg.pb"; import "./util"; import * as runtime from "./runtime"; import * as timers from "./timers"; -import * as path from "path"; function start(cwd: string, argv: string[]): void { // TODO parse arguments. const inputFn = argv[1]; - const fn = path.resolve(cwd, inputFn); - const m = runtime.FileModule.load(fn); - m.compileAndRun(); + const mod = runtime.resolveModule(inputFn, cwd + "/"); + mod.compileAndRun(); } V8Worker2.recv((ab: ArrayBuffer) => { @@ -3,7 +3,6 @@ package main; message Msg { string error = 1; - oneof payload { StartMsg start = 10; SourceCodeFetchMsg source_code_fetch = 11; @@ -15,17 +14,24 @@ message Msg { } } -// START message StartMsg { string cwd = 1; repeated string argv = 2; } -message SourceCodeFetchMsg { string filename = 1; } +message SourceCodeFetchMsg { + string module_specifier = 1; + string containing_file = 2; +} message SourceCodeFetchResMsg { - string source_code = 1; - string output_code = 2; + // If it's a non-http module, moduleName and filename will be the same. + // For http modules, moduleName is its resolved http URL, and filename + // is the location of the locally downloaded source code. + string moduleName = 1; + string filename = 2; + string source_code = 3; + string output_code = 4; // Non-empty only if cached. } message SourceCodeCacheMsg { @@ -1,7 +1,5 @@ import { main as pb } from "./msg.pb"; - -// TODO move this to types.ts -type TypedArray = Uint8Array | Float32Array | Int32Array; +import { TypedArray, ModuleInfo } from "./types"; export function exit(code = 0): void { sendMsgFromObject({ @@ -10,13 +8,13 @@ export function exit(code = 0): void { } export function sourceCodeFetch( - filename: string -): { sourceCode: string; outputCode: string } { + moduleSpecifier: string, + containingFile: string +): ModuleInfo { const res = sendMsgFromObject({ - sourceCodeFetch: { filename } + sourceCodeFetch: { moduleSpecifier, containingFile } }); - const { sourceCode, outputCode } = res.sourceCodeFetchRes; - return { sourceCode, outputCode }; + return res.sourceCodeFetchRes; } export function sourceCodeCache( diff --git a/runtime.ts b/runtime.ts index f3fb6765b..01dbf76ac 100644 --- a/runtime.ts +++ b/runtime.ts @@ -1,11 +1,12 @@ // Glossary // outputCode = generated javascript code // sourceCode = typescript code (or input javascript code) -// fileName = an unresolved raw fileName. // moduleName = a resolved module name +// fileName = an unresolved raw fileName. +// for http modules , its the path to the locally downloaded +// version. import * as ts from "typescript"; -import * as path from "path"; import * as util from "./util"; import { log } from "./util"; import * as os from "./os"; @@ -16,29 +17,31 @@ const EOL = "\n"; // This class represents a module. We call it FileModule to make it explicit // that each module represents a single file. // Access to FileModule instances should only be done thru the static method -// FileModule.load(). FileModules are executed upon first load. +// FileModule.load(). FileModules are NOT executed upon first load, only when +// compileAndRun is called. export class FileModule { scriptVersion: string = undefined; - sourceCode: string; - outputCode: string; readonly exports = {}; private static readonly map = new Map<string, FileModule>(); - private constructor(readonly fileName: string) { + constructor( + readonly fileName: string, + readonly sourceCode = "", + public outputCode = "" + ) { FileModule.map.set(fileName, this); - - // Load typescript code (sourceCode) and maybe load compiled javascript - // (outputCode) from cache. If cache is empty, outputCode will be null. - const { sourceCode, outputCode } = os.sourceCodeFetch(this.fileName); - this.sourceCode = sourceCode; - this.outputCode = outputCode; - this.scriptVersion = "1"; + if (outputCode !== "") { + this.scriptVersion = "1"; + } } - compileAndRun() { + compileAndRun(): void { if (!this.outputCode) { // If there is no cached outputCode, the compile the code. - util.assert(this.sourceCode && this.sourceCode.length > 0); + util.assert( + this.sourceCode != null && this.sourceCode.length > 0, + `Have no source code from ${this.fileName}` + ); const compiler = Compiler.instance(); this.outputCode = compiler.compile(this.fileName); os.sourceCodeCache(this.fileName, this.sourceCode, this.outputCode); @@ -48,12 +51,7 @@ export class FileModule { } static load(fileName: string): FileModule { - let m = this.map.get(fileName); - if (m == null) { - m = new this(fileName); - util.assert(this.map.has(fileName)); - } - return m; + return this.map.get(fileName); } static getScriptsWithSourceCode(): string[] { @@ -97,26 +95,26 @@ export function makeDefine(fileName: string): AmdDefine { return localDefine; } -function resolveModuleName(moduleName: string, containingFile: string): string { - if (isUrl(moduleName)) { - // Remove the "http://" from the start of the string. - const u = new URL(moduleName); - const withoutProtocol = u.toString().replace(u.protocol + "//", ""); - const name2 = "/$remote$/" + withoutProtocol; - return name2; - } else if (moduleName.startsWith("/")) { - throw Error("Absolute paths not supported"); - } else { - // Relative import. - const containingDir = path.dirname(containingFile); - const resolvedFileName = path.join(containingDir, moduleName); - util.log("relative import", { - containingFile, - moduleName, - resolvedFileName - }); - return resolvedFileName; - } +export function resolveModule( + moduleSpecifier: string, + containingFile: string +): FileModule { + // We ask golang to sourceCodeFetch. It will load the sourceCode and if + // there is any outputCode cached, it will return that as well. + const { filename, sourceCode, outputCode } = os.sourceCodeFetch( + moduleSpecifier, + containingFile + ); + util.log("resolveModule", { containingFile, moduleSpecifier, filename }); + return new FileModule(filename, sourceCode, outputCode); +} + +function resolveModuleName( + moduleSpecifier: string, + containingFile: string +): string { + const mod = resolveModule(moduleSpecifier, containingFile); + return mod.fileName; } function execute(fileName: string, outputCode: string): void { @@ -171,7 +169,6 @@ class Compiler { os.exit(1); } - util.log("compile output", output); util.assert(!output.emitSkipped); const outputCode = output.outputFiles[0].text; @@ -199,15 +196,14 @@ class TypeScriptHost implements ts.LanguageServiceHost { getScriptSnapshot(fileName: string): ts.IScriptSnapshot | undefined { util.log("getScriptSnapshot", fileName); const m = FileModule.load(fileName); - if (m.sourceCode) { - return ts.ScriptSnapshot.fromString(m.sourceCode); - } else { - return undefined; - } + util.assert(m != null); + util.assert(m.sourceCode.length > 0); + return ts.ScriptSnapshot.fromString(m.sourceCode); } fileExists(fileName: string): boolean { - throw Error("not implemented"); + util.log("fileExist", fileName); + return true; } readFile(path: string, encoding?: string): string | undefined { @@ -231,7 +227,9 @@ class TypeScriptHost implements ts.LanguageServiceHost { getDefaultLibFileName(options: ts.CompilerOptions): string { util.log("getDefaultLibFileName"); - return ts.getDefaultLibFileName(options); + const fn = ts.getDefaultLibFileName(options); + const m = resolveModule(fn, "/$asset$/"); + return m.fileName; } resolveModuleNames( @@ -248,12 +246,6 @@ class TypeScriptHost implements ts.LanguageServiceHost { } } -function isUrl(p: string): boolean { - return ( - p.startsWith("//") || p.startsWith("http://") || p.startsWith("https://") - ); -} - const formatDiagnosticsHost: ts.FormatDiagnosticsHost = { getCurrentDirectory(): string { return "."; |