From d4f72e18bedda726a74ec0960e59e304055f9039 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Thu, 11 Oct 2018 16:56:50 -0400 Subject: Improve tools/unit_tests.py (#958) Checks the output more carefully. The first line of output from js/unit_tests.ts should be something like "running 96 tests" And the last line should be something like "test result: ok. 96 passed; 0 failed; 0 ignored; 0 measured; 36 filtered out" This parses those strings and make sure they align. This will catch silent death bugs. --- tools/testdata/unit_test_output1.txt | 238 +++++++++++++++++++++++++++++++ tools/testdata/unit_test_output2.txt | 71 ++++++++++ tools/testdata/unit_test_output3.txt | 268 +++++++++++++++++++++++++++++++++++ 3 files changed, 577 insertions(+) create mode 100644 tools/testdata/unit_test_output1.txt create mode 100644 tools/testdata/unit_test_output2.txt create mode 100644 tools/testdata/unit_test_output3.txt (limited to 'tools/testdata') diff --git a/tools/testdata/unit_test_output1.txt b/tools/testdata/unit_test_output1.txt new file mode 100644 index 000000000..4b208319f --- /dev/null +++ b/tools/testdata/unit_test_output1.txt @@ -0,0 +1,238 @@ +running 96 tests +test permSerialization_permW0N0E0 +... ok +test permFromStringThrows_permW0N0E0 +... ok +test compilerInstance_permW0N0E0 +... ok +test compilerRun_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerRunMultiModule_permW0N0E0 +... ok +test compilerRunCircularDependency_permW0N0E0 +Compiling modA +Compiling modB +... ok +test compilerResolveModule_permW0N0E0 +... ok +test compilerGetModuleDependencies_permW0N0E0 +... ok +test compilerGetCompilationSettings_permW0N0E0 +... ok +test compilerGetNewLine_permW0N0E0 +... ok +test compilerGetScriptFileNames_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerRecompileFlag_permW0N0E0 +Compiling /root/project/foo/bar.ts +Compiling /root/project/foo/bar.ts +... ok +test compilerGetScriptKind_permW0N0E0 +... ok +test compilerGetScriptVersion_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerGetScriptVersionUnknown_permW0N0E0 +... ok +test compilerGetScriptSnapshot_permW0N0E0 +... ok +test compilerGetCurrentDirectory_permW0N0E0 +... ok +test compilerGetDefaultLibFileName_permW0N0E0 +... ok +test compilerUseCaseSensitiveFileNames_permW0N0E0 +... ok +test compilerReadFile_permW0N0E0 +... ok +test compilerFileExists_permW0N0E0 +... ok +test compilerResolveModuleNames_permW0N0E0 +... ok +test consoleTestAssert_permW0N0E0 +... ok +test consoleTestStringifyComplexObjects_permW0N0E0 +... ok +test consoleTestStringifyCircular_permW0N0E0 +... ok +test consoleTestStringifyWithDepth_permW0N0E0 +... ok +test consoleTestError_permW0N0E0 +... ok +test consoleDetachedLog_permW0N0E0 +Hello world +Hello world +Hello world +Hello world +Hello world +Hello world +... ok +test fetchPerm_permW0N0E0 +... ok +test headersAppend_permW0N0E0 +... ok +test newHeaderTest_permW0N0E0 +... ok +test newHeaderWithSequence_permW0N0E0 +... ok +test newHeaderWithRecord_permW0N0E0 +... ok +test newHeaderWithHeadersInstance_permW0N0E0 +... ok +test headerAppendSuccess_permW0N0E0 +... ok +test headerSetSuccess_permW0N0E0 +... ok +test headerHasSuccess_permW0N0E0 +... ok +test headerDeleteSuccess_permW0N0E0 +... ok +test headerGetSuccess_permW0N0E0 +... ok +test headerForEachSuccess_permW0N0E0 +... ok +test envFailure_permW0N0E0 +... ok +test filesStdioFileDescriptors_permW0N0E0 +... ok +test filesCopyToStdout_permW0N0E0 +{ + "name": "deno", + "devDependencies": { + "@types/base64-js": "^1.2.5", + "@types/flatbuffers": "^1.9.0", + "@types/source-map-support": "^0.4.1", + "@types/text-encoding": "0.0.33", + "base64-js": "^1.3.0", + "flatbuffers": "^1.9.0", + "magic-string": "^0.22.5", + "prettier": "^1.14.0", + "rollup": "^0.63.2", + "rollup-plugin-alias": "^1.4.0", + "rollup-plugin-analyzer": "^2.1.0", + "rollup-plugin-commonjs": "^9.1.3", + "rollup-plugin-node-globals": "^1.2.1", + "rollup-plugin-node-resolve": "^3.3.0", + "rollup-plugin-string": "^2.0.2", + "rollup-plugin-typescript2": "^0.16.1", + "rollup-pluginutils": "^2.3.0", + "source-map-support": "^0.5.6", + "text-encoding": "0.6.4", + "tslint": "^5.10.0", + "tslint-eslint-rules": "^5.3.1", + "tslint-no-circular-imports": "^0.5.0", + "typescript": "3.0.3" + } +} +bytes written 860 +... ok +test readFileSyncSuccess_permW0N0E0 +... ok +test readFileSyncNotFound_permW0N0E0 +... ok +test readFileSuccess_permW0N0E0 +... ok +test readDirSyncNotDir_permW0N0E0 +... ok +test readDirSyncNotFound_permW0N0E0 +... ok +test writeFileSyncPerm_permW0N0E0 +... ok +test writeFilePerm_permW0N0E0 +... ok +test copyFileSyncPerm_permW0N0E0 +... ok +test copyFilePerm_permW0N0E0 +... ok +test mkdirSyncPerm_permW0N0E0 +... ok +test makeTempDirSyncPerm_permW0N0E0 +... ok +test statSyncSuccess_permW0N0E0 +... ok +test statSyncNotFound_permW0N0E0 +... ok +test lstatSyncSuccess_permW0N0E0 +... ok +test lstatSyncNotFound_permW0N0E0 +... ok +test statSuccess_permW0N0E0 +... ok +test statNotFound_permW0N0E0 +... ok +test lstatSuccess_permW0N0E0 +... ok +test lstatNotFound_permW0N0E0 +... ok +test renameSyncPerm_permW0N0E0 +... ok +test readlinkSyncNotFound_permW0N0E0 +... ok +test blobString_permW0N0E0 +... ok +test blobBuffer_permW0N0E0 +... ok +test blobSlice_permW0N0E0 +... ok +test timeoutSuccess_permW0N0E0 +... ok +test timeoutArgs_permW0N0E0 +... ok +test timeoutCancelSuccess_permW0N0E0 +... ok +test timeoutCancelMultiple_permW0N0E0 +... ok +test timeoutCancelInvalidSilentFail_permW0N0E0 +... ok +test intervalSuccess_permW0N0E0 +... ok +test intervalCancelSuccess_permW0N0E0 +... ok +test intervalOrdering_permW0N0E0 +... ok +test intervalCancelInvalidSilentFail_permW0N0E0 +... ok +test symlinkSyncPerm_permW0N0E0 +... ok +test platformTransform_permW0N0E0 +... ok +test atobSuccess_permW0N0E0 +... ok +test btoaSuccess_permW0N0E0 +... ok +test btoaFailed_permW0N0E0 +... ok +test truncateSyncPerm_permW0N0E0 +... ok +test truncatePerm_permW0N0E0 +... ok +test evalErrorFormatted_permW0N0E0 +... ok +test createExecTimeColumnsRegularData_permW0N0E0 +... ok +test createExecTimeColumnsIrregularData_permW0N0E0 +... ok +test createBinarySizeColumnsRegularData_permW0N0E0 +... ok +test createBinarySizeColumnsIrregularData_permW0N0E0 +... ok +test createThreadCountColumnsRegularData_permW0N0E0 +... ok +test createThreadCountColumnsIrregularData_permW0N0E0 +... ok +test createSyscallCountColumnsRegularData_permW0N0E0 +... ok +test createSyscallCountColumnsIrregularData_permW0N0E0 +... ok +test createSha1ListRegularData_permW0N0E0 +... ok +test formatBytesPatterns_permW0N0E0 +... ok +test formatSecondsPatterns_permW0N0E0 +... ok +test getTravisDataSuccess_permW0N0E0 +... ok + +test result: ok. 96 passed; 0 failed; 0 ignored; 0 measured; 36 filtered out + diff --git a/tools/testdata/unit_test_output2.txt b/tools/testdata/unit_test_output2.txt new file mode 100644 index 000000000..5913d3b90 --- /dev/null +++ b/tools/testdata/unit_test_output2.txt @@ -0,0 +1,71 @@ +running 96 tests +test permSerialization_permW0N0E0 +... ok +test permFromStringThrows_permW0N0E0 +... ok +test compilerInstance_permW0N0E0 +... ok +test compilerRun_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerRunMultiModule_permW0N0E0 +... ok +test compilerRunCircularDependency_permW0N0E0 +Compiling modA +Compiling modB +... ok +test compilerResolveModule_permW0N0E0 +... ok +test compilerGetModuleDependencies_permW0N0E0 +... ok +test compilerGetCompilationSettings_permW0N0E0 +... ok +test compilerGetNewLine_permW0N0E0 +... ok +test compilerGetScriptFileNames_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerRecompileFlag_permW0N0E0 +Compiling /root/project/foo/bar.ts +Compiling /root/project/foo/bar.ts +... ok +test compilerGetScriptKind_permW0N0E0 +... ok +test compilerGetScriptVersion_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerGetScriptVersionUnknown_permW0N0E0 +... ok +test compilerGetScriptSnapshot_permW0N0E0 +... ok +test compilerGetCurrentDirectory_permW0N0E0 +... ok +test compilerGetDefaultLibFileName_permW0N0E0 +... ok +test compilerUseCaseSensitiveFileNames_permW0N0E0 +... ok +test compilerReadFile_permW0N0E0 +... ok +test compilerFileExists_permW0N0E0 +... ok +test compilerResolveModuleNames_permW0N0E0 +... ok +test consoleTestAssert_permW0N0E0 +... ok +test consoleTestStringifyComplexObjects_permW0N0E0 +... ok +test consoleTestStringifyCircular_permW0N0E0 +... ok +test consoleTestStringifyWithDepth_permW0N0E0 +... ok +test consoleTestError_permW0N0E0 +... ok +test consoleDetachedLog_permW0N0E0 +Hello world +Hello world +Hello world +Hello world +Hello world +Hello world +... ok +test fetchPerm_permW0N0E0 diff --git a/tools/testdata/unit_test_output3.txt b/tools/testdata/unit_test_output3.txt new file mode 100644 index 000000000..402261e76 --- /dev/null +++ b/tools/testdata/unit_test_output3.txt @@ -0,0 +1,268 @@ +Compiling /Users/rld/src/deno/js/unit_tests.ts +Compiling /Users/rld/src/deno/js/compiler_test.ts +Compiling /Users/rld/src/deno/js/test_util.ts +Compiling /Users/rld/src/deno/js/testing/testing.ts +Compiling /Users/rld/src/deno/js/testing/util.ts +Compiling /Users/rld/src/deno/js/console_test.ts +Compiling /Users/rld/src/deno/js/console.ts +Compiling /Users/rld/src/deno/js/fetch_test.ts +Compiling /Users/rld/src/deno/js/os_test.ts +Compiling /Users/rld/src/deno/js/files_test.ts +Compiling /Users/rld/src/deno/js/read_file_test.ts +Compiling /Users/rld/src/deno/js/read_dir_test.ts +Compiling /Users/rld/src/deno/js/write_file_test.ts +Compiling /Users/rld/src/deno/js/copy_file_test.ts +Compiling /Users/rld/src/deno/js/mkdir_test.ts +Compiling /Users/rld/src/deno/js/make_temp_dir_test.ts +Compiling /Users/rld/src/deno/js/stat_test.ts +Compiling /Users/rld/src/deno/js/rename_test.ts +Compiling /Users/rld/src/deno/js/read_link_test.ts +Compiling /Users/rld/src/deno/js/blob_test.ts +Compiling /Users/rld/src/deno/js/timers_test.ts +Compiling /Users/rld/src/deno/js/symlink_test.ts +Compiling /Users/rld/src/deno/js/platform_test.ts +Compiling /Users/rld/src/deno/js/text_encoding_test.ts +Compiling /Users/rld/src/deno/js/net_test.ts +Compiling /Users/rld/src/deno/js/trace_test.ts +Compiling /Users/rld/src/deno/js/truncate_test.ts +Compiling /Users/rld/src/deno/js/v8_source_maps_test.ts +Compiling /Users/rld/src/deno/website/app_test.js +Compiling /Users/rld/src/deno/website/app.js +running 96 tests +test permSerialization_permW0N0E0 +... ok +test permFromStringThrows_permW0N0E0 +... ok +test compilerInstance_permW0N0E0 +... ok +test compilerRun_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerRunMultiModule_permW0N0E0 +... ok +test compilerRunCircularDependency_permW0N0E0 +Compiling modA +Compiling modB +... ok +test compilerResolveModule_permW0N0E0 +... ok +test compilerGetModuleDependencies_permW0N0E0 +... ok +test compilerGetCompilationSettings_permW0N0E0 +... ok +test compilerGetNewLine_permW0N0E0 +... ok +test compilerGetScriptFileNames_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerRecompileFlag_permW0N0E0 +Compiling /root/project/foo/bar.ts +Compiling /root/project/foo/bar.ts +... ok +test compilerGetScriptKind_permW0N0E0 +... ok +test compilerGetScriptVersion_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerGetScriptVersionUnknown_permW0N0E0 +... ok +test compilerGetScriptSnapshot_permW0N0E0 +... ok +test compilerGetCurrentDirectory_permW0N0E0 +... ok +test compilerGetDefaultLibFileName_permW0N0E0 +... ok +test compilerUseCaseSensitiveFileNames_permW0N0E0 +... ok +test compilerReadFile_permW0N0E0 +... ok +test compilerFileExists_permW0N0E0 +... ok +test compilerResolveModuleNames_permW0N0E0 +... ok +test consoleTestAssert_permW0N0E0 +... ok +test consoleTestStringifyComplexObjects_permW0N0E0 +... ok +test consoleTestStringifyCircular_permW0N0E0 +... ok +test consoleTestStringifyWithDepth_permW0N0E0 +... ok +test consoleTestError_permW0N0E0 +... ok +test consoleDetachedLog_permW0N0E0 +Hello world +Hello world +Hello world +Hello world +Hello world +Hello world +... ok +test fetchPerm_permW0N0E0 +... ok +test headersAppend_permW0N0E0 +... ok +test newHeaderTest_permW0N0E0 +... ok +test newHeaderWithSequence_permW0N0E0 +... ok +test newHeaderWithRecord_permW0N0E0 +... ok +test newHeaderWithHeadersInstance_permW0N0E0 +... ok +test headerAppendSuccess_permW0N0E0 +... ok +test headerSetSuccess_permW0N0E0 +... ok +test headerHasSuccess_permW0N0E0 +... ok +test headerDeleteSuccess_permW0N0E0 +... ok +test headerGetSuccess_permW0N0E0 +... ok +test headerForEachSuccess_permW0N0E0 +... ok +test envFailure_permW0N0E0 +... ok +test filesStdioFileDescriptors_permW0N0E0 +... ok +test filesCopyToStdout_permW0N0E0 +{ + "name": "deno", + "devDependencies": { + "@types/base64-js": "^1.2.5", + "@types/flatbuffers": "^1.9.0", + "@types/source-map-support": "^0.4.1", + "@types/text-encoding": "0.0.33", + "base64-js": "^1.3.0", + "flatbuffers": "^1.9.0", + "magic-string": "^0.22.5", + "prettier": "^1.14.0", + "rollup": "^0.63.2", + "rollup-plugin-alias": "^1.4.0", + "rollup-plugin-analyzer": "^2.1.0", + "rollup-plugin-commonjs": "^9.1.3", + "rollup-plugin-node-globals": "^1.2.1", + "rollup-plugin-node-resolve": "^3.3.0", + "rollup-plugin-string": "^2.0.2", + "rollup-plugin-typescript2": "^0.16.1", + "rollup-pluginutils": "^2.3.0", + "source-map-support": "^0.5.6", + "text-encoding": "0.6.4", + "tslint": "^5.10.0", + "tslint-eslint-rules": "^5.3.1", + "tslint-no-circular-imports": "^0.5.0", + "typescript": "3.0.3" + } +} +bytes written 860 +... ok +test readFileSyncSuccess_permW0N0E0 +... ok +test readFileSyncNotFound_permW0N0E0 +... ok +test readFileSuccess_permW0N0E0 +... ok +test readDirSyncNotDir_permW0N0E0 +... ok +test readDirSyncNotFound_permW0N0E0 +... ok +test writeFileSyncPerm_permW0N0E0 +... ok +test writeFilePerm_permW0N0E0 +... ok +test copyFileSyncPerm_permW0N0E0 +... ok +test copyFilePerm_permW0N0E0 +... ok +test mkdirSyncPerm_permW0N0E0 +... ok +test makeTempDirSyncPerm_permW0N0E0 +... ok +test statSyncSuccess_permW0N0E0 +... ok +test statSyncNotFound_permW0N0E0 +... ok +test lstatSyncSuccess_permW0N0E0 +... ok +test lstatSyncNotFound_permW0N0E0 +... ok +test statSuccess_permW0N0E0 +... ok +test statNotFound_permW0N0E0 +... ok +test lstatSuccess_permW0N0E0 +... ok +test lstatNotFound_permW0N0E0 +... ok +test renameSyncPerm_permW0N0E0 +... ok +test readlinkSyncNotFound_permW0N0E0 +... ok +test blobString_permW0N0E0 +... ok +test blobBuffer_permW0N0E0 +... ok +test blobSlice_permW0N0E0 +... ok +test timeoutSuccess_permW0N0E0 +... ok +test timeoutArgs_permW0N0E0 +... ok +test timeoutCancelSuccess_permW0N0E0 +... ok +test timeoutCancelMultiple_permW0N0E0 +... ok +test timeoutCancelInvalidSilentFail_permW0N0E0 +... ok +test intervalSuccess_permW0N0E0 +... ok +test intervalCancelSuccess_permW0N0E0 +... ok +test intervalOrdering_permW0N0E0 +... ok +test intervalCancelInvalidSilentFail_permW0N0E0 +... ok +test symlinkSyncPerm_permW0N0E0 +... ok +test platformTransform_permW0N0E0 +... ok +test atobSuccess_permW0N0E0 +... ok +test btoaSuccess_permW0N0E0 +... ok +test btoaFailed_permW0N0E0 +... ok +test truncateSyncPerm_permW0N0E0 +... ok +test truncatePerm_permW0N0E0 +... ok +test evalErrorFormatted_permW0N0E0 +... ok +test createExecTimeColumnsRegularData_permW0N0E0 +... ok +test createExecTimeColumnsIrregularData_permW0N0E0 +... ok +test createBinarySizeColumnsRegularData_permW0N0E0 +... ok +test createBinarySizeColumnsIrregularData_permW0N0E0 +... ok +test createThreadCountColumnsRegularData_permW0N0E0 +... ok +test createThreadCountColumnsIrregularData_permW0N0E0 +... ok +test createSyscallCountColumnsRegularData_permW0N0E0 +... ok +test createSyscallCountColumnsIrregularData_permW0N0E0 +... ok +test createSha1ListRegularData_permW0N0E0 +... ok +test formatBytesPatterns_permW0N0E0 +... ok +test formatSecondsPatterns_permW0N0E0 +... ok +test getTravisDataSuccess_permW0N0E0 +... ok + +test result: ok. 96 passed; 0 failed; 0 ignored; 0 measured; 36 filtered out + -- cgit v1.2.3