Skip to content

Commit fb0ba8c

Browse files
paldepindpaldepind
authored
Merge pull request #22860 from github/js/typescript-cleanup
JS: TypeScript extractor cleanup
2 parents 00fd5ae + 0610208 commit fb0ba8c

4 files changed

Lines changed: 5 additions & 357 deletions

File tree

‎javascript/extractor/lib/typescript/src/ast_extractor.ts‎

Lines changed: 3 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import * as ts from "./typescript";
2-
import { Project } from "./common";
32

43
/**
54
* Interface that exposes some internal properties we rely on, as well as
@@ -27,27 +26,6 @@ export interface Token {
2726
text: string;
2827
}
2928

30-
function hasOwnProperty(o: object, p: string) {
31-
return o && Object.prototype.hasOwnProperty.call(o, p);
32-
}
33-
34-
// build our own copy of `ts.SyntaxKind` without `First*`/`Last*` markers
35-
let SyntaxKind: { [id: number]: string } = [];
36-
for (let p in ts.SyntaxKind) {
37-
if (!hasOwnProperty(ts.SyntaxKind, p)) {
38-
continue;
39-
}
40-
// skip numeric indices
41-
if (+p === +p) {
42-
continue;
43-
}
44-
// skip `First*`/`Last*`
45-
if (p.substring(0, 5) === "First" || p.substring(0, 4) === "Last") {
46-
continue;
47-
}
48-
SyntaxKind[ts.SyntaxKind[p] as any] = p;
49-
}
50-
5129
let skipWhiteSpace = /(?:\s|\/\/.*|\/\*[^]*?\*\/)*/g;
5230

5331
/**
@@ -61,18 +39,7 @@ function forEachNode(ast: ts.Node, callback: (node: ts.Node) => void) {
6139
visit(ast);
6240
}
6341

64-
function tryGetTypeOfNode(typeChecker: ts.TypeChecker, node: AugmentedNode): ts.Type | null {
65-
try {
66-
return typeChecker.getTypeAtLocation(node);
67-
} catch (e) {
68-
let sourceFile = node.getSourceFile();
69-
let { line, character } = sourceFile.getLineAndCharacterOfPosition(node.pos);
70-
console.warn(`Could not compute type of ${ts.SyntaxKind[node.kind]} at ${sourceFile.fileName}:${line + 1}:${character + 1}`);
71-
return null;
72-
}
73-
}
74-
75-
export function augmentAst(ast: AugmentedSourceFile, code: string, project: Project | null) {
42+
export function augmentAst(ast: AugmentedSourceFile, code: string) {
7643
ast.$lineStarts = ast.getLineStarts();
7744

7845
/**
@@ -164,10 +131,9 @@ export function augmentAst(ast: AugmentedSourceFile, code: string, project: Proj
164131
}
165132

166133
if (ts.isVariableDeclarationList(node as any)) {
167-
let tz = ts as any;
168-
if (typeof tz.isLet === "function" && tz.isLet(node) || (node.flags & ts.NodeFlags.Let)) {
134+
if (node.flags & ts.NodeFlags.Let) {
169135
node.$declarationKind = "let";
170-
} else if (typeof tz.isConst === "function" && tz.isConst(node) || (node.flags & ts.NodeFlags.Const)) {
136+
} else if (node.flags & ts.NodeFlags.Const) {
171137
node.$declarationKind = "const";
172138
} else {
173139
node.$declarationKind = "var";

‎javascript/extractor/lib/typescript/src/common.ts‎

Lines changed: 0 additions & 35 deletions
This file was deleted.

‎javascript/extractor/lib/typescript/src/main.ts‎

Lines changed: 2 additions & 216 deletions
Original file line numberDiff line numberDiff line change
@@ -36,23 +36,13 @@ import * as readline from "readline";
3636
import * as ts from "./typescript";
3737
import * as ast_extractor from "./ast_extractor";
3838

39-
import { Project } from "./common";
40-
import { VirtualSourceRoot } from "./virtual_source_root";
41-
4239
// Remove limit on stack trace depth.
4340
Error.stackTraceLimit = Infinity;
4441

4542
interface ParseCommand {
4643
command: "parse";
4744
filename: string;
4845
}
49-
interface LoadCommand {
50-
tsConfig: string;
51-
sourceRoot: string | null;
52-
virtualSourceRoot: string | null;
53-
packageEntryPoints: [string, string][];
54-
packageJsonFiles: [string, string][];
55-
}
5646
interface ResetCommand {
5747
command: "reset";
5848
}
@@ -70,93 +60,15 @@ type Command = ParseCommand | ResetCommand | QuitCommand | PrepareFilesCommand |
7060

7161
/** The state to be shared between commands. */
7262
class State {
73-
public project: Project = null;
74-
7563
/** List of files that have been requested. */
7664
public pendingFiles: string[] = [];
7765
public pendingFileIndex = 0;
7866

7967
/** Next response to be delivered. */
8068
public pendingResponse: string = null;
81-
82-
/** Map from `package.json` files to their contents. */
83-
public parsedPackageJson = new Map<string, any>();
84-
85-
/** Map from `package.json` files to the file referenced in its `types` or `typings` field. */
86-
public packageTypings = new Map<string, string | undefined>();
87-
88-
/** Map from file path to the enclosing `package.json` file, if any. Will not traverse outside node_modules. */
89-
public enclosingPackageJson = new Map<string, string | undefined>();
9069
}
9170
let state = new State();
9271

93-
const reloadMemoryThresholdMb = getEnvironmentVariable("SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD", Number, 1000);
94-
95-
function getPackageJson(file: string): any {
96-
let cache = state.parsedPackageJson;
97-
if (cache.has(file)) return cache.get(file);
98-
let result = getPackageJsonRaw(file);
99-
cache.set(file, result);
100-
return result;
101-
}
102-
103-
function getPackageJsonRaw(file: string): any {
104-
if (!ts.sys.fileExists(file)) return undefined;
105-
try {
106-
let json = JSON.parse(ts.sys.readFile(file));
107-
if (typeof json !== 'object') return undefined;
108-
return json;
109-
} catch (e) {
110-
return undefined;
111-
}
112-
}
113-
114-
function getPackageTypings(file: string): string | undefined {
115-
let cache = state.packageTypings;
116-
if (cache.has(file)) return cache.get(file);
117-
let result = getPackageTypingsRaw(file);
118-
cache.set(file, result);
119-
return result;
120-
}
121-
122-
function getPackageTypingsRaw(packageJsonFile: string): string | undefined {
123-
let json = getPackageJson(packageJsonFile);
124-
if (json == null) return undefined;
125-
let typings = json.types || json.typings; // "types" and "typings" are aliases
126-
if (typeof typings !== 'string') return undefined;
127-
let absolutePath = pathlib.join(pathlib.dirname(packageJsonFile), typings);
128-
if (ts.sys.directoryExists(absolutePath)) {
129-
absolutePath = pathlib.join(absolutePath, 'index.d.ts');
130-
} else if (!absolutePath.endsWith('.ts')) {
131-
absolutePath += '.d.ts';
132-
}
133-
if (!ts.sys.fileExists(absolutePath)) return undefined;
134-
return ts.sys.resolvePath(absolutePath);
135-
}
136-
137-
function getEnclosingPackageJson(file: string): string | undefined {
138-
let cache = state.packageTypings;
139-
if (cache.has(file)) return cache.get(file);
140-
let result = getEnclosingPackageJsonRaw(file);
141-
cache.set(file, result);
142-
return result;
143-
}
144-
145-
function getEnclosingPackageJsonRaw(file: string): string | undefined {
146-
let packageJson = pathlib.join(file, 'package.json');
147-
if (ts.sys.fileExists(packageJson)) {
148-
return packageJson;
149-
}
150-
if (pathlib.basename(file) === 'node_modules') {
151-
return undefined;
152-
}
153-
let dirname = pathlib.dirname(file);
154-
if (dirname.length < file.length) {
155-
return getEnclosingPackageJson(dirname);
156-
}
157-
return undefined;
158-
}
159-
16072
/**
16173
* Debugging method for finding cycles in the TypeScript AST. Should not be used in production.
16274
*
@@ -330,7 +242,6 @@ function extractFile(filename: string): string {
330242
function prepareNextFile() {
331243
if (state.pendingResponse != null) return;
332244
if (state.pendingFileIndex < state.pendingFiles.length) {
333-
checkMemoryUsage();
334245
let nextFilename = state.pendingFiles[state.pendingFileIndex];
335246
state.pendingResponse = extractFile(nextFilename);
336247
}
@@ -368,13 +279,12 @@ function isExtractableSourceFile(ast: ast_extractor.AugmentedSourceFile): boolea
368279
}
369280

370281
/**
371-
* Gets the AST and source code for the given file, either from
372-
* an already-open project, or by parsing the file.
282+
* Gets the AST and source code by parsing the given file.
373283
*/
374284
function getAstForFile(filename: string): ts.SourceFile {
375285
let { ast, code } = parseSingleFile(filename);
376286
if (ast != null && isExtractableSourceFile(ast)) {
377-
ast_extractor.augmentAst(ast, code, null);
287+
ast_extractor.augmentAst(ast, code);
378288
}
379289
return ast;
380290
}
@@ -401,7 +311,6 @@ function parseSingleFile(filename: string): { ast: ts.SourceFile, code: string }
401311
// parse `filename` with minimial checking
402312
let compilerOptions: ts.CompilerOptions = {
403313
experimentalDecorators: true,
404-
experimentalAsyncFunctions: true,
405314
jsx: ts.JsxEmit.Preserve,
406315
noResolve: true,
407316
};
@@ -413,98 +322,6 @@ function parseSingleFile(filename: string): { ast: ts.SourceFile, code: string }
413322
return { ast, code };
414323
}
415324

416-
/**
417-
* Matches a path segment referencing a package in a node_modules folder, and extracts
418-
* two capture groups: the package name, and the relative path in the package.
419-
*
420-
* For example `lib/node_modules/@foo/bar/src/index.js` extracts the capture groups [`@foo/bar`, `src/index.js`].
421-
*/
422-
const nodeModulesRex = /[/\\]node_modules[/\\]((?:@[\w.-]+[/\\])?\w[\w.-]*)[/\\](.*)/;
423-
424-
interface LoadedConfig {
425-
config: ts.ParsedCommandLine;
426-
basePath: string;
427-
packageEntryPoints: Map<string, string>;
428-
packageJsonFiles: Map<string, string>;
429-
virtualSourceRoot: VirtualSourceRoot;
430-
ownFiles: string[];
431-
}
432-
433-
function loadTsConfig(command: LoadCommand): LoadedConfig {
434-
let tsConfig = ts.readConfigFile(command.tsConfig, ts.sys.readFile);
435-
let basePath = pathlib.dirname(command.tsConfig);
436-
437-
let packageEntryPoints = new Map(command.packageEntryPoints);
438-
let packageJsonFiles = new Map(command.packageJsonFiles);
439-
let virtualSourceRoot = new VirtualSourceRoot(command.sourceRoot, command.virtualSourceRoot);
440-
441-
/**
442-
* Rewrites path segments of form `node_modules/PACK/suffix` to be relative to
443-
* the location of package PACK in the source tree, if it exists.
444-
*/
445-
function redirectNodeModulesPath(path: string) {
446-
let nodeModulesMatch = nodeModulesRex.exec(path);
447-
if (nodeModulesMatch == null) return null;
448-
let packageName = nodeModulesMatch[1];
449-
let packageJsonFile = packageJsonFiles.get(packageName);
450-
if (packageJsonFile == null) return null;
451-
let packageDir = pathlib.dirname(packageJsonFile);
452-
let suffix = nodeModulesMatch[2];
453-
let finalPath = pathlib.join(packageDir, suffix);
454-
if (!ts.sys.fileExists(finalPath)) return null;
455-
return finalPath;
456-
}
457-
458-
/**
459-
* Create the host passed to the tsconfig.json parser.
460-
*
461-
* We override its file system access in case there is an "extends"
462-
* clause pointing into "./node_modules", which must be redirected to
463-
* the location of an installed package or a checked-in package.
464-
*/
465-
let parseConfigHost: ts.ParseConfigHost = {
466-
useCaseSensitiveFileNames: true,
467-
readDirectory: (rootDir, extensions, excludes?, includes?, depth?) => {
468-
// Perform the glob matching in both real and virtual source roots.
469-
let exclusions = excludes == null ? [] : [...excludes];
470-
if (virtualSourceRoot.virtualSourceRoot != null) {
471-
// qltest puts the virtual source root inside the real source root (.testproj).
472-
// Make sure we don't find files inside the virtual source root in this pass.
473-
exclusions.push(virtualSourceRoot.virtualSourceRoot);
474-
}
475-
let originalResults = ts.sys.readDirectory(rootDir, extensions, exclusions, includes, depth)
476-
let virtualDir = virtualSourceRoot.toVirtualPath(rootDir);
477-
if (virtualDir == null) {
478-
return originalResults;
479-
}
480-
// Make sure glob matching does not to discover anything in node_modules.
481-
let virtualExclusions = excludes == null ? [] : [...excludes];
482-
virtualExclusions.push('**/node_modules/**/*');
483-
let virtualResults = ts.sys.readDirectory(virtualDir, extensions, virtualExclusions, includes, depth)
484-
return [...originalResults, ...virtualResults];
485-
},
486-
fileExists: (path: string) => {
487-
return ts.sys.fileExists(path)
488-
|| virtualSourceRoot.toVirtualPathIfFileExists(path) != null
489-
|| redirectNodeModulesPath(path) != null;
490-
},
491-
readFile: (path: string) => {
492-
if (!ts.sys.fileExists(path)) {
493-
let virtualPath = virtualSourceRoot.toVirtualPathIfFileExists(path);
494-
if (virtualPath != null) return ts.sys.readFile(virtualPath);
495-
virtualPath = redirectNodeModulesPath(path);
496-
if (virtualPath != null) return ts.sys.readFile(virtualPath);
497-
}
498-
return ts.sys.readFile(path);
499-
}
500-
};
501-
let config = ts.parseJsonConfigFileContent(tsConfig.config, parseConfigHost, basePath);
502-
503-
let ownFiles = config.fileNames.map(file => pathlib.resolve(file));
504-
505-
return { config, basePath, packageJsonFiles, packageEntryPoints, virtualSourceRoot, ownFiles };
506-
}
507-
508325
function handleResetCommand(command: ResetCommand) {
509326
reset();
510327
console.log(JSON.stringify({
@@ -533,37 +350,6 @@ function reset() {
533350
state = new State();
534351
}
535352

536-
function getEnvironmentVariable<T>(name: string, parse: (x: string) => T, defaultValue: T) {
537-
let value = process.env[name];
538-
return value != null ? parse(value) : defaultValue;
539-
}
540-
541-
/**
542-
* Whether the memory usage was last observed to be above the threshold for restarting the TypeScript compiler.
543-
*
544-
* This is to prevent repeatedly restarting the compiler if the GC does not immediately bring us below the
545-
* threshold again.
546-
*/
547-
let hasReloadedSinceExceedingThreshold = false;
548-
549-
/**
550-
* If memory usage has moved above a the threshold, reboot the TypeScript compiler instance.
551-
*
552-
* Make sure to call this only when stdout has been flushed.
553-
*/
554-
function checkMemoryUsage() {
555-
let bytesUsed = process.memoryUsage().heapUsed;
556-
let megabytesUsed = bytesUsed / 1000000;
557-
if (!hasReloadedSinceExceedingThreshold && megabytesUsed > reloadMemoryThresholdMb && state.project != null) {
558-
console.warn('Restarting TypeScript compiler due to memory usage');
559-
state.project.reload();
560-
hasReloadedSinceExceedingThreshold = true;
561-
}
562-
else if (hasReloadedSinceExceedingThreshold && megabytesUsed < reloadMemoryThresholdMb) {
563-
hasReloadedSinceExceedingThreshold = false;
564-
}
565-
}
566-
567353
function runReadLineInterface() {
568354
reset();
569355
let rl = readline.createInterface({ input: process.stdin, output: process.stdout });

0 commit comments

Comments
 (0)