Buckets:
| import { TokenType as tt} from "./parser/tokenizer/types"; | |
| import isAsyncOperation from "./util/isAsyncOperation"; | |
| export default class TokenProcessor { | |
| __init() {this.resultCode = ""} | |
| // Array mapping input token index to optional string index position in the | |
| // output code. | |
| __init2() {this.resultMappings = new Array(this.tokens.length)} | |
| __init3() {this.tokenIndex = 0} | |
| constructor( | |
| code, | |
| tokens, | |
| isFlowEnabled, | |
| disableESTransforms, | |
| helperManager, | |
| ) {;this.code = code;this.tokens = tokens;this.isFlowEnabled = isFlowEnabled;this.disableESTransforms = disableESTransforms;this.helperManager = helperManager;TokenProcessor.prototype.__init.call(this);TokenProcessor.prototype.__init2.call(this);TokenProcessor.prototype.__init3.call(this);} | |
| /** | |
| * Snapshot the token state in a way that can be restored later, useful for | |
| * things like lookahead. | |
| * | |
| * resultMappings do not need to be copied since in all use cases, they will | |
| * be overwritten anyway after restore. | |
| */ | |
| snapshot() { | |
| return { | |
| resultCode: this.resultCode, | |
| tokenIndex: this.tokenIndex, | |
| }; | |
| } | |
| restoreToSnapshot(snapshot) { | |
| this.resultCode = snapshot.resultCode; | |
| this.tokenIndex = snapshot.tokenIndex; | |
| } | |
| /** | |
| * Remove and return the code generated since the snapshot, leaving the | |
| * current token position in-place. Unlike most TokenProcessor operations, | |
| * this operation can result in input/output line number mismatches because | |
| * the removed code may contain newlines, so this operation should be used | |
| * sparingly. | |
| */ | |
| dangerouslyGetAndRemoveCodeSinceSnapshot(snapshot) { | |
| const result = this.resultCode.slice(snapshot.resultCode.length); | |
| this.resultCode = snapshot.resultCode; | |
| return result; | |
| } | |
| reset() { | |
| this.resultCode = ""; | |
| this.resultMappings = new Array(this.tokens.length); | |
| this.tokenIndex = 0; | |
| } | |
| matchesContextualAtIndex(index, contextualKeyword) { | |
| return ( | |
| this.matches1AtIndex(index, tt.name) && | |
| this.tokens[index].contextualKeyword === contextualKeyword | |
| ); | |
| } | |
| identifierNameAtIndex(index) { | |
| // TODO: We need to process escapes since technically you can have unicode escapes in variable | |
| // names. | |
| return this.identifierNameForToken(this.tokens[index]); | |
| } | |
| identifierNameAtRelativeIndex(relativeIndex) { | |
| return this.identifierNameForToken(this.tokenAtRelativeIndex(relativeIndex)); | |
| } | |
| identifierName() { | |
| return this.identifierNameForToken(this.currentToken()); | |
| } | |
| identifierNameForToken(token) { | |
| return this.code.slice(token.start, token.end); | |
| } | |
| rawCodeForToken(token) { | |
| return this.code.slice(token.start, token.end); | |
| } | |
| stringValueAtIndex(index) { | |
| return this.stringValueForToken(this.tokens[index]); | |
| } | |
| stringValue() { | |
| return this.stringValueForToken(this.currentToken()); | |
| } | |
| stringValueForToken(token) { | |
| // This is used to identify when two imports are the same and to resolve TypeScript enum keys. | |
| // Ideally we'd process escapes within the strings, but for now we pretty much take the raw | |
| // code. | |
| return this.code.slice(token.start + 1, token.end - 1); | |
| } | |
| matches1AtIndex(index, t1) { | |
| return this.tokens[index].type === t1; | |
| } | |
| matches2AtIndex(index, t1, t2) { | |
| return this.tokens[index].type === t1 && this.tokens[index + 1].type === t2; | |
| } | |
| matches3AtIndex(index, t1, t2, t3) { | |
| return ( | |
| this.tokens[index].type === t1 && | |
| this.tokens[index + 1].type === t2 && | |
| this.tokens[index + 2].type === t3 | |
| ); | |
| } | |
| matches1(t1) { | |
| return this.tokens[this.tokenIndex].type === t1; | |
| } | |
| matches2(t1, t2) { | |
| return this.tokens[this.tokenIndex].type === t1 && this.tokens[this.tokenIndex + 1].type === t2; | |
| } | |
| matches3(t1, t2, t3) { | |
| return ( | |
| this.tokens[this.tokenIndex].type === t1 && | |
| this.tokens[this.tokenIndex + 1].type === t2 && | |
| this.tokens[this.tokenIndex + 2].type === t3 | |
| ); | |
| } | |
| matches4(t1, t2, t3, t4) { | |
| return ( | |
| this.tokens[this.tokenIndex].type === t1 && | |
| this.tokens[this.tokenIndex + 1].type === t2 && | |
| this.tokens[this.tokenIndex + 2].type === t3 && | |
| this.tokens[this.tokenIndex + 3].type === t4 | |
| ); | |
| } | |
| matches5(t1, t2, t3, t4, t5) { | |
| return ( | |
| this.tokens[this.tokenIndex].type === t1 && | |
| this.tokens[this.tokenIndex + 1].type === t2 && | |
| this.tokens[this.tokenIndex + 2].type === t3 && | |
| this.tokens[this.tokenIndex + 3].type === t4 && | |
| this.tokens[this.tokenIndex + 4].type === t5 | |
| ); | |
| } | |
| matchesContextual(contextualKeyword) { | |
| return this.matchesContextualAtIndex(this.tokenIndex, contextualKeyword); | |
| } | |
| matchesContextIdAndLabel(type, contextId) { | |
| return this.matches1(type) && this.currentToken().contextId === contextId; | |
| } | |
| previousWhitespaceAndComments() { | |
| let whitespaceAndComments = this.code.slice( | |
| this.tokenIndex > 0 ? this.tokens[this.tokenIndex - 1].end : 0, | |
| this.tokenIndex < this.tokens.length ? this.tokens[this.tokenIndex].start : this.code.length, | |
| ); | |
| if (this.isFlowEnabled) { | |
| whitespaceAndComments = whitespaceAndComments.replace(/@flow/g, ""); | |
| } | |
| return whitespaceAndComments; | |
| } | |
| replaceToken(newCode) { | |
| this.resultCode += this.previousWhitespaceAndComments(); | |
| this.appendTokenPrefix(); | |
| this.resultMappings[this.tokenIndex] = this.resultCode.length; | |
| this.resultCode += newCode; | |
| this.appendTokenSuffix(); | |
| this.tokenIndex++; | |
| } | |
| replaceTokenTrimmingLeftWhitespace(newCode) { | |
| this.resultCode += this.previousWhitespaceAndComments().replace(/[^\r\n]/g, ""); | |
| this.appendTokenPrefix(); | |
| this.resultMappings[this.tokenIndex] = this.resultCode.length; | |
| this.resultCode += newCode; | |
| this.appendTokenSuffix(); | |
| this.tokenIndex++; | |
| } | |
| removeInitialToken() { | |
| this.replaceToken(""); | |
| } | |
| removeToken() { | |
| this.replaceTokenTrimmingLeftWhitespace(""); | |
| } | |
| /** | |
| * Remove all code until the next }, accounting for balanced braces. | |
| */ | |
| removeBalancedCode() { | |
| let braceDepth = 0; | |
| while (!this.isAtEnd()) { | |
| if (this.matches1(tt.braceL)) { | |
| braceDepth++; | |
| } else if (this.matches1(tt.braceR)) { | |
| if (braceDepth === 0) { | |
| return; | |
| } | |
| braceDepth--; | |
| } | |
| this.removeToken(); | |
| } | |
| } | |
| copyExpectedToken(tokenType) { | |
| if (this.tokens[this.tokenIndex].type !== tokenType) { | |
| throw new Error(`Expected token ${tokenType}`); | |
| } | |
| this.copyToken(); | |
| } | |
| copyToken() { | |
| this.resultCode += this.previousWhitespaceAndComments(); | |
| this.appendTokenPrefix(); | |
| this.resultMappings[this.tokenIndex] = this.resultCode.length; | |
| this.resultCode += this.code.slice( | |
| this.tokens[this.tokenIndex].start, | |
| this.tokens[this.tokenIndex].end, | |
| ); | |
| this.appendTokenSuffix(); | |
| this.tokenIndex++; | |
| } | |
| copyTokenWithPrefix(prefix) { | |
| this.resultCode += this.previousWhitespaceAndComments(); | |
| this.appendTokenPrefix(); | |
| this.resultCode += prefix; | |
| this.resultMappings[this.tokenIndex] = this.resultCode.length; | |
| this.resultCode += this.code.slice( | |
| this.tokens[this.tokenIndex].start, | |
| this.tokens[this.tokenIndex].end, | |
| ); | |
| this.appendTokenSuffix(); | |
| this.tokenIndex++; | |
| } | |
| appendTokenPrefix() { | |
| const token = this.currentToken(); | |
| if (token.numNullishCoalesceStarts || token.isOptionalChainStart) { | |
| token.isAsyncOperation = isAsyncOperation(this); | |
| } | |
| if (this.disableESTransforms) { | |
| return; | |
| } | |
| if (token.numNullishCoalesceStarts) { | |
| for (let i = 0; i < token.numNullishCoalesceStarts; i++) { | |
| if (token.isAsyncOperation) { | |
| this.resultCode += "await "; | |
| this.resultCode += this.helperManager.getHelperName("asyncNullishCoalesce"); | |
| } else { | |
| this.resultCode += this.helperManager.getHelperName("nullishCoalesce"); | |
| } | |
| this.resultCode += "("; | |
| } | |
| } | |
| if (token.isOptionalChainStart) { | |
| if (token.isAsyncOperation) { | |
| this.resultCode += "await "; | |
| } | |
| if (this.tokenIndex > 0 && this.tokenAtRelativeIndex(-1).type === tt._delete) { | |
| if (token.isAsyncOperation) { | |
| this.resultCode += this.helperManager.getHelperName("asyncOptionalChainDelete"); | |
| } else { | |
| this.resultCode += this.helperManager.getHelperName("optionalChainDelete"); | |
| } | |
| } else if (token.isAsyncOperation) { | |
| this.resultCode += this.helperManager.getHelperName("asyncOptionalChain"); | |
| } else { | |
| this.resultCode += this.helperManager.getHelperName("optionalChain"); | |
| } | |
| this.resultCode += "(["; | |
| } | |
| } | |
| appendTokenSuffix() { | |
| const token = this.currentToken(); | |
| if (token.isOptionalChainEnd && !this.disableESTransforms) { | |
| this.resultCode += "])"; | |
| } | |
| if (token.numNullishCoalesceEnds && !this.disableESTransforms) { | |
| for (let i = 0; i < token.numNullishCoalesceEnds; i++) { | |
| this.resultCode += "))"; | |
| } | |
| } | |
| } | |
| appendCode(code) { | |
| this.resultCode += code; | |
| } | |
| currentToken() { | |
| return this.tokens[this.tokenIndex]; | |
| } | |
| currentTokenCode() { | |
| const token = this.currentToken(); | |
| return this.code.slice(token.start, token.end); | |
| } | |
| tokenAtRelativeIndex(relativeIndex) { | |
| return this.tokens[this.tokenIndex + relativeIndex]; | |
| } | |
| currentIndex() { | |
| return this.tokenIndex; | |
| } | |
| /** | |
| * Move to the next token. Only suitable in preprocessing steps. When | |
| * generating new code, you should use copyToken or removeToken. | |
| */ | |
| nextToken() { | |
| if (this.tokenIndex === this.tokens.length) { | |
| throw new Error("Unexpectedly reached end of input."); | |
| } | |
| this.tokenIndex++; | |
| } | |
| previousToken() { | |
| this.tokenIndex--; | |
| } | |
| finish() { | |
| if (this.tokenIndex !== this.tokens.length) { | |
| throw new Error("Tried to finish processing tokens before reaching the end."); | |
| } | |
| this.resultCode += this.previousWhitespaceAndComments(); | |
| return {code: this.resultCode, mappings: this.resultMappings}; | |
| } | |
| isAtEnd() { | |
| return this.tokenIndex === this.tokens.length; | |
| } | |
| } | |
Xet Storage Details
- Size:
- 10.2 kB
- Xet hash:
- d2b1e91cf4af089026a8e7f24d4f2087bde73fed023d718766e835eb8312a2a3
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.