src/vs/workbench/services/textMate/browser/textMateWorker.ts/TextMateWorkerModel/_ensureTokens
class TextMateWorkerModel:	private _ensureTokens(): void {
		if (!this._tokenizationStateStore) {
			return;
		}
		const builder = new ContiguousMultilineTokensBuilder();
		const lineCount = this._lines.length;

		// Validate all states up to and including endLineIndex
		for (let lineIndex = this._tokenizationStateStore.invalidLineStartIndex; lineIndex < lineCount; lineIndex++) {
			const text = this._lines[lineIndex];
			const lineStartState = this._tokenizationStateStore.getBeginState(lineIndex);

			const r = this._tokenizationStateStore.tokenizationSupport.tokenizeEncoded(text, true, lineStartState!);
			LineTokens.convertToEndOffset(r.tokens, text.length);
			builder.add(lineIndex + 1, r.tokens);
			this._tokenizationStateStore.setEndState(lineCount, lineIndex, r.endState);
			lineIndex = this._tokenizationStateStore.invalidLineStartIndex - 1; // -1 because the outer loop increments it
		}

		this._worker._setTokens(this._uri, this._versionId, builder.serialize());
	}