src/vs/workbench/services/textMate/common/TMTokenization.ts/TMTokenization/tokenizeEncoded
class TMTokenization:	public tokenizeEncoded(line: string, hasEOL: boolean, state: StackElement): EncodedTokenizationResult {
		const textMateResult = this._grammar.tokenizeLine2(line, state, 500);

		if (textMateResult.stoppedEarly) {
			console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`);
			// return the state at the beginning of the line
			return new EncodedTokenizationResult(textMateResult.tokens, state);
		}

		if (this._containsEmbeddedLanguages) {
			let seenLanguages = this._seenLanguages;
			let tokens = textMateResult.tokens;

			// Must check if any of the embedded languages was hit
			for (let i = 0, len = (tokens.length >>> 1); i < len; i++) {
				let metadata = tokens[(i << 1) + 1];
				let languageId = TokenMetadata.getLanguageId(metadata);

				if (!seenLanguages[languageId]) {
					seenLanguages[languageId] = true;
					this._onDidEncounterLanguage.fire(languageId);
				}
			}
		}

		let endState: StackElement;
		// try to save an object if possible
		if (state.equals(textMateResult.ruleStack)) {
			endState = state;
		} else {
			endState = textMateResult.ruleStack;

		}

		return new EncodedTokenizationResult(textMateResult.tokens, endState);
	}