src/vs/editor/common/model/tokenizationTextModelPart.ts/TokenizationTextModelPart/setTokens
class TokenizationTextModelPart:	public setTokens(
		tokens: ContiguousMultilineTokens[],
		backgroundTokenizationCompleted: boolean = false
	): void {
		if (tokens.length !== 0) {
			const ranges: { fromLineNumber: number; toLineNumber: number }[] = [];

			for (let i = 0, len = tokens.length; i < len; i++) {
				const element = tokens[i];
				let minChangedLineNumber = 0;
				let maxChangedLineNumber = 0;
				let hasChange = false;
				for (
					let lineNumber = element.startLineNumber;
					lineNumber <= element.endLineNumber;
					lineNumber++
				) {
					if (hasChange) {
						this._tokens.setTokens(
							this._languageId,
							lineNumber - 1,
							this._textModel.getLineLength(lineNumber),
							element.getLineTokens(lineNumber),
							false
						);
						maxChangedLineNumber = lineNumber;
					} else {
						const lineHasChange = this._tokens.setTokens(
							this._languageId,
							lineNumber - 1,
							this._textModel.getLineLength(lineNumber),
							element.getLineTokens(lineNumber),
							true
						);
						if (lineHasChange) {
							hasChange = true;
							minChangedLineNumber = lineNumber;
							maxChangedLineNumber = lineNumber;
						}
					}
				}
				if (hasChange) {
					ranges.push({
						fromLineNumber: minChangedLineNumber,
						toLineNumber: maxChangedLineNumber,
					});
				}
			}

			if (ranges.length > 0) {
				this._emitModelTokensChangedEvent({
					tokenizationSupportChanged: false,
					semanticTokensApplied: false,
					ranges: ranges,
				});
			}
		}
		this.handleTokenizationProgress(backgroundTokenizationCompleted);
	}