src/vs/editor/common/services/semanticTokensProviderStyling.ts/toMultilineTokens2
export function toMultilineTokens2(tokens: SemanticTokens, styling: SemanticTokensProviderStyling, languageId: string): SparseMultilineTokens[] {
	const srcData = tokens.data;
	const tokenCount = (tokens.data.length / 5) | 0;
	const tokensPerArea = Math.max(Math.ceil(tokenCount / SemanticColoringConstants.DesiredMaxAreas), SemanticColoringConstants.DesiredTokensPerArea);
	const result: SparseMultilineTokens[] = [];

	let tokenIndex = 0;
	let lastLineNumber = 1;
	let lastStartCharacter = 0;
	while (tokenIndex < tokenCount) {
		const tokenStartIndex = tokenIndex;
		let tokenEndIndex = Math.min(tokenStartIndex + tokensPerArea, tokenCount);

		// Keep tokens on the same line in the same area...
		if (tokenEndIndex < tokenCount) {

			let smallTokenEndIndex = tokenEndIndex;
			while (smallTokenEndIndex - 1 > tokenStartIndex && srcData[5 * smallTokenEndIndex] === 0) {
				smallTokenEndIndex--;
			}

			if (smallTokenEndIndex - 1 === tokenStartIndex) {
				// there are so many tokens on this line that our area would be empty, we must now go right
				let bigTokenEndIndex = tokenEndIndex;
				while (bigTokenEndIndex + 1 < tokenCount && srcData[5 * bigTokenEndIndex] === 0) {
					bigTokenEndIndex++;
				}
				tokenEndIndex = bigTokenEndIndex;
			} else {
				tokenEndIndex = smallTokenEndIndex;
			}
		}

		let destData = new Uint32Array((tokenEndIndex - tokenStartIndex) * 4);
		let destOffset = 0;
		let areaLine = 0;
		let prevLineNumber = 0;
		let prevStartCharacter = 0;
		let prevEndCharacter = 0;
		while (tokenIndex < tokenEndIndex) {
			const srcOffset = 5 * tokenIndex;
			const deltaLine = srcData[srcOffset];
			const deltaCharacter = srcData[srcOffset + 1];
			// Casting both `lineNumber` and `startCharacter` here to uint32 using `|0`
			// to do checks below with the actual value that will be inserted in the Uint32Array result
			const lineNumber = (lastLineNumber + deltaLine) | 0;
			const startCharacter = (deltaLine === 0 ? (lastStartCharacter + deltaCharacter) | 0 : deltaCharacter);
			const length = srcData[srcOffset + 2];
			const tokenTypeIndex = srcData[srcOffset + 3];
			const tokenModifierSet = srcData[srcOffset + 4];
			const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet, languageId);

			if (metadata !== SemanticTokensProviderStylingConstants.NO_STYLING) {
				if (areaLine === 0) {
					areaLine = lineNumber;
				}
				if (prevLineNumber === lineNumber && prevEndCharacter > startCharacter) {
					styling.warnOverlappingSemanticTokens(lineNumber, startCharacter + 1);
					if (prevStartCharacter < startCharacter) {
						// the previous token survives after the overlapping one
						destData[destOffset - 4 + 2] = startCharacter;
					} else {
						// the previous token is entirely covered by the overlapping one
						destOffset -= 4;
					}
				}
				destData[destOffset] = lineNumber - areaLine;
				destData[destOffset + 1] = startCharacter;
				destData[destOffset + 2] = startCharacter + length;
				destData[destOffset + 3] = metadata;
				destOffset += 4;

				prevLineNumber = lineNumber;
				prevStartCharacter = startCharacter;
				prevEndCharacter = startCharacter + length;
			}

			lastLineNumber = lineNumber;
			lastStartCharacter = startCharacter;
			tokenIndex++;
		}

		if (destOffset !== destData.length) {
			destData = destData.subarray(0, destOffset);
		}

		const tokens = SparseMultilineTokens.create(areaLine, destData);
		result.push(tokens);
	}

	return result;
}