def get_tokenized_datasets(
        self,
        datasets: DatasetDict,
        text_columns: Sequence[str] = ("sentence",),
        **kwargs,
    ) -> DatasetDict:
        """
        Converts one or more dataset into sequences of tokens, using the tokenizer.

        Parameters
        ----------
        datasets: DatasetDict
            A dictionary of `datasets.Dataset` that have to be encoded.
        text_columns: str
            A list containing the text column names, whose text sequences has to be encoded.

        Returns
        -------
        DatasetDict
            A dictionary of tokenized datasets.
        """
        assert (
            len(text_columns) == 1
        ), "Only one text column should be passed when the task is MaskedLM."

        def _tokenize_fn(
            batch: Dict[str, List[Union[str, int]]],
        ) -> Dict[str, List[int]]:
            tokenized_inputs = self.tokenizer(
                *[batch[col] for col in text_columns],
                truncation=False,
                return_special_tokens_mask=True,
            )
            # Concatenate all texts
            keys = list(tokenized_inputs.keys())
            tokenized_inputs_cat = {k: sum(tokenized_inputs[k], []) for k in keys}
            # Compute length of concatenated texts
            total_length = len(tokenized_inputs_cat[keys[0]])
            # We drop the last chunk if it's smaller than chunk_size
            total_length = (total_length // self.max_length) * self.max_length
            # Split by chunks of max_len
            tokenized_inputs = {
                k: [
                    t[i : i + self.max_length]
                    for i in range(0, total_length, self.max_length)
                ]
                for k, t in tokenized_inputs_cat.items()
            }
            return tokenized_inputs

        tokenized_datasets = datasets.map(
            _tokenize_fn,
            batched=True,
            remove_columns=datasets[list(datasets.keys())[0]].column_names,
        )
        return tokenized_datasets

        Converts one or more dataset into sequences of tokens, using the tokenizer.

        Parameters
        ----------
        datasets: DatasetDict
            A dictionary of `datasets.Dataset` that have to be encoded.
        text_columns: str
            A list containing the text column names, whose text sequences has to be encoded.

        Returns
        -------
        DatasetDict
            A dictionary of tokenized datasets.
        
fortuna/data/dataset/huggingface_datasets.py