def __init__(self, length_function: Callable[[str], int] =TOKEN_ESTIMATOR.estimate_tokens, separator: str = "\n\n", **kwargs: Any):
        """Create a new TextSplitter for htmls from extracted pdfs."""
        super().__init__(**kwargs)
        self._table_tags = HTML_TABLE_TAGS
        self._separators = separator or ["\n\n", "\n", " ", ""]
        self._length_function = length_function
        self._noise = 50 # tokens to accommodate differences in token calculation, we don't want the chunking-on-the-fly to inadvertently chunk anything due to token calc mismatch
Create a new TextSplitter for htmls from extracted pdfs.
scripts/data_utils.py