def convert_to_unicode(input_text, encoding="utf-8"):
    """Converts intput_text to Unicode. Input must be utf-8."""
    if isinstance(input_text, str):
        return input_text
    elif isinstance(input_text, bytes):
        return input_text.decode(encoding, "ignore")
Converts intput_text to Unicode. Input must be utf-8.
utils_nlp/dataset/preprocess.py