def test_issue768(text, expected_tokens):
    """Allow zero-width 'infix' token during the tokenization process."""
    SPLIT_INFIX = r"(?<=[{a}]\')(?=[{a}])".format(a=ALPHA)

    class FrenchTest(Language):
        class Defaults(BaseDefaults):
            infixes = TOKENIZER_INFIXES + [SPLIT_INFIX]

    fr_tokenizer_w_infix = FrenchTest().tokenizer
    tokens = fr_tokenizer_w_infix(text)
    assert len(tokens) == 2
    assert [t.text for t in tokens] == expected_tokens
Allow zero-width 'infix' token during the tokenization process.
spacy/tests/lang/fr/test_prefix_suffix_infix.py