@Override
    public String[] split(String text) {
        for (Pattern regexp : CONTRACTIONS2)
            text = regexp.matcher(text).replaceAll("$1 $2");

        for (Pattern regexp : CONTRACTIONS3)
            text = regexp.matcher(text).replaceAll("$1 $2 $3");

        text = DELIMITERS[0].matcher(text).replaceAll(" $1 ");
        text = DELIMITERS[1].matcher(text).replaceAll(" $1");
        text = DELIMITERS[2].matcher(text).replaceAll(" $1");
        text = DELIMITERS[3].matcher(text).replaceAll(" . ");

        String[] words = WHITESPACE.split(text);
        if (words.length > 1 && words[words.length-1].equals(".")) {
            if (EnglishAbbreviations.contains(words[words.length-2])) {
                words[words.length-2] = words[words.length-2] + ".";
            }
        }

        return words;
    }
@Test
    public void testTokenizeNonLatinChars() {
        System.out.println("tokenize words containing non-Latin chars");
        // See https://en.wikipedia.org/wiki/Zero-width_non-joiner
        String text = "می‌خواهم   עֲו‌ֹנֹת   Auf‌lage";
        String[] expResult = {"می‌خواهم", "עֲו‌ֹנֹת", "Auf‌lage"};

        SimpleTokenizer instance = new SimpleTokenizer();
        String[] result = instance.split(text);

        assertEquals(expResult.length, result.length);
        for (int i = 0; i < result.length; i++) {
            assertEquals(expResult[i], result[i]);
        }
    }