package thridparty;

/**
 * @Author https://www.cnblogs.com/ChurchYim/p/8427373.html
 *
 * Changed by Mara
 * My updates are marked with #####.
 */
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URL;

public class EncodingDetect {

    public static int MAX_CHECK_BYTES = 20000000;  // ##### Added by Mara

    public static String detect(String path) {
        BytesEncodingDetect s = new BytesEncodingDetect();
        String fileCode = BytesEncodingDetect.javaname[s.detectEncoding(new File(path))];
        return fileCode;
    }

    public static String detect(File file) {
        BytesEncodingDetect s = new BytesEncodingDetect();
        String fileCode = BytesEncodingDetect.javaname[s.detectEncoding(file)];
        return fileCode;
    }

    public static String detect(byte[] contents) {
        BytesEncodingDetect s = new BytesEncodingDetect();
        String fileCode = BytesEncodingDetect.javaname[s.detectEncoding(contents)];
        return fileCode;
    }

    public static String detect(URL url) {
        BytesEncodingDetect s = new BytesEncodingDetect();
        String fileCode = BytesEncodingDetect.javaname[s.detectEncoding(url)];
        return fileCode;
    }

    static class BytesEncodingDetect extends Encoding {

        int GBFreq[][];

        int GBKFreq[][];

        int Big5Freq[][];

        int Big5PFreq[][];

        int EUC_TWFreq[][];

        int KRFreq[][];

        int JPFreq[][];

        public boolean debug;

        public BytesEncodingDetect() {
            super();
            debug = false;
            GBFreq = new int[94][94];
            GBKFreq = new int[126][191];
            Big5Freq = new int[94][158];
            Big5PFreq = new int[126][191];
            EUC_TWFreq = new int[94][94];
            KRFreq = new int[94][94];
            JPFreq = new int[94][94];
            initialize_frequencies();
        }

        public int detectEncoding(URL testurl) {
            byte[] rawtext = new byte[10000];
            int bytesread = 0, byteoffset = 0;
            int guess = OTHER;
            InputStream is;
            try {
                is = testurl.openStream();
                while ((bytesread = is.read(rawtext, byteoffset, rawtext.length - byteoffset)) > 0) {
                    byteoffset += bytesread;
                }
                is.close();
                guess = detectEncoding(rawtext);
            } catch (Exception e) {
                System.err.println("Error loading or using URL " + e.toString());
                guess = -1;
            }
            return guess;
        }

        public int detectEncoding(File testfile) {
            byte[] rawtext = new byte[MAX_CHECK_BYTES];  // ##### Updated by Mara
            try ( FileInputStream fileis = new FileInputStream(testfile)) {
                fileis.read(rawtext);
            } catch (Exception e) {
                return OTHER;
            }
            return detectEncoding(rawtext);
        }

        public int detectEncoding(byte[] rawtext) {
            int[] scores;
            int index, maxscore = 0;
            int encoding_guess = OTHER;
            scores = new int[TOTALTYPES];
            // Assign Scores
            scores[GB2312] = gb2312_probability(rawtext);
            scores[GBK] = gbk_probability(rawtext);
            scores[GB18030] = gb18030_probability(rawtext);
            scores[HZ] = hz_probability(rawtext);
            scores[BIG5] = big5_probability(rawtext);
            scores[CNS11643] = euc_tw_probability(rawtext);
            scores[ISO2022CN] = iso_2022_cn_probability(rawtext);
            scores[UTF8] = utf8_probability(rawtext);
            scores[UNICODE] = utf16_probability(rawtext);
            scores[EUC_KR] = euc_kr_probability(rawtext);
            scores[CP949] = cp949_probability(rawtext);
            scores[JOHAB] = 0;
            scores[ISO2022KR] = iso_2022_kr_probability(rawtext);
            scores[ASCII] = ascii_probability(rawtext);
            scores[SJIS] = sjis_probability(rawtext);
            scores[EUC_JP] = euc_jp_probability(rawtext);
            scores[ISO2022JP] = iso_2022_jp_probability(rawtext);
            scores[UNICODET] = 0;
            scores[UNICODES] = 0;
            scores[ISO2022CN_GB] = 0;
            scores[ISO2022CN_CNS] = 0;
            scores[OTHER] = 0;
            // Tabulate Scores
            for (index = 0; index < TOTALTYPES; index++) {
                if (debug) {
                    System.err.println("Encoding " + nicename[index] + " score " + scores[index]);
                }
                if (scores[index] > maxscore) {
                    encoding_guess = index;
                    maxscore = scores[index];
                }
            }
            // Return OTHER if nothing scored above 50
            if (maxscore <= 50) {
                encoding_guess = OTHER;
            }
            return encoding_guess;
        }

        /*
         * Function: gb2312_probability Argument: pointer to byte array Returns : number from 0 to 100 representing
         * probability text in array uses GB-2312 encoding
         */
        int gb2312_probability(byte[] rawtext) {
            int i, rawtextlen = 0;
            int dbchars = 1, gbchars = 1;
            long gbfreq = 0, totalfreq = 1;
            float rangeval = 0, freqval = 0;
            int row, column;
            // Stage 1: Check to see if characters fit into acceptable ranges
            rawtextlen = rawtext.length;
            for (i = 0; i < rawtextlen - 1; ++i) {
                // System.err.println(rawtext[i]);
                if (rawtext[i] >= 0) {
                    // asciichars++;
                } else {
                    dbchars++;
                    if ((byte) 0xA1 <= rawtext[i] && rawtext[i] <= (byte) 0xF7 && (byte) 0xA1 <= rawtext[i + 1]
                            && rawtext[i + 1] <= (byte) 0xFE) {
                        gbchars++;
                        totalfreq += 500;
                        row = rawtext[i] + 256 - 0xA1;
                        column = rawtext[i + 1] + 256 - 0xA1;
                        if (GBFreq[row][column] != 0) {
                            gbfreq += GBFreq[row][column];
                        } else if (15 <= row && row < 55) {
                            // In GB high-freq character range
                            gbfreq += 200;
                        }
                    }
                    i++;
                }
            }
            rangeval = 50 * ((float) gbchars / (float) dbchars);
            freqval = 50 * ((float) gbfreq / (float) totalfreq);
            return (int) (rangeval + freqval);
        }

        /*
         * Function: gbk_probability Argument: pointer to byte array Returns : number from 0 to 100 representing
         * probability text in array uses GBK encoding
         */
        int gbk_probability(byte[] rawtext) {
            int i, rawtextlen = 0;
            int dbchars = 1, gbchars = 1;
            long gbfreq = 0, totalfreq = 1;
            float rangeval = 0, freqval = 0;
            int row, column;
            // Stage 1: Check to see if characters fit into acceptable ranges
            rawtextlen = rawtext.length;
            for (i = 0; i < rawtextlen - 1; ++i) {
                // System.err.println(rawtext[i]);
                if (rawtext[i] >= 0) {
                    // asciichars++;
                } else {
                    dbchars++;
                    if ((byte) 0xA1 <= rawtext[i] && rawtext[i] <= (byte) 0xF7
                            && // Original GB range
                            (byte) 0xA1 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0xFE) {
                        gbchars++;
                        totalfreq += 500;
                        row = rawtext[i] + 256 - 0xA1;
                        column = rawtext[i + 1] + 256 - 0xA1;
                        // System.out.println("original row " + row + " column " +
                        // column);
                        if (GBFreq[row][column] != 0) {
                            gbfreq += GBFreq[row][column];
                        } else if (15 <= row && row < 55) {
                            gbfreq += 200;
                        }
                    } else if ((byte) 0x81 <= rawtext[i] && rawtext[i] <= (byte) 0xFE
                            && // Extended GB range
                            (((byte) 0x80 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0xFE)
                            || ((byte) 0x40 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0x7E))) {
                        gbchars++;
                        totalfreq += 500;
                        row = rawtext[i] + 256 - 0x81;
                        if (0x40 <= rawtext[i + 1] && rawtext[i + 1] <= 0x7E) {
                            column = rawtext[i + 1] - 0x40;
                        } else {
                            column = rawtext[i + 1] + 256 - 0x40;
                        }
                        // System.out.println("extended row " + row + " column " +
                        // column + " rawtext[i] " + rawtext[i]);
                        if (GBKFreq[row][column] != 0) {
                            gbfreq += GBKFreq[row][column];
                        }
                    }
                    i++;
                }
            }
            rangeval = 50 * ((float) gbchars / (float) dbchars);
            freqval = 50 * ((float) gbfreq / (float) totalfreq);
            // For regular GB files, this would give the same score, so I handicap
            // it slightly
            return (int) (rangeval + freqval) - 1;
        }

        /*
         * Function: gb18030_probability Argument: pointer to byte array Returns : number from 0 to 100 representing
         * probability text in array uses GBK encoding
         */
        int gb18030_probability(byte[] rawtext) {
            int i, rawtextlen = 0;
            int dbchars = 1, gbchars = 1;
            long gbfreq = 0, totalfreq = 1;
            float rangeval = 0, freqval = 0;
            int row, column;
            // Stage 1: Check to see if characters fit into acceptable ranges
            rawtextlen = rawtext.length;
            for (i = 0; i < rawtextlen - 1; ++i) {
                // System.err.println(rawtext[i]);
                if (rawtext[i] >= 0) {
                    // asciichars++;
                } else {
                    dbchars++;
                    if ((byte) 0xA1 <= rawtext[i] && rawtext[i] <= (byte) 0xF7
                            && // Original GB range
                            i + 1 < rawtextlen && (byte) 0xA1 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0xFE) {
                        gbchars++;
                        totalfreq += 500;
                        row = rawtext[i] + 256 - 0xA1;
                        column = rawtext[i + 1] + 256 - 0xA1;
                        // System.out.println("original row " + row + " column " +
                        // column);
                        if (GBFreq[row][column] != 0) {
                            gbfreq += GBFreq[row][column];
                        } else if (15 <= row && row < 55) {
                            gbfreq += 200;
                        }
                    } else if ((byte) 0x81 <= rawtext[i] && rawtext[i] <= (byte) 0xFE
                            && // Extended GB range
                            i + 1 < rawtextlen && (((byte) 0x80 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0xFE)
                            || ((byte) 0x40 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0x7E))) {
                        gbchars++;
                        totalfreq += 500;
                        row = rawtext[i] + 256 - 0x81;
                        if (0x40 <= rawtext[i + 1] && rawtext[i + 1] <= 0x7E) {
                            column = rawtext[i + 1] - 0x40;
                        } else {
                            column = rawtext[i + 1] + 256 - 0x40;
                        }
                        // System.out.println("extended row " + row + " column " +
                        // column + " rawtext[i] " + rawtext[i]);
                        if (GBKFreq[row][column] != 0) {
                            gbfreq += GBKFreq[row][column];
                        }
                    } else if ((byte) 0x81 <= rawtext[i] && rawtext[i] <= (byte) 0xFE
                            && // Extended GB range
                            i + 3 < rawtextlen && (byte) 0x30 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0x39
                            && (byte) 0x81 <= rawtext[i + 2] && rawtext[i + 2] <= (byte) 0xFE && (byte) 0x30 <= rawtext[i + 3]
                            && rawtext[i + 3] <= (byte) 0x39) {
                        gbchars++;
                        /*
                         * totalfreq += 500; row = rawtext[i] + 256 - 0x81; if (0x40 <= rawtext[i+1] && rawtext[i+1] <=
                         * 0x7E) { column = rawtext[i+1] - 0x40; } else { column = rawtext[i+1] + 256 - 0x40; }
                         * //System.out.println("extended row " + row + " column " + column + " rawtext[i] " +
                         * rawtext[i]); if (GBKFreq[row][column] != 0) { gbfreq += GBKFreq[row][column]; }
                         */
                    }
                    i++;
                }
            }
            rangeval = 50 * ((float) gbchars / (float) dbchars);
            freqval = 50 * ((float) gbfreq / (float) totalfreq);
            // For regular GB files, this would give the same score, so I handicap
            // it slightly
            return (int) (rangeval + freqval) - 1;
        }

        /*
         * Function: hz_probability Argument: byte array Returns : number from 0 to 100 representing probability text in
         * array uses HZ encoding
         */
        int hz_probability(byte[] rawtext) {
            int i, rawtextlen;
            int hzchars = 0, dbchars = 1;
            long hzfreq = 0, totalfreq = 1;
            float rangeval = 0, freqval = 0;
            int hzstart = 0, hzend = 0;
            int row, column;
            rawtextlen = rawtext.length;
            for (i = 0; i < rawtextlen; ++i) {
                if (rawtext[i] == '~') {
                    if (rawtext[i + 1] == '{') {
                        hzstart++;
                        i += 2;
                        while (i < rawtextlen - 1) {
                            if (rawtext[i] == 0x0A || rawtext[i] == 0x0D) {
                                break;
                            } else if (rawtext[i] == '~' && rawtext[i + 1] == '}') {
                                hzend++;
                                i++;
                                break;
                            } else if ((0x21 <= rawtext[i] && rawtext[i] <= 0x77) && (0x21 <= rawtext[i + 1] && rawtext[i + 1] <= 0x77)) {
                                hzchars += 2;
                                row = rawtext[i] - 0x21;
                                column = rawtext[i + 1] - 0x21;
                                totalfreq += 500;
                                if (GBFreq[row][column] != 0) {
                                    hzfreq += GBFreq[row][column];
                                } else if (15 <= row && row < 55) {
                                    hzfreq += 200;
                                }
                            } else if ((0xA1 <= rawtext[i] && rawtext[i] <= 0xF7) && (0xA1 <= rawtext[i + 1] && rawtext[i + 1] <= 0xF7)) {
                                hzchars += 2;
                                row = rawtext[i] + 256 - 0xA1;
                                column = rawtext[i + 1] + 256 - 0xA1;
                                totalfreq += 500;
                                if (GBFreq[row][column] != 0) {
                                    hzfreq += GBFreq[row][column];
                                } else if (15 <= row && row < 55) {
                                    hzfreq += 200;
                                }
                            }
                            dbchars += 2;
                            i += 2;
                        }
                    } else if (rawtext[i + 1] == '}') {
                        hzend++;
                        i++;
                    } else if (rawtext[i + 1] == '~') {
                        i++;
                    }
                }
            }
            if (hzstart > 4) {
                rangeval = 50;
            } else if (hzstart > 1) {
                rangeval = 41;
            } else if (hzstart > 0) { // Only 39 in case the sequence happened to
                // occur
                rangeval = 39; // in otherwise non-Hz text
            } else {
                rangeval = 0;
            }
            freqval = 50 * ((float) hzfreq / (float) totalfreq);
            return (int) (rangeval + freqval);
        }

        /**
         * Function: big5_probability Argument: byte array Returns : number from
         * 0 to 100 representing probability text in array uses Big5 encoding
         */
        int big5_probability(byte[] rawtext) {
            int i, rawtextlen = 0;
            int dbchars = 1, bfchars = 1;
            float rangeval = 0, freqval = 0;
            long bffreq = 0, totalfreq = 1;
            int row, column;
            // Check to see if characters fit into acceptable ranges
            rawtextlen = rawtext.length;
            for (i = 0; i < rawtextlen - 1; ++i) {
                if (rawtext[i] >= 0) {
                    // asciichars++;
                } else {
                    dbchars++;
                    if ((byte) 0xA1 <= rawtext[i] && rawtext[i] <= (byte) 0xF9
                            && (((byte) 0x40 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0x7E)
                            || ((byte) 0xA1 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0xFE))) {
                        bfchars++;
                        totalfreq += 500;
                        row = rawtext[i] + 256 - 0xA1;
                        if (0x40 <= rawtext[i + 1] && rawtext[i + 1] <= 0x7E) {
                            column = rawtext[i + 1] - 0x40;
                        } else {
                            column = rawtext[i + 1] + 256 - 0x61;
                        }
                        if (Big5Freq[row][column] != 0) {
                            bffreq += Big5Freq[row][column];
                        } else if (3 <= row && row <= 37) {
                            bffreq += 200;
                        }
                    }
                    i++;
                }
            }
            rangeval = 50 * ((float) bfchars / (float) dbchars);
            freqval = 50 * ((float) bffreq / (float) totalfreq);
            return (int) (rangeval + freqval);
        }

        /*
         * Function: big5plus_probability Argument: pointer to unsigned char array Returns : number from 0 to 100
         * representing probability text in array uses Big5+ encoding
         */
        int big5plus_probability(byte[] rawtext) {
            int i, rawtextlen = 0;
            int dbchars = 1, bfchars = 1;
            long bffreq = 0, totalfreq = 1;
            float rangeval = 0, freqval = 0;
            int row, column;
            // Stage 1: Check to see if characters fit into acceptable ranges
            rawtextlen = rawtext.length;
            for (i = 0; i < rawtextlen - 1; ++i) {
                // System.err.println(rawtext[i]);
                if (rawtext[i] >= 128) {
                    // asciichars++;
                } else {
                    dbchars++;
                    if (0xA1 <= rawtext[i] && rawtext[i] <= 0xF9
                            && // Original Big5 range
                            ((0x40 <= rawtext[i + 1] && rawtext[i + 1] <= 0x7E) || (0xA1 <= rawtext[i + 1] && rawtext[i + 1] <= 0xFE))) {
                        bfchars++;
                        totalfreq += 500;
                        row = rawtext[i] - 0xA1;
                        if (0x40 <= rawtext[i + 1] && rawtext[i + 1] <= 0x7E) {
                            column = rawtext[i + 1] - 0x40;
                        } else {
                            column = rawtext[i + 1] - 0x61;
                        }
                        // System.out.println("original row " + row + " column " +
                        // column);
                        if (Big5Freq[row][column] != 0) {
                            bffreq += Big5Freq[row][column];
                        } else if (3 <= row && row < 37) {
                            bffreq += 200;
                        }
                    } else if (0x81 <= rawtext[i] && rawtext[i] <= 0xFE
                            && // Extended Big5 range
                            ((0x40 <= rawtext[i + 1] && rawtext[i + 1] <= 0x7E) || (0x80 <= rawtext[i + 1] && rawtext[i + 1] <= 0xFE))) {
                        bfchars++;
                        totalfreq += 500;
                        row = rawtext[i] - 0x81;
                        if (0x40 <= rawtext[i + 1] && rawtext[i + 1] <= 0x7E) {
                            column = rawtext[i + 1] - 0x40;
                        } else {
                            column = rawtext[i + 1] - 0x40;
                        }
                        // System.out.println("extended row " + row + " column " +
                        // column + " rawtext[i] " + rawtext[i]);
                        if (Big5PFreq[row][column] != 0) {
                            bffreq += Big5PFreq[row][column];
                        }
                    }
                    i++;
                }
            }
            rangeval = 50 * ((float) bfchars / (float) dbchars);
            freqval = 50 * ((float) bffreq / (float) totalfreq);
            // For regular Big5 files, this would give the same score, so I handicap
            // it slightly
            return (int) (rangeval + freqval) - 1;
        }

        /*
         * Function: euc_tw_probability Argument: byte array Returns : number from 0 to 100 representing probability
         * text in array uses EUC-TW (CNS 11643) encoding
         */
        int euc_tw_probability(byte[] rawtext) {
            int i, rawtextlen = 0;
            int dbchars = 1, cnschars = 1;
            long cnsfreq = 0, totalfreq = 1;
            float rangeval = 0, freqval = 0;
            int row, column;
            // Check to see if characters fit into acceptable ranges
            // and have expected frequency of use
            rawtextlen = rawtext.length;
            for (i = 0; i < rawtextlen - 1; ++i) {
                if (rawtext[i] >= 0) { // in ASCII range
                    // asciichars++;
                } else { // high bit set
                    dbchars++;
                    if (i + 3 < rawtextlen && (byte) 0x8E == rawtext[i] && (byte) 0xA1 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0xB0
                            && (byte) 0xA1 <= rawtext[i + 2] && rawtext[i + 2] <= (byte) 0xFE && (byte) 0xA1 <= rawtext[i + 3]
                            && rawtext[i + 3] <= (byte) 0xFE) { // Planes 1 - 16
                        cnschars++;
                        // System.out.println("plane 2 or above CNS char");
                        // These are all less frequent chars so just ignore freq
                        i += 3;
                    } else if ((byte) 0xA1 <= rawtext[i] && rawtext[i] <= (byte) 0xFE
                            && // Plane 1
                            (byte) 0xA1 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0xFE) {
                        cnschars++;
                        totalfreq += 500;
                        row = rawtext[i] + 256 - 0xA1;
                        column = rawtext[i + 1] + 256 - 0xA1;
                        if (EUC_TWFreq[row][column] != 0) {
                            cnsfreq += EUC_TWFreq[row][column];
                        } else if (35 <= row && row <= 92) {
                            cnsfreq += 150;
                        }
                        i++;
                    }
                }
            }
            rangeval = 50 * ((float) cnschars / (float) dbchars);
            freqval = 50 * ((float) cnsfreq / (float) totalfreq);
            return (int) (rangeval + freqval);
        }

        /*
         * Function: iso_2022_cn_probability Argument: byte array Returns : number from 0 to 100 representing
         * probability text in array uses ISO 2022-CN encoding WORKS FOR BASIC CASES, BUT STILL NEEDS MORE WORK
         */
        int iso_2022_cn_probability(byte[] rawtext) {
            int i, rawtextlen = 0;
            int dbchars = 1, isochars = 1;
            long isofreq = 0, totalfreq = 1;
            float rangeval = 0, freqval = 0;
            int row, column;
            // Check to see if characters fit into acceptable ranges
            // and have expected frequency of use
            rawtextlen = rawtext.length;
            for (i = 0; i < rawtextlen - 1; ++i) {
                if (rawtext[i] == (byte) 0x1B && i + 3 < rawtextlen) { // Escape
                    // char ESC
                    if (rawtext[i + 1] == (byte) 0x24 && rawtext[i + 2] == 0x29 && rawtext[i + 3] == (byte) 0x41) { // GB
                        // Escape
                        // $
                        // )
                        // A
                        i += 4;
                        while (rawtext[i] != (byte) 0x1B) {
                            dbchars++;
                            if ((0x21 <= rawtext[i] && rawtext[i] <= 0x77) && (0x21 <= rawtext[i + 1] && rawtext[i + 1] <= 0x77)) {
                                isochars++;
                                row = rawtext[i] - 0x21;
                                column = rawtext[i + 1] - 0x21;
                                totalfreq += 500;
                                if (GBFreq[row][column] != 0) {
                                    isofreq += GBFreq[row][column];
                                } else if (15 <= row && row < 55) {
                                    isofreq += 200;
                                }
                                i++;
                            }
                            i++;
                        }
                    } else if (i + 3 < rawtextlen && rawtext[i + 1] == (byte) 0x24 && rawtext[i + 2] == (byte) 0x29
                            && rawtext[i + 3] == (byte) 0x47) {
                        // CNS Escape $ ) G
                        i += 4;
                        while (rawtext[i] != (byte) 0x1B) {
                            dbchars++;
                            if ((byte) 0x21 <= rawtext[i] && rawtext[i] <= (byte) 0x7E && (byte) 0x21 <= rawtext[i + 1]
                                    && rawtext[i + 1] <= (byte) 0x7E) {
                                isochars++;
                                totalfreq += 500;
                                row = rawtext[i] - 0x21;
                                column = rawtext[i + 1] - 0x21;
                                if (EUC_TWFreq[row][column] != 0) {
                                    isofreq += EUC_TWFreq[row][column];
                                } else if (35 <= row && row <= 92) {
                                    isofreq += 150;
                                }
                                i++;
                            }
                            i++;
                        }
                    }
                    if (rawtext[i] == (byte) 0x1B && i + 2 < rawtextlen && rawtext[i + 1] == (byte) 0x28 && rawtext[i + 2] == (byte) 0x42) { // ASCII:
                        // ESC
                        // ( B
                        i += 2;
                    }
                }
            }
            rangeval = 50 * ((float) isochars / (float) dbchars);
            freqval = 50 * ((float) isofreq / (float) totalfreq);
            // System.out.println("isochars dbchars isofreq totalfreq " + isochars +
            // " " + dbchars + " " + isofreq + " " + totalfreq + "
            // " + rangeval + " " + freqval);
            return (int) (rangeval + freqval);
            // return 0;
        }

        /*
         * Function: utf8_probability Argument: byte array Returns : number from 0 to 100 representing probability text
         * in array uses UTF-8 encoding of Unicode
         */
        int utf8_probability(byte[] rawtext) {
            int score = 0;
            int i, rawtextlen = 0;
            int goodbytes = 0, asciibytes = 0;
            // Maybe also use UTF8 Byte Order Mark: EF BB BF
            // Check to see if characters fit into acceptable ranges
            rawtextlen = rawtext.length;
            for (i = 0; i < rawtextlen; ++i) {
                if ((rawtext[i] & (byte) 0x7F) == rawtext[i]) { // One byte
                    asciibytes++;
                    // Ignore ASCII, can throw off count
                } else if (-64 <= rawtext[i] && rawtext[i] <= -33
                        && // Two bytes
                        i + 1 < rawtextlen && -128 <= rawtext[i + 1] && rawtext[i + 1] <= -65) {
                    goodbytes += 2;
                    i++;
                } else if (-32 <= rawtext[i] && rawtext[i] <= -17
                        && // Three bytes
                        i + 2 < rawtextlen && -128 <= rawtext[i + 1] && rawtext[i + 1] <= -65 && -128 <= rawtext[i + 2]
                        && rawtext[i + 2] <= -65) {
                    goodbytes += 3;
                    i += 2;
                }
            }
            if (asciibytes == rawtextlen) {
                return 0;
            }
            score = (int) (100 * ((float) goodbytes / (float) (rawtextlen - asciibytes)));
            // System.out.println("rawtextlen " + rawtextlen + " goodbytes " +
            // goodbytes + " asciibytes " + asciibytes + " score " +
            // score);
            // If not above 98, reduce to zero to prevent coincidental matches
            // Allows for some (few) bad formed sequences
            if (score > 98) {
                return score;
            } else if (score > 95 && goodbytes > 30) {
                return score;
            } else {
                return 0;
            }
        }

        /*
         * Function: utf16_probability Argument: byte array Returns : number from 0 to 100 representing probability text
         * in array uses UTF-16 encoding of Unicode, guess based on BOM // NOT VERY GENERAL, NEEDS MUCH MORE WORK
         */
        int utf16_probability(byte[] rawtext) {
            // int score = 0;
            // int i, rawtextlen = 0;
            // int goodbytes = 0, asciibytes = 0;
            if (rawtext.length > 1 && ((byte) 0xFE == rawtext[0] && (byte) 0xFF == rawtext[1])
                    || // Big-endian
                    ((byte) 0xFF == rawtext[0] && (byte) 0xFE == rawtext[1])) { // Little-endian
                return 100;
            }
            return 0;
            /*
             * // Check to see if characters fit into acceptable ranges rawtextlen = rawtext.length; for (i = 0; i <
             * rawtextlen; i++) { if ((rawtext[i] & (byte)0x7F) == rawtext[i]) { // One byte goodbytes += 1;
             * asciibytes++; } else if ((rawtext[i] & (byte)0xDF) == rawtext[i]) { // Two bytes if (i+1 < rawtextlen &&
             * (rawtext[i+1] & (byte)0xBF) == rawtext[i+1]) { goodbytes += 2; i++; } } else if ((rawtext[i] &
             * (byte)0xEF) == rawtext[i]) { // Three bytes if (i+2 < rawtextlen && (rawtext[i+1] & (byte)0xBF) ==
             * rawtext[i+1] && (rawtext[i+2] & (byte)0xBF) == rawtext[i+2]) { goodbytes += 3; i+=2; } } }
             *
             * score = (int)(100 * ((float)goodbytes/(float)rawtext.length)); // An all ASCII file is also a good UTF8
             * file, but I'd rather it // get identified as ASCII. Can delete following 3 lines otherwise if (goodbytes
             * == asciibytes) { score = 0; } // If not above 90, reduce to zero to prevent coincidental matches if
             * (score > 90) { return score; } else { return 0; }
             */
        }

        /*
         * Function: ascii_probability Argument: byte array Returns : number from 0 to 100 representing probability text
         * in array uses all ASCII Description: Sees if array has any characters not in ASCII range, if so, score is
         * reduced
         */
        int ascii_probability(byte[] rawtext) {
            int score = 75;
            int i, rawtextlen;
            rawtextlen = rawtext.length;
            for (i = 0; i < rawtextlen; ++i) {
                if (rawtext[i] < 0) {
                    score = score - 5;
                } else if (rawtext[i] == (byte) 0x1B) { // ESC (used by ISO 2022)
