def get_tokcnt_from_log(log_path, by_billions=True):
    """
    input: path to the log file
    output: Tuple of (path, token_count)
    """
    print(f"[INFO] Checking token count log from {log_path}")
    match_path_pattern = r'Counting tokens in\s+(.*)'
    match_tokcnt_pattern = r'Total number of tokens:\s+(\d+)'

    with open(log_path, 'r') as f:
        log = f.read()
        path = re.search(match_path_pattern, log).group(1)
        tokcnt = int(re.search(match_tokcnt_pattern, log).group(1))
        if by_billions:
            tokcnt = tokcnt / 1e9
            # into string x.xxxB
            tokcnt = f"{tokcnt:.3f}B"
        return (path, tokcnt)

    input: path to the log file
    output: Tuple of (path, token_count)
    
finetune/core/parse_mixture.py