def update_meta_without_tokenization(data_dir: str):
    """Update meta information for the given data directory.

    Args:
        data_dir (str): data directory path.
    """
    folder = Path(data_dir)
    jsonl_filepaths = [str(p) for p in folder.glob("**/*.jsonl")]
    sub_dirs = [str(p) for p in folder.glob("*")]
    print(f"Sub directories (tot={len(sub_dirs)}): {sub_dirs}")
    print(f"#Total Jsonl files: {len(jsonl_filepaths)}")

    mp.cpu_count()
    bar = tqdm(range(len(jsonl_filepaths)), desc="Preparing meta")
    with mp.Pool(mp.cpu_count()) as pool:
        for _ in pool.imap_unordered(prepare_meta, jsonl_filepaths):
            bar.update()
Update meta information for the given data directory.

    Args:
        data_dir (str): data directory path.
    
smoe/utils/tokenize.py