def _read_from_files(self, source_file):
        """
        <s> <image> image token </image> sentence </s>
        <s> sentence  <image> image token </image> </s>

        """
        file_path = os.path.join(self.data_dir, source_file)
        
        if not os.path.exists(file_path):
            print('| file {} not exists'.format(file_path), flush=True)
            return iter([]) # skip bad file

        try:
            with open(file_path, 'r', encoding='utf8') as f:
                lines = f.read().strip().split('\n')
        except:
            return iter([]) # skip bad file

        for doc_jsonstr in lines:
            try:
                obj = json.loads(doc_jsonstr)
                if int(obj['width']) < 200 or int(obj['height']) < 200:
                    continue
                
                line = obj['caption']
                spm_tokenizer=self.tokenizer
                if isinstance(spm_tokenizer, GPT2BPE):
                    tokens = spm_tokenizer.encode(line).split(' ')
                else:
                    tokens = spm_tokenizer.encode(line, out_type=str)
                tokenized_tokens = LaionLoader.fs_encode_line(self.dictionary, tokens, append_eos=False)
                image_tokens = [image_code_to_token(i) for i in obj['input_ids']]
                image_tokens = LaionLoader.fs_encode_line(self.dictionary, image_tokens, append_eos=False)
                r = random.random()
                doc = [self.dictionary.bos()]
                if r < 0.5:
                    doc.append(self.dictionary.index(BOI_SYMBOL))
                    doc.extend(image_tokens)
                    doc.append(self.dictionary.index(EOI_SYMBOL))
                    doc.extend(tokenized_tokens)
                else:
                    doc.extend(tokenized_tokens)
                    doc.append(self.dictionary.index(BOI_SYMBOL))
                    doc.extend(image_tokens)
                    doc.append(self.dictionary.index(EOI_SYMBOL))
                doc.append(self.dictionary.eos())
                yield doc
            except:
                continue

        <s> <image> image token </image> sentence </s>
        <s> sentence  <image> image token </image> </s>

        
kosmos-2/torchscale/examples/fairseq/tasks/data/laion_loader.py