def __call__(
            self, tokens: Union[Tuple, List[List[str]]], entity_pos: Union[Tuple, List[List[Tuple]]],
            entity_tags: Union[Tuple, List[List[str]]],
    ) -> Tuple[List, List, List, List, List]:
        """
        Tokenize and create masks; recalculate the entity positions regarding the document boarders.
        Args:
            tokens: List of tokens of each document: List[List[tokens in doc]]
            entity_pos: start and end positions of the entities' mentions
            entity_tags: NER tag of the entities
        Return:
            input_ids: List[List[int]],
            attention_mask: List[List[int]],
            entity_poss: List[
                            List[
                                List[(entity1_mention1_start_id, entity1_mention1_end_id), ...],
                                List[(entity2_mention1_start_id, entity2_mention1_end_id), ...]
                            ]
                        ]
            entity_tags: List[List[int]]
            nf_samples: List[int] - contains the information about whether the corresponding sample is real sample or
                fake (for testing): 0 means the sample is real, 1 - it is fake.
        """

        _ = self.tokenizer.add_special_tokens(self.special_tokens_dict)

        input_ids, attention_mask, upd_entity_pos, upd_entity_tags, nf_samples = [], [], [], [], []

        # this workaround is for proper testing: for an unknown reason during test in test_quick_start.py
        # each input list is transformed into a tuple, e.g., tokens -> tuple(tokens, ).
        # todo: refactoring
        if type(tokens) == tuple and type(entity_pos) == tuple and type(entity_tags) == tuple:
            tokens = tokens[0]
            entity_pos = entity_pos[0]
            entity_tags = entity_tags[0]

        for n_sample, (doc, ent_pos, ent_tags) in enumerate(zip(tokens, entity_pos, entity_tags)):

            # valid scenario
            if isinstance(ent_pos, list) and len(ent_pos) == 2:
                count = 0
                doc_wordpiece_tokens = []

                entity1_pos_start = list(zip(*ent_pos[0]))[0]  # first entity mentions' start positions
                entity1_pos_end = list(zip(*ent_pos[0]))[1]  # first entity mentions' end positions
                entity2_pos_start = list(zip(*ent_pos[1]))[0]  # second entity mentions' start positions
                entity2_pos_end = list(zip(*ent_pos[1]))[1]  # second entity mentions' end positions

                upd_entity1_pos_start, upd_entity2_pos_start, upd_entity1_pos_end, upd_entity2_pos_end = [], [], [], []
                for n, token in enumerate(doc):
                    if n in entity1_pos_start:
                        doc_wordpiece_tokens.append(self.special_token)
                        upd_entity1_pos_start.append(count)
                        count += 1

                    if n in entity1_pos_end:
                        doc_wordpiece_tokens.append(self.special_token)
                        count += 1
                        upd_entity1_pos_end.append(count)

                    if n in entity2_pos_start:
                        doc_wordpiece_tokens.append(self.special_token)
                        upd_entity2_pos_start.append(count)
                        count += 1

                    if n in entity2_pos_end:
                        doc_wordpiece_tokens.append(self.special_token)
                        count += 1
                        upd_entity2_pos_end.append(count)

                    word_tokens = self.tokenizer.tokenize(token)
                    doc_wordpiece_tokens += word_tokens
                    count += len(word_tokens)

                # special case when the entity is the last in the doc
                if len(doc) in entity1_pos_end:
                    doc_wordpiece_tokens.append(self.special_token)
                    count += 1
                    upd_entity1_pos_end.append(count)
                if len(doc) in entity2_pos_end:
                    doc_wordpiece_tokens.append(self.special_token)
                    count += 1
                    upd_entity2_pos_end.append(count)
                    word_tokens = self.tokenizer.tokenize(token)
                    doc_wordpiece_tokens += word_tokens
                    count += len(word_tokens)

                upd_entity_1_pos = list(zip(upd_entity1_pos_start, upd_entity1_pos_end))
                upd_entity_2_pos = list(zip(upd_entity2_pos_start, upd_entity2_pos_end))

                # text entities for self check
                upd_entity1_text = [doc_wordpiece_tokens[ent_m[0]:ent_m[1]] for ent_m in upd_entity_1_pos]
                upd_entity2_text = [doc_wordpiece_tokens[ent_m[0]:ent_m[1]] for ent_m in upd_entity_2_pos]

                enc_entity_tags = self.encode_ner_tag(ent_tags)

                encoding = self.tokenizer.encode_plus(
                    doc_wordpiece_tokens[:self.max_seq_length],     # truncate tokens
                    add_special_tokens=True,
                    truncation=True,
                    max_length=self.max_seq_length,
                    pad_to_max_length=True,
                    return_attention_mask=True
                )
                upd_entity_pos.append([upd_entity_1_pos, upd_entity_2_pos])
                nf_samples.append(0)

            # api test scenario
            else:
                # for api test: dump values of entity tags and entity pos
                encoding = self.tokenizer.encode_plus(
                    doc,
                    add_special_tokens=True,
                    truncation=True,
                    max_length=self.max_seq_length,
                    pad_to_max_length=True,
                    return_attention_mask=True
                )
                upd_entity_pos.append([[(0, 1)], [(0, 1)]])
                enc_entity_tags = self.encode_ner_tag([self.default_tag] * 2)
                nf_samples.append(1)

            input_ids.append(encoding['input_ids'])
            attention_mask.append(encoding['attention_mask'])
            upd_entity_tags.append(enc_entity_tags)

        return input_ids, attention_mask, upd_entity_pos, upd_entity_tags, nf_samples

        Tokenize and create masks; recalculate the entity positions regarding the document boarders.
        Args:
            tokens: List of tokens of each document: List[List[tokens in doc]]
            entity_pos: start and end positions of the entities' mentions
            entity_tags: NER tag of the entities
        Return:
            input_ids: List[List[int]],
            attention_mask: List[List[int]],
            entity_poss: List[
                            List[
                                List[(entity1_mention1_start_id, entity1_mention1_end_id), ...],
                                List[(entity2_mention1_start_id, entity2_mention1_end_id), ...]
                            ]
                        ]
            entity_tags: List[List[int]]
            nf_samples: List[int] - contains the information about whether the corresponding sample is real sample or
                fake (for testing): 0 means the sample is real, 1 - it is fake.
        
deeppavlov/models/preprocessors/re_preprocessor.py