def insert_data_and_finetune(
        self,
        new_train_dataset_batch: EmbeddingQAFinetuneDataset,
        new_val_dataset_batch: Optional[EmbeddingQAFinetuneDataset] = None,
    ):
        """
        Insert data and finetune. This should only be done if the new data you are inserting does not conflict with the already existing data. It's important to not finetune multiple times as this can cause the embeddings to lose semantic meaning since they will become further from the original embeddings.
        """
        try:
            import numpy as np
        except ImportError:
            raise ImportError(NUMPY_IMPORT_ERROR_MSG)

        new_corpus_batch = new_train_dataset_batch.corpus
        # if any of the new ids are already in the existing corpus, raise an error
        if any(id in self.corpus for id in new_corpus_batch):
            raise ValueError(
                f"ID {id} already exists in the existing corpus. New IDs must be unique."
            )

        # get the embeddings for the new corpus
        new_corpus_initial_embeddings_batch = self._get_corpus_embeddings(
            new_corpus_batch
        )

        existing_corpus_embeddings = self.corpus_embeddings

        new_train_dataset = self._format_dataset(
            new_train_dataset_batch, new_corpus_batch
        )
        new_val_dataset = self._format_dataset(new_val_dataset_batch, new_corpus_batch)

        new_corpus_embeddings_batch = self.nudge.finetune_embeddings(
            embeddings=new_corpus_initial_embeddings_batch,
            train_set=new_train_dataset,
            val_set=new_val_dataset,
            # runs faster by filtering the embeddings which will not have any queries
            nontrain_embeddings=existing_corpus_embeddings,
            val_batch_size=256,
            gamma=None,
        )

        self.corpus_embeddings = np.concatenate(
            [existing_corpus_embeddings, new_corpus_embeddings_batch]
        )

        Insert data and finetune. This should only be done if the new data you are inserting does not conflict with the already existing data. It's important to not finetune multiple times as this can cause the embeddings to lose semantic meaning since they will become further from the original embeddings.
        
llama-index-experimental/llama_index/experimental/nudge/base.py