package org.ohdsi.webapi.cohortcharacterization;

import com.cosium.spring.data.jpa.entity.graph.domain.EntityGraph;
import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.collect.ImmutableList;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.ohdsi.analysis.Utils;
import org.ohdsi.analysis.WithId;
import org.ohdsi.analysis.cohortcharacterization.design.CohortCharacterization;
import org.ohdsi.analysis.cohortcharacterization.design.StandardFeatureAnalysisType;
import org.ohdsi.circe.helper.ResourceHelper;
import org.ohdsi.featureExtraction.FeatureExtraction;
import org.ohdsi.hydra.Hydra;
import org.ohdsi.sql.SqlSplit;
import org.ohdsi.sql.SqlTranslate;
import org.ohdsi.webapi.Constants;
import org.ohdsi.webapi.JobInvalidator;
import org.ohdsi.webapi.cohortcharacterization.converter.SerializedCcToCcConverter;
import org.ohdsi.webapi.cohortcharacterization.domain.CcFeAnalysisEntity;
import org.ohdsi.webapi.cohortcharacterization.domain.CcGenerationEntity;
import org.ohdsi.webapi.cohortcharacterization.domain.CcParamEntity;
import org.ohdsi.webapi.cohortcharacterization.domain.CcStrataConceptSetEntity;
import org.ohdsi.webapi.cohortcharacterization.domain.CcStrataEntity;
import org.ohdsi.webapi.cohortcharacterization.domain.CohortCharacterizationEntity;
import org.ohdsi.webapi.cohortcharacterization.dto.AbstractTemporalResult;
import org.ohdsi.webapi.cohortcharacterization.dto.CcDistributionStat;
import org.ohdsi.webapi.cohortcharacterization.dto.CcExportDTO;
import org.ohdsi.webapi.cohortcharacterization.dto.CcPrevalenceStat;
import org.ohdsi.webapi.cohortcharacterization.dto.CcResult;
import org.ohdsi.webapi.cohortcharacterization.dto.CcShortDTO;
import org.ohdsi.webapi.cohortcharacterization.dto.CcTemporalAnnualResult;
import org.ohdsi.webapi.cohortcharacterization.dto.CcTemporalResult;
import org.ohdsi.webapi.cohortcharacterization.dto.CcVersionFullDTO;
import org.ohdsi.webapi.cohortcharacterization.dto.CohortCharacterizationDTO;
import org.ohdsi.webapi.cohortcharacterization.dto.ExecutionResultRequest;
import org.ohdsi.webapi.cohortcharacterization.dto.ExportExecutionResultRequest;
import org.ohdsi.webapi.cohortcharacterization.dto.GenerationResults;
import org.ohdsi.webapi.cohortcharacterization.report.AnalysisItem;
import org.ohdsi.webapi.cohortcharacterization.report.AnalysisResultItem;
import org.ohdsi.webapi.cohortcharacterization.report.ComparativeItem;
import org.ohdsi.webapi.cohortcharacterization.report.ExportItem;
import org.ohdsi.webapi.cohortcharacterization.report.PrevalenceItem;
import org.ohdsi.webapi.cohortcharacterization.report.Report;
import org.ohdsi.webapi.cohortcharacterization.report.TemporalAnnualItem;
import org.ohdsi.webapi.cohortcharacterization.report.TemporalItem;
import org.ohdsi.webapi.cohortcharacterization.repository.AnalysisGenerationInfoEntityRepository;
import org.ohdsi.webapi.cohortcharacterization.repository.CcConceptSetRepository;
import org.ohdsi.webapi.cohortcharacterization.repository.CcFeAnalysisRepository;
import org.ohdsi.webapi.cohortcharacterization.repository.CcGenerationEntityRepository;
import org.ohdsi.webapi.cohortcharacterization.repository.CcParamRepository;
import org.ohdsi.webapi.cohortcharacterization.repository.CcRepository;
import org.ohdsi.webapi.cohortcharacterization.repository.CcStrataRepository;
import org.ohdsi.webapi.cohortcharacterization.specification.CohortCharacterizationImpl;
import org.ohdsi.webapi.cohortdefinition.CohortDefinition;
import org.ohdsi.webapi.cohortdefinition.event.CohortDefinitionChangedEvent;
import org.ohdsi.webapi.common.DesignImportService;
import org.ohdsi.webapi.common.generation.AnalysisGenerationInfoEntity;
import org.ohdsi.webapi.common.generation.GenerationUtils;
import org.ohdsi.webapi.conceptset.ConceptSetExport;
import org.ohdsi.webapi.feanalysis.FeAnalysisService;
import org.ohdsi.webapi.feanalysis.domain.FeAnalysisCriteriaEntity;
import org.ohdsi.webapi.feanalysis.domain.FeAnalysisEntity;
import org.ohdsi.webapi.feanalysis.domain.FeAnalysisWithCriteriaEntity;
import org.ohdsi.webapi.feanalysis.domain.FeAnalysisWithStringEntity;
import org.ohdsi.webapi.feanalysis.event.FeAnalysisChangedEvent;
import org.ohdsi.webapi.job.GeneratesNotification;
import org.ohdsi.webapi.job.JobExecutionResource;
import org.ohdsi.webapi.service.AbstractDaoService;
import org.ohdsi.webapi.service.FeatureExtractionService;
import org.ohdsi.webapi.service.JobService;
import org.ohdsi.webapi.service.VocabularyService;
import org.ohdsi.webapi.shiro.Entities.UserEntity;
import org.ohdsi.webapi.shiro.annotations.CcGenerationId;
import org.ohdsi.webapi.shiro.annotations.DataSourceAccess;
import org.ohdsi.webapi.shiro.annotations.SourceKey;
import org.ohdsi.webapi.source.Source;
import org.ohdsi.webapi.source.SourceInfo;
import org.ohdsi.webapi.source.SourceService;
import org.ohdsi.webapi.sqlrender.SourceAwareSqlRender;
import org.ohdsi.webapi.tag.dto.TagNameListRequestDTO;
import org.ohdsi.webapi.util.CancelableJdbcTemplate;
import org.ohdsi.webapi.util.EntityUtils;
import org.ohdsi.webapi.util.ExceptionUtils;
import org.ohdsi.webapi.util.ExportUtil;
import org.ohdsi.webapi.util.NameUtils;
import org.ohdsi.webapi.util.SessionUtils;
import org.ohdsi.webapi.util.SourceUtils;
import org.ohdsi.webapi.util.TempFileUtils;
import org.ohdsi.webapi.versioning.domain.CharacterizationVersion;
import org.ohdsi.webapi.versioning.domain.Version;
import org.ohdsi.webapi.versioning.domain.VersionBase;
import org.ohdsi.webapi.versioning.domain.VersionType;
import org.ohdsi.webapi.versioning.dto.VersionDTO;
import org.ohdsi.webapi.versioning.dto.VersionUpdateDTO;
import org.ohdsi.webapi.versioning.service.VersionService;
import org.springframework.batch.core.BatchStatus;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.job.builder.SimpleJobBuilder;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.annotation.DependsOn;
import org.springframework.context.event.EventListener;
import org.springframework.core.convert.ConversionService;
import org.springframework.core.convert.support.GenericConversionService;
import org.springframework.core.env.Environment;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import javax.annotation.PostConstruct;
import javax.persistence.EntityManager;
import javax.ws.rs.InternalServerErrorException;
import javax.ws.rs.NotFoundException;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;

import static java.util.stream.Collectors.groupingBy;
import static org.ohdsi.analysis.cohortcharacterization.design.CcResultType.DISTRIBUTION;
import static org.ohdsi.analysis.cohortcharacterization.design.CcResultType.PREVALENCE;
import static org.ohdsi.webapi.Constants.GENERATE_COHORT_CHARACTERIZATION;
import static org.ohdsi.webapi.Constants.Params.COHORT_CHARACTERIZATION_ID;
import static org.ohdsi.webapi.Constants.Params.JOB_AUTHOR;
import static org.ohdsi.webapi.Constants.Params.JOB_NAME;
import static org.ohdsi.webapi.Constants.Params.SOURCE_ID;
import org.ohdsi.webapi.security.PermissionService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.PageImpl;

@Service
@Transactional
@DependsOn({"ccExportDTOToCcEntityConverter", "cohortDTOToCohortDefinitionConverter", "feAnalysisDTOToFeAnalysisConverter"})
public class CcServiceImpl extends AbstractDaoService implements CcService, GeneratesNotification {

    private static final String GENERATION_NOT_FOUND_ERROR = "generation cannot be found by id %d";
    private static final String[] PARAMETERS_RESULTS = {"cohort_characterization_generation_id", "threshold_level", "vocabulary_schema"};
    private static final String[] PARAMETERS_RESULTS_FILTERED = {"cohort_characterization_generation_id", "threshold_level",
            "analysis_ids", "cohort_ids", "vocabulary_schema"};
    private static final String[] PARAMETERS_COUNT = {"cohort_characterization_generation_id", "vocabulary_schema"};
    private static final String[] PREVALENCE_STATS_PARAMS = {"cdm_database_schema", "cdm_results_schema", "cc_generation_id", "analysis_id", "cohort_id", "covariate_id"};
    private final String QUERY_RESULTS = ResourceHelper.GetResourceAsString("/resources/cohortcharacterizations/sql/queryResults.sql");
    private final String QUERY_TEMPORAL_RESULTS = ResourceHelper.GetResourceAsString("/resources/cohortcharacterizations/sql/queryTemporalResults.sql");
    private final String QUERY_TEMPORAL_ANNUAL_RESULTS = ResourceHelper.GetResourceAsString("/resources/cohortcharacterizations/sql/queryTemporalAnnualResults.sql");
    private final String QUERY_COUNT = ResourceHelper.GetResourceAsString("/resources/cohortcharacterizations/sql/queryCountWithoutThreshold.sql");
    private final String DELETE_RESULTS = ResourceHelper.GetResourceAsString("/resources/cohortcharacterizations/sql/deleteResults.sql");
    private final String DELETE_EXECUTION = ResourceHelper.GetResourceAsString("/resources/cohortcharacterizations/sql/deleteExecution.sql");
    private final String QUERY_PREVALENCE_STATS = ResourceHelper.GetResourceAsString("/resources/cohortcharacterizations/sql/queryCovariateStatsVocab.sql");

    private final String HYDRA_PACKAGE = "/resources/cohortcharacterizations/hydra/CohortCharacterization_v0.0.1.zip";

    private final static List<String> INCOMPLETE_STATUSES = ImmutableList.of(BatchStatus.STARTED, BatchStatus.STARTING, BatchStatus.STOPPING, BatchStatus.UNKNOWN)
            .stream().map(BatchStatus::name).collect(Collectors.toList());

    private Map<String, FeatureExtraction.PrespecAnalysis> prespecAnalysisMap = FeatureExtraction.getNameToPrespecAnalysis();

    private final EntityGraph defaultEntityGraph = EntityUtils.fromAttributePaths(
            "cohortDefinitions",
            "featureAnalyses",
            "stratas",
            "parameters",
            "createdBy",
            "modifiedBy"
    );

    private final List<String[]> executionPrevalenceHeaderLines = new ArrayList<String[]>() {{
        add(new String[]{"Analysis ID", "Analysis name", "Strata ID",
                "Strata name", "Cohort ID", "Cohort name", "Covariate ID", "Covariate name", "Covariate short name",
                "Count", "Percent"});
    }};

    private final List<String[]> executionDistributionHeaderLines = new ArrayList<String[]>() {{
        add(new String[]{"Analysis ID", "Analysis name", "Strata ID",
                "Strata name", "Cohort ID", "Cohort name", "Covariate ID", "Covariate name", "Covariate short name", "Value field","Missing Means Zero",
                "Count", "Avg", "StdDev", "Min", "P10", "P25", "Median", "P75", "P90", "Max"});
    }};

    private final List<String[]> executionComparativeHeaderLines = new ArrayList<String[]>() {{
        add(new String[]{"Analysis ID", "Analysis name", "Strata ID",
                "Strata name", "Target cohort ID", "Target cohort name", "Comparator cohort ID", "Comparator cohort name",
                "Covariate ID", "Covariate name", "Covariate short name", "Target count", "Target percent",
                "Comparator count", "Comparator percent", "Std. Diff Of Mean"});
    }};

    private final CcRepository repository;
    private final CcParamRepository paramRepository;
    private final CcStrataRepository strataRepository;
    private final CcConceptSetRepository conceptSetRepository;
    private final FeAnalysisService analysisService;
    private final CcGenerationEntityRepository ccGenerationRepository;
    private final FeatureExtractionService featureExtractionService;
    private final DesignImportService designImportService;
    private final AnalysisGenerationInfoEntityRepository analysisGenerationInfoEntityRepository;
    private final SourceService sourceService;
    private final GenerationUtils generationUtils;
    private final EntityManager entityManager;
    private final ApplicationEventPublisher eventPublisher;

    private final JobRepository jobRepository;
    private final SourceAwareSqlRender sourceAwareSqlRender;
    private final JobService jobService;
    private final JobInvalidator jobInvalidator;
    private final GenericConversionService genericConversionService;
    private final VocabularyService vocabularyService;
    private final CcFeAnalysisRepository ccFeAnalysisRepository;
    private VersionService<CharacterizationVersion> versionService;
    
    private PermissionService permissionService;

    @Value("${security.defaultGlobalReadPermissions}")
    private boolean defaultGlobalReadPermissions;
    
    private final Environment env;

    public CcServiceImpl(
            final CcRepository ccRepository,
            final CcParamRepository paramRepository,
            final CcStrataRepository strataRepository,
            final CcConceptSetRepository conceptSetRepository,
            final FeAnalysisService analysisService,
            final CcGenerationEntityRepository ccGenerationRepository,
            final FeatureExtractionService featureExtractionService,
            final ConversionService conversionService,
            final DesignImportService designImportService,
            final JobRepository jobRepository,
            final AnalysisGenerationInfoEntityRepository analysisGenerationInfoEntityRepository,
            final SourceService sourceService,
            final GenerationUtils generationUtils,
            final SourceAwareSqlRender sourceAwareSqlRender,
            final EntityManager entityManager,
            final JobService jobService,
            final ApplicationEventPublisher eventPublisher,
            final JobInvalidator jobInvalidator,
            final VocabularyService vocabularyService,
            CcFeAnalysisRepository ccFeAnalysisRepository, final VersionService<CharacterizationVersion> versionService,
            final PermissionService permissionService,
            @Qualifier("conversionService") final GenericConversionService genericConversionService,
            Environment env) {
        this.repository = ccRepository;
        this.paramRepository = paramRepository;
        this.strataRepository = strataRepository;
        this.conceptSetRepository = conceptSetRepository;
        this.analysisService = analysisService;
        this.ccGenerationRepository = ccGenerationRepository;
        this.featureExtractionService = featureExtractionService;
        this.designImportService = designImportService;
        this.jobRepository = jobRepository;
        this.analysisGenerationInfoEntityRepository = analysisGenerationInfoEntityRepository;
        this.sourceService = sourceService;
        this.generationUtils = generationUtils;
        this.sourceAwareSqlRender = sourceAwareSqlRender;
        this.entityManager = entityManager;
        this.jobService = jobService;
        this.eventPublisher = eventPublisher;
        this.jobInvalidator = jobInvalidator;
        this.vocabularyService = vocabularyService;
        this.ccFeAnalysisRepository = ccFeAnalysisRepository;
        this.permissionService = permissionService;
        this.genericConversionService = genericConversionService;
        this.versionService = versionService;
        this.env = env;
        SerializedCcToCcConverter.setConversionService(conversionService);
    }

    @Override
    public CohortCharacterizationEntity createCc(final CohortCharacterizationEntity entity) {
        entity.setCreatedBy(getCurrentUser());
        entity.setCreatedDate(new Date());
        return saveCc(entity);
    }

    private CohortCharacterizationEntity saveCc(final CohortCharacterizationEntity entity) {
        CohortCharacterizationEntity savedEntity = repository.save(entity);

        for(CcStrataEntity strata: entity.getStratas()){
          strata.setCohortCharacterization(savedEntity);
          strataRepository.save(strata);
        }

        for(CcParamEntity param: entity.getParameters()){
          param.setCohortCharacterization(savedEntity);
          paramRepository.save(param);
        }

        for(CcFeAnalysisEntity analysis : entity.getCcFeatureAnalyses()) {
            analysis.setCohortCharacterization(savedEntity);
            ccFeAnalysisRepository.save(analysis);
        }

        entityManager.flush();
        entityManager.refresh(savedEntity);

        savedEntity = findByIdWithLinkedEntities(savedEntity.getId());

        Date modifiedDate = savedEntity.getModifiedDate();
        savedEntity.setModifiedDate(null);
        final String serialized = this.serializeCc(savedEntity);
        savedEntity.setHashCode(serialized.hashCode());
        savedEntity.setModifiedDate(modifiedDate);

        return repository.save(savedEntity);
    }

    @EventListener
    @Transactional
    @Override
    public void onCohortDefinitionChanged(CohortDefinitionChangedEvent event) {

        List<CohortCharacterizationEntity> ccList = repository.findByCohortDefinition(event.getCohortDefinition());
        ccList.forEach(this::saveCc);
    }

    @EventListener
    @Transactional
    public void onFeAnalysisChanged(FeAnalysisChangedEvent event) {

        List<CohortCharacterizationEntity> ccList = repository.findByFeatureAnalysis(event.getFeAnalysis());
        ccList.forEach(this::saveCc);
    }

    @Override
    @Transactional
    public void assignTag(Long id, int tagId) {
        CohortCharacterizationEntity entity = findById(id);
        assignTag(entity, tagId);
    }

    @Override
    @Transactional
    public void unassignTag(Long id, int tagId) {
        CohortCharacterizationEntity entity = findById(id);
        unassignTag(entity, tagId);
    }

    @Override
    public int getCountCcWithSameName(Long id, String name) {
        return repository.getCountCcWithSameName(id, name);
    }

    @Override
    public void deleteCc(Long ccId) {
        repository.delete(ccId);
    }

    @Override
    public CohortCharacterizationEntity updateCc(final CohortCharacterizationEntity entity) {
        final CohortCharacterizationEntity foundEntity = repository.findById(entity.getId())
                .orElseThrow(() -> new NotFoundException("CC entity isn't found"));

        updateLinkedFields(entity, foundEntity);

        if (StringUtils.isNotEmpty(entity.getName())) {
            foundEntity.setName(entity.getName());
        }
        foundEntity.setDescription(entity.getDescription());
        foundEntity.setStratifiedBy(entity.getStratifiedBy());
        if (Objects.nonNull(entity.getStrataOnly())) {
          foundEntity.setStrataOnly(entity.getStrataOnly());
        }

        foundEntity.setModifiedDate(new Date());
        foundEntity.setModifiedBy(getCurrentUser());

        return saveCc(foundEntity);
    }

    private void updateLinkedFields(final CohortCharacterizationEntity entity, final CohortCharacterizationEntity foundEntity) {
        updateConceptSet(entity, foundEntity);
        updateParams(entity, foundEntity);
        updateAnalyses(entity, foundEntity);
        updateCohorts(entity, foundEntity);
        updateStratas(entity, foundEntity);
    }

  private void updateConceptSet(CohortCharacterizationEntity entity, CohortCharacterizationEntity foundEntity) {
      if (Objects.nonNull(foundEntity.getConceptSetEntity()) && Objects.nonNull(entity.getConceptSetEntity())) {
        foundEntity.getConceptSetEntity().setRawExpression(entity.getConceptSetEntity().getRawExpression());
      } else if (Objects.nonNull(entity.getConceptSetEntity())) {
        CcStrataConceptSetEntity cse = new CcStrataConceptSetEntity();
        cse.setCohortCharacterization(foundEntity);
        cse.setRawExpression(entity.getConceptSetEntity().getRawExpression());
        foundEntity.setConceptSetEntity(cse);
      } else {
        foundEntity.setConceptSetEntity(null);
      }
  }

  private void updateStratas(CohortCharacterizationEntity entity, CohortCharacterizationEntity foundEntity) {
        final List<CcStrataEntity> stratasToDelete = getLinksToDelete(foundEntity,
                existingLink -> entity.getStratas().stream().noneMatch(newLink -> Objects.equals(newLink.getId(), existingLink.getId())),
                CohortCharacterizationEntity::getStratas);
        foundEntity.getStratas().removeAll(stratasToDelete);
        strataRepository.delete(stratasToDelete);
        Map<Long, CcStrataEntity> strataEntityMap = foundEntity.getStratas().stream()
                .collect(Collectors.toMap(CcStrataEntity::getId, s -> s));

        List<CcStrataEntity> updatedStratas = entity.getStratas().stream().map(updated -> {
            updated.setCohortCharacterization(foundEntity);
            if (Objects.nonNull(updated.getId())) {
                CcStrataEntity strata = strataEntityMap.get(updated.getId());
                // strata will be null in case of importing new characterization
                if (strata == null) {
                    return updated;
                }
                if (StringUtils.isNotBlank(updated.getName())) {
                    strata.setName(updated.getName());
                }
                strata.setExpressionString(updated.getExpressionString());
                return strata;
            } else {
                return updated;
            }
        }).collect(Collectors.toList());
        entity.setStratas(new HashSet<>(strataRepository.save(updatedStratas)));
    }

    private void updateCohorts(final CohortCharacterizationEntity entity, final CohortCharacterizationEntity foundEntity) {
        foundEntity.getCohortDefinitions().clear();
        foundEntity.getCohortDefinitions().addAll(entity.getCohortDefinitions());
    }

    private void updateAnalyses(final CohortCharacterizationEntity entity, final CohortCharacterizationEntity foundEntity) {
        ccFeAnalysisRepository.delete(foundEntity.getCcFeatureAnalyses());
        foundEntity.getCcFeatureAnalyses().clear();
        foundEntity.getCcFeatureAnalyses().addAll(entity.getCcFeatureAnalyses());
        ccFeAnalysisRepository.save(foundEntity.getCcFeatureAnalyses());
    }

    private <T extends WithId> List<T> getLinksToDelete(final CohortCharacterizationEntity foundEntity,
                                                        Predicate<? super T> filterPredicate,
                                                        Function<CohortCharacterizationEntity, Set<T>> getter) {
        return getter.apply(foundEntity)
                .stream()
                .filter(filterPredicate)
                .collect(Collectors.toList());
    }

    private void updateParams(final CohortCharacterizationEntity entity, final CohortCharacterizationEntity foundEntity) {
        updateOrCreateParams(entity, foundEntity);
        deleteParams(entity, foundEntity);
    }

    private void deleteParams(final CohortCharacterizationEntity entity, final CohortCharacterizationEntity foundEntity) {
        final Map<String, CcParamEntity> nameToParamFromInputMap = buildParamNameToParamMap(entity);
        List<CcParamEntity> paramsForDelete  = getLinksToDelete(foundEntity,
                parameter -> !nameToParamFromInputMap.containsKey(parameter.getName()),
                CohortCharacterizationEntity::getParameters);
        foundEntity.getParameters().removeAll(paramsForDelete);
        paramRepository.delete(paramsForDelete);
    }

    private void updateOrCreateParams(final CohortCharacterizationEntity entity, final CohortCharacterizationEntity foundEntity) {
        final Map<String, CcParamEntity> nameToParamFromDbMap = buildParamNameToParamMap(foundEntity);
        final List<CcParamEntity> paramsForCreateOrUpdate = new ArrayList<>();
        for (final CcParamEntity parameter : entity.getParameters()) {
            final CcParamEntity entityFromMap = nameToParamFromDbMap.get(parameter.getName());
            parameter.setCohortCharacterization(foundEntity);
            if (entityFromMap == null) {
                paramsForCreateOrUpdate.add(parameter);
            } else if (!StringUtils.equals(entityFromMap.getValue(), parameter.getValue())) {
                entityFromMap.setValue(parameter.getValue());
                paramsForCreateOrUpdate.add(entityFromMap);
            }
        }
        paramRepository.save(paramsForCreateOrUpdate);
    }

    @Override
    public CohortCharacterizationEntity importCc(final CohortCharacterizationEntity entity) {
        cleanIds(entity);

        final CohortCharacterizationEntity newCohortCharacterization = new CohortCharacterizationEntity();
        newCohortCharacterization.setName(entity.getName());
        final CohortCharacterizationEntity persistedCohortCharacterization = this.createCc(newCohortCharacterization);

        updateParams(entity, persistedCohortCharacterization);
        updateStratas(entity, persistedCohortCharacterization);
        updateConceptSet(entity, persistedCohortCharacterization);
        
        importCohorts(entity, persistedCohortCharacterization);

        final CohortCharacterizationEntity savedEntity = saveCc(persistedCohortCharacterization);
        List<Integer> savedAnalysesIds = importAnalyses(entity, savedEntity);

        eventPublisher.publishEvent(new CcImportEvent(savedAnalysesIds));

        return savedEntity;
    }

    @Override
    public String getNameForCopy(String dtoName) {
        return NameUtils.getNameForCopy(dtoName, this::getNamesLike, repository.findByName(dtoName));
    }

    @Override
    public String getNameWithSuffix(String dtoName) {
        return NameUtils.getNameWithSuffix(dtoName, this::getNamesLike);
    }

    @Override
    public String serializeCc(final Long id) {

        return Utils.serialize(exportCc(id), true);
    }

    @Override
    public String serializeCc(final CohortCharacterizationEntity cohortCharacterizationEntity) {
        return new SerializedCcToCcConverter().convertToDatabaseColumn(cohortCharacterizationEntity);
    }

    private CohortCharacterizationImpl exportCc(final Long id) {
      final CohortCharacterizationEntity cohortCharacterizationEntity = repository.findById(id)
              .orElseThrow(() -> new IllegalArgumentException("Cohort characterization cannot be found by id: " + id));
      CohortCharacterizationImpl cc = genericConversionService.convert(cohortCharacterizationEntity, CohortCharacterizationImpl.class);
      ExportUtil.clearCreateAndUpdateInfo(cc);
      cc.getFeatureAnalyses().forEach(ExportUtil::clearCreateAndUpdateInfo);
      cc.getCohorts().forEach(ExportUtil::clearCreateAndUpdateInfo);
      cc.setOrganizationName(env.getRequiredProperty("organization.name"));
      return cc;
    }

    @Override
    public CohortCharacterizationEntity findById(final Long id) {
        return repository.findById(id)
                .orElseThrow(() -> new IllegalArgumentException("Cohort characterization with id: " + id + " cannot be found"));
    }

    @Override
    public CohortCharacterizationEntity findByIdWithLinkedEntities(final Long id) {
        return repository.findOne(id, defaultEntityGraph);
    }

    @Override
    @DataSourceAccess
    public CohortCharacterization findDesignByGenerationId(@CcGenerationId final Long id) {
        final AnalysisGenerationInfoEntity entity = analysisGenerationInfoEntityRepository.findById(id)
                .orElseThrow(() -> new IllegalArgumentException("Analysis with id: " + id + " cannot be found"));
        return genericConversionService.convert(Utils.deserialize(entity.getDesign(),
                new TypeReference<CcExportDTO>() {}), CohortCharacterizationEntity.class);
    }

    @Override
    public Page<CohortCharacterizationEntity> getPageWithLinkedEntities(final Pageable pageable) {
      return repository.findAll(pageable, defaultEntityGraph);
      
    }

    @Override
    public Page<CohortCharacterizationEntity> getPage(final Pageable pageable) {
      List<CohortCharacterizationEntity> ccList = repository.findAll()
              .stream().filter(!defaultGlobalReadPermissions ? entity -> permissionService.hasReadAccess(entity) : entity -> true)
              .collect(Collectors.toList());
      return getPageFromResults(pageable, ccList);
    }
    
    private Page<CohortCharacterizationEntity> getPageFromResults(Pageable pageable, List<CohortCharacterizationEntity> results) {
      // Calculate the start and end indices for the current page
      int startIndex = pageable.getPageNumber() * pageable.getPageSize();
      int endIndex = Math.min(startIndex + pageable.getPageSize(), results.size());      
      
      return new PageImpl<>(results.subList(startIndex, endIndex), pageable, results.size());
    }

    @Override
    public void hydrateAnalysis(Long analysisId, String packageName, OutputStream out) {

      if (packageName == null || !Utils.isAlphaNumeric(packageName)) {
        throw new IllegalArgumentException("The package name must be alphanumeric only.");
      }
      CohortCharacterizationImpl analysis = exportCc(analysisId);
      analysis.setPackageName(packageName);
      String studySpecs = Utils.serialize(analysis, true);
      Hydra hydra = new Hydra(studySpecs);
      File skeletonFile = null;
      try {
        skeletonFile = TempFileUtils.copyResourceToTempFile(HYDRA_PACKAGE, "cc-", ".zip");
        hydra.setExternalSkeletonFileName(skeletonFile.getAbsolutePath());
        hydra.hydrate(out);
      } catch (IOException e) {
        log.error("Failed to hydrate cohort characterization", e);
        throw new InternalServerErrorException(e);
      } finally {
        FileUtils.deleteQuietly(skeletonFile);
      }
    }

    @Override
    @DataSourceAccess
    public JobExecutionResource generateCc(final Long id, @SourceKey final String sourceKey) {

        CcService ccService = this;
        Source source = getSourceRepository().findBySourceKey(sourceKey);

        JobParametersBuilder builder = new JobParametersBuilder();

        builder.addString(JOB_NAME, String.format("Generating cohort characterization %d : %s (%s)", id, source.getSourceName(), source.getSourceKey()));
        builder.addString(COHORT_CHARACTERIZATION_ID, String.valueOf(id));
        builder.addString(SOURCE_ID, String.valueOf(source.getSourceId()));
        builder.addString(JOB_AUTHOR, getCurrentUserLogin());

        CancelableJdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source);

        SimpleJobBuilder generateCohortJob = generationUtils.buildJobForCohortBasedAnalysisTasklet(
                GENERATE_COHORT_CHARACTERIZATION,
                source,
                builder,
                jdbcTemplate,
                chunkContext -> {
                    Long ccId = Long.valueOf(chunkContext.getStepContext().getJobParameters().get(COHORT_CHARACTERIZATION_ID).toString());
                    return ccService.findById(ccId).getCohortDefinitions();
                },
                new GenerateCohortCharacterizationTasklet(
                        jdbcTemplate,
                        getTransactionTemplate(),
                        ccService,
                        analysisGenerationInfoEntityRepository,
                        sourceService,
                        userRepository
                )
        );

        final JobParameters jobParameters = builder.toJobParameters();

        return jobService.runJob(generateCohortJob.build(), jobParameters);
    }

    @Override
    public List<CcGenerationEntity> findGenerationsByCcId(final Long id) {
        return ccGenerationRepository.findByCohortCharacterizationIdOrderByIdDesc(id, EntityUtils.fromAttributePaths("source"));
    }

    @Override
    public CcGenerationEntity findGenerationById(final Long id) {
