/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.gobblin.configuration;

import java.nio.charset.Charset;
import java.util.concurrent.TimeUnit;

import com.google.common.base.Charsets;


/**
 * A central place for all Gobblin configuration property keys.
 */
public class ConfigurationKeys {

  /**
   * System configuration properties.
   */
  // Default file system URI for all file storage
  // Overwritable by more specific configuration properties
  public static final String FS_URI_KEY = "fs.uri";

  // Local file system URI
  public static final String LOCAL_FS_URI = "file:///";

  // Comma-separated list of framework jars to include
  public static final String FRAMEWORK_JAR_FILES_KEY = "framework.jars";

  public static final String PST_TIMEZONE_NAME = "America/Los_Angeles";

  /**
   * State store configuration properties.
   */
  // State store type.  References an alias or factory class name
  public static final String STATE_STORE_TYPE_KEY = "state.store.type";
  public static final String DATASET_STATE_STORE_PREFIX = "dataset";
  public static final String DATASET_STATE_STORE_TYPE_KEY = DATASET_STATE_STORE_PREFIX + ".state.store.type";
  public static final String STATE_STORE_FACTORY_CLASS_KEY = "state.store.factory.class";
  public static final String INTERMEDIATE_STATE_STORE_PREFIX = "intermediate";
  public static final String INTERMEDIATE_STATE_STORE_TYPE_KEY = INTERMEDIATE_STATE_STORE_PREFIX + ".state.store.type";
  public static final String DEFAULT_STATE_STORE_TYPE = "fs";
  public static final String STATE_STORE_TYPE_NOOP = "noop";
  // are the job.state files stored using the state store?
  public static final String JOB_STATE_IN_STATE_STORE = "state.store.jobStateInStateStore";
  public static final boolean DEFAULT_JOB_STATE_IN_STATE_STORE = false;

  public static final String CONFIG_RUNTIME_PREFIX = "gobblin.config.runtime.";
  // Root directory where task state files are stored
  public static final String STATE_STORE_ROOT_DIR_KEY = "state.store.dir";
  // File system URI for file-system-based task store
  public static final String STATE_STORE_FS_URI_KEY = "state.store.fs.uri";
  // Thread pool size for listing dataset state store
  public static final String THREADPOOL_SIZE_OF_LISTING_FS_DATASET_STATESTORE =
      "state.store.threadpoolSizeOfListingFsDatasetStateStore";
  public static final int DEFAULT_THREADPOOL_SIZE_OF_LISTING_FS_DATASET_STATESTORE = 10;
  // Enable / disable state store
  public static final String STATE_STORE_ENABLED = "state.store.enabled";
  public static final String STATE_STORE_COMPRESSED_VALUES_KEY = "state.store.compressedValues";
  public static final boolean DEFAULT_STATE_STORE_COMPRESSED_VALUES = true;
  // DB state store configuration
  public static final String STATE_STORE_DB_JDBC_DRIVER_KEY = "state.store.db.jdbc.driver";
  public static final String DEFAULT_STATE_STORE_DB_JDBC_DRIVER = "com.mysql.cj.jdbc.Driver";
  public static final String STATE_STORE_DB_URL_KEY = "state.store.db.url";
  public static final String STATE_STORE_DB_USER_KEY = "state.store.db.user";

  public static final String STATE_STORE_DB_SSL_MODE_ENABLED = "state.store.db.sslModeEnabled";

  public static final String STATE_STORE_DB_PASSWORD_KEY = "state.store.db.password";
  public static final String STATE_STORE_DB_TABLE_KEY = "state.store.db.table";
  public static final String DEFAULT_STATE_STORE_DB_TABLE = "gobblin_job_state";
  public static final String MYSQL_GET_MAX_RETRIES = "mysql.get.max.retries";
  public static final int DEFAULT_MYSQL_GET_MAX_RETRIES = 3;

  public static final String DATASETURN_STATESTORE_NAME_PARSER = "state.store.datasetUrnStateStoreNameParser";

  /**
   * Error Pattern Store related to configuration properties.
   */
  public static final String ERROR_REGEX_DB_TABLE_KEY = "state.store.db.regexTableName";
  public static final String ERROR_CATEGORIES_DB_TABLE_KEY = "state.store.db.categoriesTableName";
  public static final String ERROR_REGEX_VARCHAR_SIZE_KEY = "error.regex.max.varchar.size";
  public static final String ERROR_CATEGORY_VARCHAR_SIZE_KEY = "error.category.max.varchar.size";
  public static final String DEFAULT_DB_ERROR_CATEGORY_KEY = "error.category.default";
  /**
   * Job scheduler configuration properties.
   */
  // Job retriggering
  public static final String JOB_RETRIGGERING_ENABLED = "job.retriggering.enabled";
  public static final String DEFAULT_JOB_RETRIGGERING_ENABLED = "true";
  public static final String LOAD_SPEC_BATCH_SIZE = "load.spec.batch.size";
  public static final int DEFAULT_LOAD_SPEC_BATCH_SIZE = 500;
  public static final String SKIP_SCHEDULING_FLOWS_AFTER_NUM_DAYS = "skip.scheduling.flows.after.num.days";
  public static final int DEFAULT_NUM_DAYS_TO_SKIP_AFTER = 365;
  // Mysql Dag Action Store configuration
  public static final String MYSQL_DAG_ACTION_STORE_PREFIX = "MysqlDagActionStore.";
  public static final String MYSQL_DAG_ACTION_STORE_TABLE_RETENTION_PERIOD_SECONDS_KEY = MYSQL_DAG_ACTION_STORE_PREFIX + "retentionPeriodSeconds";
  public static final long DEFAULT_MYSQL_DAG_ACTION_STORE_TABLE_RETENTION_PERIOD_SEC_KEY = 3 * 24 * 60 * 60; // (3 days in seconds)
  //  Dag Action Store Change Monitor configuration
  public static final String DAG_ACTION_STORE_MONITOR_EXECUTOR_THREADS="dagActionStoreChangeMonitor.executor.numThreads";
  public static final String DAG_ACTION_STORE_MONITOR_EXECUTOR_TIMEOUT_SECONDS="dagActionStoreChangeMonitor.executor.timeout.seconds";
  // Scheduler lease determination store configuration
  public static final String SCHEDULER_LEASE_ARBITER_NAME = "SchedulerFlowLaunchLeaseArbiter";
  public static final String PROCESSING_LEASE_ARBITER_NAME = "DagActionProcessingLeaseArbiter";
  public static final String MYSQL_LEASE_ARBITER_PREFIX = "MysqlMultiActiveLeaseArbiter";
  public static final String MULTI_ACTIVE_CONSTANTS_DB_TABLE_KEY = MYSQL_LEASE_ARBITER_PREFIX + ".constantsTable";
  public static final String LEASE_DETERMINATION_STORE_DB_TABLE_KEY = MYSQL_LEASE_ARBITER_PREFIX + "." + STATE_STORE_DB_TABLE_KEY;
  public static final String LEASE_DETERMINATION_TABLE_RETENTION_PERIOD_MILLIS_KEY = MYSQL_LEASE_ARBITER_PREFIX + ".retentionPeriodMillis";
  public static final long DEFAULT_LEASE_DETERMINATION_TABLE_RETENTION_PERIOD_MILLIS = 3 * 24 * 60 * 60 * 1000; // (3 days in ms)
  // Refers to the event we originally tried to acquire a lease which achieved `consensus` among participants through
  // the database
  public static final String SCHEDULER_PRESERVED_CONSENSUS_EVENT_TIME_MILLIS_KEY = "preservedConsensusEventTimeMillis";
  // Time the reminder event Trigger is supposed to fire from the scheduler
  public static final String SCHEDULER_EXPECTED_REMINDER_TIME_MILLIS_KEY = "expectedReminderTimeMillis";
  // Event time of flow action to orchestrate using the multi-active lease arbiter
  public static final String ORCHESTRATOR_TRIGGER_EVENT_TIME_MILLIS_KEY = "orchestratorTriggerEventTimeMillis";
  public static final String ORCHESTRATOR_TRIGGER_EVENT_TIME_DEFAULT_VAL = "0";
  public static final String FLOW_IS_REMINDER_EVENT_KEY = "isReminderEvent";
  public static final String SCHEDULER_EVENT_EPSILON_MILLIS_KEY = MYSQL_LEASE_ARBITER_PREFIX + ".epsilonMillis";
  public static final int DEFAULT_SCHEDULER_EVENT_EPSILON_MILLIS = 2000;
  // Note: linger should be on the order of seconds even though we measure in millis
  public static final String SCHEDULER_EVENT_LINGER_MILLIS_KEY = MYSQL_LEASE_ARBITER_PREFIX + ".lingerMillis";
  public static final int DEFAULT_SCHEDULER_EVENT_LINGER_MILLIS = 90000;
  public static final String SCHEDULER_MAX_BACKOFF_MILLIS_KEY = MYSQL_LEASE_ARBITER_PREFIX + ".maxBackoffMillis";
  public static final int DEFAULT_SCHEDULER_MAX_BACKOFF_MILLIS = 10000;

  // Job executor thread pool size
  public static final String JOB_EXECUTOR_THREAD_POOL_SIZE_KEY = "jobexecutor.threadpool.size";
  public static final int DEFAULT_JOB_EXECUTOR_THREAD_POOL_SIZE = 5;
  // Job configuration file monitor polling interval in milliseconds
  public static final String JOB_CONFIG_FILE_MONITOR_POLLING_INTERVAL_KEY = "jobconf.monitor.interval";
  public static final long DEFAULT_JOB_CONFIG_FILE_MONITOR_POLLING_INTERVAL = 30000;
  public static final long DISABLED_JOB_CONFIG_FILE_MONITOR_POLLING_INTERVAL = -1L;
  // Directory where all job configuration files are stored WHEN ALL confs reside in local FS.
  public static final String JOB_CONFIG_FILE_DIR_KEY = "jobconf.dir";

  // Path where all job configuration files stored
  public static final String JOB_CONFIG_FILE_GENERAL_PATH_KEY = "jobconf.fullyQualifiedPath";
  // Job configuration file extensions
  public static final String JOB_CONFIG_FILE_EXTENSIONS_KEY = "jobconf.extensions";
  public static final String DEFAULT_JOB_CONFIG_FILE_EXTENSIONS = "pull,job";
  // Whether the scheduler should wait for running jobs to complete during shutdown.
  // Note this only applies to jobs scheduled by the built-in Quartz-based job scheduler.
  public static final String SCHEDULER_WAIT_FOR_JOB_COMPLETION_KEY = "scheduler.wait.for.job.completion";
  public static final String DEFAULT_SCHEDULER_WAIT_FOR_JOB_COMPLETION = Boolean.TRUE.toString();

  public static final String TASK_TIMEOUT_SECONDS = "task.timeout.seconds";
  public static final long DEFAULT_TASK_TIMEOUT_SECONDS = 60 * 60;

  /**
   * Task executor and state tracker configuration properties.
   */
  public static final String TASK_EXECUTOR_THREADPOOL_SIZE_KEY = "taskexecutor.threadpool.size";
  public static final String TASK_STATE_TRACKER_THREAD_POOL_CORE_SIZE_KEY = "tasktracker.threadpool.coresize";
  public static final String TASK_RETRY_THREAD_POOL_CORE_SIZE_KEY = "taskretry.threadpool.coresize";
  public static final int DEFAULT_TASK_EXECUTOR_THREADPOOL_SIZE = 2;
  public static final int DEFAULT_TASK_STATE_TRACKER_THREAD_POOL_CORE_SIZE = 1;
  public static final int DEFAULT_TASK_RETRY_THREAD_POOL_CORE_SIZE = 1;

  /**
   * Common flow configuration properties.
   */
  public static final String FLOW_NAME_KEY = "flow.name";
  public static final String FLOW_GROUP_KEY = "flow.group";
  public static final String FLOW_EDGE_ID_KEY = "flow.edgeId";
  public static final String FLOW_DESCRIPTION_KEY = "flow.description";
  public static final String FLOW_EXECUTION_ID_KEY = "flow.executionId";
  public static final String FLOW_FAILURE_OPTION = "flow.failureOption";
  public static final String FLOW_APPLY_RETENTION = "flow.applyRetention";
  public static final String FLOW_APPLY_INPUT_RETENTION = "flow.applyInputRetention";
  public static final String FLOW_ALLOW_CONCURRENT_EXECUTION = "flow.allowConcurrentExecution";
  public static final String FLOW_EXPLAIN_KEY = "flow.explain";
  public static final String FLOW_UNSCHEDULE_KEY = "flow.unschedule";
  public static final String FLOW_OWNING_GROUP_KEY = "flow.owningGroup";
  public static final String FLOW_SPEC_EXECUTOR = "flow.edge.specExecutors";
  public static final String RM_HOST_KEY = "hadoop.resource.manager.rpc";

  /**
   * Common topology configuration properties.
   */
  public static final String TOPOLOGY_NAME_KEY = "topology.name";
  public static final String TOPOLOGY_GROUP_KEY = "topology.group";
  public static final String TOPOLOGY_DESCRIPTION_KEY = "topology.description";

  /**
   * Common job configuration properties.
   */
  public static final String JOB_NAME_KEY = "job.name";
  public static final String JOB_GROUP_KEY = "job.group";
  public static final String JOB_TAG_KEY = "job.tag";
  public static final String JOB_DESCRIPTION_KEY = "job.description";
  public static final String JOB_ATTEMPT_ID = "job.attemptId";
  public static final String JOB_CURRENT_GENERATION = "job.currentGeneration";
  // Job launcher type
  public static final String JOB_LAUNCHER_TYPE_KEY = "launcher.type";
  public static final String JOB_SCHEDULE_KEY = "job.schedule";
  public static final String JOB_LISTENERS_KEY = "job.listeners";
  // Type of the job lock
  public static final String JOB_LOCK_TYPE = "job.lock.type";
  //Directory that stores task staging data and task output data.
  public static final String TASK_DATA_ROOT_DIR_KEY = "task.data.root.dir";
  public static final String SOURCE_CLASS_KEY = "source.class";
  public static final String CONVERTER_CLASSES_KEY = "converter.classes";
  public static final String CONVERTER_INITIALIZERS_SERIALIZED_MEMENTOS_KEY = "converter.initializers.serialized.mementos";
  public static final String RECORD_STREAM_PROCESSOR_CLASSES_KEY = "recordStreamProcessor.classes";
  public static final String FORK_OPERATOR_CLASS_KEY = "fork.operator.class";
  public static final String DEFAULT_FORK_OPERATOR_CLASS = "org.apache.gobblin.fork.IdentityForkOperator";
  public static final String JOB_COMMIT_POLICY_KEY = "job.commit.policy";
  public static final String DEFAULT_JOB_COMMIT_POLICY = "full";
  public static final String JOB_TARGET_COMPLETION_DURATION_IN_MINUTES_KEY = "job.duration.target.completion.in.minutes";
  public static final long DEFAULT_JOB_TARGET_COMPLETION_DURATION_IN_MINUTES = 120;

  public static final String PARTIAL_FAIL_TASK_FAILS_JOB_COMMIT = "job.commit.partial.fail.task.fails.job.commit";
  // If true, commit of different datasets will be performed in parallel
  // only turn on if publisher is thread-safe
  public static final String PARALLELIZE_DATASET_COMMIT = "job.commit.parallelize";
  public static final boolean DEFAULT_PARALLELIZE_DATASET_COMMIT = false;
  /** Only applicable if {@link #PARALLELIZE_DATASET_COMMIT} is true. */
  public static final String DATASET_COMMIT_THREADS = "job.commit.parallelCommits";
  public static final int DEFAULT_DATASET_COMMIT_THREADS = 20;

  public static final String WORK_UNIT_RETRY_POLICY_KEY = "workunit.retry.policy";
  public static final String WORK_UNIT_RETRY_ENABLED_KEY = "workunit.retry.enabled";
  public static final String WORK_UNIT_CREATION_TIME_IN_MILLIS = "workunit.creation.time.in.millis";
  public static final String WORK_UNIT_CREATION_AND_RUN_INTERVAL = "workunit.creation.and.run.interval";
  public static final String WORK_UNIT_ENABLE_TRACKING_LOGS = "workunit.enableTrackingLogs";

  public static final String JOB_DEPENDENCIES = "job.dependencies";
  public static final String JOB_FORK_ON_CONCAT = "job.forkOnConcat";
  public static final String JOB_RUN_ONCE_KEY = "job.runonce";
  public static final String JOB_DISABLED_KEY = "job.disabled";
  public static final String JOB_JAR_FILES_KEY = "job.jars";
  public static final String JOB_LOCAL_FILES_KEY = "job.local.files";
  public static final String JOB_HDFS_FILES_KEY = "job.hdfs.files";
  public static final String JOB_JAR_HDFS_FILES_KEY = "job.hdfs.jars";
  public static final String JOB_LOCK_ENABLED_KEY = "job.lock.enabled";
  public static final String JOB_MAX_FAILURES_KEY = "job.max.failures";
  public static final int DEFAULT_JOB_MAX_FAILURES = 1;
  public static final String MAX_TASK_RETRIES_KEY = "task.maxretries";
  public static final int DEFAULT_MAX_TASK_RETRIES = 5;
  public static final String TASK_RETRY_INTERVAL_IN_SEC_KEY = "task.retry.intervalinsec";
  public static final long DEFAULT_TASK_RETRY_INTERVAL_IN_SEC = 300;
  public static final String OVERWRITE_CONFIGS_IN_STATESTORE = "overwrite.configs.in.statestore";
  public static final boolean DEFAULT_OVERWRITE_CONFIGS_IN_STATESTORE = false;
  public static final String CLEANUP_STAGING_DATA_PER_TASK = "cleanup.staging.data.per.task";
  public static final boolean DEFAULT_CLEANUP_STAGING_DATA_PER_TASK = true;
  public static final String CLEANUP_STAGING_DATA_BY_INITIALIZER = "cleanup.staging.data.by.initializer";
  public static final String CLEANUP_OLD_JOBS_DATA = "cleanup.old.job.data";
  public static final boolean DEFAULT_CLEANUP_OLD_JOBS_DATA = false;
  public static final String MAXIMUM_JAR_COPY_RETRY_TIMES_KEY = JOB_JAR_FILES_KEY + ".uploading.retry.maximum";
  public static final String USER_DEFINED_STATIC_STAGING_DIR = "user.defined.static.staging.dir";
  public static final String USER_DEFINED_STAGING_DIR_FLAG = "user.defined.staging.dir.flag";

  public static final String QUEUED_TASK_TIME_MAX_SIZE = "taskexecutor.queued_task_time.history.max_size";
  public static final int DEFAULT_QUEUED_TASK_TIME_MAX_SIZE = 2048;
  public static final String QUEUED_TASK_TIME_MAX_AGE = "taskexecutor.queued_task_time.history.max_age";
  public static final long DEFAULT_QUEUED_TASK_TIME_MAX_AGE = TimeUnit.HOURS.toMillis(1);

  /**
   * Optional property to specify whether existing data in databases can be overwritten during ingestion jobs
   */
  public static final String ALLOW_JDBC_RECORD_OVERWRITE = "allow.jdbc.record.overwrite";

  /**
   * Optional property to specify a default Authenticator class for a job
   */
  public static final String DEFAULT_AUTHENTICATOR_CLASS = "job.default.authenticator.class";

  /** Optional, for user to specified which template to use, inside .job file */
  public static final String JOB_TEMPLATE_PATH = "job.template";

  /**
   * Configuration property used only for job configuration file's template
   */
  public static final String REQUIRED_ATRRIBUTES_LIST = "gobblin.template.required_attributes";

  /**
   * Configuration for emitting job events
   */
  public static final String EVENT_METADATA_GENERATOR_CLASS_KEY = "event.metadata.generator.class";
  public static final String DEFAULT_EVENT_METADATA_GENERATOR_CLASS_KEY = "noop";

  /**
   * Configuration for dynamic configuration generation
   */
  public static final String DYNAMIC_CONFIG_GENERATOR_CLASS_KEY = "dynamicConfigGenerator.class";
  public static final String DEFAULT_DYNAMIC_CONFIG_GENERATOR_CLASS_KEY = "noop";

  /**
   * Configuration properties used internally.
   */
  public static final String JOB_ID_KEY = "job.id";
  public static final String JOB_KEY_KEY = "job.key";
  public static final String TASK_ID_KEY = "task.id";
  public static final String TASK_KEY_KEY = "task.key";
  public static final String TASK_START_TIME_MILLIS_KEY = "task.startTimeMillis";
  public static final String TASK_ATTEMPT_ID_KEY = "task.AttemptId";
  public static final String JOB_CONFIG_FILE_PATH_KEY = "job.config.path";
  public static final String TASK_FAILURE_EXCEPTION_KEY = "task.failure.exception";
  public static final String TASK_ISSUES_KEY = "task.issues";
  public static final String JOB_FAILURE_EXCEPTION_KEY = "job.failure.exception";
  public static final String TASK_RETRIES_KEY = "task.retries";
  public static final String TASK_IGNORE_CLOSE_FAILURES = "task.ignoreCloseFailures";
  //A boolean config to allow skipping task interrupt on cancellation. Useful for example when thread manages
  // a Kafka consumer which when interrupted during a poll() leaves the consumer in a corrupt state that prevents
  // the consumer being closed subsequently, leading to a potential resource leak.
  public static final String TASK_INTERRUPT_ON_CANCEL = "task.interruptOnCancel";
  public static final String JOB_FAILURES_KEY = "job.failures";
  public static final String JOB_TRACKING_URL_KEY = "job.tracking.url";
  public static final String FORK_STATE_KEY = "fork.state";
  public static final String JOB_STATE_FILE_PATH_KEY = "job.state.file.path";
  public static final String JOB_STATE_DISTRIBUTED_CACHE_NAME = "job.state.distributed.cache.name";

  /**
   * Dataset-related configuration properties;
   */
  // This property is used to specify the URN of a dataset a job or WorkUnit extracts data for
  public static final String DATASET_URN_KEY = "dataset.urn";
  public static final String GLOBAL_WATERMARK_DATASET_URN = "__globalDatasetWatermark";
  public static final String DEFAULT_DATASET_URN = "";
  public static final String DATASET_QUALITY_STATUS_KEY = "dataset.quality.status";
  public static final String ENFORCE_DATA_QUALITY_FAILURE_KEY = "data.quality.enforce.failure";
  public static final Boolean DEFAULT_ENFORCE_DATA_QUALITY_FAILURE = false;

  /**
   * Work unit related configuration properties.
   */
  public static final String WORK_UNIT_LOW_WATER_MARK_KEY = "workunit.low.water.mark";
  public static final String WORK_UNIT_HIGH_WATER_MARK_KEY = "workunit.high.water.mark";
  public static final String WORK_UNIT_SKIP_KEY = "workunit.skip";

  /**
   * Work unit runtime state related configuration properties.
   */
  public static final String WORK_UNIT_WORKING_STATE_KEY = "workunit.working.state";
  public static final String WORK_UNIT_STATE_RUNTIME_HIGH_WATER_MARK = "workunit.state.runtime.high.water.mark";
  public static final String WORK_UNIT_STATE_ACTUAL_HIGH_WATER_MARK_KEY = "workunit.state.actual.high.water.mark";
  public static final String WORK_UNIT_DATE_PARTITION_KEY = "workunit.source.date.partition";
  public static final String WORK_UNIT_DATE_PARTITION_NAME = "workunit.source.date.partitionName";
  public static final String WORK_UNIT_GENERATOR_FAILURE_IS_FATAL = "workunit.generator.failure.is.fatal";
  public static final boolean DEFAULT_WORK_UNIT_FAST_FAIL_ENABLED = true;

  /**
   * Task execution properties.
   */
  public static final String TASK_SYNCHRONOUS_EXECUTION_MODEL_KEY = "task.execution.synchronousExecutionModel";
  public static final boolean DEFAULT_TASK_SYNCHRONOUS_EXECUTION_MODEL = true;

  /**
   * Watermark interval related configuration properties.
   */
  public static final String WATERMARK_INTERVAL_VALUE_KEY = "watermark.interval.value";

  /**
   * DEFAULT LOOKBACK TIME KEY property
   */
  public static final String DEFAULT_COPY_LOOKBACK_TIME_KEY = "copy.lookbackTime";

  /**
   * Extract related configuration properties.
   */
  public static final String EXTRACT_TABLE_TYPE_KEY = "extract.table.type";
  public static final String EXTRACT_NAMESPACE_NAME_KEY = "extract.namespace";
  public static final String EXTRACT_TABLE_NAME_KEY = "extract.table.name";
  public static final String EXTRACT_EXTRACT_ID_KEY = "extract.extract.id";
  public static final String EXTRACT_IS_FULL_KEY = "extract.is.full";
  public static final String DEFAULT_EXTRACT_IS_FULL = "false";
  public static final String EXTRACT_FULL_RUN_TIME_KEY = "extract.full.run.time";
  public static final String EXTRACT_PRIMARY_KEY_FIELDS_KEY = "extract.primary.key.fields";
  public static final String EXTRACT_DELTA_FIELDS_KEY = "extract.delta.fields";
  public static final String EXTRACT_SCHEMA = "extract.schema";
  public static final String EXTRACT_LIMIT_ENABLED_KEY = "extract.limit.enabled";
  public static final boolean DEFAULT_EXTRACT_LIMIT_ENABLED = false;
  public static final String EXTRACT_ID_TIME_ZONE = "extract.extractIdTimeZone";
  public static final String DEFAULT_EXTRACT_ID_TIME_ZONE = "UTC";
  public static final String EXTRACT_SALESFORCE_BULK_API_MIN_WAIT_TIME_IN_MILLIS_KEY =
      "extract.salesforce.bulkApi.minWaitTimeInMillis";
  public static final long DEFAULT_EXTRACT_SALESFORCE_BULK_API_MIN_WAIT_TIME_IN_MILLIS = 60 * 1000L; // 1 min
  public static final String EXTRACT_SALESFORCE_BULK_API_MAX_WAIT_TIME_IN_MILLIS_KEY =
      "extract.salesforce.bulkApi.maxWaitTimeInMillis";
  public static final long DEFAULT_EXTRACT_SALESFORCE_BULK_API_MAX_WAIT_TIME_IN_MILLIS = 10 * 60 * 1000L; // 10 min

  /**
   * Converter configuration properties.
   */
  public static final String CONVERTER_AVRO_DATE_FORMAT = "converter.avro.date.format";
  public static final String CONVERTER_AVRO_DATE_TIMEZONE = "converter.avro.date.timezone";
  public static final String CONVERTER_AVRO_TIME_FORMAT = "converter.avro.time.format";
  public static final String CONVERTER_AVRO_TIMESTAMP_FORMAT = "converter.avro.timestamp.format";
  public static final String CONVERTER_AVRO_BINARY_CHARSET = "converter.avro.binary.charset";
  public static final String CONVERTER_AVRO_MAX_CONVERSION_FAILURES = "converter.avro.max.conversion.failures";
  public static final long DEFAULT_CONVERTER_AVRO_MAX_CONVERSION_FAILURES = 0;
  public static final String CONVERTER_CSV_TO_JSON_DELIMITER = "converter.csv.to.json.delimiter";
  public static final String CONVERTER_FILTER_FIELD_NAME = "converter.filter.field";
  public static final String CONVERTER_FILTER_FIELD_VALUE = "converter.filter.value";
  public static final String CONVERTER_IS_EPOCH_TIME_IN_SECONDS = "converter.is.epoch.time.in.seconds";
  public static final String CONVERTER_AVRO_EXTRACTOR_FIELD_PATH = "converter.avro.extractor.field.path";
  public static final String CONVERTER_STRING_FILTER_PATTERN = "converter.string.filter.pattern";
  public static final String CONVERTER_STRING_SPLITTER_DELIMITER = "converter.string.splitter.delimiter";
  public static final String CONVERTER_STRING_SPLITTER_SHOULD_TRIM_RESULTS =
      "converter.string.splitter.shouldITrimResults";
  public static final boolean DEFAULT_CONVERTER_STRING_SPLITTER_SHOULD_TRIM_RESULTS = false;
  public static final String CONVERTER_CSV_TO_JSON_ENCLOSEDCHAR = "converter.csv.to.json.enclosedchar";
  public static final String DEFAULT_CONVERTER_CSV_TO_JSON_ENCLOSEDCHAR = "\0";
  public static final String CONVERTER_AVRO_FIELD_PICK_FIELDS = "converter.avro.fields";
  public static final String CONVERTER_AVRO_JDBC_ENTRY_FIELDS_PAIRS = "converter.avro.jdbc.entry_fields_pairs";
  public static final String CONVERTER_SKIP_FAILED_RECORD = "converter.skipFailedRecord";
  public static final String CONVERTER_AVRO_SCHEMA_KEY = "converter.avroSchema";
  public static final String CONVERTER_IGNORE_FIELDS = "converter.ignoreFields";

  /**
   * Fork operator configuration properties.
   */
  public static final String FORK_BRANCHES_KEY = "fork.branches";
  public static final String FORK_BRANCH_NAME_KEY = "fork.branch.name";
  public static final String FORK_BRANCH_ID_KEY = "fork.branch.id";
  public static final String DEFAULT_FORK_BRANCH_NAME = "fork_";
  public static final String FORK_RECORD_QUEUE_CAPACITY_KEY = "fork.record.queue.capacity";
  public static final int DEFAULT_FORK_RECORD_QUEUE_CAPACITY = 100;
  public static final String FORK_RECORD_QUEUE_TIMEOUT_KEY = "fork.record.queue.timeout";
  public static final long DEFAULT_FORK_RECORD_QUEUE_TIMEOUT = 1000;
  public static final String FORK_RECORD_QUEUE_TIMEOUT_UNIT_KEY = "fork.record.queue.timeout.unit";
  public static final String DEFAULT_FORK_RECORD_QUEUE_TIMEOUT_UNIT = TimeUnit.MILLISECONDS.name();
  public static final String FORK_MAX_WAIT_MININUTES = "fork.max.wait.minutes";
  public static final long DEFAULT_FORK_MAX_WAIT_MININUTES = 60;
  public static final String FORK_FINISHED_CHECK_INTERVAL = "fork.finished.check.interval";
  public static final long DEFAULT_FORK_FINISHED_CHECK_INTERVAL = 1000;
  public static final String FORK_CLOSE_WRITER_ON_COMPLETION = "fork.closeWriterOnCompletion";
  public static final boolean DEFAULT_FORK_CLOSE_WRITER_ON_COMPLETION = false;


  /**
   * Writer configuration properties.
   */
  public static final String WRITER_PREFIX = "writer";
  public static final String WRITER_DESTINATION_TYPE_KEY = WRITER_PREFIX + ".destination.type";
  public static final String WRITER_OUTPUT_FORMAT_KEY = WRITER_PREFIX + ".output.format";
  public static final String WRITER_FILE_SYSTEM_URI = WRITER_PREFIX + ".fs.uri";
  public static final String WRITER_STAGING_DIR = WRITER_PREFIX + ".staging.dir";
  public static final String WRITER_STAGING_TABLE = WRITER_PREFIX + ".staging.table";
  public static final String WRITER_TRUNCATE_STAGING_TABLE = WRITER_PREFIX + ".truncate.staging.table";
  public static final String WRITER_OUTPUT_DIR = WRITER_PREFIX + ".output.dir";
  public static final String WRITER_BUILDER_CLASS = WRITER_PREFIX + ".builder.class";
  public static final String WRITER_INITIALIZER_SERIALIZED_MEMENTO_KEY = "writer.initializer.serialized.memento";
  public static final String DEFAULT_WRITER_BUILDER_CLASS = "org.apache.gobblin.writer.AvroDataWriterBuilder";
  public static final String WRITER_FILE_NAME = WRITER_PREFIX + ".file.name";
  public static final String WRITER_FILE_PATH = WRITER_PREFIX + ".file.path";

  @Deprecated // Use WRITER_FILE_PATH instead as configuration is not type safe with WRITER_FILE_PATH when stored in config
  public static final String WRITER_FILE_PATH_TYPE_DEPRECATED = WRITER_PREFIX + ".file.path.type";

  public static final String WRITER_FILE_PATH_TYPE = WRITER_PREFIX + ".file.pathType";
  public static final String WRITER_FILE_OWNER = WRITER_PREFIX + ".file.owner";
  public static final String WRITER_FILE_GROUP = WRITER_PREFIX + ".file.group";
  public static final String WRITER_FILE_REPLICATION_FACTOR = WRITER_PREFIX + ".file.replication.factor";
  public static final String WRITER_FILE_BLOCK_SIZE = WRITER_PREFIX + ".file.block.size";
  public static final String WRITER_FILE_PERMISSIONS = WRITER_PREFIX + ".file.permissions";
  public static final String WRITER_DIR_PERMISSIONS = WRITER_PREFIX + ".dir.permissions";
  public static final String WRITER_BUFFER_SIZE = WRITER_PREFIX + ".buffer.size";
  public static final String WRITER_PRESERVE_FILE_NAME = WRITER_PREFIX + ".preserve.file.name";
  public static final String WRITER_DEFLATE_LEVEL = WRITER_PREFIX + ".deflate.level";
  public static final String WRITER_CODEC_TYPE = WRITER_PREFIX + ".codec.type";
  public static final String WRITER_EAGER_INITIALIZATION_KEY = WRITER_PREFIX + ".eager.initialization";
  public static final String WRITER_PARTITIONER_CLASS = WRITER_PREFIX + ".partitioner.class";
  public static final String WRITER_SKIP_NULL_RECORD = WRITER_PREFIX + ".skipNullRecord";
  public static final boolean DEFAULT_WRITER_EAGER_INITIALIZATION = false;
  public static final String WRITER_GROUP_NAME = WRITER_PREFIX + ".group.name";
  public static final String DEFAULT_WRITER_FILE_BASE_NAME = "part";
  public static final int DEFAULT_DEFLATE_LEVEL = 9;
  public static final int DEFAULT_BUFFER_SIZE = 4096;
  public static final String DEFAULT_WRITER_FILE_PATH_TYPE = "default";
  public static final String SIMPLE_WRITER_DELIMITER = "simple.writer.delimiter";
  public static final String SIMPLE_WRITER_PREPEND_SIZE = "simple.writer.prepend.size";
  public static final String WRITER_ADD_TASK_TIMESTAMP = WRITER_PREFIX + ".addTaskTimestamp";

  // Internal use only - used to send metadata to publisher
  public static final String WRITER_METADATA_KEY = WRITER_PREFIX + "._internal.metadata";
  public static final String WRITER_PARTITION_PATH_KEY = WRITER_PREFIX + "._internal.partition.path";

  /**
   * Writer configuration properties used internally.
   */
  public static final String WRITER_FINAL_OUTPUT_FILE_PATHS = WRITER_PREFIX + ".final.output.file.paths";
  public static final String WRITER_RECORDS_WRITTEN = WRITER_PREFIX + ".records.written";
  public static final String WRITER_BYTES_WRITTEN = WRITER_PREFIX + ".bytes.written";
  public static final String WRITER_EARLIEST_TIMESTAMP = WRITER_PREFIX + ".earliest.timestamp";
  public static final String WRITER_AVERAGE_TIMESTAMP = WRITER_PREFIX + ".average.timestamp";
  public static final String WRITER_COUNT_METRICS_FROM_FAILED_TASKS = WRITER_PREFIX + ".jobTaskSummary.countMetricsFromFailedTasks";

  /**
   * Configuration properties used by the quality checker.
   */
  public static final String QUALITY_CHECKER_PREFIX = "qualitychecker";
  public static final String TASK_LEVEL_POLICY_RESULT_KEY = "gobblin.task.level.policy.result";
  public static final String TASK_LEVEL_POLICY_LIST = QUALITY_CHECKER_PREFIX + ".task.policies";
  public static final String TASK_LEVEL_POLICY_LIST_TYPE = QUALITY_CHECKER_PREFIX + ".task.policy.types";
  public static final String ROW_LEVEL_POLICY_LIST = QUALITY_CHECKER_PREFIX + ".row.policies";
  public static final String ROW_LEVEL_POLICY_LIST_TYPE = QUALITY_CHECKER_PREFIX + ".row.policy.types";
  public static final String ROW_LEVEL_ERR_FILE = QUALITY_CHECKER_PREFIX + ".row.err.file";
  public static final String QUALITY_CHECKER_TIMEZONE = QUALITY_CHECKER_PREFIX + ".timezone";
  public static final String DEFAULT_QUALITY_CHECKER_TIMEZONE = PST_TIMEZONE_NAME;
  public static final String CLEAN_ERR_DIR = QUALITY_CHECKER_PREFIX + ".clean.err.dir";
