global_config.ini

[DEFAULT]
pipeline_name = wf_rtf
retention = 0.0001
;below variables are dynamically read by spark-driver from global_config.ini
;s3_bucket = s3://bucket-id_env
;s3_pipeline_bucket= s3://bucket-id_env/pipeline
rtf_pipeline_path = ${s3_pipeline_bucket}/${pipeline_name}
gdpr_rtf_requests_incr = ${rtf_pipeline_path}/working/gdpr/rtf_user_requests_incr
gdpr_rtf_requests_backup = ${rtf_pipeline_path}/backup/rtf_user_requests_incr
gdpr_rtf_requests_processed = ${s3_bucket}/target/bastlevel_low/gdpr/rtf_user_requests
gdpr_rtf_log = ${s3_bucket}/target/bastlevel_low/gdpr/rtf_log
gdpr_rtf_retention = ${retention}
all_config_ssm_key = /data-pipelines/snowflake/login,/data-pipelines/pipelines_global_config
snowflake_source_name = net.snowflake.spark.snowflake
gdpr_rtf_metadata_github_repo_url = <link>/rtf_metadata.yml
gdpr_rtf_metadata = ${rtf_pipeline_path}/working/rtf_metadata/rtf_metadata.yml
gdpr_rtf_processed = ${rtf_pipeline_path}/working/forgotten_users
;gdpr_rtf_metadata_repo_override = optional-override-to-select-manual-metadata-repo
mbus_port = <port>
mbus_destination_name = <mbus queue name>
mbus_service_id_snowflake = snowflake
mbus_service_id_s3 = datalake
mbus_no_of_parallel_task = 200
mbus_force_timeout = 120
lock_file_path_prefix = _locks
lock_file_name = _rtf
lock_file_pattern = .transaction_lock
;no_of_function_retry=10 by default, change accordingly
;dry_run_flag=Y by default, better to use as driver arguments like in airflow
;rtf_full_run=N by default
;rtf_full_run_from_date=1970-01-01 by default only used when rtf_full_run=Y
;send_mbus_ack=Y by default, change accordingly

[test]
gdpr_rtf_requests_incr = tests/test_wf_rtf/input/gdpr_rtf_requests_incr
mbus_server = <mbus test lb>
no_of_function_retry = 1

[dev]
mbus_server = <mbus dev lb>

[prod]
mbus_server = <mbus lb>