sample_function_sql.py

def get_elt_queries():
	"""Gets parameterised queries for ELT"""
	return {
		# Parameters: service
		'input_data': """
			select
				data,
				added_ts
			from
				XXX.TABLE
			where
				service = @service
			order by
				added_ts desc
		""",
		# Parameters: tid
		'output_merge': """
			merge into target t
			using (
				select *, 
				@tid as tid,
				timestamp_utc as last_updated_ts, 
				tid as last_tid 
				from input_table
			) s
			on t.some_id = s.some_id
			when matched then
			update set
				t.last_updated_ts = s.last_updated_ts,
				t.last_tid = s.last_tid
			when not matched then
				insert row;
		"""
	}

# Deploy a wrapper around this function to wake up on file drop of some file
def sample_transform_table():
	input_data = get_input_df('input_data', 'some_service') # Returns pandas dataframe
	r = service_trace_id('sample_transform_table', ...., input_data['tid'].unique().tolist())
	tid = json.loads(r).get('tid')
	# When writing data back,
	# For new rows, insert this new tid into into tid column.
	# For updating rows, insert this new tid into prev_tid column.
  
	# Check out merge example to do all in one step (when possible)