kshitijcode

import aiohttp
import argparse
import asyncio
import json
import logging
import re
import tiktoken
import time
from status_tracker import StatusTracker
from api_request import APIRequest

kshitijcode

 annotations:
    service.beta.kubernetes.io/azure-load-balancer-internal: "true"
    #https://cloud-provider-azure.sigs.k8s.io/topics/pls-integration/
    service.beta.kubernetes.io/azure-pls-create: "true"
    #create private link service with name private-link-demo
    service.beta.kubernetes.io/azure-pls-name: "private-link-demo"

kshitijcode

pipeline_run_result = pipelineutils.run_and_observe_pipeline(
    azure_credential, synapse_endpoint, pipeline_name, masterpipeline_processed_params)

assert pipeline_run_result == constants.PIPELINE_SUCCESS_STATUS

# Check for Data in processed Zone
cursor = sql_connection_client.cursor()
cursor.execute(
    "SELECT COUNT(*) AS COUNT FROM [dbo].[{0}]".format(table_processed))
row = cursor.fetchone()

kshitijcode

table_raw = "integrationtest_users"
table_processed = "processed_integrationtest_user"

# Trigger the Master Pipeline for Processed to processed Zone
masterpipeline_processed_params = {
    "lookUpTables": [{
        "SourceTableSchemaName": "dbo",
        "SourceTableName": table_raw,
        "SinkTableSchemaName": "dbo",
        "SinkTableName": table_processed,

kshitijcode

"lookUpTables": [{
    "SourceTableSchemaName": "dbo",
    "SourceTableName": table_raw,
    "SinkTableSchemaName": "dbo",
    "SinkTableName": table_processed,
    "HasIncrementalData": "false"
}],
"sourceDatabase": database,

kshitijcode

assert pipeline_run_result == constants.PIPELINE_SUCCESS_STATUS

# Check for Data in Raw Zone
parquet_dataframe = adls.read_parquet_file_from_ADLS(
    adls_connection_client, container_name, target_path)
num_of_rows = len(parquet_dataframe.index)

# Assert
print(f"Number of Rows Fetched : { num_of_rows }\n")
assert num_of_rows >=1

kshitijcode

pipeline_run_result = pipelineutils.run_and_observe_pipeline(
    azure_credential, synapse_endpoint, pipeline_name, masterpipeline_raw_params)

kshitijcode

"basePath": base_path,
"filePath": "user_*.parquet",
"targetTable": target_table,
"badDataTable": "bad_users",
"containerName": container_name,
"archivePath": "archive",
"storageAccountName": storage_account_name,
"readFromSparkTables": False,
"database": database,

kshitijcode

{"registration_dttm":28529000000000,"id":1,"first_name":"Amanda","last_name":"Jordan","email":"ajordan0@com.com","gender":"Female","ip_address":"1.197.201.2","cc":"6759521864920116","country":"Indonesia","birthdate":"3/8/1971","salary":49756.53,"title":"Internal Auditor","comments":"1E+02"}
{"registration_dttm":61443000000000,"id":2,"first_name":"Albert","last_name":"Freeman","email":"afreeman1@is.gd","gender":"Male","ip_address":"218.111.175.34","cc":"","country":"Canada","birthdate":"1/16/1968","salary":150280.17,"title":"Accountant IV","comments":""}
{"registration_dttm":4171000000000,"id":3,"first_name":"Evelyn","last_name":"Morgan","email":"emorgan2@altervista.org","gender":"Female","ip_address":"7.161.136.94","cc":"6767119071901597","country":"Russia","birthdate":"2/1/1960","salary":144972.51,"title":"Structural Engineer","comments":""}
{"registration_dttm":2181000000000,"id":4,"first_name":"Denise","last_name":"Riley","email":"driley3@gmpg.org","gender":"Female","ip_address":"140.35.109.83","cc":"35760

kshitijcode

{"registration_dttm":28529000000000,"id":1,"first_name":"Amanda","last_name":"Jordan","email":"ajordan0@com.com","gender":"Female","ip_address":"1.197.201.2","cc":"6759521864920116","country":"Indonesia","birthdate":"3/8/1971","salary":49756.53,"title":"Internal Auditor","comments":"1E+02"}
{"registration_dttm":61443000000000,"id":2,"first_name":"Albert","last_name":"Freeman","email":"afreeman1@is.gd","gender":"Male","ip_address":"218.111.175.34","cc":"","country":"Canada","birthdate":"1/16/1968","salary":150280.17,"title":"Accountant IV","comments":""}
{"registration_dttm":4171000000000,"id":3,"first_name":"Evelyn","last_name":"Morgan","email":"emorgan2@altervista.org","gender":"Female","ip_address":"7.161.136.94","cc":"6767119071901597","country":"Russia","birthdate":"2/1/1960","salary":144972.51,"title":"Structural Engineer","comments":""}
{"registration_dttm":2181000000000,"id":4,"first_name":"Denise","last_name":"Riley","email":"driley3@gmpg.org","gender":"Female","ip_address":"140.35.109.83","cc":"35760