mdrakiburrahman

// Kafka Topic ➡ Delta tables: `scene` via Spark Streaming
// `SOURCE`: Connect to Kafka topic as a streaming Dataframe: `raw_DF`

import org.apache.spark.sql.types._
import org.apache.spark.sql.functions._
import org.apache.spark.sql._
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.functions.from_json

// Pull from Key Vault for non-sandbox

mdrakiburrahman

URL="http://localhost:9200"
from=20
to=30
for i in `seq $from $to`
do
  # Delete index if older than i days
  DATE=`date -d "$dataset_date - $i days" +%Y.%m.%d`
  echo "Deleting day: $DATE"
  curl -XDELETE "$URL/logstash-$DATE" # Comment out to see what range is deleted
done

mdrakiburrahman

apiVersion: security.openshift.io/v1
kind: SecurityContextConstraints
metadata:
  name: arc-data-scc
allowHostDirVolumePlugin: false
allowHostIPC: false
allowHostNetwork: false
allowHostPID: false
allowHostPorts: false
allowPrivilegeEscalation: true

mdrakiburrahman

-- Create DB
CREATE DATABASE raki_pitr_test

CREATE TABLE table1 (ID int, value nvarchar(10))
GO

INSERT INTO table1 VALUES (1, 'demo1')
INSERT INTO table1 VALUES (2, 'demo2')

mdrakiburrahman

# the Namespace and ServiceAccount name that is used for the config
namespace=arc
serviceAccount=arcOnboard

######################
# actual script starts
set -o errexit

secretName=$(kubectl --namespace $namespace get serviceAccount $serviceAccount -o jsonpath='{.secrets[0].name}')
ca=$(kubectl --namespace $namespace get secret/$secretName -o jsonpath='{.data.ca\.crt}')

mdrakiburrahman

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: arc-data-deployer-cluster-role
rules:
  # CRDs in general
  - apiGroups: ["apiextensions.k8s.io"]
    resources: ["customresourcedefinitions"]
    verbs: ['create', 'update', 'patch', 'delete', 'list', 'get', 'watch'] # Can figure out lesser scope if needed
  # All Arc Data apiGroups, I think some of these may be redundant - K8s doesn't give an easy way to get custom apiGroups so I put everything in

mdrakiburrahman

# Correct date
cat <<EOF | kubectl create -f -
apiVersion: tasks.sql.arcdata.microsoft.com/v1
kind: SqlManagedInstanceRestoreTask
metadata:
  name: sql-restore-raki-correct
  namespace: arc
spec:
  source:
    name: sql-gp-1

mdrakiburrahman

# Login Into Azure Using Service Principal
 az_cli_cmd  login --service-principal -u az_client_id -p az_client_secret --tenant az_tenant_id

# Set Azure Subscription
 az_cli_cmd  account set --subscription az_subscription_id

# Create AZ Resource Group: Arc K8s
 az_cli_cmd  group create -l az_location -n az_resource_group

# OCP Login

mdrakiburrahman

CREATE DATABASE Ldw
      COLLATE Latin1_General_100_BIN2_UTF8;

USE Ldw;
CREATE MASTER KEY ENCRYPTION BY PASSWORD = 'MyRand0mPa33W0rd1!';

CREATE DATABASE SCOPED CREDENTIAL WorkspaceIdentity
WITH IDENTITY = 'Managed Identity';
GO

mdrakiburrahman

{
   "resourceLogs": [
      {
         "resource": {},
         "scopeLogs": [
            {
               "scope": {},
               "logRecords": [
                  {
                     "timeUnixNano": "1652102891470000028",