rajkrrsingh

#!/usr/bin/env bash

set -o errtrace
set -o nounset
set -o pipefail

# Quick steps to create the VPC/Public Subnet/EC2 in the aws account.

output=$(aws sts get-caller-identity 2>&1)

rajkrrsingh

lets take a sample code to demonste what happen when java tries to resolve the ip for the give hostname.

import java.net.InetAddress;
import java.net.UnknownHostException;

public class DNSClient {
    public static void main(String[] args) {
 InetAddress address = null;

rajkrrsingh

import logging
import os
import pandas
from impala.dbapi import connect
from impala.util import as_pandas
logging.basicConfig(level=logging.DEBUG)

HIVE_HS2_HOST='HIVE_HOST_ENDPOINT'
conn = connect(host=HIVE_HS2_HOST,port=443,

rajkrrsingh

IAM

List users

aws iam list-users

Current user

aws iam get-user

List access key

rajkrrsingh

package org.example;


import java.net.URI;
import java.net.URLEncoder;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;

rajkrrsingh

Following steps can be used to debug java application running on Remote Cluster, for sake of similicity I am using bare minimum pod config to spin the HS2 pod on the cluster.

1. Build a docker image as per https://gist.github.com/rajkrrsingh/e2a5d02a3a895c7f7e8f1d6d0e71e0e9, The image push the debug config -agentlib:jdwp=transport=dt_socket,address=8000,server=y,suspend=n to the JVM opts, the same can be achieved through the ConfigMap descriptor.

2. Create HS2 Deployments

apiVersion: apps/v1

rajkrrsingh

Installation and setup

curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
chmod 700 get_helm.sh
./get_helm.sh
helm version
helm repo add stable https://kubernetes-charts.storage.googleapis.com/

helm search and install app

rajkrrsingh

Create Non-ACID table

create external table nonacid_n1 (key int, a1 string, value string) stored as orc;
insert into nonacid_n1 values(1, 'a11', 'val1');
insert into nonacid_n1 values(2, 'a12', 'val2');

+-----------------+----------------+-------------------+
| nonacid_n1.key  | nonacid_n1.a1  | nonacid_n1.value  |
+-----------------+----------------+-------------------+
| 1               | a11            | val1              |

rajkrrsingh

CDP-DC kafka useful kafka commands

-- list topics
kafka-topics --list --bootstrap-server `hostname -f`:9092

-- create topic 
kafka-topics --create --bootstrap-server `hostname -f`:9092 --replication-factor 1 --partitions 1 --topic kafkatopic

-- produce messages
kafka-console-producer --broker-list `hostname -f`:9092 --topic kafkatopic

rajkrrsingh

Golang App

mkdir golang-web-app
cd golang-web-app
vim main.go
package main

import (
	"fmt"
	"log"