febridev

IP=$1
LPORT=$2
RPORT=$2
ps aux | grep id_rsa | grep ${LPORT} | awk '{print $2}' | tail -1 | xargs kill -9
ssh -i ~/.ssh/id_rsa -o ExitOnForwardFailure=yes -f -N -L ${LPORT}:${IP}:${RPORT} <userssh>@<ipjumphost>

pgcli --host 127.0.0.1 --port $LPORT --username <userdatabase> --dbname STG_FINANCING

febridev

# First-time build can take upto 10 mins.

FROM apache/airflow:2.2.3

ENV AIRFLOW_HOME=/opt/airflow
ENV KAGGLE_USERNAME=<username>
ENV KAGGLE_KEY=<kaggle-key>

USER root
RUN apt-get update -qq && apt-get install vim -qqq && apt-get install wget -qq

febridev

version: '3'
services:
    postgres:
        image: postgres:13
        networks:
            - default_net
        env_file:
            - .env
        volumes:
            - postgres-db-volume:/var/lib/postgresql/data

febridev

use master

restore filelistonly from disk = 'E:\Backup\restore\dbSF_BizNet_NEC_20220321_094054.bak'


-- restore database
RESTORE DATABASE dbSF_BizNet_NEC FROM  DISK = 'E:\Backup\restore\dbSF_BizNet_NEC_20220321_094054.bak' 
WITH MOVE 'SunFish_Data' TO 'D:\sql2016\SFNEC_DEMO_Data.mdf',  
     MOVE 'SunFish_Log' TO 'L:\Logfiles\SFNEC_Log.ldf',  
RECOVERY,REPLACE

febridev

import os
from dataIngestion import download_from_kaggle
from datetime import datetime
from airflow import DAG
from airflow.operators.bash import BashOperator
from airflow.operators.python import PythonOperator
# from dotenv import load_dotenv

# load_dotenv()

febridev

import os
import logging

from time import time
from kaggle.api.kaggle_api_extended import KaggleApi
 

def download_from_kaggle():
    dataset = 'yamaerenay/spotify-dataset-19212020-600k-tracks'
    path = 'kaggle'

febridev

{
    "query_block": {
      "select_id": 1,
      "r_loops": 1,
      "r_total_time_ms": 88634,
      "filesort": {
        "sort_key": "tgospfeed.modified_date desc",
        "r_loops": 1,
        "r_total_time_ms": 0.5509,
        "r_limit": 10,

febridev

SELECT s.session_id,
r.status,
r.blocking_session_id 'Blk_by',
r.wait_type,
wait_resource,
r.wait_time,
r.cpu_time,
r.logical_reads,
r.reads,
r.writes,

febridev

PLAN_TABLE_OUTPUT
Plan hash value: 3233107407
 
---------------------------------------------------------------------------------------------------------------------
| Id  | Operation                               | Name                      | Rows  | Bytes | Cost (%CPU)| Time     |
---------------------------------------------------------------------------------------------------------------------
|   0 | SELECT STATEMENT                        |                           |     7 |  9093 | 11069   (1)| 00:00:01 |
|   1 |  SORT ORDER BY                          |                           |     7 |  9093 | 11069   (1)| 00:00:01 |
|*  2 |   HASH JOIN RIGHT OUTER                 |                           |     7 |  9093 | 11068   (1)| 00:00:01 |
|*  3 |    TABLE ACCESS FULL                    | TGOSPEMPFAVORITE          |     1 |    80 |     2   (0)| 00:00:01 |

febridev

DELIMITER ;
/*!50003 SET sql_mode              = @saved_sql_mode */ ;
/*!50003 SET character_set_client  = @saved_cs_client */ ;
/*!50003 SET character_set_results = @saved_cs_results */ ;
/*!50003 SET collation_connection  = @saved_col_connection */ ;
/*!50003 DROP PROCEDURE IF EXISTS `spPayrollProcess` */;
/*!50003 SET @saved_cs_client      = @@character_set_client */ ;
/*!50003 SET @saved_cs_results     = @@character_set_results */ ;
/*!50003 SET @saved_col_connection = @@collation_connection */ ;
/*!50003 SET character_set_client  = latin1 */ ;