Skip to content

Instantly share code, notes, and snippets.

View saumalya75's full-sized avatar

Saumalya Sarkar saumalya75

  • Wipro Limited
  • Kolkata
View GitHub Profile
@saumalya75
saumalya75 / RecordRejectionHandling-Gist
Last active January 22, 2023 11:10
Glue-Spark code to handle malformed records while reading csv from s3
import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job
from awsglue.dynamicframe import DynamicFrame
# Importing required field types to define schema
from pyspark.sql.types import StringType, StructType, StructField, IntegerType
import boto3
import datetime
from botocore.exceptions import ClientError
from cloudformation_validator.ValidateUtility import ValidateUtility
s3_client = boto3.client("s3")
s3_resource = boto3.resource("s3")
cfn_client = boto3.client('cloudformation')
S3_BUCKET = 'linter-testing-bucket'
import json
import boto3
import datetime
from sqlfluff import lint
s3_client = boto3.client("s3")
s3_resource = boto3.resource("s3")
S3_BUCKET = 'linter-testing-bucket'
SQL_DIALECT = 'redshift'
EXCLUDED_RULES = ['L031'] #Using alias is allowed
@saumalya75
saumalya75 / py_code_linter.py
Last active April 6, 2022 05:58
This lambda is able to lint any python file
import boto3
import datetime
from pylint import epylint as lint
'''
{
"py_file_key": "input-scripts/python-scripts/input_python_script.py",
"py_file_bucket": "linter-testing-bucket"
}
@saumalya75
saumalya75 / gist:426b0e03738e7a966286c7f091dfad9d
Last active April 22, 2021 11:31
MINIO file extraction
import os
import boto3
import logging
import tempfile
import geopandas
MINIO_BUCKET = '<minio_bucket_name>'
SHAPE_FILE_NAME = '<shape_file_path_name>'
SHX_FILE_NAME = '<shx_file_path_name>'
class House:
def __init__(self, name: str, symbol: str, student_list: list):
self.name = name
self.symbol = symbol
self.student_list = student_list
self.current_student_id = -1
def __len__(self):
return len(self.student_list)
class House:
def __init__(self, name: str, symbol: str, student_list: list):
self.name = name
self.symbol = symbol
self.student_list = student_list
self.current_student_id = -1
def __len__(self):
return len(self.student_list)
class House:
def __init__(self, name: str, symbol: str, student_list: list):
self.name = name
self.symbol = symbol
self.student_list = student_list
self.current_student_id = -1
def __len__(self):
return len(self.student_list)
class House:
def __init__(self, name: str, symbol: str, student_list: list):
self.name = name
self.symbol = symbol
self.student_list = student_list
def __len__(self):
return len(self.student_list)
def __iter__(self):
class House:
def __init__(self, name: str, symbol: str, student_list: list):
self.name = name
self.symbol = symbol
self.student_list = student_list
def __len__(self):
return len(self.student_list)