valerybriz

import sys
from pyspark.sql import SparkSession
import logging


APP_NAME = "Elastic Bulk loader"
INPUT_PATH = "gs://PATH/TO/THE/PARQUETS"
OUTPUT_PATH = "gs://PATH/TO/THE/PARQUETS"
ELASTIC_HOST = "http://localhost"
WRITE_MODE = "append"

valerybriz

-- monitorear queries con los costos más altos
WITH WAREHOUSE_SIZE AS
(
     SELECT WAREHOUSE_SIZE, NODES
       FROM (
              SELECT 'XSMALL' AS WAREHOUSE_SIZE, 1 AS NODES
              UNION ALL
              SELECT 'SMALL' AS WAREHOUSE_SIZE, 2 AS NODES
              UNION ALL
              SELECT 'MEDIUM' AS WAREHOUSE_SIZE, 4 AS NODES

valerybriz

import boto3
from settings import AWS_PROFILE

database_name = "test"
new_database_name = "newdb"
table_name = "table_name"
new_table_name = "new_table_name"
session = boto3.session.Session(profile_name=AWS_PROFILE)

client = session.client("glue")

valerybriz

# Hits the database
team_queryset = Team.objects.select_related("user").get(name="tigers")
# The second line doesn't hit the database, because team_queryset.user has been previously prepopulated
user = team_queryset.user

valerybriz

# Hits the database
team_queryset = Team.objects.get(name="tigers")
# Hits the database again to get the related User objects
user = team_queryset.user

valerybriz

Team.objects.all().prefetch_related('users')

valerybriz

user_queryset = User.objects.all()
# The first hit to the database confirms if at least one object exists.
if user_queryset.exists():
  # Another database hit to start fetching the rows in batches.
  for user in user_queryset.iterator():
    print(user.name)

valerybriz

for user in User.objects.exclude(email__isnull=True).iterator():
  print(user.name)

valerybriz

# Create the query filtering teams named tigers
team = Team.objects.get(name="tigers")
team.users.all() # query performed
team.users.all() # query performed again

valerybriz

# Create the query filtering users with email myemail@gmail.com
users = User.objects.get(email="myemail@gmail.com")
users.name # Hit the database and retrieve the name value
users.name # cached version, no database access