JavierArredondo

from pyspark.sql.functions import col
from pyspark.sql.types import (
    DoubleType,
    FloatType,
    IntegerType,
    StringType,
    StructField,
    StructType,
)

JavierArredondo

#!/bin/bash
CLUSTER='localhost:9092'
ZOOKEEPER='localhost:2181'

echo "Checking Topics"
TOPICS=`/etc/kafka/bin/kafka-topics.sh --bootstrap-server $CLUSTER --list`
for topic in ${TOPICS}
do
    echo "Processing $topic"
    N_OFFSET=`/etc/kafka/bin/kafka-log-dirs.sh --bootstrap-server $CLUSTER --topic-list $topic --describe | grep '^{' | jq '[ ..|.size? | numbers ] | add'`

JavierArredondo

#!/bin/bash
HELP="Usage: bootstrap-dask [OPTIONS]
Example AWS EMR Bootstrap Action to install and configure Dask and Jupyter
By default it does the following things:
- Installs miniconda
- Installs dask, distributed, dask-yarn, pyarrow, and s3fs. This list can be
  extended using the --conda-packages flag below.
- Packages this environment for distribution to the workers.
- Installs and starts a jupyter notebook server running on port 8888. This can
  be disabled with the --no-jupyter flag below.

JavierArredondo

import requests

url = "https://api.alerce.online/ztf/dr5/v1/light_curve/"

querystring = {"ra":"0","dec":"0","radius":"3"}

headers = {"Content-Type": "application/json"}

response = requests.request("GET", url, headers=headers, params=querystring)

JavierArredondo

FROM ubuntu:20.04

RUN apt-get update && apt-get upgrade && apt-get install -y wget

WORKDIR /app
COPY script.sh /app

CMD ["/app/script.sh"]