bstadlbauer

# Copyright (C) 2015-2021 Blackshark.ai GmbH. All Rights reserved. www.blackshark.ai
import logging
import multiprocessing as mp
import uuid
from typing import Optional, List

from flytekit import map_task, workflow, task, Resources
from flytekit.interfaces.data.data_proxy import FileAccessProvider
from flytekit.loggers import logger
from flytekit.remote import FlyteRemote

bstadlbauer

import time

import geopandas as geopd
from shapely.geometry import Point, Polygon
import numpy as np

_NUMBER_OF_POINTS = int(1e6)

np.random.seed(123)

bstadlbauer

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
This example demonstrates jitclasses and deferred types for writing a
singly-linked-list.
"""
from __future__ import print_function, absolute_import
from collections import OrderedDict
import numpy as np

bstadlbauer

import datetime
import pandas as pd

_TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M'

time_ranges = [
    (datetime.datetime(2019, 6, 16, 14, 2), datetime.datetime(2019, 6, 17, 7, 2)),
    (datetime.datetime(2019, 8, 15, 12, 30), datetime.datetime(2019, 8, 17, 8, 30))
]

bstadlbauer

import socket

import pyspark
from pip._internal.operations import freeze

NUMBER_OF_OPERATIONS = 100_000
NUMBER_OF_PARTITIONS = 10_000

def get_environment_and_hostname(_):
    hostname = socket.gethostname()