ZaxR

2023-11-20T14:19:11.202-0600 [INFO]  backend/local: apply calling Apply
2023-11-20T14:19:11.202-0600 [DEBUG] Building and walking apply graph for NormalMode plan

2023-11-20T14:19:11.203-0600 [DEBUG] Resource state not found for node "module.ds-cc-creation-test.bitbucket_repository.repo", instance module.ds-cc-creation-test.bitbucket_repository.repo
2023-11-20T14:19:11.203-0600 [DEBUG] adding implicit provider configuration provider["registry.terraform.io/drfaust92/bitbucket"], implied first by module.ds-cc-creation-test.bitbucket_repository.repo
2023-11-20T14:19:11.203-0600 [DEBUG] ProviderTransformer: "module.ds-cc-creation-test.bitbucket_repository.repo" (*terraform.NodeApplyableResourceInstance) needs module.ds-cc-creation-test.provider["registry.terraform.io/drfaust92/bitbucket"]
2023-11-20T14:19:11.203-0600 [DEBUG] ProviderTransformer: "module.ds-cc-creation-test.bitbucket_repository.repo (expand)" (*terraform.nodeExpandApplyableResource) needs module.ds-cc-creation-test.provider["registry.terraform.io/

ZaxR

#! /usr/bin/env python3
"""CLV acquisition predictions, triggered daily."""
from datetime import datetime, timedelta
from typing import Optional

from airflow import DAG

from helpers import k8s, mlops, mlops_factories, settings

from helpers.custom_operators import ModifiedKubernetesPodOperator, SnapshotStartDateOperator, UpdateConfOperator

ZaxR

from tensorflow.keras import backend as K, metrics
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.layers import concatenate, Dense, Input, Layer
from tensorflow.keras.models import Model, model_from_json
from tensorflow.python.framework.ops import disable_eager_execution

import tensorflow as tf
from tensorflow import keras

ZaxR

!pip install --upgrade google-cloud-monitoring
from google.cloud import monitoring_v3

import time


def create_custom_ts(metric_name="my_metric"):
    series = monitoring_v3.TimeSeries()
    # https://cloud.google.com/monitoring/custom-metrics/creating-metrics
    series.metric.type = f"custom.googleapis.com/{metric_name}"

ZaxR

These instructions are to create a local, named Docker volume that stores authenticated Google service account credentials, for mounting to local Docker containers. This setup should only be required once, as the volume persists even after connected containers are stopped/removed.

TREAT THIS VOLUME AS CREDENTIALS. The volume stores sensitive information and should never leave your local machine.

Steps

1. Create the service account credentials file to authenticate google services.
   This file should be saved in ~/.config/gcloud/ on Mac/Linux,

ZaxR

# OTHER COOL FUNCS #
import collections
from itertools import chain
from sys import getsizeof
from typing import List, Optional, Tuple, Union


def get_scaled_units(number: Union[float, int], unit_scale_map: Optional[List[Tuple]] = None) -> str:
    """Scale `number` to units that result in the lowest number above 1, given a `unit_scale_map`.
    

ZaxR

import numpy as np

# Data
tensor = np.array([
    [ 9.8348e+18,  4.5845e-41, -3.5873e-11],
    [ 3.0950e-41,  1.2141e-40,  3.8102e-38],
    [ 5.3741e-30,  4.8419e+30,  7.7765e+31],
    [ 4.7987e+30,  4.9796e-37,  2.1325e-41],
    [ 2.4230e+21,  1.6045e-37,  1.9106e-28]
])

ZaxR

# https://docs.pytest.org/en/latest/example/parametrize.html#apply-indirect-on-particular-arguments

import pytest


class SomeClass:
    def __init__(self, a, b, c):
        self.a = a
        self.b = b
        self.c = c

ZaxR

""" Demo Model class that can be used generally for ML projects.

Attempts to solve the following problems:
1. Ensure data preprocessing  is consistent between data for model training and prediction
2. Have a common architecture for any data types / ML model types/libraries (as long as a consistent API is ued)
3. Allow for easy swapping of preprocessing, modeling, and/or postprocessing
4. Enforce an input (and potentially output) data schema(s)

Bonus:
- Follow's sklearn's estimator/transformer/predictor APIs, allowing use of sklearn Pipelines and GridSearchCV.

ZaxR

!apt install chromium-chromedriver -q
!pip install google_images_download selenium -q

import sys

from fastai.vision import *
from google.colab import drive, files
from google_images_download import google_images_download
from selenium import webdriver  # needed for google_images_download