andrewm4894

#!/usr/bin/env bash

scope="production"

# stop on all errors
set -e

if [ $UID -ne 0 ]
then
	echo >&2 "Hey! sudo me: sudo ${0}"

andrewm4894

# node ar 1min
 template: ml_1min_node_ar
       on: anomaly_detection.anomaly_rate
    class: Anomaly
     type: System
component: Node
   lookup: average -1m foreach anomaly_rate
    units: %
    every: 30s

andrewm4894

andrewm4894

andrewm4894

#!/usr/bin/python
"""
A python script to run many "airflow run dags trigger {dag} -e {execution_datetime}" commands via the airflow rest api.

Example usage:

python airflow_trigger_dags.py --dag 'dev_dag' --start '2021-10-01 00:00:01' --end '2021-10-31 00:00:01'
python airflow_trigger_dags.py -d 'dev_dag' -s '2022-05-20 00:00:01' -e '2022-05-24 00:00:01'

Example usage to just trigger dag for now:

andrewm4894

import sqlite3
from sqlite3 import Error

 
def create_connection(db_file):
    """ create a database connection to the SQLite database
        specified by the db_file
    :param db_file: database file
    :return: Connection object or None
    """

andrewm4894

sudo apt-get update

# install python3
#sudo apt-get install python3.7
#sudo apt install python3
sudo apt-get install python3-pip
sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10

# update
sudo apt-get update

andrewm4894

# -*- coding: utf-8 -*-
import json
import base64
import logging
import os
from datetime import datetime
from urllib.parse import urlparse
import praw
from bs4 import BeautifulSoup
import pandas as pd

andrewm4894

def calc_batches(train_min: int, train_max: int, train_every: int, n: int) -> dict:
    batches = dict()
    batch = 0
    for row in range(1,n+1):
        if row < train_min:
            pass
        elif row == train_min:
            batches[batch] = dict(start=0, end=row)
        elif row % train_every == 0:
            batch += 1

andrewm4894

def calc_batches(train_max: int, train_every: int, n: int) -> dict:
    batches = dict()
    # loop over up to as many records as you have
    for batch in range(n):
        # work out the start of the batch, with a max() to handle first batch
        start = max(train_every * batch, 1)
        # work out the end of the batch, with a min() to handle last batch
        end = min(train_max+(train_every * batch), n)
        # add batch info to the dictionary
        batches[batch+1] = {"start": start, "end": end}