Skip to content

Instantly share code, notes, and snippets.

@wiseman
wiseman / agent.py
Last active June 27, 2024 14:32
Langchain example: self-debugging
from io import StringIO
import sys
from typing import Dict, Optional
from langchain.agents import load_tools
from langchain.agents import initialize_agent
from langchain.agents.tools import Tool
from langchain.llms import OpenAI
@tmarthal
tmarthal / task-dag-creation.py
Created February 23, 2017 01:31
DAG Creation from within a PythonOperator task DOES NOT WORK
# -*- coding: utf-8 -*-
from airflow.operators.http_operator import SimpleHttpOperator
from airflow.operators.postgres_operator import PostgresOperator
from airflow.operators.subdag_operator import SubDagOperator
from airflow.operators.sensors import SqlSensor
from airflow.hooks.postgres_hook import PostgresHook
from airflow.operators.python_operator import PythonOperator
from airflow.models import Variable, DAG
@metaskills
metaskills / command.sh
Last active November 4, 2022 09:44
Ubuntu 16.04 Install Latest FreeTDS
$ sudo apt-get install wget
$ sudo apt-get install build-essential
$ sudo apt-get install libc6-dev
$ wget ftp://ftp.freetds.org/pub/freetds/stable/freetds-1.00.27.tar.gz
$ tar -xzf freetds-1.00.27.tar.gz
$ cd freetds-1.00.27
$ ./configure --prefix=/usr/local --with-tdsver=7.3
$ make
$ make install
@rahul-pande
rahul-pande / pg_download_operator.py
Created September 5, 2016 07:48
Airflow Postgres Download Operator
# Airflow Operator to download results of a sql query to a file on the worker
# Pass chunksize parameter to download large tables without the
# worker running out of memory
import logging
from airflow.hooks.postgres_hook import PostgresHook
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
@erikbern
erikbern / install-tensorflow.sh
Last active June 26, 2023 00:40
Installing TensorFlow on EC2
# Note – this is not a bash script (some of the steps require reboot)
# I named it .sh just so Github does correct syntax highlighting.
#
# This is also available as an AMI in us-east-1 (virginia): ami-cf5028a5
#
# The CUDA part is mostly based on this excellent blog post:
# http://tleyden.github.io/blog/2014/10/25/cuda-6-dot-5-on-aws-gpu-instance-running-ubuntu-14-dot-04/
# Install various packages
sudo apt-get update
@SavvyGuard
SavvyGuard / botos3upload.py
Last active February 8, 2023 14:56
Use boto to upload directory into s3
import boto
import boto.s3
import os.path
import sys
# Fill these in - you get them when you sign up for S3
AWS_ACCESS_KEY_ID = ''
AWS_ACCESS_KEY_SECRET = ''
# Fill in info on data to upload
@prggmr
prggmr / python_gxmlimport.py
Last active February 20, 2023 21:35
Tool to import enormous XML files into mysql using lxml.etree
def does_exist(tbl, where = None, where_field = 'id'):
"""Builds a MySQL SELECT statement
"""
where_clause = "WHERE `%s` = %%s" % where_field
query = "SELECT COUNT(`id`) AS total FROM %s %s" % (
tbl, where_clause
)
cur.execute(query, (where,))
return cur.rownumber != 0