| import os | |
| from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline | |
| # set environment variable HUGGING_FACE_API_KEY=hf_************ | |
| HUGGING_FACE_API_KEY = os.environ.get("HUGGING_FACE_API_KEY") | |
| model_id = "microsoft/Phi-3-mini-4k-instruct" | |
| tokenizer = AutoTokenizer.from_pretrained(model_id, legacy=False) | |
| model = AutoModelForCausalLM.from_pretrained( | |
| model_id, |
Dagster provides a mechanism to create pipeline DAGs which can be executed in a reproducible way. The DAGs can be visualized and run through a UI.
Details here: https://docs.dagster.io/getting-started
In order to run dagster, we first need to make sure we run python in a docker container.
The following code will create the python env in a docker container and will be removed right after we exit.
| cd ~ | |
| mkdir notebooks | |
| docker run --rm -it -p 8888:8888 -v /Users/akshaykarnawat/notebooks:/home/notebook python:3.8-slim-buster /bin/bash | |
| pip install jupyter | |
| cd /home/notebook | |
| jupyter notebook . --ip 0.0.0.0 --no-browser --allow-root | |
| # local on my host go to --> http://127.0.0.1:8888 and enter in the token |
| Running a local s3 like object storage to practice | |
| - Docker Desktop | |
| - MinIO | |
| - python | |
| - boto3 | |
| Install Docker desktop | |
| https://docs.docker.com/desktop/#download-and-install |
| from os import popen, system | |
| from time import sleep | |
| # Note: this requires brightness package to be installed from homebrew | |
| # https://brew.sh | |
| # brew update; brew upgrade; brew install brightness | |
| # #future -- rewite as shell script..no need for py | |
| CURRENT_BRIGHTNESS = 1 |
| [{"MCC":"0742","Merchant Type":"Veterinary Services"},{"MCC":"0763","Merchant Type":"Agricultural Co-operatives"},{"MCC":"0780","Merchant Type":"Horticultural Services"},{"MCC":"0780","Merchant Type":"Landscaping Services"},{"MCC":"1520","Merchant Type":"General Contractors-Residential and Commercial"},{"MCC":"1711","Merchant Type":"Air Conditioning Contractors Sales and Installation"},{"MCC":"1711","Merchant Type":"Heating Contractors Sales, Service, Installation"},{"MCC":"1731","Merchant Type":"Electrical Contractors"},{"MCC":"1740","Merchant Type":"Insulation Contractors"},{"MCC":"1740","Merchant Type":"Masonry, Stonework Contractors"},{"MCC":"1740","Merchant Type":"Plastering Contractors"},{"MCC":"1740","Merchant Type":"Stonework and Masonry Contractors"},{"MCC":"1740","Merchant Type":"Tile Settings Contractors"},{"MCC":"1750","Merchant Type":"Carpentry Contractors"},{"MCC":"1761","Merchant Type":"Roofing-Contractors"},{"MCC":"1761","Merchant Type":"Sheet Metal Work-Contractors"},{"MCC":"1761","Merchant T |
| import requests | |
| import json | |
| import time | |
| import os | |
| LOC_TO_STORE = '/Volumes/ExPac/dev/python/leet_code_problems' | |
| LEET_CODE_URL = 'https://leetcode.com' | |
| LEET_CODE_GRAPHQL_URL = 'https://leetcode.com/graphql' |
| # the structure of the code needs to change and the script needs to be parameterized with func... | |
| # but gets the job done for the data needed | |
| from urllib.parse import urljoin | |
| from bs4 import BeautifulSoup | |
| from datetime import datetime | |
| import pandas as pd | |
| import requests | |
| import zipfile |
| def commentGenerator(comment, length=80): | |
| commentList = comment.split(' ') | |
| breakBy = int(len(commentList) / ((len(comment)+len(commentList)) / length)) | |
| body = ''.join(['# {0} \n'.format(' '.join(commentList[i: i+breakBy])) for i in range(0, len(commentList), breakBy)]) | |
| box = '#+{0}+\n'.format(''.join(['=' for i in range(length-3)])) | |
| return box + body + box |
| from pyquil.quil import Program | |
| import pyquil.api as api | |
| from pyquil.gates import * | |
| import numpy as np | |
| #open quantum virtual machine and establish connection | |
| qvm = api.QVMConnection() | |
| print("measure 0,0") | |
| p = Program() |