Skip to content

Instantly share code, notes, and snippets.

View xhinker's full-sized avatar

Andrew Zhu xhinker

View GitHub Profile
@xhinker
xhinker / upload_file_chunks.py
Created October 7, 2022 07:42
Upload large file to Azure blob storage
def upload_file_chunks(self,blob_file_path,local_file_path):
'''
Upload large file to blob
'''
import uuid
try:
blob_client = self.container_client.get_blob_client(blob_file_path)
# upload data
block_list=[]
chunk_size=1024*1024*4
@xhinker
xhinker / az_kmeans.py
Created May 23, 2022 21:31
kmeans using Python
import numpy as np
import random
def grouping(points,cps) -> dict:
'''
This function will group the points based on the center points(cps), and return
result in a dictionary object
{
cluster_index: [(x1,y1),(x2,y2)]
...
@xhinker
xhinker / simulate_functions.py
Created April 6, 2022 05:50
simulate any function with NN
import numpy as np
import matplotlib.pyplot as plt
X = np.array([*range(-20,20)],dtype=np.float32)
X = X*0.1
y = [x**3+ x**2 -3*x -1 for x in X]
plt.plot(X,y,'ro')
import torch
import torch.nn as nn
import torch.nn.functional as F
@xhinker
xhinker / verify_cnn.py
Created March 29, 2022 01:23
Verify CNN result
from torch.autograd import Variable
with torch.no_grad():
n_correct = 0
n_samples = 0
for images,labels in test_loader:
#images = images.reshape(-1,28*28)
images = Variable(images)
#labels = labels.to(device)
outputs = model.forward(images)
_,predicted = torch.max(outputs.data,1) # need figure this out in detail
@xhinker
xhinker / az_cnn.py
Created March 29, 2022 01:22
AZ CNN
from torch.autograd import Variable
import torch.nn as nn
class AZ_CNN(nn.Module):
def __init__(self,epochs=10,sizes=[784,16,16,10]):
super(AZ_CNN,self).__init__()
self.sizes = sizes
self.epochs = epochs
self.conv1 = nn.Sequential(
@xhinker
xhinker / verify_nn_wo_conv.py
Created March 29, 2022 01:19
Verify NN without convolution
with torch.no_grad():
n_correct = 0
n_samples = 0
for images,labels in test_loader:
images = images.reshape(-1,28*28)
#labels = labels.to(device)
outputs = model.forward(images)
_,predicted = torch.max(outputs.data,1) # need figure this out in detail
n_samples += labels.size(0)
n_correct += (predicted==labels).sum().item() # this is amazing operation, saved so many lines of code
@xhinker
xhinker / nn_in_pytorch.py
Last active March 29, 2022 01:19
NN in PyTorch
import torch.nn as nn
class AZ_NN(nn.Module):
def __init__(self,epochs=10,sizes=[784,16,16,10]):
super(AZ_NN,self).__init__()
self.sizes = sizes
self.epochs = epochs
self.layer1 = nn.Linear(self.sizes[0],self.sizes[1])
self.relu = nn.ReLU()
self.layer2 = nn.Linear(self.sizes[1],self.sizes[2])
self.relu = nn.ReLU()
@xhinker
xhinker / show_image_from_dl.py
Created March 29, 2022 01:08
show image from pytorch dataloader
import matplotlib.pyplot as plt
sample = iter(train_loader).next()
data = sample[0]
label = sample[1]
#pick image from the first batch,first channel
image = data[0][0]
plt.imshow(image,cmap='gray')
plt.title(label=label[0].item())
@xhinker
xhinker / conv2d_one_image.py
Created March 28, 2022 17:58
PyTorch Conv2d apply to a single image
import torch.nn as nn
import matplotlib.pyplot as plt
conv1 = nn.Conv2d(
in_channels = 1
,out_channels = 1
,kernel_size = 3
,stride = 1
,padding = 1
)
@xhinker
xhinker / load_mnist_pytorch.py
Created March 28, 2022 16:08
Download and load MNIST using PyTorch
from torchvision import datasets
from torchvision.transforms import ToTensor
train_data = datasets.MNIST(
root = './data'
,train = True
,transform = ToTensor()
,download = True
)
test_data = datasets.MNIST(