Skip to content

Instantly share code, notes, and snippets.

@hasibzunair
Last active March 9, 2022 08:04
Show Gist options
  • Save hasibzunair/a23aa11e9846274700d674cd9085bcdf to your computer and use it in GitHub Desktop.
Save hasibzunair/a23aa11e9846274700d674cd9085bcdf to your computer and use it in GitHub Desktop.
My references for colab commands.
# get API key from kaggle
# open github nb with colab
# https://colab.research.google.com/github/
!pip install -U -q kaggle
!mkdir -p ~/.kaggle
#upload API key
from google.colab import files
uploaded = files.upload()
for fn in uploaded.keys():
print('User uploaded file "{name}" with length {length} bytes'.format(
name=fn, length=len(uploaded[fn])))
!cp kaggle.json ~/.kaggle/
!kaggle datasets list
!kaggle datasets download -d iarunava/cell-images-for-detecting-malaria
!ls
os.listdir()
!unzip 'cell-images-for-detecting-malaria.zip'
os.listdir("cell_images")
# zip folder
!zip -r files.zip files
# save data
np.save("x_val.npy", x_val)
# see if files actually saved
os.listdir()
# Step 1
#---------------------------------------
from google.colab import files
files.download("x_val.npy")
#---------------------------------------
# Step 2
from google.colab import files
with open('example.txt', 'w') as f:
f.write('some content')
files.download('example.txt')
#---------------------------------------
# see RAM usage:
def spec():
# memory footprint support libraries/code
!ln -sf /opt/bin/nvidia-smi /usr/bin/nvidia-smi
!pip install gputil
!pip install psutil
!pip install humanize
import psutil
import humanize
import os
import GPUtil as GPU
GPUs = GPU.getGPUs()
# XXX: only one GPU on Colab and isn’t guaranteed
gpu = GPUs[0]
def printm():
process = psutil.Process(os.getpid())
print("Gen RAM Free: " + humanize.naturalsize( psutil.virtual_memory().available ), " | Proc size: " + humanize.naturalsize( process.memory_info().rss))
print("GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB".format(gpu.memoryFree, gpu.memoryUsed, gpu.memoryUtil*100, gpu.memoryTotal))
printm()
# move or copy files in folder
!mkdir files
!cp y_val.npy files
!mv y_val.npy files
!os.listdir("files/")
#----------------------------------------------------------
# save files in drive
from google.colab import drive
drive.mount('/content/gdrive')
PATH_MAIN = os.path.abspath("gdrive/My Drive/")
PATH_MAIN
# set path to save file
record_path = os.path.join(stor_path, 'model/record{}.csv'.format(exp_name))
# Load file from folder
img = np.load("{}/1.npy".format(stor_path))
with open('/content/gdrive/My Drive/foo.txt', 'w') as f:
f.write('Hello Google Drive!')
!cat /content/gdrive/My\ Drive/foo.txt
# or
with open('/content/gdrive/My Drive/Colab Notebooks/rbc notebooks/array.npy', 'w') as f:
f.write("array.npy")
# another way to get data from drive
!cp "/content/gdrive/My Drive/models.zip" test_folder
!unzip test/models.zip
os.listdir("models")
#----------------------------------------------------------
# run cmd command in python script
import subprocess
out = subprocess.Popen(["ls"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
stdout,stderr = out.communicate()
stdout.split()[0] # parse desired output
print(stdout)
#----------------------------------------------------------
# ref: https://medium.com/tensorflow/colab-an-easy-way-to-learn-and-use-tensorflow-d74d1686e309
# To determine which version you're using:
!pip show tensorflow
# For the current version:
!pip install --upgrade tensorflow
# For a specific version:
!pip install tensorflow==1.2
# For the latest nightly build:
!pip install tf-nightly
# import library not on colab
!pip install -q matplotlib-venn
!apt-get -qq install -y libfluidsynth1
# https://pypi.python.org/pypi/pydot
!apt-get -qq install -y graphviz && pip install -q pydot
import pydot
!apt-get -qq install python-cartopy python3-cartopy
import cartopy
@hasibzunair
Copy link
Author

hasibzunair commented Apr 22, 2019

Go to env

conda env export > environment.yml

Make new env from yml file (first line of yml is the env name, edit it if needed)

conda env create -f environment.yml 

@hasibzunair
Copy link
Author

@rawstar134
Copy link

How to run c and c++ and other programming languages on google colab? Please follow this article, and it gives a better idea if you don't have enough machines to run the program.
https://debuggingsolution.blogspot.com/2021/10/run-c-program-in-google-colab.html

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment