Skip to content

Instantly share code, notes, and snippets.

@philipturner

philipturner/s4tf-on-colab-example-1.ipynb

Created January 31, 2022 14:45
Show Gist options
  • Save philipturner/b155095efbc2e9669cf884181a98e560 to your computer and use it in GitHub Desktop.
Save philipturner/b155095efbc2e9669cf884181a98e560 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
s4tf-on-colab-example-1.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"id": "AYBuByXUvXHe"
},
"outputs": [],
"source": [
"# # Install *correct* CMake\n",
"\n",
"# # download this file onto personal computer bc the link's broke:\n",
"# # https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-linux-x86_64.sh\n",
"# # upload file to `/content` in Colab before running this cell (45 MB)\n",
"# # it takes a painfully long time to upload bc all networks throttle upload speeds\n",
"# !mv /content/cmake-3.22.1-linux-x86_64.sh /content/cmake-installer.sh\n",
"\n",
"# # the output appears to stall indefinitely, but you actually\n",
"# # have to interact with it to \"read\" the license (--MORE--)\n",
"# #\n",
"# # just continually press \"enter\" until it asks for y/n\n",
"# # do \"y\" for the first one, then \"n\" for the next\n",
"# # the next few bash commands assume there's no subdirectory\n",
"# !chmod +x /content/cmake-installer.sh\n",
"# !mkdir /opt/cmake\n",
"# !cd /opt/cmake && bash /content/cmake-installer.sh\n",
"\n",
"# !cmake --version\n",
"# !export PATH=\"/opt/cmake/bin:$PATH\" && cmake --version\n",
"# # you have to prepend the above PATH before every command\n",
"# # where you invoke CMake (similar to the Swift toolchain)\n",
"# #\n",
"# # in fact, just do this:\n",
"# # !export PATH=\"/opt/swift/toolchain/usr/bin:/opt/cmake/bin:$PATH\" && do whatever\n",
"\n",
"# # this should produce CMake 3.22.1, NOT 3.12.0\n",
"# !export PATH=\"/opt/swift/toolchain/usr/bin:/opt/cmake/bin:$PATH\" && cmake --version"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "T_ayCNRHOu_N"
},
"outputs": [],
"source": [
"# # Install *correct* Bazel\n",
"\n",
"# # same stuff as with CMake, but this time use the link:\n",
"# # https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-installer-linux-x86_64.sh\n",
"# !mv /content/bazel-3.1.0-installer-linux-x86_64.sh /content/bazel-installer.sh\n",
"# !mkdir /opt/bazel\n",
"# !cd /opt/bazel && bash /content/bazel-installer.sh\n",
"\n",
"# # this should produce Bazel 3.1.0, NOT 6.0.0\n",
"# !bazel --version"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "aV2829XuDFu2",
"outputId": "91cf259c-aec0-4a07-806c-2a6f078e573d"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
" % Total % Received % Xferd Average Speed Time Time Time Current\n",
" Dload Upload Total Spent Left Speed\n",
"100 567M 100 567M 0 0 22.8M 0 0:00:24 0:00:24 --:--:-- 24.7M\n",
"Reading package lists... Done\n",
"Building dependency tree \n",
"Reading state information... Done\n",
"The following NEW packages will be installed:\n",
" ninja-build\n",
"0 upgraded, 1 newly installed, 0 to remove and 37 not upgraded.\n",
"Need to get 93.3 kB of archives.\n",
"After this operation, 296 kB of additional disk space will be used.\n",
"Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 ninja-build amd64 1.8.2-1 [93.3 kB]\n",
"Fetched 93.3 kB in 1s (164 kB/s)\n",
"Selecting previously unselected package ninja-build.\n",
"(Reading database ... 155229 files and directories currently installed.)\n",
"Preparing to unpack .../ninja-build_1.8.2-1_amd64.deb ...\n",
"Unpacking ninja-build (1.8.2-1) ...\n",
"Setting up ninja-build (1.8.2-1) ...\n",
"Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
"Swift version 5.6-dev (LLVM 93bcb8342e7352f, Swift f2132728b7d0b1f)\n",
"Target: x86_64-unknown-linux-gnu\n"
]
}
],
"source": [
"# Install everything else\n",
"\n",
"# using 2021-11-12 rn, could use something earlier or later\n",
"# note that the toolchain is 450 MB, yet it downloads faster than the \n",
"# upload from your personal computer in the previous cell (45 MB). wtf?????\n",
"!curl \"https://download.swift.org/development/ubuntu1804/swift-DEVELOPMENT-SNAPSHOT-2021-11-12-a/swift-DEVELOPMENT-SNAPSHOT-2021-11-12-a-ubuntu18.04.tar.gz\" | tar -xz\n",
"!mkdir /opt/swift\n",
"!mv \"swift-DEVELOPMENT-SNAPSHOT-2021-11-12-a-ubuntu18.04\" \"/opt/swift/toolchain\"\n",
"\n",
"!apt install ninja-build\n",
"# this command might be useful later for patching broken dynamic libraries\n",
"# !apt install patchelf\n",
"\n",
"# this should produce Swift 5.6-dev, NOT \"... not found\"\n",
"!export PATH=\"/opt/swift/toolchain/usr/bin:/opt/cmake/bin:$PATH\" && swift --version"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "_sHDGnYjDXU7",
"outputId": "8e33ab37-bac5-4b4a-b503-927240115c48"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Cloning into 'swift-apis'...\n",
"remote: Enumerating objects: 8596, done.\u001b[K\n",
"remote: Counting objects: 100% (191/191), done.\u001b[K\n",
"remote: Compressing objects: 100% (131/131), done.\u001b[K\n",
"remote: Total 8596 (delta 98), reused 108 (delta 59), pack-reused 8405\u001b[K\n",
"Receiving objects: 100% (8596/8596), 23.55 MiB | 22.86 MiB/s, done.\n",
"Resolving deltas: 100% (6039/6039), done.\n"
]
}
],
"source": [
"# !git clone --single-branch -b patch-5 https://github.com/philipturner/swift-for-tensorflow\n",
"!git clone --single-branch -b fan/resurrection https://github.com/ProfFan/swift-apis\n",
"# !git clone --single-branch -b main https://github.com/tensorflow/swift-apis"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"id": "Kh8KBqLbKfEM"
},
"outputs": [],
"source": [
"# DON'T ACTUALLY RUN THIS CELL!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",
"# force-commented out the command\n",
"# !export PATH=\"/opt/swift/toolchain/usr/bin:/opt/cmake/bin:$PATH\" && cmake -B out -G Ninja -S swift-apis -D CMAKE_BUILD_TYPE=Release"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"id": "GTIShZJONlAI"
},
"outputs": [],
"source": [
"# DON'T ACTUALLY RUN THIS CELL!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",
"# force-commented out the command\n",
"# !export PATH=\"/opt/swift/toolchain/usr/bin:/opt/cmake/bin:$PATH\" && cmake --build out\n",
"# I actually got this to start building, but it takes so freaking long\n",
"# I paused it and will instead use the pre-built binary on GitHub"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"id": "6z1qUQquXQGo",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "2c630b60-9d7b-4a9c-ad5b-8ed957ccaca2"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
" % Total % Received % Xferd Average Speed Time Time Time Current\n",
" Dload Upload Total Spent Left Speed\n",
"100 56.8M 100 56.8M 0 0 84.2M 0 --:--:-- --:--:-- --:--:-- 84.1M\n",
"Archive: tensorflow-binary.zip\n",
" creating: Library/\n",
" creating: Library/tensorflow-2.4.0/\n",
" creating: Library/tensorflow-2.4.0/usr/\n",
" creating: Library/tensorflow-2.4.0/usr/include/\n",
" creating: Library/tensorflow-2.4.0/usr/include/tensorflow/\n",
" creating: Library/tensorflow-2.4.0/usr/include/tensorflow/core/\n",
" creating: Library/tensorflow-2.4.0/usr/include/tensorflow/core/platform/\n",
" inflating: Library/tensorflow-2.4.0/usr/include/tensorflow/core/platform/ctstring_internal.h \n",
" inflating: Library/tensorflow-2.4.0/usr/include/tensorflow/core/platform/ctstring.h \n",
" creating: Library/tensorflow-2.4.0/usr/include/tensorflow/c/\n",
" inflating: Library/tensorflow-2.4.0/usr/include/tensorflow/c/tf_status.h \n",
" inflating: Library/tensorflow-2.4.0/usr/include/tensorflow/c/tf_attrtype.h \n",
" inflating: Library/tensorflow-2.4.0/usr/include/tensorflow/c/tf_tstring.h \n",
" inflating: Library/tensorflow-2.4.0/usr/include/tensorflow/c/c_api_experimental.h \n",
" creating: Library/tensorflow-2.4.0/usr/include/tensorflow/c/eager/\n",
" inflating: Library/tensorflow-2.4.0/usr/include/tensorflow/c/eager/c_api.h \n",
" inflating: Library/tensorflow-2.4.0/usr/include/tensorflow/c/c_api.h \n",
" inflating: Library/tensorflow-2.4.0/usr/include/tensorflow/c/tf_file_statistics.h \n",
" inflating: Library/tensorflow-2.4.0/usr/include/tensorflow/c/tf_datatype.h \n",
" inflating: Library/tensorflow-2.4.0/usr/include/tensorflow/c/c_api_macros.h \n",
" inflating: Library/tensorflow-2.4.0/usr/include/tensorflow/c/tf_tensor.h \n",
" creating: Library/tensorflow-2.4.0/usr/lib/\n",
" inflating: Library/tensorflow-2.4.0/usr/lib/libx10_optimizers_optimizer.so \n",
" inflating: Library/tensorflow-2.4.0/usr/lib/libx10_optimizers_tensor_visitor_plan.so \n",
" inflating: Library/tensorflow-2.4.0/usr/lib/libx10.so \n",
" inflating: Library/tensorflow-2.4.0/usr/lib/libx10_training_loop.so \n",
"rm: cannot remove '/Library': No such file or directory\n"
]
}
],
"source": [
"!curl https://storage.googleapis.com/swift-tensorflow-artifacts/oneoff-builds/tensorflow-ubuntu1804-x86.zip --output tensorflow-binary.zip\n",
"!unzip tensorflow-binary.zip\n",
"!rm -r /Library\n",
"!cp -r /content/Library /Library\n",
"!rm -r /content/Library"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"id": "Y5hq71TSZTrF"
},
"outputs": [],
"source": [
"# CMake freezes\n",
"# !export PATH=\"/opt/swift/toolchain/usr/bin:/opt/cmake/bin:$PATH\" && cmake -B out -G Ninja -S swift-apis -D CMAKE_BUILD_TYPE=Release -D X10_LIBRARY=/Library/tensorflow-2.4.0/usr/lib/libx10.so -D X10_INCLUDE_DIRS=/Library/tensorflow-2.4.0/usr/include"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"id": "bkb2AU4xZ1Lo"
},
"outputs": [],
"source": [
"# CMake freezes\n",
"# !export PATH=\"/opt/swift/toolchain/usr/bin:/opt/cmake/bin:$PATH\" && cmake --build out"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "-jT6YSnKadCE"
},
"outputs": [],
"source": [
"# -c release -Xswiftc -Onone -Xswiftc -enable-testing\n",
"!cd swift-apis && export PATH=\"/opt/swift/toolchain/usr/bin:$PATH\" && swift build --verbose -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -I/Library/tensorflow-2.4.0/usr/include -Xlinker -L/Library/tensorflow-2.4.0/usr/lib"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"id": "R4myR5K1U7qm"
},
"outputs": [],
"source": [
"# !rm -r swift-apis"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"id": "XssHE4NYY1hv"
},
"outputs": [],
"source": [
"# !zip -r content-apis-build-zip /content/swift-apis/.build"
]
},
{
"cell_type": "code",
"source": [
"# this doesn't work because I can't force it to emit a dynamic library\n",
"# if I could force a dylib, then I could link it in `compile.sh` and possibly\n",
"# resolve the module lookup failure\n",
"#\n",
"# !rm -r swift-numerics\n",
"# !git clone --single-branch -b main https://github.com/apple/swift-numerics\n",
"# !cd swift-numerics && export PATH=\"/opt/swift/toolchain/usr/bin:$PATH\" && swift build --verbose -emit-library"
],
"metadata": {
"id": "Ftv-8zWNJJzT"
},
"execution_count": 13,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# no dynamic libraries exist here\n",
"# !cd /content/swift-numerics/.build/debug && ls"
],
"metadata": {
"id": "aBFMlmCLJM7l"
},
"execution_count": 14,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# !cd /content/swift-apis/.build/debug && ls"
],
"metadata": {
"id": "alpCs9bqJNEJ"
},
"execution_count": 15,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# # Download files from s4tf-colab-experiments\n",
"# !rm -r s4tf-colab-experiments\n",
"# !rm -r compile.sh\n",
"# !rm -r scripts\n",
"\n",
"# !git clone https://github.com/philipturner/s4tf-colab-experiments\n",
"# !mv /content/s4tf-colab-experiments/compile.sh /content/compile.sh\n",
"# !mkdir /content/scripts\n",
"# !mv /content/s4tf-colab-experiments/script.swift /content/scripts/script.swift"
],
"metadata": {
"id": "8rbe0XCjKdqu"
},
"execution_count": 16,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# !rm -r s4tf-on-colab-package\n",
"# !git clone https://github.com/philipturner/s4tf-on-colab-package.git\n",
"# !cp -r /Library /content/s4tf-on-colab-package/Library\n",
"# !cd s4tf-on-colab-package && export PATH=\"/opt/swift/toolchain/usr/bin:$PATH\" && swift build"
],
"metadata": {
"id": "UNguzgnpptCU"
},
"execution_count": 17,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# !cd scripts && export LD_LIBRARY_PATH=\"/content/swift-apis/.build/debug:$LD_LIBRARY_PATH\" && bash /content/compile.sh script.swift"
],
"metadata": {
"id": "lcJoAHWWJfa4"
},
"execution_count": 18,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# !/content/scripts/script"
],
"metadata": {
"id": "ev2yW4wDKSof"
},
"execution_count": 19,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# !cd / && touch testfile.txt"
],
"metadata": {
"id": "MHaCcL51cTSq"
},
"execution_count": 20,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# !cd / && cp /Library/tensorflow-2.4.0/usr/lib/libx10.so libx10.so\n",
"!cd swift-apis && export PATH=\"/opt/swift/toolchain/usr/bin:$PATH\" && export LD_LIBRARY_PATH=\"/Library/tensorflow-2.4.0/usr/lib:$LD_LIBRARY_PATH\" && swift test --verbose -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -I/Library/tensorflow-2.4.0/usr/include -Xlinker -L/Library/tensorflow-2.4.0/usr/lib\n",
"\n",
"# How to use tests for arbitrary code:\n",
"# Go to the `LinuxMain.swift` file in the `Tests` directory\n",
"# Replace all of its contents with your own code\n",
"# Don't forget to explicitly import `TensorFlow`!!!"
],
"metadata": {
"id": "pgVQ9AVqa5u2"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# !cd / && cp /Library/tensorflow-2.4.0/usr/lib/libx10.so /content/swift-apis/libx10.so"
],
"metadata": {
"id": "tbqV_EO_c9Vw"
},
"execution_count": 22,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# !ls /"
],
"metadata": {
"id": "3JxvnTF4c_E9"
},
"execution_count": 23,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# !ls /Library/tensorflow-2.4.0/usr/lib"
],
"metadata": {
"id": "ea3Gemdbbwwc"
},
"execution_count": 24,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# !git clone https://github.com/tensorflow/swift\n",
"# !zip -r -O -v /content/swift/docs/site\n",
"# !zip --help\n",
"# !zip -r s4tf-tutorials-zip /content/swift/docs/site"
],
"metadata": {
"id": "YJgM_6ippqUp"
},
"execution_count": 25,
"outputs": []
}
],
"metadata": {
"colab": {
"collapsed_sections": [],
"name": "CompilingSwift2.ipynb",
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment