test dbxcli-extras.ipynb

test-dbxcli-extras.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "test dbxcli-extras.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyO+e3Pu9mRcTZy+DkH0m7wZ",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/shadiakiki1986/7c478d451a4221d464d7bcfd5fc6a914/test-dbxcli-extras.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n-oavperJADj"
      },
      "source": [
        "Examples for https://github.com/shadiakiki1986/dbxcli-extras"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bCDTZn8NBhJO"
      },
      "source": [
        "# install"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y8uvxn6x7yCB"
      },
      "source": [
        "import os\n",
        "if not os.path.exists(\"/usr/bin/dbxcli\"):\n",
        "  !wget --quiet https://github.com/dropbox/dbxcli/releases/download/v3.0.0/dbxcli-linux-amd64\n",
        "  !chmod +x dbxcli-linux-amd64\n",
        "  !sudo mv dbxcli-linux-amd64 /usr/bin/dbxcli"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uw8XLF-i7qiq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7bcbad3f-7bd1-4ba4-bb08-093253bbd5f4"
      },
      "source": [
        "tag_use = \"v0.0.7\"\n",
        "!pip3 uninstall --yes --quiet dbxcli_extras # uninstall existing version to force re-install if exists\n",
        "!pip3 install --quiet git+https://github.com/shadiakiki1986/dbxcli-extras.git@{tag_use}"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[33mWARNING: Skipping dbxcli-extras as it is not installed.\u001b[0m\n",
            "\u001b[K     |████████████████████████████████| 72 kB 696 kB/s \n",
            "\u001b[K     |████████████████████████████████| 577 kB 9.3 MB/s \n",
            "\u001b[K     |████████████████████████████████| 162 kB 66.5 MB/s \n",
            "\u001b[K     |████████████████████████████████| 49 kB 5.1 MB/s \n",
            "\u001b[?25h  Building wheel for dbxcli-extras (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gbzYOJ8WMgwD"
      },
      "source": [
        "# for internal development only: install patch\n",
        "#!pip3 show dbxcli_extras\n",
        "#!ls /usr/local/lib/python3.7/dist-packages/dbxcli_extras/\n",
        "#!mv sync.py /usr/local/lib/python3.7/dist-packages/dbxcli_extras/\n",
        "# No need to restart kernel after mv since I'm using it as CLI not python package"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ISk9Dnv48dgF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5c741dde-1dec-4a66-9e65-83caf12d7a2f"
      },
      "source": [
        "# dropbox authentication by displaying a URL and prompting for the code displayed at the URL\n",
        "!dbxcli du"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1. Go to https://www.dropbox.com/1/oauth2/authorize?client_id=07o23gulcj8qi69&response_type=code&state=state\n",
            "2. Click \"Allow\" (you might have to log in first).\n",
            "3. Copy the authorization code.\n",
            "Enter the authorization code here: oZOSHA8Y2C0AAAAAAAB0dHOglcVPzBW1vUkWgYolJ0o\n",
            "Used: 287 GiB\n",
            "Type: individual\n",
            "Allocated: 2.0 TiB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3crWQrilBked"
      },
      "source": [
        "# common"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nmkzryQe-ycK"
      },
      "source": [
        "# name of directory to use for testing\n",
        "# It contains a random suffix to avoid collision with existing folders\n",
        "test_dir = \"dbxcli_extras_test_dir_kyNeEK8e\""
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZyCAcg8r-ea2"
      },
      "source": [
        "## sync"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OeiP1vvZG97B"
      },
      "source": [
        "Prepare directory"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-LImRjBM8pVe"
      },
      "source": [
        "# create local dir structure\n",
        "!rm -rf $test_dir\n",
        "!mkdir $test_dir\n",
        "!touch $test_dir/file_1\n",
        "!touch $test_dir/file_2\n",
        "\n",
        "# make sure it doesn't already exist in dropbox\n",
        "# Expect \"Error: path/not_found/..\"\n",
        "!dbxcli rm -f /$test_dir"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PtfTHzvzHDEu"
      },
      "source": [
        "Run sync the first time."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7GtRGkSdHCkH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "70c4e544-4739-4ca5-abdc-f215b291f4c2"
      },
      "source": [
        "# run sync\n",
        "!dbxcli_extras sync --verbosity=1 $test_dir /$test_dir\n",
        "\n",
        "# check they exist\n",
        "!dbxcli ls /$test_dir\n",
        "\n",
        "#Expect\n",
        "#File uploaded: dbxcli_extras_test_dir_kyNeEK8e/file_1\n",
        "#File uploaded: dbxcli_extras_test_dir_kyNeEK8e/file_2\n",
        "#/dbxcli_extras_test_dir_kyNeEK8e/file_1 \n",
        "#/dbxcli_extras_test_dir_kyNeEK8e/file_2"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "File uploaded to dropbox: dbxcli_extras_test_dir_kyNeEK8e/file_1\n",
            "File uploaded to dropbox: dbxcli_extras_test_dir_kyNeEK8e/file_2\n",
            "/dbxcli_extras_test_dir_kyNeEK8e/file_1 \n",
            "/dbxcli_extras_test_dir_kyNeEK8e/file_2"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pg9Pw_SyG_5q"
      },
      "source": [
        "Add a file and run sync again"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YaR9T1TG9FIT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "de7a1633-235f-4328-99cb-703182f9ad2f"
      },
      "source": [
        "# add a new file\n",
        "!touch $test_dir/file_3\n",
        "\n",
        "# sync again\n",
        "!dbxcli_extras sync --verbosity=1 $test_dir /$test_dir\n",
        "\n",
        "# Expect\n",
        "#File already exists: dbxcli_extras_test_dir_kyNeEK8e/file_1\n",
        "#File already exists: dbxcli_extras_test_dir_kyNeEK8e/file_2\n",
        "#File uploaded: dbxcli_extras_test_dir_kyNeEK8e/file_3"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "File already exists in dropbox and hash is the same (checked from cache): file_1\n",
            "File already exists in dropbox and hash is the same (checked from cache): file_2\n",
            "File uploaded to dropbox: dbxcli_extras_test_dir_kyNeEK8e/file_3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dkRcb0C5gbD5"
      },
      "source": [
        "Update a file and run sync again"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UuMx0LK4gdGT",
        "outputId": "5b0616c2-66dd-4363-d5f2-234058d4c96b"
      },
      "source": [
        "# update a file contents\n",
        "!date > $test_dir/file_3\n",
        "\n",
        "# sync again\n",
        "!dbxcli_extras sync --verbosity=1 $test_dir /$test_dir\n",
        "\n",
        "# Expect\n",
        "#File already exists: dbxcli_extras_test_dir_kyNeEK8e/file_1\n",
        "#File already exists: dbxcli_extras_test_dir_kyNeEK8e/file_2\n",
        "#File already exists but content changed: dbxcli_extras_test_dir_kyNeEK8e/file_3\n",
        "#File uploaded: dbxcli_extras_test_dir_kyNeEK8e/file_3"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "File already exists in dropbox and hash is the same (checked from cache): file_1\n",
            "File already exists in dropbox and hash is the same (checked from cache): file_2\n",
            "File already exists in dropbox but content hash changed (checked local file): file_3\n",
            "File uploaded to dropbox: dbxcli_extras_test_dir_kyNeEK8e/file_3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4FGh02slVrjx"
      },
      "source": [
        "Delete a file locally and run sync again"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mRX8zWApVuGk",
        "outputId": "eedebed6-93b6-4544-edd0-d606dd0a5a65"
      },
      "source": [
        "!rm $test_dir/file_3\n",
        "\n",
        "!dbxcli_extras sync --verbosity=1 $test_dir /$test_dir\n",
        "\n",
        "# Expect\n",
        "#File already exists: dbxcli_extras_test_dir_kyNeEK8e/file_1\n",
        "#File already exists: dbxcli_extras_test_dir_kyNeEK8e/file_2\n",
        "#Deleting remote file: '/dbxcli_extras_test_dir_kyNeEK8e/file_3'"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "File already exists in dropbox and hash is the same (checked from cache): file_1\n",
            "File already exists in dropbox and hash is the same (checked from cache): file_2\n",
            "Deleting remote file: '/dbxcli_extras_test_dir_kyNeEK8e/file_3'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Clear the dropbox copy, create `file_{3,4,5,6}`, and start the sync from `file_3` onwards (skipping `file_1` and `file_2`)"
      ],
      "metadata": {
        "id": "gXhOD8Xq9Cst"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!dbxcli rm -f /$test_dir\n",
        "!touch $test_dir/file_3\n",
        "!touch $test_dir/file_4\n",
        "!touch $test_dir/file_5\n",
        "!touch $test_dir/file_6\n",
        "\n",
        "!dbxcli_extras sync --verbosity=1 --start-from=\"$test_dir/file_3\" $test_dir /$test_dir\n",
        "\n",
        "# Expect\n",
        "#File uploaded to dropbox: dbxcli_extras_test_dir_kyNeEK8e/file_3\n",
        "#File uploaded to dropbox: dbxcli_extras_test_dir_kyNeEK8e/file_4\n",
        "#File uploaded to dropbox: dbxcli_extras_test_dir_kyNeEK8e/file_5\n",
        "#File uploaded to dropbox: dbxcli_extras_test_dir_kyNeEK8e/file_6"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vhcbi24R9HUC",
        "outputId": "2d982c6f-4b8a-4c31-9697-587cf5d9cf45"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "File uploaded to dropbox: dbxcli_extras_test_dir_kyNeEK8e/file_3\n",
            "File uploaded to dropbox: dbxcli_extras_test_dir_kyNeEK8e/file_4\n",
            "File uploaded to dropbox: dbxcli_extras_test_dir_kyNeEK8e/file_5\n",
            "File uploaded to dropbox: dbxcli_extras_test_dir_kyNeEK8e/file_6\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g35Yq1KI-h9i"
      },
      "source": [
        "# clean up\n",
        "!rm -rf $test_dir\n",
        "!dbxcli rm -f /$test_dir"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nNNiJ7pp-ghz"
      },
      "source": [
        "## getr"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YvKv_OvbG220"
      },
      "source": [
        "Prepare directories"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N_oCv-5D-he8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b944b980-3a07-4f05-d4d3-931fb2bdd2d9"
      },
      "source": [
        "# create dir structure\n",
        "!rm -rf $test_dir\n",
        "!mkdir $test_dir\n",
        "!touch $test_dir/file_1\n",
        "!mkdir $test_dir/subdir_1\n",
        "!touch $test_dir/subdir_1/file_2\n",
        "\n",
        "# make sure it doesn't already exist in dropbox\n",
        "!dbxcli rm -f /$test_dir\n",
        "\n",
        "# upload\n",
        "!dbxcli put $test_dir/file_1 /$test_dir/file_1\n",
        "!dbxcli put $test_dir/subdir_1/file_2 /$test_dir/subdir_1/file_2\n",
        "\n",
        "# check uploaded\n",
        "!dbxcli ls /$test_dir\n",
        "!echo \"\"\n",
        "!dbxcli ls /$test_dir/subdir_1\n",
        "\n",
        "# Expect\n",
        "#Error: path/not_found/.\n",
        "#Uploading 0 B/0 B\n",
        "#Uploading 0 B/0 B\n",
        "#/dbxcli_extras_test_dir_kyNeEK8e/subdir_1 \n",
        "#/dbxcli_extras_test_dir_kyNeEK8e/file_1\n",
        "#/dbxcli_extras_test_dir_kyNeEK8e/subdir_1/file_2 "
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Error: path/not_found/..\n",
            "Uploading 0 B/0 B\n",
            "Uploading 0 B/0 B\n",
            "/dbxcli_extras_test_dir_kyNeEK8e/subdir_1 \n",
            "/dbxcli_extras_test_dir_kyNeEK8e/file_1\n",
            "/dbxcli_extras_test_dir_kyNeEK8e/subdir_1/file_2 \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r0X7nbpdG44b"
      },
      "source": [
        "Run recursive get"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3l-NcDECAdb5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dbc8637c-6262-46cc-cf12-5bc467220abc"
      },
      "source": [
        "# remove local copy but keep empty folder\n",
        "!rm -rf $test_dir\n",
        "!mkdir -p $test_dir\n",
        "\n",
        "# assert exists but empty\n",
        "import os\n",
        "assert os.path.exists(test_dir)\n",
        "\n",
        "import glob\n",
        "assert len(glob.glob(f\"test_dir/*\"))==0\n",
        "\n",
        "# recursive get with CLI\n",
        "!dbxcli_extras getr --verbosity=1 \"/$test_dir\" \"$test_dir\"\n",
        "\n",
        "# assert files found\n",
        "assert os.path.exists(f\"{test_dir}/file_1\")\n",
        "assert os.path.exists(f\"{test_dir}/subdir_1/file_2\")\n",
        "\n",
        "# Expect\n",
        "#Created /dbxcli_extras_test_dir_kyNeEK8e/subdir_1\n",
        "#Downloaded /dbxcli_extras_test_dir_kyNeEK8e/subdir_1/file_2\n",
        "#Downloaded /dbxcli_extras_test_dir_kyNeEK8e/file_1"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Created /dbxcli_extras_test_dir_kyNeEK8e/subdir_1\n",
            "Downloaded /dbxcli_extras_test_dir_kyNeEK8e/subdir_1/file_2\n",
            "Downloaded /dbxcli_extras_test_dir_kyNeEK8e/file_1\n"
          ]
        }
      ]
    }
  ]
}