deploy_couchdb_to_flyio_v2_with_swap.ipynb

deploy_couchdb_to_flyio_v2_with_swap.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/vrtmrz/37c3efd7842e49947aaaa7f665e5020a/deploy_couchdb_to_flyio_v2_with_swap.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "History:\n",
        "- 18, May, 2023: Initial.\n",
        "- 19, Jun., 2023: Patched for enabling swap.\n",
        "- 22, Aug., 2023: Generating Setup-URI implemented.\n",
        "- 7, Nov., 2023: Fixed the issue of TOML editing.\n",
        "- 5 Feb. 2024 : Refined version is here: https://gist.github.com/vrtmrz/9402b101746e08e969b1a4f5f0deb465"
      ],
      "metadata": {
        "id": "HiRV7G8Gk1Rs"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2Vh0mEQEZuAK"
      },
      "outputs": [],
      "source": [
        "# Configurations\n",
        "import os\n",
        "os.environ['region']=\"nrt\"\n",
        "os.environ['couchUser']=\"alkcsa93\"\n",
        "os.environ['couchPwd']=\"c349usdfnv48fsasd\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SPmbB0jZauQ1"
      },
      "outputs": [],
      "source": [
        "# Delete once (Do not care about `cannot remove './fly.toml': No such file or directory`)\n",
        "!rm ./fly.toml"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Nze7QoxLZ7Yx"
      },
      "outputs": [],
      "source": [
        "# Installation\n",
        "# You have to set up your account in here.\n",
        "!curl -L https://fly.io/install.sh | sh\n",
        "!/root/.fly/bin/flyctl auth signup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MVJwsIYrbgtx"
      },
      "outputs": [],
      "source": [
        "# Generate server\n",
        "!/root/.fly/bin/flyctl launch  --auto-confirm --generate-name --detach --no-deploy --region ${region}\n",
        "!/root/.fly/bin/fly volumes create --region ${region} couchdata --size 2 --yes"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check the toml once.\n",
        "!cat fly.toml"
      ],
      "metadata": {
        "id": "2RSoO9o-i2TT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zUtPZLVnbvdQ"
      },
      "outputs": [],
      "source": [
        "# Modify the TOML and generate Dockerfile\n",
        "!pip install mergedeep\n",
        "from mergedeep import merge\n",
        "import toml\n",
        "fly = toml.load('fly.toml')\n",
        "override = {\n",
        "    \"http_service\":{\n",
        "        \"internal_port\":5984\n",
        "    },\n",
        "    \"build\":{\n",
        "        \"dockerfile\":\"./Dockerfile\"\n",
        "    },\n",
        "    \"mounts\":{\n",
        "        \"source\":\"couchdata\",\n",
        "        \"destination\":\"/opt/couchdb/data\"\n",
        "    },\n",
        "    \"env\":{\n",
        "        \"COUCHDB_USER\":os.environ['couchUser'],\n",
        "        \"ERL_FLAGS\":\"-couch_ini /opt/couchdb/etc/default.ini /opt/couchdb/etc/default.d/ /opt/couchdb/etc/local.d /opt/couchdb/etc/local.ini /opt/couchdb/data/persistence.ini\",\n",
        "    }\n",
        "}\n",
        "out = merge(fly,override)\n",
        "with open('fly.toml', 'wt') as fp:\n",
        "    toml.dump(out, fp)\n",
        "    fp.close()\n",
        "\n",
        "# Make the Dockerfile to modify the permission of the ini file. If you want to use a specific version,  you should change `latest` here.\n",
        "dockerfile = '''FROM couchdb:latest\n",
        "RUN sed -i '2itouch /opt/couchdb/data/persistence.ini && chmod +w /opt/couchdb/data/persistence.ini && fallocate -l 512M /swapfile && chmod 0600 /swapfile && mkswap /swapfile && echo 10 > /proc/sys/vm/swappiness && swapon /swapfile && echo 1 > /proc/sys/vm/overcommit_memory' /docker-entrypoint.sh\n",
        "'''\n",
        "with open(\"./Dockerfile\",\"wt\") as fp:\n",
        "    fp.write(dockerfile)\n",
        "    fp.close()\n",
        "\n",
        "!echo ------\n",
        "!cat fly.toml\n",
        "!echo ------\n",
        "!cat Dockerfile"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xWdsTCI6bzk2"
      },
      "outputs": [],
      "source": [
        "# Configure password\n",
        "!/root/.fly/bin/flyctl secrets set COUCHDB_PASSWORD=${couchPwd}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "k0WIQlShcXGa"
      },
      "outputs": [],
      "source": [
        "# Deploy server\n",
        "# Be sure to shutdown after the test.\n",
        "!/root/.fly/bin/flyctl deploy --detach --remote-only\n",
        "!/root/.fly/bin/flyctl status"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0ySggkdlfq7M"
      },
      "outputs": [],
      "source": [
        "import subprocess, json\n",
        "result = subprocess.run([\"/root/.fly/bin/flyctl\",\"status\",\"-j\"], capture_output=True, text=True)\n",
        "if result.returncode==0:\n",
        "    hostname = json.loads(result.stdout)[\"Hostname\"]\n",
        "    os.environ['couchHost']=\"https://%s\" % (hostname)\n",
        "    print(\"Your couchDB server is https://%s/\" % (hostname))\n",
        "else:\n",
        "    print(\"Something occured.\")\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Finish setting up the CouchDB\n",
        "# Please repeat until the request is  completed without error messages\n",
        "# i.e., You have to redo this block while \"curl: (35) OpenSSL SSL_connect: Connection reset by peer in connection to xxxx\" is showing.\n",
        "#\n",
        "# Note: A few minutes migth be required to be booted.\n",
        "!curl -X POST \"${couchHost}/_cluster_setup\" -H \"Content-Type: application/json\" -d \"{\\\"action\\\":\\\"enable_single_node\\\",\\\"username\\\":\\\"${couchUser}\\\",\\\"password\\\":\\\"${couchPwd}\\\",\\\"bind_address\\\":\\\"0.0.0.0\\\",\\\"port\\\":5984,\\\"singlenode\\\":true}\"  --user \"${couchUser}:${couchPwd}\""
      ],
      "metadata": {
        "id": "cGlSzVqlQG_z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Please repeat until all lines are completed without error messages\n",
        "!curl -X PUT \"${couchHost}/_node/nonode@nohost/_config/chttpd/require_valid_user\" -H \"Content-Type: application/json\" -d '\"true\"' --user \"${couchUser}:${couchPwd}\"\n",
        "!curl -X PUT \"${couchHost}/_node/nonode@nohost/_config/chttpd_auth/require_valid_user\" -H \"Content-Type: application/json\" -d '\"true\"' --user \"${couchUser}:${couchPwd}\"\n",
        "!curl -X PUT \"${couchHost}/_node/nonode@nohost/_config/httpd/WWW-Authenticate\" -H \"Content-Type: application/json\" -d '\"Basic realm=\\\"couchdb\\\"\"' --user \"${couchUser}:${couchPwd}\"\n",
        "!curl -X PUT \"${couchHost}/_node/nonode@nohost/_config/httpd/enable_cors\" -H \"Content-Type: application/json\" -d '\"true\"' --user \"${couchUser}:${couchPwd}\"\n",
        "!curl -X PUT \"${couchHost}/_node/nonode@nohost/_config/chttpd/enable_cors\" -H \"Content-Type: application/json\" -d '\"true\"' --user \"${couchUser}:${couchPwd}\"\n",
        "!curl -X PUT \"${couchHost}/_node/nonode@nohost/_config/chttpd/max_http_request_size\" -H \"Content-Type: application/json\" -d '\"4294967296\"' --user \"${couchUser}:${couchPwd}\"\n",
        "!curl -X PUT \"${couchHost}/_node/nonode@nohost/_config/couchdb/max_document_size\" -H \"Content-Type: application/json\" -d '\"50000000\"' --user \"${couchUser}:${couchPwd}\"\n",
        "!curl -X PUT \"${couchHost}/_node/nonode@nohost/_config/cors/credentials\" -H \"Content-Type: application/json\" -d '\"true\"' --user \"${couchUser}:${couchPwd}\"\n",
        "!curl -X PUT \"${couchHost}/_node/nonode@nohost/_config/cors/origins\" -H \"Content-Type: application/json\" -d '\"app://obsidian.md,capacitor://localhost,http://localhost\"' --user \"${couchUser}:${couchPwd}\""
      ],
      "metadata": {
        "id": "JePzrsHypY18"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now, our CouchDB has been surely installed and configured. Cheers!\n",
        "\n",
        "In the steps that follow, create a setup-URI.\n",
        "\n",
        "This URI could be imported directly into Self-hosted LiveSync, to configure the use of the CouchDB which we configured now."
      ],
      "metadata": {
        "id": "YfSOomsoXbGS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Database config\n",
        "import random, string\n",
        "\n",
        "def randomname(n):\n",
        "   return ''.join(random.choices(string.ascii_letters + string.digits, k=n))\n",
        "\n",
        "# The database name\n",
        "os.environ['database']=\"obsidiannote\"\n",
        "# The passphrase to E2EE\n",
        "os.environ['passphrase']=randomname(20)\n",
        "\n",
        "print(\"Your database:\"+os.environ['database'])\n",
        "print(\"Your passphrase:\"+os.environ['passphrase'])"
      ],
      "metadata": {
        "id": "416YncOqXdNn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Install deno for make setup uri\n",
        "!curl -fsSL https://deno.land/x/install/install.sh | sh"
      ],
      "metadata": {
        "id": "C4d7C0HAXgsr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Fetch module for encrypting a Setup URI\n",
        "!curl -o encrypt.ts https://gist.githubusercontent.com/vrtmrz/f9d1d95ee2ca3afa1a924a2c6759b854/raw/d7a070d864a6f61403d8dc74208238d5741aeb5a/encrypt.ts"
      ],
      "metadata": {
        "id": "hQL_Dx-PXise"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Make buttons!\n",
        "from IPython.display import HTML\n",
        "result = subprocess.run([\"/root/.deno/bin/deno\",\"run\",\"-A\",\"encrypt.ts\"], capture_output=True, text=True)\n",
        "text=\"\"\n",
        "if result.returncode==0:\n",
        "  text = result.stdout.strip()\n",
        "  result = HTML(f\"<button onclick=navigator.clipboard.writeText('{text}')>Copy setup uri</button><br>Importing passphrase is `welcome`. <br>If you want to synchronise in live mode, please apply a preset after setup.)\")\n",
        "else:\n",
        "  result = \"Failed to encrypt the setup URI\"\n",
        "result"
      ],
      "metadata": {
        "id": "o0gX_thFXlIZ"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "private_outputs": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}

@vrtmrz Thanks for building this powerful sync tool! I tried to follow this Colab snippet and it worked in starting the sync service. However, my fly.io app still often gets hanged (no http response at all, probably being stopped?). Shall I add the

auto_stop_machines = false
auto_start_machines = false

Per vrtmrz/obsidian-livesync#221 (comment) ?

P.S.: I followed this to update a running app's config and it works well now https://community.fly.io/t/db-seems-to-be-pause-how-do-i-resume/13053/2