How to copy UDPs from one openEO backend to another

migrate-udps.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bc4cf1ab-0900-49e1-a53e-759c8ad074f1",
   "metadata": {},
   "source": [
    "\n",
    "# How to migrate UDPs?\n",
    "\n",
    "How to copy user-defined processes (UDPs) stored under the `/process_graphs` sub-API from one backend to another?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cc8da6c2-f029-437b-a087-8c22d4cd27eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openeo"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7da41a8-4ceb-429e-86ef-3066e9887cbe",
   "metadata": {},
   "source": [
    "## Create connections\n",
    "\n",
    "Source backend to copy from"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ac2e64f6-8088-4c91-bf36-ca56e5422f36",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Authenticated using refresh token.\n"
     ]
    }
   ],
   "source": [
    "source_backend = openeo.connect(\n",
    "    \"openeo.vito.be\",\n",
    ").authenticate_oidc()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e75e48bb-7062-46b6-ae43-2c70836a0009",
   "metadata": {},
   "source": [
    "Target backend to copy to"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5f06fdeb-b809-4a9e-9581-500e43b9fe80",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Authenticated using refresh token.\n"
     ]
    }
   ],
   "source": [
    "target_backend = openeo.connect(\n",
    "    \"openeo.terrascope.be\",\n",
    "    # Avoid unhelpful warnings from broken UDP validation. \n",
    "    # Also see https://github.com/Open-EO/openeo-python-driver/issues/198\n",
    "    auto_validate=False, \n",
    ").authenticate_oidc()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "336a4acc-9768-4cd8-9136-0fca620157c7",
   "metadata": {},
   "source": [
    "## List UDPs\n",
    "\n",
    "List UDP ids available at the source backend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "880259f4-a062-450d-a009-5ebf3f694219",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['3plus5', 'add3', 'add35', 'fahrenheit_to_celsius', 'load_foobar', 'test01']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source_udp_ids = [u[\"id\"] for u in source_backend.list_user_defined_processes()]\n",
    "source_udp_ids"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29f4f02d-e9dd-42be-b029-34b2b4186623",
   "metadata": {},
   "source": [
    "Just to be sure, also check what's already at target backend."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "62315bd6-c3ed-4310-b4be-948e4a200f33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['add35', 'test01']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_udp_ids = [u[\"id\"] for u in target_backend.list_user_defined_processes()]\n",
    "target_udp_ids"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a021624d-bdc0-4a96-aa12-cfdf760bce59",
   "metadata": {},
   "source": [
    "## Do the copy work\n",
    "\n",
    "Start with some helpers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d139844b-d36f-48dc-adf6-65bcc1dbc241",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_udp(process_id: str, *, backend: openeo.Connection) -> dict:\n",
    "    udp_dump = backend.user_defined_process(process_id).describe()\n",
    "    print(f\"Found {process_id!r} at {backend.root_url}: {repr(udp_dump)[:100]}...\")\n",
    "    assert udp_dump[\"id\"] == process_id\n",
    "    return udp_dump\n",
    "\n",
    "def store_udp(udp_dump: dict, *, backend: openeo.Connection) -> None:\n",
    "    process_id = udp_dump.pop(\"id\")\n",
    "    print(f\"Storing {process_id!r} at {backend.root_url}\")\n",
    "    backend.save_user_defined_process(\n",
    "        user_defined_process_id=process_id,\n",
    "        **udp_dump,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d8c147b4-31a2-4d23-9ba1-9dff051a40e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- 3plus5 ---\n",
      "Found '3plus5' at https://openeo.vito.be/openeo/1.2/: {'id': '3plus5', 'links': [{'href': 'https://openeo.vito.be/openeo/1.2/processes/u:0aee24f6aaf35f49f...\n",
      "Storing '3plus5' at https://openeo.terrascope.be/openeo/1.2/\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lippenss/src/openeo/openeo-python-client/openeo/rest/connection.py:1093: UserWarning: Defining user-defined process '3plus5' without parameters\n",
      "  warnings.warn(\"Defining user-defined process {u!r} without parameters\".format(u=user_defined_process_id))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- add3 ---\n",
      "Found 'add3' at https://openeo.vito.be/openeo/1.2/: {'id': 'add3', 'links': [{'href': 'https://openeo.vito.be/openeo/1.2/processes/u:0aee24f6aaf35f49fde...\n",
      "Storing 'add3' at https://openeo.terrascope.be/openeo/1.2/\n",
      "--- add35 ---\n",
      "add35 already exists: skipping\n",
      "--- fahrenheit_to_celsius ---\n",
      "Found 'fahrenheit_to_celsius' at https://openeo.vito.be/openeo/1.2/: {'id': 'fahrenheit_to_celsius', 'parameters': [{'description': 'Degrees Fahrenheit.', 'name': 'f', '...\n",
      "Storing 'fahrenheit_to_celsius' at https://openeo.terrascope.be/openeo/1.2/\n",
      "--- load_foobar ---\n",
      "Found 'load_foobar' at https://openeo.vito.be/openeo/1.2/: {'id': 'load_foobar', 'parameters': [], 'process_graph': {'loadcollection1': {'arguments': {'bands':...\n",
      "Storing 'load_foobar' at https://openeo.terrascope.be/openeo/1.2/\n",
      "--- test01 ---\n",
      "test01 already exists: skipping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lippenss/src/openeo/openeo-python-client/openeo/rest/connection.py:1093: UserWarning: Defining user-defined process 'load_foobar' without parameters\n",
      "  warnings.warn(\"Defining user-defined process {u!r} without parameters\".format(u=user_defined_process_id))\n"
     ]
    }
   ],
   "source": [
    "for udp_id in source_udp_ids:\n",
    "    print(\"---\", udp_id, \"---\")\n",
    "    \n",
    "    if udp_id in target_udp_ids:\n",
    "        print(f\"{udp_id} already exists: skipping\")\n",
    "        continue\n",
    "        \n",
    "    udp = get_udp(udp_id, backend=source_backend)\n",
    "    store_udp(udp, backend=target_backend)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67ca0e38-52b8-4417-823f-8067f15319bd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe6fa3b6-5ee1-4e8d-8b57-bd17ed99611e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14fbe9f7-1056-42e3-9958-985ffe66b2dd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "openeo-python-client-py313",
   "language": "python",
   "name": "openeo-python-client-py313"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}