Mimic the functionality of glob when using paramiko.

paramiko_glob.ipynb

{
  "cells": [
    {
      "metadata": {
        "init_cell": true,
        "collapsed": false,
        "ExecuteTime": {
          "start_time": "2016-06-28T11:48:07.416857",
          "end_time": "2016-06-28T11:48:07.668957"
        },
        "trusted": true
      },
      "cell_type": "code",
      "source": "# Find all txt-files on remote machine.\n# \n# /scratch/lehre/edvprak/u300509/\n# ├── bar\n# │   ├── bar1\n# │   │   ├── test.nc\n# │   │   └── test.txt\n# │   └── bar2\n# │       └── test.txt\n# └── foo\n#     ├── test2.txt\n#     └── test.pdf\n# \n# 4 directories, 5 files\n\nimport os\nimport re\nimport stat\nimport tempfile\n\nimport paramiko\n\n\ndef paramiko_glob(path, pattern, sftp):\n    \"\"\"Search recursively for files matching a given pattern.\n    \n    Parameters:\n        path (str): Path to directory on remote machine.\n        pattern (str): Python re [0] pattern for filenames.\n        sftp (SFTPClient): paramiko SFTPClient.\n        \n    [0] https://docs.python.org/2/library/re.html\n        \n    \"\"\"\n    p = re.compile(pattern)\n    root = sftp.listdir(path)\n    file_list = []\n    \n    # Loop over all entries in given path...\n    for f in (os.path.join(path, entry) for entry in root):\n        f_stat = sftp.stat(f)\n        # ... if it is a directory call paramiko_glob recursively.\n        if stat.S_ISDIR(f_stat.st_mode):\n            file_list += paramiko_glob(f, pattern, sftp)\n        # ... if it is a file, check the name pattern and append it to file_list.\n        elif p.match(f):\n            file_list.append(f)\n    return file_list\n\n\n# Setup paramiko connection.\nssh = paramiko.SSHClient()\nssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())\nssh.connect('lehre3')\nsftp = ssh.open_sftp()\n\n# Actucal call of paramiko_glob.\nfiles = paramiko_glob('/scratch/lehre/edvprak/u300509', '.*\\.txt', sftp)\nprint(files)\n\n# Path to temporary directory in RAM disk.\ntmpdir = '/dev/shm/u300509/'\nfor f in files:\n    # Copy remote file to local tmp path and read it.\n    local_tmp = tempfile.mktemp(prefix=tmpdir)\n    sftp.get(f, local_tmp)\n    with open(local_tmp) as ftmp:\n        print(ftmp.readlines())\n    os.remove(local_tmp)\n        \n# Remove temporary files and close connections.\nsftp.close()\nssh.close()",
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": "['/scratch/lehre/edvprak/u300509/foo/test2.txt', '/scratch/lehre/edvprak/u300509/bar/bar1/test.txt', '/scratch/lehre/edvprak/u300509/bar/bar2/test.txt']\n['I am test2.txt\\n']\n['I am test.txt in bar1\\n']\n['I am test.txt in bar2\\n']\n",
          "name": "stdout"
        }
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3",
      "language": "python"
    },
    "gist": {
      "id": "ddda28208f7658d93f8238ad88bd45f2",
      "data": {
        "description": "Mimic the functionality of glob when using paramiko.",
        "public": true
      }
    },
    "language_info": {
      "nbconvert_exporter": "python",
      "version": "3.5.1",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "name": "python",
      "mimetype": "text/x-python",
      "file_extension": ".py",
      "pygments_lexer": "ipython3"
    },
    "toc": {
      "toc_threshold": 6,
      "toc_window_display": false,
      "toc_number_sections": true,
      "toc_cell": false
    },
    "_draft": {
      "nbviewer_url": "https://gist.github.com/ddda28208f7658d93f8238ad88bd45f2"
    },
    "widgets": {
      "state": {},
      "version": "1.1.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}