Hackmty.ipynb

hackmty.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Hackmty.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/Geobm/ee1c60556e54d1b5652282471289dc84/hackmty.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s3QA3QL_dCxR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 367
        },
        "outputId": "da700237-d458-49c1-8ac7-02b56dcd3d75"
      },
      "source": [
        "!pip install nltk\n",
        "\n",
        "!pip install gensim\n",
        "import nltk\n",
        "import json \n",
        "from nltk.corpus import stopwords \n",
        "from nltk.tokenize import word_tokenize \n",
        "import pandas as pd\n",
        "from nltk.tokenize import word_tokenize\n",
        "import numpy as np\n",
        "import re\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem.porter import PorterStemmer\n",
        "from nltk \n",
        "import difflib\n",
        "import requests\n",
        "import os\n",
        "from pprint import pprint\n",
        "import difflib\n",
        "import itertools\n",
        "from flask import Flask, jsonify"
      ],
      "execution_count": 155,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (3.2.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk) (1.15.0)\n",
            "Requirement already satisfied: gensim in /usr/local/lib/python3.6/dist-packages (3.6.0)\n",
            "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.4.1)\n",
            "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.6/dist-packages (from gensim) (2.1.0)\n",
            "Requirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.15.0)\n",
            "Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.18.5)\n",
            "Requirement already satisfied: boto in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (2.49.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (2.23.0)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (1.14.48)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (2020.6.20)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (3.0.4)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim) (0.10.0)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim) (0.3.3)\n",
            "Requirement already satisfied: botocore<1.18.0,>=1.17.48 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim) (1.17.48)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.18.0,>=1.17.48->boto3->smart-open>=1.2.1->gensim) (2.8.1)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.18.0,>=1.17.48->boto3->smart-open>=1.2.1->gensim) (0.15.2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S8gNF3ctP9hH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "cf212c17-808d-4a0f-b52f-ffa41622a006"
      },
      "source": [
        "\n",
        "github_token = \"99ad5dd512e16341e613145d10c8f603c615d633\"\n",
        "\n",
        "token = os.getenv('GITHUB_TOKEN', github_token)\n",
        "\n",
        "usernames = \"jg-11,andreamoronzavala,irving-estrada,geobm\"\n",
        "\n",
        "names = usernames.split(',')\n",
        "\n",
        "repo_description = []\n",
        "repo_name = []\n",
        "aux = 0\n",
        "final_list = []\n",
        "\n",
        "for name in names:\n",
        "  query_url = f\"https://api.github.com/users/{name}/repos\"\n",
        "  params = {\n",
        "      \"state\": \"open\",\n",
        "  }\n",
        "  headers = {'Authorization': f'token {token}'}\n",
        "  response = requests.get(query_url, headers=headers, params=params)\n",
        "  result = response.json()\n",
        "\n",
        "for i in range(len(result)):\n",
        "  if result[i]['description'] is not None:\n",
        "    repo_name.append(result[i]['name'])\n",
        "    repo_description.append(result[i]['description'])\n",
        "final_results = list(zip(repo_name,repo_description))\n",
        "#print(json.dumps([dict(final_results)]))\n",
        "\n",
        "\n",
        "tokenized_text = [word for word in nltk.tokenize.word_tokenize(str(repo_description)[1:]) if len(word) > 1]\n",
        "#print(tokenized_text)\n",
        "fdist = FreqDist(tokenized_text)\n",
        "text1 = ''.split(str(tokenized_text))\n",
        "fdist1 = nltk.FreqDist(tokenized_text)\n",
        "print (fdist1.most_common(10))"
      ],
      "execution_count": 202,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('training', 2), ('course', 2), ('the', 2), ('and', 2), ('for', 2), (\"'edX\", 1), (\"'Angular\", 1), ('CRUD', 1), ('front-end', 1), (\"'Advanced\", 1)]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}