Created
May 1, 2020 23:30
-
-
Save faizankshaikh/2de80586a0300dab2a1c7a17786bb9b0 to your computer and use it in GitHub Desktop.
PyTorch_vs_TensorFlow_v2.ipynb
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "nbformat": 4, | |
| "nbformat_minor": 0, | |
| "metadata": { | |
| "colab": { | |
| "name": "PyTorch_vs_TensorFlow_v2.ipynb", | |
| "provenance": [], | |
| "authorship_tag": "ABX9TyPSmlISIDCyI2tLvg8EGyoY", | |
| "include_colab_link": true | |
| }, | |
| "kernelspec": { | |
| "name": "python3", | |
| "display_name": "Python 3" | |
| } | |
| }, | |
| "cells": [ | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "view-in-github", | |
| "colab_type": "text" | |
| }, | |
| "source": [ | |
| "<a href=\"https://colab.research.google.com/gist/faizankshaikh/2de80586a0300dab2a1c7a17786bb9b0/pytorch_vs_tensorflow_v2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "X1FpZ3TEtJQ0", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "!pip install openreview-py" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "B6y_hJN22Pfe", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "!pip install pipreqs" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "jIS6cI71xzv5", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "import os\n", | |
| "import re\n", | |
| "import sys\n", | |
| "import requests\n", | |
| "import openreview\n", | |
| "import pandas as pd\n", | |
| "\n", | |
| "from random import choice\n", | |
| "from urllib.parse import urlparse" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "6zI1Vh1QxzyX", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "client = openreview.Client(baseurl=\"https://openreview.net\")" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "EaFb-H5Gxz4H", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "blind_notes = {note.id: note for note in openreview.tools.iterget_notes(client, invitation = 'ICLR.cc/2020/Conference/-/Blind_Submission', details='original')}" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "Zkwd8dfZxz7v", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "all_decision_notes = openreview.tools.iterget_notes(client, invitation = 'ICLR.cc/2020/Conference/Paper.*/-/Decision')" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "vpvdGgX0tUqp", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "accepted_submissions = [blind_notes[decision_note.forum] for decision_note in all_decision_notes if 'Accept' in decision_note.content['decision']]" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "Y-B1PGOW1WiB", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 34 | |
| }, | |
| "outputId": "4ee5c674-d4c0-4358-e6d6-9aa12a20eac9" | |
| }, | |
| "source": [ | |
| "len(accepted_submissions)" | |
| ], | |
| "execution_count": 7, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/plain": [ | |
| "687" | |
| ] | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "execution_count": 7 | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "4lR_Tz2Z3Yy5", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "code_present = 0\n", | |
| "code_links = []\n", | |
| "for note in accepted_submissions:\n", | |
| " try:\n", | |
| " code_links.append(note.content['code'])\n", | |
| " # print(\"code found\")\n", | |
| " code_present += 1\n", | |
| " except:\n", | |
| " print(\"Unexpected error:\", sys.exc_info()[0])" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "I5XsTqPSzd5k", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 34 | |
| }, | |
| "outputId": "9e4f31ec-317c-4426-e5f3-16cbd9123722" | |
| }, | |
| "source": [ | |
| "code_present" | |
| ], | |
| "execution_count": 9, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/plain": [ | |
| "344" | |
| ] | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "execution_count": 9 | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "4S6_b3HmNf87", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 34 | |
| }, | |
| "outputId": "87438834-3282-4e49-c9af-1083de83d53c" | |
| }, | |
| "source": [ | |
| "urlparse(choice(code_links))" | |
| ], | |
| "execution_count": 10, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/plain": [ | |
| "ParseResult(scheme='https', netloc='github.com', path='/google-research/augmix', params='', query='', fragment='')" | |
| ] | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "execution_count": 10 | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "7LfRhRoiQQUG", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "code_links_df = pd.DataFrame({'links': code_links})" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "twndVsQiSlaZ", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "code_links_df['domains'] = code_links_df.links.apply(lambda x: urlparse(x)[1])" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "W4uZyQQ3Qx2S", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 503 | |
| }, | |
| "outputId": "499fd717-51f5-4071-f7df-bf37a225828e" | |
| }, | |
| "source": [ | |
| "code_links_df.domains.value_counts()" | |
| ], | |
| "execution_count": 13, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/plain": [ | |
| "github.com 268\n", | |
| "drive.google.com 28\n", | |
| "www.dropbox.com 9\n", | |
| "anonymous.4open.science 8\n", | |
| "bit.ly 5\n", | |
| "storage.googleapis.com 3\n", | |
| "sites.google.com 2\n", | |
| "s000.tinyupload.com 2\n", | |
| "toiaydcdyywlhzvlob.github.io 1\n", | |
| "nikaashpuri.github.io 1\n", | |
| "docs.google.com 1\n", | |
| "www.github.com 1\n", | |
| "www.daml.in.tum.de 1\n", | |
| "wgrathwohl.github.io 1\n", | |
| "anonfile.com 1\n", | |
| "www.robots.ox.ac.uk 1\n", | |
| "www.cs.cmu.edu 1\n", | |
| "whyu.me 1\n", | |
| "danijar.com 1\n", | |
| "www.sendspace.com 1\n", | |
| "goo.gl 1\n", | |
| "clevrer.csail.mit.edu 1\n", | |
| "rohitgirdhar.github.io 1\n", | |
| "nitishgupta.github.io 1\n", | |
| "dap.csail.mit.edu 1\n", | |
| "automated-discovery.github.io 1\n", | |
| "mega.nz 1\n", | |
| "Name: domains, dtype: int64" | |
| ] | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "execution_count": 13 | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "1BBkCJmtVVDB", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "temp_link = \"\"\n", | |
| "def clean_github_link(link):\n", | |
| " link = link.strip()\n", | |
| " if not link[-4:] == \".git\":\n", | |
| " return link + \".git\"\n", | |
| " else:\n", | |
| " return link\n", | |
| "\n", | |
| "github_repo_links = code_links_df.loc[code_links_df.domains == 'github.com'].links.apply(clean_github_link).values" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "0BeWVTO3l4Vx", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 764 | |
| }, | |
| "outputId": "7d756297-70b3-4479-e707-d0718743096a" | |
| }, | |
| "source": [ | |
| "# takes about 24 minutes to download\n", | |
| "for link in github_repo_links:\n", | |
| " !git clone $link --depth 1 --quiet" | |
| ], | |
| "execution_count": 15, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "text": [ | |
| "fatal: repository 'https://github.com/tensorflow/addons/blob/master/tensorflow_addons/optimizers/lamb.py.git/' not found\n", | |
| "fatal: repository 'https://github.com/carloderamo/shared/tree/master.git/' not found\n", | |
| "Cloning into 'proxsgd'...\n", | |
| "remote: Enumerating objects: 59, done.\u001b[K\n", | |
| "remote: Counting objects: 100% (59/59), done.\u001b[K\n", | |
| "remote: Compressing objects: 100% (45/45), done.\u001b[K\n", | |
| "remote: Total 59 (delta 19), reused 33 (delta 12), pack-reused 0\u001b[K\n", | |
| "Unpacking objects: 100% (59/59), done.\n", | |
| "/bin/bash: https://github.com/cc-hpc-itwm/proxsgd.git: No such file or directory\n", | |
| "fatal: repository 'https://github.com/suraj-nair-1/google-research/tree/master/hierarchical_foresight.git/' not found\n", | |
| "fatal: repository 'https://github.com/google-research/google-research/tree/master/cfq.git/' not found\n", | |
| "remote: Not Found\n", | |
| "fatal: repository 'https://github.com/hangg7/deformable-kernels/.git/' not found\n", | |
| "fatal: repository 'https://github.com/google-research/google-research/tree/master/meta_learning_without_memorization.git/' not found\n", | |
| "remote: Not Found\n", | |
| "fatal: repository 'https://github.com/GRAM-nets.git/' not found\n", | |
| "fatal: could not read Username for 'https://github.com': No such device or address\n", | |
| "remote: Not Found\n", | |
| "fatal: repository 'http://github.com/AvigdorZ.git/' not found\n", | |
| "fatal: repository 'https://github.com/google-research/language/tree/master/language/bert_extraction.git/' not found\n", | |
| "remote: Not Found\n", | |
| "fatal: repository 'https://github.com/anonymous-sushi-armadillo.git/' not found\n", | |
| "fatal: repository 'https://github.com/NeurEXT/NEXT-learning-to-plan/blob/master/main.ipynb.git/' not found\n", | |
| "fatal: repository 'https://github.com/tensorflow/federated/tree/master/tensorflow_federated/python/research/gans.git/' not found\n", | |
| "remote: Not Found\n", | |
| "fatal: repository 'https://github.com/snap-stanford/pretrain-gnns/.git/' not found\n", | |
| "remote: Not Found\n", | |
| "fatal: repository 'https://github.com/PKU-AI-Edge/DGN/.git/' not found\n", | |
| "fatal: repository 'https://github.com/google-research/google-research/tree/master/weak_disentangle.git/' not found\n", | |
| "fatal: repository 'https://github.com/google/trax/tree/master/trax/models/reformer.git/' not found\n", | |
| "Cloning into 'neural-tangent-kernel-UCI'...\n", | |
| "remote: Enumerating objects: 38, done.\u001b[K\n", | |
| "remote: Counting objects: 100% (38/38), done.\u001b[K\n", | |
| "remote: Compressing objects: 100% (30/30), done.\u001b[K\n", | |
| "remote: Total 38 (delta 16), reused 19 (delta 6), pack-reused 0\u001b[K\n", | |
| "Unpacking objects: 100% (38/38), done.\n", | |
| "/bin/bash: https://drive.google.com/open?id=1SdgWmhEcnm4qyaM9xrkN01VF9tj40WZS.git: No such file or directory\n", | |
| "remote: Not Found\n", | |
| "fatal: repository 'https://github.com/nathandelara/Spectral-Embedding-of-Regularized-Block-Models/.git/' not found\n", | |
| "fatal: could not read Username for 'https://github.com': No such device or address\n", | |
| "remote: Not Found\n", | |
| "fatal: repository 'https://github.com/deepsphere.git/' not found\n", | |
| "fatal: could not read Username for 'https://github.com': No such device or address\n" | |
| ], | |
| "name": "stdout" | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "XU-Ez7Rv_-Cj", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 208 | |
| }, | |
| "outputId": "f2577855-7a4f-4fb2-b314-4e5478927a37" | |
| }, | |
| "source": [ | |
| "code_links_df.loc[code_links_df.domains == 'github.com'].links.apply(lambda x: urlparse(x)[2].split(\"/\")[1]).value_counts().head(10)" | |
| ], | |
| "execution_count": 40, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/plain": [ | |
| "google-research 10\n", | |
| "facebookresearch 4\n", | |
| "TAMU-VITA 3\n", | |
| "P2333 2\n", | |
| "cornell-zhang 2\n", | |
| "epfml 2\n", | |
| "TonghanWang 2\n", | |
| "eth-sri 2\n", | |
| "haebeom-lee 2\n", | |
| "tensorflow 2\n", | |
| "Name: links, dtype: int64" | |
| ] | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "execution_count": 40 | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "sB1EuXzku6fz", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 54 | |
| }, | |
| "outputId": "72694f55-2b1f-419a-ac62-34857e706f97" | |
| }, | |
| "source": [ | |
| "root='.'\n", | |
| "dirlist = [ item for item in os.listdir(root) if os.path.isdir(os.path.join(root, item)) ]\n", | |
| "print(dirlist)" | |
| ], | |
| "execution_count": 16, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "text": [ | |
| "['.config', 'KP2D', 'CN-DPM', 'GLAD', 'bert_score', 'delay_stability', 'word2ket', 'UniversalCertificationTheory', 'smile-mi-estimator', 'SI-NI-FGSM', 'mamus', 'impact-driven-exploration', 'InfoGraph', 'IGMC', 'stable-nalu', 'KG-A2C', 'hilloc', 'Multi-View-Information-Bottleneck', 'Incremental-Learning', 'GraphSAINT', 'mbu-content-tansfer', 'complex-order', 'augmix', 'space2vec', 'sparse_deconvolution', 'MABN', 'nonnormal-init', 'DeepSpeechDistances', 'Target-Agnostic-Attack', 'iclr_paper_mdfa', 'ICLR2020-PADGN', 'selection-via-proxy', 'kWTA-Activation', 'VHE-GAN', 'IDA', 'UCB', 'deep-skill-chaining', 'VL-BERT', 'meta-learning-curiosity-algorithms', 'mmr-universal', 'Data-Independent-Neural-Pruning-via-Coresets', 'LAMOL', 'CS-GNN', 'fedavgpy', 'eval-nas', 'IBA-paper-code', 'PAC-confidence-set', 'MAD', 'attackbox', 'cophy', 'FreeLB', 'PlasmaML', 'habitat-api', 'node', 'RidgeRegression', 'degen', 'ICLR-2020', 'CompGCN', 'synthfeedback', 'geom-gcn', 'knnlm', 'ddsp', 'metadrop', 'learning-circuits', 'pytorch-ensembles', 'NADST', 'codes', 'ALBERT', 'gsplinets', 'neural-tangent-kernel-UCI', 'rio-paper', 'RLFAT', 'How-to-0wn-NAS-in-Your-Spare-Time', 'phattacks', 'four_things_batch_norm', 'Explorer', 'neural-imaging', 'QCNN', 'sparsity-normalization', 'NDQ', 'ASN', 'difftaichi', 'group_DRO', 'mma_training', 'EMPIR', 'AtomNAS', 'Regularized_autoencoders-RAE-', 'APoT_Quantization', 'kge', 'Neural-SLAM', 'certifiedpatchdefense', 'LMRS', 'estimating-gradients-without-replacement', 'Distributional-Signatures', 'GansFallingShort', 'Equivalence', 'neural_3d_mapping', 'ZMZM-ICLR-2020', 'WhiteNoiseAnalysis', 'multi-agent-emergence-environments', 'BayesOpt_Attack', 'OptimalStrategiesAgainstGenerativeAttacks', 'NSM', 'dnn-gating', 'lnfmm', 'Mixup-Inference', 'RepDistiller', 'PairNorm', 'adversarial-negative-sampling', 'why-clipping-accelerates', 'saliency_maps', 'conservative-uncertainty-estimation-random-priors', 'local-ensembles', 'max-margin', 'AdvectiveNet-An-Eulerian-Lagrangian-Fluidic-Reservoir-for-Point-Cloud-Processing', 'hybrid-snn-conversion', 'guiding-synthesizers', 'gnn-asymptotics', 'GraN-DAG', 's2p', 'DPMPN', 'CoDAIL', 'MMT', '2simplicialtransformer', 'UVR-NMT', 'SV-RL', 'Early-Bird-Tickets', 'Learning-From-Rules', 'triple-wins', 'sparse-embed', 'PPLM', 'provable_pruning', 'lookahead_pruning', 'proxsgd', 'GLISTA', 'e2efold', 'lord', 'ChocoSGD', 'NAS-Bench-201', 'edward2', 'anonymous_iclr2020_apd_code', 'order-learning', 'RAdam', 'GAT-Generative-Adversarial-Training', 'relational-ssm', 'learning_to_retrieve_reasoning_paths', 'deeprl_network', 'dynamics-aware-embeddings', 'Neural_Iterated_Learning', 'packnet-sfm', 'macer', 'ZO-L2L', 'NLIL', 'warpgrad', 'GraphsFewShot', 'UGATIT', 'gnn-comparison', 'CROWN-IBP', 'RL-based-Graph2Seq-for-NQG', 'epciclr2020', 'query2box', 'adversarial-policies', 'FasterSeg', 'A2L', 'fair_flearn', 'RealnessGAN', 'DropEdge', 'rl-reliability-metrics', 'c-swm', 'rewinding-iclr20-public', 'sensitive-subspace-robustness', 'ml-capsules-inverted-attention-routing', 'protein-ebm', 'FedMA', 'GA', 'SRU_for_GCI', 'MetaBO', 'overparameterized_convolutional_generators', 'GraphMemoryNet', 'RobustDARTS', 'Permutation-Equivariant-Seq2Seq', 'Symplectic-ODENet', 'once-for-all', 'RL-Indirect-imitation', 'CLCL', 'remixmatch', 'pytorch-blockswap', 'Max-Mahalanobis-Training', 'joint-align', 'DeepHoyer', 'convcnp', 'Learning-to-Group', 'nasbench-1shot1', 'genesis', 'deconvolution', 'ACMC_ICLR', 'Understanding-NAS', 'mvae', 'EITI-EDTI', 'DynamicSaprseTraining', 'CausalOptimizationAnon', 'sequential-knowledge-transformer', 'Pseudo_Lidar_V2', 'l2b', 'bsuite', 'ifl-tpp', 'RaCT_CF', 'Certify_Topk', 'BinaryDuo', 'ARML', 'dads', 'imd', 'ExpressGNN', 'PCMC-Net', 'editable', 'electra', 'MaSS', 'Explanation_by_Progressive_Exaggeration', 'deep-graph-matching-consensus', 'zebrafish-learning', 'bert-nmt', 'interaction_interpretability', 'attention-cnn', 'hypercl', 'iclr20-lcn', 'ExpectedInformationMaximization', 'TREMBA', 'LambdaNet', 'hijacking', 'Deep-SAD-PyTorch', 'TruthOrBackpropaganda', 'pcl2pcl-gan-pub', 'What-Can-Neural-Networks-Reason-About', 'GraphZoom', 'fspool', 'counterfactually-augmented-data', 'pau', 'STOVE', 'xfer', 'FNA', 'NAS-Benchmark', 'coordination', 'blackbox-adv-examples-signhunter', 'code-for-paper', 'netrand', 'Variational-Recurrent-Models', 'Robustness-Verification-for-Transformers', 'mixout', 'sample_data']\n" | |
| ], | |
| "name": "stdout" | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "kUCH7Ljp2V_o", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "dirlist.remove('.config')\n", | |
| "dirlist.remove('sample_data')" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "1QTwTNgV2XzN", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 34 | |
| }, | |
| "outputId": "55f3b70e-7544-4613-c2d0-84f61611c0db" | |
| }, | |
| "source": [ | |
| "len(dirlist)" | |
| ], | |
| "execution_count": 18, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/plain": [ | |
| "247" | |
| ] | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "execution_count": 18 | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "3bvBYnVC2aV7", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "# takes about 10 minutes to run\n", | |
| "for repo in dirlist:\n", | |
| " path = \"/content/\" + repo\n", | |
| " if os.path.exists(path + \"/requirements.txt\"):\n", | |
| " pass\n", | |
| " else:\n", | |
| " !pipreqs $path" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "e97_Cmun3A4o", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "has_req_cnt = no_req_cnt = 0\n", | |
| "for repo in dirlist:\n", | |
| " path = \"/content/\" + repo\n", | |
| " if os.path.exists(path + \"/requirements.txt\"):\n", | |
| " has_req_cnt += 1\n", | |
| " else:\n", | |
| " no_req_cnt += 1" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "VlE_rhPw-1a2", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 34 | |
| }, | |
| "outputId": "022ade5d-c7a4-49ac-d547-ddff3d422343" | |
| }, | |
| "source": [ | |
| "has_req_cnt, no_req_cnt" | |
| ], | |
| "execution_count": 48, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/plain": [ | |
| "(237, 10)" | |
| ] | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "execution_count": 48 | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "SKPIZD2_JwNY", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "with open(\"/content/\" + dirlist[4] + \"/\" + \"requirements.txt\", \"r\") as f:\n", | |
| " tools = f.readlines()" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "a7zUQfHsPQQQ", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 34 | |
| }, | |
| "outputId": "62e310f4-848c-467b-b913-7bf89459c847" | |
| }, | |
| "source": [ | |
| "\",\".join(tools).lower()" | |
| ], | |
| "execution_count": 106, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/plain": [ | |
| "'torch==1.5.0+cu101\\n'" | |
| ] | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "execution_count": 106 | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "0behZVYwHdrT", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 191 | |
| }, | |
| "outputId": "a03a11ba-23b9-4f63-c1a1-bde477539f39" | |
| }, | |
| "source": [ | |
| "all_repo_names = []\n", | |
| "all_tool_names = []\n", | |
| "for repo in dirlist:\n", | |
| " try:\n", | |
| " repo_name = repo\n", | |
| " with open(\"/content/\" + repo + \"/\" + \"requirements.txt\", \"r\") as f:\n", | |
| " tools = f.readlines()\n", | |
| " tool_names = \",\".join(tools).lower()\n", | |
| "\n", | |
| " all_repo_names.append(repo_name)\n", | |
| " all_tool_names.append(tool_names)\n", | |
| " except:\n", | |
| " print(\"Unexpected error for \", repo, sys.exc_info()[0])" | |
| ], | |
| "execution_count": 107, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "text": [ | |
| "Unexpected error for SI-NI-FGSM <class 'FileNotFoundError'>\n", | |
| "Unexpected error for space2vec <class 'FileNotFoundError'>\n", | |
| "Unexpected error for synthfeedback <class 'FileNotFoundError'>\n", | |
| "Unexpected error for QCNN <class 'FileNotFoundError'>\n", | |
| "Unexpected error for GraN-DAG <class 'FileNotFoundError'>\n", | |
| "Unexpected error for GLISTA <class 'FileNotFoundError'>\n", | |
| "Unexpected error for ACMC_ICLR <class 'FileNotFoundError'>\n", | |
| "Unexpected error for PCMC-Net <class 'FileNotFoundError'>\n", | |
| "Unexpected error for pcl2pcl-gan-pub <class 'FileNotFoundError'>\n", | |
| "Unexpected error for NAS-Benchmark <class 'FileNotFoundError'>\n" | |
| ], | |
| "name": "stdout" | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "lp60yWHeJmOX", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 206 | |
| }, | |
| "outputId": "8dcc8970-0bb9-4b4f-8ba4-ed2ef282c8ec" | |
| }, | |
| "source": [ | |
| "all_tools = pd.DataFrame({\"all_repo_names\": all_repo_names, \"all_tool_names\":all_tool_names})\n", | |
| "all_tools.head()" | |
| ], | |
| "execution_count": 108, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/html": [ | |
| "<div>\n", | |
| "<style scoped>\n", | |
| " .dataframe tbody tr th:only-of-type {\n", | |
| " vertical-align: middle;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe tbody tr th {\n", | |
| " vertical-align: top;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe thead th {\n", | |
| " text-align: right;\n", | |
| " }\n", | |
| "</style>\n", | |
| "<table border=\"1\" class=\"dataframe\">\n", | |
| " <thead>\n", | |
| " <tr style=\"text-align: right;\">\n", | |
| " <th></th>\n", | |
| " <th>all_repo_names</th>\n", | |
| " <th>all_tool_names</th>\n", | |
| " </tr>\n", | |
| " </thead>\n", | |
| " <tbody>\n", | |
| " <tr>\n", | |
| " <th>0</th>\n", | |
| " <td>KP2D</td>\n", | |
| " <td>\\n</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>1</th>\n", | |
| " <td>CN-DPM</td>\n", | |
| " <td>ipdb\\n,jupyterlab\\n,matplotlib\\n,numpy\\n,pyyam...</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>2</th>\n", | |
| " <td>GLAD</td>\n", | |
| " <td>matplotlib==3.2.1\\n,networkx==2.4\\n,pandas==1....</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>3</th>\n", | |
| " <td>bert_score</td>\n", | |
| " <td># pytorch\\n,torch>=1.0.0\\n,# progress bars in ...</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>4</th>\n", | |
| " <td>delay_stability</td>\n", | |
| " <td>scipy==1.4.1\\n,matplotlib==3.2.1\\n,torchvision...</td>\n", | |
| " </tr>\n", | |
| " </tbody>\n", | |
| "</table>\n", | |
| "</div>" | |
| ], | |
| "text/plain": [ | |
| " all_repo_names all_tool_names\n", | |
| "0 KP2D \\n\n", | |
| "1 CN-DPM ipdb\\n,jupyterlab\\n,matplotlib\\n,numpy\\n,pyyam...\n", | |
| "2 GLAD matplotlib==3.2.1\\n,networkx==2.4\\n,pandas==1....\n", | |
| "3 bert_score # pytorch\\n,torch>=1.0.0\\n,# progress bars in ...\n", | |
| "4 delay_stability scipy==1.4.1\\n,matplotlib==3.2.1\\n,torchvision..." | |
| ] | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "execution_count": 108 | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "_0iqcGCbQVF9", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "all_tools.to_csv('all_tools.csv', index=False)" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "opq-MF95Qjnq", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "all_tools = pd.read_csv('all_tools.csv')" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "frzmPnsNQj9i", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 206 | |
| }, | |
| "outputId": "c16afcc7-260a-4adc-ccd4-8a7a104dad8c" | |
| }, | |
| "source": [ | |
| "all_tools.head()" | |
| ], | |
| "execution_count": 3, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/html": [ | |
| "<div>\n", | |
| "<style scoped>\n", | |
| " .dataframe tbody tr th:only-of-type {\n", | |
| " vertical-align: middle;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe tbody tr th {\n", | |
| " vertical-align: top;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe thead th {\n", | |
| " text-align: right;\n", | |
| " }\n", | |
| "</style>\n", | |
| "<table border=\"1\" class=\"dataframe\">\n", | |
| " <thead>\n", | |
| " <tr style=\"text-align: right;\">\n", | |
| " <th></th>\n", | |
| " <th>all_repo_names</th>\n", | |
| " <th>all_tool_names</th>\n", | |
| " </tr>\n", | |
| " </thead>\n", | |
| " <tbody>\n", | |
| " <tr>\n", | |
| " <th>0</th>\n", | |
| " <td>KP2D</td>\n", | |
| " <td>\\n</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>1</th>\n", | |
| " <td>CN-DPM</td>\n", | |
| " <td>ipdb\\n,jupyterlab\\n,matplotlib\\n,numpy\\n,pyyam...</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>2</th>\n", | |
| " <td>GLAD</td>\n", | |
| " <td>matplotlib==3.2.1\\n,networkx==2.4\\n,pandas==1....</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>3</th>\n", | |
| " <td>bert_score</td>\n", | |
| " <td># pytorch\\n,torch>=1.0.0\\n,# progress bars in ...</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>4</th>\n", | |
| " <td>delay_stability</td>\n", | |
| " <td>scipy==1.4.1\\n,matplotlib==3.2.1\\n,torchvision...</td>\n", | |
| " </tr>\n", | |
| " </tbody>\n", | |
| "</table>\n", | |
| "</div>" | |
| ], | |
| "text/plain": [ | |
| " all_repo_names all_tool_names\n", | |
| "0 KP2D \\n\n", | |
| "1 CN-DPM ipdb\\n,jupyterlab\\n,matplotlib\\n,numpy\\n,pyyam...\n", | |
| "2 GLAD matplotlib==3.2.1\\n,networkx==2.4\\n,pandas==1....\n", | |
| "3 bert_score # pytorch\\n,torch>=1.0.0\\n,# progress bars in ...\n", | |
| "4 delay_stability scipy==1.4.1\\n,matplotlib==3.2.1\\n,torchvision..." | |
| ] | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "execution_count": 3 | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "YnYUKwoeU-E9", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 34 | |
| }, | |
| "outputId": "b4214270-38f7-4ca7-c002-3b2d0f17ff5e" | |
| }, | |
| "source": [ | |
| "all_tools.shape" | |
| ], | |
| "execution_count": 4, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/plain": [ | |
| "(237, 2)" | |
| ] | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "execution_count": 4 | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "jFx9OXgCgtQs", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "def give_score(tool_name, offset=0):\n", | |
| " num = all_tools.all_tool_names.str.contains(tool_name).sum()\n", | |
| " num += offset\n", | |
| " print(\"Count and total percent of\", tool_name, \"is\", num, \"and\", round((num / all_tools.shape[0])*100, 4), \"%\")" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "YKZ3d8nTQxMX", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 34 | |
| }, | |
| "outputId": "16782401-dbff-457b-9312-4b5d5ab17dd3" | |
| }, | |
| "source": [ | |
| "give_score(\"torch\")" | |
| ], | |
| "execution_count": 6, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "text": [ | |
| "Count and total percent of torch is 155 and 65.4008 %\n" | |
| ], | |
| "name": "stdout" | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "MkL-dd9yRnf0", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 34 | |
| }, | |
| "outputId": "82614c50-889b-4d14-a46c-1e57cd01e466" | |
| }, | |
| "source": [ | |
| "give_score(\"tensorflow\")" | |
| ], | |
| "execution_count": 7, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "text": [ | |
| "Count and total percent of tensorflow is 85 and 35.865 %\n" | |
| ], | |
| "name": "stdout" | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "uBuE-VLSSQmS", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 34 | |
| }, | |
| "outputId": "c2c32569-e760-426d-b055-f55de4c08697" | |
| }, | |
| "source": [ | |
| "give_score(\"keras\")" | |
| ], | |
| "execution_count": 8, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "text": [ | |
| "Count and total percent of keras is 23 and 9.7046 %\n" | |
| ], | |
| "name": "stdout" | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "WiEluy-xSFD_", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 34 | |
| }, | |
| "outputId": "63c6db75-6945-4b69-a239-1be6e3f54401" | |
| }, | |
| "source": [ | |
| "give_score(\"transformers\")" | |
| ], | |
| "execution_count": 9, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "text": [ | |
| "Count and total percent of transformers is 10 and 4.2194 %\n" | |
| ], | |
| "name": "stdout" | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "CVtqVpmiSpub", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 34 | |
| }, | |
| "outputId": "d442cc97-97a1-4cb1-b714-e57bd96f0a70" | |
| }, | |
| "source": [ | |
| "give_score(\"tensorboard\")" | |
| ], | |
| "execution_count": 10, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "text": [ | |
| "Count and total percent of tensorboard is 57 and 24.0506 %\n" | |
| ], | |
| "name": "stdout" | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "wE7u_qHSVaRs", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 34 | |
| }, | |
| "outputId": "882e3f1c-7630-43ea-a907-c20621fa6f62" | |
| }, | |
| "source": [ | |
| "give_score(\"numpy\")" | |
| ], | |
| "execution_count": 11, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "text": [ | |
| "Count and total percent of numpy is 204 and 86.0759 %\n" | |
| ], | |
| "name": "stdout" | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "cCmjWFUQa1wy", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 34 | |
| }, | |
| "outputId": "d7d077fa-5561-43e1-e3a8-1afdde48bf8e" | |
| }, | |
| "source": [ | |
| "give_score(\"gym\")" | |
| ], | |
| "execution_count": 12, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "text": [ | |
| "Count and total percent of gym is 25 and 10.5485 %\n" | |
| ], | |
| "name": "stdout" | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "kF_iM1Mjcp9d", | |
| "colab_type": "code", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 34 | |
| }, | |
| "outputId": "ad490da1-5d45-4418-d515-5e35c62dbbaa" | |
| }, | |
| "source": [ | |
| "give_score(\"networkx\")" | |
| ], | |
| "execution_count": 13, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "text": [ | |
| "Count and total percent of networkx is 25 and 10.5485 %\n" | |
| ], | |
| "name": "stdout" | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "Qo8_PX43iwLu", | |
| "colab_type": "code", | |
| "colab": {} | |
| }, | |
| "source": [ | |
| "" | |
| ], | |
| "execution_count": 0, | |
| "outputs": [] | |
| } | |
| ] | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment