Crawling Melon and analyzing Korean song lyrics for a group assignment.

notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import dataset, re, requests\n",
    "\n",
    "from bs4 import BeautifulSoup\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "melon_headers = { 'User-Agent': 'Student' }\n",
    "\n",
    "def get_melon_lyrics(songid):\n",
    "    url = f'https://www.melon.com/webplayer/getLyrics.json?songId={songid}'\n",
    "    res = requests.get(url, headers=melon_headers)\n",
    "    \n",
    "    try:\n",
    "        return '\\n'.join([ caption['text'] for caption in res.json()['lylics'] ])\n",
    "    except err:\n",
    "        return None\n",
    "\n",
    "def get_top_songs(year=2018, offset=1, lyrics=True):\n",
    "    url = f'https://www.melon.com/chart/age/list.htm?idx={offset}&chartType=YE&chartGenre=KPOP&chartDate={year}&moved=Y'\n",
    "    res = requests.get(url, headers=melon_headers)\n",
    "    soup = BeautifulSoup(res.text, 'html.parser')\n",
    "\n",
    "    results = []\n",
    "    i = (offset - 1) * 50\n",
    "\n",
    "    for song in soup.find_all(class_='wrap_song_info'):\n",
    "        i += 1\n",
    "\n",
    "        if song.strong is None:\n",
    "            songid = f'unknown{i:02d}'\n",
    "            title = song.find(class_='ellipsis rank01').text.strip()\n",
    "        else:\n",
    "            songid = re.search(r\"(\\d+?)'\\)\", song.strong.a['href']).group(1)\n",
    "            title  = song.strong.a.text.strip()\n",
    "\n",
    "        artist = song.find(class_='ellipsis rank02').a.text.strip()\n",
    "        album  = song.find(class_='ellipsis rank03').a.text.strip()\n",
    "\n",
    "        results.append({\n",
    "            'id'    : f'{year}_{songid}',\n",
    "            'songid': songid,\n",
    "            'rank'  : i,\n",
    "            'year'  : year,\n",
    "            'title' : title,\n",
    "            'artist': artist,\n",
    "            'album' : album,\n",
    "            'lyrics': get_melon_lyrics(songid) if lyrics and song.strong else None\n",
    "        })\n",
    "    \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "db = dataset.connect('sqlite:///lyrics.db')\n",
    "table = db.get_table('lyrics', primary_id='id', primary_type=db.types.string(13))\n",
    "\n",
    "if len(table) == 0:\n",
    "    for year in range(2009, 2019):\n",
    "        for song in get_top_songs(year=year, lyrics=True):\n",
    "            table.insert(song)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[OrderedDict([('id', '2012_unknown74'),\n",
       "              ('songid', 'unknown74'),\n",
       "              ('rank', 74),\n",
       "              ('year', 2012),\n",
       "              ('title', 'I 돈 Care (Feat. 서수민PD)'),\n",
       "              ('artist', '용감한 녀석들'),\n",
       "              ('album', '용감한 녀석들'),\n",
       "              ('lyrics', None)]),\n",
       " OrderedDict([('id', '2009_unknown72'),\n",
       "              ('songid', 'unknown72'),\n",
       "              ('rank', 72),\n",
       "              ('year', 2009),\n",
       "              ('title', '눈물아 안녕'),\n",
       "              ('artist', '아이비'),\n",
       "              ('album', 'I Be..'),\n",
       "              ('lyrics', None)])]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_lyrics = list(table.find(lyrics=None))\n",
    "no_lyrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame(table.all())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id        object\n",
       "songid    object\n",
       "rank       int64\n",
       "year       int64\n",
       "title     object\n",
       "artist    object\n",
       "album     object\n",
       "lyrics    object\n",
       "dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rank</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>998.000000</td>\n",
       "      <td>998.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>50.454910</td>\n",
       "      <td>2013.506012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>28.891827</td>\n",
       "      <td>2.872668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>2009.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>25.250000</td>\n",
       "      <td>2011.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>50.000000</td>\n",
       "      <td>2014.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>75.750000</td>\n",
       "      <td>2016.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>2018.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             rank         year\n",
       "count  998.000000   998.000000\n",
       "mean    50.454910  2013.506012\n",
       "std     28.891827     2.872668\n",
       "min      1.000000  2009.000000\n",
       "25%     25.250000  2011.000000\n",
       "50%     50.000000  2014.000000\n",
       "75%     75.750000  2016.000000\n",
       "max    100.000000  2018.000000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "songs = data[data.lyrics.notnull()]\n",
    "songs.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def songs_that_contain(word):\n",
    "    return len(songs[songs.lyrics.str.contains(word)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "600"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "songs_that_contain('사랑')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "156"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "songs_that_contain('love')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "46"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "songs_that_contain('안녕')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "songs_that_contain('밥')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "120"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "songs_that_contain('그래 ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "songs_that_contain('그렉')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "songs_that_contain('정호')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "149"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "songs_that_contain('baby')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.lucypark.kr/courses/2015-ba/text-mining.html\n",
    "import nltk, konlpy.tag\n",
    "\n",
    "kkma = konlpy.tag.Kkma()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000 of 1000 songs processed.\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import clear_output\n",
    "\n",
    "songs = []\n",
    "songi = 0\n",
    "\n",
    "for song in table:\n",
    "    clear_output(wait=True)\n",
    "\n",
    "    if song['lyrics'] is not None:\n",
    "        songs.append(song)\n",
    "        song['pos'] = nltk.Text(kkma.pos(song['lyrics']))\n",
    "        \n",
    "    songi += 1\n",
    "    print(songi, 'of 1000 songs processed.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "songs = pd.DataFrame(songs)\n",
    "songs['love'] = [ any(token == '사랑' for token, _ in text.tokens) for text in songs['pos'] ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "csv = songs[['artist', 'title', 'year', 'rank', 'love']].to_csv(header=['Artist', 'Title', 'Year', 'Rank', 'Mentions love?'], index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

results.csv