Skip to content

Instantly share code, notes, and snippets.

@withLinda

withLinda/Fundamentals.ipynb

Created January 24, 2023 08:49
Show Gist options
  • Save withLinda/fcc83d5ee2153629f09fdf7ac99646f8 to your computer and use it in GitHub Desktop.
Save withLinda/fcc83d5ee2153629f09fdf7ac99646f8 to your computer and use it in GitHub Desktop.
Download ZIP
Scrapr Fundamentals
Display the source blob
Display the rendered blob
Raw
Fundamentals.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"pycharm": {
"name": "#%%# Getting response code from URL\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<Response [200]>\n"
]
}
],
"source": [
"import requests\n",
"my_response = requests.get(url=\"https://books.toscrape.com/catalogue/tipping-the-velvet_999/index.html\")\n",
"print(my_response)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
"<!DOCTYPE html>\n",
"<!--[if lt IE 7]> <html lang=\"en-us\" class=\"no-js lt-ie9 lt-ie8 lt-ie7\"> <![endif]-->\n",
"<!--[if IE 7]> <html lang=\"en-us\" class=\"no-js lt-ie9 lt-ie8\"> <![endif]-->\n",
"<!--[if IE 8]> <html lang=\"en-us\" class=\"no-js lt-ie9\"> <![endif]-->\n",
"<!--[if gt IE 8]><!--> <html lang=\"en-us\" class=\"no-js\"> <!--<![endif]-->\n",
" <head>\n",
" <title>\n",
" Tipping the Velvet | Books to Scrape - Sandbox\n",
"</title>\n",
"\n",
" <meta http-equiv=\"content-type\" content=\"text/html; charset=UTF-8\" />\n",
" <meta name=\"created\" content=\"24th Jun 2016 09:29\" />\n",
" <meta name=\"description\" content=\"\n",
" &quot;Erotic and absorbing...Written with starling power.&quot;--&quot;The New York Times Book Review &quot; Nan King, an oyster girl, is captivated by the music hall phenomenon Kitty Butler, a male impersonator extraordinaire treading the boards in Canterbury. Through a friend at the box office, Nan manages to visit all her shows and finally meet her heroine. Soon after, she becomes Kitty&#39;s &quot;Erotic and absorbing...Written with starling power.&quot;--&quot;The New York Times Book Review &quot; Nan King, an oyster girl, is captivated by the music hall phenomenon Kitty Butler, a male impersonator extraordinaire treading the boards in Canterbury. Through a friend at the box office, Nan manages to visit all her shows and finally meet her heroine. Soon after, she becomes Kitty&#39;s dresser and the two head for the bright lights of Leicester Square where they begin a glittering career as music-hall stars in an all-singing and dancing double act. At the same time, behind closed doors, they admit their attraction to each other and their affair begins. ...more\n",
"\" />\n",
" <meta name=\"viewport\" content=\"width=device-width\" />\n",
" <meta name=\"robots\" content=\"NOARCHIVE,NOCACHE\" />\n",
"\n",
" <!-- Le HTML5 shim, for IE6-8 support of HTML elements -->\n",
" <!--[if lt IE 9]>\n",
" <script src=\"//html5shim.googlecode.com/svn/trunk/html5.js\"></script>\n",
" <![endif]-->\n",
"\n",
" \n",
" <link rel=\"shortcut icon\" href=\"../../static/oscar/favicon.ico\" />\n",
" \n",
"\n",
" \n",
" \n",
" \n",
" \n",
" <link rel=\"stylesheet\" type=\"text/css\" href=\"../../static/oscar/css/styles.css\" />\n",
" \n",
" <link rel=\"stylesheet\" href=\"../../static/oscar/js/bootstrap-datetimepicker/bootstrap-datetimepicker.css\" />\n",
" <link rel=\"stylesheet\" type=\"text/css\" href=\"../../static/oscar/css/datetimepicker.css\" />\n",
"\n",
"\n",
" \n",
" \n",
"\n",
" \n",
"\n",
" \n",
" \n",
" \n",
"\n",
" \n",
" </head>\n",
"\n",
" <body id=\"default\" class=\"default\">\n",
" \n",
" \n",
" \n",
" \n",
" <header class=\"header container-fluid\">\n",
" <div class=\"page_inner\">\n",
" <div class=\"row\">\n",
" <div class=\"col-sm-8 h1\"><a href=\"../../index.html\">Books to Scrape</a><small> We love being scraped!</small>\n",
"</div>\n",
"\n",
" \n",
" </div>\n",
" </div>\n",
" </header>\n",
"\n",
" \n",
" \n",
" <div class=\"container-fluid page\">\n",
" <div class=\"page_inner\">\n",
" \n",
"<ul class=\"breadcrumb\">\n",
" <li>\n",
" <a href=\"../../index.html\">Home</a>\n",
" </li>\n",
" \n",
" \n",
" <li>\n",
" <a href=\"../category/books_1/index.html\">Books</a>\n",
" </li>\n",
" \n",
" <li>\n",
" <a href=\"../category/books/historical-fiction_4/index.html\">Historical Fiction</a>\n",
" </li>\n",
" \n",
" <li class=\"active\">Tipping the Velvet</li>\n",
"\n",
" \n",
" \n",
" \n",
"</ul>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
"\n",
"\n",
"<div id=\"messages\">\n",
"\n",
"</div>\n",
"\n",
" \n",
" <div class=\"content\">\n",
" \n",
"\n",
" \n",
" <div id=\"promotions\">\n",
" \n",
" </div>\n",
"\n",
" \n",
" <div id=\"content_inner\">\n",
"\n",
"<article class=\"product_page\"><!-- Start of product page -->\n",
"\n",
" <div class=\"row\">\n",
"\n",
" \n",
" <div class=\"col-sm-6\">\n",
" \n",
"\n",
"\n",
"\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" \n",
" <div id=\"product_gallery\" class=\"carousel\">\n",
" <div class=\"thumbnail\">\n",
" <div class=\"carousel-inner\">\n",
" <div class=\"item active\">\n",
" \n",
" \n",
" <img src=\"../../media/cache/08/e9/08e94f3731d7d6b760dfbfbc02ca5c62.jpg\" alt=\"Tipping the Velvet\" />\n",
" \n",
" \n",
" </div>\n",
" </div>\n",
" </div>\n",
" </div>\n",
"\n",
" \n",
"\n",
"\n",
" </div>\n",
" \n",
"\n",
" \n",
" <div class=\"col-sm-6 product_main\">\n",
" \n",
" \n",
" <h1>Tipping the Velvet</h1>\n",
"\n",
" \n",
" \n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" \n",
" <p class=\"price_color\">£53.74</p>\n",
" \n",
"\n",
"<p class=\"instock availability\">\n",
" <i class=\"icon-ok\"></i>\n",
" \n",
" In stock (20 available)\n",
" \n",
"</p>\n",
"\n",
" \n",
"\n",
" \n",
" \n",
"\n",
"\n",
"\n",
" <p class=\"star-rating One\">\n",
" <i class=\"icon-star\"></i>\n",
" <i class=\"icon-star\"></i>\n",
" <i class=\"icon-star\"></i>\n",
" <i class=\"icon-star\"></i>\n",
" <i class=\"icon-star\"></i>\n",
"\n",
" <!-- <small><a href=\"/catalogue/tipping-the-velvet_999/reviews/\">\n",
" \n",
" \n",
" 0 customer reviews\n",
" \n",
" </a></small>\n",
" -->&nbsp;\n",
"\n",
"\n",
"<!-- \n",
" <a id=\"write_review\" href=\"/catalogue/tipping-the-velvet_999/reviews/add/#addreview\" class=\"btn btn-success btn-sm\">\n",
" Write a review\n",
" </a>\n",
"\n",
" --></p>\n",
"\n",
" \n",
"\n",
" <hr/>\n",
"\n",
" <div class=\"alert alert-warning\" role=\"alert\"><strong>Warning!</strong> This is a demo website for web scraping purposes. Prices and ratings here were randomly assigned and have no real meaning.</div>\n",
"\n",
"\n",
" \n",
" \n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" \n",
" </div><!-- /col-sm-6 -->\n",
" \n",
"\n",
" </div><!-- /row -->\n",
"\n",
" \n",
" \n",
" <div id=\"product_description\" class=\"sub-header\">\n",
" <h2>Product Description</h2>\n",
" </div>\n",
" <p>\"Erotic and absorbing...Written with starling power.\"--\"The New York Times Book Review \" Nan King, an oyster girl, is captivated by the music hall phenomenon Kitty Butler, a male impersonator extraordinaire treading the boards in Canterbury. Through a friend at the box office, Nan manages to visit all her shows and finally meet her heroine. Soon after, she becomes Kitty's \"Erotic and absorbing...Written with starling power.\"--\"The New York Times Book Review \" Nan King, an oyster girl, is captivated by the music hall phenomenon Kitty Butler, a male impersonator extraordinaire treading the boards in Canterbury. Through a friend at the box office, Nan manages to visit all her shows and finally meet her heroine. Soon after, she becomes Kitty's dresser and the two head for the bright lights of Leicester Square where they begin a glittering career as music-hall stars in an all-singing and dancing double act. At the same time, behind closed doors, they admit their attraction to each other and their affair begins. ...more</p>\n",
" \n",
" \n",
"\n",
" \n",
" <div class=\"sub-header\">\n",
" <h2>Product Information</h2>\n",
" </div>\n",
" <table class=\"table table-striped\">\n",
" \n",
" <tr>\n",
" <th>UPC</th><td>90fa61229261140a</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th>Product Type</th><td>Books</td>\n",
" </tr>\n",
"\n",
" \n",
" \n",
" <tr>\n",
" <th>Price (excl. tax)</th><td>£53.74</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th>Price (incl. tax)</th><td>£53.74</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Tax</th><td>£0.00</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th>Availability</th>\n",
" <td>In stock (20 available)</td>\n",
" </tr>\n",
" \n",
" \n",
" \n",
" <tr>\n",
" <th>Number of reviews</th>\n",
" <td>0</td>\n",
" </tr>\n",
" \n",
" </table>\n",
" \n",
"\n",
" \n",
" \n",
" <section>\n",
" <div id=\"reviews\" class=\"sub-header\">\n",
" </div>\n",
" </section>\n",
" \n",
" \n",
"\n",
" \n",
" \n",
" \n",
"\n",
" \n",
"\n",
"\n",
"\n",
" \n",
" <div class=\"sub-header\">\n",
" <h2>Products you recently viewed</h2>\n",
" </div>\n",
"\n",
" <ul class=\"row\">\n",
" \n",
" <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" <article class=\"product_pod\">\n",
" \n",
" <div class=\"image_container\">\n",
" \n",
" \n",
" <a href=\"../a-light-in-the-attic_1000/index.html\"><img src=\"../../media/cache/2c/da/2cdad67c44b002e7ead0cc35693c0e8b.jpg\" alt=\"A Light in the Attic\" class=\"thumbnail\"></a>\n",
" \n",
" \n",
" </div>\n",
" \n",
"\n",
" \n",
" \n",
" <p class=\"star-rating Three\">\n",
" <i class=\"icon-star\"></i>\n",
" <i class=\"icon-star\"></i>\n",
" <i class=\"icon-star\"></i>\n",
" <i class=\"icon-star\"></i>\n",
" <i class=\"icon-star\"></i>\n",
" </p>\n",
" \n",
" \n",
"\n",
" \n",
" <h3><a href=\"../a-light-in-the-attic_1000/index.html\" title=\"A Light in the Attic\">A Light in the ...</a></h3>\n",
" \n",
"\n",
" \n",
" <div class=\"product_price\">\n",
" \n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" \n",
" <p class=\"price_color\">£51.77</p>\n",
" \n",
"\n",
"<p class=\"instock availability\">\n",
" <i class=\"icon-ok\"></i>\n",
" \n",
" In stock\n",
" \n",
"</p>\n",
"\n",
" \n",
" \n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" \n",
" <form>\n",
" <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n",
" </form>\n",
"\n",
"\n",
" \n",
" </div>\n",
" \n",
" </article>\n",
"\n",
"</li>\n",
" \n",
" </ul>\n",
" \n",
"\n",
"\n",
"\n",
"</article><!-- End of product page -->\n",
"</div>\n",
" </div>\n",
" </div>\n",
" </div>\n",
" \n",
"\n",
" \n",
"<footer class=\"footer container-fluid\">\n",
" \n",
" \n",
" \n",
"</footer>\n",
"\n",
"\n",
" \n",
" \n",
" \n",
" <!-- jQuery -->\n",
" <script src=\"http://ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js\"></script>\n",
" <script>window.jQuery || document.write('<script src=\"../../static/oscar/js/jquery/jquery-1.9.1.min.js\"><\\/script>')</script>\n",
" \n",
" \n",
"\n",
"\n",
" \n",
" \n",
" \n",
" \n",
" <!-- Twitter Bootstrap -->\n",
" <script type=\"text/javascript\" src=\"../../static/oscar/js/bootstrap3/bootstrap.min.js\"></script>\n",
" <!-- Oscar -->\n",
" <script src=\"../../static/oscar/js/oscar/ui.js\" type=\"text/javascript\" charset=\"utf-8\"></script>\n",
"\n",
" <script src=\"../../static/oscar/js/bootstrap-datetimepicker/bootstrap-datetimepicker.js\" type=\"text/javascript\" charset=\"utf-8\"></script>\n",
" <script src=\"../../static/oscar/js/bootstrap-datetimepicker/locales/bootstrap-datetimepicker.all.js\" type=\"text/javascript\" charset=\"utf-8\"></script>\n",
"\n",
"\n",
" \n",
" \n",
" \n",
" \n",
"\n",
" \n",
"\n",
"\n",
"\n",
" \n",
" <script type=\"text/javascript\">\n",
" $(function() {\n",
" \n",
" \n",
" oscar.init();\n",
"\n",
" });\n",
" </script>\n",
"\n",
" \n",
" <!-- Version: N/A -->\n",
" \n",
" </body>\n",
"</html>\n",
"\n"
]
}
],
"source": [
"import requests\n",
"my_response = requests.get(url=\"https://books.toscrape.com/catalogue/tipping-the-velvet_999/index.html\")\n",
"print(my_response.content)"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
"<!DOCTYPE html>\n",
"<!--[if lt IE 7]> <html lang=\"en-us\" class=\"no-js lt-ie9 lt-ie8 lt-ie7\"> <![endif]-->\n",
"<!--[if IE 7]> <html lang=\"en-us\" class=\"no-js lt-ie9 lt-ie8\"> <![endif]-->\n",
"<!--[if IE 8]> <html lang=\"en-us\" class=\"no-js lt-ie9\"> <![endif]-->\n",
"<!--[if gt IE 8]><!--> <html lang=\"en-us\" class=\"no-js\"> <!--<![endif]-->\n",
" <head>\n",
" <title>\n",
" Tipping the Velvet | Books to Scrape - Sandbox\n",
"</title>\n",
"\n",
" <meta http-equiv=\"content-type\" content=\"text/html; charset=UTF-8\" />\n",
" <meta name=\"created\" content=\"24th Jun 2016 09:29\" />\n",
" <meta name=\"description\" content=\"\n",
" &quot;Erotic and absorbing...Written with starling power.&quot;--&quot;The New York Times Book Review &quot; Nan King, an oyster girl, is captivated by the music hall phenomenon Kitty Butler, a male impersonator extraordinaire treading the boards in Canterbury. Through a friend at the box office, Nan manages to visit all her shows and finally meet her heroine. Soon after, she becomes Kitty&#39;s &quot;Erotic and absorbing...Written with starling power.&quot;--&quot;The New York Times Book Review &quot; Nan King, an oyster girl, is captivated by the music hall phenomenon Kitty Butler, a male impersonator extraordinaire treading the boards in Canterbury. Through a friend at the box office, Nan manages to visit all her shows and finally meet her heroine. Soon after, she becomes Kitty&#39;s dresser and the two head for the bright lights of Leicester Square where they begin a glittering career as music-hall stars in an all-singing and dancing double act. At the same time, behind closed doors, they admit their attraction to each other and their affair begins. ...more\n",
"\" />\n",
" <meta name=\"viewport\" content=\"width=device-width\" />\n",
" <meta name=\"robots\" content=\"NOARCHIVE,NOCACHE\" />\n",
"\n",
" <!-- Le HTML5 shim, for IE6-8 support of HTML elements -->\n",
" <!--[if lt IE 9]>\n",
" <script src=\"//html5shim.googlecode.com/svn/trunk/html5.js\"></script>\n",
" <![endif]-->\n",
"\n",
" \n",
" <link rel=\"shortcut icon\" href=\"../../static/oscar/favicon.ico\" />\n",
" \n",
"\n",
" \n",
" \n",
" \n",
" \n",
" <link rel=\"stylesheet\" type=\"text/css\" href=\"../../static/oscar/css/styles.css\" />\n",
" \n",
" <link rel=\"stylesheet\" href=\"../../static/oscar/js/bootstrap-datetimepicker/bootstrap-datetimepicker.css\" />\n",
" <link rel=\"stylesheet\" type=\"text/css\" href=\"../../static/oscar/css/datetimepicker.css\" />\n",
"\n",
"\n",
" \n",
" \n",
"\n",
" \n",
"\n",
" \n",
" \n",
" \n",
"\n",
" \n",
" </head>\n",
"\n",
" <body id=\"default\" class=\"default\">\n",
" \n",
" \n",
" \n",
" \n",
" <header class=\"header container-fluid\">\n",
" <div class=\"page_inner\">\n",
" <div class=\"row\">\n",
" <div class=\"col-sm-8 h1\"><a href=\"../../index.html\">Books to Scrape</a><small> We love being scraped!</small>\n",
"</div>\n",
"\n",
" \n",
" </div>\n",
" </div>\n",
" </header>\n",
"\n",
" \n",
" \n",
" <div class=\"container-fluid page\">\n",
" <div class=\"page_inner\">\n",
" \n",
"<ul class=\"breadcrumb\">\n",
" <li>\n",
" <a href=\"../../index.html\">Home</a>\n",
" </li>\n",
" \n",
" \n",
" <li>\n",
" <a href=\"../category/books_1/index.html\">Books</a>\n",
" </li>\n",
" \n",
" <li>\n",
" <a href=\"../category/books/historical-fiction_4/index.html\">Historical Fiction</a>\n",
" </li>\n",
" \n",
" <li class=\"active\">Tipping the Velvet</li>\n",
"\n",
" \n",
" \n",
" \n",
"</ul>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
"\n",
"\n",
"<div id=\"messages\">\n",
"\n",
"</div>\n",
"\n",
" \n",
" <div class=\"content\">\n",
" \n",
"\n",
" \n",
" <div id=\"promotions\">\n",
" \n",
" </div>\n",
"\n",
" \n",
" <div id=\"content_inner\">\n",
"\n",
"<article class=\"product_page\"><!-- Start of product page -->\n",
"\n",
" <div class=\"row\">\n",
"\n",
" \n",
" <div class=\"col-sm-6\">\n",
" \n",
"\n",
"\n",
"\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" \n",
" <div id=\"product_gallery\" class=\"carousel\">\n",
" <div class=\"thumbnail\">\n",
" <div class=\"carousel-inner\">\n",
" <div class=\"item active\">\n",
" \n",
" \n",
" <img src=\"../../media/cache/08/e9/08e94f3731d7d6b760dfbfbc02ca5c62.jpg\" alt=\"Tipping the Velvet\" />\n",
" \n",
" \n",
" </div>\n",
" </div>\n",
" </div>\n",
" </div>\n",
"\n",
" \n",
"\n",
"\n",
" </div>\n",
" \n",
"\n",
" \n",
" <div class=\"col-sm-6 product_main\">\n",
" \n",
" \n",
" <h1>Tipping the Velvet</h1>\n",
"\n",
" \n",
" \n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" \n",
" <p class=\"price_color\">£53.74</p>\n",
" \n",
"\n",
"<p class=\"instock availability\">\n",
" <i class=\"icon-ok\"></i>\n",
" \n",
" In stock (20 available)\n",
" \n",
"</p>\n",
"\n",
" \n",
"\n",
" \n",
" \n",
"\n",
"\n",
"\n",
" <p class=\"star-rating One\">\n",
" <i class=\"icon-star\"></i>\n",
" <i class=\"icon-star\"></i>\n",
" <i class=\"icon-star\"></i>\n",
" <i class=\"icon-star\"></i>\n",
" <i class=\"icon-star\"></i>\n",
"\n",
" <!-- <small><a href=\"/catalogue/tipping-the-velvet_999/reviews/\">\n",
" \n",
" \n",
" 0 customer reviews\n",
" \n",
" </a></small>\n",
" -->&nbsp;\n",
"\n",
"\n",
"<!-- \n",
" <a id=\"write_review\" href=\"/catalogue/tipping-the-velvet_999/reviews/add/#addreview\" class=\"btn btn-success btn-sm\">\n",
" Write a review\n",
" </a>\n",
"\n",
" --></p>\n",
"\n",
" \n",
"\n",
" <hr/>\n",
"\n",
" <div class=\"alert alert-warning\" role=\"alert\"><strong>Warning!</strong> This is a demo website for web scraping purposes. Prices and ratings here were randomly assigned and have no real meaning.</div>\n",
"\n",
"\n",
" \n",
" \n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" \n",
" </div><!-- /col-sm-6 -->\n",
" \n",
"\n",
" </div><!-- /row -->\n",
"\n",
" \n",
" \n",
" <div id=\"product_description\" class=\"sub-header\">\n",
" <h2>Product Description</h2>\n",
" </div>\n",
" <p>\"Erotic and absorbing...Written with starling power.\"--\"The New York Times Book Review \" Nan King, an oyster girl, is captivated by the music hall phenomenon Kitty Butler, a male impersonator extraordinaire treading the boards in Canterbury. Through a friend at the box office, Nan manages to visit all her shows and finally meet her heroine. Soon after, she becomes Kitty's \"Erotic and absorbing...Written with starling power.\"--\"The New York Times Book Review \" Nan King, an oyster girl, is captivated by the music hall phenomenon Kitty Butler, a male impersonator extraordinaire treading the boards in Canterbury. Through a friend at the box office, Nan manages to visit all her shows and finally meet her heroine. Soon after, she becomes Kitty's dresser and the two head for the bright lights of Leicester Square where they begin a glittering career as music-hall stars in an all-singing and dancing double act. At the same time, behind closed doors, they admit their attraction to each other and their affair begins. ...more</p>\n",
" \n",
" \n",
"\n",
" \n",
" <div class=\"sub-header\">\n",
" <h2>Product Information</h2>\n",
" </div>\n",
" <table class=\"table table-striped\">\n",
" \n",
" <tr>\n",
" <th>UPC</th><td>90fa61229261140a</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th>Product Type</th><td>Books</td>\n",
" </tr>\n",
"\n",
" \n",
" \n",
" <tr>\n",
" <th>Price (excl. tax)</th><td>£53.74</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th>Price (incl. tax)</th><td>£53.74</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Tax</th><td>£0.00</td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th>Availability</th>\n",
" <td>In stock (20 available)</td>\n",
" </tr>\n",
" \n",
" \n",
" \n",
" <tr>\n",
" <th>Number of reviews</th>\n",
" <td>0</td>\n",
" </tr>\n",
" \n",
" </table>\n",
" \n",
"\n",
" \n",
" \n",
" <section>\n",
" <div id=\"reviews\" class=\"sub-header\">\n",
" </div>\n",
" </section>\n",
" \n",
" \n",
"\n",
" \n",
" \n",
" \n",
"\n",
" \n",
"\n",
"\n",
"\n",
" \n",
" <div class=\"sub-header\">\n",
" <h2>Products you recently viewed</h2>\n",
" </div>\n",
"\n",
" <ul class=\"row\">\n",
" \n",
" <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" <article class=\"product_pod\">\n",
" \n",
" <div class=\"image_container\">\n",
" \n",
" \n",
" <a href=\"../a-light-in-the-attic_1000/index.html\"><img src=\"../../media/cache/2c/da/2cdad67c44b002e7ead0cc35693c0e8b.jpg\" alt=\"A Light in the Attic\" class=\"thumbnail\"></a>\n",
" \n",
" \n",
" </div>\n",
" \n",
"\n",
" \n",
" \n",
" <p class=\"star-rating Three\">\n",
" <i class=\"icon-star\"></i>\n",
" <i class=\"icon-star\"></i>\n",
" <i class=\"icon-star\"></i>\n",
" <i class=\"icon-star\"></i>\n",
" <i class=\"icon-star\"></i>\n",
" </p>\n",
" \n",
" \n",
"\n",
" \n",
" <h3><a href=\"../a-light-in-the-attic_1000/index.html\" title=\"A Light in the Attic\">A Light in the ...</a></h3>\n",
" \n",
"\n",
" \n",
" <div class=\"product_price\">\n",
" \n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" \n",
" <p class=\"price_color\">£51.77</p>\n",
" \n",
"\n",
"<p class=\"instock availability\">\n",
" <i class=\"icon-ok\"></i>\n",
" \n",
" In stock\n",
" \n",
"</p>\n",
"\n",
" \n",
" \n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" \n",
" <form>\n",
" <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n",
" </form>\n",
"\n",
"\n",
" \n",
" </div>\n",
" \n",
" </article>\n",
"\n",
"</li>\n",
" \n",
" </ul>\n",
" \n",
"\n",
"\n",
"\n",
"</article><!-- End of product page -->\n",
"</div>\n",
" </div>\n",
" </div>\n",
" </div>\n",
" \n",
"\n",
" \n",
"<footer class=\"footer container-fluid\">\n",
" \n",
" \n",
" \n",
"</footer>\n",
"\n",
"\n",
" \n",
" \n",
" \n",
" <!-- jQuery -->\n",
" <script src=\"http://ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js\"></script>\n",
" <script>window.jQuery || document.write('<script src=\"../../static/oscar/js/jquery/jquery-1.9.1.min.js\"><\\/script>')</script>\n",
" \n",
" \n",
"\n",
"\n",
" \n",
" \n",
" \n",
" \n",
" <!-- Twitter Bootstrap -->\n",
" <script type=\"text/javascript\" src=\"../../static/oscar/js/bootstrap3/bootstrap.min.js\"></script>\n",
" <!-- Oscar -->\n",
" <script src=\"../../static/oscar/js/oscar/ui.js\" type=\"text/javascript\" charset=\"utf-8\"></script>\n",
"\n",
" <script src=\"../../static/oscar/js/bootstrap-datetimepicker/bootstrap-datetimepicker.js\" type=\"text/javascript\" charset=\"utf-8\"></script>\n",
" <script src=\"../../static/oscar/js/bootstrap-datetimepicker/locales/bootstrap-datetimepicker.all.js\" type=\"text/javascript\" charset=\"utf-8\"></script>\n",
"\n",
"\n",
" \n",
" \n",
" \n",
" \n",
"\n",
" \n",
"\n",
"\n",
"\n",
" \n",
" <script type=\"text/javascript\">\n",
" $(function() {\n",
" \n",
" \n",
" oscar.init();\n",
"\n",
" });\n",
" </script>\n",
"\n",
" \n",
" <!-- Version: N/A -->\n",
" \n",
" </body>\n",
"</html>\n",
"\n"
]
}
],
"source": [
"import requests\n",
"my_response = requests.get(url=\"https://books.toscrape.com/catalogue/tipping-the-velvet_999/index.html\")\n",
"print(my_response.text)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'Date': 'Fri, 09 Dec 2022 04:02:48 GMT', 'Content-Type': 'text/html', 'Content-Length': '10918', 'Connection': 'keep-alive', 'Last-Modified': 'Thu, 26 May 2022 21:15:15 GMT', 'ETag': '\"628fede3-2aa6\"', 'Accept-Ranges': 'bytes', 'Strict-Transport-Security': 'max-age=0; includeSubDomains; preload'}\n"
]
}
],
"source": [
"import requests\n",
"my_response = requests.get(url=\"https://books.toscrape.com/catalogue/tipping-the-velvet_999/index.html\")\n",
"print(my_response.headers)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'User-Agent': 'python-requests/2.28.1', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}\n"
]
}
],
"source": [
"import requests\n",
"my_response = requests.get(url=\"https://books.toscrape.com/catalogue/tipping-the-velvet_999/index.html\")\n",
"print(my_response.request.headers)"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['<p>Hello World!</p>', '<p>Enjoy DataCamp!</p>']\n",
"['Hello World!', 'Enjoy DataCamp!']\n"
]
}
],
"source": [
"from scrapy import Selector\n",
"html = '''\n",
"<html>\n",
"<body>\n",
"<div class=\"hello datacamp\">\n",
"<p>Hello World!</p>\n",
"</div>\n",
"<p>Enjoy DataCamp!</p>\n",
"</body>\n",
"</html>\n",
"'''\n",
"sel = Selector(text=html)\n",
"text1 = sel.xpath(\"//p\").extract()\n",
"text2 = sel.xpath(\"//p/text()\").extract()\n",
"print(text1)\n",
"print(text2)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Day 1 - Beginner - Working with Variables in Python to Manage Data\n",
"Skills Assessment\n",
"Day 1 Goals: what we will make by the end of the day\n",
"FAQ: Can I Use PyCharm/VSCode/ Another Local Code Editor?\n",
"Printing to the Console in Python\n",
"Join Our Class on Coding Rooms\n",
"[Interactive Coding Exercise] Printing\n",
"String Manipulation and Code Intelligence\n",
"[Interactive Coding Exercise] Debugging Practice\n",
"The Python Input Function\n",
"[Interactive Coding Exercise] Input Function\n",
"Python Variables\n",
"[Interactive Coding Exercise] Variables\n",
"Variable Naming\n",
"Variable Naming Quiz\n",
"Day 1 Project: Band Name Generator\n",
"Congratulations! Well done!\n",
"Day 2 - Beginner - Understanding Data Types and How to Manipulate Strings\n",
"Day 2 Goals: what we will make by the end of the day\n",
"Python Primitive Data Types\n",
"Data Types Quiz\n",
"Type Error, Type Checking and Type Conversion\n",
"[Interactive Coding Exercise] Data Types\n",
"Mathematical Operations in Python\n",
"[Interactive Coding Exercise] BMI Calculator\n",
"Number Manipulation and F Strings in Python\n",
"[Interactive Coding Exercise] Life in Weeks\n",
"Mathematical Operations Quiz\n",
"Day 2 Project: Tip Calculator\n",
"You are already in the top 50%\n",
"Day 3 - Beginner - Control Flow and Logical Operators\n",
"Day 3 Goals: what we will make by the end of the day\n",
"Control Flow with if / else and Conditional Operators\n",
"[Interactive Coding Exercise] Odd or Even? Introducing the Modulo\n",
"Nested if statements and elif statements\n",
"[Interactive Coding Exercise] BMI 2.0\n",
"[Interactive Coding Exercise] Leap Year\n",
"Multiple If Statements in Succession\n",
"[Interactive Coding Exercise] Pizza Order Practice\n",
"Logical Operators\n",
"[Interactive Coding Exercise] Love Calculator\n",
"Day 3 Project: Treasure Island\n",
"Share and Show off your Project!\n",
"Day 4 - Beginner - Randomisation and Python Lists\n",
"Day 4 Goals: what we will make by the end of the day\n",
"Random Module\n",
"[Interactive Coding Exercise] Heads or Tails\n",
"Understanding the Offset and Appending Items to Lists\n",
"[Interactive Coding Exercise] Banker Roulette - Who will pay the bill?\n",
"IndexErrors and Working with Nested Lists\n",
"List and IndexError Quiz\n",
"[Interactive Coding Exercise] Treasure Map\n",
"Day 4 Project: Rock Paper Scissors\n",
"Programming is like going to the Gym\n",
"Day 5 - Beginner - Python Loops\n",
"Day 5 Goals: what we will make by the end of the day\n",
"Using the for loop with Python Lists\n",
"[Interactive Coding Exercise] Average Height\n",
"[Interactive Coding Exercise] High Score\n",
"for loops and the range() function\n",
"[Interactive Coding Exercise] Adding Even Numbers\n",
"[Interactive Coding Exercise] The FizzBuzz Job Interview Question\n",
"Day 5 Project: Create a Password Generator\n",
"Hard Work and Perseverance beats Raw Talent Every Time\n",
"Day 6 - Beginner - Python Functions Karel\n",
"Day 6 Goals: what we will make by the end of the day\n",
"Defining and Calling Python Functions\n",
"The Hurdles Loop Challenge\n",
"Indentation in Python\n",
"Code Indentation Quiz\n",
"While Loops\n",
"Hurdles Challenge using While Loops\n",
"Jumping over Hurdles with Variable Heights\n",
"Final Project: Escaping the Maze\n",
"Why is this *so* Hard?! Can I really do this?\n",
"Day 7 - Beginner - Hangman\n",
"Day 7 Goals: what we will make by the end of the day\n",
"How to break a Complex Problem down into a Flow Chart\n",
"Challenge 1 - Picking a Random Words and Checking Answers\n",
"Challenge 1 Solution - How to Check the User's Answer\n",
"Challenge 2 - Replacing Blanks with Guesses\n",
"Challenge 2 Solution - How to Replace the Blanks\n",
"Challenge 3 - Checking if the Player has Won\n",
"Challenge 3 Solution - How to Check if the Player Won\n",
"Challenge 4 - Keeping Track of the Player's Lives\n",
"Challenge 4 Solution - How to Keep Track of the Player's Lives\n",
"Challenge 5 - Improving the User Experience\n",
"Challenge 5 Solution - How to Add ASCII Art and Improve the UI\n",
"The Benefits of Daily Practice\n",
"Day 8 - Beginner - Function Parameters Caesar Cipher\n",
"Day 8 Goals: what we will make by the end of the day\n",
"Functions with Inputs\n",
"[Interactive Coding Exercise] Paint Area Calculator\n",
"[Interactive Coding Exercise] Prime Number Checker\n",
"Caesar Cipher Part 1 - Encryption\n",
"Caesar Cipher Part 2 - Decryption\n",
"Caesar Cipher Part 3 - Reorganising our Code\n",
"Caesar Cipher Part 4 - User Experience Improvements Final Touches\n",
"How You Can *Stay* Motivated\n",
"Day 9 - Beginner - Dictionaries, Nesting and the Secret Auction\n",
"Day 9 Goals: what we will make by the end of the day\n",
"The Python Dictionary: Deep Dive\n",
"[Interactive Coding Exercise] Grading Program\n",
"Nesting Lists and Dictionaries\n",
"[Interactive Coding Exercise] Dictionary in List\n",
"Python Dictionaries Quiz\n",
"The Secret Auction Program Instructions and Flow Chart\n",
"Solution and Complete Code for the Secret Auction Program\n",
"Motivation and the Accountability Trick\n",
"Day 10 - Beginner - Functions with Outputs\n",
"Day 10 Goals: what we will make by the end of the day\n",
"Multiple return values\n",
"[Interactive Coding Exercise] Days in Month\n",
"Docstrings\n",
"Functions Quiz\n",
"Calculator Part 1: Combining Dictionaries and Functions\n",
"Print vs. Return\n",
"While Loops, Flags and Recursion\n",
"Calculator Finishing Touches and Bug Fixes\n",
"How to Get a Good Night's Sleep\n",
"Day 11 - Beginner - The Blackjack Capstone Project\n",
"Day 11 Goals: what we will make by the end of the day\n",
"Blackjack Program Requirements and Game Rules\n",
"Hint 4 5 Solution Walkthrough\n",
"Hint 6-8 Solution Walkthrough\n",
"Hint 9 Solution Walkthrough: Refactoring and calling calculate_score()\n",
"Hint 10-12 Solution Walkthrough\n",
"Hint 13 Solution Walkthrough\n",
"A Solid Foundation goes a Long Way\n",
"Day 12 - Beginner - Scope Number Guessing Game\n",
"Does Python Have Block Scope?\n",
"How to Modify a Global Variable\n",
"Python Constants and Global Scope\n",
"Scope Quiz\n",
"Introducing the Final Project: The Number Guessing Game\n",
"Solution Walkthrough to the Number Guessing Game\n",
"Don't be too hard on yourself\n",
"Day 13 - Beginner - Debugging: How to Find and Fix Errors in your Code\n",
"Describe the Problem\n",
"Reproduce the Bug\n",
"Play Computer and Evaluate Each Line\n",
"Fixing Errors and Watching for Red Underlines\n",
"Squash bugs with a print() Statement\n",
"Bringing out the BIG Gun: Using a Debugger\n",
"Final Debugging Tips\n",
"[Interactive Coding Exercise] Debugging Odd or Even\n",
"[Interactive Coding Exercise] Debugging Leap Year\n",
"[Interactive Coding Exercise] Debugging FizzBuzz\n",
"Building Confidence\n",
"Day 14 - Beginner - Higher Lower Game Project\n",
"Introduction Program Requirements for the Higher Lower Game\n",
"Solution Walkthrough of the Higher Lower Game\n",
"Study Tip: Set Reminders in Your Calendar to Review\n",
"Day 15 - Intermediate - Local Development Environment Setup the Coffee Machine\n",
"Installing Python Locally on Your Computer\n",
"Download PyCharm for Windows or Mac\n",
"PyCharm's Charming Features (while you wait for the download to finish)\n",
"How to Install PyCharm on Windows\n",
"Installing PyCharm on Mac\n",
"Introduction Requirements for the Coffee Machine Project\n",
"Solution Walkthrough for the Coffee Machine Code\n",
"Location, Location, Location - Pavlov's Coding Corner\n",
"Day 16 - Intermediate - Object Oriented Programming (OOP)\n",
"Why do we need OOP and how does it work?\n",
"How to use OOP: Classes and Objects\n",
"Constructing Objects and Accessing their Attributes and Methods\n",
"How to Add Python Packages and use PyPi\n",
"Practice Modifying Object Attributes and Calling Methods\n",
"Python Objects Quiz\n",
"Building the Coffee Machine in OOP\n",
"Walkthrough and Solution for the OOP Coffee Machine\n",
"Don't forget to review occasionally\n",
"Day 17 - Intermediate - The Quiz Project the Benefits of OOP\n",
"Day 17 Goals: what we will make by the end of the day\n",
"How to create your own Class in Python\n",
"Working with Attributes, Class Constructors and the __init__() Function\n",
"Adding Methods to a Class\n",
"Quiz Project Part 1: Creating the Question Class\n",
"Quiz Project Part 2: Creating the List of Question Objects from the Data\n",
"Quiz Project Part 3: The QuizBrain and the next_question() Method\n",
"Quiz Project Part 4: How to continue showing new Questions\n",
"Quiz Project Part 5: Checking Answers and Keeping Score\n",
"The Benefits of OOP: Use Open Trivia DB to Get New Questions\n",
"Run for that Bus!\n",
"Day 18 - Intermediate - Turtle the Graphical User Interface (GUI)\n",
"Day 18 Goals: what we will make by the end of the day\n",
"Understanding Turtle Graphics and How to use the Documentation\n",
"Turtle Challenge 1 - Draw a Square\n",
"Importing Modules, Installing Packages, and Working with Aliases\n",
"Turtle Challenge 2 - Draw a Dashed Line\n",
"Turtle Challenge 3 - Drawing Different Shapes\n",
"Turtle Challenge 4 - Generate a Random Walk\n",
"Python Tuples and How to Generate Random RGB Colours\n",
"Turtle Challenge 5 - Draw a Spirograph\n",
"The Hirst Painting Project Part 1 - How to Extract RGB Values from Images\n",
"The Hirst Painting Project Part 2 - Drawing the Dots\n",
"Space out your study sessions and stay consistent\n",
"Day 19 - Intermediate - Instances, State and Higher Order Functions\n",
"Day 19 Goals: what we will make by the end of the day\n",
"Python Higher Order Functions Event Listeners\n",
"Challenge: Make an Etch-A-Sketch App\n",
"Object State and Instances\n",
"Understanding the Turtle Coordinate System\n",
"Turtle Coordinate System Quiz\n",
"Aaaand, we're off to the races!\n",
"Expand on the Solutions\n",
"Day 20 - Intermediate - Build the Snake Game Part 1: Animation Coordinates\n",
"Day 20 Goals: what we will make by the end of the day\n",
"Screen Setup and Creating a Snake Body\n",
"Animating the Snake Segments on Screen\n",
"Create a Snake Class Move to OOP\n",
"How to Control the Snake with a Keypress\n",
"Programming is not Memorising\n",
"Day 21 - Intermediate - Build the Snake Game Part 2: Inheritance List Slicing\n",
"Class Inheritance\n",
"Inheritance Quiz\n",
"Detect Collisions with Food\n",
"Create a Scoreboard and Keep Score\n",
"Detect Collisions with the Wall\n",
"Detect Collisions with your own Tail\n",
"How to Slice Lists Tuples in Python\n",
"Stay motivated by remembering the reason you signed up\n",
"Day 22 - Intermediate - Build Pong: The Famous Arcade Game\n",
"Day 22 Goals: what you will make by the end of the day\n",
"Set up the Main Screen\n",
"Create a Paddle that responds to Key Presses\n",
"Write the Paddle Class and Create the Second Paddle\n",
"Write the Ball Class and Make the Ball Move\n",
"Add the Ball Bouncing Logic\n",
"How to Detect Collisions with the Paddle\n",
"How to Detect when the Ball goes Out of Bounds\n",
"Score Keeping and Changing the Ball Speed\n",
"Picturing fears: even the worst-case scenario is not so scary\n",
"Day 23 - Intermediate - The Turtle Crossing Capstone Project\n",
"Day 23 Goals: what you will make by the end of the day\n",
"Choose Your Difficulty\n",
"How to use the Starter Code\n",
"Step 1 - Check out how the game play works\n",
"Step 2 - Break down the Problem\n",
"Solution to Step 3 - Create the Player Behaviour\n",
"Solution to Step 4 - Create the Car Behaviour\n",
"Solution to Step 5 - Detect when the Turtle collides with a Car *squish*\n",
"Solution to Step 6 - Detect when the Player has reached the other side\n",
"Solution to Step 7 - Add the Scoreboard and Game Over sequence\n",
"This course is not about typing out code\n",
"Day 24 - Intermediate - Files, Directories and Paths\n",
"Day 24 Goals: what you will make by the end of the day\n",
"Add a High Score to the Snake Game\n",
"How to Open, Read, and Write to Files using the \"with\" Keyword\n",
"Challenge: Read and Write the High Score to a File in Snake\n",
"Understand Relative and Absolute File Paths\n",
"File Paths Quiz\n",
"Introducing the Mail Merge Challenge\n",
"Solution Walkthrough for the Mail Merge Project\n",
"What's the correct solution? What's the best answer? What's the right way?\n",
"Day 25 - Intermediate - Working with CSV Data and the Pandas Library\n",
"Day 25 Goals: what we will make by the end of the day\n",
"Reading CSV Data in Python\n",
"DataFrames Series: Working with Rows Columns\n",
"The Great Squirrel Census Data Analysis (with Pandas!)\n",
"U.S. States Game Part 1: Setup\n",
"U.S. States Game Part 2: Challenge with .csv\n",
"U.S. States Game Part 3: Saving Data to .csv\n",
"Day 26 - Intermediate - List Comprehension and the NATO Alphabet\n",
"Day 26 Goals: what you will make by the end of the day\n",
"How to Create Lists using List Comprehension\n",
"[Interactive Coding Exercise] Squaring Numbers\n",
"[Interactive Coding Exercise] Filtering Even Numbers\n",
"[Interactive Coding Exercise] Data Overlap\n",
"Apply List Comprehension to the U.S. States Game\n",
"How to use Dictionary Comprehension\n",
"[Interactive Coding Exercise] Dictionary Comprehension 1\n",
"[Interactive Coding Exercise] Dictionary Comprehension 2\n",
"How to Iterate over a Pandas DataFrame\n",
"Introducing the NATO Alphabet Project\n",
"Solution Walkthrough for the NATO Alphabet Project\n",
"Day 27 - Intermediate - Tkinter, *args, **kwargs and Creating GUI Programs\n",
"Day 27 Goals: what we will make by the end of the day\n",
"History of GUI and Introduction to Tkinter\n",
"Creating Windows and Labels with Tkinter\n",
"Setting Default Values for Optional Arguments inside a Function Header\n",
"Default Values Quiz\n",
"*args: Many Positional Arguments\n",
"**kwargs: Many Keyword Arguments\n",
"Optional Arguments, *args and **kwargs Quiz\n",
"Buttons, Entry, and Setting Component Options\n",
"Other Tkinter Widgets: Radiobuttons, Scales, Checkbuttons and more\n",
"Tkinter Layout Managers: pack(), place() and grid()\n",
"Mile to Kilometers Converter Project\n",
"Day 28 - Intermediate - Tkinter, Dynamic Typing and the Pomodoro GUI Application\n",
"Day 28 Goals: what we will make by the end of the day\n",
"How to work with the Canvas Widget and Add Images to Tkinter\n",
"Challenge - Complete the Application's User Interface (UI)\n",
"Add a Count Down Mechanism\n",
"Dynamic Typing Explained\n",
"Setting Different Timer Sessions and Values\n",
"Adding Checkmarks and Resetting the Application\n",
"Day 29 - Intermediate - Building a Password Manager GUI App with Tkinter\n",
"Day 29 Goals: what we will make by the end of the day\n",
"Challenge 1 - Working with Images and Setting up the Canvas\n",
"Challenge 2 - Use grid() and columnspan to Complete the User Interface\n",
"Solution to the Creating the Grid Layout\n",
"Challenge 3 - Saving Data to File\n",
"Dialog Boxes and Pop-Ups in Tkinter\n",
"Generate a Password Copy it to the Clipboard\n",
"Day 30 - Intermediate - Errors, Exceptions and JSON Data: Improving the Password\n",
"Day 30 Goals: what you will make by the end of the day\n",
"Catching Exceptions: The try catch except finally Pattern\n",
"Raising your own Exceptions\n",
"[Interactive Coding Exercise] IndexError Handling\n",
"[Interactive Coding Exercise] KeyError Handling\n",
"Code Exercise: Exception Handling in the NATO Phonetic Alphabet Project\n",
"Write, read and update JSON data in the Password Manager\n",
"Challenge 1 - Handling Exceptions in the Password Manager\n",
"Challenge 2 - Search for a Website in the Password Manager\n",
"Day 31 - Intermediate - Flash Card App Capstone Project\n",
"Day 31 Goals: what you will make by the end of the day\n",
"Step 1 - Create the User Interface (UI) with Tkinter\n",
"Solution Walkthrough for Creating the UI\n",
"Step 2 - Create New Flash Cards\n",
"Solution Walkthrough for Creating New Flash Cards\n",
"Step 3 - Flip the Cards!\n",
"Solution Walkthrough for Flipping Cards\n",
"Step 4 - Save Your Progress\n",
"Solution Walkthrough for Saving Progress\n",
"Day 32 - Intermediate+ Send Email (smtplib) Manage Dates (datetime)\n",
"Day 32 Goals: what we will make by the end of the day\n",
"A Note About the Next Lesson: Google SMTP Port\n",
"How to Send Emails with Python using SMTP\n",
"Working with the datetime Module\n",
"Challenge 1 - Send Motivational Quotes on Mondays via Email\n",
"Automated Birthday Wisher Project Challenge\n",
"Solution Walkthrough for the Automated Birthday Wisher\n",
"Run Your Python Code in the Cloud!\n",
"Day 33 - Intermediate+ API Endpoints API Parameters - ISS Overhead Notifier\n",
"Day 33 Goals: what you will make by the end of the day\n",
"What are Application Programming Interfaces (APIs)?\n",
"API Endpoints and Making API Calls\n",
"Working with Responses: HTTP Codes, Exceptions JSON Data\n",
"Challenge - Build a Kanye Quotes App using the Kanye Rest API\n",
"Understand API Parameters: Match Sunset Times with the Current Time\n",
"ISS Overhead Notifier Project - Challenge Solution\n",
"Day 34 - Intermediate+ API Practice - Creating a GUI Quiz App\n",
"Day 34 Goals: what you will make by the end of the day\n",
"Trivia Question API Challenge\n",
"Solution Walkthrough for getting Trivia Questions\n",
"Unescaping HTML Entities\n",
"Class based Tkinter UI\n",
"Python Typing Showing the Next Question in the GUI\n",
"Python Typing: Type Hints and Arrows -\n",
"Check the Answer\n",
"Give Feedback to the Player, Keep Score and Fix the Bugs =)\n",
"Day 35 - Intermediate+ Keys, Authentication Environment Variables: Send SMS\n",
"What is API Authentication and Why Do We Need to Authenticate Ourselves?\n",
"Using API Keys to Authenticate and Get the Weather from OpenWeatherMap\n",
"Challenge - Check if it Will Rain in the Next 12 Hours\n",
"Sending SMS via the Twilio API\n",
"Use PythonAnywhere to Automate the Python Script\n",
"Understanding Environment Variables and Hiding API Keys\n",
"Day 36 - Intermediate+ Stock Trading News Alert Project\n",
"Day 36 Goals: what you will make by the end of the day\n",
"Choose Your Destiny!\n",
"Solution Walkthrough for Step 1 - Check for Stock Price Movements\n",
"Solution Walkthrough for Step 2 - Get the News Articles\n",
"Solution Walkthrough for Step 3 - Send the SMS Messages\n",
"Day 37 - Intermediate+ Habit Tracking Project: API Post Requests Headers\n",
"Day 37 Goals: what you will make by the end of the day\n",
"HTTP Post Requests\n",
"Advanced Authentication using an HTTP Header\n",
"Challenge: Add a Pixel to the Habit Tracker using a Post Request\n",
"Autofilling today's date using strftime\n",
"How to use HTTP Put and Delete Requests\n",
"Day 38 - Intermediate+ Workout Tracking Using Google Sheets\n",
"Day 38 Goals: what you will make by the end of the day\n",
"Step 1 - Setup API Credentials and Google Spreadsheet\n",
"Step 2 - Get Exercise Stats with Natural Language Queries\n",
"Step 3 - Setup Your Google Sheet with Sheety\n",
"Step 4 - Saving Data into Google Sheets\n",
"Step 5 - Authenticate Your Sheety API\n",
"Step 6 - Environment Variables in Repl.it\n",
"Day 39 - Intermediate+ Capstone Part 1: Flight Deal Finder\n",
"Day 39 Goals: what you will make by the end of the day\n",
"Step 1 - Choose Your Path and Download the Starting Project\n",
"Step 2 - Use Sheety to Read and Write Data to the Google Sheet\n",
"Step 3 - Get the IATA Codes using the Kiwi Partners API\n",
"Step 4 - Search for Cheap Flights\n",
"Step 5 - If Flight Price Lower than in Google Sheet send an SMS\n",
"Day 40 - Intermediate+ Capstone Part 2: Flight Club\n",
"Day 40 Goals: what you will make by the end of the day\n",
"Step 1 - Create the Customer Acquisition Code\n",
"Step 2 - Download the Starting Project\n",
"Step 3 - Exception Handling for Destinations without Flights\n",
"Step 4 - Destinations without Direct Flights\n",
"Step 5 - Email all our customers\n",
"Day 41 - Web Foundation - Introduction to HTML\n",
"Day 41 Goals: what you will make by the end of the day\n",
"How Does the Internet Actually Work?\n",
"How Do Websites Actually Work?\n",
"Optional: Install the Atom Text Editor used in the Video Lessons\n",
"Introduction to HTML\n",
"The Anatomy of an HTML Tag\n",
"What we're building - HTML Personal Site\n",
"What is The HTML Boilerplate?\n",
"How to Structure Text in HTML\n",
"HTML Lists\n",
"HTML Image Elements\n",
"HTML Links and Anchor Tags\n",
"Day 42 - Web Foundation - Intermediate HTML\n",
"Day 42 Goals: what you will make by the end of the day\n",
"HTML Tables\n",
"Using HTML Tables for Layout\n",
"HTML Tables Code Challenge\n",
"How to Type Emojis\n",
"HTML Tables Solution Walkthrough\n",
"HTML Forms\n",
"Forms in Practice - Create a Contact Me Form\n",
"HTML Challenge\n",
"Publish Your Website!\n",
"Day 43 - Web Foundation - Introduction to CSS\n",
"Day 43 Goals: what you will make by the end of the day\n",
"Introduction to CSS\n",
"Inline CSS\n",
"Internal CSS\n",
"External CSS\n",
"How to Debug CSS Code\n",
"The Anatomy of CSS Syntax\n",
"CSS Selectors\n",
"Classes vs. Ids\n",
"CSS Quiz\n",
"Day 44 - Web Foundation - Intermediate CSS\n",
"Day 44 Goals: what you will make by the end of the day\n",
"What We'll Make - Stylised Personal Site\n",
"What Are Favicons?\n",
"HTML Divs\n",
"The Box Model of Website Styling\n",
"CSS Display Property\n",
"CSS Static and Relative Positioning\n",
"Absolute positioning\n",
"The Dark Art of Centering Elements with CSS\n",
"Font Styling in Our Personal Site\n",
"Adding Content to Our Website\n",
"CSS Sizing\n",
"Font Properties Challenge 1 - Change the Font Colour\n",
"Font Properties Challenge 2 - Change the Font Weight\n",
"Font Properties Challenge 3 - Change the Line Height\n",
"CSS Font Property Challenge Solutions\n",
"CSS Float and Clear\n",
"CSS Challenge\n",
"Stylised Personal Site Solution Walkthrough\n",
"[Optional] Get More Practice HTML and CSS\n",
"Day 45 - Intermediate+ Web Scraping with Beautiful Soup\n",
"Day 45 Goals: what you will make by the end of the day\n",
"Parsing HTML and Making Soup\n",
"Finding and Selecting Particular Elements with BeautifulSoup\n",
"Beautiful Soup Exercises\n",
"Scraping a Live Website\n",
"Is Web Scraping Legal?\n",
"100 Movies that You Must Watch\n",
"Day 46 - Intermediate+ Create a Spotify Playlist using the Musical Time Machine\n",
"Day 46 Goals: what you will make by the end of the day\n",
"Step 1 - Scraping the Billboard Hot 100\n",
"Step 2 - Authentication with Spotify\n",
"Step 3 - Search Spotify for the Songs from Step 1\n",
"Step 4 - Creating and Adding to Spotify Playlist\n",
"Day 47 - Intermediate+ Create an Automated Amazon Price Tracker\n",
"Day 47 Goals: what you will make by the end of the day\n",
"Step 1 - Use BeautifulSoup to Scrape the Product Price\n",
"Step 2 - Email Alert When Price Below Preset Value\n",
"Day 48 - Intermediate+ Selenium Webdriver Browser and Game Playing Bot\n",
"Day 48 Goals: what you will make by the end of the day\n",
"How to Install Set Up Selenium\n",
"How to Find and Select Elements on a Website with Selenium\n",
"Challenge: Use Selenium to Scrape Website Data\n",
"Challenge: Use Selenium in a Blank Project Scrape a Different Piece of Data\n",
"How to Automate Filling Out Forms and Clicking Buttons with Selenium\n",
"The Cookie Clicker Project\n",
"Challenge: Create an Automated Game Playing Bot\n",
"Day 49 - Intermediate+ Automating Job Applications on LinkedIn\n",
"Day 49 Goals: what you will make by the end of the day\n",
"Step 1 - Setup Your LinkedIn Account\n",
"Step 2 - Automatically Login\n",
"Step 3 - Apply for a Job\n",
"Step 4 - Apply for all the jobs\n",
"Day 50 - Intermediate+ Auto Tinder Swiping Bot\n",
"Day 50 Goals: what you will make by the end of the day\n",
"Step 1 - Setup your account on Tinder\n",
"Step 2 - Navigate to Login Page\n",
"Step 3 - Login with Facebook\n",
"Step 4 - Dismiss all requests\n",
"Step 5 - Hit Like!\n",
"Day 51 - Intermediate+ Internet Speed Twitter Complaint Bot\n",
"Day 51 Goals: what you will make by the end of the day\n",
"Step 1 - Setup Your Twitter Account\n",
"Step 2 - Create a Class\n",
"Step 3 - Get Internet Speeds\n",
"Step 4 - Building a Twitter Bot to Tweet at your Internet Provider\n",
"Day 52 - Intermediate+ Instagram Follower Bot\n",
"Day 52 Goals: what you will make by the end of the day\n",
"Step 1 - Get Your Instagram Credentials\n",
"Step 2 - Create a Class\n",
"Step 3 - Login to Instagram\n",
"Step 4 - Find the followers of the target account\n",
"Step 5 - Follow all the followers\n",
"Day 53 - Intermediate+ Web Scraping Capstone - Data Entry Job Automation\n",
"Capstone Project Program Requirements\n",
"HINTS SOLUTION\n",
"Day 54 - Intermediate+ Introduction to Web Development with Flask\n",
"Understanding Backend Web Development with Python\n",
"Create your First Web Server with Flask\n",
"Understand the Command Line on Windows and Mac\n",
"__name__ and __main__ : Special Attributes built into Python\n",
"Python Functions as First Class Objects: Passing Nesting Functions\n",
"Understanding Python Decorator Functions and the @ Syntax\n",
"[Interactive Coding Exercise] Create Your Own Python Decorator\n",
"Day 55 - Intermediate+ HTML URL Parsing in Flask and the Higher Lower Game\n",
"Day 55 Goals: what you will make by the end of the day\n",
"Working Flask URL Paths and the Flask Debugger\n",
"Rendering HTML Elements with Flask\n",
"Challenge: Use Python Decorators to Style HTML Tags\n",
"Advanced Decorators with *args and **kwargs\n",
"[Interactive Coding Exercise] Advanced Decorators\n",
"Final Project - Higher or Lower URLs\n",
"Day 56 - Intermediate+ Rendering HTML/Static files and Using Website Templates\n",
"Day 56 Goals: what you will make by the end of the day\n",
"Rendering HTML Files with Flask\n",
"Serving Static Files using Flask\n",
"How to Use Website Templates to Speed Up Web Development\n",
"Final Project - Name Card Website Template\n",
"Solution and Walkthrough for the Name Card Final Project\n",
"Day 57 - Intermediate+ Templating with Jinja in Flask Applications\n",
"Day 57 Goals: what you will make by the end of the day\n",
"Using Jinja to Produce Dynamic HTML Pages\n",
"Challenge: Combining Jinja Templating with APIs\n",
"Multiline Statements with Jinja\n",
"URL Building with Flask\n",
"Blog Capstone Project Part 1 - Templating\n",
"Day 58 - Web Foundation Bootstrap\n",
"Day 58 Goals: What You'll Learn By the End of Today\n",
"What is Bootstrap?\n",
"Installing Bootstrap\n",
"Web Design 101 - Wireframing\n",
"The Bootstrap Navigation Bar\n",
"What We'll Make - Tindog\n",
"Download the Starting Files\n",
"Setting Up Our New Project\n",
"The Bootstrap Grid Layout System\n",
"Getting Custom Fonts and Montserrat Black to Work\n",
"Adding Grid Layouts to Our Website\n",
"A Note About CSS Link Order\n",
"Bootstrap Containers\n",
"Bootstrap Buttons and Font Awesome\n",
"Styling Our Website Challenges and Solutions\n",
"Bootstrap Challenge 1\n",
"Solution to Bootstrap Challenge 1\n",
"The Bootstrap Carousel Part 1\n",
"The Bootstrap Carousel Part 2\n",
"Bootstrap Cards\n",
"CSS Z-index and Stacking Order\n",
"Advanced CSS - Media Query Breakpoints\n",
"Bootstrap Challenge 2\n",
"Solution to Bootstrap Challenge 2\n",
"Code Refactoring\n",
"Refactor Our Website Part 1\n",
"Advanced CSS - Combining Selectors\n",
"Refactoring Our Website Part 2\n",
"Advanced CSS - Selector Priority\n",
"Completing the Website\n",
"Day 59 - Advanced - Blog Capstone Project Part 2 - Adding Styling\n",
"Day 59 Goals: What you'll make by the end of today\n",
"Step 1 - Download the starting project\n",
"Step 2 - Get the home page to work\n",
"Step 3 - Fix the header and footer\n",
"Step 4 - Using Jinja Include fo Render Templates\n",
"Step 5 - Make the About and Contact Pages Work\n",
"Step 6 - Fetch and render the blog posts from an API\n",
"Step 7 - Rendering Individual Posts\n",
"Day 60 - Advanced - Make POST Requests with Flask and HTML Forms\n",
"Day 60 goals - Make the Contact Form Work\n",
"HTML Forms Revision - Creating a Form from Scratch\n",
"Handle POST Requests with Flask Servers\n",
"POST Requests in Flask Solution\n",
"Getting the Contact Form to Work\n",
"Sending Email with smtplib\n",
"Day 61 - Advanced - Building Advanced Forms with Flask-WTForms\n",
"Day 61 Goals: Building Advanced Forms\n",
"Installing Flask-WTF\n",
"Creating Forms with Flask-WTF\n",
"Code Improvements for Our WTForms\n",
"Adding Validation to Forms with Flask-WTF\n",
"Receiving Form Data with WTForms\n",
"Inheriting Templates Using Jinja2\n",
"Using Flask-Bootstrap as an Inherited Template\n",
"Flask-Bootstrap Supports WTForms\n",
"Day 62 - Advanced - Flask, WTForms, Bootstrap and CSV - Coffee Wifi Project\n",
"Download the Starting Project\n",
"Look at the Desired Final Product\n",
"Check Off Each Requirement\n",
"Day 63 - Advanced - Databases and with SQLite and SQLAlchemy\n",
"Day 63 Goals: Creating a Virtual Bookshelf\n",
"Download the Starting Project\n",
"Make the Website Work\n",
"What Happens When You Refresh the Server?\n",
"SQLite Databases\n",
"SQLAlchemy\n",
"CRUD Operations with SQLAlchemy\n",
"Build a SQLite Database into the Flask Website\n",
"Day 64 - Advanced -My Top 10 Movies Website\n",
"Day 64 Goals: What We'll Build\n",
"Download the Starting Project\n",
"Requirement 1 - Be Able to View Movie List Items\n",
"Requirement 2 - Be Able to Edit a Movie's Rating and Review\n",
"Requirement 3 - Be Able to Delete Movies from the Database\n",
"Requirement 4 - Be Able to Add New Movies Via the Add Page\n",
"Requirement 5 - Be Able to Sort and Rank the Movies By Rating\n",
"Day 65 - Web Design School - How to Create a Website that People will Love\n",
"Introduction to Web Design\n",
"Understanding Color Theory\n",
"Understanding Typography and How to Choose Fonts\n",
"Manage ATTENTION with effective User Interface (UI) Design\n",
"User Experience (UX) Design\n",
"Web Design in Practice - Let's apply what we've learnt!\n",
"Day 66 - Advanced - Building Your Own API with RESTful Routing\n",
"Day 66 Goals: Build Your Own REST API Service\n",
"What is REST?\n",
"Download the Starting Project\n",
"HTTP GET - a Random Cafe\n",
"HTTP GET - All the Cafes\n",
"HTTP GET - Find a Cafe\n",
"Postman - The all in one API Testing Tool\n",
"HTTP POST - A New Cafe\n",
"HTTP PUT vs. PATCH\n",
"HTTP PATCH - A Cafe's Coffee Price\n",
"HTTP DELETE - A Cafe that's Closed\n",
"Build Documentation for Your API\n",
"Day 67 - Advanced - Blog Capstone Project Part 3 - RESTful Routing\n",
"Day 67 Goals: Building a RESTful Blog with Editing!\n",
"Download the Starting Project\n",
"Requirement 1 - Be Able to GET Blog Post Items\n",
"Requirement 2 - Be Able to POST a New Blog Post\n",
"Requirement 3 - Be Able to Edit Existing Blog Posts\n",
"Requirement 4- Be Able DELETE Blog Posts\n",
"Day 68 - Advanced - Authentication with Flask\n",
"Day 68 Goals - Login and Registering Users with Authentication\n",
"What is Authentication?\n",
"Download the Starting Project\n",
"Register New Users\n",
"Downloading Files\n",
"Encryption and Hashing\n",
"How to Hack Passwords 101\n",
"Salting Passwords\n",
"Hashing Passwords using Werkzeug\n",
"Authenticating Users with Flask-Login\n",
"Flask Flash Messages\n",
"Passing Authentication Status to Templates\n",
"Day 69 - Advanced - Blog Capstone Project Part 4 - Adding Users\n",
"Day 69 Goals - Adding Users to Our Blog Project\n",
"Download the Starting Project\n",
"Requirement 1 - Register New Users\n",
"Requirement 2 - Login Registered Users\n",
"Requirement 3 - Protect Routes\n",
"Creating Relational Databases\n",
"Requirement 4 - Allow Any User to Add Comments to BlogPosts\n",
"Day 70 - Advanced - Deploying Your Web Application with Heroku\n",
"Day 70 Goals - Learn to Deploy Your Website\n",
"Version Control and Git\n",
"What is GitHub?\n",
"Step 1 - Upload Your Project to GitHub\n",
"Step 2 - Use gunicorn and Heroku to host your website\n",
"Step 3 - Setup a WSGI server with gunicorn\n",
"Step 4 - Upgrade SQLite Database to PostgreSQL\n",
"Day 71 - Advanced - Data Exploration with Pandas: College Major v.s. Your Salary\n",
"Day 71 Goals: what you will make by the end of the day\n",
"Getting Set Up for Data Science\n",
"Upload the Data and Read the .csv File\n",
"Preliminary Data Exploration and Data Cleaning with Pandas\n",
"Accessing Columns and Individual Cells in a Dataframe\n",
"Solution: Highest and Lowest Earning Degrees\n",
"Sorting Values Adding Columns: Majors with the Most Potential vs Lowest Risk\n",
"Solution: Degrees with the Highest Potential\n",
"Grouping and Pivoting Data with Pandas\n",
"Learning Points Summary\n",
"Day 72 - Advanced - Data Visualisation with Matplotlib: Programming Languages\n",
"Day 72 Goals: what you will make by the end of the day\n",
"Download and Open the Starter Notebook\n",
"Solution: Preliminary Data Exploration\n",
"Solution: Analysis by Programming Language\n",
"Data Cleaning: Working with Time Stamps\n",
"Data Manipulation: Pivoting DataFrames\n",
"Data Visualisation with Matplotlib\n",
"Multi-Line Charts with Matplotib\n",
"Smoothing out Time-Series Data\n",
"Programming Language Data Analysis\n",
"Learning Points Summary\n",
"Day 73 - Advanced - Aggregate Merge Data with Pandas: Analyse the LEGO Dataset\n",
"Day 73 Goals: what you will make by the end of the day\n",
"Use HTML Markdown to Make Your Notebook Look Pretty\n",
"Solution: Exploring the LEGO Brick Colours\n",
"Find the Oldest and Largest LEGO Sets\n",
"Visualise the Number of Sets Published over Time\n",
"How to use the Pandas .agg() function\n",
"Superimposing Line Charts with Separate Axes\n",
"Scatter Plots: Average Number of Parts per LEGO Set\n",
"Relational Database Schemas: Primary and Foreign Keys\n",
"How to Merge DataFrames and Create Bar Charts\n",
"Learning Points Summary\n",
"Day 74 - Advanced - Google Trends Data: Resampling and Visualising Time Series\n",
"Day 74 Goals: what you will make by the end of the day\n",
"Data Exploration - Making Sense of Google Search Data\n",
"Data Cleaning - Resampling Time Series Data\n",
"Data Visualisation - Tesla Line Charts in Matplotlib\n",
"Using Locators and DateFormatters to generate Tick Marks on a Time Line\n",
"Data Visualisation - Bitcoin: Line Style and Markers\n",
"Data Visualisation - Unemployment: How to use Grids\n",
"Data Visualisation - Unemployment: The Effect of New Data\n",
"Learning Points Summary\n",
"Day 75 - Advanced - Beautiful Plotly Charts Analysing the Android App Store\n",
"Day 75 Goals: what you will make by the end of the day\n",
"Data Cleaning: Removing NaN Values and Duplicates\n",
"Preliminary Exploration: The Highest Ratings, Most Reviews, and Largest Size\n",
"Data Visualisation with Plotly: Create Pie and Donut Charts\n",
"Numeric Type Conversions for the Installations Price Data\n",
"Plotly Bar Charts Scatter Plots: The Most Competitive Popular App Categories\n",
"Extracting Nested Column Data using .stack()\n",
"Grouped Bar Charts and Box Plots with Plotly\n",
"Learning Points Summary\n",
"Day 76 - Advanced - Computation with NumPy and N-Dimensional Arrays\n",
"Day 76 Goals: what you will make by the end of the day\n",
"NumPy's ndarray - Incredible Power at Your Fingertips!\n",
"Generating and Manipulating ndarrays\n",
"Broadcasting, Scalars and Matrix Multiplication\n",
"Manipulating Images as ndarrays\n",
"Learning Points Summary\n",
"Day 77 - Advanced - Linear Regression and Data Visualisation with Seaborn\n",
"Day 77 Goals: what you will make by the end of the day\n",
"Explore and Clean the Data\n",
"Investigate the Films that had Zero Revenue\n",
"Filter on Multiple Conditions: International Films\n",
"Seaborn Data Visualisation: Bubble Charts\n",
"Floor Division: A Trick to Convert Years to Decades\n",
"Plotting Linear Regressions with Seaborn\n",
"Use scikit-learn to Run Your Own Regression\n",
"Learning Points Summary\n",
"Day 78 - Advanced - Analysing the Nobel Prize with Plotly, Matplotlib Seaborn\n",
"Day 78 Goals: what you will make by the end of the day\n",
"Update Packages in Google Colab Explore and Clean the Dataset\n",
"plotly Bar Donut Charts: Analyse Prize Categories Women Winning Prizes\n",
"Using Matplotlib to Visualise Trends over Time\n",
"A Choropleth Map and the Countries with the Most Prizes\n",
"Create Sunburst Charts for a Detailed Regional Breakdown of Research Locations\n",
"Unearthing Patterns in the Laureate Age at the Time of the Award\n",
"Learning Points Summary\n",
"Day 79 - Advanced - The Tragic Discovery of Handwashing: t-Tests Distributions\n",
"Day 79 Goals: what you will make by the end of the day\n",
"Preliminary Data Exploration and Visualising Births Deaths at Vienna Hospital\n",
"Analysing the Yearly Data Split By Clinic\n",
"The Effect of Handwashing\n",
"Visualising Distributions and Testing for Statistical Significance\n",
"Learning Points Summary\n",
"Day 80 - Advanced - Capstone Project - Predict House Prices\n",
"Day 80 Goals: what you will make by the end of the day\n",
"Solution Learning Points\n",
"Day 81 - Professional Portfolio Project - [Python Scripting]\n",
"The Road to Becoming a Professional Developer\n",
"Text to Morse Code Converter\n",
"Day 82 - Professional Portfolio Project - [Python Web Development]\n",
"Where are the Videos and the Solution Code?\n",
"Portfolio Website\n",
"Day 83 - Professional Portfolio Project - [Python Scripting]\n",
"Tic Tac Toe\n",
"Day 84 - Professional Portfolio Project - [GUI]\n",
"Image Watermarking Desktop App\n",
"Day 85 - Professional Portfolio Project - [GUI]\n",
"Typing Speed Test\n",
"Day 86 - Professional Portfolio Project - [Game]\n",
"Breakout Game\n",
"Day 87 - Professional Portfolio Project - [Web Development]\n",
"Cafe and Wifi Website\n",
"Day 88 - Professional Portfolio Project - [Web Development]\n",
"Todo List\n",
"Day 89 - Professional Portfolio Project - [GUI Desktop App]\n",
"Disappearing Text Writing App\n",
"Day 90 - Professional Portfolio Project - [HTTP Requests APIs]\n",
"Convert PDF to Audiobook\n",
"Day 91 - Professional Portfolio Project - [Image Processing Data Science]\n",
"Image Colour Palette Generator\n",
"Day 92 - Professional Portfolio Project - [Web Scraping]\n",
"Custom Web Scraper\n",
"Day 93 - Professional Portfolio Project - [GUI Automation]\n",
"Automate the Google Dinosaur Game\n",
"Day 94 - Professional Portfolio Project - [Game]\n",
"Space Invaders\n",
"Day 95 - Professional Portfolio Project - [HTTP Requests APIs]\n",
"Custom API Based Website\n",
"Day 96 - Professional Portfolio Project - [Web Development]\n",
"An Online Shop\n",
"Day 97 - Professional Portfolio Project - [Python Automation]\n",
"Custom Automation\n",
"Day 98 - Professional Portfolio Project - [Data Science]\n",
"Analyse and Visualise the Space Race\n",
"Day 99 - Professional Portfolio Project - [Data Science]\n",
"Analyse Deaths involving Police in the United States\n",
"Day 100 - Professional Portfolio Project - [Data Science]\n",
"Predict Earnings using Multivariable Regression\n",
"Final Stretch\n",
"Recording of our Live AMA (aka AAA - Ask Angela Anything)\n",
"Study With Me\n",
"Bonus Lecture: Check out my other courses\n"
]
}
],
"source": [
"#Printing list\n",
"from lxml import etree\n",
"parser = etree.XMLParser(recover=True)\n",
"parsed_file = etree.parse(\"/Users/linda/Downloads/webpage100daysOfCode.html\", parser=parser)\n",
"title_element = parsed_file.xpath(\"//span[@class='section--section-title--8blTh' or \"\n",
" \"@class='section--item-title--2k1DQ']\")\n",
"\n",
"for li in title_element:\n",
" text = ''.join(map(str.strip, li.xpath(\".//text()\")))\n",
" print(text)\n"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"This is the title\n",
"Hello World\n"
]
}
],
"source": [
"from lxml import etree\n",
"tree = etree.parse(\"/Users/linda/Documents/src/web_page.html\")\n",
"title_element = tree.xpath(\"//title/text()\")[0]\n",
"print(title_element)\n",
"\n",
"paragraph_element = tree.xpath(\"//p/text()\")[0]\n",
"print(paragraph_element)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"This is the title\n",
"Hello World\n"
]
}
],
"source": [
"#Printing list\n",
"from lxml import etree\n",
"html = etree.parse('/Users/linda/Documents/src/web_page.html')\n",
"html_element = html.getroot()\n",
"\n",
"title = html_element.cssselect('title')[0]\n",
"print(title.text)\n",
"\n",
"p = html_element.cssselect('p')[0]\n",
"print(p.text)\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Web Scraping with Python using Requests, LXML and Splash\n",
"Created by: Ahmed Rafik\n"
]
}
],
"source": [
"html = etree.parse('/Users/linda/Documents/src/web_page.html')\n",
"html_element = html.getroot()\n",
"\n",
"\n",
"list_items = html_element.cssselect('li')\n",
"for li in list_items:\n",
" a = li.cssselect('a')\n",
" if len(a) == 0:\n",
" print(li.text)\n",
" else:\n",
" print(f\"{li.text.strip()} {a[0].text}\")"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(b'<html lang=\"en\">\\n\\n<head>\\n <title>This is the title</title>\\n</head'\n",
" b'>\\n\\n<body>\\n <p>Hello World</p>\\n <ul>\\n <li id=\"myID\">Web '\n",
" b'Scraping with Python using Requests, LXML and Splash</li>\\n <li cl'\n",
" b'ass=\"myClass\">Created by:\\n <a href=\"https://twitter.com/Ahmed'\n",
" b'Rafik__\">Ahmed Rafik</a>\\n </li>\\n </ul>\\n</body>\\n\\n</html>')\n"
]
}
],
"source": [
"from pprint import pp\n",
"html = etree.parse('/Users/linda/Documents/src/web_page.html')\n",
"pp(etree.tostring(html))\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['Hello World!']\n",
"['Thanks for Watching!']\n",
"['Choose DataCamp!']\n"
]
}
],
"source": [
"tree = etree.parse(\"/Users/linda/Documents/src/xpath_datacamp.html\")\n",
"test1 = tree.xpath(\"/html/body/div[1]/p/text()\")\n",
"test2 = tree.xpath(\"/html/body/div[2]/p/text()\")\n",
"test3 = tree.xpath(\"/html/body/div/div/p/text()\")\n",
"print(test1)\n",
"print(test2)\n",
"print(test3)\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"pycharm": {
"name": "#%%\n"
},
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<!DOCTYPE html>\n",
"\n",
"<html>\n",
"<head>\n",
"<meta charset=\"utf-8\"/>\n",
"<title>Turtle Shellter</title>\n",
"<link href=\"https://fonts.googleapis.com/css?family=Poppins\" rel=\"stylesheet\"/>\n",
"<link href=\"style.css\" rel=\"stylesheet\" type=\"text/css\"/>\n",
"</head>\n",
"<body>\n",
"<div class=\"banner\">\n",
"<h1>The Shellter</h1>\n",
"<span class=\"brag\">The #1 Turtle Adoption website!</span>\n",
"</div>\n",
"<div class=\"about\">\n",
"<p class=\"text\">Click to learn more about each turtle</p>\n",
"</div>\n",
"<div class=\"grid\">\n",
"<div class=\"box adopt\">\n",
"<a class=\"more-info\" href=\"aesop.html\"><img class=\"headshot\" src=\"aesop.png\"/></a>\n",
"<p>Aesop</p>\n",
"</div>\n",
"<div class=\"box adopt\">\n",
"<a class=\"more-info\" href=\"caesar.html\"><img class=\"headshot\" src=\"caesar.png\"/></a>\n",
"<p>Caesar</p>\n",
"</div>\n",
"<div class=\"box adopt\">\n",
"<a class=\"more-info\" href=\"sulla.html\"><img class=\"headshot\" src=\"sulla.png\"/></a>\n",
"<p>Sulla</p>\n",
"</div>\n",
"<div class=\"box adopt\">\n",
"<a class=\"more-info\" href=\"spyro.html\"><img class=\"headshot\" src=\"spyro.png\"/></a>\n",
"<p>Spyro</p>\n",
"</div>\n",
"<div class=\"box adopt\">\n",
"<a class=\"more-info\" href=\"zelda.html\"><img class=\"headshot\" src=\"zelda.png\"/></a>\n",
"<p>Zelda</p>\n",
"</div>\n",
"<div class=\"box adopt\">\n",
"<a class=\"more-info\" href=\"bandicoot.html\"><img class=\"headshot\" src=\"bandicoot.png\"/></a>\n",
"<p>Bandicoot</p>\n",
"</div>\n",
"<div class=\"box adopt\">\n",
"<a class=\"more-info\" href=\"hal.html\"><img class=\"headshot\" src=\"hal.png\"/></a>\n",
"<p>Hal</p>\n",
"</div>\n",
"<div class=\"box adopt\">\n",
"<a class=\"more-info\" href=\"mock.html\"><img class=\"headshot\" src=\"mock.png\"/></a>\n",
"<p>Mock</p>\n",
"</div>\n",
"<div class=\"box adopt\">\n",
"<a class=\"more-info\" href=\"sparrow.html\"><img class=\"headshot\" src=\"sparrow.png\"/></a>\n",
"<p>Captain Sparrow</p>\n",
"</div>\n",
"</div>\n",
"<script>(function(){var js = \"window['__CF$cv$params']={r:'7765ce0ecf9155d2',m:'08DF3EUtkgSm09GVK6OvO0rcNOgOwsmmdfX8VXwovWI-1670505694-0-AdlGQPj55QC2ovOAw1dazRwmYqP3rLAZW5xn+y3txvy7Z0Hm1qqu1qb9GCPiL2EicPjjX7jivEPBOn85BU8HXj6cSkjjMjMmZUQfRVsG3Q5f8hrCPl2M3ky5XU4H2SQoT1wMwcfohE34KAnvmo3tqjECQOKW/i3QEX57M2xBSIVD',s:[0x78867f051c,0x44ce671996],u:'/cdn-cgi/challenge-platform/h/g'};var now=Date.now()/1000,offset=14400,ts=''+(Math.floor(now)-Math.floor(now%offset)),_cpo=document.createElement('script');_cpo.nonce='',_cpo.src='/cdn-cgi/challenge-platform/h/g/scripts/alpha/invisible.js?ts='+ts,document.getElementsByTagName('head')[0].appendChild(_cpo);\";var _0xh = document.createElement('iframe');_0xh.height = 1;_0xh.width = 1;_0xh.style.position = 'absolute';_0xh.style.top = 0;_0xh.style.left = 0;_0xh.style.border = 'none';_0xh.style.visibility = 'hidden';document.body.appendChild(_0xh);function handler() {var _0xi = _0xh.contentDocument || _0xh.contentWindow.document;if (_0xi) {var _0xj = _0xi.createElement('script');_0xj.nonce = '';_0xj.innerHTML = js;_0xi.getElementsByTagName('head')[0].appendChild(_0xj);}}if (document.readyState !== 'loading') {handler();} else if (window.addEventListener) {document.addEventListener('DOMContentLoaded', handler);} else {var prev = document.onreadystatechange || function () {};document.onreadystatechange = function (e) {prev(e);if (document.readyState !== 'loading') {document.onreadystatechange = prev;handler();}};}})();</script></body>\n",
"</html>\n",
"\n"
]
}
],
"source": [
"import requests\n",
"from bs4 import BeautifulSoup\n",
"webpage_response = requests.get('https://content.codecademy.com/courses/beautifulsoup/shellter.html')\n",
"webpage = webpage_response.content\n",
"soup = BeautifulSoup(webpage, 'html.parser')\n",
"print(soup)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['Set Me Free']\n"
]
}
],
"source": [
"from lxml import html\n",
"\n",
"url='https://books.toscrape.com/catalogue/set-me-free_988/index.html'\n",
"resp = requests.get(url, headers={\n",
" 'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Mobile Safari/537.36'\n",
"})\n",
"\n",
"tree = html.fromstring(html=resp.text)\n",
"test = tree.xpath(\"//div[@class='col-sm-6 product_main']/h1/text()\")\n",
"print(test)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"17.46\n"
]
}
],
"source": [
"import re \n",
"\n",
"url='https://books.toscrape.com/catalogue/set-me-free_988/index.html'\n",
"resp = requests.get(url, headers={\n",
" 'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Mobile Safari/537.36'\n",
"})\n",
"\n",
"tree = html.fromstring(html=resp.text)\n",
"test = tree.xpath(\"//div[@class='col-sm-6 product_main']/p[@class='price_color']/text()\")[0]\n",
"price_digitonly = ''.join(map(str, re.findall(r\"\\d+.\\d+|\\d+\", test)))\n",
"print(price_digitonly)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"19\n"
]
}
],
"source": [
"url='https://books.toscrape.com/catalogue/set-me-free_988/index.html'\n",
"resp = requests.get(url, headers={\n",
" 'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Mobile Safari/537.36'\n",
"})\n",
"\n",
"tree = html.fromstring(html=resp.text)\n",
"test = tree.xpath(\"//div[@class='col-sm-6 product_main']/ p[@class='instock availability']/text()\")[1]\n",
"in_stock = re.findall(r\"\\d+\", test)\n",
"print(''.join(map(str, in_stock)))"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Aaron Ledbetter’s future had been planned out for him since before he was born. Each year, the Ledbetter family vacation on Tybee Island gave Aaron a chance to briefly free himself from his family’s expectations. When he meets Jonas “Lucky” Luckett, a caricature artist in town with the traveling carnival, he must choose between the life that’s been mapped out for him, and Aaron Ledbetter’s future had been planned out for him since before he was born. Each year, the Ledbetter family vacation on Tybee Island gave Aaron a chance to briefly free himself from his family’s expectations. When he meets Jonas “Lucky” Luckett, a caricature artist in town with the traveling carnival, he must choose between the life that’s been mapped out for him, and the chance at true love. ...more\n"
]
}
],
"source": [
"url='https://books.toscrape.com/catalogue/set-me-free_988/index.html'\n",
"resp = requests.get(url, headers={\n",
" 'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Mobile Safari/537.36'\n",
"})\n",
"\n",
"tree = html.fromstring(html=resp.text)\n",
"test = tree.xpath(\"//article[@class='product_page']/p/text()\")\n",
"print(''.join(map(str, test)))"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[<Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>,\n",
" <Selector xpath=\"//div[@id='search_resultsRows']/a\" data='<a href=\"https://store.steampowered.c...'>]\n"
]
}
],
"source": [
"import requests\n",
"from scrapy.http import TextResponse\n",
"from pprint import pp\n",
"\n",
"r = requests.get('https://store.steampowered.com/search/?filter=topsellers')\n",
"response = TextResponse(r.url, body=r.text, encoding='utf-8')\n",
"nxgames = response.xpath(\"//div[@id='search_resultsRows']/a\")\n",
"\n",
"pp(nxgames)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://store.steampowered.com/app/570/Dota_2/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1811260/EA_SPORTS_FIFA_23/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1938090/Call_of_Duty_Modern_Warfare_II/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/730/CounterStrike_Global_Offensive/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1462040/FINAL_FANTASY_VII_REMAKE_INTERGRADE/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1659420/UNCHARTED_Legacy_of_Thieves_Collection/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/2167580/Summoners_War_Chronicles/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1172470/Apex_Legends/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1237320/Sonic_Frontiers/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1593500/God_of_War/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1361210/Warhammer_40000_Darktide/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1549250/UNDECEMBER/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1608070/CRISIS_CORE_FINAL_FANTASY_VII_REUNION/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/578080/PUBG_BATTLEGROUNDS/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/255710/Cities_Skylines/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/236390/War_Thunder/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1904540/Football_Manager_2023/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/230410/Warframe/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1063730/New_World/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1583230/High_On_Life/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/238960/Path_of_Exile/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1245620/ELDEN_RING/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1817070/Marvels_SpiderMan_Remastered/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1476970/IdleOn__The_Idle_MMO/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1222140/Detroit_Become_Human/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/39210/FINAL_FANTASY_XIV_Online/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/2208920/Assassins_Creed_Valhalla/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/548430/Deep_Rock_Galactic/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/985810/GrandChase/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1817190/Marvels_SpiderMan_Miles_Morales/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1091500/Cyberpunk_2077/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1544020/The_Callisto_Protocol/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1449850/YuGiOh_Master_Duel/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1527950/Wartales/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/435150/Divinity_Original_Sin_2__Definitive_Edition/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/534380/Dying_Light_2_Stay_Human/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1629520/A_Little_to_the_Left/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/368260/Marvels_Midnight_Suns/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/227300/Euro_Truck_Simulator_2/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1144200/Ready_or_Not/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/582660/Black_Desert/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1172620/Sea_of_Thieves/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1210320/Potion_Craft_Alchemist_Simulator/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1085660/Destiny_2/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1846380/Need_for_Speed_Unbound/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1203220/NARAKA_BLADEPOINT/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/997070/Marvels_Avengers/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/1284210/Guild_Wars_2/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/812140/Assassins_Creed_Odyssey/?snr=1_7_7_7000_150_1\n",
"https://store.steampowered.com/app/703080/Planet_Zoo/?snr=1_7_7_7000_150_1\n"
]
}
],
"source": [
"import requests\n",
"from scrapy.http import TextResponse\n",
"\n",
"r = requests.get('https://store.steampowered.com/search/?filter=topsellers')\n",
"response = TextResponse(r.url, body=r.text, encoding='utf-8')\n",
"nxgames = response.xpath(\"//div[@id='search_resultsRows']/a\")\n",
"\n",
"\n",
"# get the game_url\n",
"for nxgame in nxgames:\n",
" nxurl = nxgame.xpath(\".//@href\").get()\n",
" print(nxurl)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://cdn.akamai.steamstatic.com/steam/apps/730/capsule_sm_120.jpg?t=1668125812\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1938090/capsule_sm_120.jpg?t=1671472823\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1172470/capsule_sm_120.jpg?t=1670431796\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1245620/capsule_sm_120.jpg?t=1671759466\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1811260/capsule_sm_120_alt_assets_0.jpg?t=1671142874\n",
"https://cdn.akamai.steamstatic.com/steam/apps/570/capsule_sm_120.jpg?t=1666237243\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1174180/capsule_sm_120.jpg?t=1671485009\n",
"https://cdn.akamai.steamstatic.com/steam/apps/236390/capsule_sm_120.jpg?t=1671733778\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1144200/capsule_sm_120.jpg?t=1670541133\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1091500/capsule_sm_120.jpg?t=1663663573\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1583230/capsule_sm_120.jpg?t=1671088821\n",
"https://cdn.akamai.steamstatic.com/steam/apps/271590/capsule_sm_120.jpg?t=1671485100\n",
"https://cdn.akamai.steamstatic.com/steam/apps/252490/capsule_sm_120_alt_assets_20.jpg?t=1671160999\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1593500/capsule_sm_120.jpg?t=1650554420\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1551360/capsule_sm_120.jpg?t=1668017884\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1085660/capsule_sm_120.jpg?t=1671639469\n",
"https://cdn.akamai.steamstatic.com/steam/apps/962130/capsule_sm_120.jpg?t=1669729959\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1222670/capsule_sm_120.jpg?t=1668103304\n",
"https://cdn.akamai.steamstatic.com/steam/apps/578080/capsule_sm_120.jpg?t=1671222003\n",
"https://cdn.akamai.steamstatic.com/steam/apps/453480/capsule_sm_120.jpg?t=1672112841\n",
"https://cdn.akamai.steamstatic.com/steam/apps/648800/capsule_sm_120.jpg?t=1655744208\n",
"https://cdn.akamai.steamstatic.com/steam/apps/294100/capsule_sm_120.jpg?t=1666905455\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1172620/capsule_sm_120.jpg?t=1670607777\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1687950/capsule_sm_120.jpg?t=1671645108\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1919590/capsule_sm_120.jpg?t=1669997894\n",
"https://cdn.akamai.steamstatic.com/steam/apps/359550/capsule_sm_120.jpg?t=1671482386\n",
"https://cdn.akamai.steamstatic.com/steam/apps/814380/capsule_sm_120.jpg?t=1669076148\n",
"https://cdn.akamai.steamstatic.com/steam/apps/990080/capsule_sm_120.jpg?t=1668268562\n",
"https://cdn.akamai.steamstatic.com/steam/apps/289070/capsule_sm_120.jpg?t=1671048172\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1817190/capsule_sm_120.jpg?t=1668787110\n",
"https://cdn.akamai.steamstatic.com/steam/apps/108600/capsule_sm_120.jpg?t=1667832656\n",
"https://cdn.akamai.steamstatic.com/steam/apps/255710/capsule_sm_120.jpg?t=1670837127\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1361210/capsule_sm_120.jpg?t=1670979917\n",
"https://cdn.akamai.steamstatic.com/steam/apps/292030/capsule_sm_120.jpg?t=1670976384\n",
"https://cdn.akamai.steamstatic.com/steam/apps/381210/capsule_sm_120.jpg?t=1669150759\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1446780/capsule_sm_120.jpg?t=1670323598\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1817070/capsule_sm_120.jpg?t=1667406675\n",
"https://cdn.akamai.steamstatic.com/steam/apps/306130/capsule_sm_120.jpg?t=1667312669\n",
"https://cdn.akamai.steamstatic.com/steam/apps/2208920/capsule_sm_120.jpg?t=1671135934\n",
"https://cdn.akamai.steamstatic.com/steam/apps/242760/capsule_sm_120.jpg?t=1666811027\n",
"https://cdn.akamai.steamstatic.com/steam/apps/227300/capsule_sm_120.jpg?t=1668166019\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1332010/capsule_sm_120.jpg?t=1670349423\n",
"https://cdn.akamai.steamstatic.com/steam/apps/975370/capsule_sm_120.jpg?t=1670966411\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1142710/capsule_sm_120.jpg?t=1669220074\n",
"https://cdn.akamai.steamstatic.com/steam/apps/311210/capsule_sm_120.jpg?t=1646763462\n",
"https://cdn.akamai.steamstatic.com/steam/apps/413150/capsule_sm_120.jpg?t=1666917466\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1426210/capsule_sm_120.jpg?t=1666121755\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1286680/capsule_sm_120.jpg?t=1660273090\n",
"https://cdn.akamai.steamstatic.com/steam/apps/916440/capsule_sm_120.jpg?t=1671121447\n",
"https://cdn.akamai.steamstatic.com/steam/apps/1904540/capsule_sm_120.jpg?t=1668082258\n"
]
}
],
"source": [
"import requests\n",
"from scrapy.http import TextResponse\n",
"\n",
"r = requests.get('https://store.steampowered.com/search/?filter=topsellers')\n",
"response = TextResponse(r.url, body=r.text, encoding='utf-8')\n",
"nxgames = response.xpath(\"//div[@id='search_resultsRows']/a\")\n",
"\n",
"# get the game_image\n",
"for nxgame in nxgames:\n",
" img_url = nxgame.xpath(\".//div[@class='col search_capsule']/img/@src\").get()\n",
" print(img_url)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dota 2\n",
"EA SPORTS™ FIFA 23\n",
"Call of Duty®: Modern Warfare® II\n",
"Counter-Strike: Global Offensive\n",
"FINAL FANTASY VII REMAKE INTERGRADE\n",
"UNCHARTED™: Legacy of Thieves Collection\n",
"Summoners War: Chronicles\n",
"Warhammer 40,000: Darktide\n",
"Sonic Frontiers\n",
"God of War\n",
"Apex Legends™\n",
"CRISIS CORE –FINAL FANTASY VII– REUNION\n",
"PUBG: BATTLEGROUNDS\n",
"UNDECEMBER\n",
"War Thunder\n",
"Cities: Skylines\n",
"Football Manager 2023\n",
"New World\n",
"Path of Exile\n",
"IdleOn - The Idle MMO\n",
"Warframe\n",
"FINAL FANTASY XIV Online\n",
"High On Life\n",
"ELDEN RING\n",
"Marvel’s Spider-Man Remastered\n",
"Yu-Gi-Oh! Master Duel\n",
"Assassin's Creed Valhalla\n",
"Deep Rock Galactic\n",
"Detroit: Become Human\n",
"Need for Speed™ Unbound\n",
"Marvel’s Spider-Man: Miles Morales\n",
"Divinity: Original Sin 2 - Definitive Edition\n",
"Cyberpunk 2077\n",
"World of Warships\n",
"Sea of Thieves\n",
"The Callisto Protocol™\n",
"Wartales\n",
"Black Desert\n",
"Marvel's Midnight Suns\n",
"Dying Light 2 Stay Human\n",
"Ready or Not\n",
"Euro Truck Simulator 2\n",
"Assassin's Creed® Odyssey\n",
"Destiny 2\n",
"A Little to the Left\n",
"Potion Craft: Alchemist Simulator\n",
"Marvel's Avengers\n",
"Guild Wars 2\n",
"NARAKA: BLADEPOINT\n",
"World of Tanks Blitz\n"
]
}
],
"source": [
"import requests\n",
"from scrapy.http import TextResponse\n",
"\n",
"r = requests.get('https://store.steampowered.com/search/?filter=topsellers')\n",
"response = TextResponse(r.url, body=r.text, encoding='utf-8')\n",
"nxgames = response.xpath(\"//div[@id='search_resultsRows']/a\")\n",
"\n",
"# get the game_name\n",
"for nxgame in nxgames:\n",
" game_name = nxgame.xpath(\".//span[@class='title']/text()\").get()\n",
" print(game_name)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"9 Jul, 2013\n",
"29 Sep, 2022\n",
"27 Oct, 2022\n",
"21 Aug, 2012\n",
"17 Jun, 2022\n",
"19 Oct, 2022\n",
"9 Nov, 2022\n",
"30 Nov, 2022\n",
"7 Nov, 2022\n",
"14 Jan, 2022\n",
"4 Nov, 2020\n",
"13 Dec, 2022\n",
"21 Dec, 2017\n",
"11 Oct, 2022\n",
"15 Aug, 2013\n",
"10 Mar, 2015\n",
"7 Nov, 2022\n",
"28 Sep, 2021\n",
"23 Oct, 2013\n",
"2 Apr, 2021\n",
"25 Mar, 2013\n",
"18 Feb, 2014\n",
"13 Dec, 2022\n",
"24 Feb, 2022\n",
"12 Aug, 2022\n",
"18 Jan, 2022\n",
"6 Dec, 2022\n",
"13 May, 2020\n",
"18 Jun, 2020\n",
"1 Dec, 2022\n",
"18 Nov, 2022\n",
"14 Sep, 2017\n",
"9 Dec, 2020\n",
"15 Nov, 2017\n",
"3 Jun, 2020\n",
"1 Dec, 2022\n",
"1 Dec, 2021\n",
"17 Sep, 2019\n",
"1 Dec, 2022\n",
"3 Feb, 2022\n",
"17 Dec, 2021\n",
"12 Oct, 2012\n",
"5 Oct, 2018\n",
"1 Oct, 2019\n",
"8 Nov, 2022\n",
"13 Dec, 2022\n",
"4 Sep, 2020\n",
"23 Aug, 2022\n",
"11 Aug, 2021\n",
"9 Nov, 2016\n"
]
}
],
"source": [
"import requests\n",
"from scrapy.http import TextResponse\n",
"\n",
"r = requests.get('https://store.steampowered.com/search/?filter=topsellers')\n",
"response = TextResponse(r.url, body=r.text, encoding='utf-8')\n",
"nxgames = response.xpath(\"//div[@id='search_resultsRows']/a\")\n",
"\n",
"# get the release_date of the game\n",
"for nxgame in nxgames:\n",
" release_date = nxgame.xpath(\".//div[@class='col search_released responsive_secondrow']/text()\").get()\n",
" print(release_date)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['platform_img win']\n",
"['platform_img win', 'platform_img mac', 'platform_img linux']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win', 'platform_img mac', 'platform_img linux']\n",
"['platform_img win']\n",
"['platform_img win', 'platform_img mac', 'platform_img linux', 'vr_supported']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win', 'platform_img mac']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win', 'platform_img mac']\n",
"['platform_img win', 'platform_img mac', 'platform_img linux']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win', 'platform_img mac', 'platform_img linux']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win', 'platform_img mac', 'platform_img linux']\n",
"['platform_img win', 'platform_img mac', 'platform_img linux']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win', 'platform_img mac']\n",
"['platform_img win', 'platform_img mac', 'platform_img linux']\n",
"['platform_img win', 'vr_supported']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win', 'platform_img mac']\n",
"['platform_img win', 'platform_img mac', 'platform_img linux']\n",
"['platform_img win', 'platform_img mac', 'platform_img linux']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win']\n",
"['platform_img win', 'platform_img mac']\n"
]
}
],
"source": [
"import requests\n",
"from scrapy.http import TextResponse\n",
"\n",
"r = requests.get('https://store.steampowered.com/search/?filter=topsellers')\n",
"response = TextResponse(r.url, body=r.text, encoding='utf-8')\n",
"nxgames = response.xpath(\"//div[@id='search_resultsRows']/a\")\n",
"\n",
"# get the platforms of the game\n",
"for nxgame in nxgames:\n",
" platforms = nxgame.xpath(\".//span[contains(@class, 'platform_img') or @class='vr_supported']/@class\").getall()\n",
" print(platforms)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['linux', 'vr']"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import re\n",
"text =\" platform_img linux vr_supported \"\n",
"re.findall(r\"win|mac|linux|vr\", text)"
]
},
{
"cell_type": "code",
"execution_count": 51,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['win']\n",
"['mac']\n",
"['linux']\n",
"['vr_supported']\n"
]
}
],
"source": [
"nx_classes = [\n",
" 'platform_img win',\n",
" 'platform_img mac',\n",
" 'platform_img linux',\n",
" 'vr_supported',\n",
"]\n",
"\n",
"ps = []\n",
"for c in nx_classes:\n",
" p = re.findall(r\"win|mac|linux|vr_supported\", c)\n",
" print(p)"
]
},
{
"cell_type": "code",
"execution_count": 54,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"win\n",
"mac\n",
"linux\n",
"vr_supported\n",
"['Windows', 'Mac os', 'Linux', 'VR Supported']\n"
]
}
],
"source": [
"# Using Regex\n",
"nx_classes = [\n",
" 'platform_img win',\n",
" 'platform_img mac',\n",
" 'platform_img linux',\n",
" 'vr_supported',\n",
"]\n",
"# get platforms from list of classes\n",
"def get_platforms(list_c):\n",
" ps = []\n",
" for c in list_c:\n",
" p = ''.join(re.findall(r\"win|mac|linux|vr_supported\", c))\n",
" if p == 'win':\n",
" ps.append('Windows')\n",
" if p == 'mac':\n",
" ps.append('Mac os')\n",
" if p == 'linux':\n",
" ps.append('Linux')\n",
" if p == 'vr_supported':\n",
" ps.append('VR Supported')\n",
" print(p)\n",
" return ps\n",
"\n",
"print(get_platforms(nx_classes))"
]
},
{
"cell_type": "code",
"execution_count": 56,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"123\n",
"123\n"
]
}
],
"source": [
"# https://stackoverflow.com/questions/5618878/how-to-convert-list-to-string\n",
"# How to convert list to string\n",
"\n",
"xs = ['1', '2', '3']\n",
"s = ''.join(xs)\n",
"print(s)\n",
"\n",
"# If the list contains integers, convert the elements to string before joining them:\n",
"xs = [1, 2, 3]\n",
"s = ''.join(str(x) for x in xs)\n",
"print(s)"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['Windows', 'Mac os', 'Linux', 'VR Supported']\n"
]
}
],
"source": [
"# Using split\n",
"classes = [\n",
" 'platform_img win',\n",
" 'platform_img mac',\n",
" 'platform_img linux',\n",
" 'vr_supported',\n",
"]\n",
"\n",
"\n",
"def get_platforms(list_classes):\n",
" platforms = []\n",
" for item in list_classes:\n",
" platform = item.split(' ')[-1]\n",
" if platform == 'win':\n",
" platforms.append('Windows')\n",
" if platform == 'mac':\n",
" platforms.append('Mac os')\n",
" if platform == 'linux':\n",
" platforms.append('Linux')\n",
" if platform == 'vr_supported':\n",
" platforms.append('VR Supported')\n",
"\n",
" return platforms\n",
"\n",
"\n",
"print(get_platforms(classes))"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Very Positive 84% of the 556,413 user reviews for this game are positive.\n",
"Very Positive 88% of the 6,823,333 user reviews for this game are positive.\n",
"Mixed 62% of the 270,296 user reviews for this game are positive.\n",
"Very Positive 91% of the 453,948 user reviews for this game are positive.\n",
"Mixed 49% of the 42,445 user reviews for this game are positive.\n",
"Very Positive 82% of the 1,900,742 user reviews for this game are positive.\n",
"Very Positive 89% of the 305,681 user reviews for this game are positive.\n",
"Mostly Positive 78% of the 356,780 user reviews for this game are positive.\n",
"Very Positive 92% of the 71,701 user reviews for this game are positive.\n",
"Mostly Positive 79% of the 504,700 user reviews for this game are positive. This product has experienced one or more periods of off-topic review activity. Based on your preferences, the reviews within these periods have been excluded from this product's Review Score.\n",
"Very Positive 90% of the 4,540 user reviews for this game are positive.\n",
"Very Positive 85% of the 1,331,066 user reviews for this game are positive.\n",
"Very Positive 87% of the 702,643 user reviews for this game are positive.\n",
"Overwhelmingly Positive 97% of the 57,089 user reviews for this game are positive.\n",
"Very Positive 83% of the 511,074 user reviews for this game are positive.\n",
"Very Positive 88% of the 91,819 user reviews for this game are positive.\n",
"Very Positive 89% of the 36,786 user reviews for this game are positive.\n",
"Very Positive 86% of the 80,120 user reviews for this game are positive.\n",
"Mixed 56% of the 2,120,350 user reviews for this game are positive.\n",
"Overwhelmingly Positive 98% of the 126,902 user reviews for this game are positive.\n",
"Very Positive 90% of the 221,719 user reviews for this game are positive.\n",
"Very Positive 93% of the 208,243 user reviews for this game are positive.\n",
"Mostly Positive 75% of the 13,463 user reviews for this game are positive.\n",
"Very Positive 87% of the 926,966 user reviews for this game are positive.\n",
"Mixed 58% of the 10,137 user reviews for this game are positive.\n",
"Overwhelmingly Positive 97% of the 17,449 user reviews for this game are positive.\n",
"No reviews\n",
"Very Positive 84% of the 175,792 user reviews for this game are positive.\n",
"Very Positive 94% of the 150,684 user reviews for this game are positive.\n",
"Very Positive 93% of the 114,817 user reviews for this game are positive.\n",
"Mixed 61% of the 44,826 user reviews for this game are positive.\n",
"Overwhelmingly Positive 96% of the 623,188 user reviews for this game are positive. This product has experienced one or more periods of off-topic review activity. Based on your preferences, the reviews within these periods have been excluded from this product's Review Score.\n",
"Very Positive 94% of the 5,940 user reviews for this game are positive.\n",
"Very Positive 93% of the 159,990 user reviews for this game are positive.\n",
"Very Positive 81% of the 456,125 user reviews for this game are positive.\n",
"Very Positive 87% of the 40,094 user reviews for this game are positive.\n",
"Very Positive 83% of the 105,593 user reviews for this game are positive.\n",
"Overwhelmingly Positive 97% of the 37,081 user reviews for this game are positive.\n",
"Overwhelmingly Positive 95% of the 369,054 user reviews for this game are positive.\n",
"Overwhelmingly Positive 97% of the 435,437 user reviews for this game are positive. This product has experienced one or more periods of off-topic review activity. Based on your preferences, the reviews within these periods have been excluded from this product's Review Score.\n",
"Mixed 63% of the 3,402 user reviews for this game are positive.\n",
"Overwhelmingly Positive 97% of the 95,520 user reviews for this game are positive.\n",
"Very Positive 85% of the 86,455 user reviews for this game are positive.\n",
"Overwhelmingly Positive 96% of the 14,394 user reviews for this game are positive.\n",
"Mostly Positive 71% of the 43,837 user reviews for this game are positive.\n",
"Mostly Positive 77% of the 5,821 user reviews for this game are positive.\n",
"Overwhelmingly Positive 98% of the 441,840 user reviews for this game are positive.\n",
"Mostly Positive 79% of the 4,725 user reviews for this game are positive.\n",
"Overwhelmingly Positive 95% of the 96,176 user reviews for this game are positive.\n",
"Overwhelmingly Positive 95% of the 326,563 user reviews for this game are positive.\n"
]
}
],
"source": [
"import requests\n",
"from scrapy.http import TextResponse\n",
"from w3lib.html import remove_tags #---> for removing html tags\n",
"\n",
"r = requests.get('https://store.steampowered.com/search/?filter=topsellers')\n",
"response = TextResponse(r.url, body=r.text, encoding='utf-8')\n",
"nxgames = response.xpath(\"//div[@id='search_resultsRows']/a\")\n",
"\n",
"# get the review_summary of the game\n",
"for g in nxgames:\n",
" y = g.xpath(\".//span[contains(@class, 'search_review_summary')]/@data-tooltip-html\").get()\n",
" try: \n",
" y = y.replace(\"<br>\", \" \")\n",
" except AttributeError:\n",
" y = \"No reviews\"\n",
" print(y)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"None\n",
"None\n",
"15%\n",
"30%\n",
"60%\n",
"None\n",
"67%\n",
"None\n",
"25%\n",
"50%\n",
"None\n",
"None\n",
"33%\n",
"40%\n",
"None\n",
"35%\n",
"33%\n",
"None\n",
"None\n",
"50%\n",
"33%\n",
"60%\n",
"20%\n",
"None\n",
"65%\n",
"30%\n",
"None\n",
"33%\n",
"50%\n",
"None\n",
"80%\n",
"90%\n",
"25%\n",
"65%\n",
"70%\n",
"25%\n",
"50%\n",
"77%\n",
"70%\n",
"20%\n",
"75%\n",
"67%\n",
"67%\n",
"50%\n",
"None\n",
"25%\n",
"40%\n",
"67%\n",
"75%\n",
"20%\n"
]
}
],
"source": [
"import requests\n",
"from scrapy.http import TextResponse\n",
"\n",
"r = requests.get('https://store.steampowered.com/search/?filter=topsellers')\n",
"response = TextResponse(r.url, body=r.text, encoding='utf-8')\n",
"nxgames = response.xpath(\"//div[@id='search_resultsRows']/a\")\n",
"\n",
"# get the discount_rate of the game\n",
"for g in nxgames:\n",
" y = g.xpath(\".//div[contains(@class, 'col search_discount responsive_secondrow')]/span/text()\").get()\n",
" if y:\n",
" y = y.lstrip('-')\n",
" print(y)\n",
" else:\n",
" print(y)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"None\n",
"Rp 1 025 000\n",
"Rp 759 000\n",
"None\n",
"None\n",
"None\n",
"None\n",
"None\n",
"Rp 729 000\n",
"None\n",
"Rp 729 000\n",
"None\n",
"Rp 1 029 000\n",
"Rp 530 000\n",
"Rp 220 000\n",
"None\n",
"Rp 259 999\n",
"None\n",
"None\n",
"Rp 619 000\n",
"None\n",
"Rp 369 999\n",
"None\n",
"None\n",
"None\n",
"None\n",
"Rp 849 000\n",
"Rp 619 000\n",
"Rp 139 999\n",
"None\n",
"Rp 879 000\n",
"None\n",
"Rp 729 000\n",
"Rp 239 000\n",
"Rp 399 000\n",
"Rp 249 999\n",
"None\n",
"None\n",
"Rp 119 999\n",
"Rp 440 278\n",
"Rp 379 999\n",
"Rp 350 999\n",
"None\n",
"Rp 549 999\n",
"None\n",
"None\n",
"Rp 849 000\n",
"None\n",
"None\n",
"Rp 859 000\n"
]
}
],
"source": [
"import requests\n",
"from scrapy.http import TextResponse\n",
"\n",
"r = requests.get('https://store.steampowered.com/search/?filter=topsellers')\n",
"response = TextResponse(r.url, body=r.text, encoding='utf-8')\n",
"nxgames = response.xpath(\"//div[@id='search_resultsRows']/a\")\n",
"\n",
"# get the discount price (only) of the game\n",
"for g in nxgames:\n",
" y = g.xpath(\".//div[contains(@class,'col search_price discounted')]/span/strike/text()\").get()\n",
" print(y)"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"No info\n",
"No info\n",
"Free to Play\n",
"No info\n",
"Free to Play\n",
"No info\n",
"No info\n",
"Rp 249 999\n",
"Free to Play\n",
"No info\n",
"No info\n",
"Free to Play\n",
"No info\n",
"No info\n",
"Free To Play\n",
"No info\n",
"Free To Play\n",
"No info\n",
"No info\n",
"No info\n",
"Rp 439 999\n",
"No info\n",
"No info\n",
"No info\n",
"No info\n",
"No info\n",
"No info\n",
"No info\n",
"No info\n",
"Rp 749 000\n",
"No info\n",
"No info\n",
"No info\n",
"No info\n",
"No info\n",
"Free to Play\n",
"No info\n",
"No info\n",
"Rp 245 999\n",
"No info\n",
"No info\n",
"No info\n",
"No info\n",
"No info\n",
"No info\n",
"No info\n",
"No info\n",
"No info\n",
"No info\n",
"No info\n"
]
}
],
"source": [
"import requests\n",
"from scrapy.http import TextResponse\n",
"\n",
"r = requests.get('https://store.steampowered.com/search/?filter=topsellers')\n",
"response = TextResponse(r.url, body=r.text, encoding='utf-8')\n",
"nxgames = response.xpath(\"//div[@id='search_resultsRows']/a\")\n",
"\n",
"# get the no discount price (only) of the game\n",
"for g in nxgames:\n",
" y = g.xpath(\"normalize-space(.//div[@class='col search_price responsive_secondrow']/text())\").get()\n",
" if y:\n",
" print(y)\n",
" else:\n",
" print(\"No info\")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Rp 1 025 000\n",
"Rp 599 000\n",
"Free to Play\n",
"Rp 759 000\n",
"Free to Play\n",
"Rp 640 000\n",
"Rp 249 999\n",
"Rp 199 999\n",
"Free to Play\n",
"Rp 699 999\n",
"Rp 729 000\n",
"Free to Play\n",
"Rp 289 999\n",
"no price info\n",
"Rp 699 000\n",
"Free to Play\n",
"Rp 459 000\n",
"Free to Play\n",
"Rp 729 000\n",
"Rp 108 999\n",
"Rp 879 000\n",
"Rp 798 000\n",
"Rp 149 999\n",
"Free to Play\n",
"Rp 439 999\n",
"Rp 219 999\n",
"Rp 169 999\n",
"Rp 135 999\n",
"Rp 749 000\n",
"Rp 619 000\n",
"Rp 553 999\n",
"Rp 729 000\n",
"Rp 205 000\n",
"Rp 359 999\n",
"Rp 259 999\n",
"Rp 600 000\n",
"Rp 659 000\n",
"Rp 135 999\n",
"Rp 245 999\n",
"Rp 266 000\n",
"Rp 849 000\n",
"Rp 108 999\n",
"Rp 479 000\n",
"Rp 659 000\n",
"Rp 790 000\n",
"Rp 648 999\n",
"Rp 1 029 000\n",
"Rp 169 900\n",
"Rp 249 999\n",
"Rp 800 000\n"
]
}
],
"source": [
"# normalize-space\n",
"# https://developer.mozilla.org/en-US/docs/Web/XPath/Functions/normalize-space\n",
"\n",
"import requests\n",
"from scrapy.http import TextResponse\n",
"import re\n",
"\n",
"def has_numbers(inputString):\n",
" return bool(re.search(r'\\d', inputString))\n",
"\n",
"def has_words(inputString):\n",
" return bool(re.search(r'\\w', inputString))\n",
"\n",
"r = requests.get('https://store.steampowered.com/search/?filter=topsellers')\n",
"response = TextResponse(r.url, body=r.text, encoding='utf-8')\n",
"nxgames = response.xpath(\"//div[@id='search_resultsRows']/a\")\n",
"\n",
"# get all (the original price, free to play, or no price info) of the game\n",
"for g in nxgames:\n",
" y = g.xpath(\".//div[contains(@class,'col search_price discounted')]/span/strike/text()\").get()\n",
" if y:\n",
" print(y)\n",
" else:\n",
" y = g.xpath(\"normalize-space(.//div[@class='col search_price responsive_secondrow']/text())\").get()\n",
" if has_numbers(y):\n",
" print(y)\n",
" elif has_words(y):\n",
" print(\"Free to Play\")\n",
" else:\n",
" print(\"no price info\")\n"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Rp 871 250 \n",
"Rp 419 300 \n",
"Free to play\n",
"Rp 303 600 \n",
"Free to play\n",
"Rp 211 200 \n",
"Free to play\n",
"Rp 149 999 \n",
"Free to play\n",
"Rp 349 999 \n",
"Rp 437 400 \n",
"Rp 194 299 \n",
"Free to play\n",
"no discount info\n",
"Free to play\n",
"Rp 454 350 \n",
"Rp 307 530 \n",
"Free to play\n",
"Rp 546 750 \n",
"Rp 659 250 \n",
"Rp 175 999 \n",
"Rp 84 999 \n",
"Rp 73 029 \n",
"Free to play\n",
"Rp 91 119 \n",
"Rp 119 999 \n",
"Free to play\n",
"Rp 558 600 \n",
"Rp 82 000 \n",
"Free to play\n",
"Rp 364 500 \n",
"Rp 230 650 \n",
"Rp 204 270 \n",
"Rp 77 999 \n",
"Rp 71 999 \n",
"Rp 276 999 \n",
"Rp 60 000 \n",
"Rp 47 599 \n",
"Free to play\n",
"Rp 79 800 \n",
"Rp 25 069 \n",
"Rp 424 500 \n",
"Rp 264 000 \n",
"Rp 329 500 \n",
"Rp 632 000 \n",
"Rp 486 749 \n",
"Rp 42 475 \n",
"Rp 149 999 \n",
"Rp 82 499 \n",
"Rp 586 530 \n"
]
}
],
"source": [
"import requests\n",
"from scrapy.http import TextResponse\n",
"\n",
"def has_numbers(inputString):\n",
" return bool(re.search(r'\\d', inputString))\n",
"def has_words(inputString):\n",
" return bool(re.search(r'\\w', inputString))\n",
"\n",
"r = requests.get('https://store.steampowered.com/search/?filter=topsellers')\n",
"response = TextResponse(r.url, body=r.text, encoding='utf-8')\n",
"nxgames = response.xpath(\"//div[@id='search_resultsRows']/a\")\n",
"\n",
"# get the discount price (only) of the game\n",
"for g in nxgames:\n",
" y = g.xpath(\".//div[@class='col search_price discounted responsive_secondrow']/br/following-sibling::text()\").get()\n",
" z = g.xpath(\"normalize-space(.//div[@class='col search_price responsive_secondrow']/text())\").get()\n",
" if y is not None and has_numbers(y):\n",
" print(y)\n",
" elif has_words(z):\n",
" print(\"Free to play\")\n",
" else:\n",
" print(\"no discount info\")"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Rp 871 250\n",
"Rp 419 300\n"
]
}
],
"source": [
"# Extract price after <br> with \"following-sibling\" (Xpath axes)\n",
"\n",
"import lxml.html\n",
"\n",
"html = \"\"\"\n",
"<div class=\"col search_pdc r_secondrow\">\n",
" <div class=\"col search_price discounted\">\n",
" <span style=\"color: #888888;\"></span><br />\n",
" Rp 871 250\n",
" </div>\n",
"</div>\n",
"<div class=\"col search_pdc r_secondrow\">\n",
" <div class=\"col search_price discounted\">\n",
" <span style=\"color: #888888;\"></span><br />\n",
" Rp 419 300\n",
" </div>\n",
"</div>\n",
"<div class=\"col search_pdc r_secondrow\" >\n",
" <div class=\"col search_discount\"></div>\n",
" <div class=\"col search_price\">\n",
" Free to Play\n",
" </div>\n",
"</div>\n",
"\"\"\"\n",
"\n",
"# Parse the HTML using lxml.html\n",
"doc = lxml.html.fromstring(html)\n",
"\n",
"# Find all elements matching the XPath expression\n",
"elements = doc.xpath('//div[@class=\"col search_price discounted\"]/br/following-sibling::text()')\n",
"\n",
"# Loop through the elements and extract the text\n",
"for element in elements:\n",
" text = element.strip()\n",
" print(text)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Rp 871 250 \n",
"Rp 419 300 \n",
"Free to play\n",
"Rp 303 600 \n",
"Free to play\n",
"Rp 211 200 \n",
"Free to play\n",
"Rp 149 999 \n",
"Free to play\n",
"Rp 349 999 \n",
"Rp 437 400 \n",
"Free to play\n",
"Rp 194 299 \n",
"no discount info\n",
"Free to play\n",
"Rp 454 350 \n",
"Rp 307 530 \n",
"Free to play\n",
"Rp 546 750 \n",
"Rp 659 250 \n",
"Rp 73 029 \n",
"Rp 175 999 \n",
"Free to play\n",
"Rp 91 119 \n",
"Rp 84 999 \n",
"Rp 119 999 \n",
"Rp 558 600 \n",
"Free to play\n",
"Rp 82 000 \n",
"Rp 364 500 \n",
"Free to play\n",
"Rp 230 650 \n",
"Rp 276 999 \n",
"Rp 204 270 \n",
"Rp 77 999 \n",
"Rp 71 999 \n",
"Rp 60 000 \n",
"Rp 47 599 \n",
"Free to play\n",
"Rp 79 800 \n",
"Rp 25 069 \n",
"Rp 264 000 \n",
"Rp 424 500 \n",
"Rp 329 500 \n",
"Rp 632 000 \n",
"Rp 486 749 \n",
"Rp 149 999 \n",
"Rp 82 499 \n",
"Rp 586 530 \n",
"Rp 42 475 \n"
]
}
],
"source": [
"# Extract price after <br> with text()[2]\n",
"\n",
"import requests\n",
"from scrapy.http import TextResponse\n",
"\n",
"def has_numbers(inputString):\n",
" return bool(re.search(r'\\d', inputString))\n",
"def has_words(inputString):\n",
" return bool(re.search(r'\\w', inputString))\n",
"\n",
"r = requests.get('https://store.steampowered.com/search/?filter=topsellers')\n",
"response = TextResponse(r.url, body=r.text, encoding='utf-8')\n",
"nxgames = response.xpath(\"//div[@id='search_resultsRows']/a\")\n",
"\n",
"# get the discount price (only) of the game\n",
"for g in nxgames:\n",
" y = g.xpath(\".//div[@class='col search_price discounted responsive_secondrow']/text()[2]\").get()\n",
" z = g.xpath(\"normalize-space(.//div[@class='col search_price responsive_secondrow']/text())\").get()\n",
" if y is not None and has_numbers(y):\n",
" print(y)\n",
" elif has_words(z):\n",
" print(\"Free to play\")\n",
" else:\n",
" print(\"no discount info\")"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Rp 871 250\n",
"Rp 419 300\n"
]
}
],
"source": [
"# Extract price after <br> with BeautifulSoup\n",
"\n",
"from bs4 import BeautifulSoup\n",
"\n",
"html = \"\"\"\n",
"<div class=\"col search_pdc r_secondrow\">\n",
" <div class=\"col search_price discounted\">\n",
" <span style=\"color: #888888;\"></span><br />\n",
" Rp 871 250\n",
" </div>\n",
"</div>\n",
"<div class=\"col search_pdc r_secondrow\">\n",
" <div class=\"col search_price discounted\">\n",
" <span style=\"color: #888888;\"></span><br />\n",
" Rp 419 300\n",
" </div>\n",
"</div>\n",
"<div class=\"col search_pdc r_secondrow\" >\n",
" <div class=\"col search_discount\"></div>\n",
" <div class=\"col search_price\">\n",
" Free to Play\n",
" </div>\n",
"</div>\n",
"\"\"\"\n",
"\n",
"soup = BeautifulSoup(html, 'html.parser')\n",
"\n",
"for div in soup.find_all('div', {'class': 'col search_price discounted'}):\n",
" text = div.text.strip().split('\\n')[-1]\n",
" print(text)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [],
"source": [
"# an example of using an ItemLoader in Python, including using a Selector to extract data from an HTML response and using a loop to load multiple items, using a working URL:\n",
"\n",
"import scrapy\n",
"from scrapy.loader import ItemLoader\n",
"from scrapy.item import Item, Field\n",
"from scrapy.selector import Selector\n",
"\n",
"class MyItem(Item):\n",
" # define the fields for your item here like:\n",
" name = Field()\n",
" description = Field()\n",
"\n",
"# create a Scrapy Spider class to handle the request and response\n",
"class MySpider(scrapy.Spider):\n",
" name = 'myspider'\n",
" start_urls = ['https://www.imdb.com/chart/top/?ref_=nv_mv_250']\n",
"\n",
" def parse(self, response):\n",
" # use the Selector to extract data from the HTML response\n",
" sel = Selector(response)\n",
"\n",
" # loop through all the items on the page\n",
" for item_sel in sel.xpath('//tbody[@class=\"lister-list\"]/tr'):\n",
" # create an instance of the ItemLoader\n",
" loader = ItemLoader(item=MyItem(), selector=item_sel)\n",
"\n",
" # load data into the Item using the various add_* methods\n",
" # you can use add_xpath to extract data using an XPath expression\n",
" loader.add_xpath('name', './/td[@class=\"titleColumn\"]/a/text()')\n",
" loader.add_xpath('description', './/td[@class=\"ratingColumn imdbRating\"]/strong/text()')\n",
"\n",
" # once all data is loaded, you can extract the Item\n",
" item = loader.load_item()\n",
" yield loader.load_item()\n",
"\n",
" # you can then access the data of the Item like a dictionary\n",
" print(item['name'])\n",
" print(item['description'])\n",
"\n",
"# Run using \"scrapy runspider name_of_the_file.py\""
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://store.steampowered.com/search/?sort_by=&sort_order=0&filter=topsellers&supportedlang=english&page=2\n"
]
}
],
"source": [
"#Next page testing\n",
"\n",
"import requests\n",
"from scrapy.http import TextResponse\n",
"\n",
"r = requests.get('https://store.steampowered.com/search/?filter=topsellers')\n",
"response = TextResponse(r.url, body=r.text, encoding='utf-8')\n",
"next_page = response.xpath(\"//a[@class='pagebtn' and text()='>']/@href\").get()\n",
"\n",
"print(next_page)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Hello, World!\n",
"Hello, World!\n",
"default_value\n",
"None\n"
]
}
],
"source": [
"# In Scrapy, the .get() and .extract_first() methods are both used to extract data from a Selector object. \n",
"# The main difference between these two methods is that .get() returns a default value if the element is not found, \n",
"# while .extract_first() returns None if the element is not found.\n",
"\n",
"from scrapy.selector import Selector\n",
"\n",
"html = '<html><body><p>Hello, World!</p></body></html>'\n",
"\n",
"# Create a Selector object\n",
"sel = Selector(text=html)\n",
"\n",
"# Extract the text of the first <p> element using .get()\n",
"text = sel.xpath('//p/text()').get()\n",
"print(text) # Output: \"Hello, World!\"\n",
"\n",
"# Extract the text of the first <p> element using .extract_first()\n",
"text = sel.xpath('//p/text()').extract_first()\n",
"print(text) # Output: \"Hello, World!\"\n",
"\n",
"# Extract the text of a non-existent element using .get()\n",
"text = sel.xpath('//div/text()').get('default_value')\n",
"print(text) # Output: \"default_value\"\n",
"\n",
"# Extract the text of a non-existent element using .extract_first()\n",
"text = sel.xpath('//div/text()').extract_first()\n",
"print(text) # Output: None\n"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"no price info\n",
"no price info\n",
"Rp 1 025 000\n",
"Rp 599 000\n",
"Rp 759 000\n",
"no price info\n",
"Rp 640 000\n",
"no price info\n",
"Rp 199 999\n",
"Rp 699 999\n",
"no price info\n",
"no price info\n",
"Rp 289 999\n",
"Rp 729 000\n",
"no price info\n",
"Rp 699 000\n",
"Rp 459 000\n",
"no price info\n",
"no price info\n",
"Rp 169 999\n",
"Rp 219 999\n",
"Rp 135 999\n",
"Rp 205 000\n",
"no price info\n",
"Rp 659 000\n",
"no price info\n",
"Rp 798 000\n",
"Rp 600 000\n",
"Rp 729 000\n",
"no price info\n",
"Rp 108 999\n",
"Rp 359 999\n",
"Rp 729 000\n",
"Rp 259 999\n",
"Rp 135 999\n",
"Rp 266 000\n",
"Rp 553 999\n",
"Rp 879 000\n",
"Rp 108 999\n",
"Rp 169 900\n",
"Rp 149 999\n",
"Rp 619 000\n",
"Rp 800 000\n",
"no price info\n",
"Rp 659 000\n",
"Rp 648 999\n",
"Rp 115 999\n",
"Rp 619 000\n",
"Rp 220 999\n",
"Rp 89 999\n"
]
}
],
"source": [
"import requests\n",
"from scrapy.http import TextResponse\n",
"import re\n",
"\n",
"def has_numbers(inputString):\n",
" return bool(re.search(r'\\d', inputString))\n",
"\n",
"def has_words(inputString):\n",
" return bool(re.search(r'\\w', inputString))\n",
"\n",
"r = requests.get('https://store.steampowered.com/search/?filter=topsellers')\n",
"response = TextResponse(r.url, body=r.text, encoding='utf-8')\n",
"nxgames = response.xpath(\"//div[@id='search_resultsRows']/a\")\n",
"\n",
"# get all (the original price, free to play, or no price info) of the game\n",
"for g in nxgames:\n",
" y = g.xpath(\".//div[contains(@class,'col search_price discounted')]/span/strike/text()\").get()\n",
" if y:\n",
" print(y)\n",
" else:\n",
" print(\"no price info\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.15"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment