This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
function is_valid_range_(r) { | |
return (r.length > 0 && r[0].length === 2); | |
} | |
function get_range_values_(r) { | |
var values = [] | |
for(var i=0;i<r.length;i++) { | |
values.push({ text: r[i][0].trim(), count: r[i][1] }); | |
} | |
return values; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
sitemap_index_url="https://www.searchenginejournal.com/sitemap_index.xml" | |
from bs4 import BeautifulSoup | |
import requests | |
sitemap_index = {} | |
r = requests.get(sitemap_index_url) | |
xml = r.text |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
sitemaps = {} | |
for (sitemap_url, lasmod) in sitemap_index.items(): | |
if(sitemap_url.find("post") > 0): | |
print(sitemap_url) | |
if 1: # for testing | |
r = requests.get(sitemap_url) | |
xml = r.text |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas as pd | |
print(pd.__version__) #should be 0.23 or later | |
df = pd.DataFrame.from_dict(sitemaps, orient="index", columns=['lastmod']) | |
df.head(10) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from collections import Counter | |
import re | |
import nltk | |
from nltk.corpus import stopwords | |
nltk.download('stopwords') | |
from urllib.parse import urlparse |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
cnt=Counter() | |
english_stopwords = set(stopwords.words('english')) | |
for path in df.path: | |
words = re.split("[-/]", path) | |
for word in words: | |
if len(word) > 0 and word not in english_stopwords and not word.isdigit(): | |
cnt[word] += 1 | |
cnt.most_common(25) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator | |
import matplotlib.pyplot as plt | |
word_cloud = [x[0] for x in cnt.most_common(25)] | |
word_cloud_obj = WordCloud(max_words=25, background_color="white").generate(" ".join(word_cloud)) | |
#word_cloud_obj = WordCloud().generate(" ".join(word_cloud)) #default with ugly black background | |
plt.imshow(word_cloud_obj, interpolation='bilinear') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def get_category(path): | |
words = re.split("[-/]", path) | |
for word in words: | |
if word in word_cloud: | |
return word | |
return "other" | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
google_df = df[df["category"] == "google"] | |
first = google_df[:1000] | |
second = google_df[1000:2000] | |
third = google_df[2000:3000] | |
last = google_df[3000:] | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
high_value_pages=df[df.path.str.contains("adwords|facebook|ads|media", regex=True)] | |
import numpy as np | |
high_value_pages["fake_transactions"]=np.random.randint(1, 200, high_value_pages.shape[0]) | |
high_value_pages=high_value_pages.reset_index() | |
fake_transaction_pages=high_value_pages[["path", "fake_transactions"]] |
OlderNewer