jpchagas

jpchagas

import os
import pandas as pd

data_path = os.getcwd() + "/data/instagram_comments/"
dfs = []
for file in files:
    current_df = pd.read_csv(data_path + file)
    dfs.append(current_df)
final_df = pd.concat(dfs)
final_selected_df = final_df[['Unnamed: 1','Likes','Comment','Profile URL','Comment URL']]

jpchagas

import scrapy


class PensadorSpider(scrapy.Spider):
    name = "pensador"
    allowed_domains = ["www.pensador.com"]
    start_urls = ["https://www.pensador.com/autor/machado_de_assis/"]

    def parse(self, response):
        author = response.xpath("//*[@id='content']/div[1]/h1/text()").get()

jpchagas

import scrapy 

class PensadorCrawlerItem(scrapy.Item):
    author = scrapy.Field()
    sentence = scrapy.Field() 
    book_title = scrapy.Field()
    share_amount = scrapy.Field()

jpchagas

import scrapy
from items import PensadorCrawlerItem

class PensadorSpider(scrapy.Spider):
    name = "pensador"
    allowed_domains = ["www.pensador.com"]
    start_urls = ["https://www.pensador.com/autor/machado_de_assis/"]

    def parse(self, response):
        pensador_item = PensadorCrawlerItem()

jpchagas

from itemadapter import ItemAdapter


class PensadorCrawlerPipeline:
    def process_item(self, item, spider):
        return item

jpchagas

jpchagas

import pandas as pd

imdb_df['text_pt'] = imdb_df['text_pt'].str.lower()
instagram_df['comment'] = instagram_df['comment'].str.lower()

jpchagas

import pandas as pd

imdb_df['classification']= imdb_df['sentiment'].replace(["neg","pos"],[0,1])

jpchagas

import emoji

comment_demojize = []
for i in range(len(instagram_df)):
    if type(instagram_df['comment'][i]) is str:
        comment_demojize.append(emoji.demojize(instagram_df['comment'][i], language='pt').replace("::", " ").replace(":", "").replace("_", " "))
    else:
        comment_demojize.append(instagram_df['comment'][i])
instagram_df['comment'] = comment_demojize