index.js

import needle from 'needle'
import Q from 'q'
import cheerio from 'cheerio'
import he from 'he'
import queryString from 'query-string'
import { CronJob } from 'cron'
import { MongoClient } from 'mongodb'
import redis from 'redis'
import getFirstPhone from './getFirstPhone'
import getDistrict from './getDistrict'
import getMoney from './getMoney'
import getLocation from './getLocation'
import fbUser from './fbUser'
import isTrash from './isTrash'

(async () => {
  const {
    COOKIE,
    GID,
    PUBLISHER,
    MONGO_URL,
  } = JSON.parse(process.env.SETTINGS)
  const R = redis.createClient()
  const db = await MongoClient.connect(MONGO_URL)
  const Feeds = db.collection('feeds')
  const FBUsers = db.collection('fbUsers')
  const options = {
    headers: {
      userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
      cookie: COOKIE,
    },
  }
  const main = async () => {
    console.time('GET')
    const gURL = `https://mbasic.facebook.com/groups/${GID}`
    const responses = await Q.nfcall(needle.get, gURL, options)
    console.timeEnd('GET')
    const { body, statusCode } = responses[0]
    if (statusCode !== 200) throw Error('User is invalid')
    const feeds = []
    const articles = cheerio.load(he.decode(body))('#m_group_stories_container').children().eq(0).children()
    for (let i = 0; i < articles.length; i++) {
      const article = articles.eq(i)
      const justNow = article.find('abbr').text() === 'Just now' || article.find('abbr').text() === 'Vừa xong'
      console.log(article.find('abbr').text())
      if (justNow) {
        const actionDiv = article.children().eq(1).children().eq(1).children().eq(0)
        const imageDiv = article.children().eq(0).children().eq(2)
        const fbUser = article.children().eq(0).children().eq(0).find('a').eq(0)
        const fbLink = article.children().eq(0).children().eq(0).find('a').eq(1)
        let contentDiv = article.children().eq(0).children().eq(1)
        for (let j = 2; j < 10; j++) {
          if (contentDiv.text().length) break;
          contentDiv = article.children().eq(0).children().eq(j)
	}
        contentDiv.find('br').replaceWith('\n')
        const ptags = contentDiv.find('p')
        for (let j = 0; j < ptags.length; j++) {
          const p = ptags.eq(j)
          const text = p.text()
          p.replaceWith(`${text}\n`)
        }
        const message = contentDiv
          .text()
          .replace('See Translation', '')
          .replace('(Sold)', '')
          .replace('(Đã bán)', '')
          .replace('Thành phố Hồ Chí Minh', '')
          .replace('Hồ Chí Minh', '')
          .replace('MIỄN PHÍ', ' ')
          .replace('Vietnam', '')
          .replace('Thành phố Sài Gòn')
          .replace(/(\d+\.?)+ ₫/, '')
          .replace(/\n$/, '')
          .replace('Contact Seller', ' ')
	  .replace('Liên hệ với người bán', '')
        const name = fbUser.text()
        const phone = getFirstPhone(message) || ''
        const link = decodeURIComponent(fbLink.attr('href'))
        const images = imageDiv.find('img')
        if (link && !isTrash(message)) {
          const feedId = `${GID}_${actionDiv.attr('id').replace('like_', '')}`
          const vanity = ''
          let fbId = "000001";
	  try {
            fbId = link.split('content_owner_id_new.')[1].split(':')[0];
	  } catch(e) {}
          const feed = {}
          const hasKey = await Q.nfcall(R.get.bind(R), feedId)
          if (hasKey) continue
          let { lat, lng } = getLocation(message)
          if (lat === 0 && lng === 0) {
            const nLoc = getDistrict(message)
            lat = nLoc.lat
            lng = nLoc.lng
          }
          const { fee, cod } = getMoney(message)
          for (let j = 0; j < images.length; j++) {
            const image = images.eq(j).attr('src')
            if (image.includes('fbcdn')) {
              feed.image = image
            }
          }
          if (fbId) {
            feed.fbId = fbId || ''
          } else {
            feed.vanity = vanity || ''
          }
          feeds.push(Object.assign({
            name,
            message,
            createdAt: Date.now(),
            feedId,
            phone,
            lat,
            lng,
            fee,
            cod,
          }, feed))
          R.set(feedId, true)
        }
      }
    }
    console.log(feeds)
    if (feeds.length) {
      needle.post(PUBLISHER, { orders: feeds }, { json: true })
      Feeds.insertMany(feeds)
      fbUser(FBUsers, feeds, COOKIE)
    }
  }
  new CronJob('* * * * * *', async() => {
    await main()
  }, null, true, 'America/Los_Angeles')
})()