Data collection for Appbase search

github.js

const got = require('got');

const AUTH_KEY = ''

;(async () => {
  try {
    const { data, headers } = await got.post({
      url: 'https://api.github.com/graphql',
      headers: { Authorization: `bearer ${AUTH_KEY}` },
      json: {
        query: `
          query {
            repository(name: "reactivesearch", owner: "appbaseio") {
              discussions(orderBy: {field: UPDATED_AT, direction: DESC}, first: 100) {
                edges {
                  node {
                    bodyText
                    answer {
                      bodyText
                      url
                      createdAt
                    }
                    url
                    createdAt
                    category {
                      name
                    }
                    labels {
                      nodes {
                        name
                      }
                    }
                    upvoteCount
                  }
                  cursor
                }
                totalCount
                pageInfo {
                  hasNextPage
                  endCursor
                }
              }
            }
          }
        `
      },
    }).json();

    const discussions = (data?.repository?.discussions?.edges || []).filter(({node:_q} = {}) => _q?.answer)
    console.log(discussions.length);
    if (discussions?.pageInfo?.hasNextPage) {
      // make request with discussions?.pageInfo?.endCursor as after:
      // can use discussions?.totalCount / 100 to get pages too
    }

    const normalised_data = discussions.map(({ node: _q }) => {
      return {
        "question": _q.bodyText,
        "accepted_answer": _q.answer?.bodyText,
        "url": _q.url,
        "date_of_question": Date.parse(_q.createdAt?.replace(/ /g, '')),
        "date_of_answer": Date.parse(_q.answer?.createdAt?.replace(/ /g, '')),
        "source": "github",
        "tags": [_q.category.name],
        // concat _q.labels.nodes[].name later
        "upvotes": _q.upvoteCount,
      }
    })

    console.log(normalised_data[0])
    // Use the normalised_data for indexing
  } catch (error) {
    console.log(error);
  }
})();

stackoverflow.js

const axios = require('axios');

const FILTER = '1Yh*9Ph7pvJcDf3gv*oO6K-I_EHpCF-x)WLoa'
const ACCESS_TOKEN = ''
// how to get access_token -> https://stackoverflow.com/a/62605055/2130750
const KEY = ''

;(async () => {
  try {
    const data = []
    let page = 1
    let has_more = true
    while(has_more) {
      const { data: _data } = await axios.get(`https://api.stackexchange.com/2.3/search/advanced?page=${page}&order=desc&sort=activity&accepted=True&tagged=ReactiveSearch&site=stackoverflow&filter=${FILTER}&access_token=${ACCESS_TOKEN}&key=${KEY}`)
      console.log(page, _data?.has_more)
      data.push(...(_data?.items || []))
      has_more = _data?.has_more
      page += 1
    }
    console.log(data.length)
    const normalised_data = data.map(_q => {
      const accepted_answer = _q.answers.filter(_a => _a.answer_id === _q.accepted_answer_id)[0]
      return {
        "question": _q.body_markdown,
        "accepted_answer": accepted_answer?.body_markdown,
        "url": _q.link,
        "date_of_question": _q.creation_date,
        "date_of_answer": accepted_answer.creation_date,
        "source": "stackoverflow",
        "tags": _q.tags,
        "upvotes": _q.up_vote_count,
      }
    })

    console.log(normalised_data[0])
    // Use the normalised_data for indexing
  } catch (error) {
    console.log(error);
  }
})();