This is to generate OSS tracker csv in the node root directory

oss_tracker_update.js

"use strict"

/*
    ********* Steps to Use this script in node projects
    1. copy this script to oss_tracker_update.js and place it the root directory
    2. replace "<GIT-USERNAME>" & "<GIT-PASSWORD>" with your git name and pwd(This is because, there is a limit to github api, If you use credentials we can call 5 K api calls / hour)
    3. run node oss_tracker_update.js
    4. you will get ossTracker.csv in the same directory
*/

const path = require("path");
const fs = require("fs");
// Need to install axios, Ramda and scrape-it NPM packages first
const axios = require("axios")
const R = require("ramda")

const packagejson = require("./package.json");
const packageLockjson = require("./package-lock.json");

// Right now only dependencies were considered, If you want dev - dependencies also change dependentKey to devDependencies
const dependentKey = "dependencies" // "dependencies" or "devDependencies"

// Fill gituser name and password
const gituserName = "<GIT-USERNAME>"
const gitPassword = "<GIT-PASSWORD>"

const dir = path.resolve(__dirname)
const outputFile = path.join(dir, "ossTracker.csv")

// Headers for csv
const headers = ["package_name", "stars", "license", "mentioned_version", "current_version", "homepage", "repository", "forks", "description", "last_updated"]

const getNPMDetails = async (packageName) => {
  try {
    const npmApiURL = "https://registry.npmjs.org/" + packageName
    // console.log(npmApiURL)
    const response = await axios.get(npmApiURL);
    return response.data

  } catch (error) {
    console.log("NPM error", error);
  }
}

const getGitrepoDetails = async (url) => {
  try {
    const githubURL = R.split("/", url)
    const username = githubURL[3]
    const repoName = githubURL[4]
    if (username && repoName) {
      const gitApiURL = "https://api.github.com/repos/" + username + "/" + repoName
      // console.log(gitApiURL)
      const response = await axios.get(gitApiURL, {
        auth: {
          username: gituserName,
          password: gitPassword
        }
      });
      return response.data
    } else {
      return {}
    }
  } catch (error) {
    console.log("Github error", error);
  }
}

const generateOSSTracker = async (inputObj) => {
  await R.mapObjIndexed(async (version, packageName, o) => {
    const data = await getNPMDetails(packageName)
    if (data) {
      const repodetails = {}

      repodetails.license = data.license
      repodetails.description = data.description ? R.replace(/\,/g, "", data.description) : data.description
      repodetails.homepage = data.homepage

      if (R.path(["repository", "type"], data) === "git") {
        repodetails.repository = R.pipe(
          R.replace(".git", ""),
          R.replace("git+", ""),
          R.replace("ssh://", "https://"),
          R.replace("git://", "https://"),
          R.replace("git@github.com", "github.com")
        )(R.path(["repository", "url"], data))
      }

      repodetails["mentioned_version"] = version
      repodetails["package_name"] = packageName
      repodetails["current_version"] = R.path(["dependencies", packageName, "version"], packageLockjson)
      repodetails["latest_version"] = repodetails["version"]
      if (repodetails.repository) {
        const githubDetails = await getGitrepoDetails(repodetails.repository)
        if (githubDetails) {
          repodetails["forks"] = githubDetails.forks_count
          repodetails["stars"] = githubDetails.stargazers_count
          repodetails["last_updated"] = githubDetails.updated_at
          const desc = githubDetails.description
          repodetails["description"] = desc ? R.replace(/\,/g, "", desc) : desc
          repodetails["license"] = repodetails["license"] ? repodetails["license"] : R.path(["license", "name"], githubDetails)
        }
      }
      //   console.log(repodetails)

      let line = R.map((x) => {
        return repodetails[x]
      }, headers)

      line = R.concat(R.join(",", line), "\r\n")

      fs.appendFile(outputFile, line, function (err) {
        if (err) {
          console.log("Error: ", err)
        } else {
          //   console.log("...")
        }
      })
    }
  }, inputObj)
}


fs.truncate(outputFile, 0, function () {
  fs.appendFile(outputFile, R.concat(R.join(",", headers), "\r\n"), function (err) {
    if (err) {
      console.log("Error: ", err)
    } else {
      generateOSSTracker(packagejson[dependentKey])
    }
  })
})