Transform the opawg Podcast User Agent list into a jsonl file for BigQuery

README.md

1. Generate a jsonl file with all the Podcast User Agents.

    node agents.js
   

2. Use the BigQuery console to create a table and load the podcast-user-agents.jsonl file into the new table.

agents.js

'use strict';

const fs = require('fs');
const https = require('https');

let results = [];

const url =
  'https://raw.githubusercontent.com/opawg/user-agents/master/src/user-agents.json';

https.get(url, (res) => {
  res.setEncoding('utf8');
  let body = '';
  res.on('data', (data) => {
    body += data;
  });
  res.on('end', () => {
    for (let ua of JSON.parse(body)) {
      for (let regex of ua.user_agents) {
        results.push({
          regex: regex,
          os: ua.os || null,
          device: ua.device || null,
          app: ua.app || null,
          bot: ua.bot || false,
        });
      }
    }

    fs.writeFileSync(
      'podcast-user-agents.jsonl',
      results.reduce((previous, current) => {
        return previous + JSON.stringify(current) + '\n';
      }, '')
    );
  });
});