A node.js script to grab the beatsaver.com maps and playlists

.gitignore

maps.json
playlists.json

index.js

const startTime = Date.now();
const fs = require("fs");

// Endpoints
const mapsEndpoint = "https://api.beatsaver.com/maps/latest";
const playlistsEndpoint = "https://api.beatsaver.com/playlists/latest";

/**
 * Formats seconds as hours, minutes, and seconds.
 * @param {number} seconds 
 * @returns 
 */
function formatSeconds(seconds) {
  // Calculate hours, minutes, and remaining seconds
  const hours = Math.floor(seconds / 3600);
  const minutes = Math.floor((seconds % 3600) / 60);
  const remainingSeconds = seconds % 60;

  // Format the result
  const formattedTime = `${hours}h ${minutes}m ${remainingSeconds}s`;

  return formattedTime;
}

/**
 * Attempts to parse the specified file as json returning the specified default value if an error occurs.
 * @param {string} filePath The file path to read and parse.
 * @param {any} defaultValue The value to return if there's an error parsing the file. 
 * @returns
 */
async function readJson(filePath, defaultValue = undefined) {
  return new Promise((resolve, reject) => {
    fs.readFile(filePath, 'utf8', (err, data) => {
      if (err) {
        resolve(defaultValue);
      } else {
        try {
          let parsedData = JSON.parse(data);
          resolve(parsedData);
        } catch {
          resolve(defaultValue);
        }
      }
    });
  })
}

/**
 * Writes text as a utf-8 encoded file.
 * @param {string} filename The filename to write.
 * @param {string} text The text to write.
 * @returns {Promise<void>}
 */
function writeTextToFile(filename, text) {
  return new Promise((resolve, reject) => {
    fs.writeFile(filename, text, 'utf8', (err) => {
      if (err) {
        reject(err);
      } else {
        resolve();
      }
    });
  });
}

/**
 * Gets the content-length of a remote URL.
 * @param {string} url The URL.
 * @returns {number|null} The content length as provided in the response header, or null if not present.
 */
async function getContentLength(url) {
  try {
    const response = await fetch(url, { method: 'HEAD' });

    if (!response.ok) {
      throw new Error(`HTTP error! status: ${response.status}`);
    }

    const contentLength = response.headers.get('Content-Length');

    if (contentLength) {
      return parseInt(contentLength, 10);
    } else {
      console.warn('Content-Length header not found in response');
      return null;
    }
  } catch (error) {
    console.error('Error fetching content length:', error);
    return null;
  }
}

/**
 * Fetches a URL and returns it as a JSON-parsed object.
 * @param {string} url The URL.
 * @returns 
 */
async function getJson(url) {
  console.log(`Fetching: ${url}`);

  const response = await fetch(url);
  return await response.json();
}

/**
 * 
 * @param {string} endpoint The API endpoint.
 * @param {number} perPage 1 - 100
 * @param {string} before You probably want this. Supplying the uploaded time of the last item in the previous page will get you another page. YYYY-MM-DDTHH:MM:SS+00:00
 * @param {string} after Like before but will get you data more recent than the time supplied. YYYY-MM-DDTHH:MM:SS+00:00
 */
async function getDatedData(endpoint, perPage, before, after) {
  perPage = perPage || 100;
  let params = {
    pageSize: perPage,
    sort: "UPDATED"
  };

  if (before != null) {
    params.before = before;
  }

  if (after != null) {
    params.after = after;
  }

  return await getJson(`${endpoint}?${new URLSearchParams(params).toString()}`)
}

/**
 * 
 * @param {string} endpoint The API endpoint.
 * @param {string | null} after The date to fetch until, or all data. This will get you data more recent than the time supplied. YYYY-MM-DDTHH:MM:SS+00:00
 */
async function getAllDatedDataAfter(endpoint, after) {
  let items = [];
  let lastDate = null;

  mainLoop:
  while (true) { // Loop until break.  In a perfect world this would have some kind of cancellation token.
    let data = await getDatedData(endpoint, 100, lastDate, after);

    if (data.docs.length == 0) {
      break mainLoop;
    }

    itemLoop:
    for (var i = 0; i < data.docs.length; i++) {
      let doc = data.docs[i];
      lastDate = doc.updatedAt;

      if (after != null && after == lastDate) {
        break mainLoop;
      }

      items.push(doc);
    }
  }

  return items;
}

/**
 * Gets the details for the specified playlist
 * @param {number} id The ID of the playlist
 * @param {number} page The specific page to fetch, if omitted it will fetch all maps.
 * @returns 
 */
async function getPlaylistDetails(id, page) {
  if (page == null) {
    let page = 0;
    let playlist = await getPlaylistDetails(id, page++);

    if (playlist.maps.length == 0) {
      return playlist;
    }

    while (true) {
      let data = await getPlaylistDetails(id, page++);

      playlist.maps = [...playlist.maps, ...data.maps];

      if (data.maps.length == 0) {
        break;
      }
    }

    return playlist;
  } else {
    return await getJson(`https://api.beatsaver.com/playlists/id/${id}/${page}`);
  }
}

/**
 * Processes the endpoints and saves the data to json files.
 * @param {string} endpoint The endpoint to process.
 * @param {string} filename The json filename to read/write.
 */
async function processEndpoint(endpoint, filename) {
  let oldData = await readJson(filename, []);
  let newData = await getAllDatedDataAfter(endpoint, Object.values(oldData).length > 0 ? Object.values(oldData)[0].updatedAt : null);

  if (endpoint == playlistsEndpoint) {
    for (var i = 0; i < newData.length; i++) {
      let doc = newData[i];
      let details = await getPlaylistDetails(doc.playlistId);

      doc.maps = details.maps;
    }
  }

  let newKeys = newData.map((doc => doc.id || doc.playlistId));
  oldData.forEach(doc => {
    if (newKeys.indexOf(doc.id || doc.playlistId) == -1) {
      newData.push(doc);
    }
  });
  await writeTextToFile(filename, JSON.stringify(newData, null, "\t"));
}

// Async wrapper to use await.
(async () => {
  await processEndpoint(mapsEndpoint, "./maps.json");
  await processEndpoint(playlistsEndpoint, "./playlists.json");

  console.log(`Process finished in ${formatSeconds((Date.now() - startTime) / 1000)}`)
})();

package.json

{
  "name": "beatsaver-data-grabber",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "",
  "license": "ISC"
}