Download all CC-BY Google.Poly models

getAssets.js

// Step 2: Use the generated lists to fetch the assets.
// Only store glTF V2, metadata, and thumbnail.
// Other formats like OBJ, FBX, glTF V1 are ignored
// No API key needed here
// Checks to see if each file already exists before downloading so if the script gets interrupted just
// restart it and it'll continue where it left off

const fs = require("fs").promises;
const got = require("got");
const path = require("path");
const sanitize = require("sanitize-filename");
const { performance } = require("perf_hooks");

async function createDirectory(pathname) {
  pathname = pathname.replace(/^\.*\/|\/?[^\/]+\.[a-z]+|\/$/g, ""); // Remove leading directory markers, and remove ending /file-name.extension
  await fs.mkdir(path.resolve(pathname), { recursive: true });
}

async function fileOrfolderExists(pathName) {
  try {
    await fs.access(pathName);
    return true;
  } catch {
    return false;
  }
}

async function fetchModel(name, modelPath, assetData) {
  const t0 = performance.now();

  await createDirectory(modelPath);

  if (!(await fileOrfolderExists(`${modelPath}/data.json`))) {
    await fs.writeFile(
      `${modelPath}/data.json`,
      JSON.stringify(assetData, null, 4)
    );
  }

  // Lots of formats are specified, save only glTF V2
  const glTFData = Object.values(assetData.formats).filter(
    (format) => format.formatType === "GLTF2"
  )[0];

  const promises = [];
  const filenames = [];

  if (
    assetData.thumbnail &&
    !(await fileOrfolderExists(`${modelPath}/${"thumbnail.png"}`))
  ) {
    promises.push(got(assetData.thumbnail.url).buffer());
    filenames.push("thumbnail.png");
  }

  if (!(await fileOrfolderExists(`${modelPath}/${"model.gltf"}`))) {
    promises.push(got(glTFData.root.url).buffer());
    filenames.push("model.gltf");
  }

  if (glTFData.resources) {
    for (const resource of glTFData.resources) {
      const filename = resource.relativePath.split("/").pop();

      if (!(await fileOrfolderExists(`${modelPath}/${filename}`))) {
        promises.push(got(resource.url).buffer());
        filenames.push(filename);
      }
    }
  }

  if (filenames.length) {
    console.log(`Fetching model: ${name}`);
  }

  const buffers = await Promise.all(promises).catch((err) =>
    console.error(err)
  );

  for (const [index, filename] of filenames.entries()) {
    await fs
      .writeFile(`${modelPath}/${filename}`, buffers[index])
      .catch((err) => console.error(err));
    console.log(`Wrote file: ${filename}`);
  }

  if (filenames.length) {
    const t1 = performance.now();
    console.log(`Done in: ${((t1 - t0) / 1000).toFixed(4)}s`);
  }
}

async function getModelsFromList(assetsList) {
  for (const assetData of assetsList.assets) {
    const name = sanitize(assetData.displayName);
    
    const id = JSON.parse(rawData).name.split("/").pop();

    const modelPath = path.resolve(`models/${name}_${id}`);

    await fetchModel(name, modelPath, assetData).catch((err) =>
      console.error(err)
    );
  }
}

let page = 0;
const baseFile = "assetLists/assets_";

async function getAllModels() {
  await createDirectory("models");

  const listPath = `${baseFile}${page}.json`;

  // continue loading the asset lists until the last is reached
  if (await fileOrfolderExists(listPath)) {
    console.log(`Loading models from: ${listPath}`);
    const rawData = await fs.readFile(listPath);
    const assetsList = JSON.parse(rawData);

    await getModelsFromList(assetsList).catch((err) => {
      console.error(err);
      console.timeEnd(`Done in: `);
    });
    page++;
    await getAllModels().catch((err) => console.error(err));
  } else {
    console.log("No more files! You did it! :D");
  }
}

getAllModels().catch((err) => console.error(err));

getLists.js

// Step 1: generate the lists. 
// Each fetched list contains 100 models
// These will be saved in assetsLists/assets_0.json, assetsLists/assets_0.json, assetsLists/assets_0.json etc.
// The lists only contain CC-BY models and no tilt brush models
// This will return >800 lists with 100 models in each. Refer here for ways to filter the list:
// https://developers.google.com/poly/reference/api/rest/v1/assets/list

const got = require("got");
const fs = require("fs").promises;

// Get an API key here
// https://developers.google.com/poly/develop/api
const API_KEY = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";
const API_URL = "https://poly.googleapis.com/v1/assets";

let page = 0;

const getModelLists = async (pageToken = "") => {
  const rawdata = await got(
    `${API_URL}?format=GLTF2&key=${API_KEY}&pageToken=${pageToken}&pageSize=100`
  );
  await fs.writeFile(`assetsLists/assets_${page}.json`, rawdata.body);

  const nextPageToken = JSON.parse(rawdata.body).nextPageToken;
  if (nextPageToken) {
    page++;
    getModels(nextPageToken);
  }
};

getModelLists();

Where is getModels() function defined?

Ah line #24 should be getModelLists(nextPageToken)

Oops, sorry! 😅

Still a couple of weird names not being sanitised properly like "\"...better be GRYFFINDOR!\"".

There are a couple of issues in this script:

const filename = resource.relativePath.split("/").pop();

Some assets have really weird names (like GRYFFINDOR above) or way worse. ESpecially when they have ... at the start of the name it causes weird things to happen. So this should be sanitise(filename) - unfortunately, that will mess up the references to the assets within the .gltf file.

Second, fs.writeFile can't handle models above a certain size. Technically this should be 1gb but I found it happening with smaller models. This gives an error about "can't write a string larger than...". This causes the script to crash and you will need to restart it.