Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Firebase Hosting Fetch All Files

Fetch All Files from Firebase Hosting

This script fetches all of the files from the currently deployed version of a Firebase Hosting site. You must be signed in via the Firebase CLI and have "Site Viewer" permission on the site in question to be able to properly run the script.

Running via NPX

npx https://gist.github.com/mbleigh/9c8680cf319ace2f506f57380da66e7d <site_name>

Where <site_name> is your Firebase Hosting site (e.g. my-site from my-site.firebaseapp.com). The files will be downloaded into a folder <site_name>_<version_id> where the version ID is the current version of your site.

#!/usr/bin/env node
const requireAuth = require('firebase-tools/lib/requireAuth');
const api = require('firebase-tools/lib/api');
const fs = require('fs-extra');
const request = require('request');
if (!process.argv[2]) {
console.error(`
ERROR: Must supply a site name. Usage:
node fetchFiles.js <site_name>`);
process.exit(1);
}
const site = process.argv[2];
async function getLatestVersionName() {
const result = await api.request('GET', `/v1beta1/sites/${site}/releases?pageSize=1`, {
auth: true,
origin: api.hostingApiOrigin,
});
const release = (result.body.releases || [])[0];
if (release) {
return release.version.name;
}
return null;
}
const LIST_PAGE_SIZE = 1000;
async function listFiles(versionName, existing = [], pageToken = null) {
const result = await api.request('GET', `/v1beta1/${versionName}/files?pageSize=${LIST_PAGE_SIZE}${pageToken ? `&pageToken=${pageToken}` : ''}`, {auth: true, origin: api.hostingApiOrigin});
result.body.files.forEach(file => existing.push(file.path));
if (result.body.nextPageToken) {
return await listFiles(versionName, existing, result.body.nextPageToken);
}
return existing;
}
const MAX_FETCHES = 100;
(async function() {
try {
await requireAuth({}, ['https://www.googleapis.com/auth/cloud-platform']);
const v = await getLatestVersionName();
const vid = v.split('/')[v.split('/').length - 1];
const toFetch = await listFiles(v);
const dirName = `${site}_${vid}`;
let fetchesOutstanding = 0;
let fetchCount = 0;
function fetch() {
if (fetchesOutstanding >= MAX_FETCHES) {
return;
} else if (toFetch.length === 0) {
console.log();
console.log("Complete. Fetched", fetchCount, "files.");
return;
}
const f = toFetch.shift();
console.log('Fetching', f);
fetchesOutstanding++;
fetchCount++;
fs.ensureFileSync(dirName + f);
const q = request(`https://${site}.firebaseapp.com${f}`)
const ws = fs.createWriteStream(dirName + f);
q.pipe(ws);
ws.on('finish', () => {
console.log('Fetched ', f);
fetchesOutstanding--;
fetch();
});
}
fetch();
} catch (e) {
console.error("ERROR:", e.stack);
process.exit(1);
}
})();
{
"name": "firebase-hosting-fetch-version",
"version": "0.1.0",
"bin": "./fetchFiles.js",
"dependencies": {
"firebase-tools": "^7.4.0",
"fs-extra": "^8.1.0",
"request": "^2.88.0"
}
}
@douwevdijk
Copy link

douwevdijk commented Aug 30, 2020

Create thank you. Is there a possibility to exclude folders ( e.g. node_modules )

@GuilhermeCouto
Copy link

GuilhermeCouto commented Sep 23, 2020

Why am I getting this error?

npm ERR! code ENOENT
npm ERR! syscall spawn git
npm ERR! path git
npm ERR! errno -4058
npm ERR! enoent Error while executing:
npm ERR! enoent undefined ls-remote -h -t https://gist.github.com/9c8680cf319ace2f506f57380da66e7d.git
npm ERR! enoent
npm ERR! enoent
npm ERR! enoent spawn git ENOENT
npm ERR! enoent This is related to npm not being able to find a file.
npm ERR! enoent

npm ERR! A complete log of this run can be found in:
npm ERR! C:\Users\COUTO-PC\AppData\Roaming\npm-cache_logs\2020-09-23T21_49_34_369Z-debug.log
Install for [ 'git+https://gist.github.com/9c8680cf319ace2f506f57380da66e7d.git' ] failed with code 4294963238

@axelmierczuk
Copy link

axelmierczuk commented Sep 25, 2020

Same as @GuilhermeCouto

@axelmierczuk
Copy link

axelmierczuk commented Sep 25, 2020

@GuilhermeCouto You need to have git installed.

@GuilhermeCouto
Copy link

GuilhermeCouto commented Sep 25, 2020

I got it by downloading the files and running:
npm install
node fetchFiles.js <site_name>

@saran141998
Copy link

saran141998 commented Sep 26, 2020

Why I am getting this?

FirebaseError: HTTP Error: 404, Requested entity was not found.
at module.exports (C:\Projects\React\NodeFetch\node_modules\firebase-tools\lib\responseToError.js:38:12)
at Request._callback (C:\Projects\React\NodeFetch\node_modules\firebase-tools\lib\api.js:40:35)
at Request.self.callback (C:\Projects\React\NodeFetch\node_modules\request\request.js:185:22)
at Request.emit (events.js:315:20)
at Request. (C:\Projects\React\NodeFetch\node_modules\request\request.js:1154:10)
at Request.emit (events.js:315:20)
at IncomingMessage. (C:\Projects\React\NodeFetch\node_modules\request\request.js:1076:12)
at Object.onceWrapper (events.js:421:28)
at IncomingMessage.emit (events.js:327:22)
at endReadableNT (_stream_readable.js:1220:12)

@anubhavbagri
Copy link

anubhavbagri commented Sep 29, 2020

Why I am getting this?

FirebaseError: HTTP Error: 404, Requested entity was not found.
at module.exports (C:\Projects\React\NodeFetch\node_modules\firebase-tools\lib\responseToError.js:38:12)
at Request._callback (C:\Projects\React\NodeFetch\node_modules\firebase-tools\lib\api.js:40:35)
at Request.self.callback (C:\Projects\React\NodeFetch\node_modules\request\request.js:185:22)
at Request.emit (events.js:315:20)
at Request. (C:\Projects\React\NodeFetch\node_modules\request\request.js:1154:10)
at Request.emit (events.js:315:20)
at IncomingMessage. (C:\Projects\React\NodeFetch\node_modules\request\request.js:1076:12)
at Object.onceWrapper (events.js:421:28)
at IncomingMessage.emit (events.js:327:22)
at endReadableNT (_stream_readable.js:1220:12)

I'm getting the same error. Could someone help?

@dradeng
Copy link

dradeng commented Oct 3, 2020

Same as the last two guys above

@DangerDay
Copy link

DangerDay commented Oct 7, 2020

Getting the same error: FirebaseError: HTTP Error: 404, Requested entity was not found.

@antiochdev
Copy link

antiochdev commented Oct 26, 2020

@GuilhermeCouto followed his lead on this... 👆🏼

@dagalti
Copy link

dagalti commented Nov 26, 2020

@dradgeng, @DangerDay npx https://gist.github.com/mbleigh/9c8680cf319ace2f506f57380da66e7d <site_name>

check <site_name> . thats not with firebaseapp.com .

@sudheerb9
Copy link

sudheerb9 commented Dec 23, 2020

Hi, I tried using this with one of my firebase apps, it resulted in the following error.
(node:7084) Warning: Accessing non-existent property 'padLevels' of module exports inside circular dependency (Use node --trace-warnings ... to show where the warning was created)
ERROR: FirebaseError: HTTP Error: 404, Requested entity was not found.
at module.exports (C:\Users\sreed\AppData\Roaming\npm-cache_npx\7084\node_modules\firebase-hosting-fetch-version\node_modules\firebase-tools\lib\responseToError.js:38:12)
at Request._callback (C:\Users\sreed\AppData\Roaming\npm-cache_npx\7084\node_modules\firebase-hosting-fetch-version\node_modules\firebase-tools\lib\api.js:40:35)
at Request.self.callback (C:\Users\sreed\AppData\Roaming\npm-cache_npx\7084\node_modules\firebase-hosting-fetch-version\node_modules\request\request.js:185:22)
at Request.emit (events.js:315:20)
at Request. (C:\Users\sreed\AppData\Roaming\npm-cache_npx\7084\node_modules\firebase-hosting-fetch-version\node_modules\request\request.js:1154:10)
at Request.emit (events.js:315:20)
at IncomingMessage. (C:\Users\sreed\AppData\Roaming\npm-cache_npx\7084\node_modules\firebase-hosting-fetch-version\node_modules\request\request.js:1076:12)
at Object.onceWrapper (events.js:421:28)
at IncomingMessage.emit (events.js:327:22)
at endReadableNT (internal/streams/readable.js:1327:12)

Please help me resolving this.
Thanks in advance

@minhthongle12
Copy link

minhthongle12 commented Jan 28, 2021

In your advice, I have this question: " "Site Viewer" permission on the site" How to do this ?.
I did all the things you have shown except "site viewer" cause idk how to do this.

After those things, I got this error:
(node:8648) Warning: Accessing non-existent property 'padLevels' of module exports inside circular dependency
(Use node --trace-warnings ... to show where the warning was created)
ERROR: FirebaseError: HTTP Error: 404, Not Found

Please help me, I really need the source code to review and learn for an upcoming interview.
God blesses you.
Thankssssssss

@KurisuWong
Copy link

KurisuWong commented Jan 28, 2021

Suggest modify line 60 to
const f = encodeURI(toFetch.shift());
to prevent TypeError [ERR_UNESCAPED_CHARACTERS] Request path contains unescaped characters

@Cadris
Copy link

Cadris commented Feb 1, 2021

(node:7784) Warning: Accessing non-existent property 'padLevels' of module exports inside circular dependency (Use node --trace-warnings ... to show where the warning was created) ERROR: FirebaseError: HTTP Error: 404, Not Found at module.exports (C:\Users\Administrator\AppData\Roaming\npm-cache\_npx\7784\node_modules\firebase-hosting-fetch-version\node_modules\firebase-tools\lib\responseToError.js:38:12) at Request._callback (C:\Users\Administrator\AppData\Roaming\npm-cache\_npx\7784\node_modules\firebase-hosting-fetch-version\node_modules\firebase-tools\lib\api.js:40:35) at Request.self.callback (C:\Users\Administrator\AppData\Roaming\npm-cache\_npx\7784\node_modules\firebase-hosting-fetch-version\node_modules\request\request.js:185:22) at Request.emit (events.js:315:20) at Request.<anonymous> (C:\Users\Administrator\AppData\Roaming\npm-cache\_npx\7784\node_modules\firebase-hosting-fetch-version\node_modules\request\request.js:1154:10) at Request.emit (events.js:315:20) at IncomingMessage.<anonymous> (C:\Users\Administrator\AppData\Roaming\npm-cache\_npx\7784\node_modules\firebase-hosting-fetch-version\node_modules\request\request.js:1076:12) at Object.onceWrapper (events.js:421:28) at IncomingMessage.emit (events.js:327:22) at endReadableNT (_stream_readable.js:1327:12)

I am Getting this error.
I am the site owner .

@jinmanpak
Copy link

jinmanpak commented Feb 19, 2021

This is work

I tried like this:
(1) my site name is HelloWorld so...
(2) tried below - npx https://gist.github.com/mbleigh/9c8680cf319ace2f506f57380da66e7d HelloWorld

This is really working! Thank you so much you saved my project and time!!!

@Harsh-Arora
Copy link

Harsh-Arora commented Mar 2, 2021

This is work

I tried like this:
(1) my site name is HelloWorld so...
(2) tried below - npx https://gist.github.com/mbleigh/9c8680cf319ace2f506f57380da66e7d HelloWorld

This is really working! Thank you so much you saved my project and time!!!

It is working for me too. just use your app's name without firebaseapp.com

@Harsh-Arora
Copy link

Harsh-Arora commented Mar 2, 2021

In your advice, I have this question: " "Site Viewer" permission on the site" How to do this ?.
I did all the things you have shown except "site viewer" cause idk how to do this.

After those things, I got this error:
(node:8648) Warning: Accessing non-existent property 'padLevels' of module exports inside circular dependency
(Use node --trace-warnings ... to show where the warning was created)
ERROR: FirebaseError: HTTP Error: 404, Not Found

Please help me, I really need the source code to review and learn for an upcoming interview.
God blesses you.
Thankssssssss

Use npx https://gist.github.com/mbleigh/9c8680cf319ace2f506f57380da66e7d app name

Note, use app name without firebaseapp.com

@KaveeshaC
Copy link

KaveeshaC commented Mar 10, 2021

I have error calling that "Unexpected token function"

@mdmitry1
Copy link

mdmitry1 commented Nov 3, 2021

In order to make this code work on my machine, I've changed line 44 from:

await requireAuth({}, ['https://www.googleapis.com/auth/cloud-platform']);

to

await requireAuth.requireAuth({}, ['https://www.googleapis.com/auth/cloud-platform']);

My node version is 16.13.0.
firebase-tools version 9.21.0

OS version:

Ubuntu 20.04.3 LTS
4.4.0-19041-Microsoft #1237-Microsoft Sat Sep 11 14:32:00 PST 2021 x86_64 x86_64 x86_64 GNU/Linux

@mdmitry1
Copy link

mdmitry1 commented Nov 3, 2021

In continuation to my previous comment https://gist.github.com/mbleigh/9c8680cf319ace2f506f57380da66e7d#gistcomment-3949726
Change has been needed, because I've used node instead of npx, as recommended in README.md.
Now I run again using npx and everything worked well.

@carnivash
Copy link

carnivash commented Dec 2, 2021

Thanks @KurisuWong worked like a charm.

@hg2355
Copy link

hg2355 commented Dec 20, 2021

after the files are download - how do you actually view the code? in the files download, I don't see any familiar code from the react-application that was my latest deployment to firebase.

@Bodrie
Copy link

Bodrie commented Jan 24, 2022

Great tool! Thank you!

@jgtaylor123
Copy link

jgtaylor123 commented Feb 13, 2022

I got the "Warning: Accessing non-existent property 'padLevels' of module exports inside circular dependency" error too.

Changed the site to exclude "firebaseapp.com" and it worked like a charm. Thanks.

@mariohenriquelopes3
Copy link

mariohenriquelopes3 commented Apr 7, 2022

thank you! is there something similar for fetch functions?

@raviganwal
Copy link

raviganwal commented Apr 24, 2022

Using the above method I am only able to fetch the compiled version of dart files, not the actual source code. If anyone knows a workaround please help.

@mbleigh
Copy link
Author

mbleigh commented Apr 24, 2022

@vojta001
Copy link

vojta001 commented May 3, 2022

Should your files contain characters unacceptable in URLs, use this path to escape them first

diff --git a/fetchFiles.js b/fetchFiles.js
index 78e59ab..3409fc2 100644
--- a/fetchFiles.js
+++ b/fetchFiles.js
@@ -37,6 +37,11 @@ async function listFiles(versionName, existing = [], pageToken = null) {
   return existing;
 }

+function escapePathForURL(path) {
+  const components = path.split('/');
+  return components.map(encodeURIComponent).join('/');
+}
+
 const MAX_FETCHES = 100;

 (async function() {
@@ -62,7 +67,7 @@ const MAX_FETCHES = 100;
       fetchesOutstanding++;
       fetchCount++;
       fs.ensureFileSync(dirName + f);
-      const q = request(`https://${site}.firebaseapp.com${f}`)
+      const q = request(`https://${site}.firebaseapp.com${escapePathForURL(f)}`)
       const ws = fs.createWriteStream(dirName + f);
       q.pipe(ws);
       ws.on('finish', () => {

@mbleigh Feel free to apply it to your Gist.

@alexhollender
Copy link

alexhollender commented Jun 21, 2022

Is it possible that this library has somehow become un-registered from npm? I've tried a few different things[1] but keep getting this error:
Screen Shot 2022-06-21 at 3 52 18 PM

not sure if this is any indication, but this link doesn't work: https://registry.npmjs.org/firebase-hosting-fetch-version

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment