Firestore maintainancew

*new*.txt

< Citations.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=Citations\&withSubCollections\=1 --data-binary @- ; \
< Concours.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=Concours\&withSubCollections\=1 --data-binary @- ; \
< Conseils.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=Conseils\&withSubCollections\=1 --data-binary @- ; \
< Devises.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=Devises\&withSubCollections\=1 --data-binary @- ; \
< Dictionnary_mot_interdit.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=Dictionnary_mot_interdit\&withSubCollections\=1 --data-binary @- ; \
< Gestion_abonnement.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=Gestion_abonnement\&withSubCollections\=1 --data-binary @- ; \
< Jeux.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=Jeux\&withSubCollections\=1 --data-binary @- ; \
< Notif_complementaires.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=Notif_complementaires\&withSubCollections\=1 --data-binary @- ; \
< Progression.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=Progression\&withSubCollections\=1 --data-binary @- ; \
< code_abo.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=code_abo\&withSubCollections\=1 --data-binary @- ; \
< const_appli.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=const_appli\&withSubCollections\=1 --data-binary @- ; \
< envies.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=envies\&withSubCollections\=1 --data-binary @- ; \
< groupes.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=groupes\&withSubCollections\=1 --data-binary @- ; \
< imageBack.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=imageBack\&withSubCollections\=1 --data-binary @- ; \
< langages.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=langages\&withSubCollections\=1 --data-binary @- ; \
< messagePrive.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=messagePrive\&withSubCollections\=1 --data-binary @- ; \
< notif_comments.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=notif_comments\&withSubCollections\=1 --data-binary @- ; \
< notif_conseils.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=notif_conseils\&withSubCollections\=1 --data-binary @- ; \
< notif_like.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=notif_like\&withSubCollections\=1 --data-binary @- ; \
< notif_posts.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=notif_posts\&withSubCollections\=1 --data-binary @- ; \
< parrains.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=parrains\&withSubCollections\=1 --data-binary @- ; \
< posts.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=posts\&withSubCollections\=1 --data-binary @- ; \
< rating.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=rating\&withSubCollections\=1 --data-binary @- ; \
< signalements.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=signalements\&withSubCollections\=1 --data-binary @- ; \
< tchat.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=tchat\&withSubCollections\=1 --data-binary @- ; \
< user_cites.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=user_cites\&withSubCollections\=1 --data-binary @- ; \
< user_sauv.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=user_sauv\&withSubCollections\=1 --data-binary @- ; \
< users_classement.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=users_classement\&withSubCollections\=1 --data-binary @- ; \
< users_pseudo.jsonl | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=users_pseudo\&withSubCollections\=1 --data-binary @- ; \

< users.jsonl jq -sc '.[:5000][] | .data.firebaseToken |= null' | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=users\&withSubCollections\=1 --data-binary @- ; \
< users.jsonl jq -sc '.[5000:10000][] | .data.firebaseToken |= null' | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=users\&withSubCollections\=1 --data-binary @- ; \
< users.jsonl jq -sc '.[10000:15000][] | .data.firebaseToken |= null' | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=users\&withSubCollections\=1 --data-binary @- ; \
< users.jsonl jq -sc '.[15000:20000][] | .data.firebaseToken |= null' | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=users\&withSubCollections\=1 --data-binary @- ; \
< users.jsonl jq -sc '.[20000:25000][] | .data.firebaseToken |= null' | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=users\&withSubCollections\=1 --data-binary @- ; \
< users.jsonl jq -sc '.[25000:30000][] | .data.firebaseToken |= null' | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=users\&withSubCollections\=1 --data-binary @- ; \
< users.jsonl jq -sc '.[30000:35000][] | .data.firebaseToken |= null' | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=users\&withSubCollections\=1 --data-binary @- ; \
< users.jsonl jq -sc '.[35000:40000][] | .data.firebaseToken |= null' | curl -vv -H 'Content-Type: application/octet-stream' http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=importCol\&col\=users\&withSubCollections\=1 --data-binary @- ; \


# DELETE

curl http://127.0.0.1:5001/hatchi-mobile/us-central1/maintainance\?secret\=$SECRET\&action\=DANGEROUSLY_delCol\&col\=tchat 

maintainance.ts

/**
 *  Maintainances
 */
export const maintainance = functions.https.onRequest(async (req, res) => {
  if (req.query.secret !== SECRET) {
    res.status(500).json({ error: "what do you want from me?" });
    return;
  }

  const allowActions = [
    "count",
    "listCol",
    "getCol",
    "importCol",
    "DANGEROUSLY_delCol",
  ] as const;

  const action = req.query.action as unknown as typeof allowActions[number];

  if (action === "count") {
    const cols = (req.query.col as string).split(","); // let it crash
    const count: { [k: string]: number } = {};
    for (const col of cols) {
      const snapshot = await db.collection(col).select().get();
      count[col] = snapshot.size;

      // const snapshot = await admin
      //   .firestore()
      //   .collection(col)
      //   .where("userId", "==", "user_16")
      //   .orderBy("date", "desc")
      //   .get();
      // count[col] = snapshot.docs.map((it) => it.data()) as any;

      // const snapshot = await admin
      //   .firestore()
      //   .collection(col)
      //   // .where("userId", "==", "user_16")
      //   .orderBy("date", "desc")
      //   .get();
      // count[col] = flowRight(
      //   // fromPairs,
      //   // map(([k, v]) => [k, v.length]),
      //   // toPairs,
      //   groupBy("userId"),
      //   map((it: any) => it.data())
      // )(snapshot.docs) as any;
    }
    res.json({ count });
  } else if (action === "getCol") {
    const rootCol = db.collection(req.query.col as string); // let it crash
    const withSubCollections = !!req.query.withSubCollections;

    /**
     * should be pairs of doc/col/doc/col... of the req.query.col as root collection
     */
    let subCollectionsQueue = new Set<string>();
    let pending: Promise<unknown>[] = [];

    const snapshot =
      rootCol.stream() as unknown as AsyncIterableIterator<admin.firestore.DocumentData>;

    for await (const doc of snapshot) {
      res.write(JSON.stringify({ id: doc.id, data: doc.data() }));
      if (withSubCollections) {
        pending.push(
          (async () => {
            const subCols = await doc.ref.listCollections();
            if (subCols.length > 0) {
              for (const subCol of subCols) {
                if (!subCollectionsQueue.has(subCol.id)) {
                  console.log("  queue subCollection", [
                    req.query.col,
                    // doc.id,
                    subCol.id,
                  ]);
                  subCollectionsQueue.add(subCol.id);
                }
              }
            }
          })()
        );
      }
    }

    if (withSubCollections) {
      if (pending.length > 0) {
        console.log(`wait for pending operations: ${req.query.col}`);
        await Promise.all(pending);
        pending = [];
      }

      while (subCollectionsQueue.size > 0) {
        const path = subCollectionsQueue.values().next().value;
        subCollectionsQueue.delete(path);
        const col = db.collectionGroup(path);
        const snapshot =
          col.stream() as unknown as AsyncIterableIterator<admin.firestore.DocumentData>;
        for await (const doc of snapshot) {
          pending.push(
            (async () => {
              const subCols = await doc.ref.listCollections();
              if (subCols.length > 0) {
                for (const subCol of subCols) {
                  if (!subCollectionsQueue.has(subCol.id)) {
                    console.log("  queue subCollection", [
                      req.query.col,
                      // ...path,
                      // doc.id,
                      subCol.id,
                    ]);
                    subCollectionsQueue.add(subCol.id);
                  }
                }
              }
            })()
          );
          let parent = doc.ref.parent;
          let paths = [parent.id, doc.id];
          while (true) {
            parent = parent.parent;
            if (!parent) {
              break;
            }
            paths.unshift(parent.id);
          }
          const rootName = paths.shift();
          if (rootName === req.query.col) {
            res.write(
              JSON.stringify({
                id: paths.join("/"),
                data: doc.data(),
              })
            );
          }
        }
        if (pending.length > 0) {
          console.log(
            `wait for pending operations: ${req.query.col},... ${path}`
          );
          await Promise.all(pending);
          pending = [];
        }
      }
    }

    res.end();
  } else if (action === "listCol") {
    const cols = await db.listCollections();
    res.end(cols.map((it) => it.id).join("\n"));
  } else if (action === "importCol") {
    const col = db.collection(req.query.col as string); // let it crash
    const withSubCollections = !!req.query.withSubCollections;
    // req is already drained into req.body due to default firebase's middlewares
    // so stream no longer make sense here...
    const pipeline = Readable.from(req.body).pipe(
      jsonlParser()
    ) as unknown as AsyncIterableIterator<{
      value: { id: string; data: unknown };
    }>;
    let count = 0;
    let batch = db.batch();
    let cap = 0;

    for await (const item of pipeline) {
      count += 1;
      if (cap > 400) {
        await batch.commit();
        cap = 0;
        batch = db.batch();
      }

      cap = cap + 1;
      if (withSubCollections) {
        const path = item.value.id.split("/");
        let ref = col.doc(path.shift()!);
        if (path.length > 1) {
          ref = ref.collection(path.shift()!).doc(path.shift()!);
        }
        batch.create(ref, item.value.data);
      } else {
        const ref = col.doc(item.value.id);
        batch.create(ref, item.value.data);
      }
    }

    await batch.commit();
    res.json({ count });
  } else if (action === "DANGEROUSLY_delCol") {
    const cols = (req.query.col as string).split(","); // let it crash
    const count: { [k: string]: number } = {};
    for (const col of cols) {
      const snapshot = await db.collection(col).select().get();
      let batch = db.batch();
      let cap = 0;
      for (const item of snapshot.docs) {
        if (cap > 400) {
          await batch.commit();
          cap = 0;
          batch = db.batch();
        }

        cap = cap + 1;
        batch.delete(item.ref);
      }
      await batch.commit();
      count[col] = snapshot.size;
    }
    res.json({ count });
  } else {
    res.end(allowActions.join("\n"));
  }
});