How to read a lot of documents (1M ) from a collection in Cloud Firestore?-CodePudding

The code below fails with error 9 FAILED_PRECONDITION: The requested snapshot version is too old.

const ref = db.collection('Collection');
const snapshot = await ref.get();
snapshot.forEach((doc,index) => {
   ...use data
})

Getting all documents from one collection in Firestore

EDIT:

getData();


async function getData(doc) {

  let snapshot = await global.db.collection('Collection').orderBy().startAfter(doc).limit(5000).get();

  const last = snapshot.docs[snapshot.docs.length - 1];
  
  snapshot.forEach((doc,index) => {
      //...use data
  })


  if (snapshot.docs.length < 5000) {
    return;
  }
  else {
    getData(last)
  }
}

EDIT 2 (works, a bit slow, reads sequentially, 5000 docs at a time):

let snapshot = null;
let totalIndex = 0;

await getData();

async function getData(doc) {

  if (!doc) {
    const first = await global.db.collection("collection").doc("docID");
    snapshot = await global.db.collection('collection').orderBy(admin.firestore.FieldPath.documentId()).startAt(first).limit(5000).get();
  }
  else {
    snapshot = await global.db.collection('Prompts').orderBy(admin.firestore.FieldPath.documentId()).startAfter(doc).limit(5000).get();
  }

  const last = snapshot.docs[snapshot.docs.length - 1];
  
  snapshot.forEach((doc,index) => {
     console.log(totalIndex  );
     //...use data
  })


  if (snapshot.docs.length < 5000) {
    return;
  }
  else {
    getData(last)
  }
}

CodePudding user response：

Since you're initially calling getData() without any arguments, that leads to doc being undefined in that function body. And calling startAfter(undefined) is not valid.

What you'll want to do is optionally adding that startAfter with something like:

async function getData(doc) {
  let query = global.db.collection('Collection').orderBy();
  if (doc) {
    query = query.startAfter(doc);
  }
  let snapshot = await query.limit(5000).get();
  ...