Delete documents using a transaction and query - Firestore

Question

I have an filmOrders collection and when a document is created there, it creates documents in 7 other collections. I wanted to delete a document from filmOrders and delete all documents from the other collections that have that ID from filmOrders.

To make this happen, the ID of the document in filmOrders is stored in those documents and then I can just query the ID in the other 7 collections and delete them via a transaction.

The issue is, I'm not sure if I'm doing this correctly. I've read this question which says you can't use a query reference in a transaction. But below I'm querying the db, getting the docs and then using a transaction to delete the docs.

Is this the right way to do this? It's possible for more documents to get added with an ID from filmOrders so I'm not sure if those will get deleted while this transaction is running

"use strict";
const functions = require("firebase-functions");
const admin = require("firebase-admin");

exports.deleteOrderAndInformation = functions.https.onCall(
  async (data, context) => {
    const { admin: adminToken, pa } = context.auth.token;

    if (!adminToken && !pa)
      throw new functions.https.HttpsError(
        "permission-denied",
        "Insufficent permissions to delete order"
      );

    try {
      const db = admin.firestore();
      await db.runTransaction(async t => {
        //we have to identify the docs to use in the transaction ahead of time
        //https://stackoverflow.com/questions/50071700/can-we-not-query-collections-inside-transactions
        const { orderId } = data;
        const orderRef = db.doc(`filmOrders/${orderId}`);

        //get all batches and events that had that orderID in it
        const batchCols = [
          "extrusionBatches",
          "printingBatches",
          "laminationBatches",
          "slittingBatches",
          "conversionBatches",
          "filmBatches",
          "filmEvents"
        ];

        const queriesPromises = batchCols.map(col => {
          return db
            .collection(col)
            .where("orderId", "==", orderId)
            .get();
        });

        //this will return all the query snapshots
        const queriesResolved = await Promise.all(queriesPromises);

        //extract the document refs
        const docRefs = queriesResolved.map(qSnapshot => {
          return qSnapshot.docs.map(doc => doc.ref);
        });

        const docRefsFlat = docRefs.flat();

        const tDocs = await t.getAll(...docRefsFlat, orderRef);
        const tDelete = tDocs.map(doc => {
          return t.delete(doc.ref);
        });

        await Promise.all(tDelete);
      });
    } catch (error) {
      throw new functions.https.HttpsError("cancelled", error.toString());
    }
  }
);

Answer 1

This probably better done on a delete trigger for the filmOrders collection. Either way, do the query before beginning the transaction. The cloud function should return the result of runTransaction, and shouldn't await the deletes, which don't return promises...

async function getOtherRefs(orderId) {
  // your query of all the collections, flattening the results
}

exports.deleteOrderAndInformation = functions.https.onCall(async (orderId, context) => {
  const otherRefs = await getOtherRefs(orderId);
  const allRefs = [...otherRefs, db.doc(`filmOrders/${orderId}`)];
  
  return db.runTransaction(transaction => {
    const docs = await transaction.getAll(allRefs);
    docs.forEach(doc => transaction.delete(doc.ref));
  })
});

Answer 2

Rather than use an ordinary transaction for this, it'd be more efficient to make use of a Batched Write as you are only writing data and you don't care about the current document state. Like ordinary transactions, batched writes make atomic (all-or-nothing) changes to your database.

Importantly, you should bake in support for handling deleting more than 500 documents at once, which is the limit of Firestore Transactions. Which can be done using:

async function deleteDocs(db, docRefs) {
  return Promise.all(
    Array
      .from({length: Math.ceil(docRefs.length/500)})
      .map((_, i) => docRefs.slice(500*i, 500*(i+1))) // <-- splits docRefs into arrays with at most 500 entries
      .map((docRefsInChunk) => {
        // create a new batch that will delete all refs in docRefsInChunk
        const batch = db.batch();
        docRefsInChunk.forEach(ref => batch.delete(ref));

        // commit the result, but sink the error to not error-out `Promise.all`
        return batch.commit()
          .then(
            () => ({ success: true }),
            (error) => ({ success: false, error, docRefs: docRefsInChunk })
          );
      })
  );
}

Next, we need to handle failed deletions. If you fail to delete one or more documents, we should create a document somewhere in your database that we can use to reattempt the deletion later manually or using another Cloud Function.

async function queueDeleteRetry(db, { errors, docRefs }) {
  // Create a new "Retry deletion" document reference
  const retryDocRef = db.collection("_server/retry/delete").doc();
  
  // Collate list of targets for the deletion by pulling their paths
  const docPaths = docRefs.map(ref => ref.path);
  
  // Set the data on the retry document
  return retryDocRef
    .set({
      attempts: 0,
      causes: errors.map(({ code, message, name, stack }) => ({
        code: code || null,
        message,
        name,
        stack: stack || null
      })),
      targets: docPaths,
      timestamp: Date.now()
    })
    .then(
      () => ({ queued: true, path: retryDocRef.path }),
      (error) => ({ queued: false, error, targets: docPaths })
    );
}

Applying these into your code gives:

exports.deleteOrderAndInformation = functions.https.onCall(
  async (data, context) => {
    const { admin: adminToken, pa } = context.auth.token;
    
    // TODO: Check assumption
    const { orderId } = data;

    if (!adminToken && !pa)
      throw new functions.https.HttpsError(
        "permission-denied",
        "Insufficent permissions to delete order"
      );

    if (!orderId)
      throw new functions.https.HttpsError(
        "invalid-argument",
        "Missing or falsy property \"orderId\""
      );

    try {
      const db = admin.firestore();
      
      // collections that may contain documents to be cleaned up
      const batchCols = [
        "extrusionBatches",
        "printingBatches",
        "laminationBatches",
        "slittingBatches",
        "conversionBatches",
        "filmBatches",
        "filmEvents"
      ];

      // find all the documents references that are linked to this order
      const queriedDocRefArrayPromises = batchCols.map(col => {
        return db
          .collection(col)
          .where("orderId", "==", orderId)
          .get()
          .then(() => qSnapshot.docs.map(doc => doc.ref)); // <- pull out DocumentReferences here for performance
      });
      
      // get all the document references as one array
      const docRefs = (await Promise.all(queriedDocRefArrayPromises))
        .flat();
      
      // include the original order reference
      const orderRef = db.doc(`filmOrders/${orderId}`);
      docRefs.push(orderRef);
      
      // attempt deletion
      const deleteBatchResults = await deleteDocs(db, docRefs);
      
      const failedResults = deleteBatchResults
        .reduce((info, result) => {
          if (!result.success) {
            info.errors.push(result.error);
            info.docRefs.push(...result.docRefs);
          }
          
          return info;
        }, { errors: [], docRefs: [] });
      
      if (failedResults.errors.length === 0) {
        // done! return response to finish!
        return {
          success: true,
          message: `Order #${orderId} and all linked documents were deleted successfully!`
        }
      }
      
      // some/all deletions failed
      
      // Create a document containing all the failed deletions
      const queueResult = await queueDeleteRetry(db, failedResults);
      
      // If it couldn't create a document, log the same info
      if (!queueResult.queued) {
        functions.logger.error({
          message: "Failed to delete AND queue deletion retry of documents",
          targets: queueResult.targets,
          "logging.googleapis.com/labels": {
            customError: "failed-deletion" // <- custom label to search for them
          }
        });
      }
      
      // calculate counts
      const totalCount = docRefs.length;
      const failedCount = failedResults.docRefs.length;
      
      // return info about failure to finish
      return { 
        success: false,
        message: totalCount === failedCount
          ? `Failed to delete all documents related to order #${orderId}!`
          : `Failed to delete ${failedCount}/${totalCount} documents related to order #${orderId}!`,
        detail: {
          orderId,
          counts: {
            failed: failedCount
            total: totalCount
          },
          retryQueued: queueResult.queued
            ? { path: queueResult.path } // <- this can be used on the client to track the retry status/intervene manually
            : false
        }
      }
    } catch (error) {
      throw new functions.https.HttpsError("cancelled", error.toString());
    }
  }
);

The above code returns one of the following responses:

{
  "success": true,
  "message": "Order #8dZZohPZ8aoTv1eB7N2F and all linked documents were deleted successfully"
}

{ 
  "success": false,
  "message": "Failed to delete all documents related to order #8dZZohPZ8aoTv1eB7N2F!",
  "detail": {
    "orderId": "8dZZohPZ8aoTv1eB7N2F",
    "counts": {
      "failed": 8,
      "total": 8
    },
    "retryQueued": {
      "path": "_server/retry/delete/Li8U4ZV6k6sDvIN3pju5"
    }
  }
}

{ 
  "success": false,
  "message": "Failed to delete 42/542 documents related to order #8dZZohPZ8aoTv1eB7N2F!",
  "detail": {
    "orderId": "8dZZohPZ8aoTv1eB7N2F",
    "counts": {
      "failed": 42,
      "total": 542
    },
    "retryQueued": false // <- when retry couldn't be created and log was used
  }
}