How to handle multipart/form-data req in express backend with google cloud functions in 2022? Researching this issue has taken a lot of my time with no result. I have tries multiple ways and they work fine locally but not when deployed.
All of the following code works locally but not when deployed
Errors are always the same
- Unexpected end of form when deployed
- Busboy is not a constructor which when I fix with removing the new Busboy gives the former error
// // // Processing multipart/form-data files middleware
// // const busboy = require("busboy"); // for parsing multipart/form-data
// // const path = require("path");
// // const fs = require("fs");
// // const asyncHandler = require("express-async-handler");
// // const processFileMiddleware = asyncHandler((req, res, next) => {
// // const busboyInstance = busboy({ headers: req.headers });
// // busboyInstance.on("file", async (fieldname, file, filename, encoding, mimetype) => {
// // console.log(filename);
// // // this logs the following:
// // const filepath = "./uploads/" await filename.filename;
// // console.log(filepath)
// // const writeStream = fs.createWriteStream(filepath);
// // file.pipe(writeStream);
// // file.on("end", () => {
// // // console.log("File [" fieldname "] Finished");
// // req.file = { originalFilename: filename, filepath };
// // next();
// // }
// // );
// // });
// // busboyInstance.on("finish", () => {
// // console.log("Done parsing form!");
// // next();
// // });
// // req.pipe(busboyInstance);
// // });
// // module.exports = { processFileMiddleware };
// const path = require('path');
// const os = require('os');
// const fs = require('fs');
// // Node.js doesn't have a built-in multipart/form-data parsing library.
// // Instead, we can use the 'busboy' library from NPM to parse these requests.
// const Busboy = require('busboy');
// const processFileMiddleware = (req, res, next) => {
// if (req.method !== 'POST') {
// // Return a "method not allowed" error
// return res.status(405).end();
// }
// console.log('Processing file...');
// const busboy = Busboy({headers: req.headers});
// const tmpdir = "./uploads/";
// // This object will accumulate all the fields, keyed by their name
// const fields = {};
// // This object will accumulate all the uploaded files, keyed by their name.
// const uploads = {};
// const fileWrites = [];
// // This code will process each file uploaded.
// busboy.on('file', (fieldname, file, {filename, encoding}) => {
// // Note: os.tmpdir() points to an in-memory file system on GCF
// // Thus, any files in it must fit in the instance's memory.
// console.log(`Processed file ${filename}`);
// const filepath = path.join(tmpdir, filename);
// uploads['file'] = filepath;
// console.log(filepath)
// const writeStream = fs.createWriteStream(filepath);
// file.pipe(writeStream);
// // File was processed by Busboy; wait for it to be written.
// // Note: GCF may not persist saved files across invocations.
// // Persistent files must be kept in other locations
// // (such as Cloud Storage buckets).
// const promise = new Promise((resolve, reject) => {
// file.on('end', () => {
// writeStream.end();
// });
// writeStream.on('finish', resolve);
// writeStream.on('error', reject);
// });
// fileWrites.push(promise);
// });
// // Triggered once all uploaded files are processed by Busboy.
// // We still need to wait for the disk writes (saves) to complete.
// busboy.on('finish', async () => {
// await Promise.all(fileWrites);
// /**
// * TODO(developer): Process saved files here
// */
// for (const file in uploads) {
// fs.unlinkSync(uploads[file]);
// }
// res.send();
// });
// // busboy.end(req.body);
// console.log(req.body)
// // The raw bytes of the upload will be in req.rawBody. Send it to
// // Busboy, and get a callback when it's finished.
// busboy.pipe(req.rawBody);
// res.send();
// process incoming request
// upload the file to the bucket
// const Busboy = require("busboy");
// const path = require("path");
// const os = require("os");
// const fs = require("fs");
// const asyncHandler = require("express-async-handler");
// const { Storage } = require("@google-cloud/storage");
// const { uploadFile } = require("../cloudstorage/uploadFile");
// const random = (() => {
// const buf = Buffer.alloc(16);
// return () => randomFileSync(buf).toString("hex");
// })();
// const processFileMiddleware = asyncHandler((req, res, next) => {
// if (req.method !== "POST") {
// // Return a "method not allowed" error
// return res.status(405).end();
// }
// console.log("Processing file...");
// const bb = Busboy({ headers: req.headers });
// bb.on("file", async (name, file, { filename }) => {
// const saveTo = path.join(os.tmpdir(), filename);
// file.pipe(fs.createWriteStream(saveTo));
// console.log(`File ${filename} saved to ${saveTo}`);
// try {
// await uploadFile(file, filename);
// } catch (err) {
// console.log(err);
// }
// });
// bb.on("finish", () => {
// // res.writeHead(200, { 'Connection': 'close' });
// next();
// });
// // console.log(Object.keys(req), bb);
// // docs say this is needed
// req.pipe(bb);
// req.on("end", () => {
// console.log("Done parsing form!");
// next();
// });
// });
// module.exports = { processFileMiddleware };
const asyncHandler = require("express-async-handler");
const processFileMiddleware = asyncHandler((req, res) => {
const BusBoy = require("busboy");
const path = require("path");
const os = require("os");
const fs = require("fs");
const uuid = require("uuid");
const busboy = BusBoy({ headers: req.headers });
let imageToBeUploaded = {};
let imageFileName;
// String for image token
let generatedToken = uuid.v4();
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
console.log(fieldname, file, filename, encoding, mimetype);
if (mimetype !== "image/jpeg" && mimetype !== "image/png") {
return res.status(400).json({ error: "Wrong file type submitted" });
}
// my.image.png => ['my', 'image', 'png']
const imageExtension = filename.split(".")[filename.split(".").length - 1];
// 32756238461724837.png
imageFileName = `${Math.round(
Math.random() * 1000000000000
).toString()}.${imageExtension}`;
const filepath = path.join(os.tmpdir(), imageFileName);
imageToBeUploaded = { filepath, mimetype };
file.pipe(fs.createWriteStream(filepath));
});
busboy.on("finish", () => {
admin
.storage()
.bucket()
.upload(imageToBeUploaded.filepath, {
resumable: false,
metadata: {
metadata: {
contentType: imageToBeUploaded.mimetype,
//Generate token to be appended to imageUrl
firebaseStorageDownloadTokens: generatedToken,
},
},
});
});
busboy.end(req.rawBody);
});
module.exports = { processFileMiddleware };
How do the real professionals handle this? I am looking for any insight.
edit 1 The error
node:events:505 throw er;
// Unhandled 'error' event ^ Error: Unexpected end of form at Multipart._final (functions/node_modules/busboy/lib/types/multipart.js:588:17) at callFinal (node:internal/streams/writable:695:27) at prefinish
(node:internal/streams/writable:724:7) at finishMaybe
(node:internal/streams/writable:734:5) at Multipart.Writable.end
(node:internal/streams/writable:632:5)
The frontend which is sending the req has the following code:
const API_URL = "http://192.168.1.2:4000/api/admissionForm/submit";
const uploadFile = async (file: any) => {
const token = JSON.parse(sessionStorage.getItem("user") || "{}").token;
console.log(Object.keys(file))
try {
const config = {
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "multipart/form-data",
},
};
const res = await axios.post(API_URL, file, config);
return res.data;
} catch (err: any) {
if (err.response.status === 500) {
console.log("There was a problem with the server");
} else {
console.log(err.response.data.msg);
}
}
};
The html part
<form onSubmit={handleUpload}>
<input
type="file"
className=""
id="inputFile01"
style={{ visibility: "hidden" }}
onChange={changeHandler}
name={"file"}
/>
<button type="submit">
Upload
</button>
</form>
changeHandler() code
const changeHandler = (e: SyntheticEvent) => {
const target = e.target as HTMLInputElement;
let selectedList = target.files as FileList;
console.log(selectedList);
let selected = selectedList[0] as File;
setFile(selected);
};
const handleUpload = async (e: SyntheticEvent) => {
e.preventDefault();
console.log(file);
if (file) {
const formData = new FormData();
formData.append("file", file);
try {
const res = await uploadFile(formData);
console.log(res);
} catch (error) {
console.log(error);
}
}
};
Some research shows that this is a Cloud functions issue but those research are from 2-3 years old and some of those threads tells me that this is fixed now. Please show me how. I am asking this question in Oct 2022.
CodePudding user response：
Firebase Functions are built on top of the functions-framework-nodejs package (or at least, an internal variant of the same code), where it parses the body for you before your code even gets the chance to execute. This behaviour is documented here.
This means that when you try to feed busboy the request's body stream, it has already been drained - resulting in your "Unexpected end of form" error message. To process the original body of the request, you need to access it via the rawBody property added to the request by the framework.
This means that you need to swap out
busboy.end(req.body);
for
busboy.end(req.rawBody);