Download file from third party server and upload to S3-CodePudding

I have a Lambda Node function which is called by a webhook from a thirdparty server. The TP server sends a file download URL and some other data.

The download URL is temporary, so I need to push the file to an S3 for long term storage.

The rudimentary function below, downloads the file and then tries to upload to the S3. This works when the file is a plain text, but images/pdfs etcs are corrupted when they reach the S3.

const AWS = require("aws-sdk");
const https = require('https');
const path = require('path');

const s3 = new AWS.S3({apiVersion: '2006-03-01'});

exports.handler = async (event, context, callback) => {

  var payload = event.body;
  const url_host = payload.host;
  const url_path = payload.path; //URL of file which needs to be downloaded

  const get_params = {
    host: url_host,
    path: url_path,
    port: 443,
    method: 'GET',
    headers: { }
  };

  var resp = await https_get_processor(get_params); //File downloaded here 

  var uploadParams = {
    Bucket: "bucket_name", 
    Key: '',
    Body: resp //Preparing to upload the received file
  };

  uploadParams.Key = path.basename(url_path); //Generating filename

  s3.upload (uploadParams, function (err, data) {
    if (err) {
      console.log("Error", err);
    } if (data) {
      console.log("Upload Success", data.Location);
    }
  });

  response = {...} //Generic Response
  return response;
};

async function https_get_processor(get_params)
{
  return await new Promise((resolve, reject) =>
  {
    var data = "";

    const req = https.request(get_params, res => {

      res.on('data', chunk => { data  = chunk }) 

      res.on('end', () => 
      {
        resolve(data);
      })

    });
  
    req.on('error', (err) => {
      reject(err);
    });

    req.end();
  });
}

CodePudding user response：

Response is a Buffer in such case, so try changing request processing by pushing each chunk into an array, and then merge Buffer chunks and pass them.

Try this:

var data = [];

const req = https.request(get_params, res => {

  res.on('data', chunk => data.push(chunk)) 

  res.on('end', () => 
  {
    resolve(Buffer.concat(data));
  })