Home » Nodejs » How to wait for a stream to finish piping? (Nodejs)

How to wait for a stream to finish piping? (Nodejs)

Posted by: admin November 30, 2017 Leave a comment

Questions:

I have a for loop array of promises, so I used Promise.all to go through them and called then afterwards.

let promises = [];
promises.push(promise1);
promises.push(promise2);
promises.push(promise3);

Promise.all(promises).then((responses) => {
  for (let i = 0; i < promises.length; i++) {
    if (promise.property === something) {
      //do something
    } else {
      let file = fs.createWriteStream('./hello.pdf');
      let stream = responses[i].pipe(file);
      /*
         I WANT THE PIPING AND THE FOLLOWING CODE 
         TO RUN BEFORE NEXT ITERATION OF FOR LOOP
      */
      stream.on('finish', () => {
        //extract the text out of the pdf
        extract(filePath, {splitPages: false}, (err, text) => {
        if (err) {
          console.log(err);
        } else {
          arrayOfDocuments[i].text_contents = text;
        }
      });
    });    
  }
}

promise1, promise2, and promise3 are some http requests, and if one of them is an application/pdf, then I write it to a stream and parse the text out of it. But this code runs the next iteration before parsing the test out of the pdf. Is there a way to make the code wait until the piping to the stream and extracting are finished before moving on to the next iteration?

Answers:

Something like the following would also work. I use this pattern fairly often:

let promises = [];
promises.push(promise1);
promises.push(promise2);
promises.push(promise3);

function doNext(){
  if(!promises.length) return;
  promises.shift().then((resolved) =>{
    if(resolved.property === something){
      ...
      doNext();
    }else{
      let file = fs.createWriteStream('./hello.pdf');
      let stream = resolved.pipe(file);
      stream.on('finish', () =>{
        ...
        doNext();
      });
    }

  })
}
doNext();

or break up the handler to a controller and Promisified handler:

function streamOrNot(obj){
  return new Promise(resolve, reject){
    if(obj.property === something){
      resolve();
      return;
    }
    let file = fs.createWriteStream...;
    stream.on('finish', () =>{
      ...
      resolve();
    });
  }
}

function doNext(){
  if(!promises.length) return;
  return promises.shift().then(streamOrNot).then(doNext);
}

doNext()

Questions:
Answers:

You can write the else part inside a self invoked function. So that the handling of stream will happen in parallel

(function(i) {
    let file = fs.createWriteStream('./hello.pdf');
    let stream = responses[i].pipe(file);
  /*
     I WANT THE PIPING AND THE FOLLOWING CODE 
     TO RUN BEFORE NEXT ITERATION OF FOR LOOP
  */
    stream.on('finish', () => {
      //extract the text out of the pdf
      extract(filePath, {splitPages: false}, (err, text) => {
      if (err) {
        console.log(err);
      } 
      else {
        arrayOfDocuments[i].text_contents = text;
      }
    });
  });    
})(i) 

Else you can handle the streaming part as part of the original/individual promise itself.

As of now you are creating the promise and adding it to array, instead of that you add promise.then to the array(which is also a promise). And inside the handler to then you do your streaming stuff.

Questions:
Answers:

Use bluebird promises to get access to the mapSeries function which does exactly what you want to do:

const Promise = require('bluebird')

let promises = [];
promises.push(promise1);
promises.push(promise2);
promises.push(promise3);

Promise.all([promise1, promise2, promise3])
  .mapSeries((response) => {
    if (response.property === something) return 'something'
    const file = fs.createWriteStream('./hello.pdf');
    const stream = response.pipe(file)
    return new Promise((resolve, reject) => {
      stream.on('finish', () => {
        extract(filePath, (err, text) => {
          if (err) reject(err)
          else resolve(text)
        })
      })
    })
  })
  .map((text, index) => {
    arrayOfDocuments[index].text_contents = text
  })

Bluebird docs