// Note the s3 file must have access policy applied
/*
When the Http Stream instance is destroyed, the 'close' event will be emitted. Because Http2Stream is an instance of stream.Duplex, the 'end' event will also be emitted if the stream data is currently flowing. The 'error' event may also be emitted if httpstream.destroy() was called with an Error passed as the first argument.
*/
const http = require('http');
const csv = require('csv-parser')
var arr = [];
var count = 0
http.get('some s3 file path', (res) => {
// console.log('statusCode:', res.statusCode);
// console.log('headers:', res.headers);
res.pipe(csv()).on('data', (d) => {
count++;
console.log('count : ',count)
arr.push(d);
// this is watermark or minimum csv data requirement
if(arr.length > 10) {
res.destroy()
a(arr)
} else {
console.log("no data")
}
});
res.on('close', (d) => {
console.log("
Data :
", arr.length);
});
res.on('error', (e) => {
console.error(e);
})
});