node.js - Optimal design pattern - functions which process multiple files -
goal create distinct functions separate out work of loading multiple (xml) files , parsing them. in 1 function, nested callbacks begin ugly. in other words, don't want this:
// explore directory fs.readdir(path, function (err, files) { if(err) throw err; // touch each file files.foreach(function(file) { fs.readfile(path+file, function(err, data) { if (err) throw err; someasyncfunction ( function (someasyncfunctionresult) { // work, call async function... nestedasynchfunction ( function (nestedasyncfunctionresult) { // final work here, x levels deep. ouch! }); }); }); }); }); instead, want 1 function reads files , puts each file's xml payload array of objects returned caller (each object represents name of file , xml in file). here's function might load reports array:
function loadreports (callback) { var path = "./downloaded-reports/"; var reports = []; // there 2 files in path.... fs.readdir(path, function (err, files) { if(err) throw err; files.foreach(function(file) { fs.readfile(path+file, function(err, data) { if (err) throw err; reports.push({ report: file, xml: data.tostring()}); //gets called twice, makes strangeness in calling function callback(null, reports); }); }); // callback won't work here, returns null reports b/c haven't been processed yet //callback(null, reports); }); } ...and here's function call 1 above:
function parsereports() { loadreports( function(err, data) { console.log ("loadreports callback"); var reportxml = new jsxml.xml(data[0].xml); var datasources = reportxml.child('datasources').child('datasource').child('connection').attribute("dbname").tostring(); console.log(json.stringify(datasources,null, 2)); // more async done below } ); } as can see in loadreports() comments, can't callback work right. either calls before array has been populated @ all, or calls twice - once each fs.readfile operation.
so...what best way deal sort of situation? in brief - what's best design pattern function processes multiple things asynchronously, calls when "things" have been processed? simpler better. need use sort of queuing module q or node-queue?
thanks much!
edit: works inside deepest loop in terms of not hitting callback twice, seems kludge:
fs.readdir(path, function (err, files) { if(err) throw err; files.foreach(function(file) { fs.readfile(path+file, function(err, data) { if (err) throw err; reports.push({ report: file, xml: data.tostring()}); // works, seems hacky. if (reports.length = files.length) callback(null, reports); }); }); });
Comments
Post a Comment