node.js - multiple layers of closures and synchronous javascript

Question

This is an extension to my previous question, which received a very explanatory answer. It turns out that I did not provide enough context to my app to make the question useful enough for my actual situation.

Here is a route within my Express app:

var eventbriteService = require('../apiRequests/eventbriteService');
var queue = require('queue-async');    
app.get('/events', function (req, res, next) {
    queue()
        .defer(eventbriteService.getEventbriteEvents)
        .await(function(err, data) {
            if (err) {return next(err);}    
            console.log("here are the events from routes" ,data);
        });
});

This route calls the following service:

exports.getEventbriteEvents = function (cb) {

    var eventbriteEvents = [];

    request({
        uri: "https://www.eventbrite.com/json/event_search?app_key=R3DQQXPXBTSMUWVNOV&city=Austin&date=2014-10-01&2014-10-15",
        method: "GET", timeout: 10000, followRedirect: true, maxRedirects: 10,
    }, function(err, response, body){
        if (err) return cb(err);

        try {
            var eventsJSON = JSON.parse(body);
            var eventsWithAllFields = eventsJSON.events;
            var totalEventsNumber = parseInt(eventsWithAllFields[0].summary.total_items);

            for (var i = 1, l = eventsWithAllFields.length; i < l; i++) {

                var eventObject = {
                    name: eventsWithAllFields[i].event.title,
                    images: []
                };

                var jsdom = require('jsdom');
                var arrayOfImgs = [];
                jsdom.env({
                    html: eventsWithAllFields[i].event.description,
                    scripts: ["http://code.jquery.com/jquery.js"],
                    done: function(evtobj, errors, window) {
                        window.$('img').each(function(){
                            var imgSrc = window.$(this).attr('src');
                            console.log(imgSrc);
                            evtobj.images.push(imgSrc);
                        });
                        eventbriteEvents.push(evtobj);
                    }.bind(null, eventObject)
                });

            }

        } catch(err) {
            console.log(err);
            return cb(err);
        }

        console.log(eventbriteEvents);
        cb(null, eventbriteEvents);

    });

};

This is my console output:

{}
[]
here are the events from routes []
https://evbdn.eventbrite.com/s3-s3/eventlogos/90039995/about.png
https://evbdn.eventbrite.com/s3-s3/eventlogos/90039995/bluedawntour1.jpg

The code is executing in the following order:

1. eventbrite service logs the array of eventbrite objects (empty)
2. route logs array of eventbrite objects (empty)
3. eventbrite service uses jsdom to find the existing imgs in the html

Obviously this is out of sync, and I am quite confused on rectifying this, considering the many layers of callbacks, closures, and queue-async.

I'm using the queue-async library to define a callback in the route which ends up being populated by the eventbrite service. This was working fine until I recently added the jsdom html-parsing functionality. In addition to solving this immediate problem, I am looking for help structuring my mental model of callbacks, closures, and synchronous code.

Answer 1

The problem is that you're calling your callback when your for loop is finished, but the for loop is calling an asynchronous function (jsdom.env) on each pass of its loop. What ends up happening is the for loop finishes looping before the functions it calls are complete.

What you need is something that will call a callback when all those asynchronous functions are complete. Since you're already using queue-async elsewhere, let's just use that (see comments in modified code):

var queue = require('queue-async');

exports.getEventbriteEvents = function (cb) {
    request({
        uri: "https://www.eventbrite.com/json/event_search?app_key=<redacted>&city=Austin&date=2014-10-01&2014-10-15",
        method: "GET", timeout: 10000, followRedirect: true, maxRedirects: 10,
    }, function(err, response, body){
        if (err) return cb(err);

        try {
            var jsdomQueue = queue();
            var eventsJSON = JSON.parse(body);
            var eventsWithAllFields = eventsJSON.events;
            var totalEventsNumber = parseInt(eventsWithAllFields[0].summary.total_items);

            for (var i = 1, l = eventsWithAllFields.length; i < l; i++) {

                var eventObject = {
                    name: eventsWithAllFields[i].event.title,
                    images: []
                };

                var jsdom = require('jsdom');

                // push ("defer") the jsdom.env async function on to the jsdomQueue
                // note: we have to move the .bind for that to still work
                jsdomQueue.defer(function(i, evtobj, deferCallback) {
                    jsdom.env({
                        html: eventsWithAllFields[i].event.description,
                        scripts: ["http://code.jquery.com/jquery.js"],
                        done: function(errors, window) {
                            window.$('img').each(function(){
                                var imgSrc = window.$(this).attr('src');
                                console.log(imgSrc);
                                evtobj.images.push(imgSrc);
                            });

                            // call the deferCallback with the evtobj
                            deferCallback(null, evtobj);
                        }
                    });
                }.bind(null, i, eventObject));

            }

            // wait for all the previously deferred functions to finish.
            // the objects passed to deferCallback above will be in the
            // eventbriteEvents array.
            jsdomQueue.awaitAll(function(err, eventbriteEvents) {
                if (err) {
                    // any additional error handling
                    cb(err);
                } else {
                    // now you can call your callback
                    console.log(eventbriteEvents);
                    cb(null, eventbriteEvents);
                }
            });

        } catch(err) {
            console.log(err);
            return cb(err);
        }

    });

};