Creating an fs-based blog system using Promises

gistfile1.js

/**
 * The problem: I have a Dumb Blog System that is based on a directory structure like so:
 * .
 * |-- A Blog
 * |    |--postOne.txt
 * |    |--postTwo.txt
 * |    |--postThree.txt
 * |    |--PUBLISHED
 * |-- B Blog
 * |-- thisScript.js
 * |-- otherRandomFiles.config.js
 *
 * Blogs are in directories that end with blog.
 * They contain text files, which are our posts.
 * The PUBLISHED file tells us which posts are published.
 *
 * Given such a directory, the Dumb Blog system spits out the following model:
 * [
 *  {
 *    name: 'A Blog',
 *    posts: [
 *             {
 *               name: 'postOne',
 *               text: 'Ramblings of postOne'
 *             },
 *             ..
 *           ]
 *   },
 *   ...
 *  ]
 *
 * This means I have make three fs commands I need to run sequentially:
 *   1. Get the name of all the blogs
 *   2. Get the list of all published post names
 *   3. Get the content of all published posts
 */

// Here's how I solve this problem using sync functions
fs.readdirSync('.')
        .filter(function (fileName) {
            return fileName.match(/Blog$/);
        })// < - [ 'A Blog', 'B Blog']

        .map(function (blogName) {
            return {
                name: blogName,
                posts: fs.readFileSync( blogName + '/PUBLISHED', {encoding: 'utf8'}).split('\n')
            };
        }) // <- [{name: 'B Blog', posts: ['postOne', 'postTwo']}..]
        .map(function (blog) {
            blog.posts.forEach(function (fileName, postIndex) {
                blog.posts[postIndex] = {
                    title: fileName.charAt(0).toUpperCase() + fileName.slice(1),
                    text: fs.readFileSync(dir + '/' + blog.name + '/' + fileName + '.txt', {encoding: 'utf8'})
                }
            });
            return blog;
        }) // <- Our final model!
        .map(function (blog) {
            console.log(blog);
        })
;

// Here's how I solve this problem using callbacks
fs.readdir('.', function (fileNames) {
    fileName.filter(function (fileName) {
        return fileName.match(/Blog$/);
    })
            .map(function (blogName) {
                fs.readFile(blogName + '/PUBLISHED', function (blogPostNames) {
                    blogPostNames.split('\n')
                            .map(function (blogPostName) {
                                fs.readFile(blogName + '/' + blogPostName + '.txt', function (postText) {
                                    // Oh gawd, we're entering callback hell!!
                                    // No sense continuing this failed example, you get the idea
                                })
                            })
                    ;
                })
            })
    ;
});

// Let me use Promises, imagine for a sec that async fs methods did return a promise
readdir('.')
        .then(function (fileNames) {
            return Promise.all(fileNames.filter(function (fileName) {
                return fileName.match(/Blog$/);
            }).map(function (blogName) {

                // I can't just return a promise from fs.readFile, or I'll lose the name of the blog in the next then()
                // function. I have no choice but to nest a function again >_<
                // So much from saving us from callback hell. I could have used named functions in the previous solution!
                return readPublished(blogName);
            }))
        })
        .then(function (blogs) {
            blogs = blogs.map(function (blog) {
                
                // The problem is I want to call the next then() when the promises in blog.posts resolve.
                // Here is a different approach than creating a nested named function like in the previous then().
                // My alternative solution is to use utility functions to turn promises nested inside arrays and objects
                // into a single promise that passes the resolved object into the next then().
                blog.posts.map(function(postName){
                    
                    // Imagine the dePromisifier resolves flat objects with promises in them. I learned about this technique here:
                    // http://willi.am/blog/2014/06/15/resolving-javascript-objects-containing-promises/
                    return dePromisifier({
                        name: blog.name,
                        posts: readFile(blogs.name + '/' + postName + '.txt', {encoding: 'utf8'})
                    });
                });


                // Resolve an array of Promises into a single promise
                blogs.posts = Promise.all(blog.posts);

                // Once again, we have a flat object with a Promise in it
                return dePromisifier(blog);
            });

            // Darn it! We still have an array of promises, use Promise.all again to ensure the next then() is called
            // when we read the blog post files
            return Promise.all(blogs);
        })
        .then(function(blogs){
            // At last blogs is our final model! Man, it would have been better if I just wrote this code like I did at 
            // the beginning, with sync functions, but with async functions. Would async/await allow me to do that?
            // Even though they are based on Promises?
        })
;

function readPublished(blogName) {
    return fs.readFile(dir + '/' + blogName + '/PUBLISHED', {encoding: 'utf8'})
            .then(function (publishedPostNames) {
                return {
                    name: blogName,
                    posts: publishedPostNames.split('\n')
                };
            })
}

I feel like you might be overcomplicating the chain by trying to read all blogs and all posts as one thing. If you separate the two process out as two functions, things are made much simpler. There's a little bit of nesting - but these are mostly due to map functions and cannot be eliminated.

Here's my take on it:

'use strict';
function readBlog(name) {
  return readFile(path.join(dir, blogName, 'PUBLISHED'), 'utf8')
    .then(function (postNames) {
      return Promise.all(postNames.split('\n').map(function (postName) {
        return readFile(path.join(name, postName + '.txt'), 'utf8');
      }))
    });
}

function readAllBlogs() {
  var blogNames;
  return readdir('.')
    .then(function (fileNames) {
      return fileNames.filter(function (fileName) {
        return fileName.match(/Blog$/);
      });
    })
    .then(function (_blogNames) {
      blogNames = _blogNames;
      return Promise.all(blogNames.map(readBlog));
    })
    .then(function (postSets) {
      return postSets.map(function (postSet, index) {
        return {
          name: blogNames[index],
          posts: postSet
        };
      });
    });
}

readAllBlogs();

Just for fun, here would be the code above in ES7 async/await syntax (with lets and arrow functions):

'use strict';
async function readBlog(name) {
  let postNames = await readFile(path.join(dir, blogName, 'PUBLISHED'), 'utf8');
  return await Promise.all(
    postNames.split('\n')
      .map((postName) => readFile(path.join(name, postName + '.txt'), 'utf8'))
  );
}

async function readAllBlogs() {
  let fileNames = await readdir('.');
  let blogNames = fileNames.filter((fileName) => fileName.match(/Blog$/));
  let postSets = await Promise.all(blogNames.map(readBlog));
  return postSets.map((postSet, index) => {
    return {
      name: blogNames[index],
      posts: postSet
    };
  });
}

readAllBlogs();

Heh, yeah, I do tend to overcomplicate matters when left to my own device. I appreciate the levelheaded feedback from other developers, so thank you Keith 👍

In review, the heart of this problem is being dependent on data from asynchronous functions and forcing code to be sequential as that dependent data gets built up. I was on a quest to eradicate callbacks as that is what Promises are supposed to save us from. However, given your experienced advice, it seems that quest was misguided.

Your solution, seems to me, as a way to organize the code into functions that are passed in the dependent asynchronous data. I was doing something like that with my readPublished function above, but I was seeing that as “the wrong solution.” Not so wrong, now that I see your readBlog function. All the readFile calls inside it are doable because they have access to readBlog’s name parameter. In fact, I can take your strategy further and read posts (or any further, deeper, organization of asynchronous data):

function readPost(name, blogName) {
    return readFile(path(blogName, name + '.txt'), 'utf8')
            .then(function(postContent){
                return {
                    name: name,
                    text: postContent
                }
            })
}

function readBlog(name) {
    return readFile(path.join(dir, blogName, 'PUBLISHED'), 'utf8')
            .then(function (postNames) {
                return Promise.all(postNames.split('\n').map(function (postName) {
                    return readPost(postName, name);
                }))
            });
}

The solution I was working towards was building up dependent async data in an object and have that object passed through a chain of operations that modify it. This would result in Promises nested in this object. The next then() would be held off until those Promises resolve. However, this required using operations like Promise.all or the dePromisifier, which automated multiple then()s to be chained. Your solution, where you isolate dependent async data into a named function’s scope, seems more proper in comparison and would likely result in less computation. It also seems to lend itself to async/await better :)

Last item, I just relearned the Node path module thanks to you. It should tidy up my code a bit more now. Totally forgot about that 😅