javascript outer function finishing execution before inner function can modify return variable – asynch callback problems

I’m trying to write a simple recursive file walker using node’s fs API and am making a basic error with callbacks, but am having a hard time seeing it.

the final return the_files; executes and returns an empty array [] instead of one filled with the file objects

the debug statement does log a populated array to the console with Dirent objects in the array.

the testdir structure is:

/testdir/a.txt
/testdir/b.txt
/testdir/c.txt
/testdir/somedir/d.txt

what am i missing?

function getTheFiles(mydir){
    let the_files = [];
    
    fs.readdir(mydir, {withFileTypes: true}, function (err, files){

        if (err){
            return console.log('Unable to scan directory: ' + err);
        }

        files.forEach(function (file) {
            if (file.name[0] === '.') return; //skip dotfiles
            if (file.isDirectory()){
                getTheFiles(path.resolve(mydir, file.name));
            }
            if (file.isFile()){
                the_files.push(file);
                console.log(the_files); //debug
            }
        })
    })
    return the_files;
}
//output: []

I’ve gone through some excellent SO answers and blogs several times, but have not been able to massage my code into the right format:

Ref 1: Why is my variable unaltered after I modify it inside of a function? – Asynchronous code reference

Ref 2: Get data from fs.readFile

Ref 3: forEach does not play well with async

Ref 4: Why is my variable unaltered after I modify it inside of a function? – Asynchronous code reference

Ref 5: How do you get a list of the names of all files present in a directory in Node.js? – doesn’t help you do things with the result of the dir walk

Here’s another attempt where I tried calling a function with the result of the async readdir –

function async_ver_getTheFiles(dir){
    fs.readdir(dir, {withFileTypes: true}, function read(err, files){
        if (err){
            throw err;
        }
        filePacker(files);
    });

}

function filePacker(files){
    let the_files = [];
    for (const file of files){
        if (file.name[0] === '.') continue; //skip dotfiles
        if (file.isFile()){
            the_files.push(file.name);
        }
        if (file.isDirectory()){
            async_ver_getTheFiles(file.name);
        }
    }
    console.log(the_files); //returns the files
    return the_files;
}

var result = async_ver_getTheFiles(dir);
console.log(result); //returns undefined
//output: returns the undefined result before printing the correct result from within filePacker()

I do have a sync version working fine:

function getTheFiles(dir){
    let the_files = [];
    let files = [];

    try {
        files = fs.readdirSync(dir, {withFileTypes: true});
    } 
    catch (error) {
        console.error('failed calling fsreaddirSync(dir, {withFileTypes: true} on value dir: '' + dir + ''n');
        console.error(error);
    }

    for (const file of files){
        if (file.name[0] === '.') return; //skip dotfiles
        if (file.isFile()){
            let absolutePath = path.resolve(dir, file.name);
            the_files.push(absolutePath);
        }
        else if (file.isDirectory()){
            the_files.push(getTheFiles(path.resolve(dir, file.name)).toString()); //should be the_files.push(...getTheFiles(blah)) instead - see accepted answer
        }
}
    return the_files;
}
//output:
<path>/a.txt
<path>/b.txt
<path>/c.txt
<path>/somedir/d.txt

Answer

Maybe the whole thing gets a bit easier if you switch from the callback form of the fs API to the promise based version of the API. Ie, instead of

const fs = require('fs');

use

const fs = require('fs').promises;

This way, your fs function won’t have a signature like

fs.readdir(dir, (err, files) => {})

anymore but will look like the following

fs.readdir(dir) : Promise<string[]>

and you can use them like the following

let files = await fs.readdir("/foo/bar");

This way, you can (more or less) take your synchronous approach, and everywhere, where you now have a

let baz = fs.foobarSync(...)

you can then use

let baz = await fs.foobar();

Thus your code would be like (see also the comments in the code)

// require the promise based API
let fs = require('fs').promises;

// be sure to make this function async
// so you can use "await" inside of it
async function getTheFiles(dir){
  let the_files = [];
  let files = [];

  try {
   files = await fs.readdir(dir, {withFileTypes: true});
  } 
  catch (error) {
    console.error('...');
    console.error(error);
  }

  for (const file of files){
    if (file.name[0] === '.') continue; // <-- use continue here instead of return to skip this file and check also the remaining files
    
    if (file.isFile()){
      let absolutePath = path.resolve(dir, file.name);
      the_files.push(absolutePath);
    }
    else if (file.isDirectory()){
      // get all files from the subdirectory
      let subdirfiles = await getTheFiles(path.resolve(dir, file.name))

      // as getTheFiles returns an array
      // you can use the spread syntax to push all
      // elements of that array intto the_files array
      the_files.push(...subdirfiles);
    }
  }
  return the_files;
}


getTheFiles("/foo/bar").then(files => {
  console.log(files);
})
.catch(e => {
  console.log(e);
})