Underlying functions for crawling Learn directories.

spike.ts

import fs from "fs-extra";
import path from "path";
import matter from "gray-matter";

const BASE_PATH = "./sample";

// Utility function to list directories or files in a given path
async function listDirectories(parentPath: string): Promise<string[]> {
  const entries = await fs.readdir(parentPath);
  return entries.filter((entry) =>
    fs.statSync(path.join(parentPath, entry)).isDirectory(),
  );
}

async function listFiles(parentPath: string): Promise<string[]> {
  const entries = await fs.readdir(parentPath);
  return entries.filter((entry) =>
    fs.statSync(path.join(parentPath, entry)).isFile(),
  );
}

// Function to get all courses
async function getAllCourses(): Promise<string[]> {
  return listDirectories(BASE_PATH);
}

// Function to get a single course
async function getCourse(courseName: string): Promise<string | null> {
  const courses = await getAllCourses();
  if (courses.includes(courseName)) {
    return courseName;
  }
  return null;
}

// We'll need to be able to get a list of the modules — in order — from the config
async function getAllModulesInOrder(courseName: string): Promise<any> {
  const configPath = path.join(
    BASE_PATH,
    `${courseName}/tutorial-site/config.js`,
  );
  try {
    const config = require(configPath);
    console.log(config);
    return config;
  } catch (error) {
    console.error("Error loading config:", error);
    throw error;
  }
}

// Function to get all modules within a course
async function getAllModules(courseName: string): Promise<string[]> {
  const courseContent = `${courseName}/tutorial-site/content`;
  const coursePath = path.join(BASE_PATH, courseContent);

  const [dirPaths, filePaths] = await Promise.all([
    listDirectories(coursePath).then((dirs) =>
      dirs.map((dir) => path.join(coursePath, dir)),
    ),
    listFiles(coursePath)
      .then((files) =>
        files.filter((file) => file.endsWith(".md") || file.endsWith(".mdx")),
      )
      .then((files) => files.map((file) => path.join(coursePath, file))),
  ]);

  return [...dirPaths, ...filePaths];
}

// Function to get a single module
async function getModule(
  courseName: string,
  moduleName: string,
): Promise<string[] | null> {
  const modules = await getAllModules(courseName);
  const modulePaths = modules.filter((m) => m.includes(`${moduleName}`));
  if (modulePaths.length > 0) {
    return modulePaths;
  }
  return null;
}

async function getAllLessons(
  courseName: string,
  moduleName: string,
): Promise<string[]> {
  const modulePaths = await getModule(courseName, moduleName);
  if (!modulePaths) {
    return [];
  }

  const lessons = [];
  for (const modulePath of modulePaths) {
    if (fs.lstatSync(modulePath).isDirectory()) {
      const files = await listFiles(modulePath);
      lessons.push(...files);
    } else if (
      path.basename(modulePath, path.extname(modulePath)) === moduleName
    ) {
      lessons.push(modulePath);
    }
  }
  return lessons;
}

// Function to get a single lesson
async function getLesson(
  courseName: string,
  moduleName: string,
  fileName: string,
): Promise<any> {
  const filePathInModule = path.join(
    BASE_PATH,
    `${courseName}/tutorial-site/content/${moduleName}/${fileName}`,
  );
  const filePathInContent = path.join(
    BASE_PATH,
    `${courseName}/tutorial-site/content/${fileName}`,
  );

  let content;
  if (fs.existsSync(filePathInModule)) {
    content = await fs.promises.readFile(filePathInModule, "utf8");
  } else if (fs.existsSync(filePathInContent)) {
    content = await fs.promises.readFile(filePathInContent, "utf8");
  } else {
    throw new Error(
      `File ${fileName} not found in module ${moduleName} or content directory`,
    );
  }

  return matter(content);
}

// How we can use this
async function main() {
  console.log(await getAllCourses());
  console.log(await getAllModules("hasura-v3"));
  console.log(await getAllModulesInOrder("hasura-v3"));
  // console.log(await getModule("hasura-v3", "builds"));
  // console.log(await getAllLessons("hasura-v3", "builds"));
  // console.log(await getLesson("hasura-v3", "builds", "1-create-a-build.md"));
}

main();