statamic.dev reference to Dash Docset

statamicDevReferenceDashDocset.js

// npm install axios cheerio sqlite3 plist fs-extra
// WIP, too busy to finish this
const axios = require('axios');
const cheerio = require('cheerio');
const fs = require('fs-extra');
const path = require('path');
const sqlite3 = require('sqlite3').verbose();
const plist = require('plist');

const BASE_URL = 'https://statamic.dev';
const SECTIONS = ['Fieldtypes', 'Modifiers', 'Repositories', 'Tags', 'Variables', 'Widgets'];
const OUTPUT_DIR = path.join(__dirname, 'statamic-reference');
const DOCSET_DIR = path.join(__dirname, 'Statamic.docset');
const DOCUMENTS_DIR = path.join(DOCSET_DIR, 'Contents', 'Resources', 'Documents');

async function downloadPage(url) {
    const response = await axios.get(url);
    return response.data;
}

async function downloadDocumentation() {
    const mainUrl = `${BASE_URL}/reference`;
    await fs.ensureDir(OUTPUT_DIR);
    const mainPage = await downloadPage(mainUrl);
    await fs.outputFile(path.join(OUTPUT_DIR, 'index.html'), mainPage);
    console.log('Downloaded main documentation page');
}

async function parseAndDownloadLinks() {
    const indexPath = path.join(OUTPUT_DIR, 'index.html');
    const content = await fs.readFile(indexPath, 'utf-8');
    const $ = cheerio.load(content);

    const links = [];

    $('#side-nav nav button').each((i, elem) => {
        const sectionName = $(elem).text().trim();
        if (SECTIONS.includes(sectionName)) {
            $(elem).next('ul').find('a').each((j, linkElem) => {
                const link = $(linkElem).attr('href');
                const title = $(linkElem).text().trim();
                if (title && !title.startsWith('All')) {
                    links.push({ section: sectionName, title, link: `${BASE_URL}${link}` });
                }
            });
        }
    });

    for (const linkObj of links) {
        const outputPath = path.join(OUTPUT_DIR, linkObj.section, `${linkObj.title}.html`);
        if (!fs.existsSync(outputPath)) {
            const pageContent = await downloadPage(linkObj.link);
            await fs.outputFile(outputPath, pageContent);
            console.log(`Downloaded ${linkObj.title}`);
        }
    }

    console.log('Downloaded all linked pages');
    return links;
}

async function extractContent(entries) {
    for (const entry of entries) {
        const entryPath = path.join(OUTPUT_DIR, entry.section, `${entry.title}.html`);
        const content = await fs.readFile(entryPath, 'utf-8');
        const $ = cheerio.load(content);

        // Remove unwanted sections
        $('#breadcrumbs, #table-of-contents').remove();

        // Update internal links
        $('a').each((i, link) => {
            const href = $(link).attr('href');
            if (href && href.startsWith('/')) {
                const newHref = href.replace('/', '');
                $(link).attr('href', newHref + '.html');
            }
        });

        // Extract and save content
        const extractedContent = $('#content').html();
        const docPath = path.join(DOCUMENTS_DIR, entry.section, `${entry.title}.html`);
        await fs.outputFile(docPath, extractedContent);

        console.log(`Extracted content for ${entry.title}`);
    }
}

function createSQLiteIndex(entries) {
    const dbPath = path.join(DOCSET_DIR, 'Contents', 'Resources', 'docSet.dsidx');
    const db = new sqlite3.Database(dbPath);

    db.serialize(() => {
        db.run('CREATE TABLE searchIndex(id INTEGER PRIMARY KEY, name TEXT, type TEXT, path TEXT);');
        db.run('CREATE UNIQUE INDEX anchor ON searchIndex (name, type, path);');

        const stmt = db.prepare('INSERT OR IGNORE INTO searchIndex(name, type, path) VALUES (?, ?, ?)');
        entries.forEach(entry => {
            const docPath = path.join(entry.section, `${entry.title}.html`);
            stmt.run(entry.title, entry.section, docPath);
        });
        stmt.finalize();
    });

    db.close();
}

function createPlist() {
    const plistContent = plist.build({
        CFBundleIdentifier: 'statamic',
        CFBundleName: 'Statamic',
        DocSetPlatformFamily: 'statamic',
        isDashDocset: true,
        dashIndexFilePath: 'index.html'
    });

    const plistPath = path.join(DOCSET_DIR, 'Contents', 'Info.plist');
    fs.outputFileSync(plistPath, plistContent);
    console.log('Created Info.plist');
}

(async () => {
    await downloadDocumentation();
    const docEntries = await parseAndDownloadLinks();

    await fs.ensureDir(DOCUMENTS_DIR);
    await extractContent(docEntries);

    createSQLiteIndex(docEntries);
    createPlist();

    console.log('Docset creation complete');
})();