Simple sloppy directory-based sqlite VFS. MIT License

dirfs.cxx

/* Copyright © 2018 Taylor C. Richberger <taywee@gmx.com>
 * This code is released under the MIT license
 */

#include <memory>
#include <iostream>
#include <iterator>
#include <sstream>
#include <fstream>
#include <vector>
#include <algorithm>

#include <dirent.h>
#include <fcntl.h>
#include <climits>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <sys/stat.h>
#include <unistd.h>

#include <sqlite3.h>

class DirFile : public sqlite3_file {
    private:
        const std::unique_ptr<sqlite3_io_methods> methods;
        const std::string path;
        const int flags;
        const size_t blocksize;
        std::vector<char> buffer;
        bool blockopen;
        bool blockmodified;
        size_t openblock;
        size_t blockcount;

    public:
        DirFile(const std::string &path, int flags, size_t blocksize) : methods(new sqlite3_io_methods), path(path), flags(flags), blocksize(blocksize), blockopen(false), blockcount(0) {
            pMethods = methods.get();
            methods->iVersion = 1;
            methods->xClose = xxClose;
            methods->xRead = xxRead;
            methods->xWrite = xxWrite;
            methods->xTruncate = xxTruncate;
            methods->xSync = xxSync;
            methods->xFileSize = xxFileSize;
            methods->xLock = xxLock;
            methods->xUnlock = xxUnlock;
            methods->xCheckReservedLock = xxCheckReservedLock;
            methods->xFileControl = xxFileControl;
            methods->xSectorSize = xxSectorSize;
            methods->xDeviceCharacteristics = xxDeviceCharacteristics;
            buffer.resize(blocksize);

            std::ostringstream blockcountfile;
            blockcountfile << path << "/blockcount";
            std::ifstream input(blockcountfile.str());
            if (input) {
                input >> blockcount;
            } else {
                std::ofstream output(blockcountfile.str());
                output << blockcount;
            }
            std::cout << "Opened " << path << " with blockcount of " << blockcount << std::endl;
        }

        ~DirFile() {
            Sync();
        }

        static int xxClose(sqlite3_file *file) {
            static_cast<DirFile *>(file)->~DirFile();
            return SQLITE_OK;
        }

        template <typename ...Types>
        static int xxRead(sqlite3_file *file, Types...args) {
            return static_cast<DirFile *>(file)->Read(args...);
        }

        template <typename ...Types>
        static int xxWrite(sqlite3_file *file, Types...args) {
            return static_cast<DirFile *>(file)->Write(args...);
        }

        template <typename ...Types>
        static int xxTruncate(sqlite3_file *file, Types...args) {
            return static_cast<DirFile *>(file)->Truncate(args...);
        }

        static int xxSync(sqlite3_file *file, int) {
            static_cast<DirFile *>(file)->Sync();
            return SQLITE_OK;
        }

        template <typename ...Types>
        static int xxFileSize(sqlite3_file *file, Types...args) {
            return static_cast<DirFile *>(file)->FileSize(args...);
        }

        template <typename ...Types>
        static int xxLock(sqlite3_file *file, Types...args) {
            return static_cast<DirFile *>(file)->Lock(args...);
        }

        template <typename ...Types>
        static int xxUnlock(sqlite3_file *file, Types...args) {
            return static_cast<DirFile *>(file)->Unlock(args...);
        }

        template <typename ...Types>
        static int xxCheckReservedLock(sqlite3_file *file, Types...args) {
            return static_cast<DirFile *>(file)->CheckReservedLock(args...);
        }

        template <typename ...Types>
        static int xxFileControl(sqlite3_file *file, Types...args) {
            return static_cast<DirFile *>(file)->FileControl(args...);
        }

        template <typename ...Types>
        static int xxSectorSize(sqlite3_file *file, Types...args) {
            return static_cast<DirFile *>(file)->SectorSize(args...);
        }

        static int xxDeviceCharacteristics(sqlite3_file *file) {
            return static_cast<DirFile *>(file)->DeviceCharacteristics();
        }

        /**
         * Sync the buffer to the currently open block if necessary.
         *
         * blockmodified is always false after this call.
         */
        void Sync() {
            std::cout << "Sync " << path << std::endl;
            if (blockopen && blockmodified) {
                std::cout << "Really sync" << std::endl;
                std::ostringstream filename;
                filename << path << '/' << openblock;
                std::ofstream output(filename.str(), std::ios_base::binary);
                output.write(buffer.data(), buffer.size());
                blockmodified = false;
            }
        }

        /**
         * Ensure the size is at least minsize, resizing if necessary.
         *
         * This does not touch the existing block or its buffer at all.
         */
        void EnsureSize(const size_t minsize) {
            std::cout << "EnsureSize " << minsize << std::endl;
            bool resized = false;
            for (; blockcount < minsize; ++blockcount) {
                std::ostringstream filename;
                filename << path << '/' << blockcount;
                std::ofstream output(filename.str(), std::ios_base::binary);
                std::fill_n(std::ostream_iterator<char>(output), blocksize, '\0');
                resized = true;
            }
            if (resized) {
                std::ostringstream filename;
                filename << path << "/blockcount";
                std::ofstream output(filename.str());
                output << blockcount;
            }
        }

        /// Open the named block, flushing the current if necessary.
        void OpenBlock(const size_t blocktoopen, bool write) {
            std::cout << "OpenBlock " << blocktoopen << ", write? " << write << std::endl;
            if (!blockopen || (blocktoopen != openblock)) {
                std::cout << "Actually opening block" << std::endl;
                Sync();
                if (write) {
                    EnsureSize(blocktoopen + 1);
                }
                std::ostringstream filename;
                filename << path << '/' << blocktoopen;
                std::ifstream input(filename.str(), std::ios_base::binary);
                input.read(buffer.data(), buffer.size());
                if (static_cast<size_t>(input.gcount()) != blocksize) {
                    throw std::runtime_error("The file was not the right size");
                }
                blockopen = true;
                blockmodified = false;
                openblock = blocktoopen;
            }
        }

        int Read(void *zBuf, int iAmt, sqlite_int64 iOfst) {
            std::cout << "Read " << path << ' ' << iAmt << " from " << iOfst << std::endl;
            size_t blocktoopen = iOfst / blocksize;
            size_t blockoffset = iOfst % blocksize;

            char *buf = reinterpret_cast<char *>(zBuf);

            while (iAmt > 0) {
                if (blocktoopen >= blockcount) {
                    return SQLITE_IOERR_SHORT_READ;
                }

                OpenBlock(blocktoopen, false);

                size_t thisreadamount;

                auto it = buffer.begin();
                std::advance(it, blockoffset);

                // We need more than just this block
                if (iAmt + blockoffset > blocksize) {
                    std::cout << "iamt: " << iAmt << std::endl;
                    std::cout << "blocksize: " << blocksize << std::endl;
                    std::cout << "blockoffset: " << blockoffset << std::endl;
                    thisreadamount = blocksize - blockoffset;
                    std::cout << "thisreadamount: " << thisreadamount << std::endl;
                    ++blocktoopen;
                    blockoffset = 0;
                } else {
                    thisreadamount = iAmt;
                }

                std::copy_n(it, thisreadamount, buf);
                iAmt -= thisreadamount;
                buf += thisreadamount;
            }
            return SQLITE_OK;
        }

        int Write(const void *zBuf, int iAmt, sqlite_int64 iOfst) {
            std::cout << "Write " << path << ' ' << iAmt << " at " << iOfst << std::endl;
            size_t blocktoopen = iOfst / blocksize;
            size_t blockoffset = iOfst % blocksize;


            const char *buf = reinterpret_cast<const char *>(zBuf);

            while (iAmt > 0) {
                // If the buffer doesn't contain the target block
                OpenBlock(blocktoopen, true);

                size_t thiswriteamount;

                auto it = buffer.begin();
                std::advance(it, blockoffset);

                // We need more than just this block
                if (iAmt + blockoffset > blocksize) {
                    std::cout << "iamt: " << iAmt << std::endl;
                    std::cout << "blocksize: " << blocksize << std::endl;
                    std::cout << "blockoffset: " << blockoffset << std::endl;
                    thiswriteamount = blocksize - blockoffset;
                    std::cout << "thiswriteamount: " << thiswriteamount << std::endl;
                    ++blocktoopen;
                    blockoffset = 0;
                } else {
                    thiswriteamount = iAmt;
                }

                std::copy_n(buf, thiswriteamount, it);
                blockmodified = true;

                iAmt -= thiswriteamount;
                buf += thiswriteamount;
            }
            return SQLITE_OK;
        }

        int Truncate(sqlite_int64 size) {
            std::cout << "Truncate" << std::endl;
            const size_t newblocksize = (size + blocksize - 1) / blocksize;
            for (size_t deleteblock = newblocksize; deleteblock < blockcount; ++deleteblock) {
                std::ostringstream filename;
                filename << path << '/' << deleteblock;
                remove(filename.str().c_str());
            }
            blockcount = newblocksize;
            std::ostringstream filename;
            filename << path << "/blockcount";
            std::ofstream output(filename.str());
            output << blockcount;
            if (blockcount <= openblock) {
                blockmodified = false;
                blockopen = false;
            }
            return SQLITE_OK;
        }

        int FileSize(sqlite_int64 *pSize) {
            std::cout << "FileSize " << path << std::endl;
            *pSize = blockcount * blocksize;
            return SQLITE_OK;
        }

        int Lock(int) {
            std::cout << "Lock " << path << std::endl;
            return SQLITE_OK;
        }

        int Unlock(int) {
            std::cout << "Unlock " << path << std::endl;
            return SQLITE_OK;
        }

        int CheckReservedLock(int *pResOut) {
            std::cout << "CheckReservedLock " << path << std::endl;
            *pResOut = 0;
            return SQLITE_OK;
        }

        int FileControl(int, void *) {
            std::cout << "FileControl " << path << std::endl;
            return SQLITE_OK;
        }

        int SectorSize() {
            std::cout << "SectorSize " << path << std::endl;
            return 0;
        }

        int DeviceCharacteristics() {
            std::cout << "DeviceCharacteristics " << path << std::endl;
            return 0;
        }
};

class DirVFS : public sqlite3_vfs {
    public:
        DirVFS() {
            iVersion = 1;
            szOsFile = sizeof(DirFile);
            mxPathname = PATH_MAX - 1;
            zName = "dirvfs";
            pAppData = nullptr;
            xOpen = Open;
            xDelete = Delete;
            xAccess = Access;
            xFullPathname = FullPathname;
        }

        static int Open(sqlite3_vfs * /*pVfs*/, const char *zName, sqlite3_file *pFile, int flags, int *pOutFlags) {
            std::cout << "Opening " << zName << std::endl;
            std::cout << "Value for this: " << reinterpret_cast<const void*>(sqlite3_uri_parameter(zName, "this")) << std::endl;
            if (zName == 0) {
                return SQLITE_IOERR;
            }

            static const unsigned int RWCREATE = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE;
            const bool rwcreate = (flags & RWCREATE) == RWCREATE;
            const bool rw = (flags & SQLITE_OPEN_READWRITE) != 0;
            const bool ro = (flags & SQLITE_OPEN_READONLY) != 0;
            DIR *dir = opendir(zName);

            if (!dir) {
                if (rwcreate) {
                    mkdir(zName, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
                } else {
                    return SQLITE_CANTOPEN;
                }
                dir = opendir(zName);

                if (!dir) {
                    return SQLITE_CANTOPEN;
                }
            }
            closedir(dir);

            int oflags = 0;

            if (ro) {
                oflags |= O_RDONLY;
            } else if (rw) {
                oflags |= O_RDWR;
            }

            if (pOutFlags) {
                *pOutFlags = flags;
            }

            // TODO: make the block size configurable
            new (pFile) DirFile(zName, oflags, 4096);
            return SQLITE_OK;
        }

        static int Delete(sqlite3_vfs * /*pVfs*/, const char *zName, int /* syncDir */) {
            std::cout << "Delete" << std::endl;
            const std::string name{zName};
            DIR *dir = opendir(zName);
            if (!dir) {
                return SQLITE_IOERR_DELETE;
            }
            errno = 0;
            for (dirent *entry = readdir(dir); entry != nullptr; entry = readdir(dir)) {
                const std::string filename{entry->d_name};
                if (filename == "." || filename == "..") {
                    continue;
                }
                const std::string path = name + "/" + filename;
                if (remove(path.c_str()) == -1) {
                    return SQLITE_IOERR_DELETE;
                }
                
            }
            if (errno) {
                return SQLITE_IOERR_DELETE;
            }
            closedir(dir);
            if (remove(name.c_str()) == -1) {
                return SQLITE_IOERR_DELETE;
            }
            return SQLITE_OK;
        }

        static int Access(sqlite3_vfs * /*pVfs*/, const char *zName, int flags, int *pResOut) {
            std::cout << "Access" << std::endl;
            int mode;
            switch (flags) {
                case SQLITE_ACCESS_EXISTS:
                    mode = F_OK;
                    break;
                case SQLITE_ACCESS_READ:
                    mode = R_OK | X_OK;
                    break;
                case SQLITE_ACCESS_READWRITE:
                    mode = R_OK | W_OK | X_OK;
                    break;
                default:
                    return SQLITE_IOERR;
                    break;
            }
            const int rc = access(zName, mode);
            *pResOut = (rc == 0);
            return SQLITE_OK;
        }

        static int FullPathname(sqlite3_vfs * pVfs, const char *zName, const int nPathOut, char *zPathOut) {
            std::cout << "FullPathname" << std::endl;
            std::ostringstream outpath;

            if (zName[0] != '/') {
                char cwd[pVfs->mxPathname];
                if (!getcwd(cwd, pVfs->mxPathname)) {
                    return SQLITE_IOERR;
                }
                outpath << cwd << '/';
            }
            if (zName[0] != '\0') {
                outpath << zName;
            }

            const auto strpath = outpath.str();

            if (strpath.size() < static_cast<const unsigned int>(nPathOut)) {
                std::copy(strpath.begin(), strpath.end(), zPathOut);
                zPathOut[strpath.size()] = '\0';
            } else {
                return SQLITE_IOERR;
            }

            return SQLITE_OK;
        }
};

int main(int argc, char **argv)
{
    DirVFS dirvfs;
    sqlite3_vfs_register(&dirvfs, 0);
    sqlite3 *db;
    int rc = sqlite3_open_v2(argv[1], &db, SQLITE_OPEN_URI | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, "dirvfs");
    std::cout << "rc: " << rc << std::endl;
    sqlite3_stmt *statement;
    rc = sqlite3_prepare_v2(db, "CREATE TABLE foo (a INTEGER)", -1, &statement, nullptr);
    std::cout << "rc: " << rc << std::endl;
    rc = sqlite3_step(statement);
    std::cout << "rc: " << rc << std::endl;
    rc = sqlite3_close(db);
    std::cout << "rc: " << rc << std::endl;
    return 0;
}