OPNsense WebDAV backup plugin

WebDAV.php

<?php
///usr/local/opnsense/mvc/app/library/OPNsense/Backup/WebDAV.php
/*
 * Copyright (C) 2018 Deciso B.V.
 * Copyright (C) 2018 Fabian Franz
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
 * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

namespace OPNsense\Backup;

use OPNsense\Core\Config;

/**
 * Class WebDAV backup
 * @package OPNsense\Backup
 */
class WebDAV extends Base implements IBackupProvider
{
    /**
     * get required (user interface) fields for backup connector
     * @return array configuration fields, types and description
     */
    public function getConfigurationFields()
    {
        $fields = array(
            array(
                "name" => "enabled",
                "type" => "checkbox",
                "label" => gettext("Enable"),
                "value" => null
            ),
            array(
                "name" => "url",
                "type" => "text",
                "label" => gettext("URL"),
                "help" => gettext("The Base URL to WebDAV without trailing slash. For example: https://dav.example.com"),
                "value" => null
            ),
            array(
                "name" => "user",
                "type" => "text",
                "label" => gettext("User Name"),
                "help" => gettext("The name you use for logging into your WebDAV account"),
                "value" => null
            ),
            array(
                "name" => "password",
                "type" => "password",
                "label" => gettext("Password"),
                "help" => gettext("The password you use for logging into your WebDAV account"),
                "value" => null
            ),
            array(
                "name" => "password_encryption",
                "type" => "password",
                "label" => gettext("Encryption Password (Optional)"),
                "help" => gettext("A password to encrypt your configuration"),
                "value" => null
            ),
            array(
                "name" => "backupdir",
                "type" => "text",
                "label" => gettext("Directory Name without leading slash, starting from user's root"),
                "value" => 'OPNsense-Backup'
            )
        );
        $webdav = new WebDAVSettings();
        foreach ($fields as &$field) {
            $field['value'] = (string)$webdav->getNodeByReference($field['name']);
        }
        return $fields;
    }

    /**
     * backup provider name
     * @return string user friendly name
     */
    public function getName()
    {
        return gettext("WebDAV");
    }

    /**
     * validate and set configuration
     * @param array $conf configuration array
     * @return array of validation errors when not saved
     * @throws \OPNsense\Base\ModelException
     * @throws \ReflectionException
     */
    public function setConfiguration($conf)
    {
        $webdav = new WebDAVSettings();
        $this->setModelProperties($webdav, $conf);
        $validation_messages = $this->validateModel($webdav);
        if (empty($validation_messages)) {
            $webdav->serializeToConfig();
            Config::getInstance()->save();
        }
        return $validation_messages;
    }

    /**
     * perform backup
     * @return array filelist
     * @throws \OPNsense\Base\ModelException
     * @throws \ReflectionException
     */
    public function backup()
    {
        $cnf = Config::getInstance();
        $webdav = new WebDAVSettings();
        if ($cnf->isValid() && !empty((string)$webdav->enabled)) {
            $config = $cnf->object();
            $url = (string)$webdav->url;
            $username = (string)$webdav->user;
            $password = (string)$webdav->password;
            $backupdir = (string)$webdav->backupdir;
            $crypto_password = (string)$webdav->password_encryption;
            $hostname = $config->system->hostname . '.' . $config->system->domain;
            $configname = 'config-' . $hostname . '-' .  date('Y-m-d_H_i_s') . '.xml';
            // backup source data to local strings (plain/encrypted)
            $confdata = file_get_contents('/conf/config.xml');
            if (!empty($crypto_password)) {
                $confdata = $this->encrypt($confdata, $crypto_password);
            }
            // Check if destination directory exists, create (full path) if not
            try {
                $this->create_directory($url, $username, $password, $backupdir);
            } catch (\Exception $e) {
                return array();
            }
            try {
                $this->upload_file_content(
                    $url,
                    $username,
                    $password,
                    $backupdir,
                    $configname,
                    $confdata
                );
                // do not list directories
                return array_filter(
                    $this->listFiles($url, $username, $password, "/$backupdir/", false),
                    function ($filename) {
                        return (substr($filename, -1) !== '/');
                    }
                );
            } catch (\Exception $e) {
                return array();
            }
        }
    }

    /**
     * dir listing
     * @param string $url remote location
     * @param string $username username
     * @param string $password password to use
     * @param string $directory location to list
     * @param bool $only_dirs only list directories
     * @return array
     * @throws \Exception
     */
    public function listFiles($url, $username, $password, $directory = '/', $only_dirs = true)
    {
        $result = $this->curl_request(
            "$url$directory",
            $username,
            $password,
            'PROPFIND',
            "Error while fetching filelist from WebDAV '{$directory}' path"
        );
        //remove line breaks from xml string
        $xml = preg_replace("/\r?\n/", '', $result['response']);

        // workaround - simplexml seems to be broken when using namespaces - remove them.
        //$xml = str_replace(['<D:', '</D:', '<d:', '</d:', '<lp1:', '</lp1:'], ['<', '</', '<', '</', '<', '</'], $xml);
        // better workaround: remove any namespace
        $xml = preg_replace('/\s+xmlns:[^=]+="[^"]*"/', '', $xml);
        $xml = preg_replace('/\b(\w+)\:/i', '', $xml);

        $xml = simplexml_load_string($xml);
        $ret = array();

        //parse URL for a check if path exists
        $parsedUrl = parse_url($url);

        foreach ($xml->children() as $response) {
            // d:response
            if ($response->getName() == 'response') {

                $fileurl = (string)$response->href;

                //check if URL has a path
                if (isset($parsedUrl['path'])) {
                    //URL DOES have a path - extracting path from fileurl
                    $dirname = explode($parsedUrl['path'], $fileurl, 2)[1];
                }
                else {
                    //URL does NOT have a path
                    $dirname = $fileurl;
                }

                if (
                    $response->propstat->prop->resourcetype->children()->count() > 0 &&
                    $response->propstat->prop->resourcetype->children()[0]->getName() == 'collection' &&
                    $only_dirs
                ) {

                    $ret[] = $dirname;

                } elseif (!$only_dirs) {

                    $ret[] = $dirname;

                }
            }
        }
        return $ret;
    }

    /**
     * upload file
     * @param string $url remote location
     * @param string $username remote user
     * @param string $password password to use
     * @param string $backupdir remote directory
     * @param string $filename filename to use
     * @param string $local_file_content contents to save
     * @throws \Exception when upload fails
     */
    public function upload_file_content($url, $username, $password, $backupdir, $filename, $local_file_content)
    {
        $this->curl_request(
            $url . "/$backupdir/$filename",
            $username,
            $password,
            'PUT',
            'cannot execute PUT',
            $local_file_content
        );
    }

    /**
     * create new remote directory if doesn't exist
     * @param string $url remote location
     * @param string $username remote user
     * @param string $password password to use
     * @param string $backupdir remote directory
     * @throws \Exception when create dir fails
     */
    public function create_directory($url, $username, $password, $backupdir)
    {
        $parent_path = dirname($backupdir);
        
        try {
            $directories = $this->listFiles($url, $username, $password, "/{$parent_path}");
        } catch (\Exception $e) {
            if ($backupdir == ".") {
                // We cannot create root, if we reached here there's some other problem
                syslog(LOG_ERR, "Check WebDAV configuration parameters");
                return false;
            }
            // If error assume dir doesn't exist. Create parent folder
            if ($this->create_directory($url, $username, $password, $parent_path) === false) {
                throw new \Exception();
            }
        }

        // if path exists ok
        if (in_array("/{$backupdir}/", $directories)) {
            return;
        }
        
        // create backupdir, because path does not exist
        $this->curl_request(
            $url . "/{$backupdir}",
            $username,
            $password,
            'MKCOL',
            'cannot execute MKCOL'
        );
    }

    /**
     * @param string $url remote location
     * @param string $username remote user
     * @param string $password password to use
     * @param string $method http method, PUT, GET, ...
     * @param string $error_message message to log on failure
     * @param null|string $postdata http body
     * @param array $headers HTTP headers
     * @return array response status
     * @throws \Exception when request fails
     */
    public function curl_request(
        $url,
        $username,
        $password,
        $method,
        $error_message,
        $postdata = null,
        $headers = array('User-Agent: OPNsense Firewall')
    ) {
        //workaround for Hetzner Storagebox
        if ($method == "PROPFIND") {
        array_push($headers, 'Depth: 1');
        }
    $curl = curl_init();
        curl_setopt_array($curl, array(
            CURLOPT_URL => $url,
            CURLOPT_CUSTOMREQUEST => $method, // Create a file in WebDAV is PUT
            CURLOPT_RETURNTRANSFER => true, // Do not output the data to STDOUT
            CURLOPT_VERBOSE => 0,           // same here
            CURLOPT_MAXREDIRS => 0,         // no redirects
            CURLOPT_TIMEOUT => 60,          // maximum time: 1 min
            CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
            CURLOPT_USERPWD => $username . ":" . $password,
            CURLOPT_HTTPHEADER => $headers
        ));
        if ($postdata != null) {
            curl_setopt($curl, CURLOPT_POSTFIELDS, $postdata);
        }
        $response = curl_exec($curl);
        $err = curl_error($curl);
        $info = curl_getinfo($curl);
        if (!($info['http_code'] == 200 || $info['http_code'] == 207 || $info['http_code'] == 201) || $err) {
            syslog(LOG_ERR, $error_message);
            syslog(LOG_ERR, json_encode($info));
            throw new \Exception();
        }
        curl_close($curl);
        return array('response' => $response, 'info' => $info);
    }

    /**
     * Is this provider enabled
     * @return boolean enabled status
     * @throws \OPNsense\Base\ModelException
     * @throws \ReflectionException
     */
    public function isEnabled()
    {
        $webdav = new WebDAVSettings();
        return (string)$webdav->enabled === "1";
    }
}

WebDAVSettings.php

<?php
///usr/local/opnsense/mvc/app/models/OPNsense/Backup/WebDAVSettings.php
/**
 *    Copyright (C) 2018 Fabian Franz
 *
 *    All rights reserved.
 *
 *    Redistribution and use in source and binary forms, with or without
 *    modification, are permitted provided that the following conditions are met:
 *
 *    1. Redistributions of source code must retain the above copyright notice,
 *       this list of conditions and the following disclaimer.
 *
 *    2. Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *
 *    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
 *    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
 *    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 *    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
 *    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 *    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 *    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 *    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 *    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 *    POSSIBILITY OF SUCH DAMAGE.
 *
 */

namespace OPNsense\Backup;

use OPNsense\Base\BaseModel;

/**
 * Class WebDAV
 * @package Backup
 */
class WebDAVSettings extends BaseModel
{
}

WebDAVSettings.xml

<model>
    <mount>//system/backup/webdav</mount>
    <version>1.0.0</version>
    <description>OPNsense WebDAV Backup Settings</description>
    <items>
        <enabled type="BooleanField">
            <default>0</default>
            <Required>Y</Required>
        </enabled>
        <url type="TextField">
            <Required>N</Required>
            <mask>/^https?:\/\/.*[^\/]$/</mask>
            <ValidationMessage>The URL must be valid without a trailing slash. For example: https://dav.example.com</ValidationMessage>
            <Constraints>
                <check001>
                    <ValidationMessage>A URL for the WebDAV server must be set.</ValidationMessage>
                    <type>DependConstraint</type>
                    <addFields>
                        <field1>enabled</field1>
                    </addFields>
                </check001>
            </Constraints>
        </url>
        <user type="TextField">
            <Constraints>
                <check001>
                    <ValidationMessage>A user for the WebDAV server must be set.</ValidationMessage>
                    <type>DependConstraint</type>
                    <addFields>
                        <field1>enabled</field1>
                    </addFields>
                </check001>
            </Constraints>
        </user>
        <password type="TextField">
            <Constraints>
                <check001>
                    <ValidationMessage>A password for the WebDAV server must be set.</ValidationMessage>
                    <type>DependConstraint</type>
                    <addFields>
                        <field1>enabled</field1>
                    </addFields>
                </check001>
            </Constraints>
        </password>
        <password_encryption type="TextField">
            <Required>N</Required>
        </password_encryption>
        <backupdir type="TextField">
            <Required>Y</Required>
            <mask>/^([\w%+\-]+\/)*[\w+%\-]+$/</mask>
            <default>OPNsense-Backup</default>
            <ValidationMessage>The Backup Directory can only consist of alphanumeric characters, dash, underscores and slash. No leading or trailing slash.</ValidationMessage>
        </backupdir>
    </items>
</model>

File locations:
/usr/local/opnsense/mvc/app/library/OPNsense/Backup/WebDAV.php
/usr/local/opnsense/mvc/app/models/OPNsense/Backup/WebDAVSettings.php
/usr/local/opnsense/mvc/app/models/OPNsense/Backup/WebDAVSettings.xml

Hey @cretl! Just as a heads up, with the latest version of OPNsense (24.7.3), there's an issue with the script when the backup directory already exists. Logs show an HTTP error 405 with a "cannot execute MKCOL" error.

If I comment out lines 147-152 of WebDAV.php, which are responsible for trying to create the directory on each run of the script, the script will run successfully (so long as the backup directory exists). The script will also work correctly the first time, when the directory does NOT exist, but will fail every subsequent try with the above error.

I know the code was more or less a copy/paste from the stock Nextcloud code, so I'm not sure if you're willing to poke at it, but there's probably a better way to check for existing folders on the WebDAV server, which should fix this issue.

Hey @cretl! Just as a heads up, with the latest version of OPNsense (24.7.3), there's an issue with the script when the backup directory already exists. Logs show an HTTP error 405 with a "cannot execute MKCOL" error.

If I comment out lines 147-152 of WebDAV.php, which are responsible for trying to create the directory on each run of the script, the script will run successfully (so long as the backup directory exists). The script will also work correctly the first time, when the directory does NOT exist, but will fail every subsequent try with the above error.

I know the code was more or less a copy/paste from the stock Nextcloud code, so I'm not sure if you're willing to poke at it, but there's probably a better way to check for existing folders on the WebDAV server, which should fix this issue.

Hi, I tried it myself and can't find any error with my test setups (SFTPGo WebDAV Server and Hetzner Storage Box WebDAV Server) on version 24.7.3.
Did your setup work before?

I know there are different WebDAV implementations, and I also had to poke around a lot to get it to work for my use case (SFTPGo WebDAV Server and Hetzner Storage Box WebDAV Server). I had to change the way the directory checks are done. This may also be the reason why there is no official WebDAV backup plugin since the script depends heavily on the server's WebDAV implementation.

Update:
I think I know what I can try to change. I'll update this comment if I succeed.

I had previously been using Nextcloud, with the default plugin, which was working. I'm currently migrating my systems from Nextcloud to FileRun, hence the need for the WebDAV over Nextcloud plugin. FileRun works pretty seamlessly with Nextcloud's desktop software, but there may be some differences in WebDAV deployment between them.

While I was troubleshooting issues with FileRun's WebDAV setup, I also tested using Nextcloud's WebDAV connection with your WebDAV plugin (using the full directory that's normally hard-coded with the Nextcloud plugin), and it had the same error as I was getting with FileRun, with the same behavior (it would work the first time when no directory existed, but not again once the directory existed). I'm not sure if it's an implementation issue or otherwise, but I did find it odd that Nextcloud had the same issue when using your plugin, but worked without issue on the Nextcloud plugin.

I'm not sure why the "Try" command is breaking the script in this case, but if it's not an easily fixable solution (alternate means of directory checking / creating), I would almost exclude the directory creation from the code, and instead create a note in the plugin that the directory needs to exist prior to saving the credential info. I'd love to see this pulled into the repository as well (or add it to an unofficial repository that users can add manually, just to avoid manually copying files and adjusting ownership/permissions).

The problem is caused by the path in the URL (e.g. nextcloud/remote.php/dav/files/USERNAME/). The script does not recognize the folder if it already exists. In the first run it creates the folder if it does not exist. In the second run it tries to create the existing folder again, because it does not recognize it. This is why the MKCOL command fails. I will add a fix for this.

@JaredC01 Try the new revision of the script. It works for me with a "normal" WebDAV (SFTPGo) server and the Nextcloud WebDAV implementation.
The difference is, that it handles the "path" of the URL.

@JaredC01 Try the new revision of the script. It works for me with a "normal" WebDAV (SFTPGo) server and the Nextcloud WebDAV implementation. The difference is, that it handles the "path" of the URL.

Uploaded to my router and replaced my modified WebDAV.php file with the updated one above. Save/Test with existing directory works, as does directory creation with a new directory. Seems it's all working as expected now!

Edit: Tested on a second OPNsense router, and it's working there as well.

Great! Thanks for the feedback!