File sync

file_sync.cpp

/*
# Author: Qichunren
# Email:  whyruby@gmail.com
*/

#include "files_sync.h"
#include "nt_config.h"
#include "nt_utils.h"
#include <QDir>
#include <QTimer>

#include <QNetworkReply>
#include <QJsonDocument>
#include <QJsonArray>
#include <QJsonObject>

FilesSync::FilesSync(QObject *parent) :
    QObject(parent),
    _reply(nullptr),
    _download_reply(nullptr),
    _request_queue_timer(new QTimer(this)),
    _initial_sync(true)
{
    connect(_request_queue_timer, SIGNAL(timeout()), this, SLOT(request_queue_slot()));
}

void FilesSync::httpFinished()
{
    qDebug() << "/api/v1/files.json httpFinished";
    if(_reply->error()) {
        qDebug() << "Got error in http files.json finished.";
        _reply->deleteLater();
        _reply = nullptr;
        return;
    }

    QByteArray data = _reply->readAll();
    QString url_str = _reply->url().toString();
    if(url_str.contains("/api/v1/files.json"))
    {
        process_files_json_data(data);
    }
    else
    {
        qDebug() << "Not handle url reply yet:" << url_str;
    }
    _reply->deleteLater();
    _reply = nullptr;
}

void FilesSync::httpDownloadFinished()
{
    qDebug() << "httpDownloadFinished";
    if(_download_reply->error()) {
        qDebug() << "Got error in http download finished." << _download_reply->errorString();
        _download_reply->deleteLater();
        _download_reply = nullptr;
        return;
    }

    QByteArray data = _download_reply->readAll();
    QString url_str = _download_reply->url().toString();
    if(url_str.contains("/api/v1/files/"))
    {
        // http://192.168.10.198:3000/api/v1/files/PIS.db -> PIS.db
        // http://192.168.10.198:3000/api/v1/files/audios/Hello.mp3 -> audios/Hello.mp3
        const QString file_name = url_str.section('/', 6, -1);
        process_file_download(data, file_name);
    }
    else
    {
        qDebug() << "Not handle url reply yet:" << url_str;
    }
    _download_reply->deleteLater();
    _download_reply = nullptr;
}

void FilesSync::httpReadyRead()
{
    // qDebug() << "files.json ready read.";
}

void FilesSync::httpDownloadReadyRead()
{
    // qDebug() << "download ready read.";
}

void FilesSync::request_queue_slot()
{
    if(_request_queue.isEmpty())
    {
        _request_queue_timer->stop();
        return;
    }
    if(_download_reply && !_download_reply->isFinished())
    {
        return;
    }
    else
    {
        QString file_url = _request_queue.takeFirst();
        const QUrl url(file_url);
        start_download_request(url);
    }
}

void FilesSync::start_request(const QUrl &requestedUrl)
{
    _reply = _network_access_manager.get(QNetworkRequest(requestedUrl));
    connect(_reply, SIGNAL(finished()), this, SLOT(httpFinished()));
    connect(_reply, SIGNAL(readyRead()), this, SLOT(httpReadyRead()));
}

void FilesSync::start_download_request(const QUrl &requestedUrl)
{
    _download_reply = _network_access_manager.get(QNetworkRequest(requestedUrl));
    connect(_download_reply, SIGNAL(finished()), this, SLOT(httpDownloadFinished()));
    connect(_download_reply, SIGNAL(readyRead()), this, SLOT(httpDownloadReadyRead()));
}

void FilesSync::sync_files(const QString & pis_ip)
{
    if(_request_queue_timer->isActive())
    {
        return;
    }
    _host_ip = pis_ip;
    qDebug() << "sync_files" << pis_ip;
    const QUrl url(QString("http://%1:3000/api/v1/files.json").arg(pis_ip));
    start_request(url);
}

void FilesSync::download(const QString & file_url)
{
    qDebug() << "Download file:" << file_url;
    _request_queue.append(file_url);
    if(!_request_queue_timer->isActive())
    {
        _request_queue_timer->start(500);
    }
}

void FilesSync::process_files_json_data(const QByteArray & json_data)
{
    QJsonDocument files_json_doc = QJsonDocument::fromJson(json_data);
    if(files_json_doc.isNull())
    {
        qDebug() << "Got invalid json data from files.json, data size:" << json_data.size();
        return;
    }
    qDebug() << "httpReadyRead files_json is valid, data size:" << json_data.size();// << files_json_doc;
    if(files_json_doc.isObject())
    {
        QJsonObject files_json_doc_object = files_json_doc.object();
        QJsonArray files_value_array = files_json_doc_object["files"].toArray();
        qDebug() << "File count:" << files_value_array.size();
        for(int i = 0; i < files_value_array.size(); i++)
        {
            QJsonValue value = files_value_array.at(i);
            if(!value.isObject())
            {
                return;
            }
            QJsonObject file_json_object = value.toObject();
            QString file_name = file_json_object["name"].toString();
            QString data_dir_path = NTConfig.data_dir_path();
            QDir data_dir = QDir(data_dir_path);
            QString full_file_name = data_dir.filePath(file_name);
            const QFileInfo file_info(full_file_name);
            // qDebug() << "File:" <<  file_name;
            const QString file_url = QString("http://%1:3000/api/v1/files/%2").arg(_host_ip).arg(file_name);
            if(!file_info.exists())
            {
                qDebug() << "File" << full_file_name << "not exist, try download it";
                download(file_url);
            }
            else
            {
                QByteArray md5_bytes = NtUtils::file_check_sum(full_file_name, QCryptographicHash::Md5);
                QString file_md5 = QString(md5_bytes.toHex());
                if(file_md5 != file_json_object["md5"].toString() )
                {
                    qDebug() << "File" << full_file_name << "is diff, try sync from remote";
                    download(file_url);
                }
            }
        }
    }
}

void FilesSync::process_file_download(const QByteArray & data, const QString & saved_file_name)
{
    QString data_dir_path = NTConfig.data_dir_path();
    QDir data_dir(data_dir_path);

    // "ppp_test/上行_盛华_离站.mp3" -> ppp_test
    // "ppp_test2/audios/上行_盛华_离站.mp3" -> ppp_test2/audios
    // "上行_盛华_离站.mp3" -> ""
    QString sub_dir = saved_file_name.section('/', 0, -2);
    if(!sub_dir.isEmpty())
    {
        data_dir.mkpath(sub_dir);
    }
    QString full_file_path = data_dir.filePath(saved_file_name);
    qDebug() << "Try to save" << saved_file_name << "to" << full_file_path;
    QFile dfile(full_file_path);
    if(dfile.open(QIODevice::WriteOnly))
    {
        dfile.write(data);
        dfile.close();
        qDebug() << "Success save" << saved_file_name;
    }
    else
    {
        qDebug() << "failed to save" << saved_file_name;
    }
}

file_sync.h

/*
# Author: Qichunren
# Email:  whyruby@gmail.com
*/

#ifndef FILES_SYNC_H
#define FILES_SYNC_H

#include <QObject>
#include <QNetworkAccessManager>

class QNetworkReply;
class QFile;
class QTimer;

/**
 * @brief The FilesSync class get files list with http requst from master PIS host, sync files diff in 120 seconds.
 */
class FilesSync : public QObject
{
    Q_OBJECT
public:
    explicit FilesSync(QObject *parent = nullptr);
    void sync_files(const QString & pis_ip);
    void download(const QString & file_url);

signals:

private slots:
    void httpFinished();
    void httpReadyRead();

    void httpDownloadFinished();
    void httpDownloadReadyRead();

    void request_queue_slot();

private:
    void start_request(const QUrl &requestedUrl);
    void start_download_request(const QUrl &requestedUrl);
    void process_files_json_data(const QByteArray & json_data);
    void process_file_download(const QByteArray & data, const QString & saved_file_name);

    QList<QString> _request_queue;
    QNetworkAccessManager _network_access_manager;
    QNetworkReply * _reply;
    QNetworkReply * _download_reply;
    QTimer * _request_queue_timer;
    QString _host_ip;
    bool _initial_sync;
};

#endif // FILES_SYNC_H