windows下Qt如何检测海量文件和文件夹的变动

[墨香年少 32]

watcherthread.h

#ifndef WATCHERTHREAD_H
#define WATCHERTHREAD_H

#include <QThread>
#include <QAtomicInt>
#include <windows.h>

class WatcherThread : public QThread
{
    Q_OBJECT
public:
    WatcherThread(const QString &path) : QThread(), _path(path + (path.endsWith(QLatin1Char('/')) ? QString() : QStringLiteral("/"))), _directory(0), _resultEvent(0), _stopEvent(0), _done(false){}
    ~WatcherThread();
    void stop();

protected:
    QString longWinPath(const QString &inpath);
    QString pathtoUNC(const QString &_str);
    void run();
    void watchChanges(size_t fileNotifyBufferSize,bool *increaseBufferSize);
    void closeHandle();

signals:
    void changed(const QString &path,FILE_NOTIFY_INFORMATION *);
    void lostChanges();
    void ready();

private:
    QString _path;
    HANDLE _directory;
    HANDLE _resultEvent;
    HANDLE _stopEvent;
    QAtomicInt _done;
    FILE_NOTIFY_INFORMATION *pFileNotifyBuffer;
};

#endif // WATCHERTHREAD_H

watcherthread.cpp

#include "watcherthread.h"

#include <QDir>
#include <QUrl>
#include <QFile>

#ifdef Q_OS_WIN
#include <windows.h>
#include <windef.h>
#include <winbase.h>
#include <fcntl.h>
#include <io.h>
#endif

#include <QtDebug>


void WatcherThread::watchChanges(size_t fileNotifyBufferSize,
                                 bool *increaseBufferSize)
{
    *increaseBufferSize = false;
    const QString longPath = _path;

    _directory = CreateFileW(
                longPath.toStdWString().data(),
                FILE_LIST_DIRECTORY,
                FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE,
                nullptr,
                OPEN_EXISTING,
                FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED,
                nullptr);

    if (_directory == INVALID_HANDLE_VALUE) {
        const auto error = GetLastError();
        qDebug() << "Failed to create handle ," << error;
        _directory = 0;
        return;
    }

    OVERLAPPED overlapped;
    overlapped.hEvent = _resultEvent;

    // QVarLengthArray ensures the stack-buffer is aligned like double and qint64.
    QVarLengthArray<char, 4096 * 10> fileNotifyBuffer;
    fileNotifyBuffer.resize(static_cast<int>(fileNotifyBufferSize));

    const size_t fileNameBufferSize = 4096;
    TCHAR fileNameBuffer[fileNameBufferSize];


    pFileNotifyBuffer = reinterpret_cast<FILE_NOTIFY_INFORMATION *>(fileNotifyBuffer.data());

    while (!_done) {
        ResetEvent(_resultEvent);
        DWORD dwBytesReturned = 0;
        if (!ReadDirectoryChangesW(_directory, pFileNotifyBuffer,
                                   static_cast<DWORD>(fileNotifyBufferSize), true,
                                   FILE_NOTIFY_CHANGE_FILE_NAME
                                   | FILE_NOTIFY_CHANGE_DIR_NAME
                                   | FILE_NOTIFY_CHANGE_LAST_WRITE
                                   | FILE_NOTIFY_CHANGE_ATTRIBUTES, // attributes are for vfs pin state changes
                                   &dwBytesReturned,
                                   &overlapped,
                                   nullptr)) {
            const DWORD errorCode = GetLastError();
            if (errorCode == ERROR_NOTIFY_ENUM_DIR) {
                qDebug() << "The buffer for changes overflowed! Triggering a generic change and resizing";
                emit changed(_path,pFileNotifyBuffer);
                *increaseBufferSize = true;
            } else {
                qDebug() << "ReadDirectoryChangesW error";
            }
            break;
        }

        emit ready();

        HANDLE handles[] = { _resultEvent, _stopEvent };
        DWORD result = WaitForMultipleObjects(
                    2, handles,
                    false, // awake once one of them arrives
                    INFINITE);
        const auto error = GetLastError();
        if (result == 1) {
            qDebug() << "Received stop event, aborting folder watcher thread";
            break;
        }
        if (result != 0) {
            qDebug() << "WaitForMultipleObjects failed";
            break;
        }

        bool ok = GetOverlappedResult(_directory, &overlapped, &dwBytesReturned, false);
        if (!ok) {
            const DWORD errorCode = GetLastError();
            if (errorCode == ERROR_NOTIFY_ENUM_DIR) {
                qDebug() << "The buffer for changes overflowed! Triggering a generic change and resizing";
                emit lostChanges();
                emit changed(_path,pFileNotifyBuffer);
                *increaseBufferSize = true;
            } else {
                qDebug() << "GetOverlappedResult error";
            }
            break;
        }

        FILE_NOTIFY_INFORMATION *curEntry = pFileNotifyBuffer;
        forever {
            const int len = curEntry->FileNameLength / sizeof(wchar_t);
            QString longfile = longPath + QString::fromWCharArray(curEntry->FileName, len);

            // Unless the file was removed or renamed, get its full long name
            // TODO: We could still try expanding the path in the tricky cases...
            if (curEntry->Action != FILE_ACTION_REMOVED
                    && curEntry->Action != FILE_ACTION_RENAMED_OLD_NAME) {
                const auto wfile = longfile.toStdWString();
                const int longNameSize = GetLongPathNameW(wfile.data(), fileNameBuffer, fileNameBufferSize);
                const auto error = GetLastError();
                if (longNameSize > 0) {
                    longfile = QString::fromWCharArray(fileNameBuffer, longNameSize);
                } else {
                    qDebug() << "Error converting file name" << longfile << "to full length, keeping original name.";
                }
            }

#if QT_VERSION < QT_VERSION_CHECK(5, 14, 0)
            // The prefix is needed for native Windows functions before Windows 10, version 1607
            const bool hasLongPathPrefix = longPath.startsWith(QStringLiteral("\\\\?\\"));
            if (hasLongPathPrefix)
            {
                longfile.remove(0, 4);
            }
#endif
            longfile = QDir::cleanPath(longfile);

            // Skip modifications of folders: One of these is triggered for changes
            // and new files in a folder, probably because of the folder's mtime
            // changing. We don't need them.
            const bool skip = curEntry->Action == FILE_ACTION_MODIFIED
                    && QFileInfo(longfile).isDir();

            if (!skip) {
                emit changed(longfile,curEntry);
            } else {
                //qDebug() << "Skipping syncing of" << longfile;
            }

            if (curEntry->NextEntryOffset == 0) {
                break;
            }
            // FILE_NOTIFY_INFORMATION has no fixed size and the offset is in bytes therefor we first need to cast to char
            curEntry = reinterpret_cast<FILE_NOTIFY_INFORMATION *>(reinterpret_cast<char*>(curEntry) + curEntry->NextEntryOffset);
        }
    }

    CancelIo(_directory);
    closeHandle();
}

void WatcherThread::closeHandle()
{
    if (_directory) {
        CloseHandle(_directory);
        _directory = nullptr;
    }
}

void WatcherThread::run()
{
    _resultEvent = CreateEvent(nullptr, true, false, nullptr);
    _stopEvent = CreateEvent(nullptr, true, false, nullptr);

    // If this buffer fills up before we've extracted its data we will lose
    // change information. Therefore start big.
    size_t bufferSize = 4096 * 10;
    const size_t maxBuffer = 64 * 1024;

    while (!_done) {
        bool increaseBufferSize = false;
        watchChanges(bufferSize, &increaseBufferSize);

        if (increaseBufferSize) {
            bufferSize = qMin(bufferSize * 2, maxBuffer);
        } else if (!_done) {
            // Other errors shouldn't actually happen,
            // so sleep a bit to avoid running into the same error case in a
            // tight loop.
            sleep(2);
        }
    }
}

WatcherThread::~WatcherThread()
{
    closeHandle();
}

void WatcherThread::stop()
{
    _done = 1;
    SetEvent(_stopEvent);
}

Dlg.h

#ifndef DLG_H
#define DLG_H

#include <QDialog>
#include <QDir>
#include <QSet>
#include <windows.h>
#include "watcherthread.h"

QT_BEGIN_NAMESPACE
namespace Ui { class Dlg; }
QT_END_NAMESPACE

class Dlg : public QDialog
{
    Q_OBJECT

public:
    Dlg(QWidget *parent = nullptr);
    ~Dlg();

    void msg(QString);
signals:
    /** Emitted when one of the watched directories or one
     *  of the contained files is changed. */
    void pathChanged(const QString &path);

private slots:
    void on_btn_start_clicked();
    void changeDetected(const QString &,FILE_NOTIFY_INFORMATION *);
    void lostChanges();

    void on_btn_stop_clicked();

    void changeDetected(const QStringList &paths);

private:
    Ui::Dlg *ui;
    WatcherThread *_thread;

    QElapsedTimer _timer;
    QSet<QString> _lastPaths;

    void appendSubPaths(QDir dir, QStringList& subPaths);


};
#endif // DLG_H

Dlg.cpp

#pragma execution_character_set("utf-8")

#include "Dlg.h"
#include "ui_Dlg.h"


#include <stdlib.h>
#include <stdio.h>
#include <tchar.h>
#include <qDebug>
#include <QFileInfo>
#include <QDir>
#include <QSet>

Dlg::Dlg(QWidget *parent)
    : QDialog(parent)
    , ui(new Ui::Dlg)
{
    ui->setupUi(this);


}

Dlg::~Dlg()
{
    delete ui;
}

void Dlg::msg(QString txt)
{
    ui->plainTextEdit->appendHtml(txt+"\n");
}


void Dlg::on_btn_start_clicked()
{
    qDebug() << "开始......";
    _thread = new WatcherThread("D:\\TEST\\");

    connect(_thread, SIGNAL(changed(const QString &,FILE_NOTIFY_INFORMATION *)),this, SLOT(changeDetected(const QString &,FILE_NOTIFY_INFORMATION *)));
    connect(_thread, SIGNAL(lostChanges()),this, SLOT(lostChanges()));
    //connect(_thread, &WatcherThread::ready,this, [this]() { qDebug() << "触发了 ready"; });

    _thread->start();
    _timer.start();
}

void Dlg::changeDetected(const QString & path,FILE_NOTIFY_INFORMATION * curEntry)
{


    switch(curEntry->Action)
    {
    case FILE_ACTION_ADDED:  //1
        qDebug() << "文件被创建" << path;
        break;
    case FILE_ACTION_REMOVED: //2
        qDebug() << "文件被删除" << path;
        break;
    case FILE_ACTION_MODIFIED: //3
        qDebug() << "文件被修改" << path;
        break;
    case FILE_ACTION_RENAMED_OLD_NAME: //4
        qDebug() << "旧文件名:" << path;
        break;
    case FILE_ACTION_RENAMED_NEW_NAME: //5
        qDebug() << "新文件名:" << path;
        break;
    default:
        qDebug() << "未知操作";
        break;
    }

        QFileInfo fileInfo(path);
        QStringList paths(path);
        if (fileInfo.isDir()) {
            QDir dir(path);
            appendSubPaths(dir, paths);
        }
        changeDetected(paths);
}

void Dlg::lostChanges()
{
    qDebug() << "触发了 lostChanges";
}


void Dlg::on_btn_stop_clicked()
{
    _thread->stop();
    qDebug() << "结束......";
}


void Dlg::changeDetected(const QStringList &paths)
{
    // TODO: this shortcut doesn't look very reliable:
    //   - why is the timeout only 1 second?
    //   - what if there is more than one file being updated frequently?
    //   - why do we skip the file altogether instead of e.g. reducing the upload frequency?

    // Check if the same path was reported within the last second.
    QSet<QString> pathsSet = paths.toSet();
    if (pathsSet == _lastPaths && _timer.elapsed() < 1000) {
        // the same path was reported within the last second. Skip.
        return;
    }
    _lastPaths = pathsSet;
    _timer.restart();

    QSet<QString> changedPaths;

    // ------- handle ignores:
    for (int i = 0; i < paths.size(); ++i) {
        QString path = paths[i];
        changedPaths.insert(path);
    }
    if (changedPaths.isEmpty()) {
        return;
    }


    foreach (const QString &path, changedPaths) {
        //qDebug() << "已修改:" << path;
    }
}

void Dlg::appendSubPaths(QDir dir, QStringList& subPaths) {
    QStringList newSubPaths = dir.entryList(QDir::NoDotAndDotDot | QDir::Dirs | QDir::Files);
    for (int i = 0; i < newSubPaths.size(); i++) {
        QString path = dir.path() + "/" + newSubPaths[i];
        QFileInfo fileInfo(path);
        subPaths.append(path);
        if (fileInfo.isDir()) {
            QDir dir(path);
            appendSubPaths(dir, subPaths);
        }
    }
}