-
Notifications
You must be signed in to change notification settings - Fork 975
fix(discovery): check quota before upload (during discovery) #9777
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -1,3 +1,3 @@ | ||||||
| /* | ||||||
| * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors | ||||||
| * SPDX-FileCopyrightText: 2014 ownCloud GmbH | ||||||
|
|
@@ -241,7 +241,7 @@ | |||||
|
|
||||||
| // Delete the stales chunk on the server. | ||||||
| if (account()->capabilities().chunkingNg()) { | ||||||
| for (uint transferId : std::as_const(ids)) { | ||||||
|
Check warning on line 244 in src/libsync/syncengine.cpp
|
||||||
| if (!transferId) | ||||||
| continue; // Was not a chunked upload | ||||||
| QUrl url = Utility::concatUrlPath(account()->url(), QLatin1String("remote.php/dav/uploads/") + account()->davUser() + QLatin1Char('/') + QString::number(transferId)); | ||||||
|
|
@@ -397,7 +397,7 @@ | |||||
|
|
||||||
| // Update on-disk virtual file metadata | ||||||
| if (modificationHappened && item->_type == ItemTypeVirtualFile) { | ||||||
| auto r = _syncOptions._vfs->updateMetadata(*item, filePath, {}); | ||||||
|
Check warning on line 400 in src/libsync/syncengine.cpp
|
||||||
| if (!r) { | ||||||
| item->_status = SyncFileItem::Status::NormalError; | ||||||
| item->_instruction = CSYNC_INSTRUCTION_ERROR; | ||||||
|
|
@@ -474,7 +474,7 @@ | |||||
| _needsUpdate = true; | ||||||
|
|
||||||
| // Insert sorted | ||||||
| auto it = std::lower_bound( _syncItems.begin(), _syncItems.end(), item ); // the _syncItems is sorted | ||||||
|
Check warning on line 477 in src/libsync/syncengine.cpp
|
||||||
| _syncItems.insert( it, item ); | ||||||
|
|
||||||
| slotNewItem(item); | ||||||
|
|
@@ -501,15 +501,15 @@ | |||||
| const auto e2EeLockedFolders = _journal->e2EeLockedFolders(); | ||||||
|
|
||||||
| if (!e2EeLockedFolders.isEmpty()) { | ||||||
| for (const auto &e2EeLockedFolder : e2EeLockedFolders) { | ||||||
|
Check warning on line 504 in src/libsync/syncengine.cpp
|
||||||
| const auto folderId = e2EeLockedFolder.first; | ||||||
|
Check warning on line 505 in src/libsync/syncengine.cpp
|
||||||
| qCInfo(lcEngine()) << "start unlock job for folderId:" << folderId; | ||||||
| const auto folderToken = EncryptionHelper::decryptStringAsymmetric(_account->e2e()->getCertificateInformation(), _account->e2e()->paddingMode(), *_account->e2e(), e2EeLockedFolder.second); | ||||||
| if (!folderToken) { | ||||||
| qCWarning(lcEngine()) << "decrypt failed"; | ||||||
| return; | ||||||
| } | ||||||
| // TODO: We need to rollback changes done to metadata in case we have an active lock, this needs to be implemented on the server first | ||||||
|
Check warning on line 512 in src/libsync/syncengine.cpp
|
||||||
| const auto unlockJob = new OCC::UnlockEncryptFolderApiJob(_account, folderId, *folderToken, _journal, this); | ||||||
| unlockJob->setShouldRollbackMetadataChanges(true); | ||||||
| unlockJob->start(); | ||||||
|
|
@@ -638,7 +638,7 @@ | |||||
| _discoveryPhase->_account = _account; | ||||||
| _discoveryPhase->_excludes = _excludedFiles.data(); | ||||||
| const QString excludeFilePath = _localPath + QStringLiteral(".sync-exclude.lst"); | ||||||
| if (FileSystem::fileExists(excludeFilePath)) { | ||||||
|
Check warning on line 641 in src/libsync/syncengine.cpp
|
||||||
| _discoveryPhase->_excludes->addExcludeFilePath(excludeFilePath); | ||||||
| _discoveryPhase->_excludes->reloadExcludeFiles(); | ||||||
| } | ||||||
|
|
@@ -721,7 +721,7 @@ | |||||
| const auto databaseFingerprint = _journal->dataFingerprint(); | ||||||
| _discoveryPhase->_dataFingerprint = databaseFingerprint; | ||||||
| ProcessDirectoryJob::PathTuple path = {}; | ||||||
| path._local = path._original = path._server = path._target = singleItemDiscoveryOptions().discoveryPath; | ||||||
|
Check warning on line 724 in src/libsync/syncengine.cpp
|
||||||
|
|
||||||
| SyncJournalFileRecord rec; | ||||||
| const auto localQueryMode = _journal->getFileRecord(singleItemDiscoveryOptions().discoveryDirItem->_file, &rec) && rec.isValid() | ||||||
|
|
@@ -917,7 +917,7 @@ | |||||
| detectFileLock(item); | ||||||
| } | ||||||
|
|
||||||
| void SyncEngine::slotPropagationFinished(OCC::SyncFileItem::Status status) | ||||||
|
Check warning on line 920 in src/libsync/syncengine.cpp
|
||||||
| { | ||||||
| if (_propagator->_anotherSyncNeeded && _anotherSyncNeeded == NoFollowUpSync) { | ||||||
| _anotherSyncNeeded = ImmediateFollowUp; | ||||||
|
|
@@ -1042,6 +1042,31 @@ | |||||
| } | ||||||
| } | ||||||
|
|
||||||
| void SyncEngine::seedPropagatorQuota() | ||||||
| { | ||||||
| // Seed the propagator's per-folder quota cache from the quota data gathered | ||||||
| // during the discovery phase (PROPFIND responses). The propagator is recreated | ||||||
| // fresh every sync cycle, so without this seeding the pre-upload quota check in | ||||||
| // PropagateUploadFileCommon::startUploadFile() would always pass on the first | ||||||
| // attempt of a new cycle, causing a full upload that the server then rejects with | ||||||
| // HTTP 507 and wasting bandwidth. By initialising the cache here, the pre-upload | ||||||
| // check can block oversized files from the very first attempt of each cycle. | ||||||
| for (const auto &syncItem : std::as_const(_syncItems)) { | ||||||
| if (!syncItem->isDirectory() || syncItem->_folderQuota.bytesAvailable < 0) { | ||||||
| continue; | ||||||
| } | ||||||
| // OwncloudPropagator keys _folderQuota by QFileInfo::path() of the file | ||||||
| // being uploaded, which yields "." for items at the root of the sync | ||||||
| // folder. Normalise the empty-string root path accordingly. | ||||||
| const auto key = syncItem->_file.isEmpty() ? QStringLiteral(".") : syncItem->_file; | ||||||
| // Only insert when there is no tighter bound already present (e.g. set | ||||||
| // from a prior HTTP 507 reply within the same cycle). | ||||||
| if (!_propagator->_folderQuota.contains(key)) { | ||||||
| _propagator->_folderQuota.insert(key, syncItem->_folderQuota.bytesAvailable); | ||||||
| } | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| void SyncEngine::finishSync() | ||||||
| { | ||||||
| auto databaseFingerprint = _journal->dataFingerprint(); | ||||||
|
|
@@ -1094,6 +1119,10 @@ | |||||
| _propagator = QSharedPointer<OwncloudPropagator>( | ||||||
| new OwncloudPropagator(_account, _localPath, _remotePath, _journal, _bulkUploadBlackList)); | ||||||
| _propagator->setSyncOptions(_syncOptions); | ||||||
|
|
||||||
| // Must be called before _propagator->start(std::move(_syncItems)) below, | ||||||
| // because start() transfers ownership of _syncItems and leaves it empty. | ||||||
| seedPropagatorQuota(); | ||||||
| connect(_propagator.data(), &OwncloudPropagator::itemCompleted, | ||||||
| this, &SyncEngine::slotItemCompleted); | ||||||
| connect(_propagator.data(), &OwncloudPropagator::progress, | ||||||
|
|
@@ -1167,7 +1196,7 @@ | |||||
| ++deletionCounter; | ||||||
| } | ||||||
| }); | ||||||
| if (!result) { | ||||||
|
Check failure on line 1199 in src/libsync/syncengine.cpp
|
||||||
| qCDebug(lcEngine()) << "unable to find the number of files within a deleted folder:" << oneItem->_file; | ||||||
| } | ||||||
| } else { | ||||||
|
|
@@ -1177,7 +1206,7 @@ | |||||
| } | ||||||
| const auto filesDeletedThresholdExceeded = deletionCounter > ConfigFile().deleteFilesThreshold(); | ||||||
|
|
||||||
| if ((allFilesDeleted || filesDeletedThresholdExceeded) && displayDialog) { | ||||||
|
Check warning on line 1209 in src/libsync/syncengine.cpp
|
||||||
| qCWarning(lcEngine) << "Many files are going to be deleted, asking the user"; | ||||||
| int side = 0; // > 0 means more deleted on the server. < 0 means more deleted on the client | ||||||
| for (const auto &it : std::as_const(_syncItems)) { | ||||||
|
|
@@ -1228,7 +1257,7 @@ | |||||
| { | ||||||
| QPointer<QObject> guard = new QObject(); | ||||||
| QPointer<QObject> self = this; | ||||||
| auto callback = [this, self, guard](bool cancel) -> void { | ||||||
|
Check warning on line 1260 in src/libsync/syncengine.cpp
|
||||||
| // use a guard to ensure its only called once... | ||||||
| // qpointer to self to ensure we still exist | ||||||
| if (!guard || !self) { | ||||||
|
|
@@ -1315,14 +1344,14 @@ | |||||
| // This invariant is used in SyncEngine::shouldDiscoverLocally | ||||||
| QString prev; | ||||||
| auto it = _localDiscoveryPaths.begin(); | ||||||
| while(it != _localDiscoveryPaths.end()) { | ||||||
| if (!prev.isNull() && it->startsWith(prev) && (prev.endsWith('/') || *it == prev || it->at(prev.size()) <= '/')) { | ||||||
| it = _localDiscoveryPaths.erase(it); | ||||||
| } else { | ||||||
| prev = *it; | ||||||
| ++it; | ||||||
| } | ||||||
| } | ||||||
|
Check warning on line 1354 in src/libsync/syncengine.cpp
|
||||||
| } | ||||||
|
|
||||||
| void SyncEngine::setSingleItemDiscoveryOptions(const SingleItemDiscoveryOptions &singleItemDiscoveryOptions) | ||||||
|
|
@@ -1335,7 +1364,7 @@ | |||||
| return _singleItemDiscoveryOptions; | ||||||
| } | ||||||
|
|
||||||
| void SyncEngine::setFilesystemPermissionsReliable(bool reliable) | ||||||
|
Check warning on line 1367 in src/libsync/syncengine.cpp
|
||||||
| { | ||||||
| _filesystemPermissionsReliable = reliable; | ||||||
| } | ||||||
|
|
@@ -1486,7 +1515,8 @@ | |||||
|
|
||||||
| void SyncEngine::slotInsufficientRemoteStorage() | ||||||
| { | ||||||
| auto msg = tr("There is insufficient space available on the server for some uploads."); | ||||||
| auto msg = tr("Upload paused: one or more files exceed your remaining Nextcloud storage quota. " | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
| "Free up server space or contact your administrator to increase your quota."); | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
| if (_uniqueErrors.contains(msg)) { | ||||||
| return; | ||||||
| } | ||||||
|
|
@@ -1573,7 +1603,7 @@ | |||||
| it != _discoveryPhase->_filesNeedingScheduledSync.cend(); | ||||||
| ++it) { | ||||||
|
|
||||||
| const auto file = it.key(); | ||||||
|
Check warning on line 1606 in src/libsync/syncengine.cpp
|
||||||
| const auto syncScheduledSecs = it.value(); | ||||||
|
|
||||||
| // We don't want to schedule syncs again for files we have already discovered needing a | ||||||
|
|
||||||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -8,7 +8,7 @@ | |||||
| * any purpose. | ||||||
| */ | ||||||
|
|
||||||
| #include <QtTest> | ||||||
| #include "syncenginetestutils.h" | ||||||
| #include <syncengine.h> | ||||||
| #include <localdiscoverytracker.h> | ||||||
|
|
@@ -870,106 +870,83 @@ | |||||
|
|
||||||
| void testDiscoveryUsesCorrectQuotaSource() | ||||||
| { | ||||||
| //setup sync folder | ||||||
| // Verifies that the discovery-phase quota check uses the FRESH | ||||||
| // PROPFIND value (from the current sync cycle) rather than the | ||||||
| // stale value persisted in the journal DB from a previous cycle. | ||||||
|
|
||||||
| FakeFolder fakeFolder{FileInfo{}}; | ||||||
|
|
||||||
| // create folder | ||||||
| const QString folderA("A"); | ||||||
| fakeFolder.localModifier().mkdir(folderA); | ||||||
| fakeFolder.remoteModifier().mkdir(folderA); | ||||||
| fakeFolder.remoteModifier().setFolderQuota(folderA, {0, 500}); | ||||||
|
|
||||||
| // sync folderA | ||||||
| // Initial sync — folder A is created, DB stores quota = 500 | ||||||
| ItemCompletedSpy syncSpy(fakeFolder); | ||||||
| QVERIFY(fakeFolder.syncOnce()); | ||||||
| QCOMPARE(syncSpy.findItem(folderA)->_status, SyncFileItem::Status::NoStatus); | ||||||
|
|
||||||
| // check db quota for folderA - bytesAvailable is 500 | ||||||
| SyncJournalFileRecord recordFolderA; | ||||||
| QVERIFY(fakeFolder.syncJournal().getFileRecord(folderA, &recordFolderA)); | ||||||
| QCOMPARE(recordFolderA._folderQuota.bytesAvailable, 500); | ||||||
|
|
||||||
| // add fileNameA to folderA - size < quota in db | ||||||
| // ── Case 1: upload succeeds when size < fresh PROPFIND quota ── | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
| const QString fileNameA("A/A.data"); | ||||||
| fakeFolder.localModifier().insert(fileNameA, 200); | ||||||
|
|
||||||
| // set different quota for folderA - remote change does not change etag yet | ||||||
| fakeFolder.remoteModifier().setFolderQuota(folderA, {0, 0}); | ||||||
|
|
||||||
| // sync filenameA - size == quota => success | ||||||
| syncSpy.clear(); | ||||||
| QVERIFY(fakeFolder.syncOnce()); | ||||||
| QCOMPARE(syncSpy.findItem(fileNameA)->_status, SyncFileItem::Status::Success); | ||||||
|
|
||||||
| // add smallFile to folderA - size < quota in db | ||||||
| const QString smallFile("A/smallFile.data"); | ||||||
| fakeFolder.localModifier().insert(smallFile, 100); | ||||||
| // ── Case 2: fresh PROPFIND value (0) wins over stale DB (500) ── | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
| // Reduce quota to 0 without invalidating the etag. The DB still | ||||||
| // holds 500, but the root PROPFIND now returns 0 for folder A. | ||||||
| // A new file must be blocked proactively. | ||||||
| fakeFolder.remoteModifier().setFolderQuota(folderA, {0, 0}); | ||||||
|
|
||||||
| const QString fileBlocked("A/blocked.data"); | ||||||
| fakeFolder.localModifier().insert(fileBlocked, 100); | ||||||
|
|
||||||
| // sync smallFile - size < quota in db => success => update quota in db | ||||||
| syncSpy.clear(); | ||||||
| QVERIFY(fakeFolder.syncOnce()); | ||||||
| QCOMPARE(syncSpy.findItem(smallFile)->_status, SyncFileItem::Status::Success); | ||||||
| QVERIFY(!fakeFolder.syncOnce()); | ||||||
| QCOMPARE(syncSpy.findItem(fileBlocked)->_status, SyncFileItem::Status::DetailError); | ||||||
|
|
||||||
| // create remoteFileA - size > bytes available | ||||||
| // ── Case 3: downloads are not affected by upload quota ── | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
| const QString remoteFileA("A/remoteA.data"); | ||||||
| fakeFolder.remoteModifier().insert(remoteFileA, 200); | ||||||
|
|
||||||
| // sync remoteFile - it is a download | ||||||
| syncSpy.clear(); | ||||||
| QVERIFY(fakeFolder.syncOnce()); | ||||||
| // Sync still fails overall (blocked.data remains blocked), but the | ||||||
| // download of remoteFileA must succeed independently. | ||||||
| QVERIFY(!fakeFolder.syncOnce()); | ||||||
| QCOMPARE(syncSpy.findItem(remoteFileA)->_status, SyncFileItem::Status::Success); | ||||||
| QCOMPARE(syncSpy.findItem(fileBlocked)->_status, SyncFileItem::Status::DetailError); | ||||||
|
|
||||||
| // check db quota for folderA - bytesAvailable have changed to 0 due to new PROPFIND | ||||||
| QVERIFY(fakeFolder.syncJournal().getFileRecord(folderA, &recordFolderA)); | ||||||
| QCOMPARE(recordFolderA._folderQuota.bytesAvailable, 0); | ||||||
|
|
||||||
| // create local fileNameB - size < quota in db | ||||||
| const QString fileNameB("A/B.data"); | ||||||
| fakeFolder.localModifier().insert(fileNameB, 0); | ||||||
|
|
||||||
| // set different quota for folderA - remote change does not change etag yet | ||||||
| // ── Case 4: after quota increase, previously blocked file uploads ── | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
| fakeFolder.remoteModifier().setFolderQuota(folderA, {500, 600}); | ||||||
|
|
||||||
| // sync fileNameB - size < quota in db => success | ||||||
| syncSpy.clear(); | ||||||
| QVERIFY(fakeFolder.syncOnce()); | ||||||
| QCOMPARE(syncSpy.findItem(fileNameB)->_status, SyncFileItem::Status::Success); | ||||||
|
|
||||||
| // create remoteFileB - it is a download | ||||||
| const QString remoteFileB("A/remoteB.data"); | ||||||
| fakeFolder.remoteModifier().insert(remoteFileA, 100); | ||||||
| QCOMPARE(syncSpy.findItem(fileBlocked)->_status, SyncFileItem::Status::Success); | ||||||
|
|
||||||
| // create local fileNameC - size < quota in db | ||||||
| const QString fileNameC("A/C.data"); | ||||||
| fakeFolder.localModifier().insert(fileNameC, 0); | ||||||
|
|
||||||
| // sync filenameC - size < quota in db => success | ||||||
| syncSpy.clear(); | ||||||
| QVERIFY(fakeFolder.syncOnce()); | ||||||
| QCOMPARE(syncSpy.findItem(fileNameC)->_status, SyncFileItem::Status::Success); | ||||||
|
|
||||||
| // check db quota for folderA - bytesAvailable have changed to 600 due to new PROPFIND | ||||||
| // DB now reflects the fresh PROPFIND value (600) | ||||||
| QVERIFY(fakeFolder.syncJournal().getFileRecord(folderA, &recordFolderA)); | ||||||
| QCOMPARE(recordFolderA._folderQuota.bytesAvailable, 600); | ||||||
|
|
||||||
| QCOMPARE(syncSpy.findItem(remoteFileB)->_status, SyncFileItem::Status::NoStatus); | ||||||
|
|
||||||
| // create local fileNameD - size > quota in db | ||||||
| // ── Case 5: file exceeding fresh quota is blocked ── | ||||||
| const QString fileNameD("A/D.data"); | ||||||
| fakeFolder.localModifier().insert(fileNameD, 700); | ||||||
|
|
||||||
| // sync fileNameD - size > quota in db => error | ||||||
| syncSpy.clear(); | ||||||
| QVERIFY(!fakeFolder.syncOnce()); | ||||||
| QCOMPARE(syncSpy.findItem(fileNameD)->_status, SyncFileItem::Status::NormalError); | ||||||
| QCOMPARE(syncSpy.findItem(fileNameD)->_status, SyncFileItem::Status::DetailError); | ||||||
|
|
||||||
| // create local fileNameE - size < quota in db | ||||||
| // ── Case 6: file within quota succeeds even while another is blocked ── | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
| const QString fileNameE("A/E.data"); | ||||||
| fakeFolder.localModifier().insert(fileNameE, 400); | ||||||
|
|
||||||
| // sync fileNameE - size < quota in db => success | ||||||
| syncSpy.clear(); | ||||||
| QVERIFY(!fakeFolder.syncOnce()); | ||||||
| QVERIFY(!fakeFolder.syncOnce()); // fileNameD still blocked | ||||||
| QCOMPARE(syncSpy.findItem(fileNameE)->_status, SyncFileItem::Status::Success); | ||||||
| } | ||||||
|
|
||||||
|
|
||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.