feat: make block indexing configurable (#10608)
This adds a new folder-level configuration `FullBlockIndex`. It controls whether we maintain the block index for a given folder -- currently that's always true, now it becomes possible to turn off. The block index is used for lookup of blocks across files and folders. Effectively, when syncing a change, for each block, we check: 1. Is the block already present in the old version of the file? If so, we can reuse (copy) it without network transfer. **This check is always possible.** 2. Is the block already present in any other file in this folder or other folders? If so we can copy it. **This check is only possible with the full block index.** 3. We must transfer the block over the network. Maintaining the full block index is costly in time, I/O and database size. With this PR, maintaining the full block index becomes the default for send-receive and receive-only folders only, with it disabled for send-only and receive-encrypted folders. The block index is never useful for encrypted folders, as blocks are encrypted separate for each file. It is also not useful for send-only folders by themselves, though the data in the send-only folder could be reused by other receive-type folders if it were enabled. For very large folders it may make sense to disable the full block index regardless of folder type and just accept the resulting decrease in data reuse. Disabling or enabling the option in the GUI causes the index to be destroyed or rebuilt accordingly. https://github.com/syncthing/docs/pull/1005 --------- Signed-off-by: Jakob Borg <jakob@kastelo.net>
This commit is contained in:
Jakob Borg
@@ -50,6 +50,7 @@
|
||||
"Automatically create or share folders that this device advertises at the default path.": "Automatically create or share folders that this device advertises at the default path.",
|
||||
"Available debug logging facilities:": "Available debug logging facilities:",
|
||||
"Be careful!": "Be careful!",
|
||||
"Block Indexing": "Block Indexing",
|
||||
"Body:": "Body:",
|
||||
"Bugs": "Bugs",
|
||||
"Cancel": "Cancel",
|
||||
@@ -251,6 +252,7 @@
|
||||
"Log tailing paused. Scroll to the bottom to continue.": "Log tailing paused. Scroll to the bottom to continue.",
|
||||
"Login failed, see Syncthing logs for details.": "Login failed, see Syncthing logs for details.",
|
||||
"Logs": "Logs",
|
||||
"Maintain an index of all blocks in the folder, enabling reuse of blocks from other files when syncing changes. Disable to reduce database size at the cost of not being able to reuse blocks across files.": "Maintain an index of all blocks in the folder, enabling reuse of blocks from other files when syncing changes. Disable to reduce database size at the cost of not being able to reuse blocks across files.",
|
||||
"Major Upgrade": "Major Upgrade",
|
||||
"Mass actions": "Mass actions",
|
||||
"Maximum Age": "Maximum Age",
|
||||
@@ -394,6 +396,7 @@
|
||||
"Staggered": "Staggered",
|
||||
"Staggered File Versioning": "Staggered File Versioning",
|
||||
"Start Browser": "Start Browser",
|
||||
"Starting": "Starting",
|
||||
"Statistics": "Statistics",
|
||||
"Stay logged in": "Stay logged in",
|
||||
"Stopped": "Stopped",
|
||||
|
||||
@@ -382,7 +382,7 @@
|
||||
<div class="col-md-6" aria-labelledby="folder_list" role="region" >
|
||||
<h3 id="folder_list"><span translate>Folders</span><span ng-if="folderList().length > 1"> ({{folderList().length}})</span></h3>
|
||||
<div ng-repeat="(folderGroupName, groupedFolders) in foldersGrouped">
|
||||
<h4 ng-if="folderGroupName !== ''">{{ folderGroupName }}
|
||||
<h4 ng-if="folderGroupName !== ''">{{ folderGroupName }}
|
||||
<span ng-if="groupedFolders.length > 1 && folderGroupName.length > 0"> ({{groupedFolders.length}})</span>
|
||||
</h4>
|
||||
<div class="panel-group" id="folders-{{ $index }}">
|
||||
@@ -501,6 +501,13 @@
|
||||
<span ng-if="folder.type == 'receiveencrypted'" translate>Receive Encrypted</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th><span class="far fa-fw fa-book"></span> <span translate>Block Indexing</span></th>
|
||||
<td class="text-right">
|
||||
<span ng-if="folder.blockIndexing" translate>Yes</span>
|
||||
<span ng-if="!folder.blockIndexing" translate>No</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr ng-if="folder.ignorePerms">
|
||||
<th><span class="far fa-fw fa-minus-square"></span> <span translate>Ignore Permissions</span></th>
|
||||
<td class="text-right">
|
||||
@@ -776,7 +783,7 @@
|
||||
<!-- Remote devices -->
|
||||
<h3><span translate>Remote Devices</span> <span ng-if="otherDevices().length > 1"> ({{otherDevices().length}})</span></h3>
|
||||
<div ng-repeat="(deviceGroupName, groupedDevices) in devicesGrouped">
|
||||
<h4>{{ deviceGroupName }}
|
||||
<h4>{{ deviceGroupName }}
|
||||
<span ng-if="groupedDevices.length > 1 && deviceGroupName.length > 0"> ({{groupedDevices.length}})</span>
|
||||
</h4>
|
||||
<div class="panel-group" id="devices-{{ $index }}">
|
||||
|
||||
@@ -571,7 +571,7 @@ angular.module('syncthing.core')
|
||||
};
|
||||
|
||||
// myID is watched as $scope.otherDevices() relies on this
|
||||
// and it can potenitally not be loaded due to this function
|
||||
// and it can potenitally not be loaded due to this function
|
||||
// scope being called in an undetermistic manner
|
||||
$scope.$watch('myID', function(myID) {
|
||||
if (myID) {
|
||||
@@ -579,7 +579,7 @@ angular.module('syncthing.core')
|
||||
const otherDevices = $scope.otherDevices();
|
||||
for (var id in otherDevices) {
|
||||
if ($scope.devicesGrouped[otherDevices[id].group] === undefined) {
|
||||
$scope.devicesGrouped[otherDevices[id].group] = [];
|
||||
$scope.devicesGrouped[otherDevices[id].group] = [];
|
||||
}
|
||||
$scope.devicesGrouped[otherDevices[id].group].push(otherDevices[id]);
|
||||
};
|
||||
@@ -595,7 +595,7 @@ angular.module('syncthing.core')
|
||||
$scope.folders[folder].devices.forEach(function (deviceCfg) {
|
||||
refreshCompletion(deviceCfg.deviceID, folder);
|
||||
});
|
||||
|
||||
|
||||
if ($scope.foldersGrouped[$scope.folders[folder].group] === undefined) {
|
||||
$scope.foldersGrouped[$scope.folders[folder].group] = [];
|
||||
}
|
||||
@@ -612,7 +612,7 @@ angular.module('syncthing.core')
|
||||
}
|
||||
}
|
||||
|
||||
// Sort firstly by the top level key of the object and then by
|
||||
// Sort firstly by the top level key of the object and then by
|
||||
// prop name provided for the array of objects for each key.
|
||||
// If the prop returns has an empty value, then use the
|
||||
// fallback prop provided.
|
||||
@@ -1122,7 +1122,7 @@ angular.module('syncthing.core')
|
||||
if (status == 'paused') {
|
||||
return 'default';
|
||||
}
|
||||
if (status === 'syncing' || status === 'sync-preparing' || status === 'scanning' || status === 'cleaning') {
|
||||
if (status === 'syncing' || status === 'sync-preparing' || status === 'scanning' || status === 'cleaning' || status === 'starting') {
|
||||
return 'primary';
|
||||
}
|
||||
if (status === 'unknown') {
|
||||
@@ -1320,6 +1320,7 @@ angular.module('syncthing.core')
|
||||
case 'scan-waiting':
|
||||
case 'sync-preparing':
|
||||
case 'sync-waiting':
|
||||
case 'starting':
|
||||
return 'fa-hourglass-half';
|
||||
case 'cleaning':
|
||||
return 'fa-recycle';
|
||||
@@ -1355,6 +1356,8 @@ angular.module('syncthing.core')
|
||||
return $translate.instant('Failed Items');
|
||||
case 'idle':
|
||||
return $translate.instant('Up to Date');
|
||||
case 'starting':
|
||||
return $translate.instant('Starting');
|
||||
case 'localadditions':
|
||||
return $translate.instant('Local Additions');
|
||||
case 'localunencrypted':
|
||||
@@ -2334,6 +2337,12 @@ angular.module('syncthing.core')
|
||||
} else {
|
||||
$scope.currentFolder.fsWatcherEnabled = true;
|
||||
}
|
||||
var type = $scope.currentFolder.type;
|
||||
if ($scope.currentFolder._editing !== 'existing') {
|
||||
// Never automatically change block indexing, only suggest
|
||||
// the value on new folder creation.
|
||||
$scope.currentFolder.blockIndexing = (type === 'sendreceive' || type === 'receiveonly');
|
||||
}
|
||||
$scope.setFSWatcherIntervalDefault();
|
||||
};
|
||||
|
||||
|
||||
@@ -387,6 +387,17 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-6 form-group">
|
||||
<label>
|
||||
<input type="checkbox" ng-model="currentFolder.blockIndexing" /> <span translate>Block Indexing</span>
|
||||
</label>
|
||||
<p translate class="help-block">
|
||||
Maintain an index of all blocks in the folder, enabling reuse of blocks from other files when syncing changes. Disable to reduce database size at the cost of not being able to reuse blocks across files.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row" ng-if="currentFolder.syncXattrs || currentFolder.sendXattrs">
|
||||
<div class="col-md-12">
|
||||
<p>
|
||||
|
||||
Reference in New Issue
Block a user