chore: stop treating dirs as having size 128 (#10750)
Resp. directories for the database, and both dirs and symlinks for `FileInfo.FileSize`. Instead handle it in the UI progress percentage. We even already have special cases there for deletions, might as well handle directories just like any other zero-sized needed item there. This went through the very thorough testing of running it on my laptop, the migration was applied and it seemed to be working fine after. --------- Signed-off-by: Simon Frei <freisim93@gmail.com>
This commit is contained in:
@@ -1139,20 +1139,20 @@ angular.module('syncthing.core')
|
|||||||
};
|
};
|
||||||
|
|
||||||
$scope.syncPercentage = function (folder) {
|
$scope.syncPercentage = function (folder) {
|
||||||
if (typeof $scope.model[folder] === 'undefined') {
|
var model = $scope.model[folder];
|
||||||
|
if (typeof model === 'undefined') {
|
||||||
return 100;
|
return 100;
|
||||||
}
|
}
|
||||||
if ($scope.model[folder].needTotalItems === 0) {
|
if (model.needTotalItems === 0) {
|
||||||
return 100;
|
return 100;
|
||||||
}
|
}
|
||||||
if (($scope.model[folder].needBytes == 0 && $scope.model[folder].needDeletes > 0) || $scope.model[folder].globalBytes == 0) {
|
if (model.needBytes == 0 && model.needTotalItems > 0) {
|
||||||
// We don't need any data, but we have deletes that we need
|
// We don't need any data, but we have deletes, directories,
|
||||||
// to do. Drop down the completion percentage to indicate
|
// symlinks or zero byte files that we need to do. Drop down the
|
||||||
// that we have stuff to do.
|
// completion percentage to indicate that we have stuff to do.
|
||||||
// Do the same thing in case we only have zero byte files to sync.
|
|
||||||
return 95;
|
return 95;
|
||||||
}
|
}
|
||||||
return progressIntegerPercentage($scope.model[folder].inSyncBytes, $scope.model[folder].globalBytes);
|
return progressIntegerPercentage(model.inSyncBytes, model.globalBytes);
|
||||||
};
|
};
|
||||||
|
|
||||||
$scope.scanPercentage = function (folder) {
|
$scope.scanPercentage = function (folder) {
|
||||||
|
|||||||
@@ -29,7 +29,6 @@ import (
|
|||||||
const (
|
const (
|
||||||
folderID = "test"
|
folderID = "test"
|
||||||
blockSize = 128 << 10
|
blockSize = 128 << 10
|
||||||
dirSize = 128
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBasics(t *testing.T) {
|
func TestBasics(t *testing.T) {
|
||||||
@@ -72,11 +71,11 @@ func TestBasics(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
const (
|
const (
|
||||||
localSize = (1+2+3)*blockSize + dirSize
|
localSize = (1+2+3)*blockSize
|
||||||
remoteSize = (3 + 4 + 5) * blockSize
|
remoteSize = (3 + 4 + 5) * blockSize
|
||||||
globalSize = (2+3+3+4+5)*blockSize + dirSize
|
globalSize = (2+3+3+4+5)*blockSize
|
||||||
needSizeLocal = remoteSize
|
needSizeLocal = remoteSize
|
||||||
needSizeRemote = (2+3)*blockSize + dirSize
|
needSizeRemote = (2+3)*blockSize
|
||||||
)
|
)
|
||||||
|
|
||||||
t.Run("SchemaVersion", func(t *testing.T) {
|
t.Run("SchemaVersion", func(t *testing.T) {
|
||||||
|
|||||||
@@ -111,10 +111,6 @@ func (s *folderDB) Update(device protocol.DeviceID, fs []protocol.FileInfo, opti
|
|||||||
f.BlocksHash = nil
|
f.BlocksHash = nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if f.Type == protocol.FileInfoTypeDirectory {
|
|
||||||
f.Size = 128 // synthetic directory size
|
|
||||||
}
|
|
||||||
|
|
||||||
// Insert the file.
|
// Insert the file.
|
||||||
//
|
//
|
||||||
// If it is a remote file, set remote_sequence otherwise leave it at
|
// If it is a remote file, set remote_sequence otherwise leave it at
|
||||||
|
|||||||
@@ -0,0 +1,12 @@
|
|||||||
|
-- Copyright (C) 2026 The Syncthing Authors.
|
||||||
|
--
|
||||||
|
-- This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||||
|
-- You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
-- All non-file entries should have size zero.
|
||||||
|
-- Directories were previously stored with a "synthetic" size of 128.
|
||||||
|
UPDATE files
|
||||||
|
SET size = 0
|
||||||
|
WHERE type != 0
|
||||||
|
;
|
||||||
@@ -106,6 +106,10 @@ func (i indirectFI) FileInfo() (protocol.FileInfo, error) {
|
|||||||
fi.Blocks = bl.Blocks
|
fi.Blocks = bl.Blocks
|
||||||
}
|
}
|
||||||
fi.Name = osutil.NativeFilename(fi.Name)
|
fi.Name = osutil.NativeFilename(fi.Name)
|
||||||
|
// Undo earlier behaviour of setting a synthetic size on dirs.
|
||||||
|
if fi.Type != protocol.FileInfoTypeFile && fi.Size > 0 {
|
||||||
|
fi.Size = 0
|
||||||
|
}
|
||||||
return protocol.FileInfoFromDB(&fi), nil
|
return protocol.FileInfoFromDB(&fi), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -1774,7 +1774,7 @@ func fileIntfJSONMap(f protocol.FileInfo) map[string]interface{} {
|
|||||||
out := map[string]interface{}{
|
out := map[string]interface{}{
|
||||||
"name": f.FileName(),
|
"name": f.FileName(),
|
||||||
"type": f.FileType().String(),
|
"type": f.FileType().String(),
|
||||||
"size": f.FileSize(),
|
"size": f.Size,
|
||||||
"deleted": f.IsDeleted(),
|
"deleted": f.IsDeleted(),
|
||||||
"invalid": f.IsInvalid(),
|
"invalid": f.IsInvalid(),
|
||||||
"ignored": f.IsIgnored(),
|
"ignored": f.IsIgnored(),
|
||||||
|
|||||||
@@ -163,12 +163,11 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
|
|||||||
// We now have a newer file than the rest of the cluster. Global state should reflect this.
|
// We now have a newer file than the rest of the cluster. Global state should reflect this.
|
||||||
|
|
||||||
size = mustV(m.GlobalSize("ro"))
|
size = mustV(m.GlobalSize("ro"))
|
||||||
const sizeOfDir = 128
|
if size.Files != 1 || size.Bytes != int64(len(oldData)) {
|
||||||
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) {
|
|
||||||
t.Fatalf("Global: expected no change due to the new file: %+v", size)
|
t.Fatalf("Global: expected no change due to the new file: %+v", size)
|
||||||
}
|
}
|
||||||
size = mustV(m.LocalSize("ro", protocol.LocalDeviceID))
|
size = mustV(m.LocalSize("ro", protocol.LocalDeviceID))
|
||||||
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
|
if size.Files != 1 || size.Bytes != int64(len(newData)) {
|
||||||
t.Fatalf("Local: expected the new file to be reflected: %+v", size)
|
t.Fatalf("Local: expected the new file to be reflected: %+v", size)
|
||||||
}
|
}
|
||||||
size = mustV(m.NeedSize("ro", protocol.LocalDeviceID))
|
size = mustV(m.NeedSize("ro", protocol.LocalDeviceID))
|
||||||
@@ -185,11 +184,11 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
|
|||||||
m.Revert("ro")
|
m.Revert("ro")
|
||||||
|
|
||||||
size = mustV(m.GlobalSize("ro"))
|
size = mustV(m.GlobalSize("ro"))
|
||||||
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) {
|
if size.Files != 1 || size.Bytes != int64(len(oldData)) {
|
||||||
t.Fatalf("Global: expected the global size to revert: %+v", size)
|
t.Fatalf("Global: expected the global size to revert: %+v", size)
|
||||||
}
|
}
|
||||||
size = mustV(m.LocalSize("ro", protocol.LocalDeviceID))
|
size = mustV(m.LocalSize("ro", protocol.LocalDeviceID))
|
||||||
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
|
if size.Files != 1 || size.Bytes != int64(len(newData)) {
|
||||||
t.Fatalf("Local: expected the local size to remain: %+v", size)
|
t.Fatalf("Local: expected the local size to remain: %+v", size)
|
||||||
}
|
}
|
||||||
size = mustV(m.NeedSize("ro", protocol.LocalDeviceID))
|
size = mustV(m.NeedSize("ro", protocol.LocalDeviceID))
|
||||||
|
|||||||
@@ -1787,10 +1787,12 @@ func TestGlobalDirectoryTree(t *testing.T) {
|
|||||||
b := func(isfile bool, path ...string) protocol.FileInfo {
|
b := func(isfile bool, path ...string) protocol.FileInfo {
|
||||||
typ := protocol.FileInfoTypeDirectory
|
typ := protocol.FileInfoTypeDirectory
|
||||||
var blocks []protocol.BlockInfo
|
var blocks []protocol.BlockInfo
|
||||||
|
var size int64
|
||||||
|
|
||||||
if isfile {
|
if isfile {
|
||||||
typ = protocol.FileInfoTypeFile
|
typ = protocol.FileInfoTypeFile
|
||||||
blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
|
blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
|
||||||
|
size = 0xa
|
||||||
}
|
}
|
||||||
seq++
|
seq++
|
||||||
return protocol.FileInfo{
|
return protocol.FileInfo{
|
||||||
@@ -1798,7 +1800,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
|
|||||||
Type: typ,
|
Type: typ,
|
||||||
ModifiedS: 0x666,
|
ModifiedS: 0x666,
|
||||||
Blocks: blocks,
|
Blocks: blocks,
|
||||||
Size: 0xa,
|
Size: size,
|
||||||
Sequence: seq,
|
Sequence: seq,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1814,7 +1816,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
|
|||||||
return &TreeEntry{
|
return &TreeEntry{
|
||||||
Name: name,
|
Name: name,
|
||||||
ModTime: time.Unix(0x666, 0),
|
ModTime: time.Unix(0x666, 0),
|
||||||
Size: 128,
|
Size: 0,
|
||||||
Type: protocol.FileInfoTypeDirectory.String(),
|
Type: protocol.FileInfoTypeDirectory.String(),
|
||||||
Children: entries,
|
Children: entries,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -389,16 +389,6 @@ func (f FileInfo) HasPermissionBits() bool {
|
|||||||
return !f.NoPermissions
|
return !f.NoPermissions
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f FileInfo) FileSize() int64 {
|
|
||||||
if f.Deleted {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
if f.IsDirectory() || f.IsSymlink() {
|
|
||||||
return SyntheticDirectorySize
|
|
||||||
}
|
|
||||||
return f.Size
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f FileInfo) BlockSize() int {
|
func (f FileInfo) BlockSize() int {
|
||||||
if f.RawBlockSize < MinBlockSize {
|
if f.RawBlockSize < MinBlockSize {
|
||||||
return MinBlockSize
|
return MinBlockSize
|
||||||
|
|||||||
@@ -56,7 +56,8 @@ const (
|
|||||||
// DesiredPerFileBlocks is the number of blocks we aim for per file
|
// DesiredPerFileBlocks is the number of blocks we aim for per file
|
||||||
DesiredPerFileBlocks = 2000
|
DesiredPerFileBlocks = 2000
|
||||||
|
|
||||||
SyntheticDirectorySize = 128
|
// We used to send this size for directories and still accept that for compat.
|
||||||
|
deprecatedSyntheticDirectorySize = 128
|
||||||
|
|
||||||
// don't bother compressing messages smaller than this many bytes
|
// don't bother compressing messages smaller than this many bytes
|
||||||
compressionThreshold = 128
|
compressionThreshold = 128
|
||||||
@@ -640,7 +641,7 @@ func checkFileInfoConsistency(f FileInfo) error {
|
|||||||
// Only files should have blocks
|
// Only files should have blocks
|
||||||
return errNonFileHasBlocks
|
return errNonFileHasBlocks
|
||||||
|
|
||||||
case f.IsDirectory() && f.Size != 0 && f.Size != SyntheticDirectorySize:
|
case f.IsDirectory() && f.Size != 0 && f.Size != deprecatedSyntheticDirectorySize:
|
||||||
// Directories should be size zero or the synthetic directory size
|
// Directories should be size zero or the synthetic directory size
|
||||||
return errNonFileHasSize
|
return errNonFileHasSize
|
||||||
|
|
||||||
|
|||||||
@@ -464,7 +464,7 @@ func TestCheckConsistency(t *testing.T) {
|
|||||||
fi: FileInfo{
|
fi: FileInfo{
|
||||||
Name: "foo",
|
Name: "foo",
|
||||||
Type: FileInfoTypeDirectory,
|
Type: FileInfoTypeDirectory,
|
||||||
Size: SyntheticDirectorySize,
|
Size: deprecatedSyntheticDirectorySize,
|
||||||
},
|
},
|
||||||
ok: true,
|
ok: true,
|
||||||
},
|
},
|
||||||
@@ -492,7 +492,7 @@ func TestCheckConsistency(t *testing.T) {
|
|||||||
Name: "foo",
|
Name: "foo",
|
||||||
Type: FileInfoTypeSymlink,
|
Type: FileInfoTypeSymlink,
|
||||||
SymlinkTarget: []byte("bar"),
|
SymlinkTarget: []byte("bar"),
|
||||||
Size: SyntheticDirectorySize,
|
Size: deprecatedSyntheticDirectorySize,
|
||||||
},
|
},
|
||||||
ok: false,
|
ok: false,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -41,9 +41,9 @@ type testfileList []testfile
|
|||||||
|
|
||||||
var testdata = testfileList{
|
var testdata = testfileList{
|
||||||
{"afile", 4, "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"},
|
{"afile", 4, "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"},
|
||||||
{"dir1", 128, ""},
|
{"dir1", 0, ""},
|
||||||
{filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"},
|
{filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"},
|
||||||
{"dir2", 128, ""},
|
{"dir2", 0, ""},
|
||||||
{filepath.Join("dir2", "cfile"), 4, "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c"},
|
{filepath.Join("dir2", "cfile"), 4, "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c"},
|
||||||
{"excludes", 37, "df90b52f0c55dba7a7a940affe482571563b1ac57bd5be4d8a0291e7de928e06"},
|
{"excludes", 37, "df90b52f0c55dba7a7a940affe482571563b1ac57bd5be4d8a0291e7de928e06"},
|
||||||
{"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"},
|
{"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"},
|
||||||
@@ -601,7 +601,7 @@ func (l fileList) testfiles() testfileList {
|
|||||||
if len(f.Blocks) > 1 {
|
if len(f.Blocks) > 1 {
|
||||||
panic("simple test case stuff only supports a single block per file")
|
panic("simple test case stuff only supports a single block per file")
|
||||||
}
|
}
|
||||||
testfiles[i] = testfile{name: f.Name, length: f.FileSize()}
|
testfiles[i] = testfile{name: f.Name, length: f.Size}
|
||||||
if len(f.Blocks) == 1 {
|
if len(f.Blocks) == 1 {
|
||||||
testfiles[i].hash = fmt.Sprintf("%x", f.Blocks[0].Hash)
|
testfiles[i].hash = fmt.Sprintf("%x", f.Blocks[0].Hash)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user