all: Add copy-on-write filesystem support (fixes #4271) (#6746)

lib/model/folder_sendrecv.go

@@ -986,13 +986,13 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, sn
 	if f.versioner != nil {
 		err = f.CheckAvailableSpace(source.Size)
 		if err == nil {
-			err = osutil.Copy(f.fs, f.fs, source.Name, tempName)
+			err = osutil.Copy(f.CopyRangeMethod, f.fs, f.fs, source.Name, tempName)
 			if err == nil {
 				err = f.inWritableDir(f.versioner.Archive, source.Name)
 			}
 		}
 	} else {
-		err = osutil.RenameOrCopy(f.fs, f.fs, source.Name, tempName)
+		err = osutil.RenameOrCopy(f.CopyRangeMethod, f.fs, f.fs, source.Name, tempName)
 	}
 	if err != nil {
 		return err
@@ -1296,7 +1296,7 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
 					return true
 				}
 
-				_, err = f.limitedWriteAt(dstFd, buf, block.Offset)
+				err = f.limitedWriteAt(dstFd, buf, block.Offset)
 				if err != nil {
 					state.fail(errors.Wrap(err, "dst write"))
 				}
@@ -1314,13 +1314,14 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
 
 			if !found {
 				found = f.model.finder.Iterate(folders, block.Hash, func(folder, path string, index int32) bool {
-					fs := folderFilesystems[folder]
-					fd, err := fs.Open(path)
+					ffs := folderFilesystems[folder]
+					fd, err := ffs.Open(path)
 					if err != nil {
 						return false
 					}
 
-					_, err = fd.ReadAt(buf, int64(state.file.BlockSize())*int64(index))
+					srcOffset := int64(state.file.BlockSize()) * int64(index)
+					_, err = fd.ReadAt(buf, srcOffset)
 					fd.Close()
 					if err != nil {
 						return false
@@ -1331,7 +1332,15 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
 						return false
 					}
 
-					_, err = f.limitedWriteAt(dstFd, buf, block.Offset)
+					if f.CopyRangeMethod != fs.CopyRangeMethodStandard {
+						err = f.withLimiter(func() error {
+							dstFd.mut.Lock()
+							defer dstFd.mut.Unlock()
+							return fs.CopyRange(f.CopyRangeMethod, fd, dstFd.fd, srcOffset, block.Offset, int64(block.Size))
+						})
+					} else {
+						err = f.limitedWriteAt(dstFd, buf, block.Offset)
+					}
 					if err != nil {
 						state.fail(errors.Wrap(err, "dst write"))
 					}
@@ -1480,7 +1489,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
 		}
 
 		// Save the block data we got from the cluster
-		_, err = f.limitedWriteAt(fd, buf, state.block.Offset)
+		err = f.limitedWriteAt(fd, buf, state.block.Offset)
 		if err != nil {
 			state.fail(errors.Wrap(err, "save"))
 		} else {
@@ -1535,7 +1544,7 @@ func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCu
 
 	// Replace the original content with the new one. If it didn't work,
 	// leave the temp file in place for reuse.
-	if err := osutil.RenameOrCopy(f.fs, f.fs, tempName, file.Name); err != nil {
+	if err := osutil.RenameOrCopy(f.CopyRangeMethod, f.fs, f.fs, tempName, file.Name); err != nil {
 		return err
 	}
 
@@ -2006,12 +2015,19 @@ func (f *sendReceiveFolder) inWritableDir(fn func(string) error, path string) er
 	return inWritableDir(fn, f.fs, path, f.IgnorePerms)
 }
 
-func (f *sendReceiveFolder) limitedWriteAt(fd io.WriterAt, data []byte, offset int64) (int, error) {
+func (f *sendReceiveFolder) limitedWriteAt(fd io.WriterAt, data []byte, offset int64) error {
+	return f.withLimiter(func() error {
+		_, err := fd.WriteAt(data, offset)
+		return err
+	})
+}
+
+func (f *sendReceiveFolder) withLimiter(fn func() error) error {
 	if err := f.writeLimiter.takeWithContext(f.ctx, 1); err != nil {
-		return 0, err
+		return err
 	}
 	defer f.writeLimiter.give(1)
-	return fd.WriteAt(data, offset)
+	return fn()
 }
 
 // A []FileError is sent as part of an event and will be JSON serialized.

lib/model/folder_sendrecv_test.go

@@ -980,7 +980,7 @@ func TestDeleteBehindSymlink(t *testing.T) {
 	must(t, ffs.MkdirAll(link, 0755))
 	fi := createFile(t, file, ffs)
 	f.updateLocalsFromScanning([]protocol.FileInfo{fi})
-	must(t, osutil.RenameOrCopy(ffs, destFs, file, "file"))
+	must(t, osutil.RenameOrCopy(fs.CopyRangeMethodStandard, ffs, destFs, file, "file"))
 	must(t, ffs.RemoveAll(link))
 
 	if err := osutil.DebugSymlinkForTestsOnly(destFs.URI(), filepath.Join(ffs.URI(), link)); err != nil {

lib/model/model.go

@@ -341,7 +341,7 @@ func (m *model) addAndStartFolderLockedWithIgnores(cfg config.FolderConfiguratio
 	var ver versioner.Versioner
 	if cfg.Versioning.Type != "" {
 		var err error
-		ver, err = versioner.New(ffs, cfg.Versioning)
+		ver, err = versioner.New(cfg)
 		if err != nil {
 			panic(fmt.Errorf("creating versioner: %w", err))
 		}

lib/model/model_test.go

@@ -3636,10 +3636,10 @@ func TestRenameSameFile(t *testing.T) {
 	}
 
 	must(t, ffs.Rename("file", "file1"))
-	must(t, osutil.Copy(ffs, ffs, "file1", "file0"))
-	must(t, osutil.Copy(ffs, ffs, "file1", "file2"))
-	must(t, osutil.Copy(ffs, ffs, "file1", "file3"))
-	must(t, osutil.Copy(ffs, ffs, "file1", "file4"))
+	must(t, osutil.Copy(fs.CopyRangeMethodStandard, ffs, ffs, "file1", "file0"))
+	must(t, osutil.Copy(fs.CopyRangeMethodStandard, ffs, ffs, "file1", "file2"))
+	must(t, osutil.Copy(fs.CopyRangeMethodStandard, ffs, ffs, "file1", "file3"))
+	must(t, osutil.Copy(fs.CopyRangeMethodStandard, ffs, ffs, "file1", "file4"))
 
 	m.ScanFolders()
 

lib/model/sharedpullerstate.go

@@ -7,7 +7,6 @@
 package model
 
 import (
-	"io"
 	"time"
 
 	"github.com/pkg/errors"
@@ -118,7 +117,7 @@ func (w *lockedWriterAt) SyncClose(fsync bool) error {
 
 // tempFile returns the fd for the temporary file, reusing an open fd
 // or creating the file as necessary.
-func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
+func (s *sharedPullerState) tempFile() (*lockedWriterAt, error) {
 	s.mut.Lock()
 	defer s.mut.Unlock()