Skip to content

Commit

Permalink
lib/model: Compare all items with global on scan (fixes syncthing#7740)…
Browse files Browse the repository at this point in the history 
… (syncthing#7791)
  • Loading branch information
@imsodin
imsodin authored Jun 27, 2021
1 parent 69ce121 commit 445a82f
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 80 deletions.
160 changes: 81 additions & 79 deletions lib/model/folder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -473,16 +473,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
f.setState(FolderScanning)
f.clearScanErrors(subDirs)

batch := db.NewFileInfoBatch(func(fs []protocol.FileInfo) error {
if err := f.getHealthErrorWithoutIgnores(); err != nil {
l.Debugf("Stopping scan of folder %s due to: %s", f.Description(), err)
return err
}
f.updateLocalsFromScanning(fs)
return nil
})

batchAppend := f.scanSubdirsBatchAppendFunc(batch)
batch := f.newScanBatch()

// Schedule a pull after scanning, but only if we actually detected any
// changes.
Expand All @@ -494,7 +485,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
}
}()

changesHere, err := f.scanSubdirsChangedAndNew(subDirs, batch, batchAppend)
changesHere, err := f.scanSubdirsChangedAndNew(subDirs, batch)
changes += changesHere
if err != nil {
return err
Expand All @@ -513,7 +504,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
// Do a scan of the database for each prefix, to check for deleted and
// ignored files.

changesHere, err = f.scanSubdirsDeletedAndIgnored(subDirs, batch, batchAppend)
changesHere, err = f.scanSubdirsDeletedAndIgnored(subDirs, batch)
changes += changesHere
if err != nil {
return err
Expand All @@ -527,58 +518,57 @@ func (f *folder) scanSubdirs(subDirs []string) error {
return nil
}

type batchAppendFunc func(protocol.FileInfo, *db.Snapshot) bool

func (f *folder) scanSubdirsBatchAppendFunc(batch *db.FileInfoBatch) batchAppendFunc {
// Resolve items which are identical with the global state.
switch f.Type {
case config.FolderTypeReceiveOnly:
return func(fi protocol.FileInfo, snap *db.Snapshot) bool {
switch gf, ok := snap.GetGlobal(fi.Name); {
case !ok:
case gf.IsEquivalentOptional(fi, f.modTimeWindow, false, false, protocol.FlagLocalReceiveOnly):
// What we have locally is equivalent to the global file.
fi.Version = gf.Version
l.Debugf("%v scanning: Merging identical locally changed item with global", f, fi)
fallthrough
case fi.IsDeleted() && (gf.IsReceiveOnlyChanged() || gf.IsDeleted()):
// Our item is deleted and the global item is our own
// receive only file or deleted too. In the former
// case we can't delete file infos, so we just
// pretend it is a normal deleted file (nobody
// cares about that).
l.Debugf("%v scanning: Marking item as not locally changed", f, fi)
fi.LocalFlags &^= protocol.FlagLocalReceiveOnly
}
batch.Append(fi)
return true
type scanBatch struct {
*db.FileInfoBatch
f *folder
}

func (f *folder) newScanBatch() *scanBatch {
b := &scanBatch{
f: f,
}
b.FileInfoBatch = db.NewFileInfoBatch(func(fs []protocol.FileInfo) error {
if err := b.f.getHealthErrorWithoutIgnores(); err != nil {
l.Debugf("Stopping scan of folder %s due to: %s", b.f.Description(), err)
return err
}
case config.FolderTypeReceiveEncrypted:
return func(fi protocol.FileInfo, _ *db.Snapshot) bool {
// This is a "virtual" parent directory of encrypted files.
// We don't track it, but check if anything still exists
// within and delete it otherwise.
if fi.IsDirectory() && protocol.IsEncryptedParent(fs.PathComponents(fi.Name)) {
if names, err := f.mtimefs.DirNames(fi.Name); err == nil && len(names) == 0 {
f.mtimefs.Remove(fi.Name)
}
return false
}
// Any local change must not be sent as index entry to
// remotes and show up as an error in the UI.
fi.LocalFlags = protocol.FlagLocalReceiveOnly
batch.Append(fi)
return true
b.f.updateLocalsFromScanning(fs)
return nil
})
return b
}

// Append adds the fileinfo to the batch for updating, and does a few checks.
// It returns false if the checks result in the file not going to be updated or removed.
func (b *scanBatch) Append(fi protocol.FileInfo, snap *db.Snapshot) bool {
// Check for a "virtual" parent directory of encrypted files. We don't track
// it, but check if anything still exists within and delete it otherwise.
if b.f.Type == config.FolderTypeReceiveEncrypted && fi.IsDirectory() && protocol.IsEncryptedParent(fs.PathComponents(fi.Name)) {
if names, err := b.f.mtimefs.DirNames(fi.Name); err == nil && len(names) == 0 {
b.f.mtimefs.Remove(fi.Name)
}
default:
return func(fi protocol.FileInfo, _ *db.Snapshot) bool {
batch.Append(fi)
return true
return false
}
// Resolve receive-only items which are identical with the global state or
// the global item is our own receive-only item.
// Except if they are in a receive-encrypted folder and are locally added.
// Those must never be sent in index updates and thus must retain the flag.
switch gf, ok := snap.GetGlobal(fi.Name); {
case !ok:
case gf.IsReceiveOnlyChanged():
if b.f.Type == config.FolderTypeReceiveOnly && fi.Deleted {
l.Debugf("%v scanning: Marking deleted item as not locally changed", b.f, fi)
fi.LocalFlags &^= protocol.FlagLocalReceiveOnly
}
case gf.IsEquivalentOptional(fi, b.f.modTimeWindow, false, false, protocol.FlagLocalReceiveOnly):
l.Debugf("%v scanning: Replacing scanned file info with global as it's equivalent", b.f, fi)
fi = gf
}
b.FileInfoBatch.Append(fi)
return true
}

func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *db.FileInfoBatch, batchAppend batchAppendFunc) (int, error) {
func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *scanBatch) (int, error) {
changes := 0
snap, err := f.dbSnapshot()
if err != nil {
Expand Down Expand Up @@ -630,15 +620,15 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *db.FileInfoBa
return changes, err
}

if batchAppend(res.File, snap) {
if batch.Append(res.File, snap) {
changes++
}

switch f.Type {
case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted:
default:
if nf, ok := f.findRename(snap, res.File, alreadyUsedOrExisting); ok {
if batchAppend(nf, snap) {
if batch.Append(nf, snap) {
changes++
}
}
Expand All @@ -648,7 +638,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *db.FileInfoBa
return changes, nil
}

func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileInfoBatch, batchAppend batchAppendFunc) (int, error) {
func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch) (int, error) {
var toIgnore []db.FileInfoTruncated
ignoredParent := ""
changes := 0
Expand Down Expand Up @@ -679,7 +669,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileIn
for _, file := range toIgnore {
l.Debugln("marking file as ignored", file)
nf := file.ConvertToIgnoredFileInfo()
if batchAppend(nf, snap) {
if batch.Append(nf, snap) {
changes++
}
if err := batch.FlushIfFull(); err != nil {
Expand Down Expand Up @@ -709,7 +699,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileIn

l.Debugln("marking file as ignored", file)
nf := file.ConvertToIgnoredFileInfo()
if batchAppend(nf, snap) {
if batch.Append(nf, snap) {
changes++
}

Expand Down Expand Up @@ -739,23 +729,35 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileIn
nf.Version = protocol.Vector{}
}
l.Debugln("marking file as deleted", nf)
if batchAppend(nf, snap) {
changes++
}
case file.IsDeleted() && file.IsReceiveOnlyChanged() && f.Type == config.FolderTypeReceiveOnly && len(snap.Availability(file.Name)) == 0:
file.Version = protocol.Vector{}
file.LocalFlags &^= protocol.FlagLocalReceiveOnly
l.Debugln("marking deleted item that doesn't exist anywhere as not receive-only", file)
if batchAppend(file.ConvertDeletedToFileInfo(), snap) {
if batch.Append(nf, snap) {
changes++
}
case file.IsDeleted() && file.IsReceiveOnlyChanged() && f.Type != config.FolderTypeReceiveOnly:
// No need to bump the version for a file that was and is
// deleted and just the folder type/local flags changed.
file.LocalFlags &^= protocol.FlagLocalReceiveOnly
l.Debugln("removing receive-only flag on deleted item", file)
if batchAppend(file.ConvertDeletedToFileInfo(), snap) {
changes++
case file.IsDeleted() && file.IsReceiveOnlyChanged():
switch f.Type {
case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted:
if gf, _ := snap.GetGlobal(file.Name); gf.IsDeleted() {
// Our item is deleted and the global item is deleted too. We just
// pretend it is a normal deleted file (nobody cares about that).
// Except if this is a receive-encrypted folder and it
// is a locally added file. Those must never be sent
// in index updates and thus must retain the flag.
if f.Type == config.FolderTypeReceiveEncrypted && gf.IsReceiveOnlyChanged() {
return true
}
l.Debugf("%v scanning: Marking globally deleted item as not locally changed: %v", f, file.Name)
file.LocalFlags &^= protocol.FlagLocalReceiveOnly
if batch.Append(file.ConvertDeletedToFileInfo(), snap) {
changes++
}
}
default:
// No need to bump the version for a file that was and is
// deleted and just the folder type/local flags changed.
file.LocalFlags &^= protocol.FlagLocalReceiveOnly
l.Debugln("removing receive-only flag on deleted item", file)
if batch.Append(file.ConvertDeletedToFileInfo(), snap) {
changes++
}
}
}

Expand All @@ -772,7 +774,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileIn
for _, file := range toIgnore {
l.Debugln("marking file as ignored", f)
nf := file.ConvertToIgnoredFileInfo()
if batchAppend(nf, snap) {
if batch.Append(nf, snap) {
changes++
}
if iterError = batch.FlushIfFull(); iterError != nil {
Expand Down
4 changes: 3 additions & 1 deletion lib/model/folder_recvenc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@ type receiveEncryptedFolder struct {
}

func newReceiveEncryptedFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *util.Semaphore) service {
return &receiveEncryptedFolder{newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)}
f := &receiveEncryptedFolder{newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)}
f.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files
return f
}

func (f *receiveEncryptedFolder) Revert() {
Expand Down

0 comments on commit 445a82f

Please sign in to comment.