From 506dfb879d550ae7627f231635646067456c6fd5 Mon Sep 17 00:00:00 2001 From: Daniel Milde Date: Sun, 14 Apr 2024 22:23:19 +0200 Subject: [PATCH] feat: delete directory items in parallel --- README.md | 16 +++++++ cmd/gdu/app/app.go | 6 +++ pkg/analyze/file.go | 2 +- pkg/analyze/stored.go | 3 +- pkg/remove/parallel.go | 58 +++++++++++++++++++++++ pkg/remove/parallel_linux_test.go | 42 +++++++++++++++++ pkg/remove/parallel_test.go | 69 +++++++++++++++++++++++++++ tui/tui.go | 5 ++ tui/tui_test.go | 78 +++++++++++++++++++++++++++++++ 9 files changed, 277 insertions(+), 2 deletions(-) create mode 100644 pkg/remove/parallel.go create mode 100644 pkg/remove/parallel_linux_test.go create mode 100644 pkg/remove/parallel_test.go diff --git a/README.md b/README.md index fb0f2eef1..8b3e6381b 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,22 @@ style: background-color: "#ff0000" ``` +## Deletion in background and in parallel (experimental) + +Gdu can delete items in the background, thus not blocking the UI for additional work. +To enable: + +``` +echo "delete-in-background: true" >> ~/.gdu.yaml +``` + +Directory items can be also deleted in parallel, which can increase the speed of deletion. +To enable: + +``` +echo "delete-in-parallel: true" >> ~/.gdu.yaml +``` + ## Memory usage ### Automatic balancing diff --git a/cmd/gdu/app/app.go b/cmd/gdu/app/app.go index 4ac669665..96bc77e31 100644 --- a/cmd/gdu/app/app.go +++ b/cmd/gdu/app/app.go @@ -76,6 +76,7 @@ type Flags struct { WriteConfig bool `yaml:"-"` ChangeCwd bool `yaml:"change-cwd"` DeleteInBackground bool `yaml:"delete-in-background"` + DeleteInParallel bool `yaml:"delete-in-parallel"` Style Style `yaml:"style"` Sorting Sorting `yaml:"sorting"` } @@ -286,6 +287,11 @@ func (a *App) createUI() (UI, error) { ui.SetDeleteInBackground() }) } + if a.Flags.DeleteInParallel { + opts = append(opts, func(ui *tui.UI) { + ui.SetDeleteInParallel() + }) + } ui = tui.CreateUI( a.TermApp, diff --git a/pkg/analyze/file.go b/pkg/analyze/file.go index a933d8fc1..8184ec7dc 100644 --- a/pkg/analyze/file.go +++ b/pkg/analyze/file.go @@ -230,7 +230,7 @@ func (f *Dir) UpdateStats(linkedItems fs.HardLinkedItems) { f.Usage = totalUsage } -// RemoveFile panics on file +// RemoveFile removes item from dir, updates size and item count func (f *Dir) RemoveFile(item fs.Item) { f.m.Lock() defer f.m.Unlock() diff --git a/pkg/analyze/stored.go b/pkg/analyze/stored.go index 79bdd037c..4e0ba1041 100644 --- a/pkg/analyze/stored.go +++ b/pkg/analyze/stored.go @@ -282,7 +282,8 @@ func (f *StoredDir) SetFiles(files fs.Files) { f.Files = files } -// RemoveFile panics on file +// RemoveFile removes file from stored directory +// It also updates size and item count of parent directories func (f *StoredDir) RemoveFile(item fs.Item) { if !DefaultStorage.IsOpen() { f.dbLock.Lock() diff --git a/pkg/remove/parallel.go b/pkg/remove/parallel.go new file mode 100644 index 000000000..1f9c57e67 --- /dev/null +++ b/pkg/remove/parallel.go @@ -0,0 +1,58 @@ +package remove + +import ( + "os" + "runtime" + "sync" + + "github.com/dundee/gdu/v5/pkg/fs" +) + +var concurrencyLimit = make(chan struct{}, 3*runtime.GOMAXPROCS(0)) + +// RemoveItemFromDirParallel removes item from dir +func RemoveItemFromDirParallel(dir fs.Item, item fs.Item) error { + if !item.IsDir() { + return RemoveItemFromDir(dir, item) + } + errChan := make(chan error, 1) // we show only first error + var wait sync.WaitGroup + + // remove all files in the directory in parallel + for _, file := range item.GetFilesLocked() { + wait.Add(1) + go func(itemPath string) { + concurrencyLimit <- struct{}{} + defer func() { <-concurrencyLimit }() + + err := os.RemoveAll(itemPath) + if err != nil { + select { + // write error to channel if it's empty + case errChan <- err: + default: + } + } + wait.Done() + }(file.GetPath()) + } + + wait.Wait() + + // check if there was an error + select { + case err := <-errChan: + return err + default: + } + + // remove the directory itself + err := os.RemoveAll(item.GetPath()) + if err != nil { + return err + } + + // update parent directory + dir.RemoveFile(item) + return nil +} diff --git a/pkg/remove/parallel_linux_test.go b/pkg/remove/parallel_linux_test.go new file mode 100644 index 000000000..1bd05199e --- /dev/null +++ b/pkg/remove/parallel_linux_test.go @@ -0,0 +1,42 @@ +//go:build linux +// +build linux + +package remove + +import ( + "os" + "testing" + + "github.com/dundee/gdu/v5/internal/testdir" + "github.com/dundee/gdu/v5/pkg/analyze" + "github.com/stretchr/testify/assert" +) + +func TestRemoveItemFromDirParallelWithErr(t *testing.T) { + fin := testdir.CreateTestDir() + defer fin() + + err := os.Chmod("test_dir/nested", 0) + assert.Nil(t, err) + defer func() { + err = os.Chmod("test_dir/nested", 0755) + assert.Nil(t, err) + }() + + dir := &analyze.Dir{ + File: &analyze.File{ + Name: "test_dir", + }, + BasePath: ".", + } + + subdir := &analyze.Dir{ + File: &analyze.File{ + Name: "nested", + Parent: dir, + }, + } + + err = RemoveItemFromDirParallel(dir, subdir) + assert.Contains(t, err.Error(), "permission denied") +} diff --git a/pkg/remove/parallel_test.go b/pkg/remove/parallel_test.go new file mode 100644 index 000000000..6196d7875 --- /dev/null +++ b/pkg/remove/parallel_test.go @@ -0,0 +1,69 @@ +package remove + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/dundee/gdu/v5/internal/testdir" + "github.com/dundee/gdu/v5/pkg/analyze" + "github.com/dundee/gdu/v5/pkg/fs" +) + +func TestRemoveFileParallel(t *testing.T) { + dir := &analyze.Dir{ + File: &analyze.File{ + Name: "xxx", + Size: 5, + Usage: 12, + }, + ItemCount: 3, + BasePath: ".", + } + + subdir := &analyze.Dir{ + File: &analyze.File{ + Name: "yyy", + Size: 4, + Usage: 8, + Parent: dir, + }, + ItemCount: 2, + } + file := &analyze.File{ + Name: "zzz", + Size: 3, + Usage: 4, + Parent: subdir, + } + dir.Files = fs.Files{subdir} + subdir.Files = fs.Files{file} + + err := RemoveItemFromDirParallel(subdir, file) + assert.Nil(t, err) + + assert.Equal(t, 0, len(subdir.Files)) + assert.Equal(t, 1, subdir.ItemCount) + assert.Equal(t, int64(1), subdir.Size) + assert.Equal(t, int64(4), subdir.Usage) + assert.Equal(t, 1, len(dir.Files)) + assert.Equal(t, 2, dir.ItemCount) + assert.Equal(t, int64(2), dir.Size) +} + +func TestRemoveDirParallel(t *testing.T) { + fin := testdir.CreateTestDir() + defer fin() + + analyzer := analyze.CreateAnalyzer() + dir := analyzer.AnalyzeDir( + "test_dir", func(_, _ string) bool { return false }, false, + ).(*analyze.Dir) + analyzer.GetDone().Wait() + dir.UpdateStats(make(fs.HardLinkedItems)) + + subdir := dir.Files[0].(*analyze.Dir) + + err := RemoveItemFromDirParallel(dir, subdir) + assert.Nil(t, err) +} diff --git a/tui/tui.go b/tui/tui.go index 03a35d49e..57edb985e 100644 --- a/tui/tui.go +++ b/tui/tui.go @@ -217,6 +217,11 @@ func (ui *UI) SetChangeCwdFn(fn func(string) error) { ui.changeCwdFn = fn } +// SetDeleteInParallel sets the flag to delete files in parallel +func (ui *UI) SetDeleteInParallel() { + ui.remover = remove.RemoveItemFromDirParallel +} + // StartUILoop starts tview application func (ui *UI) StartUILoop() error { return ui.app.Run() diff --git a/tui/tui_test.go b/tui/tui_test.go index 6974de1b1..abf3d9556 100644 --- a/tui/tui_test.go +++ b/tui/tui_test.go @@ -330,6 +330,29 @@ func TestDeleteSelected(t *testing.T) { assert.NoDirExists(t, "test_dir/nested") } +func TestDeleteSelectedInParallel(t *testing.T) { + fin := testdir.CreateTestDir() + defer fin() + + ui := getAnalyzedPathMockedApp(t, false, true, false) + ui.done = make(chan struct{}) + ui.SetDeleteInParallel() + + assert.Equal(t, 1, ui.table.GetRowCount()) + + ui.table.Select(0, 0) + + ui.deleteSelected(false) + + <-ui.done + + for _, f := range ui.app.(*testapp.MockedApp).GetUpdateDraws() { + f() + } + + assert.NoDirExists(t, "test_dir/nested") +} + func TestDeleteSelectedInBackground(t *testing.T) { fin := testdir.CreateTestDir() defer fin() @@ -354,6 +377,31 @@ func TestDeleteSelectedInBackground(t *testing.T) { assert.NoDirExists(t, "test_dir/nested") } +func TestDeleteSelectedInBackgroundAndParallel(t *testing.T) { + fin := testdir.CreateTestDir() + defer fin() + + ui := getAnalyzedPathMockedApp(t, true, true, false) + ui.remover = testanalyze.RemoveItemFromDirWithSleep + ui.done = make(chan struct{}) + ui.SetDeleteInBackground() + ui.SetDeleteInParallel() + + assert.Equal(t, 1, ui.table.GetRowCount()) + + ui.table.Select(0, 0) + + ui.deleteSelected(false) + + <-ui.done + + for _, f := range ui.app.(*testapp.MockedApp).GetUpdateDraws() { + f() + } + + assert.NoDirExists(t, "test_dir/nested") +} + func TestDeleteSelectedInBackgroundBW(t *testing.T) { fin := testdir.CreateTestDir() defer fin() @@ -568,6 +616,36 @@ func TestDeleteMarkedInBackgroundWithStorage(t *testing.T) { assert.NoFileExists(t, "test_dir/nested/file2") } +func TestDeleteMarkedInBackgroundWithStorageAndParallel(t *testing.T) { + fin := testdir.CreateTestDir() + defer fin() + + ui := getAnalyzedPathMockedApp(t, false, true, false) + ui.SetAnalyzer(analyze.CreateStoredAnalyzer("/tmp/badger")) + ui.SetDeleteInBackground() + ui.SetDeleteInParallel() + + assert.Equal(t, 1, ui.table.GetRowCount()) + + ui.fileItemSelected(0, 0) // nested + + ui.markedRows[1] = struct{}{} // subnested + ui.markedRows[2] = struct{}{} // file2 + + ui.deleteMarked(false) + + <-ui.done // wait for deletion of subnested + <-ui.done // wait for deletion of file2 + + for _, f := range ui.app.(*testapp.MockedApp).GetUpdateDraws() { + f() + } + + assert.DirExists(t, "test_dir/nested") + assert.NoDirExists(t, "test_dir/nested/subnested") + assert.NoFileExists(t, "test_dir/nested/file2") +} + func TestDeleteMarkedInBackgroundWithErr(t *testing.T) { fin := testdir.CreateTestDir() defer fin()