Skip to content

Commit

Permalink
Use 32KB buffer for copyFile() (reduces copy time by 30%) (#1749)
Browse files Browse the repository at this point in the history
* Use 32KB buffer for `copyFile()` (reduces copy time by 30%)

This makes `:paste` comparable in performance to `cp --reflink=never`
for large files.

For large number of small files improvements are less substantial
(compared to `cp -r --reflink=never`), though still noticeable.

In both cases the copy takes about 30% less time than with `buf` size at 4096.

32KB is the same number `io.Copy()` uses internally (when it can), and is about where improvements stop.

Context:
#1685 (comment)

* Apply suggested changes

* Just delete incomplete an file on all errors

* Mention new progress UI update frequency

* Return the old (each 4MB copied) progress update frequency

(progress UI updates don't seem to impact performance much)
  • Loading branch information
MahouShoujoMivutilde authored Jun 14, 2024
1 parent 0c72034 commit 2d3cc97
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 21 deletions.
4 changes: 2 additions & 2 deletions app.go
Original file line number Diff line number Diff line change
Expand Up @@ -332,8 +332,8 @@ func (app *app) loop() {
return
case n := <-app.nav.copyBytesChan:
app.nav.copyBytes += n
// n is usually 4096B so update roughly per 4096B x 1024 = 4MB copied
if app.nav.copyUpdate++; app.nav.copyUpdate >= 1024 {
// n is usually 32*1024B (default io.Copy() buffer) so update roughly per 32KB x 128 = 4MB copied
if app.nav.copyUpdate++; app.nav.copyUpdate >= 128 {
app.nav.copyUpdate = 0
app.ui.draw(app.nav)
}
Expand Down
42 changes: 23 additions & 19 deletions copy.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,24 @@ import (
"github.com/djherbis/times"
)

type ProgressWriter struct {
writer io.Writer
nums chan<- int64
}

func NewProgressWriter(writer io.Writer, nums chan<- int64) *ProgressWriter {
return &ProgressWriter{
writer: writer,
nums: nums,
}
}

func (progressWriter *ProgressWriter) Write(b []byte) (int, error) {
n, err := progressWriter.writer.Write(b)
progressWriter.nums <- int64(n)
return n, err
}

func copySize(srcs []string) (int64, error) {
var total int64

Expand Down Expand Up @@ -47,8 +65,6 @@ func copyFile(src, dst string, preserve []string, info os.FileInfo, nums chan in
}
}

buf := make([]byte, 4096)

r, err := os.Open(src)
if err != nil {
return err
Expand All @@ -60,23 +76,11 @@ func copyFile(src, dst string, preserve []string, info os.FileInfo, nums chan in
return err
}

for {
n, err := r.Read(buf)
if err != nil && err != io.EOF {
w.Close()
os.Remove(dst)
return err
}

if n == 0 {
break
}

if _, err := w.Write(buf[:n]); err != nil {
return err
}

nums <- int64(n)
_, err = io.Copy(NewProgressWriter(w, nums), r)
if err != nil {
w.Close()
os.Remove(dst)
return err
}

if err := w.Close(); err != nil {
Expand Down

0 comments on commit 2d3cc97

Please sign in to comment.