Skip to content

Commit 060996e

Browse files
authored
Merge pull request #34 from ovh/perf/stream
Improve chunks reader performance
2 parents 6063698 + 3300088 commit 060996e

File tree

4 files changed

+90
-26
lines changed

4 files changed

+90
-26
lines changed

go.mod

+4-4
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,20 @@ require (
77
github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75
88
github.com/ovh/configstore v0.6.2
99
github.com/pelletier/go-toml v1.8.0
10-
github.com/stretchr/testify v1.8.2
11-
golang.org/x/crypto v0.23.0
10+
github.com/stretchr/testify v1.9.0
11+
golang.org/x/crypto v0.25.0
1212
gopkg.in/alecthomas/kingpin.v2 v2.2.6
1313
gopkg.in/yaml.v2 v2.4.0
1414
)
1515

1616
require (
1717
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
18-
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
18+
github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect
1919
github.com/davecgh/go-spew v1.1.1 // indirect
2020
github.com/fsnotify/fsnotify v1.7.0 // indirect
2121
github.com/ghodss/yaml v1.0.0 // indirect
2222
github.com/pmezard/go-difflib v1.0.0 // indirect
23-
golang.org/x/sys v0.20.0 // indirect
23+
golang.org/x/sys v0.22.0 // indirect
2424
gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2 // indirect
2525
gopkg.in/yaml.v3 v3.0.1 // indirect
2626
)

go.sum

+10-10
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ github.com/SSSaaS/sssa-golang v0.0.0-20170502204618-d37d7782d752 h1:NMpC6M+PtNND
44
github.com/SSSaaS/sssa-golang v0.0.0-20170502204618-d37d7782d752/go.mod h1:PbJ8S5YaSYAvDPTiEuUsBHQwTUlPs6VM+Av8Oi3v570=
55
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
66
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
7-
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs=
8-
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
7+
github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 h1:t3eaIm0rUkzbrIewtiFmMK5RXHej2XnoXNhxVsAYUfg=
8+
github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs=
99
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
1010
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
1111
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -25,24 +25,24 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
2525
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
2626
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
2727
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
28-
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
28+
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
2929
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
3030
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
3131
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
32-
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
33-
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
34-
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
35-
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
32+
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
33+
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
34+
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
35+
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
36+
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
3637
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
3738
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
38-
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
39-
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
39+
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
40+
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
4041
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
4142
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
4243
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
4344
gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2 h1:+j1SppRob9bAgoYmsdW9NNBdKZfgYuWpqnYHv78Qt8w=
4445
gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
45-
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
4646
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
4747
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
4848
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

stream/stream.go

+54-12
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"bytes"
55
"encoding/binary"
66
"io"
7+
"sync"
78

89
"github.com/ovh/symmecrypt"
910
_ "github.com/ovh/symmecrypt/keyloader"
@@ -42,29 +43,54 @@ func (k key) DecryptPipe(r io.Reader, w io.Writer, extra ...[]byte) error {
4243
return nil
4344
}
4445

46+
var buffers = sync.Pool{
47+
New: func() interface{} {
48+
return new(bytes.Buffer)
49+
},
50+
}
51+
52+
func getBuffer() (*bytes.Buffer, error) {
53+
if b, ok := buffers.Get().(*bytes.Buffer); ok {
54+
b.Reset()
55+
return b, nil
56+
}
57+
58+
panic("buffers is not of type *bytes.Buffer")
59+
}
60+
61+
func putBuffer(buf *bytes.Buffer) {
62+
buffers.Put(buf)
63+
}
64+
4565
var _ io.WriteCloser = new(chunksWriter)
4666

4767
type chunksWriter struct {
4868
destination io.Writer
4969
k symmecrypt.Key
5070
extras [][]byte
5171
chunkSize int
52-
currentChunkWriter *bytes.Buffer
72+
buf *bytes.Buffer
5373
currentChunkBytesWritten int
5474
}
5575

5676
func (w *chunksWriter) encryptCurrentChunk() error {
57-
if w.currentChunkWriter == nil && w.currentChunkBytesWritten == 0 {
77+
if w.buf == nil && w.currentChunkBytesWritten == 0 {
5878
return nil
5979
}
60-
currentChunk := w.currentChunkWriter.Bytes()
80+
6181
// first step: encrypt the chunks
62-
btes, err := w.k.Encrypt(currentChunk, w.extras...)
82+
btes, err := w.k.Encrypt(w.buf.Bytes(), w.extras...)
6383
if err != nil {
6484
return err
6585
}
6686

6787
// then write into the destination writer the len of the encrypted chunks
88+
headerBytes, err := getBuffer()
89+
if err != nil {
90+
return err
91+
}
92+
defer putBuffer(headerBytes)
93+
6894
headerBuf := make([]byte, binary.MaxVarintLen32)
6995
binary.PutUvarint(headerBuf, uint64(len(btes)))
7096
if _, err := w.destination.Write(headerBuf); err != nil {
@@ -76,7 +102,8 @@ func (w *chunksWriter) encryptCurrentChunk() error {
76102

77103
// finally reset the current chunk
78104
w.currentChunkBytesWritten = 0
79-
w.currentChunkWriter = nil
105+
putBuffer(w.buf)
106+
w.buf = nil
80107

81108
return err
82109
}
@@ -85,8 +112,13 @@ func (w *chunksWriter) Write(p []byte) (int, error) {
85112
if len(p) == 0 {
86113
return len(p), nil
87114
}
88-
if w.currentChunkWriter == nil {
89-
w.currentChunkWriter = new(bytes.Buffer)
115+
if w.buf == nil {
116+
chunkWriter, err := getBuffer()
117+
if err != nil {
118+
return 0, err
119+
}
120+
121+
w.buf = chunkWriter
90122
w.currentChunkBytesWritten = 0
91123
}
92124

@@ -97,7 +129,7 @@ func (w *chunksWriter) Write(p []byte) (int, error) {
97129
}
98130

99131
if w.currentChunkBytesWritten+len(p) == w.chunkSize {
100-
n, err := w.currentChunkWriter.Write(p)
132+
n, err := w.buf.Write(p)
101133
if err != nil {
102134
return n, err
103135
}
@@ -110,7 +142,7 @@ func (w *chunksWriter) Write(p []byte) (int, error) {
110142

111143
x := w.chunkSize - w.currentChunkBytesWritten
112144
if len(p) < x {
113-
n, err := w.currentChunkWriter.Write(p)
145+
n, err := w.buf.Write(p)
114146
w.currentChunkBytesWritten += int(n)
115147
return n, err
116148
} else {
@@ -176,19 +208,29 @@ func NewReader(r io.Reader, k symmecrypt.Key, chunkSize int, extras ...[]byte) i
176208
}
177209

178210
func (r *chunksReader) readNewChunk() error {
211+
headerBtes, err := getBuffer()
212+
if err != nil {
213+
return err
214+
}
215+
defer putBuffer(headerBtes)
216+
179217
// read the chunksize
180-
var headerBtes = new(bytes.Buffer)
181218
if _, err := io.CopyN(headerBtes, r.src, binary.MaxVarintLen32); err != nil {
182219
return err
183220
}
184221

185-
n, err := binary.ReadUvarint(bytes.NewReader(headerBtes.Bytes())) // READ THE HEADER BUFFER
222+
n, err := binary.ReadUvarint(headerBtes) // READ THE HEADER BUFFER
186223
if err != nil {
187224
return err
188225
}
189226

190227
// read the chunk content
191-
var btsBuff = new(bytes.Buffer)
228+
btsBuff, err := getBuffer()
229+
if err != nil {
230+
return err
231+
}
232+
defer putBuffer(btsBuff)
233+
192234
if _, err := io.CopyN(btsBuff, r.src, int64(n)); err != nil && err != io.EOF {
193235
return err
194236
}

stream/stream_test.go

+22
Original file line numberDiff line numberDiff line change
@@ -70,5 +70,27 @@ func TestIncompleteRead(t *testing.T) {
7070
assert.Contains(t, err.Error(), "EOF")
7171

7272
require.Equal(t, 32*1024+10, nbBytesReaden1+nbBytesReaden2)
73+
}
74+
75+
func BenchmarkStream(b *testing.B) {
76+
for i := 0; i < b.N; i++ {
77+
clearContent := make([]byte, 32*1024+10)
78+
rand.Read(clearContent) // nolint
79+
80+
k, _ := keyloader.LoadKey("test")
7381

82+
var bufWriter bytes.Buffer
83+
streamWriter := stream.NewWriter(&bufWriter, k, 32*1024)
84+
_, err := io.Copy(streamWriter, bytes.NewReader(clearContent))
85+
require.NoError(b, err)
86+
87+
streamReader := stream.NewReader(strings.NewReader(bufWriter.String()), k, 32*1024)
88+
var firstPart = make([]byte, 32*1024)
89+
_, err = streamReader.Read(firstPart)
90+
require.NoError(b, err)
91+
92+
var secondPart = make([]byte, 32*1024)
93+
_, err = streamReader.Read(secondPart)
94+
require.NoError(b, err)
95+
}
7496
}

0 commit comments

Comments
 (0)