Skip to content

Commit 4f25bfc

Browse files
authored
Add msgpack serializers to stringset (#2107)
Add binary (de)serializers using msgp to stringsets. Should we move this to to `minio/pkg` instead?
1 parent 6651105 commit 4f25bfc

File tree

4 files changed

+181
-2
lines changed

4 files changed

+181
-2
lines changed

go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,14 @@ require (
1313
github.com/minio/crc64nvme v1.0.1
1414
github.com/minio/md5-simd v1.1.2
1515
github.com/rs/xid v1.6.0
16+
github.com/tinylib/msgp v1.3.0
1617
golang.org/x/crypto v0.36.0
1718
golang.org/x/net v0.38.0
1819
)
1920

2021
require (
2122
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
23+
github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect
2224
github.com/stretchr/testify v1.9.0 // indirect
2325
golang.org/x/sys v0.31.0 // indirect
2426
golang.org/x/text v0.23.0 // indirect

go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,16 @@ github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY
1717
github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
1818
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
1919
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
20+
github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY=
21+
github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
2022
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
2123
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
2224
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
2325
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
2426
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
2527
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
28+
github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww=
29+
github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
2630
golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
2731
golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
2832
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=

pkg/set/msgp.go

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
/*
2+
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
3+
* Copyright 2015-2025 MinIO, Inc.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package set
19+
20+
import "github.com/tinylib/msgp/msgp"
21+
22+
// EncodeMsg encodes the message to the writer.
23+
// Values are stored as a slice of strings or nil.
24+
func (s StringSet) EncodeMsg(writer *msgp.Writer) error {
25+
if s == nil {
26+
return writer.WriteNil()
27+
}
28+
err := writer.WriteArrayHeader(uint32(len(s)))
29+
if err != nil {
30+
return err
31+
}
32+
sorted := s.ToByteSlices()
33+
for _, k := range sorted {
34+
err = writer.WriteStringFromBytes(k)
35+
if err != nil {
36+
return err
37+
}
38+
}
39+
return nil
40+
}
41+
42+
// MarshalMsg encodes the message to the bytes.
43+
// Values are stored as a slice of strings or nil.
44+
func (s StringSet) MarshalMsg(bytes []byte) ([]byte, error) {
45+
if s == nil {
46+
return msgp.AppendNil(bytes), nil
47+
}
48+
if len(s) == 0 {
49+
return msgp.AppendArrayHeader(bytes, 0), nil
50+
}
51+
bytes = msgp.AppendArrayHeader(bytes, uint32(len(s)))
52+
sorted := s.ToByteSlices()
53+
for _, k := range sorted {
54+
bytes = msgp.AppendStringFromBytes(bytes, k)
55+
}
56+
return bytes, nil
57+
}
58+
59+
// DecodeMsg decodes the message from the reader.
60+
func (s *StringSet) DecodeMsg(reader *msgp.Reader) error {
61+
if reader.IsNil() {
62+
*s = nil
63+
return reader.Skip()
64+
}
65+
sz, err := reader.ReadArrayHeader()
66+
if err != nil {
67+
return err
68+
}
69+
dst := *s
70+
if dst == nil {
71+
dst = make(StringSet, sz)
72+
} else {
73+
for k := range dst {
74+
delete(dst, k)
75+
}
76+
}
77+
for i := uint32(0); i < sz; i++ {
78+
var k string
79+
k, err = reader.ReadString()
80+
if err != nil {
81+
return err
82+
}
83+
dst[k] = struct{}{}
84+
}
85+
*s = dst
86+
return nil
87+
}
88+
89+
// UnmarshalMsg decodes the message from the bytes.
90+
func (s *StringSet) UnmarshalMsg(bytes []byte) ([]byte, error) {
91+
if msgp.IsNil(bytes) {
92+
*s = nil
93+
return bytes[msgp.NilSize:], nil
94+
}
95+
// Read the array header
96+
sz, bytes, err := msgp.ReadArrayHeaderBytes(bytes)
97+
if err != nil {
98+
return nil, err
99+
}
100+
dst := *s
101+
if dst == nil {
102+
dst = make(StringSet, sz)
103+
} else {
104+
for k := range dst {
105+
delete(dst, k)
106+
}
107+
}
108+
for i := uint32(0); i < sz; i++ {
109+
var k string
110+
k, bytes, err = msgp.ReadStringBytes(bytes)
111+
if err != nil {
112+
return nil, err
113+
}
114+
dst[k] = struct{}{}
115+
}
116+
*s = dst
117+
return bytes, nil
118+
}
119+
120+
// Msgsize returns the maximum size of the message.
121+
func (s StringSet) Msgsize() int {
122+
if s == nil {
123+
return msgp.NilSize
124+
}
125+
if len(s) == 0 {
126+
return msgp.ArrayHeaderSize
127+
}
128+
size := msgp.ArrayHeaderSize
129+
for key := range s {
130+
size += msgp.StringPrefixSize + len(key)
131+
}
132+
return size
133+
}
134+
135+
// MarshalBinary encodes the receiver into a binary form and returns the result.
136+
func (s StringSet) MarshalBinary() ([]byte, error) {
137+
return s.MarshalMsg(nil)
138+
}
139+
140+
// AppendBinary appends the binary representation of itself to the end of b
141+
func (s StringSet) AppendBinary(b []byte) ([]byte, error) {
142+
return s.MarshalMsg(b)
143+
}
144+
145+
// UnmarshalBinary decodes the binary representation of itself from b
146+
func (s *StringSet) UnmarshalBinary(b []byte) error {
147+
_, err := s.UnmarshalMsg(b)
148+
return err
149+
}

pkg/set/stringset.go

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,30 @@ func (set StringSet) ToSlice() []string {
3737
return keys
3838
}
3939

40+
// ToByteSlices - returns StringSet as a sorted
41+
// slice of byte slices, using only one allocation.
42+
func (set StringSet) ToByteSlices() [][]byte {
43+
length := 0
44+
for k := range set {
45+
length += len(k)
46+
}
47+
// Preallocate the slice with the total length of all strings
48+
// to avoid multiple allocations.
49+
dst := make([]byte, length)
50+
51+
// Add keys to this...
52+
keys := make([][]byte, 0, len(set))
53+
for k := range set {
54+
n := copy(dst, k)
55+
keys = append(keys, dst[:n])
56+
dst = dst[n:]
57+
}
58+
sort.Slice(keys, func(i, j int) bool {
59+
return string(keys[i]) < string(keys[j])
60+
})
61+
return keys
62+
}
63+
4064
// IsEmpty - returns whether the set is empty or not.
4165
func (set StringSet) IsEmpty() bool {
4266
return len(set) == 0
@@ -178,7 +202,7 @@ func NewStringSet() StringSet {
178202

179203
// CreateStringSet - creates new string set with given string values.
180204
func CreateStringSet(sl ...string) StringSet {
181-
set := make(StringSet)
205+
set := make(StringSet, len(sl))
182206
for _, k := range sl {
183207
set.Add(k)
184208
}
@@ -187,7 +211,7 @@ func CreateStringSet(sl ...string) StringSet {
187211

188212
// CopyStringSet - returns copy of given set.
189213
func CopyStringSet(set StringSet) StringSet {
190-
nset := NewStringSet()
214+
nset := make(StringSet, len(set))
191215
for k, v := range set {
192216
nset[k] = v
193217
}

0 commit comments

Comments
 (0)