Skip to content

Commit

Permalink
feat: add a first test
Browse files Browse the repository at this point in the history
  • Loading branch information
FlorianLoch committed Feb 13, 2024
1 parent c1e8f0d commit af8e9c9
Show file tree
Hide file tree
Showing 7 changed files with 249 additions and 16 deletions.
3 changes: 3 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,15 @@ go 1.21.6
require (
github.com/alitto/pond v1.8.3
github.com/deckarep/golang-set/v2 v2.6.0
github.com/h2non/gock v1.2.0
github.com/hashicorp/go-retryablehttp v0.7.5
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213
github.com/schollz/progressbar/v3 v3.14.1
go.uber.org/mock v0.4.0
)

require (
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
Expand Down
8 changes: 8 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM=
github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
github.com/h2non/gock v1.2.0 h1:K6ol8rfrRkUOefooBC8elXoaNGYkpp7y2qcxGG6BzUE=
github.com/h2non/gock v1.2.0/go.mod h1:tNhoxHYW2W42cYkYb1WqzdbYIieALC99kpYr7rH/BQk=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
Expand All @@ -17,6 +21,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
Expand All @@ -28,6 +34,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
Expand Down
9 changes: 0 additions & 9 deletions lib.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
const (
defaultDataDir = "./.hibp-data"
defaultEndpoint = "https://api.pwnedpasswords.com/range/"
defaultCheckETag = true
defaultWorkers = 50
DefaultStateFile = "./.hibp-data/state"
lastRange = 0xFFFFF
Expand All @@ -25,7 +24,6 @@ type ProgressFunc func(lowest, current, to, processed, remaining int64) error
type syncConfig struct {
dataDir string
endpoint string
checkETag bool
minWorkers int
progressFn ProgressFunc
stateFile io.ReadWriteSeeker
Expand All @@ -45,12 +43,6 @@ func WithEndpoint(endpoint string) SyncOption {
}
}

func WithCheckETag(checkETag bool) SyncOption {
return func(c *syncConfig) {
c.checkETag = checkETag
}
}

func WithMinWorkers(workers int) SyncOption {
return func(c *syncConfig) {
c.minWorkers = workers
Expand All @@ -73,7 +65,6 @@ func Sync(options ...SyncOption) error {
config := &syncConfig{
dataDir: defaultDataDir,
endpoint: defaultEndpoint,
checkETag: defaultCheckETag,
minWorkers: defaultWorkers,
progressFn: func(_, _, _, _, _ int64) error { return nil },
}
Expand Down
12 changes: 10 additions & 2 deletions storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,22 @@ import (
)

const (
fileMode = 0666 // TODO ???
dirMode = 0744 // TODO ???
dirMode = 0o755 // TODO ???
)

type storage interface {
Save(key, etag string, data []byte) error
LoadETag(key string) (string, error)
LoadData(key string) (io.ReadCloser, error)
}

type fsStorage struct {
dataDir string
writeLock sync.Mutex
}

var _ storage = (*fsStorage)(nil)

func (f *fsStorage) Save(key, etag string, data []byte) error {
// We need to synchronize calls to Save because we don't want to create the same parent directory for several files
// at the same time.
Expand Down Expand Up @@ -54,6 +61,7 @@ func (f *fsStorage) LoadETag(key string) (string, error) {
if errors.Is(err, os.ErrNotExist) {
return "", nil
}

return "", fmt.Errorf("opening file %q: %w", f.filePath(key), err)
}
defer file.Close()
Expand Down
8 changes: 4 additions & 4 deletions sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"sync/atomic"
)

func _sync(from, to int64, client *hibpClient, storage *fsStorage, pool *pond.WorkerPool, onProgress ProgressFunc) error {
func _sync(from, to int64, client *hibpClient, store storage, pool *pond.WorkerPool, onProgress ProgressFunc) error {
var (
mErr error
errLock sync.Mutex
Expand All @@ -30,15 +30,15 @@ func _sync(from, to int64, client *hibpClient, storage *fsStorage, pool *pond.Wo
err := func() error {
inFlightSet.Add(current)

etag, _ := storage.LoadETag(rangePrefix)
etag, _ := store.LoadETag(rangePrefix)

resp, err := client.RequestRange(rangePrefix, etag)
if err != nil {
return fmt.Errorf("requesting range: %w", err)
return err
}

if !resp.NotModified {
if err := storage.Save(rangePrefix, resp.ETag, resp.Data); err != nil {
if err := store.Save(rangePrefix, resp.ETag, resp.Data); err != nil {
return fmt.Errorf("saving range: %w", err)
}
}
Expand Down
223 changes: 223 additions & 0 deletions sync_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,223 @@
package hibpsync

import (
"github.com/alitto/pond"
"github.com/h2non/gock"
"io"
"net/http"
"reflect"
"sync/atomic"
"testing"

"go.uber.org/mock/gomock"
)

const baseURL = "https://api.pwnedpasswords.com"

func TestSync(t *testing.T) {
httpClient := &http.Client{}
gock.InterceptClient(httpClient)

gock.New(baseURL).
Get("/range/00000").
Reply(200).
AddHeader("ETag", "etag").
BodyString("data1")
gock.New(baseURL).
Get("/range/00001").
MatchHeader("If-None-Match", "etag received earlier").
Reply(http.StatusNotModified).
AddHeader("ETag", "etag received earlier").
BodyString("data2")
gock.New(baseURL).
Get("/range/00002").
Reply(200).
AddHeader("ETag", "etag").
BodyString("data3")
gock.New(baseURL).
Get("/range/00003").
Reply(200).
AddHeader("ETag", "etag").
BodyString("data4")
gock.New(baseURL).
Get("/range/00004").
Reply(200).
AddHeader("ETag", "etag").
BodyString("data5")
gock.New(baseURL).
Get("/range/00005").
Reply(200).
AddHeader("ETag", "etag").
BodyString("data6")
gock.New(baseURL).
Get("/range/00006").
Reply(200).
AddHeader("ETag", "etag").
BodyString("data7")
gock.New(baseURL).
Get("/range/00007").
Reply(200).
AddHeader("ETag", "etag").
BodyString("data8")
gock.New(baseURL).
Get("/range/00008").
Reply(200).
AddHeader("ETag", "etag").
BodyString("data9")
gock.New(baseURL).
Get("/range/00009").
Reply(200).
AddHeader("ETag", "etag").
BodyString("data10")
gock.New(baseURL).
Get("/range/0000A").
Reply(200).
AddHeader("ETag", "etag").
BodyString("data11")
gock.New(baseURL).
Get("/range/0000B").
Reply(200).
AddHeader("ETag", "etag").
BodyString("data12")

client := &hibpClient{
endpoint: defaultEndpoint,
httpClient: httpClient,
}

ctrl := gomock.NewController(t)
storageMock := NewMockstorage(ctrl)

storageMock.EXPECT().LoadETag("00000").Return("", nil)
storageMock.EXPECT().Save("00000", "etag", []byte("data1")).Return(nil)
storageMock.EXPECT().LoadETag("00001").Return("etag received earlier", nil)
// 00001 does not need to be written as its ETag has not changed
storageMock.EXPECT().LoadETag("00002").Return("", nil)
storageMock.EXPECT().Save("00002", "etag", []byte("data3")).Return(nil)
storageMock.EXPECT().LoadETag("00003").Return("", nil)
storageMock.EXPECT().Save("00003", "etag", []byte("data4")).Return(nil)
storageMock.EXPECT().LoadETag("00004").Return("", nil)
storageMock.EXPECT().Save("00004", "etag", []byte("data5")).Return(nil)
storageMock.EXPECT().LoadETag("00005").Return("", nil)
storageMock.EXPECT().Save("00005", "etag", []byte("data6")).Return(nil)
storageMock.EXPECT().LoadETag("00006").Return("", nil)
storageMock.EXPECT().Save("00006", "etag", []byte("data7")).Return(nil)
storageMock.EXPECT().LoadETag("00007").Return("", nil)
storageMock.EXPECT().Save("00007", "etag", []byte("data8")).Return(nil)
storageMock.EXPECT().LoadETag("00008").Return("", nil)
storageMock.EXPECT().Save("00008", "etag", []byte("data9")).Return(nil)
storageMock.EXPECT().LoadETag("00009").Return("", nil)
storageMock.EXPECT().Save("00009", "etag", []byte("data10")).Return(nil)
storageMock.EXPECT().LoadETag("0000A").Return("", nil)
storageMock.EXPECT().Save("0000A", "etag", []byte("data11")).Return(nil)
storageMock.EXPECT().LoadETag("0000B").Return("", nil)
storageMock.EXPECT().Save("0000B", "etag", []byte("data12")).Return(nil)

var callCounter atomic.Int64

progressFn := func(_, _, _, processed, _ int64) error {
callCounter.Add(1)

if processed != 10 && processed != 12 {
t.Fatalf("progressFn called at unexpected point in time: %d", processed)
}

return nil
}

// Create the pool with some arbitrary configuration
pool := pond.New(3, 3)

if err := _sync(0, 12, client, storageMock, pool, progressFn); err != nil {
t.Fatalf("unexpected error: %v", err)
}

if callCounter.Load() != 2 {
t.Fatalf("unexpected number of calls to progressFn: %d", callCounter.Load())
}

if !ctrl.Satisfied() {
t.Fatalf("there are unsatisfied expectations")
}

if gock.IsPending() {
t.Fatalf("there are pending mocks")
}
}

// TODO: We will need further testcases ensuring the library works fine even in error conditions

// Code generated by MockGen. DO NOT EDIT.
// Source: storage.go
//
// Generated by this command:
//
// mockgen -source storage.go
//

// Mockstorage is a mock of storage interface.
type Mockstorage struct {
ctrl *gomock.Controller
recorder *MockstorageMockRecorder
}

// MockstorageMockRecorder is the mock recorder for Mockstorage.
type MockstorageMockRecorder struct {
mock *Mockstorage
}

// NewMockstorage creates a new mock instance.
func NewMockstorage(ctrl *gomock.Controller) *Mockstorage {
mock := &Mockstorage{ctrl: ctrl}
mock.recorder = &MockstorageMockRecorder{mock}
return mock
}

// EXPECT returns an object that allows the caller to indicate expected use.
func (m *Mockstorage) EXPECT() *MockstorageMockRecorder {
return m.recorder
}

// LoadData mocks base method.
func (m *Mockstorage) LoadData(key string) (io.ReadCloser, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "LoadData", key)
ret0, _ := ret[0].(io.ReadCloser)
ret1, _ := ret[1].(error)
return ret0, ret1
}

// LoadData indicates an expected call of LoadData.
func (mr *MockstorageMockRecorder) LoadData(key any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadData", reflect.TypeOf((*Mockstorage)(nil).LoadData), key)
}

// LoadETag mocks base method.
func (m *Mockstorage) LoadETag(key string) (string, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "LoadETag", key)
ret0, _ := ret[0].(string)
ret1, _ := ret[1].(error)
return ret0, ret1
}

// LoadETag indicates an expected call of LoadETag.
func (mr *MockstorageMockRecorder) LoadETag(key any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadETag", reflect.TypeOf((*Mockstorage)(nil).LoadETag), key)
}

// Save mocks base method.
func (m *Mockstorage) Save(key, etag string, data []byte) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Save", key, etag, data)
ret0, _ := ret[0].(error)
return ret0
}

// Save indicates an expected call of Save.
func (mr *MockstorageMockRecorder) Save(key, etag, data any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*Mockstorage)(nil).Save), key, etag, data)
}
2 changes: 1 addition & 1 deletion upstream.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func (h *hibpClient) RequestRange(rangePrefix, etag string) (*hibpResponse, erro
mErr = errors.Join(mErr, err)
}

return nil, fmt.Errorf("requesting range %d: %w", rangePrefix, mErr)
return nil, fmt.Errorf("requesting range %q: %w", rangePrefix, mErr)
}

func (h *hibpClient) request(req *http.Request) (*hibpResponse, error) {
Expand Down

0 comments on commit af8e9c9

Please sign in to comment.