From af8e9c9a16c6d231c7b685e3041e826217da0af4 Mon Sep 17 00:00:00 2001 From: Florian Loch Date: Tue, 13 Feb 2024 15:35:45 +0100 Subject: [PATCH] feat: add a first test --- go.mod | 3 + go.sum | 8 ++ lib.go | 9 --- storage.go | 12 ++- sync.go | 8 +- sync_test.go | 223 +++++++++++++++++++++++++++++++++++++++++++++++++++ upstream.go | 2 +- 7 files changed, 249 insertions(+), 16 deletions(-) create mode 100644 sync_test.go diff --git a/go.mod b/go.mod index d6e41e9..aa0b1d2 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,15 @@ go 1.21.6 require ( github.com/alitto/pond v1.8.3 github.com/deckarep/golang-set/v2 v2.6.0 + github.com/h2non/gock v1.2.0 github.com/hashicorp/go-retryablehttp v0.7.5 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 github.com/schollz/progressbar/v3 v3.14.1 + go.uber.org/mock v0.4.0 ) require ( + github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect diff --git a/go.sum b/go.sum index 060228c..8af5605 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,10 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/h2non/gock v1.2.0 h1:K6ol8rfrRkUOefooBC8elXoaNGYkpp7y2qcxGG6BzUE= +github.com/h2non/gock v1.2.0/go.mod h1:tNhoxHYW2W42cYkYb1WqzdbYIieALC99kpYr7rH/BQk= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= @@ -17,6 +21,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -28,6 +34,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= diff --git a/lib.go b/lib.go index 63bebda..2990f11 100644 --- a/lib.go +++ b/lib.go @@ -14,7 +14,6 @@ import ( const ( defaultDataDir = "./.hibp-data" defaultEndpoint = "https://api.pwnedpasswords.com/range/" - defaultCheckETag = true defaultWorkers = 50 DefaultStateFile = "./.hibp-data/state" lastRange = 0xFFFFF @@ -25,7 +24,6 @@ type ProgressFunc func(lowest, current, to, processed, remaining int64) error type syncConfig struct { dataDir string endpoint string - checkETag bool minWorkers int progressFn ProgressFunc stateFile io.ReadWriteSeeker @@ -45,12 +43,6 @@ func WithEndpoint(endpoint string) SyncOption { } } -func WithCheckETag(checkETag bool) SyncOption { - return func(c *syncConfig) { - c.checkETag = checkETag - } -} - func WithMinWorkers(workers int) SyncOption { return func(c *syncConfig) { c.minWorkers = workers @@ -73,7 +65,6 @@ func Sync(options ...SyncOption) error { config := &syncConfig{ dataDir: defaultDataDir, endpoint: defaultEndpoint, - checkETag: defaultCheckETag, minWorkers: defaultWorkers, progressFn: func(_, _, _, _, _ int64) error { return nil }, } diff --git a/storage.go b/storage.go index 9f6355d..39e10d9 100644 --- a/storage.go +++ b/storage.go @@ -11,15 +11,22 @@ import ( ) const ( - fileMode = 0666 // TODO ??? - dirMode = 0744 // TODO ??? + dirMode = 0o755 // TODO ??? ) +type storage interface { + Save(key, etag string, data []byte) error + LoadETag(key string) (string, error) + LoadData(key string) (io.ReadCloser, error) +} + type fsStorage struct { dataDir string writeLock sync.Mutex } +var _ storage = (*fsStorage)(nil) + func (f *fsStorage) Save(key, etag string, data []byte) error { // We need to synchronize calls to Save because we don't want to create the same parent directory for several files // at the same time. @@ -54,6 +61,7 @@ func (f *fsStorage) LoadETag(key string) (string, error) { if errors.Is(err, os.ErrNotExist) { return "", nil } + return "", fmt.Errorf("opening file %q: %w", f.filePath(key), err) } defer file.Close() diff --git a/sync.go b/sync.go index 0f6e58a..7feff7e 100644 --- a/sync.go +++ b/sync.go @@ -10,7 +10,7 @@ import ( "sync/atomic" ) -func _sync(from, to int64, client *hibpClient, storage *fsStorage, pool *pond.WorkerPool, onProgress ProgressFunc) error { +func _sync(from, to int64, client *hibpClient, store storage, pool *pond.WorkerPool, onProgress ProgressFunc) error { var ( mErr error errLock sync.Mutex @@ -30,15 +30,15 @@ func _sync(from, to int64, client *hibpClient, storage *fsStorage, pool *pond.Wo err := func() error { inFlightSet.Add(current) - etag, _ := storage.LoadETag(rangePrefix) + etag, _ := store.LoadETag(rangePrefix) resp, err := client.RequestRange(rangePrefix, etag) if err != nil { - return fmt.Errorf("requesting range: %w", err) + return err } if !resp.NotModified { - if err := storage.Save(rangePrefix, resp.ETag, resp.Data); err != nil { + if err := store.Save(rangePrefix, resp.ETag, resp.Data); err != nil { return fmt.Errorf("saving range: %w", err) } } diff --git a/sync_test.go b/sync_test.go new file mode 100644 index 0000000..629cb1c --- /dev/null +++ b/sync_test.go @@ -0,0 +1,223 @@ +package hibpsync + +import ( + "github.com/alitto/pond" + "github.com/h2non/gock" + "io" + "net/http" + "reflect" + "sync/atomic" + "testing" + + "go.uber.org/mock/gomock" +) + +const baseURL = "https://api.pwnedpasswords.com" + +func TestSync(t *testing.T) { + httpClient := &http.Client{} + gock.InterceptClient(httpClient) + + gock.New(baseURL). + Get("/range/00000"). + Reply(200). + AddHeader("ETag", "etag"). + BodyString("data1") + gock.New(baseURL). + Get("/range/00001"). + MatchHeader("If-None-Match", "etag received earlier"). + Reply(http.StatusNotModified). + AddHeader("ETag", "etag received earlier"). + BodyString("data2") + gock.New(baseURL). + Get("/range/00002"). + Reply(200). + AddHeader("ETag", "etag"). + BodyString("data3") + gock.New(baseURL). + Get("/range/00003"). + Reply(200). + AddHeader("ETag", "etag"). + BodyString("data4") + gock.New(baseURL). + Get("/range/00004"). + Reply(200). + AddHeader("ETag", "etag"). + BodyString("data5") + gock.New(baseURL). + Get("/range/00005"). + Reply(200). + AddHeader("ETag", "etag"). + BodyString("data6") + gock.New(baseURL). + Get("/range/00006"). + Reply(200). + AddHeader("ETag", "etag"). + BodyString("data7") + gock.New(baseURL). + Get("/range/00007"). + Reply(200). + AddHeader("ETag", "etag"). + BodyString("data8") + gock.New(baseURL). + Get("/range/00008"). + Reply(200). + AddHeader("ETag", "etag"). + BodyString("data9") + gock.New(baseURL). + Get("/range/00009"). + Reply(200). + AddHeader("ETag", "etag"). + BodyString("data10") + gock.New(baseURL). + Get("/range/0000A"). + Reply(200). + AddHeader("ETag", "etag"). + BodyString("data11") + gock.New(baseURL). + Get("/range/0000B"). + Reply(200). + AddHeader("ETag", "etag"). + BodyString("data12") + + client := &hibpClient{ + endpoint: defaultEndpoint, + httpClient: httpClient, + } + + ctrl := gomock.NewController(t) + storageMock := NewMockstorage(ctrl) + + storageMock.EXPECT().LoadETag("00000").Return("", nil) + storageMock.EXPECT().Save("00000", "etag", []byte("data1")).Return(nil) + storageMock.EXPECT().LoadETag("00001").Return("etag received earlier", nil) + // 00001 does not need to be written as its ETag has not changed + storageMock.EXPECT().LoadETag("00002").Return("", nil) + storageMock.EXPECT().Save("00002", "etag", []byte("data3")).Return(nil) + storageMock.EXPECT().LoadETag("00003").Return("", nil) + storageMock.EXPECT().Save("00003", "etag", []byte("data4")).Return(nil) + storageMock.EXPECT().LoadETag("00004").Return("", nil) + storageMock.EXPECT().Save("00004", "etag", []byte("data5")).Return(nil) + storageMock.EXPECT().LoadETag("00005").Return("", nil) + storageMock.EXPECT().Save("00005", "etag", []byte("data6")).Return(nil) + storageMock.EXPECT().LoadETag("00006").Return("", nil) + storageMock.EXPECT().Save("00006", "etag", []byte("data7")).Return(nil) + storageMock.EXPECT().LoadETag("00007").Return("", nil) + storageMock.EXPECT().Save("00007", "etag", []byte("data8")).Return(nil) + storageMock.EXPECT().LoadETag("00008").Return("", nil) + storageMock.EXPECT().Save("00008", "etag", []byte("data9")).Return(nil) + storageMock.EXPECT().LoadETag("00009").Return("", nil) + storageMock.EXPECT().Save("00009", "etag", []byte("data10")).Return(nil) + storageMock.EXPECT().LoadETag("0000A").Return("", nil) + storageMock.EXPECT().Save("0000A", "etag", []byte("data11")).Return(nil) + storageMock.EXPECT().LoadETag("0000B").Return("", nil) + storageMock.EXPECT().Save("0000B", "etag", []byte("data12")).Return(nil) + + var callCounter atomic.Int64 + + progressFn := func(_, _, _, processed, _ int64) error { + callCounter.Add(1) + + if processed != 10 && processed != 12 { + t.Fatalf("progressFn called at unexpected point in time: %d", processed) + } + + return nil + } + + // Create the pool with some arbitrary configuration + pool := pond.New(3, 3) + + if err := _sync(0, 12, client, storageMock, pool, progressFn); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if callCounter.Load() != 2 { + t.Fatalf("unexpected number of calls to progressFn: %d", callCounter.Load()) + } + + if !ctrl.Satisfied() { + t.Fatalf("there are unsatisfied expectations") + } + + if gock.IsPending() { + t.Fatalf("there are pending mocks") + } +} + +// TODO: We will need further testcases ensuring the library works fine even in error conditions + +// Code generated by MockGen. DO NOT EDIT. +// Source: storage.go +// +// Generated by this command: +// +// mockgen -source storage.go +// + +// Mockstorage is a mock of storage interface. +type Mockstorage struct { + ctrl *gomock.Controller + recorder *MockstorageMockRecorder +} + +// MockstorageMockRecorder is the mock recorder for Mockstorage. +type MockstorageMockRecorder struct { + mock *Mockstorage +} + +// NewMockstorage creates a new mock instance. +func NewMockstorage(ctrl *gomock.Controller) *Mockstorage { + mock := &Mockstorage{ctrl: ctrl} + mock.recorder = &MockstorageMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *Mockstorage) EXPECT() *MockstorageMockRecorder { + return m.recorder +} + +// LoadData mocks base method. +func (m *Mockstorage) LoadData(key string) (io.ReadCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadData", key) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LoadData indicates an expected call of LoadData. +func (mr *MockstorageMockRecorder) LoadData(key any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadData", reflect.TypeOf((*Mockstorage)(nil).LoadData), key) +} + +// LoadETag mocks base method. +func (m *Mockstorage) LoadETag(key string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadETag", key) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LoadETag indicates an expected call of LoadETag. +func (mr *MockstorageMockRecorder) LoadETag(key any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadETag", reflect.TypeOf((*Mockstorage)(nil).LoadETag), key) +} + +// Save mocks base method. +func (m *Mockstorage) Save(key, etag string, data []byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Save", key, etag, data) + ret0, _ := ret[0].(error) + return ret0 +} + +// Save indicates an expected call of Save. +func (mr *MockstorageMockRecorder) Save(key, etag, data any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*Mockstorage)(nil).Save), key, etag, data) +} diff --git a/upstream.go b/upstream.go index c1506cc..3315373 100644 --- a/upstream.go +++ b/upstream.go @@ -42,7 +42,7 @@ func (h *hibpClient) RequestRange(rangePrefix, etag string) (*hibpResponse, erro mErr = errors.Join(mErr, err) } - return nil, fmt.Errorf("requesting range %d: %w", rangePrefix, mErr) + return nil, fmt.Errorf("requesting range %q: %w", rangePrefix, mErr) } func (h *hibpClient) request(req *http.Request) (*hibpResponse, error) {