Switch to HTTP retry library (#26)

Co-authored-by: Victor Elias <[email protected]>

clients/callback_client.go

@@ -7,41 +7,40 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/livepeer/catalyst-api/config"
 )
 
-const max_retries = 3
-const backoff_millis = 200
-
 type CallbackClient struct {
-	httpClient *http.Client
+	httpClient *retryablehttp.Client
 }
 
 func NewCallbackClient() CallbackClient {
+	client := retryablehttp.NewClient()
+	client.RetryMax = 2                          // Retry a maximum of this+1 times
+	client.RetryWaitMin = 200 * time.Millisecond // Wait at least this long between retries
+	client.RetryWaitMax = 1 * time.Second        // Wait at most this long between retries (exponential backoff)
+	client.HTTPClient = &http.Client{
+		Timeout: 5 * time.Second, // Give up on requests that take more than this long
+	}
+
 	return CallbackClient{
-		httpClient: &http.Client{
-			Timeout: 5 * time.Second,
-		},
+		httpClient: client,
 	}
 }
 
-func (c CallbackClient) DoWithRetries(r *http.Request) error {
-	var resp *http.Response
-	var err error
-	for x := 0; x < max_retries; x++ {
-		resp, err = c.httpClient.Do(r)
-		if err == nil && resp.StatusCode == http.StatusOK {
-			return nil
-		}
-
-		// Back off to give us more chance of succeeding during a network blip
-		time.Sleep(backoff_millis * time.Millisecond)
+func (c CallbackClient) DoWithRetries(r *retryablehttp.Request) error {
+	resp, err := c.httpClient.Do(r)
+	if err != nil {
+		return fmt.Errorf("failed to send callback to %q. Error: %s", r.URL.String(), err)
 	}
+	defer resp.Body.Close()
 
-	if err != nil {
-		return fmt.Errorf("failed to send callback to %q. Error: %q", r.URL.String(), err)
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf("failed to send callback to %q. HTTP Code: %d", r.URL.String(), resp.StatusCode)
 	}
-	return fmt.Errorf("failed to send callback to %q. Response Status Code: %d", r.URL.String(), resp.StatusCode)
+
+	return nil
 }
 
 func (c CallbackClient) SendTranscodeStatus(url string, status TranscodeStatus, completionRatio float32) error {
@@ -56,7 +55,7 @@ func (c CallbackClient) SendTranscodeStatus(url string, status TranscodeStatus,
 		return err
 	}
 
-	r, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(j))
+	r, err := retryablehttp.NewRequest(http.MethodPost, url, bytes.NewReader(j))
 	if err != nil {
 		return err
 	}
@@ -76,7 +75,7 @@ func (c CallbackClient) SendTranscodeStatusError(callbackURL, errorMsg string) e
 		return err
 	}
 
-	r, err := http.NewRequest(http.MethodPost, callbackURL, bytes.NewReader(j))
+	r, err := retryablehttp.NewRequest(http.MethodPost, callbackURL, bytes.NewReader(j))
 	if err != nil {
 		return err
 	}

clients/callback_client_test.go

@@ -68,7 +68,7 @@ func TestItEventuallyStopsRetrying(t *testing.T) {
 	err := client.SendTranscodeStatus(svr.URL, TranscodeStatusCompleted, 1)
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "failed to send callback")
-	require.Contains(t, err.Error(), "Response Status Code: 500")
+	require.Contains(t, err.Error(), "giving up after 3 attempt(s)")
 	require.Equal(t, 3, tries, "Expected the client to retry on failed callbacks")
 }
 

go.mod

@@ -10,10 +10,13 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0
 )
 
+require github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-kit/log v0.2.0 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.1
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect

go.sum

@@ -60,6 +60,12 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ=
+github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=