Setup sticky session for Kerberos and NTML HTTP Authentication

When server responds with `WWW-Authenticate: Negotiate`, save VCAP_ID cookie on response to client so that subsequent request with `Authorization: Negotiate ...` will be directed to the same application instance. See [RFC-4559](https://www.ietf.org/rfc/rfc4559.txt) Signed-off-by: Josh Russett <[email protected]>
cloudfoundry · Jan 31, 2024 · 3de570d · 3de570d
1 parent 1b1e6ac
commit 3de570d
Show file tree

Hide file tree

Showing 12 changed files with 294 additions and 151 deletions.
diff --git a/handlers/helpers.go b/handlers/helpers.go
@@ -67,17 +67,24 @@ func EndpointIteratorForRequest(request *http.Request, loadBalanceMethod string,
 	if err != nil {
 		return nil, fmt.Errorf("could not find reqInfo in context")
 	}
-	return reqInfo.RoutePool.Endpoints(loadBalanceMethod, getStickySession(request, stickySessionCookieNames), azPreference, az), nil
+	stickyEndpointID, mustBeSticky := GetStickySession(request, stickySessionCookieNames)
+	return reqInfo.RoutePool.Endpoints(loadBalanceMethod, stickyEndpointID, mustBeSticky, azPreference, az), nil
 }
 
-func getStickySession(request *http.Request, stickySessionCookieNames config.StringSet) string {
+func GetStickySession(request *http.Request, stickySessionCookieNames config.StringSet) (string, bool) {
+	containsAuthNegotiateHeader := strings.HasPrefix(strings.ToLower(request.Header.Get("Authorization")), "negotiate")
+	if containsAuthNegotiateHeader {
+		if sticky, err := request.Cookie(VcapCookieId); err == nil {
+			return sticky.Value, true
+		}
+	}
 	// Try choosing a backend using sticky session
 	for stickyCookieName, _ := range stickySessionCookieNames {
 		if _, err := request.Cookie(stickyCookieName); err == nil {
 			if sticky, err := request.Cookie(VcapCookieId); err == nil {
-				return sticky.Value
+				return sticky.Value, false
 			}
 		}
 	}
-	return ""
+	return "", false
 }
diff --git a/proxy/round_tripper/proxy_round_tripper.go b/proxy/round_tripper/proxy_round_tripper.go
@@ -135,9 +135,9 @@ func (rt *roundTripper) RoundTrip(originalRequest *http.Request) (*http.Response
 		return nil, errors.New("ProxyResponseWriter not set on context")
 	}
 
-	stickyEndpointID := getStickySession(request, rt.config.StickySessionCookieNames)
+	stickyEndpointID, mustBeSticky := handlers.GetStickySession(request, rt.config.StickySessionCookieNames)
 	numberOfEndpoints := reqInfo.RoutePool.NumEndpoints()
-	iter := reqInfo.RoutePool.Endpoints(rt.config.LoadBalance, stickyEndpointID, rt.config.LoadBalanceAZPreference, rt.config.Zone)
+	iter := reqInfo.RoutePool.Endpoints(rt.config.LoadBalance, stickyEndpointID, mustBeSticky, rt.config.LoadBalanceAZPreference, rt.config.Zone)
 
 	// The selectEndpointErr needs to be tracked separately. If we get an error
 	// while selecting an endpoint we might just have run out of routes. In
@@ -389,9 +389,8 @@ func setupStickySession(
 
 	requestContainsStickySessionCookies := originalEndpointId != ""
 	requestNotSentToRequestedApp := originalEndpointId != endpoint.PrivateInstanceId
-	containsAuthNegotiateHeader := strings.ToLower(response.Header.Get("Authorization")) == "negotiate"
-
-	shouldSetVCAPID := containsAuthNegotiateHeader || (requestContainsStickySessionCookies && requestNotSentToRequestedApp)
+	containsAuthNegotiateHeader := strings.HasPrefix(strings.ToLower(response.Header.Get("WWW-Authenticate")), "negotiate")
+	shouldSetVCAPID := (containsAuthNegotiateHeader || requestContainsStickySessionCookies) && requestNotSentToRequestedApp
 
 	secure := false
 	maxAge := 0
@@ -443,18 +442,6 @@ func setupStickySession(
 	}
 }
 
-func getStickySession(request *http.Request, stickySessionCookieNames config.StringSet) string {
-	// Try choosing a backend using sticky session
-	for stickyCookieName, _ := range stickySessionCookieNames {
-		if _, err := request.Cookie(stickyCookieName); err == nil {
-			if sticky, err := request.Cookie(VcapCookieId); err == nil {
-				return sticky.Value
-			}
-		}
-	}
-	return ""
-}
-
 func requestSentToRouteService(request *http.Request) bool {
 	sigHeader := request.Header.Get(routeservice.HeaderKeySignature)
 	rsUrl := request.Header.Get(routeservice.HeaderKeyForwardedURL)

diff --git a/proxy/round_tripper/proxy_round_tripper_test.go b/proxy/round_tripper/proxy_round_tripper_test.go
@@ -269,7 +269,7 @@ var _ = Describe("ProxyRoundTripper", func() {
 					res, err := proxyRoundTripper.RoundTrip(req)
 					Expect(err).NotTo(HaveOccurred())
 
-					iter := routePool.Endpoints("", "", AZPreference, AZ)
+					iter := routePool.Endpoints("", "", false, AZPreference, AZ)
 					ep1 := iter.Next(0)
 					ep2 := iter.Next(1)
 					Expect(ep1.PrivateInstanceId).To(Equal(ep2.PrivateInstanceId))
@@ -427,7 +427,7 @@ var _ = Describe("ProxyRoundTripper", func() {
 					_, err := proxyRoundTripper.RoundTrip(req)
 					Expect(err).To(MatchError(ContainSubstring("tls: handshake failure")))
 
-					iter := routePool.Endpoints("", "", AZPreference, AZ)
+					iter := routePool.Endpoints("", "", false, AZPreference, AZ)
 					ep1 := iter.Next(0)
 					ep2 := iter.Next(1)
 					Expect(ep1).To(Equal(ep2))
@@ -1035,7 +1035,7 @@ var _ = Describe("ProxyRoundTripper", func() {
 				}
 
 				setAuthorizationNegotiateHeader := func(resp *http.Response) (response *http.Response) {
-					resp.Header.Add("Authorization", "Negotiate")
+					resp.Header.Add("WWW-Authenticate", "Negotiate SOME-TOKEN")
 					return resp
 				}
 
@@ -1161,7 +1161,7 @@ var _ = Describe("ProxyRoundTripper", func() {
 						})
 					})
 
-					Context("when there is an 'Authorization: Negotiate' header set on the response", func() {
+					Context("when there is an 'WWW-Authenticate: Negotiate ...' header set on the response", func() {
 						BeforeEach(func() {
 							transport.RoundTripStub = func(req *http.Request) (*http.Response, error) {
 								resp := &http.Response{StatusCode: http.StatusTeapot, Header: make(map[string][]string)}
@@ -1486,7 +1486,29 @@ var _ = Describe("ProxyRoundTripper", func() {
 					})
 				})
 
+				Context("when VCAP_ID cookie and 'Authorization: Negotiate ...' header are on the request", func() {
+					BeforeEach(func() {
+						req.AddCookie(&http.Cookie{
+							Name:  round_tripper.VcapCookieId,
+							Value: "id-2",
+						})
+						req.Header.Add("Authorization", "Negotiate SOME-TOKEN")
+						transport.RoundTripStub = func(req *http.Request) (*http.Response, error) {
+							Expect(req.URL.Host).To(Equal("1.1.1.1:9092"))
+							resp := &http.Response{StatusCode: http.StatusTeapot, Header: make(map[string][]string)}
+							return resp, nil
+						}
+					})
+
+					It("will select the previous backend and VCAP_ID is set on the response", func() {
+						Consistently(func() error {
+							_, err := proxyRoundTripper.RoundTrip(req)
+							return err
+						}).ShouldNot(HaveOccurred())
+					})
+				})
 			})
+
 			Context("when endpoint timeout is not 0", func() {
 				var reqCh chan *http.Request
 				BeforeEach(func() {

diff --git a/proxy/session_affinity_test.go b/proxy/session_affinity_test.go
@@ -13,7 +13,7 @@ import (
 
 const StickyCookieKey = "JSESSIONID"
 
-var _ = Describe("Session Affinity", func() {
+var _ = Describe("Session Affinity with JSESSIONID", func() {
 	var done chan bool
 	var jSessionIdCookie *http.Cookie
 

diff --git a/registry/registry_test.go b/registry/registry_test.go
@@ -374,7 +374,7 @@ var _ = Describe("RouteRegistry", func() {
 					Expect(r.NumEndpoints()).To(Equal(1))
 
 					p := r.Lookup("foo.com")
-					Expect(p.Endpoints("", "", azPreference, az).Next(0).ModificationTag).To(Equal(modTag))
+					Expect(p.Endpoints("", "", false, azPreference, az).Next(0).ModificationTag).To(Equal(modTag))
 				})
 			})
 
@@ -396,7 +396,7 @@ var _ = Describe("RouteRegistry", func() {
 						Expect(r.NumEndpoints()).To(Equal(1))
 
 						p := r.Lookup("foo.com")
-						Expect(p.Endpoints("", "", azPreference, az).Next(0).ModificationTag).To(Equal(modTag))
+						Expect(p.Endpoints("", "", false, azPreference, az).Next(0).ModificationTag).To(Equal(modTag))
 					})
 
 					Context("updating an existing route with an older modification tag", func() {
@@ -416,7 +416,7 @@ var _ = Describe("RouteRegistry", func() {
 							Expect(r.NumEndpoints()).To(Equal(1))
 
 							p := r.Lookup("foo.com")
-							ep := p.Endpoints("", "", azPreference, az).Next(0)
+							ep := p.Endpoints("", "", false, azPreference, az).Next(0)
 							Expect(ep.ModificationTag).To(Equal(modTag))
 							Expect(ep).To(Equal(endpoint2))
 						})
@@ -435,7 +435,7 @@ var _ = Describe("RouteRegistry", func() {
 						Expect(r.NumEndpoints()).To(Equal(1))
 
 						p := r.Lookup("foo.com")
-						Expect(p.Endpoints("", "", azPreference, az).Next(0).ModificationTag).To(Equal(modTag))
+						Expect(p.Endpoints("", "", false, azPreference, az).Next(0).ModificationTag).To(Equal(modTag))
 					})
 				})
 			})
@@ -703,7 +703,7 @@ var _ = Describe("RouteRegistry", func() {
 			Expect(r.NumUris()).To(Equal(1))
 
 			p1 := r.Lookup("foo/bar")
-			iter := p1.Endpoints("", "", azPreference, az)
+			iter := p1.Endpoints("", "", false, azPreference, az)
 			Expect(iter.Next(0).CanonicalAddr()).To(Equal("192.168.1.1:1234"))
 
 			p2 := r.Lookup("foo")
@@ -799,7 +799,7 @@ var _ = Describe("RouteRegistry", func() {
 			p2 := r.Lookup("FOO")
 			Expect(p1).To(Equal(p2))
 
-			iter := p1.Endpoints("", "", azPreference, az)
+			iter := p1.Endpoints("", "", false, azPreference, az)
 			Expect(iter.Next(0).CanonicalAddr()).To(Equal("192.168.1.1:1234"))
 		})
 
@@ -818,7 +818,7 @@ var _ = Describe("RouteRegistry", func() {
 
 			p := r.Lookup("bar")
 			Expect(p).ToNot(BeNil())
-			e := p.Endpoints("", "", azPreference, az).Next(0)
+			e := p.Endpoints("", "", false, azPreference, az).Next(0)
 			Expect(e).ToNot(BeNil())
 			Expect(e.CanonicalAddr()).To(MatchRegexp("192.168.1.1:123[4|5]"))
 
@@ -833,13 +833,13 @@ var _ = Describe("RouteRegistry", func() {
 
 			p := r.Lookup("foo.wild.card")
 			Expect(p).ToNot(BeNil())
-			e := p.Endpoints("", "", azPreference, az).Next(0)
+			e := p.Endpoints("", "", false, azPreference, az).Next(0)
 			Expect(e).ToNot(BeNil())
 			Expect(e.CanonicalAddr()).To(Equal("192.168.1.2:1234"))
 
 			p = r.Lookup("foo.space.wild.card")
 			Expect(p).ToNot(BeNil())
-			e = p.Endpoints("", "", azPreference, az).Next(0)
+			e = p.Endpoints("", "", false, azPreference, az).Next(0)
 			Expect(e).ToNot(BeNil())
 			Expect(e.CanonicalAddr()).To(Equal("192.168.1.2:1234"))
 		})
@@ -853,7 +853,7 @@ var _ = Describe("RouteRegistry", func() {
 
 			p := r.Lookup("not.wild.card")
 			Expect(p).ToNot(BeNil())
-			e := p.Endpoints("", "", azPreference, az).Next(0)
+			e := p.Endpoints("", "", false, azPreference, az).Next(0)
 			Expect(e).ToNot(BeNil())
 			Expect(e.CanonicalAddr()).To(Equal("192.168.1.1:1234"))
 		})
@@ -885,7 +885,7 @@ var _ = Describe("RouteRegistry", func() {
 				p := r.Lookup("dora.app.com/env?foo=bar")
 
 				Expect(p).ToNot(BeNil())
-				iter := p.Endpoints("", "", azPreference, az)
+				iter := p.Endpoints("", "", false, azPreference, az)
 				Expect(iter.Next(0).CanonicalAddr()).To(Equal("192.168.1.1:1234"))
 			})
 
@@ -894,7 +894,7 @@ var _ = Describe("RouteRegistry", func() {
 				p := r.Lookup("dora.app.com/env/abc?foo=bar&baz=bing")
 
 				Expect(p).ToNot(BeNil())
-				iter := p.Endpoints("", "", azPreference, az)
+				iter := p.Endpoints("", "", false, azPreference, az)
 				Expect(iter.Next(0).CanonicalAddr()).To(Equal("192.168.1.1:1234"))
 			})
 		})
@@ -914,7 +914,7 @@ var _ = Describe("RouteRegistry", func() {
 			p1 := r.Lookup("foo/extra/paths")
 			Expect(p1).ToNot(BeNil())
 
-			iter := p1.Endpoints("", "", azPreference, az)
+			iter := p1.Endpoints("", "", false, azPreference, az)
 			Expect(iter.Next(0).CanonicalAddr()).To(Equal("192.168.1.1:1234"))
 		})
 
@@ -926,7 +926,7 @@ var _ = Describe("RouteRegistry", func() {
 			p1 := r.Lookup("foo?fields=foo,bar")
 			Expect(p1).ToNot(BeNil())
 
-			iter := p1.Endpoints("", "", azPreference, az)
+			iter := p1.Endpoints("", "", false, azPreference, az)
 			Expect(iter.Next(0).CanonicalAddr()).To(Equal("192.168.1.1:1234"))
 		})
 
@@ -962,7 +962,7 @@ var _ = Describe("RouteRegistry", func() {
 			Expect(r.NumEndpoints()).To(Equal(2))
 
 			p := r.LookupWithInstance("bar.com/foo", appId, appIndex)
-			e := p.Endpoints("", "", azPreference, az).Next(0)
+			e := p.Endpoints("", "", false, azPreference, az).Next(0)
 
 			Expect(e).ToNot(BeNil())
 			Expect(e.CanonicalAddr()).To(MatchRegexp("192.168.1.1:1234"))
@@ -976,7 +976,7 @@ var _ = Describe("RouteRegistry", func() {
 			Expect(r.NumEndpoints()).To(Equal(2))
 
 			p := r.LookupWithInstance("bar.com/foo", appId, appIndex)
-			e := p.Endpoints("", "", azPreference, az).Next(0)
+			e := p.Endpoints("", "", false, azPreference, az).Next(0)
 
 			Expect(e).ToNot(BeNil())
 			Expect(e.CanonicalAddr()).To(MatchRegexp("192.168.1.1:1234"))
@@ -1169,7 +1169,7 @@ var _ = Describe("RouteRegistry", func() {
 
 			p := r.Lookup("foo")
 			Expect(p).ToNot(BeNil())
-			Expect(p.Endpoints("", "", azPreference, az).Next(0)).To(Equal(endpoint))
+			Expect(p.Endpoints("", "", false, azPreference, az).Next(0)).To(Equal(endpoint))
 
 			p = r.Lookup("bar")
 			Expect(p).To(BeNil())

diff --git a/route/endpoint_iterator_benchmark_test.go b/route/endpoint_iterator_benchmark_test.go
@@ -69,13 +69,13 @@ func setupEndpointIterator(total int, azDistribution int, strategy string) route
 	var lb route.EndpointIterator
 	switch strategy {
 	case "round-robin":
-		lb = route.NewRoundRobin(pool, "", false, localAZ)
+		lb = route.NewRoundRobin(pool, "", false, false, localAZ)
 	case "round-robin-locally-optimistic":
-		lb = route.NewRoundRobin(pool, "", true, localAZ)
+		lb = route.NewRoundRobin(pool, "", false, true, localAZ)
 	case "least-connection":
-		lb = route.NewLeastConnection(pool, "", false, localAZ)
+		lb = route.NewLeastConnection(pool, "", false, false, localAZ)
 	case "least-connection-locally-optimistic":
-		lb = route.NewLeastConnection(pool, "", true, localAZ)
+		lb = route.NewLeastConnection(pool, "", false, true, localAZ)
 	default:
 		panic("invalid load balancing strategy")
 	}

diff --git a/route/leastconnection.go b/route/leastconnection.go
@@ -8,16 +8,18 @@ import (
 type LeastConnection struct {
 	pool                  *EndpointPool
 	initialEndpoint       string
+	mustBeSticky          bool
 	lastEndpoint          *Endpoint
 	randomize             *rand.Rand
 	locallyOptimistic     bool
 	localAvailabilityZone string
 }
 
-func NewLeastConnection(p *EndpointPool, initial string, locallyOptimistic bool, localAvailabilityZone string) EndpointIterator {
+func NewLeastConnection(p *EndpointPool, initial string, mustBeSticky bool, locallyOptimistic bool, localAvailabilityZone string) EndpointIterator {
 	return &LeastConnection{
 		pool:                  p,
 		initialEndpoint:       initial,
+		mustBeSticky:          mustBeSticky,
 		randomize:             rand.New(rand.NewSource(time.Now().UnixNano())),
 		locallyOptimistic:     locallyOptimistic,
 		localAvailabilityZone: localAvailabilityZone,
@@ -28,11 +30,17 @@ func (r *LeastConnection) Next(attempt int) *Endpoint {
 	var e *endpointElem
 	if r.initialEndpoint != "" {
 		e = r.pool.findById(r.initialEndpoint)
-		r.initialEndpoint = ""
-
 		if e != nil && e.isOverloaded() {
 			e = nil
 		}
+
+		if e == nil && r.mustBeSticky {
+			return nil
+		}
+
+		if !r.mustBeSticky {
+			r.initialEndpoint = ""
+		}
 	}
 
 	if e != nil {