Merge pull request #69 from thomasheartman/main

## Fix: Fix variant distribution

This PR does a few things, all related to using a new seed for ensuring a fair distribution for variants.

- Update the client spec tests. The commit has been changed to one that has backported the changes related to variant hashing as described in the background section.
- Removed the override tests from the client spec tests we run. There is no logic to handle overrides in this SDKs, so those tests would always fail.
- Update the way we test variant tests for the client spec (we previously only checked the enabled state, which is not sufficient).
- Fix bugs in distribution algorithms for gradual rollout and variant selection (we were using `>` instead of `>=`).

## Background
After a customer reported that variant distribution seemed skewed we performed some testing and found that since we use the same hash string for both gradual rollout and variant allocation we'd reduced the set of groups we could get to whatever percentage our gradual rollout was set.

## Example
Take a gradualRollout of 10%, this will select normalized hashes between 1 and 10, when we then again hash the same string that gave us between 1 and 10, but with modulo 1000 for variants, this will only give us 100 possible groups, instead of the expected 1000.

## Fix
Force the normalization to accept a seed, and make sure to use a new seed when normalizing the variant distribution hash.

src/client.rs

@@ -591,12 +591,12 @@ where
         }
         let identifier = identifier.unwrap();
         let total_weight = feature.variants.iter().map(|v| v.value.weight as u32).sum();
-        strategy::normalised_hash(&group, identifier, total_weight)
+        strategy::normalised_variant_hash(&group, identifier, total_weight)
             .map(|selected_weight| {
                 let mut counter: u32 = 0;
                 for variant in feature.variants.iter().as_ref() {
                     counter += variant.value.weight as u32;
-                    if counter > selected_weight {
+                    if counter >= selected_weight {
                         variant.count.fetch_add(1, Ordering::Relaxed);
                         return variant.into();
                     }
@@ -1320,7 +1320,7 @@ mod tests {
             ],
             enabled: true,
         };
-        assert_eq!(variant2, c.get_variant(UserFeatures::two, &uid1));
+        assert_eq!(variant1, c.get_variant(UserFeatures::two, &uid1));
         assert_eq!(variant2, c.get_variant(UserFeatures::two, &session1));
         assert_eq!(variant1, c.get_variant(UserFeatures::two, &host1));
     }
@@ -1395,7 +1395,7 @@ mod tests {
             ],
             enabled: true,
         };
-        assert_eq!(variant2, c.get_variant_str("two", &uid1));
+        assert_eq!(variant1, c.get_variant_str("two", &uid1));
         assert_eq!(variant2, c.get_variant_str("two", &session1));
         assert_eq!(variant1, c.get_variant_str("two", &host1));
     }

src/strategy.rs

@@ -107,7 +107,7 @@ pub fn partial_rollout(group: &str, variable: Option<&String>, rollout: u32) ->
         100 => true,
         rollout => {
             if let Ok(normalised) = normalised_hash(group, variable, 100) {
-                rollout > normalised
+                rollout >= normalised
             } else {
                 false
             }
@@ -119,13 +119,36 @@ pub fn partial_rollout(group: &str, variable: Option<&String>, rollout: u32) ->
 /// required for extension strategies, but reusing this is probably a good idea
 /// for consistency across implementations.
 pub fn normalised_hash(group: &str, identifier: &str, modulus: u32) -> std::io::Result<u32> {
+    normalised_hash_internal(group, identifier, modulus, 0)
+}
+
+const VARIANT_NORMALIZATION_SEED: u32 = 86028157;
+
+/// Calculates a hash for **variant distribution** in the standard way
+/// expected for Unleash clients. This differs from the
+/// [`normalised_hash`] function in that it uses a different seed to
+///  ensure a fair distribution.
+pub fn normalised_variant_hash(
+    group: &str,
+    identifier: &str,
+    modulus: u32,
+) -> std::io::Result<u32> {
+    normalised_hash_internal(group, identifier, modulus, VARIANT_NORMALIZATION_SEED)
+}
+
+fn normalised_hash_internal(
+    group: &str,
+    identifier: &str,
+    modulus: u32,
+    seed: u32,
+) -> std::io::Result<u32> {
     // See https://github.com/stusmall/murmur3/pull/16 : .chain may avoid
     // copying in the general case, and may be faster (though perhaps
     // benchmarking would be useful - small datasizes here could make the best
     // path non-obvious) - but until murmur3 is fixed, we need to provide it
     // with a single string no matter what.
     let mut reader = Cursor::new(format!("{}:{}", &group, &identifier));
-    murmur3_32(&mut reader, 0).map(|hash_result| hash_result % modulus)
+    murmur3_32(&mut reader, seed).map(|hash_result| hash_result % modulus + 1)
 }
 
 // Build a closure to handle session id rollouts, parameterised by groupId and a
@@ -861,4 +884,22 @@ mod tests {
     fn normalised_hash() {
         assert!(50 > super::normalised_hash("AB12A", "122", 100).unwrap());
     }
+
+    #[test]
+    fn test_normalized_hash() {
+        assert_eq!(73, super::normalised_hash("gr1", "123", 100).unwrap());
+        assert_eq!(25, super::normalised_hash("groupX", "999", 100).unwrap());
+    }
+
+    #[test]
+    fn test_normalised_variant_hash() {
+        assert_eq!(
+            96,
+            super::normalised_variant_hash("gr1", "123", 100).unwrap()
+        );
+        assert_eq!(
+            60,
+            super::normalised_variant_hash("groupX", "999", 100).unwrap()
+        );
+    }
 }

tests/clientspec.rs

@@ -38,6 +38,21 @@ mod tests {
         enabled: bool,
     }
 
+    impl PartialEq<client::Variant> for VariantResult {
+        fn eq(&self, other: &client::Variant) -> bool {
+            let payload_matches = match &self._payload {
+                Some(payload) => match (other.payload.get("type"), other.payload.get("value")) {
+                    (Some(_type), Some(value)) => {
+                        &payload._type == _type && &payload._value == value
+                    }
+                    _ => false,
+                },
+                None => other.payload.get("type").is_none() && other.payload.get("value").is_none(),
+            };
+            self.enabled == other.enabled && self._name == other.name && payload_matches
+        }
+    }
+
     #[derive(Debug, Deserialize)]
     struct VariantTest {
         description: String,
@@ -134,11 +149,11 @@ mod tests {
                 }
                 Tests::VariantTests { variant_tests } => {
                     for test in variant_tests {
-                        let result =
-                            c.is_enabled_str(&test.toggle_name, Some(&test.context), false);
+                        let result = c.get_variant_str(&test.toggle_name, &test.context);
+
                         assert_eq!(
-                            test.expected_result.enabled, result,
-                            "Test '{}' failed: got {} instead of {:?}",
+                            test.expected_result, result,
+                            "Test '{}' failed: got {:?} instead of {:?}",
                             test.description, result, test.expected_result
                         );
                     }