NonNullUnicodeSetAttribute for blind cred and script upgrades (#166)

CHANGELOG.md

@@ -1,7 +1,5 @@
 # Changelog
 
-# 2
-
 ## 3.0.0
 
 * This is a breaking release, if you're using blind credentials. This change
@@ -11,6 +9,12 @@
   This is due to a breaking change in pynamodb itself, which requires using
   specific versions of pynamodb to migrate the underlying data.
 
+## 2.0.1
+
+* Added additional logging in the v1 routes.
+* Updated the migration script to include both Service and BlindCredential
+  migrations, as well as checks to ensure the migration was successful.
+
 ## 2.0.0
 WARNING: If you upgrade to this version, any new writes to blind credentials
 will be in a format that is only compatible in 1.11.0 forward. If you've

confidant/models/blind_credential.py

@@ -3,7 +3,6 @@
 from pynamodb.models import Model
 from pynamodb.attributes import (
     UnicodeAttribute,
-    UnicodeSetAttribute,
     NumberAttribute,
     LegacyBooleanAttribute,
     UTCDateTimeAttribute,
@@ -14,6 +13,9 @@
 from confidant.app import app
 from confidant.models.session_cls import DDBSession
 from confidant.models.connection_cls import DDBConnection
+from confidant.models.non_null_unicode_set_attribute import (
+    NonNullUnicodeSetAttribute
+)
 
 
 class DataTypeDateIndex(GlobalSecondaryIndex):
@@ -40,7 +42,7 @@ class Meta:
     data_type_date_index = DataTypeDateIndex()
     name = UnicodeAttribute()
     credential_pairs = JSONAttribute()
-    credential_keys = UnicodeSetAttribute(default=set([]), null=True)
+    credential_keys = NonNullUnicodeSetAttribute(default=set([]), null=True)
     enabled = LegacyBooleanAttribute(default=True)
     data_key = JSONAttribute()
     cipher_version = NumberAttribute()

confidant/models/non_null_unicode_set_attribute.py

@@ -0,0 +1,19 @@
+from pynamodb.attributes import UnicodeSetAttribute
+
+
+class NonNullUnicodeSetAttribute(UnicodeSetAttribute):
+    def __get__(self, instance, value):
+        '''
+        Override UnicodeSetAttribute's __get__ method to return a set, rather
+        than None if the attribute isn't set.
+        '''
+        if instance:
+            # Get the attribute. If the object doesn't have the attribute,
+            # ensure we return a set.
+            _value = instance.attribute_values.get(self.attr_name, set())
+            # Attribute is assigned to None, return a set instead.
+            if _value is None:
+                _value = set()
+            return _value
+        else:
+            return self

confidant/models/service.py

@@ -3,7 +3,6 @@
 from pynamodb.models import Model
 from pynamodb.attributes import (
     UnicodeAttribute,
-    UnicodeSetAttribute,
     NumberAttribute,
     UTCDateTimeAttribute,
     LegacyBooleanAttribute
@@ -13,24 +12,9 @@
 from confidant.app import app
 from confidant.models.session_cls import DDBSession
 from confidant.models.connection_cls import DDBConnection
-
-
-class NonNullUnicodeSetAttribute(UnicodeSetAttribute):
-    def __get__(self, instance, value):
-        '''
-        Override UnicodeSetAttribute's __get__ method to return a set, rather
-        than None if the attribute isn't set.
-        '''
-        if instance:
-            # Get the attribute. If the object doesn't have the attribute,
-            # ensure we return a set.
-            _value = instance.attribute_values.get(self.attr_name, set())
-            # Attribute is assigned to None, return a set instead.
-            if _value is None:
-                _value = set()
-            return _value
-        else:
-            return self
+from confidant.models.non_null_unicode_set_attribute import (
+    NonNullUnicodeSetAttribute
+)
 
 
 class DataTypeDateIndex(GlobalSecondaryIndex):

confidant/routes/v1.py

@@ -125,6 +125,7 @@ def get_service(id):
     try:
         credentials = _get_credentials(service.credentials)
     except KeyError:
+        logging.exception('KeyError occurred in getting credentials')
         return jsonify({'error': 'Decryption error.'}), 500
     blind_credentials = _get_blind_credentials(service.blind_credentials)
     return jsonify({
@@ -145,6 +146,9 @@ def get_archive_service_revisions(id):
     try:
         service = Service.get(id)
     except DoesNotExist:
+        logging.warning(
+            'Item with id {0} does not exist.'.format(id)
+        )
         return jsonify({}), 404
     if (service.data_type != 'service' and
             service.data_type != 'archive-service'):
@@ -365,6 +369,9 @@ def get_credential(id):
     try:
         cred = Credential.get(id)
     except DoesNotExist:
+        logging.warning(
+            'Item with id {0} does not exist.'.format(id)
+        )
         return jsonify({}), 404
     if (cred.data_type != 'credential' and
             cred.data_type != 'archive-credential'):
@@ -404,6 +411,9 @@ def get_archive_credential_revisions(id):
     try:
         cred = Credential.get(id)
     except DoesNotExist:
+        logging.warning(
+            'Item with id {0} does not exist.'.format(id)
+        )
         return jsonify({}), 404
     if (cred.data_type != 'credential' and
             cred.data_type != 'archive-credential'):
@@ -852,6 +862,9 @@ def get_blind_credential(id):
     try:
         cred = BlindCredential.get(id)
     except DoesNotExist:
+        logging.warning(
+            'Item with id {0} does not exist.'.format(id)
+        )
         return jsonify({}), 404
     if (cred.data_type != 'blind-credential' and
             cred.data_type != 'archive-blind-credential'):
@@ -904,6 +917,9 @@ def get_archive_blind_credential_revisions(id):
         return jsonify({}), 404
     if (cred.data_type != 'blind-credential' and
             cred.data_type != 'archive-blind-credential'):
+        logging.warning(
+            'Item with id {0} does not exist.'.format(id)
+        )
         return jsonify({}), 404
     revisions = []
     _range = range(1, cred.revision + 1)

confidant/scripts/manage.py

@@ -6,7 +6,10 @@
 from confidant.scripts.utils import CreateDynamoTables
 from confidant.scripts.bootstrap import GenerateSecretsBootstrap
 from confidant.scripts.bootstrap import DecryptSecretsBootstrap
-from confidant.scripts.migrate import MigrateSetAttribute
+from confidant.scripts.migrate import (
+    MigrateBlindCredentialSetAttribute,
+    MigrateServiceSetAttribute,
+)
 
 manager = Manager(app.app)
 
@@ -26,7 +29,10 @@
 manager.add_command("create_dynamodb_tables", CreateDynamoTables)
 
 # Migration scripts
-manager.add_command("migrate_set_attribute", MigrateSetAttribute)
+manager.add_command("migrate_blind_cred_set_attribute",
+                    MigrateBlindCredentialSetAttribute)
+manager.add_command("migrate_service_set_attribute",
+                    MigrateServiceSetAttribute)
 
 
 def main():

confidant/scripts/migrate.py

@@ -4,16 +4,134 @@
 
 from confidant.app import app
 from confidant.models.blind_credential import BlindCredential
+from confidant.models.service import Service
+
+import json
+import six
+from pynamodb.attributes import Attribute, UnicodeAttribute
+from pynamodb.constants import STRING_SET
+from pynamodb.models import Model
 
 
 app.logger.addHandler(logging.StreamHandler(sys.stdout))
 app.logger.setLevel(logging.INFO)
 
 
-class MigrateSetAttribute(Command):
+def is_old_unicode_set(values):
+    if not values:
+        return False
+    return sum([x.startswith('"') for x in values]) > 0
+
+
+class SetMixin(object):
+    """
+    Adds (de)serialization methods for sets
+    """
+    def serialize(self, value):
+        """
+        Serializes a set
+
+        Because dynamodb doesn't store empty attributes,
+        empty sets return None
+        """
+        if value is not None:
+            try:
+                iter(value)
+            except TypeError:
+                value = [value]
+            if len(value):
+                return [json.dumps(val) for val in sorted(value)]
+        return None
+
+    def deserialize(self, value):
+        """
+        Deserializes a set
+        """
+        if value and len(value):
+            return set([json.loads(val) for val in value])
+
+
+class NewUnicodeSetAttribute(SetMixin, Attribute):
+    """
+    A unicode set
+    """
+    attr_type = STRING_SET
+    null = True
+
+    def element_serialize(self, value):
+        """
+        This serializes unicode / strings out as unicode strings.
+        It does not touch the value if it is already a unicode str
+        :param value:
+        :return:
+        """
+        if isinstance(value, six.text_type):
+            return value
+        return six.u(str(value))
+
+    def element_deserialize(self, value):
+        return value
+
+    def serialize(self, value):
+        if value is not None:
+            try:
+                iter(value)
+            except TypeError:
+                value = [value]
+            if len(value):
+                return [self.element_serialize(val) for val in sorted(value)]
+        return None
+
+    def deserialize(self, value):
+        if value and len(value):
+            return set([self.element_deserialize(val) for val in value])
+
+
+class GeneralCredentialModel(Model):
+    class Meta(BlindCredential.Meta):
+        pass
+
+    id = UnicodeAttribute(hash_key=True)
+    credential_keys = NewUnicodeSetAttribute(default=set([]), null=True)
+
+
+class GeneralServiceModel(Model):
+    class Meta(Service.Meta):
+        pass
+
+    id = UnicodeAttribute(hash_key=True)
+    credentials = NewUnicodeSetAttribute(default=set(), null=True)
+    blind_credentials = NewUnicodeSetAttribute(default=set(), null=True)
+
+
+class MigrateBlindCredentialSetAttribute(Command):
 
     def run(self):
+        total = 0
+        fail = 0
         app.logger.info('Migrating UnicodeSetAttribute in BlindCredential')
         for cred in BlindCredential.data_type_date_index.query(
                 'blind-credential'):
             cred.save()
+            new_cred = GeneralCredentialModel.get(cred.id)
+            if is_old_unicode_set(new_cred.credential_keys):
+                fail += 1
+            total += 1
+        print("Fail: {}, Total: {}".format(fail, total))
+
+
+class MigrateServiceSetAttribute(Command):
+
+    def run(self):
+        total = 0
+        fail = 0
+        app.logger.info('Migrating UnicodeSetAttribute in Service')
+        for service in Service.data_type_date_index.query(
+                'service'):
+            service.save()
+            new_service = GeneralServiceModel.get(service.id)
+            if (is_old_unicode_set(new_service.credentials) or
+                    is_old_unicode_set(new_service.blind_credentials)):
+                fail += 1
+            total += 1
+        print("Fail: {}, Total: {}".format(fail, total))

docs/source/basics/upgrade.html.markdown

@@ -12,9 +12,7 @@ document breaking changes and how to upgrade when they occur.
 ## Upgrading to 2.0.0 or 3.0.0
 
 Due to breaking changes in PynamoDB, to upgrade to 2.0.0 or 3.0.0 may require
-some data migration. It's only necessary to perform a data migration if you're
-using blind credentials. If you're not using blind credentials, this change
-isn't breaking and you can upgrade without migration.
+some data migration.
 
 PynamoDB changed its data model over a series of releases, which requires
 the upgrade path for Confidant to follow the same model. To upgrade to 3.0.0,
@@ -25,16 +23,20 @@ versions of Confidant.
 
 ### Performing the data migration
 
-Confidant 2.0.0 ships with a maintenance script for the data migration:
+Confidant 2.0.1 ships with two maintenance scripts for the data migration:
 
 ```bash
 cd /srv/confidant
 source venv/bin/activate
 
 # Encrypt the data
-python manage.py migrate_set_attribute
+python manage.py migrate_blind_cred_set_attribute
+python manage.py migrate_service_set_attribute
 ```
 
+These scripts may fail intermittently. If any failures are occur, retry the
+script until all objects are fully migrated.
+
 2.0.0 ships with the ability to enable a maintenance mode, which you may want
 to enable when upgrading to 2.0.0. Putting Confidant into maintenance mode
 will disallow any writes via the API, ensuring that blind credentials with the