Merge pull request #146 from fako/software-update

Python 3.8

.travis.yml

@@ -1,6 +1,8 @@
 language: python
 python:
   - "3.6"
+  - "3.7"
+  - "3.8"
 # command to install dependencies
 before_install:
   - sudo apt-get install libblas-dev liblapack-dev libatlas-base-dev gfortran

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.6
+FROM python:3.8
 
 RUN apt-get update && \
     apt-get install -y vim binutils libproj-dev gdal-bin gettext default-jdk
@@ -32,8 +32,8 @@ RUN python manage.py collectstatic --noinput
 # We're switching user to a non-priviliged user
 # The Python packages directory and datagrowth package needs to belong to that user
 # for dynamic packaging (see entrypoint)
-RUN chown app:app /usr/local/lib/python3.6/site-packages
-RUN chown -R app:app /usr/local/lib/python3.6/site-packages/datagrowth*
+RUN chown app:app /usr/local/lib/python3.8/site-packages
+RUN chown -R app:app /usr/local/lib/python3.8/site-packages/datagrowth*
 USER app:app
 
 # Compiling translations

src/core/migrations/0003_alter_data_hash.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-11-14 15:01
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0002_empty_schema_default'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='httpresourcemock',
+            name='data_hash',
+            field=models.CharField(blank=True, db_index=True, default='', max_length=255),
+        ),
+    ]

src/core/migrations/0004_on_delete.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-11-20 17:29
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0003_alter_data_hash'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='collective',
+            name='community_type',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'),
+        ),
+        migrations.AlterField(
+            model_name='communitymock',
+            name='current_growth',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth'),
+        ),
+        migrations.AlterField(
+            model_name='communitymock',
+            name='kernel_type',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType'),
+        ),
+        migrations.AlterField(
+            model_name='growth',
+            name='community_type',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'),
+        ),
+        migrations.AlterField(
+            model_name='growth',
+            name='input_type',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'),
+        ),
+        migrations.AlterField(
+            model_name='growth',
+            name='output_type',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'),
+        ),
+        migrations.AlterField(
+            model_name='individual',
+            name='community_type',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'),
+        ),
+        migrations.AlterField(
+            model_name='manifestation',
+            name='community_type',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'),
+        ),
+    ]

src/core/models/organisms/collective.py

@@ -1,15 +1,15 @@
 from django.apps import apps
 from django.db import models
 from django.contrib.contenttypes.fields import GenericForeignKey, ContentType
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 
 from datagrowth.datatypes import CollectionBase
 
 
 class Collective(CollectionBase):
 
     community = GenericForeignKey(ct_field="community_type", fk_field="community_id")
-    community_type = models.ForeignKey(ContentType, related_name="+")
+    community_type = models.ForeignKey(ContentType, related_name="+", on_delete=models.PROTECT)
     community_id = models.PositiveIntegerField()
 
     @classmethod

src/core/models/organisms/community.py

@@ -49,9 +49,9 @@ def collections(self):
     def documents(self):
         return self.individual_set
 
-    current_growth = models.ForeignKey('core.Growth', null=True)
+    current_growth = models.ForeignKey('core.Growth', null=True, on_delete=models.SET_NULL)
     kernel = GenericForeignKey(ct_field="kernel_type", fk_field="kernel_id")
-    kernel_type = models.ForeignKey(ContentType, null=True, blank=True)
+    kernel_type = models.ForeignKey(ContentType, null=True, blank=True, on_delete=models.PROTECT)
     kernel_id = models.PositiveIntegerField(null=True, blank=True)
 
     created_at = models.DateTimeField(auto_now_add=True)
@@ -282,7 +282,7 @@ def grow(self, *args):
         result = None
         if self.state in [CommunityState.NEW]:
             log.info("Preparing community")
-            self.state = CommunityState.ASYNC if self.config.async else CommunityState.SYNC
+            self.state = CommunityState.ASYNC if self.config.asynchronous else CommunityState.SYNC
             self.setup_growth(*args)
             self.current_growth = self.next_growth()
             self.save()  # in between save because next operations may take long and community needs to be claimed.

src/core/models/organisms/growth.py

@@ -2,6 +2,7 @@
 from operator import xor
 from collections import Iterator
 
+from django.apps import apps
 from django.db import models
 from django.contrib.contenttypes.fields import GenericForeignKey, ContentType
 from django.core.exceptions import ValidationError
@@ -54,24 +55,24 @@ class ContributeType(object):
 class Growth(models.Model, ProcessorMixin):
 
     community = GenericForeignKey(ct_field="community_type", fk_field="community_id")
-    community_type = models.ForeignKey(ContentType, related_name="+")
+    community_type = models.ForeignKey(ContentType, related_name="+", on_delete=models.PROTECT)
     community_id = models.PositiveIntegerField()
 
     type = models.CharField(max_length=255)
     config = ConfigurationField(
         namespace="growth",
-        private=["args", "kwargs", "async"]
+        private=["args", "kwargs", "asynchronous"]
     )
 
     process = models.CharField(max_length=255, choices=PROCESS_CHOICE_LIST)
     contribute = models.CharField(max_length=255, choices=PROCESS_CHOICE_LIST, null=True, blank=True)
     contribute_type = models.CharField(max_length=255, choices=CONTRIBUTE_TYPE_CHOICES, null=True, blank=True)
 
     input = GenericForeignKey(ct_field="input_type", fk_field="input_id")
-    input_type = models.ForeignKey(ContentType, related_name="+", null=True, blank=True)
+    input_type = models.ForeignKey(ContentType, related_name="+", null=True, blank=True, on_delete=models.PROTECT)
     input_id = models.PositiveIntegerField(null=True, blank=True)
     output = GenericForeignKey(ct_field="output_type", fk_field="output_id")
-    output_type = models.ForeignKey(ContentType, related_name="+", null=True, blank=True)
+    output_type = models.ForeignKey(ContentType, related_name="+", null=True, blank=True, on_delete=models.PROTECT)
     output_id = models.PositiveIntegerField(null=True, blank=True)
 
     result_id = models.CharField(max_length=255, null=True, blank=True)
@@ -91,7 +92,7 @@ def begin(self):  # TODO: test sample size to unlock
 
         self.config = self.community.config.to_dict(protected=True)  # TODO: make this += operation instead
 
-        processor, method, args_type = self.prepare_process(self.process, async=self.config.async)
+        processor, method, args_type = self.prepare_process(self.process, asynchronous=self.config.asynchronous)
         assert args_type == ArgumentsTypes.NORMAL and isinstance(self.input, (Individual, DocumentBase)) or \
             args_type == ArgumentsTypes.BATCH and isinstance(self.input, (Collective, CollectionBase)), \
             "Unexpected arguments type '{}' for input of class {}".format(args_type, self.input.__class__.__name__)
@@ -106,7 +107,7 @@ def begin(self):  # TODO: test sample size to unlock
         else:
             raise AssertionError("Growth.input is of unexpected type {}".format(type(self.input)))
 
-        if not self.config.async:
+        if not self.config.asynchronous:
             self.state = GrowthState.CONTRIBUTE
         else:
             self.state = GrowthState.PROCESSING
@@ -123,7 +124,7 @@ def finish(self, result):
             GrowthState.PROCESSING, GrowthState.COMPLETE, GrowthState.PARTIAL, GrowthState.CONTRIBUTE
         ], "Can't finish a growth that is in state {}".format(self.state)
 
-        processor, method, args_type = self.prepare_process(self.process, async=self.config.async)
+        processor, method, args_type = self.prepare_process(self.process, asynchronous=self.config.asynchronous)
 
         if self.state == GrowthState.PROCESSING:
             try:
@@ -238,7 +239,7 @@ def clean(self):
 
     @property
     def resources(self):
-        Resource = get_any_model(self.config.resource)
+        Resource = apps.get_model(self.config.resource)
         Type = ContentType.objects.get_for_model(self)
         return Resource.objects.filter(retainer_type__pk=Type.id, retainer_id=self.id)
 

src/core/models/organisms/individual.py

@@ -1,6 +1,6 @@
 from django.db import models
 from django.contrib.contenttypes.fields import GenericForeignKey, ContentType
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 
 
 from datagrowth.datatypes import DocumentBase, DocumentMysql
@@ -9,10 +9,10 @@
 class Individual(DocumentMysql, DocumentBase):
 
     community = GenericForeignKey(ct_field="community_type", fk_field="community_id")
-    community_type = models.ForeignKey(ContentType, related_name="+")
+    community_type = models.ForeignKey(ContentType, related_name="+", on_delete=models.PROTECT)
     community_id = models.PositiveIntegerField()
 
-    collective = models.ForeignKey('core.Collective', null=True)
+    collective = models.ForeignKey('core.Collective', null=True, on_delete=models.CASCADE)
 
     @property
     def collection(self):

src/core/models/organisms/tests/community.py

@@ -362,9 +362,9 @@ def test_grow_async(self, begin_growth):
         self.assertFalse(finish_growth.called)
         self.assertEqual(self.instance.state, CommunityState.READY)
 
-    @patch('core.tasks.http.get_resource_link', return_value=HttpResourceMock())
+    @patch('datagrowth.resources.http.tasks.get_resource_link', return_value=HttpResourceMock())
     def test_grow_sync(self, get_resource_link):
-        self.instance.config.async = False
+        self.instance.config.asynchronous = False
         self.set_callback_mocks()
         self.assertEqual(self.instance.state, CommunityState.NEW)
         done = self.instance.grow()

src/core/models/organisms/tests/growth.py

@@ -93,24 +93,24 @@ def setUp(self):
         self.processor = HttpResourceProcessor
 
     def test_begin_with_individual_input_async(self):
-        with patch('core.tasks.http.send.s', return_value=MockTask) as send_s:
+        with patch('datagrowth.resources.http.tasks.send.s', return_value=MockTask) as send_s:
             self.new.begin()
         MockTask.delay.assert_called_once_with(1024, 768, name="modest")
         self.assertEqual(self.new.result_id, "result-id")
         self.assertEqual(self.new.state, GrowthState.PROCESSING)
         self.assertFalse(self.new.is_finished)
 
     def test_begin_with_individual_input_sync(self):
-        self.new.config = {"async": False}
-        with patch('core.tasks.http.send.s', return_value=MockTask) as send_s:
+        self.new.config = {"asynchronous": False}
+        with patch('datagrowth.resources.http.tasks.send.s', return_value=MockTask) as send_s:
             self.new.begin()
         MockTask.delay.assert_called_once_with(1024, 768, name="modest")
         self.assertEqual(self.new.result_id, None)
         self.assertEqual(self.new.state, GrowthState.CONTRIBUTE)
         self.assertFalse(self.new.is_finished)
 
     def test_begin_with_collective_input_async(self):
-        with patch('core.tasks.http.send_mass.s', return_value=MockTask) as send_mass_s:
+        with patch('datagrowth.resources.http.tasks.send_mass.s', return_value=MockTask) as send_mass_s:
             self.collective_input.begin()
         MockTask.delay.assert_called_once_with(
             [["nested value 0"], ["nested value 1"], ["nested value 2"]],
@@ -121,8 +121,8 @@ def test_begin_with_collective_input_async(self):
         self.assertFalse(self.collective_input.is_finished)
 
     def test_begin_with_collective_input_sync(self):
-        self.collective_input.config = {"async": False}
-        with patch('core.tasks.http.send_mass.s', return_value=MockTask) as send_mass_s:
+        self.collective_input.config = {"asynchronous": False}
+        with patch('datagrowth.resources.http.tasks.send_mass.s', return_value=MockTask) as send_mass_s:
             self.collective_input.begin()
         MockTask.delay.assert_called_once_with(
             [["nested value 0"], ["nested value 1"], ["nested value 2"]],

src/core/models/resources/manifestation.py

@@ -21,7 +21,7 @@ class Manifestation(Resource):
     data = JSONField(null=True)
 
     community = GenericForeignKey(ct_field="community_type", fk_field="community_id")
-    community_type = models.ForeignKey(ContentType, related_name="+")
+    community_type = models.ForeignKey(ContentType, related_name="+", on_delete=models.PROTECT)
     community_id = models.PositiveIntegerField()
 
     task = models.CharField(max_length=255, null=True, blank=True)
@@ -34,7 +34,7 @@ def generate_config(allowed_config, **kwargs):
         config = {key: value for key, value in kwargs.items() if key in allowed_config}
         return config
 
-    def get_data(self, async=False):
+    def get_data(self, asynchronous=False):
         from core.tasks import get_manifestation_data
         if self.data:
             return self.data
@@ -49,7 +49,7 @@ def get_data(self, async=False):
             else:
                 raise AssertionError("get_data is not handling AsyncResult with status: {}".format(result.status))
             self.status = celery_status_code(result.status)
-        elif async:
+        elif asynchronous:
             self.task = get_manifestation_data.delay(self.id)
             self.status = celery_status_code(PENDING)
             self.save()

src/core/models/resources/tests/manifestation.py

@@ -36,8 +36,8 @@ def test_get_data_sync(self):
     def test_get_data_async(self, task_delay):
         # Test processing
         try:
-            self.instance.get_data(async=True)
-            self.fail("Expected get_data to raise DSProcessUnfinished when manifesting async")
+            self.instance.get_data(asynchronous=True)
+            self.fail("Expected get_data to raise DSProcessUnfinished when manifesting asynchronously")
         except DSProcessUnfinished:
             pass
         self.assertEqual(self.instance.status, 8)

src/core/processors/mixins.py

@@ -5,20 +5,17 @@
 
 class ProcessorMixin(object):
 
-    def prepare_process(self, process, async=False, class_config=None, extra_config=None):
+    def prepare_process(self, process, asynchronous=False, class_config=None):
         """
         Creates an instance of the processor based on requested process with a correct config set.
         Processors get loaded from core.processors
         It returns the processor and the method that should be invoked.
 
         :param process: A dotted string indicating the processor and method that represent the process.
+        :param asynchronous: Whether to process asynchronously or not
+        :param class_config: The configuration that should be used to supplement the processor configuration
         :return: processor, method
         """
-        if extra_config is not None:
-            warnings.warn("extra_config argument for prepare_process is deprecated, use class_config instead",
-                          DeprecationWarning)
-        class_config = class_config if isinstance(class_config, dict) else \
-            extra_config  # TODO: extra config is obsolete
         assert isinstance(class_config, (dict, type(None))), \
             "Class config given to prepare_process should be None or a dictionary"
         if class_config is not None:
@@ -27,7 +24,7 @@ def prepare_process(self, process, async=False, class_config=None, extra_config=
         processor_name, method_name = Processor.get_processor_components(process)
         processor = Processor.create_processor(processor_name, self.config.to_dict(protected=True))
         method, args_type = processor.get_processor_method(method_name)
-        if async:
+        if asynchronous:
             method = getattr(method, "delay")
         if not callable(method):
             raise AssertionError("{} is not a callable property on {}.".format(method_name, processor))

src/core/processors/resources/base.py

@@ -1,7 +1,7 @@
+from django.apps import apps
 from celery.result import AsyncResult, states as TaskStates
 
-from core.processors.base import Processor
-from core.utils.helpers import get_any_model
+from datagrowth.processors.base import Processor
 from core.exceptions import DSProcessUnfinished, DSProcessError
 
 
@@ -48,5 +48,5 @@ def results(self, result):
     @property
     def resource(self):
         if not self._resource:
-            self._resource = get_any_model(self.config.resource)
+            self._resource = apps.get_model(self.config.resource)
         return self._resource