diff --git a/.travis.yml b/.travis.yml index b1ac4bf2..c48faef6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ language: python python: - "3.6" + - "3.7" + - "3.8" # command to install dependencies before_install: - sudo apt-get install libblas-dev liblapack-dev libatlas-base-dev gfortran diff --git a/Dockerfile b/Dockerfile index 3b408e36..f2447bba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6 +FROM python:3.8 RUN apt-get update && \ apt-get install -y vim binutils libproj-dev gdal-bin gettext default-jdk @@ -32,8 +32,8 @@ RUN python manage.py collectstatic --noinput # We're switching user to a non-priviliged user # The Python packages directory and datagrowth package needs to belong to that user # for dynamic packaging (see entrypoint) -RUN chown app:app /usr/local/lib/python3.6/site-packages -RUN chown -R app:app /usr/local/lib/python3.6/site-packages/datagrowth* +RUN chown app:app /usr/local/lib/python3.8/site-packages +RUN chown -R app:app /usr/local/lib/python3.8/site-packages/datagrowth* USER app:app # Compiling translations diff --git a/src/core/migrations/0003_alter_data_hash.py b/src/core/migrations/0003_alter_data_hash.py new file mode 100644 index 00000000..c7f54ebc --- /dev/null +++ b/src/core/migrations/0003_alter_data_hash.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-14 15:01 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0002_empty_schema_default'), + ] + + operations = [ + migrations.AlterField( + model_name='httpresourcemock', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + ] diff --git a/src/core/migrations/0004_on_delete.py b/src/core/migrations/0004_on_delete.py new file mode 100644 index 00000000..3777319d --- /dev/null +++ b/src/core/migrations/0004_on_delete.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-20 17:29 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0003_alter_data_hash'), + ] + + operations = [ + migrations.AlterField( + model_name='collective', + name='community_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'), + ), + migrations.AlterField( + model_name='communitymock', + name='current_growth', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth'), + ), + migrations.AlterField( + model_name='communitymock', + name='kernel_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType'), + ), + migrations.AlterField( + model_name='growth', + name='community_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'), + ), + migrations.AlterField( + model_name='growth', + name='input_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'), + ), + migrations.AlterField( + model_name='growth', + name='output_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'), + ), + migrations.AlterField( + model_name='individual', + name='community_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'), + ), + migrations.AlterField( + model_name='manifestation', + name='community_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'), + ), + ] diff --git a/src/core/models/organisms/collective.py b/src/core/models/organisms/collective.py index 8293c40f..7bca8c00 100644 --- a/src/core/models/organisms/collective.py +++ b/src/core/models/organisms/collective.py @@ -1,7 +1,7 @@ from django.apps import apps from django.db import models from django.contrib.contenttypes.fields import GenericForeignKey, ContentType -from django.core.urlresolvers import reverse +from django.urls import reverse from datagrowth.datatypes import CollectionBase @@ -9,7 +9,7 @@ class Collective(CollectionBase): community = GenericForeignKey(ct_field="community_type", fk_field="community_id") - community_type = models.ForeignKey(ContentType, related_name="+") + community_type = models.ForeignKey(ContentType, related_name="+", on_delete=models.PROTECT) community_id = models.PositiveIntegerField() @classmethod diff --git a/src/core/models/organisms/community.py b/src/core/models/organisms/community.py index 080026d8..854e97b0 100644 --- a/src/core/models/organisms/community.py +++ b/src/core/models/organisms/community.py @@ -49,9 +49,9 @@ def collections(self): def documents(self): return self.individual_set - current_growth = models.ForeignKey('core.Growth', null=True) + current_growth = models.ForeignKey('core.Growth', null=True, on_delete=models.SET_NULL) kernel = GenericForeignKey(ct_field="kernel_type", fk_field="kernel_id") - kernel_type = models.ForeignKey(ContentType, null=True, blank=True) + kernel_type = models.ForeignKey(ContentType, null=True, blank=True, on_delete=models.PROTECT) kernel_id = models.PositiveIntegerField(null=True, blank=True) created_at = models.DateTimeField(auto_now_add=True) @@ -282,7 +282,7 @@ def grow(self, *args): result = None if self.state in [CommunityState.NEW]: log.info("Preparing community") - self.state = CommunityState.ASYNC if self.config.async else CommunityState.SYNC + self.state = CommunityState.ASYNC if self.config.asynchronous else CommunityState.SYNC self.setup_growth(*args) self.current_growth = self.next_growth() self.save() # in between save because next operations may take long and community needs to be claimed. diff --git a/src/core/models/organisms/growth.py b/src/core/models/organisms/growth.py index 85f8711e..508798b5 100644 --- a/src/core/models/organisms/growth.py +++ b/src/core/models/organisms/growth.py @@ -2,6 +2,7 @@ from operator import xor from collections import Iterator +from django.apps import apps from django.db import models from django.contrib.contenttypes.fields import GenericForeignKey, ContentType from django.core.exceptions import ValidationError @@ -54,13 +55,13 @@ class ContributeType(object): class Growth(models.Model, ProcessorMixin): community = GenericForeignKey(ct_field="community_type", fk_field="community_id") - community_type = models.ForeignKey(ContentType, related_name="+") + community_type = models.ForeignKey(ContentType, related_name="+", on_delete=models.PROTECT) community_id = models.PositiveIntegerField() type = models.CharField(max_length=255) config = ConfigurationField( namespace="growth", - private=["args", "kwargs", "async"] + private=["args", "kwargs", "asynchronous"] ) process = models.CharField(max_length=255, choices=PROCESS_CHOICE_LIST) @@ -68,10 +69,10 @@ class Growth(models.Model, ProcessorMixin): contribute_type = models.CharField(max_length=255, choices=CONTRIBUTE_TYPE_CHOICES, null=True, blank=True) input = GenericForeignKey(ct_field="input_type", fk_field="input_id") - input_type = models.ForeignKey(ContentType, related_name="+", null=True, blank=True) + input_type = models.ForeignKey(ContentType, related_name="+", null=True, blank=True, on_delete=models.PROTECT) input_id = models.PositiveIntegerField(null=True, blank=True) output = GenericForeignKey(ct_field="output_type", fk_field="output_id") - output_type = models.ForeignKey(ContentType, related_name="+", null=True, blank=True) + output_type = models.ForeignKey(ContentType, related_name="+", null=True, blank=True, on_delete=models.PROTECT) output_id = models.PositiveIntegerField(null=True, blank=True) result_id = models.CharField(max_length=255, null=True, blank=True) @@ -91,7 +92,7 @@ def begin(self): # TODO: test sample size to unlock self.config = self.community.config.to_dict(protected=True) # TODO: make this += operation instead - processor, method, args_type = self.prepare_process(self.process, async=self.config.async) + processor, method, args_type = self.prepare_process(self.process, asynchronous=self.config.asynchronous) assert args_type == ArgumentsTypes.NORMAL and isinstance(self.input, (Individual, DocumentBase)) or \ args_type == ArgumentsTypes.BATCH and isinstance(self.input, (Collective, CollectionBase)), \ "Unexpected arguments type '{}' for input of class {}".format(args_type, self.input.__class__.__name__) @@ -106,7 +107,7 @@ def begin(self): # TODO: test sample size to unlock else: raise AssertionError("Growth.input is of unexpected type {}".format(type(self.input))) - if not self.config.async: + if not self.config.asynchronous: self.state = GrowthState.CONTRIBUTE else: self.state = GrowthState.PROCESSING @@ -123,7 +124,7 @@ def finish(self, result): GrowthState.PROCESSING, GrowthState.COMPLETE, GrowthState.PARTIAL, GrowthState.CONTRIBUTE ], "Can't finish a growth that is in state {}".format(self.state) - processor, method, args_type = self.prepare_process(self.process, async=self.config.async) + processor, method, args_type = self.prepare_process(self.process, asynchronous=self.config.asynchronous) if self.state == GrowthState.PROCESSING: try: @@ -238,7 +239,7 @@ def clean(self): @property def resources(self): - Resource = get_any_model(self.config.resource) + Resource = apps.get_model(self.config.resource) Type = ContentType.objects.get_for_model(self) return Resource.objects.filter(retainer_type__pk=Type.id, retainer_id=self.id) diff --git a/src/core/models/organisms/individual.py b/src/core/models/organisms/individual.py index eb9e454f..56cd71ed 100644 --- a/src/core/models/organisms/individual.py +++ b/src/core/models/organisms/individual.py @@ -1,6 +1,6 @@ from django.db import models from django.contrib.contenttypes.fields import GenericForeignKey, ContentType -from django.core.urlresolvers import reverse +from django.urls import reverse from datagrowth.datatypes import DocumentBase, DocumentMysql @@ -9,10 +9,10 @@ class Individual(DocumentMysql, DocumentBase): community = GenericForeignKey(ct_field="community_type", fk_field="community_id") - community_type = models.ForeignKey(ContentType, related_name="+") + community_type = models.ForeignKey(ContentType, related_name="+", on_delete=models.PROTECT) community_id = models.PositiveIntegerField() - collective = models.ForeignKey('core.Collective', null=True) + collective = models.ForeignKey('core.Collective', null=True, on_delete=models.CASCADE) @property def collection(self): diff --git a/src/core/models/organisms/tests/community.py b/src/core/models/organisms/tests/community.py index 3fd0847d..0a548a5b 100644 --- a/src/core/models/organisms/tests/community.py +++ b/src/core/models/organisms/tests/community.py @@ -362,9 +362,9 @@ def test_grow_async(self, begin_growth): self.assertFalse(finish_growth.called) self.assertEqual(self.instance.state, CommunityState.READY) - @patch('core.tasks.http.get_resource_link', return_value=HttpResourceMock()) + @patch('datagrowth.resources.http.tasks.get_resource_link', return_value=HttpResourceMock()) def test_grow_sync(self, get_resource_link): - self.instance.config.async = False + self.instance.config.asynchronous = False self.set_callback_mocks() self.assertEqual(self.instance.state, CommunityState.NEW) done = self.instance.grow() diff --git a/src/core/models/organisms/tests/growth.py b/src/core/models/organisms/tests/growth.py index bb2df9da..2a19b68b 100644 --- a/src/core/models/organisms/tests/growth.py +++ b/src/core/models/organisms/tests/growth.py @@ -93,7 +93,7 @@ def setUp(self): self.processor = HttpResourceProcessor def test_begin_with_individual_input_async(self): - with patch('core.tasks.http.send.s', return_value=MockTask) as send_s: + with patch('datagrowth.resources.http.tasks.send.s', return_value=MockTask) as send_s: self.new.begin() MockTask.delay.assert_called_once_with(1024, 768, name="modest") self.assertEqual(self.new.result_id, "result-id") @@ -101,8 +101,8 @@ def test_begin_with_individual_input_async(self): self.assertFalse(self.new.is_finished) def test_begin_with_individual_input_sync(self): - self.new.config = {"async": False} - with patch('core.tasks.http.send.s', return_value=MockTask) as send_s: + self.new.config = {"asynchronous": False} + with patch('datagrowth.resources.http.tasks.send.s', return_value=MockTask) as send_s: self.new.begin() MockTask.delay.assert_called_once_with(1024, 768, name="modest") self.assertEqual(self.new.result_id, None) @@ -110,7 +110,7 @@ def test_begin_with_individual_input_sync(self): self.assertFalse(self.new.is_finished) def test_begin_with_collective_input_async(self): - with patch('core.tasks.http.send_mass.s', return_value=MockTask) as send_mass_s: + with patch('datagrowth.resources.http.tasks.send_mass.s', return_value=MockTask) as send_mass_s: self.collective_input.begin() MockTask.delay.assert_called_once_with( [["nested value 0"], ["nested value 1"], ["nested value 2"]], @@ -121,8 +121,8 @@ def test_begin_with_collective_input_async(self): self.assertFalse(self.collective_input.is_finished) def test_begin_with_collective_input_sync(self): - self.collective_input.config = {"async": False} - with patch('core.tasks.http.send_mass.s', return_value=MockTask) as send_mass_s: + self.collective_input.config = {"asynchronous": False} + with patch('datagrowth.resources.http.tasks.send_mass.s', return_value=MockTask) as send_mass_s: self.collective_input.begin() MockTask.delay.assert_called_once_with( [["nested value 0"], ["nested value 1"], ["nested value 2"]], diff --git a/src/core/models/resources/manifestation.py b/src/core/models/resources/manifestation.py index 95e931c5..26f8bf92 100644 --- a/src/core/models/resources/manifestation.py +++ b/src/core/models/resources/manifestation.py @@ -21,7 +21,7 @@ class Manifestation(Resource): data = JSONField(null=True) community = GenericForeignKey(ct_field="community_type", fk_field="community_id") - community_type = models.ForeignKey(ContentType, related_name="+") + community_type = models.ForeignKey(ContentType, related_name="+", on_delete=models.PROTECT) community_id = models.PositiveIntegerField() task = models.CharField(max_length=255, null=True, blank=True) @@ -34,7 +34,7 @@ def generate_config(allowed_config, **kwargs): config = {key: value for key, value in kwargs.items() if key in allowed_config} return config - def get_data(self, async=False): + def get_data(self, asynchronous=False): from core.tasks import get_manifestation_data if self.data: return self.data @@ -49,7 +49,7 @@ def get_data(self, async=False): else: raise AssertionError("get_data is not handling AsyncResult with status: {}".format(result.status)) self.status = celery_status_code(result.status) - elif async: + elif asynchronous: self.task = get_manifestation_data.delay(self.id) self.status = celery_status_code(PENDING) self.save() diff --git a/src/core/models/resources/tests/manifestation.py b/src/core/models/resources/tests/manifestation.py index 02311552..3dfa2bb8 100644 --- a/src/core/models/resources/tests/manifestation.py +++ b/src/core/models/resources/tests/manifestation.py @@ -36,8 +36,8 @@ def test_get_data_sync(self): def test_get_data_async(self, task_delay): # Test processing try: - self.instance.get_data(async=True) - self.fail("Expected get_data to raise DSProcessUnfinished when manifesting async") + self.instance.get_data(asynchronous=True) + self.fail("Expected get_data to raise DSProcessUnfinished when manifesting asynchronously") except DSProcessUnfinished: pass self.assertEqual(self.instance.status, 8) diff --git a/src/core/processors/mixins.py b/src/core/processors/mixins.py index 74d6a6cf..817506d4 100644 --- a/src/core/processors/mixins.py +++ b/src/core/processors/mixins.py @@ -5,20 +5,17 @@ class ProcessorMixin(object): - def prepare_process(self, process, async=False, class_config=None, extra_config=None): + def prepare_process(self, process, asynchronous=False, class_config=None): """ Creates an instance of the processor based on requested process with a correct config set. Processors get loaded from core.processors It returns the processor and the method that should be invoked. :param process: A dotted string indicating the processor and method that represent the process. + :param asynchronous: Whether to process asynchronously or not + :param class_config: The configuration that should be used to supplement the processor configuration :return: processor, method """ - if extra_config is not None: - warnings.warn("extra_config argument for prepare_process is deprecated, use class_config instead", - DeprecationWarning) - class_config = class_config if isinstance(class_config, dict) else \ - extra_config # TODO: extra config is obsolete assert isinstance(class_config, (dict, type(None))), \ "Class config given to prepare_process should be None or a dictionary" if class_config is not None: @@ -27,7 +24,7 @@ def prepare_process(self, process, async=False, class_config=None, extra_config= processor_name, method_name = Processor.get_processor_components(process) processor = Processor.create_processor(processor_name, self.config.to_dict(protected=True)) method, args_type = processor.get_processor_method(method_name) - if async: + if asynchronous: method = getattr(method, "delay") if not callable(method): raise AssertionError("{} is not a callable property on {}.".format(method_name, processor)) diff --git a/src/core/processors/resources/base.py b/src/core/processors/resources/base.py index dadaaee9..2e121145 100644 --- a/src/core/processors/resources/base.py +++ b/src/core/processors/resources/base.py @@ -1,7 +1,7 @@ +from django.apps import apps from celery.result import AsyncResult, states as TaskStates -from core.processors.base import Processor -from core.utils.helpers import get_any_model +from datagrowth.processors.base import Processor from core.exceptions import DSProcessUnfinished, DSProcessError @@ -48,5 +48,5 @@ def results(self, result): @property def resource(self): if not self._resource: - self._resource = get_any_model(self.config.resource) + self._resource = apps.get_model(self.config.resource) return self._resource diff --git a/src/core/processors/resources/http.py b/src/core/processors/resources/http.py index 43fb7fc0..445a4b3a 100644 --- a/src/core/processors/resources/http.py +++ b/src/core/processors/resources/http.py @@ -1,9 +1,7 @@ -import requests - from datagrowth.configuration import DEFAULT_CONFIGURATION +from datagrowth.resources.http.tasks import send, send_mass +from datagrowth.configuration import ConfigurationProperty from core.processors.resources.base import ResourceProcessor -from core.tasks.http import send, send_mass -from core.utils.configuration import ConfigurationProperty class HttpResourceProcessor(ResourceProcessor): @@ -17,14 +15,6 @@ class HttpResourceProcessor(ResourceProcessor): namespace="http_resource" ) - ####################################################### - # Getters - ####################################################### - - @classmethod - def get_session(cls, config): - return requests.Session() - ####################################################### # TASKS ####################################################### @@ -35,30 +25,26 @@ def get_session(cls, config): def fetch(self): return send.s( method="get", - config=self.config.to_dict(private=True, protected=True), - session=self.__class__.__name__ + config=self.config.to_dict(private=True, protected=True) ) @property def fetch_mass(self): return send_mass.s( method="get", - config=self.config.to_dict(private=True, protected=True), - session=self.__class__.__name__ + config=self.config.to_dict(private=True, protected=True) ) @property def submit(self): return send.s( method="post", - config=self.config.to_dict(private=True, protected=True), - session=self.__class__.__name__ + config=self.config.to_dict(private=True, protected=True) ) @property def submit_mass(self): return send_mass.s( method="post", - config=self.config.to_dict(private=True, protected=True), - session=self.__class__.__name__ + config=self.config.to_dict(private=True, protected=True) ) diff --git a/src/core/processors/resources/manifest.py b/src/core/processors/resources/manifest.py index 45e4ac45..500ead18 100644 --- a/src/core/processors/resources/manifest.py +++ b/src/core/processors/resources/manifest.py @@ -23,7 +23,7 @@ class ManifestProcessor(ResourceProcessor): def __init__(self, config): config.update({ - "_resource": "Manifestation" + "_resource": "core.Manifestation" }) super(ManifestProcessor, self).__init__(config) assert "_community" in config or "community" in config, \ diff --git a/src/core/processors/tests/mixins.py b/src/core/processors/tests/mixins.py index 86e69727..c0e03151 100644 --- a/src/core/processors/tests/mixins.py +++ b/src/core/processors/tests/mixins.py @@ -9,13 +9,13 @@ class TestProcessorMixin(object): def test_prepare_process_async(self): - process, method, args_type = self.instance.prepare_process(self.instance.process, async=True) + process, method, args_type = self.instance.prepare_process(self.instance.process, asynchronous=True) self.assertIsInstance(process, HttpResourceProcessor) self.assertTrue(callable(method)) self.assertIsInstance(method, MethodType) # 'delay' method of the Signature class def test_prepare_process_sync(self): - process, method, args_type = self.instance.prepare_process(self.instance.process, async=False) + process, method, args_type = self.instance.prepare_process(self.instance.process, asynchronous=False) self.assertIsInstance(process, HttpResourceProcessor) self.assertTrue(callable(method)) self.assertIsInstance(method, Signature) # the callable 'Signature' is a sync version of a task @@ -24,7 +24,7 @@ def test_prepare_process_batch(self): for batch_method in self.processor.ARGS_BATCH_METHODS: process, method, args_type = self.instance.prepare_process( "{}.{}".format(self.processor.__name__, batch_method), - async=False + asynchronous=False ) self.assertEqual(args_type, ArgumentsTypes.BATCH) @@ -33,11 +33,11 @@ def test_prepare_process_normal(self): print(normal_method) process, method, args_type = self.instance.prepare_process( "{}.{}".format(self.processor.__name__, normal_method), - async=True + asynchronous=True ) self.assertEqual(args_type, ArgumentsTypes.NORMAL) def test_config(self): self.instance.config = {"test": "config"} - process, method, args_type = self.instance.prepare_process(self.instance.process, async=True) + process, method, args_type = self.instance.prepare_process(self.instance.process, asynchronous=True) self.assertEqual(process.config.test, "config") diff --git a/src/core/processors/tests/resources.py b/src/core/processors/tests/resources.py index 87eb2916..e79c3f93 100644 --- a/src/core/processors/tests/resources.py +++ b/src/core/processors/tests/resources.py @@ -24,7 +24,7 @@ def setUp(self): defaults=None ) self.config.update({ - "resource": "HttpResourceMock", + "resource": "core.HttpResourceMock", }) self.prc = HttpResourceProcessor(config=self.config.to_dict(protected=True, private=True)) self.prc._send = MockTask @@ -32,7 +32,7 @@ def setUp(self): self.session = MockRequests MockTask.reset_mock() - @patch("core.tasks.http.send.s") + @patch("datagrowth.resources.http.tasks.send.s") def test_fetch(self, send_s): null = self.prc.fetch self.assertTrue(send_s.called) @@ -41,7 +41,7 @@ def test_fetch(self, send_s): self.assertIsInstance(kwargs["config"], dict) self.assertTrue(kwargs["config"].get("_resource")) - @patch("core.tasks.http.send_mass.s") + @patch("datagrowth.resources.http.tasks.send_mass.s") def test_fetch_mass(self, send_mass_s): null = self.prc.fetch_mass self.assertTrue(send_mass_s.called) @@ -50,7 +50,7 @@ def test_fetch_mass(self, send_mass_s): self.assertIsInstance(kwargs["config"], dict) self.assertTrue(kwargs["config"].get("_resource")) - @patch("core.tasks.http.send.s") + @patch("datagrowth.resources.http.tasks.send.s") def test_submit(self, send_s): null = self.prc.submit self.assertTrue(send_s.called) @@ -59,7 +59,7 @@ def test_submit(self, send_s): self.assertIsInstance(kwargs["config"], dict) self.assertTrue(kwargs["config"].get("_resource")) - @patch("core.tasks.http.send_mass.s") + @patch("datagrowth.resources.http.tasks.send_mass.s") def test_submit_mass(self, send_mass_s): null = self.prc.submit_mass self.assertTrue(send_mass_s.called) diff --git a/src/core/tasks/tests/http.py b/src/core/tasks/tests/http.py deleted file mode 100644 index 0320c95a..00000000 --- a/src/core/tasks/tests/http.py +++ /dev/null @@ -1,262 +0,0 @@ -from datetime import datetime - -from mock import patch -import requests - -from django.test import TestCase -from django.utils import six - -from core.tasks.http import send, send_serie, send_mass, get_resource_link, load_session -from core.utils.configuration import ConfigurationType -from core.tests.mocks.requests import MockRequestsWithAgent, MockRequests -from core.tests.mocks.http import HttpResourceMock - - -class TestHTTPTasksBase(TestCase): - - fixtures = ["test-http-resource-mock"] - method = "" - - def setUp(self): - super(TestHTTPTasksBase, self).setUp() - self.config = ConfigurationType( - namespace="http_resource", - private=["_resource", "_continuation_limit"], - defaults=None - ) - self.config.update({ - "resource": "HttpResourceMock", - }) - self.session = MockRequests - - def get_args_list(self, queries): - if self.method == "get": - return [[query] for query in queries] - elif self.method == "post": - return [[] for query in queries] - else: - raise Exception("{} does not have a valid method specified.".format(self.__class__.__name__)) - - def get_kwargs_list(self, queries): - if self.method == "get": - return [{} for query in queries] - elif self.method == "post": - return [{"query": query} for query in queries] - else: - raise Exception("{} does not have a valid method specified.".format(self.__class__.__name__)) - - def check_results(self, results, expected_length): - self.assertEqual(len(results), expected_length) - for pk in results: - self.assertIsInstance(pk, six.integer_types) - self.assertGreater(pk, 0) - - -class TestSendMassTaskBase(TestHTTPTasksBase): - - def test_send_mass(self): - args_list = self.get_args_list(["test", "test2", "404"]) - kwargs_list = self.get_kwargs_list(["test", "test2", "404"]) - start = datetime.now() - scc, err = send_mass(args_list, kwargs_list, method=self.method, config=self.config, session=MockRequests) - end = datetime.now() - duration = (end - start).total_seconds() - self.assertLess(duration, 0.1) - self.check_results(scc, 2) - self.check_results(err, 1) - - def test_send_mass_intervals(self): - self.config.interval_duration = 250 # 0.25 secs - args_list = self.get_args_list(["test", "test2"]) - kwargs_list = self.get_kwargs_list(["test", "test2"]) - start = datetime.now() - scc, err = send_mass(args_list, kwargs_list, method=self.method, config=self.config, session=MockRequests) - end = datetime.now() - duration = (end - start).total_seconds() - self.assertGreater(duration, 0.5) - self.assertLess(duration, 1) - self.check_results(scc, 2) - self.check_results(err, 0) - - def test_send_mass_continuation_prohibited(self): - args_list = self.get_args_list(["test", "next", "404"]) - kwargs_list = self.get_kwargs_list(["test", "next", "404"]) - scc, err = send_mass(args_list, kwargs_list, method=self.method, config=self.config, session=MockRequests) - self.check_results(scc, 2) - self.check_results(err, 1) - - def test_send_mass_continuation(self): - self.config.continuation_limit = 10 - args_list = self.get_args_list(["test", "next", "404"]) - kwargs_list = self.get_kwargs_list(["test", "next", "404"]) - scc, err = send_mass( - args_list, - kwargs_list, - method=self.method, - config=self.config, - session=MockRequests - ) - self.check_results(scc, 3) - self.check_results(err, 1) - - @patch("core.tasks.http.send_serie", return_value=([], [],)) - def test_send_mass_concat_arguments(self, send_serie): - self.config.concat_args_size = 3 - self.config.concat_args_symbol = "|" - scc, err = send_mass( - [[1], [2], [3], [4], [5, 5], [6], [7]], - [{}, {}, {}, {}, {}, {}, {}], - method=self.method, - config=self.config, - session=MockRequests - ) - send_serie.assert_called_with( - [["1|2|3"], ["4|5|5|6"], ["7"]], [{}, {}, {}], - method=self.method, - config=self.config, - session=MockRequests - ) - - def test_send_inserted_session_provider(self): - self.skipTest("not tested") - - -class TestSendMassTaskGet(TestSendMassTaskBase): - method = "get" - - -class TestSendMassTaskPost(TestSendMassTaskBase): - method = "post" - - -class TestSendTaskGet(TestHTTPTasksBase): - - method = "get" - - def test_send(self): - # Test makes equivalent call of HttpResourceProcessor.fetch.delay("test") - scc, err = send("test", method=self.method, config=self.config, session=self.session) - self.check_results(scc, 1) - self.check_results(err, 0) - # Similar but with a cached result - scc, err = send("success", method=self.method, config=self.config, session=self.session) - self.check_results(scc, 1) - self.check_results(err, 0) - # And with an error response - scc, err = send("404", method=self.method, config=self.config, session=self.session) - self.check_results(scc, 0) - self.check_results(err, 1) - scc, err = send("500", method=self.method, config=self.config, session=self.session) - self.check_results(scc, 0) - self.check_results(err, 1) - - def test_send_continuation_prohibited(self): - scc, err = send("next", method=self.method, config=self.config, session=self.session) - self.check_results(scc, 1) - self.check_results(err, 0) - - def test_send_continuation(self): - self.config.continuation_limit = 10 - scc, err = send("next", method=self.method, config=self.config, session=self.session) - self.check_results(scc, 2) - self.check_results(err, 0) - - def test_send_inserted_session(self): - scc, err = send("test", method=self.method, config=self.config, session=MockRequestsWithAgent) - self.check_results(scc, 1) - self.check_results(err, 0) - link = HttpResourceMock.objects.get(id=scc[0]) - self.assertIn("user-agent", link.head) - - def test_send_inserted_session_provider(self): - self.skipTest("not tested") - - -class TestSendTaskPost(TestHTTPTasksBase): - - method = "post" - - def test_send(self): - # Test makes equivalent call of HttpResourceProcessor.fetch.delay("test") - scc, err = send(query="test", method=self.method, config=self.config, session=self.session) - self.check_results(scc, 1) - self.check_results(err, 0) - # Similar but with a cached result - scc, err = send(query="success", method=self.method, config=self.config, session=self.session) - self.check_results(scc, 1) - self.check_results(err, 0) - # And with an error response - scc, err = send(query="404", method=self.method, config=self.config, session=self.session) - self.check_results(scc, 0) - self.check_results(err, 1) - scc, err = send(query="500", method=self.method, config=self.config, session=self.session) - self.check_results(scc, 0) - self.check_results(err, 1) - - def test_send_continuation_prohibited(self): - scc, err = send(query="next", method=self.method, config=self.config, session=self.session) - self.check_results(scc, 1) - self.check_results(err, 0) - - def test_send_continuation(self): - self.config.continuation_limit = 10 - scc, err = send(query="next", method=self.method, config=self.config, session=self.session) - self.check_results(scc, 2) - self.check_results(err, 0) - - def test_send_inserted_session(self): - scc, err = send(query="test", method=self.method, config=self.config, session=MockRequestsWithAgent) - self.check_results(scc, 1) - self.check_results(err, 0) - link = HttpResourceMock.objects.get(id=scc[0]) - self.assertIn("user-agent", link.head) - - def test_send_inserted_session_provider(self): - self.skipTest("not tested") - - -class TestSendSerieTaskGet(TestHTTPTasksBase): - - def test_case(self): - # TODO: very similar to TestSendMassTaskGet, refactor? - self.skipTest("not tested") - - def test_send_inserted_session_provider(self): - self.skipTest("not tested") - - def test_send_serie_intervals(self): - self.skipTest("not tested") - - -class TestSendSerieTaskPost(TestHTTPTasksBase): - - def test_case(self): - # TODO: very similar to TestSendMassTaskPost, refactor? - self.skipTest("not tested") - - def test_send_inserted_session_provider(self): - self.skipTest("not tested") - - -class TestGetResourceLink(TestHTTPTasksBase): - - def test_get_link(self): - self.config.update({"test": "test"}) - session = requests.Session() - session.cookies = {"test": "test"} - link = get_resource_link(config=self.config, session=session) - self.assertIsInstance(link, HttpResourceMock) - self.assertIsNone(link.id) - self.assertIsNone(link.request) - self.assertFalse(hasattr(link.config, 'resource')) - self.assertEqual(link.config.test, 'test') - self.assertEqual(link.session.cookies, {"test": "test"}) - - -class TestLoadSession(TestCase): - - def test_load_session(self): - self.skipTest("not tested") - - def test_preload_session(self): - self.skipTest("not tested") diff --git a/src/core/tests/fixtures/test-community.json b/src/core/tests/fixtures/test-community.json index bde0cb6d..e1ba9f60 100644 --- a/src/core/tests/fixtures/test-community.json +++ b/src/core/tests/fixtures/test-community.json @@ -26,7 +26,7 @@ "completed_at": null, "state": "Asynchronous", "current_growth": 1, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\"}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\"}", "purge_at": null, "signature": "test" }, @@ -43,7 +43,7 @@ "completed_at": "2014-06-05T16:17:54.000", "state": "Asynchronous", "current_growth": 1, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\", \"setting1\": \"const\"}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\", \"setting1\": \"const\"}", "purge_at": null, "signature": "setting1=const&test-multiple" }, @@ -60,7 +60,7 @@ "completed_at": "2015-06-05T16:17:54.000", "state": "Asynchronous", "current_growth": 3, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\", \"setting1\": \"const\"}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\", \"setting1\": \"const\"}", "purge_at": null, "signature": "setting1=const&test-multiple" }, @@ -77,7 +77,7 @@ "completed_at": "2015-06-05T16:17:54.000", "state": "Synchronous", "current_growth": 3, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\"}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\"}", "purge_at": null, "signature": "setting1=const&test-ready" }, @@ -94,7 +94,7 @@ "completed_at": "2016-06-05T16:17:54.000", "state": "Ready", "current_growth": 3, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\"}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\"}", "purge_at": null, "signature": "setting1=const&test-empty" }, @@ -111,7 +111,7 @@ "completed_at": "2016-06-05T16:17:54.000", "state": "Ready", "current_growth": 3, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\", \"include_odd\": true, \"include_even\": false}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\", \"include_odd\": true, \"include_even\": false}", "purge_at": null, "signature": "setting1=const&test-ready" }, @@ -128,7 +128,7 @@ "completed_at": "2017-06-05T16:17:54.000", "state": "Ready", "current_growth": 3, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\", \"include_odd\": true, \"include_even\": true}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\", \"include_odd\": true, \"include_even\": true}", "purge_at": null, "signature": "setting1=const&test-ready" }, @@ -148,7 +148,7 @@ "input_type": null, "output_type": ["core", "collective"], "is_finished": true, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\"}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\"}", "community_type": ["core", "communitymock"], "community_id": 2 }, @@ -168,7 +168,7 @@ "input_type": ["core", "collective"], "output_type": ["core", "collective"], "is_finished": false, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\"}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\"}", "community_type": ["core", "communitymock"], "community_id": 2 }, @@ -188,7 +188,7 @@ "input_type": ["core", "collective"], "output_type": ["core", "collective"], "is_finished": false, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\"}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\"}", "community_type": ["core", "communitymock"], "community_id": 4 }, diff --git a/src/core/tests/fixtures/test-growth.json b/src/core/tests/fixtures/test-growth.json index 6e0f8dba..48a1d225 100644 --- a/src/core/tests/fixtures/test-growth.json +++ b/src/core/tests/fixtures/test-growth.json @@ -35,7 +35,7 @@ "collective" ], "is_finished": false, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\", \"args\": [\"$.width\", \"$.height\"], \"kwargs\": {\"name\": \"$.name\"}}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\", \"args\": [\"$.width\", \"$.height\"], \"kwargs\": {\"name\": \"$.name\"}}", "community_type": ["core", "communitymock"], "community_id": 1 }, @@ -61,7 +61,7 @@ "collective" ], "is_finished": false, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\"}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\"}", "community_type": ["core", "communitymock"], "community_id": 1 }, @@ -87,7 +87,7 @@ "collective" ], "is_finished": true, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\"}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\"}", "community_type": ["core", "communitymock"], "community_id": 1 }, @@ -113,7 +113,7 @@ "collective" ], "is_finished": false, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\"}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\"}", "community_type": ["core", "communitymock"], "community_id": 1 }, @@ -139,7 +139,7 @@ "collective" ], "is_finished": false, - "config": "{\"objective\": {\"@\": \"$\", \"value\":\"$.value\", \"extra\":\"$.extra\"}, \"resource\": \"HttpResourceMock\", \"args\": [\"$.value\"], \"kwargs\": {\"context\": \"$.context\"}}", + "config": "{\"objective\": {\"@\": \"$\", \"value\":\"$.value\", \"extra\":\"$.extra\"}, \"resource\": \"core.HttpResourceMock\", \"args\": [\"$.value\"], \"kwargs\": {\"context\": \"$.context\"}}", "community_type": ["core", "communitymock"], "community_id": 1 }, @@ -165,7 +165,7 @@ "collective" ], "is_finished": false, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\"}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\"}", "community_type": ["core", "communitymock"], "community_id": 1 }, @@ -191,7 +191,7 @@ "individual" ], "is_finished": false, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\"}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\"}", "community_type": ["core", "communitymock"], "community_id": 1 }, diff --git a/src/core/tests/fixtures/test-manifestation.json b/src/core/tests/fixtures/test-manifestation.json index ddc51194..743e9622 100644 --- a/src/core/tests/fixtures/test-manifestation.json +++ b/src/core/tests/fixtures/test-manifestation.json @@ -9,7 +9,7 @@ "completed_at": "2016-06-05T16:17:54.000", "state": "Ready", "current_growth": null, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\", \"include_odd\": true, \"include_even\": false}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\", \"include_odd\": true, \"include_even\": false}", "purge_at": null, "signature": "setting1=const&test-ready" }, diff --git a/src/core/tests/fixtures/test-organisms.json b/src/core/tests/fixtures/test-organisms.json index 463eba7e..19e753e3 100644 --- a/src/core/tests/fixtures/test-organisms.json +++ b/src/core/tests/fixtures/test-organisms.json @@ -9,7 +9,7 @@ "completed_at": null, "state": "New", "current_growth": null, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\"}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\"}", "purge_at": null }, "model": "core.CommunityMock", @@ -75,7 +75,7 @@ "completed_at": null, "state": "New", "current_growth": null, - "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"HttpResourceMock\"}", + "config": "{\"objective\": {\"@\": \"$.dict.list\", \"value\":\"$\", \"#context\":\"$.dict.test\"}, \"resource\": \"core.HttpResourceMock\"}", "purge_at": null }, "model": "core.CommunityMock", diff --git a/src/core/tests/mocks/community.py b/src/core/tests/mocks/community.py index 158ba212..3412c50f 100644 --- a/src/core/tests/mocks/community.py +++ b/src/core/tests/mocks/community.py @@ -11,7 +11,7 @@ class CommunityMock(Community): "config": { "_args": ["$.test"], "_kwargs": {}, - "_resource": "HttpResourceMock", + "_resource": "core.HttpResourceMock", "_objective": { "@": "$.dict.list", "value": "$", @@ -42,7 +42,7 @@ class CommunityMock(Community): "config": { "_args": ["$.value"], "_kwargs": {}, - "_resource": "HttpResourceMock", + "_resource": "core.HttpResourceMock", "_objective": { "@": "$.dict.list", "value": "$", @@ -68,7 +68,7 @@ class CommunityMock(Community): "config": { "_args": ["$.value"], "_kwargs": {}, - "_resource": "HttpResourceMock", + "_resource": "core.HttpResourceMock", }, "input": "@phase2", "contribute": None, diff --git a/src/core/tests/tests.py b/src/core/tests/tests.py index e62abc91..9c5c1f1c 100644 --- a/src/core/tests/tests.py +++ b/src/core/tests/tests.py @@ -16,8 +16,6 @@ from core.models.organisms.tests.individual import TestIndividual from core.models.resources.tests.manifestation import TestManifestationResource -from core.tasks.tests.http import (TestSendMassTaskGet, TestSendMassTaskPost, TestSendTaskGet, TestSendTaskPost, - TestSendSerieTaskGet, TestSendSerieTaskPost, TestGetResourceLink, TestLoadSession) from core.tasks.tests.manifestation import TestManifestationTasks from core.views.tests.collective import TestCollectiveView, TestCollectiveContentView diff --git a/src/core/utils/data/numeric_features.py b/src/core/utils/data/numeric_features.py index 934e3e32..421d0ac2 100644 --- a/src/core/utils/data/numeric_features.py +++ b/src/core/utils/data/numeric_features.py @@ -82,8 +82,8 @@ def load_content(self, content_callable=None, feature_names=None): # the column partially exists # we add what is new and update existing rows missing = series.index.difference(self.data[column].index) - new[series.name] = series.loc[list(missing.get_values())] - update[series.name] = series.loc[list(intersection.get_values())] + new[series.name] = series.loc[list(missing.values)] + update[series.name] = series.loc[list(intersection.values)] self.data = self.data.append(new, sort=False) self.data.update(update) @@ -113,7 +113,7 @@ def get_feature_value(self, feature_name, feature_callable, content, content_has def get_feature_series(self, feature_name, feature_callable, content_callable=None, context=None): contents = content_callable() if content_callable else self.content() - series = pd.Series(name=feature_name) + series = pd.Series(name=feature_name, dtype="float64") for content in contents: content_hash = self.get_content_hash(content) identifier = self.get_identifier(content) @@ -138,7 +138,7 @@ def load_features(self, callables): self.features.update(features) feature_names = features.keys() for column in feature_names: - self.data[column] = pd.Series() + self.data[column] = pd.Series(dtype="float64") if self.content is not None: self.load_content(feature_names=feature_names) @@ -162,7 +162,7 @@ def reset(self, features=None, content=None): self.load_features(features) elif self.features is not None and content is not None: for column in self.features.keys(): - self.data[column] = pd.Series() + self.data[column] = pd.Series(dtype="float64") self.load_content(content) def rank_by_params(self, params, limit=20): diff --git a/src/core/utils/helpers.py b/src/core/utils/helpers.py index ed22bcf7..38e879f9 100644 --- a/src/core/utils/helpers.py +++ b/src/core/utils/helpers.py @@ -38,7 +38,7 @@ def merge_iter(*iterables, **kwargs): # TODO: test to unlock, works bad with em except StopIteration: del iterables[i] if not iterables: - raise + return def iroundrobin(*iterables): # TODO: test to unlock diff --git a/src/core/utils/tests/data.py b/src/core/utils/tests/data.py index b1412b16..22487041 100644 --- a/src/core/utils/tests/data.py +++ b/src/core/utils/tests/data.py @@ -143,7 +143,7 @@ def set_language_to_fr(test): def test_init(self): sorted_feature_names = ["is_dutch", "is_english", "value_number"] - self.assertEquals( + self.assertEqual( sorted(self.frame.features.keys()), sorted_feature_names ) @@ -206,7 +206,7 @@ def test_init_file(self): file_path="test/path/to/frame.pkl" ) sorted_feature_names = ["is_dutch", "is_english", "value_number"] - self.assertEquals( + self.assertEqual( sorted(frame.features.keys()), sorted_feature_names ) @@ -264,7 +264,7 @@ def test_adding_features(self): ]) assert_frame_equal(frame.data, self.test_frame, check_like=True) sorted_feature_names = ["is_dutch", "is_english", "value_number"] - self.assertEquals( + self.assertEqual( sorted(self.frame.features.keys()), sorted_feature_names ) @@ -308,7 +308,7 @@ def test_resetting_features_and_content(self): self.test_frame_extra = self.test_frame_extra.drop(labels="is_dutch", axis=1) assert_frame_equal(frame.data, self.test_frame_extra, check_like=True) sorted_feature_names = ["is_english", "value_number"] - self.assertEquals( + self.assertEqual( sorted(frame.features.keys()), sorted_feature_names ) @@ -329,7 +329,7 @@ def test_resetting_features(self): self.test_frame = self.test_frame.drop(labels="is_dutch", axis=1) assert_frame_equal(frame.data, self.test_frame, check_like=True) sorted_feature_names = ["is_english", "value_number"] - self.assertEquals( + self.assertEqual( sorted(frame.features.keys()), sorted_feature_names ) @@ -349,7 +349,7 @@ def test_resetting_features_no_content(self): self.test_frame = self.test_frame.drop(labels="is_dutch", axis=1) assert_frame_equal(frame.data, self.test_frame[0:0], check_like=True) sorted_feature_names = ["is_english", "value_number"] - self.assertEquals( + self.assertEqual( sorted(frame.features.keys()), sorted_feature_names ) @@ -377,7 +377,7 @@ def test_clean_params(self): for function in [str, int, float]: test_params["is_dutch"] = function(test_params["is_dutch"]) cleaned_params = self.frame.clean_params(test_params) - self.assertEquals(cleaned_params, {"is_dutch": 1.0, "value_number": 2.0}) + self.assertEqual(cleaned_params, {"is_dutch": 1.0, "value_number": 2.0}) test_error_params = { "is_dutch": "1", @@ -391,9 +391,9 @@ def test_clean_params(self): def test_rank_by_params(self): ranking = self.frame.rank_by_params({"is_dutch": 1, "value_number": 1}) - self.assertEquals(ranking, [5, 8, 6, 4, 7]) + self.assertEqual(ranking, [5, 8, 6, 4, 7]) ranking = self.frame.rank_by_params({"is_dutch": 0.5, "value_number": -1, "is_english": 2, "is_french": 100}) - self.assertEquals(ranking, [7, 8, 6, 4, 5]) + self.assertEqual(ranking, [7, 8, 6, 4, 5]) def test_get_content_hash(self): self.skipTest("not tested") diff --git a/src/core/views/community.py b/src/core/views/community.py index eb00b29f..26c67706 100644 --- a/src/core/views/community.py +++ b/src/core/views/community.py @@ -5,7 +5,7 @@ from django.shortcuts import Http404 from django.template.response import TemplateResponse from django.views.generic import View -from django.core.urlresolvers import reverse +from django.urls import reverse from rest_framework.views import APIView from rest_framework.response import Response @@ -35,7 +35,7 @@ class CommunityView(APIView): @classmethod def _get_response_from_manifestation(cls, manifestation): response_data = copy(cls.RESPONSE_DATA) - manifestation_data = manifestation.get_data(async=manifestation.community.ASYNC_MANIFEST) + manifestation_data = manifestation.get_data(asynchronous=manifestation.community.ASYNC_MANIFEST) if not manifestation_data: return Response(None, HTTP_204_NO_CONTENT) results_key = "results" if isinstance(manifestation_data, list) else "result" diff --git a/src/datascope/requirements/development.txt b/src/datascope/requirements/development.txt index 02f2cb3f..da0ce081 100644 --- a/src/datascope/requirements/development.txt +++ b/src/datascope/requirements/development.txt @@ -1,7 +1,7 @@ -r minimal.txt # Extended production dependencies -psycopg2-binary==2.8.3 +psycopg2-binary==2.8.4 spacy==2.0.18 spacy-arguing-lexicon==0.0.3 IndicoIo==0.15.4 @@ -11,7 +11,7 @@ django-extensions==1.9.0 Sphinx==1.3.1 sphinx-rtd-theme==0.1.8 pipdeptree==0.10.1 -invoke==1.4.1 +docker-compose==1.27.4 # SpaCy language models https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-2.0.0/en_core_web_lg-2.0.0.tar.gz diff --git a/src/datascope/requirements/minimal.txt b/src/datascope/requirements/minimal.txt index e7236fc6..b5a9f975 100644 --- a/src/datascope/requirements/minimal.txt +++ b/src/datascope/requirements/minimal.txt @@ -11,10 +11,10 @@ six==1.11.0 jsonschema==2.4.0 html5lib==1.0b8 beautifulsoup4==4.6.0 -numpy==1.17.0 -pandas==0.24.0 -scipy==1.3.0 -scikit-learn==0.21.3 +numpy==1.19.4 +pandas==1.1.4 +scipy==1.5.4 +scikit-learn==0.23.2 tqdm==4.19.8 # Celery plugins diff --git a/src/datascope/requirements/production.txt b/src/datascope/requirements/production.txt index 2f43d07d..97b46fa8 100644 --- a/src/datascope/requirements/production.txt +++ b/src/datascope/requirements/production.txt @@ -1,10 +1,10 @@ -r minimal.txt https://projects.unbit.it/downloads/uwsgi-lts.tar.gz -psycopg2-binary==2.8.3 +psycopg2-binary==2.8.4 spacy==2.0.18 spacy-arguing-lexicon==0.0.3 IndicoIo==0.15.4 # Datagrowth -datagrowth==0.15.5 +datagrowth==0.16.3 diff --git a/src/datascope/settings_test.py b/src/datascope/settings_test.py index c42ee9ed..43889cf4 100644 --- a/src/datascope/settings_test.py +++ b/src/datascope/settings_test.py @@ -35,6 +35,7 @@ # testing basic functionality "name_namespace_configuration": "namespace configuration", "global_global_configuration": "global configuration", + "global_asynchronous": True, "global_async": True, "global_user_agent": "DataGrowth (test)", "global_token": "", diff --git a/src/datascope/urls.py b/src/datascope/urls.py index 0069c0b1..d94c4522 100644 --- a/src/datascope/urls.py +++ b/src/datascope/urls.py @@ -9,9 +9,10 @@ from core import views as core_views from wiki_feed.urls import urlpatterns as wiki_feed_patterns from visual_translations.urls import urlpatterns as visual_translations_patterns -from future_fashion.urls import urlpatterns as future_fashion_patterns, mediapatterns +from future_fashion import urls as future_fashion_urls +from future_fashion.urls import mediapatterns from wiki_scope.urls import urlpatterns as wiki_scope_patterns -from online_discourse.urls import urlpatterns as online_discourse_patterns +from online_discourse import urls as online_discourse_urls admin.autodiscover() @@ -41,8 +42,8 @@ ############################################# datagrowth_patterns = [ - url(r'^future-fashion/', include(future_fashion_patterns, namespace="future-fashion")), - url(r'^discourse-search/', include(online_discourse_patterns, namespace="online-discourse")), + url(r'^future-fashion/', include(future_fashion_urls)), + url(r'^discourse-search/', include(online_discourse_urls)), ] @@ -52,9 +53,9 @@ urlpatterns = [ url(r'^$', views.index, name="datascope-index"), - url(r'^data/v1/', include(legacy_patterns, namespace="v1")), + url(r'^data/v1/', include((legacy_patterns, "v1",))), url(r'^api/v1/auth/token/?$', rest_views.obtain_auth_token), - url(r'^api/v1/', include(datagrowth_patterns, namespace="api-v1")), + url(r'^api/v1/', include((datagrowth_patterns, "api-v1",))), url(r'^admin/', include(admin.site.urls)), url(r'^health/?$', views.health_check), ] diff --git a/src/future_fashion/migrations/0003_alter_data_hash.py b/src/future_fashion/migrations/0003_alter_data_hash.py new file mode 100644 index 00000000..aa843566 --- /dev/null +++ b/src/future_fashion/migrations/0003_alter_data_hash.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-14 15:01 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('future_fashion', '0002_empty_schema_default'), + ] + + operations = [ + migrations.AlterField( + model_name='brandrecognitionservice', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='clothingimagedownload', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='clothingtyperecognitionservice', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='imagefeatures', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + ] diff --git a/src/future_fashion/migrations/0004_on_delete.py b/src/future_fashion/migrations/0004_on_delete.py new file mode 100644 index 00000000..84509a07 --- /dev/null +++ b/src/future_fashion/migrations/0004_on_delete.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-20 17:29 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('future_fashion', '0003_alter_data_hash'), + ] + + operations = [ + migrations.AlterField( + model_name='clothingdatacommunity', + name='current_growth', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth'), + ), + migrations.AlterField( + model_name='clothingdatacommunity', + name='kernel_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType'), + ), + migrations.AlterField( + model_name='clothinginventorycommunity', + name='current_growth', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth'), + ), + migrations.AlterField( + model_name='clothinginventorycommunity', + name='kernel_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType'), + ), + migrations.AlterField( + model_name='collection', + name='community_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'), + ), + migrations.AlterField( + model_name='document', + name='community_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'), + ), + ] diff --git a/src/future_fashion/models/data.py b/src/future_fashion/models/data.py index 1835f764..d7fa8fef 100644 --- a/src/future_fashion/models/data.py +++ b/src/future_fashion/models/data.py @@ -123,7 +123,7 @@ class ClothingDataCommunity(Community): "product": "el.find(class_='title').get_text() if el.find(class_='title') else ''", "brand": "el.find(class_='brand').get_text() if el.find(class_='brand') else ''", }, - "_resource": "KledingListing", + "_resource": "sources.KledingListing", "_continuation_limit": 100, "_interval_duration": 1000 }, @@ -138,7 +138,7 @@ class ClothingDataCommunity(Community): "config": { "_args": ["$.image"], "_kwargs": {}, - "_resource": "ClothingImageDownload", + "_resource": "future_fashion.ClothingImageDownload", "_interval_duration": 100 }, "schema": {}, diff --git a/src/future_fashion/models/inventory.py b/src/future_fashion/models/inventory.py index c222ffb6..b73ee7e9 100644 --- a/src/future_fashion/models/inventory.py +++ b/src/future_fashion/models/inventory.py @@ -35,7 +35,7 @@ def documents(self): "config": { "_args": [], "_kwargs": {"image": "$.path"}, - "_resource": "ClothingTypeRecognitionService", + "_resource": "future_fashion.ClothingTypeRecognitionService", "_objective": { "@": "$", "path": "$.path", diff --git a/src/future_fashion/models/storage.py b/src/future_fashion/models/storage.py index 498222f6..d3e6b2a9 100644 --- a/src/future_fashion/models/storage.py +++ b/src/future_fashion/models/storage.py @@ -7,7 +7,7 @@ class Collection(DocumentCollectionMixin, CollectionBase): community = GenericForeignKey(ct_field="community_type", fk_field="community_id") - community_type = models.ForeignKey(ContentType, related_name="+") + community_type = models.ForeignKey(ContentType, related_name="+", on_delete=models.PROTECT) community_id = models.PositiveIntegerField() def init_document(self, data, collection=None): @@ -23,5 +23,5 @@ def init_document(self, data, collection=None): class Document(DocumentPostgres, DocumentBase): community = GenericForeignKey(ct_field="community_type", fk_field="community_id") - community_type = models.ForeignKey(ContentType, related_name="+") + community_type = models.ForeignKey(ContentType, related_name="+", on_delete=models.PROTECT) community_id = models.PositiveIntegerField() diff --git a/src/future_fashion/urls.py b/src/future_fashion/urls.py index 4c330c49..cfc7f9ba 100644 --- a/src/future_fashion/urls.py +++ b/src/future_fashion/urls.py @@ -8,6 +8,7 @@ from future_fashion import views +app_name = "future-fashion" urlpatterns = [ url( r'^inventory/service/(?P.+)/?$', diff --git a/src/nautilus/migrations/0001_squashed_0006_on_delete.py b/src/nautilus/migrations/0001_squashed_0006_on_delete.py new file mode 100644 index 00000000..2a86796a --- /dev/null +++ b/src/nautilus/migrations/0001_squashed_0006_on_delete.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-21 12:50 +from __future__ import unicode_literals + +import core.processors.mixins +import datagrowth.configuration.fields +from django.db import migrations, models +import django.db.models.deletion +import json_field.fields + + +class Migration(migrations.Migration): + + replaces = [('nautilus', '0001_initial'), ('nautilus', '0002_auto_20171119_1315'), ('nautilus', '0003_auto_20181202_1424'), ('nautilus', '0004_auto_20190402_0928'), ('nautilus', '0005_alter_data_hash'), ('nautilus', '0006_on_delete')] + + initial = True + + dependencies = [ + ('core', '0017_auto_20170315_1639'), + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.CreateModel( + name='LocaforaLogin', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='LocaforaOrderOverviewCommunity', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('signature', models.CharField(db_index=True, max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('kernel_id', models.PositiveIntegerField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('completed_at', models.DateTimeField(blank=True, null=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('state', models.CharField(choices=[('Aborted', 'Aborted'), ('Asynchronous', 'Asynchronous'), ('New', 'New'), ('Ready', 'Ready'), ('Retry', 'Retry'), ('Synchronous', 'Synchronous')], default='New', max_length=255)), + ('current_growth', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth')), + ('kernel_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType')), + ], + options={ + 'verbose_name': 'Locafora order overview community', + 'verbose_name_plural': 'Locafora order overview communities', + }, + bases=(models.Model, core.processors.mixins.ProcessorMixin), + ), + migrations.CreateModel( + name='LocaforaOrders', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'verbose_name': 'Locafora order', + 'verbose_name_plural': 'Locafora orders', + }, + ), + ] diff --git a/src/nautilus/migrations/0005_alter_data_hash.py b/src/nautilus/migrations/0005_alter_data_hash.py new file mode 100644 index 00000000..8fc56c48 --- /dev/null +++ b/src/nautilus/migrations/0005_alter_data_hash.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-14 15:01 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nautilus', '0004_auto_20190402_0928'), + ] + + operations = [ + migrations.AlterField( + model_name='locaforalogin', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='locaforaorders', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + ] diff --git a/src/nautilus/migrations/0006_on_delete.py b/src/nautilus/migrations/0006_on_delete.py new file mode 100644 index 00000000..ab135796 --- /dev/null +++ b/src/nautilus/migrations/0006_on_delete.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-20 17:29 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('nautilus', '0005_alter_data_hash'), + ] + + operations = [ + migrations.AlterField( + model_name='locaforaorderoverviewcommunity', + name='current_growth', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth'), + ), + migrations.AlterField( + model_name='locaforaorderoverviewcommunity', + name='kernel_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType'), + ), + ] diff --git a/src/nautilus/models/community.py b/src/nautilus/models/community.py index dc700f60..94fdfa47 100644 --- a/src/nautilus/models/community.py +++ b/src/nautilus/models/community.py @@ -20,7 +20,7 @@ class LocaforaOrderOverviewCommunity(Community): "config": { "_args": [], "_kwargs": {}, - "_resource": "LocaforaOrders", + "_resource": "nautilus.LocaforaOrders", "_objective": { "@": "$.data", "first_name": "$.payer_first_name", @@ -28,7 +28,7 @@ class LocaforaOrderOverviewCommunity(Community): "orders": "$.order_shop_products" }, "_auth": { - "resource": "LocaforaLogin", + "resource": "nautilus.LocaforaLogin", "credentials": { "email": "email@fakoberkers.nl", "password": getattr(settings, 'LOCAFORA_PASSWORD', '') diff --git a/src/online_discourse/migrations/0004_alter_data_hash.py b/src/online_discourse/migrations/0004_alter_data_hash.py new file mode 100644 index 00000000..56be4cc3 --- /dev/null +++ b/src/online_discourse/migrations/0004_alter_data_hash.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-14 15:01 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('online_discourse', '0003_elastic_search'), + ] + + operations = [ + migrations.AlterField( + model_name='complexityanalysis', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='webcontentdownload', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='webtextresource', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + ] diff --git a/src/online_discourse/migrations/0005_on_delete.py b/src/online_discourse/migrations/0005_on_delete.py new file mode 100644 index 00000000..88139dcd --- /dev/null +++ b/src/online_discourse/migrations/0005_on_delete.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-20 17:29 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('online_discourse', '0004_alter_data_hash'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='community_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'), + ), + migrations.AlterField( + model_name='discoursesearchcommunity', + name='current_growth', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth'), + ), + migrations.AlterField( + model_name='discoursesearchcommunity', + name='kernel_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType'), + ), + migrations.AlterField( + model_name='document', + name='community_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'), + ), + ] diff --git a/src/online_discourse/models/elastic.py b/src/online_discourse/models/elastic.py index eb0ff838..d9b74341 100644 --- a/src/online_discourse/models/elastic.py +++ b/src/online_discourse/models/elastic.py @@ -17,7 +17,7 @@ class ElasticIndex(models.Model): signature = models.CharField(max_length=255) language = models.CharField(max_length=5, choices=settings.ELASTIC_SEARCH_ANALYSERS.items()) - dataset = models.ForeignKey(DiscourseSearchCommunity, related_name="indices") + dataset = models.ForeignKey(DiscourseSearchCommunity, related_name="indices", on_delete=models.CASCADE) configuration = JSONField(blank=True) error_count = models.IntegerField(default=0) diff --git a/src/online_discourse/models/search.py b/src/online_discourse/models/search.py index d99cf506..2642bcb6 100644 --- a/src/online_discourse/models/search.py +++ b/src/online_discourse/models/search.py @@ -43,7 +43,7 @@ class DiscourseSearchCommunity(CommunityCollectionDocumentMixin, Community): "config": { "_args": ["$.query", "$.quantity"], "_kwargs": {}, - "_resource": "GoogleText", + "_resource": "sources.GoogleText", "_objective": { "@": "$.items", "#term": "$.queries.request.0.searchTerms", @@ -64,7 +64,7 @@ class DiscourseSearchCommunity(CommunityCollectionDocumentMixin, Community): "config": { "_args": ["$.url"], "_kwargs": {}, - "_resource": "WebContentDownload", + "_resource": "sources.WebContentDownload", "_update_key": "url", "$language": "en", # TODO: think of a way to include language as a parameter without using config # Keys below are used by WebContentDownload to format its output @@ -82,7 +82,7 @@ class DiscourseSearchCommunity(CommunityCollectionDocumentMixin, Community): "config": { "_args": ["$.home"], "_kwargs": {}, - "_resource": "WebContentDownload", + "_resource": "sources.WebContentDownload", "_update_key": "home", # Keys below are used by WebContentDownload to format its output "url_key": "home", @@ -100,7 +100,7 @@ class DiscourseSearchCommunity(CommunityCollectionDocumentMixin, Community): "config": { "_args": ["$.resourcePath"], "_kwargs": {}, - "_resource": "WebTextTikaResource", + "_resource": "online_discourse.WebTextTikaResource", "_objective": { "#resourcePath": "$.resourcePath", "#title": "$.title", diff --git a/src/online_discourse/models/sources/wizenoze.py b/src/online_discourse/models/sources/wizenoze.py index e861f9e0..f8639d44 100644 --- a/src/online_discourse/models/sources/wizenoze.py +++ b/src/online_discourse/models/sources/wizenoze.py @@ -30,7 +30,7 @@ class ComplexityAnalysis(HttpResource): "url": "$.url", "languageCode": "en" }, - "_resource": "ComplexityAnalysis", + "_resource": "online_discourse.ComplexityAnalysis", "_objective": { "@": "$", "url": "$.metadata.url", diff --git a/src/online_discourse/models/storage.py b/src/online_discourse/models/storage.py index d9eba915..504b4503 100644 --- a/src/online_discourse/models/storage.py +++ b/src/online_discourse/models/storage.py @@ -7,7 +7,7 @@ class Collection(DocumentCollectionMixin, CollectionBase): community = GenericForeignKey(ct_field="community_type", fk_field="community_id") - community_type = models.ForeignKey(ContentType, related_name="+") + community_type = models.ForeignKey(ContentType, related_name="+", on_delete=models.PROTECT) community_id = models.PositiveIntegerField() def init_document(self, data, collection=None): @@ -23,7 +23,7 @@ def init_document(self, data, collection=None): class Document(DocumentPostgres, DocumentBase): community = GenericForeignKey(ct_field="community_type", fk_field="community_id") - community_type = models.ForeignKey(ContentType, related_name="+") + community_type = models.ForeignKey(ContentType, related_name="+", on_delete=models.PROTECT) community_id = models.PositiveIntegerField() def to_search(self): diff --git a/src/online_discourse/urls.py b/src/online_discourse/urls.py index 4d3b9995..c3184d95 100644 --- a/src/online_discourse/urls.py +++ b/src/online_discourse/urls.py @@ -6,6 +6,7 @@ from online_discourse import views +app_name = "online-discourse" urlpatterns = [ url( r'^service/(?P.+)/$', @@ -35,6 +36,6 @@ router.register( 'discourses', views.DiscourseViewSet, - base_name=DiscourseSearchCommunity.get_name() + basename=DiscourseSearchCommunity.get_name() ) urlpatterns += router.urls diff --git a/src/open_data/migrations/0001_squashed_0007_on_delete.py b/src/open_data/migrations/0001_squashed_0007_on_delete.py new file mode 100644 index 00000000..ba283b57 --- /dev/null +++ b/src/open_data/migrations/0001_squashed_0007_on_delete.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-21 12:51 +from __future__ import unicode_literals + +import core.processors.mixins +import datagrowth.configuration.fields +from django.db import migrations, models +import django.db.models.deletion +import json_field.fields + + +class Migration(migrations.Migration): + + replaces = [('open_data', '0001_initial'), ('open_data', '0002_auto_20170320_1518'), ('open_data', '0003_officialannouncementsdocumentnetherlands_officialannouncementsnetherlands'), ('open_data', '0004_auto_20181202_1424'), ('open_data', '0005_auto_20190402_0928'), ('open_data', '0006_alter_data_hash'), ('open_data', '0007_on_delete')] + + initial = True + + dependencies = [ + ('core', '0017_auto_20170315_1639'), + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.CreateModel( + name='DutchParliamentarySeatingTranscriptsCommunity', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('signature', models.CharField(db_index=True, max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('kernel_id', models.PositiveIntegerField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('completed_at', models.DateTimeField(blank=True, null=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('state', models.CharField(choices=[('Aborted', 'Aborted'), ('Asynchronous', 'Asynchronous'), ('New', 'New'), ('Ready', 'Ready'), ('Retry', 'Retry'), ('Synchronous', 'Synchronous')], default='New', max_length=255)), + ('current_growth', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth')), + ('kernel_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType')), + ], + options={ + 'verbose_name_plural': 'Dutch parliamentary seating transcripts', + 'verbose_name': 'Dutch parliamentary seating transcripts', + }, + bases=(models.Model, core.processors.mixins.ProcessorMixin), + ), + migrations.CreateModel( + name='OfficialAnnouncementsDocumentNetherlands', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('status', models.PositiveIntegerField(default=0)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'verbose_name': 'Official announcements document (Dutch)', + 'verbose_name_plural': 'Official announcements document (Dutch)', + }, + ), + migrations.CreateModel( + name='OfficialAnnouncementsNetherlands', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('status', models.PositiveIntegerField(default=0)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'verbose_name': 'Official announcements (Dutch)', + 'verbose_name_plural': 'Official announcements (Dutch)', + }, + ), + ] diff --git a/src/open_data/migrations/0006_alter_data_hash.py b/src/open_data/migrations/0006_alter_data_hash.py new file mode 100644 index 00000000..5e0057c9 --- /dev/null +++ b/src/open_data/migrations/0006_alter_data_hash.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-14 15:01 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('open_data', '0005_auto_20190402_0928'), + ] + + operations = [ + migrations.AlterField( + model_name='officialannouncementsdocumentnetherlands', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='officialannouncementsnetherlands', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + ] diff --git a/src/open_data/migrations/0007_on_delete.py b/src/open_data/migrations/0007_on_delete.py new file mode 100644 index 00000000..d27346d4 --- /dev/null +++ b/src/open_data/migrations/0007_on_delete.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-20 17:29 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('open_data', '0006_alter_data_hash'), + ] + + operations = [ + migrations.AlterField( + model_name='dutchparliamentaryseatingtranscriptscommunity', + name='current_growth', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth'), + ), + migrations.AlterField( + model_name='dutchparliamentaryseatingtranscriptscommunity', + name='kernel_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType'), + ), + ] diff --git a/src/open_data/models/dutch_parliament/community.py b/src/open_data/models/dutch_parliament/community.py index f87d18b4..a7ebd769 100644 --- a/src/open_data/models/dutch_parliament/community.py +++ b/src/open_data/models/dutch_parliament/community.py @@ -15,7 +15,7 @@ class DutchParliamentarySeatingTranscriptsCommunity(Community): "config": { "_args": ["$.start_date", "$.end_date"], "_kwargs": {}, - "_resource": "OfficialAnnouncementsNetherlands", + "_resource": "open_data.OfficialAnnouncementsNetherlands", "_objective": { "@": "soup.select('.lijst a')", "link": "el.get('href')", @@ -35,7 +35,7 @@ class DutchParliamentarySeatingTranscriptsCommunity(Community): "config": { "_args": ["$.link"], "_kwargs": {}, - "_resource": "OfficialAnnouncementsDocumentNetherlands", + "_resource": "open_data.OfficialAnnouncementsDocumentNetherlands", "_objective": { "@": "soup.find_all(class_='spreekbeurt')", "#seating": "{key: value for key, value in zip(['year', 'number', 'date', 'published_at'], soup.find(class_='nummer').stripped_strings)}", diff --git a/src/package.json b/src/package.json index 4840bea9..5c8d6cfc 100644 --- a/src/package.json +++ b/src/package.json @@ -1,6 +1,6 @@ { "name": "datascope", - "version": "0.16.6", + "version": "0.17.1", "data": { "digital-ocean": { "data/online_discourse/dumps": "online_discourse/dumps" diff --git a/src/setup_utrecht/migrations/0007_alter_data_hash.py b/src/setup_utrecht/migrations/0007_alter_data_hash.py new file mode 100644 index 00000000..3bce021b --- /dev/null +++ b/src/setup_utrecht/migrations/0007_alter_data_hash.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-14 15:01 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('setup_utrecht', '0006_auto_20190402_0928'), + ] + + operations = [ + migrations.AlterField( + model_name='redditimagedownload', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='redditlist', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='redditpermalink', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='uniformimagedownload', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + ] diff --git a/src/setup_utrecht/migrations/0008_on_delete.py b/src/setup_utrecht/migrations/0008_on_delete.py new file mode 100644 index 00000000..9eb7b334 --- /dev/null +++ b/src/setup_utrecht/migrations/0008_on_delete.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-20 17:29 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('setup_utrecht', '0007_alter_data_hash'), + ] + + operations = [ + migrations.AlterField( + model_name='redditscrapecommunity', + name='current_growth', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth'), + ), + migrations.AlterField( + model_name='redditscrapecommunity', + name='kernel_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType'), + ), + migrations.AlterField( + model_name='uniformimagescommunity', + name='current_growth', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth'), + ), + migrations.AlterField( + model_name='uniformimagescommunity', + name='kernel_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType'), + ), + ] diff --git a/src/setup_utrecht/models/i48_trolls.py b/src/setup_utrecht/models/i48_trolls.py index 2de4902b..3dcc96ab 100644 --- a/src/setup_utrecht/models/i48_trolls.py +++ b/src/setup_utrecht/models/i48_trolls.py @@ -14,7 +14,7 @@ class RedditScrapeCommunity(Community): "config": { "_args": ["$.subreddit"], "_kwargs": {}, - "_resource": "RedditList", + "_resource": "setup_utrecht.RedditList", "_objective": { "@": "soup.find_all(id=lambda el_id: el_id and el_id.startswith('thing_'))", "id": "el.get('id').split('_')[-1]", @@ -38,7 +38,7 @@ class RedditScrapeCommunity(Community): "config": { "_args": ["$.details_link"], "_kwargs": {}, - "_resource": "RedditPermalink", + "_resource": "setup_utrecht.RedditPermalink", "_objective": { "@": "soup.find_all(id=lambda el_id: el_id and el_id.startswith('media-preview-'))", "id": "el.get('id').split('-')[-1]", @@ -58,7 +58,7 @@ class RedditScrapeCommunity(Community): "config": { "_args": ["$.details_link"], "_kwargs": {}, - "_resource": "RedditPermalink", + "_resource": "setup_utrecht.RedditPermalink", "_objective": { "@": "soup.find_all(id=lambda el_id: el_id and el_id.startswith('thing_'))[1:]", "id": "el.get('id').split('_')[-1]", @@ -79,7 +79,7 @@ class RedditScrapeCommunity(Community): "config": { "_args": ["$.media_preview"], "_kwargs": {}, - "_resource": "RedditImageDownload", + "_resource": "setup_utrecht.RedditImageDownload", "_interval_duration": 2000, }, "schema": {}, diff --git a/src/setup_utrecht/models/womr.py b/src/setup_utrecht/models/womr.py index 73481f17..58ed5b25 100644 --- a/src/setup_utrecht/models/womr.py +++ b/src/setup_utrecht/models/womr.py @@ -19,7 +19,7 @@ class UniformImagesCommunity(Community): "config": { "_args": ["$.word", "$.country", "$.image_quantity"], "_kwargs": {}, - "_resource": "GoogleImage", + "_resource": "sources.GoogleImage", "_objective": { "@": "$.items", "#word": "$.queries.request.0.searchTerms", @@ -42,7 +42,7 @@ class UniformImagesCommunity(Community): "config": { "_args": ["$.url", "$.word"], "_kwargs": {}, - "_resource": "UniformImageDownload" + "_resource": "setup_utrecht.UniformImageDownload" }, "schema": {}, "errors": None, diff --git a/src/sources/migrations/0001_squashed_0029_alter_data_hash.py b/src/sources/migrations/0001_squashed_0029_alter_data_hash.py new file mode 100644 index 00000000..d1129e4f --- /dev/null +++ b/src/sources/migrations/0001_squashed_0029_alter_data_hash.py @@ -0,0 +1,417 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-21 12:52 +from __future__ import unicode_literals + +import datagrowth.configuration.fields +from django.db import migrations, models +import django.db.models.deletion +import json_field.fields +import sources.models.wikipedia.mixins + + +class Migration(migrations.Migration): + + replaces = [('sources', '0001_initial'), ('sources', '0002_wikipediasearch'), ('sources', '0003_wikipediarecentchanges'), ('sources', '0004_wikipedialistpages'), ('sources', '0005_wikipediatranslate'), ('sources', '0006_googleimage'), ('sources', '0007_googletranslate'), ('sources', '0008_imagedownload'), ('sources', '0009_auto_20150828_1659'), ('sources', '0010_auto_20160225_2307'), ('sources', '0011_wikidataitems'), ('sources', '0012_auto_20160228_2019'), ('sources', '0013_imagefeatures'), ('sources', '0014_wikipediacategories_wikipediacategorymembers'), ('sources', '0015_wikipediapageviewdetails'), ('sources', '0016_wikipediatransclusions'), ('sources', '0017_wikipediarevisions'), ('sources', '0018_wikipediaedit_wikipedialogin_wikipediatoken'), ('sources', '0019_officialannouncementsdocumentnetherlands_officialannouncementsnetherlands'), ('sources', '0020_auto_20170428_1123'), ('sources', '0021_kledinglisting'), ('sources', '0022_googletext'), ('sources', '0023_auto_20171003_1235'), ('sources', '0024_auto_20171119_1315'), ('sources', '0025_auto_20180818_1844'), ('sources', '0026_auto_20180818_1854'), ('sources', '0027_auto_20180912_0931'), ('sources', '0028_auto_20181202_1424'), ('sources', '0029_alter_data_hash')] + + initial = True + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.CreateModel( + name='WikipediaSearch', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + 'verbose_name_plural': 'Wikipedia searches', + 'verbose_name': 'Wikipedia search', + }, + ), + migrations.CreateModel( + name='WikipediaRecentChanges', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + 'verbose_name': 'Wikipedia recent changes', + 'verbose_name_plural': 'Wikipedia recent changes', + }, + ), + migrations.CreateModel( + name='WikipediaListPages', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + 'verbose_name': 'Wikipedia list pages', + 'verbose_name_plural': 'Wikipedia list pages', + }, + bases=(models.Model, sources.models.wikipedia.mixins.WikipediaImagesMixin), + ), + migrations.CreateModel( + name='WikipediaTranslate', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='GoogleImage', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='GoogleTranslate', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='ImageDownload', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='WikiDataItems', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + 'verbose_name': 'Wikidata items', + 'verbose_name_plural': 'Wikidata items', + }, + ), + migrations.CreateModel( + name='WikipediaCategories', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + 'verbose_name_plural': 'Wikipedia categories', + 'verbose_name': 'Wikipedia category', + }, + ), + migrations.CreateModel( + name='WikipediaCategoryMembers', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + 'verbose_name_plural': 'Wikipedia category members', + 'verbose_name': 'Wikipedia category members', + }, + ), + migrations.CreateModel( + name='WikipediaPageviewDetails', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='WikipediaTransclusions', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + 'verbose_name': 'Wikipedia transclusions', + 'verbose_name_plural': 'Wikipedia transclusions', + }, + ), + migrations.CreateModel( + name='WikipediaRevisions', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'verbose_name': 'Wikipedia revisions', + 'verbose_name_plural': 'Wikipedia revisions', + }, + ), + migrations.CreateModel( + name='WikipediaEdit', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='WikipediaLogin', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='WikipediaToken', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='KledingListing', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='GoogleText', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('status', models.PositiveIntegerField(default=0)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/src/sources/migrations/0029_alter_data_hash.py b/src/sources/migrations/0029_alter_data_hash.py new file mode 100644 index 00000000..f6671088 --- /dev/null +++ b/src/sources/migrations/0029_alter_data_hash.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-14 15:01 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sources', '0028_auto_20181202_1424'), + ] + + operations = [ + migrations.AlterField( + model_name='googleimage', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='googletext', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='googletranslate', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='imagedownload', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='kledinglisting', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='wikidataitems', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='wikipediacategories', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='wikipediacategorymembers', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='wikipediaedit', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='wikipedialistpages', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='wikipedialogin', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='wikipediapageviewdetails', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='wikipediarecentchanges', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='wikipediarevisions', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='wikipediasearch', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='wikipediatoken', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='wikipediatransclusions', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='wikipediatranslate', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + ] diff --git a/src/visual_translations/migrations/0001_squashed_0010_on_delete.py b/src/visual_translations/migrations/0001_squashed_0010_on_delete.py new file mode 100644 index 00000000..f5226984 --- /dev/null +++ b/src/visual_translations/migrations/0001_squashed_0010_on_delete.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-21 12:53 +from __future__ import unicode_literals + +import core.processors.mixins +import datagrowth.configuration.fields +from django.db import migrations, models +import django.db.models.deletion +import json_field.fields + + +class Migration(migrations.Migration): + + replaces = [('visual_translations', '0001_initial'), ('visual_translations', '0002_auto_20160219_1442'), ('visual_translations', '0003_auto_20160225_2307'), ('visual_translations', '0004_auto_20160328_1112'), ('visual_translations', '0005_visualtranslationsbriccommunity'), ('visual_translations', '0006_auto_20160417_2047'), ('visual_translations', '0007_webimagedownload'), ('visual_translations', '0008_auto_20190402_0928'), ('visual_translations', '0009_alter_data_hash'), ('visual_translations', '0010_on_delete')] + + initial = True + + dependencies = [ + ('core', '0004_communitymock'), + ('contenttypes', '0002_remove_content_type_name'), + ('core', '0014_auto_20160226_1306'), + ] + + operations = [ + migrations.CreateModel( + name='VisualTranslationsCommunity', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('signature', models.CharField(db_index=True, max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('kernel_id', models.PositiveIntegerField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('completed_at', models.DateTimeField(blank=True, null=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('views', models.IntegerField(default=0)), + ('state', models.CharField(choices=[('Aborted', 'Aborted'), ('Asynchronous', 'Asynchronous'), ('New', 'New'), ('Ready', 'Ready'), ('Synchronous', 'Synchronous')], default='New', max_length=255)), + ('current_growth', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='core.Growth')), + ('kernel_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'verbose_name': 'Visual translation', + 'verbose_name_plural': 'Visual translations', + }, + bases=(models.Model, core.processors.mixins.ProcessorMixin), + ), + migrations.RunSQL( + sql="UPDATE django_content_type\n SET model = 'visualtranslationseucommunity'\n WHERE model = 'visualtranslationscommunity' AND\n app_label = 'visual_translations';", + reverse_sql="UPDATE django_content_type\n SET model = 'visualtranslationscommunity'\n WHERE model = 'visualtranslationseucommunity' AND\n app_label = 'visual_translations';", + ), + migrations.RenameModel( + old_name='VisualTranslationsCommunity', + new_name='VisualTranslationsEUCommunity', + ), + migrations.CreateModel( + name='VisualTranslationsBRICCommunity', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('signature', models.CharField(db_index=True, max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('kernel_id', models.PositiveIntegerField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('completed_at', models.DateTimeField(blank=True, null=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('state', models.CharField(choices=[('Aborted', 'Aborted'), ('Asynchronous', 'Asynchronous'), ('New', 'New'), ('Ready', 'Ready'), ('Retry', 'Retry'), ('Synchronous', 'Synchronous')], default='New', max_length=255)), + ('current_growth', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth')), + ('kernel_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType')), + ], + options={ + 'verbose_name_plural': 'Visual translations (BRIC)', + 'verbose_name': 'Visual translation (BRIC)', + }, + bases=(models.Model, core.processors.mixins.ProcessorMixin), + ), + migrations.AlterModelOptions( + name='visualtranslationseucommunity', + options={'verbose_name': 'Visual translation (EU)', 'verbose_name_plural': 'Visual translations (EU)'}, + ), + migrations.CreateModel( + name='WebImageDownload', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uri', models.CharField(db_index=True, default=None, max_length=255)), + ('status', models.PositiveIntegerField(default=0)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('retainer_id', models.PositiveIntegerField(blank=True, null=True)), + ('data_hash', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('request', json_field.fields.JSONField(default=None, help_text='Enter a valid JSON object')), + ('head', json_field.fields.JSONField(default='{}', help_text='Enter a valid JSON object')), + ('body', models.TextField(blank=True, default=None, null=True)), + ('retainer_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + }, + ), + migrations.RemoveField( + model_name='visualtranslationseucommunity', + name='views', + ), + migrations.AlterField( + model_name='visualtranslationseucommunity', + name='state', + field=models.CharField(choices=[('Aborted', 'Aborted'), ('Asynchronous', 'Asynchronous'), ('New', 'New'), ('Ready', 'Ready'), ('Retry', 'Retry'), ('Synchronous', 'Synchronous')], default='New', max_length=255), + ), + migrations.AlterField( + model_name='visualtranslationseucommunity', + name='current_growth', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth'), + ), + migrations.AlterField( + model_name='visualtranslationseucommunity', + name='kernel_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType'), + ), + ] diff --git a/src/visual_translations/migrations/0009_alter_data_hash.py b/src/visual_translations/migrations/0009_alter_data_hash.py new file mode 100644 index 00000000..a6f91cf5 --- /dev/null +++ b/src/visual_translations/migrations/0009_alter_data_hash.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-14 15:01 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('visual_translations', '0008_auto_20190402_0928'), + ] + + operations = [ + migrations.AlterField( + model_name='webimagedownload', + name='data_hash', + field=models.CharField(blank=True, db_index=True, default='', max_length=255), + ), + ] diff --git a/src/visual_translations/migrations/0010_on_delete.py b/src/visual_translations/migrations/0010_on_delete.py new file mode 100644 index 00000000..0322bf7c --- /dev/null +++ b/src/visual_translations/migrations/0010_on_delete.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-20 17:29 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('visual_translations', '0009_alter_data_hash'), + ] + + operations = [ + migrations.AlterField( + model_name='visualtranslationsbriccommunity', + name='current_growth', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth'), + ), + migrations.AlterField( + model_name='visualtranslationsbriccommunity', + name='kernel_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType'), + ), + migrations.AlterField( + model_name='visualtranslationseucommunity', + name='current_growth', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth'), + ), + migrations.AlterField( + model_name='visualtranslationseucommunity', + name='kernel_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType'), + ), + ] diff --git a/src/visual_translations/models/bric.py b/src/visual_translations/models/bric.py index 9caed71c..30928c4e 100644 --- a/src/visual_translations/models/bric.py +++ b/src/visual_translations/models/bric.py @@ -22,7 +22,7 @@ class VisualTranslationsBRICCommunity(Community): "config": { "_args": ["$.query", "$.translate_to"], "_kwargs": {}, - "_resource": "WikipediaTranslate", + "_resource": "sources.WikipediaTranslate", "_objective": { "@": "$.page.iwlinks", "language": "$.prefix", @@ -43,7 +43,7 @@ class VisualTranslationsBRICCommunity(Community): "config": { "_args": ["$.word", "$.country"], "_kwargs": {}, - "_resource": "GoogleImage", + "_resource": "sources.GoogleImage", "_objective": { "@": "$.items", "#word": "$.queries.request.0.searchTerms", diff --git a/src/visual_translations/models/eu.py b/src/visual_translations/models/eu.py index 5adc9189..79c5d362 100644 --- a/src/visual_translations/models/eu.py +++ b/src/visual_translations/models/eu.py @@ -26,7 +26,7 @@ class VisualTranslationsEUCommunity(Community): "config": { "_args": ["$.translate_from", "$.translate_to", "$.query"], "_kwargs": {}, - "_resource": "GoogleTranslate", + "_resource": "sources.GoogleTranslate", "_objective": { "@": "$", "language": "$.language", @@ -47,7 +47,7 @@ class VisualTranslationsEUCommunity(Community): "config": { "_args": ["$.word", "$.country", "$.images_quantity"], "_kwargs": {}, - "_resource": "GoogleImage", + "_resource": "sources.GoogleImage", "_objective": { "@": "$.items", "#word": "$.queries.request.0.searchTerms", @@ -71,7 +71,7 @@ class VisualTranslationsEUCommunity(Community): "config": { "_args": ["$.url"], "_kwargs": {}, - "_resource": "WebImageDownload" + "_resource": "visual_translations.WebImageDownload" }, "schema": {}, "errors": None, diff --git a/src/wiki_feed/fixtures/breaking-news.json b/src/wiki_feed/fixtures/breaking-news.json index 1db6b586..cb06b7ce 100644 --- a/src/wiki_feed/fixtures/breaking-news.json +++ b/src/wiki_feed/fixtures/breaking-news.json @@ -5,7 +5,7 @@ "completed_at": null, "purge_at": null, - "config": "{\"async\": false, \"start_time\": 1509580800.0, \"end_time\": 1509667200.0}", + "config": "{\"asynchronous\": false, \"start_time\": 1509580800.0, \"end_time\": 1509667200.0}", "current_growth": 1, "created_at": "2017-11-03T01:26:03", "signature": "recent_changes", @@ -31,7 +31,7 @@ "process": "HttpResourceProcessor.fetch_mass", "type": "wikidata", "state": "Complete", - "config": "{\"user_agent\": \"WikiFeedBot (DataScope v0.4.0)\", \"_inline_key\": \"wikidata\", \"end_time\": 1509667200.0, \"_continuation_limit\": 1000, \"start_time\": 1509580800.0, \"_objective\": {\"@\": \"$\", \"wikidata\": \"$.id\", \"claims\": \"$.claims\", \"references\": \"$.references\", \"description\": \"$.description\"}, \"_resource\": \"WikiDataItems\", \"_concat_args_size\": 50}", + "config": "{\"user_agent\": \"WikiFeedBot (DataScope v0.4.0)\", \"_inline_key\": \"wikidata\", \"end_time\": 1509667200.0, \"_continuation_limit\": 1000, \"start_time\": 1509580800.0, \"_objective\": {\"@\": \"$\", \"wikidata\": \"$.id\", \"claims\": \"$.claims\", \"references\": \"$.references\", \"description\": \"$.description\"}, \"_resource\": \"sources.WikiDataItems\", \"_concat_args_size\": 50}", "output_type": [ "core", "collective" diff --git a/src/wiki_feed/fixtures/features.json b/src/wiki_feed/fixtures/features.json index 78f9ca98..c311efb7 100644 --- a/src/wiki_feed/fixtures/features.json +++ b/src/wiki_feed/fixtures/features.json @@ -5,7 +5,7 @@ "completed_at": null, "purge_at": null, - "config": "{\"async\": false, \"start_time\": 1509580800.0, \"end_time\": 1509667200.0}", + "config": "{\"asynchronous\": false, \"start_time\": 1509580800.0, \"end_time\": 1509667200.0}", "current_growth": 1, "created_at": "2017-11-03T01:26:03", "signature": "recent_changes", @@ -31,7 +31,7 @@ "process": "HttpResourceProcessor.fetch_mass", "type": "wikidata", "state": "Complete", - "config": "{\"user_agent\": \"WikiFeedBot (DataScope v0.4.0)\", \"_inline_key\": \"wikidata\", \"end_time\": 1509667200.0, \"_continuation_limit\": 1000, \"start_time\": 1509580800.0, \"_objective\": {\"@\": \"$\", \"wikidata\": \"$.id\", \"claims\": \"$.claims\", \"references\": \"$.references\", \"description\": \"$.description\"}, \"_resource\": \"WikiDataItems\", \"_concat_args_size\": 50}", + "config": "{\"user_agent\": \"WikiFeedBot (DataScope v0.4.0)\", \"_inline_key\": \"wikidata\", \"end_time\": 1509667200.0, \"_continuation_limit\": 1000, \"start_time\": 1509580800.0, \"_objective\": {\"@\": \"$\", \"wikidata\": \"$.id\", \"claims\": \"$.claims\", \"references\": \"$.references\", \"description\": \"$.description\"}, \"_resource\": \"sources.WikiDataItems\", \"_concat_args_size\": 50}", "output_type": [ "core", "collective" diff --git a/src/wiki_feed/migrations/0001_squashed_0005_on_delete.py b/src/wiki_feed/migrations/0001_squashed_0005_on_delete.py new file mode 100644 index 00000000..d45edbf7 --- /dev/null +++ b/src/wiki_feed/migrations/0001_squashed_0005_on_delete.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-21 12:53 +from __future__ import unicode_literals + +import core.processors.mixins +import datagrowth.configuration.fields +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + replaces = [('wiki_feed', '0001_initial'), ('wiki_feed', '0002_auto_20170201_1741'), ('wiki_feed', '0003_wikifeedpublishcommunity'), ('wiki_feed', '0004_auto_20190402_0928'), ('wiki_feed', '0005_on_delete')] + + initial = True + + dependencies = [ + ('core', '0017_auto_20170315_1639'), + ('core', '0016_auto_20160627_2118'), + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.CreateModel( + name='WikiFeedCommunity', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('signature', models.CharField(db_index=True, max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('kernel_id', models.PositiveIntegerField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('completed_at', models.DateTimeField(blank=True, null=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('state', models.CharField(choices=[('Aborted', 'Aborted'), ('Asynchronous', 'Asynchronous'), ('New', 'New'), ('Ready', 'Ready'), ('Retry', 'Retry'), ('Synchronous', 'Synchronous')], default='New', max_length=255)), + ('current_growth', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth')), + ('kernel_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType')), + ], + options={ + 'verbose_name': 'Wiki feed', + 'verbose_name_plural': 'Wiki feeds', + }, + bases=(models.Model, core.processors.mixins.ProcessorMixin), + ), + migrations.CreateModel( + name='WikiFeedPublishCommunity', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('signature', models.CharField(db_index=True, max_length=255)), + ('config', datagrowth.configuration.fields.ConfigurationField()), + ('kernel_id', models.PositiveIntegerField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('completed_at', models.DateTimeField(blank=True, null=True)), + ('purge_at', models.DateTimeField(blank=True, null=True)), + ('state', models.CharField(choices=[('Aborted', 'Aborted'), ('Asynchronous', 'Asynchronous'), ('New', 'New'), ('Ready', 'Ready'), ('Retry', 'Retry'), ('Synchronous', 'Synchronous')], default='New', max_length=255)), + ('current_growth', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth')), + ('kernel_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType')), + ], + options={ + 'verbose_name': 'Wiki feed publication', + 'verbose_name_plural': 'Wiki feed publications', + }, + bases=(models.Model, core.processors.mixins.ProcessorMixin), + ), + ] diff --git a/src/wiki_feed/migrations/0005_on_delete.py b/src/wiki_feed/migrations/0005_on_delete.py new file mode 100644 index 00000000..32ceeb10 --- /dev/null +++ b/src/wiki_feed/migrations/0005_on_delete.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-20 17:29 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('wiki_feed', '0004_auto_20190402_0928'), + ] + + operations = [ + migrations.AlterField( + model_name='wikifeedcommunity', + name='current_growth', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth'), + ), + migrations.AlterField( + model_name='wikifeedcommunity', + name='kernel_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType'), + ), + migrations.AlterField( + model_name='wikifeedpublishcommunity', + name='current_growth', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth'), + ), + migrations.AlterField( + model_name='wikifeedpublishcommunity', + name='kernel_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType'), + ), + ] diff --git a/src/wiki_feed/models.py b/src/wiki_feed/models.py index fb425460..236d00fe 100644 --- a/src/wiki_feed/models.py +++ b/src/wiki_feed/models.py @@ -28,7 +28,7 @@ class WikiFeedCommunity(Community): "config": { "_args": [], "_kwargs": {}, - "_resource": "WikipediaRecentChanges", + "_resource": "sources.WikipediaRecentChanges", "_objective": { "@": "$.query.recentchanges", "pageid": "$.pageid", @@ -51,7 +51,7 @@ class WikiFeedCommunity(Community): "config": { "_args": ["$.pageid"], "_kwargs": {}, - "_resource": "WikipediaListPages", + "_resource": "sources.WikipediaListPages", "_objective": { "@": "$.query.pages", "pageid": "$.pageid", @@ -76,7 +76,7 @@ class WikiFeedCommunity(Community): "config": { "_args": ["$.wikidata"], "_kwargs": {}, - "_resource": "WikiDataItems", + "_resource": "sources.WikiDataItems", "_objective": { "@": "$", "wikidata": "$.id", @@ -100,7 +100,7 @@ class WikiFeedCommunity(Community): # "config": { # "_args": ["$.title"], # "_kwargs": {}, - # "_resource": "WikipediaPageviewDetails", + # "_resource": "sources.WikipediaPageviewDetails", # "_objective": { # "@": "$.items", # "title": "$.article", @@ -274,7 +274,7 @@ class WikiFeedPublishCommunity(Community): "config": { "_args": ["$.feed_page"], "_kwargs": {}, - "_resource": "WikipediaTransclusions", + "_resource": "sources.WikipediaTransclusions", "_objective": { "@": "$.query.pages", "title": "$.title" @@ -294,7 +294,7 @@ class WikiFeedPublishCommunity(Community): "config": { "_args": ["$.title"], "_kwargs": {}, - "_resource": "WikipediaRevisions", + "_resource": "sources.WikipediaRevisions", "_objective": { "@": "$.page.revisions", "#namespace": "$.page.ns", @@ -320,7 +320,7 @@ class WikiFeedPublishCommunity(Community): "_kwargs": "$.feed.modules", "_community": "WikiFeedCommunity", "_update_key": "service", - "_resource": "Manifestation" + "_resource": "core.Manifestation" }, "schema": {}, "errors": {}, @@ -336,7 +336,7 @@ class WikiFeedPublishCommunity(Community): "title": "$.title", "text": "$.text" }, - "_resource": "WikipediaEdit", + "_resource": "sources.WikipediaEdit", "_username": getattr(settings, 'WIKI_USER', ''), "_password": getattr(settings, 'WIKI_PASSWORD', '') }, diff --git a/src/wiki_feed/views.py b/src/wiki_feed/views.py index 19806624..7a8dd2d2 100644 --- a/src/wiki_feed/views.py +++ b/src/wiki_feed/views.py @@ -1,6 +1,6 @@ from django.shortcuts import redirect from django.template.response import SimpleTemplateResponse -from django.core.urlresolvers import reverse +from django.urls import reverse from django.conf import settings from rest_framework import status diff --git a/src/wiki_scope/migrations/0003_on_delete.py b/src/wiki_scope/migrations/0003_on_delete.py new file mode 100644 index 00000000..bc0bb4f9 --- /dev/null +++ b/src/wiki_scope/migrations/0003_on_delete.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-20 17:29 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('wiki_scope', '0002_auto_20190402_0928'), + ] + + operations = [ + migrations.AlterField( + model_name='wikipediacategorysimilaritycommunity', + name='current_growth', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Growth'), + ), + migrations.AlterField( + model_name='wikipediacategorysimilaritycommunity', + name='kernel_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='contenttypes.ContentType'), + ), + ] diff --git a/src/wiki_scope/models/category_similarity.py b/src/wiki_scope/models/category_similarity.py index ba925cc7..2a6c2f29 100644 --- a/src/wiki_scope/models/category_similarity.py +++ b/src/wiki_scope/models/category_similarity.py @@ -17,7 +17,7 @@ class WikipediaCategorySimilarityCommunity(Community): "config": { "_args": ["$.title"], "_kwargs": {}, - "_resource": "WikipediaSearch", + "_resource": "sources.WikipediaSearch", "_objective": { "@": "$.query.pages", "pageid": "$.pageid", @@ -42,7 +42,7 @@ class WikipediaCategorySimilarityCommunity(Community): "config": { "_args": ["$.title"], "_kwargs": {}, - "_resource": "WikipediaCategories", + "_resource": "sources.WikipediaCategories", "_objective": { "@": "$.query.pages", "pageid": "$.pageid", @@ -62,7 +62,7 @@ class WikipediaCategorySimilarityCommunity(Community): "config": { "_args": ["$.title"], "_kwargs": {}, - "_resource": "WikipediaCategoryMembers", + "_resource": "sources.WikipediaCategoryMembers", "_objective": { "@": "$.query.pages", "pageid": "$.pageid", diff --git a/src/wiki_scope/models/location.py b/src/wiki_scope/models/location.py index 5ceba414..60009b96 100644 --- a/src/wiki_scope/models/location.py +++ b/src/wiki_scope/models/location.py @@ -9,7 +9,7 @@ class CityCelebrities(object): ("radius", { "process": "HttpResourceProcessor.fetch", "config": { - "_resource": "WikiGeoLocation", + "_resource": "sources.WikiGeoLocation", "_objective": {} }, "success": "ExtractProcessor.extract_resource", @@ -21,7 +21,7 @@ class CityCelebrities(object): ("backlinks", { "process": "HttpResourceProcessor.fetch_mass", "config": { - "_resource": "WikiBacklinks", + "_resource": "sources.WikiBacklinks", "_objective": {} }, "success": "ExtractProcessor.extract_resource", @@ -33,7 +33,7 @@ class CityCelebrities(object): ("people_filter", { "process": "HttpResourceProcessor.submit_mass", "config": { - "_resource": "WikiDataFilter", + "_resource": "sources.WikiDataFilter", "concat_args_with": ",", "wdq_template": "ITEMS[{}] AND CLAIM[31:5] AND NOCLAIM[570]" }, @@ -46,7 +46,7 @@ class CityCelebrities(object): ("people_text", { "process": "HttpResourceProcessor.fetch_mass", "config": { - "_resource": "WikiSearch", + "_resource": "sources.WikiSearch", "concat_args_with": ",", "_objective": {} }, @@ -60,7 +60,7 @@ class CityCelebrities(object): ("location_text", { "process": "HttpResourceProcessor.fetch_mass", "config": { - "_resource": "WikiSearch", + "_resource": "sources.WikiSearch", "concat_with": ",", "_objective": {} },