From 4c36b58556cfc9a9ff1397640967814153f2e9ee Mon Sep 17 00:00:00 2001 From: gersona Date: Sat, 24 Aug 2024 19:05:53 +0300 Subject: [PATCH 01/11] cleanup stale glossaries translation delete and glossary languages sync --- weblate/glossary/tasks.py | 37 +++++++++++++++++++++++++++-- weblate/glossary/tests.py | 19 ++++++++++++++- weblate/trans/models/translation.py | 6 ++++- 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/weblate/glossary/tasks.py b/weblate/glossary/tasks.py index 6942a80d1703..5551e79ed6cd 100644 --- a/weblate/glossary/tasks.py +++ b/weblate/glossary/tasks.py @@ -4,10 +4,13 @@ from __future__ import annotations +from django.db import transaction + from weblate.lang.models import Language -from weblate.trans.models import Component +from weblate.trans.models import Component, Project, Translation from weblate.utils.celery import app from weblate.utils.lock import WeblateLockTimeoutError +from weblate.utils.stats import prefetch_stats @app.task( @@ -16,7 +19,11 @@ retry_backoff=60, ) def sync_glossary_languages(pk: int, component: Component | None = None) -> None: - """Add missing glossary languages.""" + """Add missing glossary languages and delete empty stale glossaries.""" + # Delete stale glossaries + cleanup_stale_glossaries(component.project) + + # Add missing glossary languages if component is None: component = Component.objects.get(pk=pk) @@ -40,6 +47,32 @@ def sync_glossary_languages(pk: int, component: Component | None = None) -> None component.create_translations_task() +@app.task(trail=False, autoretry_for=(Project.DoesNotExist, WeblateLockTimeoutError)) +def cleanup_stale_glossaries(project: int | Project) -> None: + if isinstance(project, int): + project = Project.objects.get(pk=project) + + languages_in_non_glossary_components: set[int] = set( + Translation.objects.filter( + component__project=project, component__is_glossary=False + ).values_list("language_id", flat=True) + ) + + glossary_translations = prefetch_stats( + Translation.objects.filter( + component__project=project, component__is_glossary=True + ) + ) + for glossary in glossary_translations: + if ( + glossary.stats.translated == 0 + and glossary.language_id not in languages_in_non_glossary_components + ): + glossary.delete() + transaction.on_commit(glossary.stats.update_parents) + transaction.on_commit(glossary.component.schedule_update_checks) + + @app.task( trail=False, autoretry_for=(Component.DoesNotExist, WeblateLockTimeoutError), diff --git a/weblate/glossary/tests.py b/weblate/glossary/tests.py index bbd8af0c64ba..7d8ea9829e43 100644 --- a/weblate/glossary/tests.py +++ b/weblate/glossary/tests.py @@ -11,7 +11,11 @@ from django.urls import reverse from weblate.glossary.models import get_glossary_terms, get_glossary_tsv -from weblate.glossary.tasks import sync_terminology +from weblate.glossary.tasks import ( + cleanup_stale_glossaries, + sync_terminology, +) +from weblate.lang.models import Language from weblate.trans.models import Unit from weblate.trans.tests.test_views import ViewTestCase from weblate.trans.tests.utils import get_test_file @@ -472,3 +476,16 @@ def test_tsv(self) -> None: lines = list(reader) self.assertEqual(len(lines), 163) self.assertTrue(all(len(line) == 2 for line in lines)) + + def test_stale_glossaries_cleanup(self) -> None: + initial_count = self.glossary_component.translation_set.count() + # delete one translation + german = Language.objects.get(code="de") + self.component.translation_set.get(language=german).remove(self.user) + + cleanup_stale_glossaries(self.project.id) + + # check that only the one glossary has been deleted + self.assertEqual( + self.glossary_component.translation_set.count(), initial_count - 1 + ) diff --git a/weblate/trans/models/translation.py b/weblate/trans/models/translation.py index dc2bf1d8d761..7a1413337bdd 100644 --- a/weblate/trans/models/translation.py +++ b/weblate/trans/models/translation.py @@ -1337,7 +1337,9 @@ def get_export_url(self): return self.component.get_export_url() def remove(self, user: User) -> None: - """Remove translation from the VCS.""" + """Remove translation from the Database and VCS.""" + from weblate.glossary.tasks import cleanup_stale_glossaries + author = user.get_author_name() # Log self.log_info("removing %s as %s", self.filenames, author) @@ -1367,6 +1369,8 @@ def remove(self, user: User) -> None: author=user, ) + cleanup_stale_glossaries.delay(self.component.project.id) + def handle_store_change( self, request, user, previous_revision: str, change=None ) -> None: From bb00fef5869a5a77f48435ffe885ddd0d3cc13d0 Mon Sep 17 00:00:00 2001 From: gersona Date: Sat, 24 Aug 2024 19:18:21 +0300 Subject: [PATCH 02/11] changelog update --- docs/changes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index 25962eda24d2..66cd486a5be5 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -7,6 +7,8 @@ Not yet released. **Improvements** +* Stale empty glossaries are now automatically removed + **Bug fixes** * Support for using Docker network names in automatic suggestion settings. From 7fd34c2f6f13d9c50c90da0decf2a386abce4ec6 Mon Sep 17 00:00:00 2001 From: gersona Date: Mon, 26 Aug 2024 09:28:40 +0300 Subject: [PATCH 03/11] test fix --- weblate/api/tests.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/weblate/api/tests.py b/weblate/api/tests.py index a7593003b4ad..0a0c919e9543 100644 --- a/weblate/api/tests.py +++ b/weblate/api/tests.py @@ -3295,7 +3295,11 @@ def test_add_plural(self) -> None: ) def test_delete(self) -> None: - start_count = Translation.objects.count() + def _translation_count(): + # ignore glossaries as stale glossaries are also cleaned out + return Translation.objects.filter(component__is_glossary=False).count() + + start_count = _translation_count() self.do_request( "api:translation-detail", self.translation_kwargs, method="delete", code=403 ) @@ -3306,7 +3310,8 @@ def test_delete(self) -> None: superuser=True, code=204, ) - self.assertEqual(Translation.objects.count(), start_count - 1) + + self.assertEqual(_translation_count(), start_count - 1) class UnitAPITest(APIBaseTest): From 7d58c16d315416728624488a8172cca56201e3c0 Mon Sep 17 00:00:00 2001 From: gers Date: Mon, 26 Aug 2024 16:46:58 +0300 Subject: [PATCH 04/11] Update weblate/glossary/tasks.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michal Čihař --- weblate/glossary/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weblate/glossary/tasks.py b/weblate/glossary/tasks.py index 5551e79ed6cd..bd88624d80b9 100644 --- a/weblate/glossary/tasks.py +++ b/weblate/glossary/tasks.py @@ -68,7 +68,7 @@ def cleanup_stale_glossaries(project: int | Project) -> None: glossary.stats.translated == 0 and glossary.language_id not in languages_in_non_glossary_components ): - glossary.delete() + glossary.remove() transaction.on_commit(glossary.stats.update_parents) transaction.on_commit(glossary.component.schedule_update_checks) From 4890446cd772b29cda1e0ce04f0d906567fbcf90 Mon Sep 17 00:00:00 2001 From: gersona Date: Tue, 17 Sep 2024 19:30:40 +0300 Subject: [PATCH 05/11] update glossary.tasks.cleanup_stale_glossaries --- weblate/glossary/tasks.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/weblate/glossary/tasks.py b/weblate/glossary/tasks.py index bd88624d80b9..e8661d482fe4 100644 --- a/weblate/glossary/tasks.py +++ b/weblate/glossary/tasks.py @@ -61,13 +61,10 @@ def cleanup_stale_glossaries(project: int | Project) -> None: glossary_translations = prefetch_stats( Translation.objects.filter( component__project=project, component__is_glossary=True - ) + ).exclude(language__id__in=languages_in_non_glossary_components) ) for glossary in glossary_translations: - if ( - glossary.stats.translated == 0 - and glossary.language_id not in languages_in_non_glossary_components - ): + if glossary.stats.translated == 0: glossary.remove() transaction.on_commit(glossary.stats.update_parents) transaction.on_commit(glossary.component.schedule_update_checks) From c1fd930a420cee483ce404ac97ac1de39b15dbbd Mon Sep 17 00:00:00 2001 From: Benjamin Alan Jamie Date: Wed, 18 Sep 2024 13:20:04 +0200 Subject: [PATCH 06/11] sentence correction --- docs/changes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changes.rst b/docs/changes.rst index 055abdc19c78..53a394ed5efe 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -10,7 +10,7 @@ Not yet released. **Improvements** -* Stale empty glossaries are now automatically removed +* Stale, empty glossaries are now automatically removed. * :ref`mt-deepl` now supports specifying translation context. **Bug fixes** From e4f8633964e5636d9f2c0c75dba233ba7b427278 Mon Sep 17 00:00:00 2001 From: gersona Date: Thu, 19 Sep 2024 10:36:57 +0300 Subject: [PATCH 07/11] fix missing argument TypeError --- weblate/glossary/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/weblate/glossary/tasks.py b/weblate/glossary/tasks.py index e8661d482fe4..39aa7b0c1b0a 100644 --- a/weblate/glossary/tasks.py +++ b/weblate/glossary/tasks.py @@ -6,6 +6,7 @@ from django.db import transaction +from weblate.auth.models import get_anonymous from weblate.lang.models import Language from weblate.trans.models import Component, Project, Translation from weblate.utils.celery import app @@ -65,7 +66,7 @@ def cleanup_stale_glossaries(project: int | Project) -> None: ) for glossary in glossary_translations: if glossary.stats.translated == 0: - glossary.remove() + glossary.remove(get_anonymous()) transaction.on_commit(glossary.stats.update_parents) transaction.on_commit(glossary.component.schedule_update_checks) From 763ecb3c114a4823bc6155c5b573852669cd1b39 Mon Sep 17 00:00:00 2001 From: gersona Date: Mon, 30 Sep 2024 11:11:02 +0300 Subject: [PATCH 08/11] only cleanup glossaries on translation delete --- weblate/glossary/tasks.py | 53 +++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/weblate/glossary/tasks.py b/weblate/glossary/tasks.py index 39aa7b0c1b0a..2e99a23adca9 100644 --- a/weblate/glossary/tasks.py +++ b/weblate/glossary/tasks.py @@ -5,6 +5,7 @@ from __future__ import annotations from django.db import transaction +from django.db.models import F from weblate.auth.models import get_anonymous from weblate.lang.models import Language @@ -20,11 +21,7 @@ retry_backoff=60, ) def sync_glossary_languages(pk: int, component: Component | None = None) -> None: - """Add missing glossary languages and delete empty stale glossaries.""" - # Delete stale glossaries - cleanup_stale_glossaries(component.project) - - # Add missing glossary languages + """Add missing glossary languages.""" if component is None: component = Component.objects.get(pk=pk) @@ -50,6 +47,18 @@ def sync_glossary_languages(pk: int, component: Component | None = None) -> None @app.task(trail=False, autoretry_for=(Project.DoesNotExist, WeblateLockTimeoutError)) def cleanup_stale_glossaries(project: int | Project) -> None: + """ + Delete stale glossaries. + + A glossary translation is considered stale when it meets the following conditions: + - glossary.language is not used in any other non-glossary components + - glossary.language is different from glossary.component.source_language + - It has no translation + + Stale glossary is not removed if: + - the component only has one glossary component + - if is managed outside weblate (i.e repo != 'local:') + """ if isinstance(project, int): project = Project.objects.get(pk=project) @@ -62,13 +71,41 @@ def cleanup_stale_glossaries(project: int | Project) -> None: glossary_translations = prefetch_stats( Translation.objects.filter( component__project=project, component__is_glossary=True - ).exclude(language__id__in=languages_in_non_glossary_components) + ) + .prefetch() + .exclude(language__id__in=languages_in_non_glossary_components) + .exclude(language=F("component__source_language")) ) + + component_to_check = [] + + def can_delete(_glossary: Translation) -> bool: + """ + Check if a glossary can be deleted. + + It is possible to delete a glossary if: + - it has no translations + - it is not the only glossary in the project + - it is managed by Weblate (i.e. repo == 'local:') + """ + # TODO: optimize DB hits + return all( + [ + _glossary.stats.translated == 0, + len(_glossary.component.project.glossaries) > 1, + _glossary.component.repo == "local:", + ] + ) + for glossary in glossary_translations: - if glossary.stats.translated == 0: + if can_delete(glossary): glossary.remove(get_anonymous()) transaction.on_commit(glossary.stats.update_parents) - transaction.on_commit(glossary.component.schedule_update_checks) + if glossary.component not in component_to_check: + component_to_check.append(glossary.component) + + for component in component_to_check: + transaction.on_commit(component.schedule_update_checks) @app.task( From 3eb408d0d2516fb2a6d601e597b667749090db37 Mon Sep 17 00:00:00 2001 From: gersona Date: Mon, 30 Sep 2024 13:23:27 +0300 Subject: [PATCH 09/11] update tests --- weblate/api/tests.py | 2 +- weblate/glossary/tasks.py | 1 - weblate/glossary/tests.py | 26 ++++++++++++++++++++++++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/weblate/api/tests.py b/weblate/api/tests.py index e1648ba8964e..2660806f925f 100644 --- a/weblate/api/tests.py +++ b/weblate/api/tests.py @@ -3324,7 +3324,7 @@ def test_add_plural(self) -> None: def test_delete(self) -> None: def _translation_count(): - # ignore glossaries as stale glossaries are also cleaned out + # exclude glossaries because stale glossaries are also cleaned out return Translation.objects.filter(component__is_glossary=False).count() start_count = _translation_count() diff --git a/weblate/glossary/tasks.py b/weblate/glossary/tasks.py index 2e99a23adca9..8bf3f4c73e2d 100644 --- a/weblate/glossary/tasks.py +++ b/weblate/glossary/tasks.py @@ -88,7 +88,6 @@ def can_delete(_glossary: Translation) -> bool: - it is not the only glossary in the project - it is managed by Weblate (i.e. repo == 'local:') """ - # TODO: optimize DB hits return all( [ _glossary.stats.translated == 0, diff --git a/weblate/glossary/tests.py b/weblate/glossary/tests.py index 7d8ea9829e43..1471758845ae 100644 --- a/weblate/glossary/tests.py +++ b/weblate/glossary/tests.py @@ -479,13 +479,35 @@ def test_tsv(self) -> None: def test_stale_glossaries_cleanup(self) -> None: initial_count = self.glossary_component.translation_set.count() - # delete one translation + + # check glossary not deleted because it has a valid translation + cleanup_stale_glossaries(self.project.id) + self.assertEqual(self.glossary_component.translation_set.count(), initial_count) + + # delete translation german = Language.objects.get(code="de") self.component.translation_set.get(language=german).remove(self.user) + # check glossary not deleted because project only has one glossary cleanup_stale_glossaries(self.project.id) + self.assertEqual(self.glossary_component.translation_set.count(), initial_count) + self._create_component( + "po", "po/*.po", project=self.project, is_glossary=True, name="Glossary-2" + ) + + # check glossary not deleted because glossary is managed outside weblate + self.glossary_component.repo = "git://example.com/test/project.git" + self.glossary_component.save() - # check that only the one glossary has been deleted + cleanup_stale_glossaries(self.project.id) + self.assertEqual(self.glossary_component.translation_set.count(), initial_count) + + # make glossary managed by weblate + self.glossary_component.repo = "local:" + self.glossary_component.save() + + # check that one glossary has been deleted + cleanup_stale_glossaries(self.project.id) self.assertEqual( self.glossary_component.translation_set.count(), initial_count - 1 ) From 2b5441d43e1d2331396ad4114120930936adc625 Mon Sep 17 00:00:00 2001 From: gersona Date: Fri, 4 Oct 2024 17:10:43 +0300 Subject: [PATCH 10/11] cleanup task trigger update --- weblate/glossary/tasks.py | 1 - weblate/trans/models/translation.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/weblate/glossary/tasks.py b/weblate/glossary/tasks.py index 8bf3f4c73e2d..d53bed5ab876 100644 --- a/weblate/glossary/tasks.py +++ b/weblate/glossary/tasks.py @@ -91,7 +91,6 @@ def can_delete(_glossary: Translation) -> bool: return all( [ _glossary.stats.translated == 0, - len(_glossary.component.project.glossaries) > 1, _glossary.component.repo == "local:", ] ) diff --git a/weblate/trans/models/translation.py b/weblate/trans/models/translation.py index d434ecf389f5..b09e94086736 100644 --- a/weblate/trans/models/translation.py +++ b/weblate/trans/models/translation.py @@ -1391,8 +1391,8 @@ def remove(self, user: User) -> None: user=user, author=user, ) - - cleanup_stale_glossaries.delay(self.component.project.id) + if not self.component.is_glossary: + cleanup_stale_glossaries.delay(self.component.project.id) def handle_store_change( self, From 8672c244541d95228ece47222b97adece49d7b66 Mon Sep 17 00:00:00 2001 From: gersona Date: Tue, 8 Oct 2024 08:39:44 +0300 Subject: [PATCH 11/11] adjust test to changed condition --- weblate/glossary/tests.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/weblate/glossary/tests.py b/weblate/glossary/tests.py index 1471758845ae..c34d501b54c3 100644 --- a/weblate/glossary/tests.py +++ b/weblate/glossary/tests.py @@ -478,27 +478,20 @@ def test_tsv(self) -> None: self.assertTrue(all(len(line) == 2 for line in lines)) def test_stale_glossaries_cleanup(self) -> None: + # setup: make glossary managed outside weblate + self.glossary_component.repo = "git://example.com/test/project.git" + self.glossary_component.save() + initial_count = self.glossary_component.translation_set.count() # check glossary not deleted because it has a valid translation cleanup_stale_glossaries(self.project.id) self.assertEqual(self.glossary_component.translation_set.count(), initial_count) - # delete translation + # delete translation: should trigger cleanup_stale_glossary task german = Language.objects.get(code="de") self.component.translation_set.get(language=german).remove(self.user) - # check glossary not deleted because project only has one glossary - cleanup_stale_glossaries(self.project.id) - self.assertEqual(self.glossary_component.translation_set.count(), initial_count) - self._create_component( - "po", "po/*.po", project=self.project, is_glossary=True, name="Glossary-2" - ) - - # check glossary not deleted because glossary is managed outside weblate - self.glossary_component.repo = "git://example.com/test/project.git" - self.glossary_component.save() - cleanup_stale_glossaries(self.project.id) self.assertEqual(self.glossary_component.translation_set.count(), initial_count)