feat(component): Add component filtering based on key (#12032)

* Added key_filter field in Component model
* Added key filter in Component setting form and implemented key filtering in Translation.check_sync
* Applied changes based on comment and mitigated migration conflict
* Improved the disable for key_filted in the UI and the filtering condition
* Added tests and drop_key_filter_cache()
* Improved the implementation of firing check_sync when saving and added a test
* Updated code based on suggestions
* Updated the validation error message

Co-authored-by: Benjamin Alan Jamie <[email protected]>

docs/admin/projects.rst

@@ -910,6 +910,21 @@ Some examples of filtering:
 | Include all files (default)   | ``^[^.]+$``           |
 +-------------------------------+-----------------------+
 
+
+.. _component-key_filter:
+
+Key filter
+++++++++++
+
+A regular expression that is used to filter units by their keys. It displays only
+those units whose keys match the regular expression that was set
+as the value of this field.
+
+.. note::
+
+    This filter is only available for components with monolingual file formats.
+
+
 .. _component-variant_regex:
 
 Variants regular expression

docs/changes.rst

@@ -5,6 +5,7 @@ Not yet released.
 
 **New features**
 
+* Added :ref:`component-key_filter` in the component.
 * :ref:`Searching` now supports filtering by object path and :ref:`date-search`.
 * Merge requests credentials can now be passed in the repository URL, see :ref:`settings-credentials`.
 * :ref:`mt-azure-openai` automatic suggestion service.

weblate/api/serializers.py

@@ -585,6 +585,7 @@ class Meta:
             "commit_pending_age",
             "auto_lock_error",
             "language_regex",
+            "key_filter",
             "variant_regex",
             "zipfile",
             "docfile",

weblate/trans/forms.py

@@ -1491,6 +1491,7 @@ class Meta:
             "template",
             "intermediate",
             "language_regex",
+            "key_filter",
             "variant_regex",
             "restricted",
             "auto_lock_error",
@@ -1513,6 +1514,7 @@ def __init__(self, request: AuthenticatedHttpRequest, *args, **kwargs) -> None:
         self.fields["links"].queryset = request.user.managed_projects.exclude(
             pk=self.instance.project.pk
         )
+
         self.helper.layout = Layout(
             TabHolder(
                 Tab(
@@ -1603,6 +1605,7 @@ def __init__(self, request: AuthenticatedHttpRequest, *args, **kwargs) -> None:
                         "file_format",
                         "filemask",
                         "language_regex",
+                        "key_filter",
                         "source_language",
                     ),
                     Fieldset(
@@ -1650,6 +1653,15 @@ def clean(self) -> None:
         if self.hide_restricted:
             data["restricted"] = self.instance.restricted
 
+        if (
+            self.instance
+            and self.instance.key_filter
+            and not self.instance.file_format_cls.monolingual
+        ):
+            raise ValidationError(
+                gettext("To use the key filter, the file format must be monolingual.")
+            )
+
 
 class ComponentCreateForm(SettingsBaseForm, ComponentDocsMixin, ComponentAntispamMixin):
     """Component creation form."""
@@ -1677,6 +1689,7 @@ class Meta:
             "license",
             "language_code_style",
             "language_regex",
+            "key_filter",
             "source_language",
             "is_glossary",
         ]

weblate/trans/migrations/0024_component_key_filter.py

@@ -0,0 +1,29 @@
+# Copyright © Michal Čihař <[email protected]>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# Generated by Django 5.1.1 on 2024-09-17 09:47
+
+from django.db import migrations
+
+import weblate.trans.fields
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("trans", "0023_alter_label_description"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="component",
+            name="key_filter",
+            field=weblate.trans.fields.RegexField(
+                blank=True,
+                default="",
+                help_text="Regular expression used to filter keys. This is only available for monolingual formats.",
+                max_length=500,
+                verbose_name="Key filter",
+            ),
+        ),
+    ]

weblate/trans/models/component.py

@@ -770,6 +770,16 @@ class Component(models.Model, PathMixin, CacheKeyMixin, ComponentCategoryMixin):
     local_revision = models.CharField(max_length=200, default="", blank=True)
     processed_revision = models.CharField(max_length=200, default="", blank=True)
 
+    key_filter = RegexField(
+        verbose_name=gettext_lazy("Key filter"),
+        max_length=500,
+        default="",
+        help_text=gettext_lazy(
+            "Regular expression used to filter keys. This is only available for monolingual formats."
+        ),
+        blank=True,
+    )
+
     objects = ComponentQuerySet.as_manager()
 
     is_lockable = True
@@ -818,6 +828,11 @@ def save(self, *args, **kwargs) -> None:
         changed_template = False
         changed_variant = False
         create = True
+
+        # Sets the key_filter to blank if the file format is bilingual
+        if self.key_filter and not self.file_format_cls.monolingual:
+            self.key_filter = ""
+
         if self.id:
             old = Component.objects.get(pk=self.id)
             changed_git = (
@@ -835,9 +850,13 @@ def save(self, *args, **kwargs) -> None:
                 or (old.edit_template != self.edit_template)
                 or (old.new_base != self.new_base)
                 or changed_template
+                or old.key_filter != self.key_filter
             )
             if changed_setup:
                 old.commit_pending("changed setup", None)
+                if old.key_filter != self.key_filter:
+                    self.drop_key_filter_cache()
+
             changed_variant = old.variant_regex != self.variant_regex
             # Generate change entries for changes
             self.generate_changes(old)
@@ -852,6 +871,7 @@ def save(self, *args, **kwargs) -> None:
                 old.component_set.update(repo=self.get_repo_link_url())
             if changed_git:
                 self.drop_repository_cache()
+
             create = False
         elif self.is_glossary:
             # Creating new glossary
@@ -3386,6 +3406,11 @@ def drop_addons_cache(self) -> None:
         if "addons_cache" in self.__dict__:
             del self.__dict__["addons_cache"]
 
+    def drop_key_filter_cache(self) -> None:
+        """Invalidate the cached value of key_filter."""
+        if "key_filter_re" in self.__dict__:
+            del self.__dict__["key_filter_re"]
+
     def load_intermediate_store(self):
         """Load translate-toolkit store for intermediate."""
         store = self.file_format_cls(
@@ -3793,6 +3818,11 @@ def all_repo_components(self):
     def start_sentry_span(self, op: str):
         return sentry_sdk.start_span(op=op, description=self.full_slug)
 
+    @cached_property
+    def key_filter_re(self) -> re.Pattern:
+        """Provide the cached version of key_filter."""
+        return re.compile(self.key_filter)
+
 
 @receiver(m2m_changed, sender=Component.links.through)
 @disable_for_loaddata

weblate/trans/models/translation.py

@@ -437,6 +437,18 @@ def check_sync(self, force=False, request=None, change=None) -> None:  # noqa: C
                                 unit.source = translated_unit.source
                         except UnitNotFoundError:
                             pass
+                    if (
+                        self.component.file_format_cls.monolingual
+                        and self.component.key_filter_re
+                        and self.component.key_filter_re.match(unit.context) is None
+                    ):
+                        # This is where the key filtering take place
+                        self.log_info(
+                            "Doesn't match with key_filter, skipping: %s (%s)",
+                            unit.context,
+                            repr(unit.source),
+                        )
+                        continue
 
                     try:
                         id_hash = unit.id_hash

weblate/trans/tests/test_component.py

@@ -1058,3 +1058,49 @@ def test_readonly(self) -> None:
 
         # It should be now read only
         self.assertEqual(source.unit_set.all()[0].state, STATE_READONLY)
+
+
+class ComponentKeyFilterTest(ViewTestCase):
+    """Test the key filtering implementation in Component."""
+
+    def create_component(self):
+        return self.create_android(key_filter=r"^tr")
+
+    def test_get_key_filter_re(self) -> None:
+        self.assertEqual(self.component.key_filter_re.pattern, "^tr")
+
+    def test_get_filtered_result(self) -> None:
+        translation = self.component.translation_set.get(language_code="en")
+        units = translation.unit_set.all()
+        self.assertEqual(units.count(), 1)
+        self.assertEqual(units.all()[0].context, "try")
+
+    def test_change_key_filter(self) -> None:
+        self.component.key_filter = r"^th"
+        self.component.save()
+        self.assertEqual(self.component.key_filter_re.pattern, "^th")
+        translations = self.component.translation_set.all()
+        for translation in translations:
+            units = translation.unit_set.all()
+            self.assertEqual(units.count(), 1)
+            self.assertEqual(units.all()[0].context, "thanks")
+
+        self.component.key_filter = ""
+        self.component.save()
+        self.assertEqual(self.component.key_filter_re.pattern, "")
+        translations = self.component.translation_set.all()
+        for translation in translations:
+            units = translation.unit_set.all()
+            self.assertEqual(len(units), 4)
+
+    def test_bilingual_component(self):
+        project = self.component.project
+        component = self.create_po(
+            name="Bilingual Test", project=project, key_filter=r"^tr"
+        )
+        self.assertRaisesMessage(
+            ValidationError,
+            "To use the key filter, the file format must be monolingual.",
+        )
+        self.assertEqual(component.key_filter, "")
+        self.assertEqual(component.key_filter_re.pattern, "")