Skip to content

Commit

Permalink
bug-1921849: fix deprecated type: multi_field from super_search_fields
Browse files Browse the repository at this point in the history
  • Loading branch information
@relud
relud committed Oct 3, 2024
1 parent 649428f commit c68e0ec
Show file tree
Hide file tree
Showing 3 changed files with 31 additions and 62 deletions.
7 changes: 0 additions & 7 deletions socorro/external/es/crashstorage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,13 +236,6 @@ def build_document(src, crash_document, fields, all_keys):
# Fix values so they index correctly
storage_type = field.get("type", field["storage_mapping"].get("type"))

if (
storage_type == "multi_field"
and glom.glom(field, "storage_mapping.fields.full.type", default="")
== "string"
):
storage_type = "string"

if storage_type == "string":
analyzer = field.get("analyzer", field["storage_mapping"].get("analyzer"))
if analyzer == "keyword":
Expand Down
68 changes: 25 additions & 43 deletions socorro/external/es/super_search_fields.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,8 +206,8 @@ def is_doc_values_friendly(storage_value):
if not storage_type:
return False

# object, and multi_field storages don't work with doc_values=True
if storage_type in ("object", "multi_field"):
# object storages don't work with doc_values=True
if storage_type == "object":
return False

# analyzed string storages don't work with doc_values=True
Expand Down Expand Up @@ -796,10 +796,9 @@ def apply_schema_properties(fields, schema):
"query_type": "string",
"storage_mapping": {
"fields": {
"Android_Model": {"type": "string"},
"full": {"index": "not_analyzed", "type": "string"},
},
"type": "multi_field",
"type": "string",
},
},
"android_packagename": keyword_field(name="android_packagename"),
Expand Down Expand Up @@ -855,14 +854,11 @@ def apply_schema_properties(fields, schema):
"query_type": "string",
"storage_mapping": {
"fields": {
"AsyncShutdownTimeout": {
"analyzer": "standard",
"index": "analyzed",
"type": "string",
},
"full": {"index": "not_analyzed", "type": "string"},
},
"type": "multi_field",
"analyzer": "standard",
"index": "analyzed",
"type": "string",
},
},
"available_page_file": {
Expand Down Expand Up @@ -1038,14 +1034,11 @@ def apply_schema_properties(fields, schema):
"query_type": "string",
"storage_mapping": {
"fields": {
"cpu_info": {
"analyzer": "standard",
"index": "analyzed",
"type": "string",
},
"full": {"index": "not_analyzed", "type": "string"},
},
"type": "multi_field",
"analyzer": "standard",
"index": "analyzed",
"type": "string",
},
},
"cpu_microcode_version": {
Expand Down Expand Up @@ -1120,13 +1113,10 @@ def apply_schema_properties(fields, schema):
"storage_mapping": {
"fields": {
"full": {"index": "not_analyzed", "type": "string"},
"dumper_error": {
"type": "string",
"index": "analyzed",
"analyzer": "standard",
},
},
"type": "multi_field",
"type": "string",
"index": "analyzed",
"analyzer": "standard",
},
},
"em_check_compatibility": {
Expand Down Expand Up @@ -1713,14 +1703,11 @@ def apply_schema_properties(fields, schema):
"name": "platform",
"namespace": "processed_crash",
"query_type": "enum",
# FIXME(willkg): storage_mapping should either be not_analyzed or analyzed as a
# keyword
"storage_mapping": {
"fields": {
"full": {"index": "not_analyzed", "type": "string"},
"os_name": {"type": "string"},
},
"type": "multi_field",
"type": "string",
},
},
"platform_pretty_version": {
Expand Down Expand Up @@ -1767,11 +1754,10 @@ def apply_schema_properties(fields, schema):
"query_type": "enum",
"storage_mapping": {
"fields": {
"plugin_filename": {"index": "analyzed", "type": "string"},
"PluginFilename": {"index": "analyzed", "type": "string"},
"full": {"index": "not_analyzed", "type": "string"},
},
"type": "multi_field",
"index": "analyzed",
"type": "string",
},
},
"plugin_name": {
Expand All @@ -1786,11 +1772,10 @@ def apply_schema_properties(fields, schema):
"query_type": "enum",
"storage_mapping": {
"fields": {
"PluginName": {"index": "analyzed", "type": "string"},
"plugin_name": {"index": "analyzed", "type": "string"},
"full": {"index": "not_analyzed", "type": "string"},
},
"type": "multi_field",
"index": "analyzed",
"type": "string",
},
},
"plugin_version": {
Expand All @@ -1805,11 +1790,10 @@ def apply_schema_properties(fields, schema):
"query_type": "enum",
"storage_mapping": {
"fields": {
"PluginVersion": {"index": "analyzed", "type": "string"},
"plugin_version": {"index": "analyzed", "type": "string"},
"full": {"index": "not_analyzed", "type": "string"},
},
"type": "multi_field",
"index": "analyzed",
"type": "string",
},
},
"possible_bit_flips_max_confidence": integer_field(
Expand Down Expand Up @@ -1863,9 +1847,9 @@ def apply_schema_properties(fields, schema):
"storage_mapping": {
"fields": {
"full": {"index": "not_analyzed", "type": "string"},
"product": {"index": "analyzed", "type": "string"},
},
"type": "multi_field",
"index": "analyzed",
"type": "string",
},
},
"productid": {
Expand Down Expand Up @@ -1925,9 +1909,9 @@ def apply_schema_properties(fields, schema):
"storage_mapping": {
"fields": {
"full": {"index": "not_analyzed", "type": "string"},
"reason": {"analyzer": "standard", "type": "string"},
},
"type": "multi_field",
"analyzer": "standard",
"type": "string",
},
},
"release_channel": {
Expand Down Expand Up @@ -1993,9 +1977,8 @@ def apply_schema_properties(fields, schema):
"storage_mapping": {
"fields": {
"full": {"index": "not_analyzed", "type": "string"},
"signature": {"type": "string"},
},
"type": "multi_field",
"type": "string",
},
},
"stackwalk_version": keyword_field(
Expand Down Expand Up @@ -2139,9 +2122,8 @@ def apply_schema_properties(fields, schema):
"storage_mapping": {
"fields": {
"full": {"index": "not_analyzed", "type": "string"},
"user_comments": {"type": "string"},
},
"type": "multi_field",
"type": "string",
},
},
"useragent_locale": {
Expand Down
18 changes: 6 additions & 12 deletions socorro/tests/external/es/test_super_search_fields.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -252,24 +252,18 @@ def test_add_doc_values():

storage_mapping = {
"fields": {
"AsyncShutdownTimeout": {
"analyzer": "standard",
"index": "analyzed",
"type": "string",
},
"full": {"index": "not_analyzed", "type": "string"},
},
"type": "multi_field",
"analyzer": "standard",
"index": "analyzed",
"type": "string",
}
add_doc_values(storage_mapping)
assert storage_mapping == {
"fields": {
"AsyncShutdownTimeout": {
"analyzer": "standard",
"index": "analyzed",
"type": "string",
},
"full": {"index": "not_analyzed", "type": "string", "doc_values": True},
},
"type": "multi_field",
"analyzer": "standard",
"index": "analyzed",
"type": "string",
}

0 comments on commit c68e0ec

Please sign in to comment.