Merge pull request #200 from American-Institutes-for-Research/HEA-820/nom-de-mesure-locale-regex

Hea 820/nom de mesure locale regex

Author: rhunwicks

pipelines/assets/livelihood_activity.py

@@ -243,6 +243,7 @@ def get_livelihood_activity_regexes() -> list:
         "nbr_pattern": r"(?:n[bo]?r?e?|no)\.?",
         "vendu_pattern": r"(?:quantité )?vendu(?:e|s|ss|es|ses)?",
         "separator_pattern": r" ?[:-]?",
+        "name_of_local_measure_pattern": r"(?:name of (?:meas(?:ure)?\.?)|nom(?: (?:de la mesure(?: locale)?|de mesure locale|du mesure|d'unité|mesure locale|unité de mesure))?)",
     }
     # Compile the regexes
     compiled_regexes = []

pipelines/assets/livelihood_activity_regexes.json

@@ -122,7 +122,7 @@
         "times_per_month"
     ],
     [
-        "name of measure",
+        "{name_of_local_measure_pattern}",
         null,
         false,
         "name_of_local_measure"
@@ -223,12 +223,6 @@
         false,
         "times_per_month"
     ],
-    [
-        "(?:nom de la mesure|nom du mesure|nom d'unité|nom mesure locale)",
-        null,
-        false,
-        "name_of_local_measure"
-    ],
     [
         "(?:poids? de la mesure|poids? du mesure|poids? d'unité)",
         null,
@@ -500,7 +494,19 @@
         "quantity_sold"
     ],
     [
-        "{product_pattern}: name of meas\\.",
+        "autre culture(?: de rente)?{separator_pattern} (?:type|nom)",
+        null,
+        true,
+        "product__name"
+    ],
+    [
+        "(?:autre nouriture de base|autre legumineuse|autre culture)?{separator_pattern} ?\\(?{product_pattern}\\)?{separator_pattern} ?{name_of_local_measure_pattern}",
+        null,
+        true,
+        "name_of_local_measure"
+    ],
+    [
+        "{product_pattern}{separator_pattern} {name_of_local_measure_pattern}:?\\.?",
         null,
         true,
         "name_of_local_measure"
@@ -757,12 +763,6 @@
         false,
         "number_of_local_measures"
     ],
-    [
-        "autre culture(?: de rente)?{separator_pattern} (?:type|nom)",
-        null,
-        true,
-        "product__name"
-    ],
     [
         "autre culture{separator_pattern} \\(?{product_pattern}\\)?(?: type)?",
         null,
@@ -847,12 +847,6 @@
         true,
         "expenditure"
     ],
-    [
-        "(?:autre nouriture de base|autre legumineuse|autre culture)?{separator_pattern} ?\\(?{product_pattern}\\)?{separator_pattern} ?(?:nom de la mesure locale?|nom mesure locale|nom du mesure|nom unité de mesure|nom)",
-        null,
-        true,
-        "name_of_local_measure"
-    ],
     [
         "{product_pattern} achetée?: quantité ?\\(?{unit_of_measure_pattern}\\)?",
         null,

pipelines_tests/test_assets/test_livelihood_activity_regexes.json

@@ -940,4 +940,35 @@
         "product_id": "petty trade",
         "attribute": "people_per_household"
     }
+,
+  "huile: nom du mesure": {
+    "is_start": true,
+    "product_id": "huile",
+    "attribute": "name_of_local_measure"
+  },
+  "sel - nom de la mesure locale": {
+    "is_start": true,
+    "product_id": "sel",
+    "attribute": "name_of_local_measure"
+  },
+  "sucre: nom mesure locale": {
+    "is_start": true,
+    "product_id": "sucre",
+    "attribute": "name_of_local_measure"
+  },
+  "eau pour humains: nom de mesure locale:.": {
+    "is_start": true,
+    "product_id": "eau pour humains",
+    "attribute": "name_of_local_measure"
+  },
+  "eau pour betail: nom de la mesure locale": {
+    "is_start": true,
+    "product_id": "eau pour betail",
+    "attribute": "name_of_local_measure"
+  },
+  "farine de ble - nom de la mesure locale": {
+    "is_start": true,
+    "product_id": "farine de ble",
+    "attribute": "name_of_local_measure"
+  }
 }