Feature Constructor: Evaluate categorical variables to strings

Author: janezd

Orange/widgets/data/owfeatureconstructor.py

@@ -101,6 +101,13 @@ def selected_row(view):
 
 
 class FeatureEditor(QFrame):
+    ExpressionTooltip = """
+Use variable names as values in expression.
+Categorical features are passed as strings
+(note the change in behaviour from Orange 3.30).
+
+""".lstrip()
+
     FUNCTIONS = dict(chain([(key, val) for key, val in math.__dict__.items()
                             if not key.startswith("_")],
                            [(key, val) for key, val in builtins.__dict__.items()
@@ -231,7 +238,8 @@ def insert_into_expression(self, what):
 
 
 class ContinuousFeatureEditor(FeatureEditor):
-    ExpressionTooltip = "A numeric expression"
+    ExpressionTooltip = "A numeric expression\n\n" \
+                        + FeatureEditor.ExpressionTooltip
 
     def editorData(self):
         return ContinuousDescriptor(
@@ -242,7 +250,7 @@ def editorData(self):
 
 
 class DateTimeFeatureEditor(FeatureEditor):
-    ExpressionTooltip = \
+    ExpressionTooltip = FeatureEditor.ExpressionTooltip + \
         "Result must be a string in ISO-8601 format " \
         "(e.g. 2019-07-30T15:37:27 or a part thereof),\n" \
         "or a number of seconds since Jan 1, 1970."
@@ -255,7 +263,7 @@ def editorData(self):
 
 
 class DiscreteFeatureEditor(FeatureEditor):
-    ExpressionTooltip = \
+    ExpressionTooltip = FeatureEditor.ExpressionTooltip + \
         "Result must be a string, if values are not explicitly given\n" \
         "or a zero-based integer indices into a list of values given below."
 
@@ -292,7 +300,8 @@ def editorData(self):
 
 
 class StringFeatureEditor(FeatureEditor):
-    ExpressionTooltip = "A string expression"
+    ExpressionTooltip = "A string expression\n\n" \
+                        + FeatureEditor.ExpressionTooltip
 
     def editorData(self):
         return StringDescriptor(
@@ -551,6 +560,19 @@ def reserved_names(self, idx_=None):
                      if idx != idx_]
         return set(varnames)
 
+    @staticmethod
+    def purged_values(descriptors, domain):
+        expr = re.compile(r"(^|\W)("
+                          + "|".join(f"{var.name}"
+                                     for var in chain(domain.variables, domain.metas))
+                          + r")\.value(\W|$)")
+        return [
+            descriptor._replace(
+                expression=expr.sub(
+                    lambda mo: "".join(mo.group(1, 2, 3)),
+                    descriptor.expression))
+            for descriptor in descriptors]
+
     @Inputs.data
     @check_sql_input
     def setData(self, data=None):
@@ -572,6 +594,8 @@ def setData(self, data=None):
                 selmodel = self.featureview.selectionModel()
                 selmodel.selectionChanged.disconnect(
                     self._on_selectedVariableChanged)
+                self.descriptors = self.purged_values(
+                    self.descriptors, data.domain)
 
                 self.featuremodel[:] = list(self.descriptors)
                 self.setCurrentIndex(self.currentIndex)
@@ -1092,9 +1116,10 @@ def __call__(self, instance, *_):
             return [self(inst) for inst in instance]
         else:
             try:
-                args = [str(instance[var])
-                        if instance.domain[var].is_string else instance[var]
-                        for _, var in self.args]
+                args = [str(instance[var]) if var.is_string
+                    else var.values[int(instance[var])] if var.is_discrete
+                    else instance[var]
+                    for _, var in self.args]
                 y = self.func(*args)
             # user's expression can contain arbitrary errors
             # this also covers missing attributes

Orange/widgets/data/tests/test_owfeatureconstructor.py

@@ -8,6 +8,8 @@
 
 import numpy as np
 
+from orangewidget.settings import Context
+
 from Orange.data import (Table, Domain, StringVariable,
                          ContinuousVariable, DiscreteVariable, TimeVariable)
 from Orange.widgets.tests.base import WidgetTest
@@ -366,14 +368,59 @@ def test_discrete_no_values(self):
 
         discreteFeatureEditor.valuesedit.setText("A")
         discreteFeatureEditor.nameedit.setText("D1")
-        discreteFeatureEditor.expressionedit.setText("iris")
+        discreteFeatureEditor.expressionedit.setText("ord(iris[0])")
         self.widget.addFeature(
             discreteFeatureEditor.editorData()
         )
         self.assertFalse(self.widget.Error.more_values_needed.is_shown())
         self.widget.apply()
         self.assertTrue(self.widget.Error.more_values_needed.is_shown())
 
+    def test_purge_values(self):
+        v = [ContinuousVariable(name) for name in ("ana", "berta", "cilka")]
+        domain = Domain(v[:1], v[1], v[2:])
+
+        desc = self.widget.purged_values(
+            [ContinuousDescriptor(
+                 "y", "ana.value + berta.value + cilka.value", 1)], domain)[0]
+        self.assertEqual(desc.expression, "ana + berta + cilka")
+
+        desc = self.widget.purged_values(
+            [ContinuousDescriptor(
+                 "y", "ana.value + dani.value + cilka.value", 1)], domain)[0]
+        self.assertEqual(desc.expression, "ana + dani.value + cilka")
+
+        desc = self.widget.purged_values(
+            [ContinuousDescriptor(
+                "y", "sin(ana.value) + berta.value**2+cilka.value", 1)], domain)[0]
+        self.assertEqual(desc.expression, "sin(ana) + berta**2+cilka")
+
+        desc = self.widget.purged_values(
+            [ContinuousDescriptor(
+                "y", "sin(ana.value) + list(dict(a=2).values())", 1)], domain)[0]
+        self.assertEqual(desc.expression, "sin(ana) + list(dict(a=2).values())")
+
+    def test_purge_values_in_context(self):
+        names = ("ana", "berta", "cilka")
+        v = [DiscreteVariable(name, values=tuple("012")) for name in names]
+        domain = Domain(v)
+        data = Table.from_domain(domain, 5)
+
+        settings = {
+            "context_settings":
+            [Context(
+                attributes=dict.fromkeys(names, 1), metas={},
+                values=dict(
+                    descriptors=[ContinuousDescriptor(
+                        "y", "int(ana.value) + int(berta.value)", 1)],
+                    currentIndex=0)
+             )]
+        }
+        widget = self.create_widget(OWFeatureConstructor, settings)
+        self.send_signal(widget.Inputs.data, data)
+        self.assertEqual(widget.descriptors[0].expression,
+                         "int(ana) + int(berta)")
+
 
 class TestFeatureEditor(unittest.TestCase):
     def test_has_functions(self):

doc/visual-programming/source/widgets/data/featureconstructor.md

@@ -11,7 +11,7 @@ Add new features to your dataset.
 
 - Data: dataset with additional features
 
-The **Feature Constructor** allows you to manually add features (columns) into your dataset. The new feature can be a computation of an existing one or a combination of several (addition, subtraction, etc.). You can choose what type of feature it will be (discrete, continuous or string) and what its parameters are (name, value, expression). For continuous variables you only have to construct an expression in Python.
+The **Feature Constructor** allows you to manually add features (columns) into your dataset. The new feature can be a computation of an existing one or a combination of several (addition, subtraction, etc.). You can choose what type of feature it will be (categorical, numeric or text) and what its parameters are (name, value, expression). For numeric variables you only have to construct an expression in Python.
 
 ![](images/feature-constructor1-stamped.png)
 
@@ -24,7 +24,9 @@ The **Feature Constructor** allows you to manually add features (columns) into y
 7. Produce a report
 8. Press *Send* to communicate changes
 
-For discrete variables, however, there's a bit more work. First add or remove the values you want for the new feature. Then select the base value and the expression. In the example below, we have constructed an expression with 'if lower than' and defined three conditions; the program ascribes 0 (which we renamed to lower) if the original value is lower than 6, 1 (mid) if it is lower than 7 and 2 (higher) for all the other values. Notice that we use an underscore for the feature name (e.g. petal\_length).
+For categorical variables, the expression can return either strings or indices into a list of possible values. In the below example, we defined three values, "lower", "mid" and "higher". The expression must then return zero-based indices into that list. In the example below, we have constructed an expression which returns 0 (referring to "lower") if the value of sepal_length is below 6, 1 ("mid") if it is lower than 7 and 2 ("higher") otherwise. Notice that we replace spaces in the variable name with undersore (e.g. petal\_length).
+
+When the expression resturns strings, the list of values must be left empty.
 
 ![](images/feature-constructor2-stamped.png)
 
@@ -38,6 +40,8 @@ For discrete variables, however, there's a bit more work. First add or remove th
 8. Produce a report
 9. Press *Send* to communicate changes
 
+**Note**: before version 3.30, categorical values in expressions evaluated to indices into their lists of values; e.g., if a feature had values "red", "yellow" and "green", a values of "yellow" would evaluate to 2. This has been changed: categorical values are now evaluated as strings. The previous behaviour was unstable and can no longer be achieved in Feature Constructor.
+
 Example
 -------