Merge pull request #274 from ntumlgroup/tree-pr

Add linear tree classifier

docs/api/linear.rst

@@ -25,6 +25,8 @@ The simplest usage is::
 
 .. autofunction:: train_binary_and_multiclass
 
+.. autofunction:: train_tree
+
 .. autofunction:: predict_values
 
 

libmultilabel/linear/__init__.py

@@ -1,4 +1,5 @@
 from .linear import *
 from .metrics import get_metrics, tabulate_metrics
 from .preprocessor import *
+from .tree import *
 from .utils import *

libmultilabel/linear/linear.py

@@ -14,13 +14,55 @@
            'predict_values']
 
 
-def train_1vsrest(y: sparse.csr_matrix, x: sparse.csr_matrix, options: str):
+class FlatModel:
+    def __init__(self, weights: np.matrix,
+                 bias: float,
+                 thresholds: 'float | np.ndarray',
+                 ):
+        self.weights = weights
+        self.bias = bias
+        self.thresholds = thresholds
+
+    def predict_values(self, x: sparse.csr_matrix) -> np.ndarray:
+        """Calculates the decision values associated with x.
+
+        Args:
+            x (sparse.csr_matrix): A matrix with dimension number of instances * number of features.
+
+        Returns:
+            np.ndarray: A matrix with dimension number of instances * number of classes.
+        """
+        bias = self.bias
+        bias_col = np.full((x.shape[0], 1 if bias > 0 else 0), bias)
+        num_feature = self.weights.shape[0]
+        num_feature -= 1 if bias > 0 else 0
+        if x.shape[1] < num_feature:
+            x = sparse.hstack([
+                x,
+                np.zeros((x.shape[0], num_feature - x.shape[1])),
+                bias_col,
+            ], 'csr')
+        else:
+            x = sparse.hstack([
+                x[:, :num_feature],
+                bias_col,
+            ], 'csr')
+
+        return (x * self.weights).A + self.thresholds
+
+
+def train_1vsrest(y: sparse.csr_matrix,
+                  x: sparse.csr_matrix,
+                  options: str,
+                  verbose: bool = True
+                  ) -> FlatModel:
     """Trains a linear model for multiabel data using a one-vs-rest strategy.
 
     Args:
         y (sparse.csr_matrix): A 0/1 matrix with dimensions number of instances * number of classes.
         x (sparse.csr_matrix): A matrix with dimensions number of instances * number of features.
         options (str): The option string passed to liblinear.
+        verbose (bool, optional): Output extra progress information. Defaults to True.
 
     Returns:
         A model which can be used in predict_values.
@@ -33,12 +75,15 @@ def train_1vsrest(y: sparse.csr_matrix, x: sparse.csr_matrix, options: str):
     num_feature = x.shape[1]
     weights = np.zeros((num_feature, num_class), order='F')
 
-    logging.info(f'Training one-vs-rest model on {num_class} labels')
-    for i in tqdm(range(num_class)):
+    if verbose:
+        logging.info(f'Training one-vs-rest model on {num_class} labels')
+    for i in tqdm(range(num_class), disable=not verbose):
         yi = y[:, i].toarray().reshape(-1)
         weights[:, i] = do_train(2*yi - 1, x, options).ravel()
 
-    return {'weights': np.asmatrix(weights), '-B': bias, 'threshold': 0}
+    return FlatModel(weights=np.asmatrix(weights),
+                     bias=bias,
+                     thresholds=0)
 
 
 def prepare_options(x: sparse.csr_matrix, options: str) -> 'tuple[sparse.csr_matrix, str, float]':
@@ -84,9 +129,13 @@ def prepare_options(x: sparse.csr_matrix, options: str) -> 'tuple[sparse.csr_mat
     return x, options, bias
 
 
-def train_thresholding(y: sparse.csr_matrix, x: sparse.csr_matrix, options: str):
+def train_thresholding(y: sparse.csr_matrix,
+                       x: sparse.csr_matrix,
+                       options: str,
+                       verbose: bool = True
+                       ) -> FlatModel:
     """Trains a linear model for multilabel data using a one-vs-rest strategy
-    and cross-validation to pick an optimal decision threshold for Macro-F1.
+    and cross-validation to pick optimal decision thresholds for Macro-F1.
     Outperforms train_1vsrest in most aspects at the cost of higher
     time complexity.
     See user guide for more details.
@@ -95,6 +144,7 @@ def train_thresholding(y: sparse.csr_matrix, x: sparse.csr_matrix, options: str)
         y (sparse.csr_matrix): A 0/1 matrix with dimensions number of instances * number of classes.
         x (sparse.csr_matrix): A matrix with dimensions number of instances * number of features.
         options (str): The option string passed to liblinear.
+        verbose (bool, optional): Output extra progress information. Defaults to True.
 
     Returns:
         A model which can be used in predict_values.
@@ -108,14 +158,17 @@ def train_thresholding(y: sparse.csr_matrix, x: sparse.csr_matrix, options: str)
     weights = np.zeros((num_feature, num_class), order='F')
     thresholds = np.zeros(num_class)
 
-    logging.info(f'Training thresholding model on {num_class} labels')
-    for i in tqdm(range(num_class)):
+    if verbose:
+        logging.info(f'Training thresholding model on {num_class} labels')
+    for i in tqdm(range(num_class), disable=not verbose):
         yi = y[:, i].toarray().reshape(-1)
         w, t = thresholding_one_label(2*yi - 1, x, options)
         weights[:, i] = w.ravel()
         thresholds[i] = t
 
-    return {'weights': np.asmatrix(weights), '-B': bias, 'threshold': thresholds}
+    return FlatModel(weights=np.asmatrix(weights),
+                     bias=bias,
+                     thresholds=thresholds)
 
 
 def thresholding_one_label(y: np.ndarray,
@@ -318,7 +371,11 @@ def fmeasure(y_true: np.ndarray, y_pred: np.ndarray) -> float:
     return F
 
 
-def train_cost_sensitive(y: sparse.csr_matrix, x: sparse.csr_matrix, options: str):
+def train_cost_sensitive(y: sparse.csr_matrix,
+                         x: sparse.csr_matrix,
+                         options: str,
+                         verbose: bool = True
+                         ) -> FlatModel:
     """Trains a linear model for multilabel data using a one-vs-rest strategy
     and cross-validation to pick an optimal asymmetric misclassification cost
     for Macro-F1.
@@ -330,6 +387,7 @@ def train_cost_sensitive(y: sparse.csr_matrix, x: sparse.csr_matrix, options: st
         y (sparse.csr_matrix): A 0/1 matrix with dimensions number of instances * number of classes.
         x (sparse.csr_matrix): A matrix with dimensions number of instances * number of features.
         options (str): The option string passed to liblinear.
+        verbose (bool, optional): Output extra progress information. Defaults to True.
 
     Returns:
         A model which can be used in predict_values.
@@ -342,14 +400,17 @@ def train_cost_sensitive(y: sparse.csr_matrix, x: sparse.csr_matrix, options: st
     num_feature = x.shape[1]
     weights = np.zeros((num_feature, num_class), order='F')
 
-    logging.info(
-        f'Training cost-sensitive model for Macro-F1 on {num_class} labels')
-    for i in tqdm(range(num_class)):
+    if verbose:
+        logging.info(
+            f'Training cost-sensitive model for Macro-F1 on {num_class} labels')
+    for i in tqdm(range(num_class), disable=not verbose):
         yi = y[:, i].toarray().reshape(-1)
         w = cost_sensitive_one_label(2*yi - 1, x, options)
         weights[:, i] = w.ravel()
 
-    return {'weights': np.asmatrix(weights), '-B': bias, 'threshold': 0}
+    return FlatModel(weights=np.asmatrix(weights),
+                     bias=bias,
+                     thresholds=0)
 
 
 def cost_sensitive_one_label(y: np.ndarray,
@@ -415,7 +476,11 @@ def cross_validate(y: np.ndarray,
     return 2*pred - 1
 
 
-def train_cost_sensitive_micro(y: sparse.csr_matrix, x: sparse.csr_matrix, options: str):
+def train_cost_sensitive_micro(y: sparse.csr_matrix,
+                               x: sparse.csr_matrix,
+                               options: str,
+                               verbose: bool = True
+                               ) -> FlatModel:
     """Trains a linear model for multilabel data using a one-vs-rest strategy
     and cross-validation to pick an optimal asymmetric misclassification cost
     for Micro-F1.
@@ -427,6 +492,7 @@ def train_cost_sensitive_micro(y: sparse.csr_matrix, x: sparse.csr_matrix, optio
         y (sparse.csr_matrix): A 0/1 matrix with dimensions number of instances * number of classes.
         x (sparse.csr_matrix): A matrix with dimensions number of instances * number of features.
         options (str): The option string passed to liblinear.
+        verbose (bool, optional): Output extra progress information. Defaults to True.
 
     Returns:
         A model which can be used in predict_values.
@@ -444,11 +510,12 @@ def train_cost_sensitive_micro(y: sparse.csr_matrix, x: sparse.csr_matrix, optio
     param_space = [1, 1.33, 1.8, 2.5, 3.67, 6, 13]
     bestScore = -np.Inf
 
-    logging.info(
-        f'Training cost-sensitive model for Micro-F1 on {num_class} labels')
+    if verbose:
+        logging.info(
+            f'Training cost-sensitive model for Micro-F1 on {num_class} labels')
     for a in param_space:
         tp = fn = fp = 0
-        for i in tqdm(range(num_class)):
+        for i in tqdm(range(num_class), disable=not verbose):
             yi = y[:, i].toarray().reshape(-1)
             yi = 2*yi - 1
 
@@ -469,16 +536,23 @@ def train_cost_sensitive_micro(y: sparse.csr_matrix, x: sparse.csr_matrix, optio
         w = do_train(2*yi - 1, x, final_options)
         weights[:, i] = w.ravel()
 
-    return {'weights': np.asmatrix(weights), '-B': bias, 'threshold': 0}
+    return FlatModel(weights=np.asmatrix(weights),
+                     bias=bias,
+                     thresholds=0)
 
 
-def train_binary_and_multiclass(y: sparse.csr_matrix, x: sparse.csr_matrix, options: str):
+def train_binary_and_multiclass(y: sparse.csr_matrix,
+                                x: sparse.csr_matrix,
+                                options: str,
+                                verbose: bool = True
+                                ) -> FlatModel:
     """Trains a linear model for binary and multi-class data.
 
     Args:
         y (sparse.csr_matrix): A 0/1 matrix with dimensions number of instances * number of classes.
         x (sparse.csr_matrix): A matrix with dimensions number of instances * number of features.
         options (str): The option string passed to liblinear.
+        verbose (bool, optional): Output extra progress information. Defaults to True.
 
     Returns:
         A model which can be used in predict_values.
@@ -508,9 +582,11 @@ def train_binary_and_multiclass(y: sparse.csr_matrix, x: sparse.csr_matrix, opti
         weights[:, train_labels] = w
 
     # For labels not appeared in training, assign thresholds to -inf so they won't be predicted.
-    threshold = np.full(num_labels, -np.inf)
-    threshold[train_labels] = 0
-    return {'weights': np.asmatrix(weights), '-B': bias, 'threshold': threshold}
+    thresholds = np.full(num_labels, -np.inf)
+    thresholds[train_labels] = 0
+    return FlatModel(weights=np.asmatrix(weights),
+                     bias=bias,
+                     thresholds=thresholds)
 
 
 def predict_values(model, x: sparse.csr_matrix) -> np.ndarray:
@@ -523,23 +599,7 @@ def predict_values(model, x: sparse.csr_matrix) -> np.ndarray:
     Returns:
         np.ndarray: A matrix with dimension number of instances * number of classes.
     """
-    bias = model['-B']
-    bias_col = np.full((x.shape[0], 1 if bias > 0 else 0), bias)
-    num_feature = model['weights'].shape[0]
-    num_feature -= 1 if bias > 0 else 0
-    if x.shape[1] < num_feature:
-        x = sparse.hstack([
-            x,
-            np.zeros((x.shape[0], num_feature - x.shape[1])),
-            bias_col,
-        ], 'csr')
-    else:
-        x = sparse.hstack([
-            x[:, :num_feature],
-            bias_col,
-        ], 'csr')
-
-    return (x * model['weights']).A + model['threshold']
+    return model.predict_values(x)
 
 
 def get_topk_labels(label_mapping: np.ndarray, preds: np.ndarray, top_k: int = 5) -> 'list[list[str]]':