GH-60: Regularization breaks LogisticRegressionAnalysis in 2.14.0

Correcting logistic regression regularization and adding more unit tests.

Sources/Accord.Statistics/Analysis/LogisticRegressionAnalysis.cs

@@ -153,7 +153,9 @@ public class LogisticRegressionAnalysis : IRegressionAnalysis
         private string outputName;
 
         private double[,] sourceMatrix;
-        private double[] result;
+        private double[] result;
+
+        double regularization = 1e-10;
 
         private LogisticCoefficientCollection coefficientCollection;
 
@@ -176,6 +178,7 @@ public LogisticRegressionAnalysis(double[][] inputs, double[] outputs)
             // Initial argument checking
             if (inputs == null)
                 throw new ArgumentNullException("inputs");
+
             if (outputs == null)
                 throw new ArgumentNullException("outputs");
 
@@ -264,6 +267,18 @@ public double Tolerance
         {
             get { return tolerance; }
             set { tolerance = value; }
+        }
+
+        /// <summary>
+        ///   Gets or sets the regularization value to be
+        ///   added in the objective function. Default is
+        ///   1e-10.
+        /// </summary>
+        /// 
+        public double Regularization
+        {
+            get { return regularization; }
+            set { regularization = value; }
         }
 
         /// <summary>
@@ -537,7 +552,9 @@ private bool compute()
             double delta;
             int iteration = 0;
 
-            var learning = new IterativeReweightedLeastSquares(regression);
+            var learning = new IterativeReweightedLeastSquares(regression);
+
+            learning.Regularization = regularization;
 
             do // learning iterations until convergence
             {
@@ -564,7 +581,9 @@ private void computeInner()
 
             // Perform likelihood-ratio tests against diminished nested models
             var innerModel = new LogisticRegression(inputCount - 1);
-            var learning = new IterativeReweightedLeastSquares(innerModel);
+            var learning = new IterativeReweightedLeastSquares(innerModel);
+
+            learning.Regularization = regularization;
 
             for (int i = 0; i < inputCount; i++)
             {