Updates LightGBM from 2.X.X to 3.X.X (#6880)

* updated lightGBM version

* finished LightGBM baseline update

* removed accidental enums

* Fixed LightGBM Test

* fixed lightgbm

* fixed version

* fixed test with LightGBM

* fixed test result

Author: michaelgsharp

eng/Versions.props

@@ -24,7 +24,7 @@
     <SystemConfigurationConfigurationManagerVersion>6.0.1</SystemConfigurationConfigurationManagerVersion>
     <SystemIOFileSystemAccessControl>4.5.0</SystemIOFileSystemAccessControl>
     <SystemMemoryVersion>4.5.5</SystemMemoryVersion>
-    <SystemNumericsTensorsVersion>8.0.0-rtm.23523.3</SystemNumericsTensorsVersion>
+    <SystemNumericsTensorsVersion>8.0.0</SystemNumericsTensorsVersion>
     <SystemReflectionEmitLightweightVersion>4.3.0</SystemReflectionEmitLightweightVersion>
     <SystemReflectionEmitVersion>4.3.0</SystemReflectionEmitVersion>
     <SystemRuntimeCompilerServicesUnsafeVersion>6.0.0</SystemRuntimeCompilerServicesUnsafeVersion>
@@ -35,7 +35,7 @@
     <!-- Other product dependencies -->
     <ApacheArrowVersion>11.0.0</ApacheArrowVersion>
     <GoogleProtobufVersion>3.19.6</GoogleProtobufVersion>
-    <LightGBMVersion>2.3.1</LightGBMVersion>
+    <LightGBMVersion>3.3.5</LightGBMVersion>
     <MicrosoftCodeAnalysisAnalyzersVersion>3.3.0</MicrosoftCodeAnalysisAnalyzersVersion>
     <MicrosoftCodeAnalysisCSharpVersion>3.9.0</MicrosoftCodeAnalysisCSharpVersion>
     <MicrosoftDotNetInteractiveVersion>1.0.0-beta.23509.3</MicrosoftDotNetInteractiveVersion>
@@ -88,7 +88,7 @@
     <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
     <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
     <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
-    <SystemDataSqlClientVersion>4.6.1</SystemDataSqlClientVersion>
+    <SystemDataSqlClientVersion>4.8.5</SystemDataSqlClientVersion>
     <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
     <XunitCombinatorialVersion>1.2.7</XunitCombinatorialVersion>
     <XUnitVersion>2.4.2</XUnitVersion>

src/Microsoft.ML.LightGbm/WrappedLightGbmBooster.cs

@@ -80,14 +80,14 @@ internal unsafe string GetModelString()
             byte[] buffer = new byte[bufLen];
             int size = 0;
             fixed (byte* ptr = buffer)
-                LightGbmInterfaceUtils.Check(WrappedLightGbmInterface.BoosterSaveModelToString(Handle, 0, BestIteration, bufLen, ref size, ptr));
+                LightGbmInterfaceUtils.Check(WrappedLightGbmInterface.BoosterSaveModelToString(Handle, 0, BestIteration, 0, bufLen, ref size, ptr));
             // If buffer size is not enough, reallocate buffer and get again.
             if (size > bufLen)
             {
                 bufLen = size;
                 buffer = new byte[bufLen];
                 fixed (byte* ptr = buffer)
-                    LightGbmInterfaceUtils.Check(WrappedLightGbmInterface.BoosterSaveModelToString(Handle, 0, BestIteration, bufLen, ref size, ptr));
+                    LightGbmInterfaceUtils.Check(WrappedLightGbmInterface.BoosterSaveModelToString(Handle, 0, BestIteration, 0, bufLen, ref size, ptr));
             }
             byte[] content = new byte[size];
             Array.Copy(buffer, content, size);

src/Microsoft.ML.LightGbm/WrappedLightGbmInterface.cs

@@ -190,6 +190,7 @@ public static extern int BoosterCreate(SafeDataSetHandle trainset,
         public static extern unsafe int BoosterSaveModelToString(SafeBoosterHandle handle,
             int startIteration,
             int numIteration,
+            int featureImportanceType,
             int bufferLen,
             ref int outLen,
             byte* outStr);
@@ -213,7 +214,7 @@ public static extern unsafe int BoosterGetEval(SafeBoosterHandle handle, int dat
         #region API predict
         [DllImport(DllName, EntryPoint = "LGBM_BoosterPredictForMat", CallingConvention = CallingConvention.StdCall)]
         public static extern unsafe int BoosterPredictForMat(SafeBoosterHandle handle, IntPtr data, CApiDType dataType, int nRow, int nCol, int isRowMajor,
-            int predictType, int numIteration, [MarshalAs(UnmanagedType.LPStr)] string parameters, ref int outLen, double* outResult);
+            int predictType, int startIteration, int numIteration, [MarshalAs(UnmanagedType.LPStr)] string parameters, ref int outLen, double* outResult);
         #endregion
 
         #region API parallel

test/BaselineOutput/Common/FeatureContribution/LightGbmBinary.tsv

@@ -12,7 +12,7 @@
 #@   col=FeatureContributions:R4:32-37
 #@   col=FeatureContributions:R4:38-43
 #@ }
-950	757	692	720	297	7515	1	1	950	757	692	720	297	7515	0.956696868	0.760804	0.7872582	0.754716933	0.297893673	0.7578661	0.0871070847	0.116535939	0	0	0	1	18	4:-1	10:-0.0126099735	11:1	16:-0.262451649	17:20.81302
-459	961	0	659	274	2147	0	0	459	961	0	659	274	2147	0.462235659	0.965829134	0	0.690775633	0.27482447	0.21651876	24	1:1	6:-0.0739851445	9:-0.0561627634	11:-1	13:0.103110082	17:-1	19:2.238687	23:-21.7116222
-672	275	0	65	195	9818	1	1	672	275	0	65	195	9818	0.6767372	0.2763819	0	0.06813417	0.195586756	0.990116954	24	0:0.09341861	5:1	7:-0.99999994	10:-0.267225862	13:-0.0506075844	17:1	19:-0.9821341	23:19.4068565
-186	301	0	681	526	1456	0	0	186	301	0	681	526	1456	0.187311172	0.302512556	0	0.713836432	0.527582765	0.1468334	24	4:1	6:-0.0786664	9:-0.05971634	11:-1	16:0.0128529193	17:-1	22:0.262451649	23:-20.4196148
+950	757	692	720	297	7515	1	1	950	757	692	720	297	7515	0.95669687	0.760804	0.7872582	0.75471693	0.29789367	0.7578661	0.048468035	0.15627365	0	0	0	1	18	3:-1	4:-0.86065125	9:-0.008285429	11:1	15:-0.1600628	17:19.318588
+459	961	0	659	274	2147	0	0	459	961	0	659	274	2147	0.46223566	0.96582913	0	0.69077563	0.27482447	0.21651876	24	1:1	6:-0.10307922	9:-0.03593468	11:-0.99999994	13:0.069840066	17:-0.99999994	19:1.518843	23:-21.747446
+672	275	0	65	195	9818	1	1	672	275	0	65	195	9818	0.6767372	0.2763819	0	0.06813417	0.19558676	0.99011695	0.051555283	0	0	0.008813183	0	1	18	1:-1	4:-0.08602048	7:-0.08817758	11:1	13:-1.6014589	17:18.161747
+186	301	0	681	526	1456	0	0	186	301	0	681	526	1456	0.18731117	0.30251256	0	0.71383643	0.52758276	0.1468334	24	4:1	6:-0.13100865	7:-0.05215903	11:-1	16:0.006898874	17:-1	22:0.13775826	23:-19.968224

test/BaselineOutput/Common/FeatureContribution/LightGbmRanking.tsv

@@ -13,7 +13,7 @@
 #@   col=FeatureContributions:R4:33-38
 #@   col=FeatureContributions:R4:39-44
 #@ }
-950	757	692	720	297	7515	4	1	0	950	757	692	720	297	7515	0.956696868	0.760804	0.7872582	0.754716933	0.297893673	0.7578661	24	3:0.383134037	5:1	6:-0.9192525	10:-1	16:-0.2646893	17:1	22:-0.5723001	23:2.162158
-459	961	0	659	274	2147	1	1	0	459	961	0	659	274	2147	0.462235659	0.965829134	0	0.690775633	0.27482447	0.21651876	1	0.164955541	0	0.08735034	0	0	18	5:-1	6:0.110308476	11:-1	12:1.25388193	17:-11.36705
-672	275	0	65	195	9818	4	1	0	672	275	0	65	195	9818	0.6767372	0.2763819	0	0.06813417	0.195586756	0.990116954	24	0:0.0193515085	5:1	9:-0.902932465	10:-0.99999994	16:-0.1030465	17:1	22:-0.7474492	23:7.253514
-186	301	0	681	526	1456	0	1	0	186	301	0	681	526	1456	0.187311172	0.302512556	0	0.713836432	0.527582765	0.1468334	24	3:0.99999994	6:-0.0397184677	7:-0.0161588676	11:-1	15:0.008556744	17:-1	21:0.109527007	23:-12.8000793
+950	757	692	720	297	7515	4	1	0	950	757	692	720	297	7515	0.95669687	0.760804	0.7872582	0.75471693	0.29789367	0.7578661	0.18475686	0.19982447	0	0	0	1	18	3:-0.12083593	4:-1	10:-0.16583169	11:1	16:-0.2693773	17:1.6244018
+459	961	0	659	274	2147	1	1	0	459	961	0	659	274	2147	0.46223566	0.96582913	0	0.69077563	0.27482447	0.21651876	24	0:1	7:-0.021717148	9:-0.029168284	11:-1	12:0.09023979	17:-1	18:1.0248182	23:-11.35661
+672	275	0	65	195	9818	4	1	0	672	275	0	65	195	9818	0.6767372	0.2763819	0	0.06813417	0.19558676	0.99011695	0.036158063	0	0	0	0	1	0	-1	0	-0.50675523	-0.4690159	0	12	1:-0.06919657	5:1	7:-0.57434577	11:8.300206
+186	301	0	681	526	1456	0	1	0	186	301	0	681	526	1456	0.18731117	0.30251256	0	0.71383643	0.52758276	0.1468334	24	0:0.81803715	1:1	9:-0.027629191	11:-0.99999994	13:0.02057122	17:-0.99999994	19:0.24663319	23:-11.989234

test/BaselineOutput/Common/FeatureContribution/LightGbmRegressionWithCategoricalSplit.tsv

@@ -23,7 +23,7 @@
 #@   col=FeatureContributions:R4:47-55
 #@   col=FeatureContributions:R4:56-64
 #@ }
-CMT	1	1	0.7088812	1271	1.64874518	3.8	1.0118916	CRD	17.5	17.5	0	1	0	1	0	0	1	0	0	1	1	0	1	0	0	0.7088812	1.64874518	1.0118916	36	4:0.107879594	7:0.725665748	8:1	15:-1	24:-0.0418495	26:1	33:-0.370121539	35:8.844109
-CMT	1	1	0.7088812	474	0.6148743	1.5	0.3994309	CRD	8	8	0	1	0	1	0	0	1	0	0	1	1	0	1	0	0	0.7088812	0.6148743	0.3994309	36	4:1	15:-0.0364986733	16:-0.847436965	17:-1	22:0.011381451	26:-1	31:0.115415707	35:-10.1406841
-CMT	1	1	0.7088812	637	0.8263184	1.4	0.372802168	CRD	8.5	8.5	0	1	0	1	0	0	1	0	0	1	1	0	1	0	0	0.7088812	0.8263184	0.372802168	36	4:1	15:-0.0366709046	16:-0.5593253	17:-1	22:0.0182117485	26:-1	31:0.183812216	35:-10.0930576
-CMT	1	1	0.7088812	181	0.234793767	0.6	0.159772366	CSH	4.5	4.5	0	1	0	1	0	1	0	1	0	1	1	0	0	1	0	0.7088812	0.234793767	0.159772366	36	6:1	13:-0.293414325	16:-0.7202999	17:-1	24:0.0291313324	26:-1	33:0.33991462	35:-11.6683512
+CMT	1	1	0.7088812	1271	1.6487452	3.8	1.0118916	CRD	17.5	17.5	0	1	0	1	0	0	1	0	0	1	1	0	1	0	0	0.7088812	1.6487452	1.0118916	36	4:0.09394213	7:0.71750724	8:1	26:1	35:9.400241
+CMT	1	1	0.7088812	474	0.6148743	1.5	0.3994309	CRD	8	8	0	1	0	1	0	0	1	0	0	1	1	0	1	0	0	0.7088812	0.6148743	0.3994309	36	4:1	12:-0.15849221	16:-0.80323946	17:-1	22:0.24369338	26:-1	31:2.1028545	35:-8.6291
+CMT	1	1	0.7088812	637	0.8263184	1.4	0.37280217	CRD	8.5	8.5	0	1	0	1	0	0	1	0	0	1	1	0	1	0	0	0.7088812	0.8263184	0.37280217	36	4:1	12:-0.15364607	16:-0.37586385	17:-1	22:0.22522934	26:-1	31:1.9944918	35:-8.855382
+CMT	1	1	0.7088812	181	0.23479377	0.6	0.15977237	CSH	4.5	4.5	0	1	0	1	0	1	0	1	0	1	1	0	0	1	0	0.7088812	0.23479377	0.15977237	36	3:1	13:-0.8467557	16:-0.7164732	17:-1	21:0.8161843	26:-1	30:8.588971	35:-10.523323

test/BaselineOutput/Common/LightGBM/LightGBMDart-CV-breast-cancer.dart-out.txt

@@ -12,42 +12,42 @@ Confusion table
           ||======================
 PREDICTED || positive | negative | Recall
 TRUTH     ||======================
- positive ||      124 |       10 | 0.9254
+ positive ||      125 |        9 | 0.9328
  negative ||        9 |      219 | 0.9605
           ||======================
-Precision ||   0.9323 |   0.9563 |
-OVERALL 0/1 ACCURACY: 0.947514
-LOG LOSS/instance:  0.303740
+Precision ||   0.9328 |   0.9605 |
+OVERALL 0/1 ACCURACY: 0.950276
+LOG LOSS/instance:  0.298701
 Test-set entropy (prior Log-Loss/instance): 0.950799
-LOG-LOSS REDUCTION (RIG): 0.680543
-AUC:                0.984944
+LOG-LOSS REDUCTION (RIG): 0.685843
+AUC:                0.985271
 TEST POSITIVE RATIO:	0.3175 (107.0/(107.0+230.0))
 Confusion table
           ||======================
 PREDICTED || positive | negative | Recall
 TRUTH     ||======================
- positive ||       97 |       10 | 0.9065
- negative ||       10 |      220 | 0.9565
+ positive ||       99 |        8 | 0.9252
+ negative ||        8 |      222 | 0.9652
           ||======================
-Precision ||   0.9065 |   0.9565 |
-OVERALL 0/1 ACCURACY: 0.940653
-LOG LOSS/instance:  0.297583
+Precision ||   0.9252 |   0.9652 |
+OVERALL 0/1 ACCURACY: 0.952522
+LOG LOSS/instance:  0.288593
 Test-set entropy (prior Log-Loss/instance): 0.901650
-LOG-LOSS REDUCTION (RIG): 0.669957
-AUC:                0.991833
+LOG-LOSS REDUCTION (RIG): 0.679928
+AUC:                0.992950
 
 OVERALL RESULTS
 ---------------------------------------
-AUC:                0.988388 (0.0034)
-Accuracy:           0.944083 (0.0034)
-Positive precision: 0.919436 (0.0129)
-Positive recall:    0.915958 (0.0094)
-Negative precision: 0.956427 (0.0001)
-Negative recall:    0.958524 (0.0020)
-Log-loss:           0.300661 (0.0031)
-Log-loss reduction: 0.675250 (0.0053)
-F1 Score:           0.917691 (0.0111)
-AUPRC:              0.972137 (0.0107)
+AUC:                0.989111 (0.0038)
+Accuracy:           0.951399 (0.0011)
+Positive precision: 0.929035 (0.0038)
+Positive recall:    0.929035 (0.0038)
+Negative precision: 0.962872 (0.0023)
+Negative recall:    0.962872 (0.0023)
+Log-loss:           0.293647 (0.0051)
+Log-loss reduction: 0.682885 (0.0030)
+F1 Score:           0.929035 (0.0038)
+AUPRC:              0.973375 (0.0114)
 
 ---------------------------------------
 Physical memory usage(MB): %Number%

test/BaselineOutput/Common/LightGBM/LightGBMDart-CV-breast-cancer.dart-rp.txt

@@ -1,4 +1,4 @@
 LightGBM
 AUC	Accuracy	Positive precision	Positive recall	Negative precision	Negative recall	Log-loss	Log-loss reduction	F1 Score	AUPRC	/iter	/lr	/nl	/mil	/booster	/nt	Learner Name	Train Dataset	Test Dataset	Results File	Run Time	Physical Memory	Virtual Memory	Command Line	Settings	
-0.988388	0.944083	0.919436	0.915958	0.956427	0.958524	0.300661	0.67525	0.917691	0.972137	10	0.2	20	10	dart	1	LightGBM	%Data%		%Output%	99	0	0	maml.exe CV tr=LightGBM{nt=1 iter=10 booster=dart lr=0.2 mil=10 nl=20} threads=- cache=- dout=%Output% loader=Text{sparse- col=Attr:TX:6 col=Label:0 col=Features:1-5,6,7-9} data=%Data% seed=1	/iter:10;/lr:0.2;/nl:20;/mil:10;/booster:dart;/nt:1	
+0.989111	0.951399	0.929035	0.929035	0.962872	0.962872	0.293647	0.682885	0.929035	0.973375	10	0.2	20	10	dart	1	LightGBM	%Data%		%Output%	99	0	0	maml.exe CV tr=LightGBM{nt=1 iter=10 booster=dart lr=0.2 mil=10 nl=20} threads=- cache=- dout=%Output% loader=Text{sparse- col=Attr:TX:6 col=Label:0 col=Features:1-5,6,7-9} data=%Data% seed=1	/iter:10;/lr:0.2;/nl:20;/mil:10;/booster:dart;/nt:1