T M V A Multiclass

This macro provides a simple example for the training and testing of the TMVA multiclass classification

  • Project : TMVA - a Root-integrated toolkit for multivariate data analysis
  • Package : TMVA
  • Root Macro: TMVAMulticlass

Author: Andreas Hoecker
This notebook tutorial was automatically generated with ROOTBOOK-izer from the macro found in the ROOT repository on Monday, December 06, 2021 at 10:51 AM.

In [1]:
%%cpp -d
#include <cstdlib>
#include <iostream>
#include <map>
#include <string>

#include "TFile.h"
#include "TTree.h"
#include "TString.h"
#include "TSystem.h"
#include "TROOT.h"


#include "TMVA/Tools.h"
#include "TMVA/Factory.h"
#include "TMVA/DataLoader.h"
#include "TMVA/TMVAMultiClassGui.h"


using namespace TMVA;

Arguments are defined.

In [2]:
TString myMethodList = "";

This loads the library

In [3]:
TMVA::Tools::Instance();

To get access to the gui and all tmva macros

 TString tmva_dir(TString(gRootDir) + "/tmva");
 if(gSystem->Getenv("TMVASYS"))
    tmva_dir = TString(gSystem->Getenv("TMVASYS"));
 gROOT->SetMacroPath(tmva_dir + "/test/:" + gROOT->GetMacroPath() );
 gROOT->ProcessLine(".L TMVAMultiClassGui.C");

Default MVA methods to be trained + tested

In [4]:
std::map<std::string,int> Use;
Use["MLP"]             = 1;
Use["BDTG"]            = 1;
#ifdef R__HAS_TMVAGPU
Use["DL_CPU"]          = 1;
Use["DL_GPU"]          = 1;
#else
Use["DL_CPU"]          = 1;
Use["DL_GPU"]          = 0;
#endif
Use["FDA_GA"]          = 0;
Use["PDEFoam"]         = 1;

In [5]:
std::cout << std::endl;
std::cout << "==> Start TMVAMulticlass" << std::endl;

if (myMethodList != "") {
   for (std::map<std::string,int>::iterator it = Use.begin(); it != Use.end(); it++) it->second = 0;

   std::vector<TString> mlist = TMVA::gTools().SplitString( myMethodList, ',' );
   for (UInt_t i=0; i<mlist.size(); i++) {
      std::string regMethod(mlist[i]);

      if (Use.find(regMethod) == Use.end()) {
         std::cout << "Method \"" << regMethod << "\" not known in TMVA under this name. Choose among the following:" << std::endl;
         for (std::map<std::string,int>::iterator it = Use.begin(); it != Use.end(); it++) std::cout << it->first << " ";
         std::cout << std::endl;
         return;
      }
      Use[regMethod] = 1;
   }
}
==> Start TMVAMulticlass

Create a new root output file.

In [6]:
TString outfileName = "TMVAMulticlass.root";
TFile* outputFile = TFile::Open( outfileName, "RECREATE" );

TMVA::Factory *factory = new TMVA::Factory( "TMVAMulticlass", outputFile,
                                            "!V:!Silent:Color:!DrawProgressBar:Transformations=I;D;P;G,D:AnalysisType=multiclass" );
TMVA::DataLoader *dataloader=new TMVA::DataLoader("dataset");

dataloader->AddVariable( "var1", 'F' );
dataloader->AddVariable( "var2", "Variable 2", "", 'F' );
dataloader->AddVariable( "var3", "Variable 3", "units", 'F' );
dataloader->AddVariable( "var4", "Variable 4", "units", 'F' );

TFile *input(0);
TString fname = "./tmva_example_multiclass.root";
if (!gSystem->AccessPathName( fname )) {
   input = TFile::Open( fname ); // check if file in local directory exists
}
else {
   TFile::SetCacheFileDir(".");
   input = TFile::Open("http://root.cern.ch/files/tmva_multiclass_example.root", "CACHEREAD");
}
if (!input) {
   std::cout << "ERROR: could not open data file" << std::endl;
   exit(1);
}
std::cout << "--- TMVAMulticlass: Using input file: " << input->GetName() << std::endl;

TTree *signalTree  = (TTree*)input->Get("TreeS");
TTree *background0 = (TTree*)input->Get("TreeB0");
TTree *background1 = (TTree*)input->Get("TreeB1");
TTree *background2 = (TTree*)input->Get("TreeB2");

gROOT->cd( outfileName+TString(":/") );
dataloader->AddTree    (signalTree,"Signal");
dataloader->AddTree    (background0,"bg0");
dataloader->AddTree    (background1,"bg1");
dataloader->AddTree    (background2,"bg2");

dataloader->PrepareTrainingAndTestTree( "", "SplitMode=Random:NormMode=NumEvents:!V" );

if (Use["BDTG"]) // gradient boosted decision trees
   factory->BookMethod( dataloader,  TMVA::Types::kBDT, "BDTG", "!H:!V:NTrees=1000:BoostType=Grad:Shrinkage=0.10:UseBaggedBoost:BaggedSampleFraction=0.50:nCuts=20:MaxDepth=2");
if (Use["MLP"]) // neural network
   factory->BookMethod( dataloader,  TMVA::Types::kMLP, "MLP", "!H:!V:NeuronType=tanh:NCycles=1000:HiddenLayers=N+5,5:TestRate=5:EstimatorType=MSE");
if (Use["FDA_GA"]) // functional discriminant with GA minimizer
   factory->BookMethod( dataloader,  TMVA::Types::kFDA, "FDA_GA", "H:!V:Formula=(0)+(1)*x0+(2)*x1+(3)*x2+(4)*x3:ParRanges=(-1,1);(-10,10);(-10,10);(-10,10);(-10,10):FitMethod=GA:PopSize=300:Cycles=3:Steps=20:Trim=True:SaveBestGen=1" );
if (Use["PDEFoam"]) // PDE-Foam approach
   factory->BookMethod( dataloader,  TMVA::Types::kPDEFoam, "PDEFoam", "!H:!V:TailCut=0.001:VolFrac=0.0666:nActiveCells=500:nSampl=2000:nBin=5:Nmin=100:Kernel=None:Compress=T" );


if (Use["DL_CPU"]) {
   TString layoutString("Layout=TANH|100,TANH|50,TANH|10,LINEAR");
   TString trainingStrategyString("TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,"
                                  "TestRepetitions=1,ConvergenceSteps=10,BatchSize=100");
   TString nnOptions("!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:"
                     "WeightInitialization=XAVIERUNIFORM:Architecture=GPU");
   nnOptions.Append(":");
   nnOptions.Append(layoutString);
   nnOptions.Append(":");
   nnOptions.Append(trainingStrategyString);
   factory->BookMethod(dataloader, TMVA::Types::kDL, "DL_CPU", nnOptions);
}
if (Use["DL_GPU"]) {
   TString layoutString("Layout=TANH|100,TANH|50,TANH|10,LINEAR");
   TString trainingStrategyString("TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,"
                                  "TestRepetitions=1,ConvergenceSteps=10,BatchSize=100");
   TString nnOptions("!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:"
                     "WeightInitialization=XAVIERUNIFORM:Architecture=GPU");
   nnOptions.Append(":");
   nnOptions.Append(layoutString);
   nnOptions.Append(":");
   nnOptions.Append(trainingStrategyString);
   factory->BookMethod(dataloader, TMVA::Types::kDL, "DL_GPU", nnOptions);
}
--- TMVAMulticlass: Using input file: ./files/tmva_multiclass_example.root
DataSetInfo              : [dataset] : Added class "Signal"
                         : Add Tree TreeS of type Signal with 2000 events
DataSetInfo              : [dataset] : Added class "bg0"
                         : Add Tree TreeB0 of type bg0 with 2000 events
DataSetInfo              : [dataset] : Added class "bg1"
                         : Add Tree TreeB1 of type bg1 with 2000 events
DataSetInfo              : [dataset] : Added class "bg2"
                         : Add Tree TreeB2 of type bg2 with 2000 events
                         : Dataset[dataset] : Class index : 0  name : Signal
                         : Dataset[dataset] : Class index : 1  name : bg0
                         : Dataset[dataset] : Class index : 2  name : bg1
                         : Dataset[dataset] : Class index : 3  name : bg2
Factory                  : Booking method: BDTG
                         : 
                         : the option NegWeightTreatment=InverseBoostNegWeights does not exist for BoostType=Grad
                         : --> change to new default NegWeightTreatment=Pray
                         : Rebuilding Dataset dataset
                         : Building event vectors for type 2 Signal
                         : Dataset[dataset] :  create input formulas for tree TreeS
                         : Building event vectors for type 2 bg0
                         : Dataset[dataset] :  create input formulas for tree TreeB0
                         : Building event vectors for type 2 bg1
                         : Dataset[dataset] :  create input formulas for tree TreeB1
                         : Building event vectors for type 2 bg2
                         : Dataset[dataset] :  create input formulas for tree TreeB2
DataSetFactory           : [dataset] : Number of events in input trees
                         : 
                         : 
                         : 
                         : 
                         : Number of training and testing events
                         : ---------------------------------------------------------------------------
                         : Signal -- training events            : 1000
                         : Signal -- testing events             : 1000
                         : Signal -- training and testing events: 2000
                         : bg0    -- training events            : 1000
                         : bg0    -- testing events             : 1000
                         : bg0    -- training and testing events: 2000
                         : bg1    -- training events            : 1000
                         : bg1    -- testing events             : 1000
                         : bg1    -- training and testing events: 2000
                         : bg2    -- training events            : 1000
                         : bg2    -- testing events             : 1000
                         : bg2    -- training and testing events: 2000
                         : 
DataSetInfo              : Correlation matrix (Signal):
                         : ----------------------------------------
                         :             var1    var2    var3    var4
                         :    var1:  +1.000  +0.397  +0.623  +0.832
                         :    var2:  +0.397  +1.000  +0.716  +0.737
                         :    var3:  +0.623  +0.716  +1.000  +0.859
                         :    var4:  +0.832  +0.737  +0.859  +1.000
                         : ----------------------------------------
DataSetInfo              : Correlation matrix (bg0):
                         : ----------------------------------------
                         :             var1    var2    var3    var4
                         :    var1:  +1.000  +0.365  +0.592  +0.811
                         :    var2:  +0.365  +1.000  +0.708  +0.740
                         :    var3:  +0.592  +0.708  +1.000  +0.859
                         :    var4:  +0.811  +0.740  +0.859  +1.000
                         : ----------------------------------------
DataSetInfo              : Correlation matrix (bg1):
                         : ----------------------------------------
                         :             var1    var2    var3    var4
                         :    var1:  +1.000  +0.407  +0.610  +0.834
                         :    var2:  +0.407  +1.000  +0.710  +0.741
                         :    var3:  +0.610  +0.710  +1.000  +0.851
                         :    var4:  +0.834  +0.741  +0.851  +1.000
                         : ----------------------------------------
DataSetInfo              : Correlation matrix (bg2):
                         : ----------------------------------------
                         :             var1    var2    var3    var4
                         :    var1:  +1.000  -0.647  -0.016  -0.013
                         :    var2:  -0.647  +1.000  +0.015  +0.002
                         :    var3:  -0.016  +0.015  +1.000  -0.024
                         :    var4:  -0.013  +0.002  -0.024  +1.000
                         : ----------------------------------------
DataSetFactory           : [dataset] :  
                         : 
Factory                  : Booking method: MLP
                         : 
MLP                      : Building Network. 
                         : Initializing weights
Factory                  : Booking method: PDEFoam
                         : 
Factory                  : Booking method: DL_CPU
                         : 
                         : Parsing option string: 
                         : ... "!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:WeightInitialization=XAVIERUNIFORM:Architecture=GPU:Layout=TANH|100,TANH|50,TANH|10,LINEAR:TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,TestRepetitions=1,ConvergenceSteps=10,BatchSize=100"
                         : The following options are set:
                         : - By User:
                         :     <none>
                         : - Default:
                         :     Boost_num: "0" [Number of times the classifier will be boosted]
                         : Parsing option string: 
                         : ... "!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:WeightInitialization=XAVIERUNIFORM:Architecture=GPU:Layout=TANH|100,TANH|50,TANH|10,LINEAR:TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,TestRepetitions=1,ConvergenceSteps=10,BatchSize=100"
                         : The following options are set:
                         : - By User:
                         :     V: "True" [Verbose output (short form of "VerbosityLevel" below - overrides the latter one)]
                         :     VarTransform: "N" [List of variable transformations performed before training, e.g., "D_Background,P_Signal,G,N_AllClasses" for: "Decorrelation, PCA-transformation, Gaussianisation, Normalisation, each for the given class of events ('AllClasses' denotes all events of all classes, if no class indication is given, 'All' is assumed)"]
                         :     H: "False" [Print method-specific help message]
                         :     Layout: "TANH|100,TANH|50,TANH|10,LINEAR" [Layout of the network.]
                         :     ErrorStrategy: "CROSSENTROPY" [Loss function: Mean squared error (regression) or cross entropy (binary classification).]
                         :     WeightInitialization: "XAVIERUNIFORM" [Weight initialization strategy]
                         :     Architecture: "GPU" [Which architecture to perform the training on.]
                         :     TrainingStrategy: "Optimizer=ADAM,LearningRate=1e-3,TestRepetitions=1,ConvergenceSteps=10,BatchSize=100" [Defines the training strategies.]
                         : - Default:
                         :     VerbosityLevel: "Default" [Verbosity level]
                         :     CreateMVAPdfs: "False" [Create PDFs for classifier outputs (signal and background)]
                         :     IgnoreNegWeightsInTraining: "False" [Events with negative weights are ignored in the training (but are included for testing and performance evaluation)]
                         :     InputLayout: "0|0|0" [The Layout of the input]
                         :     BatchLayout: "0|0|0" [The Layout of the batch]
                         :     RandomSeed: "0" [Random seed used for weight initialization and batch shuffling]
                         :     ValidationSize: "20%" [Part of the training data to use for validation. Specify as 0.2 or 20% to use a fifth of the data set as validation set. Specify as 100 to use exactly 100 events. (Default: 20%)]
DL_CPU                   : [dataset] : Create Transformation "N" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
<ERROR>                         : CUDA backend not enabled. Please make sure you have CUDA installed and it was successfully detected by CMAKE by using -Dtmva-gpu=On  
                         : Will now use instead the CPU architecture !
                         : Will now use the CPU architecture with BLAS and IMT support !
[TFile::Cp] Total 0.12 MB	|====================| 100.00 % [62.0 MB/s]
Info in <TFile::OpenFromCache>: using local cache copy of http://root.cern.ch/files/tmva_multiclass_example.root [./files/tmva_multiclass_example.root]

Train mvas using the set of training events

In [7]:
factory->TrainAllMethods();
Factory                  : Train all methods
Factory                  : [dataset] : Create Transformation "I" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
Factory                  : [dataset] : Create Transformation "D" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
Factory                  : [dataset] : Create Transformation "P" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
Factory                  : [dataset] : Create Transformation "G" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
Factory                  : [dataset] : Create Transformation "D" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
TFHandler_Factory        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.047647     1.0025   [    -3.6592     3.2645 ]
                         :     var2:    0.32647     1.0646   [    -3.6891     3.7877 ]
                         :     var3:    0.11493     1.1230   [    -4.5727     4.5640 ]
                         :     var4:  -0.076531     1.2652   [    -4.8486     5.0412 ]
                         : -----------------------------------------------------------
                         : Preparing the Decorrelation transformation...
TFHandler_Factory        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.082544     1.0000   [    -3.6274     3.1017 ]
                         :     var2:    0.36715     1.0000   [    -3.3020     3.4950 ]
                         :     var3:   0.066865     1.0000   [    -2.9882     3.3086 ]
                         :     var4:   -0.20593     1.0000   [    -3.3088     2.8423 ]
                         : -----------------------------------------------------------
                         : Preparing the Principle Component (PCA) transformation...
TFHandler_Factory        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1: 5.7502e-10     1.8064   [    -8.0344     7.8312 ]
                         :     var2:-1.6078e-11    0.90130   [    -2.6765     2.7523 ]
                         :     var3: 3.0841e-10    0.73386   [    -2.6572     2.2255 ]
                         :     var4:-2.6886e-10    0.62168   [    -1.7384     2.2297 ]
                         : -----------------------------------------------------------
                         : Preparing the Gaussian transformation...
                         : Preparing the Decorrelation transformation...
TFHandler_Factory        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.013510     1.0000   [    -2.6520     6.2074 ]
                         :     var2:  0.0096839     1.0000   [    -2.8402     6.3073 ]
                         :     var3:   0.010397     1.0000   [    -3.0251     5.8860 ]
                         :     var4:  0.0053980     1.0000   [    -3.0998     5.7078 ]
                         : -----------------------------------------------------------
                         : Ranking input variables (method unspecific)...
Factory                  : Train method: BDTG for Multiclass classification
                         : 
                         : Training 1000 Decision Trees ... patience please
                         : Elapsed time for training with 4000 events: 5.51 sec         
                         : Dataset[dataset] : Create results for training
                         : Dataset[dataset] : Multiclass evaluation of BDTG on training sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 1.9 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating xml weight file: dataset/weights/TMVAMulticlass_BDTG.weights.xml
                         : Creating standalone class: dataset/weights/TMVAMulticlass_BDTG.class.C
                         : TMVAMulticlass.root:/dataset/Method_BDT/BDTG
Factory                  : Training finished
                         : 
Factory                  : Train method: MLP for Multiclass classification
                         : 
                         : Training Network
                         : 
                         : Elapsed time for training with 4000 events: 25.6 sec         
                         : Dataset[dataset] : Create results for training
                         : Dataset[dataset] : Multiclass evaluation of MLP on training sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.0098 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating xml weight file: dataset/weights/TMVAMulticlass_MLP.weights.xml
                         : Creating standalone class: dataset/weights/TMVAMulticlass_MLP.class.C
                         : Write special histos to file: TMVAMulticlass.root:/dataset/Method_MLP/MLP
Factory                  : Training finished
                         : 
Factory                  : Train method: PDEFoam for Multiclass classification
                         : 
                         : Build up multiclass foam 0
                         : Elapsed time: 0.668 sec                                 
                         : Build up multiclass foam 1
                         : Elapsed time: 0.675 sec                                 
                         : Build up multiclass foam 2
                         : Elapsed time: 0.683 sec                                 
                         : Build up multiclass foam 3
                         : Elapsed time: 0.473 sec                                 
                         : Elapsed time for training with 4000 events: 2.67 sec         
                         : Dataset[dataset] : Create results for training
                         : Dataset[dataset] : Multiclass evaluation of PDEFoam on training sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.116 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating xml weight file: dataset/weights/TMVAMulticlass_PDEFoam.weights.xml
                         : writing foam MultiClassFoam0 to file
                         : writing foam MultiClassFoam1 to file
                         : writing foam MultiClassFoam2 to file
                         : writing foam MultiClassFoam3 to file
                         : Foams written to file: dataset/weights/TMVAMulticlass_PDEFoam.weights_foams.root
                         : Creating standalone class: dataset/weights/TMVAMulticlass_PDEFoam.class.C
Factory                  : Training finished
                         : 
Factory                  : Train method: DL_CPU for Multiclass classification
                         : 
TFHandler_DL_CPU         : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070769    0.28960   [    -1.0000     1.0000 ]
                         :     var2:   0.074130    0.28477   [    -1.0000     1.0000 ]
                         :     var3:   0.026106    0.24582   [    -1.0000     1.0000 ]
                         :     var4:  -0.034951    0.25587   [    -1.0000     1.0000 ]
                         : -----------------------------------------------------------
                         : Start of deep neural network training on CPU using MT,  nthreads = 1
                         : 
TFHandler_DL_CPU         : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070769    0.28960   [    -1.0000     1.0000 ]
                         :     var2:   0.074130    0.28477   [    -1.0000     1.0000 ]
                         :     var3:   0.026106    0.24582   [    -1.0000     1.0000 ]
                         :     var4:  -0.034951    0.25587   [    -1.0000     1.0000 ]
                         : -----------------------------------------------------------
                         : *****   Deep Learning Network *****
DEEP NEURAL NETWORK:   Depth = 4  Input = ( 1, 1, 4 )  Batch size = 100  Loss function = C
	Layer 0	 DENSE Layer: 	 ( Input =     4 , Width =   100 ) 	Output = (  1 ,   100 ,   100 ) 	 Activation Function = Tanh
	Layer 1	 DENSE Layer: 	 ( Input =   100 , Width =    50 ) 	Output = (  1 ,   100 ,    50 ) 	 Activation Function = Tanh
	Layer 2	 DENSE Layer: 	 ( Input =    50 , Width =    10 ) 	Output = (  1 ,   100 ,    10 ) 	 Activation Function = Tanh
	Layer 3	 DENSE Layer: 	 ( Input =    10 , Width =     4 ) 	Output = (  1 ,   100 ,     4 ) 	 Activation Function = Identity
                         : Using 3200 events for training and 800 for testing
                         : Compute initial loss  on the validation data 
                         : Training phase 1 of 1:  Optimizer ADAM (beta1=0.9,beta2=0.999,eps=1e-07) Learning rate = 0.001 regularization 0 minimum error = 0.695616
                         : --------------------------------------------------------------
                         :      Epoch |   Train Err.   Val. Err.  t(s)/epoch   t(s)/Loss   nEvents/s Conv. Steps
                         : --------------------------------------------------------------
                         :    Start epoch iteration ...
                         :          1 Minimum Test error found - save the configuration 
                         :          1 |     0.597304    0.522158   0.0774837  0.00679087     45266.3           0
                         :          2 Minimum Test error found - save the configuration 
                         :          2 |     0.490409    0.468759   0.0834249  0.00676786     41744.4           0
                         :          3 Minimum Test error found - save the configuration 
                         :          3 |     0.454151    0.439093   0.0784392  0.00676175     44644.4           0
                         :          4 Minimum Test error found - save the configuration 
                         :          4 |     0.432276     0.41912   0.0788645  0.00679717     44402.9           0
                         :          5 Minimum Test error found - save the configuration 
                         :          5 |     0.415428    0.402763   0.0791809  0.00682297     44224.6           0
                         :          6 Minimum Test error found - save the configuration 
                         :          6 |     0.401166    0.389133   0.0795744  0.00688221     44021.2           0
                         :          7 Minimum Test error found - save the configuration 
                         :          7 |     0.389709    0.377451   0.0797014  0.00689821     43954.1           0
                         :          8 Minimum Test error found - save the configuration 
                         :          8 |     0.380432    0.368194   0.0854597  0.00695364     40761.2           0
                         :          9 Minimum Test error found - save the configuration 
                         :          9 |      0.37217    0.362431   0.0799754   0.0069234     43804.4           0
                         :         10 Minimum Test error found - save the configuration 
                         :         10 |     0.364649    0.355291   0.0801383  0.00694583     43720.4           0
                         :         11 Minimum Test error found - save the configuration 
                         :         11 |     0.358332    0.348185   0.0858279  0.00698866     40588.9           0
                         :         12 Minimum Test error found - save the configuration 
                         :         12 |      0.35203    0.344987   0.0804439  0.00698885     43564.1           0
                         :         13 Minimum Test error found - save the configuration 
                         :         13 |     0.346576    0.336884   0.0806055  0.00705023     43504.7           0
                         :         14 Minimum Test error found - save the configuration 
                         :         14 |     0.341285     0.33019   0.0808038  0.00701706     43368.2           0
                         :         15 Minimum Test error found - save the configuration 
                         :         15 |     0.334833    0.327835   0.0808745  0.00701969     43328.2           0
                         :         16 Minimum Test error found - save the configuration 
                         :         16 |     0.328739    0.318213   0.0809734  0.00703752     43280.7           0
                         :         17 Minimum Test error found - save the configuration 
                         :         17 |      0.32327    0.312659   0.0866112  0.00707241       40232           0
                         :         18 Minimum Test error found - save the configuration 
                         :         18 |     0.316606    0.305142   0.0812624  0.00706165     43126.3           0
                         :         19 Minimum Test error found - save the configuration 
                         :         19 |     0.310506    0.299109   0.0812668   0.0070654     43125.9           0
                         :         20 Minimum Test error found - save the configuration 
                         :         20 |     0.305639    0.297377   0.0815857  0.00710138     42962.1           0
                         :         21 |     0.301007    0.297919    0.081541  0.00703537     42949.8           1
                         :         22 Minimum Test error found - save the configuration 
                         :         22 |     0.298163    0.289633   0.0817485  0.00713269     42886.3           0
                         :         23 Minimum Test error found - save the configuration 
                         :         23 |     0.294933    0.285856   0.0872924  0.00714715     39927.5           0
                         :         24 Minimum Test error found - save the configuration 
                         :         24 |     0.290818    0.279593   0.0818076  0.00711568     42842.7           0
                         :         25 Minimum Test error found - save the configuration 
                         :         25 |     0.287524    0.279143   0.0818854  0.00713215     42807.5           0
                         :         26 Minimum Test error found - save the configuration 
                         :         26 |     0.284723    0.272313   0.0821651  0.00714974     42657.9           0
                         :         27 |     0.282887    0.272548   0.0819597  0.00707976     42735.1           1
                         :         28 |      0.28032    0.272859   0.0820016  0.00708063     42711.7           2
                         :         29 Minimum Test error found - save the configuration 
                         :         29 |       0.2784    0.268143   0.0877031   0.0071899       39745           0
                         :         30 Minimum Test error found - save the configuration 
                         :         30 |     0.276262    0.267761   0.0821845  0.00714737     42645.5           0
                         :         31 Minimum Test error found - save the configuration 
                         :         31 |     0.273774    0.263942   0.0821147  0.00715141     42687.6           0
                         :         32 Minimum Test error found - save the configuration 
                         :         32 |     0.272566    0.261333   0.0823159  0.00718977       42595           0
                         :         33 |     0.271035    0.261731   0.0822685   0.0071174     42580.9           1
                         :         34 Minimum Test error found - save the configuration 
                         :         34 |     0.268251    0.255583   0.0823558   0.0072564     42610.2           0
                         :         35 |     0.266858    0.257978   0.0878678  0.00712114     39630.1           1
                         :         36 |     0.265592    0.259902   0.0823332  0.00711213     42541.3           2
                         :         37 Minimum Test error found - save the configuration 
                         :         37 |     0.264023    0.254156   0.0823428  0.00718634     42577.8           0
                         :         38 |     0.263148    0.255776   0.0824985  0.00712541     42455.5           1
                         :         39 Minimum Test error found - save the configuration 
                         :         39 |     0.261071    0.248841   0.0825055  0.00720998     42499.2           0
                         :         40 |     0.260485    0.253057   0.0823971  0.00711795     42508.4           1
                         :         41 |     0.260384    0.254589   0.0879361  0.00711948     39595.8           2
                         :         42 Minimum Test error found - save the configuration 
                         :         42 |     0.257806    0.247609   0.0825324  0.00720158     42479.3           0
                         :         43 |     0.256476    0.250376   0.0824321  0.00712085     42490.3           1
                         :         44 |     0.256244    0.249063    0.087967  0.00714547     39593.4           2
                         :         45 Minimum Test error found - save the configuration 
                         :         45 |      0.25454    0.245295   0.0826965  0.00724066     42408.9           0
                         :         46 Minimum Test error found - save the configuration 
                         :         46 |     0.251213    0.245175    0.082679  0.00727269     42436.8           0
                         :         47 Minimum Test error found - save the configuration 
                         :         47 |     0.251539    0.241517   0.0826818  0.00720249     42395.7           0
                         :         48 |     0.249799    0.242497   0.0826172  0.00714901       42402           1
                         :         49 |     0.248668    0.241583   0.0826225   0.0071469     42397.8           2
                         :         50 Minimum Test error found - save the configuration 
                         :         50 |     0.247279    0.237702   0.0882791   0.0072698     39501.7           0
                         :         51 |     0.245812    0.241902   0.0826963  0.00716196     42364.8           1
                         :         52 |     0.246269    0.238754    0.082801  0.00715057     42299.8           2
                         :         53 Minimum Test error found - save the configuration 
                         :         53 |     0.243819    0.237344   0.0830212  0.00725953     42237.7           0
                         :         54 |     0.242433    0.239087   0.0828715  0.00717207     42272.4           1
                         :         55 Minimum Test error found - save the configuration 
                         :         55 |     0.242132     0.23681   0.0830442  0.00728305       42238           0
                         :         56 Minimum Test error found - save the configuration 
                         :         56 |     0.240703    0.233793   0.0885974  0.00727546     39349.8           0
                         :         57 Minimum Test error found - save the configuration 
                         :         57 |      0.23941    0.232329   0.0829496  0.00722611       42259           0
                         :         58 Minimum Test error found - save the configuration 
                         :         58 |     0.237635    0.231642   0.0847931  0.00730051     41294.3           0
                         :         59 |     0.237061    0.232498   0.0831547  0.00717453     42116.2           1
                         :         60 |     0.235519    0.236265   0.0829835  0.00717735     42212.9           2
                         :         61 |     0.235014    0.232064   0.0829036  0.00717511     42256.2           3
                         :         62 |     0.233812    0.233544   0.0885525  0.00721297     39341.3           4
                         :         63 Minimum Test error found - save the configuration 
                         :         63 |     0.232634    0.227488   0.0831275  0.00727251     42185.8           0
                         :         64 |     0.231558     0.22887   0.0830558  0.00720044     42185.5           1
                         :         65 Minimum Test error found - save the configuration 
                         :         65 |      0.23095    0.227281   0.0831744  0.00726852     42157.5           0
                         :         66 |     0.230471    0.233835   0.0830823  0.00718741     42163.6           1
                         :         67 |     0.231067    0.229637    0.083016  0.00717991     42196.3           2
                         :         68 Minimum Test error found - save the configuration 
                         :         68 |     0.229309     0.22625   0.0886864  0.00730523     39321.2           0
                         :         69 Minimum Test error found - save the configuration 
                         :         69 |     0.227516    0.224026   0.0832093  0.00725981     42133.3           0
                         :         70 |     0.227211     0.22746   0.0830955  0.00719087     42158.2           1
                         :         71 |     0.227322    0.229836   0.0831328   0.0071943     42139.3           2
                         :         72 |     0.226712    0.227303    0.083061  0.00718687     42175.1           3
                         :         73 Minimum Test error found - save the configuration 
                         :         73 |     0.225912    0.223372   0.0832397  0.00730346     42140.6           0
                         :         74 |      0.22475    0.228457   0.0885391  0.00720213     39342.5           1
                         :         75 Minimum Test error found - save the configuration 
                         :         75 |     0.224297    0.221799   0.0833174   0.0073102     42101.3           0
                         :         76 |     0.223029    0.227199   0.0832482  0.00721607     42087.5           1
                         :         77 |     0.223301    0.223684   0.0832586  0.00720191     42073.9           2
                         :         78 |     0.222636    0.229846   0.0832091    0.007211     42106.3           3
                         :         79 Minimum Test error found - save the configuration 
                         :         79 |     0.223394    0.221415    0.083289  0.00728462     42102.8           0
                         :         80 |     0.223529    0.222094   0.0887836  0.00723653     39241.1           1
                         :         81 |     0.220818    0.221865   0.0832152  0.00720133     42097.6           2
                         :         82 Minimum Test error found - save the configuration 
                         :         82 |     0.219487     0.21913   0.0888921   0.0127848     42045.9           0
                         :         83 |     0.219802    0.222954   0.0833347  0.00720752     42034.9           1
                         :         84 |     0.219634    0.225046   0.0832948  0.00720777     42057.1           2
                         :         85 |     0.220655    0.221942   0.0833924  0.00722083     42010.4           3
                         :         86 |     0.217743    0.221989   0.0835128  0.00723893     41954.1           4
                         :         87 |     0.217976     0.21923   0.0833888  0.00722408     42014.2           5
                         :         88 |     0.220568    0.223771   0.0888962  0.00725275     39194.8           6
                         :         89 |     0.218072    0.220376   0.0833971  0.00723071     42013.3           7
                         :         90 Minimum Test error found - save the configuration 
                         :         90 |     0.216195    0.216997   0.0834831  0.00732232     42016.4           0
                         :         91 |     0.216021    0.220576   0.0834574  0.00722826     41978.7           1
                         :         92 |     0.215071    0.217822   0.0833665  0.00723646     42033.3           2
                         :         93 Minimum Test error found - save the configuration 
                         :         93 |     0.215093    0.216673   0.0834587   0.0072864       42010           0
                         :         94 Minimum Test error found - save the configuration 
                         :         94 |     0.215688    0.215327    0.088896  0.00731678     39225.7           0
                         :         95 |     0.214218    0.217976   0.0833687  0.00721532     42020.5           1
                         :         96 |     0.213946    0.215956   0.0833611    0.007208     42020.6           2
                         :         97 |     0.213378    0.217035   0.0834671  0.00722834     41973.4           3
                         :         98 Minimum Test error found - save the configuration 
                         :         98 |     0.213936    0.212797   0.0834221  0.00730188     42038.8           0
                         :         99 |     0.212378    0.218232   0.0834296  0.00723797     41999.4           1
                         :        100 |     0.211941    0.216577    0.089933   0.0081407     39123.5           2
                         :        101 |     0.212026    0.216444   0.0835544   0.0072324     41927.6           3
                         :        102 |     0.211864    0.215142    0.083467  0.00722543     41971.9           4
                         :        103 |     0.211397    0.217738    0.083445  0.00722326     41982.8           5
                         :        104 |     0.212796    0.214716   0.0834153  0.00723351     42004.8           6
                         :        105 |     0.210558    0.215806   0.0833774  0.00722788     42022.6           7
                         :        106 Minimum Test error found - save the configuration 
                         :        106 |     0.210595    0.211269   0.0890448  0.00734594     39168.2           0
                         :        107 |     0.210147    0.216867    0.083464  0.00723141     41976.8           1
                         :        108 |     0.209146    0.212258    0.083528  0.00724229     41947.5           2
                         :        109 Minimum Test error found - save the configuration 
                         :        109 |     0.209576    0.211032   0.0837837  0.00733974     41860.7           0
                         :        110 |     0.209159    0.215979   0.0834572  0.00723069     41980.1           1
                         :        111 |     0.209015    0.216405    0.083409   0.0072121     41996.5           2
                         :        112 Minimum Test error found - save the configuration 
                         :        112 |     0.209132    0.210583   0.0892296  0.00737799     39095.2           0
                         :        113 Minimum Test error found - save the configuration 
                         :        113 |     0.209101    0.208944   0.0836542  0.00732372       41923           0
                         :        114 |     0.207761    0.213929   0.0835014   0.0072326     41956.9           1
                         :        115 |     0.207503    0.215393   0.0891302  0.00725904     39085.8           2
                         :        116 |     0.206906    0.212021   0.0837161  0.00725495     41851.3           3
                         :        117 |       0.2068    0.212291   0.0836445  0.00724894     41887.3           4
                         :        118 |     0.206242    0.213229   0.0836541   0.0072451     41879.9           5
                         :        119 |     0.205988    0.212385   0.0834937  0.00723883     41964.5           6
                         :        120 |     0.207363    0.211808    0.083563  0.00724518     41929.9           7
                         :        121 |     0.205325    0.212163   0.0892845  0.00724589       39006           8
                         :        122 |       0.2059    0.210297   0.0834737  0.00722639     41968.7           9
                         :        123 |     0.205118    0.209336   0.0835346  0.00730465     41978.2          10
                         :        124 |     0.204695    0.210202   0.0835364  0.00724086     41942.2          11
                         : 
                         : Elapsed time for training with 4000 events: 10.4 sec         
                         : Dataset[dataset] : Create results for training
                         : Dataset[dataset] : Multiclass evaluation of DL_CPU on training sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.106 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating xml weight file: dataset/weights/TMVAMulticlass_DL_CPU.weights.xml
                         : Creating standalone class: dataset/weights/TMVAMulticlass_DL_CPU.class.C
Factory                  : Training finished
                         : 
                         : Ranking input variables (method specific)...
BDTG                     : Ranking result (top variable is best ranked)
                         : --------------------------------------
                         : Rank : Variable  : Variable Importance
                         : --------------------------------------
                         :    1 : var4      : 3.117e-01
                         :    2 : var1      : 2.504e-01
                         :    3 : var2      : 2.430e-01
                         :    4 : var3      : 1.949e-01
                         : --------------------------------------
MLP                      : Ranking result (top variable is best ranked)
                         : -----------------------------
                         : Rank : Variable  : Importance
                         : -----------------------------
                         :    1 : var4      : 6.076e+01
                         :    2 : var2      : 4.824e+01
                         :    3 : var1      : 2.116e+01
                         :    4 : var3      : 1.692e+01
                         : -----------------------------
PDEFoam                  : Ranking result (top variable is best ranked)
                         : --------------------------------------
                         : Rank : Variable  : Variable Importance
                         : --------------------------------------
                         :    1 : var4      : 2.991e-01
                         :    2 : var1      : 2.930e-01
                         :    3 : var3      : 2.365e-01
                         :    4 : var2      : 1.714e-01
                         : --------------------------------------
                         : No variable ranking supplied by classifier: DL_CPU
TH1.Print Name  = TrainingHistory_DL_CPU_trainingError, Entries= 0, Total sum= 31.9933
TH1.Print Name  = TrainingHistory_DL_CPU_valError, Entries= 0, Total sum= 31.5428
Factory                  : === Destroy and recreate all methods via weight files for testing ===
                         : 
                         : Reading weight file: dataset/weights/TMVAMulticlass_BDTG.weights.xml
                         : Reading weight file: dataset/weights/TMVAMulticlass_MLP.weights.xml
MLP                      : Building Network. 
                         : Initializing weights
                         : Reading weight file: dataset/weights/TMVAMulticlass_PDEFoam.weights.xml
                         : Read foams from file: dataset/weights/TMVAMulticlass_PDEFoam.weights_foams.root
                         : Reading weight file: dataset/weights/TMVAMulticlass_DL_CPU.weights.xml

Evaluate all mvas using the set of test events

In [8]:
factory->TestAllMethods();
Factory                  : Test all methods
Factory                  : Test method: BDTG for Multiclass classification performance
                         : 
                         : Dataset[dataset] : Create results for testing
                         : Dataset[dataset] : Multiclass evaluation of BDTG on testing sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.973 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
Factory                  : Test method: MLP for Multiclass classification performance
                         : 
                         : Dataset[dataset] : Create results for testing
                         : Dataset[dataset] : Multiclass evaluation of MLP on testing sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.00951 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
Factory                  : Test method: PDEFoam for Multiclass classification performance
                         : 
                         : Dataset[dataset] : Create results for testing
                         : Dataset[dataset] : Multiclass evaluation of PDEFoam on testing sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.124 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
Factory                  : Test method: DL_CPU for Multiclass classification performance
                         : 
                         : Dataset[dataset] : Create results for testing
                         : Dataset[dataset] : Multiclass evaluation of DL_CPU on testing sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.104 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...

Evaluate and compare performance of all configured mvas

In [9]:
factory->EvaluateAllMethods();
Factory                  : Evaluate all methods
                         : Evaluate multiclass classification method: BDTG
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
TFHandler_BDTG           : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070153     1.0224   [    -4.0592     3.5808 ]
                         :     var2:    0.30372     1.0460   [    -3.6952     3.7877 ]
                         :     var3:    0.12152     1.1222   [    -3.6800     3.9200 ]
                         :     var4:  -0.072602     1.2766   [    -4.8486     4.2221 ]
                         : -----------------------------------------------------------
                         : Evaluate multiclass classification method: MLP
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
TFHandler_MLP            : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070153     1.0224   [    -4.0592     3.5808 ]
                         :     var2:    0.30372     1.0460   [    -3.6952     3.7877 ]
                         :     var3:    0.12152     1.1222   [    -3.6800     3.9200 ]
                         :     var4:  -0.072602     1.2766   [    -4.8486     4.2221 ]
                         : -----------------------------------------------------------
                         : Evaluate multiclass classification method: PDEFoam
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
TFHandler_PDEFoam        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070153     1.0224   [    -4.0592     3.5808 ]
                         :     var2:    0.30372     1.0460   [    -3.6952     3.7877 ]
                         :     var3:    0.12152     1.1222   [    -3.6800     3.9200 ]
                         :     var4:  -0.072602     1.2766   [    -4.8486     4.2221 ]
                         : -----------------------------------------------------------
                         : Evaluate multiclass classification method: DL_CPU
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
TFHandler_DL_CPU         : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.077270    0.29534   [    -1.1155     1.0914 ]
                         :     var2:   0.068045    0.27981   [    -1.0016     1.0000 ]
                         :     var3:   0.027548    0.24565   [   -0.80459    0.85902 ]
                         :     var4:  -0.034157    0.25816   [    -1.0000    0.83435 ]
                         : -----------------------------------------------------------
TFHandler_DL_CPU         : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.077270    0.29534   [    -1.1155     1.0914 ]
                         :     var2:   0.068045    0.27981   [    -1.0016     1.0000 ]
                         :     var3:   0.027548    0.24565   [   -0.80459    0.85902 ]
                         :     var4:  -0.034157    0.25816   [    -1.0000    0.83435 ]
                         : -----------------------------------------------------------
                         : 
                         : 1-vs-rest performance metrics per class
                         : -------------------------------------------------------------------------------------------------------
                         : 
                         : Considers the listed class as signal and the other classes
                         : as background, reporting the resulting binary performance.
                         : A score of 0.820 (0.850) means 0.820 was acheived on the
                         : test set and 0.850 on the training set.
                         : 
                         : Dataset        MVA Method     ROC AUC        Sig [email protected]=0.01 Sig [email protected]=0.10 Sig [email protected]=0.30 
                         : Name:          / Class:       test  (train)  test  (train)  test  (train)  test  (train)  
                         : 
                         : dataset        BDTG           
                         : ------------------------------
                         :                Signal         0.968 (0.978)  0.508 (0.605)  0.914 (0.945)  0.990 (0.996)  
                         :                bg0            0.910 (0.931)  0.256 (0.288)  0.737 (0.791)  0.922 (0.956)  
                         :                bg1            0.947 (0.954)  0.437 (0.511)  0.833 (0.856)  0.971 (0.971)  
                         :                bg2            0.978 (0.982)  0.585 (0.678)  0.951 (0.956)  0.999 (0.996)  
                         : 
                         : dataset        MLP            
                         : ------------------------------
                         :                Signal         0.970 (0.975)  0.596 (0.632)  0.933 (0.938)  0.988 (0.993)  
                         :                bg0            0.929 (0.934)  0.303 (0.298)  0.787 (0.793)  0.949 (0.961)  
                         :                bg1            0.962 (0.967)  0.467 (0.553)  0.881 (0.906)  0.985 (0.992)  
                         :                bg2            0.975 (0.979)  0.629 (0.699)  0.929 (0.940)  0.998 (0.998)  
                         : 
                         : dataset        PDEFoam        
                         : ------------------------------
                         :                Signal         0.916 (0.928)  0.294 (0.382)  0.744 (0.782)  0.932 (0.952)  
                         :                bg0            0.837 (0.848)  0.109 (0.147)  0.519 (0.543)  0.833 (0.851)  
                         :                bg1            0.890 (0.902)  0.190 (0.226)  0.606 (0.646)  0.923 (0.929)  
                         :                bg2            0.967 (0.972)  0.510 (0.527)  0.900 (0.926)  0.993 (0.998)  
                         : 
                         : dataset        DL_CPU         
                         : ------------------------------
                         :                Signal         0.976 (0.977)  0.609 (0.673)  0.937 (0.940)  0.993 (0.994)  
                         :                bg0            0.924 (0.932)  0.330 (0.308)  0.786 (0.778)  0.936 (0.955)  
                         :                bg1            0.964 (0.966)  0.469 (0.525)  0.894 (0.899)  0.991 (0.994)  
                         :                bg2            0.981 (0.981)  0.688 (0.699)  0.938 (0.937)  1.000 (0.999)  
                         : 
                         : -------------------------------------------------------------------------------------------------------
                         : 
                         : 
                         : Confusion matrices for all methods
                         : -------------------------------------------------------------------------------------------------------
                         : 
                         : Does a binary comparison between the two classes given by a 
                         : particular row-column combination. In each case, the class 
                         : given by the row is considered signal while the class given 
                         : by the column index is considered background.
                         : 
                         : === Showing confusion matrix for method : BDTG           
                         : (Signal Efficiency for Background Efficiency 0.01%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.497 (0.373)  0.710 (0.693)  0.680 (0.574) 
                         :  bg0            0.271 (0.184)  -              0.239 (0.145)  0.705 (0.667) 
                         :  bg1            0.855 (0.766)  0.369 (0.222)  -              0.587 (0.578) 
                         :  bg2            0.714 (0.585)  0.705 (0.581)  0.648 (0.601)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.10%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.911 (0.853)  0.991 (0.981)  0.945 (0.913) 
                         :  bg0            0.833 (0.774)  -              0.654 (0.582)  0.930 (0.901) 
                         :  bg1            0.971 (0.980)  0.716 (0.681)  -              0.871 (0.862) 
                         :  bg2            0.976 (0.951)  0.971 (0.973)  0.936 (0.941)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.30%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.978 (0.957)  0.999 (1.000)  0.998 (0.997) 
                         :  bg0            0.965 (0.926)  -              0.874 (0.835)  0.991 (0.976) 
                         :  bg1            1.000 (0.999)  0.916 (0.894)  -              0.988 (0.985) 
                         :  bg2            0.999 (0.999)  0.997 (0.999)  0.996 (0.997)  -             
                         : 
                         : === Showing confusion matrix for method : MLP            
                         : (Signal Efficiency for Background Efficiency 0.01%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.465 (0.490)  0.974 (0.953)  0.632 (0.498) 
                         :  bg0            0.320 (0.269)  -              0.224 (0.250)  0.655 (0.627) 
                         :  bg1            0.943 (0.920)  0.341 (0.275)  -              0.632 (0.687) 
                         :  bg2            0.665 (0.642)  0.697 (0.680)  0.706 (0.598)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.10%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.865 (0.854)  0.996 (0.994)  0.908 (0.907) 
                         :  bg0            0.784 (0.776)  -              0.666 (0.655)  0.919 (0.895) 
                         :  bg1            0.998 (0.998)  0.791 (0.785)  -              0.912 (0.902) 
                         :  bg2            0.943 (0.903)  0.946 (0.939)  0.924 (0.928)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.30%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.978 (0.964)  0.997 (0.997)  0.993 (0.986) 
                         :  bg0            0.952 (0.924)  -              0.936 (0.928)  0.992 (0.990) 
                         :  bg1            1.000 (1.000)  0.945 (0.936)  -              0.998 (0.995) 
                         :  bg2            0.994 (0.985)  0.998 (0.998)  0.998 (0.998)  -             
                         : 
                         : === Showing confusion matrix for method : PDEFoam        
                         : (Signal Efficiency for Background Efficiency 0.01%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.289 (0.233)  0.467 (0.436)  0.421 (0.332) 
                         :  bg0            0.100 (0.045)  -              0.132 (0.116)  0.540 (0.313) 
                         :  bg1            0.209 (0.434)  0.153 (0.092)  -              0.347 (0.323) 
                         :  bg2            0.560 (0.552)  0.445 (0.424)  0.501 (0.506)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.10%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.665 (0.640)  0.854 (0.822)  0.807 (0.790) 
                         :  bg0            0.538 (0.520)  -              0.415 (0.374)  0.843 (0.833) 
                         :  bg1            0.885 (0.886)  0.542 (0.491)  -              0.728 (0.646) 
                         :  bg2            0.928 (0.890)  0.956 (0.959)  0.847 (0.895)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.30%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.898 (0.878)  0.971 (0.950)  0.982 (0.975) 
                         :  bg0            0.828 (0.810)  -              0.696 (0.676)  0.954 (0.951) 
                         :  bg1            0.951 (0.966)  0.803 (0.745)  -              0.958 (0.966) 
                         :  bg2            0.998 (0.991)  0.998 (0.996)  0.998 (0.993)  -             
                         : 
                         : === Showing confusion matrix for method : DL_CPU         
                         : (Signal Efficiency for Background Efficiency 0.01%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.522 (0.484)  0.933 (0.938)  0.690 (0.631) 
                         :  bg0            0.308 (0.256)  -              0.240 (0.242)  0.564 (0.530) 
                         :  bg1            0.902 (0.891)  0.290 (0.302)  -              0.634 (0.684) 
                         :  bg2            0.673 (0.685)  0.719 (0.677)  0.698 (0.735)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.10%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.893 (0.896)  0.995 (0.995)  0.921 (0.927) 
                         :  bg0            0.743 (0.736)  -              0.705 (0.733)  0.908 (0.872) 
                         :  bg1            0.995 (0.992)  0.798 (0.779)  -              0.880 (0.902) 
                         :  bg2            0.950 (0.930)  0.950 (0.948)  0.927 (0.944)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.30%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.978 (0.975)  0.999 (1.000)  0.998 (1.000) 
                         :  bg0            0.940 (0.910)  -              0.939 (0.923)  0.985 (0.973) 
                         :  bg1            1.000 (0.998)  0.951 (0.954)  -              1.000 (0.995) 
                         :  bg2            0.999 (0.999)  0.999 (1.000)  0.999 (1.000)  -             
                         : 
                         : -------------------------------------------------------------------------------------------------------
                         : 
Dataset:dataset          : Created tree 'TestTree' with 4000 events
                         : 
Dataset:dataset          : Created tree 'TrainTree' with 4000 events
                         : 
Factory                  : Thank you for using TMVA!
                         : For citation information, please visit: http://tmva.sf.net/citeTMVA.html

Save the output

In [10]:
outputFile->Close();

std::cout << "==> Wrote root file: " << outputFile->GetName() << std::endl;
std::cout << "==> TMVAMulticlass is done!" << std::endl;

delete factory;
delete dataloader;
==> Wrote root file: TMVAMulticlass.root
==> TMVAMulticlass is done!

Launch the gui for the root macros

In [11]:
if (!gROOT->IsBatch()) TMVAMultiClassGui( outfileName );