TMVAMulticlass

This macro provides a simple example for the training and testing of the TMVA multiclass classification

  • Project : TMVA - a Root-integrated toolkit for multivariate data analysis
  • Package : TMVA
  • Root Macro: TMVAMulticlass

Author: Andreas Hoecker
This notebook tutorial was automatically generated with ROOTBOOK-izer from the macro found in the ROOT repository on Monday, August 15, 2022 at 09:30 AM.

In [1]:
%%cpp -d
#include <cstdlib>
#include <iostream>
#include <map>
#include <string>

#include "TFile.h"
#include "TTree.h"
#include "TString.h"
#include "TSystem.h"
#include "TROOT.h"


#include "TMVA/Tools.h"
#include "TMVA/Factory.h"
#include "TMVA/DataLoader.h"
#include "TMVA/TMVAMultiClassGui.h"


using namespace TMVA;

Arguments are defined.

In [2]:
TString myMethodList = "";

This loads the library

In [3]:
TMVA::Tools::Instance();

to get access to the GUI and all tmva macros

TString tmva_dir(TString(gRootDir) + "/tmva"); if(gSystem->Getenv("TMVASYS")) tmva_dir = TString(gSystem->Getenv("TMVASYS")); gROOT->SetMacroPath(tmva_dir + "/test/:" + gROOT->GetMacroPath() ); gROOT->ProcessLine(".L TMVAMultiClassGui.C");


Default MVA methods to be trained + tested

In [4]:
std::map<std::string,int> Use;
Use["MLP"]             = 1;
Use["BDTG"]            = 1;
#ifdef R__HAS_TMVAGPU
Use["DL_CPU"]          = 1;
Use["DL_GPU"]          = 1;
#else
Use["DL_CPU"]          = 1;
Use["DL_GPU"]          = 0;
#endif
Use["FDA_GA"]          = 0;
Use["PDEFoam"]         = 1;

In [5]:
std::cout << std::endl;
std::cout << "==> Start TMVAMulticlass" << std::endl;

if (myMethodList != "") {
   for (std::map<std::string,int>::iterator it = Use.begin(); it != Use.end(); it++) it->second = 0;

   std::vector<TString> mlist = TMVA::gTools().SplitString( myMethodList, ',' );
   for (UInt_t i=0; i<mlist.size(); i++) {
      std::string regMethod(mlist[i]);

      if (Use.find(regMethod) == Use.end()) {
         std::cout << "Method \"" << regMethod << "\" not known in TMVA under this name. Choose among the following:" << std::endl;
         for (std::map<std::string,int>::iterator it = Use.begin(); it != Use.end(); it++) std::cout << it->first << " ";
         std::cout << std::endl;
         return;
      }
      Use[regMethod] = 1;
   }
}
==> Start TMVAMulticlass

Create a new root output file.

In [6]:
TString outfileName = "TMVAMulticlass.root";
TFile* outputFile = TFile::Open( outfileName, "RECREATE" );

TMVA::Factory *factory = new TMVA::Factory( "TMVAMulticlass", outputFile,
                                            "!V:!Silent:Color:!DrawProgressBar:Transformations=I;D;P;G,D:AnalysisType=multiclass" );
TMVA::DataLoader *dataloader=new TMVA::DataLoader("dataset");

dataloader->AddVariable( "var1", 'F' );
dataloader->AddVariable( "var2", "Variable 2", "", 'F' );
dataloader->AddVariable( "var3", "Variable 3", "units", 'F' );
dataloader->AddVariable( "var4", "Variable 4", "units", 'F' );

TFile *input(0);
TString fname = "./tmva_example_multiclass.root";
if (!gSystem->AccessPathName( fname )) {
   input = TFile::Open( fname ); // check if file in local directory exists
}
else {
   TFile::SetCacheFileDir(".");
   input = TFile::Open("http://root.cern.ch/files/tmva_multiclass_example.root", "CACHEREAD");
}
if (!input) {
   std::cout << "ERROR: could not open data file" << std::endl;
   exit(1);
}
std::cout << "--- TMVAMulticlass: Using input file: " << input->GetName() << std::endl;

TTree *signalTree  = (TTree*)input->Get("TreeS");
TTree *background0 = (TTree*)input->Get("TreeB0");
TTree *background1 = (TTree*)input->Get("TreeB1");
TTree *background2 = (TTree*)input->Get("TreeB2");

gROOT->cd( outfileName+TString(":/") );
dataloader->AddTree    (signalTree,"Signal");
dataloader->AddTree    (background0,"bg0");
dataloader->AddTree    (background1,"bg1");
dataloader->AddTree    (background2,"bg2");

dataloader->PrepareTrainingAndTestTree( "", "SplitMode=Random:NormMode=NumEvents:!V" );

if (Use["BDTG"]) // gradient boosted decision trees
   factory->BookMethod( dataloader,  TMVA::Types::kBDT, "BDTG", "!H:!V:NTrees=1000:BoostType=Grad:Shrinkage=0.10:UseBaggedBoost:BaggedSampleFraction=0.50:nCuts=20:MaxDepth=2");
if (Use["MLP"]) // neural network
   factory->BookMethod( dataloader,  TMVA::Types::kMLP, "MLP", "!H:!V:NeuronType=tanh:NCycles=1000:HiddenLayers=N+5,5:TestRate=5:EstimatorType=MSE");
if (Use["FDA_GA"]) // functional discriminant with GA minimizer
   factory->BookMethod( dataloader,  TMVA::Types::kFDA, "FDA_GA", "H:!V:Formula=(0)+(1)*x0+(2)*x1+(3)*x2+(4)*x3:ParRanges=(-1,1);(-10,10);(-10,10);(-10,10);(-10,10):FitMethod=GA:PopSize=300:Cycles=3:Steps=20:Trim=True:SaveBestGen=1" );
if (Use["PDEFoam"]) // PDE-Foam approach
   factory->BookMethod( dataloader,  TMVA::Types::kPDEFoam, "PDEFoam", "!H:!V:TailCut=0.001:VolFrac=0.0666:nActiveCells=500:nSampl=2000:nBin=5:Nmin=100:Kernel=None:Compress=T" );


if (Use["DL_CPU"]) {
   TString layoutString("Layout=TANH|100,TANH|50,TANH|10,LINEAR");
   TString trainingStrategyString("TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,"
                                  "TestRepetitions=1,ConvergenceSteps=10,BatchSize=100");
   TString nnOptions("!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:"
                     "WeightInitialization=XAVIERUNIFORM:Architecture=GPU");
   nnOptions.Append(":");
   nnOptions.Append(layoutString);
   nnOptions.Append(":");
   nnOptions.Append(trainingStrategyString);
   factory->BookMethod(dataloader, TMVA::Types::kDL, "DL_CPU", nnOptions);
}
if (Use["DL_GPU"]) {
   TString layoutString("Layout=TANH|100,TANH|50,TANH|10,LINEAR");
   TString trainingStrategyString("TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,"
                                  "TestRepetitions=1,ConvergenceSteps=10,BatchSize=100");
   TString nnOptions("!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:"
                     "WeightInitialization=XAVIERUNIFORM:Architecture=GPU");
   nnOptions.Append(":");
   nnOptions.Append(layoutString);
   nnOptions.Append(":");
   nnOptions.Append(trainingStrategyString);
   factory->BookMethod(dataloader, TMVA::Types::kDL, "DL_GPU", nnOptions);
}
--- TMVAMulticlass: Using input file: ./files/tmva_multiclass_example.root
DataSetInfo              : [dataset] : Added class "Signal"
                         : Add Tree TreeS of type Signal with 2000 events
DataSetInfo              : [dataset] : Added class "bg0"
                         : Add Tree TreeB0 of type bg0 with 2000 events
DataSetInfo              : [dataset] : Added class "bg1"
                         : Add Tree TreeB1 of type bg1 with 2000 events
DataSetInfo              : [dataset] : Added class "bg2"
                         : Add Tree TreeB2 of type bg2 with 2000 events
                         : Dataset[dataset] : Class index : 0  name : Signal
                         : Dataset[dataset] : Class index : 1  name : bg0
                         : Dataset[dataset] : Class index : 2  name : bg1
                         : Dataset[dataset] : Class index : 3  name : bg2
Factory                  : Booking method: BDTG
                         : 
                         : the option NegWeightTreatment=InverseBoostNegWeights does not exist for BoostType=Grad
                         : --> change to new default NegWeightTreatment=Pray
                         : Rebuilding Dataset dataset
                         : Building event vectors for type 2 Signal
                         : Dataset[dataset] :  create input formulas for tree TreeS
                         : Building event vectors for type 2 bg0
                         : Dataset[dataset] :  create input formulas for tree TreeB0
                         : Building event vectors for type 2 bg1
                         : Dataset[dataset] :  create input formulas for tree TreeB1
                         : Building event vectors for type 2 bg2
                         : Dataset[dataset] :  create input formulas for tree TreeB2
DataSetFactory           : [dataset] : Number of events in input trees
                         : 
                         : 
                         : 
                         : 
                         : Number of training and testing events
                         : ---------------------------------------------------------------------------
                         : Signal -- training events            : 1000
                         : Signal -- testing events             : 1000
                         : Signal -- training and testing events: 2000
                         : bg0    -- training events            : 1000
                         : bg0    -- testing events             : 1000
                         : bg0    -- training and testing events: 2000
                         : bg1    -- training events            : 1000
                         : bg1    -- testing events             : 1000
                         : bg1    -- training and testing events: 2000
                         : bg2    -- training events            : 1000
                         : bg2    -- testing events             : 1000
                         : bg2    -- training and testing events: 2000
                         : 
DataSetInfo              : Correlation matrix (Signal):
                         : ----------------------------------------
                         :             var1    var2    var3    var4
                         :    var1:  +1.000  +0.397  +0.623  +0.832
                         :    var2:  +0.397  +1.000  +0.716  +0.737
                         :    var3:  +0.623  +0.716  +1.000  +0.859
                         :    var4:  +0.832  +0.737  +0.859  +1.000
                         : ----------------------------------------
DataSetInfo              : Correlation matrix (bg0):
                         : ----------------------------------------
                         :             var1    var2    var3    var4
                         :    var1:  +1.000  +0.365  +0.592  +0.811
                         :    var2:  +0.365  +1.000  +0.708  +0.740
                         :    var3:  +0.592  +0.708  +1.000  +0.859
                         :    var4:  +0.811  +0.740  +0.859  +1.000
                         : ----------------------------------------
DataSetInfo              : Correlation matrix (bg1):
                         : ----------------------------------------
                         :             var1    var2    var3    var4
                         :    var1:  +1.000  +0.407  +0.610  +0.834
                         :    var2:  +0.407  +1.000  +0.710  +0.741
                         :    var3:  +0.610  +0.710  +1.000  +0.851
                         :    var4:  +0.834  +0.741  +0.851  +1.000
                         : ----------------------------------------
DataSetInfo              : Correlation matrix (bg2):
                         : ----------------------------------------
                         :             var1    var2    var3    var4
                         :    var1:  +1.000  -0.647  -0.016  -0.013
                         :    var2:  -0.647  +1.000  +0.015  +0.002
                         :    var3:  -0.016  +0.015  +1.000  -0.024
                         :    var4:  -0.013  +0.002  -0.024  +1.000
                         : ----------------------------------------
DataSetFactory           : [dataset] :  
                         : 
Factory                  : Booking method: MLP
                         : 
MLP                      : Building Network. 
                         : Initializing weights
Factory                  : Booking method: PDEFoam
                         : 
Factory                  : Booking method: DL_CPU
                         : 
                         : Parsing option string: 
                         : ... "!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:WeightInitialization=XAVIERUNIFORM:Architecture=GPU:Layout=TANH|100,TANH|50,TANH|10,LINEAR:TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,TestRepetitions=1,ConvergenceSteps=10,BatchSize=100"
                         : The following options are set:
                         : - By User:
                         :     <none>
                         : - Default:
                         :     Boost_num: "0" [Number of times the classifier will be boosted]
                         : Parsing option string: 
                         : ... "!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:WeightInitialization=XAVIERUNIFORM:Architecture=GPU:Layout=TANH|100,TANH|50,TANH|10,LINEAR:TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,TestRepetitions=1,ConvergenceSteps=10,BatchSize=100"
                         : The following options are set:
                         : - By User:
                         :     V: "True" [Verbose output (short form of "VerbosityLevel" below - overrides the latter one)]
                         :     VarTransform: "N" [List of variable transformations performed before training, e.g., "D_Background,P_Signal,G,N_AllClasses" for: "Decorrelation, PCA-transformation, Gaussianisation, Normalisation, each for the given class of events ('AllClasses' denotes all events of all classes, if no class indication is given, 'All' is assumed)"]
                         :     H: "False" [Print method-specific help message]
                         :     Layout: "TANH|100,TANH|50,TANH|10,LINEAR" [Layout of the network.]
                         :     ErrorStrategy: "CROSSENTROPY" [Loss function: Mean squared error (regression) or cross entropy (binary classification).]
                         :     WeightInitialization: "XAVIERUNIFORM" [Weight initialization strategy]
                         :     Architecture: "GPU" [Which architecture to perform the training on.]
                         :     TrainingStrategy: "Optimizer=ADAM,LearningRate=1e-3,TestRepetitions=1,ConvergenceSteps=10,BatchSize=100" [Defines the training strategies.]
                         : - Default:
                         :     VerbosityLevel: "Default" [Verbosity level]
                         :     CreateMVAPdfs: "False" [Create PDFs for classifier outputs (signal and background)]
                         :     IgnoreNegWeightsInTraining: "False" [Events with negative weights are ignored in the training (but are included for testing and performance evaluation)]
                         :     InputLayout: "0|0|0" [The Layout of the input]
                         :     BatchLayout: "0|0|0" [The Layout of the batch]
                         :     RandomSeed: "0" [Random seed used for weight initialization and batch shuffling]
                         :     ValidationSize: "20%" [Part of the training data to use for validation. Specify as 0.2 or 20% to use a fifth of the data set as validation set. Specify as 100 to use exactly 100 events. (Default: 20%)]
DL_CPU                   : [dataset] : Create Transformation "N" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
<ERROR>                         : CUDA backend not enabled. Please make sure you have CUDA installed and it was successfully detected by CMAKE by using -Dtmva-gpu=On  
                         : Will now use instead the CPU architecture !
                         : Will now use the CPU architecture with BLAS and IMT support !
[TFile::Cp] Total 0.12 MB	|====================| 100.00 % [29.0 MB/s]
Info in <TFile::OpenFromCache>: using local cache copy of http://root.cern.ch/files/tmva_multiclass_example.root [./files/tmva_multiclass_example.root]

Train MVAs using the set of training events

In [7]:
factory->TrainAllMethods();
Factory                  : Train all methods
Factory                  : [dataset] : Create Transformation "I" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
Factory                  : [dataset] : Create Transformation "D" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
Factory                  : [dataset] : Create Transformation "P" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
Factory                  : [dataset] : Create Transformation "G" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
Factory                  : [dataset] : Create Transformation "D" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
TFHandler_Factory        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.047647     1.0025   [    -3.6592     3.2645 ]
                         :     var2:    0.32647     1.0646   [    -3.6891     3.7877 ]
                         :     var3:    0.11493     1.1230   [    -4.5727     4.5640 ]
                         :     var4:  -0.076531     1.2652   [    -4.8486     5.0412 ]
                         : -----------------------------------------------------------
                         : Preparing the Decorrelation transformation...
TFHandler_Factory        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.082544     1.0000   [    -3.6274     3.1017 ]
                         :     var2:    0.36715     1.0000   [    -3.3020     3.4950 ]
                         :     var3:   0.066865     1.0000   [    -2.9882     3.3086 ]
                         :     var4:   -0.20593     1.0000   [    -3.3088     2.8423 ]
                         : -----------------------------------------------------------
                         : Preparing the Principle Component (PCA) transformation...
TFHandler_Factory        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1: 5.7502e-10     1.8064   [    -8.0344     7.8312 ]
                         :     var2:-1.6078e-11    0.90130   [    -2.6765     2.7523 ]
                         :     var3: 3.0841e-10    0.73386   [    -2.6572     2.2255 ]
                         :     var4:-2.6886e-10    0.62168   [    -1.7384     2.2297 ]
                         : -----------------------------------------------------------
                         : Preparing the Gaussian transformation...
                         : Preparing the Decorrelation transformation...
TFHandler_Factory        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.013510     1.0000   [    -2.6520     6.2074 ]
                         :     var2:  0.0096839     1.0000   [    -2.8402     6.3073 ]
                         :     var3:   0.010397     1.0000   [    -3.0251     5.8860 ]
                         :     var4:  0.0053980     1.0000   [    -3.0998     5.7078 ]
                         : -----------------------------------------------------------
                         : Ranking input variables (method unspecific)...
Factory                  : Train method: BDTG for Multiclass classification
                         : 
                         : Training 1000 Decision Trees ... patience please
                         : Elapsed time for training with 4000 events: 5.5 sec         
                         : Dataset[dataset] : Create results for training
                         : Dataset[dataset] : Multiclass evaluation of BDTG on training sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 2.25 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating xml weight file: dataset/weights/TMVAMulticlass_BDTG.weights.xml
                         : Creating standalone class: dataset/weights/TMVAMulticlass_BDTG.class.C
                         : TMVAMulticlass.root:/dataset/Method_BDT/BDTG
Factory                  : Training finished
                         : 
Factory                  : Train method: MLP for Multiclass classification
                         : 
                         : Training Network
                         : 
                         : Elapsed time for training with 4000 events: 26.9 sec         
                         : Dataset[dataset] : Create results for training
                         : Dataset[dataset] : Multiclass evaluation of MLP on training sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.00968 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating xml weight file: dataset/weights/TMVAMulticlass_MLP.weights.xml
                         : Creating standalone class: dataset/weights/TMVAMulticlass_MLP.class.C
                         : Write special histos to file: TMVAMulticlass.root:/dataset/Method_MLP/MLP
Factory                  : Training finished
                         : 
Factory                  : Train method: PDEFoam for Multiclass classification
                         : 
                         : Build up multiclass foam 0
                         : Elapsed time: 0.668 sec                                 
                         : Build up multiclass foam 1
                         : Elapsed time: 0.672 sec                                 
                         : Build up multiclass foam 2
                         : Elapsed time: 0.68 sec                                 
                         : Build up multiclass foam 3
                         : Elapsed time: 0.473 sec                                 
                         : Elapsed time for training with 4000 events: 2.67 sec         
                         : Dataset[dataset] : Create results for training
                         : Dataset[dataset] : Multiclass evaluation of PDEFoam on training sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.122 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating xml weight file: dataset/weights/TMVAMulticlass_PDEFoam.weights.xml
                         : writing foam MultiClassFoam0 to file
                         : writing foam MultiClassFoam1 to file
                         : writing foam MultiClassFoam2 to file
                         : writing foam MultiClassFoam3 to file
                         : Foams written to file: dataset/weights/TMVAMulticlass_PDEFoam.weights_foams.root
                         : Creating standalone class: dataset/weights/TMVAMulticlass_PDEFoam.class.C
Factory                  : Training finished
                         : 
Factory                  : Train method: DL_CPU for Multiclass classification
                         : 
TFHandler_DL_CPU         : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070769    0.28960   [    -1.0000     1.0000 ]
                         :     var2:   0.074130    0.28477   [    -1.0000     1.0000 ]
                         :     var3:   0.026106    0.24582   [    -1.0000     1.0000 ]
                         :     var4:  -0.034951    0.25587   [    -1.0000     1.0000 ]
                         : -----------------------------------------------------------
                         : Start of deep neural network training on CPU using MT,  nthreads = 1
                         : 
TFHandler_DL_CPU         : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070769    0.28960   [    -1.0000     1.0000 ]
                         :     var2:   0.074130    0.28477   [    -1.0000     1.0000 ]
                         :     var3:   0.026106    0.24582   [    -1.0000     1.0000 ]
                         :     var4:  -0.034951    0.25587   [    -1.0000     1.0000 ]
                         : -----------------------------------------------------------
                         : *****   Deep Learning Network *****
DEEP NEURAL NETWORK:   Depth = 4  Input = ( 1, 1, 4 )  Batch size = 100  Loss function = C
	Layer 0	 DENSE Layer: 	 ( Input =     4 , Width =   100 ) 	Output = (  1 ,   100 ,   100 ) 	 Activation Function = Tanh
	Layer 1	 DENSE Layer: 	 ( Input =   100 , Width =    50 ) 	Output = (  1 ,   100 ,    50 ) 	 Activation Function = Tanh
	Layer 2	 DENSE Layer: 	 ( Input =    50 , Width =    10 ) 	Output = (  1 ,   100 ,    10 ) 	 Activation Function = Tanh
	Layer 3	 DENSE Layer: 	 ( Input =    10 , Width =     4 ) 	Output = (  1 ,   100 ,     4 ) 	 Activation Function = Identity
                         : Using 3200 events for training and 800 for testing
                         : Compute initial loss  on the validation data 
                         : Training phase 1 of 1:  Optimizer ADAM (beta1=0.9,beta2=0.999,eps=1e-07) Learning rate = 0.001 regularization 0 minimum error = 0.699236
                         : --------------------------------------------------------------
                         :      Epoch |   Train Err.   Val. Err.  t(s)/epoch   t(s)/Loss   nEvents/s Conv. Steps
                         : --------------------------------------------------------------
                         :    Start epoch iteration ...
                         :          1 Minimum Test error found - save the configuration 
                         :          1 |     0.604372    0.529418   0.0772409  0.00674995     45395.9           0
                         :          2 Minimum Test error found - save the configuration 
                         :          2 |     0.497992    0.478158   0.0781199  0.00675667       44841           0
                         :          3 Minimum Test error found - save the configuration 
                         :          3 |     0.457398    0.436838   0.0785542  0.00671628     44544.7           0
                         :          4 Minimum Test error found - save the configuration 
                         :          4 |     0.426172    0.410455   0.0789744  0.00677487     44321.6           0
                         :          5 Minimum Test error found - save the configuration 
                         :          5 |     0.404454    0.390868   0.0790305  0.00679945     44302.3           0
                         :          6 Minimum Test error found - save the configuration 
                         :          6 |     0.387869     0.37444   0.0795631  0.00684091     44003.1           0
                         :          7 Minimum Test error found - save the configuration 
                         :          7 |     0.374986    0.361737   0.0798163  0.00689799     43884.7           0
                         :          8 Minimum Test error found - save the configuration 
                         :          8 |     0.364288    0.352754   0.0799516  0.00689863     43803.8           0
                         :          9 Minimum Test error found - save the configuration 
                         :          9 |     0.355396    0.343603   0.0801542   0.0069119     43690.6           0
                         :         10 Minimum Test error found - save the configuration 
                         :         10 |     0.348053    0.334751   0.0804571  0.00693318     43523.3           0
                         :         11 Minimum Test error found - save the configuration 
                         :         11 |      0.34087    0.327428   0.0804678  0.00695492     43529.8           0
                         :         12 Minimum Test error found - save the configuration 
                         :         12 |     0.333748    0.322223   0.0805616  0.00693852     43464.7           0
                         :         13 Minimum Test error found - save the configuration 
                         :         13 |     0.327707    0.318874   0.0806664  0.00694186     43404.8           0
                         :         14 Minimum Test error found - save the configuration 
                         :         14 |     0.322189    0.312153   0.0808424  0.00696566     43315.4           0
                         :         15 Minimum Test error found - save the configuration 
                         :         15 |      0.31645     0.30809   0.0809194  0.00702524     43305.2           0
                         :         16 Minimum Test error found - save the configuration 
                         :         16 |     0.312232    0.297616   0.0811457  0.00698899     43151.8           0
                         :         17 |     0.306532    0.298588   0.0808881  0.00691508       43259           1
                         :         18 Minimum Test error found - save the configuration 
                         :         18 |     0.301695    0.292649   0.0812106  0.00701157     43127.3           0
                         :         19 Minimum Test error found - save the configuration 
                         :         19 |     0.297976    0.288957   0.0813563  0.00703661     43057.3           0
                         :         20 Minimum Test error found - save the configuration 
                         :         20 |     0.293729    0.286235   0.0812113  0.00703722     43141.7           0
                         :         21 Minimum Test error found - save the configuration 
                         :         21 |     0.291712    0.282334   0.0815932  0.00707329     42941.5           0
                         :         22 Minimum Test error found - save the configuration 
                         :         22 |     0.288016    0.279891   0.0818565  0.00708846       42799           0
                         :         23 Minimum Test error found - save the configuration 
                         :         23 |     0.285466    0.276433    0.081589  0.00704529     42927.9           0
                         :         24 Minimum Test error found - save the configuration 
                         :         24 |     0.284396    0.275731   0.0817617  0.00708487     42851.3           0
                         :         25 Minimum Test error found - save the configuration 
                         :         25 |     0.282144    0.270861   0.0819019  0.00706888     42761.9           0
                         :         26 Minimum Test error found - save the configuration 
                         :         26 |     0.279215    0.269299   0.0818114  0.00707465       42817           0
                         :         27 Minimum Test error found - save the configuration 
                         :         27 |     0.277087    0.268943   0.0820144  0.00707509     42701.2           0
                         :         28 Minimum Test error found - save the configuration 
                         :         28 |     0.276917     0.26753   0.0819058  0.00711275     42784.7           0
                         :         29 |     0.275599    0.269405   0.0817928  0.00699468     42781.8           1
                         :         30 Minimum Test error found - save the configuration 
                         :         30 |     0.273018    0.261289   0.0819135  0.00710616     42776.6           0
                         :         31 |     0.270671    0.266158   0.0819018  0.00701478       42731           1
                         :         32 Minimum Test error found - save the configuration 
                         :         32 |     0.270106    0.259422   0.0819048  0.00710818     42782.7           0
                         :         33 Minimum Test error found - save the configuration 
                         :         33 |     0.268994    0.258706   0.0818556  0.00710181     42807.2           0
                         :         34 |     0.267698    0.261389   0.0822688  0.00704425     42539.3           1
                         :         35 Minimum Test error found - save the configuration 
                         :         35 |      0.26616    0.258059   0.0820875  0.00709988     42673.7           0
                         :         36 Minimum Test error found - save the configuration 
                         :         36 |      0.26601    0.253813   0.0819168  0.00710269     42772.7           0
                         :         37 Minimum Test error found - save the configuration 
                         :         37 |     0.264751    0.253195    0.082168  0.00715783     42660.9           0
                         :         38 |     0.263012    0.258205    0.082204  0.00707889     42595.6           1
                         :         39 |     0.261306    0.253345   0.0820695  0.00701985     42638.5           2
                         :         40 Minimum Test error found - save the configuration 
                         :         40 |     0.261246    0.251963   0.0822425   0.0071251       42600           0
                         :         41 Minimum Test error found - save the configuration 
                         :         41 |     0.259955    0.250096   0.0823506  0.00713593     42544.9           0
                         :         42 Minimum Test error found - save the configuration 
                         :         42 |     0.259069    0.249179   0.0823498  0.00715196     42554.4           0
                         :         43 |     0.258739    0.250026   0.0821991  0.00707173     42594.3           1
                         :         44 Minimum Test error found - save the configuration 
                         :         44 |     0.256942    0.247543   0.0826042  0.00720921     42443.2           0
                         :         45 |     0.256897    0.248616   0.0822506  0.00705474     42555.5           1
                         :         46 |     0.255175    0.249608   0.0822474  0.00704657     42552.7           2
                         :         47 Minimum Test error found - save the configuration 
                         :         47 |     0.253298     0.24551   0.0824827  0.00714582     42475.8           0
                         :         48 Minimum Test error found - save the configuration 
                         :         48 |     0.253196    0.242325    0.082758  0.00718228     42341.6           0
                         :         49 Minimum Test error found - save the configuration 
                         :         49 |     0.252129    0.241723   0.0826506  0.00716241     42390.7           0
                         :         50 |     0.251503    0.243669   0.0823047  0.00706012       42528           1
                         :         51 |     0.251222    0.245384   0.0825058  0.00707391     42422.4           2
                         :         52 |     0.249759      0.2454   0.0826447  0.00706156     42337.5           3
                         :         53 Minimum Test error found - save the configuration 
                         :         53 |     0.248561    0.241326   0.0824341  0.00714273     42501.6           0
                         :         54 Minimum Test error found - save the configuration 
                         :         54 |     0.247587    0.239201    0.082334  0.00714265     42558.1           0
                         :         55 |     0.246378    0.241513   0.0823272  0.00706769     42519.5           1
                         :         56 |     0.246296    0.240773   0.0823639  0.00706968     42499.9           2
                         :         57 |     0.244262    0.241992     0.08229  0.00706211     42537.4           3
                         :         58 Minimum Test error found - save the configuration 
                         :         58 |     0.244645    0.238555   0.0824887  0.00717985     42491.7           0
                         :         59 |     0.242186     0.24257    0.082375  0.00709244     42506.5           1
                         :         60 Minimum Test error found - save the configuration 
                         :         60 |     0.243911    0.236815   0.0825982  0.00716661     42422.5           0
                         :         61 Minimum Test error found - save the configuration 
                         :         61 |      0.24227    0.235453   0.0827258  0.00715761     42345.8           0
                         :         62 |     0.242191    0.236623   0.0825647  0.00707602     42390.5           1
                         :         63 |     0.239722    0.235932   0.0828399  0.00706909     42232.6           2
                         :         64 |     0.238518     0.23627   0.0827383  0.00713046     42323.7           3
                         :         65 Minimum Test error found - save the configuration 
                         :         65 |     0.238237    0.234464   0.0826661  0.00717237     42387.7           0
                         :         66 |     0.236483    0.238578   0.0826611  0.00710232     42351.1           1
                         :         67 Minimum Test error found - save the configuration 
                         :         67 |     0.235006    0.234462   0.0828714  0.00719006     42282.6           0
                         :         68 Minimum Test error found - save the configuration 
                         :         68 |     0.235142    0.232501   0.0827725  0.00718885     42337.2           0
                         :         69 |     0.233196    0.234841   0.0826967  0.00708668     42322.4           1
                         :         70 |     0.233081    0.234245   0.0840308  0.00711359     41603.2           2
                         :         71 Minimum Test error found - save the configuration 
                         :         71 |     0.231521    0.229923   0.0828589  0.00720757     42299.3           0
                         :         72 Minimum Test error found - save the configuration 
                         :         72 |      0.23144    0.229784   0.0838805  0.00822658     42297.9           0
                         :         73 |     0.231643     0.23023   0.0829272  0.00713094     42218.4           1
                         :         74 |      0.22967    0.231797   0.0827792  0.00710718     42287.8           2
                         :         75 |     0.230049    0.234823   0.0838294  0.00712676     41719.5           3
                         :         76 |     0.230001    0.234356   0.0829409  0.00711146       42200           4
                         :         77 Minimum Test error found - save the configuration 
                         :         77 |     0.229294    0.229759   0.0832333  0.00723579     42106.6           0
                         :         78 Minimum Test error found - save the configuration 
                         :         78 |     0.229381    0.226976   0.0829395  0.00720192     42251.1           0
                         :         79 Minimum Test error found - save the configuration 
                         :         79 |     0.226834    0.225879   0.0830153  0.00723259       42226           0
                         :         80 |     0.225986    0.231091   0.0830012  0.00711189     42166.7           1
                         :         81 Minimum Test error found - save the configuration 
                         :         81 |     0.225474     0.22517   0.0831648  0.00723575     42144.6           0
                         :         82 |     0.224879    0.231712    0.083103  0.00712122     42115.4           1
                         :         83 Minimum Test error found - save the configuration 
                         :         83 |     0.224904    0.224285   0.0830226  0.00721276     42210.9           0
                         :         84 |     0.224129    0.226868   0.0827902  0.00711008     42283.2           1
                         :         85 |     0.223667    0.229014   0.0829299  0.00713232     42217.7           2
                         :         86 |     0.223789    0.227676   0.0829853  0.00712524     42182.9           3
                         :         87 |     0.221128    0.226124   0.0828603  0.00712349     42251.6           4
                         :         88 |      0.22038    0.226922   0.0832731  0.00712653     42024.2           5
                         :         89 Minimum Test error found - save the configuration 
                         :         89 |     0.219445    0.223602   0.0834729  0.00736352     42044.8           0
                         :         90 |     0.219899    0.226905    0.082973  0.00714221     42199.2           1
                         :         91 |     0.220012    0.231605   0.0830026   0.0071301       42176           2
                         :         92 Minimum Test error found - save the configuration 
                         :         92 |     0.219438    0.221775   0.0830729  0.00725424       42206           0
                         :         93 Minimum Test error found - save the configuration 
                         :         93 |     0.218938    0.220615   0.0833847   0.0072306     42020.1           0
                         :         94 |     0.218324    0.223047   0.0829021   0.0071415     42238.3           1
                         :         95 Minimum Test error found - save the configuration 
                         :         95 |     0.217736    0.219169   0.0830501  0.00723811     42209.7           0
                         :         96 |     0.216423    0.222323    0.083189  0.00717171     42095.7           1
                         :         97 |     0.216422    0.223474   0.0833513  0.00714357     41990.5           2
                         :         98 |     0.217097    0.220869   0.0829963  0.00714209     42186.2           3
                         :         99 Minimum Test error found - save the configuration 
                         :         99 |     0.214494    0.218837   0.0832524  0.00724059     42098.7           0
                         :        100 |     0.214441    0.221284   0.0834503  0.00724127     41989.8           1
                         :        101 |     0.214376    0.220118   0.0832401  0.00716044     42061.2           2
                         :        102 |     0.214661    0.220914    0.083511  0.00715513       41909           3
                         :        103 Minimum Test error found - save the configuration 
                         :        103 |     0.214747    0.218733   0.0832953  0.00724143     42075.4           0
                         :        104 Minimum Test error found - save the configuration 
                         :        104 |     0.213931    0.217085   0.0833626  0.00723667     42035.6           0
                         :        105 |     0.212603    0.217966   0.0838573  0.00716698     41726.2           1
                         :        106 |     0.212862    0.217321   0.0840114  0.00715686     41637.1           2
                         :        107 |     0.211562     0.22026   0.0833402  0.00715285     42001.7           3
                         :        108 Minimum Test error found - save the configuration 
                         :        108 |     0.211437    0.215041   0.0832959  0.00727479     42093.6           0
                         :        109 |     0.213408    0.218698   0.0869193  0.00717715     40129.3           1
                         :        110 |     0.210328    0.215546   0.0832404  0.00716262     42062.2           2
                         :        111 Minimum Test error found - save the configuration 
                         :        111 |     0.210833     0.21424   0.0834483  0.00726703     42005.1           0
                         :        112 Minimum Test error found - save the configuration 
                         :        112 |      0.21116      0.2137   0.0834628  0.00724607     41985.5           0
                         :        113 |      0.20885    0.219499   0.0833383  0.00716434     42009.1           1
                         :        114 |     0.209649    0.217633   0.0834124   0.0071837     41978.9           2
                         :        115 |      0.20895    0.218021   0.0833497  0.00717042     42006.2           3
                         :        116 |     0.209097    0.213958   0.0832192  0.00716586     42075.8           4
                         :        117 |     0.209452    0.217995   0.0833322  0.00716771     42014.3           5
                         :        118 |     0.208437    0.217135   0.0889332  0.00723123     39166.7           6
                         :        119 |     0.207647    0.217218   0.0835874   0.0071958     41889.4           7
                         :        120 |     0.208039    0.214528   0.0834454  0.00717693     41957.1           8
                         :        121 |     0.208942    0.219319   0.0834242  0.00718889     41975.3           9
                         :        122 |     0.208158    0.214118   0.0835028   0.0071816       41928          10
                         :        123 |     0.207143    0.215855   0.0835285  0.00718106     41913.6          11
                         : 
                         : Elapsed time for training with 4000 events: 10.2 sec         
                         : Dataset[dataset] : Create results for training
                         : Dataset[dataset] : Multiclass evaluation of DL_CPU on training sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.105 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating xml weight file: dataset/weights/TMVAMulticlass_DL_CPU.weights.xml
                         : Creating standalone class: dataset/weights/TMVAMulticlass_DL_CPU.class.C
Factory                  : Training finished
                         : 
                         : Ranking input variables (method specific)...
BDTG                     : Ranking result (top variable is best ranked)
                         : --------------------------------------
                         : Rank : Variable  : Variable Importance
                         : --------------------------------------
                         :    1 : var4      : 3.117e-01
                         :    2 : var1      : 2.504e-01
                         :    3 : var2      : 2.430e-01
                         :    4 : var3      : 1.949e-01
                         : --------------------------------------
MLP                      : Ranking result (top variable is best ranked)
                         : -----------------------------
                         : Rank : Variable  : Importance
                         : -----------------------------
                         :    1 : var4      : 6.076e+01
                         :    2 : var2      : 4.824e+01
                         :    3 : var1      : 2.116e+01
                         :    4 : var3      : 1.692e+01
                         : -----------------------------
PDEFoam                  : Ranking result (top variable is best ranked)
                         : --------------------------------------
                         : Rank : Variable  : Variable Importance
                         : --------------------------------------
                         :    1 : var4      : 2.991e-01
                         :    2 : var1      : 2.930e-01
                         :    3 : var3      : 2.365e-01
                         :    4 : var2      : 1.714e-01
                         : --------------------------------------
                         : No variable ranking supplied by classifier: DL_CPU
TH1.Print Name  = TrainingHistory_DL_CPU_trainingError, Entries= 0, Total sum= 31.7863
TH1.Print Name  = TrainingHistory_DL_CPU_valError, Entries= 0, Total sum= 31.3878
Factory                  : === Destroy and recreate all methods via weight files for testing ===
                         : 
                         : Reading weight file: dataset/weights/TMVAMulticlass_BDTG.weights.xml
                         : Reading weight file: dataset/weights/TMVAMulticlass_MLP.weights.xml
MLP                      : Building Network. 
                         : Initializing weights
                         : Reading weight file: dataset/weights/TMVAMulticlass_PDEFoam.weights.xml
                         : Read foams from file: dataset/weights/TMVAMulticlass_PDEFoam.weights_foams.root
                         : Reading weight file: dataset/weights/TMVAMulticlass_DL_CPU.weights.xml

Evaluate all MVAs using the set of test events

In [8]:
factory->TestAllMethods();
Factory                  : Test all methods
Factory                  : Test method: BDTG for Multiclass classification performance
                         : 
                         : Dataset[dataset] : Create results for testing
                         : Dataset[dataset] : Multiclass evaluation of BDTG on testing sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 1.12 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
Factory                  : Test method: MLP for Multiclass classification performance
                         : 
                         : Dataset[dataset] : Create results for testing
                         : Dataset[dataset] : Multiclass evaluation of MLP on testing sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.0101 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
Factory                  : Test method: PDEFoam for Multiclass classification performance
                         : 
                         : Dataset[dataset] : Create results for testing
                         : Dataset[dataset] : Multiclass evaluation of PDEFoam on testing sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.124 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
Factory                  : Test method: DL_CPU for Multiclass classification performance
                         : 
                         : Dataset[dataset] : Create results for testing
                         : Dataset[dataset] : Multiclass evaluation of DL_CPU on testing sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.105 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...

Evaluate and compare performance of all configured MVAs

In [9]:
factory->EvaluateAllMethods();
Factory                  : Evaluate all methods
                         : Evaluate multiclass classification method: BDTG
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
TFHandler_BDTG           : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070153     1.0224   [    -4.0592     3.5808 ]
                         :     var2:    0.30372     1.0460   [    -3.6952     3.7877 ]
                         :     var3:    0.12152     1.1222   [    -3.6800     3.9200 ]
                         :     var4:  -0.072602     1.2766   [    -4.8486     4.2221 ]
                         : -----------------------------------------------------------
                         : Evaluate multiclass classification method: MLP
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
TFHandler_MLP            : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070153     1.0224   [    -4.0592     3.5808 ]
                         :     var2:    0.30372     1.0460   [    -3.6952     3.7877 ]
                         :     var3:    0.12152     1.1222   [    -3.6800     3.9200 ]
                         :     var4:  -0.072602     1.2766   [    -4.8486     4.2221 ]
                         : -----------------------------------------------------------
                         : Evaluate multiclass classification method: PDEFoam
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
TFHandler_PDEFoam        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070153     1.0224   [    -4.0592     3.5808 ]
                         :     var2:    0.30372     1.0460   [    -3.6952     3.7877 ]
                         :     var3:    0.12152     1.1222   [    -3.6800     3.9200 ]
                         :     var4:  -0.072602     1.2766   [    -4.8486     4.2221 ]
                         : -----------------------------------------------------------
                         : Evaluate multiclass classification method: DL_CPU
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
TFHandler_DL_CPU         : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.077270    0.29534   [    -1.1155     1.0914 ]
                         :     var2:   0.068045    0.27981   [    -1.0016     1.0000 ]
                         :     var3:   0.027548    0.24565   [   -0.80459    0.85902 ]
                         :     var4:  -0.034157    0.25816   [    -1.0000    0.83435 ]
                         : -----------------------------------------------------------
TFHandler_DL_CPU         : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.077270    0.29534   [    -1.1155     1.0914 ]
                         :     var2:   0.068045    0.27981   [    -1.0016     1.0000 ]
                         :     var3:   0.027548    0.24565   [   -0.80459    0.85902 ]
                         :     var4:  -0.034157    0.25816   [    -1.0000    0.83435 ]
                         : -----------------------------------------------------------
                         : 
                         : 1-vs-rest performance metrics per class
                         : -------------------------------------------------------------------------------------------------------
                         : 
                         : Considers the listed class as signal and the other classes
                         : as background, reporting the resulting binary performance.
                         : A score of 0.820 (0.850) means 0.820 was acheived on the
                         : test set and 0.850 on the training set.
                         : 
                         : Dataset        MVA Method     ROC AUC        Sig [email protected]=0.01 Sig [email protected]=0.10 Sig [email protected]=0.30 
                         : Name:          / Class:       test  (train)  test  (train)  test  (train)  test  (train)  
                         : 
                         : dataset        BDTG           
                         : ------------------------------
                         :                Signal         0.968 (0.978)  0.508 (0.605)  0.914 (0.945)  0.990 (0.996)  
                         :                bg0            0.910 (0.931)  0.256 (0.288)  0.737 (0.791)  0.922 (0.956)  
                         :                bg1            0.947 (0.954)  0.437 (0.511)  0.833 (0.856)  0.971 (0.971)  
                         :                bg2            0.978 (0.982)  0.585 (0.678)  0.951 (0.956)  0.999 (0.996)  
                         : 
                         : dataset        MLP            
                         : ------------------------------
                         :                Signal         0.970 (0.975)  0.596 (0.632)  0.933 (0.938)  0.988 (0.993)  
                         :                bg0            0.929 (0.934)  0.303 (0.298)  0.787 (0.793)  0.949 (0.961)  
                         :                bg1            0.962 (0.967)  0.467 (0.553)  0.881 (0.906)  0.985 (0.992)  
                         :                bg2            0.975 (0.979)  0.629 (0.699)  0.929 (0.940)  0.998 (0.998)  
                         : 
                         : dataset        PDEFoam        
                         : ------------------------------
                         :                Signal         0.916 (0.928)  0.294 (0.382)  0.744 (0.782)  0.932 (0.952)  
                         :                bg0            0.837 (0.848)  0.109 (0.147)  0.519 (0.543)  0.833 (0.851)  
                         :                bg1            0.890 (0.902)  0.190 (0.226)  0.606 (0.646)  0.923 (0.929)  
                         :                bg2            0.967 (0.972)  0.510 (0.527)  0.900 (0.926)  0.993 (0.998)  
                         : 
                         : dataset        DL_CPU         
                         : ------------------------------
                         :                Signal         0.977 (0.976)  0.620 (0.644)  0.947 (0.940)  0.992 (0.994)  
                         :                bg0            0.928 (0.933)  0.332 (0.293)  0.794 (0.792)  0.941 (0.959)  
                         :                bg1            0.965 (0.967)  0.482 (0.554)  0.899 (0.897)  0.987 (0.995)  
                         :                bg2            0.982 (0.981)  0.691 (0.690)  0.955 (0.942)  1.000 (0.998)  
                         : 
                         : -------------------------------------------------------------------------------------------------------
                         : 
                         : 
                         : Confusion matrices for all methods
                         : -------------------------------------------------------------------------------------------------------
                         : 
                         : Does a binary comparison between the two classes given by a 
                         : particular row-column combination. In each case, the class 
                         : given by the row is considered signal while the class given 
                         : by the column index is considered background.
                         : 
                         : === Showing confusion matrix for method : BDTG           
                         : (Signal Efficiency for Background Efficiency 0.01%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.497 (0.373)  0.710 (0.693)  0.680 (0.574) 
                         :  bg0            0.271 (0.184)  -              0.239 (0.145)  0.705 (0.667) 
                         :  bg1            0.855 (0.766)  0.369 (0.222)  -              0.587 (0.578) 
                         :  bg2            0.714 (0.585)  0.705 (0.581)  0.648 (0.601)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.10%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.911 (0.853)  0.991 (0.981)  0.945 (0.913) 
                         :  bg0            0.833 (0.774)  -              0.654 (0.582)  0.930 (0.901) 
                         :  bg1            0.971 (0.980)  0.716 (0.681)  -              0.871 (0.862) 
                         :  bg2            0.976 (0.951)  0.971 (0.973)  0.936 (0.941)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.30%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.978 (0.957)  0.999 (1.000)  0.998 (0.997) 
                         :  bg0            0.965 (0.926)  -              0.874 (0.835)  0.991 (0.976) 
                         :  bg1            1.000 (0.999)  0.916 (0.894)  -              0.988 (0.985) 
                         :  bg2            0.999 (0.999)  0.997 (0.999)  0.996 (0.997)  -             
                         : 
                         : === Showing confusion matrix for method : MLP            
                         : (Signal Efficiency for Background Efficiency 0.01%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.465 (0.490)  0.974 (0.953)  0.632 (0.498) 
                         :  bg0            0.320 (0.269)  -              0.224 (0.250)  0.655 (0.627) 
                         :  bg1            0.943 (0.920)  0.341 (0.275)  -              0.632 (0.687) 
                         :  bg2            0.665 (0.642)  0.697 (0.680)  0.706 (0.598)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.10%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.865 (0.854)  0.996 (0.994)  0.908 (0.907) 
                         :  bg0            0.784 (0.776)  -              0.666 (0.655)  0.919 (0.895) 
                         :  bg1            0.998 (0.998)  0.791 (0.785)  -              0.912 (0.902) 
                         :  bg2            0.943 (0.903)  0.946 (0.939)  0.924 (0.928)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.30%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.978 (0.964)  0.997 (0.997)  0.993 (0.986) 
                         :  bg0            0.952 (0.924)  -              0.936 (0.928)  0.992 (0.990) 
                         :  bg1            1.000 (1.000)  0.945 (0.936)  -              0.998 (0.995) 
                         :  bg2            0.994 (0.985)  0.998 (0.998)  0.998 (0.998)  -             
                         : 
                         : === Showing confusion matrix for method : PDEFoam        
                         : (Signal Efficiency for Background Efficiency 0.01%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.289 (0.233)  0.467 (0.436)  0.421 (0.332) 
                         :  bg0            0.100 (0.045)  -              0.132 (0.116)  0.540 (0.313) 
                         :  bg1            0.209 (0.434)  0.153 (0.092)  -              0.347 (0.323) 
                         :  bg2            0.560 (0.552)  0.445 (0.424)  0.501 (0.506)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.10%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.665 (0.640)  0.854 (0.822)  0.807 (0.790) 
                         :  bg0            0.538 (0.520)  -              0.415 (0.374)  0.843 (0.833) 
                         :  bg1            0.885 (0.886)  0.542 (0.491)  -              0.728 (0.646) 
                         :  bg2            0.928 (0.890)  0.956 (0.959)  0.847 (0.895)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.30%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.898 (0.878)  0.971 (0.950)  0.982 (0.975) 
                         :  bg0            0.828 (0.810)  -              0.696 (0.676)  0.954 (0.951) 
                         :  bg1            0.951 (0.966)  0.803 (0.745)  -              0.958 (0.966) 
                         :  bg2            0.998 (0.991)  0.998 (0.996)  0.998 (0.993)  -             
                         : 
                         : === Showing confusion matrix for method : DL_CPU         
                         : (Signal Efficiency for Background Efficiency 0.01%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.465 (0.483)  0.930 (0.931)  0.733 (0.703) 
                         :  bg0            0.313 (0.326)  -              0.244 (0.244)  0.563 (0.529) 
                         :  bg1            0.939 (0.921)  0.337 (0.282)  -              0.668 (0.728) 
                         :  bg2            0.687 (0.677)  0.705 (0.687)  0.667 (0.711)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.10%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.879 (0.887)  0.993 (0.992)  0.925 (0.931) 
                         :  bg0            0.762 (0.757)  -              0.715 (0.745)  0.887 (0.853) 
                         :  bg1            0.996 (0.988)  0.790 (0.781)  -              0.892 (0.910) 
                         :  bg2            0.942 (0.937)  0.953 (0.965)  0.931 (0.960)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.30%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.981 (0.976)  0.999 (1.000)  0.998 (1.000) 
                         :  bg0            0.954 (0.929)  -              0.928 (0.919)  0.990 (0.986) 
                         :  bg1            1.000 (0.998)  0.949 (0.954)  -              1.000 (0.998) 
                         :  bg2            0.998 (0.998)  0.999 (1.000)  0.998 (1.000)  -             
                         : 
                         : -------------------------------------------------------------------------------------------------------
                         : 
Dataset:dataset          : Created tree 'TestTree' with 4000 events
                         : 
Dataset:dataset          : Created tree 'TrainTree' with 4000 events
                         : 
Factory                  : Thank you for using TMVA!
                         : For citation information, please visit: http://tmva.sf.net/citeTMVA.html

Save the output

In [10]:
outputFile->Close();

std::cout << "==> Wrote root file: " << outputFile->GetName() << std::endl;
std::cout << "==> TMVAMulticlass is done!" << std::endl;

delete factory;
delete dataloader;
==> Wrote root file: TMVAMulticlass.root
==> TMVAMulticlass is done!

Launch the GUI for the root macros

In [11]:
if (!gROOT->IsBatch()) TMVAMultiClassGui( outfileName );