Brush C++ API
A flexible interpretable machine learning framework
Loading...
Searching...
No Matches
evaluation.cpp
Go to the documentation of this file.
1#include "evaluation.h"
2
3namespace Brush{
4namespace Eval{
5
6
7// fitness of population
8template<ProgramType T>
10 int island,
11 const Dataset& data,
12 const Parameters& params,
13 bool fit,
14 bool validation,
15 bool force_fit,
16 Dataset* validation_data
17 )
18{
19 auto indices = pop.get_island_indexes(island);
20
21 for (unsigned i = 0; i<indices.size(); ++i)
22 {
23 auto& ind_ptr = pop.individuals.at(indices.at(i));
24
25 // Skip nullptr individuals (offspring slots not yet filled)
26 if (!ind_ptr) {
27 HANDLE_ERROR_THROW(fmt::format(
28 "Evaluation::update_fitness - attempted to update fitness of a nullptr. "
29 "This suggests population integrity was damaged by another manipulation method."));
30
31 }
32
33 Individual<T>& ind = *ind_ptr.get(); // we are modifying it, so operator[] wont work
34
35 if (false) // pass
36 {
37 ind.fitness.set_loss(MAX_FLT);
38 ind.fitness.set_loss_v(MAX_FLT);
39
40 ind.error = MAX_FLT*VectorXf::Ones(data.y.size());
41 }
42 else
43 {
44 // assign weights to individual
45 if (fit && (force_fit || ind.get_is_fitted() == false))
46 {
47 ind.program.fit(data.get_training_data(), params.class_weights);
48 }
49
50 assign_fit(ind, data, params, validation, validation_data);
51 }
52 }
53}
54
55// assign loss to program
56template<ProgramType T>
58 const Parameters& params, bool val,
59 Dataset* validation_data)
60{
62
63 VectorXf errors;
64 using PT = ProgramType;
65
66 Dataset train = data.get_training_data();
67 float f = S.score(ind, train, errors, params);
68 ind.error = errors;
69
70 float f_v = f;
71 if (validation_data != nullptr) {
72 VectorXf val_errors;
73 f_v = S.score(ind, *validation_data, val_errors, params);
74 }
75 else if (data.use_validation) {
76 Dataset validation = data.get_validation_data();
77
78 // when calculating validation score, we should not let
79 // it write in errors vector. That would avoid validation data leakage
80 VectorXf val_errors;
81 f_v = S.score(ind, validation, val_errors, params);
82
83 // if (val) // never use validation data here. This is used in lexicase selection
84 // ind.error = val_errors;
85 }
86
87 float error_weight = Individual<T>::weightsMap[params.scorer];
88 if (std::isnan(f) || std::isinf(f))
89 f = error_weight > 0 ? -MAX_FLT : MAX_FLT;
90 if (std::isnan(f_v) || std::isinf(f_v))
91 f_v = error_weight > 0 ? -MAX_FLT : MAX_FLT;
92
93 // This is what is going to determine the weights for the individual's fitness.
94 // OBS: always use get_objectives, as it will replace the "scorer" string with
95 // the actual scorer function name
96 ind.set_objectives(params.get_objectives());
97
98 // when we use these setters, it updates its previous values references
99 ind.fitness.set_loss(f);
100 ind.fitness.set_loss_v(f_v);
101 ind.fitness.set_size(ind.get_size());
104 ind.fitness.set_depth(ind.get_depth());
105
106 vector<float> values;
107 values.resize(0);
108
109 // TODO: implement a better way of switching between train and val
110 // without the burden of calculating stuff everytime
111 for (const auto& n : ind.get_objectives())
112 {
113 // TODO: this should be done in the fitness class when calling wvalues() or values()
114 if (n.compare(params.scorer)==0)
115 values.push_back(val ? f_v : f);
116 else if (n.compare("complexity")==0)
117 values.push_back(ind.get_complexity());
118 else if (n.compare("linear_complexity")==0)
119 values.push_back(ind.get_linear_complexity());
120 else if (n.compare("size")==0)
121 values.push_back(ind.get_size());
122 else if (n.compare("depth")==0)
123 values.push_back(ind.get_depth());
124 else
125 HANDLE_ERROR_THROW(n+" is not a known objective");
126 }
127
128 // will use inner attributes to set the fitness object
129 ind.fitness.set_values(values);
130}
131
132} // Pop
133} // Brush
134
holds variable type data.
Definition data.h:51
Dataset get_validation_data() const
Definition data.cpp:384
bool use_validation
Definition data.h:98
ArrayXf y
length N array, the target label
Definition data.h:90
Dataset get_training_data() const
Definition data.cpp:383
Class for evaluating the fitness of individuals in a population.
Definition evaluation.h:27
void assign_fit(Individual< T > &ind, const Dataset &data, const Parameters &params, bool val=false, Dataset *validation_data=nullptr)
Assign fitness to an individual.
void update_fitness(Population< T > &pop, int island, const Dataset &data, const Parameters &params, bool fit=true, bool validation=true, bool force_fit=false, Dataset *validation_data=nullptr)
Update the fitness of individuals in a population.
Definition evaluation.cpp:9
size_t n_evaluations
number of calls to assign_fit (each one scores an individual on the training partition and,...
Definition evaluation.h:95
static std::map< std::string, float > weightsMap
set parent ids using id values
Definition individual.h:163
unsigned int get_complexity() const
Definition individual.h:125
unsigned int get_size() const
Definition individual.h:123
VectorXf error
training error (used in lexicase selectors)
Definition individual.h:35
Fitness fitness
aggregate fitness score
Definition individual.h:37
vector< string > get_objectives() const
Definition individual.h:179
void set_objectives(vector< string > objs)
Definition individual.h:180
unsigned int get_depth() const
Definition individual.h:124
unsigned int get_linear_complexity() const
Definition individual.h:126
Program< T > program
executable data structure
Definition individual.h:17
bool get_is_fitted() const
Definition individual.h:140
vector< size_t > get_island_indexes(int island)
Definition population.h:39
vector< std::shared_ptr< Individual< T > > > individuals
Definition population.h:19
#define HANDLE_ERROR_THROW(err)
Definition error.h:27
< nsga2 selection operator for getting the front
Definition bandit.cpp:3
ProgramType PT
Definition program.h:39
ProgramType
Definition types.h:70
Namespace containing scoring functions for evaluation metrics.
void set_linear_complexity(unsigned int new_lc)
Definition fitness.h:80
void set_complexity(unsigned int new_c)
Definition fitness.h:75
void set_loss_v(float f_v)
Definition fitness.h:67
void set_depth(unsigned int new_d)
Definition fitness.h:85
void set_values(vector< float > &v)
Definition fitness.h:129
void set_size(unsigned int new_s)
Definition fitness.h:71
void set_loss(float f)
Definition fitness.h:63
vector< string > get_objectives() const
Definition params.h:148
vector< float > class_weights
weights for each class
Definition params.h:69
string scorer
actual loss function used, determined by error
Definition params.h:66
Program< PType > & fit(const Dataset &d)
Definition program.h:150