Brush C++ API
A flexible interpretable machine learning framework
Loading...
Searching...
No Matches
data.h
Go to the documentation of this file.
1/* Brush
2copyright 2020 William La Cava
3license: GNU/GPL v3
4*/
5
6#ifndef DATA_H
7#define DATA_H
8
9// internal includes
10#include "../init.h"
11#include "../util/utils.h"
12#include "../util/error.h"
13#include "../util/logger.h"
14#include "../util/rnd.h"
15#include "timeseries.h"
16//external includes
17#include <variant>
18#include <optional>
19
20namespace Brush
21{
22
23extern map<DataType,string> DataTypeName;
24extern map<string,DataType> DataNameType;
25ostream& operator<<(ostream& os, DataType n);
26
27
28namespace Data
29{
34
35
37State check_type(const ArrayXf& x, const string t);
38DataType StateType(const State& arg);
39
40template<typename StateRef>
41State cast_type(const ArrayXf& x, const StateRef& x_ref);
42
44
50class Dataset
51{
52 //TODO: make this a json object that allows elements to be fetched by
53 //name
54 //Dataset(ArrayXXf& X, ArrayXf& y, std::map<string,
55 //std::pair<vector<ArrayXf>, vector<ArrayXf>>>& Z): X(X), y(y), Z(Z){}
56 private:
57 vector<size_t> training_data_idx;
58 vector<size_t> validation_data_idx;
59 // A batch epoch records row indices and batch boundaries. Classification
60 // epochs may repeat an exhausted minority class to keep it represented.
61 // These are mutable because get_batch advances the sampler while the
62 // dataset values themselves remain immutable.
63 mutable vector<size_t> batch_cycle_idx;
64 mutable vector<size_t> batch_cycle_sizes;
65 mutable size_t batch_cycle_cursor = 0;
66 mutable size_t batch_cycle_batch_cursor = 0;
68 vector<string> feature_name_order_;
69
70 public:
72 std::vector<DataType> unique_data_types;
73
75 std::vector<DataType> feature_types;
76
78 std::vector<string> feature_names;
79
81 std::unordered_map<DataType,vector<string>> features_of_type;
82
84 std::map<string, State> features;
85
86 // TODO: this should probably be a more complex type to include feature type
87 // and potentially other info, like arbitrary relations between features
88
90 ArrayXf y;
91
94 std::optional<std::reference_wrapper<const ArrayXXf>> Xref;
95
100
104
105 Dataset operator()(const vector<size_t>& idx) const;
106
109 void init();
110
112 map<string,State> make_features(const ArrayXXf& X,
113 const map<string, State>& Z = {},
114 const vector<string>& vn = {},
115 const vector<string>& ft = {}
116 );
117
119 map<string,State> copy_and_make_features(const ArrayXXf& X,
120 const Dataset& ref_dataset,
121 const vector<string>& vn = {}
122 );
123
125 Dataset(std::map<string, State>& d,
126 const Ref<const ArrayXf>& y_ = ArrayXf(),
127 bool c = false,
128 float validation_size = 0.0,
129 float batch_size = 1.0,
130 bool shuffle_split = false
131 )
132 : features(d)
133 , y(y_)
134 , classification(c)
138 , use_batch(batch_size > 0.0 && batch_size < 1.0)
140 {init();};
141
143 Dataset(const ArrayXXf& X,
144 const Ref<const ArrayXf>& y_ = ArrayXf(),
145 const vector<string>& vn = {},
146 const map<string, State>& Z = {},
147 const vector<string>& ft = {},
148 bool c = false,
149 float validation_size = 0.0,
150 float batch_size = 1.0,
151 bool shuffle_split = false
152 )
153 : features(make_features(X,Z,vn,ft))
154 , y(y_)
155 , classification(c)
159 , use_batch(batch_size > 0.0 && batch_size < 1.0)
161 {
162 init();
163 Xref = optional<reference_wrapper<const ArrayXXf>>{X};
164 }
165
167 Dataset(const ArrayXXf& X, const vector<string>& vn,
168 const vector<string>& ft = {},
169 bool c = false,
170 float validation_size = 0.0,
171 float batch_size = 1.0,
172 bool shuffle_split = false
173 )
174 : classification(c)
175 , features(make_features(X,map<string, State>{},vn,ft))
179 , use_batch(batch_size > 0.0 && batch_size < 1.0)
181 {
182 init();
183 Xref = optional<reference_wrapper<const ArrayXXf>>{X};
184 }
185
189 Dataset(const ArrayXXf& X, const Dataset& ref_dataset,
190 const vector<string>& vn
191 )
192 : classification(ref_dataset.classification)
193 , features(copy_and_make_features(X,ref_dataset,vn))
194 , validation_size(0.0)
195 , use_validation(false)
196 , batch_size(1.0)
197 , use_batch(false)
198 , shuffle_split(false)
199 {
200 init();
201 Xref = optional<reference_wrapper<const ArrayXXf>>{X};
202 }
203
204 void print() const
205 {
206 fmt::print("Dataset contains {} samples and {} features\n",
208 );
209 for (auto& [key, value] : this->features)
210 {
211 if (std::holds_alternative<ArrayXf>(value))
212 fmt::print("{} <ArrayXf>: {}\n", key, std::get<ArrayXf>(value));
213 else if (std::holds_alternative<ArrayXi>(value))
214 fmt::print("{} <ArrayXi>: {}\n", key, std::get<ArrayXi>(value));
215 else if (std::holds_alternative<ArrayXb>(value))
216 fmt::print("{} <ArrayXb>: {}\n", key, std::get<ArrayXb>(value));
217 }
218
219 };
220 auto get_X() const
221 {
222 if (!Xref.has_value())
223 HANDLE_ERROR_THROW("Dataset does not hold a reference to X.");
224 return this->Xref.value().get();
225 }
226
227 // inner partition of original dataset for train and validation.
228 // if split is not set, then training = validation.
231 vector<string> get_feature_types() const;
232
233 inline int get_n_samples() const {
234 return std::visit(
235 [&](auto&& arg) -> int { return int(arg.size());},
236 features.begin()->second
237 );
238 };
239 inline int get_n_features() const { return this->features.size(); };
242 Dataset get_batch() const;
243
244 float get_batch_size();
245 void set_batch_size(float new_size);
246
247 DataType get_feature_type(const string& name) const
248 {
249 DataType feature_type = DataType::ArrayF;
250
251 bool has_feature = false;
252 for (auto& [ftype, names] : features_of_type){
253 auto it = std::find(names.begin(), names.end(), name);
254
255 if (it != names.end())
256 {
257 feature_type = ftype;
258 has_feature = true;
259 break;
260 }
261 }
262
263 if (!has_feature)
264 HANDLE_ERROR_THROW(fmt::format("Couldn't find feature {} in data\n",name));
265
266 return feature_type;
267 };
268
269 std::array<Dataset, 2> split(const ArrayXb& mask) const;
270
271 State operator[](std::string name) const
272 {
273 if (this->features.find(name) == features.end())
274 HANDLE_ERROR_THROW(fmt::format("Couldn't find feature {} in data\n",name));
275 return this->features.at(name);
276 };
277
278 /* template<> ArrayXb get<ArrayXb>(std::string name) */
279}; // class data
280
281// TODO: serialization of features in order to nlohmann to work
282// NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Dataset,
283// features,
284// y,
285// classification,
286// validation_size,
287// use_validation,
288// batch_size,
289// use_batch,
290// shuffle_split
291// );
292
293// // read csv
294// Dataset read_csv(const std::string & path, MatrixXf& X, VectorXf& y,
295// vector<string>& names, vector<char> &dtypes, bool& binary_endpoint, char sep) ;
296
297} // data
298
299extern const map<DataType,std::type_index> DataTypeID;
300extern map<std::type_index,DataType> DataIDType;
301
302} // Brush
303
304// format overload for DataType
305template <> struct fmt::formatter<Brush::DataType>: formatter<string_view> {
306 template <typename FormatContext>
307 auto format(Brush::DataType x, FormatContext& ctx) const {
308 return formatter<string_view>::format(Brush::DataTypeName.at(x), ctx);
309 }
310};
311
312// TODO: fmt overload for Data
313// template <> struct fmt::formatter<Brush::Data::Dataset>: formatter<string_view> {
314// template <typename FormatContext>
315// auto format(Brush::Data::Dataset& x, FormatContext& ctx) const {
316 // return formatter<string_view>::format(Brush::DataTypeName.at(x), ctx);
317// }
318// };
319
320#endif
holds variable type data.
Definition data.h:51
Dataset(const ArrayXXf &X, const vector< string > &vn, const vector< string > &ft={}, bool c=false, float validation_size=0.0, float batch_size=1.0, bool shuffle_split=false)
Definition data.h:167
bool classification
whether this is a classification problem
Definition data.h:93
vector< string > feature_name_order_
stores the original feature name order before map sorting
Definition data.h:68
Dataset get_validation_data() const
Definition data.cpp:384
std::map< string, State > features
dataset features, as key value pairs
Definition data.h:84
DataType get_feature_type(const string &name) const
Definition data.h:247
int get_n_samples() const
Definition data.h:233
Dataset(std::map< string, State > &d, const Ref< const ArrayXf > &y_=ArrayXf(), bool c=false, float validation_size=0.0, float batch_size=1.0, bool shuffle_split=false)
Definition data.h:125
size_t batch_cycle_batch_cursor
Definition data.h:66
vector< size_t > training_data_idx
Definition data.h:57
auto get_X() const
Definition data.h:220
Dataset get_batch() const
Select the next batch without replacement. Classification batches preserve the prevalence of the rema...
Definition data.cpp:185
vector< string > get_feature_types() const
Definition data.cpp:386
std::vector< DataType > feature_types
types of data in the features.
Definition data.h:75
void print() const
Definition data.h:204
std::unordered_map< DataType, vector< string > > features_of_type
map from data types to features having that type.
Definition data.h:81
float batch_size
percentage of training data size to use in each batch. if 1.0, then all data is used
Definition data.h:102
bool use_validation
Definition data.h:98
int get_n_features() const
Definition data.h:239
Dataset(const ArrayXXf &X, const Dataset &ref_dataset, const vector< string > &vn)
Definition data.h:189
State operator[](std::string name) const
Definition data.h:271
std::optional< std::reference_wrapper< const ArrayXXf > > Xref
Definition data.h:94
std::vector< string > feature_names
names of the feature types as string representations.
Definition data.h:78
std::vector< DataType > unique_data_types
keeps track of the unique data types in the dataset.
Definition data.h:72
map< string, State > copy_and_make_features(const ArrayXXf &X, const Dataset &ref_dataset, const vector< string > &vn={})
turns input into a feature map, with feature types copied from a reference
Definition data.cpp:631
map< string, State > make_features(const ArrayXXf &X, const map< string, State > &Z={}, const vector< string > &vn={}, const vector< string > &ft={})
turns input data into a feature map
Definition data.cpp:560
void init()
call init at the end of constructors to define metafeatures of the data.
Definition data.cpp:419
float validation_size
percentage of original data used for train. if 0.0, then all data is used for train and validation
Definition data.h:97
vector< size_t > validation_data_idx
Definition data.h:58
vector< size_t > batch_cycle_idx
Definition data.h:63
Dataset(const ArrayXXf &X, const Ref< const ArrayXf > &y_=ArrayXf(), const vector< string > &vn={}, const map< string, State > &Z={}, const vector< string > &ft={}, bool c=false, float validation_size=0.0, float batch_size=1.0, bool shuffle_split=false)
Definition data.h:143
ArrayXf y
length N array, the target label
Definition data.h:90
float get_batch_size()
Definition data.cpp:549
Dataset get_training_data() const
Definition data.cpp:383
vector< size_t > batch_cycle_sizes
Definition data.h:64
void set_batch_size(float new_size)
Definition data.cpp:550
Dataset operator()(const vector< size_t > &idx) const
return a slice of the data using indices idx
Definition data.cpp:139
size_t batch_cycle_cursor
Definition data.h:65
holds variable type data.
Definition data.h:51
#define HANDLE_ERROR_THROW(err)
Definition error.h:27
namespace containing Data structures used in Brush
Definition data.cpp:49
State cast_type(const ArrayXf &x, const StateRef &x_ref)
Definition data.cpp:128
DataType StateType(const State &arg)
Definition data.cpp:67
State check_type(const ArrayXf &x, const string t)
determines data types of columns of matrix X.
Definition data.cpp:71
std::variant< ArrayXb, ArrayXi, ArrayXf, ArrayXXb, ArrayXXi, ArrayXXf, TimeSeriesb, TimeSeriesi, TimeSeriesf, ArrayXbJet, ArrayXiJet, ArrayXfJet, ArrayXXbJet, ArrayXXiJet, ArrayXXfJet, TimeSeriesbJet, TimeSeriesiJet, TimeSeriesfJet > State
defines the possible types of data flowing thru nodes.
Definition types.h:140
std::array< Array< T,-1, 1 >, 2 > split(const Array< T,-1, 1 > &v, const ArrayXb &mask)
split Eigen matrix or array into two by mask
Definition utils.h:510
< nsga2 selection operator for getting the front
Definition bandit.cpp:3
ostream & operator<<(ostream &os, DataType n)
Eigen::Array< bool, Eigen::Dynamic, 1 > ArrayXb
Definition types.h:39
DataType
data types.
Definition types.h:143
map< DataType, string > DataTypeName
Definition data.cpp:14
const map< DataType, std::type_index > DataTypeID
Definition data.cpp:36
map< std::type_index, DataType > DataIDType
Definition data.cpp:47
map< string, DataType > DataNameType
Definition data.cpp:34
DataType
data types.
Definition types.h:143
std::variant< ArrayXb, ArrayXi, ArrayXf, ArrayXXb, ArrayXXi, ArrayXXf, TimeSeriesb, TimeSeriesi, TimeSeriesf, ArrayXbJet, ArrayXiJet, ArrayXfJet, ArrayXXbJet, ArrayXXiJet, ArrayXXfJet, TimeSeriesbJet, TimeSeriesiJet, TimeSeriesfJet > State
defines the possible types of data flowing thru nodes.
Definition types.h:140
auto format(Brush::DataType x, FormatContext &ctx) const
Definition data.h:307