40template<
typename StateRef>
94 std::optional<std::reference_wrapper<const ArrayXXf>>
Xref;
113 const map<string, State>& Z = {},
114 const vector<string>& vn = {},
115 const vector<string>& ft = {}
121 const vector<string>& vn = {}
126 const Ref<const ArrayXf>& y_ = ArrayXf(),
144 const Ref<const ArrayXf>& y_ = ArrayXf(),
145 const vector<string>& vn = {},
146 const map<string, State>& Z = {},
147 const vector<string>& ft = {},
163 Xref = optional<reference_wrapper<const ArrayXXf>>{X};
167 Dataset(
const ArrayXXf& X,
const vector<string>& vn,
168 const vector<string>& ft = {},
183 Xref = optional<reference_wrapper<const ArrayXXf>>{X};
190 const vector<string>& vn
201 Xref = optional<reference_wrapper<const ArrayXXf>>{X};
206 fmt::print(
"Dataset contains {} samples and {} features\n",
209 for (
auto& [key, value] : this->features)
211 if (std::holds_alternative<ArrayXf>(value))
212 fmt::print(
"{} <ArrayXf>: {}\n", key, std::get<ArrayXf>(value));
213 else if (std::holds_alternative<ArrayXi>(value))
214 fmt::print(
"{} <ArrayXi>: {}\n", key, std::get<ArrayXi>(value));
215 else if (std::holds_alternative<ArrayXb>(value))
216 fmt::print(
"{} <ArrayXb>: {}\n", key, std::get<ArrayXb>(value));
222 if (!
Xref.has_value())
224 return this->Xref.value().get();
235 [&](
auto&& arg) ->
int {
return int(arg.size());},
251 bool has_feature =
false;
253 auto it = std::find(names.begin(), names.end(), name);
255 if (it != names.end())
257 feature_type = ftype;
269 std::array<Dataset, 2>
split(
const ArrayXb& mask)
const;
273 if (this->features.find(name) ==
features.end())
275 return this->features.at(name);
299extern const map<DataType,std::type_index>
DataTypeID;
300extern map<std::type_index,DataType>
DataIDType;
306 template <
typename FormatContext>
holds variable type data.
Dataset(const ArrayXXf &X, const vector< string > &vn, const vector< string > &ft={}, bool c=false, float validation_size=0.0, float batch_size=1.0, bool shuffle_split=false)
bool classification
whether this is a classification problem
vector< string > feature_name_order_
stores the original feature name order before map sorting
Dataset get_validation_data() const
std::map< string, State > features
dataset features, as key value pairs
DataType get_feature_type(const string &name) const
int get_n_samples() const
Dataset(std::map< string, State > &d, const Ref< const ArrayXf > &y_=ArrayXf(), bool c=false, float validation_size=0.0, float batch_size=1.0, bool shuffle_split=false)
size_t batch_cycle_batch_cursor
vector< size_t > training_data_idx
Dataset get_batch() const
Select the next batch without replacement. Classification batches preserve the prevalence of the rema...
vector< string > get_feature_types() const
std::vector< DataType > feature_types
types of data in the features.
std::unordered_map< DataType, vector< string > > features_of_type
map from data types to features having that type.
float batch_size
percentage of training data size to use in each batch. if 1.0, then all data is used
int get_n_features() const
Dataset(const ArrayXXf &X, const Dataset &ref_dataset, const vector< string > &vn)
State operator[](std::string name) const
std::optional< std::reference_wrapper< const ArrayXXf > > Xref
std::vector< string > feature_names
names of the feature types as string representations.
std::vector< DataType > unique_data_types
keeps track of the unique data types in the dataset.
map< string, State > copy_and_make_features(const ArrayXXf &X, const Dataset &ref_dataset, const vector< string > &vn={})
turns input into a feature map, with feature types copied from a reference
map< string, State > make_features(const ArrayXXf &X, const map< string, State > &Z={}, const vector< string > &vn={}, const vector< string > &ft={})
turns input data into a feature map
void init()
call init at the end of constructors to define metafeatures of the data.
float validation_size
percentage of original data used for train. if 0.0, then all data is used for train and validation
vector< size_t > validation_data_idx
vector< size_t > batch_cycle_idx
Dataset(const ArrayXXf &X, const Ref< const ArrayXf > &y_=ArrayXf(), const vector< string > &vn={}, const map< string, State > &Z={}, const vector< string > &ft={}, bool c=false, float validation_size=0.0, float batch_size=1.0, bool shuffle_split=false)
ArrayXf y
length N array, the target label
Dataset get_training_data() const
vector< size_t > batch_cycle_sizes
void set_batch_size(float new_size)
Dataset operator()(const vector< size_t > &idx) const
return a slice of the data using indices idx
size_t batch_cycle_cursor
holds variable type data.
#define HANDLE_ERROR_THROW(err)
namespace containing Data structures used in Brush
State cast_type(const ArrayXf &x, const StateRef &x_ref)
DataType StateType(const State &arg)
State check_type(const ArrayXf &x, const string t)
determines data types of columns of matrix X.
std::variant< ArrayXb, ArrayXi, ArrayXf, ArrayXXb, ArrayXXi, ArrayXXf, TimeSeriesb, TimeSeriesi, TimeSeriesf, ArrayXbJet, ArrayXiJet, ArrayXfJet, ArrayXXbJet, ArrayXXiJet, ArrayXXfJet, TimeSeriesbJet, TimeSeriesiJet, TimeSeriesfJet > State
defines the possible types of data flowing thru nodes.
std::array< Array< T,-1, 1 >, 2 > split(const Array< T,-1, 1 > &v, const ArrayXb &mask)
split Eigen matrix or array into two by mask
< nsga2 selection operator for getting the front
ostream & operator<<(ostream &os, DataType n)
Eigen::Array< bool, Eigen::Dynamic, 1 > ArrayXb
map< DataType, string > DataTypeName
const map< DataType, std::type_index > DataTypeID
map< std::type_index, DataType > DataIDType
map< string, DataType > DataNameType
std::variant< ArrayXb, ArrayXi, ArrayXf, ArrayXXb, ArrayXXi, ArrayXXf, TimeSeriesb, TimeSeriesi, TimeSeriesf, ArrayXbJet, ArrayXiJet, ArrayXfJet, ArrayXXbJet, ArrayXXiJet, ArrayXXfJet, TimeSeriesbJet, TimeSeriesiJet, TimeSeriesfJet > State
defines the possible types of data flowing thru nodes.