ECF 1.5
AlgXCS.cpp
1#include "../ECF_base.h"
2#include "../SelRandomOp.h"
3#include "../SelFitnessProportionalOp.h"
4#include "../SelWorstOp.h"
5#include "../ECF_macro.h"
6
7#include "AlgXCS.h"
8#include "math.h"
9#include "time.h"
10#include "../Logger.h"
11#include <string>
12//
13//#undef XCS_STEP_DEBUG
14//#undef XCS_DEBUG
15
16#pragma region Intro comments
17//References:
18// [1] An algorithmic Description of XCS, Martin V.
19// Butz and Stewart W. Wilson
20// [2] Rule-Based Evolutionary Online Learning Systems
21// Martin V. Butz
22
23/*System characteristics:
24 - single-step problems
25 - multi-step problems
26*/
27#pragma endregion
28
29//Compare function used for sorting classifier by fitness value
30bool cmp (ClassifierP a, ClassifierP b){
31 return a->ind->fitness->getValue() > b->ind->fitness->getValue();
32}
33using namespace std;
34
35XCS::XCS()
36{
37 name_ = "XCS";
38
39 //TODO: izbacit randomOp i worstOp
40 selRandomOp = static_cast<SelectionOperatorP> (new SelRandomOp);
41 selFitPropOp = static_cast<SelectionOperatorP> (new SelFitnessProportionalOp);
42 selWorstOp = static_cast<SelectionOperatorP> (new SelWorstOp);
43
44 params = static_cast<XCSParamsP> ( new XCSParams(name_));
45}
46
47
48//Method for registering algorithm parameters
49void XCS::registerParameters(StateP state)
50{
51 params->registerParams(state->getRegistry());
52
53 //Defining aditional genotype used for storing classifier parameters
54 ClassifierParamsP params = static_cast<ClassifierParamsP> (new ClassifierParams(0,0,0));
55 state->addGenotype(params);
56
57}
58bool XCS::initialize(StateP state)
59{
60 if (!Classifier::checkState(state)) {
61 throw ("");
62 }
63
64 selRandomOp->initialize(state);
65 selWorstOp->initialize(state);
66 selFitPropOp->initialize(state);
67
68 params->readParams(state->getRegistry());
69
70 time = 0;
71
72 ClassifierParamsP clParams = static_cast<ClassifierParamsP> (new ClassifierParams(0,0,0));
73 clParams->setGenotypeId(3);
74 clParams->initialize(state);
75 params->initF_ = clParams->F_;
76
77 //Environment initialization
78 environment = boost::dynamic_pointer_cast<Environment> (evalOp_);
79
80 if (!environment->checkState(state)) {
81 throw ("");
82 }
83
84 try {
85 if (!environment->initialize()){
86 throw (std::string("failed to initialize"));
87 }
88 }catch (std::string text){
89 ECF_LOG_ERROR(state, "Environment error: "+text);
90 throw ("");
91 }
92
93 return true;
94}
95void XCS::printPopulation(){
96 //sort(populationSet.begin(), populationSet.end(), cmp);
97 for (uint i = 0; i < populationSet.size(); i++)
98 populationSet[i]->print();
99}
100
101bool XCS::advanceGeneration(StateP state, DemeP deme) {
102
103 if (state->getGenerationNo() == 0){
104
105 //creating population set [P]
106 ClassifierP classP;
107 PopulationP pop = state->getPopulation();
108
109 for (uint i = 0; i < pop->size(); i++){
110 for (uint j = 0; j < pop->at(i)->size(); j++){
111 classP = static_cast<ClassifierP>
112 (new Classifier(params, time, pop->at(i)->at(j), state));
113 populationSet.push_back(classP);
114 }
115 }
116
117 for (uint i = 0; i < deme->size(); i++)
118 deme->at(i)->fitness->setValue(params->initF_);
119 environment->reset();
120 }
121
122 std::vector<ClassifierP> lastActionSet; //[A]-1
123 double lastReward = 0;
124 GenotypeP lastInput;
125
126 //main generation loop
127 do {
128
129#ifdef XCS_STEP_DEBUG
130 std::cout << "Press any key to continue..." << std::endl;
131 std::cin.get();
132#endif
133 time++;
134 std::vector<ClassifierP> matchSet; //[M]
135 std::vector<ClassifierP> actionSet;//[A]
136
137 //Getting input from environment
138 GenotypeP input = environment->getInput();
139
140#ifdef XCS_DEBUG
141 std::cout << "Input value: ";
142 Classifier::printBitString(boost::dynamic_pointer_cast<BitString::BitString> (input));
143 std::cout << std::endl;
144#endif
145 //Generate match set [M]
146 matchSet = generateMatchSet(state, deme, input); //[M]
147
148#ifdef XCS_DEBUG
149 std::cout << "Match set [M]: "<< std::endl;
150 for (uint i = 0; i < matchSet.size(); i++)
151 matchSet[i]->print();
152#endif
153
154 //creating prediction array PA
155 std::map<int, double> PA = generatePA(matchSet);
156 //selecting action id
157
158 int actionId = selectActionId(state, PA);
159#ifdef XCS_DEBUG
160 std::cout << "action id = " << actionId<< std::endl;
161#endif
162
163 //generating action set [A]
164 actionSet = generateActionSet(matchSet, actionId);
165
166#ifdef XCS_DEBUG
167 std::cout << "\nAction set [A]: "<< std::endl;
168 for (uint i = 0; i < actionSet.size(); i++)
169 actionSet[i]->print();
170#endif
171
172 //executing selected action
173 IndividualP ind = static_cast<IndividualP> (new Individual);
174 ind->push_back(input);
175 ind->push_back(actionSet[0]->getAction());
176
177 //getting reward from environment
178 evaluate(ind);
179 double reward = ind->fitness->getValue();
180
181#ifdef XCS_DEBUG
182 std::cout << "Reward: " << reward << std::endl;
183#endif
184
185 if (!lastActionSet.empty()){
186 double P = lastReward + params->gama_ * getMaxFromPA(PA).second;
187
188 updateActionSet(lastActionSet, deme, P, state);
189 if (!environment->isExploit()) {
190 runGA(lastActionSet, lastInput, deme, state);
191 }
192 }
193 if (environment->isOver()) {
194
195 if (!environment->isExploit()) {
196 updateActionSet(actionSet, deme, reward, state);
197
198 runGA(actionSet, input, deme, state);
199 lastActionSet.clear();
200 }
201
202 } else {
203 lastActionSet = actionSet;
204 lastReward = reward;
205 lastInput = input;
206 }
207 } while (!environment->isOver());
208
209 environment->nextTrial();
210
211#ifdef XCS_DEBUG
212 std::cout << "Classifiers:" << std::endl;
213 printPopulation();
214 std::cout << " ===== advanceGeneration end ====="<< std::endl;
215#endif
216
217 return true;
218}
219
220std::vector<ClassifierP> XCS::generateMatchSet(StateP state, DemeP deme, GenotypeP input) {
221
222 //(!) napomena mora vrijediti deme->at(i) == vClassifier[i].ind
223 std::vector<ClassifierP> matchSet;
224
225 while(matchSet.empty()){
226
227 for (uint i = 0; i < populationSet.size(); ++i){
228 if (populationSet[i]->doesMatch(input))
229 matchSet.push_back(populationSet[i]);
230 }
231
232 uint noDiffActions = (uint) getActionsFromMs(matchSet).size();
233 if (noDiffActions < params->mna_){
234
235 ClassifierP coverCl = cover(state, deme, input, matchSet);
236 deleteFromPopulation(state, deme);
237 matchSet.clear();
238 }
239 }
240
241 return matchSet;
242}
243
244std::set<int> XCS::getActionsFromMs (std::vector<ClassifierP> matchSet){
245
246 std::set<int> actions;
247 for (uint i = 0; i < matchSet.size(); ++i){
248 actions.insert(matchSet[i]->getActionId());
249 }
250 return actions;
251}
252
253//Creates cover classifier according to input value
254//and inserts it in populationSet and deme
255ClassifierP XCS::cover (StateP state, DemeP deme, GenotypeP input, std::vector<ClassifierP> matchSet){
256
257 IndividualP newInd = static_cast<IndividualP> (new Individual(state));
258
259 ClassifierP newClassifier = static_cast<ClassifierP> (new
260 Classifier (params,time, newInd, state));
261
262 //classifier must match input so that it can be added in [M]
263 newClassifier->cover(getActionsFromMs(matchSet), input, state);
264 newInd->index = (uint)deme->size();
265
266 populationSet.push_back(newClassifier);
267 deme->push_back(newInd);
268
269 assert(deme->size() == populationSet.size());
270
271 return newClassifier;
272}
273
274//Generates prediction array from match set
275std::map<int, double> XCS::generatePA(std::vector<ClassifierP> matchSet){
276 std::map<int, double> PA;
277 std::map<int, double> fsa; //fitness sum array
278
279 for (uint i = 0; i < matchSet.size(); ++i){
280
281 ClassifierP cl = matchSet[i];
282 double fitness = cl->getFitness();
283 int action = cl->getActionId();
284
285 if(PA[action] == NULL) {
286 PA[action] = cl->getPrediction() * fitness;
287 fsa[action] = 0;
288 } else
289 PA[action] += cl->getPrediction() * fitness;
290 fsa[action] += fitness;
291 }
292 for (std::map<int, double>::iterator it = PA.begin(); it != PA.end(); ++it){
293 PA[it->first] /= (fsa[it->first] == 0 ? 1 : fsa[it->first]);
294 }
295 return PA;
296}
297
298
299int XCS:: selectActionId(StateP state, std::map<int, double> PA){
300 double rnd = state->getRandomizer()->getRandomDouble();
301 int actionId = PA.begin()->first;
302
303 //if it is exploit experiment
304 if (environment->isExploit())
305 return getMaxFromPA(PA).first;
306
307 if (rnd < params->p_explore_) {
308 //exploration
309 rnd = state->getRandomizer()->getRandomDouble();
310 int i = 1;
311 for (std::map<int, double>::const_iterator it = PA.begin(); it != PA.end(); ++it){
312 if (i > PA.size() * rnd){
313 actionId = it->first;
314 break;
315 }
316 i++;
317 }
318 }else {
319 //exploitation
320 actionId = getMaxFromPA(PA).first;
321 }
322 return actionId;
323}
324
325std::vector<ClassifierP> XCS::generateActionSet(std::vector<ClassifierP> matchSet, int actionId){
326 std::vector<ClassifierP> actionSet;
327 for (uint i = 0; i < matchSet.size(); i++)
328 if (matchSet[i]->getActionId() == actionId)
329 actionSet.push_back(matchSet[i]);
330 return actionSet;
331}
332
333void XCS::updateActionSet(std::vector<ClassifierP> actionSet, DemeP deme, double reward, StateP state){
334
335 ClassifierP cl;
336 double sum = 0;
337 std::vector<double> accuracy;
338
339 int numSum = 0;
340 for (uint i = 0; i < actionSet.size(); ++i)
341 numSum += actionSet[i]->getNumerosity();
342
343 //update exp, eps, p and as
344 for (uint i = 0; i < actionSet.size(); ++i){
345 cl = actionSet[i];
346
347 double exp = cl->getExperience() + 1;
348 cl->setExperience(exp);
349
350 double eps = cl->getError();
351 double p = cl->getPrediction();
352 double as = cl->getActSetSize();
353
354 if (exp < 1 / params->beta_) {
355 p += (reward - p) / exp;
356 eps += (fabs(reward - p) - eps) / exp;
357 as += (numSum - as) / exp;
358 } else {
359 p += params->beta_ * (reward - p);
360 eps += params->beta_ * (fabs(reward - p) - eps);
361 as += params->beta_ * (numSum - as);
362 }
363
364 if (eps < params->eps0_)
365 accuracy.push_back(1);
366 else
367 accuracy.push_back( params->alpha_ * pow(eps / params->eps0_, -params->accExp_) );
368
369 sum += accuracy[i] * cl->getNumerosity();
370
371 cl->setPrediction(p);
372 cl->setError(eps);
373 cl->setActSetSize(as);
374 }
375
376 //update fitness
377 for (uint i = 0; i < actionSet.size(); ++i){
378
379 cl = actionSet[i];
380
381 double F = cl->getFitness();
382 F += params->beta_ * (accuracy[i] * cl->getNumerosity() / sum - F) ;
383 cl->setFitness(F);
384 }
385
386 actionSetSubsumption(&actionSet, deme, state);
387
388}
389
390double XCS::getAsTimeSum(std::vector<ClassifierP> as) {
391
392 double sumNum = 0, sumTsNum = 0;
393 for (uint i = 0; i < as.size(); ++i){
394 sumTsNum += as[i]->getTimeStamp() * as[i]->getNumerosity();
395 sumNum += as[i]->getNumerosity();
396 }
397
398 return sumTsNum / sumNum;
399}
400
401void XCS::runGA(std::vector<ClassifierP> actionSet, GenotypeP genInput, DemeP deme, StateP state){
402
403 if (time - getAsTimeSum(actionSet) < params->thresholdGA_) return;
404
405 std::vector<IndividualP> tournament, vActionSet;
406
407 for (uint i = 0; i < actionSet.size(); i++) {
408 if (!actionSet[i]->valid) continue;
409 vActionSet.push_back(actionSet[i]->ind);
410 actionSet[i]->setTimeStamp(time);
411 }
412 if (vActionSet.size() < 1) return;
413
414 IndividualP parent[2];
415 parent[0] = selFitPropOp->select(vActionSet);
416 parent[1] = selFitPropOp->select(vActionSet);
417
418 ClassifierP clParent[2];
419 clParent[0] = populationSet[parent[0]->index];
420 clParent[1] = populationSet[parent[1]->index];
421
422 assert(clParent[0]->getActionId() == clParent[1]->getActionId());
423 ClassifierP clChild[2];
424
425 for (int i = 0; i < 2; ++i) {
426
427 clChild[i] = static_cast<ClassifierP> (new Classifier(clParent[i]));
428
429 clChild[i]->setNumerosity(1);
430 clChild[i]->setExperience(0);
431
432 double f = clChild[i]->getFitness();
433
434 if (state->getRandomizer()->getRandomDouble() < params->pCrossover_) {
435 mate(parent[0], parent[1], clChild[i]->ind);
436
437 clChild[i]->setAction(clParent[0]->getAction());
438
439 assert(clChild[i]->getActionId() == clParent[i]->getActionId());
440
441 clChild[i]->setPrediction((clParent[0]->getPrediction() + clParent[1]->getPrediction()) / 2);
442 clChild[i]->setError((clParent[0]->getError() + clParent[1]->getError()) / 2);
443
444 f = (clParent[0]->getFitness() + clParent[1]->getFitness()) / 2;
445 }
446
447 clChild[i]->setFitness( 0.1 * f);
448
449 clChild[i]->mutateRule(genInput, state);
450 clChild[i]->mutateAction(state);
451
452 if (clParent[0]->doesSubsume(clChild[i])) {
453 clParent[0]->setNumerosity(clParent[0]->getNumerosity() + 1);
454 } else if (clParent[1]->doesSubsume(clChild[i])) {
455 clParent[1]->setNumerosity(clParent[1]->getNumerosity() + 1);
456 } else {
457 clChild[i]->ind->index = (uint)deme->size();
458 populationSet.push_back(clChild[i]);
459 deme->push_back(clChild[i]->ind);
460 }
461 assert(deme->size() == populationSet.size());
462
463 deleteFromPopulation(state, deme);
464 }
465}
466
467std::pair<int, double> XCS::getMaxFromPA(std::map<int, double> PA){
468
469 if (PA.empty()) return std::make_pair(-1, -1.);
470
471 std::pair<int, double> maxPA = *PA.begin();
472 for (std::map<int, double>::iterator it = PA.begin(); it != PA.end(); ++it){
473 if (it->second > maxPA.second)
474 maxPA = *it;
475 }
476 return maxPA;
477}
478
479void XCS::deleteFromPopulation(StateP state, DemeP deme) {
480
481 int sumNum = 0;
482 double sumFit = 0;
483
484 for ( uint i = 0; i < populationSet.size(); ++i) {
485 sumNum += populationSet[i]->getNumerosity();
486 sumFit += populationSet[i]->getFitness();
487 }
488 if (sumNum <= (int) params->popSize_) return;
489
490 double avFitInPop = sumFit / sumNum;
491 double sumVote = 0;
492
493 for ( uint i = 0; i < populationSet.size(); ++i) {
494 sumVote += populationSet[i]->getDeletionVote(avFitInPop);
495 }
496 double choicePoint = state->getRandomizer()->getRandomDouble() * sumVote;
497 sumVote = 0;
498 for ( uint i = 0; i < populationSet.size(); ++i) {
499 ClassifierP cl = populationSet[i];
500 sumVote += cl->getDeletionVote(avFitInPop);
501 if (sumVote > choicePoint) {
502 int n = cl->getNumerosity();
503 if (n > 1){
504 cl->setNumerosity(n-1);
505 } else {
506 removeFromPopSet(cl, deme);
507 }
508 return;
509 }
510 }
511 assert(deme->size() == populationSet.size());
512
513}
514
515void XCS::removeFromPopSet(ClassifierP cl, DemeP deme){
516
517 IndividualP lastInd = deme->at(deme->size()-1);
518 ClassifierP lastCl = populationSet[populationSet.size()-1];
519
520 lastInd->index = cl->ind->index;
521
522 populationSet[cl->ind->index] = lastCl;
523 populationSet.erase(populationSet.begin() + populationSet.size() - 1);
524
525 deme->at(cl->ind->index) = lastInd;
526 deme->erase(deme->begin() + deme->size() - 1);
527
528 assert(deme->size() == populationSet.size());
529 cl->valid = false;
530}
531
532void XCS::actionSetSubsumption(std::vector<ClassifierP> *actionSet, DemeP deme, StateP state){
533 ClassifierP cl;
534 bool clSet = false;
535
536 for (uint i = 0; i < actionSet->size(); ++i){
537
538 ClassifierP c = actionSet->at(i);
539 if (c->couldSubsume()){
540
541 int clDcb = c->numOfDCBits();
542 double rnd = state->getRandomizer()->getRandomDouble();
543 if (!clSet || clDcb > cl->numOfDCBits() || (clDcb == cl->numOfDCBits() && rnd < 0.5 )) {
544 cl = c;
545 clSet = true;
546 }
547 }
548 }
549 if (clSet) {
550 for (uint i = 0; i < actionSet->size(); ++i){
551 ClassifierP c = actionSet->at(i);
552 if (cl->isMoreGeneral(c)){
553 cl->setNumerosity(cl->getNumerosity() + c->getNumerosity());
554 actionSet->erase(actionSet->begin() + i);
555 removeFromPopSet(c,deme);
556 }
557 }
558 }
559}
EvaluateOpP evalOp_
sptr to evaluation operator (set by the system)
Definition: Algorithm.h:82
std::string name_
algorithm name
Definition: Algorithm.h:23
bool mate(IndividualP p1, IndividualP p2, IndividualP child)
Helper function: crossover two individuals.
Definition: Algorithm.h:285
void evaluate(IndividualP ind)
Helper function: evaluate an individual.
Definition: Algorithm.h:157
Classifier class that holds all parameters and pointer to individual to which the parameters belong.
Definition: Classifier.h:20
Classifier data structure in XCS algorithm.
Individual class - inherits a vector of Genotype objects.
Definition: Individual.h:12
Fitness proportional (and inverse proportional) individual selection operator.
Random individual selection operator.
Definition: SelRandomOp.h:12
Worst individual selection operator.
Definition: SelWorstOp.h:11
bool advanceGeneration(StateP state, DemeP deme)
Perform a single generation on a single deme.
Definition: AlgXCS.cpp:101
bool initialize(StateP state)
Initialize the algorithm, read parameters from the system, do a sanity check.
Definition: AlgXCS.cpp:58
void registerParameters(StateP state)
Register algorithm's parameters (if any).
Definition: AlgXCS.cpp:49
Parameters for the XCS algorithm.
Definition: XCSParams.h:12