13 #include "book_make.h"
\r
15 #include "move_do.h"
\r
16 #include "move_legal.h"
\r
23 static const int COUNT_MAX = 16384;
\r
25 static const int NIL = -1;
\r
49 static double MinScore;
\r
50 static bool RemoveWhite, RemoveBlack;
\r
51 static bool Uniform;
\r
53 static book_t Book[1];
\r
57 static void book_clear ();
\r
58 static void book_insert (const char file_name[]);
\r
59 static void book_filter ();
\r
60 static void book_sort ();
\r
61 static void book_save (const char file_name[]);
\r
63 static int find_entry (const board_t * board, int move);
\r
64 static void resize ();
\r
65 static void halve_stats (uint64 key);
\r
67 static bool keep_entry (int pos);
\r
69 static int entry_score (const entry_t * entry);
\r
71 static int key_compare (const void * p1, const void * p2);
\r
73 static void write_integer (FILE * file, int size, uint64 n);
\r
79 void book_make(int argc, char * argv[]) {
\r
82 const char * pgn_file;
\r
83 const char * bin_file;
\r
86 my_string_set(&pgn_file,"book.pgn");
\r
89 my_string_set(&bin_file,"book.bin");
\r
94 RemoveWhite = false;
\r
95 RemoveBlack = false;
\r
98 for (i = 1; i < argc; i++) {
\r
102 } else if (my_string_equal(argv[i],"make-book")) {
\r
106 } else if (my_string_equal(argv[i],"-pgn")) {
\r
109 if (argv[i] == NULL) my_fatal("book_make(): missing argument\n");
\r
111 my_string_set(&pgn_file,argv[i]);
\r
113 } else if (my_string_equal(argv[i],"-bin")) {
\r
116 if (argv[i] == NULL) my_fatal("book_make(): missing argument\n");
\r
118 my_string_set(&bin_file,argv[i]);
\r
120 } else if (my_string_equal(argv[i],"-max-ply")) {
\r
123 if (argv[i] == NULL) my_fatal("book_make(): missing argument\n");
\r
125 MaxPly = atoi(argv[i]);
\r
128 } else if (my_string_equal(argv[i],"-min-game")) {
\r
131 if (argv[i] == NULL) my_fatal("book_make(): missing argument\n");
\r
133 MinGame = atoi(argv[i]);
\r
136 } else if (my_string_equal(argv[i],"-min-score")) {
\r
139 if (argv[i] == NULL) my_fatal("book_make(): missing argument\n");
\r
141 MinScore = atof(argv[i]) / 100.0;
\r
142 ASSERT(MinScore>=0.0&&MinScore<=1.0);
\r
144 } else if (my_string_equal(argv[i],"-only-white")) {
\r
146 RemoveWhite = false;
\r
147 RemoveBlack = true;
\r
149 } else if (my_string_equal(argv[i],"-only-black")) {
\r
151 RemoveWhite = true;
\r
152 RemoveBlack = false;
\r
154 } else if (my_string_equal(argv[i],"-uniform")) {
\r
160 my_fatal("book_make(): unknown option \"%s\"\n",argv[i]);
\r
166 printf("inserting games ...\n");
\r
167 book_insert(pgn_file);
\r
169 printf("filtering entries ...\n");
\r
172 printf("sorting entries ...\n");
\r
175 printf("saving entries ...\n");
\r
176 book_save(bin_file);
\r
178 printf("all done!\n");
\r
183 static void book_clear() {
\r
188 Book->mask = (Book->alloc * 2) - 1;
\r
190 Book->entry = (entry_t *) my_malloc(Book->alloc*sizeof(entry_t));
\r
193 Book->hash = (sint32 *) my_malloc((Book->alloc*2)*sizeof(sint32));
\r
194 for (index = 0; index < Book->alloc*2; index++) {
\r
195 Book->hash[index] = NIL;
\r
201 static void book_insert(const char file_name[]) {
\r
211 ASSERT(file_name!=NULL);
\r
218 pgn_open(pgn,file_name);
\r
220 while (pgn_next_game(pgn)) {
\r
222 board_start(board);
\r
227 } else if (my_string_equal(pgn->result,"1-0")) {
\r
229 } else if (my_string_equal(pgn->result,"0-1")) {
\r
233 while (pgn_next_move(pgn,string,256)) {
\r
235 if (ply < MaxPly) {
\r
237 move = move_from_san(string,board);
\r
239 if (move == MoveNone || !move_is_legal(move,board)) {
\r
240 my_fatal("book_insert(): illegal move \"%s\" at line %d, column %d,game %d\n",string,pgn->move_line,pgn->move_column,pgn->game_nb);
\r
243 pos = find_entry(board,move);
\r
245 Book->entry[pos].n++;
\r
246 Book->entry[pos].sum += result+1;
\r
248 if (Book->entry[pos].n >= COUNT_MAX) {
\r
249 halve_stats(board->key);
\r
252 move_do(board,move);
\r
258 if (pgn->game_nb % 10000 == 0) printf("%d games ...\n",pgn->game_nb);
\r
263 printf("%d game%s.\n",pgn->game_nb,(pgn->game_nb>2)?"s":"");
\r
264 printf("%d entries.\n",Book->size);
\r
271 static void book_filter() {
\r
279 for (src = 0; src < Book->size; src++) {
\r
280 if (keep_entry(src)) Book->entry[dst++] = Book->entry[src];
\r
283 ASSERT(dst>=0&&dst<=Book->size);
\r
286 printf("%d entries.\n",Book->size);
\r
291 static void book_sort() {
\r
293 // sort keys for binary search
\r
295 qsort(Book->entry,Book->size,sizeof(entry_t),&key_compare);
\r
300 static void book_save(const char file_name[]) {
\r
305 ASSERT(file_name!=NULL);
\r
307 file = fopen(file_name,"wb");
\r
308 if (file == NULL) my_fatal("book_save(): can't open file \"%s\" for writing: %s\n",file_name,strerror(errno));
\r
312 for (pos = 0; pos < Book->size; pos++) {
\r
314 ASSERT(keep_entry(pos));
\r
316 write_integer(file,8,Book->entry[pos].key);
\r
317 write_integer(file,2,Book->entry[pos].move);
\r
318 write_integer(file,2,entry_score(&Book->entry[pos]));
\r
319 write_integer(file,2,0);
\r
320 write_integer(file,2,0);
\r
328 static int find_entry(const board_t * board, int move) {
\r
334 ASSERT(board!=NULL);
\r
335 ASSERT(move_is_ok(move));
\r
337 ASSERT(move_is_legal(move,board));
\r
345 for (index = key & Book->mask; (pos=Book->hash[index]) != NIL; index = (index+1) & Book->mask) {
\r
347 ASSERT(pos>=0&&pos<Book->size);
\r
349 if (Book->entry[pos].key == key && Book->entry[pos].move == move) {
\r
350 return pos; // found
\r
356 ASSERT(Book->size<=Book->alloc);
\r
358 if (Book->size == Book->alloc) {
\r
360 // allocate more memory
\r
364 for (index = key & Book->mask; Book->hash[index] != NIL; index = (index+1) & Book->mask)
\r
368 // create a new entry
\r
370 ASSERT(Book->size<Book->alloc);
\r
371 pos = Book->size++;
\r
373 Book->entry[pos].key = key;
\r
374 Book->entry[pos].move = move;
\r
375 Book->entry[pos].n = 0;
\r
376 Book->entry[pos].sum = 0;
\r
377 Book->entry[pos].colour = board->turn;
\r
379 // insert into the hash table
\r
381 ASSERT(index>=0&&index<Book->alloc*2);
\r
382 ASSERT(Book->hash[index]==NIL);
\r
383 Book->hash[index] = pos;
\r
385 ASSERT(pos>=0&&pos<Book->size);
\r
392 static void resize() {
\r
398 ASSERT(Book->size==Book->alloc);
\r
401 Book->mask = (Book->alloc * 2) - 1;
\r
404 size += Book->alloc * sizeof(entry_t);
\r
405 size += (Book->alloc*2) * sizeof(sint32);
\r
406 if (size >= 1048576) printf("allocating %gMB ...\n",double(size)/1048576.0);
\r
409 Book->entry = (entry_t *) my_realloc(Book->entry,Book->alloc*sizeof(entry_t));
\r
410 Book->hash = (sint32 *) my_realloc(Book->hash,(Book->alloc*2)*sizeof(sint32));
\r
412 // rebuild hash table
\r
414 for (index = 0; index < Book->alloc*2; index++) {
\r
415 Book->hash[index] = NIL;
\r
418 for (pos = 0; pos < Book->size; pos++) {
\r
420 for (index = Book->entry[pos].key & Book->mask; Book->hash[index] != NIL; index = (index+1) & Book->mask)
\r
423 ASSERT(index>=0&&index<Book->alloc*2);
\r
424 Book->hash[index] = pos;
\r
430 static void halve_stats(uint64 key) {
\r
437 for (index = key & Book->mask; (pos=Book->hash[index]) != NIL; index = (index+1) & Book->mask) {
\r
439 ASSERT(pos>=0&&pos<Book->size);
\r
441 if (Book->entry[pos].key == key) {
\r
442 Book->entry[pos].n = (Book->entry[pos].n + 1) / 2;
\r
443 Book->entry[pos].sum = (Book->entry[pos].sum + 1) / 2;
\r
450 static bool keep_entry(int pos) {
\r
452 const entry_t * entry;
\r
456 ASSERT(pos>=0&&pos<Book->size);
\r
458 entry = &Book->entry[pos];
\r
460 // if (entry->n == 0) return false;
\r
461 if (entry->n < MinGame) return false;
\r
463 if (entry->sum == 0) return false;
\r
465 score = (double(entry->sum) / double(entry->n)) / 2.0;
\r
466 ASSERT(score>=0.0&&score<=1.0);
\r
468 if (score < MinScore) return false;
\r
470 colour = entry->colour;
\r
472 if ((RemoveWhite && colour_is_white(colour))
\r
473 || (RemoveBlack && colour_is_black(colour))) {
\r
477 if (entry_score(entry) == 0) return false; // REMOVE ME?
\r
484 static int entry_score(const entry_t * entry) {
\r
488 ASSERT(entry!=NULL);
\r
490 // score = entry->n; // popularity
\r
491 score = entry->sum; // "expectancy"
\r
493 if (Uniform) score = 1;
\r
502 static int key_compare(const void * p1, const void * p2) {
\r
504 const entry_t * entry_1, * entry_2;
\r
509 entry_1 = (const entry_t *) p1;
\r
510 entry_2 = (const entry_t *) p2;
\r
512 if (entry_1->key > entry_2->key) {
\r
514 } else if (entry_1->key < entry_2->key) {
\r
517 return entry_score(entry_2) - entry_score(entry_1); // highest score first
\r
523 static void write_integer(FILE * file, int size, uint64 n) {
\r
528 ASSERT(file!=NULL);
\r
529 ASSERT(size>0&&size<=8);
\r
530 ASSERT(size==8||n>>(size*8)==0);
\r
532 for (i = size-1; i >= 0; i--) {
\r
533 b = (n >> (i*8)) & 0xFF;
\r
534 ASSERT(b>=0&&b<256);
\r
539 // end of book_make.cpp
\r