Changeset 239

Timestamp:

03/09/07 16:38:11 (1 year ago)

Author:

ug

Message:

Handling of duplicates within GameList, also during process.
Better access to processing results.

Files:

• 06/libkombilo/Makefile (modified) (1 diff)
• 06/libkombilo/abstractboard.cpp (modified) (1 diff)
• 06/libkombilo/cpptest.cpp (modified) (4 diffs)
• 06/libkombilo/libkombilo.i (modified) (1 diff)
• 06/libkombilo/process.py (modified) (2 diffs)
• 06/libkombilo/search.cpp (modified) (30 diffs)
• 06/libkombilo/search.h (modified) (13 diffs)
• 06/libkombilo/testsearch.py (modified) (3 diffs)

06/libkombilo/Makefile

@@ -2 +2 @@
-        g++ -o cpptest -lsqlite3 -lboost_filesystem cpptest.o search.o sgfparser.o abstractboard.o
-
-
+ search.h abstractboard.h sgfparser.h
-        g++ -c cpptest.cpp
-
-
+ search.h abstractboard.h sgfparser.h
-        g++ -c search.cpp
-
-
+ sgfparser.h
-        g++ -c sgfparser.cpp
-
-
+ abstractboard.h
-        g++ -c abstractboard.cpp
-

06/libkombilo/abstractboard.cpp

@@ -84 +84 @@
-  if (DEBUG_ABSTRACTBOARD) printf("AB::abstractBoard(int) 0\n");
-  boardsize = bs;
-
+ // FIXME
-  status = new char[boardsize*boardsize+1];
-  for (int i = 0; i < boardsize*boardsize; i++)

06/libkombilo/cpptest.cpp

@@ -29 +29 @@
-    gl.start_processing();
-    directory_iterator end_itr;
-
+// 
-    // string path = "/home/ug/go/gtl/reviews";
-
+
+
-    for(directory_iterator it(path); it != end_itr; ++it) {
-      string n = it->string();
@@ -46 +47 @@
-        }
-        infile.close();
-
+
+
+
+
-      }
-    }
-    gl.finalize_processing();
-Processed %d games
+Now %d games in db
-  }
-  printf("%d games.\n", gl.size());
@@ -90 +94 @@
-  
-  // -------------------- do pattern search --------------------------------------
-
+// 
-
-  // ------------------- print some information about current list of games ------------
-
+// 
-  // vector<string> res = gl.currentEntriesAsStrings();
-  // for(vector<string>::iterator it = res.begin(); it != res.end(); it++)
@@ -100 +104 @@
-
-  // ------------------- print some statistics ------------------------------------------
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-    }
-
-
-
-
-
+
-
-
-
-
-
-
-
-
-
-
-
+
-  }
-

06/libkombilo/libkombilo.i

@@ -5 +5 @@
-namespace std {
-  %template(vectors) vector<string>;
+  %template(vectori) vector<int>;
-};
-

06/libkombilo/process.py

@@ -31 +31 @@
-    for filename in filenames:
-        # print filename
-        counter += 1
-        try:
-            file = open(filename)
@@ -41 +40 @@
-
-        path, fn = os.path.split(filename)
-
+
+
+
+
+
+
-    gl.finalize_processing()
-    print 'Processed %d games in %.2f seconds' % (counter, time.time()-starttime)

06/libkombilo/search.cpp

@@ -858 +858 @@
-void Algorithm::branchpoint_process() {}
-void Algorithm::endOfVariation_process() {}
-
+bool commit
-void Algorithm::finalize_process() {}
-int Algorithm::readDB(sqlite3* DB) { return 0; }
@@ -877 +877 @@
-  sprintf(sql, "create table if not exists algo_signature_%d ( id integer primary key, signature varchar(12) );", boardsize);
-  int rc = sqlite3_exec(db, sql, 0, 0, 0);
+  if (rc != SQLITE_OK) throw DBError();
+  sprintf(sql, "create index if not exists sig_idx on algo_signature_%d(signature);", boardsize);
+  rc = sqlite3_exec(db, sql, 0, 0, 0);
-  if (rc != SQLITE_OK) throw DBError();
-}
@@ -972 +975 @@
-}
-
-
-
+
+
+
+
+
+
+
+
+
-  delete [] signature;
+}
+
+void Algo_signature::finalize_process() {
+}
+
+char* Algo_signature::get_current_signature() {
+  return symmetrize(signature, boardsize);
+}
+
+vector<int> Algo_signature::search_signature(char* sig) {
+  // to be used during processing! (because we need the db)
-  char sql[100];
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-Algo_finalpos::Algo_finalpos(int bsize) : Algorithm(bsize) {
@@ -1048 +1078 @@
-}
-
-
-
-
-
+
+
+
+
+
+
-  delete [] fp;
-}
@@ -1226 +1258 @@
-}
-
+bool Algo_finalpos::equal(unsigned int i1, unsigned int i2) { 
+  // not to be used during processing
+  // i1, i2 correspond to game id's
+  sqlite3_stmt *ppStmt=0;
+  char sql[100];
+  sprintf(sql, "select data from algo_finalpos_%d where id = %d or id = %d", boardsize, i1, i2);
+  int rc = sqlite3_prepare(db, sql, -1, &ppStmt, 0);
+  if (rc != SQLITE_OK || ppStmt==0) return false; // FIXME: catch certain errors, (and/or throw DBError?)
+  if (sqlite3_step(ppStmt) == SQLITE_ROW) {
+    char* dd1 = (char*)sqlite3_column_blob(ppStmt, 0);
+    char* d1 = new char[100];
+    for(int i=0; i<100; i++) d1[i] = dd1[i]; // FIXME: is this necessary?
+    if (sqlite3_step(ppStmt) == SQLITE_ROW) {
+      char* d2 = (char*)sqlite3_column_blob(ppStmt, 0);
+      for(int i=0; i<100; i++) 
+        if (d1[i] != d2[i]) {
+          delete [] d1;
+          sqlite3_finalize(ppStmt);
+          return false;
+        }
+      delete [] d1;
+      sqlite3_finalize(ppStmt);
+      return true;
+    }
+    delete [] d1;
+  }
+  sqlite3_finalize(ppStmt);
+  return false;
+}
+
+bool Algo_finalpos::equals_current(unsigned int id1) { 
+  // to be used only during processing
+  // id1 here corresponds to a game id
+  bool result = true;
+  sqlite3_stmt *ppStmt=0;
+  char sql[100];
+  sprintf(sql, "select data from algo_finalpos_%d where id = %d", boardsize, id1);
+  int rc = sqlite3_prepare(db, sql, -1, &ppStmt, 0);
+  if (rc != SQLITE_OK || ppStmt==0) return false; // FIXME: catch certain errors, (and/or throw DBError?)
+  if (sqlite3_step(ppStmt) == SQLITE_ROW) {
+    char* d = (char*)sqlite3_column_blob(ppStmt, 0);
+    for(int i=0; i<100; i++) 
+      if (d[i] != fp[i]) {
+        result = false;
+        break;
+      }
+  } else result = false;
+  sqlite3_finalize(ppStmt);
+  return result;
+}
+
-
-Algo_movelist::Algo_movelist(int bsize) : Algorithm(bsize) {
@@ -1335 +1418 @@
-}
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
-  delete [] fpC;
-}
@@ -2056 +2141 @@
-#endif
-
+HashFEntry::HashFEntry(hashtype HASHCODE, char* BUF, int LENGTH) {
+  hashCode = HASHCODE;
+  buf = BUF;
+  length = LENGTH;
+}
+
+HashFEntry::HashFEntry(const HashFEntry& hfe) {
+  hashCode = hfe.hashCode;
+  length = hfe.length;
+  buf = new char[length];
+  for(int i=0; i<length; i++) buf[i] = hfe.buf[i];
+}
+
+HashFEntry::~HashFEntry() {
+  if (buf) {
+    delete [] buf;
+    buf = 0;
+  }
+}
+
-HashhitF::HashhitF(int GAMEID, char ORIENTATION, char* blob) {
-  gameid = GAMEID;
@@ -2135 +2240 @@
-    buf[6+2*i] = mn.data[i]%256;
-  }
+  hash_vector.push_back(HashFEntry(hashCode, buf, 5+2*mn.length));
+  return 0;
+}
+
+int Algo_hash_full::insert_all_hashes() {
-  char sql[100];
-  sprintf(sql, "insert into algo_hash_full_%d (hash, gameid, hit) values (?,?,?);", boardsize);
@@ -2140 +2250 @@
-  int rc = sqlite3_prepare(db, sql, -1, &ppStmt, 0);
-  if (rc != SQLITE_OK || ppStmt==0) return rc;
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
-  rc = sqlite3_finalize(ppStmt);
-  if (rc != SQLITE_OK) return rc;
-delete [] buf
+hash_vector.clear()
-  return 0; // success
-}
@@ -2264 +2378 @@
-}
-
-
+bool commit
-  for(vector<pair<hashtype, ExtendedMoveNumber>* >::iterator it = lfc->begin(); it != lfc->end(); it++) {
-    Move* continuation = 0;
@@ -2271 +2385 @@
-    delete *it;
-  }
+  if (commit) {
+    int rc = insert_all_hashes();
+    if (rc) printf("ouch %d\n",rc);
+  } else hash_vector.clear();
-  delete lfc;
-  delete moveNumber;
@@ -2420 +2538 @@
-
-int Algo_hash::insert_hash(hashtype hashCode, int pos) {
-
+
+
+
+
+
+
-  char sql[200];
-  sprintf(sql, "insert into algo_hash_%d_%s (hash, gameid, position) values (?,?,?);", boardsize, dbnameext.c_str());
@@ -2426 +2549 @@
-  int rc = sqlite3_prepare(db, sql, -1, &ppStmt, 0);
-  if (rc != SQLITE_OK || ppStmt==0) return rc;
-  rc = sqlite3_bind_int64(ppStmt, 1, hashCode);
-  if (rc != SQLITE_OK) return rc;
-  rc = sqlite3_bind_int(ppStmt, 2, gid);
-  if (rc != SQLITE_OK) return rc;
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-  rc = sqlite3_finalize(ppStmt);
-  if (rc != SQLITE_OK) return rc;
@@ -2481 +2609 @@
-      it->changed = false;
-      pair<hashtype,int> phi = it->cHC();
-
+
+
-    }
-  }
@@ -2522 +2651 @@
-}
-
-
+bool commit
-  for(vector<HashInstance>::iterator it = hi->begin(); it != hi->end(); it++)
-    it->finalize();
+  if (commit) {
+    int rc = insert_all_hashes();
+    if (rc) printf("ouch %d\n",rc);
+    hash_vector.clear();
+  } else hash_vector.clear();
-}
- 
@@ -2678 +2812 @@
-        }
-      }
-
+< 3 || ns 
-
-      // make sure all hash keys are unique
@@ -3357 +3491 @@
-  if (rootNodeTags.find("RE") == string::npos) rootNodeTags += ",RE";
-  if (rootNodeTags.find("DT") == string::npos) rootNodeTags += ",DT";
+
+  algos |= ALGO_FINALPOS | ALGO_MOVELIST; // these are mandatory at the moment
-}
-
@@ -3441 +3577 @@
-
-GameList::GameList(char* DBNAME, string ORDERBY, string FORMAT, ProcessOptions* p_options, int cache) throw(DBError) {
+  duplicates = 0;
-  labels = 0;
-  continuations = 0;
@@ -3671 +3808 @@
-  if (labels) delete [] labels;
-  if (continuations) delete [] continuations;
+  if (duplicates) delete duplicates;
-  delete [] dbname;
-  if (all) {
@@ -3780 +3918 @@
-
-void GameList::makeIndexHit(int index, vector<Hit* > * hits) {
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
-}
-
-void GameList::makeIndexCandidate(int index, vector<Candidate* > * candidates) {
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
-}
-
@@ -3895 +4013 @@
-  if (rc != SQLITE_OK) throw DBError();
-  return result;
+}
+
+void GameList::insert_duplicate(int i1, int i2, vector<vector<int> >* dupl) {
+  int ii1 = get_current_index_CL(i1);
+  int ii2 = get_current_index_CL(i2);
+  // printf("insert_duplicate %d, %d\n", ii1, ii2);
+  if (ii1 == -1 || ii2 == -1) return;
+  bool inserted = false;
+  for(vector<vector<int> >::iterator it = dupl->begin(); it != dupl->end(); it++) {
+    vector<int>::iterator i = it->begin();
+    while(i != it->end() && *i != ii1 && *i != ii2) i++;
+    if (i == it->end()) continue;
+    int insert = ii1;
+    if (*i == ii1) insert = ii2;
+    while (i != it->end() && *i != insert) i++;
+    if (i == it->end()) it->push_back(insert);
+    sort(it->begin(), it->end());
+    inserted = true;
+    break;
+  }
+  if (!inserted) {
+    vector<int> new_list;
+    if (ii1 < ii2) {
+      new_list.push_back(ii1);
+      new_list.push_back(ii2);
+    } else {
+      new_list.push_back(ii2);
+      new_list.push_back(ii1);
+    }
+    dupl->push_back(new_list);
+  }
+}
+
+int GameList::find_duplicates(int bs, bool strict) throw(DBError) {
+  int bs_index = 0;
+  if (duplicates) delete duplicates;
+  duplicates = new vector<vector<int> >;
+  if (strict) {
+    vector<int>::iterator it = boardsizes.begin();
+    while (it != boardsizes.end() && *it != bs) {
+      bs_index++;
+      it++;
+    }
+    if (it == boardsizes.end()) {
+      return 0;
+    }
+  }
+  sort(currentList->begin(), currentList->end());
+  sqlite3_stmt *ppStmt=0;
+  char sql[200];
+  sprintf(sql, "select as1.id,as2.id from algo_signature_%d as1 join algo_signature_%d as2 on as1.signature = as2.signature where as1.id < as2.id;", bs, bs);
+  int rc = sqlite3_prepare(db, sql, -1, &ppStmt, 0);
+  if (rc != SQLITE_OK || ppStmt==0) throw DBError();
+  do {
+    rc = sqlite3_step(ppStmt);
+    if (rc != SQLITE_DONE && rc != SQLITE_ROW) throw DBError();
+    if (rc == SQLITE_ROW) {
+      if (!strict || ((Algo_finalpos*)algo_ps[20*bs_index+algo_finalpos])->equal(sqlite3_column_int(ppStmt, 0), sqlite3_column_int(ppStmt, 1)))
+        insert_duplicate(sqlite3_column_int(ppStmt, 0), sqlite3_column_int(ppStmt, 1), duplicates);
+    }
+  } while (rc == SQLITE_ROW);
+  rc = sqlite3_finalize(ppStmt);
+  if (rc != SQLITE_OK) throw DBError();
+  sort(currentList->begin(), currentList->end(), sndcomp);
+  return duplicates->size();
+}
+
+vector<int> GameList::retrieve_duplicates_VI(unsigned int i) {
+  if (i>=duplicates->size()) return vector<int>();
+  return (*duplicates)[i];
+}
+
+int* GameList::retrieve_duplicates_PI(unsigned int i) {
+  if (i>=duplicates->size()) return 0;
+  int* result = new int[(*duplicates)[i].size()+1];
+  int j = 0;
+  for(vector<int>::iterator it = (*duplicates)[i].begin(); it != (*duplicates)[i].end(); it++)
+    result[j++] = *it;
+  result[(*duplicates)[i].size()] = -1;
+  return result;
+}
+
+
+int GameList::get_current_index(int id, int* start) {
+  // use this in between start_sorted() and end_sorted() only!
+  int end = oldList->size();
+  int m = *start;
+  while (*start < end) {
+    m = (end+*start)/2;
+    if (id == (*oldList)[m].first) {
+      *start = m;
+      return m;
+    } else {
+      if (id < (*oldList)[m].first) end = m;
+      else *start = m+1;
+    }
+  }
+  return -1; 
+}
+
+int GameList::get_current_index_CL(int id, int start) {
+  // use this in between start_sorted() and end_sorted() only!
+  int end = currentList->size();
+  int m = start;
+  while (start < end) {
+    m = (end+start)/2;
+    if (id == (*currentList)[m].first) return m;
+    else {
+      if (id < (*currentList)[m].first) end = m;
+      else start = m+1;
+    }
+  }
+  return -1; 
-}
-
@@ -3996 +4227 @@
-
-string GameList::currentEntryAsString(int i) {
-
-
+
+
+
-}
-
@@ -4278 +4510 @@
-}
-
-
+
+
+
-  const char* dbtree = "";
-  if (DBTREE) dbtree = DBTREE;
@@ -4286 +4520 @@
-    c = new Cursor(sgf, 1); // parse sgf sloppily
-  } catch (SGFError) {
-1
+0
-  }
-
@@ -4294 +4528 @@
-  while (root) {
-    current++;
+    int return_val = 0;
-    // if (!(current%1000)) {
-    //  char* sql = "end transaction;";
@@ -4319 +4554 @@
-    if (sz=="") sz = "19";
-    int bs = atoi(sz.c_str());
+    if (bs < 3) { 
+      return_val |= UNACCEPTABLE_BOARDSIZE; 
+      process_results_vector.push_back(return_val);
+      delete [] rootNodeProperties;
+      root = root->down;
+      pos++;
+      continue;
+    }
-    int algo_offset = -1;
-    int bs_ctr = 0;
@@ -4447 +4690 @@
-    int game_id = sqlite3_last_insert_rowid(db);
-
-    // evaluate tags
-    if ((*rootNodeProperties)[posHA] != "") { // handicap game
-      char sql[100];
-      sprintf(sql, "insert into GAME_TAGS (game_id, tag_id) values (%d, %d);", game_id, HANDI_TAG);
-      rc = sqlite3_exec(db, sql, 0, 0, 0);
-      if (rc != SQLITE_OK)  throw DBError();
-    }
-    if ((*rootNodeProperties)[posWR].find('p') != string::npos ||
-        (*rootNodeProperties)[posBR].find('p') != string::npos) { 
-      // at least one of the players is professional
-      char sql[100];
-      sprintf(sql, "insert into GAME_TAGS (game_id, tag_id) values (%d, %d);", game_id, PROFESSIONAL_TAG);
-      rc = sqlite3_exec(db, sql, 0, 0, 0);
-      if (rc != SQLITE_OK)  throw DBError();
-    }
-
-    delete rootNodeProperties;
-
-    // printf("play through the game\n");
+    bool commit = true;
-
-    Node* currentN = root;
@@ -4600 +4827 @@
-        } 
-      } catch (SGFError) {
+        return_val |= SGF_ERROR;
-        caughtSGFError = true;
+        if (OMIT_GAMES_WITH_SGF_ERRORS) {
+          commit = false;
+          // (FIXME should exit from the loop here)
+        }
-      }
-
@@ -4614 +4846 @@
-      }
-
-
+ // stop here with this branch
-      else currentN = currentN->next;
-
@@ -4634 +4866 @@
-    } // while
-    {
+      // check for duplicates (if desired)
+      bool is_duplicate = false;
+      if (flags & (CHECK_FOR_DUPLICATES|CHECK_FOR_DUPLICATES_STRICT)) {
+        char* sig = ((Algo_signature*)algo_ps[20*algo_offset])->get_current_signature();
+        vector<int> all_duplicates = ((Algo_signature*)algo_ps[20*algo_offset])->search_signature(sig);
+        if (all_duplicates.size()) {
+          // printf("dupl %d\n", all_duplicates.size());
+          is_duplicate = true;
+          if ((flags & CHECK_FOR_DUPLICATES_STRICT) && (p_op->algos & ALGO_FINALPOS)) {
+            vector<int>::iterator d_it = all_duplicates.begin();
+            while (d_it != all_duplicates.end() && !((Algo_finalpos*)algo_ps[20*algo_offset + algo_finalpos])->equals_current(*d_it))
+              d_it++;
+            if (d_it == all_duplicates.end()) is_duplicate = false;
+          }
+          if (is_duplicate) {
+            return_val |= IS_DUPLICATE;
+            if (flags & OMIT_DUPLICATES) commit = false;
+          }
+        }
+        delete [] sig;
+      }
+      
+      if (commit) {
+        // evaluate tags
+        if ((*rootNodeProperties)[posHA] != "") { // handicap game
+          char sql[100];
+          sprintf(sql, "insert into GAME_TAGS (game_id, tag_id) values (%d, %d);", game_id, HANDI_TAG);
+          rc = sqlite3_exec(db, sql, 0, 0, 0);
+          if (rc != SQLITE_OK)  throw DBError();
+        }
+        if ((*rootNodeProperties)[posWR].find('p') != string::npos ||
+            (*rootNodeProperties)[posBR].find('p') != string::npos) { 
+          // at least one of the players is professional
+          char sql[100];
+          sprintf(sql, "insert into GAME_TAGS (game_id, tag_id) values (%d, %d);", game_id, PROFESSIONAL_TAG);
+          rc = sqlite3_exec(db, sql, 0, 0, 0);
+          if (rc != SQLITE_OK)  throw DBError();
+        }
+      } else {
+        return_val |= NOT_INSERTED_INTO_DB;
+        char sql[200];
+        sprintf(sql, "delete from GAMES where id=%d", game_id);
+        rc = sqlite3_exec(db, sql, 0, 0, 0);
+        if (rc) printf("ouch %d\n", rc);
+      }
-      for(int a=20*algo_offset; a < 20*(algo_offset+1); a++) {
-        // printf("endgame %d\n", a);
-
+commit
-      }
-    }
+    delete rootNodeProperties;
+    process_results_vector.push_back(return_val);
-    root = root->down;
-    pos++;
-  }
-  delete c;
-
-
+
+
+
+
+
+
+
+
-
-VarInfo::VarInfo(Node* N, abstractBoard* B, int I) {

06/libkombilo/search.h

@@ -201 +201 @@
-    virtual void branchpoint_process();
-    virtual void endOfVariation_process();
-
+bool commit=true
-    virtual void finalize_process();
-    virtual int readDB(sqlite3* DB);
@@ -225 +225 @@
-    void branchpoint_process();
-    void endOfVariation_process();
-
+bool commit=true
-    void finalize_process();
-
-    int counter;
-    char* signature;
+    char* get_current_signature();
+    std::vector<int> search_signature(char* sig);
-  private:
-    bool main_variation;
@@ -248 +250 @@
-    void branchpoint_process();
-    void endOfVariation_process();
-
+bool commit=true
-    void finalize_process();
-
@@ -256 +258 @@
-    int readDB(sqlite3* DB);
-    int search(PatternList& patternList, GameList& gl, SearchOptions& options);
+
+    bool equal(unsigned int id1, unsigned int id2); // id1, id2 refer to id's in the database!
+    bool equals_current(unsigned int id1);
-};
-
@@ -317 +322 @@
-    void branchpoint_process();
-    void endOfVariation_process();
-
+bool commit=true
-    void finalize_process();
-    int readDB(sqlite3* DB);
@@ -329 +334 @@
-};
-
+class HashFEntry {
+  public:
+    hashtype hashCode;
+    char* buf;
+    int length;
+
+    HashFEntry(hashtype HASHCODE, char* BUF, int LENGTH);
+    HashFEntry(const HashFEntry& hfe);
+    ~HashFEntry();
+};
-
-class HashhitF { // hashing hit for full board search
@@ -374 +389 @@
-    void branchpoint_process();
-    void endOfVariation_process() throw(DBError);
-
+bool commit=true
-    void finalize_process();
-    int search(PatternList& patternList, GameList& gl, SearchOptions& options, sqlite3* db);
-
-    int insert_hash(hashtype hashCode, ExtendedMoveNumber& mn, Move* continuation);
-    hashtype compute_hashkey(Pattern& pattern);
-
+    int maxNumStones;
+    int numStones;
+  private:
-    hashtype currentHashCode;
-    ExtendedMoveNumber* moveNumber;
-    std::vector<std::pair<hashtype, ExtendedMoveNumber>* > *lfc; // hash code + move number, still looking for continuation
-    std::stack<HashVarInfo>* branchpoints;
-
-
+
+
+
-};
-
@@ -439 +457 @@
-    virtual void branchpoint_process();
-    virtual void endOfVariation_process();
-
+bool commit=true
-    virtual void finalize_process();
-    virtual int search(PatternList& patternList, GameList& gl, SearchOptions& options, sqlite3* db);
-
+    virtual std::pair<hashtype,int> compute_hashkey(PatternList& pl, int CS);
+    static const hashtype hashCodes[];
+    std::string dbnameext;
+    std::vector<HashInstance>* hi;
+    int maxNumStones;
+    std::vector<std::pair<hashtype, int> > hash_vector;
-    virtual int insert_hash(hashtype hashCod, int pos);
-
-
-
-
-
-
+
-};
-
@@ -569 +588 @@
-    ~VarInfo();
-};
+
+// process flags (used to determine the behavior for individual games - in contrast to
+// options which apply to the whole GameList and are given in ProcessOptions)
+const int CHECK_FOR_DUPLICATES = 1; // check for duplicates using the signature
+const int CHECK_FOR_DUPLICATES_STRICT = 2; // check for duplicates using the final position
+                                           // (if ALGO_FINAPOS is available)
+const int OMIT_DUPLICATES = 4;
+const int OMIT_GAMES_WITH_SGF_ERRORS = 8; 
+const int INDEX_OUT_OF_RANGE = 16;
+
+// process return values
+// 0:   SGF error occurred when parsing the "tree structure" (i.e. before parsing the individual nodes)
+//      database was not changed
+// n>0: n games were processed, use process_results to access the individual results 
+
+// flags used in process_results
+const int UNACCEPTABLE_BOARDSIZE = 1; // (database not changed) 
+const int SGF_ERROR = 2;
+    // SGF error occurred when playing through the game 
+    // (and the rest of the concerning variation was not used).
+    // Depending on OMIT_GAMES_WITH_SGF_ERRORS, everything before this node (and other variations, 
+    // if any) was inserted, or the database was not changed.
+const int IS_DUPLICATE = 4;
+const int NOT_INSERTED_INTO_DB = 8;
+
-
-class GameList {
@@ -606 +650 @@
-    // ------- processing SGF games (to populate the db) --------------------------
-    void start_processing(int PROCESSVARIATIONS=-1) throw(DBError);
-
+
+
+
-    void finalize_processing() throw(DBError);
-    
+    // int remove_game(int index); // TODO
+    // int remove_all_current_games();
+
-    // ------- pattern search -----------------------------------------------------
-    // options is copied in the search method (if != 0), so the caller has to free the pointer
@@ -629 +678 @@
-    void deleteTag(int tag, int i = -1) throw(DBError);
-    std::vector<int> getTags(int i, int tag=0) throw(DBError); // note the order of arguments!
+    
+    // ------- duplicates ---------------------------------------------------------
+    int find_duplicates(int bs, bool strict=false) throw(DBError); // return number of duplicate array
+    std::vector<int> retrieve_duplicates_VI(unsigned int i);
+    int* retrieve_duplicates_PI(unsigned int i); // same as above, but returns Pointer to Int 
+                                                 // (an array terminated by -1)
+                                                 // The caller must free the pointer himself
+                                                 // (before calling find_duplicates again).
-
-    // ------- misc ---------------------------------------------------------------
@@ -659 +716 @@
-    void makeIndexHit(int index, std::vector<Hit* > *hits);
-    void setCurrentFromIndex(int index);