Changeset 217

Timestamp:

02/07/07 21:35:35 (1 year ago)

Author:

ug

Message:

Split up database into 3 files to allow for faster start-up.

Files:

• 06/libkombilo/cpptest.cpp (modified) (1 diff)
• 06/libkombilo/process.py (modified) (2 diffs)
• 06/libkombilo/search.cpp (modified) (28 diffs)
• 06/libkombilo/search.h (modified) (2 diffs)
• 06/libkombilo/testsearch.py (modified) (1 diff)

06/libkombilo/cpptest.cpp

@@ -29 +29 @@
-    gl.start_processing();
-    directory_iterator end_itr;
-
+// 
-    // string path = "/home/ug/go/gtl/reviews";
-// string path = "/home/ug/go/gogod06/1999
+string path = "/home/ug/go/gogod06/2000
-    for(directory_iterator it(path); it != end_itr; ++it) {
-      string n = it->string();

06/libkombilo/process.py

@@ -10 +10 @@
-
-try: 
-
+*
-except: 
-    pass
@@ -45 +45 @@
-    print 'Processed %d games in %.2f seconds' % (counter, time.time()-starttime)
-
-
-# 
+# 
+
-# filelist = glob.glob('/home/ug/go/KGS2005/*.sgf')
-

06/libkombilo/search.cpp

@@ -969 +969 @@
-
-void Algo_finalpos::initialize_process(sqlite3* DB) throw(DBError) {
+  // printf("init Algo_finalpos\n");
-  db = DB;
-  char sql[100];
-  sprintf(sql, "create table if not exists algo_finalpos_%d ( id integer primary key, data blob );", boardsize);
-  int rc = sqlite3_exec(db, sql, 0, 0, 0);
-
+
+
+
+
+
-}
-
@@ -1030 +1035 @@
-  }
-  data = new vector<char* >;
+  int rc = sqlite3_exec(db, "begin transaction;", 0, 0, 0);
+  if (rc) throw DBError();
-  sqlite3_stmt *ppStmt=0;
-  char sql[100];
-  sprintf(sql, "select data from algo_finalpos_%d order by id", boardsize);
-int 
+
-  if (rc != SQLITE_OK || ppStmt==0) return rc; // FIXME: catch certain errors, (and/or throw DBError?)
-  while (sqlite3_step(ppStmt) == SQLITE_ROW) {
@@ -1048 +1055 @@
-  rc = sqlite3_finalize(ppStmt);
-  if (rc != SQLITE_OK) return rc;
+  rc = sqlite3_exec(db, "commit;", 0, 0, 0);
+  if (rc != SQLITE_OK) throw DBError();
-  return 0;
-}
@@ -1207 +1216 @@
-
-void Algo_movelist::initialize_process(sqlite3* DB) throw(DBError) {
+  // printf("init Algo_movelist\n");
-  db = DB;
-  char sql[100];
@@ -1212 +1222 @@
-  int rc = sqlite3_exec(db, sql, 0, 0, 0);
-  if (rc != SQLITE_OK) throw DBError();
+  // printf("init Algo_movelist\n");
-}                                         
-
@@ -1322 +1333 @@
-  db = DB;
-
+  int rc = sqlite3_exec(db, "begin transaction;", 0, 0, 0);
+  if (rc) throw DBError();
+
-  sqlite3_stmt *ppStmt=0;
-  char sql[100];
-  sprintf(sql, "select movelist,fpC from algo_movelist_%d order by id", boardsize);
-int 
+
-  if (rc != SQLITE_OK || ppStmt==0) return rc; // FIXME: catch certain errors, (and/or throw DBError?)
-  while (sqlite3_step(ppStmt) == SQLITE_ROW) {
@@ -1349 +1363 @@
-  if (rc != SQLITE_OK) return rc;
-
+  rc = sqlite3_exec(db, "commit;", 0, 0, 0);
+  if (rc != SQLITE_OK) throw DBError();
-  // printf("data sizes %d, %d, %d\n", data1->size(), data1l->size(), data2->size());
-  return 0;
@@ -3365 +3381 @@
-  labels = 0;
-  continuations = 0;
-1
+2
-  strcpy(dbname, DBNAME);
+  db = algo_db1 = algo_db2 = 0;
+
+  dbname[strlen(DBNAME)] = '1';
+  dbname[strlen(DBNAME)+1] = 0;
+  int rc = sqlite3_open(dbname, &algo_db1); 
+  if (rc) {
+    sqlite3_close(algo_db1);
+    algo_db1 = 0;
+    throw DBError();
+  }
+  rc = sqlite3_busy_timeout(algo_db1, 200);
+  if (rc) throw DBError();
+  rc = sqlite3_exec(algo_db1, "pragma synchronous = off;", 0, 0, 0);
+  if (rc) throw DBError();
+  rc = sqlite3_exec(algo_db1, "pragma cache_size = 100000;", 0, 0, 0);
+  if (rc) throw DBError();
+  dbname[strlen(DBNAME)] = '2';
+  dbname[strlen(DBNAME)+1] = 0;
+  rc = sqlite3_open(dbname, &algo_db2); 
+  if (rc) {
+    sqlite3_close(algo_db2);
+    algo_db2 = 0;
+    throw DBError();
+  }
+  rc = sqlite3_busy_timeout(algo_db2, 200);
+  if (rc) throw DBError();
+  rc = sqlite3_exec(algo_db2, "pragma synchronous = off;", 0, 0, 0);
+  if (rc) throw DBError();
+  rc = sqlite3_exec(algo_db2, "pragma cache_size = 700000;", 0, 0, 0);
+  if (rc) throw DBError();
-
-  // try to retrieve basic options from database
-
+
+
-  if (rc) {
-    sqlite3_close(db);
@@ -3379 +3426 @@
-  rc = sqlite3_exec(db, "pragma synchronous = off;", 0, 0, 0);
-  if (rc) throw DBError();
-7
+1
-  if (rc) throw DBError();
-
@@ -3423 +3470 @@
-    if (rc != SQLITE_OK) throw DBError();
-  }
-  // rc = sqlite3_close(db);
-  // if (rc != SQLITE_OK) throw DBError();
-  // db = 0;
-
-
-  // printf("set up Algorithm instances\n");
@@ -3482 +3525 @@
-  int ctr = algo_ps.size()/20;
-  // printf("add algos %d %d %d\n", bs, ctr, p_op->algos);
-
+
+
+
+
-
-  algo_ps[20*ctr] = new Algo_signature(bs);
-
+
+
-    algo_ps[algo_finalpos+20*ctr] = new Algo_finalpos(bs);
-
+
+
+
-    algo_ps[algo_movelist+20*ctr] = new Algo_movelist(bs);
-
+
+
+
-    algo_ps[algo_hash_full+20*ctr] = new Algo_hash_full(bs, p_op->algo_hash_corner_maxNumStones);
-
+
+
+
-    algo_ps[algo_hash_corner+20*ctr] = new Algo_hash_corner(bs, 7, p_op->algo_hash_corner_maxNumStones);
+    algo_dbs[algo_hash_corner+20*ctr] = algo_db2;
+  }
-  // for(int a=20*ctr; a<20*(ctr+1); a++) printf("aa %d %p\n", a, algo_ps[a]);
-  // if (algos & ALGO_HASH_SIDE) 
@@ -3513 +3568 @@
-  
-  int rc;
-  // db = 0;
-  // rc = sqlite3_open(dbname, &db);
-  // if (rc) {
-  //   fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db));
-  //   sqlite3_close(db);
-  //   db = 0;
-  //   throw DBError();
-  // }
-  rc = sqlite3_exec(db, "begin transaction;", 0, 0, 0);
-  if (rc) throw DBError();
@@ -3538 +3585 @@
-  
-  readPlayersList();
+  rc = sqlite3_exec(db, "commit;", 0, 0, 0);
+  if (rc != SQLITE_OK) throw DBError();
-
-  if (rc == SQLITE_OK && !readDBs) {
-    for(unsigned int a=0; a < 20*boardsizes.size(); a++) {
-db
+algo_dbs[a]
-    }
-    readDBs = 1;
-  }
-  rc = sqlite3_exec(db, "commit;", 0, 0, 0);
-  if (rc != SQLITE_OK) throw DBError();
-  // printf("read.\n");
-
-  // rc = sqlite3_close(db);
-  // if (rc != SQLITE_OK) throw DBError();
-  // db = 0;
-  reset();
-  // printf("leave readDB\n");
@@ -3793 +3837 @@
-
-void GameList::gisearch(char* sql, int complete) throw(DBError) {
-  bool dbWasOpen = false;
-  if (!db) {
-    int rc = sqlite3_open(dbname, &db);
-    if (rc) {
-      sqlite3_close(db);
-      db = 0;
-      throw DBError();
-    }
-  } else dbWasOpen = true;
-  if (start_sorted() == 0) { 
-    if (!dbWasOpen) {
-      int rc = sqlite3_exec(db, "pragma synchronous = off;", 0, 0, 0);
-      if (rc) throw DBError();
-      rc = sqlite3_exec(db, "pragma cache_size = 300000;", 0, 0, 0);
-      if (rc) throw DBError();
-    }
-    string query;
-    if (!complete) query = "select id from games where ";
@@ -3817 +3846 @@
-    if( rc!=SQLITE_OK ) throw DBError();
-
-    if (!dbWasOpen) {
-      rc = sqlite3_close(db);
-      if( rc!=SQLITE_OK ) throw DBError();
-      db = 0;
-    }
-    end_sorted();
-  }
@@ -3897 +3921 @@
-  }
-  int index = (*all)[(*currentList)[i].second]->id;
-  // int rc = sqlite3_open(dbname, &db); 
-  // if (rc) {
-  //   sqlite3_close(db);
-  //   db = 0;
-  //   throw DBError();
-  // }
-  char* prop = 0;
-  char sql[200];
@@ -3909 +3927 @@
-  int rc = sqlite3_exec(db, sql, getpropcallback, &prop, 0);
-  if (rc != SQLITE_OK) throw DBError();
-  // sqlite3_close(db);
-  // db = 0;
-
-  if (!prop) return "";
@@ -3929 +3945 @@
-  }
-  int index = (*all)[(*currentList)[i].second]->id;
-  // int rc = sqlite3_open(dbname, &db); 
-  // if (rc) {
-  //   sqlite3_close(db);
-  //   db = 0;
-  //   throw DBError();
-  // }
-  char* prop = 0;
-  char sql[200];
@@ -3941 +3951 @@
-  int rc = sqlite3_exec(db, sql, getpropcallback, &prop, 0);
-  if (rc != SQLITE_OK) throw DBError();
-  // sqlite3_close(db);
-  // db = 0;
-
-  if (!prop) return "";
@@ -3956 +3964 @@
-  sizeX = pattern.sizeX; // need this in lookupLabel
-  PatternList pl(pattern, searchOptions->fixedColor, searchOptions->nextMove);
-  // int rc = sqlite3_open(dbname, &db); 
-  // if (rc) {
-  //   sqlite3_close(db);
-  //   db = 0;
-  //   throw DBError();
-  // }
-  // rc = sqlite3_exec(db, "pragma synchronous = off;", 0, 0, 0);
-  // if (rc) throw DBError();
-  // rc = sqlite3_exec(db, "pragma cache_size = 300000;", 0, 0, 0);
-  // if (rc) throw DBError();
-
-  if (boardsizes.size() != 1 || boardsizes[0] != pattern.boardsize) {
@@ -3973 +3971 @@
-  }
-  if (!readDBs) {
-db
+algo_dbs[a]
-    readDBs = 1;
-  }
@@ -3980 +3978 @@
-  // FULL BOARD PATTERN?
-  if ((searchOptions->algos & ALGO_HASH_FULL) && pattern.sizeX == pattern.boardsize && pattern.sizeY == pattern.boardsize && algo_ps[algo_hash_full]) {
-db
+algo_dbs[algo_hash_full]
-    if (hash_result == 1) {
-    } else if (hash_result == 0) {
@@ -3991 +3989 @@
-    // CORNER PATTERN?
-    if ((searchOptions->algos & ALGO_HASH_FULL) && pattern.sizeX >= 7 && pattern.sizeY >= 7 && algo_ps[algo_hash_corner]) {
-db
+algo_dbs[algo_hash_corner]
-      if (hash_result == 0) {
-        if (searchOptions->algos & ALGO_MOVELIST && algo_ps[algo_movelist])
@@ -4005 +4003 @@
-    }
-  }
-  // sqlite3_close(db);
-  // db = 0;
-  if (labels) delete [] labels;
-  labels = pl.sortContinuations();
@@ -4120 +4116 @@
-
-  int rc;
-  // rc = sqlite3_open(dbname, &db); 
-  // if (rc) {
-  //   sqlite3_close(db);
-  //   db = 0;
-  //   throw DBError();
-  // }
-  // rc = sqlite3_exec(db, "pragma synchronous = off;", 0, 0, 0);
-  // if (rc) throw DBError();
-  // rc = sqlite3_exec(db, "pragma cache_size = 300000;", 0, 0, 0);
-  // if (rc) throw DBError();
-
-  createGamesDB();
@@ -4135 +4121 @@
-  char* sql = "begin transaction;";
-  rc = sqlite3_exec(db, sql, 0, 0, 0);
-
+
+
+
+
+
-  current = 0;
-
+
+
+
-}
-
@@ -4147 +4139 @@
-    sqlite3_close(db);
-    db = 0;
+    throw DBError();
+  }
+  rc = sqlite3_exec(algo_db1, "commit;", 0, 0, 0);
+  if (rc != SQLITE_OK) {
+    sqlite3_close(algo_db1);
+    algo_db1 = 0;
+    throw DBError();
+  }
+  rc = sqlite3_exec(algo_db2, "commit;", 0, 0, 0);
+  if (rc != SQLITE_OK) {
+    sqlite3_close(algo_db2);
+    algo_db2 = 0;
-    throw DBError();
-  }
@@ -4223 +4227 @@
-        // printf("a %d\n", a);
-        // printf("%p\n", algo_ps[a]);
-db
+algo_dbs[a]
-      }
-    }

06/libkombilo/search.h

@@ -574 +574 @@
-    std::vector<int> boardsizes;
-    std::vector<algo_p> algo_ps;
+    std::vector<sqlite3*> algo_dbs;
-    std::vector<GameListEntry* > * all;
-    std::vector<std::pair<int,int> > * currentList; // pair of game id and position within all
@@ -655 +656 @@
-    int SGFtagsSize;
-    int sizeX; // keeps track of width of search pattern during search
+    sqlite3* algo_db1;
+    sqlite3* algo_db2;
-    ProcessOptions* p_op;
-    std::vector<std::string>* SGFtags;

06/libkombilo/testsearch.py

@@ -14 +14 @@
-# print gl.size(), 'games in the database'
-
-
+
+
-# p = Pattern(CENTER_PATTERN, 19, 3, 3, '.X.XXXXOX')
-# 
+
-# p = Pattern(CENTER_PATTERN, 19, 5, 4, '..XOO'+ '...XX'+ '.....'+ '..X..') 
-so = SearchOptions()