searchPY.py

Revision 174, 26.4 kB (checked in by ug, 2 years ago)
Worked some more on pattern.cc ... not done yet, though.

Line
1	# File: searchPY.py
2
3	## Copyright (C) 2001-4 Ulrich Goertz (u@g0ertz.de)
4
5	## This file is part of Kombilo 0.6, a go database program.
6
7	## This program is free software; you can redistribute it and/or modify
8	## it under the terms of the GNU General Public License as published by
9	## the Free Software Foundation; either version 2 of the License, or
10	## (at your option) any later version.
11
12	## This program is distributed in the hope that it will be useful,
13	## but WITHOUT ANY WARRANTY; without even the implied warranty of
14	## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	## GNU General Public License for more details.
16
17	## You should have received a copy of the GNU General Public License
18	## along with this program (see doc/license.txt); if not, write to the Free Software
19	## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20	## The GNU GPL is also currently available at
21	## http://www.gnu.org/copyleft/gpl.html
22
23	import time
24	import os
25	import glob
26	from string import split, find, join, strip, replace, digits, lower, maketrans, translate, count
27	import re
28	from array import *
29	from copy import deepcopy
30
31	# try:
32	# from abstractBoard import *
33	# except ImportError:
34	from abstractBoardPY import *
35
36	try:
37	from pattern import *
38	except ImportError:
39	print "Using patternPY"
40	from patternPY import *
41	from algosPY import *
42
43	try:
44	from sgfpars import Node, Cursor, SGFError, SGFescape
45	except ImportError:
46	from sgfparser import Node, Cursor, SGFError, SGFescape
47
48	# --------------------------------------------------------------------------
49
50	class stdout_message:
51	def __init__(self): pass
52	def insert(self, dummy, text): print text
53
54	class no_message:
55	def __init__(self): pass
56	def insert(self, dummy, text): pass
57
58	# -------------------------------------------------------------------------
59
60	class dummy_progbar:
61	def __init__(self):
62	pass
63
64	def clear(self):
65	pass
66
67	def redraw(self, x):
68	print x
69
70	def write(self, text):
71	pass
72
73	# -------------------------------------------------------------------------
74
75	class SGFDatabase:
76
77	def __init__(self, boardsize, progBar):
78	self.boardsize = boardsize
79	if not progBar is None: self.progBar = progBar
80	else: self.progBar = dummy_progbar()
81	self.datapath = ('', '')
82	self.sgfpath = ''
83	self.disabled = 1
84	self.algos = 1
85	self.namelist = None
86	self.current = None
87	self.results = None
88
89
90	def clear(self):
91	self.namelist = None
92	self.current = None
93	self.results = None
94
95
96	def addSGF(self, filename, datap, messages, algos):
97	""" Add single SGF file to database. """
98	pass
99
100
101	def deleteSGF(self):
102	""" Delete single SGF file from database. """
103	pass
104
105
106	def getFilename(self, index):
107	s = self.namelist[index][0]
108	if find(s, '[') != -1:
109	s, s2 = split(s, '[')
110	gameNumber = int(strip(s2)[:-1])
111	else:
112	gameNumber = 0
113
114	if s[-1] == '.': filename = s[:-1] # no extension
115	elif s[-2:] == '.m': filename = s + 'gt' # extension '.mgt'
116	else: filename = s + '.sgf' # extension '.sgf'
117
118	return strip(os.path.join(self.sgfpath, filename)), gameNumber
119
120
121	def process(self, dbpath, datap, algos, filenames, messages=None,
122	sloppySGF = 1, duplicateCheck = 1, strictDuplicateCheck = 0,
123	processVariations = 0, filelist=None, gamelist = None):
124	"""
125	Process the database dbpath, i.e. translate the SGF files into the
126	.db files that the pattern search function uses.
127
128	After retrieving some information from the root node of the game
129	(names of the players etc.), which will be used in the namelist,
130	this method goes through the game, and passes all relevant information
131	(stone placements/moves/captures/start, end of variations) to instances
132	of the selected Algorithm classes.
133
134	The Algorithm classes to be used are selected via bit-flags in algos.
135
136	A gamelist parameter can be given; the games in the game list will then
137	be taken into account for the duplicate check.
138	"""
139
140	if not messages: messages = no_message()
141
142	currentTime = time.time()
143	if self.progBar: self.progBar.clear()
144
145	messages.insert('end', 'Processing ' + dbpath + ' ...\n')
146
147	algolist = [Algo_finalpos(self.boardsize)]
148	# always use the finalpos algorithm, and put it in the first position
149	# since it's also used for the duplicateCheck
150
151	for algo in ALL_ALGOS.keys():
152	if algo != 1 and algos & algo:
153	algolist.append(ALL_ALGOS[algo](self.boardsize))
154
155	namelist = ['kombilo06', `algos`] # FIXME: add boardsize here?
156	siglist = []
157
158	oa = ord('a')
159
160	if filelist is None:
161	if filenames == *'.sgf'**:
162	filelist = glob.glob(os.path.join(dbpath,*'.sgf'**))
163	elif filenames == *'.sgf, .mgt'*:
164	filelist = glob.glob(os.path.join(dbpath,*'.sgf')) + glob.glob(os.path.join(dbpath, '.mgt'*))
165	else:
166	filelist = glob.glob(os.path.join(dbpath,*''**))
167
168	filelist.sort()
169
170	for algo in algolist: algo.initialize_process(len(filelist))
171
172	total = len(filelist)
173	gameCtr = 0 # counts games which are added to the new database
174	fileCtr = 0 # counts files which are processed
175
176	for f in filelist:
177	if self.progBar and fileCtr % 10 == 0:
178	self.progBar.redraw(fileCtr*1.0/total)
179	fileCtr += 1
180
181	# if self.stopAddDBVar.get() == 1: # FIXME?!
182	# showinfo('Stop processing',
183	# 'Processing will be stopped after this database has been finished.\n')
184	# self.stopAddDBVar.set(2)
185
186	if not os.path.isfile(f): continue
187
188	if find(f, '\|%') != -1 or find(f, '[') != -1:
189	messages.insert('end', 'Invalid characters in filename ' + f + ', skipped.\n')
190	continue
191
192	try:
193	file = open(f)
194	s = file.read()
195	file.close()
196	except IOError:
197	messages.insert('end', 'I/O Error: Cannot read file ' + f + ', skipped.\n')
198	continue
199
200	try:
201	cursor = Cursor(s, sloppySGF)
202	except:
203	messages.insert('end', 'Error in SGF file '+f+': Skipped.\n')
204	continue
205
206	if cursor.root.numChildren > 1:
207	collection = 1
208	messages.insert('end', 'Warning: ' + f + ' is a collection;' +\
209	' for performance reasons it would be better to' + \
210	' split it into files with single game records.\n')
211	elif cursor.root.numChildren == 1:
212	collection = 0
213	else:
214	messages.insert('end', 'File '+f+' does not contain a game record. Skipped.\n')
215	continue
216
217	for indexColl in range(cursor.root.numChildren):
218	if collection: iColl = '[' + `indexColl` + ']'
219	else: iColl = ''
220
221	try:
222	cursor.game(indexColl)
223	except SGFError:
224	messages.insert('end', 'SGF Error in game ' + iColl + ' of file ' + f + ', skipped.\n')
225	continue
226
227	try:
228	if cursor.currentNode().has_key('SZ') and strip(cursor.currentNode()['SZ'][0]) != `self.boardsize`:
229	messages.insert('end', 'SGF file '+ f + iColl +\
230	' does not contain a %sx%s game. Skipped.\n' \
231	% (`self.boardsize`, `self.boardsize`))
232	continue
233
234	if cursor.currentNode().has_key('PB'):
235	playerB = replace(cursor.currentNode()['PB'][0], '\|%', ' ')
236	playerB = replace(playerB, '\r', ' ')
237	playerB = replace(playerB, '\n', ' ')
238	else: playerB = '?'
239	if cursor.currentNode().has_key('PW'):
240	playerW = replace(cursor.currentNode()['PW'][0], '\|%', ' ')
241	playerW = replace(playerW, '\r', ' ')
242	playerW = replace(playerW, '\n', ' ')
243	else: playerW = '?'
244
245	if cursor.currentNode().has_key('RE') and strip(cursor.currentNode()['RE'][0]):
246	result = strip(cursor.currentNode()['RE'][0])[0] # We assume that the first letter in
247	# the RE tag is the winner's color
248	else: result = '-'
249
250	if cursor.currentNode().has_key('DT') and cursor.currentNode()['DT'][0]:
251	date_str = cursor.currentNode()['DT'][0]
252
253	m = re.search('(^\|[^\d])(\d\d\d\d-\d\d-\d\d)($\|[^\d])', date_str)
254
255	try:
256	date = m.group(2)
257	except (AttributeError, ValueError):
258	m = re.search('(^\|[^\d])(\d\d\d\d-\d\d)($\|[^\d])', date_str)
259	try:
260	date = m.group(2)
261	except (AttributeError, ValueError):
262	m = re.search('(^\|[^\d])(\d\d\d\d)($\|[^\d])', date_str)
263	try:
264	date = m.group(2)
265	except (AttributeError, ValueError):
266	date = ''
267	else:
268	date = ''
269
270	signature1 = ''
271	signature2 = ''
272
273	except:
274	messages.insert('end', 'SGF Error in root node of game ' + iColl + \
275	', file ' + f + ', skipped.\n')
276	continue
277
278	counter = -1
279	try:
280
281	# now we will go through the game ...
282	# ---------------- this should be done in C++
283
284	for algo in algolist: algo.newgame_process()
285
286	b = abstractBoard()
287	self.branchpoints = [] # for processing variations
288
289	while 1:
290	counter += 1
291
292	try:
293	if cursor.currentNode().has_key('B') or cursor.currentNode().has_key('W'):
294	if cursor.currentNode().has_key('B'): co = 'B'
295	else: co = 'W'
296
297	p = cursor.currentNode()[co][0]
298	if len(p) != 2:
299	if len(p) != 0:
300	messages.insert('end', 'SGF Error in file ' + f + '\n')
301	break
302	assert len(p) == 2 # jump to end of loop if len(p) == 0 (i.e. if "pass")
303
304	x, y = ord(p[0])-oa, ord(p[1])-oa
305
306	if 0 <= x < self.boardsize and 0 <= y < self.boardsize:
307	if not b.play((x, y), co):
308	# print co, p
309	messages.insert('end', 'Illegal move in file ' + f \
310	+ ' (' + `counter+1` + ')\n')
311	break
312	else:
313	if (x,y) != (19,19): # FIXME for different boardsize???
314	messages.insert('end', 'Invalid move in file '+f+
315	' (' + `counter+1` + ')\n')
316	break
317
318	assert 0 <= x < self.boardsize and 0 <= y < self.boardsize
319	# jump to end of loop if pass ??? FIXME
320
321	if b.len_undostack(): captures = b.get_cap_last()
322	else: assert 0
323
324	if counter+1 in [20, 40, 60]:
325	signature1 = signature1 + chr(x + oa) + chr(y + oa)
326	if counter+1 in [31, 51, 71]:
327	signature2 = signature2 + chr(x + oa) + chr(y + oa)
328
329	for algo in algolist:
330	if co == 'B': algo.B_process(x, y, captures)
331	else: algo.W_process(x, y, captures)
332
333	else: # check for AE, AB, AW tags:
334
335	if cursor.currentNode().has_key('AB'):
336	for p in cursor.currentNode()['AB']:
337	if len(p) != 2: continue
338	x, y = ord(p[0])-oa, ord(p[1])-oa
339	if not (0 <= x < self.boardsize and 0 <= y < self.boardsize):
340	messages.insert('end', 'Invalid position in AE tag, in file ' + f + '\n')
341
342	assert 0 <= x < self.boardsize and 0 <= y < self.boardsize
343	b.play((x, y), 'black')
344	for algo in algolist: algo.AB_process(x, y)
345
346	if cursor.currentNode().has_key('AW'):
347	for p in cursor.currentNode()['AW']:
348	if len(p) != 2: continue
349	x, y = ord(p[0])-oa, ord(p[1])-oa
350	if not (0 <= x < self.boardsize and 0 <= y < self.boardsize):
351	messages.insert('end', 'Invalid position in AE tag, in file '\
352	+ f + '\n')
353
354	assert 0 <= x < self.boardsize and 0 <= y < self.boardsize
355	b.play((x, y), 'white')
356	for algo in algolist: algo.AW_process(x, y)
357
358	if cursor.currentNode().has_key('AE'):
359	for p in cursor.currentNode()['AE']:
360	if len(p) != 2: continue
361	x, y = ord(p[0])-oa, ord(p[1])-oa
362	if not (0 <= x < self.boardsize and 0 <= y < self.boardsize):
363	messages.insert('end', 'Invalid position in AE tag, in file '\
364	+ f + '\n')
365
366	assert 0 <= x < self.boardsize and 0 <= y < self.boardsize
367	assert b.getStatus(x,y) != " "
368
369	if b.status[(x,y)] == 'black': removed = 'B'
370	elif b.status[(x,y)] == 'white': removed = 'W'
371	b.remove((x, y))
372	for algo in algolist: algo.AE_process(x, y, removed)
373
374	except AssertionError:
375	print 'oops' # FIXME ?!
376	pass
377
378	for algo in algolist: algo.endOfNode_process()
379
380	if processVariations and cursor.noChildren() > 1:
381	# several variations start from this node
382	for algo in algolist: algo.branchpoint_process()
383	self.branchpoints.append((cursor.currentN, b.copy(), 0))
384	print 'branchpoint'
385	if not cursor.atEnd(): cursor.next()
386	elif self.branchpoints:
387	for algo in algolist: algo.endOfVariation_process()
388	# print 'eov process'
389	cursor.currentN, b, whichVar = self.branchpoints.pop()
390	if whichVar+2 < cursor.noChildren():
391	for algo in algolist: algo.branchpoint_process()
392	self.branchpoints.append((cursor.currentN, b.copy(), whichVar+1))
393	cursor.next(whichVar+1)
394	else:
395	break
396	# ---------------- end of C++ part
397
398	except SGFError:
399	messages.insert('end', 'File '+f+iColl+': SGF error in node '+`counter+3`+'.\n')
400	except ImportError: # FIXME
401	messages.insert('end', 'File ' + f + iColl + ': Error in node ' + `counter+3` + '.\n')
402
403	try:
404	# compute Dyer signature (moves 20, 40, 60, 31, 51, 71 in sgf coordinates -
405	# this will mostly characterise the game uniquely and allow to search
406	# for duplicates in an easy way)
407
408	while len(signature1) < 6: signature1 += '??'
409	while len(signature2) < 6: signature2 += '??'
410	signature = signature1 + signature2
411	signature = symmetrizeSig(signature, self.boardsize)
412
413	# Now check for duplicates; first in the currently processed database,
414	# then in the others in self.DBlist[`self.boardsize`]. If games with the
415	# same signature are found, the final position is compared. If those agree,
416	# too, it is concluded that the games are identical.
417
418	if duplicateCheck and \
419	(counter > 20 or (counter > 5 and strictDuplicateCheck)):
420
421	duplist = []
422	for sli in range(len(siglist)):
423	if siglist[sli] == signature and \
424	(not strictDuplicateCheck or algolist[0].duplicateCheck(sli)):
425	duplist.append(split(namelist[sli], '\|%')[0]) # FIXME ?!
426
427	if gamelist:
428	for db in gamelist.DBlist[`self.boardsize`]:
429	if db.disabled: continue
430	for i in range(len(db.namelist)):
431	g = db.namelist[i]
432	if signature == g[4]:
433	if not strictDuplicateCheck:
434	duplist.append(os.path.join(db.sgfpath, g[0]))
435	else:
436	if algolist[0].duplicateCheck(i, db.datapath):
437	duplist.append(os.path.join(db.sgfpath, g[0]))
438
439	if strictDuplicateCheck:
440	dupText = ' is the same as '
441	else:
442	dupText = ' is probably the same as '
443
444	if duplist:
445	ff = join(duplist, ', ')
446	if duplicateCheck == 1:
447	messages.insert('end', 'Duplicate: ' + os.path.split(f)[1] +\
448	iColl+ dupText + ff + '.\n')
449	elif duplicateCheck == 3:
450	messages.insert('end', 'Duplicate: '+os.path.split(f)[1]+iColl+\
451	dupText + ff + ', omitted.\n')
452	continue
453	elif duplicateCheck == 2:
454	if askyesno('Duplicate', 'File ' +os.path.split(f)[1]+iColl\
455	+ ' is the same as ' + ff + '. Include it?'):
456	messages.insert('end', 'Duplicate: ' + os.path.split(f)[1]+iColl\
457	+ dupText + ff + '\n')
458	else:
459	messages.insert('end', 'Duplicate: ' + os.path.split(f)[1]\
460	+ dupText + ff + ', omitted.\n')
461	continue
462
463	siglist.append(signature)
464
465	except ImportError: # FIXME
466	messages.insert('end', 'File ' + f +iColl + ': Error computing the Dyer signature.\n')
467	signature = '????????????'
468
469	# ----------------------------------
470
471	for algo in algolist: algo.endgame_process()
472
473	if os.path.split(f)[1][-4:] == '.sgf': filenameDB = os.path.split(f)[1][:-4]
474	elif os.path.split(f)[1][-4:] == '.mgt': filenameDB = os.path.split(f)[1][:-2]
475	else: filenameDB = os.path.split(f)[1] + '.'
476
477	namelist.append(filenameDB + iColl + '\|%' + playerB + '\|%' + playerW + '\|%' \
478	+ result + '\|%' + signature + '\|%' + date)
479
480	gameCtr += 1
481
482
483	if gameCtr:
484	try:
485	for algo in algolist: algo.finalize_process(datap)
486	except IOError:
487	messages.insert('end', 'I/O Error: Could not write database files (Directory write-protected?).\n')
488	return
489
490	try:
491	file = open(os.path.join(datap[0], 'namelist' + datap[1] + '.db'), 'w')
492	for s in namelist:
493	file.write(s+'\n')
494	file.close()
495	except IOError:
496	messages.insert('end', 'I/O Error: Could not write namelist.db file.\n')
497
498	if self.progBar:
499	self.progBar.redraw(1)
500	self.progBar.write('%1.1f seconds' % (time.time() - currentTime))
501
502	messages.insert('end', '%1.1f seconds' % (time.time() - currentTime))
503	messages.insert('end', '... finished.\n')
504
505	self.datapath = datap
506	self.sgfpath = dbpath
507	self.algos = algos
508	self.namelist = [split(x[:-1], '\|%') for x in namelist[2:]]
509	self.disabled = 0
510	self.current = array('L', range(len(namelist)-2))
511	self.results = []
512
513
514	def validNamelist(self, namelist):
515	"""
516	Return true if the list namelist seems to contain a valid namelist for
517	this Kombilo version.
518	"""
519
520	try:
521	assert namelist[0][:-1]=='kombilo06'
522	int(namelist[1][:-1])
523	filename, PB, PW, res, sig, date = split(namelist[2][:-1], '\|%')
524	except: return 0
525	return 1
526
527
528	def loadDB(self, sgfpath=None, datapath=None, disabled=0):
529
530	if not datapath is None: self.datapath = datapath
531	if not sgfpath is None: self.sgfpath = sgfpath
532
533	self.namelist= []
534	self.disabled = disabled
535
536	try:
537	file = open(os.path.join(self.datapath[0], 'namelist' + self.datapath[1] + '.db'))
538	namelist = file.readlines()
539	file.close()
540	except:
541	return -2
542
543	try:
544	assert self.validNamelist(namelist)
545	algos = int(namelist[1][:-1])
546	except:
547	return -1
548
549	try:
550	self.namelist = [split(x[:-1], '\|%') for x in namelist[2:]]
551	if not disabled:
552	self.current = array('L', range(len(namelist)-2))
553	self.results = []
554	else:
555	self.current = array('L')
556	self.results = []
557	self.algos = algos
558	except: return -3
559
560	return 1
561
562
563	def start(self):
564	""" Iterate through self.current. Continue with 'next()'. """
565
566	if len(self.current) and not self.disabled:
567	self.currentIndexWithinCurrent = 0
568	self.newCurrent = array('L')
569	try: self.oldMatchlists = self.matchlists
570	except AttributeError: pass
571	self.matchlists = []
572	self.results = []
573	return self.current[0]
574
575
576	def next(self):
577	self.currentIndexWithinCurrent += 1
578	if self.currentIndexWithinCurrent >= len(self.current):
579	self.current = self.newCurrent
580	return None
581
582	return self.current[self.currentIndexWithinCurrent]
583
584
585	def makeCurrentCandidate(self, matchList):
586	self.newCurrent.append(self.current[self.currentIndexWithinCurrent])
587	self.matchlists.append(matchList)
588
589
590	def makeCurrentHit(self, results = None):
591	""" results is list of Hit's (see algosPY.py) """
592	win = self.getCurrentWinner()
593	self.newCurrent.append(self.current[self.currentIndexWithinCurrent])
594	if not results is None and not type(results) == type([]): print 'Error' # FIXME
595	self.results.append(results)
596
597
598	def discardCurrent(self):
599	pass
600
601
602	def getCurrentMatchlist(self):
603	try: return self.oldMatchlists[self.currentIndexWithinCurrent]
604	except: return []
605
606
607	def getCurrentWinner(self):
608	currentWithinDB = self.current[self.currentIndexWithinCurrent]
609	return self.namelist[currentWithinDB][3]
610
611	# ---------------------------------------------------------------------------------------
612
613	def symmetrizeSig(s, boardsize):
614	""" Given a signature s, compute the 'rotated'/'mirrored' signatures,
615	and return the first, w.r.t. lexicographic order, of all these.
616	Games which differ only by a symmetry of the board will thus have the
617	same symmetrized signature. """
618
619	l = []
620	oa = ord('a')
621	for i in range(6): l.append((ord(s[2i])-oa, ord(s[2i+1])-oa))
622
623	m = []
624	for f in Pattern.flips:
625	k1 = [ f(x[0], x[1], boardsize-1, boardsize-1) for x in l]
626	k2 = []
627	for i in range(6):
628	if s[2i] == '?': k2.append('?'*)
629	else: k2.append(chr(k1[i][0]+oa))
630	if s[2i+1] == '?': k2.append('?'*)
631	else: k2.append(chr(k1[i][1]+oa))
632	m.append(join(k2, ''))
633
634	m.sort()
635	return m[0]
636
637	# ---------------------------------------------------------------------------------------
638

Note: See TracBrowser for help on using the browser.

root/06/devel-old/searchPY.py

Download in other formats: