python Realizes Lianliankan Assistance (Image Recognition)

Personal interest, using python to realize the auxiliary program of Lianliankan, and summarize the realization process and knowledge points.

General thinking

1. Get the window of Lianliankan program and put it in front of it
2. Screenshot of the game interface, cut each 1 small icon and form a 2-dimensional list composed of small icons
3. Traverse the 2D list of pictures, and convert the 2D list into a 2D array composed of numbers. The same values of pictures are the same.
4. Traverse the 2-dimensional array, find the object that can be eliminated, and realize the algorithm:

Two icons are adjacent. (1 line connection) Two icons are in the same column, and the icons in the middle are all empty (the value is 0) (connected by 1 line) Connect the two lines, turn once, and all icons on the path are empty. (2-wire connection) Connect with 3 lines, turn twice, and all icons on the path are empty. (3-wire connection) Click the two icons respectively and set the corresponding 2D data value to 0

Problems encountered in the implementation process

Picture cutting

im = image.crop((left,top,right,bottom))
//image.crop Parameter is 1 List or tuple, in the order of (left,top,right,bottom)

Find the game running window

hdwd = win32gui.FindWindow(0,wdname)
#  Set to front display 
win32gui.SetForegroundWindow(hdwd)

Don't click the window to minimize, it can't pop up after clicking.

Scale the picture and turn it to grayscale

img1 = im1.resize((20, 20), Image.ANTIALIAS).convert('L')

Image. ANTIALIAS is an anti-aliasing option, and the picture has no burrs.

Get the RGB value of each point in the picture

pi1 = list(img1.getdata())

Each element of the list is a 3-digit value, which represents the RGB value of this point. The list pi1 has 400 elements. (Because the picture is 20*20)

Mouse click elimination

PyMouse. click () This method is double-clicked by default and changed to PyMouse. press () or PyMouse. release ()

Judge the similarity of pictures Hamming distance, average hash

def compare_img(self,im1,im2):
 img1 = im1.resize((20, 20), Image.ANTIALIAS).convert('L')
 img2 = im2.resize((20, 20), Image.ANTIALIAS).convert('L')
 pi1 = list(img1.getdata())
 pi2 = list(img2.getdata())
 avg1 = sum(pi1) / len(pi1)
 avg2 = sum(pi2) / len(pi2)
 hash1 = "".join(map(lambda p: "1" if p > avg1 else "0", pi1))
 hash2 = "".join(map(lambda p: "1" if p > avg2 else "0", pi2))
 match = 0
 for i in range(len(hash1)):
 if hash1[i] != hash2[i]:
 match += 1
 # match = sum(map(operator.ne, hash1, hash2))
 # match  The smaller the value, the higher the similarity 
 return match

Calculating histogram

from PIL import Image

#  Convert a picture to RGB
def make_regalur_image(img, size=(8, 8)):
 gray_image = img.resize(size).convert('RGB')
 return gray_image


#  Calculating histogram 
def hist_similar(lh, rh):
 assert len(lh) == len(rh)
 hist = sum(1 - (0 if l == r else float(abs(l - r)) / max(l, r)) for l, r in zip(lh, rh)) / len(lh)
 return hist


#  Calculate similarity 
def calc_similar(li, ri):
 calc_sim = hist_similar(li.histogram(), ri.histogram())
 return calc_sim


if __name__ == '__main__':
 image1 = Image.open('1-10.jpg')
 image1 = make_regalur_image(image1)
 image2 = Image.open('2-11.jpg')
 image2 = make_regalur_image(image2)
 print(" The similarity between pictures is ", calc_similar(image1, image2))
 #  Value in [0,1] The larger the value, the higher the similarity 

Cosine similarity of pictures

from PIL import Image
from numpy import average, dot, linalg


#  Unify pictures 1 Chemical treatment 
def get_thum(image, size=(64, 64), greyscale=False):
 #  Utilization image Reset the image size , Image.ANTIALIAS For high quality 
 image = image.resize(size, Image.ANTIALIAS)
 if greyscale:
 #  Convert a picture to L Mode, which is a gray image, and each pixel uses 8 A bit Denote 
 image = image.convert('L')
 return image


#  Calculate the cosine distance of the picture 
def image_similarity_vectors_via_numpy(image1, image2):
 image1 = get_thum(image1)
 image2 = get_thum(image2)
 images = [image1, image2]
 vectors = []
 norms = []
 for image in images:
 vector = []
 for pixel_tuple in image.getdata():
 vector.append(average(pixel_tuple))
 vectors.append(vector)
 # linalg=linear (Linear) +algebra (Algebra), norm The norm is represented 
 #  Find the norm of a picture? ? 
 norms.append(linalg.norm(vector, 2))
 a, b = vectors
 a_norm, b_norm = norms
 # dot Returns the dot product, and the 2 Dimensional array (matrix) for calculation 
 res = dot(a / a_norm, b / b_norm)
 return res

if __name__ == '__main__':

 image1 = Image.open('1-9.jpg')
 image2 = Image.open('8-6.jpg')
 cosin = image_similarity_vectors_via_numpy(image1, image2)
 print(' Cosine similarity of pictures ', cosin)
 #  Value in [0,1] The larger the value, the higher the similarity , The calculation amount is large and the efficiency is low 

Complete code

import win32gui
import time
from PIL import ImageGrab , Image
import numpy as np
from pymouse import PyMouse


class GameAuxiliaries(object):
 def __init__(self):
 self.wdname = r' Pet Lianliankan Classic Edition 2, Pet Lianliankan Classic Edition 2 Games ,4399 Games  www.4399.com - Google Chrome'
 # self.wdname = r'main.swf - PotPlayer'
 self.image_list = {}
 self.m = PyMouse()
 def find_game_wd(self,wdname):
 #  Get the window handle 
 hdwd = win32gui.FindWindow(0,wdname)
 #  Set to front display 
 win32gui.SetForegroundWindow(hdwd)
 time.sleep(1)

 def get_img(self):
 image = ImageGrab.grab((417, 289, 884, 600))
 # image = ImageGrab.grab((417, 257, 885, 569))
 image.save('1.jpg','JPEG')
 for x in range(1,9):
 self.image_list[x] = {}
 for y in range(1,13):
 top = (x - 1) * 38 + (x-2)
 left =(y - 1) * 38 +(y-2)
 right = y * 38 + (y-1)
 bottom = x * 38 +(x -1)
 if top < 0:
 top = 0
 if left < 0 :
 left = 0
 im_temp = image.crop((left,top,right,bottom))
 im = im_temp.crop((1,1,37,37))
 im.save('{}-{}.jpg'.format(x,y))
 self.image_list[x][y]=im

 #  Determine whether the two pictures are the same. Hamming distance, average hash 
 def compare_img(self,im1,im2):
 img1 = im1.resize((20, 20), Image.ANTIALIAS).convert('L')
 img2 = im2.resize((20, 20), Image.ANTIALIAS).convert('L')
 pi1 = list(img1.getdata())
 pi2 = list(img2.getdata())
 avg1 = sum(pi1) / len(pi1)
 avg2 = sum(pi2) / len(pi2)
 hash1 = "".join(map(lambda p: "1" if p > avg1 else "0", pi1))
 hash2 = "".join(map(lambda p: "1" if p > avg2 else "0", pi2))
 match = 0
 for i in range(len(hash1)):
 if hash1[i] != hash2[i]:
 match += 1
 # match = sum(map(operator.ne, hash1, hash2))
 # match  The smaller the value, the higher the similarity 
 return match


 #  Convert a picture matrix into a digital matrix 

 def create_array(self):
 array = np.zeros((10,14),dtype=np.int32)
 img_type_list = []
 for row in range(1,len(self.image_list)+1):
 for col in range(1,len(self.image_list[1])+1):
 # im = Image.open('{}-{}.jpg'.format(row,col))
 im = self.image_list[row][col]
 for img in img_type_list:
 match = self.compare_img(im,img)
 # match = test2.image_similarity_vectors_via_numpy(im,img)
 if match <15:
 array[row][col] = img_type_list.index(img) +1

 break
 else:
 img_type_list.append(im)
 array[row][col] = len(img_type_list)

 return array

 def row_zero(self,x1,y1,x2,y2,array):
 ''' Icons in the middle of the same picture are all empty '''
 if x1 == x2:
 min_y = min(y1,y2)
 max_y = max(y1,y2)
 if max_y - min_y == 1:
 return True
 for y in range(min_y+1,max_y):
 if array[x1][y] != 0 :
 return False
 return True
 else:
 return False

 def col_zero(self,x1,y1,x2,y2,array):
 ''' Same picture, same column '''
 if y1 == y2:
 min_x = min(x1,x2)
 max_x = max(x1,x2)
 if max_x - min_x == 1:
 return True
 for x in range(min_x+1,max_x):
 if array[x][y1] != 0 :
 return False
 return True
 else:
 return False

 def two_line(self,x1,y1,x2,y2,array):
 ''' Connect two lines and turn 1 Times '''
 for row in range(1,9):
 for col in range(1,13):
 if row == x1 and col == y2 and array[row][col]==0 and self.row_zero(x1,y1,row,col,array) and self.col_zero(x2,y2,row,col,array):
 return True
 if row == x2 and col == y1 and array[row][col]==0 and self.row_zero(x2,y2,row,col,array) and self.col_zero(x1,y1,row,col,array):
 return True
 return False

 def three_line(self,x1,y1,x2,y2,array):
 '''3 Connect lines and turn twice '''
 for row1 in range(10):
 for col1 in range(14):
 for row2 in range(10):
 for col2 in range(14):
 if array[row1][col1] == array[row2][col2] == 0 and self.row_zero(x1,y1,row1,col1,array) and self.row_zero(x2,y2,row2,col2,array) and self.col_zero(row1,col1,row2,col2,array):
 return True
 if array[row1][col1] == array[row2][col2] == 0 and self.col_zero(x1,y1,row1,col1,array) and self.col_zero(x2,y2,row2,col2,array) and self.row_zero(row1,col1,row2,col2,array):
 return True
 if array[row1][col1] == array[row2][col2] == 0 and self.row_zero(x2,y2,row1,col1,array) and self.row_zero(x1,y1,row2,col2,array) and self.col_zero(row1,col1,row2,col2,array):
 return True
 if array[row1][col1] == array[row2][col2] == 0 and self.col_zero(x2,y2,row1,col1,array) and self.col_zero(x1,y1,row2,col2,array) and self.row_zero(row1,col1,row2,col2,array):
 return True
 return False


 def mouse_click(self,x,y):

 top = (x - 1) * 38 + (x - 2)
 left = (y - 1) * 38 + (y - 2)
 right = y * 38 + (y - 1)
 bottom = x * 38 + (x - 1)
 if top < 0:
 top = 0
 if left < 0:
 left = 0

 self.m.press(int(417+(left+right)/2) ,int(289+(top+bottom)/2) )

 def find_same_img(self,array):

 for x1 in range(1,9):
 for y1 in range(1,13):
 if array[x1][y1] == 0:
 continue
 for x2 in range(1,9):
 for y2 in range(1,13):
 if x1==x2 and y1 == y2:
 continue
 if array[x2][y2] == 0 :
 continue
 if array[x1][y1] != array[x2][y2] :
 continue
 if array[x1][y1] ==array[x2][y2] and (self.row_zero(x1,y1,x2,y2,array) or self.col_zero(x1,y1,x2,y2,array) or self.two_line(x1,y1,x2,y2,array) or self.three_line(x1,y1,x2,y2,array)):
 print(" It can be eliminated! x{}y{}  And  x{}y{}".format(x1,y1,x2,y2))
 self.mouse_click(x1,y1)
 time.sleep(0.1)
 self.mouse_click(x2,y2)
 time.sleep(0.1)
 array[x1][y1]=array[x2][y2]=0


 def run(self):
 # Find the game running window 
 self.find_game_wd(self.wdname)
 #  Screenshot, cut into small icons 
 self.get_img()
 #  Convert a picture matrix into a digital matrix 
 array = self.create_array()
 print(array)
 #  Traverse the matrix, find the items that can be eliminated, and click Eliminate 
 for i in range(10):
 self.find_same_img(array)
 print(array)


if __name__ == '__main__':
 ga = GameAuxiliaries()
 ga.run()

Summarize

In fact, the program can't fully realize the auxiliary function, mainly because the better rules can't be found when cutting pictures, which makes it difficult to identify pictures. The scale and judgment threshold can't find a balance point. If the threshold is too large, different icons will be identified as the same, while the threshold is too small, and the same icons will be judged as different.

For more wonderful articles about python games, please click to view the following topics:

python Tetris Game Collection

python Classic Games Summary

python WeChat Jump 1 Jump Game Collection