图片拼接 --全景图合成

开发环境

• python3
• opencv-contrib-python—3.4.2.16
• opencv-python—3.4.2.16
• PyQt5 — 5.15.6

基本思路

• SIFT特征提取
• FLANN 特征匹配
• 单应性矩阵
• 仿射变换
• 图片融合
• 最大内接矩形裁剪
• GUI界面显示

代码程序

完整工程：https://gitee.com/wangchaosun/image_merge

图片融合

merge_pic.py

import numpy as np
import cv2

LEFTDIR = 1
RIGHTDIR = 2
#  get sift ,flann Machine
def getMachine():
    FLANN_INDEX_KDTREE = 1
    index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
    search_params = dict(checks=50)
    flann = cv2.FlannBasedMatcher(index_params, search_params)
    sift = cv2.xfeatures2d_SIFT().create()
    return sift,flann

def imgProcess(img,top,bot,left,right):
    imgBord = cv2.copyMakeBorder(img,top,bot,left,right,cv2.BORDER_CONSTANT,value=(0,0,0))
    imgGray = cv2.cvtColor(imgBord,cv2.COLOR_BGR2GRAY)
    return imgBord,imgGray

def findEdgeDot(img,x1,x2,y1,y2):
    dotsum = 0
    for i in range(x1,x2+1):
        for j in range(y1,y2+1):
            if not img.item(j,i):
                dotsum +=1 
    return dotsum 

def getSmallOuterRect(img):
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    thresh,binary=cv2.threshold(gray,1,255,cv2.THRESH_BINARY)
    image,contours,hierarchy = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    areaList = []
    for contour in contours:
        area = cv2.contourArea(contour)
        areaList.append(area)
    return cv2.boundingRect(contours[np.argmax(areaList)])

def getMaxInnerRect(img,step): # 输入的图像是二进制的
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    thresh,binary=cv2.threshold(gray,1,255,cv2.THRESH_BINARY)
    x = 0
    y = 0
    h,w = binary.shape
    topdot  =  findEdgeDot(binary,x,x+w-1,y,y)
    botdot  =  findEdgeDot(binary,x,x+w-1,y+h-1,y+h-1)
    lefdot  =  findEdgeDot(binary,x,x,y,y+h-1)
    rigdot  =  findEdgeDot(binary,x+w-1,x+w-1,y,y+h-1)
    edgedot = [topdot,botdot,lefdot,rigdot]
    while topdot or botdot or lefdot or rigdot :
        maxedge = max(edgedot)
        if maxedge == topdot:
            y += step
            h -= step 
        elif maxedge == botdot:
            h -= step
        elif maxedge == lefdot:
            x += step
            w -= step
        else:
            w -= step
        topdot  =  findEdgeDot(binary,x,x+w-1,y,y)
        botdot  =  findEdgeDot(binary,x,x+w-1,y+h-1,y+h-1)
        lefdot  =  findEdgeDot(binary,x,x,y,y+h-1)
        rigdot  =  findEdgeDot(binary,x+w-1,x+w-1,y,y+h-1)
        edgedot = [topdot,botdot,lefdot,rigdot]
    return x,y,w,h
            
def mergeImge(img1,img2,sift,flann):
    srcImg,img1gray = imgProcess(img1,img1.shape[0]//2,img1.shape[0]//2,img1.shape[1]//2,img1.shape[1]//2)
    testImg,img2gray= imgProcess(img2,img2.shape[0]//2,img2.shape[0]//2,img2.shape[1]//2,img2.shape[1]//2)
    
    # find the keypoints and descriptors with SIFT
    kp1, des1 = sift.detectAndCompute(img1gray, None)
    kp2, des2 = sift.detectAndCompute(img2gray, None)
    # FLANN parameters
    matches = flann.knnMatch(des1, des2, k=2)
    # Need to draw only good matches, so create a mask
    matchesMask = [[0, 0] for i in range(len(matches))]

    good = []
    pts1 = []
    pts2 = []
    # ratio test as per Lowe's paper
    for i, (m, n) in enumerate(matches):
        if m.distance < 0.7*n.distance:
            good.append(m)
            pts2.append(kp2[m.trainIdx].pt)
            pts1.append(kp1[m.queryIdx].pt)
            matchesMask[i] = [1, 0]

    # draw_params = dict(matchColor=(0, 255, 0),
    #                    singlePointColor=(255, 0, 0),
    #                    matchesMask=matchesMask,
    #                    flags=0)
    #img3 = cv2.drawMatchesKnn(img1gray, kp1, img2gray, kp2, matches, None, **draw_params)
    rows, cols = srcImg.shape[:2]
    MIN_MATCH_COUNT = 10
    if len(good) > MIN_MATCH_COUNT:
        src_pts = np.float32([kp1[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
        dst_pts = np.float32([kp2[m.trainIdx].pt for m in good]).reshape(-1, 1, 2)
        M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
        warpImg = cv2.warpPerspective(testImg, np.array(M), (testImg.shape[0]*2, testImg.shape[1]*2), flags=cv2.WARP_INVERSE_MAP)
        direction = -1
        # overlap region
        for col in range(0, cols):
            if srcImg[:, col].any() and warpImg[:, col].any():
                left = col
                break
        if srcImg[:, left-1].any():
            direction = LEFTDIR
        else: 
            direction = RIGHTDIR

        for col in range(cols-1, 0, -1):
            if srcImg[:, col].any() and warpImg[:, col].any():
                right = col
                break

        # get max region
        res = np.zeros([rows, cols, 3], np.uint8)
        for row in range(0, rows):
            for col in range(0, cols):
                if not srcImg[row, col].any():
                    res[row, col] = warpImg[row, col]
                elif not warpImg[row, col].any():
                    res[row, col] = srcImg[row, col]
                else:
                    srcImgLen = float(abs(col - left))
                    testImgLen = float(abs(col - right))
                    alpha = 1- srcImgLen / (srcImgLen + testImgLen) # 离得越近权重越大
                    if direction == LEFTDIR:
                        alpha = 1-alpha
                    res[row, col] = np.clip(srcImg[row, col] * (1-alpha) + warpImg[row, col] * alpha, 0, 255)
 
        # opencv is bgr, matplotlib is rgb
        x,y,w,h = getSmallOuterRect(res)
        resImg = res[y:y+h,x:x+w]
        x,y,w,h = getMaxInnerRect(resImg,2)
        outimg = resImg[y:y+h,x:x+w]

        return (True,resImg,outimg)

        #res = cv2.cvtColor(res, cv2.COLOR_BGR2RGB)
        # show the result
        # plt.figure()
        # plt.imshow(res)
        # plt.show()
    else:
        return (False)


if __name__ == "__main__":
    img1 = cv2.imread("./img/test1.jpg")
    img2 = cv2.imread("./img/test2.jpg")
    sift,flann = getMachine()
    res = mergeImge(img1,img2,sift,flann)
    if(res[0]):
        cv2.imshow("res",res[2])
        cv2.waitKey()

GUI界面

gui.py

import sys
import cv2 
from merge_pic import mergeImge,getMachine
from PyQt5.QtWidgets import  QApplication,QPushButton,QFileDialog,QMainWindow,QMessageBox,QLabel
from PyQt5.QtGui import QIcon,QImage, QPixmap

class myGUI(QMainWindow):
    def __init__(self):
        super().__init__()
        self.img1 = None
        self.img2 = None
        self.outimg = None
        self.outimg_state = 0
        self.imgNum = 0
        self.sift,self.flann = getMachine()
        self.initUI()

    def initUI(self): 
        self.setFixedSize(1000,800)                    
        self.setWindowTitle('IMAGE MERGE')  
        #self.statusBar()
        self.setWindowIcon(QIcon('./source/imgsrc/icon.png')) 
        self.imglbl = QLabel(self)
        #self.imglbl.setScaledContents (True)  
        self.imglbl.resize(900,700)
        self.imglbl.move(50,50)

        choiceImg = QPushButton('加载图片', self)  
        choiceImg.setFixedSize(100,50)
        choiceImg.move((1000//3-100)//2, 30) 
        choiceImg.clicked.connect(self.openFile)


        mergeImg = QPushButton('拼接图片', self)  
        mergeImg.setFixedSize(100,50)
        mergeImg.move((1000//3-100)//2+1000//3, 30) 
        mergeImg.clicked.connect(self.merge)

        saveImg = QPushButton('保存图片', self)  
        saveImg.setFixedSize(100,50)
        saveImg.move((1000//3-100)//2+1000//3*2, 30) 
        saveImg.clicked.connect(self.saveFile)

        self.statusBar().showMessage("请加载图片!!!")
        

        self.show() 
    def saveFile(self):
        if self.outimg_state:
            filter = "Images (*.jpg);;Images (*.bmp);;Images (*.png)"
            fname = QFileDialog.getSaveFileName(self, 'Save file', './output/',filter)
            cv2.imwrite(fname[0],self.outimg)
        else:
            self.statusBar().showMessage("没有拼接成功的图片!!!")
    def merge(self):
        if self.imgNum == 2:
            self.statusBar().showMessage("正在拼接中，请耐心等待~~~")
            QApplication.processEvents()
            res = mergeImge(self.img1,self.img2,self.sift,self.flann)
            self.img2 = None
            if not res[0]:
                self.statusBar().showMessage("没有足够的特征点,拼接失败!!!")
                self.imgNum = 0
                self.img1 = None
            else:
                self.imgNum = 1
                self.img1 = res[1]
                self.outimg = res[2]
                outimg = self.outimg.copy()
                self.outimg_state=1
                if outimg.shape[1] >900:
                    outimg = cv2.resize(outimg,(900,int(900/outimg.shape[1] * outimg.shape[0])))
                if outimg.shape[0] >700:
                    outimg = cv2.resize(outimg,(int(700/outimg.shape[0] * outimg.shape[1]),700))
                imgrgb = cv2.cvtColor(outimg,cv2.COLOR_BGR2RGB)
                w = imgrgb.shape[1]  # 获取图像大小
                h = imgrgb.shape[0]
                frame = QImage(imgrgb.data, w,h,w*3,QImage.Format_RGB888)
                pix = QPixmap.fromImage(frame)
                self.imglbl.setPixmap (pix)
                self.statusBar().showMessage("继续拼接-->请加载新的图片")
        else:
            self.statusBar().showMessage("图片数量不足，请继续添加...")

    def openFile(self):
        fname = QFileDialog.getOpenFileName(self, 'Choice Image', './img/')
        if not self.imgNum:
            self.img1 = cv2.imread(fname[0])
            if self.img1 is None:
                self.statusBar().showMessage("加载图片失败,请重新选择!!!")
                QMessageBox.warning(self,'Warning', '无效的图片!   ')
            else:
                self.imgNum += 1
                self.statusBar().showMessage("还需加载一张图片")
        elif self.imgNum==1:
            self.img2 = cv2.imread(fname[0])
            if self.img2 is None:
                self.statusBar().showMessage("加载图片失败,请重新选择!!!")
                QMessageBox.warning(self,'Warning', '无效的图片!   ')
            else:
                self.imgNum += 1
                self.statusBar().showMessage("已加载两张图片，可以拼接!!!")   
if __name__ == '__main__': 
    app = QApplication(sys.argv)

    ex = myGUI()
    sys.exit(app.exec_())  

结果展示

原始图片

拼接结果

1. 两张
   
2. 三张