SuperPoint学习---demo代码理解
如果不是要改进SuperPoint的模型的话，并没有必要重新训练。
SP在准确度上相⽐SIFT、SURF传统视觉⽅法有了提升，⽽且作者也提供了不错的预训练模型，如果仅需要提取特征点，或提取单应矩阵，可直接从demo的代码⼊⼿。
花了⼀整晚拜读了match_features_demo.py的代码，在关键处都加了中⽂注释。便于将SuperPoint强⼤的特征提取能⼒快速应⽤于新的idea~
除了tensorflow，opencv和numpy再⽆其它依赖，并且tf也不需要gpu版，简单易⽤
(2020.4.17)
不过有⼀个问题，这个版本的superpoint强在可以⽤⾃⼰的数据训练，但是如果仅仅⽤于提取特征的话，还是应该⽤magicleap版。
对于⼀张240x320的图⽚，该版本在我的5500U上需要1.5s左右，⽽原版只需0.7s。速度差了⼀倍，⽽拼接质量相似
感觉sp论⽂⾥讲的实时性并没有那么实时，或者说只有在120x160这样的迷你图⽚才可以达到5fps左右的勉强实时，然⽽opencv⾥优化后的sift可以在0.05s内完成）
##Author: /rpautrat/SuperPoint
aside是什么意思import argpar
from pathlib import Path
import cv2
import numpy as np
import tensorflow as tf # noqa: E402
from ttings import EXPER_PATH # noqa: E402
def extract_SIFT_keypoints_and_descriptors(img):
gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
sift = cv2.xfeatures2d.SIFT_create()
kp, desc = sift.detectAndCompute(np.squeeze(gray_img),None)
return kp, desc
def extract_superpoint_keypoints_and_descriptors(keypoint_map, descriptor_map,
keep_k_points=1000):
def lect_k_best(points, k):
colorref""" Select the k most probable points (and strip their proba).
points has shape (num_points, 3) where the last coordinate is the proba. """
sorted_prob = points[points[:,2].argsort(),:2]
start =min(k, points.shape[0])
return sorted_prob[-start:,:]
# Extract keypoints
keypoints = np.where(keypoint_map >0)
prob = keypoint_map[keypoints[0], keypoints[1]]
#分别是所有的横坐标和所有的纵坐标，取出来的是所有概率值
wds是什么keypoints = np.stack([keypoints[0], keypoints[1], prob], axis=-1)
#此时的keypoints是(n,3),n是疑似特征点的个数，3是横坐标，纵坐标，概率值
keypoints = lect_k_best(keypoints, keep_k_points)
keypoints = keypoints.astype(int)
# Get descriptors for keypoints
desc = descriptor_map[keypoints[:,0], keypoints[:,1]]#（n,256),n个关键点，256个描述符
# Convert from just pts to cv2.KeyPoints
# Convert from just pts to cv2.KeyPoints
keypoints =[cv2.KeyPoint(p[1], p[0],1)for p in keypoints]
return keypoints, desc
def match_descriptors(kp1, desc1, kp2, desc2):
# Match the keypoints with the warped_keypoints with nearest neighbor arch
bf = cv2.BFMatcher(cv2.NORM_L2, crossCheck=True)
##是⼀个暴⼒匹配的对象，取desc1中⼀个描述⼦，再与desc2中的所有描述⼦计算欧式距离
matches = bf.match(desc1, desc2)
##上⼀⾏的返回值类似元组的集合（i，j）代表第⼀个集合的第i个点的最佳匹配是第⼆个集合的第j个点
matches_idx = np.array([m.queryIdx for m in matches])
m_kp1 =[kp1[idx]for idx in matches_idx]
matches_idx = np.array([m.trainIdx for m in matches])
m_kp2 =[kp2[idx]for idx in matches_idx]
####m_kp1是第⼀张图⽚的特征点，m_kp2是第⼆张图⽚的特征点，此时它们已经⼀⼀对应了（⾄少是最对应的，距离最⼩的return m_kp1, m_kp2, matches
dcmpdef compute_homography(matched_kp1, matched_kp2):
##matched_kp1本来是KeyPoint对象，转化成了（n,2)的numpy数组，n是特征点个数，2是横、纵坐标
matched_pts1 = cv2.KeyPoint_convert(matched_kp1)
matched_pts2 = cv2.KeyPoint_convert(matched_kp2)
# Estimate the homography between the matches using RANSAC
H, inliers = cv2.findHomography(matched_pts1[:,[1,0]],##通过两个图像的点集合，来寻找单应变换
matched_pts2[:,[1,0]],
cv2.RANSAC)## H便是（3，3）的单应矩阵
inliers = inliers.flatten()#inliers 是正确的匹配
return H, inliers
def preprocess_image(img_file, img_size):
img = cv2.imread(img_file, cv2.IMREAD_COLOR)
img = size(img, img_size)
img_orig = py()
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)nsync
img = np.expand_dims(img,2)#(480,640,1)
img = img.astype(np.float32)# int --->> float ，以便下⾯归⼀化
img_preprocesd = img /255.
# cv2.imshow("orig",img)
如何激发学生的学习兴趣
# cv2.waitKey(0)
return img_preprocesd, img_orig
if __name__ =='__main__':
parr = argpar.ArgumentParr()
parr = argpar.ArgumentParr(description='Compute the homography \
between two images with the SuperPoint feature matches.')
parr.add_argument('weights_name',type=str)
parr.add_argument('img1_path',type=str)
parr.add_argument('img2_path',type=str)
parr.add_argument('--H',type=int, default=480,
help='The height in pixels to resize the images to. \
insistence
(default:480)')
parr.add_argument('--W',type=int, default=640,
help='The width in pixels to resize the images to. \
(default:640)')
parr.add_argument('--k_best',type=int, default=1000,
help='Maximum number of keypoints to keep \
(default:1000)')
args = parr.par_args()
args = parr.par_args()
weights_name = args.weights_name
img1_file = args.img1_path
img2_file = args.img2_path
img_size =(args.W, args.H)
keep_k_best = args.k_best
weights_root_dir = Path(EXPER_PATH,'saved_models')
weights_root_dir.mkdir(parents=True, exist_ok=True)
weights_dir = Path(weights_root_dir, weights_name)
graph = tf.Graph()
女孩的英文with tf.Session(graph=graph)as ss:
tf.saved_model.loader.load(ss,
[tf.saved_model.tag_constants.SERVING],
str(weights_dir))
input_img_tensor = _tensor_by_name('superpoint/image:0')
output_prob_nms_tensor = _tensor_by_name('superpoint/prob_nms:0')
output_desc_tensors = _tensor_by_name('superpoint/descriptors:0')
img1, img1_orig = preprocess_image(img1_file, img_size)
out1 = ss.run([output_prob_nms_tensor, output_desc_tensors],
feed_dict={input_img_tensor: np.expand_dims(img1,0)})
keypoint_map1 = np.squeeze(out1[0])
#将数组中维度为1的维度去掉（1，480，640）---> (480,640)
descriptor_map1 = np.squeeze(out1[1])
kp1, desc1 = extract_superpoint_keypoints_and_descriptors(
keypoint_map1, descriptor_map1, keep_k_best)
#kp1是⼀个list，成员是KeyPoint类，desc1是（n个关键点，256）
img2, img2_orig = preprocess_image(img2_file, img_size)
out2 = ss.run([output_prob_nms_tensor, output_desc_tensors],
feed_dict={input_img_tensor: np.expand_dims(img2,0)})
keypoint_map2 = np.squeeze(out2[0])
descriptor_map2 = np.squeeze(out2[1])
kp2, desc2 = extract_superpoint_keypoints_and_descriptors(
keypoint_map2, descriptor_map2, keep_k_best)
the colony
>>##此时两张图⽚的特征点和描述符已分别保存于kp1，desc1，kp2，desc2>>#### Match and get rid of outliers
m_kp1, m_kp2, matches = match_descriptors(kp1, desc1, kp2, desc2)
##m_kp1是第⼀张图⽚的特征点，m_kp2是第⼆张图⽚的特征点，此时它们已经⼀⼀对应了（⾄少是最对应的，距离最⼩的
H, inliers = compute_homography(m_kp1, m_kp2)
>>> H是单应变换的3x3矩阵，inliers 是成功的匹配，可以理解为在每个匹配上的⼀个bool值的listsalad# Draw SuperPoint matches
matches = np.array(matches)[inliers.astype(bool)].tolist()
##matches原来是所有的所有特征点之间的最匹配（欧⽒距离最短），现在使⽤inliers过滤，只剩下了真正成功的匹配（可以理解为加了阈值
matched_img = cv2.drawMatches(img1_orig, kp1, img2_orig, kp2, matches,
None, matchColor=(0,255,0),##匹配⽤绿⾊（BGR）
singlePointColor=(0,0,255))##未匹配⽤红⾊
cv2.imshow("SuperPoint matches", matched_img)
# Compare SIFT matches
sift_kp1, sift_desc1 = extract_SIFT_keypoints_and_descriptors(img1_orig)
sift_kp2, sift_desc2 = extract_SIFT_keypoints_and_descriptors(img2_orig)
sift_m_kp1, sift_m_kp2, sift_matches = match_descriptors(
sift_kp1, sift_desc1, sift_kp2, sift_desc2)
sift_H, sift_inliers = compute_homography(sift_m_kp1, sift_m_kp2)
# Draw SIFT matches
# Draw SIFT matches
sift_matches = np.array(sift_matches)[sift_inliers.astype(bool)].tolist() sift_matched_img = cv2.drawMatches(img1_orig, sift_kp1, img2_orig, sift_kp2, sift_matches,None,
matchColor=(0,255,0),
singlePointColor=(0,0,255))
cv2.imshow("SIFT matches", sift_matched_img)
cv2.waitKey(0)
