From 0f4c9266524e3b3836ea1bcbae0dc14f1adf7d0b Mon Sep 17 00:00:00 2001 From: d <215442894@qq.com> Date: Sat, 22 Jun 2024 22:25:07 +0800 Subject: [PATCH 1/9] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/model.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/model.py b/modules/model.py index 57539fd..fdb8ab2 100644 --- a/modules/model.py +++ b/modules/model.py @@ -136,16 +136,16 @@ def forward(self, x): x = self.norm(x) #main backbone - x1 = self.block1(x) - x2 = self.block2(x1 + self.skip1(x)) - x3 = self.block3(x2) - x4 = self.block4(x3) - x5 = self.block5(x4) + x1 = self.block1(x) # x1: [B, 24, H/4, W/4] + x2 = self.block2(x1 + self.skip1(x)) # x2: [B, 24, H/4, W/4] + x3 = self.block3(x2) # x3: [B, 64, H/8, W/8] + x4 = self.block4(x3) # x4: [B, 64, H/16, W/16] + x5 = self.block5(x4) # x5: [B, 64, H/32, W/32] #pyramid fusion - x4 = F.interpolate(x4, (x3.shape[-2], x3.shape[-1]), mode='bilinear') - x5 = F.interpolate(x5, (x3.shape[-2], x3.shape[-1]), mode='bilinear') - feats = self.block_fusion( x3 + x4 + x5 ) + x4 = F.interpolate(x4, (x3.shape[-2], x3.shape[-1]), mode='bilinear') # x4: [B, 64, H/8, W/8] + x5 = F.interpolate(x5, (x3.shape[-2], x3.shape[-1]), mode='bilinear') # x5: [B, 64, H/8, W/8] + feats = self.block_fusion( x3 + x4 + x5 ) # feats: [B, 64, H/8, W/8] #heads heatmap = self.heatmap_head(feats) # Reliability map From 83af450da9cdfc7c82ba4db46d586f829ef16dce Mon Sep 17 00:00:00 2001 From: d <215442894@qq.com> Date: Sat, 22 Jun 2024 23:11:27 +0800 Subject: [PATCH 2/9] :sparkles: add onnx converter --- convert_onnx.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 convert_onnx.py diff --git a/convert_onnx.py b/convert_onnx.py new file mode 100644 index 0000000..9c3df33 --- /dev/null +++ b/convert_onnx.py @@ -0,0 +1,20 @@ +import os +import torch +from modules.xfeat import XFeat +from modules.xfeat import XFeatModel + +# os.environ['CUDA_VISIBLE_DEVICES'] = '' #Force CPU, comment for GPU + +# set the model to evaluation mode +net = XFeatModel().eval() + +# Random input +x = torch.randn(1, 3, 720, 720) + +# export to ONNX +torch.onnx.export(net, x, "xfeat.onnx", verbose=True, + input_names=['input'], + output_names=['output_feats', "output_keypoints", "output_heatmap"], + opset_version=11) + +print("ONNX model saved as xfeat.onnx") \ No newline at end of file From 94a312aeb2f418eafeb2be9320ea7308e51083f4 Mon Sep 17 00:00:00 2001 From: d <215442894@qq.com> Date: Tue, 25 Jun 2024 23:06:21 +0800 Subject: [PATCH 3/9] :sparkles: check the exported onnx model with onnxruntime --- convert_onnx.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/convert_onnx.py b/convert_onnx.py index 9c3df33..1fa76cc 100644 --- a/convert_onnx.py +++ b/convert_onnx.py @@ -2,6 +2,7 @@ import torch from modules.xfeat import XFeat from modules.xfeat import XFeatModel +import onnxruntime as ort # os.environ['CUDA_VISIBLE_DEVICES'] = '' #Force CPU, comment for GPU @@ -9,7 +10,7 @@ net = XFeatModel().eval() # Random input -x = torch.randn(1, 3, 720, 720) +x = torch.randn(1, 3, 640, 640) # export to ONNX torch.onnx.export(net, x, "xfeat.onnx", verbose=True, @@ -17,4 +18,21 @@ output_names=['output_feats', "output_keypoints", "output_heatmap"], opset_version=11) -print("ONNX model saved as xfeat.onnx") \ No newline at end of file +print("ONNX model saved as xfeat.onnx") + +# check the onnx model with onnxruntime +ort_session = ort.InferenceSession("xfeat.onnx") +print("ONNX model loaded successfully") + +outputs = ort_session.run(None, {"input": x.numpy()}) + +# pytorch model outputs +torch_outputs = net(x) + +# compare the outputs +for i in range(len(outputs)): + print(f"onnx output shape {i}: {outputs[i].shape}") + print(f"torch output shape {i}: {torch_outputs[i].shape}") + print(f"Output {i} comparison: {torch.allclose(torch_outputs[i], torch.tensor(outputs[i]))}") + print(f'Output {i} max diff: {torch.max(torch.abs(torch_outputs[i] - torch.tensor(outputs[i])))}') + print("\n") From 41e9d4471fe10f6462620c912de5f01c54dc120e Mon Sep 17 00:00:00 2001 From: d <215442894@qq.com> Date: Sun, 30 Jun 2024 18:03:52 +0800 Subject: [PATCH 4/9] :bug: the input image should be grayscale --- convert_onnx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert_onnx.py b/convert_onnx.py index 1fa76cc..4fc6979 100644 --- a/convert_onnx.py +++ b/convert_onnx.py @@ -10,7 +10,7 @@ net = XFeatModel().eval() # Random input -x = torch.randn(1, 3, 640, 640) +x = torch.randn(1, 1, 640, 640) # export to ONNX torch.onnx.export(net, x, "xfeat.onnx", verbose=True, From b95fb01f3c27b52e3ee1ce9117a65daf29a38e91 Mon Sep 17 00:00:00 2001 From: d <215442894@qq.com> Date: Mon, 1 Jul 2024 00:13:51 +0800 Subject: [PATCH 5/9] :bug: forget to load the pretained model --- convert_onnx.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/convert_onnx.py b/convert_onnx.py index 4fc6979..b1745ac 100644 --- a/convert_onnx.py +++ b/convert_onnx.py @@ -9,6 +9,9 @@ # set the model to evaluation mode net = XFeatModel().eval() +# load the pretrained weights +net.load_state_dict(torch.load("weights/xfeat.pt", map_location=torch.device('cpu'))) + # Random input x = torch.randn(1, 1, 640, 640) From 039e1c5b4e9b550196d88dad779d9ded5a532986 Mon Sep 17 00:00:00 2001 From: d <215442894@qq.com> Date: Sat, 13 Jul 2024 17:29:39 +0800 Subject: [PATCH 6/9] make the align_corners option work --- modules/interpolator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/interpolator.py b/modules/interpolator.py index 699c1b8..d7594aa 100644 --- a/modules/interpolator.py +++ b/modules/interpolator.py @@ -29,5 +29,5 @@ def forward(self, x, pos, H, W): [B, N, C] sampled channels at 2d positions """ grid = self.normgrid(pos, H, W).unsqueeze(-2).to(x.dtype) - x = F.grid_sample(x, grid, mode = self.mode , align_corners = False) + x = F.grid_sample(x, grid, mode = self.mode , align_corners = self.align_corners) return x.permute(0,2,3,1).squeeze(-2) \ No newline at end of file From 58ced41656c77530b3fd29f12cb185d25eaaeaaa Mon Sep 17 00:00:00 2001 From: d <215442894@qq.com> Date: Sat, 13 Jul 2024 17:31:06 +0800 Subject: [PATCH 7/9] add detect example --- detect_example.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 detect_example.py diff --git a/detect_example.py b/detect_example.py new file mode 100644 index 0000000..28feb7e --- /dev/null +++ b/detect_example.py @@ -0,0 +1,40 @@ +import os +import numpy as np +import torch +import tqdm +import cv2 + +from modules.xfeat import XFeat + +# os.environ['CUDA_VISIBLE_DEVICES'] = '' #Force CPU, comment for GPU + +xfeat = XFeat() + +# read image +img = cv2.imread('D:/work/xfeatc/data/1.png', cv2.IMREAD_GRAYSCALE) + +# convert to tensor [1, 1, H, W] +x = torch.tensor(img, dtype = torch.float32) +x = x[None, None, :, :] / 255.0 + +print(x.shape) + +points = xfeat.detectAndCompute(x, top_k = 2)[0] + +print("----------------") + +# draw on image +cimg = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) +for p in points['keypoints']: + cv2.circle(cimg, (int(p[0]), int(p[1])), 3, (255, 0, 0), -1) + +cv2.imshow('image', cimg) +cv2.waitKey(0) + +# print keypoints +for p in points['keypoints']: + print("{}, {}".format(p[0], p[1])) + +# print descriptors +for d in points['descriptors']: + print(d) \ No newline at end of file From ff3b5ff78c7f4d8ce4e394e26dac69a0d9b8a415 Mon Sep 17 00:00:00 2001 From: d <215442894@qq.com> Date: Sat, 28 Sep 2024 13:44:28 +0800 Subject: [PATCH 8/9] permute the feats and keypoints from [B,C,H,W] to [B,H,W,C] --- modules/model.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/model.py b/modules/model.py index fdb8ab2..e6a42a5 100644 --- a/modules/model.py +++ b/modules/model.py @@ -125,8 +125,8 @@ def forward(self, x): input: x -> torch.Tensor(B, C, H, W) grayscale or rgb images return: - feats -> torch.Tensor(B, 64, H/8, W/8) dense local features - keypoints -> torch.Tensor(B, 65, H/8, W/8) keypoint logit map + feats -> torch.Tensor(B, H/8, W/8, 64) dense local features + keypoints -> torch.Tensor(B, H/8, W/8, 65) keypoint logit map heatmap -> torch.Tensor(B, 1, H/8, W/8) reliability map """ @@ -151,4 +151,8 @@ def forward(self, x): heatmap = self.heatmap_head(feats) # Reliability map keypoints = self.keypoint_head(self._unfold2d(x, ws=8)) #Keypoint map logits - return feats, keypoints, heatmap + # convert feats and keypoints from [B, C, H, W] to [B, H, W, C] + out_feats = feats.permute(0, 2, 3, 1) + out_keypoints = keypoints.permute(0, 2, 3, 1) + + return out_feats, out_keypoints, heatmap From fb9f4618099fc79ff462e8297eacb2cd287a7716 Mon Sep 17 00:00:00 2001 From: d <215442894@qq.com> Date: Sat, 28 Sep 2024 16:17:41 +0800 Subject: [PATCH 9/9] :fire: remove instance noramlization layer --- modules/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/model.py b/modules/model.py index e6a42a5..4ddb8e2 100644 --- a/modules/model.py +++ b/modules/model.py @@ -133,7 +133,7 @@ def forward(self, x): #dont backprop through normalization with torch.no_grad(): x = x.mean(dim=1, keepdim = True) - x = self.norm(x) + # x = self.norm(x) #main backbone x1 = self.block1(x) # x1: [B, 24, H/4, W/4]