ADD Matches graph for creating 360 panorama

[phamkhactu]

Hi all,

As the issue #54 #14 .
I know matches_graph as below:

A --> B 
A --> D
B --> C
B --> D
D --> E 

How can I pass the graph to stitch image?

As the trick in #14 . I saw that maybe I need create correctly keypoints matching as matches_graph I expected. First of all, I use the tutorial for reproducer the stitch:

import cv2
import numpy as np
from stitching.images import Images
import glob 
from stitching.feature_detector import FeatureDetector
from stitching.feature_matcher import FeatureMatcher
from stitching.subsetter import Subsetter
from stitching.camera_estimator import CameraEstimator
from stitching.camera_adjuster import CameraAdjuster
from stitching.camera_wave_corrector import WaveCorrector
from stitching.warper import Warper
from stitching.timelapser import Timelapser
from stitching.cropper import Cropper
from stitching.seam_finder import SeamFinder
from stitching.exposure_error_compensator import ExposureErrorCompensator
from stitching.blender import Blender
import glob

weir_imgs = glob.glob("my_data/*.jpg")
images = Images.of(weir_imgs)

medium_imgs = list(images.resize(Images.Resolution.MEDIUM))
low_imgs = list(images.resize(Images.Resolution.LOW))
final_imgs = list(images.resize(Images.Resolution.FINAL))


finder = FeatureDetector()
features = [finder.detect_features(img) for img in medium_imgs]


matcher = cv2.detail_BestOf2NearestMatcher()
matches = []
range_width = -1
for i in range(len(features)):
    for j in range(len(features)):
        if 0 < j-i <= range_width:
            match = matcher.apply(features[i], features[j])
            match.src_img_idx = i
            match.dst_img_idx = j
            matches.append(match)
        elif 0 < i-j <= range_width:
            match_to_copy = matches[j * len(features) + i]
            
            match = cv2.detail.MatchesInfo()

            # swap src and dst, invert homography
            match.src_img_idx = match_to_copy.dst_img_idx
            match.dst_img_idx = match_to_copy.src_img_idx
            match.H = np.linalg.inv(match_to_copy.H)

            match.inliers_mask = match_to_copy.inliers_mask
            match.num_inliers = match.num_inliers
            match.confidence = match_to_copy.confidence

            dmatches = []
            for dmatch_to_copy in match_to_copy.matches:
                dmatch = cv2.DMatch()
                dmatch.distance = dmatch_to_copy.distance
                dmatch.imgIdx = dmatch_to_copy.imgIdx
                # swap queryIdx and trainIdx
                dmatch.queryIdx = dmatch_to_copy.trainIdx
                dmatch.trainIdx = dmatch_to_copy.queryIdx
                
                dmatches.append(dmatch)

            match.matches = dmatches

            matches.append(match)
        else:
            matches.append(cv2.detail.MatchesInfo())


subsetter = Subsetter()

dot_notation = subsetter.get_matches_graph(images.names, matches)


indices = subsetter.get_indices_to_keep(features, matches)


medium_imgs = subsetter.subset_list(medium_imgs, indices)
low_imgs = subsetter.subset_list(low_imgs, indices)
final_imgs = subsetter.subset_list(final_imgs, indices)
features = subsetter.subset_list(features, indices)
matches = subsetter.subset_matches(matches, indices)


camera_estimator = CameraEstimator(estimator='homography')
camera_adjuster = CameraAdjuster(adjuster='ray', refinement_mask='xxxxx')
wave_corrector = WaveCorrector(wave_correct_kind='horiz')

cameras = camera_estimator.estimate(features, matches)
cameras = camera_adjuster.adjust(features, matches, cameras)
cameras = wave_corrector.correct(cameras)


warper = Warper(warper_type='spherical')
warper.set_scale(cameras)

low_sizes = images.get_scaled_img_sizes(Images.Resolution.LOW)
camera_aspect = images.get_ratio(Images.Resolution.MEDIUM, Images.Resolution.LOW)  # since cameras were obtained on medium imgs

warped_low_imgs = list(warper.warp_images(low_imgs, cameras, camera_aspect))
warped_low_masks = list(warper.create_and_warp_masks(low_sizes, cameras, camera_aspect))
low_corners, low_sizes = warper.warp_rois(low_sizes, cameras, camera_aspect)

final_sizes = images.get_scaled_img_sizes(Images.Resolution.FINAL)
camera_aspect = images.get_ratio(Images.Resolution.MEDIUM, Images.Resolution.FINAL)

warped_final_imgs = list(warper.warp_images(final_imgs, cameras, camera_aspect))
warped_final_masks = list(warper.create_and_warp_masks(final_sizes, cameras, camera_aspect))
final_corners, final_sizes = warper.warp_rois(final_sizes, cameras, camera_aspect)



cropper = Cropper(crop=False)
mask = cropper.estimate_panorama_mask(warped_low_imgs, warped_low_masks, low_corners, low_sizes)
lir = cropper.estimate_largest_interior_rectangle(mask)

low_corners = cropper.get_zero_center_corners(low_corners)
rectangles = cropper.get_rectangles(low_corners, low_sizes)




cropper.prepare(warped_low_imgs, warped_low_masks, low_corners, low_sizes)

cropped_low_masks = list(cropper.crop_images(warped_low_masks))
cropped_low_imgs = list(cropper.crop_images(warped_low_imgs))
low_corners, low_sizes = cropper.crop_rois(low_corners, low_sizes)

lir_aspect = images.get_ratio(Images.Resolution.LOW, Images.Resolution.FINAL)  # since lir was obtained on low imgs
cropped_final_masks = list(cropper.crop_images(warped_final_masks, lir_aspect))
cropped_final_imgs = list(cropper.crop_images(warped_final_imgs, lir_aspect))
final_corners, final_sizes = cropper.crop_rois(final_corners, final_sizes, lir_aspect)


seam_finder = SeamFinder(finder='dp_color')
seam_masks = seam_finder.find(cropped_low_imgs, low_corners, cropped_low_masks)
seam_masks = [seam_finder.resize(seam_mask, mask) for seam_mask, mask in zip(seam_masks, cropped_final_masks)]


compensator = ExposureErrorCompensator()

compensator.feed(low_corners, cropped_low_imgs, cropped_low_masks)

compensated_imgs = [compensator.apply(idx, corner, img, mask) 
                    for idx, (img, mask, corner) 
                    in enumerate(zip(cropped_final_imgs, cropped_final_masks, final_corners))]


blender = Blender(blender_type='multiband', blend_strength=5)

blender.prepare(final_corners, final_sizes)
for img, mask, corner in zip(compensated_imgs, seam_masks, final_corners):
    blender.feed(img, mask, corner)
panorama, _ = blender.blend()

cv2.imwrite("test.jpg", panorama)

The code above equal to simply code below:

paths = glob.glob("my_data/*.jpg")
panorama = stitcher.stitch(paths)
cv2.imwrite(f"tree/final.jpg", panorama)

With 2 difference way:
First I get:

Second I get:

image links

As a result show that the second methods very good, but some case not good, you need define the graph. That is why I ask the question and prefer first method.

I am very happy for your supporting

[phamkhactu]

I total agree that images <https://github.com/c060604/panorama/tree/master/pics> not contain fully for creating 360 image (only 24images), I run separately ``` [a1.jpg,a2.jpg, b1.jpg, b2.jpg, c1.jpg, c2.jpg] .... [a7.jpg,a8.jpg, b7.jpg, b8.jpg, c7.jpg, c8.jpg] ``` I have a total of 4 small parts and then I combine them to stitch again. As you see the result in the first case. That's why I want to define graph matches to bypass to stitch. I think maybe I must modify in here: ``` matcher = cv2.detail_BestOf2NearestMatcher() matches = [] range_width = -1 for i in range(len(features)): for j in range(len(features)): if 0 < j-i <= range_width: match = matcher.apply(features[i], features[j]) match.src_img_idx = i match.dst_img_idx = j matches.append(match) elif 0 < i-j <= range_width: match_to_copy = matches[j * len(features) + i] match = cv2.detail.MatchesInfo() # swap src and dst, invert homography match.src_img_idx = match_to_copy.dst_img_idx match.dst_img_idx = match_to_copy.src_img_idx match.H = np.linalg.inv(match_to_copy.H) match.inliers_mask = match_to_copy.inliers_mask match.num_inliers = match.num_inliers match.confidence = match_to_copy.confidence dmatches = [] for dmatch_to_copy in match_to_copy.matches: dmatch = cv2.DMatch() dmatch.distance = dmatch_to_copy.distance dmatch.imgIdx = dmatch_to_copy.imgIdx # swap queryIdx and trainIdx dmatch.queryIdx = dmatch_to_copy.trainIdx dmatch.trainIdx = dmatch_to_copy.queryIdx dmatches.append(dmatch) match.matches = dmatches matches.append(match) else: matches.append(cv2.detail.MatchesInfo()) ``` I tried with code from scratch, but the result from scratch not good as original you suggestion: ```stitcher.Stitch(images)``` Vào Th 4, 18 thg 10, 2023 vào lúc 00:15 Lukas Weber < ***@***.***> đã viết:

…

[image: 05_timelapse_img1] <https://user-images.githubusercontent.com/32765578/275922133-4937a546-9a65-479e-a19b-7c2e7d4d01f6.jpg> [image: 05_timelapse_img2] <https://user-images.githubusercontent.com/32765578/275922071-d2df074d-3413-4b1b-9d93-62bbfb8e03a9.jpg> [image: 05_timelapse_img3] <https://user-images.githubusercontent.com/32765578/275922120-0abdd29c-f962-4951-854e-a865dfce5470.jpg> If you run a, b and c seperately and try to stitch the results, you can see in the timelapse quite nicely that the image dataset is of poor quality to create a 360 degree panorama for the whole scene. Several parts are missing. — Reply to this email directly, view it on GitHub <#162 (comment)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AKDIB5F7FRLSJ6CHREI7UPDX724LVAVCNFSM6AAAAAA6EFANV6VHI2DSMVQWIX3LMV43SRDJONRXK43TNFXW4Q3PNVWWK3TUHM3TGMBWGU4TS> . You are receiving this because you authored the thread.Message ID: ***@***.***>

[lukasalexanderweber]

I've had another look on the dataset.

I tried to stitch all a*, b* and c* images to get a feeling.

stitching a* the algorithm has a hard time to connect a4 with a5. Maybe artifical features or feature masks can help here.

b* stitches quite well

c* is very bad

So maybe solve the a* problem, stitch it with b* and try to include as many c* as possible to get a fine panorama of the scene

[phamkhactu]

oh, okie, I tried, it works. Thanks