SAM 是Mata發(fā)布的“Segment Anything Model”可以準確識別和提取圖像中的對象。 它可以分割任何的圖片，但是如果需要分割特定的物體，則需要需要點、框的特定提示才能準確分割圖像。 所以本文將介紹一種稱為 Grounding Dino 的技術(shù)來自動生成 SAM 進行分割所需的框。
除了分割以外，我們還可以通過將 SAM 與 Grounding Dino 和 Stable Diffusion 相結(jié)合，獲得高度準確圖像分割結(jié)果，并且對分割后的圖像進行細微的更改。
下面就是我們需要的所有的包：
`%cd /content
!git clone https://github.com/IDEA-Research/Grounded-Segment-Anything
%cd /content/Grounded-Segment-Anything
!pip install -q-r requirements.txt
%cd /content/Grounded-Segment-Anything/GroundingDINO
!pip install -q .
%cd /content/Grounded-Segment-Anything/segment_anything
!pip install -q .
%cd /content/Grounded-Segment-Anything
導入必要的包：
importos, sys
sys.path.append(os.path.join(os.getcwd(), "GroundingDINO"))
importargparse
importcopy
fromIPython.displayimportdisplay
fromPILimportImage, ImageDraw, ImageFont
fromtorchvision.opsimportbox_convert
# Grounding DINO
importGroundingDINO.groundingdino.datasets.transformsasT
fromGroundingDINO.groundingdino.modelsimportbuild_model
fromGroundingDINO.groundingdino.utilimportbox_ops
fromGroundingDINO.groundingdino.util.slconfigimportSLConfig
fromGroundingDINO.groundingdino.util.utilsimportclean_state_dict, get_phrases_from_posmap
fromGroundingDINO.groundingdino.util.inferenceimportannotate, load_image, predict
importsupervisionassv
# segment anything
fromsegment_anythingimportbuild_sam, SamPredictor
importcv2
importnumpyasnp
importmatplotlib.pyplotasplt
# diffusers
importPIL
importrequests
importtorch
fromioimportBytesIO
fromdiffusersimportStableDiffusionInpaintPipeline
fromhuggingface_hubimporthf_hub_download
然后我們設置處理的設備：
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
然后我們創(chuàng)建一個 GroundingDino 模型的實例。
defload_model_hf(repo_id, filename, ckpt_config_filename, device='cpu'):
cache_config_file=hf_hub_download(repo_id=repo_id, filename=ckpt_config_filename)
args=SLConfig.fromfile(cache_config_file)
args.device=device
model=build_model(args)
cache_file=hf_hub_download(repo_id=repo_id, filename=filename)
checkpoint=torch.load(cache_file, map_location=device)
log=model.load_state_dict(clean_state_dict(checkpoint['model']), strict=False)
print("Model loaded from {} \n => {}".format(cache_file, log))
_=model.eval()
returnmodel
ckpt_repo_id="ShilongLiu/GroundingDINO"
ckpt_filenmae="groundingdino_swinb_cogcoor.pth"
ckpt_config_filename="GroundingDINO_SwinB.cfg.py"
groundingdino_model=load_model_hf(ckpt_repo_id, ckpt_filenmae, ckpt_config_filename, device)
下面開始創(chuàng)建SAM 模型，定義模型并創(chuàng)建一個實例。
! wget https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth
sam_checkpoint ='sam_vit_h_4b8939.pth'
sam_predictor = SamPredictor(build_sam(checkpoint=sam_checkpoint).to(device))
這里我們使用與訓練的 vit_h 模型，下面就是擴散模型了：
sd_pipe=StableDiffusionInpaintPipeline.from_pretrained(
"stabilityai/stable-diffusion-2-inpainting",
torch_dtype=torch.float16,
).to(device)
然后我們開始測試：
# Load image
defdownload_image(url, image_file_path):
r=requests.get(url, timeout=4.0)
ifr.status_code!=requests.codes.ok:
assertFalse, 'Status code error: {}.'.format(r.status_code)
withImage.open(BytesIO(r.content)) asim:
im.save(image_file_path)
print('Image downloaded from url: {} and saved to: {}.'.format(url, image_file_path))
local_image_path="assets/inpaint_demo.jpg"
image_url="https://images.rawpixel.com/image_800/cHJpdmF0ZS9sci9pbWFnZXMvd2Vic2l0ZS8yMDIyLTA1L3Vwd2s2MTc3Nzk0MS13aWtpbWVkaWEtaW1hZ2Uta293YnN1MHYuanBn.jpg"
download_image(image_url, local_image_path)
image_source, image=load_image(local_image_path)
Image.fromarray(image_source)
先使用Grounding Dino 進行檢測：
# detect object using grounding DINO
defdetect(image, text_prompt, model, box_threshold=0.3, text_threshold=0.25):
boxes, logits, phrases=predict(
model=model,
image=image,
caption=text_prompt,
box_threshold=box_threshold,
text_threshold=text_threshold
)
annotated_frame=annotate(image_source=image_source, boxes=boxes, logits=logits, phrases=phrases)
annotated_frame=annotated_frame[...,::-1] # BGR to RGB
returnannotated_frame, boxes
annotated_frame, detected_boxes=detect(image, text_prompt="bench", model=groundingdino_model)
Image.fromarray(annotated_frame)
讓我們看看結(jié)果：
然后使用 SAM 分割這個狐貍：
defsegment(image, sam_model, boxes):
sam_model.set_image(image)
H, W, _=image.shape
boxes_xyxy=box_ops.box_cxcywh_to_xyxy(boxes) *torch.Tensor([W, H, W, H])
transformed_boxes=sam_model.transform.apply_boxes_torch(boxes_xyxy.to(device), image.shape[:2])
masks, _, _=sam_model.predict_torch(
point_coords=None,
point_labels=None,
boxes=transformed_boxes,
multimask_output=False,
)
returnmasks.cpu()
defdraw_mask(mask, image, random_color=True):
ifrandom_color:
color=np.concatenate([np.random.random(3), np.array([0.8])], axis=0)
else:
color=np.array([30/255, 144/255, 255/255, 0.6])
h, w=mask.shape[-2:]
mask_image=mask.reshape(h, w, 1) *color.reshape(1, 1, -1)
annotated_frame_pil=Image.fromarray(image).convert("RGBA")
mask_image_pil=Image.fromarray((mask_image.cpu().numpy() *255).astype(np.uint8)).convert("RGBA")
returnnp.array(Image.alpha_composite(annotated_frame_pil, mask_image_pil))
segmented_frame_masks=segment(image_source, sam_predictor, boxes=detected_boxes)
annotated_frame_with_mask=draw_mask(segmented_frame_masks[0][0], annotated_frame)
Image.fromarray(annotated_frame_with_mask)
這樣就可以通過上面的分割結(jié)果為的擴散模型生成掩碼：
# create mask images
mask=segmented_frame_masks[0][0].cpu().numpy()
inverted_mask= ((1-mask) *255).astype(np.uint8)
image_source_pil=Image.fromarray(image_source)
image_mask_pil=Image.fromarray(mask)
inverted_image_mask_pil=Image.fromarray(inverted_mask)
display(*[image_source_pil, image_mask_pil, inverted_image_mask_pil])
繪時我們還需要一個背景的掩碼，這個就是上面掩碼的反操作
defgenerate_image(image, mask, prompt, negative_prompt, pipe, seed):
# resize for inpainting
w, h=image.size
in_image=image.resize((512, 512))
in_mask=mask.resize((512, 512))
generator=torch.Generator(device).manual_seed(seed)
result=pipe(image=in_image, mask_image=in_mask, prompt=prompt, negative_prompt=negative_prompt, generator=generator)
result=result.images[0]
returnresult.resize((w, h))
然后我們可以開始改圖，輸入一個提示：
prompt=" a brown bulldog"
negative_prompt="low resolution, ugly"
seed=-1# for reproducibility
generated_image=generate_image(image=image_source_pil, mask=image_mask_pil, prompt=prompt, negative_prompt=negative_prompt, pipe=sd_pipe, seed=seed)
generated_image
或者用上面的背景掩碼來修改背景：
prompt="a hill with grasses ,weak sunlight "
negative_prompt="people, low resolution, ugly"
seed=32# for reproducibility
generated_image=generate_image(image_source_pil, inverted_image_mask_pil, prompt, negative_prompt, sd_pipe, seed)
generated_image
可以看到效果還是很好的
SAM、Grounding Dino 和 Stable Diffusion 的組合為我們提供了強大的工具。這些技術(shù)為探索令人興奮的圖像處理世界提供了堅實的基礎 并為藝術(shù)家和開發(fā)者提供巨大的創(chuàng)造潛力。
如果你想在線測試，這里有完整的源代碼：
https://avoid.overfit.cn/post/e9e083807a434935910c8116c85c8375文章來源：http://www.zghlxwxcb.cn/news/detail-437379.html
作者：Amir Shakiba文章來源地址http://www.zghlxwxcb.cn/news/detail-437379.html
到了這里，關于使用Dino+SAM+Stable diffusion 自動進行圖片的修改的文章就介紹完了。如果您還想了解更多內(nèi)容，請在右上角搜索TOY模板網(wǎng)以前的文章或繼續(xù)瀏覽下面的相關文章，希望大家以后多多支持TOY模板網(wǎng)！
