how to apply segmentation on objects only

i have this image which is an output from my object detection model

i wanted to apply segmentation on this image so that my mask will be like that

i used grabcut algorithm but the results was too bad here's my code

img=cv2.imread(testpath+imgname)
    mask=np.zeros(img.shape[:2],np.uint8)

    bgModel=np.zeros((1,65),np.float64)
    fgModel=np.zeros((1,65),np.float64)
    tmpimage=image
    masks=[]
    for i in recs:
        cv2.grabCut(img,mask,i,bgModel,fgModel,5,cv2.GC_INIT_WITH_RECT)
        mask2=np.where((mask==2)|(mask==0),0,255).astype('uint8')
        masks.append(mask2)
        #img=image*mask2[:,:,np.newaxis]
    finalmask=np.zeros(img.shape[:2],np.uint8)
    for i in range(len(masks)):
       finalmask=finalmask+masks[i]



    # for i in range(len(finalmask)):
    #     for j in range(len(finalmask[i][:])):
    #         for k in recs:
    #             if ik[0] or ik[2]:
    #                 finalmask[i][j]=0
    #


    #plt.imshow(finalmask)
    colormask = np.zeros(img.shape, np.uint8)

    for i in range(len(masks)):
        # generating random color


        #color = tuple(np.random.choice(range(256), size=3))
        color=dic[segclasses[i]]

        # create another mask to place color in white regions
        im = np.zeros(img.shape, np.uint8)

        im[masks[i] == 255] = color

        # add with final mask

        colormask = colormask + im
    #plt.imshow(colormask)
    #plt.savefig('masks/figure1.png')
    cv2.imwrite(outpath+imgname,colormask)

# plt.colorbar()
#plt.xticks([]),plt.yticks([])
#plt.show()
#cv2.imwrite('final.jpg',finalmask)
    c=c+1

print('Done')

if i tested on this image

it gives me this mask

which is not what i wanted so what seem to be the problem here