U-Net for Crack Segmentation

I used a U-Net model that was built for Oxford Pet Segmentation to a crack segmentation project. Without transfer learning, model works fine for pet segmentation but not for crack segmentation. What could be the reason?

I know there are codes for Crack Segmentation with U-Net but I want to learn why the code for pet doesn't work well for crack. Thanks in advance.

def double_conv_block(x,n_filters):
    
    x=layers.Conv2D(n_filters,3,padding=same,activation=relu,kernel_initializer=he_normal)(x)
    x=layers.Conv2D(n_filters,3,padding=same,activation=relu,kernel_initializer=he_normal)(x)
    return x
     

def downsample_block(x,n_filters):
    f=double_conv_block(x,n_filters)
    p=layers.MaxPool2D(2)(f)
    p=layers.Dropout(0.3)(p)
    return f,p


def upsample_block(x,conv_features,n_filters):

    x=layers.Conv2DTranspose(n_filters,3,2,padding=same)(x)
    x=layers.concatenate([x,conv_features])
    x=layers.Dropout(0.3)(x)
    x=double_conv_block(x,n_filters)
    return x

    inputs=layers.Input(shape=(128,128,3))
    # encoder: contracting path - downsample
    f1,p1=downsample_block(inputs,64)
    f2,p2=downsample_block(p1,128)
    f3,p3=downsample_block(p2,256)
    f4,p4=downsample_block(p3,512)

  

    bottleneck=double_conv_block(p4,1024)

  # decoder: expanding path - upsample
    u6=upsample_block(bottleneck,f4,512)
    u7=upsample_block(u6,f3,256)
    u8=upsample_block(u7,f2,128)
    u9=upsample_block(u8,f1,64)

    outputs=layers.Conv2D(1,1,padding=same,activation = sigmoid)(u9)