ViLD running zeroshot evaluation on custom dataset

[Jawing]

INFO:tensorflow:Done running local_init_op.
I0719 11:49:25.955098 139723603752768 session_manager.py:530] Done running local_init_op.
2022-07-19 11:54:59.144640: W tensorflow/core/grappler/optimizers/loop_optimizer.cc:907] Skipping loop optimization for Merge node with control input: generate_detections/nms_detections_3_255/PartitionedCall/cond/branch_executed/_138007
Traceback (most recent call last):
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/client/session.py", line 1377, in _do_call
    return fn(*args)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/client/session.py", line 1360, in _run_fn
    return self._call_tf_sessionrun(options, feed_dict, fetch_list,
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/client/session.py", line 1453, in _call_tf_sessionrun
    return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
tensorflow.python.framework.errors_impl.InvalidArgumentError: 2 root error(s) found.
  (0) INVALID_ARGUMENT: Incompatible shapes: [1,1,1203] vs. [256,1000,6]
         [[{{node frcnn_layer_0/fast_rcnn_head/BroadcastTo_2}}]]
         [[stack_1/_80625]]
  (1) INVALID_ARGUMENT: Incompatible shapes: [1,1,1203] vs. [256,1000,6]
         [[{{node frcnn_layer_0/fast_rcnn_head/BroadcastTo_2}}]]
0 successful operations.
0 derived errors ignored.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/wanjiz/tpu/models/official/detection/main.py", line 184, in <module>
    tf.app.run(main)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/platform/app.py", line 36, in run
    _run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 312, in run
    _run_main(main, args)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 258, in _run_main
    sys.exit(main(argv))
  File "/home/wanjiz/tpu/models/official/detection/main.py", line 160, in main
    executor.evaluate(eval_input_fn, eval_times, ckpt)
  File "/home/wanjiz/tpu/models/official/detection/executor/tpu_executor.py", line 204, in evaluate
    outputs = six.next(predictor)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 642, in predict
    preds_evaluated = mon_sess.run(predictions)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/training/monitored_session.py", line 782, in run
    return self._sess.run(
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/training/monitored_session.py", line 1311, in run
    return self._sess.run(
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/training/monitored_session.py", line 1416, in run
    raise six.reraise(*original_exc_info)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/six.py", line 719, in reraise
    raise value
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/training/monitored_session.py", line 1401, in run
    return self._sess.run(*args, **kwargs)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/training/monitored_session.py", line 1469, in run
    outputs = _WrappedSession.run(
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/training/monitored_session.py", line 1232, in run
    return self._sess.run(*args, **kwargs)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/client/session.py", line 967, in run
    result = self._run(None, fetches, feed_dict, options_ptr,
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/client/session.py", line 1190, in _run
    results = self._do_run(handle, final_targets, final_fetches,
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/client/session.py", line 1370, in _do_run
    return self._do_call(_run_fn, feeds, fetches, targets, options,
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/client/session.py", line 1396, in _do_call
    raise type(e)(node_def, op, message)  # pylint: disable=no-value-for-parameter
tensorflow.python.framework.errors_impl.InvalidArgumentError: Graph execution error:

Detected at node 'frcnn_layer_0/fast_rcnn_head/BroadcastTo_2' defined at (most recent call last):
    File "/home/wanjiz/tpu/models/official/detection/main.py", line 184, in <module>
      tf.app.run(main)
    File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 312, in run
      _run_main(main, args)
    File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 258, in _run_main
      sys.exit(main(argv))
    File "/home/wanjiz/tpu/models/official/detection/main.py", line 160, in main
      executor.evaluate(eval_input_fn, eval_times, ckpt)
    File "/home/wanjiz/tpu/models/official/detection/executor/tpu_executor.py", line 204, in evaluate
      outputs = six.next(predictor)
    File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 623, in predict
      estimator_spec = self._call_model_fn(features, None, ModeKeys.PREDICT,
    File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 1174, in _call_model_fn
      model_fn_results = self._model_fn(features=features, **kwargs)
    File "/home/wanjiz/tpu/models/official/detection/modeling/model_builder.py", line 51, in __call__
      return self._model.predict(features)
    File "/home/wanjiz/tpu/models/official/detection/modeling/base_model.py", line 389, in predict
      outputs = self.build_outputs(images, labels, mode=mode_keys.PREDICT)
    File "/home/wanjiz/tpu/models/official/detection/modeling/base_model.py", line 206, in build_outputs
      outputs = self._build_outputs(images, labels, mode)
    File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_model.py", line 174, in _build_outputs
      distill_class_outputs) = self._frcnn_head_fn(roi_features, is_training)
    File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_head.py", line 325, in __call__
      class_outputs = _divide_no_nan(class_outputs, classifier_norm[None,
    File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_head.py", line 38, in _divide_no_nan
      tf.greater_equal(tf.broadcast_to(y, x.get_shape()), -epsilon),
Node: 'frcnn_layer_0/fast_rcnn_head/BroadcastTo_2'
Detected at node 'frcnn_layer_0/fast_rcnn_head/BroadcastTo_2' defined at (most recent call last):
    File "/home/wanjiz/tpu/models/official/detection/main.py", line 184, in <module>
      tf.app.run(main)
    File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 312, in run
      _run_main(main, args)
    File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 258, in _run_main
      sys.exit(main(argv))
    File "/home/wanjiz/tpu/models/official/detection/main.py", line 160, in main
      executor.evaluate(eval_input_fn, eval_times, ckpt)
    File "/home/wanjiz/tpu/models/official/detection/executor/tpu_executor.py", line 204, in evaluate
      outputs = six.next(predictor)
    File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 623, in predict
      estimator_spec = self._call_model_fn(features, None, ModeKeys.PREDICT,
    File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 1174, in _call_model_fn
      model_fn_results = self._model_fn(features=features, **kwargs)
    File "/home/wanjiz/tpu/models/official/detection/modeling/model_builder.py", line 51, in __call__
      return self._model.predict(features)
    File "/home/wanjiz/tpu/models/official/detection/modeling/base_model.py", line 389, in predict
      outputs = self.build_outputs(images, labels, mode=mode_keys.PREDICT)
    File "/home/wanjiz/tpu/models/official/detection/modeling/base_model.py", line 206, in build_outputs
      outputs = self._build_outputs(images, labels, mode)
    File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_model.py", line 174, in _build_outputs
      distill_class_outputs) = self._frcnn_head_fn(roi_features, is_training)
    File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_head.py", line 325, in __call__
      class_outputs = _divide_no_nan(class_outputs, classifier_norm[None,
    File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_head.py", line 38, in _divide_no_nan
      tf.greater_equal(tf.broadcast_to(y, x.get_shape()), -epsilon),
Node: 'frcnn_layer_0/fast_rcnn_head/BroadcastTo_2'
2 root error(s) found.
  (0) INVALID_ARGUMENT: Incompatible shapes: [1,1,1203] vs. [256,1000,6]
         [[{{node frcnn_layer_0/fast_rcnn_head/BroadcastTo_2}}]]
         [[stack_1/_80625]]
  (1) INVALID_ARGUMENT: Incompatible shapes: [1,1,1203] vs. [256,1000,6]
         [[{{node frcnn_layer_0/fast_rcnn_head/BroadcastTo_2}}]]
0 successful operations.
0 derived errors ignored.

Original stack trace for 'frcnn_layer_0/fast_rcnn_head/BroadcastTo_2':
  File "/home/wanjiz/tpu/models/official/detection/main.py", line 184, in <module>
    tf.app.run(main)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/platform/app.py", line 36, in run
    _run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 312, in run
    _run_main(main, args)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 258, in _run_main
    sys.exit(main(argv))
  File "/home/wanjiz/tpu/models/official/detection/main.py", line 160, in main
    executor.evaluate(eval_input_fn, eval_times, ckpt)
  File "/home/wanjiz/tpu/models/official/detection/executor/tpu_executor.py", line 204, in evaluate
    outputs = six.next(predictor)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 623, in predict
    estimator_spec = self._call_model_fn(features, None, ModeKeys.PREDICT,
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 1174, in _call_model_fn
    model_fn_results = self._model_fn(features=features, **kwargs)
  File "/home/wanjiz/tpu/models/official/detection/modeling/model_builder.py", line 51, in __call__
    return self._model.predict(features)
  File "/home/wanjiz/tpu/models/official/detection/modeling/base_model.py", line 389, in predict
    outputs = self.build_outputs(images, labels, mode=mode_keys.PREDICT)
  File "/home/wanjiz/tpu/models/official/detection/modeling/base_model.py", line 206, in build_outputs
    outputs = self._build_outputs(images, labels, mode)
  File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_model.py", line 174, in _build_outputs
    distill_class_outputs) = self._frcnn_head_fn(roi_features, is_training)
  File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_head.py", line 325, in __call__
    class_outputs = _divide_no_nan(class_outputs, classifier_norm[None,
  File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_head.py", line 38, in _divide_no_nan
    tf.greater_equal(tf.broadcast_to(y, x.get_shape()), -epsilon),
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/ops/gen_array_ops.py", line 876, in broadcast_to
    _, _, _op, _outputs = _op_def_library._apply_op_helper(
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/framework/op_def_library.py", line 797, in _apply_op_helper
    op = g._create_op_internal(op_type_name, inputs, dtypes=None,
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/framework/ops.py", line 3754, in _create_op_internal
    ret = Operation(
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/framework/ops.py", line 2133, in __init__
    self._traceback = tf_stack.extract_stack_for_node(self._c_op)

Have been getting this error when running the inference script with custom class prompt and dataset with the vild_resnet.yaml configs altered as shown below.

anchor:
  anchor_size: 8
  aspect_ratios: [1.0, 2.0, 0.5]
  num_scales: 1
architecture:
  backbone: resnet
  feat_distill_weight: 0.5
  filter_distill_boxes_size: 0
  include_mask: true
  mask_target_size: 28
  max_level: 6
  max_num_rois: 300
  min_level: 2
  multilevel_features: fpn
  normalize_feat_during_training: true
  num_classes: 7
  parser: vild_parser
  pre_parser: null
  space_to_depth_block_size: 1
  use_bfloat16: false
  visual_feature_dim: 512
  visual_feature_distill: vanilla
batch_norm_activation:
  activation: relu
  batch_norm_epsilon: 0.0001
  batch_norm_momentum: 0.997
  batch_norm_trainable: true
  use_sync_bn: true
dropblock:
  dropblock_keep_prob: null
  dropblock_size: null
enable_summary: false
eval:
  eval_batch_size: 8
  eval_dataset_type: tfrecord
  eval_samples: 2956
  eval_timeout: null
  min_eval_interval: 5
  num_steps_per_eval: 1000
  per_category_metrics: true
  skip_eval_loss: false
  suffix: ''
  type: lvis_box
  use_json_file: true
fpn:
  fpn_feat_dims: 256
  use_batch_norm: true
  use_separable_conv: false
frcnn_box_loss:
  huber_loss_delta: 1.0
frcnn_class_loss:
  mask_rare: false
frcnn_head:
  class_agnostic_bbox_pred: true
  clip_dim: 512
  fc_dims: 1024
  normalize_classifier: true
  normalize_visual: true
  num_convs: 4
  num_fcs: 2
  num_filters: 256
  temperature: 100.0
  use_batch_norm: true
  use_separable_conv: false
mask_sampling:
  num_mask_samples_per_image: 128
mrcnn_head:
  class_agnostic_mask_pred: true
  num_convs: 4
  num_filters: 256
  use_batch_norm: true
  use_separable_conv: false
postprocess:
  apply_nms: true
  apply_sigmoid: false
  discard_background: false
  max_total_size: 300
  nms_iou_threshold: 0.5
  nms_version: v1
  pre_nms_num_boxes: 1000
  score_threshold: 0.0
  use_batched_nms: false
predict:
  predict_batch_size: 8
resnet:
  init_drop_connect_rate: null
  resnet_depth: 50
roi_proposal:
  rpn_min_size_threshold: 0.0
  rpn_nms_threshold: 0.7
  rpn_post_nms_top_k: 1000
  rpn_pre_nms_top_k: 2000
  rpn_score_threshold: 0.0
  test_rpn_min_size_threshold: 0.0
  test_rpn_nms_threshold: 0.7
  test_rpn_post_nms_top_k: 1000
  test_rpn_pre_nms_top_k: 1000
  test_rpn_score_threshold: 0.0
  use_batched_nms: false
roi_sampling:
  bg_iou_thresh_hi: 0.5
  bg_iou_thresh_lo: 0.0
  cascade_iou_thresholds: null
  fg_fraction: 0.25
  fg_iou_thresh: 0.5
  mix_gt_boxes: true
  num_samples_per_image: 512
rpn_box_loss:
  huber_loss_delta: 0.1111111111111111
rpn_head:
  anchors_per_location: null
  cast_to_float32: true
  num_convs: 2
  num_filters: 256
  use_batch_norm: true
  use_separable_conv: false
rpn_score_loss:
  rpn_batch_size_per_im: 256
train:
  checkpoint:
    path: ''
    prefix: ''
    skip_variables_regex: ''
  frozen_variable_prefix: frcnn_layer_0/fast_rcnn_head/class-predict
  gradient_clip_norm: 0.0
  input_partition_dims: null
  iterations_per_loop: 100
  l2_weight_decay: 4.0e-05
  learning_rate:
    init_learning_rate: 0.32
    learning_rate_levels: [0.032, 0.0032]
    learning_rate_steps: [162000, 171000, 175500]
    type: step
    warmup_learning_rate: 0.0032
    warmup_steps: 1000
  losses: all
  num_cores_per_replica: null
  optimizer:
    momentum: 0.9
    type: momentum
  pre_parser_dataset:
    dataset_type: tfrecord
    file_pattern: ''
  regularization_variable_regex: .*(kernel|weight):0$
  space_to_depth_block_size: 1
  total_steps: 180000
  train_batch_size: 256
  train_dataset_type: tfrecord
  transpose_input: true
type: vild
use_tpu: false
vild_parser:
  aug_rand_hflip: true
  aug_scale_max: 2.0
  aug_scale_min: 0.1
  copy_paste: false
  mask_crop_size: 112
  max_num_instances: 300
  output_size: [1024, 1024]
  regenerate_source_id: false
  rpn_batch_size_per_im: 256
  rpn_fg_fraction: 0.5
  rpn_match_threshold: 0.7
  rpn_unmatched_threshold: 0.3
  skip_crowd_during_training: true

Also I've set frcnn_class_loss.mask_rare = false

since I kept on getting this error. every directory is structure the same way as it is mentioned here

Traceback (most recent call last):
  File "/home/wanjiz/tpu/models/official/detection/main.py", line 184, in <module>
    tf.app.run(main)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/platform/app.py", line 36, in run
    _run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 312, in run
    _run_main(main, args)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 258, in _run_main
    sys.exit(main(argv))
  File "/home/wanjiz/tpu/models/official/detection/main.py", line 116, in main
    model_fn = model_builder.ModelFn(params)
  File "/home/wanjiz/tpu/models/official/detection/modeling/model_builder.py", line 26, in __init__
    self._model = factory.model_generator(params)
  File "/home/wanjiz/tpu/models/official/detection/modeling/factory.py", line 41, in model_generator
    model_fn = vild_model.ViLDModel(params)
  File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_model.py", line 73, in __init__
    self._frcnn_class_loss_fn = vild_losses.FastrcnnClassLoss(
  File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_losses.py", line 33, in __init__
    self._rare_mask = np.array(np.load(f), dtype=np.float32)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/numpy/lib/npyio.py", line 414, in load
    magic = fid.read(N)
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/lib/io/file_io.py", line 114, in read
    self._preread_check()
  File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/lib/io/file_io.py", line 76, in _preread_check
    self._read_buf = _pywrap_file_io.BufferedInputStream(
tensorflow.python.framework.errors_impl.NotFoundError: ; No such file or directory

Please share some insight on how to evaluate ViLD on custom dataset. Thanks!