My tensorflow saved module has multiple input, and each input has batch shape, such as (None, 6, 50), when I load frozen saved module to TVM relay, I need to specify the batch to a constant such as 1, and then I can only send 1 batch data to run the model in TVM。
def get_graph_def_from_saved_model(model_path):
model = tf.keras.models.load_model(model_path)
full_model = tf.function(lambda x: model(x))
full_model = full_model.get_concrete_function(
(tf.TensorSpec(shape=[1, 587], dtype=model.inputs[0].dtype, name="dense_input"),
tf.TensorSpec(shape=[1, 53], dtype=model.inputs[1].dtype, name="sparse_ids_input"),
tf.TensorSpec(shape=[1, 53], dtype=model.inputs[2].dtype, name="sparse_wgt_input"),
tf.TensorSpec(shape=[1, 6, 50], dtype=model.inputs[3].dtype, name="seq_50_input"),
),
)
frozen_func = convert_variables_to_constants_v2(full_model)
return frozen_func.graph.as_graph_def()
# input for tvm
model_params = {
"dense_input": tf.fill([1, 587], 1.0),
"sparse_ids_input": tf.fill([1, 53], 1),
"sparse_wgt_input": tf.fill([1, 53], 1.0),
"seq_50_input": tf.fill([1, 6, 50], 1),
}
module.set_input(**model_params )
module.run()
tvm_output = module.get_output(0)
Current online infer service receive differerent batch input data, how can I infer by dynamic batch input data in TVM ?
Or is there any other suggestion?