TVMError: Check failed: arg.dtype() == value.dtype() (int32 vs. int64): Error during compile function

Hi Experts:

We had the issue when compiling keras model on Ubuntu 18.04 with CUDA 11.0, and compiling works with llvm and work all well on Windows[TVM:0.7.0]. Bellow snippet is part of our compiling code:

And bellow log is the error stack:

Traceback (most recent call last):
  File "tvm_from_keras.py", line 124, in <module>
    params=relay_params)
  File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/relay/build_module.py", line 259, in build
    graph_json, mod, params = bld_mod.build(mod, target, target_host, params)
  File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/relay/build_module.py", line 126, in build
    self._build(mod, target, target_host)
  File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/_ffi/_ctypes/packed_func.py", line 237, in __call__
    raise get_last_ffi_error()
tvm._ffi.base.TVMError: Traceback (most recent call last):
  [bt] (8) /usr/local/lib/libtvm.so(tvm::relay::ExprFunctor<std::vector<tvm::relay::backend::GraphNodeRef, std::allocator<tvm::relay::backend::GraphNodeRef> > (tvm::RelayExpr const&)>::InitVTable()::{lambda(tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<std::vector<tvm::relay::backend::GraphNodeRef, std::allocator<tvm::relay::backend::GraphNodeRef> > (tvm::RelayExpr const&)>*)#6}::_FUN(tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<std::vector<tvm::relay::backend::GraphNodeRef, std::allocator<tvm::relay::backend::GraphNodeRef> > (tvm::RelayExpr const&)>*)+0x27) [0x7f2658458f67]
  [bt] (7) /usr/local/lib/libtvm.so(tvm::relay::backend::GraphRuntimeCodegen::VisitExpr_(tvm::relay::CallNode const*)+0x10cf) [0x7f265846975f]
  [bt] (6) /usr/local/lib/libtvm.so(tvm::relay::backend::GraphRuntimeCodegen::GraphAddCallNode(tvm::relay::CallNode const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xf0) [0x7f2658464ca0]
  [bt] (5) /usr/local/lib/libtvm.so(tvm::relay::backend::MemoizedExprTranslator<std::vector<tvm::relay::backend::GraphNodeRef, std::allocator<tvm::relay::backend::GraphNodeRef> > >::VisitExpr(tvm::RelayExpr const&)+0x193) [0x7f265846c593]
  [bt] (4) /usr/local/lib/libtvm.so(tvm::relay::ExprFunctor<std::vector<tvm::relay::backend::GraphNodeRef, std::allocator<tvm::relay::backend::GraphNodeRef> > (tvm::RelayExpr const&)>::InitVTable()::{lambda(tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<std::vector<tvm::relay::backend::GraphNodeRef, std::allocator<tvm::relay::backend::GraphNodeRef> > (tvm::RelayExpr const&)>*)#6}::_FUN(tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<std::vector<tvm::relay::backend::GraphNodeRef, std::allocator<tvm::relay::backend::GraphNodeRef> > (tvm::RelayExpr const&)>*)+0x27) [0x7f2658458f67]
  [bt] (3) /usr/local/lib/libtvm.so(tvm::relay::backend::GraphRuntimeCodegen::VisitExpr_(tvm::relay::CallNode const*)+0xf15) [0x7f26584695a5]
  [bt] (2) /usr/local/lib/libtvm.so(+0xf53ce5) [0x7f265843cce5]
  [bt] (1) /usr/local/lib/libtvm.so(tvm::relay::CompileEngineImpl::LowerInternal(tvm::relay::CCacheKey const&)+0xbff) [0x7f265844852f]
  [bt] (0) /usr/local/lib/libtvm.so(+0x10f995b) [0x7f26585e295b]
  File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/relay/backend/_backend.py", line 49, in lower
    f = tvm.driver.lower(sch, inputs, name=func_name)
  File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/driver/build_module.py", line 207, in lower
    mod = optimize(mod)
  File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/ir/transform.py", line 127, in __call__
    return _ffi_transform_api.RunPass(self, mod)
  File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/_ffi/_ctypes/packed_func.py", line 237, in __call__
    raise get_last_ffi_error()
  [bt] (8) /usr/local/lib/libtvm.so(tvm::tir::StmtMutator::VisitStmt_(tvm::tir::SeqStmtNode const*)+0x68) [0x7f2657d6b858]
  [bt] (7) /usr/local/lib/libtvm.so(void tvm::runtime::Array<tvm::tir::Stmt, void>::MutateByApply<tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}>(tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1})+0xb8) [0x7f2657d704f8]
  [bt] (6) /usr/local/lib/libtvm.so(tvm::tir::StmtMutator::VisitStmt(tvm::tir::Stmt const&)+0x137) [0x7f26579d7ce7]
  [bt] (5) /usr/local/lib/libtvm.so(tvm::tir::StmtFunctor<tvm::tir::Stmt (tvm::tir::Stmt const&)>::InitVTable()::{lambda(tvm::runtime::ObjectRef const&, tvm::tir::StmtFunctor<tvm::tir::Stmt (tvm::tir::Stmt const&)>*)#2}::_FUN(tvm::runtime::ObjectRef const&, tvm::tir::StmtFunctor<tvm::tir::Stmt (tvm::tir::Stmt const&)>*)+0x27) [0x7f26579c7387]
  [bt] (4) /usr/local/lib/libtvm.so(tvm::tir::StorageFlattener::VisitStmt_(tvm::tir::AttrStmtNode const*)+0x52e) [0x7f2657e6829e]
  [bt] (3) /usr/local/lib/libtvm.so(tvm::tir::StorageFlattener::HandleBufferBindScope(tvm::tir::AttrStmtNode const*)+0xc86) [0x7f2657e64fd6]
  [bt] (2) /usr/local/lib/libtvm.so(tvm::tir::ArgBinder::BindBuffer(tvm::tir::Buffer const&, tvm::tir::Buffer const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x638) [0x7f2657d93028]
  [bt] (1) /usr/local/lib/libtvm.so(tvm::tir::ArgBinder::Bind_(tvm::PrimExpr const&, tvm::PrimExpr const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x286) [0x7f2657d91916]
  [bt] (0) /usr/local/lib/libtvm.so(+0x8a81f2) [0x7f2657d911f2]
  File "/home/d_v_d/apache-tvm-src-v0.7.0.rc0-incubating/src/tir/transforms/arg_binder.cc", line 52
  File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/_ffi/_ctypes/packed_func.py", line 81, in cfun
    rv = local_pyfunc(*pyargs)
  File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/relay/backend/_backend.py", line 57, in lower
    raise RuntimeError(msg)
  File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/relay/backend/_backend.py", line 49, in lower
    f = tvm.driver.lower(sch, inputs, name=func_name)
  File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/driver/build_module.py", line 207, in lower
    mod = optimize(mod)
  File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/ir/transform.py", line 127, in __call__
    return _ffi_transform_api.RunPass(self, mod)
  File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/_ffi/_ctypes/packed_func.py", line 237, in __call__
    raise get_last_ffi_error()
  [bt] (8) /usr/local/lib/libtvm.so(tvm::tir::StmtMutator::VisitStmt_(tvm::tir::SeqStmtNode const*)+0x68) [0x7f2657d6b858]
  [bt] (7) /usr/local/lib/libtvm.so(void tvm::runtime::Array<tvm::tir::Stmt, void>::MutateByApply<tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}>(tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1})+0xb8) [0x7f2657d704f8]
  [bt] (6) /usr/local/lib/libtvm.so(tvm::tir::StmtMutator::VisitStmt(tvm::tir::Stmt const&)+0x137) [0x7f26579d7ce7]
  [bt] (5) /usr/local/lib/libtvm.so(tvm::tir::StmtFunctor<tvm::tir::Stmt (tvm::tir::Stmt const&)>::InitVTable()::{lambda(tvm::runtime::ObjectRef const&, tvm::tir::StmtFunctor<tvm::tir::Stmt (tvm::tir::Stmt const&)>*)#2}::_FUN(tvm::runtime::ObjectRef const&, tvm::tir::StmtFunctor<tvm::tir::Stmt (tvm::tir::Stmt const&)>*)+0x27) [0x7f26579c7387]
  [bt] (4) /usr/local/lib/libtvm.so(tvm::tir::StorageFlattener::VisitStmt_(tvm::tir::AttrStmtNode const*)+0x52e) [0x7f2657e6829e]
  [bt] (3) /usr/local/lib/libtvm.so(tvm::tir::StorageFlattener::HandleBufferBindScope(tvm::tir::AttrStmtNode const*)+0xc86) [0x7f2657e64fd6]
  [bt] (2) /usr/local/lib/libtvm.so(tvm::tir::ArgBinder::BindBuffer(tvm::tir::Buffer const&, tvm::tir::Buffer const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x638) [0x7f2657d93028]
  [bt] (1) /usr/local/lib/libtvm.so(tvm::tir::ArgBinder::Bind_(tvm::PrimExpr const&, tvm::PrimExpr const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x286) [0x7f2657d91916]
  [bt] (0) /usr/local/lib/libtvm.so(+0x8a81f2) [0x7f2657d911f2]
  File "/home/d_v_d/apache-tvm-src-v0.7.0.rc0-incubating/src/tir/transforms/arg_binder.cc", line 52
TVMError: Check failed: arg.dtype() == value.dtype() (int32 vs. int64) :
During handling of the above exception, another exception occurred:

TVMError: Check failed: arg.dtype() == value.dtype() (int32 vs. int64) :
Error during compile function
-----------------------------
#[version = "0.0.5"]
fn (%p0: Tensor[(1, 1280, 4, 4), float32], %p1: Tensor[(64, 1280, 1, 1), float32], %p2: Tensor[(1, 64, 1, 1), float32], Primitive=1) -> Tensor[(1, 64, 4, 4), float32] {
  %0 = nn.conv2d(%p0, %p1, padding=[0, 0, 0, 0], channels=64, kernel_size=[1, 1]) /* ty=Tensor[(1, 64, 4, 4), float32] */;
  add(%0, %p2) /* ty=Tensor[(1, 64, 4, 4), float32] */
} 

Any suggestions would be appreciate!!

Thanks in advance!!

And the Keras information:

Keras                2.2.5
Keras-Applications   1.0.8
Keras-Preprocessing  1.1.2
keras2onnx           1.7.0
tensorboard          1.14.0
tensorflow           1.14.0
tensorflow-estimator 1.14.0