Hi Experts:
We had the issue when compiling keras model on Ubuntu 18.04 with CUDA 11.0, and compiling works with llvm and work all well on Windows[TVM:0.7.0]. Bellow snippet is part of our compiling code:
And bellow log is the error stack:
Traceback (most recent call last):
File "tvm_from_keras.py", line 124, in <module>
params=relay_params)
File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/relay/build_module.py", line 259, in build
graph_json, mod, params = bld_mod.build(mod, target, target_host, params)
File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/relay/build_module.py", line 126, in build
self._build(mod, target, target_host)
File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/_ffi/_ctypes/packed_func.py", line 237, in __call__
raise get_last_ffi_error()
tvm._ffi.base.TVMError: Traceback (most recent call last):
[bt] (8) /usr/local/lib/libtvm.so(tvm::relay::ExprFunctor<std::vector<tvm::relay::backend::GraphNodeRef, std::allocator<tvm::relay::backend::GraphNodeRef> > (tvm::RelayExpr const&)>::InitVTable()::{lambda(tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<std::vector<tvm::relay::backend::GraphNodeRef, std::allocator<tvm::relay::backend::GraphNodeRef> > (tvm::RelayExpr const&)>*)#6}::_FUN(tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<std::vector<tvm::relay::backend::GraphNodeRef, std::allocator<tvm::relay::backend::GraphNodeRef> > (tvm::RelayExpr const&)>*)+0x27) [0x7f2658458f67]
[bt] (7) /usr/local/lib/libtvm.so(tvm::relay::backend::GraphRuntimeCodegen::VisitExpr_(tvm::relay::CallNode const*)+0x10cf) [0x7f265846975f]
[bt] (6) /usr/local/lib/libtvm.so(tvm::relay::backend::GraphRuntimeCodegen::GraphAddCallNode(tvm::relay::CallNode const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xf0) [0x7f2658464ca0]
[bt] (5) /usr/local/lib/libtvm.so(tvm::relay::backend::MemoizedExprTranslator<std::vector<tvm::relay::backend::GraphNodeRef, std::allocator<tvm::relay::backend::GraphNodeRef> > >::VisitExpr(tvm::RelayExpr const&)+0x193) [0x7f265846c593]
[bt] (4) /usr/local/lib/libtvm.so(tvm::relay::ExprFunctor<std::vector<tvm::relay::backend::GraphNodeRef, std::allocator<tvm::relay::backend::GraphNodeRef> > (tvm::RelayExpr const&)>::InitVTable()::{lambda(tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<std::vector<tvm::relay::backend::GraphNodeRef, std::allocator<tvm::relay::backend::GraphNodeRef> > (tvm::RelayExpr const&)>*)#6}::_FUN(tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<std::vector<tvm::relay::backend::GraphNodeRef, std::allocator<tvm::relay::backend::GraphNodeRef> > (tvm::RelayExpr const&)>*)+0x27) [0x7f2658458f67]
[bt] (3) /usr/local/lib/libtvm.so(tvm::relay::backend::GraphRuntimeCodegen::VisitExpr_(tvm::relay::CallNode const*)+0xf15) [0x7f26584695a5]
[bt] (2) /usr/local/lib/libtvm.so(+0xf53ce5) [0x7f265843cce5]
[bt] (1) /usr/local/lib/libtvm.so(tvm::relay::CompileEngineImpl::LowerInternal(tvm::relay::CCacheKey const&)+0xbff) [0x7f265844852f]
[bt] (0) /usr/local/lib/libtvm.so(+0x10f995b) [0x7f26585e295b]
File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/relay/backend/_backend.py", line 49, in lower
f = tvm.driver.lower(sch, inputs, name=func_name)
File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/driver/build_module.py", line 207, in lower
mod = optimize(mod)
File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/ir/transform.py", line 127, in __call__
return _ffi_transform_api.RunPass(self, mod)
File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/_ffi/_ctypes/packed_func.py", line 237, in __call__
raise get_last_ffi_error()
[bt] (8) /usr/local/lib/libtvm.so(tvm::tir::StmtMutator::VisitStmt_(tvm::tir::SeqStmtNode const*)+0x68) [0x7f2657d6b858]
[bt] (7) /usr/local/lib/libtvm.so(void tvm::runtime::Array<tvm::tir::Stmt, void>::MutateByApply<tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}>(tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1})+0xb8) [0x7f2657d704f8]
[bt] (6) /usr/local/lib/libtvm.so(tvm::tir::StmtMutator::VisitStmt(tvm::tir::Stmt const&)+0x137) [0x7f26579d7ce7]
[bt] (5) /usr/local/lib/libtvm.so(tvm::tir::StmtFunctor<tvm::tir::Stmt (tvm::tir::Stmt const&)>::InitVTable()::{lambda(tvm::runtime::ObjectRef const&, tvm::tir::StmtFunctor<tvm::tir::Stmt (tvm::tir::Stmt const&)>*)#2}::_FUN(tvm::runtime::ObjectRef const&, tvm::tir::StmtFunctor<tvm::tir::Stmt (tvm::tir::Stmt const&)>*)+0x27) [0x7f26579c7387]
[bt] (4) /usr/local/lib/libtvm.so(tvm::tir::StorageFlattener::VisitStmt_(tvm::tir::AttrStmtNode const*)+0x52e) [0x7f2657e6829e]
[bt] (3) /usr/local/lib/libtvm.so(tvm::tir::StorageFlattener::HandleBufferBindScope(tvm::tir::AttrStmtNode const*)+0xc86) [0x7f2657e64fd6]
[bt] (2) /usr/local/lib/libtvm.so(tvm::tir::ArgBinder::BindBuffer(tvm::tir::Buffer const&, tvm::tir::Buffer const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x638) [0x7f2657d93028]
[bt] (1) /usr/local/lib/libtvm.so(tvm::tir::ArgBinder::Bind_(tvm::PrimExpr const&, tvm::PrimExpr const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x286) [0x7f2657d91916]
[bt] (0) /usr/local/lib/libtvm.so(+0x8a81f2) [0x7f2657d911f2]
File "/home/d_v_d/apache-tvm-src-v0.7.0.rc0-incubating/src/tir/transforms/arg_binder.cc", line 52
File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/_ffi/_ctypes/packed_func.py", line 81, in cfun
rv = local_pyfunc(*pyargs)
File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/relay/backend/_backend.py", line 57, in lower
raise RuntimeError(msg)
File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/relay/backend/_backend.py", line 49, in lower
f = tvm.driver.lower(sch, inputs, name=func_name)
File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/driver/build_module.py", line 207, in lower
mod = optimize(mod)
File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/ir/transform.py", line 127, in __call__
return _ffi_transform_api.RunPass(self, mod)
File "/usr/local/lib/python3.6/dist-packages/tvm-0.7.0-py3.6-linux-x86_64.egg/tvm/_ffi/_ctypes/packed_func.py", line 237, in __call__
raise get_last_ffi_error()
[bt] (8) /usr/local/lib/libtvm.so(tvm::tir::StmtMutator::VisitStmt_(tvm::tir::SeqStmtNode const*)+0x68) [0x7f2657d6b858]
[bt] (7) /usr/local/lib/libtvm.so(void tvm::runtime::Array<tvm::tir::Stmt, void>::MutateByApply<tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}>(tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1})+0xb8) [0x7f2657d704f8]
[bt] (6) /usr/local/lib/libtvm.so(tvm::tir::StmtMutator::VisitStmt(tvm::tir::Stmt const&)+0x137) [0x7f26579d7ce7]
[bt] (5) /usr/local/lib/libtvm.so(tvm::tir::StmtFunctor<tvm::tir::Stmt (tvm::tir::Stmt const&)>::InitVTable()::{lambda(tvm::runtime::ObjectRef const&, tvm::tir::StmtFunctor<tvm::tir::Stmt (tvm::tir::Stmt const&)>*)#2}::_FUN(tvm::runtime::ObjectRef const&, tvm::tir::StmtFunctor<tvm::tir::Stmt (tvm::tir::Stmt const&)>*)+0x27) [0x7f26579c7387]
[bt] (4) /usr/local/lib/libtvm.so(tvm::tir::StorageFlattener::VisitStmt_(tvm::tir::AttrStmtNode const*)+0x52e) [0x7f2657e6829e]
[bt] (3) /usr/local/lib/libtvm.so(tvm::tir::StorageFlattener::HandleBufferBindScope(tvm::tir::AttrStmtNode const*)+0xc86) [0x7f2657e64fd6]
[bt] (2) /usr/local/lib/libtvm.so(tvm::tir::ArgBinder::BindBuffer(tvm::tir::Buffer const&, tvm::tir::Buffer const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x638) [0x7f2657d93028]
[bt] (1) /usr/local/lib/libtvm.so(tvm::tir::ArgBinder::Bind_(tvm::PrimExpr const&, tvm::PrimExpr const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x286) [0x7f2657d91916]
[bt] (0) /usr/local/lib/libtvm.so(+0x8a81f2) [0x7f2657d911f2]
File "/home/d_v_d/apache-tvm-src-v0.7.0.rc0-incubating/src/tir/transforms/arg_binder.cc", line 52
TVMError: Check failed: arg.dtype() == value.dtype() (int32 vs. int64) :
During handling of the above exception, another exception occurred:
TVMError: Check failed: arg.dtype() == value.dtype() (int32 vs. int64) :
Error during compile function
-----------------------------
#[version = "0.0.5"]
fn (%p0: Tensor[(1, 1280, 4, 4), float32], %p1: Tensor[(64, 1280, 1, 1), float32], %p2: Tensor[(1, 64, 1, 1), float32], Primitive=1) -> Tensor[(1, 64, 4, 4), float32] {
%0 = nn.conv2d(%p0, %p1, padding=[0, 0, 0, 0], channels=64, kernel_size=[1, 1]) /* ty=Tensor[(1, 64, 4, 4), float32] */;
add(%0, %p2) /* ty=Tensor[(1, 64, 4, 4), float32] */
}
Any suggestions would be appreciate!!
Thanks in advance!!