when i use the vm.compile to compile a relay Module with some dynamic ops like argwhere i meet this error.
I have no idea about the reason for this error.
Is anyone meet this error before? Or how can i debug this error.
Please help me!Thanks very much!!
INFO 2022-08-30 09:15:32,082 te_compiler.py:255] Using injective.cuda for divide based on highest priority (10)
INFO 2022-08-30 09:15:32,137 te_compiler.py:255] Using topk.cuda for topk based on highest priority (10)
[09:15:32] /root/tvmnew/apache-tvm-src-v0.9.0/src/te/schedule/bound.cc:119: not in feed graph consumer = extern(topk_gpu, 0x561848ff3610)
[09:15:32] /root/tvmnew/apache-tvm-src-v0.9.0/src/te/schedule/bound.cc:119: not in feed graph consumer = extern(topk_gpu, 0x561848ff3610)
[09:15:32] /root/tvmnew/apache-tvm-src-v0.9.0/src/te/schedule/bound.cc:119: not in feed graph consumer = extern(topk_gpu, 0x561848ff3610)
INFO 2022-08-30 09:15:32,214 te_compiler.py:255] Using injective.cuda for cast based on highest priority (10)
INFO 2022-08-30 09:15:32,224 te_compiler.py:255] Using injective.cuda for equal based on highest priority (10)
INFO 2022-08-30 09:15:32,229 te_compiler.py:188] Using argwhere.cuda for argwhere based on highest priority (10)
/opt/conda/lib/python3.9/ast.py:407: DeprecationWarning: visit_Num is deprecated; add visit_Constant
return visitor(node)
/opt/conda/lib/python3.9/ast.py:407: DeprecationWarning: visit_Str is deprecated; add visit_Constant
return visitor(node)
Traceback (most recent call last):
File "/root/ernie/moe/paddle_tvm_vm_build.py", line 152, in <module>
exe = vm.compile(mod, target=target, params=params)
File "/root/tvmnew/apache-tvm-src-v0.9.0/python/tvm/relay/backend/vm.py", line 67, in compile
compiler.lower(mod, target, target_host)
File "/root/tvmnew/apache-tvm-src-v0.9.0/python/tvm/relay/backend/vm.py", line 126, in lower
self._lower(mod, raw_targets)
File "/root/tvmnew/apache-tvm-src-v0.9.0/python/tvm/_ffi/_ctypes/packed_func.py", line 237, in __call__
raise get_last_ffi_error()
tvm._ffi.base.TVMError: Traceback (most recent call last):
48: TVMFuncCall
47: tvm::runtime::PackedFuncObj::Extractor<tvm::runtime::PackedFuncSubObj<tvm::relay::vm::VMCompiler::GetFunction(std::string const&, tvm::runtime::ObjectPtr<tvm::runtime::Object> const&)::{lambda(tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*)#1}> >::Call(tvm::runtime::PackedFuncObj const*, tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*)
46: tvm::relay::vm::VMCompiler::Lower(tvm::IRModule, tvm::runtime::Array<tvm::Target, void> const&)
45: tvm::relay::vm::VMCompiler::LowerImpl(tvm::IRModule)
44: tvm::relay::vm::VMCompiler::OptimizeModuleImpl(tvm::IRModule)
43: tvm::transform::Pass::operator()(tvm::IRModule) const
42: tvm::transform::Pass::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
41: tvm::transform::SequentialNode::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
40: tvm::transform::Pass::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
39: tvm::transform::SequentialNode::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
38: tvm::transform::Pass::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
37: tvm::transform::ModulePassNode::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
36: tvm::runtime::PackedFuncObj::Extractor<tvm::runtime::PackedFuncSubObj<tvm::runtime::TypedPackedFunc<tvm::IRModule (tvm::IRModule, tvm::transform::PassContext)>::AssignTypedLambda<tvm::relay::tec::LowerTE(tvm::runtime::String, tvm::CompilationConfig, std::function<void (tvm::BaseFunc)>)::{lambda(tvm::IRModule, tvm::transform::PassContext)#1}>(tvm::relay::tec::LowerTE(tvm::runtime::String, tvm::CompilationConfig, std::function<void (tvm::BaseFunc)>)::{lambda(tvm::IRModule, tvm::transform::PassContext)#1})::{lambda(tvm::runtime::TVMArgs const&, tvm::runtime::TVMRetValue*)#1}> >::Call(tvm::runtime::PackedFuncObj const*, tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*)
35: tvm::relay::tec::LowerTE(tvm::IRModule const&, tvm::runtime::String const&, std::function<void (tvm::BaseFunc)>, tvm::CompilationConfig)
34: tvm::transform::Pass::operator()(tvm::IRModule) const
33: tvm::transform::Pass::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
32: tvm::relay::transform::FunctionPassNode::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
31: tvm::runtime::PackedFuncObj::Extractor<tvm::runtime::PackedFuncSubObj<tvm::runtime::TypedPackedFunc<tvm::relay::Function (tvm::relay::Function, tvm::IRModule, tvm::transform::PassContext)>::AssignTypedLambda<tvm::relay::tec::LowerTensorExpr(tvm::runtime::String const&, tvm::relay::tec::TECompiler, std::function<void (tvm::BaseFunc)>, tvm::CompilationConfig)::{lambda(tvm::relay::Function, tvm::IRModule, tvm::transform::PassContext)#1}>(tvm::relay::tec::LowerTensorExpr(tvm::runtime::String const&, tvm::relay::tec::TECompiler, std::function<void (tvm::BaseFunc)>, tvm::CompilationConfig)::{lambda(tvm::relay::Function, tvm::IRModule, tvm::transform::PassContext)#1})::{lambda(tvm::runtime::TVMArgs const&, tvm::runtime::TVMRetValue*)#1}> >::Call(tvm::runtime::PackedFuncObj const*, tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*)
30: tvm::relay::ExprMutator::VisitExpr(tvm::RelayExpr const&)
29: tvm::relay::ExprFunctor<tvm::RelayExpr (tvm::RelayExpr const&)>::VisitExpr(tvm::RelayExpr const&)
28: _ZZN3tvm5relay11ExprFunc
27: tvm::relay::transform::DeviceAwareExprMutator::VisitExpr_(tvm::relay::FunctionNode const*)
26: tvm::relay::tec::LowerTensorExprMutator::DeviceAwareVisitExpr_(tvm::relay::FunctionNode const*)
25: _ZN3tvm5relay9
24: tvm::relay::ExprMutator::VisitExpr_(tvm::relay::FunctionNode const*)
23: tvm::relay::ExprMutator::VisitExpr(tvm::RelayExpr const&)
22: tvm::relay::ExprFunctor<tvm::RelayExpr (tvm::RelayExpr const&)>::VisitExpr(tvm::RelayExpr const&)
21: _ZZN3tvm5relay11ExprFunc
20: tvm::relay::transform::DeviceAwareExprMutator::VisitExpr_(tvm::relay::LetNode const*)
19: tvm::relay::tec::LowerTensorExprMutator::PreVisitLetBinding_(tvm::relay::Var const&, tvm::RelayExpr const&)
18: tvm::relay::ExprMutator::VisitExpr(tvm::RelayExpr const&)
17: tvm::relay::ExprFunctor<tvm::RelayExpr (tvm::RelayExpr const&)>::VisitExpr(tvm::RelayExpr const&)
16: _ZZN3tvm5relay11ExprFunc
15: tvm::relay::transform::DeviceAwareExprMutator::VisitExpr_(tvm::relay::CallNode const*)
14: tvm::relay::ExprMutator::VisitExpr(tvm::RelayExpr const&)
13: tvm::relay::ExprFunctor<tvm::RelayExpr (tvm::RelayExpr const&)>::VisitExpr(tvm::RelayExpr const&)
12: _ZZN3tvm5relay11ExprFunc
11: tvm::relay::transform::DeviceAwareExprMutator::VisitExpr_(tvm::relay::CallNode const*)
10: tvm::relay::tec::LowerTensorExprMutator::DeviceAwareVisitExpr_(tvm::relay::CallNode const*)
9: tvm::relay::tec::LowerTensorExprMutator::MakeLoweredCall(tvm::BaseFunc const&, tvm::GlobalVar const&, tvm::runtime::Array<tvm::RelayExpr, void>, tvm::Span, tvm::Target const&, tvm::runtime::Map<tvm::GlobalVar, tvm::BaseFunc, void, void> const&)
8: tvm::relay::tec::TECompilerImpl::LowerShapeFunc(tvm::relay::tec::CCacheKey const&)
7: tvm::relay::tec::TECompilerImpl::LowerShapeFuncInternal(tvm::relay::tec::CCacheKey const&)
6: tvm::relay::tec::ShapeFuncFor(tvm::relay::Function const&, tvm::Target const&, std::function<std::string (std::string)>)
5: tvm::relay::tec::MakeShapeFunc::Create(tvm::relay::Function const&, tvm::Target const&, std::function<std::string (std::string)>)
4: tvm::relay::tec::MakeShapeFunc::VisitExpr(tvm::RelayExpr const&)
3: tvm::relay::backend::MemoizedExprTranslator<tvm::runtime::Array<tvm::te::Tensor, void> >::VisitExpr(tvm::RelayExpr const&)
2: _ZZN3tvm5relay11ExprFunc
1: tvm::relay::tec::MakeShapeFunc::VisitExpr_(tvm::relay::CallNode const*)
0: tvm::runtime::PackedFuncObj::Extractor<tvm::runtime::PackedFuncSubObj<TVMFuncCreateFromCFunc::{lambda(tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*)#2}> >::Call(tvm::runtime::PackedFuncObj const*, tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*)
File "/root/tvmnew/apache-tvm-src-v0.9.0/python/tvm/_ffi/_ctypes/packed_func.py", line 81, in cfun
rv = local_pyfunc(*pyargs)
File "/root/tvmnew/apache-tvm-src-v0.9.0/python/tvm/relay/op/_transform.py", line 670, in argwhere_shape_func
return [_argwhere_shape_func_3d(inputs[0])]
File "/opt/conda/lib/python3.9/site-packages/decorator.py", line 232, in fun
return caller(func, *(extras + args), **kw)
File "/root/tvmnew/apache-tvm-src-v0.9.0/python/tvm/te/hybrid/__init__.py", line 60, in wrapped_func
return source_to_op(src, args, func.__globals__, closure_vars)
File "/root/tvmnew/apache-tvm-src-v0.9.0/python/tvm/te/hybrid/parser.py", line 640, in source_to_op
parser = parse_python(src, args, symbols, closure_vars)
File "/root/tvmnew/apache-tvm-src-v0.9.0/python/tvm/te/hybrid/parser.py", line 610, in parse_python
parser.parsed_body = parser.visit(root)
File "/opt/conda/lib/python3.9/ast.py", line 407, in visit
return visitor(node)
File "/root/tvmnew/apache-tvm-src-v0.9.0/python/tvm/te/hybrid/parser.py", line 229, in visit_Module
return self.visit(node.body[0])
File "/opt/conda/lib/python3.9/ast.py", line 407, in visit
return visitor(node)
File "/root/tvmnew/apache-tvm-src-v0.9.0/python/tvm/te/hybrid/parser.py", line 242, in visit_FunctionDef
res = visit_list_to_block(self.visit, node.body)
File "/root/tvmnew/apache-tvm-src-v0.9.0/python/tvm/te/hybrid/parser.py", line 58, in visit_list_to_block
lst = [visit(stmt) for stmt in lst if not utils.is_docstring(stmt)]
File "/root/tvmnew/apache-tvm-src-v0.9.0/python/tvm/te/hybrid/parser.py", line 58, in <listcomp>
lst = [visit(stmt) for stmt in lst if not utils.is_docstring(stmt)]
File "/opt/conda/lib/python3.9/ast.py", line 407, in visit
return visitor(node)
File "/root/tvmnew/apache-tvm-src-v0.9.0/python/tvm/te/hybrid/parser.py", line 363, in visit_Assign
res = tvm.tir.ProducerStore(lhs, rhs, args)
File "/root/tvmnew/apache-tvm-src-v0.9.0/python/tvm/tir/stmt.py", line 295, in __init__
self.__init_handle_by_constructor__(
File "/root/tvmnew/apache-tvm-src-v0.9.0/python/tvm/_ffi/_ctypes/object.py", line 136, in __init_handle_by_constructor__
handle = __init_by_constructor__(fconstructor, args)
File "/root/tvmnew/apache-tvm-src-v0.9.0/python/tvm/_ffi/_ctypes/packed_func.py", line 260, in __init_handle_by_constructor__
raise get_last_ffi_error()
3: TVMFuncCall
2: _ZN3tvm7runtime13PackedFun
1: tvm::runtime::TypedPackedFunc<tvm::tir::ProducerStore (tvm::tir::DataProducer, tvm::PrimExpr, tvm::runtime::Array<tvm::PrimExpr, void>, tvm::Span)>::AssignTypedLambda<tvm::tir::{lambda(tvm::tir::DataProducer, tvm::PrimExpr, tvm::runtime::Array<tvm::PrimExpr, void>, tvm::Span)#19}>(tvm::tir::{lambda(tvm::tir::DataProducer, tvm::PrimExpr, tvm::runtime::Array<tvm::PrimExpr, void>, tvm::Span)#19}, std::string)::{lambda(tvm::runtime::TVMArgs const&, tvm::runtime::TVMRetValue*)#1}::operator()(tvm::runtime::TVMArgs const&, tvm::runtime::TVMRetValue*) const
0: tvm::runtime::TVMMovableArgValueWithContext_::operator tvm::runtime::Array<tvm::PrimExpr, void><tvm::runtime::Array<tvm::PrimExpr, void> >() const
4: TVMFuncCall
3: _ZN3tvm7runtime13PackedFun
2: tvm::runtime::TypedPackedFunc<tvm::tir::ProducerStore (tvm::tir::DataProducer, tvm::PrimExpr, tvm::runtime::Array<tvm::PrimExpr, void>, tvm::Span)>::AssignTypedLambda<tvm::tir::{lambda(tvm::tir::DataProducer, tvm::PrimExpr, tvm::runtime::Array<tvm::PrimExpr, void>, tvm::Span)#19}>(tvm::tir::{lambda(tvm::tir::DataProducer, tvm::PrimExpr, tvm::runtime::Array<tvm::PrimExpr, void>, tvm::Span)#19}, std::string)::{lambda(tvm::runtime::TVMArgs const&, tvm::runtime::TVMRetValue*)#1}::operator()(tvm::runtime::TVMArgs const&, tvm::runtime::TVMRetValue*) const
1: tvm::runtime::TVMMovableArgValueWithContext_::operator tvm::runtime::Array<tvm::PrimExpr, void><tvm::runtime::Array<tvm::PrimExpr, void> >() const
0: tvm::runtime::Array<tvm::PrimExpr, void> tvm::runtime::TVMPODValue_::AsObjectRef<tvm::runtime::Array<tvm::PrimExpr, void> >() const
File "/root/tvmnew/apache-tvm-src-v0.9.0/include/tvm/runtime/packed_func.h", line 777
TVMError: In function tir.ProducerStore(0: tir.DataProducer, 1: PrimExpr, 2: Array<PrimExpr>, 3: Span) -> tir.ProducerStore: error while converting argument 2: [09:15:32] /root/tvmnew/apache-tvm-src-v0.9.0/include/tvm/runtime/packed_func.h:1863:
---------------------------------------------------------------
An error occurred during the execution of TVM.
For more information, please see: https://tvm.apache.org/docs/errors.html
---------------------------------------------------------------
Check failed: (!checked_type.defined()) is false: Expected Array[PrimExpr], but got IntImm