Crash after adding optimization: "FuseOps"

sqchao · February 15, 2022, 3:32pm

Question description:

After adding the “FuseOps” pass, Crash: TVMError: not handled was thrown when executing relay.build().

The script is as follows:

import tvm
from tvm import relay
from tvm.ir.transform import Sequential

var_0 = relay.var("var_0", dtype = "uint64", shape = ()) # shape=()
var_1 = relay.zeros_like(var_0) # shape=()
var_2 = relay.var("var_2", dtype = "uint64", shape = (9, 4, 10)) # shape=(9, 4, 10)
var_3 = relay.multiply(var_1, var_2) # shape=(9, 4, 10)
tuple = relay.Tuple([var_3,])
F = relay.Function([var_0,var_2,], tuple)

mod = tvm.IRModule()
mod['main'] = F
mod = relay.transform.InferType()(mod)
print(mod.astext(show_meta_data=False))
graph, lib, params = relay.build(mod, target='llvm')

seq = Sequential([
        relay.transform.FuseOps(3),
])
mod = seq(mod)
graph, lib, params = relay.build(mod, target='llvm')

Crash message:

Traceback (most recent call last):
  File "output1-sim.py", line 23, in <module>
    graph, lib, params = relay.build(mod, target='llvm')
  File "/workplace/software/tvm/tvm-new/python/tvm/relay/build_module.py", line 449, in build
    graph_json, runtime_mod, params = bld_mod.build(
  File "/workplace/software/tvm/tvm-new/python/tvm/relay/build_module.py", line 189, in build
    self._build(mod, target, target_host, executor, runtime, mod_name)
  File "/workplace/software/tvm/tvm-new/python/tvm/_ffi/_ctypes/packed_func.py", line 237, in __call__
    raise get_last_ffi_error()
tvm._ffi.base.TVMError: Traceback (most recent call last):
  58: TVMFuncCall
  57: _ZNSt17_Function_handlerIFvN3tvm7runtime7TVMArgsEPNS1_11
  56: tvm::relay::backend::RelayBuildModule::GetFunction(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, tvm::runtime::ObjectPtr<tvm::runtime::Object> const&)::{lambda(tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*)#3}::operator()(tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*) const
  55: tvm::relay::backend::RelayBuildModule::Build(tvm::IRModule, tvm::runtime::Map<tvm::Integer, tvm::Target, void, void> const&, tvm::Target const&, tvm::relay::Executor const&, tvm::relay::Runtime const&, tvm::runtime::String)
  54: tvm::relay::backend::RelayBuildModule::BuildRelay(tvm::IRModule, tvm::runtime::String const&)
  53: tvm::relay::backend::ExecutorCodegen::Codegen(tvm::IRModule, tvm::relay::Function const&, tvm::runtime::String)
  52: void tvm::relay::backend::ExecutorCodegen::CallFunc<tvm::IRModule, tvm::relay::Function, tvm::runtime::String>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, tvm::IRModule, tvm::relay::Function, tvm::runtime::String)
  51: _ZNSt17_Function_handlerIFvN3tvm7runtime7TVMArgsEPNS1_11
  50: tvm::relay::backend::GraphExecutorCodegenModule::GetFunction(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, tvm::runtime::ObjectPtr<tvm::runtime::Object> const&)::{lambda(tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*)#2}::operator()(tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*) const
  49: tvm::relay::backend::GraphExecutorCodegen::Codegen(tvm::IRModule, tvm::relay::Function, tvm::runtime::String)
  48: tvm::transform::Pass::operator()(tvm::IRModule) const
  47: tvm::transform::Pass::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
  46: tvm::transform::SequentialNode::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
  45: tvm::transform::Pass::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
  44: tvm::transform::ModulePassNode::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
  43: std::_Function_handler<void (tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*), tvm::runtime::TypedPackedFunc<tvm::IRModule (tvm::IRModule, tvm::transform::PassContext)>::AssignTypedLambda<tvm::relay::tec::LowerTEPass(tvm::runtime::String const&, std::function<void (tvm::BaseFunc)>, tvm::VirtualDevice)::$_8>(tvm::relay::tec::LowerTEPass(tvm::runtime::String const&, std::function<void (tvm::BaseFunc)>, tvm::VirtualDevice)::$_8)::{lambda(tvm::runtime::TVMArgs const&, tvm::runtime::TVMRetValue*)#1}>::_M_invoke(std::_Any_data const&, tvm::runtime::TVMArgs&&, tvm::runtime::TVMRetValue*&&)
  42: tvm::relay::tec::LowerTE(tvm::IRModule const&, tvm::runtime::String const&, std::function<void (tvm::BaseFunc)>, tvm::VirtualDevice)
  41: tvm::transform::Pass::operator()(tvm::IRModule) const
  40: tvm::transform::Pass::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
  39: tvm::relay::transform::FunctionPassNode::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
  38: std::_Function_handler<void (tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*), tvm::runtime::TypedPackedFunc<tvm::relay::Function (tvm::relay::Function, tvm::IRModule, tvm::transform::PassContext)>::AssignTypedLambda<tvm::relay::tec::LowerTensorExpr(tvm::runtime::String const&, tvm::relay::tec::TECompiler, std::function<void (tvm::BaseFunc)>, tvm::VirtualDevice)::$_7>(tvm::relay::tec::LowerTensorExpr(tvm::runtime::String const&, tvm::relay::tec::TECompiler, std::function<void (tvm::BaseFunc)>, tvm::VirtualDevice)::$_7)::{lambda(tvm::runtime::TVMArgs const&, tvm::runtime::TVMRetValue*)#1}>::_M_invoke(std::_Any_data const&, tvm::runtime::TVMArgs&&, tvm::runtime::TVMRetValue*&&)
  37: tvm::relay::ExprMutator::VisitExpr(tvm::RelayExpr const&)
  36: tvm::relay::ExprFunctor<tvm::RelayExpr (tvm::RelayExpr const&)>::VisitExpr(tvm::RelayExpr const&)
  35: tvm::NodeFunctor<tvm::RelayExpr (tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<tvm::RelayExpr (tvm::RelayExpr const&)>*)>::operator()(tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<tvm::RelayExpr (tvm::RelayExpr const&)>*) const
  34: _ZZN3tvm5relay11ExprFunc
  33: tvm::relay::transform::DeviceAwareExprMutator::VisitExpr_(tvm::relay::FunctionNode const*)
  32: tvm::relay::tec::LowerTensorExprMutator::DeviceAwareVisitExpr_(tvm::relay::FunctionNode const*)
  31: _ZN3tvm5relay9
  30: tvm::relay::ExprMutator::VisitExpr_(tvm::relay::FunctionNode const*)
  29: tvm::relay::ExprMutator::VisitExpr(tvm::RelayExpr const&)
  28: tvm::relay::ExprFunctor<tvm::RelayExpr (tvm::RelayExpr const&)>::VisitExpr(tvm::RelayExpr const&)
  27: tvm::NodeFunctor<tvm::RelayExpr (tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<tvm::RelayExpr (tvm::RelayExpr const&)>*)>::operator()(tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<tvm::RelayExpr (tvm::RelayExpr const&)>*) const
  26: _ZZN3tvm5relay11ExprFunc
  25: tvm::relay::ExprMutator::VisitExpr_(tvm::relay::TupleNode const*)
  24: tvm::relay::ExprMutator::VisitExpr(tvm::RelayExpr const&)
  23: tvm::relay::ExprFunctor<tvm::RelayExpr (tvm::RelayExpr const&)>::VisitExpr(tvm::RelayExpr const&)
  22: tvm::NodeFunctor<tvm::RelayExpr (tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<tvm::RelayExpr (tvm::RelayExpr const&)>*)>::operator()(tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<tvm::RelayExpr (tvm::RelayExpr const&)>*) const
  21: _ZZN3tvm5relay11ExprFunc
  20: tvm::relay::transform::DeviceAwareExprMutator::VisitExpr_(tvm::relay::CallNode const*)
  19: tvm::relay::tec::LowerTensorExprMutator::DeviceAwareVisitExpr_(tvm::relay::CallNode const*)
  18: tvm::relay::tec::LowerTensorExprMutator::MakeLoweredCall(tvm::relay::Function, tvm::runtime::Array<tvm::RelayExpr, void>, tvm::Span, tvm::Target)
  17: tvm::relay::tec::TECompilerImpl::Lower(tvm::relay::tec::CCacheKey const&, tvm::runtime::String)
  16: tvm::relay::tec::TECompilerImpl::Lower(tvm::relay::tec::CCacheKey const&, std::function<tvm::runtime::String (tvm::runtime::String)>)
  15: tvm::relay::tec::TECompilerImpl::LowerInternal(tvm::relay::tec::CCacheKey const&, std::function<tvm::runtime::String (tvm::runtime::String)>)
  14: tvm::relay::tec::PrimFuncFor(tvm::relay::Function const&, tvm::Target const&, std::function<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >)>)
  13: tvm::relay::tec::ScheduleBuilder::Create(tvm::relay::Function const&, std::function<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >)>)
  12: tvm::relay::backend::MemoizedExprTranslator<tvm::runtime::Array<tvm::te::Tensor, void> >::VisitExpr(tvm::RelayExpr const&)
  11: tvm::relay::ExprFunctor<tvm::runtime::Array<tvm::te::Tensor, void> (tvm::RelayExpr const&)>::VisitExpr(tvm::RelayExpr const&)
  10: tvm::NodeFunctor<tvm::runtime::Array<tvm::te::Tensor, void> (tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<tvm::runtime::Array<tvm::te::Tensor, void> (tvm::RelayExpr const&)>*)>::operator()(tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<tvm::runtime::Array<tvm::te::Tensor, void> (tvm::RelayExpr const&)>*) const
  9: _ZZN3tvm5relay11ExprFunc
  8: tvm::relay::tec::ScheduleBuilder::VisitExpr_(tvm::relay::CallNode const*)
  7: tvm::relay::backend::MemoizedExprTranslator<tvm::runtime::Array<tvm::te::Tensor, void> >::VisitExpr(tvm::RelayExpr const&)
  6: tvm::relay::ExprFunctor<tvm::runtime::Array<tvm::te::Tensor, void> (tvm::RelayExpr const&)>::VisitExpr(tvm::RelayExpr const&)
  5: tvm::NodeFunctor<tvm::runtime::Array<tvm::te::Tensor, void> (tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<tvm::runtime::Array<tvm::te::Tensor, void> (tvm::RelayExpr const&)>*)>::operator()(tvm::runtime::ObjectRef const&, tvm::relay::ExprFunctor<tvm::runtime::Array<tvm::te::Tensor, void> (tvm::RelayExpr const&)>*) const
  4: _ZZN3tvm5relay11ExprFunc
  3: tvm::relay::tec::ScheduleBuilder::VisitExpr_(tvm::relay::ConstantNode const*)
  2: tvm::te::compute(tvm::runtime::Array<tvm::PrimExpr, void>, std::function<tvm::PrimExpr (tvm::runtime::Array<tvm::tir::Var, void> const&)>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >,tvm::runtime::Map<tvm::runtime::String, tvm::runtime::ObjectRef, void, void>)
  1: _ZNSt17_Functi
  0: tvm::relay::tec::ScheduleBuilder::VisitExpr_(tvm::relay::ConstantNode const*)::{lambda(tvm::runtime::Array<tvm::tir::Var, void> const&)#1}::operator()(tvm::runtime::Array<tvm::tir::Var, void> const&) const
  File "/workplace/software/tvm/tvm-new/src/relay/backend/te_compiler_cache.cc", line 236
TVMError: not handled

AndrewZhaoLuo · February 15, 2022, 8:56pm

This seems like a simple error to fix, but I don’t understand the underlying code well. @mbs-octoml maybe knows what is going on?

masahi · February 15, 2022, 9:22pm

That’s not how you use FuseOps. FuseOps is always called during relay.build, you are not supposed to call it directly.

mbs-octoml · February 16, 2022, 5:15pm

Not all our passes respect the ‘Primitive’ attribute on already-fused Functions, so I suspect the implied invariant expected by the schedule builder when beginning lowering of a primitive function to TE has been broken. As it happens I’m having to fix these sorts of glitches as part of my work on Collage, so we could leave this open and I’ll come back to it once PRs start hitting main.

sqchao · February 18, 2022, 2:25pm

Thanks for your kind reply!

I though it was illegal to use these APIs in this way.