Cannot build LSTM model in relay.testing

Questions
#1

I’m trying to build LSTM model defined in relay.testing, but got an TVMError.

from tvm import relay
from tvm.relay import testing
import tvm
from tvm.contrib import graph_runtime

mod, params = relay.testing.lstm.get_workload(4, 4)

print(mod.astext(show_meta_data=False))
target = "llvm"

with tvm.transform.PassContext(opt_level=3):
    lib = relay.build(mod, target, params=params)

Error message:

  File "<stdin>", line 2, in <module>
  File "/home/rocm-user/tvm/python/tvm/relay/build_module.py", line 275, in build
    graph_json, mod, params = bld_mod.build(mod, target, target_host, params)
  File "/home/rocm-user/tvm/python/tvm/relay/build_module.py", line 138, in build
    self._build(mod, target, target_host)
  File "/home/rocm-user/tvm/python/tvm/_ffi/_ctypes/packed_func.py", line 237, in __call__
    raise get_last_ffi_error()
tvm._ffi.base.TVMError: Traceback (most recent call last):
  [bt] (8) /home/rocm-user/tvm/build/libtvm.so(tvm::relay::ExprVisitor::VisitExpr(tvm::RelayExpr const&)+0x7b) [0x7f8fe2e66ccb]
  [bt] (7) /home/rocm-user/tvm/build/libtvm.so(tvm::relay::ExprFunctor<void (tvm::RelayExpr const&)>::VisitExpr(tvm::RelayExpr const&)+0x7b) [0x7f8fe2e1bdeb]
  [bt] (6) /home/rocm-user/tvm/build/libtvm.so(tvm::relay::StorageAllocaBaseVisitor::VisitExpr_(tvm::relay::LetNode const*)+0x29) [0x7f8fe2dcd6b9]
  [bt] (5) /home/rocm-user/tvm/build/libtvm.so(tvm::relay::StorageAllocaBaseVisitor::GetToken(tvm::RelayExpr const&)+0x12) [0x7f8fe2dc6a02]
  [bt] (4) /home/rocm-user/tvm/build/libtvm.so(tvm::relay::ExprVisitor::VisitExpr(tvm::RelayExpr const&)+0x7b) [0x7f8fe2e66ccb]
  [bt] (3) /home/rocm-user/tvm/build/libtvm.so(tvm::relay::ExprFunctor<void (tvm::RelayExpr const&)>::VisitExpr(tvm::RelayExpr const&)+0x7b) [0x7f8fe2e1bdeb]
  [bt] (2) /home/rocm-user/tvm/build/libtvm.so(tvm::relay::StorageAllocaInit::VisitExpr_(tvm::relay::CallNode const*)+0x2e) [0x7f8fe2dcaa6e]
  [bt] (1) /home/rocm-user/tvm/build/libtvm.so(tvm::relay::StorageAllocaInit::CreateToken(tvm::RelayExprNode const*, bool)+0x619) [0x7f8fe2dc9d19]
  [bt] (0) /home/rocm-user/tvm/build/libtvm.so(+0x162e3b2) [0x7f8fe2dc63b2]
  File "/home/rocm-user/tvm/src/relay/backend/graph_plan_memory.cc", line 151
TVMError: 
---------------------------------------------------------------
An internal invariant was violated during the execution of TVM.
Please read TVM's error reporting guidelines.
More details can be found here: https://discuss.tvm.ai/t/error-reporting/7793.
---------------------------------------------------------------
  Check failed: ttype == false:
Auto-scheduling for lstm operator
#2

Is this an internal bug or my mistake?

Could any help me here please? :sob:

#3

@Francis22 I also got this error before. It seems that in TVM LSTM only works with the interpreter. Also wonder if this is bug or simply an unsupported feature.

executor = relay.build_module.create_executor('debug', mod, ...).evaluate()
#4

Hi, is there any update about this issue?

I am trying to tune an LSTM using auto-scheduler but failed to build it too.

The error message is now: (commit 5014b7)

One or more operators have not been tuned. Please tune your model for better performance. Use DEBUG logging level to see more details.
Traceback (most recent call last):
  File "relay_lstm.py", line 12, in <module>
    lib = relay.build(mod, target, params=params)
  File "/root/tvm/python/tvm/relay/build_module.py", line 455, in build
    mod_name=mod_name,
  File "/root/tvm/python/tvm/relay/build_module.py", line 189, in build
    self._build(mod, target, target_host, executor, runtime, mod_name)
  File "/root/tvm/python/tvm/_ffi/_ctypes/packed_func.py", line 237, in __call__
    raise get_last_ffi_error()
tvm._ffi.base.TVMError: Traceback (most recent call last):
  35: TVMFuncCall
  34: _ZNSt17_Function_handlerIFvN
  33: tvm::relay::backend::RelayBuildModule::GetFunction(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, tvm::runtime::ObjectPtr<tvm::runtime::Object> const&)::{lambda(tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*)#3}::operator()(tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*) const
  32: tvm::relay::backend::RelayBuildModule::BuildRelay(tvm::IRModule, tvm::runtime::String const&)
  31: std::_Function_handler<void (tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*), tvm::relay::backend::GraphExecutorCodegenModule::GetFunction(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, tvm::runtime::ObjectPtr<tvm::runtime::Object> const&)::{lambda(tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*)#2}>::_M_invoke(std::_Any_data const&, tvm::runtime::TVMArgs&&, tvm::runtime::TVMRetValue*&&)
  30: tvm::relay::backend::GraphExecutorCodegen::Codegen(tvm::IRModule, tvm::relay::Function, tvm::runtime::String)
  29: tvm::transform::Pass::operator()(tvm::IRModule) const
  28: tvm::transform::Pass::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
  27: tvm::transform::SequentialNode::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
  26: tvm::transform::Pass::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
  25: tvm::transform::ModulePassNode::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
  24: std::_Function_handler<void (tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*), tvm::runtime::TypedPackedFunc<tvm::IRModule (tvm::IRModule, tvm::transform::PassContext)>::AssignTypedLambda<tvm::relay::tec::LowerTEPass(tvm::runtime::String const&, std::function<void (tvm::BaseFunc)>, tvm::SEScope)::{lambda(tvm::IRModule, tvm::transform::PassContext)#1}>(tvm::relay::tec::LowerTEPass(tvm::runtime::String const&, std::function<void (tvm::BaseFunc)>, tvm::SEScope)::{lambda(tvm::IRModule, tvm::transform::PassContext)#1})::{lambda(tvm::runtime::TVMArgs const&, tvm::runtime::TVMRetValue*)#1}>::_M_invoke(std::_Any_data const&, tvm::runtime::TVMArgs&&, tvm::runtime::TVMRetValue*&&)
  23: tvm::relay::tec::LowerTE(tvm::IRModule const&, tvm::runtime::String const&, std::function<void (tvm::BaseFunc)>, tvm::SEScope)
  22: tvm::transform::Pass::operator()(tvm::IRModule) const
  21: tvm::transform::Pass::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
  20: tvm::relay::transform::FunctionPassNode::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
  19: std::_Function_handler<void (tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*), tvm::runtime::TypedPackedFunc<tvm::relay::Function (tvm::relay::Function, tvm::IRModule, tvm::transform::PassContext)>::AssignTypedLambda<tvm::relay::tec::LowerTensorExpr(tvm::runtime::String const&, tvm::relay::tec::TECompiler, std::function<void (tvm::BaseFunc)>, tvm::SEScope)::{lambda(tvm::relay::Function, tvm::IRModule, tvm::transform::PassContext)#1}>(tvm::relay::tec::LowerTensorExpr(tvm::runtime::String const&, tvm::relay::tec::TECompiler, std::function<void (tvm::BaseFunc)>, tvm::SEScope)::{lambda(tvm::relay::Function, tvm::IRModule, tvm::transform::PassContext)#1})::{lambda(tvm::runtime::TVMArgs const&, tvm::runtime::TVMRetValue*)#1}>::_M_invoke(std::_Any_data const&, tvm::runtime::TVMArgs&&, tvm::runtime::TVMRetValue*&&)
  18: tvm::relay::ExprMutator::VisitExpr(tvm::RelayExpr const&)
  17: tvm::relay::ExprFunctor<tvm::RelayExpr (tvm::RelayExpr const&)>::VisitExpr(tvm::RelayExpr const&)
  16: _ZZN3tvm5relay11ExprFunctorIFNS_9RelayExprERKS2_EE10InitVTableEvENUlR
  15: tvm::relay::transform::DeviceAwareExprMutator::VisitExpr_(tvm::relay::FunctionNode const*)
  14: tvm::relay::tec::LowerTensorExprMutator::DeviceAwareVisitExpr_(tvm::relay::FunctionNode const*)
  13: _ZN3tvm5relay9transform22DeviceAwareExprMutator21DeviceAwar
  12: tvm::relay::ExprMutator::VisitExpr_(tvm::relay::FunctionNode const*)
  11: tvm::relay::ExprMutator::VisitExpr(tvm::RelayExpr const&)
  10: tvm::relay::ExprFunctor<tvm::RelayExpr (tvm::RelayExpr const&)>::VisitExpr(tvm::RelayExpr const&)
  9: _ZZN3tvm5relay11ExprFunctorIFNS_9RelayExprERKS2_EE10InitVTableEvENUlR
  8: tvm::relay::transform::DeviceAwareExprMutator::VisitExpr_(tvm::relay::LetNode const*)
  7: tvm::relay::tec::LowerTensorExprMutator::PreVisitLetBinding_(tvm::relay::Var const&, tvm::RelayExpr const&)
  6: tvm::relay::ExprMutator::VisitExpr(tvm::RelayExpr const&)
  5: tvm::relay::ExprFunctor<tvm::RelayExpr (tvm::RelayExpr const&)>::VisitExpr(tvm::RelayExpr const&)
  4: _ZZN3tvm5relay11ExprFunctorIFNS_9RelayExprERKS2_EE10InitVTableEvENUlR
  3: tvm::relay::transform::DeviceAwareExprMutator::VisitExpr_(tvm::relay::CallNode const*)
  2: tvm::relay::tec::LowerTensorExprMutator::DeviceAwareVisitExpr_(tvm::relay::CallNode const*)
  1: std::_Function_handler<void (tvm::BaseFunc), tvm::relay::backend::GraphExecutorCodegen::Codegen(tvm::IRModule, tvm::relay::Function, tvm::runtime::String)::{lambda(tvm::BaseFunc)#1}>::_M_invoke(std::_Any_data const&, tvm::BaseFunc&&)
  0: tvm::relay::tec::UpdateFunctionMetadata(tvm::BaseFunc, tvm::runtime::Map<tvm::runtime::String, tvm::relay::backend::FunctionInfo, void, void>&, tvm::Integer)
  File "/root/tvm/src/relay/backend/te_compiler.cc", line 982
TVMError: Check failed: (prim_fns) is false: primitive functions not set on Relay function by TECompiler.