RPC error with TVM 0.9.dev0

microTVM
#1

Hi @areusch,

I am trying to deploy basic perceptron network (and other other simple fully connected neural networks) on stmf767zi, nrf5340dk boards with TVM 0.9.dev0 version. During runtime, a RPC error occurs without any specific error code/message as follows:

RPCError Traceback (most recent call last) in

in build_flash(mod, params, input_mcu, node, iteration)

~/tvm/python/tvm/contrib/debugger/debug_executor.py in run(self, **input_dict) 258 259 # Step 1. Execute the graph → 260 self._run_debug() 261 # Step 2. Dump the output tensors to the dump folder 262 self.debug_datum.dump_output_tensor()

~/tvm/python/tvm/contrib/debugger/debug_executor.py in _run_debug(self) 215 “”" 216 # Get timing. → 217 self.debug_datum._time_list = [[float(t)] for t in self.run_individual(10, 1, 1)] 218 219 # Get outputs.

~/tvm/python/tvm/contrib/debugger/debug_executor.py in run_individual(self, number, repeat, min_repeat_ms) 267 268 def run_individual(self, number, repeat=1, min_repeat_ms=0): → 269 ret = self._run_individual(number, repeat, min_repeat_ms) 270 return ret.strip(",").split(",") if ret else [] 271

~/tvm/python/tvm/_ffi/_ctypes/packed_func.py in call(self, *args) 235 != 0 236 ): → 237 raise get_last_ffi_error() 238 _ = temp_args 239 _ = args

RPCError: Traceback (most recent call last): 13: TVMFuncCall 12: std::_Function_handler<void (tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*), tvm::runtime::GraphExecutorDebug::GetFunction(std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&, tvm::runtime::ObjectPtrtvm::runtime::Object const&)::{lambda(tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*)#4}>::_M_invoke(std::_Any_data const&, tvm::runtime::TVMArgs&&, tvm::runtime::TVMRetValue*&&) 11: tvm::runtime::GraphExecutorDebug::RunIndividual[abi:cxx11](int, int, int) 10: tvm::runtime::GraphExecutorDebug::RunOpRPC(int, int, int, int) 9: tvm::runtime::TypedPackedFunc<tvm::runtime::PackedFunc (tvm::runtime::Optionaltvm::runtime::Module, std::__cxx11::basic_string<char, std::char_traits, std::allocator >, int, int, int, int, int, std::__cxx11::basic_string<char, std::char_traits, std::allocator >)>::AssignTypedLambda<tvm::runtime::{lambda(tvm::runtime::Optionaltvm::runtime::Module, std::__cxx11::basic_string<char, std::char_traits, std::allocator >, int, int, int, int, int, std::__cxx11::basic_string<char, std::char_traits, std::allocator >)#1}>(tvm::runtime::{lambda(tvm::runtime::Optionaltvm::runtime::Module, std::__cxx11::basic_string<char, std::char_traits, std::allocator >, int, int, int, int, int, std::__cxx11::basic_string<char, std::char_traits, std::allocator >)#1}, std::__cxx11::basic_string<char, std::char_traits, std::allocator >)::{lambda(tvm::runtime::TVMArgs const&, tvm::runtime::TVMRetValue*)#1}::operator()(tvm::runtime::TVMArgs const, tvm::runtime::TVMRetValue) const 8: tvm::runtime::RPCModuleNode::GetTimeEvaluator(std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&, DLDevice, int, int, int, std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&) 7: _ZNSt17_Function_handlerIFvN3tvm7runtime7TVMArgsEPNS1_11 6: tvm::runtime::RPCWrappedFunc::operator()(tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*) const 5: tvm::runtime::RPCClientSession::CallFunc(void*, TVMValue const*, int const*, int, std::function<void (tvm::runtime::TVMArgs)> const&) 4: tvm::runtime::RPCEndpoint::CallFunc(void*, TVMValue const*, int const*, int, std::function<void (tvm::runtime::TVMArgs)>) 3: tvm::runtime::RPCEndpoint::HandleUntilReturnEvent(bool, std::function<void (tvm::runtime::TVMArgs)>) 2: tvm::runtime::RPCEndpoint::EventHandler::HandleNextEvent(bool, bool, std::function<void (tvm::runtime::TVMArgs)>) 1: tvm::runtime::RPCEndpoint::EventHandler::HandleProcessPacket(std::function<void (tvm::runtime::TVMArgs)>) 0: tvm::runtime::RPCEndpoint::EventHandler::HandleReturn(tvm::runtime::RPCCode, std::function<void (tvm::runtime::TVMArgs)>) File “/home/vagrant/tvm/src/runtime/rpc/rpc_endpoint.cc”, line 377 RPCError: Error caught from RPC call:

Surprisingly, I am able to deploy a simple LSTM network (without any fully connected layers but just a LSTM block) with the same TVM version. Do you have any idea what might be wrong? I was able to do this with TVM 0.8.dev0.

#2

@asnecemnnit could you post the code snippet you’re using to build (specifically the target param and any executor= and runtime= params you might be using with 0.9dev0)? Initially I suspect #9352 which landed just after 0.8 and could influence your compile config, but there are a few other possibilities as well.

#3

Here is the code snippet:

def _make_session(temp_dir, zephyr_board, west_cmd, mod):
        config_main_stack_size = None

        project_options = {
            "project_type": "host_driven",
            "west_cmd": west_cmd,
            "verbose": False,
            "zephyr_board": zephyr_board,
        }
        if config_main_stack_size is not None:
            project_options["config_main_stack_size"] = config_main_stack_size

        project = tvm.micro.generate_project(
            temp_dir,
            mod,
            temp_dir + "/project",
            project_options,
        )
        project.build()
        project.flash()
        return tvm.micro.Session(project.transport())

    TARGET = tvm.target.target.micro("stm32f746xx") # Specifying target hardware architecture
    BOARD = "nucleo_f746zg" # Specifying exact board name, used by zephyr later to create binary

    repo_root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], encoding='utf-8').strip()
    project_dir = os.path.join(repo_root, "apps", "microtvm", "zephyr", "template_project")
    temp_file_directory = repo_root

    def build_flash(mod,TARGET,params,input_mcu,node,iteration):
         RUNTIME = Runtime(“crt”, {“system-lib”: True})
         with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}):
             c_mod = relay.build(mod, target=TARGET, runtime=RUNTIME, params=params)
         with _make_session(project_dir, BOARD, "west", c_mod) as session:
            graph_mod = tvm.micro.create_local_debug_executor(graph, session.get_system_lib(), session.device, dump_root= temp_file_directory)
            graph_mod.set_input(**c_mod.get_params())    
            graph_mod.run() 
            tvm_output = graph_mod.get_output(0).asnumpy()