Error while running tflite model with vulkan backend on android

dabhinav10 · February 21, 2024, 9:08am

@masahi @csullivan @Lunderberg @tqchen @yzhliu Hi, I have a tflite model I’d like to run via TVM on Android using tvm_rpc with Vulkan Backend. My host tvm build is compiling with vulkan enabled but the target android runtime isn’t compiling with vulkan.

Using

mod, params = relay.frontend.from_tflite(
    tflite_model, shape_dict={input_tensor: input_shape}, dtype_dict={input_tensor: input_dtype}
)
test_target = "vulkan"
arch = "arm64"
target = tvm.target.Target("llvm -mtriple=arm64-linux-android")
target = tvm.target.Target(test_target, host=target)
with tvm.transform.PassContext(opt_level=3):
        lib = relay.build(mod, target=target, params=params)
fcompile = ndk.create_shared if run_on_device else None
lib.export_library(lib_fname, fcompile)

The relay is built but when running using:

import tvm
import numpy as np
from tvm import te
from tvm.contrib import graph_executor as runtime

# ctx = remote.cl(0)
#ctx = remote.cpu(0)
ctx = remote.vulkan(0)
print("ctx:", ctx)
# Transfer the model lib to remote device
remote.upload(lib_fname)
# Load the remote module
rlib = remote.load_module(lib_fname)

# Create a runtime executor module
module = runtime.GraphModule(rlib["default"](ctx))

# Run
module.run()

# Benchmark the performance

ftime = module.module.time_evaluator("run", ctx, number=1, repeat=10)
prof_res = np.array(ftime().results) * 1000
print("Mean inference time (std dev): %.2f ms (%.2f ms)" % (np.mean(prof_res), np.std(prof_res)))

I’m getting this error:

RPCError: Error caught from RPC call: [15:58:19] /home/tvm-code/abhinav/clones/open_src/vulkan_ostvm/tvm/src/runtime/library_module.cc:122: Binary was created using {vulkan} but a loader of that name is not registered. Available loaders are const_loader, GraphExecutorFactory, static_library, relax.Executable, AotExecutorFactory, opencl, metadata_module, VMExecutable, metadata, GraphRuntimeFactory. Perhaps you need to recompile with this runtime enabled.

Below is the configuration for host and target build. Host Build:

mkdir -p build
cd build
cp ../cmake/config.cmake .

echo set\(USE_RPC ON\) >> config.cmake
echo set\(USE_GRAPH_EXECUTOR ON\) >> config.cmake
echo set\(USE_LIBBACKTRACE AUTO\) >> config.cmake
echo set\(USE_LLVM llvm-config-10\) >> config.cmake
echo set\(USE_VULKAN ON\) >> config.cmake

cmake ..
make

Got these logs in output:

-- Vulkan_INCLUDE_DIRS=/usr/include/usr/include/spirv-tools/usr/include/spirv/unified1/usr/include/spirv/unified1
-- Vulkan_LIBRARY=/usr/lib/x86_64-linux-gnu/libvulkan.so
-- Vulkan_SPIRV_TOOLS_LIBRARY=/usr/lib/x86_64-linux-gnu/libSPIRV-Tools.a

So the host build is finding vulkan libraries and tools from root and is building with vulkan enabled.

Android Target Build:

Env Variables:

export TVM_ROOT=/home/tvm-code/abhinav/clones/open_src/vulkan_ostvm/tvm/
export TVM_HOME=/home/tvm-code/abhinav/clones/open_src/vulkan_ostvm/tvm/
export ANDROID_NDK_ROOT=/home/tvm-code/Android/Ndk/android-ndk-r25c/
export ANDROID_NDK_HOME=$ANDROID_NDK_ROOT
export NDK_ROOT=$ANDROID_NDK_ROOT
export ANDROID_NDK=$ANDROID_NDK_ROOT
export PYTHONPATH=$TVM_ROOT/python:$PYTHONPATH
export TVM_NDK_CC=$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android28-clang++
export Vulkan_LIBRARY=/home/tvm-code/Android/Ndk/android-ndk-r25c/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/aarch64-linux-android/28/libvulkan.so

mkdir -p build-test
cd build-test
cp ../cmake/config.cmake .

echo set\(USE_OPENCL ON\) >> config.cmake
echo set\(USE_LLVM llvm-config-10\) >> config.cmake
echo set\(USE_RPC ON\) >> config.cmake
echo set\(USE_CPP_RPC ON\) >> config.cmake
echo set\(USE_CPP_RTVM ON\) >> config.cmake
echo set\(USE_SORT ON\) >> config.cmake
echo set\(USE_VULKAN ON\) >> config.cmake
echo set\(USE_GRAPH_EXECUTOR ON\) >> config.cmake
echo set\(USE_LIBBACKTRACE AUTO\) >> config.cmake
echo set\(USE_KALLOC_ALIGNMENT 32\) >> config.cmake
echo set\(ANDROID_ABI arm64-v8a\) >> config.cmake
echo set\(ANDROID_PLATFORM android-28\) >> config.cmake
echo set\(MACHINE_NAME aarch64-linux-gnu\) >> config.cmake

cmake -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK_HOME}/build/cmake/android.toolchain.cmake" \
   -DANDROID_ABI=arm64-v8a \
   -DANDROID_PLATFORM=android-28 \
   -DUSE_VULKAN=ON \
   -DANDROID_NATIVE_API_LEVEL=23 ..

make tvm_runtime tvm_rpc rtvm

After this I tried

readelf -a libtvm_runtime.so | grep vulkan

and it outputs nothing.

srkreddy1238 · February 22, 2024, 6:02am

Hi Abhinav,

As a functionality, I enabled Vulkan for Android a while back and it worked. Didn’t try recent times though.

Probably you may look into cmake console log (Vulkan related enables) and build log (to make sure Vulkan runtime files are compiled).

Siva

dabhinav10 · February 22, 2024, 10:16am

@srkreddy1238 As you can see from the below logs there aren’t any vulkan related enables in the build logs though its including given directories in shown in cmake log. It is setting these variables according to FindVulkan.cmake. Also tried building only tvm_runtime but got same output.

  if(CMAKE_SYSTEM_NAME STREQUAL "Android")
    set(VULKAN_NDK_SRC ${CMAKE_ANDROID_NDK}/sources/third_party/vulkan/src)
    set(Vulkan_INCLUDE_DIRS ${VULKAN_NDK_SRC}/include)
    set(Vulkan_FOUND TRUE)
    message(STATUS "Android Vulkan_INCLUDE_DIRS=" ${Vulkan_INCLUDE_DIRS})
    message(STATUS "Skip finding SPIRV in Android, make sure you only build tvm runtime.")
    return()
  endif()

-- Android Vulkan_INCLUDE_DIRS=/home/tvm-code/Android/Ndk/android-ndk-r25c/sources/third_party/vulkan/src/include
-- Skip finding SPIRV in Android, make sure you only build tvm runtime.

CMake logs:

-- The C compiler identification is Clang 14.0.7
-- The CXX compiler identification is Clang 14.0.7
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Check for working C compiler: /home/tvm-code/Android/Ndk/android-ndk-r25c/toolchains/llvm/prebuilt/linux-x86_64/bin/clang - skipped
-- Detecting C compile features
-- Detecting C compile features - done
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Check for working CXX compiler: /home/tvm-code/Android/Ndk/android-ndk-r25c/toolchains/llvm/prebuilt/linux-x86_64/bin/clang++ - skipped
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Forbidding undefined symbols in shared library, using -Wl,--no-undefined on platform Android
-- Building for Android
-- Build with RPC support...
-- Build with Graph Executor support...
-- Build with profiler...
-- Build with AOT Executor support...
-- Could NOT find GTest (missing: GTEST_LIBRARY GTEST_INCLUDE_DIR GTEST_MAIN_LIBRARY)
-- Build Alloc alignment set to 32
-- Didn't find the path to CCACHE, disabling ccache
-- Performing Test SUPPORT_CXX17
-- Performing Test SUPPORT_CXX17 - Success
-- VTA build with VTA_HW_PATH=/home/tvm-code/abhinav/clones/open_src/vulkan_ostvm/tvm/3rdparty/vta-hw
-- Build VTA runtime with target: sim
-- Enabled runtime search for OpenCL library location
-- Using gtest from Android NDK
-- Building OpenCL-Gtests
-- Android Vulkan_INCLUDE_DIRS=/home/tvm-code/Android/Ndk/android-ndk-r25c/sources/third_party/vulkan/src/include
-- Skip finding SPIRV in Android, make sure you only build tvm runtime.
-- Use llvm-config=llvm-config-10
-- LLVM libdir: /usr/lib/llvm-10/lib
-- LLVM cmakedir: /usr/lib/llvm-10/lib/cmake/llvm
-- Found LLVM_INCLUDE_DIRS=/usr/lib/llvm-10/include
-- Found LLVM_DEFINITIONS=-D_GNU_SOURCE;-D__STDC_CONSTANT_MACROS;-D__STDC_FORMAT_MACROS;-D__STDC_LIMIT_MACROS
-- Found LLVM_LIBS=/usr/lib/llvm-10/lib/libLLVM-10.so
-- Found TVM_LLVM_VERSION=100
-- Build with LLVM
-- Set TVM_LLVM_VERSION=100
-- Build with contrib.random
-- Build with contrib.sort
-- Build with contrib.hybriddump
-- Git found: /usr/bin/git
-- Found TVM_GIT_COMMIT_HASH=daa37e7e954f77006da4f2994bf544a1e9c62fce
-- Found TVM_GIT_COMMIT_TIME=2024-02-15 14:10:10 -0500
-- Could NOT find LIBBACKTRACE (missing: LIBBACKTRACE_STATIC_LIBRARY LIBBACKTRACE_INCLUDE_DIR)
-- Building with TVM Map...
-- Build with thread support...
-- Performing Test CMAKE_HAVE_LIBC_PTHREAD
-- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Failed
-- Check if compiler accepts -pthread
-- Check if compiler accepts -pthread - yes
-- Found Threads: TRUE
-- CLANG_VERSION 14.0
-- Setting enhanced clang warning flags
-- Performing Test FILE_PREFIX_MAP_SUPPORTED
-- Performing Test FILE_PREFIX_MAP_SUPPORTED - Success
-- Added "-fuse-ld=lld" to linker flags -static-libstdc++ -Wl,--build-id=sha1 -Wl,--no-rosegment -Wl,--fatal-warnings -Wl,--no-undefined -Qunused-arguments
-- Build without FlashInfer
-- Configuring done (2.9s)
-- Generating done (0.2s)
-- Build files have been written to: /home/tvm-code/abhinav/clones/open_src/vulkan_ostvm/tvm/build-test

Build logs:

[  0%] Building CXX object CMakeFiles/tvm_libinfo_objs.dir/src/support/libinfo.cc.o
[  0%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/builtin_fp16.cc.o
[  0%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/c_runtime_api.cc.o
[  0%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/const_loader_module.cc.o
[  0%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/container.cc.o
[  0%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/cpu_device_api.cc.o
[ 12%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/debug.cc.o
[ 12%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/disco/bcast_session.cc.o
[ 12%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/disco/builtin.cc.o
[ 12%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/disco/process_session.cc.o
[ 12%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/dso_library.cc.o
[ 12%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/disco/disco_worker.cc.o
[ 12%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/disco/loader.cc.o
[ 12%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/disco/session.cc.o
[ 12%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/disco/threaded_session.cc.o
[ 25%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/file_utils.cc.o
[ 25%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/library_module.cc.o
[ 25%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/logging.cc.o
[ 25%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/memory/memory_manager.cc.o
[ 25%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/metadata.cc.o
[ 25%] Built target tvm_libinfo_objs
[ 25%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/minrpc/minrpc_logger.cc.o
[ 25%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/module.cc.o
[ 25%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/name_transforms.cc.o
[ 25%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/ndarray.cc.o
[ 37%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/nvtx.cc.o
[ 37%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/object.cc.o
[ 37%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/packed_func.cc.o
[ 37%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/profiling.cc.o
[ 37%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/registry.cc.o
[ 37%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/relax_vm/builtin.cc.o
[ 37%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/relax_vm/bytecode.cc.o
[ 37%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/relax_vm/executable.cc.o
[ 50%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/relax_vm/lm_support.cc.o
[ 50%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/relax_vm/ndarray_cache_support.cc.o
[ 50%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/relax_vm/paged_kv_cache.cc.o
[ 50%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/relax_vm/vm.cc.o
[ 50%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/source_utils.cc.o
[ 50%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/static_library.cc.o
[ 50%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/system_library.cc.o
[ 50%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/thread_pool.cc.o
[ 50%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/threading_backend.cc.o
[ 62%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/vm/bytecode.cc.o
[ 62%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/vm/executable.cc.o
[ 62%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/vm/vm.cc.o
[ 62%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/workspace_pool.cc.o
[ 62%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/rpc/rpc_channel.cc.o
[ 62%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/rpc/rpc_device_api.cc.o
[ 62%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/rpc/rpc_endpoint.cc.o
[ 62%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/rpc/rpc_event_impl.cc.o
[ 62%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/rpc/rpc_local_session.cc.o
[ 75%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/rpc/rpc_module.cc.o
[ 75%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/rpc/rpc_pipe_impl.cc.o
[ 75%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/rpc/rpc_server_env.cc.o
[ 75%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/rpc/rpc_session.cc.o
[ 75%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/rpc/rpc_socket_impl.cc.o
[ 75%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/graph_executor/graph_executor.cc.o
[ 75%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/graph_executor/graph_executor_factory.cc.o
[ 75%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/graph_executor/debug/graph_executor_debug.cc.o
[ 75%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/vm/profiler/vm.cc.o
[ 87%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/aot_executor/aot_executor.cc.o
[ 87%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/aot_executor/aot_executor_factory.cc.o
[ 87%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/opencl/opencl_device_api.cc.o
[ 87%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/opencl/opencl_module.cc.o
[ 87%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/opencl/opencl_module_spirv.cc.o
[ 87%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/opencl/texture_pool.cc.o
[ 87%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/opencl/opencl_wrapper/opencl_wrapper.cc.o
[ 87%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/contrib/random/random.cc.o
[100%] Building CXX object CMakeFiles/tvm_runtime_objs.dir/src/runtime/contrib/sort/sort.cc.o
[100%] Built target tvm_runtime_objs
[100%] Linking CXX shared library libtvm_runtime.so
[100%] Built target tvm_runtime
[  0%] Built target tvm_libinfo_objs
[100%] Built target tvm_runtime_objs
[100%] Built target tvm_runtime
[100%] Building CXX object apps/cpp_rpc/CMakeFiles/tvm_rpc.dir/main.cc.o
[100%] Building CXX object apps/cpp_rpc/CMakeFiles/tvm_rpc.dir/rpc_env.cc.o
[100%] Building CXX object apps/cpp_rpc/CMakeFiles/tvm_rpc.dir/rpc_server.cc.o
[100%] Linking CXX executable ../../tvm_rpc
[100%] Built target tvm_rpc
[  0%] Built target tvm_libinfo_objs
[100%] Built target tvm_runtime_objs
[100%] Built target tvm_runtime
[100%] Building CXX object apps/cpp_rtvm/CMakeFiles/rtvm.dir/main.cc.o
[100%] Building CXX object apps/cpp_rtvm/CMakeFiles/rtvm.dir/tvm_runner.cc.o
[100%] Building CXX object apps/cpp_rtvm/CMakeFiles/rtvm.dir/__/__/3rdparty/cnpy/cnpy.cpp.o
/home/tvm-code/abhinav/clones/open_src/vulkan_ostvm/tvm/3rdparty/cnpy/cnpy.cpp:64:13: warning: unused variable 'major_version' [-Wunused-variable]
    uint8_t major_version = *reinterpret_cast<uint8_t*>(buffer+6);
            ^
/home/tvm-code/abhinav/clones/open_src/vulkan_ostvm/tvm/3rdparty/cnpy/cnpy.cpp:93:10: warning: unused variable 'littleEndian' [-Wunused-variable]
    bool littleEndian = (header[loc1] == '<' || header[loc1] == '|' ? true : false);
         ^
/home/tvm-code/abhinav/clones/open_src/vulkan_ostvm/tvm/3rdparty/cnpy/cnpy.cpp:65:13: warning: unused variable 'minor_version' [-Wunused-variable]
    uint8_t minor_version = *reinterpret_cast<uint8_t*>(buffer+7);
            ^
/home/tvm-code/abhinav/clones/open_src/vulkan_ostvm/tvm/3rdparty/cnpy/cnpy.cpp:144:10: warning: unused variable 'littleEndian' [-Wunused-variable]
    bool littleEndian = (header[loc1] == '<' || header[loc1] == '|' ? true : false);
         ^
/home/tvm-code/abhinav/clones/open_src/vulkan_ostvm/tvm/3rdparty/cnpy/cnpy.cpp:163:50: warning: variable 'comment_len' set but not used [-Wunused-but-set-variable]
    uint16_t disk_no, disk_start, nrecs_on_disk, comment_len;
                                                 ^
/home/tvm-code/abhinav/clones/open_src/vulkan_ostvm/tvm/3rdparty/cnpy/cnpy.cpp:163:35: warning: variable 'nrecs_on_disk' set but not used [-Wunused-but-set-variable]
    uint16_t disk_no, disk_start, nrecs_on_disk, comment_len;
                                  ^
/home/tvm-code/abhinav/clones/open_src/vulkan_ostvm/tvm/3rdparty/cnpy/cnpy.cpp:163:23: warning: variable 'disk_start' set but not used [-Wunused-but-set-variable]
    uint16_t disk_no, disk_start, nrecs_on_disk, comment_len;
                      ^
/home/tvm-code/abhinav/clones/open_src/vulkan_ostvm/tvm/3rdparty/cnpy/cnpy.cpp:163:14: warning: variable 'disk_no' set but not used [-Wunused-but-set-variable]
    uint16_t disk_no, disk_start, nrecs_on_disk, comment_len;
             ^
/home/tvm-code/abhinav/clones/open_src/vulkan_ostvm/tvm/3rdparty/cnpy/cnpy.cpp:199:9: warning: variable 'err' set but not used [-Wunused-but-set-variable]
    int err;
        ^
/home/tvm-code/abhinav/clones/open_src/vulkan_ostvm/tvm/apps/cpp_rtvm/main.cc:252:9: warning: unused variable 'total_time' [-Wunused-variable]
    int total_time = 0;
        ^
1 warning generated.
9 warnings generated.
[100%] Linking CXX executable ../../rtvm
[100%] Built target rtvm

Also I’ve read on the forum that https://github.com/apache/tvm/blob/main/src/runtime/vulkan/vulkan_device_api.cc this file should get included in the .so which isn’t there.

pfk123 · September 27, 2024, 2:18pm

Hello,

I encounter the same problem. After some digging I have found problematic construction in cmake. I think main entry is cmake/modules/Vulkan.cmake. At the begining of that file, we can see that find_vulkan is first command. FindVulkan.cmake has suspicius statement in CMAKE_SYSTEM_NAME STREQUAL "Android" condition, i.e. return statement, which breaks FindVulka.cmake macro.

The question is, how to set USE_VULKAN variable. ON causes problems. I would like to set path to SDK, but how can I specify that with Android NDK? Do Android NDK contains Vulkan SDK?

@srkreddy1238 sorry, for necromancing this thread, but do you remember, or have notes about how you compiled tvm_runtime with vulkan? How do you specified Vulkan SDK?

–edit–

I have temporary fixed problem. I’m not sure about further consequences… But if you want:

In FindVulkan.cmake remove return statement in mentioned condition
Remove/Comment-out last top-level condition if(Vulkan_FOUND).
To cmake command add something like -DVulkan_LIBRARY=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/aarch64-linux-android/24/libvulkan.so
Cmake output:

-- Android Vulkan_INCLUDE_DIRS=/home/piotr/android/sdk/ndk/26.1.10909125/sources/third_party/vulkan/src/include
-- Skip finding SPIRV in Android, make sure you only build tvm runtime.

Make output:

[ 71%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/static_library.cc.o
[ 71%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/system_library.cc.o
[ 71%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/thread_pool.cc.o
[ 71%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/threading_backend.cc.o
[ 71%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/vm/bytecode.cc.o
[ 71%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/vm/executable.cc.o
[ 71%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/vm/vm.cc.o
[ 71%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/workspace_pool.cc.o
[ 71%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/disco/distributed/socket_session.cc.o
....                                    
[ 71%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/vulkan/vulkan_amdrgp.cc.o
[ 85%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/vulkan/vulkan_buffer.cc.o
[ 85%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/vulkan/vulkan_common.cc.o
[ 85%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/vulkan/vulkan_device.cc.o
[ 85%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/vulkan/vulkan_device_api.cc.o
[ 85%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/vulkan/vulkan_instance.cc.o
[ 85%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/vulkan/vulkan_module.cc.o
[ 85%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/vulkan/vulkan_stream.cc.o
[ 85%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/vulkan/vulkan_wrapped_func.cc.o
[ 85%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/contrib/random/random.cc.o
[100%] Building CXX object tvm/CMakeFiles/tvm_runtime_objs.dir/src/runtime/contrib/sort/sort.cc.o
[100%] Built target tvm_runtime_objs
[100%] Linking CXX shared library libtvm_runtime.so
[100%] Built target tvm_runtime
➜  build

–edit2–

I had this error:

libc++abi: terminating due to uncaught exception of type tvm::runtime::InternalError: [12:44:05] 
/tvm/src/runtime/library_module.cc:122: 
Binary was created using {vulkan} but a loader of that name is not registered. 
Available loaders are static_library, relax.Executable, metadata_module, VMExecutable, metadata, const_loader. 
Perhaps you need to recompile with this runtime enabled.
Stack trace not available when DMLC_LOG_STACK_TRACE is disabled at compile time.

Now I have:

Build fingerprint: 'HUAWEI/ADY-L29/HWADY:12/HUAWEIADY-L29/104.2.0.161C432:user/release-keys'
#00 0x0000000000098cb0 /apex/com.android.runtime/lib64/bionic/libc.so (abort+164) (BuildId: 82f181b5d9da8999785cca84af3dad4b)
#01 0x00000000012ad014 /vendor/lib64/libbishenggpucompiler_v200.so (BuildId: a9d776435b8c2dae5d84700b7af45276)
#02 0x00000000011c87bc /vendor/lib64/libbishenggpucompiler_v200.so (BuildId: a9d776435b8c2dae5d84700b7af45276)
#03 0x00000000011be0b4 /vendor/lib64/libbishenggpucompiler_v200.so (BuildId: a9d776435b8c2dae5d84700b7af45276)
#04 0x00000000011b4c64 /vendor/lib64/libbishenggpucompiler_v200.so (BuildId: a9d776435b8c2dae5d84700b7af45276)
#05 0x0000000001319a54 /vendor/lib64/libbishenggpucompiler_v200.so (BuildId: a9d776435b8c2dae5d84700b7af45276)
#06 0x00000000013159ac /vendor/lib64/libbishenggpucompiler_v200.so (BuildId: a9d776435b8c2dae5d84700b7af45276)
#07 0x000000000177de70 /vendor/lib64/libbishenggpucompiler_v200.so (BuildId: a9d776435b8c2dae5d84700b7af45276)
#08 0x00000000012efbd8 /vendor/lib64/libbishenggpucompiler_v200.so (BuildId: a9d776435b8c2dae5d84700b7af45276)
#09 0x00000000012ef5dc /vendor/lib64/libbishenggpucompiler_v200.so (BuildId: a9d776435b8c2dae5d84700b7af45276)
#10 0x00000000013c4ce0 /vendor/lib64/libbishenggpucompiler_v200.so (BuildId: a9d776435b8c2dae5d84700b7af45276)
#11 0x0000000001726d80 /vendor/lib64/libbishenggpucompiler_v200.so (BuildId: a9d776435b8c2dae5d84700b7af45276)
#12 0x00000000017255e4 /vendor/lib64/libbishenggpucompiler_v200.so (BuildId: a9d776435b8c2dae5d84700b7af45276)
#13 0x000000000176c0f0 /vendor/lib64/libbishenggpucompiler_v200.so (BuildId: a9d776435b8c2dae5d84700b7af45276)
#14 0x000000000176b664 /vendor/lib64/libbishenggpucompiler_v200.so (BuildId: a9d776435b8c2dae5d84700b7af45276)
#15 0x000000000176b4f0 /vendor/lib64/libbishenggpucompiler_v200.so (BuildId: a9d776435b8c2dae5d84700b7af45276)
#16 0x000000000187df74 /vendor/lib64/libbishenggpucompiler_v200.so (cmpbe_v2_compile_multiple_shaders+7552) (BuildId: a9d776435b8c2dae5d84700b7af45276)
#17 0x00000000001277ec /vendor/lib64/hw/vulkan.hvgr_v200.so (BuildId: 2dd0d5421137d613f3cad2d5a5fc4e37)
#18 0x0000000000122ce4 /vendor/lib64/hw/vulkan.hvgr_v200.so (BuildId: 2dd0d5421137d613f3cad2d5a5fc4e37)
#19 0x0000000000122700 /vendor/lib64/hw/vulkan.hvgr_v200.so (BuildId: 2dd0d5421137d613f3cad2d5a5fc4e37)
#20 0x000000000036a840 /vendor/lib64/hw/vulkan.hvgr_v200.so (BuildId: 2dd0d5421137d613f3cad2d5a5fc4e37)
#21 0x00000000003547ec /vendor/lib64/hw/vulkan.hvgr_v200.so (BuildId: 2dd0d5421137d613f3cad2d5a5fc4e37)
#22 0x00000000002c79d0 /vendor/lib64/hw/vulkan.hvgr_v200.so (BuildId: 2dd0d5421137d613f3cad2d5a5fc4e37)
#23 0x00000000002ecfd0 /data/local/tmp/mlc/libtvm_runtime.so
#24 0x00000000002ec020 /data/local/tmp/mlc/libtvm_runtime.so
#25 0x00000000002f2578 /data/local/tmp/mlc/libtvm_runtime.so
#26 0x00000000002f2484 /data/local/tmp/mlc/libtvm_runtime.so
#27 0x00000000001645a0 /data/local/tmp/mlc/libtvm_runtime.so
#28 0x00000000003faf48 /data/local/tmp/mlc/gpt2-medium-q0f16-vulkan-aarch64.so
#29 0x00000000003ee650 /data/local/tmp/mlc/gpt2-medium-q0f16-vulkan-aarch64.so
#30 0x00000000001cc068 /data/local/tmp/mlc/libtvm_runtime.so
#31 0x000000000028b4a8 /data/local/tmp/mlc/libtvm_runtime.so
#32 0x000000000028e634 /data/local/tmp/mlc/libtvm_runtime.so
#33 0x000000000028d8bc /data/local/tmp/mlc/libtvm_runtime.so
#34 0x000000000028d0e8 /data/local/tmp/mlc/libtvm_runtime.so
#35 0x0000000000295d20 /data/local/tmp/mlc/libtvm_runtime.so
#36 0x000000000028b684 /data/local/tmp/mlc/libtvm_runtime.so
#37 0x00000000002969f4 /data/local/tmp/mlc/libtvm_runtime.so
#38 0x000000000037a538 /data/local/tmp/mlc/llm_benchmark
tvm::runtime::PackedFuncObj::CallPacked(tvm::runtime::TVMArgs, tvm::runtime::TVMRetValue*) const
/home/piotr/projects/odai-meta/mlc-llm-519/3rdparty/tvm/include/tvm/runtime/packed_func.h:1256:3
tvm::runtime::TVMRetValue tvm::runtime::PackedFunc::operator()<tvm::runtime::NDArray&, tvm::runtime::NDArray&, tvm::runtime::NDArray&>(tvm::runtime::NDArray&, tvm::runtime::NDArray&, tvm::runtime::NDArray&) const
/home/piotr/projects/odai-meta/mlc-llm-519/3rdparty/tvm/include/tvm/runtime/packed_func.h:1784:9
mlc::llm::serve::GPUSampler::SampleOnGPU(tvm::runtime::NDArray, tvm::runtime::NDArray, tvm::runtime::NDArray, bool, bool, int, std::__ndk1::vector<int, std::__ndk1::allocator<int>> const&)
/home/piotr/projects/odai-meta/mlc-llm-519/cpp/serve/sampler/gpu_sampler.cc:582:36
#39 0x0000000000378cd0 /data/local/tmp/mlc/llm_benchmark
mlc::llm::serve::GPUSampler::ChunkSampleTokensImpl(tvm::runtime::NDArray, std::__ndk1::vector<int, std::__ndk1::allocator<int>> const&, tvm::runtime::Array<mlc::llm::serve::GenerationConfig, void> const&, std::__ndk1::vector<mlc::llm::RandomGenerator*, std::__ndk1::allocator<mlc::llm::RandomGenerator*>> const&, bool)
/home/piotr/projects/odai-meta/mlc-llm-519/cpp/serve/sampler/gpu_sampler.cc:464:9
#40 0x00000000003785e4 /data/local/tmp/mlc/llm_benchmark
mlc::llm::serve::GPUSampler::BatchSampleTokensImpl(tvm::runtime::NDArray, std::__ndk1::vector<int, std::__ndk1::allocator<int>> const&, tvm::runtime::Array<tvm::runtime::String, void> const&, tvm::runtime::Array<mlc::llm::serve::GenerationConfig, void> const&, std::__ndk1::vector<mlc::llm::RandomGenerator*, std::__ndk1::allocator<mlc::llm::RandomGenerator*>> const&, bool)
/home/piotr/projects/odai-meta/mlc-llm-519/cpp/serve/sampler/gpu_sampler.cc:384:24
#41 0x0000000000375aec /data/local/tmp/mlc/llm_benchmark
mlc::llm::serve::GPUSampler::BatchSampleTokensWithProbAfterTopP(tvm::runtime::NDArray, std::__ndk1::vector<int, std::__ndk1::allocator<int>> const&, tvm::runtime::Array<tvm::runtime::String, void> const&, tvm::runtime::Array<mlc::llm::serve::GenerationConfig, void> const&, std::__ndk1::vector<mlc::llm::RandomGenerator*, std::__ndk1::allocator<mlc::llm::RandomGenerator*>> const&)
/home/piotr/projects/odai-meta/mlc-llm-519/cpp/serve/sampler/gpu_sampler.cc:197:12
#42 0x00000000003148cc /data/local/tmp/mlc/llm_benchmark
mlc::llm::serve::NewRequestPrefillActionObj::Step(mlc::llm::serve::EngineState)
/home/piotr/projects/odai-meta/mlc-llm-519/cpp/serve/engine_actions/new_request_prefill.cc:245:58
#43 0x00000000002d3270 /data/local/tmp/mlc/llm_benchmark
mlc::llm::serve::EngineImpl::Step()
/home/piotr/projects/odai-meta/mlc-llm-519/cpp/serve/engine.cc:650:38
#44 0x000000000037f954 /data/local/tmp/mlc/llm_benchmark
mlc::llm::serve::ThreadedEngineImpl::RunBackgroundLoop()
/home/piotr/projects/odai-meta/mlc-llm-519/cpp/serve/threaded_engine.cc:182:29
#45 0x0000000000213b70 /data/local/tmp/mlc/llm_benchmark
benchmark_llm(std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&, std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&, std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&)::$_1::operator()() const
/home/piotr/projects/odai-meta/mlc-llm-519/cpp/llm_benchmark.cpp:64:17
decltype(std::declval<benchmark_llm(std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&, std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&, std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&)::$_1>()()) std::__ndk1::__invoke[abi:v170000]<benchmark_llm(std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&, std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&, std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&)::$_1>(benchmark_llm(std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&, std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&, std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&)::$_1&&)
/home/piotr/android/sdk/ndk/26.1.10909125/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/__functional/invoke.h:394:23
void std::__ndk1::__thread_execute[abi:v170000]<std::__ndk1::unique_ptr<std::__ndk1::__thread_struct, std::__ndk1::default_delete<std::__ndk1::__thread_struct>>, benchmark_llm(std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&, std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&, std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&)::$_1>(std::__ndk1::tuple<std::__ndk1::unique_ptr<std::__ndk1::__thread_struct, std::__ndk1::default_delete<std::__ndk1::__thread_struct>>, benchmark_llm(std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&, std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&, std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&)::$_1>&, std::__ndk1::__tuple_indices<>)
/home/piotr/android/sdk/ndk/26.1.10909125/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/thread:282:5
void* std::__ndk1::__thread_proxy[abi:v170000]<std::__ndk1::tuple<std::__ndk1::unique_ptr<std::__ndk1::__thread_struct, std::__ndk1::default_delete<std::__ndk1::__thread_struct>>, benchmark_llm(std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&, std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&, std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char>> const&)::$_1>>(void*)
/home/piotr/android/sdk/ndk/26.1.10909125/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/thread:293:5
#46 0x00000000000fc370 /apex/com.android.runtime/lib64/bionic/libc.so (__pthread_start(void*)+176) (BuildId: 82f181b5d9da8999785cca84af3dad4b)
#47 0x000000000009a4fc /apex/com.android.runtime/lib64/bionic/libc.so (__start_thread+40) (BuildId: 82f181b5d9da8999785cca84af3dad4b)
Crash dump is completed

Similar setup on vulkan, on x86 is working.

–edit3–

I think the problem is with spirv. It has sources, and headers, and not-cmake in NDK. I think it can be build with similarly to desribet here Vulkan shader compilers on Android | Android NDK | Android Developers. I have compiled SPIRV-Tools, but the error is the same. Btw. this are lines in cmake to include SPIRV:

SET(VULKAN_LIBRARY_PATH "${ANDROID_NDK}/sources/third_party/shaderc/third_party/spirv-tools")
SET(Vulkan_INCLUDE_DIRS "${ANDROID_NDK}/sources/third_party/shaderc/third_party/spirv-tools/include")
find_library(Vulkan_SPIRV_TOOLS_LIBRARY SPIRV-Tools
HINTS ${__use_khronos_spirv}/lib ${VULKAN_LIBRARY_PATH} ${VULKAN_LIBRARY_PATH}/spirv-tools ${VULKAN_SDK}/lib ${VULKAN_LIBRARY_PATH}/obj/local/arm64-v8a/)
find_path(_libspirv libspirv.h HINTS ${__use_khronos_spirv}/include ${Vulkan_INCLUDE_DIRS}/ PATH_SUFFIXES vulkan spirv-tools)
find_path(_spirv spirv.hpp HINTS ${__use_khronos_spirv}/include ${VULKAN_LIBRARY_PATH}/external/spirv-headers/include/ PATH_SUFFIXES vulkan SPIRV spirv/unified1 spirv-headers)
find_path(_glsl_std GLSL.std.450.h HINTS ${VULKAN_LIBRARY_PATH}/external/spirv-headers/include/ PATH_SUFFIXES vulkan SPIRV spirv/unified1 spirv-headers)

srkreddy1238 · October 2, 2024, 2:58am

We are working on Vulkan recent times to make its performance on par with OpenCL (Texture ) for Adreno targets. There are few hacks we used for making it work for Adreno. You expect a formal PR soon here …