[Solved]Undefined symbol: cudaGetDevice

Questions
#1

When trying to execute “Get Started with TVM” example, following error occurred:

import tvm
import numpy as np
# Global declarations of environment.
tgt_host="llvm"
# Change it to respective GPU if gpu is enabled Ex: cuda, opencl
tgt="cuda"
n = tvm.var("n")
A = tvm.placeholder((n,), name='A')
B = tvm.placeholder((n,), name='B')
C = tvm.compute(A.shape, lambda i: A[i] + B[i], name="C")
print(type(C))
s = tvm.create_schedule(C.op)
bx, tx = s[C].split(C.op.axis[0], factor=64)
if tgt == "cuda":
    s[C].bind(bx, tvm.thread_axis("blockIdx.x"))
    s[C].bind(tx, tvm.thread_axis("threadIdx.x"))
fadd = tvm.build(s, [A, B, C], tgt, target_host=tgt_host, name="myadd")

Traceback (most recent call last):
  File "/usr/lib/python3.5/code.py", line 91, in runcode
    exec(code, self.locals)
  File "<input>", line 1, in <module>
  File "/home/chicheng/ide/pycharm-2018.1.4/helpers/pydev/_pydev_bundle/pydev_import_hook.py", line 19, in do_import
    module = self._system_import(name, *args, **kwargs)
  File "/home/chicheng/.local/lib/python3.5/site-packages/tvm-0.4.0-py3.5-linux-x86_64.egg/tvm/__init__.py", line 5, in <module>
    from . import tensor
  File "/home/chicheng/ide/pycharm-2018.1.4/helpers/pydev/_pydev_bundle/pydev_import_hook.py", line 19, in do_import
    module = self._system_import(name, *args, **kwargs)
  File "/home/chicheng/ide/pycharm-2018.1.4/helpers/pydev/_pydev_bundle/pydev_import_hook.py", line 19, in do_import
    module = self._system_import(name, *args, **kwargs)
  File "/home/chicheng/.local/lib/python3.5/site-packages/tvm-0.4.0-py3.5-linux-x86_64.egg/tvm/tensor.py", line 4, in <module>
    from ._ffi.node import NodeBase, NodeGeneric, register_node, convert_to_node
  File "/home/chicheng/ide/pycharm-2018.1.4/helpers/pydev/_pydev_bundle/pydev_import_hook.py", line 19, in do_import
    module = self._system_import(name, *args, **kwargs)
  File "/home/chicheng/.local/lib/python3.5/site-packages/tvm-0.4.0-py3.5-linux-x86_64.egg/tvm/_ffi/node.py", line 8, in <module>
    from .node_generic import NodeGeneric, convert_to_node, const
  File "/home/chicheng/ide/pycharm-2018.1.4/helpers/pydev/_pydev_bundle/pydev_import_hook.py", line 19, in do_import
    module = self._system_import(name, *args, **kwargs)
  File "/home/chicheng/.local/lib/python3.5/site-packages/tvm-0.4.0-py3.5-linux-x86_64.egg/tvm/_ffi/node_generic.py", line 7, in <module>
    from .base import string_types
  File "/home/chicheng/ide/pycharm-2018.1.4/helpers/pydev/_pydev_bundle/pydev_import_hook.py", line 19, in do_import
    module = self._system_import(name, *args, **kwargs)
  File "/home/chicheng/.local/lib/python3.5/site-packages/tvm-0.4.0-py3.5-linux-x86_64.egg/tvm/_ffi/base.py", line 43, in <module>
    _LIB, _LIB_NAME = _load_lib()
  File "/home/chicheng/.local/lib/python3.5/site-packages/tvm-0.4.0-py3.5-linux-x86_64.egg/tvm/_ffi/base.py", line 35, in _load_lib
    lib = ctypes.CDLL(lib_path[0], ctypes.RTLD_GLOBAL)
  File "/usr/lib/python3.5/ctypes/__init__.py", line 347, in __init__
    self._handle = _dlopen(self._name, mode)
OSError: /home/chicheng/.local/lib/python3.5/site-packages/tvm-0.4.0-py3.5-linux-x86_64.egg/tvm/libtvm.so: undefined symbol: cudaGetDevice


nm -g libtvm.so | grep cuda
                 U cudaDeviceGetAttribute
                 U cudaEventCreate
                 U cudaEventDestroy
                 U cudaEventRecord
                 U cudaFree
                 U cudaGetDevice
                 U cudaGetDeviceProperties
                 U cudaGetErrorString
                 U cudaMalloc
                 U cudaMemcpy
                 U cudaMemcpyAsync
                 U cudaMemcpyPeerAsync
                 U cudaSetDevice
                 U cudaStreamCreate
                 U cudaStreamDestroy
                 U cudaStreamSynchronize
                 U cudaStreamWaitEvent
00000000002758f0 T _ZN3tvm6target4cudaERKSt6vectorINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESaIS7_EE
0000000000611150 W _ZN3tvm7runtime13CUDADeviceAPI7GPUCopyEPKvPvm14cudaMemcpyKindP11CUstream_st

TVM is compiled with llvm-6.0 and cuda 9.0. config.cmake is as follows:

#--------------------------------------------------------------------
#  Template custom cmake configuration for compiling
#
#  This file is used to override the build options in build.
#  If you want to change the configuration, please use the following
#  steps. Assume you are on the root directory. First copy the this
#  file so that any local changes will be ignored by git
#
#  $ mkdir build
#  $ cp cmake/config.cmake build
#
#  Next modify the according entries, and then compile by
#
#  $ cd build
#  $ cmake ..
#
#  Then buld in parallel with 8 threads
#
#  $ make -j8
#--------------------------------------------------------------------

#---------------------------------------------
# Backend runtimes.
#---------------------------------------------

# Whether enable CUDA during compile,
#
# Possible values:
# - ON: enable CUDA with cmake's auto search
# - OFF: disbale CUDA
# - /path/to/cuda: use specific path to cuda toolkit
# set(USE_CUDA OFF)
set(USE_CUDA /usr/local/cuda-9.0)

# Whether enable ROCM runtime
#
# Possible values:
# - ON: enable ROCM with cmake's auto search
# - OFF: disbale ROCM
# - /path/to/rocm: use specific path to rocm
set(USE_ROCM OFF)

# Whether enable OpenCL runtime
# set(USE_OPENCL OFF)
set(USE_OPENCL OFF)

# Whether enable Metal runtime
set(USE_METAL OFF)

# Whether enable Vulkan runtime
#
# Possible values:
# - ON: enable Vulkan with cmake's auto search
# - OFF: disbale vulkan
# - /path/to/vulkan-sdk: use specific path to vulkan-sdk
set(USE_VULKAN OFF)

# Whether enable OpenGL runtime
set(USE_OPENGL OFF)

# Whether enable RPC runtime
set(USE_RPC ON)

# Whether enable tiny embedded graph runtime.
set(USE_GRAPH_RUNTIME ON)

# Whether enable additional graph debug functions
set(USE_GRAPH_RUNTIME_DEBUG OFF)

# Whether build with LLVM support
# Requires LLVM version >= 4.0
#
# Possible values:
# - ON: enable llvm with cmake's find search
# - OFF: disbale llvm
# - /path/to/llvm-config: enable specific LLVM when multiple llvm-dev is available.
# set(USE_LLVM OFF)
set(USE_LLVM /usr/lib/llvm-6.0/bin/llvm-config)

#---------------------------------------------
# Contrib libraries
#---------------------------------------------
# Whether use BLAS, choices: openblas, mkl, atlas, apple
set(USE_BLAS none)

# /path/to/mkl: mkl root path when use mkl blas library
# set(USE_MKL_PATH /opt/intel/mkl) for UNIX
# set(USE_MKL_PATH ../IntelSWTools/compilers_and_libraries_2018/windows/mkl) for WIN32
set(USE_MKL_PATH none)

# Whether use contrib.random in runtime
set(USE_RANDOM OFF)

# Whether use NNPack
set(USE_NNPACK OFF)

# Whether use CuDNN
set(USE_CUDNN ON)

# Whether use cuBLAS
set(USE_CUBLAS ON)

# Whether use MIOpen
set(USE_MIOPEN OFF)

# Whether use MPS
set(USE_MPS OFF)

# Whether use rocBlas
set(USE_ROCBLAS OFF)

# Whether use contrib sort
set(USE_SORT OFF)

The cmake generation and compilation went through without any problem.

#2

Just solved problem by changing set(USE_CUDA /usr/local/cuda-9.0) to set(USE_CUDA on); relinked ‘/usr/local/cuda’ to ‘/usr/local/cuda-9.0’ (originally pointing to 8.0); and in cmake-gui, set CUDA_USE_STATIC_CUDA_RUNTIME to false. Looks like current cmake system does not have support to non-default cuda toolkit location.

#3

I don’t believe this should be marked as “[Solved]”.

Isn’t customized CUDA path being supported in https://github.com/dmlc/tvm/pull/1270 ?

1 Like
#4

@tqchen

I believe this issue is on this line https://github.com/dmlc/tvm/blob/master/cmake/util/FindCUDA.cmake#L35

It should be spelled as “CUDA_CUDART_LIBRARY” instead of “CUDA_CUDAT_LIBRARY”.

I can submit a PR to fix this if it’s okay with you.

2 Likes
#5