Move dpctl_ext.tensor to dpnp.tensor

.github/workflows/build-sphinx.yml

@@ -26,7 +26,7 @@ jobs:
     name: Build and Deploy Docs
 
     runs-on: ubuntu-22.04
-    timeout-minutes: 60
+    timeout-minutes: 90
 
     permissions:
       # Needed to cancel any previous runs that are not completed for a given workflow

.github/workflows/generate_coverage.yaml

@@ -11,7 +11,7 @@ jobs:
     name: Generate coverage and push to Coveralls.io
 
     runs-on: ubuntu-latest
-    timeout-minutes: 120
+    timeout-minutes: 150
 
     permissions:
       # Needed to cancel any previous runs that are not completed for a given workflow
@@ -122,7 +122,7 @@ jobs:
         uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2
         with:
           shell: bash
-          timeout_minutes: 60
+          timeout_minutes: 120
           max_attempts: 5
           retry_on: error
           command: |

.gitignore

@@ -28,11 +28,8 @@ dpnp_pytest.*
 example3
 
 *dpnp_backend*
+dpnp/include/
 dpnp/**/*.cpython*.so
 dpnp/**/*.pyd
 *~
 core
-
-# TODO: revert to `dpctl/`
-# when dpnp fully migrates dpctl/tensor
-dpctl_ext/**/*.cpython*.so

CMakeLists.txt

@@ -37,12 +37,23 @@ project(
 )
 
 option(DPNP_GENERATE_COVERAGE "Enable build DPNP with coverage instrumentation" OFF)
+option(
+    DPNP_TENSOR_GENERATE_COVERAGE_FOR_PYBIND11_EXTENSIONS
+    "Build dpnp tensor pybind11 offloading extensions with coverage instrumentation"
+    OFF
+)
 option(DPNP_BACKEND_TESTS "Enable building of DPNP backend test suite" OFF)
 option(
     DPNP_WITH_REDIST
     "Build DPNP assuming DPC++ redistributable is installed into Python prefix"
     OFF
 )
+option(
+    DPNP_TENSOR_OFFLOAD_COMPRESS
+    "Build using offload section compression feature of DPC++ to reduce \
+size of shared object with offloading sections"
+    OFF
+)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
@@ -344,5 +355,15 @@ if(DEFINED SKBUILD)
     set(_ignore_me ${SKBUILD})
 endif()
 
+# DpnpTensorCAPI: Interface library for dpnp tensor C-API
+# Provides access to:
+# 1. Public C-API headers from dpnp/include
+# 2. Generated Cython headers via per-target header interface libraries
+
+add_library(DpnpTensorCAPI INTERFACE)
+target_include_directories(
+    DpnpTensorCAPI
+    INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/dpnp/include
+)
+
 add_subdirectory(dpnp)
-add_subdirectory(dpctl_ext)

dpnp/CMakeLists.txt

@@ -86,11 +86,107 @@ function(build_dpnp_cython_ext _trgt _src _dest)
     install(TARGETS ${_trgt} LIBRARY DESTINATION ${_dest})
 endfunction()
 
+function(build_dpnp_tensor_ext _trgt _src _dest)
+    set(options SYCL)
+    cmake_parse_arguments(BUILD_DPNP_TENSOR "${options}" "RELATIVE_PATH" "" ${ARGN})
+    add_cython_target(${_trgt} ${_src} CXX OUTPUT_VAR _generated_src)
+    set(_cythonize_trgt "${_trgt}_cythonize_pyx")
+    python_add_library(${_trgt} MODULE WITH_SOABI ${_generated_src})
+    if(BUILD_DPNP_SYCL)
+        add_sycl_to_target(TARGET ${_trgt} SOURCES ${_generated_src})
+        target_compile_options(${_trgt} PRIVATE -fno-sycl-id-queries-fit-in-int)
+        target_link_options(${_trgt} PRIVATE -fsycl-device-code-split=per_kernel)
+        if(DPNP_TENSOR_OFFLOAD_COMPRESS)
+            target_link_options(${_trgt} PRIVATE --offload-compress)
+        endif()
+        if(_dpnp_sycl_targets)
+            # make fat binary
+            target_compile_options(
+                ${_trgt}
+                PRIVATE ${_dpnp_sycl_target_compile_options}
+            )
+            target_link_options(${_trgt} PRIVATE ${_dpnp_sycl_target_link_options})
+        endif()
+    endif()
+    target_link_libraries(${_trgt} PRIVATE Python::NumPy)
+    if(DPNP_GENERATE_COVERAGE)
+        target_compile_definitions(${_trgt} PRIVATE CYTHON_TRACE=1 CYTHON_TRACE_NOGIL=1)
+        if(BUILD_DPNP_TENSOR_SYCL)
+            target_compile_options(${_trgt} PRIVATE -fno-sycl-use-footer)
+        endif()
+    endif()
+    # Dpctl
+    target_include_directories(${_trgt} PRIVATE ${Dpctl_INCLUDE_DIR})
+    target_link_directories(${_trgt} PRIVATE ${Dpctl_INCLUDE_DIR}/..)
+    target_link_libraries(${_trgt} PRIVATE DPCTLSyclInterface)
+    set(_linker_options "LINKER:${DPNP_LDFLAGS}")
+    target_link_options(${_trgt} PRIVATE ${_linker_options})
+    get_filename_component(_name_wle ${_generated_src} NAME_WLE)
+    get_filename_component(_generated_src_dir ${_generated_src} DIRECTORY)
+    set(_generated_public_h "${_generated_src_dir}/${_name_wle}.h")
+    set(_generated_api_h "${_generated_src_dir}/${_name_wle}_api.h")
+
+    # TODO: create separate folder inside build folder that contains only
+    #   headers related to this target and appropriate folder structure to
+    #   eliminate shadow dependencies
+    # Go up two levels to build root for "dpnp/tensor/_usmarray.h" resolution
+    get_filename_component(_parent_dir ${_generated_src_dir} DIRECTORY)
+    get_filename_component(_build_root ${_parent_dir} DIRECTORY)
+    # TODO: do not set directory if we did not generate header
+    target_include_directories(${_trgt} INTERFACE ${_build_root})
+    set(_rpath_value "$ORIGIN")
+    if(BUILD_DPNP_TENSOR_RELATIVE_PATH)
+        set(_rpath_value "${_rpath_value}/${BUILD_DPNP_TENSOR_RELATIVE_PATH}")
+    endif()
+    if(DPNP_WITH_REDIST)
+        set(_rpath_value "${_rpath_value}:${_rpath_value}/../../..")
+    endif()
+    set_target_properties(${_trgt} PROPERTIES INSTALL_RPATH ${_rpath_value})
+
+    install(TARGETS ${_trgt} LIBRARY DESTINATION ${_dest})
+    install(
+        FILES ${_generated_api_h}
+        DESTINATION ${CMAKE_INSTALL_PREFIX}/dpnp/include/${_dest}
+        OPTIONAL
+    )
+    install(
+        FILES ${_generated_public_h}
+        DESTINATION ${CMAKE_INSTALL_PREFIX}/dpnp/include/${_dest}
+        OPTIONAL
+    )
+    if(DPNP_GENERATE_COVERAGE)
+        get_filename_component(_original_src_dir ${_src} DIRECTORY)
+        file(RELATIVE_PATH _rel_dir ${CMAKE_SOURCE_DIR} ${_original_src_dir})
+        install(FILES ${_generated_src} DESTINATION ${CMAKE_INSTALL_PREFIX}/${_rel_dir})
+    endif()
+
+    # Create target with headers only, because python is managing all the
+    # library imports at runtime
+    set(_trgt_headers ${_trgt}_headers)
+    add_library(${_trgt_headers} INTERFACE)
+    add_dependencies(${_trgt_headers} ${_trgt})
+    get_target_property(_trgt_headers_dir ${_trgt} INTERFACE_INCLUDE_DIRECTORIES)
+    target_include_directories(${_trgt_headers} INTERFACE ${_trgt_headers_dir})
+endfunction()
+
 function(build_dpnp_cython_ext_with_backend _trgt _src _dest)
     build_dpnp_cython_ext(${_trgt} ${_src} ${_dest})
     target_link_libraries(${_trgt} PRIVATE dpnp_backend_library)
 endfunction()
 
+# Install dpnp C-API headers
+install(
+    DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/
+    DESTINATION ${CMAKE_INSTALL_PREFIX}/dpnp/include
+    FILES_MATCHING
+    REGEX "\\.h(pp)?$"
+)
+
+# Skip tensor build for coverage to reduce memory usage on Public CI
+if(NOT DPNP_SKIP_TENSOR_BUILD)
+    add_subdirectory(tensor)
+endif()
+
 add_subdirectory(backend)
 add_subdirectory(backend/extensions/blas)
 add_subdirectory(backend/extensions/fft)

dpnp/__init__.py

@@ -28,7 +28,6 @@
 
 import os
 import sys
-import warnings
 
 mypath = os.path.dirname(os.path.realpath(__file__))
 
@@ -61,10 +60,9 @@
                 [os.getenv("PATH", ""), dll_path]
             )
 
-# Borrowed from DPCTL
-with warnings.catch_warnings():
-    warnings.simplefilter("ignore", DeprecationWarning)
-    from dpctl.tensor import __array_api_version__, DLDeviceType
+# TODO: revert to `from dpctl.tensor...`
+# when dpnp fully migrates dpctl/tensor
+from dpctl_ext.tensor import __array_api_version__, DLDeviceType
 
 from .dpnp_array import dpnp_array as ndarray
 from .dpnp_array_api_info import __array_namespace_info__

dpnp/backend/extensions/blas/CMakeLists.txt

@@ -39,6 +39,8 @@ set(_module_src
 pybind11_add_module(${python_module_name} MODULE ${_module_src})
 add_sycl_to_target(TARGET ${python_module_name} SOURCES ${_module_src})
 
+target_link_libraries(${python_module_name} PRIVATE DpnpTensorCAPI)
+
 if(_dpnp_sycl_targets)
     # make fat binary
     target_compile_options(
@@ -68,7 +70,7 @@ target_include_directories(
     PRIVATE
         ${CMAKE_CURRENT_SOURCE_DIR}/../common
         ${CMAKE_SOURCE_DIR}/dpnp/backend/include
-        ${CMAKE_SOURCE_DIR}/dpctl_ext/tensor/libtensor/include
+        ${CMAKE_SOURCE_DIR}/dpnp/tensor/libtensor/include
 )
 
 # treat below headers as system to suppress the warnings there during the build

dpnp/backend/extensions/fft/CMakeLists.txt

@@ -33,6 +33,8 @@ set(_module_src ${CMAKE_CURRENT_SOURCE_DIR}/fft_py.cpp)
 pybind11_add_module(${python_module_name} MODULE ${_module_src})
 add_sycl_to_target(TARGET ${python_module_name} SOURCES ${_module_src})
 
+target_link_libraries(${python_module_name} PRIVATE DpnpTensorCAPI)
+
 if(_dpnp_sycl_targets)
     # make fat binary
     target_compile_options(
@@ -61,7 +63,7 @@ target_include_directories(
     ${python_module_name}
     PRIVATE
         ${CMAKE_SOURCE_DIR}/dpnp/backend/include
-        ${CMAKE_SOURCE_DIR}/dpctl_ext/tensor/libtensor/include
+        ${CMAKE_SOURCE_DIR}/dpnp/tensor/libtensor/include
 )
 
 # treat below headers as system to suppress the warnings there during the build

dpnp/backend/extensions/indexing/CMakeLists.txt

@@ -36,6 +36,8 @@ set(_module_src
 pybind11_add_module(${python_module_name} MODULE ${_module_src})
 add_sycl_to_target(TARGET ${python_module_name} SOURCES ${_module_src})
 
+target_link_libraries(${python_module_name} PRIVATE DpnpTensorCAPI)
+
 if(_dpnp_sycl_targets)
     # make fat binary
     target_compile_options(
@@ -65,7 +67,7 @@ target_include_directories(
     PRIVATE
         ${CMAKE_CURRENT_SOURCE_DIR}/../common
         ${CMAKE_SOURCE_DIR}/dpnp/backend/include
-        ${CMAKE_SOURCE_DIR}/dpctl_ext/tensor/libtensor/include
+        ${CMAKE_SOURCE_DIR}/dpnp/tensor/libtensor/include
 )
 
 # treat below headers as system to suppress the warnings there during the build