Merge pull request #182 from maciekpac/release/ccl_2021.16.2

Intel(R) oneAPI Collective Communications Library (oneCCL) 2021.16.2

Author: maciekpac

CMakeLists.txt

@@ -343,7 +343,7 @@ endif()
 
 set(CCL_MAJOR_VERSION     "2021")
 set(CCL_MINOR_VERSION     "16")
-set(CCL_UPDATE_VERSION    "1")
+set(CCL_UPDATE_VERSION    "2")
 set(CCL_PRODUCT_STATUS    "Gold")
 string(TIMESTAMP CCL_PRODUCT_BUILD_DATE "%Y-%m-%dT %H:%M:%SZ")
 get_vcs_properties("git")

README.md

@@ -31,7 +31,7 @@ oneCCL is governed by the [UXL Foundation](http://www.uxlfoundation.org) and is
 - [Contribute](#contribute)
 - [License](#license)
 - [Security Policy](#security-policy)
-  
+
 ## Prerequisites 
 
 See [System Requirements](https://www.intel.com/content/www/us/en/developer/articles/system-requirements/oneapi-collective-communication-library-system-requirements.html) to learn about hardware and software requirements before getting started with oneCCL.

doc/rst/source/conf.py

@@ -139,5 +139,5 @@ def get_git_last_updated_date():
     'extra_footer': '<p align="right"><a href="https://www.intel.com/content/www/us/en/privacy/intel-cookie-notice.html">Cookies</a></p><br><div>Last updated: ' + last_updated + '</div>'
 }
 
-html_theme_options["logo"] = {"text": "oneCCL Documentation 2021.16.1"}
+html_theme_options["logo"] = {"text": "oneCCL Documentation 2021.16.2"}
 

doc/rst/source/env-variables.rst

@@ -2148,7 +2148,7 @@ Profiling
 #########
 
 CCL_ITT_LEVEL
-*************
+-------------
 
 **Syntax**
 
@@ -2174,10 +2174,10 @@ CCL_ITT_LEVEL
 
 Set this environment variable to specify Intel\ |reg|\  Instrumentation and Tracing Technology (ITT) profiling level.
 Once the environment variable is enabled (`value>0`), it is possible to collect and display profiling
-data for |product_short| using tools such as Intel\ |reg|\  VTune\ |tm|\  Profiler and `Unified Tracing and Profiling Tool <https://github.com/intel/pti-gpu/tree/master/tools/unitrace>`_.
+data for |product_short| using tools such as Intel\ |reg|\  VTune\ |tm|\  Profiler and `Unified Tracing and Profiling Tool<https://github.com/intel/pti-gpu/tree/master/tools/unitrace>_`.
 
 CCL_PROFILING_ENABLE 
-********************
+---------------------
 
 **Syntax**
 

include/oneapi/ccl/config.h

@@ -688,17 +688,11 @@ atl_status_t atl_mpi_ctx::check_impi_env(const atl_attr_t& attr) {
 
     if (!getenv("ONEAPI_ROOT") && !getenv("I_MPI_ROOT")) {
         atl_mpi_lib_type_t type = ATL_MPI_LIB_IMPI;
-        LOG_ERROR("CCL/MPI uses ",
-                  mpi_lib_infos[type].version_prefix_1,
-                  " but neither I_MPI_ROOT nor ONEAPI_ROOT is set. ",
-                  "Please source ",
-                  mpi_lib_infos[type].kind_value,
-                  " version of ",
-                  mpi_lib_infos[type].version_prefix_1,
-                  " (",
-                  mpi_lib_infos[type].min_version_value,
-                  " or higher version).");
-        return ATL_STATUS_FAILURE;
+        LOG_INFO("oneCCL MPI network transport layer is using ",
+                 mpi_lib_infos[type].version_prefix_1,
+                 " but $I_MPI_ROOT is not set.",
+                 " Transport variables will be initialized automatically.",
+                 " To override them run `source $I_MPI_ROOT/env/vars.sh`");
     }
 
     return ATL_STATUS_SUCCESS;

src/atl/mpi/atl_mpi_ctx.cpp

@@ -156,7 +156,9 @@ void print_error(int error, lib_info_t& info) {
                  " - path contains invalid characters");
     }
     else if (error == CCL_LOAD_LB_DLOPEN_ERROR) {
-        LOG_WARN("could not open the library: ", info.path.c_str(), " - ", dlerror());
+        // Log as `info`, because in some cases we have to test multiple paths
+        // and we do not want to write excess information into users screen
+        LOG_INFO("could not open the library: ", info.path.c_str(), " - ", dlerror());
     }
 }
 

src/common/api_wrapper/api_wrapper.cpp

@@ -13,9 +13,12 @@
  See the License for the specific language governing permissions and
  limitations under the License.
 */
+#include <dlfcn.h>
+#include <libgen.h>
 #include <sys/stat.h>
 
 #include "common/api_wrapper/api_wrapper.hpp"
+#include "common/log/log.hpp"
 #include "common/api_wrapper/ofi_api_wrapper.hpp"
 
 namespace ccl {
@@ -64,20 +67,105 @@ std::string get_ofi_lib_path() {
     return ofi_lib_path;
 }
 
-bool ofi_api_init() {
-    bool ret = true;
+static std::string get_relative_ccl_root_path() {
+    Dl_info info;
+
+    if (dladdr((void*)ccl::get_library_version, &info)) {
+        char libccl_path[PATH_MAX];
+
+        if (realpath(info.dli_fname, libccl_path) != nullptr) {
+            // We have to use `realpath`, so the dirname will work correctly,
+            // because if there's any symlink like `..` in the path it will not work
+            libccl_path[PATH_MAX - 1] = '\0';
+
+            // Remove `libccl.so` from the path to get directory like `$CCL_ROOT/lib`
+            char* libccl_dir = dirname(libccl_path);
+            // Remove the `lib` from the path the get just the `CCL_ROOT`
+            char* ccl_root_cstr = dirname(libccl_dir);
+
+            auto ccl_root = std::string(ccl_root_cstr);
 
+            return ccl_root;
+        }
+    }
+
+    return {};
+}
+
+static bool load_libfabric() {
     ofi_lib_info.ops = &ofi_lib_ops;
     ofi_lib_info.fn_names = ofi_fn_names;
     ofi_lib_info.path = get_ofi_lib_path();
 
     int error = load_library(ofi_lib_info);
-    if (error != CCL_LOAD_LB_SUCCESS) {
-        print_error(error, ofi_lib_info);
-        ret = false;
+    if (error == CCL_LOAD_LB_SUCCESS) {
+        return true;
+    }
+
+    print_error(error, ofi_lib_info);
+    LOG_INFO("Retrying to load libfabric.so using relative path");
+
+    auto realtive_root = get_relative_ccl_root_path();
+    if (realtive_root.empty()) {
+        return false;
+    }
+
+    // Path up to IMPI 2021.14
+    ofi_lib_info.path = realtive_root + "/lib/libfabric/libfabric.so";
+    error = load_library(ofi_lib_info);
+    if (error == CCL_LOAD_LB_SUCCESS) {
+        return true;
+    }
+
+    // Path in IMPI 2021.15
+    ofi_lib_info.path = realtive_root + "/lib/libfabric.so";
+    error = load_library(ofi_lib_info);
+    if (error == CCL_LOAD_LB_SUCCESS) {
+        return true;
+    }
+
+    print_error(error, ofi_lib_info);
+    return false;
+}
+
+static void setup_providers() {
+    const char* fi_provider_path = getenv("FI_PROVIDER_PATH");
+    if (fi_provider_path != nullptr) {
+        LOG_DEBUG("FI_PROVIDER_PATH is already set to: ", fi_provider_path);
+        return;
+    }
+
+    char libfabric_path[PATH_MAX];
+    dlinfo(ofi_lib_info.handle, RTLD_DI_ORIGIN, &libfabric_path);
+
+    // Add realpath to resolve any symlinks and get the absolute path
+    char real_libfabric_path[PATH_MAX];
+    if (!realpath(libfabric_path, real_libfabric_path)) {
+        LOG_ERROR("Failed to resolve libfabric realpath: ", strerror(errno));
+        return;
+    }
+
+    std::string primary_path = std::string(real_libfabric_path);
+    std::string secondary_path = primary_path + "/prov";
+
+    // Construct the full provider path with colon separator
+    std::string full_provider_path = primary_path + ":" + secondary_path;
+
+    if (setenv("FI_PROVIDER_PATH", full_provider_path.c_str(), 1) != 0) {
+        LOG_ERROR("Failed to set FI_PROVIDER_PATH with error: ", strerror(errno));
+        return;
+    }
+
+    LOG_DEBUG("FI_PROVIDER_PATH set to: ", full_provider_path);
+}
+
+bool ofi_api_init() {
+    if (load_libfabric() == false) {
+        return false;
     }
 
-    return ret;
+    setup_providers();
+    return true;
 }
 
 void ofi_api_fini() {

src/common/api_wrapper/ofi_api_wrapper.cpp

@@ -663,13 +663,19 @@ void env_data::parse() {
         if (ccl_root.empty()) {
             // CCL_ROOT and ONEAPI_ROOT are missing
             Dl_info info;
+
             if (dladdr((void*)ccl::get_library_version, &info)) {
                 char libccl_path[PATH_MAX];
-                std::strncpy(libccl_path, info.dli_fname, PATH_MAX - 1);
-                libccl_path[PATH_MAX - 1] = '\0';
 
-                char* libccl_dir = dirname(libccl_path);
-                ccl_root = dirname(libccl_dir);
+                if (realpath(info.dli_fname, libccl_path) != nullptr) {
+                    // We have to use `realpath`, so the dirname will work correctly
+                    libccl_path[PATH_MAX - 1] = '\0';
+
+                    char* libccl_dir = dirname(libccl_path);
+                    char* ccl_root_cstr = dirname(libccl_dir);
+
+                    ccl_root = std::string(ccl_root_cstr);
+                }
             }
         }
 

src/common/env/env.cpp

@@ -1,5 +1,5 @@
 Intel(R) oneAPI Collective Communications Library (oneCCL) 
-2021.16.1 Third Party Programs File
+2021.16.2 Third Party Programs File
 
 This file is the "third-party-programs.txt" file specified in the associated 
 Intel end user license agreement for the Intel software you are licensing.