File offline-build-patches-and-use-system-simdjson.p... of Package milvus

Overview Repositories Revisions Requests Users Attributes Meta

File offline-build-patches-and-use-system-simdjson.patch of Package milvus (Revision 8227e033839cd5d2b6faef1647e0f646)

Currently displaying revision 8227e033839cd5d2b6faef1647e0f646 , Show latest

From ee669467ce9bd2137a6327b6ba6fe6d448d62487 Mon Sep 17 00:00:00 2001
From: Christian Goll <cgoll@suse.com>
Date: Tue, 30 Jul 2024 14:19:49 +0200
Subject: [PATCH] offline build patches and use system simdjson

Signed-off-by: Christian Goll <cgoll@suse.com>
---
 internal/core/CMakeLists.txt                  | 15 ++++++----
 internal/core/src/bitset/CMakeLists.txt       |  2 ++
 internal/core/src/common/FieldData.cpp        |  2 +-
 internal/core/src/common/Json.h               |  6 ----
 internal/core/src/query/CMakeLists.txt        |  2 ++
 .../src/query/visitors/ExecExprVisitor.cpp    |  1 -
 internal/core/thirdparty/CMakeLists.txt       | 30 ++++++++++++++-----
 .../core/thirdparty/knowhere/CMakeLists.txt   |  2 +-
 .../thirdparty/milvus-storage/CMakeLists.txt  |  2 +-
 .../milvus-storage_CMakeLists.txt             |  2 +-
 .../core/thirdparty/tantivy/CMakeLists.txt    |  6 ++--
 scripts/download_milvus_proto.sh              |  2 +-
 12 files changed, 44 insertions(+), 28 deletions(-)

diff --git a/internal/core/CMakeLists.txt b/internal/core/CMakeLists.txt
index 1c22ee51..3093d386 100644
--- a/internal/core/CMakeLists.txt
+++ b/internal/core/CMakeLists.txt
@@ -95,12 +95,15 @@ set( CMAKE_INSTALL_RPATH "/usr/lib" "${CMAKE_INSTALL_PREFIX}/lib" "${CMAKE_INSTA
 set( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE )
 
 # **************************** Dependencies ****************************
-list( APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/conan )
-include( ${CMAKE_BINARY_DIR}/conan/conanbuildinfo.cmake )
-set( CONAN_DISABLE_CHECK_COMPILER ON )
-conan_basic_setup( KEEP_RPATHS )
-
-set( CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${CONAN_PROTOBUF_ROOT})
+option(OFFLINEBUILD "Build in offline mode" OFF)
+if ( NOT OFFLINEBUILD )
+  list( APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/conan )
+  include( ${CMAKE_BINARY_DIR}/conan/conanbuildinfo.cmake )
+  set( CONAN_DISABLE_CHECK_COMPILER ON )
+  conan_basic_setup( KEEP_RPATHS )
+
+  set( CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${CONAN_PROTOBUF_ROOT})
+endif()
 
 include( CTest )
 include( BuildUtils )
diff --git a/internal/core/src/bitset/CMakeLists.txt b/internal/core/src/bitset/CMakeLists.txt
index 8b2137ca..98705a8c 100644
--- a/internal/core/src/bitset/CMakeLists.txt
+++ b/internal/core/src/bitset/CMakeLists.txt
@@ -39,3 +39,5 @@ elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm*")
 endif()
 
 add_library(milvus_bitset ${BITSET_SRCS})
+
+install(TARGETS milvus_bitset DESTINATION "${CMAKE_INSTALL_LIBDIR}")
diff --git a/internal/core/src/common/FieldData.cpp b/internal/core/src/common/FieldData.cpp
index 639e6678..519b5c1e 100644
--- a/internal/core/src/common/FieldData.cpp
+++ b/internal/core/src/common/FieldData.cpp
@@ -22,7 +22,7 @@
 #include "common/Exception.h"
 #include "common/FieldDataInterface.h"
 #include "common/Json.h"
-#include "simdjson/padded_string.h"
+#include "simdjson.h"
 
 namespace milvus {
 
diff --git a/internal/core/src/common/Json.h b/internal/core/src/common/Json.h
index 708e94de..bd42689e 100644
--- a/internal/core/src/common/Json.h
+++ b/internal/core/src/common/Json.h
@@ -28,12 +28,6 @@
 #include "common/EasyAssert.h"
 #include "simdjson.h"
 #include "fmt/core.h"
-#include "simdjson/common_defs.h"
-#include "simdjson/dom/array.h"
-#include "simdjson/dom/document.h"
-#include "simdjson/dom/element.h"
-#include "simdjson/error.h"
-#include "simdjson/padded_string.h"
 
 namespace milvus {
 using document = simdjson::ondemand::document;
diff --git a/internal/core/src/query/CMakeLists.txt b/internal/core/src/query/CMakeLists.txt
index 6e9c2e4b..1546a585 100644
--- a/internal/core/src/query/CMakeLists.txt
+++ b/internal/core/src/query/CMakeLists.txt
@@ -32,3 +32,5 @@ set(MILVUS_QUERY_SRCS
 add_library(milvus_query ${MILVUS_QUERY_SRCS})
 
 target_link_libraries(milvus_query milvus_index milvus_bitset)
+
+install(TARGETS milvus_query DESTINATION "${CMAKE_INSTALL_LIBDIR}")
diff --git a/internal/core/src/query/visitors/ExecExprVisitor.cpp b/internal/core/src/query/visitors/ExecExprVisitor.cpp
index d877bbdd..4ba21610 100644
--- a/internal/core/src/query/visitors/ExecExprVisitor.cpp
+++ b/internal/core/src/query/visitors/ExecExprVisitor.cpp
@@ -34,7 +34,6 @@
 #include "query/Relational.h"
 #include "query/Utils.h"
 #include "segcore/SegmentGrowingImpl.h"
-#include "simdjson/error.h"
 #include "query/PlanProto.h"
 #include "index/SkipIndex.h"
 #include "index/Meta.h"
diff --git a/internal/core/thirdparty/CMakeLists.txt b/internal/core/thirdparty/CMakeLists.txt
index eb1806ac..adabc53e 100644
--- a/internal/core/thirdparty/CMakeLists.txt
+++ b/internal/core/thirdparty/CMakeLists.txt
@@ -12,6 +12,7 @@
 #-------------------------------------------------------------------------------
 # Using default c and cxx compiler in our build tree
 # Thirdpart cxx and c flags
+include(FindPkgConfig)
 append_flags(CMAKE_CXX_FLAGS FLAGS "-O3 -fPIC -Wno-error -fopenmp -Wno-macro-redefined")
 
 if (NOT KNOWHERE_VERBOSE_THIRDPARTY_BUILD)
@@ -30,20 +31,35 @@ set(FETCHCONTENT_QUIET     OFF)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-add_subdirectory(knowhere)
-
 add_subdirectory(boost_ext)
-add_subdirectory(rocksdb)
-add_subdirectory(rdkafka)
-add_subdirectory(simdjson)
+if (OFFLINEBUILD)
+    find_package(RocksDB REQUIRED)
+else()
+    add_subdirectory(rocksdb)
+endif()
+if (OFFLINEBUILD)
+    pkg_check_modules(rdkafka REQUIRED)
+else()
+    add_subdirectory(rdkafka)
+endif()
+if (OFFLINEBUILD)
+    find_package(simdjson REQUIRED)
+else()
+    add_subdirectory(simdjson)
+endif()
 if (USE_OPENDAL)
     add_subdirectory(opendal)
 endif()
-add_subdirectory(tantivy)
 
+add_subdirectory(tantivy)
 add_subdirectory(milvus-storage)
+add_subdirectory(knowhere)
 
 if (LINUX)
-    add_subdirectory(jemalloc)
+    if (OFFLINEBUILD)
+        pkg_check_modules(jemalloc REQUIRED)
+    else()
+        add_subdirectory(jemalloc)
+    endif()
 endif()
 
diff --git a/internal/core/thirdparty/knowhere/CMakeLists.txt b/internal/core/thirdparty/knowhere/CMakeLists.txt
index 8df5dd71..d97f58d1 100644
--- a/internal/core/thirdparty/knowhere/CMakeLists.txt
+++ b/internal/core/thirdparty/knowhere/CMakeLists.txt
@@ -42,7 +42,7 @@ FetchContent_Declare(
         knowhere
         GIT_REPOSITORY  ${GIT_REPOSITORY}
         GIT_TAG         ${KNOWHERE_VERSION}
-        SOURCE_DIR      ${CMAKE_CURRENT_BINARY_DIR}/knowhere-src
+        SOURCE_DIR      ${CMAKE_CURRENT_SOURCE_DIR}/knowhere-src
         BINARY_DIR      ${CMAKE_CURRENT_BINARY_DIR}/knowhere-build
         DOWNLOAD_DIR    ${THIRDPARTY_DOWNLOAD_PATH} )
 
diff --git a/internal/core/thirdparty/milvus-storage/CMakeLists.txt b/internal/core/thirdparty/milvus-storage/CMakeLists.txt
index a67a7cae..c25e294e 100644
--- a/internal/core/thirdparty/milvus-storage/CMakeLists.txt
+++ b/internal/core/thirdparty/milvus-storage/CMakeLists.txt
@@ -23,7 +23,7 @@ FetchContent_Declare(
         milvus-storage
         GIT_REPOSITORY  "https://github.com/milvus-io/milvus-storage.git"
         GIT_TAG         ${MILVUS_STORAGE_VERSION}
-        SOURCE_DIR      ${CMAKE_CURRENT_BINARY_DIR}/milvus-storage-src
+        SOURCE_DIR      ${CMAKE_CURRENT_SOURCE_DIR}/milvus-storage-src
         BINARY_DIR      ${CMAKE_CURRENT_BINARY_DIR}/milvus-storage-build
         SOURCE_SUBDIR   cpp
         PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/milvus-storage_CMakeLists.txt <SOURCE_DIR>/cpp/CMakeLists.txt
diff --git a/internal/core/thirdparty/milvus-storage/milvus-storage_CMakeLists.txt b/internal/core/thirdparty/milvus-storage/milvus-storage_CMakeLists.txt
index 135765c9..62cbcc4e 100644
--- a/internal/core/thirdparty/milvus-storage/milvus-storage_CMakeLists.txt
+++ b/internal/core/thirdparty/milvus-storage/milvus-storage_CMakeLists.txt
@@ -23,7 +23,7 @@ file(GLOB_RECURSE SRC_FILES src/*.cpp src/*.cc)
 message(STATUS "SRC_FILES: ${SRC_FILES}")
 add_library(milvus-storage ${SRC_FILES})
 target_include_directories(milvus-storage PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/milvus-storage ${CMAKE_CURRENT_SOURCE_DIR}/src)
-target_link_libraries(milvus-storage PUBLIC arrow::arrow Boost::boost protobuf::protobuf AWS::aws-sdk-cpp-core glog::glog)
+target_link_libraries(milvus-storage PUBLIC Arrow::arrow_shared Boost::boost protobuf::libprotobuf glog::glog)
 if (USE_OPENDAL)
   target_link_libraries(milvus-storage PUBLIC opendal)
 endif()
diff --git a/internal/core/thirdparty/tantivy/CMakeLists.txt b/internal/core/thirdparty/tantivy/CMakeLists.txt
index f4d92892..9bdcc71f 100644
--- a/internal/core/thirdparty/tantivy/CMakeLists.txt
+++ b/internal/core/thirdparty/tantivy/CMakeLists.txt
@@ -6,8 +6,8 @@ else ()
     set(TARGET_DIR "release")
 endif ()
 
-set(TANTIVY_LIB_DIR "${CMAKE_INSTALL_PREFIX}/lib")
-set(TANTIVY_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/include")
+set(TANTIVY_LIB_DIR "$ENV{HOME}${CMAKE_INSTALL_PREFIX}/lib")
+set(TANTIVY_INCLUDE_DIR "$ENV{HOME}${CMAKE_INSTALL_PREFIX}/include")
 set(TANTIVY_NAME "libtantivy_binding${CMAKE_STATIC_LIBRARY_SUFFIX}")
 
 set(LIB_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_DIR}/${TANTIVY_NAME}")
@@ -51,7 +51,7 @@ set_target_properties(tantivy_binding
         PROPERTIES
         IMPORTED_GLOBAL TRUE
         IMPORTED_LOCATION "${TANTIVY_LIB_DIR}/${TANTIVY_NAME}"
-        INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/include")
+        INTERFACE_INCLUDE_DIRECTORIES "${TANTIVY_INCLUDE_DIR}")
 
 if (CMAKE_BUILD_TYPE STREQUAL "Debug")
         add_compile_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address)
diff --git a/scripts/download_milvus_proto.sh b/scripts/download_milvus_proto.sh
index 28919e9c..e94551a7 100755
--- a/scripts/download_milvus_proto.sh
+++ b/scripts/download_milvus_proto.sh
@@ -2,9 +2,9 @@
 
 SCRIPTS_DIR=$(dirname "$0")
 THIRD_PARTY_DIR=$SCRIPTS_DIR/../cmake_build/thirdparty
-API_VERSION=$(go list -m github.com/milvus-io/milvus-proto/go-api/v2 | awk -F' ' '{print $2}')
 
 if [ ! -d "$THIRD_PARTY_DIR/milvus-proto" ]; then
+  API_VERSION=$(go list -m github.com/milvus-io/milvus-proto/go-api/v2 | awk -F' ' '{print $2}')
   mkdir -p $THIRD_PARTY_DIR
   pushd $THIRD_PARTY_DIR
   git clone https://github.com/milvus-io/milvus-proto.git
-- 
2.43.0

Places

File offline-build-patches-and-use-system-simdjson.patch of Package milvus (Revision 8227e033839cd5d2b6faef1647e0f646)

Places