File offline-build-patches-and-use-system-simdjson.p... of Package milvus

Overview Repositories Revisions Requests Users Attributes Meta

File offline-build-patches-and-use-system-simdjson.patch of Package milvus (Revision 894aa7533e00c27aefdee3fff5377fdb)

Currently displaying revision 894aa7533e00c27aefdee3fff5377fdb , Show latest

From 8517f1f22ddefeab9cbebc8d6b094e0e052c6748 Mon Sep 17 00:00:00 2001
From: Christian Goll <cgoll@suse.com>
Date: Wed, 24 Jul 2024 16:34:01 +0200
Subject: [PATCH] offline build patches and use system simdjson

Signed-off-by: Christian Goll <cgoll@suse.com>
---
 internal/core/CMakeLists.txt                  | 15 ++++++----
 internal/core/src/common/FieldData.cpp        |  2 +-
 internal/core/src/common/Json.h               |  6 ----
 .../src/query/visitors/ExecExprVisitor.cpp    |  1 -
 internal/core/thirdparty/CMakeLists.txt       | 30 ++++++++++++++-----
 .../core/thirdparty/knowhere/CMakeLists.txt   |  2 +-
 .../thirdparty/milvus-storage/CMakeLists.txt  |  2 +-
 .../core/thirdparty/tantivy/CMakeLists.txt    |  6 ++--
 scripts/download_milvus_proto.sh              |  2 +-
 9 files changed, 39 insertions(+), 27 deletions(-)

diff --git a/internal/core/CMakeLists.txt b/internal/core/CMakeLists.txt
index 1c22ee512..3093d3866 100644
--- a/internal/core/CMakeLists.txt
+++ b/internal/core/CMakeLists.txt
@@ -95,12 +95,15 @@ set( CMAKE_INSTALL_RPATH "/usr/lib" "${CMAKE_INSTALL_PREFIX}/lib" "${CMAKE_INSTA
 set( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE )
 
 # **************************** Dependencies ****************************
-list( APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/conan )
-include( ${CMAKE_BINARY_DIR}/conan/conanbuildinfo.cmake )
-set( CONAN_DISABLE_CHECK_COMPILER ON )
-conan_basic_setup( KEEP_RPATHS )
-
-set( CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${CONAN_PROTOBUF_ROOT})
+option(OFFLINEBUILD "Build in offline mode" OFF)
+if ( NOT OFFLINEBUILD )
+  list( APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/conan )
+  include( ${CMAKE_BINARY_DIR}/conan/conanbuildinfo.cmake )
+  set( CONAN_DISABLE_CHECK_COMPILER ON )
+  conan_basic_setup( KEEP_RPATHS )
+
+  set( CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${CONAN_PROTOBUF_ROOT})
+endif()
 
 include( CTest )
 include( BuildUtils )
diff --git a/internal/core/src/common/FieldData.cpp b/internal/core/src/common/FieldData.cpp
index 639e66785..519b5c1ef 100644
--- a/internal/core/src/common/FieldData.cpp
+++ b/internal/core/src/common/FieldData.cpp
@@ -22,7 +22,7 @@
 #include "common/Exception.h"
 #include "common/FieldDataInterface.h"
 #include "common/Json.h"
-#include "simdjson/padded_string.h"
+#include "simdjson.h"
 
 namespace milvus {
 
diff --git a/internal/core/src/common/Json.h b/internal/core/src/common/Json.h
index 708e94de2..bd42689ef 100644
--- a/internal/core/src/common/Json.h
+++ b/internal/core/src/common/Json.h
@@ -28,12 +28,6 @@
 #include "common/EasyAssert.h"
 #include "simdjson.h"
 #include "fmt/core.h"
-#include "simdjson/common_defs.h"
-#include "simdjson/dom/array.h"
-#include "simdjson/dom/document.h"
-#include "simdjson/dom/element.h"
-#include "simdjson/error.h"
-#include "simdjson/padded_string.h"
 
 namespace milvus {
 using document = simdjson::ondemand::document;
diff --git a/internal/core/src/query/visitors/ExecExprVisitor.cpp b/internal/core/src/query/visitors/ExecExprVisitor.cpp
index d877bbdd0..4ba216108 100644
--- a/internal/core/src/query/visitors/ExecExprVisitor.cpp
+++ b/internal/core/src/query/visitors/ExecExprVisitor.cpp
@@ -34,7 +34,6 @@
 #include "query/Relational.h"
 #include "query/Utils.h"
 #include "segcore/SegmentGrowingImpl.h"
-#include "simdjson/error.h"
 #include "query/PlanProto.h"
 #include "index/SkipIndex.h"
 #include "index/Meta.h"
diff --git a/internal/core/thirdparty/CMakeLists.txt b/internal/core/thirdparty/CMakeLists.txt
index eb1806ac5..adabc53eb 100644
--- a/internal/core/thirdparty/CMakeLists.txt
+++ b/internal/core/thirdparty/CMakeLists.txt
@@ -12,6 +12,7 @@
 #-------------------------------------------------------------------------------
 # Using default c and cxx compiler in our build tree
 # Thirdpart cxx and c flags
+include(FindPkgConfig)
 append_flags(CMAKE_CXX_FLAGS FLAGS "-O3 -fPIC -Wno-error -fopenmp -Wno-macro-redefined")
 
 if (NOT KNOWHERE_VERBOSE_THIRDPARTY_BUILD)
@@ -30,20 +31,35 @@ set(FETCHCONTENT_QUIET     OFF)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-add_subdirectory(knowhere)
-
 add_subdirectory(boost_ext)
-add_subdirectory(rocksdb)
-add_subdirectory(rdkafka)
-add_subdirectory(simdjson)
+if (OFFLINEBUILD)
+    find_package(RocksDB REQUIRED)
+else()
+    add_subdirectory(rocksdb)
+endif()
+if (OFFLINEBUILD)
+    pkg_check_modules(rdkafka REQUIRED)
+else()
+    add_subdirectory(rdkafka)
+endif()
+if (OFFLINEBUILD)
+    find_package(simdjson REQUIRED)
+else()
+    add_subdirectory(simdjson)
+endif()
 if (USE_OPENDAL)
     add_subdirectory(opendal)
 endif()
-add_subdirectory(tantivy)
 
+add_subdirectory(tantivy)
 add_subdirectory(milvus-storage)
+add_subdirectory(knowhere)
 
 if (LINUX)
-    add_subdirectory(jemalloc)
+    if (OFFLINEBUILD)
+        pkg_check_modules(jemalloc REQUIRED)
+    else()
+        add_subdirectory(jemalloc)
+    endif()
 endif()
 
diff --git a/internal/core/thirdparty/knowhere/CMakeLists.txt b/internal/core/thirdparty/knowhere/CMakeLists.txt
index 8df5dd71e..d97f58d18 100644
--- a/internal/core/thirdparty/knowhere/CMakeLists.txt
+++ b/internal/core/thirdparty/knowhere/CMakeLists.txt
@@ -42,7 +42,7 @@ FetchContent_Declare(
         knowhere
         GIT_REPOSITORY  ${GIT_REPOSITORY}
         GIT_TAG         ${KNOWHERE_VERSION}
-        SOURCE_DIR      ${CMAKE_CURRENT_BINARY_DIR}/knowhere-src
+        SOURCE_DIR      ${CMAKE_CURRENT_SOURCE_DIR}/knowhere-src
         BINARY_DIR      ${CMAKE_CURRENT_BINARY_DIR}/knowhere-build
         DOWNLOAD_DIR    ${THIRDPARTY_DOWNLOAD_PATH} )
 
diff --git a/internal/core/thirdparty/milvus-storage/CMakeLists.txt b/internal/core/thirdparty/milvus-storage/CMakeLists.txt
index a67a7cae8..c25e294e2 100644
--- a/internal/core/thirdparty/milvus-storage/CMakeLists.txt
+++ b/internal/core/thirdparty/milvus-storage/CMakeLists.txt
@@ -23,7 +23,7 @@ FetchContent_Declare(
         milvus-storage
         GIT_REPOSITORY  "https://github.com/milvus-io/milvus-storage.git"
         GIT_TAG         ${MILVUS_STORAGE_VERSION}
-        SOURCE_DIR      ${CMAKE_CURRENT_BINARY_DIR}/milvus-storage-src
+        SOURCE_DIR      ${CMAKE_CURRENT_SOURCE_DIR}/milvus-storage-src
         BINARY_DIR      ${CMAKE_CURRENT_BINARY_DIR}/milvus-storage-build
         SOURCE_SUBDIR   cpp
         PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/milvus-storage_CMakeLists.txt <SOURCE_DIR>/cpp/CMakeLists.txt
diff --git a/internal/core/thirdparty/tantivy/CMakeLists.txt b/internal/core/thirdparty/tantivy/CMakeLists.txt
index f4d928922..9bdcc71f1 100644
--- a/internal/core/thirdparty/tantivy/CMakeLists.txt
+++ b/internal/core/thirdparty/tantivy/CMakeLists.txt
@@ -6,8 +6,8 @@ else ()
     set(TARGET_DIR "release")
 endif ()
 
-set(TANTIVY_LIB_DIR "${CMAKE_INSTALL_PREFIX}/lib")
-set(TANTIVY_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/include")
+set(TANTIVY_LIB_DIR "$ENV{HOME}${CMAKE_INSTALL_PREFIX}/lib")
+set(TANTIVY_INCLUDE_DIR "$ENV{HOME}${CMAKE_INSTALL_PREFIX}/include")
 set(TANTIVY_NAME "libtantivy_binding${CMAKE_STATIC_LIBRARY_SUFFIX}")
 
 set(LIB_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_DIR}/${TANTIVY_NAME}")
@@ -51,7 +51,7 @@ set_target_properties(tantivy_binding
         PROPERTIES
         IMPORTED_GLOBAL TRUE
         IMPORTED_LOCATION "${TANTIVY_LIB_DIR}/${TANTIVY_NAME}"
-        INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/include")
+        INTERFACE_INCLUDE_DIRECTORIES "${TANTIVY_INCLUDE_DIR}")
 
 if (CMAKE_BUILD_TYPE STREQUAL "Debug")
         add_compile_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address)
diff --git a/scripts/download_milvus_proto.sh b/scripts/download_milvus_proto.sh
index 28919e9c9..e94551a71 100755
--- a/scripts/download_milvus_proto.sh
+++ b/scripts/download_milvus_proto.sh
@@ -2,9 +2,9 @@
 
 SCRIPTS_DIR=$(dirname "$0")
 THIRD_PARTY_DIR=$SCRIPTS_DIR/../cmake_build/thirdparty
-API_VERSION=$(go list -m github.com/milvus-io/milvus-proto/go-api/v2 | awk -F' ' '{print $2}')
 
 if [ ! -d "$THIRD_PARTY_DIR/milvus-proto" ]; then
+  API_VERSION=$(go list -m github.com/milvus-io/milvus-proto/go-api/v2 | awk -F' ' '{print $2}')
   mkdir -p $THIRD_PARTY_DIR
   pushd $THIRD_PARTY_DIR
   git clone https://github.com/milvus-io/milvus-proto.git
-- 
2.43.0

Places

File offline-build-patches-and-use-system-simdjson.patch of Package milvus (Revision 894aa7533e00c27aefdee3fff5377fdb)

Places