Go version
$ go version go version go1.20 darwin/arm64
Operating system and Processor architecture?
$ go env GO111MODULE="on" GOARCH="arm64" GOBIN="" GOEXE="" GOEXPERIMENT="" GOFLAGS="" GOHOSTARCH="arm64" GOHOSTOS="darwin" GOINSECURE="" GONOPROXY="" GONOSUMDB="" GOOS="darwin" GOPRIVATE="" GOPROXY="https://proxy.golang.org,direct" GOROOT="/usr/local/go" GOSUMDB="sum.golang.org" GOTMPDIR="" GOTOOLDIR="/usr/local/go/pkg/tool/darwin_arm64" GOVCS="" GOVERSION="go1.20" GCCGO="gccgo" AR="ar" CC="clang" CXX="clang++" CGO_ENABLED="1" GOMOD="/Users/me/GolandProjects/0learning/faiss-cgo-kmeans/go.mod" GOWORK="" CGO_CFLAGS="-O2 -g" CGO_CPPFLAGS="" CGO_CXXFLAGS="-O2 -g" CGO_FFLAGS="-O2 -g" CGO_LDFLAGS="-O2 -g" PKG_CONFIG="pkg-config"
Summary
I am trying to write a self-dependent go-faiss wrapper for invoking the KMEANS function of FAISS. Repo: https://github.com/arjunsk/faiss-cgo-kmeans
This approach is inspired from https://github.com/andyalm/faissmask/tree/master/FaissMask/runtimes
The FAISS static library libfaiss_c.a is built using -DBUILD_SHARED_LIBS=OFF
When I link that library with CGO, I am getting
Error Log# faiss-go/pkg/ivf.test
/usr/local/go/pkg/tool/darwin_arm64/link: running clang failed: exit status 1
Undefined symbols for architecture arm64:
"faiss::Clustering::Clustering(int, int)", referenced from:
_faiss_Clustering_new in libfaiss_c.a(Clustering_c.cpp.o)
"faiss::Clustering::Clustering(int, int, faiss::ClusteringParameters const&)", referenced from:
_faiss_Clustering_new_with_params in libfaiss_c.a(Clustering_c.cpp.o)
"faiss::kmeans_clustering(unsigned long, unsigned long, unsigned long, float const*, float*)", referenced from:
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
"faiss::ClusteringParameters::ClusteringParameters()", referenced from:
_faiss_ClusteringParameters_init in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_new_with_params in libfaiss_c.a(Clustering_c.cpp.o)
"std::exception_ptr::exception_ptr(std::exception_ptr const&)", referenced from:
_faiss_get_last_error in libfaiss_c.a(error_impl.cpp.o)
"std::exception_ptr::~exception_ptr()", referenced from:
_faiss_get_last_error in libfaiss_c.a(error_impl.cpp.o)
thread-local wrapper routine for faiss_last_exception in libfaiss_c.a(error_impl.cpp.o)
_faiss_Clustering_new in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_new_with_params in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering.cold.1 in libfaiss_c.a(Clustering_c.cpp.o)
...
"std::exception_ptr::operator=(std::exception_ptr const&)", referenced from:
_faiss_Clustering_new in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_new_with_params in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering.cold.1 in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering.cold.2 in libfaiss_c.a(Clustering_c.cpp.o)
"std::runtime_error::runtime_error(char const*)", referenced from:
_faiss_Clustering_new in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_new_with_params in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
"std::runtime_error::runtime_error(std::runtime_error const&)", referenced from:
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::runtime_error>(std::runtime_error) in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
"std::runtime_error::~runtime_error()", referenced from:
_faiss_Clustering_new in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::runtime_error>(std::runtime_error) in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_new_with_params in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering.cold.2 in libfaiss_c.a(Clustering_c.cpp.o)
"std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>::basic_string(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&)", referenced from:
std::exception_ptr std::make_exception_ptr[abi:v15006]<faiss::FaissException>(faiss::FaissException) in libfaiss_c.a(Clustering_c.cpp.o)
faiss::FaissException::FaissException(faiss::FaissException const&) in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
"std::exception::~exception()", referenced from:
_faiss_Clustering_new in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::exception>(std::exception) in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_new_with_params in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering.cold.1 in libfaiss_c.a(Clustering_c.cpp.o)
"std::exception::~exception()", referenced from:
std::exception_ptr std::make_exception_ptr[abi:v15006]<faiss::FaissException>(faiss::FaissException) in libfaiss_c.a(Clustering_c.cpp.o)
faiss::FaissException::FaissException(faiss::FaissException const&) in libfaiss_c.a(Clustering_c.cpp.o)
faiss::FaissException::~FaissException() in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
"std::current_exception()", referenced from:
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::runtime_error>(std::runtime_error) in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::exception>(std::exception) in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<faiss::FaissException>(faiss::FaissException) in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
"std::rethrow_exception(std::exception_ptr)", referenced from:
_faiss_get_last_error in libfaiss_c.a(error_impl.cpp.o)
"std::terminate()", referenced from:
___clang_call_terminate in libfaiss_c.a(Clustering_c.cpp.o)
"typeinfo for faiss::FaissException", referenced from:
std::exception_ptr std::make_exception_ptr[abi:v15006]<faiss::FaissException>(faiss::FaissException) in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
GCC_except_table21 in libfaiss_c.a(Clustering_c.cpp.o)
GCC_except_table28 in libfaiss_c.a(Clustering_c.cpp.o)
GCC_except_table29 in libfaiss_c.a(Clustering_c.cpp.o)
GCC_except_table31 in libfaiss_c.a(Clustering_c.cpp.o)
...
"typeinfo for std::runtime_error", referenced from:
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::runtime_error>(std::runtime_error) in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
"typeinfo for std::exception", referenced from:
GCC_except_table0 in libfaiss_c.a(error_impl.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::exception>(std::exception) in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
GCC_except_table21 in libfaiss_c.a(Clustering_c.cpp.o)
GCC_except_table28 in libfaiss_c.a(Clustering_c.cpp.o)
GCC_except_table29 in libfaiss_c.a(Clustering_c.cpp.o)
...
"vtable for faiss::FaissException", referenced from:
std::exception_ptr std::make_exception_ptr[abi:v15006]<faiss::FaissException>(faiss::FaissException) in libfaiss_c.a(Clustering_c.cpp.o)
faiss::FaissException::FaissException(faiss::FaissException const&) in libfaiss_c.a(Clustering_c.cpp.o)
faiss::FaissException::~FaissException() in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
NOTE: a missing vtable usually means the first non-inline virtual member function has no definition.
"vtable for std::exception", referenced from:
_faiss_Clustering_new in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::exception>(std::exception) in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_new_with_params in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
NOTE: a missing vtable usually means the first non-inline virtual member function has no definition.
"operator delete(void*)", referenced from:
_faiss_Clustering_new in libfaiss_c.a(Clustering_c.cpp.o)
faiss::FaissException::~FaissException() in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_new_with_params in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
"operator new(unsigned long)", referenced from:
_faiss_Clustering_new in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_new_with_params in libfaiss_c.a(Clustering_c.cpp.o)
"___cxa_allocate_exception", referenced from:
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::runtime_error>(std::runtime_error) in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::exception>(std::exception) in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<faiss::FaissException>(faiss::FaissException) in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
"___cxa_begin_catch", referenced from:
_faiss_get_last_error in libfaiss_c.a(error_impl.cpp.o)
_faiss_Clustering_new in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::runtime_error>(std::runtime_error) in libfaiss_c.a(Clustering_c.cpp.o)
___clang_call_terminate in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::exception>(std::exception) in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<faiss::FaissException>(faiss::FaissException) in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_new_with_params in libfaiss_c.a(Clustering_c.cpp.o)
...
"___cxa_end_catch", referenced from:
_faiss_get_last_error in libfaiss_c.a(error_impl.cpp.o)
_faiss_Clustering_new in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::runtime_error>(std::runtime_error) in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::exception>(std::exception) in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<faiss::FaissException>(faiss::FaissException) in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_new_with_params in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
...
"___cxa_free_exception", referenced from:
std::exception_ptr std::make_exception_ptr[abi:v15006]<faiss::FaissException>(faiss::FaissException) in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
"___cxa_throw", referenced from:
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::runtime_error>(std::runtime_error) in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::exception>(std::exception) in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<faiss::FaissException>(faiss::FaissException) in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_train in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_kmeans_clustering in libfaiss_c.a(Clustering_c.cpp.o)
"___gxx_personality_v0", referenced from:
_faiss_get_last_error in libfaiss_c.a(error_impl.cpp.o)
_faiss_Clustering_new in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::runtime_error>(std::runtime_error) in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<std::exception>(std::exception) in libfaiss_c.a(Clustering_c.cpp.o)
std::exception_ptr std::make_exception_ptr[abi:v15006]<faiss::FaissException>(faiss::FaissException) in libfaiss_c.a(Clustering_c.cpp.o)
faiss::FaissException::FaissException(faiss::FaissException const&) in libfaiss_c.a(Clustering_c.cpp.o)
_faiss_Clustering_new_with_params in libfaiss_c.a(Clustering_c.cpp.o)
...
ld: symbol(s) not found for architecture arm64
clang: error: linker command failed with exit code 1 (use -v to see invocation)
I am using these CGO headers
/*
#cgo darwin LDFLAGS: -L${SRCDIR}/../../cgo/thirdparty/runtimes/osx-arm64/native -lfaiss_c
#include <stdlib.h>
#include <faiss/c_api/Clustering_c.h>
#include <faiss/c_api/impl/AuxIndexStructures_c.h>
#include <faiss/c_api/index_factory_c.h>
#include <faiss/c_api/error_c.h>
*/
However, this CGO code works when we build libfaiss_c.dylib using -DBUILD_SHARED_LIBS=ON and the libfaiss_c.dylib is added to the /usr/local/lib. The CGO header would now look like the below
/*
#cgo LDFLAGS: -lfaiss_c
#include <stdlib.h>
#include <faiss/c_api/Clustering_c.h>
#include <faiss/c_api/impl/AuxIndexStructures_c.h>
#include <faiss/c_api/index_factory_c.h>
#include <faiss/c_api/error_c.h>
*/
Please help me in linking the *.a file with CGO from a custom path.
Update 1
When I have these files in the /usr/local/lib
The test runs
These files written in the /usr/local/lib is created using the below make file
GIT_COMMAND ?= git
FAISS_COMMIT ?= d87888b13e7eb339bb9c45825e9d20def6665171
all: faiss
.PHONY: faiss
faiss:
if [ -e faiss ]; then \
cd faiss && ${GIT_COMMAND} fetch origin && git reset --hard $(FAISS_COMMIT); \
else \
${GIT_COMMAND} clone https://github.com/facebookresearch/faiss.git && cd faiss && git reset --hard $(FAISS_COMMIT); \
fi
cd faiss && \
export CMAKE_PREFIX_PATH=/opt/homebrew/opt/openblas:/opt/homebrew/opt/libomp:/opt/homebrew && \
cmake -B build -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_C_API=ON -DBUILD_SHARED_LIBS=ON -DFAISS_ENABLE_PYTHON=OFF . && \
make -C build && \
sudo make -C build install && \
sudo cp build/c_api/libfaiss_c.dylib /usr/local/lib/libfaiss_c.dylib
Update 2
For the go-rocks db library, https://github.com/linxGnu/grocksdb#build
they also recommend installing/building the rocksdb library locally and linking that with the cgo interface.
Will this approach of building static library "*.a" independently and shipping it as a self-contained library work?
Aucun commentaire:
Enregistrer un commentaire