Skip to content

Commit 537d368

Browse files
pvaretpvaret
committed
Added support for systemd socket activation.
1 parent b8e09f0 commit 537d368

File tree

6 files changed

+115
-24
lines changed

6 files changed

+115
-24
lines changed

‎CMakeLists.txt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,9 @@ option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE})
8484
option(LLAMA_BUILD_SERVER "llama: build server example" ${LLAMA_STANDALONE})
8585

8686
# 3rd party libs
87-
option(LLAMA_CURL "llama: use libcurl to download model from an URL" ON)
88-
option(LLAMA_LLGUIDANCE "llama-common: include LLGuidance library for structured output in common utils" OFF)
87+
option(LLAMA_CURL "llama: use libcurl to download model from an URL" ON)
88+
option(LLAMA_LLGUIDANCE "llama-common: include LLGuidance library for structured output in common utils" OFF)
89+
option(LLAMA_SERVER_SYSTEMD "llama-server: support systemd socket activation and readiness notification (linux only)" OFF)
8990

9091
# Required for relocatable CMake package
9192
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)

‎common/CMakeLists.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,10 @@ if (LLAMA_LLGUIDANCE)
133133
set(LLAMA_COMMON_EXTRA_LIBS ${LLAMA_COMMON_EXTRA_LIBS} llguidance ${LLGUIDANCE_PLATFORM_LIBS})
134134
endif ()
135135

136+
if (UNIX AND NOT APPLE AND LLAMA_SERVER_SYSTEMD)
137+
target_compile_definitions(${TARGET} PRIVATE LLAMA_CPP_SYSTEMD_SUPPORT)
138+
endif()
139+
136140
target_include_directories(${TARGET} PUBLIC . ../vendor)
137141
target_compile_features (${TARGET} PUBLIC cxx_std_17)
138142
target_link_libraries (${TARGET} PRIVATE ${LLAMA_COMMON_EXTRA_LIBS} PUBLIC llama Threads::Threads)

‎common/arg.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2978,6 +2978,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
29782978
params.port = value;
29792979
}
29802980
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_PORT"));
2981+
#ifdef LLAMA_CPP_SYSTEMD_SUPPORT
2982+
add_opt(common_arg({ "--systemd" },
2983+
string_format("use systemd socket and readiness notification (default: %s)",
2984+
params.use_systemd ? "enabled" : "disabled"),
2985+
[](common_params & params) { params.use_systemd = true; })
2986+
.set_examples({ LLAMA_EXAMPLE_SERVER })
2987+
.set_env("LLAMA_ARG_SYSTEMD"));
2988+
#endif // LLAMA_CPP_SYSTEMD_SUPPORT
29812989
add_opt(common_arg(
29822990
{"--path"}, "PATH",
29832991
string_format("path to serve static files from (default: %s)", params.public_path.c_str()),

‎common/common.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -436,6 +436,10 @@ struct common_params {
436436
int reasoning_budget = -1;
437437
bool prefill_assistant = true; // if true, any trailing assistant message will be prefilled into the response
438438

439+
#ifdef LLAMA_CPP_SYSTEMD_SUPPORT
440+
bool use_systemd = false; // use systemd socket and readiness notification
441+
#endif
442+
439443
std::vector<std::string> api_keys;
440444

441445
std::string ssl_file_key = ""; // NOLINT

‎tools/server/CMakeLists.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,12 @@ if (WIN32)
4747
TARGET_LINK_LIBRARIES(${TARGET} PRIVATE ws2_32)
4848
endif()
4949

50+
if (UNIX AND NOT APPLE AND LLAMA_SERVER_SYSTEMD)
51+
message(STATUS "LLAMA_SERVER_SYSTEMD is ON, enabling systemd support")
52+
find_package(PkgConfig REQUIRED)
53+
pkg_check_modules(SYSTEMD REQUIRED libsystemd)
54+
target_link_libraries(${TARGET} PRIVATE ${SYSTEMD_LIBRARIES})
55+
target_compile_definitions(${TARGET} PRIVATE LLAMA_CPP_SYSTEMD_SUPPORT)
56+
endif()
57+
5058
target_compile_features(${TARGET} PRIVATE cxx_std_17)

‎tools/server/server.cpp‎

Lines changed: 88 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@
3131
#include <unordered_map>
3232
#include <unordered_set>
3333

34+
#ifdef LLAMA_CPP_SYSTEMD_SUPPORT
35+
# include <sys/socket.h>
36+
# include <sys/stat.h>
37+
# include <systemd/sd-daemon.h>
38+
#endif // LLAMA_CPP_SYSTEMD_SUPPORT
39+
3440
using json = nlohmann::ordered_json;
3541

3642
constexpr int HTTP_POLLING_SECONDS = 1;
@@ -4068,6 +4074,38 @@ inline void signal_handler(int signal) {
40684074
shutdown_handler(signal);
40694075
}
40704076

4077+
#ifdef LLAMA_CPP_SYSTEMD_SUPPORT
4078+
// Subclass of httplib::Server that adds systemd socket activation support on systems
4079+
// where that's available.
4080+
class SystemdServer : public httplib::Server {
4081+
public:
4082+
bool setup_sd_socket() {
4083+
int n = sd_listen_fds(0);
4084+
if (n != 1) {
4085+
LOG_ERR("%s: sd_listen_fds() returned %d\n", __func__, n);
4086+
return false;
4087+
}
4088+
4089+
int fd = SD_LISTEN_FDS_START;
4090+
struct stat statbuf;
4091+
if (fstat(fd, &statbuf) == -1 || !S_ISSOCK(statbuf.st_mode)) {
4092+
LOG_ERR("%s: fstat() failed or fd is not a socket\n", __func__);
4093+
return false;
4094+
}
4095+
4096+
LOG_INF("%s: using systemd socket fd %d\n", __func__, fd);
4097+
svr_sock_ = fd;
4098+
return true;
4099+
}
4100+
};
4101+
#endif // LLAMA_CPP_SYSTEMD_SUPPORT
4102+
4103+
#ifdef LLAMA_CPP_SYSTEMD_SUPPORT
4104+
# define NEW_SERVER (new SystemdServer())
4105+
#else
4106+
# define NEW_SERVER (new httplib::Server())
4107+
#endif // LLAMA_CPP_SYSTEMD_SUPPORT
4108+
40714109
int main(int argc, char ** argv) {
40724110
// own arguments required by this example
40734111
common_params params;
@@ -4098,14 +4136,14 @@ int main(int argc, char ** argv) {
40984136
);
40994137
} else {
41004138
LOG_INF("Running without SSL\n");
4101-
svr.reset(new httplib::Server());
4139+
svr.reset(NEW_SERVER);
41024140
}
41034141
#else
41044142
if (params.ssl_file_key != "" && params.ssl_file_cert != "") {
41054143
LOG_ERR("Server is built without SSL support\n");
41064144
return 1;
41074145
}
4108-
svr.reset(new httplib::Server());
4146+
svr.reset(NEW_SERVER);
41094147
#endif
41104148

41114149
std::atomic<server_state> state{SERVER_STATE_LOADING_MODEL};
@@ -5280,24 +5318,38 @@ int main(int argc, char ** argv) {
52805318
};
52815319

52825320
bool was_bound = false;
5283-
bool is_sock = false;
5284-
if (string_ends_with(std::string(params.hostname), ".sock")) {
5285-
is_sock = true;
5286-
LOG_INF("%s: setting address family to AF_UNIX\n", __func__);
5287-
svr->set_address_family(AF_UNIX);
5288-
// bind_to_port requires a second arg, any value other than 0 should
5289-
// simply get ignored
5290-
was_bound = svr->bind_to_port(params.hostname, 8080);
5291-
} else {
5292-
LOG_INF("%s: binding port with default address family\n", __func__);
5293-
// bind HTTP listen port
5294-
if (params.port == 0) {
5295-
int bound_port = svr->bind_to_any_port(params.hostname);
5296-
if ((was_bound = (bound_port >= 0))) {
5297-
params.port = bound_port;
5298-
}
5321+
bool is_sock = false;
5322+
5323+
#ifdef LLAMA_CPP_SYSTEMD_SUPPORT
5324+
bool using_sd_socket = false;
5325+
if (params.use_systemd) {
5326+
was_bound = static_cast<SystemdServer *>(svr.get())->setup_sd_socket();
5327+
using_sd_socket = was_bound;
5328+
if (!was_bound) {
5329+
LOG_INF("%s: couldn't set up systemd socket; falling back to opening host:port socket\n", __func__);
5330+
}
5331+
}
5332+
#endif // LLAMA_CPP_SYSTEMD_SUPPORT
5333+
5334+
if (!was_bound) {
5335+
if (string_ends_with(std::string(params.hostname), ".sock")) {
5336+
is_sock = true;
5337+
LOG_INF("%s: setting address family to AF_UNIX\n", __func__);
5338+
svr->set_address_family(AF_UNIX);
5339+
// bind_to_port requires a second arg, any value other than 0 should
5340+
// simply get ignored
5341+
was_bound = svr->bind_to_port(params.hostname, 8080);
52995342
} else {
5300-
was_bound = svr->bind_to_port(params.hostname, params.port);
5343+
LOG_INF("%s: binding port with default address family\n", __func__);
5344+
// bind HTTP listen port
5345+
if (params.port == 0) {
5346+
int bound_port = svr->bind_to_any_port(params.hostname);
5347+
if ((was_bound = (bound_port >= 0))) {
5348+
params.port = bound_port;
5349+
}
5350+
} else {
5351+
was_bound = svr->bind_to_port(params.hostname, params.port);
5352+
}
53015353
}
53025354
}
53035355

@@ -5326,6 +5378,12 @@ int main(int argc, char ** argv) {
53265378
ctx_server.init();
53275379
state.store(SERVER_STATE_READY);
53285380

5381+
#ifdef LLAMA_CPP_SYSTEMD_SUPPORT
5382+
if (params.use_systemd) {
5383+
sd_notify(0, "READY=1");
5384+
}
5385+
#endif
5386+
53295387
LOG_INF("%s: model loaded\n", __func__);
53305388

53315389
// print sample chat example to make it clear which template is used
@@ -5360,9 +5418,17 @@ int main(int argc, char ** argv) {
53605418
SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);
53615419
#endif
53625420

5363-
LOG_INF("%s: server is listening on %s - starting the main loop\n", __func__,
5364-
is_sock ? string_format("unix://%s", params.hostname.c_str()).c_str() :
5365-
string_format("http://%s:%d", params.hostname.c_str(), params.port).c_str());
5421+
#ifdef LLAMA_CPP_SYSTEMD_SUPPORT
5422+
if (using_sd_socket) {
5423+
LOG_INF("%s: server is listening on systemd socket - starting the main loop\n", __func__);
5424+
} else {
5425+
#endif // LLAMA_CPP_SYSTEMD_SUPPORT
5426+
LOG_INF("%s: server is listening on %s - starting the main loop\n", __func__,
5427+
is_sock ? string_format("unix://%s", params.hostname.c_str()).c_str() :
5428+
string_format("http://%s:%d", params.hostname.c_str(), params.port).c_str());
5429+
#ifdef LLAMA_CPP_SYSTEMD_SUPPORT
5430+
}
5431+
#endif // LLAMA_CPP_SYSTEMD_SUPPORT
53665432

53675433
// this call blocks the main thread until queue_tasks.terminate() is called
53685434
ctx_server.queue_tasks.start_loop();

0 commit comments

Comments
 (0)