From cf830945738c848eea5dac0e2c8acbcdb8c3fd6d Mon Sep 17 00:00:00 2001 From: pl752 Date: Wed, 23 Apr 2025 21:08:34 +0500 Subject: [PATCH 1/5] Sigint rework in mtmd vision example --- examples/llava/mtmd-cli.cpp | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/examples/llava/mtmd-cli.cpp b/examples/llava/mtmd-cli.cpp index e80845a2c5469..8596fefcaee69 100644 --- a/examples/llava/mtmd-cli.cpp +++ b/examples/llava/mtmd-cli.cpp @@ -24,7 +24,9 @@ #include #endif -static bool g_is_generating = false; +//volatile, because of signal being an interrupt +static volatile bool g_is_generating = false; +static volatile bool is_app_running = true; /** * Please note that this is NOT a production-ready stuff. @@ -50,8 +52,10 @@ static void sigint_handler(int signo) { g_is_generating = false; } else { console::cleanup(); - LOG("\nInterrupted by user\n"); - _exit(130); + if(!is_app_running) { + _exit(1); + } + is_app_running = false; } } } @@ -167,7 +171,7 @@ struct decode_embd_batch { static int generate_response(mtmd_cli_context & ctx, common_sampler * smpl, int n_predict) { llama_tokens generated_tokens; for (int i = 0; i < n_predict; i++) { - if (i > n_predict || !g_is_generating) { + if (i > n_predict || !g_is_generating || !is_app_running) { printf("\n"); break; } @@ -184,6 +188,11 @@ static int generate_response(mtmd_cli_context & ctx, common_sampler * smpl, int printf("%s", common_token_to_piece(ctx.lctx, token_id).c_str()); fflush(stdout); + if(!is_app_running) { + printf("\n"); + break; + } + // eval the token common_batch_clear(ctx.batch); common_batch_add(ctx.batch, token_id, ctx.n_past++, {0}, true); @@ -206,6 +215,7 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg, std::vect LOG_DBG("formatted_chat.prompt: %s\n", formatted_chat.prompt.c_str()); for (auto & fname : images_fname) { + if(!is_app_running) return 0; mtmd_bitmap bitmap; if (mtmd_helper_bitmap_init_from_file(fname.c_str(), bitmap)) { LOG_ERR("Unable to load image %s\n", fname.c_str()); @@ -219,6 +229,9 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg, std::vect text.add_special = add_bos; text.parse_special = true; mtmd_input_chunks chunks; + + if(!is_app_running) return 0; + int32_t res = mtmd_tokenize(ctx.ctx_vision.get(), chunks, text, bitmaps); if (res != 0) { LOG_ERR("Unable to tokenize prompt, res = %d\n", res); @@ -276,6 +289,8 @@ int main(int argc, char ** argv) { #endif } + if(!is_app_running) return 130; + if (is_single_turn) { g_is_generating = true; if (params.prompt.find("<__image__>") == std::string::npos) { @@ -287,7 +302,7 @@ int main(int argc, char ** argv) { if (eval_message(ctx, msg, params.image, true)) { return 1; } - if (generate_response(ctx, smpl, n_predict)) { + if (is_app_running && generate_response(ctx, smpl, n_predict)) { return 1; } @@ -302,12 +317,13 @@ int main(int argc, char ** argv) { std::vector images_fname; std::string content; - while (true) { + while (is_app_running) { g_is_generating = false; LOG("\n> "); console::set_display(console::user_input); std::string line; console::readline(line, false); + if(!is_app_running) break; console::set_display(console::reset); line = string_strip(line); if (line.empty()) { @@ -335,6 +351,7 @@ int main(int argc, char ** argv) { msg.role = "user"; msg.content = content; int ret = eval_message(ctx, msg, images_fname, is_first_msg); + if(!is_app_running) break; if (ret == 2) { // non-fatal error images_fname.clear(); @@ -352,6 +369,7 @@ int main(int argc, char ** argv) { is_first_msg = false; } } + if(!is_app_running) LOG("\nInterrupted by user\n"); llama_perf_context_print(ctx.lctx); - return 0; + return is_app_running ? 0 : 130; } From 160b170a6191819d642c4c53ca3c772f7a0144f4 Mon Sep 17 00:00:00 2001 From: pl752 Date: Wed, 23 Apr 2025 21:30:05 +0500 Subject: [PATCH 2/5] Applied suggestions on mtmd-cli PR --- examples/llava/mtmd-cli.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/llava/mtmd-cli.cpp b/examples/llava/mtmd-cli.cpp index 8596fefcaee69..a39e99c7741db 100644 --- a/examples/llava/mtmd-cli.cpp +++ b/examples/llava/mtmd-cli.cpp @@ -26,7 +26,7 @@ //volatile, because of signal being an interrupt static volatile bool g_is_generating = false; -static volatile bool is_app_running = true; +static volatile bool g_is_interrupted = false; /** * Please note that this is NOT a production-ready stuff. @@ -52,10 +52,10 @@ static void sigint_handler(int signo) { g_is_generating = false; } else { console::cleanup(); - if(!is_app_running) { + if (g_is_interrupted) { _exit(1); } - is_app_running = false; + g_is_interrupted = true; } } } @@ -171,7 +171,7 @@ struct decode_embd_batch { static int generate_response(mtmd_cli_context & ctx, common_sampler * smpl, int n_predict) { llama_tokens generated_tokens; for (int i = 0; i < n_predict; i++) { - if (i > n_predict || !g_is_generating || !is_app_running) { + if (i > n_predict || !g_is_generating || g_is_interrupted) { printf("\n"); break; } @@ -188,7 +188,7 @@ static int generate_response(mtmd_cli_context & ctx, common_sampler * smpl, int printf("%s", common_token_to_piece(ctx.lctx, token_id).c_str()); fflush(stdout); - if(!is_app_running) { + if (g_is_interrupted) { printf("\n"); break; } @@ -215,7 +215,7 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg, std::vect LOG_DBG("formatted_chat.prompt: %s\n", formatted_chat.prompt.c_str()); for (auto & fname : images_fname) { - if(!is_app_running) return 0; + if (g_is_interrupted) return 0; mtmd_bitmap bitmap; if (mtmd_helper_bitmap_init_from_file(fname.c_str(), bitmap)) { LOG_ERR("Unable to load image %s\n", fname.c_str()); @@ -230,7 +230,7 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg, std::vect text.parse_special = true; mtmd_input_chunks chunks; - if(!is_app_running) return 0; + if (g_is_interrupted) return 0; int32_t res = mtmd_tokenize(ctx.ctx_vision.get(), chunks, text, bitmaps); if (res != 0) { @@ -289,7 +289,7 @@ int main(int argc, char ** argv) { #endif } - if(!is_app_running) return 130; + if (g_is_interrupted) return 130; if (is_single_turn) { g_is_generating = true; @@ -302,7 +302,7 @@ int main(int argc, char ** argv) { if (eval_message(ctx, msg, params.image, true)) { return 1; } - if (is_app_running && generate_response(ctx, smpl, n_predict)) { + if (g_is_interrupted && generate_response(ctx, smpl, n_predict)) { return 1; } @@ -317,13 +317,13 @@ int main(int argc, char ** argv) { std::vector images_fname; std::string content; - while (is_app_running) { + while (!g_is_interrupted) { g_is_generating = false; LOG("\n> "); console::set_display(console::user_input); std::string line; console::readline(line, false); - if(!is_app_running) break; + if (g_is_interrupted) break; console::set_display(console::reset); line = string_strip(line); if (line.empty()) { @@ -351,7 +351,7 @@ int main(int argc, char ** argv) { msg.role = "user"; msg.content = content; int ret = eval_message(ctx, msg, images_fname, is_first_msg); - if(!is_app_running) break; + if (g_is_interrupted) break; if (ret == 2) { // non-fatal error images_fname.clear(); @@ -369,7 +369,7 @@ int main(int argc, char ** argv) { is_first_msg = false; } } - if(!is_app_running) LOG("\nInterrupted by user\n"); + if (g_is_interrupted) LOG("\nInterrupted by user\n"); llama_perf_context_print(ctx.lctx); - return is_app_running ? 0 : 130; + return g_is_interrupted ? 130 : 0; } From cadaa284aed24ad7852b05fa74549d54b1c9e587 Mon Sep 17 00:00:00 2001 From: pl752 Date: Wed, 23 Apr 2025 22:56:07 +0500 Subject: [PATCH 3/5] Forgot to invert one of the conditions --- examples/llava/mtmd-cli.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llava/mtmd-cli.cpp b/examples/llava/mtmd-cli.cpp index a39e99c7741db..be2ac13a0255a 100644 --- a/examples/llava/mtmd-cli.cpp +++ b/examples/llava/mtmd-cli.cpp @@ -302,7 +302,7 @@ int main(int argc, char ** argv) { if (eval_message(ctx, msg, params.image, true)) { return 1; } - if (g_is_interrupted && generate_response(ctx, smpl, n_predict)) { + if (!g_is_interrupted && generate_response(ctx, smpl, n_predict)) { return 1; } From f6951e7ac374e930dea4ce0d5de7ee93d2bde9e7 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Wed, 23 Apr 2025 20:32:27 +0200 Subject: [PATCH 4/5] Update examples/llava/mtmd-cli.cpp --- examples/llava/mtmd-cli.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llava/mtmd-cli.cpp b/examples/llava/mtmd-cli.cpp index be2ac13a0255a..025ba39d80c75 100644 --- a/examples/llava/mtmd-cli.cpp +++ b/examples/llava/mtmd-cli.cpp @@ -24,7 +24,7 @@ #include #endif -//volatile, because of signal being an interrupt +// volatile, because of signal being an interrupt static volatile bool g_is_generating = false; static volatile bool g_is_interrupted = false; From 0e58d408230ca94c7876c055c8f885d828847b38 Mon Sep 17 00:00:00 2001 From: pl752 Date: Thu, 24 Apr 2025 00:10:47 +0500 Subject: [PATCH 5/5] Removed redundant exit check --- examples/llava/mtmd-cli.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/llava/mtmd-cli.cpp b/examples/llava/mtmd-cli.cpp index 025ba39d80c75..89af7331a1658 100644 --- a/examples/llava/mtmd-cli.cpp +++ b/examples/llava/mtmd-cli.cpp @@ -215,7 +215,6 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg, std::vect LOG_DBG("formatted_chat.prompt: %s\n", formatted_chat.prompt.c_str()); for (auto & fname : images_fname) { - if (g_is_interrupted) return 0; mtmd_bitmap bitmap; if (mtmd_helper_bitmap_init_from_file(fname.c_str(), bitmap)) { LOG_ERR("Unable to load image %s\n", fname.c_str());