50 lines
3.4 KiB
Diff
50 lines
3.4 KiB
Diff
diff --git a/tools/server/server.cpp b/tools/server/server.cpp
|
|
index 868d17b418312a2d7898f4b355eef76f509195c5..d2a035baeb46f868c72592d48a64c5ede41efee9 100644
|
|
--- a/tools/server/server.cpp
|
|
+++ b/tools/server/server.cpp
|
|
@@ -1875,7 +1875,7 @@ struct server_slot {
|
|
GGML_ASSERT(task);
|
|
|
|
auto previous_msg = chat_msg;
|
|
- SRV_DBG("Parsing chat message: %s\n", generated_text.c_str());
|
|
+ //SRV_DBG("Parsing chat message: %s\n", generated_text.c_str());
|
|
auto new_msg = common_chat_parse(
|
|
generated_text,
|
|
/* is_partial= */ stop != STOP_TYPE_EOS,
|
|
@@ -1920,13 +1920,17 @@ struct server_slot {
|
|
}
|
|
|
|
void print_timings() const {
|
|
- const double t_prompt = t_prompt_processing / n_prompt_tokens_processed;
|
|
- const double n_prompt_second = 1e3 / t_prompt_processing * n_prompt_tokens_processed;
|
|
+ const double t_copyout_total = t_inference_outcopy / 32 * 31;
|
|
+ const double t_copyout_decode = (t_inference_outcopy - t_prefill_inference_outcopy) / 32 * 31;
|
|
+ const double t_copyout_prefill = t_prefill_inference_outcopy / 32 * 31;
|
|
|
|
- const double t_gen = t_token_generation / n_decoded;
|
|
- const double n_gen_second = 1e3 / t_token_generation * n_decoded;
|
|
+ const double t_prompt = (t_prompt_processing - t_copyout_prefill) / n_prompt_tokens_processed;
|
|
+ const double n_prompt_second = 1e3 / (t_prompt_processing - t_copyout_prefill) * n_prompt_tokens_processed;
|
|
|
|
- const double t_total = t_prompt_processing + t_token_generation;
|
|
+ const double t_gen = (t_token_generation - t_copyout_decode) / n_decoded;
|
|
+ const double n_gen_second = 1e3 / (t_token_generation - t_copyout_decode) * n_decoded;
|
|
+
|
|
+ const double t_total = t_prompt_processing + t_token_generation -t_copyout_total ;
|
|
const double t_framework_other = t_total - t_inference_accumulated - t_inference_incopy - t_inference_outcopy - t_sampling_accumulated - t_unknown;
|
|
|
|
const double t_prompt_inference = t_prefill_inference_accumulated / n_prompt_tokens_processed;
|
|
@@ -1950,9 +1954,9 @@ struct server_slot {
|
|
" [Device] Unknown : %10.2f ms \n"
|
|
" [CPU] Sampling: %10.2f ms \n"
|
|
" [CPU] Fw Ovhd : %10.2f ms (Batch Prep, Logic, etc.)\n",
|
|
- t_prompt_processing, n_prompt_tokens_processed, t_prompt, n_prompt_second,
|
|
- t_token_generation, n_decoded, t_gen, n_gen_second,
|
|
- t_prompt_processing + t_token_generation, n_prompt_tokens_processed + n_decoded,
|
|
+ t_prompt_processing - t_copyout_prefill , n_prompt_tokens_processed, t_prompt, n_prompt_second,
|
|
+ t_token_generation - t_copyout_decode , n_decoded, t_gen, n_gen_second,
|
|
+ t_prompt_processing + t_token_generation - t_copyout_total, n_prompt_tokens_processed + n_decoded,
|
|
t_inference_accumulated, t_prefill_inference_accumulated, t_prompt_inference, t_inference_accumulated - t_prefill_inference_accumulated, t_gen_inference,
|
|
t_inference_incopy, t_prefill_inference_incopy, t_prompt_incopy, t_inference_incopy - t_prefill_inference_incopy, t_gen_incopy,
|
|
t_inference_outcopy, t_prefill_inference_outcopy, t_prompt_outcopy, t_inference_outcopy - t_prefill_inference_outcopy, t_gen_outcopy,
|