diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index bbda7dea90..1ca8c1733d 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -27,6 +27,7 @@ varnishd_SOURCES = \ cache/cache_conn_pool.c \ cache/cache_deliver_proc.c \ cache/cache_director.c \ + cache/cache_drain.c \ cache/cache_esi_deliver.c \ cache/cache_esi_fetch.c \ cache/cache_esi_parse.c \ diff --git a/bin/varnishd/acceptor/cache_acceptor_tcp.c b/bin/varnishd/acceptor/cache_acceptor_tcp.c index 56fb84f301..1f5ed0ad0b 100644 --- a/bin/varnishd/acceptor/cache_acceptor_tcp.c +++ b/bin/varnishd/acceptor/cache_acceptor_tcp.c @@ -398,6 +398,7 @@ vca_tcp_make_session(struct worker *wrk, void *arg) vca_pace_good(); wrk->stats->sess_conn++; + DRAIN_IncSess(); if (wa->acceptlsock->test_heritage) { vca_tcp_sockopt_test(wa->acceptlsock, sp); diff --git a/bin/varnishd/acceptor/cache_acceptor_uds.c b/bin/varnishd/acceptor/cache_acceptor_uds.c index 7387b8d5cc..42412bd904 100644 --- a/bin/varnishd/acceptor/cache_acceptor_uds.c +++ b/bin/varnishd/acceptor/cache_acceptor_uds.c @@ -348,6 +348,7 @@ vca_uds_make_session(struct worker *wrk, void *arg) vca_pace_good(); wrk->stats->sess_conn++; + DRAIN_IncSess(); if (wa->acceptlsock->test_heritage) { vca_uds_sockopt_test(wa->acceptlsock, sp); diff --git a/bin/varnishd/cache/cache_cli.c b/bin/varnishd/cache/cache_cli.c index 3670a1bc41..0d9eb8c957 100644 --- a/bin/varnishd/cache/cache_cli.c +++ b/bin/varnishd/cache/cache_cli.c @@ -47,6 +47,7 @@ pthread_t cli_thread; static struct lock cli_mtx; static int add_check; static struct VCLS *cache_cls; +static volatile int cli_wakeup = 0; /* * The CLI commandlist is split in three: @@ -108,11 +109,20 @@ CLI_Run(void) cli->auth = 255; // Non-zero to disable paranoia in vcli_serve do { - i = VCLS_Poll(cache_cls, cli, -1); + i = VCLS_Poll(cache_cls, cli, 100); + if (cli_wakeup) + break; } while (i == 0); VSL(SLT_CLI, NO_VXID, "EOF on CLI connection, worker stops"); } +void +CLI_Wakeup(void) +{ + + cli_wakeup = 1; +} + /*--------------------------------------------------------------------*/ static struct cli_proto cli_cmds[] = { diff --git a/bin/varnishd/cache/cache_drain.c b/bin/varnishd/cache/cache_drain.c new file mode 100644 index 0000000000..219a349714 --- /dev/null +++ b/bin/varnishd/cache/cache_drain.c @@ -0,0 +1,147 @@ +/*- + * Copyright (c) 2026 Varnish Software AS + * SPDX-License-Identifier: BSD-2-Clause + * + * Connection draining support for graceful shutdown + * + * When draining is started, Varnish stops accepting new connections + * and adds "Connection: close" headers to responses. The child process + * monitors active sessions and exits when all sessions have closed or + * the timeout expires. + */ + +#include "config.h" + +#include "cache_varnishd.h" +#include "acceptor/cache_acceptor.h" +#include "vcli_serve.h" +#include "vtim.h" +#include "vnum.h" + +static struct lock drain_mtx; +static int draining = 0; +static vtim_real drain_deadline = 0.0; +static volatile long n_active_sess = 0; +static volatile int drain_exiting = 0; + +/*-------------------------------------------------------------------- + * Signal drain completion by waking the CLI thread. + * This causes CLI_Run() to return, triggering clean child shutdown. + * Uses atomic compare-and-swap to ensure we only signal once. + */ + +static void +drain_signal_exit(void) +{ + + if (__sync_bool_compare_and_swap(&drain_exiting, 0, 1)) + CLI_Wakeup(); +} + +/*-------------------------------------------------------------------- + * Monitor thread that waits for drain timeout or session completion. + */ + +static void * +drain_monitor(void *arg) +{ + + (void)arg; + + while (VTIM_real() < drain_deadline) { + if (n_active_sess <= 0) + drain_signal_exit(); + VTIM_sleep(0.1); + } + + /* Timeout expired */ + drain_signal_exit(); + return (NULL); +} + +/*--------------------------------------------------------------------*/ + +int +DRAIN_Active(void) +{ + int r; + + Lck_Lock(&drain_mtx); + r = draining; + Lck_Unlock(&drain_mtx); + return (r); +} + +void +DRAIN_IncSess(void) +{ + + (void)__sync_fetch_and_add(&n_active_sess, 1); +} + +void +DRAIN_DecSess(void) +{ + long n; + + n = __sync_sub_and_fetch(&n_active_sess, 1); + + /* If draining and no more sessions, exit immediately */ + if (n <= 0 && DRAIN_Active()) + drain_signal_exit(); +} + +void +DRAIN_Start(vtim_dur timeout) +{ + pthread_t thr; + + Lck_Lock(&drain_mtx); + if (draining) { + Lck_Unlock(&drain_mtx); + return; + } + draining = 1; + drain_deadline = VTIM_real() + timeout; + Lck_Unlock(&drain_mtx); + + /* Stop accepting new connections */ + VCA_Shutdown(); + + /* Wake idle workers to release VCL references faster */ + Pool_WakeIdle(); + + /* Start monitor thread for timeout */ + PTOK(pthread_create(&thr, NULL, drain_monitor, NULL)); +} + +static void v_matchproto_(cli_func_t) +cli_drain(struct cli *cli, const char * const *av, void *priv) +{ + vtim_dur timeout; + + (void)priv; + + timeout = VNUM_duration(av[2]); + if (isnan(timeout) || timeout < 0) { + VCLI_SetResult(cli, CLIS_PARAM); + VCLI_Out(cli, "Invalid timeout"); + return; + } + + DRAIN_Start(timeout); + VCLI_Out(cli, "Draining started"); +} + +static struct cli_proto drain_cmds[] = { + { CLICMD_DEBUG_DRAIN, "d", cli_drain }, + { NULL } +}; + +void +DRAIN_Init(void) +{ + + Lck_New(&drain_mtx, lck_drain); + CLI_AddFuncs(drain_cmds); +} diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index 95ed67244f..723922dff5 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -543,6 +543,7 @@ child_main(int sigmagic, size_t altstksz) CLI_AddFuncs(debug_cmds); CLI_AddFuncs(child_cmds); + DRAIN_Init(); #ifdef WITH_PERSISTENT_STORAGE /* Wait for persistent storage to load if asked to */ diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index 9933365c9d..e18fada2f4 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -110,6 +110,31 @@ Pool_PurgeStat(unsigned nobj) Lck_Unlock(&wstat_mtx); } +/*-------------------------------------------------------------------- + * Wake all idle workers so they can release their cached VCL references. + * Called when entering drain mode to speed up graceful shutdown. + */ + +void +Pool_WakeIdle(void) +{ + struct pool *pp; + struct pool_task *pt; + struct worker *wrk; + + Lck_Lock(&pool_mtx); + VTAILQ_FOREACH(pp, &pools, list) { + Lck_Lock(&pp->mtx); + VTAILQ_FOREACH(pt, &pp->idle_queue, list) { + wrk = pt->priv; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + PTOK(pthread_cond_signal(&wrk->cond)); + } + Lck_Unlock(&pp->mtx); + } + Lck_Unlock(&pool_mtx); +} + /*-------------------------------------------------------------------- * Special function to summ stats */ diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index e50c8f56c8..107f23b63d 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -600,6 +600,8 @@ SES_Delete(struct sess *sp, stream_close_t reason, vtim_real now) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(reason, STREAM_CLOSE_MAGIC); + DRAIN_DecSess(); + if (reason != SC_NULL) SES_Close(sp, reason); assert(sp->fd < 0); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index a2edf20833..716f8febb8 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -267,6 +267,14 @@ void VBO_Init(void); void CLI_Init(void); void CLI_Run(void); void CLI_AddFuncs(struct cli_proto *p); +void CLI_Wakeup(void); + +/* cache_drain.c */ +int DRAIN_Active(void); +void DRAIN_Init(void); +void DRAIN_Start(vtim_dur timeout); +void DRAIN_IncSess(void); +void DRAIN_DecSess(void); /* cache_expire.c */ void EXP_Init(void); @@ -416,6 +424,7 @@ void Pool_Sumstat(const struct worker *w); int Pool_TrySumstat(const struct worker *wrk); void Pool_PurgeStat(unsigned nobj); int Pool_Task_Any(struct pool_task *task, enum task_prio prio); +void Pool_WakeIdle(void); void pan_pool(struct vsb *); /* cache_range.c */ diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 9719043378..b68e8648a8 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -444,7 +444,8 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk) tmo = now + 1.; else if (wrk->wpriv->vcl == NULL) tmo = INFINITY; - else if (DO_DEBUG(DBG_VTC_MODE)) + else if (DO_DEBUG(DBG_VTC_MODE) || + DRAIN_Active()) tmo = now + 1.; else tmo = now + 60.; diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c index edb32573e7..df6309275b 100644 --- a/bin/varnishd/http1/cache_http1_deliver.c +++ b/bin/varnishd/http1/cache_http1_deliver.c @@ -81,6 +81,10 @@ V1D_Deliver(struct req *req, int sendbody) CHECK_OBJ_ORNULL(req->boc, BOC_MAGIC); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); + /* Force connection close during draining */ + if (DRAIN_Active() && req->doclose == SC_NULL) + req->doclose = SC_RESP_CLOSE; + if (req->doclose == SC_NULL && http_HdrIs(req->resp, H_Connection, "close")) { req->doclose = SC_RESP_CLOSE; diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 1499267974..fb60710e2c 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -35,6 +35,7 @@ #include +#include #include #include #include @@ -56,6 +57,7 @@ #include "vev.h" #include "vfil.h" #include "vlu.h" +#include "vnum.h" #include "vtim.h" #include "common/heritage.h" @@ -83,6 +85,8 @@ static const char * const ch_state[] = { [CH_DIED] = "died, (restarting)", }; +static int mgt_draining = 0; /* Drain in progress */ + static struct vev *ev_poker; static struct vev *ev_listen; static struct vlu *child_std_vlu; @@ -273,8 +277,10 @@ child_poker(const struct vev *e, int what) (void)e; (void)what; - if (child_state != CH_RUNNING) + if (child_state != CH_RUNNING) { + ev_poker = NULL; /* VEV will free, clear our pointer */ return (1); + } if (child_pid < 0) return (0); if (mgt_cli_askchild(&status, &r, "ping\n") || strncmp("PONG ", r, 5)) { @@ -632,8 +638,10 @@ mgt_reap_child(void) mgt_launch_child(NULL); else if (child_state == CH_DIED) child_state = CH_STOPPED; - else if (child_state == CH_STOPPING) + else if (child_state == CH_STOPPING) { child_state = CH_STOPPED; + mgt_draining = 0; + } } /*===================================================================== @@ -758,15 +766,67 @@ mch_cli_server_start(struct cli *cli, const char * const *av, void *priv) static void v_matchproto_(cli_func_t) mch_cli_server_stop(struct cli *cli, const char * const *av, void *priv) { + vtim_dur timeout = 0.0; + unsigned status; + char *r = NULL; + char cmd[64]; - (void)av; (void)priv; - if (child_state == CH_RUNNING) { - MCH_Stop_Child(); - } else { + + if (child_state != CH_RUNNING) { VCLI_SetResult(cli, CLIS_CANT); VCLI_Out(cli, "Child in state %s", ch_state[child_state]); + return; } + + /* Parse optional -t parameter */ + if (av[2] != NULL) { + if (strcmp(av[2], "-t") != 0) { + VCLI_SetResult(cli, CLIS_PARAM); + VCLI_Out(cli, "Unknown option: %s", av[2]); + return; + } + /* Default to shutdown_timeout parameter */ + timeout = mgt_param.shutdown_timeout; + /* Check for optional timeout value */ + if (av[3] != NULL) { + /* Try duration format first, then bare number */ + timeout = VNUM_duration(av[3]); + if (isnan(timeout)) + timeout = VNUM(av[3]); + if (isnan(timeout) || timeout < 0) { + VCLI_SetResult(cli, CLIS_PARAM); + VCLI_Out(cli, "Invalid timeout value: %s", av[3]); + return; + } + } + } + + if (timeout > 0.0) { + /* + * Send drain command to child. The child will stop + * accepting new connections, add Connection: close to + * responses, and exit when all sessions are closed or + * the timeout expires. The manager's event loop will + * detect the child exit and call mgt_reap_child(). + */ + bprintf(cmd, "debug.drain %.3fs\n", timeout); + if (mgt_cli_askchild(&status, &r, "%s", cmd) != 0) { + VCLI_SetResult(cli, CLIS_COMMS); + VCLI_Out(cli, "Failed to start drain: %s", + r ? r : "communication error"); + free(r); + return; + } + free(r); + + child_state = CH_STOPPING; + mgt_draining = 1; + VCLI_Out(cli, "Draining for %.0f seconds", timeout); + return; + } + + MCH_Stop_Child(); } static void v_matchproto_(cli_func_t) diff --git a/bin/varnishtest/tests/m00062.vtc b/bin/varnishtest/tests/m00062.vtc new file mode 100644 index 0000000000..4aa14b5f0c --- /dev/null +++ b/bin/varnishtest/tests/m00062.vtc @@ -0,0 +1,60 @@ +varnishtest "Test graceful shutdown with connection draining" + +server s1 { + rxreq + txresp -body "OK" +} -start + +varnish v1 -vcl+backend {} -start + +# Test 1: stop -t with explicit timeout (non-blocking) +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run + +# stop -t returns immediately, child exits when draining completes +varnish v1 -cliexpect "Draining for" "stop -t 1" + +# Wait for child to exit (draining completes quickly with no active sessions) +# Need extra time for VCL cleanup and manager to reap child +delay 2 +varnish v1 -cliexpect "stopped" status + +# Restart for next test +varnish v1 -cliok "start" + +server s1 -wait +server s1 { + rxreq + txresp -body "OK" +} -start + +# Test 2: stop -t uses shutdown_timeout parameter +varnish v1 -cliok "param.set shutdown_timeout 1" + +client c2 { + txreq + rxresp + expect resp.status == 200 +} -run + +varnish v1 -cliexpect "Draining for" "stop -t" +delay 2 +varnish v1 -cliexpect "stopped" status + +# Restart for next test +varnish v1 -cliok "start" + +# Test 3: Invalid option rejected +varnish v1 -clierr 106 "stop -x" +varnish v1 -clierr 106 "stop -t invalid" + +# Test 4: stop (without -t) when running does immediate stop +varnish v1 -cliok "stop" +varnish v1 -cliexpect "stopped" status + +# Test 5: stop when already stopped +varnish v1 -clierr 300 "stop" +varnish v1 -clierr 300 "stop -t 5" diff --git a/bin/varnishtest/tests/m00063.vtc b/bin/varnishtest/tests/m00063.vtc new file mode 100644 index 0000000000..e933adba2e --- /dev/null +++ b/bin/varnishtest/tests/m00063.vtc @@ -0,0 +1,41 @@ +varnishtest "Test Connection: close header during draining" + +# This test verifies that during draining mode, Varnish sends +# Connection: close headers to signal clients to close keep-alive connections + +barrier b1 cond 2 + +server s1 { + rxreq + # Signal that we received the request + barrier b1 sync + # Small delay to ensure drain starts while response is being prepared + delay 0.5 + txresp -body "OK" +} -start + +varnish v1 -vcl+backend {} -start + +# Start client in background - it will wait at barrier +client c1 { + txreq -hdr "Connection: keep-alive" + rxresp + expect resp.status == 200 + # During draining, Connection: close should be set + expect resp.http.Connection == "close" +} -start + +# Wait for request to reach the backend +barrier b1 sync + +# Now start draining - the request is in flight +# Use debug.drain directly since stop -t is non-blocking +varnish v1 -cliok "debug.drain 10s" + +# Wait for client to complete +client c1 -wait + +# Child should exit after draining (no active sessions) +# Need to wait a bit for child to finish cleanup and manager to reap it +delay 2 +varnish v1 -cliexpect "stopped" status diff --git a/bin/varnishtest/tests/m00064.vtc b/bin/varnishtest/tests/m00064.vtc new file mode 100644 index 0000000000..b7eb2a9a31 --- /dev/null +++ b/bin/varnishtest/tests/m00064.vtc @@ -0,0 +1,38 @@ +varnishtest "Test debug.drain CLI command" + +# This test verifies the debug.drain command works in the child process + +barrier b1 cond 2 + +server s1 { + rxreq + barrier b1 sync + delay 0.3 + txresp -body "OK" +} -start + +varnish v1 -vcl+backend {} -start + +# Verify debug.drain command syntax error handling +varnish v1 -clierr 104 "debug.drain" +varnish v1 -clierr 106 "debug.drain invalid" + +# Test direct drain command with active request +client c1 { + txreq -hdr "Connection: keep-alive" + rxresp + expect resp.status == 200 + expect resp.http.Connection == "close" +} -start + +barrier b1 sync + +# Directly invoke drain in child +varnish v1 -cliok "debug.drain 10s" + +client c1 -wait + +# Child should exit after draining (sessions drained) +# Need extra time for VCL cleanup and manager to reap child +delay 2 +varnish v1 -cliexpect "stopped" status diff --git a/bin/varnishtest/tests/m00065.vtc b/bin/varnishtest/tests/m00065.vtc new file mode 100644 index 0000000000..0a46979857 --- /dev/null +++ b/bin/varnishtest/tests/m00065.vtc @@ -0,0 +1,26 @@ +varnishtest "Test drain early exit when sessions close" + +# This test verifies that draining exits early when all sessions close +# instead of always waiting for the full timeout + +server s1 { + rxreq + txresp -body "OK" +} -start + +varnish v1 -vcl+backend {} -start + +# Make a request to establish a session +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run + +# Start drain with a long timeout - should exit early since no sessions +# Use 10 second timeout - test will fail if we actually wait that long +varnish v1 -cliexpect "Draining for" "stop -t 10" + +# Child should exit quickly (within 1-2 seconds) since there are no sessions +delay 2 +varnish v1 -cliexpect "stopped" status diff --git a/include/tbl/cli_cmds.h b/include/tbl/cli_cmds.h index dd2235935d..549cd3d245 100644 --- a/include/tbl/cli_cmds.h +++ b/include/tbl/cli_cmds.h @@ -210,10 +210,13 @@ CLI_CMD(PARAM_SET, CLI_CMD(SERVER_STOP, "stop", - "stop", + "stop [-t []]", "Stop the Varnish cache process.", - "", - 0, 0 + " -t Drain connections for up to before stopping.\n" + " -t Drain using the shutdown_timeout parameter value.\n\n" + " During draining, new connections are rejected and existing\n" + " connections receive 'Connection: close' headers.", + 0, 2 ) CLI_CMD(SERVER_START, @@ -375,6 +378,14 @@ CLI_CMD(DEBUG_SHUTDOWN_DELAY, 1, 1 ) +CLI_CMD(DEBUG_DRAIN, + "debug.drain", + "debug.drain ", + "Enter connection draining mode.", + "", + 1, 1 +) + CLI_CMD(DEBUG_XID, "debug.xid", "debug.xid [ []]", diff --git a/include/tbl/locks.h b/include/tbl/locks.h index 2199270e27..36a158992a 100644 --- a/include/tbl/locks.h +++ b/include/tbl/locks.h @@ -46,6 +46,7 @@ LOCK(probe) LOCK(sess) LOCK(conn_pool) LOCK(dead_pool) +LOCK(drain) LOCK(vbe) LOCK(vcapace) LOCK(vcashut) diff --git a/include/tbl/params.h b/include/tbl/params.h index e66a653cb6..e6726528d5 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -382,6 +382,23 @@ PARAM_SIMPLE( "If cli_timeout is longer than startup_timeout, it is used instead." ) +PARAM_SIMPLE( + /* name */ shutdown_timeout, + /* type */ timeout, + /* min */ "0.000", + /* max */ NULL, + /* def */ "0.000", + /* units */ "seconds", + /* descr */ + "Default timeout for graceful shutdown draining.\n" + "When 'stop -t' is used without a value, or when the manager\n" + "process receives SIGINT or SIGTERM, this parameter provides\n" + "the draining timeout. During draining, Varnish stops accepting\n" + "new connections and responds with 'Connection: close' headers\n" + "until all active sessions complete or the timeout expires.\n" + "A value of 0 means immediate shutdown (no draining)." +) + PARAM_SIMPLE( /* name */ clock_skew, /* type */ uint,