Sentinel invoking the reconfig script
Sentinel client reconfig script parameter: sentinel client-reconfig-script
The reconfig script parameter is configured in the sentinel node’s sentinel.conf, where you can specify the path to the reconfig script. When the master node is switched due to a failover, the sentinel node calls this script.
sentinel client-reconfig-script mymaster /var/redis/reconfig.shArguments passed to the script:
start, and leader or observer.
void sentinelCallClientReconfScript(sentinelRedisInstance *master, int role, char *state, sentinelAddr *from, sentinelAddr *to) {
char fromport[32], toport[32];
if (master->client_reconfig_script == NULL) return;
ll2string(fromport,sizeof(fromport),from->port);
ll2string(toport,sizeof(toport),to->port);
// add the script to the pending execution queue
sentinelScheduleScriptExecution(master->client_reconfig_script,
master->name,
(role == SENTINEL_LEADER) ? "leader" : "observer",
state, announceSentinelAddr(from), fromport,
announceSentinelAddr(to), toport, NULL);
}
void sentinelScheduleScriptExecution(char *path, ...) {
sentinelScriptJob *sj;
// add to the tail of the queue, which is sentinel.scripts_queue
listAddNodeTail(sentinel.scripts_queue,sj);
}Every sentinel has a script execution queue that holds script execution jobs; each job contains the script path, arguments, and so on.
struct sentinelState {
int running_scripts; /* Number of scripts in execution right now. */
list *scripts_queue; /* Queue of user scripts to execute. */
// ...
} sentinel;The script execution jobs are run by the sentinelRunPendingScripts function, which iterates the script execution queue and runs the first job.
/* Run pending scripts if we are not already at max number of running
* scripts. */
void sentinelRunPendingScripts(void) {
listNode *ln;
listIter li;
mstime_t now = mstime();
/* Find jobs that are not running and run them, from the top to the
* tail of the queue, so we run older jobs first. */
listRewind(sentinel.scripts_queue,&li);
while (sentinel.running_scripts < SENTINEL_SCRIPT_MAX_RUNNING &&
(ln = listNext(&li)) != NULL)
{
sentinelScriptJob *sj = ln->value;
pid_t pid;
/* Skip if already running. */
if (sj->flags & SENTINEL_SCRIPT_RUNNING) continue;
/* Skip if it's a retry, but not enough time has elapsed. */
if (sj->start_time && sj->start_time > now) continue;
sj->flags |= SENTINEL_SCRIPT_RUNNING;
sj->start_time = mstime();
sj->retry_num++;
pid = fork(); // fork a child process to execute the script job
if (pid == -1) {
/* Parent (fork error).
* We report fork errors as signal 99, in order to unify the
* reporting with other kind of errors. */
sentinelEvent(LL_WARNING,"-script-error",NULL,
"%s %d %d", sj->argv[0], 99, 0);
sj->flags &= ~SENTINEL_SCRIPT_RUNNING;
sj->pid = 0;
} else if (pid == 0) {
/* Child */
tlsCleanup();
// the child process executes the script
execve(sj->argv[0],sj->argv,environ);
/* If we are here an error occurred. */
_exit(2); /* Don't retry execution. */
} else {
// parent process
sentinel.running_scripts++;
sj->pid = pid;
sentinelEvent(LL_DEBUG,"+script-child",NULL,"%ld",(long)pid);
}
}
}Who calls sentinelRunPendingScripts? It is invoked from serverCron via sentinelTimer.
int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
/* Run the Sentinel timer if we are in sentinel mode. */
if (server.sentinel_mode) sentinelTimer();
}
void sentinelTimer(void) {
sentinelCheckTiltCondition();
sentinelHandleDictOfRedisInstances(sentinel.masters);
sentinelRunPendingScripts(); // script execution jobs
sentinelCollectTerminatedScripts();
sentinelKillTimedoutScripts();
/* We continuously change the frequency of the Redis "timer interrupt"
* in order to desynchronize every Sentinel from every other.
* This non-determinism avoids that Sentinels started at the same time
* exactly continue to stay synchronized asking to be voted at the
* same time again and again (resulting in nobody likely winning the
* election because of split brain voting). */
server.hz = CONFIG_DEFAULT_HZ + rand() % CONFIG_DEFAULT_HZ;
}serverCron is the timed event handler registered in Redis’s event loop. It bears the core responsibility for the server’s background maintenance tasks:
main(int argc, char **argv) {
initServer() {
// register the timed task, serverCron
aeCreateTimeEvent(server.el, 1, serverCron, NULL, NULL) == AE_ERR)；
}
aeMain(server.el); // start the event loop
}When is the reconfig script triggered?
Triggered after a master change is learned from a hello message
This is triggered when the __sentinel__:hello channel subscribed by the Sentinel node receives a message—the core mechanism of inter-Sentinel communication.
Via Sentinel’s PUBLISH command: Sentinel processes the PUBLISH command sent to the __sentinel__:hello channel.
void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privdata) {
// get and process the hello message
sentinelProcessHelloMessage(r->element[2]->str, r->element[2]->len);
}This function processes hello messages received from master or slave instances via Pub/Sub, or messages sent directly to this sentinel via a (fake) PUBLISH command. If the master name specified in the message is unknown, the message is discarded.
void sentinelProcessHelloMessage(char *hello, int hello_len) {
if (numtokens == 8) {
// if the received master config epoch is newer than the local epoch
/* Update master info if received configuration is newer. */
if (si && master->config_epoch < master_config_epoch) {
// update the master's epoch
master->config_epoch = master_config_epoch;
// if the master address or port changed
if (master_port != master->addr->port ||
!sentinelAddrEqualsHostname(master->addr, token[5]))
{
sentinelAddr *old_addr;
sentinelEvent(LL_WARNING,"+config-update-from",si,"%@");
sentinelEvent(LL_WARNING,"+switch-master",
master,"%s %s %d %s %d",
master->name,
announceSentinelAddr(master->addr), master->addr->port,
token[5], master_port);
old_addr = dupSentinelAddr(master->addr);
// update the master address
sentinelResetMasterAndChangeAddress(master, token[5], master_port);
// run the reconfig script
sentinelCallClientReconfScript(master,
SENTINEL_OBSERVER,"start",
old_addr,master->addr);
releaseSentinelAddr(old_addr);
}
}
}
}Sentinel receives an INFO command from a monitored Redis instance
This is triggered when sentinel receives an info command from a monitored Redis instance, and processes the info data.
void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) {
sds *lines;
// handle the master -> slave role switch
/* Handle master -> slave role switch. */
if ((ri->flags & SRI_MASTER) && role == SRI_SLAVE) {
/* Nothing to do, but masters claiming to be slaves are
* considered to be unreachable by Sentinel, so eventually
* a failover will be triggered. */
}
// handle the slave -> master role switch
/* Handle slave -> master role switch. */
if ((ri->flags & SRI_SLAVE) && role == SRI_MASTER) {
/* If this is a promoted slave we can change state to the
* failover state machine. */
// failover completed successfully, the promoted slave reports itself as master
if ((ri->flags & SRI_PROMOTED) &&
(ri->master->flags & SRI_FAILOVER_IN_PROGRESS) &&
(ri->master->failover_state ==
SENTINEL_FAILOVER_STATE_WAIT_PROMOTION))
{
/* Now that we are sure the slave was reconfigured as a master
* set the master configuration epoch to the epoch we won the
* election to perform this failover. This will force the other
* Sentinels to update their config (assuming there is not
* a newer one already available). */
ri->master->config_epoch = ri->master->failover_epoch;
ri->master->failover_state = SENTINEL_FAILOVER_STATE_RECONF_SLAVES;
ri->master->failover_state_change_time = mstime();
sentinelFlushConfig();
sentinelEvent(LL_WARNING,"+promoted-slave",ri,"%@");
if (sentinel.simfailure_flags &
SENTINEL_SIMFAILURE_CRASH_AFTER_PROMOTION)
sentinelSimFailureCrash();
sentinelEvent(LL_WARNING,"+failover-state-reconf-slaves",
ri->master,"%@");
// run the client reconfig script
sentinelCallClientReconfScript(ri->master,SENTINEL_LEADER,
"start",ri->master->addr,ri->addr);
// send a hello message to update the cluster view
sentinelForceHelloUpdateForMaster(ri->master);
} else {
// unexpected slave -> master, not an expected failover result; try to reconfigure the instance as a slave
/* A slave turned into a master. We want to force our view and
* reconfigure as slave. Wait some time after the change before
* going forward, to receive new configs if any. */
mstime_t wait_time = sentinel_publish_period*4;
if (!(ri->flags & SRI_PROMOTED) &&
sentinelMasterLooksSane(ri->master) &&
sentinelRedisInstanceNoDownFor(ri,wait_time) &&
mstime() - ri->role_reported_time > wait_time)
{
int retval = sentinelSendSlaveOf(ri,ri->master->addr);
if (retval == C_OK)
sentinelEvent(LL_NOTICE,"+convert-to-slave",ri,"%@");
}
}
}
}