哨兵调用重配置脚本
哨兵客户端重配置脚本参数：sentinel client-reconfig-script
哨兵节点sentinel.conf中配置重配置脚本参数，可以指定重配置脚本的路径。当主节点因发生failover而切换时，哨兵节点会调用这个脚本
sentinel client-reconfig-script mymaster /var/redis/reconfig.sh脚本传入的参数：
start，leader或observer
void sentinelCallClientReconfScript(sentinelRedisInstance *master, int role, char *state, sentinelAddr *from, sentinelAddr *to) {
char fromport[32], toport[32];
if (master->client_reconfig_script == NULL) return;
ll2string(fromport,sizeof(fromport),from->port);
ll2string(toport,sizeof(toport),to->port);
// 将脚本添加到待执行队列中
sentinelScheduleScriptExecution(master->client_reconfig_script,
master->name,
(role == SENTINEL_LEADER) ? "leader" : "observer",
state, announceSentinelAddr(from), fromport,
announceSentinelAddr(to), toport, NULL);
}
void sentinelScheduleScriptExecution(char *path, ...) {
sentinelScriptJob *sj;
// 添加到队列尾部，队列为sentinel.scripts_queue
listAddNodeTail(sentinel.scripts_queue,sj);
}每个哨兵都有一个脚本执行队列，脚本执行队列中存放的是脚本执行任务，每个任务都包含脚本路径、参数等。
struct sentinelState {
int running_scripts; /* Number of scripts in execution right now. */
list *scripts_queue; /* Queue of user scripts to execute. */
// ...
} sentinel;脚本执行任务会通过sentinelRunPendingScripts函数执行，该函数会遍历脚本执行队列，并执行第一个任务。
/* Run pending scripts if we are not already at max number of running
* scripts. */
void sentinelRunPendingScripts(void) {
listNode *ln;
listIter li;
mstime_t now = mstime();
/* Find jobs that are not running and run them, from the top to the
* tail of the queue, so we run older jobs first. */
listRewind(sentinel.scripts_queue,&li);
while (sentinel.running_scripts < SENTINEL_SCRIPT_MAX_RUNNING &&
(ln = listNext(&li)) != NULL)
{
sentinelScriptJob *sj = ln->value;
pid_t pid;
/* Skip if already running. */
if (sj->flags & SENTINEL_SCRIPT_RUNNING) continue;
/* Skip if it's a retry, but not enough time has elapsed. */
if (sj->start_time && sj->start_time > now) continue;
sj->flags |= SENTINEL_SCRIPT_RUNNING;
sj->start_time = mstime();
sj->retry_num++;
pid = fork(); // fork子进程去执行脚本任务
if (pid == -1) {
/* Parent (fork error).
* We report fork errors as signal 99, in order to unify the
* reporting with other kind of errors. */
sentinelEvent(LL_WARNING,"-script-error",NULL,
"%s %d %d", sj->argv[0], 99, 0);
sj->flags &= ~SENTINEL_SCRIPT_RUNNING;
sj->pid = 0;
} else if (pid == 0) {
/* Child */
tlsCleanup();
// 子进程执行脚本
execve(sj->argv[0],sj->argv,environ);
/* If we are here an error occurred. */
_exit(2); /* Don't retry execution. */
} else {
// 父进程
sentinel.running_scripts++;
sj->pid = pid;
sentinelEvent(LL_DEBUG,"+script-child",NULL,"%ld",(long)pid);
}
}
}谁会调用sentinelRunPendingScripts函数呢？是serverCron函数中通过sentinelTimer会调用。
int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
/* Run the Sentinel timer if we are in sentinel mode. */
if (server.sentinel_mode) sentinelTimer();
}
void sentinelTimer(void) {
sentinelCheckTiltCondition();
sentinelHandleDictOfRedisInstances(sentinel.masters);
sentinelRunPendingScripts(); // 脚本执行任务
sentinelCollectTerminatedScripts();
sentinelKillTimedoutScripts();
/* We continuously change the frequency of the Redis "timer interrupt"
* in order to desynchronize every Sentinel from every other.
* This non-determinism avoids that Sentinels started at the same time
* exactly continue to stay synchronized asking to be voted at the
* same time again and again (resulting in nobody likely winning the
* election because of split brain voting). */
server.hz = CONFIG_DEFAULT_HZ + rand() % CONFIG_DEFAULT_HZ;
}serverCron是Redis事件循环中注册的定时事件处理器，承担着服务端后台维护任务的核心职责：
main(int argc, char **argv) {
initServer() {
// 注册定时任务，serverCron
aeCreateTimeEvent(server.el, 1, serverCron, NULL, NULL) == AE_ERR)；
}
aeMain(server.el); // 启动事件循环
}什么时候会触发重配置脚本？
hello消息中获取到主节点变更信息后触发
当Sentinel节点订阅的__sentinel__:hello通道收到消息时触发，这是Sentinel节点间通信的核心机制。
通过Sentinel的PUBLISH命令：Sentinel会处理发送到__sentinel__:hello通道的PUBLISH命令
void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privdata) {
// 获取hello消息，处理hello消息
sentinelProcessHelloMessage(r->element[2]->str, r->element[2]->len);
}这个函数处理通过Pub/Sub从master或slave实例接收到的hello消息，或者通过Sentinel的(fake) PUBLISH命令直接发送到这个sentinel的消息。如果消息中指定的master名称未知，该消息将被丢弃。
void sentinelProcessHelloMessage(char *hello, int hello_len) {
if (numtokens == 8) {
// 如果收到的master配置epoch比本地的epoch新
/* Update master info if received configuration is newer. */
if (si && master->config_epoch < master_config_epoch) {
// 更新master的epoch
master->config_epoch = master_config_epoch;
// 如果master地址或者端口变更
if (master_port != master->addr->port ||
!sentinelAddrEqualsHostname(master->addr, token[5]))
{
sentinelAddr *old_addr;
sentinelEvent(LL_WARNING,"+config-update-from",si,"%@");
sentinelEvent(LL_WARNING,"+switch-master",
master,"%s %s %d %s %d",
master->name,
announceSentinelAddr(master->addr), master->addr->port,
token[5], master_port);
old_addr = dupSentinelAddr(master->addr);
// 更新master地址
sentinelResetMasterAndChangeAddress(master, token[5], master_port);
// 运行重配置脚本
sentinelCallClientReconfScript(master,
SENTINEL_OBSERVER,"start",
old_addr,master->addr);
releaseSentinelAddr(old_addr);
}
}
}
}Sentinel从监控的Redis实例收到Info命令
sentinel从监控的redis实例收到info命令时触发，处理info信息。
void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) {
sds *lines;
// 处理master变为slave的情况
/* Handle master -> slave role switch. */
if ((ri->flags & SRI_MASTER) && role == SRI_SLAVE) {
/* Nothing to do, but masters claiming to be slaves are
* considered to be unreachable by Sentinel, so eventually
* a failover will be triggered. */
}
// 处理slave变为master的情况
/* Handle slave -> master role switch. */
if ((ri->flags & SRI_SLAVE) && role == SRI_MASTER) {
/* If this is a promoted slave we can change state to the
* failover state machine. */
// 故障转移成功完成，被提升的slave报告自己是master，
if ((ri->flags & SRI_PROMOTED) &&
(ri->master->flags & SRI_FAILOVER_IN_PROGRESS) &&
(ri->master->failover_state ==
SENTINEL_FAILOVER_STATE_WAIT_PROMOTION))
{
/* Now that we are sure the slave was reconfigured as a master
* set the master configuration epoch to the epoch we won the
* election to perform this failover. This will force the other
* Sentinels to update their config (assuming there is not
* a newer one already available). */
ri->master->config_epoch = ri->master->failover_epoch;
ri->master->failover_state = SENTINEL_FAILOVER_STATE_RECONF_SLAVES;
ri->master->failover_state_change_time = mstime();
sentinelFlushConfig();
sentinelEvent(LL_WARNING,"+promoted-slave",ri,"%@");
if (sentinel.simfailure_flags &
SENTINEL_SIMFAILURE_CRASH_AFTER_PROMOTION)
sentinelSimFailureCrash();
sentinelEvent(LL_WARNING,"+failover-state-reconf-slaves",
ri->master,"%@");
// 运行客户端重配置脚本
sentinelCallClientReconfScript(ri->master,SENTINEL_LEADER,
"start",ri->master->addr,ri->addr);
// 发送hello消息更新集群视图
sentinelForceHelloUpdateForMaster(ri->master);
} else {
// 意外的slave变为master，不是预期的故障转移结果，尝试将该实例重新配置为slave
/* A slave turned into a master. We want to force our view and
* reconfigure as slave. Wait some time after the change before
* going forward, to receive new configs if any. */
mstime_t wait_time = sentinel_publish_period*4;
if (!(ri->flags & SRI_PROMOTED) &&
sentinelMasterLooksSane(ri->master) &&
sentinelRedisInstanceNoDownFor(ri,wait_time) &&
mstime() - ri->role_reported_time > wait_time)
{
int retval = sentinelSendSlaveOf(ri,ri->master->addr);
if (retval == C_OK)
sentinelEvent(LL_NOTICE,"+convert-to-slave",ri,"%@");
}
}
}
}