corosync  2.3.4
exec/votequorum.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2009-2014 Red Hat, Inc.
3  *
4  * All rights reserved.
5  *
6  * Authors: Christine Caulfield (ccaulfie@redhat.com)
7  * Fabio M. Di Nitto (fdinitto@redhat.com)
8  *
9  * This software licensed under BSD license, the text of which follows:
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions are met:
13  *
14  * - Redistributions of source code must retain the above copyright notice,
15  * this list of conditions and the following disclaimer.
16  * - Redistributions in binary form must reproduce the above copyright notice,
17  * this list of conditions and the following disclaimer in the documentation
18  * and/or other materials provided with the distribution.
19  * - Neither the name of the MontaVista Software, Inc. nor the names of its
20  * contributors may be used to endorse or promote products derived from this
21  * software without specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTIBUTORS "AS IS"
24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
27  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33  * THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 #include <config.h>
37 
38 #include <sys/types.h>
39 #include <sys/stat.h>
40 #include <fcntl.h>
41 #include <stdint.h>
42 #include <unistd.h>
43 
44 #include <qb/qbipc_common.h>
45 
46 #include "quorum.h"
47 #include <corosync/corodefs.h>
48 #include <corosync/list.h>
49 #include <corosync/logsys.h>
50 #include <corosync/coroapi.h>
51 #include <corosync/icmap.h>
52 #include <corosync/votequorum.h>
54 
55 #include "service.h"
56 #include "util.h"
57 
58 LOGSYS_DECLARE_SUBSYS ("VOTEQ");
59 
60 /*
61  * interface with corosync
62  */
63 
64 static struct corosync_api_v1 *corosync_api;
65 
66 /*
67  * votequorum global config vars
68  */
69 
70 
71 static char qdevice_name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN];
72 static struct cluster_node *qdevice = NULL;
73 static unsigned int qdevice_timeout = VOTEQUORUM_QDEVICE_DEFAULT_TIMEOUT;
74 static unsigned int qdevice_sync_timeout = VOTEQUORUM_QDEVICE_DEFAULT_SYNC_TIMEOUT;
75 static uint8_t qdevice_can_operate = 1;
76 static void *qdevice_reg_conn = NULL;
77 static uint8_t qdevice_master_wins = 0;
78 
79 static uint8_t two_node = 0;
80 
81 static uint8_t wait_for_all = 0;
82 static uint8_t wait_for_all_status = 0;
83 
84 static enum {ATB_NONE, ATB_LOWEST, ATB_HIGHEST, ATB_LIST} auto_tie_breaker = ATB_NONE;
85 static int lowest_node_id = -1;
86 static int highest_node_id = -1;
87 
88 #define DEFAULT_LMS_WIN 10000
89 static uint8_t last_man_standing = 0;
90 static uint32_t last_man_standing_window = DEFAULT_LMS_WIN;
91 
92 static uint8_t allow_downscale = 0;
93 static uint32_t ev_barrier = 0;
94 
95 static uint8_t ev_tracking = 0;
96 static uint32_t ev_tracking_barrier = 0;
97 static int ev_tracking_fd = -1;
98 
99 /*
100  * votequorum_exec defines/structs/forward definitions
101  */
104  struct qb_ipc_request_header header __attribute__((aligned(8)));
105  uint32_t nodeid;
106  uint32_t votes;
107  uint32_t expected_votes;
108  uint32_t flags;
109 } __attribute__((packed));
110 
112  struct qb_ipc_request_header header __attribute__((aligned(8)));
113  uint32_t nodeid;
114  uint32_t value;
115  uint8_t param;
116  uint8_t _pad0;
117  uint8_t _pad1;
118  uint8_t _pad2;
119 } __attribute__((packed));
120 
122  struct qb_ipc_request_header header __attribute__((aligned(8)));
123  uint32_t operation;
125 } __attribute__((packed));
126 
128  struct qb_ipc_request_header header __attribute__((aligned(8)));
131 } __attribute__((packed));
132 
133 /*
134  * votequorum_exec onwire version (via totem)
135  */
136 
137 #include "votequorum.h"
138 
139 /*
140  * votequorum_exec onwire messages (via totem)
141  */
142 
143 #define MESSAGE_REQ_EXEC_VOTEQUORUM_NODEINFO 0
144 #define MESSAGE_REQ_EXEC_VOTEQUORUM_RECONFIGURE 1
145 #define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_REG 2
146 #define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_RECONFIGURE 3
147 
148 static void votequorum_exec_send_expectedvotes_notification(void);
149 static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context);
150 
151 #define VOTEQUORUM_RECONFIG_PARAM_EXPECTED_VOTES 1
152 #define VOTEQUORUM_RECONFIG_PARAM_NODE_VOTES 2
153 #define VOTEQUORUM_RECONFIG_PARAM_CANCEL_WFA 3
154 
155 static int votequorum_exec_send_reconfigure(uint8_t param, unsigned int nodeid, uint32_t value);
156 
157 /*
158  * used by req_exec_quorum_qdevice_reg
159  */
160 #define VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER 0
161 #define VOTEQUORUM_QDEVICE_OPERATION_REGISTER 1
162 
163 /*
164  * votequorum internal node status/view
165  */
166 
167 #define NODE_FLAGS_QUORATE 1
168 #define NODE_FLAGS_LEAVING 2
169 #define NODE_FLAGS_WFASTATUS 4
170 #define NODE_FLAGS_FIRST 8
171 #define NODE_FLAGS_QDEVICE_REGISTERED 16
172 #define NODE_FLAGS_QDEVICE_ALIVE 32
173 #define NODE_FLAGS_QDEVICE_CAST_VOTE 64
174 #define NODE_FLAGS_QDEVICE_MASTER_WINS 128
175 
176 typedef enum {
180 } nodestate_t;
181 
182 struct cluster_node {
183  int node_id;
185  uint32_t votes;
186  uint32_t expected_votes;
187  uint32_t flags;
188  struct list_head list;
189 };
190 
191 /*
192  * votequorum internal quorum status
193  */
194 
195 static uint8_t quorum;
196 static uint8_t cluster_is_quorate;
197 
198 /*
199  * votequorum membership data
200  */
201 
202 static struct cluster_node *us;
203 static struct list_head cluster_members_list;
204 static unsigned int quorum_members[PROCESSOR_COUNT_MAX];
205 static unsigned int previous_quorum_members[PROCESSOR_COUNT_MAX];
206 static unsigned int atb_nodelist[PROCESSOR_COUNT_MAX];
207 static int quorum_members_entries = 0;
208 static int previous_quorum_members_entries = 0;
209 static int atb_nodelist_entries = 0;
210 static struct memb_ring_id quorum_ringid;
211 
212 /*
213  * pre allocate all cluster_nodes + one for qdevice
214  */
215 static struct cluster_node cluster_nodes[PROCESSOR_COUNT_MAX+2];
216 static int cluster_nodes_entries = 0;
217 
218 /*
219  * votequorum tracking
220  */
221 struct quorum_pd {
222  unsigned char track_flags;
225  struct list_head list;
226  void *conn;
227 };
228 
229 static struct list_head trackers_list;
230 
231 /*
232  * votequorum timers
233  */
234 
235 static corosync_timer_handle_t qdevice_timer;
236 static int qdevice_timer_set = 0;
237 static corosync_timer_handle_t last_man_standing_timer;
238 static int last_man_standing_timer_set = 0;
239 static int sync_nodeinfo_sent = 0;
240 static int sync_wait_for_poll_or_timeout = 0;
241 
242 /*
243  * Service Interfaces required by service_message_handler struct
244  */
245 
246 static int sync_in_progress = 0;
247 
248 static void votequorum_sync_init (
249  const unsigned int *trans_list,
250  size_t trans_list_entries,
251  const unsigned int *member_list,
252  size_t member_list_entries,
253  const struct memb_ring_id *ring_id);
254 
255 static int votequorum_sync_process (void);
256 static void votequorum_sync_activate (void);
257 static void votequorum_sync_abort (void);
258 
259 static quorum_set_quorate_fn_t quorum_callback;
260 
261 /*
262  * votequorum_exec handler and definitions
263  */
264 
265 static char *votequorum_exec_init_fn (struct corosync_api_v1 *api);
266 static int votequorum_exec_exit_fn (void);
267 static int votequorum_exec_send_nodeinfo(uint32_t nodeid);
268 
269 static void message_handler_req_exec_votequorum_nodeinfo (
270  const void *message,
271  unsigned int nodeid);
272 static void exec_votequorum_nodeinfo_endian_convert (void *message);
273 
274 static void message_handler_req_exec_votequorum_reconfigure (
275  const void *message,
276  unsigned int nodeid);
277 static void exec_votequorum_reconfigure_endian_convert (void *message);
278 
279 static void message_handler_req_exec_votequorum_qdevice_reg (
280  const void *message,
281  unsigned int nodeid);
282 static void exec_votequorum_qdevice_reg_endian_convert (void *message);
283 
284 static void message_handler_req_exec_votequorum_qdevice_reconfigure (
285  const void *message,
286  unsigned int nodeid);
287 static void exec_votequorum_qdevice_reconfigure_endian_convert (void *message);
288 
289 static struct corosync_exec_handler votequorum_exec_engine[] =
290 {
291  { /* 0 */
292  .exec_handler_fn = message_handler_req_exec_votequorum_nodeinfo,
293  .exec_endian_convert_fn = exec_votequorum_nodeinfo_endian_convert
294  },
295  { /* 1 */
296  .exec_handler_fn = message_handler_req_exec_votequorum_reconfigure,
297  .exec_endian_convert_fn = exec_votequorum_reconfigure_endian_convert
298  },
299  { /* 2 */
300  .exec_handler_fn = message_handler_req_exec_votequorum_qdevice_reg,
301  .exec_endian_convert_fn = exec_votequorum_qdevice_reg_endian_convert
302  },
303  { /* 3 */
304  .exec_handler_fn = message_handler_req_exec_votequorum_qdevice_reconfigure,
305  .exec_endian_convert_fn = exec_votequorum_qdevice_reconfigure_endian_convert
306  },
307 };
308 
309 /*
310  * Library Handler and Functions Definitions
311  */
312 
313 static int quorum_lib_init_fn (void *conn);
314 
315 static int quorum_lib_exit_fn (void *conn);
316 
317 static void qdevice_timer_fn(void *arg);
318 
319 static void message_handler_req_lib_votequorum_getinfo (void *conn,
320  const void *message);
321 
322 static void message_handler_req_lib_votequorum_setexpected (void *conn,
323  const void *message);
324 
325 static void message_handler_req_lib_votequorum_setvotes (void *conn,
326  const void *message);
327 
328 static void message_handler_req_lib_votequorum_trackstart (void *conn,
329  const void *message);
330 
331 static void message_handler_req_lib_votequorum_trackstop (void *conn,
332  const void *message);
333 
334 static void message_handler_req_lib_votequorum_qdevice_register (void *conn,
335  const void *message);
336 
337 static void message_handler_req_lib_votequorum_qdevice_unregister (void *conn,
338  const void *message);
339 
340 static void message_handler_req_lib_votequorum_qdevice_update (void *conn,
341  const void *message);
342 
343 static void message_handler_req_lib_votequorum_qdevice_poll (void *conn,
344  const void *message);
345 
346 static void message_handler_req_lib_votequorum_qdevice_master_wins (void *conn,
347  const void *message);
348 
349 static struct corosync_lib_handler quorum_lib_service[] =
350 {
351  { /* 0 */
352  .lib_handler_fn = message_handler_req_lib_votequorum_getinfo,
354  },
355  { /* 1 */
356  .lib_handler_fn = message_handler_req_lib_votequorum_setexpected,
358  },
359  { /* 2 */
360  .lib_handler_fn = message_handler_req_lib_votequorum_setvotes,
362  },
363  { /* 3 */
364  .lib_handler_fn = message_handler_req_lib_votequorum_trackstart,
366  },
367  { /* 4 */
368  .lib_handler_fn = message_handler_req_lib_votequorum_trackstop,
370  },
371  { /* 5 */
372  .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_register,
374  },
375  { /* 6 */
376  .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_unregister,
378  },
379  { /* 7 */
380  .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_update,
382  },
383  { /* 8 */
384  .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_poll,
386  },
387  { /* 9 */
388  .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_master_wins,
390  }
391 };
392 
393 static struct corosync_service_engine votequorum_service_engine = {
394  .name = "corosync vote quorum service v1.0",
395  .id = VOTEQUORUM_SERVICE,
396  .priority = 2,
397  .private_data_size = sizeof (struct quorum_pd),
398  .allow_inquorate = CS_LIB_ALLOW_INQUORATE,
399  .flow_control = COROSYNC_LIB_FLOW_CONTROL_REQUIRED,
400  .lib_init_fn = quorum_lib_init_fn,
401  .lib_exit_fn = quorum_lib_exit_fn,
402  .lib_engine = quorum_lib_service,
403  .lib_engine_count = sizeof (quorum_lib_service) / sizeof (struct corosync_lib_handler),
404  .exec_init_fn = votequorum_exec_init_fn,
405  .exec_exit_fn = votequorum_exec_exit_fn,
406  .exec_engine = votequorum_exec_engine,
407  .exec_engine_count = sizeof (votequorum_exec_engine) / sizeof (struct corosync_exec_handler),
408  .sync_init = votequorum_sync_init,
409  .sync_process = votequorum_sync_process,
410  .sync_activate = votequorum_sync_activate,
411  .sync_abort = votequorum_sync_abort
412 };
413 
415 {
416  return (&votequorum_service_engine);
417 }
418 
419 static struct default_service votequorum_service[] = {
420  {
421  .name = "corosync_votequorum",
422  .ver = 0,
424  },
425 };
426 
427 /*
428  * common/utility macros/functions
429  */
430 
431 #define max(a,b) (((a) > (b)) ? (a) : (b))
432 
433 #define list_iterate(v, head) \
434  for (v = (head)->next; v != head; v = v->next)
435 
436 static void node_add_ordered(struct cluster_node *newnode)
437 {
438  struct cluster_node *node = NULL;
439  struct list_head *tmp;
440  struct list_head *newlist = &newnode->list;
441 
442  ENTER();
443 
444  list_iterate(tmp, &cluster_members_list) {
445  node = list_entry(tmp, struct cluster_node, list);
446  if (newnode->node_id < node->node_id) {
447  break;
448  }
449  }
450 
451  if (!node) {
452  list_add(&newnode->list, &cluster_members_list);
453  } else {
454  newlist->prev = tmp->prev;
455  newlist->next = tmp;
456  tmp->prev->next = newlist;
457  tmp->prev = newlist;
458  }
459 
460  LEAVE();
461 }
462 
463 static struct cluster_node *allocate_node(unsigned int nodeid)
464 {
465  struct cluster_node *cl = NULL;
466  struct list_head *tmp;
467 
468  ENTER();
469 
470  if (cluster_nodes_entries <= PROCESSOR_COUNT_MAX + 1) {
471  cl = (struct cluster_node *)&cluster_nodes[cluster_nodes_entries];
472  cluster_nodes_entries++;
473  } else {
474  list_iterate(tmp, &cluster_members_list) {
475  cl = list_entry(tmp, struct cluster_node, list);
476  if (cl->state == NODESTATE_DEAD) {
477  break;
478  }
479  }
480  /*
481  * this should never happen
482  */
483  if (!cl) {
484  log_printf(LOGSYS_LEVEL_CRIT, "Unable to find memory for node %u data!!", nodeid);
485  goto out;
486  }
487  list_del(tmp);
488  }
489 
490  memset(cl, 0, sizeof(struct cluster_node));
491  cl->node_id = nodeid;
492  if (nodeid != VOTEQUORUM_QDEVICE_NODEID) {
493  node_add_ordered(cl);
494  }
495 
496 out:
497  LEAVE();
498 
499  return cl;
500 }
501 
502 static struct cluster_node *find_node_by_nodeid(unsigned int nodeid)
503 {
504  struct cluster_node *node;
505  struct list_head *tmp;
506 
507  ENTER();
508 
509  if (nodeid == us->node_id) {
510  LEAVE();
511  return us;
512  }
513 
514  if (nodeid == VOTEQUORUM_QDEVICE_NODEID) {
515  LEAVE();
516  return qdevice;
517  }
518 
519  list_iterate(tmp, &cluster_members_list) {
520  node = list_entry(tmp, struct cluster_node, list);
521  if (node->node_id == nodeid) {
522  LEAVE();
523  return node;
524  }
525  }
526 
527  LEAVE();
528  return NULL;
529 }
530 
531 static void get_lowest_node_id(void)
532 {
533  struct cluster_node *node = NULL;
534  struct list_head *tmp;
535 
536  ENTER();
537 
538  lowest_node_id = us->node_id;
539 
540  list_iterate(tmp, &cluster_members_list) {
541  node = list_entry(tmp, struct cluster_node, list);
542  if ((node->state == NODESTATE_MEMBER) &&
543  (node->node_id < lowest_node_id)) {
544  lowest_node_id = node->node_id;
545  }
546  }
547  log_printf(LOGSYS_LEVEL_DEBUG, "lowest node id: %d us: %d", lowest_node_id, us->node_id);
548  icmap_set_uint32("runtime.votequorum.lowest_node_id", lowest_node_id);
549 
550  LEAVE();
551 }
552 
553 static void get_highest_node_id(void)
554 {
555  struct cluster_node *node = NULL;
556  struct list_head *tmp;
557 
558  ENTER();
559 
560  highest_node_id = us->node_id;
561 
562  list_iterate(tmp, &cluster_members_list) {
563  node = list_entry(tmp, struct cluster_node, list);
564  if ((node->state == NODESTATE_MEMBER) &&
565  (node->node_id > highest_node_id)) {
566  highest_node_id = node->node_id;
567  }
568  }
569  log_printf(LOGSYS_LEVEL_DEBUG, "highest node id: %d us: %d", highest_node_id, us->node_id);
570  icmap_set_uint32("runtime.votequorum.highest_node_id", highest_node_id);
571 
572  LEAVE();
573 }
574 
575 static int check_low_node_id_partition(void)
576 {
577  struct cluster_node *node = NULL;
578  struct list_head *tmp;
579  int found = 0;
580 
581  ENTER();
582 
583  list_iterate(tmp, &cluster_members_list) {
584  node = list_entry(tmp, struct cluster_node, list);
585  if ((node->state == NODESTATE_MEMBER) &&
586  (node->node_id == lowest_node_id)) {
587  found = 1;
588  }
589  }
590 
591  LEAVE();
592  return found;
593 }
594 
595 static int check_high_node_id_partition(void)
596 {
597  struct cluster_node *node = NULL;
598  struct list_head *tmp;
599  int found = 0;
600 
601  ENTER();
602 
603  list_iterate(tmp, &cluster_members_list) {
604  node = list_entry(tmp, struct cluster_node, list);
605  if ((node->state == NODESTATE_MEMBER) &&
606  (node->node_id == highest_node_id)) {
607  found = 1;
608  }
609  }
610 
611  LEAVE();
612  return found;
613 }
614 
615 static int is_in_nodelist(int nodeid, unsigned int *members, int entries)
616 {
617  int i;
618  ENTER();
619 
620  for (i=0; i<entries; i++) {
621  if (nodeid == members[i]) {
622  LEAVE();
623  return 1;
624  }
625  }
626  LEAVE();
627  return 0;
628 }
629 
630 /*
631  * The algorithm for a list of time-breaker nodes is:
632  * travel the list of nodes in the auto_tie_breaker list,
633  * if the node IS in our current partition, check if the
634  * nodes earlier in the atb list are in the 'previous' partition;
635  * If none are found then we are safe to be quorate, if any are
636  * then we cannot be as we don't know if that node is up or down.
637  * If we don't have a node in the current list we are NOT quorate.
638  * Obviously if we find the first node in the atb list in our
639  * partition then we are quorate.
640  *
641  * Special cases lowest nodeid, and highest nodeid are handled separately.
642  */
643 static int check_auto_tie_breaker(void)
644 {
645  int i, j;
646  int res;
647  ENTER();
648 
649  if (auto_tie_breaker == ATB_LOWEST) {
650  res = check_low_node_id_partition();
651  log_printf(LOGSYS_LEVEL_DEBUG, "ATB_LOWEST decision: %d", res);
652  LEAVE();
653  return res;
654  }
655  if (auto_tie_breaker == ATB_HIGHEST) {
656  res = check_high_node_id_partition();
657  log_printf(LOGSYS_LEVEL_DEBUG, "ATB_HIGHEST decision: %d", res);
658  LEAVE();
659  return res;
660  }
661 
662  /* Assume ATB_LIST, we should never be called for ATB_NONE */
663  for (i=0; i < atb_nodelist_entries; i++) {
664  if (is_in_nodelist(atb_nodelist[i], quorum_members, quorum_members_entries)) {
665  /*
666  * Node is in our partition, if any of its predecessors are
667  * in the previous quorum partition then it might be in the
668  * 'other half' (as we've got this far without seeing it here)
669  * and so we can't be quorate.
670  */
671  for (j=0; j<i; j++) {
672  if (is_in_nodelist(atb_nodelist[j], previous_quorum_members, previous_quorum_members_entries)) {
673  log_printf(LOGSYS_LEVEL_DEBUG, "ATB_LIST found node %d in previous partition but not here, quorum denied", atb_nodelist[j]);
674  LEAVE();
675  return 0;
676  }
677  }
678 
679  /*
680  * None of the other list nodes were in the previous partition, if there
681  * are enough votes, we can be quorate
682  */
683  log_printf(LOGSYS_LEVEL_DEBUG, "ATB_LIST found node %d in current partition, we can be quorate", atb_nodelist[i]);
684  LEAVE();
685  return 1;
686  }
687  }
688  log_printf(LOGSYS_LEVEL_DEBUG, "ATB_LIST found no list nodes in current partition, we cannot be quorate");
689  LEAVE();
690  return 0;
691 }
692 
693 /*
694  * atb_string can be either:
695  * 'lowest'
696  * 'highest'
697  * a list of nodeids
698  */
699 static void parse_atb_string(char *atb_string)
700 {
701  char *ptr;
702  long num;
703 
704  ENTER();
705  auto_tie_breaker = ATB_NONE;
706 
707  if (!strcmp(atb_string, "lowest"))
708  auto_tie_breaker = ATB_LOWEST;
709 
710  if (!strcmp(atb_string, "highest"))
711  auto_tie_breaker = ATB_HIGHEST;
712 
713  if (atoi(atb_string)) {
714 
715  atb_nodelist_entries = 0;
716  ptr = atb_string;
717  do {
718  num = strtol(ptr, &ptr, 10);
719  if (num) {
720  log_printf(LOGSYS_LEVEL_DEBUG, "ATB nodelist[%d] = %d", atb_nodelist_entries, num);
721  atb_nodelist[atb_nodelist_entries++] = num;
722  }
723  } while (num);
724 
725  if (atb_nodelist_entries) {
726  auto_tie_breaker = ATB_LIST;
727  }
728  }
729  icmap_set_uint32("runtime.votequorum.atb_type", auto_tie_breaker);
730  log_printf(LOGSYS_LEVEL_DEBUG, "ATB type = %d", auto_tie_breaker);
731 
732  /* Make sure we got something */
733  if (auto_tie_breaker == ATB_NONE) {
734  log_printf(LOGSYS_LEVEL_WARNING, "auto_tie_breaker_nodes is not valid. It must be 'lowest', 'highest' or a space-separated list of node IDs. auto_tie_breaker is disabled");
735  auto_tie_breaker = ATB_NONE;
736  }
737  LEAVE();
738 }
739 
740 static int check_qdevice_master(void)
741 {
742  struct cluster_node *node = NULL;
743  struct list_head *tmp;
744  int found = 0;
745 
746  ENTER();
747 
748  list_iterate(tmp, &cluster_members_list) {
749  node = list_entry(tmp, struct cluster_node, list);
750  if ((node->state == NODESTATE_MEMBER) &&
753  found = 1;
754  }
755  }
756 
757  LEAVE();
758  return found;
759 }
760 
761 static void decode_flags(uint32_t flags)
762 {
763  ENTER();
764 
766  "flags: quorate: %s Leaving: %s WFA Status: %s First: %s Qdevice: %s QdeviceAlive: %s QdeviceCastVote: %s QdeviceMasterWins: %s",
767  (flags & NODE_FLAGS_QUORATE)?"Yes":"No",
768  (flags & NODE_FLAGS_LEAVING)?"Yes":"No",
769  (flags & NODE_FLAGS_WFASTATUS)?"Yes":"No",
770  (flags & NODE_FLAGS_FIRST)?"Yes":"No",
771  (flags & NODE_FLAGS_QDEVICE_REGISTERED)?"Yes":"No",
772  (flags & NODE_FLAGS_QDEVICE_ALIVE)?"Yes":"No",
773  (flags & NODE_FLAGS_QDEVICE_CAST_VOTE)?"Yes":"No",
774  (flags & NODE_FLAGS_QDEVICE_MASTER_WINS)?"Yes":"No");
775 
776  LEAVE();
777 }
778 
779 /*
780  * load/save are copied almost pristine from totemsrp,c
781  */
782 static int load_ev_tracking_barrier(void)
783 {
784  int res = 0;
785  char filename[PATH_MAX];
786 
787  ENTER();
788 
789  snprintf(filename, sizeof(filename) - 1, "%s/ev_tracking", get_run_dir());
790 
791  ev_tracking_fd = open(filename, O_RDWR, 0700);
792  if (ev_tracking_fd != -1) {
793  res = read (ev_tracking_fd, &ev_tracking_barrier, sizeof(uint32_t));
794  if (res == sizeof (uint32_t)) {
795  LEAVE();
796  return 0;
797  }
798  }
799 
800  ev_tracking_barrier = 0;
801  umask(0);
802  ev_tracking_fd = open (filename, O_CREAT|O_RDWR, 0700);
803  if (ev_tracking_fd != -1) {
804  res = write (ev_tracking_fd, &ev_tracking_barrier, sizeof (uint32_t));
805  if ((res == -1) || (res != sizeof (uint32_t))) {
807  "Unable to write to %s", filename);
808  }
809  LEAVE();
810  return 0;
811  }
813  "Unable to create %s file", filename);
814 
815  LEAVE();
816 
817  return -1;
818 }
819 
820 static void update_wait_for_all_status(uint8_t wfa_status)
821 {
822  ENTER();
823 
824  wait_for_all_status = wfa_status;
825  if (wait_for_all_status) {
827  } else {
828  us->flags &= ~NODE_FLAGS_WFASTATUS;
829  }
830  icmap_set_uint8("runtime.votequorum.wait_for_all_status",
831  wait_for_all_status);
832 
833  LEAVE();
834 }
835 
836 static void update_two_node(void)
837 {
838  ENTER();
839 
840  icmap_set_uint8("runtime.votequorum.two_node", two_node);
841 
842  LEAVE();
843 }
844 
845 static void update_ev_barrier(uint32_t expected_votes)
846 {
847  ENTER();
848 
849  ev_barrier = expected_votes;
850  icmap_set_uint32("runtime.votequorum.ev_barrier", ev_barrier);
851 
852  LEAVE();
853 }
854 
855 static void update_qdevice_can_operate(uint8_t status)
856 {
857  ENTER();
858 
859  qdevice_can_operate = status;
860  icmap_set_uint8("runtime.votequorum.qdevice_can_operate", qdevice_can_operate);
861 
862  LEAVE();
863 }
864 
865 static void update_qdevice_master_wins(uint8_t allow)
866 {
867  ENTER();
868 
869  qdevice_master_wins = allow;
870  icmap_set_uint8("runtime.votequorum.qdevice_master_wins", qdevice_master_wins);
871 
872  LEAVE();
873 }
874 
875 static void update_ev_tracking_barrier(uint32_t ev_t_barrier)
876 {
877  int res;
878 
879  ENTER();
880 
881  ev_tracking_barrier = ev_t_barrier;
882  icmap_set_uint32("runtime.votequorum.ev_tracking_barrier", ev_tracking_barrier);
883 
884  if (lseek (ev_tracking_fd, 0, SEEK_SET) != 0) {
886  "Unable to update ev_tracking_barrier on disk data!!!");
887  LEAVE();
888  return;
889  }
890 
891  res = write (ev_tracking_fd, &ev_tracking_barrier, sizeof (uint32_t));
892  if (res != sizeof (uint32_t)) {
894  "Unable to update ev_tracking_barrier on disk data!!!");
895  }
896  fdatasync(ev_tracking_fd);
897 
898  LEAVE();
899 }
900 
901 /*
902  * quorum calculation core bits
903  */
904 
905 static int calculate_quorum(int allow_decrease, unsigned int max_expected, unsigned int *ret_total_votes)
906 {
907  struct list_head *nodelist;
908  struct cluster_node *node;
909  unsigned int total_votes = 0;
910  unsigned int highest_expected = 0;
911  unsigned int newquorum, q1, q2;
912  unsigned int total_nodes = 0;
913 
914  ENTER();
915 
916  if ((allow_downscale) && (allow_decrease) && (max_expected)) {
917  max_expected = max(ev_barrier, max_expected);
918  }
919 
920  list_iterate(nodelist, &cluster_members_list) {
921  node = list_entry(nodelist, struct cluster_node, list);
922 
923  log_printf(LOGSYS_LEVEL_DEBUG, "node %u state=%d, votes=%u, expected=%u",
924  node->node_id, node->state, node->votes, node->expected_votes);
925 
926  if (node->state == NODESTATE_MEMBER) {
927  if (max_expected) {
928  node->expected_votes = max_expected;
929  } else {
930  highest_expected = max(highest_expected, node->expected_votes);
931  }
932  total_votes += node->votes;
933  total_nodes++;
934  }
935  }
936 
938  log_printf(LOGSYS_LEVEL_DEBUG, "node 0 state=1, votes=%u", qdevice->votes);
939  total_votes += qdevice->votes;
940  total_nodes++;
941  }
942 
943  if (max_expected > 0) {
944  highest_expected = max_expected;
945  }
946 
947  /*
948  * This quorum calculation is taken from the OpenVMS Cluster Systems
949  * manual, but, then, you guessed that didn't you
950  */
951  q1 = (highest_expected + 2) / 2;
952  q2 = (total_votes + 2) / 2;
953  newquorum = max(q1, q2);
954 
955  /*
956  * Normally quorum never decreases but the system administrator can
957  * force it down by setting expected votes to a maximum value
958  */
959  if (!allow_decrease) {
960  newquorum = max(quorum, newquorum);
961  }
962 
963  /*
964  * The special two_node mode allows each of the two nodes to retain
965  * quorum if the other fails. Only one of the two should live past
966  * fencing (as both nodes try to fence each other in split-brain.)
967  * Also: if there are more than two nodes, force us inquorate to avoid
968  * any damage or confusion.
969  */
970  if (two_node && total_nodes <= 2) {
971  newquorum = 1;
972  }
973 
974  if (ret_total_votes) {
975  *ret_total_votes = total_votes;
976  }
977 
978  LEAVE();
979  return newquorum;
980 }
981 
982 static void are_we_quorate(unsigned int total_votes)
983 {
984  int quorate;
985  int quorum_change = 0;
986 
987  ENTER();
988 
989  /*
990  * wait for all nodes to show up before granting quorum
991  */
992 
993  if ((wait_for_all) && (wait_for_all_status)) {
994  if (total_votes != us->expected_votes) {
996  "Waiting for all cluster members. "
997  "Current votes: %d expected_votes: %d",
998  total_votes, us->expected_votes);
999  cluster_is_quorate = 0;
1000  return;
1001  }
1002  update_wait_for_all_status(0);
1003  }
1004 
1005  if (quorum > total_votes) {
1006  quorate = 0;
1007  } else {
1008  quorate = 1;
1009  get_lowest_node_id();
1010  get_highest_node_id();
1011  }
1012 
1013  if ((auto_tie_breaker != ATB_NONE) &&
1014  (total_votes == (us->expected_votes / 2)) &&
1015  (check_auto_tie_breaker() == 1)) {
1016  quorate = 1;
1017  }
1018 
1019  if ((qdevice_master_wins) &&
1020  (!quorate) &&
1021  (check_qdevice_master() == 1)) {
1022  log_printf(LOGSYS_LEVEL_DEBUG, "node is quorate as part of master_wins partition");
1023  quorate = 1;
1024  }
1025 
1026  if (cluster_is_quorate && !quorate) {
1027  quorum_change = 1;
1028  log_printf(LOGSYS_LEVEL_DEBUG, "quorum lost, blocking activity");
1029  }
1030  if (!cluster_is_quorate && quorate) {
1031  quorum_change = 1;
1032  log_printf(LOGSYS_LEVEL_DEBUG, "quorum regained, resuming activity");
1033  }
1034 
1035  cluster_is_quorate = quorate;
1036  if (cluster_is_quorate) {
1037  us->flags |= NODE_FLAGS_QUORATE;
1038  } else {
1039  us->flags &= ~NODE_FLAGS_QUORATE;
1040  }
1041 
1042  if (wait_for_all) {
1043  if (quorate) {
1044  update_wait_for_all_status(0);
1045  } else {
1046  update_wait_for_all_status(1);
1047  }
1048  }
1049 
1050  if ((quorum_change) &&
1051  (sync_in_progress == 0)) {
1052  quorum_callback(quorum_members, quorum_members_entries,
1053  cluster_is_quorate, &quorum_ringid);
1054  }
1055 
1056  LEAVE();
1057 }
1058 
1059 static void get_total_votes(unsigned int *totalvotes, unsigned int *current_members)
1060 {
1061  unsigned int total_votes = 0;
1062  unsigned int cluster_members = 0;
1063  struct list_head *nodelist;
1064  struct cluster_node *node;
1065 
1066  ENTER();
1067 
1068  list_iterate(nodelist, &cluster_members_list) {
1069  node = list_entry(nodelist, struct cluster_node, list);
1070  if (node->state == NODESTATE_MEMBER) {
1071  cluster_members++;
1072  total_votes += node->votes;
1073  }
1074  }
1075 
1076  if (qdevice->votes) {
1077  total_votes += qdevice->votes;
1078  cluster_members++;
1079  }
1080 
1081  *totalvotes = total_votes;
1082  *current_members = cluster_members;
1083 
1084  LEAVE();
1085 }
1086 
1087 /*
1088  * Recalculate cluster quorum, set quorate and notify changes
1089  */
1090 static void recalculate_quorum(int allow_decrease, int by_current_nodes)
1091 {
1092  unsigned int total_votes = 0;
1093  unsigned int cluster_members = 0;
1094 
1095  ENTER();
1096 
1097  get_total_votes(&total_votes, &cluster_members);
1098 
1099  if (!by_current_nodes) {
1100  cluster_members = 0;
1101  }
1102 
1103  /*
1104  * Keep expected_votes at the highest number of votes in the cluster
1105  */
1106  log_printf(LOGSYS_LEVEL_DEBUG, "total_votes=%d, expected_votes=%d", total_votes, us->expected_votes);
1107  if (total_votes > us->expected_votes) {
1108  us->expected_votes = total_votes;
1109  votequorum_exec_send_expectedvotes_notification();
1110  }
1111 
1112  if ((ev_tracking) &&
1113  (us->expected_votes > ev_tracking_barrier)) {
1114  update_ev_tracking_barrier(us->expected_votes);
1115  }
1116 
1117  quorum = calculate_quorum(allow_decrease, cluster_members, &total_votes);
1118  are_we_quorate(total_votes);
1119 
1120  votequorum_exec_send_quorum_notification(NULL, 0L);
1121 
1122  LEAVE();
1123 }
1124 
1125 /*
1126  * configuration bits and pieces
1127  */
1128 
1129 static int votequorum_read_nodelist_configuration(uint32_t *votes,
1130  uint32_t *nodes,
1131  uint32_t *expected_votes)
1132 {
1133  icmap_iter_t iter;
1134  const char *iter_key;
1135  char tmp_key[ICMAP_KEYNAME_MAXLEN];
1136  uint32_t our_pos, node_pos;
1137  uint32_t nodecount = 0;
1138  uint32_t nodelist_expected_votes = 0;
1139  uint32_t node_votes = 0;
1140  int res = 0;
1141 
1142  ENTER();
1143 
1144  if (icmap_get_uint32("nodelist.local_node_pos", &our_pos) != CS_OK) {
1146  "No nodelist defined or our node is not in the nodelist");
1147  return 0;
1148  }
1149 
1150  iter = icmap_iter_init("nodelist.node.");
1151 
1152  while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) {
1153 
1154  res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, tmp_key);
1155  if (res != 2) {
1156  continue;
1157  }
1158 
1159  if (strcmp(tmp_key, "ring0_addr") != 0) {
1160  continue;
1161  }
1162 
1163  nodecount++;
1164 
1165  snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.quorum_votes", node_pos);
1166  if (icmap_get_uint32(tmp_key, &node_votes) != CS_OK) {
1167  node_votes = 1;
1168  }
1169 
1170  nodelist_expected_votes = nodelist_expected_votes + node_votes;
1171 
1172  if (node_pos == our_pos) {
1173  *votes = node_votes;
1174  }
1175  }
1176 
1177  *expected_votes = nodelist_expected_votes;
1178  *nodes = nodecount;
1179 
1180  icmap_iter_finalize(iter);
1181 
1182  LEAVE();
1183 
1184  return 1;
1185 }
1186 
1187 static int votequorum_qdevice_is_configured(uint32_t *qdevice_votes)
1188 {
1189  char *qdevice_model = NULL;
1190  int ret = 0;
1191 
1192  ENTER();
1193 
1194  if (icmap_get_string("quorum.device.model", &qdevice_model) == CS_OK) {
1195  if (strlen(qdevice_model)) {
1196  if (icmap_get_uint32("quorum.device.votes", qdevice_votes) != CS_OK) {
1197  *qdevice_votes = -1;
1198  }
1199  if (icmap_get_uint32("quorum.device.timeout", &qdevice_timeout) != CS_OK) {
1200  qdevice_timeout = VOTEQUORUM_QDEVICE_DEFAULT_TIMEOUT;
1201  }
1202  if (icmap_get_uint32("quorum.device.sync_timeout", &qdevice_sync_timeout) != CS_OK) {
1203  qdevice_sync_timeout = VOTEQUORUM_QDEVICE_DEFAULT_SYNC_TIMEOUT;
1204  }
1205  update_qdevice_can_operate(1);
1206  ret = 1;
1207  }
1208 
1209  free(qdevice_model);
1210  }
1211 
1212  LEAVE();
1213 
1214  return ret;
1215 }
1216 
1217 #define VOTEQUORUM_READCONFIG_STARTUP 0
1218 #define VOTEQUORUM_READCONFIG_RUNTIME 1
1219 
1220 static char *votequorum_readconfig(int runtime)
1221 {
1222  uint32_t node_votes = 0, qdevice_votes = 0;
1223  uint32_t node_expected_votes = 0, expected_votes = 0;
1224  uint32_t node_count = 0;
1225  uint8_t atb = 0;
1226  int have_nodelist, have_qdevice;
1227  char *atb_string = NULL;
1228  char *error = NULL;
1229 
1230  ENTER();
1231 
1232  log_printf(LOGSYS_LEVEL_DEBUG, "Reading configuration (runtime: %d)", runtime);
1233 
1234  /*
1235  * gather basic data here
1236  */
1237  icmap_get_uint32("quorum.expected_votes", &expected_votes);
1238  have_nodelist = votequorum_read_nodelist_configuration(&node_votes, &node_count, &node_expected_votes);
1239  have_qdevice = votequorum_qdevice_is_configured(&qdevice_votes);
1240  icmap_get_uint8("quorum.two_node", &two_node);
1241 
1242  /*
1243  * do config verification and enablement
1244  */
1245 
1246  if ((!have_nodelist) && (!expected_votes)) {
1247  if (!runtime) {
1248  error = (char *)"configuration error: nodelist or quorum.expected_votes must be configured!";
1249  } else {
1250  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: nodelist or quorum.expected_votes must be configured!");
1251  log_printf(LOGSYS_LEVEL_CRIT, "will continue with current runtime data");
1252  }
1253  goto out;
1254  }
1255 
1256  /*
1257  * two_node and qdevice are not compatible in the same config.
1258  * try to make an educated guess of what to do
1259  */
1260 
1261  if ((two_node) && (have_qdevice)) {
1262  if (!runtime) {
1263  error = (char *)"configuration error: two_node and quorum device cannot be configured at the same time!";
1264  goto out;
1265  } else {
1266  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: two_node and quorum device cannot be configured at the same time!");
1267  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
1268  log_printf(LOGSYS_LEVEL_CRIT, "quorum device is registered, disabling two_node");
1269  two_node = 0;
1270  } else {
1271  log_printf(LOGSYS_LEVEL_CRIT, "quorum device is not registered, allowing two_node");
1272  update_qdevice_can_operate(0);
1273  }
1274  }
1275  }
1276 
1277  /*
1278  * Enable special features
1279  */
1280  if (!runtime) {
1281  if (two_node) {
1282  wait_for_all = 1;
1283  }
1284 
1285  icmap_get_uint8("quorum.allow_downscale", &allow_downscale);
1286  icmap_get_uint8("quorum.wait_for_all", &wait_for_all);
1287  icmap_get_uint8("quorum.last_man_standing", &last_man_standing);
1288  icmap_get_uint32("quorum.last_man_standing_window", &last_man_standing_window);
1289  icmap_get_uint8("quorum.expected_votes_tracking", &ev_tracking);
1290  icmap_get_uint8("quorum.auto_tie_breaker", &atb);
1291  icmap_get_string("quorum.auto_tie_breaker_node", &atb_string);
1292 
1293  if (!atb) {
1294  auto_tie_breaker = ATB_NONE;
1295  if (atb_string) {
1297  "auto_tie_breaker_node: is meaningless if auto_tie_breaker is set to 0");
1298  }
1299  }
1300 
1301  if (atb && atb_string) {
1302  parse_atb_string(atb_string);
1303  }
1304  free(atb_string);
1305 
1306  /* allow_downscale requires ev_tracking */
1307  if (allow_downscale) {
1308  ev_tracking = 1;
1309  }
1310 
1311  if (ev_tracking) {
1312  if (load_ev_tracking_barrier() < 0) {
1313  LEAVE();
1314  return ((char *)"Unable to load ev_tracking file!");
1315  }
1316  update_ev_tracking_barrier(ev_tracking_barrier);
1317  }
1318 
1319  }
1320 
1321  /*
1322  * quorum device is not compatible with last_man_standing and auto_tie_breaker
1323  * neither lms or atb can be set at runtime, so there is no need to check for
1324  * runtime incompatibilities, but qdevice can be configured _after_ LMS and ATB have
1325  * been enabled at startup.
1326  */
1327 
1328  if ((have_qdevice) && (last_man_standing)) {
1329  if (!runtime) {
1330  error = (char *)"configuration error: quorum.device is not compatible with last_man_standing";
1331  goto out;
1332  } else {
1333  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device is not compatible with last_man_standing");
1334  log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations");
1335  update_qdevice_can_operate(0);
1336  }
1337  }
1338 
1339  if ((have_qdevice) && (auto_tie_breaker != ATB_NONE)) {
1340  if (!runtime) {
1341  error = (char *)"configuration error: quorum.device is not compatible with auto_tie_breaker";
1342  goto out;
1343  } else {
1344  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device is not compatible with auto_tie_breaker");
1345  log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations");
1346  update_qdevice_can_operate(0);
1347  }
1348  }
1349 
1350  if ((have_qdevice) && (wait_for_all)) {
1351  if (!runtime) {
1352  error = (char *)"configuration error: quorum.device is not compatible with wait_for_all";
1353  goto out;
1354  } else {
1355  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device is not compatible with wait_for_all");
1356  log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations");
1357  update_qdevice_can_operate(0);
1358  }
1359  }
1360 
1361  if ((have_qdevice) && (allow_downscale)) {
1362  if (!runtime) {
1363  error = (char *)"configuration error: quorum.device is not compatible with allow_downscale";
1364  goto out;
1365  } else {
1366  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device is not compatible with allow_downscale");
1367  log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations");
1368  update_qdevice_can_operate(0);
1369  }
1370  }
1371 
1372  /*
1373  * if user specifies quorum.expected_votes + quorum.device but NOT the device.votes
1374  * we don't know what the quorum device should vote.
1375  */
1376 
1377  if ((expected_votes) && (have_qdevice) && (qdevice_votes == -1)) {
1378  if (!runtime) {
1379  error = (char *)"configuration error: quorum.device.votes must be specified when quorum.expected_votes is set";
1380  goto out;
1381  } else {
1382  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device.votes must be specified when quorum.expected_votes is set");
1383  log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations");
1384  update_qdevice_can_operate(0);
1385  }
1386  }
1387 
1388  /*
1389  * if user specifies a node list with uneven votes and no device.votes
1390  * we cannot autocalculate the votes
1391  */
1392 
1393  if ((have_qdevice) &&
1394  (qdevice_votes == -1) &&
1395  (have_nodelist) &&
1396  (node_count != node_expected_votes)) {
1397  if (!runtime) {
1398  error = (char *)"configuration error: quorum.device.votes must be specified when not all nodes votes 1";
1399  goto out;
1400  } else {
1401  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device.votes must be specified when not all nodes votes 1");
1402  log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations");
1403  update_qdevice_can_operate(0);
1404  }
1405  }
1406 
1407  /*
1408  * validate quorum device votes vs expected_votes
1409  */
1410 
1411  if ((qdevice_votes > 0) && (expected_votes)) {
1412  int delta = expected_votes - qdevice_votes;
1413  if (delta < 2) {
1414  if (!runtime) {
1415  error = (char *)"configuration error: quorum.device.votes is too high or expected_votes is too low";
1416  goto out;
1417  } else {
1418  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device.votes is too high or expected_votes is too low");
1419  log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations");
1420  update_qdevice_can_operate(0);
1421  }
1422  }
1423  }
1424 
1425  /*
1426  * automatically calculate device votes and adjust expected_votes from nodelist
1427  */
1428 
1429  if ((have_qdevice) &&
1430  (qdevice_votes == -1) &&
1431  (!expected_votes) &&
1432  (have_nodelist) &&
1433  (node_count == node_expected_votes)) {
1434  qdevice_votes = node_expected_votes - 1;
1435  node_expected_votes = node_expected_votes + qdevice_votes;
1436  }
1437 
1438  /*
1439  * set this node votes and expected_votes
1440  */
1441  log_printf(LOGSYS_LEVEL_DEBUG, "ev_tracking=%d, ev_tracking_barrier = %d: expected_votes = %d\n", ev_tracking, ev_tracking_barrier, expected_votes);
1442 
1443  if (ev_tracking) {
1444  expected_votes = ev_tracking_barrier;
1445  }
1446 
1447  if (have_nodelist) {
1448  us->votes = node_votes;
1449  us->expected_votes = node_expected_votes;
1450  } else {
1451  us->votes = 1;
1452  icmap_get_uint32("quorum.votes", &us->votes);
1453  }
1454 
1455  if (expected_votes) {
1457  }
1458 
1459  /*
1460  * set qdevice votes
1461  */
1462 
1463  if (!have_qdevice) {
1464  qdevice->votes = 0;
1465  }
1466 
1467  if (qdevice_votes != -1) {
1468  qdevice->votes = qdevice_votes;
1469  }
1470 
1471  update_ev_barrier(us->expected_votes);
1472  update_two_node();
1473  if (wait_for_all) {
1474  update_wait_for_all_status(1);
1475  }
1476 
1477 out:
1478  LEAVE();
1479  return error;
1480 }
1481 
1482 static void votequorum_refresh_config(
1483  int32_t event,
1484  const char *key_name,
1485  struct icmap_notify_value new_val,
1486  struct icmap_notify_value old_val,
1487  void *user_data)
1488 {
1489  int old_votes, old_expected_votes;
1490  uint8_t reloading;
1491  uint8_t cancel_wfa;
1492 
1493  ENTER();
1494 
1495  /*
1496  * If a full reload is in progress then don't do anything until it's done and
1497  * can reconfigure it all atomically
1498  */
1499  if (icmap_get_uint8("config.totemconfig_reload_in_progress", &reloading) == CS_OK && reloading) {
1500  return ;
1501  }
1502 
1503  icmap_get_uint8("quorum.cancel_wait_for_all", &cancel_wfa);
1504  if (strcmp(key_name, "quorum.cancel_wait_for_all") == 0 &&
1505  cancel_wfa >= 1) {
1506  icmap_set_uint8("quorum.cancel_wait_for_all", 0);
1507  votequorum_exec_send_reconfigure(VOTEQUORUM_RECONFIG_PARAM_CANCEL_WFA,
1508  us->node_id, 0);
1509  return;
1510  }
1511 
1512  old_votes = us->votes;
1513  old_expected_votes = us->expected_votes;
1514 
1515  /*
1516  * Reload the configuration
1517  */
1518  votequorum_readconfig(VOTEQUORUM_READCONFIG_RUNTIME);
1519 
1520  /*
1521  * activate new config
1522  */
1523  votequorum_exec_send_nodeinfo(us->node_id);
1524  votequorum_exec_send_nodeinfo(VOTEQUORUM_QDEVICE_NODEID);
1525  if (us->votes != old_votes) {
1526  votequorum_exec_send_reconfigure(VOTEQUORUM_RECONFIG_PARAM_NODE_VOTES,
1527  us->node_id, us->votes);
1528  }
1529  if (us->expected_votes != old_expected_votes) {
1530  votequorum_exec_send_reconfigure(VOTEQUORUM_RECONFIG_PARAM_EXPECTED_VOTES,
1531  us->node_id, us->expected_votes);
1532  }
1533 
1534  LEAVE();
1535 }
1536 
1537 static void votequorum_exec_add_config_notification(void)
1538 {
1539  icmap_track_t icmap_track_nodelist = NULL;
1540  icmap_track_t icmap_track_quorum = NULL;
1541  icmap_track_t icmap_track_reload = NULL;
1542 
1543  ENTER();
1544 
1545  icmap_track_add("nodelist.",
1547  votequorum_refresh_config,
1548  NULL,
1549  &icmap_track_nodelist);
1550 
1551  icmap_track_add("quorum.",
1553  votequorum_refresh_config,
1554  NULL,
1555  &icmap_track_quorum);
1556 
1557  icmap_track_add("config.totemconfig_reload_in_progress",
1559  votequorum_refresh_config,
1560  NULL,
1561  &icmap_track_reload);
1562 
1563  LEAVE();
1564 }
1565 
1566 /*
1567  * votequorum_exec core
1568  */
1569 
1570 static int votequorum_exec_send_reconfigure(uint8_t param, unsigned int nodeid, uint32_t value)
1571 {
1573  struct iovec iov[1];
1574  int ret;
1575 
1576  ENTER();
1577 
1584 
1587 
1588  iov[0].iov_base = (void *)&req_exec_quorum_reconfigure;
1589  iov[0].iov_len = sizeof(req_exec_quorum_reconfigure);
1590 
1591  ret = corosync_api->totem_mcast (iov, 1, TOTEM_AGREED);
1592 
1593  LEAVE();
1594  return ret;
1595 }
1596 
1597 static int votequorum_exec_send_nodeinfo(uint32_t nodeid)
1598 {
1600  struct iovec iov[1];
1601  struct cluster_node *node;
1602  int ret;
1603 
1604  ENTER();
1605 
1606  node = find_node_by_nodeid(nodeid);
1607  if (!node) {
1608  return -1;
1609  }
1610 
1615  if (nodeid != VOTEQUORUM_QDEVICE_NODEID) {
1616  decode_flags(node->flags);
1617  }
1618 
1620  req_exec_quorum_nodeinfo.header.size = sizeof(req_exec_quorum_nodeinfo);
1621 
1622  iov[0].iov_base = (void *)&req_exec_quorum_nodeinfo;
1623  iov[0].iov_len = sizeof(req_exec_quorum_nodeinfo);
1624 
1625  ret = corosync_api->totem_mcast (iov, 1, TOTEM_AGREED);
1626 
1627  LEAVE();
1628  return ret;
1629 }
1630 
1631 static int votequorum_exec_send_qdevice_reconfigure(const char *oldname, const char *newname)
1632 {
1634  struct iovec iov[1];
1635  int ret;
1636 
1637  ENTER();
1638 
1643 
1644  iov[0].iov_base = (void *)&req_exec_quorum_qdevice_reconfigure;
1645  iov[0].iov_len = sizeof(req_exec_quorum_qdevice_reconfigure);
1646 
1647  ret = corosync_api->totem_mcast (iov, 1, TOTEM_AGREED);
1648 
1649  LEAVE();
1650  return ret;
1651 }
1652 
1653 static int votequorum_exec_send_qdevice_reg(uint32_t operation, const char *qdevice_name_req)
1654 {
1656  struct iovec iov[1];
1657  int ret;
1658 
1659  ENTER();
1660 
1664  strcpy(req_exec_quorum_qdevice_reg.qdevice_name, qdevice_name_req);
1665 
1666  iov[0].iov_base = (void *)&req_exec_quorum_qdevice_reg;
1667  iov[0].iov_len = sizeof(req_exec_quorum_qdevice_reg);
1668 
1669  ret = corosync_api->totem_mcast (iov, 1, TOTEM_AGREED);
1670 
1671  LEAVE();
1672  return ret;
1673 }
1674 
1675 static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context)
1676 {
1678  struct list_head *tmp;
1679  struct cluster_node *node;
1680  int cluster_members = 0;
1681  int i = 0;
1682  int size;
1683  char buf[sizeof(struct res_lib_votequorum_notification) + sizeof(struct votequorum_node) * (PROCESSOR_COUNT_MAX + 2)];
1684 
1685  ENTER();
1686 
1687  list_iterate(tmp, &cluster_members_list) {
1688  node = list_entry(tmp, struct cluster_node, list);
1689  cluster_members++;
1690  }
1691  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
1692  cluster_members++;
1693  }
1694 
1695  size = sizeof(struct res_lib_votequorum_notification) + sizeof(struct votequorum_node) * cluster_members;
1696 
1697  res_lib_votequorum_notification = (struct res_lib_votequorum_notification *)&buf;
1698  res_lib_votequorum_notification->quorate = cluster_is_quorate;
1699  res_lib_votequorum_notification->node_list_entries = cluster_members;
1700  res_lib_votequorum_notification->ring_id.nodeid = quorum_ringid.rep.nodeid;
1701  res_lib_votequorum_notification->ring_id.seq = quorum_ringid.seq;
1702  res_lib_votequorum_notification->context = context;
1703  list_iterate(tmp, &cluster_members_list) {
1704  node = list_entry(tmp, struct cluster_node, list);
1705  res_lib_votequorum_notification->node_list[i].nodeid = node->node_id;
1706  res_lib_votequorum_notification->node_list[i++].state = node->state;
1707  }
1708  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
1709  res_lib_votequorum_notification->node_list[i].nodeid = VOTEQUORUM_QDEVICE_NODEID;
1710  res_lib_votequorum_notification->node_list[i++].state = qdevice->state;
1711  }
1712  res_lib_votequorum_notification->header.id = MESSAGE_RES_VOTEQUORUM_NOTIFICATION;
1713  res_lib_votequorum_notification->header.size = size;
1714  res_lib_votequorum_notification->header.error = CS_OK;
1715 
1716  /* Send it to all interested parties */
1717  if (conn) {
1718  int ret = corosync_api->ipc_dispatch_send(conn, &buf, size);
1719  LEAVE();
1720  return ret;
1721  } else {
1722  struct quorum_pd *qpd;
1723 
1724  list_iterate(tmp, &trackers_list) {
1725  qpd = list_entry(tmp, struct quorum_pd, list);
1726  res_lib_votequorum_notification->context = qpd->tracking_context;
1727  corosync_api->ipc_dispatch_send(qpd->conn, &buf, size);
1728  }
1729  }
1730 
1731  LEAVE();
1732 
1733  return 0;
1734 }
1735 
1736 static void votequorum_exec_send_expectedvotes_notification(void)
1737 {
1739  struct quorum_pd *qpd;
1740  struct list_head *tmp;
1741 
1742  ENTER();
1743 
1744  log_printf(LOGSYS_LEVEL_DEBUG, "Sending expected votes callback");
1745 
1750 
1751  list_iterate(tmp, &trackers_list) {
1752  qpd = list_entry(tmp, struct quorum_pd, list);
1756  }
1757 
1758  LEAVE();
1759 }
1760 
1761 static void exec_votequorum_qdevice_reconfigure_endian_convert (void *message)
1762 {
1763  ENTER();
1764 
1765  LEAVE();
1766 }
1767 
1768 static void message_handler_req_exec_votequorum_qdevice_reconfigure (
1769  const void *message,
1770  unsigned int nodeid)
1771 {
1773 
1774  ENTER();
1775 
1776  log_printf(LOGSYS_LEVEL_DEBUG, "Received qdevice name change req from node %u [from: %s to: %s]",
1777  nodeid,
1778  req_exec_quorum_qdevice_reconfigure->oldname,
1779  req_exec_quorum_qdevice_reconfigure->newname);
1780 
1781  if (!strcmp(req_exec_quorum_qdevice_reconfigure->oldname, qdevice_name)) {
1782  log_printf(LOGSYS_LEVEL_DEBUG, "Allowing qdevice rename");
1783  memset(qdevice_name, 0, VOTEQUORUM_QDEVICE_MAX_NAME_LEN);
1784  strcpy(qdevice_name, req_exec_quorum_qdevice_reconfigure->newname);
1785  /*
1786  * TODO: notify qdevices about name change?
1787  * this is not relevant for now and can wait later on since
1788  * qdevices are local only and libvotequorum is not final
1789  */
1790  }
1791 
1792  LEAVE();
1793 }
1794 
1795 static void exec_votequorum_qdevice_reg_endian_convert (void *message)
1796 {
1798 
1799  ENTER();
1800 
1801  req_exec_quorum_qdevice_reg->operation = swab32(req_exec_quorum_qdevice_reg->operation);
1802 
1803  LEAVE();
1804 }
1805 
1806 static void message_handler_req_exec_votequorum_qdevice_reg (
1807  const void *message,
1808  unsigned int nodeid)
1809 {
1812  int wipe_qdevice_name = 1;
1813  struct cluster_node *node = NULL;
1814  struct list_head *tmp;
1815  cs_error_t error = CS_OK;
1816 
1817  ENTER();
1818 
1819  log_printf(LOGSYS_LEVEL_DEBUG, "Received qdevice op %u req from node %u [%s]",
1820  req_exec_quorum_qdevice_reg->operation,
1821  nodeid, req_exec_quorum_qdevice_reg->qdevice_name);
1822 
1823  switch(req_exec_quorum_qdevice_reg->operation)
1824  {
1826  if (nodeid != us->node_id) {
1827  if (!strlen(qdevice_name)) {
1828  log_printf(LOGSYS_LEVEL_DEBUG, "Remote qdevice name recorded");
1829  strcpy(qdevice_name, req_exec_quorum_qdevice_reg->qdevice_name);
1830  }
1831  LEAVE();
1832  return;
1833  }
1834 
1835  /*
1836  * protect against the case where we broadcast qdevice registration
1837  * to new memebers, we receive the message back, but there is no registration
1838  * connection in progress
1839  */
1840  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
1841  LEAVE();
1842  return;
1843  }
1844 
1845  /*
1846  * this should NEVER happen
1847  */
1848  if (!qdevice_reg_conn) {
1849  log_printf(LOGSYS_LEVEL_WARNING, "Unable to determine origin of the qdevice register call!");
1850  LEAVE();
1851  return;
1852  }
1853 
1854  /*
1855  * registering our own device in this case
1856  */
1857  if (!strlen(qdevice_name)) {
1858  strcpy(qdevice_name, req_exec_quorum_qdevice_reg->qdevice_name);
1859  }
1860 
1861  /*
1862  * check if it is our device or something else
1863  */
1864  if ((!strncmp(req_exec_quorum_qdevice_reg->qdevice_name,
1865  qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN))) {
1867  votequorum_exec_send_nodeinfo(VOTEQUORUM_QDEVICE_NODEID);
1868  votequorum_exec_send_nodeinfo(us->node_id);
1869  } else {
1871  "A new qdevice with different name (new: %s old: %s) is trying to register!",
1872  req_exec_quorum_qdevice_reg->qdevice_name, qdevice_name);
1873  error = CS_ERR_EXIST;
1874  }
1875 
1878  res_lib_votequorum_status.header.error = error;
1879  corosync_api->ipc_response_send(qdevice_reg_conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status));
1880  qdevice_reg_conn = NULL;
1881  break;
1883  list_iterate(tmp, &cluster_members_list) {
1884  node = list_entry(tmp, struct cluster_node, list);
1885  if ((node->state == NODESTATE_MEMBER) &&
1886  (node->flags & NODE_FLAGS_QDEVICE_REGISTERED)) {
1887  wipe_qdevice_name = 0;
1888  }
1889  }
1890 
1891  if (wipe_qdevice_name) {
1892  memset(qdevice_name, 0, VOTEQUORUM_QDEVICE_MAX_NAME_LEN);
1893  }
1894 
1895  break;
1896  }
1897  LEAVE();
1898 }
1899 
1900 static void exec_votequorum_nodeinfo_endian_convert (void *message)
1901 {
1902  struct req_exec_quorum_nodeinfo *nodeinfo = message;
1903 
1904  ENTER();
1905 
1906  nodeinfo->nodeid = swab32(nodeinfo->nodeid);
1907  nodeinfo->votes = swab32(nodeinfo->votes);
1908  nodeinfo->expected_votes = swab32(nodeinfo->expected_votes);
1909  nodeinfo->flags = swab32(nodeinfo->flags);
1910 
1911  LEAVE();
1912 }
1913 
1914 static void message_handler_req_exec_votequorum_nodeinfo (
1915  const void *message,
1916  unsigned int sender_nodeid)
1917 {
1918  const struct req_exec_quorum_nodeinfo *req_exec_quorum_nodeinfo = message;
1919  struct cluster_node *node = NULL;
1920  int old_votes;
1921  int old_expected;
1922  uint32_t old_flags;
1923  nodestate_t old_state;
1924  int new_node = 0;
1925  int allow_downgrade = 0;
1926  int by_node = 0;
1927  unsigned int nodeid = req_exec_quorum_nodeinfo->nodeid;
1928 
1929  ENTER();
1930 
1931  log_printf(LOGSYS_LEVEL_DEBUG, "got nodeinfo message from cluster node %u", sender_nodeid);
1932  log_printf(LOGSYS_LEVEL_DEBUG, "nodeinfo message[%u]: votes: %d, expected: %d flags: %d",
1933  nodeid,
1934  req_exec_quorum_nodeinfo->votes,
1935  req_exec_quorum_nodeinfo->expected_votes,
1936  req_exec_quorum_nodeinfo->flags);
1937 
1938  if (nodeid != VOTEQUORUM_QDEVICE_NODEID) {
1939  decode_flags(req_exec_quorum_nodeinfo->flags);
1940  }
1941 
1942  node = find_node_by_nodeid(nodeid);
1943  if (!node) {
1944  node = allocate_node(nodeid);
1945  new_node = 1;
1946  }
1947  if (!node) {
1948  corosync_api->error_memory_failure();
1949  LEAVE();
1950  return;
1951  }
1952 
1953  if (new_node) {
1954  old_votes = 0;
1955  old_expected = 0;
1956  old_state = NODESTATE_DEAD;
1957  old_flags = 0;
1958  } else {
1959  old_votes = node->votes;
1960  old_expected = node->expected_votes;
1961  old_state = node->state;
1962  old_flags = node->flags;
1963  }
1964 
1965  if (nodeid == VOTEQUORUM_QDEVICE_NODEID) {
1966  struct cluster_node *sender_node = find_node_by_nodeid(sender_nodeid);
1967 
1968  assert(sender_node != NULL);
1969 
1970  if ((!cluster_is_quorate) &&
1971  (sender_node->flags & NODE_FLAGS_QUORATE)) {
1972  node->votes = req_exec_quorum_nodeinfo->votes;
1973  } else {
1974  node->votes = max(node->votes, req_exec_quorum_nodeinfo->votes);
1975  }
1976  goto recalculate;
1977  }
1978 
1979  /* Update node state */
1980  node->flags = req_exec_quorum_nodeinfo->flags;
1981  node->votes = req_exec_quorum_nodeinfo->votes;
1982  node->state = NODESTATE_MEMBER;
1983 
1984  if (node->flags & NODE_FLAGS_LEAVING) {
1985  node->state = NODESTATE_LEAVING;
1986  allow_downgrade = 1;
1987  by_node = 1;
1988  }
1989 
1990  if ((!cluster_is_quorate) &&
1991  (node->flags & NODE_FLAGS_QUORATE)) {
1992  allow_downgrade = 1;
1993  us->expected_votes = req_exec_quorum_nodeinfo->expected_votes;
1994  }
1995 
1996  if (node->flags & NODE_FLAGS_QUORATE || (ev_tracking)) {
1997  node->expected_votes = req_exec_quorum_nodeinfo->expected_votes;
1998  } else {
1999  node->expected_votes = us->expected_votes;
2000  }
2001 
2002  if ((last_man_standing) && (node->votes > 1)) {
2003  log_printf(LOGSYS_LEVEL_WARNING, "Last Man Standing feature is supported only when all"
2004  "cluster nodes votes are set to 1. Disabling LMS.");
2005  last_man_standing = 0;
2006  if (last_man_standing_timer_set) {
2007  corosync_api->timer_delete(last_man_standing_timer);
2008  last_man_standing_timer_set = 0;
2009  }
2010  }
2011 
2012 recalculate:
2013  if ((new_node) ||
2014  (nodeid == us->node_id) ||
2015  (node->flags & NODE_FLAGS_FIRST) ||
2016  (old_votes != node->votes) ||
2017  (old_expected != node->expected_votes) ||
2018  (old_flags != node->flags) ||
2019  (old_state != node->state)) {
2020  recalculate_quorum(allow_downgrade, by_node);
2021  }
2022 
2023  if ((wait_for_all) &&
2024  (!(node->flags & NODE_FLAGS_WFASTATUS)) &&
2025  (node->flags & NODE_FLAGS_QUORATE)) {
2026  update_wait_for_all_status(0);
2027  }
2028 
2029  LEAVE();
2030 }
2031 
2032 static void exec_votequorum_reconfigure_endian_convert (void *message)
2033 {
2034  struct req_exec_quorum_reconfigure *reconfigure = message;
2035 
2036  ENTER();
2037 
2038  reconfigure->nodeid = swab32(reconfigure->nodeid);
2039  reconfigure->value = swab32(reconfigure->value);
2040 
2041  LEAVE();
2042 }
2043 
2044 static void message_handler_req_exec_votequorum_reconfigure (
2045  const void *message,
2046  unsigned int nodeid)
2047 {
2049  struct cluster_node *node;
2050  struct list_head *nodelist;
2051 
2052  ENTER();
2053 
2054  log_printf(LOGSYS_LEVEL_DEBUG, "got reconfigure message from cluster node %u for %u",
2055  nodeid, req_exec_quorum_reconfigure->nodeid);
2056 
2057  switch(req_exec_quorum_reconfigure->param)
2058  {
2060  list_iterate(nodelist, &cluster_members_list) {
2061  node = list_entry(nodelist, struct cluster_node, list);
2062  if (node->state == NODESTATE_MEMBER) {
2063  node->expected_votes = req_exec_quorum_reconfigure->value;
2064  }
2065  }
2066  votequorum_exec_send_expectedvotes_notification();
2067  update_ev_barrier(req_exec_quorum_reconfigure->value);
2068  if (ev_tracking) {
2069  us->expected_votes = max(us->expected_votes, ev_tracking_barrier);
2070  }
2071  recalculate_quorum(1, 0); /* Allow decrease */
2072  break;
2073 
2075  node = find_node_by_nodeid(req_exec_quorum_reconfigure->nodeid);
2076  if (!node) {
2077  LEAVE();
2078  return;
2079  }
2080  node->votes = req_exec_quorum_reconfigure->value;
2081  recalculate_quorum(1, 0); /* Allow decrease */
2082  break;
2083 
2085  update_wait_for_all_status(0);
2086  log_printf(LOGSYS_LEVEL_INFO, "wait_for_all_status reset by user on node %d.",
2087  req_exec_quorum_reconfigure->nodeid);
2088  recalculate_quorum(0, 0);
2089 
2090  break;
2091 
2092  }
2093 
2094  LEAVE();
2095 }
2096 
2097 static int votequorum_exec_exit_fn (void)
2098 {
2099  int ret = 0;
2100 
2101  ENTER();
2102 
2103  /*
2104  * tell the other nodes we are leaving
2105  */
2106 
2107  if (allow_downscale) {
2108  us->flags |= NODE_FLAGS_LEAVING;
2109  ret = votequorum_exec_send_nodeinfo(us->node_id);
2110  }
2111 
2112  if ((ev_tracking) && (ev_tracking_fd != -1)) {
2113  close(ev_tracking_fd);
2114  }
2115 
2116 
2117  LEAVE();
2118  return ret;
2119 }
2120 
2121 static char *votequorum_exec_init_fn (struct corosync_api_v1 *api)
2122 {
2123  char *error = NULL;
2124 
2125  ENTER();
2126 
2127  /*
2128  * make sure we start clean
2129  */
2130  list_init(&cluster_members_list);
2131  list_init(&trackers_list);
2132  qdevice = NULL;
2133  us = NULL;
2134  memset(cluster_nodes, 0, sizeof(cluster_nodes));
2135 
2136  /*
2137  * Allocate a cluster_node for qdevice
2138  */
2139  qdevice = allocate_node(VOTEQUORUM_QDEVICE_NODEID);
2140  if (!qdevice) {
2141  LEAVE();
2142  return ((char *)"Could not allocate node.");
2143  }
2144  qdevice->votes = 0;
2145  memset(qdevice_name, 0, VOTEQUORUM_QDEVICE_MAX_NAME_LEN);
2146 
2147  /*
2148  * Allocate a cluster_node for us
2149  */
2150  us = allocate_node(corosync_api->totem_nodeid_get());
2151  if (!us) {
2152  LEAVE();
2153  return ((char *)"Could not allocate node.");
2154  }
2155 
2156  icmap_set_uint32("runtime.votequorum.this_node_id", us->node_id);
2157 
2158  us->state = NODESTATE_MEMBER;
2159  us->votes = 1;
2160  us->flags |= NODE_FLAGS_FIRST;
2161 
2162  error = votequorum_readconfig(VOTEQUORUM_READCONFIG_STARTUP);
2163  if (error) {
2164  return error;
2165  }
2166  recalculate_quorum(0, 0);
2167 
2168  /*
2169  * Listen for changes
2170  */
2171  votequorum_exec_add_config_notification();
2172 
2173  /*
2174  * Start us off with one node
2175  */
2176  votequorum_exec_send_nodeinfo(us->node_id);
2177 
2178  LEAVE();
2179 
2180  return (NULL);
2181 }
2182 
2183 /*
2184  * votequorum service core
2185  */
2186 
2187 static void votequorum_last_man_standing_timer_fn(void *arg)
2188 {
2189  ENTER();
2190 
2191  last_man_standing_timer_set = 0;
2192  if (cluster_is_quorate) {
2193  recalculate_quorum(1,1);
2194  }
2195 
2196  LEAVE();
2197 }
2198 
2199 static void votequorum_sync_init (
2200  const unsigned int *trans_list, size_t trans_list_entries,
2201  const unsigned int *member_list, size_t member_list_entries,
2202  const struct memb_ring_id *ring_id)
2203 {
2204  int i, j;
2205  int found;
2206  int left_nodes;
2207  struct cluster_node *node;
2208 
2209  ENTER();
2210 
2211  sync_in_progress = 1;
2212  sync_nodeinfo_sent = 0;
2213  sync_wait_for_poll_or_timeout = 0;
2214 
2215  if (member_list_entries > 1) {
2216  us->flags &= ~NODE_FLAGS_FIRST;
2217  }
2218 
2219  /*
2220  * we don't need to track which nodes have left directly,
2221  * since that info is in the node db, but we need to know
2222  * if somebody has left for last_man_standing
2223  */
2224  left_nodes = 0;
2225  for (i = 0; i < quorum_members_entries; i++) {
2226  found = 0;
2227  for (j = 0; j < member_list_entries; j++) {
2228  if (quorum_members[i] == member_list[j]) {
2229  found = 1;
2230  break;
2231  }
2232  }
2233  if (found == 0) {
2234  left_nodes = 1;
2235  node = find_node_by_nodeid(quorum_members[i]);
2236  if (node) {
2237  node->state = NODESTATE_DEAD;
2238  }
2239  }
2240  }
2241 
2242  if (last_man_standing) {
2243  if (((member_list_entries >= quorum) && (left_nodes)) ||
2244  ((member_list_entries <= quorum) && (auto_tie_breaker != ATB_NONE) && (check_low_node_id_partition() == 1))) {
2245  if (last_man_standing_timer_set) {
2246  corosync_api->timer_delete(last_man_standing_timer);
2247  last_man_standing_timer_set = 0;
2248  }
2249  corosync_api->timer_add_duration((unsigned long long)last_man_standing_window*1000000,
2250  NULL, votequorum_last_man_standing_timer_fn,
2251  &last_man_standing_timer);
2252  last_man_standing_timer_set = 1;
2253  }
2254  }
2255 
2256  memcpy(previous_quorum_members, quorum_members, sizeof(unsigned int) * quorum_members_entries);
2257  previous_quorum_members_entries = quorum_members_entries;
2258 
2259  memcpy(quorum_members, member_list, sizeof(unsigned int) * member_list_entries);
2260  quorum_members_entries = member_list_entries;
2261  memcpy(&quorum_ringid, ring_id, sizeof(*ring_id));
2262 
2263  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED && us->flags & NODE_FLAGS_QDEVICE_ALIVE) {
2264  /*
2265  * Reset poll timer. Sync waiting is interrupted on valid qdevice poll or after timeout
2266  */
2267  if (qdevice_timer_set) {
2268  corosync_api->timer_delete(qdevice_timer);
2269  }
2270  corosync_api->timer_add_duration((unsigned long long)qdevice_sync_timeout*1000000, qdevice,
2271  qdevice_timer_fn, &qdevice_timer);
2272  qdevice_timer_set = 1;
2273  sync_wait_for_poll_or_timeout = 1;
2274 
2275  log_printf(LOGSYS_LEVEL_INFO, "waiting for quorum device %s poll (but maximum for %u ms)",
2276  qdevice_name, qdevice_sync_timeout);
2277  }
2278 
2279  LEAVE();
2280 }
2281 
2282 static int votequorum_sync_process (void)
2283 {
2284 
2285  if (!sync_nodeinfo_sent) {
2286  votequorum_exec_send_nodeinfo(us->node_id);
2287  votequorum_exec_send_nodeinfo(VOTEQUORUM_QDEVICE_NODEID);
2288  if (strlen(qdevice_name)) {
2289  votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_REGISTER,
2290  qdevice_name);
2291  }
2292  sync_nodeinfo_sent = 1;
2293  }
2294 
2295  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED && sync_wait_for_poll_or_timeout) {
2296  /*
2297  * Waiting for qdevice to poll with new ringid or timeout
2298  */
2299 
2300  return (-1);
2301  }
2302 
2303  return 0;
2304 }
2305 
2306 static void votequorum_sync_activate (void)
2307 {
2308  recalculate_quorum(0, 0);
2309  quorum_callback(quorum_members, quorum_members_entries,
2310  cluster_is_quorate, &quorum_ringid);
2311  sync_in_progress = 0;
2312 }
2313 
2314 static void votequorum_sync_abort (void)
2315 {
2316 
2317 }
2318 
2320  quorum_set_quorate_fn_t q_set_quorate_fn)
2321 {
2322  char *error;
2323 
2324  ENTER();
2325 
2326  if (q_set_quorate_fn == NULL) {
2327  return ((char *)"Quorate function not set");
2328  }
2329 
2330  corosync_api = api;
2331  quorum_callback = q_set_quorate_fn;
2332 
2333  error = corosync_service_link_and_init(corosync_api,
2334  &votequorum_service[0]);
2335  if (error) {
2336  return (error);
2337  }
2338 
2339  LEAVE();
2340 
2341  return (NULL);
2342 }
2343 
2344 /*
2345  * Library Handler init/fini
2346  */
2347 
2348 static int quorum_lib_init_fn (void *conn)
2349 {
2350  struct quorum_pd *pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn);
2351 
2352  ENTER();
2353 
2354  list_init (&pd->list);
2355  pd->conn = conn;
2356 
2357  LEAVE();
2358  return (0);
2359 }
2360 
2361 static int quorum_lib_exit_fn (void *conn)
2362 {
2363  struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn);
2364 
2365  ENTER();
2366 
2367  if (quorum_pd->tracking_enabled) {
2368  list_del (&quorum_pd->list);
2369  list_init (&quorum_pd->list);
2370  }
2371 
2372  LEAVE();
2373 
2374  return (0);
2375 }
2376 
2377 /*
2378  * library internal functions
2379  */
2380 
2381 static void qdevice_timer_fn(void *arg)
2382 {
2383  ENTER();
2384 
2385  if ((!(us->flags & NODE_FLAGS_QDEVICE_ALIVE)) ||
2386  (!qdevice_timer_set)) {
2387  LEAVE();
2388  return;
2389  }
2390 
2391  us->flags &= ~NODE_FLAGS_QDEVICE_ALIVE;
2393  log_printf(LOGSYS_LEVEL_INFO, "lost contact with quorum device %s", qdevice_name);
2394  votequorum_exec_send_nodeinfo(us->node_id);
2395 
2396  qdevice_timer_set = 0;
2397  sync_wait_for_poll_or_timeout = 0;
2398 
2399  LEAVE();
2400 }
2401 
2402 /*
2403  * Library Handler Functions
2404  */
2405 
2406 static void message_handler_req_lib_votequorum_getinfo (void *conn, const void *message)
2407 {
2410  struct cluster_node *node;
2411  unsigned int highest_expected = 0;
2412  unsigned int total_votes = 0;
2413  cs_error_t error = CS_OK;
2414  uint32_t nodeid = req_lib_votequorum_getinfo->nodeid;
2415 
2416  ENTER();
2417 
2418  log_printf(LOGSYS_LEVEL_DEBUG, "got getinfo request on %p for node %u", conn, req_lib_votequorum_getinfo->nodeid);
2419 
2420  if (nodeid == VOTEQUORUM_QDEVICE_NODEID) {
2421  nodeid = us->node_id;
2422  }
2423 
2424  node = find_node_by_nodeid(nodeid);
2425  if (node) {
2426  struct cluster_node *iternode;
2427  struct list_head *nodelist;
2428 
2429  list_iterate(nodelist, &cluster_members_list) {
2430  iternode = list_entry(nodelist, struct cluster_node, list);
2431 
2432  if (iternode->state == NODESTATE_MEMBER) {
2433  highest_expected =
2434  max(highest_expected, iternode->expected_votes);
2435  total_votes += iternode->votes;
2436  }
2437  }
2438 
2439  if (node->flags & NODE_FLAGS_QDEVICE_CAST_VOTE) {
2440  total_votes += qdevice->votes;
2441  }
2442 
2443  switch(node->state) {
2444  case NODESTATE_MEMBER:
2446  break;
2447  case NODESTATE_DEAD:
2449  break;
2450  case NODESTATE_LEAVING:
2452  break;
2453  default:
2455  break;
2456  }
2460  res_lib_votequorum_getinfo.highest_expected = highest_expected;
2461 
2466 
2467  if (two_node) {
2469  }
2470  if (cluster_is_quorate) {
2472  }
2473  if (wait_for_all) {
2475  }
2476  if (last_man_standing) {
2478  }
2479  if (auto_tie_breaker != ATB_NONE) {
2481  }
2482  if (allow_downscale) {
2484  }
2485 
2487  strcpy(res_lib_votequorum_getinfo.qdevice_name, qdevice_name);
2489 
2490  if (node->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2492  }
2493  if (node->flags & NODE_FLAGS_QDEVICE_ALIVE) {
2495  }
2496  if (node->flags & NODE_FLAGS_QDEVICE_CAST_VOTE) {
2498  }
2499  if (node->flags & NODE_FLAGS_QDEVICE_MASTER_WINS) {
2501  }
2502  } else {
2503  error = CS_ERR_NOT_EXIST;
2504  }
2505 
2508  res_lib_votequorum_getinfo.header.error = error;
2510  log_printf(LOGSYS_LEVEL_DEBUG, "getinfo response error: %d", error);
2511 
2512  LEAVE();
2513 }
2514 
2515 static void message_handler_req_lib_votequorum_setexpected (void *conn, const void *message)
2516 {
2519  cs_error_t error = CS_OK;
2520  unsigned int newquorum;
2521  unsigned int total_votes;
2522  uint8_t allow_downscale_status = 0;
2523 
2524  ENTER();
2525 
2526  allow_downscale_status = allow_downscale;
2527  allow_downscale = 0;
2528 
2529  /*
2530  * Validate new expected votes
2531  */
2532  newquorum = calculate_quorum(1, req_lib_votequorum_setexpected->expected_votes, &total_votes);
2533  allow_downscale = allow_downscale_status;
2534  if (newquorum < total_votes / 2 ||
2535  newquorum > total_votes) {
2536  error = CS_ERR_INVALID_PARAM;
2537  goto error_exit;
2538  }
2539 
2540  votequorum_exec_send_reconfigure(VOTEQUORUM_RECONFIG_PARAM_EXPECTED_VOTES, us->node_id,
2541  req_lib_votequorum_setexpected->expected_votes);
2542 
2543 error_exit:
2546  res_lib_votequorum_status.header.error = error;
2548 
2549  LEAVE();
2550 }
2551 
2552 static void message_handler_req_lib_votequorum_setvotes (void *conn, const void *message)
2553 {
2556  struct cluster_node *node;
2557  unsigned int newquorum;
2558  unsigned int total_votes;
2559  unsigned int saved_votes;
2560  cs_error_t error = CS_OK;
2561  unsigned int nodeid;
2562 
2563  ENTER();
2564 
2565  nodeid = req_lib_votequorum_setvotes->nodeid;
2566  node = find_node_by_nodeid(nodeid);
2567  if (!node) {
2568  error = CS_ERR_NAME_NOT_FOUND;
2569  goto error_exit;
2570  }
2571 
2572  /*
2573  * Check votes is valid
2574  */
2575  saved_votes = node->votes;
2576  node->votes = req_lib_votequorum_setvotes->votes;
2577 
2578  newquorum = calculate_quorum(1, 0, &total_votes);
2579 
2580  if (newquorum < total_votes / 2 ||
2581  newquorum > total_votes) {
2582  node->votes = saved_votes;
2583  error = CS_ERR_INVALID_PARAM;
2584  goto error_exit;
2585  }
2586 
2587  votequorum_exec_send_reconfigure(VOTEQUORUM_RECONFIG_PARAM_NODE_VOTES, nodeid,
2588  req_lib_votequorum_setvotes->votes);
2589 
2590 error_exit:
2593  res_lib_votequorum_status.header.error = error;
2595 
2596  LEAVE();
2597 }
2598 
2599 static void message_handler_req_lib_votequorum_trackstart (void *conn,
2600  const void *message)
2601 {
2604  struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn);
2605 
2606  ENTER();
2607  /*
2608  * If an immediate listing of the current cluster membership
2609  * is requested, generate membership list
2610  */
2611  if (req_lib_votequorum_trackstart->track_flags & CS_TRACK_CURRENT ||
2612  req_lib_votequorum_trackstart->track_flags & CS_TRACK_CHANGES) {
2613  log_printf(LOGSYS_LEVEL_DEBUG, "sending initial status to %p", conn);
2614  votequorum_exec_send_quorum_notification(conn, req_lib_votequorum_trackstart->context);
2615  }
2616 
2617  /*
2618  * Record requests for tracking
2619  */
2620  if (req_lib_votequorum_trackstart->track_flags & CS_TRACK_CHANGES ||
2621  req_lib_votequorum_trackstart->track_flags & CS_TRACK_CHANGES_ONLY) {
2622 
2623  quorum_pd->track_flags = req_lib_votequorum_trackstart->track_flags;
2624  quorum_pd->tracking_enabled = 1;
2625  quorum_pd->tracking_context = req_lib_votequorum_trackstart->context;
2626 
2627  list_add (&quorum_pd->list, &trackers_list);
2628  }
2629 
2632  res_lib_votequorum_status.header.error = CS_OK;
2634 
2635  LEAVE();
2636 }
2637 
2638 static void message_handler_req_lib_votequorum_trackstop (void *conn,
2639  const void *message)
2640 {
2642  struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn);
2643  int error = CS_OK;
2644 
2645  ENTER();
2646 
2647  if (quorum_pd->tracking_enabled) {
2648  error = CS_OK;
2649  quorum_pd->tracking_enabled = 0;
2650  list_del (&quorum_pd->list);
2651  list_init (&quorum_pd->list);
2652  } else {
2653  error = CS_ERR_NOT_EXIST;
2654  }
2655 
2658  res_lib_votequorum_status.header.error = error;
2660 
2661  LEAVE();
2662 }
2663 
2664 static void message_handler_req_lib_votequorum_qdevice_register (void *conn,
2665  const void *message)
2666 {
2669  cs_error_t error = CS_OK;
2670 
2671  ENTER();
2672 
2673  if (!qdevice_can_operate) {
2674  log_printf(LOGSYS_LEVEL_INFO, "Registration of quorum device is disabled by incorrect corosync.conf. See logs for more information");
2675  error = CS_ERR_ACCESS;
2676  goto out;
2677  }
2678 
2679  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2680  if ((!strncmp(req_lib_votequorum_qdevice_register->name,
2681  qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN))) {
2682  goto out;
2683  } else {
2685  "A new qdevice with different name (new: %s old: %s) is trying to re-register!",
2686  req_lib_votequorum_qdevice_register->name, qdevice_name);
2687  error = CS_ERR_EXIST;
2688  goto out;
2689  }
2690  } else {
2691  if (qdevice_reg_conn != NULL) {
2693  "Registration request already in progress");
2694  error = CS_ERR_TRY_AGAIN;
2695  goto out;
2696  }
2697  qdevice_reg_conn = conn;
2698  if (votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_REGISTER,
2699  req_lib_votequorum_qdevice_register->name) != 0) {
2701  "Unable to send qdevice registration request to cluster");
2702  error = CS_ERR_TRY_AGAIN;
2703  qdevice_reg_conn = NULL;
2704  } else {
2705  LEAVE();
2706  return;
2707  }
2708  }
2709 
2710 out:
2711 
2714  res_lib_votequorum_status.header.error = error;
2716 
2717  LEAVE();
2718 }
2719 
2720 static void message_handler_req_lib_votequorum_qdevice_unregister (void *conn,
2721  const void *message)
2722 {
2725  cs_error_t error = CS_OK;
2726 
2727  ENTER();
2728 
2729  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2730  if (strncmp(req_lib_votequorum_qdevice_unregister->name, qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN)) {
2731  error = CS_ERR_INVALID_PARAM;
2732  goto out;
2733  }
2734  if (qdevice_timer_set) {
2735  corosync_api->timer_delete(qdevice_timer);
2736  qdevice_timer_set = 0;
2737  sync_wait_for_poll_or_timeout = 0;
2738  }
2739  us->flags &= ~NODE_FLAGS_QDEVICE_REGISTERED;
2740  us->flags &= ~NODE_FLAGS_QDEVICE_ALIVE;
2743  votequorum_exec_send_nodeinfo(us->node_id);
2744  votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER,
2745  req_lib_votequorum_qdevice_unregister->name);
2746  } else {
2747  error = CS_ERR_NOT_EXIST;
2748  }
2749 
2750 out:
2753  res_lib_votequorum_status.header.error = error;
2755 
2756  LEAVE();
2757 }
2758 
2759 static void message_handler_req_lib_votequorum_qdevice_update (void *conn,
2760  const void *message)
2761 {
2764  cs_error_t error = CS_OK;
2765 
2766  ENTER();
2767 
2768  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2769  if (strncmp(req_lib_votequorum_qdevice_update->oldname, qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN)) {
2770  error = CS_ERR_INVALID_PARAM;
2771  goto out;
2772  }
2773  votequorum_exec_send_qdevice_reconfigure(req_lib_votequorum_qdevice_update->oldname,
2774  req_lib_votequorum_qdevice_update->newname);
2775  } else {
2776  error = CS_ERR_NOT_EXIST;
2777  }
2778 
2779 out:
2782  res_lib_votequorum_status.header.error = error;
2784 
2785  LEAVE();
2786 }
2787 
2788 static void message_handler_req_lib_votequorum_qdevice_poll (void *conn,
2789  const void *message)
2790 {
2793  cs_error_t error = CS_OK;
2794  uint32_t oldflags;
2795 
2796  ENTER();
2797 
2798  if (!qdevice_can_operate) {
2799  error = CS_ERR_ACCESS;
2800  goto out;
2801  }
2802 
2803  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2804  if (!(req_lib_votequorum_qdevice_poll->ring_id.nodeid == quorum_ringid.rep.nodeid &&
2805  req_lib_votequorum_qdevice_poll->ring_id.seq == quorum_ringid.seq)) {
2806  log_printf(LOGSYS_LEVEL_DEBUG, "Received poll ring id (%u.%"PRIu64") != last sync "
2807  "ring id (%u.%"PRIu64"). Ignoring poll call.",
2808  req_lib_votequorum_qdevice_poll->ring_id.nodeid, req_lib_votequorum_qdevice_poll->ring_id.seq,
2809  quorum_ringid.rep.nodeid, quorum_ringid.seq);
2810  error = CS_ERR_MESSAGE_ERROR;
2811  goto out;
2812  }
2813  if (strncmp(req_lib_votequorum_qdevice_poll->name, qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN)) {
2814  error = CS_ERR_INVALID_PARAM;
2815  goto out;
2816  }
2817 
2818  if (qdevice_timer_set) {
2819  corosync_api->timer_delete(qdevice_timer);
2820  qdevice_timer_set = 0;
2821  }
2822 
2823  oldflags = us->flags;
2824 
2826 
2827  if (req_lib_votequorum_qdevice_poll->cast_vote) {
2829  } else {
2831  }
2832 
2833  if (us->flags != oldflags) {
2834  votequorum_exec_send_nodeinfo(us->node_id);
2835  }
2836 
2837  corosync_api->timer_add_duration((unsigned long long)qdevice_timeout*1000000, qdevice,
2838  qdevice_timer_fn, &qdevice_timer);
2839  qdevice_timer_set = 1;
2840  sync_wait_for_poll_or_timeout = 0;
2841  } else {
2842  error = CS_ERR_NOT_EXIST;
2843  }
2844 
2845 out:
2848  res_lib_votequorum_status.header.error = error;
2850 
2851  LEAVE();
2852 }
2853 
2854 static void message_handler_req_lib_votequorum_qdevice_master_wins (void *conn,
2855  const void *message)
2856 {
2859  cs_error_t error = CS_OK;
2860  uint32_t oldflags = us->flags;
2861 
2862  ENTER();
2863 
2864  if (!qdevice_can_operate) {
2865  error = CS_ERR_ACCESS;
2866  goto out;
2867  }
2868 
2869  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2870  if (strncmp(req_lib_votequorum_qdevice_master_wins->name, qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN)) {
2871  error = CS_ERR_INVALID_PARAM;
2872  goto out;
2873  }
2874 
2875  if (req_lib_votequorum_qdevice_master_wins->allow) {
2877  } else {
2879  }
2880 
2881  if (us->flags != oldflags) {
2882  votequorum_exec_send_nodeinfo(us->node_id);
2883  }
2884 
2885  update_qdevice_master_wins(req_lib_votequorum_qdevice_master_wins->allow);
2886  } else {
2887  error = CS_ERR_NOT_EXIST;
2888  }
2889 
2890 out:
2893  res_lib_votequorum_status.header.error = error;
2895 
2896  LEAVE();
2897 }
uint32_t expected_votes
char name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
char newname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define VOTEQUORUM_INFO_QUORATE
#define TOTEM_AGREED
Definition: coroapi.h:89
const char * name
Definition: coroapi.h:432
uint32_t votes
uint32_t nodeid
char oldname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define VOTEQUORUM_READCONFIG_STARTUP
const char * get_run_dir(void)
Definition: util.c:174
const char * icmap_iter_next(icmap_iter_t iter, size_t *value_len, icmap_value_types_t *type)
Definition: icmap.c:1103
#define NODE_FLAGS_WFASTATUS
void(* lib_handler_fn)(void *conn, const void *msg)
Definition: coroapi.h:418
#define LOGSYS_LEVEL_INFO
Definition: logsys.h:73
uint32_t value
struct list_head * next
Definition: list.h:47
#define NODE_FLAGS_QUORATE
void(* timer_delete)(corosync_timer_handle_t timer_handle)
Definition: coroapi.h:193
#define VOTEQUORUM_QDEVICE_DEFAULT_SYNC_TIMEOUT
struct list_head list
void icmap_iter_finalize(icmap_iter_t iter)
Definition: icmap.c:1124
#define VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER
#define MESSAGE_REQ_EXEC_VOTEQUORUM_RECONFIGURE
#define max(a, b)
int(* timer_add_duration)(unsigned long long nanoseconds_in_future, void *data, void(*timer_nf)(void *data), corosync_timer_handle_t *handle)
Definition: coroapi.h:181
#define list_iterate(v, head)
char * votequorum_init(struct corosync_api_v1 *api, quorum_set_quorate_fn_t q_set_quorate_fn)
nodestate_t
#define VOTEQUORUM_RECONFIG_PARAM_CANCEL_WFA
int tracking_enabled
char name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
char qdevice_name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define CS_TRACK_CURRENT
Definition: corotypes.h:74
#define NODE_FLAGS_QDEVICE_MASTER_WINS
nodestate_t state
#define VOTEQUORUM_INFO_LAST_MAN_STANDING
struct message_header header
Definition: totemsrp.c:60
#define VOTEQUORUM_INFO_WAIT_FOR_ALL
#define NODE_FLAGS_QDEVICE_CAST_VOTE
uint32_t operation
char name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_RECONFIGURE
#define VOTEQUORUM_INFO_TWONODE
char qdevice_name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
Definition: list.h:46
#define VOTEQUORUM_INFO_QDEVICE_REGISTERED
#define log_printf(level, format, args...)
Definition: logsys.h:217
void(* exec_handler_fn)(const void *msg, unsigned int nodeid)
Definition: coroapi.h:423
char oldname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define VOTEQUORUM_QDEVICE_NODEID
#define VOTEQUORUM_INFO_QDEVICE_MASTER_WINS
#define VOTEQUORUM_NODESTATE_MEMBER
#define CS_TRACK_CHANGES
Definition: corotypes.h:75
#define SERVICE_ID_MAKE(a, b)
Definition: coroapi.h:411
#define ICMAP_TRACK_DELETE
Definition: icmap.h:77
uint32_t expected_votes
#define ICMAP_KEYNAME_MAXLEN
Definition: icmap.h:48
#define VOTEQUORUM_QDEVICE_OPERATION_REGISTER
cs_error_t icmap_get_uint8(const char *key_name, uint8_t *u8)
Definition: icmap.c:842
#define VOTEQUORUM_INFO_ALLOW_DOWNSCALE
#define LOGSYS_LEVEL_WARNING
Definition: logsys.h:71
#define ICMAP_TRACK_MODIFY
Definition: icmap.h:78
#define VOTEQUORUM_INFO_QDEVICE_ALIVE
void *(* ipc_private_data_get)(void *conn)
Definition: coroapi.h:208
cs_error_t icmap_set_uint32(const char *key_name, uint32_t value)
Definition: icmap.c:611
void * user_data
Definition: sam.c:126
struct list_head list
unsigned int nodeid
Definition: coroapi.h:96
#define CS_TRACK_CHANGES_ONLY
Definition: corotypes.h:76
#define ICMAP_TRACK_ADD
Definition: icmap.h:76
uint32_t flags
char name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
Linked list API.
struct totem_ip_address rep
Definition: coroapi.h:104
#define COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED
Definition: coroapi.h:129
cs_error_t
Definition: corotypes.h:78
unsigned char track_flags
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:74
LOGSYS_DECLARE_SUBSYS("VOTEQ")
typedef __attribute__
int(* ipc_dispatch_send)(void *conn, const void *msg, size_t mlen)
Definition: coroapi.h:215
cs_error_t icmap_get_uint32(const char *key_name, uint32_t *u32)
Definition: icmap.c:866
uint8_t param
uint32_t quorate
Definition: sam.c:133
#define swab32(x)
Definition: swab.h:43
int(* totem_mcast)(const struct iovec *iovec, unsigned int iov_len, unsigned int guarantee)
Definition: coroapi.h:233
#define VOTEQUORUM_INFO_AUTO_TIE_BREAKER
struct corosync_service_engine * votequorum_get_service_engine_ver0(void)
#define ENTER
Definition: logsys.h:218
int(* ipc_response_send)(void *conn, const void *msg, size_t mlen)
Definition: coroapi.h:210
char * corosync_service_link_and_init(struct corosync_api_v1 *corosync_api, struct default_service *service)
Link and initialize a service.
Definition: service.c:117
char newname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define VOTEQUORUM_NODESTATE_LEAVING
#define PROCESSOR_COUNT_MAX
Definition: coroapi.h:83
void(* error_memory_failure)(void) __attribute__((noreturn))
Definition: coroapi.h:375
#define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_REG
#define VOTEQUORUM_READCONFIG_RUNTIME
struct list_head * prev
Definition: list.h:48
#define MESSAGE_REQ_EXEC_VOTEQUORUM_NODEINFO
#define VOTEQUORUM_RECONFIG_PARAM_NODE_VOTES
#define VOTEQUORUM_QDEVICE_MAX_NAME_LEN
qb_loop_timer_handle corosync_timer_handle_t
Definition: coroapi.h:64
cs_error_t icmap_get_string(const char *key_name, char **str)
Definition: icmap.c:896
#define LOGSYS_LEVEL_CRIT
Definition: logsys.h:69
char oldname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define NODE_FLAGS_LEAVING
#define list_entry(ptr, type, member)
Definition: list.h:84
char newname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define COROSYNC_LIB_FLOW_CONTROL_REQUIRED
Definition: coroapi.h:128
#define LOGSYS_LEVEL_NOTICE
Definition: logsys.h:72
unsigned long long seq
Definition: coroapi.h:105
cs_error_t icmap_set_uint8(const char *key_name, uint8_t value)
Definition: icmap.c:587
#define VOTEQUORUM_NODESTATE_DEAD
#define VOTEQUORUM_INFO_QDEVICE_CAST_VOTE
#define VOTEQUORUM_QDEVICE_DEFAULT_TIMEOUT
const char * name
Definition: service.h:43
icmap_iter_t icmap_iter_init(const char *prefix)
Definition: icmap.c:1097
struct memb_ring_id ring_id
Definition: totemsrp.c:64
uint64_t tracking_context
void(* quorum_set_quorate_fn_t)(const unsigned int *view_list, size_t view_list_entries, int quorate, struct memb_ring_id *)
Definition: exec/quorum.h:42
unsigned int(* totem_nodeid_get)(void)
Definition: coroapi.h:227
#define VOTEQUORUM_RECONFIG_PARAM_EXPECTED_VOTES
#define DEFAULT_LMS_WIN
#define LEAVE
Definition: logsys.h:219
#define NODE_FLAGS_QDEVICE_ALIVE
qb_map_iter_t * icmap_iter_t
Definition: icmap.h:121
#define NODE_FLAGS_QDEVICE_REGISTERED
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Definition: icmap.c:1167
#define NODE_FLAGS_FIRST
struct qb_ipc_request_header header __attribute__((aligned(8)))
#define ICMAP_TRACK_PREFIX
Definition: icmap.h:84