root/daemons/controld/controld_te_callbacks.c

/* */

DEFINITIONS

1. shutdown_lock_cleared
2. te_update_diff_v1
3. process_lrm_resource_diff
4. process_resource_updates
5. extract_node_uuid
6. abort_unless_down
7. process_op_deletion
8. process_delete_diff
9. process_node_state_diff
10. process_status_diff
11. process_cib_diff
12. te_update_diff_v2
13. te_update_diff
14. process_te_message
15. cib_action_updated
16. action_timer_callback

   1 /*
   2  * Copyright 2004-2022 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/stat.h>
  13 
  14 #include <crm/crm.h>
  15 #include <crm/common/xml.h>
  16 #include <crm/common/xml_internal.h>
  17 #include <crm/msg_xml.h>
  18 #include <crm/cluster.h>        /* For ONLINESTATUS etc */
  19 
  20 #include <pacemaker-controld.h>
  21 
  22 void te_update_confirm(const char *event, xmlNode * msg);
  23 
  24 extern char *te_uuid;
  25 gboolean shuttingdown = FALSE;
  26 pcmk__graph_t *transition_graph;
  27 crm_trigger_t *transition_trigger = NULL;
  28 
  29 #define RSC_OP_PREFIX "//" XML_TAG_DIFF_ADDED "//" XML_TAG_CIB \
  30                       "//" XML_LRM_TAG_RSC_OP "[@" XML_ATTR_ID "='"
  31 
  32 // An explicit shutdown-lock of 0 means the lock has been cleared
  33 static bool
  34 shutdown_lock_cleared(xmlNode *lrm_resource)
     /*  */
  35 {
  36     time_t shutdown_lock = 0;
  37 
  38     return (crm_element_value_epoch(lrm_resource, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
  39                                     &shutdown_lock) == pcmk_ok)
  40            && (shutdown_lock == 0);
  41 }
  42 
  43 static void
  44 te_update_diff_v1(const char *event, xmlNode *diff)
     /*  */
  45 {
  46     int lpc, max;
  47     xmlXPathObject *xpathObj = NULL;
  48     GString *rsc_op_xpath = NULL;
  49 
  50     CRM_CHECK(diff != NULL, return);
  51 
  52     xml_log_patchset(LOG_TRACE, __func__, diff);
  53     if (cib_config_changed(NULL, NULL, &diff)) {
  54         abort_transition(INFINITY, pcmk__graph_restart, "Non-status change",
  55                          diff);
  56         goto bail;              /* configuration changed */
  57     }
  58 
  59     /* Tickets Attributes - Added/Updated */
  60     xpathObj =
  61         xpath_search(diff,
  62                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_TICKETS);
  63     if (numXpathResults(xpathObj) > 0) {
  64         xmlNode *aborted = getXpathResult(xpathObj, 0);
  65 
  66         abort_transition(INFINITY, pcmk__graph_restart,
  67                          "Ticket attribute: update", aborted);
  68         goto bail;
  69 
  70     }
  71     freeXpathObject(xpathObj);
  72 
  73     /* Tickets Attributes - Removed */
  74     xpathObj =
  75         xpath_search(diff,
  76                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_TICKETS);
  77     if (numXpathResults(xpathObj) > 0) {
  78         xmlNode *aborted = getXpathResult(xpathObj, 0);
  79 
  80         abort_transition(INFINITY, pcmk__graph_restart,
  81                          "Ticket attribute: removal", aborted);
  82         goto bail;
  83     }
  84     freeXpathObject(xpathObj);
  85 
  86     /* Transient Attributes - Removed */
  87     xpathObj =
  88         xpath_search(diff,
  89                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//"
  90                      XML_TAG_TRANSIENT_NODEATTRS);
  91     if (numXpathResults(xpathObj) > 0) {
  92         xmlNode *aborted = getXpathResult(xpathObj, 0);
  93 
  94         abort_transition(INFINITY, pcmk__graph_restart,
  95                          "Transient attribute: removal", aborted);
  96         goto bail;
  97 
  98     }
  99     freeXpathObject(xpathObj);
 100 
 101     // Check for lrm_resource entries
 102     xpathObj = xpath_search(diff,
 103                             "//" F_CIB_UPDATE_RESULT
 104                             "//" XML_TAG_DIFF_ADDED
 105                             "//" XML_LRM_TAG_RESOURCE);
 106     max = numXpathResults(xpathObj);
 107 
 108     /*
 109      * Updates by, or in response to, graph actions will never affect more than
 110      * one resource at a time, so such updates indicate an LRM refresh. In that
 111      * case, start a new transition rather than check each result individually,
 112      * which can result in _huge_ speedups in large clusters.
 113      *
 114      * Unfortunately, we can only do so when there are no pending actions.
 115      * Otherwise, we could mistakenly throw away those results here, and
 116      * the cluster will stall waiting for them and time out the operation.
 117      */
 118     if ((transition_graph->pending == 0) && (max > 1)) {
 119         crm_debug("Ignoring resource operation updates due to history refresh of %d resources",
 120                   max);
 121         crm_log_xml_trace(diff, "lrm-refresh");
 122         abort_transition(INFINITY, pcmk__graph_restart, "History refresh",
 123                          NULL);
 124         goto bail;
 125     }
 126 
 127     if (max == 1) {
 128         xmlNode *lrm_resource = getXpathResult(xpathObj, 0);
 129 
 130         if (shutdown_lock_cleared(lrm_resource)) {
 131             // @TODO would be more efficient to abort once after transition done
 132             abort_transition(INFINITY, pcmk__graph_restart,
 133                              "Shutdown lock cleared", lrm_resource);
 134             // Still process results, so we stop timers and update failcounts
 135         }
 136     }
 137     freeXpathObject(xpathObj);
 138 
 139     /* Process operation updates */
 140     xpathObj =
 141         xpath_search(diff,
 142                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_LRM_TAG_RSC_OP);
 143     max = numXpathResults(xpathObj);
 144     if (max > 0) {
 145         int lpc = 0;
 146 
 147         for (lpc = 0; lpc < max; lpc++) {
 148             xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
 149             const char *node = get_node_id(rsc_op);
 150 
 151             process_graph_event(rsc_op, node);
 152         }
 153     }
 154     freeXpathObject(xpathObj);
 155 
 156     /* Detect deleted (as opposed to replaced or added) actions - eg. crm_resource -C */
 157     xpathObj = xpath_search(diff, "//" XML_TAG_DIFF_REMOVED "//" XML_LRM_TAG_RSC_OP);
 158     max = numXpathResults(xpathObj);
 159     for (lpc = 0; lpc < max; lpc++) {
 160         const char *op_id = NULL;
 161         xmlXPathObject *op_match = NULL;
 162         xmlNode *match = getXpathResult(xpathObj, lpc);
 163 
 164         CRM_LOG_ASSERT(match != NULL);
 165         if(match == NULL) { continue; };
 166 
 167         op_id = ID(match);
 168 
 169         if (rsc_op_xpath == NULL) {
 170             rsc_op_xpath = g_string_new(RSC_OP_PREFIX);
 171         } else {
 172             g_string_truncate(rsc_op_xpath, sizeof(RSC_OP_PREFIX) - 1);
 173         }
 174         pcmk__g_strcat(rsc_op_xpath, op_id, "']", NULL);
 175 
 176         op_match = xpath_search(diff, (const char *) rsc_op_xpath->str);
 177         if (numXpathResults(op_match) == 0) {
 178             /* Prevent false positives by matching cancelations too */
 179             const char *node = get_node_id(match);
 180             pcmk__graph_action_t *cancelled = get_cancel_action(op_id, node);
 181 
 182             if (cancelled == NULL) {
 183                 crm_debug("No match for deleted action %s (%s on %s)",
 184                           (const char *) rsc_op_xpath->str, op_id, node);
 185                 abort_transition(INFINITY, pcmk__graph_restart,
 186                                  "Resource op removal", match);
 187                 freeXpathObject(op_match);
 188                 goto bail;
 189 
 190             } else {
 191                 crm_debug("Deleted lrm_rsc_op %s on %s was for graph event %d",
 192                           op_id, node, cancelled->id);
 193             }
 194         }
 195 
 196         freeXpathObject(op_match);
 197     }
 198 
 199   bail:
 200     freeXpathObject(xpathObj);
 201     if (rsc_op_xpath != NULL) {
 202         g_string_free(rsc_op_xpath, TRUE);
 203     }
 204 }
 205 
 206 static void
 207 process_lrm_resource_diff(xmlNode *lrm_resource, const char *node)
     /*  */
 208 {
 209     for (xmlNode *rsc_op = pcmk__xml_first_child(lrm_resource); rsc_op != NULL;
 210          rsc_op = pcmk__xml_next(rsc_op)) {
 211         process_graph_event(rsc_op, node);
 212     }
 213     if (shutdown_lock_cleared(lrm_resource)) {
 214         // @TODO would be more efficient to abort once after transition done
 215         abort_transition(INFINITY, pcmk__graph_restart, "Shutdown lock cleared",
 216                          lrm_resource);
 217     }
 218 }
 219 
 220 static void
 221 process_resource_updates(const char *node, xmlNode *xml, xmlNode *change,
     /*  */
 222                          const char *op, const char *xpath)
 223 {
 224     xmlNode *rsc = NULL;
 225 
 226     if (xml == NULL) {
 227         return;
 228     }
 229 
 230     if (strcmp(TYPE(xml), XML_CIB_TAG_LRM) == 0) {
 231         xml = first_named_child(xml, XML_LRM_TAG_RESOURCES);
 232         CRM_CHECK(xml != NULL, return);
 233     }
 234 
 235     CRM_CHECK(strcmp(TYPE(xml), XML_LRM_TAG_RESOURCES) == 0, return);
 236 
 237     /*
 238      * Updates by, or in response to, TE actions will never contain updates
 239      * for more than one resource at a time, so such updates indicate an
 240      * LRM refresh.
 241      *
 242      * In that case, start a new transition rather than check each result
 243      * individually, which can result in _huge_ speedups in large clusters.
 244      *
 245      * Unfortunately, we can only do so when there are no pending actions.
 246      * Otherwise, we could mistakenly throw away those results here, and
 247      * the cluster will stall waiting for them and time out the operation.
 248      */
 249     if ((transition_graph->pending == 0)
 250         && xml->children && xml->children->next) {
 251 
 252         crm_log_xml_trace(change, "lrm-refresh");
 253         abort_transition(INFINITY, pcmk__graph_restart, "History refresh",
 254                          NULL);
 255         return;
 256     }
 257 
 258     for (rsc = pcmk__xml_first_child(xml); rsc != NULL;
 259          rsc = pcmk__xml_next(rsc)) {
 260         crm_trace("Processing %s", ID(rsc));
 261         process_lrm_resource_diff(rsc, node);
 262     }
 263 }
 264 
 265 static char *extract_node_uuid(const char *xpath) 
     /*  */
 266 {
 267     char *mutable_path = strdup(xpath);
 268     char *node_uuid = NULL;
 269     char *search = NULL;
 270     char *match = NULL;
 271 
 272     match = strstr(mutable_path, "node_state[@id=\'");
 273     if (match == NULL) {
 274         free(mutable_path);
 275         return NULL;
 276     }
 277     match += strlen("node_state[@id=\'");
 278 
 279     search = strchr(match, '\'');
 280     if (search == NULL) {
 281         free(mutable_path);
 282         return NULL;
 283     }
 284     search[0] = 0;
 285 
 286     node_uuid = strdup(match);
 287     free(mutable_path);
 288     return node_uuid;
 289 }
 290 
 291 static void
 292 abort_unless_down(const char *xpath, const char *op, xmlNode *change,
     /*  */
 293                   const char *reason)
 294 {
 295     char *node_uuid = NULL;
 296     pcmk__graph_action_t *down = NULL;
 297 
 298     if(!pcmk__str_eq(op, "delete", pcmk__str_casei)) {
 299         abort_transition(INFINITY, pcmk__graph_restart, reason, change);
 300         return;
 301     }
 302 
 303     node_uuid = extract_node_uuid(xpath);
 304     if(node_uuid == NULL) {
 305         crm_err("Could not extract node ID from %s", xpath);
 306         abort_transition(INFINITY, pcmk__graph_restart, reason, change);
 307         return;
 308     }
 309 
 310     down = match_down_event(node_uuid);
 311     if (down == NULL) {
 312         crm_trace("Not expecting %s to be down (%s)", node_uuid, xpath);
 313         abort_transition(INFINITY, pcmk__graph_restart, reason, change);
 314     } else {
 315         crm_trace("Expecting changes to %s (%s)", node_uuid, xpath);
 316     }
 317     free(node_uuid);
 318 }
 319 
 320 static void
 321 process_op_deletion(const char *xpath, xmlNode *change)
     /*  */
 322 {
 323     char *mutable_key = strdup(xpath);
 324     char *key;
 325     char *node_uuid;
 326 
 327     // Extract the part of xpath between last pair of single quotes
 328     key = strrchr(mutable_key, '\'');
 329     if (key != NULL) {
 330         *key = '\0';
 331         key = strrchr(mutable_key, '\'');
 332     }
 333     if (key == NULL) {
 334         crm_warn("Ignoring malformed CIB update (resource deletion of %s)",
 335                  xpath);
 336         free(mutable_key);
 337         return;
 338     }
 339     ++key;
 340 
 341     node_uuid = extract_node_uuid(xpath);
 342     if (confirm_cancel_action(key, node_uuid) == FALSE) {
 343         abort_transition(INFINITY, pcmk__graph_restart,
 344                          "Resource operation removal", change);
 345     }
 346     free(mutable_key);
 347     free(node_uuid);
 348 }
 349 
 350 static void
 351 process_delete_diff(const char *xpath, const char *op, xmlNode *change)
     /*  */
 352 {
 353     if (strstr(xpath, "/" XML_LRM_TAG_RSC_OP "[")) {
 354         process_op_deletion(xpath, change);
 355 
 356     } else if (strstr(xpath, "/" XML_CIB_TAG_LRM "[")) {
 357         abort_unless_down(xpath, op, change, "Resource state removal");
 358 
 359     } else if (strstr(xpath, "/" XML_CIB_TAG_STATE "[")) {
 360         abort_unless_down(xpath, op, change, "Node state removal");
 361 
 362     } else {
 363         crm_trace("Ignoring delete of %s", xpath);
 364     }
 365 }
 366 
 367 static void
 368 process_node_state_diff(xmlNode *state, xmlNode *change, const char *op,
     /*  */
 369                         const char *xpath)
 370 {
 371     xmlNode *lrm = first_named_child(state, XML_CIB_TAG_LRM);
 372 
 373     process_resource_updates(ID(state), lrm, change, op, xpath);
 374 }
 375 
 376 static void
 377 process_status_diff(xmlNode *status, xmlNode *change, const char *op,
     /*  */
 378                     const char *xpath)
 379 {
 380     for (xmlNode *state = pcmk__xml_first_child(status); state != NULL;
 381          state = pcmk__xml_next(state)) {
 382         process_node_state_diff(state, change, op, xpath);
 383     }
 384 }
 385 
 386 static void
 387 process_cib_diff(xmlNode *cib, xmlNode *change, const char *op,
     /*  */
 388                  const char *xpath)
 389 {
 390     xmlNode *status = first_named_child(cib, XML_CIB_TAG_STATUS);
 391     xmlNode *config = first_named_child(cib, XML_CIB_TAG_CONFIGURATION);
 392 
 393     if (status) {
 394         process_status_diff(status, change, op, xpath);
 395     }
 396     if (config) {
 397         abort_transition(INFINITY, pcmk__graph_restart,
 398                          "Non-status-only change", change);
 399     }
 400 }
 401 
 402 static void
 403 te_update_diff_v2(xmlNode *diff)
     /*  */
 404 {
 405     crm_log_xml_trace(diff, "Patch:Raw");
 406 
 407     for (xmlNode *change = pcmk__xml_first_child(diff); change != NULL;
 408          change = pcmk__xml_next(change)) {
 409 
 410         xmlNode *match = NULL;
 411         const char *name = NULL;
 412         const char *xpath = crm_element_value(change, XML_DIFF_PATH);
 413 
 414         // Possible ops: create, modify, delete, move
 415         const char *op = crm_element_value(change, XML_DIFF_OP);
 416 
 417         // Ignore uninteresting updates
 418         if (op == NULL) {
 419             continue;
 420 
 421         } else if (xpath == NULL) {
 422             crm_trace("Ignoring %s change for version field", op);
 423             continue;
 424 
 425         } else if ((strcmp(op, "move") == 0)
 426                    && (strstr(xpath,
 427                               "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION
 428                               "/" XML_CIB_TAG_RESOURCES) == NULL)) {
 429             /* We still need to consider moves within the resources section,
 430              * since they affect placement order.
 431              */
 432             crm_trace("Ignoring move change at %s", xpath);
 433             continue;
 434         }
 435 
 436         // Find the result of create/modify ops
 437         if (strcmp(op, "create") == 0) {
 438             match = change->children;
 439 
 440         } else if (strcmp(op, "modify") == 0) {
 441             match = first_named_child(change, XML_DIFF_RESULT);
 442             if(match) {
 443                 match = match->children;
 444             }
 445 
 446         } else if (!pcmk__str_any_of(op, "delete", "move", NULL)) {
 447             crm_warn("Ignoring malformed CIB update (%s operation on %s is unrecognized)",
 448                      op, xpath);
 449             continue;
 450         }
 451 
 452         if (match) {
 453             if (match->type == XML_COMMENT_NODE) {
 454                 crm_trace("Ignoring %s operation for comment at %s", op, xpath);
 455                 continue;
 456             }
 457             name = (const char *)match->name;
 458         }
 459 
 460         crm_trace("Handling %s operation for %s%s%s",
 461                   op, (xpath? xpath : "CIB"),
 462                   (name? " matched by " : ""), (name? name : ""));
 463 
 464         if (strstr(xpath, "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION)) {
 465             abort_transition(INFINITY, pcmk__graph_restart,
 466                              "Configuration change", change);
 467             break; // Won't be packaged with operation results we may be waiting for
 468 
 469         } else if (strstr(xpath, "/" XML_CIB_TAG_TICKETS)
 470                    || pcmk__str_eq(name, XML_CIB_TAG_TICKETS, pcmk__str_none)) {
 471             abort_transition(INFINITY, pcmk__graph_restart,
 472                              "Ticket attribute change", change);
 473             break; // Won't be packaged with operation results we may be waiting for
 474 
 475         } else if (strstr(xpath, "/" XML_TAG_TRANSIENT_NODEATTRS "[")
 476                    || pcmk__str_eq(name, XML_TAG_TRANSIENT_NODEATTRS,
 477                                    pcmk__str_none)) {
 478             abort_unless_down(xpath, op, change, "Transient attribute change");
 479             break; // Won't be packaged with operation results we may be waiting for
 480 
 481         } else if (strcmp(op, "delete") == 0) {
 482             process_delete_diff(xpath, op, change);
 483 
 484         } else if (name == NULL) {
 485             crm_warn("Ignoring malformed CIB update (%s at %s has no result)",
 486                      op, xpath);
 487 
 488         } else if (strcmp(name, XML_TAG_CIB) == 0) {
 489             process_cib_diff(match, change, op, xpath);
 490 
 491         } else if (strcmp(name, XML_CIB_TAG_STATUS) == 0) {
 492             process_status_diff(match, change, op, xpath);
 493 
 494         } else if (strcmp(name, XML_CIB_TAG_STATE) == 0) {
 495             process_node_state_diff(match, change, op, xpath);
 496 
 497         } else if (strcmp(name, XML_CIB_TAG_LRM) == 0) {
 498             process_resource_updates(ID(match), match, change, op, xpath);
 499 
 500         } else if (strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
 501             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
 502 
 503             process_resource_updates(local_node, match, change, op, xpath);
 504             free(local_node);
 505 
 506         } else if (strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
 507             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
 508 
 509             process_lrm_resource_diff(match, local_node);
 510             free(local_node);
 511 
 512         } else if (strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
 513             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
 514 
 515             process_graph_event(match, local_node);
 516             free(local_node);
 517 
 518         } else {
 519             crm_warn("Ignoring malformed CIB update (%s at %s has unrecognized result %s)",
 520                      op, xpath, name);
 521         }
 522     }
 523 }
 524 
 525 void
 526 te_update_diff(const char *event, xmlNode * msg)
     /*  */
 527 {
 528     xmlNode *diff = NULL;
 529     const char *op = NULL;
 530     int rc = -EINVAL;
 531     int format = 1;
 532     int p_add[] = { 0, 0, 0 };
 533     int p_del[] = { 0, 0, 0 };
 534 
 535     CRM_CHECK(msg != NULL, return);
 536     crm_element_value_int(msg, F_CIB_RC, &rc);
 537 
 538     if (transition_graph == NULL) {
 539         crm_trace("No graph");
 540         return;
 541 
 542     } else if (rc < pcmk_ok) {
 543         crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc));
 544         return;
 545 
 546     } else if (transition_graph->complete
 547                && fsa_state != S_IDLE
 548                && fsa_state != S_TRANSITION_ENGINE
 549                && fsa_state != S_POLICY_ENGINE) {
 550         crm_trace("Filter state=%s (complete)", fsa_state2string(fsa_state));
 551         return;
 552     }
 553 
 554     op = crm_element_value(msg, F_CIB_OPERATION);
 555     diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
 556 
 557     xml_patch_versions(diff, p_add, p_del);
 558     crm_debug("Processing (%s) diff: %d.%d.%d -> %d.%d.%d (%s)", op,
 559               p_del[0], p_del[1], p_del[2], p_add[0], p_add[1], p_add[2],
 560               fsa_state2string(fsa_state));
 561 
 562     crm_element_value_int(diff, "format", &format);
 563     switch (format) {
 564         case 1:
 565             te_update_diff_v1(event, diff);
 566             break;
 567         case 2:
 568             te_update_diff_v2(diff);
 569             break;
 570         default:
 571             crm_warn("Ignoring malformed CIB update (unknown patch format %d)",
 572                      format);
 573     }
 574 }
 575 
 576 void
 577 process_te_message(xmlNode * msg, xmlNode * xml_data)
     /*  */
 578 {
 579     const char *value = NULL;
 580     xmlXPathObject *xpathObj = NULL;
 581     int nmatches = 0;
 582 
 583     CRM_CHECK(msg != NULL, return);
 584 
 585     // Transition requests must specify transition engine as subsystem
 586     value = crm_element_value(msg, F_CRM_SYS_TO);
 587     if (pcmk__str_empty(value)
 588         || !pcmk__str_eq(value, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
 589         crm_info("Received invalid transition request: subsystem '%s' not '"
 590                  CRM_SYSTEM_TENGINE "'", pcmk__s(value, ""));
 591         return;
 592     }
 593 
 594     // Only the lrm_invoke command is supported as a transition request
 595     value = crm_element_value(msg, F_CRM_TASK);
 596     if (!pcmk__str_eq(value, CRM_OP_INVOKE_LRM, pcmk__str_none)) {
 597         crm_info("Received invalid transition request: command '%s' not '"
 598                  CRM_OP_INVOKE_LRM "'", pcmk__s(value, ""));
 599         return;
 600     }
 601 
 602     // Transition requests must be marked as coming from the executor
 603     value = crm_element_value(msg, F_CRM_SYS_FROM);
 604     if (!pcmk__str_eq(value, CRM_SYSTEM_LRMD, pcmk__str_none)) {
 605         crm_info("Received invalid transition request: from '%s' not '"
 606                  CRM_SYSTEM_LRMD "'", pcmk__s(value, ""));
 607         return;
 608     }
 609 
 610     crm_debug("Processing transition request with ref='%s' origin='%s'",
 611               pcmk__s(crm_element_value(msg, F_CRM_REFERENCE), ""),
 612               pcmk__s(crm_element_value(msg, F_ORIG), ""));
 613 
 614     xpathObj = xpath_search(xml_data, "//" XML_LRM_TAG_RSC_OP);
 615     nmatches = numXpathResults(xpathObj);
 616     if (nmatches == 0) {
 617         crm_err("Received transition request with no results (bug?)");
 618     } else {
 619         for (int lpc = 0; lpc < nmatches; lpc++) {
 620             xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
 621             const char *node = get_node_id(rsc_op);
 622 
 623             process_graph_event(rsc_op, node);
 624         }
 625     }
 626     freeXpathObject(xpathObj);
 627 }
 628 
 629 void
 630 cib_action_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
     /*  */
 631 {
 632     if (rc < pcmk_ok) {
 633         crm_err("Update %d FAILED: %s", call_id, pcmk_strerror(rc));
 634     }
 635 }
 636 
 637 /*!
 638  * \brief Handle a timeout in node-to-node communication
 639  *
 640  * \param[in,out] data  Pointer to graph action
 641  *
 642  * \return FALSE (indicating that source should be not be re-added)
 643  */
 644 gboolean
 645 action_timer_callback(gpointer data)
     /*  */
 646 {
 647     pcmk__graph_action_t *action = (pcmk__graph_action_t *) data;
 648     const char *task = NULL;
 649     const char *on_node = NULL;
 650     const char *via_node = NULL;
 651 
 652     CRM_CHECK(data != NULL, return FALSE);
 653 
 654     stop_te_timer(action);
 655 
 656     task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
 657     on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 658     via_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 659 
 660     if (transition_graph->complete) {
 661         crm_notice("Node %s did not send %s result (via %s) within %dms "
 662                    "(ignoring because transition not in progress)",
 663                    (on_node? on_node : ""), (task? task : "unknown action"),
 664                    (via_node? via_node : "controller"), action->timeout);
 665     } else {
 666         /* fail the action */
 667 
 668         crm_err("Node %s did not send %s result (via %s) within %dms "
 669                 "(action timeout plus cluster-delay)",
 670                 (on_node? on_node : ""), (task? task : "unknown action"),
 671                 (via_node? via_node : "controller"),
 672                 action->timeout + transition_graph->network_delay);
 673         pcmk__log_graph_action(LOG_ERR, action);
 674 
 675         pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
 676 
 677         te_action_confirmed(action, transition_graph);
 678         abort_transition(INFINITY, pcmk__graph_restart, "Action lost", NULL);
 679 
 680         // Record timeout in the CIB if appropriate
 681         if ((action->type == pcmk__rsc_graph_action)
 682             && controld_action_is_recordable(task)) {
 683             controld_record_action_timeout(action);
 684         }
 685     }
 686 
 687     return FALSE;
 688 }

/* */