Kea 2.2.0
ha_service.cc
Go to the documentation of this file.
1// Copyright (C) 2018-2022 Internet Systems Consortium, Inc. ("ISC")
2//
3// This Source Code Form is subject to the terms of the Mozilla Public
4// License, v. 2.0. If a copy of the MPL was not distributed with this
5// file, You can obtain one at http://mozilla.org/MPL/2.0/.
6
7#include <config.h>
8
9#include <command_creator.h>
10#include <ha_log.h>
11#include <ha_service.h>
12#include <ha_service_states.h>
14#include <cc/data.h>
16#include <config/timeouts.h>
17#include <dhcp/iface_mgr.h>
18#include <dhcpsrv/cfgmgr.h>
19#include <dhcpsrv/lease_mgr.h>
22#include <http/date_time.h>
23#include <http/response_json.h>
26#include <util/stopwatch.h>
27#include <boost/pointer_cast.hpp>
28#include <boost/make_shared.hpp>
29#include <boost/weak_ptr.hpp>
30#include <functional>
31#include <sstream>
32
33using namespace isc::asiolink;
34using namespace isc::config;
35using namespace isc::data;
36using namespace isc::dhcp;
37using namespace isc::hooks;
38using namespace isc::http;
39using namespace isc::log;
40using namespace isc::util;
41namespace ph = std::placeholders;
42
43namespace {
44
46class CommandUnsupportedError : public CtrlChannelError {
47public:
48 CommandUnsupportedError(const char* file, size_t line, const char* what) :
49 CtrlChannelError(file, line, what) {}
50};
51
52}
53
54namespace isc {
55namespace ha {
56
66
67HAService::HAService(const IOServicePtr& io_service, const NetworkStatePtr& network_state,
68 const HAConfigPtr& config, const HAServerType& server_type)
69 : io_service_(io_service), network_state_(network_state), config_(config),
70 server_type_(server_type), client_(), listener_(), communication_state_(),
71 query_filter_(config), mutex_(), pending_requests_(),
72 lease_update_backlog_(config->getDelayedUpdatesLimit()),
73 sync_complete_notified_(false) {
74
75 if (server_type == HAServerType::DHCPv4) {
77
78 } else {
80 }
81
82 network_state_->reset(NetworkState::Origin::HA_COMMAND);
83
85
86 // Create the client and(or) listener as appropriate.
87 if (!config_->getEnableMultiThreading()) {
88 // Not configured for multi-threading, start a client in ST mode.
89 client_.reset(new HttpClient(*io_service_, 0));
90 } else {
91 // Create an MT-mode client.
93 config_->getHttpClientThreads(), true));
94
95 // If we're configured to use our own listener create and start it.
96 if (config_->getHttpDedicatedListener()) {
97 // Get the server address and port from this server's URL.
98 auto my_url = config_->getThisServerConfig()->getUrl();
99 IOAddress server_address(IOAddress::IPV4_ZERO_ADDRESS());
100 try {
101 // Since we do not currently support hostname resolution,
102 // we need to make sure we have an IP address here.
103 server_address = IOAddress(my_url.getStrippedHostname());
104 } catch (const std::exception& ex) {
105 isc_throw(Unexpected, "server Url:" << my_url.getStrippedHostname()
106 << " is not a valid IP address");
107 }
108
109 // Fetch how many threads the listener will use.
110 uint32_t listener_threads = config_->getHttpListenerThreads();
111
112 // Fetch the TLS context.
113 auto tls_context = config_->getThisServerConfig()->getTlsContext();
114
115 // Instantiate the listener.
116 listener_.reset(new CmdHttpListener(server_address, my_url.getPort(),
117 listener_threads, tls_context));
118 // Set the command filter when enabled.
119 if (config_->getRestrictCommands()) {
120 if (server_type == HAServerType::DHCPv4) {
121 CmdResponseCreator::command_accept_list_ =
123 } else {
124 CmdResponseCreator::command_accept_list_ =
126 }
127 }
128 }
129 }
130
132 .arg(HAConfig::HAModeToString(config->getHAMode()))
133 .arg(HAConfig::PeerConfig::roleToString(config->getThisServerConfig()->getRole()));
134}
135
137 // Stop client and/or listener.
139
140 network_state_->reset(NetworkState::Origin::HA_COMMAND);
141}
142
143void
145 StateModel::defineEvents();
146
147 defineEvent(HA_HEARTBEAT_COMPLETE_EVT, "HA_HEARTBEAT_COMPLETE_EVT");
148 defineEvent(HA_LEASE_UPDATES_COMPLETE_EVT, "HA_LEASE_UPDATES_COMPLETE_EVT");
149 defineEvent(HA_SYNCING_FAILED_EVT, "HA_SYNCING_FAILED_EVT");
150 defineEvent(HA_SYNCING_SUCCEEDED_EVT, "HA_SYNCING_SUCCEEDED_EVT");
151 defineEvent(HA_MAINTENANCE_NOTIFY_EVT, "HA_MAINTENANCE_NOTIFY_EVT");
152 defineEvent(HA_MAINTENANCE_START_EVT, "HA_MAINTENANCE_START_EVT");
153 defineEvent(HA_MAINTENANCE_CANCEL_EVT, "HA_MAINTENANCE_CANCEL_EVT");
154 defineEvent(HA_SYNCED_PARTNER_UNAVAILABLE_EVT, "HA_SYNCED_PARTNER_UNAVAILABLE_EVT");
155}
156
157void
159 StateModel::verifyEvents();
160
169}
170
171void
173 StateModel::defineStates();
174
176 std::bind(&HAService::backupStateHandler, this),
177 config_->getStateMachineConfig()->getStateConfig(HA_BACKUP_ST)->getPausing());
178
181 config_->getStateMachineConfig()->getStateConfig(HA_COMMUNICATION_RECOVERY_ST)->getPausing());
182
184 std::bind(&HAService::normalStateHandler, this),
185 config_->getStateMachineConfig()->getStateConfig(HA_HOT_STANDBY_ST)->getPausing());
186
188 std::bind(&HAService::normalStateHandler, this),
189 config_->getStateMachineConfig()->getStateConfig(HA_LOAD_BALANCING_ST)->getPausing());
190
192 std::bind(&HAService::inMaintenanceStateHandler, this),
193 config_->getStateMachineConfig()->getStateConfig(HA_IN_MAINTENANCE_ST)->getPausing());
194
196 std::bind(&HAService::partnerDownStateHandler, this),
197 config_->getStateMachineConfig()->getStateConfig(HA_PARTNER_DOWN_ST)->getPausing());
198
201 config_->getStateMachineConfig()->getStateConfig(HA_PARTNER_IN_MAINTENANCE_ST)->getPausing());
202
204 std::bind(&HAService::passiveBackupStateHandler, this),
205 config_->getStateMachineConfig()->getStateConfig(HA_PASSIVE_BACKUP_ST)->getPausing());
206
208 std::bind(&HAService::readyStateHandler, this),
209 config_->getStateMachineConfig()->getStateConfig(HA_READY_ST)->getPausing());
210
212 std::bind(&HAService::syncingStateHandler, this),
213 config_->getStateMachineConfig()->getStateConfig(HA_SYNCING_ST)->getPausing());
214
216 std::bind(&HAService::terminatedStateHandler, this),
217 config_->getStateMachineConfig()->getStateConfig(HA_TERMINATED_ST)->getPausing());
218
220 std::bind(&HAService::waitingStateHandler, this),
221 config_->getStateMachineConfig()->getStateConfig(HA_WAITING_ST)->getPausing());
222}
223
224void
226 if (doOnEntry()) {
229
230 // Log if the state machine is paused.
232 }
233
234 // There is nothing to do in that state. This server simply receives
235 // lease updates from the partners.
237}
238
239void
241 if (doOnEntry()) {
244
245 // Log if the state machine is paused.
247 }
248
250
253
254 // Check if the clock skew is still acceptable. If not, transition to
255 // the terminated state.
256 } else if (shouldTerminate()) {
258
259 } else if (isPartnerStateInvalid()) {
261
262 } else {
263
264 // Transitions based on the partner's state.
265 switch (communication_state_->getPartnerState()) {
268 break;
269
272 break;
273
276 break;
277
278 case HA_TERMINATED_ST:
280 break;
281
283 if (shouldPartnerDown()) {
285
286 } else {
288 }
289 break;
290
291 case HA_WAITING_ST:
292 case HA_SYNCING_ST:
293 case HA_READY_ST:
294 // The partner seems to be waking up, perhaps after communication-recovery.
295 // If our backlog queue is overflown we need to synchronize our lease database.
296 // There is no need to send ha-reset to the partner because the partner is
297 // already synchronizing its lease database.
298 if (!communication_state_->isCommunicationInterrupted() &&
301 } else {
302 // Backlog was not overflown, so there is no need to synchronize our
303 // lease database. Let's wait until our partner completes synchronization
304 // and transitions to the load-balancing state.
306 }
307 break;
308
309 default:
310 // If the communication is still interrupted, let's continue sitting
311 // in this state until it is resumed or until the transition to the
312 // partner-down state, depending on what happens first.
313 if (communication_state_->isCommunicationInterrupted()) {
315 break;
316 }
317
318 // The communication has been resumed. The partner server must be in a state
319 // in which it can receive outstanding lease updates we collected. The number of
320 // outstanding lease updates must not exceed the configured limit. Finally, the
321 // lease updates must be successfully sent. If that all works, we will transition
322 // to the normal operation.
323 if ((communication_state_->getPartnerState() == getNormalState()) ||
324 (communication_state_->getPartnerState() == HA_COMMUNICATION_RECOVERY_ST)) {
326 // If our lease backlog was overflown or we were unable to send lease
327 // updates to the partner we should notify the partner that it should
328 // synchronize the lease database. We do it by sending ha-reset command.
329 if (sendHAReset()) {
331 }
332 break;
333 }
334 // The backlog was not overflown and we successfully sent our lease updates.
335 // We can now transition to the normal operation state. If the partner
336 // fails to send his outstanding lease updates to us it should send the
337 // ha-reset command to us.
339 break;
340 }
341
342 // The partner appears to be in unexpected state, we have exceeded the number
343 // of lease updates in a backlog or an attempt to send lease updates failed.
344 // In all these cases we follow plan B and transition to the waiting state.
345 // The server will then attempt to synchronize the entire lease database.
347 }
348 }
349
350 // When exiting this state we must ensure that lease updates backlog is cleared.
351 if (doOnExit()) {
353 }
354}
355
356void
358 // If we are transitioning from another state, we have to define new
359 // serving scopes appropriate for the new state. We don't do it if
360 // we remain in this state.
361 if (doOnEntry()) {
364
365 // Log if the state machine is paused.
367 }
368
370
373 return;
374 }
375
376 // Check if the clock skew is still acceptable. If not, transition to
377 // the terminated state.
378 if (shouldTerminate()) {
380 return;
381 }
382
383 // Check if the partner state is valid per current configuration. If it is
384 // in an invalid state let's transition to the waiting state and stay there
385 // until the configuration is corrected.
386 if (isPartnerStateInvalid()) {
388 return;
389 }
390
391 switch (communication_state_->getPartnerState()) {
394 break;
395
398 break;
399
402 break;
403
404 case HA_TERMINATED_ST:
406 break;
407
409 if (shouldPartnerDown()) {
411
412 } else if (config_->amAllowingCommRecovery()) {
414
415 } else {
417 }
418 break;
419
420 default:
422 }
423
424 if (doOnExit()) {
425 // Do nothing here but doOnExit() call clears the "on exit" flag
426 // when transitioning to the communication-recovery state. In that
427 // state we need this flag to be cleared.
428 }
429}
430
431void
433 // If we are transitioning from another state, we have to define new
434 // serving scopes appropriate for the new state. We don't do it if
435 // we remain in this state.
436 if (doOnEntry()) {
437 // In this state the server remains silent and waits for being
438 // shutdown.
441
442 // Log if the state machine is paused.
444
446 }
447
449
450 // We don't transition out of this state unless explicitly mandated
451 // by the administrator via a dedicated command which cancels
452 // the maintenance.
454}
455
456void
458 // If we are transitioning from another state, we have to define new
459 // serving scopes appropriate for the new state. We don't do it if
460 // we remain in this state.
461 if (doOnEntry()) {
462
463 bool maintenance = (getLastEvent() == HA_MAINTENANCE_START_EVT);
464
465 // It may be administratively disabled to handle partner's scope
466 // in case of failure. If this is the case we'll just handle our
467 // default scope (or no scope at all). The user will need to
468 // manually enable this server to handle partner's scope.
469 // If we're in the maintenance mode we serve all scopes because
470 // it is not a failover situation.
471 if (maintenance || config_->getThisServerConfig()->isAutoFailover()) {
473 } else {
475 }
477
478 // Log if the state machine is paused.
480
481 if (maintenance) {
482 // If we ended up in the partner-down state as a result of
483 // receiving the ha-maintenance-start command let's log it.
485 }
486
488 // Partner sent the ha-sync-complete-notify command to indicate that
489 // it has successfully synchronized its lease database but this server
490 // was unable to send heartbeat to this server. Enable the DHCP service
491 // and continue serving the clients in the partner-down state until the
492 // communication with the partner is fixed.
494 }
495
497
500 return;
501 }
502
503 // Check if the clock skew is still acceptable. If not, transition to
504 // the terminated state.
505 if (shouldTerminate()) {
507 return;
508 }
509
510 // Check if the partner state is valid per current configuration. If it is
511 // in an invalid state let's transition to the waiting state and stay there
512 // until the configuration is corrected.
513 if (isPartnerStateInvalid()) {
515 return;
516 }
517
518 switch (communication_state_->getPartnerState()) {
523 break;
524
525 case HA_READY_ST:
526 // If partner allocated new leases for which it didn't send lease updates
527 // to us we should synchronize our database.
528 if (communication_state_->hasPartnerNewUnsentUpdates()) {
530 } else {
531 // We did not miss any lease updates. There is no need to synchronize
532 // the database.
534 }
535 break;
536
537 case HA_TERMINATED_ST:
539 break;
540
541 default:
543 }
544}
545
546void
548 // If we are transitioning from another state, we have to define new
549 // serving scopes appropriate for the new state. We don't do it if
550 // we remain in this state.
551 if (doOnEntry()) {
553
555
556 // Log if the state machine is paused.
558
560 }
561
563
564 if (isModelPaused()) {
566 return;
567 }
568
569 // Check if the clock skew is still acceptable. If not, transition to
570 // the terminated state.
571 if (shouldTerminate()) {
573 return;
574 }
575
576 switch (communication_state_->getPartnerState()) {
579 break;
580 default:
582 }
583}
584
585void
587 // If we are transitioning from another state, we have to define new
588 // serving scopes appropriate for the new state. We don't do it if
589 // we remain in this state.
590 if (doOnEntry()) {
593
594 // In the passive-backup state we don't send heartbeat.
595 communication_state_->stopHeartbeat();
596
597 // Log if the state machine is paused.
599 }
601}
602
603void
605 // If we are transitioning from another state, we have to define new
606 // serving scopes appropriate for the new state. We don't do it if
607 // we remain in this state.
608 if (doOnEntry()) {
611
612 // Log if the state machine is paused.
614 }
615
617
620 return;
621 }
622
623 // Check if the clock skew is still acceptable. If not, transition to
624 // the terminated state.
625 if (shouldTerminate()) {
627 return;
628 }
629
630 // Check if the partner state is valid per current configuration. If it is
631 // in an invalid state let's transition to the waiting state and stay there
632 // until the configuration is corrected.
633 if (isPartnerStateInvalid()) {
635 return;
636 }
637
638 switch (communication_state_->getPartnerState()) {
643 break;
644
647 break;
648
651 break;
652
653 case HA_READY_ST:
654 // If both servers are ready, the primary server "wins" and is
655 // transitioned first.
656 if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::PRIMARY) {
659 } else {
661 }
662 break;
663
664 case HA_TERMINATED_ST:
666 break;
667
669 if (shouldPartnerDown()) {
671
672 } else {
674 }
675 break;
676
677 default:
679 }
680}
681
682void
684 // If we are transitioning from another state, we have to define new
685 // serving scopes appropriate for the new state. We don't do it if
686 // we remain in this state.
687 if (doOnEntry()) {
690
691 // Log if the state machine is paused.
693 }
694
697 return;
698 }
699
700 // Check if the clock skew is still acceptable. If not, transition to
701 // the terminated state.
702 if (shouldTerminate()) {
704 return;
705 }
706
707 // Check if the partner state is valid per current configuration. If it is
708 // in an invalid state let's transition to the waiting state and stay there
709 // until the configuration is corrected.
710 if (isPartnerStateInvalid()) {
712 return;
713 }
714
715 // We don't want to perform synchronous attempt to synchronize with
716 // a partner until we know that the partner is responding. Therefore,
717 // we wait for the heartbeat to complete successfully before we
718 // initiate the synchronization.
719 switch (communication_state_->getPartnerState()) {
720 case HA_TERMINATED_ST:
722 return;
723
725 // If the partner appears to be offline, let's transition to the partner
726 // down state. Otherwise, we'd be stuck trying to synchronize with a
727 // dead partner.
728 if (shouldPartnerDown()) {
730
731 } else {
733 }
734 break;
735
736 default:
737 // We don't want the heartbeat to interfere with the synchronization,
738 // so let's temporarily stop it.
739 communication_state_->stopHeartbeat();
740
741 // Timeout is configured in milliseconds. Need to convert to seconds.
742 unsigned int dhcp_disable_timeout =
743 static_cast<unsigned int>(config_->getSyncTimeout() / 1000);
744 if (dhcp_disable_timeout == 0) {
745 ++dhcp_disable_timeout;
746 }
747
748 // Perform synchronous leases update.
749 std::string status_message;
750 int sync_status = synchronize(status_message,
751 config_->getFailoverPeerConfig()->getName(),
752 dhcp_disable_timeout);
753
754 // If the leases synchronization was successful, let's transition
755 // to the ready state.
756 if (sync_status == CONTROL_RESULT_SUCCESS) {
758
759 } else {
760 // If the synchronization was unsuccessful we're back to the
761 // situation that the partner is unavailable and therefore
762 // we stay in the syncing state.
764 }
765 }
766
767 // Make sure that the heartbeat is re-enabled.
769}
770
771void
773 // If we are transitioning from another state, we have to define new
774 // serving scopes appropriate for the new state. We don't do it if
775 // we remain in this state.
776 if (doOnEntry()) {
779
780 // In the terminated state we don't send heartbeat.
781 communication_state_->stopHeartbeat();
782
783 // Log if the state machine is paused.
785
787 }
788
790}
791
792void
794 // If we are transitioning from another state, we have to define new
795 // serving scopes appropriate for the new state. We don't do it if
796 // we remain in this state.
797 if (doOnEntry()) {
800
801 // Log if the state machine is paused.
803 }
804
805 // Only schedule the heartbeat for non-backup servers.
806 if ((config_->getHAMode() != HAConfig::PASSIVE_BACKUP) &&
807 (config_->getThisServerConfig()->getRole() != HAConfig::PeerConfig::BACKUP)) {
809 }
810
813 return;
814 }
815
816 // Backup server must remain in its own state.
817 if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP) {
819 return;
820 }
821
822 // We're not a backup server, so we're either primary or secondary. If this is
823 // a passive-backup mode of operation, we're primary and we should transition
824 // to the passive-backup state.
825 if (config_->getHAMode() == HAConfig::PASSIVE_BACKUP) {
827 return;
828 }
829
830 // Check if the clock skew is still acceptable. If not, transition to
831 // the terminated state.
832 if (shouldTerminate()) {
834 return;
835 }
836
837 // Check if the partner state is valid per current configuration. If it is
838 // in an invalid state let's sit in the waiting state until the configuration
839 // is corrected.
840 if (isPartnerStateInvalid()) {
842 return;
843 }
844
845 switch (communication_state_->getPartnerState()) {
852 case HA_READY_ST:
853 // If we're configured to not synchronize lease database, proceed directly
854 // to the "ready" state.
855 verboseTransition(config_->amSyncingLeases() ? HA_SYNCING_ST : HA_READY_ST);
856 break;
857
858 case HA_SYNCING_ST:
860 break;
861
862 case HA_TERMINATED_ST:
863 // We have checked above whether the clock skew is exceeding the threshold
864 // and we should terminate. If we're here, it means that the clock skew
865 // is acceptable. The partner may be still in the terminated state because
866 // it hasn't been restarted yet. Probably, this server is the first one
867 // being restarted after syncing the clocks. Let's just sit in the waiting
868 // state until the partner gets restarted.
871 break;
872
873 case HA_WAITING_ST:
874 // If both servers are waiting, the primary server 'wins' and is
875 // transitioned to the next state first.
876 if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::PRIMARY) {
877 // If we're configured to not synchronize lease database, proceed directly
878 // to the "ready" state.
879 verboseTransition(config_->amSyncingLeases() ? HA_SYNCING_ST : HA_READY_ST);
880
881 } else {
883 }
884 break;
885
887 if (shouldPartnerDown()) {
889
890 } else {
892 }
893 break;
894
895 default:
897 }
898}
899
900void
901HAService::verboseTransition(const unsigned state) {
902 // Get current and new state name.
903 std::string current_state_name = getStateLabel(getCurrState());
904 std::string new_state_name = getStateLabel(state);
905
906 // Turn them to upper case so as they are better visible in the logs.
907 boost::to_upper(current_state_name);
908 boost::to_upper(new_state_name);
909
910 if (config_->getHAMode() != HAConfig::PASSIVE_BACKUP) {
911 // If this is load-balancing or hot-standby mode we also want to log
912 // partner's state.
913 auto partner_state = communication_state_->getPartnerState();
914 std::string partner_state_name = getStateLabel(partner_state);
915 boost::to_upper(partner_state_name);
916
917 // Log the transition.
919 .arg(current_state_name)
920 .arg(new_state_name)
921 .arg(partner_state_name);
922
923 } else {
924 // In the passive-backup mode we don't know the partner's state.
926 .arg(current_state_name)
927 .arg(new_state_name);
928 }
929
930 // If we're transitioning directly from the "waiting" to "ready"
931 // state it indicates that the database synchronization is
932 // administratively disabled. Let's remind the user about this
933 // configuration setting.
934 if ((state == HA_READY_ST) && (getCurrState() == HA_WAITING_ST)) {
936 }
937
938 // Do the actual transition.
939 transition(state, getNextEvent());
940
941 // Inform the administrator whether or not lease updates are generated.
942 // Updates are never generated by a backup server so it doesn't make
943 // sense to log anything for the backup server.
944 if ((config_->getHAMode() != HAConfig::PASSIVE_BACKUP) &&
945 (config_->getThisServerConfig()->getRole() != HAConfig::PeerConfig::BACKUP)) {
946 if (shouldSendLeaseUpdates(config_->getFailoverPeerConfig())) {
948 .arg(new_state_name);
949
950 } else if (!config_->amSendingLeaseUpdates()) {
951 // Lease updates are administratively disabled.
953 .arg(new_state_name);
954
955 } else {
956 // Lease updates are not administratively disabled, but they
957 // are not issued because this is the backup server or because
958 // in this state the server should not generate lease updates.
960 .arg(new_state_name);
961 }
962 }
963}
964
965int
967 if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP) {
968 return (HA_BACKUP_ST);
969 }
970
971 switch (config_->getHAMode()) {
973 return (HA_LOAD_BALANCING_ST);
975 return (HA_HOT_STANDBY_ST);
976 default:
977 return (HA_PASSIVE_BACKUP_ST);
978 }
979}
980
981bool
983 if (isModelPaused()) {
985 unpauseModel();
986 return (true);
987 }
988 return (false);
989}
990
991void
993 // Inform the administrator if the state machine is paused.
994 if (isModelPaused()) {
995 std::string state_name = stateToString(getCurrState());
996 boost::to_upper(state_name);
998 .arg(state_name);
999 }
1000}
1001
1002void
1005}
1006
1007bool
1009 return (inScopeInternal(query4));
1010}
1011
1012bool
1014 return (inScopeInternal(query6));
1015}
1016
1017template<typename QueryPtrType>
1018bool
1019HAService::inScopeInternal(QueryPtrType& query) {
1020 // Check if the query is in scope (should be processed by this server).
1021 std::string scope_class;
1022 const bool in_scope = query_filter_.inScope(query, scope_class);
1023 // Whether or not the query is going to be processed by this server,
1024 // we associate the query with the appropriate class.
1025 query->addClass(dhcp::ClientClass(scope_class));
1026 // The following is the part of the server failure detection algorithm.
1027 // If the query should be processed by the partner we need to check if
1028 // the partner responds. If the number of unanswered queries exceeds a
1029 // configured threshold, we will consider the partner to be offline.
1030 if (!in_scope && communication_state_->isCommunicationInterrupted()) {
1031 communication_state_->analyzeMessage(query);
1032 }
1033 // Indicate if the query is in scope.
1034 return (in_scope);
1035}
1036
1037void
1039 std::string current_state_name = getStateLabel(getCurrState());
1040 boost::to_upper(current_state_name);
1041
1042 // DHCP service should be enabled in the following states.
1043 const bool should_enable = ((getCurrState() == HA_COMMUNICATION_RECOVERY_ST) ||
1050
1051 if (!should_enable && network_state_->isServiceEnabled()) {
1052 std::string current_state_name = getStateLabel(getCurrState());
1053 boost::to_upper(current_state_name);
1055 .arg(config_->getThisServerName())
1056 .arg(current_state_name);
1057 network_state_->disableService(NetworkState::Origin::HA_COMMAND);
1058
1059 } else if (should_enable && !network_state_->isServiceEnabled()) {
1060 std::string current_state_name = getStateLabel(getCurrState());
1061 boost::to_upper(current_state_name);
1063 .arg(config_->getThisServerName())
1064 .arg(current_state_name);
1065 network_state_->enableService(NetworkState::Origin::HA_COMMAND);
1066 }
1067}
1068
1069bool
1071 // Checking whether the communication with the partner is OK is the
1072 // first step towards verifying if the server is up.
1073 if (communication_state_->isCommunicationInterrupted()) {
1074 // If the communication is interrupted, we also have to check
1075 // whether the partner answers DHCP requests. The only cases
1076 // when we don't (can't) do it are: the hot standby configuration
1077 // in which this server is a primary and when the DHCP service is
1078 // disabled so we can't analyze incoming traffic. Note that the
1079 // primary server can't check delayed responses to the partner
1080 // because the partner doesn't respond to any queries in this
1081 // configuration.
1082 if (network_state_->isServiceEnabled() &&
1083 ((config_->getHAMode() == HAConfig::LOAD_BALANCING) ||
1084 (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::STANDBY))) {
1085 return (communication_state_->failureDetected());
1086 }
1087
1088 // Hot standby / primary case.
1089 return (true);
1090 }
1091
1092 // Shouldn't transition to the partner down state.
1093 return (false);
1094}
1095
1096bool
1098 // Check if skew is fatally large.
1099 bool should_terminate = communication_state_->clockSkewShouldTerminate();
1100
1101 // If not issue a warning if it's getting large.
1102 if (!should_terminate) {
1103 communication_state_->clockSkewShouldWarn();
1104 }
1105
1106 return (should_terminate);
1107}
1108
1109bool
1112}
1113
1114bool
1116 switch (communication_state_->getPartnerState()) {
1118 if (config_->getHAMode() != HAConfig::LOAD_BALANCING) {
1120 return (true);
1121 }
1122 break;
1123
1124 case HA_HOT_STANDBY_ST:
1125 if (config_->getHAMode() != HAConfig::HOT_STANDBY) {
1127 return (true);
1128 }
1129 break;
1130
1132 if (config_->getHAMode() != HAConfig::LOAD_BALANCING) {
1134 return (true);
1135 }
1136 break;
1137
1138 default:
1139 ;
1140 }
1141 return (false);
1142}
1143
1144size_t
1146 const dhcp::Lease4CollectionPtr& leases,
1147 const dhcp::Lease4CollectionPtr& deleted_leases,
1148 const hooks::ParkingLotHandlePtr& parking_lot) {
1149
1150 // Get configurations of the peers. Exclude this instance.
1151 HAConfig::PeerConfigMap peers_configs = config_->getOtherServersConfig();
1152
1153 size_t sent_num = 0;
1154
1155 // Schedule sending lease updates to each peer.
1156 for (auto p = peers_configs.begin(); p != peers_configs.end(); ++p) {
1157 HAConfig::PeerConfigPtr conf = p->second;
1158
1159 // Check if the lease updates should be queued. This is the case when the
1160 // server is in the communication-recovery state. Queued lease updates may
1161 // be sent when the communication is re-established.
1162 if (shouldQueueLeaseUpdates(conf)) {
1163 // Lease updates for deleted leases.
1164 for (auto l = deleted_leases->begin(); l != deleted_leases->end(); ++l) {
1166 }
1167
1168 // Lease updates for new allocations and updated leases.
1169 for (auto l = leases->begin(); l != leases->end(); ++l) {
1171 }
1172
1173 continue;
1174 }
1175
1176 // Check if the lease update should be sent to the server. If we're in
1177 // the partner-down state we don't send lease updates to the partner.
1178 if (!shouldSendLeaseUpdates(conf)) {
1179 // If we decide to not send the lease updates to an active partner, we
1180 // should make a record of it in the communication state. The partner
1181 // can check if there were any unsent lease updates when he determines
1182 // whether it should synchronize its database or not when it recovers
1183 // from the partner-down state.
1184 if (conf->getRole() != HAConfig::PeerConfig::BACKUP) {
1185 communication_state_->increaseUnsentUpdateCount();
1186 }
1187 continue;
1188 }
1189
1190 // Lease updates for deleted leases.
1191 for (auto l = deleted_leases->begin(); l != deleted_leases->end(); ++l) {
1193 parking_lot);
1194 }
1195
1196 // Lease updates for new allocations and updated leases.
1197 for (auto l = leases->begin(); l != leases->end(); ++l) {
1199 parking_lot);
1200 }
1201
1202 // If we're contacting a backup server from which we don't expect a
1203 // response prior to responding to the DHCP client we don't count
1204 // it.
1205 if ((config_->amWaitingBackupAck() || (conf->getRole() != HAConfig::PeerConfig::BACKUP))) {
1206 ++sent_num;
1207 }
1208 }
1209
1210 return (sent_num);
1211}
1212
1213size_t
1215 const dhcp::Lease6CollectionPtr& leases,
1216 const dhcp::Lease6CollectionPtr& deleted_leases,
1217 const hooks::ParkingLotHandlePtr& parking_lot) {
1218
1219 // Get configurations of the peers. Exclude this instance.
1220 HAConfig::PeerConfigMap peers_configs = config_->getOtherServersConfig();
1221
1222 size_t sent_num = 0;
1223
1224 // Schedule sending lease updates to each peer.
1225 for (auto p = peers_configs.begin(); p != peers_configs.end(); ++p) {
1226 HAConfig::PeerConfigPtr conf = p->second;
1227
1228 // Check if the lease updates should be queued. This is the case when the
1229 // server is in the communication-recovery state. Queued lease updates may
1230 // be sent when the communication is re-established.
1231 if (shouldQueueLeaseUpdates(conf)) {
1232 for (auto l = deleted_leases->begin(); l != deleted_leases->end(); ++l) {
1234 }
1235
1236 // Lease updates for new allocations and updated leases.
1237 for (auto l = leases->begin(); l != leases->end(); ++l) {
1239 }
1240
1241 continue;
1242 }
1243
1244 // Check if the lease update should be sent to the server. If we're in
1245 // the partner-down state we don't send lease updates to the partner.
1246 if (!shouldSendLeaseUpdates(conf)) {
1247 // If we decide to not send the lease updates to an active partner, we
1248 // should make a record of it in the communication state. The partner
1249 // can check if there were any unsent lease updates when he determines
1250 // whether it should synchronize its database or not when it recovers
1251 // from the partner-down state.
1252 if (conf->getRole() != HAConfig::PeerConfig::BACKUP) {
1253 communication_state_->increaseUnsentUpdateCount();
1254 }
1255 continue;
1256 }
1257
1258 // If we're contacting a backup server from which we don't expect a
1259 // response prior to responding to the DHCP client we don't count
1260 // it.
1261 if (config_->amWaitingBackupAck() || (conf->getRole() != HAConfig::PeerConfig::BACKUP)) {
1262 ++sent_num;
1263 }
1264
1265 // Send new/updated leases and deleted leases in one command.
1266 asyncSendLeaseUpdate(query, conf, CommandCreator::createLease6BulkApply(leases, deleted_leases),
1267 parking_lot);
1268 }
1269
1270 return (sent_num);
1271}
1272
1273template<typename QueryPtrType>
1274bool
1276 const ParkingLotHandlePtr& parking_lot) {
1277 if (MultiThreadingMgr::instance().getMode()) {
1278 std::lock_guard<std::mutex> lock(mutex_);
1279 return (leaseUpdateCompleteInternal(query, parking_lot));
1280 } else {
1281 return (leaseUpdateCompleteInternal(query, parking_lot));
1282 }
1283}
1284
1285template<typename QueryPtrType>
1286bool
1287HAService::leaseUpdateCompleteInternal(QueryPtrType& query,
1288 const ParkingLotHandlePtr& parking_lot) {
1289 auto it = pending_requests_.find(query);
1290
1291 // If there are no more pending requests for this query, let's unpark
1292 // the DHCP packet.
1293 if (it == pending_requests_.end() || (--pending_requests_[query] <= 0)) {
1294 parking_lot->unpark(query);
1295
1296 // If we have unparked the packet we can clear pending requests for
1297 // this query.
1298 if (it != pending_requests_.end()) {
1299 pending_requests_.erase(it);
1300 }
1301 return (true);
1302 }
1303 return (false);
1304}
1305
1306template<typename QueryPtrType>
1307void
1309 if (MultiThreadingMgr::instance().getMode()) {
1310 std::lock_guard<std::mutex> lock(mutex_);
1311 updatePendingRequestInternal(query);
1312 } else {
1313 updatePendingRequestInternal(query);
1314 }
1315}
1316
1317template<typename QueryPtrType>
1318void
1319HAService::updatePendingRequestInternal(QueryPtrType& query) {
1320 if (pending_requests_.count(query) == 0) {
1321 pending_requests_[query] = 1;
1322 } else {
1323 ++pending_requests_[query];
1324 }
1325}
1326
1327template<typename QueryPtrType>
1328void
1329HAService::asyncSendLeaseUpdate(const QueryPtrType& query,
1330 const HAConfig::PeerConfigPtr& config,
1331 const ConstElementPtr& command,
1332 const ParkingLotHandlePtr& parking_lot) {
1333 // Create HTTP/1.1 request including our command.
1334 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
1335 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
1336 HostHttpHeader(config->getUrl().getStrippedHostname()));
1337 config->addBasicAuthHttpHeader(request);
1338 request->setBodyAsJson(command);
1339 request->finalize();
1340
1341 // Response object should also be created because the HTTP client needs
1342 // to know the type of the expected response.
1343 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
1344
1345 // When possible we prefer to pass weak pointers to the queries, rather
1346 // than shared pointers, to avoid memory leaks in case cross reference
1347 // between the pointers.
1348 boost::weak_ptr<typename QueryPtrType::element_type> weak_query(query);
1349
1350 // Schedule asynchronous HTTP request.
1351 client_->asyncSendRequest(config->getUrl(), config->getTlsContext(),
1352 request, response,
1353 [this, weak_query, parking_lot, config]
1354 (const boost::system::error_code& ec,
1355 const HttpResponsePtr& response,
1356 const std::string& error_str) {
1357 // Get the shared pointer of the query. The server should keep the
1358 // pointer to the query and then park it. Therefore, we don't really
1359 // expect it to be null. If it is null, something is really wrong.
1360 QueryPtrType query = weak_query.lock();
1361 if (!query) {
1362 isc_throw(Unexpected, "query is null while receiving response from"
1363 " HA peer. This is programmatic error");
1364 }
1365
1366 // There are three possible groups of errors during the lease update.
1367 // One is the IO error causing issues in communication with the peer.
1368 // Another one is an HTTP parsing error. The last type of error is
1369 // when non-success error code is returned in the response carried
1370 // in the HTTP message or if the JSON response is otherwise broken.
1371
1372 bool lease_update_success = true;
1373
1374 // Handle first two groups of errors.
1375 if (ec || !error_str.empty()) {
1376 LOG_WARN(ha_logger, HA_LEASE_UPDATE_COMMUNICATIONS_FAILED)
1377 .arg(query->getLabel())
1378 .arg(config->getLogLabel())
1379 .arg(ec ? ec.message() : error_str);
1380
1381 // Communication error, so let's drop parked packet. The DHCP
1382 // response will not be sent.
1383 lease_update_success = false;
1384
1385 } else {
1386
1387 // Handle third group of errors.
1388 try {
1389 int rcode = 0;
1390 auto args = verifyAsyncResponse(response, rcode);
1391 // In the v6 case the server may return a list of failed lease
1392 // updates and we should log them.
1393 logFailedLeaseUpdates(query, args);
1394
1395 } catch (const std::exception& ex) {
1397 .arg(query->getLabel())
1398 .arg(config->getLogLabel())
1399 .arg(ex.what());
1400
1401 // Error while doing an update. The DHCP response will not be sent.
1402 lease_update_success = false;
1403 }
1404 }
1405
1406 // We don't care about the result of the lease update to the backup server.
1407 // It is a best effort update.
1408 if ((config->getRole() != HAConfig::PeerConfig::BACKUP) && !lease_update_success) {
1409 // If we were unable to communicate with the partner we set partner's
1410 // state as unavailable.
1411 communication_state_->setPartnerState("unavailable");
1412 }
1413
1414 // It is possible to configure the server to not wait for a response from
1415 // the backup server before we unpark the packet and respond to the client.
1416 // Here we check if we're dealing with such situation.
1417 if (config_->amWaitingBackupAck() || (config->getRole() != HAConfig::PeerConfig::BACKUP)) {
1418 // We're expecting a response from the backup server or it is not
1419 // a backup server and the lease update was unsuccessful. In such
1420 // case the DHCP exchange fails.
1421 if (!lease_update_success) {
1422 parking_lot->drop(query);
1423 }
1424 } else {
1425 // This was a response from the backup server and we're configured to
1426 // not wait for their acknowledgments, so there is nothing more to do.
1427 return;
1428 }
1429
1430 if (leaseUpdateComplete(query, parking_lot)) {
1431 // If we have finished sending the lease updates we need to run the
1432 // state machine until the state machine finds that additional events
1433 // are required, such as next heartbeat or a lease update. The runModel()
1434 // may transition to another state, schedule asynchronous tasks etc.
1435 // Then it returns control to the DHCP server.
1436 runModel(HA_LEASE_UPDATES_COMPLETE_EVT);
1437 }
1438 },
1440 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
1441 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
1442 std::bind(&HAService::clientCloseHandler, this, ph::_1)
1443 );
1444
1445 // The number of pending requests is the number of requests for which we
1446 // expect an acknowledgment prior to responding to the DHCP clients. If
1447 // we're configured to wait for the acks from the backups or it is not
1448 // a backup increase the number of pending requests.
1449 if (config_->amWaitingBackupAck() || (config->getRole() != HAConfig::PeerConfig::BACKUP)) {
1450 // Request scheduled, so update the request counters for the query.
1451 updatePendingRequest(query);
1452 }
1453}
1454
1455bool
1456HAService::shouldSendLeaseUpdates(const HAConfig::PeerConfigPtr& peer_config) const {
1457 // Never send lease updates if they are administratively disabled.
1458 if (!config_->amSendingLeaseUpdates()) {
1459 return (false);
1460 }
1461
1462 // Always send updates to the backup server.
1463 if (peer_config->getRole() == HAConfig::PeerConfig::BACKUP) {
1464 return (true);
1465 }
1466
1467 // Never send updates if this is a backup server.
1468 if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP) {
1469 return (false);
1470 }
1471
1472 // In other case, whether we send lease updates or not depends on our
1473 // state.
1474 switch (getCurrState()) {
1475 case HA_HOT_STANDBY_ST:
1478 return (true);
1479
1480 default:
1481 ;
1482 }
1483
1484 return (false);
1485}
1486
1487bool
1488HAService::shouldQueueLeaseUpdates(const HAConfig::PeerConfigPtr& peer_config) const {
1489 if (!config_->amSendingLeaseUpdates()) {
1490 return (false);
1491 }
1492
1493 if (peer_config->getRole() == HAConfig::PeerConfig::BACKUP) {
1494 return (false);
1495 }
1496
1497 return (getCurrState() == HA_COMMUNICATION_RECOVERY_ST);
1498}
1499
1500void
1501HAService::logFailedLeaseUpdates(const PktPtr& query,
1502 const ConstElementPtr& args) const {
1503 // If there are no arguments, it means that the update was successful.
1504 if (!args || (args->getType() != Element::map)) {
1505 return;
1506 }
1507
1508 // Instead of duplicating the code between the failed-deleted-leases and
1509 // failed-leases, let's just have one function that does it for both.
1510 auto log_proc = [](const PktPtr query, const ConstElementPtr& args,
1511 const std::string& param_name, const log::MessageID& mesid) {
1512
1513 // Check if there are any failed leases.
1514 auto failed_leases = args->get(param_name);
1515
1516 // The failed leases must be a list.
1517 if (failed_leases && (failed_leases->getType() == Element::list)) {
1518 // Go over the failed leases and log each of them.
1519 for (int i = 0; i < failed_leases->size(); ++i) {
1520 auto lease = failed_leases->get(i);
1521 if (lease->getType() == Element::map) {
1522
1523 // ip-address
1524 auto ip_address = lease->get("ip-address");
1525
1526 // lease type
1527 auto lease_type = lease->get("type");
1528
1529 // error-message
1530 auto error_message = lease->get("error-message");
1531
1532 LOG_INFO(ha_logger, mesid)
1533 .arg(query->getLabel())
1534 .arg(lease_type && (lease_type->getType() == Element::string) ?
1535 lease_type->stringValue() : "(unknown)")
1536 .arg(ip_address && (ip_address->getType() == Element::string) ?
1537 ip_address->stringValue() : "(unknown)")
1538 .arg(error_message && (error_message->getType() == Element::string) ?
1539 error_message->stringValue() : "(unknown)");
1540 }
1541 }
1542 }
1543 };
1544
1545 // Process "failed-deleted-leases"
1546 log_proc(query, args, "failed-deleted-leases", HA_LEASE_UPDATE_DELETE_FAILED_ON_PEER);
1547
1548 // Process "failed-leases".
1549 log_proc(query, args, "failed-leases", HA_LEASE_UPDATE_CREATE_UPDATE_FAILED_ON_PEER);
1550}
1551
1553HAService::processStatusGet() const {
1554 ElementPtr ha_servers = Element::createMap();
1555
1556 // Local part
1557 ElementPtr local = Element::createMap();
1559 role = config_->getThisServerConfig()->getRole();
1560 std::string role_txt = HAConfig::PeerConfig::roleToString(role);
1561 local->set("role", Element::create(role_txt));
1562 int state = getCurrState();
1563 try {
1564 local->set("state", Element::create(stateToString(state)));
1565
1566 } catch (...) {
1567 // Empty string on error.
1568 local->set("state", Element::create(std::string()));
1569 }
1570 std::set<std::string> scopes = query_filter_.getServedScopes();
1571 ElementPtr list = Element::createList();
1572 for (std::string scope : scopes) {
1573 list->add(Element::create(scope));
1574 }
1575 local->set("scopes", list);
1576 ha_servers->set("local", local);
1577
1578 // Do not include remote server information if this is a backup server or
1579 // we're in the passive-backup mode.
1580 if ((config_->getHAMode() == HAConfig::PASSIVE_BACKUP) ||
1581 (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP)) {
1582 return (ha_servers);
1583 }
1584
1585 // Remote part
1586 ElementPtr remote = communication_state_->getReport();
1587
1588 try {
1589 role = config_->getFailoverPeerConfig()->getRole();
1590 std::string role_txt = HAConfig::PeerConfig::roleToString(role);
1591 remote->set("role", Element::create(role_txt));
1592
1593 } catch (...) {
1594 remote->set("role", Element::create(std::string()));
1595 }
1596 ha_servers->set("remote", remote);
1597
1598 return (ha_servers);
1599}
1600
1602HAService::processHeartbeat() {
1603 ElementPtr arguments = Element::createMap();
1604 std::string state_label = getState(getCurrState())->getLabel();
1605 arguments->set("state", Element::create(state_label));
1606
1607 std::string date_time = HttpDateTime().rfc1123Format();
1608 arguments->set("date-time", Element::create(date_time));
1609
1610 auto scopes = query_filter_.getServedScopes();
1611 ElementPtr scopes_list = Element::createList();
1612 for (auto scope : scopes) {
1613 scopes_list->add(Element::create(scope));
1614 }
1615 arguments->set("scopes", scopes_list);
1616
1617 arguments->set("unsent-update-count",
1618 Element::create(static_cast<int64_t>(communication_state_->getUnsentUpdateCount())));
1619
1620 return (createAnswer(CONTROL_RESULT_SUCCESS, "HA peer status returned.",
1621 arguments));
1622}
1623
1625HAService::processHAReset() {
1626 if (getCurrState() == HA_WAITING_ST) {
1627 return (createAnswer(CONTROL_RESULT_SUCCESS, "HA state machine already in WAITING state."));
1628 }
1629 verboseTransition(HA_WAITING_ST);
1630 runModel(NOP_EVT);
1631 return (createAnswer(CONTROL_RESULT_SUCCESS, "HA state machine reset."));
1632}
1633
1634void
1635HAService::asyncSendHeartbeat() {
1636 HAConfig::PeerConfigPtr partner_config = config_->getFailoverPeerConfig();
1637
1638 // If the sync_complete_notified_ is true it means that the partner
1639 // notified us that it had completed lease database synchronization.
1640 // We confirm that the partner is operational by sending the heartbeat
1641 // to it. Regardless if the partner responds to our heartbeats or not,
1642 // we should clear this flag. But, since we need the current value in
1643 // the async call handler, we save it in the local variable before
1644 // clearing it.
1645 bool sync_complete_notified = sync_complete_notified_;
1646 sync_complete_notified_ = false;
1647
1648 // Create HTTP/1.1 request including our command.
1649 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
1650 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
1651 HostHttpHeader(partner_config->getUrl().getStrippedHostname()));
1652 partner_config->addBasicAuthHttpHeader(request);
1653 request->setBodyAsJson(CommandCreator::createHeartbeat(server_type_));
1654 request->finalize();
1655
1656 // Response object should also be created because the HTTP client needs
1657 // to know the type of the expected response.
1658 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
1659
1660 // Schedule asynchronous HTTP request.
1661 client_->asyncSendRequest(partner_config->getUrl(),
1662 partner_config->getTlsContext(),
1663 request, response,
1664 [this, partner_config, sync_complete_notified]
1665 (const boost::system::error_code& ec,
1666 const HttpResponsePtr& response,
1667 const std::string& error_str) {
1668
1669 // There are three possible groups of errors during the heartbeat.
1670 // One is the IO error causing issues in communication with the peer.
1671 // Another one is an HTTP parsing error. The last type of error is
1672 // when non-success error code is returned in the response carried
1673 // in the HTTP message or if the JSON response is otherwise broken.
1674
1675 bool heartbeat_success = true;
1676
1677 // Handle first two groups of errors.
1678 if (ec || !error_str.empty()) {
1679 LOG_WARN(ha_logger, HA_HEARTBEAT_COMMUNICATIONS_FAILED)
1680 .arg(partner_config->getLogLabel())
1681 .arg(ec ? ec.message() : error_str);
1682 heartbeat_success = false;
1683
1684 } else {
1685
1686 // Handle third group of errors.
1687 try {
1688 // Response must contain arguments and the arguments must
1689 // be a map.
1690 int rcode = 0;
1691 ConstElementPtr args = verifyAsyncResponse(response, rcode);
1692 if (!args || args->getType() != Element::map) {
1693 isc_throw(CtrlChannelError, "returned arguments in the response"
1694 " must be a map");
1695 }
1696 // Response must include partner's state.
1697 ConstElementPtr state = args->get("state");
1698 if (!state || state->getType() != Element::string) {
1699 isc_throw(CtrlChannelError, "server state not returned in response"
1700 " to a ha-heartbeat command or it is not a string");
1701 }
1702 // Remember the partner's state. This may throw if the returned
1703 // state is invalid.
1704 communication_state_->setPartnerState(state->stringValue());
1705
1706 ConstElementPtr date_time = args->get("date-time");
1707 if (!date_time || date_time->getType() != Element::string) {
1708 isc_throw(CtrlChannelError, "date-time not returned in response"
1709 " to a ha-heartbeat command or it is not a string");
1710 }
1711 // Note the time returned by the partner to calculate the clock skew.
1712 communication_state_->setPartnerTime(date_time->stringValue());
1713
1714 // Remember the scopes served by the partner.
1715 try {
1716 auto scopes = args->get("scopes");
1717 communication_state_->setPartnerScopes(scopes);
1718
1719 } catch (...) {
1720 // We don't want to fail if the scopes are missing because
1721 // this would be incompatible with old HA hook library
1722 // versions. We may make it mandatory one day, but during
1723 // upgrades of existing HA setup it would be a real issue
1724 // if we failed here.
1725 }
1726
1727 // unsent-update-count was not present in earlier HA versions.
1728 // Let's check if the partner has sent the parameter. We initialized
1729 // the counter to 0, and it remains 0 if the partner doesn't send it.
1730 // It effectively means that we don't track partner's unsent updates
1731 // as in the earlier HA versions.
1732 auto unsent_update_count = args->get("unsent-update-count");
1733 if (unsent_update_count) {
1734 if (unsent_update_count->getType() != Element::integer) {
1735 isc_throw(CtrlChannelError, "unsent-update-count returned in"
1736 " the ha-heartbeat response is not an integer");
1737 }
1738 communication_state_->setPartnerUnsentUpdateCount(static_cast<uint64_t>
1739 (unsent_update_count->intValue()));
1740 }
1741
1742 } catch (const std::exception& ex) {
1744 .arg(partner_config->getLogLabel())
1745 .arg(ex.what());
1746 heartbeat_success = false;
1747 }
1748 }
1749
1750 // If heartbeat was successful, let's mark the connection with the
1751 // peer as healthy.
1752 if (heartbeat_success) {
1753 communication_state_->poke();
1754
1755 } else {
1756 // We were unable to retrieve partner's state, so let's mark it
1757 // as unavailable.
1758 communication_state_->setPartnerState("unavailable");
1759 // Log if the communication is interrupted.
1760 if (communication_state_->isCommunicationInterrupted()) {
1762 .arg(partner_config->getName());
1763 }
1764 }
1765
1766 startHeartbeat();
1767 // Even though the partner notified us about the synchronization completion,
1768 // we still can't communicate with the partner. Let's continue serving
1769 // the clients until the link is fixed.
1770 if (sync_complete_notified && !heartbeat_success) {
1771 postNextEvent(HA_SYNCED_PARTNER_UNAVAILABLE_EVT);
1772 }
1773 // Whatever the result of the heartbeat was, the state machine needs
1774 // to react to this. Let's run the state machine until the state machine
1775 // finds that some new events are required, i.e. next heartbeat or
1776 // lease update. The runModel() may transition to another state, schedule
1777 // asynchronous tasks etc. Then it returns control to the DHCP server.
1778 runModel(HA_HEARTBEAT_COMPLETE_EVT);
1779 },
1781 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
1782 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
1783 std::bind(&HAService::clientCloseHandler, this, ph::_1)
1784 );
1785}
1786
1787void
1788HAService::scheduleHeartbeat() {
1789 if (!communication_state_->isHeartbeatRunning()) {
1790 startHeartbeat();
1791 }
1792}
1793
1794void
1795HAService::startHeartbeat() {
1796 if (config_->getHeartbeatDelay() > 0) {
1797 communication_state_->startHeartbeat(config_->getHeartbeatDelay(),
1798 std::bind(&HAService::asyncSendHeartbeat,
1799 this));
1800 }
1801}
1802
1803void
1804HAService::asyncDisableDHCPService(HttpClient& http_client,
1805 const std::string& server_name,
1806 const unsigned int max_period,
1807 PostRequestCallback post_request_action) {
1808 HAConfig::PeerConfigPtr remote_config = config_->getPeerConfig(server_name);
1809
1810 // Create HTTP/1.1 request including our command.
1811 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
1812 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
1813 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
1814
1815 remote_config->addBasicAuthHttpHeader(request);
1816 request->setBodyAsJson(CommandCreator::createDHCPDisable(max_period,
1817 server_type_));
1818 request->finalize();
1819
1820 // Response object should also be created because the HTTP client needs
1821 // to know the type of the expected response.
1822 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
1823
1824 // Schedule asynchronous HTTP request.
1825 http_client.asyncSendRequest(remote_config->getUrl(),
1826 remote_config->getTlsContext(),
1827 request, response,
1828 [this, remote_config, post_request_action]
1829 (const boost::system::error_code& ec,
1830 const HttpResponsePtr& response,
1831 const std::string& error_str) {
1832
1833 // There are three possible groups of errors during the heartbeat.
1834 // One is the IO error causing issues in communication with the peer.
1835 // Another one is an HTTP parsing error. The last type of error is
1836 // when non-success error code is returned in the response carried
1837 // in the HTTP message or if the JSON response is otherwise broken.
1838
1839 int rcode = 0;
1840 std::string error_message;
1841
1842 // Handle first two groups of errors.
1843 if (ec || !error_str.empty()) {
1844 error_message = (ec ? ec.message() : error_str);
1845 LOG_ERROR(ha_logger, HA_DHCP_DISABLE_COMMUNICATIONS_FAILED)
1846 .arg(remote_config->getLogLabel())
1847 .arg(error_message);
1848
1849 } else {
1850
1851 // Handle third group of errors.
1852 try {
1853 static_cast<void>(verifyAsyncResponse(response, rcode));
1854
1855 } catch (const std::exception& ex) {
1856 error_message = ex.what();
1858 .arg(remote_config->getLogLabel())
1859 .arg(error_message);
1860 }
1861 }
1862
1863 // If there was an error communicating with the partner, mark the
1864 // partner as unavailable.
1865 if (!error_message.empty()) {
1866 communication_state_->setPartnerState("unavailable");
1867 }
1868
1869 // Invoke post request action if it was specified.
1870 if (post_request_action) {
1871 post_request_action(error_message.empty(),
1872 error_message,
1873 rcode);
1874 }
1875 },
1877 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
1878 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
1879 std::bind(&HAService::clientCloseHandler, this, ph::_1)
1880 );
1881}
1882
1883void
1884HAService::asyncEnableDHCPService(HttpClient& http_client,
1885 const std::string& server_name,
1886 PostRequestCallback post_request_action) {
1887 HAConfig::PeerConfigPtr remote_config = config_->getPeerConfig(server_name);
1888
1889 // Create HTTP/1.1 request including our command.
1890 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
1891 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
1892 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
1893 remote_config->addBasicAuthHttpHeader(request);
1894 request->setBodyAsJson(CommandCreator::createDHCPEnable(server_type_));
1895 request->finalize();
1896
1897 // Response object should also be created because the HTTP client needs
1898 // to know the type of the expected response.
1899 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
1900
1901 // Schedule asynchronous HTTP request.
1902 http_client.asyncSendRequest(remote_config->getUrl(),
1903 remote_config->getTlsContext(),
1904 request, response,
1905 [this, remote_config, post_request_action]
1906 (const boost::system::error_code& ec,
1907 const HttpResponsePtr& response,
1908 const std::string& error_str) {
1909
1910 // There are three possible groups of errors during the heartbeat.
1911 // One is the IO error causing issues in communication with the peer.
1912 // Another one is an HTTP parsing error. The last type of error is
1913 // when non-success error code is returned in the response carried
1914 // in the HTTP message or if the JSON response is otherwise broken.
1915
1916 int rcode = 0;
1917 std::string error_message;
1918
1919 // Handle first two groups of errors.
1920 if (ec || !error_str.empty()) {
1921 error_message = (ec ? ec.message() : error_str);
1922 LOG_ERROR(ha_logger, HA_DHCP_ENABLE_COMMUNICATIONS_FAILED)
1923 .arg(remote_config->getLogLabel())
1924 .arg(error_message);
1925
1926 } else {
1927
1928 // Handle third group of errors.
1929 try {
1930 static_cast<void>(verifyAsyncResponse(response, rcode));
1931
1932 } catch (const std::exception& ex) {
1933 error_message = ex.what();
1935 .arg(remote_config->getLogLabel())
1936 .arg(error_message);
1937 }
1938 }
1939
1940 // If there was an error communicating with the partner, mark the
1941 // partner as unavailable.
1942 if (!error_message.empty()) {
1943 communication_state_->setPartnerState("unavailable");
1944 }
1945
1946 // Invoke post request action if it was specified.
1947 if (post_request_action) {
1948 post_request_action(error_message.empty(),
1949 error_message,
1950 rcode);
1951 }
1952 },
1954 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
1955 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
1956 std::bind(&HAService::clientCloseHandler, this, ph::_1)
1957 );
1958}
1959
1960void
1961HAService::localDisableDHCPService() {
1962 network_state_->disableService(NetworkState::Origin::HA_COMMAND);
1963}
1964
1965void
1966HAService::localEnableDHCPService() {
1967 network_state_->enableService(NetworkState::Origin::HA_COMMAND);
1968}
1969
1970void
1971HAService::asyncSyncLeases() {
1972 PostSyncCallback null_action;
1973
1974 // Timeout is configured in milliseconds. Need to convert to seconds.
1975 unsigned int dhcp_disable_timeout =
1976 static_cast<unsigned int>(config_->getSyncTimeout() / 1000);
1977 if (dhcp_disable_timeout == 0) {
1978 // Ensure that we always use at least 1 second timeout.
1979 dhcp_disable_timeout = 1;
1980 }
1981
1982 asyncSyncLeases(*client_, config_->getFailoverPeerConfig()->getName(),
1983 dhcp_disable_timeout, LeasePtr(), null_action);
1984}
1985
1986void
1987HAService::asyncSyncLeases(http::HttpClient& http_client,
1988 const std::string& server_name,
1989 const unsigned int max_period,
1990 const dhcp::LeasePtr& last_lease,
1991 PostSyncCallback post_sync_action,
1992 const bool dhcp_disabled) {
1993 // Synchronization starts with a command to disable DHCP service of the
1994 // peer from which we're fetching leases. We don't want the other server
1995 // to allocate new leases while we fetch from it. The DHCP service will
1996 // be disabled for a certain amount of time and will be automatically
1997 // re-enabled if we die during the synchronization.
1998 asyncDisableDHCPService(http_client, server_name, max_period,
1999 [this, &http_client, server_name, max_period, last_lease,
2000 post_sync_action, dhcp_disabled]
2001 (const bool success, const std::string& error_message, const int) {
2002
2003 // If we have successfully disabled the DHCP service on the peer,
2004 // we can start fetching the leases.
2005 if (success) {
2006 // The last argument indicates that disabling the DHCP
2007 // service on the partner server was successful.
2008 asyncSyncLeasesInternal(http_client, server_name, max_period,
2009 last_lease, post_sync_action, true);
2010
2011 } else {
2012 post_sync_action(success, error_message, dhcp_disabled);
2013 }
2014 });
2015}
2016
2017void
2018HAService::asyncSyncLeasesInternal(http::HttpClient& http_client,
2019 const std::string& server_name,
2020 const unsigned int max_period,
2021 const dhcp::LeasePtr& last_lease,
2022 PostSyncCallback post_sync_action,
2023 const bool dhcp_disabled) {
2024
2025 HAConfig::PeerConfigPtr partner_config = config_->getFailoverPeerConfig();
2026
2027 // Create HTTP/1.1 request including our command.
2028 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2029 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2030 HostHttpHeader(partner_config->getUrl().getStrippedHostname()));
2031 partner_config->addBasicAuthHttpHeader(request);
2032 if (server_type_ == HAServerType::DHCPv4) {
2033 request->setBodyAsJson(CommandCreator::createLease4GetPage(
2034 boost::dynamic_pointer_cast<Lease4>(last_lease), config_->getSyncPageLimit()));
2035
2036 } else {
2037 request->setBodyAsJson(CommandCreator::createLease6GetPage(
2038 boost::dynamic_pointer_cast<Lease6>(last_lease), config_->getSyncPageLimit()));
2039 }
2040 request->finalize();
2041
2042 // Response object should also be created because the HTTP client needs
2043 // to know the type of the expected response.
2044 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2045
2046 // Schedule asynchronous HTTP request.
2047 http_client.asyncSendRequest(partner_config->getUrl(),
2048 partner_config->getTlsContext(),
2049 request, response,
2050 [this, partner_config, post_sync_action, &http_client, server_name,
2051 max_period, dhcp_disabled]
2052 (const boost::system::error_code& ec,
2053 const HttpResponsePtr& response,
2054 const std::string& error_str) {
2055
2056 // Holds last lease received on the page of leases. If the last
2057 // page was hit, this value remains null.
2058 LeasePtr last_lease;
2059
2060 // There are three possible groups of errors during the heartbeat.
2061 // One is the IO error causing issues in communication with the peer.
2062 // Another one is an HTTP parsing error. The last type of error is
2063 // when non-success error code is returned in the response carried
2064 // in the HTTP message or if the JSON response is otherwise broken.
2065
2066 std::string error_message;
2067
2068 // Handle first two groups of errors.
2069 if (ec || !error_str.empty()) {
2070 error_message = (ec ? ec.message() : error_str);
2071 LOG_ERROR(ha_logger, HA_LEASES_SYNC_COMMUNICATIONS_FAILED)
2072 .arg(partner_config->getLogLabel())
2073 .arg(error_message);
2074
2075 } else {
2076 // Handle third group of errors.
2077 try {
2078 int rcode = 0;
2079 ConstElementPtr args = verifyAsyncResponse(response, rcode);
2080
2081 // Arguments must be a map.
2082 if (args && (args->getType() != Element::map)) {
2083 isc_throw(CtrlChannelError,
2084 "arguments in the received response must be a map");
2085 }
2086
2087 ConstElementPtr leases = args->get("leases");
2088 if (!leases || (leases->getType() != Element::list)) {
2089 isc_throw(CtrlChannelError,
2090 "server response does not contain leases argument or this"
2091 " argument is not a list");
2092 }
2093
2094 // Iterate over the leases and update the database as appropriate.
2095 const auto& leases_element = leases->listValue();
2096
2097 LOG_INFO(ha_logger, HA_LEASES_SYNC_LEASE_PAGE_RECEIVED)
2098 .arg(leases_element.size())
2099 .arg(server_name);
2100
2101 for (auto l = leases_element.begin(); l != leases_element.end(); ++l) {
2102 try {
2103
2104 if (server_type_ == HAServerType::DHCPv4) {
2105 Lease4Ptr lease = Lease4::fromElement(*l);
2106
2107 // Check if there is such lease in the database already.
2108 Lease4Ptr existing_lease = LeaseMgrFactory::instance().getLease4(lease->addr_);
2109 if (!existing_lease) {
2110 // There is no such lease, so let's add it.
2111 LeaseMgrFactory::instance().addLease(lease);
2112
2113 } else if (existing_lease->cltt_ < lease->cltt_) {
2114 // If the existing lease is older than the fetched lease, update
2115 // the lease in our local database.
2116 // Update lease current expiration time with value received from the
2117 // database. Some database backends reject operations on the lease if
2118 // the current expiration time value does not match what is stored.
2119 Lease::syncCurrentExpirationTime(*existing_lease, *lease);
2120 LeaseMgrFactory::instance().updateLease4(lease);
2121
2122 } else {
2123 LOG_DEBUG(ha_logger, DBGLVL_TRACE_BASIC, HA_LEASE_SYNC_STALE_LEASE4_SKIP)
2124 .arg(lease->addr_.toText())
2125 .arg(lease->subnet_id_);
2126 }
2127
2128 // If we're not on the last page and we're processing final lease on
2129 // this page, let's record the lease as input to the next
2130 // lease4-get-page command.
2131 if ((leases_element.size() >= config_->getSyncPageLimit()) &&
2132 (l + 1 == leases_element.end())) {
2133 last_lease = boost::dynamic_pointer_cast<Lease>(lease);
2134 }
2135
2136 } else {
2137 Lease6Ptr lease = Lease6::fromElement(*l);
2138
2139 // Check if there is such lease in the database already.
2140 Lease6Ptr existing_lease = LeaseMgrFactory::instance().getLease6(lease->type_,
2141 lease->addr_);
2142 if (!existing_lease) {
2143 // There is no such lease, so let's add it.
2144 LeaseMgrFactory::instance().addLease(lease);
2145
2146 } else if (existing_lease->cltt_ < lease->cltt_) {
2147 // If the existing lease is older than the fetched lease, update
2148 // the lease in our local database.
2149 // Update lease current expiration time with value received from the
2150 // database. Some database backends reject operations on the lease if
2151 // the current expiration time value does not match what is stored.
2152 Lease::syncCurrentExpirationTime(*existing_lease, *lease);
2153 LeaseMgrFactory::instance().updateLease6(lease);
2154
2155 } else {
2156 LOG_DEBUG(ha_logger, DBGLVL_TRACE_BASIC, HA_LEASE_SYNC_STALE_LEASE6_SKIP)
2157 .arg(lease->addr_.toText())
2158 .arg(lease->subnet_id_);
2159 }
2160
2161 // If we're not on the last page and we're processing final lease on
2162 // this page, let's record the lease as input to the next
2163 // lease6-get-page command.
2164 if ((leases_element.size() >= config_->getSyncPageLimit()) &&
2165 (l + 1 == leases_element.end())) {
2166 last_lease = boost::dynamic_pointer_cast<Lease>(lease);
2167 }
2168 }
2169
2170 } catch (const std::exception& ex) {
2171 LOG_WARN(ha_logger, HA_LEASE_SYNC_FAILED)
2172 .arg((*l)->str())
2173 .arg(ex.what());
2174 }
2175 }
2176
2177 } catch (const std::exception& ex) {
2178 error_message = ex.what();
2180 .arg(partner_config->getLogLabel())
2181 .arg(error_message);
2182 }
2183 }
2184
2185 // If there was an error communicating with the partner, mark the
2186 // partner as unavailable.
2187 if (!error_message.empty()) {
2188 communication_state_->setPartnerState("unavailable");
2189
2190 } else if (last_lease) {
2191 // This indicates that there are more leases to be fetched.
2192 // Therefore, we have to send another leaseX-get-page command.
2193 asyncSyncLeases(http_client, server_name, max_period, last_lease,
2194 post_sync_action, dhcp_disabled);
2195 return;
2196 }
2197
2198 // Invoke post synchronization action if it was specified.
2199 if (post_sync_action) {
2200 post_sync_action(error_message.empty(),
2201 error_message,
2202 dhcp_disabled);
2203 }
2204 },
2205 HttpClient::RequestTimeout(config_->getSyncTimeout()),
2206 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
2207 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
2208 std::bind(&HAService::clientCloseHandler, this, ph::_1)
2209 );
2210
2211}
2212
2214HAService::processSynchronize(const std::string& server_name,
2215 const unsigned int max_period) {
2216 std::string answer_message;
2217 int sync_status = synchronize(answer_message, server_name, max_period);
2218 return (createAnswer(sync_status, answer_message));
2219}
2220
2221int
2222HAService::synchronize(std::string& status_message, const std::string& server_name,
2223 const unsigned int max_period) {
2224 IOService io_service;
2225 HttpClient client(io_service);
2226
2227 asyncSyncLeases(client, server_name, max_period, Lease4Ptr(),
2228 [&](const bool success, const std::string& error_message,
2229 const bool dhcp_disabled) {
2230 // If there was a fatal error while fetching the leases, let's
2231 // log an error message so as it can be included in the response
2232 // to the controlling client.
2233 if (!success) {
2234 status_message = error_message;
2235 }
2236
2237 // Whether or not there was an error while fetching the leases,
2238 // we need to re-enable the DHCP service on the peer if the
2239 // DHCP service was disabled in the course of synchronization.
2240 if (dhcp_disabled) {
2241 // If the synchronization was completed successfully let's
2242 // try to send the ha-sync-complete-notify command to the
2243 // partner.
2244 if (success) {
2245 asyncSyncCompleteNotify(client, server_name,
2246 [&](const bool success,
2247 const std::string& error_message,
2248 const int rcode) {
2249 // This command may not be supported by the partner when it
2250 // runs an older Kea version. In that case, send the dhcp-enable
2251 // command as in previous Kea version.
2253 asyncEnableDHCPService(client, server_name,
2254 [&](const bool success,
2255 const std::string& error_message,
2256 const int) {
2257 // It is possible that we have already recorded an error
2258 // message while synchronizing the lease database. Don't
2259 // override the existing error message.
2260 if (!success && status_message.empty()) {
2261 status_message = error_message;
2262 }
2263
2264 // The synchronization process is completed, so let's break
2265 // the IO service so as we can return the response to the
2266 // controlling client.
2267 io_service.stop();
2268 });
2269
2270 } else {
2271 // ha-sync-complete-notify command was delivered to the partner.
2272 // The synchronization process ends here.
2273 if (!success && status_message.empty()) {
2274 status_message = error_message;
2275 }
2276
2277 io_service.stop();
2278 }
2279 });
2280
2281 } else {
2282 // Synchronization was unsuccessful. Send the dhcp-enable command to
2283 // re-enable the DHCP service. Note, that we don't send the
2284 // ha-sync-complete-notify command in this case. It is only sent in
2285 // the case when synchronization ends successfully.
2286 asyncEnableDHCPService(client, server_name,
2287 [&](const bool success,
2288 const std::string& error_message,
2289 const int) {
2290 if (!success && status_message.empty()) {
2291 status_message = error_message;
2292 }
2293
2294 // The synchronization process is completed, so let's break
2295 // the IO service so as we can return the response to the
2296 // controlling client.
2297 io_service.stop();
2298
2299 });
2300 }
2301
2302 } else {
2303 // Also stop IO service if there is no need to enable DHCP
2304 // service.
2305 io_service.stop();
2306 }
2307 });
2308
2309 LOG_INFO(ha_logger, HA_SYNC_START).arg(server_name);
2310
2311 // Measure duration of the synchronization.
2312 Stopwatch stopwatch;
2313
2314 // Run the IO service until it is stopped by any of the callbacks. This
2315 // makes it synchronous.
2316 io_service.run();
2317
2318 // End measuring duration.
2319 stopwatch.stop();
2320
2321 // If an error message has been recorded, return an error to the controlling
2322 // client.
2323 if (!status_message.empty()) {
2324 postNextEvent(HA_SYNCING_FAILED_EVT);
2325
2327 .arg(server_name)
2328 .arg(status_message);
2329
2330 return (CONTROL_RESULT_ERROR);
2331
2332 }
2333
2334 // Everything was fine, so let's return a success.
2335 status_message = "Lease database synchronization complete.";
2336 postNextEvent(HA_SYNCING_SUCCEEDED_EVT);
2337
2339 .arg(server_name)
2340 .arg(stopwatch.logFormatLastDuration());
2341
2342 return (CONTROL_RESULT_SUCCESS);
2343}
2344
2345void
2346HAService::asyncSendLeaseUpdatesFromBacklog(HttpClient& http_client,
2347 const HAConfig::PeerConfigPtr& config,
2348 PostRequestCallback post_request_action) {
2349 if (lease_update_backlog_.size() == 0) {
2350 post_request_action(true, "", CONTROL_RESULT_SUCCESS);
2351 return;
2352 }
2353
2354 ConstElementPtr command;
2355 if (server_type_ == HAServerType::DHCPv4) {
2357 Lease4Ptr lease = boost::dynamic_pointer_cast<Lease4>(lease_update_backlog_.pop(op_type));
2358 if (op_type == LeaseUpdateBacklog::ADD) {
2359 command = CommandCreator::createLease4Update(*lease);
2360 } else {
2361 command = CommandCreator::createLease4Delete(*lease);
2362 }
2363
2364 } else {
2365 command = CommandCreator::createLease6BulkApply(lease_update_backlog_);
2366 }
2367
2368 // Create HTTP/1.1 request including our command.
2369 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2370 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2371 HostHttpHeader(config->getUrl().getStrippedHostname()));
2372 config->addBasicAuthHttpHeader(request);
2373 request->setBodyAsJson(command);
2374 request->finalize();
2375
2376 // Response object should also be created because the HTTP client needs
2377 // to know the type of the expected response.
2378 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2379
2380 http_client.asyncSendRequest(config->getUrl(), config->getTlsContext(),
2381 request, response,
2382 [this, &http_client, config, post_request_action]
2383 (const boost::system::error_code& ec,
2384 const HttpResponsePtr& response,
2385 const std::string& error_str) {
2386
2387 int rcode = 0;
2388 std::string error_message;
2389
2390 if (ec || !error_str.empty()) {
2391 error_message = (ec ? ec.message() : error_str);
2392 LOG_WARN(ha_logger, HA_LEASES_BACKLOG_COMMUNICATIONS_FAILED)
2393 .arg(config->getLogLabel())
2394 .arg(ec ? ec.message() : error_str);
2395
2396 } else {
2397 // Handle third group of errors.
2398 try {
2399 auto args = verifyAsyncResponse(response, rcode);
2400 } catch (const std::exception& ex) {
2401 error_message = ex.what();
2403 .arg(config->getLogLabel())
2404 .arg(ex.what());
2405 }
2406 }
2407
2408 // Recursively send all outstanding lease updates or break when an
2409 // error occurs. In DHCPv6, this is a single iteration because we use
2410 // lease6-bulk-apply, which combines many lease updates in a single
2411 // transaction. In the case of DHCPv4, each update is sent in its own
2412 // transaction.
2413 if (error_message.empty()) {
2414 asyncSendLeaseUpdatesFromBacklog(http_client, config, post_request_action);
2415 } else {
2416 post_request_action(error_message.empty(), error_message, rcode);
2417 }
2418 });
2419}
2420
2421bool
2422HAService::sendLeaseUpdatesFromBacklog() {
2423 auto num_updates = lease_update_backlog_.size();
2424 if (num_updates == 0) {
2426 return (true);
2427 }
2428
2429 IOService io_service;
2430 HttpClient client(io_service);
2431 auto remote_config = config_->getFailoverPeerConfig();
2432 bool updates_successful = true;
2433
2435 .arg(num_updates)
2436 .arg(remote_config->getName());
2437
2438 asyncSendLeaseUpdatesFromBacklog(client, remote_config,
2439 [&](const bool success, const std::string&, const int) {
2440 io_service.stop();
2441 updates_successful = success;
2442 });
2443
2444 // Measure duration of the updates.
2445 Stopwatch stopwatch;
2446
2447 // Run the IO service until it is stopped by the callback. This makes it synchronous.
2448 io_service.run();
2449
2450 // End measuring duration.
2451 stopwatch.stop();
2452
2453 if (updates_successful) {
2455 .arg(remote_config->getName())
2456 .arg(stopwatch.logFormatLastDuration());
2457 }
2458
2459 return (updates_successful);
2460}
2461
2462void
2463HAService::asyncSendHAReset(HttpClient& http_client,
2464 const HAConfig::PeerConfigPtr& config,
2465 PostRequestCallback post_request_action) {
2466 ConstElementPtr command = CommandCreator::createHAReset(server_type_);
2467
2468 // Create HTTP/1.1 request including our command.
2469 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2470 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2471 HostHttpHeader(config->getUrl().getStrippedHostname()));
2472 config->addBasicAuthHttpHeader(request);
2473 request->setBodyAsJson(command);
2474 request->finalize();
2475
2476 // Response object should also be created because the HTTP client needs
2477 // to know the type of the expected response.
2478 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2479
2480 http_client.asyncSendRequest(config->getUrl(), config->getTlsContext(),
2481 request, response,
2482 [this, config, post_request_action]
2483 (const boost::system::error_code& ec,
2484 const HttpResponsePtr& response,
2485 const std::string& error_str) {
2486
2487 int rcode = 0;
2488 std::string error_message;
2489
2490 if (ec || !error_str.empty()) {
2491 error_message = (ec ? ec.message() : error_str);
2492 LOG_WARN(ha_logger, HA_RESET_COMMUNICATIONS_FAILED)
2493 .arg(config->getLogLabel())
2494 .arg(ec ? ec.message() : error_str);
2495
2496 } else {
2497 // Handle third group of errors.
2498 try {
2499 auto args = verifyAsyncResponse(response, rcode);
2500 } catch (const std::exception& ex) {
2501 error_message = ex.what();
2503 .arg(config->getLogLabel())
2504 .arg(ex.what());
2505 }
2506 }
2507
2508 post_request_action(error_message.empty(), error_message, rcode);
2509 });
2510}
2511
2512bool
2513HAService::sendHAReset() {
2514 IOService io_service;
2515 HttpClient client(io_service);
2516 auto remote_config = config_->getFailoverPeerConfig();
2517 bool reset_successful = true;
2518
2519 asyncSendHAReset(client, remote_config,
2520 [&](const bool success, const std::string&, const int) {
2521 io_service.stop();
2522 reset_successful = success;
2523 });
2524
2525 // Run the IO service until it is stopped by the callback. This makes it synchronous.
2526 io_service.run();
2527
2528 return (reset_successful);
2529}
2530
2532HAService::processScopes(const std::vector<std::string>& scopes) {
2533 try {
2534 query_filter_.serveScopes(scopes);
2535 adjustNetworkState();
2536
2537 } catch (const std::exception& ex) {
2538 return (createAnswer(CONTROL_RESULT_ERROR, ex.what()));
2539 }
2540
2541 return (createAnswer(CONTROL_RESULT_SUCCESS, "New HA scopes configured."));
2542}
2543
2545HAService::processContinue() {
2546 if (unpause()) {
2547 return (createAnswer(CONTROL_RESULT_SUCCESS, "HA state machine continues."));
2548 }
2549 return (createAnswer(CONTROL_RESULT_SUCCESS, "HA state machine is not paused."));
2550}
2551
2553HAService::processMaintenanceNotify(const bool cancel) {
2554 if (cancel) {
2555 if (getCurrState() != HA_IN_MAINTENANCE_ST) {
2556 return (createAnswer(CONTROL_RESULT_ERROR, "Unable to cancel the"
2557 " maintenance for the server not in the"
2558 " in-maintenance state."));
2559 }
2560
2561 postNextEvent(HA_MAINTENANCE_CANCEL_EVT);
2562 verboseTransition(getPrevState());
2563 runModel(NOP_EVT);
2564 return (createAnswer(CONTROL_RESULT_SUCCESS, "Server maintenance canceled."));
2565 }
2566
2567 switch (getCurrState()) {
2568 case HA_BACKUP_ST:
2570 case HA_TERMINATED_ST:
2571 // The reason why we don't return an error result here is that we have to
2572 // have a way to distinguish between the errors caused by the communication
2573 // issues and the cases when there is no communication error but the server
2574 // is not allowed to enter the in-maintenance state. In the former case, the
2575 // partner would go to partner-down. In the case signaled by the special
2576 // result code entering the maintenance state is not allowed.
2577 return (createAnswer(HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED,
2578 "Unable to transition the server from the "
2579 + stateToString(getCurrState()) + " to"
2580 " in-maintenance state."));
2581 default:
2582 verboseTransition(HA_IN_MAINTENANCE_ST);
2583 runModel(HA_MAINTENANCE_NOTIFY_EVT);
2584 }
2585 return (createAnswer(CONTROL_RESULT_SUCCESS, "Server is in-maintenance state."));
2586}
2587
2589HAService::processMaintenanceStart() {
2590 switch (getCurrState()) {
2591 case HA_BACKUP_ST:
2594 case HA_TERMINATED_ST:
2595 return (createAnswer(CONTROL_RESULT_ERROR, "Unable to transition the server from"
2596 " the " + stateToString(getCurrState()) + " to"
2597 " partner-in-maintenance state."));
2598 default:
2599 ;
2600 }
2601
2602 HAConfig::PeerConfigPtr remote_config = config_->getFailoverPeerConfig();
2603
2604 // Create HTTP/1.1 request including ha-maintenance-notify command
2605 // with the cancel flag set to false.
2606 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2607 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2608 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
2609 remote_config->addBasicAuthHttpHeader(request);
2610 request->setBodyAsJson(CommandCreator::createMaintenanceNotify(false, server_type_));
2611 request->finalize();
2612
2613 // Response object should also be created because the HTTP client needs
2614 // to know the type of the expected response.
2615 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2616
2617 IOService io_service;
2618 HttpClient client(io_service);
2619
2620 boost::system::error_code captured_ec;
2621 std::string captured_error_message;
2622 int captured_rcode = 0;
2623
2624 // Schedule asynchronous HTTP request.
2625 client.asyncSendRequest(remote_config->getUrl(),
2626 remote_config->getTlsContext(),
2627 request, response,
2628 [this, remote_config, &io_service, &captured_ec, &captured_error_message,
2629 &captured_rcode]
2630 (const boost::system::error_code& ec,
2631 const HttpResponsePtr& response,
2632 const std::string& error_str) {
2633
2634 io_service.stop();
2635
2636 // There are three possible groups of errors. One is the IO error
2637 // causing issues in communication with the peer. Another one is
2638 // an HTTP parsing error. The last type of error is when non-success
2639 // error code is returned in the response carried in the HTTP message
2640 // or if the JSON response is otherwise broken.
2641
2642 std::string error_message;
2643
2644 // Handle first two groups of errors.
2645 if (ec || !error_str.empty()) {
2646 error_message = (ec ? ec.message() : error_str);
2647 LOG_ERROR(ha_logger, HA_MAINTENANCE_NOTIFY_COMMUNICATIONS_FAILED)
2648 .arg(remote_config->getLogLabel())
2649 .arg(error_message);
2650
2651 } else {
2652
2653 // Handle third group of errors.
2654 try {
2655 static_cast<void>(verifyAsyncResponse(response, captured_rcode));
2656
2657 } catch (const std::exception& ex) {
2658 error_message = ex.what();
2660 .arg(remote_config->getLogLabel())
2661 .arg(error_message);
2662 }
2663 }
2664
2665 // If there was an error communicating with the partner, mark the
2666 // partner as unavailable.
2667 if (!error_message.empty()) {
2668 communication_state_->setPartnerState("unavailable");
2669 }
2670
2671 captured_ec = ec;
2672 captured_error_message = error_message;
2673 },
2675 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
2676 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
2677 std::bind(&HAService::clientCloseHandler, this, ph::_1)
2678 );
2679
2680 // Run the IO service until it is stopped by any of the callbacks. This
2681 // makes it synchronous.
2682 io_service.run();
2683
2684 // If there was a communication problem with the partner we assume that
2685 // the partner is already down while we receive this command.
2686 if (captured_ec || (captured_rcode == CONTROL_RESULT_ERROR)) {
2687 postNextEvent(HA_MAINTENANCE_START_EVT);
2688 verboseTransition(HA_PARTNER_DOWN_ST);
2689 runModel(NOP_EVT);
2691 "Server is now in the partner-down state as its"
2692 " partner appears to be offline for maintenance."));
2693
2694 } else if (captured_rcode == CONTROL_RESULT_SUCCESS) {
2695 // If the partner responded indicating no error it means that the
2696 // partner has been transitioned to the in-maintenance state. In that
2697 // case we transition to the partner-in-maintenance state.
2698 postNextEvent(HA_MAINTENANCE_START_EVT);
2699 verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST);
2700 runModel(NOP_EVT);
2701
2702 } else {
2703 // Partner server returned a special status code which means that it can't
2704 // transition to the partner-in-maintenance state.
2705 return (createAnswer(CONTROL_RESULT_ERROR, "Unable to transition to the"
2706 " partner-in-maintenance state. The partner server responded"
2707 " with the following message to the ha-maintenance-notify"
2708 " command: " + captured_error_message + "."));
2709
2710 }
2711
2713 "Server is now in the partner-in-maintenance state"
2714 " and its partner is in-maintenance state. The partner"
2715 " can be now safely shut down."));
2716}
2717
2719HAService::processMaintenanceCancel() {
2720 if (getCurrState() != HA_PARTNER_IN_MAINTENANCE_ST) {
2721 return (createAnswer(CONTROL_RESULT_ERROR, "Unable to cancel maintenance"
2722 " request because the server is not in the"
2723 " partner-in-maintenance state."));
2724 }
2725
2726 HAConfig::PeerConfigPtr remote_config = config_->getFailoverPeerConfig();
2727
2728 // Create HTTP/1.1 request including ha-maintenance-notify command
2729 // with the cancel flag set to true.
2730 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2731 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2732 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
2733 remote_config->addBasicAuthHttpHeader(request);
2734 request->setBodyAsJson(CommandCreator::createMaintenanceNotify(true, server_type_));
2735 request->finalize();
2736
2737 // Response object should also be created because the HTTP client needs
2738 // to know the type of the expected response.
2739 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2740
2741 IOService io_service;
2742 HttpClient client(io_service);
2743
2744 std::string error_message;
2745
2746 // Schedule asynchronous HTTP request.
2747 client.asyncSendRequest(remote_config->getUrl(),
2748 remote_config->getTlsContext(),
2749 request, response,
2750 [this, remote_config, &io_service, &error_message]
2751 (const boost::system::error_code& ec,
2752 const HttpResponsePtr& response,
2753 const std::string& error_str) {
2754
2755 io_service.stop();
2756
2757 // Handle first two groups of errors.
2758 if (ec || !error_str.empty()) {
2759 error_message = (ec ? ec.message() : error_str);
2760 LOG_ERROR(ha_logger, HA_MAINTENANCE_NOTIFY_CANCEL_COMMUNICATIONS_FAILED)
2761 .arg(remote_config->getLogLabel())
2762 .arg(error_message);
2763
2764 } else {
2765
2766 // Handle third group of errors.
2767 try {
2768 int rcode = 0;
2769 static_cast<void>(verifyAsyncResponse(response, rcode));
2770
2771 } catch (const std::exception& ex) {
2772 error_message = ex.what();
2774 .arg(remote_config->getLogLabel())
2775 .arg(error_message);
2776 }
2777 }
2778
2779 // If there was an error communicating with the partner, mark the
2780 // partner as unavailable.
2781 if (!error_message.empty()) {
2782 communication_state_->setPartnerState("unavailable");
2783 }
2784 },
2786 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
2787 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
2788 std::bind(&HAService::clientCloseHandler, this, ph::_1)
2789 );
2790
2791 // Run the IO service until it is stopped by any of the callbacks. This
2792 // makes it synchronous.
2793 io_service.run();
2794
2795 // There was an error in communication with the partner or the
2796 // partner was unable to revert its state.
2797 if (!error_message.empty()) {
2799 "Unable to cancel maintenance. The partner server responded"
2800 " with the following message to the ha-maintenance-notify"
2801 " command: " + error_message + "."));
2802 }
2803
2804 // Successfully reverted partner's state. Let's also revert our state to the
2805 // previous one.
2806 postNextEvent(HA_MAINTENANCE_CANCEL_EVT);
2807 verboseTransition(getPrevState());
2808 runModel(NOP_EVT);
2809
2811 "Server maintenance successfully canceled."));
2812}
2813
2814void
2815HAService::asyncSyncCompleteNotify(HttpClient& http_client,
2816 const std::string& server_name,
2817 PostRequestCallback post_request_action) {
2818 HAConfig::PeerConfigPtr remote_config = config_->getPeerConfig(server_name);
2819
2820 // Create HTTP/1.1 request including our command.
2821 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2822 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2823 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
2824
2825 remote_config->addBasicAuthHttpHeader(request);
2826 request->setBodyAsJson(CommandCreator::createSyncCompleteNotify(server_type_));
2827 request->finalize();
2828
2829 // Response object should also be created because the HTTP client needs
2830 // to know the type of the expected response.
2831 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2832
2833 // Schedule asynchronous HTTP request.
2834 http_client.asyncSendRequest(remote_config->getUrl(),
2835 remote_config->getTlsContext(),
2836 request, response,
2837 [this, remote_config, post_request_action]
2838 (const boost::system::error_code& ec,
2839 const HttpResponsePtr& response,
2840 const std::string& error_str) {
2841
2842 // There are three possible groups of errors. One is the IO error
2843 // causing issues in communication with the peer. Another one is an
2844 // HTTP parsing error. The last type of error is when non-success
2845 // error code is returned in the response carried in the HTTP message
2846 // or if the JSON response is otherwise broken.
2847
2848 int rcode = 0;
2849 std::string error_message;
2850
2851 // Handle first two groups of errors.
2852 if (ec || !error_str.empty()) {
2853 error_message = (ec ? ec.message() : error_str);
2854 LOG_ERROR(ha_logger, HA_SYNC_COMPLETE_NOTIFY_COMMUNICATIONS_FAILED)
2855 .arg(remote_config->getLogLabel())
2856 .arg(error_message);
2857
2858 } else {
2859
2860 // Handle third group of errors.
2861 try {
2862 static_cast<void>(verifyAsyncResponse(response, rcode));
2863
2864 } catch (const CommandUnsupportedError& ex) {
2866
2867 } catch (const std::exception& ex) {
2868 error_message = ex.what();
2870 .arg(remote_config->getLogLabel())
2871 .arg(error_message);
2872 }
2873 }
2874
2875 // If there was an error communicating with the partner, mark the
2876 // partner as unavailable.
2877 if (!error_message.empty()) {
2878 communication_state_->setPartnerState("unavailable");
2879 }
2880
2881 // Invoke post request action if it was specified.
2882 if (post_request_action) {
2883 post_request_action(error_message.empty(),
2884 error_message,
2885 rcode);
2886 }
2887 },
2889 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
2890 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
2891 std::bind(&HAService::clientCloseHandler, this, ph::_1)
2892 );
2893}
2894
2896HAService::processSyncCompleteNotify() {
2897 if (getCurrState() == HA_PARTNER_DOWN_ST) {
2898 sync_complete_notified_ = true;
2899 } else {
2900 localEnableDHCPService();
2901 }
2903 "Server successfully notified about the synchronization completion."));
2904}
2905
2907HAService::verifyAsyncResponse(const HttpResponsePtr& response, int& rcode) {
2908 // Set the return code to error in case of early throw.
2909 rcode = CONTROL_RESULT_ERROR;
2910 // The response must cast to JSON type.
2911 HttpResponseJsonPtr json_response =
2912 boost::dynamic_pointer_cast<HttpResponseJson>(response);
2913 if (!json_response) {
2914 isc_throw(CtrlChannelError, "no valid HTTP response found");
2915 }
2916
2917 // Body holds the response to our command.
2918 ConstElementPtr body = json_response->getBodyAsJson();
2919 if (!body) {
2920 isc_throw(CtrlChannelError, "no body found in the response");
2921 }
2922
2923 // Body should contain a list of responses from multiple servers.
2924 if (body->getType() != Element::list) {
2925 // Some control agent errors are returned as a map.
2926 if (body->getType() == Element::map) {
2927 ElementPtr list = Element::createList();
2928 ElementPtr answer = Element::createMap();
2929 answer->set(CONTROL_RESULT, Element::create(rcode));
2930 ConstElementPtr text = body->get(CONTROL_TEXT);
2931 if (text) {
2932 answer->set(CONTROL_TEXT, text);
2933 }
2934 list->add(answer);
2935 body = list;
2936 } else {
2937 isc_throw(CtrlChannelError, "body of the response must be a list");
2938 }
2939 }
2940
2941 // There must be at least one response.
2942 if (body->empty()) {
2943 isc_throw(CtrlChannelError, "list of responses must not be empty");
2944 }
2945
2946 // Check if the status code of the first response. We don't support multiple
2947 // at this time, because we always send a request to a single location.
2948 ConstElementPtr args = parseAnswer(rcode, body->get(0));
2949 if ((rcode != CONTROL_RESULT_SUCCESS) &&
2950 (rcode != CONTROL_RESULT_EMPTY)) {
2951 std::ostringstream s;
2952 // Include an error text if available.
2953 if (args && args->getType() == Element::string) {
2954 s << args->stringValue() << ", ";
2955 }
2956 // Include an error code.
2957 s << "error code " << rcode;
2958
2960 isc_throw(CommandUnsupportedError, s.str());
2961 } else {
2962 isc_throw(CtrlChannelError, s.str());
2963 }
2964 }
2965
2966 return (args);
2967}
2968
2969bool
2970HAService::clientConnectHandler(const boost::system::error_code& ec, int tcp_native_fd) {
2971
2972 // If client is running it's own IOService we do NOT want to
2973 // register the socket with IfaceMgr.
2974 if (client_->getThreadIOService()) {
2975 return (true);
2976 }
2977
2978 // If things look OK register the socket with Interface Manager. Note
2979 // we don't register if the FD is < 0 to avoid an exception throw.
2980 // It is unlikely that this will occur but we want to be liberal
2981 // and avoid issues.
2982 if ((!ec || (ec.value() == boost::asio::error::in_progress))
2983 && (tcp_native_fd >= 0)) {
2984 // External socket callback is a NOP. Ready events handlers are
2985 // run by an explicit call IOService ready in kea-dhcp<n> code.
2986 // We are registering the socket only to interrupt main-thread
2987 // select().
2988 IfaceMgr::instance().addExternalSocket(tcp_native_fd,
2989 std::bind(&HAService::socketReadyHandler, this, ph::_1)
2990 );
2991 }
2992
2993 // If ec.value() == boost::asio::error::already_connected, we should already
2994 // be registered, so nothing to do. If it is any other value, then connect
2995 // failed and Connection logic should handle that, not us, so no matter
2996 // what happens we're returning true.
2997 return (true);
2998}
2999
3000void
3001HAService::socketReadyHandler(int tcp_native_fd) {
3002 // If the socket is ready but does not belong to one of our client's
3003 // ongoing transactions, we close it. This will unregister it from
3004 // IfaceMgr and ensure the client starts over with a fresh connection
3005 // if it needs to do so.
3006 client_->closeIfOutOfBand(tcp_native_fd);
3007}
3008
3009void
3010HAService::clientCloseHandler(int tcp_native_fd) {
3011 if (tcp_native_fd >= 0) {
3012 IfaceMgr::instance().deleteExternalSocket(tcp_native_fd);
3013 }
3014};
3015
3016size_t
3017HAService::pendingRequestSize() {
3018 if (MultiThreadingMgr::instance().getMode()) {
3019 std::lock_guard<std::mutex> lock(mutex_);
3020 return (pending_requests_.size());
3021 } else {
3022 return (pending_requests_.size());
3023 }
3024}
3025
3026template<typename QueryPtrType>
3027int
3028HAService::getPendingRequest(const QueryPtrType& query) {
3029 if (MultiThreadingMgr::instance().getMode()) {
3030 std::lock_guard<std::mutex> lock(mutex_);
3031 return (getPendingRequestInternal(query));
3032 } else {
3033 return (getPendingRequestInternal(query));
3034 }
3035}
3036
3037template<typename QueryPtrType>
3038int
3039HAService::getPendingRequestInternal(const QueryPtrType& query) {
3040 if (pending_requests_.count(query) == 0) {
3041 return (0);
3042 } else {
3043 return (pending_requests_[query]);
3044 }
3045}
3046
3047void
3048HAService::checkPermissionsClientAndListener() {
3049 // Since this function is used as CS callback all exceptions must be
3050 // suppressed (except the @ref MultiThreadingInvalidOperation), unlikely
3051 // though they may be.
3052 // The @ref MultiThreadingInvalidOperation is propagated to the scope of the
3053 // @ref MultiThreadingCriticalSection constructor.
3054 try {
3055 if (client_) {
3056 client_->checkPermissions();
3057 }
3058
3059 if (listener_) {
3060 listener_->checkPermissions();
3061 }
3062 } catch (const isc::MultiThreadingInvalidOperation& ex) {
3064 .arg(ex.what());
3065 // The exception needs to be propagated to the caller of the
3066 // @ref MultiThreadingCriticalSection constructor.
3067 throw;
3068 } catch (const std::exception& ex) {
3070 .arg(ex.what());
3071 }
3072}
3073
3074void
3075HAService::startClientAndListener() {
3076 // Add critical section callbacks.
3077 MultiThreadingMgr::instance().addCriticalSectionCallbacks("HA_MT",
3078 std::bind(&HAService::checkPermissionsClientAndListener, this),
3079 std::bind(&HAService::pauseClientAndListener, this),
3080 std::bind(&HAService::resumeClientAndListener, this));
3081
3082 if (client_) {
3083 client_->start();
3084 }
3085
3086 if (listener_) {
3087 listener_->start();
3088 }
3089}
3090
3091void
3092HAService::pauseClientAndListener() {
3093 // Since this function is used as CS callback all exceptions must be
3094 // suppressed, unlikely though they may be.
3095 try {
3096 if (client_) {
3097 client_->pause();
3098 }
3099
3100 if (listener_) {
3101 listener_->pause();
3102 }
3103 } catch (const std::exception& ex) {
3105 .arg(ex.what());
3106 }
3107}
3108
3109void
3110HAService::resumeClientAndListener() {
3111 // Since this function is used as CS callback all exceptions must be
3112 // suppressed, unlikely though they may be.
3113 try {
3114 if (client_) {
3115 client_->resume();
3116 }
3117
3118 if (listener_) {
3119 listener_->resume();
3120 }
3121 } catch (std::exception& ex) {
3123 .arg(ex.what());
3124 }
3125}
3126
3127void
3128HAService::stopClientAndListener() {
3129 // Remove critical section callbacks.
3130 MultiThreadingMgr::instance().removeCriticalSectionCallbacks("HA_MT");
3131
3132 if (client_) {
3133 client_->stop();
3134 }
3135
3136 if (listener_) {
3137 listener_->stop();
3138 }
3139}
3140
3141// Explicit instantiations.
3142template int HAService::getPendingRequest(const Pkt4Ptr&);
3143template int HAService::getPendingRequest(const Pkt6Ptr&);
3144
3145} // end of namespace isc::ha
3146} // end of namespace isc
if(!(yy_init))
virtual const char * what() const
Returns a C-style character string of the cause of the exception.
Exception thrown when a worker thread is trying to stop or pause the respective thread pool (which wo...
A generic exception that is thrown when an unexpected error condition occurs.
A multi-threaded HTTP listener that can process API commands requests.
A standard control channel exception that is thrown if a function is there is a problem with one of t...
static data::ConstElementPtr createLease4Delete(const dhcp::Lease4 &lease4)
Creates lease4-del command.
static std::unordered_set< std::string > ha_commands4_
List of commands used by the High Availability in v4.
static data::ConstElementPtr createLease4Update(const dhcp::Lease4 &lease4)
Creates lease4-update command.
static data::ConstElementPtr createLease6BulkApply(const dhcp::Lease6CollectionPtr &leases, const dhcp::Lease6CollectionPtr &deleted_leases)
Creates lease6-bulk-apply command.
static std::unordered_set< std::string > ha_commands6_
List of commands used by the High Availability in v6.
Holds communication state between DHCPv4 servers.
Holds communication state between DHCPv6 servers.
Role
Server's role in the High Availability setup.
Definition: ha_config.h:70
static std::string roleToString(const HAConfig::PeerConfig::Role &role)
Returns role name.
Definition: ha_config.cc:79
std::map< std::string, PeerConfigPtr > PeerConfigMap
Map of the servers' configurations.
Definition: ha_config.h:232
static std::string HAModeToString(const HAMode &ha_mode)
Returns HA mode name.
Definition: ha_config.cc:225
boost::shared_ptr< PeerConfig > PeerConfigPtr
Pointer to the server's configuration.
Definition: ha_config.h:229
static const int HA_MAINTENANCE_START_EVT
ha-maintenance-start command received.
Definition: ha_service.h:62
bool inScope(dhcp::Pkt4Ptr &query4)
Checks if the DHCPv4 query should be processed by this server.
Definition: ha_service.cc:1008
void adjustNetworkState()
Enables or disables network state depending on the served scopes.
Definition: ha_service.cc:1038
void stopClientAndListener()
Stop the client and(or) listener instances.
Definition: ha_service.cc:3128
int getNormalState() const
Returns normal operation state for the current configuration.
Definition: ha_service.cc:966
bool shouldQueueLeaseUpdates(const HAConfig::PeerConfigPtr &peer_config) const
Checks if the lease updates should be queued.
Definition: ha_service.cc:1488
static const int HA_HEARTBEAT_COMPLETE_EVT
Finished heartbeat command.
Definition: ha_service.h:47
bool isMaintenanceCanceled() const
Convenience method checking if the current state is a result of canceling the maintenance.
Definition: ha_service.cc:1110
void asyncSendLeaseUpdate(const QueryPtrType &query, const HAConfig::PeerConfigPtr &config, const data::ConstElementPtr &command, const hooks::ParkingLotHandlePtr &parking_lot)
Asynchronously sends lease update to the peer.
Definition: ha_service.cc:1329
void verboseTransition(const unsigned state)
Transitions to a desired state and logs it.
Definition: ha_service.cc:901
bool sendLeaseUpdatesFromBacklog()
Attempts to send all lease updates from the backlog synchronously.
Definition: ha_service.cc:2422
config::CmdHttpListenerPtr listener_
HTTP listener instance used to receive and respond to HA commands and lease updates.
Definition: ha_service.h:1163
bool leaseUpdateComplete(QueryPtrType &query, const hooks::ParkingLotHandlePtr &parking_lot)
Handle last pending request for this query.
Definition: ha_service.cc:1275
HAConfigPtr config_
Pointer to the HA hooks library configuration.
Definition: ha_service.h:1153
bool shouldTerminate() const
Indicates if the server should transition to the terminated state as a result of high clock skew.
Definition: ha_service.cc:1097
void terminatedStateHandler()
Handler for "terminated" state.
Definition: ha_service.cc:772
dhcp::NetworkStatePtr network_state_
Pointer to the state of the DHCP service (enabled/disabled).
Definition: ha_service.h:1150
HAService(const asiolink::IOServicePtr &io_service, const dhcp::NetworkStatePtr &network_state, const HAConfigPtr &config, const HAServerType &server_type=HAServerType::DHCPv4)
Constructor.
Definition: ha_service.cc:67
void scheduleHeartbeat()
Schedules asynchronous heartbeat to a peer if it is not scheduled.
Definition: ha_service.cc:1788
void passiveBackupStateHandler()
Handler for "passive-backup" state.
Definition: ha_service.cc:586
QueryFilter query_filter_
Selects queries to be processed/dropped.
Definition: ha_service.h:1169
static const int HA_MAINTENANCE_NOTIFY_EVT
ha-maintenance-notify command received.
Definition: ha_service.h:59
static const int HA_SYNCED_PARTNER_UNAVAILABLE_EVT
The heartbeat command failed after receiving ha-sync-complete-notify command from the partner.
Definition: ha_service.h:69
void inMaintenanceStateHandler()
Handler for the "in-maintenance" state.
Definition: ha_service.cc:432
virtual void verifyEvents()
Verifies events used by the HA service.
Definition: ha_service.cc:158
void conditionalLogPausedState() const
Logs if the server is paused in the current state.
Definition: ha_service.cc:992
bool unpause()
Unpauses the HA state machine with logging.
Definition: ha_service.cc:982
static const int HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED
Control result returned in response to ha-maintenance-notify.
Definition: ha_service.h:72
void serveDefaultScopes()
Instructs the HA service to serve default scopes.
Definition: ha_service.cc:1003
size_t asyncSendLeaseUpdates(const dhcp::Pkt4Ptr &query, const dhcp::Lease4CollectionPtr &leases, const dhcp::Lease4CollectionPtr &deleted_leases, const hooks::ParkingLotHandlePtr &parking_lot)
Schedules asynchronous IPv4 leases updates.
Definition: ha_service.cc:1145
static const int HA_SYNCING_SUCCEEDED_EVT
Lease database synchronization succeeded.
Definition: ha_service.h:56
bool sendHAReset()
Sends ha-reset command to partner synchronously.
Definition: ha_service.cc:2513
std::function< void(const bool, const std::string &, const int)> PostRequestCallback
Callback invoked when request was sent and a response received or an error occurred.
Definition: ha_service.h:82
virtual void defineEvents()
Defines events used by the HA service.
Definition: ha_service.cc:144
asiolink::IOServicePtr io_service_
Pointer to the IO service object shared between this hooks library and the DHCP server.
Definition: ha_service.h:1147
CommunicationStatePtr communication_state_
Holds communication state with a peer.
Definition: ha_service.h:1166
LeaseUpdateBacklog lease_update_backlog_
Backlog of DHCP lease updates.
Definition: ha_service.h:1283
virtual ~HAService()
Destructor.
Definition: ha_service.cc:136
static const int HA_SYNCING_FAILED_EVT
Lease database synchronization failed.
Definition: ha_service.h:53
static const int HA_MAINTENANCE_CANCEL_EVT
ha-maintenance-cancel command received.
Definition: ha_service.h:65
void readyStateHandler()
Handler for "ready" state.
Definition: ha_service.cc:604
virtual void defineStates()
Defines states of the HA service.
Definition: ha_service.cc:172
void backupStateHandler()
Handler for the "backup" state.
Definition: ha_service.cc:225
void communicationRecoveryHandler()
Handler for the "communication-recovery" state.
Definition: ha_service.cc:240
bool isPartnerStateInvalid() const
Indicates if the partner's state is invalid.
Definition: ha_service.cc:1115
int synchronize(std::string &status_message, const std::string &server_name, const unsigned int max_period)
Synchronizes lease database with a partner.
Definition: ha_service.cc:2222
void normalStateHandler()
Handler for the "hot-standby" and "load-balancing" states.
Definition: ha_service.cc:357
void waitingStateHandler()
Handler for "waiting" state.
Definition: ha_service.cc:793
bool shouldSendLeaseUpdates(const HAConfig::PeerConfigPtr &peer_config) const
Checks if the lease updates should be sent as result of leases allocation or release.
Definition: ha_service.cc:1456
static const int HA_LEASE_UPDATES_COMPLETE_EVT
Finished lease updates commands.
Definition: ha_service.h:50
void partnerDownStateHandler()
Handler for "partner-down" state.
Definition: ha_service.cc:457
http::HttpClientPtr client_
HTTP client instance used to send HA commands and lease updates.
Definition: ha_service.h:1159
void updatePendingRequest(QueryPtrType &query)
Update pending request counter for this query.
Definition: ha_service.cc:1308
bool shouldPartnerDown() const
Indicates if the server should transition to the partner down state.
Definition: ha_service.cc:1070
std::function< void(const bool, const std::string &, const bool)> PostSyncCallback
Callback invoked when lease database synchronization is complete.
Definition: ha_service.h:91
void syncingStateHandler()
Handler for "syncing" state.
Definition: ha_service.cc:683
void partnerInMaintenanceStateHandler()
Handler for "partner-in-maintenance" state.
Definition: ha_service.cc:547
bool push(const OpType op_type, const dhcp::LeasePtr &lease)
Appends lease update to the queue.
OpType
Type of the lease update (operation type).
void clear()
Removes all lease updates from the queue.
bool wasOverflown()
Checks if the queue was overflown.
bool inScope(const dhcp::Pkt4Ptr &query4, std::string &scope_class) const
Checks if this server should process the DHCPv4 query.
void serveFailoverScopes()
Enable scopes required in failover case.
void serveDefaultScopes()
Serve default scopes for the given HA mode.
void serveNoScopes()
Disables all scopes.
Represents HTTP Host header.
Definition: http_header.h:68
HTTP client class.
Definition: client.h:87
void asyncSendRequest(const Url &url, const asiolink::TlsContextPtr &tls_context, const HttpRequestPtr &request, const HttpResponsePtr &response, const RequestHandler &request_callback, const RequestTimeout &request_timeout=RequestTimeout(10000), const ConnectHandler &connect_callback=ConnectHandler(), const HandshakeHandler &handshake_callback=HandshakeHandler(), const CloseHandler &close_callback=CloseHandler())
Queues new asynchronous HTTP request for a given URL.
Definition: client.cc:1967
This class parses and generates time values used in HTTP.
Definition: date_time.h:41
std::string rfc1123Format() const
Returns time value formatted as specified in RFC 1123.
Definition: date_time.cc:30
const EventPtr & getEvent(unsigned int value)
Fetches the event referred to by value.
Definition: state_model.cc:186
std::string getStateLabel(const int state) const
Fetches the label associated with an state value.
Definition: state_model.cc:421
void unpauseModel()
Unpauses state model.
Definition: state_model.cc:276
bool isModelPaused() const
Returns whether or not the model is paused.
Definition: state_model.cc:415
void postNextEvent(unsigned int event)
Sets the next event to the given event value.
Definition: state_model.cc:320
void defineState(unsigned int value, const std::string &label, StateHandler handler, const StatePausing &state_pausing=STATE_PAUSE_NEVER)
Adds an state value and associated label to the set of states.
Definition: state_model.cc:196
bool doOnExit()
Checks if on exit flag is true.
Definition: state_model.cc:347
unsigned int getNextEvent() const
Fetches the model's next event.
Definition: state_model.cc:373
void defineEvent(unsigned int value, const std::string &label)
Adds an event value and associated label to the set of events.
Definition: state_model.cc:170
void transition(unsigned int state, unsigned int event)
Sets up the model to transition into given state with a given event.
Definition: state_model.cc:264
bool doOnEntry()
Checks if on entry flag is true.
Definition: state_model.cc:339
static const int NOP_EVT
Signifies that no event has occurred.
Definition: state_model.h:292
void startModel(const int start_state)
Begins execution of the model.
Definition: state_model.cc:100
unsigned int getLastEvent() const
Fetches the model's last event.
Definition: state_model.cc:367
unsigned int getCurrState() const
Fetches the model's current state.
Definition: state_model.cc:355
Utility class to measure code execution times.
Definition: stopwatch.h:35
void stop()
Stops the stopwatch.
Definition: stopwatch.cc:35
std::string logFormatLastDuration() const
Returns the last measured duration in the format directly usable in log messages.
Definition: stopwatch.cc:75
This file contains several functions and constants that are used for handling commands and responses ...
#define isc_throw(type, stream)
A shortcut macro to insert known values into exception arguments.
An abstract API for lease database.
#define LOG_ERROR(LOGGER, MESSAGE)
Macro to conveniently test error output and log it.
Definition: macros.h:32
#define LOG_INFO(LOGGER, MESSAGE)
Macro to conveniently test info output and log it.
Definition: macros.h:20
#define LOG_WARN(LOGGER, MESSAGE)
Macro to conveniently test warn output and log it.
Definition: macros.h:26
const int CONTROL_RESULT_EMPTY
Status code indicating that the specified command was completed correctly, but failed to produce any ...
const char * CONTROL_TEXT
String used for storing textual description ("text")
constexpr long TIMEOUT_DEFAULT_HTTP_CLIENT_REQUEST
Timeout for the HTTP clients awaiting a response to a request.
Definition: timeouts.h:38
const int CONTROL_RESULT_ERROR
Status code indicating a general failure.
const int CONTROL_RESULT_COMMAND_UNSUPPORTED
Status code indicating that the specified command is not supported.
ConstElementPtr createAnswer(const int status_code, const std::string &text, const ConstElementPtr &arg)
ConstElementPtr parseAnswer(int &rcode, const ConstElementPtr &msg)
const char * CONTROL_RESULT
String used for result, i.e. integer status ("result")
const int CONTROL_RESULT_SUCCESS
Status code indicating a successful operation.
boost::shared_ptr< const Element > ConstElementPtr
Definition: data.h:27
boost::shared_ptr< Element > ElementPtr
Definition: data.h:24
boost::shared_ptr< isc::dhcp::Pkt > PktPtr
A pointer to either Pkt4 or Pkt6 packet.
Definition: pkt.h:797
std::string ClientClass
Defines a single class name.
Definition: classify.h:42
boost::shared_ptr< Lease4Collection > Lease4CollectionPtr
A shared pointer to the collection of IPv4 leases.
Definition: lease.h:501
boost::shared_ptr< Pkt4 > Pkt4Ptr
A pointer to Pkt4 object.
Definition: pkt4.h:544
boost::shared_ptr< Lease > LeasePtr
Pointer to the lease object.
Definition: lease.h:22
boost::shared_ptr< NetworkState > NetworkStatePtr
Pointer to the NetworkState object.
boost::shared_ptr< Lease6Collection > Lease6CollectionPtr
A shared pointer to the collection of IPv6 leases.
Definition: lease.h:665
boost::shared_ptr< Pkt6 > Pkt6Ptr
A pointer to Pkt6 packet.
Definition: pkt6.h:28
boost::shared_ptr< Lease4 > Lease4Ptr
Pointer to a Lease4 structure.
Definition: lease.h:284
const isc::log::MessageID HA_INVALID_PARTNER_STATE_LOAD_BALANCING
Definition: ha_messages.h:51
const isc::log::MessageID HA_RESUME_CLIENT_LISTENER_FAILED
Definition: ha_messages.h:93
const isc::log::MessageID HA_LOCAL_DHCP_ENABLE
Definition: ha_messages.h:76
const isc::log::MessageID HA_LEASES_BACKLOG_NOTHING_TO_SEND
Definition: ha_messages.h:58
const isc::log::MessageID HA_LEASES_BACKLOG_FAILED
Definition: ha_messages.h:57
const isc::log::MessageID HA_SYNC_FAILED
Definition: ha_messages.h:103
const isc::log::MessageID HA_TERMINATED_RESTART_PARTNER
Definition: ha_messages.h:108
const int HA_PASSIVE_BACKUP_ST
In passive-backup state with a single active server and backup servers.
const int HA_HOT_STANDBY_ST
Hot standby state.
const isc::log::MessageID HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY
Definition: ha_messages.h:49
const isc::log::MessageID HA_LEASES_BACKLOG_SUCCESS
Definition: ha_messages.h:60
const int HA_COMMUNICATION_RECOVERY_ST
Communication recovery state.
const isc::log::MessageID HA_STATE_MACHINE_CONTINUED
Definition: ha_messages.h:96
isc::log::Logger ha_logger("ha-hooks")
Definition: ha_log.h:17
const isc::log::MessageID HA_LEASES_SYNC_FAILED
Definition: ha_messages.h:62
const isc::log::MessageID HA_SYNC_SUCCESSFUL
Definition: ha_messages.h:106
const int HA_UNAVAILABLE_ST
Special state indicating that this server is unable to communicate with the partner.
const isc::log::MessageID HA_CONFIG_LEASE_UPDATES_DISABLED_REMINDER
Definition: ha_messages.h:33
const isc::log::MessageID HA_SERVICE_STARTED
Definition: ha_messages.h:95
const int HA_TERMINATED_ST
HA service terminated state.
const int HA_IN_MAINTENANCE_ST
In maintenance state.
const int HA_LOAD_BALANCING_ST
Load balancing state.
const isc::log::MessageID HA_DHCP_ENABLE_FAILED
Definition: ha_messages.h:42
const isc::log::MessageID HA_LEASE_UPDATE_DELETE_FAILED_ON_PEER
Definition: ha_messages.h:71
const isc::log::MessageID HA_LEASES_BACKLOG_START
Definition: ha_messages.h:59
const isc::log::MessageID HA_SYNC_START
Definition: ha_messages.h:105
const isc::log::MessageID HA_HEARTBEAT_FAILED
Definition: ha_messages.h:44
const int HA_PARTNER_DOWN_ST
Partner down state.
const isc::log::MessageID HA_LEASE_UPDATES_ENABLED
Definition: ha_messages.h:68
const isc::log::MessageID HA_INVALID_PARTNER_STATE_HOT_STANDBY
Definition: ha_messages.h:50
const isc::log::MessageID HA_STATE_MACHINE_PAUSED
Definition: ha_messages.h:97
const isc::log::MessageID HA_TERMINATED
Definition: ha_messages.h:107
const isc::log::MessageID HA_DHCP_DISABLE_FAILED
Definition: ha_messages.h:40
boost::shared_ptr< HAConfig > HAConfigPtr
Pointer to the High Availability configuration structure.
Definition: ha_config.h:786
const isc::log::MessageID HA_MAINTENANCE_STARTED_IN_PARTNER_DOWN
Definition: ha_messages.h:85
const int HA_PARTNER_IN_MAINTENANCE_ST
Partner in-maintenance state.
const isc::log::MessageID HA_MAINTENANCE_NOTIFY_FAILED
Definition: ha_messages.h:81
const int HA_WAITING_ST
Server waiting state, i.e. waiting for another server to be ready.
HAServerType
Lists possible server types for which HA service is created.
const int HA_BACKUP_ST
Backup state.
const isc::log::MessageID HA_PAUSE_CLIENT_LISTENER_ILLEGAL
Definition: ha_messages.h:89
const isc::log::MessageID HA_PAUSE_CLIENT_LISTENER_FAILED
Definition: ha_messages.h:88
const isc::log::MessageID HA_MAINTENANCE_SHUTDOWN_SAFE
Definition: ha_messages.h:83
const isc::log::MessageID HA_MAINTENANCE_NOTIFY_CANCEL_FAILED
Definition: ha_messages.h:79
const isc::log::MessageID HA_LEASE_UPDATES_DISABLED
Definition: ha_messages.h:67
const isc::log::MessageID HA_LOCAL_DHCP_DISABLE
Definition: ha_messages.h:75
const int HA_SYNCING_ST
Synchronizing database state.
const isc::log::MessageID HA_RESET_FAILED
Definition: ha_messages.h:91
const isc::log::MessageID HA_STATE_TRANSITION
Definition: ha_messages.h:98
const isc::log::MessageID HA_CONFIG_LEASE_SYNCING_DISABLED_REMINDER
Definition: ha_messages.h:30
std::string stateToString(int state)
Returns state name.
const int HA_READY_ST
Server ready state, i.e. synchronized database, can enable DHCP service.
const isc::log::MessageID HA_SYNC_COMPLETE_NOTIFY_FAILED
Definition: ha_messages.h:101
const isc::log::MessageID HA_COMMUNICATION_INTERRUPTED
Definition: ha_messages.h:20
const isc::log::MessageID HA_MAINTENANCE_STARTED
Definition: ha_messages.h:84
const isc::log::MessageID HA_LEASE_UPDATE_CREATE_UPDATE_FAILED_ON_PEER
Definition: ha_messages.h:70
const isc::log::MessageID HA_LEASE_UPDATE_FAILED
Definition: ha_messages.h:72
const isc::log::MessageID HA_STATE_TRANSITION_PASSIVE_BACKUP
Definition: ha_messages.h:99
boost::shared_ptr< ParkingLotHandle > ParkingLotHandlePtr
Pointer to the parking lot handle.
Definition: parking_lots.h:381
boost::shared_ptr< PostHttpRequestJson > PostHttpRequestJsonPtr
Pointer to PostHttpRequestJson.
boost::shared_ptr< HttpResponseJson > HttpResponseJsonPtr
Pointer to the HttpResponseJson object.
Definition: response_json.h:24
boost::shared_ptr< HttpResponse > HttpResponsePtr
Pointer to the HttpResponse object.
Definition: response.h:78
const char * MessageID
Definition: message_types.h:15
Definition: edns.h:19
Defines the logger used by the top-level component of kea-lfc.
HTTP request/response timeout value.
Definition: client.h:90