From fca73dcbd98f6aa8fefad65e12743eab73336af7 Mon Sep 17 00:00:00 2001 From: karezche <64801825+karenc-bq@users.noreply.github.com> Date: Fri, 10 Jul 2026 07:01:42 -0700 Subject: [PATCH] chore: gdb regions --- ...rora_initial_connection_strategy_plugin.py | 54 ++- .../cluster_topology_monitor.py | 406 ++++++++++++++++- ...concrete_monitoring_connection_handlers.py | 194 ++++++++ .../database_dialect.py | 45 +- .../gdb_failover_plugin.py | 55 ++- .../gdb_read_write_splitting_plugin.py | 54 ++- .../monitoring_connection_handler.py | 336 ++++++++++++++ ...dvanced_python_wrapper_messages.properties | 14 + .../utils/accessible_regions.py | 59 +++ .../gdb_monitoring_connection_priority.py | 204 +++++++++ .../utils/monitoring_connection_priority.py | 73 +++ .../utils/properties.py | 26 ++ .../GlobalDatabases.md | 6 + .../UsingGlobalAuroraAccessibleRegions.md | 52 +++ .../UsingMonitoringConnectionPriority.md | 129 ++++++ tests/unit/test_accessible_regions.py | 161 +++++++ .../test_aio_aurora_initial_connection.py | 1 + ..._connection_strategy_accessible_regions.py | 117 +++++ .../test_cluster_topology_monitor_item8.py | 421 ++++++++++++++++++ tests/unit/test_gdb_failover_plugin.py | 366 +++++++++++++++ ...test_gdb_monitoring_connection_priority.py | 187 ++++++++ .../test_gdb_read_write_splitting_plugin.py | 174 ++++++++ ...ora_topology_monitor_accessible_regions.py | 83 ++++ .../test_monitoring_connection_handler.py | 156 +++++++ .../test_monitoring_connection_priority.py | 75 ++++ 25 files changed, 3405 insertions(+), 43 deletions(-) create mode 100644 aws_advanced_python_wrapper/concrete_monitoring_connection_handlers.py create mode 100644 aws_advanced_python_wrapper/monitoring_connection_handler.py create mode 100644 aws_advanced_python_wrapper/utils/accessible_regions.py create mode 100644 aws_advanced_python_wrapper/utils/gdb_monitoring_connection_priority.py create mode 100644 aws_advanced_python_wrapper/utils/monitoring_connection_priority.py create mode 100644 docs/using-the-python-wrapper/using-plugins/UsingGlobalAuroraAccessibleRegions.md create mode 100644 docs/using-the-python-wrapper/using-plugins/UsingMonitoringConnectionPriority.md create mode 100644 tests/unit/test_accessible_regions.py create mode 100644 tests/unit/test_aurora_initial_connection_strategy_accessible_regions.py create mode 100644 tests/unit/test_cluster_topology_monitor_item8.py create mode 100644 tests/unit/test_gdb_failover_plugin.py create mode 100644 tests/unit/test_gdb_monitoring_connection_priority.py create mode 100644 tests/unit/test_gdb_read_write_splitting_plugin.py create mode 100644 tests/unit/test_global_aurora_topology_monitor_accessible_regions.py create mode 100644 tests/unit/test_monitoring_connection_handler.py create mode 100644 tests/unit/test_monitoring_connection_priority.py diff --git a/aws_advanced_python_wrapper/aurora_initial_connection_strategy_plugin.py b/aws_advanced_python_wrapper/aurora_initial_connection_strategy_plugin.py index 70b7f510e..0811f6d04 100644 --- a/aws_advanced_python_wrapper/aurora_initial_connection_strategy_plugin.py +++ b/aws_advanced_python_wrapper/aurora_initial_connection_strategy_plugin.py @@ -16,7 +16,8 @@ from enum import Enum from time import perf_counter_ns, sleep -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Set, Tuple +from typing import (TYPE_CHECKING, Callable, Dict, FrozenSet, List, Optional, + Sequence, Set, Tuple) if TYPE_CHECKING: from aws_advanced_python_wrapper.driver_dialect import DriverDialect @@ -29,6 +30,8 @@ from aws_advanced_python_wrapper.host_availability import HostAvailability from aws_advanced_python_wrapper.hostinfo import HostInfo, HostRole from aws_advanced_python_wrapper.plugin import Plugin, PluginFactory +from aws_advanced_python_wrapper.utils.accessible_regions import \ + parse as parse_accessible_regions from aws_advanced_python_wrapper.utils.log import Logger from aws_advanced_python_wrapper.utils.messages import Messages from aws_advanced_python_wrapper.utils.properties import (Properties, @@ -106,6 +109,7 @@ def __init__(self, plugin_service: PluginService, props: Properties): self._plugin_service: PluginService = plugin_service self._rds_utils = RdsUtils() self._host_list_provider_service: Optional[HostListProviderService] = None + self._accessible_regions: Optional[FrozenSet[str]] = parse_accessible_regions(props) self._retry_delay_ms: int = WrapperProperties.OPEN_CONNECTION_RETRY_INTERVAL_MS.get_int(props) self._open_connection_retry_timeout_ns: int = \ @@ -277,7 +281,7 @@ def _wait_for_topology_and_connect_to_instance( "AuroraInitialConnectionStrategyPlugin.WaitingForTopology", self._wait_for_initial_topology_ms, original_connect_host.host) - # Deviation from JDBC: force_monitoring_refresh_host_list takes seconds, and host list + # force_monitoring_refresh_host_list takes seconds, and host list # providers without monitor support raise instead of returning their host list. timeout_sec = self._wait_for_initial_topology_ms / 1000 try: @@ -326,7 +330,7 @@ def _get_instance_substitution_strategy( return InstanceSubstitutionStrategy.SUBSTITUTE_WITH_WRITER if url_type == RdsUrlType.RDS_WRITER_CLUSTER: - writer = self._get_writer() + writer = self._find_writer(self._plugin_service.all_hosts) if writer is None or not self._rds_utils.is_rds_instance(writer.host): return InstanceSubstitutionStrategy.DO_NOT_SUBSTITUTE @@ -405,7 +409,7 @@ def _get_role_to_verify( return HostRole.WRITER if url_type == RdsUrlType.RDS_WRITER_CLUSTER: - writer = self._get_writer() + writer = self._find_writer(self._plugin_service.all_hosts) if (writer is not None and self._rds_utils.is_rds_instance(writer.host) and self._rds_utils.is_same_region(writer.host, original_host)): # The cluster writer endpoint belongs to the same region as the current writer; it's active. @@ -446,7 +450,12 @@ def _get_candidate_host( return original_connect_host if substitution_strategy is InstanceSubstitutionStrategy.SUBSTITUTE_WITH_WRITER: - return self._get_writer() + # Filter by accessible regions BEFORE picking the writer so a writer in + # an unreachable region is never selected (no-op unless + # gdb_accessible_regions is set on a Global Aurora dialect); the + # candidate host is chosen from the filtered host list here. + available_hosts = self._filter_by_accessible_regions(self._plugin_service.all_hosts) + return self._find_writer(available_hosts) # SUBSTITUTE_WITH_ANY has no specific target role, so to_target_role() returns None target_role = substitution_strategy.to_target_role() @@ -457,17 +466,23 @@ def _get_candidate_host( "AuroraInitialConnectionStrategyPlugin.UnsupportedStrategy", self._selection_strategy)) try: + # Filter to accessible regions BEFORE any strategy/region selection so + # a candidate is never chosen from an unreachable region (no-op unless + # gdb_accessible_regions is set on a Global Aurora dialect). + available_hosts = self._filter_by_accessible_regions(self._plugin_service.hosts) + aws_region = self._rds_utils.get_rds_region(original_connect_host.host) \ if url_type.has_region else None if aws_region: hosts_in_region: List[HostInfo] = [ - host for host in self._plugin_service.hosts + host for host in available_hosts if (host_region := self._rds_utils.get_rds_region(host.host)) is not None and aws_region.casefold() == host_region.casefold()] return self._plugin_service.get_host_info_by_strategy( target_role, self._selection_strategy, hosts_in_region) - return self._plugin_service.get_host_info_by_strategy(target_role, self._selection_strategy) + return self._plugin_service.get_host_info_by_strategy( + target_role, self._selection_strategy, available_hosts) except Exception: # Unable to find a candidate host. return None @@ -479,8 +494,14 @@ def _set_initial_connection_host_info( and host_info is not None): self._host_list_provider_service.initial_connection_host_info = host_info - def _get_writer(self) -> Optional[HostInfo]: - for host in self._plugin_service.all_hosts: + @staticmethod + def _find_writer(hosts: Sequence[HostInfo]) -> Optional[HostInfo]: + """Return the first WRITER in ``hosts``, or ``None``. + + Does NOT filter by accessible regions — the caller decides whether to + pass an already-filtered list. + """ + for host in hosts: if host.role == HostRole.WRITER: return host return None @@ -502,6 +523,21 @@ def _close_connection(self, connection: Optional[Connection]): def _delay(self, delay_ms: int): sleep(delay_ms / 1000) + def _filter_by_accessible_regions(self, hosts: Sequence[HostInfo]) -> List[HostInfo]: + """Filter hosts down to the configured ``gdb_accessible_regions``. + + Returns the list unchanged when no accessible-regions restriction is + set. Filtering is delegated to the dialect's ``filter_available_hosts`` + (a no-op default; Global Aurora dialects filter by region), so this is a + pass-through for non-Global clusters. + """ + if self._accessible_regions is None: + return list(hosts) + dialect = self._plugin_service.database_dialect + if dialect is None: + return list(hosts) + return dialect.filter_available_hosts(hosts, self._accessible_regions) + class AuroraInitialConnectionStrategyPluginFactory(PluginFactory): @staticmethod diff --git a/aws_advanced_python_wrapper/cluster_topology_monitor.py b/aws_advanced_python_wrapper/cluster_topology_monitor.py index fabda15d9..756ee72e4 100644 --- a/aws_advanced_python_wrapper/cluster_topology_monitor.py +++ b/aws_advanced_python_wrapper/cluster_topology_monitor.py @@ -19,12 +19,18 @@ from abc import ABC, abstractmethod from concurrent.futures import ThreadPoolExecutor from time import perf_counter_ns -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING, Dict, FrozenSet, List, Optional, Tuple +from aws_advanced_python_wrapper.concrete_monitoring_connection_handlers import ( + AuroraMonitoringConnectionHandler, GdbMonitoringConnectionHandler) from aws_advanced_python_wrapper.errors import AwsWrapperError from aws_advanced_python_wrapper.host_availability import HostAvailability from aws_advanced_python_wrapper.hostinfo import HostInfo, Topology from aws_advanced_python_wrapper.utils import services_container +from aws_advanced_python_wrapper.utils.accessible_regions import \ + is_in_accessible_region +from aws_advanced_python_wrapper.utils.accessible_regions import \ + parse as parse_accessible_regions from aws_advanced_python_wrapper.utils.atomic import AtomicReference from aws_advanced_python_wrapper.utils.decorators import \ is_connection_abandoned @@ -37,6 +43,8 @@ from aws_advanced_python_wrapper.utils.utils import LogUtils if TYPE_CHECKING: + from aws_advanced_python_wrapper.monitoring_connection_handler import \ + MonitoringConnectionHandler from aws_advanced_python_wrapper.pep249 import Connection from aws_advanced_python_wrapper.plugin_service import PluginService from aws_advanced_python_wrapper.utils.properties import Properties @@ -120,6 +128,17 @@ def __init__(self, plugin_service: PluginService, topology_utils: TopologyUtils, self._host_threads_latest_topology: AtomicReference[Optional[Topology]] = AtomicReference(None) self._is_verified_writer_connection = False + # Retained even after _writer_host_info is cleared; node threads use it + # as the baseline (last known writer) for writer-change detection. + self._last_known_writer_host_info: AtomicReference[Optional[HostInfo]] = AtomicReference(None) + + self._host_threads_connections: Dict[str, Tuple[HostInfo, ThreadSafeConnectionHolder]] = {} + self._host_threads_map_lock = threading.Lock() + self._reader_topologies_by_id: Dict[str, Topology] = {} + self._completed_one_cycle: Dict[str, bool] = {} + self._stable_topologies_start_nano = 0 + self._reader_observed_writer_host_info: AtomicReference[Optional[HostInfo]] = AtomicReference(None) + self._high_refresh_rate_end_time_nano = 0 self._stop = threading.Event() self._monitor_thread: Optional[threading.Thread] = None @@ -131,8 +150,30 @@ def __init__(self, plugin_service: PluginService, topology_utils: TopologyUtils, if WrapperProperties.CONNECT_TIMEOUT_SEC.get(self._monitoring_properties) is None: WrapperProperties.CONNECT_TIMEOUT_SEC.set(self._monitoring_properties, self.DEFAULT_CONNECT_TIMEOUT_SEC) + # Handler that manages the priority of the background monitoring + # connection and asynchronously upgrades it to a higher-priority node. + self._connection_handler: MonitoringConnectionHandler = self._create_connection_handler() + self._start_monitoring() + STABLE_TOPOLOGY_DURATION_NANO = 15 * 1_000_000_000 # 15 seconds in nanoseconds + + def get_stable_topologies_duration_ns(self) -> int: + return ClusterTopologyMonitorImpl.STABLE_TOPOLOGY_DURATION_NANO + + def _create_connection_handler(self) -> MonitoringConnectionHandler: + return AuroraMonitoringConnectionHandler( + self._monitoring_connection, + self._plugin_service, + self._topology_utils, + self._properties, + self._monitoring_properties, + self.wake_up_monitoring_loop) + + def wake_up_monitoring_loop(self) -> None: + """Notify the main monitoring loop (e.g. when an async upgrade completes).""" + self._request_to_update_topology.set() + def force_refresh(self, should_verify_writer: bool, timeout_sec: float) -> Topology: current_time_nano = time.time_ns() if (self._ignore_new_topology_requests_end_time_nano > 0 and @@ -206,9 +247,11 @@ def process_event(self, event: EventBase) -> None: self._submitted_hosts.clear() self._host_threads_writer_host_info.set(None) self._host_threads_latest_topology.set(None) + self._connection_handler.close() self._monitoring_connection.clear() self._is_verified_writer_connection = False self._writer_host_info.set(None) + self._last_known_writer_host_info.set(None) self._high_refresh_rate_end_time_nano = 0 def close(self) -> None: @@ -220,7 +263,10 @@ def close(self) -> None: if self._monitor_thread and self._monitor_thread.is_alive(): self._monitor_thread.join(self.MONITOR_TERMINATION_TIMEOUT_SEC) - # Step 3: Now safe to close connections - no threads are using them + # Step 3: Now safe to close connections - no threads are using them. + # Close the handler first so any in-flight async upgrade thread is + # cancelled and its connection released before we clear the rest. + self._connection_handler.close() self._monitoring_connection.clear() self._close_connection_from_ref(self._host_threads_writer_connection) self._close_connection_from_ref(self._host_threads_reader_connection) @@ -248,11 +294,21 @@ def _monitor(self) -> None: if hosts and not self._is_verified_writer_connection: logger.debug("ClusterTopologyMonitor.StartingHostMonitoringThreads", self._cluster_id) - writer_host_info = self._writer_host_info.get() - for host_info in hosts: + self._reader_observed_writer_host_info.set(None) + monitored_hosts = self._filter_hosts_for_host_monitoring(hosts) + # some_regions_inaccessible is true when monitoring + # filtering dropped a host, i.e. only for GDB with an + # accessible-regions restriction that actually applied. + some_regions_inaccessible = len(monitored_hosts) < len(hosts) + # Baseline writer for reader-observed writer-change + # detection: the last writer we believed in, retained + # even after _writer_host_info was cleared. + baseline_writer = self._last_known_writer_host_info.get() + for host_info in monitored_hosts: if host_info.host not in self._submitted_hosts: try: - worker = self._get_host_monitor(host_info, writer_host_info) + worker = self._get_host_monitor( + host_info, baseline_writer, some_regions_inaccessible) self._get_host_executor_service().submit(worker) self._submitted_hosts[host_info.host] = True except Exception as e: @@ -265,9 +321,13 @@ def _monitor(self) -> None: writer_connection = self._host_threads_writer_connection.get() if (writer_connection is not None and writer_host_info is not None): logger.debug("ClusterTopologyMonitor.WriterPickedUpFromHostMonitors", self._cluster_id, writer_host_info.host) - # Transfer the writer connection to monitoring connection - self._monitoring_connection.set(writer_connection, close_previous=True) + # Offer the writer connection to the handler, which + # sets it as the monitoring connection and seeds its + # priority index. In panic mode the monitoring + # connection is None, so the handler always accepts. + self._connection_handler.accept_connection(writer_connection, True, writer_host_info) self._writer_host_info.set(writer_host_info) + self._last_known_writer_host_info.set(writer_host_info) self._is_verified_writer_connection = True self._high_refresh_rate_end_time_nano = ( time.time_ns() + self.HIGH_REFRESH_PERIOD_AFTER_PANIC_NANO) @@ -283,13 +343,39 @@ def _monitor(self) -> None: self._submitted_hosts.clear() continue + # Item 8: a reader worker observed a writer change while + # some regions are inaccessible. No node thread may be + # able to reach the new writer to verify it directly, so + # exit panic mode by harvesting the reader connections the + # workers already opened. + reader_observed_writer = self._reader_observed_writer_host_info.get() + if reader_observed_writer is not None: + logger.debug("ClusterTopologyMonitor.WriterChangeObservedByReader", + self._cluster_id, reader_observed_writer.host) + if self._adopt_harvested_monitoring_connection(reader_observed_writer, self._get_stored_hosts()): + self._writer_host_info.set(reader_observed_writer) + self._last_known_writer_host_info.set(reader_observed_writer) + self._is_verified_writer_connection = True + self._high_refresh_rate_end_time_nano = ( + time.time_ns() + self.HIGH_REFRESH_PERIOD_AFTER_PANIC_NANO) + if self._ignore_new_topology_requests_end_time_nano == -1: + self._ignore_new_topology_requests_end_time_nano = 0 + else: + self._ignore_new_topology_requests_end_time_nano = ( + time.time_ns() + self.IGNORE_TOPOLOGY_REQUEST_NANO) + continue + # Update host monitors with new topology host_threads_topology = self._host_threads_latest_topology.get() if host_threads_topology is not None and not self._host_threads_stop.is_set(): - for host_info in host_threads_topology: + monitored_hosts = self._filter_hosts_for_host_monitoring(host_threads_topology) + some_regions_inaccessible = len(monitored_hosts) < len(host_threads_topology) + baseline_writer = self._last_known_writer_host_info.get() + for host_info in monitored_hosts: if host_info.host not in self._submitted_hosts: try: - worker = self._get_host_monitor(host_info, self._writer_host_info.get()) + worker = self._get_host_monitor( + host_info, baseline_writer, some_regions_inaccessible) self._get_host_executor_service().submit(worker) self._submitted_hosts[host_info.host] = True except Exception as e: @@ -297,6 +383,12 @@ def _monitor(self) -> None: "ClusterTopologyMonitor.ExceptionStartingHostMonitor", self._cluster_id, host_info.host, e) + # Item 8: if node threads never verified a writer (e.g. it + # lives in an inaccessible region) but the readers we did + # probe agree on a stable topology, harvest their + # connections to exit panic mode. + self._check_for_stable_reader_topologies() + self._delay(True) else: # Regular mode @@ -311,6 +403,25 @@ def _monitor(self) -> None: self._writer_host_info.set(None) continue + # Refresh the retained writer baseline from the freshly + # fetched topology so that, if the monitoring connection later + # breaks, panic-mode host threads have an accurate baseline for + # writer-change detection. Unlike _writer_host_info, this + # baseline is never cleared on fetch failure. + topology_writer = next( + (h for h in hosts if h.role == HostRole.WRITER), None) + if topology_writer is not None: + self._last_known_writer_host_info.set(topology_writer) + + # Non-blocking: check for a completed async upgrade and, if + # the current connection is not the highest priority, kick off + # a new upgrade attempt for a higher-priority host. Filter the + # candidates so upgrades never target hosts in inaccessible + # regions when gdb_accessible_regions is set (base override is + # a no-op, so this is a pass-through for non-GDB monitors). + self._connection_handler.attempt_connection_upgrade( + self._filter_hosts_for_host_monitoring(hosts)) + current_time_nano = time.time_ns() if (self._high_refresh_rate_end_time_nano > 0 and current_time_nano > self._high_refresh_rate_end_time_nano): @@ -333,8 +444,150 @@ def _monitor(self) -> None: def _is_in_panic_mode(self) -> bool: return self._monitoring_connection.get() is None or not self._is_verified_writer_connection - def _get_host_monitor(self, host_info: HostInfo, writer_host_info: Optional[HostInfo]): - return HostMonitor(self, host_info, writer_host_info) + def _get_host_monitor(self, host_info: HostInfo, writer_host_info: Optional[HostInfo], + some_regions_inaccessible: bool = False): + return HostMonitor(self, host_info, writer_host_info, some_regions_inaccessible) + + def _filter_hosts_for_host_monitoring(self, hosts: Topology) -> Topology: + return hosts + + def _adopt_harvested_monitoring_connection( + self, writer_host_info: Optional[HostInfo], topology: Topology) -> bool: + """Stop host monitors, then offer every connection they harvested into + ``_host_threads_connections`` to the handler, which adopts the best one + as the monitoring connection. + + Ordering is critical for process safety: ``_shutdown_host_executor()`` + joins the executor (``shutdown(wait=True)``) so no worker thread is still + touching a harvested connection before we hand any off — this preserves + the invariant behind the documented psycopg use-after-free fix. We join + WITHOUT ``_close_host_monitors`` because that would empty the harvest map + the workers just populated. Returns ``True`` when the handler adopted a + connection (panic mode can exit). + """ + # Join all workers first; their finally-blocks populate the map. + self._shutdown_host_executor() + self._submitted_hosts.clear() + + with self._host_threads_map_lock: + connections: List[Tuple[HostInfo, ThreadSafeConnectionHolder]] = \ + list(self._host_threads_connections.values()) + + if not connections: + self._clear_host_threads_state() + return False + + selected = self._connection_handler.accept_connections( + connections, writer_host_info, topology) + + # Close every harvested connection the handler did not adopt. + selected_key = self._host_and_port(selected) if selected is not None else None + with self._host_threads_map_lock: + for key, (_, holder) in self._host_threads_connections.items(): + if selected_key is None or key != selected_key: + holder.clear() + self._host_threads_connections.clear() + self._reader_topologies_by_id.clear() + self._completed_one_cycle.clear() + self._stable_topologies_start_nano = 0 + self._reader_observed_writer_host_info.set(None) + return selected is not None + + def _check_for_stable_reader_topologies(self) -> None: + """When host threads never verified a writer (e.g. it lives in an + inaccessible region) but every reader we probed agrees on the same + topology for ``get_stable_topologies_duration_ns()``, harvest the reader + connections and exit panic mode. + """ + latest_hosts = self._get_stored_hosts() + if not latest_hosts: + self._stable_topologies_start_nano = 0 + return + + # Only require completion from hosts we actually monitor; a subclass may + # filter the topology (GDB drops inaccessible regions), and those hosts + # would otherwise appear perpetually incomplete. + monitored_ids = [ + self._host_and_port(h) for h in self._filter_hosts_for_host_monitoring(latest_hosts)] + + with self._host_threads_map_lock: + for host_id in monitored_ids: + if not self._completed_one_cycle.get(host_id, False): + # Not every monitored reader has attempted a cycle yet. + self._stable_topologies_start_nano = 0 + return + + reader_topologies = list(self._reader_topologies_by_id.values()) + if not reader_topologies: + self._stable_topologies_start_nano = 0 + return + + reader_topology = reader_topologies[0] + # Do the reader-observed topologies all agree? Compare on + # (host, port, availability, role) — weight is excluded. + signatures = {self._topology_signature(t) for t in reader_topologies} + if len(signatures) != 1: + self._stable_topologies_start_nano = 0 + return + + if self._stable_topologies_start_nano == 0: + self._stable_topologies_start_nano = time.time_ns() + stable_since = self._stable_topologies_start_nano + + if time.time_ns() <= stable_since + self.get_stable_topologies_duration_ns(): + return + + # Reader topologies have been consistent long enough; treat them as + # accurate and try to adopt one of the reader connections. + with self._host_threads_map_lock: + self._stable_topologies_start_nano = 0 + self._update_topology_cache(reader_topology) + + if self._monitoring_connection.get() is not None: + return + + logger.debug("ClusterTopologyMonitor.StableReaderTopologiesExit", self._cluster_id) + # Adopt with the live writer (typically None here — the writer is in an + # inaccessible region) and the agreed reader topology. + # _adopt_harvested_monitoring_connection joins all workers first (their + # finally-blocks populate the connection map), so we intentionally do NOT + # pre-check the map for emptiness here: while workers are alive the map is + # empty by design, and the join is what fills it. + if self._adopt_harvested_monitoring_connection(self._writer_host_info.get(), reader_topology): + self._is_verified_writer_connection = True + + @staticmethod + def _topology_signature(topology: Topology): + return tuple( + (h.host, h.port, h.availability, h.role) for h in topology) + + def _harvest_connection(self, host_info: HostInfo, connection: Connection) -> None: + """Move ownership of a worker's live connection into the harvest map via + a fresh ThreadSafeConnectionHolder. A pre-existing entry for the same + host is closed to avoid leaks.""" + key = self._host_and_port(host_info) + holder = ThreadSafeConnectionHolder(connection) + with self._host_threads_map_lock: + previous = self._host_threads_connections.get(key) + self._host_threads_connections[key] = (host_info, holder) + if previous is not None: + previous[1].clear() + + def _mark_cycle_completed(self, host_info: HostInfo) -> None: + with self._host_threads_map_lock: + self._completed_one_cycle[self._host_and_port(host_info)] = True + + def _record_reader_topology(self, host_info: HostInfo, topology: Topology) -> None: + with self._host_threads_map_lock: + self._reader_topologies_by_id[self._host_and_port(host_info)] = topology + + def _clear_host_threads_state(self) -> None: + with self._host_threads_map_lock: + self._host_threads_connections.clear() + self._reader_topologies_by_id.clear() + self._completed_one_cycle.clear() + self._stable_topologies_start_nano = 0 + self._reader_observed_writer_host_info.set(None) def _open_any_connection_and_update_topology(self) -> Topology: writer_verified_by_this_thread = False @@ -370,6 +623,12 @@ def _open_any_connection_and_update_topology(self) -> Topology: host_id=writer_id) self._writer_host_info.set(writer_host_info) + self._last_known_writer_host_info.set(writer_host_info) + # Seed the handler's priority index with this writer + # connection so a later async upgrade is evaluated + # against the correct baseline. + self._connection_handler.accept_connection(conn, True, writer_host_info) + logger.debug("ClusterTopologyMonitor.WriterMonitoringConnection", self._cluster_id, writer_host_info.host) except Exception: @@ -408,6 +667,10 @@ def _close_connection_from_ref(self, connection: AtomicReference[Optional[Connec connection_to_close: Optional[Connection] = connection.get_and_set(None) self._close_connection(connection_to_close) + @staticmethod + def _host_and_port(host_info: HostInfo) -> str: + return f"{host_info.host}:{host_info.port}" + def _host_thread_connection_cleanup(self) -> None: writer_connection = self._host_threads_writer_connection.get_and_set(None) if self._monitoring_connection.get() != writer_connection: @@ -417,12 +680,37 @@ def _host_thread_connection_cleanup(self) -> None: if self._monitoring_connection.get() != reader_connection: self._close_connection(reader_connection) - def _close_host_monitors(self) -> None: - self._host_threads_stop.set() + self._clean_up_harvested_connections() + + def _clean_up_harvested_connections(self) -> None: + """Close every item-8 harvested connection except the active monitoring + one, then empty the harvest map.""" + current_monitoring = self._monitoring_connection.get() + with self._host_threads_map_lock: + entries = list(self._host_threads_connections.values()) + self._host_threads_connections.clear() + self._reader_topologies_by_id.clear() + self._completed_one_cycle.clear() + self._stable_topologies_start_nano = 0 + for _, holder in entries: + if current_monitoring is not None and holder.get() is current_monitoring: + # Don't close the active monitoring connection; just detach it. + holder.get_and_set(None, close_previous=False) + else: + holder.clear() + self._reader_observed_writer_host_info.set(None) + def _shutdown_host_executor(self) -> None: + """Stop and join all host-monitoring workers WITHOUT touching the + harvest map, so a subsequent harvest can read the connections the + workers handed off in their finally-blocks.""" + self._host_threads_stop.set() thread_pool_executor = self._thread_pool_executor.get_and_set(None) if thread_pool_executor is not None: thread_pool_executor.shutdown(wait=True, cancel_futures=True) + + def _close_host_monitors(self) -> None: + self._shutdown_host_executor() self._host_thread_connection_cleanup() self._submitted_hosts.clear() @@ -497,10 +785,18 @@ def _update_topology_cache(self, hosts: Topology) -> None: class HostMonitor: def __init__(self, monitor: ClusterTopologyMonitorImpl, host_info: HostInfo, - writer_host_info: Optional[HostInfo]): + writer_host_info: Optional[HostInfo], + some_regions_inaccessible: bool = False): self._monitor: ClusterTopologyMonitorImpl = monitor self._host_info = host_info + # Per-worker baseline writer for reader-observed writer-change + # detection, seeded from the monitor's last known writer. self._writer_host_info = writer_host_info + # Snapshot of whether some regions were inaccessible when this worker was + # created. When True the worker signals a panic-mode exit on an observed + # writer change (item 8) and hands its connection off to the monitor's + # harvest map on shutdown. + self._some_regions_inaccessible = some_regions_inaccessible self._writer_changed = False self._connection_attempts = 0 @@ -508,6 +804,7 @@ def __call__(self) -> None: connection = None update_topology = False start_time = time.time() + handed_off = False try: while not self._monitor._host_threads_stop.is_set(): @@ -577,12 +874,32 @@ def __call__(self) -> None: update_topology = True self._reader_thread_fetch_topology(connection) + # This worker has attempted at least one full cycle. Mark it so + # the main loop's stable-reader-topology check (item 8) does not + # conclude stability before every monitored reader has tried. + if self._some_regions_inaccessible: + self._monitor._mark_cycle_completed(self._host_info) + time.sleep(0.1) except Exception as ex: logger.debug("HostMonitor.Exception", self._host_info.host, ex) finally: - self._monitor._close_connection(connection) + # Item 8: when some regions are inaccessible, hand any live + # connection off to the monitor's harvest map instead of closing it, + # so the main loop can adopt it to exit panic mode. Ownership moves + # to a fresh holder; `handed_off` prevents the trailing close from + # touching it. This runs + # only after the worker has stopped using the connection, and the main + # loop only reads the map after joining all workers, so there is no + # cross-thread use of a connection being closed. + if self._some_regions_inaccessible: + self._monitor._mark_cycle_completed(self._host_info) + if connection is not None and not self._monitor._stop.is_set(): + self._monitor._harvest_connection(self._host_info, connection) + handed_off = True + if not handed_off: + self._monitor._close_connection(connection) elapsed_time = (time.time() - start_time) * 1000 logger.debug("HostMonitor.MonitorCompleted", self._host_info.host, elapsed_time) @@ -598,6 +915,10 @@ def _reader_thread_fetch_topology(self, connection: Connection) -> None: return self._monitor._host_threads_latest_topology.set(hosts) + # Record this reader's observed topology so the main loop's + # stable-reader-topology check (item 8) can compare across readers. + if self._some_regions_inaccessible: + self._monitor._record_reader_topology(self._host_info, hosts) if self._writer_changed: self._monitor._update_topology_cache(hosts) @@ -611,6 +932,20 @@ def _reader_thread_fetch_topology(self, connection: Connection) -> None: logger.debug("HostMonitor.WriterHostChanged", self._writer_host_info.host, latest_writer_host.host) self._monitor._update_topology_cache(hosts) + # Item 8: signal a panic-mode exit only when some regions are + # inaccessible. In that case no node thread may be able to reach the + # new writer to confirm it via get_writer_id_if_connected(), so a + # reader-observed writer change is the fastest way out of panic mode. + # When all regions are accessible we defer to the standard exit path + # (a node thread connecting to the new writer reports it directly), + # which is more reliable since it also verifies a live writer + # connection. CAS-from-None ensures only the first observer wins. + if (self._some_regions_inaccessible + and self._monitor._reader_observed_writer_host_info.compare_and_set( + None, latest_writer_host)): + logger.debug("HostMonitor.WriterChangeExitTriggered", latest_writer_host.host) + self._monitor._host_threads_stop.set() + def _calculate_backoff_with_jitter(self, attempt: int) -> int: backoff = ClusterTopologyMonitorImpl.INITIAL_BACKOFF_MS * (2 ** min(attempt, 6)) backoff = min(backoff, ClusterTopologyMonitorImpl.MAX_BACKOFF_MS) @@ -630,6 +965,10 @@ def __init__( high_refresh_rate_ns: int, instance_templates_by_region: dict[str, HostInfo] ): + self._instance_templates_by_region = instance_templates_by_region + self._global_topology_utils = topology_utils + self._accessible_regions: Optional[FrozenSet[str]] = parse_accessible_regions(props) + super().__init__( plugin_service, topology_utils, @@ -640,8 +979,41 @@ def __init__( refresh_rate_ns, high_refresh_rate_ns ) - self._instance_templates_by_region = instance_templates_by_region - self._global_topology_utils = topology_utils + + # Global Databases need a longer stable-topology window than standard Aurora + # (30s vs 15s) because cross-region topology changes take longer to settle. + GDB_STABLE_TOPOLOGY_DURATION_NANO = 30 * 1_000_000_000 # 30 seconds in nanoseconds + + def get_stable_topologies_duration_ns(self) -> int: + return GlobalAuroraTopologyMonitor.GDB_STABLE_TOPOLOGY_DURATION_NANO + + def _create_connection_handler(self) -> MonitoringConnectionHandler: + return GdbMonitoringConnectionHandler( + self._monitoring_connection, + self._plugin_service, + self._topology_utils, + self._properties, + self._monitoring_properties, + self._writer_host_info, + self.wake_up_monitoring_loop) + + def _filter_hosts_for_host_monitoring(self, hosts: Topology) -> Topology: + if self._accessible_regions is None: + return hosts + return tuple( + host for host in hosts + if is_in_accessible_region(host.host, self._accessible_regions, self._rds_utils) + ) + + def _open_any_connection_and_update_topology(self) -> Topology: + if self._accessible_regions is not None: + region = self._rds_utils.get_rds_region(self._initial_host_info.host) + if region is not None and region.casefold() not in self._accessible_regions: + raise AwsWrapperError( + Messages.get_formatted( + "GlobalAuroraTopologyMonitor.InitialHostNotInAccessibleRegion", + self._initial_host_info.host, region, self._accessible_regions)) + return super()._open_any_connection_and_update_topology() def _get_instance_template(self, instance_id: str, connection: Connection) -> HostInfo: region = self._global_topology_utils.get_region(instance_id, connection) diff --git a/aws_advanced_python_wrapper/concrete_monitoring_connection_handlers.py b/aws_advanced_python_wrapper/concrete_monitoring_connection_handlers.py new file mode 100644 index 000000000..1eb0cfffa --- /dev/null +++ b/aws_advanced_python_wrapper/concrete_monitoring_connection_handlers.py @@ -0,0 +1,194 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import TYPE_CHECKING, Callable, List, Optional, Sequence, Tuple + +from aws_advanced_python_wrapper.hostinfo import HostInfo, HostRole +from aws_advanced_python_wrapper.monitoring_connection_handler import \ + AbstractMonitoringConnectionHandler +from aws_advanced_python_wrapper.utils.gdb_monitoring_connection_priority import \ + GdbMonitoringConnectionPriority +from aws_advanced_python_wrapper.utils.log import Logger +from aws_advanced_python_wrapper.utils.monitoring_connection_priority import \ + MonitoringConnectionPriority +from aws_advanced_python_wrapper.utils.properties import WrapperProperties +from aws_advanced_python_wrapper.utils.rds_utils import RdsUtils + +if TYPE_CHECKING: + from aws_advanced_python_wrapper.host_list_provider import TopologyUtils + from aws_advanced_python_wrapper.hostinfo import Topology + from aws_advanced_python_wrapper.plugin_service import PluginService + from aws_advanced_python_wrapper.utils.atomic import AtomicReference + from aws_advanced_python_wrapper.utils.properties import Properties + from aws_advanced_python_wrapper.utils.thread_safe_connection_holder import \ + ThreadSafeConnectionHolder + +logger = Logger(__name__) + + +class AuroraMonitoringConnectionHandler( + AbstractMonitoringConnectionHandler[MonitoringConnectionPriority]): + """Standard Aurora monitoring connection handler. + + Uses :class:`MonitoringConnectionPriority` (role-only preferences) read from + the ``monitoring_connection_priority`` property. + """ + + def __init__( + self, + monitoring_connection: ThreadSafeConnectionHolder, + plugin_service: PluginService, + topology_utils: TopologyUtils, + props: Properties, + monitoring_properties: Properties, + upgrade_ready_notifier: Optional[Callable[[], None]] = None): + priorities = MonitoringConnectionPriority.parse_list( + WrapperProperties.MONITORING_CONNECTION_PRIORITY.get(props)) + super().__init__( + monitoring_connection, plugin_service, topology_utils, + monitoring_properties, priorities, upgrade_ready_notifier) + + writer_idx = -1 + reader_idx = -1 + for i, priority in enumerate(self._priorities): + if writer_idx < 0 and priority.is_satisfied_by(True): + writer_idx = i + if reader_idx < 0 and priority.is_satisfied_by(False): + reader_idx = i + self._writer_priority_index = writer_idx + self._reader_priority_index = reader_idx + + def _get_priority_index(self, host: HostInfo, is_writer: bool) -> int: + return self._writer_priority_index if is_writer else self._reader_priority_index + + def _find_hosts_for_priority(self, priority_index: int, hosts: Sequence[HostInfo]) -> List[HostInfo]: + priority = self._priorities[priority_index] + if priority is MonitoringConnectionPriority.STRICT_WRITER: + return [h for h in hosts if h.role == HostRole.WRITER] + if priority is MonitoringConnectionPriority.STRICT_READER: + return [h for h in hosts if h.role == HostRole.READER] + if priority is MonitoringConnectionPriority.WRITER_OR_READER: + return list(hosts) + return [] + + def _get_upgrade_thread_name(self) -> str: + return "atmu" + + +class GdbMonitoringConnectionHandler( + AbstractMonitoringConnectionHandler[GdbMonitoringConnectionPriority]): + """Global Aurora Database monitoring connection handler. + + Uses :class:`GdbMonitoringConnectionPriority` with region and + primary/secondary awareness, read from the + ``gdb_monitoring_connection_priority`` property. The cluster's primary region + is derived on demand from the current writer host. + """ + + def __init__( + self, + monitoring_connection: ThreadSafeConnectionHolder, + plugin_service: PluginService, + topology_utils: TopologyUtils, + props: Properties, + monitoring_properties: Properties, + writer_host_info: AtomicReference[Optional[HostInfo]], + upgrade_ready_notifier: Optional[Callable[[], None]] = None): + priorities = GdbMonitoringConnectionPriority.parse_list( + WrapperProperties.GDB_MONITORING_CONNECTION_PRIORITY.get(props)) + super().__init__( + monitoring_connection, plugin_service, topology_utils, + monitoring_properties, priorities, upgrade_ready_notifier) + self._rds_utils = RdsUtils() + self._writer_host_info = writer_host_info + + def accept_connections( + self, + connections: Sequence[Tuple[HostInfo, ThreadSafeConnectionHolder]], + writer_host_info: Optional[HostInfo], + topology: Optional[Topology]) -> Optional[HostInfo]: + with self._lock: + if not connections: + return None + + # The primary region is seeded from the just-detected writer (if any), + # else falls back to the cached writer. + if writer_host_info is not None: + primary_region: Optional[str] = self._rds_utils.get_rds_region(writer_host_info.host) + else: + primary_region = self._get_primary_region() + + best: Optional[Tuple[HostInfo, ThreadSafeConnectionHolder]] = None + best_index = self._effective_index(-1) + for host, holder in connections: + if holder is None or holder.get() is None: + continue + is_writer = (writer_host_info is not None + and self._host_and_port(writer_host_info) == self._host_and_port(host)) + effective_index = self._effective_index( + self._determine_priority_index(host, is_writer, primary_region)) + if best is None or effective_index < best_index: + best_index = effective_index + best = (host, holder) + + if best is None: + return None + + best_host, best_holder = best + best_conn = best_holder.get_and_set(None, close_previous=False) + self._monitoring_connection.set(best_conn, close_previous=True) + self._current_priority_index = best_index + logger.debug("MonitoringConnectionHandler.ConnectionAccepted", + best_host.host, best_host.role, + self._format_priority_index(best_index)) + return best_host + + def _get_priority_index(self, host: HostInfo, is_writer: bool) -> int: + return self._determine_priority_index(host, is_writer, self._get_primary_region()) + + def _find_hosts_for_priority(self, priority_index: int, hosts: Sequence[HostInfo]) -> List[HostInfo]: + priority = self._priorities[priority_index] + return priority.find_matching_hosts(list(hosts), self._get_primary_region(), self._rds_utils) + + def _get_upgrade_thread_name(self) -> str: + return "gatmu" + + def _get_primary_region(self) -> Optional[str]: + writer = self._writer_host_info.get() + if writer is not None: + return self._rds_utils.get_rds_region(writer.host) + return None + + def _determine_priority_index( + self, host: HostInfo, is_writer: bool, primary_region: Optional[str]) -> int: + effective_host = self._with_role(host, HostRole.WRITER if is_writer else HostRole.READER) + for i, priority in enumerate(self._priorities): + if priority.is_satisfied_by(effective_host, primary_region, self._rds_utils): + return i + return -1 + + @staticmethod + def _with_role(host: HostInfo, role: HostRole) -> HostInfo: + if host.role == role: + return host + return HostInfo( + host=host.host, + port=host.port, + role=role, + availability=host.availability, + weight=host.weight, + host_id=host.host_id, + last_update_time=host.last_update_time) diff --git a/aws_advanced_python_wrapper/database_dialect.py b/aws_advanced_python_wrapper/database_dialect.py index 53bab7ee6..17c4ce1f5 100644 --- a/aws_advanced_python_wrapper/database_dialect.py +++ b/aws_advanced_python_wrapper/database_dialect.py @@ -14,8 +14,8 @@ from __future__ import annotations -from typing import (TYPE_CHECKING, Callable, ClassVar, Dict, Optional, - Protocol, Tuple, runtime_checkable) +from typing import (TYPE_CHECKING, Callable, ClassVar, Dict, FrozenSet, List, + Optional, Protocol, Sequence, Tuple, runtime_checkable) from aws_advanced_python_wrapper.driver_info import DriverInfo from aws_advanced_python_wrapper.host_list_provider import ( @@ -40,6 +40,8 @@ UnsupportedOperationError) from aws_advanced_python_wrapper.hostinfo import HostInfo, HostRole from aws_advanced_python_wrapper.utils import services_container +from aws_advanced_python_wrapper.utils.accessible_regions import \ + is_in_accessible_region from aws_advanced_python_wrapper.utils.decorators import \ preserve_transaction_status_with_timeout from aws_advanced_python_wrapper.utils.log import Logger @@ -175,6 +177,19 @@ def get_host_list_provider_supplier(self, plugin_service: PluginService) -> Call def prepare_conn_props(self, props: Properties): ... + def filter_available_hosts( + self, + hosts: Sequence[HostInfo], + accessible_regions: Optional[FrozenSet[str]], + ) -> List[HostInfo]: + """Filter hosts by accessible regions. + + Non-multi-region dialects return the input unchanged. Global Aurora + dialects override this to exclude hosts whose region is not in the + accessible set. + """ + return list(hosts) + class DatabaseDialectProvider(Protocol): def get_dialect(self, driver_dialect: str, props: Properties) -> Optional[DatabaseDialect]: @@ -626,6 +641,19 @@ def get_host_list_provider_supplier(self, plugin_service: PluginService) -> Call props, GlobalAuroraTopologyUtils(self, props)) + def filter_available_hosts( + self, + hosts: Sequence[HostInfo], + accessible_regions: Optional[FrozenSet[str]], + ) -> List[HostInfo]: + if not accessible_regions: + return list(hosts) + rds_utils = RdsUtils() + return [ + host for host in hosts + if is_in_accessible_region(host.host, accessible_regions, rds_utils) + ] + class GlobalAuroraPgDialect(AuroraPgDialect, GlobalAuroraTopologyDialect): _GLOBAL_STATUS_TABLE_EXISTS_QUERY = "SELECT 'pg_catalog.aurora_global_db_status'::pg_catalog.regproc" @@ -683,6 +711,19 @@ def get_host_list_provider_supplier(self, plugin_service: PluginService) -> Call props, GlobalAuroraTopologyUtils(self, props)) + def filter_available_hosts( + self, + hosts: Sequence[HostInfo], + accessible_regions: Optional[FrozenSet[str]], + ) -> List[HostInfo]: + if not accessible_regions: + return list(hosts) + rds_utils = RdsUtils() + return [ + host for host in hosts + if is_in_accessible_region(host.host, accessible_regions, rds_utils) + ] + class MultiAzClusterMysqlDialect(MysqlDatabaseDialect, TopologyAwareDatabaseDialect): _TOPOLOGY_QUERY = "SELECT id, endpoint, port FROM mysql.rds_topology" diff --git a/aws_advanced_python_wrapper/gdb_failover_plugin.py b/aws_advanced_python_wrapper/gdb_failover_plugin.py index adf5cf0be..e1bee885b 100644 --- a/aws_advanced_python_wrapper/gdb_failover_plugin.py +++ b/aws_advanced_python_wrapper/gdb_failover_plugin.py @@ -15,7 +15,7 @@ from __future__ import annotations import time -from typing import TYPE_CHECKING, Callable, List, Optional +from typing import TYPE_CHECKING, Callable, FrozenSet, List, Optional if TYPE_CHECKING: from aws_advanced_python_wrapper.plugin_service import PluginService @@ -26,6 +26,10 @@ from aws_advanced_python_wrapper.failover_v2_plugin import FailoverV2Plugin from aws_advanced_python_wrapper.hostinfo import HostInfo, HostRole from aws_advanced_python_wrapper.plugin import Plugin, PluginFactory +from aws_advanced_python_wrapper.utils.accessible_regions import \ + is_in_accessible_region +from aws_advanced_python_wrapper.utils.accessible_regions import \ + parse as parse_accessible_regions from aws_advanced_python_wrapper.utils.gdb_failover_mode import GdbFailoverMode from aws_advanced_python_wrapper.utils.log import Logger from aws_advanced_python_wrapper.utils.messages import Messages @@ -46,6 +50,7 @@ def __init__(self, plugin_service: PluginService, props: Properties): super().__init__(plugin_service, props) self._home_region: Optional[str] = None + self._accessible_regions: Optional[FrozenSet[str]] = None self._active_home_failover_mode: Optional[GdbFailoverMode] = None self._inactive_home_failover_mode: Optional[GdbFailoverMode] = None self._retry_util = RetryUtil() @@ -86,6 +91,14 @@ def _init_failover_mode(self) -> None: logger.debug("FailoverPlugin.ParameterValue", "failover_home_region", self._home_region) + self._accessible_regions = parse_accessible_regions(self._properties) + if self._accessible_regions is not None: + logger.debug("FailoverPlugin.ParameterValue", "gdb_accessible_regions", self._accessible_regions) + if self._home_region.casefold() not in self._accessible_regions: + raise AwsWrapperError(Messages.get_formatted( + "GdbFailoverPlugin.HomeRegionNotInAccessibleRegions", + self._home_region, self._accessible_regions)) + self._active_home_failover_mode = GdbFailoverMode.from_value( WrapperProperties.ACTIVE_HOME_FAILOVER_MODE.get(self._properties)) self._inactive_home_failover_mode = GdbFailoverMode.from_value( @@ -106,6 +119,12 @@ def _is_home_region(self, region: Optional[str]) -> bool: return self._home_region is not None and region is not None \ and self._home_region.casefold() == region.casefold() + def _is_out_of_home_region(self, region: Optional[str]) -> bool: + return region is not None and not self._is_home_region(region) + + def _is_in_accessible_region(self, host: HostInfo) -> bool: + return is_in_accessible_region(host.host, self._accessible_regions, self._rds_helper) + def _is_strict_writer_failover_mode(self) -> bool: current_region = self._rds_helper.get_rds_region(self._plugin_service.current_host_info.host) if self._is_home_region(current_region): @@ -181,45 +200,59 @@ def _failover_with_mode( failover_end_time: float) -> None: match mode: case GdbFailoverMode.STRICT_WRITER: + if not self._is_in_accessible_region(writer_candidate): + self._inc(self._failover_writer_triggered_counter) + self._inc(self._failover_writer_failed_counter) + writer_region = self._rds_helper.get_rds_region(writer_candidate.host) + raise FailoverFailedError(Messages.get_formatted( + "GdbFailoverPlugin.WriterNotInAccessibleRegion", + writer_region, self._accessible_regions)) self._failover_to_writer(writer_candidate, failover_end_time) case GdbFailoverMode.STRICT_HOME_READER: self._failover_to_allowed_host( lambda: [host for host in self._plugin_service.hosts if host.role == HostRole.READER - and self._is_home_region(self._rds_helper.get_rds_region(host.host))], + and self._is_home_region(self._rds_helper.get_rds_region(host.host)) + and self._is_in_accessible_region(host)], HostRole.READER, failover_end_time) case GdbFailoverMode.STRICT_OUT_OF_HOME_READER: self._failover_to_allowed_host( lambda: [host for host in self._plugin_service.hosts if host.role == HostRole.READER - and not self._is_home_region(self._rds_helper.get_rds_region(host.host))], + and self._is_out_of_home_region(self._rds_helper.get_rds_region(host.host)) + and self._is_in_accessible_region(host)], HostRole.READER, failover_end_time) case GdbFailoverMode.STRICT_ANY_READER: self._failover_to_allowed_host( - lambda: [host for host in self._plugin_service.hosts if host.role == HostRole.READER], + lambda: [host for host in self._plugin_service.hosts + if host.role == HostRole.READER + and self._is_in_accessible_region(host)], HostRole.READER, failover_end_time) case GdbFailoverMode.HOME_READER_OR_WRITER: self._failover_to_allowed_host( lambda: [host for host in self._plugin_service.hosts - if host.role == HostRole.WRITER - or (host.role == HostRole.READER - and self._is_home_region(self._rds_helper.get_rds_region(host.host)))], + if (host.role == HostRole.WRITER + or (host.role == HostRole.READER + and self._is_home_region(self._rds_helper.get_rds_region(host.host)))) + and self._is_in_accessible_region(host)], None, failover_end_time) case GdbFailoverMode.OUT_OF_HOME_READER_OR_WRITER: self._failover_to_allowed_host( lambda: [host for host in self._plugin_service.hosts - if host.role == HostRole.WRITER - or (host.role == HostRole.READER - and not self._is_home_region(self._rds_helper.get_rds_region(host.host)))], + if (host.role == HostRole.WRITER + or (host.role == HostRole.READER + and self._is_out_of_home_region(self._rds_helper.get_rds_region(host.host)))) + and self._is_in_accessible_region(host)], None, failover_end_time) case GdbFailoverMode.ANY_READER_OR_WRITER: self._failover_to_allowed_host( - lambda: list(self._plugin_service.hosts), + lambda: [host for host in self._plugin_service.hosts + if self._is_in_accessible_region(host)], None, failover_end_time) case _: diff --git a/aws_advanced_python_wrapper/gdb_read_write_splitting_plugin.py b/aws_advanced_python_wrapper/gdb_read_write_splitting_plugin.py index 638bd5424..78a49ce8c 100644 --- a/aws_advanced_python_wrapper/gdb_read_write_splitting_plugin.py +++ b/aws_advanced_python_wrapper/gdb_read_write_splitting_plugin.py @@ -14,12 +14,16 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Callable, List, Optional +from typing import TYPE_CHECKING, Callable, FrozenSet, List, Optional from aws_advanced_python_wrapper.errors import ReadWriteSplittingError from aws_advanced_python_wrapper.plugin import Plugin, PluginFactory from aws_advanced_python_wrapper.read_write_splitting_plugin import \ ReadWriteSplittingPlugin +from aws_advanced_python_wrapper.utils.accessible_regions import \ + is_in_accessible_region +from aws_advanced_python_wrapper.utils.accessible_regions import \ + parse as parse_accessible_regions from aws_advanced_python_wrapper.utils.log import Logger from aws_advanced_python_wrapper.utils.messages import Messages from aws_advanced_python_wrapper.utils.properties import (Properties, @@ -60,6 +64,7 @@ def __init__(self, plugin_service: PluginService, props: Properties): WrapperProperties.GDB_ENABLE_GLOBAL_WRITE_FORWARDING.get_bool(props) ) self._home_region: Optional[str] = None + self._accessible_regions: Optional[FrozenSet[str]] = None self._initialized: bool = False def connect( @@ -107,8 +112,32 @@ def _init_settings(self, init_host_info: HostInfo, props: Properties) -> None: self._home_region, ) + self._accessible_regions = parse_accessible_regions(props) + if self._accessible_regions is not None: + logger.debug( + "GdbReadWriteSplittingPlugin.ParameterValue", + WrapperProperties.GDB_ACCESSIBLE_REGIONS.name, + self._accessible_regions, + ) + if home_region.casefold() not in self._accessible_regions: + raise ReadWriteSplittingError( + Messages.get_formatted( + "GdbReadWriteSplittingPlugin.HomeRegionNotInAccessibleRegions", + self._home_region, self._accessible_regions, + ) + ) + def _initialize_writer_connection(self) -> None: writer_host = self._get_writer_host_info() + if writer_host is not None and not self._is_in_accessible_region(writer_host): + writer_region = self._rds_utils.get_rds_region(writer_host.host) + raise ReadWriteSplittingError( + Messages.get_formatted( + "GdbReadWriteSplittingPlugin.WriterNotInAccessibleRegion", + writer_host.host, writer_region, self._accessible_regions, + ) + ) + if writer_host is not None and self._is_writer_outside_home_region(writer_host): if self._enable_global_write_forwarding: logger.debug( @@ -130,6 +159,16 @@ def _initialize_writer_connection(self) -> None: def _set_writer_connection( self, writer_conn: Connection, writer_host_info: HostInfo ) -> None: + if not self._is_in_accessible_region(writer_host_info): + self._close_connection(writer_conn) + writer_region = self._rds_utils.get_rds_region(writer_host_info.host) + raise ReadWriteSplittingError( + Messages.get_formatted( + "GdbReadWriteSplittingPlugin.WriterNotInAccessibleRegion", + writer_host_info.host, writer_region, self._accessible_regions, + ) + ) + if self._is_writer_outside_home_region(writer_host_info): self._close_connection(writer_conn) raise ReadWriteSplittingError( @@ -142,12 +181,16 @@ def _set_writer_connection( super()._set_writer_connection(writer_conn, writer_host_info) def _get_reader_host_candidates(self) -> List[HostInfo]: + candidates = [ + host for host in self._plugin_service.hosts + if self._is_in_accessible_region(host) + ] + if not self._restrict_reader_to_home_region: - return super()._get_reader_host_candidates() + return candidates hosts_in_region = [ - host - for host in self._plugin_service.hosts + host for host in candidates if self._is_in_home_region(host) ] @@ -167,6 +210,9 @@ def _is_writer_outside_home_region(self, host_info: HostInfo) -> bool: and not self._is_in_home_region(host_info) ) + def _is_in_accessible_region(self, host_info: HostInfo) -> bool: + return is_in_accessible_region(host_info.host, self._accessible_regions, self._rds_utils) + def _is_in_home_region(self, host_info: HostInfo) -> bool: if self._home_region is None: return True diff --git a/aws_advanced_python_wrapper/monitoring_connection_handler.py b/aws_advanced_python_wrapper/monitoring_connection_handler.py new file mode 100644 index 000000000..b8f8bc98f --- /dev/null +++ b/aws_advanced_python_wrapper/monitoring_connection_handler.py @@ -0,0 +1,336 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import random +import threading +from abc import ABC, abstractmethod +from typing import (TYPE_CHECKING, Callable, Generic, List, Optional, Sequence, + Tuple, TypeVar) + +from aws_advanced_python_wrapper.utils.log import Logger +from aws_advanced_python_wrapper.utils.thread_safe_connection_holder import \ + ThreadSafeConnectionHolder + +if TYPE_CHECKING: + from aws_advanced_python_wrapper.host_list_provider import TopologyUtils + from aws_advanced_python_wrapper.hostinfo import HostInfo, Topology + from aws_advanced_python_wrapper.pep249 import Connection + from aws_advanced_python_wrapper.plugin_service import PluginService + from aws_advanced_python_wrapper.utils.properties import Properties + +logger = Logger(__name__) + +P = TypeVar("P") + +# Sentinel priority index representing "does not match any configured priority". +# A large value so unmatched hosts sort last for effective ordering. +_NO_PRIORITY_INDEX = 2 ** 63 - 1 + + +class MonitoringConnectionHandler(ABC): + """Handles the monitoring connection lifecycle. + + The topology monitor accepts whatever connection it obtains first, then this + handler asynchronously upgrades to a higher-priority connection. + """ + + @abstractmethod + def accept_connection(self, conn: Connection, is_writer: bool, host_info: Optional[HostInfo]) -> bool: + """Offer a single connection to the handler. + + Returns ``True`` if the connection was accepted as the monitoring + connection, ``False`` if it was rejected (the caller should close it). + """ + ... + + @abstractmethod + def accept_connections( + self, + connections: Sequence[Tuple[HostInfo, ThreadSafeConnectionHolder]], + writer_host_info: Optional[HostInfo], + topology: Optional[Topology]) -> Optional[HostInfo]: + """Offer a batch of harvested connections; adopt the best by priority. + + ``connections`` is a sequence of ``(host, holder)`` pairs. A sequence of + pairs is used rather than a mapping keyed by host because + :class:`HostInfo` is unhashable; the semantics are identical. + + Returns the host of the connection that was selected and set as the + monitoring connection, or ``None`` if none was selected. + """ + ... + + @abstractmethod + def attempt_connection_upgrade(self, current_topology: Optional[Topology]) -> None: + """Non-blocking attempt to upgrade to a higher-priority host.""" + ... + + @abstractmethod + def close(self) -> None: + """Cancel pending upgrade attempts and release held connections.""" + ... + + +class AbstractMonitoringConnectionHandler(MonitoringConnectionHandler, Generic[P]): + """Priority-driven monitoring connection lifecycle shared by concrete handlers. + + A thread-safe connection wrapper is provided by + :class:`ThreadSafeConnectionHolder`, and the async upgrade runs on a single + background :class:`threading.Thread` guarded by a :class:`threading.Event`. + + Lower priority index == higher preference. ``priorities`` is a list of + priority objects (a ``MonitoringConnectionPriority`` or + ``GdbMonitoringConnectionPriority``); this base class is agnostic to their + concrete type and defers role/region reasoning to the subclass hooks + :meth:`_get_priority_index` and :meth:`_find_hosts_for_priority`. + """ + + # Seconds to wait for the async upgrade thread to finish on close(). + UPGRADE_JOIN_TIMEOUT_SEC = 5.0 + + def __init__( + self, + monitoring_connection: ThreadSafeConnectionHolder, + plugin_service: PluginService, + topology_utils: TopologyUtils, + monitoring_properties: Properties, + priorities: Sequence[P], + upgrade_ready_notifier: Optional[Callable[[], None]] = None): + self._monitoring_connection = monitoring_connection + self._upgrade_connection = ThreadSafeConnectionHolder(None) + self._plugin_service = plugin_service + self._topology_utils = topology_utils + self._monitoring_properties = monitoring_properties + self._priorities: List[P] = list(priorities) + self._upgrade_ready_notifier = upgrade_ready_notifier + + self._lock = threading.RLock() + self._current_priority_index = -1 + self._upgrade_thread: Optional[threading.Thread] = None + self._upgrade_done = threading.Event() + self._upgrade_cancelled = threading.Event() + self._upgrade_connected_host: Optional[HostInfo] = None + + # ---- Subclass hooks ------------------------------------------------- + + @abstractmethod + def _get_priority_index(self, host: HostInfo, is_writer: bool) -> int: + """Return the priority index for ``host`` (or -1 when it matches none).""" + ... + + @abstractmethod + def _find_hosts_for_priority(self, priority_index: int, hosts: Sequence[HostInfo]) -> List[HostInfo]: + """Return hosts matching the priority at ``priority_index``.""" + ... + + @abstractmethod + def _get_upgrade_thread_name(self) -> str: + ... + + # ---- Helpers -------------------------------------------------------- + + @staticmethod + def _effective_index(priority_index: int) -> int: + return priority_index if priority_index >= 0 else _NO_PRIORITY_INDEX + + @staticmethod + def _format_priority_index(index: int) -> str: + return "" if index == _NO_PRIORITY_INDEX else str(index) + + def _find_upgrade_candidates(self, hosts: Topology) -> List[List[HostInfo]]: + candidates_by_priority: List[List[HostInfo]] = [] + limit = min(self._current_priority_index, len(self._priorities)) + for i in range(limit): + matching = self._find_hosts_for_priority(i, hosts) + if matching: + candidates_by_priority.append(matching) + return candidates_by_priority + + # ---- MonitoringConnectionHandler ------------------------------------ + + def accept_connection(self, conn: Connection, is_writer: bool, host_info: Optional[HostInfo]) -> bool: + with self._lock: + priority_index = -1 if host_info is None else self._get_priority_index(host_info, is_writer) + effective_index = self._effective_index(priority_index) + host_label = host_info.host if host_info is not None else "unknown" + + if self._monitoring_connection.get() is None or self._current_priority_index < 0: + self._monitoring_connection.set(conn, close_previous=True) + self._current_priority_index = effective_index + logger.debug("MonitoringConnectionHandler.ConnectionAccepted", + host_label, "WRITER" if is_writer else "READER", + self._format_priority_index(effective_index)) + return True + + if effective_index < self._current_priority_index: + self._monitoring_connection.set(conn, close_previous=True) + self._current_priority_index = effective_index + logger.debug("MonitoringConnectionHandler.ConnectionAccepted", + host_label, "WRITER" if is_writer else "READER", + self._format_priority_index(effective_index)) + return True + + logger.debug("MonitoringConnectionHandler.ConnectionRejected", + host_label, is_writer, + self._format_priority_index(self._current_priority_index), + self._format_priority_index(effective_index)) + return False + + def accept_connections( + self, + connections: Sequence[Tuple[HostInfo, ThreadSafeConnectionHolder]], + writer_host_info: Optional[HostInfo], + topology: Optional[Topology]) -> Optional[HostInfo]: + with self._lock: + if not connections: + return None + + best: Optional[Tuple[HostInfo, ThreadSafeConnectionHolder]] = None + best_index = _NO_PRIORITY_INDEX + for host, holder in connections: + if holder is None or holder.get() is None: + continue + is_writer = (writer_host_info is not None + and self._host_and_port(writer_host_info) == self._host_and_port(host)) + effective_index = self._effective_index(self._get_priority_index(host, is_writer)) + if best is None or effective_index < best_index: + best_index = effective_index + best = (host, holder) + + if best is None: + return None + + best_host, best_holder = best + # Detach the connection from the holder without closing it, then + # adopt it as the monitoring connection. + best_conn = best_holder.get_and_set(None, close_previous=False) + self._monitoring_connection.set(best_conn, close_previous=True) + self._current_priority_index = best_index + logger.debug("MonitoringConnectionHandler.ConnectionAccepted", + best_host.host, best_host.role, + self._format_priority_index(best_index)) + return best_host + + def attempt_connection_upgrade(self, current_topology: Optional[Topology]) -> None: + with self._lock: + if self._current_priority_index <= 0: + return + + thread = self._upgrade_thread + if thread is not None: + if not self._upgrade_done.is_set(): + # Upgrade attempt still running. + return + + # Safe to read _upgrade_connected_host / _upgrade_connection + # here: the worker publishes both BEFORE calling + # _upgrade_done.set(), and we only reach this point after + # _upgrade_done.is_set() returned True (see _start_upgrade_thread + # for the memory-ordering contract). + conn = self._upgrade_connection.get() + connected_host = self._upgrade_connected_host + if conn is not None and connected_host is not None: + try: + is_writer = self._topology_utils.get_writer_id_if_connected( + conn, self._plugin_service.driver_dialect) is not None + except Exception: + self._upgrade_connection.set(None, close_previous=True) + self._reset_upgrade_state() + return + + new_index = self._get_priority_index(connected_host, is_writer) + if 0 <= new_index < self._current_priority_index: + # Adopt the upgraded connection without closing it. + self._upgrade_connection.set(None, close_previous=False) + self._monitoring_connection.set(conn, close_previous=True) + self._current_priority_index = new_index + logger.debug("MonitoringConnectionHandler.UpgradedMonitoringConnection", + connected_host.host, str(self._priorities[new_index]), + self._format_priority_index(new_index)) + else: + self._upgrade_connection.set(None, close_previous=True) + elif conn is not None: + self._upgrade_connection.set(None, close_previous=True) + self._reset_upgrade_state() + + if self._upgrade_thread is None and current_topology is not None: + candidates_by_priority = self._find_upgrade_candidates(current_topology) + # Flatten buckets highest-priority-first. Within each priority + # bucket the hosts are equivalent, so shuffle before flattening to + # spread monitoring-connection load across them. + candidates: List[HostInfo] = [] + for bucket in candidates_by_priority: + random.shuffle(bucket) + candidates.extend(bucket) + if not candidates: + return + + self._start_upgrade_thread(candidates) + + def close(self) -> None: + with self._lock: + self._upgrade_cancelled.set() + thread = self._upgrade_thread + if thread is not None and thread.is_alive(): + thread.join(self.UPGRADE_JOIN_TIMEOUT_SEC) + with self._lock: + self._upgrade_connection.clear() + self._reset_upgrade_state() + self._upgrade_cancelled.clear() + self._current_priority_index = -1 + + # ---- Internal ------------------------------------------------------- + + @staticmethod + def _host_and_port(host: HostInfo) -> str: + return f"{host.host}:{host.port}" + + def _reset_upgrade_state(self) -> None: + self._upgrade_thread = None + self._upgrade_connected_host = None + self._upgrade_done.clear() + + def _start_upgrade_thread(self, candidates: List[HostInfo]) -> None: + self._upgrade_done.clear() + self._upgrade_cancelled.clear() + + def _run() -> None: + for candidate in candidates: + if self._upgrade_cancelled.is_set(): + break + try: + conn = self._plugin_service.force_connect(candidate, self._monitoring_properties) + # Memory-ordering contract: publish the host and the + # connection BEFORE _upgrade_done.set() below. The reader in + # attempt_connection_upgrade holds self._lock but reads these + # fields without it, gated only on _upgrade_done.is_set(); + # Event.set()/is_set() provide the happens-before barrier + # (via the Event's internal lock). Do NOT reorder these three + # statements or move _upgrade_done.set() earlier. + self._upgrade_connected_host = candidate + self._upgrade_connection.set(conn, close_previous=True) + if self._upgrade_ready_notifier is not None: + self._upgrade_ready_notifier() + break + except Exception as ex: + logger.debug("MonitoringConnectionHandler.UpgradeAttemptFailed", + candidate.host, ex) + self._upgrade_done.set() + + thread = threading.Thread( + target=_run, name=self._get_upgrade_thread_name(), daemon=True) + self._upgrade_thread = thread + thread.start() diff --git a/aws_advanced_python_wrapper/resources/aws_advanced_python_wrapper_messages.properties b/aws_advanced_python_wrapper/resources/aws_advanced_python_wrapper_messages.properties index b01d488fb..b80a4cb6f 100644 --- a/aws_advanced_python_wrapper/resources/aws_advanced_python_wrapper_messages.properties +++ b/aws_advanced_python_wrapper/resources/aws_advanced_python_wrapper_messages.properties @@ -94,12 +94,20 @@ ClusterTopologyMonitor.TimeoutSetToZero=[ClusterTopologyMonitor, clusterId: '{}' ClusterTopologyMonitor.StartingHostMonitoringThreads=[ClusterTopologyMonitor, clusterId: '{}'] Starting host monitoring threads. ClusterTopologyMonitor.ExceptionStartingHostMonitor=[ClusterTopologyMonitor, clusterId: '{}'] Exception starting monitor for host '{}': '{}'. ClusterTopologyMonitor.WriterPickedUpFromHostMonitors=[ClusterTopologyMonitor, clusterId: '{}'] The writer host detected by the host monitors was picked up by the topology monitor: '{}'. +ClusterTopologyMonitor.WriterChangeObservedByReader=[ClusterTopologyMonitor, clusterId: '{}'] A reader host monitor observed a writer change to '{}' while some regions are inaccessible; exiting panic mode via harvested reader connections. +ClusterTopologyMonitor.StableReaderTopologiesExit=[ClusterTopologyMonitor, clusterId: '{}'] Reader topologies have been stable without a verified writer; harvesting a reader connection to exit panic mode. ClusterTopologyMonitor.ExceptionDuringMonitoringStop=[ClusterTopologyMonitor, clusterId: '{}'] Stopping cluster topology monitoring after unhandled exception was thrown in monitoring thread '{}'. ClusterTopologyMonitor.ClosingMonitor=[ClusterTopologyMonitor, clusterId: '{}'] Closing monitor. ClusterTopologyMonitor.OpenedMonitoringConnection=[ClusterTopologyMonitor, clusterId: '{}'] Opened monitoring connection to host '{}'. ClusterTopologyMonitor.WriterMonitoringConnection=[ClusterTopologyMonitor, clusterId: '{}'] The monitoring connection is connected to a writer: '{}'. ClusterTopologyMonitor.ErrorFetchingTopology=[ClusterTopologyMonitor, clusterId: '{}'] An error occurred while querying for topology: {} ClusterTopologyMonitor.CannotCreateExecutorWhenStopped=[ClusterTopologyMonitor, clusterId: '{}'] Monitor is stopped, cannot create executor. + +MonitoringConnectionHandler.ConnectionAccepted=[MonitoringConnectionHandler] Accepted monitoring connection to host '{}' (role: {}) at priority index {}. +MonitoringConnectionHandler.ConnectionRejected=[MonitoringConnectionHandler] Rejected monitoring connection to host '{}' (isWriter: {}); current priority index {} is better than or equal to offered index {}. +MonitoringConnectionHandler.UpgradedMonitoringConnection=[MonitoringConnectionHandler] Upgraded monitoring connection to host '{}' (priority: {}, index {}). +MonitoringConnectionHandler.UpgradeAttemptFailed=[MonitoringConnectionHandler] Failed to open an upgrade connection to host '{}': {} +GdbMonitoringConnectionHandler.UnrecognizedPriority=[GdbMonitoringConnectionHandler] Unrecognized 'gdb_monitoring_connection_priority' value '{}'. It does not match a known priority variant and does not look like an AWS region, so it will be treated as a region literal that never matches. This is likely a typo. ClusterTopologyMonitor.ResetEventReceived=[ClusterTopologyMonitor] MonitorResetEvent received for cluster '{}'. conftest.ExceptionWhileObtainingInstanceIDs=[conftest] An exception was thrown while attempting to obtain the cluster's instance IDs: '{}' @@ -200,6 +208,8 @@ GdbFailoverPlugin.IsHomeRegion=[GdbFailover] Global Database primary region is h GdbFailoverPlugin.CurrentFailoverMode=[GdbFailover] Current failover mode: {} GdbFailoverPlugin.FailoverElapsed=[GdbFailover] Failover elapsed: {}ms. GdbFailoverPlugin.UnsupportedFailoverMode=[GdbFailover] Unsupported failover mode: {} +GdbFailoverPlugin.WriterNotInAccessibleRegion=[GdbFailover] Writer is in region '{}' which is not in the list of accessible regions {}. +GdbFailoverPlugin.HomeRegionNotInAccessibleRegions=[GdbFailover] Home region '{}' is not included in the list of accessible regions {}. The home region must be accessible. GlobalDbFailoverMode.InvalidValue=[GdbFailoverMode] Invalid Global Database failover mode value: '{}'. @@ -231,6 +241,7 @@ HostMonitor.InvalidWriterQuery=[HostMonitor] The writer topology query is invali HostMonitor.Exception=[HostMonitor] Host monitor for host {} is exiting due to an unknown exception: {} HostMonitor.MonitorCompleted=[HostMonitor] Host monitor for {} completed in {} ms. HostMonitor.WriterHostChanged=[HostMonitor] Writer host changed from {} to {}. +HostMonitor.WriterChangeExitTriggered=[HostMonitor] Reader observed a writer change to {} while some regions are inaccessible; signaling panic-mode exit. HostMonitoringPlugin.ActivatedMonitoring=[HostMonitoringPlugin] Executing method '{}', monitoring is activated. HostMonitoringPlugin.ClusterEndpointHostInfo=[HostMonitoringPlugin] The HostInfo to monitor is associated with a cluster endpoint. The plugin will attempt to identify the connected database instance. @@ -341,6 +352,7 @@ GlobalAuroraTopologyUtils.detectedGdbPatterns=[GlobalAuroraTopologyUtils] Detect GlobalAuroraTopologyUtils.invalidInstanceTemplate=[GlobalAuroraTopologyUtils] Invalid instance template pattern: {} GlobalAuroraTopologyMonitor.cannotFindRegionTemplate=[GlobalAuroraTopologyMonitor] Cannot find cluster template for region {}. +GlobalAuroraTopologyMonitor.InitialHostNotInAccessibleRegion=[GlobalAuroraTopologyMonitor] Initial host '{}' is in region '{}' which is not in the list of accessible regions {}. MultiAzTopologyUtils.UnableToParseInstanceName=[MultiAzTopologyUtils] The MultiAzTopologyUtils was unable to parse the instance name from the endpoint returned by the topology query. @@ -465,6 +477,8 @@ GdbReadWriteSplittingPlugin.CantConnectWriterOutOfHomeRegion=[GdbReadWriteSplitt GdbReadWriteSplittingPlugin.NoAvailableReadersInHomeRegion=[GdbReadWriteSplittingPlugin] No available reader hosts in home region '{}'. GdbReadWriteSplittingPlugin.ParameterValue=[GdbReadWriteSplittingPlugin] {}={} GdbReadWriteSplittingPlugin.EnabledGwf=[GdbReadWriteSplittingPlugin] The current primary writer region is '{}' and is not within the home region. Keeping the current connection and letting Global Write Forwarding redirect writes to the primary region. +GdbReadWriteSplittingPlugin.WriterNotInAccessibleRegion=[GdbReadWriteSplittingPlugin] Writer '{}' is in region '{}' which is not in the list of accessible regions {}. +GdbReadWriteSplittingPlugin.HomeRegionNotInAccessibleRegions=[GdbReadWriteSplittingPlugin] Home region '{}' is not included in the list of accessible regions {}. The home region must be accessible. SqlAlchemyPooledConnectionProvider.PoolNone=[SqlAlchemyPooledConnectionProvider] Attempted to find or create a pool for '{}' but the result of the attempt evaluated to None. SqlAlchemyPooledConnectionProvider.UnableToCreateDefaultKey=[SqlAlchemyPooledConnectionProvider] Unable to create a default key for internal connection pools. By default, the user parameter is used, but the given user evaluated to None or the empty string (""). Please ensure you have passed a valid user in the connection properties. diff --git a/aws_advanced_python_wrapper/utils/accessible_regions.py b/aws_advanced_python_wrapper/utils/accessible_regions.py new file mode 100644 index 000000000..352ba70fe --- /dev/null +++ b/aws_advanced_python_wrapper/utils/accessible_regions.py @@ -0,0 +1,59 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import TYPE_CHECKING, FrozenSet, Optional + +from aws_advanced_python_wrapper.utils.properties import (Properties, + WrapperProperties) + +if TYPE_CHECKING: + from aws_advanced_python_wrapper.utils.rds_utils import RdsUtils + + +def parse(props: Properties) -> Optional[FrozenSet[str]]: + """Parse the ``gdb_accessible_regions`` property into an immutable set of + normalized (lowercased, trimmed) region names. + + Returns ``None`` when the property is unset or empty, meaning all regions + are considered accessible (no restriction). + """ + raw = WrapperProperties.GDB_ACCESSIBLE_REGIONS.get(props) + if not raw or not raw.strip(): + return None + + regions = frozenset( + region.strip().casefold() + for region in raw.split(",") + if region.strip() + ) + return regions if regions else None + + +def is_in_accessible_region( + host: str, + accessible_regions: Optional[FrozenSet[str]], + rds_utils: RdsUtils) -> bool: + """Return whether ``host`` lies in one of the ``accessible_regions``. + + When ``accessible_regions`` is ``None`` or empty (no restriction), every + host is considered accessible. Otherwise the host's region is parsed from + its endpoint; a host whose region cannot be parsed or is not in the set is + excluded. + """ + if not accessible_regions: + return True + region = rds_utils.get_rds_region(host) + return region is not None and region.casefold() in accessible_regions diff --git a/aws_advanced_python_wrapper/utils/gdb_monitoring_connection_priority.py b/aws_advanced_python_wrapper/utils/gdb_monitoring_connection_priority.py new file mode 100644 index 000000000..7287b7437 --- /dev/null +++ b/aws_advanced_python_wrapper/utils/gdb_monitoring_connection_priority.py @@ -0,0 +1,204 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, List, Optional + +from aws_advanced_python_wrapper.hostinfo import HostInfo, HostRole +from aws_advanced_python_wrapper.utils.log import Logger + +if TYPE_CHECKING: + from aws_advanced_python_wrapper.utils.rds_utils import RdsUtils + +logger = Logger(__name__) + +_STRICT_WRITER_PREFIX = "strict-writer-" +_STRICT_READER_PREFIX = "strict-reader-" +_PRIMARY = "primary" +_SECONDARY = "secondary" +_DEFAULT_VALUE = "strict-writer-primary" + +# AWS region identifiers look like "us-east-1", "eu-west-2", "ap-southeast-1". +# Any unrecognized token that does not match this shape is most likely a typo +# (e.g. "strict-wrtier-primary"); mirrors the Node.js wrapper's REGION_SHAPE. +_REGION_SHAPE = re.compile(r"^[a-z]{2}-[a-z]+-\d+$") + + +class GdbMonitoringConnectionPriority: + """Region-aware monitoring connection priority for Aurora Global Databases. + + A priority describes the kind of host the topology monitor prefers for its + background connection, combining an optional required role, an optional + required region, and primary/secondary-region flags. + + Supported priority strings: + + - ``strict-writer-primary`` — writer in the primary region. + - ``strict-reader-primary`` — reader in the primary region. + - ``strict-reader-secondary`` — reader in any secondary region. + - ``strict-writer-`` / ``strict-reader-`` — that role in the + named region. ``strict-writer-secondary`` is rejected (a writer cannot be + in a secondary region). + - ```` — any host in the named region. + """ + + def __init__( + self, + required_role: Optional[HostRole], + required_region: Optional[str], + require_primary: bool, + require_secondary: bool, + original_value: str): + self._required_role = required_role + self._required_region = required_region + self._require_primary = require_primary + self._require_secondary = require_secondary + self._original_value = original_value + + @property + def required_role(self) -> Optional[HostRole]: + return self._required_role + + @property + def required_region(self) -> Optional[str]: + return self._required_region + + @property + def require_primary(self) -> bool: + return self._require_primary + + @property + def require_secondary(self) -> bool: + return self._require_secondary + + @classmethod + def from_value(cls, value: Optional[str]) -> Optional[GdbMonitoringConnectionPriority]: + if value is None or not value.strip(): + return None + + trimmed = value.strip().lower() + + if trimmed.startswith(_STRICT_WRITER_PREFIX): + suffix = trimmed[len(_STRICT_WRITER_PREFIX):] + if not suffix: + return None + if suffix == _PRIMARY: + return cls(HostRole.WRITER, None, True, False, trimmed) + if suffix == _SECONDARY: + # A writer cannot live in a secondary region for an Aurora + # Global Database (only the primary region has a writer). + return None + return cls(HostRole.WRITER, suffix, False, False, trimmed) + + if trimmed.startswith(_STRICT_READER_PREFIX): + suffix = trimmed[len(_STRICT_READER_PREFIX):] + if not suffix: + return None + if suffix == _PRIMARY: + return cls(HostRole.READER, None, True, False, trimmed) + if suffix == _SECONDARY: + return cls(HostRole.READER, None, False, True, trimmed) + return cls(HostRole.READER, suffix, False, False, trimmed) + + # Any token without a known prefix is treated as a bare region literal. + # If it does not even look like an AWS region identifier, it is almost + # certainly a typo that will never match a host, so warn to aid + # diagnosis rather than coercing it silently. + if not _REGION_SHAPE.match(trimmed): + logger.debug("GdbMonitoringConnectionHandler.UnrecognizedPriority", value) + return cls(None, trimmed, False, False, trimmed) + + @classmethod + def parse_list(cls, value: Optional[str]) -> List[GdbMonitoringConnectionPriority]: + """Parse a comma-separated priority list. + + Defaults to ``[strict-writer-primary]`` when the value is unset/empty or + when no item parses to a valid priority. Unlike the plain + :class:`MonitoringConnectionPriority`, duplicates are **not** dropped. + """ + result: List[GdbMonitoringConnectionPriority] = [] + if value is None or not value.strip(): + result.append(cls(HostRole.WRITER, None, True, False, _DEFAULT_VALUE)) + return result + + for item in value.split(","): + priority = cls.from_value(item) + if priority is not None: + result.append(priority) + + if not result: + result.append(cls(HostRole.WRITER, None, True, False, _DEFAULT_VALUE)) + return result + + def is_satisfied_by( + self, + host: HostInfo, + primary_region: Optional[str], + rds_utils: RdsUtils) -> bool: + if self._required_role is not None and host.role != self._required_role: + return False + + host_region = rds_utils.get_rds_region(host.host) + + if self._require_primary: + if primary_region is None or host_region is None \ + or primary_region.casefold() != host_region.casefold(): + return False + if self._require_secondary: + if primary_region is None or host_region is None \ + or primary_region.casefold() == host_region.casefold(): + return False + + if self._required_region is not None: + if host_region is None or self._required_region.casefold() != host_region.casefold(): + return False + + return True + + def find_matching_host( + self, + hosts: List[HostInfo], + primary_region: Optional[str], + rds_utils: RdsUtils) -> Optional[HostInfo]: + for host in hosts: + if self.is_satisfied_by(host, primary_region, rds_utils): + return host + return None + + def find_matching_hosts( + self, + hosts: List[HostInfo], + primary_region: Optional[str], + rds_utils: RdsUtils) -> List[HostInfo]: + return [host for host in hosts if self.is_satisfied_by(host, primary_region, rds_utils)] + + def __str__(self) -> str: + return self._original_value + + def __repr__(self) -> str: + return f"GdbMonitoringConnectionPriority({self._original_value!r})" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, GdbMonitoringConnectionPriority): + return NotImplemented + return (self._required_role == other._required_role + and self._required_region == other._required_region + and self._require_primary == other._require_primary + and self._require_secondary == other._require_secondary) + + def __hash__(self) -> int: + return hash((self._required_role, self._required_region, + self._require_primary, self._require_secondary)) diff --git a/aws_advanced_python_wrapper/utils/monitoring_connection_priority.py b/aws_advanced_python_wrapper/utils/monitoring_connection_priority.py new file mode 100644 index 000000000..7743c8a1e --- /dev/null +++ b/aws_advanced_python_wrapper/utils/monitoring_connection_priority.py @@ -0,0 +1,73 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from enum import Enum +from typing import List, Optional + + +class MonitoringConnectionPriority(Enum): + """Priority of the topology monitor's background connection. + + The topology monitor accepts whatever connection it obtains first, then + asynchronously upgrades to a higher-priority connection. + """ + + STRICT_WRITER = "strict-writer" + STRICT_READER = "strict-reader" + WRITER_OR_READER = "writer-or-reader" + + @classmethod + def from_value(cls, value: Optional[str]) -> Optional[MonitoringConnectionPriority]: + if value is None: + return None + return _NAME_TO_VALUE.get(value.strip().lower()) + + @classmethod + def parse_list(cls, value: Optional[str]) -> List[MonitoringConnectionPriority]: + """Parse a comma-separated priority list. + + Defaults to ``[STRICT_WRITER]`` when the value is unset/empty or when no + item parses to a known priority. Duplicates are dropped, order preserved. + """ + result: List[MonitoringConnectionPriority] = [] + if value is None or not value.strip(): + result.append(cls.STRICT_WRITER) + return result + + for item in value.split(","): + priority = cls.from_value(item.strip()) + if priority is not None and priority not in result: + result.append(priority) + + if not result: + result.append(cls.STRICT_WRITER) + return result + + def is_satisfied_by(self, is_writer: bool) -> bool: + if self is MonitoringConnectionPriority.STRICT_WRITER: + return is_writer + if self is MonitoringConnectionPriority.STRICT_READER: + return not is_writer + if self is MonitoringConnectionPriority.WRITER_OR_READER: + return True + return False + + +_NAME_TO_VALUE = { + "strict-writer": MonitoringConnectionPriority.STRICT_WRITER, + "strict-reader": MonitoringConnectionPriority.STRICT_READER, + "writer-or-reader": MonitoringConnectionPriority.WRITER_OR_READER, +} diff --git a/aws_advanced_python_wrapper/utils/properties.py b/aws_advanced_python_wrapper/utils/properties.py index 25be0ecd5..d2f079c88 100644 --- a/aws_advanced_python_wrapper/utils/properties.py +++ b/aws_advanced_python_wrapper/utils/properties.py @@ -717,6 +717,32 @@ class WrapperProperties: False, ) + GDB_ACCESSIBLE_REGIONS = WrapperProperty( + "gdb_accessible_regions", + "Comma-separated list of AWS regions accessible by the application. " + "When set, failover, topology monitoring, and read/write splitting " + "will only consider nodes in these regions.", + None, + ) + + MONITORING_CONNECTION_PRIORITY = WrapperProperty( + "monitoring_connection_priority", + "Comma-separated priority list for the topology monitor's background " + "connection. Values: 'strict-writer', 'strict-reader', " + "'writer-or-reader'. The monitor accepts any connection initially, " + "then asynchronously upgrades to a higher-priority one.", + "strict-writer", + ) + + GDB_MONITORING_CONNECTION_PRIORITY = WrapperProperty( + "gdb_monitoring_connection_priority", + "Comma-separated, region-aware priority list for the Global Database " + "topology monitor's background connection. Values combine role, region, " + "and primary/secondary awareness, e.g. 'strict-writer-primary', " + "'strict-reader-secondary', 'strict-reader-us-east-1', 'us-west-2'.", + "strict-writer-primary", + ) + class PropertiesUtils: _MONITORING_PROPERTY_PREFIX = "monitoring-" diff --git a/docs/using-the-python-wrapper/GlobalDatabases.md b/docs/using-the-python-wrapper/GlobalDatabases.md index 7989b4fa5..dc0c9baef 100644 --- a/docs/using-the-python-wrapper/GlobalDatabases.md +++ b/docs/using-the-python-wrapper/GlobalDatabases.md @@ -123,6 +123,12 @@ The `global_cluster_instance_host_patterns` parameter is **required** for Aurora - Different cluster identifiers for each region (e.g., `XYZ1`, `XYZ2`) - Example: `us-east-2:?.XYZ1.us-east-2.rds.amazonaws.com,us-west-2:?.XYZ2.us-west-2.rds.amazonaws.com` +### Restricting to Accessible Regions +If your application can only reach a subset of the regions the global cluster spans, use the `gdb_accessible_regions` property to restrict host selection to those regions. See [Restricting Aurora Global Databases to Accessible Regions](./using-plugins/UsingGlobalAuroraAccessibleRegions.md). + +### Monitoring Connection Priority +To control which host role or region the topology monitor uses for its background connection, use the `monitoring_connection_priority` / `gdb_monitoring_connection_priority` properties. See [Monitoring Connection Priority](./using-plugins/UsingMonitoringConnectionPriority.md). + ### Authentication Plugins Compatible with GDB - [IAM Authentication Plugin](./using-plugins/UsingTheIamAuthenticationPlugin.md) - [Federated Authentication Plugin](./using-plugins/UsingTheFederatedAuthPlugin.md) diff --git a/docs/using-the-python-wrapper/using-plugins/UsingGlobalAuroraAccessibleRegions.md b/docs/using-the-python-wrapper/using-plugins/UsingGlobalAuroraAccessibleRegions.md new file mode 100644 index 000000000..af1e2a39d --- /dev/null +++ b/docs/using-the-python-wrapper/using-plugins/UsingGlobalAuroraAccessibleRegions.md @@ -0,0 +1,52 @@ +# Restricting Aurora Global Databases to Accessible Regions + +When using [Aurora Global Databases](../GlobalDatabases.md), an application may only be able to reach a subset of the regions the global cluster spans (for example, because of network routing, VPC peering, or security constraints). The `gdb_accessible_regions` property restricts the AWS Advanced Python Wrapper to a set of reachable AWS regions, excluding hosts in all other regions from host selection. + +## `gdb_accessible_regions` + +| Property | Value | Default | +|----------------------------|--------------------------------------------------------------------------------------------------------------------|-----------------------------------------| +| `gdb_accessible_regions` | Comma-separated list of AWS region names the application can reach (for example, `us-east-1,us-west-2`). Region names are matched case-insensitively and surrounding whitespace is trimmed. | Unset — **all regions are accessible** (no restriction). | + +When the property is unset (or empty), no filtering is applied and every region in the global cluster is treated as accessible. + +**Key constraint:** the **home region must be included** in `gdb_accessible_regions`. If it is not, the connection fails at initialization with an error, because the home region must always be reachable. + +## Behavior by Component + +The accessible-regions filter is applied **before** all other selection logic — role preference, failover mode, home-region restriction, and initial-connection strategy all operate on the already-filtered host list. + +| Component | Behavior | +|-----------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------| +| [GDB Failover plugin](./UsingTheGdbFailoverPlugin.md) | Validates the home region at init. In `strict-writer` mode, **fails loudly** (`FailoverFailedError`) if the new writer is in an inaccessible region. In all other modes, filters out inaccessible-region hosts before candidate selection. | +| [GDB Read/Write Splitting plugin](./UsingTheGdbReadWriteSplittingPlugin.md) | Validates the home region at init. Rejects (`ReadWriteSplittingError`) a writer in an inaccessible region. Filters readers by accessible region **before** applying the home-region restriction. | +| Aurora Initial Connection Strategy plugin | Excludes inaccessible-region hosts before selecting a host by strategy. | +| Global Aurora topology monitor | Skips node-monitoring workers for hosts in inaccessible regions, and fails if the initial host is itself in an inaccessible region. | + +### Fail-loud, not silent fallback + +Consistent with the [AWS Advanced Python Wrapper](https://github.com/aws/aws-advanced-python-wrapper), the filter is a **hard restriction**. When the writer is in an inaccessible region, the wrapper raises an error rather than silently connecting to an unreachable or unintended host. When reader filtering leaves no candidates in accessible regions, the wrapper does **not** fall back to the unfiltered host list. + +## Example + +```python +from aws_advanced_python_wrapper import AwsWrapperConnection +from psycopg import Connection + +with AwsWrapperConnection.connect( + Connection.connect, + "host=my-global-db.global-xyz.global.rds.amazonaws.com dbname=mydb user=admin password=pwd", + plugins="initial_connection,failover2,efm2", + wrapper_dialect="global-aurora-pg", + cluster_id="1", + global_cluster_instance_host_patterns="us-east-1:?.abc123.us-east-1.rds.amazonaws.com,us-west-2:?.def456.us-west-2.rds.amazonaws.com", + # Only us-east-1 and us-west-2 are reachable from this application. + gdb_accessible_regions="us-east-1,us-west-2", + autocommit=True +) as awsconn: + awscursor = awsconn.cursor() + awscursor.execute("SELECT pg_catalog.aurora_db_instance_identifier()") + print(awscursor.fetchone()) +``` + +> **Note:** the home region (derived from the connection endpoint or `failover_home_region` / `gdb_rw_home_region`) must appear in `gdb_accessible_regions`. diff --git a/docs/using-the-python-wrapper/using-plugins/UsingMonitoringConnectionPriority.md b/docs/using-the-python-wrapper/using-plugins/UsingMonitoringConnectionPriority.md new file mode 100644 index 000000000..8494d6096 --- /dev/null +++ b/docs/using-the-python-wrapper/using-plugins/UsingMonitoringConnectionPriority.md @@ -0,0 +1,129 @@ +# Monitoring Connection Priority + +The monitoring connection priority properties let you control which kind of host the topology monitor connects to for its background monitoring connection. This is useful both for standard Aurora clusters and for [Aurora Global Databases](../GlobalDatabases.md), where you may want to keep monitoring traffic on a particular host role or region. + +## Overview + +The topology monitor maintains a background connection it uses to observe cluster topology changes. By default it prefers a **writer** connection, which provides the most accurate and timely topology information. These properties let you change that preference. + +Two properties are available: + +- **`monitoring_connection_priority`** — for standard Aurora clusters. Selects the host role used for the monitoring connection. +- **`gdb_monitoring_connection_priority`** — for Aurora Global Databases. Extends the standard property with region-aware and primary/secondary-aware values. + +Both accept a **comma-separated, ordered priority list**. The monitor accepts whatever connection it obtains first, then **asynchronously upgrades** to a higher-priority host (see [Async upgrade behavior](#async-upgrade-behavior)) without blocking the monitoring loop. + +## Configuration Properties + +| Property | Value | Required | Description | Default | +| ------------------------------------- | :------: | :------: | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------- | +| `monitoring_connection_priority` | `String` | No | Comma-separated ordered priority list for the topology monitor's background connection. Values: `strict-writer`, `strict-reader`, `writer-or-reader`. | `strict-writer` | +| `gdb_monitoring_connection_priority` | `String` | No | Comma-separated, region-aware ordered priority list for the Global Database topology monitor. See [GDB values](#gdb-values-gdb_monitoring_connection_priority) below. | `strict-writer-primary` | + +Only one of these applies at a time: `gdb_monitoring_connection_priority` is used by the Global Aurora topology monitor; `monitoring_connection_priority` is used by the standard Aurora topology monitor. + +## Priority Values + +### Standard values (`monitoring_connection_priority`) + +| Value | Description | +| ------------------ | ------------------------------------------------------------------------------------------------------- | +| `strict-writer` | Prefer a **writer** host for the monitoring connection. | +| `strict-reader` | Prefer a **reader** host for the monitoring connection. | +| `writer-or-reader` | Any host is acceptable (no role preference). | + +Parsing notes: + +- Unrecognized tokens are ignored. Duplicate values are dropped, order preserved. +- If the value is unset, empty, or no token parses to a known value, it defaults to `strict-writer`. + +### GDB values (`gdb_monitoring_connection_priority`) + +| Value | Description | +| --------------------------- | -------------------------------------------------------------------------------------------------------------- | +| `strict-writer-primary` | A **writer** in the **primary** region of the Global Database. | +| `strict-reader-primary` | A **reader** in the **primary** region. | +| `strict-reader-secondary` | A **reader** in any **secondary** (non-primary) region. | +| `strict-writer-` | A **writer** in the named region, e.g. `strict-writer-us-east-1`. | +| `strict-reader-` | A **reader** in the named region, e.g. `strict-reader-us-west-2`. | +| `` | Any host (writer or reader) in the named AWS region, e.g. `us-west-2`. | + +Parsing notes: + +- **`strict-writer-secondary` is rejected** — a writer cannot exist in a secondary region of a Global Database (only the primary region has a writer). The token is skipped. +- There is **no** `writer-or-reader-primary` / `writer-or-reader-secondary` value. To target any host in a region, use the bare `` form. +- Any token that is not one of the `strict-writer-*` / `strict-reader-*` forms is treated as a **bare region literal**. A typo (for example `strict-wrtier-primary`) is therefore silently accepted as a region name that will simply never match — double-check spelling. +- Unlike the standard property, duplicate GDB values are **not** dropped (order preserved). +- If the value is unset, empty, or no token parses, it defaults to `strict-writer-primary`. + +The "primary region" is the region of the current writer host; a "secondary region" is any other region the global cluster spans. + +## Usage + +### Standard Aurora cluster + +```python +from aws_advanced_python_wrapper import AwsWrapperConnection +from psycopg import Connection + +with AwsWrapperConnection.connect( + Connection.connect, + "host=my-cluster.cluster-xyz.us-east-1.rds.amazonaws.com dbname=mydb user=admin password=pwd", + plugins="failover2,efm2", + monitoring_connection_priority="writer-or-reader", + autocommit=True +) as awsconn: + awscursor = awsconn.cursor() + awscursor.execute("SELECT 1") + print(awscursor.fetchone()) +``` + +### Aurora Global Database + +```python +from aws_advanced_python_wrapper import AwsWrapperConnection +from psycopg import Connection + +with AwsWrapperConnection.connect( + Connection.connect, + "host=my-global-db.global-xyz.global.rds.amazonaws.com dbname=mydb user=admin password=pwd", + plugins="initial_connection,gdb_failover,efm2", + wrapper_dialect="global-aurora-pg", + failover_home_region="us-west-2", + global_cluster_instance_host_patterns="us-east-1:?.abc123.us-east-1.rds.amazonaws.com,us-west-2:?.def456.us-west-2.rds.amazonaws.com", + gdb_monitoring_connection_priority="strict-writer-primary", + autocommit=True +) as awsconn: + awscursor = awsconn.cursor() + awscursor.execute("SELECT pg_catalog.aurora_db_instance_identifier()") + print(awscursor.fetchone()) +``` + +### Keeping monitoring traffic in a specific region + +```python +# Direct the monitoring connection to any host in us-west-2 to reduce +# cross-region monitoring latency. +gdb_monitoring_connection_priority="us-west-2" +``` + +## Async upgrade behavior + +The monitor does not block waiting for its preferred host. It accepts the first connection it can obtain, then in the background attempts to upgrade to a higher-priority host from the current priority list. For example, with `strict-writer-primary` configured but the primary writer temporarily unreachable, the monitor may connect to another host and upgrade once the primary writer becomes reachable. The upgrade runs on a background worker and never stalls the monitoring loop. + +## Interaction with accessible regions + +When [`gdb_accessible_regions`](./UsingGlobalAuroraAccessibleRegions.md) is configured, the accessible-regions filter is applied **first**: upgrade candidates and monitored hosts in inaccessible regions are excluded before the priority list is consulted. + +> [!WARNING] +> If `gdb_monitoring_connection_priority` names a region (or a `strict-*-` value) that is not in `gdb_accessible_regions`, that priority can never match a monitored host. Keep the two properties consistent. + +When the writer lives in an inaccessible region and no host monitor can reach it directly, the monitor exits panic mode by adopting a harvested reader connection (reader-consensus / stable-reader-topology exit), so monitoring still proceeds against reachable hosts. + +## Tuning guidance + +- Use `strict-writer` (the default) for most applications — writer connections give the most accurate, timely topology. +- Use `strict-reader` to keep monitoring load off the writer, accepting slightly more delayed topology updates. +- Use `writer-or-reader` for maximum monitoring availability when any host is acceptable. +- For Global Databases, prefer `strict-writer-primary` to read topology from the primary region's writer. +- Use a bare `` to keep monitoring traffic local and reduce cross-region latency. diff --git a/tests/unit/test_accessible_regions.py b/tests/unit/test_accessible_regions.py new file mode 100644 index 000000000..b6d4827fc --- /dev/null +++ b/tests/unit/test_accessible_regions.py @@ -0,0 +1,161 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from aws_advanced_python_wrapper.database_dialect import ( + GlobalAuroraMysqlDialect, GlobalAuroraPgDialect, MysqlDatabaseDialect) +from aws_advanced_python_wrapper.hostinfo import HostInfo, HostRole +from aws_advanced_python_wrapper.utils.accessible_regions import parse +from aws_advanced_python_wrapper.utils.properties import Properties + + +class TestParseAccessibleRegions: + def test_returns_none_when_property_not_set(self): + props = Properties() + assert parse(props) is None + + def test_returns_none_when_empty_string(self): + props = Properties() + props["gdb_accessible_regions"] = "" + assert parse(props) is None + + def test_returns_none_when_whitespace_only(self): + props = Properties() + props["gdb_accessible_regions"] = " " + assert parse(props) is None + + def test_parses_single_region(self): + props = Properties() + props["gdb_accessible_regions"] = "us-east-1" + result = parse(props) + assert result == frozenset({"us-east-1"}) + + def test_parses_multiple_regions(self): + props = Properties() + props["gdb_accessible_regions"] = "us-east-1,us-west-2,eu-central-1" + result = parse(props) + assert result == frozenset({"us-east-1", "us-west-2", "eu-central-1"}) + + def test_normalizes_to_lowercase(self): + props = Properties() + props["gdb_accessible_regions"] = "US-EAST-1,Us-West-2" + result = parse(props) + assert result == frozenset({"us-east-1", "us-west-2"}) + + def test_trims_whitespace(self): + props = Properties() + props["gdb_accessible_regions"] = " us-east-1 , us-west-2 " + result = parse(props) + assert result == frozenset({"us-east-1", "us-west-2"}) + + def test_skips_empty_entries(self): + props = Properties() + props["gdb_accessible_regions"] = "us-east-1,,us-west-2," + result = parse(props) + assert result == frozenset({"us-east-1", "us-west-2"}) + + def test_returns_frozenset(self): + props = Properties() + props["gdb_accessible_regions"] = "us-east-1" + result = parse(props) + assert isinstance(result, frozenset) + + +class TestDialectFilterAvailableHosts: + @staticmethod + def _make_host(host: str, role: HostRole = HostRole.READER) -> HostInfo: + return HostInfo(host=host, role=role) + + def _sample_hosts(self): + return [ + self._make_host("instance1.cluster-xyz.us-east-1.rds.amazonaws.com", HostRole.WRITER), + self._make_host("instance2.cluster-ro-xyz.us-east-1.rds.amazonaws.com", HostRole.READER), + self._make_host("instance3.cluster-xyz.us-west-2.rds.amazonaws.com", HostRole.READER), + self._make_host("instance4.cluster-ro-xyz.eu-central-1.rds.amazonaws.com", HostRole.READER), + ] + + def test_global_aurora_mysql_filters_by_region(self): + dialect = GlobalAuroraMysqlDialect() + regions = frozenset({"us-east-1", "us-west-2"}) + hosts = self._sample_hosts() + + filtered = dialect.filter_available_hosts(hosts, regions) + + assert len(filtered) == 3 + for h in filtered: + assert "eu-central-1" not in h.host + + def test_global_aurora_pg_filters_by_region(self): + dialect = GlobalAuroraPgDialect() + regions = frozenset({"us-east-1"}) + hosts = self._sample_hosts() + + filtered = dialect.filter_available_hosts(hosts, regions) + + assert len(filtered) == 2 + for h in filtered: + assert "us-east-1" in h.host + + def test_returns_all_when_no_restriction(self): + dialect = GlobalAuroraMysqlDialect() + hosts = self._sample_hosts() + + assert dialect.filter_available_hosts(hosts, None) == hosts + + def test_returns_all_when_empty_frozenset(self): + dialect = GlobalAuroraMysqlDialect() + hosts = self._sample_hosts() + + assert dialect.filter_available_hosts(hosts, frozenset()) == hosts + + def test_non_global_dialect_returns_all(self): + dialect = MysqlDatabaseDialect() + hosts = self._sample_hosts() + regions = frozenset({"us-east-1"}) + + result = dialect.filter_available_hosts(hosts, regions) + assert result == hosts + + def test_returns_list_for_tuple_input(self): + # The method accepts any Sequence and always returns a list, so the + # return type matches its annotation even for tuple input. + dialect = GlobalAuroraMysqlDialect() + hosts = tuple(self._sample_hosts()) + + result = dialect.filter_available_hosts(hosts, None) + assert isinstance(result, list) + assert result == list(hosts) + + def test_case_insensitive_region_matching(self): + dialect = GlobalAuroraMysqlDialect() + regions = frozenset({"us-east-1"}) + hosts = [ + self._make_host("instance1.cluster-xyz.us-east-1.rds.amazonaws.com", HostRole.WRITER), + ] + + filtered = dialect.filter_available_hosts(hosts, regions) + assert len(filtered) == 1 + + def test_excludes_hosts_without_parseable_region(self): + dialect = GlobalAuroraMysqlDialect() + regions = frozenset({"us-east-1"}) + hosts = [ + self._make_host("instance1.cluster-xyz.us-east-1.rds.amazonaws.com", HostRole.WRITER), + self._make_host("custom-domain.example.com", HostRole.READER), + ] + + filtered = dialect.filter_available_hosts(hosts, regions) + assert len(filtered) == 1 + assert "us-east-1" in filtered[0].host diff --git a/tests/unit/test_aio_aurora_initial_connection.py b/tests/unit/test_aio_aurora_initial_connection.py index 052f2eb01..156a69b7c 100644 --- a/tests/unit/test_aio_aurora_initial_connection.py +++ b/tests/unit/test_aio_aurora_initial_connection.py @@ -678,6 +678,7 @@ def test_candidate_host_substitute_with_any_raises_unsupported_strategy(): svc.get_host_info_by_strategy.assert_not_called() + def test_endpoint_substitution_role_on_instance_endpoint_raises(): """Substitution cannot be requested for an instance endpoint.""" props_overrides = {"endpoint_substitution_role": "writer"} diff --git a/tests/unit/test_aurora_initial_connection_strategy_accessible_regions.py b/tests/unit/test_aurora_initial_connection_strategy_accessible_regions.py new file mode 100644 index 000000000..c7686702f --- /dev/null +++ b/tests/unit/test_aurora_initial_connection_strategy_accessible_regions.py @@ -0,0 +1,117 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from aws_advanced_python_wrapper.aurora_initial_connection_strategy_plugin import \ + AuroraInitialConnectionStrategyPlugin +from aws_advanced_python_wrapper.database_dialect import ( + GlobalAuroraPgDialect, MysqlDatabaseDialect) +from aws_advanced_python_wrapper.hostinfo import HostInfo, HostRole +from aws_advanced_python_wrapper.utils.properties import (Properties, + WrapperProperties) +from aws_advanced_python_wrapper.utils.rds_url_type import RdsUrlType + +WRITER_HOME = HostInfo("writer.cluster-xyz.us-west-1.rds.amazonaws.com", 5432, HostRole.WRITER) +READER_HOME = HostInfo("reader1.xyz.us-west-1.rds.amazonaws.com", 5432, HostRole.READER) +READER_OUT = HostInfo("reader2.xyz.us-east-1.rds.amazonaws.com", 5432, HostRole.READER) +WRITER_OUT = HostInfo("writer.cluster-xyz.us-east-1.rds.amazonaws.com", 5432, HostRole.WRITER) + +ALL_HOSTS = (WRITER_HOME, READER_HOME, READER_OUT, WRITER_OUT) + + +@pytest.fixture +def plugin_service_mock(): + mock = MagicMock() + mock.all_hosts = ALL_HOSTS + mock.database_dialect = GlobalAuroraPgDialect() + return mock + + +def _make_plugin(plugin_service_mock, accessible_regions=None): + props = Properties() + if accessible_regions is not None: + WrapperProperties.GDB_ACCESSIBLE_REGIONS.set(props, accessible_regions) + return AuroraInitialConnectionStrategyPlugin(plugin_service_mock, props) + + +class TestFilterByAccessibleRegions: + def test_no_restriction_returns_all(self, plugin_service_mock): + plugin = _make_plugin(plugin_service_mock) + result = plugin._filter_by_accessible_regions(ALL_HOSTS) + assert list(result) == list(ALL_HOSTS) + + def test_filters_by_region(self, plugin_service_mock): + plugin = _make_plugin(plugin_service_mock, "us-west-1") + result = plugin._filter_by_accessible_regions(ALL_HOSTS) + assert result == [WRITER_HOME, READER_HOME] + + def test_non_global_dialect_returns_all(self, plugin_service_mock): + plugin_service_mock.database_dialect = MysqlDatabaseDialect() + plugin = _make_plugin(plugin_service_mock, "us-west-1") + result = plugin._filter_by_accessible_regions(ALL_HOSTS) + # Non-global dialect's default filter_available_hosts is a no-op. + assert list(result) == list(ALL_HOSTS) + + def test_none_dialect_returns_all(self, plugin_service_mock): + plugin_service_mock.database_dialect = None + plugin = _make_plugin(plugin_service_mock, "us-west-1") + result = plugin._filter_by_accessible_regions(ALL_HOSTS) + assert list(result) == list(ALL_HOSTS) + + +class TestFindWriter: + """`_find_writer` is unfiltered by design; accessible-region filtering is + applied by the caller.""" + + def test_returns_first_writer_unfiltered(self, plugin_service_mock): + plugin = _make_plugin(plugin_service_mock, "us-west-1") + # Passed the raw host list, it returns the first writer regardless of region. + assert plugin._find_writer(ALL_HOSTS) == WRITER_HOME + + def test_returns_out_of_region_writer_when_not_pre_filtered(self, plugin_service_mock): + plugin = _make_plugin(plugin_service_mock, "us-west-1") + # An out-of-region writer is still returned — filtering is the caller's job. + assert plugin._find_writer((READER_HOME, WRITER_OUT)) == WRITER_OUT + + def test_returns_none_when_no_writer(self, plugin_service_mock): + plugin = _make_plugin(plugin_service_mock, "us-west-1") + assert plugin._find_writer((READER_HOME, READER_OUT)) is None + + +class TestGetCandidateHostWriter: + """The SUBSTITUTE_WITH_WRITER branch of `_get_candidate_host` filters by + accessible regions before picking the writer.""" + + def _candidate(self, plugin, original_host): + from aws_advanced_python_wrapper.aurora_initial_connection_strategy_plugin import \ + InstanceSubstitutionStrategy + return plugin._get_candidate_host( + original_host, + RdsUrlType.RDS_WRITER_CLUSTER, + InstanceSubstitutionStrategy.SUBSTITUTE_WITH_WRITER) + + def test_writer_in_accessible_region_returned(self, plugin_service_mock): + plugin = _make_plugin(plugin_service_mock, "us-west-1") + assert self._candidate(plugin, WRITER_HOME) == WRITER_HOME + + def test_writer_filtered_out_returns_none(self, plugin_service_mock): + # Only the out-of-region writer exists; it is filtered out before selection. + plugin_service_mock.all_hosts = (READER_HOME, WRITER_OUT) + plugin = _make_plugin(plugin_service_mock, "us-west-1") + assert self._candidate(plugin, WRITER_OUT) is None diff --git a/tests/unit/test_cluster_topology_monitor_item8.py b/tests/unit/test_cluster_topology_monitor_item8.py new file mode 100644 index 000000000..0b425035b --- /dev/null +++ b/tests/unit/test_cluster_topology_monitor_item8.py @@ -0,0 +1,421 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit coverage for the item-8 (``some_regions_inaccessible``) panic-exit +machinery of ``ClusterTopologyMonitorImpl`` — the paths that let a GDB cluster +exit panic mode when the writer lives in an inaccessible region. + +These tests exercise the *decision logic* in isolation with mocked node +threads: connection harvesting (``_adopt_harvested_monitoring_connection``), the stable-reader +fallback (``_check_for_stable_reader_topologies``), reader-observed +writer-change detection (``HostMonitor._reader_thread_fetch_topology``), and the +worker connection hand-off (``_harvest_connection``). They do not spin up real +monitoring threads — a bare monitor is built via ``__new__`` and only the +attributes each method touches are populated (same approach as +``test_global_aurora_topology_monitor_accessible_regions.py``).""" + +from __future__ import annotations + +import threading +from typing import List, Optional, Tuple +from unittest.mock import MagicMock + +import pytest + +from aws_advanced_python_wrapper.cluster_topology_monitor import ( + ClusterTopologyMonitorImpl, HostMonitor) +from aws_advanced_python_wrapper.hostinfo import HostInfo, HostRole +from aws_advanced_python_wrapper.utils.atomic import AtomicReference +from aws_advanced_python_wrapper.utils.thread_safe_connection_holder import \ + ThreadSafeConnectionHolder + +WRITER = HostInfo("writer.cluster-xyz.us-west-1.rds.amazonaws.com", 5432, HostRole.WRITER) +READER_A = HostInfo("reader1.xyz.us-west-1.rds.amazonaws.com", 5432, HostRole.READER) +READER_B = HostInfo("reader2.xyz.us-west-1.rds.amazonaws.com", 5432, HostRole.READER) +NEW_WRITER = HostInfo("writer2.cluster-xyz.us-east-1.rds.amazonaws.com", 5432, HostRole.WRITER) + + +def _host_and_port(h: HostInfo) -> str: + return f"{h.host}:{h.port}" + + +def _bare_monitor(*, accessible_filter=None) -> ClusterTopologyMonitorImpl: + """Build a monitor without running its constructor (which starts threads). + + ``accessible_filter`` optionally overrides ``_filter_hosts_for_host_monitoring`` + to simulate the GDB subclass dropping inaccessible hosts. + """ + monitor = ClusterTopologyMonitorImpl.__new__(ClusterTopologyMonitorImpl) + monitor._cluster_id = "cluster-xyz" + monitor._host_threads_map_lock = threading.Lock() + monitor._host_threads_connections = {} + monitor._reader_topologies_by_id = {} + monitor._completed_one_cycle = {} + monitor._stable_topologies_start_nano = 0 + monitor._reader_observed_writer_host_info = AtomicReference(None) + monitor._host_threads_stop = threading.Event() + monitor._host_threads_latest_topology = AtomicReference(None) + monitor._thread_pool_executor = AtomicReference(None) + monitor._submitted_hosts = {} + monitor._monitoring_connection = ThreadSafeConnectionHolder(None) + monitor._writer_host_info = AtomicReference(None) + monitor._connection_handler = MagicMock() + + if accessible_filter is not None: + monitor._filter_hosts_for_host_monitoring = accessible_filter # type: ignore[method-assign] + return monitor + + +class TestHarvestConnections: + def test_returns_false_and_clears_when_no_connections(self): + monitor = _bare_monitor() + # No harvested connections in the map. + result = monitor._adopt_harvested_monitoring_connection(None, (WRITER, READER_A)) + + assert result is False + # State cleared; handler never consulted. + assert monitor._host_threads_connections == {} + monitor._connection_handler.accept_connections.assert_not_called() + + def test_adopts_selected_and_closes_the_rest(self): + monitor = _bare_monitor() + adopted_conn, dropped_conn = MagicMock(name="adopted"), MagicMock(name="dropped") + adopted_holder = ThreadSafeConnectionHolder(adopted_conn) + dropped_holder = ThreadSafeConnectionHolder(dropped_conn) + monitor._host_threads_connections = { + _host_and_port(READER_A): (READER_A, adopted_holder), + _host_and_port(READER_B): (READER_B, dropped_holder), + } + # Handler adopts READER_A. + monitor._connection_handler.accept_connections.return_value = READER_A + + result = monitor._adopt_harvested_monitoring_connection(None, (READER_A, READER_B)) + + assert result is True + monitor._connection_handler.accept_connections.assert_called_once() + # The non-adopted connection is closed; the adopted one is left alone. + dropped_conn.close.assert_called_once() + adopted_conn.close.assert_not_called() + # Map is emptied after harvest. + assert monitor._host_threads_connections == {} + + def test_joins_workers_before_reading_map(self): + """The executor must be joined (wait=True) BEFORE the harvest reads the + map, so no worker is still touching a connection being handed off.""" + monitor = _bare_monitor() + order: List[str] = [] + + executor = MagicMock() + executor.shutdown.side_effect = lambda **kw: order.append("shutdown") + monitor._thread_pool_executor = AtomicReference(executor) + + holder = ThreadSafeConnectionHolder(MagicMock()) + monitor._host_threads_connections = {_host_and_port(READER_A): (READER_A, holder)} + + def _accept(conns, writer, topo): + order.append("accept_connections") + return READER_A + monitor._connection_handler.accept_connections.side_effect = _accept + + monitor._adopt_harvested_monitoring_connection(None, (READER_A,)) + + assert order == ["shutdown", "accept_connections"] + executor.shutdown.assert_called_once_with(wait=True, cancel_futures=True) + + def test_no_selection_closes_all(self): + monitor = _bare_monitor() + c1, c2 = MagicMock(), MagicMock() + monitor._host_threads_connections = { + _host_and_port(READER_A): (READER_A, ThreadSafeConnectionHolder(c1)), + _host_and_port(READER_B): (READER_B, ThreadSafeConnectionHolder(c2)), + } + monitor._connection_handler.accept_connections.return_value = None + + result = monitor._adopt_harvested_monitoring_connection(None, (READER_A, READER_B)) + + assert result is False + c1.close.assert_called_once() + c2.close.assert_called_once() + + +class TestHandOffConnection: + def test_moves_connection_into_map(self): + monitor = _bare_monitor() + conn = MagicMock() + + monitor._harvest_connection(READER_A, conn) + + key = _host_and_port(READER_A) + assert key in monitor._host_threads_connections + host, holder = monitor._host_threads_connections[key] + assert host == READER_A + assert holder.get() is conn + conn.close.assert_not_called() + + def test_replacing_existing_entry_closes_previous(self): + monitor = _bare_monitor() + old_conn, new_conn = MagicMock(name="old"), MagicMock(name="new") + monitor._harvest_connection(READER_A, old_conn) + + monitor._harvest_connection(READER_A, new_conn) + + old_conn.close.assert_called_once() + _, holder = monitor._host_threads_connections[_host_and_port(READER_A)] + assert holder.get() is new_conn + + +class TestReaderObservedWriterChange: + def _worker(self, monitor, *, some_regions_inaccessible: bool, + baseline_writer: Optional[HostInfo]) -> HostMonitor: + worker = HostMonitor.__new__(HostMonitor) + worker._monitor = monitor + worker._host_info = READER_A + worker._writer_host_info = baseline_writer + worker._some_regions_inaccessible = some_regions_inaccessible + worker._writer_changed = False + worker._connection_attempts = 0 + return worker + + def _topology_with_writer(self, writer: HostInfo) -> Tuple[HostInfo, ...]: + return (writer, READER_A, READER_B) + + def test_writer_change_signals_exit_when_regions_inaccessible(self, monkeypatch): + monitor = _bare_monitor() + monitor._update_topology_cache = MagicMock() # type: ignore[method-assign] + monitor._record_reader_topology = MagicMock() # type: ignore[method-assign] + worker = self._worker( + monitor, some_regions_inaccessible=True, baseline_writer=WRITER) + + conn = MagicMock() + monkeypatch.setattr( + monitor, "_query_for_topology", + lambda c: self._topology_with_writer(NEW_WRITER), raising=False) + + worker._reader_thread_fetch_topology(conn) + + # First observer wins the CAS and signals panic-mode exit. + assert monitor._reader_observed_writer_host_info.get() == NEW_WRITER + assert monitor._host_threads_stop.is_set() + assert worker._writer_changed is True + + def test_writer_change_does_not_signal_when_all_regions_accessible(self, monkeypatch): + monitor = _bare_monitor() + monitor._update_topology_cache = MagicMock() # type: ignore[method-assign] + monitor._record_reader_topology = MagicMock() # type: ignore[method-assign] + worker = self._worker( + monitor, some_regions_inaccessible=False, baseline_writer=WRITER) + + conn = MagicMock() + monkeypatch.setattr( + monitor, "_query_for_topology", + lambda c: self._topology_with_writer(NEW_WRITER), raising=False) + + worker._reader_thread_fetch_topology(conn) + + # Writer change is detected, but no panic-exit signal (defer to the + # standard node-thread-verifies-writer path). + assert worker._writer_changed is True + assert monitor._reader_observed_writer_host_info.get() is None + assert not monitor._host_threads_stop.is_set() + + def test_no_signal_when_writer_unchanged(self, monkeypatch): + monitor = _bare_monitor() + monitor._update_topology_cache = MagicMock() # type: ignore[method-assign] + monitor._record_reader_topology = MagicMock() # type: ignore[method-assign] + worker = self._worker( + monitor, some_regions_inaccessible=True, baseline_writer=WRITER) + + conn = MagicMock() + monkeypatch.setattr( + monitor, "_query_for_topology", + lambda c: self._topology_with_writer(WRITER), raising=False) + + worker._reader_thread_fetch_topology(conn) + + assert worker._writer_changed is False + assert monitor._reader_observed_writer_host_info.get() is None + assert not monitor._host_threads_stop.is_set() + + def test_only_first_observer_wins_cas(self, monkeypatch): + monitor = _bare_monitor() + monitor._update_topology_cache = MagicMock() # type: ignore[method-assign] + monitor._record_reader_topology = MagicMock() # type: ignore[method-assign] + # A different worker already recorded an observed writer change. + already = HostInfo("writer3.xyz.us-east-1.rds.amazonaws.com", 5432, HostRole.WRITER) + monitor._reader_observed_writer_host_info.set(already) + + worker = self._worker( + monitor, some_regions_inaccessible=True, baseline_writer=WRITER) + conn = MagicMock() + monkeypatch.setattr( + monitor, "_query_for_topology", + lambda c: self._topology_with_writer(NEW_WRITER), raising=False) + + worker._reader_thread_fetch_topology(conn) + + # CAS-from-None fails; the earlier observation is retained. + assert monitor._reader_observed_writer_host_info.get() == already + + +class TestCheckForStableReaderTopologies: + def _prime(self, monitor, hosts, *, monkeypatch, stored=None): + stored = hosts if stored is None else stored + monkeypatch.setattr(monitor, "_get_stored_hosts", lambda: stored, raising=False) + monitor._update_topology_cache = MagicMock() # type: ignore[method-assign] + + def test_waits_until_every_monitored_reader_completed_a_cycle(self, monkeypatch): + monitor = _bare_monitor() + hosts = (READER_A, READER_B) + self._prime(monitor, hosts, monkeypatch=monkeypatch) + # Only READER_A has completed a cycle. + monitor._completed_one_cycle = {_host_and_port(READER_A): True} + monitor._reader_topologies_by_id = {_host_and_port(READER_A): hosts} + + monitor._check_for_stable_reader_topologies() + + # Not stable yet; timer not started, no harvest. + assert monitor._stable_topologies_start_nano == 0 + monitor._connection_handler.accept_connections.assert_not_called() + + def test_resets_timer_when_reader_topologies_disagree(self, monkeypatch): + monitor = _bare_monitor() + hosts = (READER_A, READER_B) + self._prime(monitor, hosts, monkeypatch=monkeypatch) + monitor._completed_one_cycle = { + _host_and_port(READER_A): True, _host_and_port(READER_B): True} + # Two readers disagree on topology. + monitor._reader_topologies_by_id = { + _host_and_port(READER_A): (READER_A, READER_B), + _host_and_port(READER_B): (READER_A,), + } + monitor._stable_topologies_start_nano = 1 # pretend a timer was running + + monitor._check_for_stable_reader_topologies() + + assert monitor._stable_topologies_start_nano == 0 + monitor._connection_handler.accept_connections.assert_not_called() + + def test_starts_timer_when_agreement_first_reached(self, monkeypatch): + monitor = _bare_monitor() + hosts = (READER_A, READER_B) + self._prime(monitor, hosts, monkeypatch=monkeypatch) + monitor._completed_one_cycle = { + _host_and_port(READER_A): True, _host_and_port(READER_B): True} + monitor._reader_topologies_by_id = { + _host_and_port(READER_A): hosts, + _host_and_port(READER_B): hosts, + } + monitor.get_stable_topologies_duration_ns = lambda: 10 ** 12 # type: ignore[method-assign] + + monitor._check_for_stable_reader_topologies() + + # Timer started this cycle; duration not yet elapsed, so no harvest. + assert monitor._stable_topologies_start_nano != 0 + monitor._connection_handler.accept_connections.assert_not_called() + + def test_harvests_after_stability_duration_elapses(self, monkeypatch): + monitor = _bare_monitor() + hosts = (READER_A, READER_B) + self._prime(monitor, hosts, monkeypatch=monkeypatch) + monitor._completed_one_cycle = { + _host_and_port(READER_A): True, _host_and_port(READER_B): True} + monitor._reader_topologies_by_id = { + _host_and_port(READER_A): hosts, + _host_and_port(READER_B): hosts, + } + # Timer already started far enough in the past that stability elapsed. + monitor._stable_topologies_start_nano = 1 + monitor.get_stable_topologies_duration_ns = lambda: 0 # type: ignore[method-assign] + # A harvested reader connection is available and gets adopted. + holder = ThreadSafeConnectionHolder(MagicMock()) + monitor._host_threads_connections = {_host_and_port(READER_A): (READER_A, holder)} + monitor._connection_handler.accept_connections.return_value = READER_A + # No verified writer (it's in an inaccessible region). + monitor._is_verified_writer_connection = False + + monitor._check_for_stable_reader_topologies() + + # Topology cache updated with the agreed reader topology, and the + # harvest adopted a connection to exit panic mode. + monitor._update_topology_cache.assert_called_once_with(hosts) + monitor._connection_handler.accept_connections.assert_called_once() + assert monitor._is_verified_writer_connection is True + + def test_skips_harvest_when_monitoring_connection_already_present(self, monkeypatch): + monitor = _bare_monitor() + hosts = (READER_A, READER_B) + self._prime(monitor, hosts, monkeypatch=monkeypatch) + monitor._completed_one_cycle = { + _host_and_port(READER_A): True, _host_and_port(READER_B): True} + monitor._reader_topologies_by_id = { + _host_and_port(READER_A): hosts, + _host_and_port(READER_B): hosts, + } + monitor._stable_topologies_start_nano = 1 + monitor.get_stable_topologies_duration_ns = lambda: 0 # type: ignore[method-assign] + # Monitoring connection already established → no harvest needed. + monitor._monitoring_connection = ThreadSafeConnectionHolder(MagicMock()) + + monitor._check_for_stable_reader_topologies() + + monitor._connection_handler.accept_connections.assert_not_called() + + def test_no_stored_hosts_resets_timer(self, monkeypatch): + monitor = _bare_monitor() + self._prime(monitor, (), monkeypatch=monkeypatch, stored=()) + monitor._stable_topologies_start_nano = 123 + + monitor._check_for_stable_reader_topologies() + + assert monitor._stable_topologies_start_nano == 0 + + def test_stability_only_requires_monitored_hosts(self, monkeypatch): + """A GDB subclass filters out inaccessible hosts; those must not block + stability by appearing perpetually incomplete.""" + monitor = _bare_monitor( + accessible_filter=lambda hosts: tuple(h for h in hosts if h != READER_B)) + stored = (READER_A, READER_B) # READER_B is inaccessible + self._prime(monitor, stored, monkeypatch=monkeypatch, stored=stored) + # Only the monitored host (READER_A) completed a cycle. + monitor._completed_one_cycle = {_host_and_port(READER_A): True} + monitor._reader_topologies_by_id = {_host_and_port(READER_A): (READER_A,)} + monitor.get_stable_topologies_duration_ns = lambda: 10 ** 12 # type: ignore[method-assign] + + monitor._check_for_stable_reader_topologies() + + # READER_B never completed a cycle, but it's not monitored, so stability + # is not blocked: the timer starts. + assert monitor._stable_topologies_start_nano != 0 + + +class TestClearHostThreadsState: + def test_clears_all_item8_state(self): + monitor = _bare_monitor() + monitor._host_threads_connections = {"x": (READER_A, ThreadSafeConnectionHolder(None))} + monitor._reader_topologies_by_id = {"x": (READER_A,)} + monitor._completed_one_cycle = {"x": True} + monitor._stable_topologies_start_nano = 999 + monitor._reader_observed_writer_host_info.set(NEW_WRITER) + + monitor._clear_host_threads_state() + + assert monitor._host_threads_connections == {} + assert monitor._reader_topologies_by_id == {} + assert monitor._completed_one_cycle == {} + assert monitor._stable_topologies_start_nano == 0 + assert monitor._reader_observed_writer_host_info.get() is None + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/unit/test_gdb_failover_plugin.py b/tests/unit/test_gdb_failover_plugin.py new file mode 100644 index 000000000..00906c5bc --- /dev/null +++ b/tests/unit/test_gdb_failover_plugin.py @@ -0,0 +1,366 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from unittest.mock import MagicMock + +import psycopg +import pytest + +from aws_advanced_python_wrapper.errors import (AwsWrapperError, + FailoverFailedError, + FailoverSuccessError) +from aws_advanced_python_wrapper.gdb_failover_plugin import ( + GdbFailoverPlugin, GdbFailoverPluginFactory) +from aws_advanced_python_wrapper.hostinfo import HostInfo, HostRole +from aws_advanced_python_wrapper.utils.gdb_failover_mode import GdbFailoverMode +from aws_advanced_python_wrapper.utils.properties import (Properties, + WrapperProperties) +from aws_advanced_python_wrapper.utils.rds_url_type import RdsUrlType + +# Region-tagged endpoints used across the tests. +HOME_REGION = "us-west-1" +OUT_REGION = "us-east-1" + +WRITER_HOME = HostInfo("writer.cluster-xyz.us-west-1.rds.amazonaws.com", 5432, HostRole.WRITER) +READER_HOME = HostInfo("reader1.xyz.us-west-1.rds.amazonaws.com", 5432, HostRole.READER) +READER_OUT = HostInfo("reader2.xyz.us-east-1.rds.amazonaws.com", 5432, HostRole.READER) +WRITER_OUT = HostInfo("writer.cluster-xyz.us-east-1.rds.amazonaws.com", 5432, HostRole.WRITER) + + +@pytest.fixture +def plugin_service_mock(): + mock = MagicMock() + mock.network_bound_methods = {"*"} + mock.current_host_info = WRITER_HOME + mock.current_connection = MagicMock(spec=psycopg.Connection) + mock.driver_dialect.network_bound_methods = {"Connection.execute", "Connection.commit"} + mock.driver_dialect.is_closed.return_value = False + mock.is_network_exception.return_value = True + mock.is_in_transaction = False + mock.hosts = [WRITER_HOME, READER_HOME, READER_OUT] + mock.all_hosts = mock.hosts + mock.get_telemetry_factory.return_value.open_telemetry_context.return_value = None + return mock + + +@pytest.fixture +def properties(): + props = Properties() + WrapperProperties.FAILOVER_TIMEOUT_SEC.set(props, "60") + WrapperProperties.TELEMETRY_FAILOVER_ADDITIONAL_TOP_TRACE.set(props, "false") + return props + + +@pytest.fixture +def gdb_plugin(plugin_service_mock, properties): + return GdbFailoverPlugin(plugin_service_mock, properties) + + +def _host_list_provider_with_host(host: str): + hlps = MagicMock() + hlps.initial_connection_host_info = HostInfo(host, 5432, HostRole.WRITER) + return hlps + + +class TestGlobalDbFailoverMode: + def test_from_value_hyphenated(self): + assert GdbFailoverMode.from_value("strict-writer") == GdbFailoverMode.STRICT_WRITER + assert GdbFailoverMode.from_value("home-reader-or-writer") == GdbFailoverMode.HOME_READER_OR_WRITER + + def test_from_value_underscored(self): + assert GdbFailoverMode.from_value("strict_any_reader") == GdbFailoverMode.STRICT_ANY_READER + + def test_from_value_case_insensitive(self): + assert GdbFailoverMode.from_value("STRICT-WRITER") == GdbFailoverMode.STRICT_WRITER + + def test_from_value_none_or_empty(self): + assert GdbFailoverMode.from_value(None) is None + assert GdbFailoverMode.from_value(" ") is None + + def test_from_value_invalid(self): + with pytest.raises(AwsWrapperError): + GdbFailoverMode.from_value("bogus-mode") + + +class TestGdbFailoverPluginFactory: + def test_factory_returns_plugin(self, plugin_service_mock, properties): + plugin = GdbFailoverPluginFactory.get_instance(plugin_service_mock, properties) + assert isinstance(plugin, GdbFailoverPlugin) + + +class TestGdbFailoverInitMode: + def test_init_mode_uses_explicit_home_region_and_modes(self, gdb_plugin, properties): + WrapperProperties.FAILOVER_HOME_REGION.set(properties, HOME_REGION) + WrapperProperties.ACTIVE_HOME_FAILOVER_MODE.set(properties, "strict-writer") + WrapperProperties.INACTIVE_HOME_FAILOVER_MODE.set(properties, "strict-any-reader") + gdb_plugin._host_list_provider_service = _host_list_provider_with_host( + "gdb.global-xyz.global.rds.amazonaws.com") + + gdb_plugin._init_failover_mode() + + assert gdb_plugin._home_region == HOME_REGION + assert gdb_plugin._active_home_failover_mode == GdbFailoverMode.STRICT_WRITER + assert gdb_plugin._inactive_home_failover_mode == GdbFailoverMode.STRICT_ANY_READER + + def test_init_mode_parses_region_from_endpoint(self, gdb_plugin): + gdb_plugin._host_list_provider_service = _host_list_provider_with_host(WRITER_HOME.host) + + gdb_plugin._init_failover_mode() + + assert gdb_plugin._home_region == "us-west-1" + # Writer cluster endpoint default is STRICT_WRITER. + assert gdb_plugin._active_home_failover_mode == GdbFailoverMode.STRICT_WRITER + assert gdb_plugin._inactive_home_failover_mode == GdbFailoverMode.STRICT_WRITER + + def test_init_mode_default_for_reader_endpoint(self, gdb_plugin): + reader_cluster = "db.cluster-ro-xyz.us-west-1.rds.amazonaws.com" + gdb_plugin._host_list_provider_service = _host_list_provider_with_host(reader_cluster) + + gdb_plugin._init_failover_mode() + + assert gdb_plugin._active_home_failover_mode == GdbFailoverMode.HOME_READER_OR_WRITER + assert gdb_plugin._inactive_home_failover_mode == GdbFailoverMode.HOME_READER_OR_WRITER + + def test_init_mode_missing_region_raises(self, gdb_plugin): + # IP address endpoint has no region and no home region property set. + gdb_plugin._host_list_provider_service = _host_list_provider_with_host("10.0.0.1") + + with pytest.raises(AwsWrapperError): + gdb_plugin._init_failover_mode() + + def test_init_mode_idempotent(self, gdb_plugin): + gdb_plugin._host_list_provider_service = _host_list_provider_with_host(WRITER_HOME.host) + gdb_plugin._init_failover_mode() + gdb_plugin._rds_url_type = RdsUrlType.RDS_READER_CLUSTER # sentinel + gdb_plugin._init_failover_mode() + # Second call returns early without recomputing. + assert gdb_plugin._rds_url_type == RdsUrlType.RDS_READER_CLUSTER + + +class TestGdbFailover: + def test_failover_refresh_failed(self, gdb_plugin): + gdb_plugin._home_region = HOME_REGION + gdb_plugin._active_home_failover_mode = GdbFailoverMode.STRICT_WRITER + gdb_plugin._inactive_home_failover_mode = GdbFailoverMode.STRICT_WRITER + gdb_plugin._plugin_service.force_monitoring_refresh_host_list.return_value = False + + with pytest.raises(FailoverFailedError): + gdb_plugin._failover() + + def test_failover_no_writer_found(self, gdb_plugin): + gdb_plugin._home_region = HOME_REGION + gdb_plugin._active_home_failover_mode = GdbFailoverMode.STRICT_WRITER + gdb_plugin._inactive_home_failover_mode = GdbFailoverMode.STRICT_WRITER + gdb_plugin._plugin_service.force_monitoring_refresh_host_list.return_value = True + gdb_plugin._plugin_service.all_hosts = [READER_HOME, READER_OUT] + + with pytest.raises(FailoverFailedError): + gdb_plugin._failover() + + def test_failover_in_home_uses_active_mode(self, gdb_plugin): + gdb_plugin._home_region = HOME_REGION + gdb_plugin._active_home_failover_mode = GdbFailoverMode.STRICT_WRITER + gdb_plugin._inactive_home_failover_mode = GdbFailoverMode.STRICT_ANY_READER + gdb_plugin._plugin_service.force_monitoring_refresh_host_list.return_value = True + gdb_plugin._plugin_service.all_hosts = [WRITER_HOME, READER_HOME, READER_OUT] + captured = {} + + def fake_failover_with_mode(mode, writer_candidate, end_time): + captured["mode"] = mode + + gdb_plugin._failover_with_mode = MagicMock(side_effect=fake_failover_with_mode) + gdb_plugin._throw_failover_success_exception = MagicMock(side_effect=FailoverSuccessError()) + + with pytest.raises(FailoverSuccessError): + gdb_plugin._failover() + + assert captured["mode"] == GdbFailoverMode.STRICT_WRITER + + def test_failover_out_of_home_uses_inactive_mode(self, gdb_plugin): + gdb_plugin._home_region = HOME_REGION + gdb_plugin._active_home_failover_mode = GdbFailoverMode.STRICT_WRITER + gdb_plugin._inactive_home_failover_mode = GdbFailoverMode.STRICT_ANY_READER + gdb_plugin._plugin_service.force_monitoring_refresh_host_list.return_value = True + # New writer is in the out-of-home region. + gdb_plugin._plugin_service.all_hosts = [WRITER_OUT, READER_HOME, READER_OUT] + captured = {} + + gdb_plugin._failover_with_mode = MagicMock(side_effect=lambda mode, w, e: captured.update(mode=mode)) + gdb_plugin._throw_failover_success_exception = MagicMock(side_effect=FailoverSuccessError()) + + with pytest.raises(FailoverSuccessError): + gdb_plugin._failover() + + assert captured["mode"] == GdbFailoverMode.STRICT_ANY_READER + + +class TestGdbFailoverWithMode: + def test_strict_writer_dispatch(self, gdb_plugin): + gdb_plugin._failover_to_writer = MagicMock() + gdb_plugin._failover_with_mode(GdbFailoverMode.STRICT_WRITER, WRITER_HOME, 0.0) + gdb_plugin._failover_to_writer.assert_called_once() + + def test_strict_home_reader_filters_home_readers(self, gdb_plugin): + gdb_plugin._home_region = HOME_REGION + gdb_plugin._plugin_service.hosts = [WRITER_HOME, READER_HOME, READER_OUT] + captured = {} + gdb_plugin._failover_to_allowed_host = MagicMock( + side_effect=lambda supplier, role, end: captured.update(hosts=supplier(), role=role)) + + gdb_plugin._failover_with_mode(GdbFailoverMode.STRICT_HOME_READER, WRITER_HOME, 0.0) + + assert captured["role"] == HostRole.READER + assert captured["hosts"] == [READER_HOME] + + def test_strict_out_of_home_reader_filters_out_readers(self, gdb_plugin): + gdb_plugin._home_region = HOME_REGION + gdb_plugin._plugin_service.hosts = [WRITER_HOME, READER_HOME, READER_OUT] + captured = {} + gdb_plugin._failover_to_allowed_host = MagicMock( + side_effect=lambda supplier, role, end: captured.update(hosts=supplier(), role=role)) + + gdb_plugin._failover_with_mode(GdbFailoverMode.STRICT_OUT_OF_HOME_READER, WRITER_HOME, 0.0) + + assert captured["role"] == HostRole.READER + assert captured["hosts"] == [READER_OUT] + + def test_strict_any_reader_selects_all_readers(self, gdb_plugin): + gdb_plugin._home_region = HOME_REGION + gdb_plugin._plugin_service.hosts = [WRITER_HOME, READER_HOME, READER_OUT] + captured = {} + gdb_plugin._failover_to_allowed_host = MagicMock( + side_effect=lambda supplier, role, end: captured.update(hosts=supplier(), role=role)) + + gdb_plugin._failover_with_mode(GdbFailoverMode.STRICT_ANY_READER, WRITER_HOME, 0.0) + + assert captured["role"] == HostRole.READER + assert READER_HOME in captured["hosts"] + assert READER_OUT in captured["hosts"] + assert len(captured["hosts"]) == 2 + + def test_home_reader_or_writer_includes_writer_and_home_readers(self, gdb_plugin): + gdb_plugin._home_region = HOME_REGION + gdb_plugin._plugin_service.hosts = [WRITER_OUT, READER_HOME, READER_OUT] + captured = {} + gdb_plugin._failover_to_allowed_host = MagicMock( + side_effect=lambda supplier, role, end: captured.update(hosts=supplier(), role=role)) + + gdb_plugin._failover_with_mode(GdbFailoverMode.HOME_READER_OR_WRITER, WRITER_OUT, 0.0) + + assert captured["role"] is None + assert WRITER_OUT in captured["hosts"] + assert READER_HOME in captured["hosts"] + assert READER_OUT not in captured["hosts"] + assert len(captured["hosts"]) == 2 + + def test_any_reader_or_writer_includes_all(self, gdb_plugin): + gdb_plugin._home_region = HOME_REGION + gdb_plugin._plugin_service.hosts = [WRITER_HOME, READER_HOME, READER_OUT] + captured = {} + gdb_plugin._failover_to_allowed_host = MagicMock( + side_effect=lambda supplier, role, end: captured.update(hosts=supplier(), role=role)) + + gdb_plugin._failover_with_mode(GdbFailoverMode.ANY_READER_OR_WRITER, WRITER_HOME, 0.0) + + assert captured["role"] is None + assert len(captured["hosts"]) == 3 + for host in (WRITER_HOME, READER_HOME, READER_OUT): + assert host in captured["hosts"] + + +class TestGdbFailoverUnsupportedMethods: + def test_failover_reader_unsupported(self, gdb_plugin): + with pytest.raises(AwsWrapperError): + gdb_plugin._failover_reader() + + def test_failover_writer_unsupported(self, gdb_plugin): + with pytest.raises(AwsWrapperError): + gdb_plugin._failover_writer() + + +class TestGdbFailoverAccessibleRegions: + def test_init_mode_home_region_not_accessible_raises(self, gdb_plugin, properties): + WrapperProperties.GDB_ACCESSIBLE_REGIONS.set(properties, OUT_REGION) + gdb_plugin._host_list_provider_service = _host_list_provider_with_host(WRITER_HOME.host) + + with pytest.raises(AwsWrapperError): + gdb_plugin._init_failover_mode() + + def test_init_mode_home_region_accessible_ok(self, gdb_plugin, properties): + WrapperProperties.GDB_ACCESSIBLE_REGIONS.set(properties, f"{HOME_REGION},{OUT_REGION}") + gdb_plugin._host_list_provider_service = _host_list_provider_with_host(WRITER_HOME.host) + + gdb_plugin._init_failover_mode() + + assert gdb_plugin._home_region == HOME_REGION + assert gdb_plugin._accessible_regions == frozenset({HOME_REGION, OUT_REGION}) + + def test_is_in_accessible_region_no_restriction_allows_all(self, gdb_plugin): + gdb_plugin._accessible_regions = None + assert gdb_plugin._is_in_accessible_region(WRITER_OUT) is True + + def test_is_in_accessible_region_filters(self, gdb_plugin): + gdb_plugin._accessible_regions = frozenset({HOME_REGION}) + assert gdb_plugin._is_in_accessible_region(WRITER_HOME) is True + assert gdb_plugin._is_in_accessible_region(READER_OUT) is False + + def test_strict_writer_inaccessible_region_raises_and_counts(self, gdb_plugin): + gdb_plugin._accessible_regions = frozenset({HOME_REGION}) + gdb_plugin._failover_to_writer = MagicMock() + gdb_plugin._failover_writer_triggered_counter = MagicMock() + gdb_plugin._failover_writer_failed_counter = MagicMock() + + with pytest.raises(FailoverFailedError): + gdb_plugin._failover_with_mode(GdbFailoverMode.STRICT_WRITER, WRITER_OUT, 0.0) + + gdb_plugin._failover_to_writer.assert_not_called() + gdb_plugin._failover_writer_triggered_counter.inc.assert_called_once() + gdb_plugin._failover_writer_failed_counter.inc.assert_called_once() + + def test_strict_writer_accessible_region_dispatches(self, gdb_plugin): + gdb_plugin._accessible_regions = frozenset({HOME_REGION}) + gdb_plugin._failover_to_writer = MagicMock() + + gdb_plugin._failover_with_mode(GdbFailoverMode.STRICT_WRITER, WRITER_HOME, 0.0) + + gdb_plugin._failover_to_writer.assert_called_once() + + def test_strict_any_reader_excludes_inaccessible(self, gdb_plugin): + gdb_plugin._home_region = HOME_REGION + gdb_plugin._accessible_regions = frozenset({HOME_REGION}) + gdb_plugin._plugin_service.hosts = [WRITER_HOME, READER_HOME, READER_OUT] + captured = {} + gdb_plugin._failover_to_allowed_host = MagicMock( + side_effect=lambda supplier, role, end: captured.update(hosts=supplier(), role=role)) + + gdb_plugin._failover_with_mode(GdbFailoverMode.STRICT_ANY_READER, WRITER_HOME, 0.0) + + assert captured["hosts"] == [READER_HOME] + + def test_any_reader_or_writer_excludes_inaccessible(self, gdb_plugin): + gdb_plugin._home_region = HOME_REGION + gdb_plugin._accessible_regions = frozenset({HOME_REGION}) + gdb_plugin._plugin_service.hosts = [WRITER_HOME, READER_HOME, READER_OUT, WRITER_OUT] + captured = {} + gdb_plugin._failover_to_allowed_host = MagicMock( + side_effect=lambda supplier, role, end: captured.update(hosts=supplier(), role=role)) + + gdb_plugin._failover_with_mode(GdbFailoverMode.ANY_READER_OR_WRITER, WRITER_HOME, 0.0) + + assert WRITER_HOME in captured["hosts"] + assert READER_HOME in captured["hosts"] + assert READER_OUT not in captured["hosts"] + assert WRITER_OUT not in captured["hosts"] diff --git a/tests/unit/test_gdb_monitoring_connection_priority.py b/tests/unit/test_gdb_monitoring_connection_priority.py new file mode 100644 index 000000000..1222978a8 --- /dev/null +++ b/tests/unit/test_gdb_monitoring_connection_priority.py @@ -0,0 +1,187 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from aws_advanced_python_wrapper.hostinfo import HostInfo, HostRole +from aws_advanced_python_wrapper.utils.gdb_monitoring_connection_priority import \ + GdbMonitoringConnectionPriority as Priority +from aws_advanced_python_wrapper.utils.rds_utils import RdsUtils + +rds_utils = RdsUtils() + +WRITER_EAST = HostInfo("instance1.cluster-xyz.us-east-1.rds.amazonaws.com", 5432, HostRole.WRITER) +READER_EAST = HostInfo("instance2.cluster-ro-xyz.us-east-1.rds.amazonaws.com", 5432, HostRole.READER) +READER_WEST = HostInfo("instance3.cluster-ro-xyz.us-west-2.rds.amazonaws.com", 5432, HostRole.READER) +WRITER_WEST = HostInfo("instance1.cluster-xyz.us-west-2.rds.amazonaws.com", 5432, HostRole.WRITER) + + +class TestFromValue: + def test_strict_writer_primary(self): + p = Priority.from_value("strict-writer-primary") + assert p is not None + assert p.required_role is HostRole.WRITER + assert p.required_region is None + assert p.require_primary is True + assert p.require_secondary is False + + def test_strict_reader_primary(self): + p = Priority.from_value("strict-reader-primary") + assert p is not None + assert p.required_role is HostRole.READER + assert p.required_region is None + assert p.require_primary is True + assert p.require_secondary is False + + def test_strict_reader_secondary(self): + p = Priority.from_value("strict-reader-secondary") + assert p is not None + assert p.required_role is HostRole.READER + assert p.required_region is None + assert p.require_primary is False + assert p.require_secondary is True + + def test_strict_writer_region(self): + p = Priority.from_value("strict-writer-us-east-1") + assert p is not None + assert p.required_role is HostRole.WRITER + assert p.required_region == "us-east-1" + assert p.require_primary is False + assert p.require_secondary is False + + def test_strict_reader_region(self): + p = Priority.from_value("strict-reader-us-west-2") + assert p is not None + assert p.required_role is HostRole.READER + assert p.required_region == "us-west-2" + + def test_plain_region(self): + p = Priority.from_value("us-east-1") + assert p is not None + assert p.required_role is None + assert p.required_region == "us-east-1" + assert p.require_primary is False + assert p.require_secondary is False + + def test_null_and_empty(self): + assert Priority.from_value(None) is None + assert Priority.from_value("") is None + assert Priority.from_value(" ") is None + + def test_invalid_prefix_no_suffix(self): + assert Priority.from_value("strict-writer-") is None + assert Priority.from_value("strict-reader-") is None + + def test_strict_writer_secondary_rejected(self): + assert Priority.from_value("strict-writer-secondary") is None + + def test_typo_logs_and_becomes_region_literal(self, mocker): + debug = mocker.patch( + "aws_advanced_python_wrapper.utils.gdb_monitoring_connection_priority.logger.debug") + # A typo does not look like an AWS region, so it logs a debug message + # but is still accepted as a (never-matching) region literal. + p = Priority.from_value("strict-wrtier-primary") + assert p is not None + assert p.required_role is None + assert p.required_region == "strict-wrtier-primary" + debug.assert_called_once_with( + "GdbMonitoringConnectionHandler.UnrecognizedPriority", "strict-wrtier-primary") + + def test_valid_region_does_not_log(self, mocker): + debug = mocker.patch( + "aws_advanced_python_wrapper.utils.gdb_monitoring_connection_priority.logger.debug") + assert Priority.from_value("us-east-1") is not None + debug.assert_not_called() + + def test_prefixed_region_does_not_log(self, mocker): + debug = mocker.patch( + "aws_advanced_python_wrapper.utils.gdb_monitoring_connection_priority.logger.debug") + # strict-writer- / strict-reader- take the prefix path + # and never reach the region-literal fallback, so nothing is logged. + assert Priority.from_value("strict-writer-us-east-1") is not None + assert Priority.from_value("strict-reader-us-west-2") is not None + debug.assert_not_called() + + +class TestParseList: + def test_default(self): + result = Priority.parse_list(None) + assert len(result) == 1 + assert str(result[0]) == "strict-writer-primary" + + def test_multiple_values(self): + result = Priority.parse_list("strict-writer-primary,strict-reader-us-east-1,us-west-2") + assert [str(p) for p in result] == [ + "strict-writer-primary", "strict-reader-us-east-1", "us-west-2"] + + def test_skips_invalid(self): + result = Priority.parse_list("strict-writer-,strict-reader-primary") + assert len(result) == 1 + assert str(result[0]) == "strict-reader-primary" + + +class TestIsSatisfiedBy: + def test_writer_in_primary_region(self): + p = Priority.from_value("strict-writer-primary") + assert p.is_satisfied_by(WRITER_EAST, "us-east-1", rds_utils) is True + assert p.is_satisfied_by(WRITER_EAST, "us-west-2", rds_utils) is False + + def test_reader_in_primary_region(self): + p = Priority.from_value("strict-reader-primary") + assert p.is_satisfied_by(READER_EAST, "us-east-1", rds_utils) is True + assert p.is_satisfied_by(READER_EAST, "us-west-2", rds_utils) is False + + def test_reader_in_secondary_region(self): + p = Priority.from_value("strict-reader-secondary") + # Primary is us-east-1, so us-west-2 reader is secondary. + assert p.is_satisfied_by(READER_WEST, "us-east-1", rds_utils) is True + # Primary is us-west-2, so us-west-2 reader is NOT secondary. + assert p.is_satisfied_by(READER_WEST, "us-west-2", rds_utils) is False + + def test_writer_in_specific_region(self): + p = Priority.from_value("strict-writer-us-east-1") + assert p.is_satisfied_by(WRITER_EAST, "us-east-1", rds_utils) is True + assert p.is_satisfied_by(WRITER_WEST, "us-east-1", rds_utils) is False + + def test_reader_in_specific_region(self): + p = Priority.from_value("strict-reader-us-west-2") + assert p.is_satisfied_by(READER_WEST, "us-east-1", rds_utils) is True + assert p.is_satisfied_by(READER_EAST, "us-east-1", rds_utils) is False + + def test_any_node_in_region(self): + p = Priority.from_value("us-east-1") + assert p.is_satisfied_by(WRITER_EAST, "us-east-1", rds_utils) is True + assert p.is_satisfied_by(READER_EAST, "us-east-1", rds_utils) is True + assert p.is_satisfied_by(WRITER_WEST, "us-east-1", rds_utils) is False + + def test_role_check_rejects_wrong_role(self): + p = Priority.from_value("strict-writer-primary") + assert p.is_satisfied_by(READER_EAST, "us-east-1", rds_utils) is False + + +class TestFindMatchingHost: + def test_finds_first_match(self): + p = Priority.from_value("strict-reader-us-west-2") + hosts = [WRITER_EAST, READER_EAST, READER_WEST] + assert p.find_matching_host(hosts, "us-east-1", rds_utils) == READER_WEST + + def test_returns_none_when_no_match(self): + p = Priority.from_value("strict-writer-us-west-2") + hosts = [WRITER_EAST, READER_EAST] + assert p.find_matching_host(hosts, "us-east-1", rds_utils) is None + + def test_find_matching_hosts_returns_all(self): + p = Priority.from_value("us-east-1") + hosts = [WRITER_EAST, READER_EAST, READER_WEST] + assert p.find_matching_hosts(hosts, "us-east-1", rds_utils) == [WRITER_EAST, READER_EAST] diff --git a/tests/unit/test_gdb_read_write_splitting_plugin.py b/tests/unit/test_gdb_read_write_splitting_plugin.py new file mode 100644 index 000000000..9204ca260 --- /dev/null +++ b/tests/unit/test_gdb_read_write_splitting_plugin.py @@ -0,0 +1,174 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from unittest.mock import MagicMock + +import psycopg +import pytest + +from aws_advanced_python_wrapper.errors import ReadWriteSplittingError +from aws_advanced_python_wrapper.gdb_read_write_splitting_plugin import \ + GdbReadWriteSplittingPlugin +from aws_advanced_python_wrapper.hostinfo import HostInfo, HostRole +from aws_advanced_python_wrapper.utils.properties import (Properties, + WrapperProperties) + +HOME_REGION = "us-west-1" +OUT_REGION = "us-east-1" + +WRITER_HOME = HostInfo("writer.cluster-xyz.us-west-1.rds.amazonaws.com", 5432, HostRole.WRITER) +READER_HOME = HostInfo("reader1.xyz.us-west-1.rds.amazonaws.com", 5432, HostRole.READER) +READER_OUT = HostInfo("reader2.xyz.us-east-1.rds.amazonaws.com", 5432, HostRole.READER) +WRITER_OUT = HostInfo("writer.cluster-xyz.us-east-1.rds.amazonaws.com", 5432, HostRole.WRITER) + + +@pytest.fixture +def plugin_service_mock(): + mock = MagicMock() + mock.hosts = [WRITER_HOME, READER_HOME, READER_OUT] + mock.current_host_info = WRITER_HOME + mock.current_connection = MagicMock(spec=psycopg.Connection) + mock.is_in_transaction = False + return mock + + +@pytest.fixture +def props(): + return Properties() + + +@pytest.fixture +def gdb_rw_plugin(plugin_service_mock, props): + plugin = GdbReadWriteSplittingPlugin(plugin_service_mock, props) + # Baseline: home region set, no accessible-region restriction, no + # home-region reader/writer restriction unless a test enables it. + plugin._home_region = HOME_REGION + return plugin + + +class TestInitSettingsAccessibleRegions: + def test_home_region_not_accessible_raises(self, plugin_service_mock): + props = Properties() + WrapperProperties.GDB_RW_HOME_REGION.set(props, HOME_REGION) + WrapperProperties.GDB_ACCESSIBLE_REGIONS.set(props, OUT_REGION) + plugin = GdbReadWriteSplittingPlugin(plugin_service_mock, props) + + with pytest.raises(ReadWriteSplittingError): + plugin._init_settings(WRITER_HOME, props) + + def test_home_region_accessible_ok(self, plugin_service_mock): + props = Properties() + WrapperProperties.GDB_RW_HOME_REGION.set(props, HOME_REGION) + WrapperProperties.GDB_ACCESSIBLE_REGIONS.set(props, f"{HOME_REGION},{OUT_REGION}") + plugin = GdbReadWriteSplittingPlugin(plugin_service_mock, props) + + plugin._init_settings(WRITER_HOME, props) + + assert plugin._accessible_regions == frozenset({HOME_REGION, OUT_REGION}) + + +class TestIsInAccessibleRegion: + def test_no_restriction_allows_all(self, gdb_rw_plugin): + gdb_rw_plugin._accessible_regions = None + assert gdb_rw_plugin._is_in_accessible_region(WRITER_OUT) is True + + def test_filters_by_region(self, gdb_rw_plugin): + gdb_rw_plugin._accessible_regions = frozenset({HOME_REGION}) + assert gdb_rw_plugin._is_in_accessible_region(WRITER_HOME) is True + assert gdb_rw_plugin._is_in_accessible_region(READER_OUT) is False + + +class TestInitializeWriterConnection: + def test_inaccessible_writer_raises(self, gdb_rw_plugin): + gdb_rw_plugin._accessible_regions = frozenset({HOME_REGION}) + gdb_rw_plugin._get_writer_host_info = MagicMock(return_value=WRITER_OUT) + + with pytest.raises(ReadWriteSplittingError): + gdb_rw_plugin._initialize_writer_connection() + + def test_accessible_writer_delegates_to_super(self, gdb_rw_plugin, monkeypatch): + gdb_rw_plugin._accessible_regions = frozenset({HOME_REGION}) + gdb_rw_plugin._restrict_writer_to_home_region = False + gdb_rw_plugin._get_writer_host_info = MagicMock(return_value=WRITER_HOME) + called = {} + monkeypatch.setattr( + "aws_advanced_python_wrapper.read_write_splitting_plugin." + "ReadWriteSplittingPlugin._initialize_writer_connection", + lambda self: called.setdefault("super", True), + ) + + gdb_rw_plugin._initialize_writer_connection() + + assert called.get("super") is True + + +class TestSetWriterConnection: + def test_inaccessible_writer_closes_and_raises(self, gdb_rw_plugin): + gdb_rw_plugin._accessible_regions = frozenset({HOME_REGION}) + gdb_rw_plugin._close_connection = MagicMock() + conn = MagicMock(spec=psycopg.Connection) + + with pytest.raises(ReadWriteSplittingError): + gdb_rw_plugin._set_writer_connection(conn, WRITER_OUT) + + gdb_rw_plugin._close_connection.assert_called_once_with(conn) + + +class TestGetReaderHostCandidates: + def test_filters_out_inaccessible_readers(self, gdb_rw_plugin): + gdb_rw_plugin._accessible_regions = frozenset({HOME_REGION}) + gdb_rw_plugin._restrict_reader_to_home_region = False + + candidates = gdb_rw_plugin._get_reader_host_candidates() + + assert WRITER_HOME in candidates + assert READER_HOME in candidates + assert READER_OUT not in candidates + + def test_no_fallback_when_all_readers_inaccessible(self, gdb_rw_plugin): + # Hard restriction: no fallback to the unfiltered host list. + gdb_rw_plugin._accessible_regions = frozenset({"eu-central-1"}) + gdb_rw_plugin._restrict_reader_to_home_region = False + + candidates = gdb_rw_plugin._get_reader_host_candidates() + + assert candidates == [] + + def test_unrestricted_returns_all_hosts(self, gdb_rw_plugin): + gdb_rw_plugin._accessible_regions = None + gdb_rw_plugin._restrict_reader_to_home_region = False + + candidates = gdb_rw_plugin._get_reader_host_candidates() + + assert candidates == [WRITER_HOME, READER_HOME, READER_OUT] + + def test_home_region_restriction_applied_after_accessible_filter(self, gdb_rw_plugin): + gdb_rw_plugin._accessible_regions = frozenset({HOME_REGION, OUT_REGION}) + gdb_rw_plugin._restrict_reader_to_home_region = True + + candidates = gdb_rw_plugin._get_reader_host_candidates() + + # READER_OUT is accessible but not in the home region, so it is dropped + # by the home-region restriction that runs on the post-filter set. + assert READER_HOME in candidates + assert READER_OUT not in candidates + + def test_home_region_restriction_no_home_readers_raises(self, gdb_rw_plugin): + gdb_rw_plugin._accessible_regions = frozenset({OUT_REGION}) + gdb_rw_plugin._restrict_reader_to_home_region = True + + with pytest.raises(ReadWriteSplittingError): + gdb_rw_plugin._get_reader_host_candidates() diff --git a/tests/unit/test_global_aurora_topology_monitor_accessible_regions.py b/tests/unit/test_global_aurora_topology_monitor_accessible_regions.py new file mode 100644 index 000000000..f95e693fc --- /dev/null +++ b/tests/unit/test_global_aurora_topology_monitor_accessible_regions.py @@ -0,0 +1,83 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import pytest + +from aws_advanced_python_wrapper.cluster_topology_monitor import \ + GlobalAuroraTopologyMonitor +from aws_advanced_python_wrapper.errors import AwsWrapperError +from aws_advanced_python_wrapper.hostinfo import HostInfo, HostRole +from aws_advanced_python_wrapper.utils.rds_utils import RdsUtils + +WRITER_HOME = HostInfo("writer.cluster-xyz.us-west-1.rds.amazonaws.com", 5432, HostRole.WRITER) +READER_HOME = HostInfo("reader1.xyz.us-west-1.rds.amazonaws.com", 5432, HostRole.READER) +READER_OUT = HostInfo("reader2.xyz.us-east-1.rds.amazonaws.com", 5432, HostRole.READER) + + +def _bare_monitor(accessible_regions, initial_host=WRITER_HOME): + # The base ClusterTopologyMonitorImpl constructor spins up background + # threads; build a bare instance and set only the attributes the + # accessible-regions methods touch. + monitor = GlobalAuroraTopologyMonitor.__new__(GlobalAuroraTopologyMonitor) + monitor._accessible_regions = accessible_regions + monitor._rds_utils = RdsUtils() + monitor._initial_host_info = initial_host + return monitor + + +class TestFilterHostsForNodeMonitoring: + def test_no_restriction_returns_all(self): + monitor = _bare_monitor(None) + hosts = (WRITER_HOME, READER_HOME, READER_OUT) + assert monitor._filter_hosts_for_host_monitoring(hosts) is hosts + + def test_filters_inaccessible_hosts(self): + monitor = _bare_monitor(frozenset({"us-west-1"})) + hosts = (WRITER_HOME, READER_HOME, READER_OUT) + + filtered = monitor._filter_hosts_for_host_monitoring(hosts) + + assert filtered == (WRITER_HOME, READER_HOME) + + +class TestOpenAnyConnectionInitialHostValidation: + def test_initial_host_inaccessible_raises(self): + monitor = _bare_monitor(frozenset({"us-west-1"}), initial_host=READER_OUT) + + with pytest.raises(AwsWrapperError): + monitor._open_any_connection_and_update_topology() + + def test_initial_host_accessible_delegates_to_super(self, monkeypatch): + monitor = _bare_monitor(frozenset({"us-west-1"}), initial_host=WRITER_HOME) + sentinel = (WRITER_HOME,) + monkeypatch.setattr( + "aws_advanced_python_wrapper.cluster_topology_monitor." + "ClusterTopologyMonitorImpl._open_any_connection_and_update_topology", + lambda self: sentinel, + ) + + assert monitor._open_any_connection_and_update_topology() is sentinel + + def test_no_restriction_delegates_to_super(self, monkeypatch): + monitor = _bare_monitor(None, initial_host=READER_OUT) + sentinel = (READER_OUT,) + monkeypatch.setattr( + "aws_advanced_python_wrapper.cluster_topology_monitor." + "ClusterTopologyMonitorImpl._open_any_connection_and_update_topology", + lambda self: sentinel, + ) + + assert monitor._open_any_connection_and_update_topology() is sentinel diff --git a/tests/unit/test_monitoring_connection_handler.py b/tests/unit/test_monitoring_connection_handler.py new file mode 100644 index 000000000..4e291e388 --- /dev/null +++ b/tests/unit/test_monitoring_connection_handler.py @@ -0,0 +1,156 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from unittest.mock import MagicMock + +from aws_advanced_python_wrapper.concrete_monitoring_connection_handlers import ( + AuroraMonitoringConnectionHandler, GdbMonitoringConnectionHandler) +from aws_advanced_python_wrapper.hostinfo import HostInfo, HostRole +from aws_advanced_python_wrapper.utils.atomic import AtomicReference +from aws_advanced_python_wrapper.utils.properties import (Properties, + WrapperProperties) +from aws_advanced_python_wrapper.utils.thread_safe_connection_holder import \ + ThreadSafeConnectionHolder + +WRITER_EAST = HostInfo("instance1.cluster-xyz.us-east-1.rds.amazonaws.com", 5432, HostRole.WRITER) +READER_EAST = HostInfo("instance2.cluster-ro-xyz.us-east-1.rds.amazonaws.com", 5432, HostRole.READER) +READER_WEST = HostInfo("instance3.cluster-ro-xyz.us-west-2.rds.amazonaws.com", 5432, HostRole.READER) + + +def _make_aurora_handler(priority_value=None): + props = Properties() + if priority_value is not None: + WrapperProperties.MONITORING_CONNECTION_PRIORITY.set(props, priority_value) + monitoring_conn = ThreadSafeConnectionHolder(None) + return AuroraMonitoringConnectionHandler( + monitoring_conn, MagicMock(), MagicMock(), props, Properties()), monitoring_conn + + +def _make_gdb_handler(priority_value, writer_ref): + props = Properties() + WrapperProperties.GDB_MONITORING_CONNECTION_PRIORITY.set(props, priority_value) + monitoring_conn = ThreadSafeConnectionHolder(None) + return GdbMonitoringConnectionHandler( + monitoring_conn, MagicMock(), MagicMock(), props, Properties(), writer_ref), monitoring_conn + + +class TestAuroraAcceptConnection: + def test_accepts_first_connection(self): + handler, monitoring_conn = _make_aurora_handler("strict-writer") + conn = MagicMock() + assert handler.accept_connection(conn, True, WRITER_EAST) is True + assert monitoring_conn.get() is conn + + def test_default_priority_rejects_reader_after_writer(self): + # Default priority is strict-writer (index 0 for writer, no reader match). + handler, monitoring_conn = _make_aurora_handler("strict-writer") + writer_conn = MagicMock() + handler.accept_connection(writer_conn, True, WRITER_EAST) + reader_conn = MagicMock() + # Reader has no matching priority (-> _NO_PRIORITY_INDEX), worse than writer. + assert handler.accept_connection(reader_conn, False, READER_EAST) is False + assert monitoring_conn.get() is writer_conn + + def test_higher_priority_replaces(self): + # Priority list: reader first (index 0), writer second (index 1). + handler, monitoring_conn = _make_aurora_handler("strict-reader,strict-writer") + writer_conn = MagicMock() + # Writer matches index 1. + assert handler.accept_connection(writer_conn, True, WRITER_EAST) is True + reader_conn = MagicMock() + # Reader matches index 0 (higher priority) -> replaces. + assert handler.accept_connection(reader_conn, False, READER_EAST) is True + assert monitoring_conn.get() is reader_conn + + def test_lower_priority_rejected(self): + handler, monitoring_conn = _make_aurora_handler("strict-reader,strict-writer") + reader_conn = MagicMock() + handler.accept_connection(reader_conn, False, READER_EAST) # index 0 + writer_conn = MagicMock() + # Writer is index 1, worse than current index 0 -> rejected. + assert handler.accept_connection(writer_conn, True, WRITER_EAST) is False + assert monitoring_conn.get() is reader_conn + + +class TestAuroraAcceptConnections: + def test_selects_best_by_priority(self): + # writer-or-reader accepts anything at index 0; writer preferred nowhere, + # so use strict-writer,strict-reader: writer index 0, reader index 1. + handler, monitoring_conn = _make_aurora_handler("strict-writer,strict-reader") + writer_conn = MagicMock() + reader_conn = MagicMock() + connections = [ + (READER_EAST, ThreadSafeConnectionHolder(reader_conn)), + (WRITER_EAST, ThreadSafeConnectionHolder(writer_conn)), + ] + selected = handler.accept_connections(connections, WRITER_EAST, None) + assert selected == WRITER_EAST + assert monitoring_conn.get() is writer_conn + + def test_returns_none_for_empty(self): + handler, _ = _make_aurora_handler("strict-writer") + assert handler.accept_connections([], None, None) is None + + +class TestAuroraFindHostsForPriority: + def test_strict_writer_filters_writers(self): + handler, _ = _make_aurora_handler("strict-writer") + hosts = [WRITER_EAST, READER_EAST] + assert handler._find_hosts_for_priority(0, hosts) == [WRITER_EAST] + + def test_writer_or_reader_returns_all(self): + handler, _ = _make_aurora_handler("writer-or-reader") + hosts = [WRITER_EAST, READER_EAST] + assert handler._find_hosts_for_priority(0, hosts) == hosts + + +class TestAuroraClose: + def test_close_resets_priority_index(self): + handler, _ = _make_aurora_handler("strict-writer") + handler.accept_connection(MagicMock(), True, WRITER_EAST) + handler.close() + assert handler._current_priority_index == -1 + + +class TestGdbAcceptConnections: + def test_region_aware_selection_prefers_primary_writer(self): + writer_ref: AtomicReference = AtomicReference(WRITER_EAST) + handler, monitoring_conn = _make_gdb_handler( + "strict-writer-primary,strict-reader-secondary", writer_ref) + writer_conn = MagicMock() + reader_conn = MagicMock() + connections = [ + (READER_WEST, ThreadSafeConnectionHolder(reader_conn)), + (WRITER_EAST, ThreadSafeConnectionHolder(writer_conn)), + ] + # Primary region derived from writer_host_info override (us-east-1). + selected = handler.accept_connections(connections, WRITER_EAST, None) + assert selected == WRITER_EAST + assert monitoring_conn.get() is writer_conn + + def test_secondary_reader_selected_when_only_option(self): + writer_ref: AtomicReference = AtomicReference(WRITER_EAST) + handler, monitoring_conn = _make_gdb_handler("strict-reader-secondary", writer_ref) + reader_conn = MagicMock() + connections = [(READER_WEST, ThreadSafeConnectionHolder(reader_conn))] + selected = handler.accept_connections(connections, WRITER_EAST, None) + assert selected == READER_WEST + assert monitoring_conn.get() is reader_conn + + def test_primary_region_from_cached_writer(self): + writer_ref: AtomicReference = AtomicReference(WRITER_EAST) + handler, _ = _make_gdb_handler("strict-writer-primary", writer_ref) + assert handler._get_primary_region() == "us-east-1" diff --git a/tests/unit/test_monitoring_connection_priority.py b/tests/unit/test_monitoring_connection_priority.py new file mode 100644 index 000000000..93ce9bc48 --- /dev/null +++ b/tests/unit/test_monitoring_connection_priority.py @@ -0,0 +1,75 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from aws_advanced_python_wrapper.utils.monitoring_connection_priority import \ + MonitoringConnectionPriority as Priority + + +class TestFromValue: + def test_known_values(self): + assert Priority.from_value("strict-writer") is Priority.STRICT_WRITER + assert Priority.from_value("strict-reader") is Priority.STRICT_READER + assert Priority.from_value("writer-or-reader") is Priority.WRITER_OR_READER + + def test_case_insensitive(self): + assert Priority.from_value("STRICT-WRITER") is Priority.STRICT_WRITER + + def test_invalid_and_none(self): + assert Priority.from_value("invalid") is None + assert Priority.from_value(None) is None + + +class TestParseList: + def test_default_when_none(self): + assert Priority.parse_list(None) == [Priority.STRICT_WRITER] + + def test_default_when_empty(self): + assert Priority.parse_list("") == [Priority.STRICT_WRITER] + + def test_single_value(self): + assert Priority.parse_list("strict-reader") == [Priority.STRICT_READER] + + def test_multiple_values_preserve_order(self): + assert Priority.parse_list("strict-writer,strict-reader,writer-or-reader") == [ + Priority.STRICT_WRITER, Priority.STRICT_READER, Priority.WRITER_OR_READER] + + def test_with_spaces(self): + assert Priority.parse_list(" strict-reader , writer-or-reader ") == [ + Priority.STRICT_READER, Priority.WRITER_OR_READER] + + def test_ignores_duplicates(self): + assert Priority.parse_list("strict-writer,strict-writer,strict-reader") == [ + Priority.STRICT_WRITER, Priority.STRICT_READER] + + def test_ignores_invalid_values(self): + assert Priority.parse_list("invalid,strict-reader,bad-value") == [Priority.STRICT_READER] + + def test_all_invalid_falls_back_to_default(self): + assert Priority.parse_list("invalid,bad-value") == [Priority.STRICT_WRITER] + + +class TestIsSatisfiedBy: + def test_strict_writer(self): + assert Priority.STRICT_WRITER.is_satisfied_by(True) is True + assert Priority.STRICT_WRITER.is_satisfied_by(False) is False + + def test_strict_reader(self): + assert Priority.STRICT_READER.is_satisfied_by(True) is False + assert Priority.STRICT_READER.is_satisfied_by(False) is True + + def test_writer_or_reader(self): + assert Priority.WRITER_OR_READER.is_satisfied_by(True) is True + assert Priority.WRITER_OR_READER.is_satisfied_by(False) is True