From 2a3e5d9d7d2ee9d5b3df6d75c15e548b13892d93 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Sat, 6 Jun 2026 08:12:47 -0400 Subject: [PATCH 01/14] python tests: add the libpq ctypes layer and in-process Session A ctypes binding of libpq (bindings, constants, OIDs, library discovery, notification and result handling) plus a Session class providing synchronous, asynchronous, pipeline, COPY-free NOTIFY/notice and non-blocking query execution. This lets the Python test suite run SQL in-process without forking psql. --- src/test/pytest/libpq/__init__.py | 37 ++ src/test/pytest/libpq/bindings.py | 206 ++++++++++ src/test/pytest/libpq/constants.py | 130 +++++++ src/test/pytest/libpq/errors.py | 15 + src/test/pytest/libpq/findlib.py | 142 +++++++ src/test/pytest/libpq/oids.py | 151 ++++++++ src/test/pytest/libpq/pgnotify.py | 44 +++ src/test/pytest/libpq/result.py | 73 ++++ src/test/pytest/libpq/session.py | 597 +++++++++++++++++++++++++++++ 9 files changed, 1395 insertions(+) create mode 100644 src/test/pytest/libpq/__init__.py create mode 100644 src/test/pytest/libpq/bindings.py create mode 100644 src/test/pytest/libpq/constants.py create mode 100644 src/test/pytest/libpq/errors.py create mode 100644 src/test/pytest/libpq/findlib.py create mode 100644 src/test/pytest/libpq/oids.py create mode 100644 src/test/pytest/libpq/pgnotify.py create mode 100644 src/test/pytest/libpq/result.py create mode 100644 src/test/pytest/libpq/session.py diff --git a/src/test/pytest/libpq/__init__.py b/src/test/pytest/libpq/__init__.py new file mode 100644 index 0000000000..904ac7e2e7 --- /dev/null +++ b/src/test/pytest/libpq/__init__.py @@ -0,0 +1,37 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""ctypes-based libpq bindings for PostgreSQL tests. + +This is a complete in-process client: :class:`~libpq.session.Session` supports +synchronous, asynchronous and pipeline execution, LISTEN/NOTIFY, and notice +capture, so tests need neither psql subprocesses nor a third-party driver. +""" + +from . import constants, errors, oids +from .constants import ( + ConnStatusType, + ExecStatusType, + PGPing, + PGTransactionStatusType, + PostgresPollingStatusType, +) +from .errors import LibpqError, QueryError +from .result import ResultData +from .session import Session, connect, conninfo_quote + +__all__ = [ + "constants", + "errors", + "oids", + "ConnStatusType", + "ExecStatusType", + "PGPing", + "PGTransactionStatusType", + "PostgresPollingStatusType", + "LibpqError", + "QueryError", + "ResultData", + "Session", + "connect", + "conninfo_quote", +] diff --git a/src/test/pytest/libpq/bindings.py b/src/test/pytest/libpq/bindings.py new file mode 100644 index 0000000000..f2ada400f9 --- /dev/null +++ b/src/test/pytest/libpq/bindings.py @@ -0,0 +1,206 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Central ctypes prototype table for libpq. + +This is the ONLY module that assigns ``restype``/``argtypes`` to libpq +functions. Concentrating every prototype here contains the classic ctypes +footgun: a function whose ``restype`` is left at the default ``c_int`` would +sign-truncate a 64-bit pointer return on LP64 platforms and silently corrupt +it. Every entry in :data:`PROTOTYPES` sets restype and argtypes together, and +:func:`load` asserts each one was applied, so a forgotten or mistyped entry +fails loudly at load time instead of crashing mid-test. + +Opaque handles (``PGconn *``, ``PGresult *`` and the ``PGnotify *`` returned by +PQnotifies) are bound as ``c_void_p`` so the full 64-bit pointer survives and a +NULL return becomes Python ``None`` (falsy). ``Oid`` is unsigned 32-bit; the +microsecond clock and socket poll deadline are explicitly ``c_int64``. +""" + +import ctypes +import sys +from ctypes import ( + CFUNCTYPE, + POINTER, + Structure, + c_char_p, + c_int, + c_int64, + c_uint, + c_void_p, +) + +# Opaque handles and scalar aliases. The names mirror libpq's PGconn */ +# PGresult * typedefs, hence the non-PascalCase spelling. +PGconn_p = c_void_p # pylint: disable=invalid-name +PGresult_p = c_void_p # pylint: disable=invalid-name +Oid = c_uint + +_Oid_p = POINTER(c_uint) +_int_p = POINTER(c_int) +_charpp = POINTER(c_char_p) + + +class PQconninfoOption(Structure): + """A single resolved connection option, as returned by PQconninfo().""" + + _fields_ = [ + ("keyword", c_char_p), + ("envvar", c_char_p), + ("compiled", c_char_p), + ("val", c_char_p), + ("label", c_char_p), + ("dispchar", c_char_p), + ("dispsize", c_int), + ] + + +_PQconninfoOption_p = POINTER(PQconninfoOption) + +# void (*PQnoticeProcessor)(void *arg, const char *message) +NOTICE_PROCESSOR = CFUNCTYPE(None, c_void_p, c_char_p) + +# name -> (restype, [argtypes]). One line per libpq function. +PROTOTYPES = { + # --- Connection establishment / teardown ------------------------------- + "PQconnectdb": (PGconn_p, [c_char_p]), + "PQconnectdbParams": (PGconn_p, [_charpp, _charpp, c_int]), + "PQsetdbLogin": ( + PGconn_p, + [c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, c_char_p], + ), + "PQconnectStart": (PGconn_p, [c_char_p]), + "PQconnectStartParams": (PGconn_p, [_charpp, _charpp, c_int]), + "PQconnectPoll": (c_int, [PGconn_p]), + "PQresetStart": (c_int, [PGconn_p]), + "PQresetPoll": (c_int, [PGconn_p]), + "PQfinish": (None, [PGconn_p]), + "PQreset": (None, [PGconn_p]), + # --- Connection introspection ------------------------------------------ + "PQdb": (c_char_p, [PGconn_p]), + "PQuser": (c_char_p, [PGconn_p]), + "PQpass": (c_char_p, [PGconn_p]), + "PQhost": (c_char_p, [PGconn_p]), + "PQhostaddr": (c_char_p, [PGconn_p]), + "PQport": (c_char_p, [PGconn_p]), + "PQtty": (c_char_p, [PGconn_p]), + "PQoptions": (c_char_p, [PGconn_p]), + "PQstatus": (c_int, [PGconn_p]), + "PQtransactionStatus": (c_int, [PGconn_p]), + "PQparameterStatus": (c_char_p, [PGconn_p, c_char_p]), + "PQping": (c_int, [c_char_p]), + "PQpingParams": (c_int, [_charpp, _charpp, c_int]), + "PQprotocolVersion": (c_int, [PGconn_p]), + "PQserverVersion": (c_int, [PGconn_p]), + "PQerrorMessage": (c_char_p, [PGconn_p]), + "PQsocket": (c_int, [PGconn_p]), + "PQsocketPoll": (c_int, [c_int, c_int, c_int, c_int64]), + "PQgetCurrentTimeUSec": (c_int64, []), + "PQbackendPID": (c_int, [PGconn_p]), + "PQconnectionNeedsPassword": (c_int, [PGconn_p]), + "PQconnectionUsedPassword": (c_int, [PGconn_p]), + "PQconnectionUsedGSSAPI": (c_int, [PGconn_p]), + "PQclientEncoding": (c_int, [PGconn_p]), + "PQsetClientEncoding": (c_int, [PGconn_p, c_char_p]), + # --- Synchronous command execution ------------------------------------- + "PQexec": (PGresult_p, [PGconn_p, c_char_p]), + "PQexecParams": ( + PGresult_p, + [PGconn_p, c_char_p, c_int, _Oid_p, _charpp, _int_p, _int_p, c_int], + ), + "PQprepare": (PGresult_p, [PGconn_p, c_char_p, c_char_p, c_int, _Oid_p]), + "PQexecPrepared": ( + PGresult_p, + [PGconn_p, c_char_p, c_int, _charpp, _int_p, _int_p, c_int], + ), + "PQdescribePrepared": (PGresult_p, [PGconn_p, c_char_p]), + "PQdescribePortal": (PGresult_p, [PGconn_p, c_char_p]), + "PQclosePrepared": (PGresult_p, [PGconn_p, c_char_p]), + "PQclosePortal": (PGresult_p, [PGconn_p, c_char_p]), + "PQchangePassword": (PGresult_p, [PGconn_p, c_char_p, c_char_p]), + "PQclear": (None, [PGresult_p]), + # --- Result inspection ------------------------------------------------- + "PQresultStatus": (c_int, [PGresult_p]), + "PQresStatus": (c_char_p, [c_int]), + "PQresultErrorMessage": (c_char_p, [PGresult_p]), + "PQresultErrorField": (c_char_p, [PGresult_p, c_int]), + "PQntuples": (c_int, [PGresult_p]), + "PQnfields": (c_int, [PGresult_p]), + "PQbinaryTuples": (c_int, [PGresult_p]), + "PQfname": (c_char_p, [PGresult_p, c_int]), + "PQfnumber": (c_int, [PGresult_p, c_char_p]), + "PQftable": (Oid, [PGresult_p, c_int]), + "PQftablecol": (c_int, [PGresult_p, c_int]), + "PQfformat": (c_int, [PGresult_p, c_int]), + "PQftype": (Oid, [PGresult_p, c_int]), + "PQfsize": (c_int, [PGresult_p, c_int]), + "PQfmod": (c_int, [PGresult_p, c_int]), + "PQcmdStatus": (c_char_p, [PGresult_p]), + "PQoidValue": (Oid, [PGresult_p]), + "PQcmdTuples": (c_char_p, [PGresult_p]), + "PQgetvalue": (c_char_p, [PGresult_p, c_int, c_int]), + "PQgetlength": (c_int, [PGresult_p, c_int, c_int]), + "PQgetisnull": (c_int, [PGresult_p, c_int, c_int]), + "PQnparams": (c_int, [PGresult_p]), + "PQparamtype": (Oid, [PGresult_p, c_int]), + # --- Asynchronous command processing ----------------------------------- + "PQsendQuery": (c_int, [PGconn_p, c_char_p]), + "PQsendQueryParams": ( + c_int, + [PGconn_p, c_char_p, c_int, _Oid_p, _charpp, _int_p, _int_p, c_int], + ), + "PQsendPrepare": (c_int, [PGconn_p, c_char_p, c_char_p, c_int, _Oid_p]), + "PQgetResult": (PGresult_p, [PGconn_p]), + "PQisBusy": (c_int, [PGconn_p]), + "PQconsumeInput": (c_int, [PGconn_p]), + "PQsetnonblocking": (c_int, [PGconn_p, c_int]), + "PQisnonblocking": (c_int, [PGconn_p]), + "PQflush": (c_int, [PGconn_p]), + # --- Pipeline mode ----------------------------------------------------- + "PQpipelineStatus": (c_int, [PGconn_p]), + "PQenterPipelineMode": (c_int, [PGconn_p]), + "PQexitPipelineMode": (c_int, [PGconn_p]), + "PQpipelineSync": (c_int, [PGconn_p]), + "PQsendFlushRequest": (c_int, [PGconn_p]), + "PQsendPipelineSync": (c_int, [PGconn_p]), + # --- Notifications ----------------------------------------------------- + # PQnotifies returns PGnotify *; keep the raw pointer (c_void_p) so the + # original allocation can be handed back to PQfreemem. + "PQnotifies": (c_void_p, [PGconn_p]), + "PQfreemem": (None, [c_void_p]), + # Resolved connection options (the actual values libpq used, including the + # service file it settled on). PQconninfo's array must be freed with + # PQconninfoFree. + "PQconninfo": (_PQconninfoOption_p, [PGconn_p]), + "PQconninfoFree": (None, [_PQconninfoOption_p]), + # --- Notice processing ------------------------------------------------- + "PQsetNoticeProcessor": (c_void_p, [PGconn_p, NOTICE_PROCESSOR, c_void_p]), +} + + +def load(libpath): + """Open *libpath* and apply every prototype in :data:`PROTOTYPES`. + + Returns the configured ``ctypes.CDLL``. Raises if a function is missing + from the library or if any prototype failed to apply. + """ + if sys.platform == "win32": + # libpq.dll's dependent DLLs (OpenSSL, zlib, libintl, ...) are not + # beside it; they are found via PATH. winmode=0 selects the standard + # Windows search order, which consults PATH, whereas the ctypes default + # since Python 3.8 (LOAD_LIBRARY_SEARCH_DEFAULT_DIRS) does not and so + # fails to resolve them. + lib = ctypes.CDLL(libpath, winmode=0) + else: + lib = ctypes.CDLL(libpath) + for name, (restype, argtypes) in PROTOTYPES.items(): + fn = getattr(lib, name) # AttributeError here = symbol missing + fn.restype = restype + fn.argtypes = argtypes + + # Defense in depth: confirm every prototype actually took, so a forgotten + # restype can never reach a caller as a silent c_int default. + for name, (restype, _argtypes) in PROTOTYPES.items(): + applied = getattr(lib, name).restype + assert applied is restype, f"prototype not applied for {name}" + + return lib diff --git a/src/test/pytest/libpq/constants.py b/src/test/pytest/libpq/constants.py new file mode 100644 index 0000000000..d70ddbe3b0 --- /dev/null +++ b/src/test/pytest/libpq/constants.py @@ -0,0 +1,130 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""libpq enum constants used by the ctypes backend. + +The values are the integer codes from libpq-fe.h and are exposed both as +IntEnum members (so they print their symbolic name in errors) and as +module-level names so framework code can use the bare symbols. +""" + +import enum + + +class ConnStatusType(enum.IntEnum): + """Connection status codes (ConnStatusType in libpq-fe.h).""" + + CONNECTION_OK = 0 + CONNECTION_BAD = 1 + CONNECTION_STARTED = 2 + CONNECTION_MADE = 3 + CONNECTION_AWAITING_RESPONSE = 4 + CONNECTION_AUTH_OK = 5 + CONNECTION_SETENV = 6 + CONNECTION_SSL_STARTUP = 7 + CONNECTION_NEEDED = 8 + CONNECTION_CHECK_WRITABLE = 9 + CONNECTION_CONSUME = 10 + CONNECTION_GSS_STARTUP = 11 + CONNECTION_CHECK_TARGET = 12 + CONNECTION_CHECK_STANDBY = 13 + CONNECTION_ALLOCATED = 14 + + +class ExecStatusType(enum.IntEnum): + """Result status codes returned by PQresultStatus().""" + + PGRES_EMPTY_QUERY = 0 + PGRES_COMMAND_OK = 1 + PGRES_TUPLES_OK = 2 + PGRES_COPY_OUT = 3 + PGRES_COPY_IN = 4 + PGRES_BAD_RESPONSE = 5 + PGRES_NONFATAL_ERROR = 6 + PGRES_FATAL_ERROR = 7 + PGRES_COPY_BOTH = 8 + PGRES_SINGLE_TUPLE = 9 + PGRES_PIPELINE_SYNC = 10 + PGRES_PIPELINE_ABORTED = 11 + PGRES_TUPLES_CHUNK = 12 + + +class PostgresPollingStatusType(enum.IntEnum): + """Async connection polling status (PQconnectPoll()).""" + + PGRES_POLLING_FAILED = 0 + PGRES_POLLING_READING = 1 + PGRES_POLLING_WRITING = 2 + PGRES_POLLING_OK = 3 + PGRES_POLLING_ACTIVE = 4 + + +class PGPing(enum.IntEnum): + """Server status codes returned by PQping().""" + + PQPING_OK = 0 + PQPING_REJECT = 1 + PQPING_NO_RESPONSE = 2 + PQPING_NO_ATTEMPT = 3 + + +class PGTransactionStatusType(enum.IntEnum): + """Transaction status codes returned by PQtransactionStatus().""" + + PQTRANS_IDLE = 0 + PQTRANS_ACTIVE = 1 + PQTRANS_INTRANS = 2 + PQTRANS_INERROR = 3 + PQTRANS_UNKNOWN = 4 + + +# Module-level aliases for every member (CONNECTION_OK, PGRES_TUPLES_OK, ...) +# so test/framework code can use the bare names, while comparisons against +# IntEnum members still succeed. Spelled out explicitly (rather than built +# with a globals() loop) so static analysis can see the names. + +CONNECTION_OK = ConnStatusType.CONNECTION_OK +CONNECTION_BAD = ConnStatusType.CONNECTION_BAD +CONNECTION_STARTED = ConnStatusType.CONNECTION_STARTED +CONNECTION_MADE = ConnStatusType.CONNECTION_MADE +CONNECTION_AWAITING_RESPONSE = ConnStatusType.CONNECTION_AWAITING_RESPONSE +CONNECTION_AUTH_OK = ConnStatusType.CONNECTION_AUTH_OK +CONNECTION_SETENV = ConnStatusType.CONNECTION_SETENV +CONNECTION_SSL_STARTUP = ConnStatusType.CONNECTION_SSL_STARTUP +CONNECTION_NEEDED = ConnStatusType.CONNECTION_NEEDED +CONNECTION_CHECK_WRITABLE = ConnStatusType.CONNECTION_CHECK_WRITABLE +CONNECTION_CONSUME = ConnStatusType.CONNECTION_CONSUME +CONNECTION_GSS_STARTUP = ConnStatusType.CONNECTION_GSS_STARTUP +CONNECTION_CHECK_TARGET = ConnStatusType.CONNECTION_CHECK_TARGET +CONNECTION_CHECK_STANDBY = ConnStatusType.CONNECTION_CHECK_STANDBY +CONNECTION_ALLOCATED = ConnStatusType.CONNECTION_ALLOCATED + +PGRES_EMPTY_QUERY = ExecStatusType.PGRES_EMPTY_QUERY +PGRES_COMMAND_OK = ExecStatusType.PGRES_COMMAND_OK +PGRES_TUPLES_OK = ExecStatusType.PGRES_TUPLES_OK +PGRES_COPY_OUT = ExecStatusType.PGRES_COPY_OUT +PGRES_COPY_IN = ExecStatusType.PGRES_COPY_IN +PGRES_BAD_RESPONSE = ExecStatusType.PGRES_BAD_RESPONSE +PGRES_NONFATAL_ERROR = ExecStatusType.PGRES_NONFATAL_ERROR +PGRES_FATAL_ERROR = ExecStatusType.PGRES_FATAL_ERROR +PGRES_COPY_BOTH = ExecStatusType.PGRES_COPY_BOTH +PGRES_SINGLE_TUPLE = ExecStatusType.PGRES_SINGLE_TUPLE +PGRES_PIPELINE_SYNC = ExecStatusType.PGRES_PIPELINE_SYNC +PGRES_PIPELINE_ABORTED = ExecStatusType.PGRES_PIPELINE_ABORTED +PGRES_TUPLES_CHUNK = ExecStatusType.PGRES_TUPLES_CHUNK + +PGRES_POLLING_FAILED = PostgresPollingStatusType.PGRES_POLLING_FAILED +PGRES_POLLING_READING = PostgresPollingStatusType.PGRES_POLLING_READING +PGRES_POLLING_WRITING = PostgresPollingStatusType.PGRES_POLLING_WRITING +PGRES_POLLING_OK = PostgresPollingStatusType.PGRES_POLLING_OK +PGRES_POLLING_ACTIVE = PostgresPollingStatusType.PGRES_POLLING_ACTIVE + +PQPING_OK = PGPing.PQPING_OK +PQPING_REJECT = PGPing.PQPING_REJECT +PQPING_NO_RESPONSE = PGPing.PQPING_NO_RESPONSE +PQPING_NO_ATTEMPT = PGPing.PQPING_NO_ATTEMPT + +PQTRANS_IDLE = PGTransactionStatusType.PQTRANS_IDLE +PQTRANS_ACTIVE = PGTransactionStatusType.PQTRANS_ACTIVE +PQTRANS_INTRANS = PGTransactionStatusType.PQTRANS_INTRANS +PQTRANS_INERROR = PGTransactionStatusType.PQTRANS_INERROR +PQTRANS_UNKNOWN = PGTransactionStatusType.PQTRANS_UNKNOWN diff --git a/src/test/pytest/libpq/errors.py b/src/test/pytest/libpq/errors.py new file mode 100644 index 0000000000..f61d0631b0 --- /dev/null +++ b/src/test/pytest/libpq/errors.py @@ -0,0 +1,15 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Exceptions raised by the libpq wrapper.""" + + +class LibpqError(Exception): + """Base class for libpq-related errors (connection or query failure).""" + + +class PqConnectionError(LibpqError): + """Raised when a libpq connection cannot be established.""" + + +class QueryError(LibpqError): + """Raised by the *_safe query helpers when a statement fails.""" diff --git a/src/test/pytest/libpq/findlib.py b/src/test/pytest/libpq/findlib.py new file mode 100644 index 0000000000..30a80fde05 --- /dev/null +++ b/src/test/pytest/libpq/findlib.py @@ -0,0 +1,142 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Locate the libpq shared library. + +A lightweight replacement for ctypes.util that searches caller-supplied +directories first and then common system locations, honoring LD_LIBRARY_PATH / +DYLD_LIBRARY_PATH, and returns the full path to the library file. +""" + +import ctypes +import glob +import os +import sys + + +def libpq_abi_skip_reason(libdir): + """Return a reason to skip if this Python cannot load the build's libpq. + + The framework loads libpq in-process via ctypes, so the interpreter and + the library must share an ABI. The common mismatch is a 64-bit Python + against a 32-bit libpq (meson's ``-m32`` build), which otherwise fails + every test with ``OSError: wrong ELF class``. Detect it by reading the + library's ELF header rather than dlopen()ing it -- a trial dlopen of an + ASan-instrumented libpq would abort the process, not raise. Returns None + when the ABI matches, when libpq cannot be located (the normal load path + reports that), or when the file is not ELF (macOS/Windows). + """ + try: + if libdir: + path = find_lib_or_die("pq", libpath=[libdir], systempath=False) + else: + path = find_lib_or_die("pq", systempath=True) + except RuntimeError: + return None + + elf_class = _elf_class(path) + if elf_class is None: + return None + + py_bits = ctypes.sizeof(ctypes.c_void_p) * 8 + lib_bits = 64 if elf_class == 2 else 32 + if py_bits != lib_bits: + return ( + f"{py_bits}-bit Python cannot load {lib_bits}-bit libpq ({path}); " + f"the in-process libpq framework needs a {lib_bits}-bit interpreter" + ) + return None + + +def _elf_class(path): + """Return 1 (ELFCLASS32), 2 (ELFCLASS64), or None if *path* is not ELF.""" + try: + with open(path, "rb") as fh: + ident = fh.read(5) + except OSError: + return None + if ident[:4] != b"\x7fELF": + return None + return ident[4] # e_ident[EI_CLASS]: 1 = 32-bit, 2 = 64-bit + + +def find_lib_or_die(lib, libpath=None, systempath=True): + """Return the full path to the shared library named *lib* (e.g. "pq"). + + *libpath* is an iterable of directories searched first. When + *systempath* is false the common system locations are not searched (used + when the caller passes the cluster's own --libdir and wants exactly that). + Raises RuntimeError if nothing is found. + """ + search_paths = list(libpath or []) + if systempath: + search_paths.extend(_system_lib_paths()) + search_paths = _with_windows_bindir(search_paths) + + patterns = _lib_patterns(lib) + + for directory in search_paths: + if not os.path.isdir(directory): + continue + for pattern in patterns: + for match in sorted(glob.glob(os.path.join(directory, pattern))): + if os.path.isfile(match) and os.access(match, os.R_OK): + return match + + raise RuntimeError( + f"find_lib_or_die: unable to find lib{lib} in: " + ", ".join(search_paths) + ) + + +def _with_windows_bindir(paths): + """On Windows, add the sibling ``bin`` of each search directory. + + The runtime DLL is installed in ``bin`` there, while ``lib`` (which is what + ``pg_config --libdir`` reports) holds only the import library. Elsewhere + the list is returned unchanged. + """ + if sys.platform not in ("win32", "cygwin"): + return paths + expanded = [] + for directory in paths: + if directory not in expanded: + expanded.append(directory) + sibling_bin = os.path.join(os.path.dirname(directory.rstrip("\\/")), "bin") + if sibling_bin not in expanded: + expanded.append(sibling_bin) + return expanded + + +def _lib_patterns(lib): + if sys.platform == "darwin": + return (f"lib{lib}.dylib", f"lib{lib}.*.dylib") + if sys.platform in ("win32", "cygwin"): + return (f"{lib}.dll", f"lib{lib}.dll") + # Linux and other Unix-like systems + return (f"lib{lib}.so", f"lib{lib}.so.*") + + +def _system_lib_paths(): + paths = [] + + # Honor the loader's search path first, so a libpq placed on + # LD_LIBRARY_PATH / DYLD_LIBRARY_PATH wins over a system-installed one, + # matching how the dynamic linker resolves the library. + if sys.platform.startswith("linux") and os.environ.get("LD_LIBRARY_PATH"): + paths += os.environ["LD_LIBRARY_PATH"].split(os.pathsep) + if sys.platform == "darwin" and os.environ.get("DYLD_LIBRARY_PATH"): + paths += os.environ["DYLD_LIBRARY_PATH"].split(os.pathsep) + + paths += ["/usr/lib", "/usr/local/lib", "/lib"] + + if sys.platform.startswith("linux"): + paths += [ + "/usr/lib/x86_64-linux-gnu", + "/usr/lib/aarch64-linux-gnu", + "/usr/lib64", + "/lib64", + ] + + if sys.platform == "darwin": + paths += ["/opt/homebrew/lib", "/usr/local/opt/libpq/lib"] + + return paths diff --git a/src/test/pytest/libpq/oids.py b/src/test/pytest/libpq/oids.py new file mode 100644 index 0000000000..adce1e1d52 --- /dev/null +++ b/src/test/pytest/libpq/oids.py @@ -0,0 +1,151 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""PostgreSQL backend type OID constants. + +Values come from src/include/catalog/pg_type_d.h and are used to identify +column types in query results via PQftype(). +""" + +# Basic types +BOOLOID = 16 +BYTEAOID = 17 +CHAROID = 18 +NAMEOID = 19 +INT8OID = 20 +INT2OID = 21 +INT2VECTOROID = 22 +INT4OID = 23 +TEXTOID = 25 +OIDOID = 26 +TIDOID = 27 +XIDOID = 28 +CIDOID = 29 +OIDVECTOROID = 30 +JSONOID = 114 +XMLOID = 142 +XID8OID = 5069 +POINTOID = 600 +LSEGOID = 601 +PATHOID = 602 +BOXOID = 603 +POLYGONOID = 604 +LINEOID = 628 +FLOAT4OID = 700 +FLOAT8OID = 701 +UNKNOWNOID = 705 +CIRCLEOID = 718 +MONEYOID = 790 +MACADDROID = 829 +INETOID = 869 +CIDROID = 650 +MACADDR8OID = 774 +ACLITEMOID = 1033 +BPCHAROID = 1042 +VARCHAROID = 1043 +DATEOID = 1082 +TIMEOID = 1083 +TIMESTAMPOID = 1114 +TIMESTAMPTZOID = 1184 +INTERVALOID = 1186 +TIMETZOID = 1266 +BITOID = 1560 +VARBITOID = 1562 +NUMERICOID = 1700 +REFCURSOROID = 1790 +UUIDOID = 2950 +TSVECTOROID = 3614 +GTSVECTOROID = 3642 +TSQUERYOID = 3615 +JSONBOID = 3802 +JSONPATHOID = 4072 +TXID_SNAPSHOTOID = 2970 + +# Range types +INT4RANGEOID = 3904 +NUMRANGEOID = 3906 +TSRANGEOID = 3908 +TSTZRANGEOID = 3910 +DATERANGEOID = 3912 +INT8RANGEOID = 3926 + +# Multirange types +INT4MULTIRANGEOID = 4451 +NUMMULTIRANGEOID = 4532 +TSMULTIRANGEOID = 4533 +TSTZMULTIRANGEOID = 4534 +DATEMULTIRANGEOID = 4535 +INT8MULTIRANGEOID = 4536 + +# Pseudo types +RECORDOID = 2249 +RECORDARRAYOID = 2287 +CSTRINGOID = 2275 +VOIDOID = 2278 +TRIGGEROID = 2279 +EVENT_TRIGGEROID = 3838 + +# Array types +BOOLARRAYOID = 1000 +BYTEAARRAYOID = 1001 +CHARARRAYOID = 1002 +NAMEARRAYOID = 1003 +INT8ARRAYOID = 1016 +INT2ARRAYOID = 1005 +INT2VECTORARRAYOID = 1006 +INT4ARRAYOID = 1007 +TEXTARRAYOID = 1009 +OIDARRAYOID = 1028 +TIDARRAYOID = 1010 +XIDARRAYOID = 1011 +CIDARRAYOID = 1012 +OIDVECTORARRAYOID = 1013 +JSONARRAYOID = 199 +XMLARRAYOID = 143 +XID8ARRAYOID = 271 +POINTARRAYOID = 1017 +LSEGARRAYOID = 1018 +PATHARRAYOID = 1019 +BOXARRAYOID = 1020 +POLYGONARRAYOID = 1027 +LINEARRAYOID = 629 +FLOAT4ARRAYOID = 1021 +FLOAT8ARRAYOID = 1022 +CIRCLEARRAYOID = 719 +MONEYARRAYOID = 791 +MACADDRARRAYOID = 1040 +INETARRAYOID = 1041 +CIDRARRAYOID = 651 +MACADDR8ARRAYOID = 775 +ACLITEMARRAYOID = 1034 +BPCHARARRAYOID = 1014 +VARCHARARRAYOID = 1015 +DATEARRAYOID = 1182 +TIMEARRAYOID = 1183 +TIMESTAMPARRAYOID = 1115 +TIMESTAMPTZARRAYOID = 1185 +INTERVALARRAYOID = 1187 +TIMETZARRAYOID = 1270 +BITARRAYOID = 1561 +VARBITARRAYOID = 1563 +NUMERICARRAYOID = 1231 +REFCURSORARRAYOID = 2201 +UUIDARRAYOID = 2951 +TSVECTORARRAYOID = 3643 +GTSVECTORARRAYOID = 3644 +TSQUERYARRAYOID = 3645 +JSONBARRAYOID = 3807 +JSONPATHARRAYOID = 4073 +TXID_SNAPSHOTARRAYOID = 2949 +INT4RANGEARRAYOID = 3905 +NUMRANGEARRAYOID = 3907 +TSRANGEARRAYOID = 3909 +TSTZRANGEARRAYOID = 3911 +DATERANGEARRAYOID = 3913 +INT8RANGEARRAYOID = 3927 +INT4MULTIRANGEARRAYOID = 6150 +NUMMULTIRANGEARRAYOID = 6151 +TSMULTIRANGEARRAYOID = 6152 +TSTZMULTIRANGEARRAYOID = 6153 +DATEMULTIRANGEARRAYOID = 6155 +INT8MULTIRANGEARRAYOID = 6157 +CSTRINGARRAYOID = 1263 diff --git a/src/test/pytest/libpq/pgnotify.py b/src/test/pytest/libpq/pgnotify.py new file mode 100644 index 0000000000..55ba9f30be --- /dev/null +++ b/src/test/pytest/libpq/pgnotify.py @@ -0,0 +1,44 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""The PGnotify struct and a helper to read a notification. + +PQnotifies() returns a pointer to a heap-allocated PGnotify that the caller +must free with PQfreemem(); we keep that pointer (as c_void_p from bindings) +for the free, cast it to read the fields, and decode the strings BEFORE +freeing. +""" + +import ctypes + + +class PGnotify(ctypes.Structure): + """typedef struct pgNotify { char *relname; int be_pid; char *extra; }.""" + + _fields_ = [ + ("relname", ctypes.c_char_p), # notification channel name + ("be_pid", ctypes.c_int), # PID of the notifying backend + ("extra", ctypes.c_char_p), # notification payload string + ] + + +_PGnotify_p = ctypes.POINTER(PGnotify) + + +def read_notification(lib, raw): + """Turn the raw PQnotifies pointer *raw* into a dict and free it. + + Returns ``{"channel", "pid", "payload"}`` or ``None`` if *raw* is NULL. + """ + if not raw: + return None + notify = ctypes.cast(raw, _PGnotify_p).contents + # Decode while the memory is still valid (before PQfreemem). + result = { + "channel": ( + notify.relname.decode("utf-8", "replace") if notify.relname else None + ), + "pid": notify.be_pid, + "payload": (notify.extra.decode("utf-8", "replace") if notify.extra else None), + } + lib.PQfreemem(ctypes.c_void_p(raw)) + return result diff --git a/src/test/pytest/libpq/result.py b/src/test/pytest/libpq/result.py new file mode 100644 index 0000000000..1bebbd0edc --- /dev/null +++ b/src/test/pytest/libpq/result.py @@ -0,0 +1,73 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Result extraction from a libpq PGresult. + +A query result is represented by :class:`ResultData`, which exposes status, +error_message, names, types, rows, and psqlout. All values come back as text +(result format 0), so ``psqlout`` matches ``psql -A -t`` output. +""" + +from dataclasses import dataclass, field +from typing import List, Optional + +from .constants import ExecStatusType + + +def _decode(raw): + """Decode a libpq C string (bytes or None) to str/None.""" + if raw is None: + return None + return raw.decode("utf-8", "replace") + + +@dataclass +class ResultData: + """Structured form of the data returned by Session query methods.""" + + status: int + error_message: Optional[str] = None + names: List[str] = field(default_factory=list) + types: List[int] = field(default_factory=list) + rows: List[List[Optional[str]]] = field(default_factory=list) + psqlout: str = "" + + +def extract_result_data(lib, result, conn): + """Build a :class:`ResultData` from a PGresult pointer. + + On a failed status the error comes from this result + (PQresultErrorMessage), falling back to the connection-level message + (*conn*) only when the result carries no error text. + """ + status = lib.PQresultStatus(result) + res = ResultData(status=status) + + if status not in (ExecStatusType.PGRES_TUPLES_OK, ExecStatusType.PGRES_COMMAND_OK): + res.error_message = _decode(lib.PQresultErrorMessage(result)) or _decode( + lib.PQerrorMessage(conn) + ) + return res + if status == ExecStatusType.PGRES_COMMAND_OK: + return res + + ntuples = lib.PQntuples(result) + nfields = lib.PQnfields(result) + for fld in range(nfields): + res.names.append(_decode(lib.PQfname(result, fld))) + res.types.append(lib.PQftype(result, fld)) + + textrows = [] + for nrow in range(ntuples): + row = [] + for fld in range(nfields): + val = _decode(lib.PQgetvalue(result, nrow, fld)) + if (val or "") == "" and lib.PQgetisnull(result, nrow, fld): + val = None + row.append(val) + res.rows.append(row) + # join renders NULL (None) as the empty string. + textrows.append("|".join("" if v is None else v for v in row)) + + if ntuples: + res.psqlout = "\n".join(textrows) + return res diff --git a/src/test/pytest/libpq/session.py b/src/test/pytest/libpq/session.py new file mode 100644 index 0000000000..9111c5d2a9 --- /dev/null +++ b/src/test/pytest/libpq/session.py @@ -0,0 +1,597 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""A libpq session for tests. + +A :class:`Session` owns one libpq connection and runs queries in-process, so +tests do not have to spawn psql. Several methods return a +:class:`~libpq.result.ResultData`. + +Asynchronous waits use PQsocketPoll, with one-second periodic deadline checks. +""" + +import getpass +import os +import re +import sys +import time +from ctypes import c_char_p + +from . import bindings +from .constants import ( + CONNECTION_BAD, + CONNECTION_OK, + PGRES_COMMAND_OK, + PGRES_PIPELINE_ABORTED, + PGRES_PIPELINE_SYNC, + PGRES_POLLING_FAILED, + PGRES_POLLING_OK, + PGRES_POLLING_READING, + PGRES_POLLING_WRITING, + PGRES_TUPLES_OK, + PQTRANS_INERROR, +) +from .errors import PqConnectionError +from .errors import QueryError +from .pgnotify import read_notification +from .result import extract_result_data + +# Default per-operation timeout in seconds. +DEFAULT_TIMEOUT = int(os.environ.get("PG_TEST_TIMEOUT_DEFAULT") or "180") + +# Cache of loaded libpq handles, keyed by resolved library path, so multiple +# clusters with different libdirs each get the right library exactly once. +_LIBS: dict = {} + +# Last connection error, for callers that treat a failed connect as fatal but +# obtained None/raise without the libpq message handy. +connect_error = None + + +def _load_lib(libdir): + from .findlib import find_lib_or_die + + if libdir: + path = find_lib_or_die("pq", libpath=[libdir], systempath=False) + else: + path = find_lib_or_die("pq", systempath=True) + lib = _LIBS.get(path) + if lib is None: + lib = bindings.load(path) + _LIBS[path] = lib + return lib + + +def _str_array(values): + """Build a char** from a list of str/None (None -> SQL NULL), or None.""" + if not values: + return None + arr = (c_char_p * len(values))() + for i, val in enumerate(values): + arr[i] = None if val is None else val.encode("utf-8") + return arr + + +def _enc(text): + return text.encode("utf-8") if text is not None else None + + +def _dec(raw): + return raw.decode("utf-8", "replace") if raw is not None else None + + +def conninfo_quote(value): + """Escape *value* for use inside single quotes in a libpq conninfo string. + + libpq treats backslash as an escape inside single quotes, so a literal + backslash or single quote in the value must be backslash-escaped. + """ + return str(value).replace("\\", "\\\\").replace("'", "\\'") + + +class Session: + """A libpq connection with synchronous, async and pipeline helpers.""" + + def __init__( + self, + connstr=None, + node=None, + dbname="postgres", + libdir=None, + user=None, + wait=True, + timeout=DEFAULT_TIMEOUT, + ): + global connect_error # pylint: disable=global-statement + + if libdir is None and node is not None: + libdir = node.libdir + self._lib = _load_lib(libdir) + + if connstr is None: + if node is None: + raise ValueError("Session requires connstr or node") + connstr = node.connstr(dbname) + + # Pin the connecting role unless the connection string names one, so a + # stray PGUSER cannot select a role the cluster does not recognize. + if not re.search(r"\buser\s*=", connstr): + if user is None: + user = ( + os.environ.get("USERNAME") + if sys.platform == "win32" + else getpass.getuser() + ) + if user: + connstr += f" user='{conninfo_quote(user)}'" + + self.connstr = connstr + self._notices = [] + self._notice_cb = None + self._last_error = None + self._closed = False + self._timeout = timeout + lib = self._lib + + if wait: + self._conn = lib.PQconnectdb(_enc(connstr)) + if lib.PQstatus(self._conn) != CONNECTION_OK: + connect_error = _dec(lib.PQerrorMessage(self._conn)) + msg = connect_error + self.close() + raise PqConnectionError(msg) + self._setup_notice_processor() + else: + self._conn = lib.PQconnectStart(_enc(connstr)) + if lib.PQstatus(self._conn) == CONNECTION_BAD: + connect_error = _dec(lib.PQerrorMessage(self._conn)) + msg = connect_error + self.close() + raise PqConnectionError(msg) + + # -- connection lifecycle ------------------------------------------------ + + def _setup_notice_processor(self): + notices = self._notices + + def _cb(_arg, message): + notices.append(_dec(message) or "") + + # Keep a reference so libpq's stored function pointer stays valid. + self._notice_cb = bindings.NOTICE_PROCESSOR(_cb) + self._lib.PQsetNoticeProcessor(self._conn, self._notice_cb, None) + + def wait_connect(self, timeout=DEFAULT_TIMEOUT): + """Drive an async (wait=False) connection to CONNECTION_OK.""" + lib = self._lib + conn = self._conn + start = time.monotonic() + while True: + poll_res = lib.PQconnectPoll(conn) + status = lib.PQstatus(conn) + if poll_res == PGRES_POLLING_OK or status == CONNECTION_OK: + self._setup_notice_processor() + return + if poll_res == PGRES_POLLING_FAILED or status == CONNECTION_BAD: + raise PqConnectionError( + "connection failed: " + (_dec(lib.PQerrorMessage(conn)) or "") + ) + if time.monotonic() - start > timeout: + raise TimeoutError("timed out waiting for connection") + sock = lib.PQsocket(conn) + if sock >= 0: + for_read = 1 if poll_res == PGRES_POLLING_READING else 0 + for_write = 1 if poll_res == PGRES_POLLING_WRITING else 0 + end_time = lib.PQgetCurrentTimeUSec() + 1_000_000 + lib.PQsocketPoll(sock, for_read, for_write, end_time) + + def poll_connect(self): + """Single non-blocking step of async connection polling.""" + return self._lib.PQconnectPoll(self._conn) + + def close(self): + if getattr(self, "_closed", True): + return + conn = getattr(self, "_conn", None) + if conn is not None: + self._lib.PQfinish(conn) + self._conn = None + self._notice_cb = None + self._closed = True + + quit = close + + def __del__(self): + try: + self.close() + except Exception: # pylint: disable=broad-exception-caught + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def reconnect(self): + if not self._closed: + self.close() + lib = self._lib + self._conn = lib.PQconnectdb(_enc(self.connstr)) + self._closed = False + status = lib.PQstatus(self._conn) + if status == CONNECTION_OK: + self._setup_notice_processor() + else: + # Failed reconnect: finish the dead conn rather than leaving a + # half-open session whose later calls would deref a bad PGconn. + self.close() + return status + + def reconnect_and_clear(self): + status = self.reconnect() + self.clear_notices() + return status + + def conn_status(self): + return self._lib.PQstatus(self._conn) if not self._closed else None + + def connected(self): + """True if the session has a live connection (status CONNECTION_OK).""" + return not self._closed and self._lib.PQstatus(self._conn) == CONNECTION_OK + + def backend_pid(self): + return self._lib.PQbackendPID(self._conn) + + # -- notice / stderr capture -------------------------------------------- + + def conninfo_value(self, keyword): + """Return libpq's resolved value for connection option *keyword*. + + For example ``conninfo_value("servicefile")`` reports the service file + libpq actually settled on, the same value psql exposes as the + :SERVICEFILE variable. Returns None if the option has no value. + """ + lib = self._lib + opts = lib.PQconninfo(self._conn) + if not opts: + return None + try: + i = 0 + while opts[i].keyword: + if _dec(opts[i].keyword) == keyword: + return _dec(opts[i].val) + i += 1 + return None + finally: + lib.PQconninfoFree(opts) + + def get_notices_str(self): + return "".join(self._notices) + + def clear_notices(self): + # Clear in place: the notice callback holds a reference to this list. + self._notices[:] = [] + + def get_stderr(self): + stderr = self.get_notices_str() + if self._last_error is not None: + stderr += self._last_error + return stderr + + def clear_stderr(self): + self.clear_notices() + self._last_error = None + + # -- synchronous execution ---------------------------------------------- + + def do(self, *sql_statements): + """Run statements with PQexec; return the status of the last one.""" + lib = self._lib + conn = self._conn + status = None + for sql in sql_statements: + result = lib.PQexec(conn, _enc(sql)) + status = lib.PQresultStatus(result) + lib.PQclear(result) + if status != PGRES_COMMAND_OK: + return status + return status + + def query(self, sql): + """Run SQL that may return tuples; return a ResultData. + + *sql* may contain several semicolon-separated statements; their output + is collected like psql. Note, however, that the whole string is sent as + a single libpq command and therefore runs in ONE implicit transaction -- + unlike psql, which runs each statement in its own autocommit + transaction. Statements that cannot run inside a transaction block + (CREATE/DROP DATABASE, VACUUM, CHECKPOINT, CREATE/ALTER/DROP + SUBSCRIPTION, CREATE TABLESPACE, ...) must be issued one per call. + """ + lib = self._lib + conn = self._conn + + if not lib.PQsendQuery(conn, _enc(sql)): + from .result import ResultData + + return ResultData(status=-1, error_message=_dec(lib.PQerrorMessage(conn))) + + final_res = None + last_error = None + error_status = None + all_psqlout = [] + while True: + result = self._get_result() + if not result: + break + res = extract_result_data(lib, result, conn) + lib.PQclear(result) + if res.psqlout != "": + all_psqlout.append(res.psqlout) + if res.error_message is not None: + last_error = res.error_message + error_status = res.status + if res.status == PGRES_TUPLES_OK or final_res is None: + final_res = res + + if final_res is None: + from .result import ResultData + + final_res = ResultData(status=PGRES_COMMAND_OK) + + if all_psqlout: + final_res.psqlout = "\n".join(all_psqlout) + if last_error is not None: + final_res.error_message = last_error + # Reflect a later statement's error in the status even when an earlier + # statement returned tuples, so callers that check status (not just + # error_message) see the failure. + if error_status is not None: + final_res.status = error_status + self._last_error = last_error + + # A multi-statement query that errors can leave the session in an open, + # aborted transaction: libpq aborts processing of the query string at + # the error, so a trailing COMMIT (e.g. "BEGIN; ; COMMIT") never + # runs. Roll back so a later query on this reused session is not + # rejected with "current transaction is aborted". + if lib.PQtransactionStatus(conn) == PQTRANS_INERROR: + rb = lib.PQexec(conn, b"ROLLBACK") + if rb: + lib.PQclear(rb) + return final_res + + def query_safe(self, sql): + """query() that raises on error; returns the psqlout string.""" + res = self.query(sql) + if res.error_message is not None: + short = re.sub(r"\s+", " ", sql[:100]) + raise QueryError(f"query_safe failed on [{short}...]: {res.error_message}") + return res.psqlout + + def query_oneval(self, sql, missing_ok=False): + """Return the single value of a one-row, one-column query.""" + lib = self._lib + conn = self._conn + result = lib.PQexec(conn, _enc(sql)) + status = lib.PQresultStatus(result) + if status != PGRES_TUPLES_OK: + if result: + lib.PQclear(result) + raise QueryError(_dec(lib.PQerrorMessage(conn))) + ntuples = lib.PQntuples(result) + if missing_ok and not ntuples: + lib.PQclear(result) + return None + nfields = lib.PQnfields(result) + if ntuples != 1 or nfields != 1: + lib.PQclear(result) + raise QueryError(f"{ntuples} tuples != 1 or {nfields} fields != 1") + val = _dec(lib.PQgetvalue(result, 0, 0)) + if val == "" and lib.PQgetisnull(result, 0, 0): + val = None + lib.PQclear(result) + return val + + def query_tuples(self, *sql_statements): + """Run queries and return output like ``psql -A -t``.""" + # Use the pipelined path for 4+ queries. + if len(sql_statements) >= 4: + return self.query_tuples_pipelined(*sql_statements) + + results = [] + for sql in sql_statements: + res = self.query(sql) + if res.status != PGRES_TUPLES_OK: + raise QueryError(res.error_message) + # query() already built psqlout in "psql -A -t" form; skip only + # when there are no rows. + if res.rows: + results.append(res.psqlout) + return "\n".join(results) + + # -- asynchronous execution --------------------------------------------- + + def do_async(self, sql): + """Send a single statement with PQsendQuery; return bool success.""" + return bool(self._lib.PQsendQuery(self._conn, _enc(sql))) + + def _get_result(self): + """Fetch the next async result, waiting on the socket with a deadline. + + Waits for the socket to become readable -- and also writable while + PQflush() reports unsent data, since on a non-blocking connection the + request may not be fully flushed yet and waiting only for readable would + deadlock (the server cannot reply to a request it has not received). + Raises TimeoutError once the per-session timeout passes. + """ + lib = self._lib + conn = self._conn + sock = lib.PQsocket(conn) + deadline = lib.PQgetCurrentTimeUSec() + self._timeout * 1_000_000 + while lib.PQisBusy(conn): + flush = lib.PQflush(conn) + if flush < 0: + raise QueryError( + "PQflush failed: " + (_dec(lib.PQerrorMessage(conn)) or "") + ) + now = lib.PQgetCurrentTimeUSec() + if now >= deadline: + raise TimeoutError("timed out waiting for query result") + # Wake at least once a second to recheck the deadline. + end = min(now + 1_000_000, deadline) + lib.PQsocketPoll(sock, 1, 1 if flush > 0 else 0, end) + if lib.PQconsumeInput(conn) == 0: + # Connection trouble (including the server closing the socket + # right after a FATAL error, e.g. "cannot alter invalid + # database"). Stop and return whatever PQgetResult yields: any + # error result already received is reported, and a clean drop + # with no result comes back as NULL, which get_async_result() + # surfaces as None for crash-detection callers. + break + return lib.PQgetResult(conn) + + def wait_for_completion(self): + """Drain and discard all outstanding async results.""" + lib = self._lib + while True: + res = self._get_result() + if not res: + break + lib.PQclear(res) + + def get_async_result(self): + """Wait for and return the next async result as ResultData.""" + lib = self._lib + conn = self._conn + result = self._get_result() + if not result: + return None + res = extract_result_data(lib, result, conn) + lib.PQclear(result) + while True: + extra = self._get_result() + if not extra: + break + lib.PQclear(extra) + return res + + # -- password change ----------------------------------------------------- + + def set_password(self, user, password): + lib = self._lib + conn = self._conn + result = lib.PQchangePassword(conn, _enc(user), _enc(password)) + ret = extract_result_data(lib, result, conn) + lib.PQclear(result) + return ret + + # -- pipeline mode ------------------------------------------------------- + + def setnonblocking(self, val): + if self._lib.PQsetnonblocking(self._conn, val): + raise QueryError("problem setting non-blocking") + + # The camelCase names below mirror the libpq PQ* functions they wrap. + + def enterPipelineMode(self): # pylint: disable=invalid-name + return self._lib.PQenterPipelineMode(self._conn) + + def pipelineSync(self): # pylint: disable=invalid-name + return self._lib.PQpipelineSync(self._conn) + + def do_pipeline(self, statement, *args): + arr = _str_array(list(args)) + return self._lib.PQsendQueryParams( + self._conn, _enc(statement), len(args), None, arr, None, None, 0 + ) + + def query_tuples_pipelined(self, *queries): + """Run multiple queries in one pipeline round trip.""" + lib = self._lib + conn = self._conn + results = [] + + if not lib.PQenterPipelineMode(conn): + raise QueryError("Failed to enter pipeline mode") + + for sql in queries: + if not lib.PQsendQueryParams(conn, _enc(sql), 0, None, None, None, None, 0): + lib.PQexitPipelineMode(conn) + raise QueryError( + "Failed to send query: " + (_dec(lib.PQerrorMessage(conn)) or "") + ) + + if not lib.PQpipelineSync(conn): + lib.PQexitPipelineMode(conn) + raise QueryError("Failed to sync pipeline") + + for i in range(len(queries)): + result = self._get_result() + if not result: + lib.PQexitPipelineMode(conn) + raise QueryError(f"No result for query {i}") + status = lib.PQresultStatus(result) + if status == PGRES_PIPELINE_ABORTED: + lib.PQclear(result) + lib.PQexitPipelineMode(conn) + raise QueryError(f"Pipeline aborted at query {i}") + if status == PGRES_TUPLES_OK: + res = extract_result_data(lib, result, conn) + if res.rows: + tuples = [ + "|".join("" if v is None else v for v in row) + for row in res.rows + ] + results.append("\n".join(tuples)) + elif status != PGRES_COMMAND_OK: + err = _dec(lib.PQerrorMessage(conn)) or "" + lib.PQclear(result) + lib.PQexitPipelineMode(conn) + raise QueryError(f"Query {i} failed: {err}") + lib.PQclear(result) + + # Consume the NULL result that ends this query's results. + while True: + extra = lib.PQgetResult(conn) + if not extra: + break + lib.PQclear(extra) + + sync_result = self._get_result() + if sync_result: + status = lib.PQresultStatus(sync_result) + lib.PQclear(sync_result) + if status != PGRES_PIPELINE_SYNC: + lib.PQexitPipelineMode(conn) + raise QueryError(f"Expected PGRES_PIPELINE_SYNC, got {status}") + + if not lib.PQexitPipelineMode(conn): + raise QueryError("Failed to exit pipeline mode") + + return "\n".join(results) + + # -- notifications ------------------------------------------------------- + + def get_notification(self): + """Return one pending LISTEN/NOTIFY notification, or None.""" + lib = self._lib + conn = self._conn + lib.PQconsumeInput(conn) + raw = lib.PQnotifies(conn) + return read_notification(lib, raw) + + def get_all_notifications(self): + """Return all pending notifications as a list of dicts.""" + notifications = [] + while True: + notify = self.get_notification() + if notify is None: + break + notifications.append(notify) + return notifications + + +def connect(connstr=None, node=None, dbname="postgres", libdir=None, **kwargs): + """Convenience constructor for a :class:`Session`.""" + return Session(connstr=connstr, node=node, dbname=dbname, libdir=libdir, **kwargs) From 52727b09d08f8b72d561eaa0d1258d5258b70e64 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Sat, 6 Jun 2026 08:12:47 -0400 Subject: [PATCH 02/14] python tests: add the PostgresServer framework and pytest fixtures PostgresServer manages a cluster's lifecycle (initdb, start/stop/restart, promote), configuration, in-process SQL, log inspection, backup/streaming/ archiving/restore, WAL helpers, replication-slot helpers and connect_ok/ connect_fails connection assertions. PgBin runs client programs; the fixtures (pg_bin, create_pg, pg, conn, bindir, libdir) build the common test objects and tear them down automatically. Author: Jelte Fennema-Nio Reviewed-by: Andrew Dunstan --- src/test/pytest/pypg/__init__.py | 19 + src/test/pytest/pypg/_env.py | 45 + src/test/pytest/pypg/command.py | 177 ++++ src/test/pytest/pypg/fixtures.py | 371 +++++++++ src/test/pytest/pypg/server.py | 1315 ++++++++++++++++++++++++++++++ src/test/pytest/pypg/util.py | 213 +++++ 6 files changed, 2140 insertions(+) create mode 100644 src/test/pytest/pypg/__init__.py create mode 100644 src/test/pytest/pypg/_env.py create mode 100644 src/test/pytest/pypg/command.py create mode 100644 src/test/pytest/pypg/fixtures.py create mode 100644 src/test/pytest/pypg/server.py create mode 100644 src/test/pytest/pypg/util.py diff --git a/src/test/pytest/pypg/__init__.py b/src/test/pytest/pypg/__init__.py new file mode 100644 index 0000000000..e2df717155 --- /dev/null +++ b/src/test/pytest/pypg/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""pypg: PostgreSQL test framework (server management and command helpers). + +The pytest fixtures live in :mod:`pypg.fixtures` and are loaded via the +``-p pypg.fixtures`` option configured in pyproject.toml. +""" + +from .command import CommandResult, PgBin +from .server import PostgresServer +from .util import append_to_file, slurp_file + +__all__ = [ + "CommandResult", + "PgBin", + "PostgresServer", + "append_to_file", + "slurp_file", +] diff --git a/src/test/pytest/pypg/_env.py b/src/test/pytest/pypg/_env.py new file mode 100644 index 0000000000..0b9ab44a91 --- /dev/null +++ b/src/test/pytest/pypg/_env.py @@ -0,0 +1,45 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Process-environment setup for the test suite. + +Forces a C locale for stable messages and clears any inherited PG* connection +variables that would otherwise steer client programs at the wrong server. Call +:func:`prepare_environment` once before any server is started. +""" + +import os + +# PG* variables that must not leak into the test environment (they would +# override host/port/user/db that the framework sets explicitly). +_PG_VARS_TO_CLEAR = ( + "PGDATABASE", + "PGUSER", + "PGPORT", + "PGHOST", + "PGHOSTADDR", + "PGSERVICE", + "PGSSLMODE", + "PGREQUIRESSL", + "PGCONNECT_TIMEOUT", + "PGDATA", + "PGCLIENTENCODING", + "PGOPTIONS", +) + +_prepared = False + + +def prepare_environment(): + """Idempotently set C locale and clear PG* variables.""" + global _prepared # pylint: disable=global-statement + if _prepared: + return + os.environ["LC_ALL"] = "C" + os.environ["LC_MESSAGES"] = "C" + for var in _PG_VARS_TO_CLEAR: + os.environ.pop(var, None) + # Default the database to "postgres" so a connection string without an + # explicit dbname (e.g. in load-balancing tests) does not fall through to + # the OS user name. + os.environ["PGDATABASE"] = "postgres" + _prepared = True diff --git a/src/test/pytest/pypg/command.py b/src/test/pytest/pypg/command.py new file mode 100644 index 0000000000..fca68812b2 --- /dev/null +++ b/src/test/pytest/pypg/command.py @@ -0,0 +1,177 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Running programs and asserting on their results. + +A :class:`PgBin` runs binaries from a given bindir (optionally with extra +environment, e.g. a node's PGHOST/PGPORT) and asserts on exit code, stdout and +stderr. Failures raise AssertionError so pytest/pgtap report them. +""" + +import os +from dataclasses import dataclass +from typing import Optional, Sequence + +from .util import TIMEOUT_DEFAULT, run_captured + + +@dataclass +class CommandResult: + """Outcome of running a command.""" + + returncode: int + stdout: str + stderr: str + + +# Mirrors program_help_ok's line-length convention. +_MAX_HELP_LINE_LENGTH = 95 + + +def _describe(cmd: Sequence[str], result: CommandResult) -> str: + return "command: {}\nexit code: {}\nstderr:\n{}\nstdout:\n{}".format( + " ".join(str(c) for c in cmd), result.returncode, result.stderr, result.stdout + ) + + +class PgBin: + """Runs PostgreSQL binaries located in *bindir*.""" + + def __init__(self, bindir, extra_env: Optional[dict] = None): + self.bindir = str(bindir) + self.extra_env = dict(extra_env or {}) + + def command_env(self, extra_env: Optional[dict]) -> dict: + env = dict(os.environ) + env.update(self.extra_env) + if extra_env: + env.update(extra_env) + # A None value means "unset this variable" (e.g. to drop an inherited + # TZ), since subprocess env values must all be strings. + return {k: v for k, v in env.items() if v is not None} + + def resolve(self, name): + """Resolve a program name to its path within bindir if present.""" + candidate = os.path.join(self.bindir, name) + return candidate if os.path.exists(candidate) else name + + def result( + self, cmd: Sequence[str], *, extra_env=None, timeout=TIMEOUT_DEFAULT + ) -> CommandResult: + """Run *cmd* (list) and capture its result. cmd[0] is resolved in bindir. + + A wedged program is killed after *timeout* seconds (raising + subprocess.TimeoutExpired); pass timeout=None to wait indefinitely. + """ + argv = [self.resolve(cmd[0]), *map(str, cmd[1:])] + print("# Running: " + " ".join(argv)) + # Capture via files, not pipes: a program that launches a server (e.g. + # "pg_ctl start") leaves the postmaster holding the pipe open on + # Windows, which would deadlock the read. Output is decoded leniently + # since programs may emit non-UTF-8 bytes (e.g. LATIN1 object names) + # that we only regex-match. + returncode, stdout, stderr = run_captured( + argv, env=self.command_env(extra_env), timeout=timeout + ) + return CommandResult(returncode, stdout, stderr) + + # -- command_* assertions ----------------------------------------------- + + def command_ok(self, cmd, msg=None, *, extra_env=None) -> CommandResult: + res = self.result(cmd, extra_env=extra_env) + assert res.returncode == 0, ( + (msg or "command should succeed") + "\n" + _describe(cmd, res) + ) + return res + + def command_fails(self, cmd, msg=None, *, extra_env=None) -> CommandResult: + res = self.result(cmd, extra_env=extra_env) + assert res.returncode != 0, ( + (msg or "command should fail") + "\n" + _describe(cmd, res) + ) + return res + + def command_exit_is(self, cmd, code, msg=None, *, extra_env=None) -> CommandResult: + res = self.result(cmd, extra_env=extra_env) + assert res.returncode == code, ( + (msg or f"exit code should be {code}") + "\n" + _describe(cmd, res) + ) + return res + + def command_like(self, cmd, pattern, msg=None, *, extra_env=None) -> CommandResult: + import re + + res = self.result(cmd, extra_env=extra_env) + assert res.returncode == 0, ( + (msg or "command should succeed") + "\n" + _describe(cmd, res) + ) + assert res.stderr == "", (msg or "no stderr") + "\n" + _describe(cmd, res) + assert re.search(pattern, res.stdout), ( + (msg or "stdout should match") + f" /{pattern}/\n" + _describe(cmd, res) + ) + return res + + def command_fails_like( + self, cmd, pattern, msg=None, *, extra_env=None + ) -> CommandResult: + import re + + res = self.result(cmd, extra_env=extra_env) + assert res.returncode != 0, ( + (msg or "command should fail") + "\n" + _describe(cmd, res) + ) + assert re.search(pattern, res.stderr), ( + (msg or "stderr should match") + f" /{pattern}/\n" + _describe(cmd, res) + ) + return res + + def command_checks_all( + self, cmd, expected_ret, stdout_res, stderr_res, msg=None, *, extra_env=None + ) -> CommandResult: + """Check exit code plus a list of stdout and stderr regexes.""" + import re + + res = self.result(cmd, extra_env=extra_env) + label = msg or "command" + assert res.returncode == expected_ret, ( + f"{label} status (got {res.returncode} vs expected {expected_ret})\n" + + _describe(cmd, res) + ) + for pattern in stdout_res: + assert re.search( + pattern, res.stdout + ), f"{label} stdout /{pattern}/\n" + _describe(cmd, res) + for pattern in stderr_res: + assert re.search( + pattern, res.stderr + ), f"{label} stderr /{pattern}/\n" + _describe(cmd, res) + return res + + # -- program_* assertions ----------------------------------------------- + + def program_help_ok(self, name): + res = self.result([name, "--help"]) + assert res.returncode == 0, f"{name} --help exit code 0\n" + _describe( + [name, "--help"], res + ) + assert res.stdout != "", f"{name} --help goes to stdout" + assert res.stderr == "", f"{name} --help nothing to stderr:\n{res.stderr}" + long_lines = [ + ln for ln in res.stdout.splitlines() if len(ln) > _MAX_HELP_LINE_LENGTH + ] + assert not long_lines, ( + f"{name} --help maximum line length (>{_MAX_HELP_LINE_LENGTH}):\n" + + "\n".join(long_lines) + ) + + def program_version_ok(self, name): + res = self.result([name, "--version"]) + assert res.returncode == 0, f"{name} --version exit code 0\n" + _describe( + [name, "--version"], res + ) + assert res.stdout != "", f"{name} --version goes to stdout" + assert res.stderr == "", f"{name} --version nothing to stderr:\n{res.stderr}" + + def program_options_handling_ok(self, name): + res = self.result([name, "--not-a-valid-option"]) + assert res.returncode != 0, f"{name} with invalid option nonzero exit code" + assert res.stderr != "", f"{name} with invalid option prints error message" diff --git a/src/test/pytest/pypg/fixtures.py b/src/test/pytest/pypg/fixtures.py new file mode 100644 index 0000000000..7bbf92ce18 --- /dev/null +++ b/src/test/pytest/pypg/fixtures.py @@ -0,0 +1,371 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Pytest fixtures for PostgreSQL tests. + +Loaded as a plugin (``-p pypg.fixtures``). Provides the building blocks tests +use: ``pg_bin`` to run client programs, ``create_pg`` to spin up servers, and +``pg``/``conn`` for the common single-server case. Servers are cleaned up +automatically at the end of the test. +""" + +import os +import pathlib +import re +import shutil +import socket +import subprocess + +import pytest + +from . import _env +from .command import PgBin +from .server import PostgresServer +from .util import short_tempdir + + +@pytest.fixture(scope="session", autouse=True) +def _prepare_env(): + """Force C locale and clear PG* variables before anything starts.""" + _env.prepare_environment() + + +def _pg_config_value(pg_config, option): + return subprocess.run( + [pg_config, option], stdout=subprocess.PIPE, text=True, check=True + ).stdout.strip() + + +@pytest.fixture(scope="session") +def pg_config(): + path = shutil.which("pg_config") + if not path: + pytest.skip("pg_config not found on PATH") + return path + + +@pytest.fixture(scope="session") +def bindir(pg_config): + return _pg_config_value(pg_config, "--bindir") + + +@pytest.fixture(scope="session") +def libdir(pg_config): + return _pg_config_value(pg_config, "--libdir") + + +@pytest.fixture(scope="session", autouse=True) +def _check_libpq_abi(libdir): + """Skip the suite when this Python cannot load the build's libpq. + + The in-process libpq layer is loaded via ctypes, so the interpreter must + match libpq's ABI. A 64-bit Python cannot dlopen the 32-bit libpq from a + ``-m32`` build, which would otherwise fail every test with an OSError; skip + with a clear reason instead. See findlib.libpq_abi_skip_reason. + """ + from libpq.findlib import libpq_abi_skip_reason + + reason = libpq_abi_skip_reason(libdir) + if reason: + pytest.skip(reason) + + +@pytest.fixture(scope="session") +def pg_bin(bindir): + """A PgBin for running client programs that do not need a server.""" + return PgBin(bindir) + + +def _safe_node_name(request): + return re.sub(r"[^A-Za-z0-9_.-]", "_", request.node.name) + + +def _test_failed(request): + """Whether the test's setup or call phase failed. + + Reads the per-phase reports stashed by pgtap's pytest_runtest_makereport + hook; absent (e.g. plugin not active) is treated as "not failed". + """ + return any( + getattr(getattr(request.node, f"_phase_{p}", None), "failed", False) + for p in ("setup", "call") + ) + + +@pytest.fixture +def test_datadir(request, tmp_path_factory): + """The per-test directory for servers and other test data. + + Under the meson/testwrap harness this is rooted at the per-file TESTDATADIR + (as PostgreSQL::Test::Utils uses for tmp_check); for a standalone pytest run + it falls back to a directory minted from pytest's tmp_path_factory. Using + TESTDATADIR rather than pytest's shared pytest-of- base matters + because that base is created with mode 0o700: on Windows that is an + owner-only ACL the postmaster's restricted access token cannot use, so + server-created paths under it (data dirs, tablespaces) fail with + "Permission denied". + + TESTDATADIR is shared by every test function in a file, so append a + per-function subdirectory to keep functions from colliding. + """ + root = os.environ.get("TESTDATADIR") + safe = _safe_node_name(request) + if not root: + # pytest's tmp_path_factory manages its own cleanup (keeps the last + # few runs), so just hand out a fresh directory. + yield tmp_path_factory.mktemp(safe, numbered=True) + return + path = pathlib.Path(root) / safe + path.mkdir(parents=True, exist_ok=True) + yield path + # TESTDATADIR is shared by every test in the file and kept for the whole CI + # job, so removing this test's data (server data dirs, WAL, aux server + # dirs, scratch) on success keeps it from accumulating gigabytes and + # filling the runner disk. Keep it on failure for post-mortem, like the + # Perl framework. + if not _test_failed(request): + shutil.rmtree(path, ignore_errors=True) + + +@pytest.fixture +def tmp_path(test_datadir): + """Override pytest's built-in tmp_path so per-test scratch space lives + under the harness directory (TESTDATADIR) rather than pytest's own base. + + pytest creates its base (pytest-of-) with mode 0o700, which on Windows + is an owner-only ACL the postmaster's restricted access token cannot use -- + initdb, tablespaces and other server-created paths under it then fail with + "Permission denied". Rooting at TESTDATADIR (created by the harness with an + inheritable ACL) avoids that and is where servers already live. + """ + return test_datadir + + +def _free_port(): + """Return an unused TCP port number (used to name the unix socket).""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +@pytest.fixture +def create_pg(bindir, libdir, test_datadir): + """Factory creating PostgresServer instances, torn down after the test. + + ``create_pg(name="main", start=True, initdb_extra=None, + allows_streaming=False, has_archiving=False, has_restoring=False, + port=None, own_host=False)`` returns an initialized (and, by default, + started) server; ``initdb_extra`` is a list of extra arguments passed to + initdb (e.g. ``["--no-data-checksums"]``). The streaming/archiving/ + restoring flags are forwarded to :meth:`PostgresServer.init`. + + ``port`` pins an explicit port (otherwise a free one is chosen); ``own_host`` + binds the node to its own loopback address (127.0.0.1, .2, .3, ...) over + TCP. Together they let several nodes share one port, distinguished by IP + -- needed for DNS-based load-balancing tests. + + Data dirs live under the test's own data directory; the unix socket lives + in a short /tmp directory to stay within the socket path length limit. + """ + servers = [] + sockdirs = [] + # Per-test loopback-IP counter for own_host nodes (127.0.0.1, .2, .3, ...). + own_host_counter = [0] + + def _create( + name="main", + *, + start=True, + initdb_extra=None, + allows_streaming=False, + has_archiving=False, + has_restoring=False, + port=None, + own_host=False, + ): + sockdir = short_tempdir() + sockdirs.append(sockdir) + listen_host = None + if own_host: + own_host_counter[0] += 1 + if own_host_counter[0] > 254: + raise RuntimeError("too many own_host nodes") + listen_host = f"127.0.0.{own_host_counter[0]}" + server = PostgresServer( + name, + bindir, + libdir, + str(test_datadir / name), + _free_port() if port is None else port, + sockdir, + listen_host=listen_host, + ) + # Track for teardown before init()/start(), so a failure in either + # still stops a postmaster that may have been left running (start() + # can set running=True and then raise when pg_ctl times out but the + # postmaster is in fact alive). + servers.append(server) + server.init( + extra=initdb_extra, + allows_streaming=allows_streaming, + has_archiving=has_archiving, + has_restoring=has_restoring, + ) + if start: + server.start() + return server + + yield _create + + for server in servers: + server.teardown() + for sockdir in sockdirs: + shutil.rmtree(sockdir, ignore_errors=True) + + +@pytest.fixture +def tempdir_short(): + """A temporary directory with a short pathname, removed after the test. + + Some uses need a path short enough to fit tar's ~100-byte symlink-target + limit -- notably tablespace locations, whose symlinks are written into a + base backup's tar stream. The per-test data directory can exceed that (its + deep layout is especially long on macOS), so use a directory directly under + the system temp area instead. + """ + d = short_tempdir() + yield d + shutil.rmtree(d, ignore_errors=True) + + +@pytest.fixture +def pg(create_pg): + """A single started PostgresServer for the test.""" + return create_pg("main") + + +@pytest.fixture +def conn(pg): + """A persistent libpq Session connected to the ``pg`` server's postgres + database, closed at the end of the test.""" + sess = pg.connect() + yield sess + sess.close() + + +@pytest.fixture +def ldap_server(test_datadir): + """Factory creating LdapServer (slapd) instances, stopped after the test. + + ``ldap_server(rootpw, authtype)`` returns a running server (authtype is + 'users' or 'anonymous'). Skips the test if no usable slapd is found. The + SSL certs come from src/test/ssl/ssl. + """ + from . import ldapserver + + if not ldapserver.AVAILABLE: + pytest.skip(ldapserver.SETUP_ERROR) + + # repo root is four levels up from this file (src/test/pytest/pypg). + repo = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..", "..", "..") + ) + certdir = os.path.join(repo, "src", "test", "ssl", "ssl") + + servers = [] + counter = [0] + + def _create(rootpw, authtype): + counter[0] += 1 + basedir = test_datadir / f"ldap{counter[0]}" + basedir.mkdir() + server = ldapserver.LdapServer(basedir, rootpw, authtype, certdir) + servers.append(server) + return server + + yield _create + + for server in servers: + server.stop() + + +@pytest.fixture +def kerberos(test_datadir): + """Factory creating a Kerberos KDC, stopped after the test. + + ``kerberos(host, hostaddr, realm, srvnam="postgres")`` sets up a realm + + service principal and starts krb5kdc, setting KRB5_CONFIG/KRB5_KDC_PROFILE/ + KRB5CCNAME in the environment (so a postmaster started afterward inherits + them). Those env vars are restored at teardown. Skips if MIT krb5 is not + installed. + """ + from . import kerberos as krb + + if not krb.AVAILABLE: + pytest.skip(krb.SETUP_ERROR) + + saved_env = { + k: os.environ.get(k) for k in ("KRB5_CONFIG", "KRB5_KDC_PROFILE", "KRB5CCNAME") + } + kdcs = [] + counter = [0] + + def _create(host, hostaddr, realm, srvnam="postgres"): + counter[0] += 1 + basedir = test_datadir / f"krb{counter[0]}" + basedir.mkdir() + kdc = krb.Kerberos(basedir, host, hostaddr, realm, srvnam) + kdcs.append(kdc) + return kdc + + yield _create + + for kdc in kdcs: + kdc.stop() + for k, v in saved_env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + +@pytest.fixture +def oauth_server(): + """Factory starting the mock OAuth provider (oauth_server.py). + + ``oauth_server(script_path)`` returns a running OAuthServer with a ``.port`` + attribute; it is stopped at teardown. + """ + from .oauthserver import OAuthServer + + servers = [] + + def _create(script): + srv = OAuthServer(script) + servers.append(srv) + return srv + + yield _create + + for srv in servers: + srv.stop() + + +@pytest.fixture +def ssl_server(bindir, test_datadir): + """An SSLServer (OpenSSL backend) for configuring a cluster for SSL. + + Skips the test unless this build uses OpenSSL (with_ssl=openssl). Client + keys are copied, with private permissions, under the test data directory. + """ + from .ssl_server import SSLServer + + if os.environ.get("with_ssl") != "openssl": + pytest.skip("SSL not supported by this build") + + repo = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..", "..", "..") + ) + ssl_dir = os.path.join(repo, "src", "test", "ssl", "ssl") + keydir = test_datadir / "ssl-keys" + keydir.mkdir() + return SSLServer(ssl_dir, keydir, bindir) diff --git a/src/test/pytest/pypg/server.py b/src/test/pytest/pypg/server.py new file mode 100644 index 0000000000..9961b918bc --- /dev/null +++ b/src/test/pytest/pypg/server.py @@ -0,0 +1,1315 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""A managed PostgreSQL server instance for tests. + +:class:`PostgresServer` manages the lifecycle (initdb, start, stop, restart, +reload), configuration, and query execution of a server instance. Queries run +in-process through :class:`libpq.Session` (no psql subprocess); the command_* +helpers run client programs with this server's PGHOST/PGPORT. +""" + +import os +import re +import shutil +import signal +import socket +import subprocess +import time + +from libpq import Session, conninfo_quote +from libpq.errors import PqConnectionError, QueryError + +from .command import CommandResult, PgBin +from .util import ( + TIMEOUT_DEFAULT, + USE_UNIX_SOCKETS, + WINDOWS_OS, + poll_until, + run_captured, +) + + +def _pid_alive(pid): + """Whether process *pid* currently exists. + + On Windows ``os.kill(pid, 0)`` would terminate the process (any signal maps + to TerminateProcess), so probe with OpenProcess/GetExitCodeProcess instead. + """ + if WINDOWS_OS: + import ctypes + + PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 + STILL_ACTIVE = 259 + kernel32 = ctypes.windll.kernel32 + handle = kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, pid) + if not handle: + return False + try: + code = ctypes.c_ulong() + if not kernel32.GetExitCodeProcess(handle, ctypes.byref(code)): + return False + return code.value == STILL_ACTIVE + finally: + kernel32.CloseHandle(handle) + try: + os.kill(pid, 0) + except OSError: + return False + return True + + +# The breadth of this class matches the cluster-management API the tests +# need; splitting it would not make the tests clearer. +class PostgresServer: # pylint: disable=too-many-public-methods + """One initdb'd data directory and the server running on it.""" + + def __init__(self, name, bindir, libdir, basedir, port, sockdir, listen_host=None): + self.name = name + self._bindir = str(bindir) + self.libdir = str(libdir) + self.basedir = str(basedir) + self.port = int(port) + self._sockdir = str(sockdir) + # The connection host. When listen_host is given (own_host), bind that + # loopback address over TCP; combined with an explicit shared port this + # lets several nodes coexist, distinguished by IP. Otherwise use the + # socket directory with Unix-domain sockets, or 127.0.0.1 on TCP + # (Windows). Backslashes in a Windows socket path are converted to '/' + # so the value is valid in postgresql.conf. + self._own_host = listen_host is not None + if self._own_host: + self.host = listen_host + elif USE_UNIX_SOCKETS: + self.host = self._sockdir.replace("\\", "/") + else: + self.host = "127.0.0.1" + self.running = False + self._logfile_generation = 0 + os.makedirs(self.basedir, exist_ok=True) + + # -- paths / connection info -------------------------------------------- + + @property + def data_dir(self): + return os.path.join(self.basedir, "pgdata") + + @property + def logfile(self): + # Generation 0 keeps the plain "server.log" name that other tests and + # helpers expect; rotate_logfile() bumps the generation for tests that + # must keep a fresh log across a restart. + if self._logfile_generation == 0: + return os.path.join(self.basedir, "server.log") + return os.path.join(self.basedir, f"server_{self._logfile_generation}.log") + + def rotate_logfile(self): + """Switch to a fresh log file for the next start. + + Needed where the old log can't be reopened for writing (e.g. on + Windows) or a test wants to scan only the newest postmaster's output. + """ + self._logfile_generation += 1 + return self.logfile + + @property + def pidfile(self): + return os.path.join(self.data_dir, "postmaster.pid") + + @property + def backup_dir(self): + """Output path for backups taken with :meth:`backup`.""" + return os.path.join(self.basedir, "backup") + + @property + def archive_dir(self): + """Directory WAL is archived into when has_archiving is enabled.""" + return os.path.join(self.basedir, "archives") + + @property + def bindir(self): + return self._bindir + + def connstr(self, dbname="postgres"): + return ( + f"host='{conninfo_quote(self.host)}' port={self.port} " + f"dbname='{conninfo_quote(dbname)}'" + ) + + def _listen_conf_lines(self): + """postgresql.conf lines selecting the connection transport. + + Unix-domain sockets in this node's private directory, or TCP on the + loopback address (Windows, or any own_host node). + """ + if self._own_host or not USE_UNIX_SOCKETS: + return [ + f"listen_addresses = '{self.host}'", + "unix_socket_directories = ''", + ] + return [ + "listen_addresses = ''", + f"unix_socket_directories = '{self.host}'", + ] + + def raw_connect(self): + """Open and return a raw socket to the server, caller closes it. + + Connects to the Unix-domain socket (``/.s.PGSQL.``) or, on + TCP, to (host, port), for tests that speak the wire protocol directly + or just consume a connection. + """ + if USE_UNIX_SOCKETS: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(os.path.join(self.host, f".s.PGSQL.{self.port}")) + else: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((self.host, self.port)) + return sock + + def raw_connect_works(self): + """Whether :meth:`raw_connect` is usable on this platform. + + Always true on TCP. With Unix-domain sockets it needs a working + AF_UNIX implementation (absent on some Windows pythons); probe once. + """ + if USE_UNIX_SOCKETS: + if not hasattr(socket, "AF_UNIX"): + return False + try: + self.raw_connect().close() + except OSError: + return False + return True + + @property + def pg_bin(self): + """A PgBin whose environment targets this server. + + Sets PGHOST/PGPORT and defaults PGDATABASE to 'postgres', so client + programs invoked without an explicit database connect to a database + that exists. + """ + return PgBin( + self._bindir, + extra_env={ + "PGHOST": self.host, + "PGPORT": str(self.port), + "PGDATABASE": "postgres", + }, + ) + + # -- internal command runner -------------------------------------------- + + def _run(self, *argv, check=True, timeout=TIMEOUT_DEFAULT): + argv = [self.resolve(argv[0]), *map(str, argv[1:])] + print("# Running: " + " ".join(argv)) + # Capture via files, not pipes: "pg_ctl start" leaves a postmaster + # holding the pipe open on Windows, which would deadlock the read. + # A wedged command is killed after *timeout* seconds (TimeoutExpired); + # pass timeout=None to wait indefinitely. + returncode, stdout, _ = run_captured(argv, combine_stderr=True, timeout=timeout) + if check and returncode != 0: + raise RuntimeError( + f"command failed ({returncode}): {' '.join(argv)}\n{stdout}" + ) + return CommandResult(returncode, stdout, "") + + def resolve(self, name): + candidate = os.path.join(self._bindir, name) + return candidate if os.path.exists(candidate) else name + + # -- lifecycle ----------------------------------------------------------- + + def init( + self, + extra=None, + *, + allows_streaming=False, + has_archiving=False, + has_restoring=False, + wal_level=None, + ): + """Run initdb and write a test configuration. + + Keyword params (all default off, preserving the existing minimal + config): + + - ``allows_streaming``: set up postgresql.conf for replication. + Pass ``"logical"`` for ``wal_level = logical``; any other truthy + value yields ``wal_level = replica``. + - ``has_archiving``: enable ``archive_mode`` with an archive_command + that copies WAL into :attr:`archive_dir`. + - ``has_restoring``: accepted but has no effect here; restoring is + actually configured on a standby in :meth:`init_from_backup`. + - ``wal_level``: explicit override of ``wal_level``. + """ + os.makedirs(self.backup_dir, exist_ok=True) + os.makedirs(self.archive_dir, exist_ok=True) + # Pre-create the (empty) data directory so initdb takes its + # "present but empty" path instead of calling pg_mkdir_p. Python's + # makedirs tolerates a concurrent create of a shared parent, whereas + # initdb's pg_mkdir_p does not: under parallel test execution several + # initdb processes race to create a common temp ancestor and all but + # one fail with "File exists". + os.makedirs(self.data_dir, exist_ok=True) + + argv = [ + "initdb", + "-D", + self.data_dir, + "--no-sync", + "--no-instructions", + "-A", + "trust", + "--locale=C", + "--encoding=UTF8", + ] + if extra: + argv += list(extra) + self._run(*argv) + + lines = [] + if allows_streaming: + if wal_level is None: + wal_level = "logical" if allows_streaming == "logical" else "replica" + lines += [ + f"wal_level = {wal_level}", + "max_wal_senders = 10", + "max_replication_slots = 10", + "wal_log_hints = on", + "hot_standby = on", + # conservative settings to ensure we can run multiple postmasters: + "shared_buffers = 1MB", + "max_connections = 10", + # limit disk space consumption, too: + "max_wal_size = 128MB", + ] + elif wal_level is not None: + lines.append(f"wal_level = {wal_level}") + + lines.append(f"port = {self.port}") + lines += self._listen_conf_lines() + lines += ["fsync = off", ""] + self.append_conf("\n".join(lines)) + + if has_archiving: + self.enable_archiving() + + @staticmethod + def _file_copy_command(src, dst): + """A shell command that copies file *src* to *dst*. + + *src*/*dst* may embed the archive/restore ``%p``/``%f`` placeholders. + Elsewhere this is ``cp`` with forward-slash paths. On Windows it is + cmd's ``copy``; there the path's backslashes must be doubled for the + command to work once stored in postgresql.conf and to identify the + target file, and both paths are double-quoted to tolerate spaces. + """ + if WINDOWS_OS: + + def winpath(p): + return p.replace("/", "\\").replace("\\", "\\\\") + + return f'copy "{winpath(src)}" "{winpath(dst)}"' + return f'cp "{src}" "{dst}"' + + def enable_archiving(self): + """Enable WAL archiving into :attr:`archive_dir`. + + Internal helper. + """ + copy_command = self._file_copy_command("%p", f"{self.archive_dir}/%f") + self.append_conf( + "\n".join( + [ + "", + "archive_mode = on", + f"archive_command = '{copy_command}'", + "", + ] + ) + ) + + def append_conf(self, text, filename="postgresql.conf"): + with open(os.path.join(self.data_dir, filename), "a", encoding="utf-8") as fh: + if not text.endswith("\n"): + text += "\n" + fh.write(text) + + def start(self, fail_ok=False): + """Start the postmaster. Returns True on success. + + With *fail_ok* a failed start returns False instead of raising, like + Cluster::start(fail_ok => 1). If pg_ctl reports failure but a + postmaster is in fact still alive (e.g. pg_ctl timed out waiting), the + running flag is set so a later stop() cleans it up. + """ + proc = self._run( + "pg_ctl", + "-D", + self.data_dir, + "-l", + self.logfile, + "-w", + "start", + check=False, + ) + if proc.returncode == 0: + self.running = True + return True + self.running = self.postmaster_alive() + if not fail_ok: + # pg_ctl's own output rarely says why; include the server log, + # which holds the actual startup error. + try: + with open(self.logfile, encoding="utf-8", errors="replace") as fh: + log = fh.read() + except OSError: + log = "(could not read log file)" + raise RuntimeError( + f'pg_ctl start failed for node "{self.name}":\n{proc.stdout}\n' + f"--- {self.logfile} ---\n{log}" + ) + return False + + def stop(self, mode="fast", fail_ok=False): + """Stop the postmaster. Returns True on success (or if not running).""" + if not self.running: + return True + proc = self._run( + "pg_ctl", + "-D", + self.data_dir, + "-m", + mode, + "-w", + "stop", + check=not fail_ok, + ) + # Only clear running on a confirmed stop, so a failed fail_ok stop + # leaves the flag set and teardown can force-kill the survivor. + if proc.returncode == 0: + self.running = False + return True + return False + + def postmaster_pid(self): + """Return the postmaster PID from postmaster.pid, or None.""" + try: + with open(self.pidfile, encoding="utf-8") as fh: + return int(fh.readline().strip()) + except (FileNotFoundError, ValueError): + return None + + def postmaster_alive(self): + pid = self.postmaster_pid() + if pid is None: + return False + return _pid_alive(pid) + + def signal_backend(self, pid, signame): + """Send signal *signame* (e.g. "QUIT", "KILL", "TERM") to process *pid*. + + Uses ``pg_ctl kill``, which delivers the signal through the server's own + mechanism and so works on every platform (Windows has no Unix signals). + """ + self._run("pg_ctl", "kill", signame, str(pid)) + + def kill9(self): + """Hard-kill the postmaster (no chance to clean up). + + Postmaster children normally exit on their own once the postmaster is + gone; a backend stuck in a CPU-bound loop is the exception this test + relies on. + """ + pid = self.postmaster_pid() + if pid is not None: + print(f'### Killing node "{self.name}" using signal 9') + if WINDOWS_OS: + # No SIGKILL on Windows; terminate the process tree forcibly. + subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(pid)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + else: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + self.running = False + + def restart(self, mode="fast"): + self._run( + "pg_ctl", + "-D", + self.data_dir, + "-l", + self.logfile, + "-m", + mode, + "-w", + "restart", + ) + self.running = True + + def reload(self): + self._run("pg_ctl", "-D", self.data_dir, "reload") + + def promote(self): + self._run("pg_ctl", "-D", self.data_dir, "-w", "promote") + + # -- backup / streaming replication -------------------------------------- + + def backup(self, backup_name, backup_options=None): + """Take a pg_basebackup into ``backup_dir/``. + + WAL is fetched at the end of the backup unless overridden via + *backup_options* (e.g. + ``["-X", "stream"]``). The resulting backup is usable by + :meth:`init_from_backup`. + """ + backup_path = os.path.join(self.backup_dir, backup_name) + print(f'# Taking pg_basebackup {backup_name} from node "{self.name}"') + argv = [ + "pg_basebackup", + "--no-sync", + "--pgdata", + backup_path, + "--host", + self.host, + "--port", + str(self.port), + "--checkpoint", + "fast", + ] + if backup_options: + argv += list(backup_options) + self._run(*argv) + print("# Backup finished") + + def backup_fs_cold(self, backup_name): + """Take a filesystem-level cold backup (the server must be stopped). + + Copies the data directory into ``backup_dir/``, excluding + the log directory and postmaster.pid, producing a tree usable by + :meth:`init_from_backup`. + """ + dest = os.path.join(self.backup_dir, backup_name) + os.makedirs(self.backup_dir, exist_ok=True) + shutil.copytree( + self.data_dir, + dest, + symlinks=True, + ignore=shutil.ignore_patterns("log", "postmaster.pid"), + ) + + def init_from_backup( + self, + root_node, + backup_name, + *, + has_streaming=False, + has_restoring=False, + standby=True, + ): + """Initialize this node's data dir from *root_node*'s named backup. + + Plain-format backups only; tar/incremental/tablespace variants are not + supported by this framework. Does not start the node. + + - ``has_streaming``: configure ``primary_conninfo`` pointing at + *root_node* and place ``standby.signal`` (streaming replication). + - ``has_restoring``: configure a ``restore_command`` reading from + *root_node*'s archive dir. Standby mode is used by default; pass + ``standby=False`` for crash-recovery (recovery.signal) mode. + """ + backup_path = os.path.join(root_node.backup_dir, backup_name) + print( + f'# Initializing node "{self.name}" from backup "{backup_name}" ' + f'of node "{root_node.name}"' + ) + if not os.path.isdir(backup_path): + raise RuntimeError( + f'Backup "{backup_name}" does not exist at {backup_path}' + ) + + os.makedirs(self.backup_dir, exist_ok=True) + os.makedirs(self.archive_dir, exist_ok=True) + + # Copy the backup tree into this node's data directory, leaving the + # original backup unmodified. + data_path = self.data_dir + if os.path.isdir(data_path): + shutil.rmtree(data_path) + shutil.copytree(backup_path, data_path, symlinks=True) + os.chmod(data_path, 0o700) + + # Base configuration for this node. + self.append_conf( + "\n".join(["", f"port = {self.port}"] + self._listen_conf_lines() + [""]) + ) + + if has_streaming: + self.enable_streaming(root_node) + if has_restoring: + self.enable_restoring(root_node, standby) + + def enable_streaming(self, root_node): + """Configure streaming replication from *root_node* and go standby. + + Internal helper. The standby's application_name defaults to its node + name so that + :meth:`wait_for_catchup` can locate it in pg_stat_replication. + """ + print(f'### Enabling streaming replication for node "{self.name}"') + root_connstr = ( + f"host={root_node.host} port={root_node.port} " + f"application_name={self.name}" + ) + self.append_conf(f"\nprimary_conninfo='{root_connstr}'\n") + self.set_standby_mode() + + def enable_restoring(self, root_node, standby=True): + """Configure WAL restore from *root_node*'s archive dir. + + Internal helper. + """ + print(f'### Enabling WAL restore for node "{self.name}"') + copy_command = self._file_copy_command(f"{root_node.archive_dir}/%f", "%p") + self.append_conf(f"\nrestore_command = '{copy_command}'\n") + if standby: + self.set_standby_mode() + else: + self.set_recovery_mode() + + def set_standby_mode(self): + """Place a standby.signal file.""" + self.append_conf("", filename="standby.signal") + + def set_recovery_mode(self): + """Place a recovery.signal file.""" + self.append_conf("", filename="recovery.signal") + + # -- LSN / replication progress ------------------------------------------ + + _LSN_MODES = { + "insert": "pg_current_wal_insert_lsn()", + "flush": "pg_current_wal_flush_lsn()", + "write": "pg_current_wal_lsn()", + "receive": "pg_last_wal_receive_lsn()", + "replay": "pg_last_wal_replay_lsn()", + } + + def lsn(self, mode): + """Return the current LSN for *mode*. + + Valid modes: insert, flush, write, receive, replay. Returns ``None`` + if the underlying function returns an empty result. + """ + if mode not in self._LSN_MODES: + raise ValueError( + f"unknown mode for 'lsn': {mode!r}, valid modes are " + + ", ".join(self._LSN_MODES) + ) + result = self.safe_sql(f"SELECT {self._LSN_MODES[mode]}").strip() + return result if result != "" else None + + # -- WAL generation / manipulation --------------------------------------- + + def _insert_lsn_bytes(self): + """Current insert LSN as an integer byte offset (LSN - '0/0').""" + return int(self.safe_sql("SELECT pg_current_wal_insert_lsn() - '0/0'")) + + def emit_wal(self, size): + """Emit *size* bytes of WAL via pg_logical_emit_message; return end LSN. + + Returns the numeric LSN. + """ + return int( + self.safe_sql( + f"SELECT pg_logical_emit_message(true, '', repeat('a', {size})) - '0/0'" + ) + ) + + def advance_wal(self, num): + """Advance WAL by *num* segments.""" + for _ in range(num): + self.safe_sql("SELECT pg_logical_emit_message(false, '', 'foo')") + self.safe_sql("SELECT pg_switch_wal()") + + def advance_wal_out_of_record_splitting_zone(self, wal_block_size): + """Advance WAL away from a page boundary.""" + page_threshold = wal_block_size // 4 + end_lsn = self._insert_lsn_bytes() + page_offset = end_lsn % wal_block_size + while page_offset >= wal_block_size - page_threshold: + self.emit_wal(page_threshold) + end_lsn = self._insert_lsn_bytes() + page_offset = end_lsn % wal_block_size + return end_lsn + + def advance_wal_to_record_splitting_zone(self, wal_block_size): + """Advance WAL close to a page boundary.""" + record_header_size = 24 + end_lsn = self._insert_lsn_bytes() + page_offset = end_lsn % wal_block_size + # Get fairly close to the end of a page in big steps. + while page_offset <= wal_block_size - 512: + self.emit_wal(wal_block_size - page_offset - 256) + end_lsn = self._insert_lsn_bytes() + page_offset = end_lsn % wal_block_size + # Calibrate the message size to approach 8 bytes at a time. + message_size = wal_block_size - 80 + while page_offset <= wal_block_size - record_header_size: + self.emit_wal(message_size) + end_lsn = self._insert_lsn_bytes() + old_offset = page_offset + page_offset = end_lsn % wal_block_size + delta = page_offset - old_offset + if delta > 8: + message_size -= 8 + elif delta <= 0: + message_size += 8 + return end_lsn + + def write_wal(self, tli, lsn, segment_size, data): + """Write raw *data* bytes at *lsn* in the WAL.""" + segment = lsn // segment_size + offset = lsn % segment_size + path = os.path.join(self.data_dir, "pg_wal", "%08X%08X%08X" % (tli, 0, segment)) + with open(path, "r+b") as fh: + fh.seek(offset) + fh.write(data) + return path + + def wait_for_catchup(self, standby_name, mode="replay", target_lsn=None): + """Wait until *standby_name* has caught up on this (primary) node. + + Polls pg_stat_replication on self until the standby's ``_lsn`` + reaches *target_lsn* (default: this node's current write LSN, or its + replay LSN when self is itself in recovery). + + *standby_name* may be a :class:`PostgresServer` (its name is used as + the application_name) or an application_name string. Valid modes: + sent, write, flush, replay. + """ + valid_modes = ("sent", "write", "flush", "replay") + if mode not in valid_modes: + raise ValueError( + f"unknown mode {mode} for 'wait_for_catchup', valid modes are " + + ", ".join(valid_modes) + ) + + if isinstance(standby_name, PostgresServer): + standby_name = standby_name.name + + if target_lsn is None: + isrecovery = self.safe_sql("SELECT pg_is_in_recovery()").strip() + target_lsn = self.lsn("replay" if isrecovery == "t" else "write") + if target_lsn is None: + raise RuntimeError( + f"{self.name}: could not determine a target LSN for " + "wait_for_catchup (no WAL position reported yet)" + ) + + print( + f"Waiting for replication conn {standby_name}'s {mode}_lsn to pass " + f"{target_lsn} on {self.name}" + ) + # Match the connection whose application_name is *standby_name*. + # Standbys with a tool-generated primary_conninfo (pg_rewind / + # pg_basebackup --write-recovery-conf) connect without setting + # application_name and so report 'walreceiver'; fall back to that, but + # only when no connection with the requested name exists. Otherwise an + # unrelated 'walreceiver' connection (e.g. a physical standby running + # alongside a named logical subscriber) would also match, and the + # per-row query would return more than one row, which + # poll_query_until's single-"t" comparison never satisfies. + query = ( + f"SELECT '{target_lsn}' <= {mode}_lsn AND state = 'streaming' " + "FROM pg_catalog.pg_stat_replication " + f"WHERE application_name = '{standby_name}' " + " OR (application_name = 'walreceiver' " + " AND NOT EXISTS (SELECT 1 FROM pg_catalog.pg_stat_replication " + f" WHERE application_name = '{standby_name}'))" + ) + if not self.poll_query_until(query): + details = self.safe_sql("SELECT * FROM pg_catalog.pg_stat_replication") + raise TimeoutError( + "timed out waiting for catchup\n" + f"Last pg_stat_replication contents:\n{details}" + ) + print("done") + + def wait_for_event(self, backend_type, wait_event_name): + """Wait until a *backend_type* backend reaches *wait_event_name*. + + Polls pg_stat_activity; used with injection points and other tests + that synchronize on a backend reaching a specific wait event. + """ + if not self.poll_query_until( + "SELECT count(*) > 0 FROM pg_stat_activity " + f"WHERE backend_type = '{backend_type}' " + f"AND wait_event = '{wait_event_name}'" + ): + raise TimeoutError( + f"timed out waiting for {backend_type} to reach wait event " + f"'{wait_event_name}'" + ) + + def wait_for_replay_catchup(self, standby_name, base_node=None): + """Wait for *standby_name*'s replay_lsn to reach *base_node*'s flush LSN. + + *base_node* defaults to self. + """ + if base_node is None: + base_node = self + self.wait_for_catchup(standby_name, "replay", base_node.lsn("flush")) + + def wait_for_subscription_sync( + self, publisher=None, subname=None, dbname="postgres" + ): + """Wait for logical replication initial sync to complete. + + Polls pg_subscription_rel until all tables are in 'r'/'s' state; if + *publisher* is given, additionally waits for the publisher to catch up + to *subname*. + """ + print(f'Waiting for all subscriptions in "{self.name}" to synchronize data') + query = ( + "SELECT count(1) = 0 FROM pg_subscription_rel " + "WHERE srsubstate NOT IN ('r', 's')" + ) + if not self.poll_query_until(query, dbname=dbname): + details = self.safe_sql("SELECT * FROM pg_subscription_rel", dbname=dbname) + raise TimeoutError( + "timed out waiting for subscriber to synchronize data\n" + f"Last pg_subscription_rel contents:\n{details}" + ) + if publisher is not None: + if subname is None: + raise ValueError("subscription name must be specified") + publisher.wait_for_catchup(subname) + print("done") + + # -- query execution (in-process via libpq) ----------------------------- + + def _open_session(self, dbname): + """Open a Session, retrying briefly past a server that is still + starting up or in recovery. + + Just after start()/restart() (or while a node finishes crash recovery) + the server can transiently reject connections with "the database system + is starting up", "... is not yet accepting connections" or "... is in + recovery mode" before it is ready; those states are temporary for a + node that is meant to be up, so retry until the deadline rather than + fail the whole test on a startup race. + """ + deadline = time.monotonic() + TIMEOUT_DEFAULT + transient_markers = ( + "is starting up", + "not yet accepting connections", + "in recovery mode", + "Connection refused", # postmaster not listening on its port yet + "No such file or directory", # Unix socket file not created yet + ) + while True: + try: + return Session(connstr=self.connstr(dbname), libdir=self.libdir) + except PqConnectionError as exc: + transient = any(m in str(exc) for m in transient_markers) + if not transient or time.monotonic() > deadline: + raise + time.sleep(0.1) + + def session(self, dbname="postgres"): + """Open a fresh persistent libpq Session for *dbname*. + + Each call returns its own independent connection -- sessions never share + state (GUCs, temp tables, transactions). The caller owns it and should + close() it; otherwise it is dropped when the server is stopped. + """ + return self._open_session(dbname) + + def connect(self, dbname="postgres", user=None, password=None, options=None): + """Open a fresh (uncached) libpq Session with extra connection params. + + Use this when a test needs to connect as a specific role, with a + password, or with per-connection GUCs (the libpq "options" keyword, + equivalent to PGOPTIONS). The caller owns the returned Session and + should close() it. + """ + connstr = self.connstr(dbname) + if user is not None: + connstr += f" user='{conninfo_quote(user)}'" + if password is not None: + connstr += f" password='{conninfo_quote(password)}'" + if options is not None: + connstr += f" options='{conninfo_quote(options)}'" + return Session(connstr=connstr, libdir=self.libdir) + + # -- connection-attempt assertions (auth tests) ------------------------- + + def _full_connstr(self, connstr): + """Combine this node's host/port/dbname with a test *connstr*. + + Callers of :meth:`connect_ok` / :meth:`connect_fails` pass a partial + conninfo (e.g. "user=test1 require_auth=password") carrying just the + auth bits. Here we prepend the node's host/port/dbname; later keywords + win in libpq, so anything the caller specifies (dbname, user, sslmode, + ...) overrides the defaults. + """ + return f"{self.connstr('postgres')} {connstr}" + + def log_check(self, test_name, offset, *, log_like=None, log_unlike=None): + """Check the log written since *offset* against regex lists. + + Every pattern in *log_like* must match; none in *log_unlike* may. + """ + if not log_like and not log_unlike: + return + contents = self.log_content()[offset:] + for regex in log_like or []: + assert re.search(regex, contents), f"{test_name}: log matches {regex!r}" + for regex in log_unlike or []: + assert not re.search( + regex, contents + ), f"{test_name}: log does not match {regex!r}" + + def attempt_connection(self, connstr, sql): + """Connect with *connstr* via a psql subprocess and run *sql*. + + Returns ``(ok, stdout, stderr)`` (psql's exit status, stdout and + stderr). A subprocess -- not the in-process Session -- is used so the + child inherits the environment (PGPASSWORD and the like) and performs a + real connection handshake: that connection-time behavior is exactly what + the auth/SSL tests exercise, and relying on the in-process library to + read the environment is not portable. ``-w`` keeps psql from blocking + on a password prompt; ``-XAtq`` gives unaligned, tuples-only, quiet + output (no command tags like ``CREATE TABLE``), matching what the + assertions expect. + """ + argv = [ + "psql", + "-w", + "-X", + "-A", + "-q", + "-t", + "-d", + self._full_connstr(connstr), + "-c", + sql if sql is not None else "SELECT 1", + ] + res = self.pg_bin.result(argv) + return res.returncode == 0, res.stdout, res.stderr + + def connect_ok( + self, + connstr, + test_name, + *, + sql=None, + expected_stdout=None, + stdout_unlike=None, + expected_stderr=None, + log_like=None, + log_unlike=None, + ): + """Assert a connection with *connstr* succeeds. + + Connects with a psql subprocess, runs *sql* (default a trivial SELECT), + and checks stdout/stderr and the server log. *expected_stdout* (if + given) must match the query output; *stdout_unlike* (if given) must + not. + """ + if sql is None: + sql = f"SELECT $$connected with {connstr}$$" + log_location = self.log_position() + + ok, stdout, stderr = self.attempt_connection(connstr, sql) + + assert ok, f"{test_name}: connection should succeed\n{stderr}" + if expected_stdout is not None: + assert re.search( + expected_stdout, stdout + ), f"{test_name}: stdout matches {expected_stdout!r}, got {stdout!r}" + if stdout_unlike is not None: + assert not re.search(stdout_unlike, stdout), ( + f"{test_name}: stdout must not match {stdout_unlike!r}, " + f"got {stdout!r}" + ) + if expected_stderr is not None: + assert re.search( + expected_stderr, stderr + ), f"{test_name}: stderr matches {expected_stderr!r}, got {stderr!r}" + else: + assert stderr == "", f"{test_name}: no stderr, got {stderr!r}" + self.log_check( + test_name, log_location, log_like=log_like, log_unlike=log_unlike + ) + + def connect_fails( + self, + connstr, + test_name, + *, + expected_stderr=None, + log_like=None, + log_unlike=None, + ): + """Assert a connection with *connstr* fails. + + When log_like/log_unlike are given, first wait for the backend + fork/exit log records so the relevant lines are present before checking. + """ + log_location = self.log_position() + + # If the connection unexpectedly succeeds, the trivial query surfaces + # any error; "failed" is then false and the assertion below fires. + ok, _stdout, stderr = self.attempt_connection(connstr, "SELECT 1") + failed = not ok + + assert failed, f"{test_name}: connection should fail" + if expected_stderr is not None: + assert re.search( + expected_stderr, stderr + ), f"{test_name}: stderr matches {expected_stderr!r}, got {stderr!r}" + if log_like or log_unlike: + # Wait for the failed backend's log records to be flushed before + # checking. On most platforms the postmaster logs a per-backend + # "forked new client backend, pid=N socket=..." record that can be + # paired with the matching "client backend (PID N) exited" record; + # Windows does not log the fork record, so wait for the exit record + # alone (connect_fails issues one connection at a time, so the next + # backend exit after this point is the one we triggered). + if WINDOWS_OS: + fork_exit = r"client backend \(PID \d+\) exited with exit code \d" + else: + fork_exit = ( + r"(?s)DEBUG: (?:00000: )?forked new client backend, " + r"pid=(\d+) socket.*DEBUG: (?:00000: )?client backend " + r"\(PID \1\) exited with exit code \d" + ) + self.wait_for_log(fork_exit, log_location) + self.log_check( + test_name, log_location, log_like=log_like, log_unlike=log_unlike + ) + + @staticmethod + def _print_notices(sess): + """Surface a session's captured NOTICE/WARNING output to the test log.""" + notices = sess.get_notices_str() + if notices: + print("#### Begin standard error") + print(notices, end="") + print("\n#### End standard error") + + def sql(self, query, dbname="postgres"): + """Run *query* on a fresh connection and return its ResultData (does not raise). + + See :meth:`safe_sql` for the important caveat about how a multi-statement + *query* is executed as a single implicit transaction. + """ + sess = self._open_session(dbname) + try: + res = sess.query(query) + self._print_notices(sess) + return res + finally: + sess.close() + + def safe_sql(self, query, dbname="postgres"): + """Run *query* on a fresh connection; return its trimmed text output, raising on error. + + Output formatting matches ``psql -A -t`` (rows joined by newlines, + columns by ``|``). Each call opens its own short-lived in-process libpq + connection, runs the query and closes it, so calls do not share session + state (GUCs, temp tables, transaction state); use :meth:`connect` for a + persistent session. + + IMPORTANT -- multiple statements run in ONE implicit transaction. + A *query* with several semicolon-separated statements is sent as a single + libpq command, which wraps them in one implicit transaction. That + differs from psql, which sends each statement separately (one autocommit + transaction each). So a statement that cannot run inside a transaction + block -- CREATE/DROP DATABASE, CREATE/DROP/ALTER SUBSCRIPTION, CREATE + TABLESPACE, VACUUM, CHECKPOINT, REINDEX CONCURRENTLY, + PREPARE/COMMIT/ROLLBACK PREPARED, etc. -- must be issued in its own + ``safe_sql`` call rather than combined with others. Beware too that + grouping unrelated statements gives them shared-transaction semantics + they would not have under psql (e.g. statements meant to run as separate + transactions will instead see each other's uncommitted effects). + """ + sess = self._open_session(dbname) + try: + out = sess.query_safe(query) + self._print_notices(sess) + return out + finally: + sess.close() + + def poll_query_until( + self, query, expected="t", dbname="postgres", timeout=TIMEOUT_DEFAULT + ): + """Run *query* repeatedly until its output equals *expected*. + + Reuses one connection across attempts, reconnecting only if it drops + (e.g. across a server restart), rather than opening a fresh connection + every iteration. + """ + sess = None + + def _ready(): + nonlocal sess + try: + if sess is not None and not sess.connected(): + sess.close() + sess = None + if sess is None: + sess = self._open_session(dbname) + res = sess.query(query) + # An errored query (transient SQL error, or the connection + # dropped -> status -1) is not a match; let other exceptions + # (bugs, timeouts) propagate. + return res.error_message is None and res.psqlout == expected + except (QueryError, PqConnectionError): + # Server not accepting connections yet / connection dropped: + # retry (a dead session is detected via connected() next time). + return False + + try: + return poll_until(_ready, timeout=timeout) + finally: + if sess is not None: + sess.close() + + # -- logical replication slots ------------------------------------------ + + def slot(self, slot_name): + """Return pg_replication_slots columns for *slot_name* as a dict. + + Missing values -- including the case of a nonexistent slot -- come back + as empty strings. + """ + columns = [ + "plugin", + "slot_type", + "datoid", + "database", + "active", + "active_pid", + "xmin", + "catalog_xmin", + "restart_lsn", + ] + res = self.sql( + "SELECT " + ", ".join(columns) + " FROM pg_catalog.pg_replication_slots" + f" WHERE slot_name = '{slot_name}'" + ) + row = res.rows[0] if res.rows else [None] * len(columns) + return {c: ("" if v is None else str(v)) for c, v in zip(columns, row)} + + def log_standby_snapshot(self, standby, slot_name): + """Trigger the xl_running_xacts record a standby logical slot waits for. + + *self* is the primary; *standby* holds the logical *slot_name*. Waits + until the slot's restart_lsn is determined, then logs a standby + snapshot. + """ + assert standby.poll_query_until( + "SELECT restart_lsn IS NOT NULL" + " FROM pg_catalog.pg_replication_slots" + f" WHERE slot_name = '{slot_name}'" + ), "timed out waiting for logical slot to calculate its restart_lsn" + self.safe_sql("SELECT pg_log_standby_snapshot()") + + def create_logical_slot_on_standby(self, primary, slot_name, dbname="postgres"): + """Create logical slot *slot_name* on this standby. + + Starts ``pg_recvlogical --create-slot`` (which blocks until the needed + xl_running_xacts record appears), has *primary* log a standby snapshot + to produce it, then waits for slot creation to finish. + """ + argv = [ + self.resolve("pg_recvlogical"), + "--dbname", + self.connstr(dbname), + "--plugin", + "test_decoding", + "--slot", + slot_name, + "--create-slot", + ] + print("# Running (background): " + " ".join(argv)) + with subprocess.Popen( + argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) as proc: + # Arrange for the xl_running_xacts record pg_recvlogical waits for. + primary.log_standby_snapshot(self, slot_name) + out, err = proc.communicate() + assert ( + self.slot(slot_name)["slot_type"] == "logical" + ), f"could not create slot {slot_name}: stdout={out!r} stderr={err!r}" + + def pg_recvlogical_upto( + self, dbname, slot_name, endpos, timeout_secs=None, **plugin_options + ): + """Read from *slot_name* with pg_recvlogical until *endpos*. + + Returns a CommandResult; on timeout the returncode is None. + ``--no-loop`` prevents pg_recvlogical from internally retrying on error. + """ + argv = [ + self.resolve("pg_recvlogical"), + "--slot", + slot_name, + "--dbname", + self.connstr(dbname), + "--endpos", + str(endpos), + "--file", + "-", + "--no-loop", + "--start", + ] + for k, v in plugin_options.items(): + if "=" in str(k): + raise ValueError( + "= is not permitted to appear in replication option name" + ) + argv += ["--option", f"{k}={v}"] + print("# Running: " + " ".join(argv)) + try: + proc = subprocess.run( + argv, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + check=False, + timeout=timeout_secs if timeout_secs else None, + ) + except subprocess.TimeoutExpired as exc: + return CommandResult( + None, + exc.stdout if isinstance(exc.stdout, str) else "", + exc.stderr if isinstance(exc.stderr, str) else "", + ) + return CommandResult(proc.returncode, proc.stdout, proc.stderr) + + def corrupt_page_checksum(self, file, page_offset=0): + """Invert the pd_checksum field of a page in *file* (offline cluster). + + *file* is relative to the data directory. + """ + path = os.path.join(self.data_dir, file) + # Inverts the pd_checksum field (bytes 8-9 of the page header) only. + mask = b"\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff" + with open(path, "r+b") as fh: + fh.seek(page_offset) + header = bytearray(fh.read(24)) + for i, byte in enumerate(mask): + header[i] ^= byte + fh.seek(page_offset) + fh.write(header) + + # -- server log access --------------------------------------------------- + + def log_content(self): + """Return the entire current server log as text.""" + if not os.path.exists(self.logfile): + return "" + with open(self.logfile, "r", encoding="utf-8", errors="replace") as fh: + return fh.read() + + def log_position(self): + """Return the current end position in the log, for use as an offset. + + This is the character length of :meth:`log_content` (which reads the + file in text mode, normalising CRLF to LF) rather than the raw byte + size. The offset is later used to slice ``log_content()[offset:]``, so + it must be a position in that normalised text: on Windows the log file + has CRLF line endings, and a byte offset would overshoot the folded + text and skip past the lines being checked. + """ + return len(self.log_content()) + + def log_contains(self, pattern, offset=0): + """Return True if *pattern* matches the log at/after byte *offset*.""" + content = self.log_content() + return re.search(pattern, content[offset:]) is not None + + def wait_for_log(self, pattern, offset=0, timeout=TIMEOUT_DEFAULT): + """Wait until *pattern* appears in the log at/after *offset*. + + Returns the log length when matched; raises on timeout. + """ + regex = re.compile(pattern) + + def _found(): + return regex.search(self.log_content()[offset:]) is not None + + if not poll_until(_found, timeout=timeout): + raise TimeoutError(f"timed out waiting for log pattern {pattern!r}") + return len(self.log_content()) + + # -- node-scoped command_* assertions ------------------------------------ + + def command_ok(self, cmd, msg=None): + return self.pg_bin.command_ok(cmd, msg) + + def command_fails(self, cmd, msg=None): + return self.pg_bin.command_fails(cmd, msg) + + def command_like(self, cmd, pattern, msg=None): + return self.pg_bin.command_like(cmd, pattern, msg) + + def command_fails_like(self, cmd, pattern, msg=None): + return self.pg_bin.command_fails_like(cmd, pattern, msg) + + def command_exit_is(self, cmd, code, msg=None): + return self.pg_bin.command_exit_is(cmd, code, msg) + + def command_checks_all(self, cmd, expected_ret, stdout_res, stderr_res, msg=None): + return self.pg_bin.command_checks_all( + cmd, expected_ret, stdout_res, stderr_res, msg + ) + + def issues_sql_like(self, cmd, pattern, msg=None): + """Run *cmd* successfully and assert *pattern* appears in the server log. + + The cluster must have log_statement enabled for the SQL to be logged. + """ + offset = self.log_position() + self.command_ok(cmd, msg) + log = self.log_content()[offset:] + assert re.search(pattern, log), ( + msg or "issues_sql_like" + ) + f": SQL /{pattern}/ not found in server log\n{log}" + + def issues_sql_unlike(self, cmd, pattern, msg=None): + """Run *cmd* successfully and assert *pattern* does NOT appear in the log.""" + offset = self.log_position() + self.command_ok(cmd, msg) + log = self.log_content()[offset:] + assert not re.search(pattern, log), ( + msg or "issues_sql_unlike" + ) + f": SQL /{pattern}/ unexpectedly found in server log\n{log}" + + # -- teardown ------------------------------------------------------------ + + def teardown(self): + if not self.running: + return + try: + self.stop("immediate", fail_ok=True) + except Exception as exc: # pylint: disable=broad-exception-caught + # e.g. pg_ctl stop hung and hit the command timeout. + print(f'### node "{self.name}": stop failed during teardown: {exc}') + # Never leak a postmaster past the test: force-kill any survivor. + if self.postmaster_alive(): + self.kill9() diff --git a/src/test/pytest/pypg/util.py b/src/test/pytest/pypg/util.py new file mode 100644 index 0000000000..0fddc2ec8a --- /dev/null +++ b/src/test/pytest/pypg/util.py @@ -0,0 +1,213 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Small file and polling helpers used by the test framework.""" + +import os +import secrets +import shutil +import subprocess +import sys +import tempfile +import time + +# Default per-operation timeout in seconds (PG_TEST_TIMEOUT_DEFAULT, 180). +TIMEOUT_DEFAULT = int(os.environ.get("PG_TEST_TIMEOUT_DEFAULT") or "180") + +# Connection transport: use Unix-domain sockets everywhere except Windows, +# where we listen on TCP (127.0.0.1). PG_TEST_USE_UNIX_SOCKETS forces Unix +# sockets even on Windows. +WINDOWS_OS = sys.platform in ("win32", "cygwin") +USE_UNIX_SOCKETS = (not WINDOWS_OS) or ("PG_TEST_USE_UNIX_SOCKETS" in os.environ) + + +def run_captured(argv, *, env=None, combine_stderr=False, timeout=None): + """Run *argv*, capturing output through temporary files instead of pipes. + + Returns ``(returncode, stdout, stderr)`` as text. With *combine_stderr*, + stderr is folded into stdout and the returned stderr is ``""``. + + Output is captured to temporary files rather than ``subprocess.PIPE`` + because of how starting a server behaves on Windows: ``pg_ctl start`` + launches a postmaster that inherits and holds open the write end of the + parent's stdout/stderr pipe for its entire lifetime. Reading such a pipe + to end-of-file -- as subprocess does to collect output -- then blocks until + the postmaster exits, i.e. forever. A regular file handle has no + end-of-file dependency on the writer staying alive, so the parent reads the + captured output as soon as the launched program returns. + """ + out = tempfile.TemporaryFile() + err = subprocess.STDOUT if combine_stderr else tempfile.TemporaryFile() + try: + proc = subprocess.run( + argv, env=env, stdout=out, stderr=err, timeout=timeout, check=False + ) + out.seek(0) + stdout = _decode(out.read()) + if combine_stderr: + stderr = "" + else: + err.seek(0) + stderr = _decode(err.read()) + finally: + out.close() + if err is not subprocess.STDOUT: + err.close() + return proc.returncode, stdout, stderr + + +def _decode(data): + """Decode captured output as text, translating newlines like text mode. + + Programs may emit non-UTF-8 bytes (e.g. LATIN1 object names) that we only + regex-match, so decode leniently. Reading a file gives no universal-newline + handling, so fold CRLF/CR to LF to match what text-mode capture produced and + what tests expect. + """ + text = data.decode("utf-8", "replace") + return text.replace("\r\n", "\n").replace("\r", "\n") + + +def dir_symlink(target, link): + """Create *link* as a link to directory *target*. + + On Windows create a junction (via cmd's ``mklink /j``): that is what the + server expects for linked directories such as pg_wal, and an ordinary + symlink there is read back with an error. Elsewhere create a real symlink. + """ + if WINDOWS_OS: + target = target.replace("/", "\\") + link = link.replace("/", "\\") + subprocess.run(["cmd", "/c", "mklink", "/j", link, target], check=True) + else: + os.symlink(target, link) + + +def remove_dir_symlink(link): + """Remove a link created by :func:`dir_symlink`. + + On Windows the link is a directory junction, which must be removed with + os.rmdir -- os.remove/os.unlink fail on it with "Access is denied". + Elsewhere it is an ordinary symlink, removed with os.unlink. + """ + if WINDOWS_OS: + os.rmdir(link) + else: + os.unlink(link) + + +def enable_localhost_tcp(node): + """Make *node* also listen on localhost TCP (no-op except on Windows). + + pg_upgrade connects to the clusters it starts over localhost TCP on Windows + -- it never uses Unix sockets there (see the ``#if !defined(WIN32)`` guard + in src/bin/pg_upgrade/server.c). A node passed to pg_upgrade must therefore + listen on TCP even when the suite is otherwise running over Unix sockets + (PG_TEST_USE_UNIX_SOCKETS), which sets listen_addresses=''. Appending this + later wins, while the Unix socket the framework uses stays available. + + We listen on the name "localhost" rather than the literal "127.0.0.1" so + the server binds exactly what the client resolves. pg_upgrade passes no + host on Windows, so libpq connects to its default (localhost), which on an + IPv6-enabled host resolves to ::1 first. Binding only 127.0.0.1 would leave + the ::1 candidate refused; pg_upgrade's parallel task framework waits on the + connection socket with select() but without an exception set, so on Windows + a refused async connect is never reported and it hangs forever. Binding + "localhost" covers every address the client may try (and degrades to just + 127.0.0.1 where IPv6 is unavailable), so the first candidate always lands. + """ + if WINDOWS_OS: + node.append_conf("listen_addresses = 'localhost'") + + +def short_tempdir(prefix="pgt"): + """Create and return a uniquely-named directory under the system temp area. + + The short pathname keeps Unix-socket and tablespace-symlink targets within + their length limits (a unix socket path and tar's ~100-byte symlink-target + field). The caller owns the directory and is responsible for removing it. + + On Windows the directory is created with the default mode so that it + inherits the parent's access-control list. That matters when the postmaster + runs under a restricted access token (one with the Administrators group + disabled, as happens when launched from an elevated context): it must be + able to create files such as the socket lock file inside the directory, and + the owner-only DACL that a private (0o700) mode produces would deny that. + Elsewhere the directory is created private, like the rest of the framework. + """ + base = tempfile.gettempdir() + while True: + path = os.path.join(base, prefix + secrets.token_hex(8)) + try: + if sys.platform == "win32": + os.mkdir(path) + else: + os.mkdir(path, 0o700) + return path + except FileExistsError: + continue + + +def slurp_file(path, offset=0): + """Return the contents of *path* as text, from character *offset* onward. + + *offset* is a character position in the text-decoded (CRLF-normalized) + contents -- the kind of value PostgresServer.log_position() returns -- not + a raw byte offset, so it stays correct for multibyte content and on Windows + where the on-disk log has CRLF line endings. + """ + with open(path, "r", encoding="utf-8", errors="replace") as fh: + text = fh.read() + return text[offset:] if offset else text + + +def append_to_file(path, text): + """Append *text* to *path* (creating it if needed). + + newline="" disables newline translation so the bytes are written exactly + as given: on Windows the default text mode turns "\\n" into "\\r\\n", which + corrupts files with a strict line format such as backup_label. + """ + with open(path, "a", encoding="utf-8", newline="") as fh: + fh.write(text) + + +def copy_live_tree(src, dst): + """Recursively copy *src* to *dst*, tolerating entries that vanish. + + Copying a running server's data directory races with the server: a file + present when a directory is scanned (e.g. a WAL archive_status flag) can be + gone before it is copied. Such ENOENT cases are silently skipped, as a + low-level base backup must. Symlinks are recreated as symlinks. + """ + os.makedirs(dst, exist_ok=True) + try: + entries = list(os.scandir(src)) + except FileNotFoundError: + return + for entry in entries: + srcpath = os.path.join(src, entry.name) + dstpath = os.path.join(dst, entry.name) + try: + if entry.is_symlink(): + os.symlink(os.readlink(srcpath), dstpath) + elif entry.is_dir(): + copy_live_tree(srcpath, dstpath) + else: + shutil.copy2(srcpath, dstpath) + except FileNotFoundError: + # Entry vanished between the scan and the copy; skip it. + continue + + +def poll_until(predicate, timeout=TIMEOUT_DEFAULT, interval=0.1): + """Call *predicate* until it returns truthy or *timeout* seconds elapse. + + Returns True on success, False on timeout. + """ + deadline = time.monotonic() + timeout + while True: + if predicate(): + return True + if time.monotonic() > deadline: + return False + time.sleep(interval) From adabc5ff4d4eb242a9b0bdfd84a9f26096a91345 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Sat, 6 Jun 2026 08:12:47 -0400 Subject: [PATCH 03/14] python tests: add the pgtap plugin and wire pytest into the meson build pgtap is a pytest plugin that emits TAP for the meson/prove harness when TESTLOGDIR is set, and maps a whole-module skip to success. The repository pyproject.toml carries the pytest configuration. meson gains a pytest feature option and a kind=='pytest' test branch, so each directory can list pytest suites beside its tap suites. Includes the suite's own self-tests. The root pyproject.toml also configures the suite's code quality gates: black for formatting, and pylint and mypy for linting and type checking (the dev-tooling dependency group lives in src/test/pytest/pyproject.toml; none of it is needed to run the tests). The whole suite is kept black-clean, pylint 10.00/10 and mypy-clean. Author: Jelte Fennema-Nio Reviewed-by: Andrew Dunstan --- meson.build | 76 +++++++++++++ meson_options.txt | 6 ++ pyproject.toml | 114 ++++++++++++++++++++ src/test/meson.build | 1 + src/test/pytest/meson.build | 44 ++++++++ src/test/pytest/pgtap.py | 135 ++++++++++++++++++++++++ src/test/pytest/pyproject.toml | 28 +++++ src/test/pytest/pyt/test_libpq.py | 58 ++++++++++ src/test/pytest/pyt/test_replication.py | 32 ++++++ 9 files changed, 494 insertions(+) create mode 100644 pyproject.toml create mode 100644 src/test/pytest/meson.build create mode 100644 src/test/pytest/pgtap.py create mode 100644 src/test/pytest/pyproject.toml create mode 100644 src/test/pytest/pyt/test_libpq.py create mode 100644 src/test/pytest/pyt/test_replication.py diff --git a/meson.build b/meson.build index 568e0e150b..fac480daed 100644 --- a/meson.build +++ b/meson.build @@ -3951,6 +3951,32 @@ install_suites = [] testwrap = files('src/tools/testwrap') +# Detect pytest for the Python-based test suite (src/test/pytest). The suite +# is optional: it is enabled when the 'pytest' feature is enabled, or (in the +# default 'auto' mode) when a usable pytest is found. We prefer a 'pytest' +# program on PATH and fall back to "python -m pytest". +pytest_enabled = false +pytest_cmd = [] +pytest_pythonpath = meson.project_source_root() / 'src' / 'test' / 'pytest' +pytestopt = get_option('pytest') +if not pytestopt.disabled() + pytest_prog = find_program(get_option('PYTEST'), native: true, required: false) + if pytest_prog.found() + pytest_enabled = true + pytest_cmd = [pytest_prog.full_path()] + else + pytest_check = run_command( + python, '-m', 'pytest', '--version', check: false) + if pytest_check.returncode() == 0 + pytest_enabled = true + pytest_cmd = [python.full_path(), '-m', 'pytest'] + endif + endif + if not pytest_enabled and pytestopt.enabled() + error('pytest not found') + endif +endif + foreach test_dir : tests testwrap_base = [ testwrap, @@ -4118,6 +4144,56 @@ foreach test_dir : tests ) endforeach install_suites += test_group + elif kind == 'pytest' + testwrap_pytest = testwrap_base + if not pytest_enabled + testwrap_pytest += ['--skip', 'pytest not enabled'] + endif + + # Make the in-tree libpq/ and pypg/ packages importable, and put the + # temporary install (and per-directory build outputs) on PATH so client + # programs and pg_config resolve there. The 'test' subdir mirrors the + # tap branch: some dirs (e.g. libpq) build their test client programs + # into /test. + env = test_env + env.prepend('PATH', temp_install_bindir, test_dir['bd'], test_dir['bd'] / 'test') + env.prepend('PYTHONPATH', pytest_pythonpath) + + foreach name, value : t.get('env', {}) + env.set(name, value) + endforeach + + test_group = test_dir['name'] + test_kwargs = { + 'protocol': 'tap', + 'suite': test_group, + 'timeout': 1000, + 'depends': test_deps + t.get('deps', []), + 'env': env, + } + t.get('test_kwargs', {}) + + foreach onepyt : t['tests'] + # Make pytest test names prettier: drop pyt/ and .py + onepyt_p = onepyt + if onepyt_p.startswith('pyt/') + onepyt_p = onepyt.split('pyt/')[1] + endif + if onepyt_p.endswith('.py') + onepyt_p = fs.stem(onepyt_p) + endif + + test(test_dir['name'] / onepyt_p, + python, + kwargs: test_kwargs, + args: testwrap_pytest + [ + '--testgroup', test_dir['name'], + '--testname', onepyt_p, + '--', pytest_cmd, + test_dir['sd'] / onepyt, + ], + ) + endforeach + install_suites += test_group else error('unknown kind @0@ of test in @1@'.format(kind, test_dir['sd'])) endif diff --git a/meson_options.txt b/meson_options.txt index 6a793f3e47..878f10fd54 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -43,6 +43,9 @@ option('cassert', type: 'boolean', value: false, option('tap_tests', type: 'feature', value: 'auto', description: 'Enable TAP tests') +option('pytest', type: 'feature', value: 'auto', + description: 'Enable Python (pytest) test suites') + option('injection_points', type: 'boolean', value: false, description: 'Enable injection points') @@ -198,6 +201,9 @@ option('PROVE', type: 'string', value: 'prove', option('PYTHON', type: 'array', value: ['python3', 'python'], description: 'Path to python binary') +option('PYTEST', type: 'array', value: ['pytest', 'py.test'], + description: 'Path to pytest binary') + option('SED', type: 'string', value: 'gsed', description: 'Path to sed binary') diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..44a9b6015e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,114 @@ +# Copyright (c) 2025-2026, PostgreSQL Global Development Group +# +# This file exists solely to configure pytest for the Python-based test suite +# (see src/test/pytest). Placing the configuration at the repository root +# makes it the pytest "rootdir" config, so it applies to test files located +# throughout the tree (e.g. src/bin/*/pyt, contrib/*/pyt) as well as the +# infrastructure tests under src/test/pytest/pyt. + +[tool.pytest.ini_options] +minversion = "7.0" + +# Make the in-tree libpq/ and pypg/ packages importable from any test. +pythonpath = ["src/test/pytest"] + +# Load the TAP-output plugin (pgtap) and the shared fixtures (pypg.fixtures). +# Use importlib import mode so identically named test files in different +# directories (e.g. many t/001_basic) do not collide. +addopts = ["-p", "pgtap", "-p", "pypg.fixtures", "--import-mode=importlib"] + +# Tests live in pyt/ subdirectories using the test_*.py convention. +python_files = ["test_*.py"] + +# --------------------------------------------------------------------------- +# Code quality gates for the Python test suite (src/test/pytest and the pyt/ +# directories). Run from the repository root, e.g.: +# +# black --check src/test/pytest $(git ls-files '*/pyt/*.py') +# pylint src/test/pytest/libpq src/test/pytest/pypg src/test/pytest/pgtap.py +# mypy src/test/pytest +# +# These configure the tools only; installing them is a dev-only step (see the +# dependency group in src/test/pytest/pyproject.toml). They are never needed +# to run the tests. + +[tool.black] +# black's default line length; pylint is aligned to it below. +line-length = 88 + +[tool.mypy] +# No explicit python_version: type-check against the dev interpreter. The +# suite leans on ctypes/libpq and third-party libs without stubs; be pragmatic +# rather than strict, while still catching real type errors in our own code. +ignore_missing_imports = true +follow_imports = "silent" +warn_unused_ignores = true +warn_redundant_casts = true +no_implicit_optional = true +# Test trees have many same-named modules (conftest.py, test_*.py) that are +# not importable packages; these settings let mypy map files to distinct +# modules by path. +namespace_packages = true +explicit_package_bases = true + +[tool.pylint.main] +# The suite's runtime floor (requires-python in src/test/pytest/pyproject.toml). +py-version = "3.8" +# Resolve the in-tree pypg/ and libpq/ packages (pytest puts this directory on +# sys.path via the pythonpath setting above). +source-roots = ["src/test/pytest"] + +[tool.pylint.basic] +# Allow names that mirror the entities under test: node_A/node_B-style names +# (multi-node tests label nodes A/B/C for readability) and ALL_CAPS for +# function-local constants. +good-names-rgxs = ["^[a-z_][a-z0-9_]*[A-Z]{0,2}$", "^[A-Z][A-Z0-9_]*$"] + +[tool.pylint.format] +max-line-length = 88 + +[tool.pylint."messages control"] +# Curated relaxations for pytest/ctypes idioms (not bug-hiding). The checks +# that matter for a cross-platform suite (encoding, broad-except, +# with-resources, real errors) stay enabled and are satisfied in code, not +# silenced here. +disable = [ + "redefined-outer-name", # pytest passes fixtures by name on purpose + "missing-module-docstring", + "missing-function-docstring", # tests and plugin hooks self-describe + "too-few-public-methods", + "duplicate-code", # parallel tests legitimately share shape + "import-outside-toplevel", # required by the importorskip() pattern + "unused-argument", # pytest hook/fixture signatures are fixed + "wrong-import-order", # in-tree pypg/libpq misread as third-party + "fixme", # TODO/XXX are tracked intentionally + "use-dict-literal", # dict(opt=...) reads well for conn options + "consider-using-f-string", + # Formatting is owned by black; the only lines it leaves over the limit + # are long string literals (SQL, log patterns, error messages) carried + # over verbatim from the tests these were ported from. Splitting those + # strings would obscure the comparison with the originals. + "line-too-long", + # Size metrics: the test files are faithful ports of the corresponding + # Perl TAP tests, some of which are very large single scripts. Splitting + # them up would hurt reviewability against the originals, so function + # size is managed by review rather than a hard threshold. + "too-many-statements", + "too-many-locals", + "too-many-branches", + "too-many-lines", +] + +[tool.pylint.design] +# Advisory complexity metrics. A server-management harness and a ctypes/error +# wrapper are inherently "wide" (many attributes/args/branches); the Perl +# equivalents are larger still. Complexity is managed by review, not a hard +# pylint threshold, so these refactor-nudges are relaxed rather than silenced +# bug checks. +max-args = 15 +max-positional-arguments = 10 +max-attributes = 15 +max-returns = 10 +# The server-management class mirrors the breadth of the Perl Cluster +# harness, which has on the order of a hundred methods. +max-public-methods = 60 diff --git a/src/test/meson.build b/src/test/meson.build index cd45cbf57f..cc74c0dc70 100644 --- a/src/test/meson.build +++ b/src/test/meson.build @@ -26,3 +26,4 @@ if icu.found() endif subdir('perl') +subdir('pytest') diff --git a/src/test/pytest/meson.build b/src/test/pytest/meson.build new file mode 100644 index 0000000000..f0b487a7f3 --- /dev/null +++ b/src/test/pytest/meson.build @@ -0,0 +1,44 @@ +# Copyright (c) 2025-2026, PostgreSQL Global Development Group + +# Register the infrastructure self-tests. Whether they actually run is +# governed by the 'pytest' feature option, handled in the test-generation loop +# in the top-level meson.build (it emits a skip when pytest is not enabled). +tests += { + 'name': 'pytest', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'pytest': { + 'tests': [ + 'pyt/test_libpq.py', + 'pyt/test_replication.py', + ], + }, +} + +# Install the Python test framework alongside the Perl one, so out-of-tree +# (PGXS) test suites can import it. +install_data( + 'pgtap.py', + 'pyproject.toml', + install_dir: dir_pgxs / 'src/test/pytest') + +install_data( + 'libpq/__init__.py', + 'libpq/bindings.py', + 'libpq/constants.py', + 'libpq/errors.py', + 'libpq/findlib.py', + 'libpq/oids.py', + 'libpq/pgnotify.py', + 'libpq/result.py', + 'libpq/session.py', + install_dir: dir_pgxs / 'src/test/pytest/libpq') + +install_data( + 'pypg/__init__.py', + 'pypg/_env.py', + 'pypg/command.py', + 'pypg/fixtures.py', + 'pypg/server.py', + 'pypg/util.py', + install_dir: dir_pgxs / 'src/test/pytest/pypg') diff --git a/src/test/pytest/pgtap.py b/src/test/pytest/pgtap.py new file mode 100644 index 0000000000..87c6a25c9f --- /dev/null +++ b/src/test/pytest/pgtap.py @@ -0,0 +1,135 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""A pytest plugin that emits TAP output for the meson/prove harness. + +When the meson test harness runs the suite it sets TESTLOGDIR. In that case +this plugin hijacks the standard streams as early as possible: pytest's own +output is redirected to ``$TESTLOGDIR/pytest.log`` and the TAP stream (plan +line plus one ``ok``/``not ok`` per test) is written to the real stdout, which +is what meson's ``protocol: 'tap'`` consumes. + +When TESTLOGDIR is unset (a developer running ``pytest`` directly) the plugin +stays out of the way and lets pytest print normally. +""" + +import os +import sys + +import pytest + +_enabled = False +# Per-test accumulated state, keyed by nodeid, filled across setup/call/teardown. +_results: dict = {} + + +class _Tap: + def __init__(self): + self.count = 0 + + def _emit(self, *args): + print(*args, file=sys.__stdout__) + + def plan(self, num): + self._emit(f"1..{num}") + + def ok(self, name): + self.count += 1 + self._emit(f"ok {self.count} - {name}") + + def not_ok(self, name, details=""): + self.count += 1 + self._emit(f"not ok {self.count} - {name}") + # meson does not surface TAP diagnostics reliably, so send the details + # to the real stderr where they show up in the failure report. + if details: + print(details, file=sys.__stderr__) + + def skip(self, name, reason): + self.count += 1 + suffix = f" # skip {reason}" if reason else " # skip" + self._emit(f"ok {self.count} - {name}{suffix}") + + +_tap = _Tap() + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config): # noqa: ARG001 (pytest calls with config) + global _enabled # pylint: disable=global-statement + logdir = os.getenv("TESTLOGDIR") + if not logdir: + return + _enabled = True + os.makedirs(logdir, exist_ok=True) + logpath = os.path.join(logdir, "pytest.log") + # The stream intentionally stays open as stdout/stderr for the whole run. + # pylint: disable-next=consider-using-with + stream = open(logpath, "a", buffering=1, encoding="utf-8") # noqa: SIM115 + sys.stdout = stream + sys.stderr = stream + + +def pytest_collection_finish(session): + if _enabled: + _tap.plan(len(session.items)) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + """Stash each phase's report on the item so fixtures can tell, at teardown, + whether the test failed (used to keep data dirs on failure).""" + outcome = yield + setattr(item, f"_phase_{call.when}", outcome.get_result()) + + +def pytest_runtest_logreport(report): + if not _enabled: + return + rec = _results.setdefault( + report.nodeid, {"failed": False, "skipped": False, "reason": "", "details": ""} + ) + if report.failed: + rec["failed"] = True + rec["details"] += report.longreprtext + elif report.skipped: + rec["skipped"] = True + rec["reason"] = _skip_reason(report) + + +def pytest_runtest_logfinish(nodeid, location): # noqa: ARG001 + if not _enabled: + return + rec = _results.pop( + nodeid, {"failed": False, "skipped": False, "reason": "", "details": ""} + ) + # Check failed before skipped: a test can be reported skipped (setup/call) + # and then have a teardown that fails; the failure must win, otherwise a + # broken finalizer on a skipped test would be emitted as "ok # skip". + if rec["failed"]: + _tap.not_ok(nodeid, rec["details"]) + elif rec["skipped"]: + _tap.skip(nodeid, rec["reason"]) + else: + _tap.ok(nodeid) + + +def pytest_sessionfinish(session, exitstatus): # noqa: ARG001 + if not _enabled: + return + # A whole-module skip (``pytest.skip(..., allow_module_level=True)``) + # collects zero test items, which pytest reports as exit code 5 + # (NO_TESTS_COLLECTED). Under the meson harness that is a legitimate + # "entire test skipped" outcome, and pytest_collection_finish has already + # emitted a ``1..0`` TAP plan that + # meson reads as a skip. Map the exit code to success so meson does not + # treat the skip as an error. + if exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED: + session.exitstatus = pytest.ExitCode.OK + + +def _skip_reason(report): + longrepr = getattr(report, "longrepr", None) + # Skips are reported as a (path, lineno, "Skipped: reason") tuple. + if isinstance(longrepr, tuple) and len(longrepr) == 3: + return str(longrepr[2]).removeprefix("Skipped: ") + return "" diff --git a/src/test/pytest/pyproject.toml b/src/test/pytest/pyproject.toml new file mode 100644 index 0000000000..ebdee71570 --- /dev/null +++ b/src/test/pytest/pyproject.toml @@ -0,0 +1,28 @@ +# Copyright (c) 2025-2026, PostgreSQL Global Development Group + +[project] +name = "postgresql-pytest" +version = "0.1.0" +description = "Python (pytest) test infrastructure for PostgreSQL" +requires-python = ">=3.8" +# The only runtime dependency is pytest itself. The libpq layer uses the +# stdlib ctypes module, so no database driver (psycopg/asyncpg) is required. +dependencies = [ + "pytest >= 7.0", +] + +# NOTE: the active pytest configuration ([tool.pytest.ini_options]) lives in +# the repository-root pyproject.toml so it applies to test files throughout +# the tree (src/bin/.../pyt, contrib/.../pyt, ...). This file only records the +# package metadata for the in-tree libpq/ and pypg/ packages. + +# Dev tooling for the code-quality gates configured in the repository-root +# pyproject.toml. Only needed to lint/format/type-check the suite, never to +# run the tests. Install with e.g.: pip install --group dev (pip >= 25.1) +# or: pip install 'black>=24,<26' 'mypy>=1.8,<2' 'pylint>=3,<4' +[dependency-groups] +dev = [ + "black >= 24, < 26", + "mypy >= 1.8, < 2", + "pylint >= 3, < 4", +] diff --git a/src/test/pytest/pyt/test_libpq.py b/src/test/pytest/pyt/test_libpq.py new file mode 100644 index 0000000000..cee59f95df --- /dev/null +++ b/src/test/pytest/pyt/test_libpq.py @@ -0,0 +1,58 @@ +# Copyright (c) 2025-2026, PostgreSQL Global Development Group + +"""Self-tests for the in-process libpq Session layer. + +These exercise the parts of the Session API that distinguish it from a thin +synchronous wrapper: tuple results, one-value queries, LISTEN/NOTIFY, async +execution, and pipeline mode. +""" + +from libpq import ExecStatusType + + +def test_query_oneval(conn): + assert conn.query_oneval("SELECT 1") == "1" + assert conn.query_oneval("SELECT 'hello'") == "hello" + assert conn.query_oneval("SELECT NULL") is None + + +def test_query_tuples_and_metadata(conn): + res = conn.query("SELECT n, s FROM (VALUES (1, 'a'), (2, 'b')) t(n, s) ORDER BY n") + assert res.status == ExecStatusType.PGRES_TUPLES_OK + assert res.names == ["n", "s"] + assert res.rows == [["1", "a"], ["2", "b"]] + assert res.psqlout == "1|a\n2|b" + + +def test_do_and_error_capture(conn): + assert conn.do("CREATE TEMP TABLE t (a int)") == ExecStatusType.PGRES_COMMAND_OK + res = conn.query("SELECT * FROM no_such_table") + assert res.error_message is not None + assert "no_such_table" in res.error_message + + +def test_listen_notify(conn): + conn.do("LISTEN test_chan") + conn.do("NOTIFY test_chan, 'payload-1'") + note = conn.get_notification() + assert note is not None + assert note["channel"] == "test_chan" + assert note["payload"] == "payload-1" + assert note["pid"] == conn.backend_pid() + + +def test_async_query(conn): + assert conn.do_async("SELECT 42") + res = conn.get_async_result() + assert res is not None + assert res.psqlout == "42" + + +def test_pipeline(conn): + out = conn.query_tuples_pipelined("SELECT 1", "SELECT 2", "SELECT 3", "SELECT 4") + assert out == "1\n2\n3\n4" + + +def test_query_tuples_helper(conn): + # Fewer than 4 queries: non-pipelined path. + assert conn.query_tuples("SELECT 1", "SELECT 2") == "1\n2" diff --git a/src/test/pytest/pyt/test_replication.py b/src/test/pytest/pyt/test_replication.py new file mode 100644 index 0000000000..e9f681b7db --- /dev/null +++ b/src/test/pytest/pyt/test_replication.py @@ -0,0 +1,32 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Smoke test for streaming-replication support in the pypg framework.""" + + +def test_streaming_replication(create_pg): + primary = create_pg("primary", allows_streaming=True) + primary.safe_sql("CREATE TABLE t (id int)") + primary.safe_sql("INSERT INTO t SELECT generate_series(1, 10)") + + primary.backup("b1") + + standby = create_pg("standby", start=False) + standby.init_from_backup(primary, "b1", has_streaming=True, standby=True) + standby.start() + + # More rows after the standby is up. + primary.safe_sql("INSERT INTO t SELECT generate_series(11, 20)") + primary.wait_for_catchup("standby") + + # The standby must report its node name as application_name; that is what + # wait_for_catchup matches on, so assert it explicitly -- if the framework + # ever stopped setting it, wait_for_catchup would time out instead of + # failing clearly here. + assert ( + primary.safe_sql("SELECT application_name FROM pg_catalog.pg_stat_replication") + == "standby" + ) + + # The standby is read-only and should see all 20 rows. + assert standby.safe_sql("SELECT pg_is_in_recovery()") == "t" + assert standby.safe_sql("SELECT count(*) FROM t") == "20" From 80d9914a1e2fc8ed2b79cf39ec5f14236239693d Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Sat, 6 Jun 2026 08:12:47 -0400 Subject: [PATCH 04/14] python tests: add SSL, LDAP, Kerberos, OAuth and pg_regress helpers Helpers used by the heavier test suites: a pg_regress runner, an OpenSSL-backed SSL server configurator, an slapd launcher, a stand-alone Kerberos KDC, and a launcher for the mock OAuth provider. --- src/test/pytest/pypg/kerberos.py | 186 ++++++++++++++++++++ src/test/pytest/pypg/ldapserver.py | 211 +++++++++++++++++++++++ src/test/pytest/pypg/oauthserver.py | 56 ++++++ src/test/pytest/pypg/regress.py | 86 ++++++++++ src/test/pytest/pypg/ssl_server.py | 254 ++++++++++++++++++++++++++++ 5 files changed, 793 insertions(+) create mode 100644 src/test/pytest/pypg/kerberos.py create mode 100644 src/test/pytest/pypg/ldapserver.py create mode 100644 src/test/pytest/pypg/oauthserver.py create mode 100644 src/test/pytest/pypg/regress.py create mode 100644 src/test/pytest/pypg/ssl_server.py diff --git a/src/test/pytest/pypg/kerberos.py b/src/test/pytest/pypg/kerberos.py new file mode 100644 index 0000000000..4e3e093caf --- /dev/null +++ b/src/test/pytest/pypg/kerberos.py @@ -0,0 +1,186 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""A stand-alone KDC for testing PostgreSQL GSSAPI / Kerberos support. + +Import probes for MIT krb5 binaries; :data:`AVAILABLE` / :data:`SETUP_ERROR` +report usability. + +The constructor writes krb5.conf / kdc.conf, sets the KRB5_CONFIG / +KRB5_KDC_PROFILE / KRB5CCNAME environment variables (so the postmaster and +client tools started afterward inherit them), creates the realm and service +principal, and starts krb5kdc. Call :meth:`stop` (or use the ``kerberos`` +fixture) to shut the KDC down. +""" + +import os +import shutil +import signal +import socket +import subprocess +import sys + + +def _detect(): + """Locate the krb5 binaries; return a dict or None if unavailable.""" + bin_dir = sbin_dir = None + if sys.platform == "darwin": + base = ( + "/opt/homebrew/opt/krb5" + if os.path.isdir("/opt/homebrew") + else "/usr/local/opt/krb5" + ) + bin_dir, sbin_dir = base + "/bin", base + "/sbin" + elif sys.platform.startswith("freebsd"): + bin_dir, sbin_dir = "/usr/local/bin", "/usr/local/sbin" + elif sys.platform.startswith("linux"): + sbin_dir = "/usr/sbin" + + def _bin(name, d): + if d and os.path.isdir(d) and os.path.exists(os.path.join(d, name)): + return os.path.join(d, name) + return shutil.which(name) + + tools = { + "krb5_config": _bin("krb5-config", bin_dir), + "kinit": _bin("kinit", bin_dir), + "klist": _bin("klist", bin_dir), + "kdb5_util": _bin("kdb5_util", sbin_dir), + "kadmin_local": _bin("kadmin.local", sbin_dir), + "krb5kdc": _bin("krb5kdc", sbin_dir), + } + if not all(tools.values()): + return None + return tools + + +_TOOLS = _detect() +AVAILABLE = _TOOLS is not None +SETUP_ERROR = None if AVAILABLE else "MIT Kerberos 5 installation not found" + + +class Kerberos: + """A running krb5kdc with one realm and a PostgreSQL service principal. + + *basedir* is a writable scratch dir (tmp_path). *srvnam* is the Kerberos + service name (postgres), i.e. the with_krb_srvnam build value. + """ + + def __init__(self, basedir, host, hostaddr, realm, srvnam="postgres"): + if not AVAILABLE: + raise RuntimeError(SETUP_ERROR) + t = _TOOLS + + basedir = str(basedir) + self.krb5_conf = os.path.join(basedir, "krb5.conf") + self.kdc_conf = os.path.join(basedir, "kdc.conf") + self.krb5_cache = os.path.join(basedir, "krb5cc") + self.krb5_log = os.path.join(basedir, "krb5libs.log") + self.kdc_log = os.path.join(basedir, "krb5kdc.log") + self.kdc_datadir = os.path.join(basedir, "krb5kdc") + self.kdc_pidfile = os.path.join(basedir, "krb5kdc.pid") + self.keytab = os.path.join(basedir, "krb5.keytab") + self.kdc_port = _free_port() + self.realm = realm + + ver = subprocess.run( + [t["krb5_config"], "--version"], + stdout=subprocess.PIPE, + text=True, + check=True, + ).stdout + if "heimdal" in ver.lower(): + raise RuntimeError("Heimdal is not supported") + m = ver and __import__("re").search(r"Kerberos 5 release (\d+\.\d+)", ver) + krb5_version = float(m.group(1)) if m else 0.0 + + with open(self.krb5_conf, "w", encoding="utf-8") as fh: + fh.write( + f"""[logging] +default = FILE:{self.krb5_log} +kdc = FILE:{self.kdc_log} + +[libdefaults] +dns_lookup_realm = false +dns_lookup_kdc = false +default_realm = {realm} +forwardable = false +rdns = false + +[realms] +{realm} = {{ + kdc = {hostaddr}:{self.kdc_port} +}} +""" + ) + + with open(self.kdc_conf, "w", encoding="utf-8") as fh: + fh.write("[kdcdefaults]\n") + if krb5_version >= 1.15: + fh.write(f"kdc_listen = {hostaddr}:{self.kdc_port}\n") + fh.write(f"kdc_tcp_listen = {hostaddr}:{self.kdc_port}\n") + else: + fh.write(f"kdc_ports = {self.kdc_port}\n") + fh.write(f"kdc_tcp_ports = {self.kdc_port}\n") + fh.write( + f""" +[realms] +{realm} = {{ + database_name = {self.kdc_datadir}/principal + admin_keytab = FILE:{self.kdc_datadir}/kadm5.keytab + acl_file = {self.kdc_datadir}/kadm5.acl + key_stash_file = {self.kdc_datadir}/_k5.{realm} +}}""" + ) + + os.mkdir(self.kdc_datadir) + + # Make the test's config and cache files, not global ones, take effect. + # The postmaster and client tools started later inherit these. + os.environ["KRB5_CONFIG"] = self.krb5_conf + os.environ["KRB5_KDC_PROFILE"] = self.kdc_conf + os.environ["KRB5CCNAME"] = self.krb5_cache + + self._kdb5_util = t["kdb5_util"] + self._kadmin_local = t["kadmin_local"] + self._krb5kdc = t["krb5kdc"] + self._kinit = t["kinit"] + self._klist = t["klist"] + + service_principal = f"{srvnam}/{host}" + self._bail(self._kdb5_util, "create", "-s", "-P", "secret0") + self._bail(self._kadmin_local, "-q", f"addprinc -randkey {service_principal}") + self._bail( + self._kadmin_local, "-q", f"ktadd -k {self.keytab} {service_principal}" + ) + self._bail(self._krb5kdc, "-P", self.kdc_pidfile) + + @staticmethod + def _bail(*argv): + subprocess.run(list(argv), check=True) + + def create_principal(self, principal, password): + self._bail(self._kadmin_local, "-q", f"addprinc -pw {password} {principal}") + + def create_ticket(self, principal, password, forwardable=False): + cmd = [self._kinit, principal] + if forwardable: + cmd.append("-f") + subprocess.run(cmd, input=password + "\n", text=True, check=True) + subprocess.run([self._klist, "-f"], check=True) + + def stop(self): + try: + with open(self.kdc_pidfile, encoding="utf-8") as fh: + pid = int(fh.readline().strip()) + except (FileNotFoundError, ValueError): + return + try: + os.kill(pid, signal.SIGINT) + except ProcessLookupError: + pass + + +def _free_port(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] diff --git a/src/test/pytest/pypg/ldapserver.py b/src/test/pytest/pypg/ldapserver.py new file mode 100644 index 0000000000..d84567772d --- /dev/null +++ b/src/test/pytest/pypg/ldapserver.py @@ -0,0 +1,211 @@ +# Copyright (c) 2023-2026, PostgreSQL Global Development Group + +"""An OpenLDAP (slapd) server for testing pg_hba.conf ldap authentication. + +Module import probes for a usable slapd binary and the OpenLDAP schema +directory; :data:`AVAILABLE` says whether a server can be set up and +:data:`SETUP_ERROR` explains why not. +""" + +import os +import shutil +import signal +import socket +import subprocess +import sys +import time + +from .util import TIMEOUT_DEFAULT + + +def _detect_slapd(): + """Return (slapd_path, schema_dir) or (None, None) if unavailable.""" + if sys.platform == "darwin": + candidates = [ + ( + "/opt/homebrew/opt/openldap/libexec/slapd", + "/opt/homebrew/etc/openldap/schema", + ), + ("/usr/local/opt/openldap/libexec/slapd", "/usr/local/etc/openldap/schema"), + ("/opt/local/libexec/slapd", "/opt/local/etc/openldap/schema"), + ] + for slapd, schema in candidates: + if os.path.isdir(os.path.dirname(schema)) and os.path.isdir(schema): + return slapd, schema + return None, None + if sys.platform.startswith("linux"): + for schema in ("/etc/ldap/schema", "/etc/openldap/schema"): + if os.path.isdir(schema): + return "/usr/sbin/slapd", schema + return None, None + if sys.platform.startswith("freebsd"): + schema = "/usr/local/etc/openldap/schema" + if os.path.isdir(schema): + return "/usr/local/libexec/slapd", schema + return None, None + return None, None + + +SLAPD, SCHEMA_DIR = _detect_slapd() +AVAILABLE = bool(SLAPD) and os.path.exists(SLAPD or "") +SETUP_ERROR = ( + None + if AVAILABLE + else ( + "ldap tests not supported on this platform" + if sys.platform not in ("darwin",) + and not sys.platform.startswith(("linux", "freebsd")) + else "OpenLDAP server installation not found" + ) +) + + +def _free_port(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +class LdapServer: + """A running slapd instance for a single test. + + *basedir* is a writable scratch directory (e.g. tmp_path); *certdir* is the + directory holding the test SSL certs (src/test/ssl/ssl). + """ + + def __init__(self, basedir, rootpw, authtype, certdir): + if not AVAILABLE: + raise RuntimeError(SETUP_ERROR or "no suitable LDAP binaries found") + + basedir = str(basedir) + ldap_datadir = os.path.join(basedir, "openldap-data") + slapd_certs = os.path.join(basedir, "slapd-certs") + self.pidfile = os.path.join(basedir, "slapd.pid") + slapd_conf = os.path.join(basedir, "slapd.conf") + slapd_logfile = os.path.join(basedir, "slapd.log") + + self.server = "localhost" + self.port = _free_port() + self.s_port = _free_port() + self.url = f"ldap://{self.server}:{self.port}" + self.s_url = f"ldaps://{self.server}:{self.s_port}" + self.basedn = "dc=example,dc=net" + self.rootdn = "cn=Manager,dc=example,dc=net" + self.pwfile = os.path.join(basedir, "ldappassword") + + conf = f"""\ +include {SCHEMA_DIR}/core.schema +include {SCHEMA_DIR}/cosine.schema +include {SCHEMA_DIR}/nis.schema +include {SCHEMA_DIR}/inetorgperson.schema + +pidfile {self.pidfile} +logfile {slapd_logfile} + +access to * + by * read + by {authtype} auth + +database ldif +directory {ldap_datadir} + +TLSCACertificateFile {slapd_certs}/ca.crt +TLSCertificateFile {slapd_certs}/server.crt +TLSCertificateKeyFile {slapd_certs}/server.key + +suffix "dc=example,dc=net" +rootdn "{self.rootdn}" +rootpw "{rootpw}" +""" + with open(slapd_conf, "w", encoding="utf-8") as fh: + fh.write(conf) + + os.mkdir(ldap_datadir) + os.mkdir(slapd_certs) + shutil.copy( + os.path.join(certdir, "server_ca.crt"), os.path.join(slapd_certs, "ca.crt") + ) + shutil.copy( + os.path.join(certdir, "server-cn-only.crt"), + os.path.join(slapd_certs, "server.crt"), + ) + shutil.copy( + os.path.join(certdir, "server-cn-only.key"), + os.path.join(slapd_certs, "server.key"), + ) + + with open(self.pwfile, "w", encoding="utf-8") as fh: + fh.write(rootpw) + os.chmod(self.pwfile, 0o600) + + # -s0 prevents log messages ending up in syslog. + subprocess.run( + [SLAPD, "-f", slapd_conf, "-s0", "-h", f"{self.url} {self.s_url}"], + check=True, + ) + + # Wait until slapd accepts requests. + deadline = time.monotonic() + TIMEOUT_DEFAULT + while True: + rc = subprocess.run( + [ + "ldapsearch", + "-sbase", + "-H", + self.url, + "-b", + self.basedn, + "-D", + self.rootdn, + "-y", + self.pwfile, + "-n", + "objectclass=*", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ).returncode + if rc == 0: + break + if time.monotonic() > deadline: + raise RuntimeError("cannot connect to slapd") + time.sleep(0.5) + + def _env(self): + env = dict(os.environ) + env["LDAPURI"] = self.url + env["LDAPBINDDN"] = self.rootdn + return env + + def ldapadd_file(self, path): + """Add the LDIF data in *path* to the server.""" + subprocess.run( + ["ldapadd", "-x", "-y", self.pwfile, "-f", str(path)], + env=self._env(), + check=True, + ) + + def ldapsetpw(self, user, password): + """Set *user*'s password on the server.""" + subprocess.run( + ["ldappasswd", "-x", "-y", self.pwfile, "-s", password, user], + env=self._env(), + check=True, + ) + + def prop(self, *names): + """Return the requested properties (url, port, basedn, ...).""" + return [getattr(self, n) for n in names] + + def stop(self): + """Terminate the slapd instance.""" + try: + with open(self.pidfile, encoding="utf-8") as fh: + pid = int(fh.readline().strip()) + except (FileNotFoundError, ValueError): + return + try: + os.kill(pid, signal.SIGINT) + except ProcessLookupError: + pass diff --git a/src/test/pytest/pypg/oauthserver.py b/src/test/pytest/pypg/oauthserver.py new file mode 100644 index 0000000000..6a032b0abc --- /dev/null +++ b/src/test/pytest/pypg/oauthserver.py @@ -0,0 +1,56 @@ +# Copyright (c) 2025-2026, PostgreSQL Global Development Group + +"""Launches the mock OAuth authorization server used by the oauth tests. + +The actual server is the Python daemon t/oauth_server.py; this is just the glue +that starts it, reads the ephemeral port it prints to stdout, and stops it. +""" + +import os +import signal +import subprocess +import sys + + +class OAuthServer: + """A running instance of the mock OAuth provider (oauth_server.py). + + *script* is the path to oauth_server.py; it is run with its grandparent + (the module source dir) as the working directory, so it invokes + "t/oauth_server.py" from the test directory. + """ + + def __init__(self, script): + python = os.environ.get("PYTHON") or sys.executable + script = os.path.abspath(script) + cwd = os.path.dirname(os.path.dirname(script)) # the module source dir + rel = os.path.join("t", os.path.basename(script)) + + # The daemon's lifetime is managed by stop(), not a with block. + # pylint: disable-next=consider-using-with + self._proc = subprocess.Popen( + [python, rel], + cwd=cwd, + stdout=subprocess.PIPE, + text=True, + ) + # The daemon prints its port then closes stdout, so a full read blocks + # only until the port is known. + line = self._proc.stdout.readline() + if not line.strip().isdigit(): + raise RuntimeError(f"server did not advertise a valid port: {line!r}") + self.port = int(line.strip()) + + def stop(self): + if self._proc is None: + return + try: + self._proc.send_signal(signal.SIGTERM) + except ProcessLookupError: + pass + try: + self._proc.stdout.close() + except Exception: # pylint: disable=broad-exception-caught + pass + self._proc.wait() + self._proc = None diff --git a/src/test/pytest/pypg/regress.py b/src/test/pytest/pypg/regress.py new file mode 100644 index 0000000000..92760728f0 --- /dev/null +++ b/src/test/pytest/pypg/regress.py @@ -0,0 +1,86 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +"""Run the core SQL regression suite (pg_regress) against a running node. + +A handful of tests that exercise the full regression suite -- e.g. the +streaming-regression recovery test and the pg_upgrade test -- shell out to the +pg_regress C driver rather than running SQL through the in-process Session. +This helper builds that command line from the PG_REGRESS / REGRESS_SHLIB +environment variables the meson harness supplies (see test_env in meson.build). +""" + +import os +import shlex +import subprocess + +from pypg.command import CommandResult + + +def pg_regress_available(): + """True if PG_REGRESS is set (the build provides the pg_regress driver).""" + return bool(os.environ.get("PG_REGRESS")) + + +def run_pg_regress( + node, + *, + inputdir, + outputdir, + schedule=None, + tests=None, + dlpath=None, + bindir="", + max_concurrent_tests=None, + extra_opts=None, + extra_args=None, +): + """Run pg_regress against *node* and return its CommandResult. + + *inputdir* holds the sql/ and expected/ trees; *outputdir* receives + results/. Pass either a *schedule* file or an explicit list of *tests*. + *dlpath* defaults to the directory of REGRESS_SHLIB (where regress.so + lives). EXTRA_REGRESS_OPTS from the environment is honored. The caller + asserts on the returncode. + """ + pg_regress = os.environ.get("PG_REGRESS") + if not pg_regress: + raise RuntimeError("PG_REGRESS is not set; pg_regress is unavailable") + + if dlpath is None: + shlib = os.environ.get("REGRESS_SHLIB") + dlpath = os.path.dirname(shlib) if shlib else "." + + argv = [pg_regress] + # EXTRA_REGRESS_OPTS is split on whitespace. + argv += shlex.split(os.environ.get("EXTRA_REGRESS_OPTS", "")) + if extra_opts: + argv += list(extra_opts) + argv += [ + f"--dlpath={dlpath}", + f"--bindir={bindir}", + f"--host={node.host}", + f"--port={node.port}", + f"--inputdir={inputdir}", + f"--outputdir={outputdir}", + ] + if max_concurrent_tests is not None: + argv.append(f"--max-concurrent-tests={max_concurrent_tests}") + if schedule is not None: + argv.append(f"--schedule={schedule}") + if extra_args: + argv += list(extra_args) + if tests: + argv += list(tests) + + os.makedirs(outputdir, exist_ok=True) + print("# Running pg_regress: " + " ".join(argv)) + proc = subprocess.run( + argv, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) + return CommandResult(proc.returncode, proc.stdout, proc.stderr) diff --git a/src/test/pytest/pypg/ssl_server.py b/src/test/pytest/pypg/ssl_server.py new file mode 100644 index 0000000000..37985b4b24 --- /dev/null +++ b/src/test/pytest/pypg/ssl_server.py @@ -0,0 +1,254 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Configure a PostgreSQL test cluster for SSL, for the ssl regression tests. + +Only the OpenSSL backend is supported. + +Certificates and keys live in src/test/ssl/ssl (built by the source tree). +The helper installs the server certs/keys into the cluster's data directory, +copies the client keys to a private-perms temp dir (so libpq accepts them), +sets up the trustdb/certdb/... databases and test roles, and rewrites +pg_hba.conf / pg_ident.conf for hostssl + certificate auth. +""" + +import glob +import os +import shutil +import subprocess + +# Server cert/key/CA/CRL files copied into the data directory by init(). +_SERVER_GLOBS = ["server-*.crt", "server-*.key"] +_SERVER_FILES = [ + "root+client_ca.crt", + "root+server_ca.crt", + "root_ca.crt", + "root+client.crl", +] +# Client keys copied to a private temp dir with 0600 perms (0644 for the +# deliberately-wrong-perms copy). +_CLIENT_KEYS = [ + "client.key", + "client-revoked.key", + "client-der.key", + "client-encrypted-pem.key", + "client-encrypted-der.key", + "client-dn.key", + "client_ext.key", + "client-long.key", + "client-revoked-utf8.key", +] + + +class SSLServer: + """OpenSSL-backed SSL configuration for a test cluster. + + *ssl_dir* is the path to src/test/ssl/ssl; *keydir* is a writable temp dir + for the permission-adjusted client keys; *bindir* locates pg_config. + """ + + def __init__(self, ssl_dir, keydir, bindir): + self._library = "OpenSSL" + self.ssl_dir = str(ssl_dir) + self.keydir = str(keydir) + self.bindir = str(bindir) + self.key = {} + + # -- backend (OpenSSL) --------------------------------------------------- + + def _copy_glob(self, pattern, dest): + for src in glob.glob(os.path.join(self.ssl_dir, pattern)): + shutil.copy(src, os.path.join(dest, os.path.basename(src))) + + def _init_backend(self, pgdata): + for pattern in _SERVER_GLOBS: + self._copy_glob(pattern, pgdata) + for key in glob.glob(os.path.join(pgdata, "server-*.key")): + os.chmod(key, 0o600) + for name in _SERVER_FILES: + shutil.copy(os.path.join(self.ssl_dir, name), os.path.join(pgdata, name)) + crldir = os.path.join(pgdata, "root+client-crldir") + os.mkdir(crldir) + self._copy_glob("root+client-crldir/*", crldir) + + # The client private keys must not be world-readable, so work from + # copies under keydir with adjusted permissions. + # The stored paths go into connection strings; use forward slashes so + # libpq does not treat Windows backslashes as escape characters. + for keyfile in _CLIENT_KEYS: + dst = os.path.join(self.keydir, keyfile) + shutil.copy(os.path.join(self.ssl_dir, keyfile), dst) + os.chmod(dst, 0o600) + self.key[keyfile] = dst.replace("\\", "/") + # A deliberately world-readable copy, to test wrong permissions. + wrong = os.path.join(self.keydir, "client_wrongperms.key") + shutil.copy(os.path.join(self.ssl_dir, "client.key"), wrong) + os.chmod(wrong, 0o644) + self.key["client_wrongperms.key"] = wrong.replace("\\", "/") + + def sslkey(self, keyfile): + """Return an ' sslkey=' connection-string fragment.""" + return f" sslkey={self.key[keyfile]}" + + def ssl_library(self): + return self._library + + def is_libressl(self): + # HAVE_SSL_CTX_SET_CERT_CB is undefined for LibreSSL. + return not self.check_pg_config("#define HAVE_SSL_CTX_SET_CERT_CB 1") + + def check_pg_config(self, needle): + out = subprocess.run( + [os.path.join(self.bindir, "pg_config"), "--includedir-server"], + stdout=subprocess.PIPE, + text=True, + check=True, + ).stdout.strip() + try: + with open(os.path.join(out, "pg_config.h"), encoding="utf-8") as fh: + return any(needle in line for line in fh) + except FileNotFoundError: + return False + + # -- server configuration ----------------------------------------------- + + def configure_test_server_for_ssl( + self, + node, + serverhost, + servercidr, + authmethod, + *, + password=None, + password_enc=None, + extensions=None, + ): + """Set up databases/roles, enable SSL listening, and write pg_hba.""" + pgdata = node.data_dir + databases = [ + "trustdb", + "certdb", + "certdb_dn", + "certdb_dn_re", + "certdb_cn", + "verifydb", + ] + + for role in ("ssltestuser", "md5testuser", "anotheruser", "yetanotheruser"): + node.safe_sql(f"CREATE USER {role}") + for db in databases: + node.safe_sql(f"CREATE DATABASE {db}") + + if password is not None: + assert ( + password_enc is not None + ), "password_enc required when password is set" + node.safe_sql( + f"SET password_encryption='{password_enc}'; " + f"ALTER USER ssltestuser PASSWORD '{password}';" + ) + # md5testuser always has an md5-encrypted password. + node.safe_sql( + f"SET password_encryption='md5'; " + f"ALTER USER md5testuser PASSWORD '{password}';" + ) + node.safe_sql( + f"SET password_encryption='{password_enc}'; " + f"ALTER USER anotheruser PASSWORD '{password}';" + ) + + for extension in extensions or []: + for db in databases: + node.safe_sql(f"CREATE EXTENSION {extension} CASCADE;", dbname=db) + + node.append_conf( + "fsync=off\n" + "log_connections=all\n" + "log_hostname=on\n" + f"listen_addresses='{serverhost}'\n" + "log_statement=all\n" + ) + node.append_conf("include 'sslconfig.conf'") + # The SSL configuration is written into sslconfig.conf later. + with open(os.path.join(pgdata, "sslconfig.conf"), "w", encoding="utf-8"): + pass + + self._init_backend(pgdata) + + # Restart to load listen_addresses. + node.restart() + + # pg_hba must be changed after restart because hostssl requires ssl=on. + self._configure_hba_for_ssl(node, servercidr, authmethod) + + def switch_server_cert( + self, + node, + *, + certfile, + keyfile=None, + cafile=None, + crlfile=None, + crldir=None, + passphrase_cmd=None, + passphrase_cmd_reload=None, + restart=True, + ): + """Point the server at a different cert/key/ca/crl and (re)load.""" + pgdata = node.data_dir + if cafile is None: + cafile = "root+client_ca" + if crlfile is None: + crlfile = "root+client.crl" + if keyfile is None: + keyfile = certfile + + os.unlink(os.path.join(pgdata, "sslconfig.conf")) + lines = ["ssl=on"] + lines.append(f"ssl_cert_file='{certfile}.crt'") + lines.append(f"ssl_key_file='{keyfile}.key'") + lines.append(f"ssl_crl_file='{crlfile}'") + if cafile != "": + lines.append(f"ssl_ca_file='{cafile}.crt'") + else: + lines.append("ssl_ca_file=''") + if crldir is not None: + lines.append(f"ssl_crl_dir='{crldir}'") + # Lists of ECDH curves and cipher suites for syntax testing. + lines.append("ssl_groups=prime256v1:secp521r1") + lines.append("ssl_tls13_ciphers=TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256") + if passphrase_cmd is not None: + lines.append(f"ssl_passphrase_command='{passphrase_cmd}'") + if passphrase_cmd_reload is not None: + lines.append( + f"ssl_passphrase_command_supports_reload='{passphrase_cmd_reload}'" + ) + node.append_conf("\n".join(lines), filename="sslconfig.conf") + + if not restart: + return + node.restart() + + def _configure_hba_for_ssl(self, node, servercidr, authmethod): + pgdata = node.data_dir + os.unlink(os.path.join(pgdata, "pg_hba.conf")) + node.append_conf( + "# TYPE DATABASE USER ADDRESS METHOD OPTIONS\n" + f"hostssl trustdb md5testuser {servercidr} md5\n" + f"hostssl trustdb all {servercidr} {authmethod}\n" + f"hostssl verifydb ssltestuser {servercidr} {authmethod} clientcert=verify-full\n" + f"hostssl verifydb anotheruser {servercidr} {authmethod} clientcert=verify-full\n" + f"hostssl verifydb yetanotheruser {servercidr} {authmethod} clientcert=verify-ca\n" + f"hostssl certdb all {servercidr} cert\n" + f"hostssl certdb_dn all {servercidr} cert clientname=DN map=dn\n" + f"hostssl certdb_dn_re all {servercidr} cert clientname=DN map=dnre\n" + f"hostssl certdb_cn all {servercidr} cert clientname=CN map=cn\n", + filename="pg_hba.conf", + ) + os.unlink(os.path.join(pgdata, "pg_ident.conf")) + node.append_conf( + "# MAPNAME SYSTEM-USERNAME PG-USERNAME\n" + 'dn "CN=ssltestuser-dn,OU=Testing,OU=Engineering,O=PGDG" ssltestuser\n' + 'dnre "/^.*OU=Testing,.*$" ssltestuser\n' + "cn ssltestuser-dn ssltestuser\n", + filename="pg_ident.conf", + ) From 9a0eba3967deeee1ca7f8e6c8e7f397454f948e0 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Mon, 8 Jun 2026 08:26:54 -0400 Subject: [PATCH 05/14] python tests: add README for the pytest/session framework Document the Python port of the Perl TAP suite: layout, how to run the tests under meson and directly with pytest, the shared fixtures, and the PostgresServer/Session/PgBin framework classes. --- src/test/pytest/README.md | 274 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 src/test/pytest/README.md diff --git a/src/test/pytest/README.md b/src/test/pytest/README.md new file mode 100644 index 0000000000..c8d80a4549 --- /dev/null +++ b/src/test/pytest/README.md @@ -0,0 +1,274 @@ +# PostgreSQL Python tests (pytest) + +This tree holds the Python port of PostgreSQL's Perl TAP test suite. Tests are +written for [pytest](https://pytest.org) and run against a real PostgreSQL +server that the harness initializes, starts, and tears down for you. Queries go +through an **in-process libpq binding** (ctypes) rather than by spawning `psql`, +so a test can open many sessions, drive async and pipeline-mode traffic, and +inspect results without subprocess overhead. + +The framework deliberately mirrors the concepts of `PostgreSQL::Test::Cluster` +and `PostgreSQL::Test::Utils`, so a Perl `.pl` test maps fairly directly onto a +Python `test_*.py`. + +## Layout + +``` +src/test/pytest/ this directory -- the shared framework +├── pyproject.toml pytest config (also picked up from the repo root) +├── pgtap.py pytest plugin: emits TAP for the meson harness +├── pyt/ self-tests for the framework itself +├── libpq/ in-process libpq binding +│ ├── bindings.py ctypes declarations for the PQ* functions +│ ├── findlib.py locate/load libpq at runtime +│ ├── session.py Session: the connection + query API +│ ├── result.py ResultData: status, columns, rows, psql-style text +│ ├── constants.py enums (ExecStatusType, ConnStatusType, ...) +│ ├── pgnotify.py LISTEN/NOTIFY payload parsing +│ └── oids.py / errors.py type OIDs and exception types +└── pypg/ server + process management ("Cluster"/"Utils") + ├── server.py PostgresServer: init/start/stop/backup/replication + ├── command.py PgBin / CommandResult: run & assert on client programs + ├── fixtures.py the shared pytest fixtures (loaded as a plugin) + ├── util.py slurp_file, poll_until, TIMEOUT_DEFAULT + ├── regress.py pg_regress integration + └── ldapserver.py / kerberos.py / oauthserver.py / ssl_server.py +``` + +The actual tests live in `pyt/` subdirectories next to the code they cover, the +same way Perl TAP tests live in `t/`: + +``` +src/bin/psql/pyt/test_001_basic.py +src/bin/pg_rewind/pyt/test_001_basic.py +src/test/authentication/pyt/test_001_password.py +contrib//pyt/... +``` + +### Naming and discovery + +* Test files are `test_NNN_.py` (`test_001_basic.py`), matching the + `NNN_name.pl` numbering of the Perl originals. +* Test functions are `test_(...)`; pytest collects them automatically. +* Helper functions are prefixed with `_` so pytest does not collect them. +* `--import-mode=importlib` is set so that identically named files in different + directories (the many `test_001_basic.py`) do not collide. + +## Running the tests + +### With meson (the CI / harness path) + +The suite is wired into the meson build as test groups using the TAP protocol. +pytest is auto-detected: in the default `auto` mode the suite is enabled when a +usable `pytest` is found (preferring a `pytest` program on `PATH`, falling back +to `python -m pytest`). Force it on or off with `-Dpytest=enabled|disabled`, +and override the interpreter discovery with `-DPYTEST=...`. + +```bash +# build, then run a single pytest group (suite == the test_dir 'name'): +meson test -C --suite setup # once, to stage tmp_install +meson test -C --suite pg_rewind +meson test -C pytest # the framework self-tests +``` + +Under meson the harness sets `TESTLOGDIR`; the `pgtap` plugin then redirects +pytest's own chatter to `$TESTLOGDIR/pytest.log` and writes a clean TAP stream +(a `1..N` plan plus one `ok`/`not ok` per test) to stdout, which is what meson's +`protocol: 'tap'` consumes. The harness also prepends the temporary install's +`bin` to `PATH` (so `pg_config`, `initdb`, etc. resolve there) and adds this +directory to `PYTHONPATH`. + +### Directly with pytest (the developer path) + +When `TESTLOGDIR` is unset the plugin stays out of the way and pytest prints +normally. You only need `pg_config` (and the matching binaries) on `PATH` and +the framework importable. The repo-root `pyproject.toml` already sets +`pythonpath` and the required plugins, so from the repo root: + +```bash +# point at the build you want to test: +export PATH=/tmp_install/usr/local/pgsql/bin:$PATH + +pytest src/test/pytest/pyt/ # framework self-tests +pytest src/bin/pg_rewind/pyt/ -v # one suite, verbose +pytest src/test/pytest/pyt/test_libpq.py::test_pipeline # one test +``` + +`pytest` discovers `pyproject.toml` by walking up from the current directory, so +running from the repo root (or anywhere beneath it) picks up the right config: + +```toml +[tool.pytest.ini_options] +pythonpath = ["src/test/pytest"] # make libpq/ and pypg/ importable +addopts = ["-p", "pgtap", "-p", "pypg.fixtures", "--import-mode=importlib"] +python_files = ["test_*.py"] +``` + +### Requirements + +* Python 3 and `pytest` (`minversion = 7.0`). +* **No database driver** — `libpq` is bound directly via the stdlib `ctypes` + module, so psycopg/asyncpg are not needed. +* `pexpect` is an *optional* dependency, used only by tests that drive a real + terminal (e.g. psql tab-completion). Those tests `pytest.importorskip( + "pexpect")` and skip themselves when it is missing. +* External-service tests (LDAP/`slapd`, MIT Kerberos, OAuth, OpenSSL) skip + automatically when the supporting software or build option is absent. + +### Environment variables + +| Variable | Meaning | +|----------|---------| +| `PG_TEST_EXTRA` | Space-separated opt-in for expensive/unsafe suites (`ssl`, `ldap`, `kerberos`, ...). Tests that are not in the list `pytest.skip` themselves. | +| `PG_TEST_TIMEOUT_DEFAULT` | Default per-operation timeout in seconds (default `180`); backs `pypg.util.TIMEOUT_DEFAULT` and the polling helpers. | +| `TESTLOGDIR` | Set by the meson harness; switches the `pgtap` plugin into TAP-emitting mode. | + +## The session framework + +### Fixtures (`pypg.fixtures`, loaded via `-p pypg.fixtures`) + +These are the building blocks almost every test starts from. Servers and helper +processes are torn down automatically at the end of the test. + +| Fixture | Scope | What you get | +|---------|-------|--------------| +| `pg_config`, `bindir`, `libdir` | session | located `pg_config` and its reported dirs | +| `pg_bin` | session | a `PgBin` for running client programs that need no server | +| `create_pg` | function | factory: `create_pg(name="main", *, start=True, initdb_extra=None, allows_streaming=False, has_archiving=False, has_restoring=False)` → `PostgresServer` | +| `pg` | function | a single, started `PostgresServer` (`create_pg("main")`) | +| `conn` | function | a libpq `Session` on `pg`'s `postgres` database | +| `ldap_server`, `kerberos`, `oauth_server`, `ssl_server` | function | factories for the matching external services; skip when unavailable | + +A minimal test: + +```python +def test_oneval(conn): + assert conn.query_oneval("SELECT 1") == "1" + +def test_two_servers(create_pg): + primary = create_pg("primary", allows_streaming=True) + primary.backup("my_backup") + standby = create_pg("standby", start=False) + standby.init_from_backup(primary, "my_backup", has_streaming=True) + standby.start() + primary.wait_for_catchup("standby") +``` + +### `PostgresServer` (`pypg.server`) — the "Cluster" + +Created via the `create_pg` fixture. Highlights (see the source for the full +list and exact signatures): + +* **Lifecycle:** `init(...)`, `start()`, `stop(mode="fast")`, `restart()`, + `reload()`, `promote()`, `kill9()`, `teardown()`, `postmaster_pid()`. +* **Config:** `append_conf(text, filename="postgresql.conf")`, + `enable_archiving()`, `enable_streaming(root)`, `enable_restoring(root)`, + `set_standby_mode()`, `set_recovery_mode()`. +* **Queries (in-process libpq):** `session(dbname="postgres")` (cached), + `connect(...)` (uncached), `sql(query)` → `ResultData`, `safe_sql(query)` → + text (raises on error), `poll_query_until(query, expected="t", ...)`. +* **Backup & replication:** `backup()`, `backup_fs_cold()`, + `init_from_backup(...)`, `lsn(mode)`, `wait_for_catchup(...)`, + `wait_for_replay_catchup(...)`, `wait_for_subscription_sync(...)`, + `wait_for_event(...)`. +* **WAL:** `emit_wal(size)`, `advance_wal(n)`, `write_wal(...)`. +* **Logs:** `log_content()`, `log_position()`, `log_contains(pat, offset)`, + `wait_for_log(pat, offset)`, `log_check(...)`. +* **Auth assertions:** `connect_ok(...)`, `connect_fails(...)`. +* **Client-program assertions** (delegating to `PgBin`): `command_ok`, + `command_fails`, `command_like`, `command_fails_like`, `command_exit_is`, + `command_checks_all`, `issues_sql_like`, `issues_sql_unlike`. + +### `Session` (`libpq.session`) — the connection + +The in-process equivalent of a `psql` session. Returned by +`PostgresServer.session()` / `.connect()` and by the `conn` fixture. + +* **Synchronous:** `do(*sql)`, `query(sql)` → `ResultData`, `query_safe(sql)`, + `query_oneval(sql, missing_ok=False)`, `query_tuples(*sql)`. +* **Asynchronous:** `do_async(sql)`, `get_async_result()`, + `try_get_async_result()`, `wait_for_completion()`, + `wait_for_async_pattern(pattern, timeout)`. +* **Pipeline mode:** `enterPipelineMode()`, `exitPipelineMode()`, + `pipelineSync()`, `query_tuples_pipelined(*queries)`. +* **LISTEN/NOTIFY:** `get_notification()`, `get_all_notifications()`. +* **Notices / errors / stderr:** `get_notices()`, `clear_notices()`, + `get_stderr()`, `clear_stderr()`. +* **Lifecycle / introspection:** `wait_connect()`, `reconnect()`, `close()`, + `backend_pid()`, `conn_status()`, `connstr`, `conninfo_value(keyword)`. + +`query()` returns a `ResultData` dataclass: + +```python +@dataclass +class ResultData: + status: int # an ExecStatusType + error_message: Optional[str] + names: List[str] # column names + types: List[int] # column type OIDs + rows: List[List[Optional[str]]] # values as text, NULL -> None + psqlout: str # "-A -t" style rendering +``` + +```python +res = conn.query("SELECT n, s FROM (VALUES (1,'a'),(2,'b')) t(n,s) ORDER BY n") +assert res.names == ["n", "s"] +assert res.rows == [["1", "a"], ["2", "b"]] +assert res.psqlout == "1|a\n2|b" +``` + +### `PgBin` / `CommandResult` (`pypg.command`) — running client programs + +For exercising client executables (`pg_dump`, `pg_basebackup`, `psql`, ...): + +```python +@dataclass +class CommandResult: + returncode: int + stdout: str + stderr: str +``` + +`PgBin(bindir, extra_env=None)` runs a command and asserts on the outcome: +`result(cmd)`, `command_ok(cmd, msg)`, `command_fails(cmd, msg)`, +`command_exit_is(cmd, code, msg)`, `command_like(cmd, pattern, msg)`, +`command_fails_like(cmd, pattern, msg)`, +`command_checks_all(cmd, expected_ret, stdout_res, stderr_res, msg)`, plus the +`program_help_ok` / `program_version_ok` / `program_options_handling_ok` +boilerplate checks. + +### Utilities (`pypg.util`) + +`slurp_file(path, offset=0)`, `append_to_file(path, text)`, and +`poll_until(predicate, timeout=TIMEOUT_DEFAULT, interval=0.1)` — the building +block behind every `wait_for_*` / `poll_query_until` helper. + +## Per-suite fixtures (conftest.py) + +A `pyt/` directory may add its own `conftest.py` with fixtures specific to that +suite. Examples already in the tree: + +* **psql** (`src/bin/psql/pyt/conftest.py`): `interactive_psql` drives a real + psql under a pty via `pexpect` (`InteractivePsql.query_until(until, send)`), + `pytest.importorskip("pexpect")`-skipping when pexpect is absent. +* **pg_rewind** (`src/bin/pg_rewind/pyt/conftest.py`): a `rewind` fixture + yielding a `RewindTest` driver (`setup_cluster`, `start_primary`, + `create_standby`, `promote_standby`, `run_pg_rewind(mode)`, ...). +* **test_checksums** (`src/test/modules/test_checksums/pyt/conftest.py`): a + `checksums` helper for enabling/disabling and polling data-checksum state. + +Put fixtures that several suites share in `pypg/`; keep suite-specific ones in +that suite's `conftest.py`. + +## Writing a new test + +1. Create `…/pyt/test_NNN_.py` (and a `pyt/meson.build` adding the suite + to `tests` if the directory is new). +2. Start from the `pg` / `conn` / `create_pg` fixtures; reach for `pg_bin` to run + client programs. +3. Prefer the in-process `Session` over shelling out to `psql`. +4. Use `wait_for_*` / `poll_query_until` / `poll_until` instead of fixed sleeps. +5. Gate anything expensive or unsafe behind `PG_TEST_EXTRA`, and + `importorskip` / skip cleanly when an optional dependency is missing. +6. Run it directly with `pytest …/pyt/test_NNN_.py -v`, then confirm it + passes under `meson test`. From e1d635462e9780b46afc64169f3296f294c95cca Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Sat, 20 Jun 2026 10:12:14 -0400 Subject: [PATCH 06/14] python tests: convert the psql TAP suite to pytest Replace the four src/bin/psql Perl TAP tests with their pytest equivalents and switch the meson test registration from 'tap' to 'pytest'. The interactive tab-completion and pager tests drive psql through a pty via pexpect (the interactive_psql fixture in pyt/conftest.py) and skip when pexpect, readline or a working "wc -l" is unavailable. --- src/bin/psql/meson.build | 13 +- src/bin/psql/pyt/conftest.py | 137 ++++ src/bin/psql/pyt/test_001_basic.py | 750 ++++++++++++++++++++ src/bin/psql/pyt/test_010_tab_completion.py | 600 ++++++++++++++++ src/bin/psql/pyt/test_020_cancel.py | 72 ++ src/bin/psql/pyt/test_030_pager.py | 119 ++++ src/bin/psql/t/001_basic.pl | 543 -------------- src/bin/psql/t/010_tab_completion.pl | 468 ------------ src/bin/psql/t/020_cancel.pl | 51 -- src/bin/psql/t/030_pager.pl | 138 ---- 10 files changed, 1686 insertions(+), 1205 deletions(-) create mode 100644 src/bin/psql/pyt/conftest.py create mode 100644 src/bin/psql/pyt/test_001_basic.py create mode 100644 src/bin/psql/pyt/test_010_tab_completion.py create mode 100644 src/bin/psql/pyt/test_020_cancel.py create mode 100644 src/bin/psql/pyt/test_030_pager.py delete mode 100644 src/bin/psql/t/001_basic.pl delete mode 100644 src/bin/psql/t/010_tab_completion.pl delete mode 100644 src/bin/psql/t/020_cancel.pl delete mode 100644 src/bin/psql/t/030_pager.pl diff --git a/src/bin/psql/meson.build b/src/bin/psql/meson.build index 922b284526..7b90f21bad 100644 --- a/src/bin/psql/meson.build +++ b/src/bin/psql/meson.build @@ -71,13 +71,16 @@ tests += { 'name': 'psql', 'sd': meson.current_source_dir(), 'bd': meson.current_build_dir(), - 'tap': { + 'pytest': { 'env': {'with_readline': readline.found() ? 'yes' : 'no'}, + # 010_tab_completion and 030_pager drive psql through an interactive pty; + # they use pexpect via the interactive_psql fixture in pyt/conftest.py and + # skip when pexpect (or readline / a working "wc -l") is unavailable. 'tests': [ - 't/001_basic.pl', - 't/010_tab_completion.pl', - 't/020_cancel.pl', - 't/030_pager.pl', + 'pyt/test_001_basic.py', + 'pyt/test_010_tab_completion.py', + 'pyt/test_020_cancel.py', + 'pyt/test_030_pager.py', ], }, } diff --git a/src/bin/psql/pyt/conftest.py b/src/bin/psql/pyt/conftest.py new file mode 100644 index 0000000000..2879afc2bd --- /dev/null +++ b/src/bin/psql/pyt/conftest.py @@ -0,0 +1,137 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Interactive (pty-driven) psql helper for the psql test suite. + +Drives a psql session through a pseudo-terminal so that psql believes it is +interactive (readline/libedit and the pager are active), using pexpect. The +helper exposes a ``query_until`` method that sends input and reads output up to +a given pattern, with timeout tracking. + +pexpect is an optional dependency: the ``interactive_psql`` fixture issues +``pytest.importorskip("pexpect")`` so tests that need a real terminal +(010_tab_completion, 030_pager) skip cleanly where it is not installed. +""" + +import os + +import pytest + +from pypg.util import TIMEOUT_DEFAULT + + +class InteractivePsql: + """A psql session driven through a pseudo-terminal. + + psql believes it is interactive (so readline/libedit and the pager are + active). Output includes psql's prompts and the echoed input. + """ + + def __init__( + self, + pexpect_mod, + psql_path, + node, + dbname="postgres", + history_file=None, + extra_params=None, + extra_env=None, + dimensions=(24, 80), + timeout=TIMEOUT_DEFAULT, + ): + self._pexpect = pexpect_mod + self.timeout = timeout + # True if the most recent query_until() timed out. + self.timed_out = False + + # Since the invoked psql will believe it's interactive, it will use + # readline/libedit if available. Adjust the environment to prevent + # unwanted side-effects: + env = dict(os.environ) + # Redirect readline history somewhere harmless (or the caller's file). + env["PSQL_HISTORY"] = history_file or "/dev/null" + # Ignore any ~/.inputrc that could change readline's behavior. + env["INPUTRC"] = "/dev/null" + # Unset TERM so readline/libedit won't emit terminal-dependent escapes. + env.pop("TERM", None) + # Some versions of readline inspect LS_COLORS; drop it for luck. + env.pop("LS_COLORS", None) + if extra_env: + env.update(extra_env) + + args = [ + "--no-psqlrc", + "--no-align", + "--tuples-only", + "--dbname", + node.connstr(dbname), + ] + if extra_params: + args += list(extra_params) + + self.child = pexpect_mod.spawn( + psql_path, + args, + env=env, + encoding="utf-8", + timeout=timeout, + dimensions=dimensions, + ) + # Wait until psql has connected and printed its first prompt. + self.child.expect(r"=[#>] ") + + def query_until(self, until, send): + """Send *send*, then read output until regex *until* appears. + + *until* may be a regex string or a compiled pattern (use re.MULTILINE + for ``$``-anchored patterns). Returns the output produced since the + previous match -- psql prompts and echoed input included. On timeout, + sets ``timed_out`` and returns whatever was captured so far instead of + raising. + """ + self.timed_out = False + self.child.send(send) + try: + self.child.expect(until, timeout=self.timeout) + except self._pexpect.TIMEOUT: + self.timed_out = True + return self.child.before + return self.child.before + self.child.after + + def quit(self): + """Send an explicit \\q so the pty closes cleanly.""" + if self.child.isalive(): + try: + self.child.send("\\q\n") + self.child.expect(self._pexpect.EOF, timeout=self.timeout) + except (self._pexpect.TIMEOUT, self._pexpect.EOF, OSError): + pass + if self.child.isalive(): + self.child.close(force=True) + + +@pytest.fixture +def interactive_psql(bindir): + """Factory yielding :class:`InteractivePsql` sessions. + + Skips the whole test if pexpect is not installed. All sessions created + through the factory are quit at teardown. + """ + pexpect_mod = pytest.importorskip( + "pexpect", reason="pexpect is needed to drive an interactive psql" + ) + psql_path = os.path.join(bindir, "psql") + sessions = [] + + def _factory(node, dbname="postgres", **kwargs): + sess = InteractivePsql(pexpect_mod, psql_path, node, dbname, **kwargs) + sessions.append(sess) + return sess + + yield _factory + + for sess in sessions: + try: + sess.quit() + except Exception: # pylint: disable=broad-exception-caught + # Best-effort teardown; a session may already have exited. + pass diff --git a/src/bin/psql/pyt/test_001_basic.py b/src/bin/psql/pyt/test_001_basic.py new file mode 100644 index 0000000000..e82bb1c601 --- /dev/null +++ b/src/bin/psql/pyt/test_001_basic.py @@ -0,0 +1,750 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Exercise the psql binary itself. + +These tests cover psql's CLI flags, output formatting, error handling, +backslash commands and exit codes. Here psql is the program under test, so it +is run as a subprocess (via the installed binary), feeding scripts in on stdin +or via -c/-f. +""" + +import os +import re +import subprocess + +import pytest + + +# --------------------------------------------------------------------------- +# psql runner helpers backing the psql_like/psql_fails_like checks. +# --------------------------------------------------------------------------- + + +def _run_psql(pg, sql, *, replication=None, on_error_stop=True): + """Run psql, feeding *sql* on stdin, returning (ret, out, err). + + Uses --no-psqlrc --no-align --tuples-only --quiet, connecting via a connstr + (with optional ``replication=`` appended), reading the script from stdin + (-f -), with ON_ERROR_STOP on by default. stdout and stderr have a single + trailing newline removed. + """ + connstr = pg.connstr("postgres") + if replication is not None and replication != "": + connstr += f" replication={replication}" + argv = [ + os.path.join(pg.bindir, "psql"), + "--no-psqlrc", + "--no-align", + "--tuples-only", + "--quiet", + "--dbname", + connstr, + "--file", + "-", + ] + if on_error_stop: + argv += ["--variable", "ON_ERROR_STOP=1"] + print("# Running: " + " ".join(argv)) + proc = subprocess.run( + argv, + input=sql, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) + out = proc.stdout + err = proc.stderr + # Strip a single trailing newline. + if out.endswith("\n"): + out = out[:-1] + if err.endswith("\n"): + err = err[:-1] + return proc.returncode, out, err + + +def _psql_like(pg, sql, expected_stdout, test_name): + """Run *sql* and check exit 0, empty stderr, stdout matching the regex.""" + ret, stdout, stderr = _run_psql(pg, sql) + assert ret == 0, f"{test_name}: exit code 0 (got {ret}, stderr: {stderr})" + assert stderr == "", f"{test_name}: no stderr (got: {stderr})" + assert re.search( + expected_stdout, stdout + ), f"{test_name}: stdout matches /{expected_stdout.pattern}/\n{stdout}" + + +def _psql_fails_like(pg, sql, expected_stderr, test_name, replication=None): + """Run *sql* and check nonzero exit and stderr matching the regex.""" + ret, stdout, stderr = _run_psql(pg, sql, replication=replication) + assert ret != 0, f"{test_name}: exit code not 0\n{stdout}\n{stderr}" + assert re.search( + expected_stderr, stderr + ), f"{test_name}: stderr matches /{expected_stderr.pattern}/\n{stderr}" + + +def _append_to_file(path, text): + with open(path, "a", encoding="utf-8") as fh: + fh.write(text) + + +def _slurp_file(path): + with open(path, "r", encoding="utf-8", errors="replace") as fh: + return fh.read() + + +# --------------------------------------------------------------------------- +# Program-level checks that need no server. +# --------------------------------------------------------------------------- + + +def test_program_help_version_options(pg_bin): + pg_bin.program_help_ok("psql") + pg_bin.program_version_ok("psql") + pg_bin.program_options_handling_ok("psql") + + +@pytest.mark.parametrize("arg", ["commands", "variables"]) +def test_help_arg(pg_bin, arg): + # Test --help=foo, analogous to program_help_ok(). + res = pg_bin.result(["psql", f"--help={arg}"]) + assert res.returncode == 0, f"psql --help={arg} exit code 0" + assert res.stdout != "", f"psql --help={arg} goes to stdout" + assert res.stderr == "", f"psql --help={arg} nothing to stderr" + + +# --------------------------------------------------------------------------- +# Server fixture: --locale=C --encoding=UTF8 (from init defaults) plus the +# logical replication settings. +# --------------------------------------------------------------------------- + + +@pytest.fixture +def node(create_pg): + return create_pg( + "main", + allows_streaming="logical", + initdb_extra=["--locale=C", "--encoding=UTF8"], + ) + + +# --------------------------------------------------------------------------- +# Basic backslash commands. +# --------------------------------------------------------------------------- + + +def test_copyright(node): + _psql_like(node, "\\copyright", re.compile(r"Copyright"), "\\copyright") + + +def test_help_no_args(node): + _psql_like(node, "\\help", re.compile(r"ALTER"), "\\help without arguments") + + +def test_help_with_arg(node): + _psql_like(node, "\\help SELECT", re.compile(r"SELECT"), "\\help with argument") + + +def test_unsupported_replication_command(node): + # Clean handling of unsupported replication command responses. + _psql_fails_like( + node, + "START_REPLICATION 0/0", + re.compile(r"unexpected PQresultStatus: 8$"), + "handling of unexpected PQresultStatus", + replication="database", + ) + + +def test_timing_successful_query(node): + _psql_like( + node, + "\\timing on\nSELECT 1", + re.compile(r"^1$\n^Time: \d+[.,]\d\d\d ms", re.M), + "\\timing with successful query", + ) + + +def test_timing_query_error(node): + ret, stdout, _stderr = _run_psql(node, "\\timing on\nSELECT error") + assert ret != 0, "\\timing with query error: query failed" + assert re.search(r"^Time: \d+[.,]\d\d\d ms", stdout, re.M), ( + "\\timing with query error: timing output appears\n" + stdout + ) + assert not re.search(r"^Time: 0[.,]000 ms", stdout, re.M), ( + "\\timing with query error: timing was updated\n" + stdout + ) + + +def test_encoding_variable(node): + # ENCODING variable is set and updated when client encoding changes. + _psql_like( + node, + "\\echo :ENCODING\nset client_encoding = LATIN1;\n\\echo :ENCODING", + re.compile(r"^UTF8$\n^LATIN1$", re.M), + "ENCODING variable is set and updated", + ) + + +def test_listen_notify(node): + _psql_like( + node, + "LISTEN foo;\nNOTIFY foo;", + re.compile( + r'^Asynchronous notification "foo" received from server ' + r"process with PID \d+\.$", + re.M, + ), + "notification", + ) + + +def test_listen_notify_payload(node): + _psql_like( + node, + "LISTEN foo;\nNOTIFY foo, 'bar';", + re.compile( + r'^Asynchronous notification "foo" with payload "bar" received ' + r"from server process with PID \d+\.$", + re.M, + ), + "notification with payload", + ) + + +def test_server_crash(node): + # Behavior and output on server crash. + ret, out, err = _run_psql( + node, + "SELECT 'before' AS running;\n" + "SELECT pg_terminate_backend(pg_backend_pid());\n" + "SELECT 'AFTER' AS not_running;\n", + ) + assert ret == 2, f"server crash: psql exit code (got {ret})" + assert re.search(r"before", out), "server crash: output before crash" + assert not re.search(r"AFTER", out), "server crash: no output after crash" + assert re.search( + r"psql::2: FATAL: terminating connection due to administrator command\n" + r"psql::2: server closed the connection unexpectedly\n" + r"\tThis probably means the server terminated abnormally\n" + r"\tbefore or while processing the request\.\n" + r"psql::2: error: connection to server was lost", + err, + ), ( + "server crash: error message\n" + err + ) + + +# --------------------------------------------------------------------------- +# \errverbose +# +# (Not in the regular regression tests because the output contains the source +# code location which we don't want to have to update.) +# --------------------------------------------------------------------------- + + +def test_errverbose_no_previous_error(node): + _psql_like( + node, + "SELECT 1;\n\\errverbose", + re.compile(r"^1\nThere is no previous error\.$"), + "\\errverbose with no previous error", + ) + + +# There are three main ways to run a query that might affect \errverbose: the +# normal way, piecemeal retrieval using FETCH_COUNT, and using \gdesc. + +ERRVERBOSE_CASES = [ + ( + "SELECT error;\n\\errverbose", + r"\A^psql::1: ERROR: .*$\n" + r"^LINE 1: SELECT error;$\n" + r"^ *\^.*$\n" + r"^psql::2: error: ERROR: [0-9A-Z]{5}: .*$\n" + r"^LINE 1: SELECT error;$\n" + r"^ *\^.*$\n" + r"^LOCATION: .*$", + "\\errverbose after normal query with error", + ), + ( + "\\set FETCH_COUNT 1\nSELECT error;\n\\errverbose", + r"\A^psql::2: ERROR: .*$\n" + r"^LINE 1: SELECT error;$\n" + r"^ *\^.*$\n" + r"^psql::3: error: ERROR: [0-9A-Z]{5}: .*$\n" + r"^LINE 1: SELECT error;$\n" + r"^ *\^.*$\n" + r"^LOCATION: .*$", + "\\errverbose after FETCH_COUNT query with error", + ), + ( + "SELECT error\\gdesc\n\\errverbose", + r"\A^psql::1: ERROR: .*$\n" + r"^LINE 1: SELECT error$\n" + r"^ *\^.*$\n" + r"^psql::2: error: ERROR: [0-9A-Z]{5}: .*$\n" + r"^LINE 1: SELECT error$\n" + r"^ *\^.*$\n" + r"^LOCATION: .*$", + "\\errverbose after \\gdesc with error", + ), +] + + +@pytest.mark.parametrize( + "sql,pattern,name", ERRVERBOSE_CASES, ids=[c[2] for c in ERRVERBOSE_CASES] +) +def test_errverbose(node, sql, pattern, name): + _ret, _stdout, stderr = _run_psql(node, sql, on_error_stop=False) + assert re.search(pattern, stderr, re.M), f"{name}\n{stderr}" + + +# --------------------------------------------------------------------------- +# Multiple -c and -f switches. +# +# Note that we cannot test backend-side errors as tests are unstable in this +# case. +# --------------------------------------------------------------------------- + + +def test_single_transaction_multiple_switches(node, tmp_path): + tempdir = str(tmp_path) + node.safe_sql("CREATE TABLE tab_psql_single (a int);") + + def row_count(): + return node.safe_sql("SELECT count(*) FROM tab_psql_single").strip() + + # Tests with ON_ERROR_STOP. + node.command_ok( + [ + "psql", + "--no-psqlrc", + "--single-transaction", + "--set", + "ON_ERROR_STOP=1", + "--command", + "INSERT INTO tab_psql_single VALUES (1)", + "--command", + "INSERT INTO tab_psql_single VALUES (2)", + ], + "ON_ERROR_STOP, --single-transaction and multiple -c switches", + ) + assert row_count() == "2", ( + "--single-transaction commits transaction, ON_ERROR_STOP and " + "multiple -c switches" + ) + + node.command_fails( + [ + "psql", + "--no-psqlrc", + "--single-transaction", + "--set", + "ON_ERROR_STOP=1", + "--command", + "INSERT INTO tab_psql_single VALUES (3)", + "--command", + f"\\copy tab_psql_single FROM '{tempdir}/nonexistent'", + ], + "ON_ERROR_STOP, --single-transaction and multiple -c switches, error", + ) + assert row_count() == "2", ( + "client-side error rolls back transaction, ON_ERROR_STOP and " + "multiple -c switches" + ) + + # Tests mixing files and commands. + copy_sql_file = os.path.join(tempdir, "tab_copy.sql") + insert_sql_file = os.path.join(tempdir, "tab_insert.sql") + _append_to_file( + copy_sql_file, f"\\copy tab_psql_single FROM '{tempdir}/nonexistent';" + ) + _append_to_file(insert_sql_file, "INSERT INTO tab_psql_single VALUES (4);") + + node.command_ok( + [ + "psql", + "--no-psqlrc", + "--single-transaction", + "--set", + "ON_ERROR_STOP=1", + "--file", + insert_sql_file, + "--file", + insert_sql_file, + ], + "ON_ERROR_STOP, --single-transaction and multiple -f switches", + ) + assert row_count() == "4", ( + "--single-transaction commits transaction, ON_ERROR_STOP and " + "multiple -f switches" + ) + + node.command_fails( + [ + "psql", + "--no-psqlrc", + "--single-transaction", + "--set", + "ON_ERROR_STOP=1", + "--file", + insert_sql_file, + "--file", + copy_sql_file, + ], + "ON_ERROR_STOP, --single-transaction and multiple -f switches, error", + ) + assert row_count() == "4", ( + "client-side error rolls back transaction, ON_ERROR_STOP and " + "multiple -f switches" + ) + + # Tests without ON_ERROR_STOP. + # The last switch fails on \copy. The command returns a failure and the + # transaction commits. + node.command_fails( + [ + "psql", + "--no-psqlrc", + "--single-transaction", + "--file", + insert_sql_file, + "--file", + insert_sql_file, + "--command", + f"\\copy tab_psql_single FROM '{tempdir}/nonexistent'", + ], + "no ON_ERROR_STOP, --single-transaction and multiple -f/-c switches", + ) + assert row_count() == "6", ( + "client-side error commits transaction, no ON_ERROR_STOP and " + "multiple -f/-c switches" + ) + + # The last switch fails on \copy coming from an input file. The command + # returns a success and the transaction commits. + node.command_ok( + [ + "psql", + "--no-psqlrc", + "--single-transaction", + "--file", + insert_sql_file, + "--file", + insert_sql_file, + "--file", + copy_sql_file, + ], + "no ON_ERROR_STOP, --single-transaction and multiple -f switches", + ) + assert row_count() == "8", ( + "client-side error commits transaction, no ON_ERROR_STOP and " + "multiple -f switches" + ) + + # The last switch makes the command return a success, and the contents of + # the transaction commit even if there is a failure in-between. + node.command_ok( + [ + "psql", + "--no-psqlrc", + "--single-transaction", + "--command", + "INSERT INTO tab_psql_single VALUES (5)", + "--file", + copy_sql_file, + "--command", + "INSERT INTO tab_psql_single VALUES (6)", + ], + "no ON_ERROR_STOP, --single-transaction and multiple -c switches", + ) + assert row_count() == "10", ( + "client-side error commits transaction, no ON_ERROR_STOP and " + "multiple -c switches" + ) + + +def test_copy_from_with_default(node, tmp_path): + # Test \copy from with DEFAULT option. + node.safe_sql( + "CREATE TABLE copy_default (" + "id integer PRIMARY KEY, " + "text_value text NOT NULL DEFAULT 'test', " + "ts_value timestamp without time zone NOT NULL DEFAULT '2022-07-05')" + ) + + copy_default_file = str(tmp_path / "copy_default.csv") + _append_to_file(copy_default_file, "1,value,2022-07-04\n") + _append_to_file(copy_default_file, "2,placeholder,2022-07-03\n") + _append_to_file(copy_default_file, "3,placeholder,placeholder\n") + + _psql_like( + node, + f"\\copy copy_default from {copy_default_file} with " + "(format 'csv', default 'placeholder');\n" + "SELECT * FROM copy_default", + re.compile( + r"1\|value\|2022-07-04 00:00:00\n" + r"2\|test\|2022-07-03 00:00:00\n" + r"3\|test\|2022-07-05 00:00:00" + ), + "\\copy from with DEFAULT", + ) + + +# --------------------------------------------------------------------------- +# \watch +# --------------------------------------------------------------------------- + + +def test_watch_three_iterations(node): + # Note: the interval value is parsed with locale-aware strtod(); in the C + # locale the formatting is the same as Python's %g. + _psql_like( + node, + "SELECT 1 \\watch c=3 i=%g" % 0.01, + re.compile(r"1\n1\n1"), + "\\watch with 3 iterations, interval of 0.01", + ) + + +def test_watch_submillisecond(node): + # Sub-millisecond wait works, equivalent to 0. + _psql_like( + node, + "SELECT 1 \\watch c=3 i=%g" % 0.0001, + re.compile(r"1\n1\n1"), + "\\watch with 3 iterations, interval of 0.0001", + ) + + +def test_watch_zero_interval(node): + _psql_like( + node, + "\\set WATCH_INTERVAL 0\nSELECT 1 \\watch c=3", + re.compile(r"1\n1\n1"), + "\\watch with 3 iterations, interval of 0", + ) + + +def test_watch_invalid_minimum_row(node): + _psql_fails_like( + node, + "SELECT 3 \\watch m=x", + re.compile(r"incorrect minimum row count"), + "\\watch, invalid minimum row setting", + ) + + +def test_watch_minimum_rows_twice(node): + _psql_fails_like( + node, + "SELECT 3 \\watch m=1 min_rows=2", + re.compile(r"minimum row count specified more than once"), + "\\watch, minimum rows is specified more than once", + ) + + +def test_watch_two_minimum_rows(node): + _psql_like( + node, + "with x as (\n" + "\t\tselect now()-backend_start AS howlong\n" + "\t\tfrom pg_stat_activity\n" + "\t\twhere pid = pg_backend_pid()\n" + "\t ) select 123 from x where howlong < '2 seconds' \\watch i=%g m=2" % 0.5, + re.compile(r"^123$", re.M), + "\\watch, 2 minimum rows", + ) + + +WATCH_ERROR_CASES = [ + ( + "SELECT 1 \\watch -10", + r'incorrect interval value "-10"', + "\\watch, negative interval", + ), + ( + "SELECT 1 \\watch 10ab", + r'incorrect interval value "10ab"', + "\\watch, incorrect interval", + ), + ( + "SELECT 1 \\watch 10e400", + r'incorrect interval value "10e400"', + "\\watch, out-of-range interval", + ), + ( + "SELECT 1 \\watch 1 1", + r"interval value is specified more than once", + "\\watch, interval value is specified more than once", + ), + ( + "SELECT 1 \\watch c=1 c=1", + r"iteration count is specified more than once", + "\\watch, iteration count is specified more than once", + ), +] + + +@pytest.mark.parametrize( + "sql,pattern,name", WATCH_ERROR_CASES, ids=[c[2] for c in WATCH_ERROR_CASES] +) +def test_watch_errors(node, sql, pattern, name): + _psql_fails_like(node, sql, re.compile(pattern), name) + + +def test_watch_interval_variable(node): + # Check WATCH_INTERVAL. + _psql_like( + node, + "\\echo :WATCH_INTERVAL\n" + "\\set WATCH_INTERVAL 10\n" + "\\echo :WATCH_INTERVAL\n" + "\\unset WATCH_INTERVAL\n" + "\\echo :WATCH_INTERVAL", + re.compile(r"^2$\n^10$\n^2$", re.M), + "WATCH_INTERVAL variable is set and updated", + ) + _psql_fails_like( + node, + "\\set WATCH_INTERVAL 1e500", + re.compile(r"is out of range"), + "WATCH_INTERVAL variable is out of range", + ) + _psql_like( + node, + "\\echo :WATCH_INTERVAL", + re.compile(r"^2$", re.M), + "WATCH_INTERVAL variable was not altered", + ) + + +# --------------------------------------------------------------------------- +# \g output piped into a program. +# +# The program is "perl -pe ''" to simply copy the input to the output. +# --------------------------------------------------------------------------- + + +def test_g_pipe(node, tmp_path): + import shutil + + perlbin = shutil.which("perl") + if perlbin is None: + pytest.skip("perl not available for \\g pipe test") + + g_file = str(tmp_path / "g_file_1.out") + pipe_cmd = f"{perlbin} -pe '' >{g_file}" + + _psql_like( + node, f"SELECT 'one' \\g | {pipe_cmd}", re.compile(r""), "one command \\g" + ) + c1 = _slurp_file(g_file) + assert re.search(r"one", c1) + + _psql_like( + node, + f"SELECT 'two' \\; SELECT 'three' \\g | {pipe_cmd}", + re.compile(r""), + "two commands \\g", + ) + c2 = _slurp_file(g_file) + assert re.search(r"two.*three", c2, re.S) + + _psql_like( + node, + f"\\set SHOW_ALL_RESULTS 0\nSELECT 'four' \\; SELECT 'five' \\g | {pipe_cmd}", + re.compile(r""), + "two commands \\g with only last result", + ) + c3 = _slurp_file(g_file) + assert re.search(r"five", c3) + assert not re.search(r"four", c3) + + _psql_like( + node, + f"copy (values ('foo'),('bar')) to stdout \\g | {pipe_cmd}", + re.compile(r""), + "copy output passed to \\g pipe", + ) + c4 = _slurp_file(g_file) + assert re.search(r"foo.*bar", c4, re.S) + + +# --------------------------------------------------------------------------- +# COPY within pipelines. These abort the connection from the frontend so they +# cannot be tested via SQL. +# --------------------------------------------------------------------------- + + +def test_copy_from_in_pipeline(node): + node.safe_sql("CREATE TABLE psql_pipeline()") + log_location = node.log_position() + _psql_fails_like( + node, + "\\startpipeline\n" + "COPY psql_pipeline FROM STDIN;\n" + "SELECT 'val1';\n" + "\\syncpipeline\n" + "\\endpipeline", + re.compile(r"COPY in a pipeline is not supported, aborting connection"), + "COPY FROM in pipeline: fails", + ) + node.wait_for_log( + r"FATAL: .*terminating connection because protocol synchronization was lost", + log_location, + ) + + +def test_copy_to_in_pipeline(node): + node.safe_sql("CREATE TABLE psql_pipeline()") + # Remove \syncpipeline here. + _psql_fails_like( + node, + "\\startpipeline\n" + "COPY psql_pipeline TO STDOUT;\n" + "SELECT 'val1';\n" + "\\endpipeline", + re.compile(r"COPY in a pipeline is not supported, aborting connection"), + "COPY TO in pipeline: fails", + ) + + +def test_copy_meta_from_in_pipeline(node): + node.safe_sql("CREATE TABLE psql_pipeline()") + _psql_fails_like( + node, + "\\startpipeline\n" + "\\copy psql_pipeline from stdin;\n" + "SELECT 'val1';\n" + "\\syncpipeline\n" + "\\endpipeline", + re.compile(r"COPY in a pipeline is not supported, aborting connection"), + "\\copy from in pipeline: fails", + ) + + +def test_copy_meta_to_in_pipeline(node): + node.safe_sql("CREATE TABLE psql_pipeline()") + # Sync attempt after a COPY TO/FROM. + _psql_fails_like( + node, + "\\startpipeline\n" + "\\copy psql_pipeline to stdout;\n" + "\\syncpipeline\n" + "\\endpipeline", + re.compile(r"COPY in a pipeline is not supported, aborting connection"), + "\\copy to in pipeline: fails", + ) + + +def test_meta_command_in_restrict_mode(node): + _psql_fails_like( + node, + "\\restrict test\n\\! should_fail", + re.compile(r"backslash commands are restricted; only \\unrestrict is allowed"), + "meta-command in restrict mode fails", + ) diff --git a/src/bin/psql/pyt/test_010_tab_completion.py b/src/bin/psql/pyt/test_010_tab_completion.py new file mode 100644 index 0000000000..62d7d04a89 --- /dev/null +++ b/src/bin/psql/pyt/test_010_tab_completion.py @@ -0,0 +1,600 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Exercise psql's readline/libedit tab-completion machinery. + +Drives an interactive psql session through a pty (via the ``interactive_psql`` +fixture in conftest.py) and checks, for a long list of inputs, that readline +completes the partial command/identifier/filename as expected. + +This test only runs when the build is --with-readline (the ``with_readline`` +environment variable is 'yes'), and is skipped when SKIP_READLINE_TESTS is set +or when pexpect is unavailable. +""" + +import os +import re + +import pytest + + +# --------------------------------------------------------------------------- +# Helpers: check_completion / clear_query / clear_line. +# These are deliberately NOT named test_* -- they are called by the single +# test function below. +# --------------------------------------------------------------------------- + + +def check_completion(psql, send, pattern, annotation): + """Type *send* and check psql responds with output matching *pattern*. + + *pattern* is a compiled regex. Send the data, wait for its result, and + assert the captured output matches the pattern and that the query did not + time out. + """ + out = psql.query_until(pattern, send) + assert pattern.search(out) and not psql.timed_out, ( + f"{annotation}\nActual output was {out!r}\n" + f"Did not match {pattern.pattern!r}" + ) + + +def clear_query(psql): + """Clear the query buffer to start over. + + (won't work if we are inside a string literal!) + """ + check_completion( + psql, + "\\r\n", + re.compile(r"Query buffer reset.*postgres=# ", re.DOTALL), + "\\r works", + ) + + +def clear_line(psql): + """Clear the current line to start over. + + (this will work in an incomplete string literal, but it's less desirable + than clear_query because we lose evidence in the history file) + """ + check_completion(psql, "\025\n", re.compile(r"postgres=# "), "control-U works") + + +# --------------------------------------------------------------------------- +# The test. +# --------------------------------------------------------------------------- + + +def test_010_tab_completion(pg, interactive_psql, tmp_path): + # Do nothing unless the build is --with-readline. + if os.environ.get("with_readline") != "yes": + pytest.skip("readline is not supported by this build") + + # Also, skip if user has set environment variable to command that. This is + # mainly intended to allow working around some of the more broken versions + # of libedit --- some users might find them acceptable even if they won't + # pass these tests. + if "SKIP_READLINE_TESTS" in os.environ: + pytest.skip("SKIP_READLINE_TESTS is set") + + # (pexpect availability is handled by the interactive_psql fixture, which + # issues pytest.importorskip("pexpect").) + + node = pg + + # set up a few database objects + node.safe_sql( + "CREATE TABLE tab1 (c1 int primary key constraint foo not null, c2 text);\n" + "CREATE TABLE mytab123 (f1 int, f2 text);\n" + "CREATE TABLE mytab246 (f1 int, f2 text);\n" + 'CREATE TABLE "mixedName" (f1 int, f2 text);\n' + "CREATE TYPE enum1 AS ENUM ('foo', 'bar', 'baz', 'BLACK');\n" + "CREATE PUBLICATION some_publication;\n" + "CREATE TABLE fpo_test (id int4range, valid_at daterange, name text);\n" + ) + + # Use relative paths to access the tab_comp_dir subdirectory; otherwise the + # output from filename completion tests is too variable. We do this by + # creating tab_comp_dir under tmp_path and chdir'ing there for the duration + # of the test, restoring the cwd afterwards. + saved_cwd = os.getcwd() + os.chdir(str(tmp_path)) + try: + # Create some junk files for filename completion testing. + os.mkdir("tab_comp_dir") + with open("tab_comp_dir/somefile", "w", encoding="utf-8") as fh: + fh.write("some stuff\n") + with open("tab_comp_dir/afile123", "w", encoding="utf-8") as fh: + fh.write("more stuff\n") + with open("tab_comp_dir/afile456", "w", encoding="utf-8") as fh: + fh.write("other stuff\n") + + # Arrange to capture, not discard, the interactive session's history + # output. We put it under tmp_path so buildfarm runs can still capture + # it for debugging. + historyfile = str(tmp_path / "010_psql_history.txt") + + # fire up an interactive psql session + psql = interactive_psql(node, dbname="postgres", history_file=historyfile) + + _run_cases(psql) + + # send psql an explicit \q to shut it down, else pty won't close + # properly + psql.quit() + finally: + os.chdir(saved_cwd) + + +def _run_cases(psql): + # check basic command completion: SEL produces SELECT + check_completion( + psql, "SEL\t", re.compile(r"SELECT "), "complete SEL to SELECT" + ) + + clear_query(psql) + + # check case variation is honored + check_completion( + psql, "sel\t", re.compile(r"select "), "complete sel to select" + ) + + # check basic table name completion + check_completion( + psql, "* from t\t", re.compile(r"\* from tab1 "), "complete t to tab1" + ) + + clear_query(psql) + + # check table name completion with multiple alternatives + # note: readline might print a bell before the completion + check_completion( + psql, + "select * from my\t", + re.compile(r"select \* from my\a?tab"), + "complete my to mytab when there are multiple choices", + ) + + # some versions of readline/libedit require two tabs here, some only need + # one + check_completion( + psql, + "\t\t", + re.compile(r"mytab123 +mytab246"), + "offer multiple table choices", + ) + + check_completion( + psql, + "2\t", + re.compile(r"246 "), + "finish completion of one of multiple table choices", + ) + + clear_query(psql) + + # check handling of quoted names + check_completion( + psql, + 'select * from "my\t', + re.compile(r'select \* from "my\a?tab'), + 'complete "my to "mytab when there are multiple choices', + ) + + check_completion( + psql, + "\t\t", + re.compile(r'"mytab123" +"mytab246"'), + "offer multiple quoted table choices", + ) + + check_completion( + psql, + "2\t", + re.compile(r'246" '), + "finish completion of one of multiple quoted table choices", + ) + + clear_query(psql) + + # check handling of mixed-case names + check_completion( + psql, + 'select * from "mi\t', + re.compile(r'"mixedName" '), + "complete a mixed-case name", + ) + + clear_query(psql) + + # check case folding + check_completion( + psql, + "select * from TAB\t", + re.compile(r"tab1 "), + "automatically fold case", + ) + + clear_query(psql) + + # check case-sensitive keyword replacement + # note: various versions of readline/libedit handle backspacing + # differently, so just check that the replacement comes out correctly + check_completion( + psql, "\\DRD\t", re.compile(r"drds "), "complete \\DRD to \\drds" + ) + + clear_query(psql) + + # check completion of a schema-qualified name + check_completion( + psql, + "select * from pub\t", + re.compile(r"public\."), + "complete schema when relevant", + ) + + check_completion( + psql, "tab\t", re.compile(r"tab1 "), "complete schema-qualified name" + ) + + clear_query(psql) + + check_completion( + psql, + "select * from PUBLIC.t\t", + re.compile(r"public\.tab1 "), + "automatically fold case in schema-qualified name", + ) + + clear_query(psql) + + # check interpretation of referenced names + check_completion( + psql, + "alter table tab1 drop constraint t\t", + re.compile(r"tab1_pkey "), + "complete index name for referenced table", + ) + + clear_query(psql) + + check_completion( + psql, + "alter table TAB1 drop constraint t\t", + re.compile(r"tab1_pkey "), + "complete index name for referenced table, with downcasing", + ) + + clear_query(psql) + + check_completion( + psql, + 'alter table public."tab1" drop constraint t\t', + re.compile(r"tab1_pkey "), + "complete index name for referenced table, with schema and quoting", + ) + + clear_query(psql) + + # check variant where we're completing a qualified name from a refname + # (this one also checks successful completion in a multiline command) + check_completion( + psql, + "comment on constraint tab1_pkey \n on public.\t", + re.compile(r"public\.tab1"), + "complete qualified name from object reference", + ) + + clear_query(psql) + + # check filename completion + check_completion( + psql, + "\\lo_import tab_comp_dir/some\t", + re.compile(r"tab_comp_dir/somefile "), + "filename completion with one possibility", + ) + + clear_query(psql) + + # note: readline might print a bell before the completion + check_completion( + psql, + "\\lo_import tab_comp_dir/af\t", + re.compile(r"tab_comp_dir/af\a?ile"), + "filename completion with multiple possibilities", + ) + + # here we are inside a string literal 'afile*', so must use clear_line(). + clear_line(psql) + + # COPY requires quoting + check_completion( + psql, + "COPY foo FROM tab_comp_dir/some\t", + re.compile(r"'tab_comp_dir/somefile' "), + "quoted filename completion with one possibility", + ) + + clear_query(psql) + + check_completion( + psql, + "COPY foo FROM tab_comp_dir/af\t", + re.compile(r"'tab_comp_dir/afile"), + "quoted filename completion with multiple possibilities", + ) + + # some versions of readline/libedit require two tabs here, some only need + # one + # also, some will offer the whole path name and some just the file name + # the quotes might appear, too + check_completion( + psql, + "\t\t", + re.compile(r"afile123'? +'?(tab_comp_dir/)?afile456"), + "offer multiple file choices", + ) + + clear_line(psql) + + # check enum label completion + # some versions of readline/libedit require two tabs here, some only need + # one + # also, some versions will offer quotes, some will not + check_completion( + psql, + "ALTER TYPE enum1 RENAME VALUE 'ba\t\t", + re.compile(r"'?bar'? +'?baz'?"), + "offer multiple enum choices", + ) + + clear_line(psql) + + # enum labels are case sensitive, so this should complete BLACK immediately + check_completion( + psql, + "ALTER TYPE enum1 RENAME VALUE 'B\t", + re.compile(r"BLACK"), + "enum labels are case sensitive", + ) + + clear_line(psql) + + # check timezone name completion + check_completion( + psql, + "SET timezone TO am\t", + re.compile(r"'America/"), + "offer partial timezone name", + ) + + check_completion( + psql, "new_\t", re.compile(r"New_York"), "complete partial timezone name" + ) + + clear_line(psql) + + # check completion of a keyword offered in addition to object names; + # such a keyword should obey COMP_KEYWORD_CASE + for case, in_, out in ( + ("lower", "CO", "column"), + ("upper", "co", "COLUMN"), + ("preserve-lower", "co", "column"), + ("preserve-upper", "CO", "COLUMN"), + ): + check_completion( + psql, + f"\\set COMP_KEYWORD_CASE {case}\n", + re.compile(r"postgres=#"), + f"set completion case to '{case}'", + ) + check_completion( + psql, + f"alter table tab1 rename {in_}\t\t\t", + re.compile(out), + f"offer keyword {out} for input {in_}, " f"COMP_KEYWORD_CASE = {case}", + ) + clear_query(psql) + + # alternate path where keyword comes from SchemaQuery + check_completion( + psql, + "DROP TYPE big\t", + re.compile(r"DROP TYPE bigint "), + "offer keyword from SchemaQuery", + ) + + clear_query(psql) + + # check create_command_generator + check_completion( + psql, + "CREATE TY\t", + re.compile(r"CREATE TYPE "), + "check create_command_generator", + ) + + clear_query(psql) + + # check words_after_create infrastructure + check_completion( + psql, + "CREATE TABLE mytab\t\t", + re.compile(r"mytab123 +mytab246"), + "check words_after_create", + ) + + clear_query(psql) + + # check VersionedQuery infrastructure + check_completion( + psql, + "DROP PUBLIC\t \t\t", + re.compile(r"DROP PUBLICATION\s+some_publication "), + "check VersionedQuery", + ) + + clear_query(psql) + + # hits ends_with() and logic for completing in multi-line queries + check_completion( + psql, + "analyze (\n\t\t", + re.compile(r"VERBOSE"), + "check ANALYZE (VERBOSE ...", + ) + + clear_query(psql) + + # check completions for GUCs + check_completion( + psql, + "set interval\t\t", + re.compile(r"intervalstyle TO"), + "complete a GUC name", + ) + check_completion( + psql, " iso\t", re.compile(r"iso_8601 "), "complete a GUC enum value" + ) + + clear_query(psql) + + # same, for qualified GUC names + check_completion( + psql, + "DO $$begin end$$ LANGUAGE plpgsql;\n", + re.compile(r"postgres=# "), + "load plpgsql extension", + ) + + check_completion( + psql, + "set plpg\t", + re.compile(r"plpg\a?sql\."), + "complete prefix of a GUC name", + ) + check_completion( + psql, + "var\t\t", + re.compile(r"variable_conflict TO"), + "complete a qualified GUC name", + ) + check_completion( + psql, + " USE_C\t", + re.compile(r"use_column"), + "complete a qualified GUC enum value", + ) + + clear_query(psql) + + # check completions for psql variables + check_completion( + psql, + "\\set VERB\t", + re.compile(r"VERBOSITY "), + "complete a psql variable name", + ) + check_completion( + psql, "def\t", re.compile(r"default "), "complete a psql variable value" + ) + + clear_query(psql) + + check_completion( + psql, + "\\echo :VERB\t", + re.compile(r":VERBOSITY "), + "complete an interpolated psql variable name", + ) + + clear_query(psql) + + # check completion for psql variable test + check_completion( + psql, + "\\echo :{?VERB\t", + re.compile(r":\{\?VERBOSITY} "), + "complete a psql variable test", + ) + + clear_query(psql) + + # check no-completions code path + check_completion( + psql, "blarg \t\t", re.compile(r""), "check completion failure path" + ) + + clear_query(psql) + + # check COPY FROM with DEFAULT option + check_completion( + psql, + "COPY foo FROM stdin WITH ( DEF\t)", + re.compile(r"DEFAULT "), + "COPY FROM with DEFAULT completion", + ) + + clear_line(psql) + + # check tab completion for DELETE ... FOR PORTION OF + check_completion( + psql, + "DELETE FROM fpo_test F\t", + re.compile(r"FOR "), + "complete DELETE FROM F to FOR", + ) + + check_completion( + psql, "P\t", re.compile(r"PORTION "), "complete FOR P to PORTION" + ) + + check_completion(psql, "O\t", re.compile(r"OF "), "complete PORTION O to OF") + + check_completion( + psql, + "v\t", + re.compile(r"valid_at "), + "complete FOR PORTION OF offers column names", + ) + + check_completion( + psql, + "FR\t", + re.compile(r"FROM "), + "complete FOR PORTION OF FR to FROM", + ) + + clear_query(psql) + + # check tab completion for UPDATE ... FOR PORTION OF + check_completion( + psql, + "UPDATE fpo_test F\t", + re.compile(r"FOR "), + "complete UPDATE
F to FOR", + ) + + check_completion( + psql, "P\t", re.compile(r"PORTION "), "complete FOR P to PORTION" + ) + + check_completion(psql, "O\t", re.compile(r"OF "), "complete PORTION O to OF") + + check_completion( + psql, + "v\t", + re.compile(r"valid_at "), + "complete FOR PORTION OF offers column names", + ) + + check_completion( + psql, + "FR\t", + re.compile(r"FROM "), + "complete FOR PORTION OF FR to FROM", + ) + + clear_query(psql) diff --git a/src/bin/psql/pyt/test_020_cancel.py b/src/bin/psql/pyt/test_020_cancel.py new file mode 100644 index 0000000000..18ff735d3b --- /dev/null +++ b/src/bin/psql/pyt/test_020_cancel.py @@ -0,0 +1,72 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Test query canceling by sending SIGINT to a running psql. A long-running +query is started in a background psql; once the server reports the backend +is executing it (observed in-process via pg_stat_activity), SIGINT is sent to +the psql process group and the resulting cancel error is checked. +""" + +import os +import signal +import subprocess +import sys + +import pytest + +from pypg.util import TIMEOUT_DEFAULT + + +# Sending SIGINT on Windows terminates the test itself, so skip there. +pytestmark = pytest.mark.skipif( + sys.platform == "win32", + reason="sending SIGINT on Windows terminates the test itself", +) + + +def test_020_cancel(pg): + node = pg + + with subprocess.Popen( + [ + os.path.join(node.bindir, "psql"), + "--no-psqlrc", + "--set", + "ON_ERROR_STOP=1", + "-h", + node.host, + "-p", + str(node.port), + "postgres", + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + # Run in its own session/process group so we can deliver SIGINT to the + # psql process without it also hitting the test process. + start_new_session=True, + ) as psql: + try: + # Send sleep command and wait until the server has registered it. + psql.stdin.write(f"select pg_sleep({TIMEOUT_DEFAULT});\n") + psql.stdin.flush() + + assert node.poll_query_until( + "SELECT (SELECT count(*) FROM pg_stat_activity " + "WHERE query ~ '^select pg_sleep') > 0;" + ), "timed out waiting for the backend to start the sleep query" + + # Send cancel request (SIGINT to psql's process group). + psql.send_signal(signal.SIGINT) + + _stdout, stderr = psql.communicate(timeout=TIMEOUT_DEFAULT) + finally: + if psql.poll() is None: + psql.kill() + psql.communicate() + + # The query failed as expected (ON_ERROR_STOP=1 -> nonzero exit). + assert psql.returncode != 0, "query failed as expected" + assert ( + "canceling statement due to user request" in stderr + ), f"query was canceled\nstderr:\n{stderr}" diff --git a/src/bin/psql/pyt/test_030_pager.py b/src/bin/psql/pyt/test_030_pager.py new file mode 100644 index 0000000000..5cd96124d8 --- /dev/null +++ b/src/bin/psql/pyt/test_030_pager.py @@ -0,0 +1,119 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Set up "wc -l" as the pager (via PSQL_PAGER) so we can tell whether psql used +the pager, then drive an interactive psql session through a pty whose window is +sized 24x80 and check, for several queries/commands, that the pager was invoked +by matching the line-count number that "wc -l" prints. +""" + +import re +import subprocess + +import pytest + + +def _do_command(psql, send, pattern, annotation): + """Send *send*, wait for *pattern*, and assert it matched. + + *pattern* is a compiled regex. The match must be found in the captured + output and the query must not have timed out. + """ + out = psql.query_until(pattern, send) + assert pattern.search(out) and not psql.timed_out, annotation + + +def test_030_pager(pg, interactive_psql): + node = pg + + # Check that "wc -l" does what we expect, else forget it. + result = subprocess.run( + ["wc", "-l"], + input=b"foo bar\nbaz\n", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + wcstdout = result.stdout.decode().strip() + wcstderr = result.stderr.decode() + if not re.match(r"^ *2$", wcstdout) or wcstderr != "": + pytest.skip('"wc -l" is needed to run this test') + + # create a view we'll use below + node.safe_sql( + """create view public.view_030_pager as select +1 as a, +2 as b, +3 as c, +4 as d, +5 as e, +6 as f, +7 as g, +8 as h, +9 as i, +10 as j, +11 as k, +12 as l, +13 as m, +14 as n, +15 as o, +16 as p, +17 as q, +18 as r, +19 as s, +20 as t, +21 as u, +22 as v, +23 as w, +24 as x, +25 as y, +26 as z""", + ) + + # fire up an interactive psql session. We set up "wc -l" as the pager so + # we can tell whether psql used the pager, and size the pty's window to + # known values (24x80). + psql = interactive_psql( + node, + dbname="postgres", + extra_env={"PSQL_PAGER": "wc -l"}, + dimensions=(24, 80), + ) + + # Test invocation of the pager + # + # Note that interactive_psql starts psql with --no-align --tuples-only, + # and that the output string will include psql's prompts and command echo. + # So we have to test for patterns that can't match the command itself, + # and we can't assume the match will extend across a whole line (there + # might be a prompt ahead of it in the output). + + _do_command( + psql, + "SELECT 'test' AS t FROM generate_series(1,23);\n", + re.compile(r"test\r?$", re.MULTILINE), + "execute SELECT query that needs no pagination", + ) + + _do_command( + psql, + "SELECT 'test' AS t FROM generate_series(1,24);\n", + re.compile(r"24\r?$", re.MULTILINE), + "execute SELECT query that needs pagination", + ) + + _do_command( + psql, + "\\pset expanded\nSELECT generate_series(1,20) as g;\n", + re.compile(r"39\r?$", re.MULTILINE), + "execute SELECT query that needs pagination in expanded mode", + ) + + _do_command( + psql, + "\\pset tuples_only off\n\\d+ public.view_030_pager\n", + re.compile(r"55\r?$", re.MULTILINE), + "execute command with footer that needs pagination", + ) + + # send psql an explicit \q to shut it down, else pty won't close properly + psql.quit() diff --git a/src/bin/psql/t/001_basic.pl b/src/bin/psql/t/001_basic.pl deleted file mode 100644 index bbd330216a..0000000000 --- a/src/bin/psql/t/001_basic.pl +++ /dev/null @@ -1,543 +0,0 @@ - -# Copyright (c) 2021-2026, PostgreSQL Global Development Group - -use strict; -use warnings FATAL => 'all'; -use locale; - -use PostgreSQL::Test::Cluster; -use PostgreSQL::Test::Utils; -use Test::More; - -program_help_ok('psql'); -program_version_ok('psql'); -program_options_handling_ok('psql'); - -# Execute a psql command and check its output. -sub psql_like -{ - local $Test::Builder::Level = $Test::Builder::Level + 1; - - my ($node, $sql, $expected_stdout, $test_name) = @_; - - my ($ret, $stdout, $stderr) = $node->psql('postgres', $sql); - - is($ret, 0, "$test_name: exit code 0"); - is($stderr, '', "$test_name: no stderr"); - like($stdout, $expected_stdout, "$test_name: matches"); - - return; -} - -# Execute a psql command and check that it fails and check the stderr. -sub psql_fails_like -{ - local $Test::Builder::Level = $Test::Builder::Level + 1; - - my ($node, $sql, $expected_stderr, $test_name, $replication) = @_; - - # Use the context of a WAL sender, if requested by the caller. - $replication = '' unless defined($replication); - - my ($ret, $stdout, $stderr) = - $node->psql('postgres', $sql, replication => $replication); - - isnt($ret, 0, "$test_name: exit code not 0"); - like($stderr, $expected_stderr, "$test_name: matches"); - - return; -} - -# test --help=foo, analogous to program_help_ok() -foreach my $arg (qw(commands variables)) -{ - my ($stdout, $stderr); - my $result; - - $result = IPC::Run::run [ 'psql', "--help=$arg" ], - '>' => \$stdout, - '2>' => \$stderr; - ok($result, "psql --help=$arg exit code 0"); - isnt($stdout, '', "psql --help=$arg goes to stdout"); - is($stderr, '', "psql --help=$arg nothing to stderr"); -} - -my $node = PostgreSQL::Test::Cluster->new('main'); -$node->init(extra => [ '--locale=C', '--encoding=UTF8' ]); -$node->append_conf( - 'postgresql.conf', q{ -wal_level = 'logical' -max_replication_slots = 4 -max_wal_senders = 4 -}); -$node->start; - -psql_like($node, '\copyright', qr/Copyright/, '\copyright'); -psql_like($node, '\help', qr/ALTER/, '\help without arguments'); -psql_like($node, '\help SELECT', qr/SELECT/, '\help with argument'); - -# Test clean handling of unsupported replication command responses -psql_fails_like( - $node, - 'START_REPLICATION 0/0', - qr/unexpected PQresultStatus: 8$/, - 'handling of unexpected PQresultStatus', 'database'); - -# test \timing -psql_like( - $node, - '\timing on -SELECT 1', - qr/^1$ -^Time: \d+[.,]\d\d\d ms/m, - '\timing with successful query'); - -# test \timing with query that fails -{ - my ($ret, $stdout, $stderr) = - $node->psql('postgres', "\\timing on\nSELECT error"); - isnt($ret, 0, '\timing with query error: query failed'); - like( - $stdout, - qr/^Time: \d+[.,]\d\d\d ms/m, - '\timing with query error: timing output appears'); - unlike( - $stdout, - qr/^Time: 0[.,]000 ms/m, - '\timing with query error: timing was updated'); -} - -# test that ENCODING variable is set and that it is updated when -# client encoding is changed -psql_like( - $node, - '\echo :ENCODING -set client_encoding = LATIN1; -\echo :ENCODING', - qr/^UTF8$ -^LATIN1$/m, - 'ENCODING variable is set and updated'); - -# test LISTEN/NOTIFY -psql_like( - $node, - 'LISTEN foo; -NOTIFY foo;', - qr/^Asynchronous notification "foo" received from server process with PID \d+\.$/, - 'notification'); - -psql_like( - $node, - "LISTEN foo; -NOTIFY foo, 'bar';", - qr/^Asynchronous notification "foo" with payload "bar" received from server process with PID \d+\.$/, - 'notification with payload'); - -# test behavior and output on server crash -my ($ret, $out, $err) = $node->psql('postgres', - "SELECT 'before' AS running;\n" - . "SELECT pg_terminate_backend(pg_backend_pid());\n" - . "SELECT 'AFTER' AS not_running;\n"); - -is($ret, 2, 'server crash: psql exit code'); -like($out, qr/before/, 'server crash: output before crash'); -unlike($out, qr/AFTER/, 'server crash: no output after crash'); -like( - $err, - qr/psql::2: FATAL: terminating connection due to administrator command -psql::2: server closed the connection unexpectedly - This probably means the server terminated abnormally - before or while processing the request. -psql::2: error: connection to server was lost/, - 'server crash: error message'); - -# test \errverbose -# -# (This is not in the regular regression tests because the output -# contains the source code location and we don't want to have to -# update that every time it changes.) - -psql_like( - $node, - 'SELECT 1; -\errverbose', - qr/^1\nThere is no previous error\.$/, - '\errverbose with no previous error'); - -# There are three main ways to run a query that might affect -# \errverbose: The normal way, piecemeal retrieval using FETCH_COUNT, -# and using \gdesc. Test them all. - -like( - ( $node->psql( - 'postgres', - "SELECT error;\n\\errverbose", - on_error_stop => 0))[2], - qr/\A^psql::1: ERROR: .*$ -^LINE 1: SELECT error;$ -^ *^.*$ -^psql::2: error: ERROR: [0-9A-Z]{5}: .*$ -^LINE 1: SELECT error;$ -^ *^.*$ -^LOCATION: .*$/m, - '\errverbose after normal query with error'); - -like( - ( $node->psql( - 'postgres', - "\\set FETCH_COUNT 1\nSELECT error;\n\\errverbose", - on_error_stop => 0))[2], - qr/\A^psql::2: ERROR: .*$ -^LINE 1: SELECT error;$ -^ *^.*$ -^psql::3: error: ERROR: [0-9A-Z]{5}: .*$ -^LINE 1: SELECT error;$ -^ *^.*$ -^LOCATION: .*$/m, - '\errverbose after FETCH_COUNT query with error'); - -like( - ( $node->psql( - 'postgres', - "SELECT error\\gdesc\n\\errverbose", - on_error_stop => 0))[2], - qr/\A^psql::1: ERROR: .*$ -^LINE 1: SELECT error$ -^ *^.*$ -^psql::2: error: ERROR: [0-9A-Z]{5}: .*$ -^LINE 1: SELECT error$ -^ *^.*$ -^LOCATION: .*$/m, - '\errverbose after \gdesc with error'); - -# Check behavior when using multiple -c and -f switches. -# Note that we cannot test backend-side errors as tests are unstable in this -# case: IPC::Run can complain about a SIGPIPE if psql quits before reading a -# query result. -my $tempdir = PostgreSQL::Test::Utils::tempdir; -$node->safe_psql('postgres', "CREATE TABLE tab_psql_single (a int);"); - -# Tests with ON_ERROR_STOP. -$node->command_ok( - [ - 'psql', - '--no-psqlrc', - '--single-transaction', - '--set' => 'ON_ERROR_STOP=1', - '--command' => 'INSERT INTO tab_psql_single VALUES (1)', - '--command' => 'INSERT INTO tab_psql_single VALUES (2)', - ], - 'ON_ERROR_STOP, --single-transaction and multiple -c switches'); -my $row_count = - $node->safe_psql('postgres', 'SELECT count(*) FROM tab_psql_single'); -is($row_count, '2', - '--single-transaction commits transaction, ON_ERROR_STOP and multiple -c switches' -); - -$node->command_fails( - [ - 'psql', - '--no-psqlrc', - '--single-transaction', - '--set' => 'ON_ERROR_STOP=1', - '--command' => 'INSERT INTO tab_psql_single VALUES (3)', - '--command' => "\\copy tab_psql_single FROM '$tempdir/nonexistent'" - ], - 'ON_ERROR_STOP, --single-transaction and multiple -c switches, error'); -$row_count = - $node->safe_psql('postgres', 'SELECT count(*) FROM tab_psql_single'); -is($row_count, '2', - 'client-side error rolls back transaction, ON_ERROR_STOP and multiple -c switches' -); - -# Tests mixing files and commands. -my $copy_sql_file = "$tempdir/tab_copy.sql"; -my $insert_sql_file = "$tempdir/tab_insert.sql"; -append_to_file($copy_sql_file, - "\\copy tab_psql_single FROM '$tempdir/nonexistent';"); -append_to_file($insert_sql_file, 'INSERT INTO tab_psql_single VALUES (4);'); -$node->command_ok( - [ - 'psql', - '--no-psqlrc', - '--single-transaction', - '--set' => 'ON_ERROR_STOP=1', - '--file' => $insert_sql_file, - '--file' => $insert_sql_file - ], - 'ON_ERROR_STOP, --single-transaction and multiple -f switches'); -$row_count = - $node->safe_psql('postgres', 'SELECT count(*) FROM tab_psql_single'); -is($row_count, '4', - '--single-transaction commits transaction, ON_ERROR_STOP and multiple -f switches' -); - -$node->command_fails( - [ - 'psql', - '--no-psqlrc', - '--single-transaction', - '--set' => 'ON_ERROR_STOP=1', - '--file' => $insert_sql_file, - '--file' => $copy_sql_file - ], - 'ON_ERROR_STOP, --single-transaction and multiple -f switches, error'); -$row_count = - $node->safe_psql('postgres', 'SELECT count(*) FROM tab_psql_single'); -is($row_count, '4', - 'client-side error rolls back transaction, ON_ERROR_STOP and multiple -f switches' -); - -# Tests without ON_ERROR_STOP. -# The last switch fails on \copy. The command returns a failure and the -# transaction commits. -$node->command_fails( - [ - 'psql', - '--no-psqlrc', - '--single-transaction', - '--file' => $insert_sql_file, - '--file' => $insert_sql_file, - '--command' => "\\copy tab_psql_single FROM '$tempdir/nonexistent'" - ], - 'no ON_ERROR_STOP, --single-transaction and multiple -f/-c switches'); -$row_count = - $node->safe_psql('postgres', 'SELECT count(*) FROM tab_psql_single'); -is($row_count, '6', - 'client-side error commits transaction, no ON_ERROR_STOP and multiple -f/-c switches' -); - -# The last switch fails on \copy coming from an input file. The command -# returns a success and the transaction commits. -$node->command_ok( - [ - 'psql', - '--no-psqlrc', - '--single-transaction', - '--file' => $insert_sql_file, - '--file' => $insert_sql_file, - '--file' => $copy_sql_file - ], - 'no ON_ERROR_STOP, --single-transaction and multiple -f switches'); -$row_count = - $node->safe_psql('postgres', 'SELECT count(*) FROM tab_psql_single'); -is($row_count, '8', - 'client-side error commits transaction, no ON_ERROR_STOP and multiple -f switches' -); - -# The last switch makes the command return a success, and the contents of -# the transaction commit even if there is a failure in-between. -$node->command_ok( - [ - 'psql', - '--no-psqlrc', - '--single-transaction', - '--command' => 'INSERT INTO tab_psql_single VALUES (5)', - '--file' => $copy_sql_file, - '--command' => 'INSERT INTO tab_psql_single VALUES (6)' - ], - 'no ON_ERROR_STOP, --single-transaction and multiple -c switches'); -$row_count = - $node->safe_psql('postgres', 'SELECT count(*) FROM tab_psql_single'); -is($row_count, '10', - 'client-side error commits transaction, no ON_ERROR_STOP and multiple -c switches' -); - -# Test \copy from with DEFAULT option -$node->safe_psql( - 'postgres', - "CREATE TABLE copy_default ( - id integer PRIMARY KEY, - text_value text NOT NULL DEFAULT 'test', - ts_value timestamp without time zone NOT NULL DEFAULT '2022-07-05' - )" -); - -my $copy_default_sql_file = "$tempdir/copy_default.csv"; -append_to_file($copy_default_sql_file, "1,value,2022-07-04\n"); -append_to_file($copy_default_sql_file, "2,placeholder,2022-07-03\n"); -append_to_file($copy_default_sql_file, "3,placeholder,placeholder\n"); - -psql_like( - $node, - "\\copy copy_default from $copy_default_sql_file with (format 'csv', default 'placeholder'); - SELECT * FROM copy_default", - qr/1\|value\|2022-07-04 00:00:00 -2|test|2022-07-03 00:00:00 -3|test|2022-07-05 00:00:00/, - '\copy from with DEFAULT'); - -# Check \watch -# Note: the interval value is parsed with locale-aware strtod() -psql_like( - $node, sprintf('SELECT 1 \watch c=3 i=%g', 0.01), - qr/1\n1\n1/, '\watch with 3 iterations, interval of 0.01'); - -# Sub-millisecond wait works, equivalent to 0. -psql_like( - $node, sprintf('SELECT 1 \watch c=3 i=%g', 0.0001), - qr/1\n1\n1/, '\watch with 3 iterations, interval of 0.0001'); - -# Test zero interval -psql_like( - $node, '\set WATCH_INTERVAL 0 -SELECT 1 \watch c=3', - qr/1\n1\n1/, '\watch with 3 iterations, interval of 0'); - -# Check \watch minimum row count -psql_fails_like( - $node, - 'SELECT 3 \watch m=x', - qr/incorrect minimum row count/, - '\watch, invalid minimum row setting'); - -psql_fails_like( - $node, - 'SELECT 3 \watch m=1 min_rows=2', - qr/minimum row count specified more than once/, - '\watch, minimum rows is specified more than once'); - -psql_like( - $node, - sprintf( - q{with x as ( - select now()-backend_start AS howlong - from pg_stat_activity - where pid = pg_backend_pid() - ) select 123 from x where howlong < '2 seconds' \watch i=%g m=2}, 0.5), - qr/^123$/, - '\watch, 2 minimum rows'); - -# Check \watch errors -psql_fails_like( - $node, - 'SELECT 1 \watch -10', - qr/incorrect interval value "-10"/, - '\watch, negative interval'); -psql_fails_like( - $node, - 'SELECT 1 \watch 10ab', - qr/incorrect interval value "10ab"/, - '\watch, incorrect interval'); -psql_fails_like( - $node, - 'SELECT 1 \watch 10e400', - qr/incorrect interval value "10e400"/, - '\watch, out-of-range interval'); -psql_fails_like( - $node, - 'SELECT 1 \watch 1 1', - qr/interval value is specified more than once/, - '\watch, interval value is specified more than once'); -psql_fails_like( - $node, - 'SELECT 1 \watch c=1 c=1', - qr/iteration count is specified more than once/, - '\watch, iteration count is specified more than once'); - -# Check WATCH_INTERVAL -psql_like( - $node, - '\echo :WATCH_INTERVAL -\set WATCH_INTERVAL 10 -\echo :WATCH_INTERVAL -\unset WATCH_INTERVAL -\echo :WATCH_INTERVAL', - qr/^2$ -^10$ -^2$/m, - 'WATCH_INTERVAL variable is set and updated'); -psql_fails_like( - $node, - '\set WATCH_INTERVAL 1e500', - qr/is out of range/, - 'WATCH_INTERVAL variable is out of range'); -psql_like($node, '\echo :WATCH_INTERVAL', - qr/^2$/m, 'WATCH_INTERVAL variable was not altered'); - -# Test \g output piped into a program. -# The program is perl -pe '' to simply copy the input to the output. -my $g_file = "$tempdir/g_file_1.out"; -my $perlbin = $^X; -$perlbin =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os; -my $pipe_cmd = "$perlbin -pe '' >$g_file"; - -psql_like($node, "SELECT 'one' \\g | $pipe_cmd", qr//, "one command \\g"); -my $c1 = slurp_file($g_file); -like($c1, qr/one/); - -psql_like($node, "SELECT 'two' \\; SELECT 'three' \\g | $pipe_cmd", - qr//, "two commands \\g"); -my $c2 = slurp_file($g_file); -like($c2, qr/two.*three/s); - - -psql_like( - $node, - "\\set SHOW_ALL_RESULTS 0\nSELECT 'four' \\; SELECT 'five' \\g | $pipe_cmd", - qr//, - "two commands \\g with only last result"); -my $c3 = slurp_file($g_file); -like($c3, qr/five/); -unlike($c3, qr/four/); - -psql_like($node, "copy (values ('foo'),('bar')) to stdout \\g | $pipe_cmd", - qr//, "copy output passed to \\g pipe"); -my $c4 = slurp_file($g_file); -like($c4, qr/foo.*bar/s); - -# Test COPY within pipelines. These abort the connection from -# the frontend so they cannot be tested via SQL. -$node->safe_psql('postgres', 'CREATE TABLE psql_pipeline()'); -my $log_location = -s $node->logfile; -psql_fails_like( - $node, - qq{\\startpipeline -COPY psql_pipeline FROM STDIN; -SELECT 'val1'; -\\syncpipeline -\\endpipeline}, - qr/COPY in a pipeline is not supported, aborting connection/, - 'COPY FROM in pipeline: fails'); -$node->wait_for_log( - qr/FATAL: .*terminating connection because protocol synchronization was lost/, - $log_location); - -# Remove \syncpipeline here. -psql_fails_like( - $node, - qq{\\startpipeline -COPY psql_pipeline TO STDOUT; -SELECT 'val1'; -\\endpipeline}, - qr/COPY in a pipeline is not supported, aborting connection/, - 'COPY TO in pipeline: fails'); - -psql_fails_like( - $node, - qq{\\startpipeline -\\copy psql_pipeline from stdin; -SELECT 'val1'; -\\syncpipeline -\\endpipeline}, - qr/COPY in a pipeline is not supported, aborting connection/, - '\copy from in pipeline: fails'); - -# Sync attempt after a COPY TO/FROM. -psql_fails_like( - $node, - qq{\\startpipeline -\\copy psql_pipeline to stdout; -\\syncpipeline -\\endpipeline}, - qr/COPY in a pipeline is not supported, aborting connection/, - '\copy to in pipeline: fails'); - -psql_fails_like( - $node, - qq{\\restrict test -\\! should_fail}, - qr/backslash commands are restricted; only \\unrestrict is allowed/, - 'meta-command in restrict mode fails'); - -done_testing(); diff --git a/src/bin/psql/t/010_tab_completion.pl b/src/bin/psql/t/010_tab_completion.pl deleted file mode 100644 index 64e27ef87a..0000000000 --- a/src/bin/psql/t/010_tab_completion.pl +++ /dev/null @@ -1,468 +0,0 @@ - -# Copyright (c) 2021-2026, PostgreSQL Global Development Group - -use strict; -use warnings FATAL => 'all'; - -use PostgreSQL::Test::Cluster; -use PostgreSQL::Test::Utils; -use Test::More; -use Data::Dumper; - -# Do nothing unless Makefile has told us that the build is --with-readline. -if (!defined($ENV{with_readline}) || $ENV{with_readline} ne 'yes') -{ - plan skip_all => 'readline is not supported by this build'; -} - -# Also, skip if user has set environment variable to command that. -# This is mainly intended to allow working around some of the more broken -# versions of libedit --- some users might find them acceptable even if -# they won't pass these tests. -if (defined($ENV{SKIP_READLINE_TESTS})) -{ - plan skip_all => 'SKIP_READLINE_TESTS is set'; -} - -# If we don't have IO::Pty, forget it, because IPC::Run depends on that -# to support pty connections -eval { require IO::Pty; }; -if ($@) -{ - plan skip_all => 'IO::Pty is needed to run this test'; -} - -# start a new server -my $node = PostgreSQL::Test::Cluster->new('main'); -$node->init; -$node->start; - -# set up a few database objects -$node->safe_psql('postgres', - "CREATE TABLE tab1 (c1 int primary key constraint foo not null, c2 text);\n" - . "CREATE TABLE mytab123 (f1 int, f2 text);\n" - . "CREATE TABLE mytab246 (f1 int, f2 text);\n" - . "CREATE TABLE \"mixedName\" (f1 int, f2 text);\n" - . "CREATE TYPE enum1 AS ENUM ('foo', 'bar', 'baz', 'BLACK');\n" - . "CREATE PUBLICATION some_publication;\n" - . "CREATE TABLE fpo_test (id int4range, valid_at daterange, name text);\n" -); - -# In a VPATH build, we'll be started in the source directory, but we want -# to run in the build directory so that we can use relative paths to -# access the tab_comp_dir subdirectory; otherwise the output from filename -# completion tests is too variable. -if ($ENV{TESTDATADIR}) -{ - chdir $ENV{TESTDATADIR} - or die "could not chdir to \"$ENV{TESTDATADIR}\": $!"; -} - -# Create some junk files for filename completion testing. -mkdir "tab_comp_dir"; -my $FH; -open $FH, ">", "tab_comp_dir/somefile" - or die("could not create file \"tab_comp_dir/somefile\": $!"); -print $FH "some stuff\n"; -close $FH; -open $FH, ">", "tab_comp_dir/afile123" - or die("could not create file \"tab_comp_dir/afile123\": $!"); -print $FH "more stuff\n"; -close $FH; -open $FH, ">", "tab_comp_dir/afile456" - or die("could not create file \"tab_comp_dir/afile456\": $!"); -print $FH "other stuff\n"; -close $FH; - -# Arrange to capture, not discard, the interactive session's history output. -# Put it in the test log directory, so that buildfarm runs capture the result -# for possible debugging purposes. -my $historyfile = "${PostgreSQL::Test::Utils::log_path}/010_psql_history.txt"; - -# fire up an interactive psql session and configure it such that each query -# restarts the timer -my $h = $node->interactive_psql('postgres', history_file => $historyfile); -$h->set_query_timer_restart(); - -# Simple test case: type something and see if psql responds as expected -sub check_completion -{ - my ($send, $pattern, $annotation) = @_; - - # report test failures from caller location - local $Test::Builder::Level = $Test::Builder::Level + 1; - - # send the data to be sent and wait for its result - my $out = $h->query_until($pattern, $send); - my $okay = ($out =~ $pattern && !$h->{timeout}->is_expired); - ok($okay, $annotation); - # for debugging, log actual output if it didn't match - local $Data::Dumper::Terse = 1; - local $Data::Dumper::Useqq = 1; - diag 'Actual output was ' . Dumper($out) . "Did not match \"$pattern\"\n" - if !$okay; - return; -} - -# Clear query buffer to start over -# (won't work if we are inside a string literal!) -sub clear_query -{ - local $Test::Builder::Level = $Test::Builder::Level + 1; - - check_completion("\\r\n", qr/Query buffer reset.*postgres=# /s, - "\\r works"); - return; -} - -# Clear current line to start over -# (this will work in an incomplete string literal, but it's less desirable -# than clear_query because we lose evidence in the history file) -sub clear_line -{ - local $Test::Builder::Level = $Test::Builder::Level + 1; - - check_completion("\025\n", qr/postgres=# /, "control-U works"); - return; -} - -# check basic command completion: SEL produces SELECT -check_completion("SEL\t", qr/SELECT /, "complete SEL to SELECT"); - -clear_query(); - -# check case variation is honored -check_completion("sel\t", qr/select /, "complete sel to select"); - -# check basic table name completion -check_completion("* from t\t", qr/\* from tab1 /, "complete t to tab1"); - -clear_query(); - -# check table name completion with multiple alternatives -# note: readline might print a bell before the completion -check_completion( - "select * from my\t", - qr/select \* from my\a?tab/, - "complete my to mytab when there are multiple choices"); - -# some versions of readline/libedit require two tabs here, some only need one -check_completion( - "\t\t", - qr/mytab123 +mytab246/, - "offer multiple table choices"); - -check_completion("2\t", qr/246 /, - "finish completion of one of multiple table choices"); - -clear_query(); - -# check handling of quoted names -check_completion( - "select * from \"my\t", - qr/select \* from "my\a?tab/, - "complete \"my to \"mytab when there are multiple choices"); - -check_completion( - "\t\t", - qr/"mytab123" +"mytab246"/, - "offer multiple quoted table choices"); - -check_completion("2\t", qr/246" /, - "finish completion of one of multiple quoted table choices"); - -clear_query(); - -# check handling of mixed-case names -check_completion( - "select * from \"mi\t", - qr/"mixedName" /, - "complete a mixed-case name"); - -clear_query(); - -# check case folding -check_completion("select * from TAB\t", qr/tab1 /, "automatically fold case"); - -clear_query(); - -# check case-sensitive keyword replacement -# note: various versions of readline/libedit handle backspacing -# differently, so just check that the replacement comes out correctly -check_completion("\\DRD\t", qr/drds /, "complete \\DRD to \\drds"); - -clear_query(); - -# check completion of a schema-qualified name -check_completion("select * from pub\t", - qr/public\./, "complete schema when relevant"); - -check_completion("tab\t", qr/tab1 /, "complete schema-qualified name"); - -clear_query(); - -check_completion( - "select * from PUBLIC.t\t", - qr/public\.tab1 /, - "automatically fold case in schema-qualified name"); - -clear_query(); - -# check interpretation of referenced names -check_completion( - "alter table tab1 drop constraint t\t", - qr/tab1_pkey /, - "complete index name for referenced table"); - -clear_query(); - -check_completion( - "alter table TAB1 drop constraint t\t", - qr/tab1_pkey /, - "complete index name for referenced table, with downcasing"); - -clear_query(); - -check_completion( - "alter table public.\"tab1\" drop constraint t\t", - qr/tab1_pkey /, - "complete index name for referenced table, with schema and quoting"); - -clear_query(); - -# check variant where we're completing a qualified name from a refname -# (this one also checks successful completion in a multiline command) -check_completion( - "comment on constraint tab1_pkey \n on public.\t", - qr/public\.tab1/, - "complete qualified name from object reference"); - -clear_query(); - -# check filename completion -check_completion( - "\\lo_import tab_comp_dir/some\t", - qr|tab_comp_dir/somefile |, - "filename completion with one possibility"); - -clear_query(); - -# note: readline might print a bell before the completion -check_completion( - "\\lo_import tab_comp_dir/af\t", - qr|tab_comp_dir/af\a?ile|, - "filename completion with multiple possibilities"); - -# here we are inside a string literal 'afile*', so must use clear_line(). -clear_line(); - -# COPY requires quoting -check_completion( - "COPY foo FROM tab_comp_dir/some\t", - qr|'tab_comp_dir/somefile' |, - "quoted filename completion with one possibility"); - -clear_query(); - -check_completion( - "COPY foo FROM tab_comp_dir/af\t", - qr|'tab_comp_dir/afile|, - "quoted filename completion with multiple possibilities"); - -# some versions of readline/libedit require two tabs here, some only need one -# also, some will offer the whole path name and some just the file name -# the quotes might appear, too -check_completion( - "\t\t", - qr|afile123'? +'?(tab_comp_dir/)?afile456|, - "offer multiple file choices"); - -clear_line(); - -# check enum label completion -# some versions of readline/libedit require two tabs here, some only need one -# also, some versions will offer quotes, some will not -check_completion( - "ALTER TYPE enum1 RENAME VALUE 'ba\t\t", - qr|'?bar'? +'?baz'?|, - "offer multiple enum choices"); - -clear_line(); - -# enum labels are case sensitive, so this should complete BLACK immediately -check_completion( - "ALTER TYPE enum1 RENAME VALUE 'B\t", - qr|BLACK|, - "enum labels are case sensitive"); - -clear_line(); - -# check timezone name completion -check_completion("SET timezone TO am\t", - qr|'America/|, "offer partial timezone name"); - -check_completion("new_\t", qr|New_York|, "complete partial timezone name"); - -clear_line(); - -# check completion of a keyword offered in addition to object names; -# such a keyword should obey COMP_KEYWORD_CASE -foreach ( - [ 'lower', 'CO', 'column' ], - [ 'upper', 'co', 'COLUMN' ], - [ 'preserve-lower', 'co', 'column' ], - [ 'preserve-upper', 'CO', 'COLUMN' ],) -{ - my ($case, $in, $out) = @$_; - - check_completion( - "\\set COMP_KEYWORD_CASE $case\n", - qr/postgres=#/, - "set completion case to '$case'"); - check_completion("alter table tab1 rename $in\t\t\t", - qr|$out|, - "offer keyword $out for input $in, COMP_KEYWORD_CASE = $case"); - clear_query(); -} - -# alternate path where keyword comes from SchemaQuery -check_completion( - "DROP TYPE big\t", - qr/DROP TYPE bigint /, - "offer keyword from SchemaQuery"); - -clear_query(); - -# check create_command_generator -check_completion( - "CREATE TY\t", - qr/CREATE TYPE /, - "check create_command_generator"); - -clear_query(); - -# check words_after_create infrastructure -check_completion( - "CREATE TABLE mytab\t\t", - qr/mytab123 +mytab246/, - "check words_after_create"); - -clear_query(); - -# check VersionedQuery infrastructure -check_completion( - "DROP PUBLIC\t \t\t", - qr/DROP PUBLICATION\s+some_publication /, - "check VersionedQuery"); - -clear_query(); - -# hits ends_with() and logic for completing in multi-line queries -check_completion("analyze (\n\t\t", qr/VERBOSE/, - "check ANALYZE (VERBOSE ..."); - -clear_query(); - -# check completions for GUCs -check_completion( - "set interval\t\t", - qr/intervalstyle TO/, - "complete a GUC name"); -check_completion(" iso\t", qr/iso_8601 /, "complete a GUC enum value"); - -clear_query(); - -# same, for qualified GUC names -check_completion( - "DO \$\$begin end\$\$ LANGUAGE plpgsql;\n", - qr/postgres=# /, - "load plpgsql extension"); - -check_completion("set plpg\t", qr/plpg\a?sql\./, - "complete prefix of a GUC name"); -check_completion( - "var\t\t", - qr/variable_conflict TO/, - "complete a qualified GUC name"); -check_completion(" USE_C\t", - qr/use_column/, "complete a qualified GUC enum value"); - -clear_query(); - -# check completions for psql variables -check_completion("\\set VERB\t", qr/VERBOSITY /, - "complete a psql variable name"); -check_completion("def\t", qr/default /, "complete a psql variable value"); - -clear_query(); - -check_completion( - "\\echo :VERB\t", - qr/:VERBOSITY /, - "complete an interpolated psql variable name"); - -clear_query(); - -# check completion for psql variable test -check_completion( - "\\echo :{?VERB\t", - qr/:\{\?VERBOSITY} /, - "complete a psql variable test"); - -clear_query(); - -# check no-completions code path -check_completion("blarg \t\t", qr//, "check completion failure path"); - -clear_query(); - -# check COPY FROM with DEFAULT option -check_completion( - "COPY foo FROM stdin WITH ( DEF\t)", - qr/DEFAULT /, - "COPY FROM with DEFAULT completion"); - -clear_line(); - -# check tab completion for DELETE ... FOR PORTION OF -check_completion( - "DELETE FROM fpo_test F\t", - qr/FOR /, - "complete DELETE FROM
F to FOR"); - -check_completion("P\t", qr/PORTION /, "complete FOR P to PORTION"); - -check_completion("O\t", qr/OF /, "complete PORTION O to OF"); - -check_completion("v\t", qr/valid_at /, - "complete FOR PORTION OF offers column names"); - -check_completion("FR\t", qr/FROM /, - "complete FOR PORTION OF FR to FROM"); - -clear_query(); - -# check tab completion for UPDATE ... FOR PORTION OF -check_completion( - "UPDATE fpo_test F\t", - qr/FOR /, - "complete UPDATE
F to FOR"); - -check_completion("P\t", qr/PORTION /, "complete FOR P to PORTION"); - -check_completion("O\t", qr/OF /, "complete PORTION O to OF"); - -check_completion("v\t", qr/valid_at /, - "complete FOR PORTION OF offers column names"); - -check_completion("FR\t", qr/FROM /, - "complete FOR PORTION OF FR to FROM"); - -clear_query(); - -# send psql an explicit \q to shut it down, else pty won't close properly -$h->quit or die "psql returned $?"; - -# done -$node->stop; -done_testing(); diff --git a/src/bin/psql/t/020_cancel.pl b/src/bin/psql/t/020_cancel.pl deleted file mode 100644 index 08cfb95153..0000000000 --- a/src/bin/psql/t/020_cancel.pl +++ /dev/null @@ -1,51 +0,0 @@ - -# Copyright (c) 2021-2026, PostgreSQL Global Development Group - -use strict; -use warnings FATAL => 'all'; - -use PostgreSQL::Test::Cluster; -use PostgreSQL::Test::Utils; -use Test::More; -use Time::HiRes qw(usleep); - -# Test query canceling by sending SIGINT to a running psql -if ($windows_os) -{ - plan skip_all => 'sending SIGINT on Windows terminates the test itself'; -} - -my $node = PostgreSQL::Test::Cluster->new('main'); -$node->init; -$node->start; - -local %ENV = $node->_get_env(); - -my ($stdin, $stdout, $stderr); -my $h = IPC::Run::start( - [ - 'psql', '--no-psqlrc', '--set' => 'ON_ERROR_STOP=1', - ], - '<' => \$stdin, - '>' => \$stdout, - '2>' => \$stderr); - -# Send sleep command and wait until the server has registered it -$stdin = "select pg_sleep($PostgreSQL::Test::Utils::timeout_default);\n"; -pump $h while length $stdin; -$node->poll_query_until('postgres', - q{SELECT (SELECT count(*) FROM pg_stat_activity WHERE query ~ '^select pg_sleep') > 0;} -) or die "timed out"; - -# Send cancel request -$h->signal('INT'); - -my $result = finish $h; - -ok(!$result, 'query failed as expected'); -like( - $stderr, - qr/canceling statement due to user request/, - 'query was canceled'); - -done_testing(); diff --git a/src/bin/psql/t/030_pager.pl b/src/bin/psql/t/030_pager.pl deleted file mode 100644 index d3f964639d..0000000000 --- a/src/bin/psql/t/030_pager.pl +++ /dev/null @@ -1,138 +0,0 @@ - -# Copyright (c) 2021-2026, PostgreSQL Global Development Group - -use strict; -use warnings FATAL => 'all'; - -use PostgreSQL::Test::Cluster; -use PostgreSQL::Test::Utils; -use Test::More; -use Data::Dumper; - -# If we don't have IO::Pty, forget it, because IPC::Run depends on that -# to support pty connections -eval { require IO::Pty; }; -if ($@) -{ - plan skip_all => 'IO::Pty is needed to run this test'; -} - -# Check that "wc -l" does what we expect, else forget it -my $wcstdin = "foo bar\nbaz\n"; -my ($wcstdout, $wcstderr); -my $result = IPC::Run::run [ 'wc', '-l' ], - '<' => \$wcstdin, - '>' => \$wcstdout, - '2>' => \$wcstderr; -chomp $wcstdout; -if ($wcstdout !~ /^ *2$/ || $wcstderr ne '') -{ - note "wc stdout = '$wcstdout'\n"; - note "wc stderr = '$wcstderr'\n"; - plan skip_all => '"wc -l" is needed to run this test'; -} - -# We set up "wc -l" as the pager so we can tell whether psql used the pager -$ENV{PSQL_PAGER} = "wc -l"; - -# start a new server -my $node = PostgreSQL::Test::Cluster->new('main'); -$node->init; -$node->start; - -# create a view we'll use below -$node->safe_psql( - 'postgres', 'create view public.view_030_pager as select -1 as a, -2 as b, -3 as c, -4 as d, -5 as e, -6 as f, -7 as g, -8 as h, -9 as i, -10 as j, -11 as k, -12 as l, -13 as m, -14 as n, -15 as o, -16 as p, -17 as q, -18 as r, -19 as s, -20 as t, -21 as u, -22 as v, -23 as w, -24 as x, -25 as y, -26 as z'); - -# fire up an interactive psql session and configure it such that each query -# restarts the timer -my $h = $node->interactive_psql('postgres'); -$h->set_query_timer_restart(); - -# set the pty's window size to known values -# (requires undesirable chumminess with the innards of IPC::Run) -for my $pty (values %{ $h->{run}->{PTYS} }) -{ - $pty->set_winsize(24, 80); -} - -# Simple test case: type something and see if psql responds as expected -sub do_command -{ - my ($send, $pattern, $annotation) = @_; - - # report test failures from caller location - local $Test::Builder::Level = $Test::Builder::Level + 1; - - # send the data to be sent and wait for its result - my $out = $h->query_until($pattern, $send); - my $okay = ($out =~ $pattern && !$h->{timeout}->is_expired); - ok($okay, $annotation); - # for debugging, log actual output if it didn't match - local $Data::Dumper::Terse = 1; - local $Data::Dumper::Useqq = 1; - diag 'Actual output was ' . Dumper($out) . "Did not match \"$pattern\"\n" - if !$okay; - return; -} - -# Test invocation of the pager -# -# Note that interactive_psql starts psql with --no-align --tuples-only, -# and that the output string will include psql's prompts and command echo. -# So we have to test for patterns that can't match the command itself, -# and we can't assume the match will extend across a whole line (there -# might be a prompt ahead of it in the output). - -do_command( - "SELECT 'test' AS t FROM generate_series(1,23);\n", - qr/test\r?$/m, - "execute SELECT query that needs no pagination"); - -do_command( - "SELECT 'test' AS t FROM generate_series(1,24);\n", - qr/24\r?$/m, - "execute SELECT query that needs pagination"); - -do_command( - "\\pset expanded\nSELECT generate_series(1,20) as g;\n", - qr/39\r?$/m, - "execute SELECT query that needs pagination in expanded mode"); - -do_command( - "\\pset tuples_only off\n\\d+ public.view_030_pager\n", - qr/55\r?$/m, - "execute command with footer that needs pagination"); - -# send psql an explicit \q to shut it down, else pty won't close properly -$h->quit or die "psql returned $?"; - -# done -$node->stop; -done_testing(); From b3b202eaf84a5b5db65e4fd770a506f9e0d31c6b Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Sat, 20 Jun 2026 10:12:38 -0400 Subject: [PATCH 07/14] python tests: convert the pg_ctl TAP suite to pytest Replace the four src/bin/pg_ctl Perl TAP tests with their pytest equivalents and switch the meson test registration from 'tap' to 'pytest'. --- src/bin/pg_ctl/meson.build | 10 +- src/bin/pg_ctl/pyt/test_001_start_stop.py | 167 ++++++++++++++++++++++ src/bin/pg_ctl/pyt/test_002_status.py | 30 ++++ src/bin/pg_ctl/pyt/test_003_promote.py | 67 +++++++++ src/bin/pg_ctl/pyt/test_004_logrotate.py | 121 ++++++++++++++++ src/bin/pg_ctl/t/001_start_stop.pl | 125 ---------------- src/bin/pg_ctl/t/002_status.pl | 32 ----- src/bin/pg_ctl/t/003_promote.pl | 71 --------- src/bin/pg_ctl/t/004_logrotate.pl | 140 ------------------ 9 files changed, 390 insertions(+), 373 deletions(-) create mode 100644 src/bin/pg_ctl/pyt/test_001_start_stop.py create mode 100644 src/bin/pg_ctl/pyt/test_002_status.py create mode 100644 src/bin/pg_ctl/pyt/test_003_promote.py create mode 100644 src/bin/pg_ctl/pyt/test_004_logrotate.py delete mode 100644 src/bin/pg_ctl/t/001_start_stop.pl delete mode 100644 src/bin/pg_ctl/t/002_status.pl delete mode 100644 src/bin/pg_ctl/t/003_promote.pl delete mode 100644 src/bin/pg_ctl/t/004_logrotate.pl diff --git a/src/bin/pg_ctl/meson.build b/src/bin/pg_ctl/meson.build index 69fa7a2842..ba780ffca9 100644 --- a/src/bin/pg_ctl/meson.build +++ b/src/bin/pg_ctl/meson.build @@ -21,12 +21,12 @@ tests += { 'name': 'pg_ctl', 'sd': meson.current_source_dir(), 'bd': meson.current_build_dir(), - 'tap': { + 'pytest': { 'tests': [ - 't/001_start_stop.pl', - 't/002_status.pl', - 't/003_promote.pl', - 't/004_logrotate.pl', + 'pyt/test_001_start_stop.py', + 'pyt/test_002_status.py', + 'pyt/test_003_promote.py', + 'pyt/test_004_logrotate.py', ], }, } diff --git a/src/bin/pg_ctl/pyt/test_001_start_stop.py b/src/bin/pg_ctl/pyt/test_001_start_stop.py new file mode 100644 index 0000000000..26ac5e4380 --- /dev/null +++ b/src/bin/pg_ctl/pyt/test_001_start_stop.py @@ -0,0 +1,167 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Tests for pg_ctl start/stop/restart and resulting data-directory permissions.""" + +import os +import re +import stat +import time + +from pypg.util import WINDOWS_OS, short_tempdir + + +def _chmod_recursive(path, dir_mode, file_mode): + """Recursively chmod *path*, applying *dir_mode* to directories and + *file_mode* to regular files (symlinks are left untouched).""" + for root, dirs, files in os.walk(path): + os.chmod(root, dir_mode) + for d in dirs: + os.chmod(os.path.join(root, d), dir_mode) + for f in files: + full = os.path.join(root, f) + if not os.path.islink(full): + os.chmod(full, file_mode) + + +def _check_mode_recursive(path, dir_mode, file_mode): + """Check that every entry under *path* has the expected permissions. + + Returns True if all directories match *dir_mode* and all files match + *file_mode*. + """ + ok = True + for root, dirs, files in os.walk(path): + for name in [root] + [os.path.join(root, d) for d in dirs]: + actual = stat.S_IMODE(os.lstat(name).st_mode) + if actual != dir_mode: + print( + f"# Directory permissions check failed for {name}: " + f"expected {dir_mode:#o}, got {actual:#o}" + ) + ok = False + for f in files: + full = os.path.join(root, f) + if os.path.islink(full): + continue + actual = stat.S_IMODE(os.lstat(full).st_mode) + if actual != file_mode: + print( + f"# File permissions check failed for {full}: " + f"expected {file_mode:#o}, got {actual:#o}" + ) + ok = False + return ok + + +def test_start_stop(pg_bin, tmp_path): + """Exercise pg_ctl start/stop/restart and the resulting file permissions.""" + pg_bin.program_help_ok("pg_ctl") + pg_bin.program_version_ok("pg_ctl") + pg_bin.program_options_handling_ok("pg_ctl") + + pg_bin.command_exit_is( + ["pg_ctl", "start", "--pgdata", str(tmp_path / "nonexistent")], + 1, + "pg_ctl start with nonexistent directory", + ) + + data_dir = str(tmp_path / "data") + # Set up trust authentication by passing "-A trust" to initdb, which grants + # trust for the local unix socket connections this test uses. + pg_bin.command_ok( + [ + "pg_ctl", + "initdb", + "--pgdata", + data_dir, + "--options", + "--no-sync -A trust", + ], + "pg_ctl initdb", + ) + + # Use a short socket directory under /tmp to stay within the socket path + # length limit. + sockdir = short_tempdir() + try: + with open( + os.path.join(data_dir, "postgresql.conf"), "a", encoding="utf-8" + ) as conf: + conf.write("fsync = off\n") + conf.write("listen_addresses = ''\n") + # Forward slashes: backslashes in the postgresql.conf string value + # are mangled, so the socket directory would be wrong on Windows. + conf.write(f"unix_socket_directories = '{sockdir.replace(chr(92), '/')}'\n") + + log_file = str(tmp_path / "001_start_stop_server.log") + ctlcmd = ["pg_ctl", "start", "--pgdata", data_dir, "--log", log_file] + pg_bin.command_like( + ctlcmd, re.compile(r"done.*server started", re.S), "pg_ctl start" + ) + + # On Windows pg_ctl needs more than its ~2 second slop time to notice + # the already-running postmaster; without the wait the second start + # spuriously succeeds instead of failing. + if WINDOWS_OS: + time.sleep(3) + pg_bin.command_fails( + ["pg_ctl", "start", "--pgdata", data_dir], + "second pg_ctl start fails", + ) + pg_bin.command_ok( + ["pg_ctl", "stop", "--pgdata", data_dir], + "pg_ctl stop", + ) + pg_bin.command_fails( + ["pg_ctl", "stop", "--pgdata", data_dir], + "second pg_ctl stop fails", + ) + + # Log file for default permission test. + log_file = os.path.join(data_dir, "perm-test-600.log") + + pg_bin.command_ok( + ["pg_ctl", "restart", "--pgdata", data_dir, "--log", log_file], + "pg_ctl restart with server not running", + ) + + # Permissions on the log file should be default. Unix-style + # permissions are not supported on Windows, so skip the check there. + if not WINDOWS_OS: + assert os.path.isfile(log_file) + assert _check_mode_recursive(data_dir, 0o700, 0o600) + + # Log file for group access test. + log_file = os.path.join(data_dir, "perm-test-640.log") + + # Group access is not supported on Windows; skip that part there. + if not WINDOWS_OS: + pg_bin.command_ok( + ["pg_ctl", "stop", "--pgdata", data_dir], + "stop server before group permission test", + ) + + # Change the data dir mode so the log file will be created with + # group read privileges on the next start. + _chmod_recursive(data_dir, 0o750, 0o640) + + pg_bin.command_ok( + ["pg_ctl", "start", "--pgdata", data_dir, "--log", log_file], + "start server to check group permissions", + ) + + assert os.path.isfile(log_file) + assert _check_mode_recursive(data_dir, 0o750, 0o640) + + pg_bin.command_ok( + ["pg_ctl", "restart", "--pgdata", data_dir, "--log", log_file], + "pg_ctl restart with server running", + ) + + pg_bin.command_ok( + ["pg_ctl", "stop", "--pgdata", data_dir], + "stop server at end of test", + ) + finally: + # Make sure the server is down even if an assertion failed. + pg_bin.result(["pg_ctl", "stop", "--pgdata", data_dir, "-m", "immediate"]) diff --git a/src/bin/pg_ctl/pyt/test_002_status.py b/src/bin/pg_ctl/pyt/test_002_status.py new file mode 100644 index 0000000000..c0bd58a87f --- /dev/null +++ b/src/bin/pg_ctl/pyt/test_002_status.py @@ -0,0 +1,30 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Tests for pg_ctl status exit codes against missing, stopped, and running servers.""" + + +def test_status(pg_bin, create_pg, tmp_path): + """pg_ctl status reports the right exit code for missing/stopped/running.""" + pg_bin.command_exit_is( + ["pg_ctl", "status", "--pgdata", str(tmp_path / "nonexistent")], + 4, + "pg_ctl status with nonexistent directory", + ) + + node = create_pg("main", start=False) + + node.command_exit_is( + ["pg_ctl", "status", "--pgdata", node.data_dir], + 3, + "pg_ctl status with server not running", + ) + + node.start() + + node.command_exit_is( + ["pg_ctl", "status", "--pgdata", node.data_dir], + 0, + "pg_ctl status with server running", + ) + + node.stop() diff --git a/src/bin/pg_ctl/pyt/test_003_promote.py b/src/bin/pg_ctl/pyt/test_003_promote.py new file mode 100644 index 0000000000..f03a28ec9d --- /dev/null +++ b/src/bin/pg_ctl/pyt/test_003_promote.py @@ -0,0 +1,67 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Tests for pg_ctl promote, including failure cases and promoting a standby.""" + +import os + + +def test_003_promote(pg_bin, create_pg, tmp_path): + pg_bin.command_fails_like( + ["pg_ctl", "--pgdata", os.path.join(str(tmp_path), "nonexistent"), "promote"], + r"directory .* does not exist", + "pg_ctl promote with nonexistent directory", + ) + + node_primary = create_pg("primary", start=False, allows_streaming=True) + + node_primary.command_fails_like( + ["pg_ctl", "--pgdata", node_primary.data_dir, "promote"], + r"PID file .* does not exist", + "pg_ctl promote of not running instance fails", + ) + + node_primary.start() + + node_primary.command_fails_like( + ["pg_ctl", "--pgdata", node_primary.data_dir, "promote"], + r"not in standby mode", + "pg_ctl promote of primary instance fails", + ) + + node_standby = create_pg("standby", start=False) + node_primary.backup("my_backup") + node_standby.init_from_backup(node_primary, "my_backup", has_streaming=True) + node_standby.start() + + assert ( + node_standby.safe_sql("SELECT pg_is_in_recovery()") == "t" + ), "standby is in recovery" + + node_standby.command_ok( + ["pg_ctl", "--pgdata", node_standby.data_dir, "--no-wait", "promote"], + "pg_ctl --no-wait promote of standby runs", + ) + + assert node_standby.poll_query_until( + "SELECT NOT pg_is_in_recovery()" + ), "promoted standby is not in recovery" + + # same again with default wait option + node_standby = create_pg("standby2", start=False) + node_standby.init_from_backup(node_primary, "my_backup", has_streaming=True) + node_standby.start() + + assert ( + node_standby.safe_sql("SELECT pg_is_in_recovery()") == "t" + ), "standby is in recovery" + + node_standby.command_ok( + ["pg_ctl", "--pgdata", node_standby.data_dir, "promote"], + "pg_ctl promote of standby runs", + ) + + # no wait here + + assert ( + node_standby.safe_sql("SELECT pg_is_in_recovery()") == "f" + ), "promoted standby is not in recovery" diff --git a/src/bin/pg_ctl/pyt/test_004_logrotate.py b/src/bin/pg_ctl/pyt/test_004_logrotate.py new file mode 100644 index 0000000000..83808dd684 --- /dev/null +++ b/src/bin/pg_ctl/pyt/test_004_logrotate.py @@ -0,0 +1,121 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Tests for pg_ctl logrotate and the resulting log file handling.""" + +import os +import re +import time + +from pypg.util import TIMEOUT_DEFAULT, slurp_file + + +def fetch_file_name(logfiles, fmt): + """Extract the file name of a *fmt* from the contents of current_logfiles.""" + filename = None + for line in logfiles.split("\n"): + m = re.search(rf"{fmt} (.*)$", line) + if m: + filename = m.group(1) + return filename + + +def check_log_pattern(fmt, logfiles, pattern, node): + """Check for a pattern in the logs associated to one format.""" + lfname = fetch_file_name(logfiles, fmt) + + max_attempts = 10 * TIMEOUT_DEFAULT + + logcontents = "" + for _ in range(max_attempts): + logcontents = slurp_file(os.path.join(node.data_dir, lfname)) + if re.search(pattern, logcontents): + break + time.sleep(0.1) + + assert re.search(pattern, logcontents), f"found expected log file content for {fmt}" + + # While we're at it, test pg_current_logfile() function + assert ( + node.safe_sql(f"SELECT pg_current_logfile('{fmt}')") == lfname + ), f"pg_current_logfile() gives correct answer with {fmt}" + + +def test_004_logrotate(create_pg): + # Set up node with logging collector + node = create_pg("primary", start=False) + node.append_conf( + """ +logging_collector = on +log_destination = 'stderr, csvlog, jsonlog' +# these ensure stability of test results: +log_rotation_age = 0 +lc_messages = 'C' +""" + ) + + node.start() + + # Verify that log output gets to the file + + node.sql("SELECT 1/0") + + # might need to retry if logging collector process is slow... + max_attempts = 10 * TIMEOUT_DEFAULT + + current_logfiles = None + for _ in range(max_attempts): + try: + current_logfiles = slurp_file( + os.path.join(node.data_dir, "current_logfiles") + ) + break + except OSError: + time.sleep(0.1) + assert current_logfiles is not None + + print(f"# current_logfiles = {current_logfiles}") + + assert re.search( + r"^stderr log/postgresql-.*log\n" + r"csvlog log/postgresql-.*csv\n" + r"jsonlog log/postgresql-.*json$", + current_logfiles, + ), "current_logfiles is sane" + + check_log_pattern("stderr", current_logfiles, "division by zero", node) + check_log_pattern("csvlog", current_logfiles, "division by zero", node) + check_log_pattern("jsonlog", current_logfiles, "division by zero", node) + + # Sleep 2 seconds and ask for log rotation; this should result in + # output into a different log file name. + time.sleep(2) + node.command_ok(["pg_ctl", "logrotate", "-D", node.data_dir]) + + # pg_ctl logrotate doesn't wait for rotation request to be completed. + # Allow a bit of time for it to happen. + new_current_logfiles = None + for _ in range(max_attempts): + new_current_logfiles = slurp_file( + os.path.join(node.data_dir, "current_logfiles") + ) + if new_current_logfiles != current_logfiles: + break + time.sleep(0.1) + + print(f"# now current_logfiles = {new_current_logfiles}") + + assert re.search( + r"^stderr log/postgresql-.*log\n" + r"csvlog log/postgresql-.*csv\n" + r"jsonlog log/postgresql-.*json$", + new_current_logfiles, + ), "new current_logfiles is sane" + + # Verify that log output gets to this file, too + node.sql("fee fi fo fum") + + check_log_pattern("stderr", new_current_logfiles, "syntax error", node) + check_log_pattern("csvlog", new_current_logfiles, "syntax error", node) + check_log_pattern("jsonlog", new_current_logfiles, "syntax error", node) + + node.stop() diff --git a/src/bin/pg_ctl/t/001_start_stop.pl b/src/bin/pg_ctl/t/001_start_stop.pl deleted file mode 100644 index a189b379f5..0000000000 --- a/src/bin/pg_ctl/t/001_start_stop.pl +++ /dev/null @@ -1,125 +0,0 @@ - -# Copyright (c) 2021-2026, PostgreSQL Global Development Group - -use strict; -use warnings FATAL => 'all'; - -use PostgreSQL::Test::Cluster; -use PostgreSQL::Test::Utils; -use Test::More; - -my $tempdir = PostgreSQL::Test::Utils::tempdir; -my $tempdir_short = PostgreSQL::Test::Utils::tempdir_short; - -program_help_ok('pg_ctl'); -program_version_ok('pg_ctl'); -program_options_handling_ok('pg_ctl'); - -command_exit_is([ 'pg_ctl', 'start', '--pgdata' => "$tempdir/nonexistent" ], - 1, 'pg_ctl start with nonexistent directory'); - -command_ok( - [ - 'pg_ctl', 'initdb', - '--pgdata' => "$tempdir/data", - '--options' => '--no-sync' - ], - 'pg_ctl initdb'); -command_ok([ $ENV{PG_REGRESS}, '--config-auth', "$tempdir/data" ], - 'configure authentication'); -my $node_port = PostgreSQL::Test::Cluster::get_free_port(); -open my $conf, '>>', "$tempdir/data/postgresql.conf" or die $!; -print $conf "fsync = off\n"; -print $conf "port = $node_port\n"; -print $conf PostgreSQL::Test::Utils::slurp_file($ENV{TEMP_CONFIG}) - if defined $ENV{TEMP_CONFIG}; - -if ($use_unix_sockets) -{ - print $conf "listen_addresses = ''\n"; - $tempdir_short =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os; - print $conf "unix_socket_directories = '$tempdir_short'\n"; -} -else -{ - print $conf "listen_addresses = '127.0.0.1'\n"; -} -close $conf; -my $ctlcmd = [ - 'pg_ctl', 'start', - '--pgdata' => "$tempdir/data", - '--log' => "$PostgreSQL::Test::Utils::log_path/001_start_stop_server.log" -]; -command_like($ctlcmd, qr/done.*server started/s, 'pg_ctl start'); - -# sleep here is because Windows builds can't check postmaster.pid exactly, -# so they may mistake a pre-existing postmaster.pid for one created by the -# postmaster they start. Waiting more than the 2 seconds slop time allowed -# by wait_for_postmaster() prevents that mistake. -sleep 3 if ($windows_os); -command_fails([ 'pg_ctl', 'start', '--pgdata' => "$tempdir/data" ], - 'second pg_ctl start fails'); -command_ok([ 'pg_ctl', 'stop', '--pgdata' => "$tempdir/data" ], - 'pg_ctl stop'); -command_fails([ 'pg_ctl', 'stop', '--pgdata' => "$tempdir/data" ], - 'second pg_ctl stop fails'); - -# Log file for default permission test. The permissions won't be checked on -# Windows but we still want to do the restart test. -my $logFileName = "$tempdir/data/perm-test-600.log"; - -command_ok( - [ - 'pg_ctl', 'restart', - '--pgdata' => "$tempdir/data", - '--log' => $logFileName - ], - 'pg_ctl restart with server not running'); - -# Permissions on log file should be default -SKIP: -{ - skip "unix-style permissions not supported on Windows", 2 - if ($windows_os); - - ok(-f $logFileName); - ok(check_mode_recursive("$tempdir/data", 0700, 0600)); -} - -# Log file for group access test -$logFileName = "$tempdir/data/perm-test-640.log"; - -SKIP: -{ - skip "group access not supported on Windows", 3 - if ($windows_os || $Config::Config{osname} eq 'cygwin'); - - system_or_bail 'pg_ctl', 'stop', '--pgdata' => "$tempdir/data"; - - # Change the data dir mode so log file will be created with group read - # privileges on the next start - chmod_recursive("$tempdir/data", 0750, 0640); - - command_ok( - [ - 'pg_ctl', 'start', - '--pgdata' => "$tempdir/data", - '--log' => $logFileName - ], - 'start server to check group permissions'); - - ok(-f $logFileName); - ok(check_mode_recursive("$tempdir/data", 0750, 0640)); -} - -command_ok( - [ - 'pg_ctl', 'restart', - '--pgdata' => "$tempdir/data", - '--log' => $logFileName - ], - 'pg_ctl restart with server running'); - -system_or_bail 'pg_ctl', 'stop', '--pgdata' => "$tempdir/data"; - -done_testing(); diff --git a/src/bin/pg_ctl/t/002_status.pl b/src/bin/pg_ctl/t/002_status.pl deleted file mode 100644 index 1d7f9a1776..0000000000 --- a/src/bin/pg_ctl/t/002_status.pl +++ /dev/null @@ -1,32 +0,0 @@ - -# Copyright (c) 2021-2026, PostgreSQL Global Development Group - -use strict; -use warnings FATAL => 'all'; - -use PostgreSQL::Test::Cluster; -use PostgreSQL::Test::Utils; -use Test::More; - -my $tempdir = PostgreSQL::Test::Utils::tempdir; - -command_exit_is([ 'pg_ctl', 'status', '--pgdata' => "$tempdir/nonexistent" ], - 4, 'pg_ctl status with nonexistent directory'); - -my $node = PostgreSQL::Test::Cluster->new('main'); -$node->init; - -command_exit_is([ 'pg_ctl', 'status', '--pgdata' => $node->data_dir ], - 3, 'pg_ctl status with server not running'); - -system_or_bail( - 'pg_ctl', - '--log' => "$tempdir/logfile", - '--pgdata' => $node->data_dir, - '--wait', 'start'); -command_exit_is([ 'pg_ctl', 'status', '--pgdata' => $node->data_dir ], - 0, 'pg_ctl status with server running'); - -system_or_bail 'pg_ctl', 'stop', '--pgdata' => $node->data_dir; - -done_testing(); diff --git a/src/bin/pg_ctl/t/003_promote.pl b/src/bin/pg_ctl/t/003_promote.pl deleted file mode 100644 index bc3d1163ae..0000000000 --- a/src/bin/pg_ctl/t/003_promote.pl +++ /dev/null @@ -1,71 +0,0 @@ - -# Copyright (c) 2021-2026, PostgreSQL Global Development Group - -use strict; -use warnings FATAL => 'all'; - -use PostgreSQL::Test::Cluster; -use PostgreSQL::Test::Utils; -use Test::More; - -my $tempdir = PostgreSQL::Test::Utils::tempdir; - -command_fails_like( - [ 'pg_ctl', '--pgdata' => "$tempdir/nonexistent", 'promote' ], - qr/directory .* does not exist/, - 'pg_ctl promote with nonexistent directory'); - -my $node_primary = PostgreSQL::Test::Cluster->new('primary'); -$node_primary->init(allows_streaming => 1); - -command_fails_like( - [ 'pg_ctl', '--pgdata' => $node_primary->data_dir, 'promote' ], - qr/PID file .* does not exist/, - 'pg_ctl promote of not running instance fails'); - -$node_primary->start; - -command_fails_like( - [ 'pg_ctl', '--pgdata' => $node_primary->data_dir, 'promote' ], - qr/not in standby mode/, - 'pg_ctl promote of primary instance fails'); - -my $node_standby = PostgreSQL::Test::Cluster->new('standby'); -$node_primary->backup('my_backup'); -$node_standby->init_from_backup($node_primary, 'my_backup', - has_streaming => 1); -$node_standby->start; - -is($node_standby->safe_psql('postgres', 'SELECT pg_is_in_recovery()'), - 't', 'standby is in recovery'); - -command_ok( - [ - 'pg_ctl', - '--pgdata' => $node_standby->data_dir, - '--no-wait', 'promote' - ], - 'pg_ctl --no-wait promote of standby runs'); - -ok( $node_standby->poll_query_until( - 'postgres', 'SELECT NOT pg_is_in_recovery()'), - 'promoted standby is not in recovery'); - -# same again with default wait option -$node_standby = PostgreSQL::Test::Cluster->new('standby2'); -$node_standby->init_from_backup($node_primary, 'my_backup', - has_streaming => 1); -$node_standby->start; - -is($node_standby->safe_psql('postgres', 'SELECT pg_is_in_recovery()'), - 't', 'standby is in recovery'); - -command_ok([ 'pg_ctl', '--pgdata' => $node_standby->data_dir, 'promote' ], - 'pg_ctl promote of standby runs'); - -# no wait here - -is($node_standby->safe_psql('postgres', 'SELECT pg_is_in_recovery()'), - 'f', 'promoted standby is not in recovery'); - -done_testing(); diff --git a/src/bin/pg_ctl/t/004_logrotate.pl b/src/bin/pg_ctl/t/004_logrotate.pl deleted file mode 100644 index 7b19f86467..0000000000 --- a/src/bin/pg_ctl/t/004_logrotate.pl +++ /dev/null @@ -1,140 +0,0 @@ - -# Copyright (c) 2021-2026, PostgreSQL Global Development Group - -use strict; -use warnings FATAL => 'all'; - -use PostgreSQL::Test::Cluster; -use PostgreSQL::Test::Utils; -use Test::More; -use Time::HiRes qw(usleep); - -# Extract the file name of a $format from the contents of -# current_logfiles. -sub fetch_file_name -{ - my $logfiles = shift; - my $format = shift; - my @lines = split(/\n/, $logfiles); - my $filename = undef; - foreach my $line (@lines) - { - if ($line =~ /$format (.*)$/gm) - { - $filename = $1; - } - } - - return $filename; -} - -# Check for a pattern in the logs associated to one format. -sub check_log_pattern -{ - local $Test::Builder::Level = $Test::Builder::Level + 1; - - my $format = shift; - my $logfiles = shift; - my $pattern = shift; - my $node = shift; - my $lfname = fetch_file_name($logfiles, $format); - - my $max_attempts = 10 * $PostgreSQL::Test::Utils::timeout_default; - - my $logcontents; - for (my $attempts = 0; $attempts < $max_attempts; $attempts++) - { - $logcontents = slurp_file($node->data_dir . '/' . $lfname); - last if $logcontents =~ m/$pattern/; - usleep(100_000); - } - - like($logcontents, qr/$pattern/, - "found expected log file content for $format"); - - # While we're at it, test pg_current_logfile() function - is( $node->safe_psql('postgres', "SELECT pg_current_logfile('$format')"), - $lfname, - "pg_current_logfile() gives correct answer with $format"); - return; -} - -# Set up node with logging collector -my $node = PostgreSQL::Test::Cluster->new('primary'); -$node->init(); -$node->append_conf( - 'postgresql.conf', qq( -logging_collector = on -log_destination = 'stderr, csvlog, jsonlog' -# these ensure stability of test results: -log_rotation_age = 0 -lc_messages = 'C' -)); - -$node->start(); - -# Verify that log output gets to the file - -$node->psql('postgres', 'SELECT 1/0'); - -# might need to retry if logging collector process is slow... -my $max_attempts = 10 * $PostgreSQL::Test::Utils::timeout_default; - -my $current_logfiles; -for (my $attempts = 0; $attempts < $max_attempts; $attempts++) -{ - eval { - $current_logfiles = slurp_file($node->data_dir . '/current_logfiles'); - }; - last unless $@; - usleep(100_000); -} -die $@ if $@; - -note "current_logfiles = $current_logfiles"; - -like( - $current_logfiles, - qr|^stderr log/postgresql-.*log -csvlog log/postgresql-.*csv -jsonlog log/postgresql-.*json$|, - 'current_logfiles is sane'); - -check_log_pattern('stderr', $current_logfiles, 'division by zero', $node); -check_log_pattern('csvlog', $current_logfiles, 'division by zero', $node); -check_log_pattern('jsonlog', $current_logfiles, 'division by zero', $node); - -# Sleep 2 seconds and ask for log rotation; this should result in -# output into a different log file name. -sleep(2); -$node->logrotate(); - -# pg_ctl logrotate doesn't wait for rotation request to be completed. -# Allow a bit of time for it to happen. -my $new_current_logfiles; -for (my $attempts = 0; $attempts < $max_attempts; $attempts++) -{ - $new_current_logfiles = slurp_file($node->data_dir . '/current_logfiles'); - last if $new_current_logfiles ne $current_logfiles; - usleep(100_000); -} - -note "now current_logfiles = $new_current_logfiles"; - -like( - $new_current_logfiles, - qr|^stderr log/postgresql-.*log -csvlog log/postgresql-.*csv -jsonlog log/postgresql-.*json$|, - 'new current_logfiles is sane'); - -# Verify that log output gets to this file, too -$node->psql('postgres', 'fee fi fo fum'); - -check_log_pattern('stderr', $new_current_logfiles, 'syntax error', $node); -check_log_pattern('csvlog', $new_current_logfiles, 'syntax error', $node); -check_log_pattern('jsonlog', $new_current_logfiles, 'syntax error', $node); - -$node->stop(); - -done_testing(); From 94a5a1152634b04a3370b0c16503cd6da69edd22 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Sat, 20 Jun 2026 10:13:20 -0400 Subject: [PATCH 08/14] python tests: convert the libpq TAP suite to pytest Replace the six src/interfaces/libpq Perl TAP tests with their pytest equivalents and switch the meson test registration from 'tap' to 'pytest', preserving the existing env and deps. --- src/interfaces/libpq/meson.build | 14 +- src/interfaces/libpq/pyt/test_001_uri.py | 299 +++++++ src/interfaces/libpq/pyt/test_002_api.py | 19 + .../pyt/test_003_load_balance_host_list.py | 127 +++ .../libpq/pyt/test_004_load_balance_dns.py | 188 ++++ .../pyt/test_005_negotiate_encryption.py | 791 +++++++++++++++++ src/interfaces/libpq/pyt/test_006_service.py | 335 +++++++ src/interfaces/libpq/t/001_uri.pl | 288 ------ src/interfaces/libpq/t/002_api.pl | 22 - .../libpq/t/003_load_balance_host_list.pl | 94 -- .../libpq/t/004_load_balance_dns.pl | 144 --- .../libpq/t/005_negotiate_encryption.pl | 831 ------------------ src/interfaces/libpq/t/006_service.pl | 246 ------ 13 files changed, 1766 insertions(+), 1632 deletions(-) create mode 100644 src/interfaces/libpq/pyt/test_001_uri.py create mode 100644 src/interfaces/libpq/pyt/test_002_api.py create mode 100644 src/interfaces/libpq/pyt/test_003_load_balance_host_list.py create mode 100644 src/interfaces/libpq/pyt/test_004_load_balance_dns.py create mode 100644 src/interfaces/libpq/pyt/test_005_negotiate_encryption.py create mode 100644 src/interfaces/libpq/pyt/test_006_service.py delete mode 100644 src/interfaces/libpq/t/001_uri.pl delete mode 100644 src/interfaces/libpq/t/002_api.pl delete mode 100644 src/interfaces/libpq/t/003_load_balance_host_list.pl delete mode 100644 src/interfaces/libpq/t/004_load_balance_dns.pl delete mode 100644 src/interfaces/libpq/t/005_negotiate_encryption.pl delete mode 100644 src/interfaces/libpq/t/006_service.pl diff --git a/src/interfaces/libpq/meson.build b/src/interfaces/libpq/meson.build index b0ae72167a..57c64e9686 100644 --- a/src/interfaces/libpq/meson.build +++ b/src/interfaces/libpq/meson.build @@ -153,14 +153,14 @@ tests += { 'name': 'libpq', 'sd': meson.current_source_dir(), 'bd': meson.current_build_dir(), - 'tap': { + 'pytest': { 'tests': [ - 't/001_uri.pl', - 't/002_api.pl', - 't/003_load_balance_host_list.pl', - 't/004_load_balance_dns.pl', - 't/005_negotiate_encryption.pl', - 't/006_service.pl', + 'pyt/test_001_uri.py', + 'pyt/test_002_api.py', + 'pyt/test_003_load_balance_host_list.py', + 'pyt/test_004_load_balance_dns.py', + 'pyt/test_005_negotiate_encryption.py', + 'pyt/test_006_service.py', ], 'env': { 'with_ssl': ssl_library, diff --git a/src/interfaces/libpq/pyt/test_001_uri.py b/src/interfaces/libpq/pyt/test_001_uri.py new file mode 100644 index 0000000000..a9ec1ae6d4 --- /dev/null +++ b/src/interfaces/libpq/pyt/test_001_uri.py @@ -0,0 +1,299 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Table-driven test of libpq URI / connection-string parsing. Each case feeds an +input string to the build-dir ``libpq_uri_regress`` program (a single argv +argument) and checks the normalized stdout, the stderr, and the exit status. + +``libpq_uri_regress`` lives in builddir/src/interfaces/libpq/test/ and is found +via PATH (PgBin falls back to the bare name when it is not in bindir). No +server is needed. +""" + +from typing import Dict, List, Tuple, Union + +import pytest + +# List of URI tests. For each test the first element is the input string, the +# second the expected stdout and the third the expected stderr. Optionally, a +# fourth element is a dict of key/value pairs which override environment +# variables for the duration of the test. +TESTS: List[Union[Tuple[str, str, str], Tuple[str, str, str, Dict[str, str]]]] = [ + ( + r"postgresql://uri-user:secret@host:12345/db", + r"user='uri-user' password='secret' dbname='db' host='host' port='12345' (inet)", + r"", + ), + ( + r"postgresql://uri-user@host:12345/db", + r"user='uri-user' dbname='db' host='host' port='12345' (inet)", + r"", + ), + ( + r"postgresql://uri-user@host/db", + r"user='uri-user' dbname='db' host='host' (inet)", + r"", + ), + ( + r"postgresql://host:12345/db", + r"dbname='db' host='host' port='12345' (inet)", + r"", + ), + (r"postgresql://host/db", r"dbname='db' host='host' (inet)", r""), + ( + r"postgresql://uri-user@host:12345/", + r"user='uri-user' host='host' port='12345' (inet)", + r"", + ), + ( + r"postgresql://uri-user@host/", + r"user='uri-user' host='host' (inet)", + r"", + ), + (r"postgresql://uri-user@", r"user='uri-user' (local)", r""), + (r"postgresql://host:12345/", r"host='host' port='12345' (inet)", r""), + (r"postgresql://host:12345", r"host='host' port='12345' (inet)", r""), + (r"postgresql://host/db", r"dbname='db' host='host' (inet)", r""), + (r"postgresql://host/", r"host='host' (inet)", r""), + (r"postgresql://host", r"host='host' (inet)", r""), + (r"postgresql://", r"(local)", r""), + ( + r"postgresql://?hostaddr=127.0.0.1", + r"hostaddr='127.0.0.1' (inet)", + r"", + ), + ( + r"postgresql://example.com?hostaddr=63.1.2.4", + r"host='example.com' hostaddr='63.1.2.4' (inet)", + r"", + ), + (r"postgresql://%68ost/", r"host='host' (inet)", r""), + ( + r"postgresql://host/db?user=uri-user", + r"user='uri-user' dbname='db' host='host' (inet)", + r"", + ), + ( + r"postgresql://host/db?user=uri-user&port=12345", + r"user='uri-user' dbname='db' host='host' port='12345' (inet)", + r"", + ), + ( + r"postgresql://host/db?u%73er=someotheruser&port=12345", + r"user='someotheruser' dbname='db' host='host' port='12345' (inet)", + r"", + ), + ( + r"postgresql://host/db?u%7aer=someotheruser&port=12345", + r"", + r'libpq_uri_regress: invalid URI query parameter: "uzer"', + ), + ( + r"postgresql://host:12345?user=uri-user", + r"user='uri-user' host='host' port='12345' (inet)", + r"", + ), + ( + r"postgresql://host?user=uri-user", + r"user='uri-user' host='host' (inet)", + r"", + ), + ( + # Leading and trailing spaces, works. + r"postgresql://host? user = uri-user & port = 12345 ", + r"user='uri-user' host='host' port='12345' (inet)", + r"", + ), + ( + # Trailing data in parameter. + r"postgresql://host? user user = uri & port = 12345 12 ", + r"", + r'libpq_uri_regress: unexpected spaces found in " user user ", use percent-encoded spaces (%20) instead', + ), + ( + # Trailing data in value. + r"postgresql://host? user = uri-user & port = 12345 12 ", + r"", + r'libpq_uri_regress: unexpected spaces found in " 12345 12 ", use percent-encoded spaces (%20) instead', + ), + (r"postgresql://host?", r"host='host' (inet)", r""), + ( + r"postgresql://[::1]:12345/db", + r"dbname='db' host='::1' port='12345' (inet)", + r"", + ), + (r"postgresql://[::1]/db", r"dbname='db' host='::1' (inet)", r""), + ( + r"postgresql://[2001:db8::1234]/", + r"host='2001:db8::1234' (inet)", + r"", + ), + ( + r"postgresql://[200z:db8::1234]/", + r"host='200z:db8::1234' (inet)", + r"", + ), + (r"postgresql://[::1]", r"host='::1' (inet)", r""), + (r"postgres://", r"(local)", r""), + (r"postgres:///", r"(local)", r""), + (r"postgres:///db", r"dbname='db' (local)", r""), + ( + r"postgres://uri-user@/db", + r"user='uri-user' dbname='db' (local)", + r"", + ), + ( + r"postgres://?host=/path/to/socket/dir", + r"host='/path/to/socket/dir' (local)", + r"", + ), + ( + r"postgresql://host?uzer=", + r"", + r'libpq_uri_regress: invalid URI query parameter: "uzer"', + ), + ( + r"postgre://", + r"", + r'libpq_uri_regress: missing "=" after "postgre://" in connection info string', + ), + ( + r"postgres://[::1", + r"", + r'libpq_uri_regress: end of string reached when looking for matching "]" in IPv6 host address in URI: "postgres://[::1"', + ), + ( + r"postgres://[]", + r"", + r'libpq_uri_regress: IPv6 host address may not be empty in URI: "postgres://[]"', + ), + ( + r"postgres://[::1]z", + r"", + r'libpq_uri_regress: unexpected character "z" at position 17 in URI (expected ":" or "/"): "postgres://[::1]z"', + ), + ( + r"postgresql://host?zzz", + r"", + r'libpq_uri_regress: missing key/value separator "=" in URI query parameter: "zzz"', + ), + ( + r"postgresql://host?value1&value2", + r"", + r'libpq_uri_regress: missing key/value separator "=" in URI query parameter: "value1"', + ), + ( + r"postgresql://host?key=key=value", + r"", + r'libpq_uri_regress: extra key/value separator "=" in URI query parameter: "key"', + ), + ( + r"postgres://host?dbname=%XXfoo", + r"", + r'libpq_uri_regress: invalid percent-encoded token: "%XXfoo"', + ), + ( + r"postgresql://a%00b", + r"", + r'libpq_uri_regress: forbidden value %00 in percent-encoded value: "a%00b"', + ), + ( + r"postgresql://%zz", + r"", + r'libpq_uri_regress: invalid percent-encoded token: "%zz"', + ), + ( + r"postgresql://%1", + r"", + r'libpq_uri_regress: invalid percent-encoded token: "%1"', + ), + ( + r"postgresql://%", + r"", + r'libpq_uri_regress: invalid percent-encoded token: "%"', + ), + (r"postgres://@host", r"host='host' (inet)", r""), + (r"postgres://host:/", r"host='host' (inet)", r""), + (r"postgres://:12345/", r"port='12345' (local)", r""), + ( + r"postgres://otheruser@?host=/no/such/directory", + r"user='otheruser' host='/no/such/directory' (local)", + r"", + ), + ( + r"postgres://otheruser@/?host=/no/such/directory", + r"user='otheruser' host='/no/such/directory' (local)", + r"", + ), + ( + r"postgres://otheruser@:12345?host=/no/such/socket/path", + r"user='otheruser' host='/no/such/socket/path' port='12345' (local)", + r"", + ), + ( + r"postgres://otheruser@:12345/db?host=/path/to/socket", + r"user='otheruser' dbname='db' host='/path/to/socket' port='12345' (local)", + r"", + ), + ( + r"postgres://:12345/db?host=/path/to/socket", + r"dbname='db' host='/path/to/socket' port='12345' (local)", + r"", + ), + ( + r"postgres://:12345?host=/path/to/socket", + r"host='/path/to/socket' port='12345' (local)", + r"", + ), + ( + r"postgres://%2Fvar%2Flib%2Fpostgresql/dbname", + r"dbname='dbname' host='/var/lib/postgresql' (local)", + r"", + ), + # Usually the default sslmode is 'prefer' (for libraries with SSL) or + # 'disable' (for those without). This default changes to 'verify-full' if + # the system CA store is in use. + ( + r"postgresql://host?sslmode=disable", + r"host='host' sslmode='disable' (inet)", + r"", + {"PGSSLROOTCERT": "system"}, + ), + ( + r"postgresql://host?sslmode=prefer", + r"host='host' sslmode='prefer' (inet)", + r"", + {"PGSSLROOTCERT": "system"}, + ), + ( + r"postgresql://host?sslmode=verify-full", + r"host='host' (inet)", + r"", + {"PGSSLROOTCERT": "system"}, + ), +] + + +@pytest.mark.parametrize( + "case", + TESTS, + ids=[t[0] for t in TESTS], +) +def test_001_uri(pg_bin, case): + uri = case[0] + expect_stdout = case[1] + expect_stderr = case[2] + envvars = case[3] if len(case) > 3 else {} + + res = pg_bin.result(["libpq_uri_regress", uri], extra_env=envvars) + + # IPC::Run chomps trailing newlines off the captured streams. + got_stdout = res.stdout.rstrip("\n") + got_stderr = res.stderr.rstrip("\n") + + # The expected exit status is success exactly when the expected stderr is + # empty. + expect_success = expect_stderr == "" + + assert got_stdout == expect_stdout, f"stdout for {uri}" + assert got_stderr == expect_stderr, f"stderr for {uri}" + assert (res.returncode == 0) == expect_success, f"exit status for {uri}" diff --git a/src/interfaces/libpq/pyt/test_002_api.py b/src/interfaces/libpq/pyt/test_002_api.py new file mode 100644 index 0000000000..1ad28a11bd --- /dev/null +++ b/src/interfaces/libpq/pyt/test_002_api.py @@ -0,0 +1,19 @@ +# Copyright (c) 2022-2026, PostgreSQL Global Development Group + +"""Test PQsslAttribute(NULL, "library") via the libpq_testclient helper.""" + +import os + + +def test_002_api(pg_bin): + # Test PQsslAttribute(NULL, "library") + res = pg_bin.result(["libpq_testclient", "--ssl"]) + + if os.environ.get("with_ssl") == "openssl": + assert ( + res.stdout.strip() == "OpenSSL" + ), 'PQsslAttribute(NULL, "library") returns "OpenSSL"' + else: + assert ( + res.stderr.strip() == "SSL is not enabled" + ), 'PQsslAttribute(NULL, "library") returns NULL' diff --git a/src/interfaces/libpq/pyt/test_003_load_balance_host_list.py b/src/interfaces/libpq/pyt/test_003_load_balance_host_list.py new file mode 100644 index 0000000000..7a976f6f27 --- /dev/null +++ b/src/interfaces/libpq/pyt/test_003_load_balance_host_list.py @@ -0,0 +1,127 @@ +# Copyright (c) 2023-2026, PostgreSQL Global Development Group + +"""Test load balancing across the list of different hosts in the host +parameter of the connection string. + +This framework uses the in-process libpq Session, so each node's "host" is its +own socket directory (or the loopback address on TCP) and the nodes are still +distinguished by listening on different ports. We observe which node answered +by checking the executed statement in each node's server log +(log_statement = all) and counting log occurrences. +""" + +import re + +import pytest + +from libpq import Session +from libpq.errors import PqConnectionError + + +def connect_with(node, connstr, sql=None): + """Open a libpq Session with *connstr*, optionally running *sql*. + + Returns the Session on success; the caller owns it and must close it. + Raises PqConnectionError on connection failure (mirrors connect_fails). + """ + sess = Session(connstr=connstr, libdir=node.libdir) + if sql is not None: + sess.query_safe(sql) + return sess + + +def count_statements(node, sql): + """Count occurrences of "statement: " in *node*'s server log.""" + pattern = "statement: " + re.escape(sql) + return len(re.findall(pattern, node.log_content())) + + +def test_003_load_balance_host_list(create_pg): + # Cluster setup which is shared for testing both load balancing methods. + # Each node listens on its own socket directory and port; logging every + # statement lets us tell which node served a given connection. + node1 = create_pg("node1", start=False) + node2 = create_pg("node2", start=False) + node3 = create_pg("node3", start=False) + + for node in (node1, node2, node3): + node.append_conf("log_statement = all\n") + node.start() + + # Build the shared host/port lists. In this unix-socket-only framework + # each node's host is its socket directory. + hostlist = ",".join(n.host for n in (node1, node2, node3)) + portlist = ",".join(str(n.port) for n in (node1, node2, node3)) + + # load_balance_hosts doesn't accept unknown values. + with pytest.raises(PqConnectionError) as excinfo: + connect_with( + node1, + f"host={hostlist} port={portlist} dbname=postgres load_balance_hosts=doesnotexist", + ) + assert re.search( + r'invalid load_balance_hosts value: "doesnotexist"', str(excinfo.value) + ), "load_balance_hosts doesn't accept unknown values" + + # load_balance_hosts=disable should always choose the first one. + sess = connect_with( + node1, + f"host={hostlist} port={portlist} dbname=postgres load_balance_hosts=disable", + sql="SELECT 'connect1'", + ) + sess.close() + assert ( + count_statements(node1, "SELECT 'connect1'") >= 1 + ), "load_balance_hosts=disable connects to the first node" + + # Statistically the following loop with load_balance_hosts=random will + # almost certainly connect at least once to each of the nodes. The chance + # of that not happening is so small that it's negligible: + # (2/3)^50 = 1.56832855e-9 + for _ in range(1, 51): + sess = connect_with( + node1, + f"host={hostlist} port={portlist} dbname=postgres load_balance_hosts=random", + sql="SELECT 'connect2'", + ) + sess.close() + + node1_occurrences = count_statements(node1, "SELECT 'connect2'") + node2_occurrences = count_statements(node2, "SELECT 'connect2'") + node3_occurrences = count_statements(node3, "SELECT 'connect2'") + + total_occurrences = node1_occurrences + node2_occurrences + node3_occurrences + + assert node1_occurrences > 1, "received at least one connection on node1" + assert node2_occurrences > 1, "received at least one connection on node2" + assert node3_occurrences > 1, "received at least one connection on node3" + assert total_occurrences == 50, "received 50 connections across all nodes" + + node1.stop() + node2.stop() + + # load_balance_hosts=disable should continue trying hosts until it finds a + # working one. + sess = connect_with( + node3, + f"host={hostlist} port={portlist} dbname=postgres load_balance_hosts=disable", + sql="SELECT 'connect3'", + ) + sess.close() + assert ( + count_statements(node3, "SELECT 'connect3'") >= 1 + ), "load_balance_hosts=disable continues until it connects to a working node" + + # Also with load_balance_hosts=random we continue to the next nodes if + # previous ones are down. Connect a few times to make sure it's not just + # lucky. + for _ in range(1, 6): + sess = connect_with( + node3, + f"host={hostlist} port={portlist} dbname=postgres load_balance_hosts=random", + sql="SELECT 'connect4'", + ) + sess.close() + assert ( + count_statements(node3, "SELECT 'connect4'") >= 5 + ), "load_balance_hosts=random continues until it connects to a working node" diff --git a/src/interfaces/libpq/pyt/test_004_load_balance_dns.py b/src/interfaces/libpq/pyt/test_004_load_balance_dns.py new file mode 100644 index 0000000000..4845399d1e --- /dev/null +++ b/src/interfaces/libpq/pyt/test_004_load_balance_dns.py @@ -0,0 +1,188 @@ +# Copyright (c) 2023-2026, PostgreSQL Global Development Group + +"""Test load balancing based on DNS records with multiple IPs. + +This tests load balancing based on a DNS entry that contains multiple records +for different IPs. Since setting up a DNS server is more effort than we +consider reasonable to run this test, this situation is instead imitated by +using a hosts file where a single hostname maps to multiple different IP +addresses. This test requires the administrator to add the following lines to +the hosts file (if we detect that this hasn't happened we skip the test): + + 127.0.0.1 pg-loadbalancetest + 127.0.0.2 pg-loadbalancetest + 127.0.0.3 pg-loadbalancetest + +Windows or Linux are required to run this test because these OSes allow binding +to 127.0.0.2 and 127.0.0.3 addresses by default, but other OSes don't. We need +to bind to different IP addresses, so that we can use these different IP +addresses in the hosts file. + +The hosts file needs to be prepared before running this test. We don't do it +on the fly, because it requires root permissions to change the hosts file. In +CI we set up the previously mentioned rules in the hosts file, so that this load +balancing method is tested. + +This test is gated behind PG_TEST_EXTRA=load_balance, the special /etc/hosts +entries above, and a Linux/Windows host; otherwise it skips. When those are +present it runs the three nodes with create_pg(own_host=True) and a shared +port, so each binds a distinct loopback address (127.0.0.1/2/3) on the same TCP +port -- the topology the single DNS name pg-loadbalancetest needs. +""" + +import os +import re +import sys + +import pytest + +from libpq import Session +from libpq.errors import PqConnectionError + +# The hostname that the prepared hosts file maps to 127.0.0.1/2/3. +LOADBALANCE_HOST = "pg-loadbalancetest" +HOSTS_PATTERN = re.compile(r"127\.0\.0\.[1-3] pg-loadbalancetest") + + +# -- skip gating ------------------------------------------------------------- + + +def _skip_reason(): + """Return a skip reason if this test cannot run here, else None.""" + # Potentially unsafe test load_balance not enabled in PG_TEST_EXTRA. + extra = os.environ.get("PG_TEST_EXTRA", "") + if not re.search(r"\bload_balance\b", extra): + return "Potentially unsafe test load_balance not enabled in PG_TEST_EXTRA" + + # Windows or Linux are required: these OSes allow binding to 127.0.0.2 and + # 127.0.0.3 by default, but other OSes don't. + can_bind_to_127_0_0_2 = sys.platform.startswith("linux") or sys.platform == "win32" + if not can_bind_to_127_0_0_2: + return "load_balance test only supported on Linux and Windows" + + # The hosts file must contain the three pg-loadbalancetest mappings. + if sys.platform == "win32": + hosts_path = r"c:\Windows\System32\Drivers\etc\hosts" + else: + hosts_path = "/etc/hosts" + try: + with open(hosts_path, "r", encoding="utf-8", errors="replace") as fh: + hosts_content = fh.read() + except OSError: + hosts_content = "" + if len(HOSTS_PATTERN.findall(hosts_content)) != 3: + return "hosts file was not prepared for DNS load balance test" + + return None + + +# Module-level skip: in the conversion environment load_balance is not in +# PG_TEST_EXTRA, so the whole module skips cleanly and never starts a server. +_skip = _skip_reason() +if _skip is not None: + pytest.skip(_skip, allow_module_level=True) + + +# -- helpers (NOT named test_*) ---------------------------------------------- + + +def _connect_ok(node, connstr, msg, *, sql=None, log_like=None): + """Connect with *connstr*, optionally run *sql*, assert success and logs. + + Opens a fresh libpq Session (no psql subprocess), runs *sql* if given, and + -- when *log_like* patterns are supplied -- asserts each appears in + *node*'s server log. + """ + offset = node.log_position() + sess = None + try: + sess = Session(connstr=connstr, libdir=node.libdir) + if sql is not None: + sess.query_safe(sql) + except PqConnectionError as exc: # pragma: no cover - only runs when enabled + raise AssertionError(f"{msg}: connection failed: {exc}") from exc + finally: + if sess is not None: + sess.close() + + for pattern in log_like or []: + log = node.log_content()[offset:] + assert re.search( + pattern, log + ), f"{msg}: pattern /{pattern}/ not found in server log\n{log}" + + +def _occurrences(node, pattern): + return len(re.findall(pattern, node.log_content())) + + +# -- the test ---------------------------------------------------------------- + + +def test_004_load_balance_dns(create_pg): + # All three nodes bind their own loopback address (127.0.0.1/2/3) on the + # *same* TCP port, so the single DNS name pg-loadbalancetest (which the + # prepared hosts file maps to those three IPs) selects between them. + node1 = create_pg("node1", start=False, own_host=True) + node2 = create_pg("node2", start=False, own_host=True, port=node1.port) + node3 = create_pg("node3", start=False, own_host=True, port=node1.port) + + for node in (node1, node2, node3): + # log_statement = all so connect_ok's log_like checks can see the SQL. + node.append_conf("log_statement = all\n") + node.start() + + # load_balance_hosts=disable should always choose the first one. + _connect_ok( + node1, + f"host={LOADBALANCE_HOST} port={node1.port} load_balance_hosts=disable", + "load_balance_hosts=disable connects to the first node", + sql="SELECT 'connect1'", + log_like=[r"statement: SELECT 'connect1'"], + ) + + # Statistically the following loop with load_balance_hosts=random will + # almost certainly connect at least once to each of the nodes. The chance + # of that not happening is negligible: (2/3)^50 = 1.56832855e-9. + for _ in range(50): + _connect_ok( + node1, + f"host={LOADBALANCE_HOST} port={node1.port} load_balance_hosts=random", + "repeated connections with random load balancing", + sql="SELECT 'connect2'", + ) + + node1_occurrences = _occurrences(node1, r"statement: SELECT 'connect2'") + node2_occurrences = _occurrences(node2, r"statement: SELECT 'connect2'") + node3_occurrences = _occurrences(node3, r"statement: SELECT 'connect2'") + + total_occurrences = node1_occurrences + node2_occurrences + node3_occurrences + + assert node1_occurrences > 1, "received at least one connection on node1" + assert node2_occurrences > 1, "received at least one connection on node2" + assert node3_occurrences > 1, "received at least one connection on node3" + assert total_occurrences == 50, "received 50 connections across all nodes" + + node1.stop() + node2.stop() + + # load_balance_hosts=disable should continue trying hosts until it finds a + # working one. + _connect_ok( + node3, + f"host={LOADBALANCE_HOST} port={node3.port} load_balance_hosts=disable", + "load_balance_hosts=disable continues until it connects to a working node", + sql="SELECT 'connect3'", + log_like=[r"statement: SELECT 'connect3'"], + ) + + # Also with load_balance_hosts=random we continue to the next nodes if + # previous ones are down. Connect a few times to make sure it's not luck. + for _ in range(5): + _connect_ok( + node3, + f"host={LOADBALANCE_HOST} port={node3.port} load_balance_hosts=random", + "load_balance_hosts=random continues until it connects to a working node", + sql="SELECT 'connect4'", + log_like=[r"statement: SELECT 'connect4'"], + ) diff --git a/src/interfaces/libpq/pyt/test_005_negotiate_encryption.py b/src/interfaces/libpq/pyt/test_005_negotiate_encryption.py new file mode 100644 index 0000000000..6a7cd09e94 --- /dev/null +++ b/src/interfaces/libpq/pyt/test_005_negotiate_encryption.py @@ -0,0 +1,791 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Test negotiation of SSL and GSSAPI encryption. + +OVERVIEW +-------- + +Test negotiation of SSL and GSSAPI encryption. + +We test all combinations of: + +- all the libpq client options that affect the protocol negotiations + (gssencmode, sslmode, sslnegotiation) +- server accepting or rejecting the authentication due to pg_hba.conf entries +- SSL and GSS enabled/disabled in the server + +That's a lot of combinations, so we use a table-driven approach. Each +combination is represented by a line in a table. The line lists the options +specifying the test case, and an expected outcome. The expected outcome +includes whether the connection succeeds or fails, and whether it uses SSL, +GSS or no encryption. It also includes a condensed trace of what steps were +taken during the negotiation. + +See the docstring of :func:`_parse_log_events` for the EVENTS / OUTCOME table +format. + +NOTES +----- + +The whole combination table is checked as follows: + +* The connection is attempted in-process via libpq (a fresh ``Session``), not + by forking psql. The full conninfo string carries + host=/hostaddr=/user=/gssencmode=/sslmode=/sslnegotiation=, and the outcome + is the value returned by ``SELECT current_enc()`` on success, or ``fail`` if + the connection (or that query) fails. + +* The EVENTS trace is scraped from the *server* log, relying on the server's + ``trace_connection_negotiation``, ``log_connections`` and + ``log_disconnections`` output. The in-process Session drives the same wire + protocol negotiation, and the same server log lines are produced and parsed. + +Framework specifics: + +* TCP listening is configured with ``listen_addresses = '127.0.0.1'`` (the + PostgresServer fixture is unix-socket-only by default). + +* injection_points availability is probed via ``pg_available_extensions``. + +* SSL on/off is toggled by appending ``ssl = on`` / ``ssl = off`` to + postgresql.conf and reloading (a later setting wins). + +The kerberos and ssl_server fixtures are obtained lazily (via +``request.getfixturevalue``) only when the corresponding support is enabled, +so the GSS/SSL sub-blocks skip independently (and the whole module skips +cleanly when libpq_encryption is not enabled). +""" + +import os +import re + +import pytest + +from libpq import Session +from libpq.errors import PqConnectionError + +# -- module-level gating ----------------------------------------------------- + +if not re.search(r"\blibpq_encryption\b", os.environ.get("PG_TEST_EXTRA", "")): + pytest.skip( + "Potentially unsafe test libpq_encryption not enabled in PG_TEST_EXTRA", + allow_module_level=True, + ) + +# Only run the GSSAPI tests when compiled with GSSAPI support and PG_TEST_EXTRA +# includes 'kerberos'. +GSS_SUPPORTED = os.environ.get("with_gssapi") == "yes" +KERBEROS_ENABLED = bool(re.search(r"\bkerberos\b", os.environ.get("PG_TEST_EXTRA", ""))) +SSL_SUPPORTED = os.environ.get("with_ssl") == "openssl" + +HOST = "enc-test-localhost.postgresql.example.com" +HOSTADDR = "127.0.0.1" +SERVERCIDR = "127.0.0.1/32" + +DBNAME = "postgres" +GSSUSER_PASSWORD = "secret1" + +ALL_TEST_USERS = ["testuser", "ssluser", "nossluser", "gssuser", "nogssuser"] +ALL_GSSENCMODES = ["disable", "prefer", "require"] +ALL_SSLMODES = ["disable", "allow", "prefer", "require"] +ALL_SSLNEGOTIATIONS = ["postgres", "direct"] + + +# -- table parsing helpers (NOT named test_*) -------------------------------- + + +def _expand_expected_line(user, gssencmode, sslmode, sslnegotiation, expected): + """Expand '*' wildcards on a test table line into concrete keys.""" + result = {} + if user == "*": + for x in ALL_TEST_USERS: + result.update( + _expand_expected_line(x, gssencmode, sslmode, sslnegotiation, expected) + ) + elif gssencmode == "*": + for x in ALL_GSSENCMODES: + result.update( + _expand_expected_line(user, x, sslmode, sslnegotiation, expected) + ) + elif sslmode == "*": + for x in ALL_SSLMODES: + result.update( + _expand_expected_line(user, gssencmode, x, sslnegotiation, expected) + ) + elif sslnegotiation == "*": + for x in ALL_SSLNEGOTIATIONS: + result.update(_expand_expected_line(user, gssencmode, sslmode, x, expected)) + else: + result[f"{user} {gssencmode} {sslmode} {sslnegotiation}"] = expected + return result + + +_LINE_RE = re.compile(r"^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S.*)\s*->\s*(\S+)\s*$") + + +def _parse_table(table): + """Parse a test table. See the comment at the top of the file for format.""" + expected = {} + user = gssencmode = sslmode = sslnegotiation = None + + for raw_line in table.split("\n"): + # Trim comments and surrounding whitespace. + line = re.sub(r"#.*$", "", raw_line).strip() + if line == "": + continue + + m = _LINE_RE.match(line) + if not m: + raise AssertionError(f'could not parse line "{line}"') + + if m.group(1) != ".": + user = m.group(1) + if m.group(2) != ".": + gssencmode = m.group(2) + if m.group(3) != ".": + sslmode = m.group(3) + if m.group(4) != ".": + sslnegotiation = m.group(4) + + # Normalize the whitespace in the "EVENTS -> OUTCOME" part. + events = re.split(r",\s*", m.group(5)) + outcome = m.group(6) + events_str = ", ".join(events).rstrip() + events_and_outcome = f"{events_str} -> {outcome}" + + expected.update( + _expand_expected_line( + user, gssencmode, sslmode, sslnegotiation, events_and_outcome + ) + ) + return expected + + +def _parse_log_events(log_contents): + """Scrape the server log for the negotiation events. + + Each recognised log line emits one condensed event token; no events at all + is represented by ``-``. + """ + events = [] + for line in log_contents.split("\n"): + if "connection received" in line: + events.append("reconnect" if events else "connect") + if "SSLRequest accepted" in line: + events.append("sslaccept") + if "SSLRequest rejected" in line: + events.append("sslreject") + if "direct SSL connection accepted" in line: + events.append("directsslaccept") + if "direct SSL connection rejected" in line: + events.append("directsslreject") + if "GSSENCRequest accepted" in line: + events.append("gssaccept") + if "GSSENCRequest rejected" in line: + events.append("gssreject") + if "no pg_hba.conf entry" in line: + events.append("authfail") + if "connection authenticated" in line: + events.append("authok") + if "error triggered for injection point backend-" in line: + events.append("backenderror") + if "protocol version 2 error triggered" in line: + events.append("v2error") + + if not events: + events.append("-") + return events + + +# -- connection test driver (NOT named test_*) ------------------------------- + + +class _Harness: + """Holds the node and accumulates pass/fail results for the run.""" + + def __init__(self, node): + self.node = node + self.failures = [] + self.count = 0 + + def connect_test(self, connstr, expected_events_and_outcome): + """Attempt a connection and verify the events and outcome. + + The outcome is the value of ``SELECT current_enc()`` on success or + ``fail`` on any failure, and the EVENTS are scraped from the server log + lines produced since the attempt began. + """ + self.count += 1 + test_name = f" '{connstr}' -> {expected_events_and_outcome}" + + connstr_full = "" + if "dbname=" not in connstr: + connstr_full += "dbname=postgres " + if "host=" not in connstr: + connstr_full += f"host={HOST} hostaddr={HOSTADDR} " + # The framework gives each node its own port, so add it here (later + # keywords win in libpq, so an explicit port= in connstr -- there is + # none -- would still take precedence). + if "port=" not in connstr: + connstr_full += f"port={self.node.port} " + connstr_full += connstr + + # Record the current log size; afterwards we look only at new lines. + log_location = self.node.log_position() + + outcome = "fail" + stderr = "" + sess = None + try: + sess = Session(connstr=connstr_full, libdir=self.node.libdir) + res = sess.query("SELECT current_enc()") + if res.error_message is None: + outcome = res.psqlout.strip() + else: + stderr = res.error_message + except PqConnectionError as exc: + stderr = str(exc) + finally: + if sess is not None: + sess.close() + + # Parse the EVENTS from the new portion of the log file. Wait briefly + # for the disconnection record so the trailing events are present. + log_contents = self._slurp_log(log_location) + events = _parse_log_events(log_contents) + + events_and_outcome = ", ".join(events) + f" -> {outcome}" + if events_and_outcome != expected_events_and_outcome: + self.failures.append( + f"FAIL:{test_name}\n" + f" got: {events_and_outcome}\n" + f" expected: {expected_events_and_outcome}\n" + f" stderr: {stderr.strip()}" + ) + + def _slurp_log(self, offset): + """Return log text written since *offset*, allowing it to settle. + + The server writes its log records slightly asynchronously from the + client's point of view, so poll until two consecutive reads agree + (content stopped growing). An empty result is legitimate -- a purely + client-side failure (e.g. gssencmode=require with no ccache, or a + direct-SSL request rejected before connecting) never reaches the server + -- so empty-and-stable returns promptly rather than waiting out a long + timeout. + """ + import time + + deadline = time.monotonic() + 5.0 + prev = self.node.log_content()[offset:] + while True: + time.sleep(0.05) + content = self.node.log_content()[offset:] + if content == prev: + return content + prev = content + if time.monotonic() > deadline: + return content + + +def _test_matrix(harness, test_users, gssencmodes, sslmodes, sslnegotiations, expected): + """Test the cube of parameters: user, gssencmode, sslmode, sslnegotiation.""" + for test_user in test_users: + for gssencmode in gssencmodes: + for client_mode in sslmodes: + for negotiation in sslnegotiations: + key = f"{test_user} {gssencmode} {client_mode} {negotiation}" + expected_events = expected.get( + key, "" + ) + harness.connect_test( + f"user={test_user} gssencmode={gssencmode} " + f"sslmode={client_mode} sslnegotiation={negotiation}", + expected_events, + ) + + +# -- the test ---------------------------------------------------------------- + + +def test_005_negotiate_encryption(create_pg, request, tmp_path): + ### + ### Prepare test server for GSSAPI and SSL authentication, with a few + ### different test users and helper functions. We don't actually enable + ### SSL and kerberos in the server yet, we will do that later. + ### + node = create_pg("node", start=False) + node.append_conf( + f""" +listen_addresses = '{HOSTADDR}' + +# Capturing the EVENTS that occur during tests requires these settings +log_connections = 'receipt,authentication,authorization' +log_disconnections = on +trace_connection_negotiation = on +lc_messages = 'C' +""" + ) + pgdata = node.data_dir + + krb = None + if GSS_SUPPORTED and KERBEROS_ENABLED: + # note: setting up Kerberos + realm = "EXAMPLE.COM" + kerberos = request.getfixturevalue("kerberos") + krb = kerberos(HOST, HOSTADDR, realm) + node.append_conf(f"krb_server_keyfile = '{krb.keytab}'\n") + + ssl_server = None + if SSL_SUPPORTED: + ssl_server = request.getfixturevalue("ssl_server") + certdir = ssl_server.ssl_dir + import shutil + + shutil.copy( + os.path.join(certdir, "server-cn-only.crt"), + os.path.join(pgdata, "server.crt"), + ) + shutil.copy( + os.path.join(certdir, "server-cn-only.key"), + os.path.join(pgdata, "server.key"), + ) + os.chmod(os.path.join(pgdata, "server.key"), 0o600) + + # Start with SSL disabled. + node.append_conf("ssl = off\n") + + node.start() + + # Check if the extension injection_points is available, as it may be + # possible that this script is run with installcheck, where the module + # would not be installed by default. + injection_points_supported = ( + node.safe_sql( + "SELECT count(*) > 0 FROM pg_available_extensions " + "WHERE name = 'injection_points'" + ).strip() + == "t" + ) + + node.safe_sql("CREATE USER localuser;") + node.safe_sql("CREATE USER testuser;") + node.safe_sql("CREATE USER ssluser;") + node.safe_sql("CREATE USER nossluser;") + node.safe_sql("CREATE USER gssuser;") + node.safe_sql("CREATE USER nogssuser;") + if injection_points_supported: + node.safe_sql("CREATE EXTENSION injection_points;") + + unixdir = node.safe_sql("SHOW unix_socket_directories;").strip() + + # Helper function that returns the encryption method in use in the + # connection. + node.safe_sql( + r""" +CREATE FUNCTION current_enc() RETURNS text LANGUAGE plpgsql AS $$ +DECLARE + ssl_in_use bool; + gss_in_use bool; +BEGIN + ssl_in_use = (SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()); + gss_in_use = (SELECT encrypted FROM pg_stat_gssapi WHERE pid = pg_backend_pid()); + + raise log 'ssl % gss %', ssl_in_use, gss_in_use; + + IF ssl_in_use AND gss_in_use THEN + RETURN 'ssl+gss'; -- shouldn't happen + ELSIF ssl_in_use THEN + RETURN 'ssl'; + ELSIF gss_in_use THEN + RETURN 'gss'; + ELSE + RETURN 'plain'; + END IF; +END; +$$; +""" + ) + + # Only accept SSL connections from $servercidr. Our tests don't depend on + # this but seems best to keep it as narrow as possible for security reasons. + hba = ( + "# TYPE DATABASE USER ADDRESS METHOD OPTIONS\n" + "local postgres localuser trust\n" + f"host postgres testuser {SERVERCIDR} trust\n" + f"hostnossl postgres nossluser {SERVERCIDR} trust\n" + f"hostnogssenc postgres nogssuser {SERVERCIDR} trust\n" + ) + if SSL_SUPPORTED: + hba += f"hostssl postgres ssluser {SERVERCIDR} trust\n" + if GSS_SUPPORTED and KERBEROS_ENABLED: + hba += f"hostgssenc postgres gssuser {SERVERCIDR} trust\n" + with open(os.path.join(pgdata, "pg_hba.conf"), "w", encoding="utf-8") as fh: + fh.write(hba) + node.reload() + + # After the pg_hba.conf rewrite above, the only local-socket entry is for + # 'localuser', so the framework's own safe_sql (which connects as the OS + # user over the unix socket) no longer works. Run the later administrative + # statements via a dedicated localuser session. This is also what + # backend-side injection_points_attach needs. + def admin_psql(sql): + with Session( + connstr=f"host={unixdir} port={node.port} dbname=postgres " + f"user=localuser", + libdir=node.libdir, + ) as sess: + sess.query_safe(sql) + + # Ok, all prepared. Run the tests. + harness = _Harness(node) + + ### + ### Run tests with GSS and SSL disabled in the server + ### + if SSL_SUPPORTED: + test_table = r""" +# USER GSSENCMODE SSLMODE SSLNEGOTIATION EVENTS -> OUTCOME +testuser disable disable postgres connect, authok -> plain +. . allow postgres connect, authok -> plain +. . prefer postgres connect, sslreject, authok -> plain +. . require postgres connect, sslreject -> fail +. . . direct connect, directsslreject -> fail +. prefer disable postgres connect, authok -> plain +. . allow postgres connect, authok -> plain +. . prefer postgres connect, sslreject, authok -> plain +. . require postgres connect, sslreject -> fail +. . . direct connect, directsslreject -> fail + +# sslnegotiation=direct is not accepted unless sslmode=require or stronger +* * disable direct - -> fail +* * allow direct - -> fail +* * prefer direct - -> fail +""" + else: + # Compiled without SSL support + test_table = r""" +# USER GSSENCMODE SSLMODE SSLNEGOTIATION EVENTS -> OUTCOME +testuser disable disable postgres connect, authok -> plain +. . allow postgres connect, authok -> plain +. . prefer postgres connect, authok -> plain +. prefer disable postgres connect, authok -> plain +. . allow postgres connect, authok -> plain +. . prefer postgres connect, authok -> plain + +# Without SSL support, sslmode=require and sslnegotiation=direct are +# not accepted at all +* * require * - -> fail +* * * direct - -> fail +""" + + # All attempts with gssencmode=require fail without connecting because no + # credential cache has been configured in the client. (Or if GSS support + # is not compiled in, they will fail because of that.) + test_table += r""" +testuser require * * - -> fail +""" + + # note: Running tests with SSL and GSS disabled in the server + _test_matrix( + harness, + ["testuser"], + ALL_GSSENCMODES, + ALL_SSLMODES, + ALL_SSLNEGOTIATIONS, + _parse_table(test_table), + ) + + ### + ### Run tests with GSS disabled and SSL enabled in the server + ### + if SSL_SUPPORTED: + test_table = r""" +# USER GSSENCMODE SSLMODE SSLNEGOTIATION EVENTS -> OUTCOME +testuser disable disable postgres connect, authok -> plain +. . allow postgres connect, authok -> plain +. . prefer postgres connect, sslaccept, authok -> ssl +. . require postgres connect, sslaccept, authok -> ssl +. . . direct connect, directsslaccept, authok -> ssl +ssluser . disable postgres connect, authfail -> fail +. . allow postgres connect, authfail, reconnect, sslaccept, authok -> ssl +. . prefer postgres connect, sslaccept, authok -> ssl +. . require postgres connect, sslaccept, authok -> ssl +. . . direct connect, directsslaccept, authok -> ssl +nossluser . disable postgres connect, authok -> plain +. . allow postgres connect, authok -> plain +. . prefer postgres connect, sslaccept, authfail, reconnect, authok -> plain +. . require postgres connect, sslaccept, authfail -> fail +. . require direct connect, directsslaccept, authfail -> fail + +# sslnegotiation=direct is not accepted unless sslmode=require or stronger +* * disable direct - -> fail +* * allow direct - -> fail +* * prefer direct - -> fail +""" + + # Enable SSL in the server + node.append_conf("ssl = on\n") + node.reload() + + # note: Running tests with SSL enabled in server + _test_matrix( + harness, + ["testuser", "ssluser", "nossluser"], + ["disable"], + ALL_SSLMODES, + ALL_SSLNEGOTIATIONS, + _parse_table(test_table), + ) + + if injection_points_supported: + admin_psql("SELECT injection_points_attach('backend-initialize', 'error');") + harness.connect_test( + "user=testuser sslmode=prefer", "connect, backenderror -> fail" + ) + node.restart() + + admin_psql( + "SELECT injection_points_attach('backend-initialize-v2-error', 'error');" + ) + harness.connect_test( + "user=testuser sslmode=prefer", "connect, v2error -> fail" + ) + node.restart() + + admin_psql( + "SELECT injection_points_attach('backend-ssl-startup', 'error');" + ) + harness.connect_test( + "user=testuser sslmode=prefer", + "connect, sslaccept, backenderror, reconnect, authok -> plain", + ) + node.restart() + + # Disable SSL again + node.append_conf("ssl = off\n") + node.reload() + + ### + ### Run tests with GSS enabled, SSL disabled in the server + ### + if GSS_SUPPORTED and KERBEROS_ENABLED: + krb.create_principal("gssuser", GSSUSER_PASSWORD) + krb.create_ticket("gssuser", GSSUSER_PASSWORD) + + test_table = r""" +# USER GSSENCMODE SSLMODE SSLNEGOTIATION EVENTS -> OUTCOME +testuser disable disable postgres connect, authok -> plain +. . allow postgres connect, authok -> plain +. . prefer postgres connect, sslreject, authok -> plain +. . require postgres connect, sslreject -> fail +. . . direct connect, directsslreject -> fail +. prefer * postgres connect, gssaccept, authok -> gss +. prefer require direct connect, gssaccept, authok -> gss +. require * postgres connect, gssaccept, authok -> gss +. . require direct connect, gssaccept, authok -> gss + +gssuser disable disable postgres connect, authfail -> fail +. . allow postgres connect, authfail, reconnect, sslreject -> fail +. . prefer postgres connect, sslreject, authfail -> fail +. . require postgres connect, sslreject -> fail +. . . direct connect, directsslreject -> fail +. prefer * postgres connect, gssaccept, authok -> gss +. prefer require direct connect, gssaccept, authok -> gss +. require * postgres connect, gssaccept, authok -> gss +. . require direct connect, gssaccept, authok -> gss + +nogssuser disable disable postgres connect, authok -> plain +. . allow postgres connect, authok -> plain +. . prefer postgres connect, sslreject, authok -> plain +. . require postgres connect, sslreject -> fail +. . . direct connect, directsslreject -> fail +. prefer disable postgres connect, gssaccept, authfail, reconnect, authok -> plain +. . allow postgres connect, gssaccept, authfail, reconnect, authok -> plain +. . prefer postgres connect, gssaccept, authfail, reconnect, sslreject, authok -> plain +. . require postgres connect, gssaccept, authfail, reconnect, sslreject -> fail +. . . direct connect, gssaccept, authfail, reconnect, directsslreject -> fail +. require disable postgres connect, gssaccept, authfail -> fail +. . allow postgres connect, gssaccept, authfail -> fail +. . prefer postgres connect, gssaccept, authfail -> fail +. . require postgres connect, gssaccept, authfail -> fail # If both GSSAPI and sslmode are required, and GSS is not available -> fail +. . . direct connect, gssaccept, authfail -> fail # If both GSSAPI and sslmode are required, and GSS is not available -> fail + +# sslnegotiation=direct is not accepted unless sslmode=require or stronger +* * disable direct - -> fail +* * allow direct - -> fail +* * prefer direct - -> fail +""" + + # The expected events and outcomes above assume that SSL support is + # enabled. When libpq is compiled without SSL support, all attempts to + # connect with sslmode=require or sslnegotiation=direct would fail + # immediately without even connecting to the server. Skip those, + # because we tested them earlier already. + if SSL_SUPPORTED: + sslmodes, sslnegotiations = ALL_SSLMODES, ALL_SSLNEGOTIATIONS + else: + sslmodes, sslnegotiations = ["disable"], ["postgres"] + + # note: Running tests with GSS enabled in server + _test_matrix( + harness, + ["testuser", "gssuser", "nogssuser"], + ALL_GSSENCMODES, + sslmodes, + sslnegotiations, + _parse_table(test_table), + ) + + if injection_points_supported: + admin_psql("SELECT injection_points_attach('backend-initialize', 'error');") + harness.connect_test( + "user=testuser gssencmode=prefer sslmode=disable", + "connect, backenderror, reconnect, backenderror -> fail", + ) + node.restart() + + admin_psql( + "SELECT injection_points_attach('backend-initialize-v2-error', 'error');" + ) + harness.connect_test( + "user=testuser gssencmode=prefer sslmode=disable", + "connect, v2error, reconnect, v2error -> fail", + ) + node.restart() + + admin_psql( + "SELECT injection_points_attach('backend-gssapi-startup', 'error');" + ) + harness.connect_test( + "user=testuser gssencmode=prefer sslmode=disable", + "connect, gssaccept, backenderror, reconnect, authok -> plain", + ) + node.restart() + + ### + ### Tests with both GSS and SSL enabled in the server + ### + if SSL_SUPPORTED and GSS_SUPPORTED and KERBEROS_ENABLED: + # Sanity check that GSSAPI is still enabled from previous test. + harness.connect_test( + "user=testuser gssencmode=prefer sslmode=prefer", + "connect, gssaccept, authok -> gss", + ) + + # Enable SSL + node.append_conf("ssl = on\n") + node.reload() + + test_table = r""" +# USER GSSENCMODE SSLMODE SSLNEGOTIATION EVENTS -> OUTCOME +testuser disable disable postgres connect, authok -> plain +. . allow postgres connect, authok -> plain +. . prefer postgres connect, sslaccept, authok -> ssl +. . require postgres connect, sslaccept, authok -> ssl +. . . direct connect, directsslaccept, authok -> ssl +. prefer disable postgres connect, gssaccept, authok -> gss +. . allow postgres connect, gssaccept, authok -> gss +. . prefer postgres connect, gssaccept, authok -> gss +. . require postgres connect, gssaccept, authok -> gss # If both GSS and SSL is possible, GSS is chosen over SSL, even if sslmode=require +. . . direct connect, gssaccept, authok -> gss +. require disable postgres connect, gssaccept, authok -> gss +. . allow postgres connect, gssaccept, authok -> gss +. . prefer postgres connect, gssaccept, authok -> gss +. . require postgres connect, gssaccept, authok -> gss # If both GSS and SSL is possible, GSS is chosen over SSL, even if sslmode=require +. . . direct connect, gssaccept, authok -> gss + +gssuser disable disable postgres connect, authfail -> fail +. . allow postgres connect, authfail, reconnect, sslaccept, authfail -> fail +. . prefer postgres connect, sslaccept, authfail, reconnect, authfail -> fail +. . require postgres connect, sslaccept, authfail -> fail +. . . direct connect, directsslaccept, authfail -> fail +. prefer disable postgres connect, gssaccept, authok -> gss +. . allow postgres connect, gssaccept, authok -> gss +. . prefer postgres connect, gssaccept, authok -> gss +. . require postgres connect, gssaccept, authok -> gss # GSS is chosen over SSL, even though sslmode=require +. . . direct connect, gssaccept, authok -> gss +. require disable postgres connect, gssaccept, authok -> gss +. . allow postgres connect, gssaccept, authok -> gss +. . prefer postgres connect, gssaccept, authok -> gss +. . require postgres connect, gssaccept, authok -> gss # If both GSS and SSL is possible, GSS is chosen over SSL, even if sslmode=require +. . . direct connect, gssaccept, authok -> gss + +ssluser disable disable postgres connect, authfail -> fail +. . allow postgres connect, authfail, reconnect, sslaccept, authok -> ssl +. . prefer postgres connect, sslaccept, authok -> ssl +. . require postgres connect, sslaccept, authok -> ssl +. . . direct connect, directsslaccept, authok -> ssl +. prefer disable postgres connect, gssaccept, authfail, reconnect, authfail -> fail +. . allow postgres connect, gssaccept, authfail, reconnect, authfail, reconnect, sslaccept, authok -> ssl +. . prefer postgres connect, gssaccept, authfail, reconnect, sslaccept, authok -> ssl +. . require postgres connect, gssaccept, authfail, reconnect, sslaccept, authok -> ssl +. . . direct connect, gssaccept, authfail, reconnect, directsslaccept, authok -> ssl +. require disable postgres connect, gssaccept, authfail -> fail +. . allow postgres connect, gssaccept, authfail -> fail +. . prefer postgres connect, gssaccept, authfail -> fail +. . require postgres connect, gssaccept, authfail -> fail # If both GSS and SSL are required, the sslmode=require is effectively ignored and GSS is required +. . . direct connect, gssaccept, authfail -> fail + +nogssuser disable disable postgres connect, authok -> plain +. . allow postgres connect, authok -> plain +. . prefer postgres connect, sslaccept, authok -> ssl +. . require postgres connect, sslaccept, authok -> ssl +. . . direct connect, directsslaccept, authok -> ssl +. prefer disable postgres connect, gssaccept, authfail, reconnect, authok -> plain +. . allow postgres connect, gssaccept, authfail, reconnect, authok -> plain +. . prefer postgres connect, gssaccept, authfail, reconnect, sslaccept, authok -> ssl +. . require postgres connect, gssaccept, authfail, reconnect, sslaccept, authok -> ssl +. . . direct connect, gssaccept, authfail, reconnect, directsslaccept, authok -> ssl +. require disable postgres connect, gssaccept, authfail -> fail +. . allow postgres connect, gssaccept, authfail -> fail +. . prefer postgres connect, gssaccept, authfail -> fail +. . require postgres connect, gssaccept, authfail -> fail # If both GSS and SSL are required, the sslmode=require is effectively ignored and GSS is required +. . . direct connect, gssaccept, authfail -> fail + +nossluser disable disable postgres connect, authok -> plain +. . allow postgres connect, authok -> plain +. . prefer postgres connect, sslaccept, authfail, reconnect, authok -> plain +. . require postgres connect, sslaccept, authfail -> fail +. . . direct connect, directsslaccept, authfail -> fail +. prefer * postgres connect, gssaccept, authok -> gss +. . require direct connect, gssaccept, authok -> gss +. require * postgres connect, gssaccept, authok -> gss +. . require direct connect, gssaccept, authok -> gss + +# sslnegotiation=direct is not accepted unless sslmode=require or stronger +* * disable direct - -> fail +* * allow direct - -> fail +* * prefer direct - -> fail +""" + + # note: Running tests with both GSS and SSL enabled in server + _test_matrix( + harness, + ["testuser", "gssuser", "ssluser", "nogssuser", "nossluser"], + ALL_GSSENCMODES, + ALL_SSLMODES, + ALL_SSLNEGOTIATIONS, + _parse_table(test_table), + ) + + ### + ### Test negotiation over unix domain sockets. + ### + if unixdir != "": + # libpq doesn't attempt SSL or GSSAPI over Unix domain sockets. The + # server would reject them too. + harness.connect_test( + f"user=localuser gssencmode=prefer sslmode=prefer host={unixdir}", + "connect, authok -> plain", + ) + harness.connect_test( + f"user=localuser gssencmode=require sslmode=prefer host={unixdir}", + "- -> fail", + ) + + # Report all accumulated failures at once. + assert not harness.failures, ( + f"{len(harness.failures)} of {harness.count} negotiation cases failed:\n" + + "\n".join(harness.failures) + ) diff --git a/src/interfaces/libpq/pyt/test_006_service.py b/src/interfaces/libpq/pyt/test_006_service.py new file mode 100644 index 0000000000..0401c37506 --- /dev/null +++ b/src/interfaces/libpq/pyt/test_006_service.py @@ -0,0 +1,335 @@ +# Copyright (c) 2025-2026, PostgreSQL Global Development Group + +"""Tests scenarios related to the service name and the service file. + +Covers the connection options and their environment variables (PGSERVICE / +PGSERVICEFILE / PGSYSCONFDIR, and the "service" / "servicefile" connection +keywords). + +Connections are made with a psql subprocess so that the service environment +variables are inherited the way a real client sees them; a successful +connection runs the SELECT and checks its output, a failed connection exits +non-zero with an error message we match. (An in-process libpq connection is +not used here: the in-process library does not portably read the environment, +and on Windows it cannot connect to the server's socket from this process.) + +The service file points at the real, started ``pg`` server. The framework's +environment setup clears the PG* connection variables, so the only connection +information in play comes from the service file / service keywords under test. +""" + +import getpass +import os +import re +import shutil +from contextlib import contextmanager + +import pytest + +# The login role: the cluster uses trust auth, so any user connects. We pin it +# explicitly in every connection string so that nothing has to inject a +# "user=" keyword (which would corrupt the URI-form connection strings). +USER = getpass.getuser() + + +@contextmanager +def _env(**overrides): + """Temporarily set/unset environment variables, restoring on exit. + + A value of None unsets the variable for the duration of the block. + """ + saved = {} + try: + for key, val in overrides.items(): + saved[key] = os.environ.get(key) + if val is None: + os.environ.pop(key, None) + else: + os.environ[key] = val + yield + finally: + for key, val in saved.items(): + if val is None: + os.environ.pop(key, None) + else: + os.environ[key] = val + + +def _kw_user(connstr): + """Append a user= keyword to a (keyword-form) connection string.""" + return (connstr + f" user='{USER}'").strip() + + +def _uri_user(uri): + """Append a user query parameter to a URI-form connection string.""" + sep = "&" if "?" in uri else "?" + return f"{uri}{sep}user={USER}" + + +def _psql(node, connstr, sql): + """Run psql with *connstr* verbatim and return ``(ok, stdout, stderr)``. + + The connection string is passed as-is (no host/port prepended), so the + service / servicefile under test fully determines the connection target. + psql is a subprocess, so it inherits PGSERVICE / PGSERVICEFILE / + PGSYSCONFDIR from the environment. + """ + res = node.pg_bin.result( + ["psql", "-w", "-X", "-A", "-q", "-t", "-d", connstr, "-c", sql] + ) + return res.returncode == 0, res.stdout, res.stderr + + +def _connect_ok(node, connstr, expected, **env): + """Assert psql connects with *connstr* and its output matches *expected*.""" + with _env(**env): + ok, out, err = _psql(node, connstr, f"SELECT '{expected}'") + assert ok, f"connection should succeed for {connstr!r}: got {err!r}" + assert re.search(expected, out), f"stdout matches for {connstr!r}: got {out!r}" + + +def _connect_fails(node, connstr, pattern, **env): + """Assert psql fails to connect with *connstr* and *pattern* is in the error.""" + with _env(**env): + ok, _out, err = _psql(node, connstr, "SELECT 1") + assert not ok, f"connection should fail for {connstr!r}" + assert re.search( + pattern, err + ), f"error matches /{pattern}/ for {connstr!r}: got {err!r}" + + +def _connect_servicefile_is(node, connstr, expected_servicefile, **env): + """Assert psql connects and the service file it resolved matches. + + psql exposes the service file libpq actually used as the :SERVICEFILE + variable. + """ + with _env(**env): + ok, out, err = _psql(node, connstr, r"\echo :SERVICEFILE") + assert ok, f"connection should succeed for {connstr!r}: got {err!r}" + actual = out.strip() + assert actual == expected_servicefile, ( + f"resolved servicefile for {connstr!r}: expected " + f"{expected_servicefile!r}, got {actual!r}" + ) + + +@pytest.fixture +def service_setup(pg, tmp_path): + """Build the set of service files used by the tests, pointing at ``pg``. + + Returns a dict of the paths plus the base environment (PGSYSCONFDIR and a + default empty PGSERVICEFILE) that every test starts from. + """ + td = tmp_path + + # File that includes a valid service name, using a decomposed connection + # string for its contents (one parameter per line). The connection + # parameters are written unquoted, as a service file gives each value + # verbatim to the right of the "=". + srvfile_valid = td / "pg_service_valid.conf" + lines = [ + "[my_srv]", + f"host={pg.host}", + f"port={pg.port}", + "dbname=postgres", + ] + srvfile_valid.write_text("\n".join(lines) + "\n") + + # File defined with no contents, used as the default value for + # PGSERVICEFILE so that no lookup is attempted in the user's home dir. + srvfile_empty = td / "pg_service_empty.conf" + srvfile_empty.write_text("") + + # Default service file in PGSYSCONFDIR. + srvfile_default = td / "pg_service.conf" + + # Missing service file. + srvfile_missing = td / "pg_service_missing.conf" + + # Service file with a nested "service" defined. + srvfile_nested = td / "pg_service_nested.conf" + shutil.copy(srvfile_valid, srvfile_nested) + with open(srvfile_nested, "a", encoding="utf-8") as fh: + fh.write("service=invalid_srv\n") + + # Service file with a nested "servicefile" defined. + srvfile_nested_2 = td / "pg_service_nested_2.conf" + shutil.copy(srvfile_valid, srvfile_nested_2) + with open(srvfile_nested_2, "a", encoding="utf-8") as fh: + fh.write(f"servicefile={srvfile_default}\n") + + # Use forward slashes for every path. Several of these go into connection + # strings as a servicefile= value, where libpq treats backslash as an + # escape character and would mangle a Windows path; forward slashes are a + # valid path separator on Windows for files, environment values and + # libpq's own servicefile bookkeeping, so they work everywhere. + def fwd(path): + return str(path).replace("\\", "/") + + return { + "td": fwd(td), + "valid": fwd(srvfile_valid), + "empty": fwd(srvfile_empty), + "default": fwd(srvfile_default), + "missing": fwd(srvfile_missing), + "nested": fwd(srvfile_nested), + "nested_2": fwd(srvfile_nested_2), + # PGSYSCONFDIR is the fallback directory lookup of the service file. + # PGSERVICEFILE is forced to a default (empty) location so the test + # never looks at a home directory. + "base_env": {"PGSYSCONFDIR": fwd(td), "PGSERVICEFILE": fwd(srvfile_empty)}, + "node": pg, + } + + +def test_service_with_pgservicefile(service_setup): + """Combinations of service name and a valid service file via PGSERVICEFILE.""" + s = service_setup + node = s["node"] + env = dict(s["base_env"], PGSERVICEFILE=s["valid"]) + + _connect_ok(node, _kw_user("service=my_srv"), "connect1_1", **env) + _connect_ok(node, _uri_user("postgres://?service=my_srv"), "connect1_2", **env) + _connect_fails( + node, + _kw_user("service=undefined-service"), + r'definition of service "undefined-service" not found', + **env, + ) + + _connect_ok(node, _kw_user(""), "connect1_3", **dict(env, PGSERVICE="my_srv")) + _connect_fails( + node, + _kw_user(""), + r'definition of service "undefined-service" not found', + **dict(env, PGSERVICE="undefined-service"), + ) + + +def test_service_with_incorrect_pgservicefile(service_setup): + """Incorrect (missing) service file referenced by PGSERVICEFILE.""" + s = service_setup + env = dict(s["base_env"], PGSERVICEFILE=s["missing"]) + _connect_fails( + s["node"], + _kw_user("service=my_srv"), + r'service file ".*pg_service_missing\.conf" not found', + **env, + ) + + +def test_service_with_default_pg_service_conf(service_setup): + """Service file named "pg_service.conf" found in PGSYSCONFDIR.""" + s = service_setup + node = s["node"] + # Create copy of the valid file at the default PGSYSCONFDIR location. + shutil.copy(s["valid"], s["default"]) + try: + env = dict(s["base_env"]) # PGSERVICEFILE stays at the empty default + _connect_ok(node, _kw_user("service=my_srv"), "connect2_1", **env) + _connect_ok(node, _uri_user("postgres://?service=my_srv"), "connect2_2", **env) + _connect_fails( + node, + _kw_user("service=undefined-service"), + r'definition of service "undefined-service" not found', + **env, + ) + _connect_ok(node, _kw_user(""), "connect2_3", **dict(env, PGSERVICE="my_srv")) + # The given servicefile (empty) does not define the service, so it is + # found in the default pg_service.conf; libpq then reports the default + # file as the resolved servicefile. + _connect_servicefile_is( + node, + _kw_user(f"service=my_srv servicefile='{s['empty']}'"), + s["default"], + **env, + ) + _connect_fails( + node, + _kw_user(""), + r'definition of service "undefined-service" not found', + **dict(env, PGSERVICE="undefined-service"), + ) + finally: + os.unlink(s["default"]) + + +def test_service_nested(service_setup): + """Nested "service" / "servicefile" specifications are rejected.""" + s = service_setup + node = s["node"] + + _connect_fails( + node, + _kw_user("service=my_srv"), + r'nested "service" specifications not supported in service file', + **dict(s["base_env"], PGSERVICEFILE=s["nested"]), + ) + _connect_fails( + node, + _kw_user("service=my_srv"), + r'nested "servicefile" specifications not supported in service file', + **dict(s["base_env"], PGSERVICEFILE=s["nested_2"]), + ) + + +def test_servicefile_option(service_setup): + """The "servicefile" connection option works in keyword and URI forms.""" + s = service_setup + node = s["node"] + env = dict(s["base_env"]) # PGSERVICEFILE stays at the empty default + + # No backslash escaping needed on non-Windows (paths use forward slashes). + valid = s["valid"] + + _connect_ok( + node, + _kw_user(f"service=my_srv servicefile='{valid}'"), + "connect3_1", + **env, + ) + + # Encode slashes (and backslash, and colon) for the URI form. + encoded = valid.replace("\\", "%5C").replace("/", "%2F").replace(":", "%3A") + + _connect_ok( + node, + _uri_user(f"postgresql:///?service=my_srv&servicefile={encoded}"), + "connect3_2", + **env, + ) + + _connect_ok( + node, + _kw_user(f"servicefile='{valid}'"), + "connect3_3", + **dict(env, PGSERVICE="my_srv"), + ) + _connect_ok( + node, + _uri_user(f"postgresql://?servicefile={encoded}"), + "connect3_4", + **dict(env, PGSERVICE="my_srv"), + ) + + +def test_servicefile_option_priority(service_setup): + """The "servicefile" option takes priority over PGSERVICEFILE.""" + s = service_setup + node = s["node"] + valid = s["valid"] + env = dict(s["base_env"], PGSERVICEFILE="non-existent-file.conf") + + _connect_fails( + node, + _kw_user("service=my_srv"), + r'service file "non-existent-file\.conf" not found', + **env, + ) + _connect_ok( + node, + _kw_user(f"service=my_srv servicefile='{valid}'"), + "connect4_1", + **env, + ) diff --git a/src/interfaces/libpq/t/001_uri.pl b/src/interfaces/libpq/t/001_uri.pl deleted file mode 100644 index 64f257ae04..0000000000 --- a/src/interfaces/libpq/t/001_uri.pl +++ /dev/null @@ -1,288 +0,0 @@ -# Copyright (c) 2021-2026, PostgreSQL Global Development Group -use strict; -use warnings FATAL => 'all'; - -use PostgreSQL::Test::Utils; -use Test::More; -use IPC::Run; - - -# List of URIs tests. For each test the first element is the input string, the -# second the expected stdout and the third the expected stderr. Optionally, -# additional arguments may specify key/value pairs which will override -# environment variables for the duration of the test. -my @tests = ( - [ - q{postgresql://uri-user:secret@host:12345/db}, - q{user='uri-user' password='secret' dbname='db' host='host' port='12345' (inet)}, - q{}, - ], - [ - q{postgresql://uri-user@host:12345/db}, - q{user='uri-user' dbname='db' host='host' port='12345' (inet)}, q{}, - ], - [ - q{postgresql://uri-user@host/db}, - q{user='uri-user' dbname='db' host='host' (inet)}, q{}, - ], - [ - q{postgresql://host:12345/db}, - q{dbname='db' host='host' port='12345' (inet)}, q{}, - ], - [ q{postgresql://host/db}, q{dbname='db' host='host' (inet)}, q{}, ], - [ - q{postgresql://uri-user@host:12345/}, - q{user='uri-user' host='host' port='12345' (inet)}, - q{}, - ], - [ - q{postgresql://uri-user@host/}, - q{user='uri-user' host='host' (inet)}, - q{}, - ], - [ q{postgresql://uri-user@}, q{user='uri-user' (local)}, q{}, ], - [ q{postgresql://host:12345/}, q{host='host' port='12345' (inet)}, q{}, ], - [ q{postgresql://host:12345}, q{host='host' port='12345' (inet)}, q{}, ], - [ q{postgresql://host/db}, q{dbname='db' host='host' (inet)}, q{}, ], - [ q{postgresql://host/}, q{host='host' (inet)}, q{}, ], - [ q{postgresql://host}, q{host='host' (inet)}, q{}, ], - [ q{postgresql://}, q{(local)}, q{}, ], - [ - q{postgresql://?hostaddr=127.0.0.1}, q{hostaddr='127.0.0.1' (inet)}, - q{}, - ], - [ - q{postgresql://example.com?hostaddr=63.1.2.4}, - q{host='example.com' hostaddr='63.1.2.4' (inet)}, - q{}, - ], - [ q{postgresql://%68ost/}, q{host='host' (inet)}, q{}, ], - [ - q{postgresql://host/db?user=uri-user}, - q{user='uri-user' dbname='db' host='host' (inet)}, - q{}, - ], - [ - q{postgresql://host/db?user=uri-user&port=12345}, - q{user='uri-user' dbname='db' host='host' port='12345' (inet)}, - q{}, - ], - [ - q{postgresql://host/db?u%73er=someotheruser&port=12345}, - q{user='someotheruser' dbname='db' host='host' port='12345' (inet)}, - q{}, - ], - [ - q{postgresql://host/db?u%7aer=someotheruser&port=12345}, q{}, - q{libpq_uri_regress: invalid URI query parameter: "uzer"}, - ], - [ - q{postgresql://host:12345?user=uri-user}, - q{user='uri-user' host='host' port='12345' (inet)}, - q{}, - ], - [ - q{postgresql://host?user=uri-user}, - q{user='uri-user' host='host' (inet)}, - q{}, - ], - [ - # Leading and trailing spaces, works. - q{postgresql://host? user = uri-user & port = 12345 }, - q{user='uri-user' host='host' port='12345' (inet)}, - q{}, - ], - [ - # Trailing data in parameter. - q{postgresql://host? user user = uri & port = 12345 12 }, - q{}, - q{libpq_uri_regress: unexpected spaces found in " user user ", use percent-encoded spaces (%20) instead}, - ], - [ - # Trailing data in value. - q{postgresql://host? user = uri-user & port = 12345 12 }, - q{}, - q{libpq_uri_regress: unexpected spaces found in " 12345 12 ", use percent-encoded spaces (%20) instead}, - ], - [ q{postgresql://host?}, q{host='host' (inet)}, q{}, ], - [ - q{postgresql://[::1]:12345/db}, - q{dbname='db' host='::1' port='12345' (inet)}, - q{}, - ], - [ q{postgresql://[::1]/db}, q{dbname='db' host='::1' (inet)}, q{}, ], - [ - q{postgresql://[2001:db8::1234]/}, q{host='2001:db8::1234' (inet)}, - q{}, - ], - [ - q{postgresql://[200z:db8::1234]/}, q{host='200z:db8::1234' (inet)}, - q{}, - ], - [ q{postgresql://[::1]}, q{host='::1' (inet)}, q{}, ], - [ q{postgres://}, q{(local)}, q{}, ], - [ q{postgres:///}, q{(local)}, q{}, ], - [ q{postgres:///db}, q{dbname='db' (local)}, q{}, ], - [ - q{postgres://uri-user@/db}, q{user='uri-user' dbname='db' (local)}, - q{}, - ], - [ - q{postgres://?host=/path/to/socket/dir}, - q{host='/path/to/socket/dir' (local)}, - q{}, - ], - [ - q{postgresql://host?uzer=}, q{}, - q{libpq_uri_regress: invalid URI query parameter: "uzer"}, - ], - [ - q{postgre://}, - q{}, - q{libpq_uri_regress: missing "=" after "postgre://" in connection info string}, - ], - [ - q{postgres://[::1}, - q{}, - q{libpq_uri_regress: end of string reached when looking for matching "]" in IPv6 host address in URI: "postgres://[::1"}, - ], - [ - q{postgres://[]}, - q{}, - q{libpq_uri_regress: IPv6 host address may not be empty in URI: "postgres://[]"}, - ], - [ - q{postgres://[::1]z}, - q{}, - q{libpq_uri_regress: unexpected character "z" at position 17 in URI (expected ":" or "/"): "postgres://[::1]z"}, - ], - [ - q{postgresql://host?zzz}, - q{}, - q{libpq_uri_regress: missing key/value separator "=" in URI query parameter: "zzz"}, - ], - [ - q{postgresql://host?value1&value2}, - q{}, - q{libpq_uri_regress: missing key/value separator "=" in URI query parameter: "value1"}, - ], - [ - q{postgresql://host?key=key=value}, - q{}, - q{libpq_uri_regress: extra key/value separator "=" in URI query parameter: "key"}, - ], - [ - q{postgres://host?dbname=%XXfoo}, q{}, - q{libpq_uri_regress: invalid percent-encoded token: "%XXfoo"}, - ], - [ - q{postgresql://a%00b}, - q{}, - q{libpq_uri_regress: forbidden value %00 in percent-encoded value: "a%00b"}, - ], - [ - q{postgresql://%zz}, q{}, - q{libpq_uri_regress: invalid percent-encoded token: "%zz"}, - ], - [ - q{postgresql://%1}, q{}, - q{libpq_uri_regress: invalid percent-encoded token: "%1"}, - ], - [ - q{postgresql://%}, q{}, - q{libpq_uri_regress: invalid percent-encoded token: "%"}, - ], - [ q{postgres://@host}, q{host='host' (inet)}, q{}, ], - [ q{postgres://host:/}, q{host='host' (inet)}, q{}, ], - [ q{postgres://:12345/}, q{port='12345' (local)}, q{}, ], - [ - q{postgres://otheruser@?host=/no/such/directory}, - q{user='otheruser' host='/no/such/directory' (local)}, - q{}, - ], - [ - q{postgres://otheruser@/?host=/no/such/directory}, - q{user='otheruser' host='/no/such/directory' (local)}, - q{}, - ], - [ - q{postgres://otheruser@:12345?host=/no/such/socket/path}, - q{user='otheruser' host='/no/such/socket/path' port='12345' (local)}, - q{}, - ], - [ - q{postgres://otheruser@:12345/db?host=/path/to/socket}, - q{user='otheruser' dbname='db' host='/path/to/socket' port='12345' (local)}, - q{}, - ], - [ - q{postgres://:12345/db?host=/path/to/socket}, - q{dbname='db' host='/path/to/socket' port='12345' (local)}, - q{}, - ], - [ - q{postgres://:12345?host=/path/to/socket}, - q{host='/path/to/socket' port='12345' (local)}, - q{}, - ], - [ - q{postgres://%2Fvar%2Flib%2Fpostgresql/dbname}, - q{dbname='dbname' host='/var/lib/postgresql' (local)}, - q{}, - ], - # Usually the default sslmode is 'prefer' (for libraries with SSL) or - # 'disable' (for those without). This default changes to 'verify-full' if - # the system CA store is in use. - [ - q{postgresql://host?sslmode=disable}, - q{host='host' sslmode='disable' (inet)}, - q{}, - PGSSLROOTCERT => "system", - ], - [ - q{postgresql://host?sslmode=prefer}, - q{host='host' sslmode='prefer' (inet)}, - q{}, - PGSSLROOTCERT => "system", - ], - [ - q{postgresql://host?sslmode=verify-full}, - q{host='host' (inet)}, - q{}, PGSSLROOTCERT => "system", - ]); - -# test to run for each of the above test definitions -sub test_uri -{ - local $Test::Builder::Level = $Test::Builder::Level + 1; - local %ENV = %ENV; - - my $uri; - my %expect; - my %envvars; - my %result; - - ($uri, $expect{stdout}, $expect{stderr}, %envvars) = @$_; - - $expect{'exit'} = $expect{stderr} eq ''; - %ENV = (%ENV, %envvars); - - my $cmd = [ 'libpq_uri_regress', $uri ]; - $result{exit} = IPC::Run::run $cmd, - '>' => \$result{stdout}, - '2>' => \$result{stderr}; - - chomp($result{stdout}); - chomp($result{stderr}); - - # use is_deeply so there's one test result for each test above, without - # losing the information whether stdout/stderr mismatched. - is_deeply(\%result, \%expect, $uri); -} - -foreach (@tests) -{ - test_uri($_); -} - -done_testing(); diff --git a/src/interfaces/libpq/t/002_api.pl b/src/interfaces/libpq/t/002_api.pl deleted file mode 100644 index 409345836d..0000000000 --- a/src/interfaces/libpq/t/002_api.pl +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) 2022-2026, PostgreSQL Global Development Group -use strict; -use warnings FATAL => 'all'; - -use PostgreSQL::Test::Utils; -use Test::More; - -# Test PQsslAttribute(NULL, "library") -my ($out, $err) = run_command([ 'libpq_testclient', '--ssl' ]); - -if ($ENV{with_ssl} eq 'openssl') -{ - is($out, 'OpenSSL', 'PQsslAttribute(NULL, "library") returns "OpenSSL"'); -} -else -{ - is( $err, - 'SSL is not enabled', - 'PQsslAttribute(NULL, "library") returns NULL'); -} - -done_testing(); diff --git a/src/interfaces/libpq/t/003_load_balance_host_list.pl b/src/interfaces/libpq/t/003_load_balance_host_list.pl deleted file mode 100644 index 1f970ff994..0000000000 --- a/src/interfaces/libpq/t/003_load_balance_host_list.pl +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2023-2026, PostgreSQL Global Development Group -use strict; -use warnings FATAL => 'all'; -use Config; -use PostgreSQL::Test::Utils; -use PostgreSQL::Test::Cluster; -use Test::More; - -# This tests load balancing across the list of different hosts in the host -# parameter of the connection string. - -# Cluster setup which is shared for testing both load balancing methods -my $node1 = PostgreSQL::Test::Cluster->new('node1'); -my $node2 = PostgreSQL::Test::Cluster->new('node2', own_host => 1); -my $node3 = PostgreSQL::Test::Cluster->new('node3', own_host => 1); - -# Create a data directory with initdb -$node1->init(); -$node2->init(); -$node3->init(); - -# Start the PostgreSQL server -$node1->start(); -$node2->start(); -$node3->start(); - -# Start the tests for load balancing method 1 -my $hostlist = $node1->host . ',' . $node2->host . ',' . $node3->host; -my $portlist = $node1->port . ',' . $node2->port . ',' . $node3->port; - -$node1->connect_fails( - "host=$hostlist port=$portlist load_balance_hosts=doesnotexist", - "load_balance_hosts doesn't accept unknown values", - expected_stderr => qr/invalid load_balance_hosts value: "doesnotexist"/); - -# load_balance_hosts=disable should always choose the first one. -$node1->connect_ok( - "host=$hostlist port=$portlist load_balance_hosts=disable", - "load_balance_hosts=disable connects to the first node", - sql => "SELECT 'connect1'", - log_like => [qr/statement: SELECT 'connect1'/]); - -# Statistically the following loop with load_balance_hosts=random will almost -# certainly connect at least once to each of the nodes. The chance of that not -# happening is so small that it's negligible: (2/3)^50 = 1.56832855e-9 -foreach my $i (1 .. 50) -{ - $node1->connect_ok( - "host=$hostlist port=$portlist load_balance_hosts=random", - "repeated connections with random load balancing", - sql => "SELECT 'connect2'"); -} - -my $node1_occurrences = () = - $node1->log_content() =~ /statement: SELECT 'connect2'/g; -my $node2_occurrences = () = - $node2->log_content() =~ /statement: SELECT 'connect2'/g; -my $node3_occurrences = () = - $node3->log_content() =~ /statement: SELECT 'connect2'/g; - -my $total_occurrences = - $node1_occurrences + $node2_occurrences + $node3_occurrences; - -cmp_ok($node1_occurrences, '>', 1, - "received at least one connection on node1"); -cmp_ok($node2_occurrences, '>', 1, - "received at least one connection on node2"); -cmp_ok($node3_occurrences, '>', 1, - "received at least one connection on node3"); -is($total_occurrences, 50, "received 50 connections across all nodes"); - -$node1->stop(); -$node2->stop(); - -# load_balance_hosts=disable should continue trying hosts until it finds a -# working one. -$node3->connect_ok( - "host=$hostlist port=$portlist load_balance_hosts=disable", - "load_balance_hosts=disable continues until it connects to the a working node", - sql => "SELECT 'connect3'", - log_like => [qr/statement: SELECT 'connect3'/]); - -# Also with load_balance_hosts=random we continue to the next nodes if previous -# ones are down. Connect a few times to make sure it's not just lucky. -foreach my $i (1 .. 5) -{ - $node3->connect_ok( - "host=$hostlist port=$portlist load_balance_hosts=random", - "load_balance_hosts=random continues until it connects to the a working node", - sql => "SELECT 'connect4'", - log_like => [qr/statement: SELECT 'connect4'/]); -} - -done_testing(); diff --git a/src/interfaces/libpq/t/004_load_balance_dns.pl b/src/interfaces/libpq/t/004_load_balance_dns.pl deleted file mode 100644 index e1ff9a0602..0000000000 --- a/src/interfaces/libpq/t/004_load_balance_dns.pl +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright (c) 2023-2026, PostgreSQL Global Development Group -use strict; -use warnings FATAL => 'all'; -use Config; -use PostgreSQL::Test::Utils; -use PostgreSQL::Test::Cluster; -use Test::More; - -if (!$ENV{PG_TEST_EXTRA} || $ENV{PG_TEST_EXTRA} !~ /\bload_balance\b/) -{ - plan skip_all => - 'Potentially unsafe test load_balance not enabled in PG_TEST_EXTRA'; -} - -# This tests loadbalancing based on a DNS entry that contains multiple records -# for different IPs. Since setting up a DNS server is more effort than we -# consider reasonable to run this test, this situation is instead imitated by -# using a hosts file where a single hostname maps to multiple different IP -# addresses. This test requires the administrator to add the following lines to -# the hosts file (if we detect that this hasn't happened we skip the test): -# -# 127.0.0.1 pg-loadbalancetest -# 127.0.0.2 pg-loadbalancetest -# 127.0.0.3 pg-loadbalancetest -# -# Windows or Linux are required to run this test because these OSes allow -# binding to 127.0.0.2 and 127.0.0.3 addresses by default, but other OSes -# don't. We need to bind to different IP addresses, so that we can use these -# different IP addresses in the hosts file. -# -# The hosts file needs to be prepared before running this test. We don't do it -# on the fly, because it requires root permissions to change the hosts file. In -# CI we set up the previously mentioned rules in the hosts file, so that this -# load balancing method is tested. - -# Cluster setup which is shared for testing both load balancing methods -my $can_bind_to_127_0_0_2 = - $Config{osname} eq 'linux' || $PostgreSQL::Test::Utils::windows_os; - -# Checks for the requirements for testing load balancing method 2 -if (!$can_bind_to_127_0_0_2) -{ - plan skip_all => 'load_balance test only supported on Linux and Windows'; -} - -my $hosts_path; -if ($windows_os) -{ - $hosts_path = 'c:\Windows\System32\Drivers\etc\hosts'; -} -else -{ - $hosts_path = '/etc/hosts'; -} - -my $hosts_content = PostgreSQL::Test::Utils::slurp_file($hosts_path); - -my $hosts_count = () = - $hosts_content =~ /127\.0\.0\.[1-3] pg-loadbalancetest/g; -if ($hosts_count != 3) -{ - # Host file is not prepared for this test - plan skip_all => "hosts file was not prepared for DNS load balance test"; -} - -$PostgreSQL::Test::Cluster::use_tcp = 1; -$PostgreSQL::Test::Cluster::test_pghost = '127.0.0.1'; -my $port = PostgreSQL::Test::Cluster::get_free_port(); -my $node1 = PostgreSQL::Test::Cluster->new('node1', port => $port); -my $node2 = - PostgreSQL::Test::Cluster->new('node2', port => $port, own_host => 1); -my $node3 = - PostgreSQL::Test::Cluster->new('node3', port => $port, own_host => 1); - -# Create a data directory with initdb -$node1->init(); -$node2->init(); -$node3->init(); - -# Start the PostgreSQL server -$node1->start(); -$node2->start(); -$node3->start(); - -# load_balance_hosts=disable should always choose the first one. -$node1->connect_ok( - "host=pg-loadbalancetest port=$port load_balance_hosts=disable", - "load_balance_hosts=disable connects to the first node", - sql => "SELECT 'connect1'", - log_like => [qr/statement: SELECT 'connect1'/]); - - -# Statistically the following loop with load_balance_hosts=random will almost -# certainly connect at least once to each of the nodes. The chance of that not -# happening is so small that it's negligible: (2/3)^50 = 1.56832855e-9 -foreach my $i (1 .. 50) -{ - $node1->connect_ok( - "host=pg-loadbalancetest port=$port load_balance_hosts=random", - "repeated connections with random load balancing", - sql => "SELECT 'connect2'"); -} - -my $node1_occurrences = () = - $node1->log_content() =~ /statement: SELECT 'connect2'/g; -my $node2_occurrences = () = - $node2->log_content() =~ /statement: SELECT 'connect2'/g; -my $node3_occurrences = () = - $node3->log_content() =~ /statement: SELECT 'connect2'/g; - -my $total_occurrences = - $node1_occurrences + $node2_occurrences + $node3_occurrences; - -cmp_ok($node1_occurrences, '>', 1, - "received at least one connection on node1"); -cmp_ok($node2_occurrences, '>', 1, - "received at least one connection on node2"); -cmp_ok($node3_occurrences, '>', 1, - "received at least one connection on node3"); -is($total_occurrences, 50, "received 50 connections across all nodes"); - -$node1->stop(); -$node2->stop(); - -# load_balance_hosts=disable should continue trying hosts until it finds a -# working one. -$node3->connect_ok( - "host=pg-loadbalancetest port=$port load_balance_hosts=disable", - "load_balance_hosts=disable continues until it connects to a working node", - sql => "SELECT 'connect3'", - log_like => [qr/statement: SELECT 'connect3'/]); - -# Also with load_balance_hosts=random we continue to the next nodes if previous -# ones are down. Connect a few times to make sure it's not just lucky. -foreach my $i (1 .. 5) -{ - $node3->connect_ok( - "host=pg-loadbalancetest port=$port load_balance_hosts=random", - "load_balance_hosts=random continues until it connects to a working node", - sql => "SELECT 'connect4'", - log_like => [qr/statement: SELECT 'connect4'/]); -} - -done_testing(); diff --git a/src/interfaces/libpq/t/005_negotiate_encryption.pl b/src/interfaces/libpq/t/005_negotiate_encryption.pl deleted file mode 100644 index 18c100fb11..0000000000 --- a/src/interfaces/libpq/t/005_negotiate_encryption.pl +++ /dev/null @@ -1,831 +0,0 @@ - -# Copyright (c) 2021-2026, PostgreSQL Global Development Group - -# OVERVIEW -# -------- -# -# Test negotiation of SSL and GSSAPI encryption -# -# We test all combinations of: -# -# - all the libpq client options that affect the protocol negotiations -# (gssencmode, sslmode, sslnegotiation) -# - server accepting or rejecting the authentication due to -# pg_hba.conf entries -# - SSL and GSS enabled/disabled in the server -# -# That's a lot of combinations, so we use a table-driven approach. -# Each combination is represented by a line in a table. The line lists -# the options specifying the test case, and an expected outcome. The -# expected outcome includes whether the connection succeeds or fails, -# and whether it uses SSL, GSS or no encryption. It also includes a -# condensed trace of what steps were taken during the negotiation. -# That can catch cases like useless retries, or if the encryption -# methods are attempted in wrong order, even when it doesn't affect -# the final outcome. -# -# TEST TABLE FORMAT -# ----------------- -# -# Example of the test table format: -# -# # USER GSSENCMODE SSLMODE EVENTS -> OUTCOME -# testuser disable allow connect, authok -> plain -# . . prefer connect, sslaccept, authok -> ssl -# testuser require * connect, gssreject -> fail -# -# USER, GSSENCMODE and SSLMODE fields are the libpq 'user', -# 'gssencmode' and 'sslmode' options used in the test. As a shorthand, -# a single dot ('.') can be used in the USER, GSSENCMODE, and SSLMODE -# fields, to indicate "same as on previous line". A '*' can be used -# as a wildcard; it is expanded to mean all possible values of that -# field. -# -# The EVENTS field is a condensed trace of expected steps during the -# negotiation: -# -# connect: a TCP connection was established -# reconnect: TCP connection was disconnected, and a new one was established -# sslaccept: client requested SSL encryption and server accepted it -# sslreject: client requested SSL encryption but server rejected it -# gssaccept: client requested GSSAPI encryption and server accepted it -# gssreject: client requested GSSAPI encryption but server rejected it -# authok: client sent startup packet and authentication was performed successfully -# authfail: client sent startup packet but server rejected the authentication -# -# The event trace can be used to verify that the client negotiated the -# connection properly in more detail than just by looking at the -# outcome. For example, if the client opens spurious extra TCP -# connections, that would show up in the EVENTS. -# -# The OUTCOME field indicates the expected result of the test: -# -# plain: an unencrypted connection was established -# ssl: SSL connection was established -# gss: GSSAPI encrypted connection was established -# fail: the connection attempt failed -# -# Empty lines are ignored. '#' can be used to mark the rest of the -# line as a comment. - -use strict; -use warnings FATAL => 'all'; -use PostgreSQL::Test::Utils; -use PostgreSQL::Test::Cluster; -use PostgreSQL::Test::Kerberos; -use File::Basename; -use File::Copy; -use Test::More; - -if (!$ENV{PG_TEST_EXTRA} || $ENV{PG_TEST_EXTRA} !~ /\blibpq_encryption\b/) -{ - plan skip_all => - 'Potentially unsafe test libpq_encryption not enabled in PG_TEST_EXTRA'; -} - -# Only run the GSSAPI tests when compiled with GSSAPI support and -# PG_TEST_EXTRA includes 'kerberos' -my $gss_supported = $ENV{with_gssapi} eq 'yes'; -my $kerberos_enabled = - $ENV{PG_TEST_EXTRA} && $ENV{PG_TEST_EXTRA} =~ /\bkerberos\b/; -my $ssl_supported = $ENV{with_ssl} eq 'openssl'; - -### -### Prepare test server for GSSAPI and SSL authentication, with a few -### different test users and helper functions. We don't actually -### enable SSL and kerberos in the server yet, we will do that later. -### - -my $host = 'enc-test-localhost.postgresql.example.com'; -my $hostaddr = '127.0.0.1'; -my $servercidr = '127.0.0.1/32'; - -my $node = PostgreSQL::Test::Cluster->new('node'); -$node->init; -$node->append_conf( - 'postgresql.conf', qq{ -listen_addresses = '$hostaddr' - -# Capturing the EVENTS that occur during tests requires these settings -log_connections = 'receipt,authentication,authorization' -log_disconnections = on -trace_connection_negotiation = on -lc_messages = 'C' -}); -my $pgdata = $node->data_dir; - -my $dbname = 'postgres'; -my $username = 'enctest'; -my $application = '001_negotiate_encryption.pl'; - -my $gssuser_password = 'secret1'; - -my $krb; - -if ($gss_supported != 0 && $kerberos_enabled != 0) -{ - note "setting up Kerberos"; - - my $realm = 'EXAMPLE.COM'; - $krb = PostgreSQL::Test::Kerberos->new($host, $hostaddr, $realm); - $node->append_conf('postgresql.conf', - "krb_server_keyfile = '$krb->{keytab}'\n"); -} - -if ($ssl_supported != 0) -{ - my $certdir = dirname(__FILE__) . "/../../../test/ssl/ssl"; - - copy "$certdir/server-cn-only.crt", "$pgdata/server.crt" - || die "copying server.crt: $!"; - copy "$certdir/server-cn-only.key", "$pgdata/server.key" - || die "copying server.key: $!"; - chmod(0600, "$pgdata/server.key") - or die "failed to change permissions on server keys: $!"; - - # Start with SSL disabled. - $node->append_conf('postgresql.conf', "ssl = off\n"); -} - -$node->start; - -# Check if the extension injection_points is available, as it may be -# possible that this script is run with installcheck, where the module -# would not be installed by default. -my $injection_points_supported = $node->check_extension('injection_points'); - -$node->safe_psql('postgres', 'CREATE USER localuser;'); -$node->safe_psql('postgres', 'CREATE USER testuser;'); -$node->safe_psql('postgres', 'CREATE USER ssluser;'); -$node->safe_psql('postgres', 'CREATE USER nossluser;'); -$node->safe_psql('postgres', 'CREATE USER gssuser;'); -$node->safe_psql('postgres', 'CREATE USER nogssuser;'); -if ($injection_points_supported != 0) -{ - $node->safe_psql('postgres', 'CREATE EXTENSION injection_points;'); -} - -my $unixdir = $node->safe_psql('postgres', 'SHOW unix_socket_directories;'); -chomp($unixdir); - -# Helper function that returns the encryption method in use in the -# connection. -$node->safe_psql( - 'postgres', q{ -CREATE FUNCTION current_enc() RETURNS text LANGUAGE plpgsql AS $$ -DECLARE - ssl_in_use bool; - gss_in_use bool; -BEGIN - ssl_in_use = (SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()); - gss_in_use = (SELECT encrypted FROM pg_stat_gssapi WHERE pid = pg_backend_pid()); - - raise log 'ssl % gss %', ssl_in_use, gss_in_use; - - IF ssl_in_use AND gss_in_use THEN - RETURN 'ssl+gss'; -- shouldn't happen - ELSIF ssl_in_use THEN - RETURN 'ssl'; - ELSIF gss_in_use THEN - RETURN 'gss'; - ELSE - RETURN 'plain'; - END IF; -END; -$$; -}); - -# Only accept SSL connections from $servercidr. Our tests don't depend on this -# but seems best to keep it as narrow as possible for security reasons. -open my $hba, '>', "$pgdata/pg_hba.conf" or die $!; -print $hba qq{ -# TYPE DATABASE USER ADDRESS METHOD OPTIONS -local postgres localuser trust -host postgres testuser $servercidr trust -hostnossl postgres nossluser $servercidr trust -hostnogssenc postgres nogssuser $servercidr trust -}; - -print $hba qq{ -hostssl postgres ssluser $servercidr trust -} if ($ssl_supported != 0); - -print $hba qq{ -hostgssenc postgres gssuser $servercidr trust -} if ($gss_supported != 0 && $kerberos_enabled != 0); -close $hba; -$node->reload; - -# Ok, all prepared. Run the tests. - -my @all_test_users = - ('testuser', 'ssluser', 'nossluser', 'gssuser', 'nogssuser'); -my @all_gssencmodes = ('disable', 'prefer', 'require'); -my @all_sslmodes = ('disable', 'allow', 'prefer', 'require'); -my @all_sslnegotiations = ('postgres', 'direct'); - -### -### Run tests with GSS and SSL disabled in the server -### -my $test_table; -if ($ssl_supported) -{ - $test_table = q{ -# USER GSSENCMODE SSLMODE SSLNEGOTIATION EVENTS -> OUTCOME -testuser disable disable postgres connect, authok -> plain -. . allow postgres connect, authok -> plain -. . prefer postgres connect, sslreject, authok -> plain -. . require postgres connect, sslreject -> fail -. . . direct connect, directsslreject -> fail -. prefer disable postgres connect, authok -> plain -. . allow postgres connect, authok -> plain -. . prefer postgres connect, sslreject, authok -> plain -. . require postgres connect, sslreject -> fail -. . . direct connect, directsslreject -> fail - -# sslnegotiation=direct is not accepted unless sslmode=require or stronger -* * disable direct - -> fail -* * allow direct - -> fail -* * prefer direct - -> fail -}; -} -else -{ - # Compiled without SSL support - $test_table = q{ -# USER GSSENCMODE SSLMODE SSLNEGOTIATION EVENTS -> OUTCOME -testuser disable disable postgres connect, authok -> plain -. . allow postgres connect, authok -> plain -. . prefer postgres connect, authok -> plain -. prefer disable postgres connect, authok -> plain -. . allow postgres connect, authok -> plain -. . prefer postgres connect, authok -> plain - -# Without SSL support, sslmode=require and sslnegotiation=direct are -# not accepted at all -* * require * - -> fail -* * * direct - -> fail - }; -} - -# All attempts with gssencmode=require fail without connecting because -# no credential cache has been configured in the client. (Or if GSS -# support is not compiled in, they will fail because of that.) -$test_table .= q{ -testuser require * * - -> fail -}; - -note("Running tests with SSL and GSS disabled in the server"); -test_matrix($node, - ['testuser'], \@all_gssencmodes, \@all_sslmodes, \@all_sslnegotiations, - parse_table($test_table)); - - -### -### Run tests with GSS disabled and SSL enabled in the server -### -SKIP: -{ - skip "SSL not supported by this build" if $ssl_supported == 0; - - $test_table = q{ -# USER GSSENCMODE SSLMODE SSLNEGOTIATION EVENTS -> OUTCOME -testuser disable disable postgres connect, authok -> plain -. . allow postgres connect, authok -> plain -. . prefer postgres connect, sslaccept, authok -> ssl -. . require postgres connect, sslaccept, authok -> ssl -. . . direct connect, directsslaccept, authok -> ssl -ssluser . disable postgres connect, authfail -> fail -. . allow postgres connect, authfail, reconnect, sslaccept, authok -> ssl -. . prefer postgres connect, sslaccept, authok -> ssl -. . require postgres connect, sslaccept, authok -> ssl -. . . direct connect, directsslaccept, authok -> ssl -nossluser . disable postgres connect, authok -> plain -. . allow postgres connect, authok -> plain -. . prefer postgres connect, sslaccept, authfail, reconnect, authok -> plain -. . require postgres connect, sslaccept, authfail -> fail -. . require direct connect, directsslaccept, authfail -> fail - -# sslnegotiation=direct is not accepted unless sslmode=require or stronger -* * disable direct - -> fail -* * allow direct - -> fail -* * prefer direct - -> fail -}; - - # Enable SSL in the server - $node->adjust_conf('postgresql.conf', 'ssl', 'on'); - $node->reload; - - note("Running tests with SSL enabled in server"); - test_matrix($node, [ 'testuser', 'ssluser', 'nossluser' ], - ['disable'], \@all_sslmodes, \@all_sslnegotiations, - parse_table($test_table)); - - if ($injection_points_supported != 0) - { - $node->safe_psql( - 'postgres', - "SELECT injection_points_attach('backend-initialize', 'error');", - connstr => "user=localuser host=$unixdir"); - connect_test( - $node, - "user=testuser sslmode=prefer", - 'connect, backenderror -> fail'); - $node->restart; - - $node->safe_psql( - 'postgres', - "SELECT injection_points_attach('backend-initialize-v2-error', 'error');", - connstr => "user=localuser host=$unixdir"); - connect_test( - $node, - "user=testuser sslmode=prefer", - 'connect, v2error -> fail'); - $node->restart; - - $node->safe_psql( - 'postgres', - "SELECT injection_points_attach('backend-ssl-startup', 'error');", - connstr => "user=localuser host=$unixdir"); - connect_test( - $node, - "user=testuser sslmode=prefer", - 'connect, sslaccept, backenderror, reconnect, authok -> plain'); - $node->restart; - } - - # Disable SSL again - $node->adjust_conf('postgresql.conf', 'ssl', 'off'); - $node->reload; -} - -### -### Run tests with GSS enabled, SSL disabled in the server -### -SKIP: -{ - skip "GSSAPI/Kerberos not supported by this build" if $gss_supported == 0; - skip "kerberos not enabled in PG_TEST_EXTRA" if $kerberos_enabled == 0; - - $krb->create_principal('gssuser', $gssuser_password); - $krb->create_ticket('gssuser', $gssuser_password); - - $test_table = q{ -# USER GSSENCMODE SSLMODE SSLNEGOTIATION EVENTS -> OUTCOME -testuser disable disable postgres connect, authok -> plain -. . allow postgres connect, authok -> plain -. . prefer postgres connect, sslreject, authok -> plain -. . require postgres connect, sslreject -> fail -. . . direct connect, directsslreject -> fail -. prefer * postgres connect, gssaccept, authok -> gss -. prefer require direct connect, gssaccept, authok -> gss -. require * postgres connect, gssaccept, authok -> gss -. . require direct connect, gssaccept, authok -> gss - -gssuser disable disable postgres connect, authfail -> fail -. . allow postgres connect, authfail, reconnect, sslreject -> fail -. . prefer postgres connect, sslreject, authfail -> fail -. . require postgres connect, sslreject -> fail -. . . direct connect, directsslreject -> fail -. prefer * postgres connect, gssaccept, authok -> gss -. prefer require direct connect, gssaccept, authok -> gss -. require * postgres connect, gssaccept, authok -> gss -. . require direct connect, gssaccept, authok -> gss - -nogssuser disable disable postgres connect, authok -> plain -. . allow postgres connect, authok -> plain -. . prefer postgres connect, sslreject, authok -> plain -. . require postgres connect, sslreject -> fail -. . . direct connect, directsslreject -> fail -. prefer disable postgres connect, gssaccept, authfail, reconnect, authok -> plain -. . allow postgres connect, gssaccept, authfail, reconnect, authok -> plain -. . prefer postgres connect, gssaccept, authfail, reconnect, sslreject, authok -> plain -. . require postgres connect, gssaccept, authfail, reconnect, sslreject -> fail -. . . direct connect, gssaccept, authfail, reconnect, directsslreject -> fail -. require disable postgres connect, gssaccept, authfail -> fail -. . allow postgres connect, gssaccept, authfail -> fail -. . prefer postgres connect, gssaccept, authfail -> fail -. . require postgres connect, gssaccept, authfail -> fail # If both GSSAPI and sslmode are required, and GSS is not available -> fail -. . . direct connect, gssaccept, authfail -> fail # If both GSSAPI and sslmode are required, and GSS is not available -> fail - -# sslnegotiation=direct is not accepted unless sslmode=require or stronger -* * disable direct - -> fail -* * allow direct - -> fail -* * prefer direct - -> fail - }; - - # The expected events and outcomes above assume that SSL support - # is enabled. When libpq is compiled without SSL support, all - # attempts to connect with sslmode=require or - # sslnegotiation=direct would fail immediately without even - # connecting to the server. Skip those, because we tested them - # earlier already. - my ($sslmodes, $sslnegotiations); - if ($ssl_supported != 0) - { - ($sslmodes, $sslnegotiations) = - (\@all_sslmodes, \@all_sslnegotiations); - } - else - { - ($sslmodes, $sslnegotiations) = (['disable'], ['postgres']); - } - - note("Running tests with GSS enabled in server"); - test_matrix($node, [ 'testuser', 'gssuser', 'nogssuser' ], - \@all_gssencmodes, $sslmodes, $sslnegotiations, - parse_table($test_table)); - - if ($injection_points_supported != 0) - { - $node->safe_psql( - 'postgres', - "SELECT injection_points_attach('backend-initialize', 'error');", - connstr => "user=localuser host=$unixdir"); - connect_test( - $node, - "user=testuser gssencmode=prefer sslmode=disable", - 'connect, backenderror, reconnect, backenderror -> fail'); - $node->restart; - - $node->safe_psql( - 'postgres', - "SELECT injection_points_attach('backend-initialize-v2-error', 'error');", - connstr => "user=localuser host=$unixdir"); - connect_test( - $node, - "user=testuser gssencmode=prefer sslmode=disable", - 'connect, v2error, reconnect, v2error -> fail'); - $node->restart; - - $node->safe_psql( - 'postgres', - "SELECT injection_points_attach('backend-gssapi-startup', 'error');", - connstr => "user=localuser host=$unixdir"); - connect_test( - $node, - "user=testuser gssencmode=prefer sslmode=disable", - 'connect, gssaccept, backenderror, reconnect, authok -> plain'); - $node->restart; - } -} - -### -### Tests with both GSS and SSL enabled in the server -### -SKIP: -{ - skip "SSL not supported by this build" if $ssl_supported == 0; - skip "GSSAPI/Kerberos not supported by this build" if $gss_supported == 0; - skip "kerberos not enabled in PG_TEST_EXTRA" if $kerberos_enabled == 0; - - # Sanity check that GSSAPI is still enabled from previous test. - connect_test( - $node, - 'user=testuser gssencmode=prefer sslmode=prefer', - 'connect, gssaccept, authok -> gss'); - - # Enable SSL - $node->adjust_conf('postgresql.conf', 'ssl', 'on'); - $node->reload; - - $test_table = q{ -# USER GSSENCMODE SSLMODE SSLNEGOTIATION EVENTS -> OUTCOME -testuser disable disable postgres connect, authok -> plain -. . allow postgres connect, authok -> plain -. . prefer postgres connect, sslaccept, authok -> ssl -. . require postgres connect, sslaccept, authok -> ssl -. . . direct connect, directsslaccept, authok -> ssl -. prefer disable postgres connect, gssaccept, authok -> gss -. . allow postgres connect, gssaccept, authok -> gss -. . prefer postgres connect, gssaccept, authok -> gss -. . require postgres connect, gssaccept, authok -> gss # If both GSS and SSL is possible, GSS is chosen over SSL, even if sslmode=require -. . . direct connect, gssaccept, authok -> gss -. require disable postgres connect, gssaccept, authok -> gss -. . allow postgres connect, gssaccept, authok -> gss -. . prefer postgres connect, gssaccept, authok -> gss -. . require postgres connect, gssaccept, authok -> gss # If both GSS and SSL is possible, GSS is chosen over SSL, even if sslmode=require -. . . direct connect, gssaccept, authok -> gss - -gssuser disable disable postgres connect, authfail -> fail -. . allow postgres connect, authfail, reconnect, sslaccept, authfail -> fail -. . prefer postgres connect, sslaccept, authfail, reconnect, authfail -> fail -. . require postgres connect, sslaccept, authfail -> fail -. . . direct connect, directsslaccept, authfail -> fail -. prefer disable postgres connect, gssaccept, authok -> gss -. . allow postgres connect, gssaccept, authok -> gss -. . prefer postgres connect, gssaccept, authok -> gss -. . require postgres connect, gssaccept, authok -> gss # GSS is chosen over SSL, even though sslmode=require -. . . direct connect, gssaccept, authok -> gss -. require disable postgres connect, gssaccept, authok -> gss -. . allow postgres connect, gssaccept, authok -> gss -. . prefer postgres connect, gssaccept, authok -> gss -. . require postgres connect, gssaccept, authok -> gss # If both GSS and SSL is possible, GSS is chosen over SSL, even if sslmode=require -. . . direct connect, gssaccept, authok -> gss - -ssluser disable disable postgres connect, authfail -> fail -. . allow postgres connect, authfail, reconnect, sslaccept, authok -> ssl -. . prefer postgres connect, sslaccept, authok -> ssl -. . require postgres connect, sslaccept, authok -> ssl -. . . direct connect, directsslaccept, authok -> ssl -. prefer disable postgres connect, gssaccept, authfail, reconnect, authfail -> fail -. . allow postgres connect, gssaccept, authfail, reconnect, authfail, reconnect, sslaccept, authok -> ssl -. . prefer postgres connect, gssaccept, authfail, reconnect, sslaccept, authok -> ssl -. . require postgres connect, gssaccept, authfail, reconnect, sslaccept, authok -> ssl -. . . direct connect, gssaccept, authfail, reconnect, directsslaccept, authok -> ssl -. require disable postgres connect, gssaccept, authfail -> fail -. . allow postgres connect, gssaccept, authfail -> fail -. . prefer postgres connect, gssaccept, authfail -> fail -. . require postgres connect, gssaccept, authfail -> fail # If both GSS and SSL are required, the sslmode=require is effectively ignored and GSS is required -. . . direct connect, gssaccept, authfail -> fail - -nogssuser disable disable postgres connect, authok -> plain -. . allow postgres connect, authok -> plain -. . prefer postgres connect, sslaccept, authok -> ssl -. . require postgres connect, sslaccept, authok -> ssl -. . . direct connect, directsslaccept, authok -> ssl -. prefer disable postgres connect, gssaccept, authfail, reconnect, authok -> plain -. . allow postgres connect, gssaccept, authfail, reconnect, authok -> plain -. . prefer postgres connect, gssaccept, authfail, reconnect, sslaccept, authok -> ssl -. . require postgres connect, gssaccept, authfail, reconnect, sslaccept, authok -> ssl -. . . direct connect, gssaccept, authfail, reconnect, directsslaccept, authok -> ssl -. require disable postgres connect, gssaccept, authfail -> fail -. . allow postgres connect, gssaccept, authfail -> fail -. . prefer postgres connect, gssaccept, authfail -> fail -. . require postgres connect, gssaccept, authfail -> fail # If both GSS and SSL are required, the sslmode=require is effectively ignored and GSS is required -. . . direct connect, gssaccept, authfail -> fail - -nossluser disable disable postgres connect, authok -> plain -. . allow postgres connect, authok -> plain -. . prefer postgres connect, sslaccept, authfail, reconnect, authok -> plain -. . require postgres connect, sslaccept, authfail -> fail -. . . direct connect, directsslaccept, authfail -> fail -. prefer * postgres connect, gssaccept, authok -> gss -. . require direct connect, gssaccept, authok -> gss -. require * postgres connect, gssaccept, authok -> gss -. . require direct connect, gssaccept, authok -> gss - -# sslnegotiation=direct is not accepted unless sslmode=require or stronger -* * disable direct - -> fail -* * allow direct - -> fail -* * prefer direct - -> fail - }; - - note("Running tests with both GSS and SSL enabled in server"); - test_matrix( - $node, - [ 'testuser', 'gssuser', 'ssluser', 'nogssuser', 'nossluser' ], - \@all_gssencmodes, - \@all_sslmodes, - \@all_sslnegotiations, - parse_table($test_table)); -} - -### -### Test negotiation over unix domain sockets. -### -SKIP: -{ - skip "Unix domain sockets not supported" unless ($unixdir ne ""); - - # libpq doesn't attempt SSL or GSSAPI over Unix domain - # sockets. The server would reject them too. - connect_test( - $node, - "user=localuser gssencmode=prefer sslmode=prefer host=$unixdir", - 'connect, authok -> plain'); - connect_test($node, - "user=localuser gssencmode=require sslmode=prefer host=$unixdir", - '- -> fail'); -} - -done_testing(); - - -### Helper functions - -# Test the cube of parameters: user, gssencmode, sslmode, and sslnegotiation -sub test_matrix -{ - local $Test::Builder::Level = $Test::Builder::Level + 1; - - my ($pg_node, $test_users, $gssencmodes, $sslmodes, $sslnegotiations, - %expected) - = @_; - - foreach my $test_user (@{$test_users}) - { - foreach my $gssencmode (@{$gssencmodes}) - { - foreach my $client_mode (@{$sslmodes}) - { - # sslnegotiation only makes a difference if SSL is enabled. This saves a few combinations. - my ($key, $expected_events); - foreach my $negotiation (@{$sslnegotiations}) - { - $key = "$test_user $gssencmode $client_mode $negotiation"; - $expected_events = $expected{$key}; - if (!defined($expected_events)) - { - $expected_events = - ""; - } - connect_test( - $pg_node, - "user=$test_user gssencmode=$gssencmode sslmode=$client_mode sslnegotiation=$negotiation", - $expected_events); - } - } - } - } -} - -# Try to establish a connection to the server using libpq. Verify the -# negotiation events and outcome. -sub connect_test -{ - local $Test::Builder::Level = $Test::Builder::Level + 1; - - my ($node, $connstr, $expected_events_and_outcome) = @_; - - my $test_name = " '$connstr' -> $expected_events_and_outcome"; - - my $connstr_full = ""; - $connstr_full .= "dbname=postgres " unless $connstr =~ m/dbname=/; - $connstr_full .= "host=$host hostaddr=$hostaddr " - unless $connstr =~ m/host=/; - $connstr_full .= $connstr; - - # Get the current size of the logfile before running the test. - # After the test, we can then check just the new lines that have - # appeared. (This is the same approach that the $node->log_contains - # function uses). - my $log_location = -s $node->logfile; - - # XXX: Pass command with -c, because I saw intermittent test - # failures like this: - # - # ack Broken pipe: write( 13, 'SELECT current_enc()' ) at /usr/local/lib/perl5/site_perl/IPC/Run/IO.pm line 550. - # - # I think that happens if the connection fails before we write the - # query to its stdin. This test gets a lot of connection failures - # on purpose. - my ($ret, $stdout, $stderr) = $node->psql( - 'postgres', - '', - extra_params => [ - '--no-password', '--command' => 'SELECT current_enc()', - ], - connstr => "$connstr_full", - on_error_stop => 0); - - my $outcome = $ret == 0 ? $stdout : 'fail'; - - # Parse the EVENTS from the log file. - my $log_contents = - PostgreSQL::Test::Utils::slurp_file($node->logfile, $log_location); - my @events = parse_log_events($log_contents); - - # Check that the events and outcome match the expected events and - # outcome - my $events_and_outcome = join(', ', @events) . " -> $outcome"; - is($events_and_outcome, $expected_events_and_outcome, $test_name) - or diag("$stderr"); -} - -# Parse a test table. See comment at top of the file for the format. -sub parse_table -{ - my ($table) = @_; - my @lines = split /\n/, $table; - - my %expected; - - my ($user, $gssencmode, $sslmode, $sslnegotiation); - foreach my $line (@lines) - { - - # Trim comments - $line =~ s/#.*$//; - - # Trim whitespace at beginning and end - $line =~ s/^\s+//; - $line =~ s/\s+$//; - - # Ignore empty lines (includes comment-only lines) - next if $line eq ''; - - $line =~ m/^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S.*)\s*->\s*(\S+)\s*$/ - or die "could not parse line \"$line\""; - $user = $1 unless $1 eq "."; - $gssencmode = $2 unless $2 eq "."; - $sslmode = $3 unless $3 eq "."; - $sslnegotiation = $4 unless $4 eq "."; - - # Normalize the whitespace in the "EVENTS -> OUTCOME" part - my @events = split /,\s*/, $5; - my $outcome = $6; - my $events_str = join(', ', @events); - $events_str =~ s/\s+$//; # trim whitespace - my $events_and_outcome = "$events_str -> $outcome"; - - my %expanded = - expand_expected_line($user, $gssencmode, $sslmode, $sslnegotiation, - $events_and_outcome); - %expected = (%expected, %expanded); - } - return %expected; -} - -# Expand wildcards on a test table line -sub expand_expected_line -{ - my ($user, $gssencmode, $sslmode, $sslnegotiation, $expected) = @_; - - my %result; - if ($user eq '*') - { - foreach my $x (@all_test_users) - { - %result = ( - %result, - expand_expected_line( - $x, $gssencmode, $sslmode, $sslnegotiation, $expected)); - } - } - elsif ($gssencmode eq '*') - { - foreach my $x (@all_gssencmodes) - { - %result = ( - %result, - expand_expected_line( - $user, $x, $sslmode, $sslnegotiation, $expected)); - } - } - elsif ($sslmode eq '*') - { - foreach my $x (@all_sslmodes) - { - %result = ( - %result, - expand_expected_line( - $user, $gssencmode, $x, $sslnegotiation, $expected)); - } - } - elsif ($sslnegotiation eq '*') - { - foreach my $x (@all_sslnegotiations) - { - %result = ( - %result, - expand_expected_line( - $user, $gssencmode, $sslmode, $x, $expected)); - } - } - else - { - $result{"$user $gssencmode $sslmode $sslnegotiation"} = $expected; - } - return %result; -} - -# Scrape the server log for the negotiation events that match the -# EVENTS field of the test tables. -sub parse_log_events -{ - my ($log_contents) = (@_); - - my @events = (); - - my @lines = split /\n/, $log_contents; - foreach my $line (@lines) - { - push @events, "reconnect" - if $line =~ /connection received/ && scalar(@events) > 0; - push @events, "connect" - if $line =~ /connection received/ && scalar(@events) == 0; - push @events, "sslaccept" if $line =~ /SSLRequest accepted/; - push @events, "sslreject" if $line =~ /SSLRequest rejected/; - push @events, "directsslaccept" - if $line =~ /direct SSL connection accepted/; - push @events, "directsslreject" - if $line =~ /direct SSL connection rejected/; - push @events, "gssaccept" if $line =~ /GSSENCRequest accepted/; - push @events, "gssreject" if $line =~ /GSSENCRequest rejected/; - push @events, "authfail" if $line =~ /no pg_hba.conf entry/; - push @events, "authok" if $line =~ /connection authenticated/; - push @events, "backenderror" - if $line =~ /error triggered for injection point backend-/; - push @events, "v2error" - if $line =~ /protocol version 2 error triggered/; - } - - # No events at all is represented by "-" - if (scalar @events == 0) - { - push @events, "-"; - } - - return @events; -} diff --git a/src/interfaces/libpq/t/006_service.pl b/src/interfaces/libpq/t/006_service.pl deleted file mode 100644 index 8e29880731..0000000000 --- a/src/interfaces/libpq/t/006_service.pl +++ /dev/null @@ -1,246 +0,0 @@ -# Copyright (c) 2025-2026, PostgreSQL Global Development Group -use strict; -use warnings FATAL => 'all'; -use File::Copy; -use PostgreSQL::Test::Utils; -use PostgreSQL::Test::Cluster; -use Test::More; - -# This tests scenarios related to the service name and the service file, -# for the connection options and their environment variables. - -my $node = PostgreSQL::Test::Cluster->new('node'); -$node->init; -$node->start; - -# Set up a dummy node used for the connection tests, but do not start it. -# This ensures that the environment variables used for the connection do -# not interfere with the connection attempts, and that the service file's -# contents are used. -my $dummy_node = PostgreSQL::Test::Cluster->new('dummy_node'); -$dummy_node->init; - -my $td = PostgreSQL::Test::Utils::tempdir; - -# Create the set of service files used in the tests. -# File that includes a valid service name, and uses a decomposed connection -# string for its contents, split on spaces. -my $srvfile_valid = "$td/pg_service_valid.conf"; -append_to_file($srvfile_valid, "[my_srv]\n"); -foreach my $param (split(/\s+/, $node->connstr)) -{ - append_to_file($srvfile_valid, $param . "\n"); -} - -# File defined with no contents, used as default value for PGSERVICEFILE, -# so as no lookup is attempted in the user's home directory. -my $srvfile_empty = "$td/pg_service_empty.conf"; -append_to_file($srvfile_empty, ''); - -# Default service file in PGSYSCONFDIR. -my $srvfile_default = "$td/pg_service.conf"; - -# Missing service file. -my $srvfile_missing = "$td/pg_service_missing.conf"; - -# Service file with nested "service" defined. -my $srvfile_nested = "$td/pg_service_nested.conf"; -copy($srvfile_valid, $srvfile_nested) - or die "Could not copy $srvfile_valid to $srvfile_nested: $!"; -append_to_file($srvfile_nested, "service=invalid_srv\n"); - -# Service file with nested "servicefile" defined. -my $srvfile_nested_2 = "$td/pg_service_nested_2.conf"; -copy($srvfile_valid, $srvfile_nested_2) - or die "Could not copy $srvfile_valid to $srvfile_nested_2: $!"; -append_to_file($srvfile_nested_2, 'servicefile=' . $srvfile_default . "\n"); - -# Set the fallback directory lookup of the service file to the temporary -# directory of this test. PGSYSCONFDIR is used if the service file -# defined in PGSERVICEFILE cannot be found, or when a service file is -# found but not the service name. -local $ENV{PGSYSCONFDIR} = $td; -# Force PGSERVICEFILE to a default location, so as this test never -# tries to look at a home directory. This value needs to remain -# at the top of this script before running any tests, and should never -# be changed. -local $ENV{PGSERVICEFILE} = "$srvfile_empty"; - -# Checks combinations of service name and a valid service file. -{ - local $ENV{PGSERVICEFILE} = $srvfile_valid; - $dummy_node->connect_ok( - 'service=my_srv', - 'connection with correct "service" string and PGSERVICEFILE', - sql => "SELECT 'connect1_1'", - expected_stdout => qr/connect1_1/); - - $dummy_node->connect_ok( - 'postgres://?service=my_srv', - 'connection with correct "service" URI and PGSERVICEFILE', - sql => "SELECT 'connect1_2'", - expected_stdout => qr/connect1_2/); - - $dummy_node->connect_fails( - 'service=undefined-service', - 'connection with incorrect "service" string and PGSERVICEFILE', - expected_stderr => - qr/definition of service "undefined-service" not found/); - - local $ENV{PGSERVICE} = 'my_srv'; - $dummy_node->connect_ok( - '', - 'connection with correct PGSERVICE and PGSERVICEFILE', - sql => "SELECT 'connect1_3'", - expected_stdout => qr/connect1_3/); - - local $ENV{PGSERVICE} = 'undefined-service'; - $dummy_node->connect_fails( - '', - 'connection with incorrect PGSERVICE and PGSERVICEFILE', - expected_stdout => - qr/definition of service "undefined-service" not found/); -} - -# Checks case of incorrect service file. -{ - local $ENV{PGSERVICEFILE} = $srvfile_missing; - $dummy_node->connect_fails( - 'service=my_srv', - 'connection with correct "service" string and incorrect PGSERVICEFILE', - expected_stderr => - qr/service file ".*pg_service_missing.conf" not found/); -} - -# Checks case of service file named "pg_service.conf" in PGSYSCONFDIR. -{ - # Create copy of valid file - my $srvfile_default = "$td/pg_service.conf"; - copy($srvfile_valid, $srvfile_default); - - $dummy_node->connect_ok( - 'service=my_srv', - 'connection with correct "service" string and pg_service.conf', - sql => "SELECT 'connect2_1'", - expected_stdout => qr/connect2_1/); - - $dummy_node->connect_ok( - 'postgres://?service=my_srv', - 'connection with correct "service" URI and default pg_service.conf', - sql => "SELECT 'connect2_2'", - expected_stdout => qr/connect2_2/); - - $dummy_node->connect_fails( - 'service=undefined-service', - 'connection with incorrect "service" string and default pg_service.conf', - expected_stderr => - qr/definition of service "undefined-service" not found/); - - local $ENV{PGSERVICE} = 'my_srv'; - $dummy_node->connect_ok( - '', - 'connection with correct PGSERVICE and default pg_service.conf', - sql => "SELECT 'connect2_3'", - expected_stdout => qr/connect2_3/); - - my $srvfile_empty_win_cared = $srvfile_empty; - $srvfile_empty_win_cared =~ s/\\/\\\\/g; - $dummy_node->connect_ok( - q{service=my_srv servicefile='} . $srvfile_empty_win_cared . q{'}, - 'SERVICEFILE updated when service is found in default pg_service.conf', - sql => '\echo :SERVICEFILE', - expected_stdout => qr/^\Q$srvfile_default\E$/); - - local $ENV{PGSERVICE} = 'undefined-service'; - $dummy_node->connect_fails( - '', - 'connection with incorrect PGSERVICE and default pg_service.conf', - expected_stdout => - qr/definition of service "undefined-service" not found/); - - # Remove default pg_service.conf. - unlink($srvfile_default); -} - -# Checks nested service file contents. -{ - local $ENV{PGSERVICEFILE} = $srvfile_nested; - - $dummy_node->connect_fails( - 'service=my_srv', - 'connection with "service" in nested service file', - expected_stderr => - qr/nested "service" specifications not supported in service file/); - - local $ENV{PGSERVICEFILE} = $srvfile_nested_2; - - $dummy_node->connect_fails( - 'service=my_srv', - 'connection with "servicefile" in nested service file', - expected_stderr => - qr/nested "servicefile" specifications not supported in service file/ - ); -} - -# Properly escape backslashes in the path, to ensure the generation of -# correct connection strings. -my $srvfile_win_cared = $srvfile_valid; -$srvfile_win_cared =~ s/\\/\\\\/g; - -# Checks that the "servicefile" option works as expected -{ - $dummy_node->connect_ok( - q{service=my_srv servicefile='} . $srvfile_win_cared . q{'}, - 'connection with valid servicefile in connection string', - sql => "SELECT 'connect3_1'", - expected_stdout => qr/connect3_1/); - - # Encode slashes and backslash - my $encoded_srvfile = $srvfile_valid =~ s{([\\/])}{ - $1 eq '/' ? '%2F' : '%5C' - }ger; - - # Additionally encode a colon in servicefile path of Windows - $encoded_srvfile =~ s/:/%3A/g; - - $dummy_node->connect_ok( - 'postgresql:///?service=my_srv&servicefile=' . $encoded_srvfile, - 'connection with valid servicefile in URI', - sql => "SELECT 'connect3_2'", - expected_stdout => qr/connect3_2/); - - local $ENV{PGSERVICE} = 'my_srv'; - $dummy_node->connect_ok( - q{servicefile='} . $srvfile_win_cared . q{'}, - 'connection with PGSERVICE and servicefile in connection string', - sql => "SELECT 'connect3_3'", - expected_stdout => qr/connect3_3/); - - $dummy_node->connect_ok( - 'postgresql://?servicefile=' . $encoded_srvfile, - 'connection with PGSERVICE and servicefile in URI', - sql => "SELECT 'connect3_4'", - expected_stdout => qr/connect3_4/); -} - -# Check that the "servicefile" option takes priority over the PGSERVICEFILE -# environment variable. -{ - local $ENV{PGSERVICEFILE} = 'non-existent-file.conf'; - - $dummy_node->connect_fails( - 'service=my_srv', - 'connection with invalid PGSERVICEFILE', - expected_stderr => - qr/service file "non-existent-file\.conf" not found/); - - $dummy_node->connect_ok( - q{service=my_srv servicefile='} . $srvfile_win_cared . q{'}, - 'connection with both servicefile and PGSERVICEFILE', - sql => "SELECT 'connect4_1'", - expected_stdout => qr/connect4_1/); -} - -$node->teardown_node; - -done_testing(); From 43c193314a7d0dfcd0ea5b9c262a94eafd04591f Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Sat, 20 Jun 2026 10:13:47 -0400 Subject: [PATCH 09/14] python tests: convert recovery 001_stream_rep to pytest Convert the streaming-replication recovery test as a representative multi-node physical-replication example. Add a 'pytest' block to src/test/recovery/meson.build for pyt/test_001_stream_rep.py and drop the corresponding t/001_stream_rep.pl from the 'tap' list; the remaining recovery Perl tests are unchanged. --- src/test/recovery/meson.build | 10 +- src/test/recovery/pyt/test_001_stream_rep.py | 589 +++++++++++++++++ src/test/recovery/t/001_stream_rep.pl | 650 ------------------- 3 files changed, 598 insertions(+), 651 deletions(-) create mode 100644 src/test/recovery/pyt/test_001_stream_rep.py delete mode 100644 src/test/recovery/t/001_stream_rep.pl diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index 9eb8ed1142..d1ba8307af 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -10,7 +10,6 @@ tests += { 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', }, 'tests': [ - 't/001_stream_rep.pl', 't/002_archiving.pl', 't/003_recovery_targets.pl', 't/004_timeline_switch.pl', @@ -64,4 +63,13 @@ tests += { 't/053_standby_login_event_trigger.pl', ], }, + 'pytest': { + 'test_kwargs': {'priority': 40}, # recovery tests are slow, start early + 'env': { + 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', + }, + 'tests': [ + 'pyt/test_001_stream_rep.py', + ], + }, } diff --git a/src/test/recovery/pyt/test_001_stream_rep.py b/src/test/recovery/pyt/test_001_stream_rep.py new file mode 100644 index 0000000000..19f4ccb480 --- /dev/null +++ b/src/test/recovery/pyt/test_001_stream_rep.py @@ -0,0 +1,589 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Minimal test testing streaming replication.""" + +import os +import re + +from libpq import Session +from libpq.errors import PqConnectionError + + +def get_slot_xmins(node, slotname, check_expr): + """Fetch (xmin, catalog_xmin) of a slot once *check_expr* holds. + + Polls pg_replication_slots until + *check_expr* is true to reach a quiescent state, then returns the slot's + xmin and catalog_xmin (empty string for NULL, mirroring psql -At). + """ + assert node.poll_query_until( + f"SELECT {check_expr} " + "FROM pg_catalog.pg_replication_slots " + f"WHERE slot_name = '{slotname}'" + ), "Timed out waiting for slot xmins to advance" + + row = node.sql( + "SELECT xmin, catalog_xmin " + "FROM pg_catalog.pg_replication_slots " + f"WHERE slot_name = '{slotname}'" + ).rows[0] + xmin = "" if row[0] is None else row[0] + catalog_xmin = "" if row[1] is None else row[1] + return xmin, catalog_xmin + + +def advance_wal(node, num): + """Advance WAL by *num* segments.""" + for _ in range(num): + node.safe_sql("SELECT pg_logical_emit_message(false, '', 'foo')") + node.safe_sql("SELECT pg_switch_wal()") + + +def check_target_session_attrs(node1, node2, target_node, mode, status): + """Attempt to connect to node1,node2 with target_session_attrs=mode. + + Expect to connect to *target_node* (None for failure) with the given exit + *status* (0 success, 2 connection failure). + """ + connstr = ( + f"host={node1.host},{node2.host} " + f"port={node1.port},{node2.port} " + f"dbname=postgres " + f"target_session_attrs={mode}" + ) + + sess = None + ret = 0 + stdout = None + try: + sess = Session(connstr=connstr, libdir=node1.libdir) + stdout = sess.query_oneval("SHOW port") + except PqConnectionError: + ret = 2 + finally: + if sess is not None: + sess.close() + + if status == 0: + target_port = str(target_node.port) + assert status == ret and stdout == target_port, ( + f'connect to node {target_node.name} with mode "{mode}" and ' + f"{node1.name},{node2.name} listed" + ) + else: + assert status == ret and target_node is None, ( + f'fail to connect with mode "{mode}" and ' + f"{node1.name},{node2.name} listed" + ) + + +def test_001_stream_rep(create_pg): + # Initialize primary node + # + # A specific role is created to perform some tests related to replication, + # and it needs proper authentication configuration. Under the trust auth + # this framework uses, the repl_role role just has to exist, which the SQL + # below ensures. + node_primary = create_pg("primary", allows_streaming=True) + backup_name = "my_backup" + + # Take backup + node_primary.backup(backup_name) + + # Create streaming standby linking to primary + node_standby_1 = create_pg("standby_1", start=False) + node_standby_1.init_from_backup(node_primary, backup_name, has_streaming=True) + node_standby_1.start() + + # Take backup of standby 1 (not mandatory, but useful to check if + # pg_basebackup works on a standby). + node_standby_1.backup(backup_name) + + # Take a second backup of the standby while the primary is offline. + node_primary.stop() + node_standby_1.backup("my_backup_2") + node_primary.start() + + # Create second standby node linking to standby 1 + node_standby_2 = create_pg("standby_2", start=False) + node_standby_2.init_from_backup(node_standby_1, backup_name, has_streaming=True) + node_standby_2.start() + + # Reset IO statistics, for the WAL sender check with pg_stat_io. + node_primary.safe_sql("SELECT pg_stat_reset_shared('io')") + + # Create some content on primary and check its presence in standby nodes + node_primary.safe_sql("CREATE TABLE tab_int AS SELECT generate_series(1,1002) AS a") + + node_primary.safe_sql( + """ +CREATE TABLE user_logins(id serial, who text); + +CREATE FUNCTION on_login_proc() RETURNS EVENT_TRIGGER AS $$ +BEGIN + IF NOT pg_is_in_recovery() THEN + INSERT INTO user_logins (who) VALUES (session_user); + END IF; + IF session_user = 'regress_hacker' THEN + RAISE EXCEPTION 'You are not welcome!'; + END IF; +END; +$$ LANGUAGE plpgsql SECURITY DEFINER; + +CREATE EVENT TRIGGER on_login_trigger ON login EXECUTE FUNCTION on_login_proc(); +ALTER EVENT TRIGGER on_login_trigger ENABLE ALWAYS; +""" + ) + + # Wait for standbys to catch up + node_primary.wait_for_replay_catchup(node_standby_1) + node_standby_1.wait_for_replay_catchup(node_standby_2, node_primary) + + result = node_standby_1.safe_sql("SELECT count(*) FROM tab_int") + print(f"standby 1: {result}") + assert result == "1002", "check streamed content on standby 1" + + result = node_standby_2.safe_sql("SELECT count(*) FROM tab_int") + print(f"standby 2: {result}") + assert result == "1002", "check streamed content on standby 2" + + result = node_standby_1.safe_sql( + "SELECT count(*) FROM pg_stat_recovery WHERE promote_triggered IS NOT NULL" + ) + assert result == "1", "check recovery state on standby 1" + + # Likewise, but for a sequence + node_primary.safe_sql("CREATE SEQUENCE seq1") + node_primary.safe_sql("SELECT nextval('seq1')") + + # Wait for standbys to catch up + node_primary.wait_for_replay_catchup(node_standby_1) + node_standby_1.wait_for_replay_catchup(node_standby_2, node_primary) + + result = node_standby_1.safe_sql("SELECT * FROM seq1") + print(f"standby 1: {result}") + assert result == "33|0|t", "check streamed sequence content on standby 1" + + result = node_standby_2.safe_sql("SELECT * FROM seq1") + print(f"standby 2: {result}") + assert result == "33|0|t", "check streamed sequence content on standby 2" + + # Check pg_sequence_last_value() returns NULL for unlogged sequence on standby + node_primary.safe_sql("CREATE UNLOGGED SEQUENCE ulseq") + node_primary.safe_sql("SELECT nextval('ulseq')") + node_primary.wait_for_replay_catchup(node_standby_1) + assert ( + node_standby_1.safe_sql( + "SELECT pg_sequence_last_value('ulseq'::regclass) IS NULL" + ) + == "t" + ), "pg_sequence_last_value() on unlogged sequence on standby 1" + + # Check that only READ-only queries can run on standbys + res = node_standby_1.sql("INSERT INTO tab_int VALUES (1)") + assert res.error_message is not None, "read-only queries on standby 1" + res = node_standby_2.sql("INSERT INTO tab_int VALUES (1)") + assert res.error_message is not None, "read-only queries on standby 2" + + # Tests for connection parameter target_session_attrs + print('# testing connection parameter "target_session_attrs"') + + # Connect to primary in "read-write" mode with primary,standby1 list. + check_target_session_attrs( + node_primary, node_standby_1, node_primary, "read-write", 0 + ) + + # Connect to primary in "read-write" mode with standby1,primary list. + check_target_session_attrs( + node_standby_1, node_primary, node_primary, "read-write", 0 + ) + + # Connect to primary in "any" mode with primary,standby1 list. + check_target_session_attrs(node_primary, node_standby_1, node_primary, "any", 0) + + # Connect to standby1 in "any" mode with standby1,primary list. + check_target_session_attrs(node_standby_1, node_primary, node_standby_1, "any", 0) + + # Connect to primary in "primary" mode with primary,standby1 list. + check_target_session_attrs(node_primary, node_standby_1, node_primary, "primary", 0) + + # Connect to primary in "primary" mode with standby1,primary list. + check_target_session_attrs(node_standby_1, node_primary, node_primary, "primary", 0) + + # Connect to standby1 in "read-only" mode with primary,standby1 list. + check_target_session_attrs( + node_primary, node_standby_1, node_standby_1, "read-only", 0 + ) + + # Connect to standby1 in "read-only" mode with standby1,primary list. + check_target_session_attrs( + node_standby_1, node_primary, node_standby_1, "read-only", 0 + ) + + # Connect to primary in "prefer-standby" mode with primary,primary list. + check_target_session_attrs( + node_primary, node_primary, node_primary, "prefer-standby", 0 + ) + + # Connect to standby1 in "prefer-standby" mode with primary,standby1 list. + check_target_session_attrs( + node_primary, node_standby_1, node_standby_1, "prefer-standby", 0 + ) + + # Connect to standby1 in "prefer-standby" mode with standby1,primary list. + check_target_session_attrs( + node_standby_1, node_primary, node_standby_1, "prefer-standby", 0 + ) + + # Connect to standby1 in "standby" mode with primary,standby1 list. + check_target_session_attrs( + node_primary, node_standby_1, node_standby_1, "standby", 0 + ) + + # Connect to standby1 in "standby" mode with standby1,primary list. + check_target_session_attrs( + node_standby_1, node_primary, node_standby_1, "standby", 0 + ) + + # Fail to connect in "read-write" mode with standby1,standby2 list. + check_target_session_attrs(node_standby_1, node_standby_2, None, "read-write", 2) + + # Fail to connect in "primary" mode with standby1,standby2 list. + check_target_session_attrs(node_standby_1, node_standby_2, None, "primary", 2) + + # Fail to connect in "read-only" mode with primary,primary list. + check_target_session_attrs(node_primary, node_primary, None, "read-only", 2) + + # Fail to connect in "standby" mode with primary,primary list. + check_target_session_attrs(node_primary, node_primary, None, "standby", 2) + + # Test for SHOW commands using a WAL sender connection with a replication + # role. + print("# testing SHOW commands for replication connection") + + node_primary.safe_sql( + """ +CREATE ROLE repl_role REPLICATION LOGIN; +GRANT pg_read_all_settings TO repl_role;""" + ) + primary_host = node_primary.host + primary_port = node_primary.port + connstr_common = f"host={primary_host} port={primary_port} user=repl_role" + connstr_rep = f"{connstr_common} replication=1" + connstr_db = f"{connstr_common} replication=database dbname=postgres" + + # Test SHOW ALL + with Session(connstr=connstr_rep, libdir=node_primary.libdir) as sess: + res = sess.query("SHOW ALL;") + assert ( + res.error_message is None + ), "SHOW ALL with replication role and physical replication" + with Session(connstr=connstr_db, libdir=node_primary.libdir) as sess: + res = sess.query("SHOW ALL;") + assert ( + res.error_message is None + ), "SHOW ALL with replication role and logical replication" + + # Test SHOW with a user-settable parameter + with Session(connstr=connstr_rep, libdir=node_primary.libdir) as sess: + res = sess.query("SHOW work_mem;") + assert res.error_message is None, ( + "SHOW with user-settable parameter, replication role and " + "physical replication" + ) + with Session(connstr=connstr_db, libdir=node_primary.libdir) as sess: + res = sess.query("SHOW work_mem;") + assert res.error_message is None, ( + "SHOW with user-settable parameter, replication role and " + "logical replication" + ) + + # Test SHOW with a superuser-settable parameter + with Session(connstr=connstr_rep, libdir=node_primary.libdir) as sess: + res = sess.query("SHOW primary_conninfo;") + assert res.error_message is None, ( + "SHOW with superuser-settable parameter, replication role and " + "physical replication" + ) + with Session(connstr=connstr_db, libdir=node_primary.libdir) as sess: + res = sess.query("SHOW primary_conninfo;") + assert res.error_message is None, ( + "SHOW with superuser-settable parameter, replication role and " + "logical replication" + ) + + print("# testing READ_REPLICATION_SLOT command for replication connection") + + slotname = "test_read_replication_slot_physical" + + with Session(connstr=connstr_rep, libdir=node_primary.libdir) as sess: + res = sess.query("READ_REPLICATION_SLOT non_existent_slot;") + assert res.error_message is None, "READ_REPLICATION_SLOT exit code 0 on success" + assert re.search( + r"^\|\|$", res.psqlout + ), "READ_REPLICATION_SLOT returns NULL values if slot does not exist" + + with Session(connstr=connstr_rep, libdir=node_primary.libdir) as sess: + sess.query(f"CREATE_REPLICATION_SLOT {slotname} PHYSICAL RESERVE_WAL;") + + with Session(connstr=connstr_rep, libdir=node_primary.libdir) as sess: + res = sess.query(f"READ_REPLICATION_SLOT {slotname};") + assert ( + res.error_message is None + ), "READ_REPLICATION_SLOT success with existing slot" + assert re.search( + r"^physical\|[^|]*\|1$", res.psqlout + ), "READ_REPLICATION_SLOT returns tuple with slot information" + + with Session(connstr=connstr_rep, libdir=node_primary.libdir) as sess: + sess.query(f"DROP_REPLICATION_SLOT {slotname};") + + print("# switching to physical replication slot") + + # Wait for the physical WAL sender to update its IO statistics. This is + # done before the next restart, which would force a flush of its stats, and + # far enough from the reset done above to not impact the run time. + assert node_primary.poll_query_until( + "SELECT sum(reads) > 0 " + "FROM pg_catalog.pg_stat_io " + "WHERE backend_type = 'walsender' " + "AND object = 'wal'" + ), "Timed out while waiting for the walsender to update its IO statistics" + + # Switch to using a physical replication slot. We can do this without a new + # backup since physical slots can go backwards if needed. Do so on both + # standbys. Since we're going to be testing things that affect the slot + # state, also increase the standby feedback interval to ensure timely + # updates. + slotname_1, slotname_2 = "standby_1", "standby_2" + node_primary.append_conf("max_replication_slots = 4") + node_primary.restart() + res = node_primary.sql( + f"SELECT pg_create_physical_replication_slot('{slotname_1}')" + ) + assert res.error_message is None, "physical slot created on primary" + node_standby_1.append_conf(f"primary_slot_name = {slotname_1}") + node_standby_1.append_conf("wal_receiver_status_interval = 1") + node_standby_1.append_conf("max_replication_slots = 4") + node_standby_1.restart() + res = node_standby_1.sql( + f"SELECT pg_create_physical_replication_slot('{slotname_2}')" + ) + assert res.error_message is None, "physical slot created on intermediate replica" + node_standby_2.append_conf(f"primary_slot_name = {slotname_2}") + node_standby_2.append_conf("wal_receiver_status_interval = 1") + # should be able change primary_slot_name without restart + # will wait effect in get_slot_xmins above + node_standby_2.reload() + + # There's no hot standby feedback and there are no logical slots on either + # peer so xmin and catalog_xmin should be null on both slots. + xmin, catalog_xmin = get_slot_xmins( + node_primary, slotname_1, "xmin IS NULL AND catalog_xmin IS NULL" + ) + assert xmin == "", "xmin of non-cascaded slot null with no hs_feedback" + assert ( + catalog_xmin == "" + ), "catalog xmin of non-cascaded slot null with no hs_feedback" + + xmin, catalog_xmin = get_slot_xmins( + node_standby_1, slotname_2, "xmin IS NULL AND catalog_xmin IS NULL" + ) + assert xmin == "", "xmin of cascaded slot null with no hs_feedback" + assert catalog_xmin == "", "catalog xmin of cascaded slot null with no hs_feedback" + + # Replication still works? + node_primary.safe_sql("CREATE TABLE replayed(val integer);") + + def replay_check(): + newval = node_primary.safe_sql( + "INSERT INTO replayed(val) SELECT coalesce(max(val),0) + 1 " + "AS newval FROM replayed RETURNING val" + ) + node_primary.wait_for_replay_catchup(node_standby_1) + node_standby_1.wait_for_replay_catchup(node_standby_2, node_primary) + + assert ( + node_standby_1.safe_sql(f"SELECT 1 FROM replayed WHERE val = {newval}") + == "1" + ), f"standby_1 didn't replay primary value {newval}" + assert ( + node_standby_2.safe_sql(f"SELECT 1 FROM replayed WHERE val = {newval}") + == "1" + ), f"standby_2 didn't replay standby_1 value {newval}" + + replay_check() + + evttrig = node_standby_1.safe_sql( + "SELECT evtname FROM pg_event_trigger WHERE evtevent = 'login'" + ) + assert evttrig == "on_login_trigger", "Name of login trigger" + evttrig = node_standby_2.safe_sql( + "SELECT evtname FROM pg_event_trigger WHERE evtevent = 'login'" + ) + assert evttrig == "on_login_trigger", "Name of login trigger" + + print("# enabling hot_standby_feedback") + + # Enable hs_feedback. The slot should gain an xmin. We set the status + # interval so we'll see the results promptly. + node_standby_1.safe_sql("ALTER SYSTEM SET hot_standby_feedback = on;") + node_standby_1.reload() + node_standby_2.safe_sql("ALTER SYSTEM SET hot_standby_feedback = on;") + node_standby_2.reload() + replay_check() + + xmin, catalog_xmin = get_slot_xmins( + node_primary, slotname_1, "xmin IS NOT NULL AND catalog_xmin IS NULL" + ) + assert xmin != "", "xmin of non-cascaded slot non-null with hs feedback" + assert ( + catalog_xmin == "" + ), "catalog xmin of non-cascaded slot still null with hs_feedback" + + xmin1, catalog_xmin1 = get_slot_xmins( + node_standby_1, slotname_2, "xmin IS NOT NULL AND catalog_xmin IS NULL" + ) + assert xmin1 != "", "xmin of cascaded slot non-null with hs feedback" + assert ( + catalog_xmin1 == "" + ), "catalog xmin of cascaded slot still null with hs_feedback" + + print("# doing some work to advance xmin") + node_primary.safe_sql( + """ +do $$ +begin + for i in 10000..11000 loop + -- use an exception block so that each iteration eats an XID + begin + insert into tab_int values (i); + exception + when division_by_zero then null; + end; + end loop; +end$$; +""" + ) + + node_primary.safe_sql("VACUUM;") + node_primary.safe_sql("CHECKPOINT;") + + xmin2, catalog_xmin2 = get_slot_xmins(node_primary, slotname_1, f"xmin <> '{xmin}'") + print(f"primary slot's new xmin {xmin2}, old xmin {xmin}") + assert xmin2 != xmin, "xmin of non-cascaded slot with hs feedback has changed" + assert ( + catalog_xmin2 == "" + ), "catalog xmin of non-cascaded slot still null with hs_feedback unchanged" + + xmin2, catalog_xmin2 = get_slot_xmins( + node_standby_1, slotname_2, f"xmin <> '{xmin1}'" + ) + print(f"standby_1 slot's new xmin {xmin2}, old xmin {xmin1}") + assert xmin2 != xmin1, "xmin of cascaded slot with hs feedback has changed" + assert ( + catalog_xmin2 == "" + ), "catalog xmin of cascaded slot still null with hs_feedback unchanged" + + print("# disabling hot_standby_feedback") + + # Disable hs_feedback. Xmin should be cleared. + node_standby_1.safe_sql("ALTER SYSTEM SET hot_standby_feedback = off;") + node_standby_1.reload() + node_standby_2.safe_sql("ALTER SYSTEM SET hot_standby_feedback = off;") + node_standby_2.reload() + replay_check() + + xmin, catalog_xmin = get_slot_xmins( + node_primary, slotname_1, "xmin IS NULL AND catalog_xmin IS NULL" + ) + assert xmin == "", "xmin of non-cascaded slot null with hs feedback reset" + assert ( + catalog_xmin == "" + ), "catalog xmin of non-cascaded slot still null with hs_feedback reset" + + xmin, catalog_xmin = get_slot_xmins( + node_standby_1, slotname_2, "xmin IS NULL AND catalog_xmin IS NULL" + ) + assert xmin == "", "xmin of cascaded slot null with hs feedback reset" + assert ( + catalog_xmin == "" + ), "catalog xmin of cascaded slot still null with hs_feedback reset" + + print("# check change primary_conninfo without restart") + node_standby_2.append_conf("primary_slot_name = ''") + node_standby_2.enable_streaming(node_primary) + node_standby_2.reload() + + # The WAL receiver should have generated some IO statistics. + stats_reads = node_standby_1.safe_sql( + "SELECT sum(writes) > 0 FROM pg_stat_io " + "WHERE backend_type = 'walreceiver' AND object = 'wal'" + ) + assert stats_reads == "t", "WAL receiver generates statistics for WAL writes" + + # be sure do not streaming from cascade + node_standby_1.stop() + + newval = node_primary.safe_sql( + "INSERT INTO replayed(val) SELECT coalesce(max(val),0) + 1 " + "AS newval FROM replayed RETURNING val" + ) + node_primary.wait_for_catchup(node_standby_2) + is_replayed = node_standby_2.safe_sql( + f"SELECT 1 FROM replayed WHERE val = {newval}" + ) + assert is_replayed == "1", f"standby_2 didn't replay primary value {newval}" + + # Drop any existing slots on the primary, for the follow-up tests. + node_primary.safe_sql( + "SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots;" + ) + + # Test physical slot advancing and its durability. Create a new slot on + # the primary, not used by any of the standbys. This reserves WAL at + # creation. + phys_slot = "phys_slot" + node_primary.safe_sql( + f"SELECT pg_create_physical_replication_slot('{phys_slot}', true);" + ) + # Generate some WAL, and switch to a new segment, used to check that + # the previous segment is correctly getting recycled as the slot advancing + # would recompute the minimum LSN calculated across all slots. + segment_removed = node_primary.safe_sql( + "SELECT pg_walfile_name(pg_current_wal_lsn())" + ) + advance_wal(node_primary, 1) + current_lsn = node_primary.safe_sql("SELECT pg_current_wal_lsn();") + res = node_primary.sql( + f"SELECT pg_replication_slot_advance('{phys_slot}', " + f"'{current_lsn}'::pg_lsn);" + ) + assert res.error_message is None, "slot advancing with physical slot" + phys_restart_lsn_pre = node_primary.safe_sql( + "SELECT restart_lsn from pg_replication_slots " + f"WHERE slot_name = '{phys_slot}';" + ) + # Slot advance should persist across clean restarts. + node_primary.restart() + phys_restart_lsn_post = node_primary.safe_sql( + "SELECT restart_lsn from pg_replication_slots " + f"WHERE slot_name = '{phys_slot}';" + ) + assert ( + phys_restart_lsn_pre == phys_restart_lsn_post + ), "physical slot advance persists across restarts" + + # Check if the previous segment gets correctly recycled after the + # server stopped cleanly, causing a shutdown checkpoint to be generated. + primary_data = node_primary.data_dir + assert not os.path.isfile( + os.path.join(primary_data, "pg_wal", segment_removed) + ), f"WAL segment {segment_removed} recycled after physical slot advancing" + + # NOTE: the final "pg_backup_start() followed by BASE_BACKUP" and + # "BASE_BACKUP cancellation" checks are not implemented here. They drive + # the replication-protocol BASE_BACKUP command and consume/cancel its + # COPY_OUT stream, which needs libpq COPY-streaming support + # (PQgetCopyData) that the in-process Session does not provide yet. This + # section should be added once that framework support exists. diff --git a/src/test/recovery/t/001_stream_rep.pl b/src/test/recovery/t/001_stream_rep.pl deleted file mode 100644 index a4fa4b96c6..0000000000 --- a/src/test/recovery/t/001_stream_rep.pl +++ /dev/null @@ -1,650 +0,0 @@ - -# Copyright (c) 2021-2026, PostgreSQL Global Development Group - -# Minimal test testing streaming replication -use strict; -use warnings FATAL => 'all'; -use PostgreSQL::Test::Cluster; -use PostgreSQL::Test::Utils; -use Test::More; - -# Initialize primary node -my $node_primary = PostgreSQL::Test::Cluster->new('primary'); -# A specific role is created to perform some tests related to replication, -# and it needs proper authentication configuration. -$node_primary->init( - allows_streaming => 1, - auth_extra => [ '--create-role' => 'repl_role' ]); -$node_primary->start; -my $backup_name = 'my_backup'; - -# Take backup -$node_primary->backup($backup_name); - -# Create streaming standby linking to primary -my $node_standby_1 = PostgreSQL::Test::Cluster->new('standby_1'); -$node_standby_1->init_from_backup($node_primary, $backup_name, - has_streaming => 1); -$node_standby_1->start; - -# Take backup of standby 1 (not mandatory, but useful to check if -# pg_basebackup works on a standby). -$node_standby_1->backup($backup_name); - -# Take a second backup of the standby while the primary is offline. -$node_primary->stop; -$node_standby_1->backup('my_backup_2'); -$node_primary->start; - -# Create second standby node linking to standby 1 -my $node_standby_2 = PostgreSQL::Test::Cluster->new('standby_2'); -$node_standby_2->init_from_backup($node_standby_1, $backup_name, - has_streaming => 1); -$node_standby_2->start; - -# Reset IO statistics, for the WAL sender check with pg_stat_io. -$node_primary->safe_psql('postgres', "SELECT pg_stat_reset_shared('io')"); - -# Create some content on primary and check its presence in standby nodes -$node_primary->safe_psql('postgres', - "CREATE TABLE tab_int AS SELECT generate_series(1,1002) AS a"); - -$node_primary->safe_psql( - 'postgres', q{ -CREATE TABLE user_logins(id serial, who text); - -CREATE FUNCTION on_login_proc() RETURNS EVENT_TRIGGER AS $$ -BEGIN - IF NOT pg_is_in_recovery() THEN - INSERT INTO user_logins (who) VALUES (session_user); - END IF; - IF session_user = 'regress_hacker' THEN - RAISE EXCEPTION 'You are not welcome!'; - END IF; -END; -$$ LANGUAGE plpgsql SECURITY DEFINER; - -CREATE EVENT TRIGGER on_login_trigger ON login EXECUTE FUNCTION on_login_proc(); -ALTER EVENT TRIGGER on_login_trigger ENABLE ALWAYS; -}); - -# Wait for standbys to catch up -$node_primary->wait_for_replay_catchup($node_standby_1); -$node_standby_1->wait_for_replay_catchup($node_standby_2, $node_primary); - -my $result = - $node_standby_1->safe_psql('postgres', "SELECT count(*) FROM tab_int"); -print "standby 1: $result\n"; -is($result, qq(1002), 'check streamed content on standby 1'); - -$result = - $node_standby_2->safe_psql('postgres', "SELECT count(*) FROM tab_int"); -print "standby 2: $result\n"; -is($result, qq(1002), 'check streamed content on standby 2'); - -$result = $node_standby_1->safe_psql('postgres', - "SELECT count(*) FROM pg_stat_recovery WHERE promote_triggered IS NOT NULL" -); -is($result, qq(1), 'check recovery state on standby 1'); - -# Likewise, but for a sequence -$node_primary->safe_psql('postgres', - "CREATE SEQUENCE seq1; SELECT nextval('seq1')"); - -# Wait for standbys to catch up -$node_primary->wait_for_replay_catchup($node_standby_1); -$node_standby_1->wait_for_replay_catchup($node_standby_2, $node_primary); - -$result = $node_standby_1->safe_psql('postgres', "SELECT * FROM seq1"); -print "standby 1: $result\n"; -is($result, qq(33|0|t), 'check streamed sequence content on standby 1'); - -$result = $node_standby_2->safe_psql('postgres', "SELECT * FROM seq1"); -print "standby 2: $result\n"; -is($result, qq(33|0|t), 'check streamed sequence content on standby 2'); - -# Check pg_sequence_last_value() returns NULL for unlogged sequence on standby -$node_primary->safe_psql('postgres', - "CREATE UNLOGGED SEQUENCE ulseq; SELECT nextval('ulseq')"); -$node_primary->wait_for_replay_catchup($node_standby_1); -is( $node_standby_1->safe_psql( - 'postgres', - "SELECT pg_sequence_last_value('ulseq'::regclass) IS NULL"), - 't', - 'pg_sequence_last_value() on unlogged sequence on standby 1'); - -# Check that only READ-only queries can run on standbys -is($node_standby_1->psql('postgres', 'INSERT INTO tab_int VALUES (1)'), - 3, 'read-only queries on standby 1'); -is($node_standby_2->psql('postgres', 'INSERT INTO tab_int VALUES (1)'), - 3, 'read-only queries on standby 2'); - -# Tests for connection parameter target_session_attrs -note "testing connection parameter \"target_session_attrs\""; - -# Attempt to connect to $node1, then $node2, using target_session_attrs=$mode. -# Expect to connect to $target_node (undef for failure) with given $status. -sub test_target_session_attrs -{ - local $Test::Builder::Level = $Test::Builder::Level + 1; - - my $node1 = shift; - my $node2 = shift; - my $target_node = shift; - my $mode = shift; - my $status = shift; - - my $node1_host = $node1->host; - my $node1_port = $node1->port; - my $node1_name = $node1->name; - my $node2_host = $node2->host; - my $node2_port = $node2->port; - my $node2_name = $node2->name; - my $target_port = undef; - $target_port = $target_node->port if (defined $target_node); - my $target_name = undef; - $target_name = $target_node->name if (defined $target_node); - - # Build connection string for connection attempt. - my $connstr = "host=$node1_host,$node2_host "; - $connstr .= "port=$node1_port,$node2_port "; - $connstr .= "target_session_attrs=$mode"; - - # Attempt to connect, and if successful, get the server port number - # we connected to. Note we must pass the SQL command via the command - # line not stdin, else Perl may spit up trying to write to stdin of - # an already-failed psql process. - my ($ret, $stdout, $stderr) = $node1->psql( - 'postgres', - undef, - extra_params => [ - '--dbname' => $connstr, - '--command' => 'SHOW port;', - ]); - if ($status == 0) - { - is( $status == $ret && $stdout eq $target_port, - 1, - "connect to node $target_name with mode \"$mode\" and $node1_name,$node2_name listed" - ); - } - else - { - print "status = $status\n"; - print "ret = $ret\n"; - print "stdout = $stdout\n"; - print "stderr = $stderr\n"; - - is( $status == $ret && !defined $target_node, - 1, - "fail to connect with mode \"$mode\" and $node1_name,$node2_name listed" - ); - } - - return; -} - -# Connect to primary in "read-write" mode with primary,standby1 list. -test_target_session_attrs($node_primary, $node_standby_1, $node_primary, - "read-write", 0); - -# Connect to primary in "read-write" mode with standby1,primary list. -test_target_session_attrs($node_standby_1, $node_primary, $node_primary, - "read-write", 0); - -# Connect to primary in "any" mode with primary,standby1 list. -test_target_session_attrs($node_primary, $node_standby_1, $node_primary, - "any", 0); - -# Connect to standby1 in "any" mode with standby1,primary list. -test_target_session_attrs($node_standby_1, $node_primary, $node_standby_1, - "any", 0); - -# Connect to primary in "primary" mode with primary,standby1 list. -test_target_session_attrs($node_primary, $node_standby_1, $node_primary, - "primary", 0); - -# Connect to primary in "primary" mode with standby1,primary list. -test_target_session_attrs($node_standby_1, $node_primary, $node_primary, - "primary", 0); - -# Connect to standby1 in "read-only" mode with primary,standby1 list. -test_target_session_attrs($node_primary, $node_standby_1, $node_standby_1, - "read-only", 0); - -# Connect to standby1 in "read-only" mode with standby1,primary list. -test_target_session_attrs($node_standby_1, $node_primary, $node_standby_1, - "read-only", 0); - -# Connect to primary in "prefer-standby" mode with primary,primary list. -test_target_session_attrs($node_primary, $node_primary, $node_primary, - "prefer-standby", 0); - -# Connect to standby1 in "prefer-standby" mode with primary,standby1 list. -test_target_session_attrs($node_primary, $node_standby_1, $node_standby_1, - "prefer-standby", 0); - -# Connect to standby1 in "prefer-standby" mode with standby1,primary list. -test_target_session_attrs($node_standby_1, $node_primary, $node_standby_1, - "prefer-standby", 0); - -# Connect to standby1 in "standby" mode with primary,standby1 list. -test_target_session_attrs($node_primary, $node_standby_1, $node_standby_1, - "standby", 0); - -# Connect to standby1 in "standby" mode with standby1,primary list. -test_target_session_attrs($node_standby_1, $node_primary, $node_standby_1, - "standby", 0); - -# Fail to connect in "read-write" mode with standby1,standby2 list. -test_target_session_attrs($node_standby_1, $node_standby_2, undef, - "read-write", 2); - -# Fail to connect in "primary" mode with standby1,standby2 list. -test_target_session_attrs($node_standby_1, $node_standby_2, undef, - "primary", 2); - -# Fail to connect in "read-only" mode with primary,primary list. -test_target_session_attrs($node_primary, $node_primary, undef, - "read-only", 2); - -# Fail to connect in "standby" mode with primary,primary list. -test_target_session_attrs($node_primary, $node_primary, undef, "standby", 2); - -# Test for SHOW commands using a WAL sender connection with a replication -# role. -note "testing SHOW commands for replication connection"; - -$node_primary->psql( - 'postgres', " -CREATE ROLE repl_role REPLICATION LOGIN; -GRANT pg_read_all_settings TO repl_role;"); -my $primary_host = $node_primary->host; -my $primary_port = $node_primary->port; -my $connstr_common = "host=$primary_host port=$primary_port user=repl_role"; -my $connstr_rep = "$connstr_common replication=1"; -my $connstr_db = "$connstr_common replication=database dbname=postgres"; - -# Test SHOW ALL -my ($ret, $stdout, $stderr) = $node_primary->psql( - 'postgres', 'SHOW ALL;', - on_error_die => 1, - extra_params => [ '--dbname' => $connstr_rep ]); -is($ret, 0, "SHOW ALL with replication role and physical replication"); -($ret, $stdout, $stderr) = $node_primary->psql( - 'postgres', 'SHOW ALL;', - on_error_die => 1, - extra_params => [ '--dbname' => $connstr_db ]); -is($ret, 0, "SHOW ALL with replication role and logical replication"); - -# Test SHOW with a user-settable parameter -($ret, $stdout, $stderr) = $node_primary->psql( - 'postgres', 'SHOW work_mem;', - on_error_die => 1, - extra_params => [ '--dbname' => $connstr_rep ]); -is($ret, 0, - "SHOW with user-settable parameter, replication role and physical replication" -); -($ret, $stdout, $stderr) = $node_primary->psql( - 'postgres', 'SHOW work_mem;', - on_error_die => 1, - extra_params => [ '--dbname' => $connstr_db ]); -is($ret, 0, - "SHOW with user-settable parameter, replication role and logical replication" -); - -# Test SHOW with a superuser-settable parameter -($ret, $stdout, $stderr) = $node_primary->psql( - 'postgres', 'SHOW primary_conninfo;', - on_error_die => 1, - extra_params => [ '--dbname' => $connstr_rep ]); -is($ret, 0, - "SHOW with superuser-settable parameter, replication role and physical replication" -); -($ret, $stdout, $stderr) = $node_primary->psql( - 'postgres', 'SHOW primary_conninfo;', - on_error_die => 1, - extra_params => [ '--dbname' => $connstr_db ]); -is($ret, 0, - "SHOW with superuser-settable parameter, replication role and logical replication" -); - -note "testing READ_REPLICATION_SLOT command for replication connection"; - -my $slotname = 'test_read_replication_slot_physical'; - -($ret, $stdout, $stderr) = $node_primary->psql( - 'postgres', - 'READ_REPLICATION_SLOT non_existent_slot;', - extra_params => [ '--dbname' => $connstr_rep ]); -is($ret, 0, "READ_REPLICATION_SLOT exit code 0 on success"); -like($stdout, qr/^\|\|$/, - "READ_REPLICATION_SLOT returns NULL values if slot does not exist"); - -$node_primary->psql( - 'postgres', - "CREATE_REPLICATION_SLOT $slotname PHYSICAL RESERVE_WAL;", - extra_params => [ '--dbname' => $connstr_rep ]); - -($ret, $stdout, $stderr) = $node_primary->psql( - 'postgres', - "READ_REPLICATION_SLOT $slotname;", - extra_params => [ '--dbname' => $connstr_rep ]); -is($ret, 0, "READ_REPLICATION_SLOT success with existing slot"); -like($stdout, qr/^physical\|[^|]*\|1$/, - "READ_REPLICATION_SLOT returns tuple with slot information"); - -$node_primary->psql( - 'postgres', - "DROP_REPLICATION_SLOT $slotname;", - extra_params => [ '--dbname' => $connstr_rep ]); - -note "switching to physical replication slot"; - -# Wait for the physical WAL sender to update its IO statistics. This is -# done before the next restart, which would force a flush of its stats, and -# far enough from the reset done above to not impact the run time. -$node_primary->poll_query_until( - 'postgres', - qq[SELECT sum(reads) > 0 - FROM pg_catalog.pg_stat_io - WHERE backend_type = 'walsender' - AND object = 'wal'] - ) - or die - "Timed out while waiting for the walsender to update its IO statistics"; - -# Switch to using a physical replication slot. We can do this without a new -# backup since physical slots can go backwards if needed. Do so on both -# standbys. Since we're going to be testing things that affect the slot state, -# also increase the standby feedback interval to ensure timely updates. -my ($slotname_1, $slotname_2) = ('standby_1', 'standby_2'); -$node_primary->append_conf('postgresql.conf', "max_replication_slots = 4"); -$node_primary->restart; -is( $node_primary->psql( - 'postgres', - qq[SELECT pg_create_physical_replication_slot('$slotname_1');]), - 0, - 'physical slot created on primary'); -$node_standby_1->append_conf('postgresql.conf', - "primary_slot_name = $slotname_1"); -$node_standby_1->append_conf('postgresql.conf', - "wal_receiver_status_interval = 1"); -$node_standby_1->append_conf('postgresql.conf', "max_replication_slots = 4"); -$node_standby_1->restart; -is( $node_standby_1->psql( - 'postgres', - qq[SELECT pg_create_physical_replication_slot('$slotname_2');]), - 0, - 'physical slot created on intermediate replica'); -$node_standby_2->append_conf('postgresql.conf', - "primary_slot_name = $slotname_2"); -$node_standby_2->append_conf('postgresql.conf', - "wal_receiver_status_interval = 1"); -# should be able change primary_slot_name without restart -# will wait effect in get_slot_xmins above -$node_standby_2->reload; - -# Fetch xmin columns from slot's pg_replication_slots row, after waiting for -# given boolean condition to be true to ensure we've reached a quiescent state -sub get_slot_xmins -{ - my ($node, $slotname, $check_expr) = @_; - - $node->poll_query_until( - 'postgres', qq[ - SELECT $check_expr - FROM pg_catalog.pg_replication_slots - WHERE slot_name = '$slotname'; - ]) or die "Timed out waiting for slot xmins to advance"; - - my $slotinfo = $node->slot($slotname); - return ($slotinfo->{'xmin'}, $slotinfo->{'catalog_xmin'}); -} - -# There's no hot standby feedback and there are no logical slots on either peer -# so xmin and catalog_xmin should be null on both slots. -my ($xmin, $catalog_xmin) = get_slot_xmins($node_primary, $slotname_1, - "xmin IS NULL AND catalog_xmin IS NULL"); -is($xmin, '', 'xmin of non-cascaded slot null with no hs_feedback'); -is($catalog_xmin, '', - 'catalog xmin of non-cascaded slot null with no hs_feedback'); - -($xmin, $catalog_xmin) = get_slot_xmins($node_standby_1, $slotname_2, - "xmin IS NULL AND catalog_xmin IS NULL"); -is($xmin, '', 'xmin of cascaded slot null with no hs_feedback'); -is($catalog_xmin, '', - 'catalog xmin of cascaded slot null with no hs_feedback'); - -# Replication still works? -$node_primary->safe_psql('postgres', 'CREATE TABLE replayed(val integer);'); - -sub replay_check -{ - my $newval = $node_primary->safe_psql('postgres', - 'INSERT INTO replayed(val) SELECT coalesce(max(val),0) + 1 AS newval FROM replayed RETURNING val' - ); - $node_primary->wait_for_replay_catchup($node_standby_1); - $node_standby_1->wait_for_replay_catchup($node_standby_2, $node_primary); - - $node_standby_1->safe_psql('postgres', - qq[SELECT 1 FROM replayed WHERE val = $newval]) - or die "standby_1 didn't replay primary value $newval"; - $node_standby_2->safe_psql('postgres', - qq[SELECT 1 FROM replayed WHERE val = $newval]) - or die "standby_2 didn't replay standby_1 value $newval"; - return; -} - -replay_check(); - -my $evttrig = $node_standby_1->safe_psql('postgres', - "SELECT evtname FROM pg_event_trigger WHERE evtevent = 'login'"); -is($evttrig, 'on_login_trigger', 'Name of login trigger'); -$evttrig = $node_standby_2->safe_psql('postgres', - "SELECT evtname FROM pg_event_trigger WHERE evtevent = 'login'"); -is($evttrig, 'on_login_trigger', 'Name of login trigger'); - -note "enabling hot_standby_feedback"; - -# Enable hs_feedback. The slot should gain an xmin. We set the status interval -# so we'll see the results promptly. -$node_standby_1->safe_psql('postgres', - 'ALTER SYSTEM SET hot_standby_feedback = on;'); -$node_standby_1->reload; -$node_standby_2->safe_psql('postgres', - 'ALTER SYSTEM SET hot_standby_feedback = on;'); -$node_standby_2->reload; -replay_check(); - -($xmin, $catalog_xmin) = get_slot_xmins($node_primary, $slotname_1, - "xmin IS NOT NULL AND catalog_xmin IS NULL"); -isnt($xmin, '', 'xmin of non-cascaded slot non-null with hs feedback'); -is($catalog_xmin, '', - 'catalog xmin of non-cascaded slot still null with hs_feedback'); - -my ($xmin1, $catalog_xmin1) = get_slot_xmins($node_standby_1, $slotname_2, - "xmin IS NOT NULL AND catalog_xmin IS NULL"); -isnt($xmin1, '', 'xmin of cascaded slot non-null with hs feedback'); -is($catalog_xmin1, '', - 'catalog xmin of cascaded slot still null with hs_feedback'); - -note "doing some work to advance xmin"; -$node_primary->safe_psql( - 'postgres', q{ -do $$ -begin - for i in 10000..11000 loop - -- use an exception block so that each iteration eats an XID - begin - insert into tab_int values (i); - exception - when division_by_zero then null; - end; - end loop; -end$$; -}); - -$node_primary->safe_psql('postgres', 'VACUUM;'); -$node_primary->safe_psql('postgres', 'CHECKPOINT;'); - -my ($xmin2, $catalog_xmin2) = - get_slot_xmins($node_primary, $slotname_1, "xmin <> '$xmin'"); -note "primary slot's new xmin $xmin2, old xmin $xmin"; -isnt($xmin2, $xmin, 'xmin of non-cascaded slot with hs feedback has changed'); -is($catalog_xmin2, '', - 'catalog xmin of non-cascaded slot still null with hs_feedback unchanged' -); - -($xmin2, $catalog_xmin2) = - get_slot_xmins($node_standby_1, $slotname_2, "xmin <> '$xmin1'"); -note "standby_1 slot's new xmin $xmin2, old xmin $xmin1"; -isnt($xmin2, $xmin1, 'xmin of cascaded slot with hs feedback has changed'); -is($catalog_xmin2, '', - 'catalog xmin of cascaded slot still null with hs_feedback unchanged'); - -note "disabling hot_standby_feedback"; - -# Disable hs_feedback. Xmin should be cleared. -$node_standby_1->safe_psql('postgres', - 'ALTER SYSTEM SET hot_standby_feedback = off;'); -$node_standby_1->reload; -$node_standby_2->safe_psql('postgres', - 'ALTER SYSTEM SET hot_standby_feedback = off;'); -$node_standby_2->reload; -replay_check(); - -($xmin, $catalog_xmin) = get_slot_xmins($node_primary, $slotname_1, - "xmin IS NULL AND catalog_xmin IS NULL"); -is($xmin, '', 'xmin of non-cascaded slot null with hs feedback reset'); -is($catalog_xmin, '', - 'catalog xmin of non-cascaded slot still null with hs_feedback reset'); - -($xmin, $catalog_xmin) = get_slot_xmins($node_standby_1, $slotname_2, - "xmin IS NULL AND catalog_xmin IS NULL"); -is($xmin, '', 'xmin of cascaded slot null with hs feedback reset'); -is($catalog_xmin, '', - 'catalog xmin of cascaded slot still null with hs_feedback reset'); - -note "check change primary_conninfo without restart"; -$node_standby_2->append_conf('postgresql.conf', "primary_slot_name = ''"); -$node_standby_2->enable_streaming($node_primary); -$node_standby_2->reload; - -# The WAL receiver should have generated some IO statistics. -my $stats_reads = $node_standby_1->safe_psql( - 'postgres', - qq{SELECT sum(writes) > 0 FROM pg_stat_io - WHERE backend_type = 'walreceiver' AND object = 'wal'}); -is($stats_reads, 't', "WAL receiver generates statistics for WAL writes"); - -# be sure do not streaming from cascade -$node_standby_1->stop; - -my $newval = $node_primary->safe_psql('postgres', - 'INSERT INTO replayed(val) SELECT coalesce(max(val),0) + 1 AS newval FROM replayed RETURNING val' -); -$node_primary->wait_for_catchup($node_standby_2); -my $is_replayed = $node_standby_2->safe_psql('postgres', - qq[SELECT 1 FROM replayed WHERE val = $newval]); -is($is_replayed, qq(1), "standby_2 didn't replay primary value $newval"); - -# Drop any existing slots on the primary, for the follow-up tests. -$node_primary->safe_psql('postgres', - "SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots;"); - -# Test physical slot advancing and its durability. Create a new slot on -# the primary, not used by any of the standbys. This reserves WAL at creation. -my $phys_slot = 'phys_slot'; -$node_primary->safe_psql('postgres', - "SELECT pg_create_physical_replication_slot('$phys_slot', true);"); -# Generate some WAL, and switch to a new segment, used to check that -# the previous segment is correctly getting recycled as the slot advancing -# would recompute the minimum LSN calculated across all slots. -my $segment_removed = $node_primary->safe_psql('postgres', - 'SELECT pg_walfile_name(pg_current_wal_lsn())'); -chomp($segment_removed); -$node_primary->advance_wal(1); -my $current_lsn = - $node_primary->safe_psql('postgres', "SELECT pg_current_wal_lsn();"); -chomp($current_lsn); -my $psql_rc = $node_primary->psql('postgres', - "SELECT pg_replication_slot_advance('$phys_slot', '$current_lsn'::pg_lsn);" -); -is($psql_rc, '0', 'slot advancing with physical slot'); -my $phys_restart_lsn_pre = $node_primary->safe_psql('postgres', - "SELECT restart_lsn from pg_replication_slots WHERE slot_name = '$phys_slot';" -); -chomp($phys_restart_lsn_pre); -# Slot advance should persist across clean restarts. -$node_primary->restart; -my $phys_restart_lsn_post = $node_primary->safe_psql('postgres', - "SELECT restart_lsn from pg_replication_slots WHERE slot_name = '$phys_slot';" -); -chomp($phys_restart_lsn_post); -is($phys_restart_lsn_pre, $phys_restart_lsn_post, - "physical slot advance persists across restarts"); - -# Check if the previous segment gets correctly recycled after the -# server stopped cleanly, causing a shutdown checkpoint to be generated. -my $primary_data = $node_primary->data_dir; -ok(!-f "$primary_data/pg_wal/$segment_removed", - "WAL segment $segment_removed recycled after physical slot advancing"); - -note "testing pg_backup_start() followed by BASE_BACKUP"; -my $connstr = $node_primary->connstr('postgres') . " replication=database"; - -# This test requires a replication connection with a database, as it mixes -# a replication command and a SQL command. -$node_primary->command_fails_like( - [ - 'psql', - '--no-psqlrc', - '--command' => "SELECT pg_backup_start('backup', true)", - '--command' => 'BASE_BACKUP', - '--dbname' => $connstr - ], - qr/a backup is already in progress in this session/, - 'BASE_BACKUP cannot run in session already running backup'); - -note "testing BASE_BACKUP cancellation"; - -my $sigchld_bb_timeout = - IPC::Run::timer($PostgreSQL::Test::Utils::timeout_default); - -# This test requires a replication connection with a database, as it mixes -# a replication command and a SQL command. The first BASE_BACKUP is throttled -# to give enough room for the cancellation running below. The second command -# for pg_backup_stop() should fail. -my ($sigchld_bb_stdin, $sigchld_bb_stdout, $sigchld_bb_stderr) = ('', '', ''); -my $sigchld_bb = IPC::Run::start( - [ - 'psql', '--no-psqlrc', - '--command' => "BASE_BACKUP (CHECKPOINT 'fast', MAX_RATE 32);", - '--command' => 'SELECT pg_backup_stop()', - '--dbname' => $connstr - ], - '<' => \$sigchld_bb_stdin, - '>' => \$sigchld_bb_stdout, - '2>' => \$sigchld_bb_stderr, - $sigchld_bb_timeout); - -# The cancellation is issued once the database files are streamed and -# the checkpoint issued at backup start completes. -is( $node_primary->poll_query_until( - 'postgres', - "SELECT pg_cancel_backend(a.pid) FROM " - . "pg_stat_activity a, pg_stat_progress_basebackup b WHERE " - . "a.pid = b.pid AND a.query ~ 'BASE_BACKUP' AND " - . "b.phase = 'streaming database files';"), - "1", - "WAL sender sending base backup killed"); - -# The psql command should fail on pg_backup_stop(). -ok( pump_until( - $sigchld_bb, $sigchld_bb_timeout, - \$sigchld_bb_stderr, qr/backup is not in progress/), - 'base backup cleanly canceled'); -$sigchld_bb->finish(); - -done_testing(); From ba5459e3a259827b88b0f98f4eb48c3c42851108 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Sat, 20 Jun 2026 10:15:29 -0400 Subject: [PATCH 10/14] python tests: convert subscription 001_rep_changes to pytest Convert the logical-replication change-propagation test as a representative multi-node logical-replication example. Add a 'pytest' block to src/test/subscription/meson.build for pyt/test_001_rep_changes.py and drop the corresponding t/001_rep_changes.pl from the 'tap' list; the remaining subscription Perl tests are unchanged. --- src/test/subscription/meson.build | 10 +- .../subscription/pyt/test_001_rep_changes.py | 572 ++++++++++++++++ src/test/subscription/t/001_rep_changes.pl | 630 ------------------ 3 files changed, 581 insertions(+), 631 deletions(-) create mode 100644 src/test/subscription/pyt/test_001_rep_changes.py delete mode 100644 src/test/subscription/t/001_rep_changes.pl diff --git a/src/test/subscription/meson.build b/src/test/subscription/meson.build index e71e95c629..63d21509df 100644 --- a/src/test/subscription/meson.build +++ b/src/test/subscription/meson.build @@ -10,7 +10,6 @@ tests += { 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', }, 'tests': [ - 't/001_rep_changes.pl', 't/002_types.pl', 't/003_constraints.pl', 't/004_sync.pl', @@ -51,4 +50,13 @@ tests += { 't/100_bugs.pl', ], }, + 'pytest': { + 'env': { + 'with_icu': icu.found() ? 'yes' : 'no', + 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', + }, + 'tests': [ + 'pyt/test_001_rep_changes.py', + ], + }, } diff --git a/src/test/subscription/pyt/test_001_rep_changes.py b/src/test/subscription/pyt/test_001_rep_changes.py new file mode 100644 index 0000000000..59a23628ac --- /dev/null +++ b/src/test/subscription/pyt/test_001_rep_changes.py @@ -0,0 +1,572 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Basic logical replication test.""" + +import re + + +def test_001_rep_changes(create_pg): + # Initialize publisher node + node_publisher = create_pg("publisher", allows_streaming="logical") + + # Create subscriber node + node_subscriber = create_pg("subscriber") + + # Create some preexisting content on publisher + node_publisher.safe_sql( + "CREATE FUNCTION public.pg_get_replica_identity_index(int)\n" + " RETURNS regclass LANGUAGE sql AS 'SELECT 1/0'" + ) # shall not call + node_publisher.safe_sql( + "CREATE TABLE tab_notrep AS SELECT generate_series(1,10) AS a" + ) + node_publisher.safe_sql( + "CREATE TABLE tab_ins AS SELECT generate_series(1,1002) AS a" + ) + node_publisher.safe_sql( + "CREATE TABLE tab_full AS SELECT generate_series(1,10) AS a" + ) + node_publisher.safe_sql("CREATE TABLE tab_full2 (x text)") + node_publisher.safe_sql("INSERT INTO tab_full2 VALUES ('a'), ('b'), ('b')") + node_publisher.safe_sql("CREATE TABLE tab_rep (a int primary key)") + node_publisher.safe_sql( + "CREATE TABLE tab_mixed (a int primary key, b text, c numeric)" + ) + node_publisher.safe_sql("INSERT INTO tab_mixed (a, b, c) VALUES (1, 'foo', 1.1)") + node_publisher.safe_sql( + "CREATE TABLE tab_include (a int, b text, " + "CONSTRAINT covering PRIMARY KEY(a) INCLUDE(b))" + ) + node_publisher.safe_sql("CREATE TABLE tab_full_pk (a int primary key, b text)") + node_publisher.safe_sql("ALTER TABLE tab_full_pk REPLICA IDENTITY FULL") + # Let this table with REPLICA IDENTITY NOTHING, allowing only INSERT changes. + node_publisher.safe_sql("CREATE TABLE tab_nothing (a int)") + node_publisher.safe_sql("ALTER TABLE tab_nothing REPLICA IDENTITY NOTHING") + + # Replicate the changes without replica identity index + node_publisher.safe_sql("CREATE TABLE tab_no_replidentity_index(c1 int)") + node_publisher.safe_sql( + "CREATE INDEX idx_no_replidentity_index ON tab_no_replidentity_index(c1)" + ) + + # Replicate the changes without columns + node_publisher.safe_sql("CREATE TABLE tab_no_col()") + node_publisher.safe_sql("INSERT INTO tab_no_col default VALUES") + + # Setup structure on subscriber + node_subscriber.safe_sql("CREATE TABLE tab_notrep (a int)") + node_subscriber.safe_sql("CREATE TABLE tab_ins (a int)") + node_subscriber.safe_sql("CREATE TABLE tab_full (a int)") + node_subscriber.safe_sql("CREATE TABLE tab_full2 (x text)") + node_subscriber.safe_sql("CREATE TABLE tab_rep (a int primary key)") + node_subscriber.safe_sql("CREATE TABLE tab_full_pk (a int primary key, b text)") + node_subscriber.safe_sql("ALTER TABLE tab_full_pk REPLICA IDENTITY FULL") + node_subscriber.safe_sql("CREATE TABLE tab_nothing (a int)") + + # different column count and order than on publisher + node_subscriber.safe_sql( + "CREATE TABLE tab_mixed (d text default 'local', c numeric, b text, " + "a int primary key)" + ) + + # replication of the table with included index + node_subscriber.safe_sql( + "CREATE TABLE tab_include (a int, b text, " + "CONSTRAINT covering PRIMARY KEY(a) INCLUDE(b))" + ) + + # replication of the table without replica identity index + node_subscriber.safe_sql("CREATE TABLE tab_no_replidentity_index(c1 int)") + node_subscriber.safe_sql( + "CREATE INDEX idx_no_replidentity_index ON tab_no_replidentity_index(c1)" + ) + + # replication of the table without columns + node_subscriber.safe_sql("CREATE TABLE tab_no_col()") + + # Setup logical replication + publisher_connstr = ( + f"host={node_publisher.host} port={node_publisher.port} dbname=postgres" + ) + node_publisher.safe_sql("CREATE PUBLICATION tap_pub") + node_publisher.safe_sql( + "CREATE PUBLICATION tap_pub_ins_only WITH (publish = insert)" + ) + node_publisher.safe_sql( + "ALTER PUBLICATION tap_pub ADD TABLE tab_rep, tab_full, tab_full2, " + "tab_mixed, tab_include, tab_nothing, tab_full_pk, " + "tab_no_replidentity_index, tab_no_col" + ) + node_publisher.safe_sql("ALTER PUBLICATION tap_pub_ins_only ADD TABLE tab_ins") + + node_subscriber.safe_sql( + f"CREATE SUBSCRIPTION tap_sub CONNECTION '{publisher_connstr}' " + "PUBLICATION tap_pub, tap_pub_ins_only" + ) + + # Wait for initial table sync to finish + node_subscriber.wait_for_subscription_sync(node_publisher, "tap_sub") + + # Reset IO statistics, for the WAL sender check with pg_stat_io. + node_publisher.safe_sql("SELECT pg_stat_reset_shared('io')") + + result = node_subscriber.safe_sql("SELECT count(*) FROM tab_notrep") + assert result == "0", "check non-replicated table is empty on subscriber" + + result = node_subscriber.safe_sql("SELECT count(*) FROM tab_ins") + assert result == "1002", "check initial data was copied to subscriber" + + node_publisher.safe_sql("INSERT INTO tab_ins SELECT generate_series(1,50)") + node_publisher.safe_sql("DELETE FROM tab_ins WHERE a > 20") + node_publisher.safe_sql("UPDATE tab_ins SET a = -a") + + node_publisher.safe_sql("INSERT INTO tab_rep SELECT generate_series(1,50)") + node_publisher.safe_sql("DELETE FROM tab_rep WHERE a > 20") + node_publisher.safe_sql("UPDATE tab_rep SET a = -a") + + node_publisher.safe_sql("INSERT INTO tab_mixed VALUES (2, 'bar', 2.2)") + + node_publisher.safe_sql("INSERT INTO tab_full_pk VALUES (1, 'foo'), (2, 'baz')") + + node_publisher.safe_sql("INSERT INTO tab_nothing VALUES (generate_series(1,20))") + + node_publisher.safe_sql("INSERT INTO tab_include SELECT generate_series(1,50)") + node_publisher.safe_sql("DELETE FROM tab_include WHERE a > 20") + node_publisher.safe_sql("UPDATE tab_include SET a = -a") + + node_publisher.safe_sql("INSERT INTO tab_no_replidentity_index VALUES(1)") + + node_publisher.safe_sql("INSERT INTO tab_no_col default VALUES") + + node_publisher.wait_for_catchup("tap_sub") + + result = node_subscriber.safe_sql("SELECT count(*), min(a), max(a) FROM tab_ins") + assert result == "1052|1|1002", "check replicated inserts on subscriber" + + result = node_subscriber.safe_sql("SELECT count(*), min(a), max(a) FROM tab_rep") + assert result == "20|-20|-1", "check replicated changes on subscriber" + + result = node_subscriber.safe_sql("SELECT * FROM tab_mixed") + assert ( + result == "local|1.1|foo|1\nlocal|2.2|bar|2" + ), "check replicated changes with different column order" + + result = node_subscriber.safe_sql("SELECT count(*) FROM tab_nothing") + assert result == "20", "check replicated changes with REPLICA IDENTITY NOTHING" + + result = node_subscriber.safe_sql( + "SELECT count(*), min(a), max(a) FROM tab_include" + ) + assert ( + result == "20|-20|-1" + ), "check replicated changes with primary key index with included columns" + + assert ( + node_subscriber.safe_sql("SELECT c1 FROM tab_no_replidentity_index") == "1" + ), "value replicated to subscriber without replica identity index" + + result = node_subscriber.safe_sql("SELECT count(*) FROM tab_no_col") + assert result == "2", "check replicated changes for table having no columns" + + # Wait for the logical WAL sender to update its IO statistics. This is + # done before the next restart, which would force a flush of its stats, and + # far enough from the reset done above to not impact the run time. + assert node_publisher.poll_query_until( + "SELECT sum(reads) > 0\n" + " FROM pg_catalog.pg_stat_io\n" + " WHERE backend_type = 'walsender'\n" + " AND object = 'wal'" + ), "Timed out while waiting for the walsender to update its IO statistics" + + # insert some duplicate rows + node_publisher.safe_sql("INSERT INTO tab_full SELECT generate_series(1,10)") + + # Test behaviour of ALTER PUBLICATION ... DROP TABLE + # + # When a publisher drops a table from publication, it should also stop + # sending its changes to subscribers. We look at the subscriber whether it + # receives the row that is inserted to the table on the publisher after it + # is dropped from the publication. + result = node_subscriber.safe_sql("SELECT count(*), min(a), max(a) FROM tab_ins") + assert ( + result == "1052|1|1002" + ), "check rows on subscriber before table drop from publication" + + # Drop the table from publication + node_publisher.safe_sql("ALTER PUBLICATION tap_pub_ins_only DROP TABLE tab_ins") + + # Insert a row in publisher, but publisher will not send this row to + # subscriber + node_publisher.safe_sql("INSERT INTO tab_ins VALUES(8888)") + + node_publisher.wait_for_catchup("tap_sub") + + # Subscriber will not receive the inserted row, after table is dropped from + # publication, so row count should remain the same. + result = node_subscriber.safe_sql("SELECT count(*), min(a), max(a) FROM tab_ins") + assert ( + result == "1052|1|1002" + ), "check rows on subscriber after table drop from publication" + + # Delete the inserted row in publisher + node_publisher.safe_sql("DELETE FROM tab_ins WHERE a = 8888") + + # Add the table to publication again + node_publisher.safe_sql("ALTER PUBLICATION tap_pub_ins_only ADD TABLE tab_ins") + + # Refresh publication after table is added to publication + node_subscriber.safe_sql("ALTER SUBSCRIPTION tap_sub REFRESH PUBLICATION") + + # Test replication with multiple publications for a subscription such that + # the operations are performed on the table from the first publication in + # the list. + + # Create tables on publisher + node_publisher.safe_sql("CREATE TABLE temp1 (a int)") + node_publisher.safe_sql("CREATE TABLE temp2 (a int)") + + # Create tables on subscriber + node_subscriber.safe_sql("CREATE TABLE temp1 (a int)") + node_subscriber.safe_sql("CREATE TABLE temp2 (a int)") + + # Setup logical replication that will only be used for this test + node_publisher.safe_sql( + "CREATE PUBLICATION tap_pub_temp1 FOR TABLE temp1 WITH (publish = insert)" + ) + node_publisher.safe_sql("CREATE PUBLICATION tap_pub_temp2 FOR TABLE temp2") + node_subscriber.safe_sql( + f"CREATE SUBSCRIPTION tap_sub_temp1 CONNECTION '{publisher_connstr}' " + "PUBLICATION tap_pub_temp1, tap_pub_temp2" + ) + + # Wait for initial table sync to finish + node_subscriber.wait_for_subscription_sync(node_publisher, "tap_sub_temp1") + + # Subscriber table will have no rows initially + result = node_subscriber.safe_sql("SELECT count(*) FROM temp1") + assert result == "0", "check initial rows on subscriber with multiple publications" + + # Insert a row into the table that's part of first publication in + # subscriber list of publications. + node_publisher.safe_sql("INSERT INTO temp1 VALUES (1)") + + node_publisher.wait_for_catchup("tap_sub_temp1") + + # Subscriber should receive the inserted row + result = node_subscriber.safe_sql("SELECT count(*) FROM temp1") + assert result == "1", "check rows on subscriber with multiple publications" + + # Drop subscription as we don't need it anymore + node_subscriber.safe_sql("DROP SUBSCRIPTION tap_sub_temp1") + + # Drop publications as we don't need them anymore + node_publisher.safe_sql("DROP PUBLICATION tap_pub_temp1") + node_publisher.safe_sql("DROP PUBLICATION tap_pub_temp2") + + # Clean up the tables on both publisher and subscriber as we don't need them + node_publisher.safe_sql("DROP TABLE temp1") + node_publisher.safe_sql("DROP TABLE temp2") + node_subscriber.safe_sql("DROP TABLE temp1") + node_subscriber.safe_sql("DROP TABLE temp2") + + # add REPLICA IDENTITY FULL so we can update + node_publisher.safe_sql("ALTER TABLE tab_full REPLICA IDENTITY FULL") + node_subscriber.safe_sql("ALTER TABLE tab_full REPLICA IDENTITY FULL") + node_publisher.safe_sql("ALTER TABLE tab_full2 REPLICA IDENTITY FULL") + node_subscriber.safe_sql("ALTER TABLE tab_full2 REPLICA IDENTITY FULL") + node_publisher.safe_sql("ALTER TABLE tab_ins REPLICA IDENTITY FULL") + node_subscriber.safe_sql("ALTER TABLE tab_ins REPLICA IDENTITY FULL") + # tab_mixed can use DEFAULT, since it has a primary key + + # and do the updates + node_publisher.safe_sql("UPDATE tab_full SET a = a * a") + node_publisher.safe_sql("UPDATE tab_full2 SET x = 'bb' WHERE x = 'b'") + node_publisher.safe_sql("UPDATE tab_mixed SET b = 'baz' WHERE a = 1") + node_publisher.safe_sql("UPDATE tab_full_pk SET b = 'bar' WHERE a = 1") + + node_publisher.wait_for_catchup("tap_sub") + + result = node_subscriber.safe_sql("SELECT count(*), min(a), max(a) FROM tab_full") + assert ( + result == "20|1|100" + ), "update works with REPLICA IDENTITY FULL and duplicate tuples" + + result = node_subscriber.safe_sql("SELECT x FROM tab_full2 ORDER BY 1") + assert ( + result == "a\nbb\nbb" + ), "update works with REPLICA IDENTITY FULL and text datums" + + result = node_subscriber.safe_sql("SELECT * FROM tab_mixed ORDER BY a") + assert ( + result == "local|1.1|baz|1\nlocal|2.2|bar|2" + ), "update works with different column order and subscriber local values" + + result = node_subscriber.safe_sql("SELECT * FROM tab_full_pk ORDER BY a") + assert ( + result == "1|bar\n2|baz" + ), "update works with REPLICA IDENTITY FULL and a primary key" + + node_subscriber.safe_sql("DELETE FROM tab_full_pk") + node_subscriber.safe_sql("DELETE FROM tab_full WHERE a = 25") + + # Note that the current location of the log file is not grabbed immediately + # after reloading the configuration, but after sending one SQL command to + # the node so as we are sure that the reloading has taken effect. + log_location_pub = node_publisher.log_position() + log_location_sub = node_subscriber.log_position() + + node_publisher.safe_sql("UPDATE tab_full_pk SET b = 'quux' WHERE a = 1") + node_publisher.safe_sql("UPDATE tab_full SET a = a + 1 WHERE a = 25") + node_publisher.safe_sql("DELETE FROM tab_full_pk WHERE a = 2") + + node_publisher.wait_for_catchup("tap_sub") + + logfile = node_subscriber.log_content()[log_location_sub:] + assert re.search( + r'conflict detected on relation "public.tab_full_pk": ' + r"conflict=update_missing.*\n.*DETAIL:.* Could not find the row to be " + r"updated: remote row \(1, quux\), replica identity \(a\)=\(1\)", + logfile, + ), "update target row is missing" + assert re.search( + r'conflict detected on relation "public.tab_full": ' + r"conflict=update_missing.*\n.*DETAIL:.* Could not find the row to be " + r"updated: remote row \(26\), replica identity full \(25\)", + logfile, + ), "update target row is missing" + assert re.search( + r'conflict detected on relation "public.tab_full_pk": ' + r"conflict=delete_missing.*\n.*DETAIL:.* Could not find the row to be " + r"deleted: replica identity \(a\)=\(2\)", + logfile, + ), "delete target row is missing" + + node_subscriber.append_conf("log_min_messages = warning") + node_subscriber.reload() + + # check behavior with toasted values + + node_publisher.safe_sql( + "UPDATE tab_mixed SET b = repeat('xyzzy', 100000) WHERE a = 2" + ) + + node_publisher.wait_for_catchup("tap_sub") + + result = node_subscriber.safe_sql( + "SELECT a, length(b), c, d FROM tab_mixed ORDER BY a" + ) + assert ( + result == "1|3|1.1|local\n2|500000|2.2|local" + ), "update transmits large column value" + + node_publisher.safe_sql("UPDATE tab_mixed SET c = 3.3 WHERE a = 2") + + node_publisher.wait_for_catchup("tap_sub") + + result = node_subscriber.safe_sql( + "SELECT a, length(b), c, d FROM tab_mixed ORDER BY a" + ) + assert ( + result == "1|3|1.1|local\n2|500000|3.3|local" + ), "update with non-transmitted large column value" + + # check behavior with dropped columns + + # this update should get transmitted before the column goes away + node_publisher.safe_sql("UPDATE tab_mixed SET b = 'bar', c = 2.2 WHERE a = 2") + + node_publisher.safe_sql("ALTER TABLE tab_mixed DROP COLUMN b") + + node_publisher.safe_sql("UPDATE tab_mixed SET c = 11.11 WHERE a = 1") + + node_publisher.wait_for_catchup("tap_sub") + + result = node_subscriber.safe_sql("SELECT * FROM tab_mixed ORDER BY a") + assert ( + result == "local|11.11|baz|1\nlocal|2.2|bar|2" + ), "update works with dropped publisher column" + + node_subscriber.safe_sql("ALTER TABLE tab_mixed DROP COLUMN d") + + node_publisher.safe_sql("UPDATE tab_mixed SET c = 22.22 WHERE a = 2") + + node_publisher.wait_for_catchup("tap_sub") + + result = node_subscriber.safe_sql("SELECT * FROM tab_mixed ORDER BY a") + assert ( + result == "11.11|baz|1\n22.22|bar|2" + ), "update works with dropped subscriber column" + + # Verify that GUC settings supplied in the CONNECTION string take effect on + # the publisher's walsender. We do this by enabling log_statement_stats in + # the CONNECTION string later and checking that the publisher's log contains + # a QUERY STATISTICS message. + # + # First, confirm that no such QUERY STATISTICS message appears before + # enabling log_statement_stats. + logfile = node_publisher.log_content()[log_location_pub:] + assert not re.search( + r"QUERY STATISTICS", logfile + ), "log_statement_stats has not been enabled yet" + log_location_pub = node_publisher.log_position() + + # check that change of connection string and/or publication list causes + # restart of subscription workers. We check the state along with + # application_name to ensure that the walsender is (re)started. + # + # Not all of these are registered as tests as we need to poll for a change + # but the test suite will fail nonetheless when something goes wrong. + # + # Enable log_statement_stats as the change of connection string, + # which is also for the above mentioned test of GUC settings passed through + # CONNECTION. + oldpid = node_publisher.safe_sql( + "SELECT pid FROM pg_stat_replication WHERE application_name = 'tap_sub' " + "AND state = 'streaming';" + ) + node_subscriber.safe_sql( + f"ALTER SUBSCRIPTION tap_sub CONNECTION '{publisher_connstr} " + "options=''-c log_statement_stats=on'''" + ) + assert node_publisher.poll_query_until( + f"SELECT pid != {oldpid} FROM pg_stat_replication " + "WHERE application_name = 'tap_sub' AND state = 'streaming';" + ), "Timed out while waiting for apply to restart after changing CONNECTION" + + # Check that the expected QUERY STATISTICS message appears, + # which shows that log_statement_stats=on from the CONNECTION string + # was correctly passed through to and honored by the walsender. + logfile = node_publisher.log_content()[log_location_pub:] + assert re.search(r"QUERY STATISTICS", logfile), ( + "log_statement_stats in CONNECTION string had effect on publisher's " + "walsender" + ) + + oldpid = node_publisher.safe_sql( + "SELECT pid FROM pg_stat_replication WHERE application_name = 'tap_sub' " + "AND state = 'streaming';" + ) + node_subscriber.safe_sql( + "ALTER SUBSCRIPTION tap_sub SET PUBLICATION tap_pub_ins_only " + "WITH (copy_data = false)" + ) + assert node_publisher.poll_query_until( + f"SELECT pid != {oldpid} FROM pg_stat_replication " + "WHERE application_name = 'tap_sub' AND state = 'streaming';" + ), "Timed out while waiting for apply to restart after changing PUBLICATION" + + node_publisher.safe_sql("INSERT INTO tab_ins SELECT generate_series(1001,1100)") + node_publisher.safe_sql("DELETE FROM tab_rep") + + # Restart the publisher and check the state of the subscriber which + # should be in a streaming state after catching up. + node_publisher.stop("fast") + node_publisher.start() + + node_publisher.wait_for_catchup("tap_sub") + + result = node_subscriber.safe_sql("SELECT count(*), min(a), max(a) FROM tab_ins") + assert ( + result == "1152|1|1100" + ), "check replicated inserts after subscription publication change" + + result = node_subscriber.safe_sql("SELECT count(*), min(a), max(a) FROM tab_rep") + assert ( + result == "20|-20|-1" + ), "check changes skipped after subscription publication change" + + # check alter publication (relcache invalidation etc) + node_publisher.safe_sql( + "ALTER PUBLICATION tap_pub_ins_only SET (publish = 'insert, delete')" + ) + node_publisher.safe_sql("ALTER PUBLICATION tap_pub_ins_only ADD TABLE tab_full") + node_publisher.safe_sql("DELETE FROM tab_ins WHERE a > 0") + node_subscriber.safe_sql( + "ALTER SUBSCRIPTION tap_sub REFRESH PUBLICATION WITH (copy_data = false)" + ) + node_publisher.safe_sql("INSERT INTO tab_full VALUES(0)") + + node_publisher.wait_for_catchup("tap_sub") + + # Check that we don't send BEGIN and COMMIT because of empty transaction + # optimization. We have to look for the DEBUG1 log messages about that, so + # temporarily bump up the log verbosity. + node_publisher.append_conf("log_min_messages = debug1") + node_publisher.reload() + + # Note that the current location of the log file is not grabbed immediately + # after reloading the configuration, but after sending one SQL command to + # the node so that we are sure that the reloading has taken effect. + log_location_pub = node_publisher.log_position() + + node_publisher.safe_sql("INSERT INTO tab_notrep VALUES (11)") + + node_publisher.wait_for_catchup("tap_sub") + + # Poll for the DEBUG1 message: the walsender reprocesses the reloaded + # log_min_messages between decoding loops, so the message can lag the + # catchup slightly. + node_publisher.wait_for_log( + r"skipped replication of an empty transaction with XID", log_location_pub + ) + + result = node_subscriber.safe_sql("SELECT count(*) FROM tab_notrep") + assert result == "0", "check non-replicated table is empty on subscriber" + + node_publisher.append_conf("log_min_messages = warning") + node_publisher.reload() + + # note that data are different on provider and subscriber + result = node_subscriber.safe_sql("SELECT count(*), min(a), max(a) FROM tab_ins") + assert result == "1052|1|1002", "check replicated deletes after alter publication" + + result = node_subscriber.safe_sql("SELECT count(*), min(a), max(a) FROM tab_full") + assert result == "19|0|100", "check replicated insert after alter publication" + + # check restart on rename + oldpid = node_publisher.safe_sql( + "SELECT pid FROM pg_stat_replication WHERE application_name = 'tap_sub' " + "AND state = 'streaming';" + ) + node_subscriber.safe_sql("ALTER SUBSCRIPTION tap_sub RENAME TO tap_sub_renamed") + assert node_publisher.poll_query_until( + f"SELECT pid != {oldpid} FROM pg_stat_replication " + "WHERE application_name = 'tap_sub_renamed' AND state = 'streaming';" + ), "Timed out while waiting for apply to restart after renaming SUBSCRIPTION" + + # check all the cleanup + node_subscriber.safe_sql("DROP SUBSCRIPTION tap_sub_renamed") + + result = node_subscriber.safe_sql("SELECT count(*) FROM pg_subscription") + assert result == "0", "check subscription was dropped on subscriber" + + result = node_publisher.safe_sql("SELECT count(*) FROM pg_replication_slots") + assert result == "0", "check replication slot was dropped on publisher" + + result = node_subscriber.safe_sql("SELECT count(*) FROM pg_subscription_rel") + assert result == "0", "check subscription relation status was dropped on subscriber" + + result = node_publisher.safe_sql("SELECT count(*) FROM pg_replication_slots") + assert result == "0", "check replication slot was dropped on publisher" + + result = node_subscriber.safe_sql("SELECT count(*) FROM pg_replication_origin") + assert result == "0", "check replication origin was dropped on subscriber" + + node_subscriber.stop("fast") + node_publisher.stop("fast") + + # CREATE PUBLICATION while wal_level=minimal should succeed, with a WARNING + node_publisher.append_conf("\nwal_level=minimal\nmax_wal_senders=0\n") + node_publisher.start() + sess = node_publisher.connect() + try: + sess.query("BEGIN") + sess.query("CREATE TABLE skip_wal()") + sess.query("CREATE PUBLICATION tap_pub2 FOR TABLE skip_wal") + sess.query("ROLLBACK") + reterr = sess.get_notices_str() + finally: + sess.close() + assert re.search( + r"WARNING: logical decoding must be enabled to publish logical changes", reterr + ), 'CREATE PUBLICATION while "wal_level=minimal"' diff --git a/src/test/subscription/t/001_rep_changes.pl b/src/test/subscription/t/001_rep_changes.pl deleted file mode 100644 index 7d41715ed8..0000000000 --- a/src/test/subscription/t/001_rep_changes.pl +++ /dev/null @@ -1,630 +0,0 @@ - -# Copyright (c) 2021-2026, PostgreSQL Global Development Group - -# Basic logical replication test -use strict; -use warnings FATAL => 'all'; -use PostgreSQL::Test::Cluster; -use PostgreSQL::Test::Utils; -use Test::More; - -# Initialize publisher node -my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); -$node_publisher->init(allows_streaming => 'logical'); -$node_publisher->start; - -# Create subscriber node -my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); -$node_subscriber->init; -$node_subscriber->start; - -# Create some preexisting content on publisher -$node_publisher->safe_psql( - 'postgres', - "CREATE FUNCTION public.pg_get_replica_identity_index(int) - RETURNS regclass LANGUAGE sql AS 'SELECT 1/0'"); # shall not call -$node_publisher->safe_psql('postgres', - "CREATE TABLE tab_notrep AS SELECT generate_series(1,10) AS a"); -$node_publisher->safe_psql('postgres', - "CREATE TABLE tab_ins AS SELECT generate_series(1,1002) AS a"); -$node_publisher->safe_psql('postgres', - "CREATE TABLE tab_full AS SELECT generate_series(1,10) AS a"); -$node_publisher->safe_psql('postgres', "CREATE TABLE tab_full2 (x text)"); -$node_publisher->safe_psql('postgres', - "INSERT INTO tab_full2 VALUES ('a'), ('b'), ('b')"); -$node_publisher->safe_psql('postgres', - "CREATE TABLE tab_rep (a int primary key)"); -$node_publisher->safe_psql('postgres', - "CREATE TABLE tab_mixed (a int primary key, b text, c numeric)"); -$node_publisher->safe_psql('postgres', - "INSERT INTO tab_mixed (a, b, c) VALUES (1, 'foo', 1.1)"); -$node_publisher->safe_psql('postgres', - "CREATE TABLE tab_include (a int, b text, CONSTRAINT covering PRIMARY KEY(a) INCLUDE(b))" -); -$node_publisher->safe_psql('postgres', - "CREATE TABLE tab_full_pk (a int primary key, b text)"); -$node_publisher->safe_psql('postgres', - "ALTER TABLE tab_full_pk REPLICA IDENTITY FULL"); -# Let this table with REPLICA IDENTITY NOTHING, allowing only INSERT changes. -$node_publisher->safe_psql('postgres', "CREATE TABLE tab_nothing (a int)"); -$node_publisher->safe_psql('postgres', - "ALTER TABLE tab_nothing REPLICA IDENTITY NOTHING"); - -# Replicate the changes without replica identity index -$node_publisher->safe_psql('postgres', - "CREATE TABLE tab_no_replidentity_index(c1 int)"); -$node_publisher->safe_psql('postgres', - "CREATE INDEX idx_no_replidentity_index ON tab_no_replidentity_index(c1)" -); - -# Replicate the changes without columns -$node_publisher->safe_psql('postgres', "CREATE TABLE tab_no_col()"); -$node_publisher->safe_psql('postgres', - "INSERT INTO tab_no_col default VALUES"); - -# Setup structure on subscriber -$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_notrep (a int)"); -$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_ins (a int)"); -$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_full (a int)"); -$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_full2 (x text)"); -$node_subscriber->safe_psql('postgres', - "CREATE TABLE tab_rep (a int primary key)"); -$node_subscriber->safe_psql('postgres', - "CREATE TABLE tab_full_pk (a int primary key, b text)"); -$node_subscriber->safe_psql('postgres', - "ALTER TABLE tab_full_pk REPLICA IDENTITY FULL"); -$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_nothing (a int)"); - -# different column count and order than on publisher -$node_subscriber->safe_psql('postgres', - "CREATE TABLE tab_mixed (d text default 'local', c numeric, b text, a int primary key)" -); - -# replication of the table with included index -$node_subscriber->safe_psql('postgres', - "CREATE TABLE tab_include (a int, b text, CONSTRAINT covering PRIMARY KEY(a) INCLUDE(b))" -); - -# replication of the table without replica identity index -$node_subscriber->safe_psql('postgres', - "CREATE TABLE tab_no_replidentity_index(c1 int)"); -$node_subscriber->safe_psql('postgres', - "CREATE INDEX idx_no_replidentity_index ON tab_no_replidentity_index(c1)" -); - -# replication of the table without columns -$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_no_col()"); - -# Setup logical replication -my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; -$node_publisher->safe_psql('postgres', "CREATE PUBLICATION tap_pub"); -$node_publisher->safe_psql('postgres', - "CREATE PUBLICATION tap_pub_ins_only WITH (publish = insert)"); -$node_publisher->safe_psql('postgres', - "ALTER PUBLICATION tap_pub ADD TABLE tab_rep, tab_full, tab_full2, tab_mixed, tab_include, tab_nothing, tab_full_pk, tab_no_replidentity_index, tab_no_col" -); -$node_publisher->safe_psql('postgres', - "ALTER PUBLICATION tap_pub_ins_only ADD TABLE tab_ins"); - -$node_subscriber->safe_psql('postgres', - "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub, tap_pub_ins_only" -); - -# Wait for initial table sync to finish -$node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub'); - -# Reset IO statistics, for the WAL sender check with pg_stat_io. -$node_publisher->safe_psql('postgres', "SELECT pg_stat_reset_shared('io')"); - -my $result = - $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_notrep"); -is($result, qq(0), 'check non-replicated table is empty on subscriber'); - -$result = - $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_ins"); -is($result, qq(1002), 'check initial data was copied to subscriber'); - -$node_publisher->safe_psql('postgres', - "INSERT INTO tab_ins SELECT generate_series(1,50)"); -$node_publisher->safe_psql('postgres', "DELETE FROM tab_ins WHERE a > 20"); -$node_publisher->safe_psql('postgres', "UPDATE tab_ins SET a = -a"); - -$node_publisher->safe_psql('postgres', - "INSERT INTO tab_rep SELECT generate_series(1,50)"); -$node_publisher->safe_psql('postgres', "DELETE FROM tab_rep WHERE a > 20"); -$node_publisher->safe_psql('postgres', "UPDATE tab_rep SET a = -a"); - -$node_publisher->safe_psql('postgres', - "INSERT INTO tab_mixed VALUES (2, 'bar', 2.2)"); - -$node_publisher->safe_psql('postgres', - "INSERT INTO tab_full_pk VALUES (1, 'foo'), (2, 'baz')"); - -$node_publisher->safe_psql('postgres', - "INSERT INTO tab_nothing VALUES (generate_series(1,20))"); - -$node_publisher->safe_psql('postgres', - "INSERT INTO tab_include SELECT generate_series(1,50)"); -$node_publisher->safe_psql('postgres', - "DELETE FROM tab_include WHERE a > 20"); -$node_publisher->safe_psql('postgres', "UPDATE tab_include SET a = -a"); - -$node_publisher->safe_psql('postgres', - "INSERT INTO tab_no_replidentity_index VALUES(1)"); - -$node_publisher->safe_psql('postgres', - "INSERT INTO tab_no_col default VALUES"); - -$node_publisher->wait_for_catchup('tap_sub'); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT count(*), min(a), max(a) FROM tab_ins"); -is($result, qq(1052|1|1002), 'check replicated inserts on subscriber'); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT count(*), min(a), max(a) FROM tab_rep"); -is($result, qq(20|-20|-1), 'check replicated changes on subscriber'); - -$result = $node_subscriber->safe_psql('postgres', "SELECT * FROM tab_mixed"); -is( $result, qq(local|1.1|foo|1 -local|2.2|bar|2), 'check replicated changes with different column order'); - -$result = - $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_nothing"); -is($result, qq(20), 'check replicated changes with REPLICA IDENTITY NOTHING'); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT count(*), min(a), max(a) FROM tab_include"); -is($result, qq(20|-20|-1), - 'check replicated changes with primary key index with included columns'); - -is( $node_subscriber->safe_psql( - 'postgres', q(SELECT c1 FROM tab_no_replidentity_index)), - 1, - "value replicated to subscriber without replica identity index"); - -$result = - $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_no_col"); -is($result, qq(2), 'check replicated changes for table having no columns'); - -# Wait for the logical WAL sender to update its IO statistics. This is -# done before the next restart, which would force a flush of its stats, and -# far enough from the reset done above to not impact the run time. -$node_publisher->poll_query_until( - 'postgres', - qq[SELECT sum(reads) > 0 - FROM pg_catalog.pg_stat_io - WHERE backend_type = 'walsender' - AND object = 'wal'] - ) - or die - "Timed out while waiting for the walsender to update its IO statistics"; - -# insert some duplicate rows -$node_publisher->safe_psql('postgres', - "INSERT INTO tab_full SELECT generate_series(1,10)"); - -# Test behaviour of ALTER PUBLICATION ... DROP TABLE -# -# When a publisher drops a table from publication, it should also stop sending -# its changes to subscribers. We look at the subscriber whether it receives -# the row that is inserted to the table on the publisher after it is dropped -# from the publication. -$result = $node_subscriber->safe_psql('postgres', - "SELECT count(*), min(a), max(a) FROM tab_ins"); -is($result, qq(1052|1|1002), - 'check rows on subscriber before table drop from publication'); - -# Drop the table from publication -$node_publisher->safe_psql('postgres', - "ALTER PUBLICATION tap_pub_ins_only DROP TABLE tab_ins"); - -# Insert a row in publisher, but publisher will not send this row to subscriber -$node_publisher->safe_psql('postgres', "INSERT INTO tab_ins VALUES(8888)"); - -$node_publisher->wait_for_catchup('tap_sub'); - -# Subscriber will not receive the inserted row, after table is dropped from -# publication, so row count should remain the same. -$result = $node_subscriber->safe_psql('postgres', - "SELECT count(*), min(a), max(a) FROM tab_ins"); -is($result, qq(1052|1|1002), - 'check rows on subscriber after table drop from publication'); - -# Delete the inserted row in publisher -$node_publisher->safe_psql('postgres', "DELETE FROM tab_ins WHERE a = 8888"); - -# Add the table to publication again -$node_publisher->safe_psql('postgres', - "ALTER PUBLICATION tap_pub_ins_only ADD TABLE tab_ins"); - -# Refresh publication after table is added to publication -$node_subscriber->safe_psql('postgres', - "ALTER SUBSCRIPTION tap_sub REFRESH PUBLICATION"); - -# Test replication with multiple publications for a subscription such that the -# operations are performed on the table from the first publication in the list. - -# Create tables on publisher -$node_publisher->safe_psql('postgres', "CREATE TABLE temp1 (a int)"); -$node_publisher->safe_psql('postgres', "CREATE TABLE temp2 (a int)"); - -# Create tables on subscriber -$node_subscriber->safe_psql('postgres', "CREATE TABLE temp1 (a int)"); -$node_subscriber->safe_psql('postgres', "CREATE TABLE temp2 (a int)"); - -# Setup logical replication that will only be used for this test -$node_publisher->safe_psql('postgres', - "CREATE PUBLICATION tap_pub_temp1 FOR TABLE temp1 WITH (publish = insert)" -); -$node_publisher->safe_psql('postgres', - "CREATE PUBLICATION tap_pub_temp2 FOR TABLE temp2"); -$node_subscriber->safe_psql('postgres', - "CREATE SUBSCRIPTION tap_sub_temp1 CONNECTION '$publisher_connstr' PUBLICATION tap_pub_temp1, tap_pub_temp2" -); - -# Wait for initial table sync to finish -$node_subscriber->wait_for_subscription_sync($node_publisher, - 'tap_sub_temp1'); - -# Subscriber table will have no rows initially -$result = - $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM temp1"); -is($result, qq(0), - 'check initial rows on subscriber with multiple publications'); - -# Insert a row into the table that's part of first publication in subscriber -# list of publications. -$node_publisher->safe_psql('postgres', "INSERT INTO temp1 VALUES (1)"); - -$node_publisher->wait_for_catchup('tap_sub_temp1'); - -# Subscriber should receive the inserted row -$result = - $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM temp1"); -is($result, qq(1), 'check rows on subscriber with multiple publications'); - -# Drop subscription as we don't need it anymore -$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub_temp1"); - -# Drop publications as we don't need them anymore -$node_publisher->safe_psql('postgres', "DROP PUBLICATION tap_pub_temp1"); -$node_publisher->safe_psql('postgres', "DROP PUBLICATION tap_pub_temp2"); - -# Clean up the tables on both publisher and subscriber as we don't need them -$node_publisher->safe_psql('postgres', "DROP TABLE temp1"); -$node_publisher->safe_psql('postgres', "DROP TABLE temp2"); -$node_subscriber->safe_psql('postgres', "DROP TABLE temp1"); -$node_subscriber->safe_psql('postgres', "DROP TABLE temp2"); - -# add REPLICA IDENTITY FULL so we can update -$node_publisher->safe_psql('postgres', - "ALTER TABLE tab_full REPLICA IDENTITY FULL"); -$node_subscriber->safe_psql('postgres', - "ALTER TABLE tab_full REPLICA IDENTITY FULL"); -$node_publisher->safe_psql('postgres', - "ALTER TABLE tab_full2 REPLICA IDENTITY FULL"); -$node_subscriber->safe_psql('postgres', - "ALTER TABLE tab_full2 REPLICA IDENTITY FULL"); -$node_publisher->safe_psql('postgres', - "ALTER TABLE tab_ins REPLICA IDENTITY FULL"); -$node_subscriber->safe_psql('postgres', - "ALTER TABLE tab_ins REPLICA IDENTITY FULL"); -# tab_mixed can use DEFAULT, since it has a primary key - -# and do the updates -$node_publisher->safe_psql('postgres', "UPDATE tab_full SET a = a * a"); -$node_publisher->safe_psql('postgres', - "UPDATE tab_full2 SET x = 'bb' WHERE x = 'b'"); -$node_publisher->safe_psql('postgres', - "UPDATE tab_mixed SET b = 'baz' WHERE a = 1"); -$node_publisher->safe_psql('postgres', - "UPDATE tab_full_pk SET b = 'bar' WHERE a = 1"); - -$node_publisher->wait_for_catchup('tap_sub'); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT count(*), min(a), max(a) FROM tab_full"); -is($result, qq(20|1|100), - 'update works with REPLICA IDENTITY FULL and duplicate tuples'); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT x FROM tab_full2 ORDER BY 1"); -is( $result, qq(a -bb -bb), - 'update works with REPLICA IDENTITY FULL and text datums'); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT * FROM tab_mixed ORDER BY a"); -is( $result, qq(local|1.1|baz|1 -local|2.2|bar|2), - 'update works with different column order and subscriber local values'); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT * FROM tab_full_pk ORDER BY a"); -is( $result, qq(1|bar -2|baz), - 'update works with REPLICA IDENTITY FULL and a primary key'); - -$node_subscriber->safe_psql('postgres', "DELETE FROM tab_full_pk"); -$node_subscriber->safe_psql('postgres', "DELETE FROM tab_full WHERE a = 25"); - -# Note that the current location of the log file is not grabbed immediately -# after reloading the configuration, but after sending one SQL command to -# the node so as we are sure that the reloading has taken effect. -my $log_location_pub = -s $node_publisher->logfile; -my $log_location_sub = -s $node_subscriber->logfile; - -$node_publisher->safe_psql('postgres', - "UPDATE tab_full_pk SET b = 'quux' WHERE a = 1"); -$node_publisher->safe_psql('postgres', - "UPDATE tab_full SET a = a + 1 WHERE a = 25"); -$node_publisher->safe_psql('postgres', "DELETE FROM tab_full_pk WHERE a = 2"); - -$node_publisher->wait_for_catchup('tap_sub'); - -my $logfile = slurp_file($node_subscriber->logfile, $log_location_sub); -like( - $logfile, - qr/conflict detected on relation "public.tab_full_pk": conflict=update_missing.*\n.*DETAIL:.* Could not find the row to be updated: remote row \(1, quux\), replica identity \(a\)=\(1\)/m, - 'update target row is missing'); -like( - $logfile, - qr/conflict detected on relation "public.tab_full": conflict=update_missing.*\n.*DETAIL:.* Could not find the row to be updated: remote row \(26\), replica identity full \(25\)/m, - 'update target row is missing'); -like( - $logfile, - qr/conflict detected on relation "public.tab_full_pk": conflict=delete_missing.*\n.*DETAIL:.* Could not find the row to be deleted: replica identity \(a\)=\(2\)/m, - 'delete target row is missing'); - -$node_subscriber->append_conf('postgresql.conf', - "log_min_messages = warning"); -$node_subscriber->reload; - -# check behavior with toasted values - -$node_publisher->safe_psql('postgres', - "UPDATE tab_mixed SET b = repeat('xyzzy', 100000) WHERE a = 2"); - -$node_publisher->wait_for_catchup('tap_sub'); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT a, length(b), c, d FROM tab_mixed ORDER BY a"); -is( $result, qq(1|3|1.1|local -2|500000|2.2|local), - 'update transmits large column value'); - -$node_publisher->safe_psql('postgres', - "UPDATE tab_mixed SET c = 3.3 WHERE a = 2"); - -$node_publisher->wait_for_catchup('tap_sub'); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT a, length(b), c, d FROM tab_mixed ORDER BY a"); -is( $result, qq(1|3|1.1|local -2|500000|3.3|local), - 'update with non-transmitted large column value'); - -# check behavior with dropped columns - -# this update should get transmitted before the column goes away -$node_publisher->safe_psql('postgres', - "UPDATE tab_mixed SET b = 'bar', c = 2.2 WHERE a = 2"); - -$node_publisher->safe_psql('postgres', "ALTER TABLE tab_mixed DROP COLUMN b"); - -$node_publisher->safe_psql('postgres', - "UPDATE tab_mixed SET c = 11.11 WHERE a = 1"); - -$node_publisher->wait_for_catchup('tap_sub'); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT * FROM tab_mixed ORDER BY a"); -is( $result, qq(local|11.11|baz|1 -local|2.2|bar|2), - 'update works with dropped publisher column'); - -$node_subscriber->safe_psql('postgres', - "ALTER TABLE tab_mixed DROP COLUMN d"); - -$node_publisher->safe_psql('postgres', - "UPDATE tab_mixed SET c = 22.22 WHERE a = 2"); - -$node_publisher->wait_for_catchup('tap_sub'); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT * FROM tab_mixed ORDER BY a"); -is( $result, qq(11.11|baz|1 -22.22|bar|2), - 'update works with dropped subscriber column'); - -# Verify that GUC settings supplied in the CONNECTION string take effect on -# the publisher's walsender. We do this by enabling log_statement_stats in -# the CONNECTION string later and checking that the publisher's log contains a -# QUERY STATISTICS message. -# -# First, confirm that no such QUERY STATISTICS message appears before enabling -# log_statement_stats. -$logfile = slurp_file($node_publisher->logfile, $log_location_pub); -unlike( - $logfile, - qr/QUERY STATISTICS/, - 'log_statement_stats has not been enabled yet'); -$log_location_pub = -s $node_publisher->logfile; - -# check that change of connection string and/or publication list causes -# restart of subscription workers. We check the state along with -# application_name to ensure that the walsender is (re)started. -# -# Not all of these are registered as tests as we need to poll for a change -# but the test suite will fail nonetheless when something goes wrong. -# -# Enable log_statement_stats as the change of connection string, -# which is also for the above mentioned test of GUC settings passed through -# CONNECTION. -my $oldpid = $node_publisher->safe_psql('postgres', - "SELECT pid FROM pg_stat_replication WHERE application_name = 'tap_sub' AND state = 'streaming';" -); -$node_subscriber->safe_psql('postgres', - "ALTER SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr options=''-c log_statement_stats=on'''" -); -$node_publisher->poll_query_until('postgres', - "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = 'tap_sub' AND state = 'streaming';" - ) - or die - "Timed out while waiting for apply to restart after changing CONNECTION"; - -# Check that the expected QUERY STATISTICS message appears, -# which shows that log_statement_stats=on from the CONNECTION string -# was correctly passed through to and honored by the walsender. -$logfile = slurp_file($node_publisher->logfile, $log_location_pub); -like( - $logfile, - qr/QUERY STATISTICS/, - 'log_statement_stats in CONNECTION string had effect on publisher\'s walsender' -); - -$oldpid = $node_publisher->safe_psql('postgres', - "SELECT pid FROM pg_stat_replication WHERE application_name = 'tap_sub' AND state = 'streaming';" -); -$node_subscriber->safe_psql('postgres', - "ALTER SUBSCRIPTION tap_sub SET PUBLICATION tap_pub_ins_only WITH (copy_data = false)" -); -$node_publisher->poll_query_until('postgres', - "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = 'tap_sub' AND state = 'streaming';" - ) - or die - "Timed out while waiting for apply to restart after changing PUBLICATION"; - -$node_publisher->safe_psql('postgres', - "INSERT INTO tab_ins SELECT generate_series(1001,1100)"); -$node_publisher->safe_psql('postgres', "DELETE FROM tab_rep"); - -# Restart the publisher and check the state of the subscriber which -# should be in a streaming state after catching up. -$node_publisher->stop('fast'); -$node_publisher->start; - -$node_publisher->wait_for_catchup('tap_sub'); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT count(*), min(a), max(a) FROM tab_ins"); -is($result, qq(1152|1|1100), - 'check replicated inserts after subscription publication change'); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT count(*), min(a), max(a) FROM tab_rep"); -is($result, qq(20|-20|-1), - 'check changes skipped after subscription publication change'); - -# check alter publication (relcache invalidation etc) -$node_publisher->safe_psql('postgres', - "ALTER PUBLICATION tap_pub_ins_only SET (publish = 'insert, delete')"); -$node_publisher->safe_psql('postgres', - "ALTER PUBLICATION tap_pub_ins_only ADD TABLE tab_full"); -$node_publisher->safe_psql('postgres', "DELETE FROM tab_ins WHERE a > 0"); -$node_subscriber->safe_psql('postgres', - "ALTER SUBSCRIPTION tap_sub REFRESH PUBLICATION WITH (copy_data = false)" -); -$node_publisher->safe_psql('postgres', "INSERT INTO tab_full VALUES(0)"); - -$node_publisher->wait_for_catchup('tap_sub'); - -# Check that we don't send BEGIN and COMMIT because of empty transaction -# optimization. We have to look for the DEBUG1 log messages about that, so -# temporarily bump up the log verbosity. -$node_publisher->append_conf('postgresql.conf', "log_min_messages = debug1"); -$node_publisher->reload; - -# Note that the current location of the log file is not grabbed immediately -# after reloading the configuration, but after sending one SQL command to -# the node so that we are sure that the reloading has taken effect. -$log_location_pub = -s $node_publisher->logfile; - -$node_publisher->safe_psql('postgres', "INSERT INTO tab_notrep VALUES (11)"); - -$node_publisher->wait_for_catchup('tap_sub'); - -$logfile = slurp_file($node_publisher->logfile, $log_location_pub); -like( - $logfile, - qr/skipped replication of an empty transaction with XID/, - 'empty transaction is skipped'); - -$result = - $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_notrep"); -is($result, qq(0), 'check non-replicated table is empty on subscriber'); - -$node_publisher->append_conf('postgresql.conf', "log_min_messages = warning"); -$node_publisher->reload; - -# note that data are different on provider and subscriber -$result = $node_subscriber->safe_psql('postgres', - "SELECT count(*), min(a), max(a) FROM tab_ins"); -is($result, qq(1052|1|1002), - 'check replicated deletes after alter publication'); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT count(*), min(a), max(a) FROM tab_full"); -is($result, qq(19|0|100), 'check replicated insert after alter publication'); - -# check restart on rename -$oldpid = $node_publisher->safe_psql('postgres', - "SELECT pid FROM pg_stat_replication WHERE application_name = 'tap_sub' AND state = 'streaming';" -); -$node_subscriber->safe_psql('postgres', - "ALTER SUBSCRIPTION tap_sub RENAME TO tap_sub_renamed"); -$node_publisher->poll_query_until('postgres', - "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = 'tap_sub_renamed' AND state = 'streaming';" - ) - or die - "Timed out while waiting for apply to restart after renaming SUBSCRIPTION"; - -# check all the cleanup -$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub_renamed"); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT count(*) FROM pg_subscription"); -is($result, qq(0), 'check subscription was dropped on subscriber'); - -$result = $node_publisher->safe_psql('postgres', - "SELECT count(*) FROM pg_replication_slots"); -is($result, qq(0), 'check replication slot was dropped on publisher'); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT count(*) FROM pg_subscription_rel"); -is($result, qq(0), - 'check subscription relation status was dropped on subscriber'); - -$result = $node_publisher->safe_psql('postgres', - "SELECT count(*) FROM pg_replication_slots"); -is($result, qq(0), 'check replication slot was dropped on publisher'); - -$result = $node_subscriber->safe_psql('postgres', - "SELECT count(*) FROM pg_replication_origin"); -is($result, qq(0), 'check replication origin was dropped on subscriber'); - -$node_subscriber->stop('fast'); -$node_publisher->stop('fast'); - -# CREATE PUBLICATION while wal_level=minimal should succeed, with a WARNING -$node_publisher->append_conf( - 'postgresql.conf', qq( -wal_level=minimal -max_wal_senders=0 -)); -$node_publisher->start; -($result, my $retout, my $reterr) = $node_publisher->psql( - 'postgres', qq{ -BEGIN; -CREATE TABLE skip_wal(); -CREATE PUBLICATION tap_pub2 FOR TABLE skip_wal; -ROLLBACK; -}); -like( - $reterr, - qr/WARNING: logical decoding must be enabled to publish logical changes/, - 'CREATE PUBLICATION while "wal_level=minimal"'); - -done_testing(); From a298bf450e14f7da320678e8fe64ef3f709bda4a Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Fri, 12 Jun 2026 13:06:23 -0400 Subject: [PATCH 11/14] ci: run the pytest suite in CI Enable the pytest suite (-Dpytest=enabled) on all jobs. This needs pytest installed where the images do not already provide it: via MacPorts on macOS (plus pexpect for the interactive psql tests, which need a pty and so skip on Windows), via pip on the Windows VS image, and via pacman on MinGW. The AddressSanitizer job needs one accommodation: the suite loads libpq in-process via ctypes, and dlopening an ASan-instrumented libpq into an uninstrumented python aborts because the ASan runtime must come first in the link order. Preload the ASan runtime for the test step to satisfy that; it is a no-op for the already-instrumented server binaries. --- .github/workflows/pg-ci.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index 5bc5292d2a..299c09dfdf 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -88,6 +88,7 @@ env: -Dplperl=enabled -Dplpython=enabled -Dpltcl=enabled + -Dpytest=enabled -Dreadline=enabled -Dssl=openssl -Dtap_tests=enabled @@ -668,6 +669,15 @@ jobs: - name: Test world shell: *su_postgres_shell + # The pytest suite loads libpq in-process via ctypes. Here libpq is + # AddressSanitizer-instrumented, and ASan must come first in the link + # order; dlopening it into an otherwise uninstrumented python aborts + # with "ASan runtime does not come first". Preload the ASan runtime + # for the test run to satisfy that (a no-op for the already-instrumented + # server/client binaries). Scoped to this step so the build is + # unaffected; detect_leaks is already disabled via ASAN_OPTIONS. + env: + ADDITIONAL_SETUP: export LD_PRELOAD="$(gcc -print-file-name=libasan.so)" run: *meson_test_world_cmd - *linux_collect_cores_step @@ -710,6 +720,8 @@ jobs: openssl p5.34-io-tty p5.34-ipc-run + py312-pexpect + py312-pytest python312 tcl zstd @@ -869,6 +881,7 @@ jobs: -Dldap=enabled -Dplperl=enabled -Dplpython=enabled + -Dpytest=enabled -Dssl=openssl -Dtap_tests=enabled @@ -956,9 +969,11 @@ jobs: - name: Install dependencies shell: pwsh run: | - # meson is not preinstalled on windows-2022. Install via pip + # meson is not preinstalled on windows-2022. Install via pip. + # pytest enables the Python test suite (pexpect is omitted: it needs + # a pty, which Windows lacks, and the interactive tests importorskip). echo ::group::pip - python -m pip install --upgrade meson + python -m pip install --upgrade meson pytest if (!$?) { throw 'cmdfail' } echo ::endgroup:: @@ -1096,6 +1111,7 @@ jobs: ${MINGW_PACKAGE_PREFIX}-meson \ ${MINGW_PACKAGE_PREFIX}-perl \ ${MINGW_PACKAGE_PREFIX}-pkgconf \ + ${MINGW_PACKAGE_PREFIX}-python-pytest \ ${MINGW_PACKAGE_PREFIX}-readline \ ${MINGW_PACKAGE_PREFIX}-zlib \ ${MINGW_PACKAGE_PREFIX}-zstd From 53db722ae8bc83eb5c095238384db6199599845d Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Sat, 20 Jun 2026 10:32:45 -0400 Subject: [PATCH 12/14] ci: fix the SanityCheck target for the pg_ctl pytest conversion The SanityCheck job ran the TAP test pg_ctl/001_start_stop, which was removed when src/bin/pg_ctl was converted to pytest (see "python tests: convert the pg_ctl TAP suite to pytest"). Point MTEST_TARGET at the pytest equivalent pg_ctl/test_001_start_stop and enable -Dpytest on the SanityCheck setup (its minimal config uses --auto-features=disabled, so pytest must be requested explicitly; the Linux image already provides pytest). Also update the now-stale "a tap test" comment. --- .github/workflows/pg-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index 299c09dfdf..60e070d93b 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -348,6 +348,7 @@ jobs: --auto-features=disabled \ -Ddefault_library=shared \ -Dtap_tests=enabled \ + -Dpytest=enabled \ build - name: Build @@ -383,7 +384,7 @@ jobs: # Run a minimal set of tests. The main regression tests take too long # for this purpose. For now this is a random quick pg_regress style - # test, and a tap test that exercises both a frontend binary and the + # test, and a pytest test that exercises both a frontend binary and the # backend. # # To allow the command below to be reused by later tasks, we allow @@ -396,7 +397,7 @@ jobs: - name: Test shell: *su_postgres_shell env: - MTEST_TARGET: cube/regress pg_ctl/001_start_stop + MTEST_TARGET: cube/regress pg_ctl/test_001_start_stop run: &meson_test_world_cmd | ${{case(runner.os == 'Windows', '', 'ulimit -c unlimited')}} From 399d78b0fc8d8a0268017c7403f946e0a88d2c47 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Sat, 20 Jun 2026 11:06:16 -0400 Subject: [PATCH 13/14] Add autoconf/make support for the pytest test suite The pytest suites were wired into the Meson build only, so the autoconf/make build had no way to run them. After the converted Perl tests were removed, "make check" in the fully converted directories ran prove against an empty t/ and failed. Teach the make build to run pytest, mirroring the existing prove_check machinery: - configure: add --enable-pytest and locate pytest (falling back to "python -m pytest", as the Meson build does, for installations that ship only the module). - src/Makefile.global.in: add enable_pytest, PYTEST and the pytest_check / pytest_installcheck recipes. They set up the temporary install on PATH and put src/test/pytest on PYTHONPATH, then run "pytest pyt/test_*.py" from the test directory; the active pytest configuration comes from the repository-root pyproject.toml. - Wire the converted directories' check targets: psql, pg_ctl and libpq (fully converted) run pytest_check; recovery and subscription (one test each converted) run both prove_check and pytest_check so their remaining Perl tests still run. - CI: configure the Linux Autoconf job with --enable-pytest. Validated with a VPATH autoconf build: "make check" in src/bin/psql and src/interfaces/libpq runs the pytest suites to green TAP output. --- .github/workflows/pg-ci.yml | 2 +- configure | 102 +++++++++++++++++++++++++++++++++ configure.ac | 24 ++++++++ src/Makefile.global.in | 61 ++++++++++++++++++++ src/bin/pg_ctl/Makefile | 4 +- src/bin/psql/Makefile | 4 +- src/interfaces/libpq/Makefile | 4 +- src/test/recovery/Makefile | 2 + src/test/subscription/Makefile | 2 + 9 files changed, 198 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index 60e070d93b..877f51fefc 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -504,7 +504,7 @@ jobs: run: | ./configure \ --enable-cassert --enable-injection-points --enable-debug \ - --enable-tap-tests --enable-nls \ + --enable-tap-tests --enable-pytest --enable-nls \ --with-segsize-blocks=6 \ --with-libnuma \ --with-liburing \ diff --git a/configure b/configure index 5f77f3cac2..fa6d61ef35 100755 --- a/configure +++ b/configure @@ -630,6 +630,7 @@ vpath_build PG_SYSROOT PG_VERSION_NUM LDFLAGS_EX_BE +PYTEST PROVE DBTOEPUB FOP @@ -773,6 +774,7 @@ CFLAGS CC enable_injection_points PG_TEST_EXTRA +enable_pytest enable_tap_tests enable_dtrace DTRACEFLAGS @@ -851,6 +853,7 @@ enable_profiling enable_coverage enable_dtrace enable_tap_tests +enable_pytest enable_injection_points with_blocksize with_segsize @@ -1552,6 +1555,7 @@ Optional Features: --enable-coverage build with coverage testing instrumentation --enable-dtrace build with DTrace support --enable-tap-tests enable TAP tests (requires Perl and IPC::Run) + --enable-pytest enable Python (pytest) tests (requires pytest) --enable-injection-points enable injection points (for testing) --enable-depend turn on automatic dependency tracking @@ -3661,6 +3665,33 @@ fi +# Python (pytest) tests +# + + +# Check whether --enable-pytest was given. +if test "${enable_pytest+set}" = set; then : + enableval=$enable_pytest; + case $enableval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --enable-pytest option" "$LINENO" 5 + ;; + esac + +else + enable_pytest=no + +fi + + + + # # Injection points @@ -19523,6 +19554,77 @@ $as_echo "$modulestderr" >&6; } fi fi +if test "$enable_pytest" = yes; then + # Make sure we know how to run pytest. Prefer a "pytest" program, but fall + # back to "python -m pytest" (as the Meson build does), since some + # installations provide only the module and not a wrapper script. + if test -z "$PYTEST"; then + for ac_prog in pytest +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_PYTEST+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $PYTEST in + [\\/]* | ?:[\\/]*) + ac_cv_path_PYTEST="$PYTEST" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_PYTEST="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +PYTEST=$ac_cv_path_PYTEST +if test -n "$PYTEST"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTEST" >&5 +$as_echo "$PYTEST" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$PYTEST" && break +done + +else + # Report the value of PYTEST in configure's output in all cases. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for PYTEST" >&5 +$as_echo_n "checking for PYTEST... " >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTEST" >&5 +$as_echo "$PYTEST" >&6; } +fi + + if test -z "$PYTEST"; then + for pgac_python in python3 python; do + if "$pgac_python" -m pytest --version >/dev/null 2>&1; then + PYTEST="$pgac_python -m pytest" + break + fi + done + fi + if test -z "$PYTEST"; then + as_fn_error $? "pytest not found" "$LINENO" 5 + fi +fi + # If compiler will take -Wl,--as-needed (or various platform-specific # spellings thereof) then add that to LDFLAGS. This is much easier than # trying to filter LIBS to the minimum for each executable. diff --git a/configure.ac b/configure.ac index 61cee42daa..8264d5e2f9 100644 --- a/configure.ac +++ b/configure.ac @@ -231,6 +231,12 @@ AC_SUBST(enable_dtrace) PGAC_ARG_BOOL(enable, tap-tests, no, [enable TAP tests (requires Perl and IPC::Run)]) AC_SUBST(enable_tap_tests) + +# Python (pytest) tests +# +PGAC_ARG_BOOL(enable, pytest, no, + [enable Python (pytest) tests (requires pytest)]) +AC_SUBST(enable_pytest) AC_ARG_VAR(PG_TEST_EXTRA, [enable selected extra tests (overridden at runtime by PG_TEST_EXTRA environment variable)]) @@ -2502,6 +2508,24 @@ if test "$enable_tap_tests" = yes; then fi fi +if test "$enable_pytest" = yes; then + # Make sure we know how to run pytest. Prefer a "pytest" program, but fall + # back to "python -m pytest" (as the Meson build does), since some + # installations provide only the module and not a wrapper script. + PGAC_PATH_PROGS(PYTEST, pytest) + if test -z "$PYTEST"; then + for pgac_python in python3 python; do + if "$pgac_python" -m pytest --version >/dev/null 2>&1; then + PYTEST="$pgac_python -m pytest" + break + fi + done + fi + if test -z "$PYTEST"; then + AC_MSG_ERROR([pytest not found]) + fi +fi + # If compiler will take -Wl,--as-needed (or various platform-specific # spellings thereof) then add that to LDFLAGS. This is much easier than # trying to filter LIBS to the minimum for each executable. diff --git a/src/Makefile.global.in b/src/Makefile.global.in index cef1ad7f87..dd51d351c5 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -211,6 +211,7 @@ enable_dtrace = @enable_dtrace@ enable_coverage = @enable_coverage@ enable_injection_points = @enable_injection_points@ enable_tap_tests = @enable_tap_tests@ +enable_pytest = @enable_pytest@ python_includespec = @python_includespec@ python_libdir = @python_libdir@ @@ -510,6 +511,66 @@ prove_installcheck = @echo "TAP tests not enabled. Try configuring with --enable prove_check = $(prove_installcheck) endif +PYTEST = @PYTEST@ +# The active pytest configuration (plugins, import mode, and the pythonpath +# that makes the in-tree libpq/ and pypg/ packages importable) lives in the +# repository-root pyproject.toml; PYTHONPATH is also set below for robustness. +# User-supplied pytest flags such as -v can be provided in PYTEST_FLAGS. +PYTEST_FLAGS = + +ifeq ($(enable_pytest),yes) + +ifndef PGXS +define pytest_installcheck +echo "# +++ pytest install-check in $(subdir) +++" && \ +rm -rf '$(CURDIR)'/tmp_check && \ +$(MKDIR_P) '$(CURDIR)'/tmp_check && \ +cd $(srcdir) && \ + TESTLOGDIR='$(CURDIR)/tmp_check/log' \ + TESTDATADIR='$(CURDIR)/tmp_check' \ + PATH="$(bindir):$(CURDIR):$$PATH" \ + PYTHONPATH='$(abs_top_srcdir)/src/test/pytest$(if $(PYTHONPATH),:$(PYTHONPATH),)' \ + PGPORT='6$(DEF_PGPORT)' top_builddir='$(CURDIR)/$(top_builddir)' \ + PG_REGRESS='$(CURDIR)/$(top_builddir)/src/test/regress/pg_regress' \ + share_contrib_dir='$(DESTDIR)$(datadir)/$(datamoduledir)' \ + $(PYTEST) $(PYTEST_FLAGS) $(if $(PYTEST_TESTS),$(PYTEST_TESTS),pyt/test_*.py) +endef +else # PGXS case +define pytest_installcheck +echo "# +++ pytest install-check in $(subdir) +++" && \ +rm -rf '$(CURDIR)'/tmp_check && \ +$(MKDIR_P) '$(CURDIR)'/tmp_check && \ +cd $(srcdir) && \ + TESTLOGDIR='$(CURDIR)/tmp_check/log' \ + TESTDATADIR='$(CURDIR)/tmp_check' \ + PATH="$(bindir):$(CURDIR):$$PATH" \ + PYTHONPATH='$(datadir)/pgxs/src/test/pytest$(if $(PYTHONPATH),:$(PYTHONPATH),)' \ + PGPORT='6$(DEF_PGPORT)' \ + PG_REGRESS='$(top_builddir)/src/test/regress/pg_regress' \ + $(PYTEST) $(PYTEST_FLAGS) $(if $(PYTEST_TESTS),$(PYTEST_TESTS),pyt/test_*.py) +endef +endif # PGXS + +define pytest_check +echo "# +++ pytest check in $(subdir) +++" && \ +rm -rf '$(CURDIR)'/tmp_check && \ +$(MKDIR_P) '$(CURDIR)'/tmp_check && \ +cd $(srcdir) && \ + TESTLOGDIR='$(CURDIR)/tmp_check/log' \ + TESTDATADIR='$(CURDIR)/tmp_check' \ + $(with_temp_install) \ + PYTHONPATH='$(abs_top_srcdir)/src/test/pytest$(if $(PYTHONPATH),:$(PYTHONPATH),)' \ + PGPORT='6$(DEF_PGPORT)' top_builddir='$(CURDIR)/$(top_builddir)' \ + PG_REGRESS='$(CURDIR)/$(top_builddir)/src/test/regress/pg_regress' \ + share_contrib_dir='$(abs_top_builddir)/tmp_install$(datadir)/$(datamoduledir)' \ + $(PYTEST) $(PYTEST_FLAGS) $(if $(PYTEST_TESTS),$(PYTEST_TESTS),pyt/test_*.py) +endef + +else +pytest_installcheck = @echo "pytest tests not enabled. Try configuring with --enable-pytest" +pytest_check = $(pytest_installcheck) +endif + # Installation. install_bin = @install_bin@ diff --git a/src/bin/pg_ctl/Makefile b/src/bin/pg_ctl/Makefile index 5c2d418098..7c9769a645 100644 --- a/src/bin/pg_ctl/Makefile +++ b/src/bin/pg_ctl/Makefile @@ -47,7 +47,7 @@ clean distclean: rm -rf tmp_check check: - $(prove_check) + $(pytest_check) installcheck: - $(prove_installcheck) + $(pytest_installcheck) diff --git a/src/bin/psql/Makefile b/src/bin/psql/Makefile index be0032652c..5abaf138ba 100644 --- a/src/bin/psql/Makefile +++ b/src/bin/psql/Makefile @@ -80,7 +80,7 @@ clean distclean: rm -f sql_help.h sql_help.c psqlscanslash.c tab-complete.c check: - $(prove_check) + $(pytest_check) installcheck: - $(prove_installcheck) + $(pytest_installcheck) diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index 0963995eed..efc1f8e17e 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -168,10 +168,10 @@ test-build: check installcheck: export PATH := $(CURDIR)/test:$(PATH) check: test-build all - $(prove_check) + $(pytest_check) installcheck: test-build all - $(prove_installcheck) + $(pytest_installcheck) installdirs: installdirs-lib $(MKDIR_P) '$(DESTDIR)$(includedir)' '$(DESTDIR)$(includedir_internal)' '$(DESTDIR)$(datadir)' diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile index d41aaaf8ae..07fef58406 100644 --- a/src/test/recovery/Makefile +++ b/src/test/recovery/Makefile @@ -26,9 +26,11 @@ export REGRESS_SHLIB check: $(prove_check) + $(pytest_check) installcheck: $(prove_installcheck) + $(pytest_installcheck) clean distclean: rm -rf tmp_check diff --git a/src/test/subscription/Makefile b/src/test/subscription/Makefile index 1b22703dc2..5655b6538a 100644 --- a/src/test/subscription/Makefile +++ b/src/test/subscription/Makefile @@ -21,9 +21,11 @@ export enable_injection_points check: $(prove_check) + $(pytest_check) installcheck: $(prove_installcheck) + $(pytest_installcheck) clean distclean: rm -rf tmp_check From 7c243768faad151d6032d04f1b37e3c9b5d236d3 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Sat, 20 Jun 2026 12:27:30 -0400 Subject: [PATCH 14/14] python tests: name converted suites with the PostgreSQL NNN_name.py convention The converted pytest suites were named test_NNN_name.py, which required a custom python_files = ["test_*.py"] pytest setting. Rename them to NNN_name.py instead, mirroring the t/NNN_name.pl Perl originals they replace (e.g. t/001_basic.pl -> pyt/001_basic.py); this is the project's TAP file naming convention and makes the meson/prove test names match the Perl ones (psql/001_basic rather than psql/test_001_basic). The framework's own self-tests under src/test/pytest/pyt have no Perl origin and keep the pytest-default test_*.py names. pyproject.toml's python_files now lists both patterns ("[0-9]*.py" and "test_*.py"), and the make pytest_check glob uses pyt/[0-9]*.py (which also excludes conftest.py). Test *function* names keep the test_ prefix, as pytest requires. Updates the five meson test lists, src/Makefile.global.in, the CI SanityCheck target and the README accordingly. --- .github/workflows/pg-ci.yml | 2 +- pyproject.toml | 6 +++-- src/Makefile.global.in | 6 ++--- src/bin/pg_ctl/meson.build | 8 +++---- ...st_001_start_stop.py => 001_start_stop.py} | 0 .../pyt/{test_002_status.py => 002_status.py} | 0 .../{test_003_promote.py => 003_promote.py} | 0 ...test_004_logrotate.py => 004_logrotate.py} | 0 src/bin/psql/meson.build | 8 +++---- .../pyt/{test_001_basic.py => 001_basic.py} | 0 ...ab_completion.py => 010_tab_completion.py} | 0 .../pyt/{test_020_cancel.py => 020_cancel.py} | 0 .../pyt/{test_030_pager.py => 030_pager.py} | 0 src/interfaces/libpq/meson.build | 12 +++++----- .../libpq/pyt/{test_001_uri.py => 001_uri.py} | 0 .../libpq/pyt/{test_002_api.py => 002_api.py} | 0 ..._list.py => 003_load_balance_host_list.py} | 0 ...balance_dns.py => 004_load_balance_dns.py} | 0 ...ryption.py => 005_negotiate_encryption.py} | 0 .../{test_006_service.py => 006_service.py} | 0 src/test/pytest/README.md | 23 ++++++++++--------- src/test/recovery/meson.build | 2 +- ...st_001_stream_rep.py => 001_stream_rep.py} | 0 src/test/subscription/meson.build | 2 +- ..._001_rep_changes.py => 001_rep_changes.py} | 0 25 files changed, 36 insertions(+), 33 deletions(-) rename src/bin/pg_ctl/pyt/{test_001_start_stop.py => 001_start_stop.py} (100%) rename src/bin/pg_ctl/pyt/{test_002_status.py => 002_status.py} (100%) rename src/bin/pg_ctl/pyt/{test_003_promote.py => 003_promote.py} (100%) rename src/bin/pg_ctl/pyt/{test_004_logrotate.py => 004_logrotate.py} (100%) rename src/bin/psql/pyt/{test_001_basic.py => 001_basic.py} (100%) rename src/bin/psql/pyt/{test_010_tab_completion.py => 010_tab_completion.py} (100%) rename src/bin/psql/pyt/{test_020_cancel.py => 020_cancel.py} (100%) rename src/bin/psql/pyt/{test_030_pager.py => 030_pager.py} (100%) rename src/interfaces/libpq/pyt/{test_001_uri.py => 001_uri.py} (100%) rename src/interfaces/libpq/pyt/{test_002_api.py => 002_api.py} (100%) rename src/interfaces/libpq/pyt/{test_003_load_balance_host_list.py => 003_load_balance_host_list.py} (100%) rename src/interfaces/libpq/pyt/{test_004_load_balance_dns.py => 004_load_balance_dns.py} (100%) rename src/interfaces/libpq/pyt/{test_005_negotiate_encryption.py => 005_negotiate_encryption.py} (100%) rename src/interfaces/libpq/pyt/{test_006_service.py => 006_service.py} (100%) rename src/test/recovery/pyt/{test_001_stream_rep.py => 001_stream_rep.py} (100%) rename src/test/subscription/pyt/{test_001_rep_changes.py => 001_rep_changes.py} (100%) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index 877f51fefc..3e1b9f89f7 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -397,7 +397,7 @@ jobs: - name: Test shell: *su_postgres_shell env: - MTEST_TARGET: cube/regress pg_ctl/test_001_start_stop + MTEST_TARGET: cube/regress pg_ctl/001_start_stop run: &meson_test_world_cmd | ${{case(runner.os == 'Windows', '', 'ulimit -c unlimited')}} diff --git a/pyproject.toml b/pyproject.toml index 44a9b6015e..8b65e2afde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,8 +17,10 @@ pythonpath = ["src/test/pytest"] # directories (e.g. many t/001_basic) do not collide. addopts = ["-p", "pgtap", "-p", "pypg.fixtures", "--import-mode=importlib"] -# Tests live in pyt/ subdirectories using the test_*.py convention. -python_files = ["test_*.py"] +# Tests live in pyt/ subdirectories. Converted suites follow the PostgreSQL +# TAP naming convention (NNN_name.py, mirroring t/NNN_name.pl); the framework's +# own self-tests under src/test/pytest/pyt use the pytest-default test_*.py. +python_files = ["[0-9]*.py", "test_*.py"] # --------------------------------------------------------------------------- # Code quality gates for the Python test suite (src/test/pytest and the pyt/ diff --git a/src/Makefile.global.in b/src/Makefile.global.in index dd51d351c5..a8aef11e4b 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -533,7 +533,7 @@ cd $(srcdir) && \ PGPORT='6$(DEF_PGPORT)' top_builddir='$(CURDIR)/$(top_builddir)' \ PG_REGRESS='$(CURDIR)/$(top_builddir)/src/test/regress/pg_regress' \ share_contrib_dir='$(DESTDIR)$(datadir)/$(datamoduledir)' \ - $(PYTEST) $(PYTEST_FLAGS) $(if $(PYTEST_TESTS),$(PYTEST_TESTS),pyt/test_*.py) + $(PYTEST) $(PYTEST_FLAGS) $(if $(PYTEST_TESTS),$(PYTEST_TESTS),pyt/[0-9]*.py) endef else # PGXS case define pytest_installcheck @@ -547,7 +547,7 @@ cd $(srcdir) && \ PYTHONPATH='$(datadir)/pgxs/src/test/pytest$(if $(PYTHONPATH),:$(PYTHONPATH),)' \ PGPORT='6$(DEF_PGPORT)' \ PG_REGRESS='$(top_builddir)/src/test/regress/pg_regress' \ - $(PYTEST) $(PYTEST_FLAGS) $(if $(PYTEST_TESTS),$(PYTEST_TESTS),pyt/test_*.py) + $(PYTEST) $(PYTEST_FLAGS) $(if $(PYTEST_TESTS),$(PYTEST_TESTS),pyt/[0-9]*.py) endef endif # PGXS @@ -563,7 +563,7 @@ cd $(srcdir) && \ PGPORT='6$(DEF_PGPORT)' top_builddir='$(CURDIR)/$(top_builddir)' \ PG_REGRESS='$(CURDIR)/$(top_builddir)/src/test/regress/pg_regress' \ share_contrib_dir='$(abs_top_builddir)/tmp_install$(datadir)/$(datamoduledir)' \ - $(PYTEST) $(PYTEST_FLAGS) $(if $(PYTEST_TESTS),$(PYTEST_TESTS),pyt/test_*.py) + $(PYTEST) $(PYTEST_FLAGS) $(if $(PYTEST_TESTS),$(PYTEST_TESTS),pyt/[0-9]*.py) endef else diff --git a/src/bin/pg_ctl/meson.build b/src/bin/pg_ctl/meson.build index ba780ffca9..99819b225b 100644 --- a/src/bin/pg_ctl/meson.build +++ b/src/bin/pg_ctl/meson.build @@ -23,10 +23,10 @@ tests += { 'bd': meson.current_build_dir(), 'pytest': { 'tests': [ - 'pyt/test_001_start_stop.py', - 'pyt/test_002_status.py', - 'pyt/test_003_promote.py', - 'pyt/test_004_logrotate.py', + 'pyt/001_start_stop.py', + 'pyt/002_status.py', + 'pyt/003_promote.py', + 'pyt/004_logrotate.py', ], }, } diff --git a/src/bin/pg_ctl/pyt/test_001_start_stop.py b/src/bin/pg_ctl/pyt/001_start_stop.py similarity index 100% rename from src/bin/pg_ctl/pyt/test_001_start_stop.py rename to src/bin/pg_ctl/pyt/001_start_stop.py diff --git a/src/bin/pg_ctl/pyt/test_002_status.py b/src/bin/pg_ctl/pyt/002_status.py similarity index 100% rename from src/bin/pg_ctl/pyt/test_002_status.py rename to src/bin/pg_ctl/pyt/002_status.py diff --git a/src/bin/pg_ctl/pyt/test_003_promote.py b/src/bin/pg_ctl/pyt/003_promote.py similarity index 100% rename from src/bin/pg_ctl/pyt/test_003_promote.py rename to src/bin/pg_ctl/pyt/003_promote.py diff --git a/src/bin/pg_ctl/pyt/test_004_logrotate.py b/src/bin/pg_ctl/pyt/004_logrotate.py similarity index 100% rename from src/bin/pg_ctl/pyt/test_004_logrotate.py rename to src/bin/pg_ctl/pyt/004_logrotate.py diff --git a/src/bin/psql/meson.build b/src/bin/psql/meson.build index 7b90f21bad..1bfe884b3c 100644 --- a/src/bin/psql/meson.build +++ b/src/bin/psql/meson.build @@ -77,10 +77,10 @@ tests += { # they use pexpect via the interactive_psql fixture in pyt/conftest.py and # skip when pexpect (or readline / a working "wc -l") is unavailable. 'tests': [ - 'pyt/test_001_basic.py', - 'pyt/test_010_tab_completion.py', - 'pyt/test_020_cancel.py', - 'pyt/test_030_pager.py', + 'pyt/001_basic.py', + 'pyt/010_tab_completion.py', + 'pyt/020_cancel.py', + 'pyt/030_pager.py', ], }, } diff --git a/src/bin/psql/pyt/test_001_basic.py b/src/bin/psql/pyt/001_basic.py similarity index 100% rename from src/bin/psql/pyt/test_001_basic.py rename to src/bin/psql/pyt/001_basic.py diff --git a/src/bin/psql/pyt/test_010_tab_completion.py b/src/bin/psql/pyt/010_tab_completion.py similarity index 100% rename from src/bin/psql/pyt/test_010_tab_completion.py rename to src/bin/psql/pyt/010_tab_completion.py diff --git a/src/bin/psql/pyt/test_020_cancel.py b/src/bin/psql/pyt/020_cancel.py similarity index 100% rename from src/bin/psql/pyt/test_020_cancel.py rename to src/bin/psql/pyt/020_cancel.py diff --git a/src/bin/psql/pyt/test_030_pager.py b/src/bin/psql/pyt/030_pager.py similarity index 100% rename from src/bin/psql/pyt/test_030_pager.py rename to src/bin/psql/pyt/030_pager.py diff --git a/src/interfaces/libpq/meson.build b/src/interfaces/libpq/meson.build index 57c64e9686..76fac4688d 100644 --- a/src/interfaces/libpq/meson.build +++ b/src/interfaces/libpq/meson.build @@ -155,12 +155,12 @@ tests += { 'bd': meson.current_build_dir(), 'pytest': { 'tests': [ - 'pyt/test_001_uri.py', - 'pyt/test_002_api.py', - 'pyt/test_003_load_balance_host_list.py', - 'pyt/test_004_load_balance_dns.py', - 'pyt/test_005_negotiate_encryption.py', - 'pyt/test_006_service.py', + 'pyt/001_uri.py', + 'pyt/002_api.py', + 'pyt/003_load_balance_host_list.py', + 'pyt/004_load_balance_dns.py', + 'pyt/005_negotiate_encryption.py', + 'pyt/006_service.py', ], 'env': { 'with_ssl': ssl_library, diff --git a/src/interfaces/libpq/pyt/test_001_uri.py b/src/interfaces/libpq/pyt/001_uri.py similarity index 100% rename from src/interfaces/libpq/pyt/test_001_uri.py rename to src/interfaces/libpq/pyt/001_uri.py diff --git a/src/interfaces/libpq/pyt/test_002_api.py b/src/interfaces/libpq/pyt/002_api.py similarity index 100% rename from src/interfaces/libpq/pyt/test_002_api.py rename to src/interfaces/libpq/pyt/002_api.py diff --git a/src/interfaces/libpq/pyt/test_003_load_balance_host_list.py b/src/interfaces/libpq/pyt/003_load_balance_host_list.py similarity index 100% rename from src/interfaces/libpq/pyt/test_003_load_balance_host_list.py rename to src/interfaces/libpq/pyt/003_load_balance_host_list.py diff --git a/src/interfaces/libpq/pyt/test_004_load_balance_dns.py b/src/interfaces/libpq/pyt/004_load_balance_dns.py similarity index 100% rename from src/interfaces/libpq/pyt/test_004_load_balance_dns.py rename to src/interfaces/libpq/pyt/004_load_balance_dns.py diff --git a/src/interfaces/libpq/pyt/test_005_negotiate_encryption.py b/src/interfaces/libpq/pyt/005_negotiate_encryption.py similarity index 100% rename from src/interfaces/libpq/pyt/test_005_negotiate_encryption.py rename to src/interfaces/libpq/pyt/005_negotiate_encryption.py diff --git a/src/interfaces/libpq/pyt/test_006_service.py b/src/interfaces/libpq/pyt/006_service.py similarity index 100% rename from src/interfaces/libpq/pyt/test_006_service.py rename to src/interfaces/libpq/pyt/006_service.py diff --git a/src/test/pytest/README.md b/src/test/pytest/README.md index c8d80a4549..a71212ea88 100644 --- a/src/test/pytest/README.md +++ b/src/test/pytest/README.md @@ -8,8 +8,8 @@ so a test can open many sessions, drive async and pipeline-mode traffic, and inspect results without subprocess overhead. The framework deliberately mirrors the concepts of `PostgreSQL::Test::Cluster` -and `PostgreSQL::Test::Utils`, so a Perl `.pl` test maps fairly directly onto a -Python `test_*.py`. +and `PostgreSQL::Test::Utils`, so a Perl `t/NNN_name.pl` test maps fairly +directly onto a Python `pyt/NNN_name.py`. ## Layout @@ -39,20 +39,21 @@ The actual tests live in `pyt/` subdirectories next to the code they cover, the same way Perl TAP tests live in `t/`: ``` -src/bin/psql/pyt/test_001_basic.py -src/bin/pg_rewind/pyt/test_001_basic.py -src/test/authentication/pyt/test_001_password.py +src/bin/psql/pyt/001_basic.py +src/bin/pg_rewind/pyt/001_basic.py +src/test/authentication/pyt/001_password.py contrib//pyt/... ``` ### Naming and discovery -* Test files are `test_NNN_.py` (`test_001_basic.py`), matching the - `NNN_name.pl` numbering of the Perl originals. +* Converted test files are `NNN_.py` (`001_basic.py`), matching the + `NNN_name.pl` naming of the Perl originals; the framework's own self-tests + under `src/test/pytest/pyt` use the pytest-default `test_*.py`. * Test functions are `test_(...)`; pytest collects them automatically. * Helper functions are prefixed with `_` so pytest does not collect them. * `--import-mode=importlib` is set so that identically named files in different - directories (the many `test_001_basic.py`) do not collide. + directories (the many `001_basic.py`) do not collide. ## Running the tests @@ -101,7 +102,7 @@ running from the repo root (or anywhere beneath it) picks up the right config: [tool.pytest.ini_options] pythonpath = ["src/test/pytest"] # make libpq/ and pypg/ importable addopts = ["-p", "pgtap", "-p", "pypg.fixtures", "--import-mode=importlib"] -python_files = ["test_*.py"] +python_files = ["[0-9]*.py", "test_*.py"] # NNN_name.py suites + test_*.py self-tests ``` ### Requirements @@ -262,7 +263,7 @@ that suite's `conftest.py`. ## Writing a new test -1. Create `…/pyt/test_NNN_.py` (and a `pyt/meson.build` adding the suite +1. Create `…/pyt/NNN_.py` (and a `pyt/meson.build` adding the suite to `tests` if the directory is new). 2. Start from the `pg` / `conn` / `create_pg` fixtures; reach for `pg_bin` to run client programs. @@ -270,5 +271,5 @@ that suite's `conftest.py`. 4. Use `wait_for_*` / `poll_query_until` / `poll_until` instead of fixed sleeps. 5. Gate anything expensive or unsafe behind `PG_TEST_EXTRA`, and `importorskip` / skip cleanly when an optional dependency is missing. -6. Run it directly with `pytest …/pyt/test_NNN_.py -v`, then confirm it +6. Run it directly with `pytest …/pyt/NNN_.py -v`, then confirm it passes under `meson test`. diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index d1ba8307af..57ddb8043a 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -69,7 +69,7 @@ tests += { 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', }, 'tests': [ - 'pyt/test_001_stream_rep.py', + 'pyt/001_stream_rep.py', ], }, } diff --git a/src/test/recovery/pyt/test_001_stream_rep.py b/src/test/recovery/pyt/001_stream_rep.py similarity index 100% rename from src/test/recovery/pyt/test_001_stream_rep.py rename to src/test/recovery/pyt/001_stream_rep.py diff --git a/src/test/subscription/meson.build b/src/test/subscription/meson.build index 63d21509df..341d472a76 100644 --- a/src/test/subscription/meson.build +++ b/src/test/subscription/meson.build @@ -56,7 +56,7 @@ tests += { 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', }, 'tests': [ - 'pyt/test_001_rep_changes.py', + 'pyt/001_rep_changes.py', ], }, } diff --git a/src/test/subscription/pyt/test_001_rep_changes.py b/src/test/subscription/pyt/001_rep_changes.py similarity index 100% rename from src/test/subscription/pyt/test_001_rep_changes.py rename to src/test/subscription/pyt/001_rep_changes.py