diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 398e2990..2c94ec71 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -44,3 +44,77 @@ jobs: - name: Run unit tests run: bazel test //ut:unit_tests --@clickhouse_cpp//:tls=${{ matrix.tls }} --test_output=errors + + # //ut:e2e_tests is tagged `manual`, so `bazel build //...` above + # skips it; build it explicitly so the run step below doesn't pay + # for compilation under the running-server timeout. + - name: Build e2e tests + run: bazel build //ut:e2e_tests --@clickhouse_cpp//:tls=${{ matrix.tls }} + + # The e2e suite needs a live server on localhost:9000. Each OS starts + # one the same way the CMake build's per-OS workflows do, since hosted + # runners can't all run a Linux container the same way: + # * Linux — docker-compose (linux.yml) + # * macOS — the native ClickHouse build run as a process (macos.yml) + # * Windows — the Linux container under WSL2 + podman (windows_msvc.yml) + - name: Start ClickHouse (Linux, docker-compose) + if: runner.os == 'Linux' + uses: hoverkraft-tech/compose-action@v2.0.1 + with: + compose-file: ci/docker-compose.yml + down-flags: --volumes + + - name: Start ClickHouse (macOS, native binary) + if: runner.os == 'macOS' + working-directory: ${{ runner.temp }} + run: | + curl https://builds.clickhouse.com/25.12/macos-aarch64/clickhouse -o clickhouse + chmod +x ./clickhouse + sudo mkdir -p /var/lib/clickhouse /var/log/clickhouse-server + sudo chown -R "$USER" /var/lib/clickhouse /var/log/clickhouse-server + nohup ./clickhouse server --config-file="$GITHUB_WORKSPACE/ci/docker-compose/config.xml" > clickhouse.log 2>&1 & + for i in {1..60}; do + if curl -fsS http://localhost:8123/ > /dev/null; then + echo "ClickHouse is ready" + exit 0 + fi + sleep 1 + done + echo "ClickHouse failed to start" + tail -200 clickhouse.log || true + exit 1 + + - name: Enable WSL (Windows) + if: runner.os == 'Windows' + uses: Vampire/setup-wsl@v5 + with: + distribution: Ubuntu-24.04 + additional-packages: + podman + podman-compose + + - name: Start ClickHouse (Windows, WSL + podman) + if: runner.os == 'Windows' + shell: wsl-bash {0} + run: | + cd $(wslpath -u "${{ github.workspace }}/ci/") + podman-compose up -d + timeout 60s bash -c \ + 'until curl -s -o /dev/null -w "%{http_code}" http://localhost:8123 | grep -q "200"; do sleep 2; done' + curl -s http://localhost:8123/?query=SELECT%20VERSION%28%29 + + - name: Wait for ClickHouse (Linux) + if: runner.os == 'Linux' + run: | + for i in {1..60}; do + if curl -fsS http://localhost:8123/ > /dev/null; then + echo "ClickHouse is ready" + exit 0 + fi + sleep 1 + done + echo "ClickHouse failed to start" + exit 1 + + - name: Run e2e tests + run: bazel test //ut:e2e_tests --@clickhouse_cpp//:tls=${{ matrix.tls }} --test_output=errors diff --git a/BUILD.bazel b/BUILD.bazel index 4fcf37a5..f8bdc8a4 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -14,8 +14,10 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") # entirely. `USE_BORINGSSL` ifdef-guards the two OpenSSL-only # surfaces (`SSL_CONF_*` command API and `SSL_read_ex`) that # BoringSSL doesn't provide; see clickhouse/base/sslsocket.cpp. -# * cityhash / lz4 / zstd / abseil come from BCR modules instead of -# contrib/; the contrib/ trees are only used by the CMake build. +# * lz4 / zstd / abseil come from BCR modules instead of contrib/. +# cityhash is the exception: it is built from contrib/ because the +# wire protocol needs ClickHouse's frozen CityHash variant, which +# differs from the stock BCR module (see the :cityhash target). # Selects the TLS implementation: `--@clickhouse_cpp//:tls=boringssl` # (default), `--@clickhouse_cpp//:tls=openssl`, or @@ -58,6 +60,34 @@ selects.config_setting_group( ], ) +# Vendored CityHash from contrib/, NOT the BCR `@cityhash` module. +# ClickHouse's wire protocol checksums compressed blocks with a specific +# frozen CityHash128 variant (see clickhouse/base/compressed.cpp); the +# stock Google CityHash 1.0.2 that BCR packages produces *different* +# hashes, so a server rejects compressed blocks with "Checksum doesn't +# match: corrupted data". This is the exact copy the CMake build uses, +# guaranteeing byte-identical hashing and wire compatibility. +cc_library( + name = "cityhash", + srcs = [ + "contrib/cityhash/cityhash/city.cc", + "contrib/cityhash/cityhash/config.h", + ], + hdrs = [ + "contrib/cityhash/cityhash/city.h", + "contrib/cityhash/cityhash/citycrc.h", + ], + # config.h gates __builtin_expect off with `#if WIN32 || WIN64`, but + # MSVC only predefines `_WIN32`/`_WIN64` — under CMake it's CMake that + # defines WIN32, which Bazel's MSVC toolchain doesn't. Define it here + # so the GCC/Clang-only builtin isn't used on MSVC. + copts = select({ + "@platforms//os:windows": ["/DWIN32"], + "//conditions:default": [], + }), + strip_include_prefix = "contrib/cityhash/cityhash", +) + # The main clickhouse-cpp library. Downstream callers use # `#include ` and link `@clickhouse_cpp//:clickhouse`. cc_library( @@ -120,7 +150,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ "@abseil-cpp//absl/numeric:int128", - "@cityhash//:cityhash", + ":cityhash", "@lz4//:lz4", "@zstd//:zstd", ] + select({ diff --git a/MODULE.bazel b/MODULE.bazel index 7bef78a8..c11197b0 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -7,7 +7,6 @@ module(name = "clickhouse_cpp", version = "2.6.1") bazel_dep(name = "abseil-cpp", version = "20260107.1") bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "boringssl", version = "0.20260526.0") -bazel_dep(name = "cityhash", version = "1.0.2") bazel_dep(name = "lz4", version = "1.10.0.bcr.1") bazel_dep(name = "openssl", version = "3.5.5.bcr.4") bazel_dep(name = "platforms", version = "1.1.0") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 4fddc55e..f8418c72 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -61,8 +61,6 @@ "https://bcr.bazel.build/modules/boringssl/0.20260526.0/source.json": "003d2c9a7e7cc5e73dd9aeffb79e9251c5ba5c3fbcf1649a9f9657c968b516c4", "https://bcr.bazel.build/modules/buildozer/8.5.1/MODULE.bazel": "a35d9561b3fc5b18797c330793e99e3b834a473d5fbd3d7d7634aafc9bdb6f8f", "https://bcr.bazel.build/modules/buildozer/8.5.1/source.json": "e3386e6ff4529f2442800dee47ad28d3e6487f36a1f75ae39ae56c70f0cd2fbd", - "https://bcr.bazel.build/modules/cityhash/1.0.2/MODULE.bazel": "db83f8032a2908911e37ff9328d7ee8e19b45f48c5f72454753fdf847595ad8c", - "https://bcr.bazel.build/modules/cityhash/1.0.2/source.json": "6e794ffbab1b5b6bf1df7d28305b60aa0fc17647a5b7909fb8c959bbefb76887", "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", "https://bcr.bazel.build/modules/google_benchmark/1.9.5/MODULE.bazel": "8a85cfd90b1e45e6e68f1aa2aa9efce3c04add57df732571d7fd54c07e7c5143", "https://bcr.bazel.build/modules/google_benchmark/1.9.5/source.json": "0bd357fd9db30ee31d5eb4c78b1086ce3d79b4423ce76de19e8a2fa7b2fa2e10", diff --git a/ut/BUILD.bazel b/ut/BUILD.bazel index 72b0b8e5..23db16c4 100644 --- a/ut/BUILD.bazel +++ b/ut/BUILD.bazel @@ -1,12 +1,36 @@ load("@rules_cc//cc:cc_test.bzl", "cc_test") -# Hermetic subset of the googletest suite in this directory: only the -# tests that do not need a running ClickHouse server. The remaining -# files (client_ut.cpp, roundtrip_tests.cpp, Column_ut.cpp, ...) connect -# to $CLICKHOUSE_HOST and stay CMake-only for now, where CI provides a -# server via docker-compose (see .github/workflows/linux.yml). +# The googletest suite in this directory is split into two targets: # -# socket_ut.cpp is included: it spins up its own LocalTcpServer on +# //ut:unit_tests — hermetic; no external server. Runs in the sandbox +# under `bazel test`, part of every CI matrix job. +# //ut:e2e_tests — needs a live ClickHouse server on $CLICKHOUSE_HOST +# (default localhost:9000). Tagged `manual`, so it is +# excluded from `bazel test //...` and must be run +# explicitly: `bazel run //ut:e2e_tests` (or +# `bazel test //ut:e2e_tests`). CI starts a server via +# docker-compose first, mirroring the CMake build's +# Linux workflow. +# +# ssl_ut.cpp is in neither target yet: it connects to an external secure +# endpoint (play.clickhouse.com:9440) with OS-specific CA paths, which the +# local docker server doesn't provide. It stays CMake-only for now. + +# Compile flags shared by both test binaries. -I. lets tests include +# support headers via the angled `#include ` form; /bigobj +# mirrors CMake's MSVC handling (Column_ut/client_ut have many symbols). +_UT_COPTS = ["-I."] + select({ + "@platforms//os:windows": ["/bigobj"], + "//conditions:default": [], +}) + +_UT_DEPS = [ + "//:clickhouse", + "@abseil-cpp//absl/numeric:int128", + "@googletest//:gtest", +] + +# socket_ut.cpp is included here: it spins up its own LocalTcpServer on # localhost, which works inside the Bazel test sandbox. cc_test( name = "unit_tests", @@ -39,16 +63,62 @@ cc_test( "//:tls_no": [], "//conditions:default": ["ssl_link_ut.cpp"], }), - # Some tests include support headers via the angled - # `#include ` form; expose the repo root for those. - # /bigobj mirrors CMake's MSVC handling for this test binary. - copts = ["-I."] + select({ - "@platforms//os:windows": ["/bigobj"], - "//conditions:default": [], - }), - deps = [ - "//:clickhouse", - "@abseil-cpp//absl/numeric:int128", - "@googletest//:gtest", + copts = _UT_COPTS, + deps = _UT_DEPS, +) + +# Server-dependent end-to-end tests. These connect to a ClickHouse +# instance ($CLICKHOUSE_HOST / _PORT / _USER / _PASSWORD / _DB, defaulting +# to localhost:9000) and exercise the full client round-trip — the same +# set the CMake build links into clickhouse-cpp-ut and runs against the +# docker-compose server. +# +# Tagged `manual` + `external`: excluded from wildcard expansion (so a bare +# `bazel test //...` stays hermetic and green without a server) and never +# cached (results depend on live server state). Run it with: +# +# bazel run //ut:e2e_tests +# CLICKHOUSE_HOST=my-host bazel test //ut:e2e_tests --test_output=all +cc_test( + name = "e2e_tests", + size = "large", + # The suite finishes in seconds on Linux/macOS, but on Windows it runs + # against the server through WSL2's localhost forwarding, where each + # reconnect / connection-failure test pays a multi-second timeout and + # the whole suite exceeds the 900s `large` budget. `timeout` is an + # upper bound, not a fixed wait, so raising it costs nothing on the + # fast platforms. (The CMake Windows job runs the same tests with no + # Bazel timeout at all.) + timeout = "eternal", + srcs = [ + # Test cases. + "abnormal_column_names_test.cpp", + "array_of_low_cardinality_tests.cpp", + "Column_ut.cpp", + "client_ut.cpp", + "connection_failed_client_test.cpp", + "connection_failed_client_test.h", + "low_cardinality_nullable_tests.cpp", + "performance_tests.cpp", + "readonly_client_test.cpp", + "readonly_client_test.h", + "roundtrip_tests.cpp", + # Test entry point and shared support code. + "main.cpp", + "roundtrip_column.cpp", + "roundtrip_column.h", + "utils.cpp", + "utils.h", + "utils_comparison.h", + "utils_meta.h", + "utils_performance.h", + "value_generators.cpp", + "value_generators.h", + ], + copts = _UT_COPTS, + tags = [ + "external", + "manual", ], + deps = _UT_DEPS, )