diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 469d0e81c7fa..d6343c6c15c6 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -76,6 +76,18 @@ run_package_test() { export PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS NOX_FILE NOX_SESSION export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}" + # Isolate PIP cache to prevent concurrent pip file lock deadlocks + export PIP_CACHE_DIR="/tmpfs/.pip_cache_$(basename ${package_name})" + mkdir -p "$PIP_CACHE_DIR" + + # Isolate gcloud state to prevent SQLite lock deadlocks and project race conditions + export CLOUDSDK_CONFIG="/tmpfs/.gcloud_config_$(basename ${package_name})" + mkdir -p "$CLOUDSDK_CONFIG" + + # Isolate boto config (used by storage/bigquery) to prevent lock contention + export BOTO_CONFIG="/tmpfs/.boto_$(basename ${package_name})" + touch "$BOTO_CONFIG" + gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS" gcloud config set project "$PROJECT_ID" @@ -140,10 +152,66 @@ for path in `find 'packages' \ set -e if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]]; then - # Call the function - its internal exports won't affect the next loop - run_package_test "$package_name" || RETVAL=$? + PACKAGES_TO_TEST="${PACKAGES_TO_TEST} ${package_name}" else echo "No changes in ${package_name} and not a continuous build, skipping." fi done + +if [ -n "$PACKAGES_TO_TEST" ]; then + export -f run_package_test + export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR + + # 1. DYNAMIC ROUTING: Automatically detect which packages are CPU hogs by checking if they install pytest-xdist or hardcode workers + LIGHT_TO_TEST="" + HEAVY_TO_TEST="" + for pkg in $PACKAGES_TO_TEST; do + if grep -qE "pytest-xdist|-n=auto|-n=[0-9]+" "packages/$pkg/noxfile.py" "packages/$pkg/setup.py" 2>/dev/null; then + HEAVY_TO_TEST="$HEAVY_TO_TEST $pkg" + else + LIGHT_TO_TEST="$LIGHT_TO_TEST $pkg" + fi + done + + # 2. PARALLEL LANE (Live Streaming): Run light packages with a parallel job queue. + # We prefix every line with the package name so output streams LIVE and remains readable. + if [ -n "$LIGHT_TO_TEST" ]; then + echo "============================================================" + echo "Running Lightweight Packages in Parallel (4 workers max)" + echo "============================================================" + for pkg in $LIGHT_TO_TEST; do + ( + timeout 15m bash -c "run_package_test \"$pkg\" < /dev/null" 2>&1 | awk -v prefix="[$pkg]" '{print prefix, $0}' + if [ ${PIPESTATUS[0]} -ne 0 ]; then touch ".failed_$pkg"; fi + ) & + # Limit parallel background jobs to 4 + while [ $(jobs -r | wc -l) -ge 4 ]; do sleep 1; done + done + wait # Wait for all parallel jobs to finish + fi + + # 3. SEQUENTIAL VIP LANE: Run heavy packages one-by-one so they have 100% of the VM resources. + if [ -n "$HEAVY_TO_TEST" ]; then + echo "============================================================" + echo "Running CPU-Intensive Packages Sequentially" + echo "============================================================" + for pkg in $HEAVY_TO_TEST; do + if [ -n "$pkg" ]; then + echo "[$pkg] Starting sequential execution..." + timeout 25m bash -c "run_package_test \"$pkg\" < /dev/null" 2>&1 | awk -v prefix="[$pkg]" '{print prefix, $0}' + if [ ${PIPESTATUS[0]} -ne 0 ]; then touch ".failed_$pkg"; fi + fi + done + fi + + # 4. FAIL STATE EVALUATION + for failed_marker in .failed_*; do + if [ -f "$failed_marker" ]; then + failed_pkg="${failed_marker#.failed_}" + echo "--- FAILED: $failed_pkg ---" + RETVAL=1 + fi + done +fi + exit ${RETVAL} diff --git a/packages/bigframes/setup.py b/packages/bigframes/setup.py index 76b98b88d312..5d7b2aadb999 100644 --- a/packages/bigframes/setup.py +++ b/packages/bigframes/setup.py @@ -162,3 +162,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/bigquery-magics/setup.py b/packages/bigquery-magics/setup.py index 763ca3afb9bc..0bc25158ab81 100644 --- a/packages/bigquery-magics/setup.py +++ b/packages/bigquery-magics/setup.py @@ -123,3 +123,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/django-google-spanner/setup.py b/packages/django-google-spanner/setup.py index 973dc622047e..58db4adfea1e 100644 --- a/packages/django-google-spanner/setup.py +++ b/packages/django-google-spanner/setup.py @@ -73,3 +73,5 @@ extras_require=extras, python_requires=">=3.10", ) + +# trigger system test diff --git a/packages/gapic-generator/setup.py b/packages/gapic-generator/setup.py index 237281229446..a5253c90ff66 100644 --- a/packages/gapic-generator/setup.py +++ b/packages/gapic-generator/setup.py @@ -85,3 +85,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-auth/setup.py b/packages/google-auth/setup.py index cf3148130d6e..51fe2038e205 100644 --- a/packages/google-auth/setup.py +++ b/packages/google-auth/setup.py @@ -132,3 +132,5 @@ "Topic :: Internet :: WWW/HTTP", ], ) + +# trigger system test diff --git a/packages/google-cloud-access-approval/setup.py b/packages/google-cloud-access-approval/setup.py index 7da5c9780181..4e8cf3b54e62 100644 --- a/packages/google-cloud-access-approval/setup.py +++ b/packages/google-cloud-access-approval/setup.py @@ -97,3 +97,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-automl/setup.py b/packages/google-cloud-automl/setup.py index fc5d8bfa6429..a8c9e2c7510f 100644 --- a/packages/google-cloud-automl/setup.py +++ b/packages/google-cloud-automl/setup.py @@ -103,3 +103,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-bigquery-connection/setup.py b/packages/google-cloud-bigquery-connection/setup.py index 9515d7a90b2e..73b713ecfad1 100644 --- a/packages/google-cloud-bigquery-connection/setup.py +++ b/packages/google-cloud-bigquery-connection/setup.py @@ -98,3 +98,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-bigquery-datatransfer/setup.py b/packages/google-cloud-bigquery-datatransfer/setup.py index afeab9dee42b..4fd94b233945 100644 --- a/packages/google-cloud-bigquery-datatransfer/setup.py +++ b/packages/google-cloud-bigquery-datatransfer/setup.py @@ -97,3 +97,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-bigquery-reservation/setup.py b/packages/google-cloud-bigquery-reservation/setup.py index 07e6eba134b7..5fde510ff5e0 100644 --- a/packages/google-cloud-bigquery-reservation/setup.py +++ b/packages/google-cloud-bigquery-reservation/setup.py @@ -98,3 +98,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-bigquery-storage/setup.py b/packages/google-cloud-bigquery-storage/setup.py index d3a6bc437845..69a8e259ef4b 100644 --- a/packages/google-cloud-bigquery-storage/setup.py +++ b/packages/google-cloud-bigquery-storage/setup.py @@ -101,3 +101,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-bigquery/setup.py b/packages/google-cloud-bigquery/setup.py index 13aa8b6ca346..3bc371bad0c3 100644 --- a/packages/google-cloud-bigquery/setup.py +++ b/packages/google-cloud-bigquery/setup.py @@ -15,3 +15,5 @@ import setuptools # type: ignore setuptools.setup() + +# trigger system test diff --git a/packages/google-cloud-bigtable/setup.py b/packages/google-cloud-bigtable/setup.py index af210c742f7c..a386b5403787 100644 --- a/packages/google-cloud-bigtable/setup.py +++ b/packages/google-cloud-bigtable/setup.py @@ -101,3 +101,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-compute/setup.py b/packages/google-cloud-compute/setup.py index 3cf505112844..01fbe832f898 100644 --- a/packages/google-cloud-compute/setup.py +++ b/packages/google-cloud-compute/setup.py @@ -95,3 +95,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-container/setup.py b/packages/google-cloud-container/setup.py index afad62d98b9c..c7700105c1b8 100644 --- a/packages/google-cloud-container/setup.py +++ b/packages/google-cloud-container/setup.py @@ -95,3 +95,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-dataproc/setup.py b/packages/google-cloud-dataproc/setup.py index 24eddbb83a43..8f5d9774c6e1 100644 --- a/packages/google-cloud-dataproc/setup.py +++ b/packages/google-cloud-dataproc/setup.py @@ -96,3 +96,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-datastore/setup.py b/packages/google-cloud-datastore/setup.py index b0cac8c0ec50..896bd1eea4d5 100644 --- a/packages/google-cloud-datastore/setup.py +++ b/packages/google-cloud-datastore/setup.py @@ -96,3 +96,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-dlp/setup.py b/packages/google-cloud-dlp/setup.py index 7f7d1bd82cad..e4aa8cd23284 100644 --- a/packages/google-cloud-dlp/setup.py +++ b/packages/google-cloud-dlp/setup.py @@ -95,3 +95,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-dns/setup.py b/packages/google-cloud-dns/setup.py index 21f64e31a38d..f6b33dda5e0a 100644 --- a/packages/google-cloud-dns/setup.py +++ b/packages/google-cloud-dns/setup.py @@ -92,3 +92,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-error-reporting/setup.py b/packages/google-cloud-error-reporting/setup.py index 58594b2a336c..b3f076d51cf0 100644 --- a/packages/google-cloud-error-reporting/setup.py +++ b/packages/google-cloud-error-reporting/setup.py @@ -98,3 +98,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-firestore/setup.py b/packages/google-cloud-firestore/setup.py index 520b78e9bc96..6be2213c639d 100644 --- a/packages/google-cloud-firestore/setup.py +++ b/packages/google-cloud-firestore/setup.py @@ -96,3 +96,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-kms/setup.py b/packages/google-cloud-kms/setup.py index 91013ea6d3b3..0c377cbf571f 100644 --- a/packages/google-cloud-kms/setup.py +++ b/packages/google-cloud-kms/setup.py @@ -96,3 +96,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-logging/setup.py b/packages/google-cloud-logging/setup.py index 97568c52429b..ad712b682f74 100644 --- a/packages/google-cloud-logging/setup.py +++ b/packages/google-cloud-logging/setup.py @@ -100,3 +100,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-monitoring/setup.py b/packages/google-cloud-monitoring/setup.py index e549aed6fc15..35d5bbbf5860 100644 --- a/packages/google-cloud-monitoring/setup.py +++ b/packages/google-cloud-monitoring/setup.py @@ -96,3 +96,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-ndb/setup.py b/packages/google-cloud-ndb/setup.py index 8c22f5349b16..42fbb112b55b 100644 --- a/packages/google-cloud-ndb/setup.py +++ b/packages/google-cloud-ndb/setup.py @@ -88,3 +88,5 @@ def main(): if __name__ == "__main__": main() + +# trigger system test diff --git a/packages/google-cloud-os-config/setup.py b/packages/google-cloud-os-config/setup.py index 8e3a7eabbf6e..dc0ff5c268d3 100644 --- a/packages/google-cloud-os-config/setup.py +++ b/packages/google-cloud-os-config/setup.py @@ -95,3 +95,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-pubsub/setup.py b/packages/google-cloud-pubsub/setup.py index 96a086c5c694..5456c67604f9 100644 --- a/packages/google-cloud-pubsub/setup.py +++ b/packages/google-cloud-pubsub/setup.py @@ -99,3 +99,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-scheduler/setup.py b/packages/google-cloud-scheduler/setup.py index 7209982c00b6..783743fd77a3 100644 --- a/packages/google-cloud-scheduler/setup.py +++ b/packages/google-cloud-scheduler/setup.py @@ -95,3 +95,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-spanner-dbapi-driver/setup.py b/packages/google-cloud-spanner-dbapi-driver/setup.py index cfc0b06a229b..198c667b8f35 100644 --- a/packages/google-cloud-spanner-dbapi-driver/setup.py +++ b/packages/google-cloud-spanner-dbapi-driver/setup.py @@ -81,3 +81,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-spanner/setup.py b/packages/google-cloud-spanner/setup.py index e7dce1a06904..68b4cf69e195 100644 --- a/packages/google-cloud-spanner/setup.py +++ b/packages/google-cloud-spanner/setup.py @@ -117,3 +117,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-speech/setup.py b/packages/google-cloud-speech/setup.py index 46a8b8bc8298..910a858398e1 100644 --- a/packages/google-cloud-speech/setup.py +++ b/packages/google-cloud-speech/setup.py @@ -95,3 +95,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-storage/setup.py b/packages/google-cloud-storage/setup.py index 0f339a119486..d8e8e462ef47 100644 --- a/packages/google-cloud-storage/setup.py +++ b/packages/google-cloud-storage/setup.py @@ -141,3 +141,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-tasks/setup.py b/packages/google-cloud-tasks/setup.py index 36fb9a623d04..782e7aa119b4 100644 --- a/packages/google-cloud-tasks/setup.py +++ b/packages/google-cloud-tasks/setup.py @@ -96,3 +96,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-testutils/setup.py b/packages/google-cloud-testutils/setup.py index 89e592a34965..218bb20cd1ad 100644 --- a/packages/google-cloud-testutils/setup.py +++ b/packages/google-cloud-testutils/setup.py @@ -75,3 +75,5 @@ ], zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-texttospeech/setup.py b/packages/google-cloud-texttospeech/setup.py index 6787142e1078..861bbfe5500c 100644 --- a/packages/google-cloud-texttospeech/setup.py +++ b/packages/google-cloud-texttospeech/setup.py @@ -97,3 +97,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-translate/setup.py b/packages/google-cloud-translate/setup.py index c79b32162195..9180964c6dc1 100644 --- a/packages/google-cloud-translate/setup.py +++ b/packages/google-cloud-translate/setup.py @@ -97,3 +97,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-videointelligence/setup.py b/packages/google-cloud-videointelligence/setup.py index 3573c98f7480..68175819c17d 100644 --- a/packages/google-cloud-videointelligence/setup.py +++ b/packages/google-cloud-videointelligence/setup.py @@ -97,3 +97,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-cloud-vision/setup.py b/packages/google-cloud-vision/setup.py index c3b44abecd05..3a42b78540a4 100644 --- a/packages/google-cloud-vision/setup.py +++ b/packages/google-cloud-vision/setup.py @@ -95,3 +95,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/google-resumable-media/setup.py b/packages/google-resumable-media/setup.py index 44606bbfb63c..a428dfe4817f 100644 --- a/packages/google-resumable-media/setup.py +++ b/packages/google-resumable-media/setup.py @@ -66,3 +66,5 @@ 'Topic :: Internet', ], ) + +# trigger system test diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index de65ff230c85..42d8da244056 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -105,3 +105,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test diff --git a/packages/sqlalchemy-bigquery/setup.py b/packages/sqlalchemy-bigquery/setup.py index 8d7ad005c2ee..6f1752eba471 100644 --- a/packages/sqlalchemy-bigquery/setup.py +++ b/packages/sqlalchemy-bigquery/setup.py @@ -131,3 +131,5 @@ def readme(): # obsolete legitimate ones. obsoletes=["pybigquery"], ) + +# trigger system test diff --git a/packages/sqlalchemy-spanner/setup.py b/packages/sqlalchemy-spanner/setup.py index b01fd1d74301..82ffeefac0ce 100644 --- a/packages/sqlalchemy-spanner/setup.py +++ b/packages/sqlalchemy-spanner/setup.py @@ -84,3 +84,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system test