From 4f00dae56bbd500c8239449e1e693684348fd183 Mon Sep 17 00:00:00 2001
From: ohmayr <omairn@google.com>
Date: Fri, 12 Jun 2026 08:11:50 +0000
Subject: [PATCH 1/9] chore(ci): optimize kokoro system tests with concurrency

---
 .kokoro/system.sh | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/.kokoro/system.sh b/.kokoro/system.sh
index 469d0e81c7fa..491767c7dfe2 100755
--- a/.kokoro/system.sh
+++ b/.kokoro/system.sh
@@ -140,10 +140,26 @@ for path in `find 'packages' \
   set -e
 
   if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]]; then
-      # Call the function - its internal exports won't affect the next loop
-      run_package_test "$package_name" || RETVAL=$?
+      PACKAGES_TO_TEST="${PACKAGES_TO_TEST} ${package_name}"
   else
       echo "No changes in ${package_name} and not a continuous build, skipping."
   fi
 done
+
+if [ -n "$PACKAGES_TO_TEST" ]; then
+  mkdir -p .logs
+  export -f run_package_test
+  export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR
+  
+  echo "$PACKAGES_TO_TEST" | xargs -n 1 -P 8 -I {} bash -c 'run_package_test "{}" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"'
+  
+  for failed in .logs/*.failed; do
+    if [ -f "$failed" ]; then
+      echo "--- FAILED: ${failed%.failed} ---"
+      cat "${failed%.failed}.log"
+      RETVAL=1
+    fi
+  done
+fi
+
 exit ${RETVAL}

From 493ba3a0f21727e5b985a96b8aba49689c3ff906 Mon Sep 17 00:00:00 2001
From: ohmayr <omairn@google.com>
Date: Fri, 12 Jun 2026 08:12:30 +0000
Subject: [PATCH 2/9] test

---
 packages/bigframes/setup.py                          | 2 ++
 packages/bigquery-magics/setup.py                    | 2 ++
 packages/django-google-spanner/setup.py              | 2 ++
 packages/gapic-generator/setup.py                    | 2 ++
 packages/google-auth/setup.py                        | 2 ++
 packages/google-cloud-access-approval/setup.py       | 2 ++
 packages/google-cloud-automl/setup.py                | 2 ++
 packages/google-cloud-bigquery-connection/setup.py   | 2 ++
 packages/google-cloud-bigquery-datatransfer/setup.py | 2 ++
 packages/google-cloud-bigquery-reservation/setup.py  | 2 ++
 packages/google-cloud-bigquery-storage/setup.py      | 2 ++
 packages/google-cloud-bigquery/setup.py              | 2 ++
 packages/google-cloud-bigtable/setup.py              | 2 ++
 packages/google-cloud-compute/setup.py               | 2 ++
 packages/google-cloud-container/setup.py             | 2 ++
 packages/google-cloud-dataproc/setup.py              | 2 ++
 packages/google-cloud-datastore/setup.py             | 2 ++
 packages/google-cloud-dlp/setup.py                   | 2 ++
 packages/google-cloud-dns/setup.py                   | 2 ++
 packages/google-cloud-error-reporting/setup.py       | 2 ++
 packages/google-cloud-firestore/setup.py             | 2 ++
 packages/google-cloud-kms/setup.py                   | 2 ++
 packages/google-cloud-logging/setup.py               | 2 ++
 packages/google-cloud-monitoring/setup.py            | 2 ++
 packages/google-cloud-ndb/setup.py                   | 2 ++
 packages/google-cloud-os-config/setup.py             | 2 ++
 packages/google-cloud-pubsub/setup.py                | 2 ++
 packages/google-cloud-scheduler/setup.py             | 2 ++
 packages/google-cloud-spanner-dbapi-driver/setup.py  | 2 ++
 packages/google-cloud-spanner/setup.py               | 2 ++
 packages/google-cloud-speech/setup.py                | 2 ++
 packages/google-cloud-storage/setup.py               | 2 ++
 packages/google-cloud-tasks/setup.py                 | 2 ++
 packages/google-cloud-testutils/setup.py             | 2 ++
 packages/google-cloud-texttospeech/setup.py          | 2 ++
 packages/google-cloud-translate/setup.py             | 2 ++
 packages/google-cloud-videointelligence/setup.py     | 2 ++
 packages/google-cloud-vision/setup.py                | 2 ++
 packages/google-resumable-media/setup.py             | 2 ++
 packages/pandas-gbq/setup.py                         | 2 ++
 packages/sqlalchemy-bigquery/setup.py                | 2 ++
 packages/sqlalchemy-spanner/setup.py                 | 2 ++
 42 files changed, 84 insertions(+)

diff --git a/packages/bigframes/setup.py b/packages/bigframes/setup.py
index 76b98b88d312..5d7b2aadb999 100644
--- a/packages/bigframes/setup.py
+++ b/packages/bigframes/setup.py
@@ -162,3 +162,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/bigquery-magics/setup.py b/packages/bigquery-magics/setup.py
index 763ca3afb9bc..0bc25158ab81 100644
--- a/packages/bigquery-magics/setup.py
+++ b/packages/bigquery-magics/setup.py
@@ -123,3 +123,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/django-google-spanner/setup.py b/packages/django-google-spanner/setup.py
index 973dc622047e..58db4adfea1e 100644
--- a/packages/django-google-spanner/setup.py
+++ b/packages/django-google-spanner/setup.py
@@ -73,3 +73,5 @@
     extras_require=extras,
     python_requires=">=3.10",
 )
+
+# trigger system test
diff --git a/packages/gapic-generator/setup.py b/packages/gapic-generator/setup.py
index 237281229446..a5253c90ff66 100644
--- a/packages/gapic-generator/setup.py
+++ b/packages/gapic-generator/setup.py
@@ -85,3 +85,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-auth/setup.py b/packages/google-auth/setup.py
index cf3148130d6e..51fe2038e205 100644
--- a/packages/google-auth/setup.py
+++ b/packages/google-auth/setup.py
@@ -132,3 +132,5 @@
         "Topic :: Internet :: WWW/HTTP",
     ],
 )
+
+# trigger system test
diff --git a/packages/google-cloud-access-approval/setup.py b/packages/google-cloud-access-approval/setup.py
index 7da5c9780181..4e8cf3b54e62 100644
--- a/packages/google-cloud-access-approval/setup.py
+++ b/packages/google-cloud-access-approval/setup.py
@@ -97,3 +97,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-automl/setup.py b/packages/google-cloud-automl/setup.py
index fc5d8bfa6429..a8c9e2c7510f 100644
--- a/packages/google-cloud-automl/setup.py
+++ b/packages/google-cloud-automl/setup.py
@@ -103,3 +103,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-bigquery-connection/setup.py b/packages/google-cloud-bigquery-connection/setup.py
index 9515d7a90b2e..73b713ecfad1 100644
--- a/packages/google-cloud-bigquery-connection/setup.py
+++ b/packages/google-cloud-bigquery-connection/setup.py
@@ -98,3 +98,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-bigquery-datatransfer/setup.py b/packages/google-cloud-bigquery-datatransfer/setup.py
index afeab9dee42b..4fd94b233945 100644
--- a/packages/google-cloud-bigquery-datatransfer/setup.py
+++ b/packages/google-cloud-bigquery-datatransfer/setup.py
@@ -97,3 +97,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-bigquery-reservation/setup.py b/packages/google-cloud-bigquery-reservation/setup.py
index 07e6eba134b7..5fde510ff5e0 100644
--- a/packages/google-cloud-bigquery-reservation/setup.py
+++ b/packages/google-cloud-bigquery-reservation/setup.py
@@ -98,3 +98,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-bigquery-storage/setup.py b/packages/google-cloud-bigquery-storage/setup.py
index d3a6bc437845..69a8e259ef4b 100644
--- a/packages/google-cloud-bigquery-storage/setup.py
+++ b/packages/google-cloud-bigquery-storage/setup.py
@@ -101,3 +101,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-bigquery/setup.py b/packages/google-cloud-bigquery/setup.py
index 13aa8b6ca346..3bc371bad0c3 100644
--- a/packages/google-cloud-bigquery/setup.py
+++ b/packages/google-cloud-bigquery/setup.py
@@ -15,3 +15,5 @@
 import setuptools  # type: ignore
 
 setuptools.setup()
+
+# trigger system test
diff --git a/packages/google-cloud-bigtable/setup.py b/packages/google-cloud-bigtable/setup.py
index af210c742f7c..a386b5403787 100644
--- a/packages/google-cloud-bigtable/setup.py
+++ b/packages/google-cloud-bigtable/setup.py
@@ -101,3 +101,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-compute/setup.py b/packages/google-cloud-compute/setup.py
index 3cf505112844..01fbe832f898 100644
--- a/packages/google-cloud-compute/setup.py
+++ b/packages/google-cloud-compute/setup.py
@@ -95,3 +95,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-container/setup.py b/packages/google-cloud-container/setup.py
index afad62d98b9c..c7700105c1b8 100644
--- a/packages/google-cloud-container/setup.py
+++ b/packages/google-cloud-container/setup.py
@@ -95,3 +95,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-dataproc/setup.py b/packages/google-cloud-dataproc/setup.py
index 24eddbb83a43..8f5d9774c6e1 100644
--- a/packages/google-cloud-dataproc/setup.py
+++ b/packages/google-cloud-dataproc/setup.py
@@ -96,3 +96,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-datastore/setup.py b/packages/google-cloud-datastore/setup.py
index b0cac8c0ec50..896bd1eea4d5 100644
--- a/packages/google-cloud-datastore/setup.py
+++ b/packages/google-cloud-datastore/setup.py
@@ -96,3 +96,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-dlp/setup.py b/packages/google-cloud-dlp/setup.py
index 7f7d1bd82cad..e4aa8cd23284 100644
--- a/packages/google-cloud-dlp/setup.py
+++ b/packages/google-cloud-dlp/setup.py
@@ -95,3 +95,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-dns/setup.py b/packages/google-cloud-dns/setup.py
index 21f64e31a38d..f6b33dda5e0a 100644
--- a/packages/google-cloud-dns/setup.py
+++ b/packages/google-cloud-dns/setup.py
@@ -92,3 +92,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-error-reporting/setup.py b/packages/google-cloud-error-reporting/setup.py
index 58594b2a336c..b3f076d51cf0 100644
--- a/packages/google-cloud-error-reporting/setup.py
+++ b/packages/google-cloud-error-reporting/setup.py
@@ -98,3 +98,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-firestore/setup.py b/packages/google-cloud-firestore/setup.py
index 520b78e9bc96..6be2213c639d 100644
--- a/packages/google-cloud-firestore/setup.py
+++ b/packages/google-cloud-firestore/setup.py
@@ -96,3 +96,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-kms/setup.py b/packages/google-cloud-kms/setup.py
index 91013ea6d3b3..0c377cbf571f 100644
--- a/packages/google-cloud-kms/setup.py
+++ b/packages/google-cloud-kms/setup.py
@@ -96,3 +96,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-logging/setup.py b/packages/google-cloud-logging/setup.py
index 97568c52429b..ad712b682f74 100644
--- a/packages/google-cloud-logging/setup.py
+++ b/packages/google-cloud-logging/setup.py
@@ -100,3 +100,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-monitoring/setup.py b/packages/google-cloud-monitoring/setup.py
index e549aed6fc15..35d5bbbf5860 100644
--- a/packages/google-cloud-monitoring/setup.py
+++ b/packages/google-cloud-monitoring/setup.py
@@ -96,3 +96,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-ndb/setup.py b/packages/google-cloud-ndb/setup.py
index 8c22f5349b16..42fbb112b55b 100644
--- a/packages/google-cloud-ndb/setup.py
+++ b/packages/google-cloud-ndb/setup.py
@@ -88,3 +88,5 @@ def main():
 
 if __name__ == "__main__":
     main()
+
+# trigger system test
diff --git a/packages/google-cloud-os-config/setup.py b/packages/google-cloud-os-config/setup.py
index 8e3a7eabbf6e..dc0ff5c268d3 100644
--- a/packages/google-cloud-os-config/setup.py
+++ b/packages/google-cloud-os-config/setup.py
@@ -95,3 +95,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-pubsub/setup.py b/packages/google-cloud-pubsub/setup.py
index 96a086c5c694..5456c67604f9 100644
--- a/packages/google-cloud-pubsub/setup.py
+++ b/packages/google-cloud-pubsub/setup.py
@@ -99,3 +99,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-scheduler/setup.py b/packages/google-cloud-scheduler/setup.py
index 7209982c00b6..783743fd77a3 100644
--- a/packages/google-cloud-scheduler/setup.py
+++ b/packages/google-cloud-scheduler/setup.py
@@ -95,3 +95,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-spanner-dbapi-driver/setup.py b/packages/google-cloud-spanner-dbapi-driver/setup.py
index cfc0b06a229b..198c667b8f35 100644
--- a/packages/google-cloud-spanner-dbapi-driver/setup.py
+++ b/packages/google-cloud-spanner-dbapi-driver/setup.py
@@ -81,3 +81,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-spanner/setup.py b/packages/google-cloud-spanner/setup.py
index e7dce1a06904..68b4cf69e195 100644
--- a/packages/google-cloud-spanner/setup.py
+++ b/packages/google-cloud-spanner/setup.py
@@ -117,3 +117,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-speech/setup.py b/packages/google-cloud-speech/setup.py
index 46a8b8bc8298..910a858398e1 100644
--- a/packages/google-cloud-speech/setup.py
+++ b/packages/google-cloud-speech/setup.py
@@ -95,3 +95,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-storage/setup.py b/packages/google-cloud-storage/setup.py
index 0f339a119486..d8e8e462ef47 100644
--- a/packages/google-cloud-storage/setup.py
+++ b/packages/google-cloud-storage/setup.py
@@ -141,3 +141,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-tasks/setup.py b/packages/google-cloud-tasks/setup.py
index 36fb9a623d04..782e7aa119b4 100644
--- a/packages/google-cloud-tasks/setup.py
+++ b/packages/google-cloud-tasks/setup.py
@@ -96,3 +96,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-testutils/setup.py b/packages/google-cloud-testutils/setup.py
index 89e592a34965..218bb20cd1ad 100644
--- a/packages/google-cloud-testutils/setup.py
+++ b/packages/google-cloud-testutils/setup.py
@@ -75,3 +75,5 @@
     ],
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-texttospeech/setup.py b/packages/google-cloud-texttospeech/setup.py
index 6787142e1078..861bbfe5500c 100644
--- a/packages/google-cloud-texttospeech/setup.py
+++ b/packages/google-cloud-texttospeech/setup.py
@@ -97,3 +97,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-translate/setup.py b/packages/google-cloud-translate/setup.py
index c79b32162195..9180964c6dc1 100644
--- a/packages/google-cloud-translate/setup.py
+++ b/packages/google-cloud-translate/setup.py
@@ -97,3 +97,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-videointelligence/setup.py b/packages/google-cloud-videointelligence/setup.py
index 3573c98f7480..68175819c17d 100644
--- a/packages/google-cloud-videointelligence/setup.py
+++ b/packages/google-cloud-videointelligence/setup.py
@@ -97,3 +97,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-cloud-vision/setup.py b/packages/google-cloud-vision/setup.py
index c3b44abecd05..3a42b78540a4 100644
--- a/packages/google-cloud-vision/setup.py
+++ b/packages/google-cloud-vision/setup.py
@@ -95,3 +95,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/google-resumable-media/setup.py b/packages/google-resumable-media/setup.py
index 44606bbfb63c..a428dfe4817f 100644
--- a/packages/google-resumable-media/setup.py
+++ b/packages/google-resumable-media/setup.py
@@ -66,3 +66,5 @@
         'Topic :: Internet',
     ],
 )
+
+# trigger system test
diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py
index de65ff230c85..42d8da244056 100644
--- a/packages/pandas-gbq/setup.py
+++ b/packages/pandas-gbq/setup.py
@@ -105,3 +105,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test
diff --git a/packages/sqlalchemy-bigquery/setup.py b/packages/sqlalchemy-bigquery/setup.py
index 8d7ad005c2ee..6f1752eba471 100644
--- a/packages/sqlalchemy-bigquery/setup.py
+++ b/packages/sqlalchemy-bigquery/setup.py
@@ -131,3 +131,5 @@ def readme():
     # obsolete legitimate ones.
     obsoletes=["pybigquery"],
 )
+
+# trigger system test
diff --git a/packages/sqlalchemy-spanner/setup.py b/packages/sqlalchemy-spanner/setup.py
index b01fd1d74301..82ffeefac0ce 100644
--- a/packages/sqlalchemy-spanner/setup.py
+++ b/packages/sqlalchemy-spanner/setup.py
@@ -84,3 +84,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system test

From 97fae6dcb230f165cebdbd851468eb1c22dbda41 Mon Sep 17 00:00:00 2001
From: ohmayr <omairn@google.com>
Date: Fri, 12 Jun 2026 08:29:23 +0000
Subject: [PATCH 3/9] fix filename issue

---
 .kokoro/system.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.kokoro/system.sh b/.kokoro/system.sh
index 491767c7dfe2..9cd4f5f45298 100755
--- a/.kokoro/system.sh
+++ b/.kokoro/system.sh
@@ -151,7 +151,7 @@ if [ -n "$PACKAGES_TO_TEST" ]; then
   export -f run_package_test
   export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR
   
-  echo "$PACKAGES_TO_TEST" | xargs -n 1 -P 8 -I {} bash -c 'run_package_test "{}" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"'
+  echo "$PACKAGES_TO_TEST" | tr ' ' '\n' | awk 'NF' | xargs -P 8 -I {} bash -c 'run_package_test "{}" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"'
   
   for failed in .logs/*.failed; do
     if [ -f "$failed" ]; then

From 064b4226be33b461238f00f2c75b684efb47c324 Mon Sep 17 00:00:00 2001
From: ohmayr <omairn@google.com>
Date: Fri, 12 Jun 2026 17:32:43 +0000
Subject: [PATCH 4/9] drop workers from 8 to 3

---
 .kokoro/system.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.kokoro/system.sh b/.kokoro/system.sh
index 9cd4f5f45298..088770c7d6f0 100755
--- a/.kokoro/system.sh
+++ b/.kokoro/system.sh
@@ -151,7 +151,7 @@ if [ -n "$PACKAGES_TO_TEST" ]; then
   export -f run_package_test
   export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR
   
-  echo "$PACKAGES_TO_TEST" | tr ' ' '\n' | awk 'NF' | xargs -P 8 -I {} bash -c 'run_package_test "{}" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"'
+  echo "$PACKAGES_TO_TEST" | tr ' ' '\n' | awk 'NF' | xargs -P 3 -I {} bash -c 'run_package_test "{}" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"'
   
   for failed in .logs/*.failed; do
     if [ -f "$failed" ]; then

From bbbd5a65f67b21138891813e485350b7ed85f57e Mon Sep 17 00:00:00 2001
From: ohmayr <omairn@google.com>
Date: Fri, 12 Jun 2026 20:16:01 +0000
Subject: [PATCH 5/9] isolated gcloud config

---
 .kokoro/system.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.kokoro/system.sh b/.kokoro/system.sh
index 088770c7d6f0..e9c024014a18 100755
--- a/.kokoro/system.sh
+++ b/.kokoro/system.sh
@@ -75,6 +75,10 @@ run_package_test() {
   # Export variables for the duration of this function's sub-processes
   export PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS NOX_FILE NOX_SESSION
   export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}"
+  
+  # Isolate gcloud state to prevent SQLite lock deadlocks and project race conditions
+  export CLOUDSDK_CONFIG="/tmpfs/.gcloud_config_$(basename ${package_name})"
+  mkdir -p "$CLOUDSDK_CONFIG"
 
   gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS"
   gcloud config set project "$PROJECT_ID"

From 064a81f1f717d55a80ad92c9867b92b49b13281a Mon Sep 17 00:00:00 2001
From: ohmayr <omairn@google.com>
Date: Fri, 12 Jun 2026 21:29:22 +0000
Subject: [PATCH 6/9] fail fast in 15 mins for quick feedback

---
 .kokoro/system.sh | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/.kokoro/system.sh b/.kokoro/system.sh
index e9c024014a18..eccddece4513 100755
--- a/.kokoro/system.sh
+++ b/.kokoro/system.sh
@@ -76,6 +76,10 @@ run_package_test() {
   export PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS NOX_FILE NOX_SESSION
   export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}"
   
+  # Isolate PIP cache to prevent concurrent pip file lock deadlocks
+  export PIP_CACHE_DIR="/tmpfs/.pip_cache_$(basename ${package_name})"
+  mkdir -p "$PIP_CACHE_DIR"
+
   # Isolate gcloud state to prevent SQLite lock deadlocks and project race conditions
   export CLOUDSDK_CONFIG="/tmpfs/.gcloud_config_$(basename ${package_name})"
   mkdir -p "$CLOUDSDK_CONFIG"
@@ -152,10 +156,10 @@ done
 
 if [ -n "$PACKAGES_TO_TEST" ]; then
   mkdir -p .logs
-  export -f run_package_test
-  export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR
-  
-  echo "$PACKAGES_TO_TEST" | tr ' ' '\n' | awk 'NF' | xargs -P 3 -I {} bash -c 'run_package_test "{}" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"'
+  echo "Running system tests in parallel (3 workers)..."
+  mkdir -p .logs
+  # Use timeout to prevent infinite hangs, and < /dev/null to prevent stdin blocks
+  echo "$PACKAGES_TO_TEST" | tr ' ' '\n' | awk 'NF' | xargs -P 3 -I {} bash -c 'timeout 15m bash -c "run_package_test \"{}\" < /dev/null" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"'
   
   for failed in .logs/*.failed; do
     if [ -f "$failed" ]; then

From 237486c789336c4fbd8f71d5ed44e1012c050592 Mon Sep 17 00:00:00 2001
From: ohmayr <omairn@google.com>
Date: Fri, 12 Jun 2026 21:35:18 +0000
Subject: [PATCH 7/9] minor fix

---
 .kokoro/system.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.kokoro/system.sh b/.kokoro/system.sh
index eccddece4513..1b6938a652af 100755
--- a/.kokoro/system.sh
+++ b/.kokoro/system.sh
@@ -156,8 +156,10 @@ done
 
 if [ -n "$PACKAGES_TO_TEST" ]; then
   mkdir -p .logs
+  export -f run_package_test
+  export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR
+  
   echo "Running system tests in parallel (3 workers)..."
-  mkdir -p .logs
   # Use timeout to prevent infinite hangs, and < /dev/null to prevent stdin blocks
   echo "$PACKAGES_TO_TEST" | tr ' ' '\n' | awk 'NF' | xargs -P 3 -I {} bash -c 'timeout 15m bash -c "run_package_test \"{}\" < /dev/null" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"'
   

From 8f70bdd6314e80ff4b406fdf82a6cb53bc47d01c Mon Sep 17 00:00:00 2001
From: ohmayr <omairn@google.com>
Date: Fri, 12 Jun 2026 22:17:22 +0000
Subject: [PATCH 8/9] manage package based workers

---
 .kokoro/system.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.kokoro/system.sh b/.kokoro/system.sh
index 1b6938a652af..18d4eefbd5f0 100755
--- a/.kokoro/system.sh
+++ b/.kokoro/system.sh
@@ -76,6 +76,11 @@ run_package_test() {
   export PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS NOX_FILE NOX_SESSION
   export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}"
   
+  # Limit pytest-xdist to 4 workers per package. When 3 packages run in parallel,
+  # if each uses -n=auto they will spawn 16 workers each (48 total threads), 
+  # completely thrashing the CPU and causing all tests to hang/timeout.
+  export PYTEST_ADDOPTS="-n 4"
+
   # Isolate PIP cache to prevent concurrent pip file lock deadlocks
   export PIP_CACHE_DIR="/tmpfs/.pip_cache_$(basename ${package_name})"
   mkdir -p "$PIP_CACHE_DIR"
@@ -83,6 +88,10 @@ run_package_test() {
   # Isolate gcloud state to prevent SQLite lock deadlocks and project race conditions
   export CLOUDSDK_CONFIG="/tmpfs/.gcloud_config_$(basename ${package_name})"
   mkdir -p "$CLOUDSDK_CONFIG"
+  
+  # Isolate boto config (used by storage/bigquery) to prevent lock contention
+  export BOTO_CONFIG="/tmpfs/.boto_$(basename ${package_name})"
+  touch "$BOTO_CONFIG"
 
   gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS"
   gcloud config set project "$PROJECT_ID"

From 2a6aaff88d6ffebbec9e4bed1217123648a688f4 Mon Sep 17 00:00:00 2001
From: ohmayr <omairn@google.com>
Date: Fri, 12 Jun 2026 23:49:11 +0000
Subject: [PATCH 9/9] divide tests based on category

---
 .kokoro/system.sh | 63 ++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 48 insertions(+), 15 deletions(-)

diff --git a/.kokoro/system.sh b/.kokoro/system.sh
index 18d4eefbd5f0..d6343c6c15c6 100755
--- a/.kokoro/system.sh
+++ b/.kokoro/system.sh
@@ -75,11 +75,6 @@ run_package_test() {
   # Export variables for the duration of this function's sub-processes
   export PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS NOX_FILE NOX_SESSION
   export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}"
-  
-  # Limit pytest-xdist to 4 workers per package. When 3 packages run in parallel,
-  # if each uses -n=auto they will spawn 16 workers each (48 total threads), 
-  # completely thrashing the CPU and causing all tests to hang/timeout.
-  export PYTEST_ADDOPTS="-n 4"
 
   # Isolate PIP cache to prevent concurrent pip file lock deadlocks
   export PIP_CACHE_DIR="/tmpfs/.pip_cache_$(basename ${package_name})"
@@ -164,18 +159,56 @@ for path in `find 'packages' \
 done
 
 if [ -n "$PACKAGES_TO_TEST" ]; then
-  mkdir -p .logs
   export -f run_package_test
   export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR
-  
-  echo "Running system tests in parallel (3 workers)..."
-  # Use timeout to prevent infinite hangs, and < /dev/null to prevent stdin blocks
-  echo "$PACKAGES_TO_TEST" | tr ' ' '\n' | awk 'NF' | xargs -P 3 -I {} bash -c 'timeout 15m bash -c "run_package_test \"{}\" < /dev/null" > ".logs/{}.log" 2>&1 || touch ".logs/{}.failed"'
-  
-  for failed in .logs/*.failed; do
-    if [ -f "$failed" ]; then
-      echo "--- FAILED: ${failed%.failed} ---"
-      cat "${failed%.failed}.log"
+
+  # 1. DYNAMIC ROUTING: Automatically detect which packages are CPU hogs by checking if they install pytest-xdist or hardcode workers
+  LIGHT_TO_TEST=""
+  HEAVY_TO_TEST=""
+  for pkg in $PACKAGES_TO_TEST; do
+    if grep -qE "pytest-xdist|-n=auto|-n=[0-9]+" "packages/$pkg/noxfile.py" "packages/$pkg/setup.py" 2>/dev/null; then
+      HEAVY_TO_TEST="$HEAVY_TO_TEST $pkg"
+    else
+      LIGHT_TO_TEST="$LIGHT_TO_TEST $pkg"
+    fi
+  done
+
+  # 2. PARALLEL LANE (Live Streaming): Run light packages with a parallel job queue. 
+  # We prefix every line with the package name so output streams LIVE and remains readable.
+  if [ -n "$LIGHT_TO_TEST" ]; then
+    echo "============================================================"
+    echo "Running Lightweight Packages in Parallel (4 workers max)"
+    echo "============================================================"
+    for pkg in $LIGHT_TO_TEST; do
+      (
+        timeout 15m bash -c "run_package_test \"$pkg\" < /dev/null" 2>&1 | awk -v prefix="[$pkg]" '{print prefix, $0}'
+        if [ ${PIPESTATUS[0]} -ne 0 ]; then touch ".failed_$pkg"; fi
+      ) &
+      # Limit parallel background jobs to 4
+      while [ $(jobs -r | wc -l) -ge 4 ]; do sleep 1; done
+    done
+    wait # Wait for all parallel jobs to finish
+  fi
+
+  # 3. SEQUENTIAL VIP LANE: Run heavy packages one-by-one so they have 100% of the VM resources.
+  if [ -n "$HEAVY_TO_TEST" ]; then
+    echo "============================================================"
+    echo "Running CPU-Intensive Packages Sequentially"
+    echo "============================================================"
+    for pkg in $HEAVY_TO_TEST; do
+      if [ -n "$pkg" ]; then
+        echo "[$pkg] Starting sequential execution..."
+        timeout 25m bash -c "run_package_test \"$pkg\" < /dev/null" 2>&1 | awk -v prefix="[$pkg]" '{print prefix, $0}'
+        if [ ${PIPESTATUS[0]} -ne 0 ]; then touch ".failed_$pkg"; fi
+      fi
+    done
+  fi
+
+  # 4. FAIL STATE EVALUATION
+  for failed_marker in .failed_*; do
+    if [ -f "$failed_marker" ]; then
+      failed_pkg="${failed_marker#.failed_}"
+      echo "--- FAILED: $failed_pkg ---"
       RETVAL=1
     fi
   done