Ngmix v2.0 (CI mirror of #740)

.github/workflows/fast-tests.yml

@@ -0,0 +1,60 @@
+name: Fast tests
+
+# The fast inner-loop signal: the `tests/` suite minus the heavy and
+# cluster-only tiers (`-m "not slow and not candide"`), run on every push and
+# pull request. This is the quick gate that complements — does not replace —
+# `deploy-image.yml`, which builds the dev image and runs the *full* suite
+# inside it (the environment that ships). This job skips the Docker build, so
+# it returns in a couple of minutes instead of waiting on the image.
+#
+# Environment is reproduced from `uv.lock` (the repo's single source of pinned
+# versions) with `uv sync --frozen`, so CI installs exactly what a developer's
+# `.venv` does — no parallel pin set. Tests that need the astromatic binaries
+# (Source Extractor, PSFEx) self-skip via `shutil.which`, so they simply don't
+# run here; the dev-image CI exercises them.
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+  workflow_dispatch:
+
+# A newer push to the same ref cancels an in-flight run — the fast loop only
+# cares about the latest commit.
+concurrency:
+  group: fast-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  fast-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      # uv is the project's package manager; pin the Python it provisions to the
+      # `requires-python` floor (3.12) so CI matches the container.
+      - name: Set up uv
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+        with:
+          python-version: "3.12"
+          enable-cache: true
+
+      # `--frozen` installs the exact versions in uv.lock (incl. the pinned
+      # ngmix git source) — no resolution, no drift from what ships.
+      - name: Install (test extra, from lockfile)
+        run: uv sync --frozen --extra test
+
+      # The fast tier: everything under the single `tests/` root except `slow`
+      # and `candide`. Coverage gates are dropped here (this job is about speed,
+      # not the coverage report) and the deterministic Hypothesis profile is
+      # pinned, matching the dev-image CI step.
+      - name: Run fast suite
+        env:
+          HYPOTHESIS_PROFILE: ci
+          SHAPEPIPE_ON_CANDIDE: "0"
+        run: >-
+          uv run --frozen
+          pytest -m "not slow and not candide"
+          -o addopts="--strict-markers"
+          -rs

CLAUDE.md

@@ -56,7 +56,10 @@ Full detail: `docs/source/installation.md` and `docs/source/container.md`.
   I/O; `utilities/`; `canfar/` is CANFAR/cluster job orchestration. Console entry
   points (`shapepipe_run`, `summary_run`, `canfar_*`) are defined under
   `[project.scripts]`.
-- `src/shapepipe/tests/` — unit tests; `tests/unit/` holds newer test scaffolding.
+- `tests/` — the whole test suite, one discovery root: `module/` (per-module
+  unit/property/integration tests), `unit/` (structural), `science/` (fast
+  guardrails), `cluster/` (candide-only), `helpers/` (shared library code).
+  See `tests/README.md`.
 - `example/` — a runnable example pipeline (`example/config.ini`) on a single
   CFIS tile; doubles as the CI smoke test.
 - `scripts/` — shell / Python / notebook helpers (`sh/`, `python/`, `jupyter/`),

Dockerfile.jupyter

@@ -1,7 +1,7 @@
 # Dockerfile for ShapePipe Jupyter Lab
 # Uses the shared base image
 
-FROM ghcr.io/cosmostat/shapepipe:v2.0
+FROM ghcr.io/cosmostat/shapepipe:develop
 
 # Default command starts Jupyter Lab
 ENTRYPOINT ["jupyter","lab","--ip=0.0.0.0","--port=8888","--no-browser","--allow-root"]

bin/canfar_monitor.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+from shapepipe.canfar_run import run_log
+run_log()

bin/canfar_monitor_log.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+from shapepipe.canfar_run import run_monitor_log
+run_monitor_log()

bin/canfar_submit_job.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+from shapepipe.canfar_run import run_job
+run_job()

bin/shapepipe_run.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+from shapepipe.shapepipe_run import main
+main()

bin/summary_run.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+from shapepipe.summary_run import main
+main()

conftest.py

@@ -1,10 +1,106 @@
-"""Shared pytest configuration for the ShapePipe test suite."""
+"""Shared pytest configuration for the ShapePipe test suite.
+
+Markers, environment detection, and the candide skip policy live here so
+every test module — wherever it sits in the tree — sees the same rules.
+
+Markers (also declared in ``pyproject.toml`` so ``--strict-markers`` is on):
+
+* ``slow``    — heavy compute (minutes), not part of the fast inner loop.
+* ``candide`` — needs the candide cluster and/or its real on-disk data;
+  meaningless (and auto-skipped) anywhere else.
+
+A ``candide``-marked test only runs on a candide node. Everywhere else it
+is skipped with a clear reason, so the same suite is green on a laptop, in
+CI, and on the cluster — the cluster-only tests simply do not fire off it.
+"""
 
 import os
+import re
+import socket
+
+import pytest
 
 from hypothesis import settings
 
 
 settings.register_profile("ci", derandomize=True, max_examples=50)
 settings.register_profile("dev", max_examples=200)
 settings.load_profile(os.environ.get("HYPOTHESIS_PROFILE", "ci"))
+
+
+# --------------------------------------------------------------------------- #
+# Candide detection
+# --------------------------------------------------------------------------- #
+
+# Candide compute / login nodes are named c01, c03, n22..n36, etc. The login
+# host this suite is most often driven from is ``c03``. We match the candide
+# node-name families rather than a fixed list so new nodes are covered, and
+# allow an explicit override for CI or odd hostnames.
+_CANDIDE_HOST_RE = re.compile(r"^(c\d|n\d{2})", re.IGNORECASE)
+
+
+def on_candide():
+    """Return True when running on a candide node.
+
+    The check is, in order: an explicit ``SHAPEPIPE_ON_CANDIDE`` override
+    (``1``/``0``), then the hostname against the candide node-name families
+    (``c0x`` login, ``nXX`` compute). Cheap, import-safe, no cluster calls.
+    """
+    override = os.environ.get("SHAPEPIPE_ON_CANDIDE")
+    if override is not None:
+        return override == "1"
+    return bool(_CANDIDE_HOST_RE.match(socket.gethostname()))
+
+
+# --------------------------------------------------------------------------- #
+# Marker registration + skip policy
+# --------------------------------------------------------------------------- #
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers",
+        "slow: heavy compute (minutes); excluded from the fast inner loop.",
+    )
+    config.addinivalue_line(
+        "markers",
+        "candide: needs the candide cluster and/or its real data; "
+        "auto-skipped elsewhere.",
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    """Skip ``candide`` tests off-cluster.
+
+    Collection still happens everywhere — so ``pytest --collect-only`` shows
+    the cluster tests exist — they are just marked skipped at run time when
+    not on candide.
+    """
+    if on_candide():
+        return
+    skip_candide = pytest.mark.skip(
+        reason="needs candide (set SHAPEPIPE_ON_CANDIDE=1 to force)"
+    )
+    for item in items:
+        if "candide" in item.keywords:
+            item.add_marker(skip_candide)
+
+
+# --------------------------------------------------------------------------- #
+# Shared fixtures
+# --------------------------------------------------------------------------- #
+
+
+@pytest.fixture(scope="session")
+def artifacts_dir():
+    """Directory where guardrail tests drop plots + status summaries.
+
+    The artifacts SEAM: a later GitHub Pages / status step publishes from
+    here. Created on demand so a clean checkout has nothing to commit until
+    a test actually emits.
+    """
+    from pathlib import Path
+
+    path = Path(__file__).parent / "tests" / "_artifacts"
+    path.mkdir(parents=True, exist_ok=True)
+    return path

docs/source/configuration.md

@@ -84,7 +84,10 @@ The following options can be added to the `[FILE]` section of the config file
   (*e.g.* `.`, `-`, `:`, *etc.*). *optional*ly a regular expression can also be
   passed if it is preceded by `RE:` (*e.g.* `RE:-\d{9}`).
 - `NUMBER_LIST` : (`str` or `list`, *optional*) A list of number strings
-  matching the numbering scheme or a file name.
+  matching the numbering scheme or a file name. Restricts the run to these
+  numbers; every entry must match an input file found on disk, otherwise the
+  run fails at start-up. This is also how a single image is processed per
+  job (formerly the `-e`/`--exclusive` command-line flag).
 - `CORRECT_FILE_PATTERN` : (`bool`, *optional*) Option to allow substring file
   patterns. Default value is `True`.
 

docs/source/testing.md

@@ -6,22 +6,26 @@ the bundled example pipeline.
 
 ## The automated test suite
 
-The test suite runs with [pytest](https://docs.pytest.org/) and lives in two
-trees:
+The test suite runs with [pytest](https://docs.pytest.org/) under a single
+discovery root, `tests/`, with one tier per subdirectory:
 
-- `tests/unit/` — **structural** tests, one parametrized case per file: every
-  shell script parses (`bash -n`), every example config parses, every
-  `shapepipe.*` submodule imports, every `[project.scripts]` entry handles
-  `-h`, and every `*_runner.py` carries its `@module_runner` metadata. Cheap,
-  broad, and good at catching the dumb-but-real regressions (a syntax error, a
-  broken import, a renamed entry point).
-- `src/shapepipe/tests/` — **unit, property-based, and integration** tests for
+- `tests/module/` — **unit, property-based, and integration** tests for
   the analytic surface: pure helpers and geometry (coordinate round-trips,
   postage-stamp shapes, the MegaCam CCD flip), tested both by example and with
   [Hypothesis](https://hypothesis.readthedocs.io/) property tests where a
   function has a clear invariant. Integration tests that need the astromatic
   binaries (Source Extractor, PSFEx) build synthetic FITS on the fly rather
   than requiring survey data.
+- `tests/unit/` — **structural** tests, one parametrized case per file: every
+  shell script parses (`bash -n`), every example config parses, every
+  `shapepipe.*` submodule imports, every `[project.scripts]` entry handles
+  `-h`, and every `*_runner.py` carries its `@module_runner` metadata. Cheap,
+  broad, and good at catching the dumb-but-real regressions (a syntax error, a
+  broken import, a renamed entry point).
+- `tests/science/` — **fast scientific guardrails** (controlled simulations
+  with a known answer) and `tests/cluster/` — **candide guardrails** that read
+  real on-disk catalogs; the latter are marked `candide` and auto-skip off the
+  cluster. See `tests/README.md` for the full layout and markers.
 
 **CI is the single gate.** On every pull request and every push to `develop`,
 `.github/workflows/deploy-image.yml` builds the dev image and runs the suite

example/cfis/config_exp_mccd.ini

@@ -116,7 +116,7 @@ BKG_FROM_HEADER = False
 # Type of image check (optional), default not used, can be a list of
 # BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, 
 # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES
-CHECKIMAGE = BACKGROUND
+CHECKIMAGE = BACKGROUND, BACKGROUND_RMS
 
 # File name suffix for the output sextractor files (optional) SUFFIX = tile
 SUFFIX = sexcat
@@ -235,6 +235,3 @@ PSF_MODEL_PATTERN = fitted_model
 
 # PSF model separator
 PSF_MODEL_SEPARATOR = -
-
-# For multi-epoch purposes
-ME_LOG_WCS = $SP_RUN/output/run_sp_exp_Mh/merge_headers_runner/output/log_exp_headers.sqlite

example/cfis/config_exp_psfex.ini

@@ -64,6 +64,10 @@ INPUT_DIR = last:split_exp_runner, last:mask_runner
 # Read pipeline flag files created by mask module
 FILE_PATTERN = image, weight, pipeline_flag
 
+# Explicit extensions: a 3-entry FILE_PATTERN override must not fall back on
+# the decorator's 4-entry FILE_EXT default (length check fails at startup)
+FILE_EXT = .fits, .fits, .fits
+
 NUMBERING_SCHEME = -0000000-0
 
 # SExtractor executable path
@@ -112,7 +116,7 @@ BKG_FROM_HEADER = False
 # Type of image check (optional), default not used, can be a list of
 # BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, 
 # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES
-CHECKIMAGE = BACKGROUND
+CHECKIMAGE = BACKGROUND, BACKGROUND_RMS
 
 # File name suffix for the output sextractor files (optional) SUFFIX = tile
 SUFFIX = sexcat

example/cfis/config_onthefly.mask

@@ -3,7 +3,7 @@
 ## Paths to executables
 [PROGRAM_PATH]
 
-WW_PATH = ww
+WW_PATH = weightwatcher
 WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww
 
 # Indicate cds client executable if no external star catalogue is available

example/cfis/config_save.mask

@@ -3,7 +3,7 @@
 ## Paths to executables
 [PROGRAM_PATH]
 
-WW_PATH = ww
+WW_PATH = weightwatcher
 WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww
 
 # Indicate cds client executable if no external star catalogue is available