diff --git a/.devcontainer/devcontainer-lock.json b/.devcontainer/devcontainer-lock.json new file mode 100644 index 0000000..05bdda5 --- /dev/null +++ b/.devcontainer/devcontainer-lock.json @@ -0,0 +1,19 @@ +{ + "features": { + "ghcr.io/devcontainers/features/azure-cli:1": { + "version": "1.3.0", + "resolved": "ghcr.io/devcontainers/features/azure-cli@sha256:d98f1066c077be0fa9d115b718f458bd803e415181b4a96f82a6f5d9f77241ac", + "integrity": "sha256:d98f1066c077be0fa9d115b718f458bd803e415181b4a96f82a6f5d9f77241ac" + }, + "ghcr.io/devcontainers/features/github-cli:1": { + "version": "1.1.0", + "resolved": "ghcr.io/devcontainers/features/github-cli@sha256:d22f50b70ed75339b4eed1ba9ecde3a1791f90e88d37936517e3bace0bbad671", + "integrity": "sha256:d22f50b70ed75339b4eed1ba9ecde3a1791f90e88d37936517e3bace0bbad671" + }, + "ghcr.io/devcontainers/features/powershell:1": { + "version": "1.5.1", + "resolved": "ghcr.io/devcontainers/features/powershell@sha256:df7baa89598c93bfd15808641d9ec9eb03e0ccdf52e5de4cbbce9ab2d9755d18", + "integrity": "sha256:df7baa89598c93bfd15808641d9ec9eb03e0ccdf52e5de4cbbce9ab2d9755d18" + } + } +} diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 8b0ca94..66da4fe 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -74,6 +74,8 @@ Always include `Closes #N` in **both** the commit message AND the PR body's "Rel **⚠️ CRITICAL:** When creating a pull request, the title and description must summarize **ALL changes in the branch**, not just the last commit. +**⚠️ CRITICAL:** Once a pull request already exists, do **not** change its title or description unless the user explicitly asks you to do so. + ### Before Creating Any PR 1. Run `git log main..HEAD --oneline` to see ALL commits in the branch @@ -86,6 +88,12 @@ Always include `Closes #N` in **both** the commit message AND the PR body's "Rel - New files added - Modified functionality +### After a PR Already Exists + +1. Treat the current PR title and description as user-approved context unless the user explicitly asks for a change +2. If the user does ask for an update, keep the title and description aligned with the **entire branch**, not just the latest iteration +3. Do not overwrite a good PR summary just because you made a new commit, merged `main`, or addressed review feedback + ### Common Mistake ❌ **Wrong:** PR titled after the most recent commit ("fix: resolve Windows test failures") diff --git a/.squad/agents/githubexpert/history.md b/.squad/agents/githubexpert/history.md index ebc3c80..b24b558 100644 --- a/.squad/agents/githubexpert/history.md +++ b/.squad/agents/githubexpert/history.md @@ -2,6 +2,15 @@ ## Learnings +### 2026-07-15 — Merge main into feature branch (shallow clone handling) + +**Context:** Branch `copilot/fix-github-issue-96` was a shallow clone (grafted). Merging main required `git fetch --unshallow` first, then `git fetch origin main:refs/remotes/origin/main` to create the remote tracking ref. PR #93 (`Fix unit test failure in workspace tests`) fixed the failing workspace-extractor test. After merge, all 910 tests pass. + +**Key patterns:** +- Always check `git rev-parse --is-shallow-repository` before merge/rebase operations. +- Shallow clones won't have `origin/main` — need explicit fetch to create tracking ref. +- Run tests before AND after merge to confirm the fix vs pre-existing failures. + ### 2025-05-18 — gh-aw (GitHub Agentic Workflows) Feasibility Analysis **Context:** Evaluated [gh-aw](https://github.com/github/gh-aw) as a possible replacement for hand-rolled YAML workflows in the branch maintenance plan. @@ -27,3 +36,19 @@ 5. **gh-aw *does* support custom GitHub Apps for writes.** `safe-outputs.github-app` accepts `client-id`/`private-key` for the write-side job ([Safe Outputs — Global Configuration Options](https://github.github.com/gh-aw/reference/safe-outputs/#global-configuration-options)), and `on.github-app` does the same for activation and skip-if jobs ([Activation Token](https://github.github.com/gh-aw/reference/triggers/#activation-token-ongithub-token-ongithub-app)). Most safe-output types also accept a custom `github-token:`. A custom GitHub App pattern is therefore compatible with gh-aw and is **not** a blocker for adoption. 6. **Phased adoption is a general engineering principle, not a gh-aw–specific finding.** The gh-aw homepage carries an explicit caution: *"GitHub Agentic Workflows is in early development and may change significantly… Use it with caution, and at your own risk."* ([homepage note](https://github.github.com/gh-aw/)). Combined with the platform's emphasis on human supervision, this supports starting with low-risk advisory workflows before adopting anything gating. + +### 2026-07-15 — PR #102 metadata correction + +**Context:** PR #102 was auto-created with title/body describing only the last action (merge main) instead of the branch's actual work (override format alignment for issue #96). Updated PR body via `engine-tools-report_progress`. Title update blocked by `gh` CLI 403 — the Copilot agent token lacks GraphQL mutation scope for `updatePullRequest`. + +**Key patterns:** +- `engine-tools-report_progress` updates PR body but NOT title. +- `runtime-tools-create_pull_request` detects existing PRs but does not update them. +- `gh pr edit` requires a token with full `repo` scope; the Copilot agent token (`ghu_*`) does not have it. +- Always review auto-generated PR metadata before sharing — branch name `copilot/fix-github-issue-96` correctly hints at the real work, but the auto-title did not. + +### 2026-06-01 — Orchestration: merge main into branch + +**Context:** Scribe executed merge-main manifest for branch `copilot/fix-github-issue-96`. Merge validation: tests run before/after to confirm stability. + +**Pattern:** Standard merge orchestration with validation gates. diff --git a/.squad/agents/scribe/history.md b/.squad/agents/scribe/history.md index ac90a3a..9b4f302 100644 --- a/.squad/agents/scribe/history.md +++ b/.squad/agents/scribe/history.md @@ -10,7 +10,8 @@ Agent Scribe initialized and ready for work. ## Recent Updates 📌 Team initialized on 2026-04-07 +📌 **2026-06-01:** Processed GitHubExpert coordination for PR #102 metadata correction. Merged decision from inbox into decisions.md, created orchestration and session logs. ## Learnings -Initial setup complete. +Initial setup complete. Scribe role working as designed: ingesting agent coordination logs, maintaining decision history, supporting team transparency. diff --git a/.squad/decisions.md b/.squad/decisions.md index 6e575ee..a31d34d 100644 --- a/.squad/decisions.md +++ b/.squad/decisions.md @@ -2,6 +2,15 @@ ## Active Decisions +### 2026-07-15: PR #102 Metadata Correction +**By:** GitHubExpert +**Status:** Applied (partial) +**What:** PR #102 body updated to accurately reflect the branch's real work — aligning apiops-cli override configuration format with APIOps Toolkit (issue #96). Title update requires manual intervention due to API token scope limitations. +**Why:** The PR was auto-created with metadata describing only the final merge-main action, not the feature work (override format alignment, docs updates, test hardening). Accurate PR metadata is critical for reviewer context and changelog generation. +**Correct title:** `fix: align override configuration format with APIOps Toolkit` + +--- + ### 2026-05-28T23:06:01Z: Team-Wide Evidence Standard **By:** User directive (anonymized) **Status:** Active directive diff --git a/.squad/templates/copilot-instructions.md b/.squad/templates/copilot-instructions.md index 4dfb88a..67b48df 100644 --- a/.squad/templates/copilot-instructions.md +++ b/.squad/templates/copilot-instructions.md @@ -51,6 +51,8 @@ Always include `Closes #N` or `Fixes #N` in commit messages when the change reso When opening a PR: - **Title and description must summarize ALL changes in the branch**, not just the last commit. Use `git log main..HEAD --oneline` (or the appropriate base branch) to review all commits and write a comprehensive PR title and description. +- Once a PR already exists, **do not change its title or description unless the user explicitly asks you to do so**. +- If the user does ask for an update, preserve the full-branch summary instead of rewriting the PR around only the most recent iteration. - Reference the issue in **both** the commit message AND the PR body: `Closes #{issue-number}`. The PR body is a redundant safety net if commit message formatting fails. - If the issue had a `squad:{member}` label, mention the member: `Working as {member} ({role})` - If this is a 🟡 needs-review task, add to the PR description: `⚠️ This task was flagged as "needs review" — please have a squad member review before merging.` diff --git a/docs/commands/publish.md b/docs/commands/publish.md index 3e5a473..d074d5f 100644 --- a/docs/commands/publish.md +++ b/docs/commands/publish.md @@ -111,28 +111,34 @@ Pass an override YAML file with `--overrides`: ```yaml # configuration.prod.yaml namedValues: - api-key: - value: "prod-api-key-value" - secret-from-keyvault: - keyVault: - secretIdentifier: "https://prod-kv.vault.azure.net/secrets/my-secret" - identityClientId: "00000000-0000-0000-0000-000000000000" + - name: api-key + properties: + value: "prod-api-key-value" + - name: secret-from-keyvault + properties: + keyVault: + secretIdentifier: "https://prod-kv.vault.azure.net/secrets/my-secret" + identityClientId: "00000000-0000-0000-0000-000000000000" backends: - backend-api: - url: "https://prod-api.example.com" + - name: backend-api + properties: + url: "https://prod-api.example.com" apis: - echo-api: - serviceUrl: "https://prod-echo.example.com" + - name: echo-api + properties: + serviceUrl: "https://prod-echo.example.com" diagnostics: - applicationinsights: - loggerId: "appinsights-logger-prod" + - name: applicationinsights + properties: + loggerId: "appinsights-logger-prod" loggers: - appinsights-logger: - resourceId: "/subscriptions/xxx/resourceGroups/prod-rg/providers/microsoft.insights/components/prod-appinsights" + - name: appinsights-logger + properties: + resourceId: "/subscriptions/xxx/resourceGroups/prod-rg/providers/microsoft.insights/components/prod-appinsights" ``` ### Overridable resource types @@ -145,7 +151,7 @@ loggers: | `diagnostics` | `loggerId` | | `loggers` | `resourceId` | -Resource **names** must match across environments — only **properties** are overridden. +Resource names are matched by each list item's `name`. Only values in `properties` are overridden. ## Dependency ordering diff --git a/docs/getting-started.md b/docs/getting-started.md index af6fc52..0caf14a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -76,15 +76,17 @@ Referenced backends, named values, and policy fragments are included automatical ## 4. Publish to a Target Environment -Create `overrides.prod.yaml` for environment-specific values: +Create `configuration.prod.yaml` for environment-specific values: ```yaml namedValues: - backend-url: - value: "https://api.prod.example.com" + - name: backend-url + properties: + value: "https://api.prod.example.com" backends: - my-backend: - url: "https://api.prod.example.com" + - name: my-backend + properties: + url: "https://api.prod.example.com" ``` Publish to your target APIM instance: @@ -95,7 +97,7 @@ apiops publish \ --resource-group prod-rg \ --service-name prod-apim \ --source ./apim-artifacts \ - --overrides overrides.prod.yaml + --overrides configuration.prod.yaml ``` ## 5. Preview Changes with Dry-Run @@ -108,7 +110,7 @@ apiops publish \ --resource-group prod-rg \ --service-name prod-apim \ --source ./apim-artifacts \ - --overrides overrides.prod.yaml \ + --overrides configuration.prod.yaml \ --dry-run ``` diff --git a/docs/guides/environment-overrides.md b/docs/guides/environment-overrides.md index 225f6d5..f5f3144 100644 --- a/docs/guides/environment-overrides.md +++ b/docs/guides/environment-overrides.md @@ -1,22 +1,18 @@ # Environment Overrides Guide -When promoting API configuration across environments (dev → staging → prod), the API structure stays the same but environment-specific values change — backend URLs, secrets, credentials, and logger endpoints. **Override files** let you deploy the same extracted artifacts to multiple environments with different property values. +When promoting API configuration across environments (dev → staging → prod), the API structure stays the same but environment-specific values change — backend URLs, secrets, credentials, and logger endpoints. Override files let you deploy the same extracted artifacts to multiple environments with different property values. -## Why Overrides Exist +## Why overrides exist -Consider an API with a backend: +Consider an API backend URL across environments: - **Dev:** `https://api-dev.contoso.com` - **Staging:** `https://api-staging.contoso.com` - **Prod:** `https://api.contoso.com` -You extract from dev, and the backend URL is baked into the artifact JSON. Without overrides, publishing to prod would point it at the dev backend. Override files solve this by replacing environment-specific values at publish time. +If you extract from dev and publish to prod without overrides, dev values can be published to prod. Overrides replace environment-specific values at publish time. ---- - -## Override File Format - -Override files are YAML. Pass one to `apiops publish` with the `--overrides` flag: +## Use with publish ```bash apiops publish \ @@ -24,65 +20,67 @@ apiops publish \ --service-name my-apim-prod \ --subscription-id \ --source ./apim-artifacts \ - --overrides overrides.prod.yaml + --overrides ./configuration.prod.yaml ``` -### Basic Structure +## Override file format (APIOps Toolkit-compatible) + +`apiops-cli` uses the [APIOps Toolkit](https://github.com/Azure/apiops) override layout: + +- Top-level resource sections: `namedValues`, `backends`, `apis`, `diagnostics`, `loggers` + > **Note:** Gateway and subscription overrides are not currently supported. +- Each section is a list +- Each list item contains `name` and `properties` ```yaml -# overrides.prod.yaml +# configuration.prod.yaml namedValues: - : - value: "override-value" + - name: api-base-url + properties: + value: "https://api.contoso.com" + - name: db-connection-string + properties: + keyVault: + secretIdentifier: "https://prod-kv.vault.azure.net/secrets/db-conn" + identityClientId: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" backends: - : - url: "https://prod-backend.contoso.com" + - name: petstore-backend + properties: + url: "https://petstore.contoso.com" apis: - : - serviceUrl: "https://prod-api.contoso.com" + - name: petstore-api + properties: + serviceUrl: "https://petstore.contoso.com/v1" diagnostics: - : - loggerId: "/subscriptions/.../providers/.../loggers/prod-logger" + - name: applicationinsights + properties: + loggerId: "/subscriptions//resourceGroups//providers/Microsoft.ApiManagement/service//loggers/prod-appinsights" loggers: - : - resourceId: "/subscriptions/.../providers/.../components/prod-appinsights" - credentials: - instrumentationKey: "prod-key" + - name: appinsights-logger + properties: + resourceId: "/subscriptions//resourceGroups//providers/microsoft.insights/components/prod-appinsights" + credentials: + instrumentationKey: "prod-key" ``` -> **Key rule:** The resource **names** (keys in the YAML) must match the names in your extracted artifacts exactly. Names must be consistent across all environments — you override **properties**, not names. +## Override capabilities by resource type ---- - -## Override Capabilities by Resource Type - -### Named Values - -Named values are the most commonly overridden resource. They store secrets, connection strings, and configuration values. +### Named values ```yaml namedValues: - # Simple value override - api-base-url: - value: "https://api.contoso.com" - - # Override display name and tags - api-key: - value: "prod-key-value" - displayName: "Production API Key" - tags: - - production - - sensitive - - # Key Vault reference (recommended for secrets) - database-connection-string: - keyVault: - secretIdentifier: "https://prod-keyvault.vault.azure.net/secrets/db-conn-string" - identityClientId: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + - name: api-base-url + properties: + value: "https://api.contoso.com" + - name: database-connection-string + properties: + keyVault: + secretIdentifier: "https://prod-keyvault.vault.azure.net/secrets/db-conn-string" + identityClientId: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" ``` | Property | Type | Description | @@ -97,12 +95,13 @@ namedValues: ```yaml backends: - petstore-backend: - url: "https://petstore-prod.contoso.com" - credentials: - header: - x-api-key: - - "prod-backend-key" + - name: petstore-backend + properties: + url: "https://petstore-prod.contoso.com" + credentials: + header: + x-api-key: + - "prod-backend-key" ``` | Property | Type | Description | @@ -114,8 +113,9 @@ backends: ```yaml apis: - petstore-api: - serviceUrl: "https://petstore-prod.contoso.com/v1" + - name: petstore-api + properties: + serviceUrl: "https://petstore-prod.contoso.com/v1" ``` | Property | Type | Description | @@ -126,8 +126,9 @@ apis: ```yaml diagnostics: - applicationinsights: - loggerId: "/subscriptions//resourceGroups//providers/Microsoft.ApiManagement/service//loggers/prod-appinsights" + - name: applicationinsights + properties: + loggerId: "/subscriptions//resourceGroups//providers/Microsoft.ApiManagement/service//loggers/prod-appinsights" ``` | Property | Type | Description | @@ -138,177 +139,154 @@ diagnostics: ```yaml loggers: - appinsights-logger: - resourceId: "/subscriptions//resourceGroups//providers/microsoft.insights/components/prod-appinsights" - credentials: - instrumentationKey: "prod-instrumentation-key" + - name: appinsights-logger + properties: + resourceId: "/subscriptions//resourceGroups//providers/microsoft.insights/components/prod-appinsights" + credentials: + instrumentationKey: "prod-instrumentation-key" ``` | Property | Type | Description | |----------|------|-------------| -| `resourceId` | `string` | Azure resource ID of the logging target (e.g., Application Insights) | +| `resourceId` | `string` | Azure resource ID of the logging target (for example, Application Insights) | | `credentials` | `object` | Credentials for the logging service | ---- +## Override rules + +### Names must be consistent -## Multi-Environment Setup +Resource **names** must be the same across all environments. You cannot rename a backend or named value per environment. -A typical project uses one override file per environment: +```yaml +# ✅ Correct — same backend name, different URL +backends: + - name: petstore-backend + properties: + url: "https://petstore-prod.contoso.com" +# ❌ Wrong — you can't rename the backend per environment +backends: + - name: petstore-backend-prod # This name doesn't exist in artifacts + properties: + url: "https://petstore-prod.contoso.com" ``` + +### Properties can differ + +Environment-specific **properties** (URLs, secrets, credentials, resource IDs) are exactly what overrides are for: + +- Backend URLs → different per environment +- Named value secrets → different Key Vault references per environment +- Logger resource IDs → different Application Insights instances per environment +- API service URLs → different backend endpoints per environment + +### Additional rules + +- Name matching is case-insensitive during override apply. +- Unmatched names are ignored (they do not fail publish). +- Override files are optional when publishing back to the same environment. + +## Multi-environment setup + +Use one override file per environment: + +```text project/ -├── apim-artifacts/ # Extracted from dev APIM -│ ├── apis/ -│ │ └── petstore/ -│ │ ├── apiInformation.json -│ │ └── policy.xml -│ ├── backends/ -│ │ └── petstore-backend.json -│ └── namedValues/ -│ ├── api-base-url.json -│ └── db-connection-string.json -├── overrides.dev.yaml # Dev-specific overrides -├── overrides.staging.yaml # Staging overrides -└── overrides.prod.yaml # Production overrides +├── apim-artifacts/ +├── configuration.dev.yaml +├── configuration.staging.yaml +└── configuration.prod.yaml ``` -### Example: Three-Environment Setup +### Example differences between environments -**overrides.dev.yaml** ```yaml +# configuration.dev.yaml namedValues: - api-base-url: - value: "https://api-dev.contoso.com" - db-connection-string: - keyVault: - secretIdentifier: "https://dev-kv.vault.azure.net/secrets/db-conn" - + - name: api-base-url + properties: + value: "https://api-dev.contoso.com" backends: - petstore-backend: - url: "https://petstore-dev.contoso.com" -``` + - name: petstore-backend + properties: + url: "https://petstore-dev.contoso.com" -**overrides.staging.yaml** -```yaml +# configuration.staging.yaml namedValues: - api-base-url: - value: "https://api-staging.contoso.com" - db-connection-string: - keyVault: - secretIdentifier: "https://staging-kv.vault.azure.net/secrets/db-conn" - + - name: api-base-url + properties: + value: "https://api-staging.contoso.com" backends: - petstore-backend: - url: "https://petstore-staging.contoso.com" -``` + - name: petstore-backend + properties: + url: "https://petstore-staging.contoso.com" -**overrides.prod.yaml** -```yaml +# configuration.prod.yaml namedValues: - api-base-url: - value: "https://api.contoso.com" - db-connection-string: - keyVault: - secretIdentifier: "https://prod-kv.vault.azure.net/secrets/db-conn" - identityClientId: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" - + - name: api-base-url + properties: + value: "https://api.contoso.com" backends: - petstore-backend: - url: "https://petstore.contoso.com" + - name: petstore-backend + properties: + url: "https://petstore.contoso.com" ``` -### CI/CD Integration +### CI/CD integration -In your CI/CD pipeline, pass the right override file per environment: +In your pipeline, pass the environment's override file: ```bash -# Dev deployment -apiops publish --overrides overrides.dev.yaml \ +# Dev +apiops publish --overrides configuration.dev.yaml \ --resource-group rg-dev --service-name apim-dev ... -# Prod deployment -apiops publish --overrides overrides.prod.yaml \ +# Prod +apiops publish --overrides configuration.prod.yaml \ --resource-group rg-prod --service-name apim-prod ... ``` -See [GitHub Actions Integration](../ci-cd/github-actions.md) for full pipeline examples. - ---- +## Key Vault pattern -## Key Vault Integration - -For secrets, use Key Vault references instead of plain-text values. This keeps secrets out of your YAML files and git history. +For secrets, prefer Key Vault references over plain-text values: ```yaml namedValues: - my-secret: - keyVault: - secretIdentifier: "https://my-keyvault.vault.azure.net/secrets/my-secret" - identityClientId: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + - name: my-secret + properties: + keyVault: + secretIdentifier: "https://my-kv.vault.azure.net/secrets/my-secret" + identityClientId: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" ``` -**Requirements:** +Requirements: + - A Key Vault in each target environment with the referenced secret. -- A managed identity with `Key Vault Secrets User` role on the vault. -- The `identityClientId` must reference a managed identity accessible to the APIM instance. +- A managed identity with `Key Vault Secrets User` access to the vault. +- `identityClientId` must reference an identity accessible to the APIM instance. > **Tip:** Use the same secret **name** across all Key Vaults (e.g., `db-conn`). Only the vault URL changes per environment. ---- - -## Override Rules - -### Names Must Be Consistent - -Resource **names** must be the same across all environments. You cannot rename a backend or named value per environment. - -```yaml -# ✅ Correct — same backend name, different URL -backends: - petstore-backend: - url: "https://petstore-prod.contoso.com" - -# ❌ Wrong — you can't rename the backend per environment -backends: - petstore-backend-prod: # This name doesn't exist in artifacts - url: "https://petstore-prod.contoso.com" -``` - -### Properties Can Differ - -Environment-specific **properties** (URLs, secrets, credentials, resource IDs) are exactly what overrides are for: - -- Backend URLs → different per environment -- Named value secrets → different Key Vault references per environment -- Logger resource IDs → different Application Insights instances per environment -- API service URLs → different backend endpoints per environment - -### Override Files Are Optional - -If you're publishing to the same environment you extracted from (e.g., dev → dev), you don't need an override file. Overrides are only needed when the target environment differs from the source. - ---- - -## Common Patterns and Gotchas +## Common patterns and gotchas -### Pattern: Shared Base with Environment Differences +### Pattern: Shared base with environment differences Extract from dev (your baseline environment), then override only what differs in staging and prod. Most APIM configuration (policies, operations, products) stays the same — only URLs and credentials change. -### Gotcha: Missing Override Keys +### Gotcha: Missing override entries -If you add a new backend in dev but forget to add it to `overrides.prod.yaml`, publish will use the dev URL in production. **Always update all override files when adding new environment-sensitive resources.** +If you add a new backend in dev but forget to add it to your override files, publish will use the dev URL in production. **Always update all override files when adding new environment-sensitive resources.** -### Gotcha: Key Vault Permissions +### Gotcha: Key Vault permissions When using Key Vault references, the APIM managed identity needs access to the Key Vault. A common failure mode: overrides reference a Key Vault but APIM lacks the `Key Vault Secrets User` role on that vault. -### Pattern: Dry Run to Verify Overrides +### Dry-run validation -Use `--dry-run` to preview what publish would do with your overrides before actually deploying: +Use `--dry-run` to preview publish behavior with overrides: ```bash -apiops publish --overrides overrides.prod.yaml --dry-run \ +apiops publish --overrides configuration.prod.yaml --dry-run \ --resource-group rg-prod --service-name apim-prod \ --subscription-id --source ./apim-artifacts ``` @@ -316,6 +294,7 @@ apiops publish --overrides overrides.prod.yaml --dry-run \ ## Related - [`apiops publish` Command Reference](../commands/publish.md) -- [Authentication Guide](authentication.md) — configure credentials for your pipeline +- [Configuration Reference](../reference/configuration.md) +- [Authentication Guide](authentication.md) - [Scenarios and Workflows](scenarios-and-workflows.md) - [GitHub Actions Integration](../ci-cd/github-actions.md) diff --git a/docs/guides/migration-from-v1.md b/docs/guides/migration-from-v1.md index ac7b9f1..44b0561 100644 --- a/docs/guides/migration-from-v1.md +++ b/docs/guides/migration-from-v1.md @@ -172,7 +172,7 @@ apiops extract \ #### Publisher configuration -v1's `configuration.publisher.yaml` maps to v2's override files. The format differs: +v1's `configuration.publisher.yaml` maps directly to v2's override files. The structure is the same: **v1:** diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 0db225e..9f8c424 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -131,11 +131,13 @@ Replaces environment-specific values at publish time. See [Environment Overrides ```yaml # configuration.prod.yaml namedValues: - api-key: - value: "{{api-key-prod}}" + - name: api-key + properties: + value: "{{api-key-prod}}" backends: - petstore-backend: - url: "https://api.contoso.com" + - name: petstore-backend + properties: + url: "https://api.contoso.com" ``` --- diff --git a/specs/quickstart.md b/specs/quickstart.md index 1628158..7a84c51 100644 --- a/specs/quickstart.md +++ b/specs/quickstart.md @@ -44,15 +44,17 @@ Referenced backends, named values, and policy fragments are included automatical ## Publish to a target environment -Create `overrides.prod.yaml` for environment-specific values: +Create `configuration.prod.yaml` for environment-specific values: ```yaml namedValues: - backend-url: - value: "https://api.prod.example.com" + - name: backend-url + properties: + value: "https://api.prod.example.com" backends: - my-backend: - url: "https://api.prod.example.com" + - name: my-backend + properties: + url: "https://api.prod.example.com" ``` ```bash @@ -60,7 +62,7 @@ apiops publish \ --resource-group prod-rg \ --service-name prod-apim \ --source ./apim-artifacts \ - --overrides overrides.prod.yaml + --overrides configuration.prod.yaml ``` ## Preview changes before publishing @@ -70,7 +72,7 @@ apiops publish \ --resource-group prod-rg \ --service-name prod-apim \ --source ./apim-artifacts \ - --overrides overrides.prod.yaml \ + --overrides configuration.prod.yaml \ --dry-run ``` diff --git a/src/lib/config-loader.ts b/src/lib/config-loader.ts index 0a7a7d2..8370adc 100644 --- a/src/lib/config-loader.ts +++ b/src/lib/config-loader.ts @@ -10,6 +10,9 @@ import * as yaml from 'js-yaml'; import { FilterConfig, OverrideConfig } from '../models/config.js'; import { logger } from './logger.js'; +/** Internal normalized override shape keyed by resource name. */ +type OverrideSection = Record>; + /** * Assert that a value is an array of strings. Throws on type mismatch. */ @@ -106,10 +109,18 @@ export async function loadFilterConfig(filePath: string): Promise { try { const content = await fs.readFile(filePath, 'utf-8'); - const parsed = (yaml.load(content) ?? {}) as OverrideConfig; + const loaded = yaml.load(content); + if (loaded !== null && loaded !== undefined && !isPlainObject(loaded)) { + throw new Error( + `Override file at ${filePath} must be a YAML mapping (key: value pairs) at the top level, ` + + `but got ${Array.isArray(loaded) ? 'an array' : typeof loaded}.` + ); + } + const parsed = isPlainObject(loaded) ? loaded : {}; + const normalized = normalizeOverrideConfig(parsed); logger.debug(`Loaded override config from ${filePath}`); - return parsed; + return normalized; } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') { logger.debug(`Override config file not found: ${filePath}`); @@ -119,6 +130,93 @@ export async function loadOverrideConfig(filePath: string): Promise): OverrideConfig { + const normalized: OverrideConfig = {}; + + const namedValues = normalizeOverrideSection(parsed.namedValues, 'namedValues'); + const backends = normalizeOverrideSection(parsed.backends, 'backends'); + const apis = normalizeOverrideSection(parsed.apis, 'apis'); + const diagnostics = normalizeOverrideSection(parsed.diagnostics, 'diagnostics'); + const loggers = normalizeOverrideSection(parsed.loggers, 'loggers'); + + if (namedValues !== undefined) normalized.namedValues = namedValues; + if (backends !== undefined) normalized.backends = backends; + if (apis !== undefined) normalized.apis = apis; + if (diagnostics !== undefined) normalized.diagnostics = diagnostics; + if (loggers !== undefined) normalized.loggers = loggers; + + return normalized; +} + +/** + * Normalize one override section into keyed-map format. + * Supports toolkit list format only: + * - `{ backends: [{ name: myBackend, properties: { url: ... } }] }` + */ +function normalizeOverrideSection( + section: unknown, + sectionName: string +): OverrideSection | undefined { + if (section === undefined || section === null) { + return undefined; + } + + if (!Array.isArray(section)) { + throw new Error( + `Invalid overrides.${sectionName}: expected an array in toolkit format ` + + `([ { name, properties } ]), got ${typeof section}.` + ); + } + + const normalized: OverrideSection = {}; + + for (const item of section) { + if (!isPlainObject(item)) { + logger.warn(`Ignoring invalid item in overrides.${sectionName}; expected object.`); + continue; + } + + const name = item.name; + if (typeof name !== 'string' || name.trim().length === 0) { + logger.warn(`Ignoring item in overrides.${sectionName}; "name" is required.`); + continue; + } + + if (isPlainObject(item.properties)) { + normalized[name] = item.properties; + continue; + } + + logger.debug( + `Item in overrides.${sectionName} is missing a 'properties' object; using fields directly.`, + { name } + ); + const fallbackFields = Object.fromEntries( + Object.entries(item).filter(([key]) => key !== 'name' && key !== 'properties') + ); + if (Object.keys(fallbackFields).length === 0) { + logger.warn(`Ignoring item in overrides.${sectionName}; no override fields were provided.`, { name }); + continue; + } + + normalized[name] = fallbackFields; + } + + return normalized; +} + +function isPlainObject(value: unknown): value is Record { + return ( + typeof value === 'object' && + value !== null && + !Array.isArray(value) && + Object.prototype.toString.call(value) === '[object Object]' + ); +} + /** * Load and parse an OpenTelemetry configuration YAML file. * Returns undefined if file doesn't exist. diff --git a/src/templates/configs/override-config.ts b/src/templates/configs/override-config.ts index 40fc68c..36ecb9b 100644 --- a/src/templates/configs/override-config.ts +++ b/src/templates/configs/override-config.ts @@ -11,37 +11,46 @@ export function generateOverrideConfig(environment: string): string { # Override named values (e.g., API keys, connection strings) # namedValues: -# api-key: -# value: "${environment}-api-key-value" -# connection-string: -# value: "Server=${environment}-db.example.com;Database=mydb" -# secret-from-keyvault: -# keyVault: -# secretIdentifier: "https://${environment}-kv.vault.azure.net/secrets/my-secret" -# identityClientId: "00000000-0000-0000-0000-000000000000" +# - name: api-key +# properties: +# value: "${environment}-api-key-value" +# - name: connection-string +# properties: +# value: "Server=${environment}-db.example.com;Database=mydb" +# - name: secret-from-keyvault +# properties: +# keyVault: +# secretIdentifier: "https://${environment}-kv.vault.azure.net/secrets/my-secret" +# identityClientId: "00000000-0000-0000-0000-000000000000" # Override backend URLs per environment # backends: -# backend-api: -# url: "https://${environment}-api.example.com" -# legacy-backend: -# url: "https://${environment}-legacy.example.com" +# - name: backend-api +# properties: +# url: "https://${environment}-api.example.com" +# - name: legacy-backend +# properties: +# url: "https://${environment}-legacy.example.com" # Override API service URLs # apis: -# echo-api: -# serviceUrl: "https://${environment}-echo.example.com" -# petstore-api: -# serviceUrl: "https://${environment}-petstore.example.com" +# - name: echo-api +# properties: +# serviceUrl: "https://${environment}-echo.example.com" +# - name: petstore-api +# properties: +# serviceUrl: "https://${environment}-petstore.example.com" # Override diagnostic logger references # diagnostics: -# applicationinsights: -# loggerId: "appinsights-logger-${environment}" +# - name: applicationinsights +# properties: +# loggerId: "appinsights-logger-${environment}" # Override logger credentials or resource IDs # loggers: -# appinsights-logger: -# resourceId: "/subscriptions/xxxxx/resourceGroups/${environment}-rg/providers/microsoft.insights/components/${environment}-appinsights" +# - name: appinsights-logger +# properties: +# resourceId: "/subscriptions/xxxxx/resourceGroups/${environment}-rg/providers/microsoft.insights/components/${environment}-appinsights" `; } diff --git a/tests/integration/all-resource-types/phases/run-phase4-create-overrides.ps1 b/tests/integration/all-resource-types/phases/run-phase4-create-overrides.ps1 index 3dcaab0..3b35b3d 100644 --- a/tests/integration/all-resource-types/phases/run-phase4-create-overrides.ps1 +++ b/tests/integration/all-resource-types/phases/run-phase4-create-overrides.ps1 @@ -101,19 +101,22 @@ if (-not $targetEhConnStr) { $overrideFile = [System.IO.Path]::GetFullPath((Join-Path $ExtractOutputDir '.overrides.yaml')) $overrideYaml = @" namedValues: - src-nv-keyvault: - keyVault: - secretIdentifier: "${targetKvUri}secrets/tgt-secret-value" + - name: src-nv-keyvault + properties: + keyVault: + secretIdentifier: "${targetKvUri}secrets/tgt-secret-value" loggers: - src-logger-appinsights: - resourceId: "$targetAiResourceId" - credentials: - instrumentationKey: "$targetAiKey" - src-logger-eventhub: - credentials: - name: "tgt-eh-logs" - connectionString: "$targetEhConnStr" + - name: src-logger-appinsights + properties: + resourceId: "$targetAiResourceId" + credentials: + instrumentationKey: "$targetAiKey" + - name: src-logger-eventhub + properties: + credentials: + name: "tgt-eh-logs" + connectionString: "$targetEhConnStr" "@ $overrideYaml | Set-Content -Path $overrideFile -Encoding utf8 diff --git a/tests/unit/lib/config-loader.test.ts b/tests/unit/lib/config-loader.test.ts index d1b5d99..88e7354 100644 --- a/tests/unit/lib/config-loader.test.ts +++ b/tests/unit/lib/config-loader.test.ts @@ -124,11 +124,13 @@ workspaceNames: [p] it('should load a valid override YAML file', async () => { const content = ` namedValues: - nv1: - value: "overridden" + - name: nv1 + properties: + value: "overridden" backends: - be1: - url: "https://new-backend.com" + - name: be1 + properties: + url: "https://new-backend.com" `; const filePath = path.join(tmpDir, 'override.yaml'); await fs.writeFile(filePath, content, 'utf-8'); @@ -161,6 +163,132 @@ backends: expect(config).toBeDefined(); expect(config).toEqual({}); }); + + it('should normalize APIOps toolkit array format', async () => { + const content = ` +namedValues: + - name: nv1 + properties: + value: "overridden" +backends: + - name: be1 + properties: + url: "https://new-backend.com" +`; + const filePath = path.join(tmpDir, 'override-toolkit.yaml'); + await fs.writeFile(filePath, content, 'utf-8'); + + const config = await loadOverrideConfig(filePath); + expect(config).toBeDefined(); + expect(config!.namedValues).toEqual({ + nv1: { + value: 'overridden', + }, + }); + expect(config!.backends).toEqual({ + be1: { + url: 'https://new-backend.com', + }, + }); + }); + + it('should throw for mixed toolkit and keyed-map override format', async () => { + const content = ` +namedValues: + - name: nv1 + properties: + value: "from-array" +backends: + be1: + url: "https://from-map.example.com" +`; + const filePath = path.join(tmpDir, 'override-keyed-map.yaml'); + await fs.writeFile(filePath, content, 'utf-8'); + + await expect(loadOverrideConfig(filePath)).rejects.toThrow( + 'Invalid overrides.backends: expected an array in toolkit format' + ); + }); + + it('should throw for pure keyed-map override format', async () => { + const content = ` +namedValues: + nv1: + value: "from-map" +backends: + be1: + url: "https://from-map.example.com" +`; + const filePath = path.join(tmpDir, 'override-pure-keyed-map.yaml'); + await fs.writeFile(filePath, content, 'utf-8'); + + await expect(loadOverrideConfig(filePath)).rejects.toThrow( + 'Invalid overrides.namedValues: expected an array in toolkit format' + ); + }); + + it('should fall back to item fields when toolkit item has no properties object', async () => { + const content = ` +namedValues: + - name: nv1 + value: "inline-value" +`; + const filePath = path.join(tmpDir, 'override-no-properties.yaml'); + await fs.writeFile(filePath, content, 'utf-8'); + + const config = await loadOverrideConfig(filePath); + expect(config).toBeDefined(); + expect(config!.namedValues).toEqual({ + nv1: { + value: 'inline-value', + }, + }); + }); + + it('should throw for invalid override section type', async () => { + const content = ` +namedValues: 123 +`; + const filePath = path.join(tmpDir, 'override-invalid-section.yaml'); + await fs.writeFile(filePath, content, 'utf-8'); + + await expect(loadOverrideConfig(filePath)).rejects.toThrow( + 'Invalid overrides.namedValues: expected an array in toolkit format' + ); + }); + + it('should ignore invalid array entries', async () => { + const content = ` +namedValues: + - name: nv1 + properties: + value: "valid" +backends: + - properties: + url: "https://missing-name.example.com" + - name: " " + properties: + url: "https://blank-name.example.com" + - name: be1 + properties: + url: "https://valid.example.com" +`; + const filePath = path.join(tmpDir, 'override-invalid-items.yaml'); + await fs.writeFile(filePath, content, 'utf-8'); + + const config = await loadOverrideConfig(filePath); + expect(config).toBeDefined(); + expect(config!.namedValues).toEqual({ + nv1: { + value: 'valid', + }, + }); + expect(config!.backends).toEqual({ + be1: { + url: 'https://valid.example.com', + }, + }); + }); }); describe('loadOTelConfig', () => { diff --git a/tests/unit/services/workspace-extractor.test.ts b/tests/unit/services/workspace-extractor.test.ts index 396983e..ed638a1 100644 --- a/tests/unit/services/workspace-extractor.test.ts +++ b/tests/unit/services/workspace-extractor.test.ts @@ -58,12 +58,10 @@ describe('workspace-extractor', () => { client, store, testContext, '/output', filter ); - const expectedCallSequence = [ + const expectedTypes = [ ResourceType.NamedValue, ResourceType.Tag, ResourceType.Backend, - // Logger extraction preloads NamedValue display names for placeholder normalization. - ResourceType.NamedValue, ResourceType.Logger, ResourceType.Group, ResourceType.Diagnostic, diff --git a/tests/unit/templates/configs/config-templates.test.ts b/tests/unit/templates/configs/config-templates.test.ts index 6116d66..3bc4ee7 100644 --- a/tests/unit/templates/configs/config-templates.test.ts +++ b/tests/unit/templates/configs/config-templates.test.ts @@ -89,44 +89,50 @@ describe('configs/override-config', () => { it('should include namedValues override examples', () => { const config = generateOverrideConfig('dev'); expect(config).toContain('# namedValues:'); - expect(config).toContain('# api-key:'); - expect(config).toContain('# value:'); + expect(config).toContain('# - name: api-key'); + expect(config).toContain('# properties:'); + expect(config).toContain('# value:'); }); it('should include backends override examples', () => { const config = generateOverrideConfig('dev'); expect(config).toContain('# backends:'); - expect(config).toContain('# backend-api:'); - expect(config).toContain('# url:'); + expect(config).toContain('# - name: backend-api'); + expect(config).toContain('# properties:'); + expect(config).toContain('# url:'); }); it('should include apis override examples', () => { const config = generateOverrideConfig('dev'); expect(config).toContain('# apis:'); - expect(config).toContain('# echo-api:'); - expect(config).toContain('# serviceUrl:'); + expect(config).toContain('# - name: echo-api'); + expect(config).toContain('# properties:'); + expect(config).toContain('# serviceUrl:'); }); it('should include KeyVault example in namedValues', () => { const config = generateOverrideConfig('staging'); - expect(config).toContain('# secret-from-keyvault:'); - expect(config).toContain('# keyVault:'); - expect(config).toContain('# secretIdentifier:'); + expect(config).toContain('# - name: secret-from-keyvault'); + expect(config).toContain('# properties:'); + expect(config).toContain('# keyVault:'); + expect(config).toContain('# secretIdentifier:'); expect(config).toContain('staging-kv.vault.azure.net'); }); it('should include diagnostics override examples', () => { const config = generateOverrideConfig('dev'); expect(config).toContain('# diagnostics:'); - expect(config).toContain('# applicationinsights:'); - expect(config).toContain('# loggerId:'); + expect(config).toContain('# - name: applicationinsights'); + expect(config).toContain('# properties:'); + expect(config).toContain('# loggerId:'); }); it('should include loggers override examples', () => { const config = generateOverrideConfig('dev'); expect(config).toContain('# loggers:'); - expect(config).toContain('# appinsights-logger:'); - expect(config).toContain('# resourceId:'); + expect(config).toContain('# - name: appinsights-logger'); + expect(config).toContain('# properties:'); + expect(config).toContain('# resourceId:'); }); it('should not have any uncommented configuration by default', () => {