diff --git a/README.md b/README.md index c51f9565..64a28544 100644 --- a/README.md +++ b/README.md @@ -124,8 +124,10 @@ apiops publish \ ``` The publish filter uses the same YAML file and matching rules as `apiops extract --filter`. -Referenced named values, backends (including backend pool members), policy fragments, and version -sets are included automatically. Add `--no-transitive` to publish only the exact filter matches. +The resources the selected ones depend on, such as referenced named values, backends (including +backend pool members), policy fragments, version sets, tags, groups and loggers, are included +automatically; see [Transitive Dependency Behavior](docs/guides/filtering-resources.md#transitive-dependency-behavior). +Add `--no-transitive` to publish only the exact filter matches. ### `apiops init` diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 5e6f31f1..3a6232bd 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -124,7 +124,7 @@ flowchart TD ### Key Extract Behaviors - **Topological ordering** — Resources are extracted in dependency order across 4 tiers, ensuring parent resources are written before children. -- **Transitive dependency resolution** — Policy XML is scanned for references to named values, backends, and policy fragments. Referenced resources are included even if not explicitly listed in the filter. +- **Transitive dependency resolution** — Policy XML is scanned for references to named values, backends, policy fragments, loggers and schemas; APIs, diagnostics, loggers, backends and products are scanned for their version set, logger, credential named values, tags and groups. Referenced resources are included even if not explicitly listed in the filter. - **Dual output format** — Resource metadata is written as JSON (`*.info.json`), while policies are written as raw XML (`policy.xml`). - **LIST then GET** — Some APIM list endpoints omit fields. When `listOmitsFields` is set for a resource type, individual GET calls fetch the full representation. diff --git a/docs/commands/extract.md b/docs/commands/extract.md index c4cce83e..c05479b8 100644 --- a/docs/commands/extract.md +++ b/docs/commands/extract.md @@ -66,7 +66,7 @@ apiops extract \ | `--output ` | string | `./apim-artifacts` | No | Output directory path | | `--filter ` | string | — | No | Filter configuration YAML file | | `--no-transitive` | boolean | false (transitive ON) | No | Disable transitive dependency inclusion | -| `--remove-stale` | boolean | false | No | Remove stale managed artifacts in the selected filter and workspace scope after a fully successful extraction | +| `--remove-stale` | boolean | false | No | Remove stale managed artifacts in the selected filter and workspace scope, and pulled-in dependencies nothing references any more, after a fully successful extraction | ### Global flags @@ -159,7 +159,7 @@ For every key, the value semantics are: ### API sub-filters To restrict which child resources of an API are extracted (operations, -diagnostics, schemas, releases), use the **nested object entry** form under +diagnostics, schemas, releases, tag descriptions), use the **nested object entry** form under `apis:`. Each entry becomes a `name: { subKey: [...] }` map: ```yaml @@ -175,9 +175,13 @@ apis: releases: [] ``` -Supported API sub-filter keys: `operations`, `diagnostics`, `schemas`, `releases`. +Supported API sub-filter keys: `operations`, `diagnostics`, `schemas`, `releases`, `tagDescriptions`. Same value semantics as top-level keys (omitted = include all, `[]` = include none, list = allowlist). +The key of a nested entry can be a wildcard pattern (`'*'`, `'v2-*'`), so one +sub-filter applies to every matching API. An entry keyed by an API's name takes +precedence over a pattern. See [One sub-filter for many APIs](../guides/filtering-resources.md#one-sub-filter-for-many-apis). + ### Workspace sub-filters Workspaces accept the same nested object entry form. Each workspace can have @@ -197,12 +201,18 @@ Supported workspace sub-filter keys: `apis`, `backends`, `diagnostics`, `groups`, `loggers`, `namedValues`, `policyFragments`, `products`, `schemas`, `subscriptions`, `tags`, `versionSets`. +As for APIs, the key can be a wildcard pattern (`'team-*'`), and an entry keyed +by the workspace's name takes precedence over a pattern. See +[Workspace sub-resource filters](../guides/filtering-resources.md#workspace-sub-resource-filters). + ### Transitive dependencies -When extracting with a filter, transitive dependencies are **included automatically**. For example, extracting `echo-api` also extracts any backends, named values, and policy fragments that `echo-api` references in its policies. +When extracting with a filter, transitive dependencies are **included automatically**. For example, extracting `echo-api` also extracts the backends, named values and policy fragments its policies reference, its version set, its tags, and the loggers its diagnostics use. See [Transitive Dependency Behavior](../guides/filtering-resources.md#transitive-dependency-behavior) for the full list. Use `--no-transitive` to disable this behavior and extract only the explicitly listed resources. +With `--remove-stale`, a pulled-in dependency the filter otherwise leaves out (for example a tag when `tags: []`) is removed once the selected resources no longer reference it and no other artifact in the output does. A dependency another filtered extract into the same output still uses is kept. + ## Output format Extracted artifacts are organized as JSON info files and XML policy files in a directory tree: diff --git a/docs/commands/publish.md b/docs/commands/publish.md index 8d14dc32..d19bc000 100644 --- a/docs/commands/publish.md +++ b/docs/commands/publish.md @@ -53,10 +53,12 @@ apiops publish \ --filter ./configuration.extractor.yaml ``` -The filter uses the same YAML format as `apiops extract --filter`. Referenced named values, -backends (including backend pool members), policy fragments, and version sets are included by -default; use `--no-transitive` to publish only exact matches. Product, gateway, and subscription -links do not pull their API or Product targets into the publish set. +The filter uses the same YAML format as `apiops extract --filter`. The resources the selected ones +depend on — such as referenced named values, backends (including backend pool members), policy +fragments, version sets, tags, groups and loggers — are included by default; see +[Transitive Dependency Behavior](../guides/filtering-resources.md#transitive-dependency-behavior). +Use `--no-transitive` to publish only exact matches. Product, gateway, and subscription links do +not pull their API or Product targets into the publish set. ### Delete resources not in source diff --git a/docs/getting-started.md b/docs/getting-started.md index 1e4c9cd6..a9b12fbd 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -75,7 +75,7 @@ apiops extract \ --filter filter.yaml ``` -Referenced backends, named values, and policy fragments are included automatically (transitive dependencies). Use `--no-transitive` to disable this. +Referenced backends, named values, policy fragments and other dependencies are included automatically ([transitive dependencies](guides/filtering-resources.md#transitive-dependency-behavior)). Use `--no-transitive` to disable this. ## 4. Publish to a Target Environment diff --git a/docs/guides/filtering-resources.md b/docs/guides/filtering-resources.md index 4b08c976..b5bdf039 100644 --- a/docs/guides/filtering-resources.md +++ b/docs/guides/filtering-resources.md @@ -76,7 +76,7 @@ documentations: [] workspaces: [] ``` -With transitive resolution enabled (the default), any version set, backend, named value, policy fragment, or tag directly referenced by `my-own-api` or its policies is still pulled in automatically — even though those keys are set to `[]`. Use `--no-transitive` to disable that behavior. +With transitive resolution enabled (the default), the resources `my-own-api` depends on — such as the backends, named values and policy fragments its policies reference, its version set, its tags, and its diagnostics' loggers — are still pulled in automatically, even though those keys are set to `[]`. See [Transitive Dependency Behavior](#transitive-dependency-behavior) for the full list. Use `--no-transitive` to disable that behavior. The three states for every key are: @@ -228,7 +228,7 @@ apis: ### API sub-resource filters -For APIs, you can control which sub-resources (operations, diagnostics, schemas, releases) are extracted. Use an object entry instead of a plain string: +For APIs, you can control which sub-resources (operations, diagnostics, schemas, releases, tag descriptions) are extracted. Use an object entry instead of a plain string: ```yaml apis: @@ -242,6 +242,7 @@ apis: schemas: [] # Empty = exclude ALL schemas releases: - v1-release + tagDescriptions: [] # By name, as under apis//tagDescriptions/ ``` **Sub-filter rules:** @@ -249,6 +250,20 @@ apis: - If a sub-resource key is an **empty array** (`[]`), all sub-resources of that type are excluded - If a sub-resource key lists **names**, only those sub-resources are included +#### One sub-filter for many APIs + +The key of a nested entry can be a wildcard pattern (`*`, `?`) instead of an API name. Like a plain pattern, it selects the matching APIs, and its sub-filter applies to each of them. For example, to extract every API without its operations, schemas or tag descriptions: + +```yaml +apis: + - '*': + operations: [] + schemas: [] + tagDescriptions: [] +``` + +An entry keyed by an API's name takes precedence over a pattern; otherwise the first matching pattern applies. Quote patterns that start with `*` so YAML reads them as strings. + During extraction, the `operations` filter also applies to the API's OpenAPI specification (`specification.yaml` or `specification.json`, including Swagger 2.0). Excluded operations are removed from the specification, and paths with no remaining operations are removed. @@ -275,6 +290,10 @@ workspaces: Supported workspace sub-filter keys: `apis`, `backends`, `diagnostics`, `groups`, `loggers`, `namedValues`, `policyFragments`, `products`, `schemas`, `subscriptions`, `tags`, `versionSets`. +As for APIs, the key can be a wildcard pattern: `- 'team-*': { apis: [orders] }` selects every matching workspace and applies the sub-filter to each. An entry keyed by the workspace's name takes precedence over a pattern; otherwise the first matching pattern applies. + +A sub-filter can't be attached to an exclusion (`- '!team-c': {...}`), for APIs or workspaces: the exclusion still applies, but its sub-filter is ignored with a warning. + --- ## All 17 Filterable Resource Types @@ -303,20 +322,28 @@ Supported workspace sub-filter keys: `apis`, `backends`, `diagnostics`, `groups` ## Transitive Dependency Behavior -When you filter by API name, `apiops extract` automatically includes resources referenced by those APIs. This is **transitive dependency resolution** — it ensures the extracted artifacts are self-contained and publishable. +When you filter resources, `apiops extract` and `apiops publish` automatically include the resources the selected ones depend on. This is **transitive dependency resolution** — it keeps the extracted artifacts self-contained and publishable. Dependencies are followed until nothing new is found, so a pulled-in resource's own dependencies are included too. ### What Gets Auto-Included -```mermaid -flowchart TD - A[Filtered API] --> B[Backends referenced in policies] - A --> C[Named Values referenced in policies] - A --> D[Policy Fragments included in policies] - A --> E[Tags attached to the API] - A --> F[Products containing the API] - A --> G[Diagnostics configured on the API] - A --> H[Version Set the API belongs to] -``` +| Selected resource | Includes | +|-------------------|----------| +| Any policy (service, API, operation, GraphQL resolver, product) and policy fragment | Named values referenced as `{{displayName}}`; backends named by ``, `` and `embeddings-backend-id` on the semantic cache lookup policies; fragments named by ``; loggers named by ``; schemas named by validate-content's `` | +| API | Its version set, and the tags it is linked to | +| Diagnostic (service-level or on an API) | The logger it sends data to (`loggerId`) | +| Logger, backend | Named values referenced in their credentials | +| Backend pool | Its member backends | +| Product | The groups and tags it is linked to | + +A reference written as a policy expression (`backend-id="@(...)"`) is not followed, because its value is only known at run time. One set to a named value (`backend-id="{{backend-name}}"`) pulls in the named value and the resource its value names, in the same scope (except ``, where APIM doesn't substitute named values); publish reads the value with your `--overrides` applied, so each environment gets the resource its own value names. A named value that is published also brings the resource its value names whenever any policy or fragment in the source uses it that way, even one the filter leaves out: APIM checks every policy on the target that uses a named value when its value changes. A resource the filter excludes by name, or that isn't in the source, isn't published, so it has to exist on the target already. When a policy uses a named value to name a backend, logger or schema that is published in the same run, publish creates that resource before it updates the named value, and publishes policy fragments, and any backend or logger using the named value in its credentials, after it, since APIM rejects a named value change that would leave a policy using it pointing at a resource that doesn't exist. A secret or Key Vault named value's value isn't known, so the resource it names isn't pulled in; add that resource to the filter. + +A `{{...}}` reference, in a policy or in credentials, names a named value by its display name, as APIM does; when no named value has that display name, it is read as a resource name. The `namedValues` list and its exclusions (`'!name'`) use the resource name, which can differ from the display name. References inside XML comments are ignored. Inside a `` body, a `{{name}}` is followed only when a named value has that name: APIM substitutes those there and leaves every other `{{ ... }}` (`{{body.id}}`, `{{ name }}`, `{{x | upcase}}`) to Liquid, so those are skipped without an error. + +A resource the filter excludes by name (`'!name'`) is never pulled in, and links to it are not extracted: an API's or product's link to an excluded tag, or a product's link to an excluded group. Built-in groups (`administrators`, `developers`, `guests`) exist at service level in every APIM instance, so they are not pulled in either; a product's links to them are published only when the filter includes them, as for any group. The exception is the link to `administrators`, which APIM adds to every product itself: it is published whatever the `groups` filter says, unless the filter excludes `administrators` by name. + +**Not included:** the products, gateways and subscriptions an API is linked to, and the APIs a product contains. These are independent resources, not dependencies; add them to the filter to include them. On publish, links to APIs or products outside the publish set are skipped. + +> **Logger keys:** APIM stores an Application Insights logger's key as an auto-generated secret named value, named with a 24-character hex ID (its display name is `Logger-Credentials--`). It is pulled in with the logger and extracted with its value redacted. Publish skips it unless you provide an override for it: an instance that already has the logger already has its key, but on a new instance the logger can't be published without one. See [Logger credentials with auto-generated named values](environment-overrides.md#logger-credentials-with-auto-generated-named-values). ### Example @@ -325,6 +352,12 @@ Given this filter: ```yaml apis: - petstore-api +backends: [] +namedValues: [] +policyFragments: [] +loggers: [] +tags: [] +products: [] ``` If `petstore-api` has a policy that references: @@ -332,11 +365,10 @@ If `petstore-api` has a policy that references: - Named value `petstore-api-key` → **auto-included** - Policy fragment `rate-limit-fragment` → **auto-included** -And `petstore-api` is assigned to: -- Product `starter` → **auto-included** -- Tag `production` → **auto-included** - -The extract output includes all of these, even though only `apiNames` was specified in the filter. +And `petstore-api`: +- has a diagnostic that sends data to logger `appinsights` → **auto-included** +- is tagged `production` → **auto-included** +- is in product `starter` → **not included**; add `starter` to `products` to include it ### Why Transitive Matters @@ -379,7 +411,7 @@ apis: - orders-admin-api ``` -Transitive dependencies (backends, named values, policy fragments) are auto-included. +Transitive dependencies (backends, named values, policy fragments, version sets, tags and loggers) are auto-included. ### Product-Focused Extract diff --git a/docs/reference/dependency-graph.md b/docs/reference/dependency-graph.md index bb45884a..73186517 100644 --- a/docs/reference/dependency-graph.md +++ b/docs/reference/dependency-graph.md @@ -15,7 +15,7 @@ apiops-cli solves this by organizing all 34 resource types into **4 tiers**. Eac | Tier | Resources | Rule | |------|-----------|------| -| **1** | NamedValue, Tag, Gateway, VersionSet, Backend, Logger, Group, PolicyFragment, GlobalSchema, PolicyRestriction, Documentation | No dependencies — safe to process in any order | +| **1** | NamedValue, Tag, Gateway, VersionSet, Backend, Logger, Group, PolicyFragment, GlobalSchema, PolicyRestriction, Documentation | No dependencies on other tiers. Within the tier, named values go first, since backends and loggers can use them; pool backends follow the backends they contain; and a named value a policy uses to name a backend, logger or schema published in the same run goes after that resource, with policy fragments after it in that case. A backend or logger in the run that uses such a named value in its credentials goes after the named value, and these are ordered so each comes after what it needs; only where that is impossible (a cycle, as when a backend uses in its credentials the named value that names it, or a pool containing it), the named values in it go first | | **2** | Diagnostic, ServicePolicy, Product, Api | Depend only on Tier 1 resources | | **3** | ProductPolicy, ProductGroup, ProductTag, ProductApi, ProductWiki, ApiPolicy, ApiTag, ApiDiagnostic, ApiOperation, ApiSchema, ApiRelease, ApiTagDescription, ApiWiki, McpServer, GraphQLResolver, GatewayApi, Subscription | Depend on Tier 1 and/or Tier 2 resources | | **4** | ApiOperationPolicy, GraphQLResolverPolicy | Depend on Tier 3 resources | @@ -163,9 +163,12 @@ If a Tier 2 resource fails to publish, its dependents in Tier 3 and 4 will also When using `filter.yaml` to include specific resources, be aware of dependencies. -**During publish**, apiops-cli does **not** auto-include dependencies at the -artifact level — your filter (and hence your extracted artifact tree) must -already contain every required parent. For example, publishing an **ApiPolicy** +**During publish** with a filter, apiops-cli includes the resources the +selected ones depend on (named values, backends, policy fragments, version +sets, tags, groups, loggers and so on) unless `--no-transitive` is passed; see +[Transitive Dependency Behavior](../guides/filtering-resources.md#transitive-dependency-behavior). +It can only include what is in the artifact tree, so the tree must already +contain every required resource. For example, publishing an **ApiPolicy** without its parent **Api** present in the artifact tree will fail. **During extract**, apiops-cli **does** auto-follow a limited set of transitive diff --git a/schemas/v1/extractor-config.schema.json b/schemas/v1/extractor-config.schema.json index 1ea17a9b..c34d7a59 100644 --- a/schemas/v1/extractor-config.schema.json +++ b/schemas/v1/extractor-config.schema.json @@ -105,7 +105,7 @@ }, { "type": "object", - "description": "A single API name mapped to sub-resource filters for that API.", + "description": "A single API name, or a wildcard pattern such as *, mapped to sub-resource filters for the matching APIs. A name takes precedence over a pattern.", "minProperties": 1, "maxProperties": 1, "patternProperties": { @@ -124,7 +124,7 @@ }, { "type": "object", - "description": "A single workspace name mapped to sub-resource filters for that workspace.", + "description": "A single workspace name, or a wildcard pattern such as team-*, mapped to sub-resource filters for the matching workspaces. A name takes precedence over a pattern.", "minProperties": 1, "maxProperties": 1, "patternProperties": { @@ -156,6 +156,10 @@ "releases": { "$ref": "#/definitions/resourcePatternArray", "description": "Releases to extract for this API. Matching is case-insensitive and supports * and ? wildcards." + }, + "tagDescriptions": { + "$ref": "#/definitions/resourcePatternArray", + "description": "TagDescriptions to extract for this API. Matching is case-insensitive and supports * and ? wildcards." } } }, diff --git a/scripts/generate-schemas.mjs b/scripts/generate-schemas.mjs index 524a563f..73284398 100644 --- a/scripts/generate-schemas.mjs +++ b/scripts/generate-schemas.mjs @@ -192,7 +192,7 @@ const extractorSchema = { { type: 'object', description: - 'A single API name mapped to sub-resource filters for that API.', + 'A single API name, or a wildcard pattern such as *, mapped to sub-resource filters for the matching APIs. A name takes precedence over a pattern.', minProperties: 1, maxProperties: 1, patternProperties: { @@ -208,7 +208,7 @@ const extractorSchema = { { type: 'object', description: - 'A single workspace name mapped to sub-resource filters for that workspace.', + 'A single workspace name, or a wildcard pattern such as team-*, mapped to sub-resource filters for the matching workspaces. A name takes precedence over a pattern.', minProperties: 1, maxProperties: 1, patternProperties: { diff --git a/src/cli/extract-command.ts b/src/cli/extract-command.ts index 4dec2332..d9fbcdf6 100644 --- a/src/cli/extract-command.ts +++ b/src/cli/extract-command.ts @@ -13,7 +13,8 @@ import * as path from 'node:path'; import { ExtractConfig, FilterConfig } from '../models/config.js'; import { ApimServiceContext } from '../models/types.js'; import { runExtraction, ExtractionResult } from '../services/extract-service.js'; -import { shouldReconcileResource } from '../services/filter-service.js'; +import { isInRemoveStaleScope } from '../services/filter-service.js'; +import { findStaleDependencies } from '../services/stale-dependency-finder.js'; import { loadFilterConfig } from '../lib/config-loader.js'; import { logger, parseLogLevel } from '../lib/logger.js'; import { ApimClient } from '../clients/apim-client.js'; @@ -24,7 +25,7 @@ import { EXIT_FATAL, EXIT_SUCCESS } from '../lib/exit-codes.js'; import { getResourceTier, TIER_LABELS } from '../lib/dependency-graph.js'; import { formatDuration } from '../lib/format-duration.js'; import { ResourceType } from '../models/resource-types.js'; -import { getNamePart } from '../lib/resource-path.js'; +import { getNamePart, getResourceDescriptorKey } from '../lib/resource-path.js'; /** * Interface for extract command options (from CLI flags). @@ -63,7 +64,7 @@ export function createExtractCommand(): Command { .option('--output ', 'Output directory path', './apim-artifacts') .option('--filter ', 'Filter configuration YAML file') .option('--no-transitive', 'Disable transitive dependency inclusion') - .option('--remove-stale', 'Remove stale managed artifacts in the selected scope') + .option('--remove-stale', 'Remove stale managed artifacts in the selected scope, and pulled-in dependencies nothing references any more') .action(async (options: ExtractOptions, command: Command) => { const globalOpts = command.optsWithGlobals<{ logLevel?: string; @@ -153,11 +154,19 @@ export async function executeExtract( result = await dependencies.runExtraction(client, store, extractConfig); if (result.exitCode !== EXIT_FATAL) { + const removeStale = shouldRemoveStaleArtifacts(result, options.removeStale ?? false); + // Pulled-in dependencies are outside the filter's scope; remove the + // ones the selection no longer needs and nothing else references. + const staleDependencies = removeStale && filterConfig + ? await findStaleDependencies(store, outputDir, stagingDir, filterConfig) + : []; + const staleKeys = new Set(staleDependencies.map(getResourceDescriptorKey)); await store.commitStagedExtraction( stagingDir, outputDir, - descriptor => shouldReconcileResource(descriptor, filterConfig), - shouldRemoveStaleArtifacts(result, options.removeStale ?? false) + descriptor => isInRemoveStaleScope(descriptor, filterConfig) || + staleKeys.has(getResourceDescriptorKey(descriptor)), + removeStale ); } } finally { diff --git a/src/lib/arm-id.ts b/src/lib/arm-id.ts new file mode 100644 index 00000000..91c217b4 --- /dev/null +++ b/src/lib/arm-id.ts @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +/** + * Reading names and scope from APIM ARM resource IDs, such as + * `/subscriptions/{s}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{apim}/workspaces/{ws}/loggers/{name}`, + * or the relative form APIM also returns (`/loggers/{name}`). + */ + +import type { AssociationScope } from '../models/types.js'; +import { logger } from './logger.js'; + +const FULL_SERVICE_ID_PATTERN = + /\/subscriptions\/[^/]+\/resourceGroups\/[^/]+\/providers\/Microsoft\.ApiManagement\/service\/[^/]+/i; + +/** + * Decode one ARM ID segment. A malformed escape leaves the segment as it is, + * with a warning. + */ +export function decodeArmSegment(value: string): string { + try { + return decodeURIComponent(value); + } catch (error) { + logger.warn(`Unable to decode ARM resource ID segment; using the raw value: ${String(error)}`); + return value; + } +} + +/** + * The part of an ARM ID below the APIM service. Segment names are matched + * there only, so a resource group or service named `workspaces`, `products` + * and so on is not read as one. + */ +function pathBelowService(id: string): string { + return id.replace(FULL_SERVICE_ID_PATTERN, ''); +} + +/** + * The decoded name of the `{segment}` collection's resource in an ARM ID + * (case-insensitive), for example `armNameAfter(id, 'loggers')`. The path + * below the service is read as collection/name pairs, so a resource named + * like a collection (a workspace named `backends`) is not read as one. + */ +export function armNameAfter(id: string, segment: string): string | undefined { + const parts = pathBelowService(id).split('/').filter(Boolean); + for (let index = 0; index + 1 < parts.length; index += 2) { + if (parts[index].toLowerCase() === segment.toLowerCase()) { + return decodeArmSegment(parts[index + 1]); + } + } + return undefined; +} + +/** The decoded last segment of an ARM ID, or undefined when it has none. */ +export function armLastName(id: string): string | undefined { + const segment = id.split('/').filter(Boolean).at(-1); + return segment ? decodeArmSegment(segment) : undefined; +} + +/** `workspace` when the ARM ID is in a workspace, otherwise `service`. */ +export function armScope(id: string): AssociationScope { + return /\/workspaces\/[^/]+(?:\/|$)/i.test(pathBelowService(id)) ? 'workspace' : 'service'; +} + +/** + * The workspace an ARM reference points into: the one it names, none for a + * full service-level ID, and `fallback` (the referencing resource's + * workspace) for a relative reference such as `/loggers/{name}`. + */ +export function armReferenceWorkspace(id: string, fallback?: string): string | undefined { + const workspace = armNameAfter(id, 'workspaces'); + if (workspace) { + return workspace; + } + return FULL_SERVICE_ID_PATTERN.test(id) ? undefined : fallback; +} diff --git a/src/lib/config-loader.ts b/src/lib/config-loader.ts index 74669367..9b5416c9 100644 --- a/src/lib/config-loader.ts +++ b/src/lib/config-loader.ts @@ -163,8 +163,12 @@ export async function loadFilterConfig(filePath: string): Promise 0) { config.apiSubFilters = {}; - const validApiSubKeys = ['operations', 'diagnostics', 'schemas', 'releases'] as const; + const validApiSubKeys = ['operations', 'diagnostics', 'schemas', 'releases', 'tagDescriptions'] as const; for (const [apiName, sf] of Object.entries(subFilters)) { + if (apiName.startsWith('!')) { + logger.warn(`Sub-filter for exclusion '${apiName}' in '${sourceKey}' has no effect; ignoring.`); + continue; + } const apiSub: ApiSubFilter = {}; for (const subKey of validApiSubKeys) { if (subKey in sf) { @@ -188,6 +192,10 @@ export async function loadFilterConfig(filePath: string): Promise 0 - ? decodeURIComponent(lastSegment) - : undefined; + return armLastName(armId); } /** @@ -177,3 +174,50 @@ export function buildLinkPayload( }, }; } + +/** + * In a workspace, API and product tags are links listed under each tag + * (`tags/{tag}/apiLinks`, `tags/{tag}/productLinks`). Yield each tag and the + * name of the API or product it links to. A link without a readable target, + * or a failed listing, throws. + * + * @param context - Workspace-scoped context (its baseUrl includes the workspace) + * @param tagNames - Tags to list links for + * @param linkType - ApiTag (apiLinks) or ProductTag (productLinks) + */ +export async function* listWorkspaceTagLinks( + client: IApimClient, + context: ApimServiceContext, + tagNames: Iterable, + linkType: ResourceType.ApiTag | ResourceType.ProductTag +): AsyncGenerator<{ tagName: string; targetName: string }> { + const linkProperty = RESOURCE_TYPE_METADATA[linkType].workspaceLinkIdProperty; + if (!linkProperty) { + return; + } + const [targetNoun, linkName] = linkType === ResourceType.ApiTag + ? ['API', 'apiLink'] + : ['product', 'productLink']; + + for (const tagName of tagNames) { + // Do NOT set workspace on this descriptor — context.baseUrl already + // includes the workspace prefix and buildArmUri would double it. + const tagDescriptor: ResourceDescriptor = { type: ResourceType.Tag, nameParts: [tagName] }; + const targets: string[] = []; + try { + for await (const linkJson of client.listResources(context, linkType, tagDescriptor)) { + const targetName = extractNameFromLink(linkJson, linkProperty); + if (!targetName) { + throw new Error(`Failed to extract ${targetNoun} name from tag "${tagName}" ${linkName} response`); + } + targets.push(targetName); + } + } catch (error) { + logger.warn(`Failed to list ${linkName}s for tag "${tagName}": ${(error as Error).message}`); + throw error; + } + for (const targetName of targets) { + yield { tagName, targetName }; + } + } +} diff --git a/src/models/config.ts b/src/models/config.ts index 1498eef3..1555fa1c 100644 --- a/src/models/config.ts +++ b/src/models/config.ts @@ -27,6 +27,7 @@ export interface ApiSubFilter { diagnostics?: string[]; schemas?: string[]; releases?: string[]; + tagDescriptions?: string[]; } /** diff --git a/src/models/resource-types.ts b/src/models/resource-types.ts index 6686edb1..23b9752c 100644 --- a/src/models/resource-types.ts +++ b/src/models/resource-types.ts @@ -97,6 +97,27 @@ export interface ResourceTypeMetadata { readonly workspaceLinkIdProperty?: string; } +/** Groups every APIM instance has; they are never created, changed or deleted. */ +export const BUILT_IN_GROUP_NAMES: readonly string[] = ['administrators', 'developers', 'guests']; + +/** Whether `name` is one of the built-in groups (case-insensitive). */ +export function isBuiltInGroupName(name: string): boolean { + return BUILT_IN_GROUP_NAMES.includes(name.toLowerCase()); +} + +/** + * Whether a descriptor is a built-in group. Built-in groups exist only at + * service level; a workspace group with the same name is a custom group. + */ +export function isBuiltInGroup(descriptor: { type: ResourceType; nameParts: string[]; workspace?: string }): boolean { + return ( + descriptor.type === ResourceType.Group && + !descriptor.workspace && + descriptor.nameParts.length === 1 && + isBuiltInGroupName(descriptor.nameParts[0] ?? '') + ); +} + export const RESOURCE_TYPE_METADATA: Record = { [ResourceType.NamedValue]: { armPathSuffix: 'namedValues/{0}', diff --git a/src/services/api-extractor.ts b/src/services/api-extractor.ts index 2e477385..3060f5da 100644 --- a/src/services/api-extractor.ts +++ b/src/services/api-extractor.ts @@ -10,17 +10,17 @@ import { IApimClient, ApiSpecDialect } from '../clients/iapim-client.js'; import { IArtifactStore } from '../clients/iartifact-store.js'; import { ApimServiceContext, ResourceDescriptor } from '../models/types.js'; -import { ResourceType, RESOURCE_TYPE_METADATA } from '../models/resource-types.js'; +import { ResourceType } from '../models/resource-types.js'; import { FilterConfig } from '../models/config.js'; import * as yaml from 'js-yaml'; -import { extractRootApiName, shouldIncludeResource } from './filter-service.js'; +import { extractRootApiName, findApiSubFilter, shouldIncludeResource } from './filter-service.js'; import { extractResourceType, ExtractedResource } from './resource-extractor.js'; import { redactAndWarnPolicySecrets } from './secret-redactor.js'; import { logger } from '../lib/logger.js'; import { buildResourceLabel } from '../lib/resource-uri.js'; import { getNamePart } from '../lib/resource-path.js'; import { normalizeWsdl } from '../lib/wsdl-normalizer.js'; -import { isWorkspaceScope, extractNameFromLink } from '../lib/workspace-link.js'; +import { isWorkspaceScope, listWorkspaceTagLinks } from '../lib/workspace-link.js'; /** * `JSON.parse`'s reviver `context` argument and `JSON.rawJSON` are ES2024 @@ -381,10 +381,7 @@ function filterOpenApiOperations( ): string { const apiName = getNamePart(apiDescriptor.nameParts, 0); const rootName = extractRootApiName(apiName).toLowerCase(); - const subFilterKey = Object.keys(filter?.apiSubFilters ?? {}).find( - key => key.toLowerCase() === rootName - ); - if (!subFilterKey || filter?.apiSubFilters?.[subFilterKey].operations === undefined) { + if (findApiSubFilter(rootName, filter?.apiSubFilters)?.operations === undefined) { return content; } @@ -810,6 +807,7 @@ async function extractGraphQLResolvers( * @param extractedApiNames - API names already extracted for this workspace * @param outputDir - Output directory * @param workspace - Workspace name + * @param onLink - Called with each ApiTag artifact written * @returns Number of ApiTag artifacts written */ export async function extractWorkspaceApiTags( @@ -819,59 +817,36 @@ export async function extractWorkspaceApiTags( extractedTagNames: string[], extractedApiNames: Set, outputDir: string, - workspace: string + workspace: string, + onLink?: (descriptor: ResourceDescriptor, json: Record) => void ): Promise { - const linkProperty = RESOURCE_TYPE_METADATA[ResourceType.ApiTag].workspaceLinkIdProperty; - if (!linkProperty) { - return 0; - } - let count = 0; - for (const tagName of extractedTagNames) { - // List apiLinks under this tag. - // Do NOT set workspace on this descriptor — context.baseUrl already includes - // the workspace prefix and buildArmUri would double it. - const tagDescriptor: ResourceDescriptor = { - type: ResourceType.Tag, - nameParts: [tagName], - }; - - try { - for await (const linkJson of client.listResources(context, ResourceType.ApiTag, tagDescriptor)) { - const apiName = extractNameFromLink(linkJson, linkProperty); - if (!apiName) { - throw new Error(`Failed to extract API name from tag "${tagName}" apiLink response`); - } - - // Only create ApiTag artifacts for APIs that were extracted - if (!extractedApiNames.has(apiName)) { - logger.debug(`Skipping apiLink for tag "${tagName}" → API "${apiName}" (API not extracted)`); - continue; - } - - // Write the ApiTag artifact - const apiTagDescriptor: ResourceDescriptor = { - type: ResourceType.ApiTag, - nameParts: [apiName, tagName], - workspace, - }; + for await (const { tagName, targetName: apiName } of listWorkspaceTagLinks( + client, context, extractedTagNames, ResourceType.ApiTag + )) { + // Only create ApiTag artifacts for APIs that were extracted + if (!extractedApiNames.has(apiName)) { + logger.debug(`Skipping apiLink for tag "${tagName}" → API "${apiName}" (API not extracted)`); + continue; + } - // ApiTag artifacts store a minimal tag information JSON - const tagJson: Record = { - properties: { - displayName: tagName, - }, - }; + const apiTagDescriptor: ResourceDescriptor = { + type: ResourceType.ApiTag, + nameParts: [apiName, tagName], + workspace, + }; + // ApiTag artifacts store a minimal tag information JSON + const tagJson: Record = { + properties: { + displayName: tagName, + }, + }; - await store.writeResource(outputDir, apiTagDescriptor, tagJson); - logger.info(`Extracted workspace ApiTag: ${apiName}/tags/${tagName}`); - count++; - } - } catch (error) { - logger.warn(`Failed to list apiLinks for tag "${tagName}": ${(error as Error).message}`); - throw error; - } + await store.writeResource(outputDir, apiTagDescriptor, tagJson); + logger.info(`Extracted workspace ApiTag: ${apiName}/tags/${tagName}`); + onLink?.(apiTagDescriptor, tagJson); + count++; } return count; diff --git a/src/services/api-publisher.ts b/src/services/api-publisher.ts index 4fbc7fa2..5b80e6af 100644 --- a/src/services/api-publisher.ts +++ b/src/services/api-publisher.ts @@ -33,7 +33,7 @@ import { } from '../lib/resource-path.js'; import { isAutoGeneratedId, isPortalGeneratedSchemaId } from '../lib/auto-generated.js'; import { normalizeWsdlXsdImportLocations } from '../lib/wsdl-normalizer.js'; -import { resolveWorkspaceFilter, shouldIncludeResource } from './filter-service.js'; +import { findApiSubFilter, resolveWorkspaceFilter, shouldIncludeResource } from './filter-service.js'; /** * API child resource types that should be published after the API itself @@ -185,7 +185,8 @@ export async function publishApi( store, context, config, - publicationPlan + publicationPlan, + allowedDescriptors )); return { @@ -225,9 +226,7 @@ export async function planApiPublication( const effectiveFilter = apiDescriptor.workspace ? resolveWorkspaceFilter(apiDescriptor.workspace, config.filter) : config.filter; - const subFilter = Object.entries(effectiveFilter?.apiSubFilters ?? {}).find( - ([name]) => name.toLowerCase() === apiName.toLowerCase() - )?.[1]; + const subFilter = findApiSubFilter(apiName, effectiveFilter?.apiSubFilters); const allowed = allowedDescriptors ? new Set(allowedDescriptors.map(getResourceDescriptorKey)) : undefined; @@ -718,7 +717,8 @@ async function publishApiChildren( store: IArtifactStore, context: ApimServiceContext, config: PublishConfig, - plan: ApiPublicationPlan + plan: ApiPublicationPlan, + allowedDescriptors?: ResourceDescriptor[] ): Promise { const results: ResourcePublishResult[] = []; // Group resources by publish tier for dependency ordering. @@ -738,7 +738,7 @@ async function publishApiChildren( const descriptors = tierMap.get(tier) ?? []; if (descriptors.length > 0) { const tasks = descriptors.map((descriptor) => async () => { - return publishResource(client, store, context, descriptor, config); + return publishResource(client, store, context, descriptor, config, allowedDescriptors); }); const taskResults = await runParallel(tasks, 5); for (const [index, taskResult] of (taskResults ?? []).entries()) { diff --git a/src/services/delete-unmatched-service.ts b/src/services/delete-unmatched-service.ts index bbb77691..61798324 100644 --- a/src/services/delete-unmatched-service.ts +++ b/src/services/delete-unmatched-service.ts @@ -27,7 +27,7 @@ import type { IApimClient } from '../clients/iapim-client.js'; import type { IArtifactStore } from '../clients/iartifact-store.js'; import type { ApimServiceContext, ResourceDescriptor } from '../models/types.js'; import type { PublishConfig } from '../models/config.js'; -import { ResourceType } from '../models/resource-types.js'; +import { isBuiltInGroupName, ResourceType } from '../models/resource-types.js'; import { getTopologicalOrder } from '../lib/dependency-graph.js'; import { getNameFromNameParts, @@ -77,11 +77,6 @@ export function filterRevisionDeletesHandledByBaseApi( }); } -/** - * Built-in groups that should never be deleted - */ -const BUILT_IN_GROUPS = ['administrators', 'developers', 'guests']; - /** * System resources that should never be deleted */ @@ -393,7 +388,7 @@ function isSystemResource(descriptor: ResourceDescriptor): boolean { // Check built-in groups if (descriptor.type === ResourceType.Group) { - if (BUILT_IN_GROUPS.includes(ownName)) { + if (isBuiltInGroupName(ownName)) { return true; } } diff --git a/src/services/dry-run-reporter.ts b/src/services/dry-run-reporter.ts index 4431e810..745de3af 100644 --- a/src/services/dry-run-reporter.ts +++ b/src/services/dry-run-reporter.ts @@ -361,7 +361,9 @@ async function planDryRunPublications( } for (const child of apiPlan.childPuts) { if (child.type === ResourceType.ApiTag) { - const eligibility = await evaluateResourceEligibility(store, child, config); + const eligibility = await evaluateResourceEligibility( + store, child, config, undefined, targetDescriptors + ); if ( eligibility.eligible && !child.workspace && @@ -415,7 +417,9 @@ async function planDryRunPublications( const eligibility = await evaluateResourceEligibility( store, descriptor, - config + config, + undefined, + targetDescriptors ); if ( eligibility.eligible && diff --git a/src/services/extract-service.ts b/src/services/extract-service.ts index 8c1d87b4..17e46264 100644 --- a/src/services/extract-service.ts +++ b/src/services/extract-service.ts @@ -26,9 +26,14 @@ import { isSingletonType, isChildType } from '../lib/resource-path.js'; import { extractApiResources, ApiExtractionResult } from './api-extractor.js'; import { extractProductResources, ProductExtractionResult } from './product-extractor.js'; import { extractWorkspaces, WorkspaceExtractionResult } from './workspace-extractor.js'; -import { extractTransitiveDependencies } from './transitive-extractor.js'; +import { + apiDependencyResources, + collectPolicies, + extractTransitiveDependencies, +} from './transitive-extractor.js'; +import { readProductLinkDependencies } from './transitive-resolver.js'; import { redactAndWarnPolicySecrets } from './secret-redactor.js'; -import { shouldIncludeResource } from './filter-service.js'; +import { isExplicitlyExcluded, shouldIncludeResource } from './filter-service.js'; import { logger } from '../lib/logger.js'; import { buildResourceLabel } from '../lib/resource-uri.js'; import { EXIT_SUCCESS, EXIT_PARTIAL, EXIT_FATAL } from '../lib/exit-codes.js'; @@ -125,8 +130,9 @@ export async function runExtraction( await extractServicePolicy(client, store, service, outputDir, filter, result); // Phase 6: Transitive dependency resolution (if enabled) + let failedTransitive: ResourceDescriptor[] = []; if (config.includeTransitive && filter) { - await resolveAndExtractTransitive(client, store, service, outputDir, result); + failedTransitive = await resolveAndExtractTransitive(client, store, service, outputDir, filter, result); } // Phase 7: Extract workspace-scoped resources @@ -137,7 +143,8 @@ export async function runExtraction( outputDir, filter, config.includeTransitive, - result + result, + failedTransitive ); // Compute exit code @@ -285,13 +292,7 @@ async function extractApiSubResources( result.totalExtracted += subCount; // Collect policies for transitive resolution - for (let pi = 0; pi < apiResult.policies.length; pi++) { - const policyContent = apiResult.policies[pi]; - if (policyContent !== undefined) { - const key = `api:${apiName}:policy:${pi}`; - result.collectedPolicies.set(key, policyContent); - } - } + collectPolicies(result.collectedPolicies, 'api', apiName, apiResult.policies); // Track sub-resource descriptors for (const sub of [ @@ -382,13 +383,7 @@ async function extractProductSubResources( result.totalExtracted += subCount; // Collect policies for transitive resolution - for (let pi = 0; pi < prodResult.policies.length; pi++) { - const policyContent = prodResult.policies[pi]; - if (policyContent !== undefined) { - const key = `product:${productName}:policy:${pi}`; - result.collectedPolicies.set(key, policyContent); - } - } + collectPolicies(result.collectedPolicies, 'product', productName, prodResult.policies); } } } @@ -517,8 +512,9 @@ async function resolveAndExtractTransitive( store: IArtifactStore, context: ApimServiceContext, outputDir: string, + filter: FilterConfig, result: ExtractionResult -): Promise { +): Promise { logger.info('Resolving transitive dependencies...'); // Build maps for transitive resolution @@ -535,14 +531,16 @@ async function resolveAndExtractTransitive( } } - const resources = result.typeResults.flatMap((typeResult) => - typeResult.extracted + // Top-level resources, plus the API children that reference other + // resources (diagnostics and tag links). + const resources = [ + ...result.typeResults + .flatMap((typeResult) => typeResult.extracted) .filter((extracted) => extracted.status === 'success') - .map((extracted) => ({ - descriptor: extracted.descriptor, - json: extracted.json, - })) - ); + .map(({ descriptor, json }) => ({ descriptor, json })), + ...result.apiResults.flatMap(apiDependencyResources), + ]; + const linkedDependencies = await findProductLinkDependencies(store, outputDir, result); const transitiveResult = await extractTransitiveDependencies( client, store, @@ -551,11 +549,34 @@ async function resolveAndExtractTransitive( result.collectedPolicies, apiJsonMap, resources, - result.extractedDescriptors + result.extractedDescriptors, + { + linkedDependencies, + skip: (dep) => isExplicitlyExcluded(dep, filter), + } ); result.totalExtracted += transitiveResult.extractedDescriptors.length; result.totalErrors += transitiveResult.errorCount; result.extractedDescriptors.push(...transitiveResult.extractedDescriptors); + return transitiveResult.failedDescriptors; +} + +/** + * The groups and tags that extracted products link to, read back from the + * groups.json and tags.json files written for them. + */ +async function findProductLinkDependencies( + store: IArtifactStore, + outputDir: string, + result: ExtractionResult +): Promise { + const dependencies: ResourceDescriptor[] = []; + for (const typeResult of result.typeResults.filter((r) => r.type === ResourceType.Product)) { + for (const product of typeResult.extracted.filter((r) => r.status === 'success')) { + dependencies.push(...await readProductLinkDependencies(store, outputDir, product.descriptor)); + } + } + return dependencies; } /** @@ -568,10 +589,13 @@ async function extractWorkspaceResources( outputDir: string, filter: FilterConfig | undefined, includeTransitive: boolean, - result: ExtractionResult + result: ExtractionResult, + /** Dependencies the service-level transitive pass tried and failed to extract. */ + failedTransitive: ResourceDescriptor[] = [] ): Promise { const wsResults = await extractWorkspaces( - client, store, context, outputDir, filter, includeTransitive + client, store, context, outputDir, filter, includeTransitive, + [...result.extractedDescriptors, ...failedTransitive] ); result.workspaceResults = wsResults; diff --git a/src/services/filter-service.ts b/src/services/filter-service.ts index a73ca28b..64f81ade 100644 --- a/src/services/filter-service.ts +++ b/src/services/filter-service.ts @@ -68,6 +68,7 @@ const API_SUB_FILTER_KEY_MAP: Partial> [ResourceType.ApiDiagnostic]: 'diagnostics', [ResourceType.ApiSchema]: 'schemas', [ResourceType.ApiRelease]: 'releases', + [ResourceType.ApiTagDescription]: 'tagDescriptions', }; /** @@ -110,6 +111,15 @@ export function shouldIncludeResource( return false; } + // An API's link to a tag the filter excludes by name is left out with + // the tag, rather than extracted and then skipped on every publish. + if ( + descriptor.type === ResourceType.ApiTag && + isExcludedByName(ResourceType.Tag, getNamePart(descriptor.nameParts, 1), filter) + ) { + return false; + } + // Second check: does the parent have sub-resource filters? if (parentField === 'apis' && filter.apiSubFilters) { return matchesApiSubFilter(descriptor, parentName, filter.apiSubFilters); @@ -139,37 +149,24 @@ export function shouldReconcileResource( return false; } - const workspaceFilter = getWorkspaceFilter(descriptor.workspace, filter); - return shouldIncludeResource(descriptor, workspaceFilter); + return shouldIncludeResource(descriptor, resolveWorkspaceFilter(descriptor.workspace, filter)); } -function getWorkspaceFilter( - workspaceName: string, - filter?: FilterConfig -): FilterConfig | undefined { - const matchingKey = Object.keys(filter?.workspaceSubFilters ?? {}).find( - key => key.toLowerCase() === workspaceName.toLowerCase() - ); - if (!matchingKey) { - return undefined; +/** + * Whether `extract --remove-stale` manages this artifact, removing it when + * the extract does not write it again. As shouldReconcileResource, except + * that an API's tag links belong to the API even when the filter excludes + * the tag: extract doesn't write a link to an excluded tag, so one left by + * an earlier extract is removed with the API's other stale children. + */ +export function isInRemoveStaleScope(descriptor: ResourceDescriptor, filter?: FilterConfig): boolean { + if (descriptor.type === ResourceType.ApiTag) { + return shouldReconcileResource( + { type: ResourceType.Api, nameParts: [getNamePart(descriptor.nameParts, 0)], workspace: descriptor.workspace }, + filter + ); } - - const subFilter = filter?.workspaceSubFilters?.[matchingKey]; - return subFilter ? { - apis: subFilter.apis, - apiSubFilters: subFilter.apiSubFilters, - backends: subFilter.backends, - diagnostics: subFilter.diagnostics, - groups: subFilter.groups, - loggers: subFilter.loggers, - namedValues: subFilter.namedValues, - policyFragments: subFilter.policyFragments, - products: subFilter.products, - schemas: subFilter.schemas, - subscriptions: subFilter.subscriptions, - tags: subFilter.tags, - versionSets: subFilter.versionSets, - } : undefined; + return shouldReconcileResource(descriptor, filter); } /** @@ -180,11 +177,9 @@ export function resolveWorkspaceFilter( workspaceName: string, filter?: FilterConfig ): FilterConfig | undefined { - const lowerName = workspaceName.toLowerCase(); - const matchingKey = Object.keys(filter?.workspaceSubFilters ?? {}).find( - (key) => key.toLowerCase() === lowerName - ); - const subFilter = matchingKey ? filter?.workspaceSubFilters?.[matchingKey] : undefined; + // A sub-filter keyed by a pattern (for example `team-*`) applies to every + // workspace it matches; one keyed by the workspace's name takes precedence. + const subFilter = findSubFilter(workspaceName, filter?.workspaceSubFilters); return subFilter ? workspaceSubFilterToFilterConfig(subFilter) : undefined; } @@ -225,18 +220,12 @@ function matchesApiSubFilter( parentApiName: string, apiSubFilters: Record ): boolean { - // Find matching sub-filter using case-insensitive API name - const lowerParent = parentApiName.toLowerCase(); - const matchingKey = Object.keys(apiSubFilters).find( - (k) => k.toLowerCase() === lowerParent - ); - - if (!matchingKey) { + const subFilter = findApiSubFilter(parentApiName, apiSubFilters); + if (!subFilter) { // No sub-filter for this API — include all children return true; } - const subFilter = apiSubFilters[matchingKey]; const subFilterKey = API_SUB_FILTER_KEY_MAP[descriptor.type]; if (!subFilterKey) { @@ -295,6 +284,77 @@ export function extractRootApiName(name: string): string { return semiIndex >= 0 ? name.substring(0, semiIndex) : name; } +/** + * Whether the filter excludes this resource by name: its type's list has a + * `!pattern` entry that matches it. Dependency resolution respects these + * exclusions, while it still pulls in resources whose type is `[]` or that a + * list simply does not name. A workspace resource is checked against its + * workspace's sub-filter. + */ +export function isExplicitlyExcluded(descriptor: ResourceDescriptor, filter?: FilterConfig): boolean { + if (descriptor.nameParts.length === 0) { + return false; + } + const effectiveFilter = descriptor.workspace ? cachedWorkspaceFilter(descriptor.workspace, filter) : filter; + return isExcludedByName(descriptor.type, getNamePart(descriptor.nameParts, 0), effectiveFilter); +} + +/** Whether `filter` (already resolved for the resource's scope) excludes `name` of `type` by name. */ +function isExcludedByName(type: ResourceType, name: string, filter: FilterConfig | undefined): boolean { + const field = FILTER_FIELD_MAP[type]; + const list = field && filter ? (filter[field] as string[] | undefined) : undefined; + const exclusions = list?.filter((entry) => entry.startsWith('!')) ?? []; + return exclusions.length > 0 && !matchesFilter(name, exclusions); +} + +/** + * Resolved workspace filters, cached per filter object and workspace: the + * exclusion check runs for every reference during publish expansion. + */ +const workspaceFilterCache = new WeakMap>(); + +function cachedWorkspaceFilter(workspace: string, filter: FilterConfig | undefined): FilterConfig | undefined { + if (!filter) { + return undefined; + } + let byWorkspace = workspaceFilterCache.get(filter); + if (!byWorkspace) { + byWorkspace = new Map(); + workspaceFilterCache.set(filter, byWorkspace); + } + const key = workspace.toLowerCase(); + if (!byWorkspace.has(key)) { + byWorkspace.set(key, resolveWorkspaceFilter(workspace, filter)); + } + return byWorkspace.get(key); +} + +/** + * The sub-filter that applies to a named API or workspace: the one keyed by + * its name (case-insensitive), otherwise the first one keyed by a wildcard + * pattern that matches it (for example `*`). Exclusion keys (`!name`) never + * apply. + */ +function findSubFilter(name: string, subFilters: Record | undefined): T | undefined { + if (!subFilters) { + return undefined; + } + const lowerName = name.toLowerCase(); + const keys = Object.keys(subFilters).filter((key) => !key.startsWith('!')); + const key = + keys.find((k) => k.toLowerCase() === lowerName) ?? + keys.find((k) => isWildcardPattern(k) && matchesFilter(name, [k])); + return key === undefined ? undefined : subFilters[key]; +} + +/** The sub-filter that applies to an API (see {@link findSubFilter}). */ +export function findApiSubFilter( + apiName: string, + apiSubFilters: Record | undefined +): ApiSubFilter | undefined { + return findSubFilter(apiName, apiSubFilters); +} + /** * Check if a pattern contains wildcard characters (* or ?). */ diff --git a/src/services/product-extractor.ts b/src/services/product-extractor.ts index 97118722..fb7a098e 100644 --- a/src/services/product-extractor.ts +++ b/src/services/product-extractor.ts @@ -14,7 +14,9 @@ import { extractResourceName } from './resource-extractor.js'; import { redactAndWarnPolicySecrets } from './secret-redactor.js'; import { logger } from '../lib/logger.js'; import { getNamePart } from '../lib/resource-path.js'; -import { isWorkspaceScope, extractNameFromLink, extractLinkTarget } from '../lib/workspace-link.js'; +import { isWorkspaceScope, extractLinkTarget, listWorkspaceTagLinks } from '../lib/workspace-link.js'; +import { isExplicitlyExcluded } from './filter-service.js'; +import { associationTarget } from './transitive-resolver.js'; /** * Result of product-specific extraction for a single product. @@ -40,7 +42,11 @@ export async function extractProductResources( context: ApimServiceContext, productDescriptor: ResourceDescriptor, outputDir: string, - _filter?: FilterConfig, + /** + * The top-level filter (for a workspace product too): links to groups and + * tags it excludes by name are left out with them. + */ + filter?: FilterConfig, _workspace?: string ): Promise { const productName = getNamePart(productDescriptor.nameParts, 0); @@ -62,7 +68,7 @@ export async function extractProductResources( // Extract product group associations result.groups = await extractProductAssociations( - client, store, context, productDescriptor, outputDir, 'groups', () => result.errorCount++ + client, store, context, productDescriptor, outputDir, 'groups', () => result.errorCount++, filter ); // Extract product policy @@ -84,7 +90,7 @@ export async function extractProductResources( // in the workspace extractor. if (!isWorkspaceScope(context)) { result.tags = await extractProductTags( - client, store, context, productDescriptor, outputDir, () => result.errorCount++ + client, store, context, productDescriptor, outputDir, () => result.errorCount++, filter ); } @@ -101,9 +107,10 @@ async function extractProductAssociations( productDescriptor: ResourceDescriptor, outputDir: string, associationType: 'apis' | 'groups', - onError: () => void + onError: () => void, + filter?: FilterConfig ): Promise { - const entries: AssociationEntry[] = []; + let entries: AssociationEntry[] = []; const resourceType = associationType === 'apis' ? ResourceType.ProductApi : ResourceType.ProductGroup; @@ -137,6 +144,10 @@ async function extractProductAssociations( } } + if (associationType === 'groups') { + entries = withoutExcludedTargets(productDescriptor, 'groups', entries, filter); + } + // Write association file if (entries.length > 0) { await store.writeAssociation(outputDir, productDescriptor, associationType, entries); @@ -150,6 +161,20 @@ async function extractProductAssociations( return entries.map(entry => entry.name); } +/** + * A product's link entries without the ones to groups or tags the filter + * excludes by name: those are not extracted, so the links would only be + * skipped on every publish (as an API's links to excluded tags are left out). + */ +function withoutExcludedTargets( + product: ResourceDescriptor, + associationType: 'groups' | 'tags', + entries: AssociationEntry[], + filter: FilterConfig | undefined +): AssociationEntry[] { + return entries.filter((entry) => !isExplicitlyExcluded(associationTarget(product, associationType, entry), filter)); +} + /** * Extract product tags and write to artifact store as tags.json. * Only used in service scope; workspace scope is handled by extractWorkspaceProductTags(). @@ -160,9 +185,10 @@ async function extractProductTags( context: ApimServiceContext, productDescriptor: ResourceDescriptor, outputDir: string, - onError: () => void + onError: () => void, + filter?: FilterConfig ): Promise { - const entries: AssociationEntry[] = []; + let entries: AssociationEntry[] = []; const workspaceScoped = isWorkspaceScope(context); const linkProperty = RESOURCE_TYPE_METADATA[ResourceType.ProductTag].workspaceLinkIdProperty; @@ -188,6 +214,8 @@ async function extractProductTags( } } + entries = withoutExcludedTargets(productDescriptor, 'tags', entries, filter); + // Write tags association file if (entries.length > 0) { await store.writeAssociation(outputDir, productDescriptor, 'tags', entries); @@ -294,38 +322,15 @@ export async function extractWorkspaceProductTags( outputDir: string, _workspace: string ): Promise { - const linkProperty = RESOURCE_TYPE_METADATA[ResourceType.ProductTag].workspaceLinkIdProperty; - if (!linkProperty) { - return 0; - } - // Build a map of product name → tags for that product const productTagsMap = new Map(); - - for (const tagName of extractedTagNames) { - // Do NOT set workspace on this descriptor — context.baseUrl already includes - // the workspace prefix and buildArmUri would double it. - const tagDescriptor: ResourceDescriptor = { - type: ResourceType.Tag, - nameParts: [tagName], - }; - - try { - for await (const linkJson of client.listResources(context, ResourceType.ProductTag, tagDescriptor)) { - const productName = extractNameFromLink(linkJson, linkProperty); - if (!productName) { - throw new Error(`Failed to extract product name from tag "${tagName}" productLink response`); - } - - if (!productTagsMap.has(productName)) { - productTagsMap.set(productName, []); - } - productTagsMap.get(productName)!.push(tagName); - } - } catch (error) { - logger.warn(`Failed to list productLinks for tag "${tagName}": ${(error as Error).message}`); - throw error; + for await (const { tagName, targetName: productName } of listWorkspaceTagLinks( + client, context, extractedTagNames, ResourceType.ProductTag + )) { + if (!productTagsMap.has(productName)) { + productTagsMap.set(productName, []); } + productTagsMap.get(productName)!.push(tagName); } // Write tags.json for each product that has tag associations diff --git a/src/services/product-publisher.ts b/src/services/product-publisher.ts index 87b0e371..66a20352 100644 --- a/src/services/product-publisher.ts +++ b/src/services/product-publisher.ts @@ -23,6 +23,7 @@ import { parseArmUri } from '../lib/resource-uri.js'; import { isWorkspaceScope, buildLinkPayload } from '../lib/workspace-link.js'; import { isLinkAlreadyExistsError } from '../clients/apim-client.js'; import { mapDescriptor, toDeployedName } from './env-mapper.js'; +import { armNameAfter } from '../lib/arm-id.js'; export type ProductAssociationPublicationPlan = AssociationPublicationPlan; @@ -147,7 +148,7 @@ async function cleanupProductSubscriptions( if ( typeof name !== 'string' || typeof scope !== 'string' || - getScopedProductName(scope)?.toLowerCase() !== productNameLower + armNameAfter(scope, 'products')?.toLowerCase() !== productNameLower ) { continue; } @@ -170,16 +171,6 @@ async function cleanupProductSubscriptions( } } -function getScopedProductName(scope: string): string | undefined { - const segment = scope.match(/\/products\/([^/]+)\/?$/i)?.[1]; - if (!segment) return undefined; - try { - return decodeURIComponent(segment); - } catch { - return segment; - } -} - async function cleanupProductGroups( client: IApimClient, context: ApimServiceContext, @@ -291,7 +282,7 @@ export async function planProductAssociationPublications( store, targetDescriptor, config, - allowedDescriptors + { allowedDescriptors } ); const deployedDescriptor = config.envMapping ? mapDescriptor(descriptor, config.envMapping) diff --git a/src/services/publish-service.ts b/src/services/publish-service.ts index 6d1e966a..0f7847ff 100644 --- a/src/services/publish-service.ts +++ b/src/services/publish-service.ts @@ -29,6 +29,7 @@ import { } from '../lib/resource-path.js'; import { extractRootApiName, + isExplicitlyExcluded, resolveWorkspaceFilter, shouldIncludeResource, shouldReconcileResource, @@ -50,9 +51,17 @@ import { } from './delete-unmatched-service.js'; import { computeGitDiff } from './git-diff-service.js'; import { scanForRedactionMarkers } from './secret-redaction-guard.js'; -import { hasNamedValueOverride } from './override-merger.js'; +import { armNameAfter } from '../lib/arm-id.js'; +import { applyOverrides, hasNamedValueOverride } from './override-merger.js'; import { REDACTION_MARKER } from './secret-redactor.js'; -import { scanArtifactReferences } from './transitive-resolver.js'; +import { + createArtifactReferenceResolver, + createNamedValueReader, + findTransitiveDependencies, + indexNamedValueTargetTypes, + namedValueTargetName, + scanArtifactReferences, +} from './transitive-resolver.js'; import { validateAndBuildEnvMapping } from './env-mapping-validator.js'; import { mapDescriptor } from './env-mapper.js'; @@ -124,10 +133,15 @@ async function runPublishPipeline( // Step 0: Pre-flight validation — ensure the resource group and APIM service exist await client.validatePreFlight(config.service); + // The named values policies use as resource names, read from the source + // once for this run when the expansion or the Tier 1 order first needs it. + const namedValueTargets = createNamedValueTargetIndex(store, config); + // Step 1: Determine target descriptors based on incremental mode const { targetDescriptors, deletedDescriptors } = await determinePublishTargets( store, - config + config, + namedValueTargets ); // Step 1a: For env-mapping validation and policy ref rewriting we need @@ -220,7 +234,8 @@ async function runPublishPipeline( store, config.service, config, - targetDescriptors + targetDescriptors, + namedValueTargets ); // Step 4: Execute DELETEs in reverse dependency order (tier 4 → tier 1) if requested @@ -279,7 +294,8 @@ async function runPublishPipeline( */ async function determinePublishTargets( store: IArtifactStore, - config: PublishConfig + config: PublishConfig, + namedValueTargets: NamedValueTargetIndex ): Promise { let targetDescriptors: ResourceDescriptor[]; let deletedDescriptors: ResourceDescriptor[]; @@ -309,23 +325,87 @@ async function determinePublishTargets( config.sourceDir, targetDescriptors, availableDescriptors, - config.filter + config.filter, + config.overrides, + namedValueTargets ); } return { targetDescriptors, deletedDescriptors }; } +/** + * The resource types each named value in the source is used for as a resource + * name (indexNamedValueTargetTypes over every policy and policy fragment): + * built on first use and then shared, so a publish run reads the policies once + * for both the transitive expansion and the Tier 1 order. + */ +type NamedValueTargetIndex = () => Promise>>; + +function createNamedValueTargetIndex(store: IArtifactStore, config: PublishConfig): NamedValueTargetIndex { + let index: Promise>> | undefined; + return () => { + index ??= store.listResources(config.sourceDir).then((available) => + indexNamedValueTargetTypes( + store, + config.sourceDir, + available, + available, + createArtifactReferenceResolver(available, (descriptor) => store.readResource(config.sourceDir, descriptor)), + // As published: an override can replace a policy's or fragment's XML. + (descriptor, json) => applyOverrides(descriptor, json, config.overrides) + ) + ); + return index; + }; +} + async function expandTransitivePublishTargets( store: IArtifactStore, sourceDir: string, initialDescriptors: ResourceDescriptor[], availableDescriptors: ResourceDescriptor[], - filter: FilterConfig + filter: FilterConfig, + overrides: OverrideConfig | undefined, + namedValueTargets: NamedValueTargetIndex ): Promise { - const availableByKey = new Map( - availableDescriptors.map((descriptor) => [getResourceDescriptorKey(descriptor), descriptor]) + const findAvailable = createArtifactReferenceResolver( + availableDescriptors, + (descriptor) => store.readResource(sourceDir, descriptor) ); + // A named value as it is published: with this environment's overrides, + // which can name a different resource. One that can't be read only hides + // what a policy attribute set to it names; publishing the named value + // reports the error. One the filter excludes isn't published, so neither + // is what it names. + const readNamedValueJson = async (descriptor: ResourceDescriptor) => { + if (isExplicitlyExcluded(descriptor, filter)) { + return undefined; + } + try { + const json = await store.readResource(sourceDir, descriptor); + return json ? applyOverrides(descriptor, json, overrides) : undefined; + } catch (error) { + logger.warn(`Unable to read ${buildResourceLabel(descriptor)} to find what policies set to it name: ${String(error)}`); + return undefined; + } + }; + const readNamedValue = createNamedValueReader(findAvailable, readNamedValueJson); + // A published named value needs the resource its value names when any + // policy or fragment in the source uses it as a resource name + // (backend-id="{{name}}"), even one the filter leaves out: APIM checks every + // policy on the target that uses a named value when the named value + // changes, and rejects the change while the resource it names is missing. + // The policies are read once per run, on the first named value in the + // publish set. + const findNamedValueTargets = async (namedValue: ResourceDescriptor): Promise => { + const types = (await namedValueTargets()).get(getResourceDescriptorKey(namedValue)); + const json = types ? await readNamedValueJson(namedValue) : undefined; + const name = json ? namedValueTargetName(json) : undefined; + return types && name + ? [...types].map((type) => ({ type, nameParts: [name], workspace: namedValue.workspace })) + : []; + }; const expanded = [...initialDescriptors]; const included = new Set(expanded.map(getResourceDescriptorKey)); const scanQueue = [...initialDescriptors]; @@ -377,7 +457,15 @@ async function expandTransitivePublishTargets( for (let index = 0; index < scanQueue.length; index++) { let references: ResourceDescriptor[]; try { - references = await scanArtifactReferences(store, sourceDir, scanQueue[index]); + // As published: an override can change what a backend's or logger's + // credentials, a diagnostic's logger or a policy's XML reference. + references = await scanArtifactReferences( + store, sourceDir, scanQueue[index], readNamedValue, + (descriptor, json) => applyOverrides(descriptor, json, overrides) + ); + if (scanQueue[index].type === ResourceType.NamedValue) { + references.push(...await findNamedValueTargets(scanQueue[index])); + } } catch (error) { logger.warn( `Unable to scan transitive references for ${scanQueue[index].type} "${scanQueue[index].nameParts.join('/')}": ${String(error)}` @@ -385,8 +473,12 @@ async function expandTransitivePublishTargets( continue; } for (const reference of references) { - const available = availableByKey.get(getResourceDescriptorKey(reference)); - if (available && !included.has(getResourceDescriptorKey(available))) { + const available = await findAvailable(reference); + if ( + available && + !included.has(getResourceDescriptorKey(available)) && + !isExplicitlyExcluded(available, filter) + ) { included.add(getResourceDescriptorKey(available)); expanded.push(available); scanQueue.push(available); @@ -447,7 +539,8 @@ async function executePuts( store: IArtifactStore, context: ApimServiceContext, config: PublishConfig, - targetDescriptors: ResourceDescriptor[] + targetDescriptors: ResourceDescriptor[], + namedValueTargets: NamedValueTargetIndex ): Promise { const results: PublishActionResult[] = []; @@ -501,33 +594,65 @@ async function executePuts( await publishAndOutput(client, store, context, config, workspaces, targetDescriptors, results); } - // Within Tier 1 we publish in three ordered waves to satisfy implicit + // Within Tier 1 we publish in ordered waves to satisfy implicit // runtime dependencies: // - // Wave 1 — NamedValues only. + // Wave 1 — Named values, except those in wave 4. // Loggers and backends may reference named values by name in their // configuration (e.g. AppInsights instrumentation key stored as a // named value). Publishing named values first ensures they exist // before any resource that references them. // - // Wave 2 — All remaining Tier 1 resources except pool backends. - // Loggers, tags, gateways, etc. can now reference named values safely. + // Wave 2 — The remaining Tier 1 resources except pool backends and + // wave 4 (and except policy fragments when wave 4 has any). // // Wave 3 — Pool backends. // Pool backends reference individual Backend resources and must be // published after the backends they aggregate (see pool backend // ordering comments in splitPoolBackends). - if (namedValues.length > 0) { - logger.debug(`Publishing ${namedValues.length} named value(s) first (wave 1 of tier 1)`); - await publishAndOutput(client, store, context, config, namedValues, targetDescriptors, results); + // + // Wave 4 — Named values a policy uses as a resource name + // (backend-id="{{name}}") whose value names a backend, logger or + // schema published here, after that resource: APIM checks the + // policies that use a named value when it changes, so that resource + // must exist first. With them, in an order where each goes after what + // it needs, the backends and loggers whose credentials use them and + // the pools those are members of (see + // splitNamedValuesNamingResources). + // + // Wave 5 — Policy fragments, when wave 4 has resources: a fragment can + // use its named values, and APIM rejects a policy that uses a + // missing one. + const { early: earlyNamedValues, ordered } = await splitNamedValuesNamingResources( + store, + config, + namedValues, + otherTier1, + namedValueTargets + ); + if (earlyNamedValues.length > 0) { + logger.debug(`Publishing ${earlyNamedValues.length} named value(s) first (wave 1 of tier 1)`); + await publishAndOutput(client, store, context, config, earlyNamedValues, targetDescriptors, results); } + const inOrdered = new Set(ordered.flat()); const { poolBackends, regularTier1 } = await splitPoolBackends( store, config.sourceDir, - otherTier1 + otherTier1.filter((descriptor) => !inOrdered.has(descriptor)) + ); + const fragments = inOrdered.size > 0 + ? regularTier1.filter((descriptor) => descriptor.type === ResourceType.PolicyFragment) + : []; + await publishAndOutput( + client, + store, + context, + config, + regularTier1.filter((descriptor) => !fragments.includes(descriptor)), + targetDescriptors, + results ); - await publishAndOutput(client, store, context, config, regularTier1, targetDescriptors, results); if (poolBackends.length > 0) { logger.debug( `Publishing ${poolBackends.length} pool backend(s) after regular backends` @@ -542,6 +667,17 @@ async function executePuts( results ); } + if (inOrdered.size > 0) { + logger.debug( + `Publishing ${inOrdered.size} named value(s) and resource(s) in the order the named values' values need` + ); + } + for (const wave of ordered) { + await publishAndOutput(client, store, context, config, wave, targetDescriptors, results); + } + if (fragments.length > 0) { + await publishAndOutput(client, store, context, config, fragments, targetDescriptors, results); + } } else if (tier === 2) { const tier2Descriptors = filterApiRevisionsHandledByRootApis(descriptors); if (tier2Descriptors.length === 0) continue; @@ -753,6 +889,175 @@ function splitNamedValues( return { namedValues, otherTier1 }; } +/** + * The Tier 1 order around named values a policy or policy fragment in the + * source uses as the whole value of a reference attribute + * (backend-id="{{name}}", logger-id, schema-id), and whose value (with + * overrides) names a resource of that type published in this tier. APIM + * checks the policies that use a named value when the named value changes, + * rejecting the change ("its value breaks policies") while the resource it + * now names doesn't exist, so such a named value goes after that resource. + * + * `early` named values go before the rest of Tier 1. `ordered` is published + * in turn after it, wave by wave: the late named values, the backends and + * loggers whose credentials use them (after them), the resources those name + * or pools those are members of, in an order where each comes after what it + * needs. Where there is none (a cycle, as when a backend uses in its + * credentials the named value that names it), the named values in it go + * early instead. `ordered` resources other than named values are not + * published in the earlier waves. + * + * The policies are read (once) only when a named value's value matches the + * name of such a resource here; a named value that can't be read goes first, + * and publishing it reports the error. + */ +async function splitNamedValuesNamingResources( + store: IArtifactStore, + config: PublishConfig, + namedValues: ResourceDescriptor[], + otherTier1: ResourceDescriptor[], + namedValueTargets: NamedValueTargetIndex +): Promise<{ early: ResourceDescriptor[]; ordered: ResourceDescriptor[][] }> { + const resourceKey = (type: ResourceType, name: string, workspace: string | undefined) => + `${type}/${workspace ?? ''}/${name.toLowerCase()}`; + const keyOf = (descriptor: ResourceDescriptor) => + resourceKey(descriptor.type, getNamePart(descriptor.nameParts, 0), descriptor.workspace); + const byResourceKey = new Map(otherTier1.map((descriptor) => [keyOf(descriptor), descriptor])); + const named = new Set( + otherTier1.filter((descriptor) => NAMED_THROUGH_NAMED_VALUE_TYPES.has(descriptor.type)).map(keyOf) + ); + const early: ResourceDescriptor[] = []; + const late: ResourceDescriptor[] = []; + if (named.size === 0) { + return { early: namedValues, ordered: [] }; + } + // The resources each late named value names, which go before it. + const targetsOf = new Map(); + // The named values policies use as resource names, and as which types. + // Every policy in the source counts, not only the ones the filter selects: + // APIM checks every policy on the target that uses the named value, and + // this only decides the order, not what is published. + const typesFor = async (namedValue: ResourceDescriptor): Promise | undefined> => + (await namedValueTargets()).get(getResourceDescriptorKey(namedValue)); + for (const descriptor of namedValues) { + let targets: string[] = []; + try { + const json = await store.readResource(config.sourceDir, descriptor); + const name = json ? namedValueTargetName(applyOverrides(descriptor, json, config.overrides)) : undefined; + if (name && [...NAMED_THROUGH_NAMED_VALUE_TYPES].some((type) => named.has(resourceKey(type, name, descriptor.workspace)))) { + targets = [...(await typesFor(descriptor) ?? [])] + .map((type) => resourceKey(type, name, descriptor.workspace)) + .filter((target) => named.has(target)); + } + } catch { + targets = []; + } + if (targets.length > 0) { + late.push(descriptor); + targetsOf.set(descriptor, targets); + } else { + early.push(descriptor); + } + } + if (late.length === 0) { + return { early, ordered: [] }; + } + + // The backends and loggers whose own configuration (their credentials) + // uses a late named value, and the pools each backend is a member of. + const available = await store.listResources(config.sourceDir); + const resolve = createArtifactReferenceResolver(available, (descriptor) => store.readResource(config.sourceDir, descriptor)); + const lateByKey = new Map(late.map((descriptor) => [getResourceDescriptorKey(descriptor), descriptor])); + const usesOf = new Map(); + const poolsOf = new Map(); + for (const descriptor of otherTier1) { + if (descriptor.type !== ResourceType.Backend && descriptor.type !== ResourceType.Logger) { + continue; + } + let json: Record | undefined; + try { + json = await store.readResource(config.sourceDir, descriptor); + } catch { + continue; + } + if (!json) { + continue; + } + const resource = { descriptor, json: applyOverrides(descriptor, json, config.overrides) }; + const uses: ResourceDescriptor[] = []; + for (const dependency of findTransitiveDependencies(new Map(), new Map(), descriptor.workspace, [resource])) { + const target = dependency.type === ResourceType.NamedValue ? await resolve(dependency) : undefined; + const used = target ? lateByKey.get(getResourceDescriptorKey(target)) : undefined; + if (used) { + uses.push(used); + } + } + if (uses.length > 0) { + usesOf.set(descriptor, uses); + } + const pool = (resource.json.properties as Record | undefined)?.pool as Record | undefined; + for (const service of Array.isArray(pool?.services) ? pool.services : []) { + const id = (service as Record | undefined)?.id; + const member = typeof id === 'string' ? armNameAfter(id, 'backends') : undefined; + if (member) { + const memberKey = resourceKey(ResourceType.Backend, member, descriptor.workspace); + poolsOf.set(memberKey, [...(poolsOf.get(memberKey) ?? []), descriptor]); + } + } + } + + // What is ordered: the late named values, their credential users, and the + // pools those users are members of (a pool goes after its members). + const nodes = new Set([...late, ...usesOf.keys()]); + for (const node of [...nodes]) { + if (node.type === ResourceType.Backend) { + (poolsOf.get(keyOf(node)) ?? []).forEach((pool) => nodes.add(pool)); + } + } + // Each node's predecessors among the nodes. + const before = new Map>([...nodes].map((node) => [node, new Set()])); + for (const [user, uses] of usesOf) { + uses.forEach((used) => before.get(user)?.add(used)); + } + for (const [namedValue, targets] of targetsOf) { + for (const target of targets) { + const resource = byResourceKey.get(target); + if (resource && nodes.has(resource)) { + before.get(namedValue)?.add(resource); + } + } + } + for (const node of nodes) { + if (node.type === ResourceType.Backend) { + (poolsOf.get(keyOf(node)) ?? []).forEach((pool) => before.get(pool)?.add(node)); + } + } + + const ordered: ResourceDescriptor[][] = []; + const remaining = new Set(nodes); + while (remaining.size > 0) { + const wave = [...remaining].filter((node) => [...(before.get(node) ?? [])].every((needed) => !remaining.has(needed))); + if (wave.length === 0) { + // A cycle: its named values go early, before what they name. + for (const node of [...remaining].filter((candidate) => candidate.type === ResourceType.NamedValue)) { + remaining.delete(node); + early.push(node); + } + continue; + } + wave.forEach((node) => remaining.delete(node)); + ordered.push(wave); + } + return { early, ordered }; +} + +/** Tier 1 types a policy attribute can name through a named value. */ +const NAMED_THROUGH_NAMED_VALUE_TYPES: ReadonlySet = new Set([ + ResourceType.Backend, + ResourceType.Logger, + ResourceType.GlobalSchema, +]); + /** * Separates Backend descriptors that are pool backends (properties.type === "Pool") * from all other descriptors in the supplied list. diff --git a/src/services/resource-publisher.ts b/src/services/resource-publisher.ts index fe9f3729..2c3c98a3 100644 --- a/src/services/resource-publisher.ts +++ b/src/services/resource-publisher.ts @@ -15,7 +15,7 @@ import type { ResourceDescriptor, } from '../models/types.js'; import type { PublishConfig } from '../models/config.js'; -import { ResourceType, RESOURCE_TYPE_METADATA, MANAGED_GATEWAY_NAME } from '../models/resource-types.js'; +import { isBuiltInGroup, ResourceType, RESOURCE_TYPE_METADATA, MANAGED_GATEWAY_NAME } from '../models/resource-types.js'; import { applyOverrides, resolveOverrideSection } from './override-merger.js'; import { checkKeyVaultSecretAccess } from './keyvault-checker.js'; import { @@ -30,12 +30,13 @@ import { REDACTION_MARKER } from './secret-redactor.js'; import { isAssociationReferenceNotFoundError, isLinkAlreadyExistsError } from '../clients/apim-client.js'; import type { OverrideConfig, OverrideSection } from '../models/config.js'; import { buildResourceLabel } from '../lib/resource-uri.js'; -import { hasExplicitTypeFilter, resolveWorkspaceFilter, shouldIncludeResource } from './filter-service.js'; +import { hasExplicitTypeFilter, isExplicitlyExcluded, resolveWorkspaceFilter, shouldIncludeResource } from './filter-service.js'; import { findSubscriptionTargets } from './transitive-resolver.js'; import { mapDescriptor, toDeployedName } from './env-mapper.js'; import type { EnvMapping } from './env-mapper.js'; import { rewritePolicyRefs } from './policy-ref-rewriter.js'; import type { KnownArtifactSets } from '../models/config.js'; +import { armLastName, armNameAfter, armScope } from '../lib/arm-id.js'; export type { KnownArtifactSets } from '../models/config.js'; @@ -259,7 +260,9 @@ export async function publishResource( const eligibility = await evaluateResourceEligibility( store, descriptor, - config + config, + undefined, + allowedDescriptors ); if (!eligibility.eligible) { logger.warn( @@ -735,12 +738,9 @@ async function buildAssociationPublicationPlan( nameParts: [entry.name], workspace: entry.scope === 'service' ? undefined : workspace, }; - const eligibility = await evaluateAssociationEligibility( - store, - target, - config, - allowedDescriptors - ); + const eligibility = isAdministratorsLink(target, config) + ? { eligible: true } + : await evaluateAssociationEligibility(store, target, config, { allowedDescriptors }); const metadata = RESOURCE_TYPE_METADATA[type]; const workspaceScoped = !!workspace || isWorkspaceScope(context); const segment = target.type === ResourceType.Group ? 'groups' : 'apis'; @@ -764,16 +764,55 @@ async function buildAssociationPublicationPlan( return { descriptor, deployedDescriptor, target, payload, ...eligibility }; } +/** + * Whether a product's group link targets the built-in `administrators` group, + * which APIM links to every product itself (publish removes the links APIM + * adds to a new product, then publishes the extracted ones). The link is kept + * whatever the groups filter says, unless the filter excludes the group by + * name, so a filter that narrows groups neither drops it nor warns about it. + */ +function isAdministratorsLink(target: ResourceDescriptor, config: PublishConfig): boolean { + return ( + isBuiltInGroup(target) && + target.nameParts[0]?.toLowerCase() === 'administrators' && + !(config.filter && isExplicitlyExcluded(target, config.filter)) + ); +} + +export interface AssociationEligibilityOptions { + /** + * The publish set, when the caller has one. Besides marking targets as + * published, it decides availability for targets the filter lists. + */ + allowedDescriptors?: ResourceDescriptor[]; + /** + * Resources being published, used only to accept targets in it (defaults + * to allowedDescriptors). The ApiTag check passes this without + * allowedDescriptors, so an incremental publish still accepts a tag that + * exists on disk but is not in the diff. + */ + publishSet?: ResourceDescriptor[]; +} + export async function evaluateAssociationEligibility( store: IArtifactStore, target: ResourceDescriptor, config: PublishConfig, - allowedDescriptors?: ResourceDescriptor[] + { allowedDescriptors, publishSet = allowedDescriptors }: AssociationEligibilityOptions = {} ): Promise { if (!config.filter) { return { eligible: true }; } + // A target being published is linked even when the filter leaves its type + // out, because dependency resolution pulled it in (a tag or group). + if ( + !isExplicitlyExcluded(target, config.filter) && + publishSet?.some((included) => sameResourceDescriptor(included, target)) + ) { + return { eligible: true }; + } + const effectiveFilter = target.workspace ? resolveWorkspaceFilter(target.workspace, config.filter) : config.filter; @@ -782,12 +821,6 @@ export async function evaluateAssociationEligibility( } if (allowedDescriptors) { - if ( - allowedDescriptors.some((allowed) => sameResourceDescriptor(allowed, target)) - ) { - return { eligible: true }; - } - if (config.commitId) { const exists = (await store.readResource(config.sourceDir, target)) !== undefined; return exists @@ -860,9 +893,9 @@ export async function resolveAssociationDeleteDescriptor( if ( typeof targetId !== 'string' || typeof linkName !== 'string' || - getArmResourceName(targetId).toLowerCase() !== expectedTarget.toLowerCase() || + (armLastName(targetId) ?? '').toLowerCase() !== expectedTarget.toLowerCase() || (descriptor.targetScope !== undefined && - getAssociationTargetScope(targetId) !== descriptor.targetScope) + armScope(targetId) !== descriptor.targetScope) ) { continue; } @@ -877,27 +910,15 @@ export async function resolveAssociationDeleteDescriptor( }; } - function getAssociationTargetScope(resourceId: string): 'service' | 'workspace' { - return /\/workspaces\/[^/]+\//i.test(resourceId) ? 'workspace' : 'service'; - } - return undefined; } -function getArmResourceName(resourceId: string): string { - const segment = resourceId.split('/').filter(Boolean).at(-1) ?? ''; - try { - return decodeURIComponent(segment); - } catch { - return segment; - } -} - export async function evaluateResourceEligibility( store: IArtifactStore, descriptor: ResourceDescriptor, config: PublishConfig, - json?: Record + json?: Record, + publishSet?: ResourceDescriptor[] ): Promise { if (descriptor.type === ResourceType.ApiTag) { const target: ResourceDescriptor = { @@ -905,7 +926,7 @@ export async function evaluateResourceEligibility( nameParts: [getNamePart(descriptor.nameParts, 1)], workspace: descriptor.workspace, }; - return evaluateAssociationEligibility(store, target, config); + return evaluateAssociationEligibility(store, target, config, { publishSet }); } if (descriptor.type === ResourceType.ApiOperation) { @@ -1332,7 +1353,7 @@ export function normalizeDiagnosticLoggerId( return json; } - const loggerName = getArmResourceName(loggerId); + const loggerName = armLastName(loggerId); if (!loggerName) { return json; } @@ -1385,7 +1406,7 @@ export function normalizeApiVersionSetId( return json; } - const versionSetName = getArmResourceName(versionSetId); + const versionSetName = armLastName(versionSetId); if (!versionSetName) { return json; } @@ -1432,19 +1453,15 @@ function normalizeBackendPoolServices( return typedService; } - const workspaceMatch = id.match(/\/workspaces\/([^/]+)\/backends\/([^/]+)$/i); - const serviceMatch = id.match(/\/backends\/([^/]+)$/i); - const backendName = getArmResourceName(id); - if (!backendName || (!workspaceMatch && !serviceMatch)) { + const backendName = armNameAfter(id, 'backends'); + if (!backendName) { return typedService; } const deployedBackend = envMapping ? toDeployedName(backendName, ResourceType.Backend, envMapping) : backendName; - const sourceWorkspace = workspaceMatch?.[1] - ? getArmResourceName(`/workspaces/${workspaceMatch[1]}`) - : undefined; + const sourceWorkspace = armNameAfter(id, 'workspaces'); const deployedWorkspace = sourceWorkspace ? envMapping ? toDeployedName(sourceWorkspace, ResourceType.Workspace, envMapping) @@ -1651,7 +1668,9 @@ async function publishWorkspaceApiTagLink( * rewrites the reference. * * References that already use displayName (or don't match any named value - * artifact) are left unchanged. + * artifact) are left unchanged. A reference is matched as APIM matches it, + * by display name first, so one that is another named value's display name + * stays a reference to that named value. */ async function rewriteNamedValueReferences( json: Record, @@ -1671,16 +1690,36 @@ async function rewriteNamedValueReferences( const credStr = JSON.stringify(credentials); const refPattern = /\{\{([^}]+)\}\}/g; const rewriteMap = new Map(); + // The named values in the logger's scope by display name (lower case), + // read on the first reference. + let displayNames: Promise> | undefined; + const readDisplayNames = async (): Promise> => { + const byDisplayName = new Map(); + const namedValues = (await store.listResources(sourceDir)).filter( + (candidate) => candidate.type === ResourceType.NamedValue && candidate.workspace === descriptor.workspace + ); + for (const namedValue of namedValues) { + const nvJson = await store.readResource(sourceDir, namedValue).catch(() => undefined); + const displayName = nvJson + ? (applyOverrides(namedValue, nvJson, overrides).properties as Record | undefined)?.displayName + : undefined; + if (typeof displayName === 'string') { + byDisplayName.set(displayName.toLowerCase(), getNamePart(namedValue.nameParts, 0)); + } + } + return byDisplayName; + }; let match; while ((match = refPattern.exec(credStr)) !== null) { const ref = match[1]; if (rewriteMap.has(ref)) continue; - - // Try to read the named value artifact by this ref as a resource name + // The named value APIM resolves the reference to: the one with that + // display name, otherwise the one with that resource name. + const nvName = (await (displayNames ??= readDisplayNames())).get(ref.toLowerCase()) ?? ref; const nvDescriptor: ResourceDescriptor = { type: ResourceType.NamedValue, - nameParts: [ref], + nameParts: [nvName], ...(descriptor.workspace ? { workspace: descriptor.workspace } : {}), }; let nvJson = await store.readResource(sourceDir, nvDescriptor); @@ -1694,12 +1733,12 @@ async function rewriteNamedValueReferences( // use the affixed displayName so {{token}} refs resolve on the shared APIM. if (envMapping?.appliesTo.has(ResourceType.NamedValue)) { const hasDisplayNameOverride = hasExplicitPropertyOverride( - ref, + nvName, 'displayName', resolveOverrideSection(overrides, 'namedValues', descriptor.workspace), ); if (!hasDisplayNameOverride) { - const baseDisplayName = displayName ?? ref; + const baseDisplayName = displayName ?? nvName; displayName = toDeployedName(baseDisplayName, ResourceType.NamedValue, envMapping); } } diff --git a/src/services/stale-dependency-finder.ts b/src/services/stale-dependency-finder.ts new file mode 100644 index 00000000..2e632059 --- /dev/null +++ b/src/services/stale-dependency-finder.ts @@ -0,0 +1,181 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +/** + * Stale pulled-in dependencies for `extract --remove-stale`. + * + * A filtered extract pulls in resources its selection depends on (tags, + * groups, loggers, named values and so on) even when the filter leaves their + * type out. Those artifacts are outside the filter's scope, so --remove-stale + * does not remove them as it does selected resources. Other filtered extracts + * into the same output can depend on them too, so one is only stale when it + * was a dependency of this filter's selection and nothing left in the output + * still references it. + */ + +import type { IArtifactStore } from '../clients/iartifact-store.js'; +import type { FilterConfig } from '../models/config.js'; +import type { ResourceDescriptor } from '../models/types.js'; +import { getResourceDescriptorKey } from '../lib/resource-path.js'; +import { buildResourceLabel } from '../lib/resource-uri.js'; +import { logger } from '../lib/logger.js'; +import { runParallel } from '../lib/parallel-runner.js'; +import { isInRemoveStaleScope } from './filter-service.js'; +import { + createArtifactReferenceResolver, + createNamedValueReader, + REFERENCE_SCAN_CONCURRENCY, + scanArtifactReferences, + type NamedValueReader, +} from './transitive-resolver.js'; + +interface ArtifactLocation { + descriptor: ResourceDescriptor; + /** The directory the artifact is read from: staging for staged ones. */ + dir: string; +} + +/** + * The references in each artifact, read in parallel. An artifact that can't + * be read is reported in `failed` rather than failing the scan. + */ +async function scanAll( + store: IArtifactStore, + artifacts: ReadonlyArray, + readNamedValue: NamedValueReader +): Promise<{ references: ResourceDescriptor[]; failed: ResourceDescriptor[] }> { + const results = await runParallel( + artifacts.map(({ descriptor, dir }) => () => scanArtifactReferences(store, dir, descriptor, readNamedValue)), + REFERENCE_SCAN_CONCURRENCY + ); + const references: ResourceDescriptor[] = []; + const failed: ResourceDescriptor[] = []; + results.forEach((result, index) => { + if (result.status === 'fulfilled') { + references.push(...(result.value ?? [])); + } else { + logger.warn(`Unable to scan ${buildResourceLabel(artifacts[index].descriptor)} for references: ${String(result.reason)}`); + failed.push(artifacts[index].descriptor); + } + }); + return { references, failed }; +} + +/** + * The artifacts in `outputDir` that this extract's selection used to depend + * on (directly or through other pulled-in artifacts), that this extract did + * not write again, and that no artifact left after committing it references. + * + * Artifacts left in the output are scanned, in parallel, until every + * candidate is known to be needed; this only runs when there is a candidate. + * If an artifact left in the output can't be scanned, or a named value left + * there can't be read to match {{references}} by display name, nothing is + * reported stale, since it could reference any candidate. + * + * @param outputDir - The existing output + * @param stagingDir - This extract's staged output + * @param filter - The filter the extract ran with + */ +export async function findStaleDependencies( + store: IArtifactStore, + outputDir: string, + stagingDir: string, + filter: FilterConfig +): Promise { + const existing = await store.listResources(outputDir); + const staged = await store.listResources(stagingDir); + const key = getResourceDescriptorKey; + const stagedKeys = new Set(staged.map(key)); + // In step with what --remove-stale removes when not written again. + const isSelected = (descriptor: ResourceDescriptor): boolean => isInRemoveStaleScope(descriptor, filter); + // Only an artifact outside the selection that this extract did not write + // again can be stale; often there is none, and nothing needs reading. + const kept = existing.filter((descriptor) => !stagedKeys.has(key(descriptor)) && !isSelected(descriptor)); + if (kept.length === 0) { + return []; + } + + // What the selection depended on before this extract, followed through + // pulled-in artifacts (a logger's named value, say). + const readExisting = (descriptor: ResourceDescriptor) => store.readResource(outputDir, descriptor); + const resolveExisting = createArtifactReferenceResolver(existing, readExisting); + const readExistingNamedValue = createNamedValueReader(resolveExisting, readExisting); + // A dependency the extract wrote again is not a candidate, but its old + // version is still followed: what it no longer references can be stale. + const candidates = new Map(); + const followed = new Set(); + let wave: ArtifactLocation[] = existing.filter(isSelected).map((descriptor) => ({ descriptor, dir: outputDir })); + while (wave.length > 0) { + // An artifact that can't be scanned contributes no candidates, which only keeps more. + const { references } = await scanAll(store, wave, readExistingNamedValue); + wave = []; + for (const reference of references) { + const target = await resolveExisting(reference); + if (!target || isSelected(target) || followed.has(key(target))) { + continue; + } + followed.add(key(target)); + if (!stagedKeys.has(key(target))) { + candidates.set(key(target), target); + } + wave.push({ descriptor: target, dir: outputDir }); + } + } + if (candidates.size === 0) { + return []; + } + + // What is left after the commit: everything staged, and every existing + // artifact that --remove-stale keeps. A candidate anything left references + // is still needed, and so is whatever that candidate references. + const readRemaining = (descriptor: ResourceDescriptor) => + store.readResource(stagedKeys.has(key(descriptor)) ? stagingDir : outputDir, descriptor); + // A named value that can't be read may be what a {{reference}} left in + // the output names, so its candidates can't be known to be unused. + const unreadable: ResourceDescriptor[] = []; + const resolveRemaining = createArtifactReferenceResolver([...staged, ...kept], readRemaining, (descriptor) => { + unreadable.push(descriptor); + }); + const readRemainingNamedValue = createNamedValueReader(resolveRemaining, readRemaining); + const stale = new Map(candidates); + const referrers: ArtifactLocation[] = [ + ...staged.map((descriptor) => ({ descriptor, dir: stagingDir })), + ...kept.filter((descriptor) => !candidates.has(key(descriptor))).map((descriptor) => ({ descriptor, dir: outputDir })), + ]; + // Scan until nothing is left to rescue: an artifact not yet started is + // skipped once every candidate is known to be needed, and a rescued + // candidate's own references are scanned in the next round. + let round = referrers; + while (stale.size > 0 && round.length > 0) { + const rescued: ArtifactLocation[] = []; + const failed: ResourceDescriptor[] = []; + await runParallel( + round.map(({ descriptor, dir }) => async () => { + if (stale.size === 0 || failed.length > 0 || unreadable.length > 0) { + return; + } + let references: ResourceDescriptor[]; + try { + references = await scanArtifactReferences(store, dir, descriptor, readRemainingNamedValue); + } catch (error) { + logger.warn(`Unable to scan ${buildResourceLabel(descriptor)} for references: ${String(error)}`); + failed.push(descriptor); + return; + } + for (const reference of references) { + const target = await resolveRemaining(reference); + if (target && stale.delete(key(target))) { + rescued.push({ descriptor: target, dir: outputDir }); + } + } + }), + REFERENCE_SCAN_CONCURRENCY + ); + if (failed.length > 0 || unreadable.length > 0) { + logger.warn('Keeping every pulled-in dependency, since not every artifact could be checked for references'); + return []; + } + round = rescued; + } + + return [...stale.values()]; +} diff --git a/src/services/transitive-extractor.ts b/src/services/transitive-extractor.ts index 4f59fc56..0e7fc29a 100644 --- a/src/services/transitive-extractor.ts +++ b/src/services/transitive-extractor.ts @@ -10,7 +10,8 @@ import { buildResourceLabel } from '../lib/resource-uri.js'; import { logger } from '../lib/logger.js'; import { runParallel } from '../lib/parallel-runner.js'; import { redactSecrets } from './secret-redactor.js'; -import { findTransitiveDependencies } from './transitive-resolver.js'; +import { findTransitiveDependencies, scanLiquidNamedValueCandidates } from './transitive-resolver.js'; +import type { ExtractedResource } from './resource-extractor.js'; const DEFAULT_CONCURRENCY = 5; @@ -19,35 +20,88 @@ export interface TransitiveResourceArtifact { json: Record; } +/** + * Add an API's or product's extracted policies to the policies scanned for + * dependencies, keyed `api:{name}:policy:{n}` / `product:{name}:policy:{n}`. + */ +export function collectPolicies( + collected: Map, + owner: 'api' | 'product', + name: string, + policies: ReadonlyArray +): void { + policies.forEach((policy, index) => { + if (policy !== undefined) { + collected.set(`${owner}:${name}:policy:${index}`, policy); + } + }); +} + +/** + * The extracted API children that reference other resources: diagnostics + * (their logger) and tag links (the tag). + */ +export function apiDependencyResources(apiResult: { + diagnostics: ReadonlyArray; + tags: ReadonlyArray; +}): TransitiveResourceArtifact[] { + return [...apiResult.diagnostics, ...apiResult.tags] + .filter((extracted) => extracted.status === 'success') + .map(({ descriptor, json }) => ({ descriptor, json })); +} + export interface TransitiveExtractionResult { extractedDescriptors: ResourceDescriptor[]; + /** Dependencies that were not found or could not be extracted. */ + failedDescriptors: ResourceDescriptor[]; errorCount: number; } interface TransitiveTaskResult { dep: ResourceDescriptor; json?: Record; - /** Resolved to a resource already extracted under its real name. */ + /** + * Resolved to a resource already extracted or claimed by another reference, + * or to one the filter excludes by name. + */ skipped?: boolean; } /** - * Per-context (keyed by baseUrl) index of named value displayName → resource name. + * Per-context (keyed by baseUrl) index of named value displayName, and + * resource name where no display name claims it (lower case, as names are + * compared everywhere else) → resource name. * Stored as promises so concurrent tasks share a single LIST call. */ -type NamedValueNameIndex = Map>>; +type NamedValueNameIndex = Map>; + +interface NamedValueNames { + names: Map; + /** False when the named values couldn't be listed, so `names` is incomplete. */ + listed: boolean; +} async function loadNamedValueNames( client: IApimClient, context: ApimServiceContext -): Promise> { +): Promise { const byDisplayName = new Map(); + const resourceNames: string[] = []; try { for await (const json of client.listResources(context, ResourceType.NamedValue)) { const name = json.name; const displayName = (json.properties as Record | undefined)?.displayName; - if (typeof name === 'string' && typeof displayName === 'string') { - byDisplayName.set(displayName, name); + if (typeof name === 'string') { + resourceNames.push(name); + if (typeof displayName === 'string') { + byDisplayName.set(displayName.toLowerCase(), name); + } + } + } + // A reference matches a display name first, as in APIM, then a resource name. + for (const name of resourceNames) { + if (!byDisplayName.has(name.toLowerCase())) { + byDisplayName.set(name.toLowerCase(), name); } } } catch (error) { @@ -56,8 +110,23 @@ async function loadNamedValueNames( `Failed to list named values for display name resolution; ` + `falling back to resolving {{references}} as resource names: ${message}` ); + return { names: byDisplayName, listed: false }; + } + return { names: byDisplayName, listed: true }; +} + +/** The named value name index for a context, listed once and shared. */ +function namedValueNames( + client: IApimClient, + context: ApimServiceContext, + index: NamedValueNameIndex +): Promise { + let names = index.get(context.baseUrl); + if (!names) { + names = loadNamedValueNames(client, context); + index.set(context.baseUrl, names); } - return byDisplayName; + return names; } /** @@ -71,14 +140,8 @@ async function resolveNamedValueDescriptor( dep: ResourceDescriptor, index: NamedValueNameIndex ): Promise { - let names = index.get(context.baseUrl); - if (!names) { - names = loadNamedValueNames(client, context); - index.set(context.baseUrl, names); - } - const ref = dep.nameParts[0] ?? ''; - const name = (await names).get(ref); + const name = (await namedValueNames(client, context, index)).names.get(ref.toLowerCase()); if (!name || name === ref) { return dep; } @@ -87,6 +150,56 @@ async function resolveNamedValueDescriptor( return { ...dep, nameParts: [name] }; } +/** The `{{name}}`s in a policy fragment's Liquid bodies (scanLiquidNamedValueCandidates). */ +function fragmentLiquidCandidates({ descriptor, json }: TransitiveResourceArtifact): string[] { + if (descriptor.type !== ResourceType.PolicyFragment) { + return []; + } + const properties = json.properties as Record | undefined; + return [properties?.value, properties?.policyContent] + .filter((text): text is string => typeof text === 'string') + .flatMap(scanLiquidNamedValueCandidates); +} + +/** + * The named values Liquid bodies reference: each `{{name}}` candidate that a + * named value in this scope has as its display or resource name. The others + * are Liquid, skipped without a warning. `listed` is false when the named + * values couldn't be listed, so a candidate that is one may be missed. + */ +async function liquidNamedValueDependencies( + client: IApimClient, + context: ApimServiceContext, + candidates: ReadonlySet, + workspace: string | undefined, + index: NamedValueNameIndex +): Promise<{ dependencies: ResourceDescriptor[]; listed: boolean }> { + if (candidates.size === 0) { + return { dependencies: [], listed: true }; + } + const { names, listed } = await namedValueNames(client, context, index); + return { + dependencies: [...candidates] + .filter((candidate) => names.has(candidate.toLowerCase())) + .map((candidate) => ({ type: ResourceType.NamedValue, nameParts: [candidate], workspace })), + listed, + }; +} + +export interface TransitiveExtractionOptions { + /** Workspace being extracted; its resources are fetched through `context`. */ + workspace?: string; + /** Service-level context, for dependencies outside the workspace. */ + serviceContext?: ApimServiceContext; + /** Dependencies already known from links (a product's groups and tags). */ + linkedDependencies?: ResourceDescriptor[]; + /** + * Dependencies not to extract: ones the filter excludes by name, or ones + * the caller knows are extracted already. + */ + skip?: (dep: ResourceDescriptor) => boolean; +} + export async function extractTransitiveDependencies( client: IApimClient, store: IArtifactStore, @@ -96,29 +209,69 @@ export async function extractTransitiveDependencies( apis: Map>, resources: TransitiveResourceArtifact[], alreadyExtracted: ResourceDescriptor[], - workspace?: string, - serviceContext?: ApimServiceContext + { + workspace, + serviceContext, + linkedDependencies = [], + skip = () => false, + }: TransitiveExtractionOptions = {} ): Promise { - const attempted = new Set(alreadyExtracted.map(getResourceDescriptorKey)); + // A {{reference}} to an extracted named value's name can be another named + // value's display name, so named value references are only compared with + // what is extracted once resolved (claimedNamedValues). + const attempted = new Set( + alreadyExtracted.filter((descriptor) => descriptor.type !== ResourceType.NamedValue).map(getResourceDescriptorKey) + ); const claimedNamedValues = new Set(alreadyExtracted.map(getResourceDescriptorKey)); const extractedDescriptors: ResourceDescriptor[] = []; + const failedDescriptors: ResourceDescriptor[] = []; const namedValueIndex: NamedValueNameIndex = new Map(); + const contextFor = (dep: ResourceDescriptor): ApimServiceContext => + serviceContext && dep.workspace !== workspace ? serviceContext : context; let errorCount = 0; let foundDependencies = false; + let reportedUnlisted = false; + + // The {{name}}s in Liquid bodies: the policies' found once, and each + // fragment's as it is extracted. + const liquidCandidates = new Set([...policies.values()].flatMap(scanLiquidNamedValueCandidates)); + let resourcesScanned = 0; while (true) { - const newDeps = findTransitiveDependencies( - policies, - apis, - workspace, - resources - ).filter((dep) => !attempted.has(getResourceDescriptorKey(dep))); + for (; resourcesScanned < resources.length; resourcesScanned++) { + fragmentLiquidCandidates(resources[resourcesScanned]).forEach((name) => liquidCandidates.add(name)); + } + const liquid = await liquidNamedValueDependencies(client, context, liquidCandidates, workspace, namedValueIndex); + if (!liquid.listed && !reportedUnlisted) { + // A named value used only in a Liquid body may be missing from the + // output, so the extract is incomplete (and --remove-stale won't run). + reportedUnlisted = true; + errorCount++; + logger.warn('Unable to tell which {{...}} in Liquid bodies are named values, since the named values could not be listed'); + } + const found = [ + ...findTransitiveDependencies(policies, apis, workspace, resources), + ...linkedDependencies, + ...liquid.dependencies, + ].filter((dep) => !attempted.has(getResourceDescriptorKey(dep))); + const newDeps: ResourceDescriptor[] = []; + for (const dep of found) { + // A {{reference}} can be a display name, so a named value is checked + // under the resource name it resolves to, which is what an exclusion + // names. + const checked = dep.type === ResourceType.NamedValue + ? await resolveNamedValueDescriptor(client, contextFor(dep), dep, namedValueIndex) + : dep; + if (!skip(checked) && !claimedNamedValues.has(getResourceDescriptorKey(checked))) { + newDeps.push(dep); + } + } if (newDeps.length === 0) { if (!foundDependencies) { logger.debug('No additional transitive dependencies found'); } - return { extractedDescriptors, errorCount }; + return { extractedDescriptors, failedDescriptors, errorCount }; } foundDependencies = true; @@ -129,8 +282,7 @@ export async function extractTransitiveDependencies( const tasks = newDeps.map((dep) => async (): Promise => { try { - const dependencyContext = - serviceContext && dep.workspace !== workspace ? serviceContext : context; + const dependencyContext = contextFor(dep); let target = dep; if (dep.type === ResourceType.NamedValue) { target = await resolveNamedValueDescriptor(client, dependencyContext, dep, namedValueIndex); @@ -165,6 +317,9 @@ export async function extractTransitiveDependencies( } if (!value?.json) { errorCount++; + if (value) { + failedDescriptors.push(value.dep); + } continue; } diff --git a/src/services/transitive-resolver.ts b/src/services/transitive-resolver.ts index 1251e823..4e98306a 100644 --- a/src/services/transitive-resolver.ts +++ b/src/services/transitive-resolver.ts @@ -2,24 +2,59 @@ // Licensed under the MIT license. /** * Transitive dependency resolver - * Scan policies for named value refs, backend refs, policy fragment refs. - * Scan apiInformation.json for apiVersionSetId. + * Scan policies for named value, backend, policy fragment, logger and schema + * refs; apiInformation.json for apiVersionSetId; diagnostics for their + * logger; logger and backend credentials for named values; API and product + * links for their tags and groups. A policy attribute set to a named value + * (backend-id="{{name}}") also needs the resource that value names. * Fixed-point expansion; --no-transitive bypass. */ -import { FilterConfig } from '../models/config.js'; -import { ResourceType, RESOURCE_TYPE_METADATA } from '../models/resource-types.js'; -import { ResourceDescriptor } from '../models/types.js'; +import { isBuiltInGroup, ResourceType, RESOURCE_TYPE_METADATA } from '../models/resource-types.js'; +import { AssociationEntry, ResourceDescriptor } from '../models/types.js'; import type { IArtifactStore } from '../clients/iartifact-store.js'; -import { logger } from '../lib/logger.js'; +import { armNameAfter, armReferenceWorkspace } from '../lib/arm-id.js'; import { getResourceDescriptorKey } from '../lib/resource-path.js'; +import { buildResourceLabel } from '../lib/resource-uri.js'; +import { logger } from '../lib/logger.js'; +import { runParallel } from '../lib/parallel-runner.js'; + +/** Artifact files read at once while scanning references: the extract's own concurrency. */ +export const REFERENCE_SCAN_CONCURRENCY = 5; /** * Reference detection patterns for policy XML content. */ const NAMED_VALUE_PATTERN = /\{\{([^}]+)\}\}/g; -const BACKEND_PATTERN = /`, + * lets a value contain `>` (as policy expressions can). + */ +const XML_ATTRIBUTE = String.raw`\s+[\w:.-]+\s*=\s*(?:"[^"]*"|'[^']*')`; + +/** + * Build a pattern matching `attribute="value"` (either quote) anywhere on an + * `element` start tag; the value is capture group 2. + */ +function attributePattern(element: string, attribute: string): RegExp { + return new RegExp(`<${element}(?:${XML_ATTRIBUTE})*?\\s+${attribute}\\s*=\\s*(["'])(.*?)\\1`, 'g'); +} + +/** Policy attributes that name another resource, by the type they name. */ +const POLICY_REFERENCE_PATTERNS: ReadonlyArray<{ type: ResourceType; pattern: RegExp }> = [ + { type: ResourceType.Backend, pattern: attributePattern('set-backend-service', 'backend-id') }, + { type: ResourceType.Backend, pattern: attributePattern('llm-content-safety', 'backend-id') }, + { + type: ResourceType.Backend, + pattern: attributePattern('(?:azure-openai|llm)-semantic-cache-lookup', 'embeddings-backend-id'), + }, + { type: ResourceType.PolicyFragment, pattern: attributePattern('include-fragment', 'fragment-id') }, + { type: ResourceType.Logger, pattern: attributePattern('log-to-eventhub', 'logger-id') }, + // validate-content's names a schema added to the service. + { type: ResourceType.GlobalSchema, pattern: attributePattern('content', 'schema-id') }, +]; /** * Represents a discovered transitive dependency. @@ -29,6 +64,20 @@ export interface TransitiveDependency { name: string; } +/** + * A whole named value reference as the value of a policy attribute that names + * a resource, such as backend-id="{{backend-name}}": the attribute names the + * resource of `type` the named value's value names. + */ +export interface NamedValueTarget { + type: ResourceType; + /** The named value as the policy references it (normally its display name). */ + namedValue: string; +} + +/** An attribute value that is one named value reference and nothing else. */ +const WHOLE_NAMED_VALUE_PATTERN = /^\{\{([^{}]+)\}\}$/; + const POLICY_RESOURCE_TYPES = new Set([ ResourceType.ServicePolicy, ResourceType.ApiPolicy, @@ -37,48 +86,158 @@ const POLICY_RESOURCE_TYPES = new Set([ ResourceType.GraphQLResolverPolicy, ]); +/** + * The artifact types scanArtifactReferences can find references in: policies, + * and the types findTransitiveDependencies and product links read. Keep it in + * step with them. + */ +const REFERENCING_TYPES = new Set([ + ...POLICY_RESOURCE_TYPES, + ResourceType.Api, + ResourceType.ApiTag, + ResourceType.Diagnostic, + ResourceType.ApiDiagnostic, + ResourceType.Logger, + ResourceType.Backend, + ResourceType.PolicyFragment, + ResourceType.Product, +]); + +/** + * XML comments: references inside them are not in effect. Only a closed + * comment is removed, so a stray `/g; + +/** + * The body of a `` element (capture group 3): its + * `{{ ... }}` are mostly Liquid output expressions, not named value + * references; see scanLiquidNamedValueCandidates. A self-closing set-body has + * no body and is not matched. + */ +const LIQUID_BODY_PATTERN = new RegExp( + `()` + + `([\\s\\S]*?)(<\\/set-body\\s*>)`, + 'gi' +); + +/** + * A `{{name}}` whose name has only the characters a named value's name can: + * letters, digits, '.', '-' and '_'. Spaces or a filter (`|`) make it Liquid. + */ +const LIQUID_NAMED_VALUE_PATTERN = /\{\{([A-Za-z0-9._-]+)\}\}/g; + +/** + * Policy XML with the parts that cannot reference other resources removed: + * XML comments, and the bodies of Liquid templates. + */ +function stripNonReferenceContent(policyXml: string): string { + return policyXml + .replace(XML_COMMENT_PATTERN, '') + .replace(LIQUID_BODY_PATTERN, (_match, open: string, _quote: string, _body: string, close: string) => `${open}${close}`); +} + +/** + * The names a `` body may reference named values + * by: each `{{name}}` there whose name a named value could have. APIM + * substitutes a named value inside a Liquid body when one has that name, and + * leaves every other `{{ ... }}` (with spaces, a filter, or no such named + * value, like `{{body.id}}`) to Liquid. So each is a reference only when a + * named value with that name exists; callers check before following one, and + * skip the rest without an error. XML comments are ignored. + */ +export function scanLiquidNamedValueCandidates(policyXml: string): string[] { + const names = new Set(); + for (const body of policyXml.replace(XML_COMMENT_PATTERN, '').matchAll(LIQUID_BODY_PATTERN)) { + for (const match of (body[3] ?? '').matchAll(LIQUID_NAMED_VALUE_PATTERN)) { + if (match[1]) { + names.add(match[1]); + } + } + } + return [...names]; +} + +/** + * Scan text for named value references ({{displayName}}). + */ +function scanNamedValueReferences(text: string): TransitiveDependency[] { + const dependencies: TransitiveDependency[] = []; + for (const match of text.matchAll(NAMED_VALUE_PATTERN)) { + if (match[1]) { + dependencies.push({ type: ResourceType.NamedValue, name: match[1].trim() }); + } + } + return dependencies; +} + /** * Scan policy XML content for references to other resources. * - * Detects: + * Detects, with the attribute anywhere on the element and in either quote: * - Named values: {{namedValueName}} syntax - * - Backends: - * - Policy fragments: + * - Backends: , , + * and embeddings-backend-id on the semantic cache lookup policies + * - Policy fragments: + * - Loggers: + * - Schemas: validate-content's + * + * A value that is a policy expression (`@...`) or a named value reference + * (`{{...}}`, found by the named value scan) does not name a resource. + * References inside XML comments, and `{{ ... }}` inside a + * `` body, are ignored: most of those are Liquid + * output. The bare `{{name}}`s there that APIM substitutes when a named value + * has that name are found by scanLiquidNamedValueCandidates, for callers to + * check. */ export function scanPolicyReferences(policyXml: string): TransitiveDependency[] { - const dependencies: TransitiveDependency[] = []; + const scannable = stripNonReferenceContent(policyXml); + const dependencies = scanNamedValueReferences(scannable); - // Named value references - for (const match of policyXml.matchAll(NAMED_VALUE_PATTERN)) { - if (match[1]) { - dependencies.push({ - type: ResourceType.NamedValue, - name: match[1].trim(), - }); + for (const { type, value } of scanReferenceAttributes(scannable)) { + if (!value.startsWith('@') && !value.includes('{{')) { + dependencies.push({ type, name: value }); } } - // Backend references - for (const match of policyXml.matchAll(BACKEND_PATTERN)) { - if (match[1]) { - dependencies.push({ - type: ResourceType.Backend, - name: match[1].trim(), - }); + return dependencies; +} + +/** + * Scan policy XML for attributes that name a resource through a named value, + * such as backend-id="{{backend-name}}". The named value itself is found by + * scanPolicyReferences; the resource its value names is found once the named + * value's value is known. + */ +export function scanNamedValueTargets(policyXml: string): NamedValueTarget[] { + const targets: NamedValueTarget[] = []; + for (const { type, value } of scanReferenceAttributes(stripNonReferenceContent(policyXml))) { + // APIM doesn't substitute a named value in include-fragment's fragment-id: + // it looks for a fragment with that literal ID and rejects the policy. + if (type === ResourceType.PolicyFragment) { + continue; + } + const namedValue = WHOLE_NAMED_VALUE_PATTERN.exec(value)?.[1]?.trim(); + if (namedValue) { + targets.push({ type, namedValue }); } } + return targets; +} - // Policy fragment references - for (const match of policyXml.matchAll(FRAGMENT_PATTERN)) { - if (match[1]) { - dependencies.push({ - type: ResourceType.PolicyFragment, - name: match[1].trim(), - }); +/** The non-empty values of the policy attributes that name a resource, trimmed. */ +function scanReferenceAttributes(scannable: string): Array<{ type: ResourceType; value: string }> { + const attributes: Array<{ type: ResourceType; value: string }> = []; + for (const { type, pattern } of POLICY_REFERENCE_PATTERNS) { + for (const match of scannable.matchAll(pattern)) { + const value = match[2]?.trim(); + if (value) { + attributes.push({ type, value }); + } } } - - return dependencies; + return attributes; } /** @@ -97,7 +256,7 @@ export function scanApiVersionSetReference( return undefined; } - const name = extractResourceNameFromId(versionSetId, 'apiVersionSets'); + const name = armNameAfter(versionSetId, 'apiVersionSets'); if (!name) { return undefined; } @@ -108,95 +267,6 @@ export function scanApiVersionSetReference( }; } -/** - * Resolve transitive dependencies by expanding the extraction set - * until no new dependencies are found (fixed-point). - * - * @param extractedPolicies - Map of descriptor key to policy XML content - * @param extractedApis - Map of API name to API JSON (for version set refs) - * @param currentFilter - Current filter config to expand - * @returns Updated filter config with transitive dependencies included - */ -export function resolveTransitiveDependencies( - extractedPolicies: Map, - extractedApis: Map>, - currentFilter: FilterConfig -): FilterConfig { - const expanded = { ...currentFilter }; - let changed = true; - let iterations = 0; - const maxIterations = 10; // Safety limit - - while (changed && iterations < maxIterations) { - changed = false; - iterations++; - - // Scan all extracted policies for references - for (const [, policyXml] of extractedPolicies) { - const refs = scanPolicyReferences(policyXml); - - for (const ref of refs) { - if (addToFilter(expanded, ref)) { - changed = true; - } - } - } - - // Scan API information for version set references - for (const [, apiJson] of extractedApis) { - const versionSetRef = scanApiVersionSetReference(apiJson); - if (versionSetRef && addToFilter(expanded, versionSetRef)) { - changed = true; - } - } - } - - if (iterations > 1) { - logger.debug(`Transitive resolution completed in ${iterations} iterations`); - } - - return expanded; -} - -/** - * Add a transitive dependency to the filter config. - * Returns true if the filter was actually modified (new entry added). - */ -function addToFilter( - filter: FilterConfig, - dep: TransitiveDependency -): boolean { - type StringArrayField = 'namedValues' | 'backends' | 'policyFragments' | 'versionSets'; - const fieldMap: Partial> = { - [ResourceType.NamedValue]: 'namedValues', - [ResourceType.Backend]: 'backends', - [ResourceType.PolicyFragment]: 'policyFragments', - [ResourceType.VersionSet]: 'versionSets', - }; - - const field = fieldMap[dep.type]; - if (!field) { - return false; - } - - // Get current array; if undefined, the type is not filtered so no need to add - const current = filter[field]; - if (current === undefined) { - return false; - } - - // Check if already included (case-insensitive) - const lowerName = dep.name.toLowerCase(); - if (current.some((n: string) => n.toLowerCase() === lowerName)) { - return false; - } - - // Add to filter - (filter[field] as string[]) = [...current, dep.name]; - logger.debug(`Transitive: added ${dep.type} "${dep.name}" to filter`); - return true; -} - /** * Build a set of descriptors from transitive dependency scanning. * Returns additional descriptors that should be extracted. @@ -211,11 +281,13 @@ export function findTransitiveDependencies( }> = [] ): ResourceDescriptor[] { const dependencies: ResourceDescriptor[] = []; + const namedValueTargets: Array = []; for (const [, policyXml] of policies) { for (const dep of scanPolicyReferences(policyXml)) { dependencies.push({ type: dep.type, nameParts: [dep.name], workspace }); } + namedValueTargets.push(...scanNamedValueTargets(policyXml).map((target) => ({ ...target, workspace }))); } for (const [, apiJson] of apis) { @@ -228,18 +300,54 @@ export function findTransitiveDependencies( for (const { descriptor, json } of resources) { const properties = json.properties as Record | undefined; + // A diagnostic sends its data to the logger named by properties.loggerId, in the diagnostic's own + // scope. APIM returns a workspace diagnostic's loggerId without the workspace segment + // (.../service/{name}/loggers/{logger}), so the ID can't say which scope the logger is in; publish + // makes the same assumption when it rebuilds the ID. + if (descriptor.type === ResourceType.Diagnostic || descriptor.type === ResourceType.ApiDiagnostic) { + const loggerId = properties?.loggerId; + const name = typeof loggerId === 'string' ? armNameAfter(loggerId, 'loggers') : undefined; + if (name) { + dependencies.push({ + type: ResourceType.Logger, + nameParts: [name], + workspace: descriptor.workspace, + }); + } + } + + // Logger and backend credentials can hold named value references (for + // example the Application Insights key APIM stores as a named value). + if ( + (descriptor.type === ResourceType.Logger || descriptor.type === ResourceType.Backend) && + properties?.credentials !== undefined + ) { + for (const dep of scanNamedValueReferences(JSON.stringify(properties.credentials))) { + dependencies.push({ type: dep.type, nameParts: [dep.name], workspace: descriptor.workspace }); + } + } + + // An API's tag link needs the tag itself. + if (descriptor.type === ResourceType.ApiTag && descriptor.nameParts[1]) { + dependencies.push({ + type: ResourceType.Tag, + nameParts: [descriptor.nameParts[1]], + workspace: descriptor.workspace, + }); + } + if (descriptor.type === ResourceType.Backend) { const pool = isRecord(properties?.pool) ? properties.pool : undefined; const services = pool?.services; if (Array.isArray(services)) { for (const service of services) { if (isRecord(service) && typeof service.id === 'string') { - const name = extractResourceNameFromId(service.id, 'backends'); + const name = armNameAfter(service.id, 'backends'); if (name) { dependencies.push({ type: ResourceType.Backend, nameParts: [name], - workspace: workspaceFromReference(service.id, descriptor.workspace), + workspace: armReferenceWorkspace(service.id, descriptor.workspace), }); } } @@ -259,58 +367,405 @@ export function findTransitiveDependencies( workspace: descriptor.workspace, }); } + namedValueTargets.push( + ...scanNamedValueTargets(value).map((target) => ({ ...target, workspace: descriptor.workspace })) + ); } } } + dependencies.push(...findNamedValueTargetDependencies(namedValueTargets, resources)); + return deduplicateDescriptors(dependencies); } +/** + * The resources that policy attributes name through a named value + * (backend-id="{{backend-name}}"), for the named values among `resources`. + * A reference matches a named value in its own scope by display name, as APIM + * resolves it, falling back to the resource name. A secret named value's value + * isn't known, nor is a Key Vault one's, and a value that is itself a policy + * expression or named value reference names no resource here. + */ +function findNamedValueTargetDependencies( + targets: ReadonlyArray, + resources: ReadonlyArray<{ descriptor: ResourceDescriptor; json: Record }> +): ResourceDescriptor[] { + if (targets.length === 0) { + return []; + } + const namedValueKey = (name: string, workspace: string | undefined): string => + getResourceDescriptorKey({ type: ResourceType.NamedValue, nameParts: [name], workspace }); + // The name each named value's value gives, or undefined when it gives none. + const byName = new Map(); + const byDisplayName = new Map(); + for (const { descriptor, json } of resources) { + if (descriptor.type !== ResourceType.NamedValue) { + continue; + } + const name = namedValueTargetName(json); + byName.set(namedValueKey(descriptor.nameParts[0] ?? '', descriptor.workspace), name); + const displayName = (json.properties as Record | undefined)?.displayName; + if (typeof displayName === 'string') { + byDisplayName.set(namedValueKey(displayName, descriptor.workspace), name); + } + } + + const dependencies: ResourceDescriptor[] = []; + for (const { type, namedValue, workspace } of targets) { + const key = namedValueKey(namedValue, workspace); + const name = byDisplayName.has(key) ? byDisplayName.get(key) : byName.get(key); + if (name) { + dependencies.push({ type, nameParts: [name], workspace }); + } + } + return dependencies; +} + +/** + * The resource name a named value's value gives a policy attribute set to it, + * or undefined when it gives none: a secret named value's value isn't known, + * nor is a Key Vault one's, and a value that is itself a policy expression or + * named value reference names no resource here. + */ +export function namedValueTargetName(json: Record): string | undefined { + const properties = json.properties as Record | undefined; + const value = typeof properties?.value === 'string' ? properties.value.trim() : ''; + return properties?.secret === true || value.startsWith('@') || value.includes('{{') ? undefined : value || undefined; +} + +/** + * Turns an artifact's JSON into the payload that is scanned, such as the one + * publish sends with overrides applied. + */ +export type PrepareJson = (descriptor: ResourceDescriptor, json: Record) => Record; + +/** + * A policy's XML as `prepareJson` leaves it: the payload publish sends is + * `{ properties: { value, format: 'rawxml' } }`, which overrides can change. + */ +function preparePolicyContent(descriptor: ResourceDescriptor, content: string, prepareJson?: PrepareJson): unknown { + if (!prepareJson) { + return content; + } + const payload = prepareJson(descriptor, { properties: { value: content, format: 'rawxml' } }); + return (payload.properties as Record | undefined)?.value; +} + +/** + * The resource types each named value is the whole value of a reference + * attribute for (backend-id="{{name}}"), keyed by the named value's + * descriptor key. It reads the policies and policy fragments among + * `selected`, and the fragments among `available` that they include + * (), directly or through another fragment. A named value + * published without the policies that use it (only its value changed, say) + * needs the resource its value now names. + * + * @param resolve - Resolves a `{{reference}}` to the named value artifact + * @param prepareJson - Turns each policy and fragment into what is scanned, + * such as the payload publish sends with overrides applied + */ +export async function indexNamedValueTargetTypes( + store: Pick, + sourceDir: string, + selected: ReadonlyArray, + available: ReadonlyArray, + resolve: (reference: ResourceDescriptor) => Promise, + prepareJson?: PrepareJson +): Promise>> { + const isPolicy = (descriptor: ResourceDescriptor) => + POLICY_RESOURCE_TYPES.has(descriptor.type) || descriptor.type === ResourceType.PolicyFragment; + const fragments = new Map( + available + .filter((descriptor) => descriptor.type === ResourceType.PolicyFragment) + .map((descriptor) => [getResourceDescriptorKey(descriptor), descriptor]) + ); + const index = new Map>(); + const read = new Set(); + let wave = selected.filter(isPolicy); + while (wave.length > 0) { + wave.forEach((descriptor) => read.add(getResourceDescriptorKey(descriptor))); + const policies = wave; + const results = await runParallel( + policies.map((descriptor) => async (): Promise => { + if (POLICY_RESOURCE_TYPES.has(descriptor.type)) { + const content = (await store.readContent(sourceDir, descriptor, 'policy'))?.content; + return [content === undefined ? undefined : preparePolicyContent(descriptor, content, prepareJson)]; + } + const json = await store.readResource(sourceDir, descriptor); + const properties = (json && prepareJson ? prepareJson(descriptor, json) : json)?.properties as + Record | undefined; + return [properties?.value, properties?.policyContent]; + }), + REFERENCE_SCAN_CONCURRENCY + ); + wave = []; + for (const [position, result] of results.entries()) { + const descriptor = policies[position]; + if (result.status !== 'fulfilled') { + logger.warn( + `Unable to read ${buildResourceLabel(descriptor)} to find the named values its policy attributes are set to: ` + + String(result.reason) + ); + continue; + } + for (const text of result.value ?? []) { + if (typeof text !== 'string') { + continue; + } + for (const { type, namedValue } of scanNamedValueTargets(text)) { + const target = await resolve({ type: ResourceType.NamedValue, nameParts: [namedValue], workspace: descriptor.workspace }); + if (target) { + const key = getResourceDescriptorKey(target); + index.set(key, (index.get(key) ?? new Set()).add(type)); + } + } + for (const reference of scanPolicyReferences(text)) { + if (reference.type !== ResourceType.PolicyFragment) { + continue; + } + const key = getResourceDescriptorKey({ type: reference.type, nameParts: [reference.name], workspace: descriptor.workspace }); + const fragment = fragments.get(key); + if (fragment && !read.has(key)) { + read.add(key); + wave.push(fragment); + } + } + } + } + } + return index; +} + +/** + * The groups or tags a product's links (groups.json, tags.json) need. A link + * recorded at service scope targets a service-level resource even when the + * product is in a workspace. Built-in groups are left out. + */ +export function findAssociationDependencies( + owner: ResourceDescriptor, + associationType: 'groups' | 'tags', + entries: ReadonlyArray +): ResourceDescriptor[] { + return deduplicateDescriptors( + entries + .map((entry) => associationTarget(owner, associationType, entry)) + // Built-in groups exist in every APIM instance, so they are not dependencies. + .filter((target) => !isBuiltInGroup(target)) + ); +} + +/** + * The group or tag a product's link entry points to. An entry recorded at + * service scope targets a service-level resource even when the product is + * in a workspace. + */ +export function associationTarget( + owner: ResourceDescriptor, + associationType: 'groups' | 'tags', + entry: AssociationEntry +): ResourceDescriptor { + return { + type: associationType === 'groups' ? ResourceType.Group : ResourceType.Tag, + nameParts: [entry.name], + workspace: entry.scope === 'service' ? undefined : owner.workspace, + }; +} + +/** + * The groups and tags a product links to, read from its groups.json and + * tags.json association files. + */ +export async function readProductLinkDependencies( + store: Pick, + baseDir: string, + product: ResourceDescriptor +): Promise { + const dependencies: ResourceDescriptor[] = []; + for (const associationType of ['groups', 'tags'] as const) { + const entries = await store.readAssociation(baseDir, product, associationType); + dependencies.push(...findAssociationDependencies(product, associationType, entries)); + } + return dependencies; +} + +/** A named value artifact, read to learn what a policy attribute set to it names. */ +export type NamedValueReader = ( + reference: ResourceDescriptor +) => Promise<{ descriptor: ResourceDescriptor; json: Record } | undefined>; + /** * Read intrinsic dependencies from one on-disk artifact. * - * Association and subscription targets are links to independently selected - * composite resources, not transitive dependencies. + * A product's group and tag links pull in those groups and tags. API, + * product and gateway targets of links and subscriptions are independently + * selected composite resources, not transitive dependencies. + * + * @param readNamedValue - Reads the named value a `{{reference}}` resolves to, + * so a policy attribute set to one (backend-id="{{backend-name}}") also + * gives the resource its value names. Without it, only the named value is. + * @param prepareJson - Turns the artifact's JSON into what is scanned, such + * as the payload publish sends with overrides applied */ export async function scanArtifactReferences( store: IArtifactStore, sourceDir: string, - descriptor: ResourceDescriptor + descriptor: ResourceDescriptor, + readNamedValue?: NamedValueReader, + prepareJson?: PrepareJson ): Promise { + if (!REFERENCING_TYPES.has(descriptor.type)) { + return []; + } const references: ResourceDescriptor[] = []; const policies = new Map(); const apis = new Map>(); if (POLICY_RESOURCE_TYPES.has(descriptor.type)) { const content = await store.readContent(sourceDir, descriptor, 'policy'); - if (content) { - policies.set(descriptor.nameParts.join('/'), content.content); + const text = content ? preparePolicyContent(descriptor, content.content, prepareJson) : undefined; + if (typeof text === 'string') { + policies.set(descriptor.nameParts.join('/'), text); } } const infoFile = RESOURCE_TYPE_METADATA[descriptor.type]?.infoFile; - const json = POLICY_RESOURCE_TYPES.has(descriptor.type) || !infoFile?.endsWith('.json') + const stored = POLICY_RESOURCE_TYPES.has(descriptor.type) || !infoFile?.endsWith('.json') ? undefined : await store.readResource(sourceDir, descriptor); - if (json) { - if (descriptor.type === ResourceType.Api) { - apis.set(descriptor.nameParts.join('/'), json); + const json = stored && prepareJson ? prepareJson(descriptor, stored) : stored; + if (json && descriptor.type === ResourceType.Api) { + apis.set(descriptor.nameParts.join('/'), json); + } + + const resources = json ? [{ descriptor, json }] : []; + if (readNamedValue) { + const fragment = descriptor.type === ResourceType.PolicyFragment + ? (json?.properties as Record | undefined) + : undefined; + const policyTexts = [...policies.values(), fragment?.value, fragment?.policyContent] + .filter((text): text is string => typeof text === 'string'); + const namedValues = new Set(policyTexts.flatMap(scanNamedValueTargets).map((target) => target.namedValue)); + for (const namedValue of namedValues) { + const resolved = await readNamedValue({ + type: ResourceType.NamedValue, + nameParts: [namedValue], + workspace: descriptor.workspace, + }); + if (resolved) { + resources.push(resolved); + } } + } + references.push(...findTransitiveDependencies(policies, apis, descriptor.workspace, resources)); + + // A {{name}} in a Liquid body is a reference only when such a named value + // exists; one that resolves to no artifact is simply not followed. + const fragmentProperties = descriptor.type === ResourceType.PolicyFragment + ? (json?.properties as Record | undefined) + : undefined; + for (const text of [...policies.values(), fragmentProperties?.value, fragmentProperties?.policyContent]) { + if (typeof text === 'string') { + references.push( + ...scanLiquidNamedValueCandidates(text).map((name) => ({ + type: ResourceType.NamedValue, + nameParts: [name], + workspace: descriptor.workspace, + })) + ); + } } - references.push( - ...findTransitiveDependencies( - policies, - apis, - descriptor.workspace, - json ? [{ descriptor, json }] : [] - ) - ); + if (descriptor.type === ResourceType.Product) { + references.push(...await readProductLinkDependencies(store, sourceDir, descriptor)); + } return deduplicateDescriptors(references); } +/** + * Resolve the references scanArtifactReferences returns to the artifacts + * they name. Policies reference named values as {{displayName}}, which can + * differ from the resource name the artifact is stored under: as in APIM + * (and on extract), the display name decides and the resource name is the + * fallback. Named value artifacts are read once, on the first named value + * reference. + * + * @param available - The artifacts references can resolve to + * @param readResource - Reads an artifact's JSON + * @param onUnreadable - Called for a named value artifact that can't be read, + * so a {{reference}} to its display name may resolve to nothing + */ +export function createArtifactReferenceResolver( + available: ReadonlyArray, + readResource: (descriptor: ResourceDescriptor) => Promise | undefined>, + onUnreadable?: (descriptor: ResourceDescriptor) => void +): (reference: ResourceDescriptor) => Promise { + const byKey = new Map(available.map((descriptor) => [getResourceDescriptorKey(descriptor), descriptor])); + let byDisplayName: Promise> | undefined; + return async (reference) => { + const key = getResourceDescriptorKey(reference); + if (reference.type !== ResourceType.NamedValue) { + return byKey.get(key); + } + byDisplayName ??= indexNamedValuesByDisplayName(available, readResource, onUnreadable); + return (await byDisplayName).get(key) ?? byKey.get(key); + }; +} + +/** + * A NamedValueReader that resolves a `{{reference}}` with `resolve` (from + * createArtifactReferenceResolver) and reads the artifact it resolves to. + */ +export function createNamedValueReader( + resolve: (reference: ResourceDescriptor) => Promise, + readResource: (descriptor: ResourceDescriptor) => Promise | undefined> +): NamedValueReader { + // Many policies can set attributes to the same named value; read it once. + const read = new Map>(); + return (reference) => { + const key = getResourceDescriptorKey(reference); + let result = read.get(key); + if (!result) { + result = (async () => { + const descriptor = await resolve(reference); + const json = descriptor ? await readResource(descriptor) : undefined; + return descriptor && json ? { descriptor, json } : undefined; + })(); + read.set(key, result); + } + return result; + }; +} + +/** + * The named value artifacts keyed as a `{{displayName}}` reference to them + * would be (case-insensitive, like every descriptor key). + */ +async function indexNamedValuesByDisplayName( + available: ReadonlyArray, + readResource: (descriptor: ResourceDescriptor) => Promise | undefined>, + onUnreadable?: (descriptor: ResourceDescriptor) => void +): Promise> { + const index = new Map(); + for (const descriptor of available.filter((d) => d.type === ResourceType.NamedValue)) { + let json: Record | undefined; + try { + json = await readResource(descriptor); + } catch (error) { + logger.warn(`Unable to read ${buildResourceLabel(descriptor)} to match {{references}} by display name: ${String(error)}`); + onUnreadable?.(descriptor); + continue; + } + const displayName = (json?.properties as Record | undefined)?.displayName; + if (typeof displayName === 'string') { + index.set(getResourceDescriptorKey({ ...descriptor, nameParts: [displayName] }), descriptor); + } + } + return index; +} + /** * Find API or Product targets referenced by a subscription payload. * @@ -331,12 +786,12 @@ export function findSubscriptionTargets( ['apis', ResourceType.Api], ['products', ResourceType.Product], ] as const) { - const name = extractResourceNameFromId(value, segment); + const name = armNameAfter(value, segment); if (name) { references.push({ type, nameParts: [name], - workspace: workspaceFromReference(value, workspace), + workspace: armReferenceWorkspace(value, workspace), }); } } @@ -345,33 +800,6 @@ export function findSubscriptionTargets( return deduplicateDescriptors(references); } -function extractResourceNameFromId(value: string, segment: string): string | undefined { - const match = value.match(new RegExp(`(?:^|/)${segment}/([^/]+)(?:/|$)`, 'i')); - return match?.[1] ? decodeArmSegment(match[1]) : undefined; -} - -function workspaceFromReference(value: string, fallback?: string): string | undefined { - const match = value.match(/\/workspaces\/([^/]+)/i); - if (match?.[1]) { - return decodeArmSegment(match[1]); - } - - const isFullArmId = - /\/subscriptions\/[^/]+\/resourceGroups\/[^/]+\/providers\/Microsoft\.ApiManagement\/service\/[^/]+/i.test( - value - ); - return isFullArmId ? undefined : fallback; -} - -function decodeArmSegment(value: string): string { - try { - return decodeURIComponent(value); - } catch (error) { - logger.warn(`Unable to decode ARM resource ID segment; using the raw value: ${String(error)}`); - return value; - } -} - function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } diff --git a/src/services/workspace-extractor.ts b/src/services/workspace-extractor.ts index a91d3c14..9a5a269c 100644 --- a/src/services/workspace-extractor.ts +++ b/src/services/workspace-extractor.ts @@ -16,14 +16,18 @@ import { extractProductResources, extractWorkspaceProductTags } from './product- import { logger } from '../lib/logger.js'; import { getNamePart } from '../lib/resource-path.js'; import { + isExplicitlyExcluded, isWildcardPattern, resolveWorkspaceFilter, shouldIncludeResource, } from './filter-service.js'; import { + apiDependencyResources, + collectPolicies, extractTransitiveDependencies, type TransitiveResourceArtifact, } from './transitive-extractor.js'; +import { readProductLinkDependencies } from './transitive-resolver.js'; export { resolveWorkspaceFilter }; @@ -50,6 +54,9 @@ export interface WorkspaceExtractionResult { * @param context - APIM service context * @param outputDir - Output directory * @param filter - Optional filter config (workspace names come from here) + * @param includeTransitive - Whether to pull in the resources extracted ones depend on + * @param serviceExtracted - Resources the service-level extraction already + * wrote or tried, so service-level dependencies are not fetched again * @returns Results per workspace */ export async function extractWorkspaces( @@ -58,7 +65,8 @@ export async function extractWorkspaces( context: ApimServiceContext, outputDir: string, filter?: FilterConfig, - includeTransitive: boolean = false + includeTransitive: boolean = false, + serviceExtracted: ReadonlyArray = [] ): Promise { const results: WorkspaceExtractionResult[] = []; let workspaceNames: string[]; @@ -109,6 +117,13 @@ export async function extractWorkspaces( logger.info(`Extracting ${workspaceNames.length} workspace(s)...`); + // As at service level, dependencies are only resolved when a filter + // narrows what is extracted. Service-level dependencies are shared by + // every workspace, so each is fetched once. + const transitive: WorkspaceTransitiveOptions | undefined = includeTransitive && filter + ? { serviceDependencies: serviceExtracted.filter((descriptor) => !descriptor.workspace) } + : undefined; + for (const wsName of workspaceNames) { // Persist the workspace container itself so publish can recreate it // before workspace-scoped children (named values, APIs, products, etc.). @@ -126,7 +141,8 @@ export async function extractWorkspaces( const wsResult = await extractWorkspace( client, store, context, wsName, outputDir, resolveWorkspaceFilter(wsName, filter), - includeTransitive + filter, + transitive ); wsResult.errorCount += workspaceContainerError; results.push(wsResult); @@ -135,6 +151,14 @@ export async function extractWorkspaces( return results; } +interface WorkspaceTransitiveOptions { + /** + * Service-level resources already extracted; each workspace adds the ones + * it tried, extracted or failed, so they are not tried again. + */ + serviceDependencies: ResourceDescriptor[]; +} + async function extractWorkspace( client: IApimClient, store: IArtifactStore, @@ -142,7 +166,13 @@ async function extractWorkspace( workspaceName: string, outputDir: string, filter?: FilterConfig, - includeTransitive: boolean = false + /** + * The whole filter: it decides for service-level resources and, through + * its workspace sub-filters, for workspace ones. + */ + serviceFilter?: FilterConfig, + /** Set when dependencies are resolved. */ + transitive?: WorkspaceTransitiveOptions ): Promise { logger.info(`Extracting workspace "${workspaceName}"...`); @@ -202,12 +232,8 @@ async function extractWorkspace( ); const apiName = getNamePart(api.descriptor.nameParts, 0); apis.set(apiName, api.json); - for (let index = 0; index < apiResult.policies.length; index++) { - const policy = apiResult.policies[index]; - if (policy !== undefined) { - policies.set(`api:${apiName}:policy:${index}`, policy); - } - } + resources.push(...apiDependencyResources(apiResult)); + collectPolicies(policies, 'api', apiName, apiResult.policies); resourceCount += apiResult.operations.length + apiResult.tags.length + apiResult.schemas.length; @@ -226,15 +252,10 @@ async function extractWorkspace( try { const productResult = await extractProductResources( client, store, wsContext, product.descriptor, - outputDir, filter, workspaceName + outputDir, serviceFilter, workspaceName ); const productName = getNamePart(product.descriptor.nameParts, 0); - for (let index = 0; index < productResult.policies.length; index++) { - const policy = productResult.policies[index]; - if (policy !== undefined) { - policies.set(`product:${productName}:policy:${index}`, policy); - } - } + collectPolicies(policies, 'product', productName, productResult.policies); resourceCount++; errorCount += productResult.errorCount; } catch (error) { @@ -249,14 +270,38 @@ async function extractWorkspace( } } + // Workspace tag links can only be listed per tag (one apiLinks and one + // productLinks request each). When the filter leaves some tags out and + // transitive resolution is on, look on every workspace tag the filter does + // not exclude by name, so tags linked to the extracted APIs and products + // are found and pulled in below; otherwise the extracted tags are all + // there is to check. + const hasLinkOwners = extractedApiNames.size > 0 || extractedProducts.length > 0; + let linkTagNames = extractedTagNames; + if (transitive && hasLinkOwners && leavesOutUnnamed(filter?.tags)) { + const listed = await listWorkspaceTagNames(client, wsContext, workspaceName); + if (listed) { + linkTagNames = listed.filter( + (name) => !isExplicitlyExcluded( + { type: ResourceType.Tag, nameParts: [name], workspace: workspaceName }, + serviceFilter + ) + ); + } else { + // Links on the other tags may be missed, so --remove-stale must not run. + errorCount++; + } + } + // Extract workspace API tags using the tag-centric apiLinks endpoint. // This must happen after both Tags and APIs are extracted since the // workspace scope uses `tags/{tag}/apiLinks` (inverted parent-child). - if (extractedTagNames.length > 0 && extractedApiNames.size > 0) { + if (linkTagNames.length > 0 && extractedApiNames.size > 0) { try { const apiTagCount = await extractWorkspaceApiTags( - client, store, wsContext, extractedTagNames, extractedApiNames, - outputDir, workspaceName + client, store, wsContext, linkTagNames, extractedApiNames, + outputDir, workspaceName, + (descriptor, json) => resources.push({ descriptor, json }) ); resourceCount += apiTagCount; } catch (error) { @@ -267,10 +312,10 @@ async function extractWorkspace( // Extract workspace product tags using the tag-centric productLinks endpoint. // Similar to ApiTag, workspace ProductTag uses `tags/{tag}/productLinks`. - if (extractedTagNames.length > 0 && extractedProducts.length > 0) { + if (linkTagNames.length > 0 && extractedProducts.length > 0) { try { const productTagCount = await extractWorkspaceProductTags( - client, store, wsContext, extractedTagNames, extractedProducts, + client, store, wsContext, linkTagNames, extractedProducts, outputDir, workspaceName ); resourceCount += productTagCount; @@ -280,8 +325,16 @@ async function extractWorkspace( } } - if (includeTransitive && filter) { + // Without a sub-filter the whole workspace was extracted, and its resources + // depend only on resources in it: APIM doesn't let a workspace product link + // to a custom service-level group, and the built-in ones are never pulled in. + if (transitive && filter) { logger.info(`Resolving transitive dependencies for workspace "${workspaceName}"...`); + // The groups and tags the extracted products link to. + const linkedDependencies: ResourceDescriptor[] = []; + for (const product of extractedProducts) { + linkedDependencies.push(...await readProductLinkDependencies(store, outputDir, product.descriptor)); + } const transitiveResult = await extractTransitiveDependencies( client, store, @@ -290,9 +343,18 @@ async function extractWorkspace( policies, apis, resources, - extractedDescriptors, - workspaceName, - context + [...extractedDescriptors, ...transitive.serviceDependencies], + { + workspace: workspaceName, + serviceContext: context, + linkedDependencies, + skip: (dep) => isExplicitlyExcluded(dep, serviceFilter), + } + ); + // Later workspaces skip the service-level ones, including any that failed. + transitive.serviceDependencies.push( + ...[...transitiveResult.extractedDescriptors, ...transitiveResult.failedDescriptors] + .filter((descriptor) => !descriptor.workspace) ); resourceCount += transitiveResult.extractedDescriptors.length; errorCount += transitiveResult.errorCount; @@ -303,6 +365,45 @@ async function extractWorkspace( return { workspaceName, resourceCount, errorCount }; } +/** + * Whether a filter list leaves out resources it does not name: it is `[]` or + * names some to include. A list of exclusions only (`['!internal']`), or one + * including `*`, keeps everything else, so all it leaves out is excluded by + * name. + */ +function leavesOutUnnamed(list: string[] | undefined): boolean { + if (list === undefined) { + return false; + } + const includes = list.filter((entry) => !entry.startsWith('!')); + return list.length === 0 || (includes.length > 0 && !includes.includes('*')); +} + +/** + * Every tag in the workspace, or undefined if the list fails. + */ +async function listWorkspaceTagNames( + client: IApimClient, + wsContext: ApimServiceContext, + workspaceName: string +): Promise { + const names: string[] = []; + try { + for await (const json of client.listResources(wsContext, ResourceType.Tag)) { + if (typeof json.name === 'string') { + names.push(json.name); + } + } + return names; + } catch (error) { + logger.warn( + `Failed to list tags in workspace "${workspaceName}"; only links to extracted tags are included: ` + + `${(error as Error).message}` + ); + return undefined; + } +} + /** * Discover all workspace names from APIM. */ diff --git a/tests/unit/cli/extract-command.test.ts b/tests/unit/cli/extract-command.test.ts index de66f588..f5f76733 100644 --- a/tests/unit/cli/extract-command.test.ts +++ b/tests/unit/cli/extract-command.test.ts @@ -17,6 +17,11 @@ import { IArtifactStore } from '../../../src/clients/iartifact-store.js'; import { ResourceType } from '../../../src/models/resource-types.js'; import { ApiExtractionResult } from '../../../src/services/api-extractor.js'; import { TypeExtractionResult } from '../../../src/services/resource-extractor.js'; +import { ArtifactStore } from '../../../src/clients/artifact-store.js'; +import type { ExtractConfig } from '../../../src/models/config.js'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; describe('extract-command', () => { describe('createExtractCommand', () => { @@ -144,6 +149,63 @@ describe('extract-command', () => { }); }); + describe('executeExtract stale dependencies', () => { + const orders = { type: ResourceType.Api, nameParts: ['orders'] }; + + /** Extract into `output` with `filterYaml`, where the extract finds only API orders. */ + async function extractOrders(root: string, output: string, filterYaml: string, store: ArtifactStore): Promise { + const filterPath = path.join(root, 'filter.yaml'); + await fs.writeFile(filterPath, filterYaml); + const result: ExtractionResult = { + totalExtracted: 1, + totalErrors: 0, + typeResults: [], + apiResults: [], + productResults: [], + workspaceResults: [], + extractedDescriptors: [orders], + collectedPolicies: new Map(), + exitCode: 0, + }; + const runExtraction = vi.fn(async (_client: unknown, _store: unknown, config: ExtractConfig) => { + await store.writeResource(config.outputDir, orders, { name: 'orders', properties: {} }); + return result; + }); + const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + try { + await executeExtract( + { resourceGroup: 'rg-1', serviceName: 'apim-1', output, filter: filterPath, transitive: true, removeStale: true }, + { subscriptionId: 'sub-1', format: 'json' }, + { runExtraction, createClient: () => ({}) as ApimClient, createStore: () => store, exit: vi.fn() } + ); + } finally { + stdout.mockRestore(); + } + return (await store.listResources(output)) + .map((descriptor) => `${descriptor.type}:${descriptor.nameParts.join('/')}`) + .sort(); + } + + it('should remove an old link to a tag the filter excludes, and the tag', async () => { + const filterYaml = "apis: [orders]\ntags: ['!internal']\n"; + const tagName = 'internal'; + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'apiops-extract-stale-')); + const output = path.join(root, 'artifacts'); + const store = new ArtifactStore(); + try { + await store.writeResource(output, orders, { name: 'orders', properties: {} }); + await store.writeResource(output, { type: ResourceType.ApiTag, nameParts: ['orders', tagName] }, { properties: {} }); + await store.writeResource(output, { type: ResourceType.Tag, nameParts: [tagName] }, { name: tagName, properties: {} }); + await store.writeResource(output, { type: ResourceType.Tag, nameParts: ['standalone'] }, { name: 'standalone', properties: {} }); + + // Every other tag is in this filter's scope, so standalone goes too: the extract did not find it. + expect(await extractOrders(root, output, filterYaml, store)).toEqual(['Api:orders']); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + }); + describe('outputText', () => { function typeResult(type: ResourceType, successCount: number, errorCount = 0): TypeExtractionResult { return { diff --git a/tests/unit/lib/arm-id.test.ts b/tests/unit/lib/arm-id.test.ts new file mode 100644 index 00000000..36b23b1e --- /dev/null +++ b/tests/unit/lib/arm-id.test.ts @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +/** + * Unit tests for ARM resource ID helpers + */ + +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { logger } from '../../../src/lib/logger.js'; +import { + armLastName, + armNameAfter, + armReferenceWorkspace, + armScope, + decodeArmSegment, +} from '../../../src/lib/arm-id.js'; + +const SERVICE = '/subscriptions/s/resourceGroups/r/providers/Microsoft.ApiManagement/service/apim'; + +describe('arm-id', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('should decode a segment, and keep a malformed one as it is with a warning', () => { + const warn = vi.spyOn(logger, 'warn').mockImplementation(() => undefined); + + expect(decodeArmSegment('echo%20api')).toBe('echo api'); + expect(decodeArmSegment('bad%E0%A4%A')).toBe('bad%E0%A4%A'); + expect(warn).toHaveBeenCalledTimes(1); + }); + + it.each([ + ['/loggers/app-insights', 'loggers', 'app-insights'], + [`${SERVICE}/workspaces/team-a/Loggers/ws%20logger`, 'loggers', 'ws logger'], + [`${SERVICE}/apis/orders/operations/get`, 'apis', 'orders'], + [`${SERVICE}/apis/orders`, 'products', undefined], + ['/subscriptions/s/resourceGroups/products/providers/Microsoft.ApiManagement/service/apim/apis/orders', 'products', undefined], + ['/subscriptions/s/resourceGroups/r/providers/Microsoft.ApiManagement/service/loggers/loggers/l', 'loggers', 'l'], + [`${SERVICE}/workspaces/backends/backends/member`, 'backends', 'member'], + ['/workspaces/products/products/gold', 'products', 'gold'], + ])('should read the name after a segment in %s', (id, segment, expected) => { + expect(armNameAfter(id, segment)).toBe(expected); + }); + + it.each([ + [`${SERVICE}/groups/administrators`, 'administrators'], + [`${SERVICE}/apis/echo%20api/`, 'echo api'], + ['', undefined], + ])('should read the last name of %j', (id, expected) => { + expect(armLastName(id)).toBe(expected); + }); + + it.each([ + [`${SERVICE}/workspaces/team-a/groups/devs`, 'workspace'], + [`${SERVICE}/workspaces/team-a`, 'workspace'], + [`${SERVICE}/groups/administrators`, 'service'], + ['/subscriptions/s/resourceGroups/workspaces/providers/Microsoft.ApiManagement/service/apim/groups/g', 'service'], + ['/subscriptions/s/resourceGroups/r/providers/Microsoft.ApiManagement/service/workspaces/groups/g', 'service'], + [`${SERVICE.replace('Microsoft.ApiManagement', 'microsoft.apimanagement')}/WORKSPACES/ws/groups/g`, 'workspace'], + ])('should read the scope of %s', (id, expected) => { + expect(armScope(id)).toBe(expected); + }); + + it.each([ + [`${SERVICE}/workspaces/team-a/loggers/l`, 'other', 'team-a'], + [`${SERVICE}/loggers/l`, 'team-a', undefined], + ['/subscriptions/s/resourceGroups/workspaces/providers/Microsoft.ApiManagement/service/apim/loggers/l', 'team-a', undefined], + ['/loggers/l', 'team-a', 'team-a'], + ['/loggers/l', undefined, undefined], + ])('should read the workspace %s points into (referenced from %s)', (id, fallback, expected) => { + expect(armReferenceWorkspace(id, fallback)).toBe(expected); + }); +}); diff --git a/tests/unit/lib/config-loader.test.ts b/tests/unit/lib/config-loader.test.ts index 47fd3e54..46448cab 100644 --- a/tests/unit/lib/config-loader.test.ts +++ b/tests/unit/lib/config-loader.test.ts @@ -201,6 +201,49 @@ apis: }); }); + it('should load a wildcard API sub-filter with tagDescriptions (#316, #307)', async () => { + const content = ` +apis: + - '*': + operations: [] + schemas: [] + tagDescriptions: [] +`; + const filePath = path.join(tmpDir, 'wildcard-api-filter.yaml'); + await fs.writeFile(filePath, content, 'utf-8'); + + const config = await loadFilterConfig(filePath); + + expect(config!.apis).toEqual(['*']); + expect(config!.apiSubFilters).toEqual({ + '*': { operations: [], schemas: [], tagDescriptions: [] }, + }); + }); + + it('should ignore, with a warning, sub-filters keyed by an exclusion', async () => { + const warnSpy = vi.spyOn(logger, 'warn').mockImplementation(() => {}); + const content = ` +apis: + - '*' + - '!legacy': + operations: [] +workspaces: + - '!team-c': + apis: [] +`; + const filePath = path.join(tmpDir, 'exclusion-sub-filter.yaml'); + await fs.writeFile(filePath, content, 'utf-8'); + + const config = await loadFilterConfig(filePath); + + expect(config!.apis).toEqual(['*', '!legacy']); + expect(config!.apiSubFilters).toEqual({}); + expect(config!.workspaceSubFilters).toEqual({}); + expect(warnSpy).toHaveBeenCalledWith("Sub-filter for exclusion '!legacy' in 'apis' has no effect; ignoring."); + expect(warnSpy).toHaveBeenCalledWith("Sub-filter for exclusion '!team-c' in 'workspaces' has no effect; ignoring."); + warnSpy.mockRestore(); + }); + it('should load nested workspace filter entries', async () => { const content = ` workspaces: diff --git a/tests/unit/lib/workspace-link.test.ts b/tests/unit/lib/workspace-link.test.ts index 3c73b7fb..18112e67 100644 --- a/tests/unit/lib/workspace-link.test.ts +++ b/tests/unit/lib/workspace-link.test.ts @@ -1,13 +1,17 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi, afterEach } from 'vitest'; import { extractScopeFromLinkId, extractLinkTarget, buildWorkspaceResourceId, buildLinkPayload, + listWorkspaceTagLinks, } from '../../../src/lib/workspace-link.js'; import { ApimServiceContext } from '../../../src/models/types.js'; +import { ResourceType } from '../../../src/models/resource-types.js'; +import { logger } from '../../../src/lib/logger.js'; +import type { IApimClient } from '../../../src/clients/iapim-client.js'; const SERVICE_BASE = 'https://management.azure.com/subscriptions/sub/resourceGroups/rg/providers/Microsoft.ApiManagement/service/apim'; @@ -116,3 +120,51 @@ describe('buildLinkPayload', () => { expect(groupId).toContain('/workspaces/ws/groups/ws-group'); }); }); + +describe('listWorkspaceTagLinks', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + /** A client whose tag-link listings come from `links` (tag name → link responses). */ + function clientWith(links: Record> | Error>): IApimClient { + return { + listResources: async function* (_ctx: ApimServiceContext, _type: ResourceType, parent?: { nameParts: string[] }) { + const entry = links[parent?.nameParts[0] ?? ''] ?? []; + if (entry instanceof Error) throw entry; + yield* entry; + }, + } as unknown as IApimClient; + } + const collect = async (items: AsyncIterable) => { + const result: T[] = []; + for await (const item of items) result.push(item); + return result; + }; + + it('should yield each tag with the API or product its links point at', async () => { + const client = clientWith({ + public: [{ properties: { apiId: `${WORKSPACE_BASE}/apis/orders` } }, { properties: { apiId: `${WORKSPACE_BASE}/apis/billing` } }], + internal: [], + }); + + await expect(collect(listWorkspaceTagLinks(client, makeContext(WORKSPACE_BASE), ['public', 'internal'], ResourceType.ApiTag))) + .resolves.toEqual([{ tagName: 'public', targetName: 'orders' }, { tagName: 'public', targetName: 'billing' }]); + }); + + it('should throw for a link without a readable target', async () => { + const client = clientWith({ partner: [{ properties: {} }] }); + + await expect(collect(listWorkspaceTagLinks(client, makeContext(WORKSPACE_BASE), ['partner'], ResourceType.ProductTag))) + .rejects.toThrow('Failed to extract product name from tag "partner" productLink response'); + }); + + it('should warn and rethrow when listing a tag\'s links fails', async () => { + const warn = vi.spyOn(logger, 'warn').mockImplementation(() => undefined); + const client = clientWith({ public: new Error('HTTP 500') }); + + await expect(collect(listWorkspaceTagLinks(client, makeContext(WORKSPACE_BASE), ['public'], ResourceType.ApiTag))) + .rejects.toThrow('HTTP 500'); + expect(warn).toHaveBeenCalledTimes(1); + }); +}); diff --git a/tests/unit/services/api-product-extractor.test.ts b/tests/unit/services/api-product-extractor.test.ts index 23017257..b8a2d908 100644 --- a/tests/unit/services/api-product-extractor.test.ts +++ b/tests/unit/services/api-product-extractor.test.ts @@ -1493,6 +1493,65 @@ describe('product-extractor', () => { expect(result.errorCount).toBe(1); }); + it('leaves out links to groups and tags the filter excludes by name', async () => { + const client = createMockClient(); + client.listResources = async function* (_ctx, type) { + if (type === ResourceType.ProductGroup) { + yield { name: 'partners' }; + yield { name: 'support-team' }; + } + if (type === ResourceType.ProductTag) { + yield { name: 'internal' }; + yield { name: 'public' }; + } + }; + client.getResource = vi.fn().mockResolvedValue(undefined); + const store = createMockStore(); + const filter: FilterConfig = { products: ['starter'], groups: ['!partners', '*'], tags: ['!internal'] }; + + const result = await extractProductResources( + client, store, testContext, + { type: ResourceType.Product, nameParts: ['starter'] }, + '/output', filter + ); + + expect(result.groups).toEqual(['support-team']); + expect(result.tags).toEqual(['public']); + expect(store.writeAssociation).toHaveBeenCalledWith('/output', expect.anything(), 'groups', [{ name: 'support-team' }]); + expect(store.writeAssociation).toHaveBeenCalledWith('/output', expect.anything(), 'tags', [{ name: 'public' }]); + }); + + it("decides a workspace product's links by the scope each target is in", async () => { + const SERVICE = '/subscriptions/s/resourceGroups/r/providers/Microsoft.ApiManagement/service/a'; + const client = createMockClient(); + client.listResources = async function* (_ctx, type) { + if (type === ResourceType.ProductGroup) { + yield { name: 'l1', properties: { groupId: `${SERVICE}/groups/partners` } }; + yield { name: 'l2', properties: { groupId: `${SERVICE}/workspaces/team-a/groups/partners` } }; + yield { name: 'l3', properties: { groupId: `${SERVICE}/workspaces/team-a/groups/ops` } }; + } + }; + client.getResource = vi.fn().mockResolvedValue(undefined); + const store = createMockStore(); + const filter: FilterConfig = { + groups: ['!partners', '*'], + workspaceSubFilters: { 'team-a': { groups: ['!ops', '*'] } }, + }; + + const result = await extractProductResources( + client, store, { ...testContext, baseUrl: `${testContext.baseUrl}/workspaces/team-a` }, + { type: ResourceType.Product, nameParts: ['starter'], workspace: 'team-a' }, + '/output', filter, 'team-a' + ); + + // The service-level partners group is excluded at the top level; the + // workspace's own partners group is not, but its ops group is. + expect(result.groups).toEqual(['partners']); + expect(store.writeAssociation).toHaveBeenCalledWith( + '/output', expect.anything(), 'groups', [{ name: 'partners', scope: 'workspace' }] + ); + }); + it('counts failed product associations and wiki extraction', async () => { const client = createMockClient(); // eslint-disable-next-line require-yield diff --git a/tests/unit/services/api-publisher.test.ts b/tests/unit/services/api-publisher.test.ts index 6e3592fa..3f37f1b9 100644 --- a/tests/unit/services/api-publisher.test.ts +++ b/tests/unit/services/api-publisher.test.ts @@ -771,7 +771,8 @@ describe('api-publisher', () => { store, testContext, expect.objectContaining({ type: ResourceType.ApiPolicy, nameParts: ['orders-api'] }), - testConfig + testConfig, + undefined ); }); diff --git a/tests/unit/services/extract-service.test.ts b/tests/unit/services/extract-service.test.ts index 090b8209..3b35e87f 100644 --- a/tests/unit/services/extract-service.test.ts +++ b/tests/unit/services/extract-service.test.ts @@ -273,6 +273,105 @@ describe('extract-service', () => { expect(result.collectedPolicies.get('service-policy')).toBe(redactedPolicy); }); + it("should pull in an API diagnostic's logger, an API's tags, and a product's groups and tags", async () => { + const client = createMockClient({ + [ResourceType.Api]: [{ name: 'orders', properties: {} }], + [ResourceType.ApiDiagnostic]: [{ name: 'applicationinsights', properties: { loggerId: '/loggers/app-insights' } }], + [ResourceType.ApiTag]: [{ name: 'public', properties: { displayName: 'public' } }], + [ResourceType.Product]: [{ name: 'starter', properties: {} }], + }); + client.getResource = vi.fn().mockImplementation(async (_ctx, descriptor: ResourceDescriptor) => + [ResourceType.Logger, ResourceType.Tag, ResourceType.Group].includes(descriptor.type) + ? { name: descriptor.nameParts[0], properties: {} } + : undefined + ); + const store = createMockStore(); + store.readAssociation.mockImplementation(async (_dir: string, _descriptor: ResourceDescriptor, type: string) => + type === 'groups' ? [{ name: 'support-team' }] : type === 'tags' ? [{ name: 'partner' }] : [] + ); + const filter: FilterConfig = { + apis: ['orders'], + products: ['starter'], + loggers: [], + tags: [], + groups: [], + }; + + const result = await runExtraction(client, store, { + service: testContext, + outputDir: '/output', + filter, + includeTransitive: true, + logLevel: LogLevel.INFO, + }); + + expect(result.extractedDescriptors).toEqual(expect.arrayContaining([ + { type: ResourceType.Logger, nameParts: ['app-insights'], workspace: undefined }, + { type: ResourceType.Tag, nameParts: ['public'], workspace: undefined }, + { type: ResourceType.Group, nameParts: ['support-team'], workspace: undefined }, + { type: ResourceType.Tag, nameParts: ['partner'], workspace: undefined }, + ])); + }); + + it('should not try again in a workspace a service-level dependency the service pass failed to extract', async () => { + const client = createMockClient({ + [ResourceType.Product]: [{ name: 'starter', properties: {} }], + }); + client.getResource = vi.fn().mockImplementation(async (_ctx, descriptor: ResourceDescriptor) => + descriptor.type === ResourceType.Group ? undefined : { name: descriptor.nameParts[0], properties: {} } + ); + const store = createMockStore(); + store.readAssociation.mockImplementation(async (_dir: string, _descriptor: ResourceDescriptor, type: string) => + type === 'groups' ? [{ name: 'old-team', scope: 'service' }] : [] + ); + + const result = await runExtraction(client, store, { + service: testContext, + outputDir: '/output', + filter: { products: ['starter'], groups: [], workspaces: ['team-a'] }, + includeTransitive: true, + logLevel: LogLevel.INFO, + }); + + const groupGets = client.getResource.mock.calls.filter(([, d]) => (d as ResourceDescriptor).type === ResourceType.Group); + expect(groupGets).toHaveLength(1); + expect(result.workspaceResults?.[0]?.errorCount).toBe(0); + }); + + it('should not pull in a tag the filter excludes by name', async () => { + const client = createMockClient({ + [ResourceType.Api]: [{ name: 'orders', properties: {} }], + [ResourceType.ApiTag]: [ + { name: 'internal', properties: { displayName: 'internal' } }, + { name: 'public', properties: { displayName: 'public' } }, + ], + }); + client.getResource = vi.fn().mockImplementation(async (_ctx, descriptor: ResourceDescriptor) => + descriptor.type === ResourceType.Tag ? { name: descriptor.nameParts[0], properties: {} } : undefined + ); + + const store = createMockStore(); + await runExtraction(client, store, { + service: testContext, + outputDir: '/output', + filter: { apis: ['orders'], tags: ['!internal', 'none'] }, + includeTransitive: true, + logLevel: LogLevel.INFO, + }); + + const fetchedTags = client.getResource.mock.calls + .map(([, descriptor]) => descriptor as ResourceDescriptor) + .filter((descriptor) => descriptor.type === ResourceType.Tag) + .map((descriptor) => descriptor.nameParts[0]); + expect(fetchedTags).toEqual(['public']); + // The link to the excluded tag is not written either. + expect(store.writeResource).not.toHaveBeenCalledWith( + '/output', + expect.objectContaining({ type: ResourceType.ApiTag, nameParts: ['orders', 'internal'] }), + expect.anything() + ); + }); + it('should not crash during transitive dedupe when a singleton (ServicePolicy) is already extracted', async () => { // Regression: transitive resolution previously keyed the // already-extracted set on getNamePart(d.nameParts, 0), which threw diff --git a/tests/unit/services/filter-service.test.ts b/tests/unit/services/filter-service.test.ts index 169b9277..04d15093 100644 --- a/tests/unit/services/filter-service.test.ts +++ b/tests/unit/services/filter-service.test.ts @@ -11,11 +11,14 @@ import { FilterConfig } from '../../../src/models/config.js'; import { shouldIncludeResource, shouldReconcileResource, + isInRemoveStaleScope, filterResources, extractRootApiName, isWildcardPattern, wildcardToRegex, wildcardMatch, + findApiSubFilter, + isExplicitlyExcluded, } from '../../../src/services/filter-service.js'; describe('filter-service', () => { @@ -47,6 +50,37 @@ describe('filter-service', () => { filter )).toBe(false); }); + + it('should use a workspace sub-filter keyed by a pattern', () => { + const filter: FilterConfig = { + workspaces: ['team-*'], + workspaceSubFilters: { + 'team-*': { namedValues: ['workspace-value'] }, + }, + }; + + expect(shouldReconcileResource(workspaceNamedValue, filter)).toBe(true); + expect(shouldReconcileResource( + { ...workspaceNamedValue, nameParts: ['other-value'] }, + filter + )).toBe(false); + }); + }); + + describe('isInRemoveStaleScope', () => { + const internalLink: ResourceDescriptor = { type: ResourceType.ApiTag, nameParts: ['orders', 'internal'] }; + + it("should count an API's link to an excluded tag as the API's to manage", () => { + const filter: FilterConfig = { apis: ['orders'], tags: ['!internal'] }; + + expect(shouldReconcileResource(internalLink, filter)).toBe(false); + expect(isInRemoveStaleScope(internalLink, filter)).toBe(true); + expect(isInRemoveStaleScope({ type: ResourceType.Tag, nameParts: ['internal'] }, filter)).toBe(false); + }); + + it("should leave an unselected API's tag links alone", () => { + expect(isInRemoveStaleScope(internalLink, { apis: ['billing'] })).toBe(false); + }); }); describe('shouldIncludeResource', () => { @@ -345,6 +379,86 @@ describe('filter-service', () => { }); }); + describe('wildcard API sub-filters (#316) and tagDescriptions (#307)', () => { + const filter: FilterConfig = { + apis: ['*', 'orders'], + apiSubFilters: { + '*': { operations: [], schemas: [], tagDescriptions: [] }, + orders: { operations: ['get-order'] }, + }, + }; + const child = (type: ResourceType, api: string, name: string): ResourceDescriptor => ({ + type, + nameParts: [api, name], + }); + + it('should prefer a sub-filter keyed by the API name over a pattern, ignoring case, and find none without a match', () => { + expect(findApiSubFilter('ORDERS', filter.apiSubFilters)).toEqual({ operations: ['get-order'] }); + expect(findApiSubFilter('orders', { 'pet*': { schemas: [] } })).toBeUndefined(); + expect(findApiSubFilter('orders', undefined)).toBeUndefined(); + }); + + it('should exclude operations, schemas and tag descriptions from every API matched by the pattern', () => { + for (const type of [ResourceType.ApiOperation, ResourceType.ApiSchema, ResourceType.ApiTagDescription]) { + expect(shouldIncludeResource(child(type, 'petstore', 'x'), filter)).toBe(false); + } + expect(shouldIncludeResource(child(ResourceType.ApiDiagnostic, 'petstore', 'applicationinsights'), filter)).toBe(true); + }); + + it('should apply the named sub-filter to the named API only', () => { + expect(shouldIncludeResource(child(ResourceType.ApiOperation, 'orders', 'get-order'), filter)).toBe(true); + expect(shouldIncludeResource(child(ResourceType.ApiOperation, 'orders', 'delete-order'), filter)).toBe(false); + // The named sub-filter does not list schemas, so the API's schemas are all included. + expect(shouldIncludeResource(child(ResourceType.ApiSchema, 'orders', 'schema-1'), filter)).toBe(true); + }); + + it('should filter tag descriptions by name', () => { + const tagFilter: FilterConfig = { apis: ['orders'], apiSubFilters: { orders: { tagDescriptions: ['public'] } } }; + + expect(shouldIncludeResource(child(ResourceType.ApiTagDescription, 'orders', 'public'), tagFilter)).toBe(true); + expect(shouldIncludeResource(child(ResourceType.ApiTagDescription, 'orders', 'internal'), tagFilter)).toBe(false); + }); + }); + + describe('isExplicitlyExcluded', () => { + it.each([ + ['a name excluded by an exclusion entry', { tags: ['!internal', '*'] }, 'internal', true], + ['a name matched by a wildcard exclusion', { tags: ['!int*'] }, 'internal', true], + ['a name the exclusions do not match', { tags: ['!internal', '*'] }, 'public', false], + ['a type set to []', { tags: [] }, 'internal', false], + ['a list that just does not name it', { tags: ['public'] }, 'internal', false], + ])('should report %s', (_label, filter, name, expected) => { + expect(isExplicitlyExcluded({ type: ResourceType.Tag, nameParts: [name] }, filter as FilterConfig)).toBe(expected); + }); + + it("should check a workspace resource against its workspace's sub-filter", () => { + const filter: FilterConfig = { + tags: ['!shared'], + workspaces: ['team-a'], + workspaceSubFilters: { 'team-a': { tags: ['!internal'] } }, + }; + + expect(isExplicitlyExcluded({ type: ResourceType.Tag, nameParts: ['internal'], workspace: 'team-a' }, filter)).toBe(true); + expect(isExplicitlyExcluded({ type: ResourceType.Tag, nameParts: ['shared'], workspace: 'team-a' }, filter)).toBe(false); + expect(isExplicitlyExcluded({ type: ResourceType.Tag, nameParts: ['shared'] }, filter)).toBe(true); + }); + }); + + describe('links to tags excluded by name', () => { + const link = (tag: string): ResourceDescriptor => ({ type: ResourceType.ApiTag, nameParts: ['orders', tag] }); + + it("should leave out an API's link to a tag the filter excludes by name", () => { + const filter: FilterConfig = { apis: ['orders'], tags: ['!internal', '*'] }; + + expect(shouldIncludeResource(link('internal'), filter)).toBe(false); + expect(shouldIncludeResource(link('public'), filter)).toBe(true); + }); + + it('should keep links to tags the filter leaves out with [] (they are pulled in as dependencies)', () => { + expect(shouldIncludeResource(link('internal'), { apis: ['orders'], tags: [] })).toBe(true); + }); + }); + describe('extractRootApiName', () => { it('should return name as-is when no revision suffix', () => { expect(extractRootApiName('my-api')).toBe('my-api'); diff --git a/tests/unit/services/product-publisher.test.ts b/tests/unit/services/product-publisher.test.ts index 27d1a833..a5edbc6f 100644 --- a/tests/unit/services/product-publisher.test.ts +++ b/tests/unit/services/product-publisher.test.ts @@ -12,7 +12,7 @@ import { import { ResourceType } from '../../../src/models/resource-types.js'; import { ApimServiceContext, ResourceDescriptor } from '../../../src/models/types.js'; import { PublishConfig } from '../../../src/models/config.js'; -import { LogLevel } from '../../../src/lib/logger.js'; +import { LogLevel, logger } from '../../../src/lib/logger.js'; import { DEFAULT_APPLIES_TO } from '../../../src/services/env-mapper.js'; // Mock resource-publisher so product-publisher tests don't run full resource-publisher logic @@ -268,6 +268,78 @@ describe('product-publisher', () => { ); }); + it('links groups and tags that were pulled into the publish set although the filter excludes them', async () => { + const client = createMockClient(); + const store = createMockStore(); + store.readAssociation.mockImplementation(async (_dir: string, _descriptor: ResourceDescriptor, type: string) => + type === 'groups' ? [{ name: 'support-team' }] : type === 'tags' ? [{ name: 'partner' }, { name: 'internal' }] : [] + ); + const config: PublishConfig = { + ...testConfig, + filter: { products: ['my-product'], groups: [], tags: [] }, + }; + + await publishProduct(client, store, testContext, productDescriptor, config, [ + productDescriptor, + { type: ResourceType.Group, nameParts: ['support-team'] }, + { type: ResourceType.Tag, nameParts: ['partner'] }, + ]); + + const links = client.putResource.mock.calls + .map(([, descriptor]) => `${descriptor.type}:${descriptor.nameParts.join('/')}`) + .filter((key) => key.startsWith('ProductGroup') || key.startsWith('ProductTag')); + expect(links).toEqual(['ProductGroup:my-product/support-team', 'ProductTag:my-product/partner']); + }); + + it('publishes links to built-in groups only when the filter includes them', async () => { + const client = createMockClient(); + const store = createMockStore(); + store.readAssociation.mockImplementation(async (_dir: string, _descriptor: ResourceDescriptor, type: string) => + type === 'groups' ? [{ name: 'developers' }] : [] + ); + const developers: ResourceDescriptor = { type: ResourceType.Group, nameParts: ['developers'] }; + const linksFor = async (groups: string[]) => { + client.putResource.mockClear(); + // A group the filter includes is in the publish set, as after extracting it. + await publishProduct(client, store, testContext, productDescriptor, { + ...testConfig, + filter: { products: ['my-product'], groups }, + }, groups.length > 0 ? [productDescriptor, developers] : [productDescriptor]); + return client.putResource.mock.calls + .map(([, descriptor]) => `${descriptor.type}:${descriptor.nameParts.join('/')}`) + .filter((key) => key.startsWith('ProductGroup')); + }; + + expect(await linksFor([])).toEqual([]); + expect(await linksFor(['developers'])).toEqual(['ProductGroup:my-product/developers']); + }); + + it('publishes the link APIM adds to administrators whatever the groups filter, unless it excludes the group by name', async () => { + const client = createMockClient(); + const store = createMockStore(); + store.readAssociation.mockImplementation(async (_dir: string, _descriptor: ResourceDescriptor, type: string) => + type === 'groups' ? [{ name: 'administrators' }] : [] + ); + const warn = vi.spyOn(logger, 'warn'); + const linksFor = async (groups: string[]) => { + client.putResource.mockClear(); + warn.mockClear(); + await publishProduct(client, store, testContext, productDescriptor, { + ...testConfig, + filter: { products: ['my-product'], groups }, + }, [productDescriptor]); + return client.putResource.mock.calls + .map(([, descriptor]) => `${descriptor.type}:${descriptor.nameParts.join('/')}`) + .filter((key) => key.startsWith('ProductGroup')); + }; + + expect(await linksFor([])).toEqual(['ProductGroup:my-product/administrators']); + expect(await linksFor(['support-team'])).toEqual(['ProductGroup:my-product/administrators']); + expect(warn).not.toHaveBeenCalled(); + expect(await linksFor(['!administrators'])).toEqual([]); + warn.mockRestore(); + }); + it('does not let resolved targets override explicit API exclusions', async () => { const client = createMockClient(); const store = createMockStore(); diff --git a/tests/unit/services/publish-service.test.ts b/tests/unit/services/publish-service.test.ts index 6bcac061..cc441d6b 100644 --- a/tests/unit/services/publish-service.test.ts +++ b/tests/unit/services/publish-service.test.ts @@ -7,8 +7,8 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { ResourceType } from '../../../src/models/resource-types.js'; import { ResourceDescriptor, ApimServiceContext } from '../../../src/models/types.js'; -import { PublishConfig } from '../../../src/models/config.js'; -import { LogLevel } from '../../../src/lib/logger.js'; +import { FilterConfig, PublishConfig } from '../../../src/models/config.js'; +import { LogLevel, logger } from '../../../src/lib/logger.js'; import { recordRetry, withRetryResource } from '../../../src/lib/retry-tracker.js'; import { getResourceDescriptorKey } from '../../../src/lib/resource-path.js'; @@ -482,6 +482,434 @@ describe('publish-service', () => { expect(publishApi).not.toHaveBeenCalled(); }); + it("should include the backend a policy attribute names through a named value's value (#331)", async () => { + const resources = [ + { type: ResourceType.PolicyFragment, nameParts: ['route'] }, + { type: ResourceType.NamedValue, nameParts: ['backend-id'] }, + { type: ResourceType.Backend, nameParts: ['function-backend'] }, + { type: ResourceType.Backend, nameParts: ['unused'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + store.readResource.mockImplementation(async (_sourceDir, descriptor) => { + if (descriptor.type === ResourceType.PolicyFragment) { + return { properties: { value: '' } }; + } + if (descriptor.type === ResourceType.NamedValue) { + return { properties: { displayName: 'Backend_Id', value: 'function-backend' } }; + } + return { properties: { url: 'https://example.test' } }; + }); + + const publish = (filter: FilterConfig) => runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + const backendPuts = () => client.putResource.mock.calls + .filter((call) => call[1].type === ResourceType.Backend) + .map((call) => call[1].nameParts[0]); + + await publish({ policyFragments: ['route'], namedValues: [], backends: [] }); + expect(backendPuts()).toEqual(['function-backend']); + + // A named value the filter excludes isn't published, so neither is what it names. + client.putResource.mockClear(); + await publish({ policyFragments: ['route'], namedValues: ['!backend-id'], backends: [] }); + expect(backendPuts()).toEqual([]); + }); + + it("should follow the overridden value of a named value a policy attribute is set to", async () => { + const resources = [ + { type: ResourceType.PolicyFragment, nameParts: ['route'] }, + { type: ResourceType.NamedValue, nameParts: ['backend-id'] }, + { type: ResourceType.Backend, nameParts: ['dev-backend'] }, + { type: ResourceType.Backend, nameParts: ['prod-backend'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + store.readResource.mockImplementation(async (_sourceDir, descriptor) => { + if (descriptor.type === ResourceType.PolicyFragment) { + return { properties: { value: '' } }; + } + if (descriptor.type === ResourceType.NamedValue) { + return { properties: { displayName: 'Backend_Id', value: 'dev-backend' } }; + } + return { properties: { url: 'https://example.test' } }; + }); + + await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { policyFragments: ['route'], namedValues: [], backends: [] }, + overrides: { namedValues: { 'backend-id': { properties: { value: 'prod-backend' } } } }, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + const backendPuts = client.putResource.mock.calls + .filter((call) => call[1].type === ResourceType.Backend) + .map((call) => call[1].nameParts[0]); + expect(backendPuts).toEqual(['prod-backend']); + }); + + it("should pull in the named values a backend's overridden credentials use", async () => { + const resources = [ + { type: ResourceType.Backend, nameParts: ['api-backend'] }, + { type: ResourceType.NamedValue, nameParts: ['dev-key'] }, + { type: ResourceType.NamedValue, nameParts: ['prod-key'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + store.readResource.mockImplementation(async (_sourceDir, descriptor) => + descriptor.type === ResourceType.Backend + ? { properties: { url: 'https://example.test', credentials: { header: { 'x-key': ['{{dev-key}}'] } } } } + : { properties: { displayName: descriptor.nameParts[0], value: 'secret' } } + ); + + await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { backends: ['api-backend'], namedValues: [] }, + overrides: { backends: { 'api-backend': { properties: { credentials: { header: { 'x-key': ['{{prod-key}}'] } } } } } }, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + const namedValuePuts = client.putResource.mock.calls + .filter((call) => call[1].type === ResourceType.NamedValue) + .map((call) => call[1].nameParts[0]); + expect(namedValuePuts).toEqual(['prod-key']); + }); + + it('should include the backend a changed named value now names when only the named value changed', async () => { + const namedValue = { type: ResourceType.NamedValue, nameParts: ['backend-id'] }; + const resources = [ + { type: ResourceType.Api, nameParts: ['orders'] }, + { type: ResourceType.ApiPolicy, nameParts: ['orders'] }, + namedValue, + { type: ResourceType.Backend, nameParts: ['old-backend'] }, + { type: ResourceType.Backend, nameParts: ['new-backend'] }, + ]; + vi.mocked(computeGitDiff).mockResolvedValue({ changedDescriptors: [namedValue], deletedDescriptors: [] }); + const client = createMockClient(); + const store = createMockStore(resources); + store.readContent.mockImplementation(async (_sourceDir: string, descriptor: ResourceDescriptor) => + descriptor.type === ResourceType.ApiPolicy + ? { content: '' } + : undefined + ); + store.readResource.mockImplementation(async (_sourceDir, descriptor) => + descriptor.type === ResourceType.NamedValue + ? { properties: { displayName: 'Backend_Id', value: 'new-backend' } } + : { properties: {} } + ); + + await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { apis: ['orders'], namedValues: ['backend-id'], backends: [] }, + commitId: 'abc123', + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + const puts = client.putResource.mock.calls.map((call) => `${call[1].type}:${call[1].nameParts.join('/')}`).sort(); + expect(puts).toEqual(['Backend:new-backend', 'NamedValue:backend-id']); + }); + + describe('with a policy whose XML an override replaces', () => { + const servicePolicy: ResourceDescriptor = { type: ResourceType.ServicePolicy, nameParts: [] }; + const namedValue: ResourceDescriptor = { type: ResourceType.NamedValue, nameParts: ['backend-id'] }; + const overrides = { + policies: { policy: { properties: { value: '' } } }, + }; + const publishWith = async (filter: FilterConfig, commitId?: string) => { + const client = createMockClient(); + const store = createMockStore([servicePolicy, namedValue, { type: ResourceType.Backend, nameParts: ['new-backend'] }]); + store.readContent.mockImplementation(async (_sourceDir: string, descriptor: ResourceDescriptor) => + descriptor.type === ResourceType.ServicePolicy ? { content: '' } : undefined + ); + store.readResource.mockImplementation(async (_sourceDir, descriptor) => + descriptor.type === ResourceType.NamedValue + ? { properties: { displayName: 'Backend_Id', value: 'new-backend' } } + : { properties: {} } + ); + await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter, + overrides, + commitId, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + return client.putResource.mock.calls.map((call) => `${call[1].type}:${call[1].nameParts.join('/')}`).sort(); + }; + + it('should follow the overridden XML of a published policy', async () => { + expect(await publishWith({ namedValues: [], backends: [] })).toEqual([ + 'Backend:new-backend', + 'NamedValue:backend-id', + 'ServicePolicy:', + ]); + }); + + it('should include the backend a changed named value names in the overridden XML', async () => { + vi.mocked(computeGitDiff).mockResolvedValue({ changedDescriptors: [namedValue], deletedDescriptors: [] }); + + expect(await publishWith({ namedValues: ['backend-id'], backends: [] }, 'abc123')).toEqual([ + 'Backend:new-backend', + 'NamedValue:backend-id', + ]); + }); + }); + + it('should include the backend a changed named value names in a fragment a selected policy includes', async () => { + const namedValue = { type: ResourceType.NamedValue, nameParts: ['backend-id'] }; + const resources = [ + { type: ResourceType.Api, nameParts: ['orders'] }, + { type: ResourceType.ApiPolicy, nameParts: ['orders'] }, + { type: ResourceType.PolicyFragment, nameParts: ['route'] }, + namedValue, + { type: ResourceType.Backend, nameParts: ['new-backend'] }, + ]; + vi.mocked(computeGitDiff).mockResolvedValue({ changedDescriptors: [namedValue], deletedDescriptors: [] }); + const client = createMockClient(); + const store = createMockStore(resources); + store.readContent.mockImplementation(async (_sourceDir: string, descriptor: ResourceDescriptor) => + descriptor.type === ResourceType.ApiPolicy + ? { content: '' } + : undefined + ); + store.readResource.mockImplementation(async (_sourceDir, descriptor) => { + if (descriptor.type === ResourceType.PolicyFragment) { + return { properties: { value: '' } }; + } + return descriptor.type === ResourceType.NamedValue + ? { properties: { displayName: 'Backend_Id', value: 'new-backend' } } + : { properties: {} }; + }); + + await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { apis: ['orders'], namedValues: ['backend-id'], policyFragments: [], backends: [] }, + commitId: 'abc123', + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + const puts = client.putResource.mock.calls.map((call) => `${call[1].type}:${call[1].nameParts.join('/')}`).sort(); + expect(puts).toEqual(['Backend:new-backend', 'NamedValue:backend-id']); + }); + + it("should include a named value a selected fragment's Liquid body references", async () => { + const resources = [ + { type: ResourceType.PolicyFragment, nameParts: ['route'] }, + { type: ResourceType.NamedValue, nameParts: ['frag-nv'] }, + { type: ResourceType.NamedValue, nameParts: ['unused'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + store.readResource.mockImplementation(async (_sourceDir, descriptor) => { + if (descriptor.type === ResourceType.PolicyFragment) { + return { properties: { value: '{{Frag_NV}} {{body.id}}' } }; + } + return { properties: { displayName: descriptor.nameParts[0] === 'frag-nv' ? 'Frag_NV' : 'unused', value: 'x' } }; + }); + + await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { policyFragments: ['route'], namedValues: [] }, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + const namedValuePuts = client.putResource.mock.calls + .filter((call) => call[1].type === ResourceType.NamedValue) + .map((call) => call[1].nameParts[0]); + expect(namedValuePuts).toEqual(['frag-nv']); + }); + + it('should include a named value a Liquid body references, and nothing for Liquid output', async () => { + const resources = [ + { type: ResourceType.Api, nameParts: ['orders'] }, + { type: ResourceType.ApiPolicy, nameParts: ['orders'] }, + { type: ResourceType.NamedValue, nameParts: ['backend-id'] }, + { type: ResourceType.NamedValue, nameParts: ['unused'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + store.readContent.mockImplementation(async (_sourceDir: string, descriptor: ResourceDescriptor) => + descriptor.type === ResourceType.ApiPolicy + ? { content: 'nv=[{{Backend_Id}}] liquid=[{{body.id}}]' } + : undefined + ); + store.readResource.mockImplementation(async (_sourceDir, descriptor) => + descriptor.type === ResourceType.NamedValue + ? { properties: { displayName: descriptor.nameParts[0] === 'backend-id' ? 'Backend_Id' : 'unused', value: 'x' } } + : { properties: {} } + ); + + await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { apis: ['orders'], namedValues: [] }, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + const namedValuePuts = client.putResource.mock.calls + .filter((call) => call[1].type === ResourceType.NamedValue) + .map((call) => call[1].nameParts[0]); + expect(namedValuePuts).toEqual(['backend-id']); + }); + + describe('a named value a policy outside the filter uses as a resource name', () => { + const setUp = (extra: ResourceDescriptor[] = [{ type: ResourceType.Backend, nameParts: ['billing-backend'] }]) => { + const resources = [ + { type: ResourceType.Api, nameParts: ['orders'] }, + { type: ResourceType.Api, nameParts: ['billing'] }, + { type: ResourceType.ApiPolicy, nameParts: ['billing'] }, + { type: ResourceType.NamedValue, nameParts: ['billing-id'] }, + ...extra, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + store.readContent.mockImplementation(async (_sourceDir: string, descriptor: ResourceDescriptor) => + descriptor.type === ResourceType.ApiPolicy + ? { content: '' } + : undefined + ); + store.readResource.mockImplementation(async (_sourceDir, descriptor) => + descriptor.type === ResourceType.NamedValue + ? { properties: { displayName: 'billing-id', value: 'billing-backend' } } + : { properties: {} } + ); + const publish = (filter: FilterConfig) => runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + const puts = () => client.putResource.mock.calls.map((call) => `${call[1].type}:${call[1].nameParts.join('/')}`).sort(); + return { publish, puts, store }; + }; + + it('should read that policy once, though both the expansion and the order use it', async () => { + const { publish, store } = setUp(); + + await publish({ apis: ['orders'], backends: [] }); + + const billingPolicyReads = store.readContent.mock.calls.filter( + (call) => call[1].type === ResourceType.ApiPolicy && call[1].nameParts[0] === 'billing' + ); + expect(billingPolicyReads).toHaveLength(1); + }); + + it('should include the resource its value names, as APIM checks that policy on the target', async () => { + const { publish, puts } = setUp(); + + // Every named value is selected (the key is left out); backends: [] still lets dependencies in. + await publish({ apis: ['orders'], backends: [] }); + + expect(puts()).toEqual(expect.arrayContaining(['Backend:billing-backend', 'NamedValue:billing-id'])); + expect(puts()).not.toContain('Api:billing'); + }); + + it.each([ + ['the filter excludes it by name', { apis: ['orders'], backends: ['!billing-backend'] }, undefined], + ["it isn't in the source", { apis: ['orders'], backends: [] }, []], + ])('should not publish the resource when %s', async (_label, filter, extra) => { + const { publish, puts } = setUp(extra as ResourceDescriptor[] | undefined); + + await publish(filter as FilterConfig); + + expect(puts()).toContain('NamedValue:billing-id'); + expect(puts()).not.toContain('Backend:billing-backend'); + }); + }); + + it('should prefer the named value whose display name a reference uses over one with that resource name', async () => { + const resources = [ + { type: ResourceType.PolicyFragment, nameParts: ['auth'] }, + { type: ResourceType.NamedValue, nameParts: ['one'] }, + { type: ResourceType.NamedValue, nameParts: ['two'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + store.readResource.mockImplementation(async (_sourceDir, descriptor) => { + if (descriptor.type === ResourceType.PolicyFragment) { + return { properties: { value: '{{TWO}}' } }; + } + return { properties: { displayName: descriptor.nameParts[0] === 'one' ? 'two' : 'three', value: 'x' } }; + }); + + await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { policyFragments: ['auth'], namedValues: [] }, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + const namedValuePuts = client.putResource.mock.calls + .filter((call) => call[1].type === ResourceType.NamedValue) + .map((call) => call[1].nameParts[0]); + expect(namedValuePuts).toEqual(['one']); + }); + + it('should warn and match by resource name when a named value artifact cannot be read', async () => { + const warn = vi.spyOn(logger, 'warn').mockImplementation(() => undefined); + const resources = [ + { type: ResourceType.PolicyFragment, nameParts: ['auth'] }, + { type: ResourceType.NamedValue, nameParts: ['broken'] }, + { type: ResourceType.NamedValue, nameParts: ['plain'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + let namedValueReads = 0; + store.readResource.mockImplementation(async (_sourceDir, descriptor) => { + if (descriptor.type === ResourceType.PolicyFragment) { + return { properties: { value: '{{plain}}' } }; + } + if (descriptor.nameParts[0] === 'broken' && ++namedValueReads === 1) { + throw new Error('Unexpected token in JSON'); + } + return { properties: { displayName: descriptor.nameParts[0], value: 'x' } }; + }); + + await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { policyFragments: ['auth'], namedValues: [] }, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + expect(warn).toHaveBeenCalledWith(expect.stringContaining('to match {{references}} by display name')); + expect(client.putResource.mock.calls + .filter((call) => call[1].type === ResourceType.NamedValue) + .map((call) => call[1].nameParts[0])).toEqual(['plain']); + warn.mockRestore(); + }); + it('should include backend pool members transitively', async () => { const resources = [ { type: ResourceType.Backend, nameParts: ['pool'] }, @@ -520,6 +948,71 @@ describe('publish-service', () => { )).toBe(true); }); + it("should include an API diagnostic's logger, an API's tags, and a product's groups and tags transitively", async () => { + const resources = [ + { type: ResourceType.Api, nameParts: ['orders'] }, + { type: ResourceType.ApiDiagnostic, nameParts: ['orders', 'applicationinsights'] }, + { type: ResourceType.ApiTag, nameParts: ['orders', 'public'] }, + { type: ResourceType.Product, nameParts: ['starter'] }, + { type: ResourceType.Logger, nameParts: ['app-insights'] }, + { type: ResourceType.Logger, nameParts: ['unrelated'] }, + { type: ResourceType.Tag, nameParts: ['public'] }, + { type: ResourceType.Tag, nameParts: ['partner'] }, + { type: ResourceType.Group, nameParts: ['support-team'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + store.readResource.mockImplementation(async (_sourceDir, descriptor) => + descriptor.type === ResourceType.ApiDiagnostic + ? { properties: { loggerId: '/loggers/app-insights' } } + : { properties: {} } + ); + store.readAssociation.mockImplementation(async (_sourceDir, _descriptor, type) => + type === 'groups' ? [{ name: 'support-team' }] : type === 'tags' ? [{ name: 'partner' }] : [] + ); + + await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { apis: ['orders'], products: ['starter'], loggers: [], tags: [], groups: [] }, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + const put = client.putResource.mock.calls.map((call) => `${call[1].type}:${call[1].nameParts.join('/')}`); + expect(put).toEqual(expect.arrayContaining([ + 'Logger:app-insights', + 'Tag:public', + 'Tag:partner', + 'Group:support-team', + ])); + expect(put).not.toContain('Logger:unrelated'); + }); + + it('should not pull in a tag the filter excludes by name', async () => { + const resources = [ + { type: ResourceType.Api, nameParts: ['orders'] }, + { type: ResourceType.ApiTag, nameParts: ['orders', 'internal'] }, + { type: ResourceType.Tag, nameParts: ['internal'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + store.readResource.mockResolvedValue({ properties: {} }); + + await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { apis: ['orders'], tags: ['!internal'] }, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + const put = client.putResource.mock.calls.map((call) => `${call[1].type}:${call[1].nameParts.join('/')}`); + expect(put).not.toContain('Tag:internal'); + }); + it('should not include transitive dependencies when disabled', async () => { const resources = [ { type: ResourceType.Api, nameParts: ['orders'] }, @@ -1608,6 +2101,250 @@ describe('publish-service', () => { }); describe('named value ordering within tier 1', () => { + const orderingCase = async ( + extra: ResourceDescriptor[], + policy: string, + overrides?: PublishConfig['overrides'] + ): Promise => { + const publishOrder: string[] = []; + const resources: ResourceDescriptor[] = [ + { type: ResourceType.NamedValue, nameParts: ['backend-id'] }, + { type: ResourceType.NamedValue, nameParts: ['api-key'] }, + { type: ResourceType.Backend, nameParts: ['prod-backend'] }, + { type: ResourceType.Backend, nameParts: ['dev-backend'] }, + { type: ResourceType.PolicyFragment, nameParts: ['route'] }, + ...extra, + ]; + const client = createMockClient(); + vi.mocked(client.putResource).mockImplementation(async (_ctx, descriptor) => { + publishOrder.push(descriptor.nameParts.join('/')); + }); + const store = createMockStore(resources); + vi.mocked(store.readResource).mockImplementation(async (_sourceDir, descriptor) => { + if (descriptor.type === ResourceType.NamedValue) { + const value = { 'backend-id': 'dev-backend', 'api-key': 'prod-backend', 'schema-id': 'order-schema' }[descriptor.nameParts[0] ?? '']; + return { properties: { displayName: descriptor.nameParts[0], value } }; + } + if (descriptor.type === ResourceType.PolicyFragment) { + return { properties: { value: policy } }; + } + return { properties: { url: 'https://example.test' } }; + }); + + await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + overrides, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + return publishOrder; + }; + + it('should publish a named value a policy uses as a backend after the backend its value names, and before fragments', async () => { + const order = await orderingCase( + [], + '', + { namedValues: { 'backend-id': { properties: { value: 'prod-backend' } } } } + ); + + expect(order.indexOf('backend-id')).toBeGreaterThan(order.indexOf('prod-backend')); + expect(order.indexOf('backend-id')).toBeGreaterThan(order.indexOf('dev-backend')); + // The fragment uses the named value, so it goes after it. + expect(order.indexOf('route')).toBeGreaterThan(order.indexOf('backend-id')); + // api-key's value matches a backend's name, but no policy uses it as one. + expect(order.indexOf('api-key')).toBeLessThan(order.indexOf('prod-backend')); + }); + + it('should publish a named value a policy uses as a schema after the schema its value names', async () => { + const order = await orderingCase( + [ + { type: ResourceType.NamedValue, nameParts: ['schema-id'] }, + { type: ResourceType.GlobalSchema, nameParts: ['order-schema'] }, + ], + '' + ); + + expect(order.indexOf('schema-id')).toBeGreaterThan(order.indexOf('order-schema')); + expect(order.indexOf('backend-id')).toBeLessThan(order.indexOf('dev-backend')); + }); + + const recordOrder = (resources: ResourceDescriptor[]) => { + const order: string[] = []; + const client = createMockClient(); + vi.mocked(client.putResource).mockImplementation(async (_ctx, descriptor) => { + order.push(descriptor.nameParts.join('/')); + }); + return { order, client, store: createMockStore(resources) }; + }; + const publishAll = (client: ReturnType, store: ReturnType, filter?: FilterConfig) => + runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + it('should publish a named value naming a pool backend after the pool', async () => { + const { order, client, store } = recordOrder([ + { type: ResourceType.NamedValue, nameParts: ['pool-id'] }, + { type: ResourceType.Backend, nameParts: ['member'] }, + { type: ResourceType.Backend, nameParts: ['my-pool'] }, + { type: ResourceType.PolicyFragment, nameParts: ['route'] }, + ]); + vi.mocked(store.readResource).mockImplementation(async (_sourceDir, descriptor) => { + if (descriptor.type === ResourceType.NamedValue) { + return { properties: { displayName: 'pool-id', value: 'my-pool' } }; + } + if (descriptor.type === ResourceType.PolicyFragment) { + return { properties: { value: '' } }; + } + return descriptor.nameParts[0] === 'my-pool' + ? { properties: { type: 'Pool', pool: { services: [{ id: '/backends/member' }] } } } + : { properties: { url: 'https://example.test' } }; + }); + + await publishAll(client, store); + + expect(order.indexOf('my-pool')).toBeGreaterThan(order.indexOf('member')); + expect(order.indexOf('pool-id')).toBeGreaterThan(order.indexOf('my-pool')); + expect(order.indexOf('route')).toBeGreaterThan(order.indexOf('pool-id')); + }); + + it("should keep a named value first when a backend published with it uses it in its credentials", async () => { + const { order, client, store } = recordOrder([ + { type: ResourceType.NamedValue, nameParts: ['shared'] }, + { type: ResourceType.Backend, nameParts: ['prod-backend'] }, + { type: ResourceType.PolicyFragment, nameParts: ['route'] }, + ]); + vi.mocked(store.readResource).mockImplementation(async (_sourceDir, descriptor) => { + if (descriptor.type === ResourceType.NamedValue) { + return { properties: { displayName: 'shared', value: 'prod-backend' } }; + } + if (descriptor.type === ResourceType.PolicyFragment) { + return { properties: { value: '' } }; + } + return { properties: { url: 'https://example.test', credentials: { header: { 'x-route': ['{{shared}}'] } } } }; + }); + + await publishAll(client, store); + + expect(order.indexOf('shared')).toBeLessThan(order.indexOf('prod-backend')); + }); + + it('should publish a backend using a named value in its credentials after the named value and the backend it names', async () => { + const { order, client, store } = recordOrder([ + { type: ResourceType.NamedValue, nameParts: ['route'] }, + { type: ResourceType.Backend, nameParts: ['new-backend'] }, + { type: ResourceType.Backend, nameParts: ['caller'] }, + { type: ResourceType.PolicyFragment, nameParts: ['frag'] }, + ]); + vi.mocked(store.readResource).mockImplementation(async (_sourceDir, descriptor) => { + if (descriptor.type === ResourceType.NamedValue) { + return { properties: { displayName: 'route', value: 'new-backend' } }; + } + if (descriptor.type === ResourceType.PolicyFragment) { + return { properties: { value: '' } }; + } + return descriptor.nameParts[0] === 'caller' + ? { properties: { url: 'https://example.test', credentials: { header: { 'x-route': ['{{route}}'] } } } } + : { properties: { url: 'https://example.test' } }; + }); + + await publishAll(client, store); + + expect(order.indexOf('route')).toBeGreaterThan(order.indexOf('new-backend')); + expect(order.indexOf('caller')).toBeGreaterThan(order.indexOf('route')); + expect(order.indexOf('frag')).toBeGreaterThan(order.indexOf('route')); + }); + + it('should order a chain of named values and the backends using them in their credentials', async () => { + const { order, client, store } = recordOrder([ + { type: ResourceType.NamedValue, nameParts: ['route-a'] }, + { type: ResourceType.NamedValue, nameParts: ['route-b'] }, + { type: ResourceType.Backend, nameParts: ['new-a'] }, + { type: ResourceType.Backend, nameParts: ['new-b'] }, + { type: ResourceType.PolicyFragment, nameParts: ['frag'] }, + ]); + vi.mocked(store.readResource).mockImplementation(async (_sourceDir, descriptor) => { + if (descriptor.type === ResourceType.NamedValue) { + const name = descriptor.nameParts[0]; + return { properties: { displayName: name, value: name === 'route-a' ? 'new-a' : 'new-b' } }; + } + if (descriptor.type === ResourceType.PolicyFragment) { + return { + properties: { + value: '', + }, + }; + } + return descriptor.nameParts[0] === 'new-a' + ? { properties: { url: 'https://example.test', credentials: { header: { 'x-route': ['{{route-b}}'] } } } } + : { properties: { url: 'https://example.test' } }; + }); + + await publishAll(client, store); + + expect(order.indexOf('route-b')).toBeGreaterThan(order.indexOf('new-b')); + expect(order.indexOf('new-a')).toBeGreaterThan(order.indexOf('route-b')); + expect(order.indexOf('route-a')).toBeGreaterThan(order.indexOf('new-a')); + expect(order.indexOf('frag')).toBeGreaterThan(order.indexOf('route-a')); + }); + + it('should keep a named value first when a pool member uses it in its credentials', async () => { + const { order, client, store } = recordOrder([ + { type: ResourceType.NamedValue, nameParts: ['route'] }, + { type: ResourceType.Backend, nameParts: ['member'] }, + { type: ResourceType.Backend, nameParts: ['my-pool'] }, + { type: ResourceType.PolicyFragment, nameParts: ['frag'] }, + ]); + vi.mocked(store.readResource).mockImplementation(async (_sourceDir, descriptor) => { + if (descriptor.type === ResourceType.NamedValue) { + return { properties: { displayName: 'route', value: 'my-pool' } }; + } + if (descriptor.type === ResourceType.PolicyFragment) { + return { properties: { value: '' } }; + } + return descriptor.nameParts[0] === 'my-pool' + ? { properties: { type: 'Pool', pool: { services: [{ id: '/backends/member' }] } } } + : { properties: { url: 'https://example.test', credentials: { header: { 'x-route': ['{{route}}'] } } } }; + }); + + await publishAll(client, store); + + expect(order.indexOf('route')).toBeLessThan(order.indexOf('member')); + expect(order.indexOf('my-pool')).toBeGreaterThan(order.indexOf('member')); + }); + + it('should order by a policy the filter leaves out, since APIM checks it on the target', async () => { + const { order, client, store } = recordOrder([ + { type: ResourceType.Api, nameParts: ['orders'] }, + { type: ResourceType.Api, nameParts: ['billing'] }, + { type: ResourceType.ApiPolicy, nameParts: ['billing'] }, + { type: ResourceType.NamedValue, nameParts: ['billing-id'] }, + { type: ResourceType.Backend, nameParts: ['new-billing'] }, + ]); + store.readContent.mockImplementation(async (_sourceDir: string, descriptor: ResourceDescriptor) => + descriptor.type === ResourceType.ApiPolicy + ? { content: '' } + : undefined + ); + vi.mocked(store.readResource).mockImplementation(async (_sourceDir, descriptor) => + descriptor.type === ResourceType.NamedValue + ? { properties: { displayName: 'billing-id', value: 'new-billing' } } + : { properties: { url: 'https://example.test' } } + ); + + // Named values and backends are left out of the filter, so all are selected. + await publishAll(client, store, { apis: ['orders'] }); + + expect(order).not.toContain('billing'); + expect(order.indexOf('billing-id')).toBeGreaterThan(order.indexOf('new-billing')); + }); + it('should publish named values before loggers within tier 1', async () => { const publishOrder: string[] = []; diff --git a/tests/unit/services/resource-publisher.test.ts b/tests/unit/services/resource-publisher.test.ts index 8e1154e5..aab77ef8 100644 --- a/tests/unit/services/resource-publisher.test.ts +++ b/tests/unit/services/resource-publisher.test.ts @@ -749,6 +749,37 @@ describe('resource-publisher', () => { expect(client.putResource).not.toHaveBeenCalled(); }); + it('should publish an ApiTag link whose excluded Tag was pulled into the publish set', async () => { + const client = createMockClient(); + const store = createMockStore(); + store.readResource.mockResolvedValue({ properties: { displayName: 'public' } }); + const descriptor: ResourceDescriptor = { type: ResourceType.ApiTag, nameParts: ['orders', 'public'] }; + const tag: ResourceDescriptor = { type: ResourceType.Tag, nameParts: ['public'] }; + const config: PublishConfig = { ...testConfig, filter: { apis: ['orders'], tags: [] } }; + + const pulledIn = await publishResource(client, store, testContext, descriptor, config, [descriptor, tag]); + const notPulledIn = await publishResource(client, store, testContext, descriptor, config, [descriptor]); + + expect(pulledIn.status).toBe('success'); + expect(notPulledIn.status).toBe('skipped'); + expect(client.putResource).toHaveBeenCalledTimes(1); + }); + + it('should skip a link to a target in the publish set that the filter excludes by name', async () => { + const client = createMockClient(); + const store = createMockStore(); + const descriptor: ResourceDescriptor = { type: ResourceType.ApiTag, nameParts: ['orders', 'internal'] }; + const config: PublishConfig = { ...testConfig, filter: { apis: ['orders'], tags: ['!internal', '*'] } }; + + const result = await publishResource(client, store, testContext, descriptor, config, [ + descriptor, + { type: ResourceType.Tag, nameParts: ['internal'] }, + ]); + + expect(result.status).toBe('skipped'); + expect(client.putResource).not.toHaveBeenCalled(); + }); + it('should publish a GatewayApi association when its target passes an explicit filter and was extracted', async () => { const client = createMockClient(); const store = createMockStore(); @@ -2146,6 +2177,35 @@ describe('resource-publisher', () => { expect(creds.instrumentationKey).toBe('{{Logger-Credentials--abc123}}'); }); + it("should keep a reference that is another named value's display name pointing at that named value", async () => { + const client = createMockClient(); + const store = createMockStore(); + const namedValue = (name: string): ResourceDescriptor => ({ type: ResourceType.NamedValue, nameParts: [name] }); + store.listResources.mockResolvedValue([namedValue('one'), namedValue('two')]); + store.readResource.mockImplementation(async (_dir: string, desc: ResourceDescriptor) => { + if (desc.type === ResourceType.Logger) { + return { name: 'my-logger', properties: { loggerType: 'applicationInsights', credentials: { instrumentationKey: '{{two}}' } } }; + } + if (desc.type === ResourceType.NamedValue) { + return { name: desc.nameParts[0], properties: { displayName: desc.nameParts[0] === 'one' ? 'two' : 'three' } }; + } + return null; + }); + + const loggerDescriptor: ResourceDescriptor = { type: ResourceType.Logger, nameParts: ['my-logger'] }; + const sentKey = () => + (client.putResource.mock.calls.at(-1)?.[2] as { properties: { credentials: Record } }) + .properties.credentials.instrumentationKey; + + await publishResource(client, store, testContext, loggerDescriptor, testConfig); + expect(sentKey()).toBe('{{two}}'); + + // With a prefix, the reference follows that named value's deployed display name. + const envMapping = { prefix: 'dev-', suffix: '', appliesTo: new Set([ResourceType.NamedValue]) }; + await publishResource(client, store, testContext, loggerDescriptor, { ...testConfig, envMapping }); + expect(sentKey()).toBe('{{dev-two}}'); + }); + it('should keep an explicit workspace named-value displayName unprefixed in logger credentials', async () => { const client = createMockClient(); const store = createMockStore(); diff --git a/tests/unit/services/stale-dependency-finder.test.ts b/tests/unit/services/stale-dependency-finder.test.ts new file mode 100644 index 00000000..a07270d5 --- /dev/null +++ b/tests/unit/services/stale-dependency-finder.test.ts @@ -0,0 +1,321 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +/** + * Unit tests for finding stale pulled-in dependencies (extract --remove-stale) + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { ArtifactStore } from '../../../src/clients/artifact-store.js'; +import type { IArtifactStore } from '../../../src/clients/iartifact-store.js'; +import { logger } from '../../../src/lib/logger.js'; +import { ResourceType } from '../../../src/models/resource-types.js'; +import type { ResourceDescriptor } from '../../../src/models/types.js'; +import type { FilterConfig } from '../../../src/models/config.js'; +import { findStaleDependencies } from '../../../src/services/stale-dependency-finder.js'; + +const orders: ResourceDescriptor = { type: ResourceType.Api, nameParts: ['orders'] }; +const billing: ResourceDescriptor = { type: ResourceType.Api, nameParts: ['billing'] }; +const tag = (name: string): ResourceDescriptor => ({ type: ResourceType.Tag, nameParts: [name] }); +const apiTag = (api: string, name: string): ResourceDescriptor => ({ type: ResourceType.ApiTag, nameParts: [api, name] }); + +/** The filter a team extracting only `orders` uses: dependencies are pulled in. */ +const filter: FilterConfig = { apis: ['orders'], tags: [], loggers: [], namedValues: [] }; + +describe('stale-dependency-finder', () => { + const store = new ArtifactStore(); + let root: string; + let outputDir: string; + let stagingDir: string; + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'apiops-stale-')); + outputDir = path.join(root, 'output'); + stagingDir = path.join(root, 'staging'); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + const write = (dir: string, descriptor: ResourceDescriptor, properties: Record = {}) => + store.writeResource(dir, descriptor, { name: descriptor.nameParts.at(-1), properties }); + + const names = (descriptors: ResourceDescriptor[]) => + descriptors.map((descriptor) => `${descriptor.type}:${descriptor.nameParts.join('/')}`).sort(); + + it('should report a tag the selected API no longer links to', async () => { + await write(outputDir, orders); + await write(outputDir, apiTag('orders', 'public')); + await write(outputDir, tag('public')); + await write(stagingDir, orders); + + expect(names(await findStaleDependencies(store, outputDir, stagingDir, filter))).toEqual(['Tag:public']); + }); + + it('should keep a dependency another artifact in the output still references', async () => { + await write(outputDir, orders); + await write(outputDir, apiTag('orders', 'public')); + await write(outputDir, billing); + await write(outputDir, apiTag('billing', 'public')); + await write(outputDir, tag('public')); + await write(stagingDir, orders); + + expect(await findStaleDependencies(store, outputDir, stagingDir, filter)).toEqual([]); + }); + + it('should keep a dependency the extract wrote again', async () => { + await write(outputDir, orders); + await write(outputDir, apiTag('orders', 'public')); + await write(outputDir, tag('public')); + await write(stagingDir, orders); + await write(stagingDir, apiTag('orders', 'public')); + await write(stagingDir, tag('public')); + + expect(await findStaleDependencies(store, outputDir, stagingDir, filter)).toEqual([]); + }); + + it('should leave alone an out-of-scope artifact the selection never referenced', async () => { + await write(outputDir, orders); + await write(outputDir, tag('standalone')); + await write(stagingDir, orders); + + expect(await findStaleDependencies(store, outputDir, stagingDir, filter)).toEqual([]); + }); + + it("should follow a dependency's own dependencies, including named values referenced by display name", async () => { + const diagnostic: ResourceDescriptor = { type: ResourceType.ApiDiagnostic, nameParts: ['orders', 'applicationinsights'] }; + await write(outputDir, orders); + await write(outputDir, diagnostic, { loggerId: '/loggers/ai' }); + await write(outputDir, { type: ResourceType.Logger, nameParts: ['ai'] }, { credentials: { instrumentationKey: '{{AI Key}}' } }); + await write(outputDir, { type: ResourceType.NamedValue, nameParts: ['ai-key'] }, { displayName: 'AI Key' }); + await write(stagingDir, orders); + + expect(names(await findStaleDependencies(store, outputDir, stagingDir, filter))).toEqual([ + 'Logger:ai', + 'NamedValue:ai-key', + ]); + }); + + it("should follow a policy attribute set to a named value to the backend the named value's value names", async () => { + const policy = (api: string): ResourceDescriptor => ({ type: ResourceType.ApiPolicy, nameParts: [api] }); + const writePolicy = (dir: string, api: string) => + store.writeContent(dir, policy(api), '', 'policy', 'xml'); + const backendFilter: FilterConfig = { ...filter, backends: [] }; + await write(outputDir, orders); + await writePolicy(outputDir, 'orders'); + await write(outputDir, { type: ResourceType.NamedValue, nameParts: ['backend-id'] }, { displayName: 'Backend_Id', value: 'function-backend' }); + await write(outputDir, { type: ResourceType.Backend, nameParts: ['function-backend'] }); + await write(stagingDir, orders); + + expect(names(await findStaleDependencies(store, outputDir, stagingDir, backendFilter))).toEqual([ + 'Backend:function-backend', + 'NamedValue:backend-id', + ]); + + // Another API left in the output still sends requests to the backend. + await write(outputDir, billing); + await writePolicy(outputDir, 'billing'); + expect(await findStaleDependencies(store, outputDir, stagingDir, backendFilter)).toEqual([]); + }); + + it('should keep a named value only a Liquid body still uses, and report it once nothing does', async () => { + const policy: ResourceDescriptor = { type: ResourceType.ApiPolicy, nameParts: ['orders'] }; + const liquid = 'nv=[{{Backend_Id}}] liquid=[{{body.id}}]'; + await write(outputDir, orders); + await store.writeContent(outputDir, policy, liquid, 'policy', 'xml'); + await write(outputDir, { type: ResourceType.NamedValue, nameParts: ['backend-id'] }, { displayName: 'Backend_Id', value: 'x' }); + await write(stagingDir, orders); + await store.writeContent(stagingDir, policy, liquid, 'policy', 'xml'); + + expect(await findStaleDependencies(store, outputDir, stagingDir, filter)).toEqual([]); + + await store.writeContent(stagingDir, policy, '', 'policy', 'xml'); + expect(names(await findStaleDependencies(store, outputDir, stagingDir, filter))).toEqual(['NamedValue:backend-id']); + }); + + it('should report what a dependency the extract wrote again no longer references', async () => { + const diagnostic: ResourceDescriptor = { type: ResourceType.ApiDiagnostic, nameParts: ['orders', 'applicationinsights'] }; + const aiLogger: ResourceDescriptor = { type: ResourceType.Logger, nameParts: ['ai'] }; + await write(outputDir, orders); + await write(outputDir, diagnostic, { loggerId: '/loggers/ai' }); + await write(outputDir, aiLogger, { credentials: { instrumentationKey: '{{AI Key}}' } }); + await write(outputDir, { type: ResourceType.NamedValue, nameParts: ['ai-key'] }, { displayName: 'AI Key' }); + await write(stagingDir, orders); + await write(stagingDir, diagnostic, { loggerId: '/loggers/ai' }); + await write(stagingDir, aiLogger, { credentials: { connectionString: 'InstrumentationKey=00000000-0000-0000-0000-000000000000' } }); + + expect(names(await findStaleDependencies(store, outputDir, stagingDir, filter))).toEqual(['NamedValue:ai-key']); + }); + + it("should keep a dependency's own dependency that another artifact still uses", async () => { + const diagnostic: ResourceDescriptor = { type: ResourceType.ApiDiagnostic, nameParts: ['orders', 'applicationinsights'] }; + await write(outputDir, orders); + await write(outputDir, diagnostic, { loggerId: '/loggers/ai' }); + await write(outputDir, { type: ResourceType.Logger, nameParts: ['ai'] }, { credentials: { instrumentationKey: '{{AI Key}}' } }); + await write(outputDir, { type: ResourceType.NamedValue, nameParts: ['ai-key'] }, { displayName: 'AI Key' }); + await write(outputDir, { type: ResourceType.Backend, nameParts: ['legacy'] }, { credentials: { header: { key: ['{{AI Key}}'] } } }); + await write(stagingDir, orders); + + // The backend is outside the filter, so --remove-stale keeps it. + expect(names(await findStaleDependencies(store, outputDir, stagingDir, { ...filter, backends: [] }))).toEqual([ + 'Logger:ai', + ]); + }); + + it("should report a product's groups and tags it no longer links to, keeping one another product links to", async () => { + const starter: ResourceDescriptor = { type: ResourceType.Product, nameParts: ['starter'] }; + const gold: ResourceDescriptor = { type: ResourceType.Product, nameParts: ['gold'] }; + await write(outputDir, starter); + await store.writeAssociation(outputDir, starter, 'groups', [{ name: 'support-team' }, { name: 'partners' }]); + await store.writeAssociation(outputDir, starter, 'tags', [{ name: 'promo' }]); + await write(outputDir, gold); + await store.writeAssociation(outputDir, gold, 'groups', [{ name: 'partners' }]); + await write(outputDir, { type: ResourceType.Group, nameParts: ['support-team'] }); + await write(outputDir, { type: ResourceType.Group, nameParts: ['partners'] }); + await write(outputDir, tag('promo')); + await write(stagingDir, starter); + + const productFilter: FilterConfig = { products: ['starter'], groups: [], tags: [] }; + expect(names(await findStaleDependencies(store, outputDir, stagingDir, productFilter))).toEqual([ + 'Group:support-team', + 'Tag:promo', + ]); + }); + + it("should decide workspace artifacts by the workspace's sub-filter, including a link to a service-level group", async () => { + const wsOrders: ResourceDescriptor = { ...orders, workspace: 'team-a' }; + const wsStarter: ResourceDescriptor = { type: ResourceType.Product, nameParts: ['starter'], workspace: 'team-a' }; + const gold: ResourceDescriptor = { type: ResourceType.Product, nameParts: ['gold'] }; + await write(outputDir, wsOrders); + await write(outputDir, { ...apiTag('orders', 'public'), workspace: 'team-a' }); + await write(outputDir, { ...tag('public'), workspace: 'team-a' }); + await write(outputDir, wsStarter); + await store.writeAssociation(outputDir, wsStarter, 'groups', [ + { name: 'partners', scope: 'service' }, + { name: 'auditors', scope: 'service' }, + ]); + await write(outputDir, gold); + await store.writeAssociation(outputDir, gold, 'groups', [{ name: 'partners' }]); + await write(outputDir, { type: ResourceType.Group, nameParts: ['partners'] }); + await write(outputDir, { type: ResourceType.Group, nameParts: ['auditors'] }); + await write(stagingDir, wsOrders); + await write(stagingDir, wsStarter); + + const workspaceFilter: FilterConfig = { + products: [], + groups: [], + workspaceSubFilters: { 'team-*': { apis: ['orders'], products: ['starter'], tags: [] } }, + }; + // The service-level product gold (outside the filter) still links to partners. + expect(names(await findStaleDependencies(store, outputDir, stagingDir, workspaceFilter))).toEqual([ + 'Group:auditors', + 'Tag:public', + ]); + }); + + it('should report nothing when an artifact left in the output cannot be checked', async () => { + const warn = vi.spyOn(logger, 'warn').mockImplementation(() => undefined); + await write(outputDir, orders); + await write(outputDir, apiTag('orders', 'public')); + await write(outputDir, tag('public')); + await write(outputDir, billing); + await write(stagingDir, orders); + const failingStore: IArtifactStore = Object.create(store); + failingStore.readResource = async (dir, descriptor) => { + if (descriptor.nameParts[0] === 'billing') { + throw new Error('Unexpected token in JSON'); + } + return store.readResource(dir, descriptor); + }; + + try { + expect(await findStaleDependencies(failingStore, outputDir, stagingDir, { ...filter, apis: ['orders'] })).toEqual([]); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('Keeping every pulled-in dependency')); + } finally { + warn.mockRestore(); + } + }); + + it('should report nothing when a named value left in the output cannot be read to match display names', async () => { + const warn = vi.spyOn(logger, 'warn').mockImplementation(() => undefined); + const policy = (api: string): ResourceDescriptor => ({ type: ResourceType.ApiPolicy, nameParts: [api] }); + const namedValue: ResourceDescriptor = { type: ResourceType.NamedValue, nameParts: ['backend-id'] }; + await write(outputDir, orders); + await store.writeContent(outputDir, policy('orders'), '', 'policy', 'xml'); + await write(outputDir, { type: ResourceType.Backend, nameParts: ['function-backend'] }); + // billing reaches the backend through the named value's display name. + await write(outputDir, billing); + await store.writeContent(outputDir, policy('billing'), '', 'policy', 'xml'); + await write(outputDir, namedValue, { displayName: 'Backend_Id', value: 'function-backend' }); + await write(stagingDir, orders); + const failingStore: IArtifactStore = Object.create(store); + failingStore.readResource = async (dir, descriptor) => { + if (descriptor.type === ResourceType.NamedValue) { + throw new Error('Unexpected token in JSON'); + } + return store.readResource(dir, descriptor); + }; + + try { + expect(await findStaleDependencies(failingStore, outputDir, stagingDir, { ...filter, backends: [] })).toEqual([]); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('Keeping every pulled-in dependency')); + } finally { + warn.mockRestore(); + } + }); + + it('should report an excluded tag that only an old link from the selected API referenced', async () => { + await write(outputDir, orders); + await write(outputDir, apiTag('orders', 'internal')); + await write(outputDir, tag('internal')); + await write(stagingDir, orders); + + // The old link itself is removed as one of the API's children. + expect(names(await findStaleDependencies(store, outputDir, stagingDir, { apis: ['orders'], tags: ['!internal'] }))) + .toEqual(['Tag:internal']); + }); + + it('should stop scanning once every candidate is known to be needed', async () => { + await write(outputDir, orders); + await write(outputDir, apiTag('orders', 'public')); + await write(outputDir, tag('public')); + for (let index = 0; index < 30; index++) { + await write(outputDir, { type: ResourceType.Api, nameParts: [`other-${index}`] }); + } + await write(stagingDir, orders); + await write(stagingDir, billing); + await write(stagingDir, apiTag('billing', 'public')); + const countingStore: IArtifactStore = Object.create(store); + let otherReads = 0; + countingStore.readResource = async (dir, descriptor) => { + if (descriptor.nameParts[0]?.startsWith('other-')) { + otherReads++; + } + return store.readResource(dir, descriptor); + }; + + expect(await findStaleDependencies(countingStore, outputDir, stagingDir, { ...filter, apis: ['orders', 'billing'] })).toEqual([]); + // The staged link from billing rescues the only candidate, so most of the other APIs are never read. + expect(otherReads).toBeLessThan(30); + }); + + it('should read nothing when every existing artifact is selected or written again', async () => { + await write(outputDir, orders); + await write(outputDir, apiTag('orders', 'public')); + await write(outputDir, tag('public')); + await write(stagingDir, orders); + await write(stagingDir, tag('public')); + const countingStore: IArtifactStore = Object.create(store); + let reads = 0; + countingStore.readResource = async (dir, descriptor) => { + reads++; + return store.readResource(dir, descriptor); + }; + + expect(await findStaleDependencies(countingStore, outputDir, stagingDir, { apis: ['orders'] })).toEqual([]); + expect(reads).toBe(0); + }); +}); diff --git a/tests/unit/services/transitive-extractor.test.ts b/tests/unit/services/transitive-extractor.test.ts index 31ead3d4..99d4e0fe 100644 --- a/tests/unit/services/transitive-extractor.test.ts +++ b/tests/unit/services/transitive-extractor.test.ts @@ -8,6 +8,7 @@ import { describe, it, expect, vi } from 'vitest'; import { ResourceType } from '../../../src/models/resource-types.js'; import { ApimServiceContext, ResourceDescriptor } from '../../../src/models/types.js'; import { extractTransitiveDependencies } from '../../../src/services/transitive-extractor.js'; +import { logger } from '../../../src/lib/logger.js'; const testContext: ApimServiceContext = { subscriptionId: 'sub-1', @@ -90,6 +91,54 @@ describe('transitive-extractor', () => { ); }); + it('should not extract a {{displayName}} reference whose resource name is excluded', async () => { + const client = createMockClient(); + client.listResources = namedValueList({ name: 'my-secret', displayName: 'My Secret' }); + client.getResource.mockResolvedValue({ name: 'my-secret', properties: { displayName: 'My Secret' } }); + const store = createMockStore(); + + const result = await extractTransitiveDependencies( + client, store, testContext, '/output', policiesWith('My Secret'), new Map(), [], [], + { skip: (dep) => dep.nameParts[0] === 'my-secret' } + ); + + expect(result).toEqual({ extractedDescriptors: [], failedDescriptors: [], errorCount: 0 }); + expect(client.getResource).not.toHaveBeenCalled(); + expect(store.writeResource).not.toHaveBeenCalled(); + }); + + it('should extract a {{displayName}} reference whose display name is the name of an excluded named value', async () => { + const client = createMockClient(); + client.listResources = namedValueList({ name: 'one', displayName: 'two' }, { name: 'two', displayName: 'other' }); + client.getResource.mockResolvedValue({ name: 'one', properties: { displayName: 'two' } }); + const store = createMockStore(); + + const result = await extractTransitiveDependencies( + client, store, testContext, '/output', policiesWith('two'), new Map(), [], [], + { skip: (dep) => dep.nameParts[0] === 'two' } + ); + + expect(result.extractedDescriptors).toEqual([ + expect.objectContaining({ type: ResourceType.NamedValue, nameParts: ['one'] }), + ]); + }); + + it('should extract a {{displayName}} reference whose display name is the name of an extracted named value', async () => { + const client = createMockClient(); + client.listResources = namedValueList({ name: 'one', displayName: 'two' }, { name: 'two', displayName: 'three' }); + client.getResource.mockResolvedValue({ name: 'one', properties: { displayName: 'two' } }); + const store = createMockStore(); + + const result = await extractTransitiveDependencies( + client, store, testContext, '/output', policiesWith('two'), new Map(), [], + [{ type: ResourceType.NamedValue, nameParts: ['two'] }] + ); + + expect(result.extractedDescriptors).toEqual([ + expect.objectContaining({ type: ResourceType.NamedValue, nameParts: ['one'] }), + ]); + }); + it('should fall back to the reference as resource name when no displayName matches', async () => { const client = createMockClient(); client.listResources = namedValueList({ name: 'other', displayName: 'other' }); @@ -136,6 +185,22 @@ describe('transitive-extractor', () => { expect(namedValueLists).toHaveLength(1); }); + it('should match a {{displayName}} reference ignoring case', async () => { + const client = createMockClient(); + client.listResources = namedValueList({ name: 'nv-resource', displayName: 'NV_Display' }); + client.getResource.mockImplementation(async (_ctx, descriptor: ResourceDescriptor) => ({ + name: descriptor.nameParts[0], + properties: {}, + })); + const store = createMockStore(); + + const result = await extractTransitiveDependencies( + client, store, testContext, '/output', policiesWith('nv_display'), new Map(), [], [] + ); + + expect(result.extractedDescriptors.map((descriptor) => descriptor.nameParts[0])).toEqual(['nv-resource']); + }); + it('should extract each resolved resource when a resource name is also another display name', async () => { const client = createMockClient(); client.listResources = namedValueList( @@ -222,7 +287,8 @@ describe('transitive-extractor', () => { const store = createMockStore(); const result = await extractTransitiveDependencies( - client, store, wsContext, '/output', policiesWith('ws_display'), new Map(), [], [], 'team-a', testContext + client, store, wsContext, '/output', policiesWith('ws_display'), new Map(), [], [], + { workspace: 'team-a', serviceContext: testContext } ); expect(result.errorCount).toBe(0); @@ -244,4 +310,146 @@ describe('transitive-extractor', () => { expect(result.extractedDescriptors).toEqual([]); }); }); + + describe('policy attributes set to a named value (#331)', () => { + const policies = new Map([['api-1', '']]); + const backendId = { type: ResourceType.NamedValue, nameParts: ['backend-id'] }; + + it("should extract the backend named by the named value's value", async () => { + const client = createMockClient(); + client.listResources = namedValueList({ name: 'backend-id', displayName: 'Backend_Id' }); + client.getResource.mockImplementation(async (_ctx, descriptor: ResourceDescriptor) => { + if (descriptor.type === ResourceType.NamedValue) { + return { name: 'backend-id', properties: { displayName: 'Backend_Id', value: 'function-backend' } }; + } + return descriptor.type === ResourceType.Backend ? { name: descriptor.nameParts[0], properties: {} } : undefined; + }); + + const result = await extractTransitiveDependencies( + client, createMockStore(), testContext, '/output', policies, new Map(), [], [] + ); + + expect(result.errorCount).toBe(0); + expect(result.extractedDescriptors).toEqual([ + expect.objectContaining(backendId), + expect.objectContaining({ type: ResourceType.Backend, nameParts: ['function-backend'] }), + ]); + }); + + it('should follow a named value the selection already extracted', async () => { + const client = createMockClient(); + client.listResources = namedValueList({ name: 'backend-id', displayName: 'Backend_Id' }); + client.getResource.mockResolvedValue({ name: 'function-backend', properties: {} }); + + const result = await extractTransitiveDependencies( + client, createMockStore(), testContext, '/output', policies, new Map(), + [{ descriptor: backendId, json: { properties: { displayName: 'Backend_Id', value: 'function-backend' } } }], + [backendId] + ); + + expect(result.extractedDescriptors).toEqual([ + expect.objectContaining({ type: ResourceType.Backend, nameParts: ['function-backend'] }), + ]); + }); + }); +}); + +describe('extractTransitiveDependencies with Liquid bodies', () => { + it('should extract a named value a Liquid body references and skip Liquid output without an error', async () => { + const client = createMockClient(); + client.listResources = namedValueList({ name: 'backend-id', displayName: 'demo-backend-id' }); + client.getResource.mockImplementation(async (_ctx: ApimServiceContext, descriptor: ResourceDescriptor) => + descriptor.type === ResourceType.NamedValue && descriptor.nameParts[0] === 'backend-id' + ? { name: 'backend-id', properties: { displayName: 'demo-backend-id', value: 'function-backend' } } + : undefined + ); + const store = createMockStore(); + const policies = new Map([[ + 'api:liquid-test:policy:0', + 'nv=[{{demo-backend-id}}] liquid=[{{body.id}}] spaced=[{{ context.Request.Url }}]', + ]]); + + const result = await extractTransitiveDependencies(client, store, testContext, '/out', policies, new Map(), [], []); + + const fetched = client.getResource.mock.calls.map((call) => `${call[1].type}:${call[1].nameParts.join('/')}`); + expect(fetched).toEqual(['NamedValue:backend-id']); + expect(result.errorCount).toBe(0); + expect(result.extractedDescriptors.map((d) => d.nameParts[0])).toEqual(['backend-id']); + }); + + it('should report an error when the named values for a Liquid body cannot be listed', async () => { + const warn = vi.spyOn(logger, 'warn').mockImplementation(() => undefined); + const client = createMockClient(); + // eslint-disable-next-line require-yield + client.listResources = async function* () { + throw new Error('HTTP 503'); + }; + const policies = new Map([['api:orders:policy:0', '{{demo-backend-id}}']]); + + try { + const result = await extractTransitiveDependencies(client, createMockStore(), testContext, '/out', policies, new Map(), [], []); + + expect(result.errorCount).toBe(1); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('Unable to tell which {{...}} in Liquid bodies are named values')); + } finally { + warn.mockRestore(); + } + }); + + it("should follow a Liquid {{name}} in a fragment extracted in a later round", async () => { + const client = createMockClient(); + client.listResources = namedValueList({ name: 'frag-nv', displayName: 'frag-nv' }); + client.getResource.mockImplementation(async (_ctx: ApimServiceContext, descriptor: ResourceDescriptor) => + descriptor.type === ResourceType.PolicyFragment + ? { properties: { value: '{{frag-nv}} {{body.id}}' } } + : { properties: { value: 'x' } } + ); + const policies = new Map([['api:orders:policy:0', '']]); + + const result = await extractTransitiveDependencies(client, createMockStore(), testContext, '/out', policies, new Map(), [], []); + + expect(client.getResource.mock.calls.map((call) => `${call[1].type}:${call[1].nameParts.join('/')}`)).toEqual([ + 'PolicyFragment:route', + 'NamedValue:frag-nv', + ]); + expect(result.errorCount).toBe(0); + }); + + it('should look a Liquid {{name}} up in the workspace being extracted', async () => { + const client = createMockClient(); + const workspaceContext = { ...testContext, baseUrl: `${testContext.baseUrl}/workspaces/team-a` }; + client.listResources = vi.fn(async function* (ctx: ApimServiceContext, type: ResourceType) { + if (type === ResourceType.NamedValue && ctx.baseUrl === workspaceContext.baseUrl) { + yield { name: 'ws-key', properties: { displayName: 'ws-key' } }; + } + }); + client.getResource.mockResolvedValue({ properties: { value: 'x' } }); + const policies = new Map([['api:orders:policy:0', '{{ws-key}}']]); + + await extractTransitiveDependencies(client, createMockStore(), workspaceContext, '/out', policies, new Map(), [], [], { + workspace: 'team-a', + serviceContext: testContext, + }); + + expect(client.getResource.mock.calls.map((call) => call[1])).toEqual([ + { type: ResourceType.NamedValue, nameParts: ['ws-key'], workspace: 'team-a' }, + ]); + }); +}); + +describe('extractTransitiveDependencies named value lookup', () => { + it("should resolve a reference that matches no display name to the resource name's spelling, and check exclusions under it", async () => { + const client = createMockClient(); + client.listResources = namedValueList({ name: 'foo', displayName: 'Something' }); + client.getResource.mockResolvedValue({ properties: { value: 'x' } }); + + await extractTransitiveDependencies(client, createMockStore(), testContext, '/out', policiesWith('FOO'), new Map(), [], []); + expect(client.getResource.mock.calls.map((call) => call[1].nameParts)).toEqual([['foo']]); + + client.getResource.mockClear(); + await extractTransitiveDependencies(client, createMockStore(), testContext, '/out', policiesWith('FOO'), new Map(), [], [], { + skip: (dep) => dep.nameParts[0] === 'foo', + }); + expect(client.getResource).not.toHaveBeenCalled(); + }); }); diff --git a/tests/unit/services/transitive-resolver.test.ts b/tests/unit/services/transitive-resolver.test.ts index 8d05947e..e2f46eca 100644 --- a/tests/unit/services/transitive-resolver.test.ts +++ b/tests/unit/services/transitive-resolver.test.ts @@ -6,14 +6,16 @@ import { describe, it, expect, vi } from 'vitest'; import { ResourceType } from '../../../src/models/resource-types.js'; -import { FilterConfig } from '../../../src/models/config.js'; import { scanPolicyReferences, scanApiVersionSetReference, - resolveTransitiveDependencies, findTransitiveDependencies, findSubscriptionTargets, scanArtifactReferences, + findAssociationDependencies, + scanNamedValueTargets, + createNamedValueReader, + scanLiquidNamedValueCandidates, } from '../../../src/services/transitive-resolver.js'; describe('transitive-resolver', () => { @@ -122,53 +124,6 @@ describe('transitive-resolver', () => { }); }); - describe('resolveTransitiveDependencies', () => { - it('should expand filter with discovered dependencies', () => { - const policies = new Map(); - policies.set('service-policy', '{{my-secret}}'); - - const apis = new Map>(); - - const filter: FilterConfig = { - apis: ['my-api'], - namedValues: [], // Start with empty — should be expanded - }; - - const expanded = resolveTransitiveDependencies(policies, apis, filter); - expect(expanded.namedValues).toContain('my-secret'); - }); - - it('should not add to undefined filter fields (unfiltered types)', () => { - const policies = new Map(); - policies.set('policy', '{{my-secret}}'); - - const apis = new Map>(); - - // namedValues is undefined = all named values included - const filter: FilterConfig = { - apis: ['my-api'], - }; - - const expanded = resolveTransitiveDependencies(policies, apis, filter); - // Should remain undefined (no need to add — all are already included) - expect(expanded.namedValues).toBeUndefined(); - }); - - it('should not duplicate existing entries', () => { - const policies = new Map(); - policies.set('policy', '{{existing-secret}}'); - - const apis = new Map>(); - - const filter: FilterConfig = { - namedValues: ['existing-secret'], - }; - - const expanded = resolveTransitiveDependencies(policies, apis, filter); - expect(expanded.namedValues).toEqual(['existing-secret']); - }); - }); - describe('findTransitiveDependencies', () => { it('should find dependencies from policies and APIs', () => { const policies = new Map(); @@ -247,7 +202,7 @@ describe('transitive-resolver', () => { }, }), readContent: vi.fn().mockResolvedValue(undefined), - readAssociation: vi.fn(), + readAssociation: vi.fn().mockResolvedValue([]), }; const backendRefs = await scanArtifactReferences( @@ -272,7 +227,8 @@ describe('transitive-resolver', () => { '/source', { type: ResourceType.Product, nameParts: ['starter'] } )).resolves.toEqual([]); - expect(store.readAssociation).not.toHaveBeenCalled(); + // A product's API links are not dependencies, so they are not read. + expect(store.readAssociation.mock.calls.map((call) => call[2])).not.toContain('apis'); }); it('should identify subscription targets separately from transitive dependencies', () => { @@ -339,4 +295,298 @@ describe('transitive-resolver', () => { }); }); }); + + describe('dependency coverage', () => { + const SERVICE = '/subscriptions/s/resourceGroups/r/providers/Microsoft.ApiManagement/service/a'; + + it.each([ + ['an attribute before backend-id', '', ResourceType.Backend, 'func-backend'], + ['single quotes', "", ResourceType.Backend, 'quoted-backend'], + ['llm-content-safety', '', ResourceType.Backend, 'content-safety'], + ['azure-openai-semantic-cache-lookup', '', ResourceType.Backend, 'embeddings'], + ['llm-semantic-cache-lookup', '', ResourceType.Backend, 'llm-embeddings'], + ['include-fragment with another attribute first', '', ResourceType.PolicyFragment, 'frag-a'], + ['log-to-eventhub', '@(context.RequestId)', ResourceType.Logger, 'eventhub-logger'], + [ + 'validate-content schema-id', + '', + ResourceType.GlobalSchema, + 'order-schema', + ], + ])('should detect a reference written with %s', (_label, policy, type, name) => { + expect(scanPolicyReferences(policy)).toContainEqual({ type, name }); + }); + + it('should ignore references inside XML comments (#305)', () => { + const refs = scanPolicyReferences( + '' + + '' + ); + + expect(refs).toEqual([{ type: ResourceType.Backend, name: 'live-backend' }]); + }); + + it('should not let a stray " + {{outside-liquid}} + `; + + expect(scanLiquidNamedValueCandidates(policy)).toEqual(['demo-backend-id', 'body.id']); + }); + + it('should ignore Liquid output in set-body templates but keep named values elsewhere (#276)', () => { + const refs = scanPolicyReferences(` + {{real-value}} + {{context.Request.MatchedParameters["id"]}} + {{body.envelope.body.fault.faultcode}} + + {{after-liquid}} + {{not-liquid}}`); + + expect(refs).toEqual([ + { type: ResourceType.NamedValue, name: 'real-value' }, + { type: ResourceType.NamedValue, name: 'after-liquid' }, + { type: ResourceType.NamedValue, name: 'not-liquid' }, + ]); + }); + + it('should not treat policy expressions or named value placeholders as resource names', () => { + const refs = scanPolicyReferences( + '' + + '' + ); + + expect(refs).toEqual([{ type: ResourceType.NamedValue, name: 'backend-name' }]); + }); + + it.each([ + [ResourceType.Diagnostic, ['applicationinsights'], `${SERVICE}/loggers/app-insights`, undefined, undefined], + [ResourceType.ApiDiagnostic, ['orders', 'azuremonitor'], '/loggers/app-insights', 'team-a', 'team-a'], + // APIM returns a workspace diagnostic's loggerId without the workspace segment. + [ResourceType.ApiDiagnostic, ['orders', 'applicationinsights'], `${SERVICE}/loggers/app-insights`, 'team-a', 'team-a'], + ])('should follow a %s loggerId to the logger in its own scope', (type, nameParts, loggerId, workspace, loggerWorkspace) => { + const deps = findTransitiveDependencies(new Map(), new Map(), workspace, [ + { descriptor: { type, nameParts, workspace }, json: { properties: { loggerId } } }, + ]); + + expect(deps).toEqual([{ type: ResourceType.Logger, nameParts: ['app-insights'], workspace: loggerWorkspace }]); + }); + + it('should follow named values in logger and backend credentials', () => { + const deps = findTransitiveDependencies(new Map(), new Map(), undefined, [ + { + descriptor: { type: ResourceType.Logger, nameParts: ['app-insights'] }, + json: { properties: { loggerType: 'applicationInsights', credentials: { instrumentationKey: '{{Logger-Credentials--abc}}' } } }, + }, + { + descriptor: { type: ResourceType.Backend, nameParts: ['orders'], workspace: 'team-a' }, + json: { + properties: { + url: 'https://orders.example.com', + credentials: { + header: { 'x-api-key': ['{{orders-key}}'] }, + query: { code: ['{{orders-code}}'] }, + authorization: { scheme: 'Bearer', parameter: '{{orders-token}}' }, + }, + }, + }, + }, + ]); + + expect(deps).toEqual([ + { type: ResourceType.NamedValue, nameParts: ['Logger-Credentials--abc'], workspace: undefined }, + { type: ResourceType.NamedValue, nameParts: ['orders-key'], workspace: 'team-a' }, + { type: ResourceType.NamedValue, nameParts: ['orders-code'], workspace: 'team-a' }, + { type: ResourceType.NamedValue, nameParts: ['orders-token'], workspace: 'team-a' }, + ]); + }); + + it("should follow an API's tag link to the tag", () => { + const deps = findTransitiveDependencies(new Map(), new Map(), 'team-a', [ + { descriptor: { type: ResourceType.ApiTag, nameParts: ['orders', 'public'], workspace: 'team-a' }, json: {} }, + ]); + + expect(deps).toEqual([{ type: ResourceType.Tag, nameParts: ['public'], workspace: 'team-a' }]); + }); + + it("should follow a product's group and tag links, keeping service-scope targets at service level", () => { + const product = { type: ResourceType.Product, nameParts: ['starter'], workspace: 'team-a' }; + + expect(findAssociationDependencies(product, 'groups', [ + { name: 'support-team', scope: 'service' }, + { name: 'partners', scope: 'workspace' }, + { name: 'partners' }, + ])).toEqual([ + { type: ResourceType.Group, nameParts: ['support-team'], workspace: undefined }, + { type: ResourceType.Group, nameParts: ['partners'], workspace: 'team-a' }, + ]); + expect(findAssociationDependencies(product, 'tags', [{ name: 'public' }])).toEqual([ + { type: ResourceType.Tag, nameParts: ['public'], workspace: 'team-a' }, + ]); + }); + + it('should leave built-in groups out of product link dependencies', () => { + expect(findAssociationDependencies({ type: ResourceType.Product, nameParts: ['starter'] }, 'groups', [ + { name: 'Developers' }, + { name: 'guests' }, + { name: 'administrators' }, + { name: 'support-team' }, + ])).toEqual([{ type: ResourceType.Group, nameParts: ['support-team'], workspace: undefined }]); + }); + + it('should treat a workspace group with a built-in name as a custom group', () => { + const product = { type: ResourceType.Product, nameParts: ['starter'], workspace: 'team-a' }; + + expect(findAssociationDependencies(product, 'groups', [ + { name: 'developers', scope: 'workspace' }, + { name: 'administrators', scope: 'service' }, + ])).toEqual([{ type: ResourceType.Group, nameParts: ['developers'], workspace: 'team-a' }]); + }); + + describe('policy attributes set to a named value (#331)', () => { + const POLICY = ''; + const namedValue = (properties: Record, name = 'backend-id', workspace?: string) => ({ + descriptor: { type: ResourceType.NamedValue, nameParts: [name], workspace }, + json: { properties }, + }); + + it('should read each named value once, however many policies set attributes to it', async () => { + const descriptor = { type: ResourceType.NamedValue, nameParts: ['backend-id'] }; + const readResource = vi.fn(async () => ({ properties: { value: 'function-backend' } })); + const read = createNamedValueReader(async () => descriptor, readResource); + + const results = await Promise.all([read(descriptor), read(descriptor), read({ ...descriptor, nameParts: ['BACKEND-ID'] })]); + + expect(results.map((result) => result?.json)).toEqual(Array(3).fill({ properties: { value: 'function-backend' } })); + expect(readResource).toHaveBeenCalledTimes(1); + }); + + it('should find attributes whose whole value is a named value reference', () => { + expect(scanNamedValueTargets( + '' + + '' + + '' + + '' + + '' + )).toEqual([ + { type: ResourceType.Backend, namedValue: 'Backend_Id' }, + { type: ResourceType.Backend, namedValue: 'safety' }, + ]); + }); + + it("should follow the named value's value to the resource it names, matching the display name", () => { + const deps = findTransitiveDependencies(new Map([['p', POLICY]]), new Map(), undefined, [ + namedValue({ displayName: 'backend_id', value: ' function-backend ' }), + ]); + + expect(deps).toContainEqual({ type: ResourceType.Backend, nameParts: ['function-backend'], workspace: undefined }); + }); + + it('should fall back to the resource name, and let a display name match win', () => { + const byName = findTransitiveDependencies(new Map([['p', POLICY]]), new Map(), undefined, [ + namedValue({ displayName: 'Other', value: 'by-name' }, 'Backend_Id'), + ]); + const byDisplayName = findTransitiveDependencies(new Map([['p', POLICY]]), new Map(), undefined, [ + namedValue({ displayName: 'Other', value: 'by-name' }, 'Backend_Id'), + namedValue({ displayName: 'Backend_Id', value: 'by-display-name' }, 'nv-1'), + ]); + + expect(byName).toContainEqual({ type: ResourceType.Backend, nameParts: ['by-name'], workspace: undefined }); + expect(byDisplayName.filter((dep) => dep.type === ResourceType.Backend)).toEqual([ + { type: ResourceType.Backend, nameParts: ['by-display-name'], workspace: undefined }, + ]); + }); + + it.each([ + ['a secret', { displayName: 'Backend_Id', value: '***REDACTED***', secret: true }], + ['a policy expression', { displayName: 'Backend_Id', value: '@(context.Variables["b"])' }], + ['a nested reference', { displayName: 'Backend_Id', value: '{{other}}' }], + ])('should not follow %s named value', (_kind, properties) => { + const deps = findTransitiveDependencies(new Map([['p', POLICY]]), new Map(), undefined, [namedValue(properties)]); + + expect(deps.filter((dep) => dep.type === ResourceType.Backend)).toEqual([]); + }); + + it("should only match a named value in the policy's own scope", () => { + const deps = findTransitiveDependencies(new Map([['p', POLICY]]), new Map(), 'team-a', [ + namedValue({ displayName: 'Backend_Id', value: 'service-backend' }), + namedValue({ displayName: 'Backend_Id', value: 'ws-backend' }, 'backend-id', 'team-a'), + ]); + + expect(deps.filter((dep) => dep.type === ResourceType.Backend)).toEqual([ + { type: ResourceType.Backend, nameParts: ['ws-backend'], workspace: 'team-a' }, + ]); + }); + + it('should follow one in a policy fragment', () => { + const deps = findTransitiveDependencies(new Map(), new Map(), undefined, [ + { descriptor: { type: ResourceType.PolicyFragment, nameParts: ['route'] }, json: { properties: { value: POLICY } } }, + namedValue({ displayName: 'Backend_Id', value: 'function-backend' }), + ]); + + expect(deps).toContainEqual({ type: ResourceType.Backend, nameParts: ['function-backend'], workspace: undefined }); + }); + + it('should read the named value when scanning an artifact, given a reader', async () => { + const store = { + readResource: vi.fn(), + readContent: vi.fn().mockResolvedValue({ content: POLICY, format: 'xml' }), + readAssociation: vi.fn(), + }; + const readNamedValue = vi.fn().mockResolvedValue( + namedValue({ displayName: 'Backend_Id', value: 'ws-backend' }, 'backend-id', 'team-a') + ); + const policy = { type: ResourceType.ApiPolicy, nameParts: ['orders'], workspace: 'team-a' }; + + const withReader = await scanArtifactReferences(store, '/source', policy, readNamedValue); + const withoutReader = await scanArtifactReferences(store, '/source', policy); + + expect(readNamedValue).toHaveBeenCalledWith({ type: ResourceType.NamedValue, nameParts: ['Backend_Id'], workspace: 'team-a' }); + expect(withReader).toEqual([ + { type: ResourceType.NamedValue, nameParts: ['Backend_Id'], workspace: 'team-a' }, + { type: ResourceType.Backend, nameParts: ['ws-backend'], workspace: 'team-a' }, + ]); + expect(withoutReader).toEqual([{ type: ResourceType.NamedValue, nameParts: ['Backend_Id'], workspace: 'team-a' }]); + }); + }); + + it("should read a product's group and tag links when scanning it for publish", async () => { + const store = { + readResource: vi.fn().mockResolvedValue({ properties: { displayName: 'Starter' } }), + readContent: vi.fn().mockResolvedValue(undefined), + readAssociation: vi.fn(async (_dir: string, _descriptor: unknown, type: string) => + type === 'groups' ? [{ name: 'support-team' }] : [{ name: 'public' }] + ), + }; + + const refs = await scanArtifactReferences(store, '/source', { type: ResourceType.Product, nameParts: ['starter'] }); + + expect(refs).toEqual([ + { type: ResourceType.Group, nameParts: ['support-team'], workspace: undefined }, + { type: ResourceType.Tag, nameParts: ['public'], workspace: undefined }, + ]); + }); + + }); }); diff --git a/tests/unit/services/workspace-extractor.test.ts b/tests/unit/services/workspace-extractor.test.ts index e24f1b1f..a5ba8ab9 100644 --- a/tests/unit/services/workspace-extractor.test.ts +++ b/tests/unit/services/workspace-extractor.test.ts @@ -10,6 +10,7 @@ import { ApimServiceContext } from '../../../src/models/types.js'; import { FilterConfig } from '../../../src/models/config.js'; import { extractWorkspaces } from '../../../src/services/workspace-extractor.js'; import { resolveWorkspaceFilter } from '../../../src/services/workspace-extractor.js'; +import { logger } from '../../../src/lib/logger.js'; const testContext: ApimServiceContext = { subscriptionId: 'sub-1', @@ -563,7 +564,289 @@ describe('workspace-extractor', () => { }); }); + describe('workspace dependency coverage', () => { + const SERVICE = '/subscriptions/s/resourceGroups/r/providers/Microsoft.ApiManagement/service/a'; + const wsContext = { ...testContext, baseUrl: `${testContext.baseUrl}/workspaces/team-a` }; + + /** A store that keeps association files in memory so they can be read back. */ + function createAssociationStore() { + const store = createMockStore(); + const associations = new Map(); + const key = (descriptor: { nameParts: string[]; workspace?: string }, type: string) => + `${descriptor.workspace ?? ''}/${descriptor.nameParts.join('/')}/${type}`; + store.writeAssociation.mockImplementation(async (_dir, descriptor, type, entries) => { + associations.set(key(descriptor, type), entries); + }); + store.readAssociation.mockImplementation(async (_dir, descriptor, type) => + associations.get(key(descriptor, type)) ?? [] + ); + return store; + } + + it("should pull in an API diagnostic's logger, tags linked through any workspace tag, and a product's groups and tags", async () => { + const client = createMockClient(); + client.listResources = async function* (_ctx: ApimServiceContext, type: ResourceType, parent?: { nameParts: string[] }) { + if (type === ResourceType.Api) yield { name: 'orders', properties: {} }; + if (type === ResourceType.Product) yield { name: 'starter', properties: {} }; + if (type === ResourceType.Tag) { + yield { name: 'public', properties: {} }; + yield { name: 'internal', properties: {} }; + } + if (type === ResourceType.ApiDiagnostic) { + yield { name: 'applicationinsights', properties: { loggerId: `${SERVICE}/workspaces/team-a/loggers/ws-logger` } }; + } + if (type === ResourceType.ApiTag && parent?.nameParts[0] === 'public') { + yield { name: 'link-1', properties: { apiId: `${SERVICE}/workspaces/team-a/apis/orders` } }; + } + if (type === ResourceType.ProductTag && parent?.nameParts[0] === 'internal') { + yield { name: 'link-2', properties: { productId: `${SERVICE}/workspaces/team-a/products/starter` } }; + } + if (type === ResourceType.ProductGroup) { + yield { name: 'link-3', properties: { groupId: `${SERVICE}/groups/support-team` } }; + } + }; + client.getResource.mockImplementation(async (_ctx, descriptor) => { + if (descriptor.type === ResourceType.Workspace) return { name: 'team-a', properties: {} }; + if ([ResourceType.Logger, ResourceType.Tag, ResourceType.Group].includes(descriptor.type)) { + return { name: descriptor.nameParts[0], properties: {} }; + } + return undefined; + }); + const store = createAssociationStore(); + const filter: FilterConfig = { + workspaces: ['team-a'], + workspaceSubFilters: { + 'team-a': { apis: ['orders'], products: ['starter'], tags: [], loggers: [], groups: [] }, + }, + }; + + await extractWorkspaces(client, store, testContext, '/output', filter, true); + + const fetched = client.getResource.mock.calls.map(([ctx, d]) => + `${d.type}:${d.workspace ?? 'service'}:${d.nameParts.join('/')}@${ctx.baseUrl === wsContext.baseUrl ? 'workspace' : 'service'}` + ); + expect(fetched).toEqual(expect.arrayContaining([ + 'Logger:team-a:ws-logger@workspace', + 'Tag:team-a:public@workspace', + 'Tag:team-a:internal@workspace', + 'Group:service:support-team@service', + ])); + expect(store.writeResource).toHaveBeenCalledWith( + '/output', + expect.objectContaining({ type: ResourceType.ApiTag, nameParts: ['orders', 'public'], workspace: 'team-a' }), + expect.anything() + ); + }); + + it('should fall back to the extracted tags, and report an error, when listing the workspace tags fails', async () => { + const warn = vi.spyOn(logger, 'warn').mockImplementation(() => undefined); + const client = createMockClient(); + let tagListings = 0; + client.listResources = async function* (_ctx: ApimServiceContext, type: ResourceType) { + if (type === ResourceType.Api) yield { name: 'orders', properties: {} }; + if (type === ResourceType.Tag && ++tagListings > 1) throw new Error('HTTP 503'); + }; + client.getResource.mockResolvedValue({ name: 'team-a', properties: {} }); + const filter: FilterConfig = { + workspaces: ['team-a'], + workspaceSubFilters: { 'team-a': { apis: ['orders'], tags: [] } }, + }; + + const results = await extractWorkspaces(client, createAssociationStore(), testContext, '/output', filter, true); + + expect(results).toHaveLength(1); + expect(results[0].errorCount).toBe(1); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('Failed to list tags in workspace "team-a"')); + warn.mockRestore(); + }); + + it('should not pull in dependencies the workspace or top-level filter excludes by name', async () => { + const client = createMockClient(); + client.listResources = async function* (_ctx: ApimServiceContext, type: ResourceType, parent?: { nameParts: string[] }) { + if (type === ResourceType.Api) yield { name: 'orders', properties: {} }; + if (type === ResourceType.Product) yield { name: 'starter', properties: {} }; + if (type === ResourceType.Tag) { + yield { name: 'public', properties: {} }; + yield { name: 'internal', properties: {} }; + } + if (type === ResourceType.ApiTag && parent) { + yield { name: `link-${parent.nameParts[0]}`, properties: { apiId: `${SERVICE}/workspaces/team-a/apis/orders` } }; + } + if (type === ResourceType.ProductGroup) { + yield { name: 'link-a', properties: { groupId: `${SERVICE}/groups/partners` } }; + yield { name: 'link-b', properties: { groupId: `${SERVICE}/groups/support-team` } }; + yield { name: 'link-c', properties: { groupId: `${SERVICE}/workspaces/team-a/groups/developers` } }; + } + }; + client.getResource.mockImplementation(async (_ctx, descriptor) => { + if (descriptor.type === ResourceType.Workspace) return { name: 'team-a', properties: {} }; + if ([ResourceType.Tag, ResourceType.Group].includes(descriptor.type)) { + return { name: descriptor.nameParts[0], properties: {} }; + } + return undefined; + }); + const filter: FilterConfig = { + groups: ['!partners', '*'], + workspaces: ['team-a'], + workspaceSubFilters: { + 'team-a': { apis: ['orders'], products: ['starter'], tags: ['!internal', 'none'], groups: [] }, + }, + }; + + await extractWorkspaces(client, createAssociationStore(), testContext, '/output', filter, true); + + const fetched = client.getResource.mock.calls + .map(([, d]) => `${d.type}:${d.workspace ?? 'service'}:${d.nameParts.join('/')}`) + .filter((key) => key.startsWith('Tag') || key.startsWith('Group')); + expect(fetched.sort()).toEqual([ + 'Group:service:support-team', + // A workspace group named like a built-in group is a custom group. + 'Group:team-a:developers', + 'Tag:team-a:public', + ]); + }); + + /** + * A client whose workspaces each have product 'starter' linked to service + * group 'support-team'. APIM refuses that link for a custom group; it + * stands in for any service-level dependency a workspace can have. + */ + function createSharedGroupClient(workspaces: string[]) { + const client = createMockClient(); + client.listResources = async function* (ctx: ApimServiceContext, type: ResourceType) { + if (type === ResourceType.Workspace) { + for (const name of workspaces) yield { name, properties: {} }; + } + if (type === ResourceType.Product && ctx.baseUrl.includes('/workspaces/')) yield { name: 'starter', properties: {} }; + if (type === ResourceType.ProductGroup) { + yield { name: 'link-1', properties: { groupId: `${SERVICE}/groups/support-team` } }; + } + }; + client.getResource.mockImplementation(async (_ctx, descriptor) => + ({ name: descriptor.nameParts[0], properties: {} }) + ); + return client; + } + + const groupGets = (client: ReturnType) => + client.getResource.mock.calls.filter(([, d]) => d.type === ResourceType.Group).length; + + /** A filter extracting product 'starter' from each workspace through a sub-filter. */ + const starterFilter = (workspaces: string[]): FilterConfig => ({ + groups: [], + workspaces, + workspaceSubFilters: Object.fromEntries(workspaces.map((name) => [name, { products: ['starter'] }])), + }); + + it('should not resolve dependencies in a workspace extracted without a sub-filter', async () => { + const client = createSharedGroupClient(['team-a']); + const filter: FilterConfig = { groups: ['partners'], workspaces: ['team-a'] }; + + await extractWorkspaces(client, createAssociationStore(), testContext, '/output', filter, true); + + expect(groupGets(client)).toBe(0); + }); + + it('should fetch a service-level dependency once across workspaces', async () => { + const client = createSharedGroupClient(['team-a', 'team-b']); + const filter = starterFilter(['team-a', 'team-b']); + + const store = createAssociationStore(); + + await extractWorkspaces(client, store, testContext, '/output', filter, true); + + expect(groupGets(client)).toBe(1); + expect(store.writeResource.mock.calls.filter(([, d]) => d.type === ResourceType.Group)).toHaveLength(1); + }); + + it('should not fetch a service-level dependency the service extraction already wrote', async () => { + const client = createSharedGroupClient(['team-a']); + const filter = starterFilter(['team-a']); + + await extractWorkspaces( + client, createAssociationStore(), testContext, '/output', filter, true, + [{ type: ResourceType.Group, nameParts: ['support-team'] }] + ); + + expect(groupGets(client)).toBe(0); + }); + + it('should try a missing service-level dependency once across workspaces', async () => { + const client = createSharedGroupClient(['team-a', 'team-b']); + client.getResource.mockImplementation(async (_ctx, descriptor) => + descriptor.type === ResourceType.Group ? undefined : { name: descriptor.nameParts[0], properties: {} } + ); + const filter = starterFilter(['team-a', 'team-b']); + + const results = await extractWorkspaces(client, createAssociationStore(), testContext, '/output', filter, true); + + expect(groupGets(client)).toBe(1); + expect(results.map((ws) => ws.errorCount)).toEqual([1, 0]); + }); + + it.each([ + ['the tags filter only excludes by name', { apis: ['orders'], tags: ['!internal'] }], + ['no APIs or products were extracted', { apis: [], products: [], tags: [] }], + ['the tags filter includes every tag', { apis: ['orders'], tags: ['*', '!internal'] }], + ['the tags key is left out, so every tag is included', { apis: ['orders'] }], + ])('should not list the workspace tags again when %s', async (_case, subFilter) => { + const client = createMockClient(); + let tagListings = 0; + client.listResources = async function* (_ctx: ApimServiceContext, type: ResourceType) { + if (type === ResourceType.Api) yield { name: 'orders', properties: {} }; + if (type === ResourceType.Tag) { + tagListings++; + yield { name: 'public', properties: {} }; + } + }; + client.getResource.mockResolvedValue({ name: 'team-a', properties: {} }); + const filter: FilterConfig = { workspaces: ['team-a'], workspaceSubFilters: { 'team-a': subFilter } }; + + await extractWorkspaces(client, createAssociationStore(), testContext, '/output', filter, true); + + expect(tagListings).toBe(1); + }); + + it('should only use extracted tags for links without transitive resolution', async () => { + const client = createMockClient(); + const listed: ResourceType[] = []; + client.listResources = async function* (_ctx: ApimServiceContext, type: ResourceType) { + listed.push(type); + if (type === ResourceType.Api) yield { name: 'orders', properties: {} }; + }; + client.getResource.mockResolvedValue({ name: 'team-a', properties: {} }); + const filter: FilterConfig = { + workspaces: ['team-a'], + workspaceSubFilters: { 'team-a': { apis: ['orders'], tags: [] } }, + }; + + await extractWorkspaces(client, createAssociationStore(), testContext, '/output', filter, false); + + expect(listed).not.toContain(ResourceType.ApiTag); + }); + }); + describe('resolveWorkspaceFilter', () => { + it('should apply a sub-filter keyed by a pattern to every workspace it matches', () => { + const filter: FilterConfig = { + workspaces: ['team-*'], + workspaceSubFilters: { + 'team-*': { apis: ['orders'], namedValues: [] }, + 'team-b': { apis: ['billing'] }, + '!team-c': { apis: [] }, + }, + }; + + expect(resolveWorkspaceFilter('team-a', filter)).toEqual( + expect.objectContaining({ apis: ['orders'], namedValues: [] }) + ); + // A sub-filter keyed by the workspace's name takes precedence over a pattern. + expect(resolveWorkspaceFilter('TEAM-B', filter)).toEqual(expect.objectContaining({ apis: ['billing'] })); + // Exclusion keys never apply. + expect(resolveWorkspaceFilter('team-c', filter)).toEqual(expect.objectContaining({ apis: ['orders'] })); + expect(resolveWorkspaceFilter('platform', filter)).toBeUndefined(); + }); + it('should return undefined when no filter provided', () => { expect(resolveWorkspaceFilter('ws-1')).toBeUndefined(); });