Rp Source Wordpress

scripts/wp-discovery.js

#!/usr/bin/env node

const fs = require('node:fs/promises');
const path = require('node:path');
const { createProgressLogger, parseProgressArgs } = require('../../../lib/progress-log.js');
const { readEnvFile } = require('../../../lib/config-env.js');
const { buildWordPressCoverageRows, buildCoverageFile, ADAPTER_RAW_DIRS } = require('../../../lib/source-coverage.js');
const { writeAdapterSourceRegional } = require('../../../lib/source-regional.js');
const regionalCapture = require('../lib/regional-capture.js');

// Transport, auth, throttling, and pagination-header parsing live in the shared
// adapter lib so the bulk-extract reader generated by rp-import-codegen inherits
// the exact same discipline instead of re-deriving it.
const {
  DEFAULT_TIMEOUT_MS,
  DEFAULT_RATE_LIMIT_RPM,
  DEFAULT_MAX_RETRIES,
  configureRateLimit,
  buildHeaders,
  normalizeBaseUrl,
  fetchJson,
  parseTotalHeader,
} = require('../lib/wp-http.js');
const {
  classifyRoutes,
  summarizeSkippedByCategory,
  buildRegisteredRestBases,
  defaultPluginRules,
  defaultQueryFor,
  defaultQueryReasonFor,
} = require('../lib/wp-route-classifier.js');
// Plugin awareness: detection runs before classification so profile routes are
// in scope; embedded detection and Tier-B derivation run after sampling because they need
// real record payloads and the accepted route set.
const { gatherInventory, inventoryPayload } = require('./wp-plugin-inventory.js');
// discover-structure/query-structure: the only source for entities whose
// blocked[].fulfillment.kind is structure-bridge-plugin -- data with no REST route at all.
const {
  discoverStructure, queryStructure, guardedValueColumns, violatesTierRule,
  discoverEavKeys, buildEavReadPlan, MAX_PAGE_LIMIT,
} = require('../lib/wix-wp-plugin-v2-client.js');
const redirectSources = require('../lib/wp-redirect-sources.js');
const {
  pluginsRoot,
  loadProfiles,
  loadNoMigrationNeeded,
  loadRequiresDevelopment,
  loadPendingDecisions,
  loadFingerprintAliases,
  loadCutoverTasks,
  loadCmsFallbackEligible,
  CHILD_ROUTE_PLACEHOLDER,
  buildResponseEnvelopes,
  buildRequestOverrides,
  buildResponseFragmentGroups,
  buildRecordKeyFields,
} = require('../lib/plugin-knowledge.js');
const {
  buildDispositionRows,
  summarizeDispositions,
} = require('../lib/plugin-disposition.js');
const {
  cmsFallbackDecisionKeys,
  cmsFallbackDecisionShape,
  capabilityOfKey: cmsFallbackCapabilityOfKey,
  cmsFallbackResolutionInputs,
  cmsFallbackInputsDigest,
  detectedPluginsOf,
} = require('../../../lib/plugin-cms-fallback.js');
const {
  detectPlugins,
  describeProfiledEntity,
  collectRecordProperties,
  collectUnprofiledRoutes,
  collectCandidateNamespaces,
  deriveGenericEntities,
  deriveBridgePlannedEntities,
  classifyCoverage,
  coverageSummary,
  registerKeys,
} = require('../lib/wp-plugin-detect.js');
const { collectPresenceObservations, presenceByPlugin } = require('../lib/plugin-data-presence.js');
const {
  collectAllIds,
  queryInBatches,
  DEFAULT_BATCH_SIZE,
} = require('../lib/sampled-ids-batch.js');

const DEFAULT_SAMPLE_LIMIT = 3;
let progress;

function printUsage() {
  console.log(`Usage:
  node wp-discovery.js --base-url <url> --out-dir <dir> [auth options]

Required:
  --base-url <url>              WordPress site base URL, e.g. https://example.com
  --out-dir <dir>               Directory to write discovery markdown files into
  --decisions <path>            orchestration/decisions.json to read batched-ask answers from
                                (default: <out-dir>/../../orchestration/decisions.json)
  --env-file <path>             Load defaults from a project-local env file

Reclassify without refetching (no source network calls):
  node wp-discovery.js --reclassify-coverage --out-dir <dir> [--decisions <path>]
                       [--canonical-coverage <path>]
  Rebuilds <out-dir>/plugin-coverage.json from the stored plugin-inventory.json, the prior
  coverage's entityStats/review evidence, and the current profiles, registers, target
  knowledge and decisions. Also rewrites --canonical-coverage (default:
  <out-dir>/../../discovery/plugin-coverage.json when that directory exists). Exits 3 with
  outcome needs-full-discovery, writing nothing, when the stored evidence is incomplete.

Authentication options:
  --username <name>             WordPress username for Application Password auth
  --application-password <pw>   WordPress Application Password
  --api-key <token>             API key/token for custom auth setups
  --api-key-header <name>       Header name for --api-key. Defaults to Authorization
  --auth-header <'Name: Value'> Add a raw HTTP header. Can be repeated.

Optional:
  --sample-limit <n>            Number of sample records per entity. Default: 3
  --timeout-ms <n>              Request timeout in ms. Default: 60000
  --rate-limit-rpm <n>          Max requests per minute. Default: 120
  --max-retries <n>             Retries on 429/503 (honors Retry-After). Default: 3
  --commerce-mode <mode>        WooCommerce read mode: public | authenticated.
                                Defaults to public without auth, otherwise authenticated.
  --no-plugin-inventory         Skip plugin detection, Tier-B derivation, and coverage.
  --no-html-fingerprint         Skip the homepage fetch used for asset-path fingerprints.
  --include-namespace <ns>      Only inspect a namespace. Can be repeated.
  --include-route <path>        Force-sample a route. Can be repeated.
  --include-excluded-category <category>
                                Force-sample an excluded category. Can be repeated.
  --exclude-route <path>        Skip a route even if otherwise sampled. Can be repeated.
  --override-reason <text>      Reason recorded for include/exclude overrides.
  --progress-log <path>         Append progress NDJSON records to this file.
  --help                        Show this help text

Examples:
  node wp-discovery.js \
    --base-url https://example.com \
    --out-dir migrations/acme/data/wp-discovery \
    --username admin \
    --application-password 'abcd efgh ijkl mnop'

  node wp-discovery.js \
    --base-url https://example.com \
    --out-dir migrations/acme/data/wp-discovery \
    --api-key $WP_API_KEY \
    --api-key-header X-API-Key
`);
}

function parseArgs(argv) {
  const args = {
    authHeaders: [],
    includeNamespaces: [],
    includeRoutes: [],
    includeExcludedCategories: [],
    excludeRoutes: [],
    overrideReason: null,
    sampleLimit: DEFAULT_SAMPLE_LIMIT,
    timeoutMs: DEFAULT_TIMEOUT_MS,
    rateLimitRpm: DEFAULT_RATE_LIMIT_RPM,
    maxRetries: DEFAULT_MAX_RETRIES,
    commerceMode: null,
    pluginInventory: true,
    htmlFingerprint: true,
  };

  for (let i = 0; i < argv.length; i += 1) {
    const arg = argv[i];
    const next = argv[i + 1];

    switch (arg) {
      case '--help':
      case '-h':
        args.help = true;
        break;
      case '--base-url':
        args.baseUrl = next;
        i += 1;
        break;
      case '--out-dir':
        args.outDir = next;
        i += 1;
        break;
      case '--decisions':
        args.decisions = next;
        i += 1;
        break;
      case '--env-file':
        args.envFile = next;
        i += 1;
        break;
      case '--username':
        args.username = next;
        i += 1;
        break;
      case '--application-password':
        args.applicationPassword = next;
        i += 1;
        break;
      case '--api-key':
        args.apiKey = next;
        i += 1;
        break;
      case '--api-key-header':
        args.apiKeyHeader = next;
        i += 1;
        break;
      case '--auth-header':
        args.authHeaders.push(next);
        i += 1;
        break;
      case '--sample-limit':
        args.sampleLimit = Number.parseInt(next, 10);
        i += 1;
        break;
      case '--timeout-ms':
        args.timeoutMs = Number.parseInt(next, 10);
        i += 1;
        break;
      case '--rate-limit-rpm':
        args.rateLimitRpm = Number.parseInt(next, 10);
        i += 1;
        break;
      case '--max-retries':
        args.maxRetries = Number.parseInt(next, 10);
        i += 1;
        break;
      case '--commerce-mode':
        args.commerceMode = next;
        i += 1;
        break;
      case '--include-namespace':
        args.includeNamespaces.push(next);
        i += 1;
        break;
      case '--include-route':
        args.includeRoutes.push(next);
        i += 1;
        break;
      case '--include-excluded-category':
        args.includeExcludedCategories.push(next);
        i += 1;
        break;
      case '--exclude-route':
        args.excludeRoutes.push(next);
        i += 1;
        break;
      case '--override-reason':
        args.overrideReason = next;
        i += 1;
        break;
      case '--no-plugin-inventory':
        args.pluginInventory = false;
        break;
      case '--no-html-fingerprint':
        args.htmlFingerprint = false;
        break;
      case '--reclassify-coverage':
        args.reclassifyCoverage = true;
        break;
      case '--canonical-coverage':
        args.canonicalCoverage = next;
        i += 1;
        break;
      // Fixture-tree overrides for tests; both default to the installed skill.
      case '--plugins-root':
        args.pluginsRoot = next;
        i += 1;
        break;
      case '--target-knowledge-root':
        args.targetKnowledgeRoot = next;
        i += 1;
        break;
      default:
        if (arg.startsWith('--')) {
          throw new Error(`Unknown argument: ${arg}`);
        }
    }
  }

  if (!args.baseUrl) {
    args.baseUrl = process.env.WP_BASE_URL || process.env.WP_SITE_URL;
  }
  if (!args.outDir) {
    args.outDir = process.env.WP_DISCOVERY_OUT_DIR;
  }
  if (!args.apiKey) {
    args.apiKey = process.env.WP_API_KEY;
  }
  if (!args.apiKeyHeader) {
    args.apiKeyHeader = process.env.WP_API_KEY_HEADER;
  }
  if (!args.username) {
    args.username = process.env.WP_USERNAME;
  }
  if (!args.applicationPassword) {
    args.applicationPassword = process.env.WP_APPLICATION_PASSWORD;
  }
  if (args.authHeaders.length === 0 && process.env.WP_AUTH_HEADER) {
    args.authHeaders.push(process.env.WP_AUTH_HEADER);
  }

  if (!Number.isFinite(args.sampleLimit) || args.sampleLimit < 1) {
    args.sampleLimit = DEFAULT_SAMPLE_LIMIT;
  }
  if (!Number.isFinite(args.timeoutMs) || args.timeoutMs < 1000) {
    args.timeoutMs = DEFAULT_TIMEOUT_MS;
  }
  if (!Number.isFinite(args.rateLimitRpm) || args.rateLimitRpm < 1) {
    args.rateLimitRpm = DEFAULT_RATE_LIMIT_RPM;
  }
  if (!Number.isFinite(args.maxRetries) || args.maxRetries < 0) {
    args.maxRetries = DEFAULT_MAX_RETRIES;
  }
  if (args.commerceMode !== 'public' && args.commerceMode !== 'authenticated') {
    args.commerceMode = (args.username && args.applicationPassword) || args.apiKey || args.authHeaders.length > 0
      ? 'authenticated'
      : 'public';
  }

  return args;
}

async function hydrateArgsFromEnvFile(args) {
  if (!args.envFile) {
    return args;
  }

  const envValues = await readEnvFile(path.resolve(args.envFile));
  if (!args.baseUrl && envValues.WP_BASE_URL) {
    args.baseUrl = envValues.WP_BASE_URL;
  }
  if (!args.username && envValues.WP_USERNAME) {
    args.username = envValues.WP_USERNAME;
  }
  if (!args.applicationPassword && envValues.WP_APPLICATION_PASSWORD) {
    args.applicationPassword = envValues.WP_APPLICATION_PASSWORD;
  }
  if (!args.apiKey && envValues.WP_API_KEY) {
    args.apiKey = envValues.WP_API_KEY;
  }
  if (!args.apiKeyHeader && envValues.WP_API_KEY_HEADER) {
    args.apiKeyHeader = envValues.WP_API_KEY_HEADER;
  }
  if (args.authHeaders.length === 0 && envValues.WP_AUTH_HEADER) {
    args.authHeaders.push(envValues.WP_AUTH_HEADER);
  }

  return args;
}

function endpointMethods(endpoint) {
  const raw = endpoint?.methods;
  if (Array.isArray(raw)) {
    return raw.map(String);
  }
  if (typeof raw === 'string') {
    return raw.split(',').map((value) => value.trim()).filter(Boolean);
  }
  if (raw && typeof raw === 'object') {
    return Object.keys(raw);
  }
  return [];
}

function routeSegments(routePath) {
  return routePath.split('/').filter(Boolean);
}

function summarizeRelationships(record) {
  const links = record?._links;
  if (!links || typeof links !== 'object') {
    return [];
  }
  // HAL housekeeping rels carry no entity relationship signal.
  const ignored = new Set(['self', 'collection', 'about', 'curies']);
  const relationships = [];

  for (const [rel, entries] of Object.entries(links)) {
    if (ignored.has(rel)) {
      continue;
    }
    const list = Array.isArray(entries) ? entries : [entries];
    const hrefs = list.map((entry) => entry?.href).filter(Boolean);
    if (hrefs.length === 0) {
      continue;
    }
    relationships.push({
      rel,
      embeddable: list.some((entry) => entry?.embeddable === true),
      hrefs,
    });
  }

  return relationships;
}

function isParameterizedRoute(routePath) {
  return routePath.includes('(?P<');
}

function slugify(value) {
  return value
    .toLowerCase()
    .replace(/[^a-z0-9]+/g, '-')
    .replace(/^-+|-+$/g, '') || 'entity';
}

function summarizeAuthMode(args) {
  if (args.username && args.applicationPassword) {
    return 'basic-application-password';
  }
  if (args.apiKey) {
    return `api-key:${args.apiKeyHeader || 'Authorization'}`;
  }
  if (args.authHeaders.length > 0) {
    return 'custom-header';
  }
  return 'none';
}

// wix-wp-plugin-v2 0.5.0+ admits `manage_woocommerce` or `manage_options`, so
// one WordPress REST credential reaches the bridge with or without WooCommerce. There is no
// other bridge credential.
function summarizeBridgeAuthMode(args) {
  const hasRestCredential = Boolean(
    (args.username && args.applicationPassword) || args.apiKey || args.authHeaders.length > 0,
  );
  return hasRestCredential ? 'wordpress-rest-credential' : 'none';
}

// A 403 from the bridge on a site without WooCommerce means a plugin copy older than 0.5.0,
// whose routes required `manage_woocommerce` (a capability that exists only with
// WooCommerce). The fix is updating the plugin, not a different credential.
const BRIDGE_PLUGIN_UPDATE_REQUIRED = 'bridge_plugin_update_required';
function bridgeForbiddenCode(status, namespaces) {
  if (status !== 403) return null; // 401 is a missing/invalid credential, not an old plugin
  return (namespaces || []).some((namespace) => namespace.startsWith('wc/')) ? null : BRIDGE_PLUGIN_UPDATE_REQUIRED;
}

function summarizeOverrides(args) {
  return {
    includeRoutes: args.includeRoutes,
    includeNamespaces: args.includeNamespaces,
    includeExcludedCategories: args.includeExcludedCategories,
    excludeRoutes: args.excludeRoutes,
    overrideReason: args.overrideReason,
    commerceMode: args.commerceMode,
  };
}

function deriveEntityCandidates(indexJson, includeNamespaces, requestOverrides = new Map()) {
  const routes = indexJson?.routes || {};
  const candidates = [];

  for (const [routePath, routeDefinition] of Object.entries(routes)) {
    if (!routePath.startsWith('/') || routePath === '/' || isParameterizedRoute(routePath)) {
      continue;
    }

    const segments = routeSegments(routePath);
    if (segments.length < 2) {
      continue;
    }

    const namespace = segments.slice(0, 2).join('/');
    if (includeNamespaces.length > 0 && !includeNamespaces.includes(namespace)) {
      continue;
    }

    const endpoints = Array.isArray(routeDefinition?.endpoints) ? routeDefinition.endpoints : [];
    // A profile may declare that this entity's real read path is not GET (see
    // plugin-knowledge.js buildRequestOverrides) — the declared method then stands in for
    // the GET-collection requirement below, generically for any plugin/entity.
    const requestOverride = requestOverrides.get(routePath) || null;
    const getEndpoint = requestOverride
      ? endpoints.find((endpoint) => endpointMethods(endpoint).includes(requestOverride.method))
      : endpoints.find((endpoint) => endpointMethods(endpoint).includes('GET'));
    if (!getEndpoint) {
      continue;
    }

    const entityName = segments[segments.length - 1];
    const supportsPagination = Boolean(getEndpoint?.args?.page || getEndpoint?.args?.per_page);
    const hasSchema = Boolean(getEndpoint?.schema || routeDefinition?.schema);

    // The generic shape heuristic (schema or pagination or 3+ path segments) exists to
    // filter out config/dashboard noise when nothing else vouches for a route. A profile
    // explicitly declaring a request override already is that vouching — same principle as
    // profile-declared routes outranking the classifier's shape rules.
    if (!requestOverride && !hasSchema && !supportsPagination && segments.length < 3) {
      continue;
    }

    // Derive the file name from the full path after the namespace so distinct
    // routes that share a last segment (e.g. /wp/v2/categories vs.
    // /wp/v2/block-patterns/categories) do not collide onto one file.
    const pathSlug = slugify(segments.slice(2).join('-')) || slugify(entityName);

    candidates.push({
      entityName,
      namespace,
      routePath,
      endpoints,
      getEndpoint,
      routeDefinition,
      supportsPagination,
      requestOverride,
      fileName: `${slugify(namespace)}--${pathSlug}.md`,
    });
  }

  const unique = new Map();
  for (const candidate of candidates) {
    const key = `${candidate.namespace}:${candidate.routePath}`;
    if (!unique.has(key)) {
      unique.set(key, candidate);
    }
  }

  return [...unique.values()].sort((a, b) => a.routePath.localeCompare(b.routePath));
}

// Finds every route named by a "$SAMPLED_IDS:<route>" placeholder anywhere
// inside a requestOverride body, however deeply nested. A body may reference more than one
// route (e.g. two independent id lists); collecting all of them, not just the first match, is
// what lets the pagination+batch mechanism below (lib/sampled-ids-batch.js) discover every
// route it must paginate to exhaustion rather than resolving one placeholder to []. Generic
// body-walk, no field names baked in.
function findSampledIdsDependencies(value, found = new Set()) {
  if (Array.isArray(value)) {
    for (const item of value) {
      findSampledIdsDependencies(item, found);
    }
  } else if (value && typeof value === 'object') {
    for (const nested of Object.values(value)) {
      findSampledIdsDependencies(nested, found);
    }
  } else if (typeof value === 'string') {
    const match = value.match(/^\$SAMPLED_IDS:(.+)$/);
    if (match) found.add(match[1]);
  }
  return found;
}

// Resolves a dot-path ("data.items", "data.meta.count") against a response body for a
// responseEnvelope-declaring entity. Returns undefined on any missing/non-object segment
// rather than throwing, so an envelope path that stops matching (a plugin update, a stale
// profile) degrades to the safe fallback in inspectEntity instead of crashing the run.
function getAtPath(value, dotPath) {
  return String(dotPath).split('.').reduce(
    (acc, key) => (acc && typeof acc === 'object' ? acc[key] : undefined),
    value,
  );
}

// Deep-resolves a profile-declared requestBody template against a batch's ids. `idsByRoute` is
// a plain `{ route: string[] }` map — for the $SAMPLED_IDS mechanism this is
// always the CURRENT BATCH's ids for the route being paginated, resolved fresh per batch, never
// a fixed full list baked into one oversized request. Generic across any entity/route pair —
// the placeholder names the OTHER route's ids it wants, nothing plugin-specific lives here.
function resolveRequestBody(template, idsByRoute) {
  if (Array.isArray(template)) {
    return template.map((value) => resolveRequestBody(value, idsByRoute));
  }
  if (template && typeof template === 'object') {
    const resolved = {};
    for (const [key, value] of Object.entries(template)) {
      resolved[key] = resolveRequestBody(value, idsByRoute);
    }
    return resolved;
  }
  if (typeof template === 'string') {
    const match = template.match(/^\$SAMPLED_IDS:(.+)$/);
    if (match) {
      return (idsByRoute && idsByRoute[match[1]]) || [];
    }
  }
  return template;
}

// Envelope resolution + fragment-group reassembly, shared by the single-fetch path below and
// by every batch of the paginate+batch path (lib/sampled-ids-batch.js) — one profile's response
// shape must normalize the same way regardless of which path fetched it. Pushes discovery
// notes only when `notes` is provided (batched callers pass it for the first batch only, since
// the shape characteristic belongs to the endpoint, not to any one batch). Returns `{ ok,
// records }`: `ok: false` means the payload did not resolve to an array after envelope
// resolution (when declared) and fragment reassembly (when declared) — a genuine shape
// mismatch, not "zero records" — and callers on the batched path must treat that as a failure
// to defer on, never silently as an empty batch (an enveloped response whose itemsPath
// mismatches would otherwise look like an authoritative, exact zero).
function normalizeResponseRecords(payload, { envelope, fragmentGroupSize, notes } = {}) {
  let effectivePayload = payload;
  if (envelope) {
    const items = getAtPath(payload, envelope.itemsPath);
    if (Array.isArray(items)) {
      effectivePayload = items;
    } else if (notes) {
      notes.push(`responseEnvelope.itemsPath="${envelope.itemsPath}" did not resolve to an array on this response; falling back to the raw payload shape. The profile may be stale or the plugin version differs.`);
    }
  }
  if (fragmentGroupSize && Array.isArray(effectivePayload)) {
    const completeGroups = Math.floor(effectivePayload.length / fragmentGroupSize);
    const reassembled = [];
    for (let i = 0; i < completeGroups; i += 1) {
      reassembled.push(Object.assign({}, ...effectivePayload.slice(i * fragmentGroupSize, (i + 1) * fragmentGroupSize)));
    }
    if (notes && effectivePayload.length % fragmentGroupSize !== 0) {
      notes.push(`Response length ${effectivePayload.length} is not a multiple of the declared responseFragmentGroupSize=${fragmentGroupSize}; the trailing ${effectivePayload.length % fragmentGroupSize} fragment(s) were dropped rather than emitted as a broken record.`);
    }
    if (notes) {
      notes.push(`Response records were flattened into groups of ${fragmentGroupSize} single-key fragments (per profile responseFragmentGroupSize) and reassembled into one object per record.`);
    }
    effectivePayload = reassembled;
  }
  const ok = Array.isArray(effectivePayload);
  return { ok, records: ok ? effectivePayload : [] };
}

async function inspectEntity(baseUrl, headers, candidate, options) {
  const details = {
    entityName: candidate.entityName,
    namespace: candidate.namespace,
    routePath: candidate.routePath,
    fileName: candidate.fileName,
    classification: candidate.classification || null,
    methods: [...new Set(candidate.endpoints.flatMap(endpointMethods))].sort(),
    supportsPagination: candidate.supportsPagination,
    discoveryNotes: [],
    requestErrors: [],
    collectionArgs: candidate.getEndpoint?.args || null,
    schema: candidate.getEndpoint?.schema || candidate.routeDefinition?.schema || null,
    optionsSchema: null,
    sampleRecords: [],
    sampleRecordCount: 0,
    recordCount: null,
    inUse: null,
    relationships: [],
    responseShape: 'unknown',
  };

  const optionsResponse = await fetchJson(baseUrl, candidate.routePath, {
    headers,
    method: 'OPTIONS',
    timeoutMs: options.timeoutMs,
    progress: options.progress,
    progressContext: {
      step: 'inspect-endpoint',
      entity: candidate.routePath,
    },
  });

  if (optionsResponse.ok && optionsResponse.json) {
    const optionEndpoints = Array.isArray(optionsResponse.json?.endpoints) ? optionsResponse.json.endpoints : [];
    const getEndpoint = optionEndpoints.find((endpoint) => endpointMethods(endpoint).includes('GET'));
    if (getEndpoint?.args) {
      details.collectionArgs = getEndpoint.args;
    }
    if (optionsResponse.json?.schema) {
      details.optionsSchema = optionsResponse.json.schema;
      details.schema = optionsResponse.json.schema;
    } else if (getEndpoint?.schema) {
      details.optionsSchema = getEndpoint.schema;
      details.schema = getEndpoint.schema;
    }
  } else if (optionsResponse.status !== 404 && optionsResponse.status !== 405) {
    details.requestErrors.push({
      request: 'OPTIONS',
      routePath: candidate.routePath,
      status: optionsResponse.status,
      statusText: optionsResponse.statusText,
      url: optionsResponse.url,
    });
  }

  // A profile may declare that this entity's real read path is a non-GET request with a
  // JSON body — generic across any plugin/entity, not special-cased here.
  const requestOverride = options.requestOverride || null;
  const dependencyRoutes = requestOverride ? [...findSampledIdsDependencies(requestOverride.body)] : [];

  if (requestOverride && dependencyRoutes.length > 0) {
    // The $SAMPLED_IDS mechanism always means "paginate the dependency route to exhaustion,
    // then batch-query this route" — the dependency's record count is unknown, so
    // there is no bounded "small sample" variant that is safe to assume complete. This is the
    // ENTIRE meaning of a $SAMPLED_IDS placeholder now, in both discovery and any generated
    // reader (see rp-import-codegen's sourceMeta contract).
    return inspectDependentEntity(baseUrl, headers, candidate, options, details, requestOverride, dependencyRoutes);
  }

  let sampleResponse;
  if (requestOverride) {
    // A fixed-body override with no $SAMPLED_IDS placeholder — single request, unbounded by
    // definition since nothing here scales with another route's record count.
    const resolvedBody = resolveRequestBody(requestOverride.body, {});
    details.discoveryNotes.push(
      `Sampled via profile-declared ${requestOverride.method} request with a JSON body, not a plain GET collection.`,
    );
    sampleResponse = await fetchJson(baseUrl, candidate.routePath, {
      // `headers` is a real Headers instance (wp-http.js buildHeaders) — spreading it
      // would silently drop every entry including Authorization, since Headers is not a
      // plain object. Clone it properly instead.
      headers: (() => {
        const withBody = new Headers(headers);
        withBody.set('content-type', 'application/json');
        return withBody;
      })(),
      method: requestOverride.method,
      body: JSON.stringify(resolvedBody),
      timeoutMs: options.timeoutMs,
      progress: options.progress,
      progressContext: {
        step: 'inspect-endpoint',
        entity: candidate.routePath,
      },
    });
  } else {
    // Per-route default query (see ROUTE_DEFAULT_QUERY_RULES): some collection routes apply
    // a default filter when the caller sends none, so sampling with `per_page` alone reads a
    // subset and the plan under-counts. Paging is layered on top; a route with no rule is
    // unchanged.
    const defaultQuery = options.defaultQuery || {};
    const query = {
      ...defaultQuery,
      ...(candidate.supportsPagination ? { per_page: options.sampleLimit } : {}),
    };
    if (Object.keys(defaultQuery).length > 0) {
      const pairs = Object.entries(defaultQuery).map(([key, value]) => `${key}=${value}`).join('&');
      const reason = options.defaultQueryReason ? ` — ${options.defaultQueryReason}` : '';
      details.discoveryNotes.push(`Sampled with route default query ${pairs} so the full collection is counted${reason}. Generated readers must send the same parameters.`);
    }

    sampleResponse = await fetchJson(baseUrl, candidate.routePath, {
      headers,
      method: 'GET',
      query,
      timeoutMs: options.timeoutMs,
      progress: options.progress,
      progressContext: {
        step: 'inspect-endpoint',
        entity: candidate.routePath,
      },
    });
  }

  if (sampleResponse.ok) {
    const payload = sampleResponse.json;
    const totalFromHeader = parseTotalHeader(sampleResponse.headers);

    // responseEnvelope support: some plugin REST APIs (MailPoet's
    // /mailpoet/v1/* namespace, verified live 2026-08-11) wrap their records in a body path
    // instead of returning a flat array, and carry no X-WP-Total/X-WP-TotalPages headers —
    // the total lives inside the body too. Unwrap to the declared path and otherwise treat it
    // exactly like a flat array; a path that no longer resolves (stale profile, plugin
    // version drift) falls back to the raw payload rather than failing the run.
    const envelope = options.responseEnvelope || null;
    let effectivePayload = payload;
    let envelopeCount = null;
    if (envelope) {
      const items = getAtPath(payload, envelope.itemsPath);
      if (Array.isArray(items)) {
        effectivePayload = items;
        envelopeCount = envelope.countPath ? getAtPath(payload, envelope.countPath) : null;
      } else {
        details.discoveryNotes.push(`responseEnvelope.itemsPath="${envelope.itemsPath}" did not resolve to an array on this response; falling back to the raw payload shape. The profile may be stale or the plugin version differs.`);
      }
    }

    // responseFragmentGroupSize support: some plugin REST APIs (Back In Stock Notifier's
    // list_subscriber, verified live 2026-08-19) emit each logical record as N separate
    // single-key array entries in sequence instead of one merged object. Reassemble every N
    // entries into one record via Object.assign before anything downstream counts or samples
    // — a reader built from a stale sample would otherwise see 4x the real record count and a
    // schema missing whichever key happened to be sampled out of order. A trailing partial
    // group (malformed response, plugin bug) is dropped and noted rather than emitted broken.
    const fragmentGroupSize = options.responseFragmentGroupSize || null;
    if (fragmentGroupSize && Array.isArray(effectivePayload)) {
      effectivePayload = normalizeResponseRecords(effectivePayload, { fragmentGroupSize, notes: details.discoveryNotes }).records;
    }

    if (Array.isArray(effectivePayload)) {
      details.responseShape = envelope && effectivePayload !== payload ? 'enveloped-array' : 'array';
      details.sampleRecords = effectivePayload.slice(0, options.sampleLimit);
      details.sampleRecordCount = effectivePayload.length;
      const hasEnvelopeCount = typeof envelopeCount === 'number';
      details.recordCount = hasEnvelopeCount ? envelopeCount : (totalFromHeader !== null ? totalFromHeader : effectivePayload.length);
      details.inUse = details.recordCount > 0;
      if (details.responseShape === 'enveloped-array' && !hasEnvelopeCount) {
        details.discoveryNotes.push(`responseEnvelope has no working countPath (itemsPath="${envelope.itemsPath}"); recordCount reflects only the sampled page and may undercount the true total.`);
      } else if (details.responseShape === 'array' && totalFromHeader === null) {
        details.discoveryNotes.push('No X-WP-Total header returned; recordCount reflects only the sampled page and may undercount the true total.');
      }
      if (candidate.routePath.startsWith('/wc/store/v1/') && !Object.keys(sampleResponse.headers || {}).some((key) => key.toLowerCase() === 'x-wp-totalpages')) {
        details.discoveryNotes.push('WooCommerce Store API route does not advertise X-WP-TotalPages; generated readers must stop on a short page when bulk-extracting this entity.');
      }
      if (effectivePayload.length === 0) {
        details.discoveryNotes.push('Endpoint returned an empty array. The entity is advertised but appears unused (no records).');
      }
    } else if (effectivePayload && typeof effectivePayload === 'object') {
      details.responseShape = 'object';
      details.sampleRecords = [effectivePayload];
      details.sampleRecordCount = 1;
      details.recordCount = totalFromHeader !== null ? totalFromHeader : 1;
      details.inUse = details.recordCount > 0;
    } else {
      details.responseShape = typeof effectivePayload;
      details.discoveryNotes.push(`Endpoint returned a non-object payload of type ${typeof effectivePayload}.`);
    }

    const firstRecord = details.sampleRecords[0];
    if (firstRecord && typeof firstRecord === 'object') {
      details.relationships = summarizeRelationships(firstRecord);
    }
  } else {
    details.requestErrors.push({
      request: requestOverride ? requestOverride.method : 'GET',
      routePath: candidate.routePath,
      status: sampleResponse.status,
      statusText: sampleResponse.statusText,
      url: sampleResponse.url,
      body: sampleResponse.text ? sampleResponse.text.slice(0, 1000) : '',
    });
  }

  return details;
}

// Paginates every $SAMPLED_IDS dependency route to exhaustion, then batch-queries `candidate`'s
// route — split out of inspectEntity because it is a fundamentally different shape
// (two routes, many requests, an explicit fail/defer path) from a single-fetch inspection.
// Assumes a single batching axis: when a body references more than one dependency route, only
// the first drives batching (noted below) — no known profile needs two independent id lists
// batched together, and this stays honest about that boundary rather than guessing a policy.
async function inspectDependentEntity(baseUrl, headers, candidate, options, details, requestOverride, dependencyRoutes) {
  const batchRoute = dependencyRoutes[0];
  if (dependencyRoutes.length > 1) {
    details.discoveryNotes.push(
      `requestBody references multiple $SAMPLED_IDS routes (${dependencyRoutes.join(', ')}); only the first, ${batchRoute}, drives pagination/batching — a profile needing more than one independent id list is not yet supported generically.`,
    );
  }

  const paged = await collectAllIds({
    baseUrl,
    headers,
    route: batchRoute,
    timeoutMs: options.timeoutMs,
    progress: options.progress,
  });
  if (!paged.ok) {
    // Fail/defer explicitly: recordCount/inUse stay at their initial `null`, which
    // is this codebase's existing "unknown, not zero" signal, backed by a requestErrors entry
    // explaining exactly which page of which route failed — never a false empty or a silent
    // undercount from whatever ids were collected before the failure.
    details.requestErrors.push({
      request: 'GET',
      routePath: paged.failure.route,
      status: paged.failure.status,
      statusText: paged.failure.statusText,
      url: null,
      body: `Failed while paginating the $SAMPLED_IDS dependency route ${paged.failure.route} at page ${paged.failure.page}. ${candidate.routePath}'s record count and schema are unknown and must be treated as deferred, not reported as empty.`,
    });
    return details;
  }

  details.discoveryNotes.push(
    `Sampled via profile-declared ${requestOverride.method} request with a JSON body, against the FULL paginated id list of ${batchRoute} (${paged.ids.length} ids across ${Math.ceil(paged.ids.length / DEFAULT_BATCH_SIZE)} batches of up to ${DEFAULT_BATCH_SIZE}) rather than a fixed small sample.`,
  );

  let shapeNotesEmitted = false;
  const batched = await queryInBatches({
    baseUrl,
    headers,
    route: candidate.routePath,
    method: requestOverride.method,
    ids: paged.ids,
    recordKeyField: options.recordKeyField || 'id',
    buildBody: (batchIds) => resolveRequestBody(requestOverride.body, { [batchRoute]: batchIds.map(String) }),
    // Pass the RAW batch JSON through — response.json may be an enveloped object, not an
    // array, and normalizeResponseRecords (not this callback) is what unwraps it. Returning
    // its {ok, records} result unchanged is what lets queryInBatches distinguish "this batch's
    // response didn't match the declared shape" from "this batch legitimately matched zero
    // subscribers" and defer on the former instead of counting it as an exact zero.
    normalizeBatch: (rawJson) => {
      const normalized = normalizeResponseRecords(rawJson, {
        envelope: options.responseEnvelope,
        fragmentGroupSize: options.responseFragmentGroupSize,
        notes: shapeNotesEmitted ? null : details.discoveryNotes,
      });
      shapeNotesEmitted = true;
      return normalized;
    },
    timeoutMs: options.timeoutMs,
    progress: options.progress,
  });
  if (!batched.ok) {
    const shapeFailure = batched.failure.reason === 'invalid-shape';
    details.requestErrors.push({
      request: requestOverride.method,
      routePath: batched.failure.route,
      status: batched.failure.status,
      statusText: batched.failure.statusText,
      url: null,
      body: shapeFailure
        ? `Batch ${batched.failure.batchIndex + 1} (${batched.failure.batchSize} ids) returned a ${batched.failure.status} response that did not match the declared response shape (responseEnvelope/responseFragmentGroupSize). Batches already merged before this failure are discarded — ${candidate.routePath}'s record count and schema are unknown and must be treated as deferred, not reported as an exact zero.`
        : `Failed on batch ${batched.failure.batchIndex + 1} (${batched.failure.batchSize} ids). Batches already merged before this failure are discarded — ${candidate.routePath}'s record count and schema are unknown and must be treated as deferred, not reported as a partial undercount.`,
    });
    return details;
  }

  details.responseShape = 'array';
  details.sampleRecords = batched.records.slice(0, options.sampleLimit);
  details.sampleRecordCount = batched.records.length;
  // Exact, not an estimate: every dependency id was queried and every batch's records were
  // deduplicated, unlike the sampled-page count elsewhere in this function.
  details.recordCount = batched.records.length;
  details.inUse = details.recordCount > 0;
  details.discoveryNotes.push('recordCount reflects the full deduplicated merge across every batch of the complete dependency id list, not a sampled page — this total is exact.');
  if (batched.records.length === 0) {
    details.discoveryNotes.push('No records matched any batch across the full paginated dependency id list. The entity is advertised but appears unused (no records).');
  }
  const firstRecord = details.sampleRecords[0];
  if (firstRecord && typeof firstRecord === 'object') {
    details.relationships = summarizeRelationships(firstRecord);
  }
  return details;
}

function markdownJsonBlock(value) {
  if (value === null || value === undefined) {
    return '\n\n`Unavailable`';
  }

  return `\n\n\`\`\`json\n${JSON.stringify(value, null, 2)}\n\`\`\``;
}

function renderEntityFile(details) {
  const errorLines = details.requestErrors.length > 0
    ? details.requestErrors.map((error) => `- ${error.request} ${error.routePath}: ${error.status} ${error.statusText}`).join('\n')
    : '- None';

  const notes = details.discoveryNotes.length > 0
    ? details.discoveryNotes.map((note) => `- ${note}`).join('\n')
    : '- None';

  const relationshipLines = details.relationships.length > 0
    ? details.relationships
      .map((relationship) => `- \`${relationship.rel}\`${relationship.embeddable ? ' (embeddable)' : ''} → ${relationship.hrefs.join(', ')}`)
      .join('\n')
    : '- None detected (no `_links` in sample record)';

  const recordCountLabel = details.recordCount === null ? 'unknown' : String(details.recordCount);
  const inUseLabel = details.inUse === null ? 'unknown' : details.inUse ? 'yes' : 'no (advertised but empty)';
  const classificationLines = details.classification
    ? `- Discovery category: \`${details.classification.category}\`\n` +
      `- Discovery rule: \`${details.classification.ruleId}\`\n` +
      `- Discovery reason: ${details.classification.reason}\n` +
      `- Included by override: \`${details.classification.includedByOverride ? 'yes' : 'no'}\`\n` +
      `- Excluded by override: \`${details.classification.excludedByOverride ? 'yes' : 'no'}\`\n`
    : '';

  return `# ${details.entityName}\n\n` +
    `- Namespace: \`${details.namespace}\`\n` +
    `- Route: \`${details.routePath}\`\n` +
    classificationLines +
    `- Methods: ${details.methods.map((method) => `\`${method}\``).join(', ') || '`unknown`'}\n` +
    `- Response shape: \`${details.responseShape}\`\n` +
    `- Record count: \`${recordCountLabel}\`\n` +
    `- In use: \`${inUseLabel}\`\n` +
    `- Sample records captured: ${details.sampleRecords.length}\n\n` +
    `## Notes\n${notes}\n\n` +
    `## Relationships\n${relationshipLines}\n\n` +
    `## Request Errors\n${errorLines}\n\n` +
    `## Schema${markdownJsonBlock(details.schema)}\n\n` +
    `## Collection Args${markdownJsonBlock(details.collectionArgs)}\n\n` +
    `## Sample Records${markdownJsonBlock(details.sampleRecords)}\n`;
}

function renderPluginSection(context) {
  if (!context.plugins) {
    return `## Plugin Coverage\n\nPlugin inventory was not run for this capture.\n\n`;
  }

  const { detection, coverage, summary, unprofiled } = context.plugins;
  const rows = coverage
    .map((row) => {
      const via = row.status === 'migration-planned' ? `${row.via} · ${row.confidence}` : '-';
      const blocked = (row.blocked || []).length > 0
        ? row.blocked.map((blocker) => `${blocker.kind}${blocker.declined ? ' (declined)' : ''}`).join(', ')
        : '-';
      return `| ${row.capability} | \`${row.status}\` | ${via} | ${row.recognized ? 'yes' : 'no'} | ${row.plugins.join(', ') || '-'} | \`${row.channel || 'none'}\` | ${blocked} | ${row.action ? row.action : '-'} |`;
    })
    .join('\n');

  const authNote = detection.pluginListAvailable
    ? ''
    : `**\`GET /wp/v2/plugins\` was unavailable**, so installed-but-unprofiled plugins could not be enumerated. ` +
      `Detection used REST namespaces, declared routes, registered types/taxonomies, and public asset paths only. ` +
      `Re-run with an administrator credential for a complete plugin list.\n\n`;

  const installedRows = detection.installedButUnprofiled
    .map((entry) => `- \`${entry.plugin}\`${entry.name ? ` (${entry.name})` : ''}${entry.active ? '' : ' — inactive'}`)
    .join('\n') || '- None';

  const unprofiledRows = unprofiled
    .map((entry) => `- \`${entry.namespace}\`: ${entry.routes.length} route(s) accepted by shape`)
    .join('\n') || '- None';

  return `## Plugin Coverage\n\n` +
    authNote +
    `- Recognized plugins detected: ${detection.detected.length}\n` +
    `- Derived entities (no profile): ${context.plugins.genericEntities.length}\n` +
    `- Unprofiled namespaces: ${unprofiled.length}\n` +
    `- Installed but unrecognized: ${detection.installedButUnprofiled.length}\n` +
    `- Publicly fingerprinted (named, not enumerated): ${(detection.fingerprinted || []).length}\n` +
    `- Capabilities by status: ${Object.entries(summary.byStatus).map(([status, count]) => `\`${status}\`: ${count}`).join(', ') || 'none'}\n\n` +
    `| Capability | Status | Via | Recognized | Plugins | Channel | Blocked | Action needed |\n` +
    `| --- | --- | --- | --- | --- | --- | --- | --- |\n` +
    `${rows || '| None | - | - | - | - | - | - | - |'}\n\n` +
    `### Installed but unprofiled\n\n${installedRows}\n\n` +
    `### Unprofiled namespaces accepted by shape\n\n${unprofiledRows}\n\n` +
    `Full coverage evidence: [plugin-coverage.json](./plugin-coverage.json) · ` +
    `detection evidence: [plugin-inventory.json](./plugin-inventory.json)\n\n`;
}

function renderIndexFile(context) {
  const entityRows = context.entities.map((entity) => {
    const status = entity.requestErrors.length > 0 ? 'partial' : 'ok';
    const recordCountLabel = entity.recordCount === null ? '?' : String(entity.recordCount);
    const inUseLabel = entity.inUse === null ? '?' : entity.inUse ? 'yes' : 'no';
    return `| ${entity.entityName} | \`${entity.namespace}\` | \`${entity.routePath}\` | ${recordCountLabel} | ${inUseLabel} | ${entity.sampleRecords.length} | ${status} | [${entity.fileName}](./${entity.fileName}) |`;
  }).join('\n');

  const errorLines = context.entities
    .flatMap((entity) => entity.requestErrors.map((error) => `- ${entity.entityName}: ${error.request} ${error.routePath} -> ${error.status} ${error.statusText}`));
  const skippedByCategoryLines = Object.entries(context.skippedByCategory || {})
    .map(([category, count]) => `- \`${category}\`: ${count}`)
    .join('\n') || '- None';
  const overrideLines = [
    `- Include routes: ${context.overrides.includeRoutes.map((route) => `\`${route}\``).join(', ') || '`none`'}`,
    `- Include namespaces: ${context.overrides.includeNamespaces.map((namespace) => `\`${namespace}\``).join(', ') || '`none`'}`,
    `- Include excluded categories: ${context.overrides.includeExcludedCategories.map((category) => `\`${category}\``).join(', ') || '`none`'}`,
    `- Exclude routes: ${context.overrides.excludeRoutes.map((route) => `\`${route}\``).join(', ') || '`none`'}`,
    `- Override reason: ${context.overrides.overrideReason ? context.overrides.overrideReason : '`none`'}`,
  ].join('\n');

  const authGatedEntities = context.entities.filter((entity) =>
    entity.requestErrors.some((error) => error.status === 401 || error.status === 403));
  const authWarning = authGatedEntities.length > 0
    ? `## ⚠️ Incomplete Capture (Authentication)\n\n` +
      `${authGatedEntities.length} entit${authGatedEntities.length === 1 ? 'y' : 'ies'} returned 401/403 and ` +
      `could not be captured` +
      `${context.authMode === 'none' ? ' — this run used **no credentials**' : ` with auth mode \`${context.authMode}\` (insufficient scope)`}.\n` +
      `Their \`recordCount\`/\`inUse\` are unreliable: an auth-gated entity can look empty or errored even when the site uses it heavily (e.g. WooCommerce orders/customers, drafts, private fields).\n` +
      `Re-run with credentials for a complete and trustworthy capture.\n\n` +
      `Auth-gated entities: ${authGatedEntities.map((entity) => `\`${entity.entityName}\``).join(', ')}\n\n`
    : '';

  const pluginSection = renderPluginSection(context);

  return `# WordPress Discovery\n\n` +
    `- Generated at: \`${context.generatedAt}\`\n` +
    `- Base URL: \`${context.baseUrl}\`\n` +
    `- REST root: \`${context.restRoot}\`\n` +
    `- Auth mode: \`${context.authMode}\`\n` +
    `- Advertised auth providers: ${context.authProviders.map((provider) => `\`${provider}\``).join(', ') || '`none`'}\n` +
    `- Namespaces advertised: ${context.namespaces.map((namespace) => `\`${namespace}\``).join(', ') || '`none`'}\n` +
    `- Advertised routes: ${context.totalAdvertisedRoutes}\n` +
    `- Candidate routes after generic filtering: ${context.totalCandidateRoutes}\n` +
    `- Sampled backend data/metadata routes: ${context.sampledRoutes}\n` +
    `- Skipped routes: ${context.skippedRoutes}\n` +
    `- Entities documented: ${context.entities.length}\n\n` +
    authWarning +
    `## Route Scope\n\n` +
    `### Skipped Routes by Category\n\n${skippedByCategoryLines}\n\n` +
    `### Overrides\n\n${overrideLines}\n\n` +
    `Skipped route evidence: [skipped-routes.json](./skipped-routes.json)\n\n` +
    pluginSection +
    `## Discovery Summary\n\n` +
    `| Entity | Namespace | Route | Records | In use | Samples | Status | File |\n` +
    `| --- | --- | --- | ---: | --- | ---: | --- | --- |\n` +
    `${entityRows || '| None | - | - | - | - | 0 | - | - |'}\n\n` +
    `## Route Index Sample${markdownJsonBlock(context.routeIndexSample)}\n\n` +
    `## Errors\n${errorLines.length > 0 ? errorLines.join('\n') : '- None'}\n`;
}

function skippedRoutesPayload(context) {
  return {
    generatedAt: context.generatedAt,
    totalAdvertisedRoutes: context.totalAdvertisedRoutes,
    totalCandidateRoutes: context.totalCandidateRoutes,
    sampledRoutes: context.sampledRoutes,
    skippedRoutes: context.skippedRoutes,
    overrides: context.overrides,
    routes: context.classifications
      .filter((classification) => classification.effectiveAction === 'skip')
      .map((classification) => ({
        routePath: classification.routePath,
        namespace: classification.namespace,
        category: classification.category,
        reason: classification.reason,
        ruleId: classification.ruleId,
        sampleByDefault: classification.sampleByDefault,
        canIncludeByOverride: classification.canIncludeByOverride,
        includedByOverride: classification.includedByOverride,
        excludedByOverride: classification.excludedByOverride,
        effectiveAction: classification.effectiveAction,
        duplicateOf: classification.duplicateOf,
        overrideReason: classification.overrideReason,
      })),
  };
}

// orchestration/decisions.json sits two levels above the discovery out-dir in the standard
// layout (migrations/<project>/data/wp-discovery); --decisions overrides. Missing or
// unparseable files mean "no answers yet", never an error — the ask may not have run.
function loadDecisions(args) {
  const filePath = args.decisions
    || (args.outDir ? path.resolve(args.outDir, '..', '..', 'orchestration', 'decisions.json') : null);
  if (!filePath) return {};
  try {
    return JSON.parse(require('node:fs').readFileSync(filePath, 'utf8'));
  } catch {
    return {};
  }
}

// The batched ask (J1 step 9) records one decision per blocker under the key
// `pluginBlocker:<capability>:<kind>`; a value of `declined` (or `skipped`) marks it declined.
function applyBlockerDecisions(rows, decisions) {
  for (const row of rows) {
    for (const blocker of row.blocked || []) {
      const entry = decisions[`pluginBlocker:${row.capability}:${blocker.kind}`];
      if (entry && ['declined', 'skipped'].includes(entry.value)) blocker.declined = true;
    }
  }
}

function loadTargetKnowledge(domainsDir = null) {
  try {
    const {
      knowledgeRoot,
      knowledgeSummary,
    } = require('../../rp-target-wix/lib/domain-knowledge.js');
    return knowledgeSummary(domainsDir || knowledgeRoot(path.resolve(__dirname, '..', '..', 'rp-target-wix')));
  } catch (error) {
    // A partial install must not break discovery; coverage then reports CMS/native-gap
    // conservatively instead of resolving target refs.
    return { knownRefs: new Set(), capabilityRefs: new Map(), verificationByRef: new Map(), loadError: error.message };
  }
}

// The record-count note a route's sample left (e.g. no X-WP-Total header), if any.
function countNoteOf(sampled) {
  return (sampled.discoveryNotes || []).find((note) => /recordCount|undercount|record count/i.test(note)) || null;
}

// Durable record counts and sample evidence, one key per route of every detected plugin's
// profile entities (null when the route was not sampled). This is what lets coverage be
// reclassified later without a source fetch and without silently dropping counts.
function buildEntityStats(detection, sampledByRoute) {
  const stats = {};
  for (const plugin of detection?.detected || []) {
    for (const entity of plugin.entities || []) {
      if (!entity.route) continue;
      const sampled = sampledByRoute.get(entity.route);
      stats[entity.route] = sampled ? {
        recordCount: sampled.recordCount ?? null,
        inUse: sampled.inUse ?? null,
        sampleRecordCount: sampled.sampleRecordCount ?? (sampled.sampleRecords || []).length,
        countNote: countNoteOf(sampled),
      } : null;
    }
  }
  return Object.fromEntries(Object.entries(stats).sort(([a], [b]) => a.localeCompare(b)));
}

function entityStatsMap(entityStats) {
  return new Map(Object.entries(entityStats || {}).filter(([, value]) => value && typeof value === 'object'));
}

// Malformed `pluginCmsFallback:*` keys with no row to carry them (capability not detected, or
// not eligible): listed once at the top level so the mapping review can show them.
function cmsFallbackDecisionIssues(rows, decisions) {
  const carried = new Set(rows.filter((row) => row.recognized && row.cmsFallback).map((row) => row.capability));
  const issues = [];
  for (const key of cmsFallbackDecisionKeys(decisions)) {
    const shape = cmsFallbackDecisionShape(key, decisions[key]);
    if (shape.ok || carried.has(cmsFallbackCapabilityOfKey(key))) continue;
    issues.push({ key, problem: shape.problem });
  }
  return issues;
}

function cmsFallbackDigestFor({ inventory, profiles, targetKnowledge, knowledgeDir, hints, decisions }) {
  return cmsFallbackInputsDigest(cmsFallbackResolutionInputs({
    detectedPlugins: detectedPluginsOf(inventory),
    profiles,
    targetKnowledge,
    registers: {
      cmsFallbackEligible: loadCmsFallbackEligible(knowledgeDir).capabilities || [],
      requiresDevelopment: loadRequiresDevelopment(knowledgeDir).capabilities || [],
      noMigrationNeeded: hints,
    },
    decisions,
  }));
}

// The project root a raw discovery directory belongs to (`<project>/data/wp-discovery`), or null.
function projectDirOfOutDir(outDir) {
  if (!outDir) return null;
  const resolved = path.resolve(outDir);
  return path.basename(resolved) === 'wp-discovery' && path.basename(path.dirname(resolved)) === 'data'
    ? path.resolve(resolved, '../..') : null;
}

// Tier C input: the project's canonical source-schema.json entities, where project-local bridge
// read plans (PLUGIN-READ-PLANS.md) record an unprofiled plugin's own tables. Missing or
// unreadable means none; this never reads the source site.
function readProjectSourceEntities(projectDir) {
  if (!projectDir) return [];
  try {
    const schema = JSON.parse(require('node:fs').readFileSync(path.join(projectDir, 'source-schema.json'), 'utf8'));
    return schema && schema.entities ? schema.entities : [];
  } catch {
    return [];
  }
}

// Plugin detection, Tier-B derivation, and coverage classification, run after sampling so
// record payloads are available for core-embedded plugins that add no REST route.
// Routes this run really read: a sample came back (a known shape or a record count) and no
// read request failed. An inspected route that answered 401/403/500 is not read, so it never
// marks a coreCovered plugin (WooCommerce) as covered. OPTIONS failures do not count: the
// schema probe is optional and the read itself may still succeed.
function readRoutePaths(entities) {
  return (entities || []).filter((entity) => {
    if (!entity) return false;
    const readFailed = (entity.requestErrors || []).some((error) => error && error.request !== 'OPTIONS');
    const sampled = (entity.responseShape !== undefined && entity.responseShape !== 'unknown')
      || (entity.recordCount !== undefined && entity.recordCount !== null);
    return sampled && !readFailed;
  }).map((entity) => entity.routePath || entity.route).filter(Boolean);
}

function buildPluginCoverage({ args, inventory, classifications, entities, allCandidates = [], dataPresenceObservations = [] }) {
  const knowledgeDir = args.pluginsRoot ? path.resolve(args.pluginsRoot) : pluginsRoot(path.resolve(__dirname, '..'));
  const profiles = loadProfiles(knowledgeDir);
  const sampledRecordProperties = collectRecordProperties(entities);

  const detection = detectPlugins({
    profiles,
    restIndex: inventory.restIndex,
    pluginList: inventory.detection.pluginListAvailable ? inventory.pluginList : null,
    types: inventory.types,
    taxonomies: inventory.taxonomies,
    htmlSources: inventory.htmlSources || [],
    sampledRecordProperties,
    fingerprintAliases: loadFingerprintAliases(knowledgeDir),
  });

  const sampledByRoute = new Map(entities.map((entity) => [entity.routePath, entity]));
  const unprofiled = [
    ...collectUnprofiledRoutes({ classifications, detection }),
    ...collectCandidateNamespaces({ classifications, candidates: allCandidates, detection }),
  ];
  const genericEntities = deriveGenericEntities({
    types: inventory.types,
    taxonomies: inventory.taxonomies,
    classifications,
    detection,
    sampledByRoute,
  });

  const targetKnowledge = loadTargetKnowledge(args.targetKnowledgeRoot ? path.resolve(args.targetKnowledgeRoot) : null);
  const hints = loadNoMigrationNeeded(knowledgeDir);
  const cutoverTasks = loadCutoverTasks(knowledgeDir);
  const decisions = loadDecisions(args);
  const bridgeEntities = deriveBridgePlannedEntities({
    sourceEntities: readProjectSourceEntities(projectDirOfOutDir(args.outDir)),
    detection,
    capabilityRefs: targetKnowledge.capabilityRefs,
  });
  const rows = classifyCoverage({
    detection,
    genericEntities,
    bridgeEntities,
    unprofiledRoutes: unprofiled,
    targetKnowledge,
    entityStatsByRoute: sampledByRoute,
    // Only human-signed verdicts may produce a requires-development status.
    requiresDevelopmentEntries: loadRequiresDevelopment(knowledgeDir).capabilities || [],
    // Working notes for profiled capabilities we have not placed yet.
    pendingNotes: loadPendingDecisions(knowledgeDir).capabilities || [],
    // Nothing-to-move verdicts, both tiers: the slug list resolves unprofiled plugins into
    // rows (basis: list), and the signed capabilities[] register resolves profiled
    // capabilities a human decided need no migration (basis: decision).
    noMigrationNeeded: hints,
    // CMS fallback: the signed eligibility register plus this run's merchant answers
    // (`pluginCmsFallback:<capability>`). Neither alone plans a CMS write.
    cmsFallbackEntries: loadCmsFallbackEligible(knowledgeDir).capabilities || [],
    cmsFallbackDecisions: decisions,
    // No-data plugins that need a named cutover task, and the routes this run read
    // (a coreCovered plugin such as WooCommerce is covered only when its namespace was read).
    cutoverTasks,
    readRoutes: readRoutePaths(entities),
    dataPresence: presenceByPlugin({ profiles, detection, observations: dataPresenceObservations }),
  });

  // Read back the batched-ask answers so a declined blocker renders as declined, never as
  // unanswered. A declined blocker stays on the row — skipping is an answer, not silence.
  applyBlockerDecisions(rows, decisions);

  // One row per installed plugin, for customer review and for debugging this run later.
  const dispositionRows = buildDispositionRows({
    detection,
    coverage: rows,
    profiles,
    hints,
    cutoverTasks,
  });

  return {
    detection,
    genericEntities,
    bridgeEntities,
    unprofiled,
    sampledRecordProperties,
    coverage: rows,
    summary: coverageSummary(rows),
    disposition: dispositionRows,
    dispositionSummary: summarizeDispositions(dispositionRows),
    toolkitExcluded: toolkitExcludedCount(detection, hints),
    targetKnowledgeError: targetKnowledge.loadError || null,
    entityStats: buildEntityStats(detection, sampledByRoute),
    cmsFallbackDecisionIssues: cmsFallbackDecisionIssues(rows, decisions),
    cmsFallbackDigest: cmsFallbackDigestFor({ inventory: detection, profiles, targetKnowledge, knowledgeDir, hints, decisions }),
    profiles,
    dataPresenceObservations,
  };
}

// run the dataPresence checks of every detected profile that declares them. REST
// checks use the run's credentials; postmeta/posts checks need wix-wp-plugin-v2 and come back
// `unknown` without it (never "empty").
async function collectDataPresence({ args, headers, namespaces, profiles, detection }) {
  const fetchRest = async (routePath, query) => fetchJson(args.baseUrl, routePath, {
    headers, method: 'GET', query, timeoutMs: args.timeoutMs, progress,
    progressContext: { step: 'plugin-data-presence', entity: routePath },
  });
  let bridgeQuery = null;
  if (namespaces.includes('wix-wp-plugin/v2')) {
    const bridge = createBridgeTransport({ baseUrl: args.baseUrl, headers, timeoutMs: args.timeoutMs, progress });
    bridgeQuery = async (structureRequest, cursor, limit) => queryStructure({
      httpClient: bridge.httpClient, siteBaseUrl: args.baseUrl, auth: bridge.auth, structureRequest, cursor, limit,
    });
  }
  return collectPresenceObservations({ profiles, detection, fetchRest, bridgeQuery });
}

/**
 * Rebuild plugin coverage with no source network calls, after a CMS-fallback answer or a change
 * to what could resolve an in-scope capability. Inputs are durable artifacts only: the stored
 * detection (plugin-inventory.json), the prior coverage's entityStats and review evidence (copied
 * forward unchanged), and the CURRENT profiles, registers, target knowledge and decisions.
 * Returns { outcome: 'needs-full-discovery', reason } instead of a payload whenever the stored
 * evidence cannot carry every count; it never writes a row with counts silently dropped.
 */
function reclassifyPluginCoverage({ prior, inventory, decisions = {}, pluginsRoot: pluginsDir = null, targetKnowledgeRoot = null, sourceEntities = [], redirectCaptureCoverage = null, generatedAt = new Date().toISOString() }) {
  if (!prior || !prior.entityStats || typeof prior.entityStats !== 'object' || Array.isArray(prior.entityStats)) {
    return { outcome: 'needs-full-discovery', reason: 'prior plugin-coverage.json has no entityStats' };
  }
  if (!inventory || !Array.isArray(inventory.detected)) {
    return { outcome: 'needs-full-discovery', reason: 'plugin-inventory.json is missing or has no detected[]' };
  }
  const knowledgeDir = pluginsDir || pluginsRoot(path.resolve(__dirname, '..'));
  const profiles = loadProfiles(knowledgeDir);
  const profileBySlug = new Map(profiles.map((profile) => [profile.plugin, profile]));
  const targetKnowledge = loadTargetKnowledge(targetKnowledgeRoot);
  const hints = loadNoMigrationNeeded(knowledgeDir);
  const cutoverTasks = loadCutoverTasks(knowledgeDir);
  const aliases = loadFingerprintAliases(knowledgeDir).aliases || {};
  const cmsFallbackEntries = loadCmsFallbackEligible(knowledgeDir).capabilities || [];
  const requiresDevelopmentEntries = loadRequiresDevelopment(knowledgeDir).capabilities || [];

  const sampledRoutes = new Set(Object.keys(prior.entityStats).filter((route) => prior.entityStats[route]));
  const detected = inventory.detected.map((stored) => {
    const profile = profileBySlug.get(stored.plugin);
    // A detected plugin whose profile left the repo keeps its capabilities but declares no
    // entities, so nothing can resolve for it and no CMS plan survives.
    if (!profile) return { ...stored, entities: [] };
    return {
      ...stored,
      displayName: profile.displayName || stored.displayName,
      capabilities: [...(profile.capabilities || [])],
      profileVersion: profile.profileVersion || stored.profileVersion,
      // The current profile decides where the data lives (a stored run may predate the field).
      dataLocation: profile.dataLocation || null,
      entities: (profile.entities || []).map((entity) => {
        const described = describeProfiledEntity(entity, {
          routes: sampledRoutes,
          rawRoutes: {},
          properties: new Set(),
          apiBelowMinVersion: stored.apiBelowMinVersion === true,
        });
        const previous = (stored.entities || []).find((candidate) => candidate.entity === entity.entity
          && candidate.channel === entity.channel && (candidate.route || null) === (entity.route || null));
        return previous ? { ...described, channelStatus: previous.channelStatus } : described;
      }),
    };
  });

  const inputs = cmsFallbackResolutionInputs({
    detectedPlugins: detectedPluginsOf(inventory),
    profiles,
    targetKnowledge,
    registers: { cmsFallbackEligible: cmsFallbackEntries, requiresDevelopment: requiresDevelopmentEntries, noMigrationNeeded: hints },
    decisions,
  });
  for (const capability of Object.keys(inputs.capabilities)) {
    for (const plugin of detected.filter((candidate) => (candidate.capabilities || []).includes(capability))) {
      for (const entity of plugin.entities) {
        if (entity.route && !Object.prototype.hasOwnProperty.call(prior.entityStats, entity.route)) {
          return { outcome: 'needs-full-discovery', reason: `route ${entity.route} of ${plugin.plugin} (${capability}) has no stored entityStats` };
        }
      }
    }
  }

  const detection = {
    pluginListAvailable: inventory.pluginListAvailable === true,
    detected,
    installedButUnprofiled: inventory.installedButUnprofiled || [],
    // Re-resolved through the CURRENT alias map, so an alias added after discovery
    // (ctxfeed -> CTX Feed) folds a stored fingerprint into its installed plugin.
    fingerprinted: (inventory.fingerprinted || []).map((print) => refreshFingerprint(print, aliases)),
  };
  // Tier C is re-derived from the CURRENT source schema: a project-local read plan written after
  // discovery is exactly the change this rebuild exists to pick up.
  const bridgeEntities = deriveBridgePlannedEntities({ sourceEntities, detection, capabilityRefs: targetKnowledge.capabilityRefs });
  const rows = classifyCoverage({
    detection,
    genericEntities: prior.genericEntities || [],
    bridgeEntities,
    unprofiledRoutes: prior.unprofiledRoutes || [],
    targetKnowledge,
    entityStatsByRoute: entityStatsMap(prior.entityStats),
    requiresDevelopmentEntries,
    pendingNotes: loadPendingDecisions(knowledgeDir).capabilities || [],
    noMigrationNeeded: hints,
    cmsFallbackEntries,
    cmsFallbackDecisions: decisions,
    cutoverTasks,
    readRoutes: readRoutePaths(Array.isArray(sourceEntities) ? sourceEntities : Object.values(sourceEntities || {})),
    // Stored observations only: a rebuild never re-reads the site.
    dataPresence: presenceByPlugin({ profiles, detection, observations: prior.dataPresence || [] }),
  });
  applyBlockerDecisions(rows, decisions);
  const annotated = redirectCaptureCoverage
    ? redirectSources.annotateRedirectCoverageRows(rows, redirectCaptureCoverage, { providers: redirectSources.providersFromProfiles(profiles) })
    : rows;
  rows.splice(0, rows.length, ...annotated);
  const disposition = buildDispositionRows({ detection, coverage: rows, profiles, hints, cutoverTasks });

  return {
    outcome: 'reclassified',
    payload: {
      ...prior,
      generatedAt,
      summary: coverageSummary(rows),
      perPluginSummary: summarizeDispositions(disposition),
      capabilities: rows,
      perPlugin: disposition,
      statusInvariant: { ...(prior.statusInvariant || {}), toolkitExcluded: toolkitExcludedCount(detection, hints) },
      bridgeEntities,
      cmsFallbackDecisionIssues: cmsFallbackDecisionIssues(rows, decisions),
      inputs: { ...(prior.inputs || {}), cmsFallbackDigest: cmsFallbackInputsDigest(inputs), reclassifiedFrom: prior.generatedAt || null },
    },
  };
}

function refreshFingerprint(print, aliases) {
  const alias = print && aliases[print.token];
  if (!alias) return print;
  return { ...print, displayName: alias.displayName, slug: alias.slug, aliasMatched: true };
}

function toolkitExcludedCount(detection, hints) {
  const toolkit = new Set((hints?.toolkitPlugins || []).filter((entry) => entry && entry.decidedBy && entry.decidedOn).map((entry) => entry.slug));
  return (detection?.installedButUnprofiled || []).filter((entry) => registerKeys(entry.plugin).some((key) => toolkit.has(key))).length;
}

async function runReclassifyCoverage(args) {
  if (!args.outDir) throw new Error('--reclassify-coverage requires --out-dir (the raw discovery directory)');
  const outDir = path.resolve(args.outDir);
  const readOrNull = async (filePath) => {
    try {
      return JSON.parse(await fs.readFile(filePath, 'utf8'));
    } catch {
      return null;
    }
  };
  const result = reclassifyPluginCoverage({
    prior: await readOrNull(path.join(outDir, 'plugin-coverage.json')),
    inventory: await readOrNull(path.join(outDir, 'plugin-inventory.json')),
    decisions: loadDecisions({ ...args, outDir }),
    pluginsRoot: args.pluginsRoot ? path.resolve(args.pluginsRoot) : null,
    targetKnowledgeRoot: args.targetKnowledgeRoot ? path.resolve(args.targetKnowledgeRoot) : null,
    sourceEntities: readProjectSourceEntities(projectDirOfOutDir(outDir)),
    redirectCaptureCoverage: readRecordedRedirectCoverage(projectDirOfOutDir(outDir)),
  });
  if (result.outcome !== 'reclassified') return result;
  const text = `${JSON.stringify(result.payload, null, 2)}\n`;
  const targets = [path.join(outDir, 'plugin-coverage.json')];
  const canonical = args.canonicalCoverage
    ? path.resolve(args.canonicalCoverage)
    : path.resolve(outDir, '..', '..', 'discovery', 'plugin-coverage.json');
  let canonicalDirExists = Boolean(args.canonicalCoverage);
  if (!canonicalDirExists) {
    try {
      canonicalDirExists = (await fs.stat(path.dirname(canonical))).isDirectory();
    } catch {
      canonicalDirExists = false;
    }
  }
  if (canonicalDirExists) targets.push(canonical);
  for (const target of targets) {
    await fs.mkdir(path.dirname(target), { recursive: true });
    const tempPath = `${target}.${process.pid}.tmp`;
    await fs.writeFile(tempPath, text, 'utf8');
    await fs.rename(tempPath, target);
  }
  return { outcome: 'reclassified', written: targets, cmsFallbackDigest: result.payload.inputs.cmsFallbackDigest };
}

// plugin-rest-child support: a parent-scoped sub-resource (e.g.
// /wc/v3/orders/{parentId}/notes) cannot be listed on its own, so it is checked by
// substituting real ids from the already-sampled parent collection. Pure and exported so the
// id-picking and route-building logic is fixture-testable with no live site.

// Candidate parent ids for a plugin-rest-child entity, drawn from its parent collection's
// already-sampled records and capped at a small representative count — this is a check, not
// an attempt at full coverage.
function pickChildSampleParentIds(parentEntity, limit) {
  const records = parentEntity?.sampleRecords || [];
  return records
    .map((record) => (record && typeof record === 'object' ? record.id : undefined))
    .filter((id) => id !== undefined && id !== null)
    .slice(0, limit);
}

function buildChildRoutePath(template, parentId) {
  return template.replace(CHILD_ROUTE_PLACEHOLDER, encodeURIComponent(String(parentId)));
}

function describeChildSample(sample) {
  const label = `${sample.plugin} · ${sample.entity}`;
  if (sample.parentsChecked === 0) {
    return `${label}: no sampled ${sample.parentRoute} records were available to check ${sample.route} against.`;
  }
  return `${label}: checked ${sample.parentsChecked} sampled ${sample.parentRoute} record(s)' `
    + `${sample.route} — ${sample.parentsWithRecords} of ${sample.parentsChecked} returned matching records `
    + `(${sample.sampleRecords.length} sample record(s) total). Representative check only, not a full count.`;
}

// Live per-parent sampling for every detected plugin-rest-child entity. Runs after the main
// sampling loop so `sampledByRoute` already holds real parent records to draw ids from.
async function sampleChildEntities({ baseUrl, headers, timeoutMs, sampleLimit, progress, detection, sampledByRoute }) {
  const results = [];
  const parentSampleLimit = Math.min(3, sampleLimit);

  for (const detected of detection?.detected || []) {
    for (const entity of detected.entities) {
      if (entity.channel !== 'plugin-rest-child' || entity.channelStatus !== 'available') continue;

      const parentEntity = sampledByRoute.get(entity.parentRoute);
      const parentIds = pickChildSampleParentIds(parentEntity, parentSampleLimit);
      if (parentIds.length === 0) {
        results.push({
          plugin: detected.plugin,
          entity: entity.entity,
          parentRoute: entity.parentRoute,
          route: entity.route,
          parentsChecked: 0,
          parentsWithRecords: 0,
          sampleRecords: [],
        });
        continue;
      }

      const sampleRecords = [];
      let parentsWithRecords = 0;
      for (const parentId of parentIds) {
        const childRoute = buildChildRoutePath(entity.route, parentId);
        // eslint-disable-next-line no-await-in-loop -- each parent's sub-resource must be
        // fetched under the shared rate limiter, one request at a time, like inspectEntity.
        const response = await fetchJson(baseUrl, childRoute, {
          headers,
          method: 'GET',
          timeoutMs,
          progress,
          progressContext: { step: 'sample-child-entity', entity: entity.entity },
        });
        if (response.ok && Array.isArray(response.json) && response.json.length > 0) {
          parentsWithRecords += 1;
          sampleRecords.push(...response.json.slice(0, Math.max(0, sampleLimit - sampleRecords.length)));
        }
      }

      results.push({
        plugin: detected.plugin,
        entity: entity.entity,
        parentRoute: entity.parentRoute,
        route: entity.route,
        parentsChecked: parentIds.length,
        parentsWithRecords,
        sampleRecords,
      });
    }
  }

  return results;
}

// Pure -- no I/O -- so it can be unit-tested directly without mocking httpClient or the
// profile loader. Decides what query a structure-bridge-plugin entity's sample should
// run, or why none can be run at all. `table` is spread in LAST when a fulfillment declares its
// own sampleStructureRequest so a stray `table` key inside that declaration (which
// plugin-knowledge.js's validator already rejects at authoring time) can never silently
// redirect the query at a different table than this entity's fulfillment names -- this is
// the second, independent line of defense for that same invariant.
function resolveSampleStructureRequest(fulfillment, table, schema) {
  if (fulfillment.sampleStructureRequest) {
    return { sampleRequest: { ...fulfillment.sampleStructureRequest, table }, skippedReason: null };
  }
  if (schema.columns.length === 0) {
    return {
      sampleRequest: null,
      skippedReason: 'discover-structure returned zero columns for this table -- cannot build a default sample. Unexpected for a real table; the fulfillment metadata or the live schema needs review.',
    };
  }
  if (schema.columns.length > 50) {
    return {
      sampleRequest: null,
      skippedReason: `this table has ${schema.columns.length} columns, more than wix-wp-plugin-v2's own 50-column select limit -- a default "every column" sample would silently drop ${schema.columns.length - 50} of them. This entity's fulfillment must declare an explicit sampleStructureRequest choosing which columns matter; there is no safe default for a table this wide.`,
    };
  }
  // On a key/value table the guarded value column(s) are unreachable until the
  // request pins the matching key column, so a default "select every column" sample is a
  // request the plugin REFUSES with a 400 -- and a 400 here reads as "this table is broken"
  // rather than "this sample was built wrong". Drop the guarded columns and sample the key
  // names instead, which is the discovery tier and exactly what a mapper needs first: the
  // real key list for whatever plugin this site runs.
  const guarded = guardedValueColumns(schema.eavPair);
  const sampleable = schema.columns.filter((column) => !guarded.includes(column.name));
  if (sampleable.length === 0) {
    return {
      sampleRequest: null,
      skippedReason: 'every column on this table is a guarded key/value column -- nothing can be sampled without naming specific keys first.',
    };
  }
  if (guarded.length > 0) {
    return {
      sampleRequest: {
        table,
        select: sampleable.map((column) => ({ column: column.name })),
        orderBy: [{ column: schema.eavPair.keyColumn, direction: 'asc' }],
      },
      skippedReason: null,
      eavNote: `key/value table: ${guarded.join(', ')} omitted from this sample because wix-wp-plugin-v2 will not return a value until the request pins its key column (${schema.eavPair.keyColumn}). Read those with a scoped request naming the keys this sample reveals.`,
    };
  }
  const primaryColumn = schema.columns.find((column) => column.key === 'primary') || schema.columns[0];
  return {
    sampleRequest: {
      table,
      select: schema.columns.map((column) => ({ column: column.name })),
      orderBy: [{ column: primaryColumn.name, direction: 'asc' }],
    },
    skippedReason: null,
  };
}

// Every entity whose blocked[].fulfillment.kind is structure-bridge-plugin has
// no REST route at all -- the ONLY way to capture it is wix-wp-plugin-v2's own routes. Runs
// after plugin detection/coverage so it can skip a plugin that isn't actually active, and
// after the REST sampling passes so it never competes with them for the shared rate limit
// slot ordering. Never throws: a per-entity failure (plugin not installed, table renamed,
// wix-wp-plugin-v2 not installed at all) is recorded as a request error on that entity's own
// file, exactly like an auth-gated REST route already is -- it must not abort discovery.
/**
 * Accumulates the per-read counts the client derives and files them with
 * rp-telemetry once, at the end of the bridge pass.
 *
 * Wired HERE, in the real caller, rather than left as a hook nobody calls: `onSourceRead` with
 * no production caller is telemetry that exists in the code and not in the data, which is
 * indistinguishable from a run that read nothing. Failure to record is swallowed on purpose --
 * telemetry must never be able to abort a discovery run, and this whole function is already
 * built on "a per-entity failure is recorded, not thrown".
 */
function makeSourceReadCollector(outDir) {
  const totals = { discovery_queries: 0, scoped_queries: 0, tier_refusals: 0, redacted_values: 0 };
  const discoveryTables = new Set();
  const scopedTables = new Set();
  return {
    onSourceRead(counts) {
      for (const key of Object.keys(totals)) totals[key] += counts[key] || 0;
      for (const table of counts.discovery_tables || []) discoveryTables.add(table);
      for (const table of counts.scoped_tables || []) scopedTables.add(table);
    },
    countTierRefusal() {
      totals.tier_refusals += 1;
    },
    flush(progress) {
      const payload = { ...totals };
      if (discoveryTables.size > 0) payload.discovery_tables = [...discoveryTables].sort();
      if (scopedTables.size > 0) payload.scoped_tables = [...scopedTables].sort();
      if (Object.values(totals).every((n) => n === 0)) return;
      try {
        // outDir is <project>/discovery/wordpress; the project root is two levels up, the same
        // derivation the decisions.json lookup above already uses.
        const projectDir = path.resolve(outDir, '..', '..');
        // eslint-disable-next-line global-require
        require('../../rp-telemetry/lib/telemetry-recorder.js').sourceRead(projectDir, payload);
      } catch (error) {
        progress?.progress(`source-read telemetry not recorded: ${error.message}`, {
          phase: 'discovery',
          step: 'structure-bridge-plugin',
        });
      }
    },
  };
}

// Source redirect maps, captured completely during discovery (no agent step). A detected
// profile entity with a `redirectSource` block names its provider's bare tables; each is read
// in full through wix-wp-plugin-v2 and handed to wp-redirect-sources' extractor, then persisted
// by the shared recorder into state/url-preservation/. Never throws: no bridge, a missing table
// or a failed page becomes `unreadable` coverage, which can never pass for an empty map; a
// failure to save the rules becomes `not_recorded`.
// Returns null when no redirect provider is detected.
async function readBridgeTableFully({ transport, table }) {
  let schema;
  try {
    schema = await discoverStructure({ httpClient: transport.httpClient, siteBaseUrl: transport.siteBaseUrl, auth: transport.auth, table });
  } catch (error) {
    return { found: false, read: { error: `discover-structure failed: ${error.message}` } };
  }
  const { sampleRequest, skippedReason } = resolveSampleStructureRequest({}, table, schema);
  if (!sampleRequest || guardedValueColumns(schema.eavPair).length > 0) {
    return { found: true, read: { error: skippedReason || 'key/value table: a full read needs named keys' } };
  }
  const rows = [];
  const seenCursors = new Set();
  let cursor;
  for (;;) {
    let page;
    try {
      page = await queryStructure({ httpClient: transport.httpClient, siteBaseUrl: transport.siteBaseUrl, auth: transport.auth,
        structureRequest: sampleRequest, cursor, limit: MAX_PAGE_LIMIT, onSourceRead: transport.onSourceRead });
    } catch (error) {
      return { found: true, read: { error: `query-structure failed after ${rows.length} row(s): ${error.message}`, complete: false, rows } };
    }
    rows.push(...page.rows);
    const next = page.pagingMetadata.cursors.next;
    if (!page.pagingMetadata.hasNext) break;
    if (!next || seenCursors.has(next)) return { found: true, read: { error: 'paging did not advance', complete: false, rows } };
    seenCursors.add(next);
    cursor = next;
  }
  return { found: true, read: { complete: true, rows } };
}

async function captureSourceRedirects({ baseUrl, headers, namespaces, timeoutMs, progress, detection, outDir,
  pluginsDir = null, createTransport = createBridgeTransport, extractedAt = new Date().toISOString(),
  recordCapture = (args) => require('../../../lib/source-redirect-capture.js').recordSourceRedirectCapture(args) }) {
  const knowledgeDir = pluginsDir || pluginsRoot(path.resolve(__dirname, '..'));
  const providers = redirectSources.providersFromProfiles(loadProfiles(knowledgeDir));
  const detectedBySlug = new Map((detection?.detected || []).map((entry) => [entry.plugin, entry]));
  const active = Object.entries(providers).filter(([, def]) => detectedBySlug.has(def.profile));
  if (active.length === 0) return null;
  const input = {
    extractedAt,
    plugins: active.map(([, def]) => ({ slug: def.profile, active: detectedBySlug.get(def.profile).active !== false })),
    physicalTables: null,
    tables: {},
  };
  const tables = Array.from(new Set(active.flatMap(([, def]) => def.tables)));
  if (tables.length > 0 && (namespaces || []).includes('wix-wp-plugin/v2')) {
    const sourceReads = makeSourceReadCollector(outDir);
    const transport = createTransport({ baseUrl, headers, timeoutMs, progress, onSourceRead: sourceReads.onSourceRead });
    input.physicalTables = [];
    for (const table of tables) {
      progress?.progress(`Reading redirect table ${table}`, { phase: 'discovery', step: 'source-redirects', entity: table });
      const { found, read } = await readBridgeTableFully({ transport, table });
      if (found) input.physicalTables.push(table);
      input.tables[table] = read;
    }
    sourceReads.flush(progress);
  }
  const result = redirectSources.extractRedirectSources(input, { providers: Object.fromEntries(active) });
  const projectDir = projectDirOfOutDir(outDir);
  let recorded = null;
  if (projectDir) {
    try {
      recorded = await recordCapture({ projectDir, result });
    } catch (error) {
      // The rules were read but not saved (or their targets not planned): a bad state file or a
      // held lock. Discovery goes on; the coverage row says so and asks the Import Operator.
      const reason = `rules read but not recorded: ${error.message}`;
      result.coverage = { ...result.coverage,
        providers: result.coverage.providers.map((p) => ({ ...p, status: 'not_recorded', capturedStatus: p.status, reason })) };
      recorded = { error: error.message };
      progress?.progress('Source redirect maps not recorded', { phase: 'discovery', step: 'source-redirects', details: { error: error.message } });
    }
  }
  progress?.progress('Source redirect maps captured', {
    phase: 'discovery', step: 'source-redirects', count: result.rules.length, unit: 'rules',
    details: Object.fromEntries(result.coverage.providers.map((p) => [p.provider, p.status])),
  });
  return { coverage: result.coverage, rules: result.rules.length, recorded };
}

// Annotate plugin coverage with a redirect capture and rebuild the views derived from rows.
function withRedirectCapture(plugins, captureCoverage, { pluginsDir = null } = {}) {
  if (!plugins || !captureCoverage) return plugins;
  const knowledgeDir = pluginsDir || pluginsRoot(path.resolve(__dirname, '..'));
  const coverage = redirectSources.annotateRedirectCoverageRows(plugins.coverage, captureCoverage,
    { providers: redirectSources.providersFromProfiles(loadProfiles(knowledgeDir)) });
  const disposition = buildDispositionRows({ detection: plugins.detection, coverage, profiles: loadProfiles(knowledgeDir), hints: loadNoMigrationNeeded(knowledgeDir) });
  return { ...plugins, coverage, summary: coverageSummary(coverage), disposition, dispositionSummary: summarizeDispositions(disposition) };
}

// The latest recorded capture for a project, or null.
function readRecordedRedirectCoverage(projectDir) {
  if (!projectDir) return null;
  try {
    return JSON.parse(require('node:fs').readFileSync(path.join(projectDir, 'state', 'url-preservation', 'source-redirect-coverage.json'), 'utf8'));
  } catch {
    return null;
  }
}

// The one wix-wp-plugin-v2 bridge transport: discovery samples and the pre-checkpoint full
// read (plugin-full-read.js) use the same credential selection and HTTP client.
function createBridgeTransport({ baseUrl, headers, timeoutMs, progress, onSourceRead }) {
  const httpClient = {
    async get(url, { params } = {}) {
      const response = await fetchJson(baseUrl, '/wix-wp-plugin/v2/structure', {
        headers, method: 'GET', query: params, timeoutMs, progress,
        progressContext: { step: 'structure-bridge-discover', entity: params?.table },
      });
      if (!response.ok) {
        const error = new Error(`HTTP ${response.status} ${response.statusText}`);
        error.status = response.status;
        throw error;
      }
      return { data: response.json };
    },
    async post(url, { body } = {}) {
      const response = await fetchJson(baseUrl, '/wix-wp-plugin/v2/query/admin-key', {
        headers, method: 'POST', body, timeoutMs, progress,
        progressContext: { step: 'structure-bridge-query' },
      });
      if (!response.ok) {
        const error = new Error(`HTTP ${response.status} ${response.statusText}`);
        error.status = response.status;
        throw error;
      }
      return { data: response.json };
    },
  };
  return { httpClient, siteBaseUrl: baseUrl, auth: undefined, structureMethod: 'GET', queryRoute: '/wix-wp-plugin/v2/query/admin-key', onSourceRead };
}

async function sampleStructureBridgeEntities({ baseUrl, headers, namespaces, sampleLimit, timeoutMs, progress, detection, outDir }) {
  if (!namespaces.includes('wix-wp-plugin/v2')) {
    progress?.progress('wix-wp-plugin-v2 not detected on this site; skipping structure-bridge-plugin entities', {
      phase: 'discovery',
      step: 'structure-bridge-plugin',
    });
    return [];
  }

  const bridge = createBridgeTransport({ baseUrl, headers, timeoutMs, progress });
  const { httpClient, auth: bridgeAuth, structureMethod: bridgeStructureMethod, queryRoute: bridgeQueryRoute } = bridge;

  const detectedSlugs = new Set((detection?.detected || []).map((entry) => entry.plugin));
  const profiles = loadProfiles(pluginsRoot(path.resolve(__dirname, '..')));
  const results = [];

  const sourceReads = makeSourceReadCollector(outDir);

  for (const profile of profiles) {
    if (!detectedSlugs.has(profile.plugin)) continue;
    for (const entity of profile.entities || []) {
      const blockedEntries = [
        ...(entity.blocked || []),
        ...((entity.pitfalls || []).flatMap((pitfall) => pitfall.blocked || [])),
      ];
      for (const blockedEntry of blockedEntries) {
        const fulfillment = blockedEntry.fulfillment;
        if (!fulfillment || fulfillment.kind !== 'structure-bridge-plugin' || fulfillment.handlerId !== 'wix-wp-plugin-v2') continue;

        const { table } = fulfillment;
        const fileName = `${slugify('wix-wp-plugin/v2')}--${slugify(table)}.md`;
        const notes = [
          `Captured via wix-wp-plugin-v2's discover-structure (schema) and query-structure (sample rows), not the WordPress REST API -- this entity has no REST route at all (channel: ${entity.channel}).`,
          `Source plugin: ${profile.plugin}.`,
        ];
        const requestErrors = [];
        let schema = null;
        let sampleRecords = [];
        let recordCount = null;

        progress?.progress(`Inspecting structure-bridge-plugin table ${table}`, {
          phase: 'discovery',
          step: 'structure-bridge-plugin',
          entity: table,
        });

        try {
          schema = await discoverStructure({ httpClient, siteBaseUrl: baseUrl, auth: bridgeAuth, table });
        } catch (error) {
          const forbiddenCode = bridgeForbiddenCode(error.status, namespaces);
          requestErrors.push({
            request: bridgeStructureMethod, routePath: `/wix-wp-plugin/v2/structure?table=${table}`, status: error.status || 0, statusText: error.message,
            ...(forbiddenCode ? { code: forbiddenCode } : {}),
          });
          notes.push(forbiddenCode
            ? `discover-structure failed: ${error.message}. ${forbiddenCode}: this site has no WooCommerce, and a wix-wp-plugin-v2 copy older than 0.5.0 admits only manage_woocommerce. Ask the site owner to upload the current wix-wp-plugin-v2 zip (0.5.0+, which also admits an administrator), then run discovery again. If the plugin is already 0.5.0+, the Application Password user is not an administrator.`
            : `discover-structure failed: ${error.message}. wix-wp-plugin-v2 may not be fully installed/authorized on this site, or ${table} may not exist -- this entity's fulfillment metadata may need review.`);
          results.push({
            entityName: entity.entity, namespace: 'wix-wp-plugin/v2', routePath: `/structure?table=${table}`,
            classification: null, methods: ['GET'], responseShape: 'object', recordCount: null, inUse: null,
            sampleRecords: [], discoveryNotes: notes, relationships: [], requestErrors, schema: null, collectionArgs: [], fileName,
          });
          continue;
        }

        // The authoritative `table` is spread LAST so it always wins even if a declared
        // sampleStructureRequest carried its own `table` key -- plugin-knowledge.js's
        // validator already rejects that key at authoring time, but this ordering is the
        // second, independent line of defense: a profile can never make this code query a
        // different table than the one this entity's filename/notes say it describes.
        const { sampleRequest, skippedReason: sampleSkippedReason, eavNote } = resolveSampleStructureRequest(fulfillment, table, schema);
        // Surfaced to the mapper, not swallowed: "this table's values need a second, scoped
        // read" is exactly the kind of thing that turns into a silent data gap if the sample
        // merely looks a bit thin.
        if (eavNote) notes.push(eavNote);
        if (fulfillment.sampleStructureRequest) {
          notes.push('This sample uses the fulfillment\'s own declared query, not a raw "select every column" dump -- see this entity\'s blocked[].resolution for why a raw sample is not safe here.');
        }

        let queried = false;
        if (sampleSkippedReason) {
          notes.push(sampleSkippedReason);
        } else {
          try {
            const response = await queryStructure({
              httpClient, siteBaseUrl: baseUrl, auth: bridgeAuth, structureRequest: sampleRequest, limit: sampleLimit,
              onSourceRead: sourceReads.onSourceRead,
            });
            sampleRecords = response.rows;
            recordCount = response.pagingMetadata.total;
            queried = true;
            // A redaction is a finding about the SOURCE SITE -- it means a credential is
            // stored somewhere nobody expected -- so it belongs in the discovery record
            // rather than being noticed later as a value that looks oddly literal.
            if (response.redactionMetadata.redactedValueCount > 0) {
              notes.push(`${response.redactionMetadata.redactedValueCount} value(s) in ${response.redactionMetadata.redactedColumns.join(', ')} matched a credential shape and were returned as [REDACTED:secret-shaped]. This is a source-site finding, not a sampling error: something on this site keeps a secret in this column.`);
            }
            if (response.eavAccess) {
              notes.push(`Key/value table, served in the ${response.eavAccess.tier} tier: key column ${response.eavAccess.keyColumn}, guarded value column(s) ${guardedValueColumns(response.eavAccess).join(', ')}.`);
            }
          } catch (error) {
            requestErrors.push({ request: 'POST', routePath: bridgeQueryRoute, status: error.status || 0, statusText: error.message });
            notes.push(`query-structure failed: ${error.message}. Schema above is still real; no sample rows could be captured this run.`);
            // Count a tier refusal only when the REQUEST actually violates the tier rule.
            // A 400 here can equally mean a nonexistent column (an outdated profile naming a
            // column the site dropped), a malformed request, or a bad cursor -- the plugin
            // uses one error for all of them on purpose, so that a refusal never discloses
            // which column holds values. Deciding from the request instead keeps
            // `tier_refusals` meaning "something tried to reach values it had not named".
            if (error.status === 400 && violatesTierRule(sampleRequest, schema.eavPair)) {
              sourceReads.countTierRefusal();
            }
          }
        }

        // Persisted as a machine-readable sidecar (not just described in this file's own
        // "Collection Args" prose) so rp-import-codegen can consume the EXACT request
        // verbatim instead of re-deriving its own default at codegen time, which could
        // drift from this one if the table's live schema changes in between (this is a normal reader,
        // not a blocked-dependency snapshot). This filename is
        // computed unconditionally (even when nothing gets written this run) so writeOutputs
        // can clean up a STALE sidecar from a prior run when this run could not produce a
        // fresh one -- publishing only happens when the query actually succeeded (`queried`),
        // never merely because a request was built; a failed or skipped run must not leave a
        // sidecar codegen would then trust as current.
        const structureRequestArtifactFileName = `${slugify('wix-wp-plugin/v2')}--${slugify(table)}.structure-request.json`;
        const structureRequestArtifact = queried
          ? { fileName: structureRequestArtifactFileName, content: sampleRequest }
          : null;

        // A key/value table needs a SECOND artifact, because the request above deliberately
        // omits the guarded value columns -- a reader running it verbatim extracts key names
        // and no values, which is the opposite of what this capability is for. The read plan
        // is the executable second step: every distinct key, batched at the plugin's 50-key
        // cap, each batch a complete runnable request.
        const eavReadPlanFileName = `${slugify('wix-wp-plugin/v2')}--${slugify(table)}.eav-read-plan.json`;
        let eavReadPlan = null;
        // Deliberately NOT gated on `queried`. The ordinary sample is skipped for a table wider
        // than the 50-column select cap (and fails for other reasons too), but key discovery
        // selects exactly ONE column, so nothing that stopped the sample applies to it. Tying
        // the two together meant a wide key/value table -- which a busy `postmeta` absolutely
        // can be -- silently lost its only path to values.
        if (schema.eavPair) {
          try {
            const pairs = [
              { keyColumn: schema.eavPair.keyColumn, valueColumn: schema.eavPair.valueColumn },
              ...(schema.eavPair.additionalPairs || []),
            ];
            // One key sweep PER PAIR: each value column answers only to its own key column, so
            // one pair's key list cannot unlock another pair's values.
            const keysByPair = {};
            const countsByPair = {};
            let anyTruncated = false;
            let totalPages = 0;
            for (const pair of pairs) {
              // eslint-disable-next-line no-await-in-loop
              const discovered = await discoverEavKeys({
                httpClient, siteBaseUrl: baseUrl, auth: bridgeAuth, table, keyColumn: pair.keyColumn,
                onSourceRead: sourceReads.onSourceRead,
              });
              keysByPair[pair.keyColumn] = discovered.keys;
              countsByPair[pair.keyColumn] = discovered.keyCounts;
              anyTruncated = anyTruncated || discovered.truncated;
              totalPages += discovered.pages;
            }
            const content = buildEavReadPlan({
              table, eavPair: schema.eavPair, schemaColumns: schema.columns, keysByPair, countsByPair, truncated: anyTruncated,
            });
            eavReadPlan = { fileName: eavReadPlanFileName, content };
            notes.push(
              `Key/value table: ${content.distinctKeyCount} distinct key(s) across ${pairs.length} pair(s), discovered over ${totalPages} page(s). `
              + `Values are extracted by ${content.expectedBatches} scoped batch(es)`
              + (content.expectedScopedRequests === null ? '' : ` (~${content.expectedScopedRequests} paginated request(s))`)
              + ` carrying ${content.identityColumns.join(', ') || 'no'} identity column(s) -- `
              + `see ${eavReadPlanFileName}, which rp-import-codegen runs verbatim.`
              + (anyTruncated ? ' *** KEY LIST TRUNCATED: this extraction cannot be treated as whole-table. ***' : '')
              + (content.droppedIdentityColumns.length === 0 ? ''
                : ` *** WIDER THAN THE 50-COLUMN SELECT CAP: ${content.droppedIdentityColumns.length} column(s) are absent from every extracted row (${content.droppedIdentityColumns.join(', ')}). `
                  + 'Every row will be INCOMPLETE per record until the mapping narrows this table and the plan is regenerated with an explicit column choice. ***')
              + (content.orderByIsUnique ? '' : ` *** No primary or NOT-NULL unique column: batches order by ${content.orderColumn}, which can tie, so paging is best-effort. ***`),
            );
          } catch (error) {
            // Never fatal: the schema above is still real, and a discovery run must not abort
            // because one table's key sweep failed.
            notes.push(`Key discovery for the scoped-value read failed: ${error.message}. Values for this table cannot be extracted until this succeeds.`);
            requestErrors.push({ request: 'POST', routePath: bridgeQueryRoute, status: error.status || 0, statusText: error.message });
          }
        }

        results.push({
          entityName: entity.entity,
          sourceEntityRef: `plugin.${profile.plugin}.${entity.entity}`,
          sourceMeta: { plugin: profile.plugin, candidateTargetRefs: entity.candidateTargetRefs || [] },
          namespace: 'wix-wp-plugin/v2',
          routePath: `/query/admin-key (table=${table})`,
          classification: null,
          methods: ['POST'],
          responseShape: 'object',
          recordCount,
          inUse: sampleRecords.length > 0,
          sampleRecords,
          discoveryNotes: notes,
          relationships: [],
          requestErrors,
          schema,
          // Not literally REST "collection args" (no such concept here) -- reusing this
          // section to display the actual structureRequest this sample used (or an empty
          // array when none was attempted), so a human reader can see (e.g.) that
          // gift-card-activity's sample is an aggregate query, not a raw dump, without
          // having to go read the plugin profile's JSON. The machine-readable copy
          // consumers should actually parse is structureRequestArtifact below.
          collectionArgs: sampleRequest ? [sampleRequest] : [],
          fileName,
          structureRequestArtifact,
          structureRequestArtifactFileName,
          eavReadPlan,
          eavReadPlanFileName,
        });
      }
    }
  }

  // One telemetry write for the whole bridge pass, after every entity, so a resumed or
  // partially-failed run still files what it actually read.
  sourceReads.flush(progress);

  return results;
}

// Everything this run advertised or attempted but did not capture as an entity
// (401/403, failed capture, a data route skipped by rule, an unsupported route surface). Always
// written, `rows: []` when there is nothing to report; rp-discovery refuses to promote without it.
function sourceCoveragePayload(context) {
  return buildCoverageFile({
    platform: 'wordpress',
    adapter: 'wordpress',
    generatedAt: context.generatedAt,
    rows: buildWordPressCoverageRows({
      entities: context.entities,
      classifications: context.classifications,
      rawDir: ADAPTER_RAW_DIRS.wordpress,
    }),
  });
}

function pluginCoveragePayload(context) {
  const plugins = context.plugins;
  return {
    generatedAt: context.generatedAt,
    baseUrl: context.baseUrl,
    authenticated: context.authMode !== 'none',
    bridgeAuthMode: context.bridgeAuthMode,
    pluginListAvailable: plugins.detection.pluginListAvailable,
    notes: [
      ...(context.pluginNotes || []),
      ...(plugins.targetKnowledgeError
        ? [`Wix target knowledge could not be loaded (${plugins.targetKnowledgeError}); target refs were not resolved.`]
        : []),
      ...(plugins.childSamples || []).map(describeChildSample),
    ],
    childSamples: plugins.childSamples || [],
    summary: plugins.summary,
    perPluginSummary: plugins.dispositionSummary,
    // The status invariant: N installed plugins produce N per-plugin rows, recognized or not.
    // Recorded here so the check is inspectable in the artifact, not just in tests.
    statusInvariant: {
      detectedRecognized: plugins.detection.detected.length,
      installedButUnrecognized: plugins.detection.installedButUnprofiled.length,
      genericEntities: plugins.genericEntities.length,
      unprofiledNamespaces: plugins.unprofiled.length,
      pluginListAvailable: plugins.detection.pluginListAvailable,
      // This skill's own plugins (no-migration-needed.json toolkitPlugins[]): installed, not listed.
      toolkitExcluded: plugins.toolkitExcluded || 0,
    },
    capabilities: plugins.coverage,
    // One artifact, two views: the capability rows above are the contract;
    // this per-plugin projection answers the question a merchant actually asks.
    perPlugin: plugins.disposition,
    genericEntities: plugins.genericEntities,
    bridgeEntities: plugins.bridgeEntities || [],
    unprofiledRoutes: plugins.unprofiled,
    // Durable per-route counts and sample evidence, so --reclassify-coverage never refetches.
    entityStats: plugins.entityStats || {},
    // raw dataPresence observations, so --reclassify-coverage reuses them.
    dataPresence: plugins.dataPresenceObservations || [],
    // Malformed pluginCmsFallback:* answers with no capability row to carry them.
    cmsFallbackDecisionIssues: plugins.cmsFallbackDecisionIssues || [],
    // The CMS-fallback resolution-input digest. rp-mapper copies it into mapping-plan.json and
    // rp-setup-discovery into setup-plan.json; the router refuses any artifact that disagrees.
    inputs: { cmsFallbackDigest: plugins.cmsFallbackDigest || null },
  };
}

async function ensureDir(dirPath) {
  await fs.mkdir(dirPath, { recursive: true });
}

async function writeOutputs(outDir, context, progress) {
  await ensureDir(outDir);
  const planProjectDir = context.projectDir ? path.resolve(context.projectDir)
    : (path.basename(path.resolve(outDir)) === 'wp-discovery' && path.basename(path.dirname(path.resolve(outDir))) === 'data' ? path.resolve(outDir, '../..') : null);
  if (planProjectDir) {
    const { sha } = require('../lib/plugin-entity-read-plan.js');
    for (const entity of context.entities) {
      if (entity.sourceEntityRef && entity.namespace === 'wix-wp-plugin/v2') {
        entity.fileName = `wix-wp-plugin-v2--${sha(entity.sourceEntityRef)}.md`;
        entity.rawFile = entity.fileName;
      }
    }
  }
  await fs.writeFile(path.join(outDir, 'README.md'), renderIndexFile(context), 'utf8');
  progress?.progress('Wrote WordPress discovery index', { phase: 'discovery', step: 'write-artifact', artifact: path.join(outDir, 'README.md') });
  await fs.writeFile(
    path.join(outDir, 'skipped-routes.json'),
    `${JSON.stringify(skippedRoutesPayload(context), null, 2)}\n`,
    'utf8',
  );
  progress?.progress('Wrote WordPress skipped-route index', {
    phase: 'discovery',
    step: 'write-artifact',
    artifact: path.join(outDir, 'skipped-routes.json'),
  });
  await fs.writeFile(
    path.join(outDir, 'source-coverage.json'),
    `${JSON.stringify(sourceCoveragePayload(context), null, 2)}\n`,
    'utf8',
  );
  progress?.progress('Wrote WordPress source coverage', {
    phase: 'discovery',
    step: 'write-artifact',
    artifact: path.join(outDir, 'source-coverage.json'),
  });

  if (context.plugins) {
    await fs.writeFile(
      path.join(outDir, 'plugin-inventory.json'),
      `${JSON.stringify(inventoryPayload({
        generatedAt: context.generatedAt,
        baseUrl: context.baseUrl,
        authenticated: context.authMode !== 'none',
        detection: context.plugins.detection,
        notes: context.pluginNotes || [],
        unprofiled: context.plugins.unprofiled,
        profileCount: context.pluginProfileCount,
      }), null, 2)}\n`,
      'utf8',
    );
    progress?.progress('Wrote WordPress plugin inventory', {
      phase: 'discovery',
      step: 'write-artifact',
      artifact: path.join(outDir, 'plugin-inventory.json'),
    });
    await fs.writeFile(
      path.join(outDir, 'plugin-coverage.json'),
      `${JSON.stringify(pluginCoveragePayload(context), null, 2)}\n`,
      'utf8',
    );
    progress?.progress('Wrote WordPress plugin coverage', {
      phase: 'discovery',
      step: 'write-artifact',
      artifact: path.join(outDir, 'plugin-coverage.json'),
    });
    // Deliberately no plugin-disposition.md: the per-plugin view lives inside
    // plugin-coverage.json (one coverage artifact, no second document).
  }

  for (const entity of context.entities) {
    if (entity.sourceEntityRef && entity.namespace === 'wix-wp-plugin/v2'
      && planProjectDir) {
      const plans = require('../lib/plugin-entity-read-plan.js');
      const projectDir = planProjectDir;
      try {
        const artifact = entity.eavReadPlan || entity.structureRequestArtifact;
        if (!artifact || !entity.schema) throw Object.assign(new Error('discovery could not produce a supported read recipe; inspect its storage evidence'), { code: 'plan-unavailable' });
        const base = `data/wp-discovery/entity-read-plans/${plans.sha(entity.sourceEntityRef)}`;
        const evidence = plans.writeJson(projectDir, `${base}/schema.json`, entity.schema);
        const request = plans.writeJson(projectDir, `${base}/${entity.eavReadPlan ? 'eav-read-plan' : 'structure-request'}.json`, artifact.content);
        plans.writePluginEntityReadPlan({ projectDir, siteBaseUrl: context.baseUrl, sourceEntity: entity,
          plan: { schemaVersion: 1, sourceEntityRef: entity.sourceEntityRef, sourceIdentity: plans.sourceIdentity(context.baseUrl), evidenceRefs: [evidence],
            components: [{ id: 'root', table: entity.schema.table, variant: entity.eavReadPlan ? 'eav' : 'single-request', schema: entity.schema,
              readArtifacts: { [entity.eavReadPlan ? 'eavReadPlan' : 'structureRequest']: request }, scope: { where: artifact.content.where || [] } }],
            assembly: { kind: 'identity', identityColumn: artifact.content.groupBy?.[0] || entity.schema.columns.find((column) => column.key === 'primary' || (column.key === 'unique' && column.nullable === false))?.name || '' },
            cost: { expectedRequests: artifact.content.expectedScopedRequests ?? null } } });
      } catch (error) {
        plans.invalidatePluginEntityReadPlan({ projectDir, sourceEntityRef: entity.sourceEntityRef });
        entity.sourceMeta = { ...entity.sourceMeta, extractionPlan: null };
        plans.recordPluginExtractionDeferral({ projectDir, sourceEntityRef: entity.sourceEntityRef, code: error.code || 'plan-invalid', reason: error.message, producer: 'rp-source-wordpress/scripts/wp-discovery.js' });
        entity.discoveryNotes.push(`Entity plan deferred: ${error.message}`);
      }
    }
    const metadata = entity.sourceEntityRef ? `\n## Source binding\n\n${JSON.stringify({ sourceEntityRef: entity.sourceEntityRef, bridgeDesignated: true, sourceMeta: entity.sourceMeta }, null, 2)}\n` : '';
    await fs.writeFile(path.join(outDir, entity.fileName), renderEntityFile(entity) + metadata, 'utf8');
    progress?.progress(`Wrote WordPress discovery artifact for ${entity.entityName}`, {
      phase: 'discovery',
      step: 'write-artifact',
      entity: entity.routePath,
      artifact: path.join(outDir, entity.fileName),
    });
    // structure-bridge-plugin entities additionally persist the exact
    // structureRequest their sample used, as machine-readable JSON alongside the
    // human-readable markdown -- see sampleStructureBridgeEntities()'s own comment on why.
    if (entity.structureRequestArtifact) {
      const artifactPath = path.join(outDir, entity.structureRequestArtifact.fileName);
      await fs.writeFile(artifactPath, `${JSON.stringify(entity.structureRequestArtifact.content, null, 2)}\n`, 'utf8');
      progress?.progress(`Wrote structure-request artifact for ${entity.entityName}`, {
        phase: 'discovery',
        step: 'write-artifact',
        entity: entity.routePath,
        artifact: artifactPath,
      });
    } else if (entity.structureRequestArtifactFileName) {
      // This run could not produce a fresh sidecar (query skipped or failed) -- delete any
      // sidecar a PRIOR run left behind for this exact table, so rp-import-codegen can never
      // consume a stale one and mistake it for this run's (non-existent) authoritative query.
      const stalePath = path.join(outDir, entity.structureRequestArtifactFileName);
      try {
        await fs.unlink(stalePath);
        progress?.progress(`Removed stale structure-request artifact for ${entity.entityName}`, {
          phase: 'discovery',
          step: 'write-artifact',
          entity: entity.routePath,
          artifact: stalePath,
        });
      } catch (error) {
        if (error.code !== 'ENOENT') throw error;
      }
    }

    // Same publish-or-clean rule for the scoped-value read plan: a stale plan is worse than a
    // missing one, because codegen would run it and produce a confidently wrong extraction
    // against a key list the site no longer has.
    if (entity.eavReadPlan) {
      const planPath = path.join(outDir, entity.eavReadPlan.fileName);
      await fs.writeFile(planPath, `${JSON.stringify(entity.eavReadPlan.content, null, 2)}\n`, 'utf8');
      progress?.progress(`Wrote EAV read plan for ${entity.entityName}`, {
        phase: 'discovery',
        step: 'write-artifact',
        entity: entity.routePath,
        artifact: planPath,
      });
    } else if (entity.eavReadPlanFileName) {
      try {
        await fs.unlink(path.join(outDir, entity.eavReadPlanFileName));
      } catch (error) {
        if (error.code !== 'ENOENT') throw error;
      }
    }
  }
}

// The store's currency, time zone, language and country, written next to the coverage file for
// promotion. Reads are bounded (two GETs, or one when the store-settings capture already ran)
// and a denied or failed read is an explicit unknown in the file, never a discovery failure.
// Only the standard `<project>/data/wp-discovery` layout can be promoted, so any other --out-dir
// is skipped with a note.
async function captureSourceRegional({ outDir, baseUrl, headers, timeoutMs, entities, generatedAt, progress: log = null, fetch = fetchJson }) {
  const resolved = path.resolve(outDir);
  const projectDir = path.resolve(resolved, '..', '..');
  if (resolved !== path.join(projectDir, ADAPTER_RAW_DIRS.wordpress)) {
    log?.progress('Skipped source regional capture: --out-dir is not <project>/data/wp-discovery', { phase: 'discovery', step: 'regional-settings' });
    return null;
  }
  let capturedGeneral = null;
  try {
    capturedGeneral = JSON.parse(await fs.readFile(path.join(projectDir, 'data', 'store-settings-general.json'), 'utf8'));
  } catch { /* not captured (or unreadable): read the route instead */ }
  const settings = await regionalCapture.readRegionalSettings({ baseUrl, headers, fetchJson: fetch, timeoutMs, progress: log, capturedGeneral });
  const file = regionalCapture.buildWordPressRegional({
    ...settings,
    ordersEntity: entities.find((entity) => entity.routePath === regionalCapture.ORDERS_ROUTE) || null,
    generatedAt,
  });
  const target = await writeAdapterSourceRegional(projectDir, 'wordpress', file);
  log?.progress('Wrote WordPress source regional file', { phase: 'discovery', step: 'write-artifact', artifact: target });
  return file;
}

async function main() {
  const parsed = parseProgressArgs(process.argv.slice(2));
  progress = createProgressLogger({
    script: 'skills/wix-replatform/resources/rp-source-wordpress/scripts/wp-discovery.js',
    ...parsed.progress,
  });
  progress.start('WordPress discovery started', { phase: 'discovery' });

  const args = await hydrateArgsFromEnvFile(parseArgs(parsed.args));
  if (args.help) {
    printUsage();
    progress.complete('WordPress discovery help shown', { phase: 'discovery', step: 'help' });
    return;
  }

  if (args.reclassifyCoverage) {
    const result = await runReclassifyCoverage(args);
    console.log(JSON.stringify({ ok: result.outcome === 'reclassified', ...result }, null, 2));
    if (result.outcome !== 'reclassified') {
      progress.warn('Plugin coverage reclassification needs a full discovery run', { phase: 'discovery', step: 'reclassify-coverage' });
      process.exitCode = 3;
      return;
    }
    progress.complete('Plugin coverage reclassified without refetching', { phase: 'discovery', step: 'reclassify-coverage' });
    return;
  }

  if (!args.baseUrl || !args.outDir) {
    printUsage();
    progress.error('Missing required WordPress discovery arguments', { phase: 'discovery' });
    throw new Error('Missing required arguments: --base-url and --out-dir are required.');
  }

  configureRateLimit({ rateLimitRpm: args.rateLimitRpm, maxRetries: args.maxRetries });

  const headers = buildHeaders(args);
  const rootResponse = await fetchJson(args.baseUrl, '', {
    headers,
    method: 'GET',
    timeoutMs: args.timeoutMs,
    progress,
    progressContext: { step: 'rest-index' },
  });

  if (!rootResponse.ok || !rootResponse.json) {
    throw new Error(`Failed to fetch WordPress REST index from ${rootResponse.url}: ${rootResponse.status} ${rootResponse.statusText}`);
  }

  const indexJson = rootResponse.json;
  const namespaces = Array.isArray(indexJson?.namespaces) ? indexJson.namespaces : [];
  progress.progress('WordPress namespace enumeration completed', {
    phase: 'discovery',
    step: 'namespace-enumeration',
    count: namespaces.length,
    unit: 'namespaces',
  });
  // Plugin inventory pre-pass: runs before classification so profile-declared routes are
  // already in scope when routes are classified.
  let inventory = null;
  if (args.pluginInventory) {
    try {
      const gathered = await gatherInventory({
        baseUrl: args.baseUrl,
        headers,
        timeoutMs: args.timeoutMs,
        htmlFingerprint: args.htmlFingerprint,
        restIndex: indexJson,
        logger: progress,
      });
      inventory = gathered;
      progress.progress('WordPress plugin inventory completed', {
        phase: 'discovery',
        step: 'plugin-inventory',
        count: gathered.detection.detected.length,
        unit: 'plugins',
        details: {
          pluginListAvailable: gathered.detection.pluginListAvailable,
          installedButUnprofiled: gathered.detection.installedButUnprofiled.length,
        },
      });
    } catch (error) {
      // Never fail discovery over the plugin pre-pass; record and continue.
      progress.progress(`WordPress plugin inventory skipped: ${error.message}`, {
        phase: 'discovery',
        step: 'plugin-inventory',
      });
    }
  }

  const requestOverrides = buildRequestOverrides(pluginsRoot(path.resolve(__dirname, '..')));
  const candidates = deriveEntityCandidates(indexJson, args.includeNamespaces, requestOverrides);
  progress.progress('WordPress endpoint candidates derived', {
    phase: 'discovery',
    step: 'derive-endpoints',
    count: candidates.length,
    unit: 'endpoints',
  });
  // Registered post types/taxonomies are positive evidence for classification, so the
  // inventory pre-pass must run before this point (it fetches /wp/v2/types+taxonomies).
  const classifications = classifyRoutes(candidates, {
    ...summarizeOverrides(args),
    registeredRestBases: inventory ? buildRegisteredRestBases({ types: inventory.types, taxonomies: inventory.taxonomies }) : null,
  });
  // classifyRoutes() loads the checked-in plugin profiles lazily via defaultPluginRules();
  // a failed load degrades to zero plugin rules with no exception, so it must be surfaced
  // here or every plugin-owned route silently falls through to generic/unsupported handling.
  const pluginRulesError = defaultPluginRules().loadError || null;
  if (pluginRulesError) {
    progress.progress(`WordPress plugin route rules could not be loaded (${pluginRulesError}); plugin-owned routes will not be recognized this run.`, {
      phase: 'discovery',
      step: 'classify-endpoints',
    });
  }
  const candidatesByRoute = new Map(candidates.map((candidate) => [candidate.routePath, candidate]));
  const candidatesToInspect = classifications
    .filter((classification) => ['sample', 'metadata'].includes(classification.effectiveAction))
    .map((classification) => ({
      ...candidatesByRoute.get(classification.routePath),
      classification,
    }));
  const skippedByCategory = summarizeSkippedByCategory(classifications);
  const skippedRoutes = classifications.filter((classification) => classification.effectiveAction === 'skip').length;
  progress.progress('WordPress endpoint candidates classified', {
    phase: 'discovery',
    step: 'classify-endpoints',
    count: candidatesToInspect.length,
    total: candidates.length,
    unit: 'endpoints',
    details: {
      sampledRoutes: candidatesToInspect.length,
      skippedRoutes,
      skippedByCategory,
    },
  });
  const entities = [];
  const responseEnvelopes = buildResponseEnvelopes(pluginsRoot(path.resolve(__dirname, '..')));
  const responseFragmentGroups = buildResponseFragmentGroups(pluginsRoot(path.resolve(__dirname, '..')));
  const recordKeyFields = buildRecordKeyFields(pluginsRoot(path.resolve(__dirname, '..')));

  for (let i = 0; i < candidatesToInspect.length; i += 1) {
    const candidate = candidatesToInspect[i];
    progress.progress(`Inspecting WordPress endpoint ${candidate.routePath}`, {
      phase: 'discovery',
      step: 'inspect-endpoint',
      entity: candidate.routePath,
      count: i + 1,
      total: candidatesToInspect.length,
      unit: 'endpoints',
      percent: candidatesToInspect.length ? Math.round(((i + 1) / candidatesToInspect.length) * 100) : 100,
    });
    const details = await inspectEntity(args.baseUrl, headers, candidate, {
      sampleLimit: args.sampleLimit,
      timeoutMs: args.timeoutMs,
      progress,
      responseEnvelope: responseEnvelopes.get(candidate.routePath) || null,
      responseFragmentGroupSize: responseFragmentGroups.get(candidate.routePath) || null,
      recordKeyField: recordKeyFields.get(candidate.routePath) || null,
      requestOverride: candidate.requestOverride,
      defaultQuery: defaultQueryFor(candidate.routePath),
      defaultQueryReason: defaultQueryReasonFor(candidate.routePath),
    });
    entities.push(details);
  }

  // Second detection pass + Tier-B derivation + coverage. This is where plugins that add
  // no REST route (Product Bundles, ACF, Yoast) become visible, because it looks at the
  // record payload keys the sampling loop just collected.
  let plugins = null;
  if (inventory) {
    plugins = buildPluginCoverage({ args, inventory, classifications, entities, allCandidates: candidates });
    if (plugins.profiles.some((profile) => profile.dataPresence
      && plugins.detection.detected.some((detected) => detected.plugin === profile.plugin))) {
      const dataPresenceObservations = await collectDataPresence({
        args, headers, namespaces, profiles: plugins.profiles, detection: plugins.detection,
      });
      plugins = buildPluginCoverage({ args, inventory, classifications, entities, allCandidates: candidates, dataPresenceObservations });
    }
    progress.progress('WordPress plugin coverage classified', {
      phase: 'discovery',
      step: 'plugin-coverage',
      count: plugins.coverage.length,
      unit: 'capabilities',
      details: plugins.summary.byStatus,
    });

    // Third pass: plugin-rest-child entities — a parent-scoped sub-resource
    // (e.g. /wc/v3/orders/{parentId}/notes) cannot be listed on its own, so it is checked by
    // substituting real ids from the parent collection this run already sampled.
    const childSamples = await sampleChildEntities({
      baseUrl: args.baseUrl,
      headers,
      timeoutMs: args.timeoutMs,
      sampleLimit: args.sampleLimit,
      progress,
      detection: plugins.detection,
      sampledByRoute: new Map(entities.map((entity) => [entity.routePath, entity])),
    });
    if (childSamples.length > 0) {
      progress.progress('WordPress per-parent sub-resource sampling completed', {
        phase: 'discovery',
        step: 'sample-child-entity',
        count: childSamples.length,
        unit: 'entities',
      });
    }
    plugins = { ...plugins, childSamples };

    // Fourth pass: structure-bridge-plugin entities -- data with no REST route
    // at all, captured through wix-wp-plugin-v2 instead. Pushed into the same `entities`
    // array so it flows through the existing per-entity markdown writer unchanged; run
    // after detection/coverage so it can skip a plugin that isn't actually active.
    const structureBridgeEntities = await sampleStructureBridgeEntities({
      baseUrl: args.baseUrl,
      headers,
      namespaces,
      sampleLimit: args.sampleLimit,
      timeoutMs: args.timeoutMs,
      progress,
      detection: plugins.detection,
      outDir: args.outDir,
    });
    if (structureBridgeEntities.length > 0) {
      progress.progress('wix-wp-plugin-v2 structure-bridge-plugin capture completed', {
        phase: 'discovery',
        step: 'structure-bridge-plugin',
        count: structureBridgeEntities.length,
        unit: 'entities',
      });
    }
    entities.push(...structureBridgeEntities);

    // Fifth pass: source redirect maps (redirect plugins' own tables), read in full.
    let redirectCapture = null;
    try {
      redirectCapture = await captureSourceRedirects({
      baseUrl: args.baseUrl,
      headers,
      namespaces,
      timeoutMs: args.timeoutMs,
      progress,
      detection: plugins.detection,
      outDir: args.outDir,
      pluginsDir: args.pluginsRoot ? path.resolve(args.pluginsRoot) : null,
      });
    } catch (error) {
      // Never fail discovery over the redirect pass; the redirect row keeps its read blocker.
      progress.progress('Source redirect capture failed', { phase: 'discovery', step: 'source-redirects', details: { error: error.message } });
    }
    if (redirectCapture) plugins = withRedirectCapture(plugins, redirectCapture.coverage, { pluginsDir: args.pluginsRoot ? path.resolve(args.pluginsRoot) : null });
  }

  const context = {
    generatedAt: new Date().toISOString(),
    baseUrl: normalizeBaseUrl(args.baseUrl),
    restRoot: `${normalizeBaseUrl(args.baseUrl)}/wp-json`,
    authMode: summarizeAuthMode(args),
    bridgeAuthMode: summarizeBridgeAuthMode(args),
    authProviders: Object.keys(indexJson?.authentication || {}),
    namespaces,
    totalAdvertisedRoutes: Object.keys(indexJson?.routes || {}).length,
    totalCandidateRoutes: candidates.length,
    sampledRoutes: candidatesToInspect.length,
    skippedRoutes,
    skippedByCategory,
    overrides: summarizeOverrides(args),
    classifications,
    routeIndexSample: Object.keys(indexJson?.routes || {}).slice(0, 50),
    entities,
    plugins,
    pluginNotes: [
      ...(inventory ? inventory.notes : ['Plugin inventory was disabled with --no-plugin-inventory.']),
      ...(pluginRulesError
        ? [`WordPress plugin route rules could not be loaded (${pluginRulesError}); plugin-owned routes were not recognized this run.`]
        : []),
    ],
    pluginProfileCount: inventory ? inventory.profileCount : 0,
  };

  await writeOutputs(args.outDir, context, progress);
  await captureSourceRegional({
    outDir: args.outDir,
    baseUrl: args.baseUrl,
    headers,
    timeoutMs: args.timeoutMs,
    entities,
    generatedAt: context.generatedAt,
    progress,
  });

  console.log(`Wrote ${entities.length + 1} markdown files and skipped-routes.json to ${args.outDir}`);
  progress.complete('WordPress discovery completed', {
    phase: 'discovery',
    artifact: args.outDir,
    count: entities.length,
    unit: 'entities',
  });
}

if (require.main === module) {
  main().catch((error) => {
    console.error(error.stack || error.message);
    if (progress) {
      progress.error(error && error.message ? error.message : 'WordPress discovery failed', { phase: 'discovery' });
    }
    process.exitCode = 1;
  });
}

module.exports = {
  parseArgs,
  deriveEntityCandidates,
  renderIndexFile,
  renderPluginSection,
  skippedRoutesPayload,
  sourceCoveragePayload,
  buildPluginCoverage,
  readRoutePaths,
  pluginCoveragePayload,
  buildEntityStats,
  reclassifyPluginCoverage,
  runReclassifyCoverage,
  loadDecisions,
  loadTargetKnowledge,
  pickChildSampleParentIds,
  buildChildRoutePath,
  describeChildSample,
  sampleChildEntities,
  sampleStructureBridgeEntities,
  resolveSampleStructureRequest,
  captureSourceRedirects,
  readBridgeTableFully,
  withRedirectCapture,
  writeOutputs,
  captureSourceRegional,
  getAtPath,
  inspectEntity,
  resolveRequestBody,
  normalizeResponseRecords,
  findSampledIdsDependencies,
  summarizeAuthMode,
  summarizeBridgeAuthMode,
  bridgeForbiddenCode,
  BRIDGE_PLUGIN_UPDATE_REQUIRED,
  createBridgeTransport,
  hydrateArgsFromEnvFile,
  makeSourceReadCollector,
};