diff --git a/graphile/graphile-presigned-url-plugin/__tests__/storage-module-cache.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/storage-module-cache.test.ts new file mode 100644 index 0000000000..54ed68236d --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/__tests__/storage-module-cache.test.ts @@ -0,0 +1,209 @@ +/** + * Unit tests for resolveStorageConfigFromCodec. + * + * These are pure (no DB) tests covering both the single-tenant path (exact + * physical schema match) and the blueprint-pooling path, where the codec's + * build-time schema belongs to the REPRESENTATIVE tenant but the request's + * configs carry a DIFFERENT tenant's actual (differently-hashed) schema. In + * that case the resolver must fall back to matching on the LOGICAL schema + * name (the tenant hash prefix stripped from both sides). + */ + +import { resolveStorageConfigFromCodec } from '../src/storage-module-cache'; +import type { StorageModuleConfig } from '../src/types'; + +// --- Fixtures ------------------------------------------------------------- + +// Two real-shaped hashed tenant schemas that share the SAME logical suffix +// (`storage-public`). Under pooling, the shared instance is built against the +// representative tenant's schema, while requests route to other tenants. +const REPRESENTATIVE_SCHEMA = 'marketplace-db-tenant1-5e6b13b2-storage-public'; +const TENANT_SCHEMA = 'marketplace-db-tenant2-35a03232-storage-public'; + +function makeConfig( + overrides: Partial & + Pick, +): StorageModuleConfig { + return { + bucketsQualifiedName: `"${overrides.schemaName}"."${overrides.bucketsTableName}"`, + filesQualifiedName: `"${overrides.schemaName}"."${overrides.filesTableName}"`, + scope: 'app', + entityTableId: null, + entityQualifiedName: null, + endpoint: null, + publicUrlPrefix: null, + provider: null, + allowedOrigins: null, + uploadUrlExpirySeconds: 900, + downloadUrlExpirySeconds: 3600, + defaultMaxFileSize: 200 * 1024 * 1024, + maxFilenameLength: 1024, + cacheTtlSeconds: 3600, + hasPathShares: false, + maxBulkFiles: 100, + maxBulkTotalSize: 1073741824, + ...overrides, + }; +} + +function makeCodec(schemaName: string | undefined, name: string | undefined) { + return { name: name as string, extensions: { pg: { schemaName, name } } }; +} + +// --- Tests ---------------------------------------------------------------- + +describe('resolveStorageConfigFromCodec', () => { + describe('single-tenant (exact physical schema match)', () => { + it('matches a files codec by exact schema + files table name', () => { + const config = makeConfig({ + id: 'sm-1', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec(TENANT_SCHEMA, 'app_files'); + + expect(resolveStorageConfigFromCodec(codec, [config])).toBe(config); + }); + + it('matches a buckets codec by exact schema + buckets table name', () => { + const config = makeConfig({ + id: 'sm-1', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec(TENANT_SCHEMA, 'app_buckets'); + + expect(resolveStorageConfigFromCodec(codec, [config])).toBe(config); + }); + + it('matches plain (non-hashed) schemas exactly, unaffected by hash stripping', () => { + const config = makeConfig({ + id: 'sm-1', + schemaName: 'app_public', + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec('app_public', 'app_files'); + + expect(resolveStorageConfigFromCodec(codec, [config])).toBe(config); + }); + }); + + describe('blueprint pooling (logical schema match across tenants)', () => { + it('matches a files codec built for the representative tenant against another tenant config', () => { + // Build-time codec: representative tenant. Request config: a DIFFERENT tenant. + const tenantConfig = makeConfig({ + id: 'sm-tenant2', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec(REPRESENTATIVE_SCHEMA, 'app_files'); + + // Physical schemas differ; logical suffix (`storage-public`) is shared. + expect(REPRESENTATIVE_SCHEMA).not.toEqual(TENANT_SCHEMA); + expect(resolveStorageConfigFromCodec(codec, [tenantConfig])).toBe(tenantConfig); + }); + + it('matches a buckets codec built for the representative tenant against another tenant config', () => { + const tenantConfig = makeConfig({ + id: 'sm-tenant2', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec(REPRESENTATIVE_SCHEMA, 'app_buckets'); + + expect(resolveStorageConfigFromCodec(codec, [tenantConfig])).toBe(tenantConfig); + }); + + it('does NOT match when the logical schema suffix differs', () => { + // A codec whose logical schema is `analytics-public` must not resolve to a + // `storage-public` config even though both are hashed tenant schemas. + const tenantConfig = makeConfig({ + id: 'sm-tenant2', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec('marketplace-db-tenant1-5e6b13b2-analytics-public', 'app_files'); + + expect(resolveStorageConfigFromCodec(codec, [tenantConfig])).toBeNull(); + }); + }); + + describe('exact-physical priority', () => { + it('prefers an exact physical match over a logical-only match', () => { + const exactConfig = makeConfig({ + id: 'sm-exact', + schemaName: REPRESENTATIVE_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const logicalOnlyConfig = makeConfig({ + id: 'sm-logical', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec(REPRESENTATIVE_SCHEMA, 'app_files'); + + // exactConfig listed AFTER the logical-only one to prove priority is by + // match quality, not array order. + const result = resolveStorageConfigFromCodec(codec, [logicalOnlyConfig, exactConfig]); + expect(result).toBe(exactConfig); + }); + }); + + describe('non-matches and guards', () => { + it('returns null when the table name matches nothing', () => { + const config = makeConfig({ + id: 'sm-1', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec(REPRESENTATIVE_SCHEMA, 'some_unrelated_table'); + + expect(resolveStorageConfigFromCodec(codec, [config])).toBeNull(); + }); + + it('returns null when the codec has no schemaName', () => { + const config = makeConfig({ + id: 'sm-1', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec(undefined, 'app_files'); + + expect(resolveStorageConfigFromCodec(codec, [config])).toBeNull(); + }); + + it('returns null when the codec has no resolvable table name', () => { + const config = makeConfig({ + id: 'sm-1', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = { name: undefined as unknown as string, extensions: { pg: { schemaName: REPRESENTATIVE_SCHEMA } } }; + + expect(resolveStorageConfigFromCodec(codec, [config])).toBeNull(); + }); + + it('falls back to codec.name for the table name when extensions.pg.name is absent', () => { + const config = makeConfig({ + id: 'sm-1', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = { name: 'app_files', extensions: { pg: { schemaName: REPRESENTATIVE_SCHEMA } } }; + + expect(resolveStorageConfigFromCodec(codec, [config])).toBe(config); + }); + }); +}); diff --git a/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts b/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts index 6ea403fbb5..5cf993558e 100644 --- a/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts +++ b/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts @@ -354,12 +354,46 @@ export async function loadAllStorageModules( return configs; } +/** + * Strip a tenant hash prefix from a physical schema name, returning the + * logical schema name. + * + * Hashed multi-tenant schemas are named like + * `marketplace-db-tenant1-5e6b13b2-app-public`, where `-5e6b13b2-` is an + * 8-hex-char tenant hash separating the tenant prefix from the logical schema + * suffix (`app-public`). This removes everything up to and including the first + * such hash segment. If no hash segment is present (e.g. a plain schema like + * `app_public` or a control-plane schema like `metaschema_public`), the name + * is returned unchanged. + * + * (Duplicated locally rather than imported to avoid a cross-package dependency + * for a one-line helper.) + */ +function stripSchemaHashPrefix(name: string): string { + const match = /-[0-9a-f]{8}-/.exec(name); + return match ? name.slice(match.index + match[0].length) : name; +} + /** * Resolve the storage module config from a PostGraphile pgCodec. * * Matches the codec's schema + table name against cached storage modules. * Works for both files codecs (@storageFiles) and buckets codecs (@storageBuckets). * + * Matching is two-tier: + * 1. Exact physical schema match — used for single-tenant / non-pooled + * instances where the codec's build-time schema equals the request + * tenant's actual schema. Keeps existing behavior unchanged. + * 2. Logical schema match — strip the tenant hash prefix from BOTH the + * codec's schema and each config's schema before comparing. Under + * blueprint pooling a single shared instance serves every tenant of a + * given schema-shape, so the codec's build-time `schemaName` belongs to + * the representative tenant while `allConfigs` carry the request tenant's + * actual (differently-hashed) schema. Both share the same logical suffix + * (e.g. `app-public`). Safe because `allConfigs` is already filtered to a + * single databaseId, so (tableName, logicalSchema) uniquely identifies a + * module. + * * @param pgCodec - The PostGraphile codec (has extensions.pg.schemaName, name) * @param allConfigs - All storage module configs for this database * @returns The matching StorageModuleConfig or null @@ -373,10 +407,24 @@ export function resolveStorageConfigFromCodec( if (!schemaName || !tableName) return null; - return allConfigs.find((c) => + // Priority 1: exact physical schema match (non-pooled / single-tenant). + const exact = allConfigs.find((c) => (c.filesTableName === tableName && c.schemaName === schemaName) || (c.bucketsTableName === tableName && c.schemaName === schemaName), - ) || null; + ); + if (exact) return exact; + + // Priority 2: logical schema match (blueprint pooling — the codec's + // build-time schema belongs to the representative tenant; strip the tenant + // hash from both sides so the shared logical suffix lines up). + const logicalSchema = stripSchemaHashPrefix(schemaName); + return allConfigs.find((c) => { + const configLogicalSchema = stripSchemaHashPrefix(c.schemaName); + return ( + (c.filesTableName === tableName && configLogicalSchema === logicalSchema) || + (c.bucketsTableName === tableName && configLogicalSchema === logicalSchema) + ); + }) || null; } // --- Bucket metadata cache --- diff --git a/graphile/graphile-search/src/codecs/bm25-codec.ts b/graphile/graphile-search/src/codecs/bm25-codec.ts index 117ca88340..c927a4db5c 100644 --- a/graphile/graphile-search/src/codecs/bm25-codec.ts +++ b/graphile/graphile-search/src/codecs/bm25-codec.ts @@ -36,6 +36,18 @@ export interface Bm25IndexInfo { * * Key: "schemaName.tableName.columnName" * Value: Bm25IndexInfo + * + * NOTE (concurrency + blueprint pooling): this store is process-global and is + * cleared + repopulated per introspection run. Concurrent gathers in one + * process are serialized by the server's build semaphore + * (GRAPHILE_BUILD_CONCURRENCY, default 1), so a single global store is + * tolerated for now; if that guarantee ever moves, key this store by build + * object like meta-schema's per-build cache. The adapter embeds the + * schema-qualified index name from these entries as a bind VALUE at plan + * time; the pooling rewrite seam maps schema identifiers in SQL TEXT only and + * never rewrites values, so shapes carrying bm25 indexes are structurally + * excluded from pooling (see graphql/server pooling-decision.ts) and always + * get per-tenant instances. */ export const bm25IndexStore = new Map(); diff --git a/graphile/graphile-settings/src/plugins/meta-schema/table-meta-builder.ts b/graphile/graphile-settings/src/plugins/meta-schema/table-meta-builder.ts index 0b98fccb26..e71ffb7375 100644 --- a/graphile/graphile-settings/src/plugins/meta-schema/table-meta-builder.ts +++ b/graphile/graphile-settings/src/plugins/meta-schema/table-meta-builder.ts @@ -115,11 +115,28 @@ function buildTableMeta( }; } +/** + * Strip the tenant hash prefix (`-<8hex>-`) from a physical + * schema name, returning the logical name; names without a hash segment are + * returned unchanged. Local duplicate of the server-side helper to avoid a + * cross-package dependency. + */ +function stripSchemaHashPrefix(name: string): string { + const match = /-[0-9a-f]{8}-/.exec(name); + if (!match) return name; + return name.slice(match.index + match[0].length); +} + export function collectTablesMeta(build: MetaBuild): TableMeta[] { const configuredSchemas = getConfiguredSchemas(build); const context = createBuildContext(build); const seenCodecs = new Set(); const tablesMeta: TableMeta[] = []; + // Shared (pooled) instances serve many tenants: reporting the build-time + // PHYSICAL schema name would leak the representative tenant's hashed schema + // identifier to every other tenant via _meta. When schema.constructivePooled + // is set, report the logical (hash-stripped) name instead. + const pooled = !!(build.options as any)?.constructivePooled; for (const resource of Object.values(build.input.pgRegistry.pgResources || {})) { if (!isTableResource(resource)) continue; @@ -137,7 +154,9 @@ export function collectTablesMeta(build: MetaBuild): TableMeta[] { continue; } - tablesMeta.push(buildTableMeta(resource, schemaName, context)); + tablesMeta.push( + buildTableMeta(resource, pooled ? stripSchemaHashPrefix(schemaName) : schemaName, context) + ); } return tablesMeta; diff --git a/graphql/server/src/diagnostics/__tests__/metrics-sampler.test.ts b/graphql/server/src/diagnostics/__tests__/metrics-sampler.test.ts index 18154b493d..79c8cfadc2 100644 --- a/graphql/server/src/diagnostics/__tests__/metrics-sampler.test.ts +++ b/graphql/server/src/diagnostics/__tests__/metrics-sampler.test.ts @@ -46,6 +46,7 @@ describe('collectMetricsSample', () => { expect(typeof sample.counters.disposals).toBe('number'); expect(typeof sample.counters.evictions.lru).toBe('number'); expect(typeof sample.counters.builds).toBe('number'); + expect(typeof sample.counters.poolingAttaches).toBe('number'); expect(typeof sample.counters.buildQueueDepth).toBe('number'); expect(typeof sample.counters.rewritePool.rewrittenQueries).toBe('number'); expect(typeof sample.counters.introspectionFilter.swaps).toBe('number'); diff --git a/graphql/server/src/middleware/api.ts b/graphql/server/src/middleware/api.ts index 005be581fd..cd20402413 100644 --- a/graphql/server/src/middleware/api.ts +++ b/graphql/server/src/middleware/api.ts @@ -14,6 +14,7 @@ import { getPgPool } from 'pg-cache'; import errorPage50x from '../errors/50x'; import errorPage404Message from '../errors/404-message'; import { ApiConfigResult, ApiError, ApiOptions, ApiStructure, AuthSettings, DatabaseSettings, PubkeyChallengeSettings, RlsModule, WebauthnSettings } from '../types'; +import { stripSchemaHashPrefix } from './blueprint'; import './types'; const log = new Logger('api'); @@ -266,6 +267,7 @@ const toApiStructure = (row: ApiRow, opts: ApiOptions, settings: ResolvedModuleS anonRole: row.anon_role || 'anon', roleName: row.role_name || 'authenticated', schema: row.schemas || [], + logicalSchemas: (row.schemas || []).map(stripSchemaHashPrefix), apiModules: [], rlsModule: settings.rlsModule, domains: [], diff --git a/graphql/server/src/middleware/flush.ts b/graphql/server/src/middleware/flush.ts index 653d74631c..70b7bc8a72 100644 --- a/graphql/server/src/middleware/flush.ts +++ b/graphql/server/src/middleware/flush.ts @@ -2,9 +2,11 @@ import { ConstructiveOptions } from '@constructive-io/graphql-types'; import { Logger } from '@pgpmjs/logger'; import { svcCache } from '@pgpmjs/server-utils'; import { NextFunction, Request, Response } from 'express'; -import { graphileCache } from 'graphile-cache'; +import { clearMatchingEntries, graphileCache } from 'graphile-cache'; import { getPgPool } from 'pg-cache'; import './types'; // for Request type +import { isBlueprintPoolingEnabled } from './blueprint'; +import { clearPoolDecisions, clearPoolDecisionsForDatabase } from './pooling-decision'; const log = new Logger('flush'); @@ -17,6 +19,12 @@ export const flush = async ( // TODO: check bearer for a flush / special key graphileCache.delete((req as any).svc_key); svcCache.delete((req as any).svc_key); + // Under pooling the serving instance is stored under a `bp:` key, which the + // svc_key delete above cannot reach — mirror flushService's v1 semantics. + if (isBlueprintPoolingEnabled()) { + clearMatchingEntries(/^bp:/); + clearPoolDecisions(); + } res.status(200).send('OK'); return; } @@ -30,6 +38,23 @@ export const flushService = async ( const pgPool = getPgPool(opts.pg); log.info('flushing db ' + databaseId); + // Blueprint pooling: invalidate ONLY the changed database's decisions and the + // blueprint instance it was attached to. Fleet-wide `bp:` flushes on every + // schema:update turned each tenant PROVISION into a cold restart of every + // pooled instance — and the immediate rebuild raced the evicted instances' + // drain (~GB still live until release), which OOMed the 24h soak. New tenants + // have no memoized decisions, so provisioning is a no-op here (instances are + // shape-generic; a same-shape tenant attaches without any rebuild). + if (isBlueprintPoolingEnabled()) { + const bpKeys = clearPoolDecisionsForDatabase(databaseId); + for (const key of bpKeys) { + graphileCache.delete(key); + } + if (bpKeys.length > 0) { + log.info(`[pooling] flushed ${bpKeys.length} blueprint(s) for db ${databaseId}: ${bpKeys.join(', ')}`); + } + } + const api = new RegExp(`^api:${databaseId}:.*`); const schemata = new RegExp(`^schemata:${databaseId}:.*`); const meta = new RegExp(`^metaschema:api:${databaseId}`); diff --git a/graphql/server/src/middleware/graphile.ts b/graphql/server/src/middleware/graphile.ts index 288b0ea222..fe7ed33b9d 100644 --- a/graphql/server/src/middleware/graphile.ts +++ b/graphql/server/src/middleware/graphile.ts @@ -24,16 +24,23 @@ import { createConstructivePreset, makePgService } from 'graphile-settings'; import { getPgPool } from 'pg-cache'; import { getPgEnvOptions } from 'pg-env'; import './types'; // for Request type +import { isBlueprintPoolingEnabled, stripSchemaHashPrefix } from './blueprint'; +import { resolvePoolDecision } from './pooling-decision'; import { createIntrospectionFilterPool, isIntrospectionFilterEnabled } from './introspection-filter'; +import { createRewritingPool, POOL_SCHEMAS_GUC } from './rewrite-pool'; import { isGraphqlObservabilityEnabled } from '../diagnostics/observability'; import { HandlerCreationError } from '../errors/api-errors'; import { observeGraphileBuild } from './observability/graphile-build-stats'; import type { DatabaseSettings } from '../types'; import { AuthCookiePlugin } from '../plugins/auth-cookie-plugin'; +// Re-exported so flush.ts (and other callers) can invalidate pooling decisions +// through the graphile module surface. +export { clearPoolDecisions } from './pooling-decision'; + const maskErrorLog = new Logger('graphile:maskError'); const SAFE_ERROR_CODES = new Set([ @@ -279,17 +286,23 @@ export function getBuildQueueDepth(): number { export interface GraphileCounters { /** Total PostGraphile schema builds started (past coalescing + the semaphore). */ builds: number; + /** Requests routed to a shared blueprint (pooling) instance. */ + poolingAttaches: number; + /** Builds that were pooled (shared-blueprint) builds. */ + poolingBuilds: number; /** Requests that gave up waiting on a build (GRAPHILE_BUILD_TIMEOUT_MS) and got 503. */ buildWaitTimeouts: number; } /** * Cumulative in-process build counters for the metrics sampler. Mutated where a build - * starts. Zero overhead when the sampler is off — these are integer bumps on paths - * that already do real build work. + * starts and where a [pooling] attach/build happens. Zero overhead when the sampler is + * off — these are integer bumps on paths that already do real build work. */ const graphileCounters: GraphileCounters = { builds: 0, + poolingAttaches: 0, + poolingBuilds: 0, buildWaitTimeouts: 0 }; @@ -315,15 +328,32 @@ const buildPreset = ( anonRole: string, roleName: string, databaseSettings?: DatabaseSettings, + options?: { pooling?: boolean }, ): GraphileConfig.Preset => { + // When pooling, the instance is shared across tenants of the same schema-shape + // and routed per request via the rewriting pool (canonical→tenant schema- + // identifier rewrite; see ./rewrite-pool). + const pooling = options?.pooling === true; + // Pooled instances are built fully qualified against the canonical tenant's + // physical schemas; the rewriting pool swaps canonical→tenant schema + // identifiers per request (see ./rewrite-pool). Dedicated instances use the + // raw pool untouched. // Introspection filter (opt-in via GRAPHILE_INTROSPECTION_FILTER): scope the // instance's catalog introspection to the schemas it serves. Only active when // the flag is on AND we have a concrete served-schema list; otherwise the pool - // selection below is byte-identical to today. + // selection below is byte-identical to today. Pooled instances receive the + // filter through the rewriting pool; dedicated instances get a thin filter-only + // wrapper on the raw pool. const introspectionFilterActive = isIntrospectionFilterEnabled() && schemas.length > 0; - const servicePool = introspectionFilterActive - ? createIntrospectionFilterPool(pool, { servedSchemas: schemas }) - : pool; + const servicePool = pooling + ? createRewritingPool(pool, { + canonicalSchemas: schemas, + logicalName: stripSchemaHashPrefix, + introspectionFilter: introspectionFilterActive ? { servedSchemas: schemas } : false + }) + : introspectionFilterActive + ? createIntrospectionFilterPool(pool, { servedSchemas: schemas }) + : pool; const preset: GraphileConfig.Preset = { extends: [createConstructivePreset(databaseSettings)], plugins: [AuthCookiePlugin], @@ -349,6 +379,13 @@ const buildPreset = ( const req = (requestContext as { expressv4?: { req?: Request } })?.expressv4?.req; const context: Record = {}; + // De-closure the role names: read them from the resolved API on the request + // so a shared (pooled) instance uses the REQUESTING tenant's roles rather than + // whichever tenant happened to build it. Falls back to the build-time params. + const api = req?.api; + const role = api?.roleName ?? roleName; + const anon = api?.anonRole ?? anonRole; + if (req) { if (req.databaseId) { context['jwt.claims.database_id'] = req.databaseId; @@ -368,7 +405,7 @@ const buildPreset = ( if (req.token?.user_id) { const pgSettings: Record = { - role: roleName, + role, 'jwt.claims.token_id': req.token.id, 'jwt.claims.user_id': req.token.user_id, ...context, @@ -399,17 +436,30 @@ const buildPreset = ( pgSettings['request.id'] = req.requestId; } + // Pooled instances: hand the REQUESTING tenant's physical schema list to + // the rewriting pool via a transaction-local GUC; the wrapper maps the + // canonical schemas to these by logical name. Never set when not pooling. + if (pooling && api?.schema?.length) { + pgSettings[POOL_SCHEMAS_GUC] = JSON.stringify(api.schema); + } + return { pgSettings }; } } const anonSettings: Record = { - role: anonRole, + role: anon, ...context, }; if (req?.requestId) { anonSettings['request.id'] = req.requestId; } + // Pooled instances: hand the REQUESTING tenant's physical schema list to + // the rewriting pool via a transaction-local GUC; the wrapper maps the + // canonical schemas to these by logical name. Never set when not pooling. + if (pooling && api?.schema?.length) { + anonSettings[POOL_SCHEMAS_GUC] = JSON.stringify(api.schema); + } return { pgSettings: anonSettings, @@ -418,6 +468,15 @@ const buildPreset = ( }, }; + if (pooling) { + // Shared (pooled) instances must not leak the canonical tenant's hashed + // schema names through tenant-facing metadata (e.g. _meta) — plugins read + // this flag and report logical schema names instead. SQL emission stays + // fully qualified (PostGraphile default). Cast: custom schema option, + // absent from base types. + (preset as any).schema = { constructivePooled: true }; + } + return preset; }; @@ -489,14 +548,40 @@ export const graphile = (opts: ConstructiveOptions): RequestHandler => { log.error(`${label} Missing API info`); return res.status(500).send('Missing API info'); } - const key = req.svc_key; - if (!key) { + const svcKey = req.svc_key; + if (!svcKey) { log.error(`${label} Missing service cache key`); return res.status(500).send('Missing service cache key'); } const { dbname, anonRole, roleName, schema } = api; const schemaLabel = schema?.join(',') || 'unknown'; + // ========================================================================= + // Blueprint Pooling Decision (opt-in via GRAPHILE_BLUEPRINT_POOLING) + // + // When enabled, resolve a shared blueprint key so tenants of the same + // schema-shape attach to ONE instance, routed per request by the rewriting + // pool (the requesting tenant's schema list is set in grafast.context). + // Decisions are memoized per svc_key and cleared on flush. When the flag is + // OFF this block is skipped entirely: `key` stays exactly req.svc_key and no + // extra pool or queries are touched. + // ========================================================================= + const pgConfig = getPgEnvOptions({ ...opts.pg, database: dbname }); + let key = svcKey; + let pooling = false; + let pool: Pool | undefined; + if (isBlueprintPoolingEnabled() && api) { + // Blueprint keying needs the tenant pool to fingerprint the schema shape. + pool = getPgPool(pgConfig); + const decision = await resolvePoolDecision(svcKey, api, pool); + key = decision.key; + pooling = decision.pooling; + if (pooling) { + graphileCounters.poolingAttaches += 1; + log.info(`[pooling] svc=${svcKey} → ${key}`); + } + } + // ========================================================================= // Phase A: Cache Check (fast path) // ========================================================================= @@ -557,10 +642,10 @@ export const graphile = (opts: ConstructiveOptions): RequestHandler => { }); } catch (error) { // A refusal surfacing on the re-coalesce await is bound by the same - // contract as the owner path and the request-time gate below: + // contract as the owner path (763) and the request-time gate (640): // 503 SERVICE_OVERLOADED + Retry-After for every coalesced waiter, never // a generic 500. Other rejections fall through to retry the build here, - // exactly as Phase B does. + // exactly as Phase B does (605). if (error instanceof BuildRefusedError) { log.warn(`${label} ${error.message} key=${key}`); res.setHeader('Retry-After', '15'); @@ -591,17 +676,14 @@ export const graphile = (opts: ConstructiveOptions): RequestHandler => { `${label} Building PostGraphile v5 handler key=${key} db=${dbname} schemas=${schemaLabel} role=${roleName} anon=${anonRole}`, ); - const pgConfig = getPgEnvOptions({ - ...opts.pg, - database: dbname, - }); - // Route through pg-cache so the pool is tracked and can be cleaned up - // properly, preventing leaked connections during database teardown. - const pool = getPgPool(pgConfig); + // properly, preventing leaked connections during database teardown. When the + // pooling decision above already resolved the pool, reuse it (pg-cache memoizes + // regardless); otherwise resolve it here exactly as before (flag-off path). + const activePool = pool ?? getPgPool(pgConfig); // Create promise and store in in-flight map BEFORE try block - const preset = buildPreset(pool, schema || [], anonRole, roleName, api.databaseSettings); + const preset = buildPreset(activePool, schema || [], anonRole, roleName, api.databaseSettings, { pooling }); const creationPromise = (async (): Promise => { // Serialize builds process-wide: each build transiently allocates hundreds of MB, // and only same-key builds are deduped by the single-flight map above. @@ -642,6 +724,9 @@ export const graphile = (opts: ConstructiveOptions): RequestHandler => { // A build genuinely starts here: past single-flight coalescing, past the // build semaphore, and past the built-meanwhile re-check. graphileCounters.builds += 1; + if (pooling) { + graphileCounters.poolingBuilds += 1; + } return await observeGraphileBuild( { cacheKey: key,