diff --git a/CLAUDE.md b/CLAUDE.md index 99c42fb08..fb2df6e8c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,6 +18,26 @@ To build a specific package: yarn workspace @sourcebot/ build ``` +## Backend Workloads + +Use the workload system in `packages/backend` for background work. Define the queue payload and default job behavior in the shared queue registry, implement a `Workload`, and register it with the `JobManager`. + +### Execution locks + +- Key an execution lock by the logical resource being mutated, not by the job ID. Workloads that mutate the same resource must use the exact same lock key. For example, repo indexing, repo cleanup, and repo permission syncing share the per-repo lock. +- An execution lock serializes work but does not deduplicate it. Multiple jobs for one resource may still be queued and will execute one at a time. +- The lock lease is extended automatically while work is running. The workload's `AbortSignal` is aborted if extension fails or the worker shuts down. +- Abortion is cooperative. Call `signal.throwIfAborted()` before side effects and after long-running or external operations so work stops promptly after losing the lock. The signal cannot cancel an operation that has already been submitted. +- `onStarted` runs after the execution lock is acquired and immediately before `process`. `onCompleted` and `onTerminalFailure` are BullMQ event hooks and run after the processor has returned and released the lock. + +### Lifecycle state + +- In `onStarted`, upsert the workload-specific job row as `IN_PROGRESS`. If the parent resource tracks a `latest...JobId`, update that pointer in the same database transaction. +- Completion and terminal-failure hooks must always update their own historical job row by job ID. Do not condition that update on the job still being latest. Every job row should record its actual outcome. +- `onTerminalFailure` only runs after the job exhausts all retry attempts. Intermediate failures are retried without marking the lifecycle row as terminally failed. +- If a completion or failure hook publishes state onto the parent resource, use a conditional `updateMany` keyed by both the resource ID and its `latest...JobId`. This prevents an older hook from overwriting state belonging to a newer job after the execution lock has been released. +- Parent-resource state written inside `process` is already serialized by the execution lock. It does not need a latest-job conditional merely because the resource tracks the latest job ID. + ## File Naming Files should use camelCase starting with a lowercase letter: diff --git a/docs/snippets/schemas/v3/index.schema.mdx b/docs/snippets/schemas/v3/index.schema.mdx index a9603abdc..1cff30599 100644 --- a/docs/snippets/schemas/v3/index.schema.mdx +++ b/docs/snippets/schemas/v3/index.schema.mdx @@ -32,7 +32,8 @@ "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -52,7 +53,8 @@ "maxRepoGarbageCollectionJobConcurrency": { "type": "number", "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", @@ -216,7 +218,8 @@ "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -236,7 +239,8 @@ "maxRepoGarbageCollectionJobConcurrency": { "type": "number", "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", diff --git a/packages/backend/package.json b/packages/backend/package.json index 951ef1f33..50c8cf9d6 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -22,6 +22,9 @@ "vitest": "^4.1.4" }, "dependencies": { + "@bull-board/api": "6.11.2", + "@bull-board/express": "6.11.2", + "@bull-board/ui": "6.11.2", "@coderabbitai/bitbucket": "^1.1.3", "@gitbeaker/rest": "^40.5.1", "@octokit/app": "^16.1.1", @@ -35,7 +38,7 @@ "@types/express": "^5.0.0", "argparse": "^2.0.1", "azure-devops-node-api": "^15.1.1", - "bullmq": "^5.34.10", + "bullmq": "^5.81.3", "chokidar": "^4.0.3", "cross-fetch": "^4.0.0", "dotenv": "^16.4.5", @@ -46,7 +49,7 @@ "gitea-js": "^1.22.0", "glob": "^11.1.0", "http-status-codes": "^2.3.0", - "ioredis": "^5.4.2", + "ioredis": "^5.11.1", "lowdb": "^7.0.1", "micromatch": "^4.0.8", "p-limit": "^7.2.0", diff --git a/packages/backend/src/api.ts b/packages/backend/src/api.ts index 96aec050c..43ce13a8e 100644 --- a/packages/backend/src/api.ts +++ b/packages/backend/src/api.ts @@ -1,19 +1,19 @@ +import { createBullBoard } from '@bull-board/api'; +import { BullMQAdapter } from '@bull-board/api/bullMQAdapter.js'; +import { ExpressAdapter } from '@bull-board/express'; +import { Octokit } from '@octokit/rest'; +import * as Sentry from "@sentry/node"; import { PrismaClient, RepoIndexingJobType } from '@sourcebot/db'; -import * as Sentry from '@sentry/node'; -import { hasEntitlement } from './entitlements.js'; -import { createLogger, doesIdpSupportPermissionSyncing, env } from '@sourcebot/shared'; +import { createLogger, env, JOB_PRIORITIES } from '@sourcebot/shared'; import express, { NextFunction, Request, Response } from 'express'; import 'express-async-errors'; import * as http from "http"; -import { ConnectionManager } from './connectionManager.js'; -import { AccountPermissionSyncer } from './ee/accountPermissionSyncer.js'; +import z from 'zod'; +import { SINGLE_TENANT_ORG_ID } from './constants.js'; +import { isGitHubRateLimitError, isNotFound } from './errors.js'; import { PromClient } from './promClient.js'; -import { RepoIndexManager } from './repoIndexManager.js'; import { createGitHubRepoRecord } from './repoCompileUtils.js'; -import { isGitHubRateLimitError, isNotFound } from './errors.js'; -import { Octokit } from '@octokit/rest'; -import { SINGLE_TENANT_ORG_ID } from './constants.js'; -import z from 'zod'; +import type { JobManager } from './types.js'; const logger = createLogger('api'); @@ -26,14 +26,20 @@ export class Api { constructor( promClient: PromClient, private prisma: PrismaClient, - private connectionManager: ConnectionManager, - private repoIndexManager: RepoIndexManager, - private accountPermissionSyncer: AccountPermissionSyncer, + private jobManager: JobManager, ) { const app = express(); app.use(express.json()); app.use(express.urlencoded({ extended: true })); + const bullBoardAdapter = new ExpressAdapter(); + bullBoardAdapter.setBasePath('/admin/queues'); + createBullBoard({ + queues: jobManager.getQueues().map(queue => new BullMQAdapter(queue, { readOnlyMode: true })), + serverAdapter: bullBoardAdapter, + }); + app.use('/admin/queues', bullBoardAdapter.getRouter()); + // Prometheus metrics endpoint app.use('/metrics', async (_req: Request, res: Response) => { res.set('Content-Type', promClient.registry.contentType); @@ -41,9 +47,6 @@ export class Api { res.end(metrics); }); - app.post('/api/sync-connection', this.syncConnection.bind(this)); - app.post('/api/index-repo', this.indexRepo.bind(this)); - app.post('/api/trigger-account-permission-sync', this.triggerAccountPermissionSync.bind(this)); app.post(`/api/experimental/add-github-repo`, this.experimental_addGithubRepo.bind(this)); app.use((error: unknown, _req: Request, _res: Response, next: NextFunction) => { @@ -53,97 +56,10 @@ export class Api { this.server = app.listen(PORT, () => { logger.debug(`API server is running on port ${PORT}`); + logger.debug(`Bull Board is available at ${workerApiUrl.origin}/admin/queues`); }); } - private async syncConnection(req: Request, res: Response) { - const schema = z.object({ - connectionId: z.number(), - }).strict(); - - const parsed = schema.safeParse(req.body); - if (!parsed.success) { - res.status(400).json({ error: parsed.error.message }); - return; - } - - const { connectionId } = parsed.data; - const connection = await this.prisma.connection.findUnique({ - where: { - id: connectionId, - } - }); - - if (!connection) { - res.status(404).json({ error: 'Connection not found' }); - return; - } - - const [jobId] = await this.connectionManager.createJobs([connection]); - - res.status(200).json({ jobId }); - } - - private async indexRepo(req: Request, res: Response) { - const schema = z.object({ - repoId: z.number(), - }).strict(); - - const parsed = schema.safeParse(req.body); - if (!parsed.success) { - res.status(400).json({ error: parsed.error.message }); - return; - } - - const { repoId } = parsed.data; - const repo = await this.prisma.repo.findUnique({ - where: { id: repoId }, - }); - - if (!repo) { - res.status(404).json({ error: 'Repo not found' }); - return; - } - - const [jobId] = await this.repoIndexManager.createJobs([repo], RepoIndexingJobType.INDEX); - res.status(200).json({ jobId }); - } - - private async triggerAccountPermissionSync(req: Request, res: Response) { - if (env.PERMISSION_SYNC_ENABLED !== 'true' || !await hasEntitlement('permission-syncing')) { - res.status(403).json({ error: 'Permission syncing is not enabled.' }); - return; - } - - const schema = z.object({ - accountId: z.string(), - }).strict(); - - const parsed = schema.safeParse(req.body); - if (!parsed.success) { - res.status(400).json({ error: parsed.error.message }); - return; - } - - const { accountId } = parsed.data; - const account = await this.prisma.account.findUnique({ - where: { id: accountId }, - }); - - if (!account) { - res.status(404).json({ error: 'Account not found' }); - return; - } - - if (!doesIdpSupportPermissionSyncing(account.providerType)) { - res.status(400).json({ error: `Provider '${account.providerType}' does not support permission syncing.` }); - return; - } - - const jobId = await this.accountPermissionSyncer.schedulePermissionSyncForAccount(account); - res.status(200).json({ jobId }); - } - private async experimental_addGithubRepo(req: Request, res: Response) { const schema = z.object({ owner: z.string(), @@ -196,7 +112,14 @@ export class Api { create: record, }); - const [jobId ] = await this.repoIndexManager.createJobs([repo], RepoIndexingJobType.INDEX); + const jobId = await this.jobManager.trigger( + 'repo-index', + { + repoId: repo.id, + type: RepoIndexingJobType.INDEX, + }, + { priority: JOB_PRIORITIES.INTERACTIVE }, + ); res.status(200).json({ jobId, repoId: repo.id }); } diff --git a/packages/backend/src/attachmentPruneWorkload.test.ts b/packages/backend/src/attachmentPruneWorkload.test.ts new file mode 100644 index 000000000..9fe516918 --- /dev/null +++ b/packages/backend/src/attachmentPruneWorkload.test.ts @@ -0,0 +1,168 @@ +import type { PrismaClient } from "@sourcebot/db"; +import type { StorageBackend } from "@sourcebot/shared"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { createAttachmentPruneWorkload } from "./attachmentPruneWorkload.js"; + +const mocks = { + updateMany: vi.fn(), + findMany: vi.fn(), + deleteMany: vi.fn(), + storageDelete: vi.fn(), +}; + +const db = { + attachment: { + updateMany: mocks.updateMany, + findMany: mocks.findMany, + deleteMany: mocks.deleteMany, + }, +} as unknown as PrismaClient; + +const storage = { + delete: mocks.storageDelete, +} as unknown as StorageBackend; + +const logger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +}; + +const processWorkload = (ttlHours = 24) => + createAttachmentPruneWorkload({ db, storage, ttlHours }).process({ + data: {}, + jobId: "job-1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + logger, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), + }); + +describe("attachmentPruneWorkload", () => { + beforeEach(() => { + vi.restoreAllMocks(); + vi.clearAllMocks(); + mocks.updateMany.mockResolvedValue({ count: 0 }); + mocks.findMany.mockResolvedValue([]); + mocks.deleteMany.mockResolvedValue({ count: 0 }); + mocks.storageDelete.mockResolvedValue(undefined); + }); + + test("declares an hourly scheduled workload", () => { + const workload = createAttachmentPruneWorkload({ + db, + storage, + ttlHours: 24, + }); + + expect(workload.queueSpec.name).toBe("attachment-prune"); + expect(workload.concurrency).toBe(1); + expect(workload.schedule).toEqual({ + interval: "1h", + data: {}, + options: { priority: 10 }, + }); + }); + + test("does not schedule or prune when the TTL is disabled", async () => { + const workload = createAttachmentPruneWorkload({ + db, + storage, + ttlHours: 0, + }); + + expect(workload.schedule).toBeUndefined(); + await expect(processWorkload(0)).resolves.toEqual({ + pendingClaimed: 0, + committedClaimed: 0, + reclaimed: 0, + }); + expect(mocks.updateMany).not.toHaveBeenCalled(); + expect(mocks.findMany).not.toHaveBeenCalled(); + }); + + test("claims expired orphans and reclaims their tombstones", async () => { + vi.spyOn(Date, "now").mockReturnValue( + new Date("2026-08-10T12:00:00.000Z").getTime(), + ); + mocks.updateMany + .mockResolvedValueOnce({ count: 2 }) + .mockResolvedValueOnce({ count: 1 }); + mocks.findMany.mockResolvedValue([ + { id: "attachment-1", storageKey: "key-1" }, + { id: "attachment-2", storageKey: "key-2" }, + ]); + mocks.deleteMany.mockResolvedValue({ count: 2 }); + + await expect(processWorkload()).resolves.toEqual({ + pendingClaimed: 2, + committedClaimed: 1, + reclaimed: 2, + }); + + const cutoff = new Date("2026-08-09T12:00:00.000Z"); + expect(mocks.updateMany).toHaveBeenNthCalledWith(1, { + where: { + status: "PENDING", + createdAt: { lt: cutoff }, + }, + data: { status: "DELETING" }, + }); + expect(mocks.updateMany).toHaveBeenNthCalledWith(2, { + where: { + status: "COMMITTED", + createdAt: { lt: cutoff }, + chats: { none: {} }, + }, + data: { status: "DELETING" }, + }); + expect(mocks.storageDelete).toHaveBeenCalledWith("key-1"); + expect(mocks.storageDelete).toHaveBeenCalledWith("key-2"); + expect(mocks.deleteMany).toHaveBeenCalledWith({ + where: { + id: { in: ["attachment-1", "attachment-2"] }, + status: "DELETING", + }, + }); + }); + + test("leaves tombstones whose bytes could not be deleted", async () => { + mocks.findMany.mockResolvedValue([ + { id: "attachment-1", storageKey: "key-1" }, + { id: "attachment-2", storageKey: "key-2" }, + ]); + mocks.storageDelete.mockImplementation(async (key: string) => { + if (key === "key-2") { + throw new Error("Storage unavailable"); + } + }); + mocks.deleteMany.mockResolvedValue({ count: 1 }); + + await expect(processWorkload()).resolves.toEqual({ + pendingClaimed: 0, + committedClaimed: 0, + reclaimed: 1, + }); + + expect(mocks.deleteMany).toHaveBeenCalledWith({ + where: { + id: { in: ["attachment-1"] }, + status: "DELETING", + }, + }); + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining("attachment-2"), + ); + }); + + test("propagates database failures so BullMQ can retry", async () => { + const error = new Error("Database unavailable"); + mocks.updateMany.mockRejectedValueOnce(error); + + await expect(processWorkload()).rejects.toBe(error); + }); +}); diff --git a/packages/backend/src/attachmentPruneWorkload.ts b/packages/backend/src/attachmentPruneWorkload.ts new file mode 100644 index 000000000..ab600b5ab --- /dev/null +++ b/packages/backend/src/attachmentPruneWorkload.ts @@ -0,0 +1,182 @@ +import { AttachmentStatus, type PrismaClient } from "@sourcebot/db"; +import { + ATTACHMENT_PRUNE_QUEUE, + getStorageBackend, + JOB_PRIORITIES, + type StorageBackend, +} from "@sourcebot/shared"; +import type { Workload } from "./types.js"; + +const BATCH_SIZE = 1_000; +const ONE_HOUR_MS = 60 * 60 * 1000; + +interface Props { + db: PrismaClient; + ttlHours: number; + storage?: StorageBackend; +} + +interface AttachmentPruneResult { + pendingClaimed: number; + committedClaimed: number; + reclaimed: number; +} + +/** + * Reclaims orphaned attachment blobs using the `DELETING` tombstone protocol: + * an orphan is first atomically flipped to `DELETING`, then its bytes are + * deleted, and only then is the row removed. Because the row (the only durable + * handle to the bytes) outlives the byte delete, a failed byte delete is always + * retryable. + * + * Each run condemns two classes of orphan to `DELETING`, then reclaims all + * tombstones: + * + * 1. PENDING (uploaded-but-never-linked): produced when a user selects a file + * in the chat box but never sends the message. + * 2. COMMITTED with zero links: normally a committed blob is reclaimed inline + * by the chat-delete sweep in the web app, but this is the backstop for an + * interrupted sweep. + * + * @note Byte deletion goes through the shared `StorageBackend`, so the web app + * and this worker share one on-disk layout. + */ +export const createAttachmentPruneWorkload = ({ + db, + ttlHours, + storage = getStorageBackend(), +}: Props): Workload<"attachment-prune", AttachmentPruneResult> => ({ + queueSpec: ATTACHMENT_PRUNE_QUEUE, + concurrency: 1, + ...(ttlHours > 0 + ? { + schedule: { + interval: "1h", + data: {}, + options: { priority: JOB_PRIORITIES.SCHEDULED }, + }, + } + : {}), + process: async ({ logger }) => { + if (ttlHours <= 0) { + logger.debug("Attachment orphan pruning is disabled."); + return { + pendingClaimed: 0, + committedClaimed: 0, + reclaimed: 0, + }; + } + + const cutoff = new Date(Date.now() - ttlHours * ONE_HOUR_MS); + + // Each claim is atomic, so a PENDING blob committed by a concurrent + // send or a zero-link blob re-linked by a concurrent duplicate-chat + // loses the claim and is left intact. + const pendingClaimed = await db.attachment.updateMany({ + where: { + status: AttachmentStatus.PENDING, + createdAt: { lt: cutoff }, + }, + data: { status: AttachmentStatus.DELETING }, + }); + + const committedClaimed = await db.attachment.updateMany({ + where: { + status: AttachmentStatus.COMMITTED, + createdAt: { lt: cutoff }, + chats: { none: {} }, + }, + data: { status: AttachmentStatus.DELETING }, + }); + + const reclaimed = await reclaimTombstonedAttachments({ + db, + storage, + warn: (message) => logger.warn(message), + }); + + if ( + pendingClaimed.count > 0 || + committedClaimed.count > 0 || + reclaimed > 0 + ) { + logger.debug( + `Attachment prune: condemned ${pendingClaimed.count} PENDING + ` + + `${committedClaimed.count} COMMITTED orphan(s), reclaimed ${reclaimed} tombstone(s).`, + ); + } + + return { + pendingClaimed: pendingClaimed.count, + committedClaimed: committedClaimed.count, + reclaimed, + }; + }, +}); + +/** + * Deletes the bytes for every `DELETING` tombstone, then removes the row. A + * failed byte delete leaves the tombstone in place for the next scheduled run. + * Failed IDs are excluded from later batches in this run so a persistent + * storage failure cannot spin the loop. + */ +const reclaimTombstonedAttachments = async ({ + db, + storage, + warn, +}: { + db: PrismaClient; + storage: StorageBackend; + warn: (message: string) => void; +}): Promise => { + let totalReclaimed = 0; + const failedIds: string[] = []; + + while (true) { + const batch = await db.attachment.findMany({ + where: { + status: AttachmentStatus.DELETING, + id: { notIn: failedIds }, + }, + select: { id: true, storageKey: true }, + take: BATCH_SIZE, + }); + + if (batch.length === 0) { + break; + } + + const settled = await Promise.allSettled( + batch.map((attachment) => storage.delete(attachment.storageKey)), + ); + + const reclaimedIds: string[] = []; + batch.forEach((attachment, index) => { + const outcome = settled[index]; + if (outcome.status === "fulfilled") { + reclaimedIds.push(attachment.id); + } else { + warn( + `Failed to delete bytes for tombstoned attachment ${attachment.id}, will retry next run: ${outcome.reason}`, + ); + failedIds.push(attachment.id); + } + }); + + if (reclaimedIds.length > 0) { + const result = await db.attachment.deleteMany({ + where: { + id: { in: reclaimedIds }, + status: AttachmentStatus.DELETING, + }, + }); + totalReclaimed += result.count; + } + + if (batch.length < BATCH_SIZE) { + break; + } + } + + return totalReclaimed; +}; diff --git a/packages/backend/src/attachmentPruner.ts b/packages/backend/src/attachmentPruner.ts deleted file mode 100644 index ffeb534c3..000000000 --- a/packages/backend/src/attachmentPruner.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { AttachmentStatus, PrismaClient } from "@sourcebot/db"; -import { createLogger, env, getStorageBackend } from "@sourcebot/shared"; -import { setIntervalAsync } from "./utils.js"; - -const BATCH_SIZE = 1_000; -const ONE_HOUR_MS = 60 * 60 * 1000; - -const logger = createLogger('attachment-pruner'); - -/** - * Periodically reclaims orphaned attachment blobs older than the configured TTL, - * along with their stored bytes, using the `DELETING` tombstone protocol: an - * orphan is first atomically flipped to `DELETING`, then its bytes are deleted, - * and only then is the row removed. Because the row (the only durable handle to - * the bytes) outlives the byte delete, a failed byte delete is always retryable. - * - * Each tick condemns two classes of orphan to `DELETING`, then reclaims all - * tombstones: - * - * 1. PENDING (uploaded-but-never-linked): produced when a user selects a file - * in the chat box but never sends the message. - * 2. COMMITTED with zero links: normally a committed blob is reclaimed inline - * by the chat-delete sweep in the web app, but if that sweep is interrupted - * (process crash / DB error / failed byte delete) the blob is left tombstoned - * or unlinked. This is the backstop for that case. - * - * @note Byte deletion goes through the shared `StorageBackend`, so the web app - * and this worker share one on-disk layout. - */ -export class AttachmentPruner { - private interval?: NodeJS.Timeout; - private readonly storage = getStorageBackend(); - - constructor(private db: PrismaClient) {} - - startScheduler() { - const ttlHours = env.SOURCEBOT_CHAT_ATTACHMENT_ORPHAN_TTL_HOURS; - if (ttlHours <= 0) { - logger.info('SOURCEBOT_CHAT_ATTACHMENT_ORPHAN_TTL_HOURS is 0, attachment orphan pruning is disabled.'); - return; - } - - logger.debug(`Attachment pruner started. Reclaiming orphaned attachments older than ${ttlHours} hours.`); - - // Run immediately on startup, then every hour. The startup call isn't - // awaited, so log any failure here: this worker exits on - // unhandledRejection, and the recurring schedule will retry. - this.pruneOrphanedAttachments().catch((error) => { - logger.warn(`Initial attachment prune failed: ${error}`); - }); - this.interval = setIntervalAsync(() => this.pruneOrphanedAttachments(), ONE_HOUR_MS); - } - - async dispose() { - if (this.interval) { - clearInterval(this.interval); - this.interval = undefined; - } - } - - private async pruneOrphanedAttachments() { - const cutoff = new Date(Date.now() - env.SOURCEBOT_CHAT_ATTACHMENT_ORPHAN_TTL_HOURS * ONE_HOUR_MS); - - // Condemn orphans by flipping them to the DELETING tombstone. Each claim - // is atomic, so a PENDING blob committed by a concurrent send (its commit - // matches only PENDING rows) or a zero-link blob re-linked by a concurrent - // duplicate-chat loses the claim and is left intact. - // - // PENDING orphans: uploaded but the message was never sent. - const pendingClaimed = await this.db.attachment.updateMany({ - where: { - status: AttachmentStatus.PENDING, - createdAt: { lt: cutoff }, - }, - data: { status: AttachmentStatus.DELETING }, - }); - - // COMMITTED orphans: blobs left with zero links by an interrupted - // chat-delete sweep in the web app. - const committedClaimed = await this.db.attachment.updateMany({ - where: { - status: AttachmentStatus.COMMITTED, - createdAt: { lt: cutoff }, - chats: { none: {} }, - }, - data: { status: AttachmentStatus.DELETING }, - }); - - // Reclaim every tombstone: delete bytes, then the row. This also picks up - // tombstones left behind by the web app's inline reclaim (or a crashed - // earlier tick) whose byte delete failed. - const reclaimed = await this.reclaimTombstonedAttachments(); - - if (pendingClaimed.count > 0 || committedClaimed.count > 0 || reclaimed > 0) { - logger.debug( - `Attachment prune: condemned ${pendingClaimed.count} PENDING + ` + - `${committedClaimed.count} COMMITTED orphan(s), reclaimed ${reclaimed} tombstone(s).`, - ); - } - } - - /** - * Deletes the bytes for every `DELETING` tombstone, then removes the row. - * The row (the only durable handle to the bytes) is removed only after its - * bytes are confirmed gone, so a failed byte delete leaves the tombstone in - * place to be retried on the next tick — bytes can never be orphaned by a - * transient storage error. Rows whose byte delete fails this run are - * excluded from subsequent batches so a persistent failure can't spin the - * loop. - * - * @returns the number of tombstones fully reclaimed (bytes + row). - */ - private async reclaimTombstonedAttachments(): Promise { - let totalReclaimed = 0; - const failedIds: string[] = []; - - while (true) { - const batch = await this.db.attachment.findMany({ - where: { status: AttachmentStatus.DELETING, id: { notIn: failedIds } }, - select: { id: true, storageKey: true }, - take: BATCH_SIZE, - }); - - if (batch.length === 0) { - break; - } - - const settled = await Promise.allSettled( - batch.map((attachment) => this.storage.delete(attachment.storageKey))); - - const reclaimedIds: string[] = []; - batch.forEach((attachment, index) => { - const outcome = settled[index]; - if (outcome.status === 'fulfilled') { - reclaimedIds.push(attachment.id); - } else { - logger.warn(`Failed to delete bytes for tombstoned attachment ${attachment.id}, will retry next tick: ${outcome.reason}`); - failedIds.push(attachment.id); - } - }); - - if (reclaimedIds.length > 0) { - const result = await this.db.attachment.deleteMany({ - where: { id: { in: reclaimedIds }, status: AttachmentStatus.DELETING }, - }); - totalReclaimed += result.count; - } - - if (batch.length < BATCH_SIZE) { - break; - } - } - - return totalReclaimed; - } -} diff --git a/packages/backend/src/bitbucket.ts b/packages/backend/src/bitbucket.ts index 9051164a7..84f85d023 100644 --- a/packages/backend/src/bitbucket.ts +++ b/packages/backend/src/bitbucket.ts @@ -755,7 +755,7 @@ export const getReposForAuthenticatedBitbucketServerUser = async ( * @note This only covers direct user-to-repo grants. It does NOT include users who have access via: * - Project-level permissions (inherited by all repos in the project) * - Group membership - * These users will still gain access through account-driven syncing (accountPermissionSyncer). + * These users will still gain access through account-driven permission syncing. * * @see https://developer.atlassian.com/server/bitbucket/rest/v906/api-group-repository/#api-rest-api-latest-projects-projectkey-repos-reposlug-permissions-users-get */ diff --git a/packages/backend/src/configManager.test.ts b/packages/backend/src/configManager.test.ts new file mode 100644 index 000000000..2dc50c320 --- /dev/null +++ b/packages/backend/src/configManager.test.ts @@ -0,0 +1,170 @@ +import type { JobManager } from "./types.js"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => { + const watcher = { + on: vi.fn(), + close: vi.fn(), + }; + watcher.on.mockReturnValue(watcher); + + return { + watcher, + loadConfig: vi.fn(), + resolveConfigSettings: vi.fn(), + syncSearchContexts: vi.fn(), + trigger: vi.fn(), + upsertJobScheduler: vi.fn(), + removeJobScheduler: vi.fn(), + connectionFindUnique: vi.fn(), + connectionCreate: vi.fn(), + connectionUpdate: vi.fn(), + connectionFindMany: vi.fn(), + connectionDelete: vi.fn(), + }; +}); + +vi.mock("@sourcebot/shared", () => ({ + createLogger: vi.fn(() => ({ + debug: vi.fn(), + error: vi.fn(), + })), + env: { + DATA_CACHE_DIR: "test-data", + PERMISSION_SYNC_ENABLED: "false", + }, + JOB_PRIORITIES: { + INTERACTIVE: 1, + SCHEDULED: 10, + }, + loadConfig: mocks.loadConfig, + resolveConfigSettings: mocks.resolveConfigSettings, +})); + +vi.mock("chokidar", () => ({ + default: { + watch: vi.fn(() => mocks.watcher), + }, +})); + +vi.mock("./ee/syncSearchContexts.js", () => ({ + syncSearchContexts: mocks.syncSearchContexts, +})); + +vi.mock("./prisma.js", () => ({ + prisma: { + connection: { + findUnique: mocks.connectionFindUnique, + create: mocks.connectionCreate, + update: mocks.connectionUpdate, + findMany: mocks.connectionFindMany, + delete: mocks.connectionDelete, + }, + }, +})); + +import { ConfigManager } from "./configManager.js"; + +const jobManager = { + trigger: mocks.trigger, + upsertJobScheduler: mocks.upsertJobScheduler, + removeJobScheduler: mocks.removeJobScheduler, +} as unknown as JobManager; + +describe("ConfigManager", () => { + beforeEach(() => { + vi.restoreAllMocks(); + vi.clearAllMocks(); + mocks.watcher.on.mockReturnValue(mocks.watcher); + mocks.resolveConfigSettings.mockReturnValue({ + resyncConnectionIntervalMs: 86_400_000, + }); + mocks.syncSearchContexts.mockResolvedValue(undefined); + mocks.trigger.mockResolvedValue("triggered-job"); + mocks.upsertJobScheduler.mockResolvedValue("scheduled-job"); + mocks.removeJobScheduler.mockResolvedValue(true); + mocks.connectionFindUnique.mockResolvedValue(null); + mocks.connectionCreate.mockResolvedValue({ id: 42 }); + mocks.connectionUpdate.mockResolvedValue({ id: 42 }); + mocks.connectionFindMany.mockResolvedValue([]); + mocks.connectionDelete.mockResolvedValue(undefined); + }); + + test("triggers a new connection during config sync", async () => { + const connectionConfig = { + type: "github", + url: "https://github.com", + }; + mocks.loadConfig.mockResolvedValue({ + connections: { sourcebot: connectionConfig }, + contexts: {}, + }); + const manager = new ConfigManager(jobManager, "/config.json"); + + await manager.syncConfig(); + + expect(mocks.trigger).toHaveBeenCalledWith( + "connection-sync", + { connectionId: 42 }, + { priority: 1 }, + ); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "connection-sync", + "connection-sync-v1-42", + 86_400_000, + { connectionId: 42 }, + { priority: 10 }, + ); + expect(mocks.connectionCreate.mock.invocationCallOrder[0]).toBeLessThan( + mocks.upsertJobScheduler.mock.invocationCallOrder[0], + ); + expect( + mocks.upsertJobScheduler.mock.invocationCallOrder[0], + ).toBeLessThan(mocks.trigger.mock.invocationCallOrder[0]); + expect(mocks.syncSearchContexts).toHaveBeenCalledWith({ + contexts: {}, + orgId: 1, + }); + }); + + test("deletes connections removed from the config", async () => { + mocks.loadConfig.mockResolvedValue({}); + mocks.connectionFindMany.mockResolvedValue([ + { id: 42, name: "removed-connection" }, + ]); + const manager = new ConfigManager(jobManager, "/config.json"); + + await manager.syncConfig(); + + expect(mocks.connectionDelete).toHaveBeenCalledWith({ + where: { id: 42 }, + }); + expect(mocks.removeJobScheduler).toHaveBeenCalledWith( + "connection-sync", + "connection-sync-v1-42", + ); + expect( + mocks.removeJobScheduler.mock.invocationCallOrder[0], + ).toBeLessThan(mocks.connectionDelete.mock.invocationCallOrder[0]); + }); + + test("does not trigger an unchanged connection", async () => { + const connectionConfig = { + type: "github", + url: "https://github.com", + }; + mocks.loadConfig.mockResolvedValue({ + connections: { sourcebot: connectionConfig }, + }); + mocks.connectionFindUnique.mockResolvedValue({ + id: 42, + config: connectionConfig, + }); + const manager = new ConfigManager(jobManager, "/config.json"); + + await manager.syncConfig(); + + expect(mocks.upsertJobScheduler).not.toHaveBeenCalled(); + expect(mocks.trigger).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/backend/src/configManager.ts b/packages/backend/src/configManager.ts index 4d4f61ff6..26e1aaf9e 100644 --- a/packages/backend/src/configManager.ts +++ b/packages/backend/src/configManager.ts @@ -1,22 +1,29 @@ -import { Prisma, PrismaClient } from "@sourcebot/db"; -import { createLogger, env } from "@sourcebot/shared"; -import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; -import { loadConfig } from "@sourcebot/shared"; +import { Prisma } from "@sourcebot/db"; +import { + createLogger, + env, + JOB_PRIORITIES, + loadConfig, + resolveConfigSettings, +} from "@sourcebot/shared"; +import type { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import chokidar, { FSWatcher } from 'chokidar'; -import { ConnectionManager } from "./connectionManager.js"; import { SINGLE_TENANT_ORG_ID } from "./constants.js"; import { syncSearchContexts } from "./ee/syncSearchContexts.js"; import isEqual from 'fast-deep-equal'; +import { prisma } from "./prisma.js"; +import type { JobManager } from "./types.js"; const logger = createLogger('config-manager'); +const getConnectionSyncSchedulerId = (connectionId: number) => + `connection-sync-v1-${connectionId}`; export class ConfigManager { private watcher: FSWatcher; constructor( - private db: PrismaClient, - private connectionManager: ConnectionManager, - configPath: string, + private readonly jobManager: JobManager, + private readonly configPath: string, ) { this.watcher = chokidar.watch(configPath, { ignoreInitial: true, // Don't fire events for existing files @@ -28,32 +35,38 @@ export class ConfigManager { }); this.watcher.on('change', async () => { - logger.debug(`Config file ${configPath} changed. Syncing config.`); + logger.debug(`Config file ${this.configPath} changed. Syncing config.`); try { - await this.syncConfig(configPath); + await this.syncConfig(); } catch (error) { logger.error(`Failed to sync config: ${error}`); } }); - - this.syncConfig(configPath); } - private syncConfig = async (configPath: string) => { - const config = await loadConfig(configPath); + public syncConfig = async (): Promise => { + const config = await loadConfig(this.configPath); + const settings = resolveConfigSettings(config); - await this.syncConnections(config.connections); + await this.syncConnections( + config.connections, + settings.resyncConnectionIntervalMs, + ); await syncSearchContexts({ contexts: config.contexts, orgId: SINGLE_TENANT_ORG_ID, - db: this.db, }); } - private syncConnections = async (connections?: { [key: string]: ConnectionConfig }) => { + private syncConnections = async ( + connections: { [key: string]: ConnectionConfig } | undefined, + intervalMs: number, + ) => { + const connectionIdsToSync: number[] = []; + if (connections) { for (const [key, newConnectionConfig] of Object.entries(connections)) { - const existingConnection = await this.db.connection.findUnique({ + const existingConnection = await prisma.connection.findUnique({ where: { name_orgId: { name: key, @@ -73,7 +86,7 @@ export class ConfigManager { // Either update the existing connection or create a new one. const connection = existingConnection ? - await this.db.connection.update({ + await prisma.connection.update({ where: { id: existingConnection.id, }, @@ -84,7 +97,7 @@ export class ConfigManager { enforcePermissionsForPublicRepos, } }) : - await this.db.connection.create({ + await prisma.connection.create({ data: { name: key, config: newConnectionConfig as unknown as Prisma.InputJsonValue, @@ -100,15 +113,24 @@ export class ConfigManager { } }); + if (!existingConnection) { + await this.jobManager.upsertJobScheduler( + "connection-sync", + getConnectionSyncSchedulerId(connection.id), + intervalMs, + { connectionId: connection.id }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ); + } + if (connectionNeedsSyncing) { - logger.debug(`Change detected for connection '${key}' (id: ${connection.id}). Creating sync job.`); - await this.connectionManager.createJobs([connection]); + connectionIdsToSync.push(connection.id); } } } // Delete any connections that are no longer in the config. - const deletedConnections = await this.db.connection.findMany({ + const deletedConnections = await prisma.connection.findMany({ where: { isDeclarative: true, name: { @@ -118,9 +140,23 @@ export class ConfigManager { } }); + await Promise.all( + connectionIdsToSync.map((connectionId) => + this.jobManager.trigger( + "connection-sync", + { connectionId }, + { priority: JOB_PRIORITIES.INTERACTIVE }, + ), + ), + ); + for (const connection of deletedConnections) { logger.debug(`Deleting connection with name '${connection.name}'. Connection ID: ${connection.id}`); - await this.db.connection.delete({ + await this.jobManager.removeJobScheduler( + "connection-sync", + getConnectionSyncSchedulerId(connection.id), + ); + await prisma.connection.delete({ where: { id: connection.id, } @@ -131,4 +167,4 @@ export class ConfigManager { public dispose = async () => { await this.watcher.close(); } -} \ No newline at end of file +} diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts deleted file mode 100644 index c9db057e2..000000000 --- a/packages/backend/src/connectionManager.ts +++ /dev/null @@ -1,410 +0,0 @@ -import * as Sentry from "@sentry/node"; -import { Connection, ConnectionSyncJobStatus, PrismaClient } from "@sourcebot/db"; -import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; -import { createLogger, env, loadConfig } from "@sourcebot/shared"; -import { Job, Queue, Worker } from "bullmq"; -import { Redis } from 'ioredis'; -import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; -import { syncSearchContexts } from "./ee/syncSearchContexts.js"; -import { captureEvent } from "./posthog.js"; -import { PromClient } from "./promClient.js"; -import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js"; -import { Settings } from "./types.js"; -import { setIntervalAsync } from "./utils.js"; - -const LOG_TAG = 'connection-manager'; -const logger = createLogger(LOG_TAG); -const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); -const QUEUE_NAME = 'connection-sync-queue'; - -const CONNECTION_SYNC_TIMEOUT_MS = 1000 * 60 * 60 * 2; // 2 hours - -type JobPayload = { - jobId: string, - connectionId: number, - connectionName: string, - orgId: number, -}; - -type JobResult = { - repoCount: number, -} - -export class ConnectionManager { - private worker: Worker; - private queue: Queue; - private abortController: AbortController; - private interval?: NodeJS.Timeout; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - private promClient: PromClient, - ) { - this.abortController = new AbortController(); - - this.queue = new Queue(QUEUE_NAME, { - connection: redis, - defaultJobOptions: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - attempts: 2, - }, - }); - - this.worker = new Worker( - QUEUE_NAME, - this.runJob.bind(this), - { - connection: redis, - concurrency: this.settings.maxConnectionSyncJobConcurrency, - maxStalledCount: 1, - } - ); - - this.worker.on('completed', this.onJobCompleted.bind(this)); - this.worker.on('failed', this.onJobMaybeFailed.bind(this)); - this.worker.on('stalled', (jobId) => { - // Just log - BullMQ will automatically retry the job (up to maxStalledCount times). - // If all retries fail, onJobMaybeFailed will handle marking it as failed. - logger.warn(`Job ${jobId} stalled - BullMQ will retry`); - }); - this.worker.on('error', (error) => { - logger.error(`Connection syncer worker error:`, error); - }); - } - - public startScheduler() { - logger.debug('Starting scheduler'); - this.interval = setIntervalAsync(async () => { - const thresholdDate = new Date(Date.now() - this.settings.resyncConnectionIntervalMs); - const timeoutDate = new Date(Date.now() - CONNECTION_SYNC_TIMEOUT_MS); - - const connections = await this.db.connection.findMany({ - where: { - AND: [ - { - OR: [ - { syncedAt: null }, - { syncedAt: { lt: thresholdDate } }, - ] - }, - { - NOT: { - syncJobs: { - some: { - OR: [ - // Don't schedule if there are active jobs that were created within the threshold date. - // This handles the case where a job is stuck in a pending state and will never be scheduled. - { - AND: [ - { status: { in: [ConnectionSyncJobStatus.PENDING, ConnectionSyncJobStatus.IN_PROGRESS] } }, - { createdAt: { gt: timeoutDate } }, - ] - }, - // Don't schedule if there are recent failed jobs (within the threshold date). - { - AND: [ - { status: ConnectionSyncJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } - ] - } - } - } - } - ] - } - }); - - if (connections.length > 0) { - await this.createJobs(connections); - } - }, this.settings.resyncConnectionPollingIntervalMs); - } - - - public async createJobs(connections: Connection[]) { - const jobs = await this.db.connectionSyncJob.createManyAndReturn({ - data: connections.map(connection => ({ - connectionId: connection.id, - })), - include: { - connection: true, - } - }); - - for (const job of jobs) { - logger.debug(`Scheduling job ${job.id} for connection ${job.connection.name} (id: ${job.connectionId})`); - await this.queue.add( - 'connection-sync-job', - { - jobId: job.id, - connectionId: job.connectionId, - connectionName: job.connection.name, - orgId: job.connection.orgId, - }, - { jobId: job.id } - ); - - this.promClient.pendingConnectionSyncJobs.inc({ connection: job.connection.name }); - } - - return jobs.map(job => job.id); - } - - private async runJob(job: Job): Promise { - const { jobId, connectionName } = job.data; - const logger = createJobLogger(jobId); - logger.debug(`Running connection sync job ${jobId} for connection ${connectionName} (id: ${job.data.connectionId})`); - - const currentStatus = await this.db.connectionSyncJob.findUniqueOrThrow({ - where: { - id: jobId, - }, - select: { - status: true, - } - }); - - // Fail safe: if the job is not PENDING (first run) or IN_PROGRESS (retry), it indicates the job - // is in an invalid state and should be skipped. - if (currentStatus.status !== ConnectionSyncJobStatus.PENDING && currentStatus.status !== ConnectionSyncJobStatus.IN_PROGRESS) { - throw new Error(`Job ${jobId} is not in a valid state. Expected: ${ConnectionSyncJobStatus.PENDING} or ${ConnectionSyncJobStatus.IN_PROGRESS}. Actual: ${currentStatus.status}. Skipping.`); - } - - this.promClient.pendingConnectionSyncJobs.dec({ connection: connectionName }); - this.promClient.activeConnectionSyncJobs.inc({ connection: connectionName }); - - const { connection: { config: rawConnectionConfig, orgId } } = await this.db.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - status: ConnectionSyncJobStatus.IN_PROGRESS, - }, - select: { - connection: { - select: { - config: true, - orgId: true, - } - } - }, - }); - - const config = rawConnectionConfig as unknown as ConnectionConfig; - - const result = await (async () => { - switch (config.type) { - case 'github': { - return await compileGithubConfig(config, job.data.connectionId, this.abortController.signal); - } - case 'gitlab': { - return await compileGitlabConfig(config, job.data.connectionId); - } - case 'gitea': { - return await compileGiteaConfig(config, job.data.connectionId); - } - case 'gerrit': { - return await compileGerritConfig(config, job.data.connectionId); - } - case 'bitbucket': { - return await compileBitbucketConfig(config, job.data.connectionId); - } - case 'azuredevops': { - return await compileAzureDevOpsConfig(config, job.data.connectionId); - } - case 'git': { - return await compileGenericGitHostConfig(config, job.data.connectionId); - } - } - })(); - - let { repoData, warnings } = result; - - await this.db.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - warningMessages: warnings, - }, - }); - - - // Filter out any duplicates by external_id and external_codeHostUrl. - repoData = repoData.filter((repo, index, self) => { - return index === self.findIndex(r => - r.external_id === repo.external_id && - r.external_codeHostUrl === repo.external_codeHostUrl - ); - }) - - // @note: to handle orphaned Repos we delete all RepoToConnection records for this connection, - // and then recreate them when we upsert the repos. For example, if a repo is no-longer - // captured by the connection's config (e.g., it was deleted, marked archived, etc.), it won't - // appear in the repoData array above, and so the RepoToConnection record won't be re-created. - // Repos that have no RepoToConnection records are considered orphaned and can be deleted. - await this.db.$transaction(async (tx) => { - const deleteStart = performance.now(); - await tx.connection.update({ - where: { - id: job.data.connectionId, - }, - data: { - repos: { - deleteMany: {} - } - } - }); - const deleteDuration = performance.now() - deleteStart; - logger.debug(`Deleted all RepoToConnection records for connection ${connectionName} (id: ${job.data.connectionId}) in ${deleteDuration}ms`); - - const totalUpsertStart = performance.now(); - for (const repo of repoData) { - const upsertStart = performance.now(); - await tx.repo.upsert({ - where: { - external_id_external_codeHostUrl_orgId: { - external_id: repo.external_id, - external_codeHostUrl: repo.external_codeHostUrl, - orgId: orgId, - } - }, - update: repo, - create: repo, - }) - const upsertDuration = performance.now() - upsertStart; - logger.debug(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`); - } - const totalUpsertDuration = performance.now() - totalUpsertStart; - logger.debug(`Upserted ${repoData.length} repos for connection ${connectionName} (id: ${job.data.connectionId}) in ${totalUpsertDuration}ms`); - }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS }); - - return { - repoCount: repoData.length, - }; - } - - - private async onJobCompleted(job: Job, result: JobResult) { - try { - const logger = createJobLogger(job.id!); - const { connectionId, connectionName, orgId } = job.data; - - const { connection } = await this.db.connectionSyncJob.update({ - where: { - id: job.id!, - }, - data: { - status: ConnectionSyncJobStatus.COMPLETED, - completedAt: new Date(), - connection: { - update: { - syncedAt: new Date(), - } - } - }, - select: { - connection: true, - } - }); - - // After a connection has synced, we need to re-sync the org's search contexts as - // there may be new repos that match the search context's include/exclude patterns. - if (env.CONFIG_PATH) { - try { - const config = await loadConfig(env.CONFIG_PATH); - - await syncSearchContexts({ - db: this.db, - orgId, - contexts: config.contexts, - }); - } catch (err) { - logger.error(`Failed to sync search contexts for connection ${connectionId}: ${err}`); - Sentry.captureException(err); - } - } - - logger.debug(`Connection sync job ${job.id} for connection ${job.data.connectionName} (id: ${job.data.connectionId}) completed`); - - this.promClient.activeConnectionSyncJobs.dec({ connection: connectionName }); - this.promClient.connectionSyncJobSuccessTotal.inc({ connection: connectionName }); - - const config = connection.config as unknown as ConnectionConfig; - captureEvent('backend_connection_sync_job_completed', { - connectionId: connectionId, - repoCount: result.repoCount, - type: config.type, - }); - } catch (error) { - Sentry.captureException(error); - logger.error(`Exception thrown while executing lifecycle function \`onJobCompleted\`.`, error); - } - } - - private async onJobMaybeFailed(job: Job | undefined, error: Error) { - try { - if (!job) { - logger.error(`Job failed but job object is undefined. Error: ${error.message}`); - return; - } - const jobLogger = createJobLogger(job.id!); - - // @note: we need to check the job state to determine if the job failed, - // or if it is being retried. - const jobState = await job.getState(); - if (jobState !== 'failed') { - jobLogger.warn(`Job ${job.id} for connection ${job.data.connectionName} (id: ${job.data.connectionId}) failed. Retrying...`); - return; - } - - const { connection } = await this.db.connectionSyncJob.update({ - where: { id: job.id }, - data: { - status: ConnectionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: job.failedReason, - }, - select: { - connection: true, - } - }); - - this.promClient.activeConnectionSyncJobs.dec({ connection: connection.name }); - this.promClient.connectionSyncJobFailTotal.inc({ connection: connection.name }); - - jobLogger.error(`Failed job ${job.id} for connection ${connection.name} (id: ${connection.id}). Reason: ${job.failedReason}`); - - const config = connection.config as unknown as ConnectionConfig; - captureEvent('backend_connection_sync_job_failed', { - connectionId: job.data.connectionId, - type: config.type, - }); - } catch (err) { - Sentry.captureException(err); - logger.error(`Exception thrown while executing lifecycle function \`onJobMaybeFailed\`.`, err); - } - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - - // Signal all active jobs to abort - this.abortController.abort(); - - // Wait for worker to finish with timeout - await Promise.race([ - this.worker.close(), - new Promise(resolve => setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS)) - ]); - - await this.queue.close(); - } -} diff --git a/packages/backend/src/connectionWorkload.test.ts b/packages/backend/src/connectionWorkload.test.ts new file mode 100644 index 000000000..5997b8e4b --- /dev/null +++ b/packages/backend/src/connectionWorkload.test.ts @@ -0,0 +1,643 @@ +import type { PrismaClient } from "@sourcebot/db"; +import type { RepoData } from "./repoCompileUtils.js"; +import type { JobManager, ProcessContext } from "./types.js"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + connectionFindUniqueOrThrow: vi.fn(), + connectionUpdate: vi.fn(), + connectionSyncJobUpsert: vi.fn(), + connectionSyncJobUpdate: vi.fn(), + compileGithubConfig: vi.fn(), + loadConfig: vi.fn(), + syncSearchContexts: vi.fn(), + repoFindMany: vi.fn(), + repoUpsert: vi.fn(), + repoToConnectionDeleteMany: vi.fn(), + getJobSchedulerIds: vi.fn(), + upsertJobScheduler: vi.fn(), + removeJobScheduler: vi.fn(), +})); + +vi.mock("@sentry/node", () => ({ + captureException: vi.fn(), +})); + +vi.mock("@sourcebot/shared", () => ({ + CONNECTION_QUEUE: { + name: "connection-sync", + dedupKey: ({ connectionId }: { connectionId: number }) => + `connection:${connectionId}`, + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keepJobs: { + completed: { count: 50 }, + failed: { count: 50 }, + }, + keepLogs: 500, + }, + }, + JOB_PRIORITIES: { + INITIAL: 5, + SCHEDULED: 10, + }, + env: { + CONFIG_PATH: "/config.json", + CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: 60_000, + }, + PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES: [ + "github", + "gitlab", + "bitbucketCloud", + "bitbucketServer", + ], + PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS: [ + "github", + "gitlab", + "bitbucket-cloud", + "bitbucket-server", + ], + loadConfig: mocks.loadConfig, +})); + +vi.mock("./repoCompileUtils.js", () => ({ + compileAzureDevOpsConfig: vi.fn(), + compileBitbucketConfig: vi.fn(), + compileGenericGitHostConfig: vi.fn(), + compileGerritConfig: vi.fn(), + compileGiteaConfig: vi.fn(), + compileGithubConfig: mocks.compileGithubConfig, + compileGitlabConfig: vi.fn(), +})); + +vi.mock("./ee/syncSearchContexts.js", () => ({ + syncSearchContexts: mocks.syncSearchContexts, +})); + +import { + createConnectionWorkload, + persistConnectionRepositories, + reconcileRepoIndexWork, + reconcileRepoPermissionSyncWork, +} from "./connectionWorkload.js"; +import { REPO_PERMISSION_SYNC_WHERE } from "./ee/permissionSyncEligibility.js"; + +const transactionClient = { + connection: { + update: mocks.connectionUpdate, + }, + connectionSyncJob: { + upsert: mocks.connectionSyncJobUpsert, + }, +}; +const transaction = vi.fn( + (callback: (tx: typeof transactionClient) => Promise) => + callback(transactionClient), +); + +const db = { + connection: { + findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, + update: mocks.connectionUpdate, + }, + connectionSyncJob: { + upsert: mocks.connectionSyncJobUpsert, + update: mocks.connectionSyncJobUpdate, + }, + repo: { + findMany: mocks.repoFindMany, + upsert: mocks.repoUpsert, + }, + repoToConnection: { + deleteMany: mocks.repoToConnectionDeleteMany, + }, + $transaction: transaction, +} as unknown as PrismaClient; + +const jobManager = { + getJobSchedulerIds: mocks.getJobSchedulerIds, + upsertJobScheduler: mocks.upsertJobScheduler, + removeJobScheduler: mocks.removeJobScheduler, +} as unknown as JobManager; +const connectionWorkload = createConnectionWorkload({ + db, + jobManager, + permissionSyncEnabled: true, + settings: { + maxConnectionSyncJobConcurrency: 2, + reindexIntervalMs: 3_600_000, + repoDrivenPermissionSyncIntervalMs: 21_600_000, + } as never, +}); + +const data = { + connectionId: 42, +}; + +const lifecycleLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +}; + +const lifecycleContext = { + data, + jobId: "job-1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + logger: lifecycleLogger, +}; + +describe("connectionWorkload", () => { + beforeEach(() => { + vi.restoreAllMocks(); + vi.clearAllMocks(); + mocks.connectionUpdate.mockResolvedValue({}); + mocks.repoFindMany.mockResolvedValue([]); + mocks.getJobSchedulerIds.mockResolvedValue([]); + mocks.upsertJobScheduler.mockResolvedValue("scheduled-job"); + mocks.removeJobScheduler.mockResolvedValue(true); + mocks.loadConfig.mockResolvedValue({ contexts: undefined }); + mocks.syncSearchContexts.mockResolvedValue(undefined); + }); + + test("declares database-backed lifecycle hooks", () => { + expect(connectionWorkload.onStarted).toBeTypeOf("function"); + expect(connectionWorkload.onCompleted).toBeTypeOf("function"); + expect(connectionWorkload.onTerminalFailure).toBeTypeOf("function"); + }); + + test("uses a distinct execution lock for each connection", () => { + expect(connectionWorkload.executionLock).toBeDefined(); + expect( + connectionWorkload.executionLock?.resource({ connectionId: 42 }), + ).toBe("sourcebot:lock:connection:42"); + expect( + connectionWorkload.executionLock?.resource({ connectionId: 43 }), + ).toBe("sourcebot:lock:connection:43"); + expect(connectionWorkload.executionLock?.durationMs).toBe(60_000); + }); + + test("does not start syncing when execution has already been aborted", async () => { + const controller = new AbortController(); + controller.abort(new Error("Connection execution lock was lost")); + + await expect( + connectionWorkload.process({ + ...lifecycleContext, + signal: controller.signal, + updateProgress: vi.fn(), + trigger: vi.fn(), + }), + ).rejects.toThrow("Connection execution lock was lost"); + expect(mocks.connectionFindUniqueOrThrow).not.toHaveBeenCalled(); + }); + + test("marks the connection sync job as in progress when started", async () => { + await connectionWorkload.onStarted?.(lifecycleContext); + + expect(mocks.connectionSyncJobUpsert).toHaveBeenCalledWith({ + where: { + id: "job-1", + }, + update: { + status: "IN_PROGRESS", + completedAt: null, + errorMessage: null, + warningMessages: [], + }, + create: { + id: "job-1", + connectionId: 42, + status: "IN_PROGRESS", + warningMessages: [], + }, + }); + expect(mocks.connectionUpdate).toHaveBeenCalledWith({ + where: { + id: 42, + }, + data: { + latestSyncJobId: "job-1", + }, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("marks the connection sync job as completed", async () => { + await connectionWorkload.onCompleted?.(lifecycleContext, { + reposToCleanup: [], + reposToIndex: [], + }); + + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: "job-1", + }, + data: { + status: "COMPLETED", + completedAt: expect.any(Date), + errorMessage: null, + }, + }); + }); + + test("marks the connection sync job as failed after terminal failure", async () => { + await connectionWorkload.onTerminalFailure?.( + lifecycleContext, + new Error("Connection credentials expired"), + ); + + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: "job-1", + }, + data: { + status: "FAILED", + completedAt: expect.any(Date), + errorMessage: "Connection credentials expired", + }, + }); + }); + + test("orchestrates discovery, persistence, and repo work reconciliation", async () => { + const config = { + type: "github" as const, + }; + const discoveredRepo = { + external_id: "repo-4", + external_codeHostUrl: "https://github.com", + }; + mocks.connectionFindUniqueOrThrow.mockResolvedValue({ + id: 42, + name: "github", + orgId: 7, + config, + }); + mocks.compileGithubConfig.mockResolvedValue({ + repoData: [discoveredRepo], + warnings: ["Repository was archived"], + }); + mocks.repoUpsert.mockResolvedValue({ + id: 4, + name: "github.com/sourcebot/repo-4", + indexedAt: null, + }); + const trigger = vi.fn(); + const updateProgress = vi.fn(); + + const result = await connectionWorkload.process({ + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress, + trigger, + }); + + expect(mocks.compileGithubConfig).toHaveBeenCalledWith( + config, + 42, + expect.any(AbortSignal), + ); + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { id: "job-1" }, + data: { warningMessages: ["Repository was archived"] }, + }); + expect(mocks.repoUpsert).toHaveBeenCalledOnce(); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "repo-index", + "repo-index-v1-4", + 3_600_000, + { repoId: 4, type: "INDEX" }, + { priority: 10 }, + ); + expect(trigger).toHaveBeenCalledWith( + "repo-index", + { + repoId: 4, + type: "INDEX", + }, + { priority: 5 }, + ); + expect(mocks.connectionUpdate).toHaveBeenCalledWith({ + where: { id: 42 }, + data: { syncedAt: expect.any(Date) }, + }); + expect(mocks.syncSearchContexts).toHaveBeenCalledWith({ + orgId: 7, + contexts: undefined, + }); + expect(result).toEqual({ + reposToCleanup: [], + reposToIndex: [ + { id: 4, name: "github.com/sourcebot/repo-4" }, + ], + }); + expect(updateProgress).not.toHaveBeenCalled(); + }); + + test("does not mark the connection synced when repo work reconciliation fails", async () => { + mocks.connectionFindUniqueOrThrow.mockResolvedValue({ + id: 42, + name: "github", + orgId: 7, + config: { type: "github" }, + }); + mocks.compileGithubConfig.mockResolvedValue({ + repoData: [ + { + external_id: "repo-4", + external_codeHostUrl: "https://github.com", + }, + ], + warnings: [], + }); + mocks.repoUpsert.mockResolvedValue({ + id: 4, + name: "github.com/sourcebot/repo-4", + indexedAt: null, + }); + mocks.upsertJobScheduler.mockRejectedValueOnce( + new Error("Redis unavailable"), + ); + + await expect( + connectionWorkload.process({ + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), + }), + ).rejects.toThrow("Redis unavailable"); + + expect(mocks.connectionUpdate).not.toHaveBeenCalled(); + expect(mocks.syncSearchContexts).not.toHaveBeenCalled(); + }); +}); + +describe("connectionWorkload repo sync helpers", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.getJobSchedulerIds.mockResolvedValue([]); + mocks.upsertJobScheduler.mockResolvedValue("scheduled-job"); + mocks.removeJobScheduler.mockResolvedValue(true); + }); + + test("persists the discovered repository snapshot", async () => { + const indexedAt = new Date("2026-07-30T12:00:00.000Z"); + const existingRepo = { + external_id: "repo-1", + external_codeHostUrl: "https://github.com", + displayName: "sourcebot/repo-1", + connections: { + create: { + connectionId: 42, + }, + }, + }; + const newRepo = { + external_id: "repo-4", + external_codeHostUrl: "https://github.com", + displayName: "sourcebot/repo-4", + connections: { + create: { + connectionId: 42, + }, + }, + }; + mocks.repoFindMany + .mockResolvedValueOnce([{ id: 1 }, { id: 2 }, { id: 3 }]) + .mockResolvedValueOnce([ + { id: 2, name: "github.com/sourcebot/repo-2" }, + ]); + mocks.repoUpsert + .mockResolvedValueOnce({ + id: 1, + name: "github.com/sourcebot/repo-1", + indexedAt, + }) + .mockResolvedValueOnce({ + id: 4, + name: "github.com/sourcebot/repo-4", + indexedAt: null, + }); + + const result = await persistConnectionRepositories({ + db, + connectionId: 42, + orgId: 7, + discoveredRepos: [ + existingRepo, + newRepo, + newRepo, + ] as unknown as RepoData[], + }); + + expect(mocks.repoUpsert).toHaveBeenCalledTimes(2); + expect(mocks.repoUpsert).toHaveBeenNthCalledWith(1, { + where: { + external_id_external_codeHostUrl_orgId: { + external_id: "repo-1", + external_codeHostUrl: "https://github.com", + orgId: 7, + }, + }, + update: { + ...existingRepo, + connections: { + createMany: { + data: { connectionId: 42 }, + skipDuplicates: true, + }, + }, + }, + create: existingRepo, + select: { + id: true, + name: true, + indexedAt: true, + }, + }); + expect(mocks.repoToConnectionDeleteMany).toHaveBeenCalledWith({ + where: { + connectionId: 42, + repoId: { + in: [2, 3], + }, + }, + }); + expect(result).toEqual({ + currentRepos: [ + { + id: 1, + name: "github.com/sourcebot/repo-1", + indexedAt, + }, + { + id: 4, + name: "github.com/sourcebot/repo-4", + indexedAt: null, + }, + ], + unindexedRepos: [ + { id: 4, name: "github.com/sourcebot/repo-4" }, + ], + orphanedRepos: [ + { id: 2, name: "github.com/sourcebot/repo-2" }, + ], + affectedRepoIds: [1, 4, 2, 3], + }); + }); + + test("reconciles repo indexing schedules and immediate work", async () => { + const trigger = vi.fn().mockResolvedValue("job"); + const indexedAt = new Date("2026-07-30T12:00:00.000Z"); + + await reconcileRepoIndexWork({ + jobManager, + trigger: trigger as ProcessContext<"connection-sync">["trigger"], + currentRepos: [ + { id: 1, name: "repo-1", indexedAt }, + { id: 4, name: "repo-4", indexedAt: null }, + ], + unindexedRepos: [{ id: 4, name: "repo-4" }], + orphanedRepos: [{ id: 2, name: "repo-2" }], + intervalMs: 3_600_000, + }); + + expect(mocks.upsertJobScheduler).toHaveBeenNthCalledWith( + 1, + "repo-index", + "repo-index-v1-1", + 3_600_000, + { repoId: 1, type: "INDEX" }, + { priority: 10 }, + ); + expect(mocks.upsertJobScheduler).toHaveBeenNthCalledWith( + 2, + "repo-index", + "repo-index-v1-4", + 3_600_000, + { repoId: 4, type: "INDEX" }, + { priority: 10 }, + ); + expect(mocks.removeJobScheduler).toHaveBeenCalledWith( + "repo-index", + "repo-index-v1-2", + ); + expect(trigger).toHaveBeenNthCalledWith( + 1, + "repo-index", + { + repoId: 2, + type: "CLEANUP", + }, + { priority: 10 }, + ); + expect(trigger).toHaveBeenNthCalledWith( + 2, + "repo-index", + { + repoId: 4, + type: "INDEX", + }, + { priority: 5 }, + ); + expect( + mocks.upsertJobScheduler.mock.invocationCallOrder[1], + ).toBeLessThan(trigger.mock.invocationCallOrder[0]); + }); + + test("reconciles repo permission schedules and immediate work", async () => { + const trigger = vi.fn().mockResolvedValue("job"); + const permissionSyncedAt = new Date("2026-07-30T12:00:00.000Z"); + mocks.repoFindMany.mockResolvedValue([ + { id: 1, permissionSyncedAt: null }, + { id: 4, permissionSyncedAt }, + ]); + mocks.getJobSchedulerIds.mockResolvedValue([ + "repo-permission-sync-v1-1", + ]); + + await reconcileRepoPermissionSyncWork({ + db, + jobManager, + trigger: trigger as ProcessContext<"connection-sync">["trigger"], + enabled: true, + affectedRepoIds: [1, 4, 2, 3], + intervalMs: 21_600_000, + }); + + expect(mocks.repoFindMany).toHaveBeenCalledWith({ + where: { + id: { + in: [1, 4, 2, 3], + }, + ...REPO_PERMISSION_SYNC_WHERE, + }, + select: { + id: true, + permissionSyncedAt: true, + }, + }); + expect(mocks.upsertJobScheduler).toHaveBeenNthCalledWith( + 1, + "repo-permission-sync", + "repo-permission-sync-v1-1", + 21_600_000, + { repoId: 1 }, + { priority: 10 }, + ); + expect(mocks.upsertJobScheduler).toHaveBeenNthCalledWith( + 2, + "repo-permission-sync", + "repo-permission-sync-v1-4", + 21_600_000, + { repoId: 4 }, + { priority: 10 }, + ); + expect(mocks.removeJobScheduler).toHaveBeenNthCalledWith( + 1, + "repo-permission-sync", + "repo-permission-sync-v1-2", + ); + expect(mocks.removeJobScheduler).toHaveBeenNthCalledWith( + 2, + "repo-permission-sync", + "repo-permission-sync-v1-3", + ); + expect(trigger).toHaveBeenNthCalledWith( + 1, + "repo-permission-sync", + { repoId: 1 }, + { priority: 10 }, + ); + expect(trigger).toHaveBeenNthCalledWith( + 2, + "repo-permission-sync", + { repoId: 4 }, + { priority: 10 }, + ); + }); + + test("removes permission schedules without querying eligibility when disabled", async () => { + const trigger = vi.fn(); + + await reconcileRepoPermissionSyncWork({ + db, + jobManager, + trigger: trigger as ProcessContext<"connection-sync">["trigger"], + enabled: false, + affectedRepoIds: [1, 2], + intervalMs: 21_600_000, + }); + + expect(mocks.repoFindMany).not.toHaveBeenCalled(); + expect(mocks.getJobSchedulerIds).not.toHaveBeenCalled(); + expect(mocks.upsertJobScheduler).not.toHaveBeenCalled(); + expect(mocks.removeJobScheduler).toHaveBeenCalledTimes(2); + expect(trigger).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts new file mode 100644 index 000000000..31eb59e7a --- /dev/null +++ b/packages/backend/src/connectionWorkload.ts @@ -0,0 +1,524 @@ +import * as Sentry from "@sentry/node"; +import { ConnectionSyncJobStatus, PrismaClient } from "@sourcebot/db"; +import { ConnectionConfig } from "@sourcebot/schemas/v3/index.type"; +import { + CONNECTION_QUEUE, + env, + JOB_PRIORITIES, + loadConfig, +} from "@sourcebot/shared"; +import { REPO_PERMISSION_SYNC_WHERE } from "./ee/permissionSyncEligibility.js"; +import { syncSearchContexts } from "./ee/syncSearchContexts.js"; +import { + compileAzureDevOpsConfig, + compileBitbucketConfig, + compileGenericGitHostConfig, + compileGerritConfig, + compileGiteaConfig, + compileGithubConfig, + compileGitlabConfig, +} from "./repoCompileUtils.js"; +import type { RepoData } from "./repoCompileUtils.js"; +import { JobManager, ProcessContext, Settings, Workload } from "./types.js"; + +const CONNECTION_SYNC_LOCK_DURATION_MS = 60_000; + +interface Props { + db: PrismaClient; + jobManager: JobManager; + permissionSyncEnabled: boolean; + settings: Settings; +} + +interface ConnectionSyncResult { + reposToCleanup: { id: number; name: string }[]; + reposToIndex: { id: number; name: string }[]; +} + +export const createConnectionWorkload = ({ + db, + jobManager, + permissionSyncEnabled, + settings, +}: Props): Workload<"connection-sync", ConnectionSyncResult> => ({ + queueSpec: CONNECTION_QUEUE, + concurrency: settings.maxConnectionSyncJobConcurrency, + executionLock: { + resource: ({ connectionId }) => + `sourcebot:lock:connection:${connectionId}`, + durationMs: CONNECTION_SYNC_LOCK_DURATION_MS, + }, + process: async ({ + data: { connectionId }, + logger, + signal, + jobId, + trigger, + }) => { + signal.throwIfAborted(); + const connection = await db.connection.findUniqueOrThrow({ + where: { + id: connectionId, + }, + }); + signal.throwIfAborted(); + const { orgId } = connection; + + logger.info(`Syncing connection ${connectionId}`, { + connectionId, + orgId, + }); + + const { repoData, warnings } = await discoverConnectionRepositories({ + config: connection.config as unknown as ConnectionConfig, + connectionId, + signal, + }); + + signal.throwIfAborted(); + await db.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + warningMessages: warnings, + }, + }); + + logger.info(`Discovered ${repoData.length} repositories`, { + connectionId, + repositoryCount: repoData.length, + }); + + signal.throwIfAborted(); + const repoChanges = await persistConnectionRepositories({ + db, + connectionId, + orgId, + discoveredRepos: repoData, + }); + + signal.throwIfAborted(); + await reconcileRepoIndexWork({ + jobManager, + trigger, + currentRepos: repoChanges.currentRepos, + unindexedRepos: repoChanges.unindexedRepos, + orphanedRepos: repoChanges.orphanedRepos, + intervalMs: settings.reindexIntervalMs, + }); + + signal.throwIfAborted(); + await reconcileRepoPermissionSyncWork({ + db, + jobManager, + trigger, + enabled: permissionSyncEnabled, + affectedRepoIds: repoChanges.affectedRepoIds, + intervalMs: settings.repoDrivenPermissionSyncIntervalMs, + }); + + logger.info( + `Stored ${repoChanges.currentRepos.length} repositories`, + { + connectionId, + connectionName: connection.name, + repositoryCount: repoChanges.currentRepos.length, + }, + ); + + signal.throwIfAborted(); + await db.connection.update({ + where: { + id: connectionId, + }, + data: { + syncedAt: new Date(), + }, + }); + + // After a connection has synced, we need to re-sync the org's search contexts as + // there may be new repos that match the search context's include/exclude patterns. + signal.throwIfAborted(); + try { + const config = await loadConfig(env.CONFIG_PATH); + + await syncSearchContexts({ + orgId, + contexts: config.contexts, + }); + } catch (error) { + logger.error( + `Failed to sync search contexts for connection ${connectionId}`, + error, + ); + Sentry.captureException(error); + } + signal.throwIfAborted(); + + logger.info(`Connection ${connectionId} sync finished`, { + connectionId, + }); + + return { + reposToCleanup: repoChanges.orphanedRepos, + reposToIndex: repoChanges.unindexedRepos, + }; + }, + onStarted: async ({ data: { connectionId }, jobId }) => { + await db.$transaction(async (tx) => { + await tx.connectionSyncJob.upsert({ + where: { + id: jobId, + }, + update: { + status: ConnectionSyncJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + warningMessages: [], + }, + create: { + id: jobId, + connectionId, + status: ConnectionSyncJobStatus.IN_PROGRESS, + warningMessages: [], + }, + }); + await tx.connection.update({ + where: { + id: connectionId, + }, + data: { + latestSyncJobId: jobId, + }, + }); + }); + }, + onCompleted: async ({ jobId }) => { + await db.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: ConnectionSyncJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + }, + }); + }, + onTerminalFailure: async ({ jobId }, error) => { + await db.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: ConnectionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + }); + }, +}); + +export interface CurrentRepo { + id: number; + name: string; + indexedAt: Date | null; +} + +export interface ConnectionRepoChanges { + currentRepos: CurrentRepo[]; + unindexedRepos: { id: number; name: string }[]; + orphanedRepos: { id: number; name: string }[]; + affectedRepoIds: number[]; +} + +type Trigger = ProcessContext<"connection-sync">["trigger"]; + +const deduplicateRepos = (repos: RepoData[]): RepoData[] => + repos.filter( + (repo, index, allRepos) => + index === + allRepos.findIndex( + (candidate) => + candidate.external_id === repo.external_id && + candidate.external_codeHostUrl === + repo.external_codeHostUrl, + ), + ); + +export const persistConnectionRepositories = async ({ + db, + connectionId, + orgId, + discoveredRepos, +}: { + db: PrismaClient; + connectionId: number; + orgId: number; + discoveredRepos: RepoData[]; +}): Promise => { + const previouslyAssociatedRepos = await db.repo.findMany({ + where: { + connections: { + some: { + connectionId, + }, + }, + }, + select: { + id: true, + }, + }); + + const currentRepos: CurrentRepo[] = []; + for (const repo of deduplicateRepos(discoveredRepos)) { + currentRepos.push( + await db.repo.upsert({ + where: { + external_id_external_codeHostUrl_orgId: { + external_id: repo.external_id, + external_codeHostUrl: repo.external_codeHostUrl, + orgId, + }, + }, + update: { + ...repo, + connections: { + createMany: { + data: { + connectionId, + }, + skipDuplicates: true, + }, + }, + }, + create: repo, + select: { + id: true, + name: true, + indexedAt: true, + }, + }), + ); + } + + const currentRepoIds = new Set(currentRepos.map(({ id }) => id)); + const staleRepoIds = previouslyAssociatedRepos + .map(({ id }) => id) + .filter((id) => !currentRepoIds.has(id)); + + if (staleRepoIds.length > 0) { + await db.repoToConnection.deleteMany({ + where: { + connectionId, + repoId: { + in: staleRepoIds, + }, + }, + }); + } + + const orphanedRepos = + staleRepoIds.length > 0 + ? await db.repo.findMany({ + where: { + id: { + in: staleRepoIds, + }, + connections: { + none: {}, + }, + }, + select: { + id: true, + name: true, + }, + }) + : []; + + return { + currentRepos, + unindexedRepos: currentRepos + .filter(({ indexedAt }) => indexedAt === null) + .map(({ id, name }) => ({ id, name })), + orphanedRepos, + affectedRepoIds: [...new Set([...currentRepoIds, ...staleRepoIds])], + }; +}; + +export const reconcileRepoIndexWork = async ({ + jobManager, + trigger, + currentRepos, + unindexedRepos, + orphanedRepos, + intervalMs, +}: { + jobManager: JobManager; + trigger: Trigger; + currentRepos: CurrentRepo[]; + unindexedRepos: { id: number; name: string }[]; + orphanedRepos: { id: number; name: string }[]; + intervalMs: number; +}): Promise => { + await Promise.all( + currentRepos.map(({ id }) => + jobManager.upsertJobScheduler( + "repo-index", + `repo-index-v1-${id}`, + intervalMs, + { repoId: id, type: "INDEX" }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ), + ), + ); + + await Promise.all( + orphanedRepos.map(({ id }) => + jobManager.removeJobScheduler( + "repo-index", + `repo-index-v1-${id}`, + ), + ), + ); + + await Promise.all( + orphanedRepos.map(({ id }) => + trigger( + "repo-index", + { + repoId: id, + type: "CLEANUP", + }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ), + ), + ); + + await Promise.all( + unindexedRepos.map(({ id }) => + trigger( + "repo-index", + { + repoId: id, + type: "INDEX", + }, + { priority: JOB_PRIORITIES.INITIAL }, + ), + ), + ); +}; + +export const reconcileRepoPermissionSyncWork = async ({ + db, + jobManager, + trigger, + enabled, + affectedRepoIds, + intervalMs, +}: { + db: PrismaClient; + jobManager: JobManager; + trigger: Trigger; + enabled: boolean; + affectedRepoIds: number[]; + intervalMs: number; +}): Promise => { + const [eligibleRepos, existingSchedulerIds] = + enabled && affectedRepoIds.length > 0 + ? await Promise.all([ + db.repo.findMany({ + where: { + id: { + in: affectedRepoIds, + }, + ...REPO_PERMISSION_SYNC_WHERE, + }, + select: { + id: true, + permissionSyncedAt: true, + }, + }), + jobManager.getJobSchedulerIds("repo-permission-sync"), + ]) + : [[], []]; + const existingSchedulerIdSet = new Set(existingSchedulerIds); + const eligibleRepoIds = new Set(eligibleRepos.map(({ id }) => id)); + const ineligibleRepoIds = affectedRepoIds.filter( + (id) => !eligibleRepoIds.has(id), + ); + + await Promise.all( + eligibleRepos.map(({ id }) => + jobManager.upsertJobScheduler( + "repo-permission-sync", + `repo-permission-sync-v1-${id}`, + intervalMs, + { repoId: id }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ), + ), + ); + + await Promise.all( + ineligibleRepoIds.map((id) => + jobManager.removeJobScheduler( + "repo-permission-sync", + `repo-permission-sync-v1-${id}`, + ), + ), + ); + + await Promise.all( + eligibleRepos + .filter( + ({ id, permissionSyncedAt }) => + permissionSyncedAt === null || + !existingSchedulerIdSet.has( + `repo-permission-sync-v1-${id}`, + ), + ) + .map(({ id }) => + trigger( + "repo-permission-sync", + { repoId: id }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ), + ), + ); +}; + +const discoverConnectionRepositories = async ({ + config, + connectionId, + signal, +}: { + config: ConnectionConfig; + connectionId: number; + signal: AbortSignal; +}) => { + switch (config.type) { + case "github": { + return compileGithubConfig(config, connectionId, signal); + } + case "gitlab": { + return compileGitlabConfig(config, connectionId); + } + case "gitea": { + return compileGiteaConfig(config, connectionId); + } + case "gerrit": { + return compileGerritConfig(config, connectionId); + } + case "bitbucket": { + return compileBitbucketConfig(config, connectionId); + } + case "azuredevops": { + return compileAzureDevOpsConfig(config, connectionId); + } + case "git": { + return compileGenericGitHostConfig(config, connectionId); + } + } +}; diff --git a/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts b/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts new file mode 100644 index 000000000..aedd1e590 --- /dev/null +++ b/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts @@ -0,0 +1,483 @@ +import type { PrismaClient } from "@sourcebot/db"; +import type { JobLogger } from "@sourcebot/shared"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + captureException: vi.fn(), + createBitbucketCloudClient: vi.fn(), + createBitbucketServerClient: vi.fn(), + ensureFreshAccountToken: vi.fn(), + getIdentityProviderConfig: vi.fn(), + getReposForAuthenticatedBitbucketCloudUser: vi.fn(), + getReposForAuthenticatedBitbucketServerUser: vi.fn(), + hasEntitlement: vi.fn(), +})); + +vi.mock("@sentry/node", () => ({ + captureException: mocks.captureException, +})); + +vi.mock("@sourcebot/shared", async (importOriginal) => ({ + ...(await importOriginal()), + ACCOUNT_PERMISSION_SYNC_QUEUE: { + name: "account-permission-sync", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keepJobs: { + completed: { count: 50 }, + failed: { count: 50 }, + }, + keepLogs: 500, + }, + }, + getIdentityProviderConfig: mocks.getIdentityProviderConfig, +})); + +vi.mock("../entitlements.js", () => ({ + hasEntitlement: mocks.hasEntitlement, +})); + +vi.mock("../bitbucket.js", () => ({ + createBitbucketCloudClient: mocks.createBitbucketCloudClient, + createBitbucketServerClient: mocks.createBitbucketServerClient, + getReposForAuthenticatedBitbucketCloudUser: + mocks.getReposForAuthenticatedBitbucketCloudUser, + getReposForAuthenticatedBitbucketServerUser: + mocks.getReposForAuthenticatedBitbucketServerUser, +})); + +vi.mock("./tokenRefresh.js", async (importOriginal) => ({ + ...(await importOriginal()), + ensureFreshAccountToken: mocks.ensureFreshAccountToken, +})); + +import { + classifyPermissionSyncFailure, + createAccountPermissionSyncWorkload, +} from "./accountPermissionSyncWorkload.js"; +import { + PermissionSyncUpstreamError, + type PermissionSyncUpstreamErrorKind, +} from "./permissionSyncError.js"; +import { + TokenRefreshError, + type TokenRefreshErrorKind, +} from "./tokenRefresh.js"; + +const tokenRefreshError = ( + kind: TokenRefreshErrorKind, + status?: number, +): TokenRefreshError => + new TokenRefreshError(`Token refresh failed: ${kind}`, { + kind, + status, + }); + +const upstreamError = ( + kind: PermissionSyncUpstreamErrorKind, +): PermissionSyncUpstreamError => + new PermissionSyncUpstreamError(`Permission sync failed: ${kind}`, { + kind, + provider: "github", + operation: "list_accessible_repositories", + }); + +const account = { + id: "account_1", + providerId: "bitbucket-server", + issuerUrl: "https://bitbucket.example.com", + user: { email: "user@example.com" }, +}; +const accountFindUniqueOrThrow = vi.fn().mockResolvedValue(account); +const accountUpdate = vi.fn().mockResolvedValue(account); +const accountUpdateMany = vi.fn().mockResolvedValue({ count: 1 }); +const repoFindMany = vi.fn().mockResolvedValue([]); +const permissionCreateMany = vi.fn().mockResolvedValue({ count: 0 }); +const permissionDeleteMany = vi.fn().mockResolvedValue({ count: 95 }); +const permissionSyncJobUpsert = vi.fn(); +const permissionSyncJobUpdate = vi.fn().mockResolvedValue({ account }); +const transactionClient = { + account: { + update: accountUpdate, + updateMany: accountUpdateMany, + }, + accountPermissionSyncJob: { + upsert: permissionSyncJobUpsert, + update: permissionSyncJobUpdate, + }, +}; +const transaction = vi.fn( + ( + queriesOrCallback: + | Array> + | ((tx: typeof transactionClient) => Promise), + ) => + typeof queriesOrCallback === "function" + ? queriesOrCallback(transactionClient) + : Promise.all(queriesOrCallback), +); + +const db = { + account: { + findUniqueOrThrow: accountFindUniqueOrThrow, + update: accountUpdate, + updateMany: accountUpdateMany, + }, + accountToRepoPermission: { + createMany: permissionCreateMany, + deleteMany: permissionDeleteMany, + }, + repo: { + findMany: repoFindMany, + }, + accountPermissionSyncJob: { + upsert: permissionSyncJobUpsert, + update: permissionSyncJobUpdate, + }, + $transaction: transaction, +} as unknown as PrismaClient; + +const jobLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + flush: vi.fn(), +} satisfies JobLogger; + +const createWorkload = () => + createAccountPermissionSyncWorkload({ + db, + settings: { + maxAccountPermissionSyncJobConcurrency: 2, + } as never, + }); + +const lifecycleContext = { + data: { + accountId: "account_1", + }, + jobId: "job_1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + logger: jobLogger, +}; + +const processContext = { + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), +}; + +beforeEach(() => { + vi.clearAllMocks(); + mocks.hasEntitlement.mockResolvedValue(true); + mocks.ensureFreshAccountToken.mockReset().mockResolvedValue("access-token"); + mocks.getIdentityProviderConfig.mockReset().mockResolvedValue({ + provider: "bitbucket-server", + baseUrl: "https://bitbucket.example.com", + }); + mocks.createBitbucketServerClient.mockReset().mockReturnValue({}); + mocks.getReposForAuthenticatedBitbucketServerUser + .mockReset() + .mockResolvedValue([]); + accountFindUniqueOrThrow.mockResolvedValue(account); + repoFindMany.mockResolvedValue([]); + permissionCreateMany.mockResolvedValue({ count: 0 }); + permissionDeleteMany.mockResolvedValue({ count: 95 }); + permissionSyncJobUpdate.mockResolvedValue({ account }); + accountUpdateMany.mockResolvedValue({ count: 1 }); +}); + +describe("classifyPermissionSyncFailure", () => { + test("fails closed when the refresh token is rejected", () => { + expect( + classifyPermissionSyncFailure( + tokenRefreshError("refresh_token_rejected", 400), + ), + ).toEqual({ + action: "clear_permissions", + reason: "oauth_refresh_token_rejected", + }); + }); + + test.each([ + ["transient", 500], + ["configuration", 400], + ["invalid_response", undefined], + ["local_credential", undefined], + ] satisfies Array<[TokenRefreshErrorKind, number | undefined]>)( + "keeps permissions for a %s token refresh failure", + (kind, status) => { + expect( + classifyPermissionSyncFailure(tokenRefreshError(kind, status)), + ).toEqual({ + action: "preserve_permissions", + }); + }, + ); + + test("does not treat a token refresh configuration error with HTTP 401 as an API authorization failure", () => { + expect( + classifyPermissionSyncFailure( + tokenRefreshError("configuration", 401), + ), + ).toEqual({ + action: "preserve_permissions", + }); + }); + + test.each([ + ["credential_rejected", "upstream_credential_rejected"], + ["insufficient_scope", "upstream_insufficient_scope"], + ] as const)( + "fails closed for a classified %s upstream failure", + (kind, reason) => { + expect(classifyPermissionSyncFailure(upstreamError(kind))).toEqual({ + action: "clear_permissions", + reason, + }); + }, + ); + + test.each([ + "rate_limited", + "upstream_unavailable", + "forbidden", + "unknown", + ] satisfies PermissionSyncUpstreamErrorKind[])( + "keeps permissions for a classified %s upstream failure", + (kind) => { + expect(classifyPermissionSyncFailure(upstreamError(kind))).toEqual({ + action: "preserve_permissions", + }); + }, + ); + + test.each([401, 403, 410])( + "does not fail closed on an unclassified HTTP %s error", + (status) => { + const error = Object.assign(new Error(`HTTP ${status}`), { + status, + }); + expect(classifyPermissionSyncFailure(error)).toEqual({ + action: "preserve_permissions", + }); + }, + ); +}); + +describe("accountPermissionSyncWorkload", () => { + test("uses the configured concurrency and database-backed lifecycle hooks", () => { + const workload = createWorkload(); + + expect(workload.queueSpec.name).toBe("account-permission-sync"); + expect(workload.concurrency).toBe(2); + expect(workload.onStarted).toBeTypeOf("function"); + expect(workload.onCompleted).toBeTypeOf("function"); + expect(workload.onTerminalFailure).toBeTypeOf("function"); + }); + + test("uses a distinct execution lock for each account", () => { + const workload = createWorkload(); + + expect(workload.executionLock).toBeDefined(); + expect( + workload.executionLock?.resource({ accountId: "account_1" }), + ).toBe("sourcebot:lock:account:account_1"); + expect( + workload.executionLock?.resource({ accountId: "account_2" }), + ).toBe("sourcebot:lock:account:account_2"); + expect(workload.executionLock?.durationMs).toBe(60_000); + }); + + test("does not start syncing when execution has already been aborted", async () => { + const controller = new AbortController(); + controller.abort(new Error("Account execution lock was lost")); + + await expect( + createWorkload().process({ + ...processContext, + signal: controller.signal, + }), + ).rejects.toThrow("Account execution lock was lost"); + expect(mocks.hasEntitlement).not.toHaveBeenCalled(); + expect(accountFindUniqueOrThrow).not.toHaveBeenCalled(); + }); + + test("syncs the requested account", async () => { + const workload = createWorkload(); + + await workload.process(processContext); + + expect(accountFindUniqueOrThrow).toHaveBeenCalledWith({ + where: { id: "account_1" }, + include: { user: true }, + }); + expect(mocks.ensureFreshAccountToken).toHaveBeenCalledWith(account, db); + expect(mocks.createBitbucketServerClient).toHaveBeenCalledWith( + "https://bitbucket.example.com", + undefined, + "access-token", + ); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("does not run without the permission syncing entitlement", async () => { + mocks.hasEntitlement.mockResolvedValue(false); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toThrow( + "Permission syncing entitlement is not currently available.", + ); + + expect(accountFindUniqueOrThrow).not.toHaveBeenCalled(); + }); + + test("atomically records a reauthentication issue when the refresh token is rejected", async () => { + const error = tokenRefreshError("refresh_token_rejected", 400); + mocks.ensureFreshAccountToken.mockRejectedValue(error); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toBe(error); + + expect(permissionDeleteMany).toHaveBeenCalledWith({ + where: { accountId: "account_1" }, + }); + expect(accountUpdate).toHaveBeenCalledWith({ + where: { id: "account_1" }, + data: { + permissionSyncIssue: "REAUTHENTICATION_REQUIRED", + permissionSyncIssueAt: expect.any(Date), + }, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("records an issue even when permissions were cleared by an earlier attempt", async () => { + const error = tokenRefreshError("refresh_token_rejected", 400); + permissionDeleteMany.mockResolvedValue({ count: 0 }); + mocks.ensureFreshAccountToken.mockRejectedValue(error); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toBe(error); + + expect(accountUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + permissionSyncIssue: "REAUTHENTICATION_REQUIRED", + }), + }), + ); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("records an insufficient-scope issue for scope failures", async () => { + const error = upstreamError("insufficient_scope"); + mocks.getReposForAuthenticatedBitbucketServerUser.mockRejectedValue( + error, + ); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toBe(error); + + expect(accountUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + permissionSyncIssue: "INSUFFICIENT_SCOPE", + }), + }), + ); + }); + + test("preserves permissions for a transient refresh failure", async () => { + const error = tokenRefreshError("transient", 500); + mocks.ensureFreshAccountToken.mockRejectedValue(error); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toBe(error); + + expect(permissionDeleteMany).not.toHaveBeenCalled(); + expect(accountUpdate).not.toHaveBeenCalled(); + expect(transaction).not.toHaveBeenCalled(); + }); + + test("marks a job as in progress when started", async () => { + await createWorkload().onStarted?.(lifecycleContext); + + expect(permissionSyncJobUpsert).toHaveBeenCalledWith({ + where: { id: "job_1" }, + update: { + status: "IN_PROGRESS", + completedAt: null, + errorMessage: null, + }, + create: { + id: "job_1", + accountId: "account_1", + status: "IN_PROGRESS", + }, + }); + expect(accountUpdate).toHaveBeenCalledWith({ + where: { id: "account_1" }, + data: { latestPermissionSyncJobId: "job_1" }, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("marks a job completed and clears the account issue when it is still latest", async () => { + await createWorkload().onCompleted?.(lifecycleContext, undefined); + + expect(permissionSyncJobUpdate).toHaveBeenCalledWith({ + where: { id: "job_1" }, + data: { + status: "COMPLETED", + completedAt: expect.any(Date), + errorMessage: null, + }, + select: { + account: { + include: { user: true }, + }, + }, + }); + expect(accountUpdateMany).toHaveBeenCalledWith({ + where: { + id: "account_1", + latestPermissionSyncJobId: "job_1", + }, + data: { + permissionSyncedAt: expect.any(Date), + permissionSyncIssue: null, + permissionSyncIssueAt: null, + }, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("marks a job failed after terminal failure", async () => { + const error = new Error("Upstream unavailable"); + + await createWorkload().onTerminalFailure?.(lifecycleContext, error); + + expect(permissionSyncJobUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: "job_1" }, + data: { + status: "FAILED", + completedAt: expect.any(Date), + errorMessage: "Upstream unavailable", + }, + }), + ); + expect(mocks.captureException).toHaveBeenCalledWith(error, { + tags: { + jobId: "job_1", + queue: "account-permission-sync", + }, + }); + }); +}); diff --git a/packages/backend/src/ee/accountPermissionSyncWorkload.ts b/packages/backend/src/ee/accountPermissionSyncWorkload.ts new file mode 100644 index 000000000..a6bcc5573 --- /dev/null +++ b/packages/backend/src/ee/accountPermissionSyncWorkload.ts @@ -0,0 +1,567 @@ +import * as Sentry from "@sentry/node"; +import { + Account, + AccountPermissionSyncIssue, + AccountPermissionSyncJobStatus, + PermissionSyncSource, + PrismaClient, +} from "@sourcebot/db"; +import { + ACCOUNT_PERMISSION_SYNC_QUEUE, + getIdentityProviderConfig, + JobLogSink, +} from "@sourcebot/shared"; +import { + createBitbucketCloudClient, + createBitbucketServerClient, + getReposForAuthenticatedBitbucketCloudUser, + getReposForAuthenticatedBitbucketServerUser, +} from "../bitbucket.js"; +import { hasEntitlement } from "../entitlements.js"; +import { + createOctokitFromToken, + getOAuthScopesForAuthenticatedUser as getGitHubOAuthScopesForAuthenticatedUser, + getReposForAuthenticatedUser, +} from "../github.js"; +import { + createGitLabFromOAuthToken, + getOAuthScopesForAuthenticatedUser as getGitLabOAuthScopesForAuthenticatedUser, + getProjectsForAuthenticatedUser, +} from "../gitlab.js"; +import { + PermissionSyncUpstreamError, + withPermissionSyncUpstreamError, +} from "./permissionSyncError.js"; +import { ensureFreshAccountToken, TokenRefreshError } from "./tokenRefresh.js"; +import { Settings, Workload } from "../types.js"; +import { IdentityProviderConfig } from "@sourcebot/schemas/v3/index.type"; + +type AccountWithUser = Account & { user: { email: string | null } }; + +type SupportedProvider = + | "github" + | "gitlab" + | "bitbucket-cloud" + | "bitbucket-server"; +type ProviderConfig = Extract< + IdentityProviderConfig, + { provider: TProvider } +>; + +const ACCOUNT_PERMISSION_SYNC_LOCK_DURATION_MS = 60_000; + +interface ProviderPermissionSyncProps { + db: PrismaClient; + account: AccountWithUser; + accessToken: string; + config: ProviderConfig; +} + +export type PermissionCleanupReason = + | "oauth_refresh_token_rejected" + | "upstream_credential_rejected" + | "upstream_insufficient_scope"; + +export type PermissionCleanupDecision = + | { + action: "clear_permissions"; + reason: PermissionCleanupReason; + } + | { + action: "preserve_permissions"; + }; + +export const classifyPermissionSyncFailure = ( + error: unknown, +): PermissionCleanupDecision => { + // Token refresh failures have their own classification. Do not fall through + // to the generic HTTP checks because another token endpoint failure may + // also carry a 401 or 403 status. + if (error instanceof TokenRefreshError) { + return error.kind === "refresh_token_rejected" + ? { + action: "clear_permissions", + reason: "oauth_refresh_token_rejected", + } + : { action: "preserve_permissions" }; + } + + if (error instanceof PermissionSyncUpstreamError) { + if (error.kind === "credential_rejected") { + return { + action: "clear_permissions", + reason: "upstream_credential_rejected", + }; + } + if (error.kind === "insufficient_scope") { + return { + action: "clear_permissions", + reason: "upstream_insufficient_scope", + }; + } + } + + return { action: "preserve_permissions" }; +}; + +const PERMISSION_CLEANUP_DETAILS: Record< + PermissionCleanupReason, + { + message: string; + issue: AccountPermissionSyncIssue; + } +> = { + oauth_refresh_token_rejected: { + message: "OAuth refresh token rejection", + issue: AccountPermissionSyncIssue.REAUTHENTICATION_REQUIRED, + }, + upstream_credential_rejected: { + message: "upstream credential rejection", + issue: AccountPermissionSyncIssue.REAUTHENTICATION_REQUIRED, + }, + upstream_insufficient_scope: { + message: "insufficient OAuth scope", + issue: AccountPermissionSyncIssue.INSUFFICIENT_SCOPE, + }, +}; + +interface AccountPermissionSyncWorkloadDependencies { + db: PrismaClient; + settings: Settings; +} + +export const createAccountPermissionSyncWorkload = ({ + db, + settings, +}: AccountPermissionSyncWorkloadDependencies): Workload<"account-permission-sync"> => { + return { + queueSpec: ACCOUNT_PERMISSION_SYNC_QUEUE, + concurrency: settings.maxAccountPermissionSyncJobConcurrency, + executionLock: { + resource: ({ accountId }) => + `sourcebot:lock:account:${accountId}`, + durationMs: ACCOUNT_PERMISSION_SYNC_LOCK_DURATION_MS, + }, + process: async ({ + data: { accountId }, + logger: jobLogger, + signal, + }) => { + signal.throwIfAborted(); + if (!(await hasEntitlement("permission-syncing"))) { + throw new Error( + "Permission syncing entitlement is not currently available.", + ); + } + + signal.throwIfAborted(); + const account = await db.account.findUniqueOrThrow({ + where: { + id: accountId, + }, + include: { + user: true, + }, + }); + signal.throwIfAborted(); + + jobLogger.debug( + `Syncing permissions for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}...`, + ); + + try { + // Ensure the OAuth token is fresh, refreshing it if it is expired or near expiry. + const accessToken = await ensureFreshAccountToken(account, db); + signal.throwIfAborted(); + + const idpConfig = await getIdentityProviderConfig( + account.providerId, + ); + signal.throwIfAborted(); + if (!idpConfig) { + throw new Error( + "Unable to find IDP config in config.json.", + ); + } + + const repoIds = await getAccessibleRepoIds({ + db, + account, + accessToken, + config: idpConfig, + }); + + signal.throwIfAborted(); + await db.$transaction([ + db.account.update({ + where: { + id: account.id, + }, + data: { + accessibleRepos: { + deleteMany: {}, + }, + }, + }), + db.accountToRepoPermission.createMany({ + data: repoIds.map((repoId) => ({ + accountId: account.id, + repoId, + source: PermissionSyncSource.ACCOUNT_DRIVEN, + })), + skipDuplicates: true, + }), + ]); + signal.throwIfAborted(); + } catch (error) { + signal.throwIfAborted(); + // Clear cached permissions only for classified permanent failures. + // Ambiguous HTTP errors and transient upstream failures preserve the + // last successful permission state. + const cleanupDecision = classifyPermissionSyncFailure(error); + + if (cleanupDecision.action === "clear_permissions") { + const details = + PERMISSION_CLEANUP_DETAILS[cleanupDecision.reason]; + signal.throwIfAborted(); + const [{ count }] = await db.$transaction([ + db.accountToRepoPermission.deleteMany({ + where: { accountId: account.id }, + }), + db.account.update({ + where: { id: account.id }, + data: { + permissionSyncIssue: details.issue, + permissionSyncIssueAt: new Date(), + }, + }), + ]); + signal.throwIfAborted(); + const message = + error instanceof Error ? error.message : String(error); + jobLogger.warn( + `Cleared ${count} permission row(s) for account ${account.id} (user ${account.user.email ?? "unknown"}) — fail-closed cleanup triggered by ${details.message}: ${message}`, + ); + } + throw error; + } + }, + onStarted: async ({ data: { accountId }, jobId }) => { + await db.$transaction(async (tx) => { + await tx.accountPermissionSyncJob.upsert({ + where: { + id: jobId, + }, + update: { + status: AccountPermissionSyncJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + }, + create: { + id: jobId, + accountId, + status: AccountPermissionSyncJobStatus.IN_PROGRESS, + }, + }); + await tx.account.update({ + where: { + id: accountId, + }, + data: { + latestPermissionSyncJobId: jobId, + }, + }); + }); + }, + onCompleted: async ({ + data: { accountId }, + jobId, + logger: jobLogger, + }) => { + const account = await db.$transaction(async (tx) => { + const { account } = await tx.accountPermissionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: AccountPermissionSyncJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + }, + select: { + account: { + include: { + user: true, + }, + }, + }, + }); + await tx.account.updateMany({ + where: { + id: accountId, + latestPermissionSyncJobId: jobId, + }, + data: { + permissionSyncedAt: new Date(), + permissionSyncIssue: null, + permissionSyncIssueAt: null, + }, + }); + return account; + }); + + jobLogger.debug( + `Permissions synced for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}`, + ); + }, + onTerminalFailure: async ( + { data: { accountId }, jobId, logger: jobLogger }, + error, + ) => { + Sentry.captureException(error, { + tags: { + jobId, + queue: ACCOUNT_PERMISSION_SYNC_QUEUE.name, + }, + }); + + const { account } = await db.accountPermissionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: AccountPermissionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + select: { + account: { + include: { + user: true, + }, + }, + }, + }); + + jobLogger.error( + `Account permission sync job failed for account (id: ${accountId}) for user ${account.user.email ?? "unknown user (email not found)"}: ${error.message}`, + ); + }, + }; +}; + +const getAccessibleRepoIds = async ({ + db, + account, + accessToken, + config, +}: { + db: PrismaClient; + account: AccountWithUser; + accessToken: string; + config: IdentityProviderConfig; +}): Promise => { + switch (config.provider) { + case "github": + return getGitHubAccessibleRepoIds({ + db, + account, + accessToken, + config, + }); + case "gitlab": + return getGitLabAccessibleRepoIds({ + db, + account, + accessToken, + config, + }); + case "bitbucket-cloud": + return getBitbucketCloudAccessibleRepoIds({ + db, + account, + accessToken, + config, + }); + case "bitbucket-server": + return getBitbucketServerAccessibleRepoIds({ + db, + account, + accessToken, + config, + }); + default: + throw new Error(`Unsupported provider type: ${config.provider}`); + } +}; + +const getGitHubAccessibleRepoIds = async ({ + db, + account, + accessToken, + config, +}: ProviderPermissionSyncProps<"github">): Promise => { + const { octokit } = await createOctokitFromToken({ + token: accessToken, + url: config.baseUrl, + }); + + const scopes = await withPermissionSyncUpstreamError( + "github", + "inspect_token_scopes", + () => getGitHubOAuthScopesForAuthenticatedUser(octokit, accessToken), + ); + + // Token supports scope introspection (classic PAT or OAuth app token). + if (scopes !== null && !scopes.includes("repo")) { + throw new PermissionSyncUpstreamError( + `OAuth token with scopes [${scopes.join(", ")}] is missing the 'repo' scope required for permission syncing. Please re-authorize with GitHub to grant the required scope.`, + { + kind: "insufficient_scope", + provider: "github", + operation: "inspect_token_scopes", + }, + ); + } + + // Public repos do not need an explicit permission mapping. + const githubRepos = await withPermissionSyncUpstreamError( + "github", + "list_accessible_repositories", + () => getReposForAuthenticatedUser("private", octokit), + ); + const gitHubRepoIds = githubRepos.map((repo) => repo.id.toString()); + + const repos = await db.repo.findMany({ + where: { + external_codeHostType: "github", + external_id: { + in: gitHubRepoIds, + }, + ...(account.issuerUrl + ? { + external_codeHostUrl: account.issuerUrl, + } + : {}), + }, + }); + + return repos.map((repo) => repo.id); +}; + +const getGitLabAccessibleRepoIds = async ({ + db, + account, + accessToken, + config, +}: ProviderPermissionSyncProps<"gitlab">): Promise => { + const api = await createGitLabFromOAuthToken({ + oauthToken: accessToken, + url: config.baseUrl, + }); + + const scopes = await withPermissionSyncUpstreamError( + "gitlab", + "inspect_token_scopes", + () => getGitLabOAuthScopesForAuthenticatedUser(api), + ); + if (!scopes.includes("read_api")) { + throw new PermissionSyncUpstreamError( + `OAuth token with scopes [${scopes.join(", ")}] is missing the 'read_api' scope required for permission syncing.`, + { + kind: "insufficient_scope", + provider: "gitlab", + operation: "inspect_token_scopes", + }, + ); + } + + // Public and internal repos do not need an explicit permission mapping. + const gitLabProjectIds = ( + await withPermissionSyncUpstreamError( + "gitlab", + "list_accessible_repositories", + () => getProjectsForAuthenticatedUser("private", api), + ) + ).map((project) => project.id.toString()); + + const repos = await db.repo.findMany({ + where: { + external_codeHostType: "gitlab", + external_id: { + in: gitLabProjectIds, + }, + ...(account.issuerUrl + ? { + external_codeHostUrl: account.issuerUrl, + } + : {}), + }, + }); + + return repos.map((repo) => repo.id); +}; + +const getBitbucketCloudAccessibleRepoIds = async ({ + db, + account, + accessToken, +}: ProviderPermissionSyncProps<"bitbucket-cloud">): Promise => { + // Use a bearer token by omitting the user. + const client = createBitbucketCloudClient(undefined, accessToken); + const bitbucketRepos = await withPermissionSyncUpstreamError( + "bitbucket-cloud", + "list_accessible_repositories", + () => getReposForAuthenticatedBitbucketCloudUser(client), + ); + const bitbucketRepoUuids = bitbucketRepos.map((repo) => repo.uuid); + + const repos = await db.repo.findMany({ + where: { + external_codeHostType: "bitbucketCloud", + external_id: { + in: bitbucketRepoUuids, + }, + ...(account.issuerUrl + ? { + external_codeHostUrl: account.issuerUrl, + } + : {}), + }, + }); + + return repos.map((repo) => repo.id); +}; + +const getBitbucketServerAccessibleRepoIds = async ({ + db, + account, + accessToken, + config, +}: ProviderPermissionSyncProps<"bitbucket-server">): Promise => { + const client = createBitbucketServerClient( + config.baseUrl, + undefined, + accessToken, + ); + const serverRepos = await withPermissionSyncUpstreamError( + "bitbucket-server", + "list_accessible_repositories", + () => getReposForAuthenticatedBitbucketServerUser(client), + ); + const serverRepoIds = serverRepos.map((repo) => repo.id); + + const repos = await db.repo.findMany({ + where: { + external_codeHostType: "bitbucketServer", + external_id: { in: serverRepoIds }, + ...(account.issuerUrl + ? { + external_codeHostUrl: account.issuerUrl, + } + : {}), + }, + }); + + return repos.map((repo) => repo.id); +}; diff --git a/packages/backend/src/ee/accountPermissionSyncer.test.ts b/packages/backend/src/ee/accountPermissionSyncer.test.ts deleted file mode 100644 index 09b35d594..000000000 --- a/packages/backend/src/ee/accountPermissionSyncer.test.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { beforeEach, describe, expect, test, vi } from 'vitest'; - -const mocks = vi.hoisted(() => ({ - hasEntitlement: vi.fn(), -})); - -vi.mock('../entitlements.js', () => ({ - hasEntitlement: mocks.hasEntitlement, -})); - -import { AccountPermissionSyncer, classifyPermissionSyncFailure } from './accountPermissionSyncer.js'; -import { - PermissionSyncUpstreamError, - type PermissionSyncUpstreamErrorKind, -} from './permissionSyncError.js'; -import { TokenRefreshError, type TokenRefreshErrorKind } from './tokenRefresh.js'; - -const tokenRefreshError = ( - kind: TokenRefreshErrorKind, - status?: number, -): TokenRefreshError => new TokenRefreshError(`Token refresh failed: ${kind}`, { - kind, - status, -}); - -const upstreamError = ( - kind: PermissionSyncUpstreamErrorKind, -): PermissionSyncUpstreamError => new PermissionSyncUpstreamError(`Permission sync failed: ${kind}`, { - kind, - provider: 'github', - operation: 'list_accessible_repositories', -}); - -const createSyncerHarness = (syncError?: Error, permissionCount = 95) => { - const account = { - id: 'account_1', - providerId: 'bitbucket-server', - user: { email: 'user@example.com' }, - }; - const db = { - accountPermissionSyncJob: { - update: vi.fn().mockResolvedValue({ account }), - }, - accountToRepoPermission: { - deleteMany: vi.fn().mockResolvedValue({ count: permissionCount }), - }, - account: { - update: vi.fn().mockResolvedValue(account), - }, - $transaction: vi.fn((queries: Array>) => Promise.all(queries)), - }; - const syncAccountPermissions = syncError - ? vi.fn().mockRejectedValue(syncError) - : vi.fn().mockResolvedValue(undefined); - const syncer = Object.create(AccountPermissionSyncer.prototype) as { - db: typeof db; - syncAccountPermissions: typeof syncAccountPermissions; - runJob(job: { data: { jobId: string } }): Promise; - onJobCompleted(job: { data: { jobId: string } }): Promise; - }; - syncer.db = db; - syncer.syncAccountPermissions = syncAccountPermissions; - - return { - account, - db, - job: { data: { jobId: 'job_1' } }, - syncer, - }; -}; - -beforeEach(() => { - vi.clearAllMocks(); - mocks.hasEntitlement.mockResolvedValue(true); -}); - -describe('classifyPermissionSyncFailure', () => { - test('fails closed when the refresh token is rejected', () => { - expect(classifyPermissionSyncFailure(tokenRefreshError('refresh_token_rejected', 400))).toEqual({ - action: 'clear_permissions', - reason: 'oauth_refresh_token_rejected', - }); - }); - - test.each([ - ['transient', 500], - ['configuration', 400], - ['invalid_response', undefined], - ['local_credential', undefined], - ] satisfies Array<[TokenRefreshErrorKind, number | undefined]>)('keeps permissions for a %s token refresh failure', (kind, status) => { - expect(classifyPermissionSyncFailure(tokenRefreshError(kind, status))).toEqual({ - action: 'preserve_permissions', - }); - }); - - test('does not treat a token refresh configuration error with HTTP 401 as an API authorization failure', () => { - expect(classifyPermissionSyncFailure(tokenRefreshError('configuration', 401))).toEqual({ - action: 'preserve_permissions', - }); - }); - - test.each([ - ['credential_rejected', 'upstream_credential_rejected'], - ['insufficient_scope', 'upstream_insufficient_scope'], - ] as const)('fails closed for a classified %s upstream failure', (kind, reason) => { - expect(classifyPermissionSyncFailure(upstreamError(kind))).toEqual({ - action: 'clear_permissions', - reason, - }); - }); - - test.each([ - 'rate_limited', - 'upstream_unavailable', - 'forbidden', - 'unknown', - ] satisfies PermissionSyncUpstreamErrorKind[])('keeps permissions for a classified %s upstream failure', (kind) => { - expect(classifyPermissionSyncFailure(upstreamError(kind))).toEqual({ - action: 'preserve_permissions', - }); - }); - - test.each([401, 403, 410])('does not fail closed on an unclassified HTTP %s error', (status) => { - const error = Object.assign(new Error(`HTTP ${status}`), { status }); - expect(classifyPermissionSyncFailure(error)).toEqual({ - action: 'preserve_permissions', - }); - }); -}); - -describe('permission sync issue lifecycle', () => { - test('atomically records a reauthentication issue when the refresh token is rejected', async () => { - const error = tokenRefreshError('refresh_token_rejected', 400); - const { db, job, syncer } = createSyncerHarness(error); - - await expect(syncer.runJob(job)).rejects.toBe(error); - - expect(db.accountToRepoPermission.deleteMany).toHaveBeenCalledWith({ - where: { accountId: 'account_1' }, - }); - expect(db.account.update).toHaveBeenCalledWith({ - where: { id: 'account_1' }, - data: { - permissionSyncIssue: 'REAUTHENTICATION_REQUIRED', - permissionSyncIssueAt: expect.any(Date), - }, - }); - expect(db.$transaction).toHaveBeenCalledOnce(); - }); - - test('records an issue when permissions were already cleared by an earlier attempt', async () => { - const error = tokenRefreshError('refresh_token_rejected', 400); - const { db, job, syncer } = createSyncerHarness(error, 0); - - await expect(syncer.runJob(job)).rejects.toBe(error); - - expect(db.account.update).toHaveBeenCalledWith(expect.objectContaining({ - data: expect.objectContaining({ - permissionSyncIssue: 'REAUTHENTICATION_REQUIRED', - }), - })); - expect(db.$transaction).toHaveBeenCalledOnce(); - }); - - test('records an insufficient-scope issue for scope failures', async () => { - const error = upstreamError('insufficient_scope'); - const { db, job, syncer } = createSyncerHarness(error); - - await expect(syncer.runJob(job)).rejects.toBe(error); - - expect(db.account.update).toHaveBeenCalledWith(expect.objectContaining({ - data: expect.objectContaining({ - permissionSyncIssue: 'INSUFFICIENT_SCOPE', - }), - })); - }); - - test('does not record an issue or clear permissions for a transient refresh failure', async () => { - const error = tokenRefreshError('transient', 500); - const { db, job, syncer } = createSyncerHarness(error); - - await expect(syncer.runJob(job)).rejects.toBe(error); - - expect(db.accountToRepoPermission.deleteMany).not.toHaveBeenCalled(); - expect(db.account.update).not.toHaveBeenCalled(); - expect(db.$transaction).not.toHaveBeenCalled(); - }); - - test('clears the action-required issue after a successful permission sync', async () => { - const { db, job, syncer } = createSyncerHarness(); - - await syncer.onJobCompleted(job); - - expect(db.accountPermissionSyncJob.update).toHaveBeenCalledWith(expect.objectContaining({ - data: expect.objectContaining({ - account: { - update: expect.objectContaining({ - permissionSyncIssue: null, - permissionSyncIssueAt: null, - }), - }, - }), - })); - }); -}); diff --git a/packages/backend/src/ee/accountPermissionSyncer.ts b/packages/backend/src/ee/accountPermissionSyncer.ts deleted file mode 100644 index 7184a191e..000000000 --- a/packages/backend/src/ee/accountPermissionSyncer.ts +++ /dev/null @@ -1,535 +0,0 @@ -import * as Sentry from "@sentry/node"; -import { - PrismaClient, - AccountPermissionSyncIssue, - AccountPermissionSyncJobStatus, - Account, - PermissionSyncSource, -} from "@sourcebot/db"; -import { env, createLogger, getIdentityProviderConfig, PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS } from "@sourcebot/shared"; -import { hasEntitlement } from "../entitlements.js"; -import { ensureFreshAccountToken, TokenRefreshError } from "./tokenRefresh.js"; -import { DelayedError, Job, Queue, Worker } from "bullmq"; -import { Redis } from "ioredis"; -import { - createOctokitFromToken, - getOAuthScopesForAuthenticatedUser as getGitHubOAuthScopesForAuthenticatedUser, - getReposForAuthenticatedUser, -} from "../github.js"; -import { - createGitLabFromOAuthToken, - getOAuthScopesForAuthenticatedUser as getGitLabOAuthScopesForAuthenticatedUser, - getProjectsForAuthenticatedUser, -} from "../gitlab.js"; -import { createBitbucketCloudClient, createBitbucketServerClient, getReposForAuthenticatedBitbucketCloudUser, getReposForAuthenticatedBitbucketServerUser } from "../bitbucket.js"; -import { Settings } from "../types.js"; -import { setIntervalAsync } from "../utils.js"; -import { PermissionSyncUpstreamError, withPermissionSyncUpstreamError } from "./permissionSyncError.js"; - -const LOG_TAG = 'user-permission-syncer'; -const logger = createLogger(LOG_TAG); -const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); - -const QUEUE_NAME = 'accountPermissionSyncQueue'; -const POLLING_INTERVAL_MS = 1000; -const ENTITLEMENT_RETRY_DELAY_MS = 30 * 1000; - -type AccountPermissionSyncJob = { - jobId: string; -} - -export type PermissionCleanupReason = - | 'oauth_refresh_token_rejected' - | 'upstream_credential_rejected' - | 'upstream_insufficient_scope'; - -export type PermissionCleanupDecision = - | { - action: 'clear_permissions'; - reason: PermissionCleanupReason; - } - | { - action: 'preserve_permissions'; - }; - -const PERMISSION_CLEANUP_DETAILS: Record = { - oauth_refresh_token_rejected: { - message: 'OAuth refresh token rejection', - issue: AccountPermissionSyncIssue.REAUTHENTICATION_REQUIRED, - }, - upstream_credential_rejected: { - message: 'upstream credential rejection', - issue: AccountPermissionSyncIssue.REAUTHENTICATION_REQUIRED, - }, - upstream_insufficient_scope: { - message: 'insufficient OAuth scope', - issue: AccountPermissionSyncIssue.INSUFFICIENT_SCOPE, - }, -}; - -export const classifyPermissionSyncFailure = (error: unknown): PermissionCleanupDecision => { - // Token refresh failures have their own classification. Do not fall through - // to the generic HTTP checks because another token endpoint failure may - // also carry a 401 or 403 status. - if (error instanceof TokenRefreshError) { - return error.kind === 'refresh_token_rejected' - ? { action: 'clear_permissions', reason: 'oauth_refresh_token_rejected' } - : { action: 'preserve_permissions' }; - } - - if (error instanceof PermissionSyncUpstreamError) { - if (error.kind === 'credential_rejected') { - return { action: 'clear_permissions', reason: 'upstream_credential_rejected' }; - } - if (error.kind === 'insufficient_scope') { - return { action: 'clear_permissions', reason: 'upstream_insufficient_scope' }; - } - } - - return { action: 'preserve_permissions' }; -}; - -export class AccountPermissionSyncer { - private queue: Queue; - private worker: Worker; - private interval?: NodeJS.Timeout; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - ) { - this.queue = new Queue(QUEUE_NAME, { - connection: redis, - }); - this.worker = new Worker(QUEUE_NAME, this.runJob.bind(this), { - connection: redis, - concurrency: this.settings.maxAccountPermissionSyncJobConcurrency, - }); - this.worker.on('completed', this.onJobCompleted.bind(this)); - this.worker.on('failed', this.onJobFailed.bind(this)); - } - - public async startScheduler() { - logger.debug('Starting scheduler'); - - this.interval = setIntervalAsync(async () => { - if (!await hasEntitlement('permission-syncing')) { - return; - } - - const thresholdDate = new Date(Date.now() - this.settings.userDrivenPermissionSyncIntervalMs); - - const accounts = await this.db.account.findMany({ - where: { - AND: [ - { - providerType: { - in: PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS - } - }, - { - OR: [ - { permissionSyncedAt: null }, - { permissionSyncedAt: { lt: thresholdDate } }, - ] - }, - { - NOT: { - permissionSyncJobs: { - some: { - OR: [ - // Don't schedule if there are active jobs - { - status: { - in: [ - AccountPermissionSyncJobStatus.PENDING, - AccountPermissionSyncJobStatus.IN_PROGRESS, - ], - } - }, - // Don't schedule if there are recent failed jobs (within the threshold date). Note `gt` is used here since this is a inverse condition. - { - AND: [ - { status: AccountPermissionSyncJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } - ] - } - } - } - }, - ] - } - }); - - await this.schedulePermissionSync(accounts); - }, POLLING_INTERVAL_MS); - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - await this.worker.close(/* force = */ true); - await this.queue.close(); - } - - public async schedulePermissionSyncForAccount(account: Account) { - const [job] = await this.db.accountPermissionSyncJob.createManyAndReturn({ - data: [{ accountId: account.id }], - }); - - await this.queue.add('accountPermissionSyncJob', { - jobId: job.id, - }, { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - priority: 1, - }); - - return job.id; - } - - private async schedulePermissionSync(accounts: Account[]) { - // @note: we don't perform this in a transaction because - // we want to avoid the situation where a job is created and run - // prior to the transaction being committed. - const jobs = await this.db.accountPermissionSyncJob.createManyAndReturn({ - data: accounts.map(account => ({ - accountId: account.id, - })), - include: { - account: true, - } - }); - - await this.queue.addBulk(jobs.map((job) => ({ - name: 'accountPermissionSyncJob', - data: { - jobId: job.id, - }, - opts: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - // Priority 1 (high) for never-synced, Priority 2 (normal) for re-sync - priority: job.account.permissionSyncedAt === null ? 1 : 2, - } - }))) - } - - private async runJob(job: Job) { - if (!await hasEntitlement('permission-syncing')) { - await job.moveToDelayed(Date.now() + ENTITLEMENT_RETRY_DELAY_MS, job.token); - throw new DelayedError('Permission syncing entitlement is not currently available.'); - } - - const id = job.data.jobId; - const logger = createJobLogger(id); - - const { account } = await this.db.accountPermissionSyncJob.update({ - where: { - id, - }, - data: { - status: AccountPermissionSyncJobStatus.IN_PROGRESS, - }, - select: { - account: { - include: { - user: true, - } - } - } - }); - - try { - await this.syncAccountPermissions(account, logger); - } catch (error) { - // Clear cached permissions only for classified permanent failures. - // Ambiguous HTTP errors and transient upstream failures preserve the - // last successful permission state. - const cleanupDecision = classifyPermissionSyncFailure(error); - - if (cleanupDecision.action === 'clear_permissions') { - const details = PERMISSION_CLEANUP_DETAILS[cleanupDecision.reason]; - const [{ count }] = await this.db.$transaction([ - this.db.accountToRepoPermission.deleteMany({ - where: { accountId: account.id }, - }), - this.db.account.update({ - where: { id: account.id }, - data: { - permissionSyncIssue: details.issue, - permissionSyncIssueAt: new Date(), - }, - }), - ]); - const message = error instanceof Error ? error.message : String(error); - logger.warn(`Cleared ${count} permission row(s) for account ${account.id} (user ${account.user.email ?? 'unknown'}) — fail-closed cleanup triggered by ${details.message}: ${message}`); - } - throw error; - } - } - - private async syncAccountPermissions( - account: Account & { user: { email: string | null } }, - logger: ReturnType, - ) { - logger.debug(`Syncing permissions for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}...`); - - // Ensure the OAuth token is fresh, refreshing it if it is expired or near expiry. - const accessToken = await ensureFreshAccountToken(account, this.db); - - // Get a list of all repos that the user has access to from all connected accounts. - const repoIds = await (async () => { - const aggregatedRepoIds: Set = new Set(); - - const idpConfig = await getIdentityProviderConfig(account.providerId); - - if (!idpConfig) { - throw new Error(`Unable to find IDP config in config.json.`); - } - - if (idpConfig.provider === 'github') { - const { octokit } = await createOctokitFromToken({ - token: accessToken, - url: idpConfig.baseUrl, - }); - - const scopes = await withPermissionSyncUpstreamError( - 'github', - 'inspect_token_scopes', - () => getGitHubOAuthScopesForAuthenticatedUser(octokit, accessToken), - ); - - // Token supports scope introspection (classic PAT or OAuth app token) - if (scopes !== null) { - if (!scopes.includes('repo')) { - throw new PermissionSyncUpstreamError( - `OAuth token with scopes [${scopes.join(', ')}] is missing the 'repo' scope required for permission syncing. Please re-authorize with GitHub to grant the required scope.`, - { - kind: 'insufficient_scope', - provider: 'github', - operation: 'inspect_token_scopes', - }, - ); - } - } - - // @note: we only care about the private repos since we don't need to build a mapping - // for public repos. - // @see: packages/web/src/prisma.ts - const githubRepos = await withPermissionSyncUpstreamError( - 'github', - 'list_accessible_repositories', - () => getReposForAuthenticatedUser(/* visibility = */ 'private', octokit), - ); - const gitHubRepoIds = githubRepos.map(repo => repo.id.toString()); - - const repos = await this.db.repo.findMany({ - where: { - external_codeHostType: 'github', - external_id: { - in: gitHubRepoIds, - }, - ...(account.issuerUrl ? { - external_codeHostUrl: account.issuerUrl, - } : {}), - } - }); - - repos.forEach(repo => aggregatedRepoIds.add(repo.id)); - } else if (idpConfig.provider === 'gitlab') { - const api = await createGitLabFromOAuthToken({ - oauthToken: accessToken, - url: idpConfig.baseUrl, - }); - - const scopes = await withPermissionSyncUpstreamError( - 'gitlab', - 'inspect_token_scopes', - () => getGitLabOAuthScopesForAuthenticatedUser(api), - ); - if (!scopes.includes('read_api')) { - throw new PermissionSyncUpstreamError( - `OAuth token with scopes [${scopes.join(', ')}] is missing the 'read_api' scope required for permission syncing.`, - { - kind: 'insufficient_scope', - provider: 'gitlab', - operation: 'inspect_token_scopes', - }, - ); - } - - // @note: we only care about the private repos since we don't need to build a - // mapping for public or internal repos. Note that internal repos are _not_ - // enforced by permission syncing and therefore we don't need to fetch them - // here. - // - // @see: packages/web/src/prisma.ts - const gitLabProjectIds = ( - await withPermissionSyncUpstreamError( - 'gitlab', - 'list_accessible_repositories', - () => getProjectsForAuthenticatedUser('private', api), - ) - ).map(project => project.id.toString()); - - const repos = await this.db.repo.findMany({ - where: { - external_codeHostType: 'gitlab', - external_id: { - in: gitLabProjectIds, - }, - ...(account.issuerUrl ? { - external_codeHostUrl: account.issuerUrl, - } : {}), - } - }); - - repos.forEach(repo => aggregatedRepoIds.add(repo.id)); - } else if (idpConfig.provider === 'bitbucket-cloud') { - // @note: we don't pass a user here since we want to use a bearer token - // for authentication. - const client = createBitbucketCloudClient(/* user = */ undefined, accessToken) - const bitbucketRepos = await withPermissionSyncUpstreamError( - 'bitbucket-cloud', - 'list_accessible_repositories', - () => getReposForAuthenticatedBitbucketCloudUser(client), - ); - const bitbucketRepoUuids = bitbucketRepos.map(repo => repo.uuid); - - const repos = await this.db.repo.findMany({ - where: { - external_codeHostType: 'bitbucketCloud', - external_id: { - in: bitbucketRepoUuids, - }, - ...(account.issuerUrl ? { - external_codeHostUrl: account.issuerUrl, - } : {}), - } - }); - - repos.forEach(repo => aggregatedRepoIds.add(repo.id)); - } else if (idpConfig.provider === 'bitbucket-server') { - const client = createBitbucketServerClient(idpConfig.baseUrl, /* user = */ undefined, accessToken); - const serverRepos = await withPermissionSyncUpstreamError( - 'bitbucket-server', - 'list_accessible_repositories', - () => getReposForAuthenticatedBitbucketServerUser(client), - ); - const serverRepoIds = serverRepos.map(r => r.id); - - const repos = await this.db.repo.findMany({ - where: { - external_codeHostType: 'bitbucketServer', - external_id: { in: serverRepoIds }, - ...(account.issuerUrl ? { - external_codeHostUrl: account.issuerUrl, - } : {}), - } - }); - - repos.forEach(repo => aggregatedRepoIds.add(repo.id)); - } else { - throw new Error(`Unsupported provider type: ${idpConfig.provider}`); - } - - return Array.from(aggregatedRepoIds); - })(); - - await this.db.$transaction([ - this.db.account.update({ - where: { - id: account.id, - }, - data: { - accessibleRepos: { - deleteMany: {}, - } - } - }), - this.db.accountToRepoPermission.createMany({ - data: repoIds.map(repoId => ({ - accountId: account.id, - repoId, - source: PermissionSyncSource.ACCOUNT_DRIVEN, - })), - skipDuplicates: true, - }) - ]); - } - - private async onJobCompleted(job: Job) { - const logger = createJobLogger(job.data.jobId); - - const { account } = await this.db.accountPermissionSyncJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: AccountPermissionSyncJobStatus.COMPLETED, - account: { - update: { - permissionSyncedAt: new Date(), - permissionSyncIssue: null, - permissionSyncIssueAt: null, - }, - }, - completedAt: new Date(), - }, - select: { - account: { - include: { - user: true, - } - } - } - }); - - logger.debug(`Permissions synced for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}`); - } - - private async onJobFailed(job: Job | undefined, err: Error) { - const logger = createJobLogger(job?.data.jobId ?? 'unknown'); - - Sentry.captureException(err, { - tags: { - jobId: job?.data.jobId, - queue: QUEUE_NAME, - } - }); - - const errorMessage = (accountId: string, email: string) => `Account permission sync job failed for account (id: ${accountId}) for user ${email}: ${err.message}`; - - if (job) { - const { account } = await this.db.accountPermissionSyncJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: AccountPermissionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: err.message, - }, - select: { - account: { - include: { - user: true, - } - } - } - }); - - logger.error(errorMessage(account.id, account.user.email ?? 'unknown user (email not found)')); - } else { - logger.error(errorMessage('unknown account (id not found)', 'unknown user (id not found)')); - } - } -} diff --git a/packages/backend/src/ee/auditLogPruneWorkload.test.ts b/packages/backend/src/ee/auditLogPruneWorkload.test.ts new file mode 100644 index 000000000..ef6d4e724 --- /dev/null +++ b/packages/backend/src/ee/auditLogPruneWorkload.test.ts @@ -0,0 +1,142 @@ +import type { PrismaClient } from "@sourcebot/db"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { createAuditLogPruneWorkload } from "./auditLogPruneWorkload.js"; + +const mocks = { + findMany: vi.fn(), + deleteMany: vi.fn(), +}; + +const db = { + audit: { + findMany: mocks.findMany, + deleteMany: mocks.deleteMany, + }, +} as unknown as PrismaClient; + +const logger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +}; + +const processWorkload = ({ + enabled = true, + retentionDays = 180, +}: { + enabled?: boolean; + retentionDays?: number; +} = {}) => + createAuditLogPruneWorkload({ db, enabled, retentionDays }).process({ + data: {}, + jobId: "job-1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + logger, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), + }); + +describe("auditLogPruneWorkload", () => { + beforeEach(() => { + vi.restoreAllMocks(); + vi.clearAllMocks(); + mocks.findMany.mockResolvedValue([]); + mocks.deleteMany.mockResolvedValue({ count: 0 }); + }); + + test("declares a daily scheduled workload", () => { + const workload = createAuditLogPruneWorkload({ + db, + enabled: true, + retentionDays: 180, + }); + + expect(workload.queueSpec.name).toBe("audit-log-prune"); + expect(workload.concurrency).toBe(1); + expect(workload.schedule).toEqual({ + interval: "1d", + data: {}, + options: { priority: 10 }, + }); + }); + + test.each([ + { enabled: false, retentionDays: 180 }, + { enabled: true, retentionDays: 0 }, + ])( + "does not schedule or prune when disabled: %o", + async ({ enabled, retentionDays }) => { + const workload = createAuditLogPruneWorkload({ + db, + enabled, + retentionDays, + }); + + expect(workload.schedule).toBeUndefined(); + await expect( + processWorkload({ enabled, retentionDays }), + ).resolves.toEqual({ deleted: 0 }); + expect(mocks.findMany).not.toHaveBeenCalled(); + expect(mocks.deleteMany).not.toHaveBeenCalled(); + }, + ); + + test("deletes audit logs older than the retention period", async () => { + vi.spyOn(Date, "now").mockReturnValue( + new Date("2026-08-10T12:00:00.000Z").getTime(), + ); + mocks.findMany.mockResolvedValue([ + { id: "audit-1" }, + { id: "audit-2" }, + ]); + mocks.deleteMany.mockResolvedValue({ count: 2 }); + + await expect(processWorkload()).resolves.toEqual({ deleted: 2 }); + + expect(mocks.findMany).toHaveBeenCalledWith({ + where: { + timestamp: { + lt: new Date("2026-02-11T12:00:00.000Z"), + }, + }, + select: { id: true }, + take: 10_000, + }); + expect(mocks.deleteMany).toHaveBeenCalledWith({ + where: { + id: { in: ["audit-1", "audit-2"] }, + }, + }); + expect(logger.debug).toHaveBeenCalledWith( + "Pruned 2 audit log record(s).", + ); + }); + + test("continues deleting full batches", async () => { + const firstBatch = Array.from({ length: 10_000 }, (_, index) => ({ + id: `audit-${index}`, + })); + mocks.findMany + .mockResolvedValueOnce(firstBatch) + .mockResolvedValueOnce([{ id: "audit-last" }]); + mocks.deleteMany + .mockResolvedValueOnce({ count: 10_000 }) + .mockResolvedValueOnce({ count: 1 }); + + await expect(processWorkload()).resolves.toEqual({ deleted: 10_001 }); + + expect(mocks.findMany).toHaveBeenCalledTimes(2); + expect(mocks.deleteMany).toHaveBeenCalledTimes(2); + }); + + test("propagates database failures so BullMQ can retry", async () => { + const error = new Error("Database unavailable"); + mocks.findMany.mockRejectedValueOnce(error); + + await expect(processWorkload()).rejects.toBe(error); + }); +}); diff --git a/packages/backend/src/ee/auditLogPruneWorkload.ts b/packages/backend/src/ee/auditLogPruneWorkload.ts new file mode 100644 index 000000000..eb6b69a91 --- /dev/null +++ b/packages/backend/src/ee/auditLogPruneWorkload.ts @@ -0,0 +1,80 @@ +import type { PrismaClient } from "@sourcebot/db"; +import { + AUDIT_LOG_PRUNE_QUEUE, + JOB_PRIORITIES, +} from "@sourcebot/shared"; +import type { Workload } from "../types.js"; + +const BATCH_SIZE = 10_000; +const ONE_DAY_MS = 24 * 60 * 60 * 1000; + +interface Props { + db: PrismaClient; + enabled: boolean; + retentionDays: number; +} + +interface AuditLogPruneResult { + deleted: number; +} + +export const createAuditLogPruneWorkload = ({ + db, + enabled, + retentionDays, +}: Props): Workload<"audit-log-prune", AuditLogPruneResult> => ({ + queueSpec: AUDIT_LOG_PRUNE_QUEUE, + concurrency: 1, + ...(enabled && retentionDays > 0 + ? { + schedule: { + interval: "1d", + data: {}, + options: { priority: JOB_PRIORITIES.SCHEDULED }, + }, + } + : {}), + process: async ({ logger }) => { + if (!enabled || retentionDays <= 0) { + logger.debug("Audit log pruning is disabled."); + return { deleted: 0 }; + } + + const cutoff = new Date(Date.now() - retentionDays * ONE_DAY_MS); + let totalDeleted = 0; + + logger.debug( + `Pruning audit logs older than ${cutoff.toISOString()}.`, + ); + + // Delete in batches to avoid long-running transactions. + while (true) { + const batch = await db.audit.findMany({ + where: { timestamp: { lt: cutoff } }, + select: { id: true }, + take: BATCH_SIZE, + }); + + if (batch.length === 0) { + break; + } + + const result = await db.audit.deleteMany({ + where: { id: { in: batch.map(({ id }) => id) } }, + }); + totalDeleted += result.count; + + if (batch.length < BATCH_SIZE) { + break; + } + } + + logger.debug( + totalDeleted > 0 + ? `Pruned ${totalDeleted} audit log record(s).` + : "No audit log records to prune.", + ); + + return { deleted: totalDeleted }; + }, +}); diff --git a/packages/backend/src/ee/auditLogPruner.ts b/packages/backend/src/ee/auditLogPruner.ts deleted file mode 100644 index 9222b91e1..000000000 --- a/packages/backend/src/ee/auditLogPruner.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { PrismaClient } from "@sourcebot/db"; -import { createLogger, env } from "@sourcebot/shared"; -import { setIntervalAsync } from "../utils.js"; - -const BATCH_SIZE = 10_000; -const ONE_DAY_MS = 24 * 60 * 60 * 1000; - -const logger = createLogger('audit-log-pruner'); - -export class AuditLogPruner { - private interval?: NodeJS.Timeout; - - constructor(private db: PrismaClient) {} - - startScheduler() { - if (env.SOURCEBOT_EE_AUDIT_LOGGING_ENABLED !== 'true') { - logger.info('Audit logging is disabled, skipping audit log pruner.'); - return; - } - - if (env.SOURCEBOT_EE_AUDIT_RETENTION_DAYS <= 0) { - logger.info('SOURCEBOT_EE_AUDIT_RETENTION_DAYS is 0, audit log pruning is disabled.'); - return; - } - - logger.debug(`Audit log pruner started. Retaining logs for ${env.SOURCEBOT_EE_AUDIT_RETENTION_DAYS} days.`); - - // Run immediately on startup, then every 24 hours - this.pruneOldAuditLogs(); - this.interval = setIntervalAsync(() => this.pruneOldAuditLogs(), ONE_DAY_MS); - } - - async dispose() { - if (this.interval) { - clearInterval(this.interval); - this.interval = undefined; - } - } - - private async pruneOldAuditLogs() { - const cutoff = new Date(Date.now() - env.SOURCEBOT_EE_AUDIT_RETENTION_DAYS * ONE_DAY_MS); - let totalDeleted = 0; - - logger.debug(`Pruning audit logs older than ${cutoff.toISOString()}...`); - - // Delete in batches to avoid long-running transactions - while (true) { - const batch = await this.db.audit.findMany({ - where: { timestamp: { lt: cutoff } }, - select: { id: true }, - take: BATCH_SIZE, - }); - - if (batch.length === 0) break; - - const result = await this.db.audit.deleteMany({ - where: { id: { in: batch.map(r => r.id) } }, - }); - - totalDeleted += result.count; - - if (batch.length < BATCH_SIZE) break; - } - - if (totalDeleted > 0) { - logger.debug(`Pruned ${totalDeleted} audit log records.`); - } else { - logger.debug('No audit log records to prune.'); - } - } -} diff --git a/packages/backend/src/ee/permissionSyncEligibility.ts b/packages/backend/src/ee/permissionSyncEligibility.ts new file mode 100644 index 000000000..74dfcfbbf --- /dev/null +++ b/packages/backend/src/ee/permissionSyncEligibility.ts @@ -0,0 +1,25 @@ +import type { Prisma } from "@sourcebot/db"; +import { + PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, + PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS, +} from "@sourcebot/shared"; + +export const ACCOUNT_PERMISSION_SYNC_WHERE = { + providerType: { + in: [...PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS], + }, +} satisfies Prisma.AccountWhereInput; + +export const REPO_PERMISSION_SYNC_WHERE = { + isPublic: false, + external_codeHostType: { + in: PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, + }, + connections: { + some: { + connection: { + enforcePermissions: true, + }, + }, + }, +} satisfies Prisma.RepoWhereInput; diff --git a/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts b/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts new file mode 100644 index 000000000..26ebfe595 --- /dev/null +++ b/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts @@ -0,0 +1,425 @@ +import type { PrismaClient } from "@sourcebot/db"; +import type { JobLogger } from "@sourcebot/shared"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + captureException: vi.fn(), + createBitbucketCloudClient: vi.fn(), + createBitbucketServerClient: vi.fn(), + createGitLabFromPersonalAccessToken: vi.fn(), + createOctokitFromToken: vi.fn(), + getAuthCredentialsForRepo: vi.fn(), + getExplicitUserPermissionsForCloudRepo: vi.fn(), + getProjectMembers: vi.fn(), + getRepoCollaborators: vi.fn(), + getUserPermissionsForServerRepo: vi.fn(), + hasEntitlement: vi.fn(), +})); + +vi.mock("@sentry/node", () => ({ + captureException: mocks.captureException, +})); + +vi.mock("@sourcebot/shared", async (importOriginal) => ({ + ...(await importOriginal()), + REPO_PERMISSION_SYNC_QUEUE: { + name: "repo-permission-sync", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keepJobs: { + completed: { count: 50 }, + failed: { count: 50 }, + }, + keepLogs: 500, + }, + }, +})); + +vi.mock("../entitlements.js", () => ({ + hasEntitlement: mocks.hasEntitlement, +})); + +vi.mock("../utils.js", () => ({ + getAuthCredentialsForRepo: mocks.getAuthCredentialsForRepo, +})); + +vi.mock("../github.js", () => ({ + createOctokitFromToken: mocks.createOctokitFromToken, + getRepoCollaborators: mocks.getRepoCollaborators, + GITHUB_CLOUD_HOSTNAME: "github.com", +})); + +vi.mock("../gitlab.js", () => ({ + createGitLabFromPersonalAccessToken: + mocks.createGitLabFromPersonalAccessToken, + getProjectMembers: mocks.getProjectMembers, +})); + +vi.mock("../bitbucket.js", () => ({ + createBitbucketCloudClient: mocks.createBitbucketCloudClient, + createBitbucketServerClient: mocks.createBitbucketServerClient, + getExplicitUserPermissionsForCloudRepo: + mocks.getExplicitUserPermissionsForCloudRepo, + getUserPermissionsForServerRepo: mocks.getUserPermissionsForServerRepo, +})); + +import { createRepoPermissionSyncWorkload } from "./repoPermissionSyncWorkload.js"; + +const repo = { + id: 42, + name: "github.com/sourcebot-dev/sourcebot", + displayName: "sourcebot-dev/sourcebot", + external_codeHostType: "github", + external_id: "123", + metadata: {}, + connections: [], +}; +const repoFindUniqueOrThrow = vi.fn().mockResolvedValue(repo); +const repoUpdate = vi.fn().mockResolvedValue(repo); +const repoUpdateMany = vi.fn().mockResolvedValue({ count: 1 }); +const accountFindMany = vi.fn().mockResolvedValue([]); +const permissionCreateMany = vi.fn().mockResolvedValue({ count: 0 }); +const permissionSyncJobUpsert = vi.fn(); +const permissionSyncJobUpdateMany = vi.fn().mockResolvedValue({ count: 1 }); +const transactionClient = { + repo: { + update: repoUpdate, + updateMany: repoUpdateMany, + }, + repoPermissionSyncJob: { + upsert: permissionSyncJobUpsert, + updateMany: permissionSyncJobUpdateMany, + }, +}; +const transaction = vi.fn( + ( + queriesOrCallback: + | Array> + | ((tx: typeof transactionClient) => Promise), + ) => + typeof queriesOrCallback === "function" + ? queriesOrCallback(transactionClient) + : Promise.all(queriesOrCallback), +); + +const db = { + repo: { + findUniqueOrThrow: repoFindUniqueOrThrow, + update: repoUpdate, + updateMany: repoUpdateMany, + }, + account: { + findMany: accountFindMany, + }, + accountToRepoPermission: { + createMany: permissionCreateMany, + }, + repoPermissionSyncJob: { + upsert: permissionSyncJobUpsert, + updateMany: permissionSyncJobUpdateMany, + }, + $transaction: transaction, +} as unknown as PrismaClient; + +const jobLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + flush: vi.fn(), +} satisfies JobLogger; + +const createWorkload = () => + createRepoPermissionSyncWorkload({ + db, + settings: { + maxRepoPermissionSyncJobConcurrency: 3, + } as never, + }); + +const lifecycleContext = { + data: { + repoId: 42, + }, + jobId: "job_1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + logger: jobLogger, +}; + +const processContext = { + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), +}; + +beforeEach(() => { + vi.clearAllMocks(); + mocks.hasEntitlement.mockResolvedValue(true); + mocks.getAuthCredentialsForRepo.mockReset().mockResolvedValue({ + hostUrl: "https://github.com", + token: "token", + }); + mocks.createOctokitFromToken.mockReset().mockResolvedValue({ octokit: {} }); + mocks.getRepoCollaborators.mockReset().mockResolvedValue([]); + repoFindUniqueOrThrow.mockResolvedValue(repo); + permissionSyncJobUpdateMany.mockResolvedValue({ count: 1 }); + accountFindMany.mockResolvedValue([]); + repoUpdate.mockResolvedValue(repo); + repoUpdateMany.mockResolvedValue({ count: 1 }); + permissionCreateMany.mockResolvedValue({ count: 0 }); +}); + +describe("repoPermissionSyncWorkload", () => { + test("uses the configured concurrency and database-backed lifecycle hooks", () => { + const workload = createWorkload(); + + expect(workload.queueSpec.name).toBe("repo-permission-sync"); + expect(workload.concurrency).toBe(3); + expect(workload.onStarted).toBeTypeOf("function"); + expect(workload.onCompleted).toBeTypeOf("function"); + expect(workload.onTerminalFailure).toBeTypeOf("function"); + }); + + test("shares the repository execution lock with indexing and cleanup", () => { + const workload = createWorkload(); + + expect(workload.executionLock).toBeDefined(); + expect(workload.executionLock?.resource({ repoId: 42 })).toBe( + "sourcebot:lock:repo:42", + ); + expect(workload.executionLock?.durationMs).toBe(60_000); + }); + + test("does not start syncing when execution has already been aborted", async () => { + const controller = new AbortController(); + controller.abort(new Error("Repository execution lock was lost")); + + await expect( + createWorkload().process({ + ...processContext, + signal: controller.signal, + }), + ).rejects.toThrow("Repository execution lock was lost"); + expect(mocks.hasEntitlement).not.toHaveBeenCalled(); + expect(repoFindUniqueOrThrow).not.toHaveBeenCalled(); + }); + + test("syncs the requested repo with its connections", async () => { + await createWorkload().process(processContext); + + expect(repoFindUniqueOrThrow).toHaveBeenCalledWith({ + where: { id: 42 }, + include: { + connections: { + include: { + connection: true, + }, + }, + }, + }); + expect(mocks.getAuthCredentialsForRepo).toHaveBeenCalledWith( + repo, + jobLogger, + ); + expect(mocks.createOctokitFromToken).toHaveBeenCalledWith({ + token: "token", + url: undefined, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("does not run without the permission syncing entitlement", async () => { + mocks.hasEntitlement.mockResolvedValue(false); + + await expect(createWorkload().process(processContext)).rejects.toThrow( + "Permission syncing entitlement is not currently available.", + ); + + expect(repoFindUniqueOrThrow).not.toHaveBeenCalled(); + }); + + test("replaces all permissions for a complete GitHub sync", async () => { + const githubRepo = { + ...repo, + external_codeHostType: "github", + external_id: "123", + metadata: {}, + }; + repoFindUniqueOrThrow.mockResolvedValue(githubRepo); + mocks.getAuthCredentialsForRepo.mockResolvedValue({ + hostUrl: "https://github.com", + token: "token", + }); + const octokit = {}; + mocks.createOctokitFromToken.mockResolvedValue({ octokit }); + mocks.getRepoCollaborators.mockResolvedValue([{ id: 101 }]); + accountFindMany.mockResolvedValue([{ id: "account_1" }]); + + await createWorkload().process(processContext); + + expect(accountFindMany).toHaveBeenCalledWith({ + where: { + providerType: "github", + providerAccountId: { + in: ["101"], + }, + issuerUrl: "https://github.com", + }, + }); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { id: 42 }, + data: { + permittedAccounts: { + deleteMany: {}, + }, + }, + }); + expect(permissionCreateMany).toHaveBeenCalledWith({ + data: [ + { + accountId: "account_1", + repoId: 42, + source: "REPO_DRIVEN", + }, + ], + skipDuplicates: true, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("preserves account-driven permissions for a partial Bitbucket Cloud sync", async () => { + const bitbucketRepo = { + ...repo, + external_codeHostType: "bitbucketCloud", + external_id: "repo-uuid", + metadata: { + codeHostMetadata: { + bitbucketCloud: { + workspace: "sourcebot", + repoSlug: "sourcebot", + }, + }, + }, + }; + repoFindUniqueOrThrow.mockResolvedValue(bitbucketRepo); + mocks.getAuthCredentialsForRepo.mockResolvedValue({ + hostUrl: "https://bitbucket.org", + token: "token", + connectionConfig: { + user: "service-account", + }, + }); + mocks.createBitbucketCloudClient.mockReturnValue({}); + mocks.getExplicitUserPermissionsForCloudRepo.mockResolvedValue([ + { accountId: "upstream-account" }, + ]); + accountFindMany.mockResolvedValue([{ id: "account_1" }]); + + await createWorkload().process(processContext); + + expect(repoUpdate).toHaveBeenCalledWith({ + where: { id: 42 }, + data: { + permittedAccounts: { + deleteMany: { + source: "REPO_DRIVEN", + }, + }, + }, + }); + expect(permissionCreateMany).toHaveBeenCalledWith({ + data: [ + { + accountId: "account_1", + repoId: 42, + source: "REPO_DRIVEN", + }, + ], + skipDuplicates: true, + }); + }); + + test("marks a job as in progress when started", async () => { + await createWorkload().onStarted?.(lifecycleContext); + + expect(permissionSyncJobUpsert).toHaveBeenCalledWith({ + where: { id: "job_1" }, + update: { + status: "IN_PROGRESS", + completedAt: null, + errorMessage: null, + }, + create: { + id: "job_1", + repoId: 42, + status: "IN_PROGRESS", + }, + }); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { id: 42 }, + data: { latestPermissionSyncJobId: "job_1" }, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("marks a job completed and updates the repo sync timestamp when it is still latest", async () => { + await createWorkload().onCompleted?.(lifecycleContext, { + repoName: "sourcebot-dev/sourcebot", + }); + + expect(permissionSyncJobUpdateMany).toHaveBeenCalledWith({ + where: { id: "job_1" }, + data: { + status: "COMPLETED", + completedAt: expect.any(Date), + errorMessage: null, + }, + }); + expect(repoUpdateMany).toHaveBeenCalledWith({ + where: { + id: 42, + latestPermissionSyncJobId: "job_1", + }, + data: { + permissionSyncedAt: expect.any(Date), + }, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("does not fail completion after the repo has been deleted", async () => { + permissionSyncJobUpdateMany.mockResolvedValue({ count: 0 }); + repoUpdateMany.mockResolvedValue({ count: 0 }); + + await expect( + createWorkload().onCompleted?.(lifecycleContext, { + repoName: "sourcebot-dev/sourcebot", + }), + ).resolves.toBeUndefined(); + }); + + test("marks a job failed after terminal failure", async () => { + const error = new Error("Upstream unavailable"); + + await createWorkload().onTerminalFailure?.(lifecycleContext, error); + + expect(permissionSyncJobUpdateMany).toHaveBeenCalledWith({ + where: { id: "job_1" }, + data: { + status: "FAILED", + completedAt: expect.any(Date), + errorMessage: "Upstream unavailable", + }, + }); + expect(mocks.captureException).toHaveBeenCalledWith(error, { + tags: { + jobId: "job_1", + queue: "repo-permission-sync", + }, + }); + }); +}); diff --git a/packages/backend/src/ee/repoPermissionSyncWorkload.ts b/packages/backend/src/ee/repoPermissionSyncWorkload.ts new file mode 100644 index 000000000..be736057d --- /dev/null +++ b/packages/backend/src/ee/repoPermissionSyncWorkload.ts @@ -0,0 +1,443 @@ +import * as Sentry from "@sentry/node"; +import { + PermissionSyncSource, + PrismaClient, + RepoPermissionSyncJobStatus, +} from "@sourcebot/db"; +import { + JobLogSink, + REPO_PERMISSION_SYNC_QUEUE, + repoMetadataSchema, +} from "@sourcebot/shared"; +import { hasEntitlement } from "../entitlements.js"; +import { + createOctokitFromToken, + getRepoCollaborators, + GITHUB_CLOUD_HOSTNAME, +} from "../github.js"; +import { + createGitLabFromPersonalAccessToken, + getProjectMembers, +} from "../gitlab.js"; +import { + createBitbucketCloudClient, + createBitbucketServerClient, + getExplicitUserPermissionsForCloudRepo, + getUserPermissionsForServerRepo, +} from "../bitbucket.js"; +import { + RepoAuthCredentials, + RepoWithConnections, + Settings, + Workload, +} from "../types.js"; +import { getAuthCredentialsForRepo } from "../utils.js"; +import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/index.type"; + +interface RepoPermissionSyncWorkloadDependencies { + db: PrismaClient; + settings: Settings; +} + +interface RepoPermissionSyncResult { + repoName: string; +} + +const REPO_PERMISSION_SYNC_LOCK_DURATION_MS = 60_000; + +export const createRepoPermissionSyncWorkload = ({ + db, + settings, +}: RepoPermissionSyncWorkloadDependencies): Workload< + "repo-permission-sync", + RepoPermissionSyncResult +> => ({ + queueSpec: REPO_PERMISSION_SYNC_QUEUE, + concurrency: settings.maxRepoPermissionSyncJobConcurrency, + // This lock is shared with repoIndexWorkload so indexing, cleanup, and + // permission syncing are serialized for the same repository. + executionLock: { + resource: ({ repoId }) => `sourcebot:lock:repo:${repoId}`, + durationMs: REPO_PERMISSION_SYNC_LOCK_DURATION_MS, + }, + process: async ({ data: { repoId }, logger, signal }) => { + signal.throwIfAborted(); + if (!(await hasEntitlement("permission-syncing"))) { + throw new Error( + "Permission syncing entitlement is not currently available.", + ); + } + + signal.throwIfAborted(); + const repo = await db.repo.findUniqueOrThrow({ + where: { + id: repoId, + }, + include: { + connections: { + include: { + connection: true, + }, + }, + }, + }); + signal.throwIfAborted(); + + const id = repo.id; + logger.debug(`Syncing permissions for repo ${repo.displayName}...`); + + const credentials = await getAuthCredentialsForRepo(repo, logger); + signal.throwIfAborted(); + if (!credentials) { + throw new Error(`No credentials found for repo ${id}`); + } + + const { accountIds, isPartialSync = false } = + await getPermissionSyncResult({ + db, + repo, + credentials, + logger: logger, + }); + + signal.throwIfAborted(); + await db.$transaction([ + db.repo.update({ + where: { + id: repo.id, + }, + data: { + permittedAccounts: { + // @note: if this is a partial sync, we only want to delete the repo-driven permissions + // since we don't want to overwrite the account-driven permissions. + deleteMany: isPartialSync + ? { + source: PermissionSyncSource.REPO_DRIVEN, + } + : {}, + }, + }, + }), + db.accountToRepoPermission.createMany({ + data: accountIds.map((accountId) => ({ + accountId, + repoId: repo.id, + source: PermissionSyncSource.REPO_DRIVEN, + })), + skipDuplicates: true, + }), + ]); + signal.throwIfAborted(); + + return { + repoName: repo.displayName ?? repo.name, + }; + }, + onStarted: async ({ data: { repoId }, jobId }) => { + await db.$transaction(async (tx) => { + await tx.repoPermissionSyncJob.upsert({ + where: { + id: jobId, + }, + update: { + status: RepoPermissionSyncJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + }, + create: { + id: jobId, + repoId, + status: RepoPermissionSyncJobStatus.IN_PROGRESS, + }, + }); + await tx.repo.update({ + where: { + id: repoId, + }, + data: { + latestPermissionSyncJobId: jobId, + }, + }); + }); + }, + onCompleted: async ( + { data: { repoId }, jobId, logger }, + { repoName }, + ) => { + await db.$transaction(async (tx) => { + await tx.repoPermissionSyncJob.updateMany({ + where: { + id: jobId, + }, + data: { + status: RepoPermissionSyncJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + }, + }); + await tx.repo.updateMany({ + where: { + id: repoId, + latestPermissionSyncJobId: jobId, + }, + data: { + permissionSyncedAt: new Date(), + }, + }); + }); + + logger.debug(`Permissions synced for repo ${repoName}`); + }, + onTerminalFailure: async ({ data: { repoId }, jobId, logger }, error) => { + Sentry.captureException(error, { + tags: { + jobId, + queue: REPO_PERMISSION_SYNC_QUEUE.name, + }, + }); + + await db.repoPermissionSyncJob.updateMany({ + where: { + id: jobId, + }, + data: { + status: RepoPermissionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + }); + + logger.error( + `Repo permission sync job failed for repo ${repoId}: ${error.message}`, + ); + }, +}); + +interface ProviderPermissionSyncProps { + db: PrismaClient; + repo: RepoWithConnections; + credentials: RepoAuthCredentials; + logger: JobLogSink; +} + +interface PermissionSyncResult { + accountIds: string[]; + isPartialSync?: boolean; +} + +const getPermissionSyncResult = async ( + props: ProviderPermissionSyncProps, +): Promise => { + switch (props.repo.external_codeHostType) { + case "github": + return getGitHubPermissionSyncResult(props); + case "gitlab": + return getGitLabPermissionSyncResult(props); + case "bitbucketCloud": + return getBitbucketCloudPermissionSyncResult(props); + case "bitbucketServer": + return getBitbucketServerPermissionSyncResult(props); + default: + throw new Error( + `Unsupported code host type: ${props.repo.external_codeHostType}`, + ); + } +}; + +const getGitHubPermissionSyncResult = async ({ + db, + repo, + credentials, + logger, +}: ProviderPermissionSyncProps): Promise => { + const isGitHubCloud = credentials.hostUrl + ? new URL(credentials.hostUrl).hostname === GITHUB_CLOUD_HOSTNAME + : true; + const { octokit } = await createOctokitFromToken({ + token: credentials.token, + url: isGitHubCloud ? undefined : credentials.hostUrl, + }); + + // @note: this is a bit of a hack since the displayName _might_ not be set.. + // however, this property was introduced many versions ago and _should_ be set + // on each connection sync. Let's throw an error just in case. + if (!repo.displayName) { + throw new Error(`Repo ${repo.id} does not have a displayName`); + } + + const [owner, repoName] = repo.displayName.split("/"); + const collaborators = await getRepoCollaborators(owner, repoName, octokit); + const githubUserIds = collaborators.map((collaborator) => + collaborator.id.toString(), + ); + + logger.debug(`Found ${collaborators.length} collaborator(s)`, { + collaborators: collaborators.flatMap(({ email, login }) => ({email, login})), + }); + + const accounts = await db.account.findMany({ + where: { + providerType: "github", + providerAccountId: { + in: githubUserIds, + }, + issuerUrl: credentials.hostUrl, + }, + }); + + return { + accountIds: accounts.map((account) => account.id), + }; +}; + +const getGitLabPermissionSyncResult = async ({ + db, + repo, + credentials, +}: ProviderPermissionSyncProps): Promise => { + const api = await createGitLabFromPersonalAccessToken({ + token: credentials.token, + url: credentials.hostUrl, + }); + + const projectId = repo.external_id; + if (!projectId) { + throw new Error(`Repo ${repo.id} does not have an external_id`); + } + + const members = await getProjectMembers(projectId, api); + const gitlabUserIds = members.map((member) => member.id.toString()); + + const accounts = await db.account.findMany({ + where: { + providerType: "gitlab", + providerAccountId: { + in: gitlabUserIds, + }, + issuerUrl: credentials.hostUrl, + }, + }); + + return { + accountIds: accounts.map((account) => account.id), + }; +}; + +const getBitbucketCloudPermissionSyncResult = async ({ + db, + repo, + credentials, +}: ProviderPermissionSyncProps): Promise => { + const config = credentials.connectionConfig as + | BitbucketConnectionConfig + | undefined; + if (!config) { + throw new Error(`No connection config found for repo ${repo.id}`); + } + + const client = createBitbucketCloudClient(config.user, credentials.token); + + const parsedMetadata = repoMetadataSchema.safeParse(repo.metadata); + if (!parsedMetadata.success) { + throw new Error( + `Repo ${repo.id} has invalid metadata: ${JSON.stringify(parsedMetadata.error.errors)}`, + ); + } + const bitbucketCloudMetadata = + parsedMetadata.data.codeHostMetadata?.bitbucketCloud; + if (!bitbucketCloudMetadata) { + throw new Error( + `Repo ${repo.id} is missing required Bitbucket Cloud metadata (workspace/repoSlug)`, + ); + } + + const { workspace, repoSlug } = bitbucketCloudMetadata; + + // @note: The Bitbucket Cloud permissions API only returns users who have been *directly* + // granted access to this repository. Users who have access via a group added to the repo, + // via project-level membership, or via a group in a project are NOT captured here. + // These users will still gain access through account-driven permission syncing, + // but there may be a delay of up to `userDrivenPermissionSyncIntervalMs` before + // they see the repository in Sourcebot. + // @see: https://developer.atlassian.com/cloud/bitbucket/rest/api-group-repositories/#api-repositories-workspace-repo-slug-permissions-config-users-get + const users = await getExplicitUserPermissionsForCloudRepo( + client, + workspace, + repoSlug, + ); + const userAccountIds = users.map((user) => user.accountId); + + const accounts = await db.account.findMany({ + where: { + providerType: "bitbucket-cloud", + providerAccountId: { + in: userAccountIds, + }, + issuerUrl: credentials.hostUrl, + }, + }); + + return { + accountIds: accounts.map((account) => account.id), + // Since we only fetch users who have been explicitly granted access to the repo, + // this is a partial sync. + isPartialSync: true, + }; +}; + +const getBitbucketServerPermissionSyncResult = async ({ + db, + repo, + credentials, +}: ProviderPermissionSyncProps): Promise => { + const parsedMetadata = repoMetadataSchema.safeParse(repo.metadata); + if (!parsedMetadata.success) { + throw new Error( + `Repo ${repo.id} has invalid metadata: ${JSON.stringify(parsedMetadata.error.errors)}`, + ); + } + const bitbucketServerMetadata = + parsedMetadata.data.codeHostMetadata?.bitbucketServer; + if (!bitbucketServerMetadata) { + throw new Error( + `Repo ${repo.id} is missing required Bitbucket Server metadata (projectKey/repoSlug)`, + ); + } + + const { projectKey, repoSlug } = bitbucketServerMetadata; + const hostUrl = credentials.hostUrl; + + if (!hostUrl) { + throw new Error( + `No host URL found for Bitbucket Server repo ${repo.id}`, + ); + } + + // @note: This covers users with direct repo-level and project-level permissions. + // Users with access only via groups are NOT captured here. Those users will + // still gain access through account-driven permission syncing. + const client = createBitbucketServerClient( + hostUrl, + /* user = */ undefined, + credentials.token, + ); + const users = await getUserPermissionsForServerRepo( + client, + projectKey, + repoSlug, + ); + const userIds = users.map((user) => user.userId); + + const accounts = await db.account.findMany({ + where: { + providerType: "bitbucket-server", + providerAccountId: { in: userIds }, + issuerUrl: credentials.hostUrl, + }, + }); + + return { + accountIds: accounts.map((account) => account.id), + isPartialSync: true, + }; +}; diff --git a/packages/backend/src/ee/repoPermissionSyncer.ts b/packages/backend/src/ee/repoPermissionSyncer.ts deleted file mode 100644 index 536f48a09..000000000 --- a/packages/backend/src/ee/repoPermissionSyncer.ts +++ /dev/null @@ -1,436 +0,0 @@ -import * as Sentry from "@sentry/node"; -import { PermissionSyncSource, PrismaClient, Repo, RepoPermissionSyncJobStatus } from "@sourcebot/db"; -import { createLogger, PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES } from "@sourcebot/shared"; -import { env } from "@sourcebot/shared"; -import { hasEntitlement } from "../entitlements.js"; -import { DelayedError, Job, Queue, Worker } from 'bullmq'; -import { Redis } from 'ioredis'; -import { createOctokitFromToken, getRepoCollaborators, GITHUB_CLOUD_HOSTNAME } from "../github.js"; -import { createGitLabFromPersonalAccessToken, getProjectMembers } from "../gitlab.js"; -import { createBitbucketCloudClient, createBitbucketServerClient, getExplicitUserPermissionsForCloudRepo, getUserPermissionsForServerRepo } from "../bitbucket.js"; -import { repoMetadataSchema } from "@sourcebot/shared"; -import { Settings } from "../types.js"; -import { getAuthCredentialsForRepo, setIntervalAsync } from "../utils.js"; -import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/index.type"; - -type RepoPermissionSyncJob = { - jobId: string; -} - -const QUEUE_NAME = 'repoPermissionSyncQueue'; -const POLLING_INTERVAL_MS = 1000; -const ENTITLEMENT_RETRY_DELAY_MS = 30 * 1000; -const LOG_TAG = 'repo-permission-syncer'; - -const logger = createLogger(LOG_TAG); -const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); - -export class RepoPermissionSyncer { - private queue: Queue; - private worker: Worker; - private interval?: NodeJS.Timeout; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - ) { - this.queue = new Queue(QUEUE_NAME, { - connection: redis, - }); - this.worker = new Worker(QUEUE_NAME, this.runJob.bind(this), { - connection: redis, - concurrency: this.settings.maxRepoPermissionSyncJobConcurrency, - }); - this.worker.on('completed', this.onJobCompleted.bind(this)); - this.worker.on('failed', this.onJobFailed.bind(this)); - } - - public async startScheduler() { - logger.debug('Starting scheduler'); - - this.interval = setIntervalAsync(async () => { - if (!await hasEntitlement('permission-syncing')) { - return; - } - - // @todo: make this configurable - const thresholdDate = new Date(Date.now() - this.settings.repoDrivenPermissionSyncIntervalMs); - - const repos = await this.db.repo.findMany({ - // Repos need their permissions to be synced against the code host when... - where: { - AND: [ - // They are not public. Public repositories are always visible to all users, therefore we don't - // need to explicitly perform permission syncing for them. - // @see: packages/web/src/prisma.ts - { - isPublic: false - }, - // They belong to a code host that supports permissions syncing - { - external_codeHostType: { - in: PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, - } - }, - // They have at least one connection with permission enforcement enabled - { - connections: { - some: { - connection: { - enforcePermissions: true, - } - } - } - }, - // They have not been synced within the threshold date. - { - OR: [ - { permissionSyncedAt: null }, - { permissionSyncedAt: { lt: thresholdDate } }, - ], - }, - // There aren't any active or recently failed jobs. - { - NOT: { - permissionSyncJobs: { - some: { - OR: [ - // Don't schedule if there are active jobs - { - status: { - in: [ - RepoPermissionSyncJobStatus.PENDING, - RepoPermissionSyncJobStatus.IN_PROGRESS, - ], - } - }, - // Don't schedule if there are recent failed jobs (within the threshold date). Note `gt` is used here since this is a inverse condition. - { - AND: [ - { status: RepoPermissionSyncJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } - ] - } - } - } - }, - ] - } - }); - - await this.schedulePermissionSync(repos); - }, POLLING_INTERVAL_MS); - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - await this.worker.close(/* force = */ true); - await this.queue.close(); - } - - private async schedulePermissionSync(repos: Repo[]) { - // @note: we don't perform this in a transaction because - // we want to avoid the situation where a job is created and run - // prior to the transaction being committed. - const jobs = await this.db.repoPermissionSyncJob.createManyAndReturn({ - data: repos.map(repo => ({ - repoId: repo.id, - })), - include: { - repo: true, - } - }); - - await this.queue.addBulk(jobs.map((job) => ({ - name: 'repoPermissionSyncJob', - data: { - jobId: job.id, - }, - opts: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - // Priority 1 (high) for never-synced, Priority 2 (normal) for re-sync - priority: job.repo.permissionSyncedAt === null ? 1 : 2, - } - }))) - } - - private async runJob(job: Job) { - if (!await hasEntitlement('permission-syncing')) { - await job.moveToDelayed(Date.now() + ENTITLEMENT_RETRY_DELAY_MS, job.token); - throw new DelayedError('Permission syncing entitlement is not currently available.'); - } - - const id = job.data.jobId; - const logger = createJobLogger(id); - - const { repo } = await this.db.repoPermissionSyncJob.update({ - where: { - id, - }, - data: { - status: RepoPermissionSyncJobStatus.IN_PROGRESS, - }, - select: { - repo: { - include: { - connections: { - include: { - connection: true, - } - } - } - } - } - }); - - if (!repo) { - throw new Error(`Repo ${id} not found`); - } - - logger.debug(`Syncing permissions for repo ${repo.displayName}...`); - - const credentials = await getAuthCredentialsForRepo(repo, logger); - if (!credentials) { - throw new Error(`No credentials found for repo ${id}`); - } - - const { - accountIds, - isPartialSync = false, - } = await (async (): Promise<{ - accountIds: string[], - isPartialSync?: boolean - }> => { - if (repo.external_codeHostType === 'github') { - const isGitHubCloud = credentials.hostUrl ? new URL(credentials.hostUrl).hostname === GITHUB_CLOUD_HOSTNAME : true; - const { octokit } = await createOctokitFromToken({ - token: credentials.token, - url: isGitHubCloud ? undefined : credentials.hostUrl, - }); - - // @note: this is a bit of a hack since the displayName _might_ not be set.. - // however, this property was introduced many versions ago and _should_ be set - // on each connection sync. Let's throw an error just in case. - if (!repo.displayName) { - throw new Error(`Repo ${id} does not have a displayName`); - } - - const [owner, repoName] = repo.displayName.split('/'); - - const collaborators = await getRepoCollaborators(owner, repoName, octokit); - const githubUserIds = collaborators.map(collaborator => collaborator.id.toString()); - - const accounts = await this.db.account.findMany({ - where: { - providerType: 'github', - providerAccountId: { - in: githubUserIds, - }, - issuerUrl: credentials.hostUrl, - }, - }); - - return { - accountIds: accounts.map(account => account.id), - } - } else if (repo.external_codeHostType === 'gitlab') { - const api = await createGitLabFromPersonalAccessToken({ - token: credentials.token, - url: credentials.hostUrl, - }); - - const projectId = repo.external_id; - if (!projectId) { - throw new Error(`Repo ${id} does not have an external_id`); - } - - const members = await getProjectMembers(projectId, api); - const gitlabUserIds = members.map(member => member.id.toString()); - - const accounts = await this.db.account.findMany({ - where: { - providerType: 'gitlab', - providerAccountId: { - in: gitlabUserIds, - }, - issuerUrl: credentials.hostUrl, - }, - }); - - return { - accountIds: accounts.map(account => account.id), - } - } else if (repo.external_codeHostType === 'bitbucketCloud') { - const config = credentials.connectionConfig as BitbucketConnectionConfig | undefined; - if (!config) { - throw new Error(`No connection config found for repo ${id}`); - } - - const client = createBitbucketCloudClient(config.user, credentials.token); - - const parsedMetadata = repoMetadataSchema.safeParse(repo.metadata); - if (!parsedMetadata.success) { - throw new Error(`Repo ${id} has invalid metadata: ${JSON.stringify(parsedMetadata.error.errors)}`); - } - const bitbucketCloudMetadata = parsedMetadata.data.codeHostMetadata?.bitbucketCloud; - if (!bitbucketCloudMetadata) { - throw new Error(`Repo ${id} is missing required Bitbucket Cloud metadata (workspace/repoSlug)`); - } - - const { workspace, repoSlug } = bitbucketCloudMetadata; - - // @note: The Bitbucket Cloud permissions API only returns users who have been *directly* - // granted access to this repository. Users who have access via a group added to the repo, - // via project-level membership, or via a group in a project are NOT captured here. - // These users will still gain access through user-driven syncing (accountPermissionSyncer), - // but there may be a delay of up to `userDrivenPermissionSyncIntervalMs` before - // they see the repository in Sourcebot. - // @see: https://developer.atlassian.com/cloud/bitbucket/rest/api-group-repositories/#api-repositories-workspace-repo-slug-permissions-config-users-get - const users = await getExplicitUserPermissionsForCloudRepo(client, workspace, repoSlug); - const userAccountIds = users.map(u => u.accountId); - - const accounts = await this.db.account.findMany({ - where: { - providerType: 'bitbucket-cloud', - providerAccountId: { - in: userAccountIds, - }, - issuerUrl: credentials.hostUrl, - }, - }); - - return { - accountIds: accounts.map(account => account.id), - // Since we only fetch users who have been explicitly granted access to the repo, - // this is a partial sync. - isPartialSync: true, - } - } else if (repo.external_codeHostType === 'bitbucketServer') { - const parsedMetadata = repoMetadataSchema.safeParse(repo.metadata); - if (!parsedMetadata.success) { - throw new Error(`Repo ${id} has invalid metadata: ${JSON.stringify(parsedMetadata.error.errors)}`); - } - const bitbucketServerMetadata = parsedMetadata.data.codeHostMetadata?.bitbucketServer; - if (!bitbucketServerMetadata) { - throw new Error(`Repo ${id} is missing required Bitbucket Server metadata (projectKey/repoSlug)`); - } - - const { projectKey, repoSlug } = bitbucketServerMetadata; - const hostUrl = credentials.hostUrl; - - if (!hostUrl) { - throw new Error(`No host URL found for Bitbucket Server repo ${id}`); - } - - // @note: This covers users with direct repo-level and project-level permissions. - // Users with access only via groups are NOT captured here. Those users will - // still gain access through account-driven syncing (accountPermissionSyncer). - const client = createBitbucketServerClient(hostUrl, /* user = */ undefined, credentials.token); - const users = await getUserPermissionsForServerRepo(client, projectKey, repoSlug); - const userIds = users.map(u => u.userId); - - const accounts = await this.db.account.findMany({ - where: { - providerType: 'bitbucket-server', - providerAccountId: { in: userIds }, - issuerUrl: credentials.hostUrl, - } - }); - - return { - accountIds: accounts.map(account => account.id), - isPartialSync: true, - } - } - - throw new Error(`Unsupported code host type: ${repo.external_codeHostType}`); - })(); - - await this.db.$transaction([ - this.db.repo.update({ - where: { - id: repo.id, - }, - data: { - permittedAccounts: { - // @note: if this is a partial sync, we only want to delete the repo-driven permissions - // since we don't want to overwrite the account-driven permissions. - deleteMany: isPartialSync ? { - source: PermissionSyncSource.REPO_DRIVEN, - } : {}, - } - } - }), - this.db.accountToRepoPermission.createMany({ - data: accountIds.map(accountId => ({ - accountId, - repoId: repo.id, - source: PermissionSyncSource.REPO_DRIVEN, - })), - skipDuplicates: true, - }) - ]); - } - - private async onJobCompleted(job: Job) { - const logger = createJobLogger(job.data.jobId); - - const { repo } = await this.db.repoPermissionSyncJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: RepoPermissionSyncJobStatus.COMPLETED, - repo: { - update: { - permissionSyncedAt: new Date(), - } - }, - completedAt: new Date(), - }, - select: { - repo: true - } - }); - - logger.debug(`Permissions synced for repo ${repo.displayName ?? repo.name}`); - } - - private async onJobFailed(job: Job | undefined, err: Error) { - const logger = createJobLogger(job?.data.jobId ?? 'unknown'); - - Sentry.captureException(err, { - tags: { - jobId: job?.data.jobId, - queue: QUEUE_NAME, - } - }); - - const errorMessage = (repoName: string) => `Repo permission sync job failed for repo ${repoName}: ${err.message}`; - - if (job) { - const { repo } = await this.db.repoPermissionSyncJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: RepoPermissionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: err.message, - }, - select: { - repo: true - }, - }); - logger.error(errorMessage(repo.displayName ?? repo.name)); - } else { - logger.error(errorMessage('unknown repo (id not found)')); - } - } -} diff --git a/packages/backend/src/ee/syncSearchContexts.test.ts b/packages/backend/src/ee/syncSearchContexts.test.ts index 9aa1decfd..89ffec5d6 100644 --- a/packages/backend/src/ee/syncSearchContexts.test.ts +++ b/packages/backend/src/ee/syncSearchContexts.test.ts @@ -21,6 +21,17 @@ vi.mock('../entitlements.js', () => ({ getPlan: vi.fn(() => Promise.resolve('enterprise')), })); +// `syncSearchContexts` imports the prisma singleton, which builds a real client (and needs +// DATABASE_URL) at import time. Stand in for it with an object that `buildDb` re-populates +// with fresh mocks for each test, so tests stay isolated from one another. +const { prismaMock } = vi.hoisted(() => ({ + prismaMock: {} as Record, +})); + +vi.mock('../prisma.js', () => ({ + prisma: prismaMock, +})); + import { syncSearchContexts } from './syncSearchContexts.js'; // Helper to build a repo record with GitLab topics stored in metadata. @@ -64,20 +75,25 @@ const buildDb = (overrides: Partial<{ connectionFindMany: unknown[]; searchContextFindUnique: unknown; searchContextFindMany: unknown[]; -}> = {}): PrismaClient => ({ - repo: { - findMany: vi.fn().mockResolvedValue(overrides.repoFindMany ?? []), - }, - connection: { - findMany: vi.fn().mockResolvedValue(overrides.connectionFindMany ?? []), - }, - searchContext: { - findUnique: vi.fn().mockResolvedValue(overrides.searchContextFindUnique ?? null), - findMany: vi.fn().mockResolvedValue(overrides.searchContextFindMany ?? []), - upsert: vi.fn().mockResolvedValue({}), - delete: vi.fn().mockResolvedValue({}), - }, -} as unknown as PrismaClient); +}> = {}): PrismaClient => { + // Overwrite every delegate so no mock survives from a previous test. + Object.assign(prismaMock, { + repo: { + findMany: vi.fn().mockResolvedValue(overrides.repoFindMany ?? []), + }, + connection: { + findMany: vi.fn().mockResolvedValue(overrides.connectionFindMany ?? []), + }, + searchContext: { + findUnique: vi.fn().mockResolvedValue(overrides.searchContextFindUnique ?? null), + findMany: vi.fn().mockResolvedValue(overrides.searchContextFindMany ?? []), + upsert: vi.fn().mockResolvedValue({}), + delete: vi.fn().mockResolvedValue({}), + }, + }); + + return prismaMock as unknown as PrismaClient; +}; describe('syncSearchContexts - includeTopics', () => { test('includes repos whose topics match an includeTopics entry', async () => { @@ -90,7 +106,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -109,7 +124,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -127,7 +141,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['backend', 'core'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -146,7 +159,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['core-*'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -165,7 +177,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -188,7 +199,6 @@ describe('syncSearchContexts - excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -209,7 +219,6 @@ describe('syncSearchContexts - excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -230,7 +239,6 @@ describe('syncSearchContexts - excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -251,7 +259,6 @@ describe('syncSearchContexts - excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -274,7 +281,6 @@ describe('syncSearchContexts - includeTopics + excludeTopics combined', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -298,7 +304,6 @@ describe('syncSearchContexts - includeTopics combined with include globs', () => }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -319,7 +324,6 @@ describe('syncSearchContexts - includeTopics combined with include globs', () => }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -340,7 +344,6 @@ describe('syncSearchContexts - GitHub includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -359,7 +362,6 @@ describe('syncSearchContexts - GitHub includeTopics', () => { myContext: { includeTopics: ['core-*'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -377,7 +379,6 @@ describe('syncSearchContexts - GitHub includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -400,7 +401,6 @@ describe('syncSearchContexts - GitHub excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -422,7 +422,6 @@ describe('syncSearchContexts - mixed GitHub and GitLab repos', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -446,7 +445,6 @@ describe('syncSearchContexts - mixed GitHub and GitLab repos', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; diff --git a/packages/backend/src/ee/syncSearchContexts.ts b/packages/backend/src/ee/syncSearchContexts.ts index 6a02c9062..a3592a30f 100644 --- a/packages/backend/src/ee/syncSearchContexts.ts +++ b/packages/backend/src/ee/syncSearchContexts.ts @@ -1,20 +1,19 @@ import micromatch from "micromatch"; import { createLogger } from "@sourcebot/shared"; -import { PrismaClient } from "@sourcebot/db"; import { repoMetadataSchema, SOURCEBOT_SUPPORT_EMAIL } from "@sourcebot/shared"; import { hasEntitlement } from "../entitlements.js"; import { SearchContext } from "@sourcebot/schemas/v3/index.type"; +import { prisma } from "../prisma.js"; const logger = createLogger('sync-search-contexts'); interface SyncSearchContextsParams { contexts?: { [key: string]: SearchContext } | undefined; orgId: number; - db: PrismaClient; } export const syncSearchContexts = async (params: SyncSearchContextsParams) => { - const { contexts, orgId, db } = params; + const { contexts, orgId } = params; if (!await hasEntitlement("search-contexts")) { if (contexts) { @@ -25,7 +24,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { if (contexts) { for (const [key, newContextConfig] of Object.entries(contexts)) { - const allRepos = await db.repo.findMany({ + const allRepos = await prisma.repo.findMany({ where: { orgId, }, @@ -44,7 +43,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { } if(newContextConfig.includeConnections) { - const connections = await db.connection.findMany({ + const connections = await prisma.connection.findMany({ where: { orgId, name: { @@ -101,7 +100,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { } if (newContextConfig.excludeConnections) { - const connections = await db.connection.findMany({ + const connections = await prisma.connection.findMany({ where: { orgId, name: { @@ -145,7 +144,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { }); } - const currentReposInContext = (await db.searchContext.findUnique({ + const currentReposInContext = (await prisma.searchContext.findUnique({ where: { name_orgId: { name: key, @@ -157,7 +156,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { } }))?.repos ?? []; - await db.searchContext.upsert({ + await prisma.searchContext.upsert({ where: { name_orgId: { name: key, @@ -195,7 +194,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { } } - const deletedContexts = await db.searchContext.findMany({ + const deletedContexts = await prisma.searchContext.findMany({ where: { name: { notIn: Object.keys(contexts ?? {}), @@ -206,7 +205,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { for (const context of deletedContexts) { logger.debug(`Deleting search context with name '${context.name}'. ID: ${context.id}`); - await db.searchContext.delete({ + await prisma.searchContext.delete({ where: { id: context.id, } diff --git a/packages/backend/src/executionLock.test.ts b/packages/backend/src/executionLock.test.ts new file mode 100644 index 000000000..15ed9ae82 --- /dev/null +++ b/packages/backend/src/executionLock.test.ts @@ -0,0 +1,180 @@ +import type { Redis } from "ioredis"; +import Redlock, { + ExecutionError, + ResourceLockedError, + type ExecutionStats, + type RedlockAbortSignal, +} from "redlock"; +import { describe, expect, test, vi } from "vitest"; +import { RedlockExecutionLockRunner } from "./executionLock.js"; + +const LOCK_DURATION_MS = 60_000; + +const executionError = (attemptError: Error): ExecutionError => { + const client = {} as Redis; + const stats: ExecutionStats = { + membershipSize: 1, + quorumSize: 1, + votesFor: new Set(), + votesAgainst: new Map([[client, attemptError]]), + }; + + return new ExecutionError("Unable to acquire lock", [ + Promise.resolve(stats), + ]); +}; + +const contentionError = (): ExecutionError => + executionError(new ResourceLockedError("Resource is locked")); + +const lockSignal = (controller = new AbortController()) => + controller.signal as RedlockAbortSignal; + +describe("RedlockExecutionLockRunner", () => { + test("waits and retries after lock contention", async () => { + const using = vi + .fn() + .mockRejectedValueOnce(contentionError()) + .mockImplementationOnce( + async (_resources, _duration, _settings, routine) => + routine(lockSignal()), + ); + const runner = new RedlockExecutionLockRunner( + { using } as unknown as Redlock, + { retryDelayMs: 0, retryJitterMs: 0 }, + ); + + await expect( + runner.using( + "resource:1", + LOCK_DURATION_MS, + new AbortController().signal, + async () => "complete", + ), + ).resolves.toBe("complete"); + + expect(using).toHaveBeenCalledTimes(2); + expect(using).toHaveBeenLastCalledWith( + ["resource:1"], + LOCK_DURATION_MS, + { retryCount: 0 }, + expect.any(Function), + ); + }); + + test("does not retry an error thrown after lock acquisition", async () => { + const error = contentionError(); + const using = vi.fn(async (_resources, _duration, _settings, routine) => + routine(lockSignal()), + ); + const runner = new RedlockExecutionLockRunner( + { using } as unknown as Redlock, + { retryDelayMs: 0, retryJitterMs: 0 }, + ); + + await expect( + runner.using( + "resource:1", + LOCK_DURATION_MS, + new AbortController().signal, + async () => { + throw error; + }, + ), + ).rejects.toBe(error); + expect(using).toHaveBeenCalledOnce(); + }); + + test("immediately propagates non-contention acquisition errors", async () => { + const error = executionError(new Error("Redis unavailable")); + const using = vi.fn().mockRejectedValue(error); + const runner = new RedlockExecutionLockRunner( + { using } as unknown as Redlock, + { retryDelayMs: 0, retryJitterMs: 0 }, + ); + + await expect( + runner.using( + "resource:1", + LOCK_DURATION_MS, + new AbortController().signal, + async () => "unreachable", + ), + ).rejects.toBe(error); + expect(using).toHaveBeenCalledOnce(); + }); + + test("stops waiting for a contended lock during shutdown", async () => { + const using = vi.fn().mockRejectedValue(contentionError()); + const runner = new RedlockExecutionLockRunner( + { using } as unknown as Redlock, + { retryDelayMs: 60_000, retryJitterMs: 0 }, + ); + const shutdownController = new AbortController(); + + const result = runner.using( + "resource:1", + LOCK_DURATION_MS, + shutdownController.signal, + async () => "unreachable", + ); + await vi.waitFor(() => expect(using).toHaveBeenCalledOnce()); + + shutdownController.abort(); + + await expect(result).rejects.toBe(shutdownController.signal.reason); + expect(using).toHaveBeenCalledOnce(); + }); + + test("passes lease loss to the routine and rejects a successful return", async () => { + const lockController = new AbortController(); + const signal = lockSignal(lockController); + const leaseError = new Error("Unable to extend lock"); + signal.error = leaseError; + const using = vi.fn(async (_resources, _duration, _settings, routine) => + routine(signal), + ); + const runner = new RedlockExecutionLockRunner({ + using, + } as unknown as Redlock); + let workloadSignal: AbortSignal | undefined; + + const result = runner.using( + "resource:1", + LOCK_DURATION_MS, + new AbortController().signal, + async (combinedSignal) => { + workloadSignal = combinedSignal; + await new Promise((resolve) => { + combinedSignal.addEventListener("abort", () => resolve(), { + once: true, + }); + }); + return "must not succeed"; + }, + ); + await vi.waitFor(() => expect(workloadSignal).toBeDefined()); + + lockController.abort(); + + await expect(result).rejects.toBe(leaseError); + expect(workloadSignal?.aborted).toBe(true); + expect(workloadSignal?.reason).toBe(leaseError); + }); + + test("validates resource names and lock durations", async () => { + const using = vi.fn(); + const runner = new RedlockExecutionLockRunner({ + using, + } as unknown as Redlock); + const signal = new AbortController().signal; + + await expect( + runner.using("", LOCK_DURATION_MS, signal, async () => undefined), + ).rejects.toThrow("resource must not be empty"); + await expect( + runner.using("resource:1", 20_099, signal, async () => undefined), + ).rejects.toThrow("greater than or equal to 20100ms"); + expect(using).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/backend/src/executionLock.ts b/packages/backend/src/executionLock.ts new file mode 100644 index 000000000..1ccd3d284 --- /dev/null +++ b/packages/backend/src/executionLock.ts @@ -0,0 +1,222 @@ +/// + +import { createLogger } from "@sourcebot/shared"; +import type { Redis } from "ioredis"; +import Redlock, { + ExecutionError, + ResourceLockedError, + type RedlockAbortSignal, +} from "redlock"; + +const LOG_TAG = "execution-lock"; +const logger = createLogger(LOG_TAG); + +const REDLOCK_RETRY_DELAY_MS = 250; +const REDLOCK_RETRY_JITTER_MS = 100; +const REDLOCK_AUTOMATIC_EXTENSION_THRESHOLD_MS = 20_000; + +const REDLOCK_SETTINGS = { + driftFactor: 0.01, + retryCount: 0, + retryDelay: REDLOCK_RETRY_DELAY_MS, + retryJitter: REDLOCK_RETRY_JITTER_MS, + automaticExtensionThreshold: REDLOCK_AUTOMATIC_EXTENSION_THRESHOLD_MS, +} as const; + +type RedlockUsingClient = Pick; + +export interface ExecutionLockRunner { + using( + resource: string, + durationMs: number, + shutdownSignal: AbortSignal, + routine: (signal: AbortSignal) => Promise, + ): Promise; +} + +interface RedlockExecutionLockRunnerOptions { + retryDelayMs?: number; + retryJitterMs?: number; + random?: () => number; +} + +const abortReason = (signal: AbortSignal): Error => { + if (signal.reason instanceof Error) { + return signal.reason; + } + + return new Error( + signal.reason === undefined + ? "The operation was aborted" + : String(signal.reason), + ); +}; + +const abortableDelay = async ( + durationMs: number, + signal: AbortSignal, +): Promise => { + if (signal.aborted) { + throw abortReason(signal); + } + + await new Promise((resolve, reject) => { + const onAbort = () => { + clearTimeout(timeout); + reject(abortReason(signal)); + }; + const timeout = setTimeout(() => { + signal.removeEventListener("abort", onAbort); + resolve(); + }, durationMs); + + signal.addEventListener("abort", onAbort, { once: true }); + }); +}; + +const combineAbortSignals = ( + shutdownSignal: AbortSignal, + lockSignal: RedlockAbortSignal, +): { signal: AbortSignal; dispose: () => void } => { + const controller = new AbortController(); + const abortFromShutdown = () => { + controller.abort(abortReason(shutdownSignal)); + }; + const abortFromLock = () => { + controller.abort(lockSignal.error ?? abortReason(lockSignal)); + }; + + shutdownSignal.addEventListener("abort", abortFromShutdown, { + once: true, + }); + lockSignal.addEventListener("abort", abortFromLock, { once: true }); + + if (lockSignal.aborted) { + abortFromLock(); + } else if (shutdownSignal.aborted) { + abortFromShutdown(); + } + + return { + signal: controller.signal, + dispose: () => { + shutdownSignal.removeEventListener("abort", abortFromShutdown); + lockSignal.removeEventListener("abort", abortFromLock); + }, + }; +}; + +const isLockContention = async (error: unknown): Promise => { + if (!(error instanceof ExecutionError) || error.attempts.length === 0) { + return false; + } + + const attempts = await Promise.all(error.attempts); + return attempts.every( + ({ votesAgainst }) => + votesAgainst.size > 0 && + [...votesAgainst.values()].every( + (attemptError) => attemptError instanceof ResourceLockedError, + ), + ); +}; + +export class RedlockExecutionLockRunner implements ExecutionLockRunner { + private readonly retryDelayMs: number; + private readonly retryJitterMs: number; + private readonly random: () => number; + + constructor( + private readonly redlock: RedlockUsingClient, + options: RedlockExecutionLockRunnerOptions = {}, + ) { + this.retryDelayMs = options.retryDelayMs ?? REDLOCK_RETRY_DELAY_MS; + this.retryJitterMs = options.retryJitterMs ?? REDLOCK_RETRY_JITTER_MS; + this.random = options.random ?? Math.random; + } + + async using( + resource: string, + durationMs: number, + shutdownSignal: AbortSignal, + routine: (signal: AbortSignal) => Promise, + ): Promise { + if (resource.length === 0) { + throw new Error("Execution lock resource must not be empty"); + } + if ( + !Number.isInteger(durationMs) || + durationMs < REDLOCK_AUTOMATIC_EXTENSION_THRESHOLD_MS + 100 + ) { + throw new Error( + `Execution lock duration must be an integer greater than or equal to ${REDLOCK_AUTOMATIC_EXTENSION_THRESHOLD_MS + 100}ms`, + ); + } + + while (true) { + if (shutdownSignal.aborted) { + throw abortReason(shutdownSignal); + } + + let acquired = false; + try { + return await this.redlock.using( + [resource], + durationMs, + { retryCount: 0 }, + async (lockSignal) => { + acquired = true; + const combined = combineAbortSignals( + shutdownSignal, + lockSignal, + ); + + try { + if (combined.signal.aborted) { + throw abortReason(combined.signal); + } + + const result = await routine(combined.signal); + + if (combined.signal.aborted) { + throw abortReason(combined.signal); + } + + return result; + } finally { + combined.dispose(); + } + }, + ); + } catch (error) { + if (acquired || !(await isLockContention(error))) { + throw error; + } + + const delayMs = Math.max( + 0, + this.retryDelayMs + + Math.floor( + (this.random() * 2 - 1) * this.retryJitterMs, + ), + ); + await abortableDelay(delayMs, shutdownSignal); + } + } + } +} + +export const createExecutionLockRunner = ( + redis: Redis, +): ExecutionLockRunner => { + const redlock = new Redlock([redis], REDLOCK_SETTINGS); + redlock.on("error", (error: unknown) => { + if (error instanceof ResourceLockedError) { + return; + } + + logger.error("Unexpected Redlock error", error); + }); + + return new RedlockExecutionLockRunner(redlock); +}; diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index df2a18993..0b19cb082 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -2,22 +2,25 @@ import "./instrument.js"; import * as Sentry from "@sentry/node"; import { createLogger, env, getConfigSettings } from "@sourcebot/shared"; -import { prisma } from "./prisma.js"; import 'express-async-errors'; import { existsSync } from 'fs'; import { mkdir } from 'fs/promises'; -import { Api } from "./api.js"; -import { AttachmentPruner } from "./attachmentPruner.js"; import { ConfigManager } from "./configManager.js"; -import { ConnectionManager } from './connectionManager.js'; import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, SHUTDOWN_SIGNALS } from './constants.js'; -import { AccountPermissionSyncer } from "./ee/accountPermissionSyncer.js"; -import { AuditLogPruner } from "./ee/auditLogPruner.js"; -import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js'; +import { BullMQJobManager } from "./jobManager.js"; import { shutdownPosthog } from "./posthog.js"; +import { prisma } from "./prisma.js"; import { PromClient } from './promClient.js'; -import { RepoIndexManager } from "./repoIndexManager.js"; import { redis } from "./redis.js"; +import { createConnectionWorkload } from "./connectionWorkload.js"; +import { cleanupOrphanedRepoResources, createRepoIndexWorkload } from "./repoIndexWorkload.js"; +import { Api } from "./api.js"; +import { createAccountPermissionSyncWorkload } from "./ee/accountPermissionSyncWorkload.js"; +import { createRepoPermissionSyncWorkload } from "./ee/repoPermissionSyncWorkload.js"; +import { reconcileJobSchedulersAtStartup } from "./reconcileJobSchedulersAtStartup.js"; +import { hasEntitlement } from "./entitlements.js"; +import { createAttachmentPruneWorkload } from "./attachmentPruneWorkload.js"; +import { createAuditLogPruneWorkload } from "./ee/auditLogPruneWorkload.js"; const logger = createLogger('backend-entrypoint'); @@ -40,40 +43,70 @@ try { process.exit(1); } -const promClient = new PromClient(); const settings = await getConfigSettings(env.CONFIG_PATH); +const permissionSyncEnabled = + env.PERMISSION_SYNC_ENABLED === "true" && + (await hasEntitlement("permission-syncing")); -const connectionManager = new ConnectionManager(prisma, settings, redis, promClient); -const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis); -const accountPermissionSyncer = new AccountPermissionSyncer(prisma, settings, redis); -const repoIndexManager = new RepoIndexManager(prisma, settings, redis, promClient); -const configManager = new ConfigManager(prisma, connectionManager, env.CONFIG_PATH); -const auditLogPruner = new AuditLogPruner(prisma); -const attachmentPruner = new AttachmentPruner(prisma); - -connectionManager.startScheduler(); -await repoIndexManager.startScheduler(); -auditLogPruner.startScheduler(); -attachmentPruner.startScheduler(); - -if (env.PERMISSION_SYNC_ENABLED === 'true') { - if (env.PERMISSION_SYNC_REPO_DRIVEN_ENABLED === 'true') { - await repoPermissionSyncer.startScheduler(); - } - await accountPermissionSyncer.startScheduler(); -} - -const api = new Api( - promClient, - prisma, - connectionManager, - repoIndexManager, - accountPermissionSyncer, -); +const promClient = new PromClient(); logger.info('Worker started.'); +const jobManager = new BullMQJobManager(redis); + +const connectionWorkload = createConnectionWorkload({ + db: prisma, + jobManager, + permissionSyncEnabled, + settings, +}); +const repoIndexWorkload = createRepoIndexWorkload({ + db: prisma, + settings, +}); +const accountPermissionSyncWorkload = createAccountPermissionSyncWorkload({ + db: prisma, + settings, +}); +const repoPermissionSyncWorkload = createRepoPermissionSyncWorkload({ + db: prisma, + settings, +}); +const attachmentPruneWorkload = createAttachmentPruneWorkload({ + db: prisma, + ttlHours: env.SOURCEBOT_CHAT_ATTACHMENT_ORPHAN_TTL_HOURS, +}); +const auditLogPruneWorkload = createAuditLogPruneWorkload({ + db: prisma, + enabled: env.SOURCEBOT_EE_AUDIT_LOGGING_ENABLED === "true", + retentionDays: env.SOURCEBOT_EE_AUDIT_RETENTION_DAYS, +}); + +jobManager.register(connectionWorkload); +jobManager.register(repoIndexWorkload); +jobManager.register(accountPermissionSyncWorkload); +jobManager.register(repoPermissionSyncWorkload); +jobManager.register(attachmentPruneWorkload); +jobManager.register(auditLogPruneWorkload); + +const api = new Api(promClient, prisma, jobManager); + +await cleanupOrphanedRepoResources(prisma); + +const configManager = new ConfigManager(jobManager, env.CONFIG_PATH); +await configManager.syncConfig(); + +await reconcileJobSchedulersAtStartup({ + db: prisma, + jobManager, + permissionSyncEnabled, + settings, +}); + +await jobManager.start(); + + const listenToShutdownSignals = () => { const signals = SHUTDOWN_SIGNALS; @@ -88,13 +121,8 @@ const listenToShutdownSignals = () => { logger.info(`Received ${signal}, cleaning up...`); - await repoIndexManager.dispose() - await connectionManager.dispose() - await repoPermissionSyncer.dispose() - await accountPermissionSyncer.dispose() - await auditLogPruner.dispose() - await attachmentPruner.dispose() await configManager.dispose() + await jobManager.stop(); await prisma.$disconnect(); await redis.quit(); diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts new file mode 100644 index 000000000..e2259d2bd --- /dev/null +++ b/packages/backend/src/jobManager.test.ts @@ -0,0 +1,398 @@ +import { Redis } from "ioredis"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { ProcessContext, Workload } from "./types.js"; + +const mocks = vi.hoisted(() => { + const jobLogger = { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + flush: vi.fn(), + }; + return { + enqueue: vi.fn(), + upsertJobScheduler: vi.fn(), + getJobSchedulerIds: vi.fn(), + removeJobScheduler: vi.fn(), + producerClose: vi.fn(), + workerClose: vi.fn(), + executionLockUsing: vi.fn(), + jobLogger, + createBullMQJobLogger: vi.fn(() => jobLogger), + workers: [] as Array<{ + processor: (job: unknown) => Promise; + handlers: Map void>; + }>, + }; +}); + +// The module under test creates a logger at import time; stub it so importing pure helpers +// has no side effects (mirrors repoIndexManager.test.ts). +vi.mock("@sourcebot/shared", () => ({ + createLogger: vi.fn(() => ({ + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + })), + createBullMQJobLogger: mocks.createBullMQJobLogger, + scheduleToMs: vi.fn((schedule: string | number) => + typeof schedule === "number" ? schedule : 300_000, + ), + BullMQClient: class { + enqueue = mocks.enqueue; + upsertJobScheduler = mocks.upsertJobScheduler; + getJobSchedulerIds = mocks.getJobSchedulerIds; + removeJobScheduler = mocks.removeJobScheduler; + close = mocks.producerClose; + getQueue = vi.fn(() => ({ + getJobCounts: vi.fn(), + upsertJobScheduler: vi.fn(), + })); + }, +})); + +// Mock the constants module directly so its env-derived cache-dir paths don't load. +vi.mock("./constants.js", () => ({ + WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, +})); + +vi.mock("@sentry/node", () => ({ + captureException: vi.fn(), +})); + +vi.mock("./executionLock.js", () => ({ + createExecutionLockRunner: vi.fn(() => ({ + using: mocks.executionLockUsing, + })), +})); + +vi.mock("bullmq", () => ({ + Worker: class { + private readonly record: (typeof mocks.workers)[number]; + + constructor( + _name: string, + processor: (job: unknown) => Promise, + ) { + this.record = { processor, handlers: new Map() }; + mocks.workers.push(this.record); + } + + on(event: string, handler: (...args: unknown[]) => void) { + this.record.handlers.set(event, handler); + } + + close = mocks.workerClose; + }, +})); + +import { BullMQJobManager } from "./jobManager.js"; + +const createWorkload = ( + overrides: Partial> = {}, +): Workload<"connection-sync", { repoCount: number }> => ({ + queueSpec: { + name: "connection-sync", + dedupKey: ({ connectionId }) => `connection:${connectionId}`, + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keepJobs: { + completed: { count: 50 }, + failed: { count: 50 }, + }, + keepLogs: 500, + }, + }, + concurrency: 2, + process: vi.fn(async () => ({ repoCount: 3 })), + ...overrides, +}); + +const data = { connectionId: 42 }; +const job = { + id: "job-1", + queueName: "connection-sync", + data, + attemptsMade: 2, + opts: { attempts: 2 }, + log: vi.fn(), + updateProgress: vi.fn(), +}; + +describe("BullMQJobManager lifecycle", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.workers.length = 0; + mocks.enqueue.mockResolvedValue("job-1"); + mocks.upsertJobScheduler.mockResolvedValue("scheduled-job-1"); + mocks.getJobSchedulerIds.mockResolvedValue([]); + mocks.removeJobScheduler.mockResolvedValue(true); + mocks.workerClose.mockResolvedValue(undefined); + mocks.producerClose.mockResolvedValue(undefined); + mocks.executionLockUsing.mockImplementation( + async (_resource, _durationMs, _shutdownSignal, routine) => + routine(new AbortController().signal), + ); + }); + + test("delegates enqueueing to BullMQClient and returns its job id", async () => { + const manager = new BullMQJobManager({} as Redis); + const workload = createWorkload(); + manager.register(workload); + + const result = await manager.trigger("connection-sync", data, { + priority: 1, + }); + + expect(result).toBe("job-1"); + expect(mocks.enqueue).toHaveBeenCalledWith(workload.queueSpec, data, { + priority: 1, + }); + }); + + test("manages schedulers through the registered workload", async () => { + const manager = new BullMQJobManager({} as Redis); + const workload = createWorkload(); + manager.register(workload); + + await expect( + manager.upsertJobScheduler( + "connection-sync", + "connection-sync-v1-42", + 60_000, + data, + { priority: 10 }, + ), + ).resolves.toBe("scheduled-job-1"); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + workload.queueSpec, + "connection-sync-v1-42", + 60_000, + data, + { priority: 10 }, + ); + + mocks.getJobSchedulerIds.mockResolvedValue(["connection-sync-v1-42"]); + await expect( + manager.getJobSchedulerIds("connection-sync"), + ).resolves.toEqual(["connection-sync-v1-42"]); + expect(mocks.getJobSchedulerIds).toHaveBeenCalledWith( + workload.queueSpec, + ); + + await expect( + manager.removeJobScheduler( + "connection-sync", + "connection-sync-v1-42", + ), + ).resolves.toBe(true); + expect(mocks.removeJobScheduler).toHaveBeenCalledWith( + workload.queueSpec, + "connection-sync-v1-42", + ); + }); + + test("upserts a declared workload schedule when starting", async () => { + const manager = new BullMQJobManager({} as Redis); + const workload = createWorkload({ + schedule: { + interval: "5m", + data, + options: { priority: 10 }, + }, + }); + manager.register(workload); + + await manager.start(); + + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + workload.queueSpec, + "schedule:connection-sync", + "5m", + data, + { priority: 10 }, + ); + }); + + test("closes workers and producer queues when stopping", async () => { + const manager = new BullMQJobManager({} as Redis); + manager.register(createWorkload()); + await manager.start(); + + await manager.stop(); + + expect(mocks.workerClose).toHaveBeenCalledOnce(); + expect(mocks.producerClose).toHaveBeenCalledOnce(); + }); + + test("calls onStarted before processing and onCompleted after completion", async () => { + const calls: string[] = []; + const workload = createWorkload({ + onStarted: vi.fn(async ({ logger }) => { + logger.info("Lifecycle started"); + calls.push("started"); + }), + process: vi.fn(async () => { + calls.push("processed"); + return { repoCount: 3 }; + }), + onCompleted: vi.fn(async ({ logger }) => { + logger.info("Lifecycle completed"); + calls.push("completed"); + }), + }); + const manager = new BullMQJobManager({} as Redis); + manager.register(workload); + await manager.start(); + + const result = await mocks.workers[0].processor({ + ...job, + attemptsMade: 0, + }); + expect(calls).toEqual(["started", "processed"]); + + mocks.workers[0].handlers.get("completed")?.(job, result); + await vi.waitFor(() => + expect(calls).toEqual(["started", "processed", "completed"]), + ); + expect(workload.onCompleted).toHaveBeenCalledWith( + expect.objectContaining({ + data, + jobId: "job-1", + maxAttempts: 2, + logger: mocks.jobLogger, + }), + { repoCount: 3 }, + ); + expect(mocks.jobLogger.info).toHaveBeenCalledWith("Lifecycle started"); + expect(mocks.jobLogger.info).toHaveBeenCalledWith( + "Lifecycle completed", + ); + await vi.waitFor(() => { + expect(mocks.jobLogger.flush).toHaveBeenCalledTimes(2); + }); + }); + + test("runs onStarted and processing while the execution lock is held", async () => { + const calls: string[] = []; + const workloadSignal = new AbortController().signal; + mocks.executionLockUsing.mockImplementation( + async (_resource, _durationMs, _shutdownSignal, routine) => { + calls.push("lock-acquired"); + const result = await routine(workloadSignal); + calls.push("lock-released"); + return result; + }, + ); + const workload = createWorkload({ + executionLock: { + resource: ({ connectionId }) => + `sourcebot:lock:connection:${connectionId}`, + durationMs: 60_000, + }, + onStarted: vi.fn(async () => { + calls.push("started"); + }), + process: vi.fn(async ({ signal }) => { + expect(signal).toBe(workloadSignal); + calls.push("processed"); + return { repoCount: 3 }; + }), + }); + const manager = new BullMQJobManager({} as Redis); + manager.register(workload); + await manager.start(); + + await expect( + mocks.workers[0].processor({ ...job, attemptsMade: 0 }), + ).resolves.toEqual({ repoCount: 3 }); + + expect(calls).toEqual([ + "lock-acquired", + "started", + "processed", + "lock-released", + ]); + expect(mocks.executionLockUsing).toHaveBeenCalledWith( + "sourcebot:lock:connection:42", + 60_000, + expect.any(AbortSignal), + expect.any(Function), + ); + expect(mocks.jobLogger.debug).toHaveBeenCalledWith( + "Acquired workload execution lock", + { + resource: "sourcebot:lock:connection:42", + lockWaitMs: expect.any(Number), + }, + ); + expect(mocks.jobLogger.debug).toHaveBeenCalledWith( + "Finished work protected by execution lock", + { + resource: "sourcebot:lock:connection:42", + lockHeldMs: expect.any(Number), + }, + ); + }); + + test("provides the structured job logger to the workload processor", async () => { + const process = vi.fn( + async (context: ProcessContext<"connection-sync">) => { + context.logger.info("Processing connection"); + return { repoCount: 3 }; + }, + ); + const manager = new BullMQJobManager({} as Redis); + manager.register(createWorkload({ process })); + await manager.start(); + + await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); + + expect(process).toHaveBeenCalledWith( + expect.objectContaining({ + logger: mocks.jobLogger, + }), + ); + expect(mocks.jobLogger.info).toHaveBeenCalledWith( + "Processing connection", + ); + expect(mocks.jobLogger.flush).toHaveBeenCalled(); + }); + + test("reports lifecycle metadata after terminal failure", async () => { + const onTerminalFailure = vi.fn(async ({ logger }) => { + logger.error("Lifecycle failed"); + }); + const workload = createWorkload({ onTerminalFailure }); + const manager = new BullMQJobManager({} as Redis); + manager.register(workload); + await manager.start(); + + const error = new Error("failed"); + mocks.workers[0].handlers.get("failed")?.(job, error); + + await vi.waitFor(() => { + expect(onTerminalFailure).toHaveBeenCalledWith( + expect.objectContaining({ + data, + jobId: "job-1", + attemptsMade: 2, + maxAttempts: 2, + logger: mocks.jobLogger, + }), + error, + ); + }); + expect(mocks.jobLogger.error).toHaveBeenCalledWith("Lifecycle failed"); + await vi.waitFor(() => { + expect(mocks.jobLogger.flush).toHaveBeenCalledOnce(); + }); + expect(mocks.createBullMQJobLogger).toHaveBeenCalledWith( + expect.objectContaining({ id: "job-1", attemptsMade: 2 }), + expect.objectContaining({ attempt: 2 }), + ); + }); +}); diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts new file mode 100644 index 000000000..683a0fb38 --- /dev/null +++ b/packages/backend/src/jobManager.ts @@ -0,0 +1,352 @@ +import * as Sentry from "@sentry/node"; +import { + BullMQClient, + createBullMQJobLogger, + createLogger, + DataOf, + JobEnqueueOptions, + JobLogSink, + QueueName, + Schedule, + scheduleToMs, +} from "@sourcebot/shared"; +import { Job, Queue, Worker } from "bullmq"; +import { Redis } from "ioredis"; +import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; +import { createExecutionLockRunner } from "./executionLock.js"; +import type { ExecutionLockRunner } from "./executionLock.js"; +import { JobLifecycleContext, Workload } from "./types.js"; +import type { JobManager } from "./types.js"; +import { prisma } from "./prisma.js"; + +const LOG_TAG = "job-manager"; +const logger = createLogger(LOG_TAG); + +export class BullMQJobManager implements JobManager { + private readonly workloads = new Map< + string, + Workload + >(); + private readonly workers = new Map(); + private readonly bullmqClient: BullMQClient; + private readonly abortController = new AbortController(); + private readonly executionLockRunner: ExecutionLockRunner; + + constructor(private readonly connection: Redis) { + this.bullmqClient = new BullMQClient(connection); + this.executionLockRunner = createExecutionLockRunner(connection); + } + + register(workload: Workload): void { + const name = workload.queueSpec.name; + if (this.workloads.has(name)) { + throw new Error(`Workload "${name}" is already registered`); + } + this.workloads.set(name, workload); + } + + getQueues(): Queue[] { + return [...this.workloads.values()].map((workload) => + this.bullmqClient.getQueue(workload.queueSpec), + ); + } + + async start(): Promise { + if (this.workloads.size === 0) { + logger.debug( + "start() called with nothing registered; nothing to do", + ); + return; + } + + for (const workload of this.workloads.values()) { + await this.startWorkload(workload); + } + + logger.info( + `Started ${this.workloads.size} workload(s) [${[...this.workloads.keys()].join(", ")}]`, + ); + } + + async trigger( + workloadName: TName, + data: DataOf, + options?: JobEnqueueOptions, + ): Promise { + const workload = this.getWorkload(workloadName); + return this.bullmqClient.enqueue(workload.queueSpec, data, options); + } + + async upsertJobScheduler( + workloadName: TName, + schedulerId: string, + schedule: Schedule, + data: DataOf, + options?: JobEnqueueOptions, + ): Promise { + const workload = this.getWorkload(workloadName); + return this.bullmqClient.upsertJobScheduler( + workload.queueSpec, + schedulerId, + schedule, + data, + options, + ); + } + + async getJobSchedulerIds( + workloadName: TName, + ): Promise { + const workload = this.getWorkload(workloadName); + return this.bullmqClient.getJobSchedulerIds(workload.queueSpec); + } + + async removeJobScheduler( + workloadName: TName, + schedulerId: string, + ): Promise { + const workload = this.getWorkload(workloadName); + return this.bullmqClient.removeJobScheduler( + workload.queueSpec, + schedulerId, + ); + } + + async stop(): Promise { + this.abortController.abort(); + + await Promise.all( + [...this.workers.values()].map((worker) => + Promise.race([ + worker.close(), + new Promise((resolve) => + setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS), + ), + ]), + ), + ); + + await this.bullmqClient.close(); + + logger.info("Job manager stopped"); + } + + private async startWorkload( + workload: Workload, + ): Promise { + const { + queueSpec: spec, + concurrency, + executionLock, + rateLimit, + schedule, + } = workload; + + const worker = new Worker( + spec.name, + async (job) => { + const jobLogger = createBullMQJobLogger(job, { + label: `${LOG_TAG}:${spec.name}:job:${job.id ?? "unknown"}`, + }); + const lifecycleContext = this.jobLifecycleContext( + job, + jobLogger, + ); + + const process = async (signal: AbortSignal) => { + await workload.onStarted?.(lifecycleContext); + return workload.process({ + ...lifecycleContext, + signal, + updateProgress: (progress) => + job.updateProgress(progress), + trigger: (target, data, options) => + this.trigger(target, data, options), + }); + }; + + try { + if (executionLock) { + const resource = executionLock.resource(job.data); + const waitStartedAt = Date.now(); + return await this.executionLockRunner.using( + resource, + executionLock.durationMs, + this.abortController.signal, + async (signal) => { + const acquiredAt = Date.now(); + jobLogger.debug( + "Acquired workload execution lock", + { + resource, + lockWaitMs: acquiredAt - waitStartedAt, + }, + ); + + try { + return await process(signal); + } finally { + jobLogger.debug( + "Finished work protected by execution lock", + { + resource, + lockHeldMs: Date.now() - acquiredAt, + }, + ); + } + }, + ); + } + + return await process(this.abortController.signal); + } catch (error) { + jobLogger.error( + `Workload "${spec.name}" attempt failed`, + error, + ); + throw error; + } finally { + await jobLogger.flush(); + } + }, + { + connection: this.connection, + concurrency, + maxStalledCount: 1, + ...(rateLimit + ? { + limiter: { + max: rateLimit.max, + duration: scheduleToMs(rateLimit.per), + }, + } + : {}), + }, + ); + + worker.on("failed", (job, error) => { + void this.onWorkloadJobFailed(workload, job, error); + }); + worker.on("completed", (job, result) => { + void this.onWorkloadJobCompleted(workload, job, result); + }); + worker.on("error", (error) => { + logger.error(`Worker "${spec.name}" error:`, error); + }); + + this.workers.set(spec.name, worker); + + if (schedule) { + // @note: jobs produced by BullMQ's scheduler bypass the deduplication check that + // `Queue.add` goes through, so a dedup key would be silently ignored here. The + // next tick's job is only created once the current one goes active, so at most one + // run is ever queued behind the one in flight. + await this.upsertJobScheduler( + spec.name, + `schedule:${spec.name}`, + schedule.interval, + schedule.data, + schedule.options, + ); + } + } + + private getWorkload( + workloadName: TName, + ): Workload { + const workload = this.workloads.get(workloadName) as + | Workload + | undefined; + if (!workload) { + throw new Error(`Unknown workload "${workloadName}"`); + } + return workload; + } + + private async onWorkloadJobFailed( + workload: Workload, + job: Job | undefined, + error: Error, + ): Promise { + if (!job) { + return; + } + const maxAttempts = job.opts.attempts ?? 1; + const isTerminal = job.attemptsMade >= maxAttempts; + if (!isTerminal) { + logger.warn( + `Workload "${workload.queueSpec.name}" job ${job.id} failed attempt ${job.attemptsMade}/${maxAttempts}; will retry: ${error.message}`, + ); + return; + } + logger.error( + `Workload "${workload.queueSpec.name}" job ${job.id} failed terminally after ${job.attemptsMade} attempt(s): ${error.message}`, + ); + + const jobLogger = createBullMQJobLogger(job, { + label: `${LOG_TAG}:${workload.queueSpec.name}:job:${job.id ?? "unknown"}`, + attempt: Math.max(job.attemptsMade, 1), + }); + try { + await workload.onTerminalFailure?.( + this.jobLifecycleContext(job, jobLogger), + error, + ); + } catch (hookError) { + Sentry.captureException(hookError); + jobLogger.error( + `onTerminalFailure for workload "${workload.queueSpec.name}" threw`, + hookError, + ); + logger.error( + `onTerminalFailure for workload "${workload.queueSpec.name}" threw:`, + hookError, + ); + } finally { + await jobLogger.flush(); + } + } + + private async onWorkloadJobCompleted( + workload: Workload, + job: Job, + result: TResult, + ): Promise { + const jobLogger = createBullMQJobLogger(job, { + label: `${LOG_TAG}:${workload.queueSpec.name}:job:${job.id ?? "unknown"}`, + attempt: Math.max(job.attemptsMade, 1), + }); + try { + await workload.onCompleted?.( + this.jobLifecycleContext(job, jobLogger), + result, + ); + } catch (hookError) { + Sentry.captureException(hookError); + jobLogger.error( + `onCompleted for workload "${workload.queueSpec.name}" threw`, + hookError, + ); + logger.error( + `onCompleted for workload "${workload.queueSpec.name}" threw:`, + hookError, + ); + } finally { + await jobLogger.flush(); + } + } + + private jobLifecycleContext( + job: Job, + logger: JobLogSink, + ): JobLifecycleContext { + return { + data: job.data, + jobId: job.id ?? "", + attemptsMade: job.attemptsMade, + maxAttempts: job.opts.attempts ?? 1, + prisma, + logger, + }; + } +} diff --git a/packages/backend/src/reconcileJobSchedulersAtStartup.test.ts b/packages/backend/src/reconcileJobSchedulersAtStartup.test.ts new file mode 100644 index 000000000..810544242 --- /dev/null +++ b/packages/backend/src/reconcileJobSchedulersAtStartup.test.ts @@ -0,0 +1,166 @@ +import type { PrismaClient } from "@sourcebot/db"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { reconcileJobSchedulersAtStartup } from "./reconcileJobSchedulersAtStartup.js"; +import type { JobManager } from "./types.js"; + +const mocks = { + accountFindMany: vi.fn(), + connectionFindMany: vi.fn(), + repoFindMany: vi.fn(), + getJobSchedulerIds: vi.fn(), + upsertJobScheduler: vi.fn(), + removeJobScheduler: vi.fn(), +}; + +const db = { + account: { + findMany: mocks.accountFindMany, + }, + connection: { + findMany: mocks.connectionFindMany, + }, + repo: { + findMany: mocks.repoFindMany, + }, +} as unknown as PrismaClient; + +const jobManager = mocks as unknown as JobManager; + +describe("reconcileJobSchedulersAtStartup", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.accountFindMany.mockResolvedValue([ + { id: "account-1" }, + { id: "account-2" }, + ]); + mocks.connectionFindMany.mockResolvedValue([{ id: 1 }, { id: 2 }]); + mocks.repoFindMany.mockResolvedValue([{ id: 42 }, { id: 84 }]); + mocks.getJobSchedulerIds.mockResolvedValue([]); + mocks.upsertJobScheduler.mockResolvedValue("scheduled-job"); + mocks.removeJobScheduler.mockResolvedValue(true); + }); + + test("reconciles connection, repository, and permission schedulers", async () => { + await reconcileJobSchedulersAtStartup({ + db, + jobManager, + permissionSyncEnabled: true, + settings: { + resyncConnectionIntervalMs: 86_400_000, + reindexIntervalMs: 3_600_000, + userDrivenPermissionSyncIntervalMs: 43_200_000, + repoDrivenPermissionSyncIntervalMs: 21_600_000, + }, + }); + + expect(mocks.connectionFindMany).toHaveBeenCalledWith({ + select: { id: true }, + }); + expect(mocks.repoFindMany).toHaveBeenCalledWith({ + select: { id: true }, + }); + expect(mocks.accountFindMany).toHaveBeenCalledWith({ + where: { + providerType: { + in: [ + "github", + "gitlab", + "bitbucket-cloud", + "bitbucket-server", + ], + }, + }, + select: { id: true }, + }); + expect(mocks.repoFindMany).toHaveBeenCalledWith({ + where: { + isPublic: false, + external_codeHostType: { + in: [ + "github", + "gitlab", + "bitbucketCloud", + "bitbucketServer", + ], + }, + connections: { + some: { + connection: { + enforcePermissions: true, + }, + }, + }, + }, + select: { id: true }, + }); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "connection-sync", + "connection-sync-v1-1", + 86_400_000, + { connectionId: 1 }, + { priority: 10 }, + ); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "repo-index", + "repo-index-v1-42", + 3_600_000, + { repoId: 42, type: "INDEX" }, + { priority: 10 }, + ); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "account-permission-sync", + "account-permission-sync-v1-account-1", + 43_200_000, + { accountId: "account-1" }, + { priority: 10 }, + ); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "repo-permission-sync", + "repo-permission-sync-v1-42", + 21_600_000, + { repoId: 42 }, + { priority: 10 }, + ); + }); + + test("removes permission schedulers when permission syncing is disabled", async () => { + mocks.getJobSchedulerIds.mockImplementation(async (workloadName) => { + if (workloadName === "account-permission-sync") { + return ["account-permission-sync-v1-account-1"]; + } + if (workloadName === "repo-permission-sync") { + return ["repo-permission-sync-v1-42"]; + } + return []; + }); + + await reconcileJobSchedulersAtStartup({ + db, + jobManager, + permissionSyncEnabled: false, + settings: { + resyncConnectionIntervalMs: 86_400_000, + reindexIntervalMs: 3_600_000, + userDrivenPermissionSyncIntervalMs: 43_200_000, + repoDrivenPermissionSyncIntervalMs: 21_600_000, + }, + }); + + expect(mocks.accountFindMany).not.toHaveBeenCalled(); + expect(mocks.repoFindMany).toHaveBeenCalledOnce(); + expect(mocks.removeJobScheduler).toHaveBeenCalledWith( + "account-permission-sync", + "account-permission-sync-v1-account-1", + ); + expect(mocks.removeJobScheduler).toHaveBeenCalledWith( + "repo-permission-sync", + "repo-permission-sync-v1-42", + ); + expect(mocks.upsertJobScheduler).not.toHaveBeenCalledWith( + expect.stringContaining("permission-sync"), + expect.anything(), + expect.anything(), + expect.anything(), + ); + }); +}); diff --git a/packages/backend/src/reconcileJobSchedulersAtStartup.ts b/packages/backend/src/reconcileJobSchedulersAtStartup.ts new file mode 100644 index 000000000..6917388af --- /dev/null +++ b/packages/backend/src/reconcileJobSchedulersAtStartup.ts @@ -0,0 +1,175 @@ +import type { PrismaClient } from "@sourcebot/db"; +import { + ACCOUNT_PERMISSION_SYNC_SCHEDULER_ID_PREFIX, + getAccountPermissionSyncSchedulerId, + JOB_PRIORITIES, +} from "@sourcebot/shared"; +import type { + DataOf, + JobEnqueueOptions, + QueueName, + Schedule, +} from "@sourcebot/shared"; +import { + ACCOUNT_PERMISSION_SYNC_WHERE, + REPO_PERMISSION_SYNC_WHERE, +} from "./ee/permissionSyncEligibility.js"; +import type { JobManager, Settings } from "./types.js"; + +interface SchedulerTarget { + schedulerId: string; + data: DataOf; +} + +interface ReconcileOptions { + jobManager: JobManager; + workloadName: TName; + schedulerIdPrefix: string; + targets: SchedulerTarget[]; + schedule: Schedule; + jobOptions?: JobEnqueueOptions; +} + +const reconcileJobSchedulers = async ({ + jobManager, + workloadName, + schedulerIdPrefix, + targets, + schedule, + jobOptions, +}: ReconcileOptions): Promise => { + const existingSchedulerIds = new Set( + await jobManager.getJobSchedulerIds(workloadName), + ); + const desiredSchedulerIds = new Set( + targets.map(({ schedulerId }) => schedulerId), + ); + + await Promise.all( + targets.map(({ schedulerId, data }) => { + if (jobOptions) { + return jobManager.upsertJobScheduler( + workloadName, + schedulerId, + schedule, + data, + jobOptions, + ); + } + return jobManager.upsertJobScheduler( + workloadName, + schedulerId, + schedule, + data, + ); + }), + ); + + const obsoleteSchedulerIds = [...existingSchedulerIds].filter( + (schedulerId) => + schedulerId.startsWith(schedulerIdPrefix) && + !desiredSchedulerIds.has(schedulerId), + ); + await Promise.all( + obsoleteSchedulerIds.map((schedulerId) => + jobManager.removeJobScheduler(workloadName, schedulerId), + ), + ); +}; + +interface Props { + db: PrismaClient; + jobManager: JobManager; + settings: Pick< + Settings, + | "reindexIntervalMs" + | "repoDrivenPermissionSyncIntervalMs" + | "resyncConnectionIntervalMs" + | "userDrivenPermissionSyncIntervalMs" + >; + permissionSyncEnabled: boolean; +} + +export const reconcileJobSchedulersAtStartup = async ({ + db, + jobManager, + settings, + permissionSyncEnabled, +}: Props): Promise => { + const [connections, repos, accountsForPermissionSync, reposForPermissionSync] = + await Promise.all([ + db.connection.findMany({ + select: { + id: true, + }, + }), + db.repo.findMany({ + select: { + id: true, + }, + }), + permissionSyncEnabled + ? db.account.findMany({ + where: ACCOUNT_PERMISSION_SYNC_WHERE, + select: { + id: true, + }, + }) + : [], + permissionSyncEnabled + ? db.repo.findMany({ + where: REPO_PERMISSION_SYNC_WHERE, + select: { + id: true, + }, + }) + : [], + ]); + + await Promise.all([ + reconcileJobSchedulers({ + jobManager, + workloadName: "connection-sync", + schedulerIdPrefix: "connection-sync-v1-", + targets: connections.map(({ id }) => ({ + schedulerId: `connection-sync-v1-${id}`, + data: { connectionId: id }, + })), + schedule: settings.resyncConnectionIntervalMs, + jobOptions: { priority: JOB_PRIORITIES.SCHEDULED }, + }), + reconcileJobSchedulers({ + jobManager, + workloadName: "repo-index", + schedulerIdPrefix: "repo-index-v1-", + targets: repos.map(({ id }) => ({ + schedulerId: `repo-index-v1-${id}`, + data: { repoId: id, type: "INDEX" }, + })), + schedule: settings.reindexIntervalMs, + jobOptions: { priority: JOB_PRIORITIES.SCHEDULED }, + }), + reconcileJobSchedulers({ + jobManager, + workloadName: "account-permission-sync", + schedulerIdPrefix: ACCOUNT_PERMISSION_SYNC_SCHEDULER_ID_PREFIX, + targets: accountsForPermissionSync.map(({ id }) => ({ + schedulerId: getAccountPermissionSyncSchedulerId(id), + data: { accountId: id }, + })), + schedule: settings.userDrivenPermissionSyncIntervalMs, + jobOptions: { priority: JOB_PRIORITIES.SCHEDULED }, + }), + reconcileJobSchedulers({ + jobManager, + workloadName: "repo-permission-sync", + schedulerIdPrefix: "repo-permission-sync-v1-", + targets: reposForPermissionSync.map(({ id }) => ({ + schedulerId: `repo-permission-sync-v1-${id}`, + data: { repoId: id }, + })), + schedule: settings.repoDrivenPermissionSyncIntervalMs, + jobOptions: { priority: JOB_PRIORITIES.SCHEDULED }, + }), + ]); +}; diff --git a/packages/backend/src/repoIndexManager.test.ts b/packages/backend/src/repoIndexManager.test.ts deleted file mode 100644 index 684f1a826..000000000 --- a/packages/backend/src/repoIndexManager.test.ts +++ /dev/null @@ -1,912 +0,0 @@ -import type { PrismaClient, Repo } from '@sourcebot/db'; -import { RepoIndexingJobStatus, RepoIndexingJobType } from '@sourcebot/db'; -import type { Job } from 'bullmq'; -import type { Redis } from 'ioredis'; -import { afterEach, beforeEach, describe, expect, Mock, test, vi } from 'vitest'; -import type { RepoWithConnections, Settings } from './types.js'; - -// Mock modules before importing the class under test -vi.mock('@sentry/node', () => ({ - captureException: vi.fn(), -})); - -vi.mock('@sourcebot/shared', () => ({ - createLogger: vi.fn(() => ({ - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), - })), - env: { - DATA_CACHE_DIR: 'test-data', - REDIS_REMOVE_ON_COMPLETE: true, - REDIS_REMOVE_ON_FAIL: true, - }, - getRepoPath: vi.fn((repo: Repo) => ({ - path: `/test-data/repos/${repo.id}`, - isReadOnly: false, - })), - repoMetadataSchema: { - parse: vi.fn((metadata: unknown) => metadata ?? {}), - }, - repoIndexingJobMetadataSchema: { - parse: vi.fn((metadata: unknown) => metadata ?? {}), - }, -})); - -vi.mock('./constants.js', () => ({ - WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, - INDEX_CACHE_DIR: 'test-data/index', -})); - -vi.mock('./git.js', () => ({ - cloneRepository: vi.fn(), - fetchRepository: vi.fn(), - getBranches: vi.fn().mockResolvedValue([]), - getTags: vi.fn().mockResolvedValue([]), - getLocalDefaultBranch: vi.fn().mockResolvedValue('main'), - getCommitHashForRefName: vi.fn().mockResolvedValue('abc123'), - getLatestCommitTimestamp: vi.fn().mockResolvedValue(new Date()), - isPathAValidGitRepoRoot: vi.fn().mockResolvedValue(true), - isRepoEmpty: vi.fn().mockResolvedValue(false), - unsetGitConfig: vi.fn(), - upsertGitConfig: vi.fn(), - writeCommitGraph: vi.fn(), -})); - -vi.mock('./zoekt.js', () => ({ - indexGitRepository: vi.fn().mockResolvedValue({ stdout: '', stderr: '' }), -})); - -vi.mock('./posthog.js', () => ({ - captureEvent: vi.fn(), -})); - -vi.mock('./utils.js', () => ({ - getAuthCredentialsForRepo: vi.fn().mockResolvedValue(null), - getShardPrefix: vi.fn((orgId: number, repoId: number) => `${orgId}_${repoId}`), - measure: vi.fn(async (cb: () => Promise) => { - const data = await cb(); - return { data, durationMs: 100 }; - }), - setIntervalAsync: vi.fn((cb: () => void, _interval: number) => { - // Return a mock interval ID - return { unref: vi.fn() } as unknown as NodeJS.Timeout; - }), -})); - -vi.mock('fs', () => ({ - existsSync: vi.fn().mockReturnValue(false), -})); - -vi.mock('fs/promises', () => ({ - rm: vi.fn(), - readdir: vi.fn().mockResolvedValue([]), -})); - -// Mock BullMQ -const mockQueueAdd = vi.fn().mockResolvedValue(undefined); -const mockQueueClose = vi.fn().mockResolvedValue(undefined); -const mockWorkerClose = vi.fn().mockResolvedValue(undefined); -const mockWorkerOn = vi.fn(); - -vi.mock('bullmq', () => ({ - Queue: vi.fn().mockImplementation(function () { - return { - add: mockQueueAdd, - close: mockQueueClose, - }; - }), - Worker: vi.fn().mockImplementation(function (_name: string, processor: unknown) { - return { - on: mockWorkerOn, - close: mockWorkerClose, - processJob: processor, - }; - }), - DelayedError: class DelayedError extends Error { - constructor(message: string) { - super(message); - this.name = 'DelayedError'; - } - }, -})); - -// Mock Redlock -const mockRedlockUsing = vi.fn(); -vi.mock('redlock', () => ({ - default: vi.fn().mockImplementation(function () { - return { - using: mockRedlockUsing, - }; - }), - ExecutionError: class ExecutionError extends Error { - constructor(message: string) { - super(message); - this.name = 'ExecutionError'; - } - }, -})); - -// Import after mocks are set up -import { existsSync } from 'fs'; -import { readdir, rm } from 'fs/promises'; -import { ExecutionError } from 'redlock'; -import { - cloneRepository, - fetchRepository, - getBranches, - getTags, - isPathAValidGitRepoRoot, -} from './git.js'; -import { RepoIndexManager } from './repoIndexManager.js'; -import { indexGitRepository } from './zoekt.js'; - -// Helper to create mock Prisma client -const createMockPrisma = () => { - return { - repo: { - findMany: vi.fn().mockResolvedValue([]), - update: vi.fn(), - delete: vi.fn(), - }, - repoIndexingJob: { - createManyAndReturn: vi.fn().mockResolvedValue([]), - findUniqueOrThrow: vi.fn().mockResolvedValue({ status: RepoIndexingJobStatus.PENDING }), - update: vi.fn(), - }, - } as unknown as PrismaClient; -}; - -// Helper to create mock Redis -const createMockRedis = () => { - return {} as Redis; -}; - -// Helper to create mock Settings -const createMockSettings = (): Settings => ({ - maxFileSize: 2 * 1024 * 1024, - maxTrigramCount: 20000, - reindexIntervalMs: 1000 * 60 * 60, - resyncConnectionIntervalMs: 1000 * 60 * 60 * 24, - resyncConnectionPollingIntervalMs: 1000 * 1, - reindexRepoPollingIntervalMs: 1000 * 1, - maxConnectionSyncJobConcurrency: 8, - maxRepoIndexingJobConcurrency: 8, - maxRepoGarbageCollectionJobConcurrency: 8, - repoGarbageCollectionGracePeriodMs: 10 * 1000, - repoIndexTimeoutMs: 1000 * 60 * 60 * 2, - enablePublicAccess: false, - experiment_repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, - experiment_userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, - repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, - userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, - maxAccountPermissionSyncJobConcurrency: 8, - maxRepoPermissionSyncJobConcurrency: 8, -}); - -// Helper to create mock PromClient -const createMockPromClient = () => ({ - pendingRepoIndexJobs: { inc: vi.fn(), dec: vi.fn() }, - activeRepoIndexJobs: { inc: vi.fn(), dec: vi.fn() }, - repoIndexJobSuccessTotal: { inc: vi.fn() }, - repoIndexJobFailTotal: { inc: vi.fn() }, -}); - -// Helper to create a mock repo -const createMockRepo = (overrides: Partial = {}): Repo => ({ - id: 1, - name: 'test-repo', - cloneUrl: 'https://github.com/test/repo.git', - orgId: 1, - indexedAt: null, - indexedCommitHash: null, - defaultBranch: 'main', - metadata: {}, - repoIndexingStatus: 'CREATED', - latestIndexingJobStatus: null, - latestConnectionSyncJobStatus: null, - external_id: 'test-external-id', - external_codeHostType: 'github', - external_codeHostUrl: 'https://github.com', - pushedAt: null, - createdAt: new Date(), - updatedAt: new Date(), - isFork: false, - isArchived: false, - isAutoCleanupDisabled: false, - ...overrides, -} as Repo); - -// Helper to create a mock repoWithConnections -const createMockRepoWithConnections = (overrides: Partial = {}): RepoWithConnections => ({ - ...createMockRepo(), - connections: [], - ...overrides, -}); - -describe('RepoIndexManager', () => { - let mockPrisma: PrismaClient; - let mockRedis: Redis; - let mockSettings: Settings; - let mockPromClient: ReturnType; - let manager: RepoIndexManager; - - beforeEach(() => { - vi.clearAllMocks(); - mockPrisma = createMockPrisma(); - mockRedis = createMockRedis(); - mockSettings = createMockSettings(); - mockPromClient = createMockPromClient(); - - // Default redlock behavior - execute the callback immediately - mockRedlockUsing.mockImplementation(async (_keys: string[], _ttl: number, cb: (signal: AbortSignal) => Promise) => { - const signal = new AbortController().signal; - return cb(signal); - }); - }); - - afterEach(async () => { - if (manager) { - await manager.dispose(); - } - }); - - describe('Job Processing - Success', () => { - test('clones new repository when directory does not exist', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(false); - (cloneRepository as Mock).mockResolvedValue(undefined); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - // Set up mocks for job processing - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - // Simulate processing a job - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - // Get the worker processor callback - const { Worker } = await import('bullmq'); - const workerCalls = (Worker as unknown as Mock).mock.calls; - expect(workerCalls.length).toBeGreaterThan(0); - const processor = workerCalls[0][1]; - - // Execute the processor - await processor(mockJob); - - expect(cloneRepository).toHaveBeenCalledWith( - expect.objectContaining({ - cloneUrl: repo.cloneUrl, - path: expect.stringContaining(`${repo.id}`), - }) - ); - }); - - test('deletes directory and performs fresh clone when path exists but is not a valid git repo root', async () => { - const repo = createMockRepoWithConnections(); - // Path exists initially but after rm is called, it no longer exists - // First two calls return true (first check + check before delete), then false (after deletion) - (existsSync as Mock) - .mockReturnValueOnce(true) // First existsSync check - path exists - .mockReturnValueOnce(false); // Second existsSync check - path deleted, trigger clone - (isPathAValidGitRepoRoot as Mock).mockResolvedValue(false); - (cloneRepository as Mock).mockResolvedValue(undefined); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - // Should delete the invalid directory - expect(rm).toHaveBeenCalledWith( - expect.stringContaining(`${repo.id}`), - { recursive: true, force: true } - ); - - // Should perform a fresh clone after deletion - expect(cloneRepository).toHaveBeenCalledWith( - expect.objectContaining({ - cloneUrl: repo.cloneUrl, - path: expect.stringContaining(`${repo.id}`), - }) - ); - }); - - test('fetches existing repository when directory exists', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(true); - (fetchRepository as Mock).mockResolvedValue(undefined); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - expect(fetchRepository).toHaveBeenCalledWith( - expect.objectContaining({ - cloneUrl: repo.cloneUrl, - path: expect.stringContaining(`${repo.id}`), - }) - ); - }); - - test('invokes zoekt-git-index with correct arguments', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(true); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - expect(indexGitRepository).toHaveBeenCalledWith( - repo, - mockSettings, - expect.arrayContaining(['refs/heads/main']), - expect.any(Object) - ); - }); - - test('keeps default branch and truncates to the first 63 matching tags', async () => { - const newestTagsFirst = Array.from( - { length: 70 }, - (_, index) => `v${70 - index}.0.0`, - ); - const repo = createMockRepoWithConnections({ - metadata: { - tags: ['**'], - }, - }); - (existsSync as Mock).mockReturnValue(true); - (getTags as Mock).mockResolvedValue(newestTagsFirst); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - expect(indexGitRepository).toHaveBeenCalledWith( - repo, - mockSettings, - [ - 'refs/heads/main', - ...newestTagsFirst - .slice(0, 63) - .map((tag) => `refs/tags/${tag}`), - ], - expect.any(Object) - ); - }); - - test('de-duplicates the default branch before truncating matching branches', async () => { - const newestBranchesFirst = [ - 'feature/newest', - 'main', - ...Array.from( - { length: 68 }, - (_, index) => `feature/${68 - index}`, - ), - ]; - const repo = createMockRepoWithConnections({ - metadata: { - branches: ['main', 'feature/**'], - }, - }); - (existsSync as Mock).mockReturnValue(true); - (getBranches as Mock).mockResolvedValue(newestBranchesFirst); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - const revisions = (indexGitRepository as Mock).mock.calls.at(-1)?.[2] as string[]; - - expect(revisions).toHaveLength(64); - expect(revisions.filter((revision) => revision === 'refs/heads/main')).toHaveLength(1); - expect(revisions[0]).toBe('refs/heads/main'); - expect(revisions).toContain('refs/heads/feature/newest'); - expect(revisions).not.toContain('refs/heads/feature/6'); - }); - - test('updates repo.indexedAt and indexedCommitHash on completion', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(true); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - // The onJobCompleted handler reads the job via findUniqueOrThrow, then marks it - // COMPLETED and updates the repo (indexedAt, etc.) in a single repoIndexingJob.update - // with a nested repo update. - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repoId: repo.id, - repo, - metadata: {}, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ repo }); - - // Get the onCompleted handler - const onCompletedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'completed')?.[1]; - expect(onCompletedHandler).toBeDefined(); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - } as unknown as Job; - - await onCompletedHandler(mockJob); - - // The job status and indexedAt must be written together (single transaction) to - // close the race where the scheduler sees a completed job but a stale indexedAt. - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.COMPLETED, - completedAt: expect.any(Date), - repo: { - update: expect.objectContaining({ - indexedAt: expect.any(Date), - indexedCommitHash: 'abc123', - }), - }, - }), - }) - ); - }); - }); - - describe('Job Processing - Failure', () => { - test('marks job as FAILED when git clone throws', async () => { - const repo = createMockRepoWithConnections(); - const cloneError = new Error('Clone failed: authentication error'); - (existsSync as Mock).mockReturnValue(false); - (cloneRepository as Mock).mockRejectedValue(cloneError); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock) - .mockResolvedValueOnce({ type: RepoIndexingJobType.INDEX, repo }) - .mockResolvedValueOnce({ repo }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - getState: vi.fn().mockResolvedValue('failed'), - } as unknown as Job; - - // Get the onFailed handler - const onFailedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'failed')?.[1]; - expect(onFailedHandler).toBeDefined(); - - await onFailedHandler(mockJob, cloneError); - - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.FAILED, - errorMessage: cloneError.message, - }), - }) - ); - - expect(mockPromClient.repoIndexJobFailTotal.inc).toHaveBeenCalledWith({ - repo: repo.name, - type: 'index', - }); - }); - - test('marks job as FAILED when zoekt-git-index fails', async () => { - const repo = createMockRepoWithConnections(); - const indexError = new Error('zoekt-git-index: failed to index'); - (existsSync as Mock).mockReturnValue(true); - (indexGitRepository as Mock).mockRejectedValue(indexError); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock) - .mockResolvedValueOnce({ type: RepoIndexingJobType.INDEX, repo }) - .mockResolvedValueOnce({ repo }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - getState: vi.fn().mockResolvedValue('failed'), - } as unknown as Job; - - const onFailedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'failed')?.[1]; - await onFailedHandler(mockJob, indexError); - - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.FAILED, - errorMessage: indexError.message, - }), - }) - ); - }); - }); - - describe('Concurrency Control', () => { - test('prevents concurrent jobs for the same repo via redlock', async () => { - const repo = createMockRepoWithConnections(); - - // Simulate lock acquisition failure - mockRedlockUsing.mockRejectedValue(new ExecutionError('Lock already held')); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn().mockResolvedValue(undefined), - token: 'test-token', - } as unknown as Job; - - const { Worker, DelayedError } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - - // The processor should throw a DelayedError when lock cannot be acquired - await expect(processor(mockJob)).rejects.toThrow('locked'); - - // Verify moveToDelayed was called to retry later - expect(mockJob.moveToDelayed).toHaveBeenCalled(); - }); - }); - - describe('Cleanup Jobs', () => { - test('deletes repo directory and index shards', async () => { - const repo = createMockRepoWithConnections({ id: 5, orgId: 2 }); - (existsSync as Mock).mockReturnValue(true); - (readdir as Mock).mockResolvedValue(['2_5_v1.zoekt', '2_5_v2.zoekt', 'other_file.zoekt']); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.CLEANUP, - repo, - }); - - const mockJob = { - data: { - jobId: 'cleanup-job-1', - type: 'CLEANUP', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - // Should delete the repo directory - expect(rm).toHaveBeenCalledWith( - expect.stringContaining(`${repo.id}`), - { recursive: true, force: true } - ); - - // Should delete shard files matching the prefix - expect(rm).toHaveBeenCalledWith( - expect.stringContaining('2_5_v1.zoekt'), - { force: true } - ); - expect(rm).toHaveBeenCalledWith( - expect.stringContaining('2_5_v2.zoekt'), - { force: true } - ); - }); - - test('removes repo from database after cleanup', async () => { - const repo = createMockRepoWithConnections({ id: 3 }); - (existsSync as Mock).mockReturnValue(false); - (readdir as Mock).mockResolvedValue([]); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - type: RepoIndexingJobType.CLEANUP, - repoId: repo.id, - repo, - metadata: {}, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ repo }); - (mockPrisma.repo.delete as Mock).mockResolvedValue(repo); - - const onCompletedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'completed')?.[1]; - - const mockJob = { - data: { - jobId: 'cleanup-job-1', - type: 'CLEANUP', - repoId: repo.id, - repoName: repo.name, - }, - } as unknown as Job; - - await onCompletedHandler(mockJob); - - expect(mockPrisma.repo.delete).toHaveBeenCalledWith({ - where: { id: repo.id }, - }); - - expect(mockPromClient.repoIndexJobSuccessTotal.inc).toHaveBeenCalledWith({ - repo: repo.name, - type: 'cleanup', - }); - }); - }); - - describe('latestIndexingJobStatus Updates', () => { - test('sets latestIndexingJobStatus to IN_PROGRESS when job starts', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(true); - // Ensure indexGitRepository resolves for this test - (indexGitRepository as Mock).mockResolvedValue({ stdout: '', stderr: '' }); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - // Verify the first update call sets latestIndexingJobStatus to IN_PROGRESS - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.IN_PROGRESS, - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, - }, - }, - }), - }) - ); - }); - - test('sets latestIndexingJobStatus to COMPLETED when job succeeds', async () => { - const repo = createMockRepoWithConnections(); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repoId: repo.id, - repo, - metadata: {}, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ repo }); - - const onCompletedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'completed')?.[1]; - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - } as unknown as Job; - - await onCompletedHandler(mockJob); - - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.COMPLETED, - repo: { - update: expect.objectContaining({ - latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, - }), - }, - }), - }) - ); - }); - - test('sets latestIndexingJobStatus to FAILED when job fails', async () => { - const repo = createMockRepoWithConnections(); - const error = new Error('Job processing failed'); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ repo }); - - const onFailedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'failed')?.[1]; - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - getState: vi.fn().mockResolvedValue('failed'), - } as unknown as Job; - - await onFailedHandler(mockJob, error); - - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.FAILED, - errorMessage: error.message, - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, - }, - }, - }), - }) - ); - }); - }); -}); diff --git a/packages/backend/src/repoIndexManager.ts b/packages/backend/src/repoIndexManager.ts deleted file mode 100644 index aea1291dc..000000000 --- a/packages/backend/src/repoIndexManager.ts +++ /dev/null @@ -1,769 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; -import { createLogger, env, getRepoPath, Logger, getRepoIdFromPath, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata, repoMetadataSchema } from "@sourcebot/shared"; -import { DelayedError, Job, Queue, Worker } from "bullmq"; -import { existsSync } from 'fs'; -import { readdir, rm } from 'fs/promises'; -import { Redis } from 'ioredis'; -import micromatch from 'micromatch'; -import Redlock, { ExecutionError } from 'redlock'; -import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, WORKER_STOP_GRACEFUL_TIMEOUT_MS } from './constants.js'; -import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getLatestCommitTimestamp, getLocalDefaultBranch, getTags, isPathAValidGitRepoRoot, isRepoEmpty, unsetGitConfig, upsertGitConfig, writeCommitGraph } from './git.js'; -import { captureEvent } from './posthog.js'; -import { PromClient } from './promClient.js'; -import { RepoWithConnections, Settings } from "./types.js"; -import { getAuthCredentialsForRepo, getRepoIdFromShardFileName, getShardPrefix, measure, setIntervalAsync } from './utils.js'; -import { cleanupTempShards, indexGitRepository } from './zoekt.js'; - -const LOG_TAG = 'repo-index-manager'; -const logger = createLogger(LOG_TAG); -const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); -const QUEUE_NAME = 'repo-index-queue'; - -type JobPayload = { - type: 'INDEX' | 'CLEANUP'; - jobId: string; - repoId: number; - repoName: string; -}; - -// Lock TTL with auto-extension - minimizes dead lock time after crashes -const LOCK_TTL_MS = 60 * 1000; // 1 minute -const LOCK_PREFIX = `bullmq:${QUEUE_NAME}:lock:`; - -// Delay before retrying a job when the group lock cannot be acquired -const LOCK_RETRY_DELAY_MS = 5000; - -/** - * Manages the lifecycle of repository data on disk, including git working copies - * and search index shards. Handles both indexing operations (cloning/fetching repos - * and building search indexes) and cleanup operations (removing orphaned repos and - * their associated data). - * - * Uses a job queue system to process indexing and cleanup tasks asynchronously, - * with configurable concurrency limits and retry logic. Automatically schedules - * re-indexing of repos based on configured intervals and manages garbage collection - * of repos that are no longer connected to any source. - */ -export class RepoIndexManager { - private interval?: NodeJS.Timeout; - private queue: Queue; - private worker: Worker; - private redlock: Redlock; - private abortController: AbortController; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - private promClient: PromClient, - ) { - this.abortController = new AbortController(); - - this.queue = new Queue(QUEUE_NAME, { - connection: redis, - defaultJobOptions: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - attempts: 2, - }, - }); - - this.redlock = new Redlock([redis], { - retryCount: 0, // Don't retry - we'll delay the job instead - automaticExtensionThreshold: LOCK_TTL_MS / 2, // Extend when 50% of TTL remains - }); - - this.worker = new Worker( - QUEUE_NAME, - this.processJob.bind(this), - { - connection: redis, - concurrency: this.settings.maxRepoIndexingJobConcurrency, - maxStalledCount: 1, - } - ); - - this.worker.on('completed', this.onJobCompleted.bind(this)); - this.worker.on('failed', this.onJobMaybeFailed.bind(this)); - this.worker.on('stalled', (jobId) => { - // Just log - BullMQ will automatically retry the job (up to maxStalledCount times). - // If all retries fail, onJobMaybeFailed will handle marking it as failed. - logger.warn(`Job ${jobId} stalled - BullMQ will retry`); - }); - this.worker.on('error', (error) => { - logger.error(`Index syncer worker error:`, error); - }); - } - - public async startScheduler() { - logger.debug('Starting scheduler'); - // Cleanup any orphaned disk resources on startup - await this.cleanupOrphanedDiskResources(); - this.interval = setIntervalAsync(async () => { - await this.scheduleIndexJobs(); - await this.scheduleCleanupJobs(); - }, this.settings.reindexRepoPollingIntervalMs); - } - - private async scheduleIndexJobs() { - const thresholdDate = new Date(Date.now() - this.settings.reindexIntervalMs); - const timeoutDate = new Date(Date.now() - this.settings.repoIndexTimeoutMs); - - const reposToIndex = await this.db.repo.findMany({ - where: { - AND: [ - { - OR: [ - { indexedAt: null }, - { indexedAt: { lt: thresholdDate } }, - ] - }, - { - NOT: { - jobs: { - some: { - AND: [ - { - type: RepoIndexingJobType.INDEX, - }, - { - OR: [ - // Don't schedule if there are active jobs that were created within the threshold date. - // This handles the case where a job is stuck in a pending state and will never be scheduled. - { - AND: [ - { - status: { - in: [ - RepoIndexingJobStatus.PENDING, - RepoIndexingJobStatus.IN_PROGRESS, - ] - }, - }, - { - createdAt: { - gt: timeoutDate, - } - } - ] - }, - // Don't schedule if there are recent failed jobs (within the threshold date). - { - AND: [ - { status: RepoIndexingJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } - ] - } - ] - } - } - } - } - ], - }, - }); - - if (reposToIndex.length > 0) { - await this.createJobs(reposToIndex, RepoIndexingJobType.INDEX); - } - } - - private async scheduleCleanupJobs() { - const gcGracePeriodMs = new Date(Date.now() - this.settings.repoGarbageCollectionGracePeriodMs); - const timeoutDate = new Date(Date.now() - this.settings.repoIndexTimeoutMs); - - const reposToCleanup = await this.db.repo.findMany({ - where: { - connections: { - none: {} - }, - isAutoCleanupDisabled: false, - OR: [ - { indexedAt: null }, - { indexedAt: { lt: gcGracePeriodMs } }, - ], - NOT: { - jobs: { - some: { - AND: [ - { - type: RepoIndexingJobType.CLEANUP, - }, - { - status: { - in: [ - RepoIndexingJobStatus.PENDING, - RepoIndexingJobStatus.IN_PROGRESS, - ] - }, - }, - { - createdAt: { - gt: timeoutDate, - } - } - ] - } - } - } - } - }); - - if (reposToCleanup.length > 0) { - await this.createJobs(reposToCleanup, RepoIndexingJobType.CLEANUP); - } - } - - public async createJobs(repos: Repo[], type: RepoIndexingJobType) { - // @note: we don't perform this in a transaction because - // we want to avoid the situation where a job is created and run - // prior to the transaction being committed. - const jobs = await this.db.repoIndexingJob.createManyAndReturn({ - data: repos.map(repo => ({ - type, - repoId: repo.id, - })), - include: { - repo: true, - } - }); - - for (const job of jobs) { - await this.queue.add( - 'repo-index-job', - { - jobId: job.id, - type, - repoName: job.repo.name, - repoId: job.repo.id, - }, - { jobId: job.id } - ); - - const jobTypeLabel = getJobTypePrometheusLabel(type); - this.promClient.pendingRepoIndexJobs.inc({ repo: job.repo.name, type: jobTypeLabel }); - } - - return jobs.map(job => job.id); - } - - private async processJob(job: Job): Promise { - const groupId = `repo:${job.data.repoId}`; - const lockKey = `${LOCK_PREFIX}${groupId}`; - - try { - return await this.redlock.using([lockKey], LOCK_TTL_MS, async (lockSignal: AbortSignal) => { - const signal = AbortSignal.any([ - this.abortController.signal, - lockSignal, - ]); - - return await this.runJob(job, signal); - }); - } catch (error) { - if (error instanceof ExecutionError) { - // Lock could not be acquired - another job for this group is running - // Delay this job and let BullMQ retry later - // DelayedError tells BullMQ to delay without counting as a failed attempt - logger.debug(`Group ${groupId} locked, delaying job ${job.id}`); - await job.moveToDelayed(Date.now() + LOCK_RETRY_DELAY_MS, job.token); - throw new DelayedError(`Group ${groupId} locked, delaying job`); - } - throw error; - } - } - - private async runJob(job: Job, signal: AbortSignal) { - const id = job.data.jobId; - const logger = createJobLogger(id); - logger.debug(`Running ${job.data.type} job ${id} for repo ${job.data.repoName} (id: ${job.data.repoId})`); - - const currentStatus = await this.db.repoIndexingJob.findUniqueOrThrow({ - where: { - id, - }, - select: { - status: true, - } - }); - - // Fail safe: if the job is not PENDING (first run) or IN_PROGRESS (retry), it indicates the job - // is in an invalid state and should be skipped. - if ( - currentStatus.status !== RepoIndexingJobStatus.PENDING && - currentStatus.status !== RepoIndexingJobStatus.IN_PROGRESS - ) { - throw new Error(`Job ${id} is not in a valid state. Expected: ${RepoIndexingJobStatus.PENDING} or ${RepoIndexingJobStatus.IN_PROGRESS}. Actual: ${currentStatus.status}. Skipping.`); - } - - const { repo, type: jobType } = await this.db.repoIndexingJob.update({ - where: { - id, - }, - data: { - status: RepoIndexingJobStatus.IN_PROGRESS, - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, - } - } - }, - select: { - type: true, - repo: { - include: { - connections: { - include: { - connection: true, - } - } - } - } - } - }); - - const jobTypeLabel = getJobTypePrometheusLabel(jobType); - this.promClient.pendingRepoIndexJobs.dec({ repo: job.data.repoName, type: jobTypeLabel }); - this.promClient.activeRepoIndexJobs.inc({ repo: job.data.repoName, type: jobTypeLabel }); - - if (jobType === RepoIndexingJobType.INDEX) { - const revisions = await this.indexRepository(repo, logger, signal); - - await this.db.repoIndexingJob.update({ - where: { id }, - data: { - metadata: { - indexedRevisions: revisions, - } satisfies RepoIndexingJobMetadata, - }, - }); - } else if (jobType === RepoIndexingJobType.CLEANUP) { - await this.cleanupRepository(repo, logger); - } - - } - - private async indexRepository(repo: RepoWithConnections, logger: Logger, signal: AbortSignal) { - const { path: repoPath, isReadOnly } = getRepoPath(repo); - - const metadata = repoMetadataSchema.parse(repo.metadata); - - const credentials = await getAuthCredentialsForRepo(repo, logger); - const cloneUrlMaybeWithToken = credentials?.cloneUrlWithToken ?? repo.cloneUrl; - const authHeader = credentials?.authHeader ?? undefined; - - // If the repo path exists but it is not a valid git repository root, this indicates - // that the repository is in a bad state. To fix, we remove the directory and perform - // a fresh clone. - if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) { - const isValidGitRepo = await isPathAValidGitRepoRoot({ - path: repoPath, - signal, - }); - - if (!isValidGitRepo && !isReadOnly) { - logger.warn(`${repoPath} is not a valid git repository root. Deleting directory and performing fresh clone.`); - await rm(repoPath, { recursive: true, force: true }); - } - } - - if (existsSync(repoPath) && !isReadOnly) { - // @NOTE: in #483, we changed the cloning method s.t., we _no longer_ - // write the clone URL (which could contain a auth token) to the - // `remote.origin.url` entry. For the upgrade scenario, we want - // to unset this key since it is no longer needed, hence this line. - // This will no-op if the key is already unset. - // @see: https://github.com/sourcebot-dev/sourcebot/pull/483 - await unsetGitConfig({ - path: repoPath, - keys: ["remote.origin.url"], - signal, - }); - - logger.debug(`Fetching ${repo.name} (id: ${repo.id})...`); - const { durationMs } = await measure(() => fetchRepository({ - cloneUrl: cloneUrlMaybeWithToken, - authHeader, - path: repoPath, - onProgress: ({ method, stage, progress }) => { - logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) - }, - signal, - })); - const fetchDuration_s = durationMs / 1000; - - logger.debug(`Fetched ${repo.name} (id: ${repo.id}) in ${fetchDuration_s}s`); - - // Update the commit-graph after fetch. Force a full backfill the first time we - // see this repo after the --changed-paths rollout, so historical commits get - // Bloom filters. Subsequent fetches do a cheap incremental write. - const needsBackfill = !metadata.commitGraphChangedPathsBackfilledAt; - if (needsBackfill) { - logger.debug(`Backfilling changed-path Bloom filters for ${repo.name} (id: ${repo.id})...`); - } - await writeCommitGraph({ - path: repoPath, - forceBackfill: needsBackfill, - signal, - }); - } else if (!isReadOnly) { - logger.debug(`Cloning ${repo.name} (id: ${repo.id})...`); - - const { durationMs } = await measure(() => cloneRepository({ - cloneUrl: cloneUrlMaybeWithToken, - authHeader, - path: repoPath, - onProgress: ({ method, stage, progress }) => { - logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) - }, - signal - })); - const cloneDuration_s = durationMs / 1000; - - logger.debug(`Cloned ${repo.name} (id: ${repo.id}) in ${cloneDuration_s}s`); - - // Write the commit-graph for the freshly cloned repo. - await writeCommitGraph({ - path: repoPath, - signal, - }); - } - - // Record that this repo's commit-graph now includes changed-path Bloom filters - // for its full history (either freshly written during clone, or backfilled above - // during fetch). - if (!isReadOnly && !metadata.commitGraphChangedPathsBackfilledAt) { - await this.db.repo.update({ - where: { id: repo.id }, - data: { - metadata: { - ...metadata, - commitGraphChangedPathsBackfilledAt: new Date().toISOString(), - } satisfies RepoMetadata, - }, - }); - } - - // Regardless of clone or fetch, always upsert the git config for the repo. - // This ensures that the git config is always up to date for whatever we - // have in the DB. - if (metadata.gitConfig && !isReadOnly) { - await upsertGitConfig({ - path: repoPath, - gitConfig: metadata.gitConfig, - signal, - }); - } - - const defaultBranch = await getLocalDefaultBranch({ - path: repoPath, - }); - - // Ensure defaultBranch has refs/heads/ prefix for consistent searching - const defaultBranchWithPrefix = defaultBranch && !defaultBranch.startsWith('refs/') - ? `refs/heads/${defaultBranch}` - : defaultBranch; - - let revisions = defaultBranchWithPrefix ? [defaultBranchWithPrefix] : ['HEAD']; - - if (metadata.branches) { - const branchGlobs = metadata.branches - const allBranches = await getBranches(repoPath); - const matchingBranches = - allBranches - .filter((branch) => micromatch.isMatch(branch, branchGlobs)) - .map((branch) => `refs/heads/${branch}`); - - revisions = [ - ...revisions, - ...matchingBranches - ]; - } - - if (metadata.tags) { - const tagGlobs = metadata.tags; - const allTags = await getTags(repoPath); - const matchingTags = - allTags - .filter((tag) => micromatch.isMatch(tag, tagGlobs)) - .map((tag) => `refs/tags/${tag}`); - - revisions = [ - ...revisions, - ...matchingTags - ]; - } - - // De-duplicate revisions to ensure we don't have duplicate branches/tags - revisions = [...new Set(revisions)]; - - // zoekt has a limit of 64 branches/tags to index. - if (revisions.length > 64) { - logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`); - captureEvent('backend_revisions_truncated', { - repoId: repo.id, - revisionCount: revisions.length, - }); - revisions = revisions.slice(0, 64); - } - - logger.debug(`Indexing ${repo.name} (id: ${repo.id})...`); - try { - const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, revisions, signal)); - const indexDuration_s = durationMs / 1000; - logger.debug(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`); - } catch (error) { - // Clean up any temporary shard files left behind by the failed indexing operation. - // Zoekt creates .tmp files during indexing which can accumulate if indexing fails repeatedly. - logger.warn(`Indexing failed for ${repo.name} (id: ${repo.id}), cleaning up temp shard files...`); - await cleanupTempShards(repo); - throw error; - } - - return revisions; - } - - private async cleanupRepository(repo: Repo, logger: Logger) { - const { path: repoPath, isReadOnly } = getRepoPath(repo); - if (existsSync(repoPath) && !isReadOnly) { - logger.debug(`Deleting repo directory ${repoPath}`); - await rm(repoPath, { recursive: true, force: true }); - } - - const shardPrefix = getShardPrefix(repo.orgId, repo.id); - const files = (await readdir(INDEX_CACHE_DIR)).filter(file => file.startsWith(shardPrefix)); - for (const file of files) { - const filePath = `${INDEX_CACHE_DIR}/${file}`; - logger.debug(`Deleting shard file ${filePath}`); - await rm(filePath, { force: true }); - } - } - - private async onJobCompleted(job: Job) { - try { - const logger = createJobLogger(job.data.jobId); - const jobData = await this.db.repoIndexingJob.findUniqueOrThrow({ - where: { id: job.data.jobId }, - include: { - repo: true, - } - }); - - const jobTypeLabel = getJobTypePrometheusLabel(jobData.type); - // @note: capture this before the update below, since the update sets indexedAt. - const isFirstIndex = jobData.repo.indexedAt === null; - - if (jobData.type === RepoIndexingJobType.INDEX) { - const { path: repoPath } = getRepoPath(jobData.repo); - const isEmpty = await isRepoEmpty({ path: repoPath }); - const commitHash = isEmpty ? undefined : await getCommitHashForRefName({ - path: repoPath, - refName: 'HEAD', - }); - - const pushedAt = await getLatestCommitTimestamp({ path: repoPath }); - const defaultBranch = await getLocalDefaultBranch({ path: repoPath }); - - const jobMetadata = repoIndexingJobMetadataSchema.parse(jobData.metadata); - - const { repo } = await this.db.repoIndexingJob.update({ - where: { id: job.data.jobId }, - data: { - status: RepoIndexingJobStatus.COMPLETED, - completedAt: new Date(), - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, - indexedAt: new Date(), - indexedCommitHash: commitHash, - pushedAt: pushedAt, - metadata: { - ...(jobData.repo.metadata as RepoMetadata), - indexedRevisions: jobMetadata.indexedRevisions, - } satisfies RepoMetadata, - // @note: always update the default branch. While this field can be set - // during connection syncing, by setting it here we ensure that a) the - // default branch is as up to date as possible (since repo indexing happens - // more frequently than connection syncing) and b) for hosts where it is - // impossible to determine the default branch from the host's API - // (e.g., generic git url), we still set the default branch here. - defaultBranch: defaultBranch, - } - } - }, - include: { - repo: true, - } - }); - - logger.debug(`Completed index job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`); - } - else if (jobData.type === RepoIndexingJobType.CLEANUP) { - await this.db.repoIndexingJob.update({ - where: { id: job.data.jobId }, - data: { - status: RepoIndexingJobStatus.COMPLETED, - completedAt: new Date(), - } - }); - - const repo = await this.db.repo.delete({ - where: { id: jobData.repoId }, - }); - - logger.debug(`Completed cleanup job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`); - } - - // Track metrics for successful job - this.promClient.activeRepoIndexJobs.dec({ repo: job.data.repoName, type: jobTypeLabel }); - this.promClient.repoIndexJobSuccessTotal.inc({ repo: job.data.repoName, type: jobTypeLabel }); - - if (jobData.type === RepoIndexingJobType.INDEX && isFirstIndex) { - captureEvent('backend_repo_first_indexed', { - repoId: job.data.repoId, - type: jobData.repo.external_codeHostType, - }); - } - } catch (error) { - Sentry.captureException(error); - logger.error(`Exception thrown while executing lifecycle function \`onJobCompleted\`.`, error); - } - } - - private async onJobMaybeFailed(job: Job | undefined, error: Error) { - try { - if (!job) { - logger.error(`Job failed but job object is undefined. Error: ${error.message}`); - return; - } - - const jobLogger = createJobLogger(job.data.jobId); - const jobTypeLabel = getJobTypePrometheusLabel(job.data.type); - - // @note: we need to check the job state to determine if the job failed, - // or if it is being retried. - const jobState = await job.getState(); - if (jobState !== 'failed') { - jobLogger.warn(`Job ${job.id} for repo ${job.data.repoName} (id: ${job.data.repoId}) failed. Retrying... Reason: ${error.message}`); - return; - } - - const { repo } = await this.db.repoIndexingJob.update({ - where: { id: job.data.jobId }, - data: { - status: RepoIndexingJobStatus.FAILED, - completedAt: new Date(), - errorMessage: error.message, - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, - } - } - }, - select: { repo: true } - }); - - this.promClient.activeRepoIndexJobs.dec({ repo: job.data.repoName, type: jobTypeLabel }); - this.promClient.repoIndexJobFailTotal.inc({ repo: job.data.repoName, type: jobTypeLabel }); - - jobLogger.error(`Failed job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id}). Reason: ${error.message}`); - - captureEvent('backend_repo_index_job_failed', { - repoId: job.data.repoId, - jobType: job.data.type, - type: repo.external_codeHostType, - }); - } catch (err) { - Sentry.captureException(err); - logger.error(`Exception thrown while executing lifecycle function \`onJobMaybeFailed\`.`, err); - } - } - - // Scans the repos and index directories on disk and removes any entries - // that have no corresponding Repo record in the database. This handles - // edge cases where the DB and disk resources are out of sync. - private async cleanupOrphanedDiskResources() { - // --- Repo directories --- - // Dirs are named by repoId: DATA_CACHE_DIR/repos// - if (existsSync(REPOS_CACHE_DIR)) { - const entries = await readdir(REPOS_CACHE_DIR); - const repoIdToPath = new Map(); - for (const entry of entries) { - const repoPath = `${REPOS_CACHE_DIR}/${entry}`; - const repoId = getRepoIdFromPath(repoPath); - if (repoId !== undefined) { - repoIdToPath.set(repoId, repoPath); - } - } - - if (repoIdToPath.size > 0) { - const existingRepos = await this.db.repo.findMany({ - where: { id: { in: [...repoIdToPath.keys()] } }, - select: { id: true }, - }); - const existingIds = new Set(existingRepos.map(r => r.id)); - for (const [repoId, repoPath] of repoIdToPath) { - if (!existingIds.has(repoId)) { - logger.debug(`Removing orphaned repo directory with no DB record: ${repoPath}`); - await rm(repoPath, { recursive: true, force: true }); - } - } - } - } - - // --- Index shards --- - // Shard files are prefixed with _: DATA_CACHE_DIR/index/__*.zoekt - if (existsSync(INDEX_CACHE_DIR)) { - const entries = await readdir(INDEX_CACHE_DIR); - const repoIdToShards = new Map(); - for (const entry of entries) { - const repoId = getRepoIdFromShardFileName(entry); - if (repoId !== undefined) { - const shards = repoIdToShards.get(repoId) ?? []; - shards.push(entry); - repoIdToShards.set(repoId, shards); - } - } - - if (repoIdToShards.size > 0) { - const existingRepos = await this.db.repo.findMany({ - where: { id: { in: [...repoIdToShards.keys()] } }, - select: { id: true }, - }); - const existingIds = new Set(existingRepos.map(r => r.id)); - for (const [repoId, shards] of repoIdToShards) { - if (!existingIds.has(repoId)) { - for (const entry of shards) { - const shardPath = `${INDEX_CACHE_DIR}/${entry}`; - logger.debug(`Removing orphaned index shard with no DB record: ${shardPath}`); - await rm(shardPath, { force: true }); - } - } - } - } - } - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - - // Signal all active jobs to abort - this.abortController.abort(); - - // Wait for worker to finish with timeout - await Promise.race([ - this.worker.close(), - new Promise(resolve => setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS)) - ]); - - // Locks will auto-expire via TTL, no need to manually release them - await this.queue.close(); - } -} - -const getJobTypePrometheusLabel = (type: RepoIndexingJobType) => type === RepoIndexingJobType.INDEX ? 'index' : 'cleanup'; diff --git a/packages/backend/src/repoIndexWorkload.test.ts b/packages/backend/src/repoIndexWorkload.test.ts new file mode 100644 index 000000000..3bffa2b87 --- /dev/null +++ b/packages/backend/src/repoIndexWorkload.test.ts @@ -0,0 +1,328 @@ +import type { PrismaClient } from "@sourcebot/db"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { createRepoIndexWorkload } from "./repoIndexWorkload.js"; + +const fsMocks = vi.hoisted(() => ({ + existsSync: vi.fn(), + readdir: vi.fn(), + rm: vi.fn(), +})); + +vi.mock("fs", () => ({ + existsSync: fsMocks.existsSync, +})); + +vi.mock("fs/promises", () => ({ + readdir: fsMocks.readdir, + rm: fsMocks.rm, +})); + +const repoFindUnique = vi.fn(); +const repoDeleteMany = vi.fn(); +const repoIndexingJobUpsert = vi.fn(); +const repoIndexingJobUpdateMany = vi.fn(); +const repoUpdate = vi.fn(); +const repoUpdateMany = vi.fn(); + +const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => + callback({ + repoIndexingJob: { + upsert: repoIndexingJobUpsert, + updateMany: repoIndexingJobUpdateMany, + }, + repo: { + findUnique: repoFindUnique, + update: repoUpdate, + updateMany: repoUpdateMany, + }, + }), +); + +const db = { + $transaction: transaction, + repo: { + deleteMany: repoDeleteMany, + }, +} as unknown as PrismaClient; + +const workload = createRepoIndexWorkload({ + db, + settings: { + maxRepoIndexingJobConcurrency: 2, + } as never, +}); + +const lifecycleLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +}; + +const lifecycleContext = { + data: { + repoId: 42, + type: "INDEX" as const, + }, + jobId: "job-1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + logger: lifecycleLogger, +}; + +const processContext = { + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), +}; + +const eligibleRepo = { + id: 42, + name: "github.com/acme/repo", + cloneUrl: "https://github.com/acme/repo.git", + external_codeHostType: "github", + orgId: 1, + indexedAt: null, + isAutoCleanupDisabled: false, + connections: [], +}; + +describe("repoIndexWorkload", () => { + beforeEach(() => { + vi.clearAllMocks(); + fsMocks.existsSync.mockReturnValue(false); + fsMocks.readdir.mockResolvedValue([]); + fsMocks.rm.mockResolvedValue(undefined); + repoFindUnique.mockResolvedValue(eligibleRepo); + repoDeleteMany.mockResolvedValue({ count: 1 }); + repoIndexingJobUpsert.mockResolvedValue(undefined); + repoIndexingJobUpdateMany.mockResolvedValue({ count: 1 }); + repoUpdate.mockResolvedValue(undefined); + repoUpdateMany.mockResolvedValue({ count: 1 }); + }); + + test("uses the same repository execution lock for INDEX and CLEANUP", () => { + expect(workload.executionLock).toBeDefined(); + expect( + workload.executionLock?.resource({ repoId: 42, type: "INDEX" }), + ).toBe("sourcebot:lock:repo:42"); + expect( + workload.executionLock?.resource({ repoId: 42, type: "CLEANUP" }), + ).toBe("sourcebot:lock:repo:42"); + expect(workload.executionLock?.durationMs).toBe(60_000); + expect(workload.queueSpec.dedupKey).toBeUndefined(); + expect(workload.onStarted).toBeUndefined(); + expect(workload.onCompleted).toBeTypeOf("function"); + expect(workload.onTerminalFailure).toBeTypeOf("function"); + }); + + test("validates state and marks an eligible job in progress inside process", async () => { + await workload.process({ + ...processContext, + data: { repoId: 42, type: "CLEANUP" }, + }); + + expect(repoFindUnique).toHaveBeenCalledWith({ + where: { id: 42 }, + include: { + connections: { + include: { + connection: true, + }, + }, + }, + }); + expect(repoIndexingJobUpsert).toHaveBeenCalledWith({ + where: { + id: "job-1", + }, + update: { + status: "IN_PROGRESS", + completedAt: null, + errorMessage: null, + }, + create: { + id: "job-1", + repoId: 42, + type: "CLEANUP", + status: "IN_PROGRESS", + }, + }); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { + id: 42, + }, + data: { + latestIndexingJobId: "job-1", + latestIndexingJobStatus: "IN_PROGRESS", + }, + }); + expect(repoDeleteMany).toHaveBeenCalledWith({ + where: { + id: 42, + isAutoCleanupDisabled: false, + connections: { + none: {}, + }, + }, + }); + }); + + test("cleanup removes only shards belonging to the exact repository id", async () => { + fsMocks.readdir.mockResolvedValue([ + "1_42_v16.00000.zoekt", + "1_420_v16.00000.zoekt", + ]); + + await workload.process({ + ...processContext, + data: { repoId: 42, type: "CLEANUP" }, + }); + + expect(fsMocks.rm).toHaveBeenCalledWith( + expect.stringContaining("1_42_v16.00000.zoekt"), + { force: true }, + ); + expect(fsMocks.rm).not.toHaveBeenCalledWith( + expect.stringContaining("1_420_v16.00000.zoekt"), + expect.anything(), + ); + }); + + test("skips an INDEX job when the repository no longer exists", async () => { + repoFindUnique.mockResolvedValue(null); + + await workload.process(processContext); + + expect(repoIndexingJobUpsert).not.toHaveBeenCalled(); + expect(repoDeleteMany).not.toHaveBeenCalled(); + expect(fsMocks.readdir).not.toHaveBeenCalled(); + expect(lifecycleLogger.info).toHaveBeenCalledWith( + "Skipping INDEX job for repo 42: repository no longer exists", + ); + }); + + test("finishes orphaned filesystem cleanup when a CLEANUP retry finds no repo", async () => { + repoFindUnique.mockResolvedValue(null); + fsMocks.existsSync.mockReturnValue(true); + fsMocks.readdir.mockResolvedValue([ + "1_42_v16.00000.zoekt", + "1_99_v16.00000.zoekt", + ]); + + await workload.process({ + ...processContext, + data: { repoId: 42, type: "CLEANUP" }, + }); + + expect(repoIndexingJobUpsert).not.toHaveBeenCalled(); + expect(fsMocks.rm).toHaveBeenCalledWith( + expect.stringMatching(/repos\/42$/), + { recursive: true, force: true }, + ); + expect(fsMocks.rm).toHaveBeenCalledWith( + expect.stringContaining("1_42_v16.00000.zoekt"), + { force: true }, + ); + expect(fsMocks.rm).not.toHaveBeenCalledWith( + expect.stringContaining("1_99_v16.00000.zoekt"), + expect.anything(), + ); + }); + + test.each([ + { + name: "automatic cleanup is disabled", + repo: { ...eligibleRepo, isAutoCleanupDisabled: true }, + reason: "automatic cleanup is disabled", + }, + { + name: "the repository was reattached", + repo: { ...eligibleRepo, connections: [{}] }, + reason: "repository has been reattached to a connection", + }, + ])("skips CLEANUP when $name", async ({ repo, reason }) => { + repoFindUnique.mockResolvedValue(repo); + + await workload.process({ + ...processContext, + data: { repoId: 42, type: "CLEANUP" }, + }); + + expect(repoIndexingJobUpsert).not.toHaveBeenCalled(); + expect(repoDeleteMany).not.toHaveBeenCalled(); + expect(fsMocks.readdir).not.toHaveBeenCalled(); + expect(lifecycleLogger.info).toHaveBeenCalledWith( + `Skipping CLEANUP job for repo 42: ${reason}`, + ); + }); + + test("revalidates cleanup eligibility when atomically deleting the repo", async () => { + repoDeleteMany.mockResolvedValue({ count: 0 }); + + await workload.process({ + ...processContext, + data: { repoId: 42, type: "CLEANUP" }, + }); + + expect(repoIndexingJobUpsert).toHaveBeenCalled(); + expect(repoDeleteMany).toHaveBeenCalled(); + expect(fsMocks.readdir).not.toHaveBeenCalled(); + expect(lifecycleLogger.info).toHaveBeenCalledWith( + "Skipping CLEANUP job for repo 42: repository is no longer eligible for cleanup", + ); + }); + + test("marks a completed job and fences the repository summary by job id", async () => { + await workload.onCompleted?.(lifecycleContext, undefined); + + expect(repoIndexingJobUpdateMany).toHaveBeenCalledWith({ + where: { + id: "job-1", + }, + data: { + status: "COMPLETED", + completedAt: expect.any(Date), + errorMessage: null, + }, + }); + expect(repoUpdateMany).toHaveBeenCalledWith({ + where: { + id: 42, + latestIndexingJobId: "job-1", + }, + data: { + latestIndexingJobStatus: "COMPLETED", + }, + }); + }); + + test("marks a terminal failure and fences the repository summary by job id", async () => { + await workload.onTerminalFailure?.( + lifecycleContext, + new Error("Unable to clone repository"), + ); + + expect(repoIndexingJobUpdateMany).toHaveBeenCalledWith({ + where: { + id: "job-1", + }, + data: { + status: "FAILED", + completedAt: expect.any(Date), + errorMessage: "Unable to clone repository", + }, + }); + expect(repoUpdateMany).toHaveBeenCalledWith({ + where: { + id: 42, + latestIndexingJobId: "job-1", + }, + data: { + latestIndexingJobStatus: "FAILED", + }, + }); + }); +}); diff --git a/packages/backend/src/repoIndexWorkload.ts b/packages/backend/src/repoIndexWorkload.ts new file mode 100644 index 000000000..cd697e279 --- /dev/null +++ b/packages/backend/src/repoIndexWorkload.ts @@ -0,0 +1,556 @@ +import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; +import { createLogger, getRepoPath, JobLogSink, getRepoIdFromPath, RepoMetadata, repoMetadataSchema, REPO_INDEX_QUEUE } from "@sourcebot/shared"; +import { existsSync } from 'fs'; +import { readdir, rm } from 'fs/promises'; +import micromatch from 'micromatch'; +import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js'; +import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getLatestCommitTimestamp, getLocalDefaultBranch, getTags, isPathAValidGitRepoRoot, isRepoEmpty, unsetGitConfig, upsertGitConfig, writeCommitGraph } from './git.js'; +import { captureEvent } from './posthog.js'; +import { RepoWithConnections, Settings, Workload } from "./types.js"; +import { getAuthCredentialsForRepo, getRepoIdFromShardFileName, measure } from './utils.js'; +import { cleanupTempShards, indexGitRepository } from './zoekt.js'; + +const LOG_TAG = 'repo-index-workload'; +const logger = createLogger(LOG_TAG); +const REPO_INDEX_LOCK_DURATION_MS = 60_000; + +interface Props { + db: PrismaClient; + settings: Settings; +} + +export const createRepoIndexWorkload = ({ + db, + settings, +}: Props): Workload<'repo-index'> => ({ + queueSpec: REPO_INDEX_QUEUE, + concurrency: settings.maxRepoIndexingJobConcurrency, + // This lock is shared with repoPermissionSyncWorkload so indexing, cleanup, + // and permission syncing are serialized for the same repository. + executionLock: { + resource: ({ repoId }) => `sourcebot:lock:repo:${repoId}`, + durationMs: REPO_INDEX_LOCK_DURATION_MS, + }, + process: async ({ data, jobId, logger: jobLogger, signal }) => { + signal.throwIfAborted(); + const start = await prepareRepoIndexJob({ + db, + repoId: data.repoId, + type: data.type, + jobId, + }); + + if (start.action === "skip") { + jobLogger.info( + `Skipping ${data.type} job for repo ${data.repoId}: ${start.reason}`, + ); + + if (data.type === "CLEANUP" && start.repoMissing) { + signal.throwIfAborted(); + await cleanupOrphanedRepoResourcesForRepoId( + data.repoId, + jobLogger, + ); + } + return; + } + + signal.throwIfAborted(); + const { repo } = start; + jobLogger.debug(`Running ${data.type} job for repo ${repo.name} (id: ${repo.id})`); + + if (data.type === "CLEANUP") { + signal.throwIfAborted(); + const { count } = await db.repo.deleteMany({ + where: { + id: repo.id, + isAutoCleanupDisabled: false, + connections: { + none: {}, + }, + }, + }); + + if (count === 0) { + jobLogger.info( + `Skipping CLEANUP job for repo ${repo.id}: repository is no longer eligible for cleanup`, + ); + return; + } + + signal.throwIfAborted(); + await cleanupRepository(repo, jobLogger); + } else { + const isFirstIndex = repo.indexedAt === null; + const revisions = await indexRepository(db, settings, repo, jobLogger, signal); + signal.throwIfAborted(); + const { path: repoPath } = getRepoPath(repo); + const isEmpty = await isRepoEmpty({ path: repoPath }); + const commitHash = isEmpty ? undefined : await getCommitHashForRefName({ + path: repoPath, + refName: 'HEAD', + }); + const pushedAt = await getLatestCommitTimestamp({ path: repoPath }); + const defaultBranch = await getLocalDefaultBranch({ path: repoPath }); + const currentRepo = await db.repo.findUniqueOrThrow({ + where: { id: repo.id }, + select: { metadata: true }, + }); + + signal.throwIfAborted(); + await db.repo.update({ + where: { id: repo.id }, + data: { + indexedAt: new Date(), + indexedCommitHash: commitHash, + pushedAt, + metadata: { + ...(currentRepo.metadata as RepoMetadata), + indexedRevisions: revisions, + } satisfies RepoMetadata, + defaultBranch, + }, + }); + + if (isFirstIndex) { + captureEvent('backend_repo_first_indexed', { + repoId: repo.id, + type: repo.external_codeHostType, + }); + } + } + }, + onCompleted: async ({ data: { repoId }, jobId }) => { + await db.$transaction(async (tx) => { + await tx.repoIndexingJob.updateMany({ + where: { + id: jobId, + }, + data: { + status: RepoIndexingJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + }, + }); + await tx.repo.updateMany({ + where: { + id: repoId, + latestIndexingJobId: jobId, + }, + data: { + latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, + }, + }); + }); + }, + onTerminalFailure: async ({ data: { repoId }, jobId }, error) => { + await db.$transaction(async (tx) => { + await tx.repoIndexingJob.updateMany({ + where: { + id: jobId, + }, + data: { + status: RepoIndexingJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + }); + await tx.repo.updateMany({ + where: { + id: repoId, + latestIndexingJobId: jobId, + }, + data: { + latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, + }, + }); + }); + }, +}); + +type RepoIndexStartDecision = + | { + action: "run"; + repo: RepoWithConnections; + } + | { + action: "skip"; + reason: string; + repoMissing: boolean; + }; + +const prepareRepoIndexJob = async ({ + db, + repoId, + type, + jobId, +}: { + db: PrismaClient; + repoId: number; + type: "INDEX" | "CLEANUP"; + jobId: string; +}): Promise => + db.$transaction(async (tx) => { + const repo = await tx.repo.findUnique({ + where: { id: repoId }, + include: { + connections: { + include: { + connection: true, + }, + }, + }, + }); + + if (!repo) { + return { + action: "skip", + reason: "repository no longer exists", + repoMissing: true, + }; + } + + if (type === "CLEANUP" && repo.isAutoCleanupDisabled) { + return { + action: "skip", + reason: "automatic cleanup is disabled", + repoMissing: false, + }; + } + + if (type === "CLEANUP" && repo.connections.length > 0) { + return { + action: "skip", + reason: "repository has been reattached to a connection", + repoMissing: false, + }; + } + + await tx.repoIndexingJob.upsert({ + where: { + id: jobId, + }, + update: { + status: RepoIndexingJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + }, + create: { + id: jobId, + repoId, + type: RepoIndexingJobType[type], + status: RepoIndexingJobStatus.IN_PROGRESS, + }, + }); + await tx.repo.update({ + where: { + id: repoId, + }, + data: { + latestIndexingJobId: jobId, + latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, + }, + }); + + return { + action: "run", + repo, + }; + }); + +const indexRepository = async ( + db: PrismaClient, + settings: Settings, + repo: RepoWithConnections, + logger: JobLogSink, + signal: AbortSignal, +) => { + const { path: repoPath, isReadOnly } = getRepoPath(repo); + + const metadata = repoMetadataSchema.parse(repo.metadata); + + const credentials = await getAuthCredentialsForRepo(repo, logger); + const cloneUrlMaybeWithToken = credentials?.cloneUrlWithToken ?? repo.cloneUrl; + const authHeader = credentials?.authHeader ?? undefined; + + // If the repo path exists but it is not a valid git repository root, this indicates + // that the repository is in a bad state. To fix, we remove the directory and perform + // a fresh clone. + if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) { + const isValidGitRepo = await isPathAValidGitRepoRoot({ + path: repoPath, + signal, + }); + + if (!isValidGitRepo && !isReadOnly) { + logger.warn(`${repoPath} is not a valid git repository root. Deleting directory and performing fresh clone.`); + await rm(repoPath, { recursive: true, force: true }); + } + } + + if (existsSync(repoPath) && !isReadOnly) { + // @NOTE: in #483, we changed the cloning method s.t., we _no longer_ + // write the clone URL (which could contain a auth token) to the + // `remote.origin.url` entry. For the upgrade scenario, we want + // to unset this key since it is no longer needed, hence this line. + // This will no-op if the key is already unset. + // @see: https://github.com/sourcebot-dev/sourcebot/pull/483 + await unsetGitConfig({ + path: repoPath, + keys: ["remote.origin.url"], + signal, + }); + + logger.debug(`Fetching ${repo.name} (id: ${repo.id})...`); + const { durationMs } = await measure(() => fetchRepository({ + cloneUrl: cloneUrlMaybeWithToken, + authHeader, + path: repoPath, + onProgress: ({ method, stage, progress }) => { + logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) + }, + signal, + })); + const fetchDuration_s = durationMs / 1000; + + logger.debug(`Fetched ${repo.name} (id: ${repo.id}) in ${fetchDuration_s}s`); + + // Update the commit-graph after fetch. Force a full backfill the first time we + // see this repo after the --changed-paths rollout, so historical commits get + // Bloom filters. Subsequent fetches do a cheap incremental write. + const needsBackfill = !metadata.commitGraphChangedPathsBackfilledAt; + if (needsBackfill) { + logger.debug(`Backfilling changed-path Bloom filters for ${repo.name} (id: ${repo.id})...`); + } + await writeCommitGraph({ + path: repoPath, + forceBackfill: needsBackfill, + signal, + }); + } else if (!isReadOnly) { + logger.debug(`Cloning ${repo.name} (id: ${repo.id})...`); + + const { durationMs } = await measure(() => cloneRepository({ + cloneUrl: cloneUrlMaybeWithToken, + authHeader, + path: repoPath, + onProgress: ({ method, stage, progress }) => { + logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) + }, + signal + })); + const cloneDuration_s = durationMs / 1000; + + logger.debug(`Cloned ${repo.name} (id: ${repo.id}) in ${cloneDuration_s}s`); + + // Write the commit-graph for the freshly cloned repo. + await writeCommitGraph({ + path: repoPath, + signal, + }); + } + + // Record that this repo's commit-graph now includes changed-path Bloom filters + // for its full history (either freshly written during clone, or backfilled above + // during fetch). + if (!isReadOnly && !metadata.commitGraphChangedPathsBackfilledAt) { + signal.throwIfAborted(); + await db.repo.update({ + where: { id: repo.id }, + data: { + metadata: { + ...metadata, + commitGraphChangedPathsBackfilledAt: new Date().toISOString(), + } satisfies RepoMetadata, + }, + }); + } + + // Regardless of clone or fetch, always upsert the git config for the repo. + // This ensures that the git config is always up to date for whatever we + // have in the DB. + if (metadata.gitConfig && !isReadOnly) { + await upsertGitConfig({ + path: repoPath, + gitConfig: metadata.gitConfig, + signal, + }); + } + + const defaultBranch = await getLocalDefaultBranch({ + path: repoPath, + }); + + // Ensure defaultBranch has refs/heads/ prefix for consistent searching + const defaultBranchWithPrefix = defaultBranch && !defaultBranch.startsWith('refs/') + ? `refs/heads/${defaultBranch}` + : defaultBranch; + + let revisions = defaultBranchWithPrefix ? [defaultBranchWithPrefix] : ['HEAD']; + + if (metadata.branches) { + const branchGlobs = metadata.branches + const allBranches = await getBranches(repoPath); + const matchingBranches = + allBranches + .filter((branch) => micromatch.isMatch(branch, branchGlobs)) + .map((branch) => `refs/heads/${branch}`); + + revisions = [ + ...revisions, + ...matchingBranches + ]; + } + + if (metadata.tags) { + const tagGlobs = metadata.tags; + const allTags = await getTags(repoPath); + const matchingTags = + allTags + .filter((tag) => micromatch.isMatch(tag, tagGlobs)) + .map((tag) => `refs/tags/${tag}`); + + revisions = [ + ...revisions, + ...matchingTags + ]; + } + + // De-duplicate revisions to ensure we don't have duplicate branches/tags + revisions = [...new Set(revisions)]; + + // zoekt has a limit of 64 branches/tags to index. + if (revisions.length > 64) { + logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`); + captureEvent('backend_revisions_truncated', { + repoId: repo.id, + revisionCount: revisions.length, + }); + revisions = revisions.slice(0, 64); + } + + logger.debug(`Indexing ${repo.name} (id: ${repo.id})...`); + try { + signal.throwIfAborted(); + const { durationMs } = await measure(() => indexGitRepository(repo, settings, revisions, signal)); + signal.throwIfAborted(); + const indexDuration_s = durationMs / 1000; + logger.debug(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`); + } catch (error) { + if (signal.aborted) { + throw error; + } + + // Clean up any temporary shard files left behind by the failed indexing operation. + // Zoekt creates .tmp files during indexing which can accumulate if indexing fails repeatedly. + logger.warn(`Indexing failed for ${repo.name} (id: ${repo.id}), cleaning up temp shard files...`); + await cleanupTempShards(repo); + throw error; + } + + return revisions; +}; + +const cleanupRepository = async (repo: Repo, logger: JobLogSink) => { + const { path: repoPath, isReadOnly } = getRepoPath(repo); + if (existsSync(repoPath) && !isReadOnly) { + logger.debug(`Deleting repo directory ${repoPath}`); + await rm(repoPath, { recursive: true, force: true }); + } + + const files = (await readdir(INDEX_CACHE_DIR)).filter(file => getRepoIdFromShardFileName(file) === repo.id); + for (const file of files) { + const filePath = `${INDEX_CACHE_DIR}/${file}`; + logger.debug(`Deleting shard file ${filePath}`); + await rm(filePath, { force: true }); + } +}; + +const cleanupOrphanedRepoResourcesForRepoId = async ( + repoId: number, + logger: JobLogSink, +) => { + const repoPath = `${REPOS_CACHE_DIR}/${repoId}`; + if (existsSync(repoPath)) { + logger.debug(`Deleting orphaned repo directory ${repoPath}`); + await rm(repoPath, { recursive: true, force: true }); + } + + if (!existsSync(INDEX_CACHE_DIR)) { + return; + } + + const shardFiles = (await readdir(INDEX_CACHE_DIR)).filter( + (file) => getRepoIdFromShardFileName(file) === repoId, + ); + for (const file of shardFiles) { + const filePath = `${INDEX_CACHE_DIR}/${file}`; + logger.debug(`Deleting orphaned shard file ${filePath}`); + await rm(filePath, { force: true }); + } +}; + +// Scans the repos and index directories on disk and removes any entries +// that have no corresponding Repo record in the database. This handles +// edge cases where the DB and disk resources are out of sync. +export const cleanupOrphanedRepoResources = async (db: PrismaClient) => { + // --- Repo directories --- + // Dirs are named by repoId: DATA_CACHE_DIR/repos// + if (existsSync(REPOS_CACHE_DIR)) { + const entries = await readdir(REPOS_CACHE_DIR); + const repoIdToPath = new Map(); + for (const entry of entries) { + const repoPath = `${REPOS_CACHE_DIR}/${entry}`; + const repoId = getRepoIdFromPath(repoPath); + if (repoId !== undefined) { + repoIdToPath.set(repoId, repoPath); + } + } + + if (repoIdToPath.size > 0) { + const existingRepos = await db.repo.findMany({ + where: { id: { in: [...repoIdToPath.keys()] } }, + select: { id: true }, + }); + const existingIds = new Set(existingRepos.map(r => r.id)); + for (const [repoId, repoPath] of repoIdToPath) { + if (!existingIds.has(repoId)) { + logger.debug(`Removing orphaned repo directory with no DB record: ${repoPath}`); + await rm(repoPath, { recursive: true, force: true }); + } + } + } + } + + // --- Index shards --- + // Shard files are prefixed with _: DATA_CACHE_DIR/index/__*.zoekt + if (existsSync(INDEX_CACHE_DIR)) { + const entries = await readdir(INDEX_CACHE_DIR); + const repoIdToShards = new Map(); + for (const entry of entries) { + const repoId = getRepoIdFromShardFileName(entry); + if (repoId !== undefined) { + const shards = repoIdToShards.get(repoId) ?? []; + shards.push(entry); + repoIdToShards.set(repoId, shards); + } + } + + if (repoIdToShards.size > 0) { + const existingRepos = await db.repo.findMany({ + where: { id: { in: [...repoIdToShards.keys()] } }, + select: { id: true }, + }); + const existingIds = new Set(existingRepos.map(r => r.id)); + for (const [repoId, shards] of repoIdToShards) { + if (!existingIds.has(repoId)) { + for (const entry of shards) { + const shardPath = `${INDEX_CACHE_DIR}/${entry}`; + logger.debug(`Removing orphaned index shard with no DB record: ${shardPath}`); + await rm(shardPath, { force: true }); + } + } + } + } + } +}; diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 8803b48b9..1e2e6f6fd 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -1,27 +1,114 @@ -import { Connection, Repo, RepoToConnection } from "@sourcebot/db"; +import { + Connection, + PrismaClient, + Repo, + RepoToConnection, +} from "@sourcebot/db"; import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type"; +import { + DataOf, + JobEnqueueOptions, + JobLogSink, + QueueName, + QueueSpec, + Schedule, +} from "@sourcebot/shared"; +import type { Queue } from "bullmq"; export type Settings = Required; // @see : https://stackoverflow.com/a/61132308 -export type DeepPartial = T extends object ? { - [P in keyof T]?: DeepPartial; -} : T; +export type DeepPartial = T extends object + ? { + [P in keyof T]?: DeepPartial; + } + : T; // @see: https://stackoverflow.com/a/69328045 export type WithRequired = T & { [P in K]-?: T[P] }; -export type RepoWithConnections = Repo & { connections: (RepoToConnection & { connection: Connection })[] }; - +export type RepoWithConnections = Repo & { + connections: (RepoToConnection & { connection: Connection })[]; +}; export type RepoAuthCredentials = { hostUrl?: string; token: string; cloneUrlWithToken?: string; authHeader?: string; - /** The connection that configured the - * credentials for this repo. - */ connectionConfig?: ConnectionConfig; -} \ No newline at end of file +}; + +export interface JobLifecycleContext { + data: DataOf; + jobId: string; + attemptsMade: number; + maxAttempts: number; + prisma: PrismaClient; + logger: JobLogSink; +} + +export interface ProcessContext + extends JobLifecycleContext { + signal: AbortSignal; + updateProgress(progress: number | object): Promise; + trigger( + workload: T, + data: DataOf, + options?: JobEnqueueOptions, + ): Promise; +} + +export interface WorkloadExecutionLock { + resource(data: DataOf): string; + durationMs: number; +} + +export interface Workload { + queueSpec: QueueSpec; + concurrency: number; + executionLock?: WorkloadExecutionLock; + schedule?: { + interval: Schedule; + data: DataOf; + options?: JobEnqueueOptions; + }; + rateLimit?: { max: number; per: string }; + process(ctx: ProcessContext): Promise; + onStarted?(ctx: JobLifecycleContext): Promise; + onCompleted?( + ctx: JobLifecycleContext, + result: TResult, + ): Promise; + onTerminalFailure?( + ctx: JobLifecycleContext, + err: Error, + ): Promise; +} + +export interface JobManager { + register(workload: Workload): void; + getQueues(): Queue[]; + start(): Promise; + stop(): Promise; + trigger( + workloadName: TName, + data: DataOf, + options?: JobEnqueueOptions, + ): Promise; + upsertJobScheduler( + workloadName: TName, + schedulerId: string, + schedule: Schedule, + data: DataOf, + options?: JobEnqueueOptions, + ): Promise; + getJobSchedulerIds( + workloadName: TName, + ): Promise; + removeJobScheduler( + workloadName: TName, + schedulerId: string, + ): Promise; +} diff --git a/packages/backend/src/types/redlock.d.ts b/packages/backend/src/types/redlock.d.ts index 63c6b5a9b..b03e91181 100644 --- a/packages/backend/src/types/redlock.d.ts +++ b/packages/backend/src/types/redlock.d.ts @@ -1,33 +1,18 @@ -// Type declarations for redlock -// The redlock package's exports field doesn't include types, so TypeScript can't resolve them. -// This file re-exports the types from the actual .d.ts file. - -declare module 'redlock' { +// redlock@5.0.0-beta.2 bundles this declaration, but its package exports do not +// expose it to TypeScript's Node16 module resolution. Keep this narrow surface in +// sync with the upstream declaration until the package publishes a fixed export. +declare module "redlock" { import { EventEmitter } from "events"; - import { Redis as IORedisClient, Cluster as IORedisCluster } from "ioredis"; - - type Client = IORedisClient | IORedisCluster; + import { Cluster, Redis } from "ioredis"; - export type ClientExecutionResult = { - client: Client; - vote: "for"; - value: number; - } | { - client: Client; - vote: "against"; - error: Error; - }; + type Client = Redis | Cluster; - export type ExecutionStats = { + export interface ExecutionStats { readonly membershipSize: number; readonly quorumSize: number; readonly votesFor: Set; readonly votesAgainst: Map; - }; - - export type ExecutionResult = { - attempts: ReadonlyArray>; - }; + } export interface Settings { readonly driftFactor: number; @@ -38,25 +23,15 @@ declare module 'redlock' { } export class ResourceLockedError extends Error { - readonly message: string; constructor(message: string); } export class ExecutionError extends Error { - readonly message: string; - readonly attempts: ReadonlyArray>; - constructor(message: string, attempts: ReadonlyArray>); - } - - export class Lock { - readonly redlock: Redlock; - readonly resources: string[]; - readonly value: string; readonly attempts: ReadonlyArray>; - expiration: number; - constructor(redlock: Redlock, resources: string[], value: string, attempts: ReadonlyArray>, expiration: number); - release(): Promise; - extend(duration: number): Promise; + constructor( + message: string, + attempts: ReadonlyArray>, + ); } export type RedlockAbortSignal = AbortSignal & { @@ -64,32 +39,19 @@ declare module 'redlock' { }; export default class Redlock extends EventEmitter { - readonly clients: Set; - readonly settings: Settings; - readonly scripts: { - readonly acquireScript: { - value: string; - hash: string; - }; - readonly extendScript: { - value: string; - hash: string; - }; - readonly releaseScript: { - value: string; - hash: string; - }; - }; - constructor(clients: Iterable, settings?: Partial, scripts?: { - readonly acquireScript?: string | ((script: string) => string); - readonly extendScript?: string | ((script: string) => string); - readonly releaseScript?: string | ((script: string) => string); - }); - quit(): Promise; - acquire(resources: string[], duration: number, settings?: Partial): Promise; - release(lock: Lock, settings?: Partial): Promise; - extend(existing: Lock, duration: number, settings?: Partial): Promise; - using(resources: string[], duration: number, settings: Partial, routine?: (signal: RedlockAbortSignal) => Promise): Promise; - using(resources: string[], duration: number, routine: (signal: RedlockAbortSignal) => Promise): Promise; + constructor(clients: Iterable, settings?: Partial); + + using( + resources: string[], + duration: number, + settings: Partial, + routine: (signal: RedlockAbortSignal) => Promise, + ): Promise; + + using( + resources: string[], + duration: number, + routine: (signal: RedlockAbortSignal) => Promise, + ): Promise; } } diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index ba028fb20..5836bc04d 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -1,7 +1,7 @@ import { Logger } from "winston"; import { RepoAuthCredentials, RepoWithConnections } from "./types.js"; import path from 'path'; -import { env, getTokenFromConfig } from "@sourcebot/shared"; +import { env, getTokenFromConfig, JobLogSink } from "@sourcebot/shared"; import * as Sentry from "@sentry/node"; import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, BitbucketConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; import { GithubAppManager } from "./ee/githubAppManager.js"; @@ -114,7 +114,7 @@ export const fetchWithRetry = async ( // fetch the token here using the connections from the repo. Multiple connections could be referencing this repo, and each // may have their own token. This method will just pick the first connection that has a token (if one exists) and uses that. This // may technically cause syncing to fail if that connection's token just so happens to not have access to the repo it's referencing. -export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, logger?: Logger): Promise => { +export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, logger?: JobLogSink): Promise => { if (repo.external_codeHostType === 'github' && env.EXPERIMENT_ASK_GH_GITHUB_TOKEN) { logger?.debug(`Using Ask GitHub PAT for service auth for repo ${repo.displayName} hosted at ${repo.external_codeHostUrl}`); @@ -287,28 +287,3 @@ const createGitCloneUrlWithToken = (cloneUrl: string, credentials: { username?: } return url.toString(); } - - -// setInterval wrapper that ensures async callbacks are not executed concurrently. -// @see: https://mottaquikarim.github.io/dev/posts/setinterval-that-blocks-on-await/ -export const setIntervalAsync = (target: () => Promise, pollingIntervalMs: number): NodeJS.Timeout => { - const setIntervalWithPromise = Promise>( - target: T - ): (...args: Parameters) => Promise => { - return async function (...args: Parameters): Promise { - if ((target as any).isRunning) return; - - (target as any).isRunning = true; - try { - await target(...args); - } finally { - (target as any).isRunning = false; - } - }; - } - - return setInterval( - setIntervalWithPromise(target), - pollingIntervalMs - ); -} diff --git a/packages/db/prisma/migrations/20260810000000_add_latest_repo_indexing_job_id/migration.sql b/packages/db/prisma/migrations/20260810000000_add_latest_repo_indexing_job_id/migration.sql new file mode 100644 index 000000000..b0c4a67a5 --- /dev/null +++ b/packages/db/prisma/migrations/20260810000000_add_latest_repo_indexing_job_id/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Repo" ADD COLUMN "latestIndexingJobId" TEXT; diff --git a/packages/db/prisma/migrations/20260811000000_add_latest_account_permission_sync_job_id/migration.sql b/packages/db/prisma/migrations/20260811000000_add_latest_account_permission_sync_job_id/migration.sql new file mode 100644 index 000000000..dbd96beb5 --- /dev/null +++ b/packages/db/prisma/migrations/20260811000000_add_latest_account_permission_sync_job_id/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Account" ADD COLUMN "latestPermissionSyncJobId" TEXT; diff --git a/packages/db/prisma/migrations/20260811001000_add_latest_repo_permission_sync_job_id/migration.sql b/packages/db/prisma/migrations/20260811001000_add_latest_repo_permission_sync_job_id/migration.sql new file mode 100644 index 000000000..35f303b95 --- /dev/null +++ b/packages/db/prisma/migrations/20260811001000_add_latest_repo_permission_sync_job_id/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Repo" ADD COLUMN "latestPermissionSyncJobId" TEXT; diff --git a/packages/db/prisma/migrations/20260811002000_add_latest_connection_sync_job_id/migration.sql b/packages/db/prisma/migrations/20260811002000_add_latest_connection_sync_job_id/migration.sql new file mode 100644 index 000000000..62a56fcad --- /dev/null +++ b/packages/db/prisma/migrations/20260811002000_add_latest_connection_sync_job_id/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Connection" ADD COLUMN "latestSyncJobId" TEXT; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index e43f2887b..250b7e2eb 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -73,10 +73,12 @@ model Repo { permittedAccounts AccountToRepoPermission[] permissionSyncJobs RepoPermissionSyncJob[] permissionSyncedAt DateTime? /// When the permissions were last synced successfully. + latestPermissionSyncJobId String? /// The permission sync job allowed to publish the latest repo state. jobs RepoIndexingJob[] indexedAt DateTime? /// When the repo was last indexed successfully. indexedCommitHash String? /// The commit hash of the last indexed commit (on HEAD). + latestIndexingJobId String? /// The job allowed to publish the latest indexing status. latestIndexingJobStatus RepoIndexingJobStatus? /// The status of the latest indexing job. pushedAt DateTime? /// The timestamp of the most recent commit across all branches. @@ -184,6 +186,7 @@ model Connection { syncJobs ConnectionSyncJob[] /// When the connection was last synced successfully. syncedAt DateTime? + latestSyncJobId String? /// The most recently started connection sync job. /// Controls whether repository permissions are enforced for this connection. /// When `PERMISSION_SYNC_ENABLED` is false, this setting has no effect. @@ -611,6 +614,7 @@ model Account { permissionSyncJobs AccountPermissionSyncJob[] permissionSyncedAt DateTime? + latestPermissionSyncJobId String? /// The permission sync job allowed to publish the latest account state. /// Set when permission syncing fails closed and user action is required. /// Cleared after a subsequent permission sync completes successfully. diff --git a/packages/schemas/src/v3/index.schema.ts b/packages/schemas/src/v3/index.schema.ts index 48e42f300..a15b31ea3 100644 --- a/packages/schemas/src/v3/index.schema.ts +++ b/packages/schemas/src/v3/index.schema.ts @@ -31,7 +31,8 @@ const schema = { "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -51,7 +52,8 @@ const schema = { "maxRepoGarbageCollectionJobConcurrency": { "type": "number", "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", @@ -215,7 +217,8 @@ const schema = { "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -235,7 +238,8 @@ const schema = { "maxRepoGarbageCollectionJobConcurrency": { "type": "number", "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", diff --git a/packages/schemas/src/v3/index.type.ts b/packages/schemas/src/v3/index.type.ts index df59a13f4..c18bc9301 100644 --- a/packages/schemas/src/v3/index.type.ts +++ b/packages/schemas/src/v3/index.type.ts @@ -101,6 +101,7 @@ export interface Settings { */ resyncConnectionIntervalMs?: number; /** + * @deprecated * The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second. */ resyncConnectionPollingIntervalMs?: number; @@ -117,6 +118,7 @@ export interface Settings { */ maxRepoIndexingJobConcurrency?: number; /** + * @deprecated * The number of repo GC jobs to run concurrently. Defaults to 8. */ maxRepoGarbageCollectionJobConcurrency?: number; diff --git a/packages/shared/package.json b/packages/shared/package.json index 2a16111ff..7cb47f380 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -14,11 +14,13 @@ "@google-cloud/secret-manager": "^6.1.1", "@logtail/node": "^0.5.2", "@logtail/winston": "^0.5.2", + "@sentry/node": "^10.40.0", "@sourcebot/db": "workspace:*", "@sourcebot/schemas": "workspace:*", "@t3-oss/env-core": "^0.13.10", "ajv": "^8.17.1", - "ioredis": "^5.4.2", + "bullmq": "^5.81.3", + "ioredis": "^5.11.1", "micromatch": "^4.0.8", "strip-json-comments": "^5.0.1", "triple-beam": "^1.4.1", diff --git a/packages/shared/src/bullmqClient.test.ts b/packages/shared/src/bullmqClient.test.ts new file mode 100644 index 000000000..bd84c3316 --- /dev/null +++ b/packages/shared/src/bullmqClient.test.ts @@ -0,0 +1,147 @@ +import { Redis } from "ioredis"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + add: vi.fn(async () => ({ id: "job-1" })), + upsertJobScheduler: vi.fn(async () => ({ id: "scheduled-job" })), + getJobSchedulers: vi.fn(async () => [ + { key: "scheduler-1" }, + { key: "scheduler-2" }, + ]), + removeJobScheduler: vi.fn(async () => true), +})); + +vi.mock("bullmq", () => ({ + Queue: class { + add = mocks.add; + upsertJobScheduler = mocks.upsertJobScheduler; + getJobSchedulers = mocks.getJobSchedulers; + removeJobScheduler = mocks.removeJobScheduler; + }, +})); + +vi.mock("./jobLogger.js", () => ({ + DEFAULT_JOB_LOGS_MAX_ENTRIES: 500, + readBullMQJobLogs: vi.fn(), +})); + +import { BullMQClient } from "./bullmqClient.js"; +import { CONNECTION_QUEUE } from "./queue.js"; + +describe("BullMQClient", () => { + beforeEach(() => { + vi.restoreAllMocks(); + vi.clearAllMocks(); + vi.spyOn(Date, "now").mockReturnValue(1_000_000); + }); + + test("includes workload data in scheduled jobs", async () => { + const client = new BullMQClient({} as Redis); + const data = { connectionId: 42 }; + + await client.upsertJobScheduler( + CONNECTION_QUEUE, + "schedule:42", + 1_000, + data, + ); + + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "schedule:42", + { every: 1_000, startDate: 1_001_000 }, + expect.objectContaining({ + name: "connection-sync", + data, + opts: { + attempts: 4, + backoff: { + type: "exponential", + delay: 30_000, + jitter: 0.5, + }, + removeOnComplete: { age: 1_209_600 }, + removeOnFail: { age: 1_209_600 }, + keepLogs: 500, + }, + }), + ); + }); + + test("adds enqueue priority to immediate jobs", async () => { + const client = new BullMQClient({} as Redis); + + await client.enqueue( + CONNECTION_QUEUE, + { connectionId: 42 }, + { priority: 1 }, + ); + + expect(mocks.add).toHaveBeenCalledWith( + "connection-sync", + { connectionId: 42 }, + expect.objectContaining({ + priority: 1, + attempts: 4, + backoff: { + type: "exponential", + delay: 30_000, + jitter: 0.5, + }, + }), + ); + }); + + test("adds enqueue priority to scheduled jobs", async () => { + const client = new BullMQClient({} as Redis); + + await client.upsertJobScheduler( + CONNECTION_QUEUE, + "schedule:42", + 1_000, + { connectionId: 42 }, + { priority: 10 }, + ); + + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "schedule:42", + expect.any(Object), + expect.objectContaining({ + opts: expect.objectContaining({ priority: 10 }), + }), + ); + }); + + test("parses string schedules and delays the first run by one interval", async () => { + const client = new BullMQClient({} as Redis); + + await client.upsertJobScheduler( + CONNECTION_QUEUE, + "schedule:42", + "5m", + { connectionId: 42 }, + ); + + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "schedule:42", + { every: 300_000, startDate: 1_300_000 }, + expect.any(Object), + ); + }); + + test("lists scheduler ids", async () => { + const client = new BullMQClient({} as Redis); + + await expect( + client.getJobSchedulerIds(CONNECTION_QUEUE), + ).resolves.toEqual(["scheduler-1", "scheduler-2"]); + }); + + test("removes a scheduler by id", async () => { + const client = new BullMQClient({} as Redis); + + await expect( + client.removeJobScheduler(CONNECTION_QUEUE, "scheduler-1"), + ).resolves.toBe(true); + expect(mocks.removeJobScheduler).toHaveBeenCalledWith("scheduler-1"); + }); +}); diff --git a/packages/shared/src/bullmqClient.ts b/packages/shared/src/bullmqClient.ts new file mode 100644 index 000000000..4853c433b --- /dev/null +++ b/packages/shared/src/bullmqClient.ts @@ -0,0 +1,208 @@ +import { Queue } from "bullmq"; +import { randomUUID } from "crypto"; +import { Redis } from "ioredis"; +import type { + DataOf, + JobEnqueueOptions, + QueueName, + QueueSpec, +} from "./queue.js"; +import { scheduleToMs } from "./schedule.js"; +import type { Schedule } from "./schedule.js"; +import { readBullMQJobLogs } from "./jobLogger.js"; +import type { GetJobLogsOptions, JobLogs } from "./jobLogger.js"; + +export type WorkloadJobStatus = + | "PENDING" + | "IN_PROGRESS" + | "COMPLETED" + | "FAILED"; + +export interface WorkloadJob { + id: string; + data: DataOf; + status: WorkloadJobStatus; + errorMessage: string | null; +} + +type WorkloadQueue = Queue< + DataOf, + unknown, + string, + DataOf, + unknown, + string +>; + +const normalizeJobState = (state: string): WorkloadJobStatus | null => { + switch (state) { + case "waiting": + case "waiting-children": + case "delayed": + case "prioritized": + case "paused": + return "PENDING"; + case "active": + return "IN_PROGRESS"; + case "completed": + return "COMPLETED"; + case "failed": + return "FAILED"; + default: + return null; + } +}; + +export class BullMQClient { + private readonly queues = new Map(); + + constructor(private readonly connection: Redis) {} + + getQueue( + spec: QueueSpec, + ): WorkloadQueue { + const queueName = spec.name; + let queue = this.queues.get(queueName); + if (!queue) { + queue = new Queue(queueName, { connection: this.connection }); + this.queues.set(queueName, queue); + } + return queue as WorkloadQueue; + } + + async getJob( + spec: QueueSpec, + jobId: string, + ): Promise | null> { + const job = await this.getQueue(spec).getJob(jobId); + if (!job) { + return null; + } + + const status = normalizeJobState(await job.getState()); + if (!status) { + return null; + } + + return { + id: job.id ?? jobId, + data: job.data as DataOf, + status, + errorMessage: status === "FAILED" ? job.failedReason || null : null, + }; + } + + async getJobLogs( + spec: QueueSpec, + jobId: string, + options: GetJobLogsOptions = {}, + ): Promise { + return readBullMQJobLogs(this.getQueue(spec), jobId, options); + } + + async enqueue( + spec: QueueSpec, + data: DataOf, + options: JobEnqueueOptions = {}, + ): Promise { + const dedupKey = spec.dedupKey?.(data); + const queue = this.getQueue(spec); + + const requestedJobId = randomUUID(); + const job = await queue.add(spec.name, data, { + jobId: requestedJobId, + ...(dedupKey ? { deduplication: { id: dedupKey } } : {}), + ...(options.priority !== undefined + ? { priority: options.priority } + : {}), + attempts: spec.jobOptions.attempts, + backoff: { + type: spec.jobOptions.backoff.type, + delay: spec.jobOptions.backoff.delayMs, + ...(spec.jobOptions.backoff.jitter !== undefined + ? { jitter: spec.jobOptions.backoff.jitter } + : {}), + }, + removeOnComplete: spec.jobOptions.keepJobs.completed, + removeOnFail: spec.jobOptions.keepJobs.failed, + keepLogs: spec.jobOptions.keepLogs, + }); + + if (!job.id) { + throw new Error( + `BullMQ did not return an id for workload "${spec.name}"`, + ); + } + + return job.id; + } + + async upsertJobScheduler( + spec: QueueSpec, + schedulerId: string, + schedule: Schedule, + data: DataOf, + options: JobEnqueueOptions = {}, + ): Promise { + const queue = this.getQueue(spec); + const intervalMs = scheduleToMs(schedule); + + // @note: jobs produced by BullMQ's scheduler bypass the deduplication check that + // `Queue.add` goes through, so a dedup key would be silently ignored here. + const job = await queue.upsertJobScheduler( + schedulerId, + { + every: intervalMs, + startDate: Date.now() + intervalMs, + }, + { + name: spec.name, + data, + opts: { + ...(options.priority !== undefined + ? { priority: options.priority } + : {}), + attempts: spec.jobOptions.attempts, + backoff: { + type: spec.jobOptions.backoff.type, + delay: spec.jobOptions.backoff.delayMs, + ...(spec.jobOptions.backoff.jitter !== undefined + ? { jitter: spec.jobOptions.backoff.jitter } + : {}), + }, + removeOnComplete: spec.jobOptions.keepJobs.completed, + removeOnFail: spec.jobOptions.keepJobs.failed, + keepLogs: spec.jobOptions.keepLogs, + }, + }, + ); + + if (!job.id) { + throw new Error( + `BullMQ did not return an id for workload "${spec.name}"`, + ); + } + + return job.id; + } + + async getJobSchedulerIds( + spec: QueueSpec, + ): Promise { + const schedulers = await this.getQueue(spec).getJobSchedulers(); + return schedulers.map(({ key }) => key); + } + + removeJobScheduler( + spec: QueueSpec, + schedulerId: string, + ): Promise { + return this.getQueue(spec).removeJobScheduler(schedulerId); + } + + async close(): Promise { + await Promise.all( + [...this.queues.values()].map((queue) => queue.close()), + ); + } +} diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index c299ef1cc..134774a0c 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -25,7 +25,7 @@ export const DEFAULT_CONFIG_SETTINGS: ConfigSettings = { reindexRepoPollingIntervalMs: 1000 * 1, // 1 second maxConnectionSyncJobConcurrency: 8, maxRepoIndexingJobConcurrency: 8, - maxRepoGarbageCollectionJobConcurrency: 8, + maxRepoGarbageCollectionJobConcurrency: 2, repoGarbageCollectionGracePeriodMs: 10 * 1000, // 10 seconds repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours enablePublicAccess: false, // deprected, use FORCE_ENABLE_ANONYMOUS_ACCESS instead diff --git a/packages/shared/src/env.server.ts b/packages/shared/src/env.server.ts index 2edd9050c..41cb205f4 100644 --- a/packages/shared/src/env.server.ts +++ b/packages/shared/src/env.server.ts @@ -395,7 +395,6 @@ const options = { REDIS_TLS_HONOR_CIPHER_ORDER: booleanSchema.optional(), REDIS_TLS_KEY_PASSPHRASE: z.string().optional(), - CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: numberSchema.default(300000), REPO_SYNC_RETRY_BASE_SLEEP_SECONDS: numberSchema.default(60), GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS: numberSchema.default(60 * 10), diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index 6c1d8d723..da6d7a131 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -21,18 +21,17 @@ export type { export * from './lighthouseTypes.js'; export type { RepoMetadata, - RepoIndexingJobMetadata, IdentityProviderType, LicenseStatus, } from "./types.js"; export { repoMetadataSchema, - repoIndexingJobMetadataSchema, } from "./types.js"; export { base64Decode, loadJsonFile, getConfigSettings, + resolveConfigSettings, getRepoPath, getRepoIdFromPath, isCredentialsLoginEnabled, @@ -94,3 +93,48 @@ export { compareVersions, } from "./versionUtils.js"; export type { Version } from "./versionUtils.js"; +export type { + QueueName, + DataOf, + JobEnqueueOptions, + QueueSpec, + JobOptions, +} from "./queue.js"; +export { + ACCOUNT_PERMISSION_SYNC_QUEUE, + ATTACHMENT_PRUNE_QUEUE, + AUDIT_LOG_PRUNE_QUEUE, + CONNECTION_QUEUE, + DEFAULT_JOB_OPTIONS, + JOB_PRIORITIES, + REPO_INDEX_QUEUE, + REPO_PERMISSION_SYNC_QUEUE, +} from "./queue.js"; +export type { Schedule } from "./schedule.js"; +export { + ACCOUNT_PERMISSION_SYNC_SCHEDULER_ID_PREFIX, + getAccountPermissionSyncSchedulerId, + scheduleToMs, +} from "./schedule.js"; +export { + BullMQClient, +} from "./bullmqClient.js"; +export type { + WorkloadJob, + WorkloadJobStatus, +} from "./bullmqClient.js"; +export { + createBullMQJobLogger, + DEFAULT_JOB_LOGS_MAX_ENTRIES, + parseJobLogEntry, + readBullMQJobLogs, +} from "./jobLogger.js"; +export type { + GetJobLogsOptions, + JobLogEntry, + JobLogFields, + JobLogLevel, + JobLogger, + JobLogs, + JobLogSink, +} from "./jobLogger.js"; diff --git a/packages/shared/src/jobLogger.test.ts b/packages/shared/src/jobLogger.test.ts new file mode 100644 index 000000000..cf727c2ab --- /dev/null +++ b/packages/shared/src/jobLogger.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + applicationLog: vi.fn(), + applicationError: vi.fn(), +})); + +vi.mock("./logger.js", () => ({ + createLogger: vi.fn(() => ({ + log: mocks.applicationLog, + error: mocks.applicationError, + })), +})); + +import { + createBullMQJobLogger, + parseJobLogEntry, + readBullMQJobLogs, +} from "./jobLogger.js"; + +describe("createBullMQJobLogger", () => { + test("writes structured, redacted entries to BullMQ and the application logger", async () => { + const log = vi.fn().mockResolvedValue(1); + const logger = createBullMQJobLogger({ + id: "job-1", + name: "connection", + queueName: "connection", + attemptsMade: 1, + log, + }); + + logger.warn("Some repositories were skipped", { + skipped: 2, + accessToken: "do-not-store", + }); + await logger.flush(); + + expect(mocks.applicationLog).toHaveBeenCalledWith( + "warn", + "Some repositories were skipped", + { + skipped: 2, + accessToken: "[REDACTED]", + }, + ); + + const storedEntry = JSON.parse(log.mock.calls[0][0]); + expect(storedEntry).toMatchObject({ + version: 1, + level: "warn", + message: "Some repositories were skipped", + attempt: 2, + fields: { + skipped: 2, + accessToken: "[REDACTED]", + }, + }); + expect(storedEntry.timestamp).toEqual(expect.any(String)); + }); + + test("does not fail the workload when persisting a log entry fails", async () => { + const logger = createBullMQJobLogger({ + id: "job-1", + name: "connection", + queueName: "connection", + attemptsMade: 0, + log: vi.fn().mockRejectedValue(new Error("Redis unavailable")), + }); + + logger.info("Starting"); + await expect(logger.flush()).resolves.toBeUndefined(); + expect(mocks.applicationError).toHaveBeenCalled(); + }); + + test("uses the supplied attempt for post-processing lifecycle logs", async () => { + const log = vi.fn().mockResolvedValue(1); + const logger = createBullMQJobLogger( + { + id: "job-1", + name: "connection", + queueName: "connection", + attemptsMade: 2, + log, + }, + { attempt: 2 }, + ); + + logger.info("Completed"); + await logger.flush(); + + expect(JSON.parse(log.mock.calls[0][0])).toMatchObject({ attempt: 2 }); + }); +}); + +describe("readBullMQJobLogs", () => { + test("parses structured entries and preserves legacy string logs", async () => { + const structuredEntry = JSON.stringify({ + version: 1, + timestamp: "2026-07-28T03:00:00.000Z", + level: "info", + message: "Started", + attempt: 1, + }); + const queue = { + getJobLogs: vi.fn().mockResolvedValue({ + logs: [structuredEntry, "legacy log"], + count: 2, + }), + }; + + const result = await readBullMQJobLogs(queue, "job-1", { + start: 10, + end: 20, + ascending: true, + }); + + expect(queue.getJobLogs).toHaveBeenCalledWith("job-1", 10, 20, true); + expect(result).toEqual({ + logs: [ + parseJobLogEntry(structuredEntry), + { + version: 0, + timestamp: null, + level: "info", + message: "legacy log", + attempt: null, + }, + ], + count: 2, + }); + }); +}); diff --git a/packages/shared/src/jobLogger.ts b/packages/shared/src/jobLogger.ts new file mode 100644 index 000000000..f496cdab6 --- /dev/null +++ b/packages/shared/src/jobLogger.ts @@ -0,0 +1,226 @@ +import type { Job, Queue } from "bullmq"; +import { createLogger } from "./logger.js"; + +export const DEFAULT_JOB_LOGS_MAX_ENTRIES = 500; + +export type JobLogLevel = "debug" | "info" | "warn" | "error"; +export type JobLogFields = Record; + +export interface JobLogEntry { + version: 1 | 0; + timestamp: string | null; + level: JobLogLevel; + message: string; + attempt: number | null; + fields?: JobLogFields; +} + +export interface JobLogSink { + debug(message: string, fields?: unknown): void; + info(message: string, fields?: unknown): void; + warn(message: string, fields?: unknown): void; + error(message: string, fields?: unknown): void; +} + +export interface JobLogger extends JobLogSink { + flush(): Promise; +} + +export interface GetJobLogsOptions { + start?: number; + end?: number; + ascending?: boolean; +} + +export interface JobLogs { + logs: JobLogEntry[]; + count: number; +} + +type BullMQLogJob = Pick< + Job, + "id" | "name" | "queueName" | "attemptsMade" | "log" +>; +type JobLogQueue = Pick; + +const JOB_LOG_LEVELS = new Set(["debug", "info", "warn", "error"]); +const SENSITIVE_FIELD_NAME = + /authorization|cookie|credential|password|private.?key|secret|token/i; +const MAX_FIELD_DEPTH = 6; + +const sanitizeValue = ( + value: unknown, + seen: WeakSet, + depth: number, +): unknown => { + if (depth > MAX_FIELD_DEPTH) { + return "[Max depth reached]"; + } + if ( + value === null || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + return value; + } + if (typeof value === "bigint") { + return value.toString(); + } + if (typeof value === "undefined") { + return "[undefined]"; + } + if (typeof value === "symbol" || typeof value === "function") { + return String(value); + } + if (value instanceof Date) { + return value.toISOString(); + } + if (value instanceof Error) { + return { + name: value.name, + message: value.message, + stack: value.stack, + }; + } + if (seen.has(value)) { + return "[Circular]"; + } + + seen.add(value); + if (Array.isArray(value)) { + return value.map((item) => sanitizeValue(item, seen, depth + 1)); + } + + return Object.fromEntries( + Object.entries(value).map(([key, nestedValue]) => [ + key, + SENSITIVE_FIELD_NAME.test(key) + ? "[REDACTED]" + : sanitizeValue(nestedValue, seen, depth + 1), + ]), + ); +}; + +const sanitizeFields = (fields: unknown): JobLogFields | undefined => { + if (fields === undefined) { + return undefined; + } + + const sanitized = sanitizeValue(fields, new WeakSet(), 0); + if ( + sanitized !== null && + typeof sanitized === "object" && + !Array.isArray(sanitized) + ) { + return sanitized as JobLogFields; + } + return { value: sanitized }; +}; + +export const parseJobLogEntry = (rawLog: string): JobLogEntry => { + try { + const parsed = JSON.parse(rawLog) as Partial; + if ( + parsed.version === 1 && + typeof parsed.timestamp === "string" && + typeof parsed.level === "string" && + JOB_LOG_LEVELS.has(parsed.level as JobLogLevel) && + typeof parsed.message === "string" && + typeof parsed.attempt === "number" + ) { + return { + version: 1, + timestamp: parsed.timestamp, + level: parsed.level as JobLogLevel, + message: parsed.message, + attempt: parsed.attempt, + ...(parsed.fields ? { fields: parsed.fields } : {}), + }; + } + } catch { + // Older BullMQ logs were stored as plain strings. + } + + return { + version: 0, + timestamp: null, + level: "info", + message: rawLog, + attempt: null, + }; +}; + +export const readBullMQJobLogs = async ( + queue: JobLogQueue, + jobId: string, + options: GetJobLogsOptions = {}, +): Promise => { + const result = await queue.getJobLogs( + jobId, + options.start, + options.end, + options.ascending, + ); + + return { + logs: result.logs.map(parseJobLogEntry), + count: result.count, + }; +}; + +export const createBullMQJobLogger = ( + job: BullMQLogJob, + options: { + label?: string; + attempt?: number; + } = {}, +): JobLogger => { + const label = + options.label ?? `${job.queueName}:job:${job.id ?? "unknown"}`; + const attempt = options.attempt ?? job.attemptsMade + 1; + const applicationLogger = createLogger(label); + const pendingWrites = new Set>(); + + const write = ( + level: JobLogLevel, + message: string, + rawFields?: unknown, + ): void => { + const fields = sanitizeFields(rawFields); + applicationLogger.log(level, message, fields); + + const entry: JobLogEntry = { + version: 1, + timestamp: new Date().toISOString(), + level, + message, + attempt, + ...(fields ? { fields } : {}), + }; + const pendingWrite = job + .log(JSON.stringify(entry)) + .then(() => undefined) + .catch((error: unknown) => { + applicationLogger.error( + `Failed to persist a BullMQ log entry for job ${job.id ?? "unknown"}`, + error, + ); + }); + + pendingWrites.add(pendingWrite); + void pendingWrite.finally(() => { + pendingWrites.delete(pendingWrite); + }); + }; + + return { + debug: (message, fields) => write("debug", message, fields), + info: (message, fields) => write("info", message, fields), + warn: (message, fields) => write("warn", message, fields), + error: (message, fields) => write("error", message, fields), + flush: async () => { + await Promise.all([...pendingWrites]); + }, + }; +}; diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts new file mode 100644 index 000000000..993fc9b6c --- /dev/null +++ b/packages/shared/src/queue.ts @@ -0,0 +1,105 @@ +import type { KeepJobs } from "bullmq"; +import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from "./jobLogger.js"; + +export interface QueueSpec { + name: TName; + dedupKey?(data: DataOf): string; + jobOptions: JobOptions; +} + +export type JobOptions = { + attempts: number; + backoff: { + type: "fixed" | "exponential"; + delayMs: number; + jitter?: number; + }; + keepJobs: { + completed: KeepJobs; + failed: KeepJobs; + }; + keepLogs: number; +}; + +export type JobEnqueueOptions = { + priority?: number; +}; + +export const JOB_PRIORITIES = { + INTERACTIVE: 1, + INITIAL: 5, + SCHEDULED: 10, +} as const; + +const TWO_WEEKS_IN_SECONDS = 14 * 24 * 60 * 60; + +export const DEFAULT_JOB_OPTIONS: JobOptions = { + attempts: 4, + backoff: { + type: "exponential", + delayMs: 30_000, + jitter: 0.5, + }, + keepJobs: { + completed: { age: TWO_WEEKS_IN_SECONDS }, + failed: { age: TWO_WEEKS_IN_SECONDS }, + }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, +}; + +export type QueueName = keyof QueueRegistry; +export type DataOf = QueueRegistry[TName]; + +interface QueueRegistry { + "attachment-prune": Record; + "audit-log-prune": Record; + "connection-sync": { + connectionId: number; + }; + "repo-index": { + repoId: number; + type: "INDEX" | "CLEANUP"; + }; + "account-permission-sync": { + accountId: string; + }; + "repo-permission-sync": { + repoId: number; + }; +} + +export const ATTACHMENT_PRUNE_QUEUE: QueueSpec<"attachment-prune"> = { + name: "attachment-prune", + jobOptions: DEFAULT_JOB_OPTIONS, + dedupKey: () => "global", +}; + +export const AUDIT_LOG_PRUNE_QUEUE: QueueSpec<"audit-log-prune"> = { + name: "audit-log-prune", + jobOptions: DEFAULT_JOB_OPTIONS, + dedupKey: () => "global", +}; + +export const CONNECTION_QUEUE: QueueSpec<"connection-sync"> = { + name: "connection-sync", + jobOptions: DEFAULT_JOB_OPTIONS, + dedupKey: (data) => `connection:${data.connectionId}`, +}; + +export const REPO_INDEX_QUEUE: QueueSpec<"repo-index"> = { + name: "repo-index", + jobOptions: DEFAULT_JOB_OPTIONS, +}; + +export const ACCOUNT_PERMISSION_SYNC_QUEUE: QueueSpec<"account-permission-sync"> = + { + name: "account-permission-sync", + jobOptions: DEFAULT_JOB_OPTIONS, + dedupKey: (data) => `account:${data.accountId}`, + }; + +export const REPO_PERMISSION_SYNC_QUEUE: QueueSpec<"repo-permission-sync"> = { + name: "repo-permission-sync", + jobOptions: DEFAULT_JOB_OPTIONS, + dedupKey: (data) => `repo:${data.repoId}`, +}; diff --git a/packages/shared/src/redis.ts b/packages/shared/src/redis.ts index c016aa966..49e678f2a 100644 --- a/packages/shared/src/redis.ts +++ b/packages/shared/src/redis.ts @@ -1,5 +1,5 @@ import fs from "fs"; -import { Redis } from "ioredis"; +import { Redis, type RedisOptions } from "ioredis"; import { env } from "./env.server.js"; const buildTlsOptions = (): Record => { @@ -43,7 +43,9 @@ const buildTlsOptions = (): Record => { }; }; -export const createRedisClient = () => new Redis(env.REDIS_URL, { - maxRetriesPerRequest: null, - ...buildTlsOptions(), -}); +export const createRedisClient = (options: RedisOptions = {}) => + new Redis(env.REDIS_URL, { + maxRetriesPerRequest: null, + ...buildTlsOptions(), + ...options, + }); diff --git a/packages/shared/src/schedule.test.ts b/packages/shared/src/schedule.test.ts new file mode 100644 index 000000000..3aa8139c6 --- /dev/null +++ b/packages/shared/src/schedule.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from "vitest"; +import { + getAccountPermissionSyncSchedulerId, + scheduleToMs, +} from "./schedule.js"; + +test("builds an account permission sync scheduler ID", () => { + expect(getAccountPermissionSyncSchedulerId("account-1")).toBe( + "account-permission-sync-v1-account-1", + ); +}); + +describe("scheduleToMs", () => { + test.each([ + [500, 500], + ["500ms", 500], + ["30s", 30_000], + ["5m", 300_000], + ["6h", 21_600_000], + ["1d", 86_400_000], + [" 10m ", 600_000], + ])("converts %s to milliseconds", (schedule, expected) => { + expect(scheduleToMs(schedule)).toBe(expected); + }); + + test.each([ + "", + "0s", + "5", + "m", + "5x", + "1.5h", + "-5m", + "5 m", + 0, + -1, + Number.NaN, + Number.POSITIVE_INFINITY, + ])('rejects invalid schedule "%s"', (schedule) => { + expect(() => scheduleToMs(schedule)).toThrow(); + }); +}); diff --git a/packages/shared/src/schedule.ts b/packages/shared/src/schedule.ts new file mode 100644 index 000000000..29f1df6a9 --- /dev/null +++ b/packages/shared/src/schedule.ts @@ -0,0 +1,43 @@ +export type Schedule = string | number; + +export const ACCOUNT_PERMISSION_SYNC_SCHEDULER_ID_PREFIX = + "account-permission-sync-v1-"; + +export const getAccountPermissionSyncSchedulerId = ( + accountId: string, +): string => `${ACCOUNT_PERMISSION_SYNC_SCHEDULER_ID_PREFIX}${accountId}`; + +const SCHEDULE_UNITS_MS: Record = { + ms: 1, + s: 1000, + m: 1000 * 60, + h: 1000 * 60 * 60, + d: 1000 * 60 * 60 * 24, +}; + +export const scheduleToMs = (schedule: Schedule): number => { + if (typeof schedule === "number") { + if (!Number.isFinite(schedule) || schedule <= 0) { + throw new Error( + `Invalid schedule "${schedule}". Expected a positive number of milliseconds.`, + ); + } + return schedule; + } + + const match = /^(\d+)(ms|s|m|h|d)$/.exec(schedule.trim()); + if (!match) { + throw new Error( + `Invalid schedule "${schedule}". Expected e.g. "500ms", "30s", "5m", "6h", or "1d".`, + ); + } + + const intervalMs = Number(match[1]) * SCHEDULE_UNITS_MS[match[2]]; + if (!Number.isFinite(intervalMs) || intervalMs <= 0) { + throw new Error( + `Invalid schedule "${schedule}". Expected a positive finite interval.`, + ); + } + + return intervalMs; +}; diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 5951bab6b..9f93637c9 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -63,15 +63,6 @@ export const repoMetadataSchema = z.object({ export type RepoMetadata = z.infer; -export const repoIndexingJobMetadataSchema = z.object({ - /** - * A list of revisions that were indexed for the repo. - */ - indexedRevisions: z.array(z.string()).optional(), -}); - -export type RepoIndexingJobMetadata = z.infer; - export type IdentityProviderType = IdentityProviderConfig['provider']; // @see: https://docs.stripe.com/api/subscriptions/object#subscription_object-status diff --git a/packages/shared/src/utils.test.ts b/packages/shared/src/utils.test.ts index c346c2c5a..14e1de4e8 100644 --- a/packages/shared/src/utils.test.ts +++ b/packages/shared/src/utils.test.ts @@ -1,7 +1,7 @@ import { readFile } from 'fs/promises'; import { beforeEach, describe, expect, test, vi } from 'vitest'; import { DEFAULT_CONFIG_SETTINGS } from './constants.js'; -import { getConfigSettings } from './utils.js'; +import { getConfigSettings, resolveConfigSettings } from './utils.js'; // Mock fs/promises so loadConfig doesn't hit the filesystem. // The config schema has no required fields, so '{}' is valid. @@ -95,3 +95,18 @@ describe('getConfigSettings', () => { }); }); }); + +describe('resolveConfigSettings', () => { + test('resolves settings from an already-loaded config', () => { + const result = resolveConfigSettings({ + settings: { + resyncConnectionIntervalMs: 12_345, + }, + }); + + expect(result.resyncConnectionIntervalMs).toBe(12_345); + expect(result.reindexIntervalMs).toBe( + DEFAULT_CONFIG_SETTINGS.reindexIntervalMs, + ); + }); +}); diff --git a/packages/shared/src/utils.ts b/packages/shared/src/utils.ts index 848b941eb..2a1867fc4 100644 --- a/packages/shared/src/utils.ts +++ b/packages/shared/src/utils.ts @@ -4,9 +4,9 @@ import { z } from "zod"; import { DEFAULT_CONFIG_SETTINGS } from "./constants.js"; import { ConfigSettings } from "./types.js"; import { Org, Repo } from "@sourcebot/db"; +import type { SourcebotConfig } from "@sourcebot/schemas/v3/index.type"; import path from "path"; import { env, isRemotePath, loadConfig } from "./env.server.js"; -import { isAnonymousAccessAvailable } from './entitlements.js'; // From https://developer.mozilla.org/en-US/docs/Glossary/Base64#the_unicode_problem export const base64Decode = (base64: string): string => { @@ -75,14 +75,8 @@ export const loadJsonFile = async ( } -export const getConfigSettings = async (configPath?: string): Promise => { - if (!configPath) { - return DEFAULT_CONFIG_SETTINGS; - } - - const config = await loadConfig(configPath); - - return { +export const resolveConfigSettings = (config: SourcebotConfig): ConfigSettings => + ({ ...DEFAULT_CONFIG_SETTINGS, ...config.settings, // Fall back to deprecated experiment_ variants if new keys are not set. @@ -94,7 +88,14 @@ export const getConfigSettings = async (configPath?: string): Promise => { + if (!configPath) { + return DEFAULT_CONFIG_SETTINGS; } + + return resolveConfigSettings(await loadConfig(configPath)); } export const getRepoIdFromPath = (repoPath: string): number | undefined => { @@ -143,4 +144,4 @@ export const isMemberApprovalRequired = (org: Org): boolean => { } return org.memberApprovalRequired; -} \ No newline at end of file +} diff --git a/packages/shared/vitest.config.ts b/packages/shared/vitest.config.ts index 35f8c7016..30085023a 100644 --- a/packages/shared/vitest.config.ts +++ b/packages/shared/vitest.config.ts @@ -2,6 +2,7 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { + include: ['src/**/*.test.ts'], environment: 'node', watch: false, env: { diff --git a/packages/web/src/app/(app)/repos/components/repoActionsDropdown.tsx b/packages/web/src/app/(app)/repos/components/repoActionsDropdown.tsx index 4a0049892..878ba3f2b 100644 --- a/packages/web/src/app/(app)/repos/components/repoActionsDropdown.tsx +++ b/packages/web/src/app/(app)/repos/components/repoActionsDropdown.tsx @@ -6,7 +6,7 @@ import { getCodeHostInfoForRepo, isServiceError } from "@/lib/utils" import { ExternalLink, MoreHorizontal } from "lucide-react" import Link from "next/link" import { useState } from "react" -import { indexRepo } from "@/features/workerApi/actions" +import { indexRepo } from "@/features/repos/actions" import { useRouter } from "next/navigation" import { useToast } from "@/components/hooks/use-toast" import type { Repo } from "./reposTable" @@ -34,7 +34,7 @@ export const RepoActionsDropdown = ({ repo }: RepoActionsDropdownProps) => { if (!isServiceError(response)) { const { jobId } = response toast({ - description: `✅ Repository sync triggered successfully. Job ID: ${jobId}`, + description: `✅ Repository indexing scheduled. Job ID: ${jobId}`, }) router.refresh() } else { diff --git a/packages/web/src/app/(app)/repos/components/repoJobsTable.tsx b/packages/web/src/app/(app)/repos/components/repoJobsTable.tsx index a108d1b18..7f82ff796 100644 --- a/packages/web/src/app/(app)/repos/components/repoJobsTable.tsx +++ b/packages/web/src/app/(app)/repos/components/repoJobsTable.tsx @@ -27,7 +27,7 @@ import { useRouter } from "next/navigation" import { useToast } from "@/components/hooks/use-toast" import { DisplayDate } from "../../components/DisplayDate" import { LoadingButton } from "@/components/ui/loading-button" -import { indexRepo } from "@/features/workerApi/actions" +import { indexRepo } from "@/features/repos/actions" import { isServiceError } from "@/lib/utils" // @see: https://v0.app/chat/repo-indexing-status-uhjdDim8OUS @@ -202,7 +202,7 @@ export const RepoJobsTable = ({ if (!isServiceError(response)) { const { jobId } = response; toast({ - description: `✅ Repository sync triggered successfully. Job ID: ${jobId}`, + description: `✅ Repository indexing scheduled. Job ID: ${jobId}`, }) router.refresh(); } else { diff --git a/packages/web/src/app/(app)/repos/components/reposTable.tsx b/packages/web/src/app/(app)/repos/components/reposTable.tsx index cbbc51676..865f19402 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.tsx @@ -26,7 +26,7 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip import { NotificationDot } from "../../components/notificationDot" import { CodeHostType } from "@sourcebot/db" import { useHotkeys } from "react-hotkeys-hook" -import { indexRepo } from "@/features/workerApi/actions" +import { indexRepo } from "@/features/repos/actions" import { RepoActionsDropdown } from "./repoActionsDropdown" // @see: https://v0.app/chat/repo-indexing-status-uhjdDim8OUS @@ -352,7 +352,7 @@ export const ReposTable = ({ if (!isServiceError(response)) { const { jobId } = response; toast({ - description: `✅ Repository sync triggered successfully. Job ID: ${jobId}`, + description: `✅ Repository indexing scheduled. Job ID: ${jobId}`, }); router.refresh(); } else { diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx index 9277d81b7..0600a5c37 100644 --- a/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx +++ b/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx @@ -27,7 +27,7 @@ import { useRouter } from "next/navigation" import { useToast } from "@/components/hooks/use-toast" import { DisplayDate } from "@/app/(app)/components/DisplayDate" import { LoadingButton } from "@/components/ui/loading-button" -import { syncConnection } from "@/features/workerApi/actions" +import { syncConnection } from "@/features/connections/actions" import { isServiceError } from "@/lib/utils" @@ -199,7 +199,7 @@ export const ConnectionJobsTable = ({ data, connectionId }: { data: ConnectionSy if (!isServiceError(response)) { const { jobId } = response; toast({ - description: `✅ Connection synced successfully. Job ID: ${jobId}`, + description: `✅ Connection sync scheduled. Job ID: ${jobId}`, }) router.refresh(); } else { diff --git a/packages/web/src/auth.ts b/packages/web/src/auth.ts index 8f1932c0c..b6f132b9a 100644 --- a/packages/web/src/auth.ts +++ b/packages/web/src/auth.ts @@ -4,7 +4,7 @@ import NextAuth, { DefaultSession, Session, User as AuthJsUser } from "next-auth import Credentials from "next-auth/providers/credentials" import EmailProvider from "next-auth/providers/nodemailer"; import { __unsafePrisma } from "@/prisma"; -import { createLogger, env, getSMTPConnectionURL } from "@sourcebot/shared"; +import { createLogger, doesIdpSupportPermissionSyncing, env, getSMTPConnectionURL } from "@sourcebot/shared"; import { User } from '@sourcebot/db'; import 'next-auth/jwt'; import type { Provider } from "next-auth/providers"; @@ -23,7 +23,7 @@ import { captureEvent } from '@/lib/posthog'; import { isEmailCodeLoginEnabled, isCredentialsLoginEnabled } from '@sourcebot/shared' import { onCreateUser } from './features/membership/onCreateUser'; import { setSentryUser } from './lib/sentryUser'; -import { requestAccountPermissionSync } from './features/workerApi/client.server'; +import { scheduleAndTriggerAccountPermissionSync } from './ee/features/permissionSync/accountPermissionSyncQueue.server'; export const runtime = 'nodejs'; const logger = createLogger('auth'); @@ -225,17 +225,24 @@ const nextAuthResult = NextAuth(async () => ({ }) }); + // Once the latest OAuth credentials and issuer URL are persisted, + // ensure a recurring permission-sync schedule exists and trigger a + // sync for new accounts or accounts recovering from a previous issue. if ( - updatedAccount.permissionSyncIssue !== null && - env.PERMISSION_SYNC_ENABLED === 'true' + env.PERMISSION_SYNC_ENABLED === 'true' && + doesIdpSupportPermissionSyncing(updatedAccount.providerType) && + ( + updatedAccount.permissionSyncedAt === null || + updatedAccount.permissionSyncIssue !== null + ) ) { try { if (await hasEntitlement('permission-syncing')) { - await requestAccountPermissionSync(updatedAccount.id); + await scheduleAndTriggerAccountPermissionSync(updatedAccount.id); } } catch (error) { const message = error instanceof Error ? error.message : String(error); - logger.error(`Failed to schedule permission sync after reauthentication for account ${updatedAccount.id}: ${message}`); + logger.error(`Failed to schedule permission sync after authentication for account ${updatedAccount.id}: ${message}`); } } } diff --git a/packages/web/src/ee/features/permissionSync/accountPermissionSyncQueue.server.test.ts b/packages/web/src/ee/features/permissionSync/accountPermissionSyncQueue.server.test.ts new file mode 100644 index 000000000..593b5d3b2 --- /dev/null +++ b/packages/web/src/ee/features/permissionSync/accountPermissionSyncQueue.server.test.ts @@ -0,0 +1,85 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + enqueue: vi.fn(), + getConfigSettings: vi.fn(), + removeJobScheduler: vi.fn(), + upsertJobScheduler: vi.fn(), +})); + +const accountPermissionSyncQueue = { + name: 'account-permission-sync', +}; + +vi.mock('server-only', () => ({})); +vi.mock('@/lib/bullmqClient', () => ({ + getBullMQClient: () => ({ + enqueue: mocks.enqueue, + removeJobScheduler: mocks.removeJobScheduler, + upsertJobScheduler: mocks.upsertJobScheduler, + }), +})); +vi.mock('@sourcebot/shared', () => ({ + ACCOUNT_PERMISSION_SYNC_QUEUE: accountPermissionSyncQueue, + env: { CONFIG_PATH: '/config.json' }, + getAccountPermissionSyncSchedulerId: (accountId: string) => + `account-permission-sync-v1-${accountId}`, + getConfigSettings: mocks.getConfigSettings, + JOB_PRIORITIES: { + INTERACTIVE: 1, + SCHEDULED: 10, + }, +})); + +const { + removeAccountPermissionSyncScheduler, + scheduleAndTriggerAccountPermissionSync, +} = await import('./accountPermissionSyncQueue.server'); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.getConfigSettings.mockResolvedValue({ + userDrivenPermissionSyncIntervalMs: 86_400_000, + }); + mocks.upsertJobScheduler.mockResolvedValue('scheduled-job'); + mocks.enqueue.mockResolvedValue('job-1'); + mocks.removeJobScheduler.mockResolvedValue(true); +}); + +describe('scheduleAndTriggerAccountPermissionSync', () => { + test('upserts the account scheduler before enqueueing an immediate sync', async () => { + await expect( + scheduleAndTriggerAccountPermissionSync('account-1'), + ).resolves.toEqual({ jobId: 'job-1' }); + + expect(mocks.getConfigSettings).toHaveBeenCalledWith('/config.json'); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + accountPermissionSyncQueue, + 'account-permission-sync-v1-account-1', + 86_400_000, + { accountId: 'account-1' }, + { priority: 10 }, + ); + expect(mocks.enqueue).toHaveBeenCalledWith( + accountPermissionSyncQueue, + { accountId: 'account-1' }, + { priority: 1 }, + ); + expect( + mocks.upsertJobScheduler.mock.invocationCallOrder[0], + ).toBeLessThan(mocks.enqueue.mock.invocationCallOrder[0]); + }); +}); + +describe('removeAccountPermissionSyncScheduler', () => { + test('removes the scheduler using the account scheduler ID', async () => { + await expect( + removeAccountPermissionSyncScheduler('account-1'), + ).resolves.toBe(true); + + expect(mocks.removeJobScheduler).toHaveBeenCalledWith( + accountPermissionSyncQueue, + 'account-permission-sync-v1-account-1', + ); + }); +}); diff --git a/packages/web/src/ee/features/permissionSync/accountPermissionSyncQueue.server.ts b/packages/web/src/ee/features/permissionSync/accountPermissionSyncQueue.server.ts new file mode 100644 index 000000000..94054fffe --- /dev/null +++ b/packages/web/src/ee/features/permissionSync/accountPermissionSyncQueue.server.ts @@ -0,0 +1,41 @@ +import 'server-only'; + +import { getBullMQClient } from '@/lib/bullmqClient'; +import { + ACCOUNT_PERMISSION_SYNC_QUEUE, + env, + getAccountPermissionSyncSchedulerId, + getConfigSettings, + JOB_PRIORITIES, +} from '@sourcebot/shared'; + +export const scheduleAndTriggerAccountPermissionSync = async ( + accountId: string, +): Promise<{ jobId: string }> => { + const settings = await getConfigSettings(env.CONFIG_PATH); + const client = getBullMQClient(); + const data = { accountId }; + + await client.upsertJobScheduler( + ACCOUNT_PERMISSION_SYNC_QUEUE, + getAccountPermissionSyncSchedulerId(accountId), + settings.userDrivenPermissionSyncIntervalMs, + data, + { priority: JOB_PRIORITIES.SCHEDULED }, + ); + const jobId = await client.enqueue( + ACCOUNT_PERMISSION_SYNC_QUEUE, + data, + { priority: JOB_PRIORITIES.INTERACTIVE }, + ); + + return { jobId }; +}; + +export const removeAccountPermissionSyncScheduler = ( + accountId: string, +): Promise => + getBullMQClient().removeJobScheduler( + ACCOUNT_PERMISSION_SYNC_QUEUE, + getAccountPermissionSyncSchedulerId(accountId), + ); diff --git a/packages/web/src/ee/features/sso/actions.test.ts b/packages/web/src/ee/features/sso/actions.test.ts new file mode 100644 index 000000000..4cec5a018 --- /dev/null +++ b/packages/web/src/ee/features/sso/actions.test.ts @@ -0,0 +1,156 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + authContext: undefined as unknown, + hasEntitlement: vi.fn(), + removeAccountPermissionSyncScheduler: vi.fn(), + scheduleAndTriggerAccountPermissionSync: vi.fn(), +})); + +vi.mock('@/middleware/sew', () => ({ + sew: (callback: () => unknown) => callback(), +})); +vi.mock('@/middleware/withAuth', () => ({ + withAuth: (callback: (context: unknown) => unknown) => + callback(mocks.authContext), +})); +vi.mock('@/middleware/withMinimumOrgRole', () => ({ + withMinimumOrgRole: ( + _role: unknown, + _minimumRole: unknown, + callback: () => unknown, + ) => callback(), +})); +vi.mock('@/ee/features/permissionSync/accountPermissionSyncQueue.server', () => ({ + removeAccountPermissionSyncScheduler: + mocks.removeAccountPermissionSyncScheduler, + scheduleAndTriggerAccountPermissionSync: + mocks.scheduleAndTriggerAccountPermissionSync, +})); +vi.mock('@/lib/entitlements', () => ({ + hasEntitlement: mocks.hasEntitlement, +})); +vi.mock('@/lib/serviceError', () => ({ + unexpectedError: (message: string) => ({ error: message }), +})); +vi.mock('@sourcebot/shared', () => ({ + createLogger: () => ({ info: vi.fn() }), + doesIdpSupportPermissionSyncing: () => true, + env: { PERMISSION_SYNC_ENABLED: 'true' }, + getIdentityProviderConfig: vi.fn(), + getIdentityProviderConfigs: vi.fn(), +})); +vi.mock('next/headers', () => ({ + cookies: vi.fn(), +})); + +const { + triggerAccountPermissionSync, + unlinkLinkedAccountProvider, +} = await import('./actions'); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.hasEntitlement.mockResolvedValue(true); + mocks.removeAccountPermissionSyncScheduler.mockResolvedValue(true); + mocks.scheduleAndTriggerAccountPermissionSync.mockResolvedValue({ + jobId: 'job-1', + }); +}); + +describe('triggerAccountPermissionSync', () => { + test('enqueues a sync for an eligible account owned by the user', async () => { + const findFirst = vi.fn().mockResolvedValue({ providerType: 'github' }); + mocks.authContext = { + prisma: { account: { findFirst } }, + role: 'MEMBER', + user: { id: 'user-1' }, + }; + + await expect( + triggerAccountPermissionSync('account-1'), + ).resolves.toEqual({ jobId: 'job-1' }); + + expect(findFirst).toHaveBeenCalledWith({ + where: { + id: 'account-1', + userId: 'user-1', + }, + select: { + providerType: true, + }, + }); + expect( + mocks.scheduleAndTriggerAccountPermissionSync, + ).toHaveBeenCalledWith('account-1'); + }); + + test('does not enqueue a sync for an account the user does not own', async () => { + mocks.authContext = { + prisma: { + account: { findFirst: vi.fn().mockResolvedValue(null) }, + }, + role: 'MEMBER', + user: { id: 'user-1' }, + }; + + await expect( + triggerAccountPermissionSync('account-2'), + ).resolves.toEqual({ + error: 'Account does not support permission syncing', + }); + expect( + mocks.scheduleAndTriggerAccountPermissionSync, + ).not.toHaveBeenCalled(); + }); +}); + +describe('unlinkLinkedAccountProvider', () => { + test('removes account schedulers before deleting the accounts', async () => { + const findMany = vi.fn().mockResolvedValue([ + { id: 'account-1' }, + { id: 'account-2' }, + ]); + const deleteMany = vi.fn().mockResolvedValue({ count: 2 }); + mocks.authContext = { + prisma: { account: { deleteMany, findMany } }, + role: 'MEMBER', + user: { id: 'user-1' }, + }; + + await expect( + unlinkLinkedAccountProvider('github'), + ).resolves.toEqual({ success: true, count: 2 }); + + const where = { providerId: 'github', userId: 'user-1' }; + expect(findMany).toHaveBeenCalledWith({ + where, + select: { id: true }, + }); + expect(mocks.removeAccountPermissionSyncScheduler).toHaveBeenCalledTimes(2); + expect(mocks.removeAccountPermissionSyncScheduler).toHaveBeenCalledWith('account-1'); + expect(mocks.removeAccountPermissionSyncScheduler).toHaveBeenCalledWith('account-2'); + expect(deleteMany).toHaveBeenCalledWith({ where }); + expect( + mocks.removeAccountPermissionSyncScheduler.mock.invocationCallOrder[1], + ).toBeLessThan(deleteMany.mock.invocationCallOrder[0]); + }); + + test('does not delete accounts when scheduler removal fails', async () => { + const findMany = vi.fn().mockResolvedValue([{ id: 'account-1' }]); + const deleteMany = vi.fn(); + mocks.authContext = { + prisma: { account: { deleteMany, findMany } }, + role: 'MEMBER', + user: { id: 'user-1' }, + }; + mocks.removeAccountPermissionSyncScheduler.mockRejectedValue( + new Error('Redis unavailable'), + ); + + await expect( + unlinkLinkedAccountProvider('github'), + ).rejects.toThrow('Redis unavailable'); + expect(deleteMany).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/web/src/ee/features/sso/actions.ts b/packages/web/src/ee/features/sso/actions.ts index f5734c71b..23cf6a10c 100644 --- a/packages/web/src/ee/features/sso/actions.ts +++ b/packages/web/src/ee/features/sso/actions.ts @@ -8,6 +8,8 @@ import { OrgRole, type AccountPermissionSyncIssue } from "@sourcebot/db"; import { hasEntitlement } from "@/lib/entitlements"; import { createLogger, doesIdpSupportPermissionSyncing, env, getIdentityProviderConfig, getIdentityProviderConfigs } from "@sourcebot/shared"; import { cookies } from "next/headers"; +import { removeAccountPermissionSyncScheduler, scheduleAndTriggerAccountPermissionSync } from "@/ee/features/permissionSync/accountPermissionSyncQueue.server"; +import { unexpectedError } from "@/lib/serviceError"; const logger = createLogger('web-ee-sso-actions'); @@ -91,17 +93,65 @@ export const getLinkedAccounts = async () => sew(() => ) ); +export const triggerAccountPermissionSync = async (accountId: string) => sew(() => + withAuth(({ prisma, role, user }) => + withMinimumOrgRole(role, OrgRole.MEMBER, async () => { + try { + if ( + env.PERMISSION_SYNC_ENABLED !== 'true' || + !await hasEntitlement('permission-syncing') + ) { + return unexpectedError('Permission syncing is not enabled'); + } + + const account = await prisma.account.findFirst({ + where: { + id: accountId, + userId: user.id, + }, + select: { + providerType: true, + }, + }); + if ( + !account || + !doesIdpSupportPermissionSyncing(account.providerType) + ) { + return unexpectedError('Account does not support permission syncing'); + } + + return await scheduleAndTriggerAccountPermissionSync(accountId); + } catch { + return unexpectedError('Failed to trigger account permission sync'); + } + }) + ) +); export const unlinkLinkedAccountProvider = async (providerId: string) => sew(() => withAuth(async ({ prisma, role, user }) => withMinimumOrgRole(role, OrgRole.MEMBER, async () => { - const result = await prisma.account.deleteMany({ - where: { - providerId, - userId: user.id, + const where = { + providerId, + userId: user.id, + }; + const accounts = await prisma.account.findMany({ + where, + select: { + id: true, }, }); + await Promise.all( + accounts.map(({ id }) => + removeAccountPermissionSyncScheduler(id), + ), + ); + + const result = await prisma.account.deleteMany({ + where, + }); + logger.info(`Unlinked account provider ${providerId} for user ${user.id}. Deleted ${result.count} account(s).`); return { success: true, count: result.count }; diff --git a/packages/web/src/ee/features/sso/components/linkedAccountProviderCard.test.tsx b/packages/web/src/ee/features/sso/components/linkedAccountProviderCard.test.tsx index 8e54b2c55..4f6ddb3f2 100644 --- a/packages/web/src/ee/features/sso/components/linkedAccountProviderCard.test.tsx +++ b/packages/web/src/ee/features/sso/components/linkedAccountProviderCard.test.tsx @@ -24,11 +24,8 @@ vi.mock('@/components/hooks/use-toast', () => ({ })); vi.mock('@/ee/features/sso/actions', () => ({ - unlinkLinkedAccountProvider: vi.fn(), -})); - -vi.mock('@/features/workerApi/actions', () => ({ triggerAccountPermissionSync: mocks.triggerAccountPermissionSync, + unlinkLinkedAccountProvider: vi.fn(), })); vi.mock('@/app/api/(client)/client', () => ({ diff --git a/packages/web/src/ee/features/sso/components/linkedAccountProviderCard.tsx b/packages/web/src/ee/features/sso/components/linkedAccountProviderCard.tsx index ceb3ef79a..d6b378fd8 100644 --- a/packages/web/src/ee/features/sso/components/linkedAccountProviderCard.tsx +++ b/packages/web/src/ee/features/sso/components/linkedAccountProviderCard.tsx @@ -4,7 +4,7 @@ import { useEffect, useState } from "react"; import { cn, getAuthProviderInfo, unwrapServiceError } from "@/lib/utils"; import { AlertCircle, ArrowUpRight, ChevronDown, RefreshCw, Unlink } from "lucide-react"; import { ProviderIcon } from "./providerIcon"; -import { LinkedAccount } from "@/ee/features/sso/actions"; +import { LinkedAccount, triggerAccountPermissionSync, unlinkLinkedAccountProvider } from "@/ee/features/sso/actions"; import { LoadingButton } from "@/components/ui/loading-button"; import { @@ -13,8 +13,6 @@ import { DropdownMenuItem, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; -import { unlinkLinkedAccountProvider } from "@/ee/features/sso/actions"; -import { triggerAccountPermissionSync } from "@/features/workerApi/actions"; import { isServiceError } from "@/lib/utils"; import { useRouter } from "next/navigation"; import { useToast } from "@/components/hooks/use-toast"; diff --git a/packages/web/src/features/connections/actions.test.ts b/packages/web/src/features/connections/actions.test.ts new file mode 100644 index 000000000..ff9325229 --- /dev/null +++ b/packages/web/src/features/connections/actions.test.ts @@ -0,0 +1,92 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + authContext: undefined as unknown, + enqueue: vi.fn(), +})); + +const connectionQueue = { + name: 'connection-sync', +}; + +vi.mock('@/lib/bullmqClient', () => ({ + getBullMQClient: () => ({ enqueue: mocks.enqueue }), +})); +vi.mock('@/lib/serviceError', () => ({ + unexpectedError: (message: string) => ({ error: message }), +})); +vi.mock('@/middleware/sew', () => ({ + sew: (callback: () => unknown) => callback(), +})); +vi.mock('@/middleware/withAuth', () => ({ + withAuth: (callback: (context: unknown) => unknown) => + callback(mocks.authContext), +})); +vi.mock('@/middleware/withMinimumOrgRole', () => ({ + withMinimumOrgRole: ( + _role: unknown, + _minimumRole: unknown, + callback: () => unknown, + ) => callback(), +})); +vi.mock('@sourcebot/shared', () => ({ + CONNECTION_QUEUE: connectionQueue, + JOB_PRIORITIES: { INTERACTIVE: 1 }, +})); + +const { syncConnection } = await import('./actions'); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.enqueue.mockResolvedValue('job-1'); +}); + +const setAuthContext = (findFirst: ReturnType) => { + mocks.authContext = { + org: { id: 1 }, + prisma: { connection: { findFirst } }, + role: 'OWNER', + }; +}; + +describe('syncConnection', () => { + test('enqueues an interactive sync for an existing connection', async () => { + const findFirst = vi.fn().mockResolvedValue({ id: 42 }); + setAuthContext(findFirst); + + await expect(syncConnection(42)).resolves.toEqual({ jobId: 'job-1' }); + + expect(findFirst).toHaveBeenCalledWith({ + where: { + id: 42, + orgId: 1, + }, + select: { + id: true, + }, + }); + expect(mocks.enqueue).toHaveBeenCalledWith( + connectionQueue, + { connectionId: 42 }, + { priority: 1 }, + ); + }); + + test('does not enqueue a sync for a missing connection', async () => { + setAuthContext(vi.fn().mockResolvedValue(null)); + + await expect(syncConnection(42)).resolves.toEqual({ + error: 'Failed to sync connection', + }); + expect(mocks.enqueue).not.toHaveBeenCalled(); + }); + + test('returns a service error when Redis rejects the job', async () => { + setAuthContext(vi.fn().mockResolvedValue({ id: 42 })); + mocks.enqueue.mockRejectedValue(new Error('Redis unavailable')); + + await expect(syncConnection(42)).resolves.toEqual({ + error: 'Failed to sync connection', + }); + }); +}); diff --git a/packages/web/src/features/connections/actions.ts b/packages/web/src/features/connections/actions.ts new file mode 100644 index 000000000..d6563cf95 --- /dev/null +++ b/packages/web/src/features/connections/actions.ts @@ -0,0 +1,40 @@ +'use server'; + +import { getBullMQClient } from '@/lib/bullmqClient'; +import { unexpectedError } from '@/lib/serviceError'; +import { sew } from '@/middleware/sew'; +import { withAuth } from '@/middleware/withAuth'; +import { withMinimumOrgRole } from '@/middleware/withMinimumOrgRole'; +import { OrgRole } from '@sourcebot/db'; +import { CONNECTION_QUEUE, JOB_PRIORITIES } from '@sourcebot/shared'; + +export const syncConnection = async (connectionId: number) => sew(() => + withAuth(({ org, prisma, role }) => + withMinimumOrgRole(role, OrgRole.OWNER, async () => { + try { + const connection = await prisma.connection.findFirst({ + where: { + id: connectionId, + orgId: org.id, + }, + select: { + id: true, + }, + }); + if (!connection) { + return unexpectedError('Failed to sync connection'); + } + + const jobId = await getBullMQClient().enqueue( + CONNECTION_QUEUE, + { connectionId: connection.id }, + { priority: JOB_PRIORITIES.INTERACTIVE }, + ); + + return { jobId }; + } catch { + return unexpectedError('Failed to sync connection'); + } + }) + ) +); diff --git a/packages/web/src/features/repos/actions.test.ts b/packages/web/src/features/repos/actions.test.ts new file mode 100644 index 000000000..c64d5c0ac --- /dev/null +++ b/packages/web/src/features/repos/actions.test.ts @@ -0,0 +1,92 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + authContext: undefined as unknown, + enqueue: vi.fn(), +})); + +const repoIndexQueue = { + name: 'repo-index', +}; + +vi.mock('@/lib/bullmqClient', () => ({ + getBullMQClient: () => ({ enqueue: mocks.enqueue }), +})); +vi.mock('@/lib/serviceError', () => ({ + unexpectedError: (message: string) => ({ error: message }), +})); +vi.mock('@/middleware/sew', () => ({ + sew: (callback: () => unknown) => callback(), +})); +vi.mock('@/middleware/withAuth', () => ({ + withAuth: (callback: (context: unknown) => unknown) => + callback(mocks.authContext), +})); +vi.mock('@/middleware/withMinimumOrgRole', () => ({ + withMinimumOrgRole: ( + _role: unknown, + _minimumRole: unknown, + callback: () => unknown, + ) => callback(), +})); +vi.mock('@sourcebot/shared', () => ({ + JOB_PRIORITIES: { INTERACTIVE: 1 }, + REPO_INDEX_QUEUE: repoIndexQueue, +})); + +const { indexRepo } = await import('./actions'); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.enqueue.mockResolvedValue('job-1'); +}); + +const setAuthContext = (findFirst: ReturnType) => { + mocks.authContext = { + org: { id: 1 }, + prisma: { repo: { findFirst } }, + role: 'OWNER', + }; +}; + +describe('indexRepo', () => { + test('enqueues an interactive index job for an existing repo', async () => { + const findFirst = vi.fn().mockResolvedValue({ id: 42 }); + setAuthContext(findFirst); + + await expect(indexRepo(42)).resolves.toEqual({ jobId: 'job-1' }); + + expect(findFirst).toHaveBeenCalledWith({ + where: { + id: 42, + orgId: 1, + }, + select: { + id: true, + }, + }); + expect(mocks.enqueue).toHaveBeenCalledWith( + repoIndexQueue, + { repoId: 42, type: 'INDEX' }, + { priority: 1 }, + ); + }); + + test('does not enqueue an index job for a missing repo', async () => { + setAuthContext(vi.fn().mockResolvedValue(null)); + + await expect(indexRepo(42)).resolves.toEqual({ + error: 'Failed to index repo', + }); + expect(mocks.enqueue).not.toHaveBeenCalled(); + }); + + test('returns a service error when Redis rejects the job', async () => { + setAuthContext(vi.fn().mockResolvedValue({ id: 42 })); + mocks.enqueue.mockRejectedValue(new Error('Redis unavailable')); + + await expect(indexRepo(42)).resolves.toEqual({ + error: 'Failed to index repo', + }); + }); +}); diff --git a/packages/web/src/features/repos/actions.ts b/packages/web/src/features/repos/actions.ts new file mode 100644 index 000000000..286fb1ab8 --- /dev/null +++ b/packages/web/src/features/repos/actions.ts @@ -0,0 +1,40 @@ +'use server'; + +import { getBullMQClient } from '@/lib/bullmqClient'; +import { unexpectedError } from '@/lib/serviceError'; +import { sew } from '@/middleware/sew'; +import { withAuth } from '@/middleware/withAuth'; +import { withMinimumOrgRole } from '@/middleware/withMinimumOrgRole'; +import { OrgRole } from '@sourcebot/db'; +import { JOB_PRIORITIES, REPO_INDEX_QUEUE } from '@sourcebot/shared'; + +export const indexRepo = async (repoId: number) => sew(() => + withAuth(({ org, prisma, role }) => + withMinimumOrgRole(role, OrgRole.OWNER, async () => { + try { + const repo = await prisma.repo.findFirst({ + where: { + id: repoId, + orgId: org.id, + }, + select: { + id: true, + }, + }); + if (!repo) { + return unexpectedError('Failed to index repo'); + } + + const jobId = await getBullMQClient().enqueue( + REPO_INDEX_QUEUE, + { repoId: repo.id, type: 'INDEX' }, + { priority: JOB_PRIORITIES.INTERACTIVE }, + ); + + return { jobId }; + } catch { + return unexpectedError('Failed to index repo'); + } + }) + ) +); diff --git a/packages/web/src/features/workerApi/actions.ts b/packages/web/src/features/workerApi/actions.ts index 33c6c60d7..35ab05aab 100644 --- a/packages/web/src/features/workerApi/actions.ts +++ b/packages/web/src/features/workerApi/actions.ts @@ -2,77 +2,12 @@ import { sew } from "@/middleware/sew"; import { githubRateLimited, repositoryNotFound, unexpectedError } from "@/lib/serviceError"; -import { withAuth, withOptionalAuth } from "@/middleware/withAuth"; -import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; -import { OrgRole } from "@sourcebot/db"; +import { withOptionalAuth } from "@/middleware/withAuth"; import { env } from "@sourcebot/shared"; import z from "zod"; -import { requestAccountPermissionSync } from "./client.server"; const WORKER_API_URL = env.WORKER_API_URL; -export const syncConnection = async (connectionId: number) => sew(() => - withAuth(({ role }) => - withMinimumOrgRole(role, OrgRole.OWNER, async () => { - const response = await fetch(`${WORKER_API_URL}/api/sync-connection`, { - method: 'POST', - body: JSON.stringify({ - connectionId - }), - headers: { - 'Content-Type': 'application/json', - }, - }); - - if (!response.ok) { - return unexpectedError('Failed to sync connection'); - } - - const data = await response.json(); - const schema = z.object({ - jobId: z.string(), - }); - return schema.parse(data); - }) - ) -); - -export const indexRepo = async (repoId: number) => sew(() => - withAuth(({ role }) => - withMinimumOrgRole(role, OrgRole.OWNER, async () => { - const response = await fetch(`${WORKER_API_URL}/api/index-repo`, { - method: 'POST', - body: JSON.stringify({ repoId }), - headers: { - 'Content-Type': 'application/json', - }, - }); - - if (!response.ok) { - return unexpectedError('Failed to index repo'); - } - - const data = await response.json(); - const schema = z.object({ - jobId: z.string(), - }); - return schema.parse(data); - }) - ) -); - -export const triggerAccountPermissionSync = async (accountId: string) => sew(() => - withAuth(({ role }) => - withMinimumOrgRole(role, OrgRole.MEMBER, async () => { - try { - return await requestAccountPermissionSync(accountId); - } catch { - return unexpectedError('Failed to trigger account permission sync'); - } - }) - ) -); - export const addGithubRepo = async (owner: string, repo: string) => sew(() => withOptionalAuth(async () => { const response = await fetch(`${WORKER_API_URL}/api/experimental/add-github-repo`, { diff --git a/packages/web/src/features/workerApi/client.server.test.ts b/packages/web/src/features/workerApi/client.server.test.ts deleted file mode 100644 index 8096f79d9..000000000 --- a/packages/web/src/features/workerApi/client.server.test.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { beforeEach, describe, expect, test, vi } from 'vitest'; - -vi.mock('server-only', () => ({})); -vi.mock('@sourcebot/shared', () => ({ - env: { WORKER_API_URL: 'http://worker.example.com' }, -})); - -const { requestAccountPermissionSync } = await import('./client.server'); - -beforeEach(() => { - vi.unstubAllGlobals(); -}); - -describe('requestAccountPermissionSync', () => { - test('schedules an account permission sync with the worker', async () => { - const fetchMock = vi.fn().mockResolvedValue(new Response( - JSON.stringify({ jobId: 'job_1' }), - { status: 200 }, - )); - vi.stubGlobal('fetch', fetchMock); - - await expect(requestAccountPermissionSync('account_1')).resolves.toEqual({ jobId: 'job_1' }); - expect(fetchMock).toHaveBeenCalledWith( - 'http://worker.example.com/api/trigger-account-permission-sync', - expect.objectContaining({ - method: 'POST', - body: JSON.stringify({ accountId: 'account_1' }), - }), - ); - }); - - test('rejects when the worker does not accept the sync request', async () => { - vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(null, { status: 503 }))); - - await expect(requestAccountPermissionSync('account_1')) - .rejects.toThrow('Worker rejected account permission sync with HTTP 503.'); - }); -}); diff --git a/packages/web/src/features/workerApi/client.server.ts b/packages/web/src/features/workerApi/client.server.ts deleted file mode 100644 index 79b110d13..000000000 --- a/packages/web/src/features/workerApi/client.server.ts +++ /dev/null @@ -1,26 +0,0 @@ -import 'server-only'; - -import { env } from '@sourcebot/shared'; -import { z } from 'zod'; - -const accountPermissionSyncResponseSchema = z.object({ - jobId: z.string(), -}); -const WORKER_REQUEST_TIMEOUT_MS = 5000; - -export const requestAccountPermissionSync = async (accountId: string): Promise<{ jobId: string }> => { - const response = await fetch(`${env.WORKER_API_URL}/api/trigger-account-permission-sync`, { - method: 'POST', - body: JSON.stringify({ accountId }), - headers: { - 'Content-Type': 'application/json', - }, - signal: AbortSignal.timeout(WORKER_REQUEST_TIMEOUT_MS), - }); - - if (!response.ok) { - throw new Error(`Worker rejected account permission sync with HTTP ${response.status}.`); - } - - return accountPermissionSyncResponseSchema.parse(await response.json()); -}; diff --git a/packages/web/src/lib/bullmqClient.ts b/packages/web/src/lib/bullmqClient.ts new file mode 100644 index 000000000..a891bf4bd --- /dev/null +++ b/packages/web/src/lib/bullmqClient.ts @@ -0,0 +1,11 @@ +import 'server-only'; + +import { BullMQClient } from '@sourcebot/shared'; +import { getRedisClient } from './redis'; + +let client: BullMQClient | undefined; + +export function getBullMQClient() { + client ??= new BullMQClient(getRedisClient()); + return client; +} diff --git a/packages/web/src/lib/encryptedPrismaAdapter.test.ts b/packages/web/src/lib/encryptedPrismaAdapter.test.ts new file mode 100644 index 000000000..286d61760 --- /dev/null +++ b/packages/web/src/lib/encryptedPrismaAdapter.test.ts @@ -0,0 +1,79 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + removeAccountPermissionSyncScheduler: vi.fn(), +})); + +vi.mock('@auth/prisma-adapter', () => ({ + PrismaAdapter: () => ({}), +})); +vi.mock('@sourcebot/shared', () => ({ + encryptOAuthToken: (value: unknown) => value, + getIdentityProviderConfig: vi.fn(), +})); +vi.mock('@/ee/features/permissionSync/accountPermissionSyncQueue.server', () => ({ + removeAccountPermissionSyncScheduler: + mocks.removeAccountPermissionSyncScheduler, +})); + +const { EncryptedPrismaAdapter } = await import('./encryptedPrismaAdapter'); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.removeAccountPermissionSyncScheduler.mockResolvedValue(true); +}); + +describe('EncryptedPrismaAdapter.unlinkAccount', () => { + test('removes the scheduler before deleting the account', async () => { + const findUnique = vi.fn().mockResolvedValue({ id: 'account-1' }); + const deleteAccount = vi.fn().mockResolvedValue({}); + const adapter = EncryptedPrismaAdapter({ + account: { + delete: deleteAccount, + findUnique, + }, + } as never); + + await adapter.unlinkAccount?.({ + provider: 'github', + providerAccountId: 'github-user-1', + }); + + const where = { + providerId_providerAccountId: { + providerId: 'github', + providerAccountId: 'github-user-1', + }, + }; + expect(findUnique).toHaveBeenCalledWith({ + where, + select: { id: true }, + }); + expect(mocks.removeAccountPermissionSyncScheduler).toHaveBeenCalledWith( + 'account-1', + ); + expect(deleteAccount).toHaveBeenCalledWith({ where }); + expect( + mocks.removeAccountPermissionSyncScheduler.mock.invocationCallOrder[0], + ).toBeLessThan(deleteAccount.mock.invocationCallOrder[0]); + }); + + test('does not delete the account when scheduler removal fails', async () => { + const deleteAccount = vi.fn(); + const adapter = EncryptedPrismaAdapter({ + account: { + delete: deleteAccount, + findUnique: vi.fn().mockResolvedValue({ id: 'account-1' }), + }, + } as never); + mocks.removeAccountPermissionSyncScheduler.mockRejectedValue( + new Error('Redis unavailable'), + ); + + await expect(adapter.unlinkAccount?.({ + provider: 'github', + providerAccountId: 'github-user-1', + })).rejects.toThrow('Redis unavailable'); + expect(deleteAccount).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/web/src/lib/encryptedPrismaAdapter.ts b/packages/web/src/lib/encryptedPrismaAdapter.ts index 70e145032..b9a813ff9 100644 --- a/packages/web/src/lib/encryptedPrismaAdapter.ts +++ b/packages/web/src/lib/encryptedPrismaAdapter.ts @@ -2,6 +2,7 @@ import { PrismaAdapter } from "@auth/prisma-adapter"; import type { Adapter, AdapterAccount, AdapterUser } from "next-auth/adapters"; import { PrismaClient } from "@sourcebot/db"; import { encryptOAuthToken, getIdentityProviderConfig } from "@sourcebot/shared"; +import { removeAccountPermissionSyncScheduler } from "@/ee/features/permissionSync/accountPermissionSyncQueue.server"; /** * Encrypts OAuth tokens in account data before database storage @@ -58,13 +59,25 @@ export function EncryptedPrismaAdapter(prisma: PrismaClient): Adapter { return (account?.user ?? null) as AdapterUser | null; }, async unlinkAccount({ provider, providerAccountId }) { - await prisma.account.delete({ - where: { - providerId_providerAccountId: { - providerId: provider, - providerAccountId, - }, + const where = { + providerId_providerAccountId: { + providerId: provider, + providerAccountId, }, + }; + const account = await prisma.account.findUnique({ + where, + select: { + id: true, + }, + }); + + if (account) { + await removeAccountPermissionSyncScheduler(account.id); + } + + await prisma.account.delete({ + where, }); }, }; diff --git a/packages/web/src/lib/redis.ts b/packages/web/src/lib/redis.ts index 0eb6e7f92..2f6a1020f 100644 --- a/packages/web/src/lib/redis.ts +++ b/packages/web/src/lib/redis.ts @@ -4,7 +4,13 @@ import { createRedisClient } from '@sourcebot/shared'; let redis: ReturnType | undefined; +const REDIS_REQUEST_TIMEOUT_MS = 5000; + export function getRedisClient() { - redis ??= createRedisClient(); + redis ??= createRedisClient({ + commandTimeout: REDIS_REQUEST_TIMEOUT_MS, + connectTimeout: REDIS_REQUEST_TIMEOUT_MS, + maxRetriesPerRequest: 1, + }); return redis; } diff --git a/schemas/v3/index.json b/schemas/v3/index.json index 874f9f8d5..21499ef0f 100644 --- a/schemas/v3/index.json +++ b/schemas/v3/index.json @@ -30,7 +30,8 @@ "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -50,7 +51,8 @@ "maxRepoGarbageCollectionJobConcurrency": { "type": "number", "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", diff --git a/yarn.lock b/yarn.lock index 9f0096f20..8bbaed9af 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1316,6 +1316,38 @@ __metadata: languageName: node linkType: hard +"@bull-board/api@npm:6.11.2": + version: 6.11.2 + resolution: "@bull-board/api@npm:6.11.2" + dependencies: + redis-info: "npm:^3.1.0" + peerDependencies: + "@bull-board/ui": 6.11.2 + checksum: 10c0/d6a82bdd598d41c4e09dd4e8a001f49e87ed40aa535ca841c5430590f0df206ae49ac8835c2832b00cd6bdb413ef89ea7083a5ffc1b779a307744ea69d83c76d + languageName: node + linkType: hard + +"@bull-board/express@npm:6.11.2": + version: 6.11.2 + resolution: "@bull-board/express@npm:6.11.2" + dependencies: + "@bull-board/api": "npm:6.11.2" + "@bull-board/ui": "npm:6.11.2" + ejs: "npm:^3.1.10" + express: "npm:^4.21.1 || ^5.0.0" + checksum: 10c0/36cf6fb63f51f095934ba6167be8704f45d881c882d56e11e34f9b60e0dac16bc989553292942797af0f62beac2c10f116a852988dadb1bc07cd06d48e1b8ae3 + languageName: node + linkType: hard + +"@bull-board/ui@npm:6.11.2": + version: 6.11.2 + resolution: "@bull-board/ui@npm:6.11.2" + dependencies: + "@bull-board/api": "npm:6.11.2" + checksum: 10c0/7ddcb49222c09f32f63d9a55f000445cd59dd6c73620f5e30ffcf04cfa8727c91918485a7c3d17c709d2b721b4a8d32d2026c2a7f208953cd3facd602e595f03 + languageName: node + linkType: hard + "@cfworker/json-schema@npm:^4.0.2": version: 4.1.1 resolution: "@cfworker/json-schema@npm:4.1.1" @@ -3039,10 +3071,10 @@ __metadata: languageName: node linkType: hard -"@ioredis/commands@npm:^1.1.1": - version: 1.2.0 - resolution: "@ioredis/commands@npm:1.2.0" - checksum: 10c0/a5d3c29dd84d8a28b7c67a441ac1715cbd7337a7b88649c0f17c345d89aa218578d2b360760017c48149ef8a70f44b051af9ac0921a0622c2b479614c4f65b36 +"@ioredis/commands@npm:1.10.0": + version: 1.10.0 + resolution: "@ioredis/commands@npm:1.10.0" + checksum: 10c0/baf91e62d0e64ef2b5f7ca4413dc2456fe250e87483beac4a1c8ef1fe5ad0d2fcdeb9b89d4556d8ef6c7455c64a964359d729601fdb06b2f4c76c35dd59afa99 languageName: node linkType: hard @@ -3624,44 +3656,44 @@ __metadata: languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.4" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.4" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.4" conditions: os=linux & cpu=arm64 languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.4" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.4" conditions: os=linux & cpu=x64 languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.4" conditions: os=win32 & cpu=x64 languageName: node linkType: hard @@ -8978,6 +9010,9 @@ __metadata: version: 0.0.0-use.local resolution: "@sourcebot/backend@workspace:packages/backend" dependencies: + "@bull-board/api": "npm:6.11.2" + "@bull-board/express": "npm:6.11.2" + "@bull-board/ui": "npm:6.11.2" "@coderabbitai/bitbucket": "npm:^1.1.3" "@gitbeaker/rest": "npm:^40.5.1" "@octokit/app": "npm:^16.1.1" @@ -8994,7 +9029,7 @@ __metadata: "@types/node": "npm:^22.7.5" argparse: "npm:^2.0.1" azure-devops-node-api: "npm:^15.1.1" - bullmq: "npm:^5.34.10" + bullmq: "npm:^5.81.3" chokidar: "npm:^4.0.3" cross-env: "npm:^7.0.3" cross-fetch: "npm:^4.0.0" @@ -9006,7 +9041,7 @@ __metadata: gitea-js: "npm:^1.22.0" glob: "npm:^11.1.0" http-status-codes: "npm:^2.3.0" - ioredis: "npm:^5.4.2" + ioredis: "npm:^5.11.1" json-schema-to-typescript: "npm:^15.0.4" lowdb: "npm:^7.0.1" micromatch: "npm:^4.0.8" @@ -9085,14 +9120,16 @@ __metadata: "@google-cloud/secret-manager": "npm:^6.1.1" "@logtail/node": "npm:^0.5.2" "@logtail/winston": "npm:^0.5.2" + "@sentry/node": "npm:^10.40.0" "@sourcebot/db": "workspace:*" "@sourcebot/schemas": "workspace:*" "@t3-oss/env-core": "npm:^0.13.10" "@types/micromatch": "npm:^4.0.9" "@types/node": "npm:^22.7.5" ajv: "npm:^8.17.1" + bullmq: "npm:^5.81.3" cross-env: "npm:^7.0.3" - ioredis: "npm:^5.4.2" + ioredis: "npm:^5.11.1" micromatch: "npm:^4.0.8" strip-json-comments: "npm:^5.0.1" triple-beam: "npm:^1.4.1" @@ -11392,7 +11429,7 @@ __metadata: languageName: node linkType: hard -"async@npm:^3.2.3": +"async@npm:^3.2.3, async@npm:^3.2.6": version: 3.2.6 resolution: "async@npm:3.2.6" checksum: 10c0/36484bb15ceddf07078688d95e27076379cc2f87b10c03b6dd8a83e89475a3c8df5848859dd06a4c95af1e4c16fc973de0171a77f18ea00be899aca2a4f85e70 @@ -11588,19 +11625,19 @@ __metadata: linkType: hard "body-parser@npm:^2.2.1": - version: 2.3.0 - resolution: "body-parser@npm:2.3.0" + version: 2.2.2 + resolution: "body-parser@npm:2.2.2" dependencies: bytes: "npm:^3.1.2" - content-type: "npm:^2.0.0" + content-type: "npm:^1.0.5" debug: "npm:^4.4.3" - http-errors: "npm:^2.0.1" - iconv-lite: "npm:^0.7.2" + http-errors: "npm:^2.0.0" + iconv-lite: "npm:^0.7.0" on-finished: "npm:^2.4.1" - qs: "npm:^6.15.2" - raw-body: "npm:^3.0.2" - type-is: "npm:^2.1.0" - checksum: 10c0/2a8fbbdc471b588338555a3e1a597d1eb0ad0c21cf20fdc3bac5d3f8d9c3a4b19b4163575ab852a43a5dfc0df7770ced5284f979e561f1a24c2f851ce89e695a + qs: "npm:^6.14.1" + raw-body: "npm:^3.0.1" + type-is: "npm:^2.0.1" + checksum: 10c0/95a830a003b38654b75166ca765358aa92ee3d561bf0e41d6ccdde0e1a0c9783cab6b90b20eb635d23172c010b59d3563a137a738e74da4ba714463510d05137 languageName: node linkType: hard @@ -11639,16 +11676,16 @@ __metadata: linkType: hard "brace-expansion@npm:^1.1.13": - version: 1.1.18 - resolution: "brace-expansion@npm:1.1.18" + version: 1.1.14 + resolution: "brace-expansion@npm:1.1.14" dependencies: balanced-match: "npm:^1.0.0" concat-map: "npm:0.0.1" - checksum: 10c0/3432c18a9e2ebf94162d4effb62198bd0adea06a9f332b2c0188df5d5e30b1e51ea3c848b6608e47d0b857ebe1ea5b3888ed3326dd3c4f6f9645c94153cf9c14 + checksum: 10c0/b6fdac832bc4e36a753658c9ed052c2e1a2be221763b002df25d1efbf7d21724334e726a6cd5eadc72a4b19ec3efb632d629cc003bc9c62f7af7a7915ffa4385 languageName: node linkType: hard -"brace-expansion@npm:^2.0.3": +"brace-expansion@npm:^2.0.1": version: 2.1.4 resolution: "brace-expansion@npm:2.1.4" dependencies: @@ -11657,12 +11694,21 @@ __metadata: languageName: node linkType: hard +"brace-expansion@npm:^2.0.3": + version: 2.1.0 + resolution: "brace-expansion@npm:2.1.0" + dependencies: + balanced-match: "npm:^1.0.0" + checksum: 10c0/439cedf3e23d7993b37919f1d6fdc653ec21a42437ec3e7460bea9ca8b17edf7a24a633273c31d61aa4335877cf29a443f1871814131c87997a1e6223e1f1502 + languageName: node + linkType: hard + "brace-expansion@npm:^5.0.5": - version: 5.0.9 - resolution: "brace-expansion@npm:5.0.9" + version: 5.0.6 + resolution: "brace-expansion@npm:5.0.6" dependencies: balanced-match: "npm:^4.0.2" - checksum: 10c0/3dea38884a1c3c8b1c9c44a7402a0c76fca460f70cffb3127242b0b4cbf4472019e022ade021eec44838ff19f1dac2625dfd11dd459d7e1e055b0698a8d52fec + checksum: 10c0/8c919869b90f61d533b341d3340be5ee4413232ea89b8246cbc2f38eb014f1d8182785c98a006eaf6111d02dc9eeffefdc240d5ac158625b2ed084dccd4bbf9b languageName: node linkType: hard @@ -11696,18 +11742,22 @@ __metadata: languageName: node linkType: hard -"bullmq@npm:^5.34.10": - version: 5.44.3 - resolution: "bullmq@npm:5.44.3" +"bullmq@npm:^5.81.3": + version: 5.81.3 + resolution: "bullmq@npm:5.81.3" dependencies: - cron-parser: "npm:^4.9.0" - ioredis: "npm:^5.4.1" - msgpackr: "npm:^1.11.2" - node-abort-controller: "npm:^3.1.1" - semver: "npm:^7.5.4" - tslib: "npm:^2.0.0" - uuid: "npm:^9.0.0" - checksum: 10c0/2785929ef59645980e3981d6f5e3f7ef25ef0e8488281a7aa48e89c25a89ff5dde0431051f48b2edadd9675c5b780c02577724c33d5d31762009820b8d404cdb + cron-parser: "npm:4.9.0" + ioredis: "npm:5.11.1" + msgpackr: "npm:2.0.5" + node-abort-controller: "npm:3.1.1" + semver: "npm:7.8.5" + tslib: "npm:2.8.1" + peerDependencies: + redis: ">=5.0.0" + peerDependenciesMeta: + redis: + optional: true + checksum: 10c0/e28abf1f37191966d0204e8717040f99807749579ba34e6cd8ac4698f3daa0327be094e52fa52c6aab991f4cd4cdd338b6c950134ec083be166c19f82d96cde1 languageName: node linkType: hard @@ -12011,10 +12061,10 @@ __metadata: languageName: node linkType: hard -"cluster-key-slot@npm:^1.1.0": - version: 1.1.2 - resolution: "cluster-key-slot@npm:1.1.2" - checksum: 10c0/d7d39ca28a8786e9e801eeb8c770e3c3236a566625d7299a47bb71113fb2298ce1039596acb82590e598c52dbc9b1f088c8f587803e697cb58e1867a95ff94d3 +"cluster-key-slot@npm:1.1.1": + version: 1.1.1 + resolution: "cluster-key-slot@npm:1.1.1" + checksum: 10c0/079b1ae86b20e2d53308a877b08de5e830722a45c07810569d0dab4955bed569da33ac9f79998289d014adf02cca7223a0647cb0ee6548a12ab3c4f9beac1377 languageName: node linkType: hard @@ -12465,13 +12515,6 @@ __metadata: languageName: node linkType: hard -"content-type@npm:^2.0.0": - version: 2.0.0 - resolution: "content-type@npm:2.0.0" - checksum: 10c0/491539fff707d7594b0ca4fabcc084bef2a31ffa754ff0a4f80c4377e3963cff0394317f9271c24087596c97fa675bc123d61fa34ffe65b4904e7d3d3098de72 - languageName: node - linkType: hard - "convert-source-map@npm:^2.0.0": version: 2.0.0 resolution: "convert-source-map@npm:2.0.0" @@ -12542,7 +12585,7 @@ __metadata: languageName: node linkType: hard -"cron-parser@npm:^4.9.0": +"cron-parser@npm:4.9.0": version: 4.9.0 resolution: "cron-parser@npm:4.9.0" dependencies: @@ -13160,6 +13203,18 @@ __metadata: languageName: node linkType: hard +"debug@npm:4.4.3, debug@npm:^4.4.3, debug@npm:~4.4.1": + version: 4.4.3 + resolution: "debug@npm:4.4.3" + dependencies: + ms: "npm:^2.1.3" + peerDependenciesMeta: + supports-color: + optional: true + checksum: 10c0/d79136ec6c83ecbefd0f6a5593da6a9c91ec4d7ddc4b54c883d6e71ec9accb5f67a1a5e96d00a328196b5b5c86d365e98d8a3a70856aaf16b4e7b1985e67f5a6 + languageName: node + linkType: hard + "debug@npm:^3.2.7": version: 3.2.7 resolution: "debug@npm:3.2.7" @@ -13181,18 +13236,6 @@ __metadata: languageName: node linkType: hard -"debug@npm:^4.4.3, debug@npm:~4.4.1": - version: 4.4.3 - resolution: "debug@npm:4.4.3" - dependencies: - ms: "npm:^2.1.3" - peerDependenciesMeta: - supports-color: - optional: true - checksum: 10c0/d79136ec6c83ecbefd0f6a5593da6a9c91ec4d7ddc4b54c883d6e71ec9accb5f67a1a5e96d00a328196b5b5c86d365e98d8a3a70856aaf16b4e7b1985e67f5a6 - languageName: node - linkType: hard - "debug@npm:~4.3.2": version: 4.3.7 resolution: "debug@npm:4.3.7" @@ -13299,7 +13342,7 @@ __metadata: languageName: node linkType: hard -"denque@npm:^2.1.0": +"denque@npm:2.1.0": version: 2.1.0 resolution: "denque@npm:2.1.0" checksum: 10c0/f9ef81aa0af9c6c614a727cb3bd13c5d7db2af1abf9e6352045b86e85873e629690f6222f4edd49d10e4ccf8f078bbeec0794fafaf61b659c0589d0c511ec363 @@ -13451,14 +13494,14 @@ __metadata: linkType: hard "dompurify@npm:^3.3.2, dompurify@npm:^3.3.3": - version: 3.4.13 - resolution: "dompurify@npm:3.4.13" + version: 3.4.11 + resolution: "dompurify@npm:3.4.11" dependencies: "@types/trusted-types": "npm:^2.0.7" dependenciesMeta: "@types/trusted-types": optional: true - checksum: 10c0/9c2a1a71e1a1d8b77953db7a39ffc935ef4ed4f10fe40770232128c2cbfd4c3dc677d3d7bae51eb24cc52d9ff3548612dc540ecb4343e89b26e809d2be2e0cfe + checksum: 10c0/31439481c7e8fc3805d40c376936fd66936620fb1b1a31a2ec097f6165412c37f2d868e082c9ceba62bb37661c1ea132a5db4d5213434317e30df68d4aca9cc9 languageName: node linkType: hard @@ -13572,6 +13615,17 @@ __metadata: languageName: node linkType: hard +"ejs@npm:^3.1.10": + version: 3.1.10 + resolution: "ejs@npm:3.1.10" + dependencies: + jake: "npm:^10.8.5" + bin: + ejs: bin/cli.js + checksum: 10c0/52eade9e68416ed04f7f92c492183340582a36482836b11eab97b159fcdcfdedc62233a1bf0bf5e5e1851c501f2dca0e2e9afd111db2599e4e7f53ee29429ae1 + languageName: node + linkType: hard + "electron-to-chromium@npm:^1.5.73": version: 1.5.123 resolution: "electron-to-chromium@npm:1.5.123" @@ -14578,6 +14632,42 @@ __metadata: languageName: node linkType: hard +"express@npm:^4.21.1 || ^5.0.0, express@npm:^5.2.1": + version: 5.2.1 + resolution: "express@npm:5.2.1" + dependencies: + accepts: "npm:^2.0.0" + body-parser: "npm:^2.2.1" + content-disposition: "npm:^1.0.0" + content-type: "npm:^1.0.5" + cookie: "npm:^0.7.1" + cookie-signature: "npm:^1.2.1" + debug: "npm:^4.4.0" + depd: "npm:^2.0.0" + encodeurl: "npm:^2.0.0" + escape-html: "npm:^1.0.3" + etag: "npm:^1.8.1" + finalhandler: "npm:^2.1.0" + fresh: "npm:^2.0.0" + http-errors: "npm:^2.0.0" + merge-descriptors: "npm:^2.0.0" + mime-types: "npm:^3.0.0" + on-finished: "npm:^2.4.1" + once: "npm:^1.4.0" + parseurl: "npm:^1.3.3" + proxy-addr: "npm:^2.0.7" + qs: "npm:^6.14.0" + range-parser: "npm:^1.2.1" + router: "npm:^2.2.0" + send: "npm:^1.1.0" + serve-static: "npm:^2.2.0" + statuses: "npm:^2.0.1" + type-is: "npm:^2.0.1" + vary: "npm:^1.1.2" + checksum: 10c0/45e8c841ad188a41402ddcd1294901e861ee0819f632fb494f2ed344ef9c43315d294d443fb48d594e6586a3b779785120f43321417adaef8567316a55072949 + languageName: node + linkType: hard + "express@npm:^4.22.2": version: 4.22.2 resolution: "express@npm:4.22.2" @@ -14617,42 +14707,6 @@ __metadata: languageName: node linkType: hard -"express@npm:^5.2.1": - version: 5.2.1 - resolution: "express@npm:5.2.1" - dependencies: - accepts: "npm:^2.0.0" - body-parser: "npm:^2.2.1" - content-disposition: "npm:^1.0.0" - content-type: "npm:^1.0.5" - cookie: "npm:^0.7.1" - cookie-signature: "npm:^1.2.1" - debug: "npm:^4.4.0" - depd: "npm:^2.0.0" - encodeurl: "npm:^2.0.0" - escape-html: "npm:^1.0.3" - etag: "npm:^1.8.1" - finalhandler: "npm:^2.1.0" - fresh: "npm:^2.0.0" - http-errors: "npm:^2.0.0" - merge-descriptors: "npm:^2.0.0" - mime-types: "npm:^3.0.0" - on-finished: "npm:^2.4.1" - once: "npm:^1.4.0" - parseurl: "npm:^1.3.3" - proxy-addr: "npm:^2.0.7" - qs: "npm:^6.14.0" - range-parser: "npm:^1.2.1" - router: "npm:^2.2.0" - send: "npm:^1.1.0" - serve-static: "npm:^2.2.0" - statuses: "npm:^2.0.1" - type-is: "npm:^2.0.1" - vary: "npm:^1.1.2" - checksum: 10c0/45e8c841ad188a41402ddcd1294901e861ee0819f632fb494f2ed344ef9c43315d294d443fb48d594e6586a3b779785120f43321417adaef8567316a55072949 - languageName: node - linkType: hard - "extend@npm:^3.0.0, extend@npm:^3.0.2": version: 3.0.2 resolution: "extend@npm:3.0.2" @@ -14759,9 +14813,9 @@ __metadata: linkType: hard "fast-uri@npm:^3.1.2": - version: 3.1.5 - resolution: "fast-uri@npm:3.1.5" - checksum: 10c0/2bf60eb800dd610c65e17be436425dcb21c92aff3a87d442a8bccab0b7b071e88cf1a5d7d1ea946370b937e6fc0375c405c0296c10587e57de4f78be4646d1d0 + version: 3.1.2 + resolution: "fast-uri@npm:3.1.2" + checksum: 10c0/5b35641895959f3f7ab7a7b1b5542bded159346f25ec9f256817b206d50b64eda5828e90d605a2e2fc645c90519a7259c2bab2c942ee728c88b88e5be21b090d languageName: node linkType: hard @@ -14873,6 +14927,15 @@ __metadata: languageName: node linkType: hard +"filelist@npm:^1.0.4": + version: 1.0.6 + resolution: "filelist@npm:1.0.6" + dependencies: + minimatch: "npm:^5.0.1" + checksum: 10c0/6ee725bec3e1936d680a45f14439b224d9f7c71658c145addcf551dd82f03d608522eb6b191aa086b392bc3e52ed4ce0ed8d78e24b203e6c5e867560a05d1121 + languageName: node + linkType: hard + "fill-range@npm:^7.1.1": version: 7.1.1 resolution: "fill-range@npm:7.1.1" @@ -15748,9 +15811,9 @@ __metadata: linkType: hard "hono@npm:^4.11.4": - version: 4.13.0 - resolution: "hono@npm:4.13.0" - checksum: 10c0/d7cfb4d1063be19bea1982ea2a6b3cfa1840e8403a03203e86d17278e76fed11e7483974738345257543e183a935ea73ffc6d4a1e6ad7f0008451f55cdf8be7b + version: 4.12.25 + resolution: "hono@npm:4.12.25" + checksum: 10c0/9216d647fe2f39b17855b0e74913688b837e3fa9519d367c7beeec399265b36608a820928cc33ab926eee58fe2daf7e33296235b52e56dbfac0fbcd51a5e818e languageName: node linkType: hard @@ -15829,7 +15892,7 @@ __metadata: languageName: node linkType: hard -"http-errors@npm:^2.0.1, http-errors@npm:~2.0.0, http-errors@npm:~2.0.1": +"http-errors@npm:~2.0.0, http-errors@npm:~2.0.1": version: 2.0.1 resolution: "http-errors@npm:2.0.1" dependencies: @@ -15911,7 +15974,7 @@ __metadata: languageName: node linkType: hard -"iconv-lite@npm:^0.7.2, iconv-lite@npm:~0.7.0": +"iconv-lite@npm:^0.7.0, iconv-lite@npm:^0.7.2, iconv-lite@npm:~0.7.0": version: 0.7.2 resolution: "iconv-lite@npm:0.7.2" dependencies: @@ -16074,27 +16137,25 @@ __metadata: languageName: node linkType: hard -"ioredis@npm:^5.4.1, ioredis@npm:^5.4.2": - version: 5.6.0 - resolution: "ioredis@npm:5.6.0" +"ioredis@npm:5.11.1, ioredis@npm:^5.11.1": + version: 5.11.1 + resolution: "ioredis@npm:5.11.1" dependencies: - "@ioredis/commands": "npm:^1.1.1" - cluster-key-slot: "npm:^1.1.0" - debug: "npm:^4.3.4" - denque: "npm:^2.1.0" - lodash.defaults: "npm:^4.2.0" - lodash.isarguments: "npm:^3.1.0" - redis-errors: "npm:^1.2.0" - redis-parser: "npm:^3.0.0" - standard-as-callback: "npm:^2.1.0" - checksum: 10c0/a885e5146640fc448706871290ef424ffa39af561f7ee3cf1590085209a509f85e99082bdaaf3cd32fa66758aea3fc2055d1109648ddca96fac4944bf2092c30 + "@ioredis/commands": "npm:1.10.0" + cluster-key-slot: "npm:1.1.1" + debug: "npm:4.4.3" + denque: "npm:2.1.0" + redis-errors: "npm:1.2.0" + redis-parser: "npm:3.0.0" + standard-as-callback: "npm:2.1.0" + checksum: 10c0/a8b27043cf2c045dfc93f40a32ce24cf9f8b57799a37f4234c4b925c365ccf131629590f94a512f546fda2ba8ed034009c94c4933ecd44c50bc166636d929fd6 languageName: node linkType: hard "ip-address@npm:^10.1.1, ip-address@npm:^10.2.0": - version: 10.4.0 - resolution: "ip-address@npm:10.4.0" - checksum: 10c0/d7b0bd2624fd861afbae6e49036a9b56f9506eaff7ff38592b7b4492dd5c272b53f5356dc8cc019e318acf29a96c90a3d1af1df761960267d23efa96858270fa + version: 10.2.0 + resolution: "ip-address@npm:10.2.0" + checksum: 10c0/5a00aada6e922c9c69dfc800ed5d0fa3348675ebdeed0e1575f503f27ca385b5f534363c9af7ad1daf64c1f1409388cdd3cc2e9b9b0fe1c924a431378d55075a languageName: node linkType: hard @@ -16559,6 +16620,19 @@ __metadata: languageName: node linkType: hard +"jake@npm:^10.8.5": + version: 10.9.4 + resolution: "jake@npm:10.9.4" + dependencies: + async: "npm:^3.2.6" + filelist: "npm:^1.0.4" + picocolors: "npm:^1.1.1" + bin: + jake: bin/cli.js + checksum: 10c0/bb52f000340d4a32f1a3893b9abe56ef2b77c25da4dbf2c0c874a8159d082dddda50a5ad10e26060198bd645b928ba8dba3b362710f46a247e335321188c5a9c + languageName: node + linkType: hard + "jiti@npm:2.4.2": version: 2.4.2 resolution: "jiti@npm:2.4.2" @@ -16615,13 +16689,13 @@ __metadata: linkType: hard "js-yaml@npm:^4.1.1": - version: 4.3.1 - resolution: "js-yaml@npm:4.3.1" + version: 4.2.0 + resolution: "js-yaml@npm:4.2.0" dependencies: argparse: "npm:^2.0.1" bin: js-yaml: bin/js-yaml.js - checksum: 10c0/13c500ca322e0c3f8c81686e6ecda96d2ea37b45247a420c17c7db36932d6965cc27391abc2d1a104501600e7f0d947a5f8b7be6db619c4fefa87901b3512807 + checksum: 10c0/1916456c118746603b067d74bbcbb0445d9a1d5e474ad4ae775e7b20525bed902e01d9d97dd0c81fcd8d4f596162309d0eb057f4aa38f3e9647f14075e9dea45 languageName: node linkType: hard @@ -17130,11 +17204,11 @@ __metadata: linkType: hard "linkify-it@npm:^5.0.1": - version: 5.0.2 - resolution: "linkify-it@npm:5.0.2" + version: 5.0.1 + resolution: "linkify-it@npm:5.0.1" dependencies: uc.micro: "npm:^2.0.0" - checksum: 10c0/dd70b1735a13d41a2cff0a058ac3771166038f23f6aff004dd53873cf985c64b107902fe0b544a5b3d1ff6e63249cf9c648fb3ae9f285481db48f56887adb0d6 + checksum: 10c0/d06d04f1ed03be131740fc900a5e74ea1f49886b052213599e306d469d5ffe2303db76dd8f771de9f28e2b0b38852de22ec46ae597d245f8b66439b0ceb19b10 languageName: node linkType: hard @@ -17198,20 +17272,6 @@ __metadata: languageName: node linkType: hard -"lodash.defaults@npm:^4.2.0": - version: 4.2.0 - resolution: "lodash.defaults@npm:4.2.0" - checksum: 10c0/d5b77aeb702caa69b17be1358faece33a84497bcca814897383c58b28a2f8dfc381b1d9edbec239f8b425126a3bbe4916223da2a576bb0411c2cefd67df80707 - languageName: node - linkType: hard - -"lodash.isarguments@npm:^3.1.0": - version: 3.1.0 - resolution: "lodash.isarguments@npm:3.1.0" - checksum: 10c0/5e8f95ba10975900a3920fb039a3f89a5a79359a1b5565e4e5b4310ed6ebe64011e31d402e34f577eca983a1fc01ff86c926e3cbe602e1ddfc858fdd353e62d8 - languageName: node - linkType: hard - "lodash.isplainobject@npm:^4.0.6": version: 4.0.6 resolution: "lodash.isplainobject@npm:4.0.6" @@ -17226,7 +17286,7 @@ __metadata: languageName: node linkType: hard -"lodash@npm:^4.17.21": +"lodash@npm:^4.17.11, lodash@npm:^4.17.21": version: 4.18.1 resolution: "lodash@npm:4.18.1" checksum: 10c0/757228fc68805c59789e82185135cf85f05d0b2d3d54631d680ca79ec21944ec8314d4533639a14b8bcfbd97a517e78960933041a5af17ecb693ec6eecb99a27 @@ -17722,8 +17782,8 @@ __metadata: linkType: hard "mermaid@npm:^11.16.0": - version: 11.16.1 - resolution: "mermaid@npm:11.16.1" + version: 11.16.0 + resolution: "mermaid@npm:11.16.0" dependencies: "@braintree/sanitize-url": "npm:^7.1.2" "@iconify/utils": "npm:^3.0.2" @@ -17746,7 +17806,7 @@ __metadata: stylis: "npm:^4.3.6" ts-dedent: "npm:^2.2.0" uuid: "npm:^11.1.0 || ^12 || ^13 || ^14.0.0" - checksum: 10c0/a905627d5ad372914bf7c67cb9e491d50f248d3a8458e4ec920033f5ce7387bf419423909ed71d88f54a035ec42987746d64a46055bd2b2a4dc1bc4ceed38c44 + checksum: 10c0/a14f9bc9db7f1dea65b0d6c0b920236d12ad2812f2a1b11c8b39b3dfb3c22bfce39c77f6a69493203b85880d8008d345c539efe7ae6a31d3dad512833ccfb517 languageName: node linkType: hard @@ -18183,6 +18243,15 @@ __metadata: languageName: node linkType: hard +"minimatch@npm:^5.0.1": + version: 5.1.9 + resolution: "minimatch@npm:5.1.9" + dependencies: + brace-expansion: "npm:^2.0.1" + checksum: 10c0/4202718683815a7288b13e470160a4f9560cf392adef4f453927505817e01ef6b3476ecde13cfcaed17e7326dd3b69ad44eb2daeb19a217c5500f9277893f1d6 + languageName: node + linkType: hard + "minimatch@npm:^9.0.4, minimatch@npm:^9.0.5": version: 9.0.9 resolution: "minimatch@npm:9.0.9" @@ -18364,16 +18433,16 @@ __metadata: languageName: node linkType: hard -"msgpackr-extract@npm:^3.0.2": - version: 3.0.3 - resolution: "msgpackr-extract@npm:3.0.3" - dependencies: - "@msgpackr-extract/msgpackr-extract-darwin-arm64": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-darwin-x64": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-linux-arm": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-linux-arm64": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-linux-x64": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-win32-x64": "npm:3.0.3" +"msgpackr-extract@npm:^3.0.4": + version: 3.0.4 + resolution: "msgpackr-extract@npm:3.0.4" + dependencies: + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-darwin-x64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-linux-arm": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-linux-arm64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-linux-x64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-win32-x64": "npm:3.0.4" node-gyp: "npm:latest" node-gyp-build-optional-packages: "npm:5.2.2" dependenciesMeta: @@ -18391,19 +18460,19 @@ __metadata: optional: true bin: download-msgpackr-prebuilds: bin/download-prebuilds.js - checksum: 10c0/e504fd8bf86a29d7527c83776530ee6dc92dcb0273bb3679fd4a85173efead7f0ee32fb82c8410a13c33ef32828c45f81118ffc0fbed5d6842e72299894623b4 + checksum: 10c0/582a9d17abbf3019e600e948736695056280ce401fd0235ee2474e95f9952208b9f6cce4d0e355b03b7d3c5630e6c3d11fe5fc27fdedb2311cce48de464338d8 languageName: node linkType: hard -"msgpackr@npm:^1.11.2": - version: 1.11.2 - resolution: "msgpackr@npm:1.11.2" +"msgpackr@npm:2.0.5": + version: 2.0.5 + resolution: "msgpackr@npm:2.0.5" dependencies: - msgpackr-extract: "npm:^3.0.2" + msgpackr-extract: "npm:^3.0.4" dependenciesMeta: msgpackr-extract: optional: true - checksum: 10c0/7d2e81ca82c397b2352d470d6bc8f4a967fe4fe14f8fc1fc9906b23009fdfb543999b1ad29c700b8861581e0b6bf903d6f0fefb69a09375cbca6d4d802e6c906 + checksum: 10c0/7ac9820cecd44d24d2ef07994405a277509f004d4fb0a77d04e10e9071d0599e86ad53a9daf673e801518230774bdaca72b1fdcff2b516584ab58bd19362e6ad languageName: node linkType: hard @@ -18441,12 +18510,21 @@ __metadata: languageName: node linkType: hard +"nanoid@npm:^3.3.12": + version: 3.3.12 + resolution: "nanoid@npm:3.3.12" + bin: + nanoid: bin/nanoid.cjs + checksum: 10c0/ba142b7b39e11e80c16dd74b0365d407880c87c1cf7e1480956981ae940ee36060fa5b6f092cd1e315184dd19244c657bd017d03327bd3c62247d691c5e8edfb + languageName: node + linkType: hard + "nanoid@npm:^3.3.16": - version: 3.3.18 - resolution: "nanoid@npm:3.3.18" + version: 3.3.16 + resolution: "nanoid@npm:3.3.16" bin: nanoid: bin/nanoid.cjs - checksum: 10c0/b994b4e396730f8be2520923284e2040d61eaee55cc6d4935ef6d38d34bafdc46133eda4d3faea5073bda545aa6079d82b886caeac5c731cf9ac18bcc1301425 + checksum: 10c0/bbf2dcffe22d2b62d16de2711752070b539c0f644c7916f823ad6521986b2078cbe524f2d6240f58c46e9141ea0c7b87a029e100c0f7f175228cacaf30e41bba languageName: node linkType: hard @@ -18606,7 +18684,7 @@ __metadata: languageName: node linkType: hard -"node-abort-controller@npm:^3.0.1, node-abort-controller@npm:^3.1.1": +"node-abort-controller@npm:3.1.1, node-abort-controller@npm:^3.0.1": version: 3.1.1 resolution: "node-abort-controller@npm:3.1.1" checksum: 10c0/f7ad0e7a8e33809d4f3a0d1d65036a711c39e9d23e0319d80ebe076b9a3b4432b4d6b86a7fab65521de3f6872ffed36fc35d1327487c48eb88c517803403eda3 @@ -19679,7 +19757,7 @@ __metadata: languageName: node linkType: hard -"postcss@npm:^8.4.47, postcss@npm:^8.5.12, postcss@npm:^8.5.15": +"postcss@npm:^8.4.47, postcss@npm:^8.5.12": version: 8.5.25 resolution: "postcss@npm:8.5.25" dependencies: @@ -19690,6 +19768,17 @@ __metadata: languageName: node linkType: hard +"postcss@npm:^8.5.15": + version: 8.5.15 + resolution: "postcss@npm:8.5.15" + dependencies: + nanoid: "npm:^3.3.12" + picocolors: "npm:^1.1.1" + source-map-js: "npm:^1.2.1" + checksum: 10c0/7f2e63ae22fbe43aace1bf652bd99da4e90737c64194d49e51ddc9cd0f9e51ff2861a7d734379b494deffa03a880a5c65eec70bc29ee9ebaa7136dde3eee8f31 + languageName: node + linkType: hard + "postgres-array@npm:~2.0.0": version: 2.0.0 resolution: "postgres-array@npm:2.0.0" @@ -19967,8 +20056,8 @@ __metadata: linkType: hard "protobufjs@npm:^7.3.0, protobufjs@npm:^7.4.0, protobufjs@npm:^7.5.3, protobufjs@npm:^7.5.4": - version: 7.6.5 - resolution: "protobufjs@npm:7.6.5" + version: 7.6.4 + resolution: "protobufjs@npm:7.6.4" dependencies: "@protobufjs/aspromise": "npm:^1.1.2" "@protobufjs/base64": "npm:^1.1.2" @@ -19981,7 +20070,7 @@ __metadata: "@protobufjs/utf8": "npm:^1.1.1" "@types/node": "npm:>=13.7.0" long: "npm:^5.3.2" - checksum: 10c0/863eaca9c6f45bfcfb8787c545f53e9c6696507e328e3981b4643c12d35df7f2826021c10e3fd30bef342c13a8e9d306547bac9c0849ee3bf50f770a12f01dc5 + checksum: 10c0/6403eaa9c5a72cc6450c11f38fefafdde243fd806e7ac606ac8d591bc3fdaec45ae764febf83181a2d9aac51aca624e0f46dec368ceea191f7e85e2d6ccaaf93 languageName: node linkType: hard @@ -20123,7 +20212,7 @@ __metadata: languageName: node linkType: hard -"raw-body@npm:^3.0.2": +"raw-body@npm:^3.0.1": version: 3.0.2 resolution: "raw-body@npm:3.0.2" dependencies: @@ -20592,14 +20681,23 @@ __metadata: languageName: node linkType: hard -"redis-errors@npm:^1.0.0, redis-errors@npm:^1.2.0": +"redis-errors@npm:1.2.0, redis-errors@npm:^1.0.0": version: 1.2.0 resolution: "redis-errors@npm:1.2.0" checksum: 10c0/5b316736e9f532d91a35bff631335137a4f974927bb2fb42bf8c2f18879173a211787db8ac4c3fde8f75ed6233eb0888e55d52510b5620e30d69d7d719c8b8a7 languageName: node linkType: hard -"redis-parser@npm:^3.0.0": +"redis-info@npm:^3.1.0": + version: 3.1.0 + resolution: "redis-info@npm:3.1.0" + dependencies: + lodash: "npm:^4.17.11" + checksum: 10c0/ec0f31d97893c5828cec7166486d74198c92160c60073b6f2fe805cdf575a10ddcccc7641737d44b8f451355f0ab5b6c7b0d79e8fc24742b75dd625f91ffee38 + languageName: node + linkType: hard + +"redis-parser@npm:3.0.0": version: 3.0.0 resolution: "redis-parser@npm:3.0.0" dependencies: @@ -21298,6 +21396,15 @@ __metadata: languageName: node linkType: hard +"semver@npm:7.8.5, semver@npm:^7.8.5": + version: 7.8.5 + resolution: "semver@npm:7.8.5" + bin: + semver: bin/semver.js + checksum: 10c0/b1f3127a5be8125a94f37188b361c212466c292c6910adce3ec106cff5dc211ccaedc4739c11bb70fda59d6fc1f040a9bca289f4e093451521a2372e5231fe0c + languageName: node + linkType: hard + "semver@npm:^6.3.1": version: 6.3.1 resolution: "semver@npm:6.3.1" @@ -21307,7 +21414,7 @@ __metadata: languageName: node linkType: hard -"semver@npm:^7.3.5, semver@npm:^7.5.3, semver@npm:^7.5.4, semver@npm:^7.6.0, semver@npm:^7.6.3": +"semver@npm:^7.3.5, semver@npm:^7.5.3, semver@npm:^7.6.0, semver@npm:^7.6.3": version: 7.7.1 resolution: "semver@npm:7.7.1" bin: @@ -21325,15 +21432,6 @@ __metadata: languageName: node linkType: hard -"semver@npm:^7.8.5": - version: 7.8.5 - resolution: "semver@npm:7.8.5" - bin: - semver: bin/semver.js - checksum: 10c0/b1f3127a5be8125a94f37188b361c212466c292c6910adce3ec106cff5dc211ccaedc4739c11bb70fda59d6fc1f040a9bca289f4e093451521a2372e5231fe0c - languageName: node - linkType: hard - "send@npm:^1.1.0, send@npm:^1.2.0": version: 1.2.0 resolution: "send@npm:1.2.0" @@ -21393,9 +21491,9 @@ __metadata: linkType: hard "seroval@npm:~1.5.0": - version: 1.5.6 - resolution: "seroval@npm:1.5.6" - checksum: 10c0/47e4fb25305bf05fdf300cac6b0d4aaaaf1e12f15c819195afcdf512d88901a3f1f7754ac5a2de740cc0bb04e912c653fbf0129fb3e4c81ce12809e6aa5815e3 + version: 1.5.0 + resolution: "seroval@npm:1.5.0" + checksum: 10c0/aff16b14a7145388555cefd4ebd41759024ee1c2c064080fd8d4fabea4b7c89d103155cd98f5109523b8878e577da73cc6cd8abf98965f2d1f0ba19dc38317ab languageName: node linkType: hard @@ -21615,9 +21713,9 @@ __metadata: linkType: hard "shell-quote@npm:^1.6.1, shell-quote@npm:^1.8.4": - version: 1.10.0 - resolution: "shell-quote@npm:1.10.0" - checksum: 10c0/46ee59bfd972ce6a45500c44ed130dff2d0a7d6fbac9841e59d548518cad8060a06393c9a5dcbc0cede294ad80b2a2cd8c904679e09265f53efc0a0879f30961 + version: 1.8.4 + resolution: "shell-quote@npm:1.8.4" + checksum: 10c0/86c93678bc394cb81f5ddcdc87df9c95d279ef9652775cd1cd1eed361404169a8d8cbaacaeed232ab09919e36ee1e5363863570390d78571f8c22b7f6312fb40 languageName: node linkType: hard @@ -21835,12 +21933,12 @@ __metadata: linkType: hard "socket.io-parser@npm:~4.2.4": - version: 4.2.7 - resolution: "socket.io-parser@npm:4.2.7" + version: 4.2.6 + resolution: "socket.io-parser@npm:4.2.6" dependencies: "@socket.io/component-emitter": "npm:~3.1.0" debug: "npm:~4.4.1" - checksum: 10c0/16a5579718b871114ca644d688987dd3cf9292010c949fb083633eefbc1d8fdaf424691d6511d746e5c10f5c88cbd8911cd0b1e807242159f40989be90842b4e + checksum: 10c0/ba0a0b541b0a8e9d02b45c04c4c93a02331be5ea3478073c65bb9ff87032f12469c9adb309728eb90c0a352618d645ab88999c167a11c783cac861d7fd35c9d1 languageName: node linkType: hard @@ -21987,7 +22085,7 @@ __metadata: languageName: node linkType: hard -"standard-as-callback@npm:^2.1.0": +"standard-as-callback@npm:2.1.0": version: 2.1.0 resolution: "standard-as-callback@npm:2.1.0" checksum: 10c0/012677236e3d3fdc5689d29e64ea8a599331c4babe86956bf92fc5e127d53f85411c5536ee0079c52c43beb0026b5ce7aa1d834dd35dd026e82a15d1bcaead1f @@ -22526,15 +22624,15 @@ __metadata: linkType: hard "tar@npm:^7.4.3": - version: 7.5.22 - resolution: "tar@npm:7.5.22" + version: 7.5.16 + resolution: "tar@npm:7.5.16" dependencies: "@isaacs/fs-minipass": "npm:^4.0.0" chownr: "npm:^3.0.0" minipass: "npm:^7.1.2" minizlib: "npm:^3.1.0" yallist: "npm:^5.0.0" - checksum: 10c0/1311f6be85a8157ac4c9147bae43e13923d2a1aae15e4aa1bd5239e4e03d2cf53cfe103dde7f35832fbb4c938b042856bc8e9a0afd29abd05e2d1608788c4fea + checksum: 10c0/4f37f3c4bd2ca2755fd736a5df1d573c1a868ec1b1e893346aeafa95ac510f9e2fd1469420bd866cc7904799e5bd4ac62b5d4f03fe27747d6e1e373b44505c5c languageName: node linkType: hard @@ -22918,7 +23016,7 @@ __metadata: languageName: node linkType: hard -"tslib@npm:^2.0.0, tslib@npm:^2.1.0, tslib@npm:^2.4.0, tslib@npm:^2.6.2, tslib@npm:^2.7.0, tslib@npm:^2.8.0, tslib@npm:^2.8.1": +"tslib@npm:2.8.1, tslib@npm:^2.0.0, tslib@npm:^2.1.0, tslib@npm:^2.4.0, tslib@npm:^2.6.2, tslib@npm:^2.7.0, tslib@npm:^2.8.0, tslib@npm:^2.8.1": version: 2.8.1 resolution: "tslib@npm:2.8.1" checksum: 10c0/9c4759110a19c53f992d9aae23aac5ced636e99887b51b9e61def52611732872ff7668757d4e4c61f19691e36f4da981cd9485e869b4a7408d689f6bf1f14e62 @@ -22998,17 +23096,6 @@ __metadata: languageName: node linkType: hard -"type-is@npm:^2.1.0": - version: 2.1.0 - resolution: "type-is@npm:2.1.0" - dependencies: - content-type: "npm:^2.0.0" - media-typer: "npm:^1.1.0" - mime-types: "npm:^3.0.0" - checksum: 10c0/a6018f8f509de48f2c7429305e3a920e73b374fa93127dd0877ae1c2df65a5d33907caac8afb0c37a9b9fc7c49f29e3f55d668963dc845d966930b667c07f50e - languageName: node - linkType: hard - "type-is@npm:~1.6.18": version: 1.6.18 resolution: "type-is@npm:1.6.18"