Make GPT model calls in batches of 2 requests (#29944)

GitOrigin-RevId: 263c297b2bf5fb40b702f14222c6c6fadef75e8a

libraries/access-token-encryptor/Jenkinsfile

@@ -101,6 +101,15 @@ pipeline {
     always {
       junit checksName: 'access-token-encryptor test results', testResults: 'libraries/access-token-encryptor/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       sh 'rm -rf libraries/access-token-encryptor/reports'
       sh 'make clean_jenkins -j10'

libraries/access-token-encryptor/buildscript.txt

@@ -5,4 +5,5 @@ access-token-encryptor
 --esmock-loader=False
 --is-library=True
 --node-version=22.18.0
+--pipeline-owner=🚉 Platform
 --public-repo=False

libraries/fetch-utils/Jenkinsfile

@@ -101,6 +101,15 @@ pipeline {
     always {
       junit checksName: 'fetch-utils test results', testResults: 'libraries/fetch-utils/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       sh 'rm -rf libraries/fetch-utils/reports'
       sh 'make clean_jenkins -j10'

libraries/fetch-utils/buildscript.txt

@@ -5,4 +5,5 @@ fetch-utils
 --esmock-loader=False
 --is-library=True
 --node-version=22.18.0
+--pipeline-owner=🚉 Platform
 --public-repo=False

libraries/logger/Jenkinsfile

@@ -101,6 +101,15 @@ pipeline {
     always {
       junit checksName: 'logger test results', testResults: 'libraries/logger/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       sh 'rm -rf libraries/logger/reports'
       sh 'make clean_jenkins -j10'

libraries/logger/buildscript.txt

@@ -5,4 +5,5 @@ logger
 --esmock-loader=False
 --is-library=True
 --node-version=22.18.0
+--pipeline-owner=🚉 Platform
 --public-repo=False

libraries/metrics/Jenkinsfile

@@ -101,6 +101,15 @@ pipeline {
     always {
       junit checksName: 'metrics test results', testResults: 'libraries/metrics/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       sh 'rm -rf libraries/metrics/reports'
       sh 'make clean_jenkins -j10'

libraries/metrics/buildscript.txt

@@ -5,4 +5,5 @@ metrics
 --esmock-loader=False
 --is-library=True
 --node-version=22.18.0
+--pipeline-owner=🚉 Platform
 --public-repo=False

libraries/mongo-utils/Jenkinsfile

@@ -91,6 +91,15 @@ pipeline {
     }
   }
   post {
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       sh 'rm -rf libraries/mongo-utils/reports'
       sh 'make clean_jenkins -j10'

libraries/mongo-utils/buildscript.txt

@@ -5,4 +5,5 @@ mongo-utils
 --esmock-loader=False
 --is-library=True
 --node-version=22.18.0
+--pipeline-owner=🚉 Platform
 --public-repo=False

libraries/o-error/Jenkinsfile

@@ -101,6 +101,15 @@ pipeline {
     always {
       junit checksName: 'o-error test results', testResults: 'libraries/o-error/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       sh 'rm -rf libraries/o-error/reports'
       sh 'make clean_jenkins -j10'

libraries/o-error/buildscript.txt

@@ -5,4 +5,5 @@ o-error
 --esmock-loader=False
 --is-library=True
 --node-version=22.18.0
+--pipeline-owner=🚉 Platform
 --public-repo=False

libraries/object-persistor/Jenkinsfile

@@ -101,6 +101,15 @@ pipeline {
     always {
       junit checksName: 'object-persistor test results', testResults: 'libraries/object-persistor/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       sh 'rm -rf libraries/object-persistor/reports'
       sh 'make clean_jenkins -j10'

libraries/object-persistor/buildscript.txt

@@ -5,4 +5,5 @@ object-persistor
 --esmock-loader=False
 --is-library=True
 --node-version=22.18.0
+--pipeline-owner=🚉 Platform
 --public-repo=False

libraries/overleaf-editor-core/Jenkinsfile

@@ -101,6 +101,15 @@ pipeline {
     always {
       junit checksName: 'overleaf-editor-core test results', testResults: 'libraries/overleaf-editor-core/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🥑 Core"'
+          }
+        }
+      }
+    }
     cleanup {
       sh 'rm -rf libraries/overleaf-editor-core/reports'
       sh 'make clean_jenkins -j10'

libraries/overleaf-editor-core/buildscript.txt

@@ -5,4 +5,5 @@ overleaf-editor-core
 --esmock-loader=False
 --is-library=True
 --node-version=22.18.0
+--pipeline-owner=🥑 Core
 --public-repo=False

libraries/promise-utils/Jenkinsfile

@@ -101,6 +101,15 @@ pipeline {
     always {
       junit checksName: 'promise-utils test results', testResults: 'libraries/promise-utils/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       sh 'rm -rf libraries/promise-utils/reports'
       sh 'make clean_jenkins -j10'

libraries/promise-utils/buildscript.txt

@@ -5,4 +5,5 @@ promise-utils
 --esmock-loader=False
 --is-library=True
 --node-version=22.18.0
+--pipeline-owner=🚉 Platform
 --public-repo=False

libraries/ranges-tracker/Jenkinsfile

@@ -101,6 +101,15 @@ pipeline {
     always {
       junit checksName: 'ranges-tracker test results', testResults: 'libraries/ranges-tracker/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🥑 Core"'
+          }
+        }
+      }
+    }
     cleanup {
       sh 'rm -rf libraries/ranges-tracker/reports'
       sh 'make clean_jenkins -j10'

libraries/ranges-tracker/buildscript.txt

@@ -5,4 +5,5 @@ ranges-tracker
 --esmock-loader=False
 --is-library=True
 --node-version=22.18.0
+--pipeline-owner=🥑 Core
 --public-repo=False

libraries/redis-wrapper/Jenkinsfile

@@ -101,6 +101,15 @@ pipeline {
     always {
       junit checksName: 'redis-wrapper test results', testResults: 'libraries/redis-wrapper/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       sh 'rm -rf libraries/redis-wrapper/reports'
       sh 'make clean_jenkins -j10'

libraries/redis-wrapper/buildscript.txt

@@ -5,4 +5,5 @@ redis-wrapper
 --esmock-loader=False
 --is-library=True
 --node-version=22.18.0
+--pipeline-owner=🚉 Platform
 --public-repo=False

libraries/settings/Jenkinsfile

@@ -91,6 +91,15 @@ pipeline {
     }
   }
   post {
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       sh 'rm -rf libraries/settings/reports'
       sh 'make clean_jenkins -j10'

libraries/settings/buildscript.txt

@@ -5,4 +5,5 @@ settings
 --esmock-loader=False
 --is-library=True
 --node-version=22.18.0
+--pipeline-owner=🚉 Platform
 --public-repo=False

libraries/stream-utils/Jenkinsfile

@@ -101,6 +101,15 @@ pipeline {
     always {
       junit checksName: 'stream-utils test results', testResults: 'libraries/stream-utils/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       sh 'rm -rf libraries/stream-utils/reports'
       sh 'make clean_jenkins -j10'

libraries/stream-utils/buildscript.txt

@@ -5,4 +5,5 @@ stream-utils
 --esmock-loader=False
 --is-library=True
 --node-version=22.18.0
+--pipeline-owner=🚉 Platform
 --public-repo=False

server-ce/test/Jenkinsfile

@@ -357,6 +357,15 @@ pipeline {
     always {
       junit checksName: 'Server-Pro-E2E-Tests results', testResults: 'server-ce/test/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="B2B"'
+          }
+        }
+      }
+    }
     // Ensure tear down of test containers, remove CE docker images, then run general Jenkins VM cleanup.
     cleanup {
       dir('server-ce/test') {

services/chat/Jenkinsfile

@@ -142,6 +142,15 @@ pipeline {
     always {
       junit checksName: 'chat test results', testResults: 'services/chat/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🥑 Core"'
+          }
+        }
+      }
+    }
     cleanup {
       dir('services/chat') {
         sh 'make clean'

services/chat/buildscript.txt

@@ -4,4 +4,5 @@ chat
 --env-pass-through=
 --esmock-loader=False
 --node-version=22.18.0
+--pipeline-owner=🥑 Core
 --public-repo=False

services/clsi/Jenkinsfile

@@ -148,6 +148,15 @@ pipeline {
     always {
       junit checksName: 'clsi test results', testResults: 'services/clsi/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       dir('services/clsi') {
         sh 'make clean'

services/clsi/buildscript.txt

@@ -5,5 +5,6 @@ clsi
 --env-pass-through=
 --esmock-loader=False
 --node-version=22.18.0
+--pipeline-owner=🚉 Platform
 --public-repo=True
 --use-large-ci-runner=True

services/contacts/Jenkinsfile

@@ -142,6 +142,15 @@ pipeline {
     always {
       junit checksName: 'contacts test results', testResults: 'services/contacts/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="B2C"'
+          }
+        }
+      }
+    }
     cleanup {
       dir('services/contacts') {
         sh 'make clean'

services/contacts/buildscript.txt

@@ -4,4 +4,5 @@ contacts
 --env-pass-through=
 --esmock-loader=True
 --node-version=22.18.0
+--pipeline-owner=B2C
 --public-repo=False

services/docstore/Jenkinsfile

@@ -142,6 +142,15 @@ pipeline {
     always {
       junit checksName: 'docstore test results', testResults: 'services/docstore/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       dir('services/docstore') {
         sh 'make clean'

services/docstore/buildscript.txt

@@ -4,4 +4,5 @@ docstore
 --env-pass-through=
 --esmock-loader=False
 --node-version=22.18.0
+--pipeline-owner=🚉 Platform
 --public-repo=True

services/document-updater/Jenkinsfile

@@ -142,6 +142,15 @@ pipeline {
     always {
       junit checksName: 'document-updater test results', testResults: 'services/document-updater/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🥑 Core"'
+          }
+        }
+      }
+    }
     cleanup {
       dir('services/document-updater') {
         sh 'make clean'

services/document-updater/buildscript.txt

@@ -4,4 +4,5 @@ document-updater
 --env-pass-through=
 --esmock-loader=False
 --node-version=22.18.0
+--pipeline-owner=🥑 Core
 --public-repo=True

services/filestore/Jenkinsfile

@@ -169,6 +169,15 @@ pipeline {
     always {
       junit checksName: 'filestore test results', testResults: 'services/filestore/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       dir('services/filestore') {
         sh 'make clean'

services/filestore/buildscript.txt

@@ -5,6 +5,7 @@ filestore
 --env-pass-through=
 --esmock-loader=False
 --node-version=22.18.0
+--pipeline-owner=🚉 Platform
 --public-repo=True
 --test-acceptance-shards=SHARD_01_,SHARD_02_,SHARD_03_
 --use-large-ci-runner=True

services/history-v1/Jenkinsfile

@@ -142,6 +142,15 @@ pipeline {
     always {
       junit checksName: 'history-v1 test results', testResults: 'services/history-v1/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🥑 Core"'
+          }
+        }
+      }
+    }
     cleanup {
       dir('services/history-v1') {
         sh 'make clean'

services/history-v1/buildscript.txt

@@ -4,5 +4,6 @@ history-v1
 --env-pass-through=
 --esmock-loader=False
 --node-version=22.18.0
+--pipeline-owner=🥑 Core
 --public-repo=False
 --tsconfig-extra-includes=backup-deletion-app.mjs,backup-verifier-app.mjs,backup-worker-app.mjs,api/**/*,migrations/**/*,storage/**/*

services/history-v1/storage/scripts/backup.mjs

@@ -370,6 +370,25 @@ const optionDefinitions = [
     description:
       'Compare backup with original chunks. With --start-date and --end-date compares all projects in range.',
   },
+  {
+    name: 'fast',
+    type: Boolean,
+    description:
+      'Performs a fast comparison of blobs by only checking for presence and size. Only works with --compare.',
+  },
+  {
+    name: 'input',
+    type: String,
+    description:
+      'Input file containing project IDs (one per line) for batch comparison. Only works with --compare.',
+  },
+  {
+    name: 'verbose',
+    alias: 'v',
+    type: Boolean,
+    description:
+      'Enable verbose output during batch comparison. Only works with --compare and --input.',
+  },
 ]
 
 function handleOptions() {
@@ -388,7 +407,8 @@ function handleOptions() {
     !options.pending &&
     !options.init &&
     !(options.fix >= 0) &&
-    !(options.compare && options['start-date'] && options['end-date'])
+    !(options.compare && options['start-date'] && options['end-date']) &&
+    !(options.compare && options.input)
 
   if (projectIdRequired && !options.projectId) {
     console.error('Error: projectId is required')
@@ -428,14 +448,42 @@ function handleOptions() {
   if (
     options.compare &&
     !options.projectId &&
-    !(options['start-date'] && options['end-date'])
+    !(options['start-date'] && options['end-date']) &&
+    !options.input
   ) {
     console.error(
-      'Error: --compare requires either projectId or both --start-date and --end-date'
+      'Error: --compare requires either projectId, --input file, or both --start-date and --end-date'
     )
     process.exit(1)
   }
 
+  if (options.fast && !options.compare) {
+    console.error('Error: --fast can only be used with --compare')
+    process.exit(1)
+  }
+
+  if (options.input && !options.compare) {
+    console.error('Error: --input can only be used with --compare')
+    process.exit(1)
+  }
+
+  if (options.input && options.projectId) {
+    console.error('Error: --input cannot be specified with projectId')
+    process.exit(1)
+  }
+
+  if (options.input && (options['start-date'] || options['end-date'])) {
+    console.error(
+      'Error: --input cannot be specified with --start-date or --end-date'
+    )
+    process.exit(1)
+  }
+
+  if (options.verbose && !options.input) {
+    console.error('Error: --verbose can only be used with --input')
+    process.exit(1)
+  }
+
   DRY_RUN = options['dry-run'] || false
   RETRY_LIMIT = options.retries || 3
   CONCURRENCY = options.concurrency || 1
@@ -848,8 +896,60 @@ class BlobComparator {
   }
 }
 
-async function compareBackups(projectId, options) {
-  console.log(`Comparing backups for project ${projectId}`)
+const SHA1_HEX_REGEX = /^[a-f0-9]{40}$/
+
+async function getBlobListing(historyId) {
+  const backupPersistorForProject = await backupPersistor.forProject(
+    projectBlobsBucket,
+    makeProjectKey(historyId, '')
+  )
+
+  // get the blob listing
+  const projectBlobsPath = projectKey.format(historyId)
+
+  const { contents: blobList } = await backupPersistorForProject.listDirectory(
+    projectBlobsBucket,
+    projectBlobsPath
+  )
+
+  if (blobList.length === 0) {
+    return new Map()
+  }
+
+  const remoteBlobs = new Map()
+
+  for (const blobRecord of blobList) {
+    if (!blobRecord.Key) {
+      logger.debug({ blobRecord }, 'no key')
+      continue
+    }
+    const parts = blobRecord.Key.split('/')
+    const hash = parts[3] + parts[4]
+
+    if (!SHA1_HEX_REGEX.test(hash)) {
+      console.warn(`Invalid SHA1 hash for project ${historyId}: ${hash}`)
+      continue
+    }
+    remoteBlobs.set(hash, blobRecord)
+  }
+  return remoteBlobs
+}
+
+/**
+ * @typedef {Object} ComparisonError
+ * @property {string} type - Error type code (e.g., 'chunk-not-found', 'blob-hash-mismatch')
+ * @property {string} [chunkId]
+ * @property {string} historyId
+ * @property {string} [blobHash]
+ * @property {string|Error} error
+ */
+
+/**
+ * @typedef {Error & {historyId: string, errors: ComparisonError[], counters: Object}} ComparisonFailureError
+ */
+
+async function compareBackups(projectId, options, log = console.log) {
+  log(`Comparing backups for project ${projectId}`)
   const { historyId } = await getBackupStatus(projectId)
   const chunks = await getProjectChunks(historyId)
   const blobStore = new BlobStore(historyId)
@@ -864,10 +964,17 @@ async function compareBackups(projectId, options) {
   let totalBlobMatches = 0
   let totalBlobMismatches = 0
   let totalBlobsNotFound = 0
+  /** @type {ComparisonError[]} */
   const errors = []
   const blobComparator = new BlobComparator(backupPersistorForProject)
 
+  const blobsFromListing = await getBlobListing(historyId)
+
   for (const chunk of chunks) {
+    if (gracefulShutdownInitiated) {
+      throw new Error('interrupted')
+    }
+
     try {
       // Compare chunk content
       const originalChunk = await historyStore.loadRaw(historyId, chunk.id)
@@ -882,33 +989,48 @@ async function compareBackups(projectId, options) {
         const backupEndVersion = chunk.startVersion + backupChunk.changes.length
 
         if (originalStr === backupStr) {
-          console.log(
+          log(
             `✓ Chunk ${chunk.id} (v${chunk.startVersion}-v${chunk.endVersion}) matches`
           )
           totalChunkMatches++
         } else if (originalStr === JSON.stringify(JSON.parse(backupStr))) {
-          console.log(
+          log(
             `✓ Chunk ${chunk.id} (v${chunk.startVersion}-v${chunk.endVersion}) matches (after normalisation)`
           )
           totalChunkMatches++
         } else if (backupEndVersion < chunk.endVersion) {
-          console.log(
+          log(
             `✗ Chunk ${chunk.id} is ahead of backup (v${chunk.startVersion}-v${chunk.endVersion} vs v${backupStartVersion}-v${backupEndVersion})`
           )
           totalChunkMismatches++
-          errors.push({ chunkId: chunk.id, error: 'Chunk ahead of backup' })
+          errors.push({
+            type: 'chunk-ahead',
+            chunkId: chunk.id,
+            historyId,
+            error: 'Chunk ahead of backup',
+          })
         } else {
-          console.log(
+          log(
             `✗ Chunk ${chunk.id} (v${chunk.startVersion}-v${chunk.endVersion}) MISMATCH`
           )
           totalChunkMismatches++
-          errors.push({ chunkId: chunk.id, error: 'Chunk mismatch' })
+          errors.push({
+            type: 'chunk-mismatch',
+            chunkId: chunk.id,
+            historyId,
+            error: 'Chunk mismatch',
+          })
         }
       } catch (err) {
         if (err instanceof NotFoundError) {
-          console.log(`✗ Chunk ${chunk.id} not found in backup`, err.cause)
+          log(`✗ Chunk ${chunk.id} not found in backup`, err.cause)
           totalChunksNotFound++
-          errors.push({ chunkId: chunk.id, error: `Chunk not found` })
+          errors.push({
+            type: 'chunk-not-found',
+            chunkId: chunk.id,
+            historyId,
+            error: `Chunk not found`,
+          })
         } else {
           throw err
         }
@@ -921,41 +1043,103 @@ async function compareBackups(projectId, options) {
       history.findBlobHashes(blobHashes)
       const blobs = await blobStore.getBlobs(Array.from(blobHashes))
       for (const blob of blobs) {
+        if (gracefulShutdownInitiated) {
+          throw new Error('interrupted')
+        }
+
         if (GLOBAL_BLOBS.has(blob.hash)) {
           const globalBlob = GLOBAL_BLOBS.get(blob.hash)
-          console.log(
+          log(
             `  ✓ Blob ${blob.hash} is a global blob`,
             globalBlob?.demoted ? '(demoted)' : ''
           )
           continue
         }
         try {
-          const { matches, computedHash, fromCache } =
-            await blobComparator.compareBlob(historyId, blob)
-
-          if (matches) {
-            console.log(
-              `  ✓ Blob ${blob.hash} hash matches (${blob.byteLength} bytes)` +
-                (fromCache ? ' (from cache)' : '')
-            )
-            totalBlobMatches++
+          const blobListEntry = blobsFromListing.get(blob.hash)
+          if (options.fast) {
+            if (blobListEntry) {
+              if (blob.byteLength === blobListEntry.Size) {
+                // Size matches exactly
+                log(
+                  `  ✓ Blob ${blob.hash} exists on remote with expected size (${blob.byteLength} bytes)`
+                )
+                totalBlobMatches++
+                continue
+              } else if (blob.stringLength > 0 && blobListEntry.Size > 0) {
+                // Text file present with compressed size, assume valid as we are in --fast comparison mode
+                const compressionRatio = (
+                  blobListEntry.Size / blob.byteLength
+                ).toFixed(2)
+                log(
+                  `  ✓ Blob ${blob.hash} consistent with compressed data on remote (${blob.byteLength} bytes => ${blobListEntry.Size} bytes, ratio=${compressionRatio})`
+                )
+                totalBlobMatches++
+                continue
+              } else {
+                log(
+                  `  ✗ Blob ${blob.hash} size mismatch (original: ${blob.byteLength} bytes, stringLength: ${blob.stringLength}, backup: ${blobListEntry.Size} bytes)`
+                )
+                totalBlobMismatches++
+                errors.push({
+                  type: 'blob-size-mismatch',
+                  chunkId: chunk.id,
+                  historyId,
+                  blobHash: blob.hash,
+                  error: `Blob ${blob.hash} size mismatch`,
+                })
+                continue
+              }
+            } else {
+              log(
+                `  ✗ Blob ${blob.hash} not found on remote listing (${blob.byteLength} bytes, ${blob.stringLength} string length)`
+              )
+              totalBlobMismatches++
+              errors.push({
+                type: 'blob-not-found',
+                chunkId: chunk.id,
+                historyId,
+                blobHash: blob.hash,
+                error: `Blob ${blob.hash} not found`,
+              })
+              continue
+            }
           } else {
-            console.log(
-              `  ✗ Blob ${blob.hash} hash mismatch (original: ${blob.hash}, backup: ${computedHash}) (${blob.byteLength} bytes, ${blob.stringLength} string length)` +
-                (fromCache ? ' (from cache)' : '')
-            )
-            totalBlobMismatches++
-            errors.push({
-              chunkId: chunk.id,
-              error: `Blob ${blob.hash} hash mismatch`,
-            })
+            const { matches, computedHash, fromCache } =
+              await blobComparator.compareBlob(historyId, blob)
+
+            if (matches) {
+              log(
+                `  ✓ Blob ${blob.hash} hash matches (${blob.byteLength} bytes)` +
+                  (fromCache ? ' (from cache)' : '')
+              )
+              totalBlobMatches++
+              continue
+            } else {
+              log(
+                `  ✗ Blob ${blob.hash} hash mismatch (original: ${blob.hash}, backup: ${computedHash}) (${blob.byteLength} bytes, ${blob.stringLength} string length)` +
+                  (fromCache ? ' (from cache)' : '')
+              )
+              totalBlobMismatches++
+              errors.push({
+                type: 'blob-hash-mismatch',
+                chunkId: chunk.id,
+                historyId,
+                blobHash: blob.hash,
+                error: `Blob ${blob.hash} hash mismatch`,
+              })
+              continue
+            }
           }
         } catch (err) {
           if (err instanceof NotFoundError) {
-            console.log(`  ✗ Blob ${blob.hash} not found in backup`, err.cause)
+            log(`  ✗ Blob ${blob.hash} not found in backup`, err.cause)
             totalBlobsNotFound++
             errors.push({
+              type: 'blob-not-found',
               chunkId: chunk.id,
+              historyId,
+              blobHash: blob.hash,
               error: `Blob ${blob.hash} not found`,
             })
           } else {
@@ -964,31 +1148,197 @@ async function compareBackups(projectId, options) {
         }
       }
     } catch (err) {
-      console.error(`Error comparing chunk ${chunk.id}:`, err)
-      errors.push({ chunkId: chunk.id, error: err })
+      log(`Error comparing chunk ${chunk.id}:`, err)
+      errors.push({
+        type: 'error',
+        chunkId: chunk.id,
+        historyId,
+        error: err instanceof Error ? err : String(err),
+      })
     }
   }
 
   // Print summary
-  console.log('\nComparison Summary:')
-  console.log('==================')
-  console.log(`Total chunks: ${chunks.length}`)
-  console.log(`Chunk matches: ${totalChunkMatches}`)
-  console.log(`Chunk mismatches: ${totalChunkMismatches}`)
-  console.log(`Chunk not found: ${totalChunksNotFound}`)
-  console.log(`Blob matches: ${totalBlobMatches}`)
-  console.log(`Blob mismatches: ${totalBlobMismatches}`)
-  console.log(`Blob not found: ${totalBlobsNotFound}`)
-  console.log(`Errors: ${errors.length}`)
+  log('\nComparison Summary:')
+  log('==================')
+  log(`Total chunks: ${chunks.length}`)
+  log(`Chunk matches: ${totalChunkMatches}`)
+  log(`Chunk mismatches: ${totalChunkMismatches}`)
+  log(`Chunk not found: ${totalChunksNotFound}`)
+  log(`Blob matches: ${totalBlobMatches}`)
+  log(`Blob mismatches: ${totalBlobMismatches}`)
+  log(`Blob not found: ${totalBlobsNotFound}`)
+  log(`Errors: ${errors.length}`)
 
   if (errors.length > 0) {
-    console.log('\nErrors:')
+    log('\nErrors:')
     errors.forEach(({ chunkId, error }) => {
-      console.log(`  Chunk ${chunkId}: ${error}`)
+      log(`  Chunk ${chunkId}: ${error}`)
     })
-    throw new Error('Backup comparison FAILED')
+    const err = /** @type {ComparisonFailureError} */ (
+      new Error('Backup comparison FAILED')
+    )
+    err.historyId = historyId
+    err.errors = errors
+    err.counters = {
+      totalChunks: chunks.length,
+      chunkMatches: totalChunkMatches,
+      chunkMismatches: totalChunkMismatches,
+      chunksNotFound: totalChunksNotFound,
+      blobMatches: totalBlobMatches,
+      blobMismatches: totalBlobMismatches,
+      blobsNotFound: totalBlobsNotFound,
+    }
+    throw err
   } else {
-    console.log('Backup comparison successful')
+    log('Backup comparison successful')
+  }
+}
+
+/**
+ * Compare a single project and emit structured output
+ * @param {string} projectId - The project ID to compare
+ * @param {Object} options - Comparison options
+ * @param {number} projectNumber - Current project number for progress reporting
+ * @param {number} totalCount - Total number of projects
+ * @returns {Promise<boolean>} - Returns true if comparison had errors
+ */
+async function compareProjectAndEmitResult(
+  projectId,
+  options,
+  projectNumber,
+  totalCount
+) {
+  if (gracefulShutdownInitiated) {
+    return false
+  }
+
+  console.error(
+    `Processing project ${projectNumber}/${totalCount}: ${projectId}`
+  )
+
+  // Custom logger: silent by default, buffered if verbose
+  const logBuffer = []
+  const customLog = options.verbose
+    ? (...args) => logBuffer.push(args.join(' '))
+    : () => {}
+
+  try {
+    await compareBackups(projectId, options, customLog)
+    console.log(`OK: ${projectId}`)
+
+    // Output buffered logs after success
+    if (options.verbose && logBuffer.length > 0) {
+      console.error(`\n--- Verbose output for ${projectId} ---`)
+      logBuffer.forEach(line => console.error(line))
+      console.error(`--- End of output for ${projectId} ---\n`)
+    }
+
+    return false
+  } catch (err) {
+    console.log(`FAIL: ${projectId}`)
+
+    // Output buffered logs on error when verbose
+    if (options.verbose && logBuffer.length > 0) {
+      console.error(`\n--- Verbose output for ${projectId} (FAILED) ---`)
+      logBuffer.forEach(line => console.error(line))
+      console.error(`--- End of output for ${projectId} ---\n`)
+    }
+
+    // Check if this is a comparison error with attached details
+    const error = /** @type {ComparisonFailureError} */ (err)
+    if (error.errors && error.historyId) {
+      // Emit structured error lines
+      for (const errorRecord of error.errors) {
+        const {
+          type,
+          historyId,
+          blobHash,
+          chunkId,
+          error: errorDetail,
+        } = errorRecord
+        const errorMsg =
+          typeof errorDetail === 'string'
+            ? errorDetail
+            : errorDetail?.message || String(errorDetail)
+
+        // Use error type for structured output
+        switch (type) {
+          case 'blob-not-found':
+            console.log(`missing: ${projectId},${historyId},${blobHash}`)
+            break
+          case 'chunk-not-found':
+            console.log(`chunk-missing: ${projectId},${historyId},${chunkId}`)
+            break
+          case 'blob-hash-mismatch':
+            console.log(`hash-mismatch: ${projectId},${historyId},${blobHash}`)
+            break
+          case 'blob-size-mismatch':
+            console.log(`size-mismatch: ${projectId},${historyId},${blobHash}`)
+            break
+          case 'chunk-mismatch':
+            console.log(`chunk-mismatch: ${projectId},${historyId},${chunkId}`)
+            break
+          case 'chunk-ahead':
+            console.log(`chunk-ahead: ${projectId},${historyId},${chunkId}`)
+            break
+          default:
+            console.log(
+              `error: ${projectId},${historyId},${errorMsg.replace(/[,\n]/g, ' ')}`
+            )
+            break
+        }
+      }
+    } else {
+      // Generic error without details
+      const errorMsg = error?.message || String(error)
+      console.log(
+        `error: ${projectId},unknown,${errorMsg.replace(/[,\n]/g, ' ')}`
+      )
+    }
+    return true
+  }
+}
+
+async function compareProjectsFromFile(options) {
+  await ensureGlobalBlobsLoaded()
+  const limiter = pLimit(CONCURRENCY)
+  let totalErrors = 0
+  let totalProjects = 0
+
+  // Read project IDs from file
+  const fileContent = await fs.readFile(options.input, 'utf-8')
+  const projectIds = fileContent
+    .split('\n')
+    .map(line => line.trim())
+    .filter(line => line.length > 0)
+
+  console.error(`Loaded ${projectIds.length} project IDs from ${options.input}`)
+
+  const operations = projectIds.map(projectId =>
+    limiter(async () => {
+      totalProjects++
+      const hadError = await compareProjectAndEmitResult(
+        projectId,
+        options,
+        totalProjects,
+        projectIds.length
+      )
+      if (hadError) {
+        totalErrors++
+      }
+    })
+  )
+
+  await Promise.allSettled(operations)
+
+  console.error('\nComparison Summary:')
+  console.error('==================')
+  console.error(`Total projects processed: ${totalProjects}`)
+  console.error(`Projects with errors: ${totalErrors}`)
+
+  if (totalErrors > 0) {
+    throw new Error('Some project comparisons failed')
   }
 }
 
@@ -1063,7 +1413,9 @@ async function main() {
   } else if (options.init) {
     await initializeProjects(options)
   } else if (options.compare) {
-    if (options['start-date'] && options['end-date']) {
+    if (options.input) {
+      await compareProjectsFromFile(options)
+    } else if (options['start-date'] && options['end-date']) {
       await compareAllProjects(options)
     } else {
       await compareBackups(projectId, options)

services/notifications/Jenkinsfile

@@ -142,6 +142,15 @@ pipeline {
     always {
       junit checksName: 'notifications test results', testResults: 'services/notifications/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       dir('services/notifications') {
         sh 'make clean'

services/notifications/buildscript.txt

@@ -4,6 +4,7 @@ notifications
 --env-pass-through=
 --esmock-loader=False
 --node-version=22.18.0
+--pipeline-owner=🚉 Platform
 --public-repo=True
 --test-acceptance-vitest=True
 --test-unit-vitest=True

services/project-history/Jenkinsfile

@@ -142,6 +142,15 @@ pipeline {
     always {
       junit checksName: 'project-history test results', testResults: 'services/project-history/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🥑 Core"'
+          }
+        }
+      }
+    }
     cleanup {
       dir('services/project-history') {
         sh 'make clean'

services/project-history/buildscript.txt

@@ -4,4 +4,5 @@ project-history
 --env-pass-through=
 --esmock-loader=True
 --node-version=22.18.0
+--pipeline-owner=🥑 Core
 --public-repo=False

services/real-time/Jenkinsfile

@@ -142,6 +142,15 @@ pipeline {
     always {
       junit checksName: 'real-time test results', testResults: 'services/real-time/reports/junit-*.xml'
     }
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       dir('services/real-time') {
         sh 'make clean'

services/real-time/buildscript.txt

@@ -4,4 +4,5 @@ real-time
 --env-pass-through=
 --esmock-loader=False
 --node-version=22.18.0
+--pipeline-owner=🚉 Platform
 --public-repo=False

services/web/.storybook/Jenkinsfile

@@ -48,6 +48,15 @@ pipeline {
     }
   }
   post {
+    failure {
+      script {
+        if (env.BRANCH_NAME == 'main') {
+          node('built-in') {
+            sh '/usr/local/bin/open-gh-failure-issue --project="🚉 Platform"'
+          }
+        }
+      }
+    }
     cleanup {
       sh 'rm -rf services/web/data/storybook/'
       sh 'make clean_jenkins -j10'

services/web/app.mjs

@@ -1,7 +1,7 @@
 // Metrics must be initialized before importing anything else
 import { metricsModuleImportStartTime } from '@overleaf/metrics/initialize.js'
 
-import Modules from './app/src/infrastructure/Modules.js'
+import Modules from './app/src/infrastructure/Modules.mjs'
 import metrics from '@overleaf/metrics'
 import Settings from '@overleaf/settings'
 import logger from '@overleaf/logger'
@@ -13,10 +13,10 @@ import https from 'node:https'
 import Serializers from './app/src/infrastructure/LoggerSerializers.mjs'
 import Server from './app/src/infrastructure/Server.mjs'
 import QueueWorkers from './app/src/infrastructure/QueueWorkers.mjs'
-import mongodb from './app/src/infrastructure/mongodb.js'
-import mongoose from './app/src/infrastructure/Mongoose.js'
-import { triggerGracefulShutdown } from './app/src/infrastructure/GracefulShutdown.js'
-import FileWriter from './app/src/infrastructure/FileWriter.js'
+import mongodb from './app/src/infrastructure/mongodb.mjs'
+import mongoose from './app/src/infrastructure/Mongoose.mjs'
+import { triggerGracefulShutdown } from './app/src/infrastructure/GracefulShutdown.mjs'
+import FileWriter from './app/src/infrastructure/FileWriter.mjs'
 import { fileURLToPath } from 'node:url'
 
 metrics.gauge(

services/web/app/src/Features/Analytics/AnalyticsManager.mjs

@@ -1,7 +1,7 @@
 import SessionManager from '../Authentication/SessionManager.mjs'
 import UserAnalyticsIdCache from './UserAnalyticsIdCache.mjs'
 import Settings from '@overleaf/settings'
-import Metrics from '../../infrastructure/Metrics.js'
+import Metrics from '../../infrastructure/Metrics.mjs'
 import Queues from '../../infrastructure/Queues.mjs'
 import crypto, { createHash } from 'node:crypto'
 import _ from 'lodash'

services/web/app/src/Features/Analytics/AnalyticsRouter.mjs

@@ -1,7 +1,7 @@
 import AuthenticationController from './../Authentication/AuthenticationController.mjs'
 import AnalyticsController from './AnalyticsController.mjs'
 import AnalyticsProxy from './AnalyticsProxy.mjs'
-import { RateLimiter } from '../../infrastructure/RateLimiter.js'
+import { RateLimiter } from '../../infrastructure/RateLimiter.mjs'
 import RateLimiterMiddleware from '../Security/RateLimiterMiddleware.mjs'
 
 const rateLimiters = {

services/web/app/src/Features/Authentication/AuthenticationController.mjs

@@ -19,9 +19,9 @@ import AsyncFormHelper from '../Helpers/AsyncFormHelper.mjs'
 import _ from 'lodash'
 import UserAuditLogHandler from '../User/UserAuditLogHandler.mjs'
 import AnalyticsRegistrationSourceHelper from '../Analytics/AnalyticsRegistrationSourceHelper.mjs'
-import { acceptsJson } from '../../infrastructure/RequestContentTypeDetection.js'
+import { acceptsJson } from '../../infrastructure/RequestContentTypeDetection.mjs'
 import AdminAuthorizationHelper from '../Helpers/AdminAuthorizationHelper.mjs'
-import Modules from '../../infrastructure/Modules.js'
+import Modules from '../../infrastructure/Modules.mjs'
 import { expressify, promisify } from '@overleaf/promise-utils'
 import { handleAuthenticateErrors } from './AuthenticationErrors.mjs'
 import EmailHelper from '../Helpers/EmailHelper.mjs'

services/web/app/src/Features/Authentication/AuthenticationManager.mjs

@@ -1,6 +1,6 @@
 import Settings from '@overleaf/settings'
 import { User } from '../../models/User.mjs'
-import { db, ObjectId } from '../../infrastructure/mongodb.js'
+import { db, ObjectId } from '../../infrastructure/mongodb.mjs'
 import bcrypt from 'bcrypt'
 import EmailHelper from '../Helpers/EmailHelper.mjs'
 

services/web/app/src/Features/Authorization/PermissionsController.mjs

@@ -1,7 +1,7 @@
 // @ts-check
 import { ForbiddenError, UserNotFoundError } from '../Errors/Errors.js'
 import PermissionsManager from './PermissionsManager.mjs'
-import Modules from '../../infrastructure/Modules.js'
+import Modules from '../../infrastructure/Modules.mjs'
 import { expressify } from '@overleaf/promise-utils'
 import Features from '../../infrastructure/Features.mjs'
 

services/web/app/src/Features/Authorization/PermissionsManager.mjs

@@ -44,7 +44,7 @@
 import { callbackify } from 'node:util'
 
 import Errors from '../Errors/Errors.js'
-import Modules from '../../infrastructure/Modules.js'
+import Modules from '../../infrastructure/Modules.mjs'
 
 const { ForbiddenError } = Errors
 

services/web/app/src/Features/Chat/ChatController.mjs

@@ -1,5 +1,5 @@
 import { expressify } from '@overleaf/promise-utils'
-import Modules from '../../infrastructure/Modules.js'
+import Modules from '../../infrastructure/Modules.mjs'
 import ChatApiHandler from './ChatApiHandler.mjs'
 import EditorRealTimeController from '../Editor/EditorRealTimeController.mjs'
 import SessionManager from '../Authentication/SessionManager.mjs'

services/web/app/src/Features/Collaborators/CollaboratorsController.mjs

@@ -15,7 +15,7 @@ import TokenAccessHandler from '../TokenAccess/TokenAccessHandler.mjs'
 import ProjectAuditLogHandler from '../Project/ProjectAuditLogHandler.mjs'
 import LimitationsManager from '../Subscription/LimitationsManager.mjs'
 import PrivilegeLevels from '../Authorization/PrivilegeLevels.mjs'
-import { z, zz, validateReq } from '../../infrastructure/Validation.js'
+import { z, zz, validateReq } from '../../infrastructure/Validation.mjs'
 import Features from '../../infrastructure/Features.mjs'
 
 const { hasAdminAccess } = AdminAuthorizationHelper

services/web/app/src/Features/Collaborators/CollaboratorsInviteController.mjs

@@ -10,8 +10,8 @@ import EmailHelper from '../Helpers/EmailHelper.mjs'
 import EditorRealTimeController from '../Editor/EditorRealTimeController.mjs'
 import AnalyticsManager from '../Analytics/AnalyticsManager.mjs'
 import SessionManager from '../Authentication/SessionManager.mjs'
-import { RateLimiter } from '../../infrastructure/RateLimiter.js'
-import { z, zz, validateReq } from '../../infrastructure/Validation.js'
+import { RateLimiter } from '../../infrastructure/RateLimiter.mjs'
+import { z, zz, validateReq } from '../../infrastructure/Validation.mjs'
 import { expressify } from '@overleaf/promise-utils'
 import ProjectAuditLogHandler from '../Project/ProjectAuditLogHandler.mjs'
 import Errors from '../Errors/Errors.js'

services/web/app/src/Features/Collaborators/CollaboratorsRouter.mjs

@@ -2,7 +2,7 @@ import CollaboratorsController from './CollaboratorsController.mjs'
 import AuthenticationController from '../Authentication/AuthenticationController.mjs'
 import AuthorizationMiddleware from '../Authorization/AuthorizationMiddleware.mjs'
 import CollaboratorsInviteController from './CollaboratorsInviteController.mjs'
-import { RateLimiter } from '../../infrastructure/RateLimiter.js'
+import { RateLimiter } from '../../infrastructure/RateLimiter.mjs'
 import RateLimiterMiddleware from '../Security/RateLimiterMiddleware.mjs'
 import CaptchaMiddleware from '../Captcha/CaptchaMiddleware.mjs'
 import AnalyticsRegistrationSourceMiddleware from '../Analytics/AnalyticsRegistrationSourceMiddleware.mjs'

services/web/app/src/Features/Compile/ClsiCookieManager.mjs

@@ -6,7 +6,7 @@ import {
   fetchStringWithResponse,
   RequestFailedError,
 } from '@overleaf/fetch-utils'
-import RedisWrapper from '../../infrastructure/RedisWrapper.js'
+import RedisWrapper from '../../infrastructure/RedisWrapper.mjs'
 import Cookie from 'cookie'
 import logger from '@overleaf/logger'
 import Metrics from '@overleaf/metrics'

services/web/app/src/Features/Compile/CompileController.mjs

@@ -10,8 +10,8 @@ import logger from '@overleaf/logger'
 import Settings from '@overleaf/settings'
 import Errors from '../Errors/Errors.js'
 import SessionManager from '../Authentication/SessionManager.mjs'
-import { RateLimiter } from '../../infrastructure/RateLimiter.js'
-import Validation from '../../infrastructure/Validation.js'
+import { RateLimiter } from '../../infrastructure/RateLimiter.mjs'
+import Validation from '../../infrastructure/Validation.mjs'
 import ClsiCookieManagerFactory from './ClsiCookieManager.mjs'
 import Path from 'node:path'
 import AnalyticsManager from '../Analytics/AnalyticsManager.mjs'

services/web/app/src/Features/Compile/CompileManager.mjs

@@ -1,12 +1,12 @@
 import Crypto from 'node:crypto'
 import Settings from '@overleaf/settings'
-import RedisWrapper from '../../infrastructure/RedisWrapper.js'
+import RedisWrapper from '../../infrastructure/RedisWrapper.mjs'
 import ProjectGetter from '../Project/ProjectGetter.mjs'
 import ProjectRootDocManager from '../Project/ProjectRootDocManager.mjs'
 import UserGetter from '../User/UserGetter.mjs'
 import ClsiManager from './ClsiManager.mjs'
 import Metrics from '@overleaf/metrics'
-import { RateLimiter } from '../../infrastructure/RateLimiter.js'
+import { RateLimiter } from '../../infrastructure/RateLimiter.mjs'
 import UserAnalyticsIdCache from '../Analytics/UserAnalyticsIdCache.mjs'
 import { callbackify, callbackifyMultiResult } from '@overleaf/promise-utils'
 let CompileManager

services/web/app/src/Features/Contacts/ContactController.mjs

@@ -1,7 +1,7 @@
 import SessionManager from '../Authentication/SessionManager.mjs'
 import ContactManager from './ContactManager.mjs'
 import UserGetter from '../User/UserGetter.mjs'
-import Modules from '../../infrastructure/Modules.js'
+import Modules from '../../infrastructure/Modules.mjs'
 import { expressify } from '@overleaf/promise-utils'
 
 function _formatContact(contact) {

services/web/app/src/Features/Cooldown/CooldownManager.mjs

@@ -1,4 +1,4 @@
-import RedisWrapper from '../../infrastructure/RedisWrapper.js'
+import RedisWrapper from '../../infrastructure/RedisWrapper.mjs'
 import logger from '@overleaf/logger'
 import { promisify } from '@overleaf/promise-utils'
 const rclient = RedisWrapper.client('cooldown')

services/web/app/src/Features/DocumentUpdater/DocumentUpdaterController.mjs

@@ -1,7 +1,7 @@
 import logger from '@overleaf/logger'
 import DocumentUpdaterHandler from './DocumentUpdaterHandler.mjs'
 import ProjectLocator from '../Project/ProjectLocator.mjs'
-import { plainTextResponse } from '../../infrastructure/Response.js'
+import { plainTextResponse } from '../../infrastructure/Response.mjs'
 import { expressify } from '@overleaf/promise-utils'
 
 async function getDoc(req, res) {

services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.mjs

@@ -10,7 +10,7 @@ import _ from 'lodash'
 import logger from '@overleaf/logger'
 import { callbackifyAll } from '@overleaf/promise-utils'
 import ProjectGetter from '../Project/ProjectGetter.mjs'
-import Modules from '../../infrastructure/Modules.js'
+import Modules from '../../infrastructure/Modules.mjs'
 
 const REQUEST_TIMEOUT_MS = 30 * 1000
 const RESYNC_TIMEOUT_MS = 6 * 60 * 1000

services/web/app/src/Features/Documents/DocumentController.mjs

@@ -5,9 +5,9 @@ import ProjectEntityHandler from '../Project/ProjectEntityHandler.mjs'
 import ProjectEntityUpdateHandler from '../Project/ProjectEntityUpdateHandler.mjs'
 import logger from '@overleaf/logger'
 import _ from 'lodash'
-import { plainTextResponse } from '../../infrastructure/Response.js'
+import { plainTextResponse } from '../../infrastructure/Response.mjs'
 import { expressify } from '@overleaf/promise-utils'
-import Modules from '../../infrastructure/Modules.js'
+import Modules from '../../infrastructure/Modules.mjs'
 
 async function getDocument(req, res) {
   const { Project_id: projectId, doc_id: docId } = req.params

services/web/app/src/Features/Downloads/ProjectDownloadsController.mjs

@@ -14,7 +14,7 @@ import Metrics from '@overleaf/metrics'
 import ProjectGetter from '../Project/ProjectGetter.mjs'
 import ProjectZipStreamManager from './ProjectZipStreamManager.mjs'
 import DocumentUpdaterHandler from '../DocumentUpdater/DocumentUpdaterHandler.mjs'
-import { prepareZipAttachment } from '../../infrastructure/Response.js'
+import { prepareZipAttachment } from '../../infrastructure/Response.mjs'
 
 let ProjectDownloadsController
 

services/web/app/src/Features/Editor/EditorHttpController.mjs

@@ -11,7 +11,7 @@ import Errors from '../Errors/Errors.js'
 import { expressify } from '@overleaf/promise-utils'
 import Settings from '@overleaf/settings'
 import CollaboratorsGetter from '../Collaborators/CollaboratorsGetter.mjs'
-import { z, zz, validateReq } from '../../infrastructure/Validation.js'
+import { z, zz, validateReq } from '../../infrastructure/Validation.mjs'
 
 const ProjectAccess = CollaboratorsGetter.ProjectAccess
 

services/web/app/src/Features/Editor/EditorRealTimeController.mjs

@@ -12,7 +12,7 @@
  */
 import Settings from '@overleaf/settings'
 import Metrics from '@overleaf/metrics'
-import RedisWrapper from '../../infrastructure/RedisWrapper.js'
+import RedisWrapper from '../../infrastructure/RedisWrapper.mjs'
 import os from 'node:os'
 import crypto from 'node:crypto'
 let EditorRealTimeController

services/web/app/src/Features/Editor/EditorRouter.mjs

@@ -1,7 +1,7 @@
 import EditorHttpController from './EditorHttpController.mjs'
 import AuthenticationController from '../Authentication/AuthenticationController.mjs'
 import AuthorizationMiddleware from '../Authorization/AuthorizationMiddleware.mjs'
-import { RateLimiter } from '../../infrastructure/RateLimiter.js'
+import { RateLimiter } from '../../infrastructure/RateLimiter.mjs'
 import RateLimiterMiddleware from '../Security/RateLimiterMiddleware.mjs'
 
 const rateLimiters = {

services/web/app/src/Features/Email/EmailBuilder.mjs

@@ -181,6 +181,26 @@ templates.canceledSubscription = ctaTemplate({
   },
 })
 
+templates.canceledSubscriptionOrAddOn = ctaTemplate({
+  subject() {
+    return `${settings.appName} thoughts`
+  },
+  message() {
+    return [
+      `We are sorry to see you cancelled your ${settings.appName} subscription. Would you mind giving us some feedback on what the site is lacking at the moment via this quick survey?`,
+    ]
+  },
+  secondaryMessage() {
+    return ['Thank you in advance!']
+  },
+  ctaText() {
+    return 'Leave feedback'
+  },
+  ctaURL(opts) {
+    return 'https://digitalscience.qualtrics.com/jfe/form/SV_2n2aSlWgvoxXdGK'
+  },
+})
+
 templates.reactivatedSubscription = ctaTemplate({
   subject() {
     return `Subscription Reactivated - ${settings.appName}`

services/web/app/src/Features/Email/EmailSender.mjs

@@ -5,7 +5,7 @@ import Settings from '@overleaf/settings'
 import nodemailer from 'nodemailer'
 import aws from '@aws-sdk/client-ses'
 import OError from '@overleaf/o-error'
-import { RateLimiter } from '../../infrastructure/RateLimiter.js'
+import { RateLimiter } from '../../infrastructure/RateLimiter.mjs'
 import _ from 'lodash'
 
 const EMAIL_SETTINGS = Settings.email || {}

services/web/app/src/Features/Errors/ErrorController.mjs

@@ -3,7 +3,7 @@ import Errors from './Errors.js'
 import SessionManager from '../Authentication/SessionManager.mjs'
 import SamlLogHandler from '../SamlLog/SamlLogHandler.mjs'
 import HttpErrorHandler from './HttpErrorHandler.mjs'
-import { plainTextResponse } from '../../infrastructure/Response.js'
+import { plainTextResponse } from '../../infrastructure/Response.mjs'
 import { expressifyErrorHandler } from '@overleaf/promise-utils'
 
 function notFound(req, res) {

services/web/app/src/Features/Errors/HttpErrorHandler.mjs

@@ -1,6 +1,6 @@
 import logger from '@overleaf/logger'
 import Settings from '@overleaf/settings'
-import { plainTextResponse } from '../../infrastructure/Response.js'
+import { plainTextResponse } from '../../infrastructure/Response.mjs'
 
 function renderJSONError(res, message, info = {}) {
   if (info.message) {

services/web/app/src/Features/FileStore/FileStoreController.mjs

@@ -7,7 +7,7 @@ import Metrics from '@overleaf/metrics'
 import ProjectLocator from '../Project/ProjectLocator.mjs'
 import HistoryManager from '../History/HistoryManager.mjs'
 import Errors from '../Errors/Errors.js'
-import { preparePlainTextResponse } from '../../infrastructure/Response.js'
+import { preparePlainTextResponse } from '../../infrastructure/Response.mjs'
 
 async function getFile(req, res) {
   const projectId = req.params.Project_id

services/web/app/src/Features/FileStore/FileStoreHandler.mjs

@@ -7,7 +7,7 @@ import ProjectDetailsHandler from '../Project/ProjectDetailsHandler.mjs'
 import { File } from '../../models/File.mjs'
 import OError from '@overleaf/o-error'
 import { promisifyAll } from '@overleaf/promise-utils'
-import Modules from '../../infrastructure/Modules.js'
+import Modules from '../../infrastructure/Modules.mjs'
 
 const FileStoreHandler = {
   RETRY_ATTEMPTS: 3,

services/web/app/src/Features/HealthCheck/HealthCheckController.mjs

@@ -1,4 +1,4 @@
-import RedisWrapper from '../../infrastructure/RedisWrapper.js'
+import RedisWrapper from '../../infrastructure/RedisWrapper.mjs'
 import settings from '@overleaf/settings'
 import logger from '@overleaf/logger'
 import UserGetter from '../User/UserGetter.mjs'

services/web/app/src/Features/Helpers/AdminAuthorizationHelper.mjs

@@ -1,5 +1,5 @@
 import Settings from '@overleaf/settings'
-import Modules from '../../infrastructure/Modules.js'
+import Modules from '../../infrastructure/Modules.mjs'
 import { expressify } from '@overleaf/promise-utils'
 import SessionManager from '../Authentication/SessionManager.mjs'
 import logger from '@overleaf/logger'

services/web/app/src/Features/Helpers/AsyncFormHelper.mjs

@@ -1,4 +1,4 @@
-import { acceptsJson } from '../../infrastructure/RequestContentTypeDetection.js'
+import { acceptsJson } from '../../infrastructure/RequestContentTypeDetection.mjs'
 
 export default {
   redirect,

services/web/app/src/Features/Helpers/StringHelper.mjs

@@ -15,6 +15,20 @@ const JSON_ESCAPE = {
   '\u2029': '\\u2029',
 }
 
+/**
+ * Converts a snake_case string into a user friendly string with each word capitalized.
+ * @param {string} snakecaseStr
+ * @returns {string}
+ */
+export function wordifySnakecase(snakecaseStr) {
+  return snakecaseStr
+    .split('_')
+    .map(word => {
+      return word.charAt(0).toUpperCase() + word.slice(1)
+    })
+    .join(' ')
+}
+
 export default StringHelper = {
   // stringifies and escapes a json object for use in a script. This ensures that &, < and > characters are escaped,
   // along with quotes. This ensures that the string can be safely rendered into HTML. See rationale at:

services/web/app/src/Features/History/HistoryController.mjs

@@ -23,9 +23,9 @@ import HistoryManager from './HistoryManager.mjs'
 import ProjectDetailsHandler from '../Project/ProjectDetailsHandler.mjs'
 import ProjectEntityUpdateHandler from '../Project/ProjectEntityUpdateHandler.mjs'
 import RestoreManager from './RestoreManager.mjs'
-import { prepareZipAttachment } from '../../infrastructure/Response.js'
+import { prepareZipAttachment } from '../../infrastructure/Response.mjs'
 import Features from '../../infrastructure/Features.mjs'
-import { z, zz, validateReq } from '../../infrastructure/Validation.js'
+import { z, zz, validateReq } from '../../infrastructure/Validation.mjs'
 
 // Number of seconds after which the browser should send a request to revalidate
 // blobs

services/web/app/src/Features/History/HistoryManager.mjs

@@ -11,7 +11,7 @@ import OError from '@overleaf/o-error'
 import UserGetter from '../User/UserGetter.mjs'
 import ProjectGetter from '../Project/ProjectGetter.mjs'
 import HistoryBackupDeletionHandler from './HistoryBackupDeletionHandler.mjs'
-import { db, waitForDb } from '../../infrastructure/mongodb.js'
+import { db, waitForDb } from '../../infrastructure/mongodb.mjs'
 import Metrics from '@overleaf/metrics'
 import { NotFoundError } from '../Errors/Errors.js'
 

services/web/app/src/Features/History/HistoryRangesSupportMigration.mjs

@@ -7,7 +7,7 @@ import HistoryManager from '../History/HistoryManager.mjs'
 import DocumentUpdaterHandler from '../DocumentUpdater/DocumentUpdaterHandler.mjs'
 import DocstoreManager from '../Docstore/DocstoreManager.mjs'
 import ProjectOptionsHandler from '../Project/ProjectOptionsHandler.mjs'
-import mongodb from '../../infrastructure/mongodb.js'
+import mongodb from '../../infrastructure/mongodb.mjs'
 
 const { db, ObjectId, READ_PREFERENCE_SECONDARY } = mongodb
 

services/web/app/src/Features/History/HistoryRouter.mjs

@@ -1,7 +1,7 @@
 // @ts-check
 
 import Settings from '@overleaf/settings'
-import { RateLimiter } from '../../infrastructure/RateLimiter.js'
+import { RateLimiter } from '../../infrastructure/RateLimiter.mjs'
 import AuthenticationController from '../Authentication/AuthenticationController.mjs'
 import AuthorizationMiddleware from '../Authorization/AuthorizationMiddleware.mjs'
 import RateLimiterMiddleware from '../Security/RateLimiterMiddleware.mjs'

services/web/app/src/Features/History/RestoreManager.mjs

@@ -1,7 +1,7 @@
 import Settings from '@overleaf/settings'
 import Path from 'node:path'
-import FileWriter from '../../infrastructure/FileWriter.js'
-import Metrics from '../../infrastructure/Metrics.js'
+import FileWriter from '../../infrastructure/FileWriter.mjs'
+import Metrics from '../../infrastructure/Metrics.mjs'
 import FileSystemImportManager from '../Uploads/FileSystemImportManager.mjs'
 import FileTypeManager from '../Uploads/FileTypeManager.mjs'
 import EditorController from '../Editor/EditorController.mjs'

services/web/app/src/Features/InactiveData/InactiveProjectManager.mjs

@@ -5,8 +5,8 @@ import DocumentUpdaterHandler from '../DocumentUpdater/DocumentUpdaterHandler.mj
 import ProjectGetter from '../Project/ProjectGetter.mjs'
 import ProjectUpdateHandler from '../Project/ProjectUpdateHandler.mjs'
 import { Project } from '../../models/Project.mjs'
-import Modules from '../../infrastructure/Modules.js'
-import { READ_PREFERENCE_SECONDARY } from '../../infrastructure/mongodb.js'
+import Modules from '../../infrastructure/Modules.mjs'
+import { READ_PREFERENCE_SECONDARY } from '../../infrastructure/mongodb.mjs'
 import { callbackifyAll } from '@overleaf/promise-utils'
 import Metrics from '@overleaf/metrics'
 

services/web/app/src/Features/Institutions/InstitutionsAPI.mjs

@@ -10,7 +10,7 @@ import {
   InvalidInstitutionalEmailError,
 } from '../Errors/Errors.js'
 import { fetchJson, fetchNothing } from '@overleaf/fetch-utils'
-import Modules from '../../infrastructure/Modules.js'
+import Modules from '../../infrastructure/Modules.mjs'
 
 function _makeRequestOptions(options) {
   const requestOptions = {

services/web/app/src/Features/LinkedFiles/LinkedFilesController.mjs

@@ -20,9 +20,9 @@ import {
   OutputFileFetchFailedError,
   FileTooLargeError,
 } from '../Errors/Errors.js'
-import Modules from '../../infrastructure/Modules.js'
-import { plainTextResponse } from '../../infrastructure/Response.js'
-import { z, zz, validateReq } from '../../infrastructure/Validation.js'
+import Modules from '../../infrastructure/Modules.mjs'
+import { plainTextResponse } from '../../infrastructure/Response.mjs'
+import { z, zz, validateReq } from '../../infrastructure/Validation.mjs'
 import EditorRealTimeController from '../Editor/EditorRealTimeController.mjs'
 import { expressify } from '@overleaf/promise-utils'
 import ProjectOutputFileAgent from './ProjectOutputFileAgent.mjs'

services/web/app/src/Features/LinkedFiles/LinkedFilesHandler.mjs

@@ -1,4 +1,4 @@
-import FileWriter from '../../infrastructure/FileWriter.js'
+import FileWriter from '../../infrastructure/FileWriter.mjs'
 import EditorController from '../Editor/EditorController.mjs'
 import ProjectLocator from '../Project/ProjectLocator.mjs'
 import { Project } from '../../models/Project.mjs'

services/web/app/src/Features/LinkedFiles/LinkedFilesRouter.mjs

@@ -1,6 +1,6 @@
 import AuthorizationMiddleware from '../Authorization/AuthorizationMiddleware.mjs'
 import AuthenticationController from '../Authentication/AuthenticationController.mjs'
-import { RateLimiter } from '../../infrastructure/RateLimiter.js'
+import { RateLimiter } from '../../infrastructure/RateLimiter.mjs'
 import RateLimiterMiddleware from '../Security/RateLimiterMiddleware.mjs'
 import LinkedFilesController from './LinkedFilesController.mjs'
 

services/web/app/src/Features/PasswordReset/PasswordResetController.mjs

@@ -8,7 +8,7 @@ import UserSessionsManager from '../User/UserSessionsManager.mjs'
 import OError from '@overleaf/o-error'
 import EmailsHelper from '../Helpers/EmailHelper.mjs'
 import { expressify } from '@overleaf/promise-utils'
-import { z, validateReq } from '../../infrastructure/Validation.js'
+import { z, validateReq } from '../../infrastructure/Validation.mjs'
 
 const setNewUserPasswordSchema = z.object({
   body: z.object({

services/web/app/src/Features/PasswordReset/PasswordResetRouter.mjs

@@ -1,7 +1,7 @@
 import PasswordResetController from './PasswordResetController.mjs'
 import AuthenticationController from '../Authentication/AuthenticationController.mjs'
 import CaptchaMiddleware from '../../Features/Captcha/CaptchaMiddleware.mjs'
-import { RateLimiter } from '../../infrastructure/RateLimiter.js'
+import { RateLimiter } from '../../infrastructure/RateLimiter.mjs'
 import RateLimiterMiddleware from '../Security/RateLimiterMiddleware.mjs'
 
 const rateLimiter = new RateLimiter('password_reset_rate_limit', {

services/web/app/src/Features/Project/ProjectAuditLogHandler.mjs

@@ -4,6 +4,7 @@ import { callbackify } from '@overleaf/promise-utils'
 import SubscriptionLocator from '../Subscription/SubscriptionLocator.mjs'
 
 const MANAGED_GROUP_PROJECT_EVENTS = [
+  'send-invite',
   'accept-invite',
   'project-created',
   'project-deleted',

services/web/app/src/Features/Project/ProjectController.mjs

@@ -44,8 +44,8 @@ import PublicAccessLevels from '../Authorization/PublicAccessLevels.mjs'
 import TagsHandler from '../Tags/TagsHandler.mjs'
 import TutorialHandler from '../Tutorial/TutorialHandler.mjs'
 import UserUpdater from '../User/UserUpdater.mjs'
-import Modules from '../../infrastructure/Modules.js'
-import { z, zz, validateReq } from '../../infrastructure/Validation.js'
+import Modules from '../../infrastructure/Modules.mjs'
+import { z, zz, validateReq } from '../../infrastructure/Validation.mjs'
 import UserGetter from '../User/UserGetter.mjs'
 import { isStandaloneAiAddOnPlanCode } from '../Subscription/AiHelper.mjs'
 import SubscriptionController from '../Subscription/SubscriptionController.mjs'
@@ -456,6 +456,7 @@ const _ProjectController = {
       'compile-timeout-target-plans',
       'writefull-keywords-generator',
       'writefull-figure-generator',
+      'writefull-asymetric-queue-size-per-model',
       'pdf-dark-mode',
     ].filter(Boolean)
 
@@ -1308,49 +1309,49 @@ const defaultUserValues = () => ({
 })
 
 const THEME_LIST = [
-  'cobalt',
-  'dracula',
-  'eclipse',
-  'monokai',
-  'overleaf',
-  'overleaf_dark',
-  'textmate',
+  { name: 'cobalt', dark: true },
+  { name: 'dracula', dark: true },
+  { name: 'eclipse', dark: false },
+  { name: 'monokai', dark: true },
+  { name: 'overleaf', dark: false },
+  { name: 'overleaf_dark', dark: true },
+  { name: 'textmate', dark: false },
 ]
 
 const LEGACY_THEME_LIST = [
-  'ambiance',
-  'chaos',
-  'chrome',
-  'clouds',
-  'clouds_midnight',
-  'crimson_editor',
-  'dawn',
-  'dreamweaver',
-  'github',
-  'gob',
-  'gruvbox',
-  'idle_fingers',
-  'iplastic',
-  'katzenmilch',
-  'kr_theme',
-  'kuroir',
-  'merbivore',
-  'merbivore_soft',
-  'mono_industrial',
-  'nord_dark',
-  'pastel_on_dark',
-  'solarized_dark',
-  'solarized_light',
-  'sqlserver',
-  'terminal',
-  'tomorrow',
-  'tomorrow_night',
-  'tomorrow_night_blue',
-  'tomorrow_night_bright',
-  'tomorrow_night_eighties',
-  'twilight',
-  'vibrant_ink',
-  'xcode',
+  { name: 'ambiance', dark: true },
+  { name: 'chaos', dark: true },
+  { name: 'chrome', dark: false },
+  { name: 'clouds', dark: false },
+  { name: 'clouds_midnight', dark: true },
+  { name: 'crimson_editor', dark: false },
+  { name: 'dawn', dark: false },
+  { name: 'dreamweaver', dark: false },
+  { name: 'github', dark: false },
+  { name: 'gob', dark: true },
+  { name: 'gruvbox', dark: true },
+  { name: 'idle_fingers', dark: true },
+  { name: 'iplastic', dark: false },
+  { name: 'katzenmilch', dark: false },
+  { name: 'kr_theme', dark: true },
+  { name: 'kuroir', dark: false },
+  { name: 'merbivore', dark: true },
+  { name: 'merbivore_soft', dark: true },
+  { name: 'mono_industrial', dark: true },
+  { name: 'nord_dark', dark: true },
+  { name: 'pastel_on_dark', dark: true },
+  { name: 'solarized_dark', dark: true },
+  { name: 'solarized_light', dark: false },
+  { name: 'sqlserver', dark: false },
+  { name: 'terminal', dark: true },
+  { name: 'tomorrow', dark: false },
+  { name: 'tomorrow_night', dark: true },
+  { name: 'tomorrow_night_blue', dark: true },
+  { name: 'tomorrow_night_bright', dark: true },
+  { name: 'tomorrow_night_eighties', dark: true },
+  { name: 'twilight', dark: true },
+  { name: 'vibrant_ink', dark: true },
+  { name: 'xcode', dark: false },
 ]
 
 const ProjectController = {

services/web/app/src/Features/Project/ProjectDeleter.mjs

@@ -3,8 +3,8 @@ import {
   db,
   ObjectId,
   READ_PREFERENCE_SECONDARY,
-} from '../../infrastructure/mongodb.js'
-import Modules from '../../infrastructure/Modules.js'
+} from '../../infrastructure/mongodb.mjs'
+import Modules from '../../infrastructure/Modules.mjs'
 import { callbackify } from 'node:util'
 import { Project } from '../../models/Project.mjs'
 import { DeletedProject } from '../../models/DeletedProject.mjs'

services/web/app/src/Features/Project/ProjectDuplicator.mjs

@@ -20,7 +20,7 @@ import TpdsProjectFlusher from '../ThirdPartyDataStore/TpdsProjectFlusher.mjs'
 import _ from 'lodash'
 import TagsHandler from '../Tags/TagsHandler.mjs'
 import ClsiCacheManager from '../Compile/ClsiCacheManager.mjs'
-import Modules from '../../infrastructure/Modules.js'
+import Modules from '../../infrastructure/Modules.mjs'
 
 export default {
   duplicate: callbackify(duplicate),

services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandler.mjs

@@ -8,7 +8,7 @@ import OError from '@overleaf/o-error'
 import CooldownManager from '../Cooldown/CooldownManager.mjs'
 import Errors from '../Errors/Errors.js'
 import { Folder } from '../../models/Folder.mjs'
-import LockManager from '../../infrastructure/LockManager.js'
+import LockManager from '../../infrastructure/LockManager.mjs'
 import { Project } from '../../models/Project.mjs'
 import ProjectEntityHandler from './ProjectEntityHandler.mjs'
 import ProjectGetter from './ProjectGetter.mjs'

services/web/app/src/Features/Project/ProjectEntityUpdateHandler.mjs

@@ -9,7 +9,7 @@ import DocstoreManager from '../Docstore/DocstoreManager.mjs'
 import DocumentUpdaterHandler from '../../Features/DocumentUpdater/DocumentUpdaterHandler.mjs'
 import Errors from '../Errors/Errors.js'
 import FileStoreHandler from '../FileStore/FileStoreHandler.mjs'
-import LockManager from '../../infrastructure/LockManager.js'
+import LockManager from '../../infrastructure/LockManager.mjs'
 import { Project } from '../../models/Project.mjs'
 import ProjectEntityHandler from './ProjectEntityHandler.mjs'
 import ProjectGetter from './ProjectGetter.mjs'
@@ -19,7 +19,7 @@ import ProjectUpdateHandler from './ProjectUpdateHandler.mjs'
 import ProjectEntityMongoUpdateHandler from './ProjectEntityMongoUpdateHandler.mjs'
 import SafePath from './SafePath.mjs'
 import TpdsUpdateSender from '../ThirdPartyDataStore/TpdsUpdateSender.mjs'
-import FileWriter from '../../infrastructure/FileWriter.js'
+import FileWriter from '../../infrastructure/FileWriter.mjs'
 import EditorRealTimeController from '../Editor/EditorRealTimeController.mjs'
 import { callbackifyMultiResult, callbackify } from '@overleaf/promise-utils'
 import { iterablePaths } from './IterablePath.mjs'