Merge pull request #29546 from overleaf/mfb-from-joi-to-zod-real-time

RE MIGRATE from joi to zod, moving schemas to top level in file

GitOrigin-RevId: c1512be2e7d6edf52c3dc01d62f2fc2051b3d9b2

package-lock.json

@@ -52882,6 +52882,7 @@
         "@overleaf/o-error": "*",
         "@overleaf/redis-wrapper": "*",
         "@overleaf/settings": "*",
+        "@overleaf/validation-tools": "*",
         "async": "^3.2.5",
         "base64id": "0.1.0",
         "body-parser": "^1.20.3",
@@ -52890,12 +52891,12 @@
         "cookie-parser": "^1.4.6",
         "express": "^4.21.2",
         "express-session": "^1.17.1",
-        "joi": "^17.12.0",
         "lodash": "^4.17.21",
         "proxy-addr": "^2.0.7",
         "request": "^2.88.2",
         "socket.io": "github:overleaf/socket.io#0.9.19-overleaf-12",
-        "socket.io-client": "github:overleaf/socket.io-client#0.9.17-overleaf-5"
+        "socket.io-client": "github:overleaf/socket.io-client#0.9.17-overleaf-5",
+        "zod-validation-error": "^4.0.1"
       },
       "devDependencies": {
         "chai": "^4.3.6",

services/real-time/.jenkinsIncludeFile

@@ -7,6 +7,7 @@ libraries/metrics/**
 libraries/o-error/**
 libraries/redis-wrapper/**
 libraries/settings/**
+libraries/validation-tools/**
 package-lock.json
 package.json
 patches/**

services/real-time/Dockerfile

@@ -19,6 +19,7 @@ COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
 COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
 COPY libraries/redis-wrapper/package.json /overleaf/libraries/redis-wrapper/package.json
 COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
+COPY libraries/validation-tools/package.json /overleaf/libraries/validation-tools/package.json
 COPY services/real-time/package.json /overleaf/services/real-time/package.json
 COPY patches/ /overleaf/patches/
 
@@ -29,6 +30,7 @@ COPY libraries/metrics/ /overleaf/libraries/metrics/
 COPY libraries/o-error/ /overleaf/libraries/o-error/
 COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/
 COPY libraries/settings/ /overleaf/libraries/settings/
+COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/
 COPY services/real-time/ /overleaf/services/real-time/
 
 FROM app

services/real-time/Makefile

@@ -20,6 +20,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
 	$(MONOREPO)/libraries/o-error/package.json \
 	$(MONOREPO)/libraries/redis-wrapper/package.json \
 	$(MONOREPO)/libraries/settings/package.json \
+	$(MONOREPO)/libraries/validation-tools/package.json \
 	$(MONOREPO)/services/real-time/package.json \
 	$(MONOREPO)/patches/* \
 | sha256sum | cut -d '-' -f1)

services/real-time/app/js/Router.js

@@ -8,16 +8,23 @@ const WebsocketAddressManager = require('./WebsocketAddressManager')
 const bodyParser = require('body-parser')
 const base64id = require('base64id')
 const { UnexpectedArgumentsError } = require('./Errors')
-const Joi = require('joi')
+const { z, zz } = require('@overleaf/validation-tools')
+const { isZodErrorLike } = require('zod-validation-error')
 
 const HOSTNAME = require('node:os').hostname()
 const SERVER_PING_INTERVAL = 15000
 const SERVER_PING_LATENCY_THRESHOLD = 5000
 
-const JOI_OBJECT_ID = Joi.string()
-  .required()
-  .regex(/^[0-9a-f]{24}$/)
-  .message('invalid id')
+const joinDocSchema = z.object({
+  doc_id: zz.objectId(),
+  fromVersion: z.number().int().optional(),
+  options: z.object(),
+})
+
+const applyOtUpdateSchema = z.object({
+  doc_id: zz.objectId(),
+  update: z.object(),
+})
 
 let Router
 module.exports = Router = {
@@ -29,11 +36,11 @@ module.exports = Router = {
     attrs.client_id = client.id
     attrs.err = error
     attrs.method = method
-    if (Joi.isError(error)) {
+    if (isZodErrorLike(error)) {
       logger.info(attrs, 'validation error')
       let message = 'invalid'
       try {
-        message = error.details[0].message
+        message = error.issues[0].message
       } catch (e) {
         // ignore unexpected errors
         logger.warn({ error, e }, 'unexpected validation error')
@@ -193,7 +200,7 @@ module.exports = Router = {
 
       if (!isDebugging) {
         try {
-          Joi.assert(projectId, JOI_OBJECT_ID)
+          zz.objectId().parse(projectId)
         } catch (error) {
           metrics.inc('socket-io.connection', 1, {
             status: client.transport,
@@ -442,14 +449,7 @@ module.exports = Router = {
           return Router._handleInvalidArguments(client, 'joinDoc', arguments)
         }
         try {
-          Joi.assert(
-            { doc_id: docId, fromVersion, options },
-            Joi.object({
-              doc_id: JOI_OBJECT_ID,
-              fromVersion: Joi.number().integer(),
-              options: Joi.object().required(),
-            })
-          )
+          joinDocSchema.parse({ doc_id: docId, fromVersion, options })
         } catch (error) {
           return Router._handleError(callback, error, client, 'joinDoc', {
             disconnect: 1,
@@ -478,7 +478,7 @@ module.exports = Router = {
           return Router._handleInvalidArguments(client, 'leaveDoc', arguments)
         }
         try {
-          Joi.assert(docId, JOI_OBJECT_ID)
+          zz.objectId().parse(docId)
         } catch (error) {
           return Router._handleError(callback, error, client, 'joinDoc', {
             disconnect: 1,
@@ -563,13 +563,7 @@ module.exports = Router = {
           )
         }
         try {
-          Joi.assert(
-            { doc_id: docId, update },
-            Joi.object({
-              doc_id: JOI_OBJECT_ID,
-              update: Joi.object().required(),
-            })
-          )
+          applyOtUpdateSchema.parse({ doc_id: docId, update })
         } catch (error) {
           return Router._handleError(callback, error, client, 'applyOtUpdate', {
             disconnect: 1,

services/real-time/package.json

@@ -22,6 +22,7 @@
     "@overleaf/o-error": "*",
     "@overleaf/redis-wrapper": "*",
     "@overleaf/settings": "*",
+    "@overleaf/validation-tools": "*",
     "async": "^3.2.5",
     "base64id": "0.1.0",
     "body-parser": "^1.20.3",
@@ -30,12 +31,12 @@
     "cookie-parser": "^1.4.6",
     "express": "^4.21.2",
     "express-session": "^1.17.1",
-    "joi": "^17.12.0",
     "lodash": "^4.17.21",
     "proxy-addr": "^2.0.7",
     "request": "^2.88.2",
     "socket.io": "github:overleaf/socket.io#0.9.19-overleaf-12",
-    "socket.io-client": "github:overleaf/socket.io-client#0.9.17-overleaf-5"
+    "socket.io-client": "github:overleaf/socket.io-client#0.9.17-overleaf-5",
+    "zod-validation-error": "^4.0.1"
   },
   "devDependencies": {
     "chai": "^4.3.6",

services/real-time/test/acceptance/js/JoinDocTests.js

@@ -328,7 +328,7 @@ describe('joinDoc', function () {
     })
 
     it('should return an invalid id error', function () {
-      this.error.message.should.equal('invalid id')
+      this.error.message.should.equal('invalid Mongo ObjectId')
     })
 
     return it('should not have joined the doc room', function (done) {

services/web/Jenkinsfile

@@ -269,11 +269,20 @@ pipeline {
             }
           }
         }
-        stage('Test Unit ESM') {
+        stage('Test Unit ESM - Parallel') {
           steps {
             dir('services/web') {
               retry(count: 3) {
-                sh "make test_unit_esm"
+                sh "make test_unit_esm_parallel"
+              }
+            }
+          }
+        }
+        stage('Test Unit ESM - Sequential') {
+          steps {
+            dir('services/web') {
+              retry(count: 3) {
+                sh "make test_unit_esm_sequential"
               }
             }
           }

services/web/Makefile

@@ -106,11 +106,21 @@ test_unit_mocha: export COMPOSE_PROJECT_NAME=unit_test_mocha_$(BUILD_DIR_NAME)
 test_unit_mocha: mongo_migrations_for_tests
 	$(DOCKER_COMPOSE) run --rm test_unit npm run test:unit:mocha
 
-test_unit_esm: export MOCHA_ROOT_SUITE_NAME = ESM unit tests
-test_unit_esm: export COMPOSE_PROJECT_NAME=unit_test_esm_$(BUILD_DIR_NAME)
+test_unit_esm: export VITEST_ROOT_SUITE_NAME = ESM unit tests - parallel
+test_unit_esm: export COMPOSE_PROJECT_NAME=unit_test_esm_parallel_$(BUILD_DIR_NAME)
 test_unit_esm: mongo_migrations_for_tests
 	$(DOCKER_COMPOSE) run --rm test_unit npm run test:unit:esm
 
+test_unit_esm_parallel: export VITEST_ROOT_SUITE_NAME = ESM unit tests - parallel
+test_unit_esm_parallel: export COMPOSE_PROJECT_NAME=unit_test_esm_parallel_$(BUILD_DIR_NAME)
+test_unit_esm_parallel: mongo_migrations_for_tests
+	$(DOCKER_COMPOSE) run --rm test_unit npm run test:unit:esm:parallel
+
+test_unit_esm_sequential: export VITEST_ROOT_SUITE_NAME = ESM unit tests - sequential
+test_unit_esm_sequential: export COMPOSE_PROJECT_NAME=unit_test_esm_sequential_$(BUILD_DIR_NAME)
+test_unit_esm_sequential: mongo_migrations_for_tests
+	$(DOCKER_COMPOSE) run --rm test_unit npm run test:unit:esm:sequential
+
 test_unit_esm_watch: export COMPOSE_PROJECT_NAME=unit_test_esm_watch_$(BUILD_DIR_NAME)
 test_unit_esm_watch: mongo_migrations_for_tests
 	$(DOCKER_COMPOSE) run --rm test_unit npm run test:unit:esm:watch

services/web/app/src/Features/User/UserAuditLogHandler.mjs

@@ -5,6 +5,7 @@ import { callbackify } from 'node:util'
 import SubscriptionLocator from '../Subscription/SubscriptionLocator.mjs'
 
 function _canHaveNoIpAddressId(operation, info) {
+  if (operation === 'add-email' && info.script) return true
   if (operation === 'join-group-subscription') return true
   if (operation === 'leave-group-subscription') return true
   if (operation === 'must-reset-password-set') return true
@@ -15,6 +16,7 @@ function _canHaveNoIpAddressId(operation, info) {
 }
 
 function _canHaveNoInitiatorId(operation, info) {
+  if (operation === 'add-email' && info.script) return true
   if (operation === 'reset-password') return true
   if (operation === 'unlink-sso' && info.providerId === 'collabratec')
     return true

services/web/app/src/Features/User/UserUpdater.mjs

@@ -101,6 +101,7 @@ async function addEmailAddress(userId, newEmail, affiliationOptions, auditLog) {
     auditLog.initiatorId,
     auditLog.ipAddress,
     {
+      ...auditLog.info,
       newSecondaryEmail: newEmail,
     }
   )

services/web/frontend/extracted-translations.json

@@ -97,6 +97,7 @@
   "add_to_tag": "",
   "add_unlimited_ai_to_overleaf": "",
   "add_unlimited_ai_to_your_overleaf_plan": "",
+  "add_unlimited_ai_to_your_plan": "",
   "add_your_comment_here": "",
   "add_your_first_group_member_now": "",
   "added_by_on": "",

services/web/locales/en.json

@@ -112,6 +112,7 @@
   "add_to_tag": "Add to tag",
   "add_unlimited_ai_to_overleaf": "Add unlimited AI* to Overleaf",
   "add_unlimited_ai_to_your_overleaf_plan": "Add unlimited AI* to your Overleaf __planName__ plan",
+  "add_unlimited_ai_to_your_plan": "Add unlimited AI* to your __planName__ plan",
   "add_your_comment_here": "Add your comment here",
   "add_your_first_group_member_now": "Add your first group members now",
   "added": "added",
@@ -2715,7 +2716,7 @@
   "work_or_university_sso": "Work/university single sign-on",
   "work_with_non_overleaf_users": "Work with non Overleaf users",
   "work_with_other_github_users": "Work with other GitHub users",
-  "write_faster_smarter_with_overleaf_and_writefull_ai_tools": "Write faster, smarter, and with confidence with Overleaf and Writefull AI tools",
+  "write_faster_smarter_with_overleaf_and_writefull_ai_tools": "Write faster, smarter and with confidence with AI Assist",
   "writefull": "Writefull",
   "x_changes_in": "__count__ change in",
   "x_changes_in_plural": "__count__ changes in",

services/web/package.json

@@ -15,7 +15,9 @@
     "test:unit:app": "npm run test:unit:run_dir -- test/unit/src",
     "test:unit:mocha": "npm run test:unit:mocha:run_dir -- test/unit/src modules/*/test/unit/src",
     "test:unit:mocha:run_dir": "mocha --recursive --timeout 25000 --exit --grep=$MOCHA_GREP --require test/unit/bootstrap.js --extension=js",
-    "test:unit:esm": "vitest run",
+    "test:unit:esm": "npm run test:unit:esm:parallel && npm run test:unit:esm:sequential",
+    "test:unit:esm:parallel": "vitest run --project=Parallel",
+    "test:unit:esm:sequential": "vitest run --project=Sequential --no-file-parallelism",
     "test:unit:esm:watch": "vitest",
     "test:frontend": "NODE_ENV=test TZ=GMT mocha --recursive --timeout 5000 --exit --extension js,jsx,mjs,ts,tsx --grep=$MOCHA_GREP --require test/frontend/bootstrap.js --ignore '**/*.spec.{js,jsx,ts,tsx}' --ignore '**/helpers/**/*.{js,jsx,ts,tsx}' test/frontend modules/*/test/frontend",
     "test:frontend:coverage": "c8 --all --include 'frontend/js' --include 'modules/*/frontend/js' --exclude 'frontend/js/vendor' --reporter=lcov --reporter=text-summary npm run test:frontend",

services/web/scripts/re_add_deleted_emails.mjs

@@ -0,0 +1,288 @@
+// @ts-check
+
+import minimist from 'minimist'
+import { db, ObjectId } from '../app/src/infrastructure/mongodb.js'
+import fs from 'node:fs/promises'
+import * as csv from 'csv'
+import { promisify } from 'node:util'
+import { scriptRunner } from './lib/ScriptRunner.mjs'
+import Errors from '../app/src/Features/Errors/Errors.js'
+import UserGetter from '../app/src/Features/User/UserGetter.mjs'
+import { READ_PREFERENCE_SECONDARY } from '@overleaf/mongo-utils/batchedUpdate.js'
+import UserUpdater from '../app/src/Features/User/UserUpdater.mjs'
+import EmailHelper from '../app/src/Features/Helpers/EmailHelper.js'
+import AsyncLocalStorage from '../app/src/infrastructure/AsyncLocalStorage.js'
+import AnalyticsManager from '../app/src/Features/Analytics/AnalyticsManager.mjs'
+import UserAuditLogHandler from '../app/src/Features/User/UserAuditLogHandler.mjs'
+import InstitutionsAPI from '../app/src/Features/Institutions/InstitutionsAPI.mjs'
+import OError from '@overleaf/o-error'
+import EmailChangeHelper from '../app/src/Features/Analytics/EmailChangeHelper.mjs'
+import logger from '@overleaf/logger'
+
+const CSV_FILENAME = '/tmp/re_add_deleted_emails.csv'
+
+/**
+ * @type {(csvString: string) => Promise<string[][]>}
+ */
+const parseAsync = promisify(csv.parse)
+
+function usage() {
+  console.log('Usage: node re_add_deleted_emails.mjs [options]')
+  console.log(
+    'fix wrongly removed emails by remove_unconfirmed_emails (see 2025-11-05 email removal)'
+  )
+  console.log(
+    'run this script with a CSV containing user IDs and emails to re-add save in ',
+    CSV_FILENAME
+  )
+  console.log('Options:')
+  console.log('  --commit       apply the changes')
+  process.exit(0)
+}
+
+const { commit, help } = minimist(process.argv.slice(2), {
+  boolean: ['commit', 'help'],
+  alias: { help: 'h' },
+  default: { commit: false },
+})
+
+/**
+ * @param {string} email
+ */
+async function isEmailUsed(email) {
+  try {
+    await UserGetter.promises.ensureUniqueEmailAddress(email)
+    return false
+  } catch (err) {
+    if (err instanceof Errors.EmailExistsError) {
+      return true
+    }
+    throw err
+  }
+}
+
+async function consumeCsvFile(trackProgress) {
+  console.time('re_add_deleted_emails')
+
+  const csvContent = await fs.readFile(CSV_FILENAME, 'utf8')
+  const rows = await parseAsync(csvContent)
+  rows.shift() // Remove header row
+
+  const emailsByUserId = {}
+  for (const [userId, email] of rows) {
+    if (!EmailHelper.parseEmail(email)) {
+      throw new Error(`invalid email ${email}`)
+    }
+
+    if (!emailsByUserId[userId]) {
+      emailsByUserId[userId] = []
+    }
+    emailsByUserId[userId].push(email)
+  }
+
+  const userIds = Object.keys(emailsByUserId)
+
+  const counts = {
+    /** @type {string[]} */
+    processedUsers: [],
+    /** @type {string[]} */
+    userNotFound: [],
+    /** @type {string[]} */
+    emailsInUse: [],
+    /** @type {string[]} */
+    alreadyOk: [],
+    /** @type {string[]} */
+    primary: [],
+    /** @type {string[]} */
+    secondary: [],
+    /** @type {string[]} */
+    addedEmails: [],
+  }
+
+  console.log('Total emails in the CSV:', rows.length)
+  console.log('Total users in the CSV:', userIds.length)
+
+  for (const userId of userIds) {
+    const candidateEmails = emailsByUserId[userId]
+
+    const user = await db.users.findOne(
+      { _id: new ObjectId(userId) },
+      { readPreference: READ_PREFERENCE_SECONDARY }
+    )
+    if (!user) {
+      counts.userNotFound.push(userId)
+      continue
+    }
+
+    for (const email of candidateEmails) {
+      if (user.emails.some(item => item.email === email)) {
+        counts.alreadyOk.push(email)
+        continue
+      }
+
+      const isUsed = await isEmailUsed(email)
+      const isOwnPrimary = user.email === email
+
+      if (isUsed && !isOwnPrimary) {
+        counts.emailsInUse.push(email)
+        continue
+      }
+
+      if (user.email === email) counts.primary.push(email)
+      else counts.secondary.push(email)
+
+      if (commit) {
+        const auditLog = {
+          initiatorId: null,
+          ipAddress: null,
+          info: {
+            script: true,
+            note: 'fix wrongly removed unconfirmed secondary email',
+          },
+        }
+        if (isOwnPrimary) {
+          // can't use addEmailAddress for primary email because ensureUniqueEmailAddress will throw
+          // using an override instead
+          await addEmailAddressOverride(user._id, email, {}, auditLog)
+        } else {
+          await UserUpdater.promises.addEmailAddress(
+            user._id,
+            email,
+            {},
+            auditLog
+          )
+        }
+        await UserUpdater.promises.confirmEmail(user._id, email)
+      }
+      counts.addedEmails.push(email)
+    }
+
+    counts.processedUsers.push(userId)
+    trackProgress(
+      `Processed users: ${counts.processedUsers.length}/${userIds.length}`
+    )
+  }
+
+  console.log()
+  if (!commit) {
+    console.log('Dry-run, use --commit to apply changes')
+    console.log('This would be the result:')
+    console.log()
+  }
+
+  console.log('Total emails in the CSV:', rows.length)
+  console.log('Total users in the CSV:', userIds.length)
+  console.log()
+  console.log('Users not found:', counts.userNotFound.length)
+  console.log('Users not found:', JSON.stringify(counts.userNotFound))
+  console.log()
+  console.log('Already OK:', counts.alreadyOk.length)
+  console.log('Already OK:', JSON.stringify(counts.alreadyOk))
+  console.log()
+  console.log('Already in use:', counts.emailsInUse.length)
+  console.log('Already in use:', JSON.stringify(counts.emailsInUse))
+  console.log()
+  console.log('Primary:', counts.primary.length)
+  console.log('Primary:', JSON.stringify(counts.primary))
+  console.log()
+  console.log('Secondary:', counts.secondary.length)
+  console.log('Secondary:', JSON.stringify(counts.secondary))
+  console.log()
+  console.log('Added emails:', counts.addedEmails.length)
+  console.log('Added emails:', JSON.stringify(counts.addedEmails))
+  console.log()
+  console.log()
+  console.timeEnd('re_add_deleted_emails')
+  console.log()
+}
+
+async function main(trackProgress) {
+  if (help) {
+    return usage()
+  }
+  await consumeCsvFile(trackProgress)
+}
+
+try {
+  await scriptRunner(main)
+  process.exit(0)
+} catch (error) {
+  console.error(error)
+  process.exit(1)
+}
+
+async function addEmailAddressOverride(
+  userId,
+  newEmail,
+  affiliationOptions,
+  auditLog
+) {
+  AsyncLocalStorage.removeItem('userFullEmails')
+  newEmail = EmailHelper.parseEmail(newEmail)
+  if (!newEmail) {
+    throw new Error('invalid email')
+  }
+
+  // Bypass ensureUniqueEmailAddress when re-adding primary emails
+  // await UserGetter.promises.ensureUniqueEmailAddress(newEmail)
+
+  AnalyticsManager.recordEventForUserInBackground(
+    userId,
+    'secondary-email-added'
+  )
+
+  await UserAuditLogHandler.promises.addEntry(
+    userId,
+    'add-email',
+    auditLog.initiatorId,
+    auditLog.ipAddress,
+    {
+      ...auditLog.info,
+      newSecondaryEmail: newEmail,
+    }
+  )
+
+  try {
+    await InstitutionsAPI.promises.addAffiliation(
+      userId,
+      newEmail,
+      affiliationOptions
+    )
+  } catch (error) {
+    throw OError.tag(error, 'problem adding affiliation while adding email')
+  }
+
+  const createdAt = new Date()
+  let res
+  try {
+    const reversedHostname = newEmail.split('@')[1].split('').reverse().join('')
+    const update = {
+      $push: {
+        emails: { email: newEmail, createdAt, reversedHostname },
+      },
+    }
+    res = await UserUpdater.promises.updateUser(
+      { _id: userId, 'emails.email': { $ne: newEmail } },
+      update
+    )
+  } catch (error) {
+    throw OError.tag(error, 'problem updating users emails')
+  }
+
+  if (res.matchedCount !== 1) {
+    return
+  }
+
+  try {
+    await EmailChangeHelper.registerEmailCreation(userId, newEmail, {
+      // @ts-expect-error - This is copied from UserUpdater.mjs
+      createdAt: new Date(),
+      emailCreatedAt: createdAt,
+    })
+  } catch (error) {
+    logger.warn(
+      { error, userId, newEmail },
+      'Error registering email creation with analytics'
+    )
+  }
+}

services/web/scripts/remove_unconfirmed_emails.mjs

@@ -146,15 +146,15 @@ async function consumeCsvFile() {
   console.log('Total users in the CSV:', userIds.length)
 
   for (const userId of userIds) {
-    const emailsToRemove = emailsByUserId[userId]
+    const emailsToRemoveCandidates = emailsByUserId[userId]
 
     const user = await db.users.findOne({ _id: new ObjectId(userId) })
     if (!user) {
-      skippedEmail.userNotFound += emailsToRemove.length
+      skippedEmail.userNotFound += emailsToRemoveCandidates.length
       continue
     }
 
-    const emailsToRemoveNow = emailsToRemove.filter(email => {
+    const emailsToRemove = emailsToRemoveCandidates.filter(email => {
       const currentEmail = user.emails.find(e => e.email === email)
       if (!currentEmail) {
         skippedEmail.nowRemoved++
@@ -171,9 +171,9 @@ async function consumeCsvFile() {
       return true
     })
 
-    removedEmailsCount += emailsToRemoveNow.length
+    removedEmailsCount += emailsToRemove.length
 
-    if (commit && emailsToRemoveNow.length > 0) {
+    if (commit && emailsToRemove.length > 0) {
       for (const email of emailsToRemove) {
         await UserAuditLogHandler.promises.addEntry(
           userId,

services/web/test/acceptance/src/ReAddDeletedEmailsTests.mjs

@@ -0,0 +1,275 @@
+import { promisify } from 'node:util'
+import { exec } from 'node:child_process'
+import { expect } from 'chai'
+import { filterOutput } from './helpers/settings.mjs'
+import { db, ObjectId } from '../../../app/src/infrastructure/mongodb.js'
+import fs from 'node:fs/promises'
+
+const CSV_FILENAME = '/tmp/re_add_deleted_emails.csv'
+
+async function runScript(commit) {
+  const result = await promisify(exec)(
+    ['node', 'scripts/re_add_deleted_emails.mjs', commit && '--commit']
+      .filter(Boolean)
+      .join(' ')
+  )
+
+  return {
+    ...result,
+    stdout: result.stdout.split('\n').filter(filterOutput),
+  }
+}
+
+/**
+ * @param {[string, string[]][]} userEmails
+ */
+const createUsers = async userEmails =>
+  Promise.all(
+    userEmails.map(async ([email, emails]) => {
+      const _id = new ObjectId()
+      await db.users.insertOne({
+        _id,
+        email,
+        emails: emails.map(email => ({ email })),
+        features: {},
+      })
+      return _id
+    })
+  )
+
+async function generateCsv(users) {
+  const text = 'User ID,Email'
+  const userRows = users.map(user => {
+    return `${user._id.toString()},${user.email}`
+  })
+  await fs.writeFile(CSV_FILENAME, [text, ...userRows].join('\n'))
+}
+
+describe('scripts/re_add_deleted_emails', function () {
+  let userIds
+
+  afterEach(async function () {
+    try {
+      await fs.unlink(CSV_FILENAME)
+    } catch (err) {
+      // Ignore errors if file doesn't exist
+    }
+  })
+
+  describe('when user IDs dont match', function () {
+    beforeEach(async function () {
+      userIds = await createUsers([['mismatch1@xmpl.com', []]])
+      await generateCsv([{ _id: new ObjectId(), email: 'mismatch2@xmpl.com' }])
+    })
+
+    it('doesnt add new emails', async function () {
+      const { stdout } = await runScript(true)
+      expect(stdout).to.include('Total emails in the CSV: 1')
+      expect(stdout).to.include('Total users in the CSV: 1')
+      expect(stdout).to.include('Users not found: 1')
+      expect(stdout).to.include('Added emails: 0')
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(0)
+    })
+  })
+
+  describe('when email address is invalid', function () {
+    beforeEach(async function () {
+      userIds = await createUsers([['user@xmpl.com', []]])
+      await generateCsv([{ _id: userIds[0], email: 'inv@lid@xmpl.com' }])
+    })
+
+    it('throws', async function () {
+      await expect(runScript(true)).to.eventually.be.rejectedWith(
+        'invalid email'
+      )
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(0)
+    })
+    it('throws even without --commit', async function () {
+      await expect(runScript(false)).to.eventually.be.rejectedWith(
+        'invalid email'
+      )
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(0)
+    })
+  })
+
+  describe('when new email is used by another user', function () {
+    beforeEach(async function () {
+      userIds = await createUsers([
+        ['user1@xmpl.com', []],
+        ['user2@xmpl.com', ['new-email@xmpl.com']],
+      ])
+      await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }])
+    })
+
+    it('doesnt add new emails', async function () {
+      const { stdout } = await runScript(true)
+      expect(stdout).to.include('Total emails in the CSV: 1')
+      expect(stdout).to.include('Total users in the CSV: 1')
+      expect(stdout).to.include('Users not found: 0')
+      expect(stdout).to.include('Already in use: ["new-email@xmpl.com"]')
+      expect(stdout).to.include('Added emails: 0')
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(0)
+    })
+  })
+
+  describe('when the user has 0 email in the array', function () {
+    beforeEach(async function () {
+      userIds = await createUsers([['user@xmpl.com', []]])
+    })
+
+    it('adds the primary email to the user', async function () {
+      await generateCsv([{ _id: userIds[0], email: 'user@xmpl.com' }])
+      const { stdout } = await runScript(true)
+      expect(stdout).to.include('Total emails in the CSV: 1')
+      expect(stdout).to.include('Total users in the CSV: 1')
+      expect(stdout).to.include('Users not found: 0')
+      expect(stdout).to.include('Added emails: 1')
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(1)
+      expect(updatedUser.emails[0].email).to.equal('user@xmpl.com')
+      expect(updatedUser.emails[0].reversedHostname).to.equal('moc.lpmx')
+      expect(updatedUser.emails[0].confirmedAt).to.be.an.instanceof(Date)
+      expect(updatedUser.emails[0].createdAt).to.be.an.instanceof(Date)
+      expect(updatedUser.emails[0].reconfirmedAt).to.be.an.instanceof(Date)
+      const auditLogs = await db.userAuditLogEntries
+        .find({ userId: userIds[0] })
+        .toArray()
+      expect(auditLogs).to.have.length(1)
+      expect(auditLogs[0].operation).to.equal('add-email')
+      expect(auditLogs[0].info).to.deep.include({
+        script: true,
+        note: 'fix wrongly removed unconfirmed secondary email',
+        newSecondaryEmail: 'user@xmpl.com',
+      })
+    })
+
+    it('adds the secondary email to the user', async function () {
+      await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }])
+      const { stdout } = await runScript(true)
+      expect(stdout).to.include('Total emails in the CSV: 1')
+      expect(stdout).to.include('Total users in the CSV: 1')
+      expect(stdout).to.include('Users not found: 0')
+      expect(stdout).to.include('Added emails: 1')
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(1)
+      expect(updatedUser.emails[0].email).to.equal('new-email@xmpl.com')
+      expect(updatedUser.emails[0].reversedHostname).to.equal('moc.lpmx')
+      expect(updatedUser.emails[0].confirmedAt).to.be.an.instanceof(Date)
+      expect(updatedUser.emails[0].createdAt).to.be.an.instanceof(Date)
+      expect(updatedUser.emails[0].reconfirmedAt).to.be.an.instanceof(Date)
+      const auditLogs = await db.userAuditLogEntries
+        .find({ userId: userIds[0] })
+        .toArray()
+      expect(auditLogs).to.have.length(1)
+      expect(auditLogs[0].operation).to.equal('add-email')
+      expect(auditLogs[0].info).to.deep.include({
+        script: true,
+        note: 'fix wrongly removed unconfirmed secondary email',
+        newSecondaryEmail: 'new-email@xmpl.com',
+      })
+    })
+
+    it('doesnt add new emails without --commit', async function () {
+      await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }])
+      const { stdout } = await runScript(false)
+      expect(stdout).to.include('Dry-run, use --commit to apply changes')
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(0)
+    })
+  })
+  describe('when the user has several emails in the array', function () {
+    beforeEach(async function () {
+      userIds = await createUsers([
+        [
+          'user@xmpl.com',
+          ['email1@xmpl.com', 'email2@xmpl.com', 'email3@xmpl.com'],
+        ],
+      ])
+      await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }])
+    })
+
+    it('adds the email to the user', async function () {
+      const { stdout } = await runScript(true)
+
+      expect(stdout).to.include('Total emails in the CSV: 1')
+      expect(stdout).to.include('Total users in the CSV: 1')
+      expect(stdout).to.include('Users not found: 0')
+      expect(stdout).to.include('Added emails: 1')
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(4)
+
+      expect(updatedUser.emails[3].email).to.equal('new-email@xmpl.com')
+      expect(updatedUser.emails[3].reversedHostname).to.equal('moc.lpmx')
+      expect(updatedUser.emails[3].confirmedAt).to.be.an.instanceof(Date)
+      expect(updatedUser.emails[3].createdAt).to.be.an.instanceof(Date)
+      expect(updatedUser.emails[3].reconfirmedAt).to.be.an.instanceof(Date)
+    })
+
+    it('doesnt add new emails without --commit', async function () {
+      const { stdout } = await runScript(false)
+      expect(stdout).to.include('Dry-run, use --commit to apply changes')
+      const updatedUser = await db.users.findOne({ _id: userIds[0] })
+      expect(updatedUser.emails).to.have.length(3)
+    })
+  })
+
+  describe('all of the above', function () {
+    beforeEach(async function () {
+      userIds = await createUsers([
+        ['user0@xmpl.com', []],
+        ['user1@xmpl.com', []],
+        ['user2@xmpl.com', ['a@xmpl.com', 'b@xmpl.com', 'c@xmpl.com']],
+        ['user3@xmpl.com', ['d@xmpl.com', 'e@xmpl.com', 'f@xmpl.com']],
+        ['user4@xmpl.com', ['x@xmpl.com', 'y@xmpl.com', 'z@xmpl.com']],
+        ['user5@xmpl.com', ['u@xmpl.com', 'v@xmpl.com', 'w@xmpl.com']],
+      ])
+      await generateCsv([
+        { _id: userIds[1], email: 'new1@xmpl.com' },
+        { _id: userIds[1], email: 'user1@xmpl.com' },
+        { _id: userIds[1], email: 'new2@xmpl.com' },
+        { _id: userIds[2], email: 'new3@xmpl.com' },
+        { _id: userIds[2], email: 'new4@xmpl.com' },
+        { _id: userIds[2], email: 'user2@xmpl.com' },
+        { _id: userIds[2], email: 'user3@xmpl.com' },
+        { _id: userIds[3], email: 'd@xmpl.com' },
+        { _id: userIds[5], email: 'a@xmpl.com' },
+        { _id: new ObjectId(), email: 'a@xmpl.com' },
+      ])
+    })
+
+    it('updates users', async function () {
+      const { stdout } = await runScript(true)
+      expect(stdout).to.include('Total emails in the CSV: 10')
+      expect(stdout).to.include('Total users in the CSV: 5')
+      expect(stdout).to.include('Users not found: 1')
+      expect(stdout).to.include(
+        'Already in use: ["user3@xmpl.com","a@xmpl.com"]'
+      )
+      expect(stdout).to.include('Already OK: ["d@xmpl.com"]')
+      expect(stdout).to.include('Primary: ["user1@xmpl.com","user2@xmpl.com"]')
+      expect(stdout).to.include(
+        'Secondary: ["new1@xmpl.com","new2@xmpl.com","new3@xmpl.com","new4@xmpl.com"]'
+      )
+      expect(stdout).to.include(
+        'Added emails: ["new1@xmpl.com","user1@xmpl.com","new2@xmpl.com","new3@xmpl.com","new4@xmpl.com","user2@xmpl.com"]'
+      )
+
+      const user0 = await db.users.findOne({ _id: userIds[0] })
+      const user1 = await db.users.findOne({ _id: userIds[1] })
+      const user2 = await db.users.findOne({ _id: userIds[2] })
+      const user3 = await db.users.findOne({ _id: userIds[3] })
+      const user4 = await db.users.findOne({ _id: userIds[4] })
+      const user5 = await db.users.findOne({ _id: userIds[5] })
+      expect(user0.emails).to.have.length(0)
+      expect(user1.emails).to.have.length(3) // new1, user1, new2
+      expect(user2.emails).to.have.length(6) // a, b, c, new3, new4, user2
+      expect(user3.emails).to.have.length(3) // d, e, f
+      expect(user4.emails).to.have.length(3) // x, y, z
+      expect(user5.emails).to.have.length(3) // u, v, w
+    })
+  })
+})