mirrors/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-12-05 01:10:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

move migrations to shared location (#28306)

Browse Source 
* fix: correct typedef for Document in helpers.mjs

* add move-migrations codemod

* update migration paths to use shared migrations directory

* move migrations to shared location

* fix: update Dockerfile and docker-compose.ci.yml to include migrations directory

* feat: add migrations tool to workspaces in package.json

* [monorepo] Fix order of docker ignore rules

* [web] remove unused docker ignore file

* [monorepo] replace old references to migrations folder

* [server-ce] copy migrations from new place

* [migrations] Inline web scripts

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] move three web scripts over

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] add missing collection

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] remove lodash dependency

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] avoid mongodb-legacy dependency

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [monorepo] run migrations from tools/migrations

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] simplify migration for adding gitBridge feature to users

* [monorepo] run migrations from tests in all the services

* [migrations] add Jenkins pipeline for linting/formatting

* [monorepo] fixup running web migrations everywhere

* [monorepo] trigger Jenkins builds on changes to mongo migrations

* [migrations] add Jenkins pipeline for linting/formatting

* [monorepo] build scripts: update devDependencies before deps scanning

* [monorepo] build scripts: formerly depend on tools/migrations

* [monorepo] run eslint on .mjs files

* [migrations] enable more eslint rules and fix all the errors

* [rake] fix migrations:list task

---------

Co-authored-by: Jakob Ackermann <jakob.ackermann@overleaf.com>
GitOrigin-RevId: 14cf69cc1b9405bbc75adbb9a000e555500e0614
This commit is contained in:
Brian Gough
2025-10-16 08:53:25 +01:00
committed by Copybot
parent fbea855690
commit 729e0f5ac9
249 changed files with 802 additions and 984 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
libraries/access-token-encryptor/package.json
View File

@@ -5,8 +5,8 @@
"main": "index.js",
"scripts": {
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
"test:ci": "npm run test:unit",

4
libraries/fetch-utils/package.json
View File

@@ -5,8 +5,8 @@
"main": "index.js",
"scripts": {
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
"test:ci": "npm run test:unit",

4
libraries/logger/package.json
View File

@@ -13,8 +13,8 @@
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
"test:ci": "npm run test:unit",
"test:unit": "mocha --exit test/**/*.{js,cjs}",
"types:check": "tsc --noEmit"

4
libraries/metrics/package.json
View File

@@ -31,8 +31,8 @@
"typescript": "^5.0.4"
},
"scripts": {
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
"test:unit": "mocha --exit test/**/*.{js,cjs}",
"test:acceptance": "mocha --recursive --exit --grep=$MOCHA_GREP test/acceptance",
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",

4
libraries/mongo-utils/package.json
View File

@@ -6,8 +6,8 @@
"scripts": {
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
"test:unit": "mocha --exit test/**/*.{js,cjs}",
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
"test:ci": "npm run test:unit",

4
libraries/o-error/package.json
View File

@@ -18,8 +18,8 @@
],
"scripts": {
"build": "npm run --silent test",
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",

4
libraries/object-persistor/package.json
View File

@@ -6,8 +6,8 @@
"scripts": {
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
"test:unit": "mocha --exit test/**/*.{js,cjs}",
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
"test:ci": "npm run test:unit",

4
libraries/overleaf-editor-core/package.json
View File

@@ -7,8 +7,8 @@
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
"test:ci": "npm run test:unit",
"test:unit": "mocha --exit test/**/*.{js,cjs}",
"types:check": "tsc --noEmit"

4
libraries/promise-utils/package.json
View File

@@ -6,8 +6,8 @@
"scripts": {
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
"test:unit": "mocha --exit test/**/*.{js,cjs}",
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
"test:ci": "npm run test:unit",

4
libraries/ranges-tracker/package.json
View File

@@ -9,8 +9,8 @@
"author": "Overleaf (https://www.overleaf.com)",
"private": true,
"scripts": {
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",

4
libraries/redis-wrapper/package.json
View File

@@ -13,8 +13,8 @@
"repository": "github:overleaf/redis-wrapper",
"license": "ISC",
"scripts": {
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",

4
libraries/settings/package.json
View File

@@ -5,8 +5,8 @@
"repository": "overleaf/settings-module",
"main": "index.js",
"scripts": {
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",

4
libraries/stream-utils/package.json
View File

@@ -6,8 +6,8 @@
"scripts": {
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
"test:unit": "mocha --exit test/**/*.{js,cjs}",
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
"test:ci": "npm run test:unit",

4
libraries/validation-tools/package.json
View File

@@ -13,8 +13,8 @@
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
"test:ci": "npm run test:unit",
"test:unit": "vitest --config vitest.config.ts",
"types:check": "tsc --noEmit"

31
package-lock.json generated
View File

@@ -33,7 +33,8 @@
"services/third-party-references",
"services/tpdsworker",
"services/web",
"tools/saas-e2e"
"tools/saas-e2e",
"tools/migrations"
],
"dependencies": {
"patch-package": "^8.0.0"
@@ -14867,6 +14868,10 @@
"resolved": "libraries/metrics",
"link": true
},
"node_modules/@overleaf/migrations": {
"resolved": "tools/migrations",
"link": true
},
"node_modules/@overleaf/mongo-utils": {
"resolved": "libraries/mongo-utils",
"link": true
@@ -15248,7 +15253,6 @@
"integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
"dev": true,
"optional": true,
"peer": true,
"engines": {
"node": ">=14"
}
@@ -51368,6 +51372,7 @@
"mongodb": "6.12.0"
},
"devDependencies": {
"@overleaf/migrations": "*",
"acorn": "^7.1.1",
"ajv": "^6.12.0",
"chai": "^4.3.6",
@@ -51531,6 +51536,7 @@
"underscore": "~1.13.1"
},
"devDependencies": {
"@overleaf/migrations": "*",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"esmock": "^2.6.3",
@@ -51605,6 +51611,7 @@
},
"devDependencies": {
"@google-cloud/storage": "^6.10.1",
"@overleaf/migrations": "*",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"mocha": "^11.1.0",
@@ -51682,6 +51689,7 @@
"requestretry": "^7.1.0"
},
"devDependencies": {
"@overleaf/migrations": "*",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"cluster-key-slot": "^1.0.5",
@@ -51815,6 +51823,7 @@
},
"devDependencies": {
"@overleaf/fetch-utils": "*",
"@overleaf/migrations": "*",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"mocha": "^11.1.0",
@@ -51949,6 +51958,7 @@
"utf-8-validate": "^5.0.4"
},
"devDependencies": {
"@overleaf/migrations": "*",
"benny": "^3.7.1",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
@@ -52689,6 +52699,7 @@
"zod-validation-error": "^4.0.1"
},
"devDependencies": {
"@overleaf/migrations": "*",
"@types/method-override": "^3.0.0",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
@@ -52724,6 +52735,7 @@
"request": "^2.88.2"
},
"devDependencies": {
"@overleaf/migrations": "*",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"mocha": "^11.1.0",
@@ -52896,6 +52908,7 @@
"request": "^2.88.2"
},
"devDependencies": {
"@overleaf/migrations": "*",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"mocha": "^11.1.0",
@@ -53150,6 +53163,7 @@
"@overleaf/fetch-utils": "*",
"@overleaf/logger": "*",
"@overleaf/metrics": "*",
"@overleaf/migrations": "*",
"@overleaf/mongo-utils": "*",
"@overleaf/o-error": "*",
"@overleaf/object-persistor": "*",
@@ -53185,7 +53199,6 @@
"csurf": "^1.11.0",
"csv": "^6.2.5",
"dateformat": "1.0.4-1.2.3",
"east": "^2.0.2",
"ejs": "^3.1.10",
"email-addresses": "^5.0.0",
"eventsource-parser": "^1.1.2",
@@ -54545,6 +54558,18 @@
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
},
"tools/migrations": {
"name": "@overleaf/migrations",
"dependencies": {
"@overleaf/logger": "*",
"@overleaf/mongo-utils": "*",
"@overleaf/o-error": "*",
"@overleaf/promise-utils": "*",
"@overleaf/settings": "*",
"east": "2.0.3",
"mongodb": "6.12.0"
}
},
"tools/saas-e2e": {
"name": "@overleaf/saas-e2e",
"devDependencies": {

3
package.json
View File

@@ -76,6 +76,7 @@
"services/third-party-references",
"services/tpdsworker",
"services/web",
"tools/saas-e2e"
"tools/saas-e2e",
"tools/migrations"
]
}

1
server-ce/Dockerfile
View File

@@ -14,6 +14,7 @@ ADD server-ce/services.js /overleaf/services.js
ADD package.json package-lock.json /overleaf/
ADD libraries/ /overleaf/libraries/
ADD services/ /overleaf/services/
ADD tools/migrations/ /overleaf/tools/migrations/
# Add npm patches
# -----------------------

2
server-ce/init_scripts/900_run_web_migrations.sh
View File

@@ -8,6 +8,6 @@ else
fi
echo "Running migrations for $environment"
cd /overleaf/services/web
cd /overleaf/tools/migrations
/sbin/setuser www-data npm run migrations -- migrate -t "$environment"
echo "Finished migrations"

3
server-ce/test/.jenkinsIncludeFile
View File

@@ -68,7 +68,6 @@ services/web/config/settings.defaults.js
services/web/config/settings.overrides.server-pro.js
services/web/frontend/**
services/web/locales/**
services/web/migrations/**
services/web/modules/*/*
services/web/modules/*/app/**
services/web/modules/*/frontend/**
@@ -78,6 +77,8 @@ services/web/public/**
services/web/types/**
services/web/webpack-plugins/**
tools/migrations/**
.dockerignore
.eslint*
.pretter*

2
services/chat/.jenkinsIncludeFile
View File

@@ -6,8 +6,10 @@ libraries/logger/**
libraries/metrics/**
libraries/mongo-utils/**
libraries/o-error/**
libraries/promise-utils/**
libraries/settings/**
package-lock.json
package.json
patches/**
services/chat/**
tools/migrations/**

5
services/chat/Dockerfile
View File

@@ -18,9 +18,12 @@ COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
COPY libraries/mongo-utils/package.json /overleaf/libraries/mongo-utils/package.json
COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY services/chat/package.json /overleaf/services/chat/package.json
COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
COPY patches/ /overleaf/patches/
COPY tools/migrations/ /overleaf/tools/migrations/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
@@ -28,8 +31,10 @@ COPY libraries/logger/ /overleaf/libraries/logger/
COPY libraries/metrics/ /overleaf/libraries/metrics/
COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/
COPY libraries/o-error/ /overleaf/libraries/o-error/
COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY services/chat/ /overleaf/services/chat/
COPY tools/migrations/ /overleaf/tools/migrations/
FROM app
USER node

2
services/chat/Makefile
View File

@@ -19,8 +19,10 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
$(MONOREPO)/libraries/metrics/package.json \
$(MONOREPO)/libraries/mongo-utils/package.json \
$(MONOREPO)/libraries/o-error/package.json \
$(MONOREPO)/libraries/promise-utils/package.json \
$(MONOREPO)/libraries/settings/package.json \
$(MONOREPO)/services/chat/package.json \
$(MONOREPO)/tools/migrations/package.json \
$(MONOREPO)/patches/* \
| sha256sum | cut -d '-' -f1)

3
services/chat/docker-compose.yml
View File

@@ -31,10 +31,11 @@ services:
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
- ../../tools/migrations:/overleaf/tools/migrations
working_dir: /overleaf/services/chat
environment:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
LOG_LEVEL: ${LOG_LEVEL:-}

1
services/chat/package.json
View File

@@ -29,6 +29,7 @@
"mongodb": "6.12.0"
},
"devDependencies": {
"@overleaf/migrations": "*",
"acorn": "^7.1.1",
"ajv": "^6.12.0",
"chai": "^4.3.6",

1
services/chat/test/acceptance/js/helpers/ChatApp.js
View File

@@ -1,5 +1,6 @@
import { createServer } from '../../../../app/js/server.js'
import { promisify } from 'node:util'
import './MongoHelper.js'
export { db } from '../../../../app/js/mongodb.js'

10
services/chat/test/acceptance/js/helpers/MongoHelper.js Normal file
View File

@@ -0,0 +1,10 @@
import { exec } from 'node:child_process'
import { promisify } from 'node:util'
before('run migrations', async function () {
this.timeout(60_000)
await promisify(exec)(
`cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
)
})

2
services/clsi/docker-compose.yml
View File

@@ -34,7 +34,7 @@ services:
working_dir: /overleaf/services/clsi
environment:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
LOG_LEVEL: ${LOG_LEVEL:-}

2
services/contacts/.jenkinsIncludeFile
View File

@@ -6,8 +6,10 @@ libraries/logger/**
libraries/metrics/**
libraries/mongo-utils/**
libraries/o-error/**
libraries/promise-utils/**
libraries/settings/**
package-lock.json
package.json
patches/**
services/contacts/**
tools/migrations/**

5
services/contacts/Dockerfile
View File

@@ -18,9 +18,12 @@ COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
COPY libraries/mongo-utils/package.json /overleaf/libraries/mongo-utils/package.json
COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY services/contacts/package.json /overleaf/services/contacts/package.json
COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
COPY patches/ /overleaf/patches/
COPY tools/migrations/ /overleaf/tools/migrations/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
@@ -28,8 +31,10 @@ COPY libraries/logger/ /overleaf/libraries/logger/
COPY libraries/metrics/ /overleaf/libraries/metrics/
COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/
COPY libraries/o-error/ /overleaf/libraries/o-error/
COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY services/contacts/ /overleaf/services/contacts/
COPY tools/migrations/ /overleaf/tools/migrations/
FROM app
USER node

2
services/contacts/Makefile
View File

@@ -19,8 +19,10 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
$(MONOREPO)/libraries/metrics/package.json \
$(MONOREPO)/libraries/mongo-utils/package.json \
$(MONOREPO)/libraries/o-error/package.json \
$(MONOREPO)/libraries/promise-utils/package.json \
$(MONOREPO)/libraries/settings/package.json \
$(MONOREPO)/services/contacts/package.json \
$(MONOREPO)/tools/migrations/package.json \
$(MONOREPO)/patches/* \
| sha256sum | cut -d '-' -f1)

3
services/contacts/docker-compose.yml
View File

@@ -31,10 +31,11 @@ services:
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
- ../../tools/migrations:/overleaf/tools/migrations
working_dir: /overleaf/services/contacts
environment:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
LOG_LEVEL: ${LOG_LEVEL:-}

1
services/contacts/package.json
View File

@@ -31,6 +31,7 @@
"underscore": "~1.13.1"
},
"devDependencies": {
"@overleaf/migrations": "*",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"esmock": "^2.6.3",

10
services/contacts/test/acceptance/js/MongoHelper.js Normal file
View File

@@ -0,0 +1,10 @@
import { exec } from 'node:child_process'
import { promisify } from 'node:util'
before('run migrations', async function () {
this.timeout(60_000)
await promisify(exec)(
`cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
)
})

1
services/docstore/.jenkinsIncludeFile
View File

@@ -14,3 +14,4 @@ package-lock.json
package.json
patches/**
services/docstore/**
tools/migrations/**

3
services/docstore/Dockerfile
View File

@@ -23,7 +23,9 @@ COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/pack
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
COPY services/docstore/package.json /overleaf/services/docstore/package.json
COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
COPY patches/ /overleaf/patches/
COPY tools/migrations/ /overleaf/tools/migrations/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
@@ -36,6 +38,7 @@ COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
COPY services/docstore/ /overleaf/services/docstore/
COPY tools/migrations/ /overleaf/tools/migrations/
FROM app
USER node

1
services/docstore/Makefile
View File

@@ -24,6 +24,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
$(MONOREPO)/libraries/settings/package.json \
$(MONOREPO)/libraries/stream-utils/package.json \
$(MONOREPO)/services/docstore/package.json \
$(MONOREPO)/tools/migrations/package.json \
$(MONOREPO)/patches/* \
| sha256sum | cut -d '-' -f1)

3
services/docstore/docker-compose.yml
View File

@@ -31,10 +31,11 @@ services:
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
- ../../tools/migrations:/overleaf/tools/migrations
working_dir: /overleaf/services/docstore
environment:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
POSTGRES_HOST: postgres
GCS_API_ENDPOINT: http://gcs:9090
GCS_PROJECT_ID: fake

1
services/docstore/package.json
View File

@@ -37,6 +37,7 @@
},
"devDependencies": {
"@google-cloud/storage": "^6.10.1",
"@overleaf/migrations": "*",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"mocha": "^11.1.0",

1
services/docstore/test/acceptance/js/helpers/DocstoreApp.js
View File

@@ -1,5 +1,6 @@
const app = require('../../../../app')
const Settings = require('@overleaf/settings')
require('./MongoHelper')
function startApp() {
return new Promise((resolve, reject) => {

10
services/docstore/test/acceptance/js/helpers/MongoHelper.js Normal file
View File

@@ -0,0 +1,10 @@
import { exec } from 'node:child_process'
import { promisify } from 'node:util'
before('run migrations', async function () {
this.timeout(60_000)
await promisify(exec)(
`cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
)
})

1
services/document-updater/.jenkinsIncludeFile
View File

@@ -15,3 +15,4 @@ package-lock.json
package.json
patches/**
services/document-updater/**
tools/migrations/**

3
services/document-updater/Dockerfile
View File

@@ -24,7 +24,9 @@ COPY libraries/ranges-tracker/package.json /overleaf/libraries/ranges-tracker/pa
COPY libraries/redis-wrapper/package.json /overleaf/libraries/redis-wrapper/package.json
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY services/document-updater/package.json /overleaf/services/document-updater/package.json
COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
COPY patches/ /overleaf/patches/
COPY tools/migrations/ /overleaf/tools/migrations/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
@@ -38,6 +40,7 @@ COPY libraries/ranges-tracker/ /overleaf/libraries/ranges-tracker/
COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY services/document-updater/ /overleaf/services/document-updater/
COPY tools/migrations/ /overleaf/tools/migrations/
FROM app
USER node

1
services/document-updater/Makefile
View File

@@ -25,6 +25,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
$(MONOREPO)/libraries/redis-wrapper/package.json \
$(MONOREPO)/libraries/settings/package.json \
$(MONOREPO)/services/document-updater/package.json \
$(MONOREPO)/tools/migrations/package.json \
$(MONOREPO)/patches/* \
| sha256sum | cut -d '-' -f1)

3
services/document-updater/docker-compose.yml
View File

@@ -37,6 +37,7 @@ services:
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
- ../../tools/migrations:/overleaf/tools/migrations
working_dir: /overleaf/services/document-updater
environment:
ELASTIC_SEARCH_DSN: es:9200
@@ -44,7 +45,7 @@ services:
HISTORY_REDIS_HOST: redis_test
QUEUES_REDIS_HOST: redis_test
ANALYTICS_QUEUES_REDIS_HOST: redis_test
MONGO_HOST: mongo
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
LOG_LEVEL: ${LOG_LEVEL:-}

1
services/document-updater/package.json
View File

@@ -41,6 +41,7 @@
"requestretry": "^7.1.0"
},
"devDependencies": {
"@overleaf/migrations": "*",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"cluster-key-slot": "^1.0.5",

1
services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js
View File

@@ -1,4 +1,5 @@
const app = require('../../../../app')
require('./MongoHelper')
function startApp() {
return new Promise((resolve, reject) => {

10
services/document-updater/test/acceptance/js/helpers/MongoHelper.js Normal file
View File

@@ -0,0 +1,10 @@
const { exec } = require('node:child_process')
const { promisify } = require('node:util')
before('run migrations', async function () {
this.timeout(60_000)
await promisify(exec)(
`cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
)
})

2
services/filestore/docker-compose.yml
View File

@@ -38,7 +38,7 @@ services:
working_dir: /overleaf/services/filestore
environment:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
POSTGRES_HOST: postgres
AWS_S3_ENDPOINT: https://minio:9000
AWS_S3_PATH_STYLE: 'true'

1
services/history-v1/.jenkinsIncludeFile
View File

@@ -16,3 +16,4 @@ package-lock.json
package.json
patches/**
services/history-v1/**
tools/migrations/**

3
services/history-v1/Dockerfile
View File

@@ -30,7 +30,9 @@ COPY libraries/redis-wrapper/package.json /overleaf/libraries/redis-wrapper/pack
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
COPY services/history-v1/package.json /overleaf/services/history-v1/package.json
COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
COPY patches/ /overleaf/patches/
COPY tools/migrations/ /overleaf/tools/migrations/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
@@ -45,6 +47,7 @@ COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
COPY services/history-v1/ /overleaf/services/history-v1/
COPY tools/migrations/ /overleaf/tools/migrations/
FROM app
USER node

1
services/history-v1/Makefile
View File

@@ -26,6 +26,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
$(MONOREPO)/libraries/settings/package.json \
$(MONOREPO)/libraries/stream-utils/package.json \
$(MONOREPO)/services/history-v1/package.json \
$(MONOREPO)/tools/migrations/package.json \
$(MONOREPO)/patches/* \
| sha256sum | cut -d '-' -f1)

3
services/history-v1/docker-compose.yml
View File

@@ -47,6 +47,7 @@ services:
- ../../libraries:/overleaf/libraries
- minio-certs:/certs
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
- ../../tools/migrations:/overleaf/tools/migrations
working_dir: /overleaf/services/history-v1
environment:
ELASTIC_SEARCH_DSN: es:9200
@@ -54,7 +55,7 @@ services:
HISTORY_REDIS_HOST: redis_test
QUEUES_REDIS_HOST: redis_test
ANALYTICS_QUEUES_REDIS_HOST: redis_test
MONGO_HOST: mongo
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
POSTGRES_HOST: postgres
AWS_S3_ENDPOINT: https://minio:9000
AWS_S3_PATH_STYLE: 'true'

1
services/history-v1/package.json
View File

@@ -47,6 +47,7 @@
"utf-8-validate": "^5.0.4"
},
"devDependencies": {
"@overleaf/migrations": "*",
"benny": "^3.7.1",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",

20
services/history-v1/test/setup.js
View File

@@ -2,7 +2,9 @@ const chai = require('chai')
const chaiAsPromised = require('chai-as-promised')
const config = require('config')
const fetch = require('node-fetch')
const { knex, mongodb, redis } = require('../storage')
const { knex, redis } = require('../storage')
const { exec } = require('node:child_process')
const { promisify } = require('node:util')
// ensure every ObjectId has the id string as a property for correct comparisons
require('mongodb').ObjectId.cacheHexString = true
@@ -17,19 +19,9 @@ async function setupPostgresDatabase() {
async function setupMongoDatabase() {
this.timeout(60_000)
await mongodb.db.collection('projectHistoryChunks').createIndexes([
{
key: { projectId: 1, startVersion: 1 },
name: 'projectId_1_startVersion_1',
partialFilterExpression: { state: { $in: ['active', 'closed'] } },
unique: true,
},
{
key: { state: 1 },
name: 'state_1',
partialFilterExpression: { state: 'deleted' },
},
])
await promisify(exec)(
`cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
)
}
async function createGcsBuckets() {

1
services/notifications/.jenkinsIncludeFile
View File

@@ -13,3 +13,4 @@ package-lock.json
package.json
patches/**
services/notifications/**
tools/migrations/**

3
services/notifications/Dockerfile
View File

@@ -22,7 +22,9 @@ COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/pack
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY libraries/validation-tools/package.json /overleaf/libraries/validation-tools/package.json
COPY services/notifications/package.json /overleaf/services/notifications/package.json
COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
COPY patches/ /overleaf/patches/
COPY tools/migrations/ /overleaf/tools/migrations/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
@@ -34,6 +36,7 @@ COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/
COPY services/notifications/ /overleaf/services/notifications/
COPY tools/migrations/ /overleaf/tools/migrations/
FROM app
USER node

1
services/notifications/Makefile
View File

@@ -23,6 +23,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
$(MONOREPO)/libraries/settings/package.json \
$(MONOREPO)/libraries/validation-tools/package.json \
$(MONOREPO)/services/notifications/package.json \
$(MONOREPO)/tools/migrations/package.json \
$(MONOREPO)/patches/* \
| sha256sum | cut -d '-' -f1)

3
services/notifications/docker-compose.yml
View File

@@ -32,11 +32,12 @@ services:
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
- ../../tools/migrations:/overleaf/tools/migrations
- ../../tsconfig.backend.json:/overleaf/tsconfig.backend.json
working_dir: /overleaf/services/notifications
environment:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
LOG_LEVEL: ${LOG_LEVEL:-}

1
services/notifications/package.json
View File

@@ -38,6 +38,7 @@
"zod-validation-error": "^4.0.1"
},
"devDependencies": {
"@overleaf/migrations": "*",
"@types/method-override": "^3.0.0",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",

1
services/notifications/test/acceptance/js/HealthCheck.test.ts
View File

@@ -2,6 +2,7 @@ import { beforeAll, describe, it, expect } from 'vitest'
import { fetchStringWithResponse } from '@overleaf/fetch-utils'
import app from '../../../app.ts'
import logger from '@overleaf/logger'
import './MongoHelper.ts'
let runAppPromise: Promise<void> | null = null

9
services/notifications/test/acceptance/js/MongoHelper.ts Normal file
View File

@@ -0,0 +1,9 @@
import { beforeAll } from 'vitest'
import { exec } from 'node:child_process'
import { promisify } from 'node:util'
beforeAll(async function () {
await promisify(exec)(
`cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
)
}, 60_000)

1
services/project-history/.jenkinsIncludeFile
View File

@@ -15,3 +15,4 @@ package-lock.json
package.json
patches/**
services/project-history/**
tools/migrations/**

3
services/project-history/Dockerfile
View File

@@ -24,7 +24,9 @@ COPY libraries/redis-wrapper/package.json /overleaf/libraries/redis-wrapper/pack
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
COPY services/project-history/package.json /overleaf/services/project-history/package.json
COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
COPY patches/ /overleaf/patches/
COPY tools/migrations/ /overleaf/tools/migrations/
RUN cd /overleaf && npm ci --quiet
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
@@ -38,6 +40,7 @@ COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/
COPY libraries/settings/ /overleaf/libraries/settings/
COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
COPY services/project-history/ /overleaf/services/project-history/
COPY tools/migrations/ /overleaf/tools/migrations/
FROM app
USER node

1
services/project-history/Makefile
View File

@@ -25,6 +25,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
$(MONOREPO)/libraries/settings/package.json \
$(MONOREPO)/libraries/stream-utils/package.json \
$(MONOREPO)/services/project-history/package.json \
$(MONOREPO)/tools/migrations/package.json \
$(MONOREPO)/patches/* \
| sha256sum | cut -d '-' -f1)

3
services/project-history/docker-compose.yml
View File

@@ -37,6 +37,7 @@ services:
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
- ../../tools/migrations:/overleaf/tools/migrations
working_dir: /overleaf/services/project-history
environment:
ELASTIC_SEARCH_DSN: es:9200
@@ -44,7 +45,7 @@ services:
HISTORY_REDIS_HOST: redis_test
QUEUES_REDIS_HOST: redis_test
ANALYTICS_QUEUES_REDIS_HOST: redis_test
MONGO_HOST: mongo
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
LOG_LEVEL: ${LOG_LEVEL:-}

1
services/project-history/package.json
View File

@@ -42,6 +42,7 @@
"request": "^2.88.2"
},
"devDependencies": {
"@overleaf/migrations": "*",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"mocha": "^11.1.0",

10
services/project-history/test/acceptance/js/helpers/MongoHelper.js Normal file
View File

@@ -0,0 +1,10 @@
import { exec } from 'node:child_process'
import { promisify } from 'node:util'
before('run migrations', async function () {
this.timeout(60_000)
await promisify(exec)(
`cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
)
})

1
services/project-history/test/acceptance/js/helpers/ProjectHistoryApp.js
View File

@@ -1,5 +1,6 @@
import { app } from '../../../../app/js/server.js'
import { mongoClient } from '../../../../app/js/mongodb.js'
import './MongoHelper.js'
let running = false
let initPromise = null

2
services/real-time/docker-compose.yml
View File

@@ -39,7 +39,7 @@ services:
HISTORY_REDIS_HOST: redis_test
QUEUES_REDIS_HOST: redis_test
ANALYTICS_QUEUES_REDIS_HOST: redis_test
MONGO_HOST: mongo
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
LOG_LEVEL: ${LOG_LEVEL:-}

1
services/web/.dockerignore
View File

@@ -1 +0,0 @@
.gitignore

2
services/web/.eslintrc.js
View File

@@ -286,10 +286,8 @@ module.exports = {
'scripts/add_subscription_members_csv.mjs',
'scripts/analytics/helpers/GoogleBigQueryHelper.mjs',
'scripts/attach_dangling_comments_to_doc.mjs',
'scripts/back_fill_doc_rev.mjs',
'scripts/backfill_mixpanel_user_properties.mjs',
'scripts/backfill_project_image_name.mjs',
'scripts/backfill_project_invites_token_hmac.mjs',
'scripts/backfill_user_properties.mjs',
'scripts/backfill_users_sso_attribute.mjs',
'scripts/bench_bcrypt.mjs',

1
services/web/.jenkinsIncludeFile
View File

@@ -20,3 +20,4 @@ package-lock.json
package.json
patches/**
services/web/**
tools/migrations/**

3
services/web/Dockerfile
View File

@@ -39,6 +39,7 @@ COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
COPY libraries/validation-tools/package.json /overleaf/libraries/validation-tools/package.json
COPY services/web/package.json /overleaf/services/web/package.json
COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
COPY patches/ /overleaf/patches/
RUN cd /overleaf && NODE_ENV=production npm ci --quiet
@@ -72,6 +73,7 @@ COPY libraries/settings/ /overleaf/libraries/settings/
COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/
COPY services/web/ /overleaf/services/web/
COPY tools/migrations/ /overleaf/tools/migrations/
# Build the latex parser
RUN cd /overleaf/services/web && npm run 'lezer-latex:generate'
@@ -108,6 +110,7 @@ COPY libraries/settings/ /overleaf/libraries/settings/
COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/
COPY services/web/ /overleaf/services/web/
COPY tools/migrations/ /overleaf/tools/migrations/
# Omit Server Pro/CE specific scripts from SaaS image
RUN rm /overleaf/services/web/modules/server-ce-scripts -rf

2
services/web/Makefile
View File

@@ -146,7 +146,7 @@ test_unit_module: mongo_migrations_for_tests
$(MAKE) modules/$(MODULE_NAME)/test_unit
mongo_migrations_for_tests:
$(DOCKER_COMPOSE) run --rm test_unit npm run migrations -- migrate -t saas
$(DOCKER_COMPOSE) run --rm --workdir /overleaf/tools/migrations test_unit npm run migrations -- migrate -t saas
#
# Frontend tests

2
services/web/buildscript.txt
View File

@@ -1,5 +1,5 @@
web
--dependencies=
--dependencies=mongo
--docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker
--env-add=
--env-pass-through=

2
services/web/docker-compose.yml
View File

@@ -10,6 +10,7 @@ services:
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
- ../../tools/migrations:/overleaf/tools/migrations
working_dir: /overleaf/services/web
env_file: docker-compose.common.env
environment:
@@ -37,6 +38,7 @@ services:
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
- ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
- ../../tools/migrations:/overleaf/tools/migrations
user: node
working_dir: /overleaf/services/web
env_file: docker-compose.common.env

28
services/web/migrations/20190730093801_script_example.mjs
View File

@@ -1,28 +0,0 @@
/* eslint-disable no-unused-vars */
/*
* Example migration for a script:
*
* This migration demonstrates how to run a script. In this case, the example
* script will print "hello world" if there are no users in the users collection
* or "hello <name>", when User.findOne() finds something.
*/
import runScript from '../scripts/example/script_for_migration.mjs'
const tags = []
const migrate = async client => {
const { db } = client
await runScript()
}
const rollback = async client => {
const { db } = client
}
export default {
tags,
migrate,
rollback,
}

19
services/web/migrations/20210727150530_ce_sp_backfill_deleted_docs.mjs
View File

@@ -1,19 +0,0 @@
import runScript from '../scripts/back_fill_doc_name_for_deleted_docs.mjs'
const tags = ['server-ce', 'server-pro', 'saas']
const migrate = async client => {
const options = {
performCleanup: true,
letUserDoubleCheckInputsFor: 10,
}
await runScript(options)
}
const rollback = async client => {}
export default {
tags,
migrate,
rollback,
}

20
services/web/migrations/20220913125500_migrate_auditLog_to_collections.mjs
View File

@@ -1,20 +0,0 @@
import runScript from '../scripts/migrate_audit_logs.mjs'
const tags = ['server-ce', 'server-pro', 'saas']
const migrate = async () => {
const options = {
letUserDoubleCheckInputsFor: 10,
writeConcurrency: 5,
dryRun: false,
}
await runScript(options)
}
const rollback = async () => {}
export default {
tags,
migrate,
rollback,
}

19
services/web/migrations/20230817081910_back_fill_gitBridge_feature_server_pro.mjs
View File

@@ -1,19 +0,0 @@
const tags = ['server-ce', 'server-pro']
const migrate = async () => {
// Run-time import as SaaS does not ship with the server-ce-scripts module
const { default: runScript } = await import(
'../modules/server-ce-scripts/scripts/upgrade-user-features.mjs'
)
await runScript(false, {
gitBridge: 1,
})
}
const rollback = async () => {}
export default {
tags,
migrate,
rollback,
}

30
services/web/migrations/20240524135408_add_token_hmac_project_invite_tokens.mjs
View File

@@ -1,30 +0,0 @@
/* eslint-disable no-unused-vars */
import Helpers from './lib/helpers.mjs'
import runScript from '../scripts/backfill_project_invites_token_hmac.mjs'
const tags = ['server-ce', 'server-pro', 'saas']
const index = {
key: {
tokenHmac: 1,
},
name: 'tokenHmac_1',
}
const migrate = async client => {
const { db } = client
await Helpers.addIndexesToCollection(db.projectInvites, [index])
await runScript(false)
}
const rollback = async client => {
const { db } = client
await Helpers.dropIndexesFromCollection(db.projectInvites, [index])
}
export default {
tags,
migrate,
rollback,
}

7
services/web/package.json
View File

@@ -40,7 +40,6 @@
"type-check": "tsc --noEmit",
"type-check:backend": "tsc -p tsconfig.backend.json --noEmit",
"extract-translations": "i18next-scanner",
"migrations": "MONGO_SOCKET_TIMEOUT=0 east --es-modules",
"convert-themes": "node frontend/js/features/source-editor/themes/scripts/convert.js",
"cypress:open-ct": "OVERLEAF_CONFIG=$PWD/config/settings.webpack.js cypress open --component",
"cypress:run-ct": "OVERLEAF_CONFIG=$PWD/config/settings.webpack.js cypress run --component --browser chrome",
@@ -91,6 +90,7 @@
"@overleaf/fetch-utils": "*",
"@overleaf/logger": "*",
"@overleaf/metrics": "*",
"@overleaf/migrations": "*",
"@overleaf/mongo-utils": "*",
"@overleaf/o-error": "*",
"@overleaf/object-persistor": "*",
@@ -126,7 +126,6 @@
"csurf": "^1.11.0",
"csv": "^6.2.5",
"dateformat": "1.0.4-1.2.3",
"east": "^2.0.2",
"ejs": "^3.1.10",
"email-addresses": "^5.0.0",
"eventsource-parser": "^1.1.2",
@@ -194,6 +193,7 @@
"zod-validation-error": "^4.0.1"
},
"devDependencies": {
"5to6-codemod": "^1.8.0",
"@babel/cli": "^7.27.0",
"@babel/core": "^7.26.10",
"@babel/plugin-proposal-decorators": "^7.27.0",
@@ -278,7 +278,6 @@
"@writefull/core": "^1.27.27",
"@writefull/ui": "^1.27.27",
"@writefull/utils": "^1.27.27",
"5to6-codemod": "^1.8.0",
"abort-controller": "^3.0.0",
"acorn": "^7.1.1",
"acorn-walk": "^7.1.1",
@@ -354,8 +353,8 @@
"mock-fs": "^5.1.2",
"nock": "^13.5.6",
"nvd3": "^1.8.6",
"path-browserify": "^1.0.1",
"p-reflect": "^3.1.0",
"path-browserify": "^1.0.1",
"pdfjs-dist": "5.1.91",
"pirates": "^4.0.1",
"postcss": "^8.4.31",

97
services/web/scripts/back_fill_doc_name_for_deleted_docs.mjs
View File

@@ -1,97 +0,0 @@
import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
import { promiseMapWithLimit, promisify } from '@overleaf/promise-utils'
import { db } from '../app/src/infrastructure/mongodb.js'
import { fileURLToPath } from 'node:url'
import _ from 'lodash'
import { scriptRunner } from './lib/ScriptRunner.mjs'
const sleep = promisify(setTimeout)
async function main(options, trackProgress) {
if (!options) {
options = {}
}
_.defaults(options, {
writeConcurrency: parseInt(process.env.WRITE_CONCURRENCY, 10) || 10,
performCleanup: process.argv.pop() === '--perform-cleanup',
letUserDoubleCheckInputsFor: parseInt(
process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR || 10 * 1000,
10
),
})
await letUserDoubleCheckInputs(options)
await batchedUpdate(
db.projects,
// array is not empty ~ array has one item
{ 'deletedDocs.0': { $exists: true } },
async projects => {
await processBatch(projects, options)
},
{ _id: 1, deletedDocs: 1 },
undefined,
{ trackProgress }
)
}
async function processBatch(projects, options) {
await promiseMapWithLimit(
options.writeConcurrency,
projects,
async project => {
await processProject(project, options)
}
)
}
async function processProject(project, options) {
for (const doc of project.deletedDocs) {
await backFillDoc(doc)
}
if (options.performCleanup) {
await cleanupProject(project)
}
}
async function backFillDoc(doc) {
const { name, deletedAt } = doc
await db.docs.updateOne({ _id: doc._id }, { $set: { name, deletedAt } })
}
async function cleanupProject(project) {
await db.projects.updateOne(
{ _id: project._id },
{ $set: { deletedDocs: [] } }
)
}
async function letUserDoubleCheckInputs(options) {
if (options.performCleanup) {
console.error('BACK FILLING AND PERFORMING CLEANUP')
} else {
console.error(
'BACK FILLING ONLY - You will need to rerun with --perform-cleanup'
)
}
console.error(
'Waiting for you to double check inputs for',
options.letUserDoubleCheckInputsFor,
'ms'
)
await sleep(options.letUserDoubleCheckInputsFor)
}
export default main
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await scriptRunner(
async trackProgress => await main(undefined, trackProgress)
)
process.exit(0)
} catch (error) {
console.error({ error })
process.exit(1)
}
}

79
services/web/scripts/backfill_project_invites_token_hmac.mjs
View File

@@ -1,79 +0,0 @@
import { db } from '../app/src/infrastructure/mongodb.js'
import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
import minimist from 'minimist'
import CollaboratorsInviteHelper from '../app/src/Features/Collaborators/CollaboratorsInviteHelper.js'
import { fileURLToPath } from 'node:url'
const argv = minimist(process.argv.slice(2), {
boolean: ['dry-run', 'help'],
default: {
'dry-run': true,
},
})
const DRY_RUN = argv['dry-run']
async function addTokenHmacField(DRY_RUN) {
const query = { tokenHmac: { $exists: false } }
await batchedUpdate(
db.projectInvites,
query,
async invites => {
for (const invite of invites) {
console.log(
`=> Missing "tokenHmac" token in invitation: ${invite._id.toString()}`
)
if (DRY_RUN) {
console.log(
`=> DRY RUN - would add "tokenHmac" token to invitation ${invite._id.toString()}`
)
continue
}
const tokenHmac = CollaboratorsInviteHelper.hashInviteToken(
invite.token
)
await db.projectInvites.updateOne(
{ _id: invite._id },
{ $set: { tokenHmac } }
)
console.log(
`=> Added "tokenHmac" token to invitation ${invite._id.toString()}`
)
}
},
{ token: 1 }
)
}
async function main(DRY_RUN) {
await addTokenHmacField(DRY_RUN)
}
export default main
if (fileURLToPath(import.meta.url) === process.argv[1]) {
if (argv.help || argv._.length > 1) {
console.error(`Usage: node scripts/backfill_project_invites_token_hmac.mjs
Adds a "tokenHmac" field (which is a hashed version of the token) to each project invite record.
Options:
--dry-run finds invitations without HMAC token but does not do any updates
`)
process.exit(1)
}
try {
await main(DRY_RUN)
console.error('Done')
process.exit(0)
} catch (error) {
console.error(error)
process.exit(1)
}
}

2
services/web/scripts/mark_migration.mjs
View File

@@ -1,4 +1,4 @@
import Adapter from '../migrations/lib/adapter.mjs'
import Adapter from '../../../tools/migrations/lib/adapter.mjs'
import { promises as fs } from 'node:fs'
import { join, dirname } from 'node:path'
import { fileURLToPath } from 'node:url'

169
services/web/scripts/migrate_audit_logs.mjs
View File

@@ -1,169 +0,0 @@
import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
import { promiseMapWithLimit, promisify } from '@overleaf/promise-utils'
import { db, ObjectId } from '../app/src/infrastructure/mongodb.js'
import _ from 'lodash'
import { fileURLToPath } from 'node:url'
import { scriptRunner } from './lib/ScriptRunner.mjs'
const sleep = promisify(setTimeout)
async function main(options, trackProgress) {
if (!options) {
options = {}
}
_.defaults(options, {
dryRun: process.env.DRY_RUN !== 'false',
projectId: process.env.PROJECT_ID,
userId: process.env.USER_ID,
skipUsersMigration: process.env.SKIP_USERS_MIGRATION === 'true',
writeConcurrency: parseInt(process.env.WRITE_CONCURRENCY, 10) || 10,
letUserDoubleCheckInputsFor: parseInt(
process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR || 10 * 1000,
10
),
})
await letUserDoubleCheckInputs(options)
if (options.projectId) {
console.log('migrating projectId=' + options.projectId)
const project = await db.projects.findOne(
{ _id: new ObjectId(options.projectId) },
{ _id: 1, auditLog: 1 }
)
if (!project || !project.auditLog) {
console.error('unable to process project', project)
return
}
await processProjectsBatch([project], options)
} else if (options.userId) {
console.log('migrating userId=' + options.userId)
const user = await db.users.findOne(
{ _id: new ObjectId(options.userId) },
{ _id: 1, auditLog: 1 }
)
if (!user || !user.auditLog) {
console.error('unable to process user', user)
return
}
await processUsersBatch([user], options)
} else {
if (!options.skipUsersMigration) {
await batchedUpdate(
db.users,
{ auditLog: { $exists: true } },
async users => {
await processUsersBatch(users, options)
},
{ _id: 1, auditLog: 1 },
undefined,
{ trackProgress }
)
}
// most projects are processed after its owner has been processed, but only those
// users with an existing `auditLog` have been taken into consideration, leaving
// some projects orphan. This batched update processes all remaining projects.
await batchedUpdate(
db.projects,
{ auditLog: { $exists: true } },
async projects => {
await processProjectsBatch(projects, options)
},
{ _id: 1, auditLog: 1 },
undefined,
{ trackProgress }
)
}
}
async function processUsersBatch(users, options) {
if (!users || users.length <= 0) {
return
}
const entries = users
.map(user => user.auditLog.map(log => ({ ...log, userId: user._id })))
.flat()
if (!options.dryRun && entries?.length > 0) {
await db.userAuditLogEntries.insertMany(entries)
}
if (!options.dryRun) {
const userIds = users.map(user => user._id)
await db.users.updateMany(
{ _id: { $in: userIds } },
{ $unset: { auditLog: 1 } }
)
}
await promiseMapWithLimit(options.writeConcurrency, users, async user => {
const projects = await db.projects
.find(
{ owner_ref: user._id, auditLog: { $exists: true } },
{ _id: 1, auditLog: 1 }
)
.toArray()
await processProjectsBatch(projects, options)
})
}
async function processProjectsBatch(projects, options) {
if (!projects || projects.length <= 0) {
return
}
const entries = projects
.map(project =>
project.auditLog.map(log => ({ ...log, projectId: project._id }))
)
.flat()
if (!options.dryRun && entries?.length > 0) {
await db.projectAuditLogEntries.insertMany(entries)
}
if (!options.dryRun) {
const projectIds = projects.map(project => project._id)
await db.projects.updateMany(
{ _id: { $in: projectIds } },
{ $unset: { auditLog: 1 } }
)
}
}
async function letUserDoubleCheckInputs(options) {
const allOptions = {
...options,
// batchedUpdate() environment variables
BATCH_DESCENDING: process.env.BATCH_DESCENDING,
BATCH_SIZE: process.env.BATCH_SIZE,
VERBOSE_LOGGING: process.env.VERBOSE_LOGGING,
BATCH_LAST_ID: process.env.BATCH_LAST_ID,
BATCH_RANGE_END: process.env.BATCH_RANGE_END,
SKIP_USERS_MIGRATION: process.env.SKIP_USERS_MIGRATION,
}
console.error('Options:', JSON.stringify(allOptions, null, 2))
console.error(
'Waiting for you to double check inputs for',
options.letUserDoubleCheckInputsFor,
'ms'
)
await sleep(options.letUserDoubleCheckInputsFor)
}
export default main
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await scriptRunner(
async trackProgress => await main(undefined, trackProgress)
)
console.log('Done.')
process.exit(0)
} catch (error) {
console.error({ error })
process.exit(1)
}
}

20
services/web/test/acceptance/src/BackFillDocNameForDeletedDocsTests.mjs
View File

@@ -63,14 +63,11 @@ describe('BackFillDocNameForDeletedDocs', function () {
await setDeletedDocs(projectId2, deletedDocs2)
})
async function runScript(args = []) {
async function runScript() {
let result
try {
result = await promisify(exec)(
['LET_USER_DOUBLE_CHECK_INPUTS_FOR=1']
.concat(['node', 'scripts/back_fill_doc_name_for_deleted_docs.mjs'])
.concat(args)
.join(' ')
'cd ../../tools/migrations && east migrate -t saas --force 20210727150530_ce_sp_backfill_deleted_docs'
)
} catch (error) {
// dump details like exit code, stdErr and stdOut
@@ -95,20 +92,9 @@ describe('BackFillDocNameForDeletedDocs', function () {
})
}
describe('back fill only', function () {
beforeEach('run script', runScript)
checkDocsBackFilled()
it('should leave the deletedDocs as is', async function () {
expect(await getDeletedDocs(projectId1)).to.deep.equal(deletedDocs1)
expect(await getDeletedDocs(projectId2)).to.deep.equal(deletedDocs2)
})
})
describe('back fill and cleanup', function () {
beforeEach('run script with cleanup flag', async function () {
await runScript(['--perform-cleanup'])
await runScript()
})
checkDocsBackFilled()

26
services/web/test/acceptance/src/BackFillDocRevTests.mjs
View File

@@ -17,16 +17,11 @@ describe('BackFillDocRevTests', function () {
])
})
async function runScript(dryRun) {
async function runScript() {
let result
try {
result = await promisify(exec)(
[
'VERBOSE_LOGGING=true',
'node',
'scripts/back_fill_doc_rev.mjs',
dryRun,
].join(' ')
'cd ../../tools/migrations && VERBOSE_LOGGING=true east migrate -t server-ce --force 20230315170739_back_fill_doc_rev'
)
} catch (error) {
// dump details like exit code, stdErr and stdOut
@@ -47,24 +42,9 @@ describe('BackFillDocRevTests', function () {
)
}
describe('dry-run=true', function () {
beforeEach('run script', async function () {
await runScript('--dry-run=true')
})
it('should not back fill the rev', async function () {
const docs = await db.docs.find({}, { $sort: { _id: 1 } }).toArray()
expect(docs).to.deep.equal([
{ _id: docId1, deleted: true },
{ _id: docId2 },
{ _id: docId3, rev: 42 },
])
})
})
describe('dry-run=false', function () {
beforeEach('run script', async function () {
await runScript('--dry-run=false')
await runScript()
})
it('should back fill the rev', async function () {

66
services/web/test/acceptance/src/BackFillDummyDocMetaTests.mjs
View File

@@ -7,8 +7,6 @@ import { db, ObjectId } from '../../../app/src/infrastructure/mongodb.js'
const DUMMY_NAME = 'unknown.tex'
const DUMMY_TIME = new Date('2021-04-12T00:00:00.000Z')
const ONE_DAY_IN_S = 60 * 60 * 24
const BATCH_SIZE = 3
function getObjectIdFromDate(date) {
const seconds = new Date(date).getTime() / 1000
@@ -18,7 +16,6 @@ function getObjectIdFromDate(date) {
describe('BackFillDummyDocMeta', function () {
let docIds
let projectIds
let stopAtSeconds
beforeEach('create docs', async function () {
docIds = []
docIds[0] = getObjectIdFromDate('2021-04-01T00:00:00.000Z')
@@ -48,11 +45,15 @@ describe('BackFillDummyDocMeta', function () {
// two docs in the same project
projectIds[10] = projectIds[9]
projectIds[11] = projectIds[4]
stopAtSeconds = new Date('2021-04-17T00:00:00.000Z').getTime() / 1000
})
const now = new Date()
beforeEach('insert doc stubs into docs collection', async function () {
// don't look here, just drop duplicates from the list of projectIds :)
await db.projects.insertMany(
Array.from(new Set(projectIds.map(id => id.toString()))).map(_id => ({
_id: new ObjectId(_id),
}))
)
await db.docs.insertMany([
// incomplete, without deletedDocs context
{ _id: docIds[0], project_id: projectIds[0], deleted: true },
@@ -104,33 +105,18 @@ describe('BackFillDummyDocMeta', function () {
])
})
let options
async function runScript(dryRun) {
options = {
BATCH_SIZE,
CACHE_SIZE: 100,
DRY_RUN: dryRun,
FIRST_PROJECT_ID: projectIds[0].toString(),
INCREMENT_BY_S: ONE_DAY_IN_S,
STOP_AT_S: stopAtSeconds,
// start right away
LET_USER_DOUBLE_CHECK_INPUTS_FOR: 1,
}
async function runScript() {
let result
try {
result = await promisify(exec)(
Object.entries(options)
.map(([key, value]) => `${key}=${value}`)
.concat(['node', 'scripts/back_fill_dummy_doc_meta.mjs'])
.join(' ')
'cd ../../tools/migrations && east migrate -t saas --force 20210728115327_ce_sp_backfill_dummy_doc_meta'
)
} catch (error) {
// dump details like exit code, stdErr and stdOut
logger.error({ error }, 'script failed')
throw error
}
let { stderr: stdErr, stdout: stdOut } = result
stdErr = stdErr.split('\n')
let { stdout: stdOut } = result
stdOut = stdOut.split('\n').filter(filterOutput)
expect(stdOut.filter(filterOutput)).to.include.members([
@@ -145,42 +131,8 @@ describe('BackFillDummyDocMeta', function () {
`Orphaned deleted doc ${docIds[9]} (no deletedProjects entry)`,
`Orphaned deleted doc ${docIds[10]} (no deletedProjects entry)`,
])
expect(stdErr.filter(filterOutput)).to.include.members([
`Processed 9 until ${projectIds[9]}`,
'Done.',
])
}
describe('DRY_RUN=true', function () {
beforeEach('run script', async function () {
await runScript(true)
})
it('should leave docs as is', async function () {
const docs = await db.docs.find({}).toArray()
expect(docs).to.deep.equal([
{ _id: docIds[0], project_id: projectIds[0], deleted: true },
{ _id: docIds[1], project_id: projectIds[1], deleted: true },
{ _id: docIds[2], project_id: projectIds[2], deleted: true },
{ _id: docIds[3], project_id: projectIds[3], deleted: true },
{ _id: docIds[4], project_id: projectIds[4], deleted: true },
{
_id: docIds[5],
project_id: projectIds[5],
deleted: true,
name: 'foo.tex',
deletedAt: now,
},
{ _id: docIds[6], project_id: projectIds[6] },
{ _id: docIds[7], project_id: projectIds[7], deleted: true },
{ _id: docIds[8], project_id: projectIds[8], deleted: true },
{ _id: docIds[9], project_id: projectIds[9], deleted: true },
{ _id: docIds[10], project_id: projectIds[10], deleted: true },
{ _id: docIds[11], project_id: projectIds[11], deleted: true },
])
})
})
describe('DRY_RUN=false', function () {
beforeEach('run script', async function () {
await runScript(false)

2
services/web/test/acceptance/src/ConvertArchivedState.mjs
View File

@@ -120,7 +120,7 @@ describe('ConvertArchivedState', function () {
beforeEach(function (done) {
exec(
'east migrate --tag server-ce --force 20221111111111_ce_sp_convert_archived_state',
'cd ../../tools/migrations && east migrate --tag server-ce --force 20221111111111_ce_sp_convert_archived_state',
error => {
if (error) {
return done(error)

2
services/web/test/acceptance/src/ConvertEmailConfirmedAtToDates.js
View File

@@ -28,7 +28,7 @@ describe('ConvertEmailConfirmedAtToDates', function () {
beforeEach('run migration', function (done) {
exec(
'east migrate -t saas --force 20210726083523_convert_confirmedAt_strings_to_dates',
'cd ../../tools/migrations && east migrate -t saas --force 20210726083523_convert_confirmedAt_strings_to_dates',
done
)
})

2
services/web/test/acceptance/src/ConvertSplitTestAssignedAtToDates.js
View File

@@ -65,7 +65,7 @@ describe('ConvertSplitTestAssignedAtToDates', function () {
beforeEach('run migration', function (done) {
exec(
'east migrate -t saas --force 20210726083523_convert_split_tests_assigned_at_strings_to_dates',
'cd ../../tools/migrations && east migrate -t saas --force 20210726083523_convert_split_tests_assigned_at_strings_to_dates',
done
)
})

114
services/web/test/acceptance/src/RemoveDeletedUsersFromTokenAccessRefsTests.mjs
View File

@@ -22,9 +22,8 @@ describe('RemoveDeletedUsersFromTokenAccessRefsTests', function () {
const projectId3 = new ObjectId('65d726e807c024c8db43be24')
const projectId4 = new ObjectId('65d726e807c024c8db43be25')
let insertedProjects
beforeEach('insert projects', async function () {
insertedProjects = await db.projects.insertMany([
await db.projects.insertMany([
{
_id: projectId1,
tokenAccessReadAndWrite_refs: [userId1],
@@ -47,17 +46,11 @@ describe('RemoveDeletedUsersFromTokenAccessRefsTests', function () {
let stdOut
const runScript = async (dryRun, projectsList) => {
const runScript = async () => {
let result
try {
result = await promisify(exec)(
[
'VERBOSE_LOGGING=true',
'node',
'scripts/remove_deleted_users_from_token_access_refs.mjs',
dryRun,
projectsList,
].join(' ')
'cd ../../tools/migrations && east migrate -t saas --force 20240220130452_remove_deleted_users_from_token_access_refs'
)
} catch (error) {
// dump details like exit code, stdErr and stdOut
@@ -70,91 +63,6 @@ describe('RemoveDeletedUsersFromTokenAccessRefsTests', function () {
expect(stdOut).to.match(new RegExp(`User ids count: ${insertedUsersCount}`))
}
describe('dry-run=true', function () {
beforeEach('run script', async function () {
await runScript('--dry-run=true')
expect(stdOut).to.match(/doing dry run/i)
})
it('should show current user id to be removed', function () {
expect(stdOut).to.match(
new RegExp(
`Found deleted user id: ${userId2.toString()} in project: ${projectId2.toString()}`
)
)
expect(stdOut).to.match(
new RegExp(
`DRY RUN - would remove deleted ${userId2.toString()} from all projects \\(found in project ${projectId2.toString()}\\)`
)
)
expect(stdOut).to.match(
new RegExp(
`Found deleted user id: ${userId3.toString()} in project: ${projectId3.toString()}`
)
)
expect(stdOut).to.match(
new RegExp(
`DRY RUN - would remove deleted ${userId3.toString()} from all projects \\(found in project ${projectId3.toString()}\\)`
)
)
})
it('should show projects with non-existing token access fields', function () {
expect(stdOut)
.to.match(
new RegExp(
`DRY RUN - would fix non-existing token access fields in project ${projectId3.toString()}`
)
)
.and.match(
new RegExp(
`DRY RUN - would fix non-existing token access fields in project ${projectId4.toString()}`
)
)
})
it('should show the user ids (and their count) to be deleted', function () {
expect(stdOut).to.match(
new RegExp(
`DRY RUN - would delete user ids \\(2\\)\\n${userId2.toString()}\\n${userId3.toString()}`
)
)
})
it('should show the project ids (and their count) that needs fixing', function () {
expect(stdOut).to.match(
new RegExp(
`Projects with deleted user ids \\(2\\)\\n${projectId2.toString()}\\n${projectId3.toString()}`
)
)
})
it('should not fix the token access fields of projects', async function () {
const projects = await db.projects
.find({}, { $sort: { _id: 1 } })
.toArray()
expect(projects).to.deep.equal([
{
_id: projectId1,
tokenAccessReadAndWrite_refs: [userId1],
tokenAccessReadOnly_refs: [],
},
{
_id: projectId2,
tokenAccessReadAndWrite_refs: [userId2],
tokenAccessReadOnly_refs: [],
},
{
_id: projectId3,
tokenAccessReadAndWrite_refs: [userId3],
},
{
_id: projectId4,
},
])
})
})
describe('dry-run=false', function () {
beforeEach('run script', async function () {
await runScript('--dry-run=false')
@@ -237,20 +145,4 @@ describe('RemoveDeletedUsersFromTokenAccessRefsTests', function () {
])
})
})
describe('projects=projectId2', function () {
beforeEach('run script', async function () {
const projectId2 = insertedProjects.insertedIds[1]
await runScript('--dry-run=false', `--projects=${projectId2.toString()}`)
})
it('should fix only the projects provided', async function () {
const [project1, project2, project3] = await db.projects
.find({}, { $sort: { _id: 1 } })
.toArray()
expect(project1.tokenAccessReadAndWrite_refs.length).to.be.gt(0)
expect(project2.tokenAccessReadAndWrite_refs.length).to.eq(0) // deleted user removed
expect(project3.tokenAccessReadAndWrite_refs.length).to.be.gt(0)
})
})
})

24
services/web/test/acceptance/src/helpers/MongoHelper.mjs
View File

@@ -1,10 +1,11 @@
import { execFile } from 'node:child_process'
import { exec } from 'node:child_process'
import {
connectionPromise,
cleanupTestDatabase,
dropTestDatabase,
} from '../../../../app/src/infrastructure/mongodb.js'
import Settings from '@overleaf/settings'
import { promisify } from 'node:util'
const DEFAULT_ENV = 'saas'
@@ -17,21 +18,12 @@ export default {
before('drop test database', dropTestDatabase)
}
before('run migrations', function (done) {
const args = [
'run',
'migrations',
'--',
'migrate',
'-t',
Settings.env || DEFAULT_ENV,
]
execFile('npm', args, (error, stdout, stderr) => {
if (error) {
throw error
}
done()
})
before('run migrations', async function () {
this.timeout(60_000)
await promisify(exec)(
`cd ../../tools/migrations && npm run migrations -- migrate -t ${Settings.env || DEFAULT_ENV}`
)
})
afterEach('purge mongo data', cleanupTestDatabase)

3
services/web/.eastrc → tools/migrations/.eastrc
View File

@@ -1,5 +1,6 @@
{
"adapter": "./migrations/lib/adapter.mjs",
"adapter": "./lib/adapter.mjs",
"dir": ".",
"migrationNumberFormat": "dateTime",
"migrationExtension": "mjs"
}

6
tools/migrations/.jenkinsIncludeFile Normal file
View File

@@ -0,0 +1,6 @@
tools/migrations/**
package.json
package-lock.json
Makefile

0
services/web/migrations/20190720165251_create_migrations.mjs → tools/migrations/20190720165251_create_migrations.mjs
View File

0
services/web/migrations/20190912145001_create_contacts_indexes.mjs → tools/migrations/20190912145001_create_contacts_indexes.mjs
View File

0
services/web/migrations/20190912145002_create_deletedProjects_indexes.mjs → tools/migrations/20190912145002_create_deletedProjects_indexes.mjs
View File

0
services/web/migrations/20190912145003_create_deletedSubscriptions_indexes.mjs → tools/migrations/20190912145003_create_deletedSubscriptions_indexes.mjs
View File

Some files were not shown because too many files have changed in this diff Show More