mirrors/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-12-05 01:10:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #28845 from overleaf/td-async-await-doc-updater-client

Browse Source 
Convert DocUpdateClient in document-updater acceptance tests to async/await

GitOrigin-RevId: 8f2352119f8f1175c2703ed90dbbc483ed039e86
This commit is contained in:
Tim Down
2025-10-07 09:34:24 +01:00
committed by Copybot
parent 07234fd7d2
commit c104aa454e
21 changed files with 1359 additions and 3425 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
package-lock.json generated
View File

@@ -51098,6 +51098,7 @@
"services/document-updater": {
"name": "@overleaf/document-updater",
"dependencies": {
"@overleaf/fetch-utils": "*",
"@overleaf/logger": "*",
"@overleaf/metrics": "*",
"@overleaf/mongo-utils": "*",

6
services/document-updater/app/js/HttpController.js
View File

@@ -226,7 +226,11 @@ async function setDoc(req, res) {
)
timer.done()
logger.debug({ projectId, docId }, 'set doc via http')
res.json(result)
// If the document is unchanged and hasn't been updated, `result` will be
// undefined, which leads to an invalid JSON response, so we send an empty
// object instead.
res.json(result || {})
}
async function appendToDoc(req, res) {

4
services/document-updater/app/js/RedisManager.js
View File

@@ -542,7 +542,7 @@ const RedisManager = {
)
// return if no projects ready to be processed
if (!projectsReady || projectsReady.length === 0) {
return
return {}
}
// pop the oldest entry (get and remove in a multi)
const multi = rclient.multi()
@@ -552,7 +552,7 @@ const RedisManager = {
multi.zcard(keys.flushAndDeleteQueue()) // the total length of the queue (for metrics)
const reply = await multi.exec()
if (!reply || reply.length === 0) {
return
return {}
}
const [key, timestamp] = reply[0]
const queueLength = reply[2]

1
services/document-updater/package.json
View File

@@ -18,6 +18,7 @@
"types:check": "tsc --noEmit"
},
"dependencies": {
"@overleaf/fetch-utils": "*",
"@overleaf/logger": "*",
"@overleaf/metrics": "*",
"@overleaf/mongo-utils": "*",

938
services/document-updater/test/acceptance/js/ApplyingUpdatesToADocTests.js
View File

File diff suppressed because it is too large Load Diff

251
services/document-updater/test/acceptance/js/ApplyingUpdatesToProjectStructureTests.js
View File

@@ -1,4 +1,5 @@
const sinon = require('sinon')
const { setTimeout } = require('node:timers/promises')
const Settings = require('@overleaf/settings')
const rclientProjectHistory = require('@overleaf/redis-wrapper').createClient(
Settings.redis.project_history
@@ -10,6 +11,13 @@ const MockWebApi = require('./helpers/MockWebApi')
const DocUpdaterClient = require('./helpers/DocUpdaterClient')
const DocUpdaterApp = require('./helpers/DocUpdaterApp')
async function sendProjectUpdateAndWait(projectId, docId, update, version) {
await DocUpdaterClient.sendProjectUpdate(projectId, docId, update, version)
// It seems that we need to wait for a little while
await setTimeout(200)
}
describe("Applying updates to a project's structure", function () {
before(function () {
this.user_id = 'user-id-123'
@@ -17,7 +25,7 @@ describe("Applying updates to a project's structure", function () {
})
describe('renaming a file', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.fileUpdate = {
type: 'rename-file',
@@ -26,23 +34,13 @@ describe("Applying updates to a project's structure", function () {
newPathname: '/new-file-path',
}
this.updates = [this.fileUpdate]
DocUpdaterApp.ensureRunning(error => {
if (error) {
return done(error)
}
DocUpdaterClient.sendProjectUpdate(
this.project_id,
this.user_id,
this.updates,
this.version,
error => {
if (error) {
return done(error)
}
setTimeout(done, 200)
}
)
})
await DocUpdaterApp.ensureRunning()
await sendProjectUpdateAndWait(
this.project_id,
this.user_id,
this.updates,
this.version
)
})
it('should push the applied file renames to the project history api', function (done) {
@@ -70,7 +68,7 @@ describe("Applying updates to a project's structure", function () {
})
describe('deleting a file', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.fileUpdate = {
type: 'rename-file',
@@ -79,17 +77,11 @@ describe("Applying updates to a project's structure", function () {
newPathname: '',
}
this.updates = [this.fileUpdate]
DocUpdaterClient.sendProjectUpdate(
await sendProjectUpdateAndWait(
this.project_id,
this.user_id,
this.updates,
this.version,
error => {
if (error) {
return done(error)
}
setTimeout(done, 200)
}
this.version
)
})
@@ -129,19 +121,13 @@ describe("Applying updates to a project's structure", function () {
})
describe('when the document is not loaded', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
DocUpdaterClient.sendProjectUpdate(
await sendProjectUpdateAndWait(
this.project_id,
this.user_id,
this.updates,
this.version,
error => {
if (error) {
return done(error)
}
setTimeout(done, 200)
}
this.version
)
})
@@ -170,45 +156,29 @@ describe("Applying updates to a project's structure", function () {
})
describe('when the document is loaded', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
MockWebApi.insertDoc(this.project_id, this.update.id, {})
DocUpdaterClient.preloadDoc(this.project_id, this.update.id, error => {
if (error) {
return done(error)
}
sinon.spy(MockWebApi, 'getDocument')
DocUpdaterClient.sendProjectUpdate(
this.project_id,
this.user_id,
this.updates,
this.version,
error => {
if (error) {
return done(error)
}
setTimeout(done, 200)
}
)
})
await DocUpdaterClient.preloadDoc(this.project_id, this.update.id)
sinon.spy(MockWebApi, 'getDocument')
await sendProjectUpdateAndWait(
this.project_id,
this.user_id,
this.updates,
this.version
)
})
after(function () {
MockWebApi.getDocument.restore()
})
it('should update the doc', function (done) {
DocUpdaterClient.getDoc(
it('should update the doc', async function () {
const doc = await DocUpdaterClient.getDoc(
this.project_id,
this.update.id,
(error, res, doc) => {
if (error) {
return done(error)
}
doc.pathname.should.equal(this.update.newPathname)
done()
}
this.update.id
)
doc.pathname.should.equal(this.update.newPathname)
})
it('should push the applied doc renames to the project history api', function (done) {
@@ -271,19 +241,13 @@ describe("Applying updates to a project's structure", function () {
})
describe('when the documents are not loaded', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
DocUpdaterClient.sendProjectUpdate(
await sendProjectUpdateAndWait(
this.project_id,
this.user_id,
this.updates,
this.version,
error => {
if (error) {
return done(error)
}
setTimeout(done, 200)
}
this.version
)
})
@@ -348,19 +312,13 @@ describe("Applying updates to a project's structure", function () {
})
describe('when the document is not loaded', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
DocUpdaterClient.sendProjectUpdate(
await sendProjectUpdateAndWait(
this.project_id,
this.user_id,
this.updates,
this.version,
error => {
if (error) {
return done(error)
}
setTimeout(done, 200)
}
this.version
)
})
@@ -389,46 +347,29 @@ describe("Applying updates to a project's structure", function () {
})
describe('when the document is loaded', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
MockWebApi.insertDoc(this.project_id, this.update.id, {})
DocUpdaterClient.preloadDoc(this.project_id, this.update.id, error => {
if (error) {
return done(error)
}
sinon.spy(MockWebApi, 'getDocument')
DocUpdaterClient.sendProjectUpdate(
this.project_id,
this.user_id,
this.updates,
this.version,
error => {
if (error) {
return done(error)
}
setTimeout(done, 200)
}
)
})
await DocUpdaterClient.preloadDoc(this.project_id, this.update.id)
sinon.spy(MockWebApi, 'getDocument')
await sendProjectUpdateAndWait(
this.project_id,
this.user_id,
this.updates,
this.version
)
})
after(function () {
MockWebApi.getDocument.restore()
})
it('should not modify the doc', function (done) {
DocUpdaterClient.getDoc(
it('should not modify the doc', async function () {
const doc = await DocUpdaterClient.getDoc(
this.project_id,
this.update.id,
(error, res, doc) => {
if (error) {
return done(error)
}
doc.pathname.should.equal('/a/b/c.tex') // default pathname from MockWebApi
done()
}
this.update.id
)
doc.pathname.should.equal('/a/b/c.tex') // default pathname from MockWebApi
})
it('should push the applied doc update to the project history api', function (done) {
@@ -457,7 +398,7 @@ describe("Applying updates to a project's structure", function () {
})
describe('adding a file', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.fileUpdate = {
type: 'add-file',
@@ -466,17 +407,11 @@ describe("Applying updates to a project's structure", function () {
url: 'filestore.example.com',
}
this.updates = [this.fileUpdate]
DocUpdaterClient.sendProjectUpdate(
await sendProjectUpdateAndWait(
this.project_id,
this.user_id,
this.updates,
this.version,
error => {
if (error) {
return done(error)
}
setTimeout(done, 200)
}
this.version
)
})
@@ -505,7 +440,7 @@ describe("Applying updates to a project's structure", function () {
})
describe('adding a doc', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.docUpdate = {
type: 'add-doc',
@@ -514,17 +449,11 @@ describe("Applying updates to a project's structure", function () {
docLines: 'a\nb',
}
this.updates = [this.docUpdate]
DocUpdaterClient.sendProjectUpdate(
await sendProjectUpdateAndWait(
this.project_id,
this.user_id,
this.updates,
this.version,
error => {
if (error) {
return done(error)
}
setTimeout(done, 200)
}
this.version
)
})
@@ -553,7 +482,7 @@ describe("Applying updates to a project's structure", function () {
})
describe('with enough updates to flush to the history service', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.user_id = DocUpdaterClient.randomId()
this.version0 = 12345
@@ -574,29 +503,19 @@ describe("Applying updates to a project's structure", function () {
// Send updates in chunks to causes multiple flushes
const projectId = this.project_id
const userId = this.project_id
DocUpdaterClient.sendProjectUpdate(
await DocUpdaterClient.sendProjectUpdate(
projectId,
userId,
updates.slice(0, 250),
this.version0,
function (error) {
if (error) {
return done(error)
}
DocUpdaterClient.sendProjectUpdate(
projectId,
userId,
updates.slice(250),
this.version1,
error => {
if (error) {
return done(error)
}
setTimeout(done, 2000)
}
)
}
this.version0
)
await DocUpdaterClient.sendProjectUpdate(
projectId,
userId,
updates.slice(250),
this.version1
)
await setTimeout(200)
})
after(function () {
@@ -611,7 +530,7 @@ describe("Applying updates to a project's structure", function () {
})
describe('with too few updates to flush to the history service', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.user_id = DocUpdaterClient.randomId()
this.version0 = 12345
@@ -633,29 +552,19 @@ describe("Applying updates to a project's structure", function () {
// Send updates in chunks
const projectId = this.project_id
const userId = this.project_id
DocUpdaterClient.sendProjectUpdate(
await DocUpdaterClient.sendProjectUpdate(
projectId,
userId,
updates.slice(0, 10),
this.version0,
function (error) {
if (error) {
return done(error)
}
DocUpdaterClient.sendProjectUpdate(
projectId,
userId,
updates.slice(10),
this.version1,
error => {
if (error) {
return done(error)
}
setTimeout(done, 2000)
}
)
}
this.version0
)
await DocUpdaterClient.sendProjectUpdate(
projectId,
userId,
updates.slice(10),
this.version1
)
await setTimeout(200)
})
after(function () {

24
services/document-updater/test/acceptance/js/CheckRedisMongoSyncStateTests.js
View File

@@ -15,8 +15,8 @@ const rclient = require('@overleaf/redis-wrapper').createClient(
)
describe('CheckRedisMongoSyncState', function () {
beforeEach(function (done) {
DocUpdaterApp.ensureRunning(done)
beforeEach(async function () {
await DocUpdaterApp.ensureRunning()
})
beforeEach(async function () {
await rclient.flushall()
@@ -60,14 +60,14 @@ describe('CheckRedisMongoSyncState', function () {
describe('with a project', function () {
let projectId, docId
beforeEach(function (done) {
beforeEach(async function () {
projectId = DocUpdaterClient.randomId()
docId = DocUpdaterClient.randomId()
MockWebApi.insertDoc(projectId, docId, {
lines: ['mongo', 'lines'],
version: 1,
})
DocUpdaterClient.getDoc(projectId, docId, done)
await DocUpdaterClient.preloadDoc(projectId, docId)
})
it('should work when in sync', async function () {
@@ -149,14 +149,14 @@ describe('CheckRedisMongoSyncState', function () {
describe('with a project', function () {
let projectId2, docId2
beforeEach(function (done) {
beforeEach(async function () {
projectId2 = DocUpdaterClient.randomId()
docId2 = DocUpdaterClient.randomId()
MockWebApi.insertDoc(projectId2, docId2, {
lines: ['mongo', 'lines'],
version: 1,
})
DocUpdaterClient.getDoc(projectId2, docId2, done)
await DocUpdaterClient.preloadDoc(projectId2, docId2)
})
it('should work when in sync', async function () {
@@ -245,14 +245,14 @@ describe('CheckRedisMongoSyncState', function () {
describe('with more projects than the LIMIT', function () {
for (let i = 0; i < 20; i++) {
beforeEach(function (done) {
beforeEach(async function () {
const projectId = DocUpdaterClient.randomId()
const docId = DocUpdaterClient.randomId()
MockWebApi.insertDoc(projectId, docId, {
lines: ['mongo', 'lines'],
version: 1,
})
DocUpdaterClient.getDoc(projectId, docId, done)
await DocUpdaterClient.preloadDoc(projectId, docId)
})
}
@@ -278,7 +278,7 @@ describe('CheckRedisMongoSyncState', function () {
describe('with partially deleted doc', function () {
let projectId, docId
beforeEach(function (done) {
beforeEach(async function () {
projectId = DocUpdaterClient.randomId()
docId = DocUpdaterClient.randomId()
MockWebApi.insertDoc(projectId, docId, {
@@ -289,10 +289,8 @@ describe('CheckRedisMongoSyncState', function () {
lines: ['mongo', 'lines'],
version: 1,
})
DocUpdaterClient.getDoc(projectId, docId, err => {
MockWebApi.clearDocs()
done(err)
})
await DocUpdaterClient.preloadDoc(projectId, docId)
MockWebApi.clearDocs()
})
describe('with only the file-tree entry deleted', function () {
it('should flag the partial deletion', async function () {

109
services/document-updater/test/acceptance/js/DeletingADocumentTests.js
View File

@@ -1,19 +1,12 @@
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const sinon = require('sinon')
const MockProjectHistoryApi = require('./helpers/MockProjectHistoryApi')
const MockWebApi = require('./helpers/MockWebApi')
const DocUpdaterClient = require('./helpers/DocUpdaterClient')
const DocUpdaterApp = require('./helpers/DocUpdaterApp')
const { setTimeout } = require('node:timers/promises')
describe('Deleting a document', function () {
before(function (done) {
before(async function () {
this.lines = ['one', 'two', 'three']
this.version = 42
this.update = {
@@ -29,7 +22,7 @@ describe('Deleting a document', function () {
this.result = ['one', 'one and a half', 'two', 'three']
sinon.spy(MockProjectHistoryApi, 'flushProject')
DocUpdaterApp.ensureRunning(done)
await DocUpdaterApp.ensureRunning()
})
after(function () {
@@ -37,11 +30,9 @@ describe('Deleting a document', function () {
})
describe('when the updated doc exists in the doc updater', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
sinon.spy(MockWebApi, 'setDocument')
sinon.spy(MockWebApi, 'getDocument')
@@ -49,32 +40,15 @@ describe('Deleting a document', function () {
lines: this.lines,
version: this.version,
})
DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, error => {
if (error != null) {
throw error
}
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.update,
error => {
if (error != null) {
throw error
}
setTimeout(() => {
DocUpdaterClient.deleteDoc(
this.project_id,
this.doc_id,
(error, res, body) => {
if (error) return done(error)
this.statusCode = res.statusCode
setTimeout(done, 200)
}
)
}, 200)
}
)
})
await DocUpdaterClient.preloadDoc(this.project_id, this.doc_id)
await DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.update
)
await setTimeout(200)
const res = await DocUpdaterClient.deleteDoc(this.project_id, this.doc_id)
this.statusCode = res.status
})
after(function () {
@@ -92,20 +66,13 @@ describe('Deleting a document', function () {
.should.equal(true)
})
it('should need to reload the doc if read again', function (done) {
it('should need to reload the doc if read again', async function () {
MockWebApi.getDocument.resetHistory()
MockWebApi.getDocument.called.should.equals(false)
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
if (error) return done(error)
MockWebApi.getDocument
.calledWith(this.project_id, this.doc_id)
.should.equal(true)
done()
}
)
await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
MockWebApi.getDocument
.calledWith(this.project_id, this.doc_id)
.should.equal(true)
})
it('should flush project history', function () {
@@ -116,25 +83,16 @@ describe('Deleting a document', function () {
})
describe('when the doc is not in the doc updater', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
})
sinon.spy(MockWebApi, 'setDocument')
sinon.spy(MockWebApi, 'getDocument')
DocUpdaterClient.deleteDoc(
this.project_id,
this.doc_id,
(error, res, body) => {
if (error) return done(error)
this.statusCode = res.statusCode
setTimeout(done, 200)
}
)
const res = await DocUpdaterClient.deleteDoc(this.project_id, this.doc_id)
this.statusCode = res.status
})
after(function () {
@@ -150,19 +108,12 @@ describe('Deleting a document', function () {
MockWebApi.setDocument.called.should.equal(false)
})
it('should need to reload the doc if read again', function (done) {
it('should need to reload the doc if read again', async function () {
MockWebApi.getDocument.called.should.equals(false)
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
if (error) return done(error)
MockWebApi.getDocument
.calledWith(this.project_id, this.doc_id)
.should.equal(true)
done()
}
)
await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
MockWebApi.getDocument
.calledWith(this.project_id, this.doc_id)
.should.equal(true)
})
it('should flush project history', function () {

264
services/document-updater/test/acceptance/js/DeletingAProjectTests.js
View File

@@ -1,13 +1,5 @@
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const sinon = require('sinon')
const async = require('async')
const { setTimeout } = require('node:timers/promises')
const MockProjectHistoryApi = require('./helpers/MockProjectHistoryApi')
const MockWebApi = require('./helpers/MockWebApi')
@@ -15,7 +7,7 @@ const DocUpdaterClient = require('./helpers/DocUpdaterClient')
const DocUpdaterApp = require('./helpers/DocUpdaterApp')
describe('Deleting a project', function () {
beforeEach(function (done) {
beforeEach(async function () {
let docId0, docId1
this.project_id = DocUpdaterClient.randomId()
this.docs = [
@@ -50,45 +42,27 @@ describe('Deleting a project', function () {
updatedLines: ['four', 'four and a half', 'five', 'six'],
},
]
for (const doc of Array.from(this.docs)) {
for (const doc of this.docs) {
MockWebApi.insertDoc(this.project_id, doc.id, {
lines: doc.lines,
version: doc.update.v,
})
}
DocUpdaterApp.ensureRunning(done)
await DocUpdaterApp.ensureRunning()
})
describe('without updates', function () {
beforeEach(function (done) {
beforeEach(async function () {
sinon.spy(MockWebApi, 'setDocument')
sinon.spy(MockProjectHistoryApi, 'flushProject')
async.series(
this.docs.map(doc => {
return callback => {
DocUpdaterClient.preloadDoc(this.project_id, doc.id, error => {
callback(error)
})
}
}),
error => {
if (error != null) {
throw error
}
setTimeout(() => {
DocUpdaterClient.deleteProject(
this.project_id,
(error, res, body) => {
if (error) return done(error)
this.statusCode = res.statusCode
done()
}
)
}, 200)
}
)
for (const doc of this.docs) {
await DocUpdaterClient.preloadDoc(this.project_id, doc.id)
}
await setTimeout(200)
const res = await DocUpdaterClient.deleteProject(this.project_id)
this.statusCode = res.status
})
afterEach(function () {
@@ -104,32 +78,18 @@ describe('Deleting a project', function () {
MockWebApi.setDocument.should.not.have.been.called
})
it('should need to reload the docs if read again', function (done) {
it('should need to reload the docs if read again', async function () {
sinon.spy(MockWebApi, 'getDocument')
async.series(
this.docs.map(doc => {
return callback => {
MockWebApi.getDocument
.calledWith(this.project_id, doc.id)
.should.equal(false)
DocUpdaterClient.getDoc(
this.project_id,
doc.id,
(error, res, returnedDoc) => {
if (error) return done(error)
MockWebApi.getDocument
.calledWith(this.project_id, doc.id)
.should.equal(true)
callback()
}
)
}
}),
() => {
MockWebApi.getDocument.restore()
done()
}
)
for (const doc of this.docs) {
MockWebApi.getDocument
.calledWith(this.project_id, doc.id)
.should.equal(false)
await DocUpdaterClient.getDoc(this.project_id, doc.id)
MockWebApi.getDocument
.calledWith(this.project_id, doc.id)
.should.equal(true)
}
MockWebApi.getDocument.restore()
})
it('should flush each doc in project history', function () {
@@ -140,44 +100,16 @@ describe('Deleting a project', function () {
})
describe('with documents which have been updated', function () {
beforeEach(function (done) {
beforeEach(async function () {
sinon.spy(MockWebApi, 'setDocument')
sinon.spy(MockProjectHistoryApi, 'flushProject')
async.series(
this.docs.map(doc => {
return callback => {
DocUpdaterClient.preloadDoc(this.project_id, doc.id, error => {
if (error != null) {
return callback(error)
}
DocUpdaterClient.sendUpdate(
this.project_id,
doc.id,
doc.update,
error => {
callback(error)
}
)
})
}
}),
error => {
if (error != null) {
throw error
}
setTimeout(() => {
DocUpdaterClient.deleteProject(
this.project_id,
(error, res, body) => {
if (error) return done(error)
this.statusCode = res.statusCode
done()
}
)
}, 200)
}
)
for (const doc of this.docs) {
await DocUpdaterClient.preloadDoc(this.project_id, doc.id)
await DocUpdaterClient.sendUpdate(this.project_id, doc.id, doc.update)
}
await setTimeout(200)
const res = await DocUpdaterClient.deleteProject(this.project_id)
this.statusCode = res.status
})
afterEach(function () {
@@ -190,39 +122,25 @@ describe('Deleting a project', function () {
})
it('should send each document to the web api', function () {
Array.from(this.docs).map(doc =>
for (const doc of this.docs) {
MockWebApi.setDocument
.calledWith(this.project_id, doc.id, doc.updatedLines)
.should.equal(true)
)
}
})
it('should need to reload the docs if read again', function (done) {
it('should need to reload the docs if read again', async function () {
sinon.spy(MockWebApi, 'getDocument')
async.series(
this.docs.map(doc => {
return callback => {
MockWebApi.getDocument
.calledWith(this.project_id, doc.id)
.should.equal(false)
DocUpdaterClient.getDoc(
this.project_id,
doc.id,
(error, res, returnedDoc) => {
if (error) return done(error)
MockWebApi.getDocument
.calledWith(this.project_id, doc.id)
.should.equal(true)
callback()
}
)
}
}),
() => {
MockWebApi.getDocument.restore()
done()
}
)
for (const doc of this.docs) {
MockWebApi.getDocument
.calledWith(this.project_id, doc.id)
.should.equal(false)
await DocUpdaterClient.getDoc(this.project_id, doc.id)
MockWebApi.getDocument
.calledWith(this.project_id, doc.id)
.should.equal(true)
}
MockWebApi.getDocument.restore()
})
it('should flush each doc in project history', function () {
@@ -233,44 +151,18 @@ describe('Deleting a project', function () {
})
describe('with the background=true parameter from realtime and no request to flush the queue', function () {
beforeEach(function (done) {
beforeEach(async function () {
sinon.spy(MockWebApi, 'setDocument')
sinon.spy(MockProjectHistoryApi, 'flushProject')
async.series(
this.docs.map(doc => {
return callback => {
DocUpdaterClient.preloadDoc(this.project_id, doc.id, error => {
if (error != null) {
return callback(error)
}
DocUpdaterClient.sendUpdate(
this.project_id,
doc.id,
doc.update,
error => {
callback(error)
}
)
})
}
}),
error => {
if (error != null) {
throw error
}
setTimeout(() => {
DocUpdaterClient.deleteProjectOnShutdown(
this.project_id,
(error, res, body) => {
if (error) return done(error)
this.statusCode = res.statusCode
done()
}
)
}, 200)
}
for (const doc of this.docs) {
await DocUpdaterClient.preloadDoc(this.project_id, doc.id)
await DocUpdaterClient.sendUpdate(this.project_id, doc.id, doc.update)
}
await setTimeout(200)
const res = await DocUpdaterClient.deleteProjectOnShutdown(
this.project_id
)
this.statusCode = res.status
})
afterEach(function () {
@@ -292,45 +184,21 @@ describe('Deleting a project', function () {
})
describe('with the background=true parameter from realtime and a request to flush the queue', function () {
beforeEach(function (done) {
beforeEach(async function () {
sinon.spy(MockWebApi, 'setDocument')
sinon.spy(MockProjectHistoryApi, 'flushProject')
async.series(
this.docs.map(doc => {
return callback => {
DocUpdaterClient.preloadDoc(this.project_id, doc.id, error => {
if (error != null) {
return callback(error)
}
DocUpdaterClient.sendUpdate(
this.project_id,
doc.id,
doc.update,
error => {
callback(error)
}
)
})
}
}),
error => {
if (error != null) {
throw error
}
setTimeout(() => {
DocUpdaterClient.deleteProjectOnShutdown(
this.project_id,
(error, res, body) => {
if (error) return done(error)
this.statusCode = res.statusCode
// after deleting the project and putting it in the queue, flush the queue
setTimeout(() => DocUpdaterClient.flushOldProjects(done), 2000)
}
)
}, 200)
}
for (const doc of this.docs) {
await DocUpdaterClient.preloadDoc(this.project_id, doc.id)
await DocUpdaterClient.sendUpdate(this.project_id, doc.id, doc.update)
}
await setTimeout(200)
const res = await DocUpdaterClient.deleteProjectOnShutdown(
this.project_id
)
this.statusCode = res.status
// after deleting the project and putting it in the queue, flush the queue
await setTimeout(2000)
await DocUpdaterClient.flushOldProjects()
})
afterEach(function () {
@@ -343,11 +211,11 @@ describe('Deleting a project', function () {
})
it('should send each document to the web api', function () {
Array.from(this.docs).map(doc =>
for (const doc of this.docs) {
MockWebApi.setDocument
.calledWith(this.project_id, doc.id, doc.updatedLines)
.should.equal(true)
)
}
})
it('should flush to project history', function () {

107
services/document-updater/test/acceptance/js/FlushingAProjectTests.js
View File

@@ -1,26 +1,18 @@
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const sinon = require('sinon')
const async = require('async')
const { setTimeout } = require('node:timers/promises')
const MockWebApi = require('./helpers/MockWebApi')
const DocUpdaterClient = require('./helpers/DocUpdaterClient')
const DocUpdaterApp = require('./helpers/DocUpdaterApp')
describe('Flushing a project', function () {
before(function (done) {
let docId0, docId1
before(async function () {
this.project_id = DocUpdaterClient.randomId()
const docId0 = DocUpdaterClient.randomId()
const docId1 = DocUpdaterClient.randomId()
this.docs = [
{
id: (docId0 = DocUpdaterClient.randomId()),
id: docId0,
lines: ['one', 'two', 'three'],
update: {
doc: docId0,
@@ -35,7 +27,7 @@ describe('Flushing a project', function () {
updatedLines: ['one', 'one and a half', 'two', 'three'],
},
{
id: (docId1 = DocUpdaterClient.randomId()),
id: docId1,
lines: ['four', 'five', 'six'],
update: {
doc: docId1,
@@ -50,92 +42,51 @@ describe('Flushing a project', function () {
updatedLines: ['four', 'four and a half', 'five', 'six'],
},
]
for (const doc of Array.from(this.docs)) {
for (const doc of this.docs) {
MockWebApi.insertDoc(this.project_id, doc.id, {
lines: doc.lines,
version: doc.update.v,
})
}
return DocUpdaterApp.ensureRunning(done)
await DocUpdaterApp.ensureRunning()
})
return describe('with documents which have been updated', function () {
before(function (done) {
describe('with documents which have been updated', function () {
before(async function () {
sinon.spy(MockWebApi, 'setDocument')
return async.series(
this.docs.map(doc => {
return callback => {
return DocUpdaterClient.preloadDoc(
this.project_id,
doc.id,
error => {
if (error != null) {
return callback(error)
}
return DocUpdaterClient.sendUpdate(
this.project_id,
doc.id,
doc.update,
error => {
return callback(error)
}
)
}
)
}
}),
error => {
if (error != null) {
throw error
}
return setTimeout(() => {
return DocUpdaterClient.flushProject(
this.project_id,
(error, res, body) => {
if (error) return done(error)
this.statusCode = res.statusCode
return done()
}
)
}, 200)
}
)
for (const doc of this.docs) {
await DocUpdaterClient.preloadDoc(this.project_id, doc.id)
await DocUpdaterClient.sendUpdate(this.project_id, doc.id, doc.update)
}
await setTimeout(200)
const res = await DocUpdaterClient.flushProject(this.project_id)
this.statusCode = res.status
})
after(function () {
return MockWebApi.setDocument.restore()
MockWebApi.setDocument.restore()
})
it('should return a 204 status code', function () {
return this.statusCode.should.equal(204)
this.statusCode.should.equal(204)
})
it('should send each document to the web api', function () {
return Array.from(this.docs).map(doc =>
for (const doc of this.docs) {
MockWebApi.setDocument
.calledWith(this.project_id, doc.id, doc.updatedLines)
.should.equal(true)
)
}
})
return it('should update the lines in the doc updater', function (done) {
return async.series(
this.docs.map(doc => {
return callback => {
return DocUpdaterClient.getDoc(
this.project_id,
doc.id,
(error, res, returnedDoc) => {
if (error) return done(error)
returnedDoc.lines.should.deep.equal(doc.updatedLines)
return callback()
}
)
}
}),
done
)
it('should update the lines in the doc updater', async function () {
for (const doc of this.docs) {
const returnedDoc = await DocUpdaterClient.getDoc(
this.project_id,
doc.id
)
returnedDoc.lines.should.deep.equal(doc.updatedLines)
}
})
})
})

104
services/document-updater/test/acceptance/js/FlushingDocsTests.js
View File

@@ -1,26 +1,13 @@
/* eslint-disable
no-return-assign,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const sinon = require('sinon')
const { expect } = require('chai')
const async = require('async')
const { setTimeout } = require('node:timers/promises')
const MockWebApi = require('./helpers/MockWebApi')
const DocUpdaterClient = require('./helpers/DocUpdaterClient')
const DocUpdaterApp = require('./helpers/DocUpdaterApp')
describe('Flushing a doc to Mongo', function () {
before(function (done) {
before(async function () {
this.lines = ['one', 'two', 'three']
this.version = 42
this.update = {
@@ -35,83 +22,69 @@ describe('Flushing a doc to Mongo', function () {
v: this.version,
}
this.result = ['one', 'one and a half', 'two', 'three']
return DocUpdaterApp.ensureRunning(done)
await DocUpdaterApp.ensureRunning()
})
describe('when the updated doc exists in the doc updater', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
sinon.spy(MockWebApi, 'setDocument')
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
})
return DocUpdaterClient.sendUpdates(
this.project_id,
this.doc_id,
[this.update],
error => {
if (error != null) {
throw error
}
return setTimeout(() => {
return DocUpdaterClient.flushDoc(this.project_id, this.doc_id, done)
}, 200)
}
)
await DocUpdaterClient.sendUpdates(this.project_id, this.doc_id, [
this.update,
])
await setTimeout(200)
await DocUpdaterClient.flushDoc(this.project_id, this.doc_id)
})
after(function () {
return MockWebApi.setDocument.restore()
MockWebApi.setDocument.restore()
})
it('should flush the updated doc lines and version to the web api', function () {
return MockWebApi.setDocument
MockWebApi.setDocument
.calledWith(this.project_id, this.doc_id, this.result, this.version + 1)
.should.equal(true)
})
return it('should flush the last update author and time to the web api', function () {
it('should flush the last update author and time to the web api', function () {
const lastUpdatedAt = MockWebApi.setDocument.lastCall.args[5]
parseInt(lastUpdatedAt).should.be.closeTo(new Date().getTime(), 30000)
const lastUpdatedBy = MockWebApi.setDocument.lastCall.args[6]
return lastUpdatedBy.should.equal('last-author-fake-id')
lastUpdatedBy.should.equal('last-author-fake-id')
})
})
describe('when the doc does not exist in the doc updater', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
})
sinon.spy(MockWebApi, 'setDocument')
return DocUpdaterClient.flushDoc(this.project_id, this.doc_id, done)
await DocUpdaterClient.flushDoc(this.project_id, this.doc_id)
})
after(function () {
return MockWebApi.setDocument.restore()
MockWebApi.setDocument.restore()
})
return it('should not flush the doc to the web api', function () {
return MockWebApi.setDocument.called.should.equal(false)
it('should not flush the doc to the web api', function () {
MockWebApi.setDocument.called.should.equal(false)
})
})
return describe('when the web api http request takes a long time on first request', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
describe('when the web api http request takes a long time on first request', function () {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
@@ -130,33 +103,26 @@ describe('Flushing a doc to Mongo', function () {
lastUpdatedBy,
callback
) => {
if (callback == null) {
if (!callback) {
callback = function () {}
}
setTimeout(callback, t)
return (t = 0)
t = 0
}
)
return DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, done)
await DocUpdaterClient.preloadDoc(this.project_id, this.doc_id)
})
after(function () {
return MockWebApi.setDocument.restore()
MockWebApi.setDocument.restore()
})
return it('should still work', function (done) {
it('should still work', async function () {
const start = Date.now()
return DocUpdaterClient.flushDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
if (error) return done(error)
res.statusCode.should.equal(204)
const delta = Date.now() - start
expect(delta).to.be.below(20000)
return done()
}
)
const res = await DocUpdaterClient.flushDoc(this.project_id, this.doc_id)
res.status.should.equal(204)
const delta = Date.now() - start
expect(delta).to.be.below(20000)
})
})
})

223
services/document-updater/test/acceptance/js/GettingADocumentTests.js
View File

@@ -1,32 +1,22 @@
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const sinon = require('sinon')
const { expect } = require('chai')
const MockWebApi = require('./helpers/MockWebApi')
const DocUpdaterClient = require('./helpers/DocUpdaterClient')
const DocUpdaterApp = require('./helpers/DocUpdaterApp')
const { RequestFailedError } = require('@overleaf/fetch-utils')
describe('Getting a document', function () {
before(function (done) {
before(async function () {
this.lines = ['one', 'two', 'three']
this.version = 42
return DocUpdaterApp.ensureRunning(done)
await DocUpdaterApp.ensureRunning()
})
describe('when the document is not loaded', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
sinon.spy(MockWebApi, 'getDocument')
MockWebApi.insertDoc(this.project_id, this.doc_id, {
@@ -34,87 +24,66 @@ describe('Getting a document', function () {
version: this.version,
})
return DocUpdaterClient.getDoc(
this.returnedDoc = await DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, returnedDoc) => {
if (error) return done(error)
this.returnedDoc = returnedDoc
return done()
}
this.doc_id
)
})
after(function () {
return MockWebApi.getDocument.restore()
MockWebApi.getDocument.restore()
})
it('should load the document from the web API', function () {
return MockWebApi.getDocument
MockWebApi.getDocument
.calledWith(this.project_id, this.doc_id)
.should.equal(true)
})
it('should return the document lines', function () {
return this.returnedDoc.lines.should.deep.equal(this.lines)
this.returnedDoc.lines.should.deep.equal(this.lines)
})
return it('should return the document at its current version', function () {
return this.returnedDoc.version.should.equal(this.version)
it('should return the document at its current version', function () {
this.returnedDoc.version.should.equal(this.version)
})
})
describe('when the document is already loaded', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
})
return DocUpdaterClient.preloadDoc(
await DocUpdaterClient.preloadDoc(this.project_id, this.doc_id)
sinon.spy(MockWebApi, 'getDocument')
this.returnedDoc = await DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
error => {
if (error != null) {
throw error
}
sinon.spy(MockWebApi, 'getDocument')
return DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, returnedDoc) => {
if (error) return done(error)
this.returnedDoc = returnedDoc
return done()
}
)
}
this.doc_id
)
})
after(function () {
return MockWebApi.getDocument.restore()
MockWebApi.getDocument.restore()
})
it('should not load the document from the web API', function () {
return MockWebApi.getDocument.called.should.equal(false)
MockWebApi.getDocument.called.should.equal(false)
})
return it('should return the document lines', function () {
return this.returnedDoc.lines.should.deep.equal(this.lines)
it('should return the document lines', function () {
this.returnedDoc.lines.should.deep.equal(this.lines)
})
})
describe('when the request asks for some recent ops', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: (this.lines = ['one', 'two', 'three']),
})
@@ -125,159 +94,109 @@ describe('Getting a document', function () {
v,
}))
return DocUpdaterClient.sendUpdates(
await DocUpdaterClient.sendUpdates(
this.project_id,
this.doc_id,
this.updates,
error => {
if (error != null) {
throw error
}
sinon.spy(MockWebApi, 'getDocument')
return done()
}
this.updates
)
sinon.spy(MockWebApi, 'getDocument')
})
after(function () {
return MockWebApi.getDocument.restore()
MockWebApi.getDocument.restore()
})
describe('when the ops are loaded', function () {
before(function (done) {
return DocUpdaterClient.getDocAndRecentOps(
before(async function () {
this.returnedDoc = await DocUpdaterClient.getDocAndRecentOps(
this.project_id,
this.doc_id,
190,
(error, res, returnedDoc) => {
if (error) return done(error)
this.returnedDoc = returnedDoc
return done()
}
190
)
})
return it('should return the recent ops', function () {
it('should return the recent ops', function () {
this.returnedDoc.ops.length.should.equal(10)
return Array.from(this.updates.slice(190, -1)).map((update, i) =>
for (const [i, update] of this.updates.slice(190, -1).entries()) {
this.returnedDoc.ops[i].op.should.deep.equal(update.op)
)
}
})
})
return describe('when the ops are not all loaded', function () {
before(function (done) {
describe('when the ops are not all loaded', function () {
it('should return UnprocessableEntity', async function () {
// We only track 100 ops
return DocUpdaterClient.getDocAndRecentOps(
this.project_id,
this.doc_id,
10,
(error, res, returnedDoc) => {
if (error) return done(error)
this.res = res
this.returnedDoc = returnedDoc
return done()
}
await expect(
DocUpdaterClient.getDocAndRecentOps(this.project_id, this.doc_id, 10)
)
})
return it('should return UnprocessableEntity', function () {
return this.res.statusCode.should.equal(422)
.to.be.rejectedWith(RequestFailedError)
.and.eventually.have.nested.property('response.status', 422)
})
})
})
describe('when the document does not exist', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
return DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
if (error) return done(error)
this.statusCode = res.statusCode
return done()
}
)
})
return it('should return 404', function () {
return this.statusCode.should.equal(404)
it('should return 404', async function () {
const projectId = DocUpdaterClient.randomId()
const docId = DocUpdaterClient.randomId()
await expect(DocUpdaterClient.getDoc(projectId, docId))
.to.be.rejectedWith(RequestFailedError)
.and.eventually.have.nested.property('response.status', 404)
})
})
describe('when the web api returns an error', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
before(function () {
sinon
.stub(MockWebApi, 'getDocument')
.callsFake((projectId, docId, callback) => {
if (callback == null) {
callback = function () {}
}
return callback(new Error('oops'))
callback(new Error('oops'))
})
return DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
if (error) return done(error)
this.statusCode = res.statusCode
return done()
}
)
})
after(function () {
return MockWebApi.getDocument.restore()
MockWebApi.getDocument.restore()
})
return it('should return 500', function () {
return this.statusCode.should.equal(500)
it('should return 500', async function () {
const projectId = DocUpdaterClient.randomId()
const docId = DocUpdaterClient.randomId()
await expect(DocUpdaterClient.getDoc(projectId, docId))
.to.be.rejectedWith(RequestFailedError)
.and.eventually.have.nested.property('response.status', 500)
})
})
return describe('when the web api http request takes a long time', function () {
describe('when the web api http request takes a long time', function () {
before(function (done) {
this.timeout = 10000
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
sinon
.stub(MockWebApi, 'getDocument')
.callsFake((projectId, docId, callback) => {
if (callback == null) {
callback = function () {}
}
return setTimeout(callback, 30000)
setTimeout(callback, 30000)
})
return done()
done()
})
after(function () {
return MockWebApi.getDocument.restore()
MockWebApi.getDocument.restore()
})
return it('should return quickly(ish)', function (done) {
it('should return quickly(ish)', async function () {
const projectId = DocUpdaterClient.randomId()
const docId = DocUpdaterClient.randomId()
const start = Date.now()
return DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
if (error) return done(error)
res.statusCode.should.equal(500)
const delta = Date.now() - start
expect(delta).to.be.below(20000)
return done()
}
)
await expect(DocUpdaterClient.getDoc(projectId, docId))
.to.be.rejectedWith(RequestFailedError)
.and.eventually.have.nested.property('response.status', 500)
const delta = Date.now() - start
expect(delta).to.be.below(20000)
})
})
})

187
services/document-updater/test/acceptance/js/GettingProjectDocsTests.js
View File

@@ -1,176 +1,77 @@
/* eslint-disable
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const sinon = require('sinon')
const { expect } = require('chai')
const MockWebApi = require('./helpers/MockWebApi')
const DocUpdaterClient = require('./helpers/DocUpdaterClient')
const DocUpdaterApp = require('./helpers/DocUpdaterApp')
const { RequestFailedError } = require('@overleaf/fetch-utils')
describe('Getting documents for project', function () {
before(function (done) {
before(async function () {
this.lines = ['one', 'two', 'three']
this.version = 42
return DocUpdaterApp.ensureRunning(done)
await DocUpdaterApp.ensureRunning()
})
describe('when project state hash does not match', function () {
before(function (done) {
this.projectStateHash = DocUpdaterClient.randomId()
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
MockWebApi.insertDoc(this.project_id, this.doc_id, {
it('should return a 409 Conflict response', async function () {
const projectStateHash = DocUpdaterClient.randomId()
const projectId = DocUpdaterClient.randomId()
const docId = DocUpdaterClient.randomId()
MockWebApi.insertDoc(projectId, docId, {
lines: this.lines,
version: this.version,
})
return DocUpdaterClient.preloadDoc(
this.project_id,
this.doc_id,
error => {
if (error != null) {
throw error
}
return DocUpdaterClient.getProjectDocs(
this.project_id,
this.projectStateHash,
(error, res, returnedDocs) => {
if (error) return done(error)
this.res = res
this.returnedDocs = returnedDocs
return done()
}
)
}
)
})
return it('should return a 409 Conflict response', function () {
return this.res.statusCode.should.equal(409)
await DocUpdaterClient.preloadDoc(projectId, docId)
await expect(DocUpdaterClient.getProjectDocs(projectId, projectStateHash))
.to.be.rejectedWith(RequestFailedError)
.and.eventually.have.nested.property('response.status', 409)
})
})
describe('when project state hash matches', function () {
before(function (done) {
this.projectStateHash = DocUpdaterClient.randomId()
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
MockWebApi.insertDoc(this.project_id, this.doc_id, {
it('should return the documents', async function () {
const projectStateHash = DocUpdaterClient.randomId()
const projectId = DocUpdaterClient.randomId()
const docId = DocUpdaterClient.randomId()
MockWebApi.insertDoc(projectId, docId, {
lines: this.lines,
version: this.version,
})
return DocUpdaterClient.preloadDoc(
this.project_id,
this.doc_id,
error => {
if (error != null) {
throw error
}
return DocUpdaterClient.getProjectDocs(
this.project_id,
this.projectStateHash,
(error, res0, returnedDocs0) => {
if (error) return done(error)
// set the hash
this.res0 = res0
this.returnedDocs0 = returnedDocs0
return DocUpdaterClient.getProjectDocs(
this.project_id,
this.projectStateHash,
(error, res, returnedDocs) => {
if (error) return done(error)
// the hash should now match
this.res = res
this.returnedDocs = returnedDocs
return done()
}
)
}
)
}
await DocUpdaterClient.preloadDoc(projectId, docId)
// set the hash
await expect(DocUpdaterClient.getProjectDocs(projectId, projectStateHash))
.to.be.rejectedWith(RequestFailedError)
.and.eventually.have.nested.property('response.status', 409)
const returnedDocs1 = await DocUpdaterClient.getProjectDocs(
projectId,
projectStateHash
)
})
it('should return a 200 response', function () {
return this.res.statusCode.should.equal(200)
})
return it('should return the documents', function () {
return this.returnedDocs.should.deep.equal([
{ _id: this.doc_id, lines: this.lines, v: this.version },
// the hash should now match
returnedDocs1.should.deep.equal([
{ _id: docId, lines: this.lines, v: this.version },
])
})
})
return describe('when the doc has been removed', function () {
before(function (done) {
this.projectStateHash = DocUpdaterClient.randomId()
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
MockWebApi.insertDoc(this.project_id, this.doc_id, {
describe('when the doc has been removed', function () {
it('should return a 409 Conflict response', async function () {
const projectStateHash = DocUpdaterClient.randomId()
const projectId = DocUpdaterClient.randomId()
const docId = DocUpdaterClient.randomId()
MockWebApi.insertDoc(projectId, docId, {
lines: this.lines,
version: this.version,
})
return DocUpdaterClient.preloadDoc(
this.project_id,
this.doc_id,
error => {
if (error != null) {
throw error
}
return DocUpdaterClient.getProjectDocs(
this.project_id,
this.projectStateHash,
(error, res0, returnedDocs0) => {
if (error) return done(error)
// set the hash
this.res0 = res0
this.returnedDocs0 = returnedDocs0
return DocUpdaterClient.deleteDoc(
this.project_id,
this.doc_id,
(error, res, body) => {
if (error) return done(error)
// delete the doc
return DocUpdaterClient.getProjectDocs(
this.project_id,
this.projectStateHash,
(error, res1, returnedDocs) => {
if (error) return done(error)
// the hash would match, but the doc has been deleted
this.res = res1
this.returnedDocs = returnedDocs
return done()
}
)
}
)
}
)
}
)
})
return it('should return a 409 Conflict response', function () {
return this.res.statusCode.should.equal(409)
await DocUpdaterClient.preloadDoc(projectId, docId)
await expect(DocUpdaterClient.getProjectDocs(projectId, projectStateHash))
.to.be.rejectedWith(RequestFailedError)
.and.eventually.have.nested.property('response.status', 409)
await DocUpdaterClient.deleteDoc(projectId, docId)
// the hash would match, but the doc has been deleted
await expect(DocUpdaterClient.getProjectDocs(projectId, projectStateHash))
.to.be.rejectedWith(RequestFailedError)
.and.eventually.have.nested.property('response.status', 409)
})
})
})

68
services/document-updater/test/acceptance/js/PeekingADoc.js
View File

@@ -2,16 +2,18 @@ const sinon = require('sinon')
const MockWebApi = require('./helpers/MockWebApi')
const DocUpdaterClient = require('./helpers/DocUpdaterClient')
const DocUpdaterApp = require('./helpers/DocUpdaterApp')
const { expect } = require('chai')
const { RequestFailedError } = require('@overleaf/fetch-utils')
describe('Peeking a document', function () {
before(function (done) {
before(async function () {
this.lines = ['one', 'two', 'three']
this.version = 42
return DocUpdaterApp.ensureRunning(done)
await DocUpdaterApp.ensureRunning()
})
describe('when the document is not loaded', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
sinon.spy(MockWebApi, 'getDocument')
@@ -20,34 +22,22 @@ describe('Peeking a document', function () {
lines: this.lines,
version: this.version,
})
return DocUpdaterClient.peekDoc(
this.project_id,
this.doc_id,
(error, res, returnedDoc) => {
this.error = error
this.res = res
this.returnedDoc = returnedDoc
return done()
}
)
})
after(function () {
return MockWebApi.getDocument.restore()
MockWebApi.getDocument.restore()
})
it('should return a 404 response', function () {
this.res.statusCode.should.equal(404)
})
it('should not load the document from the web API', function () {
return MockWebApi.getDocument.called.should.equal(false)
it('should not load the document from the web API and should return a 404 response', async function () {
await expect(DocUpdaterClient.peekDoc(this.project_id, this.doc_id))
.to.be.rejectedWith(RequestFailedError)
.and.eventually.have.nested.property('response.status', 404)
MockWebApi.getDocument.called.should.equal(false)
})
})
describe('when the document is already loaded', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
@@ -55,46 +45,28 @@ describe('Peeking a document', function () {
lines: this.lines,
version: this.version,
})
return DocUpdaterClient.preloadDoc(
await DocUpdaterClient.preloadDoc(this.project_id, this.doc_id)
sinon.spy(MockWebApi, 'getDocument')
this.returnedDoc = await DocUpdaterClient.peekDoc(
this.project_id,
this.doc_id,
error => {
if (error != null) {
throw error
}
sinon.spy(MockWebApi, 'getDocument')
return DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, returnedDoc) => {
if (error) return done(error)
this.res = res
this.returnedDoc = returnedDoc
return done()
}
)
}
this.doc_id
)
})
after(function () {
return MockWebApi.getDocument.restore()
})
it('should return a 200 response', function () {
this.res.statusCode.should.equal(200)
MockWebApi.getDocument.restore()
})
it('should return the document lines', function () {
return this.returnedDoc.lines.should.deep.equal(this.lines)
this.returnedDoc.lines.should.deep.equal(this.lines)
})
it('should return the document version', function () {
return this.returnedDoc.version.should.equal(this.version)
this.returnedDoc.version.should.equal(this.version)
})
it('should not load the document from the web API', function () {
return MockWebApi.getDocument.called.should.equal(false)
MockWebApi.getDocument.called.should.equal(false)
})
})
})

834
services/document-updater/test/acceptance/js/RangesTests.js
View File

File diff suppressed because it is too large Load Diff

237
services/document-updater/test/acceptance/js/RejectingChangesTests.js
View File

@@ -8,12 +8,12 @@ const DocUpdaterApp = require('./helpers/DocUpdaterApp')
const sandbox = sinon.createSandbox()
describe('Rejecting Changes', function () {
before(function (done) {
DocUpdaterApp.ensureRunning(done)
before(async function () {
await DocUpdaterApp.ensureRunning()
})
describe('rejecting a single change', function () {
beforeEach(function (done) {
beforeEach(async function () {
this.project_id = DocUpdaterClient.randomId()
this.user_id = DocUpdaterClient.randomId()
this.doc = {
@@ -38,11 +38,10 @@ describe('Rejecting Changes', function () {
},
}
DocUpdaterClient.sendUpdate(
await DocUpdaterClient.sendUpdate(
this.project_id,
this.doc.id,
this.update,
done
this.update
)
})
@@ -50,93 +49,54 @@ describe('Rejecting Changes', function () {
sandbox.restore()
})
it('should reject the change and restore the original text', function (done) {
DocUpdaterClient.getDoc(
it('should reject the change and restore the original text', async function () {
const doc1 = await DocUpdaterClient.getDoc(this.project_id, this.doc.id)
expect(doc1.ranges.changes).to.have.length(1)
const change = doc1.ranges.changes[0]
expect(change.op).to.deep.equal({ i: 'quick ', p: 4 })
expect(change.id).to.equal(this.id_seed + '000001')
expect(doc1.lines).to.deep.equal([
'the quick brown fox jumps over the lazy dog',
])
const { rejectedChangeIds } = await DocUpdaterClient.rejectChanges(
this.project_id,
this.doc.id,
(error, res, data) => {
if (error != null) {
throw error
}
expect(data.ranges.changes).to.have.length(1)
const change = data.ranges.changes[0]
expect(change.op).to.deep.equal({ i: 'quick ', p: 4 })
expect(change.id).to.equal(this.id_seed + '000001')
expect(data.lines).to.deep.equal([
'the quick brown fox jumps over the lazy dog',
])
DocUpdaterClient.rejectChanges(
this.project_id,
this.doc.id,
[change.id],
this.user_id,
(error, res, body) => {
if (error != null) {
throw error
}
expect(res.statusCode).to.equal(200)
expect(body.rejectedChangeIds).to.be.an('array')
expect(body.rejectedChangeIds).to.include(change.id)
DocUpdaterClient.getDoc(
this.project_id,
this.doc.id,
(error, res, data) => {
if (error != null) {
throw error
}
expect(data.ranges.changes || []).to.have.length(0)
expect(data.lines).to.deep.equal([
'the brown fox jumps over the lazy dog',
])
done()
}
)
}
)
}
[change.id],
this.user_id
)
expect(rejectedChangeIds).to.be.an('array')
expect(rejectedChangeIds).to.include(change.id)
const doc2 = await DocUpdaterClient.getDoc(this.project_id, this.doc.id)
expect(doc2.ranges.changes || []).to.have.length(0)
expect(doc2.lines).to.deep.equal([
'the brown fox jumps over the lazy dog',
])
})
it('should return 200 status code with rejectedChangeIds on successful rejection', function (done) {
DocUpdaterClient.getDoc(
it('should return 200 status code with rejectedChangeIds on successful rejection', async function () {
const data = await DocUpdaterClient.getDoc(this.project_id, this.doc.id)
const changeId = data.ranges.changes[0].id
const { rejectedChangeIds } = await DocUpdaterClient.rejectChanges(
this.project_id,
this.doc.id,
(error, res, data) => {
if (error != null) {
throw error
}
const changeId = data.ranges.changes[0].id
DocUpdaterClient.rejectChanges(
this.project_id,
this.doc.id,
[changeId],
this.user_id,
(error, res, body) => {
if (error != null) {
throw error
}
expect(res.statusCode).to.equal(200)
expect(body.rejectedChangeIds).to.be.an('array')
expect(body.rejectedChangeIds).to.include(changeId)
done()
}
)
}
[changeId],
this.user_id
)
expect(rejectedChangeIds).to.be.an('array')
expect(rejectedChangeIds).to.include(changeId)
})
})
describe('rejecting multiple changes', function () {
beforeEach(function (done) {
beforeEach(async function () {
this.project_id = DocUpdaterClient.randomId()
this.user_id = DocUpdaterClient.randomId()
this.doc = {
@@ -174,11 +134,10 @@ describe('Rejecting Changes', function () {
},
]
DocUpdaterClient.sendUpdates(
await DocUpdaterClient.sendUpdates(
this.project_id,
this.doc.id,
this.updates,
done
this.updates
)
})
@@ -186,62 +145,36 @@ describe('Rejecting Changes', function () {
sandbox.restore()
})
it('should reject multiple changes in order', function (done) {
DocUpdaterClient.getDoc(
it('should reject multiple changes in order', async function () {
const data = await DocUpdaterClient.getDoc(this.project_id, this.doc.id)
expect(data.ranges.changes).to.have.length(2)
expect(data.lines).to.deep.equal([
'the quick brown fox jumps over the dog',
])
const changeIds = data.ranges.changes.map(change => change.id)
const { rejectedChangeIds } = await DocUpdaterClient.rejectChanges(
this.project_id,
this.doc.id,
(error, res, data) => {
if (error != null) {
throw error
}
expect(data.ranges.changes).to.have.length(2)
expect(data.lines).to.deep.equal([
'the quick brown fox jumps over the dog',
])
const changeIds = data.ranges.changes.map(change => change.id)
DocUpdaterClient.rejectChanges(
this.project_id,
this.doc.id,
changeIds,
this.user_id,
(error, res, body) => {
if (error != null) {
throw error
}
expect(res.statusCode).to.equal(200)
expect(body.rejectedChangeIds).to.be.an('array')
expect(body.rejectedChangeIds).to.have.length(2)
expect(body.rejectedChangeIds).to.include.members(changeIds)
DocUpdaterClient.getDoc(
this.project_id,
this.doc.id,
(error, res, data) => {
if (error != null) {
throw error
}
expect(data.ranges.changes || []).to.have.length(0)
expect(data.lines).to.deep.equal([
'the brown fox jumps over the lazy dog',
])
done()
}
)
}
)
}
changeIds,
this.user_id
)
expect(rejectedChangeIds).to.be.an('array')
expect(rejectedChangeIds).to.have.length(2)
expect(rejectedChangeIds).to.include.members(changeIds)
const data2 = await DocUpdaterClient.getDoc(this.project_id, this.doc.id)
expect(data2.ranges.changes || []).to.have.length(0)
expect(data2.lines).to.deep.equal([
'the brown fox jumps over the lazy dog',
])
})
})
describe('error cases', function () {
beforeEach(function (done) {
beforeEach(async function () {
this.project_id = DocUpdaterClient.randomId()
this.user_id = DocUpdaterClient.randomId()
this.doc = {
@@ -255,46 +188,32 @@ describe('Rejecting Changes', function () {
historyRangesSupport: true,
})
DocUpdaterApp.ensureRunning(done)
await DocUpdaterApp.ensureRunning()
})
it('should handle rejection of non-existent changes gracefully', function (done) {
it('should handle rejection of non-existent changes gracefully', async function () {
const nonExistentChangeId = 'nonexistent_change_id'
DocUpdaterClient.rejectChanges(
const { rejectedChangeIds } = await DocUpdaterClient.rejectChanges(
this.project_id,
this.doc.id,
[nonExistentChangeId],
this.user_id,
(error, res, body) => {
// Should still return 200 with empty rejectedChangeIds if no changes were found to reject
if (error != null) {
throw error
}
expect(res.statusCode).to.equal(200)
expect(body.rejectedChangeIds).to.be.an('array')
expect(body.rejectedChangeIds).to.have.length(0)
done()
}
this.user_id
)
// Should still return 200 with empty rejectedChangeIds if no changes were found to reject
expect(rejectedChangeIds).to.be.an('array')
expect(rejectedChangeIds).to.have.length(0)
})
it('should handle empty change_ids array', function (done) {
DocUpdaterClient.rejectChanges(
it('should handle empty change_ids array', async function () {
const { rejectedChangeIds } = await DocUpdaterClient.rejectChanges(
this.project_id,
this.doc.id,
[],
this.user_id,
(error, res, body) => {
if (error != null) {
throw error
}
expect(res.statusCode).to.equal(200)
expect(body.rejectedChangeIds).to.be.an('array')
expect(body.rejectedChangeIds).to.have.length(0)
done()
}
this.user_id
)
expect(rejectedChangeIds).to.be.an('array')
expect(rejectedChangeIds).to.have.length(0)
})
})
})

512
services/document-updater/test/acceptance/js/SettingADocumentTests.js
View File

@@ -1,5 +1,6 @@
const sinon = require('sinon')
const { expect } = require('chai')
const { setTimeout } = require('node:timers/promises')
const Settings = require('@overleaf/settings')
const docUpdaterRedis = require('@overleaf/redis-wrapper').createClient(
Settings.redis.documentupdater
@@ -10,10 +11,11 @@ const MockProjectHistoryApi = require('./helpers/MockProjectHistoryApi')
const MockWebApi = require('./helpers/MockWebApi')
const DocUpdaterClient = require('./helpers/DocUpdaterClient')
const DocUpdaterApp = require('./helpers/DocUpdaterApp')
const { RequestFailedError } = require('@overleaf/fetch-utils')
describe('Setting a document', function () {
let numberOfReceivedUpdates = 0
before(function (done) {
before(async function () {
DocUpdaterClient.subscribeToAppliedOps(() => {
numberOfReceivedUpdates++
})
@@ -36,7 +38,7 @@ describe('Setting a document', function () {
sinon.spy(MockProjectHistoryApi, 'flushProject')
sinon.spy(MockWebApi, 'setDocument')
DocUpdaterApp.ensureRunning(done)
await DocUpdaterApp.ensureRunning()
})
after(function () {
@@ -45,7 +47,7 @@ describe('Setting a document', function () {
})
describe('when the updated doc exists in the doc updater', function () {
before(function (done) {
before(async function () {
numberOfReceivedUpdates = 0
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
@@ -53,39 +55,21 @@ describe('Setting a document', function () {
lines: this.lines,
version: this.version,
})
DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, error => {
if (error) {
throw error
}
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.update,
error => {
if (error) {
throw error
}
setTimeout(() => {
DocUpdaterClient.setDocLines(
this.project_id,
this.doc_id,
this.newLines,
this.source,
this.user_id,
false,
(error, res, body) => {
if (error) {
return done(error)
}
this.statusCode = res.statusCode
this.body = body
done()
}
)
}, 200)
}
)
})
await DocUpdaterClient.preloadDoc(this.project_id, this.doc_id)
await DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.update
)
await setTimeout(200)
this.body = await DocUpdaterClient.setDocLines(
this.project_id,
this.doc_id,
this.newLines,
this.source,
this.user_id,
false
)
})
after(function () {
@@ -93,10 +77,6 @@ describe('Setting a document', function () {
MockWebApi.setDocument.resetHistory()
})
it('should return a 200 status code', function () {
this.statusCode.should.equal(200)
})
it('should emit two updates (from sendUpdate and setDocLines)', function () {
expect(numberOfReceivedUpdates).to.equal(2)
})
@@ -107,32 +87,14 @@ describe('Setting a document', function () {
.should.equal(true)
})
it('should update the lines in the doc updater', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
if (error) {
return done(error)
}
doc.lines.should.deep.equal(this.newLines)
done()
}
)
it('should update the lines in the doc updater', async function () {
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
doc.lines.should.deep.equal(this.newLines)
})
it('should bump the version in the doc updater', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
if (error) {
return done(error)
}
doc.version.should.equal(this.version + 2)
done()
}
)
it('should bump the version in the doc updater', async function () {
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
doc.version.should.equal(this.version + 2)
})
it('should leave the document in redis', function (done) {
@@ -153,51 +115,33 @@ describe('Setting a document', function () {
})
describe('when doc has the same contents', function () {
beforeEach(function (done) {
beforeEach(async function () {
numberOfReceivedUpdates = 0
DocUpdaterClient.setDocLines(
await DocUpdaterClient.setDocLines(
this.project_id,
this.doc_id,
this.newLines,
this.source,
this.user_id,
false,
(error, res, body) => {
if (error) {
return done(error)
}
this.statusCode = res.statusCode
this.body = body
done()
}
false
)
})
it('should not bump the version in doc updater', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
if (error) {
return done(error)
}
doc.version.should.equal(this.version + 2)
done()
}
)
it('should not bump the version in doc updater', async function () {
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
doc.version.should.equal(this.version + 2)
})
it('should not emit any updates', function (done) {
setTimeout(() => {
expect(numberOfReceivedUpdates).to.equal(0)
done()
}, 100) // delay by 100ms: make sure we do not check too early!
it('should not emit any updates', async function () {
// delay by 100ms: make sure we do not check too early!
await setTimeout(100)
expect(numberOfReceivedUpdates).to.equal(0)
})
})
})
describe('when the updated doc exists in the doc updater (history-ot)', function () {
before(function (done) {
before(async function () {
numberOfReceivedUpdates = 0
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
@@ -212,39 +156,21 @@ describe('Setting a document', function () {
version: this.version,
otMigrationStage: 1,
})
DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, error => {
if (error) {
throw error
}
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.historyOTUpdate,
error => {
if (error) {
throw error
}
setTimeout(() => {
DocUpdaterClient.setDocLines(
this.project_id,
this.doc_id,
this.newLines,
this.source,
this.user_id,
false,
(error, res, body) => {
if (error) {
return done(error)
}
this.statusCode = res.statusCode
this.body = body
done()
}
)
}, 200)
}
)
})
await DocUpdaterClient.preloadDoc(this.project_id, this.doc_id)
await DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.historyOTUpdate
)
await setTimeout(200)
this.body = await DocUpdaterClient.setDocLines(
this.project_id,
this.doc_id,
this.newLines,
this.source,
this.user_id,
false
)
})
after(function () {
@@ -252,10 +178,6 @@ describe('Setting a document', function () {
MockWebApi.setDocument.resetHistory()
})
it('should return a 200 status code', function () {
this.statusCode.should.equal(200)
})
it('should emit two updates (from sendUpdate and setDocLines)', function () {
expect(numberOfReceivedUpdates).to.equal(2)
})
@@ -266,32 +188,14 @@ describe('Setting a document', function () {
.should.equal(true)
})
it('should update the lines in the doc updater', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
if (error) {
return done(error)
}
doc.lines.should.deep.equal(this.newLines)
done()
}
)
it('should update the lines in the doc updater', async function () {
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
doc.lines.should.deep.equal(this.newLines)
})
it('should bump the version in the doc updater', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
if (error) {
return done(error)
}
doc.version.should.equal(this.version + 2)
done()
}
)
it('should bump the version in the doc updater', async function () {
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
doc.version.should.equal(this.version + 2)
})
it('should leave the document in redis', function (done) {
@@ -314,51 +218,33 @@ describe('Setting a document', function () {
})
describe('when doc has the same contents', function () {
beforeEach(function (done) {
beforeEach(async function () {
numberOfReceivedUpdates = 0
DocUpdaterClient.setDocLines(
this.body = await DocUpdaterClient.setDocLines(
this.project_id,
this.doc_id,
this.newLines,
this.source,
this.user_id,
false,
(error, res, body) => {
if (error) {
return done(error)
}
this.statusCode = res.statusCode
this.body = body
done()
}
false
)
})
it('should not bump the version in doc updater', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
if (error) {
return done(error)
}
doc.version.should.equal(this.version + 2)
done()
}
)
it('should not bump the version in doc updater', async function () {
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
doc.version.should.equal(this.version + 2)
})
it('should not emit any updates', function (done) {
setTimeout(() => {
expect(numberOfReceivedUpdates).to.equal(0)
done()
}, 100) // delay by 100ms: make sure we do not check too early!
it('should not emit any updates', async function () {
// delay by 100ms: make sure we do not check too early!
await setTimeout(100)
expect(numberOfReceivedUpdates).to.equal(0)
})
})
})
describe('when the updated doc does not exist in the doc updater', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
numberOfReceivedUpdates = 0
@@ -366,22 +252,15 @@ describe('Setting a document', function () {
lines: this.lines,
version: this.version,
})
DocUpdaterClient.setDocLines(
this.body = await DocUpdaterClient.setDocLines(
this.project_id,
this.doc_id,
this.newLines,
this.source,
this.user_id,
false,
(error, res, body) => {
if (error) {
return done(error)
}
this.statusCode = res.statusCode
this.body = body
setTimeout(done, 200)
}
false
)
await setTimeout(200)
})
after(function () {
@@ -389,10 +268,6 @@ describe('Setting a document', function () {
MockWebApi.setDocument.resetHistory()
})
it('should return a 200 status code', function () {
this.statusCode.should.equal(200)
})
it('should emit an update', function () {
expect(numberOfReceivedUpdates).to.equal(1)
})
@@ -442,7 +317,7 @@ describe('Setting a document', function () {
DOC_TOO_LARGE_TEST_CASES.forEach(testCase => {
describe(testCase.desc, function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
MockWebApi.insertDoc(this.project_id, this.doc_id, {
@@ -453,21 +328,24 @@ describe('Setting a document', function () {
while (JSON.stringify(this.newLines).length <= testCase.size) {
this.newLines.push('(a long line of text)'.repeat(10000))
}
DocUpdaterClient.setDocLines(
this.project_id,
this.doc_id,
this.newLines,
this.source,
this.user_id,
false,
(error, res, body) => {
if (error) {
return done(error)
}
this.statusCode = res.statusCode
setTimeout(done, 200)
try {
await DocUpdaterClient.setDocLines(
this.project_id,
this.doc_id,
this.newLines,
this.source,
this.user_id,
false
)
this.statusCode = 200
} catch (err) {
if (err instanceof RequestFailedError) {
this.statusCode = err.response.status
} else {
throw err
}
)
}
await setTimeout(200)
})
after(function () {
@@ -490,7 +368,7 @@ describe('Setting a document', function () {
})
describe('when the updated doc is large but under the bodyParser and HTTPController size limit', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
MockWebApi.insertDoc(this.project_id, this.doc_id, {
@@ -504,22 +382,15 @@ describe('Setting a document', function () {
this.newLines.push('(a long line of text)'.repeat(10000))
}
this.newLines.pop() // remove the line which took it over the limit
DocUpdaterClient.setDocLines(
this.body = await DocUpdaterClient.setDocLines(
this.project_id,
this.doc_id,
this.newLines,
this.source,
this.user_id,
false,
(error, res, body) => {
if (error) {
return done(error)
}
this.statusCode = res.statusCode
this.body = body
setTimeout(done, 200)
}
false
)
await setTimeout(200)
})
after(function () {
@@ -527,10 +398,6 @@ describe('Setting a document', function () {
MockWebApi.setDocument.resetHistory()
})
it('should return a 200 status code', function () {
this.statusCode.should.equal(200)
})
it('should send the updated doc lines to the web api', function () {
MockWebApi.setDocument
.calledWith(this.project_id, this.doc_id, this.newLines)
@@ -563,44 +430,29 @@ describe('Setting a document', function () {
})
describe('with the undo flag', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
})
DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, error => {
if (error) {
throw error
}
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.update,
error => {
if (error) {
throw error
}
// Go back to old lines, with undo flag
DocUpdaterClient.setDocLines(
this.project_id,
this.doc_id,
this.lines,
this.source,
this.user_id,
true,
(error, res, body) => {
if (error) {
return done(error)
}
this.statusCode = res.statusCode
setTimeout(done, 200)
}
)
}
)
})
await DocUpdaterClient.preloadDoc(this.project_id, this.doc_id)
await DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.update
)
// Go back to old lines, with undo flag
await DocUpdaterClient.setDocLines(
this.project_id,
this.doc_id,
this.lines,
this.source,
this.user_id,
true
)
await setTimeout(200)
})
after(function () {
@@ -608,61 +460,36 @@ describe('Setting a document', function () {
MockWebApi.setDocument.resetHistory()
})
it('should undo the tracked changes', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, data) => {
if (error) {
throw error
}
const { ranges } = data
expect(ranges.changes).to.be.undefined
done()
}
)
it('should undo the tracked changes', async function () {
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
expect(doc.ranges.changes).to.be.undefined
})
})
describe('without the undo flag', function () {
before(function (done) {
before(async function () {
this.project_id = DocUpdaterClient.randomId()
this.doc_id = DocUpdaterClient.randomId()
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
})
DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, error => {
if (error) {
throw error
}
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.update,
error => {
if (error) {
throw error
}
// Go back to old lines, without undo flag
DocUpdaterClient.setDocLines(
this.project_id,
this.doc_id,
this.lines,
this.source,
this.user_id,
false,
(error, res, body) => {
if (error) {
return done(error)
}
this.statusCode = res.statusCode
setTimeout(done, 200)
}
)
}
)
})
await DocUpdaterClient.preloadDoc(this.project_id, this.doc_id)
await DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.update
)
// Go back to old lines, without undo flag
await DocUpdaterClient.setDocLines(
this.project_id,
this.doc_id,
this.lines,
this.source,
this.user_id,
false
)
await setTimeout(200)
})
after(function () {
@@ -670,19 +497,9 @@ describe('Setting a document', function () {
MockWebApi.setDocument.resetHistory()
})
it('should not undo the tracked changes', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, data) => {
if (error) {
throw error
}
const { ranges } = data
expect(ranges.changes.length).to.equal(1)
done()
}
)
it('should not undo the tracked changes', async function () {
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
expect(doc.ranges.changes.length).to.equal(1)
})
})
})
@@ -691,7 +508,8 @@ describe('Setting a document', function () {
const lines = ['one', 'one and a half', 'two', 'three']
const userId = DocUpdaterClient.randomId()
const ts = new Date().toISOString()
beforeEach(function (done) {
beforeEach(async function () {
numberOfReceivedUpdates = 0
this.newLines = ['one', 'two', 'three']
this.project_id = DocUpdaterClient.randomId()
@@ -722,32 +540,20 @@ describe('Setting a document', function () {
version: this.version,
otMigrationStage: 1,
})
DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, error => {
if (error) {
throw error
}
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.historyOTUpdate,
error => {
if (error) {
throw error
}
DocUpdaterClient.waitForPendingUpdates(
this.project_id,
this.doc_id,
done
)
}
)
})
await DocUpdaterClient.preloadDoc(this.project_id, this.doc_id)
await DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.historyOTUpdate
)
await DocUpdaterClient.waitForPendingUpdates(this.doc_id)
})
afterEach(function () {
MockProjectHistoryApi.flushProject.resetHistory()
MockWebApi.setDocument.resetHistory()
})
it('should record tracked changes', function (done) {
docUpdaterRedis.get(
Keys.docLines({ doc_id: this.doc_id }),
@@ -776,19 +582,11 @@ describe('Setting a document', function () {
)
})
it('should apply the change', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, data) => {
if (error) {
throw error
}
expect(data.lines).to.deep.equal(this.newLines)
done()
}
)
it('should apply the change', async function () {
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
expect(doc.lines).to.deep.equal(this.newLines)
})
const cases = [
{
name: 'when resetting the content',
@@ -934,22 +732,14 @@ describe('Setting a document', function () {
for (const { name, lines, want } of cases) {
describe(name, function () {
beforeEach(function (done) {
DocUpdaterClient.setDocLines(
beforeEach(async function () {
this.body = await DocUpdaterClient.setDocLines(
this.project_id,
this.doc_id,
lines,
this.source,
userId,
false,
(error, res, body) => {
if (error) {
return done(error)
}
this.statusCode = res.statusCode
this.body = body
done()
}
false
)
})
it('should update accordingly', function (done) {

123
services/document-updater/test/acceptance/js/SizeCheckTests.js
View File

@@ -1,13 +1,15 @@
const { expect } = require('chai')
const { setTimeout } = require('node:timers/promises')
const Settings = require('@overleaf/settings')
const MockWebApi = require('./helpers/MockWebApi')
const DocUpdaterClient = require('./helpers/DocUpdaterClient')
const DocUpdaterApp = require('./helpers/DocUpdaterApp')
const { RequestFailedError } = require('@overleaf/fetch-utils')
describe('SizeChecks', function () {
before(function (done) {
DocUpdaterApp.ensureRunning(done)
before(async function () {
await DocUpdaterApp.ensureRunning()
})
beforeEach(function () {
this.version = 0
@@ -34,40 +36,27 @@ describe('SizeChecks', function () {
})
})
it('should error when fetching the doc', function (done) {
DocUpdaterClient.getDoc(this.project_id, this.doc_id, (error, res) => {
if (error) return done(error)
expect(res.statusCode).to.equal(500)
done()
})
it('should error when fetching the doc', async function () {
await expect(DocUpdaterClient.getDoc(this.project_id, this.doc_id))
.to.be.rejectedWith(RequestFailedError)
.and.eventually.have.nested.property('response.status', 500)
})
describe('when trying to update', function () {
beforeEach(function (done) {
beforeEach(async function () {
const update = {
doc: this.doc_id,
op: this.update.op,
v: this.version,
}
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
update,
error => {
if (error != null) {
throw error
}
setTimeout(done, 200)
}
)
await DocUpdaterClient.sendUpdate(this.project_id, this.doc_id, update)
await setTimeout(200)
})
it('should still error when fetching the doc', function (done) {
DocUpdaterClient.getDoc(this.project_id, this.doc_id, (error, res) => {
if (error) return done(error)
expect(res.statusCode).to.equal(500)
done()
})
it('should still error when fetching the doc', async function () {
await expect(DocUpdaterClient.getDoc(this.project_id, this.doc_id))
.to.be.rejectedWith(RequestFailedError)
.and.eventually.have.nested.property('response.status', 500)
})
})
})
@@ -91,48 +80,25 @@ describe('SizeChecks', function () {
})
})
it('should be able to fetch the doc', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
if (error) return done(error)
expect(doc.lines).to.deep.equal(this.lines)
done()
}
)
it('should be able to fetch the doc', async function () {
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
expect(doc.lines).to.deep.equal(this.lines)
})
describe('when trying to update', function () {
beforeEach(function (done) {
beforeEach(async function () {
const update = {
doc: this.doc_id,
op: this.update.op,
v: this.version,
}
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
update,
error => {
if (error != null) {
throw error
}
setTimeout(done, 200)
}
)
await DocUpdaterClient.sendUpdate(this.project_id, this.doc_id, update)
await setTimeout(200)
})
it('should not update the doc', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
if (error) return done(error)
expect(doc.lines).to.deep.equal(this.lines)
done()
}
)
it('should not update the doc', async function () {
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
expect(doc.lines).to.deep.equal(this.lines)
})
})
})
@@ -146,48 +112,25 @@ describe('SizeChecks', function () {
})
})
it('should be able to fetch the doc', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
if (error) return done(error)
expect(doc.lines).to.deep.equal(this.lines)
done()
}
)
it('should be able to fetch the doc', async function () {
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
expect(doc.lines).to.deep.equal(this.lines)
})
describe('when trying to update', function () {
beforeEach(function (done) {
beforeEach(async function () {
const update = {
doc: this.doc_id,
op: this.update.op,
v: this.version,
}
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
update,
error => {
if (error != null) {
throw error
}
setTimeout(done, 200)
}
)
await DocUpdaterClient.sendUpdate(this.project_id, this.doc_id, update)
await setTimeout(200)
})
it('should not update the doc', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
if (error) return done(error)
expect(doc.lines).to.deep.equal(this.lines)
done()
}
)
it('should not update the doc', async function () {
const doc = await DocUpdaterClient.getDoc(this.project_id, this.doc_id)
expect(doc.lines).to.deep.equal(this.lines)
})
})
})

56
services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js
View File

@@ -1,42 +1,26 @@
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS205: Consider reworking code to avoid use of IIFEs
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const app = require('../../../../app')
module.exports = {
running: false,
initing: false,
callbacks: [],
ensureRunning(callback) {
if (callback == null) {
callback = function () {}
}
if (this.running) {
return callback()
} else if (this.initing) {
return this.callbacks.push(callback)
}
this.initing = true
this.callbacks.push(callback)
function startApp() {
return new Promise((resolve, reject) => {
app.listen(3003, '127.0.0.1', error => {
if (error != null) {
throw error
if (error) {
reject(error)
} else {
resolve()
}
this.running = true
return (() => {
const result = []
for (callback of Array.from(this.callbacks)) {
result.push(callback())
}
return result
})()
})
},
})
}
let appStartedPromise
async function ensureRunning() {
if (!appStartedPromise) {
appStartedPromise = startApp()
}
await appStartedPromise
}
module.exports = {
ensureRunning,
}

348
services/document-updater/test/acceptance/js/helpers/DocUpdaterClient.js
View File

@@ -5,8 +5,8 @@ const rclient = require('@overleaf/redis-wrapper').createClient(
Settings.redis.documentupdater
)
const keys = Settings.redis.documentupdater.key_schema
const request = require('request').defaults({ jar: false })
const async = require('async')
const { fetchJson, fetchNothing } = require('@overleaf/fetch-utils')
const { setTimeout } = require('node:timers/promises')
const rclientSub = require('@overleaf/redis-wrapper').createClient(
Settings.redis.pubsub
@@ -14,6 +14,15 @@ const rclientSub = require('@overleaf/redis-wrapper').createClient(
rclientSub.subscribe('applied-ops')
rclientSub.setMaxListeners(0)
function getPendingUpdateListKey() {
const shard = _.random(0, Settings.dispatcherCount - 1)
if (shard === 0) {
return 'pending-updates-list'
} else {
return `pending-updates-list-${shard}`
}
}
module.exports = DocUpdaterClient = {
randomId() {
let str = ''
@@ -23,234 +32,177 @@ module.exports = DocUpdaterClient = {
return str
},
subscribeToAppliedOps(callback) {
rclientSub.on('message', callback)
subscribeToAppliedOps(messageHandler) {
rclientSub.on('message', messageHandler)
},
_getPendingUpdateListKey() {
const shard = _.random(0, Settings.dispatcherCount - 1)
if (shard === 0) {
return 'pending-updates-list'
} else {
return `pending-updates-list-${shard}`
}
},
sendUpdate(projectId, docId, update, callback) {
rclient.rpush(
async sendUpdate(projectId, docId, update) {
const docKey = `${projectId}:${docId}`
await rclient.rpush(
keys.pendingUpdates({ doc_id: docId }),
JSON.stringify(update),
error => {
if (error) {
return callback(error)
}
const docKey = `${projectId}:${docId}`
rclient.sadd('DocsWithPendingUpdates', docKey, error => {
if (error) {
return callback(error)
}
JSON.stringify(update)
)
await rclient.sadd('DocsWithPendingUpdates', docKey)
await rclient.rpush(getPendingUpdateListKey(), docKey)
},
rclient.rpush(
DocUpdaterClient._getPendingUpdateListKey(),
docKey,
callback
)
})
async sendUpdates(projectId, docId, updates) {
await DocUpdaterClient.preloadDoc(projectId, docId)
for (const update of updates) {
await DocUpdaterClient.sendUpdate(projectId, docId, update)
}
await DocUpdaterClient.waitForPendingUpdates(docId)
},
async waitForPendingUpdates(docId) {
const maxRetries = 30
const retryInterval = 100
for (let attempt = 0; attempt < maxRetries; attempt++) {
const length = await rclient.llen(keys.pendingUpdates({ doc_id: docId }))
if (length === 0) {
return // Success - no pending updates
}
if (attempt < maxRetries - 1) {
await setTimeout(retryInterval)
}
}
throw new Error('updates still pending after maximum retries')
},
async getDoc(projectId, docId) {
return await fetchJson(
`http://127.0.0.1:3003/project/${projectId}/doc/${docId}`
)
},
sendUpdates(projectId, docId, updates, callback) {
DocUpdaterClient.preloadDoc(projectId, docId, error => {
if (error) {
return callback(error)
}
const jobs = updates.map(update => callback => {
DocUpdaterClient.sendUpdate(projectId, docId, update, callback)
})
async.series(jobs, err => {
if (err) {
return callback(err)
}
DocUpdaterClient.waitForPendingUpdates(projectId, docId, callback)
})
})
},
waitForPendingUpdates(projectId, docId, callback) {
async.retry(
{ times: 30, interval: 100 },
cb =>
rclient.llen(keys.pendingUpdates({ doc_id: docId }), (err, length) => {
if (err) {
return cb(err)
}
if (length > 0) {
cb(new Error('updates still pending'))
} else {
cb()
}
}),
callback
async getDocAndRecentOps(projectId, docId, fromVersion) {
return await fetchJson(
`http://127.0.0.1:3003/project/${projectId}/doc/${docId}?fromVersion=${fromVersion}`
)
},
getDoc(projectId, docId, callback) {
request.get(
`http://127.0.0.1:3003/project/${projectId}/doc/${docId}`,
(error, res, body) => {
if (body != null && res.statusCode >= 200 && res.statusCode < 300) {
body = JSON.parse(body)
}
callback(error, res, body)
}
async getProjectLastUpdatedAt(projectId) {
return await fetchJson(
`http://127.0.0.1:3003/project/${projectId}/last_updated_at`
)
},
getDocAndRecentOps(projectId, docId, fromVersion, callback) {
request.get(
`http://127.0.0.1:3003/project/${projectId}/doc/${docId}?fromVersion=${fromVersion}`,
(error, res, body) => {
if (body != null && res.statusCode >= 200 && res.statusCode < 300) {
body = JSON.parse(body)
}
callback(error, res, body)
}
async preloadDoc(projectId, docId) {
await DocUpdaterClient.getDoc(projectId, docId)
},
async peekDoc(projectId, docId) {
return await fetchJson(
`http://127.0.0.1:3003/project/${projectId}/doc/${docId}/peek`
)
},
getProjectLastUpdatedAt(projectId, callback) {
request.get(
`http://127.0.0.1:3003/project/${projectId}/last_updated_at`,
(error, res, body) => {
if (body != null && res.statusCode >= 200 && res.statusCode < 300) {
body = JSON.parse(body)
}
callback(error, res, body)
}
)
},
preloadDoc(projectId, docId, callback) {
DocUpdaterClient.getDoc(projectId, docId, callback)
},
peekDoc(projectId, docId, callback) {
request.get(
`http://127.0.0.1:3003/project/${projectId}/doc/${docId}/peek`,
(error, res, body) => {
if (body != null && res.statusCode >= 200 && res.statusCode < 300) {
body = JSON.parse(body)
}
callback(error, res, body)
}
)
},
flushDoc(projectId, docId, callback) {
request.post(
async flushDoc(projectId, docId) {
return await fetchNothing(
`http://127.0.0.1:3003/project/${projectId}/doc/${docId}/flush`,
(error, res, body) => callback(error, res, body)
{ method: 'POST' }
)
},
setDocLines(projectId, docId, lines, source, userId, undoing, callback) {
request.post(
async setDocLines(projectId, docId, lines, source, userId, undoing) {
return await fetchJson(
`http://127.0.0.1:3003/project/${projectId}/doc/${docId}`,
{
url: `http://127.0.0.1:3003/project/${projectId}/doc/${docId}`,
method: 'POST',
json: {
lines,
source,
user_id: userId,
undoing,
},
},
(error, res, body) => callback(error, res, body)
)
},
deleteDoc(projectId, docId, callback) {
request.del(
`http://127.0.0.1:3003/project/${projectId}/doc/${docId}`,
(error, res, body) => callback(error, res, body)
)
},
flushProject(projectId, callback) {
request.post(`http://127.0.0.1:3003/project/${projectId}/flush`, callback)
},
deleteProject(projectId, callback) {
request.del(`http://127.0.0.1:3003/project/${projectId}`, callback)
},
deleteProjectOnShutdown(projectId, callback) {
request.del(
`http://127.0.0.1:3003/project/${projectId}?background=true&shutdown=true`,
callback
)
},
flushOldProjects(callback) {
request.get(
'http://127.0.0.1:3003/flush_queued_projects?min_delete_age=1',
callback
)
},
acceptChange(projectId, docId, changeId, callback) {
request.post(
`http://127.0.0.1:3003/project/${projectId}/doc/${docId}/change/${changeId}/accept`,
callback
)
},
acceptChanges(projectId, docId, changeIds, callback) {
request.post(
{
url: `http://127.0.0.1:3003/project/${projectId}/doc/${docId}/change/accept`,
json: { change_ids: changeIds },
},
callback
)
},
rejectChanges(projectId, docId, changeIds, userId, callback) {
request.post(
{
url: `http://127.0.0.1:3003/project/${projectId}/doc/${docId}/change/reject`,
json: { change_ids: changeIds, user_id: userId },
},
callback
)
},
removeComment(projectId, docId, comment, callback) {
request.del(
`http://127.0.0.1:3003/project/${projectId}/doc/${docId}/comment/${comment}`,
callback
)
},
getProjectDocs(projectId, projectStateHash, callback) {
request.get(
`http://127.0.0.1:3003/project/${projectId}/doc?state=${projectStateHash}`,
(error, res, body) => {
if (body != null && res.statusCode >= 200 && res.statusCode < 300) {
body = JSON.parse(body)
}
callback(error, res, body)
}
)
},
sendProjectUpdate(projectId, userId, updates, version, callback) {
request.post(
{
url: `http://127.0.0.1:3003/project/${projectId}`,
json: { userId, updates, version },
},
(error, res, body) => callback(error, res, body)
async deleteDoc(projectId, docId) {
return await fetchNothing(
`http://127.0.0.1:3003/project/${projectId}/doc/${docId}`,
{ method: 'DELETE' }
)
},
async flushProject(projectId) {
return await fetchNothing(
`http://127.0.0.1:3003/project/${projectId}/flush`,
{
method: 'POST',
}
)
},
async deleteProject(projectId) {
return await fetchNothing(`http://127.0.0.1:3003/project/${projectId}`, {
method: 'DELETE',
})
},
async deleteProjectOnShutdown(projectId) {
return await fetchNothing(
`http://127.0.0.1:3003/project/${projectId}?background=true&shutdown=true`,
{
method: 'DELETE',
}
)
},
async flushOldProjects() {
await fetchNothing(
'http://127.0.0.1:3003/flush_queued_projects?min_delete_age=1'
)
},
async acceptChange(projectId, docId, changeId) {
await fetchNothing(
`http://127.0.0.1:3003/project/${projectId}/doc/${docId}/change/${changeId}/accept`,
{ method: 'POST' }
)
},
async acceptChanges(projectId, docId, changeIds) {
await fetchNothing(
`http://127.0.0.1:3003/project/${projectId}/doc/${docId}/change/accept`,
{
method: 'POST',
json: { change_ids: changeIds },
}
)
},
async rejectChanges(projectId, docId, changeIds, userId) {
return await fetchJson(
`http://127.0.0.1:3003/project/${projectId}/doc/${docId}/change/reject`,
{
method: 'POST',
json: { change_ids: changeIds, user_id: userId },
}
)
},
async removeComment(projectId, docId, comment) {
await fetchNothing(
`http://127.0.0.1:3003/project/${projectId}/doc/${docId}/comment/${comment}`,
{ method: 'DELETE' }
)
},
async getProjectDocs(projectId, projectStateHash) {
return await fetchJson(
`http://127.0.0.1:3003/project/${projectId}/doc?state=${projectStateHash}`
)
},
async sendProjectUpdate(projectId, userId, updates, version) {
await fetchNothing(`http://127.0.0.1:3003/project/${projectId}`, {
method: 'POST',
json: { userId, updates, version },
})
},
}

387
services/document-updater/test/stress/js/run.js
View File

@@ -1,387 +0,0 @@
/* eslint-disable
no-return-assign,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS202: Simplify dynamic range loops
* DS205: Consider reworking code to avoid use of IIFEs
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const DocUpdaterClient = require('../../acceptance/js/helpers/DocUpdaterClient')
// MockWebApi = require "../../acceptance/js/helpers/MockWebApi"
const assert = require('node:assert')
const async = require('async')
const insert = function (string, pos, content) {
const result = string.slice(0, pos) + content + string.slice(pos)
return result
}
const transform = function (op1, op2) {
if (op2.p < op1.p) {
return {
p: op1.p + op2.i.length,
i: op1.i,
}
} else {
return op1
}
}
class StressTestClient {
constructor(options) {
if (options == null) {
options = {}
}
this.options = options
if (this.options.updateDelay == null) {
this.options.updateDelay = 200
}
this.project_id = this.options.project_id || DocUpdaterClient.randomId()
this.doc_id = this.options.doc_id || DocUpdaterClient.randomId()
this.pos = this.options.pos || 0
this.content = this.options.content || ''
this.client_id = DocUpdaterClient.randomId()
this.version = this.options.version || 0
this.inflight_op = null
this.charCode = 0
this.counts = {
conflicts: 0,
local_updates: 0,
remote_updates: 0,
max_delay: 0,
}
DocUpdaterClient.subscribeToAppliedOps((channel, update) => {
update = JSON.parse(update)
if (update.error != null) {
console.error(new Error(`Error from server: '${update.error}'`))
return
}
if (update.doc_id === this.doc_id) {
return this.processReply(update)
}
})
}
sendUpdate() {
const data = String.fromCharCode(65 + (this.charCode++ % 26))
this.content = insert(this.content, this.pos, data)
this.inflight_op = {
i: data,
p: this.pos++,
}
this.resendUpdate()
return (this.inflight_op_sent = Date.now())
}
resendUpdate() {
assert(this.inflight_op != null)
DocUpdaterClient.sendUpdate(this.project_id, this.doc_id, {
doc: this.doc_id,
op: [this.inflight_op],
v: this.version,
meta: {
source: this.client_id,
},
dupIfSource: [this.client_id],
})
return (this.update_timer = setTimeout(() => {
console.log(
`[${new Date()}] \t[${this.client_id.slice(
0,
4
)}] WARN: Resending update after 5 seconds`
)
return this.resendUpdate()
}, 5000))
}
processReply(update) {
if (update.op.v !== this.version) {
if (update.op.v < this.version) {
console.log(
`[${new Date()}] \t[${this.client_id.slice(
0,
4
)}] WARN: Duplicate ack (already seen version)`
)
return
} else {
console.error(
`[${new Date()}] \t[${this.client_id.slice(
0,
4
)}] ERROR: Version jumped ahead (client: ${this.version}, op: ${
update.op.v
})`
)
}
}
this.version++
if (update.op.meta.source === this.client_id) {
if (this.inflight_op != null) {
this.counts.local_updates++
this.inflight_op = null
clearTimeout(this.update_timer)
const delay = Date.now() - this.inflight_op_sent
this.counts.max_delay = Math.max(this.counts.max_delay, delay)
return this.continue()
} else {
return console.log(
`[${new Date()}] \t[${this.client_id.slice(
0,
4
)}] WARN: Duplicate ack`
)
}
} else {
assert(update.op.op.length === 1)
this.counts.remote_updates++
let externalOp = update.op.op[0]
if (this.inflight_op != null) {
this.counts.conflicts++
this.inflight_op = transform(this.inflight_op, externalOp)
externalOp = transform(externalOp, this.inflight_op)
}
if (externalOp.p < this.pos) {
this.pos += externalOp.i.length
}
return (this.content = insert(this.content, externalOp.p, externalOp.i))
}
}
continue() {
if (this.updateCount > 0) {
this.updateCount--
return setTimeout(
() => {
return this.sendUpdate()
},
this.options.updateDelay * (0.5 + Math.random())
)
} else {
return this.updateCallback()
}
}
runForNUpdates(n, callback) {
if (callback == null) {
callback = function () {}
}
this.updateCallback = callback
this.updateCount = n
return this.continue()
}
check(callback) {
if (callback == null) {
callback = function () {}
}
return DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, body) => {
if (error != null) {
throw error
}
if (body.lines == null) {
return console.error(
`[${new Date()}] \t[${this.client_id.slice(
0,
4
)}] ERROR: Invalid response from get doc (${this.doc_id})`,
body
)
}
const content = body.lines.join('\n')
const { version } = body
if (content !== this.content) {
if (version === this.version) {
console.error(
`[${new Date()}] \t[${this.client_id.slice(
0,
4
)}] Error: Client content does not match server.`
)
console.error(`Server: ${content.split('a')}`)
console.error(`Client: ${this.content.split('a')}`)
} else {
console.error(
`[${new Date()}] \t[${this.client_id.slice(
0,
4
)}] Error: Version mismatch (Server: '${version}', Client: '${
this.version
}')`
)
}
}
if (!this.isContentValid(this.content)) {
const iterable = this.content.split('')
for (let i = 0; i < iterable.length; i++) {
const chunk = iterable[i]
if (chunk != null && chunk !== 'a') {
console.log(chunk, i)
}
}
throw new Error('bad content')
}
return callback()
}
)
}
isChunkValid(chunk) {
const char = 0
for (let i = 0; i < chunk.length; i++) {
const letter = chunk[i]
if (letter.charCodeAt(0) !== 65 + (i % 26)) {
console.error(
`[${new Date()}] \t[${this.client_id.slice(0, 4)}] Invalid Chunk:`,
chunk
)
return false
}
}
return true
}
isContentValid(content) {
for (const chunk of Array.from(content.split('a'))) {
if (chunk != null && chunk !== '') {
if (!this.isChunkValid(chunk)) {
console.error(
`[${new Date()}] \t[${this.client_id.slice(0, 4)}] Invalid content`,
content
)
return false
}
}
}
return true
}
}
const checkDocument = function (projectId, docId, clients, callback) {
if (callback == null) {
callback = function () {}
}
const jobs = clients.map(client => cb => client.check(cb))
return async.parallel(jobs, callback)
}
const printSummary = function (docId, clients) {
const slot = require('cluster-key-slot')
const now = new Date()
console.log(
`[${now}] [${docId.slice(0, 4)} (slot: ${slot(docId)})] ${
clients.length
} clients...`
)
return (() => {
const result = []
for (const client of Array.from(clients)) {
console.log(
`[${now}] \t[${client.client_id.slice(0, 4)}] { local: ${
client.counts.local_updates
}, remote: ${client.counts.remote_updates}, conflicts: ${
client.counts.conflicts
}, max_delay: ${client.counts.max_delay} }`
)
result.push(
(client.counts = {
local_updates: 0,
remote_updates: 0,
conflicts: 0,
max_delay: 0,
})
)
}
return result
})()
}
const CLIENT_COUNT = parseInt(process.argv[2], 10)
const UPDATE_DELAY = parseInt(process.argv[3], 10)
const SAMPLE_INTERVAL = parseInt(process.argv[4], 10)
for (const docAndProjectId of Array.from(process.argv.slice(5))) {
;(function (docAndProjectId) {
const [projectId, docId] = Array.from(docAndProjectId.split(':'))
console.log({ projectId, docId })
return DocUpdaterClient.setDocLines(
projectId,
docId,
[new Array(CLIENT_COUNT + 2).join('a')],
null,
null,
error => {
if (error != null) {
throw error
}
return DocUpdaterClient.getDoc(projectId, docId, (error, res, body) => {
let runBatch
if (error != null) {
throw error
}
if (body.lines == null) {
return console.error(
`[${new Date()}] ERROR: Invalid response from get doc (${docId})`,
body
)
}
const content = body.lines.join('\n')
const { version } = body
const clients = []
for (
let pos = 1, end = CLIENT_COUNT, asc = end >= 1;
asc ? pos <= end : pos >= end;
asc ? pos++ : pos--
) {
;(function (pos) {
const client = new StressTestClient({
doc_id: docId,
project_id: projectId,
content,
pos,
version,
updateDelay: UPDATE_DELAY,
})
return clients.push(client)
})(pos)
}
return (runBatch = function () {
const jobs = clients.map(
client => cb =>
client.runForNUpdates(SAMPLE_INTERVAL / UPDATE_DELAY, cb)
)
return async.parallel(jobs, error => {
if (error != null) {
throw error
}
printSummary(docId, clients)
return checkDocument(projectId, docId, clients, error => {
if (error != null) {
throw error
}
return runBatch()
})
})
})()
})
}
)
})(docAndProjectId)
}