Feat/changeset ts (#6594)

* Migrated changeset

* Added more tests.

* Fixed test scopes

src/tests/backend-new/easysync-helper.ts

@@ -0,0 +1,220 @@
+import AttributePool from "../../static/js/AttributePool";
+import { Attribute } from "../../static/js/types/Attribute";
+import {StringAssembler} from "../../static/js/StringAssembler";
+import {SmartOpAssembler} from "../../static/js/SmartOpAssembler";
+import Op from "../../static/js/Op";
+import {numToString} from "../../static/js/ChangesetUtils";
+import {checkRep, pack} from "../../static/js/Changeset";
+
+export const poolOrArray = (attribs: any) => {
+  if (attribs.getAttrib) {
+    return attribs; // it's already an attrib pool
+  } else {
+    // assume it's an array of attrib strings to be split and added
+    const p = new AttributePool();
+    attribs.forEach((kv: { split: (arg0: string) => Attribute; }) => {
+      p.putAttrib(kv.split(','));
+    });
+    return p;
+  }
+};
+const randInt = (maxValue: number) => Math.floor(Math.random() * maxValue);
+const randomInlineString = (len: number) => {
+  const assem = new StringAssembler();
+  for (let i = 0; i < len; i++) {
+    assem.append(String.fromCharCode(randInt(26) + 97));
+  }
+  return assem.toString();
+};
+export const randomMultiline = (approxMaxLines: number, approxMaxCols: number) => {
+  const numParts = randInt(approxMaxLines * 2) + 1;
+  const txt = new StringAssembler();
+  txt.append(randInt(2) ? '\n' : '');
+  for (let i = 0; i < numParts; i++) {
+    if ((i % 2) === 0) {
+      if (randInt(10)) {
+        txt.append(randomInlineString(randInt(approxMaxCols) + 1));
+      } else {
+        txt.append('\n');
+      }
+    } else {
+      txt.append('\n');
+    }
+  }
+  return txt.toString();
+};
+
+const randomTwoPropAttribs = (opcode: "" | "=" | "+" | "-") => {
+  // assumes attrib pool like ['apple,','apple,true','banana,','banana,true']
+  if (opcode === '-' || randInt(3)) {
+    return '';
+  } else if (randInt(3)) { // eslint-disable-line no-dupe-else-if
+    if (opcode === '+' || randInt(2)) {
+      return `*${numToString(randInt(2) * 2 + 1)}`;
+    } else {
+      return `*${numToString(randInt(2) * 2)}`;
+    }
+  } else if (opcode === '+' || randInt(4) === 0) {
+    return '*1*3';
+  } else {
+    return ['*0*2', '*0*3', '*1*2'][randInt(3)];
+  }
+};
+
+
+const randomStringOperation = (numCharsLeft: number) => {
+  let result;
+  switch (randInt(11)) {
+    case 0:
+    {
+      // insert char
+      result = {
+        insert: randomInlineString(1),
+      };
+      break;
+    }
+    case 1:
+    {
+      // delete char
+      result = {
+        remove: 1,
+      };
+      break;
+    }
+    case 2:
+    {
+      // skip char
+      result = {
+        skip: 1,
+      };
+      break;
+    }
+    case 3:
+    {
+      // insert small
+      result = {
+        insert: randomInlineString(randInt(4) + 1),
+      };
+      break;
+    }
+    case 4:
+    {
+      // delete small
+      result = {
+        remove: randInt(4) + 1,
+      };
+      break;
+    }
+    case 5:
+    {
+      // skip small
+      result = {
+        skip: randInt(4) + 1,
+      };
+      break;
+    }
+    case 6:
+    {
+      // insert multiline;
+      result = {
+        insert: randomMultiline(5, 20),
+      };
+      break;
+    }
+    case 7:
+    {
+      // delete multiline
+      result = {
+        remove: Math.round(numCharsLeft * Math.random() * Math.random()),
+      };
+      break;
+    }
+    case 8:
+    {
+      // skip multiline
+      result = {
+        skip: Math.round(numCharsLeft * Math.random() * Math.random()),
+      };
+      break;
+    }
+    case 9:
+    {
+      // delete to end
+      result = {
+        remove: numCharsLeft,
+      };
+      break;
+    }
+    case 10:
+    {
+      // skip to end
+      result = {
+        skip: numCharsLeft,
+      };
+      break;
+    }
+  }
+  const maxOrig = numCharsLeft - 1;
+  if ('remove' in result!) {
+    result.remove = Math.min(result.remove, maxOrig);
+  } else if ('skip' in result!) {
+    result.skip = Math.min(result.skip, maxOrig);
+  }
+  return result;
+};
+
+export const randomTestChangeset = (origText: string, withAttribs?: any) => {
+  const charBank = new StringAssembler();
+  let textLeft = origText; // always keep final newline
+  const outTextAssem = new StringAssembler();
+  const opAssem = new SmartOpAssembler();
+  const oldLen = origText.length;
+
+  const nextOp = new Op();
+
+  const appendMultilineOp = (opcode: "" | "=" | "+" | "-", txt: string) => {
+    nextOp.opcode = opcode;
+    if (withAttribs) {
+      nextOp.attribs = randomTwoPropAttribs(opcode);
+    }
+    txt.replace(/\n|[^\n]+/g, (t) => {
+      if (t === '\n') {
+        nextOp.chars = 1;
+        nextOp.lines = 1;
+        opAssem.append(nextOp);
+      } else {
+        nextOp.chars = t.length;
+        nextOp.lines = 0;
+        opAssem.append(nextOp);
+      }
+      return '';
+    });
+  };
+
+  const doOp = () => {
+    const o = randomStringOperation(textLeft.length);
+    if (o!.insert) {
+      const txt = o!.insert;
+      charBank.append(txt);
+      outTextAssem.append(txt);
+      appendMultilineOp('+', txt);
+    } else if (o!.skip) {
+      const txt = textLeft.substring(0, o!.skip);
+      textLeft = textLeft.substring(o!.skip);
+      outTextAssem.append(txt);
+      appendMultilineOp('=', txt);
+    } else if (o!.remove) {
+      const txt = textLeft.substring(0, o!.remove);
+      textLeft = textLeft.substring(o!.remove);
+      appendMultilineOp('-', txt);
+    }
+  };
+
+  while (textLeft.length > 1) doOp();
+  for (let i = 0; i < 5; i++) doOp(); // do some more (only insertions will happen)
+  const outText = `${outTextAssem.toString()}\n`;
+  opAssem.endDocument();
+  const cs = pack(oldLen, outText.length, opAssem.toString(), charBank.toString());
+  checkRep(cs);
+  return [cs, outText];
+};

src/tests/backend-new/specs/StringIteratorTest.ts

@@ -0,0 +1,47 @@
+import {expect, describe, it} from 'vitest'
+import {StringIterator} from "../../../static/js/StringIterator";
+
+
+describe('Test string iterator take', function () {
+  it('should iterate over a string', async function () {
+    const str = 'Hello, world!';
+    const iter = new StringIterator(str);
+    let i = 0;
+    while (iter.remaining() > 0) {
+      expect(iter.remaining()).to.equal(str.length - i);
+      console.error(iter.remaining());
+      expect(iter.take(1)).to.equal(str.charAt(i));
+      i++;
+    }
+  });
+})
+
+
+describe('Test string iterator peek', function () {
+  it('should peek over a string', async function () {
+    const str = 'Hello, world!';
+    const iter = new StringIterator(str);
+    let i = 0;
+    while (iter.remaining() > 0) {
+      expect(iter.remaining()).to.equal(str.length - i);
+      expect(iter.peek(1)).to.equal(str.charAt(i));
+      i++;
+      iter.skip(1);
+    }
+  });
+})
+
+describe('Test string iterator skip', function () {
+  it('should throw error when skip over a string too long', async function () {
+    const str = 'Hello, world!';
+    const iter = new StringIterator(str);
+    expect(()=>iter.skip(1000)).toThrowError();
+  });
+
+  it('should skip over a string', async function () {
+    const str = 'Hello, world!';
+    const iter = new StringIterator(str);
+    iter.skip(7);
+    expect(iter.take(1)).to.equal('w');
+  });
+})

src/tests/backend-new/specs/easysync-assembler.ts

@@ -0,0 +1,224 @@
+'use strict';
+
+import {deserializeOps, opsFromAText} from '../../../static/js/Changeset';
+import padutils from '../../../static/js/pad_utils';
+import {poolOrArray} from '../easysync-helper.js';
+
+import {describe, it, expect} from 'vitest'
+import {OpAssembler} from "../../../static/js/OpAssembler";
+import {SmartOpAssembler} from "../../../static/js/SmartOpAssembler";
+import Op from "../../../static/js/Op";
+
+
+describe('easysync-assembler', function () {
+  it('opAssembler', async function () {
+    const x = '-c*3*4+6|3=az*asdf0*1*2*3+1=1-1+1*0+1=1-1+1|c=c-1';
+    const assem = new OpAssembler();
+    var opLength = 0
+    for (const op of deserializeOps(x)){
+      console.log(op)
+      assem.append(op);
+      opLength++
+    }
+    expect(assem.toString()).to.equal(x);
+  });
+
+  it('smartOpAssembler', async function () {
+    const x = '-c*3*4+6|3=az*asdf0*1*2*3+1=1-1+1*0+1=1-1+1|c=c-1';
+    const assem = new SmartOpAssembler();
+    for (const op of deserializeOps(x)) assem.append(op);
+    assem.endDocument();
+    expect(assem.toString()).to.equal(x);
+  });
+
+  it('smartOpAssembler ignore additional pure keeps (no attributes)', async function () {
+    const x = '-c*3*4+6|1+1=5';
+    const assem = new SmartOpAssembler();
+    for (const op of deserializeOps(x)) assem.append(op);
+    assem.endDocument();
+    expect(assem.toString()).to.equal('-c*3*4+6|1+1');
+  });
+
+  it('smartOpAssembler merge consecutive + ops without multiline', async function () {
+    const x = '-c*3*4+6*3*4+1*3*4+9=5';
+    const assem = new SmartOpAssembler();
+    for (const op of deserializeOps(x)) assem.append(op);
+    assem.endDocument();
+    expect(assem.toString()).to.equal('-c*3*4+g');
+  });
+
+  it('smartOpAssembler merge consecutive + ops with multiline', async function () {
+    const x = '-c*3*4+6*3*4|1+1*3*4|9+f*3*4+k=5';
+    const assem = new SmartOpAssembler();
+    for (const op of deserializeOps(x)) assem.append(op);
+    assem.endDocument();
+    expect(assem.toString()).to.equal('-c*3*4|a+m*3*4+k');
+  });
+
+  it('smartOpAssembler merge consecutive - ops without multiline', async function () {
+    const x = '-c-6-1-9=5';
+    const assem = new SmartOpAssembler();
+    for (const op of deserializeOps(x)) assem.append(op);
+    assem.endDocument();
+    expect(assem.toString()).to.equal('-s');
+  });
+
+  it('smartOpAssembler merge consecutive - ops with multiline', async function () {
+    const x = '-c-6|1-1|9-f-k=5';
+    const assem = new SmartOpAssembler();
+    for (const op of deserializeOps(x)) assem.append(op);
+    assem.endDocument();
+    expect(assem.toString()).to.equal('|a-y-k');
+  });
+
+  it('smartOpAssembler merge consecutive = ops without multiline', async function () {
+    const x = '-c*3*4=6*2*4=1*3*4=f*3*4=2*3*4=a=k=5';
+    const assem = new SmartOpAssembler();
+    for (const op of deserializeOps(x)) assem.append(op);
+    assem.endDocument();
+    expect(assem.toString()).to.equal('-c*3*4=6*2*4=1*3*4=r');
+  });
+
+  it('smartOpAssembler merge consecutive = ops with multiline', async function () {
+    const x = '-c*3*4=6*2*4|1=1*3*4|9=f*3*4|2=2*3*4=a*3*4=1=k=5';
+    const assem = new SmartOpAssembler();
+    for (const op of deserializeOps(x)) assem.append(op);
+    assem.endDocument();
+    expect(assem.toString()).to.equal('-c*3*4=6*2*4|1=1*3*4|b=h*3*4=b');
+  });
+
+  it('smartOpAssembler ignore + ops with ops.chars === 0', async function () {
+    const x = '-c*3*4+6*3*4+0*3*4+1+0*3*4+1';
+    const assem = new SmartOpAssembler();
+    for (const op of deserializeOps(x)) assem.append(op);
+    assem.endDocument();
+    expect(assem.toString()).to.equal('-c*3*4+8');
+  });
+
+  it('smartOpAssembler ignore - ops with ops.chars === 0', async function () {
+    const x = '-c-6-0-1-0-1';
+    const assem = new SmartOpAssembler();
+    for (const op of deserializeOps(x)) assem.append(op);
+    assem.endDocument();
+    expect(assem.toString()).to.equal('-k');
+  });
+
+  it('smartOpAssembler append + op with text', async function () {
+    const assem = new SmartOpAssembler();
+    const pool = poolOrArray([
+      'attr1,1',
+      'attr2,2',
+      'attr3,3',
+      'attr4,4',
+      'attr5,5',
+    ]);
+
+    padutils.warnDeprecatedFlags.disabledForTestingOnly = true;
+    try {
+      assem.appendOpWithText('+', 'test', '*3*4*5', pool);
+      assem.appendOpWithText('+', 'test', '*3*4*5', pool);
+      assem.appendOpWithText('+', 'test', '*1*4*5', pool);
+    } finally {
+      // @ts-ignore
+      delete padutils.warnDeprecatedFlags.disabledForTestingOnly;
+    }
+    assem.endDocument();
+    expect(assem.toString()).to.equal('*3*4*5+8*1*4*5+4');
+  });
+
+  it('smartOpAssembler append + op with multiline text', async function () {
+    const assem = new SmartOpAssembler();
+    const pool = poolOrArray([
+      'attr1,1',
+      'attr2,2',
+      'attr3,3',
+      'attr4,4',
+      'attr5,5',
+    ]);
+
+    padutils.warnDeprecatedFlags.disabledForTestingOnly = true;
+    try {
+      assem.appendOpWithText('+', 'test\ntest', '*3*4*5', pool);
+      assem.appendOpWithText('+', '\ntest\n', '*3*4*5', pool);
+      assem.appendOpWithText('+', '\ntest', '*1*4*5', pool);
+    } finally {
+      // @ts-ignore
+      delete padutils.warnDeprecatedFlags.disabledForTestingOnly;
+    }
+    assem.endDocument();
+    expect(assem.toString()).to.equal('*3*4*5|3+f*1*4*5|1+1*1*4*5+4');
+  });
+
+  it('smartOpAssembler clear should empty internal assemblers', async function () {
+    const x = '-c*3*4+6|3=az*asdf0*1*2*3+1=1-1+1*0+1=1-1+1|c=c-1';
+    const ops = deserializeOps(x);
+    const iter = {
+      _n: ops.next(),
+      hasNext() { return !this._n.done; },
+      next() { const v = this._n.value; this._n = ops.next(); return v as Op; },
+    };
+    const assem = new SmartOpAssembler();
+    var iter1 = iter.next()
+    assem.append(iter1);
+    var iter2 = iter.next()
+    assem.append(iter2);
+    var iter3 = iter.next()
+    assem.append(iter3);
+    console.log(assem.toString());
+    assem.clear();
+    assem.append(iter.next());
+    assem.append(iter.next());
+    console.log(assem.toString());
+    assem.clear();
+    let counter = 0;
+    while (iter.hasNext()) {
+      console.log(counter++)
+      assem.append(iter.next());
+    }
+    assem.endDocument();
+    expect(assem.toString()).to.equal('-1+1*0+1=1-1+1|c=c-1');
+  });
+
+  describe('append atext to assembler', function () {
+    const testAppendATextToAssembler = (testId: number, atext: { text: string; attribs: string; }, correctOps: string) => {
+      it(`testAppendATextToAssembler#${testId}`, async function () {
+        const assem = new SmartOpAssembler();
+        for (const op of opsFromAText(atext)) assem.append(op);
+        expect(assem.toString()).to.equal(correctOps);
+      });
+    };
+
+    testAppendATextToAssembler(1, {
+      text: '\n',
+      attribs: '|1+1',
+    }, '');
+    testAppendATextToAssembler(2, {
+      text: '\n\n',
+      attribs: '|2+2',
+    }, '|1+1');
+    testAppendATextToAssembler(3, {
+      text: '\n\n',
+      attribs: '*x|2+2',
+    }, '*x|1+1');
+    testAppendATextToAssembler(4, {
+      text: '\n\n',
+      attribs: '*x|1+1|1+1',
+    }, '*x|1+1');
+    testAppendATextToAssembler(5, {
+      text: 'foo\n',
+      attribs: '|1+4',
+    }, '+3');
+    testAppendATextToAssembler(6, {
+      text: '\nfoo\n',
+      attribs: '|2+5',
+    }, '|1+1+3');
+    testAppendATextToAssembler(7, {
+      text: '\nfoo\n',
+      attribs: '*x|2+5',
+    }, '*x|1+1*x+3');
+    testAppendATextToAssembler(8, {
+      text: '\n\n\nfoo\n',
+      attribs: '|2+2*x|2+5',
+    }, '|2+2*x|1+1*x+3');
+  });
+});

src/tests/backend-new/specs/easysync-compose.ts

@@ -0,0 +1,54 @@
+'use strict';
+
+import {applyToText, checkRep, compose} from '../../../static/js/Changeset';
+import AttributePool from '../../../static/js/AttributePool';
+import {randomMultiline, randomTestChangeset} from '../easysync-helper';
+import {expect, describe, it} from 'vitest';
+
+describe('easysync-compose', function () {
+  describe('compose', function () {
+    const testCompose = (randomSeed: number) => {
+      it(`testCompose#${randomSeed}`, async function () {
+        const p = new AttributePool();
+
+        const startText = `${randomMultiline(10, 20)}\n`;
+
+        const x1 = randomTestChangeset(startText);
+        const change1 = x1[0];
+        const text1 = x1[1];
+
+        const x2 = randomTestChangeset(text1);
+        const change2 = x2[0];
+        const text2 = x2[1];
+
+        const x3 = randomTestChangeset(text2);
+        const change3 = x3[0];
+        const text3 = x3[1];
+
+        const change12 = checkRep(compose(change1, change2, p));
+        const change23 = checkRep(compose(change2, change3, p));
+        const change123 = checkRep(compose(change12, change3, p));
+        const change123a = checkRep(compose(change1, change23, p));
+        expect(change123a).to.equal(change123);
+
+        expect(applyToText(change12, startText)).to.equal(text2);
+        expect(applyToText(change23, text1)).to.equal(text3);
+        expect(applyToText(change123, startText)).to.equal(text3);
+      });
+    };
+
+    for (let i = 0; i < 30; i++) testCompose(i);
+  });
+
+  describe('compose attributes', function () {
+    it('simpleComposeAttributesTest', async function () {
+      const p = new AttributePool();
+      p.putAttrib(['bold', '']);
+      p.putAttrib(['bold', 'true']);
+      const cs1 = checkRep('Z:2>1*1+1*1=1$x');
+      const cs2 = checkRep('Z:3>0*0|1=3$');
+      const cs12 = checkRep(compose(cs1, cs2, p));
+      expect(cs12).to.equal('Z:2>1+1*0|1=2$x');
+    });
+  });
+});

src/tests/backend-new/specs/easysync-mutations.ts

@@ -0,0 +1,320 @@
+'use strict';
+
+import {applyToAttribution, applyToText, checkRep, joinAttributionLines, mutateAttributionLines, mutateTextLines, pack} from '../../../static/js/Changeset';
+import AttributePool from '../../../static/js/AttributePool';
+import {poolOrArray} from '../easysync-helper';
+import {expect, describe,it } from "vitest";
+import {SmartOpAssembler} from "../../../static/js/SmartOpAssembler";
+import Op from "../../../static/js/Op";
+import {StringAssembler} from "../../../static/js/StringAssembler";
+import TextLinesMutator from "../../../static/js/TextLinesMutator";
+import {numToString} from "../../../static/js/ChangesetUtils";
+
+describe('easysync-mutations', function () {
+  const applyMutations = (mu: TextLinesMutator, arrayOfArrays: any[]) => {
+    arrayOfArrays.forEach((a) => {
+      // @ts-ignore
+      const result = mu[a[0]](...a.slice(1));
+      if (a[0] === 'remove' && a[3]) {
+        expect(result).to.equal(a[3]);
+      }
+    });
+  };
+
+  const mutationsToChangeset = (oldLen: number, arrayOfArrays: string[][]) => {
+    const assem = new SmartOpAssembler();
+    const op = new Op();
+    const bank = new StringAssembler();
+    let oldPos = 0;
+    let newLen = 0;
+    arrayOfArrays.forEach((a: any[]) => {
+      if (a[0] === 'skip') {
+        op.opcode = '=';
+        op.chars = a[1];
+        op.lines = (a[2] || 0);
+        assem.append(op);
+        oldPos += op.chars;
+        newLen += op.chars;
+      } else if (a[0] === 'remove') {
+        op.opcode = '-';
+        op.chars = a[1];
+        op.lines = (a[2] || 0);
+        assem.append(op);
+        oldPos += op.chars;
+      } else if (a[0] === 'insert') {
+        op.opcode = '+';
+        bank.append(a[1]);
+        op.chars = a[1].length;
+        op.lines = (a[2] || 0);
+        assem.append(op);
+        newLen += op.chars;
+      }
+    });
+    newLen += oldLen - oldPos;
+    assem.endDocument();
+    return pack(oldLen, newLen, assem.toString(), bank.toString());
+  };
+
+  const runMutationTest = (testId: number, origLines: string[], muts:any, correct: string[]) => {
+    it(`runMutationTest#${testId}`, async function () {
+      let lines = origLines.slice();
+      const mu = new TextLinesMutator(lines);
+      applyMutations(mu, muts);
+      mu.close();
+      expect(lines).to.eql(correct);
+
+      const inText = origLines.join('');
+      const cs = mutationsToChangeset(inText.length, muts);
+      lines = origLines.slice();
+      mutateTextLines(cs, lines);
+      expect(lines).to.eql(correct);
+
+      const correctText = correct.join('');
+      const outText = applyToText(cs, inText);
+      expect(outText).to.equal(correctText);
+    });
+  };
+
+  runMutationTest(1, ['apple\n', 'banana\n', 'cabbage\n', 'duffle\n', 'eggplant\n'], [
+    ['remove', 1, 0, 'a'],
+    ['insert', 'tu'],
+    ['remove', 1, 0, 'p'],
+    ['skip', 4, 1],
+    ['skip', 7, 1],
+    ['insert', 'cream\npie\n', 2],
+    ['skip', 2],
+    ['insert', 'bot'],
+    ['insert', '\n', 1],
+    ['insert', 'bu'],
+    ['skip', 3],
+    ['remove', 3, 1, 'ge\n'],
+    ['remove', 6, 0, 'duffle'],
+  ], ['tuple\n', 'banana\n', 'cream\n', 'pie\n', 'cabot\n', 'bubba\n', 'eggplant\n']);
+
+  runMutationTest(2, ['apple\n', 'banana\n', 'cabbage\n', 'duffle\n', 'eggplant\n'], [
+    ['remove', 1, 0, 'a'],
+    ['remove', 1, 0, 'p'],
+    ['insert', 'tu'],
+    ['skip', 11, 2],
+    ['insert', 'cream\npie\n', 2],
+    ['skip', 2],
+    ['insert', 'bot'],
+    ['insert', '\n', 1],
+    ['insert', 'bu'],
+    ['skip', 3],
+    ['remove', 3, 1, 'ge\n'],
+    ['remove', 6, 0, 'duffle'],
+  ], ['tuple\n', 'banana\n', 'cream\n', 'pie\n', 'cabot\n', 'bubba\n', 'eggplant\n']);
+
+  runMutationTest(3, ['apple\n', 'banana\n', 'cabbage\n', 'duffle\n', 'eggplant\n'], [
+    ['remove', 6, 1, 'apple\n'],
+    ['skip', 15, 2],
+    ['skip', 6],
+    ['remove', 1, 1, '\n'],
+    ['remove', 8, 0, 'eggplant'],
+    ['skip', 1, 1],
+  ], ['banana\n', 'cabbage\n', 'duffle\n']);
+
+  runMutationTest(4, ['15\n'], [
+    ['skip', 1],
+    ['insert', '\n2\n3\n4\n', 4],
+    ['skip', 2, 1],
+  ], ['1\n', '2\n', '3\n', '4\n', '5\n']);
+
+  runMutationTest(5, ['1\n', '2\n', '3\n', '4\n', '5\n'], [
+    ['skip', 1],
+    ['remove', 7, 4, '\n2\n3\n4\n'],
+    ['skip', 2, 1],
+  ], ['15\n']);
+
+  runMutationTest(6, ['123\n', 'abc\n', 'def\n', 'ghi\n', 'xyz\n'], [
+    ['insert', '0'],
+    ['skip', 4, 1],
+    ['skip', 4, 1],
+    ['remove', 8, 2, 'def\nghi\n'],
+    ['skip', 4, 1],
+  ], ['0123\n', 'abc\n', 'xyz\n']);
+
+  runMutationTest(7, ['apple\n', 'banana\n', 'cabbage\n', 'duffle\n', 'eggplant\n'], [
+    ['remove', 6, 1, 'apple\n'],
+    ['skip', 15, 2, true],
+    ['skip', 6, 0, true],
+    ['remove', 1, 1, '\n'],
+    ['remove', 8, 0, 'eggplant'],
+    ['skip', 1, 1, true],
+  ], ['banana\n', 'cabbage\n', 'duffle\n']);
+
+  it('mutatorHasMore', async function () {
+    const lines = ['1\n', '2\n', '3\n', '4\n'];
+    let mu;
+
+    mu = new TextLinesMutator(lines);
+    expect(mu.hasMore()).toBeTruthy();
+    mu.skip(8, 4);
+    expect(mu.hasMore()).toBeFalsy();
+    mu.close();
+    expect(mu.hasMore()).toBeFalsy();
+
+    // still 1,2,3,4
+    mu = new TextLinesMutator(lines);
+    expect(mu.hasMore()).toBeTruthy();
+    mu.remove(2, 1);
+    expect(mu.hasMore()).toBeTruthy();
+    mu.skip(2, 1);
+    expect(mu.hasMore()).toBeTruthy();
+    mu.skip(2, 1);
+    expect(mu.hasMore()).toBeTruthy();
+    mu.skip(2, 1);
+    expect(mu.hasMore()).toBeFalsy();
+    mu.insert('5\n', 1);
+    expect(mu.hasMore()).toBeFalsy();
+    mu.close();
+    expect(mu.hasMore()).toBeFalsy();
+
+    // 2,3,4,5 now
+    mu = new TextLinesMutator(lines);
+    expect(mu.hasMore()).toBeTruthy();
+    mu.remove(6, 3);
+    expect(mu.hasMore()).toBeTruthy();
+    mu.remove(2, 1);
+    expect(mu.hasMore()).toBeFalsy();
+    mu.insert('hello\n', 1);
+    expect(mu.hasMore()).toBeFalsy();
+    mu.close();
+    expect(mu.hasMore()).toBeFalsy();
+  });
+
+  describe('mutateTextLines', function () {
+    const testMutateTextLines = (testId: number, cs: string, lines: string[], correctLines: string[]) => {
+      it(`testMutateTextLines#${testId}`, async function () {
+        const a = lines.slice();
+        mutateTextLines(cs, a);
+        expect(a).to.eql(correctLines);
+      });
+    };
+
+    testMutateTextLines(1, 'Z:4<1|1-2-1|1+1+1$\nc', ['a\n', 'b\n'], ['\n', 'c\n']);
+    testMutateTextLines(2, 'Z:4>0|1-2-1|2+3$\nc\n', ['a\n', 'b\n'], ['\n', 'c\n', '\n']);
+
+    it('mutate keep only lines', async function () {
+      const lines = ['1\n', '2\n', '3\n', '4\n'];
+      const result = lines.slice();
+      const cs = 'Z:8>0*0|1=2|2=2';
+
+      mutateTextLines(cs, lines);
+      expect(result).to.eql(lines);
+    });
+  });
+
+  describe('mutate attributions', function () {
+    const testPoolWithChars = (() => {
+      const p = new AttributePool();
+      p.putAttrib(['char', 'newline']);
+      for (let i = 1; i < 36; i++) {
+        p.putAttrib(['char', numToString(i)]);
+      }
+      p.putAttrib(['char', '']);
+      return p;
+    })();
+
+    const runMutateAttributionTest = (testId: number, attribs: string[] | AttributePool, cs: string, alines: string[], outCorrect: string[]) => {
+      it(`runMutateAttributionTest#${testId}`, async function () {
+        const p = poolOrArray(attribs);
+        const alines2 = Array.prototype.slice.call(alines);
+        mutateAttributionLines(checkRep(cs), alines2, p);
+        expect(alines2).to.eql(outCorrect);
+
+        const removeQuestionMarks = (a: string) => a.replace(/\?/g, '');
+        const inMerged = joinAttributionLines(alines.map(removeQuestionMarks));
+        const correctMerged = joinAttributionLines(outCorrect.map(removeQuestionMarks));
+        const mergedResult = applyToAttribution(cs, inMerged, p);
+        expect(mergedResult).to.equal(correctMerged);
+      });
+    };
+
+    // turn 123\n 456\n 789\n into 123\n 4<b>5</b>6\n 789\n
+    runMutateAttributionTest(1,
+        ['bold,true'], 'Z:c>0|1=4=1*0=1$', ['|1+4', '|1+4', '|1+4'],
+        ['|1+4', '+1*0+1|1+2', '|1+4']);
+
+    // make a document bold
+    runMutateAttributionTest(2,
+        ['bold,true'], 'Z:c>0*0|3=c$', ['|1+4', '|1+4', '|1+4'], ['*0|1+4', '*0|1+4', '*0|1+4']);
+
+    // clear bold on document
+    runMutateAttributionTest(3,
+        ['bold,', 'bold,true'], 'Z:c>0*0|3=c$',
+        ['*1+1+1*1+1|1+1', '+1*1+1|1+2', '*1+1+1*1+1|1+1'], ['|1+4', '|1+4', '|1+4']);
+
+    // add a character on line 3 of a document with 5 blank lines, and make sure
+    // the optimization that skips purely-kept lines is working; if any attribution string
+    // with a '?' is parsed it will cause an error.
+    runMutateAttributionTest(4,
+        ['foo,bar', 'line,1', 'line,2', 'line,3', 'line,4', 'line,5'],
+        'Z:5>1|2=2+1$x', ['?*1|1+1', '?*2|1+1', '*3|1+1', '?*4|1+1', '?*5|1+1'],
+        ['?*1|1+1', '?*2|1+1', '+1*3|1+1', '?*4|1+1', '?*5|1+1']);
+
+    // based on runMutationTest#1
+    runMutateAttributionTest(5, testPoolWithChars,
+        'Z:11>7-2*t+1*u+1|2=b|2+a=2*b+1*o+1*t+1*0|1+1*b+1*u+1=3|1-3-6$tucream\npie\nbot\nbu',
+        [
+          '*a+1*p+2*l+1*e+1*0|1+1',
+          '*b+1*a+1*n+1*a+1*n+1*a+1*0|1+1',
+          '*c+1*a+1*b+2*a+1*g+1*e+1*0|1+1',
+          '*d+1*u+1*f+2*l+1*e+1*0|1+1',
+          '*e+1*g+2*p+1*l+1*a+1*n+1*t+1*0|1+1',
+        ],
+        [
+          '*t+1*u+1*p+1*l+1*e+1*0|1+1',
+          '*b+1*a+1*n+1*a+1*n+1*a+1*0|1+1',
+          '|1+6',
+          '|1+4',
+          '*c+1*a+1*b+1*o+1*t+1*0|1+1',
+          '*b+1*u+1*b+2*a+1*0|1+1',
+          '*e+1*g+2*p+1*l+1*a+1*n+1*t+1*0|1+1',
+        ]);
+
+    // based on runMutationTest#3
+    runMutateAttributionTest(6, testPoolWithChars,
+        'Z:11<f|1-6|2=f=6|1-1-8$', ['*a|1+6', '*b|1+7', '*c|1+8', '*d|1+7', '*e|1+9'],
+        ['*b|1+7', '*c|1+8', '*d+6*e|1+1']);
+
+    // based on runMutationTest#4
+    runMutateAttributionTest(7, testPoolWithChars, 'Z:3>7=1|4+7$\n2\n3\n4\n',
+        ['*1+1*5|1+2'], ['*1+1|1+1', '|1+2', '|1+2', '|1+2', '*5|1+2']);
+
+    // based on runMutationTest#5
+    runMutateAttributionTest(8, testPoolWithChars, 'Z:a<7=1|4-7$',
+        ['*1|1+2', '*2|1+2', '*3|1+2', '*4|1+2', '*5|1+2'], ['*1+1*5|1+2']);
+
+    // based on runMutationTest#6
+    runMutateAttributionTest(9, testPoolWithChars, 'Z:k<7*0+1*10|2=8|2-8$0',
+        [
+          '*1+1*2+1*3+1|1+1',
+          '*a+1*b+1*c+1|1+1',
+          '*d+1*e+1*f+1|1+1',
+          '*g+1*h+1*i+1|1+1',
+          '?*x+1*y+1*z+1|1+1',
+        ],
+        ['*0+1|1+4', '|1+4', '?*x+1*y+1*z+1|1+1']);
+
+    runMutateAttributionTest(10, testPoolWithChars, 'Z:6>4=1+1=1+1|1=1+1=1*0+1$abcd',
+        ['|1+3', '|1+3'], ['|1+5', '+2*0+1|1+2']);
+
+
+    runMutateAttributionTest(11, testPoolWithChars, 'Z:s>1|1=4=6|1+1$\n',
+        ['*0|1+4', '*0|1+8', '*0+5|1+1', '*0|1+1', '*0|1+5', '*0|1+1', '*0|1+1', '*0|1+1', '|1+1'],
+        [
+          '*0|1+4',
+          '*0+6|1+1',
+          '*0|1+2',
+          '*0+5|1+1',
+          '*0|1+1',
+          '*0|1+5',
+          '*0|1+1',
+          '*0|1+1',
+          '*0|1+1',
+          '|1+1',
+        ]);
+  });
+});

src/tests/backend-new/specs/easysync-other.test.ts

@@ -0,0 +1,166 @@
+'use strict';
+
+import {applyToAttribution, applyToText, checkRep, deserializeOps, exportedForTestingOnly, filterAttribNumbers, joinAttributionLines, makeAttribsString, makeSplice, moveOpsToNewPool, opAttributeValue, splitAttributionLines} from '../../../static/js/Changeset';
+import AttributePool from '../../../static/js/AttributePool';
+import {randomMultiline, poolOrArray} from '../easysync-helper';
+import padutils from '../../../static/js/pad_utils';
+import {describe, it, expect} from 'vitest'
+import Op from "../../../static/js/Op";
+import {MergingOpAssembler} from "../../../static/js/MergingOpAssembler";
+import {Attribute} from "../../../static/js/types/Attribute";
+
+
+describe('easysync-other', function () {
+  describe('filter attribute numbers', function () {
+    const testFilterAttribNumbers = (testId: number, cs: string, filter: Function, correctOutput: string) => {
+      it(`testFilterAttribNumbers#${testId}`, async function () {
+        const str = filterAttribNumbers(cs, filter);
+        expect(str).to.equal(correctOutput);
+      });
+    };
+
+    testFilterAttribNumbers(1, '*0*1+1+2+3*1+4*2+5*0*2*1*b*c+6',
+        (n: number) => (n % 2) === 0, '*0+1+2+3+4*2+5*0*2*c+6');
+    testFilterAttribNumbers(2, '*0*1+1+2+3*1+4*2+5*0*2*1*b*c+6',
+        (n: number) => (n % 2) === 1, '*1+1+2+3*1+4+5*1*b+6');
+  });
+
+  describe('make attribs string', function () {
+    const testMakeAttribsString = (testId: number, pool: string[], opcode: string, attribs: string | Attribute[], correctString: string) => {
+      it(`testMakeAttribsString#${testId}`, async function () {
+        const p = poolOrArray(pool);
+        padutils.warnDeprecatedFlags.disabledForTestingOnly = true;
+        try {
+          expect(makeAttribsString(opcode, attribs, p)).to.equal(correctString);
+        } finally {
+          // @ts-ignore
+          delete padutils.warnDeprecatedFlags.disabledForTestingOnly;
+        }
+      });
+    };
+
+    testMakeAttribsString(1, ['bold,'], '+', [
+      ['bold', ''],
+    ], '');
+    testMakeAttribsString(2, ['abc,def', 'bold,'], '=', [
+      ['bold', ''],
+    ], '*1');
+    testMakeAttribsString(3, ['abc,def', 'bold,true'], '+', [
+      ['abc', 'def'],
+      ['bold', 'true'],
+    ], '*0*1');
+    testMakeAttribsString(4, ['abc,def', 'bold,true'], '+', [
+      ['bold', 'true'],
+      ['abc', 'def'],
+    ], '*0*1');
+  });
+
+  describe('other', function () {
+    it('testMoveOpsToNewPool', async function () {
+      const pool1 = new AttributePool();
+      const pool2 = new AttributePool();
+
+      pool1.putAttrib(['baz', 'qux']);
+      pool1.putAttrib(['foo', 'bar']);
+
+      pool2.putAttrib(['foo', 'bar']);
+
+      expect(moveOpsToNewPool('Z:1>2*1+1*0+1$ab', pool1, pool2))
+          .to.equal('Z:1>2*0+1*1+1$ab');
+      expect(moveOpsToNewPool('*1+1*0+1', pool1, pool2)).to.equal('*0+1*1+1');
+    });
+
+    it('testMakeSplice', async function () {
+      const t = 'a\nb\nc\n';
+      let splice = makeSplice(t, 5, 0, 'def')
+      const t2 = applyToText(splice, t);
+      expect(t2).to.equal('a\nb\ncdef\n');
+    });
+
+    it('makeSplice at the end', async function () {
+      const orig = '123';
+      const ins = '456';
+      expect(applyToText(makeSplice(orig, orig.length, 0, ins), orig))
+          .to.equal(`${orig}${ins}`);
+    });
+
+    it('testToSplices', async function () {
+      const cs = checkRep('Z:z>9*0=1=4-3+9=1|1-4-4+1*0+a$123456789abcdefghijk');
+      const correctSplices = [
+        [5, 8, '123456789'],
+        [9, 17, 'abcdefghijk'],
+      ];
+      expect(exportedForTestingOnly.toSplices(cs)).to.eql(correctSplices);
+    });
+
+    it('opAttributeValue', async function () {
+      const p = new AttributePool();
+      p.putAttrib(['name', 'david']);
+      p.putAttrib(['color', 'green']);
+
+      const stringOp = (str: string) => deserializeOps(str).next().value as Op;
+
+      padutils.warnDeprecatedFlags.disabledForTestingOnly = true;
+      try {
+        expect(opAttributeValue(stringOp('*0*1+1'), 'name', p)).to.equal('david');
+        expect(opAttributeValue(stringOp('*0+1'), 'name', p)).to.equal('david');
+        expect(opAttributeValue(stringOp('*1+1'), 'name', p)).to.equal('');
+        expect(opAttributeValue(stringOp('+1'), 'name', p)).to.equal('');
+        expect(opAttributeValue(stringOp('*0*1+1'), 'color', p)).to.equal('green');
+        expect(opAttributeValue(stringOp('*1+1'), 'color', p)).to.equal('green');
+        expect(opAttributeValue(stringOp('*0+1'), 'color', p)).to.equal('');
+        expect(opAttributeValue(stringOp('+1'), 'color', p)).to.equal('');
+      } finally {
+        // @ts-ignore
+        delete padutils.warnDeprecatedFlags.disabledForTestingOnly;
+      }
+    });
+
+    describe('applyToAttribution', function () {
+      const runApplyToAttributionTest = (testId: number, attribs: string[], cs: string, inAttr: string, outCorrect: string) => {
+        it(`applyToAttribution#${testId}`, async function () {
+          const p = poolOrArray(attribs);
+          const result = applyToAttribution(checkRep(cs), inAttr, p);
+          expect(result).to.equal(outCorrect);
+        });
+      };
+
+      // turn c<b>a</b>ctus\n into a<b>c</b>tusabcd\n
+      runApplyToAttributionTest(1,
+          ['bold,', 'bold,true'], 'Z:7>3-1*0=1*1=1=3+4$abcd', '+1*1+1|1+5', '+1*1+1|1+8');
+
+      // turn "david\ngreenspan\n" into "<b>david\ngreen</b>\n"
+      runApplyToAttributionTest(2,
+          ['bold,', 'bold,true'], 'Z:g<4*1|1=6*1=5-4$', '|2+g', '*1|1+6*1+5|1+1');
+    });
+
+    describe('split/join attribution lines', function () {
+      const testSplitJoinAttributionLines = (randomSeed: number) => {
+        const stringToOps = (str: string) => {
+          const assem = new MergingOpAssembler();
+          const o = new Op('+');
+          o.chars = 1;
+          for (let i = 0; i < str.length; i++) {
+            const c = str.charAt(i);
+            o.lines = (c === '\n' ? 1 : 0);
+            o.attribs = (c === 'a' || c === 'b' ? `*${c}` : '');
+            assem.append(o);
+          }
+          return assem.toString();
+        };
+
+        it(`testSplitJoinAttributionLines#${randomSeed}`, async function () {
+          const doc = `${randomMultiline(10, 20)}\n`;
+
+          const theJoined = stringToOps(doc);
+          const theSplit = doc.match(/[^\n]*\n/g)!.map(stringToOps);
+
+          expect(splitAttributionLines(theJoined, doc)).to.eql(theSplit);
+          expect(joinAttributionLines(theSplit)).to.equal(theJoined);
+        });
+      };
+
+      for (let i = 0; i < 10; i++) testSplitJoinAttributionLines(i);
+    });
+  });
+});

src/tests/backend-new/specs/easysync-subAttribution.ts

@@ -0,0 +1,55 @@
+'use strict';
+
+import {subattribution} from '../../../static/js/Changeset';
+import {expect, describe, it} from 'vitest';
+describe('easysync-subAttribution', function () {
+  const testSubattribution = (testId: number, astr: string, start: number, end: number | undefined, correctOutput: string) => {
+    it(`subattribution#${testId}`, async function () {
+      const str = subattribution(astr, start, end);
+      expect(str).to.equal(correctOutput);
+    });
+  };
+
+  testSubattribution(1, '+1', 0, 0, '');
+  testSubattribution(2, '+1', 0, 1, '+1');
+  testSubattribution(3, '+1', 0, undefined, '+1');
+  testSubattribution(4, '|1+1', 0, 0, '');
+  testSubattribution(5, '|1+1', 0, 1, '|1+1');
+  testSubattribution(6, '|1+1', 0, undefined, '|1+1');
+  testSubattribution(7, '*0+1', 0, 0, '');
+  testSubattribution(8, '*0+1', 0, 1, '*0+1');
+  testSubattribution(9, '*0+1', 0, undefined, '*0+1');
+  testSubattribution(10, '*0|1+1', 0, 0, '');
+  testSubattribution(11, '*0|1+1', 0, 1, '*0|1+1');
+  testSubattribution(12, '*0|1+1', 0, undefined, '*0|1+1');
+  testSubattribution(13, '*0+2+1*1+3', 0, 1, '*0+1');
+  testSubattribution(14, '*0+2+1*1+3', 0, 2, '*0+2');
+  testSubattribution(15, '*0+2+1*1+3', 0, 3, '*0+2+1');
+  testSubattribution(16, '*0+2+1*1+3', 0, 4, '*0+2+1*1+1');
+  testSubattribution(17, '*0+2+1*1+3', 0, 5, '*0+2+1*1+2');
+  testSubattribution(18, '*0+2+1*1+3', 0, 6, '*0+2+1*1+3');
+  testSubattribution(19, '*0+2+1*1+3', 0, 7, '*0+2+1*1+3');
+  testSubattribution(20, '*0+2+1*1+3', 0, undefined, '*0+2+1*1+3');
+  testSubattribution(21, '*0+2+1*1+3', 1, undefined, '*0+1+1*1+3');
+  testSubattribution(22, '*0+2+1*1+3', 2, undefined, '+1*1+3');
+  testSubattribution(23, '*0+2+1*1+3', 3, undefined, '*1+3');
+  testSubattribution(24, '*0+2+1*1+3', 4, undefined, '*1+2');
+  testSubattribution(25, '*0+2+1*1+3', 5, undefined, '*1+1');
+  testSubattribution(26, '*0+2+1*1+3', 6, undefined, '');
+  testSubattribution(27, '*0+2+1*1|1+3', 0, 1, '*0+1');
+  testSubattribution(28, '*0+2+1*1|1+3', 0, 2, '*0+2');
+  testSubattribution(29, '*0+2+1*1|1+3', 0, 3, '*0+2+1');
+  testSubattribution(30, '*0+2+1*1|1+3', 0, 4, '*0+2+1*1+1');
+  testSubattribution(31, '*0+2+1*1|1+3', 0, 5, '*0+2+1*1+2');
+  testSubattribution(32, '*0+2+1*1|1+3', 0, 6, '*0+2+1*1|1+3');
+  testSubattribution(33, '*0+2+1*1|1+3', 0, 7, '*0+2+1*1|1+3');
+  testSubattribution(34, '*0+2+1*1|1+3', 0, undefined, '*0+2+1*1|1+3');
+  testSubattribution(35, '*0+2+1*1|1+3', 1, undefined, '*0+1+1*1|1+3');
+  testSubattribution(36, '*0+2+1*1|1+3', 2, undefined, '+1*1|1+3');
+  testSubattribution(37, '*0+2+1*1|1+3', 3, undefined, '*1|1+3');
+  testSubattribution(38, '*0+2+1*1|1+3', 4, undefined, '*1|1+2');
+  testSubattribution(39, '*0+2+1*1|1+3', 5, undefined, '*1|1+1');
+  testSubattribution(40, '*0+2+1*1|1+3', 1, 5, '*0+1+1*1+2');
+  testSubattribution(41, '*0+2+1*1|1+3', 2, 6, '+1*1|1+3');
+  testSubattribution(42, '*0+2+1*1+3', 2, 6, '+1*1+3');
+});