tests/editor/ul/li/ol/import/export: Introduce contentcollector.js tests & various OL/UL/LI related bugfixes

1. Introduce contentcollector.js backend tests
1. Fix issue with OL LI items not being properly numbered after import
1. Fix issue with nested OL LI items being improperly numbered on export
1. Fix issue with new lines not being introduced after lists in on import #3961
1. Sanitize HTML on the way in (import)
1. Fix ExportHTML CSS because it needs to support OL > LI > OL not OL > OL [The latter being the correct format]
1. Fix backend tests.

src/node/utils/ExportHelper.js

@@ -63,6 +63,13 @@ exports._analyzeLine = function(text, aline, apool){
         }
       }
     }
+    var opIter2 = Changeset.opIterator(aline);
+    if (opIter2.hasNext()){
+      var start = Changeset.opAttributeValue(opIter2.next(), 'start', apool);
+      if (start){
+          line.start = start;
+      }
+    }
   }
   if (lineMarker){
     line.text = text.substring(1);

src/node/utils/ExportHtml.js

@@ -316,7 +316,6 @@ function getHTMLFromAtext(pad, atext, authorColors)
     var context;
     var line = _analyzeLine(textLines[i], attribLines[i], apool);
     var lineContent = getLineHTML(line.text, line.aline);
-
     if (line.listLevel)//If we are inside a list
     {
       context = {
@@ -361,13 +360,51 @@ function getHTMLFromAtext(pad, atext, authorColors)
 
             if (prevPiece.indexOf("<ul") === 0 || prevPiece.indexOf("<ol") === 0 || prevPiece.indexOf("</li>") === 0)
             {
-              pieces.push("<li>");
+               /*
+                 uncommenting this breaks nested ols..
+                 if the previous item is NOT a ul, NOT an ol OR closing li then close the list
+                 so we consider this HTML, I inserted ** where it throws a problem in Example Wrong..
+                 <ol><li>one</li><li><ol><li>1.1</li><li><ol><li>1.1.1</li></ol></li></ol></li><li>two</li></ol>
+
+                 Note that closing the li then re-opening for another li item here is wrong.  The correct markup is
+                 <ol><li>one<ol><li>1.1<ol><li>1.1.1</li></ol></li></ol></li><li>two</li></ol>
+
+                 Exmaple Right: <ol class="number"><li>one</li><ol start="2" class="number"><li>1.1</li><ol start="3" class="number"><li>1.1.1</li></ol></li></ol><li>two</li></ol>
+                 Example Wrong: <ol class="number"><li>one</li>**</li>**<ol start="2" class="number"><li>1.1</li>**</li>**<ol start="3" class="number"><li>1.1.1</li></ol></li></ol><li>two</li></ol>
+                 So it's firing wrong where the current piece is an li and the previous piece is an ol and next piece is an ol
+                 So to remedy this we can say if next piece is NOT an OL or UL.
+                 // pieces.push("</li>");
+
+              */
+              if( (nextLine.listTypeName === 'number') && (nextLine.text === '') ){
+                // is the listTypeName check needed here?  null text might be completely fine!
+                // TODO Check against Uls
+                // don't do anything because the next item is a nested ol openener so we need to keep the li open
+              }else{
+                pieces.push("</li>");
+              }
+
+
             }
 
             if (line.listTypeName === "number")
             {
+              // We introduce line.start here, this is useful for continuing Ordered list line numbers
+              // in case you have a bullet in a list IE you Want
+              // 1. hello
+              //   * foo
+              // 2. world
+              // Without this line.start logic it would be
+              // 1. hello * foo 1. world because the bullet would kill the OL
+
+              // TODO: This logic could also be used to continue OL with indented content
+              // but that's a job for another day....
+              if(line.start){
+                pieces.push("<ol start=\""+Number(line.start)+"\" class=\"" + line.listTypeName + "\">");
+              }else{
                 pieces.push("<ol class=\"" + line.listTypeName + "\">");
               }
+            }
             else
             {
               pieces.push("<ul class=\"" + line.listTypeName + "\">");
@@ -375,14 +412,25 @@ function getHTMLFromAtext(pad, atext, authorColors)
           }
         }
       }
-
+      // if we're going up a level we shouldn't be adding..
+      if(context.lineContent){
         pieces.push("<li>", context.lineContent);
+      }
 
       // To close list elements
       if (nextLine && nextLine.listLevel === line.listLevel && line.listTypeName === nextLine.listTypeName)
       {
+        if(context.lineContent){
+          if( (nextLine.listTypeName === 'number') && (nextLine.text === '') ){
+            // is the listTypeName check needed here?  null text might be completely fine!
+            // TODO Check against Uls
+            // don't do anything because the next item is a nested ol openener so we need to keep the li open
+          }else{
             pieces.push("</li>");
           }
+
+        }
+      }
       if ((!nextLine || !nextLine.listLevel || nextLine.listLevel < line.listLevel) || (nextLine && line.listTypeName !== nextLine.listTypeName))
       {
         var nextLevel = 0;

src/node/utils/ImportHtml.js

@@ -14,15 +14,29 @@
  * limitations under the License.
  */
 
-var log4js = require('log4js');
-var Changeset = require("ep_etherpad-lite/static/js/Changeset");
-var contentcollector = require("ep_etherpad-lite/static/js/contentcollector");
-var cheerio = require("cheerio");
+const log4js            = require('log4js');
+const Changeset         = require("ep_etherpad-lite/static/js/Changeset");
+const contentcollector  = require("ep_etherpad-lite/static/js/contentcollector");
+const cheerio           = require("cheerio");
+const rehype            = require("rehype")
+const format            = require("rehype-format")
+
 
 exports.setPadHTML = function(pad, html)
 {
   var apiLogger = log4js.getLogger("ImportHtml");
 
+  var opts = {
+    indentInitial: false,
+    indent: -1
+  }
+
+  rehype()
+    .use(format, opts)
+    .process(html, function(err, output){
+      html = String(output).replace(/(\r\n|\n|\r)/gm,"");
+  })
+
   var $ = cheerio.load(html);
 
   // Appends a line break, used by Etherpad to ensure a caret is available

src/node/utils/Minify.js

@@ -23,6 +23,8 @@ var ERR = require("async-stacktrace");
 var settings = require('./Settings');
 var async = require('async');
 var fs = require('fs');
+var StringDecoder = require('string_decoder').StringDecoder;
+var CleanCSS = require('clean-css');
 var path = require('path');
 var plugins = require("ep_etherpad-lite/static/js/pluginfw/plugins");
 var RequireKernel = require('etherpad-require-kernel');

src/package.json

@@ -51,6 +51,8 @@
     "lodash.clonedeep": "4.5.0",
     "log4js": "0.6.35",
     "measured-core": "1.11.2",
+    "rehype": "^10.0.0",
+    "rehype-format": "^3.0.1",
     "nodeify": "1.0.1",
     "npm": "6.14.5",
     "openapi-backend": "2.4.1",
@@ -88,6 +90,7 @@
   },
   "scripts": {
     "test": "nyc mocha --timeout 5000 ../tests/backend/specs/api",
+    "test-contentcollector": "nyc mocha --timeout 5000 ../tests/backend/specs",
     "test-container": "nyc mocha --timeout 5000 ../tests/container/specs/api"
   },
   "version": "1.8.4",

src/static/js/contentcollector.js

@@ -247,10 +247,15 @@ function makeContentCollector(collectStyles, abrowser, apool, domInterface, clas
 
   function _enterList(state, listType)
   {
+    if(!listType) return;
     var oldListType = state.lineAttributes['list'];
     if (listType != 'none')
     {
       state.listNesting = (state.listNesting || 0) + 1;
+      // reminder that listType can be "number2", "number3" etc.
+      if(listType.indexOf("number") !== -1){
+        state.start = (state.start || 0) + 1;
+      }
     }
 
     if(listType === 'none' || !listType ){
@@ -259,19 +264,22 @@ function makeContentCollector(collectStyles, abrowser, apool, domInterface, clas
     else{
       state.lineAttributes['list'] = listType;
     }
-
     _recalcAttribString(state);
     return oldListType;
   }
 
   function _exitList(state, oldListType)
   {
-    if (state.lineAttributes['list'])
-    {
+    if (state.lineAttributes['list']) {
       state.listNesting--;
     }
-    if (oldListType && oldListType != 'none') { state.lineAttributes['list'] = oldListType; }
-    else { delete state.lineAttributes['list']; }
+    if (oldListType && oldListType != 'none') {
+      state.lineAttributes['list'] = oldListType;
+    }
+    else {
+      delete state.lineAttributes['list'];
+      delete state.lineAttributes['start'];
+    }
     _recalcAttribString(state);
   }
 
@@ -411,7 +419,14 @@ function makeContentCollector(collectStyles, abrowser, apool, domInterface, clas
         styl: null,
         cls: null
       });
-      var txt = (typeof(txtFromHook)=='object'&&txtFromHook.length==0)?dom.nodeValue(node):txtFromHook[0];
+
+      if(typeof(txtFromHook)=='object'){
+        txt = dom.nodeValue(node)
+      }else{
+        if(txtFromHook){
+          txt = txtFromHook
+        };
+      }
 
       var rest = '';
       var x = 0; // offset into original text
@@ -602,6 +617,48 @@ function makeContentCollector(collectStyles, abrowser, apool, domInterface, clas
             // See https://github.com/ether/etherpad-lite/issues/2412 for reasoning
             if(!abrowser.chrome) oldListTypeOrNull = (_enterList(state, type) || 'none');
           }
+          else if ((tname === "li")){
+            state.lineAttributes['start'] = state.start || 0;
+            _recalcAttribString(state);
+            if(state.lineAttributes.list.indexOf("number") !== -1){
+              /*
+               Nested OLs are not --> <ol><li>1</li><ol>nested</ol></ol>
+               They are           --> <ol><li>1</li><li><ol><li>nested</li></ol></li></ol>
+               Note how the <ol> item has to be inside a <li>
+               Because of this we don't increment the start number
+              */
+              if(node.parent && node.parent.name !== "ol"){
+                /*
+                TODO: start number has to increment based on indentLevel(numberX)
+                This means we have to build an object IE
+                {
+                 1: 4
+                 2: 3
+                 3: 5
+                }
+                But the browser seems to handle it fine using CSS..  Why can't we do the same
+                with exports?  We can..  But let's leave this comment in because it might be useful
+                in the future..
+                */
+                state.start++; // not if it's parent is an OL or UL.
+              }
+            }
+            // UL list items never modify the start value.
+            if(node.parent && node.parent.name === "ul"){
+              state.start++;
+              // TODO, this is hacky.
+              // Because if the first item is an UL it will increment a list no?
+              // A much more graceful way would be to say, ul increases if it's within an OL
+              // But I don't know a way to do that because we're only aware of the previous Line
+              // As the concept of parent's doesn't exist when processing each domline...
+            }
+
+          }else{
+            // Below needs more testin if it's neccesary as _exitList should take care of this.
+            // delete state.start;
+            // delete state.listNesting;
+            // _recalcAttribString(state);
+          }
           if (className2Author && cls)
           {
             var classes = cls.match(/\S+/g);
@@ -665,12 +722,16 @@ function makeContentCollector(collectStyles, abrowser, apool, domInterface, clas
     {
       if (lines.length() - 1 == startLine)
       {
-        // to solve #2412 - https://github.com/ether/etherpad-lite/issues/2412
         // added additional check to resolve https://github.com/JohnMcLear/ep_copy_paste_images/issues/20
         // this does mean that images etc can't be pasted on lists but imho that's fine
-        if(state.lineAttributes && !state.lineAttributes.list){
+
+        // If we're doing an export event we need to start a new lines
+        // Export events don't have window available.
+        // commented out to solve #2412 - https://github.com/ether/etherpad-lite/issues/2412
+        if((state.lineAttributes && !state.lineAttributes.list) || typeof window === "undefined"){
           cc.startNewLine(state);
         }
+
       }
       else
       {

src/static/js/linestylefilter.js

@@ -81,8 +81,10 @@ linestylefilter.getLineStyleFilter = function(lineLength, aline, textAndClassFun
       var classes = '';
       var isLineAttribMarker = false;
 
+      // For each attribute number
       Changeset.eachAttribNumber(attribs, function(n)
       {
+        // Give us this attributes key
         var key = apool.getAttribKey(n);
         if (key)
         {
@@ -100,8 +102,8 @@ linestylefilter.getLineStyleFilter = function(lineLength, aline, textAndClassFun
             {
               classes += ' list:' + value;
             }
-            else if (key == 'start')
-            {
+            else if (key == 'start'){
+              // Needed to introduce the correct Ordered list item start number on import
               classes += ' start:' + value;
             }
             else if (linestylefilter.ATTRIB_CLASSES[key])

src/templates/export_html.html

@@ -7,132 +7,29 @@
 <meta name="changedby" content="Etherpad">
 <meta charset="utf-8">
 <style>
-* {
-  font-family: arial, sans-serif;
-  font-size: 13px;
-  line-height: 17px;
-}
-ul.indent {
-  list-style-type: none;
-}
 ol {
-  list-style-type: none;
-  padding-left: 0;
+  counter-reset: item;
 }
-body > ol {
-  counter-reset: first second third fourth fifth sixth seventh eighth ninth tenth eleventh twelfth thirteenth fourteenth fifteenth sixteenth;
+
+ol > li {
+  counter-increment: item;
 }
+
+ol ol > li {
+  display: block;
+}
+
+ol > li {
+  display: block;
+}
+
 ol > li:before {
-  content: counter(first) ". ";
-  counter-increment: first;
+  content: counters(item, ".") ". ";
 }
-ol > ol > li:before {
-  content: counter(first) "." counter(second) ". ";
-  counter-increment: second;
-}
-ol > ol > ol > li:before {
-  content: counter(first) "." counter(second) "." counter(third) ". ";
-  counter-increment: third;
-}
-ol > ol > ol > ol > li:before {
-  content: counter(first) "." counter(second) "." counter(third) "." counter(fourth) ". ";
-  counter-increment: fourth;
-}
-ol > ol > ol > ol > ol > li:before {
-  content: counter(first) "." counter(second) "." counter(third) "." counter(fourth) "." counter(fifth) ". ";
-  counter-increment: fifth;
-}
-ol > ol > ol > ol > ol > ol > li:before {
-  content: counter(first) "." counter(second) "." counter(third) "." counter(fourth) "." counter(fifth) "." counter(sixth) ". ";
-  counter-increment: sixth;
-}
-ol > ol > ol > ol > ol > ol > ol > li:before {
-  content: counter(first) "." counter(second) "." counter(third) "." counter(fourth) "." counter(fifth) "." counter(sixth) "." counter(seventh) ". ";
-  counter-increment: seventh;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > li:before {
-  content: counter(first) "." counter(second) "." counter(third) "." counter(fourth) "." counter(fifth) "." counter(sixth) "." counter(seventh) "." counter(eighth) ". ";
-  counter-increment: eighth;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > ol > li:before {
-  content: counter(first) "." counter(second) "." counter(third) "." counter(fourth) "." counter(fifth) "." counter(sixth) "." counter(seventh) "." counter(eighth) "." counter(ninth) ". ";
-  counter-increment: ninth;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > li:before {
-  content: counter(first) "." counter(second) "." counter(third) "." counter(fourth) "." counter(fifth) "." counter(sixth) "." counter(seventh) "." counter(eighth) "." counter(ninth) "." counter(tenth) ". ";
-  counter-increment: tenth;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > li:before {
-  content: counter(first) "." counter(second) "." counter(third) "." counter(fourth) "." counter(fifth) "." counter(sixth) "." counter(seventh) "." counter(eighth) "." counter(ninth) "." counter(tenth) "." counter(eleventh) ". ";
-  counter-increment: eleventh;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > li:before {
-  content: counter(first) "." counter(second) "." counter(third) "." counter(fourth) "." counter(fifth) "." counter(sixth) "." counter(seventh) "." counter(eighth) "." counter(ninth) "." counter(tenth) "." counter(eleventh) "." counter(twelfth) ". ";
-  counter-increment: twelfth;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > li:before {
-  content: counter(first) "." counter(second) "." counter(third) "." counter(fourth) "." counter(fifth) "." counter(sixth) "." counter(seventh) "." counter(eighth) "." counter(ninth) "." counter(tenth) "." counter(eleventh) "." counter(twelfth) "." counter(thirteenth) ". ";
-  counter-increment: thirteenth;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > li:before {
-  content: counter(first) "." counter(second) "." counter(third) "." counter(fourth) "." counter(fifth) "." counter(sixth) "." counter(seventh) "." counter(eighth) "." counter(ninth) "." counter(tenth) "." counter(eleventh) "." counter(twelfth) "." counter(thirteenth) "." counter(fourteenth) ". ";
-  counter-increment: fourteenth;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > li:before {
-  content: counter(first) "." counter(second) "." counter(third) "." counter(fourth) "." counter(fifth) "." counter(sixth) "." counter(seventh) "." counter(eighth) "." counter(ninth) "." counter(tenth) "." counter(eleventh) "." counter(twelfth) "." counter(thirteenth) "." counter(fourteenth) "." counter(fifteenth) ". ";
-  counter-increment: fifteenth;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > li:before {
-  content: counter(first) "." counter(second) "." counter(third) "." counter(fourth) "." counter(fifth) "." counter(sixth) "." counter(seventh) "." counter(eighth) "." counter(ninth) "." counter(tenth) "." counter(eleventh) "." counter(twelfth) "." counter(thirteenth) "." counter(fourteenth) "." counter(fifteenth) "." counter(sixteenth) ". ";
-  counter-increment: sixteenth;
-}
-ol {
-  text-indent: 0px;
-}
-ol > ol {
-  text-indent: 10px;
-}
-ol > ol > ol {
-  text-indent: 20px;
-}
-ol > ol > ol > ol {
-  text-indent: 30px;
-}
-ol > ol > ol > ol > ol {
-  text-indent: 40px;
-}
-ol > ol > ol > ol > ol > ol {
-  text-indent: 50px;
-}
-ol > ol > ol > ol > ol > ol > ol {
-  text-indent: 60px;
-}
-ol > ol > ol > ol > ol > ol > ol > ol {
-  text-indent: 70px;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > ol {
-  text-indent: 80px;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > ol > ol {
-  text-indent: 90px;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol {
-  text-indent: 100px;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol {
-  text-indent: 110px;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol {
-  text-indent: 120px;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol {
-  text-indent: 130px;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol {
-  text-indent: 140px;
-}
-ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol > ol {
-  text-indent: 150px;
+
+ol ol > li:before {
+  content: counters(item, ".") ". ";
+  margin-left: -20px;
 }
 <%- extraCSS %>
 </style>

test.js

@@ -0,0 +1,77 @@
+const Changeset        = require("./src/static/js/Changeset");
+const contentcollector = require("./src/static/js/contentcollector");
+const AttributePool    = require("./src/static/js/AttributePool");
+const cheerio          = require("./src/node_modules/cheerio");
+const util             = require('util');
+
+const tests = {
+  bulletListInOL:{
+    description : "A bullet within an OL should not change numbering..",
+    html : "<html><body><ol><li>should be 1</li><ul><li>should be a bullet</li></ul><li>should be 2</li></ol><p></p></body></html>",
+    expectedLineAttribs : [ '*0*1*2*3+1+b', '*0*4*2*5+1+i', '*0*1*2*5+1+b', '' ],
+    expectedText: ["*should be 1","*should be a bullet","*should be 2", ""]
+  }
+}
+
+// For each test..
+for (let test in tests){
+  let testObj = tests[test];
+
+//  describe(test, function() {
+//    it(testObj.description, function(done) {
+      var $ = cheerio.load(testObj.html);  // Load HTML into Cheerio
+      var doc = $('html')[0]; // Creates a dom-like representation of HTML
+      // Create an empty attribute pool
+      var apool = new AttributePool();
+      // Convert a dom tree into a list of lines and attribute liens
+      // using the content collector object
+      var cc = contentcollector.makeContentCollector(true, null, apool);
+      cc.collectContent(doc);
+      var result = cc.finish();
+      var recievedAttributes = result.lineAttribs;
+      var expectedAttributes = testObj.expectedLineAttribs;
+      var recievedText = new Array(result.lines)
+      var expectedText = testObj.expectedText;
+
+      // Check recieved text matches the expected text
+      if(arraysEqual(recievedText[0], expectedText)){
+        console.log("PASS: Recieved Text did match Expected Text\nRecieved:", recievedText[0], "\nExpected:", testObj.expectedText)
+      }else{
+         console.error("FAIL: Recieved Text did not match Expected Text\nRecieved:", recievedText[0], "\nExpected:", testObj.expectedText)
+        // throw new Error();
+      }
+
+      // Check recieved attributes matches the expected attributes
+      if(arraysEqual(recievedAttributes, expectedAttributes)){
+        console.log("PASS: Recieved Attributes matched Expected Attributes");
+        done();
+      }else{
+        console.error("FAIL", test, testObj.description);
+        console.error("FAIL: Recieved Attributes did not match Expected Attributes\nRecieved: ", recievedAttributes, "\nExpected: ", expectedAttributes)
+        console.error("FAILING HTML", testObj.html);
+        //throw new Error();
+      }
+//    });
+
+//  });
+
+};
+
+
+
+
+function arraysEqual(a, b) {
+  if (a === b) return true;
+  if (a == null || b == null) return false;
+  if (a.length != b.length) return false;
+
+  // If you don't care about the order of the elements inside
+  // the array, you should sort both arrays here.
+  // Please note that calling sort on an array will modify that array.
+  // you might want to clone your array first.
+
+  for (var i = 0; i < a.length; ++i) {
+    if (a[i] !== b[i]) return false;
+  }
+  return true;
+}

tests/backend/specs/api/pad.js

@@ -33,7 +33,20 @@ var ulHtml = '<!doctype html><html><body><ul class="bullet"><li>one</li><li>two<
  * textually, but at least it remains standard compliant and has an equal DOM
  * structure.
  */
-var expectedHtml = '<!doctype html><html><body><ul class="bullet"><li>one</li><li>two</li><li>0</li><li>1</li><li>2<ul class="bullet"><li>3</li><li>4</ul></li></ul><ol class="number"><li>item<ol class="number"><li>item1</li><li>item2</ol></li></ol></body></html>';
+var expectedHtml = '<!doctype html><html><body><ul class="bullet"><li>one</li><li>two</li><li>0</li><li>1</li><li>2<ul class="bullet"><li>3</li><li>4</ul></li></ul><ol start="1" class="number"><li>item<ol start="2" class="number"><li>item1</li><li>item2</ol></li></ol></body></html>';
+
+/*
+ * Html document with space between list items, to test its import and
+ * verify it is exported back correctly
+ */
+var ulSpaceHtml = '<!doctype html><html><body><ul class="bullet"> <li>one</li></ul></body></html>';
+
+/*
+ * When exported back, Etherpad produces an html which is not exactly the same
+ * textually, but at least it remains standard compliant and has an equal DOM
+ * structure.
+ */
+var expectedSpaceHtml = '<!doctype html><html><body><ul class="bullet"><li>one</ul></body></html>';
 
 describe('Connectivity', function(){
   it('can connect', function(done) {
@@ -573,7 +586,7 @@ describe('setHTML', function(){
       "html":  html,
     })
     .expect(function(res){
-      if(res.body.code !== 1) throw new Error("Allowing crappy HTML to be imported")
+      if(res.body.code !== 0) throw new Error("Crappy HTML Can't be Imported[we weren't able to sanitize it']")
     })
     .expect('Content-Type', /json/)
     .expect(200, done)
@@ -618,6 +631,40 @@ describe('getHTML', function(){
   });
 })
 
+describe('setHTML', function(){
+  it('Sets the HTML of a Pad with white space between list items', function(done) {
+    api.get(endPoint('setHTML')+"&padID="+testPadId+"&html="+ulSpaceHtml)
+    .expect(function(res){
+      if(res.body.code !== 0) throw new Error("List HTML cant be imported")
+    })
+    .expect('Content-Type', /json/)
+    .expect(200, done)
+  });
+})
+
+describe('getHTML', function(){
+  it('Gets back the HTML of a Pad with complex nested lists of different types', function(done) {
+    api.get(endPoint('getHTML')+"&padID="+testPadId)
+    .expect(function(res){
+      var receivedHtml = res.body.data.html.replace("<br></body>", "</body>").toLowerCase();
+console.log(receivedHtml);
+      if (receivedHtml !== expectedSpaceHtml) {
+        throw new Error(`HTML received from export is not the one we were expecting.
+           Received:
+           ${receivedHtml}
+
+           Expected:
+           ${expectedSpaceHtml}
+
+           Which is a slightly modified version of the originally imported one:
+           ${ulSpaceHtml}`);
+      }
+    })
+    .expect('Content-Type', /json/)
+    .expect(200, done)
+  });
+})
+
 describe('createPad', function(){
   it('errors if pad can be created', function(done) {
     var badUrlChars = ["/", "%23", "%3F", "%26"];

tests/backend/specs/contentcollector.js

@@ -0,0 +1,175 @@
+const Changeset        = require("../../../src/static/js/Changeset");
+const contentcollector = require("../../../src/static/js/contentcollector");
+const AttributePool    = require("../../../src/static/js/AttributePool");
+const cheerio          = require("../../../src/node_modules/cheerio");
+const util             = require('util');
+
+const tests = {
+  nestedLi:{
+    description: "Complex nested Li",
+    html: '<!doctype html><html><body><ol><li>one</li><li><ol><li>1.1</li></ol></li><li>two</li></ol></body></html>',
+    expectedLineAttribs : [
+      '*0*1*2*3+1+3', '*0*4*2*5+1+3', '*0*1*2*5+1+3'
+    ],
+    expectedText: [
+      '*one', '*1.1', '*two'
+    ]
+  },
+  complexNest:{
+    description: "Complex list of different types",
+    html: '<!doctype html><html><body><ul class="bullet"><li>one</li><li>two</li><li>0</li><li>1</li><li>2<ul class="bullet"><li>3</li><li>4</li></ul></li></ul><ol class="number"><li>item<ol class="number"><li>item1</li><li>item2</li></ol></li></ol></body></html>',
+    expectedLineAttribs : [
+      '*0*1*2+1+3',
+      '*0*1*2+1+3',
+      '*0*1*2+1+1',
+      '*0*1*2+1+1',
+      '*0*1*2+1+1',
+      '*0*3*2+1+1',
+      '*0*3*2+1+1',
+      '*0*4*2*5+1+4',
+      '*0*6*2*7+1+5',
+      '*0*6*2*7+1+5'
+    ],
+    expectedText: [
+      '*one',   '*two',
+      '*0',     '*1',
+      '*2',     '*3',
+      '*4',     '*item',
+      '*item1', '*item2'
+    ]
+  },
+  ul: {
+    description : "Tests if uls properly get attributes",
+    html : "<html><body><ul><li>a</li><li>b</li></ul><div>div</div><p>foo</p></body></html>",
+    expectedLineAttribs : [ '*0*1*2+1+1', '*0*1*2+1+1', '+3' , '+3'],
+    expectedText: ["*a","*b", "div", "foo"]
+  }
+  ,
+  ulIndented: {
+    description : "Tests if indented uls properly get attributes",
+    html : "<html><body><ul><li>a</li><ul><li>b</li></ul><li>a</li></ul><p>foo</p></body></html>",
+    expectedLineAttribs : [ '*0*1*2+1+1', '*0*3*2+1+1', '*0*1*2+1+1', '+3' ],
+    expectedText: ["*a","*b", "*a", "foo"]
+  },
+  ol: {
+    description : "Tests if ols properly get line numbers when in a normal OL",
+    html : "<html><body><ol><li>a</li><li>b</li><li>c</li></ol><p>test</p></body></html>",
+    expectedLineAttribs : ['*0*1*2*3+1+1', '*0*1*2*3+1+1', '*0*1*2*3+1+1', '+4'],
+    expectedText: ["*a","*b","*c", "test"],
+    noteToSelf: "Ensure empty P does not induce line attribute marker, wont this break the editor?"
+  }
+  ,
+  lineDoBreakInOl:{
+    description : "A single completely empty line break within an ol should reset count if OL is closed off..",
+    html : "<html><body><ol><li>should be 1</li></ol><p>hello</p><ol><li>should be 1</li><li>should be 2</li></ol><p></p></body></html>",
+    expectedLineAttribs : [ '*0*1*2*3+1+b', '+5', '*0*1*2*4+1+b', '*0*1*2*4+1+b', '' ],
+    expectedText: ["*should be 1","hello", "*should be 1","*should be 2", ""],
+    noteToSelf: "Shouldn't include attribute marker in the <p> line"
+  },
+  bulletListInOL:{
+    description : "A bullet within an OL should not change numbering..",
+    html : "<html><body><ol><li>should be 1</li><ul><li>should be a bullet</li></ul><li>should be 2</li></ol><p></p></body></html>",
+    expectedLineAttribs : [ '*0*1*2*3+1+b', '*0*4*2*3+1+i', '*0*1*2*5+1+b', '' ],
+    expectedText: ["*should be 1","*should be a bullet","*should be 2", ""]
+  },
+  testP:{
+    description : "A single <p></p> should create a new line",
+    html : "<html><body><p></p><p></p></body></html>",
+    expectedLineAttribs : [ '', ''],
+    expectedText: ["", ""],
+    noteToSelf: "<p></p>should create a line break but not break numbering"
+  },
+  nestedOl: {
+    description : "Tests if ols properly get line numbers when in a normal OL",
+    html : "<html><body>a<ol><li>b<ol><li>c</li></ol></ol>notlist<p>foo</p></body></html>",
+    expectedLineAttribs : [ '+1', '*0*1*2*3+1+1', '*0*4*2*5+1+1', '+7', '+3' ],
+    expectedText: ["a","*b","*c", "notlist", "foo"],
+    noteToSelf: "Ensure empty P does not induce line attribute marker, wont this break the editor?"
+  },
+  nestedOl: {
+    description : "First item being an UL then subsequent being OL will fail",
+    html : "<html><body><ul><li>a<ol><li>b</li><li>c</li></ol></li></ul></body></html>",
+    expectedLineAttribs : [ '+1', '*0*1*2*3+1+1', '*0*4*2*5+1+1' ],
+    expectedText: ["a","*b","*c"],
+    noteToSelf: "Ensure empty P does not induce line attribute marker, wont this break the editor?",
+    disabled: true
+  },
+  lineDontBreakOL:{
+    description : "A single completely empty line break within an ol should NOT reset count",
+    html : "<html><body><ol><li>should be 1</li><p></p><li>should be 2</li><li>should be 3</li></ol><p></p></body></html>",
+    expectedLineAttribs : [ ],
+    expectedText: ["*should be 1","*should be 2","*should be 3"],
+    noteToSelf: "<p></p>should create a line break but not break numbering -- This is what I can't get working!",
+    disabled: true
+  }
+
+}
+
+// For each test..
+for (let test in tests){
+  let testObj = tests[test];
+
+  describe(test, function() {
+    if(testObj.disabled){
+      return xit("DISABLED:", test, function(done){
+        done();
+      })
+    }
+
+    it(testObj.description, function(done) {
+      var $ = cheerio.load(testObj.html);  // Load HTML into Cheerio
+      var doc = $('html')[0]; // Creates a dom-like representation of HTML
+      // Create an empty attribute pool
+      var apool = new AttributePool();
+      // Convert a dom tree into a list of lines and attribute liens
+      // using the content collector object
+      var cc = contentcollector.makeContentCollector(true, null, apool);
+      cc.collectContent(doc);
+      var result = cc.finish();
+      var recievedAttributes = result.lineAttribs;
+      var expectedAttributes = testObj.expectedLineAttribs;
+      var recievedText = new Array(result.lines)
+      var expectedText = testObj.expectedText;
+
+      // Check recieved text matches the expected text
+      if(arraysEqual(recievedText[0], expectedText)){
+        // console.log("PASS: Recieved Text did match Expected Text\nRecieved:", recievedText[0], "\nExpected:", testObj.expectedText)
+      }else{
+        console.error("FAIL: Recieved Text did not match Expected Text\nRecieved:", recievedText[0], "\nExpected:", testObj.expectedText)
+        throw new Error();
+      }
+
+      // Check recieved attributes matches the expected attributes
+      if(arraysEqual(recievedAttributes, expectedAttributes)){
+        // console.log("PASS: Recieved Attributes matched Expected Attributes");
+        done();
+      }else{
+        console.error("FAIL", test, testObj.description);
+        console.error("FAIL: Recieved Attributes did not match Expected Attributes\nRecieved: ", recievedAttributes, "\nExpected: ", expectedAttributes)
+        console.error("FAILING HTML", testObj.html);
+        throw new Error();
+      }
+    });
+
+  });
+
+};
+
+
+
+
+function arraysEqual(a, b) {
+  if (a === b) return true;
+  if (a == null || b == null) return false;
+  if (a.length != b.length) return false;
+
+  // If you don't care about the order of the elements inside
+  // the array, you should sort both arrays here.
+  // Please note that calling sort on an array will modify that array.
+  // you might want to clone your array first.
+
+  for (var i = 0; i < a.length; ++i) {
+    if (a[i] !== b[i]) return false;
+  }
+  return true;
+}

tests/frontend/travis/runnerBackend.sh

@@ -41,6 +41,7 @@ sleep 10
 echo "Now run the backend tests"
 cd src
 npm run test
+npm run test-contentcollector
 exit_code=$?
 
 kill $!