Merge pull request #2418 from ether/etherpad-export-and-import

Full Pad portability (Export/Import)

src/locales/en.json

@@ -45,6 +45,7 @@
   "pad.importExport.import": "Upload any text file or document",
   "pad.importExport.importSuccessful": "Successful!",
   "pad.importExport.export": "Export current pad as:",
+  "pad.importExport.exportetherpad": "Etherpad",
   "pad.importExport.exporthtml": "HTML",
   "pad.importExport.exportplain": "Plain text",
   "pad.importExport.exportword": "Microsoft Word",
@@ -130,6 +131,7 @@
   "pad.impexp.importing": "Importing...",
   "pad.impexp.confirmimport": "Importing a file will overwrite the current text of the pad. Are you sure you want to proceed?",
   "pad.impexp.convertFailed": "We were not able to import this file. Please use a different document format or copy paste manually",
+  "pad.impexp.padHasData": "We were not able to import this file because this Pad has already had changes, please import to a new pad",
   "pad.impexp.uploadFailed": "The upload failed, please try again",
   "pad.impexp.importfailed": "Import failed",
   "pad.impexp.copypaste": "Please copy paste",

src/node/handler/ExportHandler.js

@@ -4,6 +4,7 @@
 
 /*
  * 2011 Peter 'Pita' Martischka (Primary Technology Ltd)
+ * 2014 John McLear (Etherpad Foundation / McLear Ltd)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,6 +22,7 @@
 var ERR = require("async-stacktrace");
 var exporthtml = require("../utils/ExportHtml");
 var exporttxt = require("../utils/ExportTxt");
+var exportEtherpad = require("../utils/ExportEtherpad");
 var async = require("async");
 var fs = require("fs");
 var settings = require('../utils/Settings');
@@ -52,14 +54,20 @@ exports.doExport = function(req, res, padId, type)
       // if fileName is set then set it to the padId, note that fileName is returned as an array.
       if(hookFileName.length) fileName = hookFileName;
 
-
       //tell the browser that this is a downloadable file
       res.attachment(fileName + "." + type);
     
       //if this is a plain text export, we can do this directly
       // We have to over engineer this because tabs are stored as attributes and not plain text
-    
+      if(type == "etherpad"){
-      if(type == "txt")
+        exportEtherpad.getPadRaw(padId, function(err, pad){
+          if(!err){
+            res.send(pad);
+            // return;
+          }
+        });
+      }
+      else if(type == "txt")
       {
         var txt;
         var randNum;

src/node/handler/ImportHandler.js

@@ -5,6 +5,7 @@
 /*
  * 2011 Peter 'Pita' Martischka (Primary Technology Ltd)
  * 2012 Iván Eixarch
+ * 2014 John McLear (Etherpad Foundation / McLear Ltd)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -29,6 +30,7 @@ var ERR = require("async-stacktrace")
   , formidable = require('formidable')
   , os = require("os")
   , importHtml = require("../utils/ImportHtml")
+  , importEtherpad = require("../utils/ImportEtherpad")
   , log4js = require("log4js")
   , hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js");
 
@@ -53,7 +55,8 @@ exports.doImport = function(req, res, padId)
   var srcFile, destFile
     , pad
     , text
-    , importHandledByPlugin;
+    , importHandledByPlugin
+    , directDatabaseAccess;
 
   var randNum = Math.floor(Math.random()*0xFFFFFFFF);
   
@@ -83,7 +86,7 @@ exports.doImport = function(req, res, padId)
     //this allows us to accept source code files like .c or .java
     function(callback) {
       var fileEnding = path.extname(srcFile).toLowerCase()
-        , knownFileEndings = [".txt", ".doc", ".docx", ".pdf", ".odt", ".html", ".htm"]
+        , knownFileEndings = [".txt", ".doc", ".docx", ".pdf", ".odt", ".html", ".htm", ".etherpad"]
         , fileEndingKnown = (knownFileEndings.indexOf(fileEnding) > -1);
       
       //if the file ending is known, continue as normal
@@ -116,9 +119,33 @@ exports.doImport = function(req, res, padId)
         }
       });
     },
+    function(callback) {
+      var fileEnding = path.extname(srcFile).toLowerCase()
+      var fileIsEtherpad = (fileEnding === ".etherpad");
+
+      if(fileIsEtherpad){
+        // we do this here so we can see if the pad has quit ea few edits
+        padManager.getPad(padId, function(err, _pad){
+          var headCount = _pad.head;
+          if(headCount >= 10){
+            apiLogger.warn("Direct database Import attempt of a pad that already has content, we wont be doing this")
+            return callback("padHasData");
+          }else{
+            fs.readFile(srcFile, "utf8", function(err, _text){
+              directDatabaseAccess = true;
+              importEtherpad.setPadRaw(padId, _text, function(err){
+                callback();
+              });
+            });
+          }  
+        });
+      }else{
+        callback();
+      }
+    },
     //convert file to html
     function(callback) {
-      if(!importHandledByPlugin){
+      if(!importHandledByPlugin || !directDatabaseAccess){
         var fileEnding = path.extname(srcFile).toLowerCase();
         var fileIsHTML = (fileEnding === ".html" || fileEnding === ".htm");
         if (abiword && !fileIsHTML) {
@@ -141,24 +168,28 @@ exports.doImport = function(req, res, padId)
     },
     
     function(callback) {
-      if (!abiword) {
+      if (!abiword){
-        // Read the file with no encoding for raw buffer access.
+        if(!directDatabaseAccess) {
-        fs.readFile(destFile, function(err, buf) {
+          // Read the file with no encoding for raw buffer access.
-          if (err) throw err;
+          fs.readFile(destFile, function(err, buf) {
-          var isAscii = true;
+            if (err) throw err;
-          // Check if there are only ascii chars in the uploaded file
+            var isAscii = true;
-          for (var i=0, len=buf.length; i<len; i++) {
+            // Check if there are only ascii chars in the uploaded file
-            if (buf[i] > 240) {
+            for (var i=0, len=buf.length; i<len; i++) {
-              isAscii=false;
+              if (buf[i] > 240) {
-              break;
+                isAscii=false;
+                break;
+              }
             }
-          }
+            if (isAscii) {
-          if (isAscii) {
+              callback();
-            callback();
+            } else {
-          } else {
+              callback("uploadFailed");
-            callback("uploadFailed");
+            }
-          }
+          });
-        });
+        }else{
+          callback();
+        }
       } else {
         callback();
       }
@@ -175,64 +206,90 @@ exports.doImport = function(req, res, padId)
     
     //read the text
     function(callback) {
-      fs.readFile(destFile, "utf8", function(err, _text){
+      if(!directDatabaseAccess){
-        if(ERR(err, callback)) return;
+        fs.readFile(destFile, "utf8", function(err, _text){
-        text = _text;
+          if(ERR(err, callback)) return;
-        // Title needs to be stripped out else it appends it to the pad..
+          text = _text;
-        text = text.replace("<title>", "<!-- <title>");
+          // Title needs to be stripped out else it appends it to the pad..
-        text = text.replace("</title>","</title>-->");
+          text = text.replace("<title>", "<!-- <title>");
-
+          text = text.replace("</title>","</title>-->");
-        //node on windows has a delay on releasing of the file lock.  
+  
-        //We add a 100ms delay to work around this
+          //node on windows has a delay on releasing of the file lock.  
-        if(os.type().indexOf("Windows") > -1){
+          //We add a 100ms delay to work around this
-           setTimeout(function() {callback();}, 100);
+          if(os.type().indexOf("Windows") > -1){
-        } else {
+             setTimeout(function() {callback();}, 100);
-          callback();
+          } else {
-        }
+            callback();
-      });
+          }
+        });
+      }else{
+        callback();
+      }
     },
     
     //change text of the pad and broadcast the changeset
     function(callback) {
-      var fileEnding = path.extname(srcFile).toLowerCase();
+      if(!directDatabaseAccess){
-      if (abiword || fileEnding == ".htm" || fileEnding == ".html") {
+        var fileEnding = path.extname(srcFile).toLowerCase();
-        try{
+        if (abiword || fileEnding == ".htm" || fileEnding == ".html") {
-          importHtml.setPadHTML(pad, text);
+          try{
-        }catch(e){
+            importHtml.setPadHTML(pad, text);
-          apiLogger.warn("Error importing, possibly caused by malformed HTML");
+          }catch(e){
+            apiLogger.warn("Error importing, possibly caused by malformed HTML");
+          }
+        } else {
+          pad.setText(text);
         }
-      } else {
-        pad.setText(text);
       }
-      padMessageHandler.updatePadClients(pad, callback);
+
+      // Load the Pad into memory then brodcast updates to all clients
+      padManager.unloadPad(padId);
+      padManager.getPad(padId, function(err, _pad){
+        var pad = _pad;
+        padManager.unloadPad(padId);
+
+        // direct Database Access means a pad user should perform a switchToPad
+        // and not attempt to recieve updated pad data..
+        if(!directDatabaseAccess){
+          padMessageHandler.updatePadClients(pad, function(){
+            callback();
+          });
+        }else{
+          callback();
+        }
+      });
+
     },
     
     //clean up temporary files
     function(callback) {
-      //for node < 0.7 compatible
+      if(!directDatabaseAccess){
-      var fileExists = fs.exists || path.exists;
+        //for node < 0.7 compatible
-      async.parallel([
+        var fileExists = fs.exists || path.exists;
-        function(callback){
+        async.parallel([
-          fileExists (srcFile, function(exist) { (exist)? fs.unlink(srcFile, callback): callback(); });
+          function(callback){
-        },
+            fileExists (srcFile, function(exist) { (exist)? fs.unlink(srcFile, callback): callback(); });
-        function(callback){
+          },
-          fileExists (destFile, function(exist) { (exist)? fs.unlink(destFile, callback): callback(); });
+          function(callback){
-        }
+            fileExists (destFile, function(exist) { (exist)? fs.unlink(destFile, callback): callback(); });
-      ], callback);
+          }
+        ], callback);
+      }else{
+        callback();
+      }
     }
   ], function(err) {
-
     var status = "ok";
     
     //check for known errors and replace the status
-    if(err == "uploadFailed" || err == "convertFailed")
+    if(err == "uploadFailed" || err == "convertFailed" || err == "padHasData")
     {
       status = err;
       err = null;
     }
 
     ERR(err);
-  
+
     //close the connection
     res.send(
       "<head> \
@@ -243,7 +300,7 @@ exports.doImport = function(req, res, padId)
           if(navigator.userAgent.indexOf('MSIE') === -1){ \
             document.domain = document.domain; \
           } \
-          var impexp = window.parent.padimpexp.handleFrameCall('" + status + "'); \
+          var impexp = window.parent.padimpexp.handleFrameCall('" + directDatabaseAccess +"', '" + status + "'); \
         }) \
       </script>"
     , 200);

src/node/hooks/express/importexport.js

@@ -5,7 +5,7 @@ var importHandler = require('../../handler/ImportHandler');
 
 exports.expressCreateServer = function (hook_name, args, cb) {
   args.app.get('/p/:pad/:rev?/export/:type', function(req, res, next) {
-    var types = ["pdf", "doc", "txt", "html", "odt"];
+    var types = ["pdf", "doc", "txt", "html", "odt", "etherpad"];
     //send a 404 if we don't support this filetype
     if (types.indexOf(req.params.type) == -1) {
       next();

src/node/utils/ExportEtherpad.js

@@ -0,0 +1,68 @@
+/**
+ * 2014 John McLear (Etherpad Foundation / McLear Ltd)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+var async = require("async");
+var db = require("../db/DB").db;
+var ERR = require("async-stacktrace");
+
+exports.getPadRaw = function(padId, callback){
+  async.waterfall([
+  function(cb){
+
+    // Get the Pad available content keys
+    db.findKeys("pad:"+padId+"*", null, function(err,records){
+      if(!err){
+        cb(err, records);
+      }
+    })
+  },
+  function(records, cb){
+    var data = {};
+
+    async.forEachSeries(Object.keys(records), function(key, r){
+
+      // For each piece of info about a pad.
+      db.get(records[key], function(err, entry){
+        data[records[key]] = entry;
+
+        // Get the Pad Authors
+        if(entry.pool && entry.pool.numToAttrib){
+          var authors = entry.pool.numToAttrib;
+          async.forEachSeries(Object.keys(authors), function(k, c){
+            if(authors[k][0] === "author"){
+              var authorId = authors[k][1];
+
+              // Get the author info
+              db.get("globalAuthor:"+authorId, function(e, authorEntry){
+                if(!e) data["globalAuthor:"+authorId] = authorEntry;
+              });
+
+            }
+            // console.log("authorsK", authors[k]);
+            c(null);
+          });
+        }
+        r(null); // callback;
+      });
+    }, function(err){ 
+      cb(err, data);
+    })
+  }
+  ], function(err, data){
+    callback(null, data);
+  });
+}

src/node/utils/ImportEtherpad.js

@@ -0,0 +1,55 @@
+/**
+ * 2014 John McLear (Etherpad Foundation / McLear Ltd)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var log4js = require('log4js');
+var async = require("async");
+var db = require("../db/DB").db;
+
+exports.setPadRaw = function(padId, records, callback){
+  records = JSON.parse(records);
+
+  async.eachSeries(Object.keys(records), function(key, cb){
+    var value = records[key]
+
+    // we know its an author
+    if(value.padIDs){
+      // rewrite author pad ids
+      value.padIDs[padId] = 1;
+      var newKey = key;
+
+    }else{
+      // we can split it to look to see if its pad data
+      var oldPadId = key.split(":");
+
+      // we know its pad data..
+      if(oldPadId[0] === "pad"){
+
+        // so set the new pad id for the author
+        oldPadId[1] = padId;
+        
+        // and create the value
+        var newKey = oldPadId.join(":"); // create the new key
+      }
+
+    }
+    // Write the value to the server
+    db.set(newKey, value);
+
+    cb();
+  }, function(){
+    callback(null, true);
+  });
+}

src/static/css/pad.css

@@ -689,6 +689,9 @@ table#otheruserstable {
 #exportpdfa:before {
   content: "\e803";
 }
+#exportetherpada:before {
+  content: "\e806";
+}
 #exportopena:before {
   content: "\e805";
 }

src/static/js/pad_impexp.js

@@ -109,6 +109,8 @@ var padimpexp = (function()
       msg = html10n.get("pad.impexp.convertFailed");
     } else if(status === "uploadFailed"){
       msg = html10n.get("pad.impexp.uploadFailed");
+    } else if(status === "padHasData"){
+      msg = html10n.get("pad.impexp.padHasData");
     }
   
     function showError(fade)
@@ -198,6 +200,7 @@ var padimpexp = (function()
 
       // build the export links
       $("#exporthtmla").attr("href", pad_root_path + "/export/html");
+      $("#exportetherpada").attr("href", pad_root_path + "/export/etherpad");
       $("#exportplaina").attr("href", pad_root_path + "/export/txt");
 
       // activate action to import in the form
@@ -234,13 +237,13 @@ var padimpexp = (function()
       $('#importform').submit(fileInputSubmit);
       $('.disabledexport').click(cantExport);
     },
-    handleFrameCall: function(status)
+    handleFrameCall: function(directDatabaseAccess, status)
     {
       if (status !== "ok")
       {
         importFailed(status);
       }
-      
+      if(directDatabaseAccess) pad.switchToPad(clientVars.padId);
       importDone();
     },
     disable: function()

src/templates/pad.html

@@ -206,6 +206,7 @@
             <div class="column" id="exportColumn">
                 <h2 data-l10n-id="pad.importExport.export"></h2>
                 <% e.begin_block("exportColumn"); %>
+                <a id="exportetherpada" target="_blank" class="exportlink"><div class="exporttype" id="exportetherpad" data-l10n-id="pad.importExport.exportetherpad"></div></a>
                 <a id="exporthtmla" target="_blank" class="exportlink"><div class="exporttype" id="exporthtml" data-l10n-id="pad.importExport.exporthtml"></div></a>
                 <a id="exportplaina" target="_blank" class="exportlink"><div class="exporttype" id="exportplain" data-l10n-id="pad.importExport.exportplain"></div></a>
                 <a id="exportworda" target="_blank" class="exportlink"><div class="exporttype" id="exportword" data-l10n-id="pad.importExport.exportword"></div></a>