From 6dfc5f2c8868ebf7d4420e09b730f1e0dd09818f Mon Sep 17 00:00:00 2001 From: CeBe Date: Wed, 6 Mar 2013 18:16:30 +0100 Subject: [PATCH 1/8] a script that allows importing old etherpad db this script allows you to import the sql file generated with convert.js into all supported dbms, not only MySQL --- bin/importSqlFile.js | 45 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 bin/importSqlFile.js diff --git a/bin/importSqlFile.js b/bin/importSqlFile.js new file mode 100644 index 000000000..4e9b1f3ed --- /dev/null +++ b/bin/importSqlFile.js @@ -0,0 +1,45 @@ +var startTime = new Date().getTime(); +var fs = require("fs"); +var db = require("../src/node/db/DB"); +//var async = require("../src/node_modules/async"); + +var sqlFile = process.argv[2]; + +//stop if the settings file is not set +if(!sqlFile) +{ + console.error("Use: node importSqlIntoRedis.js $SQLFILE"); + process.exit(1); +} + +log("initializing db"); +db.init(function(){ + log("done"); + + log("open output file..."); + var file = fs.readFileSync(sqlFile, 'utf8'); + + var keyNo = 0; + + file.split("\n").forEach(function(l) { + if (l.substr(0, 27) == "REPLACE INTO store VALUES (") { + var pos = l.indexOf("', '"); + var key = l.substr(28, pos - 28); + var value = l.substr(pos + 4); + value = value.substr(0, value.length - 3); + db.db.set(key, value, null); + keyNo++; + } + }); + + db.db.doShutdown(function() { + log("finished, imported " + keyNo + " keys."); + process.exit(0); + }); +}); + + +function log(str) +{ + console.log((new Date().getTime() - startTime)/1000 + "\t" + str); +} \ No newline at end of file From db0d0d1f72abb0c98093a1a2b309e73c41a00dff Mon Sep 17 00:00:00 2001 From: CeBe Date: Wed, 6 Mar 2013 22:08:14 +0100 Subject: [PATCH 2/8] fixed problem with npm --- bin/importSqlFile.js | 62 +++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/bin/importSqlFile.js b/bin/importSqlFile.js index 4e9b1f3ed..b1642288b 100644 --- a/bin/importSqlFile.js +++ b/bin/importSqlFile.js @@ -1,44 +1,46 @@ var startTime = new Date().getTime(); -var fs = require("fs"); -var db = require("../src/node/db/DB"); -//var async = require("../src/node_modules/async"); -var sqlFile = process.argv[2]; +require("ep_etherpad-lite/node_modules/npm").load({}, function(er,npm) { -//stop if the settings file is not set -if(!sqlFile) -{ - console.error("Use: node importSqlIntoRedis.js $SQLFILE"); - process.exit(1); -} + var fs = require("fs"); + var db = require("ep_etherpad-lite/node/db/DB");; -log("initializing db"); -db.init(function(){ - log("done"); + var sqlFile = process.argv[2]; - log("open output file..."); - var file = fs.readFileSync(sqlFile, 'utf8'); + //stop if the settings file is not set + if(!sqlFile) + { + console.error("Use: node importSqlFile.js $SQLFILE"); + process.exit(1); + } - var keyNo = 0; + log("initializing db"); + db.init(function(){ + log("done"); - file.split("\n").forEach(function(l) { - if (l.substr(0, 27) == "REPLACE INTO store VALUES (") { - var pos = l.indexOf("', '"); - var key = l.substr(28, pos - 28); - var value = l.substr(pos + 4); - value = value.substr(0, value.length - 3); - db.db.set(key, value, null); - keyNo++; - } - }); + log("open output file..."); + var file = fs.readFileSync(sqlFile, 'utf8'); - db.db.doShutdown(function() { - log("finished, imported " + keyNo + " keys."); - process.exit(0); + var keyNo = 0; + + file.split("\n").forEach(function(l) { + if (l.substr(0, 27) == "REPLACE INTO store VALUES (") { + var pos = l.indexOf("', '"); + var key = l.substr(28, pos - 28); + var value = l.substr(pos + 4); + value = value.substr(0, value.length - 3); + db.db.set(key, value, null); + keyNo++; + } + }); + + db.db.doShutdown(function() { + log("finished, imported " + keyNo + " keys."); + process.exit(0); + }); }); }); - function log(str) { console.log((new Date().getTime() - startTime)/1000 + "\t" + str); From 4b7238c2cde98da86d07a4ea4529c8cb1786b9bc Mon Sep 17 00:00:00 2001 From: CeBe Date: Wed, 6 Mar 2013 22:28:00 +0100 Subject: [PATCH 3/8] improved output for importSqlFile --- bin/importSqlFile.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bin/importSqlFile.js b/bin/importSqlFile.js index b1642288b..b43a0622e 100644 --- a/bin/importSqlFile.js +++ b/bin/importSqlFile.js @@ -19,11 +19,12 @@ require("ep_etherpad-lite/node_modules/npm").load({}, function(er,npm) { log("done"); log("open output file..."); - var file = fs.readFileSync(sqlFile, 'utf8'); + var lines = fs.readFileSync(sqlFile, 'utf8').split("\n");; + var count = lines.length; var keyNo = 0; - file.split("\n").forEach(function(l) { + lines.forEach(function(l) { if (l.substr(0, 27) == "REPLACE INTO store VALUES (") { var pos = l.indexOf("', '"); var key = l.substr(28, pos - 28); @@ -31,8 +32,13 @@ require("ep_etherpad-lite/node_modules/npm").load({}, function(er,npm) { value = value.substr(0, value.length - 3); db.db.set(key, value, null); keyNo++; + process.stdout.write("."); + if (keyNo % 100 == 0) { + console.log(" " + keyNo + "/" + count); + } } }); + process.stdout.write("\n"); db.db.doShutdown(function() { log("finished, imported " + keyNo + " keys."); From 76fbc2960728ee9f6b7058e03ace0064b9302644 Mon Sep 17 00:00:00 2001 From: CeBe Date: Wed, 6 Mar 2013 22:36:00 +0100 Subject: [PATCH 4/8] improved output for importSqlFile --- bin/importSqlFile.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/importSqlFile.js b/bin/importSqlFile.js index b43a0622e..bdc2c4345 100644 --- a/bin/importSqlFile.js +++ b/bin/importSqlFile.js @@ -24,6 +24,7 @@ require("ep_etherpad-lite/node_modules/npm").load({}, function(er,npm) { var count = lines.length; var keyNo = 0; + process.stdout.write("Start importing " + count + " keys...\n"); lines.forEach(function(l) { if (l.substr(0, 27) == "REPLACE INTO store VALUES (") { var pos = l.indexOf("', '"); @@ -32,9 +33,8 @@ require("ep_etherpad-lite/node_modules/npm").load({}, function(er,npm) { value = value.substr(0, value.length - 3); db.db.set(key, value, null); keyNo++; - process.stdout.write("."); - if (keyNo % 100 == 0) { - console.log(" " + keyNo + "/" + count); + if (keyNo % 1000 == 0) { + process.stdout.write(" " + keyNo + "/" + count); } } }); From f2b173f566e6c53caa1f045a714c52753caec710 Mon Sep 17 00:00:00 2001 From: CeBe Date: Wed, 6 Mar 2013 22:38:18 +0100 Subject: [PATCH 5/8] improved output for importSqlFile --- bin/importSqlFile.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/importSqlFile.js b/bin/importSqlFile.js index bdc2c4345..faf6b1b9a 100644 --- a/bin/importSqlFile.js +++ b/bin/importSqlFile.js @@ -34,7 +34,7 @@ require("ep_etherpad-lite/node_modules/npm").load({}, function(er,npm) { db.db.set(key, value, null); keyNo++; if (keyNo % 1000 == 0) { - process.stdout.write(" " + keyNo + "/" + count); + process.stdout.write(" " + keyNo + "/" + count + "\n"); } } }); From 4026ba18156b0551191e7ac62a077ab16d694e25 Mon Sep 17 00:00:00 2001 From: CeBe Date: Thu, 7 Mar 2013 13:15:29 +0100 Subject: [PATCH 6/8] fixed saved data to be escaped properly --- bin/importSqlFile.js | 108 ++++++++++++++++++++++++++++++++----------- 1 file changed, 82 insertions(+), 26 deletions(-) diff --git a/bin/importSqlFile.js b/bin/importSqlFile.js index faf6b1b9a..6463088a9 100644 --- a/bin/importSqlFile.js +++ b/bin/importSqlFile.js @@ -3,7 +3,17 @@ var startTime = new Date().getTime(); require("ep_etherpad-lite/node_modules/npm").load({}, function(er,npm) { var fs = require("fs"); - var db = require("ep_etherpad-lite/node/db/DB");; + + var ueberDB = require("ep_etherpad-lite/node_modules/ueberDB"); + var settings = require("ep_etherpad-lite/node/utils/Settings"); + var log4js = require('ep_etherpad-lite/node_modules/log4js'); + + var dbWrapperSettings = { + cache: 0, + writeInterval: 100, + json: false // data is already json encoded + }; + var db = new ueberDB.database(settings.dbType, settings.dbSettings, dbWrapperSettings, log4js.getLogger("ueberDB")); var sqlFile = process.argv[2]; @@ -15,39 +25,85 @@ require("ep_etherpad-lite/node_modules/npm").load({}, function(er,npm) { } log("initializing db"); - db.init(function(){ - log("done"); + db.init(function(err) + { + //there was an error while initializing the database, output it and stop + if(err) + { + console.error("ERROR: Problem while initalizing the database"); + console.error(err.stack ? err.stack : err); + process.exit(1); + } + else + { + log("done"); - log("open output file..."); - var lines = fs.readFileSync(sqlFile, 'utf8').split("\n");; + log("open output file..."); + var lines = fs.readFileSync(sqlFile, 'utf8').split("\n"); - var count = lines.length; - var keyNo = 0; + var count = lines.length; + var keyNo = 0; - process.stdout.write("Start importing " + count + " keys...\n"); - lines.forEach(function(l) { - if (l.substr(0, 27) == "REPLACE INTO store VALUES (") { - var pos = l.indexOf("', '"); - var key = l.substr(28, pos - 28); - var value = l.substr(pos + 4); - value = value.substr(0, value.length - 3); - db.db.set(key, value, null); - keyNo++; - if (keyNo % 1000 == 0) { - process.stdout.write(" " + keyNo + "/" + count + "\n"); + process.stdout.write("Start importing " + count + " keys...\n"); + lines.forEach(function(l) { + if (l.substr(0, 27) == "REPLACE INTO store VALUES (") { + var pos = l.indexOf("', '"); + var key = l.substr(28, pos - 28); + var value = l.substr(pos + 3); + value = value.substr(0, value.length - 2); + console.log("key: " + key + " val: " + value); + console.log("unval: " + unescape(value)); + db.set(key, unescape(value), null); + keyNo++; + if (keyNo % 1000 == 0) { + process.stdout.write(" " + keyNo + "/" + count + "\n"); + } } - } - }); - process.stdout.write("\n"); + }); + process.stdout.write("\n"); - db.db.doShutdown(function() { - log("finished, imported " + keyNo + " keys."); - process.exit(0); - }); + db.doShutdown(function() { + log("finished, imported " + keyNo + " keys."); + process.exit(0); + }); + } }); }); function log(str) { console.log((new Date().getTime() - startTime)/1000 + "\t" + str); -} \ No newline at end of file +} + +unescape = function(val) { + // value is a string + if (val.substr(0, 1) == "'") { + val = val.substr(0, val.length - 1).substr(1); + + return val.replace(/\\[0nrbtZ\\'"]/g, function(s) { + switch(s) { + case "\\0": return "\0"; + case "\\n": return "\n"; + case "\\r": return "\r"; + case "\\b": return "\b"; + case "\\t": return "\t"; + case "\\Z": return "\x1a"; + default: return s.substr(1); + } + }); + } + + // value is a boolean or NULL + if (val == 'NULL') { + return null; + } + if (val == 'true') { + return true; + } + if (val == 'false') { + return false; + } + + // value is a number + return val; +}; From 70c329957d0a488f30df138f56ab5d975b0bd54b Mon Sep 17 00:00:00 2001 From: CeBe Date: Thu, 7 Mar 2013 14:05:55 +0100 Subject: [PATCH 7/8] additional ouput for importSqlFile --- bin/importSqlFile.js | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/importSqlFile.js b/bin/importSqlFile.js index 6463088a9..cd1e7df3b 100644 --- a/bin/importSqlFile.js +++ b/bin/importSqlFile.js @@ -61,6 +61,7 @@ require("ep_etherpad-lite/node_modules/npm").load({}, function(er,npm) { } }); process.stdout.write("\n"); + process.stdout.wirte("done. waiting for db to finish transaction. depended on dbms this may take some time...\n"); db.doShutdown(function() { log("finished, imported " + keyNo + " keys."); From 62c13b4c3fee68de0935648b889f509490e95caa Mon Sep 17 00:00:00 2001 From: CeBe Date: Thu, 7 Mar 2013 14:10:54 +0100 Subject: [PATCH 8/8] typo --- bin/importSqlFile.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/importSqlFile.js b/bin/importSqlFile.js index cd1e7df3b..6491cbea1 100644 --- a/bin/importSqlFile.js +++ b/bin/importSqlFile.js @@ -61,7 +61,7 @@ require("ep_etherpad-lite/node_modules/npm").load({}, function(er,npm) { } }); process.stdout.write("\n"); - process.stdout.wirte("done. waiting for db to finish transaction. depended on dbms this may take some time...\n"); + process.stdout.write("done. waiting for db to finish transaction. depended on dbms this may take some time...\n"); db.doShutdown(function() { log("finished, imported " + keyNo + " keys.");