libre-service.eu/pad.libre-service.eu-etherpad
libre-service.eu/pad.libre-service.eu-etherpad
1
0
Fork 0
mirror of https://github.com/ether/etherpad-lite.git synced 2025-01-31 19:02:59 +01:00
Code Issues Projects Releases Wiki Activity

clean

Browse source
This commit is contained in:
John McLear 2013-11-24 12:41:12 +00:00
parent 678c80d1dc
commit d6f1aa3236
27 changed files with 1622 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
bin/bin/buildDebian.sh Executable file
View file

@ -0,0 +1,43 @@
#!/usr/bin/env bash
# IMPORTANT
# Protect agaisnt mispelling a var and rm -rf /
set -u
set -e
SRC=/tmp/etherpad-deb-src
DIST=/tmp/etherpad-deb-dist
SYSROOT=${SRC}/sysroot
DEBIAN=${SRC}/DEBIAN
rm -rf ${DIST}
mkdir -p ${DIST}/
rm -rf ${SRC}
rsync -a bin/deb-src/ ${SRC}/
mkdir -p ${SYSROOT}/opt/
rsync --exclude '.git' -a . ${SYSROOT}/opt/etherpad/ --delete
find ${SRC}/ -type d -exec chmod 0755 {} \;
find ${SRC}/ -type f -exec chmod go-w {} \;
chown -R root:root ${SRC}/
let SIZE=`du -s ${SYSROOT} | sed s'/\s\+.*//'`+8
pushd ${SYSROOT}/
tar czf ${DIST}/data.tar.gz [a-z]*
popd
sed s"/SIZE/${SIZE}/" -i ${DEBIAN}/control
pushd ${DEBIAN}
tar czf ${DIST}/control.tar.gz *
popd
pushd ${DIST}/
echo 2.0 > ./debian-binary
find ${DIST}/ -type d -exec chmod 0755 {} \;
find ${DIST}/ -type f -exec chmod go-w {} \;
chown -R root:root ${DIST}/
ar r ${DIST}/etherpad-1.deb debian-binary control.tar.gz data.tar.gz
popd
rsync -a ${DIST}/etherpad-1.deb ./

70
bin/bin/buildForWindows.sh Executable file
View file

@ -0,0 +1,70 @@
#!/bin/sh
NODE_VERSION="0.8.4"
#Move to the folder where ep-lite is installed
cd `dirname $0`
#Was this script started in the bin folder? if yes move out
if [ -d "../bin" ]; then
cd "../"
fi
#Is wget installed?
hash wget > /dev/null 2>&1 || {
echo "Please install wget" >&2
exit 1
}
#Is zip installed?
hash zip > /dev/null 2>&1 || {
echo "Please install zip" >&2
exit 1
}
#Is zip installed?
hash unzip > /dev/null 2>&1 || {
echo "Please install unzip" >&2
exit 1
}
START_FOLDER=$(pwd);
echo "create a clean environment in /tmp/etherpad-lite-win..."
rm -rf /tmp/etherpad-lite-win
cp -ar . /tmp/etherpad-lite-win
cd /tmp/etherpad-lite-win
rm -rf node_modules
rm -f etherpad-lite-win.zip
echo "do a normal unix install first..."
bin/installDeps.sh || exit 1
echo "copy the windows settings template..."
cp settings.json.template settings.json
echo "resolve symbolic links..."
cp -rL node_modules node_modules_resolved
rm -rf node_modules
mv node_modules_resolved node_modules
echo "download windows node..."
cd bin
wget "http://nodejs.org/dist/v$NODE_VERSION/node.exe" -O ../node.exe
echo "remove git history to reduce folder size"
rm -rf .git/objects
echo "remove windows jsdom-nocontextify/test folder"
rm -rf /tmp/etherpad-lite-win/node_modules/ep_etherpad-lite/node_modules/jsdom-nocontextifiy/test/
rm -rf /tmp/etherpad-lite-win/src/node_modules/jsdom-nocontextifiy/test/
echo "create the zip..."
cd /tmp
zip -9 -r etherpad-lite-win.zip etherpad-lite-win
mv etherpad-lite-win.zip $START_FOLDER
echo "clean up..."
rm -rf /tmp/etherpad-lite-win
echo "Finished. You can find the zip in the Etherpad root folder, it's called etherpad-lite-win.zip"

141
bin/bin/checkPad.js Normal file
View file

@ -0,0 +1,141 @@
/*
This is a debug tool. It checks all revisions for data corruption
*/
if(process.argv.length != 3)
{
console.error("Use: node bin/checkPad.js $PADID");
process.exit(1);
}
//get the padID
var padId = process.argv[2];
//initalize the variables
var db, settings, padManager;
var npm = require("../src/node_modules/npm");
var async = require("../src/node_modules/async");
var Changeset = require("ep_etherpad-lite/static/js/Changeset");
async.series([
//load npm
function(callback) {
npm.load({}, function(er) {
callback(er);
})
},
//load modules
function(callback) {
settings = require('../src/node/utils/Settings');
db = require('../src/node/db/DB');
//intallize the database
db.init(callback);
},
//get the pad
function (callback)
{
padManager = require('../src/node/db/PadManager');
padManager.doesPadExists(padId, function(err, exists)
{
if(!exists)
{
console.error("Pad does not exist");
process.exit(1);
}
padManager.getPad(padId, function(err, _pad)
{
pad = _pad;
callback(err);
});
});
},
function (callback)
{
//create an array with key kevisions
//key revisions always save the full pad atext
var head = pad.getHeadRevisionNumber();
var keyRevisions = [];
for(var i=0;i<head;i+=100)
{
keyRevisions.push(i);
}
//run trough all key revisions
async.forEachSeries(keyRevisions, function(keyRev, callback)
{
//create an array of revisions we need till the next keyRevision or the End
var revisionsNeeded = [];
for(var i=keyRev;i<=keyRev+100 && i<=head; i++)
{
revisionsNeeded.push(i);
}
//this array will hold all revision changesets
var revisions = [];
//run trough all needed revisions and get them from the database
async.forEach(revisionsNeeded, function(revNum, callback)
{
db.db.get("pad:"+padId+":revs:" + revNum, function(err, revision)
{
revisions[revNum] = revision;
callback(err);
});
}, function(err)
{
if(err)
{
callback(err);
return;
}
//check if the pad has a pool
if(pad.pool === undefined )
{
console.error("Attribute pool is missing");
process.exit(1);
}
//check if there is a atext in the keyRevisions
if(revisions[keyRev] === undefined || revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined)
{
console.error("No atext in key revision " + keyRev);
callback();
return;
}
var apool = pad.pool;
var atext = revisions[keyRev].meta.atext;
for(var i=keyRev+1;i<=keyRev+100 && i<=head; i++)
{
try
{
//console.log("check revision " + i);
var cs = revisions[i].changeset;
atext = Changeset.applyToAText(cs, atext, apool);
}
catch(e)
{
console.error("Bad changeset at revision " + i + " - " + e.message);
callback();
return;
}
}
callback();
});
}, callback);
}
], function (err)
{
if(err) throw err;
else
{
console.log("finished");
process.exit(0);
}
});

453
bin/bin/convert.js Normal file
View file

@ -0,0 +1,453 @@
var startTime = new Date().getTime();
var fs = require("fs");
var ueberDB = require("../src/node_modules/ueberDB");
var mysql = require("../src/node_modules/ueberDB/node_modules/mysql");
var async = require("../src/node_modules/async");
var Changeset = require("ep_etherpad-lite/static/js/Changeset");
var randomString = require('ep_etherpad-lite/static/js/pad_utils').randomString;
var AttributePool = require("ep_etherpad-lite/static/js/AttributePool");
var settingsFile = process.argv[2];
var sqlOutputFile = process.argv[3];
//stop if the settings file is not set
if(!settingsFile || !sqlOutputFile)
{
console.error("Use: node convert.js $SETTINGSFILE $SQLOUTPUT");
process.exit(1);
}
log("read settings file...");
//read the settings file and parse the json
var settings = JSON.parse(fs.readFileSync(settingsFile, "utf8"));
log("done");
log("open output file...");
var sqlOutput = fs.openSync(sqlOutputFile, "w");
var sql = "SET CHARACTER SET UTF8;\n" +
"CREATE TABLE IF NOT EXISTS `store` ( \n" +
"`key` VARCHAR( 100 ) NOT NULL , \n" +
"`value` LONGTEXT NOT NULL , \n" +
"PRIMARY KEY ( `key` ) \n" +
") ENGINE = INNODB;\n" +
"START TRANSACTION;\n\n";
fs.writeSync(sqlOutput, sql);
log("done");
//set setings for ep db
var etherpadDB= new mysql.Client();
etherpadDB.host = settings.etherpadDB.host;
etherpadDB.port = settings.etherpadDB.port;
etherpadDB.database = settings.etherpadDB.database;
etherpadDB.user = settings.etherpadDB.user;
etherpadDB.password = settings.etherpadDB.password;
//get the timestamp once
var timestamp = new Date().getTime();
var padIDs;
async.series([
//get all padids out of the database...
function(callback)
{
log("get all padIds out of the database...");
etherpadDB.query("SELECT ID FROM PAD_META", [], function(err, _padIDs)
{
padIDs = _padIDs;
callback(err);
});
},
function(callback)
{
log("done");
//create a queue with a concurrency 100
var queue = async.queue(function (padId, callback)
{
convertPad(padId, function(err)
{
incrementPadStats();
callback(err);
});
}, 100);
//set the step callback as the queue callback
queue.drain = callback;
//add the padids to the worker queue
for(var i=0,length=padIDs.length;i<length;i++)
{
queue.push(padIDs[i].ID);
}
}
], function(err)
{
if(err) throw err;
//write the groups
var sql = "";
for(var proID in proID2groupID)
{
var groupID = proID2groupID[proID];
var subdomain = proID2subdomain[proID];
sql+="REPLACE INTO store VALUES (" + etherpadDB.escape("group:" + groupID) + ", " + etherpadDB.escape(JSON.stringify(groups[groupID]))+ ");\n";
sql+="REPLACE INTO store VALUES (" + etherpadDB.escape("mapper2group:subdomain:" + subdomain) + ", " + etherpadDB.escape(groupID)+ ");\n";
}
//close transaction
sql+="COMMIT;";
//end the sql file
fs.writeSync(sqlOutput, sql, undefined, "utf-8");
fs.closeSync(sqlOutput);
log("finished.");
process.exit(0);
});
function log(str)
{
console.log((new Date().getTime() - startTime)/1000 + "\t" + str);
}
var padsDone = 0;
function incrementPadStats()
{
padsDone++;
if(padsDone%100 == 0)
{
var averageTime = Math.round(padsDone/((new Date().getTime() - startTime)/1000));
log(padsDone + "/" + padIDs.length + "\t" + averageTime + " pad/s")
}
}
var proID2groupID = {};
var proID2subdomain = {};
var groups = {};
function convertPad(padId, callback)
{
var changesets = [];
var changesetsMeta = [];
var chatMessages = [];
var authors = [];
var apool;
var subdomain;
var padmeta;
async.series([
//get all needed db values
function(callback)
{
async.parallel([
//get the pad revisions
function(callback)
{
var sql = "SELECT * FROM `PAD_CHAT_TEXT` WHERE NUMID = (SELECT `NUMID` FROM `PAD_CHAT_META` WHERE ID=?)";
etherpadDB.query(sql, [padId], function(err, results)
{
if(!err)
{
try
{
//parse the pages
for(var i=0,length=results.length;i<length;i++)
{
parsePage(chatMessages, results[i].PAGESTART, results[i].OFFSETS, results[i].DATA, true);
}
}catch(e) {err = e}
}
callback(err);
});
},
//get the chat entries
function(callback)
{
var sql = "SELECT * FROM `PAD_REVS_TEXT` WHERE NUMID = (SELECT `NUMID` FROM `PAD_REVS_META` WHERE ID=?)";
etherpadDB.query(sql, [padId], function(err, results)
{
if(!err)
{
try
{
//parse the pages
for(var i=0,length=results.length;i<length;i++)
{
parsePage(changesets, results[i].PAGESTART, results[i].OFFSETS, results[i].DATA, false);
}
}catch(e) {err = e}
}
callback(err);
});
},
//get the pad revisions meta data
function(callback)
{
var sql = "SELECT * FROM `PAD_REVMETA_TEXT` WHERE NUMID = (SELECT `NUMID` FROM `PAD_REVMETA_META` WHERE ID=?)";
etherpadDB.query(sql, [padId], function(err, results)
{
if(!err)
{
try
{
//parse the pages
for(var i=0,length=results.length;i<length;i++)
{
parsePage(changesetsMeta, results[i].PAGESTART, results[i].OFFSETS, results[i].DATA, true);
}
}catch(e) {err = e}
}
callback(err);
});
},
//get the attribute pool of this pad
function(callback)
{
var sql = "SELECT `JSON` FROM `PAD_APOOL` WHERE `ID` = ?";
etherpadDB.query(sql, [padId], function(err, results)
{
if(!err)
{
try
{
apool=JSON.parse(results[0].JSON).x;
}catch(e) {err = e}
}
callback(err);
});
},
//get the authors informations
function(callback)
{
var sql = "SELECT * FROM `PAD_AUTHORS_TEXT` WHERE NUMID = (SELECT `NUMID` FROM `PAD_AUTHORS_META` WHERE ID=?)";
etherpadDB.query(sql, [padId], function(err, results)
{
if(!err)
{
try
{
//parse the pages
for(var i=0, length=results.length;i<length;i++)
{
parsePage(authors, results[i].PAGESTART, results[i].OFFSETS, results[i].DATA, true);
}
}catch(e) {err = e}
}
callback(err);
});
},
//get the pad information
function(callback)
{
var sql = "SELECT JSON FROM `PAD_META` WHERE ID=?";
etherpadDB.query(sql, [padId], function(err, results)
{
if(!err)
{
try
{
padmeta = JSON.parse(results[0].JSON).x;
}catch(e) {err = e}
}
callback(err);
});
},
//get the subdomain
function(callback)
{
//skip if this is no proPad
if(padId.indexOf("$") == -1)
{
callback();
return;
}
//get the proID out of this padID
var proID = padId.split("$")[0];
var sql = "SELECT subDomain FROM pro_domains WHERE ID = ?";
etherpadDB.query(sql, [proID], function(err, results)
{
if(!err)
{
subdomain = results[0].subDomain;
}
callback(err);
});
}
], callback);
},
function(callback)
{
//saves all values that should be written to the database
var values = {};
//this is a pro pad, let's convert it to a group pad
if(padId.indexOf("$") != -1)
{
var padIdParts = padId.split("$");
var proID = padIdParts[0];
var padName = padIdParts[1];
var groupID
//this proID is not converted so far, do it
if(proID2groupID[proID] == null)
{
groupID = "g." + randomString(16);
//create the mappers for this new group
proID2groupID[proID] = groupID;
proID2subdomain[proID] = subdomain;
groups[groupID] = {pads: {}};
}
//use the generated groupID;
groupID = proID2groupID[proID];
//rename the pad
padId = groupID + "$" + padName;
//set the value for this pad in the group
groups[groupID].pads[padId] = 1;
}
try
{
var newAuthorIDs = {};
var oldName2newName = {};
//replace the authors with generated authors
// we need to do that cause where the original etherpad saves pad local authors, the new (lite) etherpad uses them global
for(var i in apool.numToAttrib)
{
var key = apool.numToAttrib[i][0];
var value = apool.numToAttrib[i][1];
//skip non authors and anonymous authors
if(key != "author" || value == "")
continue;
//generate new author values
var authorID = "a." + randomString(16);
var authorColorID = authors[i].colorId || Math.floor(Math.random()*32);
var authorName = authors[i].name || null;
//overwrite the authorID of the attribute pool
apool.numToAttrib[i][1] = authorID;
//write the author to the database
values["globalAuthor:" + authorID] = {"colorId" : authorColorID, "name": authorName, "timestamp": timestamp};
//save in mappers
newAuthorIDs[i] = authorID;
oldName2newName[value] = authorID;
}
//save all revisions
for(var i=0;i<changesets.length;i++)
{
values["pad:" + padId + ":revs:" + i] = {changeset: changesets[i],
meta : {
author: newAuthorIDs[changesetsMeta[i].a],
timestamp: changesetsMeta[i].t,
atext: changesetsMeta[i].atext || undefined
}};
}
//save all chat messages
for(var i=0;i<chatMessages.length;i++)
{
values["pad:" + padId + ":chat:" + i] = {"text": chatMessages[i].lineText,
"userId": oldName2newName[chatMessages[i].userId],
"time": chatMessages[i].time}
}
//generate the latest atext
var fullAPool = (new AttributePool()).fromJsonable(apool);
var keyRev = Math.floor(padmeta.head / padmeta.keyRevInterval) * padmeta.keyRevInterval;
var atext = changesetsMeta[keyRev].atext;
var curRev = keyRev;
while (curRev < padmeta.head)
{
curRev++;
var changeset = changesets[curRev];
atext = Changeset.applyToAText(changeset, atext, fullAPool);
}
values["pad:" + padId] = {atext: atext,
pool: apool,
head: padmeta.head,
chatHead: padmeta.numChatMessages }
}
catch(e)
{
console.error("Error while converting pad " + padId + ", pad skipped");
console.error(e.stack ? e.stack : JSON.stringify(e));
callback();
return;
}
var sql = "";
for(var key in values)
{
sql+="REPLACE INTO store VALUES (" + etherpadDB.escape(key) + ", " + etherpadDB.escape(JSON.stringify(values[key]))+ ");\n";
}
fs.writeSync(sqlOutput, sql, undefined, "utf-8");
callback();
}
], callback);
}
/**
* This parses a Page like Etherpad uses them in the databases
* The offsets descripes the length of a unit in the page, the data are
* all values behind each other
*/
function parsePage(array, pageStart, offsets, data, json)
{
var start = 0;
var lengths = offsets.split(",");
for(var i=0;i<lengths.length;i++)
{
var unitLength = lengths[i];
//skip empty units
if(unitLength == "")
continue;
//parse the number
unitLength = Number(unitLength);
//cut the unit out of data
var unit = data.substr(start, unitLength);
//put it into the array
array[pageStart + i] = json ? JSON.parse(unit) : unit;
//update start
start+=unitLength;
}
}

10
bin/bin/convertSettings.json.template Normal file
View file

@ -0,0 +1,10 @@
{
"etherpadDB":
{
"host": "localhost",
"port": 3306,
"database": "etherpad",
"user": "etherpaduser",
"password": "yourpassword"
}
}

9
bin/bin/deb-src/DEBIAN/control Normal file
View file

@ -0,0 +1,9 @@
Package: etherpad
Version: 1.3
Section: base
Priority: optional
Architecture: i386
Installed-Size: SIZE
Depends:
Maintainer: John McLear <john@mclear.co.uk>
Description: Etherpad is a collaborative editor.

6
bin/bin/deb-src/DEBIAN/postinst Executable file
View file

@ -0,0 +1,6 @@
#!/bin/bash
# Start the services!
service etherpad start
echo "Give Etherpad about 3 minutes to install dependencies and start"
rm -f /tmp/etherpad.log /tmp/etherpad.err

26
bin/bin/deb-src/DEBIAN/preinst Executable file
View file

@ -0,0 +1,26 @@
#!/bin/bash
# Installs node if it isn't already installed
#
# Don't steamroll over a previously installed node version
# TODO provide a local version of node?
VER="0.10.4"
ARCH="x86"
if [ `arch | grep 64` ]
then
ARCH="x64"
fi
# TODO test version
if [ ! -f /usr/local/bin/node ]
then
pushd /tmp
wget -c "http://nodejs.org/dist/v${VER}/node-v${VER}-linux-${ARCH}.tar.gz"
rm -rf /tmp/node-v${VER}-linux-${ARCH}
tar xf node-v${VER}-linux-${ARCH}.tar.gz -C /tmp/
cp -a /tmp/node-v${VER}-linux-${ARCH}/* /usr/local/
fi
# Create Etherpad user
adduser --system etherpad

4
bin/bin/deb-src/DEBIAN/prerm Executable file
View file

@ -0,0 +1,4 @@
#!/bin/bash
# Stop the appserver:
service etherpad stop || true

26
bin/bin/deb-src/sysroot/etc/init/etherpad.conf Normal file
View file

@ -0,0 +1,26 @@
description "etherpad"
start on started networking
stop on runlevel [!2345]
env EPHOME=/opt/etherpad
env EPLOGS=/var/log/etherpad
env EPUSER=etherpad
respawn
pre-start script
cd $EPHOME
mkdir $EPLOGS ||true
chown $EPUSER:admin $EPLOGS ||true
chmod 0755 $EPLOGS ||true
chown -R $EPUSER:admin $EPHOME/var ||true
$EPHOME/bin/installDeps.sh >> $EPLOGS/error.log || { stop; exit 1; }
end script
script
cd $EPHOME/
exec su -s /bin/sh -c 'exec "$0" "$@"' $EPUSER -- node node_modules/ep_etherpad-lite/node/server.js \
>> $EPLOGS/access.log \
2>> $EPLOGS/error.log
end script

115
bin/bin/deb-src/sysroot/usr/share/doc/etherpad/README.md Normal file
View file

@ -0,0 +1,115 @@
# A really-real time collaborative word processor for the web
![alt text](http://i.imgur.com/zYrGkg3.gif "Etherpad in action on PrimaryPad")
# About
Etherpad is a really-real time collaborative editor maintained by the Etherpad Community.
Etherpad is written in Javascript(99.9%) on both the server and client so it's easy for developers to maintain and add new features. Because of this Etherpad has tons of customizations that you can leverage.
Etherpad is designed to be easily embeddable and provides a [HTTP API](https://github.com/ether/etherpad-lite/wiki/HTTP-API)
that allows your web application to manage pads, users and groups. It is recommended to use the [available client implementations](https://github.com/ether/etherpad-lite/wiki/HTTP-API-client-libraries) in order to interact with this API.
There is also a [jQuery plugin](https://github.com/ether/etherpad-lite-jquery-plugin) that helps you to embed Pads into your website.
There's also a full-featured plugin framework, allowing you to easily add your own features. By default your Etherpad is rather sparce and because Etherpad takes a lot of it's inspiration from Wordpress plugins are really easy to install and update. Once you have Etherpad installed you should visit the plugin page and take control.
Finally, Etherpad comes with translations into most languages! Users are automatically delivered the correct language for their local settings.
**Visit [beta.etherpad.org](http://beta.etherpad.org) to test it live.**
Also, check out the **[FAQ](https://github.com/ether/etherpad-lite/wiki/FAQ)**, really!
# Installation
Etherpad works with node v0.8 and v0.10, only. (We don't support v0.6)
## Windows
### Prebuilt windows package
This package works out of the box on any windows machine, but it's not very useful for developing purposes...
1. [Download the latest windows package](http://etherpad.org/#download)
2. Extract the folder
Now, run `start.bat` and open <http://localhost:9001> in your browser. You like it? [Next steps](#next-steps).
### Fancy install
You'll need [node.js](http://nodejs.org) and (optionally, though recommended) git.
1. Grab the source, either
- download <https://github.com/ether/etherpad-lite/zipball/master>
- or `git clone https://github.com/ether/etherpad-lite.git` (for this you need git, obviously)
2. start `bin\installOnWindows.bat`
Now, run `start.bat` and open <http://localhost:9001> in your browser.
Update to the latest version with `git pull origin`, then run `bin\installOnWindows.bat`, again.
[Next steps](#next-steps).
## GNU/Linux and other UNIX-like systems
You'll need gzip, git, curl, libssl develop libraries, python and gcc.
*For Debian/Ubuntu*: `apt-get install gzip git-core curl python libssl-dev pkg-config build-essential`
*For Fedora/CentOS*: `yum install gzip git-core curl python openssl-devel && yum groupinstall "Development Tools"`
*For FreeBSD*: `portinstall node, npm, git (optional)`
Additionally, you'll need [node.js](http://nodejs.org) installed, Ideally the latest stable version, be careful of installing nodejs from apt.
**As any user (we recommend creating a separate user called etherpad):**
1. Move to a folder where you want to install Etherpad. Clone the git repository `git clone git://github.com/ether/etherpad-lite.git`
2. Change into the new directory containing the cloned source code `cd etherpad-lite`
Now, run `bin/run.sh` and open <http://127.0.0.1:9001> in your browser.
Update to the latest version with `git pull origin`. The next start with bin/run.sh will update the dependencies.
You like it? [Next steps](#next-steps).
# Next Steps
## Tweak the settings
You can initially modify the settings in `settings.json`. (If you need to handle multiple settings files, you can pass the path to a settings file to `bin/run.sh` using the `-s|--settings` option. This allows you to run multiple Etherpad instances from the same installation.) Once you have access to your /admin section settings can be modified through the web browser.
You should use a dedicated database such as "mysql", if you are planning on using etherpad-in a production environment, since the "dirtyDB" database driver is only for testing and/or development purposes.
## Helpful resources
The [wiki](https://github.com/ether/etherpad-lite/wiki) is your one-stop resource for Tutorials and How-to's, really check it out! Also, feel free to improve these wiki pages.
Documentation can be found in `docs/`.
# Development
## Things you should know
Read this [git guide](http://learn.github.com/p/index.html) and watch this [video on getting started with Etherpad Development](http://youtu.be/67-Q26YH97E).
If you're new to node.js, start with Ryan Dahl's [Introduction to Node.js](http://youtu.be/jo_B4LTHi3I).
You can debug Etherpad using `bin/debugRun.sh`.
If you want to find out how Etherpad's `Easysync` works (the library that makes it really realtime), start with this [PDF](https://github.com/ether/etherpad-lite/raw/master/doc/easysync/easysync-full-description.pdf) (complex, but worth reading).
## Getting started
You know all this and just want to know how you can help?
Look at the [TODO list](https://github.com/ether/etherpad-lite/wiki/TODO) and our [Issue tracker](https://github.com/ether/etherpad-lite/issues). (Please consider using [jshint](http://www.jshint.com/about/), if you plan to contribute code.)
Also, and most importantly, read our [**Developer Guidelines**](https://github.com/ether/etherpad-lite/blob/master/CONTRIBUTING.md), really!
# Get in touch
Join the [mailinglist](http://groups.google.com/group/etherpad-lite-dev) and make some noise on our busy freenode irc channel [#etherpad-lite-dev](http://webchat.freenode.net?channels=#etherpad-lite-dev)!
# Modules created for this project
* [ueberDB](https://github.com/Pita/ueberDB) "transforms every database into a object key value store" - manages all database access
* [channels](https://github.com/Pita/channels) "Event channels in node.js" - ensures that ueberDB operations are atomic and in series for each key
* [async-stacktrace](https://github.com/Pita/async-stacktrace) "Improves node.js stacktraces and makes it easier to handle errors"
# Donate!
* [Flattr] (http://flattr.com/thing/71378/Etherpad-Foundation)
* Paypal - Press the donate button on [etherpad.org](http://etherpad.org)
* [Bitcoin] (https://coinbase.com/checkouts/1e572bf8a82e4663499f7f1f66c2d15a)
# License
[Apache License v2](http://www.apache.org/licenses/LICENSE-2.0.html)

28
bin/bin/debugRun.sh Executable file
View file

@ -0,0 +1,28 @@
#!/bin/sh
#Move to the folder where ep-lite is installed
cd `dirname $0`
#Was this script started in the bin folder? if yes move out
if [ -d "../bin" ]; then
cd "../"
fi
#prepare the enviroment
bin/installDeps.sh || exit 1
hash node-inspector > /dev/null 2>&1 || {
echo "You need to install node-inspector to run the tests!" >&2
echo "You can install it with npm" >&2
echo "Run: npm install -g node-inspector" >&2
exit 1
}
node-inspector &
echo "If you are new to node-inspector, take a look at this video: http://youtu.be/AOnK3NVnxL8"
node --debug node_modules/ep_etherpad-lite/node/server.js $*
#kill node-inspector before ending
kill $!

63
bin/bin/deletePad.js Normal file
View file

@ -0,0 +1,63 @@
/*
A tool for deleting pads from the CLI, because sometimes a brick is required to fix a window.
*/
if(process.argv.length != 3)
{
console.error("Use: node deletePad.js $PADID");
process.exit(1);
}
//get the padID
var padId = process.argv[2];
var db, padManager, pad, settings;
var neededDBValues = ["pad:"+padId];
var npm = require("../src/node_modules/npm");
var async = require("../src/node_modules/async");
async.series([
// load npm
function(callback) {
npm.load({}, function(er) {
if(er)
{
console.error("Could not load NPM: " + er)
process.exit(1);
}
else
{
callback();
}
})
},
// load modules
function(callback) {
settings = require('../src/node/utils/Settings');
db = require('../src/node/db/DB');
callback();
},
// intallize the database
function (callback)
{
db.init(callback);
},
// delete the pad and it's links
function (callback)
{
padManager = require('../src/node/db/PadManager');
padManager.removePad(padId, function(err){
callback(err);
});
callback();
}
], function (err)
{
if(err) throw err;
else
{
console.log("Finished deleting padId: "+padId);
process.exit();
}
});

109
bin/bin/extractPadData.js Normal file
View file

@ -0,0 +1,109 @@
/*
This is a debug tool. It helps to extract all datas of a pad and move it from an productive enviroment and to a develop enviroment to reproduce bugs there. It outputs a dirtydb file
*/
if(process.argv.length != 3)
{
console.error("Use: node extractPadData.js $PADID");
process.exit(1);
}
//get the padID
var padId = process.argv[2];
var db, dirty, padManager, pad, settings;
var neededDBValues = ["pad:"+padId];
var npm = require("../node_modules/ep_etherpad-lite/node_modules/npm");
var async = require("../node_modules/ep_etherpad-lite/node_modules/async");
async.series([
// load npm
function(callback) {
npm.load({}, function(er) {
if(er)
{
console.error("Could not load NPM: " + er)
process.exit(1);
}
else
{
callback();
}
})
},
// load modules
function(callback) {
settings = require('../node_modules/ep_etherpad-lite/node/utils/Settings');
db = require('../node_modules/ep_etherpad-lite/node/db/DB');
dirty = require("../node_modules/ep_etherpad-lite/node_modules/ueberDB/node_modules/dirty")(padId + ".db");
callback();
},
//intallize the database
function (callback)
{
db.init(callback);
},
//get the pad
function (callback)
{
padManager = require('../node_modules/ep_etherpad-lite/node/db/PadManager');
padManager.getPad(padId, function(err, _pad)
{
pad = _pad;
callback(err);
});
},
function (callback)
{
//add all authors
var authors = pad.getAllAuthors();
for(var i=0;i<authors.length;i++)
{
neededDBValues.push("globalAuthor:" + authors[i]);
}
//add all revisions
var revHead = pad.head;
for(var i=0;i<=revHead;i++)
{
neededDBValues.push("pad:"+padId+":revs:" + i);
}
//get all chat values
var chatHead = pad.chatHead;
for(var i=0;i<=chatHead;i++)
{
neededDBValues.push("pad:"+padId+":chat:" + i);
}
//get and set all values
async.forEach(neededDBValues, function(dbkey, callback)
{
db.db.db.wrappedDB.get(dbkey, function(err, dbvalue)
{
if(err) { callback(err); return}
if(dbvalue && typeof dbvalue != 'object'){
dbvalue=JSON.parse(dbvalue); // if its not json then parse it as json
}
dirty.set(dbkey, dbvalue, callback);
});
}, callback);
}
], function (err)
{
if(err) throw err;
else
{
console.log("finished");
process.exit();
}
});
//get the pad object
//get all revisions of this pad
//get all authors related to this pad
//get the readonly link releated to this pad
//get the chat entrys releated to this pad

110
bin/bin/importSqlFile.js Normal file
View file

@ -0,0 +1,110 @@
var startTime = new Date().getTime();
require("ep_etherpad-lite/node_modules/npm").load({}, function(er,npm) {
var fs = require("fs");
var ueberDB = require("ep_etherpad-lite/node_modules/ueberDB");
var settings = require("ep_etherpad-lite/node/utils/Settings");
var log4js = require('ep_etherpad-lite/node_modules/log4js');
var dbWrapperSettings = {
cache: 0,
writeInterval: 100,
json: false // data is already json encoded
};
var db = new ueberDB.database(settings.dbType, settings.dbSettings, dbWrapperSettings, log4js.getLogger("ueberDB"));
var sqlFile = process.argv[2];
//stop if the settings file is not set
if(!sqlFile)
{
console.error("Use: node importSqlFile.js $SQLFILE");
process.exit(1);
}
log("initializing db");
db.init(function(err)
{
//there was an error while initializing the database, output it and stop
if(err)
{
console.error("ERROR: Problem while initalizing the database");
console.error(err.stack ? err.stack : err);
process.exit(1);
}
else
{
log("done");
log("open output file...");
var lines = fs.readFileSync(sqlFile, 'utf8').split("\n");
var count = lines.length;
var keyNo = 0;
process.stdout.write("Start importing " + count + " keys...\n");
lines.forEach(function(l) {
if (l.substr(0, 27) == "REPLACE INTO store VALUES (") {
var pos = l.indexOf("', '");
var key = l.substr(28, pos - 28);
var value = l.substr(pos + 3);
value = value.substr(0, value.length - 2);
console.log("key: " + key + " val: " + value);
console.log("unval: " + unescape(value));
db.set(key, unescape(value), null);
keyNo++;
if (keyNo % 1000 == 0) {
process.stdout.write(" " + keyNo + "/" + count + "\n");
}
}
});
process.stdout.write("\n");
process.stdout.write("done. waiting for db to finish transaction. depended on dbms this may take some time...\n");
db.doShutdown(function() {
log("finished, imported " + keyNo + " keys.");
process.exit(0);
});
}
});
});
function log(str)
{
console.log((new Date().getTime() - startTime)/1000 + "\t" + str);
}
unescape = function(val) {
// value is a string
if (val.substr(0, 1) == "'") {
val = val.substr(0, val.length - 1).substr(1);
return val.replace(/\\[0nrbtZ\\'"]/g, function(s) {
switch(s) {
case "\\0": return "\0";
case "\\n": return "\n";
case "\\r": return "\r";
case "\\b": return "\b";
case "\\t": return "\t";
case "\\Z": return "\x1a";
default: return s.substr(1);
}
});
}
// value is a boolean or NULL
if (val == 'NULL') {
return null;
}
if (val == 'true') {
return true;
}
if (val == 'false') {
return false;
}
// value is a number
return val;
};

114
bin/bin/installDeps.sh Executable file
View file

@ -0,0 +1,114 @@
#!/bin/sh
#Move to the folder where ep-lite is installed
cd `dirname $0`
#Was this script started in the bin folder? if yes move out
if [ -d "../bin" ]; then
cd "../"
fi
#Is gnu-grep (ggrep) installed on SunOS (Solaris)
if [ $(uname) = "SunOS" ]; then
hash ggrep > /dev/null 2>&1 || {
echo "Please install ggrep (pkg install gnu-grep)" >&2
exit 1
}
fi
#Is curl installed?
hash curl > /dev/null 2>&1 || {
echo "Please install curl" >&2
exit 1
}
#Is node installed?
hash node > /dev/null 2>&1 || {
echo "Please install node.js ( http://nodejs.org )" >&2
exit 1
}
#Is npm installed?
hash npm > /dev/null 2>&1 || {
echo "Please install npm ( http://npmjs.org )" >&2
exit 1
}
#check npm version
NPM_VERSION=$(npm --version)
if [ ! $(echo $NPM_VERSION | cut -d "." -f 1) = "1" ]; then
echo "You're running a wrong version of npm, you're using $NPM_VERSION, we need 1.x" >&2
exit 1
fi
#check node version
NODE_VERSION=$(node --version)
NODE_V_MINOR=$(echo $NODE_VERSION | cut -d "." -f 1-2)
if [ ! $NODE_V_MINOR = "v0.8" ] && [ ! $NODE_V_MINOR = "v0.10" ] && [ ! $NODE_V_MINOR = "v0.11" ]; then
echo "You're running a wrong version of node, you're using $NODE_VERSION, we need v0.8.x, v0.10.x or v0.11.x" >&2
exit 1
fi
#Get the name of the settings file
settings="settings.json"
a='';
for arg in $*; do
if [ "$a" = "--settings" ] || [ "$a" = "-s" ]; then settings=$arg; fi
a=$arg
done
#Does a $settings exist? if no copy the template
if [ ! -f $settings ]; then
echo "Copy the settings template to $settings..."
cp settings.json.template $settings || exit 1
fi
echo "Ensure that all dependencies are up to date... If this is the first time you have run Etherpad please be patient."
(
mkdir -p node_modules
cd node_modules
[ -e ep_etherpad-lite ] || ln -s ../src ep_etherpad-lite
cd ep_etherpad-lite
npm install --loglevel warn
) || {
rm -rf node_modules
exit 1
}
echo "Ensure jQuery is downloaded and up to date..."
DOWNLOAD_JQUERY="true"
NEEDED_VERSION="1.9.1"
if [ -f "src/static/js/jquery.js" ]; then
if [ $(uname) = "SunOS" ]; then
VERSION=$(cat src/static/js/jquery.js | head -n 3 | ggrep -o "v[0-9]\.[0-9]\(\.[0-9]\)\?");
else
VERSION=$(cat src/static/js/jquery.js | head -n 3 | grep -o "v[0-9]\.[0-9]\(\.[0-9]\)\?");
fi
if [ ${VERSION#v} = $NEEDED_VERSION ]; then
DOWNLOAD_JQUERY="false"
fi
fi
if [ $DOWNLOAD_JQUERY = "true" ]; then
curl -lo src/static/js/jquery.js http://code.jquery.com/jquery-$NEEDED_VERSION.js || exit 1
fi
#Remove all minified data to force node creating it new
echo "Clear minfified cache..."
rm -f var/minified*
echo "ensure custom css/js files are created..."
for f in "index" "pad" "timeslider"
do
if [ ! -f "src/static/custom/$f.js" ]; then
cp "src/static/custom/js.template" "src/static/custom/$f.js" || exit 1
fi
if [ ! -f "src/static/custom/$f.css" ]; then
cp "src/static/custom/css.template" "src/static/custom/$f.css" || exit 1
fi
done
exit 0

39
bin/bin/installOnWindows.bat Normal file
View file

@ -0,0 +1,39 @@
@echo off
:: change directory to etherpad-lite root
cd /D "%~dp0\.."
:: Is node installed?
cmd /C node -e "" || ( echo "Please install node.js ( http://nodejs.org )" && exit /B 1 )
echo _
echo Checking node version...
set check_version="if(['8','10'].indexOf(process.version.split('.')[1].toString()) === -1) { console.log('You are running a wrong version of Node. Etherpad requires v0.8.x or v0.10.x'); process.exit(1) }"
cmd /C node -e %check_version% || exit /B 1
echo _
echo Ensure that all dependencies are up to date... If this is the first time you have run Etherpad please be patient.
cmd /C npm install src/ --loglevel warn || exit /B 1
echo _
echo Copying custom templates...
set custom_dir=node_modules\ep_etherpad-lite\static\custom
FOR %%f IN (index pad timeslider) DO (
if NOT EXIST "%custom_dir%\%%f.js" copy "%custom_dir%\js.template" "%custom_dir%\%%f.js"
if NOT EXIST "%custom_dir%\%%f.css" copy "%custom_dir%\css.template" "%custom_dir%\%%f.css"
)
echo _
echo Clearing cache...
del /S var\minified*
echo _
echo Setting up settings.json...
IF NOT EXIST settings.json (
echo Can't find settings.json.
echo Copying settings.json.template...
cmd /C copy settings.json.template settings.json || exit /B 1
)
echo _
echo Installed Etherpad! To run Etherpad type start.bat

9
bin/bin/jshint.sh Executable file
View file

@ -0,0 +1,9 @@
#!/bin/sh
if [ -d "../bin" ]; then
cd "../"
fi
JSHINT=./node_modules/jshint/bin/hint
$JSHINT ./node/

79
bin/bin/loadTesting/README Normal file
View file

@ -0,0 +1,79 @@
This is the new load testing file: https://bitbucket.org/rbraakman/etherpad-stresstest
BELOW is the original load testing file.
This load tester is extremely useful for testing how many dormant clients can connect to etherpad.
TODO:
Emulate characters being typed into a pad
HOW TO USE (from @mjd75) proper formatting at: https://github.com/ether/etherpad-lite/issues/360
Server 1:
Installed Node.js (etc), EtherPad and MySQL
Server 2:
Installed Xvfb and PhantomJS
I installed Xvfb following (roughly) this guide: http://blog.martin-lyness.com/archives/installing-xvfb-on-ubuntu-9-10-karmic-koala
#sudo apt-get install xvfb
#sudo apt-get install xfonts-100dpi xfonts-75dpi xfonts-scalable xfonts-cyrillic
Launched two instances of Xvfb directly from the terminal:
#Xvfb :0 -ac
#Xvfb :1 -ac
I installed PhantomJS following this guide: http://code.google.com/p/phantomjs/wiki/Installation
#sudo add-apt-repository ppa:jerome-etienne/neoip
#sudo apt-get update
#sudo apt-get install phantomjs
I created a small JavaScript file for PhatomJS to use to control the browser instances:
### BEGIN JAVASCRIPT ###
var page = new WebPage(),
t, address;
if (phantom.args.length === 0) {
console.log('Usage: loader.js <some URL>');
phantom.exit();
} else {
t = Date.now();
address = phantom.args[0];
var page = new WebPage();
page.onResourceRequested = function (request) {
console.log('Request ' + JSON.stringify(request, undefined, 4));
};
page.onResourceReceived = function (response) {
console.log('Receive ' + JSON.stringify(response, undefined, 4));
};
page.open(address);
}
### END JAVASCRIPT ###
And finally a launcher script that uses screen to run 400 instances of PhantomJS with the above script:
### BEGIN SHELL SCRIPT ###
#!/bin/bash
# connect 200 instances to display :0
for i in {1..200}
do
DISPLAY=:0 screen -d -m phantomjs loader.js http://ec2-50-17-168-xx.compute-1.amazonaws.com:9001/p/pad2 && sleep 2
done
# connect 200 instances to display :1
for i in {1..200}
do
DISPLAY=:1 screen -d -m phantomjs loader.js http://ec2-50-17-168-xx.compute-1.amazonaws.com:9001/p/pad2 && sleep 2
done
### END SHELL SCRIPT ###

16
bin/bin/loadTesting/launcher.sh Executable file
View file

@ -0,0 +1,16 @@
#!/bin/bash
# connect 500 instances to display :0
for i in {1..500}
do
echo $i
echo "Displaying Some shit"
DISPLAY=:0 screen -d -m /home/phantomjs/bin/phantomjs loader.js http://10.0.0.55:9001/p/pad2 && sleep 2
done
# connect 500 instances to display :1
for i in {1..500}
do
echo $i
DISPLAY=:1 screen -d -m /home/phantomjs/bin/phantomjs loader.js http://10.0.0.55:9001/p/pad2 && sleep 2
done

20
bin/bin/loadTesting/loader.js Normal file
View file

@ -0,0 +1,20 @@
var page = new WebPage(),
t, address;
if (phantom.args.length === 0) {
console.log('Usage: loader.js <some URL>');
phantom.exit();
} else {
t = Date.now();
address = phantom.args[0];
var page = new WebPage();
page.onResourceRequested = function (request) {
console.log('Request ' + JSON.stringify(request, undefined, 4));
};
page.onResourceReceived = function (response) {
console.log('Receive ' + JSON.stringify(response, undefined, 4));
};
page.open(address);
}

18
bin/bin/migrateDirtyDBtoMySQL.js Normal file
View file

@ -0,0 +1,18 @@
require("ep_etherpad-lite/node_modules/npm").load({}, function(er,npm) {
process.chdir(npm.root+'/..')
var settings = require("ep_etherpad-lite/node/utils/Settings");
var dirty = require("ep_etherpad-lite/node_modules/ueberDB/node_modules/dirty")('var/dirty.db');
var db = require("ep_etherpad-lite/node/db/DB");
db.init(function() {
db = db.db;
dirty.on("load", function() {
dirty.forEach(function(key, value) {
db.set(key, value);
});
});
});
});

39
bin/bin/run.sh Executable file
View file

@ -0,0 +1,39 @@
#!/bin/sh
#Move to the folder where ep-lite is installed
cd `dirname $0`
#Was this script started in the bin folder? if yes move out
if [ -d "../bin" ]; then
cd "../"
fi
ignoreRoot=0
for ARG in $*
do
if [ "$ARG" = "--root" ]; then
ignoreRoot=1
fi
done
#Stop the script if its started as root
if [ "$(id -u)" -eq 0 ] && [ $ignoreRoot -eq 0 ]; then
echo "You shouldn't start Etherpad as root!"
echo "Please type 'Etherpad rocks my socks' or supply the '--root' argument if you still want to start it as root"
read rocks
if [ ! $rocks = "Etherpad rocks my socks" ]
then
echo "Your input was incorrect"
exit 1
fi
fi
#prepare the enviroment
bin/installDeps.sh $* || exit 1
#Move to the node folder and start
echo "start..."
SCRIPTPATH=`pwd -P`
node $SCRIPTPATH/node_modules/ep_etherpad-lite/node/server.js $*

68
bin/bin/safeRun.sh Executable file
View file

@ -0,0 +1,68 @@
#!/bin/sh
#This script ensures that ep-lite is automatically restarting after an error happens
#Handling Errors
# 0 silent
# 1 email
ERROR_HANDLING=0
# Your email address which should recieve the error messages
EMAIL_ADDRESS="no-reply@example.com"
# Sets the minimun amount of time betweens the sending of error emails.
# This ensures you not get spamed while a endless reboot loop
# It's the time in seconds
TIME_BETWEEN_EMAILS=600 # 10 minutes
# DON'T EDIT AFTER THIS LINE
LAST_EMAIL_SEND=0
LOG="$1"
#Move to the folder where ep-lite is installed
cd `dirname $0`
#Was this script started in the bin folder? if yes move out
if [ -d "../bin" ]; then
cd "../"
fi
#check if a logfile parameter is set
if [ -z "${LOG}" ]; then
echo "Set a logfile as the first parameter"
exit 1
fi
shift
while [ 1 ]
do
#try to touch the file if it doesn't exist
if [ ! -f ${LOG} ]; then
touch ${LOG} || ( echo "Logfile '${LOG}' is not writeable" && exit 1 )
fi
#check if the file is writeable
if [ ! -w ${LOG} ]; then
echo "Logfile '${LOG}' is not writeable"
exit 1
fi
#start the application
bin/run.sh $@ >>${LOG} 2>>${LOG}
#Send email
if [ $ERROR_HANDLING = 1 ]; then
TIME_NOW=$(date +%s)
TIME_SINCE_LAST_SEND=$(($TIME_NOW - $LAST_EMAIL_SEND))
if [ $TIME_SINCE_LAST_SEND -gt $TIME_BETWEEN_EMAILS ]; then
printf "Server was restared at: $(date)\nThe last 50 lines of the log before the error happens:\n $(tail -n 50 ${LOG})" | mail -s "Pad Server was restarted" $EMAIL_ADDRESS
LAST_EMAIL_SEND=$TIME_NOW
fi
fi
echo "RESTART!" >>${LOG}
#Sleep 10 seconds before restart
sleep 10
done

4
bin/buildDebian.sh
View file

@ -14,10 +14,10 @@ rm -rf ${DIST}
mkdir -p ${DIST}/ mkdir -p ${DIST}/
rm -rf ${SRC} rm -rf ${SRC}
rsync -a deb-src/ ${SRC}/ rsync -a bin/deb-src/ ${SRC}/
mkdir -p ${SYSROOT}/opt/ mkdir -p ${SYSROOT}/opt/
rsync --exclude '.git' -a ../ ${SYSROOT}/opt/etherpad/ --delete rsync --exclude '.git' -a . ${SYSROOT}/opt/etherpad/ --delete
find ${SRC}/ -type d -exec chmod 0755 {} \; find ${SRC}/ -type d -exec chmod 0755 {} \;
find ${SRC}/ -type f -exec chmod go-w {} \; find ${SRC}/ -type f -exec chmod go-w {} \;

3
bin/deb-src/DEBIAN/postinst
View file

@ -2,5 +2,6 @@
# Start the services! # Start the services!
service etherpad start service etherpad start
echo "Give Etherpad about 3 minutes to install dependencies and start" echo "Give Etherpad about 3 minutes to install dependencies then visit http://localhost:9001 in your web browser"
echo "To stop etherpad type 'service etherpad stop', To restart type 'service etherpad restart'".
rm -f /tmp/etherpad.log /tmp/etherpad.err rm -f /tmp/etherpad.log /tmp/etherpad.err

4
bin/deb-src/sysroot/etc/init/etherpad.conf
View file

@ -20,7 +20,9 @@ end script
script script
cd $EPHOME/ cd $EPHOME/
exec su -s /bin/sh -c 'exec "$0" "$@"' $EPUSER -- node node_modules/ep_etherpad/node/server.js \ exec su -s /bin/sh -c 'exec "$0" "$@"' $EPUSER -- node node_modules/ep_etherpad-lite/node/server.js \
>> $EPLOGS/access.log \ >> $EPLOGS/access.log \
2>> $EPLOGS/error.log 2>> $EPLOGS/error.log
echo "Etherpad is running on http://localhost:9001 - To change settings edit /opt/etherpad/settings.json"
end script end script