/**
* Copyright 2009 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS-IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import("fastJSON");
import("sqlbase.sqlbase");
import("sqlbase.sqlcommon");
import("sqlbase.sqlobj");
import("timer");
import("sync");
import("etherpad.collab.ace.easysync2.{Changeset,AttribPool}");
import("etherpad.log");
import("etherpad.pad.padevents");
import("etherpad.pad.padutils");
import("etherpad.pad.dbwriter");
import("etherpad.pad.pad_migrations");
import("etherpad.pad.pad_security");
import("etherpad.collab.collab_server");
import("cache_utils.syncedWithCache");
jimport("net.appjet.common.util.LimitedSizeMapping");
jimport("java.lang.System.out.println");
jimport("java.util.concurrent.ConcurrentHashMap");
jimport("net.appjet.oui.GlobalSynchronizer");
jimport("net.appjet.oui.exceptionlog");
function onStartup() {
appjet.cache.pads = {};
appjet.cache.pads.meta = new ConcurrentHashMap();
appjet.cache.pads.temp = new ConcurrentHashMap();
appjet.cache.pads.revs = new ConcurrentHashMap();
appjet.cache.pads.revs10 = new ConcurrentHashMap();
appjet.cache.pads.revs100 = new ConcurrentHashMap();
appjet.cache.pads.revs1000 = new ConcurrentHashMap();
appjet.cache.pads.chat = new ConcurrentHashMap();
appjet.cache.pads.revmeta = new ConcurrentHashMap();
appjet.cache.pads.authors = new ConcurrentHashMap();
appjet.cache.pads.apool = new ConcurrentHashMap();
}
var _JSON_CACHE_SIZE = 10000;
// to clear: appjet.cache.padmodel.modelcache.map.clear()
function _getModelCache() {
return syncedWithCache('padmodel.modelcache', function(cache) {
if (! cache.map) {
cache.map = new LimitedSizeMapping(_JSON_CACHE_SIZE);
}
return cache.map;
});
}
function cleanText(txt) {
return txt.replace(/\r\n/g,'\n').replace(/\r/g,'\n').replace(/\t/g, ' ').replace(/\xa0/g, ' ');
}
/**
* Access a pad object, which is passed as an argument to
* the given padFunc, which is executed inside an exclusive lock,
* and return the result. If the pad doesn't exist, a wrapper
* object is still created and passed to padFunc, and it can
* be used to check whether the pad exists and create it.
*
* Note: padId is a GLOBAL id.
*/
function accessPadGlobal(padId, padFunc, rwMode) {
// this may make a nested call to accessPadGlobal, so do it first
pad_security.checkAccessControl(padId, rwMode);
// pad is never loaded into memory (made "active") unless it has been migrated.
// Migrations do not use accessPad, but instead access the database directly.
pad_migrations.ensureMigrated(padId);
var mode = (rwMode || "rw").toLowerCase();
if (! appjet.requestCache.padsAccessing) {
appjet.requestCache.padsAccessing = {};
}
if (appjet.requestCache.padsAccessing[padId]) {
// nested access to same pad
var p = appjet.requestCache.padsAccessing[padId];
var m = p._meta;
if (m && mode != "r") {
m.status.lastAccess = +new Date();
m.status.dirty = true;
}
return padFunc(p);
}
return doWithPadLock(padId, function() {
return sqlcommon.inTransaction(function() {
var meta = _getPadMetaData(padId); // null if pad doesn't exist yet
if (meta && ! meta.status) {
meta.status = { validated: false };
}
if (meta && mode != "r") {
meta.status.lastAccess = +new Date();
}
function getCurrentAText() {
var tempObj = pad.tempObj();
if (! tempObj.atext) {
tempObj.atext = pad.getInternalRevisionAText(meta.head);
}
return tempObj.atext;
}
function addRevision(theChangeset, author, optDatestamp) {
var atext = getCurrentAText();
var newAText = Changeset.applyToAText(theChangeset, atext, pad.pool());
Changeset.copyAText(newAText, atext); // updates pad.tempObj().atext!
var newRev = ++meta.head;
var revs = _getPadStringArray(padId, "revs");
revs.setEntry(newRev, theChangeset);
var revmeta = _getPadStringArray(padId, "revmeta");
var thisRevMeta = {t: (optDatestamp || (+new Date())),
a: getNumForAuthor(author)};
if ((newRev % meta.keyRevInterval) == 0) {
thisRevMeta.atext = atext;
}
revmeta.setJSONEntry(newRev, thisRevMeta);
updateCoarseChangesets(true);
}
function getNumForAuthor(author, dontAddIfAbsent) {
return pad.pool().putAttrib(['author',author||''], dontAddIfAbsent);
}
function getAuthorForNum(n) {
// must return null if n is an attrib number that isn't an author
var pair = pad.pool().getAttrib(n);
if (pair && pair[0] == 'author') {
return pair[1];
}
return null;
}
function updateCoarseChangesets(onlyIfPresent) {
// this is fast to run if the coarse changesets
// are up-to-date or almost up-to-date;
// if there's no coarse changeset data,
// it may take a while.
if (! meta.coarseHeads) {
if (onlyIfPresent) {
return;
}
else {
meta.coarseHeads = {10:-1, 100:-1, 1000:-1};
}
}
var head = meta.head;
// once we reach head==9, coarseHeads[10] moves
// from -1 up to 0; at head==19 it moves up to 1
var desiredCoarseHeads = {
10: Math.floor((head-9)/10),
100: Math.floor((head-99)/100),
1000: Math.floor((head-999)/1000)
};
var revs = _getPadStringArray(padId, "revs");
var revs10 = _getPadStringArray(padId, "revs10");
var revs100 = _getPadStringArray(padId, "revs100");
var revs1000 = _getPadStringArray(padId, "revs1000");
var fineArrays = [revs, revs10, revs100];
var coarseArrays = [revs10, revs100, revs1000];
var levels = [10, 100, 1000];
var dirty = false;
for(var z=0;z<3;z++) {
var level = levels[z];
var coarseArray = coarseArrays[z];
var fineArray = fineArrays[z];
while (meta.coarseHeads[level] < desiredCoarseHeads[level]) {
dirty = true;
// for example, if the current coarse head is -1,
// compose 0-9 inclusive of the finer level and call it 0
var x = meta.coarseHeads[level] + 1;
var cs = fineArray.getEntry(10 * x);
for(var i=1;i<=9;i++) {
cs = Changeset.compose(cs, fineArray.getEntry(10*x + i),
pad.pool());
}
coarseArray.setEntry(x, cs);
meta.coarseHeads[level] = x;
}
}
if (dirty) {
meta.status.dirty = true;
}
}
/////////////////// "Public" API starts here (functions used by collab_server or other modules)
var pad = {
// Operations that write to the data structure should
// set meta.dirty = true. Any pad access that isn't
// done in "read" mode also sets dirty = true.
getId: function() { return padId; },
exists: function() { return !!meta; },
create: function(optText) {
meta = {};
meta.head = -1; // incremented below by addRevision
pad.tempObj().atext = Changeset.makeAText("\n");
meta.padId = padId,
meta.keyRevInterval = 100;
meta.numChatMessages = 0;
var t = +new Date();
meta.status = { validated: true };
meta.status.lastAccess = t;
meta.status.dirty = true;
meta.supportsTimeSlider = true;
var firstChangeset = Changeset.makeSplice("\n", 0, 0,
cleanText(optText || ''));
addRevision(firstChangeset, '');
_insertPadMetaData(padId, meta);
sqlobj.insert("PAD_SQLMETA", {
id: padId, version: 2, creationTime: new Date(t), lastWriteTime: new Date(),
headRev: meta.head }); // headRev is not authoritative, just for info
padevents.onNewPad(pad);
},
destroy: function() { // you may want to collab_server.bootAllUsers first
padevents.onDestroyPad(pad);
_destroyPadStringArray(padId, "revs");
_destroyPadStringArray(padId, "revs10");
_destroyPadStringArray(padId, "revs100");
_destroyPadStringArray(padId, "revs1000");
_destroyPadStringArray(padId, "revmeta");
_destroyPadStringArray(padId, "chat");
_destroyPadStringArray(padId, "authors");
_removePadMetaData(padId);
_removePadAPool(padId);
sqlobj.deleteRows("PAD_SQLMETA", { id: padId });
meta = null;
},
writeToDB: function() {
var meta2 = {};
for(var k in meta) meta2[k] = meta[k];
delete meta2.status;
sqlbase.putJSON("PAD_META", padId, meta2);
_getPadStringArray(padId, "revs").writeToDB();
_getPadStringArray(padId, "revs10").writeToDB();
_getPadStringArray(padId, "revs100").writeToDB();
_getPadStringArray(padId, "revs1000").writeToDB();
_getPadStringArray(padId, "revmeta").writeToDB();
_getPadStringArray(padId, "chat").writeToDB();
_getPadStringArray(padId, "authors").writeToDB();
sqlbase.putJSON("PAD_APOOL", padId, pad.pool().toJsonable());
var props = { headRev: meta.head, lastWriteTime: new Date() };
_writePadSqlMeta(padId, props);
},
pool: function() {
return _getPadAPool(padId);
},
getHeadRevisionNumber: function() { return meta.head; },
getRevisionAuthor: function(r) {
var n = _getPadStringArray(padId, "revmeta").getJSONEntry(r).a;
return getAuthorForNum(Number(n));
},
getRevisionChangeset: function(r) {
return _getPadStringArray(padId, "revs").getEntry(r);
},
tempObj: function() { return _getPadTemp(padId); },
getKeyRevisionNumber: function(r) {
return Math.floor(r / meta.keyRevInterval) * meta.keyRevInterval;
},
getInternalRevisionAText: function(r) {
var cacheKey = "atext/C/"+r+"/"+padId;
var modelCache = _getModelCache();
var cachedValue = modelCache.get(cacheKey);
if (cachedValue) {
modelCache.touch(cacheKey);
//java.lang.System.out.println("HIT! "+cacheKey);
return Changeset.cloneAText(cachedValue);
}
//java.lang.System.out.println("MISS! "+cacheKey);
var revs = _getPadStringArray(padId, "revs");
var keyRev = pad.getKeyRevisionNumber(r);
var revmeta = _getPadStringArray(padId, "revmeta");
var atext = revmeta.getJSONEntry(keyRev).atext;
var curRev = keyRev;
var targetRev = r;
var apool = pad.pool();
while (curRev < targetRev) {
curRev++;
var cs = pad.getRevisionChangeset(curRev);
atext = Changeset.applyToAText(cs, atext, apool);
}
modelCache.put(cacheKey, Changeset.cloneAText(atext));
return atext;
},
getInternalRevisionText: function(r, optInfoObj) {
var atext = pad.getInternalRevisionAText(r);
var text = atext.text;
if (optInfoObj) {
if (text.slice(-1) != "\n") {
optInfoObj.badLastChar = text.slice(-1);
}
}
return text;
},
getRevisionText: function(r, optInfoObj) {
var internalText = pad.getInternalRevisionText(r, optInfoObj);
return internalText.slice(0, -1);
},
atext: function() { return Changeset.cloneAText(getCurrentAText()); },
text: function() { return pad.atext().text; },
getRevisionDate: function(r) {
var revmeta = _getPadStringArray(padId, "revmeta");
return new Date(revmeta.getJSONEntry(r).t);
},
// note: calls like appendRevision will NOT notify clients of the change!
// you must go through collab_server.
// Also, be sure to run cleanText() on any text to strip out carriage returns
// and other stuff.
appendRevision: function(theChangeset, author, optDatestamp) {
addRevision(theChangeset, author || '', optDatestamp);
},
appendChatMessage: function(obj) {
var index = meta.numChatMessages;
meta.numChatMessages++;
var chat = _getPadStringArray(padId, "chat");
chat.setJSONEntry(index, obj);
},
getNumChatMessages: function() {
return meta.numChatMessages;
},
getChatMessage: function(i) {
var chat = _getPadStringArray(padId, "chat");
return chat.getJSONEntry(i);
},
getPadOptionsObj: function() {
var data = pad.getDataRoot();
if (! data.padOptions) {
data.padOptions = {};
}
if ((! data.padOptions.guestPolicy) ||
(data.padOptions.guestPolicy == 'ask')) {
data.padOptions.guestPolicy = 'deny';
}
return data.padOptions;
},
getGuestPolicy: function() {
// allow/ask/deny
return pad.getPadOptionsObj().guestPolicy;
},
setGuestPolicy: function(policy) {
pad.getPadOptionsObj().guestPolicy = policy;
},
getDataRoot: function() {
var dataRoot = meta.dataRoot;
if (! dataRoot) {
dataRoot = {};
meta.dataRoot = dataRoot;
}
return dataRoot;
},
// returns an object, changes to which are not reflected
// in the DB; use setAuthorData for mutation
getAuthorData: function(author) {
var authors = _getPadStringArray(padId, "authors");
var n = getNumForAuthor(author, true);
if (n < 0) {
return null;
}
else {
return authors.getJSONEntry(n);
}
},
setAuthorData: function(author, data) {
var authors = _getPadStringArray(padId, "authors");
var n = getNumForAuthor(author);
authors.setJSONEntry(n, data);
},
adoptChangesetAttribs: function(cs, oldPool) {
return Changeset.moveOpsToNewPool(cs, oldPool, pad.pool());
},
eachATextAuthor: function(atext, func) {
var seenNums = {};
Changeset.eachAttribNumber(atext.attribs, function(n) {
if (! seenNums[n]) {
seenNums[n] = true;
var author = getAuthorForNum(n);
if (author) {
func(author, n);
}
}
});
},
getCoarseChangeset: function(start, numChangesets) {
updateCoarseChangesets();
if (!(numChangesets == 10 || numChangesets == 100 ||
numChangesets == 1000)) {
return null;
}
var level = numChangesets;
var x = Math.floor(start / level);
if (!(x >= 0 && x*level == start)) {
return null;
}
var cs = _getPadStringArray(padId, "revs"+level).getEntry(x);
if (! cs) {
return null;
}
return cs;
},
getSupportsTimeSlider: function() {
if (! ('supportsTimeSlider' in meta)) {
if (padutils.isProPadId(padId)) {
return true;
}
else {
return false;
}
}
else {
return !! meta.supportsTimeSlider;
}
},
setSupportsTimeSlider: function(v) {
meta.supportsTimeSlider = v;
},
get _meta() { return meta; }
};
try {
padutils.setCurrentPad(padId);
appjet.requestCache.padsAccessing[padId] = pad;
return padFunc(pad);
}
finally {
padutils.clearCurrentPad();
delete appjet.requestCache.padsAccessing[padId];
if (meta) {
if (mode != "r") {
meta.status.dirty = true;
}
if (meta.status.dirty) {
dbwriter.notifyPadDirty(padId);
}
}
}
});
});
}
/**
* Call an arbitrary function with no arguments inside an exclusive
* lock on a padId, and return the result.
*/
function doWithPadLock(padId, func) {
var lockName = "document/"+padId;
return sync.doWithStringLock(lockName, func);
}
function isPadLockHeld(padId) {
var lockName = "document/"+padId;
return GlobalSynchronizer.isHeld(lockName);
}
/**
* Get pad meta-data object, which is stored in SQL as JSON
* but cached in appjet.cache. Returns null if pad doesn't
* exist at all (does NOT create it). Requires pad lock.
*/
function _getPadMetaData(padId) {
var padMeta = appjet.cache.pads.meta.get(padId);
if (! padMeta) {
// not in cache
padMeta = sqlbase.getJSON("PAD_META", padId);
if (! padMeta) {
// not in SQL
padMeta = null;
}
else {
appjet.cache.pads.meta.put(padId, padMeta);
}
}
return padMeta;
}
/**
* Sets a pad's meta-data object, such as when creating
* a pad for the first time. Requires pad lock.
*/
function _insertPadMetaData(padId, obj) {
appjet.cache.pads.meta.put(padId, obj);
}
/**
* Removes a pad's meta data, writing through to the database.
* Used for the rare case of deleting a pad.
*/
function _removePadMetaData(padId) {
appjet.cache.pads.meta.remove(padId);
sqlbase.deleteJSON("PAD_META", padId);
}
function _getPadAPool(padId) {
var padAPool = appjet.cache.pads.apool.get(padId);
if (! padAPool) {
// not in cache
padAPool = new AttribPool();
padAPoolJson = sqlbase.getJSON("PAD_APOOL", padId);
if (padAPoolJson) {
// in SQL
padAPool.fromJsonable(padAPoolJson);
}
appjet.cache.pads.apool.put(padId, padAPool);
}
return padAPool;
}
/**
* Removes a pad's apool data, writing through to the database.
* Used for the rare case of deleting a pad.
*/
function _removePadAPool(padId) {
appjet.cache.pads.apool.remove(padId);
sqlbase.deleteJSON("PAD_APOOL", padId);
}
/**
* Get an object for a pad that's not persisted in storage,
* e.g. for tracking open connections. Creates object
* if necessary. Requires pad lock.
*/
function _getPadTemp(padId) {
var padTemp = appjet.cache.pads.temp.get(padId);
if (! padTemp) {
padTemp = {};
appjet.cache.pads.temp.put(padId, padTemp);
}
return padTemp;
}
/**
* Returns an object with methods for manipulating a string array, where name
* is something like "revs" or "chat". The object must be acquired and used
* all within a pad lock.
*/
function _getPadStringArray(padId, name) {
var padFoo = appjet.cache.pads[name].get(padId);
if (! padFoo) {
padFoo = {};
// writes go into writeCache, which is authoritative for reads;
// reads cause pages to be read into readCache
padFoo.readCache = {};
padFoo.writeCache = {};
appjet.cache.pads[name].put(padId, padFoo);
}
var tableName = "PAD_"+name.toUpperCase();
var self = {
getEntry: function(idx) {
var n = Number(idx);
if (padFoo.writeCache[n]) return padFoo.writeCache[n];
if (padFoo.readCache[n]) return padFoo.readCache[n];
sqlbase.getPageStringArrayElements(tableName, padId, n, padFoo.readCache);
return padFoo.readCache[n]; // null if not present in SQL
},
setEntry: function(idx, value) {
var n = Number(idx);
var v = String(value);
padFoo.writeCache[n] = v;
},
getJSONEntry: function(idx) {
var result = self.getEntry(idx);
if (! result) return result;
return fastJSON.parse(String(result));
},
setJSONEntry: function(idx, valueObj) {
self.setEntry(idx, fastJSON.stringify(valueObj));
},
writeToDB: function() {
sqlbase.putDictStringArrayElements(tableName, padId, padFoo.writeCache);
// copy key-vals of writeCache into readCache
var readCache = padFoo.readCache;
var writeCache = padFoo.writeCache;
for(var p in writeCache) {
readCache[p] = writeCache[p];
}
padFoo.writeCache = {};
}
};
return self;
}
/**
* Destroy a string array; writes through to the database. Must be
* called within a pad lock.
*/
function _destroyPadStringArray(padId, name) {
appjet.cache.pads[name].remove(padId);
var tableName = "PAD_"+name.toUpperCase();
sqlbase.clearStringArray(tableName, padId);
}
/**
* SELECT the row of PAD_SQLMETA for the given pad. Requires pad lock.
*/
function _getPadSqlMeta(padId) {
return sqlobj.selectSingle("PAD_SQLMETA", { id: padId });
}
function _writePadSqlMeta(padId, updates) {
sqlobj.update("PAD_SQLMETA", { id: padId }, updates);
}
// called from dbwriter
function removeFromMemory(pad) {
// safe to call if all data is written to SQL, otherwise will lose data;
var padId = pad.getId();
appjet.cache.pads.meta.remove(padId);
appjet.cache.pads.revs.remove(padId);
appjet.cache.pads.revs10.remove(padId);
appjet.cache.pads.revs100.remove(padId);
appjet.cache.pads.revs1000.remove(padId);
appjet.cache.pads.chat.remove(padId);
appjet.cache.pads.revmeta.remove(padId);
appjet.cache.pads.apool.remove(padId);
collab_server.removeFromMemory(pad);
}