diff --git a/core/language/en-GB/Fields.multids b/core/language/en-GB/Fields.multids
index 6b4823c02f9..d1eeabd4295 100644
--- a/core/language/en-GB/Fields.multids
+++ b/core/language/en-GB/Fields.multids
@@ -35,3 +35,4 @@ title: The unique name of a tiddler
toc-link: Suppresses the tiddler's link in a Table of Contents tree if set to: ''no''
type: The content type of a tiddler
version: Version information for a plugin
+_is_skinny: If present, indicates that the tiddler text field must be loaded from the server
diff --git a/core/language/en-GB/Misc.multids b/core/language/en-GB/Misc.multids
index ee1d0320ef5..861bb68995c 100644
--- a/core/language/en-GB/Misc.multids
+++ b/core/language/en-GB/Misc.multids
@@ -28,6 +28,7 @@ Error/Filter: Filter error
Error/FilterSyntax: Syntax error in filter expression
Error/IsFilterOperator: Filter Error: Unknown operand for the 'is' filter operator
Error/LoadingPluginLibrary: Error loading plugin library
+Error/NetworkErrorAlert: `
''Network Error''
It looks like the connection to the server has been lost. This may indicate a problem with your network connection. Please attempt to restore network connectivity before continuing.
''Any unsaved changes will be automatically synchronised when connectivity is restored''.`
Error/RecursiveTransclusion: Recursive transclusion error in transclude widget
Error/RetrievingSkinny: Error retrieving skinny tiddler list
Error/SavingToTWEdit: Error saving to TWEdit
diff --git a/core/modules/server/routes/get-file.js b/core/modules/server/routes/get-file.js
index 3429c4cd208..2a0ef647abf 100644
--- a/core/modules/server/routes/get-file.js
+++ b/core/modules/server/routes/get-file.js
@@ -19,22 +19,16 @@ exports.path = /^\/files\/(.+)$/;
exports.handler = function(request,response,state) {
var path = require("path"),
fs = require("fs"),
- util = require("util");
- var filename = path.resolve($tw.boot.wikiPath,"files",decodeURIComponent(state.params[0])),
+ util = require("util"),
+ suppliedFilename = decodeURIComponent(state.params[0]),
+ filename = path.resolve($tw.boot.wikiPath,"files",suppliedFilename),
extension = path.extname(filename);
fs.readFile(filename,function(err,content) {
var status,content,type = "text/plain";
if(err) {
- if(err.code === "ENOENT") {
- status = 404;
- content = "File '" + filename + "' not found";
- } else if(err.code === "EACCES") {
- status = 403;
- content = "You do not have permission to access the file '" + filename + "'";
- } else {
- status = 500;
- content = err.toString();
- }
+ console.log("Error accessing file " + filename + ": " + err.toString());
+ status = 404;
+ content = "File '" + suppliedFilename + "' not found";
} else {
status = 200;
content = content;
diff --git a/core/modules/server/routes/get-tiddlers-json.js b/core/modules/server/routes/get-tiddlers-json.js
index 3ece35ce13b..8e93733e48d 100644
--- a/core/modules/server/routes/get-tiddlers-json.js
+++ b/core/modules/server/routes/get-tiddlers-json.js
@@ -3,7 +3,7 @@ title: $:/core/modules/server/routes/get-tiddlers-json.js
type: application/javascript
module-type: route
-GET /recipes/default/tiddlers/tiddlers.json
+GET /recipes/default/tiddlers/tiddlers.json?filter=
\*/
(function() {
@@ -12,23 +12,34 @@ GET /recipes/default/tiddlers/tiddlers.json
/*global $tw: false */
"use strict";
+var DEFAULT_FILTER = "[all[tiddlers]!is[system]sort[title]]";
+
exports.method = "GET";
exports.path = /^\/recipes\/default\/tiddlers.json$/;
exports.handler = function(request,response,state) {
+ var filter = state.queryParameters.filter || DEFAULT_FILTER;
+ if($tw.wiki.getTiddlerText("$:/config/Server/AllowAllExternalFilters") !== "yes") {
+ if($tw.wiki.getTiddlerText("$:/config/Server/ExternalFilters/" + filter) !== "yes") {
+ console.log("Blocked attempt to GET /recipes/default/tiddlers/tiddlers.json with filter: " + filter);
+ response.writeHead(403);
+ response.end();
+ return;
+ }
+ }
+ var excludeFields = (state.queryParameters.exclude || "text").split(","),
+ titles = state.wiki.filterTiddlers(filter);
response.writeHead(200, {"Content-Type": "application/json"});
var tiddlers = [];
- state.wiki.forEachTiddler({sortField: "title"},function(title,tiddler) {
- var tiddlerFields = {};
- $tw.utils.each(tiddler.fields,function(field,name) {
- if(name !== "text") {
- tiddlerFields[name] = tiddler.getFieldString(name);
- }
- });
- tiddlerFields.revision = state.wiki.getChangeCount(title);
- tiddlerFields.type = tiddlerFields.type || "text/vnd.tiddlywiki";
- tiddlers.push(tiddlerFields);
+ $tw.utils.each(titles,function(title) {
+ var tiddler = state.wiki.getTiddler(title);
+ if(tiddler) {
+ var tiddlerFields = tiddler.getFieldStrings({exclude: excludeFields});
+ tiddlerFields.revision = state.wiki.getChangeCount(title);
+ tiddlerFields.type = tiddlerFields.type || "text/vnd.tiddlywiki";
+ tiddlers.push(tiddlerFields);
+ }
});
var text = JSON.stringify(tiddlers);
response.end(text,"utf8");
diff --git a/core/modules/server/server.js b/core/modules/server/server.js
index 3225b95f39d..3226cacd727 100644
--- a/core/modules/server/server.js
+++ b/core/modules/server/server.js
@@ -16,7 +16,8 @@ if($tw.node) {
var util = require("util"),
fs = require("fs"),
url = require("url"),
- path = require("path");
+ path = require("path"),
+ querystring = require("querystring");
}
/*
@@ -162,6 +163,7 @@ Server.prototype.requestHandler = function(request,response) {
state.wiki = self.wiki;
state.server = self;
state.urlInfo = url.parse(request.url);
+ state.queryParameters = querystring.parse(state.urlInfo.query);
// Get the principals authorized to access this resource
var authorizationType = this.methodMappings[request.method] || "readers";
// Check for the CSRF header if this is a write
@@ -236,6 +238,7 @@ host: optional host address (falls back to value of "host" variable)
prefix: optional prefix (falls back to value of "path-prefix" variable)
*/
Server.prototype.listen = function(port,host,prefix) {
+ var self = this;
// Handle defaults for port and host
port = port || this.get("port");
host = host || this.get("host");
@@ -244,19 +247,24 @@ Server.prototype.listen = function(port,host,prefix) {
if(parseInt(port,10).toString() !== port) {
port = process.env[port] || 8080;
}
- $tw.utils.log("Serving on " + this.protocol + "://" + host + ":" + port + prefix,"brown/orange");
- $tw.utils.log("(press ctrl-C to exit)","red");
// Warn if required plugins are missing
if(!$tw.wiki.getTiddler("$:/plugins/tiddlywiki/tiddlyweb") || !$tw.wiki.getTiddler("$:/plugins/tiddlywiki/filesystem")) {
$tw.utils.warning("Warning: Plugins required for client-server operation (\"tiddlywiki/filesystem\" and \"tiddlywiki/tiddlyweb\") are missing from tiddlywiki.info file");
}
- // Listen
+ // Create the server
var server;
if(this.listenOptions) {
server = this.transport.createServer(this.listenOptions,this.requestHandler.bind(this));
} else {
server = this.transport.createServer(this.requestHandler.bind(this));
}
+ // Display the port number after we've started listening (the port number might have been specified as zero, in which case we will get an assigned port)
+ server.on("listening",function() {
+ var address = server.address();
+ $tw.utils.log("Serving on " + self.protocol + "://" + address.address + ":" + address.port + prefix,"brown/orange");
+ $tw.utils.log("(press ctrl-C to exit)","red");
+ });
+ // Listen
return server.listen(port,host);
};
diff --git a/core/modules/startup/startup.js b/core/modules/startup/startup.js
index 4cd53dfcded..ad1416bf39c 100755
--- a/core/modules/startup/startup.js
+++ b/core/modules/startup/startup.js
@@ -128,7 +128,7 @@ exports.startup = function() {
// Set up the syncer object if we've got a syncadaptor
if($tw.syncadaptor) {
$tw.syncer = new $tw.Syncer({wiki: $tw.wiki, syncadaptor: $tw.syncadaptor});
- }
+ }
// Setup the saver handler
$tw.saverHandler = new $tw.SaverHandler({
wiki: $tw.wiki,
diff --git a/core/modules/syncer.js b/core/modules/syncer.js
index 0b84be750ea..f39646eace7 100644
--- a/core/modules/syncer.js
+++ b/core/modules/syncer.js
@@ -3,7 +3,7 @@ title: $:/core/modules/syncer.js
type: application/javascript
module-type: global
-The syncer tracks changes to the store. If a syncadaptor is used then individual tiddlers are synchronised through it. If there is no syncadaptor then the entire wiki is saved via saver modules.
+The syncer tracks changes to the store and synchronises them to a remote data store represented as a "sync adaptor"
\*/
(function(){
@@ -23,8 +23,10 @@ Syncer.prototype.titleSyncFilter = "$:/config/SyncFilter";
Syncer.prototype.titleSyncPollingInterval = "$:/config/SyncPollingInterval";
Syncer.prototype.titleSyncDisableLazyLoading = "$:/config/SyncDisableLazyLoading";
Syncer.prototype.titleSavedNotification = "$:/language/Notifications/Save/Done";
+Syncer.prototype.titleSyncThrottleInterval = "$:/config/SyncThrottleInterval";
Syncer.prototype.taskTimerInterval = 1 * 1000; // Interval for sync timer
Syncer.prototype.throttleInterval = 1 * 1000; // Defer saving tiddlers if they've changed in the last 1s...
+Syncer.prototype.errorRetryInterval = 5 * 1000; // Interval to retry after an error
Syncer.prototype.fallbackInterval = 10 * 1000; // Unless the task is older than 10s
Syncer.prototype.pollTimerInterval = 60 * 1000; // Interval for polling for changes from the adaptor
@@ -36,6 +38,7 @@ wiki: wiki to be synced
function Syncer(options) {
var self = this;
this.wiki = options.wiki;
+ // Save parameters
this.syncadaptor = options.syncadaptor;
this.disableUI = !!options.disableUI;
this.titleIsLoggedIn = options.titleIsLoggedIn || this.titleIsLoggedIn;
@@ -43,27 +46,54 @@ function Syncer(options) {
this.titleSyncFilter = options.titleSyncFilter || this.titleSyncFilter;
this.titleSavedNotification = options.titleSavedNotification || this.titleSavedNotification;
this.taskTimerInterval = options.taskTimerInterval || this.taskTimerInterval;
- this.throttleInterval = options.throttleInterval || this.throttleInterval;
+ this.throttleInterval = options.throttleInterval || parseInt(this.wiki.getTiddlerText(this.titleSyncThrottleInterval,""),10) || this.throttleInterval;
+ this.errorRetryInterval = options.errorRetryInterval || this.errorRetryInterval;
this.fallbackInterval = options.fallbackInterval || this.fallbackInterval;
this.pollTimerInterval = options.pollTimerInterval || parseInt(this.wiki.getTiddlerText(this.titleSyncPollingInterval,""),10) || this.pollTimerInterval;
this.logging = "logging" in options ? options.logging : true;
// Make a logger
this.logger = new $tw.utils.Logger("syncer" + ($tw.browser ? "-browser" : "") + ($tw.node ? "-server" : "") + (this.syncadaptor.name ? ("-" + this.syncadaptor.name) : ""),{
- colour: "cyan",
- enable: this.logging
- });
+ colour: "cyan",
+ enable: this.logging,
+ saveHistory: true
+ });
+ if(this.syncadaptor.setLoggerSaveBuffer) {
+ this.syncadaptor.setLoggerSaveBuffer(this.logger);
+ }
// Compile the dirty tiddler filter
this.filterFn = this.wiki.compileFilter(this.wiki.getTiddlerText(this.titleSyncFilter));
// Record information for known tiddlers
this.readTiddlerInfo();
- // Tasks are {type: "load"/"save"/"delete", title:, queueTime:, lastModificationTime:}
- this.taskQueue = {}; // Hashmap of tasks yet to be performed
- this.taskInProgress = {}; // Hash of tasks in progress
+ this.titlesToBeLoaded = {}; // Hashmap of titles of tiddlers that need loading from the server
+ this.titlesHaveBeenLazyLoaded = {}; // Hashmap of titles of tiddlers that have already been lazily loaded from the server
+ // Timers
this.taskTimerId = null; // Timer for task dispatch
this.pollTimerId = null; // Timer for polling server
+ // Number of outstanding requests
+ this.numTasksInProgress = 0;
// Listen out for changes to tiddlers
this.wiki.addEventListener("change",function(changes) {
- self.syncToServer(changes);
+ // Filter the changes to just include ones that are being synced
+ var filteredChanges = self.getSyncedTiddlers(function(callback) {
+ $tw.utils.each(changes,function(change,title) {
+ var tiddler = self.wiki.tiddlerExists(title) && self.wiki.getTiddler(title);
+ callback(tiddler,title);
+ });
+ });
+ if(filteredChanges.length > 0) {
+ self.processTaskQueue();
+ } else {
+ // Look for deletions of tiddlers we're already syncing
+ var outstandingDeletion = false
+ $tw.utils.each(changes,function(change,title,object) {
+ if(change.deleted && $tw.utils.hop(self.tiddlerInfo,title)) {
+ outstandingDeletion = true;
+ }
+ });
+ if(outstandingDeletion) {
+ self.processTaskQueue();
+ }
+ }
});
// Browser event handlers
if($tw.browser && !this.disableUI) {
@@ -86,6 +116,9 @@ function Syncer(options) {
$tw.rootWidget.addEventListener("tm-server-refresh",function() {
self.handleRefreshEvent();
});
+ $tw.rootWidget.addEventListener("tm-copy-syncer-logs-to-clipboard",function() {
+ $tw.utils.copyToClipboard($tw.utils.getSystemInfo() + "\n\nLog:\n" + self.logger.getBuffer());
+ });
}
// Listen out for lazyLoad events
if(!this.disableUI && $tw.wiki.getTiddlerText(this.titleSyncDisableLazyLoading) !== "yes") {
@@ -100,45 +133,83 @@ function Syncer(options) {
});
}
+/*
+Show a generic network error alert
+*/
+Syncer.prototype.showErrorAlert = function() {
+console.log($tw.language.getString("Error/NetworkErrorAlert"))
+ this.logger.alert($tw.language.getString("Error/NetworkErrorAlert"));
+};
+
+/*
+Return an array of the tiddler titles that are subjected to syncing
+*/
+Syncer.prototype.getSyncedTiddlers = function(source) {
+ return this.filterFn.call(this.wiki,source);
+};
+
+/*
+Return an array of the tiddler titles that are subjected to syncing
+*/
+Syncer.prototype.getTiddlerRevision = function(title) {
+ if(this.syncadaptor && this.syncadaptor.getTiddlerRevision) {
+ return this.syncadaptor.getTiddlerRevision(title);
+ } else {
+ return this.wiki.getTiddler(title).fields.revision;
+ }
+};
+
/*
Read (or re-read) the latest tiddler info from the store
*/
Syncer.prototype.readTiddlerInfo = function() {
// Hashmap by title of {revision:,changeCount:,adaptorInfo:}
+ // "revision" is the revision of the tiddler last seen on the server, and "changecount" is the corresponding local changecount
this.tiddlerInfo = {};
// Record information for known tiddlers
var self = this,
- tiddlers = this.filterFn.call(this.wiki);
+ tiddlers = this.getSyncedTiddlers();
$tw.utils.each(tiddlers,function(title) {
- var tiddler = self.wiki.getTiddler(title);
+ var tiddler = self.wiki.tiddlerExists(title) && self.wiki.getTiddler(title);
self.tiddlerInfo[title] = {
- revision: tiddler.fields.revision,
+ revision: self.getTiddlerRevision(title),
adaptorInfo: self.syncadaptor && self.syncadaptor.getTiddlerInfo(tiddler),
- changeCount: self.wiki.getChangeCount(title),
- hasBeenLazyLoaded: false
+ changeCount: self.wiki.getChangeCount(title)
};
});
};
-/*
-Create an tiddlerInfo structure if it doesn't already exist
-*/
-Syncer.prototype.createTiddlerInfo = function(title) {
- if(!$tw.utils.hop(this.tiddlerInfo,title)) {
- this.tiddlerInfo[title] = {
- revision: null,
- adaptorInfo: {},
- changeCount: -1,
- hasBeenLazyLoaded: false
- };
- }
-};
-
/*
Checks whether the wiki is dirty (ie the window shouldn't be closed)
*/
Syncer.prototype.isDirty = function() {
- return (this.numTasksInQueue() > 0) || (this.numTasksInProgress() > 0);
+ this.logger.log("Checking dirty status");
+ // Check tiddlers that are in the store and included in the filter function
+ var titles = this.getSyncedTiddlers();
+ for(var index=0; index tiddlerInfo.changeCount) {
+ return true;
+ }
+ } else {
+ // If the tiddler isn't known on the server then it needs to be saved to the server
+ return true;
+ }
+ }
+ }
+ // Check tiddlers that are known from the server but not currently in the store
+ titles = Object.keys(this.tiddlerInfo);
+ for(index=0; index 0 || updates.deletions.length > 0) {
+ self.processTaskQueue();
+ }
+ }
+ });
+ } else if(this.syncadaptor && this.syncadaptor.getSkinnyTiddlers) {
+ this.logger.log("Retrieving skinny tiddler list");
+ cancelNextSync();
+ this.syncadaptor.getSkinnyTiddlers(function(err,tiddlers) {
+ triggerNextSync();
// Check for errors
if(err) {
- self.logger.alert($tw.language.getString("Error/RetrievingSkinny") + ":",err);
+ self.showErrorAlert();
+ self.logger.log($tw.language.getString("Error/RetrievingSkinny") + ":",err);
return;
}
+ // Keep track of which tiddlers we already know about have been reported this time
+ var previousTitles = Object.keys(self.tiddlerInfo);
// Process each incoming tiddler
for(var t=0; t tiddlerInfo.changeCount,
+ isReadyToSave = !tiddlerInfo || !tiddlerInfo.timestampLastSaved || tiddlerInfo.timestampLastSaved < thresholdLastSaved;
+ if(hasChanged) {
+ if(isReadyToSave) {
+ return new SaveTiddlerTask(this,title);
+ } else {
+ havePending = true;
}
- // Mark that this task is no longer in progress
- delete self.taskInProgress[task.title];
- self.updateDirtyStatus();
- // Process the next task
- self.processTaskQueue.call(self);
- });
- } else {
- // Make sure we've set a time if there wasn't a task to perform, but we've still got tasks in the queue
- if(this.numTasksInQueue() > 0) {
- this.triggerTimeout();
}
}
}
-};
-
-/*
-Choose the next applicable task
-*/
-Syncer.prototype.chooseNextTask = function() {
- var self = this,
- candidateTask = null,
- now = Date.now();
- // Select the best candidate task
- $tw.utils.each(this.taskQueue,function(task,title) {
- // Exclude the task if there's one of the same name in progress
- if($tw.utils.hop(self.taskInProgress,title)) {
- return;
- }
- // Exclude the task if it is a save and the tiddler has been modified recently, but not hit the fallback time
- if(task.type === "save" && (now - task.lastModificationTime) < self.throttleInterval &&
- (now - task.queueTime) < self.fallbackInterval) {
- return;
- }
- // Exclude the task if it is newer than the current best candidate
- if(candidateTask && candidateTask.queueTime < task.queueTime) {
- return;
+ // Second, we check tiddlers that are known from the server but not currently in the store, and so need deleting on the server
+ titles = Object.keys(this.tiddlerInfo);
+ for(index=0; index 1) {
@@ -87,6 +115,22 @@ Logger.prototype.alert = function(/* args */) {
}
};
+/*
+Clear outstanding alerts
+*/
+Logger.prototype.clearAlerts = function() {
+ var self = this;
+ if($tw.browser && this.alertCount > 0) {
+ $tw.utils.each($tw.wiki.getTiddlersWithTag(ALERT_TAG),function(title) {
+ var tiddler = $tw.wiki.getTiddler(title);
+ if(tiddler.fields.component === self.componentName) {
+ $tw.wiki.deleteTiddler(title);
+ }
+ });
+ this.alertCount = 0;
+ }
+};
+
exports.Logger = Logger;
})();
diff --git a/core/modules/utils/utils.js b/core/modules/utils/utils.js
index 2495be49fa5..7787783e5c9 100644
--- a/core/modules/utils/utils.js
+++ b/core/modules/utils/utils.js
@@ -779,4 +779,22 @@ exports.strEndsWith = function(str,ending,position) {
}
};
+/*
+Return system information useful for debugging
+*/
+exports.getSystemInfo = function(str,ending,position) {
+ var results = [],
+ save = function(desc,value) {
+ results.push(desc + ": " + value);
+ };
+ if($tw.browser) {
+ save("User Agent",navigator.userAgent);
+ save("Online Status",window.navigator.onLine);
+ }
+ if($tw.node) {
+ save("Node Version",process.version);
+ }
+ return results.join("\n");
+};
+
})();
diff --git a/core/modules/wiki.js b/core/modules/wiki.js
index fc8b42eda40..2adfb0ef307 100755
--- a/core/modules/wiki.js
+++ b/core/modules/wiki.js
@@ -1233,9 +1233,9 @@ exports.getTiddlerText = function(title,defaultText) {
if(!tiddler) {
return defaultText;
}
- if(tiddler.fields.text !== undefined) {
+ if(!tiddler.hasField("_is_skinny")) {
// Just return the text if we've got it
- return tiddler.fields.text;
+ return tiddler.fields.text || "";
} else {
// Tell any listeners about the need to lazily load this tiddler
this.dispatchEvent("lazyLoad",title);
diff --git a/core/templates/html-div-skinny-tiddler.tid b/core/templates/html-div-skinny-tiddler.tid
new file mode 100644
index 00000000000..eaf388605e1
--- /dev/null
+++ b/core/templates/html-div-skinny-tiddler.tid
@@ -0,0 +1,9 @@
+title: $:/core/templates/html-div-skinny-tiddler
+
+`
diff --git a/core/wiki/config/ServerExternalFiltersDefault.tid b/core/wiki/config/ServerExternalFiltersDefault.tid
new file mode 100644
index 00000000000..7ef93ecf377
--- /dev/null
+++ b/core/wiki/config/ServerExternalFiltersDefault.tid
@@ -0,0 +1,2 @@
+title: $:/config/Server/ExternalFilters/[all[tiddlers]!is[system]sort[title]]
+text: yes
diff --git a/editions/dev/tiddlers/from tw5.com/moduletypes/SyncAdaptorModules.tid b/editions/dev/tiddlers/from tw5.com/moduletypes/SyncAdaptorModules.tid
index 6fdc6af3cfc..6f88fb9a50c 100644
--- a/editions/dev/tiddlers/from tw5.com/moduletypes/SyncAdaptorModules.tid
+++ b/editions/dev/tiddlers/from tw5.com/moduletypes/SyncAdaptorModules.tid
@@ -1,5 +1,5 @@
created: 20130825162100000
-modified: 20140814094907624
+modified: 20200113094126878
tags: dev moduletypes
title: SyncAdaptorModules
type: text/vnd.tiddlywiki
@@ -14,6 +14,8 @@ SyncAdaptorModules encapsulate storage mechanisms that can be used by the SyncMe
SyncAdaptorModules are represented as JavaScript tiddlers with the field `module-type` set to `syncadaptor`.
+See [[this pull request|https://github.com/Jermolene/TiddlyWiki5/pull/4373]] for background on the evolution of this API.
+
! Exports
The following properties should be exposed via the `exports` object:
@@ -47,12 +49,21 @@ Gets the supplemental information that the adaptor needs to keep track of for a
Returns an object storing any additional information required by the adaptor.
+!! `getTiddlerRevision(title)`
+
+Gets the revision ID associated with the specified tiddler title.
+
+|!Parameter |!Description |
+|title |Tiddler title |
+
+Returns a revision ID.
+
!! `getStatus(callback)`
Retrieves status information from the server. This method is optional.
|!Parameter |!Description |
-|callback |Callback function invoked with parameters `err,isLoggedIn,username` |
+|callback |Callback function invoked with parameters `err,isLoggedIn,username,isReadOnly` |
!! `login(username,password,callback)`
@@ -70,16 +81,39 @@ Attempts to logout of the server. This method is optional.
|!Parameter |!Description |
|callback |Callback function invoked with parameter `err` |
+!! `getUpdatedTiddlers(syncer,callback)`
+
+Retrieves the titles of tiddlers that need to be updated from the server.
+
+This method is optional. If an adaptor doesn't implement it then synchronisation will be unidirectional from the TiddlyWiki store to the adaptor, but not the other way.
+
+The syncer will use the `getUpdatedTiddlers()` method in preference to the `getSkinnyTiddlers()` method.
+
+|!Parameter |!Description |
+|syncer |Reference to the syncer object making the call |
+|callback |Callback function invoked with parameter `err,data` -- see below |
+
+The data provided by the callback is as follows:
+
+```
+{
+modifications: [],
+deletions: [],
+}
+```
+
!! `getSkinnyTiddlers(callback)`
Retrieves a list of skinny tiddlers from the server.
This method is optional. If an adaptor doesn't implement it then synchronisation will be unidirectional from the TiddlyWiki store to the adaptor, but not the other way.
+The syncer will use the `getUpdatedTiddlers()` method in preference to the `getSkinnyTiddlers()` method.
+
|!Parameter |!Description |
|callback |Callback function invoked with parameter `err,tiddlers`, where `tiddlers` is an array of tiddler field objects |
-!! `saveTiddler(tiddler,callback,tiddlerInfo)`
+!! `saveTiddler(tiddler,callback)`
Saves a tiddler to the server.
@@ -96,11 +130,16 @@ Loads a tiddler from the server.
|title |Title of tiddler to be retrieved |
|callback |Callback function invoked with parameter `err,tiddlerFields` |
-!! `deleteTiddler(title,callback,tiddlerInfo)`
+!! `deleteTiddler(title,callback,options)`
Delete a tiddler from the server.
|!Parameter |!Description |
|title |Title of tiddler to be deleted |
|callback |Callback function invoked with parameter `err` |
+|options |See below |
+
+The options parameter contains the following properties:
+
+|!Property |!Description |
|tiddlerInfo |The tiddlerInfo maintained by the syncer for this tiddler |
diff --git a/editions/tw5.com/tiddlers/concepts/TiddlerFields.tid b/editions/tw5.com/tiddlers/concepts/TiddlerFields.tid
index 0a0c07c7e6f..ced34345404 100644
--- a/editions/tw5.com/tiddlers/concepts/TiddlerFields.tid
+++ b/editions/tw5.com/tiddlers/concepts/TiddlerFields.tid
@@ -1,5 +1,5 @@
created: 20130825213300000
-modified: 20191013093910961
+modified: 20191206152031468
tags: Concepts
title: TiddlerFields
type: text/vnd.tiddlywiki
@@ -42,11 +42,13 @@ Other fields used by the core are:
|`subtitle` |<> |
|`throttle.refresh` |<> |
|`toc-link`|<>|
+|`_canonical_uri`|<>|
The TiddlyWebAdaptor uses a few more fields:
|!Field Name |!Description |
|`bag` |<> |
|`revision` |<> |
+|`_is_skinny` |<> |
Details of the fields used in this ~TiddlyWiki are shown in the [[control panel|$:/ControlPanel]] {{$:/core/ui/Buttons/control-panel}} under the <<.controlpanel-tab Info>> tab >> <<.info-tab Advanced>> sub-tab >> Tiddler Fields
diff --git a/editions/tw5.com/tiddlers/webserver/WebServer API_ Get All Tiddlers.tid b/editions/tw5.com/tiddlers/webserver/WebServer API_ Get All Tiddlers.tid
index 98119804ca5..52ca8964f15 100644
--- a/editions/tw5.com/tiddlers/webserver/WebServer API_ Get All Tiddlers.tid
+++ b/editions/tw5.com/tiddlers/webserver/WebServer API_ Get All Tiddlers.tid
@@ -1,5 +1,5 @@
created: 20181002131215403
-modified: 20190903094711346
+modified: 2020031109590546
tags: [[WebServer API]]
title: WebServer API: Get All Tiddlers
type: text/vnd.tiddlywiki
@@ -12,11 +12,23 @@ GET /recipes/default/tiddlers.json
Parameters:
-* none
+* ''filter'' - filter identifying tiddlers to be returned (optional, defaults to "[all[tiddlers]!is[system]sort[title]]")
+* ''exclude'' - comma delimited list of fields to excluded from the returned tiddlers (optional, defaults to "text")
-Response:
+In order to avoid denial of service attacks with malformed filters in the default configuration the only filter that is accepted is the default filter "[all[tiddlers]!is[system]sort[title]]"; attempts to use any other filter will result in an HTTP 403 error.
+
+To enable a particular filter, create a tiddler with the title "$:/config/Server/ExternalFilters/" concatenated with the filter text, and the text field set to "yes". For example, the TiddlyWeb plugin includes the following shadow tiddler to enable the filter that it requires:
+```
+title: $:/config/Server/ExternalFilters/[all[tiddlers]] -[[$:/isEncrypted]] -[prefix[$:/temp/]] -[prefix[$:/status/]]
+text: yes
+```
+
+It is also possible to configure the server to accept any filter by creating a tiddler titled $:/config/Server/AllowAllExternalFilters with the text "yes". This should not be done for public facing servers.
+
+Response:
* 200 OK
*> `Content-Type: application/json`
*> Body: array of all non-system tiddlers in [[TiddlyWeb JSON tiddler format]]
+* 403 Forbidden
diff --git a/editions/tw5.com/tiddlers/webserver/WebServer Parameter_ port.tid b/editions/tw5.com/tiddlers/webserver/WebServer Parameter_ port.tid
index f05da08fca3..bbfee9906a6 100644
--- a/editions/tw5.com/tiddlers/webserver/WebServer Parameter_ port.tid
+++ b/editions/tw5.com/tiddlers/webserver/WebServer Parameter_ port.tid
@@ -1,6 +1,6 @@
caption: port
created: 20180630180552254
-modified: 20180702155017130
+modified: 20191219123751824
tags: [[WebServer Parameters]]
title: WebServer Parameter: port
type: text/vnd.tiddlywiki
@@ -10,6 +10,7 @@ The [[web server configuration parameter|WebServer Parameters]] ''port'' specifi
The ''port'' parameter accepts two types of value:
* Numerical values are interpreted as a decimal port number
+** The special value 0 (zero) causes the operating system to assign an available port
* Non-numeric values are interpreted as an environment variable from which the port should be read
This example configures the server to listen on port 8090:
diff --git a/plugins/tiddlywiki/filesystem/filesystemadaptor.js b/plugins/tiddlywiki/filesystem/filesystemadaptor.js
index a346a66065f..9e0734814ff 100644
--- a/plugins/tiddlywiki/filesystem/filesystemadaptor.js
+++ b/plugins/tiddlywiki/filesystem/filesystemadaptor.js
@@ -26,6 +26,8 @@ function FileSystemAdaptor(options) {
FileSystemAdaptor.prototype.name = "filesystem";
+FileSystemAdaptor.prototype.supportsLazyLoading = false;
+
FileSystemAdaptor.prototype.isReady = function() {
// The file system adaptor is always ready
return true;
diff --git a/plugins/tiddlywiki/tiddlyweb/ServerControlPanel.tid b/plugins/tiddlywiki/tiddlyweb/ServerControlPanel.tid
deleted file mode 100644
index 9ef1e27afbb..00000000000
--- a/plugins/tiddlywiki/tiddlyweb/ServerControlPanel.tid
+++ /dev/null
@@ -1,20 +0,0 @@
-title: $:/plugins/tiddlywiki/tiddlyweb/ServerControlPanel
-caption: Server
-tags: $:/tags/ControlPanel
-
-<$reveal state="$:/status/IsLoggedIn" type="nomatch" text="yes">
-Log in to ~TiddlyWeb: <$button message="tm-login">Login$button>
-$reveal>
-<$reveal state="$:/status/IsLoggedIn" type="match" text="yes">
-Logged in as {{$:/status/UserName}} <$button message="tm-logout">Logout$button>
-$reveal>
-
-----
-
-Host configuration: <$edit-text tiddler="$:/config/tiddlyweb/host" tag="input" default=""/>
-
-
//for example, `$protocol$//$host$/folder`, where `$protocol$` is replaced by the protocol (typically `http` or `https`), and `$host$` by the host name//
-
-----
-
-<$button message="tm-server-refresh">Refresh$button> to fetch changes from the server immediately
diff --git a/plugins/tiddlywiki/tiddlyweb/config-tiddlers-filter.tid b/plugins/tiddlywiki/tiddlyweb/config-tiddlers-filter.tid
new file mode 100644
index 00000000000..04bf24613b3
--- /dev/null
+++ b/plugins/tiddlywiki/tiddlyweb/config-tiddlers-filter.tid
@@ -0,0 +1,2 @@
+title: $:/config/Server/ExternalFilters/[all[tiddlers]] -[[$:/isEncrypted]] -[prefix[$:/temp/]] -[prefix[$:/status/]]
+text: yes
diff --git a/plugins/tiddlywiki/tiddlyweb/configOfficialPluginLibrary.tid b/plugins/tiddlywiki/tiddlyweb/configOfficialPluginLibrary.tid
new file mode 100644
index 00000000000..9f0e164f414
--- /dev/null
+++ b/plugins/tiddlywiki/tiddlyweb/configOfficialPluginLibrary.tid
@@ -0,0 +1,3 @@
+title: $:/config/OfficialPluginLibrary
+
+(This core tiddler is overridden by the tiddlyweb plugin to prevent users from installing official plugins via control panel. Instead they should be installed by editing tiddlywiki.info in the root of the wiki folder)
\ No newline at end of file
diff --git a/plugins/tiddlywiki/tiddlyweb/css-tiddler.tid b/plugins/tiddlywiki/tiddlyweb/css-tiddler.tid
new file mode 100644
index 00000000000..2d7367fa93d
--- /dev/null
+++ b/plugins/tiddlywiki/tiddlyweb/css-tiddler.tid
@@ -0,0 +1,7 @@
+title: $:/core/templates/css-tiddler
+
+``
\ No newline at end of file
diff --git a/plugins/tiddlywiki/tiddlyweb/html-div-skinny-tiddler.tid b/plugins/tiddlywiki/tiddlyweb/html-div-skinny-tiddler.tid
new file mode 100644
index 00000000000..010a603b55c
--- /dev/null
+++ b/plugins/tiddlywiki/tiddlyweb/html-div-skinny-tiddler.tid
@@ -0,0 +1,9 @@
+title: $:/core/templates/html-div-skinny-tiddler
+
+`