diff --git a/node/server.js b/node/server.js index f254c2ca9..724e7a617 100644 --- a/node/server.js +++ b/node/server.js @@ -31,6 +31,7 @@ var async = require('async'); var express = require('express'); var path = require('path'); var minify = require('./utils/Minify'); +var CachingMiddleware = require('./utils/caching_middleware'); var formidable = require('formidable'); var apiHandler; var exportHandler; @@ -61,8 +62,7 @@ console.log("Report bugs at https://github.com/Pita/etherpad-lite/issues") var serverName = "Etherpad-Lite " + version + " (http://j.mp/ep-lite)"; -//cache 6 hours -exports.maxAge = 1000*60*60*6; +exports.maxAge = settings.maxAge; //set loglevel log4js.setGlobalLogLevel(settings.loglevel); @@ -155,7 +155,8 @@ async.waterfall([ }); //serve minified files - app.get('/minified/:filename', minify.minifyJS); + var assetCache = new CachingMiddleware; + app.all('/minified/:filename', assetCache.handle, minify.minifyJS); //checks for padAccess function hasPadAccess(req, res, callback) diff --git a/node/utils/Minify.js b/node/utils/Minify.js index 859ee07ae..f54dc4d3a 100644 --- a/node/utils/Minify.js +++ b/node/utils/Minify.js @@ -27,16 +27,12 @@ var cleanCSS = require('clean-css'); var jsp = require("uglify-js").parser; var pro = require("uglify-js").uglify; var path = require('path'); -var Buffer = require('buffer').Buffer; -var zlib = require('zlib'); var RequireKernel = require('require-kernel'); var server = require('../server'); -var os = require('os'); var ROOT_DIR = path.normalize(__dirname + "/../" ); var JS_DIR = ROOT_DIR + '../static/js/'; var CSS_DIR = ROOT_DIR + '../static/css/'; -var CACHE_DIR = ROOT_DIR + '../var/'; var TAR_PATH = path.join(__dirname, 'tar.json'); var tar = JSON.parse(fs.readFileSync(TAR_PATH, 'utf8')); @@ -57,160 +53,61 @@ exports.minifyJS = function(req, res, next) } else { // Not in tar list, but try anyways, if it fails, pass to `next`. jsFiles = [jsFilename]; - fs.stat(JS_DIR + jsFilename, function (error, stats) { - if (error || !stats.isFile()) { - next(); - } else { - _handle(req, res, jsFilename, jsFiles); - } - }); + _handle(req, res, jsFilename, jsFiles); } } function _handle(req, res, jsFilename, jsFiles) { res.header("Content-Type","text/javascript"); - - //minifying is enabled - if(settings.minify) - { - var result = undefined; - var latestModification = 0; - - async.series([ - //find out the highest modification date - function(callback) - { - var folders2check = [CSS_DIR, JS_DIR]; - - //go trough this two folders - async.forEach(folders2check, function(path, callback) - { - //read the files in the folder - fs.readdir(path, function(err, files) - { - if(ERR(err, callback)) return; - - //we wanna check the directory itself for changes too - files.push("."); - - //go trough all files in this folder - async.forEach(files, function(filename, callback) - { - //get the stat data of this file - fs.stat(path + "/" + filename, function(err, stats) - { - if(ERR(err, callback)) return; - - //get the modification time - var modificationTime = stats.mtime.getTime(); - - //compare the modification time to the highest found - if(modificationTime > latestModification) - { - latestModification = modificationTime; - } - - callback(); - }); - }, callback); - }); - }, callback); - }, - function(callback) - { - //check the modification time of the minified js - fs.stat(CACHE_DIR + "/minified_" + jsFilename, function(err, stats) - { - if(err && err.code != "ENOENT") - { - ERR(err, callback); - return; - } - - //there is no minfied file or there new changes since this file was generated, so continue generating this file - if((err && err.code == "ENOENT") || stats.mtime.getTime() < latestModification) - { - callback(); - } - //the minified file is still up to date, stop minifying - else - { - callback("stop"); - } - }); - }, - //load all js files - function (callback) - { - var values = []; - tarCode( - jsFiles - , function (content) {values.push(content)} - , function (err) { - if(ERR(err)) return; - result = values.join(''); - callback(); - }); - }, - //put all together and write it into a file - function(callback) - { - async.parallel([ - //write the results plain in a file - function(callback) - { - fs.writeFile(CACHE_DIR + "minified_" + jsFilename, result, "utf8", callback); - }, - //write the results compressed in a file - function(callback) - { - zlib.gzip(result, function(err, compressedResult){ - //weird gzip bug that returns 0 instead of null if everything is ok - err = err === 0 ? null : err; - - if(ERR(err, callback)) return; - - fs.writeFile(CACHE_DIR + "minified_" + jsFilename + ".gz", compressedResult, callback); - }); - } - ],callback); + lastModifiedDate(function (date) { + date = new Date(date); + res.setHeader('last-modified', date.toUTCString()); + res.setHeader('date', (new Date()).toUTCString()); + if (server.maxAge) { + var expiresDate = new Date((new Date()).getTime() + server.maxAge*1000); + res.setHeader('expires', expiresDate.toUTCString()); + res.setHeader('cache-control', 'max-age=' + server.maxAge); + } + + fs.stat(JS_DIR + jsFiles[0], function (error, stats) { + if (error) { + if (error.code == "ENOENT") { + res.writeHead(404, {}); + res.end(); + } else { + res.writeHead(500, {}); + res.end(); + } + } else if (!stats.isFile()) { + res.writeHead(404, {}); + res.end(); + } else if (new Date(req.headers['if-modified-since']) >= date) { + res.writeHead(304, {}); + res.end(); + } else { + if (req.method == 'HEAD') { + res.writeHead(200, {}); + res.end(); + } else if (req.method == 'GET') { + res.writeHead(200, {}); + tarCode( + jsFiles + , function (content) { + res.write(content); + } + , function (err) { + if(ERR(err)) return; + res.end(); + } + ); + } else { + res.writeHead(405, {'allow': 'HEAD, GET'}); + res.end(); + } } - ], function(err) - { - if(err && err != "stop") - { - if(ERR(err)) return; - } - - //check if gzip is supported by this browser - var gzipSupport = req.header('Accept-Encoding', '').indexOf('gzip') != -1; - - var pathStr; - if(gzipSupport && os.type().indexOf("Windows") == -1) - { - pathStr = path.normalize(CACHE_DIR + "minified_" + jsFilename + ".gz"); - res.header('Content-Encoding', 'gzip'); - } - else - { - pathStr = path.normalize(CACHE_DIR + "minified_" + jsFilename ); - } - - res.sendfile(pathStr, { maxAge: server.maxAge }); - }) - } - //minifying is disabled, so put the files together in one file - else - { - tarCode( - jsFiles - , function (content) {res.write(content)} - , function (err) { - if(ERR(err)) return; - res.end(); }); - } + }); } // find all includes in ace.js and embed them. @@ -232,7 +129,8 @@ function getAceFile(callback) { async.forEach(founds, function (item, callback) { var filename = item.match(/"([^"]*)"/)[1]; var type = item.match(/INCLUDE_([A-Z]+)/)[1]; - var shortFilename = (filename.match(/^..\/static\/js\/(.*)$/, '')||[])[1]; + var shortFilename = + (filename.match(/^\.\.\/static\/js\/(.*)$/, '') || [])[1]; //read the included files if (shortFilename) { @@ -279,6 +177,46 @@ function getAceFile(callback) { }); } +function lastModifiedDate(callback) { + var folders2check = [CSS_DIR, JS_DIR]; + var latestModification = 0; + //go trough this two folders + async.forEach(folders2check, function(path, callback) + { + //read the files in the folder + fs.readdir(path, function(err, files) + { + if(ERR(err, callback)) return; + + //we wanna check the directory itself for changes too + files.push("."); + + //go trough all files in this folder + async.forEach(files, function(filename, callback) + { + //get the stat data of this file + fs.stat(path + "/" + filename, function(err, stats) + { + if(ERR(err, callback)) return; + + //get the modification time + var modificationTime = stats.mtime.getTime(); + + //compare the modification time to the highest found + if(modificationTime > latestModification) + { + latestModification = modificationTime; + } + + callback(); + }); + }, callback); + }); + }, function () { + callback(latestModification); + }); +} + exports.requireDefinition = requireDefinition; function requireDefinition() { return 'var require = ' + RequireKernel.kernelSource + ';\n'; diff --git a/node/utils/Settings.js b/node/utils/Settings.js index efa226947..0d30fb688 100644 --- a/node/utils/Settings.js +++ b/node/utils/Settings.js @@ -56,6 +56,11 @@ exports.requireSession = false; */ exports.editOnly = false; +/** + * Max age that responses will have (affects caching layer). + */ +exports.maxAge = 1000*60*60*6; // 6 hours + /** * A flag that shows if minification is enabled or not */ diff --git a/node/utils/caching_middleware.js b/node/utils/caching_middleware.js new file mode 100644 index 000000000..e1d8677ea --- /dev/null +++ b/node/utils/caching_middleware.js @@ -0,0 +1,175 @@ +/* + * 2011 Peter 'Pita' Martischka (Primary Technology Ltd) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var async = require('async'); +var Buffer = require('buffer').Buffer; +var fs = require('fs'); +var path = require('path'); +var server = require('../server'); +var zlib = require('zlib'); +var util = require('util'); + +var ROOT_DIR = path.normalize(__dirname + "/../"); +var CACHE_DIR = ROOT_DIR + '../var/'; + +var responseCache = {}; + +/* + This caches and compresses 200 and 404 responses to GET and HEAD requests. + TODO: Caching and compressing are solved problems, a middleware configuration + should replace this. +*/ + +function CachingMiddleware() { +} +CachingMiddleware.prototype = new function () { + function handle(req, res, next) { + if (!(req.method == "GET" || req.method == "HEAD")) { + return next(undefined, req, res); + } + + var old_req = {}; + var old_res = {}; + + var supportsGzip = + req.header('Accept-Encoding', '').indexOf('gzip') != -1; + + var path = require('url').parse(req.url).path; + var cacheKey = (new Buffer(path)).toString('base64').replace(/[\/\+=]/g, ''); + + fs.stat(CACHE_DIR + 'minified_' + cacheKey, function (error, stats) { + var modifiedSince = (req.headers['if-modified-since'] + && new Date(req.headers['if-modified-since'])); + var lastModifiedCache = stats && stats.mtime; + if (lastModifiedCache) { + req.headers['if-modified-since'] = lastModifiedCache.toUTCString(); + } else { + delete req.headers['if-modified-since']; + } + + // Always issue get to downstream. + old_req.method = req.method; + req.method = 'GET'; + + var expirationDate = new Date(((responseCache[cacheKey] || {}).headers || {})['expires']); + if (expirationDate > new Date()) { + // Our cached version is still valid. + return respond(); + } + + var _headers = {}; + old_res.setHeader = res.setHeader; + res.setHeader = function (key, value) { + _headers[key.toLowerCase()] = value; + old_res.setHeader.call(res, key, value); + }; + + old_res.writeHead = res.writeHead; + res.writeHead = function (status, headers) { + var lastModified = (res.getHeader('last-modified') + && new Date(res.getHeader('last-modified'))); + + res.writeHead = old_res.writeHead; + if (status == 200 || status == 404) { + // Update cache + var buffer = ''; + + Object.keys(headers).forEach(function (key) { + res.setHeader(key, headers[key]); + }); + headers = _headers; + responseCache[cacheKey] = {statusCode: status, headers: headers}; + + old_res.write = res.write; + old_res.end = res.end; + res.write = function(data, encoding) { + buffer += data.toString(encoding); + }; + res.end = function(data, encoding) { + async.parallel([ + function (callback) { + var path = CACHE_DIR + 'minified_' + cacheKey; + fs.writeFile(path, buffer, function (error, stats) { + callback(); + }); + } + , function (callback) { + var path = CACHE_DIR + 'minified_' + cacheKey + '.gz'; + zlib.gzip(buffer, function(error, content) { + if (error) { + callback(); + } else { + fs.writeFile(path, content, function (error, stats) { + callback(); + }); + } + }); + } + ], respond); + }; + } else if (status == 304) { + // Nothing new changed from the cached version. + old_res.write = res.write; + old_res.end = res.end; + res.write = function(data, encoding) {}; + res.end = function(data, encoding) { respond() }; + } else { + res.writeHead(status, headers); + } + }; + + next(undefined, req, res); + + // This handles read/write synchronization as well as its predecessor, + // which is to say, not at all. + // TODO: Implement locking on write or ditch caching of gzip and use + // existing middlewares. + function respond() { + req.method = old_req.method || req.method; + res.write = old_res.write || res.write; + res.end = old_res.end || res.end; + + var headers = responseCache[cacheKey].headers; + var statusCode = responseCache[cacheKey].statusCode; + + var pathStr = CACHE_DIR + 'minified_' + cacheKey; + if (supportsGzip) { + pathStr = pathStr + '.gz'; + headers['content-encoding'] = 'gzip'; + } + + var lastModified = (headers['last-modified'] + && new Date(headers['last-modified'])); + + if (statusCode == 200 && lastModified <= modifiedSince) { + res.writeHead(304, headers); + res.end(); + } else if (req.method == 'GET') { + var readStream = fs.createReadStream(pathStr); + res.writeHead(statusCode, headers); + util.pump(readStream, res); + } else { + res.writeHead(200, headers); + res.end(); + } + } + }); + } + + this.handle = handle; +}(); + +module.exports = CachingMiddleware; diff --git a/settings.json.template b/settings.json.template index 7cd0e8195..57bc5711b 100644 --- a/settings.json.template +++ b/settings.json.template @@ -38,6 +38,10 @@ /* if true, all css & js will be minified before sending to the client. This will improve the loading performance massivly, but makes it impossible to debug the javascript/css */ "minify" : true, + + /* How long may clients use served javascript code? Without versioning this + is may cause problems during deployment. */ + "maxAge" : 1000*60*60*6, // 6 hours /* This is the path to the Abiword executable. Setting it to null, disables abiword. Abiword is needed to enable the import/export of pads*/