Merge b661ef5960 into bf42359669

node/server.js

@@ -31,6 +31,7 @@ var async = require('async');
 var express = require('express');
 var path = require('path');
 var minify = require('./utils/Minify');
+var CachingMiddleware = require('./utils/caching_middleware');
 var formidable = require('formidable');
 var apiHandler;
 var exportHandler;
@@ -61,8 +62,7 @@ console.log("Report bugs at https://github.com/Pita/etherpad-lite/issues")
 
 var serverName = "Etherpad-Lite " + version + " (http://j.mp/ep-lite)";
 
-//cache 6 hours
+exports.maxAge = settings.maxAge;
-exports.maxAge = 1000*60*60*6;
 
 //set loglevel
 log4js.setGlobalLogLevel(settings.loglevel);
@@ -155,7 +155,8 @@ async.waterfall([
     });
     
     //serve minified files
-    app.get('/minified/:filename', minify.minifyJS);
+    var assetCache = new CachingMiddleware;
+    app.all('/minified/:filename', assetCache.handle, minify.minifyJS);
     
     //checks for padAccess
     function hasPadAccess(req, res, callback)

node/utils/Minify.js

@@ -27,16 +27,12 @@ var cleanCSS = require('clean-css');
 var jsp = require("uglify-js").parser;
 var pro = require("uglify-js").uglify;
 var path = require('path');
-var Buffer = require('buffer').Buffer;
-var zlib = require('zlib');
 var RequireKernel = require('require-kernel');
 var server = require('../server');
-var os = require('os');
 
 var ROOT_DIR = path.normalize(__dirname + "/../" );
 var JS_DIR = ROOT_DIR + '../static/js/';
 var CSS_DIR = ROOT_DIR + '../static/css/';
-var CACHE_DIR = ROOT_DIR + '../var/';
 var TAR_PATH = path.join(__dirname, 'tar.json');
 var tar = JSON.parse(fs.readFileSync(TAR_PATH, 'utf8'));
 
@@ -57,160 +53,61 @@ exports.minifyJS = function(req, res, next)
   } else {
     // Not in tar list, but try anyways, if it fails, pass to `next`.
     jsFiles = [jsFilename];
-    fs.stat(JS_DIR + jsFilename, function (error, stats) {
+    _handle(req, res, jsFilename, jsFiles);
-      if (error || !stats.isFile()) {
-        next();
-      } else {
-        _handle(req, res, jsFilename, jsFiles);
-      }
-    });
   }
 }
 
 function _handle(req, res, jsFilename, jsFiles) {
   res.header("Content-Type","text/javascript");
-  
-  //minifying is enabled
-  if(settings.minify)
-  {
-    var result = undefined;
-    var latestModification = 0;
-    
-    async.series([
-      //find out the highest modification date
-      function(callback)
-      {        
-        var folders2check = [CSS_DIR, JS_DIR];
-        
-        //go trough this two folders
-        async.forEach(folders2check, function(path, callback)
-        {
-          //read the files in the folder
-          fs.readdir(path, function(err, files)
-          {
-            if(ERR(err, callback)) return;
-            
-            //we wanna check the directory itself for changes too
-            files.push(".");
-            
-            //go trough all files in this folder
-            async.forEach(files, function(filename, callback) 
-            {
-              //get the stat data of this file
-              fs.stat(path + "/" + filename, function(err, stats)
-              {
-                if(ERR(err, callback)) return;
-              
-                //get the modification time
-                var modificationTime = stats.mtime.getTime();
-              
-                //compare the modification time to the highest found
-                if(modificationTime > latestModification)
-                {
-                  latestModification = modificationTime;
-                }
-                
-                callback();
-              });
-            }, callback);
-          });
-        }, callback);
-      },
-      function(callback)
-      {
-        //check the modification time of the minified js
-        fs.stat(CACHE_DIR + "/minified_" + jsFilename, function(err, stats)
-        {
-          if(err && err.code != "ENOENT")
-          {
-            ERR(err, callback);
-            return;
-          }
-        
-          //there is no minfied file or there new changes since this file was generated, so continue generating this file
-          if((err && err.code == "ENOENT") || stats.mtime.getTime() < latestModification)
-          {
-            callback();
-          }
-          //the minified file is still up to date, stop minifying
-          else
-          {
-            callback("stop");
-          }
-        });
-      }, 
-      //load all js files
-      function (callback)
-      {
-        var values = [];
-        tarCode(
-          jsFiles
-        , function (content) {values.push(content)}
-        , function (err) {
-          if(ERR(err)) return;
 
-          result = values.join('');
+  lastModifiedDate(function (date) {
-          callback();
+    date = new Date(date);
-        });
+    res.setHeader('last-modified', date.toUTCString());
-      },
+    res.setHeader('date', (new Date()).toUTCString());
-      //put all together and write it into a file
+    if (server.maxAge) {
-      function(callback)
+      var expiresDate = new Date((new Date()).getTime() + server.maxAge*1000);
-      {
+      res.setHeader('expires', expiresDate.toUTCString());
-        async.parallel([
+      res.setHeader('cache-control', 'max-age=' + server.maxAge);
-          //write the results plain in a file
+    }
-          function(callback)
+
-          {
+    fs.stat(JS_DIR + jsFiles[0], function (error, stats) {
-            fs.writeFile(CACHE_DIR + "minified_" + jsFilename, result, "utf8", callback);
+      if (error) {
-          },
+        if (error.code == "ENOENT") {
-          //write the results compressed in a file
+          res.writeHead(404, {});
-          function(callback)
+          res.end();
-          {
+        } else {
-            zlib.gzip(result, function(err, compressedResult){
+          res.writeHead(500, {});
-              //weird gzip bug that returns 0 instead of null if everything is ok
+          res.end();
-              err = err === 0 ? null : err;
+        }
-            
+      } else if (!stats.isFile()) {
-              if(ERR(err, callback)) return;
+        res.writeHead(404, {});
-              
+        res.end();
-              fs.writeFile(CACHE_DIR + "minified_" + jsFilename + ".gz", compressedResult, callback);
+      } else if (new Date(req.headers['if-modified-since']) >= date) {
-            });
+        res.writeHead(304, {});
-          }
+        res.end();
-        ],callback);
+      } else {
+        if (req.method == 'HEAD') {
+          res.writeHead(200, {});
+          res.end();
+        } else if (req.method == 'GET') {
+          res.writeHead(200, {});
+          tarCode(
+            jsFiles
+          , function (content) {
+              res.write(content);
+            }
+          , function (err) {
+              if(ERR(err)) return;
+              res.end();
+            }
+          );
+        } else {
+          res.writeHead(405, {'allow': 'HEAD, GET'});
+          res.end();
+        }
       }
-    ], function(err)
-    {
-      if(err && err != "stop")
-      {
-        if(ERR(err)) return;
-      }
-      
-      //check if gzip is supported by this browser
-      var gzipSupport = req.header('Accept-Encoding', '').indexOf('gzip') != -1;
-      
-      var pathStr;
-      if(gzipSupport && os.type().indexOf("Windows") == -1)
-      {
-        pathStr = path.normalize(CACHE_DIR + "minified_" + jsFilename + ".gz");
-        res.header('Content-Encoding', 'gzip');
-      }
-      else
-      {
-        pathStr = path.normalize(CACHE_DIR + "minified_" + jsFilename );
-      }
-      
-      res.sendfile(pathStr, { maxAge: server.maxAge });
-    })
-  }
-  //minifying is disabled, so put the files together in one file
-  else
-  {
-    tarCode(
-      jsFiles
-    , function (content) {res.write(content)}
-    , function (err) {
-      if(ERR(err)) return;
-      res.end();
     });
-  }
+  });
 }
 
 // find all includes in ace.js and embed them.
@@ -232,7 +129,8 @@ function getAceFile(callback) {
     async.forEach(founds, function (item, callback) {
       var filename = item.match(/"([^"]*)"/)[1];
       var type = item.match(/INCLUDE_([A-Z]+)/)[1];
-      var shortFilename = (filename.match(/^..\/static\/js\/(.*)$/, '')||[])[1];
+      var shortFilename =
+          (filename.match(/^\.\.\/static\/js\/(.*)$/, '') || [])[1];
 
       //read the included files
       if (shortFilename) {
@@ -279,6 +177,46 @@ function getAceFile(callback) {
   });
 }
 
+function lastModifiedDate(callback) {
+  var folders2check = [CSS_DIR, JS_DIR];
+  var latestModification = 0;
+  //go trough this two folders
+  async.forEach(folders2check, function(path, callback)
+  {
+    //read the files in the folder
+    fs.readdir(path, function(err, files)
+    {
+      if(ERR(err, callback)) return;
+
+      //we wanna check the directory itself for changes too
+      files.push(".");
+
+      //go trough all files in this folder
+      async.forEach(files, function(filename, callback)
+      {
+        //get the stat data of this file
+        fs.stat(path + "/" + filename, function(err, stats)
+        {
+          if(ERR(err, callback)) return;
+
+          //get the modification time
+          var modificationTime = stats.mtime.getTime();
+
+          //compare the modification time to the highest found
+          if(modificationTime > latestModification)
+          {
+            latestModification = modificationTime;
+          }
+
+          callback();
+        });
+      }, callback);
+    });
+  }, function () {
+    callback(latestModification);
+  });
+}
+
 exports.requireDefinition = requireDefinition;
 function requireDefinition() {
   return 'var require = ' + RequireKernel.kernelSource + ';\n';

node/utils/Settings.js

@@ -56,6 +56,11 @@ exports.requireSession = false;
  */
 exports.editOnly = false;
 
+/**
+ * Max age that responses will have (affects caching layer).
+ */
+exports.maxAge = 1000*60*60*6; // 6 hours
+
 /**
  * A flag that shows if minification is enabled or not
  */

node/utils/caching_middleware.js

@@ -0,0 +1,175 @@
+/*
+ * 2011 Peter 'Pita' Martischka (Primary Technology Ltd)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var async = require('async');
+var Buffer = require('buffer').Buffer;
+var fs = require('fs');
+var path = require('path');
+var server = require('../server');
+var zlib = require('zlib');
+var util = require('util');
+
+var ROOT_DIR = path.normalize(__dirname + "/../");
+var CACHE_DIR = ROOT_DIR + '../var/';
+
+var responseCache = {};
+
+/*
+  This caches and compresses 200 and 404 responses to GET and HEAD requests.
+  TODO: Caching and compressing are solved problems, a middleware configuration
+  should replace this.
+*/
+
+function CachingMiddleware() {
+}
+CachingMiddleware.prototype = new function () {
+  function handle(req, res, next) {
+    if (!(req.method == "GET" || req.method == "HEAD")) {
+      return next(undefined, req, res);
+    }
+
+    var old_req = {};
+    var old_res = {};
+
+    var supportsGzip =
+        req.header('Accept-Encoding', '').indexOf('gzip') != -1;
+
+    var path = require('url').parse(req.url).path;
+    var cacheKey = (new Buffer(path)).toString('base64').replace(/[\/\+=]/g, '');
+
+    fs.stat(CACHE_DIR + 'minified_' + cacheKey, function (error, stats) {
+      var modifiedSince = (req.headers['if-modified-since']
+          && new Date(req.headers['if-modified-since']));
+      var lastModifiedCache = stats && stats.mtime;
+      if (lastModifiedCache) {
+        req.headers['if-modified-since'] = lastModifiedCache.toUTCString();
+      } else {
+        delete req.headers['if-modified-since'];
+      }
+
+      // Always issue get to downstream.
+      old_req.method = req.method;
+      req.method = 'GET';
+
+      var expirationDate = new Date(((responseCache[cacheKey] || {}).headers || {})['expires']);
+      if (expirationDate > new Date()) {
+        // Our cached version is still valid.
+        return respond();
+      }
+
+      var _headers = {};
+      old_res.setHeader = res.setHeader;
+      res.setHeader = function (key, value) {
+        _headers[key.toLowerCase()] = value;
+        old_res.setHeader.call(res, key, value);
+      };
+
+      old_res.writeHead = res.writeHead;
+      res.writeHead = function (status, headers) {
+        var lastModified = (res.getHeader('last-modified')
+            && new Date(res.getHeader('last-modified')));
+
+        res.writeHead = old_res.writeHead;
+        if (status == 200 || status == 404) {
+          // Update cache
+          var buffer = '';
+
+          Object.keys(headers).forEach(function (key) {
+            res.setHeader(key, headers[key]);
+          });
+          headers = _headers;
+          responseCache[cacheKey] = {statusCode: status, headers: headers};
+
+          old_res.write = res.write;
+          old_res.end = res.end;
+          res.write = function(data, encoding) {
+            buffer += data.toString(encoding);
+          };
+          res.end = function(data, encoding) {
+            async.parallel([
+              function (callback) {
+                var path = CACHE_DIR + 'minified_' + cacheKey;
+                fs.writeFile(path, buffer, function (error, stats) {
+                  callback();
+                });
+              }
+            , function (callback) {
+                var path = CACHE_DIR + 'minified_' + cacheKey + '.gz';
+                zlib.gzip(buffer, function(error, content) {
+                  if (error) {
+                    callback();
+                  } else {
+                    fs.writeFile(path, content, function (error, stats) {
+                      callback();
+                    });
+                  }
+                });
+              }
+            ], respond);
+          };
+        } else if (status == 304) {
+          // Nothing new changed from the cached version.
+          old_res.write = res.write;
+          old_res.end = res.end;
+          res.write = function(data, encoding) {};
+          res.end = function(data, encoding) { respond() };
+        } else {
+          res.writeHead(status, headers);
+        }
+      };
+
+      next(undefined, req, res);
+
+      // This handles read/write synchronization as well as its predecessor,
+      // which is to say, not at all.
+      // TODO: Implement locking on write or ditch caching of gzip and use
+      // existing middlewares.
+      function respond() {
+        req.method = old_req.method || req.method;
+        res.write = old_res.write || res.write;
+        res.end = old_res.end || res.end;
+
+        var headers = responseCache[cacheKey].headers;
+        var statusCode = responseCache[cacheKey].statusCode;
+
+        var pathStr = CACHE_DIR + 'minified_' + cacheKey;
+        if (supportsGzip) {
+          pathStr = pathStr + '.gz';
+          headers['content-encoding'] = 'gzip';
+        }
+
+        var lastModified = (headers['last-modified']
+            && new Date(headers['last-modified']));
+
+        if (statusCode == 200 && lastModified <= modifiedSince) {
+          res.writeHead(304, headers);
+          res.end();
+        } else if (req.method == 'GET') {
+          var readStream = fs.createReadStream(pathStr);
+          res.writeHead(statusCode, headers);
+          util.pump(readStream, res);
+        } else {
+          res.writeHead(200, headers);
+          res.end();
+        }
+      }
+    });
+  }
+
+  this.handle = handle;
+}();
+
+module.exports = CachingMiddleware;

settings.json.template

@@ -38,6 +38,10 @@
   /* if true, all css & js will be minified before sending to the client. This will improve the loading performance massivly, 
      but makes it impossible to debug the javascript/css */
   "minify" : true,
+
+  /* How long may clients use served javascript code? Without versioning this
+    is may cause problems during deployment. */
+  "maxAge" : 1000*60*60*6, // 6 hours
   
   /* This is the path to the Abiword executable. Setting it to null, disables abiword.
      Abiword is needed to enable the import/export of pads*/