mirrors/etherpad-lite
1
0
Fork 0
mirror of https://github.com/ether/etherpad-lite.git synced 2025-06-15 18:54:45 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Feat/frontend vitest (#6469)

Browse source 
* Added vitest tests.

* Added Settings tests to vitest - not working

* Added attributes and attributemap to vitest.

* Added more tests.

* Also run the vitest tests.

* Also run withoutPlugins

* Fixed pnpm lock
This commit is contained in:
SamTV12345 2024-08-16 22:55:42 +02:00 • committed by GitHub
parent babfaab4df
commit c7a2dea4d1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 1092 additions and 552 deletions
Download patch file Download diff file Expand all files Collapse all files

12
src/node/hooks/express/static.ts
View file

@ -4,11 +4,10 @@ import {MapArrayType} from "../../types/MapType";
import {PartType} from "../../types/PartType";
const fs = require('fs').promises;
const minify = require('../../utils/Minify');
const path = require('path');
import {minify} from '../../utils/Minify';
import path from 'node:path';
const plugins = require('../../../static/js/pluginfw/plugin_defs');
const settings = require('../../utils/Settings');
import CachingMiddleware from '../../utils/caching_middleware';
// Rewrite tar to include modules with no extensions and proper rooted paths.
const getTar = async () => {
@ -32,15 +31,10 @@ const getTar = async () => {
};
exports.expressPreSession = async (hookName:string, {app}:any) => {
// Cache both minified and static.
const assetCache = new CachingMiddleware();
// Cache static assets
app.all(/\/js\/(.*)/, assetCache.handle.bind(assetCache));
app.all(/\/css\/(.*)/, assetCache.handle.bind(assetCache));
// Minify will serve static files compressed (minify enabled). It also has
// file-specific hacks for ace/require-kernel/etc.
app.all('/static/:filename(*)', minify.minify);
app.all('/static/:filename(*)', minify);
// serve plugin definitions
// not very static, but served here so that client can do

1
src/node/types/PadType.ts
View file

@ -35,6 +35,7 @@ export type APool = {
clone: ()=>APool,
check: ()=>Promise<void>,
eachAttrib: (callback: (key: string, value: any)=>void)=>void,
getAttrib: (key: number)=>any,
}

142
src/node/utils/Minify.js → src/node/utils/Minify.ts
View file

@ -21,20 +21,20 @@
* limitations under the License.
*/
const settings = require('./Settings');
const fs = require('fs').promises;
const path = require('path');
const plugins = require('../../static/js/pluginfw/plugin_defs');
const mime = require('mime-types');
const Threads = require('threads');
const log4js = require('log4js');
const sanitizePathname = require('./sanitizePathname');
import {TransformResult} from "esbuild";
import mime from 'mime-types';
import log4js from 'log4js';
import {compressCSS, compressJS} from './MinifyWorker'
const settings = require('./Settings');
import {promises as fs} from 'fs';
import path from 'node:path';
const plugins = require('../../static/js/pluginfw/plugin_defs');
import sanitizePathname from './sanitizePathname';
const logger = log4js.getLogger('Minify');
const ROOT_DIR = path.join(settings.root, 'src/static/');
const threadsPool = new Threads.Pool(() => Threads.spawn(new Threads.Worker('./MinifyWorker')), 2);
const LIBRARY_WHITELIST = [
'async',
@ -48,10 +48,10 @@ const LIBRARY_WHITELIST = [
// What follows is a terrible hack to avoid loop-back within the server.
// TODO: Serve files from another service, or directly from the file system.
const requestURI = async (url, method, headers) => {
const requestURI = async (url: string | URL, method: any, headers: { [x: string]: any; }) => {
const parsedUrl = new URL(url);
let status = 500;
const content = [];
const content: any[] = [];
const mockRequest = {
url,
method,
@ -61,7 +61,7 @@ const requestURI = async (url, method, headers) => {
let mockResponse;
const p = new Promise((resolve) => {
mockResponse = {
writeHead: (_status, _headers) => {
writeHead: (_status: number, _headers: { [x: string]: any; }) => {
status = _status;
for (const header in _headers) {
if (Object.prototype.hasOwnProperty.call(_headers, header)) {
@ -69,37 +69,63 @@ const requestURI = async (url, method, headers) => {
}
}
},
setHeader: (header, value) => {
setHeader: (header: string, value: { toString: () => any; }) => {
headers[header.toLowerCase()] = value.toString();
},
header: (header, value) => {
header: (header: string, value: { toString: () => any; }) => {
headers[header.toLowerCase()] = value.toString();
},
write: (_content) => {
write: (_content: any) => {
_content && content.push(_content);
},
end: (_content) => {
end: (_content: any) => {
_content && content.push(_content);
resolve([status, headers, content.join('')]);
},
};
});
await minify(mockRequest, mockResponse);
await _minify(mockRequest, mockResponse);
return await p;
};
const requestURIs = (locations, method, headers, callback) => {
const _requestURIs = (locations: any[], method: any, headers: {
[x: string]:
/**
* This Module manages all /minified/* requests. It controls the
* minification && compression of Javascript and CSS.
*/
/*
* 2011 Peter 'Pita' Martischka (Primary Technology Ltd)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS-IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
any;
}, callback: (arg0: any[], arg1: any[], arg2: any[]) => void) => {
Promise.all(locations.map(async (loc) => {
try {
return await requestURI(loc, method, headers);
} catch (err) {
logger.debug(`requestURI(${JSON.stringify(loc)}, ${JSON.stringify(method)}, ` +
`${JSON.stringify(headers)}) failed: ${err.stack || err}`);
// @ts-ignore
`${JSON.stringify(headers)}) failed: ${err.stack || err}`);
return [500, headers, ''];
}
})).then((responses) => {
// @ts-ignore
const statuss = responses.map((x) => x[0]);
// @ts-ignore
const headerss = responses.map((x) => x[1]);
// @ts-ignore
const contentss = responses.map((x) => x[2]);
callback(statuss, headerss, contentss);
});
@ -119,11 +145,12 @@ const compatPaths = {
* @param req the Express request
* @param res the Express response
*/
const minify = async (req, res) => {
const _minify = async (req:any, res:any) => {
let filename = req.params.filename;
try {
filename = sanitizePathname(filename);
} catch (err) {
// @ts-ignore
logger.error(`sanitization of pathname "${filename}" failed: ${err.stack || err}`);
res.writeHead(404, {});
res.end();
@ -131,6 +158,7 @@ const minify = async (req, res) => {
}
// Backward compatibility for plugins that require() files from old paths.
// @ts-ignore
const newLocation = compatPaths[filename.replace(/^plugins\/ep_etherpad-lite\/static\//, '')];
if (newLocation != null) {
logger.warn(`request for deprecated path "${filename}", replacing with "${newLocation}"`);
@ -193,7 +221,7 @@ const minify = async (req, res) => {
res.writeHead(200, {});
res.end();
} else if (req.method === 'GET') {
const content = await getFileCompressed(filename, contentType);
const content = await getFileCompressed(filename, contentType as string);
res.header('Content-Type', contentType);
res.writeHead(200, {});
res.write(content);
@ -205,7 +233,7 @@ const minify = async (req, res) => {
};
// Check for the existance of the file and get the last modification date.
const statFile = async (filename, dirStatLimit) => {
const statFile = async (filename: string, dirStatLimit: number):Promise<(any | boolean)[]> => {
/*
* The only external call to this function provides an explicit value for
* dirStatLimit: this check could be removed.
@ -221,6 +249,7 @@ const statFile = async (filename, dirStatLimit) => {
try {
stats = await fs.stat(path.resolve(ROOT_DIR, filename));
} catch (err) {
// @ts-ignore
if (['ENOENT', 'ENOTDIR'].includes(err.code)) {
// Stat the directory instead.
const [date] = await statFile(path.dirname(filename), dirStatLimit - 1);
@ -234,69 +263,64 @@ const statFile = async (filename, dirStatLimit) => {
let contentCache = new Map();
const getFileCompressed = async (filename, contentType) => {
const getFileCompressed = async (filename: any, contentType: string) => {
if (contentCache.has(filename)) {
return contentCache.get(filename);
}
let content = await getFile(filename);
let content: Buffer|string = await getFile(filename);
if (!content || !settings.minify) {
return content;
} else if (contentType === 'application/javascript') {
return await new Promise((resolve) => {
threadsPool.queue(async ({compressJS}) => {
return await new Promise(async (resolve) => {
try {
logger.info('Compress JS file %s.', filename);
content = content.toString();
try {
logger.info('Compress JS file %s.', filename);
content = content.toString();
const compressResult = await compressJS(content);
if (compressResult.error) {
console.error(`Error compressing JS (${filename}) using terser`, compressResult.error);
} else {
content = compressResult.code.toString(); // Convert content obj code to string
}
let compressResult: TransformResult<{ minify: boolean }>
compressResult = await compressJS(content);
content = compressResult.code.toString(); // Convert content obj code to string
} catch (error) {
console.error('getFile() returned an error in ' +
`getFileCompressed(${filename}, ${contentType}): ${error}`);
console.error(`Error compressing JS (${filename}) using esbuild`, error);
}
contentCache.set(filename, content);
resolve(content);
});
} catch (error) {
console.error('getFile() returned an error in ' +
`getFileCompressed(${filename}, ${contentType}): ${error}`);
}
contentCache.set(filename, content);
resolve(content);
});
} else if (contentType === 'text/css') {
return await new Promise((resolve) => {
threadsPool.queue(async ({compressCSS}) => {
return await new Promise(async (resolve) => {
try {
logger.info('Compress CSS file %s.', filename);
try {
logger.info('Compress CSS file %s.', filename);
const compressResult = await compressCSS(path.resolve(ROOT_DIR,filename));
if (compressResult.error) {
console.error(`Error compressing CSS (${filename}) using terser`, compressResult.error);
} else {
content = compressResult
}
content = await compressCSS(path.resolve(ROOT_DIR, filename));
} catch (error) {
console.error(`CleanCSS.minify() returned an error on ${filename}: ${error}`);
}
contentCache.set(filename, content);
resolve(content);
});
});
} catch (e) {
console.error('getFile() returned an error in ' +
`getFileCompressed(${filename}, ${contentType}): ${e}`);
}
})
} else {
contentCache.set(filename, content);
return content;
}
};
const getFile = async (filename) => {
const getFile = async (filename: any) => {
return await fs.readFile(path.resolve(ROOT_DIR, filename));
};
exports.minify = (req, res, next) => minify(req, res).catch((err) => next(err || new Error(err)));
export const minify = (req:any, res:any, next:Function) => _minify(req, res).catch((err) => next(err || new Error(err)));
exports.requestURIs = requestURIs;
export const requestURIs = _requestURIs;
exports.shutdown = async (hookName, context) => {
await threadsPool.terminate();
export const shutdown = async (hookName: string, context:any) => {
contentCache = new Map();
};

10
src/node/utils/MinifyWorker.js → src/node/utils/MinifyWorker.ts
View file

@ -3,14 +3,13 @@
* Worker thread to minify JS & CSS files out of the main NodeJS thread
*/
import {expose} from 'threads'
import {build, transform} from 'esbuild';
/*
* Minify JS content
* @param {string} content - JS content to minify
*/
const compressJS = async (content) => {
export const compressJS = async (content: string) => {
return await transform(content, {minify: true});
}
@ -19,7 +18,7 @@ const compressJS = async (content) => {
* @param {string} filename - name of the file
* @param {string} ROOT_DIR - the root dir of Etherpad
*/
const compressCSS = async (content) => {
export const compressCSS = async (content: string) => {
const transformedCSS = await build(
{
entryPoints: [content],
@ -41,8 +40,3 @@ const compressCSS = async (content) => {
)
return transformedCSS.outputFiles[0].text
};
expose({
compressJS: compressJS,
compressCSS,
});

211
src/node/utils/caching_middleware.ts
View file

@ -1,211 +0,0 @@
'use strict';
/*
* 2011 Peter 'Pita' Martischka (Primary Technology Ltd)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS-IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {Buffer} from 'node:buffer'
import fs from 'fs';
const fsp = fs.promises;
import path from 'path';
import zlib from 'zlib';
const settings = require('./Settings');
const existsSync = require('./path_exists');
import util from 'util';
/*
* The crypto module can be absent on reduced node installations.
*
* Here we copy the approach TypeScript guys used for https://github.com/microsoft/TypeScript/issues/19100
* If importing crypto fails at runtime, we replace sha256 with djb2, which is
* weaker, but works for our case.
*
* djb2 was written in 1991 by Daniel J. Bernstein.
*
*/
import _crypto from 'crypto';
let CACHE_DIR: string|undefined = path.join(settings.root, 'var/');
CACHE_DIR = existsSync(CACHE_DIR) ? CACHE_DIR : undefined;
type Headers = {
[id: string]: string
}
type ResponseCache = {
[id: string]: {
statusCode: number
headers: Headers
}
}
const responseCache: ResponseCache = {};
const djb2Hash = (data: string) => {
const chars = data.split('').map((str) => str.charCodeAt(0));
return `${chars.reduce((prev, curr) => ((prev << 5) + prev) + curr, 5381)}`;
};
const generateCacheKeyWithSha256 =
(path: string) => _crypto.createHash('sha256').update(path).digest('hex');
const generateCacheKeyWithDjb2 =
(path: string) => Buffer.from(djb2Hash(path)).toString('hex');
let generateCacheKey: (path: string)=>string;
if (_crypto) {
generateCacheKey = generateCacheKeyWithSha256;
} else {
generateCacheKey = generateCacheKeyWithDjb2;
console.warn('No crypto support in this nodejs runtime. Djb2 (weaker) will be used.');
}
// MIMIC https://github.com/microsoft/TypeScript/commit/9677b0641cc5ba7d8b701b4f892ed7e54ceaee9a - END
/*
This caches and compresses 200 and 404 responses to GET and HEAD requests.
TODO: Caching and compressing are solved problems, a middleware configuration
should replace this.
*/
export default class CachingMiddleware {
handle(req: any, res: any, next: any) {
this._handle(req, res, next).catch((err) => next(err || new Error(err)));
}
async _handle(req: any, res: any, next: any) {
if (!(req.method === 'GET' || req.method === 'HEAD') || !CACHE_DIR) {
return next(undefined, req, res);
}
const oldReq:ResponseCache = {};
const oldRes:ResponseCache = {};
const supportsGzip =
(req.get('Accept-Encoding') || '').indexOf('gzip') !== -1;
const url = new URL(req.url, 'http://localhost');
const cacheKey = generateCacheKey(url.pathname + url.search);
const stats = await fsp.stat(`${CACHE_DIR}minified_${cacheKey}`).catch(() => {});
const modifiedSince =
req.headers['if-modified-since'] && new Date(req.headers['if-modified-since']);
if (stats != null && stats.mtime && responseCache[cacheKey]) {
req.headers['if-modified-since'] = stats.mtime.toUTCString();
} else {
delete req.headers['if-modified-since'];
}
// Always issue get to downstream.
oldReq.method = req.method;
req.method = 'GET';
// This handles read/write synchronization as well as its predecessor,
// which is to say, not at all.
// TODO: Implement locking on write or ditch caching of gzip and use
// existing middlewares.
const respond = () => {
req.method = oldReq.method || req.method;
res.write = oldRes.write || res.write;
res.end = oldRes.end || res.end;
const headers: Headers = {};
Object.assign(headers, (responseCache[cacheKey].headers || {}));
const statusCode = responseCache[cacheKey].statusCode;
let pathStr = `${CACHE_DIR}minified_${cacheKey}`;
if (supportsGzip && /application\/javascript/.test(headers['content-type'])) {
pathStr += '.gz';
headers['content-encoding'] = 'gzip';
}
const lastModified = headers['last-modified'] && new Date(headers['last-modified']);
if (statusCode === 200 && lastModified <= modifiedSince) {
res.writeHead(304, headers);
res.end();
} else if (req.method === 'GET') {
const readStream = fs.createReadStream(pathStr);
res.writeHead(statusCode, headers);
readStream.pipe(res);
} else {
res.writeHead(statusCode, headers);
res.end();
}
};
const expirationDate = new Date(((responseCache[cacheKey] || {}).headers || {}).expires);
if (expirationDate > new Date()) {
// Our cached version is still valid.
return respond();
}
const _headers:Headers = {};
oldRes.setHeader = res.setHeader;
res.setHeader = (key: string, value: string) => {
// Don't set cookies, see issue #707
if (key.toLowerCase() === 'set-cookie') return;
_headers[key.toLowerCase()] = value;
// @ts-ignore
oldRes.setHeader.call(res, key, value);
};
oldRes.writeHead = res.writeHead;
res.writeHead = (status: number, headers: Headers) => {
res.writeHead = oldRes.writeHead;
if (status === 200) {
// Update cache
let buffer = '';
Object.keys(headers || {}).forEach((key) => {
res.setHeader(key, headers[key]);
});
headers = _headers;
oldRes.write = res.write;
oldRes.end = res.end;
res.write = (data: number, encoding: number) => {
buffer += data.toString(encoding);
};
res.end = async (data: number, encoding: number) => {
await Promise.all([
fsp.writeFile(`${CACHE_DIR}minified_${cacheKey}`, buffer).catch(() => {}),
util.promisify(zlib.gzip)(buffer)
// @ts-ignore
.then((content: string) => fsp.writeFile(`${CACHE_DIR}minified_${cacheKey}.gz`, content))
.catch(() => {}),
]);
responseCache[cacheKey] = {statusCode: status, headers};
respond();
};
} else if (status === 304) {
// Nothing new changed from the cached version.
oldRes.write = res.write;
oldRes.end = res.end;
res.write = (data: number, encoding: number) => {};
res.end = (data: number, encoding: number) => { respond(); };
} else {
res.writeHead(status, headers);
}
};
next(undefined, req, res);
}
};

8
src/node/utils/sanitizePathname.ts
View file

@ -1,10 +1,8 @@
'use strict';
const path = require('path');
import path from 'path';
// Normalizes p and ensures that it is a relative path that does not reach outside. See
// https://nvd.nist.gov/vuln/detail/CVE-2015-3297 for additional context.
module.exports = (p: string, pathApi = path) => {
const sanitizeRoot = (p: string, pathApi = path) => {
// The documentation for path.normalize() says that it resolves '..' and '.' segments. The word
// "resolve" implies that it examines the filesystem to resolve symbolic links, so 'a/../b' might
// not be the same thing as 'b'. Most path normalization functions from other libraries (e.g.,
@ -21,3 +19,5 @@ module.exports = (p: string, pathApi = path) => {
if (pathApi.sep === '\\') p = p.replace(/\\/g, '/');
return p;
};
export default sanitizeRoot