From 09a4753dcc8c431c2570ea2b5f142ccf506f6df9 Mon Sep 17 00:00:00 2001 From: Richard Bown Date: Fri, 5 Oct 2018 11:07:57 +0200 Subject: [PATCH 01/16] Added notes about how to configure hosts file for docker after creating new user --- README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 268f61cb1..a02fc2bbe 100644 --- a/README.md +++ b/README.md @@ -228,14 +228,21 @@ Run with: docker run -p 8443:8443 --name solid node-solid-server ``` -Modify the config as follows: +This will enable you to login to solid on https://localhost:8443 and then create a new account +but not yet use that account. After a new account is made you will need to create an entry for +it in your local (/etc/)hosts file in line with the account and subdomain i.e. + +127.0.0.1 newsoliduser.localhost + +Then you'll be able to use solid as intended. + +You can modify the config within the docker container as follows: - Copy the config to the current directory with: `docker cp solid:/usr/src/app/config.json .` - Edit the `config.json` file - Copy the file back with `docker cp config.json solid:/usr/src/app/` - Restart the server with `docker restart solid` - ## Library Usage ### Install Dependencies From c4dc0180024c7f064b13468177c03ffe79f5f567 Mon Sep 17 00:00:00 2001 From: Arne Hassel Date: Fri, 16 Nov 2018 01:30:05 +0100 Subject: [PATCH 02/16] A simple start on the updateindex command Trying to reuse methods from invalidUsernames command --- bin/lib/cli.js | 2 ++ bin/lib/common.js | 34 +++++++++++++++++++++++++++++++++- bin/lib/invalidUsernames.js | 25 ++++--------------------- bin/lib/updateIndex.js | 20 ++++++++++++++++++++ 4 files changed, 59 insertions(+), 22 deletions(-) create mode 100644 bin/lib/updateIndex.js diff --git a/bin/lib/cli.js b/bin/lib/cli.js index 85dcb789a..889903bed 100644 --- a/bin/lib/cli.js +++ b/bin/lib/cli.js @@ -3,6 +3,7 @@ const loadInit = require('./init') const loadStart = require('./start') const loadInvalidUsernames = require('./invalidUsernames') const loadMigrateLegacyResources = require('./migrateLegacyResources') +const loadUpdateIndex = require('./updateIndex') const { spawnSync } = require('child_process') const path = require('path') @@ -13,6 +14,7 @@ module.exports = function startCli (server) { loadStart(program, server) loadInvalidUsernames(program) loadMigrateLegacyResources(program) + loadUpdateIndex(program) program.parse(process.argv) if (program.args.length === 0) program.help() diff --git a/bin/lib/common.js b/bin/lib/common.js index 80a0f6e58..433cc56de 100644 --- a/bin/lib/common.js +++ b/bin/lib/common.js @@ -1,8 +1,31 @@ -const fs = require('fs') +const fs = require('fs-extra') const extend = require('extend') const { cyan, bold } = require('colorette') +const { URL } = require('url') +const LDP = require('../../lib/ldp') +const AccountManager = require('../../lib/models/account-manager') +const SolidHost = require('../../lib/models/solid-host') +module.exports.getAccountManager = getAccountManager module.exports.loadConfig = loadConfig +module.exports.loadUsernames = loadUsernames + +/** + * Returns an instance of AccountManager + * + * @param {Object} config + * @param {Object} [options] + * @returns {AccountManager} + */ +function getAccountManager (config, options = {}) { + const ldp = new LDP(config) + const host = options.host || SolidHost.from({ port: config.port, serverUri: config.serverUri }) + return AccountManager.from({ + host, + store: ldp, + multiuser: config.multiuser + }) +} function loadConfig (program, options) { let argv = extend({}, options, { version: program.version() }) @@ -23,3 +46,12 @@ function loadConfig (program, options) { return argv } + +function loadUsernames (config) { + const files = fs.readdirSync(config.root) + const hostname = new URL(config.serverUri).hostname + const isUserDirectory = new RegExp(`.${hostname}$`) + return files + .filter(file => isUserDirectory.test(file)) + .map(userDirectory => userDirectory.substr(0, userDirectory.length - hostname.length - 1)) +} diff --git a/bin/lib/invalidUsernames.js b/bin/lib/invalidUsernames.js index f2b7493e6..d2a2aef27 100644 --- a/bin/lib/invalidUsernames.js +++ b/bin/lib/invalidUsernames.js @@ -1,17 +1,14 @@ const fs = require('fs-extra') const Handlebars = require('handlebars') const path = require('path') -const { URL } = require('url') -const { loadConfig } = require('./common') +const { getAccountManager, loadConfig, loadUsernames } = require('./common') const { isValidUsername } = require('../../lib/common/user-utils') const blacklistService = require('../../lib/services/blacklist-service') const { initConfigDir, initTemplateDirs } = require('../../lib/server-config') const { fromServerConfig } = require('../../lib/models/oidc-manager') -const AccountManager = require('../../lib/models/account-manager') const EmailService = require('../../lib/services/email-service') -const LDP = require('../../lib/ldp') const SolidHost = require('../../lib/models/solid-host') module.exports = function (program) { @@ -28,7 +25,7 @@ module.exports = function (program) { const invalidUsernames = getInvalidUsernames(config) const host = SolidHost.from({ port: config.port, serverUri: config.serverUri }) - const accountManager = getAccountManager(config, host) + const accountManager = getAccountManager(config, { host }) if (options.notify) { return notifyUsers(invalidUsernames, accountManager, config) @@ -96,23 +93,9 @@ async function deleteUsers (usernames, accountManager, config, host) { console.info(`Deleted ${deletingUsers.length} users succeeded`) } -function getAccountManager (config, host) { - const ldp = new LDP(config) - return AccountManager.from({ - host, - store: ldp, - multiuser: config.multiuser - }) -} - function getInvalidUsernames (config) { - const files = fs.readdirSync(config.root) - const hostname = new URL(config.serverUri).hostname - const isUserDirectory = new RegExp(`.${hostname}$`) - return files - .filter(file => isUserDirectory.test(file)) - .map(userDirectory => userDirectory.substr(0, userDirectory.length - hostname.length - 1)) - .filter(username => !isValidUsername(username) || !blacklistService.validate(username)) + const usernames = loadUsernames(config) + return usernames.filter(username => !isValidUsername(username) || !blacklistService.validate(username)) } function listUsernames (usernames) { diff --git a/bin/lib/updateIndex.js b/bin/lib/updateIndex.js new file mode 100644 index 000000000..5409282b3 --- /dev/null +++ b/bin/lib/updateIndex.js @@ -0,0 +1,20 @@ +const { getAccountManager, loadConfig, loadUsernames } = require('./common') +const path = require('path') + +module.exports = function (program) { + program + .command('updateindex') + .description('Update index.html in root of all PODs that haven\'t been marked otherwise') + .action((options) => { + const config = loadConfig(program, options) + const usernames = loadUsernames(config) + usernames.forEach(username => updateIndex(username, config)) + }) +} + +function updateIndex (username, config) { + const accountManager = getAccountManager(config) + const userDirectory = accountManager.accountDirFor(username) + const indexFile = path.join(userDirectory, 'index.html') + console.log(indexFile) +} From 17717aa6237a7ce9789b436ce27f70909013808b Mon Sep 17 00:00:00 2001 From: Arne Hassel Date: Fri, 16 Nov 2018 14:09:32 +0100 Subject: [PATCH 03/16] A somewhat brute force solution Will improve on it to make it easier to read and some functionality more reusable --- bin/lib/common.js | 2 +- bin/lib/updateIndex.js | 51 ++++++++++--- package-lock.json | 166 +++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 4 files changed, 210 insertions(+), 10 deletions(-) diff --git a/bin/lib/common.js b/bin/lib/common.js index 433cc56de..b193622e1 100644 --- a/bin/lib/common.js +++ b/bin/lib/common.js @@ -18,7 +18,7 @@ module.exports.loadUsernames = loadUsernames * @returns {AccountManager} */ function getAccountManager (config, options = {}) { - const ldp = new LDP(config) + const ldp = options.ldp || new LDP(config) const host = options.host || SolidHost.from({ port: config.port, serverUri: config.serverUri }) return AccountManager.from({ host, diff --git a/bin/lib/updateIndex.js b/bin/lib/updateIndex.js index 5409282b3..325005885 100644 --- a/bin/lib/updateIndex.js +++ b/bin/lib/updateIndex.js @@ -1,20 +1,53 @@ -const { getAccountManager, loadConfig, loadUsernames } = require('./common') +const fs = require('fs') const path = require('path') +const cheerio = require('cheerio') +const Handlebars = require('handlebars') +const LDP = require('../../lib/ldp') +const $rdf = require('rdflib') +const { URL } = require('url') + +const { getAccountManager, loadConfig, loadUsernames } = require('./common') +const { initConfigDir, initTemplateDirs } = require('../../lib/server-config') + +const SOLID = $rdf.Namespace('http://www.w3.org/ns/solid/terms#') module.exports = function (program) { program .command('updateindex') .description('Update index.html in root of all PODs that haven\'t been marked otherwise') - .action((options) => { + .action(async (options) => { const config = loadConfig(program, options) + const ldp = new LDP(config) + const configPath = initConfigDir(config) + const templates = initTemplateDirs(configPath) + const accountManager = getAccountManager(config, { ldp }) const usernames = loadUsernames(config) - usernames.forEach(username => updateIndex(username, config)) + const indexTemplatePath = path.join(templates.account, 'index.html') + const indexTemplateSource = fs.readFileSync(indexTemplatePath, 'utf-8') + const indexTemplate = Handlebars.compile(indexTemplateSource) + const usersProcessed = usernames.map(async name => { + const userDirectory = accountManager.accountDirFor(name) + const indexFilePath = path.join(userDirectory, 'index.html') + const indexSource = fs.readFileSync(indexFilePath, 'utf-8') + const $ = cheerio.load(indexSource) + const allowAutomaticUpdateValue = $('meta[name="solid-allow-automatic-updates"]').prop('content') + const allowAutomaticUpdate = !allowAutomaticUpdateValue || allowAutomaticUpdateValue === 'true' + if (!allowAutomaticUpdate) { + return + } + const serverUrl = new URL(config.serverUri) + const accountUrl = `${serverUrl.protocol}//${name}.${serverUrl.host}/` + const metaFileUri = `${accountUrl}/${ldp.suffixMeta}` + const metaData = await ldp.readContainerMeta(userDirectory) + const metaGraph = $rdf.graph() + $rdf.parse(metaData, metaGraph, metaFileUri, 'text/turtle') + const webIdNode = metaGraph.any(undefined, SOLID('account'), $rdf.sym(accountUrl)) + const webId = webIdNode.value + const newIndexSource = indexTemplate({ name, webId }) + fs.writeFileSync(indexFilePath, newIndexSource, 'utf-8') + }) + await Promise.all(usersProcessed) + console.log(`Processed ${usersProcessed.length} users`) }) } -function updateIndex (username, config) { - const accountManager = getAccountManager(config) - const userDirectory = accountManager.accountDirFor(username) - const indexFile = path.join(userDirectory, 'index.html') - console.log(indexFile) -} diff --git a/package-lock.json b/package-lock.json index 48aed24e6..90699e6d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -370,6 +370,11 @@ "text-encoding": "^0.6.1" } }, + "@types/node": { + "version": "10.12.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.9.tgz", + "integrity": "sha512-eajkMXG812/w3w4a1OcBlaTwsFPO5F7fJ/amy+tieQxEMWBlbV1JGSjkFM+zkHNf81Cad+dfIRA+IBkvmvdAeA==" + }, "@yarnpkg/lockfile": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz", @@ -1447,6 +1452,11 @@ } } }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" + }, "bootstrap": { "version": "3.3.7", "resolved": "http://registry.npmjs.org/bootstrap/-/bootstrap-3.3.7.tgz", @@ -1842,6 +1852,19 @@ "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", "dev": true }, + "cheerio": { + "version": "1.0.0-rc.2", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.2.tgz", + "integrity": "sha1-S59TqBsn5NXawxwP/Qz6A8xoMNs=", + "requires": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.0", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash": "^4.15.0", + "parse5": "^3.0.1" + } + }, "cipher-base": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz", @@ -2218,6 +2241,33 @@ "integrity": "sha1-ojD2T1aDEOFJgAmUB5DsmVRbyn4=", "dev": true }, + "css-select": { + "version": "1.2.0", + "resolved": "http://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", + "requires": { + "boolbase": "~1.0.0", + "css-what": "2.1", + "domutils": "1.5.1", + "nth-check": "~1.0.1" + }, + "dependencies": { + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + } + } + }, + "css-what": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.2.tgz", + "integrity": "sha512-wan8dMWQ0GUeF7DGEPVjhHemVW/vy6xUYmFzRY8RYqgA0JtXC9rJmbScBjqSu6dg9q0lwPQy6ZAmJVr3PPTvqQ==" + }, "d": { "version": "1.0.0", "resolved": "http://registry.npmjs.org/d/-/d-1.0.0.tgz", @@ -2485,11 +2535,49 @@ } } }, + "dom-serializer": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz", + "integrity": "sha1-BzxpdUbOB4DOI75KKOKT5AvDDII=", + "requires": { + "domelementtype": "~1.1.1", + "entities": "~1.1.1" + }, + "dependencies": { + "domelementtype": { + "version": "1.1.3", + "resolved": "http://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", + "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=" + } + } + }, "domain-browser": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz", "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==" }, + "domelementtype": { + "version": "1.3.0", + "resolved": "http://registry.npmjs.org/domelementtype/-/domelementtype-1.3.0.tgz", + "integrity": "sha1-sXrtguirWeUt2cGbF1bg/BhyBMI=" + }, + "domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz", + "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==", + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, "dot-prop": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz", @@ -2588,6 +2676,11 @@ "iconv-lite": "~0.4.13" } }, + "entities": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==" + }, "es5-ext": { "version": "0.10.46", "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.46.tgz", @@ -3733,6 +3826,39 @@ "resolved": "http://registry.npmjs.org/htmlescape/-/htmlescape-1.1.1.tgz", "integrity": "sha1-OgPtwiFLyjtmQko+eVk0lQnLA1E=" }, + "htmlparser2": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.0.tgz", + "integrity": "sha512-J1nEUGv+MkXS0weHNWVKJJ+UrLfePxRWpN3C9bEi9fLxL2+ggW94DQvgYVXsaT30PGwYRIZKNZXuyMhp3Di4bQ==", + "requires": { + "domelementtype": "^1.3.0", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.0.6" + }, + "dependencies": { + "readable-stream": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.0.6.tgz", + "integrity": "sha512-9E1oLoOWfhSXHGv6QlwXJim7uNzd9EVlWK+21tCU9Ju/kR0/p2AZYPz4qSchgO8PlLIH4FpZYfzwS+rEksZjIg==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "http-errors": { "version": "1.6.3", "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", @@ -5098,6 +5224,14 @@ "pify": "^3.0.0" } }, + "nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "requires": { + "boolbase": "~1.0.0" + } + }, "number-is-nan": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", @@ -6520,6 +6654,14 @@ "pbkdf2": "^3.0.3" } }, + "parse5": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", + "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", + "requires": { + "@types/node": "*" + } + }, "parseurl": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", @@ -8136,6 +8278,30 @@ "source-pane": "^1.0.3" } }, + "solid-permissions": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/solid-permissions/-/solid-permissions-0.6.0.tgz", + "integrity": "sha512-TiObEonB4l8VNa/6IOnNL0/7u3GKil4Bh/BiBB51AXOCuLahvTQ+LQc+C5KLHAykzyb9z5oM/BVOE3w8mzIm/A==", + "requires": { + "debug": "^3.1.0", + "solid-namespace": "0.1.0" + }, + "dependencies": { + "debug": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.5.tgz", + "integrity": "sha512-D61LaDQPQkxJ5AUM2mbSJRbPkNs/TmdmOeLAi1hgDkpDfIfetSrjmWhccwtuResSwMbACjx/xXQofvM9CE/aeg==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + } + } + }, "solid-ui": { "version": "0.11.12", "resolved": "https://registry.npmjs.org/solid-ui/-/solid-ui-0.11.12.tgz", diff --git a/package.json b/package.json index 05d5e45f7..7c123314d 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "busboy": "^0.2.12", "cached-path-relative": "^1.0.2", "camelize": "^1.0.0", + "cheerio": "^1.0.0-rc.2", "colorette": "^1.0.5", "commander": "^2.9.0", "cors": "^2.7.1", From 627b1fe8432c76890662f8f2474432075c04b30b Mon Sep 17 00:00:00 2001 From: Arne Hassel Date: Fri, 16 Nov 2018 15:06:07 +0100 Subject: [PATCH 04/16] Trying to make code more readable And by that hopefully more maintainable --- bin/lib/{common.js => cli-utils.js} | 0 bin/lib/invalidUsernames.js | 2 +- bin/lib/start.js | 2 +- bin/lib/updateIndex.js | 49 +++++++++++------------- default-templates/new-account/index.html | 2 + lib/common/fs-utils.js | 10 +++++ lib/common/template-utils.js | 40 +++++++++---------- lib/common/user-utils.js | 20 ++++++++++ 8 files changed, 77 insertions(+), 48 deletions(-) rename bin/lib/{common.js => cli-utils.js} (100%) diff --git a/bin/lib/common.js b/bin/lib/cli-utils.js similarity index 100% rename from bin/lib/common.js rename to bin/lib/cli-utils.js diff --git a/bin/lib/invalidUsernames.js b/bin/lib/invalidUsernames.js index d2a2aef27..1d6cd340e 100644 --- a/bin/lib/invalidUsernames.js +++ b/bin/lib/invalidUsernames.js @@ -2,7 +2,7 @@ const fs = require('fs-extra') const Handlebars = require('handlebars') const path = require('path') -const { getAccountManager, loadConfig, loadUsernames } = require('./common') +const { getAccountManager, loadConfig, loadUsernames } = require('./cli-utils') const { isValidUsername } = require('../../lib/common/user-utils') const blacklistService = require('../../lib/services/blacklist-service') const { initConfigDir, initTemplateDirs } = require('../../lib/server-config') diff --git a/bin/lib/start.js b/bin/lib/start.js index 723fec99f..222911186 100644 --- a/bin/lib/start.js +++ b/bin/lib/start.js @@ -2,7 +2,7 @@ const options = require('./options') const fs = require('fs') -const { loadConfig } = require('./common') +const { loadConfig } = require('./cli-utils') const { red, bold } = require('colorette') module.exports = function (program, server) { diff --git a/bin/lib/updateIndex.js b/bin/lib/updateIndex.js index 325005885..d962dee6a 100644 --- a/bin/lib/updateIndex.js +++ b/bin/lib/updateIndex.js @@ -1,53 +1,50 @@ const fs = require('fs') const path = require('path') const cheerio = require('cheerio') -const Handlebars = require('handlebars') const LDP = require('../../lib/ldp') -const $rdf = require('rdflib') const { URL } = require('url') -const { getAccountManager, loadConfig, loadUsernames } = require('./common') +const { compileTemplate, writeTemplate } = require('../../lib/common/template-utils') +const { getAccountManager, loadConfig, loadUsernames } = require('./cli-utils') +const { getWebId } = require('../../lib/common/user-utils') const { initConfigDir, initTemplateDirs } = require('../../lib/server-config') -const SOLID = $rdf.Namespace('http://www.w3.org/ns/solid/terms#') - module.exports = function (program) { program .command('updateindex') .description('Update index.html in root of all PODs that haven\'t been marked otherwise') .action(async (options) => { const config = loadConfig(program, options) - const ldp = new LDP(config) const configPath = initConfigDir(config) const templates = initTemplateDirs(configPath) + const indexTemplatePath = path.join(templates.account, 'index.html') + const indexTemplate = await compileTemplate(indexTemplatePath) + const ldp = new LDP(config) const accountManager = getAccountManager(config, { ldp }) const usernames = loadUsernames(config) - const indexTemplatePath = path.join(templates.account, 'index.html') - const indexTemplateSource = fs.readFileSync(indexTemplatePath, 'utf-8') - const indexTemplate = Handlebars.compile(indexTemplateSource) const usersProcessed = usernames.map(async name => { - const userDirectory = accountManager.accountDirFor(name) - const indexFilePath = path.join(userDirectory, 'index.html') - const indexSource = fs.readFileSync(indexFilePath, 'utf-8') - const $ = cheerio.load(indexSource) - const allowAutomaticUpdateValue = $('meta[name="solid-allow-automatic-updates"]').prop('content') - const allowAutomaticUpdate = !allowAutomaticUpdateValue || allowAutomaticUpdateValue === 'true' - if (!allowAutomaticUpdate) { + const accountDirectory = accountManager.accountDirFor(name) + const indexFilePath = path.join(accountDirectory, 'index.html') + if (!isUpdateAllowed(indexFilePath)) { return } - const serverUrl = new URL(config.serverUri) - const accountUrl = `${serverUrl.protocol}//${name}.${serverUrl.host}/` - const metaFileUri = `${accountUrl}/${ldp.suffixMeta}` - const metaData = await ldp.readContainerMeta(userDirectory) - const metaGraph = $rdf.graph() - $rdf.parse(metaData, metaGraph, metaFileUri, 'text/turtle') - const webIdNode = metaGraph.any(undefined, SOLID('account'), $rdf.sym(accountUrl)) - const webId = webIdNode.value - const newIndexSource = indexTemplate({ name, webId }) - fs.writeFileSync(indexFilePath, newIndexSource, 'utf-8') + const accountUrl = getAccountUrl(name, config) + const webId = await getWebId(accountDirectory, accountUrl, { ldp }) + writeTemplate(indexFilePath, indexTemplate, { name, webId }) }) await Promise.all(usersProcessed) console.log(`Processed ${usersProcessed.length} users`) }) } +function getAccountUrl (name, config) { + const serverUrl = new URL(config.serverUri) + return `${serverUrl.protocol}//${name}.${serverUrl.host}/` +} + +function isUpdateAllowed (indexFilePath) { + const indexSource = fs.readFileSync(indexFilePath, 'utf-8') + const $ = cheerio.load(indexSource) + const allowAutomaticUpdateValue = $('meta[name="solid-allow-automatic-updates"]').prop('content') + return !allowAutomaticUpdateValue || allowAutomaticUpdateValue === 'true' +} diff --git a/default-templates/new-account/index.html b/default-templates/new-account/index.html index 8364a4335..7fe96bfa7 100644 --- a/default-templates/new-account/index.html +++ b/default-templates/new-account/index.html @@ -3,6 +3,8 @@ + + {{#if name}}{{name}} – {{/if}}Solid Home diff --git a/lib/common/fs-utils.js b/lib/common/fs-utils.js index 083a2bdc0..8da615064 100644 --- a/lib/common/fs-utils.js +++ b/lib/common/fs-utils.js @@ -1,5 +1,7 @@ module.exports.copyTemplateDir = copyTemplateDir module.exports.processFile = processFile +module.exports.readFile = readFile +module.exports.writeFile = writeFile const fs = require('fs-extra') @@ -31,3 +33,11 @@ async function processFile (filePath, manipulateSourceFn) { }) }) } + +function readFile (filePath, options = 'utf-8') { + return fs.readFileSync(filePath, options) +} + +function writeFile (filePath, fileSource, options = 'utf-8') { + fs.writeFileSync(filePath, fileSource, options) +} diff --git a/lib/common/template-utils.js b/lib/common/template-utils.js index 78bbb1479..9b7c05af8 100644 --- a/lib/common/template-utils.js +++ b/lib/common/template-utils.js @@ -1,8 +1,15 @@ +module.exports.compileTemplate = compileTemplate module.exports.processHandlebarFile = processHandlebarFile +module.exports.writeTemplate = writeTemplate const Handlebars = require('handlebars') const debug = require('../debug').errors -const { processFile } = require('./fs-utils') +const { processFile, readFile, writeFile } = require('./fs-utils') + +async function compileTemplate (filePath) { + const indexTemplateSource = readFile(filePath) + return Handlebars.compile(indexTemplateSource) +} /** * Reads a file, processes it (performing template substitution), and saves @@ -14,25 +21,18 @@ const { processFile } = require('./fs-utils') * @return {Promise} */ async function processHandlebarFile (filePath, substitutions) { - return processFile(filePath, (rawSource) => processHandlebarTemplate(rawSource, substitutions)) + return processFile(filePath, (rawSource) => { + try { + const template = Handlebars.compile(rawSource) + return template(substitutions) + } catch (error) { + debug(`Error processing template: ${error}`) + return rawSource + } + }) } -/** - * Performs a Handlebars string template substitution, and returns the - * resulting string. - * - * @see https://www.npmjs.com/package/handlebars - * - * @param source {string} e.g. 'Hello, {{name}}' - * - * @return {string} Result, e.g. 'Hello, Alice' - */ -function processHandlebarTemplate (source, substitutions) { - try { - const template = Handlebars.compile(source) - return template(substitutions) - } catch (error) { - debug(`Error processing template: ${error}`) - return source - } +function writeTemplate (filePath, template, substitutions) { + const source = template(substitutions) + writeFile(filePath, source) } diff --git a/lib/common/user-utils.js b/lib/common/user-utils.js index 3c0c46a69..98356a5ba 100644 --- a/lib/common/user-utils.js +++ b/lib/common/user-utils.js @@ -1,5 +1,25 @@ +const $rdf = require('rdflib') + +const LDP = require('../ldp') + +const SOLID = $rdf.Namespace('http://www.w3.org/ns/solid/terms#') + +module.exports.getWebId = getWebId module.exports.isValidUsername = isValidUsername +async function getWebId (accountDirectory, accountUrl, options = {}) { + if (!options.ldp && !options.config) { + throw new Error('Require ldp or config set in options for getWebId') + } + const ldp = options.ldp || new LDP(options.config) + const metaFileUri = `${accountUrl}/${ldp.suffixMeta}` + const metaData = await ldp.readContainerMeta(accountDirectory) + const metaGraph = $rdf.graph() + $rdf.parse(metaData, metaGraph, metaFileUri, 'text/turtle') + const webIdNode = metaGraph.any(undefined, SOLID('account'), $rdf.sym(accountUrl)) + return webIdNode.value +} + function isValidUsername (username) { return /^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(username) } From beeeab7201c1c1419e82edac21ed433c6799fbe6 Mon Sep 17 00:00:00 2001 From: Arne Hassel Date: Tue, 20 Nov 2018 13:24:23 +0100 Subject: [PATCH 05/16] Now fetching name given a webId --- bin/lib/updateIndex.js | 20 +++++++++++++------- lib/common/user-utils.js | 21 +++++++++++++++++---- lib/ldp.js | 2 +- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/bin/lib/updateIndex.js b/bin/lib/updateIndex.js index d962dee6a..d21e2749b 100644 --- a/bin/lib/updateIndex.js +++ b/bin/lib/updateIndex.js @@ -3,10 +3,11 @@ const path = require('path') const cheerio = require('cheerio') const LDP = require('../../lib/ldp') const { URL } = require('url') +const debug = require('../../lib/debug') const { compileTemplate, writeTemplate } = require('../../lib/common/template-utils') const { getAccountManager, loadConfig, loadUsernames } = require('./cli-utils') -const { getWebId } = require('../../lib/common/user-utils') +const { getName, getWebId } = require('../../lib/common/user-utils') const { initConfigDir, initTemplateDirs } = require('../../lib/server-config') module.exports = function (program) { @@ -22,18 +23,23 @@ module.exports = function (program) { const ldp = new LDP(config) const accountManager = getAccountManager(config, { ldp }) const usernames = loadUsernames(config) - const usersProcessed = usernames.map(async name => { - const accountDirectory = accountManager.accountDirFor(name) + const usersProcessed = usernames.map(async username => { + const accountDirectory = accountManager.accountDirFor(username) const indexFilePath = path.join(accountDirectory, 'index.html') if (!isUpdateAllowed(indexFilePath)) { return } - const accountUrl = getAccountUrl(name, config) - const webId = await getWebId(accountDirectory, accountUrl, { ldp }) - writeTemplate(indexFilePath, indexTemplate, { name, webId }) + const accountUrl = getAccountUrl(username, config) + try { + const webId = await getWebId(accountDirectory, accountUrl, { ldp }) + const name = await getName(webId, { ldp }) + writeTemplate(indexFilePath, indexTemplate, { name, webId }) + } catch (err) { + debug.errors(`Failed to create new index for ${username}: ${JSON.stringify(err, null, 2)}`) + } }) await Promise.all(usersProcessed) - console.log(`Processed ${usersProcessed.length} users`) + debug.accounts(`Processed ${usersProcessed.length} users`) }) } diff --git a/lib/common/user-utils.js b/lib/common/user-utils.js index 98356a5ba..85d8bcaf8 100644 --- a/lib/common/user-utils.js +++ b/lib/common/user-utils.js @@ -3,15 +3,21 @@ const $rdf = require('rdflib') const LDP = require('../ldp') const SOLID = $rdf.Namespace('http://www.w3.org/ns/solid/terms#') +const VCARD = $rdf.Namespace('http://www.w3.org/2006/vcard/ns#') +module.exports.getName = getName module.exports.getWebId = getWebId module.exports.isValidUsername = isValidUsername +async function getName (webId, options = {}) { + const ldp = setupLDP(options) + const graph = await ldp.fetchGraph(webId) + const nameNode = graph.any($rdf.sym(webId), VCARD('fn')) + return nameNode.value +} + async function getWebId (accountDirectory, accountUrl, options = {}) { - if (!options.ldp && !options.config) { - throw new Error('Require ldp or config set in options for getWebId') - } - const ldp = options.ldp || new LDP(options.config) + const ldp = setupLDP(options) const metaFileUri = `${accountUrl}/${ldp.suffixMeta}` const metaData = await ldp.readContainerMeta(accountDirectory) const metaGraph = $rdf.graph() @@ -23,3 +29,10 @@ async function getWebId (accountDirectory, accountUrl, options = {}) { function isValidUsername (username) { return /^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(username) } + +function setupLDP (options) { + if (!options.ldp && !options.config) { + throw new Error('Require ldp or config set in options for getWebId') + } + return options.ldp || new LDP(options.config) +} diff --git a/lib/ldp.js b/lib/ldp.js index ebaa9274e..d90c70400 100644 --- a/lib/ldp.js +++ b/lib/ldp.js @@ -273,7 +273,7 @@ class LDP { * * @return {Promise} */ - async fetchGraph (uri, options) { + async fetchGraph (uri, options = {}) { const response = await fetch(uri) if (!response.ok) { const error = new Error( From 34e403d83674a96086be1846dacb6e79264038b0 Mon Sep 17 00:00:00 2001 From: Arne Hassel Date: Tue, 20 Nov 2018 13:31:38 +0100 Subject: [PATCH 06/16] Cleaning up a bit --- lib/common/template-utils.js | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/common/template-utils.js b/lib/common/template-utils.js index 9b7c05af8..435e39dd1 100644 --- a/lib/common/template-utils.js +++ b/lib/common/template-utils.js @@ -21,15 +21,27 @@ async function compileTemplate (filePath) { * @return {Promise} */ async function processHandlebarFile (filePath, substitutions) { - return processFile(filePath, (rawSource) => { - try { - const template = Handlebars.compile(rawSource) - return template(substitutions) - } catch (error) { - debug(`Error processing template: ${error}`) - return rawSource - } - }) + return processFile(filePath, (rawSource) => processHandlebarTemplate(rawSource, substitutions)) +} + +/** + * Performs a Handlebars string template substitution, and returns the + * resulting string. + * + * @see https://www.npmjs.com/package/handlebars + * + * @param source {string} e.g. 'Hello, {{name}}' + * + * @return {string} Result, e.g. 'Hello, Alice' + */ +function processHandlebarTemplate (source, substitutions) { + try { + const template = Handlebars.compile(source) + return template(substitutions) + } catch (error) { + debug(`Error processing template: ${error}`) + return source + } } function writeTemplate (filePath, template, substitutions) { From 988ee0f5712d1a3bbf86adf3245a3a895f05e3af Mon Sep 17 00:00:00 2001 From: Arne Hassel Date: Tue, 20 Nov 2018 14:01:43 +0100 Subject: [PATCH 07/16] Added a couple of tests --- test/unit/user-utils-test.js | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/unit/user-utils-test.js b/test/unit/user-utils-test.js index 9170e7eac..cd0af1fd8 100644 --- a/test/unit/user-utils-test.js +++ b/test/unit/user-utils-test.js @@ -1,8 +1,43 @@ const chai = require('chai') const expect = chai.expect const userUtils = require('../../lib/common/user-utils') +const $rdf = require('rdflib') describe('user-utils', () => { + describe('getName', () => { + let ldp + const webId = 'http://test#me' + const name = 'NAME' + + beforeEach(() => { + const store = $rdf.graph() + store.add($rdf.sym(webId), $rdf.sym('http://www.w3.org/2006/vcard/ns#fn'), $rdf.lit(name)) + ldp = { fetchGraph: () => Promise.resolve(store) } + }) + + it('should return name from graph', async () => { + const returnedName = await userUtils.getName(webId, { ldp }) + expect(returnedName).to.equal(name) + }) + }) + + describe('getWebId', () => { + let ldp + const webId = 'https://test.localhost:8443/profile/card#me' + + beforeEach(() => { + ldp = { + readContainerMeta: () => Promise.resolve(`<${webId}> .`), + suffixMeta: '.meta' + } + }) + + it('should return webId from meta file', async () => { + const returnedWebId = await userUtils.getWebId('foo', 'https://bar/', { ldp }) + expect(returnedWebId).to.equal(webId) + }) + }) + describe('isValidUsername', () => { it('should accect valid usernames', () => { const usernames = [ From abbdbab5ab89f2699b50676cdce8b39d08609219 Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Wed, 21 Nov 2018 14:47:27 +0100 Subject: [PATCH 08/16] Fix index path creation on Windows --- lib/handlers/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/handlers/index.js b/lib/handlers/index.js index a72dbde8a..4b9e20305 100644 --- a/lib/handlers/index.js +++ b/lib/handlers/index.js @@ -3,6 +3,7 @@ module.exports = handler const path = require('path') const debug = require('debug')('solid:index') const Negotiator = require('negotiator') +const url = require('url') async function handler (req, res, next) { const indexFile = 'index.html' From 7090ee0c4e84c1173e23ca8a18ec8dca9ca6aea3 Mon Sep 17 00:00:00 2001 From: Arne Hassel Date: Fri, 16 Nov 2018 15:06:07 +0100 Subject: [PATCH 09/16] Trying to make code more readable And by that hopefully more maintainable --- lib/common/template-utils.js | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/lib/common/template-utils.js b/lib/common/template-utils.js index 435e39dd1..05a24e3a4 100644 --- a/lib/common/template-utils.js +++ b/lib/common/template-utils.js @@ -21,27 +21,20 @@ async function compileTemplate (filePath) { * @return {Promise} */ async function processHandlebarFile (filePath, substitutions) { - return processFile(filePath, (rawSource) => processHandlebarTemplate(rawSource, substitutions)) + return processFile(filePath, (rawSource) => { + try { + const template = Handlebars.compile(rawSource) + return template(substitutions) + } catch (error) { + debug(`Error processing template: ${error}`) + return rawSource + } + }) } -/** - * Performs a Handlebars string template substitution, and returns the - * resulting string. - * - * @see https://www.npmjs.com/package/handlebars - * - * @param source {string} e.g. 'Hello, {{name}}' - * - * @return {string} Result, e.g. 'Hello, Alice' - */ -function processHandlebarTemplate (source, substitutions) { - try { - const template = Handlebars.compile(source) - return template(substitutions) - } catch (error) { - debug(`Error processing template: ${error}`) - return source - } +function writeTemplate (filePath, template, substitutions) { + const source = template(substitutions) + writeFile(filePath, source) } function writeTemplate (filePath, template, substitutions) { From 3b25fdaadace2c3024aee7e8b10c63983a0bbaac Mon Sep 17 00:00:00 2001 From: Arne Hassel Date: Tue, 20 Nov 2018 13:31:38 +0100 Subject: [PATCH 10/16] Cleaning up a bit --- lib/common/template-utils.js | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/common/template-utils.js b/lib/common/template-utils.js index 05a24e3a4..ab60880a0 100644 --- a/lib/common/template-utils.js +++ b/lib/common/template-utils.js @@ -21,15 +21,27 @@ async function compileTemplate (filePath) { * @return {Promise} */ async function processHandlebarFile (filePath, substitutions) { - return processFile(filePath, (rawSource) => { - try { - const template = Handlebars.compile(rawSource) - return template(substitutions) - } catch (error) { - debug(`Error processing template: ${error}`) - return rawSource - } - }) + return processFile(filePath, (rawSource) => processHandlebarTemplate(rawSource, substitutions)) +} + +/** + * Performs a Handlebars string template substitution, and returns the + * resulting string. + * + * @see https://www.npmjs.com/package/handlebars + * + * @param source {string} e.g. 'Hello, {{name}}' + * + * @return {string} Result, e.g. 'Hello, Alice' + */ +function processHandlebarTemplate (source, substitutions) { + try { + const template = Handlebars.compile(source) + return template(substitutions) + } catch (error) { + debug(`Error processing template: ${error}`) + return source + } } function writeTemplate (filePath, template, substitutions) { From 7a8230eb9b79b5a688c3bf108543f10c30d50572 Mon Sep 17 00:00:00 2001 From: Arne Hassel Date: Sat, 24 Nov 2018 16:39:47 +0100 Subject: [PATCH 11/16] Removing extend from cli-utils --- bin/lib/cli-utils.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/lib/cli-utils.js b/bin/lib/cli-utils.js index b193622e1..3ce82a1f7 100644 --- a/bin/lib/cli-utils.js +++ b/bin/lib/cli-utils.js @@ -1,5 +1,4 @@ const fs = require('fs-extra') -const extend = require('extend') const { cyan, bold } = require('colorette') const { URL } = require('url') const LDP = require('../../lib/ldp') @@ -28,7 +27,10 @@ function getAccountManager (config, options = {}) { } function loadConfig (program, options) { - let argv = extend({}, options, { version: program.version() }) + let argv = { + ...options, + version: program.version() + } let configFile = argv['configFile'] || './config.json' try { @@ -36,9 +38,7 @@ function loadConfig (program, options) { // Use flags with priority over config file const config = JSON.parse(file) - Object.keys(config).forEach((option) => { - argv[option] = argv[option] || config[option] - }) + argv = { ...config, ...argv } } catch (err) { // No file exists, not a problem console.log(cyan(bold('TIP')), 'create a config.json: `$ solid init`') From 3989aa8530d3d11fc2c76a33d32d35b227fbedc9 Mon Sep 17 00:00:00 2001 From: Arne Hassel Date: Fri, 16 Nov 2018 15:06:07 +0100 Subject: [PATCH 12/16] Trying to make code more readable And by that hopefully more maintainable --- lib/common/template-utils.js | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/lib/common/template-utils.js b/lib/common/template-utils.js index ab60880a0..0eeab0783 100644 --- a/lib/common/template-utils.js +++ b/lib/common/template-utils.js @@ -21,27 +21,20 @@ async function compileTemplate (filePath) { * @return {Promise} */ async function processHandlebarFile (filePath, substitutions) { - return processFile(filePath, (rawSource) => processHandlebarTemplate(rawSource, substitutions)) + return processFile(filePath, (rawSource) => { + try { + const template = Handlebars.compile(rawSource) + return template(substitutions) + } catch (error) { + debug(`Error processing template: ${error}`) + return rawSource + } + }) } -/** - * Performs a Handlebars string template substitution, and returns the - * resulting string. - * - * @see https://www.npmjs.com/package/handlebars - * - * @param source {string} e.g. 'Hello, {{name}}' - * - * @return {string} Result, e.g. 'Hello, Alice' - */ -function processHandlebarTemplate (source, substitutions) { - try { - const template = Handlebars.compile(source) - return template(substitutions) - } catch (error) { - debug(`Error processing template: ${error}`) - return source - } +function writeTemplate (filePath, template, substitutions) { + const source = template(substitutions) + writeFile(filePath, source) } function writeTemplate (filePath, template, substitutions) { From 16688d40db110e6344688d98fd43b6f381c0070d Mon Sep 17 00:00:00 2001 From: Arne Hassel Date: Tue, 20 Nov 2018 13:31:38 +0100 Subject: [PATCH 13/16] Cleaning up a bit --- lib/common/template-utils.js | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/common/template-utils.js b/lib/common/template-utils.js index 0eeab0783..119c37fba 100644 --- a/lib/common/template-utils.js +++ b/lib/common/template-utils.js @@ -21,15 +21,27 @@ async function compileTemplate (filePath) { * @return {Promise} */ async function processHandlebarFile (filePath, substitutions) { - return processFile(filePath, (rawSource) => { - try { - const template = Handlebars.compile(rawSource) - return template(substitutions) - } catch (error) { - debug(`Error processing template: ${error}`) - return rawSource - } - }) + return processFile(filePath, (rawSource) => processHandlebarTemplate(rawSource, substitutions)) +} + +/** + * Performs a Handlebars string template substitution, and returns the + * resulting string. + * + * @see https://www.npmjs.com/package/handlebars + * + * @param source {string} e.g. 'Hello, {{name}}' + * + * @return {string} Result, e.g. 'Hello, Alice' + */ +function processHandlebarTemplate (source, substitutions) { + try { + const template = Handlebars.compile(source) + return template(substitutions) + } catch (error) { + debug(`Error processing template: ${error}`) + return source + } } function writeTemplate (filePath, template, substitutions) { From 24a9a587aad18e0bc53721d93240f17100b241b5 Mon Sep 17 00:00:00 2001 From: Arne Hassel Date: Fri, 16 Nov 2018 15:06:07 +0100 Subject: [PATCH 14/16] Trying to make code more readable And by that hopefully more maintainable --- lib/common/template-utils.js | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/lib/common/template-utils.js b/lib/common/template-utils.js index 119c37fba..5416437fd 100644 --- a/lib/common/template-utils.js +++ b/lib/common/template-utils.js @@ -21,27 +21,20 @@ async function compileTemplate (filePath) { * @return {Promise} */ async function processHandlebarFile (filePath, substitutions) { - return processFile(filePath, (rawSource) => processHandlebarTemplate(rawSource, substitutions)) + return processFile(filePath, (rawSource) => { + try { + const template = Handlebars.compile(rawSource) + return template(substitutions) + } catch (error) { + debug(`Error processing template: ${error}`) + return rawSource + } + }) } -/** - * Performs a Handlebars string template substitution, and returns the - * resulting string. - * - * @see https://www.npmjs.com/package/handlebars - * - * @param source {string} e.g. 'Hello, {{name}}' - * - * @return {string} Result, e.g. 'Hello, Alice' - */ -function processHandlebarTemplate (source, substitutions) { - try { - const template = Handlebars.compile(source) - return template(substitutions) - } catch (error) { - debug(`Error processing template: ${error}`) - return source - } +function writeTemplate (filePath, template, substitutions) { + const source = template(substitutions) + writeFile(filePath, source) } function writeTemplate (filePath, template, substitutions) { From bfdb2cebf24b2df6585e14277376f8cfd83ab735 Mon Sep 17 00:00:00 2001 From: Arne Hassel Date: Tue, 20 Nov 2018 13:31:38 +0100 Subject: [PATCH 15/16] Cleaning up a bit --- lib/common/template-utils.js | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/common/template-utils.js b/lib/common/template-utils.js index 5416437fd..bd0d8f493 100644 --- a/lib/common/template-utils.js +++ b/lib/common/template-utils.js @@ -21,15 +21,27 @@ async function compileTemplate (filePath) { * @return {Promise} */ async function processHandlebarFile (filePath, substitutions) { - return processFile(filePath, (rawSource) => { - try { - const template = Handlebars.compile(rawSource) - return template(substitutions) - } catch (error) { - debug(`Error processing template: ${error}`) - return rawSource - } - }) + return processFile(filePath, (rawSource) => processHandlebarTemplate(rawSource, substitutions)) +} + +/** + * Performs a Handlebars string template substitution, and returns the + * resulting string. + * + * @see https://www.npmjs.com/package/handlebars + * + * @param source {string} e.g. 'Hello, {{name}}' + * + * @return {string} Result, e.g. 'Hello, Alice' + */ +function processHandlebarTemplate (source, substitutions) { + try { + const template = Handlebars.compile(source) + return template(substitutions) + } catch (error) { + debug(`Error processing template: ${error}`) + return source + } } function writeTemplate (filePath, template, substitutions) { From bf9a59e52e3275eeb1480358710b91d70a6f2408 Mon Sep 17 00:00:00 2001 From: Arne Hassel Date: Tue, 27 Nov 2018 19:00:56 +0100 Subject: [PATCH 16/16] Fixed all tests, rebased on v5.0.0 --- bin/lib/cli-utils.js | 19 +++++++++++++--- bin/lib/updateIndex.js | 20 ++++++++--------- lib/common/template-utils.js | 14 ------------ lib/common/user-utils.js | 22 +++++-------------- lib/handlers/index.js | 4 ++-- lib/ldp.js | 4 ++-- lib/resource-mapper.js | 3 ++- lib/utils.js | 17 ++++++++++++++ .../tim.localhost/write-acl/test-file$.txt | 1 + .../acl-tls/write-acl/test-file$.ttl | 1 + test/unit/user-utils-test.js | 12 +++++----- 11 files changed, 62 insertions(+), 55 deletions(-) create mode 100644 test/resources/accounts-acl/tim.localhost/write-acl/test-file$.txt create mode 100644 test/resources/acl-tls/write-acl/test-file$.ttl diff --git a/bin/lib/cli-utils.js b/bin/lib/cli-utils.js index 3ce82a1f7..b578f6614 100644 --- a/bin/lib/cli-utils.js +++ b/bin/lib/cli-utils.js @@ -6,6 +6,7 @@ const AccountManager = require('../../lib/models/account-manager') const SolidHost = require('../../lib/models/solid-host') module.exports.getAccountManager = getAccountManager +module.exports.loadAccounts = loadAccounts module.exports.loadConfig = loadConfig module.exports.loadUsernames = loadUsernames @@ -47,11 +48,23 @@ function loadConfig (program, options) { return argv } -function loadUsernames (config) { - const files = fs.readdirSync(config.root) - const hostname = new URL(config.serverUri).hostname +/** + * + * @param root + * @param [serverUri] If not set, hostname must be set + * @param [hostname] If not set, serverUri must be set + * @returns {*} + */ +function loadAccounts ({ root, serverUri, hostname }) { + const files = fs.readdirSync(root) + hostname = hostname || new URL(serverUri).hostname const isUserDirectory = new RegExp(`.${hostname}$`) return files .filter(file => isUserDirectory.test(file)) +} + +function loadUsernames ({ root, serverUri }) { + const hostname = new URL(serverUri).hostname + return loadAccounts({ root, hostname }) .map(userDirectory => userDirectory.substr(0, userDirectory.length - hostname.length - 1)) } diff --git a/bin/lib/updateIndex.js b/bin/lib/updateIndex.js index d21e2749b..8412f7210 100644 --- a/bin/lib/updateIndex.js +++ b/bin/lib/updateIndex.js @@ -4,9 +4,10 @@ const cheerio = require('cheerio') const LDP = require('../../lib/ldp') const { URL } = require('url') const debug = require('../../lib/debug') +const { readFile } = require('../../lib/common/fs-utils') const { compileTemplate, writeTemplate } = require('../../lib/common/template-utils') -const { getAccountManager, loadConfig, loadUsernames } = require('./cli-utils') +const { loadConfig, loadAccounts } = require('./cli-utils') const { getName, getWebId } = require('../../lib/common/user-utils') const { initConfigDir, initTemplateDirs } = require('../../lib/server-config') @@ -21,21 +22,20 @@ module.exports = function (program) { const indexTemplatePath = path.join(templates.account, 'index.html') const indexTemplate = await compileTemplate(indexTemplatePath) const ldp = new LDP(config) - const accountManager = getAccountManager(config, { ldp }) - const usernames = loadUsernames(config) - const usersProcessed = usernames.map(async username => { - const accountDirectory = accountManager.accountDirFor(username) - const indexFilePath = path.join(accountDirectory, 'index.html') + const accounts = loadAccounts(config) + const usersProcessed = accounts.map(async account => { + const accountDirectory = path.join(config.root, account) + const indexFilePath = path.join(accountDirectory, '/index.html') if (!isUpdateAllowed(indexFilePath)) { return } - const accountUrl = getAccountUrl(username, config) + const accountUrl = getAccountUrl(account, config) try { - const webId = await getWebId(accountDirectory, accountUrl, { ldp }) - const name = await getName(webId, { ldp }) + const webId = await getWebId(accountDirectory, accountUrl, ldp.suffixMeta, (filePath) => readFile(filePath)) + const name = await getName(webId, ldp.fetchGraph) writeTemplate(indexFilePath, indexTemplate, { name, webId }) } catch (err) { - debug.errors(`Failed to create new index for ${username}: ${JSON.stringify(err, null, 2)}`) + debug.errors(`Failed to create new index for ${account}: ${JSON.stringify(err, null, 2)}`) } }) await Promise.all(usersProcessed) diff --git a/lib/common/template-utils.js b/lib/common/template-utils.js index bd0d8f493..afb6f05e8 100644 --- a/lib/common/template-utils.js +++ b/lib/common/template-utils.js @@ -49,17 +49,3 @@ function writeTemplate (filePath, template, substitutions) { writeFile(filePath, source) } -function writeTemplate (filePath, template, substitutions) { - const source = template(substitutions) - writeFile(filePath, source) -} - -function writeTemplate (filePath, template, substitutions) { - const source = template(substitutions) - writeFile(filePath, source) -} - -function writeTemplate (filePath, template, substitutions) { - const source = template(substitutions) - writeFile(filePath, source) -} diff --git a/lib/common/user-utils.js b/lib/common/user-utils.js index 85d8bcaf8..c5bcd00b8 100644 --- a/lib/common/user-utils.js +++ b/lib/common/user-utils.js @@ -1,7 +1,5 @@ const $rdf = require('rdflib') -const LDP = require('../ldp') - const SOLID = $rdf.Namespace('http://www.w3.org/ns/solid/terms#') const VCARD = $rdf.Namespace('http://www.w3.org/2006/vcard/ns#') @@ -9,17 +7,16 @@ module.exports.getName = getName module.exports.getWebId = getWebId module.exports.isValidUsername = isValidUsername -async function getName (webId, options = {}) { - const ldp = setupLDP(options) - const graph = await ldp.fetchGraph(webId) +async function getName (webId, fetchGraph) { + const graph = await fetchGraph(webId) const nameNode = graph.any($rdf.sym(webId), VCARD('fn')) return nameNode.value } -async function getWebId (accountDirectory, accountUrl, options = {}) { - const ldp = setupLDP(options) - const metaFileUri = `${accountUrl}/${ldp.suffixMeta}` - const metaData = await ldp.readContainerMeta(accountDirectory) +async function getWebId (accountDirectory, accountUrl, suffixMeta, fetchData) { + const metaFilePath = `${accountDirectory}/${suffixMeta}` + const metaFileUri = `${accountUrl}${suffixMeta}` + const metaData = await fetchData(metaFilePath) const metaGraph = $rdf.graph() $rdf.parse(metaData, metaGraph, metaFileUri, 'text/turtle') const webIdNode = metaGraph.any(undefined, SOLID('account'), $rdf.sym(accountUrl)) @@ -29,10 +26,3 @@ async function getWebId (accountDirectory, accountUrl, options = {}) { function isValidUsername (username) { return /^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(username) } - -function setupLDP (options) { - if (!options.ldp && !options.config) { - throw new Error('Require ldp or config set in options for getWebId') - } - return options.ldp || new LDP(options.config) -} diff --git a/lib/handlers/index.js b/lib/handlers/index.js index 4b9e20305..4ebf58f43 100644 --- a/lib/handlers/index.js +++ b/lib/handlers/index.js @@ -20,7 +20,7 @@ async function handler (req, res, next) { } // redirect to the right container if missing trailing / if (req.path.lastIndexOf('/') !== req.path.length - 1) { - return res.redirect(301, path.join(req.path, '/')) + return res.redirect(301, req.path + '/') } if (requestedType && requestedType.indexOf('text/html') !== 0) { @@ -30,7 +30,7 @@ async function handler (req, res, next) { // Check if file exists in first place await ldp.exists(req.hostname, path.join(req.path, indexFile)) - res.locals.path = path.join(req.path, indexFile) + res.locals.path = url.resolve(req.path, indexFile) debug('Found an index for current path') } catch (e) { // Ignore errors diff --git a/lib/ldp.js b/lib/ldp.js index d90c70400..15e05c37d 100644 --- a/lib/ldp.js +++ b/lib/ldp.js @@ -102,7 +102,7 @@ class LDP { const { path } = await this.resourceMapper.mapUrlToFile({ url }) return await promisify(fs.readFile)(path, {'encoding': 'utf8'}) } catch (err) { - throw error(404, "Can't read file") + throw error(err.status, err.message) } } @@ -284,7 +284,7 @@ class LDP { } const body = await response.text() - return promisify(parse)(body, uri, getContentType(response.headers)) + return parse(body, uri, getContentType(response.headers)) } /** diff --git a/lib/resource-mapper.js b/lib/resource-mapper.js index 905781eac..c6f577c32 100644 --- a/lib/resource-mapper.js +++ b/lib/resource-mapper.js @@ -3,6 +3,7 @@ const URL = require('url') const { promisify } = require('util') const { types, extensions } = require('mime-types') const readdir = promisify(fs.readdir) +const HTTPError = require('./http-error') // A ResourceMapper maintains the mapping between HTTP URLs and server filenames, // following the principles of the “sweet spot” discussed in @@ -79,7 +80,7 @@ class ResourceMapper { if (isIndex) { match = '' } else { - throw new Error(`File not found: ${fullPath}`) + throw new HTTPError(404, `File not found: ${fullPath}`) } } path = `${folder}${match}` diff --git a/lib/utils.js b/lib/utils.js index d7bcd7b36..21e5f6870 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -10,6 +10,7 @@ module.exports.routeResolvedFile = routeResolvedFile module.exports.getQuota = getQuota module.exports.overQuota = overQuota module.exports.getContentType = getContentType +module.exports.parse = parse const fs = require('fs') const path = require('path') @@ -69,6 +70,22 @@ function debrack (s) { return s.substring(1, s.length - 1) } +async function parse (data, baseUri, contentType) { + const graph = $rdf.graph() + return new Promise((resolve, reject) => { + try { + return $rdf.parse(data, graph, baseUri, contentType, (err, str) => { + if (err) { + return reject(err) + } + resolve(str) + }) + } catch (err) { + return reject(err) + } + }) +} + function pathBasename (fullpath) { let bname = '' if (fullpath) { diff --git a/test/resources/accounts-acl/tim.localhost/write-acl/test-file$.txt b/test/resources/accounts-acl/tim.localhost/write-acl/test-file$.txt new file mode 100644 index 000000000..ce002f06a --- /dev/null +++ b/test/resources/accounts-acl/tim.localhost/write-acl/test-file$.txt @@ -0,0 +1 @@ + . \ No newline at end of file diff --git a/test/resources/acl-tls/write-acl/test-file$.ttl b/test/resources/acl-tls/write-acl/test-file$.ttl new file mode 100644 index 000000000..ce002f06a --- /dev/null +++ b/test/resources/acl-tls/write-acl/test-file$.ttl @@ -0,0 +1 @@ + . \ No newline at end of file diff --git a/test/unit/user-utils-test.js b/test/unit/user-utils-test.js index cd0af1fd8..67dea178d 100644 --- a/test/unit/user-utils-test.js +++ b/test/unit/user-utils-test.js @@ -16,24 +16,22 @@ describe('user-utils', () => { }) it('should return name from graph', async () => { - const returnedName = await userUtils.getName(webId, { ldp }) + const returnedName = await userUtils.getName(webId, ldp.fetchGraph) expect(returnedName).to.equal(name) }) }) describe('getWebId', () => { - let ldp + let fetchGraph const webId = 'https://test.localhost:8443/profile/card#me' + const suffixMeta = '.meta' beforeEach(() => { - ldp = { - readContainerMeta: () => Promise.resolve(`<${webId}> .`), - suffixMeta: '.meta' - } + fetchGraph = () => Promise.resolve(`<${webId}> .`) }) it('should return webId from meta file', async () => { - const returnedWebId = await userUtils.getWebId('foo', 'https://bar/', { ldp }) + const returnedWebId = await userUtils.getWebId('foo', 'https://bar/', suffixMeta, fetchGraph) expect(returnedWebId).to.equal(webId) }) })