Skip to content

GitLab

Commit 24d66718 authored by Mohamed, Fawzi Roberto (fawzi)'s avatar Mohamed, Fawzi Roberto (fawzi)
Browse files

* consolidate k8 ops in k8-data 

* better handling of failures
* entry point fixes
parent 128fe14d
Hide whitespace changes
Inline Side-by-side
app/ProxyRouter.js
View file @ 24d66718
const config = require('config')
const stringify = require('json-stringify-safe');
const http = require('http');
const fs = require('fs');
const components = require('./components');
const yaml = require('js-yaml')
const k8 = require('./kubernetes');
const k8D = require('./k8-data');
const logger = require('./logger')
function guaranteeDir(path, next) {
fs.access(path, fs.constants.F_OK | fs.constants.R_OK, (err) => {
if(err){
fs.mkdir(path, parseInt('2775', 8), (err) => {
if(err) throw err;
fs.chown(path, 1000, 1000, (err) => {
if (err)
logger.warn('Dir '+ path + ' created, error in chown: ' + stringify(err));
else
logger.info('Dir correctly created:' + path);
next();
});
});
} else {
next();
}
});
}
function guaranteeUserDir(userID, next) {
//Async version needs to be tested thorougly
guaranteeDir(config.userInfo.sharedDir + '/' + userID, function() {
guaranteeDir(config.userInfo.privateDir + '/' + userID, function() {
next();
});
});
}
/// functions that either gives the running pod or starts it
function getOrCreatePod(podName, repl, shouldCreate, next) {
k8.ns(config.k8component.namespace).pod.get(podName, function(err, result) {
if(err) {
if (shouldCreate) {
logger.info(`creating ${podName}`)
components.templateForImage(repl, function(err, template, repl) {
if(err) {
logger.error(`Cannot start pod ${podName}, error in template generation: ${stringify(err)}`);
next(err, null)
} else {
guaranteeUserDir(repl.user, function (){
const templateValue = yaml.safeLoad(template, 'utf8')
k8.ns(config.k8component.namespace).pod.post({ body: templateValue}, function(err, res2){
if(err) {
logger.error(`Cannot start pod ${podName}, error: ${stringify(err)}, \n====\ntemplate was ${template}\n====`);
next(err, null)
} else {
logger.info(`Created pod ${podName}: ${stringify(res2)}`)
next(null, res2)
}
})
})
}
});
} else {
logger.error(`Requested pod ${podName} which does not exist and should not be created, error: ${stringify(err)}`);
next(err, null)
}
} else {
//logger.debug(`looked up ${podName}: ${stringify(result)}`)
next(null, result)
}
});
}
// cache pod name -> host & port
const resolveCache = require('../safe-memory-cache/map.js')({
limit: config.app.resolveCacheNMax,
maxTTL: config.app.resolveCacheTtlMaxMs,
refreshF: function(key, value, cache) {
}
})
function resolvePod(repl, next) {
const podName = components.podNameForRepl(repl)
var v = resolveCache.get(podName)
if (v === undefined) {
getOrCreatePod(podName, repl, config.k8component.image.autoRestart, function (err, pod) {
if (err) {
next(err, null)
} else {
const portNr = pod.spec.containers[0].ports[0].containerPort
const podIp = pod.status.podIP
if (podIp) {
var ready = false
const conds = pod.status.conditions
if (pod.status && conds) {
for (icond in conds) {
let cond = conds[icond]
if (cond.type === 'Ready' && cond.status === 'True')
ready = true
}
}
if (ready) {
const res = {
host: podIp,
port: portNr
}
//logger.debug(`got ${stringify(res)} out of pod ${stringify(pod)}`)
resolveCache.set(podName, res)
next(null, res)
} else {
let secondsSinceCreation = (Date.now() - Date.parse(pod.metadata.creationTimestamp))/ 1000.0
const err = {
error: "not ready",
msg: "pod not yet ready",
status: pod.status,
host: podIp,
port: portNr,
pod: pod,
secondsSinceCreation: secondsSinceCreation
}
next(err, null)
}
} else {
let secondsSinceCreation = (Date.now() - Date.parse(pod.metadata.creationTimestamp))/ 1000.0
const err = {
error: "no ip",
msg: "ip not yet available",
status: pod.status,
host: podIp,
port: portNr,
pod: pod,
secondsSinceCreation: secondsSinceCreation
}
next(err, null)
}
}
})
} else {
next(null, v)
}
}
var ProxyRouter = function(options) {
if (!options.backend) {
throw "ProxyRouter backend required. Please provide options.backend parameter!";
......@@ -159,7 +24,7 @@ ProxyRouter.prototype.lookup = function(req, res, userID, isWebsocket, path, nex
msg: `lookup without visiting the entry point ${config.k8component.entryPoint.path} (${stringify(err)})`
}))
} else {
resolvePod(repl, function (err, target) {
k8D.resolvePod(repl, function (err, target) {
//logger.debug(`target available after ${(Date.now()-start)/1000.0}s, err: ${stringify(err)} target: ${stringify(target)}`)
if (err) {
if ((err.error === 'no ip' || err.error === 'not ready') &&
......@@ -178,10 +43,15 @@ ProxyRouter.prototype.lookup = function(req, res, userID, isWebsocket, path, nex
res.send(pageHtml)
})
return;
} else if (err.error === 'too many containers') {
components.evalHtmlTemplate("maxContainers.html", {
pods: pods
}, function(err, errorHtml) {
repl.status(503).send(errorHtml)
})
} else {
logger.error(`error starting container ${repl.podName}: ${stringify(err)}`)
let errorHtml = components.getHtmlErrorTemplate(err, "Error starting container")
logger.error(`errorHtml: ${stringify(errorHtml)}`)
if (res && res.status && res.send)
res.status(500).send(errorHtml)
}
......
app/components.js
View file @ 24d66718
......@@ -17,6 +17,9 @@ var baseRepl = {
baseUri: config.app.baseUri,
baseUriPath: url.parse(config.app.baseUri).path
};
// ensure that baseUriPath does not end with /
if (baseRepl.baseUriPath.endsWith('/'))
baseRepl.baseUriPath = baseRepl.baseUriPath.slice(0, baseRepl.baseUriPath.length -1)
const br = config.app.baseReplacements
for (k in br)
baseRepl[k] = br[k];
......@@ -86,29 +89,14 @@ function loadTemplate(templatePath, next) {
}
}
// evaluates a template, here only the most basic replacements are given, you normally need to pass in extraRepl.
// evaluates a template, here only the most basic replacements are given, you normally need to pass in replacements.
// calls next with the resolved template plus all replacements defined
function evalTemplate(templatePath, extraRepl, next) {
function evalTemplate(templatePath, replacements, next) {
loadTemplate(templatePath, function (err, template) {
if (err) {
next(err, null, undefined)
} else {
const repl = Object.assign({}, extraRepl, baseRepl)
const res = template(repl)
//logger.debug(`evaluating <<${templatePath}>> with ${stringify(repl)} gives <<${res}>>`)
next(null, res, repl)
}
})
}
// evaluates a template, here only the most basic replacements are given, you normally need to pass in extraRepl.
// calls next with the resolved template plus all replacements defined
function evalTemplate(templatePath, extraRepl, next) {
loadTemplate(templatePath, function (err, template) {
if (err) {
next(err, null, undefined)
} else {
const repl = Object.assign({}, extraRepl, baseRepl)
const repl = Object.assign({}, baseRepl, replacements)
const res = template(repl)
//logger.debug(`evaluating <<${templatePath}>> with ${stringify(repl)} gives <<${res}>>`)
next(null, res, repl)
......@@ -219,7 +207,9 @@ function infoForPodName(podName) {
}
/// gives the replacements for the user
function replacementsForUser(user, extraRepl, next) {
/// overridingRepl have highest priority,
/// injectedRepl are replacements applied last but respecting the protected keys
function replacementsForUser(user, overridingRepl, injectedRepl, next) {
var repl = {}
var keysToProtect = new Set()
var toSkip
......@@ -240,16 +230,27 @@ function replacementsForUser(user, extraRepl, next) {
let imageType = cconfig.image.imageType
const userRepl = userSettings.getAppSetting(user, 'image:' + imageType)
addRepl(userRepl)
// extraRepl overrides even protected values
if (extraRepl)
for (k in extraRepl)
repl[k] = extraRepl[k]
addRepl(injectedRepl)
// overridingRepl overrides even protected values
if (overridingRepl)
for (k in overridingRepl)
repl[k] = overridingRepl[k]
// "real" user imageType and podName overrided everything
repl['user'] = user
repl['imageType'] = imageType
repl['podName'] = podNameForRepl(repl)
delete repl.replacementsChecksum
repl.replacementsChecksum = compact_sha.objectSha(repl)
let replToChecksum = repl
if (repl.checksumSkipReStr) {
let re = new RegExp(repl.checksumSkipReStr)
replToChecksum = {}
for (var k in repl) {
let m = re.exec(k)
if (!m)
replToChecksum[k] = repl[k]
}
}
repl.replacementsChecksum = compact_sha.objectSha(replToChecksum)
next(null, repl)
}
......@@ -274,7 +275,7 @@ function cachedReplacements(req, next) {
if (repl) {
next(null, repl)
} else if (!cconfig.entryPoint.exclusiveStartPoint) {
replacementsForUser(selfUserName(req), {}, function(err, newRepl) {
replacementsForUser(selfUserName(req), {}, {}, function(err, newRepl) {
if (!req.session.replacements)
req.session.replacements = {}
req.session.replacements[imageType] = newRepl
......
app/k8-data.js
View file @ 24d66718
const config = require('config')
const k8 = require('./kubernetes');
const stringify = require('json-stringify-safe');
const fs = require('fs');
const components = require('./components');
const yaml = require('js-yaml')
const logger = require('./logger')
// cache pod name -> host & port
const resolveCache = require('../safe-memory-cache/map.js')({
limit: config.app.resolveCacheNMax,
maxTTL: config.app.resolveCacheTtlMaxMs,
refreshF: function(key, value, cache) {
}
})
// gets pods with the given labels
exports.getPods = function(labels, next) {
function getPods(labels, next) {
let selector = ""
let first = true
for (k in labels) {
......@@ -14,3 +27,275 @@ exports.getPods = function(labels, next) {
}
k8.ns(config.k8component.namespace).pods.get({ qs: { labelSelector: selector } }, next)
}
// gets json api formatted pods
// if details is true gives detailed information on the pod
function jsonApiPods(labels, next, {details = true}={}) {
getPods(labels, function(err, pods) {
if (err) {
next(err, [])
} else {
let podList = pods.items
if (podList)
podList = podList.map(function(pod){
let secondsSinceCreation = (Date.now() - Date.parse(pod.metadata.creationTimestamp))/ 1000.0
let time = secondsSinceCreation
let unit = "s"
if (time > 60) {
time = time / 60.0
unit = "m"
if (time > 60) {
time = time / 60.0
unit = "h"
if (time > 24) {
time = time / 24.0
unit = "d"
}
}
}
let status = "danger"
if (pod.metadata.deletionTimestamp) {
status = "danger"
else if (pod.status && pod.status.phase === 'Pending') {
status = "warning"
} else if (pod.status && pod.status.phase === 'Running') {
const conds = pod.status.conditions
let ready = false
if (pod.status && conds) {
for (icond in conds) {
let cond = conds[icond]
if (cond.type === 'Ready' && cond.status === 'True')
ready = true
}
if (ready)
status = "success"
else
status = "warning"
}
}
let podInfo = {
id: pod.metadata.name,
type: 'pod',
attributes: {
name: pod.metadata.name,
time: `${time.toFixed(1)} ${unit}`,
status: status
}
}
if (details)
podInfo.attributes.data = pod
return podInfo
})
next(null, podList)
}
})
}
/// Guarantees the existence of a directory
function guaranteeDir(path, next) {
fs.access(path, fs.constants.F_OK | fs.constants.R_OK, (err) => {
if(err){
fs.mkdir(path, parseInt('2775', 8), (err) => {
if(err) throw err;
fs.chown(path, 1000, 1000, (err) => {
if (err)
logger.warn('Dir '+ path + ' created, error in chown: ' + stringify(err));
else
logger.info('Dir correctly created:' + path);
next();
});
});
} else {
next();
}
});
}
/// Guarantees the existence of user dirs for the given user
function guaranteeUserDir(userID, next) {
//Async version needs to be tested thorougly
guaranteeDir(config.userInfo.sharedDir + '/' + userID, function() {
guaranteeDir(config.userInfo.privateDir + '/' + userID, function() {
next();
});
});
}
/// creates a pod
function createPod(podName, repl, next) {
if (repl.imageReStr && repl.image) {
let re = new RegExp(repl.imageReStr)
if (!re.exec(repl.image)) {
let err = {
error: 'invalid image',
detail: `Refusing to start pod ${podName} with non acceptable image ${repl.image}`
}
logger.warn(err.detail)
next(err, null)
return;
}
}
logger.info(`creating ${podName}`)
components.templateForImage(repl, function(err, template, repl) {
if(err) {
logger.error(`Cannot start pod ${podName}, error in template generation: ${stringify(err)}`);
next(err, null)
} else {
guaranteeUserDir(repl.user, function (){
jsonApiPods({user:repl.user}, function(err, pods) {
if (pods.length >= config.k8component.maxContainersPerUser) {
next({
error: 'too many containers',
msg: `Reached the maximum number of running containers for ${repl.user}: ${config.k8component.maxContainersPerUser}`,
pods: pods
}, null)
} else {
const templateValue = yaml.safeLoad(template, 'utf8')
k8.ns(config.k8component.namespace).pod.post({ body: templateValue}, function(err, res2){
if(err) {
logger.error(`Cannot start pod ${podName}, error: ${stringify(err)}, \n====\ntemplate was ${template}\n====`);
next(err, null)
} else {
logger.info(`Created pod ${podName}: ${stringify(res2)}`)
next(null, res2)
}
})
}
})
})
}
});
}
/// functions that either gives the running pod or starts it
function getOrCreatePod(podName, repl, shouldCreate, next) {
k8.ns(config.k8component.namespace).pod.get(podName, function(err, result) {
if(err || result &&
(result.status && ['Error', 'Failed', 'Succeeded'].includes(result.status.phase) ||
result.metadata && result.metadata.deletionTimestamp)) {
if (result && result.metadata && result.metadata.deletionTimestamp) {
let error = {
error: 'pod shutting down',
detail: `Pod ${podName} is shutting down, need to wait to restart`
}
logger.warn(error.detail)
next(error, null)
} else if (result && result.status && ['Error', 'Failed', 'Succeeded'].includes(result.status.phase)) {
if (shouldCreate) {
k8.ns(config.k8component.namespace).pods.delete({ name: podName }, function (err, result) {
if (!err) {
logger.info(`Deleted stopped pod ${podName} to restart it`)
createPod(podName, repl, next) // wait & return 'pod shutting down' instead?
} else {
let error = {
error: 'failed deleting pod',
detail: `Error deleting pod ${podName} while trying to restart it: ${stringify(err)}`
}
logger.warn(error.detail)
next(err, null)
}
});
} else {
let error = {
error: 'pod failed',
detail: `Requested pod ${podName} which failed but should not be created, error: ${stringify(err)}`
}
logger.error(error.detail);
next(error, null)
}
} else if (shouldCreate) {
createPod(podName, repl, next)
} else {
logger.error(`Requested pod ${podName} which does not exist and should not be created, error: ${stringify(err)}`);
next(err, null)
}
} else {
//logger.debug(`looked up ${podName}: ${stringify(result)}`)
next(null, result)
}
});
}
function resolvePod(repl, next) {
const podName = components.podNameForRepl(repl)
var v = resolveCache.get(podName)
if (v === undefined) {
getOrCreatePod(podName, repl, config.k8component.image.autoRestart, function (err, pod) {
if (err) {
next(err, null)
} else {
const portNr = pod.spec.containers[0].ports[0].containerPort
const podIp = pod.status.podIP
if (podIp) {
var ready = false
const conds = pod.status.conditions
if (pod.status && conds) {
for (icond in conds) {
let cond = conds[icond]
if (cond.type === 'Ready' && cond.status === 'True')
ready = true
}
}
if (pod.status.phase == 'Running' && ready) {
const res = {
host: podIp,
port: portNr
}
resolveCache.set(podName, res)
next(null, res)
} else {
let secondsSinceCreation = (Date.now() - Date.parse(pod.metadata.creationTimestamp))/ 1000.0
const err = {
error: "not ready",
msg: "pod not yet ready",
status: pod.status,
host: podIp,
port: portNr,
pod: pod,
secondsSinceCreation: secondsSinceCreation
}
next(err, null)
}
} else {
let secondsSinceCreation = (Date.now() - Date.parse(pod.metadata.creationTimestamp))/ 1000.0
const err = {
error: "no ip",
msg: "ip not yet available",
status: pod.status,
host: podIp,
port: portNr,
pod: pod,
secondsSinceCreation: secondsSinceCreation
}
next(err, null)
}
}
})
} else {
next(null, v)
}
}
function deletePod(podName, next) {
k8.ns(config.k8component.namespace).pods.delete({ name: podName }, function (err, result) {
resolveCache.set(podName, undefined)
if (!err) {
logger.info(`deleted pod ${podName}`)
next(null, result)
} else {
logger.warn(`Error deleting pod ${podName}: ${stringify(err)}`)
next(err, null)
}
})
}
module.exports = {
getPods: getPods,
jsonApiPods: jsonApiPods,
getOrCreatePod: getOrCreatePod,
guaranteeDir: guaranteeDir,
guaranteeUserDir: guaranteeUserDir,
resolvePod: resolvePod,
deletePod: deletePod
}
app/routes.js
View file @ 24d66718
......@@ -3,6 +3,7 @@ module.exports = function (app, redirect, config, proxyServer, proxyRouter, k8,
const logger = require('./logger')
const stringify = require('json-stringify-safe')
const k8D = require('./k8-data')
const compactSha = require('./compact-sha')
function setFrontendHeader() {
return function(req, res, next) {
......@@ -25,36 +26,144 @@ module.exports = function (app, redirect, config, proxyServer, proxyRouter, k8,
});
let cconf = config.k8component
let entryPath = components.templatize(cconf.entryPoint.path)(components.baseRepl)
logger.debug(`entryPoint: ${entryPath}`)
app.get(cconf.entryPoint.path, ensureLoggedIn('/login'), function(req, res){
extraArgs = object.create(req.query)
app.get(entryPath, ensureLoggedIn('/login'), bodyParser.json(), bodyParser.urlencoded({extended:true}), function(req, res){
function isEmpty(obj) {
for (var x in obj) { return false; }
return true;
}
extraArgs = Object.assign({},req.query)
if (!isEmpty(extraArgs) && !extraArgs.imageSubtype)
extraArgs.imageSubtype = `custom${compactSha.objectSha(extraArgs, prefix='').replace(/[-_]/g,'').slice(0,5).toLowerCase()}`
extraArgs.path = req.url
if (cconf.entryPoint.pathReStr) {
let re = new RegExp(cconf.entryPoint.pathReStr)
var iRe = 1
while (undefined !== re[iRe]) {
let iReStr = iRe.toString()
let pVal = re[iRe]
extraArgs["path" + iReStr] = pVal
extraArgs["escapedPath" + iReStr] = pVal.replace("/","%2F")
let reMatch = re.exec(req.url)
for (var iRe in reMatch) {