diff --git a/app.js b/app.js
index 0dcd608bcbd725f3d8ad4d3e395aefb132ad4789..e749c308dddf05068ab9f92dd3058082409ab8fe 100644
--- a/app.js
+++ b/app.js
@@ -36,7 +36,7 @@ function main() {
const stringify = require('json-stringify-safe');
logger.info(`Started with arguments ${stringify(args)}`)
logger.info(`Using configuration ${config.util.getEnv('NODE_ENV')} for instance ${process.env["NODE_APP_INSTANCE"]} ${stringify(config, null, 2)}`);
- if (config.app.catchErrors) {
+ if (config.app.catchErrors || config.app.debug) {
process.on('uncaughtException', (err) => {
logger.error(`UncaughtException: ${stringify(err)}`)
})
diff --git a/app/ProxyRouter.js b/app/ProxyRouter.js
index 208fc30b0bbb8eea500e25161fb0fb42d83cc48f..ec785d489e60a1687ea4c46cc9e63e82275596f6 100644
--- a/app/ProxyRouter.js
+++ b/app/ProxyRouter.js
@@ -38,9 +38,11 @@ function guaranteeUserDir(userID, next) {
/// functions that either gives the running pod or starts it
function getOrCreatePod(podName, repl, shouldCreate, next) {
+ logger.debug(`enter getOrCreatePod ${podName}`)
k8.ns(config.k8component.namespace).pod.get(podName, function(err, result) {
if(err) {
if (shouldCreate) {
+ logger.debug(`creating ${podName}`)
components.templateForImage(repl, function(err, template, repl) {
if(err) {
logger.error(`Cannot start pod ${podName}, error in template generation: ${stringify(err)}`);
@@ -49,8 +51,9 @@ function getOrCreatePod(podName, repl, shouldCreate, next) {
guaranteeUserDir(repl.user, function (){
const templateValue = yaml.safeLoad(template, 'utf8')
k8.ns(config.k8component.namespace).pod.post({ body: templateValue}, function(err, res2){
+ logger.debug(`created ${podName}`)
if(err) {
- logger.error(`Cannot start pod ${podName}, error: ${stringify(err)}, \n====\ntemplate was ${template}\n====\nexpanded to\n${templateValue}\n====`);
+ logger.error(`Cannot start pod ${podName}, error: ${stringify(err)}, \n====\ntemplate was ${template}\n====\nconverted to\n${templateValue}\n====`);
next(err, null)
} else {
logger.info(`Created pod ${podName}: ${stringify(res2)}`)
@@ -73,8 +76,8 @@ function getOrCreatePod(podName, repl, shouldCreate, next) {
// cache pod name -> host & port
const resolveCache = require('../safe-memory-cache/map.js')({
- limit: config.resolveCacheNMax,
- maxTTL: config.resolveCacheTtlMaxMs,
+ limit: config.app.resolveCacheNMax,
+ maxTTL: config.app.resolveCacheTtlMaxMs,
refreshF: function(key, value, cache) {
}
})
@@ -108,22 +111,28 @@ function resolvePod(repl, next) {
resolveCache.set(podName, res)
next(null, res)
} else {
+ let secondsSinceCreation = (Date.parse(pod.metadata.creationTimestamp) - Date.now())/ 1000.0
const err = {
error: "not ready",
msg: "pod not yet ready",
status: pod.status,
host: podIp,
- port: portNr
+ port: portNr,
+ pod: pod,
+ secondsSinceCreation: secondsSinceCreation
}
next(err, null)
}
} else {
+ let secondsSinceCreation = (Date.parse(pod.metadata.creationTimestamp) - Date.now())/ 1000.0
const err = {
error: "no ip",
msg: "ip not yet available",
status: pod.status,
host: podIp,
- port: portNr
+ port: portNr,
+ pod: pod,
+ secondsSinceCreation: secondsSinceCreation
}
next(err, null)
}
@@ -145,24 +154,40 @@ function resolvePod(repl, next) {
ProxyRouter.prototype.lookup = function(req, res, userID, isWebsocket, path, next) {
var start = Date.now()
components.cachedReplacements(req, function(err, repl) {
- //logger.debug(`replacements available after ${(Date.now()-start)/1000.0}s`)
+ logger.debug(`replacements available after ${(Date.now()-start)/1000.0}s`)
if (err) {
- logger.error(`No replacements: lookup without visiting the entry point ${config.k8component.entryPoint.path} (${stringify(err)})`)
+ logger.error(`no replacements for ${userID} in %{path}`)
+ res.send(500, components.getHtmlErrorTemplate({
+ error:"No replacements",
+ msg: `lookup without visiting the entry point ${config.k8component.entryPoint.path} (${stringify(err)})`
+ }))
} else {
resolvePod(repl, function (err, target) {
- //logger.debug(`target available after ${(Date.now()-start)/1000.0}s`)
+ logger.debug(`target available after ${(Date.now()-start)/1000.0}s, err: ${stringify(err)} target: ${stringify(target)}`)
if (err) {
- if (err.error === 'no ip' && err.status && err.status.phase === 'Pending' ||
- err.error === 'not ready') {
- logger.warn(`pod ${repl.podName} ${err.error} ${stringify(err)}`)
- res.send(reloadMsg)
+ if ((err.error === 'no ip' || err.error === 'not ready') &&
+ err.status && err.status.phase === 'Pending') {
+ let error_detail = ''
+ if (!err.secondsSinceCreation || err.secondsSinceCreation > 10)
+ error_detail = stringify(err, null, 2)
+ logger.debug(`eval reload`)
+ let repl = {
+ refreshEachS: config.app.pageReloadTime,
+ error_detail: error_detail
+ }
+ logger.debug(`repl done ${stringify(repl)}`)
+ components.evalHtmlTemplate(
+ 'reloadMsg.html', repl,
+ function(err, pageHtml) {
+ res.send(pageHtml)
+ })
+ return;
} else {
- const errorMsg = `<html><head><title>Error starting Container!</title><meta http-equiv="refresh"<body><h3>Error ${err.error} while trying to start a container for you!</h3><p>${err.msg}</p><pre>${stringify(err, null, 2 )}</pre></body></html>`;
logger.error(`error starting container ${repl.podName}: ${stringify(err)}`)
- res.send(500, errorMsg)
+ res.send(500, components.getHtmlErrorTemplate(err, "Error starting container"))
}
} else {
- // logger.debug(`Resolved to ${stringify(target)} after ${(Date.now()-start)/1000.0}s`)
+ logger.debug(`Resolved to ${stringify(target)} after ${(Date.now()-start)/1000.0}s`)
next(target);
}
})
diff --git a/app/components.js b/app/components.js
index 167f5807da96e9694e6bf1aac7d27713168da638..980a47df268ae9b0c8b3f6811a1fb189782a72f0 100644
--- a/app/components.js
+++ b/app/components.js
@@ -10,6 +10,7 @@ const url = require('url');
const compact_sha = require('./compact-sha')
const logger = require('./logger')
const stringify = require('json-stringify-safe')
+const yaml = require('js-yaml')
var baseRepl = {
baseDir: baseDir,
@@ -139,20 +140,37 @@ function getHtmlErrorTemplate(err, context = '') {
// Helper to evaluate a web page template (layout + content)
// will *always* give an html as result (it there was an error it describe the error
-function evalHtmlTemplate(htmlPath, repl, next, layout = null, context = '') {
- const layout = repl.layout || "defaultTemplate.html"
- evalTemplate("html/"+htmlPath, repl, function (err, template){
+function evalHtmlTemplate(htmlPath, repl, next, { context = '' } = {} ) {
+ logger.debug('entered evalHtmlTemplate')
+ logger.debug(`entering evalHtmlTemplate(${stringify(htmlPath)}, ${stringify(repl)},...)`)
+ evalTemplate('html/'+htmlPath, repl, function (err, template){
+ logger.debug(`eval internal template err:${stringify(err)}, body:${stringify(template)}`)
if (err) {
+ logger.debug(`returning error`)
next(err, getHtmlErrorTemplate(err, context))
} else {
- const repl2 = Object.assign({title: htmlPath, head: ''}, repl, { body: template })
- evalHtmlTemplate("html/"+layout, repl2, function(err,res){
- if (err) {
- next(err, getHtmlErrorTemplate(err, context))
- } else {
- next(nil, res)
- }
- })
+ let extraRepl = {}
+ let templateBody = template
+ let m = /\B---\B/.exec(template)
+ if (m) {
+ templateBody = template.slice(m.index + 3)
+ extraRepl = yaml.safeLoad(template.slice(0, m.index), 'utf8')
+ }
+ logger.debug(`eval layout with extraRepl: ${stringify(extraRepl)} templateBody: ${stringify(templateBody)}`)
+ const repl2 = Object.assign({title: htmlPath, head: '', layout: "defaultLayout.html"}, extraRepl, repl, { body: templateBody })
+ const layout = repl2.layout
+ if (layout) {
+ evalTemplate("htmlLayout/"+layout, repl2, function(err,res){
+ logger.debug(`evaluated template, err: ${stringify(err)}`)
+ if (err) {
+ next(err, getHtmlErrorTemplate(err, context))
+ } else {
+ next(nil, res)
+ }
+ })
+ } else {
+ next(nil, templateBody)
+ }
}
})
}
@@ -276,5 +294,7 @@ module.exports = {
cachedReplacements: cachedReplacements,
podNameForRepl: podNameForRepl,
infoForPodName: infoForPodName,
- templateForImage: templateForImage
+ templateForImage: templateForImage,
+ getHtmlErrorTemplate: getHtmlErrorTemplate,
+ evalHtmlTemplate: evalHtmlTemplate
}
diff --git a/templates/html/reloadMsg.html b/templates/html/reloadMsg.html
index 1be3159853ba4746c25c9b60813801b1f8add14c..147f73aec20ff8e1e20333fd57831dccc5bed3e9 100644
--- a/templates/html/reloadMsg.html
+++ b/templates/html/reloadMsg.html
@@ -1,2 +1,8 @@
+title: "Starting up!"
+head: "<meta http-equiv=\"refresh\" content=\"{{refreshEachS}}\" >"
+---
<h3>Please wait while we start a container for you!</h3>
<p>You might need to refresh manually (F5)...</p>p>
+<pre>
+{{error_detail}}
+</pre>
diff --git a/templates/html/defaultLayout.html b/templates/htmlLayout/defaultLayout.html
similarity index 100%
rename from templates/html/defaultLayout.html
rename to templates/htmlLayout/defaultLayout.html
diff --git a/templates/kube/defaultTemplate.yaml b/templates/kube/defaultTemplate.yaml
index 32a847b4b104b788bf4f7f3e559c9cde41a1ee5d..1e827a526b6b898cda5624f8db134431396accb6 100644
--- a/templates/kube/defaultTemplate.yaml
+++ b/templates/kube/defaultTemplate.yaml
@@ -11,6 +11,7 @@ metadata:
spec:
imagePullSecrets:
- name: garching-kube
+ restartPolicy: Never
containers:
- image: "{{image}}"
name: "{{imageType}}"
diff --git a/templates/kube/remoteVisTemplate.yaml b/templates/kube/remoteVisTemplate.yaml
index 3886b39c091e898f575242a82d3cda017d33beca..f30780750b64c669b69e7d4a5231f93d33ede0ed 100644
--- a/templates/kube/remoteVisTemplate.yaml
+++ b/templates/kube/remoteVisTemplate.yaml
@@ -4,6 +4,7 @@ metadata:
name: {{podName}}
spec:
terminationGracePeriodSeconds: 5
+ restartPolicy: Never
containers:
- image: labdev-nomad.esc.rzg.mpg.de:5000/nomadlab/nomadvis:v1.0.7
name: nomadvis