Commit 7af6ef1c authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Refactored infrastructure for staging.

parent 836f1727
...@@ -47,6 +47,9 @@ COPY --from=dependencies /install /install ...@@ -47,6 +47,9 @@ COPY --from=dependencies /install /install
COPY . /app COPY . /app
WORKDIR /app WORKDIR /app
RUN pip install -e . RUN pip install -e .
WORKDIR /app/docs
RUN make html
WORKDIR /app
RUN useradd -ms /bin/bash nomad RUN useradd -ms /bin/bash nomad
RUN mkdir -p /app/.volumes/fs; chown -R nomad /app/.volumes/fs RUN mkdir -p /app/.volumes/fs; chown -R nomad /app/.volumes/fs
USER nomad USER nomad
......
build/
node_modules/
\ No newline at end of file
# build environment
FROM node:9.6.1 as builder
RUN mkdir /app
WORKDIR /app
ENV PATH /app/node_modules/.bin:$PATH
COPY package.json /app/package.json
RUN npm install --silent
RUN npm install react-scripts@1.1.1 -g --silent
COPY . /app
RUN npm run build
# production environment
FROM nginx:1.13.9-alpine
COPY --from=builder /app/build /usr/share/nginx/html
EXPOSE 80
CMD ["nginx", "-g", "daemon off;"]
\ No newline at end of file
../../docs/.build/html
\ No newline at end of file
...@@ -23,7 +23,7 @@ function App() { ...@@ -23,7 +23,7 @@ function App() {
<Route path="/archive/:uploadHash/:calcHash" component={ArchiveCalc} /> <Route path="/archive/:uploadHash/:calcHash" component={ArchiveCalc} />
<Route path="/enc" render={() => <div>In the future, you'll see chart'n'stuff for your calculations and materials.</div>} /> <Route path="/enc" render={() => <div>In the future, you'll see chart'n'stuff for your calculations and materials.</div>} />
<Route path="/profile" render={() => <div>Profile</div>} /> <Route path="/profile" render={() => <div>Profile</div>} />
<Route path="/documentation" component={Documentation} /> <Route path="/docs" component={Documentation} />
<Route render={() => <div>Not found</div>} /> <Route render={() => <div>Not found</div>} />
</Switch> </Switch>
</Navigation> </Navigation>
......
import React, { Component } from 'react' import React, { Component } from 'react'
import HtmlToReact from 'html-to-react' import HtmlToReact from 'html-to-react'
import { withRouter } from 'react-router-dom'; import { withRouter } from 'react-router-dom'
import { HashLink as Link } from 'react-router-hash-link' import { HashLink as Link } from 'react-router-hash-link'
import './Documentation.css' import './Documentation.css'
import Url from 'url-parse' import Url from 'url-parse'
import { apiBase } from '../config'
const processNodeDefinitions = new HtmlToReact.ProcessNodeDefinitions(React) const processNodeDefinitions = new HtmlToReact.ProcessNodeDefinitions(React)
const processingInstructions = location => { const processingInstructions = location => {
...@@ -22,11 +23,11 @@ const processingInstructions = location => { ...@@ -22,11 +23,11 @@ const processingInstructions = location => {
shouldProcessNode: node => node.type === 'tag' && node.name === 'a' && node.attribs['href'] && !node.attribs['href'].startsWith('http'), shouldProcessNode: node => node.type === 'tag' && node.name === 'a' && node.attribs['href'] && !node.attribs['href'].startsWith('http'),
processNode: (node, children) => { processNode: (node, children) => {
const linkUrl = Url(node.attribs['href']) const linkUrl = Url(node.attribs['href'])
let pathname = linkUrl.pathname.replace(/^\/documentation/, '').replace(/^\//, '') let pathname = linkUrl.pathname.replace(/^\/docs/, '').replace(/^\//, '')
if (pathname === '' ) { if (pathname === '' ) {
pathname = location.pathname pathname = location.pathname
} else { } else {
pathname = `/documentation/${pathname}` pathname = `/docs/${pathname}`
} }
return (<Link smooth to={pathname + (linkUrl.hash || '#')}>{children}</Link>) return (<Link smooth to={pathname + (linkUrl.hash || '#')}>{children}</Link>)
} }
...@@ -35,7 +36,7 @@ const processingInstructions = location => { ...@@ -35,7 +36,7 @@ const processingInstructions = location => {
// We have to redirect img src attributes to the static sphynx build dir. // We have to redirect img src attributes to the static sphynx build dir.
shouldProcessNode: node => node.type === 'tag' && node.name === 'img' && node.attribs['src'] && !node.attribs['src'].startsWith('http'), shouldProcessNode: node => node.type === 'tag' && node.name === 'img' && node.attribs['src'] && !node.attribs['src'].startsWith('http'),
processNode: (node, children) => { processNode: (node, children) => {
node.attribs['src'] = `/docs/${node.attribs['src']}` node.attribs['src'] = `${apiBase}/docs/${node.attribs['src']}`
return processNodeDefinitions.processDefaultNode(node) return processNodeDefinitions.processDefaultNode(node)
} }
}, },
...@@ -56,7 +57,10 @@ class Documentation extends Component { ...@@ -56,7 +57,10 @@ class Documentation extends Component {
onRouteChanged() { onRouteChanged() {
const fetchAndUpdate = path => { const fetchAndUpdate = path => {
fetch(`/docs/${path}`) if (path === '' || path.startsWith('#')) {
path = 'index.html' + path
}
fetch(`${apiBase}/docs/${path}`)
.then(response => response.text()) .then(response => response.text())
.then(content => { .then(content => {
// extract body of html page // extract body of html page
...@@ -78,7 +82,7 @@ class Documentation extends Component { ...@@ -78,7 +82,7 @@ class Documentation extends Component {
} }
}) })
} }
const path = this.props.location.pathname.replace('/documentation/', '') const path = this.props.location.pathname.replace('/docs/', '')
fetchAndUpdate(path) fetchAndUpdate(path)
} }
......
...@@ -29,7 +29,7 @@ const toolbarTitles = { ...@@ -29,7 +29,7 @@ const toolbarTitles = {
'/repo': 'Raw Code Outputs', '/repo': 'Raw Code Outputs',
'/upload': 'Upload Your Own Data', '/upload': 'Upload Your Own Data',
'/profile': 'Your Profile', '/profile': 'Your Profile',
'/documentation': 'Documentation', '/docs': 'Documentation',
'/archive': 'Code Independent Data', '/archive': 'Code Independent Data',
'/enc': 'The Material Perspective' '/enc': 'The Material Perspective'
} }
...@@ -39,7 +39,7 @@ const toolbarThemes = { ...@@ -39,7 +39,7 @@ const toolbarThemes = {
'/repo': repoTheme, '/repo': repoTheme,
'/upload': repoTheme, '/upload': repoTheme,
'/profile': genTheme, '/profile': genTheme,
'/documentation': genTheme, '/docs': genTheme,
'/archive': archiveTheme, '/archive': archiveTheme,
'/enc': encTheme '/enc': encTheme
} }
...@@ -141,7 +141,7 @@ class Navigation extends React.Component { ...@@ -141,7 +141,7 @@ class Navigation extends React.Component {
</ListItemIcon> </ListItemIcon>
<ListItemText inset primary="Profil"/> <ListItemText inset primary="Profil"/>
</MenuItem> </MenuItem>
<MenuItem component={Link} to="/documentation" selected={ '/documentation' === pathname }> <MenuItem component={Link} to="/docs" selected={ '/docs' === pathname }>
<ListItemIcon> <ListItemIcon>
<DocumentationIcon /> <DocumentationIcon />
</ListItemIcon> </ListItemIcon>
......
...@@ -4,7 +4,7 @@ import enc from '@material-ui/core/colors/amber'; ...@@ -4,7 +4,7 @@ import enc from '@material-ui/core/colors/amber';
import secondary from '@material-ui/core/colors/blueGrey'; import secondary from '@material-ui/core/colors/blueGrey';
import { createMuiTheme } from '@material-ui/core'; import { createMuiTheme } from '@material-ui/core';
export const apiBase = 'http://localhost:5000' export const apiBase = 'http://localhost:8000'
export const genTheme = createMuiTheme({ export const genTheme = createMuiTheme({
palette: { palette: {
......
MINIO_HOST_PORT=9007
RABBITMQ_HOST_PORT=5672
REDIS_HOST_PORT=6379
ELASTIC_HOST_PORT=9200
MONGO_HOST_PORT=27017
KIBANA_HOST_PORT=5601
API_HOST_PORT=8000
GUI_HOST_PORT=8005
\ No newline at end of file
...@@ -22,10 +22,10 @@ services: ...@@ -22,10 +22,10 @@ services:
# image: minio/minio # image: minio/minio
container_name: nomad-xt-minio container_name: nomad-xt-minio
ports: ports:
- 9007:9000 - ${MINIO_HOST_PORT}:9000
volumes: volumes:
- ../.volumes/minio:/data - ../../.volumes/minio:/data
- ./config/minio:/root/.minio - ../config/minio:/root/.minio
environment: environment:
- "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" - "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE"
- "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" - "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
...@@ -42,7 +42,7 @@ services: ...@@ -42,7 +42,7 @@ services:
- "RABBITMQ_DEFAULT_PASS=rabbitmq" - "RABBITMQ_DEFAULT_PASS=rabbitmq"
- "RABBITMQ_DEFAULT_VHOST=/" - "RABBITMQ_DEFAULT_VHOST=/"
ports: ports:
- 5672:5672 - {RABITMQ_HOST_PORT}:5672
# results backend for celery # results backend for celery
redis: redis:
...@@ -50,18 +50,18 @@ services: ...@@ -50,18 +50,18 @@ services:
container_name: nomad-xt-redis container_name: nomad-xt-redis
command: redis-server --appendonly yes command: redis-server --appendonly yes
volumes: volumes:
- '../.volumes/redis:/data' - '../../.volumes/redis:/data'
ports: ports:
- '6379:6379' - '{REDIS_HOST_PORT}:6379'
# the search engine # the search engine
elastic: elastic:
image: docker.elastic.co/elasticsearch/elasticsearch:6.3.2 image: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
container_name: nomad-xt-elastic container_name: nomad-xt-elastic
volumes: volumes:
- '../.volumes/elastic:/usr/share/elasticsearch/data' - '../../.volumes/elastic:/usr/share/elasticsearch/data'
ports: ports:
- 9200:9200 - '{ELASTIC_HOST_PORT}:9200'
# the user data db # the user data db
mongo: mongo:
...@@ -71,25 +71,24 @@ services: ...@@ -71,25 +71,24 @@ services:
- MONGO_DATA_DIR=/data/db - MONGO_DATA_DIR=/data/db
- MONGO_LOG_DIR=/dev/null - MONGO_LOG_DIR=/dev/null
volumes: volumes:
- '../.volumes/mongo:/data/db' - '../../.volumes/mongo:/data/db'
ports: ports:
- 27017:27017 - '{MONGO_HOST_PORT}:27017'
command: mongod --smallfiles --logpath=/dev/null # --quiet command: mongod --smallfiles --logpath=/dev/null # --quiet
# used for centralized logging # used for centralized logging
# elk: elk:
# restart: always restart: always
# build: ./elk/ build: ./elk/
# container_name: nomad-xt-elk container_name: nomad-xt-elk
# ports: expose:
# - 5601:5601 # kibana web - 5000 # logstash beats
# - 9201:9200 # elastic search api - 5044 # logstash tcp
# # - 9300:9300 # elastic transport api, not needed by other services or host ports:
# - 5044:5044 # logstash beats - '{KIBANA_HOST_PORT}':5601 # kibana web
# - 5000:5000 # logstash tcp
# nomad processing worker # nomad processing worker
nomad-worker: worker:
build: ../ build: ../
container_name: nomad-xt-worker container_name: nomad-xt-worker
environment: environment:
...@@ -108,21 +107,20 @@ services: ...@@ -108,21 +107,20 @@ services:
- mongo - mongo
# - elk # - elk
volumes: volumes:
- '../.volumes/fs:/app/.volumes/fs' - '../../.volumes/fs:/app/.volumes/fs'
command: python -m celery worker -l info -A nomad.processing command: python -m celery worker -l info -A nomad.processing
# nomad upload handler # nomad upload handler
nomad-handler: handler:
build: ../ build: ../
container_name: nomad-xt-handler
environment: environment:
- NOMAD_MINIO_PORT=9000 - NOMAD_MINIO_PORT=9000
- NOMAD_MINIO_HOST=minio - NOMAD_MINIO_HOST=minio
- NOMAD_RABBITMQ_HOST=rabbitmq - NOMAD_RABBITMQ_HOST=rabbitmq
- NOMAD_REDIS_HOST=redis - NOMAD_REDIS_HOST=redis
- NOMAD_LOGSTASH_HOST=elk - NOMAD_LOGSTASH_HOST=elk
- NOMAD_ELASTIC_HOST=elastic
- NOMAD_MONGO_HOST=mongo - NOMAD_MONGO_HOST=mongo
- NOMAD_ELASTIC_HOST=elastic
links: links:
- minio - minio
- redis - redis
...@@ -130,6 +128,34 @@ services: ...@@ -130,6 +128,34 @@ services:
- elastic - elastic
- mongo - mongo
# - elk # - elk
volumes:
- '../.volumes/fs:/app/.volumes/fs'
command: python -m nomad.processing.handlerdaemon command: python -m nomad.processing.handlerdaemon
# nomad api
api:
build: ../
environment:
- NOMAD_MINIO_PORT=9000
- NOMAD_MINIO_HOST=minio
- NOMAD_RABBITMQ_HOST=rabbitmq
- NOMAD_REDIS_HOST=redis
- NOMAD_LOGSTASH_HOST=elk
- NOMAD_ELASTIC_HOST=elastic
- NOMAD_MONGO_HOST=mongo
links:
- minio
- redis
- rabbitmq
- elastic
- mongo
# - elk
ports:
- ${API_HOST_PORT}:8000
command: python -m gunicorn.app.wsgiapp -w 4 -b 127.0.0.1:8000 nomad.api:app
# nomad gui
gui:
build: ../gui/
links:
- nomad-api
ports:
- ${GUI_HOST_PORT}:80
\ No newline at end of file
...@@ -10,7 +10,7 @@ from nomad import users, files, search ...@@ -10,7 +10,7 @@ from nomad import users, files, search
from nomad.processing import UploadProc from nomad.processing import UploadProc
from nomad.utils import get_logger from nomad.utils import get_logger
app = Flask(__name__) app = Flask(__name__, static_url_path='/docs', static_folder='../docs/.build/html')
CORS(app) CORS(app)
api = Api(app) api = Api(app)
......
...@@ -82,7 +82,7 @@ mongo = MongoConfig( ...@@ -82,7 +82,7 @@ mongo = MongoConfig(
users_db='users' users_db='users'
) )
logstash = LogstashConfig( logstash = LogstashConfig(
enabled=False, enabled=True,
host=os.environ.get('NOMAD_LOGSTASH_HOST', 'localhost'), host=os.environ.get('NOMAD_LOGSTASH_HOST', 'localhost'),
tcp_port=int(os.environ.get('NOMAD_LOGSTASH_TCPPORT', '5000')) tcp_port=int(os.environ.get('NOMAD_LOGSTASH_TCPPORT', '5000'))
) )
...@@ -12,3 +12,4 @@ ase ...@@ -12,3 +12,4 @@ ase
numpy numpy
cython>=0.19 cython>=0.19
spglib spglib
gunicorn
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment