From f60f18ccfa1a6b4569b6900d424a49cd82499df7 Mon Sep 17 00:00:00 2001 From: Markus Scheidgen <markus.scheidgen@gmail.com> Date: Tue, 28 Apr 2020 22:30:05 +0200 Subject: [PATCH] Refactored the date histogram. --- gui/src/components/search/Search.js | 12 +- gui/src/components/search/SearchContext.js | 42 +- gui/src/components/search/UploadsChart.js | 479 ++++++++------------- gui/src/config.js | 3 +- nomad/app/api/repo.py | 32 +- 5 files changed, 238 insertions(+), 330 deletions(-) diff --git a/gui/src/components/search/Search.js b/gui/src/components/search/Search.js index 4c237c25fc..e62d17a494 100644 --- a/gui/src/components/search/Search.js +++ b/gui/src/components/search/Search.js @@ -15,7 +15,7 @@ import UploadList from './UploadsList' import GroupList from './GroupList' import ApiDialogButton from '../ApiDialogButton' import SearchIcon from '@material-ui/icons/Search' -import UploadsChart from './UploadsChart' +import UploadsHistogram from './UploadsChart' import QuantityHistogram from './QuantityHistogram' import SearchContext, { searchContext } from './SearchContext' import {objectFilter} from '../../utils' @@ -213,17 +213,13 @@ SearchEntry.propTypes = { ownerTypes: PropTypes.arrayOf(PropTypes.string) } -function UsersVisualization(props) { - const {domain, response: {metric}, setStatistics} = useContext(searchContext) +function UsersVisualization() { + const {setStatistics} = useContext(searchContext) useEffect(() => { setStatistics(['uploader']) }, []) return <div> - <Card> - <CardContent> - <UploadsChart metricsDefinitions={domain.searchMetrics}/> - </CardContent> - </Card> + <UploadsHistogram /> <QuantityHistogram quantity="uploader" title="Uploaders" /> </div> } diff --git a/gui/src/components/search/SearchContext.js b/gui/src/components/search/SearchContext.js index ffa47f739f..d403f508ff 100644 --- a/gui/src/components/search/SearchContext.js +++ b/gui/src/components/search/SearchContext.js @@ -9,6 +9,24 @@ import { useLocation, useHistory } from 'react-router-dom' import qs from 'qs' import * as searchQuantities from '../../searchQuantities.json' +const padDateNumber = number => String('00' + number).slice(-2) + +export const Dates = { + dateHistogramStartDate: '2014-12-15', + APIDate: date => date.toISOString(), + JSDate: date => new Date(date), + FormDate: date => { + date = new Date(date) + return `${date.getFullYear()}-${padDateNumber(date.getMonth())}-${padDateNumber(date.getDate())}` + }, + addSeconds: (date, interval) => new Date(date.getTime() + interval * 1000), + deltaSeconds: (from, end) => Math.round((new Date(end).getTime() - new Date(from).getTime()) / 1000), + intervalSeconds: (from, end, buckets) => Math.round((new Date(end).getTime() - new Date(from).getTime()) / (1000 * buckets)), + buckets: 50 +} + +searchQuantities['from_time'] = true +searchQuantities['until_time'] = true /** * A custom hook that reads and writes search parameters from the current URL. */ @@ -115,7 +133,8 @@ export default function SearchContext({initialRequest, initialQuery, query, chil // checks for necessity. It will update the response state, once the request has // been answered by the api. const runRequest = useCallback(() => { - const {metric, domainKey, owner} = requestRef.current + let dateHistogramInterval = null + const {metric, domainKey, owner, dateHistogram} = requestRef.current const domain = domains[domainKey] const apiRequest = { ...initialRequest, @@ -132,9 +151,23 @@ export default function SearchContext({initialRequest, initialQuery, query, chil ...requestRef.current.query, ...query } + if (dateHistogram) { + dateHistogramInterval = Dates.intervalSeconds( + apiQuery.from_time || Dates.dateHistogramStartDate, + apiQuery.until_time || new Date(), Dates.buckets) + apiQuery['date_histogram'] = true + apiQuery['interval'] = `${dateHistogramInterval}s` + } api.search(apiQuery) .then(newResponse => { - setResponse({...emptyResponse, ...newResponse, metric: metric}) + setResponse({ + ...emptyResponse, + ...newResponse, + metric: metric, + dateHistogramInterval: dateHistogramInterval, + from_time: apiQuery.from_time, + until_time: apiQuery.until_time + }) }).catch(error => { setResponse({...emptyResponse, metric: metric}) raiseError(error) @@ -174,6 +207,10 @@ export default function SearchContext({initialRequest, initialQuery, query, chil requestRef.current.groups = {...groups} }, [requestRef]) + const setDateHistogram = useCallback(dateHistogram => { + requestRef.current.dateHistogram = dateHistogram + }, [requestRef]) + const handleQueryChange = (changes, replace) => { if (changes.atoms && changes.atoms.length === 0) { changes.atoms = undefined @@ -219,6 +256,7 @@ export default function SearchContext({initialRequest, initialQuery, query, chil setOwner: setOwner, setStatisticsToRefresh: () => null, // TODO remove setStatistics: setStatistics, + setDateHistogram: setDateHistogram, update: runRequest } diff --git a/gui/src/components/search/UploadsChart.js b/gui/src/components/search/UploadsChart.js index e186e68705..d6236e35af 100644 --- a/gui/src/components/search/UploadsChart.js +++ b/gui/src/components/search/UploadsChart.js @@ -1,188 +1,118 @@ -import React from 'react' +import React, { useContext, useState, useEffect, useRef, useLayoutEffect, useCallback } from 'react' import PropTypes from 'prop-types' -import { withStyles, Select, MenuItem } from '@material-ui/core' -import Button from '@material-ui/core/Button' -import RefreshIcon from '@material-ui/icons/Refresh' +import { Select, MenuItem, Card, CardHeader, CardContent, makeStyles } from '@material-ui/core' import Grid from '@material-ui/core/Grid' import TextField from '@material-ui/core/TextField' import * as d3 from 'd3' -import { scaleBand, scalePow } from 'd3-scale' +import { scaleTime, scalePow } from 'd3-scale' import { nomadSecondaryColor } from '../../config.js' -import { searchContext } from './SearchContext' -import { compose } from 'recompose' -import { withApi } from '../api' - -class UploadsHistogramUnstyled extends React.Component { - static propTypes = { - classes: PropTypes.object.isRequired, - height: PropTypes.number.isRequired, - data: PropTypes.object, - metric: PropTypes.string.isRequired, - metricsDefinitions: PropTypes.object.isRequired, - onChanged: PropTypes.func.isRequired, - defaultScale: PropTypes.number - } - - static styles = theme => ({ - root: {}, - content: { - paddingTop: 10 - } - }) - - constructor(props) { - super(props) - this.state = { - scalePower: this.props.defaultScale || 1.0, - time: null, - from_time: 0, - until_time: 0 - } - - this.container = React.createRef() - this.svgEl = React.createRef() - } - - startDate = '2013-01-01' - - scales = [ - { - label: 'Linear', - value: 1.0 - }, - { - label: '1/2', - value: 0.5 - }, - { - label: '1/4', - value: 0.25 - }, - { - label: '1/8', - value: 0.25 - } - ] - - componentDidMount() { - const from_time = new Date(this.startDate).getTime() - const until_time = new Date().getTime() - this.handleTimeChange(from_time, 'from_time', 'all') - this.handleTimeChange(until_time, 'until_time', 'all') - } - - componentDidUpdate() { - this.updateChart() +import { searchContext, Dates } from './SearchContext' + +const useStyles = makeStyles(theme => ({ + root: { + marginTop: theme.spacing(2) + }, + content: { + paddingTop: 0, + position: 'relative', + height: 250 + }, + tooltip: { + textAlign: 'center', + position: 'absolute', + pointerEvents: 'none', + opacity: 0 + }, + tooltipContent: { + // copy of the material ui popper style + display: 'inline-block', + color: '#fff', + padding: '4px 8px', + fontSize: '0.625rem', + fontFamily: '"Roboto", "Helvetica", "Arial", sans-serif', + lineHeight: '1.4em', + borderRadius: '4px', + backgroundColor: '#616161' } - - handleQueryChange() { - const from_time = new Date(this.state.from_time) - const until_time = new Date(this.state.until_time) - this.props.onChanged(from_time.toISOString(), until_time.toISOString()) - } - - handleTimeChange(newTime, key, target) { - let date - if (!newTime) { - date = key === 'from_time' ? new Date(this.startDate) : new Date() - } else { - date = new Date(newTime) +})) + +export default function UploadsHistogram({title = 'Uploads over time', initialScale = 1, tooltips}) { + const classes = useStyles() + const containerRef = useRef() + const fromTimeFieldRef = useRef() + const untilTimeFieldRef = useRef() + const [scale, setScale] = useState(initialScale) + const {response, query, setQuery, domain, setDateHistogram} = useContext(searchContext) + + useEffect(() => { + setDateHistogram(true) + return () => { + setDateHistogram(false) } + }, []) - if (target === 'state' || target === 'all') { - if (key === 'from_time') { - if (this.state.from_time !== date.getTime()) { - this.setState({from_time: date.getTime()}) - } - } else if (key === 'until_time') { - if (this.state.until_time !== date.getTime()) { - this.setState({until_time: date.getTime()}) - } - } - } - - if (target === 'picker' || target === 'all') { - document.getElementById(key).value = date.toISOString().substring(0, 10) - } - } - - handleItemClicked(item, deltaT) { - const selected = item.time - if (selected === this.state.time) { - this.props.onChanged(null, null) - } else { - this.handleTimeChange(selected, 'from_time', 'all') - this.handleTimeChange(selected + deltaT, 'until_time', 'all') - this.handleQueryChange() - } - } - - resolveDate(name, deltaT) { - const date = new Date(parseInt(name, 10)) - const year = date.toLocaleDateString(undefined, {year: 'numeric'}) - const quarter = Math.floor((date.getMonth() + 3) / 3) - const month = date.toLocaleDateString(undefined, {month: 'short'}) - const week = (date) => { - const first = new Date(date.getFullYear(), 0, 1) - return Math.ceil((((date - first) / 86400000) + first.getDay() + 1) / 7) - } - const day = date.toLocaleDateString(undefined, {day: 'numeric'}) - const hour = date.toLocaleTimeString(undefined, {hour: 'numeric'}) - const min = date.toLocaleTimeString(undefined, {minute: 'numeric'}) - const sec = date.toLocaleTimeString(undefined, {second: 'numeric'}) - - const times = [31536000, 7776000, 2419200, 604800, 864000, 3600, 60, 1] - const diffs = times.map(t => Math.abs(t - (deltaT / 1000))) + useLayoutEffect(() => { + fromTimeFieldRef.current.value = Dates.FormDate(query.from_time || Dates.dateHistogramStartDate) + untilTimeFieldRef.current.value = Dates.FormDate(query.until_time || new Date()) + }) - const intervals = [year, 'Q' + quarter, month, 'W' + week, day, hour, min, sec] - return intervals[diffs.indexOf(Math.min(...diffs))] - } + useEffect(() => { + const {statistics, metric} = response - updateChart() { let data = [] - if (!this.props.data) { + if (!statistics.date_histogram) { return } else { - data = Object.keys(this.props.data).map(key => ({ - time: parseInt(key, 10), - // name: this.resolveDate(key), - value: this.props.data[key][this.props.metric] + data = Object.keys(statistics.date_histogram).map(key => ({ + time: Dates.JSDate(parseInt(key)), + value: statistics.date_histogram[key][metric] })) } - data.sort((a, b) => d3.ascending(a.time, b.time)) - if (data.length > 0) { - this.handleTimeChange(this.state.from_time, 'from_time', 'picker') - this.handleTimeChange(this.state.until_time, 'until_time', 'picker') - } + const fromTime = Dates.JSDate(response.from_time || Dates.dateHistogramStartDate) + const untilTime = Dates.JSDate(response.until_time || new Date()) + const interval = response.dateHistogramInterval + const clickable = (interval * Dates.buckets) > 3600 - let deltaT = 31536000000 - if (data.length > 1) { - deltaT = data[1].time - data[0].time + const handleItemClicked = item => { + if (!clickable) { + return + } + const fromTime = item.time + const untilTime = Dates.addSeconds(fromTime, interval) + setQuery({ + ...query, + from_time: Dates.APIDate(fromTime), + until_time: Dates.APIDate(untilTime) + }) } - data.forEach(d => { d.name = this.resolveDate(d.time, deltaT) }) - - const scalePower = this.state.scalePower - const width = this.container.current.offsetWidth - const height = this.props.height - const margin = Math.round(0.15 * height) - - const x = scaleBand().rangeRound([margin, width]).padding(0.1) - const y = scalePow().range([height - margin, margin]).exponent(scalePower) + const width = containerRef.current.offsetWidth + const height = 250 + const marginRight = 32 + const marginTop = 0 + const marginBottom = 16 + const y = scalePow().range([height - marginBottom, marginTop]).exponent(scale) const max = d3.max(data, d => d.value) || 0 - x.domain(data.map(d => d.name)) y.domain([0, max]) - let svg = d3.select(this.svgEl.current) - svg.attr('width', width) - svg.attr('height', height) + const x = scaleTime() + .domain([Dates.addSeconds(fromTime, -interval), Dates.addSeconds(untilTime, interval)]) + .rangeRound([marginRight, width]) + + const container = d3.select(containerRef.current) + const tooltip = container.select('.' + classes.tooltip) + .style('opacity', 0) + const tooltipContent = container.select('.' + classes.tooltipContent) + const svg = container.select('svg') + .attr('width', width) + .attr('height', height) const xAxis = d3.axisBottom(x) svg.select('.xaxis').remove() svg.append('g') - .attr('transform', `translate(0,${height - margin})`) + .attr('transform', `translate(0,${height - marginBottom})`) .attr('class', 'xaxis') .call(xAxis) @@ -198,19 +128,19 @@ class UploadsHistogramUnstyled extends React.Component { const yAxis = d3.axisLeft(y).ticks(Math.min(max, 5), '.0s') svg.select('.yaxis').remove() svg.append('g') - .attr('transform', `translate(${margin}, 0)`) + .attr('transform', `translate(${marginRight}, 0)`) .attr('class', 'yaxis') .call(yAxis) - const {label, shortLabel} = this.props.metricsDefinitions[this.props.metric] - svg.select('.ylabel').remove() - svg.append('text') - .attr('class', 'ylabel') - .attr('x', 0) - .attr('y', 10) - .attr('dy', '1em') - .attr('font-size', '12px') - .text(`${shortLabel || label}`) + const {label, shortLabel} = domain.searchMetrics[metric] + // svg.select('.ylabel').remove() + // svg.append('text') + // .attr('class', 'ylabel') + // .attr('x', 0) + // .attr('y', 0) + // .attr('dy', '1em') + // .attr('font-size', '12px') + // .text(`${shortLabel || label}`) let withData = svg .selectAll('.bar').remove().exit() @@ -222,154 +152,127 @@ class UploadsHistogramUnstyled extends React.Component { item .append('rect') .attr('class', 'bar') - .attr('x', d => x(d.name)) + .attr('x', d => x(d.time) + 1) .attr('y', d => y(d.value)) - .attr('width', x.bandwidth()) + .attr('width', d => x(Dates.addSeconds(d.time, interval)) - x(d.time) - 2) .attr('height', d => y(0) - y(d.value)) .style('fill', nomadSecondaryColor.light) - item - .style('cursor', 'pointer') - .on('click', d => this.handleItemClicked(d, deltaT)) - - svg.select('.tooltip').remove() - let tooltip = svg.append('g') - tooltip - .attr('class', 'tooltip') - .style('visibility', 'hidden') - - tooltip.append('rect') - .attr('x', 0) - .attr('rx', 6) - .attr('ry', 6) - .attr('width', 100) - .attr('height', 40) - .attr('fill', 'grey') - .style('opacity', 1.0) - - let tooltipText = tooltip.append('text') - .attr('dy', '1.2em') - .attr('font-family', 'Arial, Helvetica, sans-serif') - .attr('font-size', '10px') - .attr('fill', 'white') - .style('text-anchor', 'middle') + if (clickable) { + item + .style('cursor', 'pointer') + .on('click', handleItemClicked) + } item .on('mouseover', function(d) { - tooltip.style('visibility', 'visible') - const date = new Date(d.time) - const value = `${date.toLocaleDateString()}\n - ${date.toLocaleTimeString()} - ${d.value} ${shortLabel || label}` - tooltipText.selectAll('tspan') - .data((value).split(/\n/)).join('tspan') - .attr('x', 50) - .attr('y', (d, i) => `${i * 1.2}em`) - .text(d => d) - const xPosition = x(d.name) + 20 - const yPosition = y(d.value) - 20 - tooltip.attr('transform', `translate( ${xPosition}, ${yPosition})`) + d3.select(this).select('.background') + .style('opacity', 0.08) + if (tooltips) { + tooltip.transition() + .duration(200) + .style('opacity', 1) + tooltip + .style('left', x(d.time) + 'px') + .style('bottom', '24px') + tooltipContent.html( + `${d.time.toLocaleDateString()}-${Dates.addSeconds(d.time, interval).toLocaleDateString()} with ${d.value.toLocaleString()} ${shortLabel || label}`) + } }) - .on('mouseout', () => tooltip.style('visibility', 'hidden')) - } + .on('mouseout', function(d) { + d3.select(this).select('.background') + .style('opacity', 0) + if (tooltips) { + tooltip.transition() + .duration(200) + .style('opacity', 0) + } + }) + }) - render() { - return ( - <div> - <Grid container justify='space-between' alignItems='flex-end'> - <Grid item xs={2}> - <Select - margin='none' - id='scales' - value={this.state.scalePower} - onChange={(event) => this.setState({scalePower: event.target.value})} - label= 'scale' - > {this.scales.map(item => ( - <MenuItem - value={item.value} - key={item.label}> {item.label} - </MenuItem>))} - </Select> - </Grid> - <Grid item xs={2}> + const handleDatePickerChange = useCallback((event, key) => { + try { + const date = new Date(event.target.value).getTime() + if (date < Dates.JSDate(Dates.dateHistogramStartDate).getTime()) { + return + } + if (date > new Date().getTime()) { + return + } + const value = Dates.APIDate(new Date(event.target.value)) + setQuery({...query, [key]: value}) + } catch (error) { + } + }) + + return <Card classes={{root: classes.root}}> + <CardHeader + title={title} + titleTypographyProps={{variant: 'body1'}} + action={( + <Grid container alignItems='flex-end' spacing={2}> + <Grid item> <TextField - id='from_time' + inputRef={fromTimeFieldRef} label="from time" type="date" - onChange={(event) => this.handleTimeChange(event.target.value, 'from_time', 'state')} + defaultValue={Dates.FormDate(query.from_time || Dates.dateHistogramStartDate)} + onChange={event => handleDatePickerChange(event, 'from_time')} InputLabelProps={{ shrink: true }} /> </Grid> - <Grid item xs={2}> + <Grid item> <TextField - id='until_time' + inputRef={untilTimeFieldRef} label="until time" type="date" - onChange={(event) => this.handleTimeChange(event.target.value, 'until_time', 'state')} + defaultValue={Dates.FormDate(query.until_time || new Date())} + onChange={event => handleDatePickerChange(event, 'until_time')} InputLabelProps={{ shrink: true }} /> </Grid> - <Grid item xs={2}> - <Button - variant='outlined' - color='default' - onClick={() => this.handleQueryChange()} - > Refresh <RefreshIcon/> - </Button> + <Grid item> + <Select + value={scale} + onChange={(event) => setScale(event.target.value)} + displayEmpty + name="scale power" + > + <MenuItem value={1}>linear</MenuItem> + <MenuItem value={0.5}>1/2</MenuItem> + <MenuItem value={0.25}>1/4</MenuItem> + <MenuItem value={0.125}>1/8</MenuItem> + </Select> </Grid> </Grid> - <div ref={this.container}> - <svg ref={this.svgEl}></svg> + )} + /> + <CardContent classes={{root: classes.content}}> + <div ref={containerRef}> + <div className={classes.tooltip}> + <div className={classes.tooltipContent}></div> </div> + <svg /> </div> - ) - } + </CardContent> + </Card> } - -export const UploadsHistogram = withStyles(UploadsHistogramUnstyled.styles)(UploadsHistogramUnstyled) - -class UploadsChart extends React.Component { - static propTypes = { - classes: PropTypes.object.isRequired, - metricsDefinitions: PropTypes.object.isRequired - } - static styles = theme => ({ - root: { - marginTop: theme.spacing(1) - } - }) - - static contextType = searchContext - - componentDidMount() { - const {setStatisticsToRefresh} = this.context - setStatisticsToRefresh('date_histogram') - } - - render() { - const {classes, metricsDefinitions, ...props} = this.props - const {response: {statistics, metric}, query, setQuery} = this.context - - return ( - <Grid container spacing={2}> - <Grid item xs={12}> - <UploadsHistogram - classes={{root: classes.root}} - height={250} - defaultScale={1} - data={statistics.date_histogram} - metric={metric} - metricsDefinitions={metricsDefinitions} - onChanged={(from_time, until_time) => setQuery({...query, from_time: from_time, until_time: until_time})} - {...props} /> - </Grid> - </Grid> - ) - } +UploadsHistogram.propTypes = { + /** + * An optional title for the chart. If no title is given, the quantity is used. + */ + title: PropTypes.string, + /** + * An optional scale power that is used as the initial scale before the user + * changes it. Default is 1 (linear scale). + */ + initialScale: PropTypes.number, + /** + * Set to true to enable tooltips for each value. + */ + tooltips: PropTypes.bool } - -export default compose(withApi(false, false), withStyles(UploadsChart.styles))(UploadsChart) diff --git a/gui/src/config.js b/gui/src/config.js index e5fdf1df5d..1a144db9f5 100644 --- a/gui/src/config.js +++ b/gui/src/config.js @@ -2,7 +2,8 @@ import { createMuiTheme } from '@material-ui/core' window.nomadEnv = window.nomadEnv || {} export const appBase = window.nomadEnv.appBase.replace(/\/$/, '') -export const apiBase = 'http://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/testing-major/api' // `${appBase}/api` +export const apiBase = 'http://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/testing-major/api' +// export const apiBase = `${appBase}/api` export const optimadeBase = `${appBase}/optimade` export const guiBase = process.env.PUBLIC_URL export const matomoUrl = window.nomadEnv.matomoUrl diff --git a/nomad/app/api/repo.py b/nomad/app/api/repo.py index 7441511be5..1da7d86f81 100644 --- a/nomad/app/api/repo.py +++ b/nomad/app/api/repo.py @@ -76,29 +76,6 @@ class RepoCalcResource(Resource): return result, 200 -def resolve_interval(from_time, until_time): - if from_time is None: - from_time = datetime.fromtimestamp(0) - if until_time is None: - until_time = datetime.utcnow() - dt = rfc3339DateTime.parse(until_time) - rfc3339DateTime.parse(from_time) - - if dt.days >= 1826: - return '1y' - elif dt.days >= 731: - return '1q' - elif dt.days >= 121: - return '1M' - elif dt.days >= 28: - return '1w' - elif dt.days >= 4: - return '1d' - elif dt.total_seconds() >= 14400: - return '1h' - else: - return '1m' - - _search_request_parser = api.parser() add_pagination_parameters(_search_request_parser) add_scroll_parameters(_search_request_parser) @@ -197,7 +174,7 @@ class RepoCalcsResource(Resource): order_by = args.get('order_by', 'upload_time') date_histogram = args.get('date_histogram', False) - interval = args.get('interval', 'auto') + interval = args.get('interval', '1M') metrics: List[str] = request.args.getlist('metrics') statistics = args.get('statistics', []) except Exception as e: @@ -206,13 +183,6 @@ class RepoCalcsResource(Resource): search_request = search.SearchRequest() apply_search_parameters(search_request, args) if date_histogram: - if interval == 'auto': - try: - from_time = args.get('from_time', None) - until_time = args.get('until_time', None) - interval = resolve_interval(from_time, until_time) - except Exception: - abort(400, message='encountered error resolving time interval') search_request.date_histogram(interval=interval) try: -- GitLab