diff --git a/gui/src/components/archive/ArchiveBrowser.js b/gui/src/components/archive/ArchiveBrowser.js index 0518d4de52275ca96efe688d2338ef45b3898d54..111bb6c032b37857eb60bd52173db0637d4e8533 100644 --- a/gui/src/components/archive/ArchiveBrowser.js +++ b/gui/src/components/archive/ArchiveBrowser.js @@ -63,6 +63,7 @@ import NavigateIcon from '@material-ui/icons/MoreHoriz' import ReloadIcon from '@material-ui/icons/Replay' import UploadIcon from '@material-ui/icons/CloudUpload' import { apiBase } from '../../config' +import ReactJson from 'react-json-view' export const configState = atom({ key: 'config', @@ -530,7 +531,8 @@ class SectionAdaptor extends ArchiveAdaptor { } // Regular quantities if (property.m_annotations?.browser) { - if (property.m_annotations.browser[0].adaptor === 'RawFileAdaptor') { + const adaptor = property.m_annotations.browser[0].adaptor + if (adaptor === 'RawFileAdaptor') { const installationUrl = this.parsedBaseUrl.installationUrl const uploadId = this.parsedBaseUrl.uploadId const path = this.obj[property.name] @@ -650,6 +652,20 @@ QuantityItemPreview.propTypes = ({ def: PropTypes.object.isRequired }) +const HtmlValue = React.memo(function HtmlValue({value}) { + return
+}) +HtmlValue.propTypes = { + value: PropTypes.string +} + +const JsonValue = React.memo(function JsonValue({value}) { + return +}) +JsonValue.propTypes = { + value: PropTypes.string +} + const QuantityValue = React.memo(function QuantityValue({value, def, ...more}) { const units = useUnits() @@ -1215,6 +1231,13 @@ FullStorageQuantity.propTypes = ({ function Quantity({value, def, unit, children}) { const {prev} = useLane() + const valueComponentName = def.m_annotations?.browser?.[0]?.value_component || def.m_annotations?.browser?.[0]?.valueComponent + let valueComponent = QuantityValue + if (valueComponentName === 'HtmlValue') { + valueComponent = HtmlValue + } else if (valueComponentName === 'JsonValue') { + valueComponent = JsonValue + } return {def.m_annotations?.plot && ( @@ -1228,11 +1251,9 @@ function Quantity({value, def, unit, children}) { )} - + {React.createElement(valueComponent, { + value: value, def: def, unit: unit + })} {children} diff --git a/gui/src/components/entry/ArchiveLogView.js b/gui/src/components/entry/ArchiveLogView.js index 214ac51616eb6c47860bb56a86d3ad756f60bbe0..562ec32b5fc959c5d151cdeeaffe99745ee3a369 100644 --- a/gui/src/components/entry/ArchiveLogView.js +++ b/gui/src/components/entry/ArchiveLogView.js @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import React, { useEffect, useState } from 'react' +import React, { useCallback, useEffect, useMemo, useState } from 'react' import PropTypes from 'prop-types' import { Typography, Accordion, AccordionSummary, AccordionDetails, makeStyles, FormGroup, Button, Grid, FormControl, InputLabel, Input, Select, MenuItem, Chip } from '@material-ui/core' import ExpandMoreIcon from '@material-ui/icons/ExpandMore' @@ -28,6 +28,7 @@ import { useEntryPageContext } from './EntryPageContext' import Checkbox from '@material-ui/core/Checkbox' import FormControlLabel from '@material-ui/core/FormControlLabel' +const defaultKeys = ['event', 'logger', 'timestamp', 'level'] const logsDefaultValues = { defaultLogsToShowOnFirstMount: 5, defaultLogsToShowOnEachMount: 10 @@ -44,7 +45,7 @@ const useLogEntryStyles = makeStyles(theme => ({ const LogEntry = React.memo(function LogEntry(props) { const classes = useLogEntryStyles() - const {entry, keyNames} = props + const {entry, keys} = props const data = entry const summaryProps = {} @@ -53,10 +54,23 @@ const LogEntry = React.memo(function LogEntry(props) { } else if (data.level === 'WARNING') { summaryProps.classes = {root: classes.warning} } + + const filterBeforeColon = useCallback((key) => { + return (key === 'level' || key === 'timestamp' || key === 'logger') + }, []) + + const beforeColon = useMemo(() => keys.filter(filterBeforeColon), [filterBeforeColon, keys]) + const afterColon = useMemo(() => keys.filter(key => !filterBeforeColon(key)), [filterBeforeColon, keys]) + return ( }> - {data.level}: {(keyNames.map((key) => `${data[key]}`).join(' | '))} + + {beforeColon.length > 0 && ( + {beforeColon.map(key => `${String(data[key])}`).join(' ')}:  + )} + {(afterColon.map(key => `${String(data[key])}`).join(', '))} + ({ @@ -86,9 +100,9 @@ const useStyles = makeStyles(theme => ({ position: 'fixed !important' }, formControl: { - margin: theme.spacing(1), - minWidth: 120, - maxWidth: 350 + width: '100%', + marginX: theme.spacing(2), + marginBottom: theme.spacing(1) }, chips: { display: 'flex', @@ -105,69 +119,6 @@ const useStyles = makeStyles(theme => ({ } })) -const FilterLogsByLevel = React.memo(function FilterLogsByLevel(props) { - const {logLevels, onCheckListChanged} = props - return ( - - - Filter Logs by Level: - - {Object.keys(logLevels).map((key, i) => { - return ( - } - label={key} - /> - ) - })} - - ) -}) -FilterLogsByLevel.propTypes = { - logLevels: PropTypes.object.isRequired, - onCheckListChanged: PropTypes.func.isRequired -} - -const FilterLogTagsByKeys = React.memo(function FilterLogTagsByKeys(props) { - const {className, keyNames, onKeyNamesChanged, uniquekeys} = props - return ( - - Filter keys by: - - - ) -}) -FilterLogTagsByKeys.propTypes = { - className: PropTypes.object.isRequired, - keyNames: PropTypes.array.isRequired, - onKeyNamesChanged: PropTypes.func.isRequired, - uniquekeys: PropTypes.array.isRequired -} - export default function ArchiveLogView(props) { const classes = useStyles() const {entryId} = useEntryPageContext() @@ -179,22 +130,18 @@ export default function ArchiveLogView(props) { const [logLevels, setLogLevels] = useState({ DEBUG: true, - ERROR: true, - CRITICAL: true, + INFO: true, WARNING: true, - INFO: true + ERROR: true, + CRITICAL: true }) const [numberOfLogs, setNumberOflogs] = useState(logsDefaultValues.defaultLogsToShowOnFirstMount) - const [keyNames, setkeyNames] = useState(['parser']) + const [keys, setKeys] = useState(['event']) - const handlekeyNamesChanged = (e) => { - setkeyNames(e.target.value) - } - - const handleCheckListChanged = (e) => { - setLogLevels({...logLevels, [e.target.name]: e.target.checked}) - } + const handleCheckListChanged = useCallback((e) => { + setLogLevels(logLevels => ({...logLevels, [e.target.name]: e.target.checked})) + }, [setLogLevels]) useEffect(() => { api.post(`/entries/${entryId}/archive/query`, {required: {processing_logs: '*'}}) @@ -213,6 +160,20 @@ export default function ArchiveLogView(props) { setNumberOflogs(logsDefaultValues.defaultLogsToShowOnEachMount) }, [setData, setDoesNotExist, api, raiseError, entryId, logLevels]) + const availableKeys = useMemo(() => { + const keys = [...new Set( + (data || []).reduce((aggregatedKeys, item) => [...aggregatedKeys, ...Object.keys(item)], []))] + + keys.sort((a, b) => defaultKeys.indexOf(b) - defaultKeys.indexOf(a)) + return keys + }, [data]) + + const handlekeyNamesChanged = useCallback((e) => { + const keys = [...e.target.value] + keys.sort((a, b) => availableKeys.indexOf(a) - availableKeys.indexOf(b)) + setKeys(keys) + }, [setKeys, availableKeys]) + if (doesNotExist) { return ( @@ -224,46 +185,72 @@ export default function ArchiveLogView(props) { ) } - let content = 'loading ...' - if (data) { - const uniquekeys = [...new Set( - data.reduce((aggregatedKeys, item) => [...aggregatedKeys, ...Object.keys(item)], []))] - content = - - - - - - - - - - {data - .map((entry, i) => (logLevels[entry.level] ? : null)) - .slice(0, numberOfLogs) - .filter(el => el !== null)} - - - {(numberOfLogs > data.length || - data - .map((entry, i) => (logLevels[entry.level] ? 1 : null)) - .filter(el => el !== null).length <= numberOfLogs) ? '' : ()} - - - + if (!data) { + return loading ... } return ( - {content} + + + + + {Object.keys(logLevels).map((key, i) => { + return ( + } + label={key} + /> + ) + })} + + + + + Keys to show: + + + + + {data + .map((entry, i) => (logLevels[entry.level] ? : null)) + .slice(0, numberOfLogs) + .filter(el => el !== null)} + + + {(numberOfLogs > data.length || + data + .map((entry, i) => (logLevels[entry.level] ? 1 : null)) + .filter(el => el !== null).length <= numberOfLogs) ? '' : ()} + + + ) } -ArchiveLogView.propTypes = {} diff --git a/nomad/datamodel/metainfo/eln/__init__.py b/nomad/datamodel/metainfo/eln/__init__.py index 67d5e148c2fad3704ab74953dc8ec5ae301f0a3b..ec259779e411a181e30f53ba1d8d7bbef492f51e 100644 --- a/nomad/datamodel/metainfo/eln/__init__.py +++ b/nomad/datamodel/metainfo/eln/__init__.py @@ -25,6 +25,9 @@ from nomad.datamodel.results import ELN, Results, Material, BandGap from nomad.metainfo import Package, Quantity, Datetime, Reference, Section from nomad.datamodel.metainfo.eln.perovskite_solar_cell_database import addSolarCell +from .labfolder import LabfolderProject + + m_package = Package(name='material_library') diff --git a/nomad/datamodel/metainfo/eln/elabftw.py b/nomad/datamodel/metainfo/eln/elabftw.py new file mode 100644 index 0000000000000000000000000000000000000000..33cf4c42589bdc4f99799c2dd910b66ec3302d0a --- /dev/null +++ b/nomad/datamodel/metainfo/eln/elabftw.py @@ -0,0 +1,44 @@ +import requests + +from nomad.metainfo import Package, Quantity +from nomad.datamodel.data import EntryData + +m_package = Package(name='eLabFTW') + + +class ElabFTW(EntryData): + + def __init__(self, *args, **kwargs): + super(ElabFTW, self).__init__(*args, **kwargs) + self.logger = None + + url = Quantity( + type=str, + description='The URL of your eLabFTW experiment', + a_eln=dict(component='StringEditQuantity')) + + api_key = Quantity( + type=str, + description='Your API key to access this experiment. This can be a read-only key.', + a_eln=dict(component='StringEditQuantity')) + + title = Quantity(type=str, a_eln=dict(component='StringEditQuantity')) + content = Quantity(type=str, a_eln=dict(component='RichTextEditQuantity')) + files = Quantity(type=str, shape=['*'], a_browser=dict(adaptor='RawFileAdaptor')) + + def _get_exp_data(self, url, api_key): + endpoint = url[0:url.rindex("/")] + exp_id = url[url.find("id=") + 3:].strip("#") + api_url = f"{endpoint}/api/v1/experiments/{exp_id}" + resp = requests.get(api_url, headers={"Authorization": api_key}, verify=False) + return resp.json() + + def normalize(self, archive, logger): + super(ElabFTW, self).normalize(archive, logger) + + elabftw_data = self._get_exp_data(self.url, self.api_key) + self.title = elabftw_data["title"] + self.content = elabftw_data["body"] + + +m_package.__init_metainfo__() diff --git a/nomad/datamodel/metainfo/eln/labfolder.py b/nomad/datamodel/metainfo/eln/labfolder.py new file mode 100644 index 0000000000000000000000000000000000000000..96a0a4730a5926e8fbddd553e44cd556bee9095c --- /dev/null +++ b/nomad/datamodel/metainfo/eln/labfolder.py @@ -0,0 +1,245 @@ +import requests +import re +from urllib.parse import urlparse, parse_qs + +from nomad.metainfo import MSection, Section, Quantity, SubSection, Package, Datetime, MEnum, JSON +from nomad.datamodel.data import EntryData + +m_package = Package(name='labfolder') + + +class LabfolderImportError(Exception): + pass + + +class LabfolderElement(MSection): + m_def = Section(label_quantity='element_type') + + id = Quantity(type=str, description='the stable pointer to the element') + entry_id = Quantity(type=str, description='the id of the stable pointer to the entry') + version_id = Quantity(type=str, description='the unique id of the element') + version_date = Quantity(type=Datetime, description='the creation date of the entry element version (same with the creation date on the first version)') + creation_date = Quantity(type=Datetime, description='the creation date of the entry element (first version)') + owner_id = Quantity(type=str, description='the id of the original author') + element_type = Quantity(type=MEnum('TEXT', 'FILE'), description='Denotes that this is a file element. The value is always `FILE`') + + def download_files(self, parser, archive, logger): + pass + + +class LabfolderTextElement(LabfolderElement): + content = Quantity( + type=str, description='The text based content of this element', + a_browser=dict(value_component='HtmlValue')) + + +class LabfolderFileElement(LabfolderElement): + file_name = Quantity(type=str, description='The name of the file') + file_size = Quantity(type=int, description='The size of the file in bytes') + content_type = Quantity(type=str, description='The type of the binary content which is sent on header parameter `Content-Type`') + + file = Quantity(type=str, a_browser=dict(adaptor='RawFileAdaptor')) + + def download_files(self, parser, archive, logger): + response = parser._response(requests.get, f'/elements/file/{self.id}/download') + try: + with archive.m_context.raw_file(self.file_name, 'wb') as f: + f.write(response.content) + except Exception as e: + logger.error('could not download file', exc_info=e, data=dict(file_name=self.file_name)) + + self.file = self.file_name + + +class LabfolderImageElement(LabfolderElement): + title = Quantity(type=str, description='the title of the image element') + file_size = Quantity(type=int, description='the size of the image file in bytes') + preview_height = Quantity(type=int, description='height of the downscaled image version, in px') + preview_width = Quantity(type=int, description='width of the downscaled image version, in px') + preview_zoom = Quantity(type=float, description='image zoom in the ELN UI, in percentage') + original_file_content_type = Quantity(type=str, description='the content type of the original uploaded image file') + annotation_layer_svg = Quantity(type=str, description='The vector graphic used for the image annotation layer, defined in SVG format') + + original_image_file = Quantity(type=str, a_browser=dict(adaptor='RawFileAdaptor')) + preview_image_file = Quantity(type=str, a_browser=dict(adaptor='RawFileAdaptor')) + + def download_files(self, parser, archive, logger): + def download(path, file_quantity): + response = parser._response(requests.get, f'/elements/image/{self.id}/{path}') + + content_disposition = response.headers.get('Content-Disposition', '') + match = re.match(r'^attachment; filename="(.+)"$', content_disposition) + if match: + file_name = match.group(1) + else: + file_name = self.id + logger.warn('there is no filename for an image', data=dict(element_id=self.id)) + try: + with archive.m_context.raw_file(file_name, 'wb') as f: + f.write(response.content) + except Exception as e: + logger.error('could not download file', exc_info=e, data=dict(file_name=self.file_name)) + + self.m_set(file_quantity, file_name) + + download('original-data', LabfolderImageElement.original_image_file) + download('preview-data', LabfolderImageElement.preview_image_file) + + +class LabfolderTableElement(LabfolderElement): + title = Quantity(type=str, description='the title of the table') + content = Quantity( + type=JSON, description='The JSON content of the table element', + a_browser=dict(value_component='JsonValue')) + + +class LabfolderWellPlateElement(LabfolderElement): + title = Quantity(type=str, description='The title of the well plate template') + content = Quantity( + type=JSON, description='The title of the well plate template', + a_browser=dict(value_component='JsonValue')) + meta_data = Quantity( + type=JSON, description='JSON meta data for visualization processing, used to store information about layer colors and well identifiers', + a_browser=dict(value_component='JsonValue')) + + +_element_type_path_mapping = { + 'TEXT': 'text', + 'FILE': 'file', + 'IMAGE': 'image', + 'TABLE': 'table', + 'WELL_PLATE': 'well-plate' +} + +_element_type_section_mapping = { + 'TEXT': LabfolderTextElement, + 'FILE': LabfolderFileElement, + 'IMAGE': LabfolderImageElement, + 'TABLE': LabfolderTableElement, + 'WELL_PLATE': LabfolderWellPlateElement +} + + +class LabfolderProject(EntryData): + + def __init__(self, *args, **kwargs): + super(LabfolderProject, self).__init__(*args, **kwargs) + + self.__headers = None + self.logger = None + + project_url = Quantity( + type=str, + a_eln=dict(component='StringEditQuantity')) + password = Quantity( + type=str, + a_eln=dict(component='StringEditQuantity', type='password')) + labfolder_email = Quantity( + type=str, + a_eln=dict(component='StringEditQuantity')) + + id = Quantity(type=str) + version_id = Quantity(type=str) + author_id = Quantity(type=str) + project_id = Quantity(type=str) + version_date = Quantity(type=Datetime) + creation_date = Quantity(type=Datetime) + custom_dates = Quantity(type=Datetime, shape=['*']) + tags = Quantity(type=str, shape=['*']) + title = Quantity(type=str) + hidden = Quantity(type=bool) + editable = Quantity(type=bool) + + elements = SubSection(sub_section=LabfolderElement, repeats=True) + + def _response(self, method, url, msg='cannot do labfolder api request', **kwargs): + response = method( + f'{self._api_base_url}{url}', + headers=self._headers, **kwargs) + + if response.status_code != 200: + self.logger.error( + msg, + data=dict(status_code=response.status_code, text=response.text)) + raise LabfolderImportError() + + return response + + def _json(self, *args, **kwargs): + return self._response(*args, **kwargs).json() + + @property + def _api_base_url(self): + match = re.match(r'^(.+)/eln/notebook.*$', self.project_url) + if not match: + self.logger.error('unexpected labfolder url format', data=dict(project_url=self.project_url)) + raise LabfolderImportError() + + return f'{match.group(1)}/api/v2' + + @property + def _headers(self): + if not self.__headers: + response = requests.post( + f'{self._api_base_url}/auth/login', + json=dict(user=self.labfolder_email, password=self.password)) + + if response.status_code != 200: + self.logger.error( + 'cannot login', + data=dict(status_code=response.status_code, text=response.text)) + raise LabfolderImportError() + + self.__headers = dict(Authorization=f'Token {response.json()["token"]}') + + return self.__headers + + def normalize(self, archive, logger): + super(LabfolderProject, self).normalize(archive, logger) + self.logger = logger + + if not self.project_url or not self.labfolder_email or not self.password: + logger.error('missing information, cannot import project') + return + + try: + project_ids = parse_qs(urlparse(self.project_url).fragment[1:])['projectIds'] + except KeyError as e: + logger.error('cannot parse project ids from url', exc_info=e) + raise LabfolderImportError() + + data = self._json( + requests.get, f'/entries?project_ids={",".join(project_ids)}') + + elements = data[0]['elements'] + del data[0]['elements'] + + try: + self.m_update_from_dict(data[0]) + except Exception as e: + logger.error('cannot update archive with labfolder data', exc_info=e) + raise LabfolderImportError() + + # remove potential old content + self.elements.clear() + + for element in elements: + element_type = element['type'] + + if element_type not in _element_type_path_mapping: + logger.warn('unknown element type', data=dict(element_type=element_type)) + continue + + data = self._json( + requests.get, + f'/elements/{_element_type_path_mapping[element_type]}/{element["id"]}/version/{element["version_id"]}') + nomad_element = _element_type_section_mapping[element_type]() + nomad_element.m_update_from_dict(data) + self.elements.append(nomad_element) + + nomad_element.download_files(self, archive, logger) + + logger.info('reached the end') + + +m_package.init_metainfo()